MediaWiki  master
MultiWriteBagOStuff.php
Go to the documentation of this file.
1 <?php
23 use Wikimedia\ObjectFactory;
24 
38  protected $caches;
39 
41  protected $asyncWrites = false;
43  protected $cacheIndexes = [];
44 
46  private static $UPGRADE_TTL = 3600;
47 
69  public function __construct( $params ) {
70  parent::__construct( $params );
71 
72  if ( empty( $params['caches'] ) || !is_array( $params['caches'] ) ) {
73  throw new InvalidArgumentException(
74  __METHOD__ . ': "caches" parameter must be an array of caches'
75  );
76  }
77 
78  $this->caches = [];
79  foreach ( $params['caches'] as $cacheInfo ) {
80  if ( $cacheInfo instanceof BagOStuff ) {
81  $this->caches[] = $cacheInfo;
82  } else {
83  if ( !isset( $cacheInfo['args'] ) ) {
84  // B/C for when $cacheInfo was for ObjectCache::newFromParams().
85  // Callers intenting this to be for ObjectFactory::getObjectFromSpec
86  // should have set "args" per the docs above. Doings so avoids extra
87  // (likely harmless) params (factory/class/calls) ending up in "args".
88  $cacheInfo['args'] = [ $cacheInfo ];
89  }
90 
91  // ObjectFactory::getObjectFromSpec accepts an array, not just a callable (phan bug)
92  // @phan-suppress-next-line PhanTypeInvalidCallableArraySize
93  $this->caches[] = ObjectFactory::getObjectFromSpec( $cacheInfo );
94  }
95  }
96 
97  $this->attrMap = $this->mergeFlagMaps( $this->caches );
98 
99  $this->asyncWrites = (
100  isset( $params['replication'] ) &&
101  $params['replication'] === 'async' &&
102  is_callable( $this->asyncHandler )
103  );
104 
105  $this->cacheIndexes = array_keys( $this->caches );
106  }
107 
108  public function get( $key, $flags = 0 ) {
109  $args = func_get_args();
110 
111  if ( $this->fieldHasFlags( $flags, self::READ_LATEST ) ) {
112  // If the latest write was a delete(), we do NOT want to fallback
113  // to the other tiers and possibly see the old value. Also, this
114  // is used by merge(), which only needs to hit the primary.
115  return $this->callKeyMethodOnTierCache(
116  0,
117  __FUNCTION__,
118  self::ARG0_KEY,
119  self::RES_NONKEY,
120  $args
121  );
122  }
123 
124  $value = false;
125  $missIndexes = []; // backends checked
126  foreach ( $this->cacheIndexes as $i ) {
127  $value = $this->callKeyMethodOnTierCache(
128  $i,
129  __FUNCTION__,
130  self::ARG0_KEY,
131  self::RES_NONKEY,
132  $args
133  );
134  if ( $value !== false ) {
135  break;
136  }
137  $missIndexes[] = $i;
138  }
139 
140  if (
141  $value !== false &&
142  $this->fieldHasFlags( $flags, self::READ_VERIFIED ) &&
143  $missIndexes
144  ) {
145  // Backfill the value to the higher (and often faster/smaller) cache tiers
147  $missIndexes,
148  $this->asyncWrites,
149  'set',
150  self::ARG0_KEY,
151  self::RES_NONKEY,
152  [ $key, $value, self::$UPGRADE_TTL ]
153  );
154  }
155 
156  return $value;
157  }
158 
159  public function set( $key, $value, $exptime = 0, $flags = 0 ) {
160  return $this->callKeyWriteMethodOnTierCaches(
161  $this->cacheIndexes,
162  $this->useAsyncSecondaryWrites( $flags ),
163  __FUNCTION__,
164  self::ARG0_KEY,
165  self::RES_NONKEY,
166  func_get_args()
167  );
168  }
169 
170  public function delete( $key, $flags = 0 ) {
171  return $this->callKeyWriteMethodOnTierCaches(
172  $this->cacheIndexes,
173  $this->useAsyncSecondaryWrites( $flags ),
174  __FUNCTION__,
175  self::ARG0_KEY,
176  self::RES_NONKEY,
177  func_get_args()
178  );
179  }
180 
181  public function add( $key, $value, $exptime = 0, $flags = 0 ) {
182  // Try the write to the top-tier cache
183  $ok = $this->callKeyMethodOnTierCache(
184  0,
185  __FUNCTION__,
186  self::ARG0_KEY,
187  self::RES_NONKEY,
188  func_get_args()
189  );
190 
191  if ( $ok ) {
192  // Relay the add() using set() if it succeeded. This is meant to handle certain
193  // migration scenarios where the same store might get written to twice for certain
194  // keys. In that case, it makes no sense to return false due to "self-conflicts".
195  $okSecondaries = $this->callKeyWriteMethodOnTierCaches(
196  array_slice( $this->cacheIndexes, 1 ),
197  $this->useAsyncSecondaryWrites( $flags ),
198  'set',
199  self::ARG0_KEY,
200  self::RES_NONKEY,
201  [ $key, $value, $exptime, $flags ]
202  );
203  if ( $okSecondaries === false ) {
204  $ok = false;
205  }
206  }
207 
208  return $ok;
209  }
210 
211  public function merge( $key, callable $callback, $exptime = 0, $attempts = 10, $flags = 0 ) {
212  return $this->callKeyWriteMethodOnTierCaches(
213  $this->cacheIndexes,
214  $this->useAsyncSecondaryWrites( $flags ),
215  __FUNCTION__,
216  self::ARG0_KEY,
217  self::RES_NONKEY,
218  func_get_args()
219  );
220  }
221 
222  public function changeTTL( $key, $exptime = 0, $flags = 0 ) {
223  return $this->callKeyWriteMethodOnTierCaches(
224  $this->cacheIndexes,
225  $this->useAsyncSecondaryWrites( $flags ),
226  __FUNCTION__,
227  self::ARG0_KEY,
228  self::RES_NONKEY,
229  func_get_args()
230  );
231  }
232 
233  public function lock( $key, $timeout = 6, $exptime = 6, $rclass = '' ) {
234  // Only need to lock the first cache; also avoids deadlocks
235  return $this->callKeyMethodOnTierCache(
236  0,
237  __FUNCTION__,
238  self::ARG0_KEY,
239  self::RES_NONKEY,
240  func_get_args()
241  );
242  }
243 
244  public function unlock( $key ) {
245  // Only the first cache is locked
246  return $this->callKeyMethodOnTierCache(
247  0,
248  __FUNCTION__,
249  self::ARG0_KEY,
250  self::RES_NONKEY,
251  func_get_args()
252  );
253  }
254 
256  $timestamp,
257  callable $progress = null,
258  $limit = INF,
259  string $tag = null
260  ) {
261  $ret = false;
262  foreach ( $this->caches as $cache ) {
263  if ( $cache->deleteObjectsExpiringBefore( $timestamp, $progress, $limit, $tag ) ) {
264  $ret = true;
265  }
266  }
267 
268  return $ret;
269  }
270 
271  public function getMulti( array $keys, $flags = 0 ) {
272  // Just iterate over each key in order to handle all the backfill logic
273  $res = [];
274  foreach ( $keys as $key ) {
275  $val = $this->get( $key, $flags );
276  if ( $val !== false ) {
277  $res[$key] = $val;
278  }
279  }
280 
281  return $res;
282  }
283 
284  public function setMulti( array $valueByKey, $exptime = 0, $flags = 0 ) {
285  return $this->callKeyWriteMethodOnTierCaches(
286  $this->cacheIndexes,
287  $this->useAsyncSecondaryWrites( $flags ),
288  __FUNCTION__,
289  self::ARG0_KEYMAP,
290  self::RES_NONKEY,
291  func_get_args()
292  );
293  }
294 
295  public function deleteMulti( array $keys, $flags = 0 ) {
296  return $this->callKeyWriteMethodOnTierCaches(
297  $this->cacheIndexes,
298  $this->useAsyncSecondaryWrites( $flags ),
299  __FUNCTION__,
300  self::ARG0_KEYARR,
301  self::RES_NONKEY,
302  func_get_args()
303  );
304  }
305 
306  public function changeTTLMulti( array $keys, $exptime, $flags = 0 ) {
307  return $this->callKeyWriteMethodOnTierCaches(
308  $this->cacheIndexes,
309  $this->useAsyncSecondaryWrites( $flags ),
310  __FUNCTION__,
311  self::ARG0_KEYARR,
312  self::RES_NONKEY,
313  func_get_args()
314  );
315  }
316 
317  public function incr( $key, $value = 1, $flags = 0 ) {
318  return $this->callKeyWriteMethodOnTierCaches(
319  $this->cacheIndexes,
320  $this->useAsyncSecondaryWrites( $flags ),
321  __FUNCTION__,
322  self::ARG0_KEY,
323  self::RES_NONKEY,
324  func_get_args()
325  );
326  }
327 
328  public function decr( $key, $value = 1, $flags = 0 ) {
329  return $this->callKeyWriteMethodOnTierCaches(
330  $this->cacheIndexes,
331  $this->useAsyncSecondaryWrites( $flags ),
332  __FUNCTION__,
333  self::ARG0_KEY,
334  self::RES_NONKEY,
335  func_get_args()
336  );
337  }
338 
339  public function incrWithInit( $key, $exptime, $value = 1, $init = null, $flags = 0 ) {
340  return $this->callKeyWriteMethodOnTierCaches(
341  $this->cacheIndexes,
342  $this->useAsyncSecondaryWrites( $flags ),
343  __FUNCTION__,
344  self::ARG0_KEY,
345  self::RES_NONKEY,
346  func_get_args()
347  );
348  }
349 
350  public function getLastError() {
351  foreach ( $this->caches as $cache ) {
352  $error = $cache->getLastError();
353  if ( $error !== self::ERR_NONE ) {
354  return $error;
355  }
356  }
357 
358  return self::ERR_NONE;
359  }
360 
361  public function clearLastError() {
362  foreach ( $this->caches as $cache ) {
363  $cache->clearLastError();
364  }
365  }
366 
367  public function makeKeyInternal( $keyspace, $components ) {
368  return $this->genericKeyFromComponents( $keyspace, ...$components );
369  }
370 
371  public function makeKey( $collection, ...$components ) {
372  return $this->genericKeyFromComponents( $this->keyspace, $collection, ...$components );
373  }
374 
375  public function makeGlobalKey( $collection, ...$components ) {
376  return $this->genericKeyFromComponents( self::GLOBAL_KEYSPACE, $collection, ...$components );
377  }
378 
379  protected function convertGenericKey( $key ) {
380  return $key; // short-circuit; already uses "generic" keys
381  }
382 
383  public function addBusyCallback( callable $workCallback ) {
384  $this->caches[0]->addBusyCallback( $workCallback );
385  }
386 
387  public function setNewPreparedValues( array $valueByKey ) {
388  return $this->callKeyMethodOnTierCache(
389  0,
390  __FUNCTION__,
391  self::ARG0_KEYMAP,
392  self::RES_NONKEY,
393  func_get_args()
394  );
395  }
396 
397  public function setMockTime( &$time ) {
398  parent::setMockTime( $time );
399  foreach ( $this->caches as $cache ) {
400  $cache->setMockTime( $time );
401  }
402  }
403 
414  private function callKeyMethodOnTierCache( $index, $method, $arg0Sig, $rvSig, array $args ) {
415  return $this->caches[$index]->proxyCall( $method, $arg0Sig, $rvSig, $args );
416  }
417 
430  array $indexes,
431  $asyncSecondary,
432  $method,
433  $arg0Sig,
434  $resSig,
435  array $args
436  ) {
437  $res = null;
438 
439  if ( $asyncSecondary && array_diff( $indexes, [ 0 ] ) && $method !== 'merge' ) {
440  // Deep-clone $args to prevent misbehavior when something writes an
441  // object to the BagOStuff then modifies it afterwards, e.g. T168040.
443  }
444 
445  foreach ( $indexes as $i ) {
446  $cache = $this->caches[$i];
447 
448  if ( $i == 0 || !$asyncSecondary ) {
449  // Tier 0 store or in sync mode: write synchronously and get result
450  $storeRes = $cache->proxyCall( $method, $arg0Sig, $resSig, $args );
451  if ( $storeRes === false ) {
452  $res = false;
453  } elseif ( $res === null ) {
454  $res = $storeRes; // first synchronous result
455  }
456  } else {
457  // Secondary write in async mode: do not block this HTTP request
459  static function () use ( $cache, $method, $arg0Sig, $resSig, $args ) {
460  $cache->proxyCall( $method, $arg0Sig, $resSig, $args );
461  }
462  );
463  }
464  }
465 
466  return $res;
467  }
468 
473  private function useAsyncSecondaryWrites( $flags ) {
474  return $this->fieldHasFlags( $flags, self::WRITE_SYNC ) ? false : $this->asyncWrites;
475  }
476 }
MultiWriteBagOStuff\$cacheIndexes
int[] $cacheIndexes
List of all backing cache indexes.
Definition: MultiWriteBagOStuff.php:43
MultiWriteBagOStuff\lock
lock( $key, $timeout=6, $exptime=6, $rclass='')
Acquire an advisory lock on a key string, exclusive to the caller.
Definition: MultiWriteBagOStuff.php:233
MultiWriteBagOStuff\getLastError
getLastError()
Get the "last error" registered; clearLastError() should be called manually.
Definition: MultiWriteBagOStuff.php:350
MultiWriteBagOStuff\$asyncWrites
bool $asyncWrites
Use async secondary writes.
Definition: MultiWriteBagOStuff.php:41
MultiWriteBagOStuff\clearLastError
clearLastError()
Clear the "last error" registry.
Definition: MultiWriteBagOStuff.php:361
MultiWriteBagOStuff\merge
merge( $key, callable $callback, $exptime=0, $attempts=10, $flags=0)
Merge changes into the existing cache value (possibly creating a new one)
Definition: MultiWriteBagOStuff.php:211
MultiWriteBagOStuff\decr
decr( $key, $value=1, $flags=0)
Decrease stored value of $key by $value while preserving its TTL.
Definition: MultiWriteBagOStuff.php:328
BagOStuff
Class representing a cache/ephemeral data store.
Definition: BagOStuff.php:86
BagOStuff\genericKeyFromComponents
genericKeyFromComponents(... $components)
At a minimum, there must be a keyspace and collection name component.
Definition: BagOStuff.php:676
MultiWriteBagOStuff\changeTTL
changeTTL( $key, $exptime=0, $flags=0)
Change the expiration on a key if it exists.
Definition: MultiWriteBagOStuff.php:222
MultiWriteBagOStuff\incrWithInit
incrWithInit( $key, $exptime, $value=1, $init=null, $flags=0)
Increase the value of the given key (no TTL change) if it exists or create it otherwise.
Definition: MultiWriteBagOStuff.php:339
$res
$res
Definition: testCompression.php:57
serialize
serialize()
Definition: ApiMessageTrait.php:138
Wikimedia\LightweightObjectStore\StorageAwareness\ERR_NONE
const ERR_NONE
No storage medium error.
Definition: StorageAwareness.php:34
MultiWriteBagOStuff\setMulti
setMulti(array $valueByKey, $exptime=0, $flags=0)
Batch insertion/replace.
Definition: MultiWriteBagOStuff.php:284
MultiWriteBagOStuff\addBusyCallback
addBusyCallback(callable $workCallback)
Let a callback be run to avoid wasting time on special blocking calls.
Definition: MultiWriteBagOStuff.php:383
MultiWriteBagOStuff\makeKey
makeKey( $collection,... $components)
Make a cache key for the global keyspace and given components.
Definition: MultiWriteBagOStuff.php:371
MultiWriteBagOStuff\changeTTLMulti
changeTTLMulti(array $keys, $exptime, $flags=0)
Change the expiration of multiple keys that exist.
Definition: MultiWriteBagOStuff.php:306
MultiWriteBagOStuff\unlock
unlock( $key)
Release an advisory lock on a key string.
Definition: MultiWriteBagOStuff.php:244
BagOStuff\$asyncHandler
callable null $asyncHandler
Definition: BagOStuff.php:92
MultiWriteBagOStuff\useAsyncSecondaryWrites
useAsyncSecondaryWrites( $flags)
Definition: MultiWriteBagOStuff.php:473
$args
if( $line===false) $args
Definition: mcc.php:124
MultiWriteBagOStuff\convertGenericKey
convertGenericKey( $key)
Convert a "generic" reversible cache key into one for this cache.
Definition: MultiWriteBagOStuff.php:379
MultiWriteBagOStuff\add
add( $key, $value, $exptime=0, $flags=0)
Insert an item if it does not already exist.
Definition: MultiWriteBagOStuff.php:181
MultiWriteBagOStuff\incr
incr( $key, $value=1, $flags=0)
Increase stored value of $key by $value while preserving its TTL.
Definition: MultiWriteBagOStuff.php:317
MultiWriteBagOStuff\setMockTime
setMockTime(&$time)
Definition: MultiWriteBagOStuff.php:397
MultiWriteBagOStuff\callKeyWriteMethodOnTierCaches
callKeyWriteMethodOnTierCaches(array $indexes, $asyncSecondary, $method, $arg0Sig, $resSig, array $args)
Call a write method on the cache instances, in order, for the given tiers (indexes)
Definition: MultiWriteBagOStuff.php:429
MultiWriteBagOStuff\__construct
__construct( $params)
Definition: MultiWriteBagOStuff.php:69
BagOStuff\mergeFlagMaps
mergeFlagMaps(array $bags)
Merge the flag maps of one or more BagOStuff objects into a "lowest common denominator" map.
Definition: BagOStuff.php:594
MultiWriteBagOStuff\makeGlobalKey
makeGlobalKey( $collection,... $components)
Make a cache key for the default keyspace and given components.
Definition: MultiWriteBagOStuff.php:375
MultiWriteBagOStuff
A cache class that replicates all writes to multiple child caches.
Definition: MultiWriteBagOStuff.php:36
MultiWriteBagOStuff\$caches
BagOStuff[] $caches
Backing cache stores in order of highest to lowest tier.
Definition: MultiWriteBagOStuff.php:38
unserialize
unserialize( $serialized)
Definition: ApiMessageTrait.php:146
$cache
$cache
Definition: mcc.php:33
MultiWriteBagOStuff\callKeyMethodOnTierCache
callKeyMethodOnTierCache( $index, $method, $arg0Sig, $rvSig, array $args)
Call a method on the cache instance for the given cache tier (index)
Definition: MultiWriteBagOStuff.php:414
MultiWriteBagOStuff\makeKeyInternal
makeKeyInternal( $keyspace, $components)
Make a cache key for the given keyspace and components.
Definition: MultiWriteBagOStuff.php:367
BagOStuff\fieldHasFlags
fieldHasFlags( $field, $flags)
Definition: BagOStuff.php:584
$keys
$keys
Definition: testCompression.php:72
MultiWriteBagOStuff\deleteObjectsExpiringBefore
deleteObjectsExpiringBefore( $timestamp, callable $progress=null, $limit=INF, string $tag=null)
Delete all objects expiring before a certain date.
Definition: MultiWriteBagOStuff.php:255
MultiWriteBagOStuff\setNewPreparedValues
setNewPreparedValues(array $valueByKey)
Make a "generic" reversible cache key from the given components.
Definition: MultiWriteBagOStuff.php:387
MultiWriteBagOStuff\deleteMulti
deleteMulti(array $keys, $flags=0)
Batch deletion.
Definition: MultiWriteBagOStuff.php:295
BagOStuff\$keyspace
string $keyspace
Default keyspace; used by makeKey()
Definition: BagOStuff.php:103
MultiWriteBagOStuff\getMulti
getMulti(array $keys, $flags=0)
Get an associative array containing the item for each of the keys that have items.
Definition: MultiWriteBagOStuff.php:271
MultiWriteBagOStuff\$UPGRADE_TTL
static int $UPGRADE_TTL
TTL when a key is copied to a higher cache tier.
Definition: MultiWriteBagOStuff.php:46