MediaWiki  master
MediumSpecificBagOStuff.php
Go to the documentation of this file.
1 <?php
25 
34 abstract class MediumSpecificBagOStuff extends BagOStuff {
36  protected $locks = [];
38  protected $lastError = self::ERR_NONE;
40  protected $keyspace = 'local';
42  protected $syncTimeout;
44  protected $segmentationSize;
47 
49  private $duplicateKeyLookups = [];
51  private $reportDupes = false;
53  private $dupeTrackScheduled = false;
54 
56  protected $busyCallbacks = [];
57 
59  const SEGMENT_COMPONENT = 'segment';
60 
80  public function __construct( array $params = [] ) {
81  parent::__construct( $params );
82 
83  if ( isset( $params['keyspace'] ) ) {
84  $this->keyspace = $params['keyspace'];
85  }
86 
87  if ( !empty( $params['reportDupes'] ) && is_callable( $this->asyncHandler ) ) {
88  $this->reportDupes = true;
89  }
90 
91  $this->syncTimeout = $params['syncTimeout'] ?? 3;
92  $this->segmentationSize = $params['segmentationSize'] ?? 8388608; // 8MiB
93  $this->segmentedValueMaxSize = $params['segmentedValueMaxSize'] ?? 67108864; // 64MiB
94  }
95 
109  public function get( $key, $flags = 0 ) {
110  $this->trackDuplicateKeys( $key );
111 
112  return $this->resolveSegments( $key, $this->doGet( $key, $flags ) );
113  }
114 
119  private function trackDuplicateKeys( $key ) {
120  if ( !$this->reportDupes ) {
121  return;
122  }
123 
124  if ( !isset( $this->duplicateKeyLookups[$key] ) ) {
125  // Track that we have seen this key. This N-1 counting style allows
126  // easy filtering with array_filter() later.
127  $this->duplicateKeyLookups[$key] = 0;
128  } else {
129  $this->duplicateKeyLookups[$key] += 1;
130 
131  if ( $this->dupeTrackScheduled === false ) {
132  $this->dupeTrackScheduled = true;
133  // Schedule a callback that logs keys processed more than once by get().
134  call_user_func( $this->asyncHandler, function () {
135  $dups = array_filter( $this->duplicateKeyLookups );
136  foreach ( $dups as $key => $count ) {
137  $this->logger->warning(
138  'Duplicate get(): "{key}" fetched {count} times',
139  // Count is N-1 of the actual lookup count
140  [ 'key' => $key, 'count' => $count + 1, ]
141  );
142  }
143  } );
144  }
145  }
146  }
147 
154  abstract protected function doGet( $key, $flags = 0, &$casToken = null );
155 
165  public function set( $key, $value, $exptime = 0, $flags = 0 ) {
166  list( $entry, $usable ) = $this->makeValueOrSegmentList( $key, $value, $exptime, $flags );
167  // Only when all segments (if any) are stored should the main key be changed
168  return $usable ? $this->doSet( $key, $entry, $exptime, $flags ) : false;
169  }
170 
180  abstract protected function doSet( $key, $value, $exptime = 0, $flags = 0 );
181 
193  public function delete( $key, $flags = 0 ) {
194  if ( !$this->fieldHasFlags( $flags, self::WRITE_PRUNE_SEGMENTS ) ) {
195  return $this->doDelete( $key, $flags );
196  }
197 
198  $mainValue = $this->doGet( $key, self::READ_LATEST );
199  if ( !$this->doDelete( $key, $flags ) ) {
200  return false;
201  }
202 
203  if ( !SerializedValueContainer::isSegmented( $mainValue ) ) {
204  return true; // no segments to delete
205  }
206 
207  $orderedKeys = array_map(
208  function ( $segmentHash ) use ( $key ) {
209  return $this->makeGlobalKey( self::SEGMENT_COMPONENT, $key, $segmentHash );
210  },
212  );
213 
214  return $this->deleteMulti( $orderedKeys, $flags & ~self::WRITE_PRUNE_SEGMENTS );
215  }
216 
224  abstract protected function doDelete( $key, $flags = 0 );
225 
226  public function add( $key, $value, $exptime = 0, $flags = 0 ) {
227  list( $entry, $usable ) = $this->makeValueOrSegmentList( $key, $value, $exptime, $flags );
228  // Only when all segments (if any) are stored should the main key be changed
229  return $usable ? $this->doAdd( $key, $entry, $exptime, $flags ) : false;
230  }
231 
241  abstract protected function doAdd( $key, $value, $exptime = 0, $flags = 0 );
242 
259  public function merge( $key, callable $callback, $exptime = 0, $attempts = 10, $flags = 0 ) {
260  return $this->mergeViaCas( $key, $callback, $exptime, $attempts, $flags );
261  }
262 
272  final protected function mergeViaCas( $key, callable $callback, $exptime, $attempts, $flags ) {
273  $attemptsLeft = $attempts;
274  do {
275  $token = null; // passed by reference
276  // Get the old value and CAS token from cache
277  $this->clearLastError();
278  $currentValue = $this->resolveSegments(
279  $key,
280  $this->doGet( $key, $flags, $token )
281  );
282  if ( $this->getLastError() ) {
283  // Don't spam slow retries due to network problems (retry only on races)
284  $this->logger->warning(
285  __METHOD__ . ' failed due to read I/O error on get() for {key}.',
286  [ 'key' => $key ]
287  );
288  $success = false;
289  break;
290  }
291 
292  // Derive the new value from the old value
293  $value = call_user_func( $callback, $this, $key, $currentValue, $exptime );
294  $keyWasNonexistant = ( $currentValue === false );
295  $valueMatchesOldValue = ( $value === $currentValue );
296  unset( $currentValue ); // free RAM in case the value is large
297 
298  $this->clearLastError();
299  if ( $value === false || $exptime < 0 ) {
300  $success = true; // do nothing
301  } elseif ( $valueMatchesOldValue && $attemptsLeft !== $attempts ) {
302  $success = true; // recently set by another thread to the same value
303  } elseif ( $keyWasNonexistant ) {
304  // Try to create the key, failing if it gets created in the meantime
305  $success = $this->add( $key, $value, $exptime, $flags );
306  } else {
307  // Try to update the key, failing if it gets changed in the meantime
308  $success = $this->cas( $token, $key, $value, $exptime, $flags );
309  }
310  if ( $this->getLastError() ) {
311  // Don't spam slow retries due to network problems (retry only on races)
312  $this->logger->warning(
313  __METHOD__ . ' failed due to write I/O error for {key}.',
314  [ 'key' => $key ]
315  );
316  $success = false;
317  break;
318  }
319 
320  } while ( !$success && --$attemptsLeft );
321 
322  return $success;
323  }
324 
335  protected function cas( $casToken, $key, $value, $exptime = 0, $flags = 0 ) {
336  if ( $casToken === null ) {
337  $this->logger->warning(
338  __METHOD__ . ' got empty CAS token for {key}.',
339  [ 'key' => $key ]
340  );
341 
342  return false; // caller may have meant to use add()?
343  }
344 
345  list( $entry, $usable ) = $this->makeValueOrSegmentList( $key, $value, $exptime, $flags );
346  // Only when all segments (if any) are stored should the main key be changed
347  return $usable ? $this->doCas( $casToken, $key, $entry, $exptime, $flags ) : false;
348  }
349 
360  protected function doCas( $casToken, $key, $value, $exptime = 0, $flags = 0 ) {
361  // @TODO: the lock() call assumes that all other relavent sets() use one
362  if ( !$this->lock( $key, 0 ) ) {
363  return false; // non-blocking
364  }
365 
366  $curCasToken = null; // passed by reference
367  $this->clearLastError();
368  $this->doGet( $key, self::READ_LATEST, $curCasToken );
369  if ( is_object( $curCasToken ) ) {
370  // Using === does not work with objects since it checks for instance identity
371  throw new UnexpectedValueException( "CAS token cannot be an object" );
372  }
373  if ( $this->getLastError() ) {
374  // Fail if the old CAS token could not be read
375  $success = false;
376  $this->logger->warning(
377  __METHOD__ . ' failed due to write I/O error for {key}.',
378  [ 'key' => $key ]
379  );
380  } elseif ( $casToken === $curCasToken ) {
381  $success = $this->doSet( $key, $value, $exptime, $flags );
382  } else {
383  $success = false; // mismatched or failed
384  $this->logger->info(
385  __METHOD__ . ' failed due to race condition for {key}.',
386  [ 'key' => $key ]
387  );
388  }
389 
390  $this->unlock( $key );
391 
392  return $success;
393  }
394 
412  public function changeTTL( $key, $exptime = 0, $flags = 0 ) {
413  return $this->doChangeTTL( $key, $exptime, $flags );
414  }
415 
422  protected function doChangeTTL( $key, $exptime, $flags ) {
423  if ( !$this->lock( $key, 0 ) ) {
424  return false;
425  }
426 
427  $expiry = $this->getExpirationAsTimestamp( $exptime );
428  $delete = ( $expiry != self::TTL_INDEFINITE && $expiry < $this->getCurrentTime() );
429 
430  // Use doGet() to avoid having to trigger resolveSegments()
431  $blob = $this->doGet( $key, self::READ_LATEST );
432  if ( $blob ) {
433  if ( $delete ) {
434  $ok = $this->doDelete( $key, $flags );
435  } else {
436  $ok = $this->doSet( $key, $blob, $exptime, $flags );
437  }
438  } else {
439  $ok = false;
440  }
441 
442  $this->unlock( $key );
443 
444  return $ok;
445  }
446 
458  public function lock( $key, $timeout = 6, $expiry = 6, $rclass = '' ) {
459  // Avoid deadlocks and allow lock reentry if specified
460  if ( isset( $this->locks[$key] ) ) {
461  if ( $rclass != '' && $this->locks[$key]['class'] === $rclass ) {
462  ++$this->locks[$key]['depth'];
463  return true;
464  } else {
465  return false;
466  }
467  }
468 
469  $fname = __METHOD__;
470  $expiry = min( $expiry ?: INF, self::TTL_DAY );
471  $loop = new WaitConditionLoop(
472  function () use ( $key, $expiry, $fname ) {
473  $this->clearLastError();
474  if ( $this->add( "{$key}:lock", 1, $expiry ) ) {
475  return WaitConditionLoop::CONDITION_REACHED; // locked!
476  } elseif ( $this->getLastError() ) {
477  $this->logger->warning(
478  $fname . ' failed due to I/O error for {key}.',
479  [ 'key' => $key ]
480  );
481 
482  return WaitConditionLoop::CONDITION_ABORTED; // network partition?
483  }
484 
485  return WaitConditionLoop::CONDITION_CONTINUE;
486  },
487  $timeout
488  );
489 
490  $code = $loop->invoke();
491  $locked = ( $code === $loop::CONDITION_REACHED );
492  if ( $locked ) {
493  $this->locks[$key] = [ 'class' => $rclass, 'depth' => 1 ];
494  } elseif ( $code === $loop::CONDITION_TIMED_OUT ) {
495  $this->logger->warning(
496  "$fname failed due to timeout for {key}.",
497  [ 'key' => $key, 'timeout' => $timeout ]
498  );
499  }
500 
501  return $locked;
502  }
503 
510  public function unlock( $key ) {
511  if ( !isset( $this->locks[$key] ) ) {
512  return false;
513  }
514 
515  if ( --$this->locks[$key]['depth'] <= 0 ) {
516  unset( $this->locks[$key] );
517 
518  $ok = $this->doDelete( "{$key}:lock" );
519  if ( !$ok ) {
520  $this->logger->warning(
521  __METHOD__ . ' failed to release lock for {key}.',
522  [ 'key' => $key ]
523  );
524  }
525 
526  return $ok;
527  }
528 
529  return true;
530  }
531 
543  $timestamp,
544  callable $progress = null,
545  $limit = INF
546  ) {
547  return false;
548  }
549 
556  public function getMulti( array $keys, $flags = 0 ) {
557  $foundByKey = $this->doGetMulti( $keys, $flags );
558 
559  $res = [];
560  foreach ( $keys as $key ) {
561  // Resolve one blob at a time (avoids too much I/O at once)
562  if ( array_key_exists( $key, $foundByKey ) ) {
563  // A value should not appear in the key if a segment is missing
564  $value = $this->resolveSegments( $key, $foundByKey[$key] );
565  if ( $value !== false ) {
566  $res[$key] = $value;
567  }
568  }
569  }
570 
571  return $res;
572  }
573 
580  protected function doGetMulti( array $keys, $flags = 0 ) {
581  $res = [];
582  foreach ( $keys as $key ) {
583  $val = $this->doGet( $key, $flags );
584  if ( $val !== false ) {
585  $res[$key] = $val;
586  }
587  }
588 
589  return $res;
590  }
591 
603  public function setMulti( array $data, $exptime = 0, $flags = 0 ) {
604  if ( $this->fieldHasFlags( $flags, self::WRITE_ALLOW_SEGMENTS ) ) {
605  throw new InvalidArgumentException( __METHOD__ . ' got WRITE_ALLOW_SEGMENTS' );
606  }
607 
608  return $this->doSetMulti( $data, $exptime, $flags );
609  }
610 
617  protected function doSetMulti( array $data, $exptime = 0, $flags = 0 ) {
618  $res = true;
619  foreach ( $data as $key => $value ) {
620  $res = $this->doSet( $key, $value, $exptime, $flags ) && $res;
621  }
622 
623  return $res;
624  }
625 
636  public function deleteMulti( array $keys, $flags = 0 ) {
637  if ( $this->fieldHasFlags( $flags, self::WRITE_PRUNE_SEGMENTS ) ) {
638  throw new InvalidArgumentException( __METHOD__ . ' got WRITE_PRUNE_SEGMENTS' );
639  }
640 
641  return $this->doDeleteMulti( $keys, $flags );
642  }
643 
649  protected function doDeleteMulti( array $keys, $flags = 0 ) {
650  $res = true;
651  foreach ( $keys as $key ) {
652  $res = $this->doDelete( $key, $flags ) && $res;
653  }
654  return $res;
655  }
656 
668  public function changeTTLMulti( array $keys, $exptime, $flags = 0 ) {
669  $res = true;
670  foreach ( $keys as $key ) {
671  $res = $this->doChangeTTL( $key, $exptime, $flags ) && $res;
672  }
673 
674  return $res;
675  }
676 
677  public function incrWithInit( $key, $exptime, $value = 1, $init = null, $flags = 0 ) {
678  $init = is_int( $init ) ? $init : $value;
679  $this->clearLastError();
680  $newValue = $this->incr( $key, $value, $flags );
681  if ( $newValue === false && !$this->getLastError() ) {
682  // No key set; initialize
683  $newValue = $this->add( $key, (int)$init, $exptime, $flags ) ? $init : false;
684  if ( $newValue === false && !$this->getLastError() ) {
685  // Raced out initializing; increment
686  $newValue = $this->incr( $key, $value, $flags );
687  }
688  }
689 
690  return $newValue;
691  }
692 
700  final protected function resolveSegments( $key, $mainValue ) {
701  if ( SerializedValueContainer::isUnified( $mainValue ) ) {
702  return $this->unserialize( $mainValue->{SerializedValueContainer::UNIFIED_DATA} );
703  }
704 
705  if ( SerializedValueContainer::isSegmented( $mainValue ) ) {
706  $orderedKeys = array_map(
707  function ( $segmentHash ) use ( $key ) {
708  return $this->makeGlobalKey( self::SEGMENT_COMPONENT, $key, $segmentHash );
709  },
711  );
712 
713  $segmentsByKey = $this->doGetMulti( $orderedKeys );
714 
715  $parts = [];
716  foreach ( $orderedKeys as $segmentKey ) {
717  if ( isset( $segmentsByKey[$segmentKey] ) ) {
718  $parts[] = $segmentsByKey[$segmentKey];
719  } else {
720  return false; // missing segment
721  }
722  }
723 
724  return $this->unserialize( implode( '', $parts ) );
725  }
726 
727  return $mainValue;
728  }
729 
735  public function getLastError() {
736  return $this->lastError;
737  }
738 
743  public function clearLastError() {
744  $this->lastError = self::ERR_NONE;
745  }
746 
752  protected function setLastError( $err ) {
753  $this->lastError = $err;
754  }
755 
756  final public function addBusyCallback( callable $workCallback ) {
757  $this->busyCallbacks[] = $workCallback;
758  }
759 
770  final protected function makeValueOrSegmentList( $key, $value, $exptime, $flags ) {
771  $entry = $value;
772  $usable = true;
773 
774  if (
775  $this->fieldHasFlags( $flags, self::WRITE_ALLOW_SEGMENTS ) &&
776  !is_int( $value ) && // avoid breaking incr()/decr()
777  is_finite( $this->segmentationSize )
778  ) {
779  $segmentSize = $this->segmentationSize;
780  $maxTotalSize = $this->segmentedValueMaxSize;
781 
782  $serialized = $this->serialize( $value );
783  $size = strlen( $serialized );
784  if ( $size > $maxTotalSize ) {
785  $this->logger->warning(
786  "Value for {key} exceeds $maxTotalSize bytes; cannot segment.",
787  [ 'key' => $key ]
788  );
789  } elseif ( $size <= $segmentSize ) {
790  // The serialized value was already computed, so just use it inline
792  } else {
793  // Split the serialized value into chunks and store them at different keys
794  $chunksByKey = [];
795  $segmentHashes = [];
796  $count = intdiv( $size, $segmentSize ) + ( ( $size % $segmentSize ) ? 1 : 0 );
797  for ( $i = 0; $i < $count; ++$i ) {
798  $segment = substr( $serialized, $i * $segmentSize, $segmentSize );
799  $hash = sha1( $segment );
800  $chunkKey = $this->makeGlobalKey( self::SEGMENT_COMPONENT, $key, $hash );
801  $chunksByKey[$chunkKey] = $segment;
802  $segmentHashes[] = $hash;
803  }
804  $flags &= ~self::WRITE_ALLOW_SEGMENTS; // sanity
805  $usable = $this->setMulti( $chunksByKey, $exptime, $flags );
806  $entry = SerializedValueContainer::newSegmented( $segmentHashes );
807  }
808  }
809 
810  return [ $entry, $usable ];
811  }
812 
818  final protected function isRelativeExpiration( $exptime ) {
819  return ( $exptime !== self::TTL_INDEFINITE && $exptime < ( 10 * self::TTL_YEAR ) );
820  }
821 
835  final protected function getExpirationAsTimestamp( $exptime ) {
836  if ( $exptime == self::TTL_INDEFINITE ) {
837  return $exptime;
838  }
839 
840  return $this->isRelativeExpiration( $exptime )
841  ? intval( $this->getCurrentTime() + $exptime )
842  : $exptime;
843  }
844 
859  final protected function getExpirationAsTTL( $exptime ) {
860  if ( $exptime == self::TTL_INDEFINITE ) {
861  return $exptime;
862  }
863 
864  return $this->isRelativeExpiration( $exptime )
865  ? $exptime
866  : (int)max( $exptime - $this->getCurrentTime(), 1 );
867  }
868 
875  final protected function isInteger( $value ) {
876  if ( is_int( $value ) ) {
877  return true;
878  } elseif ( !is_string( $value ) ) {
879  return false;
880  }
881 
882  $integer = (int)$value;
883 
884  return ( $value === (string)$integer );
885  }
886 
887  public function makeKeyInternal( $keyspace, $args ) {
888  $key = $keyspace;
889  foreach ( $args as $arg ) {
890  $key .= ':' . str_replace( ':', '%3A', $arg );
891  }
892  return strtr( $key, ' ', '_' );
893  }
894 
903  public function makeGlobalKey( $class, ...$components ) {
904  return $this->makeKeyInternal( 'global', func_get_args() );
905  }
906 
915  public function makeKey( $class, ...$components ) {
916  return $this->makeKeyInternal( $this->keyspace, func_get_args() );
917  }
918 
924  public function getQoS( $flag ) {
925  return $this->attrMap[$flag] ?? self::QOS_UNKNOWN;
926  }
927 
928  public function getSegmentationSize() {
930  }
931 
932  public function getSegmentedValueMaxSize() {
934  }
935 
941  protected function serialize( $value ) {
942  return is_int( $value ) ? $value : serialize( $value );
943  }
944 
950  protected function unserialize( $value ) {
951  return $this->isInteger( $value ) ? (int)$value : unserialize( $value );
952  }
953 
957  protected function debug( $text ) {
958  if ( $this->debugMode ) {
959  $this->logger->debug( "{class} debug: $text", [ 'class' => static::class ] );
960  }
961  }
962 }
trackDuplicateKeys( $key)
Track the number of times that a given key has been used.
getMulti(array $keys, $flags=0)
Get an associative array containing the item for each of the keys that have items.
add( $key, $value, $exptime=0, $flags=0)
clearLastError()
Clear the "last error" registry.
$success
int $segmentedValueMaxSize
Bytes; maximum total size of a segmented cache value.
array [] $locks
Lock tracking.
doSetMulti(array $data, $exptime=0, $flags=0)
changeTTL( $key, $exptime=0, $flags=0)
Change the expiration on a key if it exists.
doCas( $casToken, $key, $value, $exptime=0, $flags=0)
Check and set an item.
changeTTLMulti(array $keys, $exptime, $flags=0)
Change the expiration of multiple keys that exist.
deleteMulti(array $keys, $flags=0)
Batch deletion.
addBusyCallback(callable $workCallback)
getCurrentTime()
Definition: BagOStuff.php:523
if( $line===false) $args
Definition: cdb.php:64
makeKey( $class,... $components)
Make a cache key, scoped to this instance&#39;s keyspace.
incrWithInit( $key, $exptime, $value=1, $init=null, $flags=0)
doGetMulti(array $keys, $flags=0)
Get an associative array containing the item for each of the keys that have items.
incr( $key, $value=1, $flags=0)
Increase stored value of $key by $value while preserving its TTL.
makeValueOrSegmentList( $key, $value, $exptime, $flags)
Determine the entry (inline or segment list) to store under a key to save the value.
int $segmentationSize
Bytes; chunk size of segmented cache values.
setMulti(array $data, $exptime=0, $flags=0)
Batch insertion/replace.
static newSegmented(array $segmentHashList)
makeGlobalKey( $class,... $components)
Make a global cache key.
lock( $key, $timeout=6, $expiry=6, $rclass='')
Acquire an advisory lock on a key string.
Storage medium specific cache for storing items (e.g.
doGet( $key, $flags=0, &$casToken=null)
resolveSegments( $key, $mainValue)
Get and reassemble the chunks of blob at the given key.
deleteObjectsExpiringBefore( $timestamp, callable $progress=null, $limit=INF)
Delete all objects expiring before a certain date.
getExpirationAsTimestamp( $exptime)
Convert an optionally relative timestamp to an absolute time.
doChangeTTL( $key, $exptime, $flags)
doSet( $key, $value, $exptime=0, $flags=0)
Set an item.
const WRITE_ALLOW_SEGMENTS
Definition: BagOStuff.php:84
getLastError()
Get the "last error" registered; clearLastError() should be called manually.
doDelete( $key, $flags=0)
Delete an item.
merge( $key, callable $callback, $exptime=0, $attempts=10, $flags=0)
Merge changes into the existing cache value (possibly creating a new one)
getExpirationAsTTL( $exptime)
Convert an optionally absolute expiry time to a relative time.
setLastError( $err)
Set the "last error" registry.
int $lastError
ERR_* class constant.
doDeleteMulti(array $keys, $flags=0)
mergeViaCas( $key, callable $callback, $exptime, $attempts, $flags)
foreach( $res as $row) $serialized
cas( $casToken, $key, $value, $exptime=0, $flags=0)
Check and set an item.
isInteger( $value)
Check if a value is an integer.
doAdd( $key, $value, $exptime=0, $flags=0)
Insert an item if it does not already exist.
fieldHasFlags( $field, $flags)
Definition: BagOStuff.php:493
unlock( $key)
Release an advisory lock on a key string.