MediaWiki  master
MediumSpecificBagOStuff.php
Go to the documentation of this file.
1 <?php
24 use Wikimedia\WaitConditionLoop;
25 
34 abstract class MediumSpecificBagOStuff extends BagOStuff {
36  protected $locks = [];
38  protected $syncTimeout;
40  protected $segmentationSize;
43 
45  private $duplicateKeyLookups = [];
47  private $reportDupes = false;
49  private $dupeTrackScheduled = false;
50 
52  protected $busyCallbacks = [];
53 
55  protected $preparedValues = [];
56 
58  private const SEGMENT_COMPONENT = 'segment';
59 
61  protected const PASS_BY_REF = -1;
62 
63  protected const METRIC_OP_GET = 'get';
64  protected const METRIC_OP_SET = 'set';
65  protected const METRIC_OP_DELETE = 'delete';
66  protected const METRIC_OP_CHANGE_TTL = 'change_ttl';
67  protected const METRIC_OP_ADD = 'add';
68  protected const METRIC_OP_INCR = 'incr';
69  protected const METRIC_OP_DECR = 'decr';
70  protected const METRIC_OP_CAS = 'cas';
71 
72  protected const LOCK_RCLASS = 0;
73  protected const LOCK_DEPTH = 1;
74  protected const LOCK_TIME = 2;
75  protected const LOCK_EXPIRY = 3;
76 
94  public function __construct( array $params = [] ) {
95  parent::__construct( $params );
96 
97  if ( !empty( $params['reportDupes'] ) && is_callable( $this->asyncHandler ) ) {
98  $this->reportDupes = true;
99  }
100 
101  $this->syncTimeout = $params['syncTimeout'] ?? 3;
102  $this->segmentationSize = $params['segmentationSize'] ?? 8388608; // 8MiB
103  $this->segmentedValueMaxSize = $params['segmentedValueMaxSize'] ?? 67108864; // 64MiB
104  }
105 
119  public function get( $key, $flags = 0 ) {
120  $this->trackDuplicateKeys( $key );
121 
122  return $this->resolveSegments( $key, $this->doGet( $key, $flags ) );
123  }
124 
129  private function trackDuplicateKeys( $key ) {
130  if ( !$this->reportDupes ) {
131  return;
132  }
133 
134  if ( !isset( $this->duplicateKeyLookups[$key] ) ) {
135  // Track that we have seen this key. This N-1 counting style allows
136  // easy filtering with array_filter() later.
137  $this->duplicateKeyLookups[$key] = 0;
138  } else {
139  $this->duplicateKeyLookups[$key] += 1;
140 
141  if ( $this->dupeTrackScheduled === false ) {
142  $this->dupeTrackScheduled = true;
143  // Schedule a callback that logs keys processed more than once by get().
144  call_user_func( $this->asyncHandler, function () {
145  $dups = array_filter( $this->duplicateKeyLookups );
146  foreach ( $dups as $key => $count ) {
147  $this->logger->warning(
148  'Duplicate get(): "{key}" fetched {count} times',
149  // Count is N-1 of the actual lookup count
150  [ 'key' => $key, 'count' => $count + 1, ]
151  );
152  }
153  } );
154  }
155  }
156  }
157 
164  abstract protected function doGet( $key, $flags = 0, &$casToken = null );
165 
175  public function set( $key, $value, $exptime = 0, $flags = 0 ) {
176  list( $entry, $usable ) = $this->makeValueOrSegmentList( $key, $value, $exptime, $flags );
177  // Only when all segments (if any) are stored should the main key be changed
178  return $usable ? $this->doSet( $key, $entry, $exptime, $flags ) : false;
179  }
180 
190  abstract protected function doSet( $key, $value, $exptime = 0, $flags = 0 );
191 
203  public function delete( $key, $flags = 0 ) {
204  if ( !$this->fieldHasFlags( $flags, self::WRITE_PRUNE_SEGMENTS ) ) {
205  return $this->doDelete( $key, $flags );
206  }
207 
208  $mainValue = $this->doGet( $key, self::READ_LATEST );
209  if ( !$this->doDelete( $key, $flags ) ) {
210  return false;
211  }
212 
213  if ( !SerializedValueContainer::isSegmented( $mainValue ) ) {
214  return true; // no segments to delete
215  }
216 
217  $orderedKeys = array_map(
218  function ( $segmentHash ) use ( $key ) {
219  return $this->makeGlobalKey( self::SEGMENT_COMPONENT, $key, $segmentHash );
220  },
222  );
223 
224  return $this->deleteMulti( $orderedKeys, $flags & ~self::WRITE_PRUNE_SEGMENTS );
225  }
226 
234  abstract protected function doDelete( $key, $flags = 0 );
235 
236  public function add( $key, $value, $exptime = 0, $flags = 0 ) {
237  list( $entry, $usable ) = $this->makeValueOrSegmentList( $key, $value, $exptime, $flags );
238  // Only when all segments (if any) are stored should the main key be changed
239  return $usable ? $this->doAdd( $key, $entry, $exptime, $flags ) : false;
240  }
241 
251  abstract protected function doAdd( $key, $value, $exptime = 0, $flags = 0 );
252 
269  public function merge( $key, callable $callback, $exptime = 0, $attempts = 10, $flags = 0 ) {
270  return $this->mergeViaCas( $key, $callback, $exptime, $attempts, $flags );
271  }
272 
282  final protected function mergeViaCas( $key, callable $callback, $exptime, $attempts, $flags ) {
283  $attemptsLeft = $attempts;
284  do {
285  $token = self::PASS_BY_REF; // passed by reference
286  // Get the old value and CAS token from cache
287  $watchPoint = $this->watchErrors();
288  $currentValue = $this->resolveSegments(
289  $key,
290  $this->doGet( $key, $flags, $token )
291  );
292  if ( $this->getLastError( $watchPoint ) ) {
293  // Don't spam slow retries due to network problems (retry only on races)
294  $this->logger->warning(
295  __METHOD__ . ' failed due to read I/O error on get() for {key}.',
296  [ 'key' => $key ]
297  );
298  $success = false;
299  break;
300  }
301 
302  // Derive the new value from the old value
303  $value = $callback( $this, $key, $currentValue, $exptime );
304  $keyWasNonexistant = ( $currentValue === false );
305  $valueMatchesOldValue = ( $value === $currentValue );
306  unset( $currentValue ); // free RAM in case the value is large
307 
308  $watchPoint = $this->watchErrors();
309  if ( $value === false || $exptime < 0 ) {
310  $success = true; // do nothing
311  } elseif ( $valueMatchesOldValue && $attemptsLeft !== $attempts ) {
312  $success = true; // recently set by another thread to the same value
313  } elseif ( $keyWasNonexistant ) {
314  // Try to create the key, failing if it gets created in the meantime
315  $success = $this->add( $key, $value, $exptime, $flags );
316  } else {
317  // Try to update the key, failing if it gets changed in the meantime
318  $success = $this->cas( $token, $key, $value, $exptime, $flags );
319  }
320  if ( $this->getLastError( $watchPoint ) ) {
321  // Don't spam slow retries due to network problems (retry only on races)
322  $this->logger->warning(
323  __METHOD__ . ' failed due to write I/O error for {key}.',
324  [ 'key' => $key ]
325  );
326  $success = false;
327  break;
328  }
329 
330  } while ( !$success && --$attemptsLeft );
331 
332  return $success;
333  }
334 
345  protected function cas( $casToken, $key, $value, $exptime = 0, $flags = 0 ) {
346  if ( $casToken === null ) {
347  $this->logger->warning(
348  __METHOD__ . ' got empty CAS token for {key}.',
349  [ 'key' => $key ]
350  );
351 
352  return false; // caller may have meant to use add()?
353  }
354 
355  list( $entry, $usable ) = $this->makeValueOrSegmentList( $key, $value, $exptime, $flags );
356  // Only when all segments (if any) are stored should the main key be changed
357  return $usable ? $this->doCas( $casToken, $key, $entry, $exptime, $flags ) : false;
358  }
359 
370  protected function doCas( $casToken, $key, $value, $exptime = 0, $flags = 0 ) {
371  // @TODO: the use of lock() assumes that all other relevant sets() use a lock
372  if ( !$this->lock( $key, 0 ) ) {
373  return false; // non-blocking
374  }
375 
376  $curCasToken = self::PASS_BY_REF; // passed by reference
377  $watchPoint = $this->watchErrors();
378  $this->doGet( $key, self::READ_LATEST, $curCasToken );
379  if ( is_object( $curCasToken ) ) {
380  // Using === does not work with objects since it checks for instance identity
381  throw new UnexpectedValueException( "CAS token cannot be an object" );
382  }
383  if ( $this->getLastError( $watchPoint ) ) {
384  // Fail if the old CAS token could not be read
385  $success = false;
386  $this->logger->warning(
387  __METHOD__ . ' failed due to write I/O error for {key}.',
388  [ 'key' => $key ]
389  );
390  } elseif ( $casToken === $curCasToken ) {
391  $success = $this->doSet( $key, $value, $exptime, $flags );
392  } else {
393  $success = false; // mismatched or failed
394  $this->logger->info(
395  __METHOD__ . ' failed due to race condition for {key}.',
396  [ 'key' => $key ]
397  );
398  }
399 
400  $this->unlock( $key );
401 
402  return $success;
403  }
404 
422  public function changeTTL( $key, $exptime = 0, $flags = 0 ) {
423  return $this->doChangeTTL( $key, $exptime, $flags );
424  }
425 
432  protected function doChangeTTL( $key, $exptime, $flags ) {
433  // @TODO: the use of lock() assumes that all other relevant sets() use a lock
434  if ( !$this->lock( $key, 0 ) ) {
435  return false;
436  }
437 
438  $expiry = $this->getExpirationAsTimestamp( $exptime );
439  $delete = ( $expiry != self::TTL_INDEFINITE && $expiry < $this->getCurrentTime() );
440 
441  // Use doGet() to avoid having to trigger resolveSegments()
442  $blob = $this->doGet( $key, self::READ_LATEST );
443  if ( $blob ) {
444  if ( $delete ) {
445  $ok = $this->doDelete( $key, $flags );
446  } else {
447  $ok = $this->doSet( $key, $blob, $exptime, $flags );
448  }
449  } else {
450  $ok = false;
451  }
452 
453  $this->unlock( $key );
454 
455  return $ok;
456  }
457 
465  public function lock( $key, $timeout = 6, $exptime = 6, $rclass = '' ) {
466  $exptime = min( $exptime ?: INF, self::TTL_DAY );
467 
468  $acquired = false;
469 
470  if ( isset( $this->locks[$key] ) ) {
471  // Already locked; avoid deadlocks and allow lock reentry if specified
472  if ( $rclass != '' && $this->locks[$key][self::LOCK_RCLASS] === $rclass ) {
473  ++$this->locks[$key][self::LOCK_DEPTH];
474  $acquired = true;
475  }
476  } else {
477  // Not already locked; acquire a lock on the backend
478  $lockTsUnix = $this->doLock( $key, $timeout, $exptime );
479  if ( $lockTsUnix !== null ) {
480  $this->locks[$key] = [
481  self::LOCK_RCLASS => $rclass,
482  self::LOCK_DEPTH => 1,
483  self::LOCK_TIME => $lockTsUnix,
484  self::LOCK_EXPIRY => $lockTsUnix + $exptime
485  ];
486  $acquired = true;
487  }
488  }
489 
490  return $acquired;
491  }
492 
501  protected function doLock( $key, $timeout, $exptime ) {
502  $lockTsUnix = null;
503 
504  $fname = __METHOD__;
505  $loop = new WaitConditionLoop(
506  function () use ( $key, $exptime, $fname, &$lockTsUnix ) {
507  $watchPoint = $this->watchErrors();
508  if ( $this->add( $this->makeLockKey( $key ), 1, $exptime ) ) {
509  $lockTsUnix = microtime( true );
510 
511  return WaitConditionLoop::CONDITION_REACHED; // locked!
512  } elseif ( $this->getLastError( $watchPoint ) ) {
513  $this->logger->warning(
514  "$fname failed due to I/O error for {key}.",
515  [ 'key' => $key ]
516  );
517 
518  return WaitConditionLoop::CONDITION_ABORTED; // network partition?
519  }
520 
521  return WaitConditionLoop::CONDITION_CONTINUE;
522  },
523  $timeout
524  );
525  $code = $loop->invoke();
526 
527  if ( $code === $loop::CONDITION_TIMED_OUT ) {
528  $this->logger->warning(
529  "$fname failed due to timeout for {key}.",
530  [ 'key' => $key, 'timeout' => $timeout ]
531  );
532  }
533 
534  return $lockTsUnix;
535  }
536 
543  public function unlock( $key ) {
544  $released = false;
545 
546  if ( isset( $this->locks[$key] ) ) {
547  if ( --$this->locks[$key][self::LOCK_DEPTH] > 0 ) {
548  $released = true;
549  } else {
550  $released = $this->doUnlock( $key );
551  unset( $this->locks[$key] );
552  if ( !$released ) {
553  $this->logger->warning(
554  __METHOD__ . ' failed to release lock for {key}.',
555  [ 'key' => $key ]
556  );
557  }
558  }
559  } else {
560  $this->logger->warning(
561  __METHOD__ . ' no lock to release for {key}.',
562  [ 'key' => $key ]
563  );
564  }
565 
566  return $released;
567  }
568 
575  protected function doUnlock( $key ) {
576  // Estimate the remaining TTL of the lock key
577  $curTTL = $this->locks[$key][self::LOCK_EXPIRY] - $this->getCurrentTime();
578  // Maximum expected one-way-delay for a query to reach the backend
579  $maxOWD = 0.050;
580 
581  $released = false;
582 
583  if ( ( $curTTL - $maxOWD ) > 0 ) {
584  // The lock key is extremely unlikely to expire before a deletion operation
585  // sent from this method arrives on the relevant backend server
586  $released = $this->doDelete( $this->makeLockKey( $key ) );
587  } else {
588  // It is unsafe for this method to delete the lock key due to the risk of it
589  // expiring and being claimed by another thread before the deletion operation
590  // arrives on the backend server
591  $this->logger->warning(
592  "Lock for {key} held too long ({age} sec).",
593  [ 'key' => $key, 'curTTL' => $curTTL ]
594  );
595  }
596 
597  return $released;
598  }
599 
604  protected function makeLockKey( $key ) {
605  return "$key:lock";
606  }
607 
609  $timestamp,
610  callable $progress = null,
611  $limit = INF,
612  string $tag = null
613  ) {
614  return false;
615  }
616 
623  public function getMulti( array $keys, $flags = 0 ) {
624  $foundByKey = $this->doGetMulti( $keys, $flags );
625 
626  $res = [];
627  foreach ( $keys as $key ) {
628  // Resolve one blob at a time (avoids too much I/O at once)
629  if ( array_key_exists( $key, $foundByKey ) ) {
630  // A value should not appear in the key if a segment is missing
631  $value = $this->resolveSegments( $key, $foundByKey[$key] );
632  if ( $value !== false ) {
633  $res[$key] = $value;
634  }
635  }
636  }
637 
638  return $res;
639  }
640 
647  protected function doGetMulti( array $keys, $flags = 0 ) {
648  $res = [];
649  foreach ( $keys as $key ) {
650  $val = $this->doGet( $key, $flags );
651  if ( $val !== false ) {
652  $res[$key] = $val;
653  }
654  }
655 
656  return $res;
657  }
658 
670  public function setMulti( array $valueByKey, $exptime = 0, $flags = 0 ) {
671  if ( $this->fieldHasFlags( $flags, self::WRITE_ALLOW_SEGMENTS ) ) {
672  throw new InvalidArgumentException( __METHOD__ . ' got WRITE_ALLOW_SEGMENTS' );
673  }
674 
675  return $this->doSetMulti( $valueByKey, $exptime, $flags );
676  }
677 
684  protected function doSetMulti( array $data, $exptime = 0, $flags = 0 ) {
685  $res = true;
686  foreach ( $data as $key => $value ) {
687  $res = $this->doSet( $key, $value, $exptime, $flags ) && $res;
688  }
689 
690  return $res;
691  }
692 
703  public function deleteMulti( array $keys, $flags = 0 ) {
704  if ( $this->fieldHasFlags( $flags, self::WRITE_PRUNE_SEGMENTS ) ) {
705  throw new InvalidArgumentException( __METHOD__ . ' got WRITE_PRUNE_SEGMENTS' );
706  }
707 
708  return $this->doDeleteMulti( $keys, $flags );
709  }
710 
716  protected function doDeleteMulti( array $keys, $flags = 0 ) {
717  $res = true;
718  foreach ( $keys as $key ) {
719  $res = $this->doDelete( $key, $flags ) && $res;
720  }
721  return $res;
722  }
723 
734  public function changeTTLMulti( array $keys, $exptime, $flags = 0 ) {
735  return $this->doChangeTTLMulti( $keys, $exptime, $flags );
736  }
737 
744  protected function doChangeTTLMulti( array $keys, $exptime, $flags = 0 ) {
745  $res = true;
746  foreach ( $keys as $key ) {
747  $res = $this->doChangeTTL( $key, $exptime, $flags ) && $res;
748  }
749 
750  return $res;
751  }
752 
753  public function incrWithInit( $key, $exptime, $value = 1, $init = null, $flags = 0 ) {
754  $init = is_int( $init ) ? $init : $value;
755  $watchPoint = $this->watchErrors();
756  $newValue = $this->incr( $key, $value, $flags );
757  if ( $newValue === false && !$this->getLastError( $watchPoint ) ) {
758  // No key set; initialize
759  $newValue = $this->add( $key, (int)$init, $exptime, $flags ) ? $init : false;
760  if ( $newValue === false && !$this->getLastError( $watchPoint ) ) {
761  // Raced out initializing; increment
762  $newValue = $this->incr( $key, $value, $flags );
763  }
764  }
765 
766  return $newValue;
767  }
768 
776  final protected function resolveSegments( $key, $mainValue ) {
777  if ( SerializedValueContainer::isUnified( $mainValue ) ) {
778  return $this->unserialize( $mainValue->{SerializedValueContainer::UNIFIED_DATA} );
779  }
780 
781  if ( SerializedValueContainer::isSegmented( $mainValue ) ) {
782  $orderedKeys = array_map(
783  function ( $segmentHash ) use ( $key ) {
784  return $this->makeGlobalKey( self::SEGMENT_COMPONENT, $key, $segmentHash );
785  },
787  );
788 
789  $segmentsByKey = $this->doGetMulti( $orderedKeys );
790 
791  $parts = [];
792  foreach ( $orderedKeys as $segmentKey ) {
793  if ( isset( $segmentsByKey[$segmentKey] ) ) {
794  $parts[] = $segmentsByKey[$segmentKey];
795  } else {
796  return false; // missing segment
797  }
798  }
799 
800  return $this->unserialize( implode( '', $parts ) );
801  }
802 
803  return $mainValue;
804  }
805 
806  final public function addBusyCallback( callable $workCallback ) {
807  $this->busyCallbacks[] = $workCallback;
808  }
809 
820  final protected function makeValueOrSegmentList( $key, $value, $exptime, $flags ) {
821  $entry = $value;
822  $usable = true;
823 
824  if (
825  $this->fieldHasFlags( $flags, self::WRITE_ALLOW_SEGMENTS ) &&
826  !is_int( $value ) && // avoid breaking incr()/decr()
827  is_finite( $this->segmentationSize )
828  ) {
829  $segmentSize = $this->segmentationSize;
830  $maxTotalSize = $this->segmentedValueMaxSize;
831 
832  $serialized = $this->getSerialized( $value, $key );
833  $size = strlen( $serialized );
834  if ( $size > $maxTotalSize ) {
835  $this->logger->warning(
836  "Value for {key} exceeds $maxTotalSize bytes; cannot segment.",
837  [ 'key' => $key ]
838  );
839  } elseif ( $size <= $segmentSize ) {
840  // The serialized value was already computed, so just use it inline
842  } else {
843  // Split the serialized value into chunks and store them at different keys
844  $chunksByKey = [];
845  $segmentHashes = [];
846  $count = intdiv( $size, $segmentSize ) + ( ( $size % $segmentSize ) ? 1 : 0 );
847  for ( $i = 0; $i < $count; ++$i ) {
848  $segment = substr( $serialized, $i * $segmentSize, $segmentSize );
849  $hash = sha1( $segment );
850  $chunkKey = $this->makeGlobalKey( self::SEGMENT_COMPONENT, $key, $hash );
851  $chunksByKey[$chunkKey] = $segment;
852  $segmentHashes[] = $hash;
853  }
854  $flags &= ~self::WRITE_ALLOW_SEGMENTS;
855  $usable = $this->setMulti( $chunksByKey, $exptime, $flags );
856  $entry = SerializedValueContainer::newSegmented( $segmentHashes );
857  }
858  }
859 
860  return [ $entry, $usable ];
861  }
862 
868  final protected function isRelativeExpiration( $exptime ) {
869  return ( $exptime !== self::TTL_INDEFINITE && $exptime < ( 10 * self::TTL_YEAR ) );
870  }
871 
885  final protected function getExpirationAsTimestamp( $exptime ) {
886  if ( $exptime == self::TTL_INDEFINITE ) {
887  return $exptime;
888  }
889 
890  return $this->isRelativeExpiration( $exptime )
891  ? intval( $this->getCurrentTime() + $exptime )
892  : $exptime;
893  }
894 
909  final protected function getExpirationAsTTL( $exptime ) {
910  if ( $exptime == self::TTL_INDEFINITE ) {
911  return $exptime;
912  }
913 
914  return $this->isRelativeExpiration( $exptime )
915  ? $exptime
916  : (int)max( $exptime - $this->getCurrentTime(), 1 );
917  }
918 
925  final protected function isInteger( $value ) {
926  if ( is_int( $value ) ) {
927  return true;
928  } elseif ( !is_string( $value ) ) {
929  return false;
930  }
931 
932  $integer = (int)$value;
933 
934  return ( $value === (string)$integer );
935  }
936 
937  public function makeGlobalKey( $collection, ...$components ) {
938  return $this->makeKeyInternal( self::GLOBAL_KEYSPACE, func_get_args() );
939  }
940 
941  public function makeKey( $collection, ...$components ) {
942  return $this->makeKeyInternal( $this->keyspace, func_get_args() );
943  }
944 
945  protected function convertGenericKey( $key ) {
946  $components = $this->componentsFromGenericKey( $key );
947  if ( count( $components ) < 2 ) {
948  // Legacy key not from makeKey()/makeGlobalKey(); keep it as-is
949  return $key;
950  }
951 
952  $keyspace = array_shift( $components );
953 
954  return $this->makeKeyInternal( $keyspace, $components );
955  }
956 
957  public function getQoS( $flag ) {
958  return $this->attrMap[$flag] ?? self::QOS_UNKNOWN;
959  }
960 
961  public function getSegmentationSize() {
962  return $this->segmentationSize;
963  }
964 
965  public function getSegmentedValueMaxSize() {
966  return $this->segmentedValueMaxSize;
967  }
968 
969  public function setNewPreparedValues( array $valueByKey ) {
970  $this->preparedValues = [];
971 
972  $sizes = [];
973  foreach ( $valueByKey as $key => $value ) {
974  if ( $value === false ) {
975  $sizes[] = null; // not storable, don't bother
976  continue;
977  }
978 
979  $serialized = $this->serialize( $value );
980  $sizes[] = ( $serialized !== false ) ? strlen( $serialized ) : null;
981 
982  $this->preparedValues[$key] = [ $value, $serialized ];
983  }
984 
985  return $sizes;
986  }
987 
998  protected function getSerialized( $value, $key ) {
999  // Reuse any available prepared (serialized) value
1000  if ( array_key_exists( $key, $this->preparedValues ) ) {
1001  list( $prepValue, $prepSerialized ) = $this->preparedValues[$key];
1002  // Normally, this comparison should only take a few microseconds to confirm a match.
1003  // Using "===" on variables of different types is always fast. It is also fast for
1004  // variables of matching type int, float, bool, null, and object. Lastly, it is fast
1005  // for comparing arrays/strings if they are copy-on-write references, which should be
1006  // the case at this point, assuming prepareValues() was called correctly.
1007  if ( $prepValue === $value ) {
1008  unset( $this->preparedValues[$key] );
1009 
1010  return $prepSerialized;
1011  }
1012  }
1013 
1014  $this->checkValueSerializability( $value, $key );
1015 
1016  return $this->serialize( $value );
1017  }
1018 
1028  protected function guessSerialValueSize( $value, $depth = 0, &$loops = 0 ) {
1029  // Include serialization format overhead estimates roughly based on serialize(),
1030  // without counting . Also, int/float variables use the largest case
1031  // byte size for numbers of that type; this avoids CPU overhead for large arrays.
1032  switch ( gettype( $value ) ) {
1033  case 'string':
1034  // E.g. "<type><delim1><quote><value><quote><delim2>"
1035  return strlen( $value ) + 5;
1036  case 'integer':
1037  // E.g. "<type><delim1><sign><2^63><delim2>";
1038  // ceil(log10 (2^63)) = 19
1039  return 23;
1040  case 'double':
1041  // E.g. "<type><delim1><sign><2^52><esign><2^10><delim2>"
1042  // ceil(log10 (2^52)) = 16 and ceil(log10 (2^10)) = 4
1043  return 25;
1044  case 'boolean':
1045  // E.g. "true" becomes "1" and "false" is not storable
1046  return $value ? 1 : null;
1047  case 'NULL':
1048  return 1; // "\0"
1049  case 'array':
1050  case 'object':
1051  // Give up and guess if there is too much depth
1052  if ( $depth >= 5 && $loops >= 256 ) {
1053  return 1024;
1054  }
1055 
1056  ++$loops;
1057  // E.g. "<type><delim1><brace><<Kn><Vn> for all n><brace><delim2>"
1058  $size = 5;
1059  // Note that casting to an array includes private object members
1060  foreach ( (array)$value as $k => $v ) {
1061  // Inline the recursive result here for performance
1062  $size += is_string( $k ) ? ( strlen( $k ) + 5 ) : 23;
1063  $size += $this->guessSerialValueSize( $v, $depth + 1, $loops );
1064  }
1065 
1066  return $size;
1067  default:
1068  return null; // invalid
1069  }
1070  }
1071 
1092  private function checkValueSerializability( $value, $key ) {
1093  if ( is_array( $value ) ) {
1094  $this->checkIterableMapSerializability( $value, $key );
1095  } elseif ( is_object( $value ) ) {
1096  // Note that Closure instances count as objects
1097  if ( $value instanceof stdClass ) {
1098  $this->checkIterableMapSerializability( $value, $key );
1099  } elseif ( !( $value instanceof JsonSerializable ) ) {
1100  $this->logger->warning(
1101  "{class} value for '{cachekey}'; serialization is suspect.",
1102  [ 'cachekey' => $key, 'class' => get_class( $value ) ]
1103  );
1104  }
1105  }
1106  }
1107 
1112  private function checkIterableMapSerializability( $value, $key ) {
1113  foreach ( $value as $index => $entry ) {
1114  if ( is_object( $entry ) ) {
1115  // Note that Closure instances count as objects
1116  if (
1117  !( $entry instanceof stdClass ) &&
1118  !( $entry instanceof JsonSerializable )
1119  ) {
1120  $this->logger->warning(
1121  "{class} value for '{cachekey}' at '$index'; serialization is suspect.",
1122  [ 'cachekey' => $key, 'class' => get_class( $entry ) ]
1123  );
1124 
1125  return;
1126  }
1127  }
1128  }
1129  }
1130 
1136  protected function serialize( $value ) {
1137  return is_int( $value ) ? $value : serialize( $value );
1138  }
1139 
1145  protected function unserialize( $value ) {
1146  return $this->isInteger( $value ) ? (int)$value : unserialize( $value );
1147  }
1148 
1152  protected function debug( $text ) {
1153  $this->logger->debug( "{class} debug: $text", [ 'class' => static::class ] );
1154  }
1155 
1163  protected function updateOpStats( string $op, array $keyInfo ) {
1164  $deltasByMetric = [];
1165 
1166  foreach ( $keyInfo as $indexOrKey => $keyOrSizes ) {
1167  if ( is_array( $keyOrSizes ) ) {
1168  $key = $indexOrKey;
1169  list( $sPayloadSize, $rPayloadSize ) = $keyOrSizes;
1170  } else {
1171  $key = $keyOrSizes;
1172  $sPayloadSize = null;
1173  $rPayloadSize = null;
1174  }
1175 
1176  // Metric prefix for the cache wrapper and key collection name
1177  $prefix = $this->determineKeyPrefixForStats( $key );
1178 
1179  if ( $op === self::METRIC_OP_GET ) {
1180  // This operation was either a "hit" or "miss" for this key
1181  $name = "{$prefix}.{$op}_" . ( $rPayloadSize === false ? 'miss_rate' : 'hit_rate' );
1182  } else {
1183  // There is no concept of "hit" or "miss" for this operation
1184  $name = "{$prefix}.{$op}_call_rate";
1185  }
1186  $deltasByMetric[$name] = ( $deltasByMetric[$name] ?? 0 ) + 1;
1187 
1188  if ( $sPayloadSize > 0 ) {
1189  $name = "{$prefix}.{$op}_bytes_sent";
1190  $deltasByMetric[$name] = ( $deltasByMetric[$name] ?? 0 ) + $sPayloadSize;
1191  }
1192 
1193  if ( $rPayloadSize > 0 ) {
1194  $name = "{$prefix}.{$op}_bytes_read";
1195  $deltasByMetric[$name] = ( $deltasByMetric[$name] ?? 0 ) + $rPayloadSize;
1196  }
1197  }
1198 
1199  foreach ( $deltasByMetric as $name => $delta ) {
1200  $this->stats->updateCount( $name, $delta );
1201  }
1202  }
1203 }
MediumSpecificBagOStuff\mergeViaCas
mergeViaCas( $key, callable $callback, $exptime, $attempts, $flags)
Definition: MediumSpecificBagOStuff.php:282
MediumSpecificBagOStuff\doCas
doCas( $casToken, $key, $value, $exptime=0, $flags=0)
Check and set an item.
Definition: MediumSpecificBagOStuff.php:370
MediumSpecificBagOStuff\isInteger
isInteger( $value)
Check if a value is an integer.
Definition: MediumSpecificBagOStuff.php:925
MediumSpecificBagOStuff\$reportDupes
bool $reportDupes
Definition: MediumSpecificBagOStuff.php:47
MediumSpecificBagOStuff\guessSerialValueSize
guessSerialValueSize( $value, $depth=0, &$loops=0)
Estimate the size of a variable once serialized.
Definition: MediumSpecificBagOStuff.php:1028
MediumSpecificBagOStuff\debug
debug( $text)
Definition: MediumSpecificBagOStuff.php:1152
MediumSpecificBagOStuff\incrWithInit
incrWithInit( $key, $exptime, $value=1, $init=null, $flags=0)
Increase the value of the given key (no TTL change) if it exists or create it otherwise.
Definition: MediumSpecificBagOStuff.php:753
MediumSpecificBagOStuff\doAdd
doAdd( $key, $value, $exptime=0, $flags=0)
Insert an item if it does not already exist.
MediumSpecificBagOStuff\cas
cas( $casToken, $key, $value, $exptime=0, $flags=0)
Check and set an item.
Definition: MediumSpecificBagOStuff.php:345
MediumSpecificBagOStuff\LOCK_RCLASS
const LOCK_RCLASS
Definition: MediumSpecificBagOStuff.php:72
MediumSpecificBagOStuff\$locks
array< string, array > $locks
Map of (key => (class, depth, expiry)
Definition: MediumSpecificBagOStuff.php:36
MediumSpecificBagOStuff\doLock
doLock( $key, $timeout, $exptime)
Definition: MediumSpecificBagOStuff.php:501
$serialized
foreach( $res as $row) $serialized
Definition: testCompression.php:88
MediumSpecificBagOStuff\trackDuplicateKeys
trackDuplicateKeys( $key)
Track the number of times that a given key has been used.
Definition: MediumSpecificBagOStuff.php:129
BagOStuff
Class representing a cache/ephemeral data store.
Definition: BagOStuff.php:86
MediumSpecificBagOStuff\$duplicateKeyLookups
array $duplicateKeyLookups
Definition: MediumSpecificBagOStuff.php:45
MediumSpecificBagOStuff\unlock
unlock( $key)
Release an advisory lock on a key string.
Definition: MediumSpecificBagOStuff.php:543
$success
$success
Definition: NoLocalSettings.php:42
MediumSpecificBagOStuff\serialize
serialize( $value)
Definition: MediumSpecificBagOStuff.php:1136
$res
$res
Definition: testCompression.php:57
serialize
serialize()
Definition: ApiMessageTrait.php:138
MediumSpecificBagOStuff\METRIC_OP_ADD
const METRIC_OP_ADD
Definition: MediumSpecificBagOStuff.php:67
MediumSpecificBagOStuff\getMulti
getMulti(array $keys, $flags=0)
Get an associative array containing the item for each of the keys that have items.
Definition: MediumSpecificBagOStuff.php:623
MediumSpecificBagOStuff\convertGenericKey
convertGenericKey( $key)
Convert a "generic" reversible cache key into one for this cache.
Definition: MediumSpecificBagOStuff.php:945
MediumSpecificBagOStuff\changeTTL
changeTTL( $key, $exptime=0, $flags=0)
Change the expiration on a key if it exists.
Definition: MediumSpecificBagOStuff.php:422
MediumSpecificBagOStuff\LOCK_EXPIRY
const LOCK_EXPIRY
Definition: MediumSpecificBagOStuff.php:75
MediumSpecificBagOStuff\checkValueSerializability
checkValueSerializability( $value, $key)
Log if a new cache value does not appear suitable for serialization at a quick glance.
Definition: MediumSpecificBagOStuff.php:1092
MediumSpecificBagOStuff\lock
lock( $key, $timeout=6, $exptime=6, $rclass='')
Definition: MediumSpecificBagOStuff.php:465
MediumSpecificBagOStuff\$syncTimeout
int $syncTimeout
Seconds.
Definition: MediumSpecificBagOStuff.php:38
SerializedValueContainer\isSegmented
static isSegmented( $value)
Definition: SerializedValueContainer.php:50
MediumSpecificBagOStuff\$segmentedValueMaxSize
int $segmentedValueMaxSize
Bytes; maximum total size of a segmented cache value.
Definition: MediumSpecificBagOStuff.php:42
MediumSpecificBagOStuff\makeKey
makeKey( $collection,... $components)
Make a cache key for the global keyspace and given components.
Definition: MediumSpecificBagOStuff.php:941
MediumSpecificBagOStuff\makeLockKey
makeLockKey( $key)
Definition: MediumSpecificBagOStuff.php:604
MediumSpecificBagOStuff\resolveSegments
resolveSegments( $key, $mainValue)
Get and reassemble the chunks of blob at the given key.
Definition: MediumSpecificBagOStuff.php:776
MediumSpecificBagOStuff\doChangeTTLMulti
doChangeTTLMulti(array $keys, $exptime, $flags=0)
Definition: MediumSpecificBagOStuff.php:744
MediumSpecificBagOStuff\getExpirationAsTimestamp
getExpirationAsTimestamp( $exptime)
Convert an optionally relative timestamp to an absolute time.
Definition: MediumSpecificBagOStuff.php:885
MediumSpecificBagOStuff\changeTTLMulti
changeTTLMulti(array $keys, $exptime, $flags=0)
Change the expiration of multiple keys that exist.
Definition: MediumSpecificBagOStuff.php:734
MediumSpecificBagOStuff\doSetMulti
doSetMulti(array $data, $exptime=0, $flags=0)
Definition: MediumSpecificBagOStuff.php:684
$blob
$blob
Definition: testCompression.php:70
MediumSpecificBagOStuff\doSet
doSet( $key, $value, $exptime=0, $flags=0)
Set an item.
MediumSpecificBagOStuff\deleteMulti
deleteMulti(array $keys, $flags=0)
Batch deletion.
Definition: MediumSpecificBagOStuff.php:703
MediumSpecificBagOStuff\addBusyCallback
addBusyCallback(callable $workCallback)
Let a callback be run to avoid wasting time on special blocking calls.
Definition: MediumSpecificBagOStuff.php:806
MediumSpecificBagOStuff\doGet
doGet( $key, $flags=0, &$casToken=null)
MediumSpecificBagOStuff\getQoS
getQoS( $flag)
Definition: MediumSpecificBagOStuff.php:957
MediumSpecificBagOStuff\getExpirationAsTTL
getExpirationAsTTL( $exptime)
Convert an optionally absolute expiry time to a relative time.
Definition: MediumSpecificBagOStuff.php:909
MediumSpecificBagOStuff\makeGlobalKey
makeGlobalKey( $collection,... $components)
Make a cache key for the default keyspace and given components.
Definition: MediumSpecificBagOStuff.php:937
MediumSpecificBagOStuff
Storage medium specific cache for storing items (e.g.
Definition: MediumSpecificBagOStuff.php:34
MediumSpecificBagOStuff\updateOpStats
updateOpStats(string $op, array $keyInfo)
Definition: MediumSpecificBagOStuff.php:1163
MediumSpecificBagOStuff\METRIC_OP_DECR
const METRIC_OP_DECR
Definition: MediumSpecificBagOStuff.php:69
MediumSpecificBagOStuff\METRIC_OP_CHANGE_TTL
const METRIC_OP_CHANGE_TTL
Definition: MediumSpecificBagOStuff.php:66
MediumSpecificBagOStuff\LOCK_TIME
const LOCK_TIME
Definition: MediumSpecificBagOStuff.php:74
MediumSpecificBagOStuff\LOCK_DEPTH
const LOCK_DEPTH
Definition: MediumSpecificBagOStuff.php:73
MediumSpecificBagOStuff\$dupeTrackScheduled
bool $dupeTrackScheduled
Definition: MediumSpecificBagOStuff.php:49
MediumSpecificBagOStuff\doDelete
doDelete( $key, $flags=0)
Delete an item.
MediumSpecificBagOStuff\makeValueOrSegmentList
makeValueOrSegmentList( $key, $value, $exptime, $flags)
Determine the entry (inline or segment list) to store under a key to save the value.
Definition: MediumSpecificBagOStuff.php:820
MediumSpecificBagOStuff\setMulti
setMulti(array $valueByKey, $exptime=0, $flags=0)
Batch insertion/replace.
Definition: MediumSpecificBagOStuff.php:670
MediumSpecificBagOStuff\METRIC_OP_GET
const METRIC_OP_GET
Definition: MediumSpecificBagOStuff.php:63
MediumSpecificBagOStuff\METRIC_OP_SET
const METRIC_OP_SET
Definition: MediumSpecificBagOStuff.php:64
MediumSpecificBagOStuff\__construct
__construct(array $params=[])
Definition: MediumSpecificBagOStuff.php:94
MediumSpecificBagOStuff\$segmentationSize
int $segmentationSize
Bytes; chunk size of segmented cache values.
Definition: MediumSpecificBagOStuff.php:40
MediumSpecificBagOStuff\doGetMulti
doGetMulti(array $keys, $flags=0)
Get an associative array containing the item for each of the keys that have items.
Definition: MediumSpecificBagOStuff.php:647
SerializedValueContainer\UNIFIED_DATA
const UNIFIED_DATA
Definition: SerializedValueContainer.php:13
MediumSpecificBagOStuff\getSegmentationSize
getSegmentationSize()
Definition: MediumSpecificBagOStuff.php:961
MediumSpecificBagOStuff\doUnlock
doUnlock( $key)
Definition: MediumSpecificBagOStuff.php:575
MediumSpecificBagOStuff\$busyCallbacks
callable[] $busyCallbacks
Definition: MediumSpecificBagOStuff.php:52
MediumSpecificBagOStuff\getSerialized
getSerialized( $value, $key)
Get the serialized form a value, using any applicable prepared value.
Definition: MediumSpecificBagOStuff.php:998
SerializedValueContainer\newUnified
static newUnified( $serialized)
Definition: SerializedValueContainer.php:20
MediumSpecificBagOStuff\METRIC_OP_INCR
const METRIC_OP_INCR
Definition: MediumSpecificBagOStuff.php:68
MediumSpecificBagOStuff\METRIC_OP_DELETE
const METRIC_OP_DELETE
Definition: MediumSpecificBagOStuff.php:65
MediumSpecificBagOStuff\isRelativeExpiration
isRelativeExpiration( $exptime)
Definition: MediumSpecificBagOStuff.php:868
unserialize
unserialize( $serialized)
Definition: ApiMessageTrait.php:146
MediumSpecificBagOStuff\setNewPreparedValues
setNewPreparedValues(array $valueByKey)
Make a "generic" reversible cache key from the given components.
Definition: MediumSpecificBagOStuff.php:969
MediumSpecificBagOStuff\unserialize
unserialize( $value)
Definition: MediumSpecificBagOStuff.php:1145
SerializedValueContainer\SEGMENTED_HASHES
const SEGMENTED_HASHES
Definition: SerializedValueContainer.php:14
MediumSpecificBagOStuff\deleteObjectsExpiringBefore
deleteObjectsExpiringBefore( $timestamp, callable $progress=null, $limit=INF, string $tag=null)
Delete all objects expiring before a certain date.
Definition: MediumSpecificBagOStuff.php:608
MediumSpecificBagOStuff\checkIterableMapSerializability
checkIterableMapSerializability( $value, $key)
Definition: MediumSpecificBagOStuff.php:1112
BagOStuff\fieldHasFlags
fieldHasFlags( $field, $flags)
Definition: BagOStuff.php:618
MediumSpecificBagOStuff\add
add( $key, $value, $exptime=0, $flags=0)
Insert an item if it does not already exist.
Definition: MediumSpecificBagOStuff.php:236
MediumSpecificBagOStuff\$preparedValues
array[] $preparedValues
Map of (key => (PHP variable value, serialized value))
Definition: MediumSpecificBagOStuff.php:55
$keys
$keys
Definition: testCompression.php:72
MediumSpecificBagOStuff\getSegmentedValueMaxSize
getSegmentedValueMaxSize()
Definition: MediumSpecificBagOStuff.php:965
SerializedValueContainer\isUnified
static isUnified( $value)
Definition: SerializedValueContainer.php:42
MediumSpecificBagOStuff\merge
merge( $key, callable $callback, $exptime=0, $attempts=10, $flags=0)
Merge changes into the existing cache value (possibly creating a new one)
Definition: MediumSpecificBagOStuff.php:269
MediumSpecificBagOStuff\METRIC_OP_CAS
const METRIC_OP_CAS
Definition: MediumSpecificBagOStuff.php:70
MediumSpecificBagOStuff\doDeleteMulti
doDeleteMulti(array $keys, $flags=0)
Definition: MediumSpecificBagOStuff.php:716
MediumSpecificBagOStuff\doChangeTTL
doChangeTTL( $key, $exptime, $flags)
Definition: MediumSpecificBagOStuff.php:432
SerializedValueContainer\newSegmented
static newSegmented(array $segmentHashList)
Definition: SerializedValueContainer.php:31