MediaWiki  master
MediumSpecificBagOStuff.php
Go to the documentation of this file.
1 <?php
24 use Wikimedia\WaitConditionLoop;
25 
34 abstract class MediumSpecificBagOStuff extends BagOStuff {
36  protected $locks = [];
40  protected $syncTimeout;
42  protected $segmentationSize;
45 
47  private $duplicateKeyLookups = [];
49  private $reportDupes = false;
51  private $dupeTrackScheduled = false;
52 
54  protected $busyCallbacks = [];
55 
57  protected $preparedValues = [];
58 
60  private const SEGMENT_COMPONENT = 'segment';
61 
63  protected const PASS_BY_REF = -1;
64 
65  protected const METRIC_OP_GET = 'get';
66  protected const METRIC_OP_SET = 'set';
67  protected const METRIC_OP_DELETE = 'delete';
68  protected const METRIC_OP_CHANGE_TTL = 'change_ttl';
69  protected const METRIC_OP_ADD = 'add';
70  protected const METRIC_OP_INCR = 'incr';
71  protected const METRIC_OP_DECR = 'decr';
72  protected const METRIC_OP_CAS = 'cas';
73 
74  protected const LOCK_RCLASS = 0;
75  protected const LOCK_DEPTH = 1;
76  protected const LOCK_TIME = 2;
77  protected const LOCK_EXPIRY = 3;
78 
96  public function __construct( array $params = [] ) {
97  parent::__construct( $params );
98 
99  if ( !empty( $params['reportDupes'] ) && is_callable( $this->asyncHandler ) ) {
100  $this->reportDupes = true;
101  }
102 
103  $this->syncTimeout = $params['syncTimeout'] ?? 3;
104  $this->segmentationSize = $params['segmentationSize'] ?? 8388608; // 8MiB
105  $this->segmentedValueMaxSize = $params['segmentedValueMaxSize'] ?? 67108864; // 64MiB
106  }
107 
121  public function get( $key, $flags = 0 ) {
122  $this->trackDuplicateKeys( $key );
123 
124  return $this->resolveSegments( $key, $this->doGet( $key, $flags ) );
125  }
126 
131  private function trackDuplicateKeys( $key ) {
132  if ( !$this->reportDupes ) {
133  return;
134  }
135 
136  if ( !isset( $this->duplicateKeyLookups[$key] ) ) {
137  // Track that we have seen this key. This N-1 counting style allows
138  // easy filtering with array_filter() later.
139  $this->duplicateKeyLookups[$key] = 0;
140  } else {
141  $this->duplicateKeyLookups[$key] += 1;
142 
143  if ( $this->dupeTrackScheduled === false ) {
144  $this->dupeTrackScheduled = true;
145  // Schedule a callback that logs keys processed more than once by get().
146  call_user_func( $this->asyncHandler, function () {
147  $dups = array_filter( $this->duplicateKeyLookups );
148  foreach ( $dups as $key => $count ) {
149  $this->logger->warning(
150  'Duplicate get(): "{key}" fetched {count} times',
151  // Count is N-1 of the actual lookup count
152  [ 'key' => $key, 'count' => $count + 1, ]
153  );
154  }
155  } );
156  }
157  }
158  }
159 
166  abstract protected function doGet( $key, $flags = 0, &$casToken = null );
167 
177  public function set( $key, $value, $exptime = 0, $flags = 0 ) {
178  list( $entry, $usable ) = $this->makeValueOrSegmentList( $key, $value, $exptime, $flags );
179  // Only when all segments (if any) are stored should the main key be changed
180  return $usable ? $this->doSet( $key, $entry, $exptime, $flags ) : false;
181  }
182 
192  abstract protected function doSet( $key, $value, $exptime = 0, $flags = 0 );
193 
205  public function delete( $key, $flags = 0 ) {
206  if ( !$this->fieldHasFlags( $flags, self::WRITE_PRUNE_SEGMENTS ) ) {
207  return $this->doDelete( $key, $flags );
208  }
209 
210  $mainValue = $this->doGet( $key, self::READ_LATEST );
211  if ( !$this->doDelete( $key, $flags ) ) {
212  return false;
213  }
214 
215  if ( !SerializedValueContainer::isSegmented( $mainValue ) ) {
216  return true; // no segments to delete
217  }
218 
219  $orderedKeys = array_map(
220  function ( $segmentHash ) use ( $key ) {
221  return $this->makeGlobalKey( self::SEGMENT_COMPONENT, $key, $segmentHash );
222  },
224  );
225 
226  return $this->deleteMulti( $orderedKeys, $flags & ~self::WRITE_PRUNE_SEGMENTS );
227  }
228 
236  abstract protected function doDelete( $key, $flags = 0 );
237 
238  public function add( $key, $value, $exptime = 0, $flags = 0 ) {
239  list( $entry, $usable ) = $this->makeValueOrSegmentList( $key, $value, $exptime, $flags );
240  // Only when all segments (if any) are stored should the main key be changed
241  return $usable ? $this->doAdd( $key, $entry, $exptime, $flags ) : false;
242  }
243 
253  abstract protected function doAdd( $key, $value, $exptime = 0, $flags = 0 );
254 
271  public function merge( $key, callable $callback, $exptime = 0, $attempts = 10, $flags = 0 ) {
272  return $this->mergeViaCas( $key, $callback, $exptime, $attempts, $flags );
273  }
274 
284  final protected function mergeViaCas( $key, callable $callback, $exptime, $attempts, $flags ) {
285  $attemptsLeft = $attempts;
286  do {
287  $token = self::PASS_BY_REF; // passed by reference
288  // Get the old value and CAS token from cache
289  $this->clearLastError();
290  $currentValue = $this->resolveSegments(
291  $key,
292  $this->doGet( $key, $flags, $token )
293  );
294  if ( $this->getLastError() ) {
295  // Don't spam slow retries due to network problems (retry only on races)
296  $this->logger->warning(
297  __METHOD__ . ' failed due to read I/O error on get() for {key}.',
298  [ 'key' => $key ]
299  );
300  $success = false;
301  break;
302  }
303 
304  // Derive the new value from the old value
305  $value = $callback( $this, $key, $currentValue, $exptime );
306  $keyWasNonexistant = ( $currentValue === false );
307  $valueMatchesOldValue = ( $value === $currentValue );
308  unset( $currentValue ); // free RAM in case the value is large
309 
310  $this->clearLastError();
311  if ( $value === false || $exptime < 0 ) {
312  $success = true; // do nothing
313  } elseif ( $valueMatchesOldValue && $attemptsLeft !== $attempts ) {
314  $success = true; // recently set by another thread to the same value
315  } elseif ( $keyWasNonexistant ) {
316  // Try to create the key, failing if it gets created in the meantime
317  $success = $this->add( $key, $value, $exptime, $flags );
318  } else {
319  // Try to update the key, failing if it gets changed in the meantime
320  $success = $this->cas( $token, $key, $value, $exptime, $flags );
321  }
322  if ( $this->getLastError() ) {
323  // Don't spam slow retries due to network problems (retry only on races)
324  $this->logger->warning(
325  __METHOD__ . ' failed due to write I/O error for {key}.',
326  [ 'key' => $key ]
327  );
328  $success = false;
329  break;
330  }
331 
332  } while ( !$success && --$attemptsLeft );
333 
334  return $success;
335  }
336 
347  protected function cas( $casToken, $key, $value, $exptime = 0, $flags = 0 ) {
348  if ( $casToken === null ) {
349  $this->logger->warning(
350  __METHOD__ . ' got empty CAS token for {key}.',
351  [ 'key' => $key ]
352  );
353 
354  return false; // caller may have meant to use add()?
355  }
356 
357  list( $entry, $usable ) = $this->makeValueOrSegmentList( $key, $value, $exptime, $flags );
358  // Only when all segments (if any) are stored should the main key be changed
359  return $usable ? $this->doCas( $casToken, $key, $entry, $exptime, $flags ) : false;
360  }
361 
372  protected function doCas( $casToken, $key, $value, $exptime = 0, $flags = 0 ) {
373  // @TODO: the use of lock() assumes that all other relevant sets() use a lock
374  if ( !$this->lock( $key, 0 ) ) {
375  return false; // non-blocking
376  }
377 
378  $curCasToken = self::PASS_BY_REF; // passed by reference
379  $this->clearLastError();
380  $this->doGet( $key, self::READ_LATEST, $curCasToken );
381  if ( is_object( $curCasToken ) ) {
382  // Using === does not work with objects since it checks for instance identity
383  throw new UnexpectedValueException( "CAS token cannot be an object" );
384  }
385  if ( $this->getLastError() ) {
386  // Fail if the old CAS token could not be read
387  $success = false;
388  $this->logger->warning(
389  __METHOD__ . ' failed due to write I/O error for {key}.',
390  [ 'key' => $key ]
391  );
392  } elseif ( $casToken === $curCasToken ) {
393  $success = $this->doSet( $key, $value, $exptime, $flags );
394  } else {
395  $success = false; // mismatched or failed
396  $this->logger->info(
397  __METHOD__ . ' failed due to race condition for {key}.',
398  [ 'key' => $key ]
399  );
400  }
401 
402  $this->unlock( $key );
403 
404  return $success;
405  }
406 
424  public function changeTTL( $key, $exptime = 0, $flags = 0 ) {
425  return $this->doChangeTTL( $key, $exptime, $flags );
426  }
427 
434  protected function doChangeTTL( $key, $exptime, $flags ) {
435  // @TODO: the use of lock() assumes that all other relevant sets() use a lock
436  if ( !$this->lock( $key, 0 ) ) {
437  return false;
438  }
439 
440  $expiry = $this->getExpirationAsTimestamp( $exptime );
441  $delete = ( $expiry != self::TTL_INDEFINITE && $expiry < $this->getCurrentTime() );
442 
443  // Use doGet() to avoid having to trigger resolveSegments()
444  $blob = $this->doGet( $key, self::READ_LATEST );
445  if ( $blob ) {
446  if ( $delete ) {
447  $ok = $this->doDelete( $key, $flags );
448  } else {
449  $ok = $this->doSet( $key, $blob, $exptime, $flags );
450  }
451  } else {
452  $ok = false;
453  }
454 
455  $this->unlock( $key );
456 
457  return $ok;
458  }
459 
467  public function lock( $key, $timeout = 6, $exptime = 6, $rclass = '' ) {
468  $exptime = min( $exptime ?: INF, self::TTL_DAY );
469 
470  $acquired = false;
471 
472  if ( isset( $this->locks[$key] ) ) {
473  // Already locked; avoid deadlocks and allow lock reentry if specified
474  if ( $rclass != '' && $this->locks[$key][self::LOCK_RCLASS] === $rclass ) {
475  ++$this->locks[$key][self::LOCK_DEPTH];
476  $acquired = true;
477  }
478  } else {
479  // Not already locked; acquire a lock on the backend
480  $lockTsUnix = $this->doLock( $key, $timeout, $exptime );
481  if ( $lockTsUnix !== null ) {
482  $this->locks[$key] = [
483  self::LOCK_RCLASS => $rclass,
484  self::LOCK_DEPTH => 1,
485  self::LOCK_TIME => $lockTsUnix,
486  self::LOCK_EXPIRY => $lockTsUnix + $exptime
487  ];
488  $acquired = true;
489  }
490  }
491 
492  return $acquired;
493  }
494 
503  protected function doLock( $key, $timeout, $exptime ) {
504  $lockTsUnix = null;
505 
506  $fname = __METHOD__;
507  $loop = new WaitConditionLoop(
508  function () use ( $key, $exptime, $fname, &$lockTsUnix ) {
509  $this->clearLastError();
510  if ( $this->add( $this->makeLockKey( $key ), 1, $exptime ) ) {
511  $lockTsUnix = microtime( true );
512 
513  return WaitConditionLoop::CONDITION_REACHED; // locked!
514  } elseif ( $this->getLastError() ) {
515  $this->logger->warning(
516  "$fname failed due to I/O error for {key}.",
517  [ 'key' => $key ]
518  );
519 
520  return WaitConditionLoop::CONDITION_ABORTED; // network partition?
521  }
522 
523  return WaitConditionLoop::CONDITION_CONTINUE;
524  },
525  $timeout
526  );
527  $code = $loop->invoke();
528 
529  if ( $code === $loop::CONDITION_TIMED_OUT ) {
530  $this->logger->warning(
531  "$fname failed due to timeout for {key}.",
532  [ 'key' => $key, 'timeout' => $timeout ]
533  );
534  }
535 
536  return $lockTsUnix;
537  }
538 
545  public function unlock( $key ) {
546  $released = false;
547 
548  if ( isset( $this->locks[$key] ) ) {
549  if ( --$this->locks[$key][self::LOCK_DEPTH] > 0 ) {
550  $released = true;
551  } else {
552  $released = $this->doUnlock( $key );
553  unset( $this->locks[$key] );
554  if ( !$released ) {
555  $this->logger->warning(
556  __METHOD__ . ' failed to release lock for {key}.',
557  [ 'key' => $key ]
558  );
559  }
560  }
561  } else {
562  $this->logger->warning(
563  __METHOD__ . ' no lock to release for {key}.',
564  [ 'key' => $key ]
565  );
566  }
567 
568  return $released;
569  }
570 
577  protected function doUnlock( $key ) {
578  // Estimate the remaining TTL of the lock key
579  $curTTL = $this->locks[$key][self::LOCK_EXPIRY] - $this->getCurrentTime();
580  // Maximum expected one-way-delay for a query to reach the backend
581  $maxOWD = 0.050;
582 
583  $released = false;
584 
585  if ( ( $curTTL - $maxOWD ) > 0 ) {
586  // The lock key is extremely unlikely to expire before a deletion operation
587  // sent from this method arrives on the relevant backend server
588  $released = $this->doDelete( $this->makeLockKey( $key ) );
589  } else {
590  // It is unsafe for this method to delete the lock key due to the risk of it
591  // expiring and being claimed by another thread before the deletion operation
592  // arrives on the backend server
593  $this->logger->warning(
594  "Lock for {key} held too long ({age} sec).",
595  [ 'key' => $key, 'curTTL' => $curTTL ]
596  );
597  }
598 
599  return $released;
600  }
601 
606  protected function makeLockKey( $key ) {
607  return "$key:lock";
608  }
609 
611  $timestamp,
612  callable $progress = null,
613  $limit = INF,
614  string $tag = null
615  ) {
616  return false;
617  }
618 
625  public function getMulti( array $keys, $flags = 0 ) {
626  $foundByKey = $this->doGetMulti( $keys, $flags );
627 
628  $res = [];
629  foreach ( $keys as $key ) {
630  // Resolve one blob at a time (avoids too much I/O at once)
631  if ( array_key_exists( $key, $foundByKey ) ) {
632  // A value should not appear in the key if a segment is missing
633  $value = $this->resolveSegments( $key, $foundByKey[$key] );
634  if ( $value !== false ) {
635  $res[$key] = $value;
636  }
637  }
638  }
639 
640  return $res;
641  }
642 
649  protected function doGetMulti( array $keys, $flags = 0 ) {
650  $res = [];
651  foreach ( $keys as $key ) {
652  $val = $this->doGet( $key, $flags );
653  if ( $val !== false ) {
654  $res[$key] = $val;
655  }
656  }
657 
658  return $res;
659  }
660 
672  public function setMulti( array $valueByKey, $exptime = 0, $flags = 0 ) {
673  if ( $this->fieldHasFlags( $flags, self::WRITE_ALLOW_SEGMENTS ) ) {
674  throw new InvalidArgumentException( __METHOD__ . ' got WRITE_ALLOW_SEGMENTS' );
675  }
676 
677  return $this->doSetMulti( $valueByKey, $exptime, $flags );
678  }
679 
686  protected function doSetMulti( array $data, $exptime = 0, $flags = 0 ) {
687  $res = true;
688  foreach ( $data as $key => $value ) {
689  $res = $this->doSet( $key, $value, $exptime, $flags ) && $res;
690  }
691 
692  return $res;
693  }
694 
705  public function deleteMulti( array $keys, $flags = 0 ) {
706  if ( $this->fieldHasFlags( $flags, self::WRITE_PRUNE_SEGMENTS ) ) {
707  throw new InvalidArgumentException( __METHOD__ . ' got WRITE_PRUNE_SEGMENTS' );
708  }
709 
710  return $this->doDeleteMulti( $keys, $flags );
711  }
712 
718  protected function doDeleteMulti( array $keys, $flags = 0 ) {
719  $res = true;
720  foreach ( $keys as $key ) {
721  $res = $this->doDelete( $key, $flags ) && $res;
722  }
723  return $res;
724  }
725 
736  public function changeTTLMulti( array $keys, $exptime, $flags = 0 ) {
737  return $this->doChangeTTLMulti( $keys, $exptime, $flags );
738  }
739 
746  protected function doChangeTTLMulti( array $keys, $exptime, $flags = 0 ) {
747  $res = true;
748  foreach ( $keys as $key ) {
749  $res = $this->doChangeTTL( $key, $exptime, $flags ) && $res;
750  }
751 
752  return $res;
753  }
754 
755  public function incrWithInit( $key, $exptime, $value = 1, $init = null, $flags = 0 ) {
756  $init = is_int( $init ) ? $init : $value;
757  $this->clearLastError();
758  $newValue = $this->incr( $key, $value, $flags );
759  if ( $newValue === false && !$this->getLastError() ) {
760  // No key set; initialize
761  $newValue = $this->add( $key, (int)$init, $exptime, $flags ) ? $init : false;
762  if ( $newValue === false && !$this->getLastError() ) {
763  // Raced out initializing; increment
764  $newValue = $this->incr( $key, $value, $flags );
765  }
766  }
767 
768  return $newValue;
769  }
770 
778  final protected function resolveSegments( $key, $mainValue ) {
779  if ( SerializedValueContainer::isUnified( $mainValue ) ) {
780  return $this->unserialize( $mainValue->{SerializedValueContainer::UNIFIED_DATA} );
781  }
782 
783  if ( SerializedValueContainer::isSegmented( $mainValue ) ) {
784  $orderedKeys = array_map(
785  function ( $segmentHash ) use ( $key ) {
786  return $this->makeGlobalKey( self::SEGMENT_COMPONENT, $key, $segmentHash );
787  },
789  );
790 
791  $segmentsByKey = $this->doGetMulti( $orderedKeys );
792 
793  $parts = [];
794  foreach ( $orderedKeys as $segmentKey ) {
795  if ( isset( $segmentsByKey[$segmentKey] ) ) {
796  $parts[] = $segmentsByKey[$segmentKey];
797  } else {
798  return false; // missing segment
799  }
800  }
801 
802  return $this->unserialize( implode( '', $parts ) );
803  }
804 
805  return $mainValue;
806  }
807 
813  public function getLastError() {
814  return $this->lastError;
815  }
816 
821  public function clearLastError() {
822  $this->lastError = self::ERR_NONE;
823  }
824 
830  protected function setLastError( $err ) {
831  $this->lastError = $err;
832  }
833 
834  final public function addBusyCallback( callable $workCallback ) {
835  $this->busyCallbacks[] = $workCallback;
836  }
837 
848  final protected function makeValueOrSegmentList( $key, $value, $exptime, $flags ) {
849  $entry = $value;
850  $usable = true;
851 
852  if (
853  $this->fieldHasFlags( $flags, self::WRITE_ALLOW_SEGMENTS ) &&
854  !is_int( $value ) && // avoid breaking incr()/decr()
855  is_finite( $this->segmentationSize )
856  ) {
857  $segmentSize = $this->segmentationSize;
858  $maxTotalSize = $this->segmentedValueMaxSize;
859 
860  $serialized = $this->getSerialized( $value, $key );
861  $size = strlen( $serialized );
862  if ( $size > $maxTotalSize ) {
863  $this->logger->warning(
864  "Value for {key} exceeds $maxTotalSize bytes; cannot segment.",
865  [ 'key' => $key ]
866  );
867  } elseif ( $size <= $segmentSize ) {
868  // The serialized value was already computed, so just use it inline
870  } else {
871  // Split the serialized value into chunks and store them at different keys
872  $chunksByKey = [];
873  $segmentHashes = [];
874  $count = intdiv( $size, $segmentSize ) + ( ( $size % $segmentSize ) ? 1 : 0 );
875  for ( $i = 0; $i < $count; ++$i ) {
876  $segment = substr( $serialized, $i * $segmentSize, $segmentSize );
877  $hash = sha1( $segment );
878  $chunkKey = $this->makeGlobalKey( self::SEGMENT_COMPONENT, $key, $hash );
879  $chunksByKey[$chunkKey] = $segment;
880  $segmentHashes[] = $hash;
881  }
882  $flags &= ~self::WRITE_ALLOW_SEGMENTS; // sanity
883  $usable = $this->setMulti( $chunksByKey, $exptime, $flags );
884  $entry = SerializedValueContainer::newSegmented( $segmentHashes );
885  }
886  }
887 
888  return [ $entry, $usable ];
889  }
890 
896  final protected function isRelativeExpiration( $exptime ) {
897  return ( $exptime !== self::TTL_INDEFINITE && $exptime < ( 10 * self::TTL_YEAR ) );
898  }
899 
913  final protected function getExpirationAsTimestamp( $exptime ) {
914  if ( $exptime == self::TTL_INDEFINITE ) {
915  return $exptime;
916  }
917 
918  return $this->isRelativeExpiration( $exptime )
919  ? intval( $this->getCurrentTime() + $exptime )
920  : $exptime;
921  }
922 
937  final protected function getExpirationAsTTL( $exptime ) {
938  if ( $exptime == self::TTL_INDEFINITE ) {
939  return $exptime;
940  }
941 
942  return $this->isRelativeExpiration( $exptime )
943  ? $exptime
944  : (int)max( $exptime - $this->getCurrentTime(), 1 );
945  }
946 
953  final protected function isInteger( $value ) {
954  if ( is_int( $value ) ) {
955  return true;
956  } elseif ( !is_string( $value ) ) {
957  return false;
958  }
959 
960  $integer = (int)$value;
961 
962  return ( $value === (string)$integer );
963  }
964 
965  public function makeGlobalKey( $collection, ...$components ) {
966  return $this->makeKeyInternal( self::GLOBAL_KEYSPACE, func_get_args() );
967  }
968 
969  public function makeKey( $collection, ...$components ) {
970  return $this->makeKeyInternal( $this->keyspace, func_get_args() );
971  }
972 
973  protected function convertGenericKey( $key ) {
974  $components = $this->componentsFromGenericKey( $key );
975  if ( count( $components ) < 2 ) {
976  // Legacy key not from makeKey()/makeGlobalKey(); keep it as-is
977  return $key;
978  }
979 
980  $keyspace = array_shift( $components );
981 
982  return $this->makeKeyInternal( $keyspace, $components );
983  }
984 
985  public function getQoS( $flag ) {
986  return $this->attrMap[$flag] ?? self::QOS_UNKNOWN;
987  }
988 
989  public function getSegmentationSize() {
990  return $this->segmentationSize;
991  }
992 
993  public function getSegmentedValueMaxSize() {
994  return $this->segmentedValueMaxSize;
995  }
996 
997  public function setNewPreparedValues( array $valueByKey ) {
998  $this->preparedValues = [];
999 
1000  $sizes = [];
1001  foreach ( $valueByKey as $key => $value ) {
1002  if ( $value === false ) {
1003  $sizes[] = null; // not storable, don't bother
1004  continue;
1005  }
1006 
1007  $serialized = $this->serialize( $value );
1008  $sizes[] = ( $serialized !== false ) ? strlen( $serialized ) : null;
1009 
1010  $this->preparedValues[$key] = [ $value, $serialized ];
1011  }
1012 
1013  return $sizes;
1014  }
1015 
1026  protected function getSerialized( $value, $key ) {
1027  // Reuse any available prepared (serialized) value
1028  if ( array_key_exists( $key, $this->preparedValues ) ) {
1029  list( $prepValue, $prepSerialized ) = $this->preparedValues[$key];
1030  // Normally, this comparison should only take a few microseconds to confirm a match.
1031  // Using "===" on variables of different types is always fast. It is also fast for
1032  // variables of matching type int, float, bool, null, and object. Lastly, it is fast
1033  // for comparing arrays/strings if they are copy-on-write references, which should be
1034  // the case at this point, assuming prepareValues() was called correctly.
1035  if ( $prepValue === $value ) {
1036  unset( $this->preparedValues[$key] );
1037 
1038  return $prepSerialized;
1039  }
1040  }
1041 
1042  $this->checkValueSerializability( $value, $key );
1043 
1044  return $this->serialize( $value );
1045  }
1046 
1056  protected function guessSerialValueSize( $value, $depth = 0, &$loops = 0 ) {
1057  // Include serialization format overhead estimates roughly based on serialize(),
1058  // without counting . Also, int/float variables use the largest case
1059  // byte size for numbers of that type; this avoids CPU overhead for large arrays.
1060  switch ( gettype( $value ) ) {
1061  case 'string':
1062  // E.g. "<type><delim1><quote><value><quote><delim2>"
1063  return strlen( $value ) + 5;
1064  case 'integer':
1065  // E.g. "<type><delim1><sign><2^63><delim2>";
1066  // ceil(log10 (2^63)) = 19
1067  return 23;
1068  case 'double':
1069  // E.g. "<type><delim1><sign><2^52><esign><2^10><delim2>"
1070  // ceil(log10 (2^52)) = 16 and ceil(log10 (2^10)) = 4
1071  return 25;
1072  case 'boolean':
1073  // E.g. "true" becomes "1" and "false" is not storable
1074  return $value ? 1 : null;
1075  case 'NULL':
1076  return 1; // "\0"
1077  case 'array':
1078  case 'object':
1079  // Give up and guess if there is too much depth
1080  if ( $depth >= 5 && $loops >= 256 ) {
1081  return 1024;
1082  }
1083 
1084  ++$loops;
1085  // E.g. "<type><delim1><brace><<Kn><Vn> for all n><brace><delim2>"
1086  $size = 5;
1087  // Note that casting to an array includes private object members
1088  foreach ( (array)$value as $k => $v ) {
1089  // Inline the recursive result here for performance
1090  $size += is_string( $k ) ? ( strlen( $k ) + 5 ) : 23;
1091  $size += $this->guessSerialValueSize( $v, $depth + 1, $loops );
1092  }
1093 
1094  return $size;
1095  default:
1096  return null; // invalid
1097  }
1098  }
1099 
1120  private function checkValueSerializability( $value, $key ) {
1121  if ( is_array( $value ) ) {
1122  $this->checkIterableMapSerializability( $value, $key );
1123  } elseif ( is_object( $value ) ) {
1124  // Note that Closure instances count as objects
1125  if ( $value instanceof stdClass ) {
1126  $this->checkIterableMapSerializability( $value, $key );
1127  } elseif ( !( $value instanceof JsonSerializable ) ) {
1128  $this->logger->warning(
1129  "{class} value for '{cachekey}'; serialization is suspect.",
1130  [ 'cachekey' => $key, 'class' => get_class( $value ) ]
1131  );
1132  }
1133  }
1134  }
1135 
1140  private function checkIterableMapSerializability( $value, $key ) {
1141  foreach ( $value as $index => $entry ) {
1142  if ( is_object( $entry ) ) {
1143  // Note that Closure instances count as objects
1144  if (
1145  !( $entry instanceof stdClass ) &&
1146  !( $entry instanceof JsonSerializable )
1147  ) {
1148  $this->logger->warning(
1149  "{class} value for '{cachekey}' at '$index'; serialization is suspect.",
1150  [ 'cachekey' => $key, 'class' => get_class( $entry ) ]
1151  );
1152 
1153  return;
1154  }
1155  }
1156  }
1157  }
1158 
1164  protected function serialize( $value ) {
1165  return is_int( $value ) ? $value : serialize( $value );
1166  }
1167 
1173  protected function unserialize( $value ) {
1174  return $this->isInteger( $value ) ? (int)$value : unserialize( $value );
1175  }
1176 
1180  protected function debug( $text ) {
1181  $this->logger->debug( "{class} debug: $text", [ 'class' => static::class ] );
1182  }
1183 
1191  protected function updateOpStats( string $op, array $keyInfo ) {
1192  $deltasByMetric = [];
1193 
1194  foreach ( $keyInfo as $indexOrKey => $keyOrSizes ) {
1195  if ( is_array( $keyOrSizes ) ) {
1196  $key = $indexOrKey;
1197  list( $sPayloadSize, $rPayloadSize ) = $keyOrSizes;
1198  } else {
1199  $key = $keyOrSizes;
1200  $sPayloadSize = null;
1201  $rPayloadSize = null;
1202  }
1203 
1204  // Metric prefix for the cache wrapper and key collection name
1205  $prefix = $this->determineKeyPrefixForStats( $key );
1206 
1207  if ( $op === self::METRIC_OP_GET ) {
1208  // This operation was either a "hit" or "miss" for this key
1209  $name = "{$prefix}.{$op}_" . ( $rPayloadSize === false ? 'miss_rate' : 'hit_rate' );
1210  } else {
1211  // There is no concept of "hit" or "miss" for this operation
1212  $name = "{$prefix}.{$op}_call_rate";
1213  }
1214  $deltasByMetric[$name] = ( $deltasByMetric[$name] ?? 0 ) + 1;
1215 
1216  if ( $sPayloadSize > 0 ) {
1217  $name = "{$prefix}.{$op}_bytes_sent";
1218  $deltasByMetric[$name] = ( $deltasByMetric[$name] ?? 0 ) + $sPayloadSize;
1219  }
1220 
1221  if ( $rPayloadSize > 0 ) {
1222  $name = "{$prefix}.{$op}_bytes_read";
1223  $deltasByMetric[$name] = ( $deltasByMetric[$name] ?? 0 ) + $rPayloadSize;
1224  }
1225  }
1226 
1227  foreach ( $deltasByMetric as $name => $delta ) {
1228  $this->stats->updateCount( $name, $delta );
1229  }
1230  }
1231 }
MediumSpecificBagOStuff\mergeViaCas
mergeViaCas( $key, callable $callback, $exptime, $attempts, $flags)
Definition: MediumSpecificBagOStuff.php:284
MediumSpecificBagOStuff\setLastError
setLastError( $err)
Set the "last error" registry.
Definition: MediumSpecificBagOStuff.php:830
MediumSpecificBagOStuff\doCas
doCas( $casToken, $key, $value, $exptime=0, $flags=0)
Check and set an item.
Definition: MediumSpecificBagOStuff.php:372
MediumSpecificBagOStuff\isInteger
isInteger( $value)
Check if a value is an integer.
Definition: MediumSpecificBagOStuff.php:953
MediumSpecificBagOStuff\$reportDupes
bool $reportDupes
Definition: MediumSpecificBagOStuff.php:49
MediumSpecificBagOStuff\guessSerialValueSize
guessSerialValueSize( $value, $depth=0, &$loops=0)
Estimate the size of a variable once serialized.
Definition: MediumSpecificBagOStuff.php:1056
MediumSpecificBagOStuff\debug
debug( $text)
Definition: MediumSpecificBagOStuff.php:1180
MediumSpecificBagOStuff\incrWithInit
incrWithInit( $key, $exptime, $value=1, $init=null, $flags=0)
Increase the value of the given key (no TTL change) if it exists or create it otherwise.
Definition: MediumSpecificBagOStuff.php:755
MediumSpecificBagOStuff\doAdd
doAdd( $key, $value, $exptime=0, $flags=0)
Insert an item if it does not already exist.
MediumSpecificBagOStuff\cas
cas( $casToken, $key, $value, $exptime=0, $flags=0)
Check and set an item.
Definition: MediumSpecificBagOStuff.php:347
MediumSpecificBagOStuff\LOCK_RCLASS
const LOCK_RCLASS
Definition: MediumSpecificBagOStuff.php:74
MediumSpecificBagOStuff\$locks
array< string, array > $locks
Map of (key => (class, depth, expiry)
Definition: MediumSpecificBagOStuff.php:36
MediumSpecificBagOStuff\doLock
doLock( $key, $timeout, $exptime)
Definition: MediumSpecificBagOStuff.php:503
$serialized
foreach( $res as $row) $serialized
Definition: testCompression.php:88
MediumSpecificBagOStuff\trackDuplicateKeys
trackDuplicateKeys( $key)
Track the number of times that a given key has been used.
Definition: MediumSpecificBagOStuff.php:131
BagOStuff
Class representing a cache/ephemeral data store.
Definition: BagOStuff.php:86
MediumSpecificBagOStuff\$duplicateKeyLookups
array $duplicateKeyLookups
Definition: MediumSpecificBagOStuff.php:47
MediumSpecificBagOStuff\unlock
unlock( $key)
Release an advisory lock on a key string.
Definition: MediumSpecificBagOStuff.php:545
$success
$success
Definition: NoLocalSettings.php:42
MediumSpecificBagOStuff\serialize
serialize( $value)
Definition: MediumSpecificBagOStuff.php:1164
$res
$res
Definition: testCompression.php:57
serialize
serialize()
Definition: ApiMessageTrait.php:138
MediumSpecificBagOStuff\METRIC_OP_ADD
const METRIC_OP_ADD
Definition: MediumSpecificBagOStuff.php:69
MediumSpecificBagOStuff\getMulti
getMulti(array $keys, $flags=0)
Get an associative array containing the item for each of the keys that have items.
Definition: MediumSpecificBagOStuff.php:625
Wikimedia\LightweightObjectStore\StorageAwareness\ERR_NONE
const ERR_NONE
No storage medium error.
Definition: StorageAwareness.php:34
MediumSpecificBagOStuff\convertGenericKey
convertGenericKey( $key)
Convert a "generic" reversible cache key into one for this cache.
Definition: MediumSpecificBagOStuff.php:973
MediumSpecificBagOStuff\changeTTL
changeTTL( $key, $exptime=0, $flags=0)
Change the expiration on a key if it exists.
Definition: MediumSpecificBagOStuff.php:424
MediumSpecificBagOStuff\LOCK_EXPIRY
const LOCK_EXPIRY
Definition: MediumSpecificBagOStuff.php:77
MediumSpecificBagOStuff\checkValueSerializability
checkValueSerializability( $value, $key)
Log if a new cache value does not appear suitable for serialization at a quick glance.
Definition: MediumSpecificBagOStuff.php:1120
MediumSpecificBagOStuff\lock
lock( $key, $timeout=6, $exptime=6, $rclass='')
Definition: MediumSpecificBagOStuff.php:467
MediumSpecificBagOStuff\$syncTimeout
int $syncTimeout
Seconds.
Definition: MediumSpecificBagOStuff.php:40
SerializedValueContainer\isSegmented
static isSegmented( $value)
Definition: SerializedValueContainer.php:50
MediumSpecificBagOStuff\$segmentedValueMaxSize
int $segmentedValueMaxSize
Bytes; maximum total size of a segmented cache value.
Definition: MediumSpecificBagOStuff.php:44
MediumSpecificBagOStuff\makeKey
makeKey( $collection,... $components)
Make a cache key for the global keyspace and given components.
Definition: MediumSpecificBagOStuff.php:969
MediumSpecificBagOStuff\makeLockKey
makeLockKey( $key)
Definition: MediumSpecificBagOStuff.php:606
MediumSpecificBagOStuff\resolveSegments
resolveSegments( $key, $mainValue)
Get and reassemble the chunks of blob at the given key.
Definition: MediumSpecificBagOStuff.php:778
MediumSpecificBagOStuff\doChangeTTLMulti
doChangeTTLMulti(array $keys, $exptime, $flags=0)
Definition: MediumSpecificBagOStuff.php:746
MediumSpecificBagOStuff\getExpirationAsTimestamp
getExpirationAsTimestamp( $exptime)
Convert an optionally relative timestamp to an absolute time.
Definition: MediumSpecificBagOStuff.php:913
MediumSpecificBagOStuff\changeTTLMulti
changeTTLMulti(array $keys, $exptime, $flags=0)
Change the expiration of multiple keys that exist.
Definition: MediumSpecificBagOStuff.php:736
MediumSpecificBagOStuff\doSetMulti
doSetMulti(array $data, $exptime=0, $flags=0)
Definition: MediumSpecificBagOStuff.php:686
$blob
$blob
Definition: testCompression.php:70
MediumSpecificBagOStuff\doSet
doSet( $key, $value, $exptime=0, $flags=0)
Set an item.
MediumSpecificBagOStuff\deleteMulti
deleteMulti(array $keys, $flags=0)
Batch deletion.
Definition: MediumSpecificBagOStuff.php:705
MediumSpecificBagOStuff\addBusyCallback
addBusyCallback(callable $workCallback)
Let a callback be run to avoid wasting time on special blocking calls.
Definition: MediumSpecificBagOStuff.php:834
MediumSpecificBagOStuff\doGet
doGet( $key, $flags=0, &$casToken=null)
MediumSpecificBagOStuff\getQoS
getQoS( $flag)
Definition: MediumSpecificBagOStuff.php:985
MediumSpecificBagOStuff\getExpirationAsTTL
getExpirationAsTTL( $exptime)
Convert an optionally absolute expiry time to a relative time.
Definition: MediumSpecificBagOStuff.php:937
MediumSpecificBagOStuff\makeGlobalKey
makeGlobalKey( $collection,... $components)
Make a cache key for the default keyspace and given components.
Definition: MediumSpecificBagOStuff.php:965
MediumSpecificBagOStuff
Storage medium specific cache for storing items (e.g.
Definition: MediumSpecificBagOStuff.php:34
MediumSpecificBagOStuff\updateOpStats
updateOpStats(string $op, array $keyInfo)
Definition: MediumSpecificBagOStuff.php:1191
MediumSpecificBagOStuff\METRIC_OP_DECR
const METRIC_OP_DECR
Definition: MediumSpecificBagOStuff.php:71
MediumSpecificBagOStuff\METRIC_OP_CHANGE_TTL
const METRIC_OP_CHANGE_TTL
Definition: MediumSpecificBagOStuff.php:68
MediumSpecificBagOStuff\LOCK_TIME
const LOCK_TIME
Definition: MediumSpecificBagOStuff.php:76
MediumSpecificBagOStuff\LOCK_DEPTH
const LOCK_DEPTH
Definition: MediumSpecificBagOStuff.php:75
MediumSpecificBagOStuff\$dupeTrackScheduled
bool $dupeTrackScheduled
Definition: MediumSpecificBagOStuff.php:51
MediumSpecificBagOStuff\doDelete
doDelete( $key, $flags=0)
Delete an item.
MediumSpecificBagOStuff\makeValueOrSegmentList
makeValueOrSegmentList( $key, $value, $exptime, $flags)
Determine the entry (inline or segment list) to store under a key to save the value.
Definition: MediumSpecificBagOStuff.php:848
MediumSpecificBagOStuff\setMulti
setMulti(array $valueByKey, $exptime=0, $flags=0)
Batch insertion/replace.
Definition: MediumSpecificBagOStuff.php:672
MediumSpecificBagOStuff\METRIC_OP_GET
const METRIC_OP_GET
Definition: MediumSpecificBagOStuff.php:65
MediumSpecificBagOStuff\METRIC_OP_SET
const METRIC_OP_SET
Definition: MediumSpecificBagOStuff.php:66
MediumSpecificBagOStuff\__construct
__construct(array $params=[])
Definition: MediumSpecificBagOStuff.php:96
MediumSpecificBagOStuff\$segmentationSize
int $segmentationSize
Bytes; chunk size of segmented cache values.
Definition: MediumSpecificBagOStuff.php:42
MediumSpecificBagOStuff\doGetMulti
doGetMulti(array $keys, $flags=0)
Get an associative array containing the item for each of the keys that have items.
Definition: MediumSpecificBagOStuff.php:649
SerializedValueContainer\UNIFIED_DATA
const UNIFIED_DATA
Definition: SerializedValueContainer.php:13
MediumSpecificBagOStuff\getSegmentationSize
getSegmentationSize()
Definition: MediumSpecificBagOStuff.php:989
MediumSpecificBagOStuff\doUnlock
doUnlock( $key)
Definition: MediumSpecificBagOStuff.php:577
MediumSpecificBagOStuff\$busyCallbacks
callable[] $busyCallbacks
Definition: MediumSpecificBagOStuff.php:54
MediumSpecificBagOStuff\getSerialized
getSerialized( $value, $key)
Get the serialized form a value, using any applicable prepared value.
Definition: MediumSpecificBagOStuff.php:1026
SerializedValueContainer\newUnified
static newUnified( $serialized)
Definition: SerializedValueContainer.php:20
MediumSpecificBagOStuff\METRIC_OP_INCR
const METRIC_OP_INCR
Definition: MediumSpecificBagOStuff.php:70
MediumSpecificBagOStuff\METRIC_OP_DELETE
const METRIC_OP_DELETE
Definition: MediumSpecificBagOStuff.php:67
MediumSpecificBagOStuff\isRelativeExpiration
isRelativeExpiration( $exptime)
Definition: MediumSpecificBagOStuff.php:896
unserialize
unserialize( $serialized)
Definition: ApiMessageTrait.php:146
MediumSpecificBagOStuff\setNewPreparedValues
setNewPreparedValues(array $valueByKey)
Make a "generic" reversible cache key from the given components.
Definition: MediumSpecificBagOStuff.php:997
MediumSpecificBagOStuff\unserialize
unserialize( $value)
Definition: MediumSpecificBagOStuff.php:1173
SerializedValueContainer\SEGMENTED_HASHES
const SEGMENTED_HASHES
Definition: SerializedValueContainer.php:14
MediumSpecificBagOStuff\deleteObjectsExpiringBefore
deleteObjectsExpiringBefore( $timestamp, callable $progress=null, $limit=INF, string $tag=null)
Delete all objects expiring before a certain date.
Definition: MediumSpecificBagOStuff.php:610
MediumSpecificBagOStuff\checkIterableMapSerializability
checkIterableMapSerializability( $value, $key)
Definition: MediumSpecificBagOStuff.php:1140
BagOStuff\fieldHasFlags
fieldHasFlags( $field, $flags)
Definition: BagOStuff.php:584
MediumSpecificBagOStuff\add
add( $key, $value, $exptime=0, $flags=0)
Insert an item if it does not already exist.
Definition: MediumSpecificBagOStuff.php:238
MediumSpecificBagOStuff\$preparedValues
array[] $preparedValues
Map of (key => (PHP variable value, serialized value))
Definition: MediumSpecificBagOStuff.php:57
$keys
$keys
Definition: testCompression.php:72
MediumSpecificBagOStuff\getSegmentedValueMaxSize
getSegmentedValueMaxSize()
Definition: MediumSpecificBagOStuff.php:993
SerializedValueContainer\isUnified
static isUnified( $value)
Definition: SerializedValueContainer.php:42
MediumSpecificBagOStuff\merge
merge( $key, callable $callback, $exptime=0, $attempts=10, $flags=0)
Merge changes into the existing cache value (possibly creating a new one)
Definition: MediumSpecificBagOStuff.php:271
MediumSpecificBagOStuff\METRIC_OP_CAS
const METRIC_OP_CAS
Definition: MediumSpecificBagOStuff.php:72
MediumSpecificBagOStuff\$lastError
int $lastError
ERR_* class constant.
Definition: MediumSpecificBagOStuff.php:38
MediumSpecificBagOStuff\clearLastError
clearLastError()
Clear the "last error" registry.
Definition: MediumSpecificBagOStuff.php:821
MediumSpecificBagOStuff\doDeleteMulti
doDeleteMulti(array $keys, $flags=0)
Definition: MediumSpecificBagOStuff.php:718
MediumSpecificBagOStuff\doChangeTTL
doChangeTTL( $key, $exptime, $flags)
Definition: MediumSpecificBagOStuff.php:434
SerializedValueContainer\newSegmented
static newSegmented(array $segmentHashList)
Definition: SerializedValueContainer.php:31
MediumSpecificBagOStuff\getLastError
getLastError()
Get the "last error" registered; clearLastError() should be called manually.
Definition: MediumSpecificBagOStuff.php:813