MediaWiki  master
BagOStuff.php
Go to the documentation of this file.
1 <?php
29 use Liuggio\StatsdClient\Factory\StatsdDataFactoryInterface;
30 use Psr\Log\LoggerAwareInterface;
31 use Psr\Log\LoggerInterface;
32 use Psr\Log\NullLogger;
35 use Wikimedia\ScopedCallback;
36 
81 abstract class BagOStuff implements
85  LoggerAwareInterface
86 {
88  protected $stats;
90  protected $logger;
92  protected $asyncHandler;
97  protected $wrapperInfoByPrefix = [];
98 
100  protected $attrMap = [];
101 
103  protected $keyspace;
104 
108  protected $lastErrorId = 0;
109 
111  protected static $nextErrorMonitorId = 1;
112 
115 
117  public const READ_LATEST = 1; // if supported, avoid reading stale data due to replication
118  public const READ_VERIFIED = 2; // promise that the caller handles detection of staleness
120  public const WRITE_SYNC = 4; // if supported, block until the write is fully replicated
121  public const WRITE_CACHE_ONLY = 8; // only change state of the in-memory cache
122  public const WRITE_ALLOW_SEGMENTS = 16; // allow partitioning of the value if it is large
123  public const WRITE_PRUNE_SEGMENTS = 32; // delete all the segments if the value is partitioned
124  public const WRITE_BACKGROUND = 64; // if supported, do not block on completion until the next read
125 
127  protected const GLOBAL_KEYSPACE = 'global';
129  protected const GLOBAL_PREFIX = 'global:';
131  protected const GLOBAL_PREFIX_LEN = 7;
132 
134  protected const ARG0_KEY = 0;
136  protected const ARG0_KEYARR = 1;
138  protected const ARG0_KEYMAP = 2;
140  protected const ARG0_NONKEY = 3;
141 
143  protected const RES_KEYMAP = 0;
145  protected const RES_NONKEY = 1;
146 
148  private const WRAPPER_STATS_GROUP = 0;
150  private const WRAPPER_COLLECTION_CALLBACK = 1;
151 
162  public function __construct( array $params = [] ) {
163  $this->keyspace = $params['keyspace'] ?? 'local';
164  $this->asyncHandler = $params['asyncHandler'] ?? null;
165  $this->stats = $params['stats'] ?? new NullStatsdDataFactory();
166  $this->setLogger( $params['logger'] ?? new NullLogger() );
167  }
168 
173  public function setLogger( LoggerInterface $logger ) {
174  $this->logger = $logger;
175  }
176 
181  public function getLogger(): LoggerInterface {
182  return $this->logger;
183  }
184 
198  final public function getWithSetCallback( $key, $exptime, $callback, $flags = 0 ) {
199  $value = $this->get( $key, $flags );
200 
201  if ( $value === false ) {
202  $value = $callback( $exptime );
203  if ( $value !== false && $exptime >= 0 ) {
204  $this->set( $key, $value, $exptime, $flags );
205  }
206  }
207 
208  return $value;
209  }
210 
224  abstract public function get( $key, $flags = 0 );
225 
235  abstract public function set( $key, $value, $exptime = 0, $flags = 0 );
236 
248  abstract public function delete( $key, $flags = 0 );
249 
259  abstract public function add( $key, $value, $exptime = 0, $flags = 0 );
260 
278  abstract public function merge(
279  $key,
280  callable $callback,
281  $exptime = 0,
282  $attempts = 10,
283  $flags = 0
284  );
285 
303  abstract public function changeTTL( $key, $exptime = 0, $flags = 0 );
304 
317  abstract public function lock( $key, $timeout = 6, $exptime = 6, $rclass = '' );
318 
325  abstract public function unlock( $key );
326 
343  final public function getScopedLock( $key, $timeout = 6, $expiry = 30, $rclass = '' ) {
344  $expiry = min( $expiry ?: INF, self::TTL_DAY );
345 
346  if ( !$this->lock( $key, $timeout, $expiry, $rclass ) ) {
347  return null;
348  }
349 
350  return new ScopedCallback( function () use ( $key, $expiry ) {
351  $this->unlock( $key );
352  } );
353  }
354 
367  abstract public function deleteObjectsExpiringBefore(
368  $timestamp,
369  callable $progress = null,
370  $limit = INF,
371  string $tag = null
372  );
373 
381  abstract public function getMulti( array $keys, $flags = 0 );
382 
396  abstract public function setMulti( array $valueByKey, $exptime = 0, $flags = 0 );
397 
410  abstract public function deleteMulti( array $keys, $flags = 0 );
411 
423  abstract public function changeTTLMulti( array $keys, $exptime, $flags = 0 );
424 
433  abstract public function incr( $key, $value = 1, $flags = 0 );
434 
443  abstract public function decr( $key, $value = 1, $flags = 0 );
444 
460  abstract public function incrWithInit( $key, $exptime, $value = 1, $init = null, $flags = 0 );
461 
468  public function watchErrors() {
469  return self::$nextErrorMonitorId++;
470  }
471 
490  public function getLastError( $watchPoint = 0 ) {
491  return ( $this->lastErrorId > $watchPoint ) ? $this->lastError : self::ERR_NONE;
492  }
493 
500  public function clearLastError() {
501  $this->lastError = self::ERR_NONE;
502  }
503 
510  protected function setLastError( $error ) {
511  $this->lastError = $error;
512  $this->lastErrorId = self::$nextErrorMonitorId++;
513  }
514 
535  abstract public function addBusyCallback( callable $workCallback );
536 
550  abstract public function makeKeyInternal( $keyspace, $components );
551 
562  abstract public function makeGlobalKey( $collection, ...$components );
563 
574  abstract public function makeKey( $collection, ...$components );
575 
583  public function isKeyGlobal( $key ) {
584  return ( strncmp( $key, self::GLOBAL_PREFIX, self::GLOBAL_PREFIX_LEN ) === 0 );
585  }
586 
592  public function getQoS( $flag ) {
593  return $this->attrMap[$flag] ?? self::QOS_UNKNOWN;
594  }
595 
600  public function getSegmentationSize() {
601  return INF;
602  }
603 
608  public function getSegmentedValueMaxSize() {
609  return INF;
610  }
611 
618  final protected function fieldHasFlags( $field, $flags ) {
619  return ( ( $field & $flags ) === $flags );
620  }
621 
628  final protected function mergeFlagMaps( array $bags ) {
629  $map = [];
630  foreach ( $bags as $bag ) {
631  foreach ( $bag->attrMap as $attr => $rank ) {
632  if ( isset( $map[$attr] ) ) {
633  $map[$attr] = min( $map[$attr], $rank );
634  } else {
635  $map[$attr] = $rank;
636  }
637  }
638  }
639 
640  return $map;
641  }
642 
668  abstract public function setNewPreparedValues( array $valueByKey );
669 
693  string $prefixComponent,
694  string $statsGroup,
695  callable $collectionCallback
696  ) {
697  $this->wrapperInfoByPrefix[$prefixComponent] = [
698  self::WRAPPER_STATS_GROUP => $statsGroup,
699  self::WRAPPER_COLLECTION_CALLBACK => $collectionCallback
700  ];
701  }
702 
710  final protected function genericKeyFromComponents( ...$components ) {
711  if ( count( $components ) < 2 ) {
712  throw new InvalidArgumentException( "Missing keyspace or collection name" );
713  }
714 
715  $key = '';
716  foreach ( $components as $i => $component ) {
717  if ( $i > 0 ) {
718  $key .= ':';
719  }
720  // Escape delimiter (":") and escape ("%") characters
721  $key .= strtr( $component, [ '%' => '%25', ':' => '%3A' ] );
722  }
723 
724  return $key;
725  }
726 
736  final protected function componentsFromGenericKey( $key ) {
737  // Note that the order of each corresponding search/replace pair matters
738  return str_replace( [ '%3A', '%25' ], [ ':', '%' ], explode( ':', $key ) );
739  }
740 
749  abstract protected function convertGenericKey( $key );
750 
761  protected function proxyCall(
762  string $method,
763  int $arg0Sig,
764  int $resSig,
765  array $genericArgs,
766  BagOStuff $wrapper
767  ) {
768  // Get the corresponding store-specific cache keys...
769  $storeArgs = $genericArgs;
770  switch ( $arg0Sig ) {
771  case self::ARG0_KEY:
772  $storeArgs[0] = $this->convertGenericKey( $genericArgs[0] );
773  break;
774  case self::ARG0_KEYARR:
775  foreach ( $genericArgs[0] as $i => $genericKey ) {
776  $storeArgs[0][$i] = $this->convertGenericKey( $genericKey );
777  }
778  break;
779  case self::ARG0_KEYMAP:
780  $storeArgs[0] = [];
781  foreach ( $genericArgs[0] as $genericKey => $v ) {
782  $storeArgs[0][$this->convertGenericKey( $genericKey )] = $v;
783  }
784  break;
785  }
786 
787  // Result of invoking the method with the corresponding store-specific cache keys
788  $watchPoint = $this->watchErrors();
789  $storeRes = $this->$method( ...$storeArgs );
790  $lastError = $this->getLastError( $watchPoint );
791  if ( $lastError !== self::ERR_NONE ) {
792  $wrapper->setLastError( $lastError );
793  }
794 
795  // Convert any store-specific cache keys in the result back to generic cache keys
796  if ( $resSig === self::RES_KEYMAP ) {
797  // Map of (store-specific cache key => generic cache key)
798  $genericKeyByStoreKey = array_combine( $storeArgs[0], $genericArgs[0] );
799 
800  $genericRes = [];
801  foreach ( $storeRes as $storeKey => $value ) {
802  $genericRes[$genericKeyByStoreKey[$storeKey]] = $value;
803  }
804  } else {
805  $genericRes = $storeRes;
806  }
807 
808  return $genericRes;
809  }
810 
815  protected function determineKeyPrefixForStats( $key ) {
816  $firstComponent = substr( $key, 0, strcspn( $key, ':' ) );
817 
818  $wrapperInfo = $this->wrapperInfoByPrefix[$firstComponent] ?? null;
819  if ( $wrapperInfo ) {
820  // Key has the prefix of a cache wrapper class that wraps BagOStuff
821  $collection = $wrapperInfo[self::WRAPPER_COLLECTION_CALLBACK]( $key );
822  $statsGroup = $wrapperInfo[self::WRAPPER_STATS_GROUP];
823  } else {
824  // Key came directly from BagOStuff::makeKey() or BagOStuff::makeGlobalKey()
825  // and thus has the format of "<scope>:<collection>[:<constant or variable>]..."
826  $components = explode( ':', $key, 3 );
827  // Handle legacy callers that fail to use the key building methods
828  $collection = $components[1] ?? $components[0];
829  $statsGroup = 'objectcache';
830  }
831 
832  // Replace dots because they are special in StatsD (T232907)
833  return $statsGroup . '.' . strtr( $collection, '.', '_' );
834  }
835 
841  public function getCurrentTime() {
842  return $this->wallClockOverride ?: microtime( true );
843  }
844 
850  public function setMockTime( &$time ) {
851  $this->wallClockOverride =& $time;
852  }
853 }
BagOStuff\getSegmentedValueMaxSize
getSegmentedValueMaxSize()
Definition: BagOStuff.php:608
BagOStuff\determineKeyPrefixForStats
determineKeyPrefixForStats( $key)
Definition: BagOStuff.php:815
BagOStuff\setNewPreparedValues
setNewPreparedValues(array $valueByKey)
Make a "generic" reversible cache key from the given components.
BagOStuff\registerWrapperInfoForStats
registerWrapperInfoForStats(string $prefixComponent, string $statsGroup, callable $collectionCallback)
Register info about a caching layer class that uses BagOStuff as a backing store.
Definition: BagOStuff.php:692
BagOStuff\$stats
StatsdDataFactoryInterface $stats
Definition: BagOStuff.php:88
BagOStuff\setLastError
setLastError( $error)
Set the "last error" registry due to a problem encountered during an attempted operation.
Definition: BagOStuff.php:510
BagOStuff\getQoS
getQoS( $flag)
Definition: BagOStuff.php:592
BagOStuff\add
add( $key, $value, $exptime=0, $flags=0)
Insert an item if it does not already exist.
BagOStuff\WRITE_SYNC
const WRITE_SYNC
Bitfield constants for set()/merge(); these are only advisory.
Definition: BagOStuff.php:120
BagOStuff\watchErrors
watchErrors()
Get a "watch point" token that can be used to get the "last error" to occur after now.
Definition: BagOStuff.php:468
BagOStuff\WRITE_ALLOW_SEGMENTS
const WRITE_ALLOW_SEGMENTS
Definition: BagOStuff.php:122
Wikimedia\LightweightObjectStore\ExpirationAwareness
Generic interface providing Time-To-Live constants for expirable object storage.
Definition: ExpirationAwareness.php:32
BagOStuff\getLastError
getLastError( $watchPoint=0)
Get the "last error" registry.
Definition: BagOStuff.php:490
BagOStuff\$lastErrorId
int $lastErrorId
Error event sequence number of the last error that occurred.
Definition: BagOStuff.php:108
NullStatsdDataFactory
Definition: NullStatsdDataFactory.php:10
BagOStuff
Class representing a cache/ephemeral data store.
Definition: BagOStuff.php:86
BagOStuff\genericKeyFromComponents
genericKeyFromComponents(... $components)
At a minimum, there must be a keyspace and collection name component.
Definition: BagOStuff.php:710
BagOStuff\changeTTL
changeTTL( $key, $exptime=0, $flags=0)
Change the expiration on a key if it exists.
BagOStuff\componentsFromGenericKey
componentsFromGenericKey( $key)
Extract the components from a "generic" reversible cache key.
Definition: BagOStuff.php:736
BagOStuff\$wrapperInfoByPrefix
array< string, array > $wrapperInfoByPrefix
Cache key processing callbacks and info for metrics.
Definition: BagOStuff.php:97
BagOStuff\convertGenericKey
convertGenericKey( $key)
Convert a "generic" reversible cache key into one for this cache.
BagOStuff\$logger
LoggerInterface $logger
Definition: BagOStuff.php:90
Wikimedia\LightweightObjectStore\StorageAwareness\ERR_NONE
const ERR_NONE
No storage medium error.
Definition: StorageAwareness.php:34
BagOStuff\changeTTLMulti
changeTTLMulti(array $keys, $exptime, $flags=0)
Change the expiration of multiple keys that exist.
BagOStuff\setMockTime
setMockTime(&$time)
Definition: BagOStuff.php:850
BagOStuff\isKeyGlobal
isKeyGlobal( $key)
Check whether a cache key is in the global keyspace.
Definition: BagOStuff.php:583
BagOStuff\unlock
unlock( $key)
Release an advisory lock on a key string.
BagOStuff\clearLastError
clearLastError()
Clear the "last error" registry.
Definition: BagOStuff.php:500
BagOStuff\decr
decr( $key, $value=1, $flags=0)
Decrease stored value of $key by $value while preserving its TTL.
BagOStuff\$lastError
int $lastError
BagOStuff:ERR_* constant of the last error that occurred.
Definition: BagOStuff.php:106
BagOStuff\incr
incr( $key, $value=1, $flags=0)
Increase stored value of $key by $value while preserving its TTL.
BagOStuff\setLogger
setLogger(LoggerInterface $logger)
Definition: BagOStuff.php:173
BagOStuff\$asyncHandler
callable null $asyncHandler
Definition: BagOStuff.php:92
BagOStuff\WRITE_BACKGROUND
const WRITE_BACKGROUND
Definition: BagOStuff.php:124
BagOStuff\READ_LATEST
const READ_LATEST
Bitfield constants for get()/getMulti(); these are only advisory.
Definition: BagOStuff.php:117
BagOStuff\addBusyCallback
addBusyCallback(callable $workCallback)
Let a callback be run to avoid wasting time on special blocking calls.
BagOStuff\$wallClockOverride
float null $wallClockOverride
Definition: BagOStuff.php:114
BagOStuff\WRITE_PRUNE_SEGMENTS
const WRITE_PRUNE_SEGMENTS
Definition: BagOStuff.php:123
Wikimedia\LightweightObjectStore\StorageAwareness
Generic interface providing error code and quality-of-service constants for object stores.
Definition: StorageAwareness.php:32
BagOStuff\getWithSetCallback
getWithSetCallback( $key, $exptime, $callback, $flags=0)
Get an item with the given key, regenerating and setting it if not found.
Definition: BagOStuff.php:198
BagOStuff\WRAPPER_STATS_GROUP
const WRAPPER_STATS_GROUP
Key to the metric group to use for the relevant cache wrapper.
Definition: BagOStuff.php:148
BagOStuff\mergeFlagMaps
mergeFlagMaps(array $bags)
Merge the flag maps of one or more BagOStuff objects into a "lowest common denominator" map.
Definition: BagOStuff.php:628
BagOStuff\__construct
__construct(array $params=[])
Parameters include:
Definition: BagOStuff.php:162
BagOStuff\getLogger
getLogger()
Definition: BagOStuff.php:181
BagOStuff\$nextErrorMonitorId
static int $nextErrorMonitorId
Next sequence number to use for watch/error events.
Definition: BagOStuff.php:111
BagOStuff\makeKey
makeKey( $collection,... $components)
Make a cache key for the global keyspace and given components.
BagOStuff\merge
merge( $key, callable $callback, $exptime=0, $attempts=10, $flags=0)
Merge changes into the existing cache value (possibly creating a new one)
BagOStuff\deleteMulti
deleteMulti(array $keys, $flags=0)
Batch deletion.
BagOStuff\incrWithInit
incrWithInit( $key, $exptime, $value=1, $init=null, $flags=0)
Increase the value of the given key (no TTL change) if it exists or create it otherwise.
BagOStuff\makeKeyInternal
makeKeyInternal( $keyspace, $components)
Make a cache key for the given keyspace and components.
BagOStuff\setMulti
setMulti(array $valueByKey, $exptime=0, $flags=0)
Batch insertion/replace.
BagOStuff\$attrMap
int[] $attrMap
Map of (BagOStuff:ATTR_* constant => BagOStuff:QOS_* constant)
Definition: BagOStuff.php:100
BagOStuff\READ_VERIFIED
const READ_VERIFIED
Definition: BagOStuff.php:118
BagOStuff\lock
lock( $key, $timeout=6, $exptime=6, $rclass='')
Acquire an advisory lock on a key string, exclusive to the caller.
BagOStuff\fieldHasFlags
fieldHasFlags( $field, $flags)
Definition: BagOStuff.php:618
$keys
$keys
Definition: testCompression.php:72
BagOStuff\getCurrentTime
getCurrentTime()
Definition: BagOStuff.php:841
BagOStuff\$keyspace
string $keyspace
Default keyspace; used by makeKey()
Definition: BagOStuff.php:103
IStoreKeyEncoder
Generic interface for object stores with key encoding methods.
Definition: IStoreKeyEncoder.php:9
Wikimedia\LightweightObjectStore\StorageAwareness\QOS_UNKNOWN
const QOS_UNKNOWN
Generic "unknown" value; useful for comparisons (always "good enough")
Definition: StorageAwareness.php:62
BagOStuff\proxyCall
proxyCall(string $method, int $arg0Sig, int $resSig, array $genericArgs, BagOStuff $wrapper)
Call a method on behalf of wrapper BagOStuff instance that uses "generic" keys.
Definition: BagOStuff.php:761
BagOStuff\getScopedLock
getScopedLock( $key, $timeout=6, $expiry=30, $rclass='')
Get a lightweight exclusive self-unlocking lock.
Definition: BagOStuff.php:343
BagOStuff\getSegmentationSize
getSegmentationSize()
Definition: BagOStuff.php:600
BagOStuff\makeGlobalKey
makeGlobalKey( $collection,... $components)
Make a cache key for the default keyspace and given components.
BagOStuff\deleteObjectsExpiringBefore
deleteObjectsExpiringBefore( $timestamp, callable $progress=null, $limit=INF, string $tag=null)
Delete all objects expiring before a certain date.
BagOStuff\getMulti
getMulti(array $keys, $flags=0)
Get an associative array containing the item for each of the keys that have items.
BagOStuff\WRAPPER_COLLECTION_CALLBACK
const WRAPPER_COLLECTION_CALLBACK
Key to the callback that extracts collection names from cache wrapper keys.
Definition: BagOStuff.php:150
BagOStuff\WRITE_CACHE_ONLY
const WRITE_CACHE_ONLY
Definition: BagOStuff.php:121