MediaWiki  master
WANObjectCache.php
Go to the documentation of this file.
1 <?php
22 use Liuggio\StatsdClient\Factory\StatsdDataFactoryInterface;
23 use Psr\Log\LoggerAwareInterface;
24 use Psr\Log\LoggerInterface;
25 use Psr\Log\NullLogger;
28 
123 class WANObjectCache implements
127  LoggerAwareInterface
128 {
130  protected $cache;
132  protected $processCaches = [];
134  protected $logger;
136  protected $stats;
138  protected $asyncHandler;
139 
147  protected $broadcastRoute;
149  protected $onHostRoute;
151  protected $useInterimHoldOffCaching = true;
153  protected $epoch;
155  protected $secret;
157  protected $coalesceScheme;
158 
160  private $keyHighQps;
163 
165  private $callbackDepth = 0;
167  private $warmupCache = [];
169  private $warmupKeyMisses = 0;
170 
173 
175  private const MAX_COMMIT_DELAY = 3;
177  private const MAX_READ_LAG = 7;
179  public const HOLDOFF_TTL = self::MAX_COMMIT_DELAY + self::MAX_READ_LAG + 1;
180 
182  private const LOW_TTL = 30;
184  public const TTL_LAGGED = 30;
185 
187  private const HOT_TTR = 900;
189  private const AGE_NEW = 60;
190 
192  private const TSE_NONE = -1;
193 
195  public const STALE_TTL_NONE = 0;
197  public const GRACE_TTL_NONE = 0;
199  public const HOLDOFF_TTL_NONE = 0;
200 
202  public const MIN_TIMESTAMP_NONE = 0.0;
203 
205  private const PC_PRIMARY = 'primary:1000';
206 
208  public const PASS_BY_REF = [];
209 
211  private const SCHEME_HASH_TAG = 1;
213  private const SCHEME_HASH_STOP = 2;
214 
216  private const CHECK_KEY_TTL = self::TTL_YEAR;
218  private const INTERIM_KEY_TTL = 1;
219 
221  private const LOCK_TTL = 10;
223  private const COOLOFF_TTL = 1;
225  private const RAMPUP_TTL = 30;
226 
228  private const TINY_NEGATIVE = -0.000001;
230  private const TINY_POSTIVE = 0.000001;
231 
233  private const RECENT_SET_LOW_MS = 50;
235  private const RECENT_SET_HIGH_MS = 100;
236 
238  private const GENERATION_SLOW_SEC = 3;
239 
241  private const PURGE_TIME = 0;
243  private const PURGE_HOLDOFF = 1;
244 
246  private const VERSION = 1;
247 
249  private const RES_VALUE = 0;
251  private const RES_METADATA = 1;
252 
254  public const KEY_VERSION = 'version';
256  public const KEY_AS_OF = 'asOf';
258  public const KEY_TTL = 'ttl';
260  public const KEY_CUR_TTL = 'curTTL';
262  public const KEY_TOMB_AS_OF = 'tombAsOf';
264  public const KEY_CHECK_AS_OF = 'lastCKPurge';
265 
267  private const FLD_FORMAT_VERSION = 0;
269  private const FLD_VALUE = 1;
271  private const FLD_TTL = 2;
273  private const FLD_TIME = 3;
275  private const FLD_FLAGS = 4;
277  private const FLD_VALUE_VERSION = 5;
279  private const FLD_GENERATION_TIME = 6;
280 
282  private const TYPE_VALUE = 'v';
284  private const TYPE_TIMESTAMP = 't';
286  private const TYPE_FLUX = 'f';
288  private const TYPE_MUTEX = 'm';
290  private const TYPE_INTERIM = 'i';
292  private const TYPE_COOLOFF = 'c';
293 
295  private const PURGE_VAL_PREFIX = 'PURGED:';
296 
340  public function __construct( array $params ) {
341  $this->cache = $params['cache'];
342  $this->broadcastRoute = $params['broadcastRoutingPrefix'] ?? null;
343  $this->onHostRoute = $params['onHostRoutingPrefix'] ?? null;
344  $this->epoch = $params['epoch'] ?? 0;
345  $this->secret = $params['secret'] ?? (string)$this->epoch;
346  if ( ( $params['coalesceScheme'] ?? '' ) === 'hash_tag' ) {
347  // https://redis.io/topics/cluster-spec
348  // https://github.com/twitter/twemproxy/blob/v0.4.1/notes/recommendation.md#hash-tags
349  // https://github.com/Netflix/dynomite/blob/v0.7.0/notes/recommendation.md#hash-tags
350  $this->coalesceScheme = self::SCHEME_HASH_TAG;
351  } else {
352  // https://github.com/facebook/mcrouter/wiki/Key-syntax
353  $this->coalesceScheme = self::SCHEME_HASH_STOP;
354  }
355 
356  $this->keyHighQps = $params['keyHighQps'] ?? 100;
357  $this->keyHighUplinkBps = $params['keyHighUplinkBps'] ?? ( 1e9 / 8 / 100 );
358 
359  $this->setLogger( $params['logger'] ?? new NullLogger() );
360  $this->stats = $params['stats'] ?? new NullStatsdDataFactory();
361  $this->asyncHandler = $params['asyncHandler'] ?? null;
362 
363  $this->cache->registerWrapperInfoForStats(
364  'WANCache',
365  'wanobjectcache',
366  [ __CLASS__, 'getCollectionFromSisterKey' ]
367  );
368  }
369 
373  public function setLogger( LoggerInterface $logger ) {
374  $this->logger = $logger;
375  }
376 
382  public static function newEmpty() {
383  return new static( [ 'cache' => new EmptyBagOStuff() ] );
384  }
385 
441  final public function get( $key, &$curTTL = null, array $checkKeys = [], &$info = [] ) {
442  // Note that an undeclared variable passed as $info starts as null (not the default).
443  // Also, if no $info parameter is provided, then it doesn't matter how it changes here.
444  $legacyInfo = ( $info !== self::PASS_BY_REF );
445 
446  $res = $this->fetchKeys( [ $key ], $checkKeys )[$key];
447  $value = $res[self::RES_VALUE];
448  $metadata = $res[self::RES_METADATA];
449 
450  $curTTL = $metadata[self::KEY_CUR_TTL];
451  $info = $legacyInfo ? $metadata[self::KEY_AS_OF] : $metadata;
452 
453  return $value;
454  }
455 
480  final public function getMulti(
481  array $keys,
482  &$curTTLs = [],
483  array $checkKeys = [],
484  &$info = []
485  ) {
486  // Note that an undeclared variable passed as $info starts as null (not the default).
487  // Also, if no $info parameter is provided, then it doesn't matter how it changes here.
488  $legacyInfo = ( $info !== self::PASS_BY_REF );
489 
490  $curTTLs = [];
491  $info = [];
492  $valuesByKey = [];
493 
494  $resByKey = $this->fetchKeys( $keys, $checkKeys );
495  foreach ( $resByKey as $key => $res ) {
496  $value = $res[self::RES_VALUE];
497  $metadata = $res[self::RES_METADATA];
498 
499  if ( $value !== false ) {
500  $valuesByKey[$key] = $value;
501  }
502 
503  if ( $metadata[self::KEY_CUR_TTL] !== null ) {
504  $curTTLs[$key] = $metadata[self::KEY_CUR_TTL];
505  }
506 
507  $info[$key] = $legacyInfo ? $metadata[self::KEY_AS_OF] : $metadata;
508  }
509 
510  return $valuesByKey;
511  }
512 
530  protected function fetchKeys( array $keys, array $checkKeys ) {
531  $resByKey = [];
532 
533  // List of all sister keys that need to be fetched from cache
534  $allSisterKeys = [];
535  // Order-corresponding value sister key list for the base key list ($keys)
536  $valueSisterKeys = [];
537  // Order-corresponding "flux" sister key list for the base key list ($keys) or []
538  $fluxSisterKeys = [];
539  // List of "check" sister keys to compare all value sister keys against
540  $checkSisterKeysForAll = [];
541  // Map of (base key => additional "check" sister key(s) to compare against)
542  $checkSisterKeysByKey = [];
543 
544  foreach ( $keys as $key ) {
545  $sisterKey = $this->makeSisterKey( $key, self::TYPE_VALUE, $this->onHostRoute );
546  $allSisterKeys[] = $sisterKey;
547  $valueSisterKeys[] = $sisterKey;
548  if ( $this->onHostRoute !== null ) {
549  $sisterKey = $this->makeSisterKey( $key, self::TYPE_FLUX );
550  $allSisterKeys[] = $sisterKey;
551  $fluxSisterKeys[] = $sisterKey;
552  }
553  }
554 
555  foreach ( $checkKeys as $i => $checkKeyOrKeyGroup ) {
556  // Note: avoid array_merge() inside loop in case there are many keys
557  if ( is_int( $i ) ) {
558  // Single "check" key that applies to all base keys
559  $sisterKey = $this->makeSisterKey( $checkKeyOrKeyGroup, self::TYPE_TIMESTAMP );
560  $allSisterKeys[] = $sisterKey;
561  $checkSisterKeysForAll[] = $sisterKey;
562  } else {
563  // List of "check" keys that apply to a specific base key
564  foreach ( (array)$checkKeyOrKeyGroup as $checkKey ) {
565  $sisterKey = $this->makeSisterKey( $checkKey, self::TYPE_TIMESTAMP );
566  $allSisterKeys[] = $sisterKey;
567  $checkSisterKeysByKey[$i][] = $sisterKey;
568  }
569  }
570  }
571 
572  if ( $this->warmupCache ) {
573  // Get the wrapped values of the sister keys from the warmup cache
574  $wrappedBySisterKey = $this->warmupCache;
575  $sisterKeysMissing = array_diff( $allSisterKeys, array_keys( $wrappedBySisterKey ) );
576  if ( $sisterKeysMissing ) { // sanity
577  $this->warmupKeyMisses += count( $sisterKeysMissing );
578  $wrappedBySisterKey += $this->cache->getMulti( $sisterKeysMissing );
579  }
580  } else {
581  // Fetch the wrapped values of the sister keys from the backend
582  $wrappedBySisterKey = $this->cache->getMulti( $allSisterKeys );
583  }
584 
585  // Pessimistically treat the "current time" as the time when any network I/O finished
586  $now = $this->getCurrentTime();
587 
588  // List of "check" sister key purge timestamps to compare all value sister keys against
589  $ckPurgesForAll = $this->processCheckKeys(
590  $checkSisterKeysForAll,
591  $wrappedBySisterKey,
592  $now
593  );
594  // Map of (base key => extra "check" sister key purge timestamp(s) to compare against)
595  $ckPurgesByKey = [];
596  foreach ( $checkSisterKeysByKey as $keyWithCheckKeys => $checkKeysForKey ) {
597  $ckPurgesByKey[$keyWithCheckKeys] = $this->processCheckKeys(
598  $checkKeysForKey,
599  $wrappedBySisterKey,
600  $now
601  );
602  }
603 
604  // Map of (base key => "flux" key purge timestamp to compare against)
605  $fkPurgesByKey = $this->processFluxKeys( $keys, $fluxSisterKeys, $wrappedBySisterKey );
606 
607  // Unwrap and validate any value found for each base key (under the value sister key)
608  reset( $keys );
609  foreach ( $valueSisterKeys as $valueSisterKey ) {
610  // Get the corresponding base key for this value sister key
611  $key = current( $keys );
612  next( $keys );
613 
614  if ( isset( $fkPurgesByKey[$key] ) ) {
615  // An on-host tier is in use and a "flux" sister key exists for this
616  // Treat the value sister key as if it was a tombstone with this value.
617  $wrapped = $fkPurgesByKey[$key];
618  } elseif ( array_key_exists( $valueSisterKey, $wrappedBySisterKey ) ) {
619  // Key exists as either a live value or tombstone value
620  $wrapped = $wrappedBySisterKey[$valueSisterKey];
621  } else {
622  // Key does not exist
623  $wrapped = false;
624  }
625 
626  list( $value, $metadata ) = $this->unwrap( $wrapped, $now );
627  // Include the timestamp of the newest "check" key purge/initialization
628  $metadata[self::KEY_CHECK_AS_OF] = null;
629 
630  foreach ( array_merge( $ckPurgesForAll, $ckPurgesByKey[$key] ?? [] ) as $ckPurge ) {
631  $metadata[self::KEY_CHECK_AS_OF] = max(
632  $ckPurge[self::PURGE_TIME],
633  $metadata[self::KEY_CHECK_AS_OF]
634  );
635  // Timestamp marking the end of the hold-off period for this purge
636  $holdoffDeadline = $ckPurge[self::PURGE_TIME] + $ckPurge[self::PURGE_HOLDOFF];
637  // Check if the value was generated during the hold-off period
638  if ( $value !== false && $holdoffDeadline >= $metadata[self::KEY_AS_OF] ) {
639  // How long ago this value was invalidated by *this* "check" key
640  $ago = min( $ckPurge[self::PURGE_TIME] - $now, self::TINY_NEGATIVE );
641  // How long ago this value was invalidated by *any* known "check" key
642  $metadata[self::KEY_CUR_TTL] = min( $metadata[self::KEY_CUR_TTL], $ago );
643  }
644  }
645 
646  $resByKey[$key] = [ self::RES_VALUE => $value, self::RES_METADATA => $metadata ];
647  }
648 
649  return $resByKey;
650  }
651 
658  private function processFluxKeys(
659  array $keys,
660  array $fluxSisterKeys,
661  array $wrappedBySisterKey
662  ) {
663  $purges = [];
664 
665  reset( $keys );
666  foreach ( $fluxSisterKeys as $fluxKey ) {
667  // Get the corresponding base key for this "flux" key
668  $key = current( $keys );
669  next( $keys );
670 
671  $purge = isset( $wrappedBySisterKey[$fluxKey] )
672  ? $this->parsePurgeValue( $wrappedBySisterKey[$fluxKey] )
673  : false;
674 
675  if ( $purge !== false ) {
676  $purges[$key] = $purge;
677  }
678  }
679 
680  return $purges;
681  }
682 
689  private function processCheckKeys(
690  array $checkSisterKeys,
691  array $wrappedBySisterKey,
692  float $now
693  ) {
694  $purges = [];
695 
696  foreach ( $checkSisterKeys as $timeKey ) {
697  $purge = isset( $wrappedBySisterKey[$timeKey] )
698  ? $this->parsePurgeValue( $wrappedBySisterKey[$timeKey] )
699  : false;
700 
701  if ( $purge === false ) {
702  // Key is not set or malformed; regenerate
703  $newVal = $this->makeCheckPurgeValue( $now, self::HOLDOFF_TTL );
704  $this->cache->add( $timeKey, $newVal, self::CHECK_KEY_TTL );
705  $purge = $this->parsePurgeValue( $newVal );
706  }
707 
708  $purges[] = $purge;
709  }
710 
711  return $purges;
712  }
713 
792  final public function set( $key, $value, $ttl = self::TTL_INDEFINITE, array $opts = [] ) {
793  $now = $this->getCurrentTime();
794  $lag = $opts['lag'] ?? 0;
795  $age = isset( $opts['since'] ) ? max( 0, $now - $opts['since'] ) : 0;
796  $pending = $opts['pending'] ?? false;
797  $lockTSE = $opts['lockTSE'] ?? self::TSE_NONE;
798  $staleTTL = $opts['staleTTL'] ?? self::STALE_TTL_NONE;
799  $creating = $opts['creating'] ?? false;
800  $version = $opts['version'] ?? null;
801  $walltime = $opts['walltime'] ?? null;
802 
803  if ( $ttl < 0 ) {
804  return true; // not cacheable
805  }
806 
807  // Do not cache potentially uncommitted data as it might get rolled back
808  if ( $pending ) {
809  $this->logger->info(
810  'Rejected set() for {cachekey} due to pending writes.',
811  [ 'cachekey' => $key ]
812  );
813 
814  return true; // no-op the write for being unsafe
815  }
816 
817  // Check if there is a risk of caching (stale) data that predates the last delete()
818  // tombstone due to the tombstone having expired. If so, then the behavior should depend
819  // on whether the problem is specific to this regeneration attempt or systemically affects
820  // attempts to regenerate this key. For systemic cases, the cache writes should set a low
821  // TTL so that the value at least remains cacheable. For non-systemic cases, the cache
822  // write can simply be rejected.
823  if ( $age > self::MAX_READ_LAG ) {
824  // Case A: high snapshot lag
825  if ( $walltime === null ) {
826  // Case A0: high snapshot lag without regeneration wall time info.
827  // Probably systemic; use a low TTL to avoid stampedes/uncacheability.
828  $mitigated = 'snapshot lag';
829  $mitigationTTL = self::TTL_SECOND;
830  } elseif ( ( $age - $walltime ) > self::MAX_READ_LAG ) {
831  // Case A1: value regeneration during an already long-running transaction.
832  // Probably non-systemic; rely on a less problematic regeneration attempt.
833  $mitigated = 'snapshot lag (late regeneration)';
834  $mitigationTTL = self::TTL_UNCACHEABLE;
835  } else {
836  // Case A2: value regeneration takes a long time.
837  // Probably systemic; use a low TTL to avoid stampedes/uncacheability.
838  $mitigated = 'snapshot lag (high regeneration time)';
839  $mitigationTTL = self::TTL_SECOND;
840  }
841  } elseif ( $lag === false || $lag > self::MAX_READ_LAG ) {
842  // Case B: high replication lag without high snapshot lag
843  // Probably systemic; use a low TTL to avoid stampedes/uncacheability
844  $mitigated = 'replication lag';
845  $mitigationTTL = self::TTL_LAGGED;
846  } elseif ( ( $lag + $age ) > self::MAX_READ_LAG ) {
847  // Case C: medium length request with medium replication lag
848  // Probably non-systemic; rely on a less problematic regeneration attempt
849  $mitigated = 'read lag';
850  $mitigationTTL = self::TTL_UNCACHEABLE;
851  } else {
852  // New value generated with recent enough data
853  $mitigated = null;
854  $mitigationTTL = null;
855  }
856 
857  if ( $mitigationTTL === self::TTL_UNCACHEABLE ) {
858  $this->logger->warning(
859  "Rejected set() for {cachekey} due to $mitigated.",
860  [ 'cachekey' => $key, 'lag' => $lag, 'age' => $age, 'walltime' => $walltime ]
861  );
862 
863  return true; // no-op the write for being unsafe
864  }
865 
866  // TTL to use in staleness checks (does not effect persistence layer TTL)
867  $logicalTTL = null;
868 
869  if ( $mitigationTTL !== null ) {
870  // New value generated from data that is old enough to be risky
871  if ( $lockTSE >= 0 ) {
872  // Value will have the normal expiry but will be seen as stale sooner
873  $logicalTTL = min( $ttl ?: INF, $mitigationTTL );
874  } else {
875  // Value expires sooner (leaving enough TTL for preemptive refresh)
876  $ttl = min( $ttl ?: INF, max( $mitigationTTL, self::LOW_TTL ) );
877  }
878 
879  $this->logger->warning(
880  "Lowered set() TTL for {cachekey} due to $mitigated.",
881  [ 'cachekey' => $key, 'lag' => $lag, 'age' => $age, 'walltime' => $walltime ]
882  );
883  }
884 
885  // Wrap that value with time/TTL/version metadata
886  $wrapped = $this->wrap( $value, $logicalTTL ?: $ttl, $version, $now, $walltime );
887  $storeTTL = $ttl + $staleTTL;
888 
889  if ( $creating ) {
890  $ok = $this->cache->add(
891  $this->makeSisterKey( $key, self::TYPE_VALUE ),
892  $wrapped,
893  $storeTTL
894  );
895  } else {
896  $ok = $this->cache->merge(
897  $this->makeSisterKey( $key, self::TYPE_VALUE ),
898  static function ( $cache, $key, $cWrapped ) use ( $wrapped ) {
899  // A string value means that it is a tombstone; do nothing in that case
900  return ( is_string( $cWrapped ) ) ? false : $wrapped;
901  },
902  $storeTTL,
903  1 // 1 attempt
904  );
905  }
906 
907  return $ok;
908  }
909 
972  final public function delete( $key, $ttl = self::HOLDOFF_TTL ) {
973  // Purge values must be stored under the value key so that WANObjectCache::set()
974  // can atomically merge values without accidentally undoing a recent purge and thus
975  // violating the holdoff TTL restriction.
976  $valueSisterKey = $this->makeSisterKey( $key, self::TYPE_VALUE );
977 
978  // When an on-host tier is configured, fetchKeys() relies on "flux" keys to determine
979  // whether a value from the on-host tier is still valid. A "flux" key is a short-lived
980  // key that contains the last recent purge due to delete(). This approach avoids having
981  // to purge the on-host cache service on potentially hundreds of application servers.
982 
983  if ( $ttl <= 0 ) {
984  // A client or cache cleanup script is requesting a cache purge, so there is no
985  // volatility period due to replica DB lag. Any recent change to an entity cached
986  // in this key should have triggered an appropriate purge event.
987 
988  // Remove the key from all datacenters, ignoring any on-host tier. Since on-host
989  // tier caches only use low key TTLs, setting "flux" keys here has little practical
990  // benefit; missed purges should be rare and the on-host tier will quickly correct
991  // itself in those rare cases.
992  $ok = $this->relayNonVolatilePurge( $valueSisterKey );
993  } else {
994  // A cacheable entity recently changed, so there might be a volatility period due
995  // to replica DB lag. Clients usually expect their actions to be reflected in any
996  // of their subsequent web request. This is attainable if (a) purge relay lag is
997  // lower than the time it takes for subsequent request by the client to arrive,
998  // and, (b) DB replica queries have "read-your-writes" consistency due to DB lag
999  // mitigation systems.
1000 
1001  $now = $this->getCurrentTime();
1002  // Set the key to the purge value in all datacenters
1003  $purgeBySisterKey = [ $valueSisterKey => $this->makeTombstonePurgeValue( $now ) ];
1004  // When an on-host tier is configured, invalidate it by setting "flux" keys
1005  if ( $this->onHostRoute !== null ) {
1006  $fluxSisterKey = $this->makeSisterKey( $key, self::TYPE_FLUX );
1007  $purgeBySisterKey[$fluxSisterKey] = $this->makeTombstonePurgeValue( $now );
1008  }
1009 
1010  $ok = $this->relayVolatilePurges( $purgeBySisterKey, $ttl );
1011  }
1012 
1013  $kClass = $this->determineKeyClassForStats( $key );
1014  $this->stats->increment( "wanobjectcache.$kClass.delete." . ( $ok ? 'ok' : 'error' ) );
1015 
1016  return $ok;
1017  }
1018 
1038  final public function getCheckKeyTime( $key ) {
1039  return $this->getMultiCheckKeyTime( [ $key ] )[$key];
1040  }
1041 
1103  final public function getMultiCheckKeyTime( array $keys ) {
1104  $checkSisterKeysByKey = [];
1105  foreach ( $keys as $key ) {
1106  $checkSisterKeysByKey[$key] = $this->makeSisterKey( $key, self::TYPE_TIMESTAMP );
1107  }
1108 
1109  $wrappedBySisterKey = $this->cache->getMulti( $checkSisterKeysByKey );
1110  $wrappedBySisterKey += array_fill_keys( $checkSisterKeysByKey, false );
1111 
1112  $times = [];
1113  foreach ( $checkSisterKeysByKey as $key => $checkSisterKey ) {
1114  $purge = $this->parsePurgeValue( $wrappedBySisterKey[$checkSisterKey] );
1115  if ( $purge !== false ) {
1116  $time = $purge[self::PURGE_TIME];
1117  } else {
1118  // Casting assures identical floats for the next getCheckKeyTime() calls
1119  $now = (string)$this->getCurrentTime();
1120  $this->cache->add(
1121  $checkSisterKey,
1122  $this->makeCheckPurgeValue( $now, self::HOLDOFF_TTL ),
1124  );
1125  $time = (float)$now;
1126  }
1127 
1128  $times[$key] = $time;
1129  }
1130 
1131  return $times;
1132  }
1133 
1168  final public function touchCheckKey( $key, $holdoff = self::HOLDOFF_TTL ) {
1169  $checkSisterKey = $this->makeSisterKey( $key, self::TYPE_TIMESTAMP );
1170  $ok = $this->relayVolatilePurges(
1171  [ $checkSisterKey => $this->makeCheckPurgeValue( $this->getCurrentTime(), $holdoff ) ],
1172  self::CHECK_KEY_TTL
1173  );
1174 
1175  $kClass = $this->determineKeyClassForStats( $key );
1176  $this->stats->increment( "wanobjectcache.$kClass.ck_touch." . ( $ok ? 'ok' : 'error' ) );
1177 
1178  return $ok;
1179  }
1180 
1208  final public function resetCheckKey( $key ) {
1209  $checkSisterKey = $this->makeSisterKey( $key, self::TYPE_TIMESTAMP );
1210  $ok = $this->relayNonVolatilePurge( $checkSisterKey );
1211 
1212  $kClass = $this->determineKeyClassForStats( $key );
1213  $this->stats->increment( "wanobjectcache.$kClass.ck_reset." . ( $ok ? 'ok' : 'error' ) );
1214 
1215  return $ok;
1216  }
1217 
1521  final public function getWithSetCallback(
1522  $key, $ttl, $callback, array $opts = [], array $cbParams = []
1523  ) {
1524  $version = $opts['version'] ?? null;
1525  $pcTTL = $opts['pcTTL'] ?? self::TTL_UNCACHEABLE;
1526  $pCache = ( $pcTTL >= 0 )
1527  ? $this->getProcessCache( $opts['pcGroup'] ?? self::PC_PRIMARY )
1528  : null;
1529 
1530  // Use the process cache if requested as long as no outer cache callback is running.
1531  // Nested callback process cache use is not lag-safe with regard to HOLDOFF_TTL since
1532  // process cached values are more lagged than persistent ones as they are not purged.
1533  if ( $pCache && $this->callbackDepth == 0 ) {
1534  $cached = $pCache->get( $this->getProcessCacheKey( $key, $version ), $pcTTL, false );
1535  if ( $cached !== false ) {
1536  $this->logger->debug( "getWithSetCallback($key): process cache hit" );
1537  return $cached;
1538  }
1539  }
1540 
1541  $res = $this->fetchOrRegenerate( $key, $ttl, $callback, $opts, $cbParams );
1542  list( $value, $valueVersion, $curAsOf ) = $res;
1543  if ( $valueVersion !== $version ) {
1544  // Current value has a different version; use the variant key for this version.
1545  // Regenerate the variant value if it is not newer than the main value at $key
1546  // so that purges to the main key propagate to the variant value.
1547  $this->logger->debug( "getWithSetCallback($key): using variant key" );
1548  list( $value ) = $this->fetchOrRegenerate(
1549  $this->makeGlobalKey( 'WANCache-key-variant', md5( $key ), $version ),
1550  $ttl,
1551  $callback,
1552  [ 'version' => null, 'minAsOf' => $curAsOf ] + $opts,
1553  $cbParams
1554  );
1555  }
1556 
1557  // Update the process cache if enabled
1558  if ( $pCache && $value !== false ) {
1559  $pCache->set( $this->getProcessCacheKey( $key, $version ), $value );
1560  }
1561 
1562  return $value;
1563  }
1564 
1581  private function fetchOrRegenerate( $key, $ttl, $callback, array $opts, array $cbParams ) {
1582  $checkKeys = $opts['checkKeys'] ?? [];
1583  $graceTTL = $opts['graceTTL'] ?? self::GRACE_TTL_NONE;
1584  $minAsOf = $opts['minAsOf'] ?? self::MIN_TIMESTAMP_NONE;
1585  $hotTTR = $opts['hotTTR'] ?? self::HOT_TTR;
1586  $lowTTL = $opts['lowTTL'] ?? min( self::LOW_TTL, $ttl );
1587  $ageNew = $opts['ageNew'] ?? self::AGE_NEW;
1588  $touchedCb = $opts['touchedCallback'] ?? null;
1589  $initialTime = $this->getCurrentTime();
1590 
1591  $kClass = $this->determineKeyClassForStats( $key );
1592 
1593  // Get the current key value and its metadata
1594  $res = $this->fetchKeys( [ $key ], $checkKeys )[$key];
1595  $curValue = $res[self::RES_VALUE];
1596  $curInfo = $res[self::RES_METADATA];
1597  $curTTL = $curInfo[self::KEY_CUR_TTL];
1598  // Apply any $touchedCb invalidation timestamp to get the "last purge timestamp"
1599  list( $curTTL, $LPT ) = $this->resolveCTL( $curValue, $curTTL, $curInfo, $touchedCb );
1600  // Use the cached value if it exists and is not due for synchronous regeneration
1601  if (
1602  $this->isValid( $curValue, $curInfo[self::KEY_AS_OF], $minAsOf ) &&
1603  $this->isAliveOrInGracePeriod( $curTTL, $graceTTL )
1604  ) {
1605  $preemptiveRefresh = (
1606  $this->worthRefreshExpiring( $curTTL, $curInfo[self::KEY_TTL], $lowTTL ) ||
1607  $this->worthRefreshPopular( $curInfo['asOf'], $ageNew, $hotTTR, $initialTime )
1608  );
1609  if ( !$preemptiveRefresh ) {
1610  $this->stats->timing(
1611  "wanobjectcache.$kClass.hit.good",
1612  1e3 * ( $this->getCurrentTime() - $initialTime )
1613  );
1614 
1615  return [ $curValue, $curInfo[self::KEY_VERSION], $curInfo[self::KEY_AS_OF] ];
1616  } elseif ( $this->scheduleAsyncRefresh( $key, $ttl, $callback, $opts, $cbParams ) ) {
1617  $this->logger->debug( "fetchOrRegenerate($key): hit with async refresh" );
1618  $this->stats->timing(
1619  "wanobjectcache.$kClass.hit.refresh",
1620  1e3 * ( $this->getCurrentTime() - $initialTime )
1621  );
1622 
1623  return [ $curValue, $curInfo[self::KEY_VERSION], $curInfo[self::KEY_AS_OF] ];
1624  } else {
1625  $this->logger->debug( "fetchOrRegenerate($key): hit with sync refresh" );
1626  }
1627  }
1628 
1629  // Determine if there is stale or volatile cached value that is still usable
1630  $isKeyTombstoned = ( $curInfo[self::KEY_TOMB_AS_OF] !== null );
1631  if ( $isKeyTombstoned ) {
1632  // Key is write-holed; use the (volatile) interim key as an alternative
1633  list( $possValue, $possInfo ) = $this->getInterimValue( $key, $minAsOf );
1634  // Update the "last purge time" since the $touchedCb timestamp depends on $value
1635  $LPT = $this->resolveTouched( $possValue, $LPT, $touchedCb );
1636  } else {
1637  $possValue = $curValue;
1638  $possInfo = $curInfo;
1639  }
1640 
1641  // Avoid overhead from callback runs, regeneration locks, and cache sets during
1642  // hold-off periods for the key by reusing very recently generated cached values
1643  if (
1644  $this->isValid( $possValue, $possInfo[self::KEY_AS_OF], $minAsOf, $LPT ) &&
1645  $this->isVolatileValueAgeNegligible( $initialTime - $possInfo[self::KEY_AS_OF] )
1646  ) {
1647  $this->logger->debug( "fetchOrRegenerate($key): volatile hit" );
1648  $this->stats->timing(
1649  "wanobjectcache.$kClass.hit.volatile",
1650  1e3 * ( $this->getCurrentTime() - $initialTime )
1651  );
1652 
1653  return [ $possValue, $possInfo[self::KEY_VERSION], $curInfo[self::KEY_AS_OF] ];
1654  }
1655 
1656  $lockTSE = $opts['lockTSE'] ?? self::TSE_NONE;
1657  $busyValue = $opts['busyValue'] ?? null;
1658  $staleTTL = $opts['staleTTL'] ?? self::STALE_TTL_NONE;
1659  $version = $opts['version'] ?? null;
1660 
1661  // Determine whether one thread per datacenter should handle regeneration at a time
1662  $useRegenerationLock =
1663  // Note that since tombstones no-op set(), $lockTSE and $curTTL cannot be used to
1664  // deduce the key hotness because |$curTTL| will always keep increasing until the
1665  // tombstone expires or is overwritten by a new tombstone. Also, even if $lockTSE
1666  // is not set, constant regeneration of a key for the tombstone lifetime might be
1667  // very expensive. Assume tombstoned keys are possibly hot in order to reduce
1668  // the risk of high regeneration load after the delete() method is called.
1669  $isKeyTombstoned ||
1670  // Assume a key is hot if requested soon ($lockTSE seconds) after invalidation.
1671  // This avoids stampedes when timestamps from $checkKeys/$touchedCb bump.
1672  ( $curTTL !== null && $curTTL <= 0 && abs( $curTTL ) <= $lockTSE ) ||
1673  // Assume a key is hot if there is no value and a busy fallback is given.
1674  // This avoids stampedes on eviction or preemptive regeneration taking too long.
1675  ( $busyValue !== null && $possValue === false );
1676 
1677  // If a regeneration lock is required, threads that do not get the lock will try to use
1678  // the stale value, the interim value, or the $busyValue placeholder, in that order. If
1679  // none of those are set then all threads will bypass the lock and regenerate the value.
1680  $hasLock = $useRegenerationLock && $this->claimStampedeLock( $key );
1681  if ( $useRegenerationLock && !$hasLock ) {
1682  if ( $this->isValid( $possValue, $possInfo[self::KEY_AS_OF], $minAsOf ) ) {
1683  $this->logger->debug( "fetchOrRegenerate($key): returning stale value" );
1684  $this->stats->timing(
1685  "wanobjectcache.$kClass.hit.stale",
1686  1e3 * ( $this->getCurrentTime() - $initialTime )
1687  );
1688 
1689  return [ $possValue, $possInfo[self::KEY_VERSION], $curInfo[self::KEY_AS_OF] ];
1690  } elseif ( $busyValue !== null ) {
1691  $miss = is_infinite( $minAsOf ) ? 'renew' : 'miss';
1692  $this->logger->debug( "fetchOrRegenerate($key): busy $miss" );
1693  $this->stats->timing(
1694  "wanobjectcache.$kClass.$miss.busy",
1695  1e3 * ( $this->getCurrentTime() - $initialTime )
1696  );
1697  $placeholderValue = $this->resolveBusyValue( $busyValue );
1698 
1699  return [ $placeholderValue, $version, $curInfo[self::KEY_AS_OF] ];
1700  }
1701  }
1702 
1703  // Generate the new value given any prior value with a matching version
1704  $setOpts = [];
1705  $preCallbackTime = $this->getCurrentTime();
1707  try {
1708  $value = $callback(
1709  ( $curInfo[self::KEY_VERSION] === $version ) ? $curValue : false,
1710  $ttl,
1711  $setOpts,
1712  ( $curInfo[self::KEY_VERSION] === $version ) ? $curInfo[self::KEY_AS_OF] : null,
1713  $cbParams
1714  );
1715  } finally {
1717  }
1718  $postCallbackTime = $this->getCurrentTime();
1719 
1720  // How long it took to fetch, validate, and generate the value
1721  $elapsed = max( $postCallbackTime - $initialTime, 0.0 );
1722 
1723  // How long it took to generate the value
1724  $walltime = max( $postCallbackTime - $preCallbackTime, 0.0 );
1725  $this->stats->timing( "wanobjectcache.$kClass.regen_walltime", 1e3 * $walltime );
1726 
1727  // Attempt to save the newly generated value if applicable
1728  if (
1729  // Callback yielded a cacheable value
1730  ( $value !== false && $ttl >= 0 ) &&
1731  // Current thread was not raced out of a regeneration lock or key is tombstoned
1732  ( !$useRegenerationLock || $hasLock || $isKeyTombstoned ) &&
1733  // Key does not appear to be undergoing a set() stampede
1734  $this->checkAndSetCooloff( $key, $kClass, $value, $elapsed, $hasLock )
1735  ) {
1736  // If the key is write-holed then use the (volatile) interim key as an alternative
1737  if ( $isKeyTombstoned ) {
1738  $this->setInterimValue( $key, $value, $lockTSE, $version, $walltime );
1739  } else {
1740  $finalSetOpts = [
1741  // @phan-suppress-next-line PhanUselessBinaryAddRight
1742  'since' => $setOpts['since'] ?? $preCallbackTime,
1743  'version' => $version,
1744  'staleTTL' => $staleTTL,
1745  'lockTSE' => $lockTSE, // informs lag vs performance trade-offs
1746  'creating' => ( $curValue === false ), // optimization
1747  'walltime' => $walltime
1748  ] + $setOpts;
1749  $this->set( $key, $value, $ttl, $finalSetOpts );
1750  }
1751  }
1752 
1753  $this->yieldStampedeLock( $key, $hasLock );
1754 
1755  $miss = is_infinite( $minAsOf ) ? 'renew' : 'miss';
1756  $this->logger->debug( "fetchOrRegenerate($key): $miss, new value computed" );
1757  $this->stats->timing(
1758  "wanobjectcache.$kClass.$miss.compute",
1759  1e3 * ( $this->getCurrentTime() - $initialTime )
1760  );
1761 
1762  return [ $value, $version, $curInfo[self::KEY_AS_OF] ];
1763  }
1764 
1769  private function claimStampedeLock( $key ) {
1770  $checkSisterKey = $this->makeSisterKey( $key, self::TYPE_MUTEX );
1771  // Note that locking is not bypassed due to I/O errors; this avoids stampedes
1772  return $this->cache->add( $checkSisterKey, 1, self::LOCK_TTL );
1773  }
1774 
1779  private function yieldStampedeLock( $key, $hasLock ) {
1780  if ( $hasLock ) {
1781  $checkSisterKey = $this->makeSisterKey( $key, self::TYPE_MUTEX );
1782  // The backend might be a mcrouter proxy set to broadcast DELETE to *all* the local
1783  // datacenter cache servers via OperationSelectorRoute (for increased consistency).
1784  // Since that would be excessive for these locks, use TOUCH to expire the key.
1785  $this->cache->changeTTL( $checkSisterKey, $this->getCurrentTime() - 60 );
1786  }
1787  }
1788 
1799  private function makeSisterKeys( array $baseKeys, string $type, string $route = null ) {
1800  $sisterKeys = [];
1801  foreach ( $baseKeys as $baseKey ) {
1802  $sisterKeys[] = $this->makeSisterKey( $baseKey, $type, $route );
1803  }
1804 
1805  return $sisterKeys;
1806  }
1807 
1818  private function makeSisterKey( string $baseKey, string $typeChar, string $route = null ) {
1819  if ( $this->coalesceScheme === self::SCHEME_HASH_STOP ) {
1820  // Key style: "WANCache:<base key>|#|<character>"
1821  $sisterKey = 'WANCache:' . $baseKey . '|#|' . $typeChar;
1822  } else {
1823  // Key style: "WANCache:{<base key>}:<character>"
1824  $sisterKey = 'WANCache:{' . $baseKey . '}:' . $typeChar;
1825  }
1826 
1827  if ( $route !== null ) {
1828  $sisterKey = $this->prependRoute( $sisterKey, $route );
1829  }
1830 
1831  return $sisterKey;
1832  }
1833 
1840  public static function getCollectionFromSisterKey( string $sisterKey ) {
1841  if ( substr( $sisterKey, -4 ) === '|#|v' ) {
1842  // Key style: "WANCache:<base key>|#|<character>"
1843  $collection = substr( $sisterKey, 9, strcspn( $sisterKey, ':|', 9 ) );
1844  } elseif ( substr( $sisterKey, -3 ) === '}:v' ) {
1845  // Key style: "WANCache:{<base key>}:<character>"
1846  $collection = substr( $sisterKey, 10, strcspn( $sisterKey, ':}', 10 ) );
1847  } else {
1848  $collection = 'internal';
1849  }
1850 
1851  return $collection;
1852  }
1853 
1858  private function isVolatileValueAgeNegligible( $age ) {
1859  return ( $age < mt_rand( self::RECENT_SET_LOW_MS, self::RECENT_SET_HIGH_MS ) / 1e3 );
1860  }
1861 
1883  private function checkAndSetCooloff( $key, $kClass, $value, $elapsed, $hasLock ) {
1884  $valueSisterKey = $this->makeSisterKey( $key, self::TYPE_VALUE, $this->onHostRoute );
1885  list( $estimatedSize ) = $this->cache->setNewPreparedValues( [
1886  $valueSisterKey => $value
1887  ] );
1888 
1889  if ( !$hasLock ) {
1890  // Suppose that this cache key is very popular (KEY_HIGH_QPS reads/second).
1891  // After eviction, there will be cache misses until it gets regenerated and saved.
1892  // If the time window when the key is missing lasts less than one second, then the
1893  // number of misses will not reach KEY_HIGH_QPS. This window largely corresponds to
1894  // the key regeneration time. Estimate the count/rate of cache misses, e.g.:
1895  // - 100 QPS, 20ms regeneration => ~2 misses (< 1s)
1896  // - 100 QPS, 100ms regeneration => ~10 misses (< 1s)
1897  // - 100 QPS, 3000ms regeneration => ~300 misses (100/s for 3s)
1898  $missesPerSecForHighQPS = ( min( $elapsed, 1 ) * $this->keyHighQps );
1899 
1900  // Determine whether there is enough I/O stampede risk to justify throttling set().
1901  // Estimate unthrottled set() overhead, as bps, from miss count/rate and value size,
1902  // comparing it to the per-key uplink bps limit (KEY_HIGH_UPLINK_BPS), e.g.:
1903  // - 2 misses (< 1s), 10KB value, 1250000 bps limit => 160000 bits (low risk)
1904  // - 2 misses (< 1s), 100KB value, 1250000 bps limit => 1600000 bits (high risk)
1905  // - 10 misses (< 1s), 10KB value, 1250000 bps limit => 800000 bits (low risk)
1906  // - 10 misses (< 1s), 100KB value, 1250000 bps limit => 8000000 bits (high risk)
1907  // - 300 misses (100/s), 1KB value, 1250000 bps limit => 800000 bps (low risk)
1908  // - 300 misses (100/s), 10KB value, 1250000 bps limit => 8000000 bps (high risk)
1909  // - 300 misses (100/s), 100KB value, 1250000 bps limit => 80000000 bps (high risk)
1910  if ( ( $missesPerSecForHighQPS * $estimatedSize ) >= $this->keyHighUplinkBps ) {
1911  $cooloffSisterKey = $this->makeSisterKey( $key, self::TYPE_COOLOFF );
1912  $this->cache->clearLastError();
1913  if (
1914  !$this->cache->add( $cooloffSisterKey, 1, self::COOLOFF_TTL ) &&
1915  // Don't treat failures due to I/O errors as the key being in cool-off
1916  $this->cache->getLastError() === BagOStuff::ERR_NONE
1917  ) {
1918  $this->stats->increment( "wanobjectcache.$kClass.cooloff_bounce" );
1919 
1920  return false;
1921  }
1922  }
1923  }
1924 
1925  // Corresponding metrics for cache writes that actually get sent over the write
1926  $this->stats->timing( "wanobjectcache.$kClass.regen_set_delay", 1e3 * $elapsed );
1927  $this->stats->updateCount( "wanobjectcache.$kClass.regen_set_bytes", $estimatedSize );
1928 
1929  return true;
1930  }
1931 
1940  private function resolveCTL( $value, $curTTL, $curInfo, $touchedCallback ) {
1941  if ( $touchedCallback === null || $value === false ) {
1942  return [
1943  $curTTL,
1944  max( $curInfo[self::KEY_TOMB_AS_OF], $curInfo[self::KEY_CHECK_AS_OF] )
1945  ];
1946  }
1947 
1948  $touched = $touchedCallback( $value );
1949  if ( $touched !== null && $touched >= $curInfo[self::KEY_AS_OF] ) {
1950  $curTTL = min( $curTTL, self::TINY_NEGATIVE, $curInfo[self::KEY_AS_OF] - $touched );
1951  }
1952 
1953  return [
1954  $curTTL,
1955  max(
1956  $curInfo[self::KEY_TOMB_AS_OF],
1957  $curInfo[self::KEY_CHECK_AS_OF],
1958  $touched
1959  )
1960  ];
1961  }
1962 
1970  private function resolveTouched( $value, $lastPurge, $touchedCallback ) {
1971  return ( $touchedCallback === null || $value === false )
1972  ? $lastPurge // nothing to derive the "touched timestamp" from
1973  : max( $touchedCallback( $value ), $lastPurge );
1974  }
1975 
1981  private function getInterimValue( $key, $minAsOf ) {
1982  $now = $this->getCurrentTime();
1983 
1984  if ( $this->useInterimHoldOffCaching ) {
1985  $wrapped = $this->cache->get(
1986  $this->makeSisterKey( $key, self::TYPE_INTERIM )
1987  );
1988 
1989  list( $value, $keyInfo ) = $this->unwrap( $wrapped, $now );
1990  if ( $this->isValid( $value, $keyInfo[self::KEY_AS_OF], $minAsOf ) ) {
1991  return [ $value, $keyInfo ];
1992  }
1993  }
1994 
1995  return $this->unwrap( false, $now );
1996  }
1997 
2005  private function setInterimValue( $key, $value, $ttl, $version, $walltime ) {
2006  $ttl = max( self::INTERIM_KEY_TTL, (int)$ttl );
2007 
2008  $wrapped = $this->wrap( $value, $ttl, $version, $this->getCurrentTime(), $walltime );
2009  $this->cache->merge(
2010  $this->makeSisterKey( $key, self::TYPE_INTERIM ),
2011  static function () use ( $wrapped ) {
2012  return $wrapped;
2013  },
2014  $ttl,
2015  1
2016  );
2017  }
2018 
2023  private function resolveBusyValue( $busyValue ) {
2024  return ( $busyValue instanceof Closure ) ? $busyValue() : $busyValue;
2025  }
2026 
2091  final public function getMultiWithSetCallback(
2092  ArrayIterator $keyedIds, $ttl, callable $callback, array $opts = []
2093  ) {
2094  // Batch load required keys into the in-process warmup cache
2095  $this->warmupCache = $this->fetchWrappedValuesForWarmupCache(
2096  $this->getNonProcessCachedMultiKeys( $keyedIds, $opts ),
2097  $opts['checkKeys'] ?? []
2098  );
2099  $this->warmupKeyMisses = 0;
2100 
2101  // The required callback signature includes $id as the first argument for convenience
2102  // to distinguish different items. To reuse the code in getWithSetCallback(), wrap the
2103  // callback with a proxy callback that has the standard getWithSetCallback() signature.
2104  // This is defined only once per batch to avoid closure creation overhead.
2105  $proxyCb = static function ( $oldValue, &$ttl, &$setOpts, $oldAsOf, $params )
2106  use ( $callback )
2107  {
2108  return $callback( $params['id'], $oldValue, $ttl, $setOpts, $oldAsOf );
2109  };
2110 
2111  $values = [];
2112  foreach ( $keyedIds as $key => $id ) { // preserve order
2113  $values[$key] = $this->getWithSetCallback(
2114  $key,
2115  $ttl,
2116  $proxyCb,
2117  $opts,
2118  [ 'id' => $id ]
2119  );
2120  }
2121 
2122  $this->warmupCache = [];
2123 
2124  return $values;
2125  }
2126 
2192  final public function getMultiWithUnionSetCallback(
2193  ArrayIterator $keyedIds, $ttl, callable $callback, array $opts = []
2194  ) {
2195  $checkKeys = $opts['checkKeys'] ?? [];
2196  unset( $opts['lockTSE'] ); // incompatible
2197  unset( $opts['busyValue'] ); // incompatible
2198 
2199  // Batch load required keys into the in-process warmup cache
2200  $keysByIdGet = $this->getNonProcessCachedMultiKeys( $keyedIds, $opts );
2201  $this->warmupCache = $this->fetchWrappedValuesForWarmupCache( $keysByIdGet, $checkKeys );
2202  $this->warmupKeyMisses = 0;
2203 
2204  // IDs of entities known to be in need of regeneration
2205  $idsRegen = [];
2206 
2207  // Find out which keys are missing/deleted/stale
2208  $resByKey = $this->fetchKeys( $keysByIdGet, $checkKeys );
2209  foreach ( $keysByIdGet as $id => $key ) {
2210  $res = $resByKey[$key];
2211  $value = $res[self::RES_VALUE];
2212  $metadata = $res[self::RES_METADATA];
2213  if ( $value === false || $metadata[self::KEY_CUR_TTL] < 0 ) {
2214  $idsRegen[] = $id;
2215  }
2216  }
2217 
2218  // Run the callback to populate the regeneration value map for all required IDs
2219  $newSetOpts = [];
2220  $newTTLsById = array_fill_keys( $idsRegen, $ttl );
2221  $newValsById = $idsRegen ? $callback( $idsRegen, $newTTLsById, $newSetOpts ) : [];
2222 
2223  // The required callback signature includes $id as the first argument for convenience
2224  // to distinguish different items. To reuse the code in getWithSetCallback(), wrap the
2225  // callback with a proxy callback that has the standard getWithSetCallback() signature.
2226  // This is defined only once per batch to avoid closure creation overhead.
2227  $proxyCb = static function ( $oldValue, &$ttl, &$setOpts, $oldAsOf, $params )
2228  use ( $callback, $newValsById, $newTTLsById, $newSetOpts )
2229  {
2230  $id = $params['id'];
2231 
2232  if ( array_key_exists( $id, $newValsById ) ) {
2233  // Value was already regerated as expected, so use the value in $newValsById
2234  $newValue = $newValsById[$id];
2235  $ttl = $newTTLsById[$id];
2236  $setOpts = $newSetOpts;
2237  } else {
2238  // Pre-emptive/popularity refresh and version mismatch cases are not detected
2239  // above and thus $newValsById has no entry. Run $callback on this single entity.
2240  $ttls = [ $id => $ttl ];
2241  $newValue = $callback( [ $id ], $ttls, $setOpts )[$id];
2242  $ttl = $ttls[$id];
2243  }
2244 
2245  return $newValue;
2246  };
2247 
2248  // Run the cache-aside logic using warmupCache instead of persistent cache queries
2249  $values = [];
2250  foreach ( $keyedIds as $key => $id ) { // preserve order
2251  $values[$key] = $this->getWithSetCallback(
2252  $key,
2253  $ttl,
2254  $proxyCb,
2255  $opts,
2256  [ 'id' => $id ]
2257  );
2258  }
2259 
2260  $this->warmupCache = [];
2261 
2262  return $values;
2263  }
2264 
2277  final public function reap( $key, $purgeTimestamp, &$isStale = false ) {
2278  $valueSisterKey = $this->makeSisterKey( $key, self::TYPE_VALUE );
2279 
2280  $minAsOf = $purgeTimestamp + self::HOLDOFF_TTL;
2281  $wrapped = $this->cache->get( $valueSisterKey );
2282  if ( is_array( $wrapped ) && $wrapped[self::FLD_TIME] < $minAsOf ) {
2283  $isStale = true;
2284  $this->logger->warning( "Reaping stale value key '$key'." );
2285  $ttlReap = self::HOLDOFF_TTL; // avoids races with tombstone creation
2286  $ok = $this->cache->changeTTL( $valueSisterKey, $ttlReap );
2287  if ( !$ok ) {
2288  $this->logger->error( "Could not complete reap of key '$key'." );
2289  }
2290 
2291  return $ok;
2292  }
2293 
2294  $isStale = false;
2295 
2296  return true;
2297  }
2298 
2308  final public function reapCheckKey( $key, $purgeTimestamp, &$isStale = false ) {
2309  $checkSisterKey = $this->makeSisterKey( $key, self::TYPE_TIMESTAMP );
2310 
2311  $wrapped = $this->cache->get( $checkSisterKey );
2312  $purge = $this->parsePurgeValue( $wrapped );
2313  if ( $purge && $purge[self::PURGE_TIME] < $purgeTimestamp ) {
2314  $isStale = true;
2315  $this->logger->warning( "Reaping stale check key '$key'." );
2316  $ok = $this->cache->changeTTL( $checkSisterKey, self::TTL_SECOND );
2317  if ( !$ok ) {
2318  $this->logger->error( "Could not complete reap of check key '$key'." );
2319  }
2320 
2321  return $ok;
2322  }
2323 
2324  $isStale = false;
2325 
2326  return false;
2327  }
2328 
2339  public function makeGlobalKey( $collection, ...$components ) {
2340  return $this->cache->makeGlobalKey( ...func_get_args() );
2341  }
2342 
2353  public function makeKey( $collection, ...$components ) {
2354  return $this->cache->makeKey( ...func_get_args() );
2355  }
2356 
2364  public function hash256( $component ) {
2365  return hash_hmac( 'sha256', $component, $this->secret );
2366  }
2367 
2418  final public function makeMultiKeys( array $ids, $keyCallback ) {
2419  $idByKey = [];
2420  foreach ( $ids as $id ) {
2421  // Discourage triggering of automatic makeKey() hashing in some backends
2422  if ( strlen( $id ) > 64 ) {
2423  $this->logger->warning( __METHOD__ . ": long ID '$id'; use hash256()" );
2424  }
2425  $key = $keyCallback( $id, $this );
2426  // Edge case: ignore key collisions due to duplicate $ids like "42" and 42
2427  if ( !isset( $idByKey[$key] ) ) {
2428  $idByKey[$key] = $id;
2429  } elseif ( (string)$id !== (string)$idByKey[$key] ) {
2430  throw new UnexpectedValueException(
2431  "Cache key collision; IDs ('$id','{$idByKey[$key]}') map to '$key'"
2432  );
2433  }
2434  }
2435 
2436  return new ArrayIterator( $idByKey );
2437  }
2438 
2474  final public function multiRemap( array $ids, array $res ) {
2475  if ( count( $ids ) !== count( $res ) ) {
2476  // If makeMultiKeys() is called on a list of non-unique IDs, then the resulting
2477  // ArrayIterator will have less entries due to "first appearance" de-duplication
2478  $ids = array_keys( array_fill_keys( $ids, true ) );
2479  if ( count( $ids ) !== count( $res ) ) {
2480  throw new UnexpectedValueException( "Multi-key result does not match ID list" );
2481  }
2482  }
2483 
2484  return array_combine( $ids, $res );
2485  }
2486 
2491  final public function getLastError() {
2492  $code = $this->cache->getLastError();
2493  switch ( $code ) {
2494  case BagOStuff::ERR_NONE:
2495  return self::ERR_NONE;
2496  case BagOStuff::ERR_NO_RESPONSE:
2497  return self::ERR_NO_RESPONSE;
2498  case BagOStuff::ERR_UNREACHABLE:
2499  return self::ERR_UNREACHABLE;
2500  default:
2501  return self::ERR_UNEXPECTED;
2502  }
2503  }
2504 
2508  final public function clearLastError() {
2509  $this->cache->clearLastError();
2510  }
2511 
2517  public function clearProcessCache() {
2518  $this->processCaches = [];
2519  }
2520 
2541  final public function useInterimHoldOffCaching( $enabled ) {
2542  $this->useInterimHoldOffCaching = $enabled;
2543  }
2544 
2550  public function getQoS( $flag ) {
2551  return $this->cache->getQoS( $flag );
2552  }
2553 
2617  public function adaptiveTTL( $mtime, $maxTTL, $minTTL = 30, $factor = 0.2 ) {
2618  if ( is_float( $mtime ) || ctype_digit( $mtime ) ) {
2619  $mtime = (int)$mtime; // handle fractional seconds and string integers
2620  }
2621 
2622  if ( !is_int( $mtime ) || $mtime <= 0 ) {
2623  return $minTTL; // no last-modified time provided
2624  }
2625 
2626  $age = $this->getCurrentTime() - $mtime;
2627 
2628  return (int)min( $maxTTL, max( $minTTL, $factor * $age ) );
2629  }
2630 
2635  final public function getWarmupKeyMisses() {
2636  return $this->warmupKeyMisses;
2637  }
2638 
2653  protected function relayVolatilePurges( array $purgeBySisterKey, int $ttl ) {
2654  $purgeByRouteKey = [];
2655  foreach ( $purgeBySisterKey as $sisterKey => $purge ) {
2656  if ( $this->broadcastRoute !== null ) {
2657  $routeKey = $this->prependRoute( $sisterKey, $this->broadcastRoute );
2658  } else {
2659  $routeKey = $sisterKey;
2660  }
2661  $purgeByRouteKey[$routeKey] = $purge;
2662  }
2663 
2664  if ( count( $purgeByRouteKey ) == 1 ) {
2665  $purge = reset( $purgeByRouteKey );
2666  $ok = $this->cache->set( key( $purgeByRouteKey ), $purge, $ttl );
2667  } else {
2668  $ok = $this->cache->setMulti( $purgeByRouteKey, $ttl );
2669  }
2670 
2671  return $ok;
2672  }
2673 
2682  protected function relayNonVolatilePurge( string $sisterKey ) {
2683  if ( $this->broadcastRoute !== null ) {
2684  $routeKey = $this->prependRoute( $sisterKey, $this->broadcastRoute );
2685  } else {
2686  $routeKey = $sisterKey;
2687  }
2688 
2689  return $this->cache->delete( $routeKey );
2690  }
2691 
2697  protected function prependRoute( string $sisterKey, string $route ) {
2698  if ( $sisterKey[0] === '/' ) {
2699  throw new RuntimeException( "Sister key '$sisterKey' already contains a route." );
2700  }
2701 
2702  return $route . $sisterKey;
2703  }
2704 
2716  private function scheduleAsyncRefresh( $key, $ttl, $callback, array $opts, array $cbParams ) {
2717  if ( !$this->asyncHandler ) {
2718  return false;
2719  }
2720  // Update the cache value later, such during post-send of an HTTP request. This forces
2721  // cache regeneration by setting "minAsOf" to infinity, meaning that no existing value
2722  // is considered valid. Furthermore, note that preemptive regeneration is not applicable
2723  // to invalid values, so there is no risk of infinite preemptive regeneration loops.
2724  $func = $this->asyncHandler;
2725  $func( function () use ( $key, $ttl, $callback, $opts, $cbParams ) {
2726  $opts['minAsOf'] = INF;
2727  try {
2728  $this->fetchOrRegenerate( $key, $ttl, $callback, $opts, $cbParams );
2729  } catch ( Exception $e ) {
2730  // Log some context for easier debugging
2731  $this->logger->error( 'Async refresh failed for {key}', [
2732  'key' => $key,
2733  'ttl' => $ttl,
2734  'exception' => $e
2735  ] );
2736  throw $e;
2737  }
2738  } );
2739 
2740  return true;
2741  }
2742 
2757  private function isAliveOrInGracePeriod( $curTTL, $graceTTL ) {
2758  if ( $curTTL > 0 ) {
2759  return true;
2760  } elseif ( $graceTTL <= 0 ) {
2761  return false;
2762  }
2763 
2764  $ageStale = abs( $curTTL ); // seconds of staleness
2765  $curGraceTTL = ( $graceTTL - $ageStale ); // current grace-time-to-live
2766  if ( $curGraceTTL <= 0 ) {
2767  return false; // already out of grace period
2768  }
2769 
2770  // Chance of using a stale value is the complement of the chance of refreshing it
2771  return !$this->worthRefreshExpiring( $curGraceTTL, $graceTTL, $graceTTL );
2772  }
2773 
2791  protected function worthRefreshExpiring( $curTTL, $logicalTTL, $lowTTL ) {
2792  if ( $lowTTL <= 0 ) {
2793  return false;
2794  }
2795 
2796  // T264787: avoid having keys start off with a high chance of being refreshed;
2797  // the point where refreshing becomes possible cannot precede the key lifetime.
2798  $effectiveLowTTL = min( $lowTTL, $logicalTTL ?: INF );
2799 
2800  if ( $curTTL >= $effectiveLowTTL || $curTTL <= 0 ) {
2801  return false;
2802  }
2803 
2804  $chance = ( 1 - $curTTL / $effectiveLowTTL );
2805 
2806  // @phan-suppress-next-line PhanTypeMismatchArgumentInternal
2807  $decision = ( mt_rand( 1, 1e9 ) <= 1e9 * $chance );
2808 
2809  $this->logger->debug(
2810  "worthRefreshExpiring($curTTL, $logicalTTL, $lowTTL): " .
2811  "p = $chance; refresh = " . ( $decision ? 'Y' : 'N' )
2812  );
2813 
2814  return $decision;
2815  }
2816 
2832  protected function worthRefreshPopular( $asOf, $ageNew, $timeTillRefresh, $now ) {
2833  if ( $ageNew < 0 || $timeTillRefresh <= 0 ) {
2834  return false;
2835  }
2836 
2837  $age = $now - $asOf;
2838  $timeOld = $age - $ageNew;
2839  if ( $timeOld <= 0 ) {
2840  return false;
2841  }
2842 
2843  $popularHitsPerSec = 1;
2844  // Lifecycle is: new, ramp-up refresh chance, full refresh chance.
2845  // Note that the "expected # of refreshes" for the ramp-up time range is half
2846  // of what it would be if P(refresh) was at its full value during that time range.
2847  $refreshWindowSec = max( $timeTillRefresh - $ageNew - self::RAMPUP_TTL / 2, 1 );
2848  // P(refresh) * (# hits in $refreshWindowSec) = (expected # of refreshes)
2849  // P(refresh) * ($refreshWindowSec * $popularHitsPerSec) = 1 (by definition)
2850  // P(refresh) = 1/($refreshWindowSec * $popularHitsPerSec)
2851  $chance = 1 / ( $popularHitsPerSec * $refreshWindowSec );
2852  // Ramp up $chance from 0 to its nominal value over RAMPUP_TTL seconds to avoid stampedes
2853  $chance *= ( $timeOld <= self::RAMPUP_TTL ) ? $timeOld / self::RAMPUP_TTL : 1;
2854 
2855  // @phan-suppress-next-line PhanTypeMismatchArgumentInternal
2856  $decision = ( mt_rand( 1, 1e9 ) <= 1e9 * $chance );
2857 
2858  $this->logger->debug(
2859  "worthRefreshPopular($asOf, $ageNew, $timeTillRefresh, $now): " .
2860  "p = $chance; refresh = " . ( $decision ? 'Y' : 'N' )
2861  );
2862 
2863  return $decision;
2864  }
2865 
2875  protected function isValid( $value, $asOf, $minAsOf, $purgeTime = null ) {
2876  // Avoid reading any key not generated after the latest delete() or touch
2877  $safeMinAsOf = max( $minAsOf, $purgeTime + self::TINY_POSTIVE );
2878 
2879  if ( $value === false ) {
2880  return false;
2881  } elseif ( $safeMinAsOf > 0 && $asOf < $minAsOf ) {
2882  return false;
2883  }
2884 
2885  return true;
2886  }
2887 
2896  private function wrap( $value, $ttl, $version, $now, $walltime ) {
2897  // Returns keys in ascending integer order for PHP7 array packing:
2898  // https://nikic.github.io/2014/12/22/PHPs-new-hashtable-implementation.html
2899  $wrapped = [
2900  self::FLD_FORMAT_VERSION => self::VERSION,
2901  self::FLD_VALUE => $value,
2902  self::FLD_TTL => $ttl,
2903  self::FLD_TIME => $now
2904  ];
2905  if ( $version !== null ) {
2906  $wrapped[self::FLD_VALUE_VERSION] = $version;
2907  }
2908  if ( $walltime >= self::GENERATION_SLOW_SEC ) {
2909  $wrapped[self::FLD_GENERATION_TIME] = $walltime;
2910  }
2911 
2912  return $wrapped;
2913  }
2914 
2927  private function unwrap( $wrapped, $now ) {
2928  $value = false;
2929  $info = $this->newKeyInfoPlaceholder();
2930 
2931  if ( is_array( $wrapped ) ) {
2932  // Entry expected to be a cached value; validate it
2933  if (
2934  ( $wrapped[self::FLD_FORMAT_VERSION] ?? null ) === self::VERSION &&
2935  $wrapped[self::FLD_TIME] >= $this->epoch
2936  ) {
2937  if ( $wrapped[self::FLD_TTL] > 0 ) {
2938  // Get the approximate time left on the key
2939  $age = $now - $wrapped[self::FLD_TIME];
2940  $curTTL = max( $wrapped[self::FLD_TTL] - $age, 0.0 );
2941  } else {
2942  // Key had no TTL, so the time left is unbounded
2943  $curTTL = INF;
2944  }
2945  $value = $wrapped[self::FLD_VALUE];
2946  $info[self::KEY_VERSION] = $wrapped[self::FLD_VALUE_VERSION] ?? null;
2947  $info[self::KEY_AS_OF] = $wrapped[self::FLD_TIME];
2948  $info[self::KEY_CUR_TTL] = $curTTL;
2949  $info[self::KEY_TTL] = $wrapped[self::FLD_TTL];
2950  }
2951  } else {
2952  // Entry expected to be a tombstone; parse it
2953  $purge = $this->parsePurgeValue( $wrapped );
2954  if ( $purge !== false ) {
2955  // Tombstoned keys should always have a negative current $ttl
2956  $info[self::KEY_CUR_TTL] =
2957  min( $purge[self::PURGE_TIME] - $now, self::TINY_NEGATIVE );
2958  $info[self::KEY_TOMB_AS_OF] = $purge[self::PURGE_TIME];
2959  }
2960  }
2961 
2962  return [ $value, $info ];
2963  }
2964 
2968  private function newKeyInfoPlaceholder() {
2969  return [
2970  self::KEY_VERSION => null,
2971  self::KEY_AS_OF => null,
2972  self::KEY_TTL => null,
2973  self::KEY_CUR_TTL => null,
2974  self::KEY_TOMB_AS_OF => null
2975  ];
2976  }
2977 
2982  private function determineKeyClassForStats( $key ) {
2983  $parts = explode( ':', $key, 3 );
2984  // Sanity fallback in case the key was not made by makeKey.
2985  // Replace dots because they are special in StatsD (T232907)
2986  return strtr( $parts[1] ?? $parts[0], '.', '_' );
2987  }
2988 
2998  private function parsePurgeValue( $value ) {
2999  if ( !is_string( $value ) ) {
3000  return false;
3001  }
3002 
3003  $segments = explode( ':', $value, 3 );
3004  if ( isset( $segments[2] ) ) {
3005  $prefix = $segments[0];
3006  $timestamp = (float)$segments[1];
3007  $holdoff = (int)$segments[2];
3008  } elseif ( isset( $segments[1] ) ) {
3009  $prefix = $segments[0];
3010  $timestamp = (float)$segments[1];
3011  // Value tombstones don't store hold-off TTLs
3012  $holdoff = self::HOLDOFF_TTL;
3013  } else {
3014  return false;
3015  }
3016 
3017  if ( "{$prefix}:" !== self::PURGE_VAL_PREFIX || $timestamp < $this->epoch ) {
3018  // Not a purge value or the purge value is too old
3019  return false;
3020  }
3021 
3022  return [ self::PURGE_TIME => $timestamp, self::PURGE_HOLDOFF => $holdoff ];
3023  }
3024 
3029  private function makeTombstonePurgeValue( $timestamp ) {
3030  return self::PURGE_VAL_PREFIX . number_format( $timestamp, 4, '.', '' );
3031  }
3032 
3038  private function makeCheckPurgeValue( $timestamp, int $holdoff ) {
3039  return self::PURGE_VAL_PREFIX . number_format( $timestamp, 4, '.', '' ) . ":$holdoff";
3040  }
3041 
3046  private function getProcessCache( $group ) {
3047  if ( !isset( $this->processCaches[$group] ) ) {
3048  list( , $size ) = explode( ':', $group );
3049  $this->processCaches[$group] = new MapCacheLRU( (int)$size );
3050  if ( $this->wallClockOverride !== null ) {
3051  $this->processCaches[$group]->setMockTime( $this->wallClockOverride );
3052  }
3053  }
3054 
3055  return $this->processCaches[$group];
3056  }
3057 
3063  private function getProcessCacheKey( $key, $version ) {
3064  return $key . ' ' . (int)$version;
3065  }
3066 
3072  private function getNonProcessCachedMultiKeys( ArrayIterator $keys, array $opts ) {
3073  $pcTTL = $opts['pcTTL'] ?? self::TTL_UNCACHEABLE;
3074 
3075  $keysMissing = [];
3076  if ( $pcTTL > 0 && $this->callbackDepth == 0 ) {
3077  $version = $opts['version'] ?? null;
3078  $pCache = $this->getProcessCache( $opts['pcGroup'] ?? self::PC_PRIMARY );
3079  foreach ( $keys as $key => $id ) {
3080  if ( !$pCache->has( $this->getProcessCacheKey( $key, $version ), $pcTTL ) ) {
3081  $keysMissing[$id] = $key;
3082  }
3083  }
3084  }
3085 
3086  return $keysMissing;
3087  }
3088 
3095  private function fetchWrappedValuesForWarmupCache( array $keys, array $checkKeys ) {
3096  if ( !$keys ) {
3097  return [];
3098  }
3099 
3100  // Get all the value keys to fetch...
3101  $sisterKeys = $this->makeSisterKeys( $keys, self::TYPE_VALUE, $this->onHostRoute );
3102  // Get all the flux keys to fetch...
3103  if ( $this->onHostRoute !== null ) {
3104  foreach ( $keys as $key ) {
3105  $sisterKeys[] = $this->makeSisterKey( $key, self::TYPE_FLUX );
3106  }
3107  }
3108  // Get all the "check" keys to fetch...
3109  foreach ( $checkKeys as $i => $checkKeyOrKeyGroup ) {
3110  // Note: avoid array_merge() inside loop in case there are many keys
3111  if ( is_int( $i ) ) {
3112  // Single "check" key that applies to all value keys
3113  $sisterKeys[] = $this->makeSisterKey( $checkKeyOrKeyGroup, self::TYPE_TIMESTAMP );
3114  } else {
3115  // List of "check" keys that apply to a specific value key
3116  foreach ( (array)$checkKeyOrKeyGroup as $checkKey ) {
3117  $sisterKeys[] = $this->makeSisterKey( $checkKey, self::TYPE_TIMESTAMP );
3118  }
3119  }
3120  }
3121 
3122  $wrappedBySisterKey = $this->cache->getMulti( $sisterKeys );
3123  $wrappedBySisterKey += array_fill_keys( $sisterKeys, false );
3124 
3125  return $wrappedBySisterKey;
3126  }
3127 
3132  protected function getCurrentTime() {
3133  if ( $this->wallClockOverride ) {
3134  return $this->wallClockOverride;
3135  }
3136 
3137  $clockTime = (float)time(); // call this first
3138  // microtime() uses an initial gettimeofday() call added to usage clocks.
3139  // This can severely drift from time() and the microtime() value of other threads
3140  // due to undercounting of the amount of time elapsed. Instead of seeing the current
3141  // time as being in the past, use the value of time(). This avoids setting cache values
3142  // that will immediately be seen as expired and possibly cause stampedes.
3143  return max( microtime( true ), $clockTime );
3144  }
3145 
3150  public function setMockTime( &$time ) {
3151  $this->wallClockOverride =& $time;
3152  $this->cache->setMockTime( $time );
3153  foreach ( $this->processCaches as $pCache ) {
3154  $pCache->setMockTime( $time );
3155  }
3156  }
3157 }
WANObjectCache\relayVolatilePurges
relayVolatilePurges(array $purgeBySisterKey, int $ttl)
Set a sister key to a purge value in all datacenters.
Definition: WANObjectCache.php:2653
WANObjectCache\getWithSetCallback
getWithSetCallback( $key, $ttl, $callback, array $opts=[], array $cbParams=[])
Method to fetch/regenerate a cache key.
Definition: WANObjectCache.php:1521
WANObjectCache\KEY_TTL
const KEY_TTL
Logical TTL attribute for a key.
Definition: WANObjectCache.php:258
WANObjectCache\getQoS
getQoS( $flag)
Definition: WANObjectCache.php:2550
WANObjectCache\hash256
hash256( $component)
Hash a possibly long string into a suitable component for makeKey()/makeGlobalKey()
Definition: WANObjectCache.php:2364
WANObjectCache\determineKeyClassForStats
determineKeyClassForStats( $key)
Definition: WANObjectCache.php:2982
WANObjectCache\setMockTime
setMockTime(&$time)
Definition: WANObjectCache.php:3150
WANObjectCache\$warmupCache
mixed[] $warmupCache
Temporary warm-up cache.
Definition: WANObjectCache.php:167
WANObjectCache\RES_VALUE
const RES_VALUE
The key value component of a fetchMulti() result.
Definition: WANObjectCache.php:249
WANObjectCache\SCHEME_HASH_TAG
const SCHEME_HASH_TAG
Use twemproxy-style Hash Tag key scheme (e.g.
Definition: WANObjectCache.php:211
EmptyBagOStuff
A BagOStuff object with no objects in it.
Definition: EmptyBagOStuff.php:29
WANObjectCache\KEY_VERSION
const KEY_VERSION
Version number attribute for a key; keep value for b/c (< 1.36)
Definition: WANObjectCache.php:254
WANObjectCache\$cache
BagOStuff $cache
The local datacenter cache.
Definition: WANObjectCache.php:130
WANObjectCache\fetchOrRegenerate
fetchOrRegenerate( $key, $ttl, $callback, array $opts, array $cbParams)
Do the actual I/O for getWithSetCallback() when needed.
Definition: WANObjectCache.php:1581
WANObjectCache\MAX_READ_LAG
const MAX_READ_LAG
Max expected seconds of combined lag from replication and view snapshots.
Definition: WANObjectCache.php:177
WANObjectCache\isValid
isValid( $value, $asOf, $minAsOf, $purgeTime=null)
Check if $value is not false, versioned (if needed), and not older than $minTime (if set)
Definition: WANObjectCache.php:2875
WANObjectCache\makeCheckPurgeValue
makeCheckPurgeValue( $timestamp, int $holdoff)
Definition: WANObjectCache.php:3038
WANObjectCache\TYPE_FLUX
const TYPE_FLUX
Single character component for flux keys.
Definition: WANObjectCache.php:286
WANObjectCache\FLD_VALUE
const FLD_VALUE
Key to the cached value.
Definition: WANObjectCache.php:269
WANObjectCache\MAX_COMMIT_DELAY
const MAX_COMMIT_DELAY
Max expected seconds to pass between delete() and DB commit finishing.
Definition: WANObjectCache.php:175
WANObjectCache\processFluxKeys
processFluxKeys(array $keys, array $fluxSisterKeys, array $wrappedBySisterKey)
Definition: WANObjectCache.php:658
WANObjectCache\GENERATION_SLOW_SEC
const GENERATION_SLOW_SEC
Consider value generation slow if it takes more than this many seconds.
Definition: WANObjectCache.php:238
Wikimedia\LightweightObjectStore\ExpirationAwareness
Generic interface providing Time-To-Live constants for expirable object storage.
Definition: ExpirationAwareness.php:32
WANObjectCache\TYPE_INTERIM
const TYPE_INTERIM
Single character component for interium value keys.
Definition: WANObjectCache.php:290
WANObjectCache\makeTombstonePurgeValue
makeTombstonePurgeValue( $timestamp)
Definition: WANObjectCache.php:3029
WANObjectCache\RECENT_SET_LOW_MS
const RECENT_SET_LOW_MS
Min millisecond set() backoff during hold-off (far less than INTERIM_KEY_TTL)
Definition: WANObjectCache.php:233
WANObjectCache\KEY_CUR_TTL
const KEY_CUR_TTL
Remaining TTL attribute for a key; keep value for b/c (< 1.36)
Definition: WANObjectCache.php:260
NullStatsdDataFactory
Definition: NullStatsdDataFactory.php:10
WANObjectCache\FLD_GENERATION_TIME
const FLD_GENERATION_TIME
Key to how long it took to generate the value.
Definition: WANObjectCache.php:279
WANObjectCache\FLD_FORMAT_VERSION
const FLD_FORMAT_VERSION
Key to WAN cache version number.
Definition: WANObjectCache.php:267
WANObjectCache\getMultiWithSetCallback
getMultiWithSetCallback(ArrayIterator $keyedIds, $ttl, callable $callback, array $opts=[])
Method to fetch multiple cache keys at once with regeneration.
Definition: WANObjectCache.php:2091
BagOStuff
Class representing a cache/ephemeral data store.
Definition: BagOStuff.php:86
WANObjectCache\getMultiWithUnionSetCallback
getMultiWithUnionSetCallback(ArrayIterator $keyedIds, $ttl, callable $callback, array $opts=[])
Method to fetch/regenerate multiple cache keys at once.
Definition: WANObjectCache.php:2192
WANObjectCache\resolveTouched
resolveTouched( $value, $lastPurge, $touchedCallback)
Definition: WANObjectCache.php:1970
WANObjectCache\getCollectionFromSisterKey
static getCollectionFromSisterKey(string $sisterKey)
Definition: WANObjectCache.php:1840
WANObjectCache\PASS_BY_REF
const PASS_BY_REF
Idiom for get()/getMulti() to return extra information by reference.
Definition: WANObjectCache.php:208
WANObjectCache\FLD_VALUE_VERSION
const FLD_VALUE_VERSION
Key to collection cache version number.
Definition: WANObjectCache.php:277
WANObjectCache\worthRefreshExpiring
worthRefreshExpiring( $curTTL, $logicalTTL, $lowTTL)
Check if a key is nearing expiration and thus due for randomized regeneration.
Definition: WANObjectCache.php:2791
$res
$res
Definition: testCompression.php:57
WANObjectCache\TTL_LAGGED
const TTL_LAGGED
Max TTL, in seconds, to store keys when a data sourced is lagged.
Definition: WANObjectCache.php:184
WANObjectCache\HOLDOFF_TTL_NONE
const HOLDOFF_TTL_NONE
Idiom for delete()/touchCheckKey() meaning "no hold-off period".
Definition: WANObjectCache.php:199
WANObjectCache\fetchKeys
fetchKeys(array $keys, array $checkKeys)
Fetch the value and key metadata of several keys from cache.
Definition: WANObjectCache.php:530
WANObjectCache\reapCheckKey
reapCheckKey( $key, $purgeTimestamp, &$isStale=false)
Set a "check" key to soon expire in the local cluster if it pre-dates $purgeTimestamp.
Definition: WANObjectCache.php:2308
WANObjectCache\SCHEME_HASH_STOP
const SCHEME_HASH_STOP
Use mcrouter-style Hash Stop key scheme (e.g.
Definition: WANObjectCache.php:213
WANObjectCache\getCheckKeyTime
getCheckKeyTime( $key)
Fetch the value of a timestamp "check" key.
Definition: WANObjectCache.php:1038
WANObjectCache\resolveBusyValue
resolveBusyValue( $busyValue)
Definition: WANObjectCache.php:2023
WANObjectCache\$logger
LoggerInterface $logger
Definition: WANObjectCache.php:134
WANObjectCache\$warmupKeyMisses
int $warmupKeyMisses
Key fetched.
Definition: WANObjectCache.php:169
WANObjectCache\newEmpty
static newEmpty()
Get an instance that wraps EmptyBagOStuff.
Definition: WANObjectCache.php:382
WANObjectCache\LOW_TTL
const LOW_TTL
Consider regeneration if the key will expire within this many seconds.
Definition: WANObjectCache.php:182
WANObjectCache\FLD_TIME
const FLD_TIME
Key to the cache timestamp.
Definition: WANObjectCache.php:273
WANObjectCache\clearProcessCache
clearProcessCache()
Clear the in-process caches; useful for testing.
Definition: WANObjectCache.php:2517
WANObjectCache\getInterimValue
getInterimValue( $key, $minAsOf)
Definition: WANObjectCache.php:1981
WANObjectCache\$broadcastRoute
string null $broadcastRoute
Routing prefix for values that should be broadcasted to all data centers.
Definition: WANObjectCache.php:147
WANObjectCache\$secret
string $secret
Stable secret used for hasing long strings into key components.
Definition: WANObjectCache.php:155
WANObjectCache\clearLastError
clearLastError()
Clear the "last error" registry.
Definition: WANObjectCache.php:2508
WANObjectCache\getCurrentTime
getCurrentTime()
Definition: WANObjectCache.php:3132
WANObjectCache\INTERIM_KEY_TTL
const INTERIM_KEY_TTL
Seconds to keep interim value keys for tombstoned keys around.
Definition: WANObjectCache.php:218
WANObjectCache\unwrap
unwrap( $wrapped, $now)
Definition: WANObjectCache.php:2927
WANObjectCache\fetchWrappedValuesForWarmupCache
fetchWrappedValuesForWarmupCache(array $keys, array $checkKeys)
Definition: WANObjectCache.php:3095
WANObjectCache\getLastError
getLastError()
Get the "last error" registered; clearLastError() should be called manually.
Definition: WANObjectCache.php:2491
WANObjectCache\relayNonVolatilePurge
relayNonVolatilePurge(string $sisterKey)
Remove a sister key from all datacenters.
Definition: WANObjectCache.php:2682
WANObjectCache\makeMultiKeys
makeMultiKeys(array $ids, $keyCallback)
Get an iterator of (cache key => entity ID) for a list of entity IDs.
Definition: WANObjectCache.php:2418
WANObjectCache\getMulti
getMulti(array $keys, &$curTTLs=[], array $checkKeys=[], &$info=[])
Fetch the value of several keys from cache.
Definition: WANObjectCache.php:480
WANObjectCache\__construct
__construct(array $params)
Definition: WANObjectCache.php:340
WANObjectCache\useInterimHoldOffCaching
useInterimHoldOffCaching( $enabled)
Enable or disable the use of brief caching for tombstoned keys.
Definition: WANObjectCache.php:2541
WANObjectCache\reap
reap( $key, $purgeTimestamp, &$isStale=false)
Set a key to soon expire in the local cluster if it pre-dates $purgeTimestamp.
Definition: WANObjectCache.php:2277
WANObjectCache\$onHostRoute
string null $onHostRoute
Routing prefix for value keys that support use of an on-host tier.
Definition: WANObjectCache.php:149
WANObjectCache\$keyHighQps
int $keyHighQps
Reads/second assumed during a hypothetical cache write stampede for a key.
Definition: WANObjectCache.php:160
WANObjectCache\touchCheckKey
touchCheckKey( $key, $holdoff=self::HOLDOFF_TTL)
Purge a "check" key from all datacenters, invalidating keys that use it.
Definition: WANObjectCache.php:1168
WANObjectCache\TYPE_COOLOFF
const TYPE_COOLOFF
Single character component for cool-off bounce keys.
Definition: WANObjectCache.php:292
MapCacheLRU
Handles a simple LRU key/value map with a maximum number of entries.
Definition: MapCacheLRU.php:37
WANObjectCache\RES_METADATA
const RES_METADATA
The key metadata component of a fetchMulti() result.
Definition: WANObjectCache.php:251
WANObjectCache\makeSisterKey
makeSisterKey(string $baseKey, string $typeChar, string $route=null)
Get a sister key that should be collocated with a base cache key.
Definition: WANObjectCache.php:1818
WANObjectCache\wrap
wrap( $value, $ttl, $version, $now, $walltime)
Definition: WANObjectCache.php:2896
WANObjectCache\adaptiveTTL
adaptiveTTL( $mtime, $maxTTL, $minTTL=30, $factor=0.2)
Get a TTL that is higher for objects that have not changed recently.
Definition: WANObjectCache.php:2617
WANObjectCache\getNonProcessCachedMultiKeys
getNonProcessCachedMultiKeys(ArrayIterator $keys, array $opts)
Definition: WANObjectCache.php:3072
WANObjectCache\$epoch
float $epoch
Unix timestamp of the oldest possible valid values.
Definition: WANObjectCache.php:153
WANObjectCache\KEY_TOMB_AS_OF
const KEY_TOMB_AS_OF
Tomstone timestamp attribute for a key; keep value for b/c (< 1.36)
Definition: WANObjectCache.php:262
WANObjectCache\$wallClockOverride
float null $wallClockOverride
Definition: WANObjectCache.php:172
WANObjectCache\HOLDOFF_TTL
const HOLDOFF_TTL
Seconds to tombstone keys on delete() and treat as volatile after invalidation.
Definition: WANObjectCache.php:179
Wikimedia\LightweightObjectStore\StorageAwareness
Generic interface providing error code and quality-of-service constants for object stores.
Definition: StorageAwareness.php:32
WANObjectCache\makeGlobalKey
makeGlobalKey( $collection,... $components)
Make a cache key for the global keyspace and given components.
Definition: WANObjectCache.php:2339
WANObjectCache\isVolatileValueAgeNegligible
isVolatileValueAgeNegligible( $age)
Definition: WANObjectCache.php:1858
WANObjectCache\getMultiCheckKeyTime
getMultiCheckKeyTime(array $keys)
Fetch the values of each timestamp "check" key.
Definition: WANObjectCache.php:1103
WANObjectCache\worthRefreshPopular
worthRefreshPopular( $asOf, $ageNew, $timeTillRefresh, $now)
Check if a key is due for randomized regeneration due to its popularity.
Definition: WANObjectCache.php:2832
WANObjectCache\TYPE_TIMESTAMP
const TYPE_TIMESTAMP
Single character component for timestamp check keys.
Definition: WANObjectCache.php:284
WANObjectCache\isAliveOrInGracePeriod
isAliveOrInGracePeriod( $curTTL, $graceTTL)
Check if a key is fresh or in the grace window and thus due for randomized reuse.
Definition: WANObjectCache.php:2757
WANObjectCache\RAMPUP_TTL
const RAMPUP_TTL
Seconds to ramp up the chance of regeneration due to expected time-till-refresh.
Definition: WANObjectCache.php:225
WANObjectCache\$useInterimHoldOffCaching
bool $useInterimHoldOffCaching
Whether to use "interim" caching while keys are tombstoned.
Definition: WANObjectCache.php:151
WANObjectCache\PC_PRIMARY
const PC_PRIMARY
Default process cache name and max key count.
Definition: WANObjectCache.php:205
WANObjectCache
Multi-datacenter aware caching interface.
Definition: WANObjectCache.php:128
WANObjectCache\PURGE_TIME
const PURGE_TIME
Key to the tombstone entry timestamp.
Definition: WANObjectCache.php:241
WANObjectCache\VERSION
const VERSION
Cache format version number.
Definition: WANObjectCache.php:246
WANObjectCache\getProcessCache
getProcessCache( $group)
Definition: WANObjectCache.php:3046
WANObjectCache\KEY_AS_OF
const KEY_AS_OF
Generation timestamp attribute for a key; keep value for b/c (< 1.36)
Definition: WANObjectCache.php:256
WANObjectCache\multiRemap
multiRemap(array $ids, array $res)
Get an (ID => value) map from (i) a non-unique list of entity IDs, and (ii) the list of corresponding...
Definition: WANObjectCache.php:2474
WANObjectCache\$keyHighUplinkBps
float $keyHighUplinkBps
Max tolerable bytes/second to spend on a cache write stampede for a key.
Definition: WANObjectCache.php:162
WANObjectCache\checkAndSetCooloff
checkAndSetCooloff( $key, $kClass, $value, $elapsed, $hasLock)
Check whether set() is rate-limited to avoid concurrent I/O spikes.
Definition: WANObjectCache.php:1883
WANObjectCache\processCheckKeys
processCheckKeys(array $checkSisterKeys, array $wrappedBySisterKey, float $now)
Definition: WANObjectCache.php:689
WANObjectCache\resetCheckKey
resetCheckKey( $key)
Delete a "check" key from all datacenters, invalidating keys that use it.
Definition: WANObjectCache.php:1208
WANObjectCache\TSE_NONE
const TSE_NONE
Idiom for getWithSetCallback() meaning "no cache stampede mutex".
Definition: WANObjectCache.php:192
WANObjectCache\newKeyInfoPlaceholder
newKeyInfoPlaceholder()
Definition: WANObjectCache.php:2968
WANObjectCache\claimStampedeLock
claimStampedeLock( $key)
Definition: WANObjectCache.php:1769
WANObjectCache\makeSisterKeys
makeSisterKeys(array $baseKeys, string $type, string $route=null)
Get sister keys that should be collocated with their corresponding base cache keys.
Definition: WANObjectCache.php:1799
WANObjectCache\parsePurgeValue
parsePurgeValue( $value)
Extract purge metadata from cached value if it is a valid purge value.
Definition: WANObjectCache.php:2998
WANObjectCache\HOT_TTR
const HOT_TTR
Expected time-till-refresh, in seconds, if the key is accessed once per second.
Definition: WANObjectCache.php:187
WANObjectCache\LOCK_TTL
const LOCK_TTL
Seconds to keep lock keys around.
Definition: WANObjectCache.php:221
WANObjectCache\TYPE_MUTEX
const TYPE_MUTEX
Single character component for mutex lock keys.
Definition: WANObjectCache.php:288
WANObjectCache\getProcessCacheKey
getProcessCacheKey( $key, $version)
Definition: WANObjectCache.php:3063
WANObjectCache\prependRoute
prependRoute(string $sisterKey, string $route)
Definition: WANObjectCache.php:2697
WANObjectCache\yieldStampedeLock
yieldStampedeLock( $key, $hasLock)
Definition: WANObjectCache.php:1779
WANObjectCache\PURGE_VAL_PREFIX
const PURGE_VAL_PREFIX
Value prefix of purge values.
Definition: WANObjectCache.php:295
WANObjectCache\$asyncHandler
callable null $asyncHandler
Function that takes a WAN cache callback and runs it later.
Definition: WANObjectCache.php:138
WANObjectCache\getWarmupKeyMisses
getWarmupKeyMisses()
Definition: WANObjectCache.php:2635
$keys
$keys
Definition: testCompression.php:72
WANObjectCache\KEY_CHECK_AS_OF
const KEY_CHECK_AS_OF
Highest "check" key timestamp for a key; keep value for b/c (< 1.36)
Definition: WANObjectCache.php:264
WANObjectCache\resolveCTL
resolveCTL( $value, $curTTL, $curInfo, $touchedCallback)
Definition: WANObjectCache.php:1940
WANObjectCache\$callbackDepth
int $callbackDepth
Callback stack depth for getWithSetCallback()
Definition: WANObjectCache.php:165
WANObjectCache\FLD_FLAGS
const FLD_FLAGS
Key to the flags bit field (reserved number)
Definition: WANObjectCache.php:275
WANObjectCache\$coalesceScheme
int $coalesceScheme
Scheme to use for key coalescing (Hash Tags or Hash Stops)
Definition: WANObjectCache.php:157
WANObjectCache\AGE_NEW
const AGE_NEW
Minimum key age, in seconds, for expected time-till-refresh to be considered.
Definition: WANObjectCache.php:189
IStoreKeyEncoder
Generic interface for object stores with key encoding methods.
Definition: IStoreKeyEncoder.php:9
WANObjectCache\setLogger
setLogger(LoggerInterface $logger)
Definition: WANObjectCache.php:373
WANObjectCache\TYPE_VALUE
const TYPE_VALUE
Single character component for value keys.
Definition: WANObjectCache.php:282
WANObjectCache\makeKey
makeKey( $collection,... $components)
Make a cache key using the "global" keyspace for the given components.
Definition: WANObjectCache.php:2353
WANObjectCache\CHECK_KEY_TTL
const CHECK_KEY_TTL
Seconds to keep dependency purge keys around.
Definition: WANObjectCache.php:216
WANObjectCache\GRACE_TTL_NONE
const GRACE_TTL_NONE
Idiom for set()/getWithSetCallback() meaning "no post-expiration grace period".
Definition: WANObjectCache.php:197
WANObjectCache\RECENT_SET_HIGH_MS
const RECENT_SET_HIGH_MS
Max millisecond set() backoff during hold-off (far less than INTERIM_KEY_TTL)
Definition: WANObjectCache.php:235
WANObjectCache\FLD_TTL
const FLD_TTL
Key to the original TTL.
Definition: WANObjectCache.php:271
WANObjectCache\PURGE_HOLDOFF
const PURGE_HOLDOFF
Key to the tombstone entry hold-off TTL.
Definition: WANObjectCache.php:243
WANObjectCache\STALE_TTL_NONE
const STALE_TTL_NONE
Idiom for set()/getWithSetCallback() meaning "no post-expiration persistence".
Definition: WANObjectCache.php:195
WANObjectCache\$stats
StatsdDataFactoryInterface $stats
Definition: WANObjectCache.php:136
WANObjectCache\setInterimValue
setInterimValue( $key, $value, $ttl, $version, $walltime)
Definition: WANObjectCache.php:2005
WANObjectCache\$processCaches
MapCacheLRU[] $processCaches
Map of group PHP instance caches.
Definition: WANObjectCache.php:132
WANObjectCache\scheduleAsyncRefresh
scheduleAsyncRefresh( $key, $ttl, $callback, array $opts, array $cbParams)
Schedule a deferred cache regeneration if possible.
Definition: WANObjectCache.php:2716
WANObjectCache\COOLOFF_TTL
const COOLOFF_TTL
Seconds to no-op key set() calls to avoid large blob I/O stampedes.
Definition: WANObjectCache.php:223
$type
$type
Definition: testCompression.php:52