MediaWiki  master
WANObjectCache.php
Go to the documentation of this file.
1 <?php
22 use Liuggio\StatsdClient\Factory\StatsdDataFactoryInterface;
23 use Psr\Log\LoggerAwareInterface;
24 use Psr\Log\LoggerInterface;
25 use Psr\Log\NullLogger;
28 
126 class WANObjectCache implements
130  LoggerAwareInterface
131 {
133  protected $cache;
135  protected $processCaches = [];
137  protected $logger;
139  protected $stats;
141  protected $asyncHandler;
142 
150  protected $broadcastRoute;
152  protected $useInterimHoldOffCaching = true;
154  protected $epoch;
156  protected $secret;
158  protected $coalesceScheme;
159 
161  private $keyHighQps;
164 
166  private $missLog;
167 
169  private $callbackDepth = 0;
171  private $warmupCache = [];
173  private $warmupKeyMisses = 0;
174 
177 
179  private const MAX_COMMIT_DELAY = 3;
181  private const MAX_READ_LAG = 7;
183  public const HOLDOFF_TTL = self::MAX_COMMIT_DELAY + self::MAX_READ_LAG + 1;
184 
186  private const LOW_TTL = 30;
188  public const TTL_LAGGED = 30;
189 
191  private const HOT_TTR = 900;
193  private const AGE_NEW = 60;
194 
196  private const TSE_NONE = -1;
197 
199  public const STALE_TTL_NONE = 0;
201  public const GRACE_TTL_NONE = 0;
203  public const HOLDOFF_TTL_NONE = 0;
204 
206  public const MIN_TIMESTAMP_NONE = 0.0;
207 
209  private const PC_PRIMARY = 'primary:1000';
210 
212  public const PASS_BY_REF = [];
213 
215  private const SCHEME_HASH_TAG = 1;
217  private const SCHEME_HASH_STOP = 2;
218 
220  private const CHECK_KEY_TTL = self::TTL_YEAR;
222  private const INTERIM_KEY_TTL = 1;
223 
225  private const LOCK_TTL = 10;
227  private const COOLOFF_TTL = 1;
229  private const RAMPUP_TTL = 30;
230 
232  private const TINY_NEGATIVE = -0.000001;
234  private const TINY_POSTIVE = 0.000001;
235 
237  private const RECENT_SET_LOW_MS = 50;
239  private const RECENT_SET_HIGH_MS = 100;
240 
242  private const GENERATION_HIGH_SEC = 0.2;
244  private const GENERATION_SLOW_SEC = 3.0;
245 
247  private const PURGE_TIME = 0;
249  private const PURGE_HOLDOFF = 1;
250 
252  private const VERSION = 1;
253 
255  public const KEY_VERSION = 'version';
257  public const KEY_AS_OF = 'asOf';
259  public const KEY_TTL = 'ttl';
261  public const KEY_CUR_TTL = 'curTTL';
263  public const KEY_TOMB_AS_OF = 'tombAsOf';
265  public const KEY_CHECK_AS_OF = 'lastCKPurge';
266 
268  private const RES_VALUE = 0;
270  private const RES_VERSION = 1;
272  private const RES_AS_OF = 2;
274  private const RES_TTL = 3;
276  private const RES_TOMB_AS_OF = 4;
278  private const RES_CHECK_AS_OF = 5;
280  private const RES_TOUCH_AS_OF = 6;
282  private const RES_CUR_TTL = 7;
283 
285  private const FLD_FORMAT_VERSION = 0;
287  private const FLD_VALUE = 1;
289  private const FLD_TTL = 2;
291  private const FLD_TIME = 3;
293  private const FLD_FLAGS = 4;
295  private const FLD_VALUE_VERSION = 5;
297  private const FLD_GENERATION_TIME = 6;
298 
300  private const TYPE_VALUE = 'v';
302  private const TYPE_TIMESTAMP = 't';
304  private const TYPE_MUTEX = 'm';
306  private const TYPE_INTERIM = 'i';
308  private const TYPE_COOLOFF = 'c';
309 
311  private const PURGE_VAL_PREFIX = 'PURGED';
312 
345  public function __construct( array $params ) {
346  $this->cache = $params['cache'];
347  $this->broadcastRoute = $params['broadcastRoutingPrefix'] ?? null;
348  $this->epoch = $params['epoch'] ?? 0;
349  $this->secret = $params['secret'] ?? (string)$this->epoch;
350  if ( ( $params['coalesceScheme'] ?? '' ) === 'hash_tag' ) {
351  // https://redis.io/topics/cluster-spec
352  // https://github.com/twitter/twemproxy/blob/v0.4.1/notes/recommendation.md#hash-tags
353  // https://github.com/Netflix/dynomite/blob/v0.7.0/notes/recommendation.md#hash-tags
354  $this->coalesceScheme = self::SCHEME_HASH_TAG;
355  } else {
356  // https://github.com/facebook/mcrouter/wiki/Key-syntax
357  $this->coalesceScheme = self::SCHEME_HASH_STOP;
358  }
359 
360  $this->keyHighQps = $params['keyHighQps'] ?? 100;
361  $this->keyHighUplinkBps = $params['keyHighUplinkBps'] ?? ( 1e9 / 8 / 100 );
362 
363  $this->setLogger( $params['logger'] ?? new NullLogger() );
364  $this->stats = $params['stats'] ?? new NullStatsdDataFactory();
365  $this->asyncHandler = $params['asyncHandler'] ?? null;
366 
367  $this->missLog = array_fill( 0, 10, [ '', 0.0 ] );
368 
369  $this->cache->registerWrapperInfoForStats(
370  'WANCache',
371  'wanobjectcache',
372  [ __CLASS__, 'getCollectionFromSisterKey' ]
373  );
374  }
375 
379  public function setLogger( LoggerInterface $logger ) {
380  $this->logger = $logger;
381  }
382 
388  public static function newEmpty() {
389  return new static( [ 'cache' => new EmptyBagOStuff() ] );
390  }
391 
447  final public function get( $key, &$curTTL = null, array $checkKeys = [], &$info = [] ) {
448  // Note that an undeclared variable passed as $info starts as null (not the default).
449  // Also, if no $info parameter is provided, then it doesn't matter how it changes here.
450  $legacyInfo = ( $info !== self::PASS_BY_REF );
451 
452  $res = $this->fetchKeys( [ $key ], $checkKeys )[$key];
453 
454  $curTTL = $res[self::RES_CUR_TTL];
455  $info = $legacyInfo
457  : [
458  self::KEY_VERSION => $res[self::RES_VERSION],
459  self::KEY_AS_OF => $res[self::RES_AS_OF],
460  self::KEY_TTL => $res[self::RES_TTL],
461  self::KEY_CUR_TTL => $res[self::RES_CUR_TTL],
462  self::KEY_TOMB_AS_OF => $res[self::RES_TOMB_AS_OF],
463  self::KEY_CHECK_AS_OF => $res[self::RES_CHECK_AS_OF]
464  ];
465 
466  if ( $curTTL === null || $curTTL <= 0 ) {
467  // Log the timestamp in case a corresponding set() call does not provide "walltime"
468  reset( $this->missLog );
469  unset( $this->missLog[key( $this->missLog )] );
470  $this->missLog[] = [ $key, $this->getCurrentTime() ];
471  }
472 
473  return $res[self::RES_VALUE];
474  }
475 
500  final public function getMulti(
501  array $keys,
502  &$curTTLs = [],
503  array $checkKeys = [],
504  &$info = []
505  ) {
506  // Note that an undeclared variable passed as $info starts as null (not the default).
507  // Also, if no $info parameter is provided, then it doesn't matter how it changes here.
508  $legacyInfo = ( $info !== self::PASS_BY_REF );
509 
510  $curTTLs = [];
511  $info = [];
512  $valuesByKey = [];
513 
514  $resByKey = $this->fetchKeys( $keys, $checkKeys );
515  foreach ( $resByKey as $key => $res ) {
516  if ( $res[self::RES_VALUE] !== false ) {
517  $valuesByKey[$key] = $res[self::RES_VALUE];
518  }
519 
520  if ( $res[self::RES_CUR_TTL] !== null ) {
521  $curTTLs[$key] = $res[self::RES_CUR_TTL];
522  }
523  $info[$key] = $legacyInfo
525  : [
526  self::KEY_VERSION => $res[self::RES_VERSION],
527  self::KEY_AS_OF => $res[self::RES_AS_OF],
528  self::KEY_TTL => $res[self::RES_TTL],
529  self::KEY_CUR_TTL => $res[self::RES_CUR_TTL],
530  self::KEY_TOMB_AS_OF => $res[self::RES_TOMB_AS_OF],
531  self::KEY_CHECK_AS_OF => $res[self::RES_CHECK_AS_OF]
532  ];
533  }
534 
535  return $valuesByKey;
536  }
537 
552  protected function fetchKeys( array $keys, array $checkKeys, $touchedCb = null ) {
553  $resByKey = [];
554 
555  // List of all sister keys that need to be fetched from cache
556  $allSisterKeys = [];
557  // Order-corresponding value sister key list for the base key list ($keys)
558  $valueSisterKeys = [];
559  // List of "check" sister keys to compare all value sister keys against
560  $checkSisterKeysForAll = [];
561  // Map of (base key => additional "check" sister key(s) to compare against)
562  $checkSisterKeysByKey = [];
563 
564  foreach ( $keys as $key ) {
565  $sisterKey = $this->makeSisterKey( $key, self::TYPE_VALUE );
566  $allSisterKeys[] = $sisterKey;
567  $valueSisterKeys[] = $sisterKey;
568  }
569 
570  foreach ( $checkKeys as $i => $checkKeyOrKeyGroup ) {
571  // Note: avoid array_merge() inside loop in case there are many keys
572  if ( is_int( $i ) ) {
573  // Single "check" key that applies to all base keys
574  $sisterKey = $this->makeSisterKey( $checkKeyOrKeyGroup, self::TYPE_TIMESTAMP );
575  $allSisterKeys[] = $sisterKey;
576  $checkSisterKeysForAll[] = $sisterKey;
577  } else {
578  // List of "check" keys that apply to a specific base key
579  foreach ( (array)$checkKeyOrKeyGroup as $checkKey ) {
580  $sisterKey = $this->makeSisterKey( $checkKey, self::TYPE_TIMESTAMP );
581  $allSisterKeys[] = $sisterKey;
582  $checkSisterKeysByKey[$i][] = $sisterKey;
583  }
584  }
585  }
586 
587  if ( $this->warmupCache ) {
588  // Get the wrapped values of the sister keys from the warmup cache
589  $wrappedBySisterKey = $this->warmupCache;
590  $sisterKeysMissing = array_diff( $allSisterKeys, array_keys( $wrappedBySisterKey ) );
591  if ( $sisterKeysMissing ) {
592  $this->warmupKeyMisses += count( $sisterKeysMissing );
593  $wrappedBySisterKey += $this->cache->getMulti( $sisterKeysMissing );
594  }
595  } else {
596  // Fetch the wrapped values of the sister keys from the backend
597  $wrappedBySisterKey = $this->cache->getMulti( $allSisterKeys );
598  }
599 
600  // Pessimistically treat the "current time" as the time when any network I/O finished
601  $now = $this->getCurrentTime();
602 
603  // List of "check" sister key purge timestamps to compare all value sister keys against
604  $ckPurgesForAll = $this->processCheckKeys(
605  $checkSisterKeysForAll,
606  $wrappedBySisterKey,
607  $now
608  );
609  // Map of (base key => extra "check" sister key purge timestamp(s) to compare against)
610  $ckPurgesByKey = [];
611  foreach ( $checkSisterKeysByKey as $keyWithCheckKeys => $checkKeysForKey ) {
612  $ckPurgesByKey[$keyWithCheckKeys] = $this->processCheckKeys(
613  $checkKeysForKey,
614  $wrappedBySisterKey,
615  $now
616  );
617  }
618 
619  // Unwrap and validate any value found for each base key (under the value sister key)
620  reset( $keys );
621  foreach ( $valueSisterKeys as $valueSisterKey ) {
622  // Get the corresponding base key for this value sister key
623  $key = current( $keys );
624  next( $keys );
625 
626  if ( array_key_exists( $valueSisterKey, $wrappedBySisterKey ) ) {
627  // Key exists as either a live value or tombstone value
628  $wrapped = $wrappedBySisterKey[$valueSisterKey];
629  } else {
630  // Key does not exist
631  $wrapped = false;
632  }
633 
634  $res = $this->unwrap( $wrapped, $now );
635  $value = $res[self::RES_VALUE];
636 
637  foreach ( array_merge( $ckPurgesForAll, $ckPurgesByKey[$key] ?? [] ) as $ckPurge ) {
639  $ckPurge[self::PURGE_TIME],
640  $res[self::RES_CHECK_AS_OF]
641  );
642  // Timestamp marking the end of the hold-off period for this purge
643  $holdoffDeadline = $ckPurge[self::PURGE_TIME] + $ckPurge[self::PURGE_HOLDOFF];
644  // Check if the value was generated during the hold-off period
645  if ( $value !== false && $holdoffDeadline >= $res[self::RES_AS_OF] ) {
646  // How long ago this value was purged by *this* "check" key
647  $ago = min( $ckPurge[self::PURGE_TIME] - $now, self::TINY_NEGATIVE );
648  // How long ago this value was purged by *any* known "check" key
649  $res[self::RES_CUR_TTL] = min( $res[self::RES_CUR_TTL], $ago );
650  }
651  }
652 
653  if ( $touchedCb !== null && $value !== false ) {
654  $touched = $touchedCb( $value );
655  if ( $touched !== null && $touched >= $res[self::RES_AS_OF] ) {
656  $res[self::RES_CUR_TTL] = min(
657  $res[self::RES_CUR_TTL],
658  $res[self::RES_AS_OF] - $touched,
659  self::TINY_NEGATIVE
660  );
661  }
662  } else {
663  $touched = null;
664  }
665 
666  $res[self::RES_TOUCH_AS_OF] = max( $res[self::RES_TOUCH_AS_OF], $touched );
667 
668  $resByKey[$key] = $res;
669  }
670 
671  return $resByKey;
672  }
673 
680  private function processCheckKeys(
681  array $checkSisterKeys,
682  array $wrappedBySisterKey,
683  float $now
684  ) {
685  $purges = [];
686 
687  foreach ( $checkSisterKeys as $timeKey ) {
688  $purge = isset( $wrappedBySisterKey[$timeKey] )
689  ? $this->parsePurgeValue( $wrappedBySisterKey[$timeKey] )
690  : null;
691 
692  if ( $purge === null ) {
693  $wrapped = $this->makeCheckPurgeValue( $now, self::HOLDOFF_TTL, $purge );
694  $this->cache->add( $timeKey, $wrapped, self::CHECK_KEY_TTL );
695  }
696 
697  $purges[] = $purge;
698  }
699 
700  return $purges;
701  }
702 
783  final public function set( $key, $value, $ttl = self::TTL_INDEFINITE, array $opts = [] ) {
784  $now = $this->getCurrentTime();
785  $dataReplicaLag = $opts['lag'] ?? 0;
786  $dataSnapshotLag = isset( $opts['since'] ) ? max( 0, $now - $opts['since'] ) : 0;
787  $dataCombinedLag = $dataReplicaLag + $dataSnapshotLag;
788  $dataPendingCommit = $opts['pending'] ?? null;
789  $lockTSE = $opts['lockTSE'] ?? self::TSE_NONE;
790  $staleTTL = $opts['staleTTL'] ?? self::STALE_TTL_NONE;
791  $creating = $opts['creating'] ?? false;
792  $version = $opts['version'] ?? null;
793  $walltime = $opts['walltime'] ?? $this->timeSinceLoggedMiss( $key, $now );
794 
795  if ( $ttl < 0 ) {
796  return true; // not cacheable
797  }
798 
799  // Forbid caching data that only exists within an uncommitted transaction. Also, lower
800  // the TTL when the data has a "since" time so far in the past that a delete() tombstone,
801  // made after that time, could have already expired (the key is no longer write-holed).
802  // The mitigation TTL depends on whether this data lag is assumed to systemically effect
803  // regeneration attempts in the near future. The TTL also reflects regeneration wall time.
804  if ( $dataPendingCommit ) {
805  // Case A: data comes from an uncommitted write transaction
806  $mitigated = 'pending writes';
807  // Data might never be committed; rely on a less problematic regeneration attempt
808  $mitigationTTL = self::TTL_UNCACHEABLE;
809  } elseif ( $dataSnapshotLag > self::MAX_READ_LAG ) {
810  // Case B: high snapshot lag
811  $pregenSnapshotLag = ( $walltime !== null ) ? ( $dataSnapshotLag - $walltime ) : 0;
812  if ( ( $pregenSnapshotLag + self::GENERATION_HIGH_SEC ) > self::MAX_READ_LAG ) {
813  // Case B1: generation started when transaction duration was already long
814  $mitigated = 'snapshot lag (late generation)';
815  // Probably non-systemic; rely on a less problematic regeneration attempt
816  $mitigationTTL = self::TTL_UNCACHEABLE;
817  } else {
818  // Case B2: slow generation made transaction duration long
819  $mitigated = 'snapshot lag (high generation time)';
820  // Probably systemic; use a low TTL to avoid stampedes/uncacheability
821  $mitigationTTL = self::LOW_TTL;
822  }
823  } elseif ( $dataReplicaLag === false || $dataReplicaLag > self::MAX_READ_LAG ) {
824  // Case C: low/medium snapshot lag with high replication lag
825  $mitigated = 'replication lag';
826  // Probably systemic; use a low TTL to avoid stampedes/uncacheability
827  $mitigationTTL = self::TTL_LAGGED;
828  } elseif ( $dataCombinedLag > self::MAX_READ_LAG ) {
829  $pregenCombinedLag = ( $walltime !== null ) ? ( $dataCombinedLag - $walltime ) : 0;
830  // Case D: medium snapshot lag with medium replication lag
831  if ( ( $pregenCombinedLag + self::GENERATION_HIGH_SEC ) > self::MAX_READ_LAG ) {
832  // Case D1: generation started when read lag was too high
833  $mitigated = 'read lag (late generation)';
834  // Probably non-systemic; rely on a less problematic regeneration attempt
835  $mitigationTTL = self::TTL_UNCACHEABLE;
836  } else {
837  // Case D2: slow generation made read lag too high
838  $mitigated = 'read lag (high generation time)';
839  // Probably systemic; use a low TTL to avoid stampedes/uncacheability
840  $mitigationTTL = self::LOW_TTL;
841  }
842  } else {
843  // Case E: new value generated with recent data
844  $mitigated = null;
845  // Nothing to mitigate
846  $mitigationTTL = null;
847  }
848 
849  if ( $mitigationTTL === self::TTL_UNCACHEABLE ) {
850  $this->logger->warning(
851  "Rejected set() for {cachekey} due to $mitigated.",
852  [
853  'cachekey' => $key,
854  'lag' => $dataReplicaLag,
855  'age' => $dataSnapshotLag,
856  'walltime' => $walltime
857  ]
858  );
859 
860  return true; // no-op the write for being unsafe
861  }
862 
863  // TTL to use in staleness checks (does not effect persistence layer TTL)
864  $logicalTTL = null;
865 
866  if ( $mitigationTTL !== null ) {
867  // New value was generated from data that is old enough to be risky
868  if ( $lockTSE >= 0 ) {
869  // Persist the value as long as normal, but make it count as stale sooner
870  $logicalTTL = min( $ttl ?: INF, $mitigationTTL );
871  } else {
872  // Persist the value for a shorter duration
873  $ttl = min( $ttl ?: INF, $mitigationTTL );
874  }
875 
876  $this->logger->warning(
877  "Lowered set() TTL for {cachekey} due to $mitigated.",
878  [
879  'cachekey' => $key,
880  'lag' => $dataReplicaLag,
881  'age' => $dataSnapshotLag,
882  'walltime' => $walltime
883  ]
884  );
885  }
886 
887  // Wrap that value with time/TTL/version metadata
888  $wrapped = $this->wrap( $value, $logicalTTL ?: $ttl, $version, $now, $walltime );
889  $storeTTL = $ttl + $staleTTL;
890 
891  if ( $creating ) {
892  $ok = $this->cache->add(
893  $this->makeSisterKey( $key, self::TYPE_VALUE ),
894  $wrapped,
895  $storeTTL
896  );
897  } else {
898  $ok = $this->cache->merge(
899  $this->makeSisterKey( $key, self::TYPE_VALUE ),
900  static function ( $cache, $key, $cWrapped ) use ( $wrapped ) {
901  // A string value means that it is a tombstone; do nothing in that case
902  return ( is_string( $cWrapped ) ) ? false : $wrapped;
903  },
904  $storeTTL,
905  1 // 1 attempt
906  );
907  }
908 
909  return $ok;
910  }
911 
974  final public function delete( $key, $ttl = self::HOLDOFF_TTL ) {
975  // Purge values must be stored under the value key so that WANObjectCache::set()
976  // can atomically merge values without accidentally undoing a recent purge and thus
977  // violating the holdoff TTL restriction.
978  $valueSisterKey = $this->makeSisterKey( $key, self::TYPE_VALUE );
979 
980  if ( $ttl <= 0 ) {
981  // A client or cache cleanup script is requesting a cache purge, so there is no
982  // volatility period due to replica DB lag. Any recent change to an entity cached
983  // in this key should have triggered an appropriate purge event.
984  $ok = $this->relayNonVolatilePurge( $valueSisterKey );
985  } else {
986  // A cacheable entity recently changed, so there might be a volatility period due
987  // to replica DB lag. Clients usually expect their actions to be reflected in any
988  // of their subsequent web request. This is attainable if (a) purge relay lag is
989  // lower than the time it takes for subsequent request by the client to arrive,
990  // and, (b) DB replica queries have "read-your-writes" consistency due to DB lag
991  // mitigation systems.
992  $now = $this->getCurrentTime();
993  // Set the key to the purge value in all datacenters
994  $purgeBySisterKey = [ $valueSisterKey => $this->makeTombstonePurgeValue( $now ) ];
995  $ok = $this->relayVolatilePurges( $purgeBySisterKey, $ttl );
996  }
997 
998  $kClass = $this->determineKeyClassForStats( $key );
999  $this->stats->increment( "wanobjectcache.$kClass.delete." . ( $ok ? 'ok' : 'error' ) );
1000 
1001  return $ok;
1002  }
1003 
1023  final public function getCheckKeyTime( $key ) {
1024  return $this->getMultiCheckKeyTime( [ $key ] )[$key];
1025  }
1026 
1088  final public function getMultiCheckKeyTime( array $keys ) {
1089  $checkSisterKeysByKey = [];
1090  foreach ( $keys as $key ) {
1091  $checkSisterKeysByKey[$key] = $this->makeSisterKey( $key, self::TYPE_TIMESTAMP );
1092  }
1093 
1094  $wrappedBySisterKey = $this->cache->getMulti( $checkSisterKeysByKey );
1095  $wrappedBySisterKey += array_fill_keys( $checkSisterKeysByKey, false );
1096 
1097  $now = $this->getCurrentTime();
1098  $times = [];
1099  foreach ( $checkSisterKeysByKey as $key => $checkSisterKey ) {
1100  $purge = $this->parsePurgeValue( $wrappedBySisterKey[$checkSisterKey] );
1101  if ( $purge === null ) {
1102  $wrapped = $this->makeCheckPurgeValue( $now, self::HOLDOFF_TTL, $purge );
1103  $this->cache->add( $checkSisterKey, $wrapped, self::CHECK_KEY_TTL );
1104  }
1105 
1106  $times[$key] = $purge[self::PURGE_TIME];
1107  }
1108 
1109  return $times;
1110  }
1111 
1145  final public function touchCheckKey( $key, $holdoff = self::HOLDOFF_TTL ) {
1146  $checkSisterKey = $this->makeSisterKey( $key, self::TYPE_TIMESTAMP );
1147 
1148  $now = $this->getCurrentTime();
1149  $purgeBySisterKey = [ $checkSisterKey => $this->makeCheckPurgeValue( $now, $holdoff ) ];
1150  $ok = $this->relayVolatilePurges( $purgeBySisterKey, self::CHECK_KEY_TTL );
1151 
1152  $kClass = $this->determineKeyClassForStats( $key );
1153  $this->stats->increment( "wanobjectcache.$kClass.ck_touch." . ( $ok ? 'ok' : 'error' ) );
1154 
1155  return $ok;
1156  }
1157 
1185  final public function resetCheckKey( $key ) {
1186  $checkSisterKey = $this->makeSisterKey( $key, self::TYPE_TIMESTAMP );
1187  $ok = $this->relayNonVolatilePurge( $checkSisterKey );
1188 
1189  $kClass = $this->determineKeyClassForStats( $key );
1190  $this->stats->increment( "wanobjectcache.$kClass.ck_reset." . ( $ok ? 'ok' : 'error' ) );
1191 
1192  return $ok;
1193  }
1194 
1498  final public function getWithSetCallback(
1499  $key, $ttl, $callback, array $opts = [], array $cbParams = []
1500  ) {
1501  $version = $opts['version'] ?? null;
1502  $pcTTL = $opts['pcTTL'] ?? self::TTL_UNCACHEABLE;
1503  $pCache = ( $pcTTL >= 0 )
1504  ? $this->getProcessCache( $opts['pcGroup'] ?? self::PC_PRIMARY )
1505  : null;
1506 
1507  // Use the process cache if requested as long as no outer cache callback is running.
1508  // Nested callback process cache use is not lag-safe with regard to HOLDOFF_TTL since
1509  // process cached values are more lagged than persistent ones as they are not purged.
1510  if ( $pCache && $this->callbackDepth == 0 ) {
1511  $cached = $pCache->get( $key, $pcTTL, false );
1512  if ( $cached !== false ) {
1513  $this->logger->debug( "getWithSetCallback($key): process cache hit" );
1514  return $cached;
1515  }
1516  }
1517 
1518  $res = $this->fetchOrRegenerate( $key, $ttl, $callback, $opts, $cbParams );
1519  list( $value, $valueVersion, $curAsOf ) = $res;
1520  if ( $valueVersion !== $version ) {
1521  // Current value has a different version; use the variant key for this version.
1522  // Regenerate the variant value if it is not newer than the main value at $key
1523  // so that purges to the main key propagate to the variant value.
1524  $this->logger->debug( "getWithSetCallback($key): using variant key" );
1525  list( $value ) = $this->fetchOrRegenerate(
1526  $this->makeGlobalKey( 'WANCache-key-variant', md5( $key ), $version ),
1527  $ttl,
1528  $callback,
1529  [ 'version' => null, 'minAsOf' => $curAsOf ] + $opts,
1530  $cbParams
1531  );
1532  }
1533 
1534  // Update the process cache if enabled
1535  if ( $pCache && $value !== false ) {
1536  $pCache->set( $key, $value );
1537  }
1538 
1539  return $value;
1540  }
1541 
1558  private function fetchOrRegenerate( $key, $ttl, $callback, array $opts, array $cbParams ) {
1559  $checkKeys = $opts['checkKeys'] ?? [];
1560  $graceTTL = $opts['graceTTL'] ?? self::GRACE_TTL_NONE;
1561  $minAsOf = $opts['minAsOf'] ?? self::MIN_TIMESTAMP_NONE;
1562  $hotTTR = $opts['hotTTR'] ?? self::HOT_TTR;
1563  $lowTTL = $opts['lowTTL'] ?? min( self::LOW_TTL, $ttl );
1564  $ageNew = $opts['ageNew'] ?? self::AGE_NEW;
1565  $touchedCb = $opts['touchedCallback'] ?? null;
1566  $startTime = $this->getCurrentTime();
1567 
1568  $kClass = $this->determineKeyClassForStats( $key );
1569 
1570  // Get the current key value and its metadata
1571  $curState = $this->fetchKeys( [ $key ], $checkKeys, $touchedCb )[$key];
1572  $curValue = $curState[self::RES_VALUE];
1573  // Use the cached value if it exists and is not due for synchronous regeneration
1574  if ( $this->isAcceptablyFreshValue( $curState, $graceTTL, $minAsOf ) ) {
1575  if ( !$this->isLotteryRefreshDue( $curState, $lowTTL, $ageNew, $hotTTR, $startTime ) ) {
1576  $this->stats->timing(
1577  "wanobjectcache.$kClass.hit.good",
1578  1e3 * ( $this->getCurrentTime() - $startTime )
1579  );
1580 
1581  return [ $curValue, $curState[self::RES_VERSION], $curState[self::RES_AS_OF] ];
1582  } elseif ( $this->scheduleAsyncRefresh( $key, $ttl, $callback, $opts, $cbParams ) ) {
1583  $this->logger->debug( "fetchOrRegenerate($key): hit with async refresh" );
1584  $this->stats->timing(
1585  "wanobjectcache.$kClass.hit.refresh",
1586  1e3 * ( $this->getCurrentTime() - $startTime )
1587  );
1588 
1589  return [ $curValue, $curState[self::RES_VERSION], $curState[self::RES_AS_OF] ];
1590  } else {
1591  $this->logger->debug( "fetchOrRegenerate($key): hit with sync refresh" );
1592  }
1593  }
1594 
1595  $isKeyTombstoned = ( $curState[self::RES_TOMB_AS_OF] !== null );
1596  // Use the interim key as an temporary alternative if the key is tombstoned
1597  if ( $isKeyTombstoned ) {
1598  $volState = $this->getInterimValue( $key, $minAsOf, $startTime, $touchedCb );
1599  $volValue = $volState[self::RES_VALUE];
1600  } else {
1601  $volState = $curState;
1602  $volValue = $curValue;
1603  }
1604 
1605  // During the volatile "hold-off" period that follows a purge of the key, the value
1606  // will be regenerated many times if frequently accessed. This is done to mitigate
1607  // the effects of backend replication lag as soon as possible. However, throttle the
1608  // overhead of locking and regeneration by reusing values recently written to cache
1609  // tens of milliseconds ago. Verify the "as of" time against the last purge event.
1610  $lastPurgeTime = max(
1611  // RES_TOUCH_AS_OF depends on the value (possibly from the interim key)
1612  $volState[self::RES_TOUCH_AS_OF],
1613  $curState[self::RES_TOMB_AS_OF],
1614  $curState[self::RES_CHECK_AS_OF]
1615  );
1616  $safeMinAsOf = max( $minAsOf, $lastPurgeTime + self::TINY_POSTIVE );
1617  if ( $this->isExtremelyNewValue( $volState, $safeMinAsOf, $startTime ) ) {
1618  $this->logger->debug( "fetchOrRegenerate($key): volatile hit" );
1619  $this->stats->timing(
1620  "wanobjectcache.$kClass.hit.volatile",
1621  1e3 * ( $this->getCurrentTime() - $startTime )
1622  );
1623 
1624  return [ $volValue, $volState[self::RES_VERSION], $curState[self::RES_AS_OF] ];
1625  }
1626 
1627  $lockTSE = $opts['lockTSE'] ?? self::TSE_NONE;
1628  $busyValue = $opts['busyValue'] ?? null;
1629  $staleTTL = $opts['staleTTL'] ?? self::STALE_TTL_NONE;
1630  $version = $opts['version'] ?? null;
1631 
1632  // Determine whether one thread per datacenter should handle regeneration at a time
1633  $useRegenerationLock =
1634  // Note that since tombstones no-op set(), $lockTSE and $curTTL cannot be used to
1635  // deduce the key hotness because |$curTTL| will always keep increasing until the
1636  // tombstone expires or is overwritten by a new tombstone. Also, even if $lockTSE
1637  // is not set, constant regeneration of a key for the tombstone lifetime might be
1638  // very expensive. Assume tombstoned keys are possibly hot in order to reduce
1639  // the risk of high regeneration load after the delete() method is called.
1640  $isKeyTombstoned ||
1641  // Assume a key is hot if requested soon ($lockTSE seconds) after purge.
1642  // This avoids stampedes when timestamps from $checkKeys/$touchedCb bump.
1643  (
1644  $curState[self::RES_CUR_TTL] !== null &&
1645  $curState[self::RES_CUR_TTL] <= 0 &&
1646  abs( $curState[self::RES_CUR_TTL] ) <= $lockTSE
1647  ) ||
1648  // Assume a key is hot if there is no value and a busy fallback is given.
1649  // This avoids stampedes on eviction or preemptive regeneration taking too long.
1650  ( $busyValue !== null && $volValue === false );
1651 
1652  // If a regeneration lock is required, threads that do not get the lock will try to use
1653  // the stale value, the interim value, or the $busyValue placeholder, in that order. If
1654  // none of those are set then all threads will bypass the lock and regenerate the value.
1655  $hasLock = $useRegenerationLock && $this->claimStampedeLock( $key );
1656  if ( $useRegenerationLock && !$hasLock ) {
1657  // Determine if there is stale or volatile cached value that is still usable
1658  if ( $this->isValid( $volValue, $volState[self::RES_AS_OF], $minAsOf ) ) {
1659  $this->logger->debug( "fetchOrRegenerate($key): returning stale value" );
1660  $this->stats->timing(
1661  "wanobjectcache.$kClass.hit.stale",
1662  1e3 * ( $this->getCurrentTime() - $startTime )
1663  );
1664 
1665  return [ $volValue, $volState[self::RES_VERSION], $curState[self::RES_AS_OF] ];
1666  } elseif ( $busyValue !== null ) {
1667  $miss = is_infinite( $minAsOf ) ? 'renew' : 'miss';
1668  $this->logger->debug( "fetchOrRegenerate($key): busy $miss" );
1669  $this->stats->timing(
1670  "wanobjectcache.$kClass.$miss.busy",
1671  1e3 * ( $this->getCurrentTime() - $startTime )
1672  );
1673  $placeholderValue = $this->resolveBusyValue( $busyValue );
1674 
1675  return [ $placeholderValue, $version, $curState[self::RES_AS_OF] ];
1676  }
1677  }
1678 
1679  // Generate the new value given any prior value with a matching version
1680  $setOpts = [];
1681  $preCallbackTime = $this->getCurrentTime();
1683  try {
1684  $value = $callback(
1685  ( $curState[self::RES_VERSION] === $version ) ? $curValue : false,
1686  $ttl,
1687  $setOpts,
1688  ( $curState[self::RES_VERSION] === $version ) ? $curState[self::RES_AS_OF] : null,
1689  $cbParams
1690  );
1691  } finally {
1693  }
1694  $postCallbackTime = $this->getCurrentTime();
1695 
1696  // How long it took to fetch, validate, and generate the value
1697  $elapsed = max( $postCallbackTime - $startTime, 0.0 );
1698 
1699  // How long it took to generate the value
1700  $walltime = max( $postCallbackTime - $preCallbackTime, 0.0 );
1701  $this->stats->timing( "wanobjectcache.$kClass.regen_walltime", 1e3 * $walltime );
1702 
1703  // Attempt to save the newly generated value if applicable
1704  if (
1705  // Callback yielded a cacheable value
1706  ( $value !== false && $ttl >= 0 ) &&
1707  // Current thread was not raced out of a regeneration lock or key is tombstoned
1708  ( !$useRegenerationLock || $hasLock || $isKeyTombstoned ) &&
1709  // Key does not appear to be undergoing a set() stampede
1710  $this->checkAndSetCooloff( $key, $kClass, $value, $elapsed, $hasLock )
1711  ) {
1712  // If the key is write-holed then use the (volatile) interim key as an alternative
1713  if ( $isKeyTombstoned ) {
1714  $this->setInterimValue( $key, $value, $lockTSE, $version, $postCallbackTime, $walltime );
1715  } else {
1716  $finalSetOpts = [
1717  // @phan-suppress-next-line PhanUselessBinaryAddRight,PhanCoalescingAlwaysNull
1718  'since' => $setOpts['since'] ?? $preCallbackTime,
1719  'version' => $version,
1720  'staleTTL' => $staleTTL,
1721  'lockTSE' => $lockTSE, // informs lag vs performance trade-offs
1722  'creating' => ( $curValue === false ), // optimization
1723  'walltime' => $walltime
1724  ] + $setOpts;
1725  $this->set( $key, $value, $ttl, $finalSetOpts );
1726  }
1727  }
1728 
1729  $this->yieldStampedeLock( $key, $hasLock );
1730 
1731  $miss = is_infinite( $minAsOf ) ? 'renew' : 'miss';
1732  $this->logger->debug( "fetchOrRegenerate($key): $miss, new value computed" );
1733  $this->stats->timing(
1734  "wanobjectcache.$kClass.$miss.compute",
1735  1e3 * ( $this->getCurrentTime() - $startTime )
1736  );
1737 
1738  return [ $value, $version, $curState[self::RES_AS_OF] ];
1739  }
1740 
1745  private function claimStampedeLock( $key ) {
1746  $checkSisterKey = $this->makeSisterKey( $key, self::TYPE_MUTEX );
1747  // Note that locking is not bypassed due to I/O errors; this avoids stampedes
1748  return $this->cache->add( $checkSisterKey, 1, self::LOCK_TTL );
1749  }
1750 
1755  private function yieldStampedeLock( $key, $hasLock ) {
1756  if ( $hasLock ) {
1757  $checkSisterKey = $this->makeSisterKey( $key, self::TYPE_MUTEX );
1758  $this->cache->changeTTL( $checkSisterKey, $this->getCurrentTime() - 60 );
1759  }
1760  }
1761 
1772  private function makeSisterKeys( array $baseKeys, string $type, string $route = null ) {
1773  $sisterKeys = [];
1774  foreach ( $baseKeys as $baseKey ) {
1775  $sisterKeys[] = $this->makeSisterKey( $baseKey, $type, $route );
1776  }
1777 
1778  return $sisterKeys;
1779  }
1780 
1791  private function makeSisterKey( string $baseKey, string $typeChar, string $route = null ) {
1792  if ( $this->coalesceScheme === self::SCHEME_HASH_STOP ) {
1793  // Key style: "WANCache:<base key>|#|<character>"
1794  $sisterKey = 'WANCache:' . $baseKey . '|#|' . $typeChar;
1795  } else {
1796  // Key style: "WANCache:{<base key>}:<character>"
1797  $sisterKey = 'WANCache:{' . $baseKey . '}:' . $typeChar;
1798  }
1799 
1800  if ( $route !== null ) {
1801  $sisterKey = $this->prependRoute( $sisterKey, $route );
1802  }
1803 
1804  return $sisterKey;
1805  }
1806 
1813  public static function getCollectionFromSisterKey( string $sisterKey ) {
1814  if ( substr( $sisterKey, -4 ) === '|#|v' ) {
1815  // Key style: "WANCache:<base key>|#|<character>"
1816  $collection = substr( $sisterKey, 9, strcspn( $sisterKey, ':|', 9 ) );
1817  } elseif ( substr( $sisterKey, -3 ) === '}:v' ) {
1818  // Key style: "WANCache:{<base key>}:<character>"
1819  $collection = substr( $sisterKey, 10, strcspn( $sisterKey, ':}', 10 ) );
1820  } else {
1821  $collection = 'internal';
1822  }
1823 
1824  return $collection;
1825  }
1826 
1839  private function isExtremelyNewValue( $res, $minAsOf, $now ) {
1840  if ( $res[self::RES_VALUE] === false || $res[self::RES_AS_OF] < $minAsOf ) {
1841  return false;
1842  }
1843 
1844  $age = $now - $res[self::RES_AS_OF];
1845 
1846  return ( $age < mt_rand( self::RECENT_SET_LOW_MS, self::RECENT_SET_HIGH_MS ) / 1e3 );
1847  }
1848 
1870  private function checkAndSetCooloff( $key, $kClass, $value, $elapsed, $hasLock ) {
1871  $valueSisterKey = $this->makeSisterKey( $key, self::TYPE_VALUE );
1872  list( $estimatedSize ) = $this->cache->setNewPreparedValues( [
1873  $valueSisterKey => $value
1874  ] );
1875 
1876  if ( !$hasLock ) {
1877  // Suppose that this cache key is very popular (KEY_HIGH_QPS reads/second).
1878  // After eviction, there will be cache misses until it gets regenerated and saved.
1879  // If the time window when the key is missing lasts less than one second, then the
1880  // number of misses will not reach KEY_HIGH_QPS. This window largely corresponds to
1881  // the key regeneration time. Estimate the count/rate of cache misses, e.g.:
1882  // - 100 QPS, 20ms regeneration => ~2 misses (< 1s)
1883  // - 100 QPS, 100ms regeneration => ~10 misses (< 1s)
1884  // - 100 QPS, 3000ms regeneration => ~300 misses (100/s for 3s)
1885  $missesPerSecForHighQPS = ( min( $elapsed, 1 ) * $this->keyHighQps );
1886 
1887  // Determine whether there is enough I/O stampede risk to justify throttling set().
1888  // Estimate unthrottled set() overhead, as bps, from miss count/rate and value size,
1889  // comparing it to the per-key uplink bps limit (KEY_HIGH_UPLINK_BPS), e.g.:
1890  // - 2 misses (< 1s), 10KB value, 1250000 bps limit => 160000 bits (low risk)
1891  // - 2 misses (< 1s), 100KB value, 1250000 bps limit => 1600000 bits (high risk)
1892  // - 10 misses (< 1s), 10KB value, 1250000 bps limit => 800000 bits (low risk)
1893  // - 10 misses (< 1s), 100KB value, 1250000 bps limit => 8000000 bits (high risk)
1894  // - 300 misses (100/s), 1KB value, 1250000 bps limit => 800000 bps (low risk)
1895  // - 300 misses (100/s), 10KB value, 1250000 bps limit => 8000000 bps (high risk)
1896  // - 300 misses (100/s), 100KB value, 1250000 bps limit => 80000000 bps (high risk)
1897  if ( ( $missesPerSecForHighQPS * $estimatedSize ) >= $this->keyHighUplinkBps ) {
1898  $cooloffSisterKey = $this->makeSisterKey( $key, self::TYPE_COOLOFF );
1899  $watchPoint = $this->cache->watchErrors();
1900  if (
1901  !$this->cache->add( $cooloffSisterKey, 1, self::COOLOFF_TTL ) &&
1902  // Don't treat failures due to I/O errors as the key being in cool-off
1903  $this->cache->getLastError( $watchPoint ) === self::ERR_NONE
1904  ) {
1905  $this->stats->increment( "wanobjectcache.$kClass.cooloff_bounce" );
1906 
1907  return false;
1908  }
1909  }
1910  }
1911 
1912  // Corresponding metrics for cache writes that actually get sent over the write
1913  $this->stats->timing( "wanobjectcache.$kClass.regen_set_delay", 1e3 * $elapsed );
1914  $this->stats->updateCount( "wanobjectcache.$kClass.regen_set_bytes", $estimatedSize );
1915 
1916  return true;
1917  }
1918 
1928  private function getInterimValue( $key, $minAsOf, $now, $touchedCb ) {
1929  if ( $this->useInterimHoldOffCaching ) {
1930  $interimSisterKey = $this->makeSisterKey( $key, self::TYPE_INTERIM );
1931  $wrapped = $this->cache->get( $interimSisterKey );
1932  $res = $this->unwrap( $wrapped, $now );
1933  if ( $res[self::RES_VALUE] !== false && $res[self::RES_AS_OF] >= $minAsOf ) {
1934  if ( $touchedCb !== null ) {
1935  // Update "last purge time" since the $touchedCb timestamp depends on $value
1936  // Get the new "touched timestamp", accounting for callback-checked dependencies
1937  $res[self::RES_TOUCH_AS_OF] = max(
1938  $touchedCb( $res[self::RES_VALUE] ),
1939  $res[self::RES_TOUCH_AS_OF]
1940  );
1941  }
1942 
1943  return $res;
1944  }
1945  }
1946 
1947  return $this->unwrap( false, $now );
1948  }
1949 
1958  private function setInterimValue( $key, $value, $ttl, $version, $now, $walltime ) {
1959  $ttl = max( self::INTERIM_KEY_TTL, (int)$ttl );
1960 
1961  $wrapped = $this->wrap( $value, $ttl, $version, $now, $walltime );
1962  $this->cache->merge(
1963  $this->makeSisterKey( $key, self::TYPE_INTERIM ),
1964  static function () use ( $wrapped ) {
1965  return $wrapped;
1966  },
1967  $ttl,
1968  1
1969  );
1970  }
1971 
1976  private function resolveBusyValue( $busyValue ) {
1977  return ( $busyValue instanceof Closure ) ? $busyValue() : $busyValue;
1978  }
1979 
2044  final public function getMultiWithSetCallback(
2045  ArrayIterator $keyedIds, $ttl, callable $callback, array $opts = []
2046  ) {
2047  // Batch load required keys into the in-process warmup cache
2048  $this->warmupCache = $this->fetchWrappedValuesForWarmupCache(
2049  $this->getNonProcessCachedMultiKeys( $keyedIds, $opts ),
2050  $opts['checkKeys'] ?? []
2051  );
2052  $this->warmupKeyMisses = 0;
2053 
2054  // The required callback signature includes $id as the first argument for convenience
2055  // to distinguish different items. To reuse the code in getWithSetCallback(), wrap the
2056  // callback with a proxy callback that has the standard getWithSetCallback() signature.
2057  // This is defined only once per batch to avoid closure creation overhead.
2058  $proxyCb = static function ( $oldValue, &$ttl, &$setOpts, $oldAsOf, $params )
2059  use ( $callback )
2060  {
2061  return $callback( $params['id'], $oldValue, $ttl, $setOpts, $oldAsOf );
2062  };
2063 
2064  $values = [];
2065  foreach ( $keyedIds as $key => $id ) { // preserve order
2066  $values[$key] = $this->getWithSetCallback(
2067  $key,
2068  $ttl,
2069  $proxyCb,
2070  $opts,
2071  [ 'id' => $id ]
2072  );
2073  }
2074 
2075  $this->warmupCache = [];
2076 
2077  return $values;
2078  }
2079 
2145  final public function getMultiWithUnionSetCallback(
2146  ArrayIterator $keyedIds, $ttl, callable $callback, array $opts = []
2147  ) {
2148  $checkKeys = $opts['checkKeys'] ?? [];
2149  $minAsOf = $opts['minAsOf'] ?? self::MIN_TIMESTAMP_NONE;
2150  unset( $opts['lockTSE'] ); // incompatible
2151  unset( $opts['busyValue'] ); // incompatible
2152 
2153  // Batch load required keys into the in-process warmup cache
2154  $keysByIdGet = $this->getNonProcessCachedMultiKeys( $keyedIds, $opts );
2155  $this->warmupCache = $this->fetchWrappedValuesForWarmupCache( $keysByIdGet, $checkKeys );
2156  $this->warmupKeyMisses = 0;
2157 
2158  // IDs of entities known to be in need of generation
2159  $idsRegen = [];
2160 
2161  // Find out which keys are missing/deleted/stale
2162  $resByKey = $this->fetchKeys( $keysByIdGet, $checkKeys );
2163  foreach ( $keysByIdGet as $id => $key ) {
2164  $res = $resByKey[$key];
2165  if (
2166  $res[self::RES_VALUE] === false ||
2167  $res[self::RES_CUR_TTL] < 0 ||
2168  $res[self::RES_AS_OF] < $minAsOf
2169  ) {
2170  $idsRegen[] = $id;
2171  }
2172  }
2173 
2174  // Run the callback to populate the generation value map for all required IDs
2175  $newSetOpts = [];
2176  $newTTLsById = array_fill_keys( $idsRegen, $ttl );
2177  $newValsById = $idsRegen ? $callback( $idsRegen, $newTTLsById, $newSetOpts ) : [];
2178 
2179  // The required callback signature includes $id as the first argument for convenience
2180  // to distinguish different items. To reuse the code in getWithSetCallback(), wrap the
2181  // callback with a proxy callback that has the standard getWithSetCallback() signature.
2182  // This is defined only once per batch to avoid closure creation overhead.
2183  $proxyCb = static function ( $oldValue, &$ttl, &$setOpts, $oldAsOf, $params )
2184  use ( $callback, $newValsById, $newTTLsById, $newSetOpts )
2185  {
2186  $id = $params['id'];
2187 
2188  if ( array_key_exists( $id, $newValsById ) ) {
2189  // Value was already regerated as expected, so use the value in $newValsById
2190  $newValue = $newValsById[$id];
2191  $ttl = $newTTLsById[$id];
2192  $setOpts = $newSetOpts;
2193  } else {
2194  // Pre-emptive/popularity refresh and version mismatch cases are not detected
2195  // above and thus $newValsById has no entry. Run $callback on this single entity.
2196  $ttls = [ $id => $ttl ];
2197  $newValue = $callback( [ $id ], $ttls, $setOpts )[$id];
2198  $ttl = $ttls[$id];
2199  }
2200 
2201  return $newValue;
2202  };
2203 
2204  // Run the cache-aside logic using warmupCache instead of persistent cache queries
2205  $values = [];
2206  foreach ( $keyedIds as $key => $id ) { // preserve order
2207  $values[$key] = $this->getWithSetCallback(
2208  $key,
2209  $ttl,
2210  $proxyCb,
2211  $opts,
2212  [ 'id' => $id ]
2213  );
2214  }
2215 
2216  $this->warmupCache = [];
2217 
2218  return $values;
2219  }
2220 
2233  final public function reap( $key, $purgeTimestamp, &$isStale = false ) {
2234  $valueSisterKey = $this->makeSisterKey( $key, self::TYPE_VALUE );
2235 
2236  $minAsOf = $purgeTimestamp + self::HOLDOFF_TTL;
2237  $wrapped = $this->cache->get( $valueSisterKey );
2238  if ( is_array( $wrapped ) && $wrapped[self::FLD_TIME] < $minAsOf ) {
2239  $isStale = true;
2240  $this->logger->warning( "Reaping stale value key '$key'." );
2241  $ttlReap = self::HOLDOFF_TTL; // avoids races with tombstone creation
2242  $ok = $this->cache->changeTTL( $valueSisterKey, $ttlReap );
2243  if ( !$ok ) {
2244  $this->logger->error( "Could not complete reap of key '$key'." );
2245  }
2246 
2247  return $ok;
2248  }
2249 
2250  $isStale = false;
2251 
2252  return true;
2253  }
2254 
2264  final public function reapCheckKey( $key, $purgeTimestamp, &$isStale = false ) {
2265  $checkSisterKey = $this->makeSisterKey( $key, self::TYPE_TIMESTAMP );
2266 
2267  $wrapped = $this->cache->get( $checkSisterKey );
2268  $purge = $this->parsePurgeValue( $wrapped );
2269  if ( $purge !== null && $purge[self::PURGE_TIME] < $purgeTimestamp ) {
2270  $isStale = true;
2271  $this->logger->warning( "Reaping stale check key '$key'." );
2272  $ok = $this->cache->changeTTL( $checkSisterKey, self::TTL_SECOND );
2273  if ( !$ok ) {
2274  $this->logger->error( "Could not complete reap of check key '$key'." );
2275  }
2276 
2277  return $ok;
2278  }
2279 
2280  $isStale = false;
2281 
2282  return false;
2283  }
2284 
2295  public function makeGlobalKey( $collection, ...$components ) {
2296  return $this->cache->makeGlobalKey( ...func_get_args() );
2297  }
2298 
2309  public function makeKey( $collection, ...$components ) {
2310  return $this->cache->makeKey( ...func_get_args() );
2311  }
2312 
2320  public function hash256( $component ) {
2321  return hash_hmac( 'sha256', $component, $this->secret );
2322  }
2323 
2374  final public function makeMultiKeys( array $ids, $keyCallback ) {
2375  $idByKey = [];
2376  foreach ( $ids as $id ) {
2377  // Discourage triggering of automatic makeKey() hashing in some backends
2378  if ( strlen( $id ) > 64 ) {
2379  $this->logger->warning( __METHOD__ . ": long ID '$id'; use hash256()" );
2380  }
2381  $key = $keyCallback( $id, $this );
2382  // Edge case: ignore key collisions due to duplicate $ids like "42" and 42
2383  if ( !isset( $idByKey[$key] ) ) {
2384  $idByKey[$key] = $id;
2385  } elseif ( (string)$id !== (string)$idByKey[$key] ) {
2386  throw new UnexpectedValueException(
2387  "Cache key collision; IDs ('$id','{$idByKey[$key]}') map to '$key'"
2388  );
2389  }
2390  }
2391 
2392  return new ArrayIterator( $idByKey );
2393  }
2394 
2430  final public function multiRemap( array $ids, array $res ) {
2431  if ( count( $ids ) !== count( $res ) ) {
2432  // If makeMultiKeys() is called on a list of non-unique IDs, then the resulting
2433  // ArrayIterator will have less entries due to "first appearance" de-duplication
2434  $ids = array_keys( array_fill_keys( $ids, true ) );
2435  if ( count( $ids ) !== count( $res ) ) {
2436  throw new UnexpectedValueException( "Multi-key result does not match ID list" );
2437  }
2438  }
2439 
2440  return array_combine( $ids, $res );
2441  }
2442 
2449  public function watchErrors() {
2450  return $this->cache->watchErrors();
2451  }
2452 
2470  final public function getLastError( $watchPoint = 0 ) {
2471  $code = $this->cache->getLastError( $watchPoint );
2472  switch ( $code ) {
2473  case self::ERR_NONE:
2474  return self::ERR_NONE;
2475  case self::ERR_NO_RESPONSE:
2476  return self::ERR_NO_RESPONSE;
2477  case self::ERR_UNREACHABLE:
2478  return self::ERR_UNREACHABLE;
2479  default:
2480  return self::ERR_UNEXPECTED;
2481  }
2482  }
2483 
2488  final public function clearLastError() {
2489  $this->cache->clearLastError();
2490  }
2491 
2497  public function clearProcessCache() {
2498  $this->processCaches = [];
2499  }
2500 
2521  final public function useInterimHoldOffCaching( $enabled ) {
2522  $this->useInterimHoldOffCaching = $enabled;
2523  }
2524 
2530  public function getQoS( $flag ) {
2531  return $this->cache->getQoS( $flag );
2532  }
2533 
2597  public function adaptiveTTL( $mtime, $maxTTL, $minTTL = 30, $factor = 0.2 ) {
2598  $mtime = (int)$mtime; // handle fractional seconds and string integers
2599  if ( $mtime <= 0 ) {
2600  return $minTTL; // no last-modified time provided
2601  }
2602 
2603  $age = (int)$this->getCurrentTime() - $mtime;
2604 
2605  return (int)min( $maxTTL, max( $minTTL, $factor * $age ) );
2606  }
2607 
2612  final public function getWarmupKeyMisses() {
2613  return $this->warmupKeyMisses;
2614  }
2615 
2630  protected function relayVolatilePurges( array $purgeBySisterKey, int $ttl ) {
2631  $purgeByRouteKey = [];
2632  foreach ( $purgeBySisterKey as $sisterKey => $purge ) {
2633  if ( $this->broadcastRoute !== null ) {
2634  $routeKey = $this->prependRoute( $sisterKey, $this->broadcastRoute );
2635  } else {
2636  $routeKey = $sisterKey;
2637  }
2638  $purgeByRouteKey[$routeKey] = $purge;
2639  }
2640 
2641  if ( count( $purgeByRouteKey ) == 1 ) {
2642  $purge = reset( $purgeByRouteKey );
2643  $ok = $this->cache->set( key( $purgeByRouteKey ), $purge, $ttl );
2644  } else {
2645  $ok = $this->cache->setMulti( $purgeByRouteKey, $ttl );
2646  }
2647 
2648  return $ok;
2649  }
2650 
2659  protected function relayNonVolatilePurge( string $sisterKey ) {
2660  if ( $this->broadcastRoute !== null ) {
2661  $routeKey = $this->prependRoute( $sisterKey, $this->broadcastRoute );
2662  } else {
2663  $routeKey = $sisterKey;
2664  }
2665 
2666  return $this->cache->delete( $routeKey );
2667  }
2668 
2674  protected function prependRoute( string $sisterKey, string $route ) {
2675  if ( $sisterKey[0] === '/' ) {
2676  throw new RuntimeException( "Sister key '$sisterKey' already contains a route." );
2677  }
2678 
2679  return $route . $sisterKey;
2680  }
2681 
2693  private function scheduleAsyncRefresh( $key, $ttl, $callback, array $opts, array $cbParams ) {
2694  if ( !$this->asyncHandler ) {
2695  return false;
2696  }
2697  // Update the cache value later, such during post-send of an HTTP request. This forces
2698  // cache regeneration by setting "minAsOf" to infinity, meaning that no existing value
2699  // is considered valid. Furthermore, note that preemptive regeneration is not applicable
2700  // to invalid values, so there is no risk of infinite preemptive regeneration loops.
2701  $func = $this->asyncHandler;
2702  $func( function () use ( $key, $ttl, $callback, $opts, $cbParams ) {
2703  $opts['minAsOf'] = INF;
2704  try {
2705  $this->fetchOrRegenerate( $key, $ttl, $callback, $opts, $cbParams );
2706  } catch ( Exception $e ) {
2707  // Log some context for easier debugging
2708  $this->logger->error( 'Async refresh failed for {key}', [
2709  'key' => $key,
2710  'ttl' => $ttl,
2711  'exception' => $e
2712  ] );
2713  throw $e;
2714  }
2715  } );
2716 
2717  return true;
2718  }
2719 
2728  private function isAcceptablyFreshValue( $res, $graceTTL, $minAsOf ) {
2729  if ( !$this->isValid( $res[self::RES_VALUE], $res[self::RES_AS_OF], $minAsOf ) ) {
2730  // Value does not exists or is too old
2731  return false;
2732  }
2733 
2734  $curTTL = $res[self::RES_CUR_TTL];
2735  if ( $curTTL > 0 ) {
2736  // Value is definitely still fresh
2737  return true;
2738  }
2739 
2740  // Remaining seconds during which this stale value can be used
2741  $curGraceTTL = $graceTTL + $curTTL;
2742 
2743  return ( $curGraceTTL > 0 )
2744  // Chance of using the value decreases as $curTTL goes from 0 to -$graceTTL
2745  ? !$this->worthRefreshExpiring( $curGraceTTL, $graceTTL, $graceTTL )
2746  // Value is too stale to fall in the grace period
2747  : false;
2748  }
2749 
2760  protected function isLotteryRefreshDue( $res, $lowTTL, $ageNew, $hotTTR, $now ) {
2761  $curTTL = $res[self::RES_CUR_TTL];
2762  $logicalTTL = $res[self::RES_TTL];
2763  $asOf = $res[self::RES_AS_OF];
2764 
2765  return (
2766  $this->worthRefreshExpiring( $curTTL, $logicalTTL, $lowTTL ) ||
2767  $this->worthRefreshPopular( $asOf, $ageNew, $hotTTR, $now )
2768  );
2769  }
2770 
2786  protected function worthRefreshPopular( $asOf, $ageNew, $timeTillRefresh, $now ) {
2787  if ( $ageNew < 0 || $timeTillRefresh <= 0 ) {
2788  return false;
2789  }
2790 
2791  $age = $now - $asOf;
2792  $timeOld = $age - $ageNew;
2793  if ( $timeOld <= 0 ) {
2794  return false;
2795  }
2796 
2797  $popularHitsPerSec = 1;
2798  // Lifecycle is: new, ramp-up refresh chance, full refresh chance.
2799  // Note that the "expected # of refreshes" for the ramp-up time range is half
2800  // of what it would be if P(refresh) was at its full value during that time range.
2801  $refreshWindowSec = max( $timeTillRefresh - $ageNew - self::RAMPUP_TTL / 2, 1 );
2802  // P(refresh) * (# hits in $refreshWindowSec) = (expected # of refreshes)
2803  // P(refresh) * ($refreshWindowSec * $popularHitsPerSec) = 1 (by definition)
2804  // P(refresh) = 1/($refreshWindowSec * $popularHitsPerSec)
2805  $chance = 1 / ( $popularHitsPerSec * $refreshWindowSec );
2806  // Ramp up $chance from 0 to its nominal value over RAMPUP_TTL seconds to avoid stampedes
2807  $chance *= ( $timeOld <= self::RAMPUP_TTL ) ? $timeOld / self::RAMPUP_TTL : 1;
2808 
2809  $decision = ( mt_rand( 1, 1000000000 ) <= 1000000000 * $chance );
2810 
2811  return $decision;
2812  }
2813 
2832  protected function worthRefreshExpiring( $curTTL, $logicalTTL, $lowTTL ) {
2833  if ( $lowTTL <= 0 ) {
2834  return false;
2835  }
2836 
2837  // T264787: avoid having keys start off with a high chance of being refreshed;
2838  // the point where refreshing becomes possible cannot precede the key lifetime.
2839  $effectiveLowTTL = min( $lowTTL, $logicalTTL ?: INF );
2840 
2841  if ( $curTTL >= $effectiveLowTTL || $curTTL <= 0 ) {
2842  return false;
2843  }
2844 
2845  $chance = ( 1 - $curTTL / $effectiveLowTTL );
2846 
2847  $decision = ( mt_rand( 1, 1000000000 ) <= 1000000000 * $chance );
2848 
2849  return $decision;
2850  }
2851 
2860  protected function isValid( $value, $asOf, $minAsOf ) {
2861  return ( $value !== false && $asOf >= $minAsOf );
2862  }
2863 
2872  private function wrap( $value, $ttl, $version, $now, $walltime ) {
2873  // Returns keys in ascending integer order for PHP7 array packing:
2874  // https://nikic.github.io/2014/12/22/PHPs-new-hashtable-implementation.html
2875  $wrapped = [
2876  self::FLD_FORMAT_VERSION => self::VERSION,
2877  self::FLD_VALUE => $value,
2878  self::FLD_TTL => $ttl,
2879  self::FLD_TIME => $now
2880  ];
2881  if ( $version !== null ) {
2882  $wrapped[self::FLD_VALUE_VERSION] = $version;
2883  }
2884  if ( $walltime >= self::GENERATION_SLOW_SEC ) {
2885  $wrapped[self::FLD_GENERATION_TIME] = $walltime;
2886  }
2887 
2888  return $wrapped;
2889  }
2890 
2905  private function unwrap( $wrapped, $now ) {
2906  // https://nikic.github.io/2014/12/22/PHPs-new-hashtable-implementation.html
2907  $res = [
2908  // Attributes that only depend on the fetched key value
2909  self::RES_VALUE => false,
2910  self::RES_VERSION => null,
2911  self::RES_AS_OF => null,
2912  self::RES_TTL => null,
2913  self::RES_TOMB_AS_OF => null,
2914  // Attributes that depend on caller-specific "check" keys or "touched callbacks"
2915  self::RES_CHECK_AS_OF => null,
2916  self::RES_TOUCH_AS_OF => null,
2917  self::RES_CUR_TTL => null
2918  ];
2919 
2920  if ( is_array( $wrapped ) ) {
2921  // Entry expected to be a cached value; validate it
2922  if (
2923  ( $wrapped[self::FLD_FORMAT_VERSION] ?? null ) === self::VERSION &&
2924  $wrapped[self::FLD_TIME] >= $this->epoch
2925  ) {
2926  if ( $wrapped[self::FLD_TTL] > 0 ) {
2927  // Get the approximate time left on the key
2928  $age = $now - $wrapped[self::FLD_TIME];
2929  $curTTL = max( $wrapped[self::FLD_TTL] - $age, 0.0 );
2930  } else {
2931  // Key had no TTL, so the time left is unbounded
2932  $curTTL = INF;
2933  }
2934  $res[self::RES_VALUE] = $wrapped[self::FLD_VALUE];
2935  $res[self::RES_VERSION] = $wrapped[self::FLD_VALUE_VERSION] ?? null;
2936  $res[self::RES_AS_OF] = $wrapped[self::FLD_TIME];
2937  $res[self::RES_CUR_TTL] = $curTTL;
2938  $res[self::RES_TTL] = $wrapped[self::FLD_TTL];
2939  }
2940  } else {
2941  // Entry expected to be a tombstone; parse it
2942  $purge = $this->parsePurgeValue( $wrapped );
2943  if ( $purge !== null ) {
2944  // Tombstoned keys should always have a negative "current TTL"
2945  $curTTL = min( $purge[self::PURGE_TIME] - $now, self::TINY_NEGATIVE );
2946  $res[self::RES_CUR_TTL] = $curTTL;
2948  }
2949  }
2950 
2951  return $res;
2952  }
2953 
2958  private function determineKeyClassForStats( $key ) {
2959  $parts = explode( ':', $key, 3 );
2960  // Fallback in case the key was not made by makeKey.
2961  // Replace dots because they are special in StatsD (T232907)
2962  return strtr( $parts[1] ?? $parts[0], '.', '_' );
2963  }
2964 
2973  private function parsePurgeValue( $value ) {
2974  if ( !is_string( $value ) ) {
2975  return null;
2976  }
2977 
2978  $segments = explode( ':', $value, 3 );
2979  $prefix = $segments[0];
2980  if ( $prefix !== self::PURGE_VAL_PREFIX ) {
2981  // Not a purge value
2982  return null;
2983  }
2984 
2985  $timestamp = (float)$segments[1];
2986  // makeTombstonePurgeValue() doesn't store hold-off TTLs
2987  $holdoff = isset( $segments[2] ) ? (int)$segments[2] : self::HOLDOFF_TTL;
2988 
2989  if ( $timestamp < $this->epoch ) {
2990  // Purge value is too old
2991  return null;
2992  }
2993 
2994  return [ self::PURGE_TIME => $timestamp, self::PURGE_HOLDOFF => $holdoff ];
2995  }
2996 
3001  private function makeTombstonePurgeValue( float $timestamp ) {
3002  return self::PURGE_VAL_PREFIX . ':' . (int)$timestamp;
3003  }
3004 
3011  private function makeCheckPurgeValue( float $timestamp, int $holdoff, array &$purge = null ) {
3012  $normalizedTime = (int)$timestamp;
3013  // Purge array that matches what parsePurgeValue() would have returned
3014  $purge = [ self::PURGE_TIME => (float)$normalizedTime, self::PURGE_HOLDOFF => $holdoff ];
3015 
3016  return self::PURGE_VAL_PREFIX . ":$normalizedTime:$holdoff";
3017  }
3018 
3023  private function getProcessCache( $group ) {
3024  if ( !isset( $this->processCaches[$group] ) ) {
3025  list( , $size ) = explode( ':', $group );
3026  $this->processCaches[$group] = new MapCacheLRU( (int)$size );
3027  if ( $this->wallClockOverride !== null ) {
3028  $this->processCaches[$group]->setMockTime( $this->wallClockOverride );
3029  }
3030  }
3031 
3032  return $this->processCaches[$group];
3033  }
3034 
3040  private function getNonProcessCachedMultiKeys( ArrayIterator $keys, array $opts ) {
3041  $pcTTL = $opts['pcTTL'] ?? self::TTL_UNCACHEABLE;
3042 
3043  $keysMissing = [];
3044  if ( $pcTTL > 0 && $this->callbackDepth == 0 ) {
3045  $pCache = $this->getProcessCache( $opts['pcGroup'] ?? self::PC_PRIMARY );
3046  foreach ( $keys as $key => $id ) {
3047  if ( !$pCache->has( $key, $pcTTL ) ) {
3048  $keysMissing[$id] = $key;
3049  }
3050  }
3051  }
3052 
3053  return $keysMissing;
3054  }
3055 
3062  private function fetchWrappedValuesForWarmupCache( array $keys, array $checkKeys ) {
3063  if ( !$keys ) {
3064  return [];
3065  }
3066 
3067  // Get all the value keys to fetch...
3068  $sisterKeys = $this->makeSisterKeys( $keys, self::TYPE_VALUE );
3069  // Get all the "check" keys to fetch...
3070  foreach ( $checkKeys as $i => $checkKeyOrKeyGroup ) {
3071  // Note: avoid array_merge() inside loop in case there are many keys
3072  if ( is_int( $i ) ) {
3073  // Single "check" key that applies to all value keys
3074  $sisterKeys[] = $this->makeSisterKey( $checkKeyOrKeyGroup, self::TYPE_TIMESTAMP );
3075  } else {
3076  // List of "check" keys that apply to a specific value key
3077  foreach ( (array)$checkKeyOrKeyGroup as $checkKey ) {
3078  $sisterKeys[] = $this->makeSisterKey( $checkKey, self::TYPE_TIMESTAMP );
3079  }
3080  }
3081  }
3082 
3083  $wrappedBySisterKey = $this->cache->getMulti( $sisterKeys );
3084  $wrappedBySisterKey += array_fill_keys( $sisterKeys, false );
3085 
3086  return $wrappedBySisterKey;
3087  }
3088 
3094  private function timeSinceLoggedMiss( $key, $now ) {
3095  for ( end( $this->missLog ); $miss = current( $this->missLog ); prev( $this->missLog ) ) {
3096  if ( $miss[0] === $key ) {
3097  return ( $now - $miss[1] );
3098  }
3099  }
3100 
3101  return null;
3102  }
3103 
3108  protected function getCurrentTime() {
3109  if ( $this->wallClockOverride ) {
3110  return $this->wallClockOverride;
3111  }
3112 
3113  $clockTime = (float)time(); // call this first
3114  // microtime() uses an initial gettimeofday() call added to usage clocks.
3115  // This can severely drift from time() and the microtime() value of other threads
3116  // due to undercounting of the amount of time elapsed. Instead of seeing the current
3117  // time as being in the past, use the value of time(). This avoids setting cache values
3118  // that will immediately be seen as expired and possibly cause stampedes.
3119  return max( microtime( true ), $clockTime );
3120  }
3121 
3126  public function setMockTime( &$time ) {
3127  $this->wallClockOverride =& $time;
3128  $this->cache->setMockTime( $time );
3129  foreach ( $this->processCaches as $pCache ) {
3130  $pCache->setMockTime( $time );
3131  }
3132  }
3133 }
WANObjectCache\relayVolatilePurges
relayVolatilePurges(array $purgeBySisterKey, int $ttl)
Set a sister key to a purge value in all datacenters.
Definition: WANObjectCache.php:2630
WANObjectCache\getWithSetCallback
getWithSetCallback( $key, $ttl, $callback, array $opts=[], array $cbParams=[])
Method to fetch/regenerate a cache key.
Definition: WANObjectCache.php:1498
WANObjectCache\KEY_TTL
const KEY_TTL
Logical TTL attribute for a key.
Definition: WANObjectCache.php:259
WANObjectCache\getQoS
getQoS( $flag)
Definition: WANObjectCache.php:2530
WANObjectCache\hash256
hash256( $component)
Hash a possibly long string into a suitable component for makeKey()/makeGlobalKey()
Definition: WANObjectCache.php:2320
WANObjectCache\determineKeyClassForStats
determineKeyClassForStats( $key)
Definition: WANObjectCache.php:2958
WANObjectCache\setMockTime
setMockTime(&$time)
Definition: WANObjectCache.php:3126
WANObjectCache\$warmupCache
mixed[] $warmupCache
Temporary warm-up cache.
Definition: WANObjectCache.php:171
Wikimedia\LightweightObjectStore\StorageAwareness\ERR_NO_RESPONSE
const ERR_NO_RESPONSE
Storage medium failed to yield a complete response to an operation.
Definition: StorageAwareness.php:36
WANObjectCache\RES_VALUE
const RES_VALUE
Value for a key.
Definition: WANObjectCache.php:268
WANObjectCache\SCHEME_HASH_TAG
const SCHEME_HASH_TAG
Use twemproxy-style Hash Tag key scheme (e.g.
Definition: WANObjectCache.php:215
EmptyBagOStuff
A BagOStuff object with no objects in it.
Definition: EmptyBagOStuff.php:29
WANObjectCache\KEY_VERSION
const KEY_VERSION
Version number attribute for a key; keep value for b/c (< 1.36)
Definition: WANObjectCache.php:255
WANObjectCache\$cache
BagOStuff $cache
The local datacenter cache.
Definition: WANObjectCache.php:133
WANObjectCache\fetchOrRegenerate
fetchOrRegenerate( $key, $ttl, $callback, array $opts, array $cbParams)
Do the actual I/O for getWithSetCallback() when needed.
Definition: WANObjectCache.php:1558
WANObjectCache\MAX_READ_LAG
const MAX_READ_LAG
Max expected seconds of combined lag from replication and "view snapshots".
Definition: WANObjectCache.php:181
WANObjectCache\FLD_VALUE
const FLD_VALUE
Key to the cached value; stored in blobs.
Definition: WANObjectCache.php:287
WANObjectCache\MAX_COMMIT_DELAY
const MAX_COMMIT_DELAY
Max expected seconds to pass between delete() and DB commit finishing.
Definition: WANObjectCache.php:179
WANObjectCache\GENERATION_SLOW_SEC
const GENERATION_SLOW_SEC
Consider value generation slow if it takes this many seconds or more.
Definition: WANObjectCache.php:244
WANObjectCache\isExtremelyNewValue
isExtremelyNewValue( $res, $minAsOf, $now)
Check if a key value is non-false, new enough, and has an "as of" time almost equal to now.
Definition: WANObjectCache.php:1839
Wikimedia\LightweightObjectStore\ExpirationAwareness
Generic interface providing Time-To-Live constants for expirable object storage.
Definition: ExpirationAwareness.php:32
WANObjectCache\TYPE_INTERIM
const TYPE_INTERIM
Single character component for interium value keys.
Definition: WANObjectCache.php:306
WANObjectCache\RECENT_SET_LOW_MS
const RECENT_SET_LOW_MS
Min millisecond set() backoff during hold-off (far less than INTERIM_KEY_TTL)
Definition: WANObjectCache.php:237
WANObjectCache\isLotteryRefreshDue
isLotteryRefreshDue( $res, $lowTTL, $ageNew, $hotTTR, $now)
Check if a key is due for randomized regeneration due to near-expiration/popularity.
Definition: WANObjectCache.php:2760
WANObjectCache\KEY_CUR_TTL
const KEY_CUR_TTL
Remaining TTL attribute for a key; keep value for b/c (< 1.36)
Definition: WANObjectCache.php:261
NullStatsdDataFactory
Definition: NullStatsdDataFactory.php:10
WANObjectCache\FLD_GENERATION_TIME
const FLD_GENERATION_TIME
Key to how long it took to generate the value; stored in blobs.
Definition: WANObjectCache.php:297
WANObjectCache\FLD_FORMAT_VERSION
const FLD_FORMAT_VERSION
Key to WAN cache version number; stored in blobs.
Definition: WANObjectCache.php:285
WANObjectCache\getMultiWithSetCallback
getMultiWithSetCallback(ArrayIterator $keyedIds, $ttl, callable $callback, array $opts=[])
Method to fetch multiple cache keys at once with regeneration.
Definition: WANObjectCache.php:2044
BagOStuff
Class representing a cache/ephemeral data store.
Definition: BagOStuff.php:86
WANObjectCache\getMultiWithUnionSetCallback
getMultiWithUnionSetCallback(ArrayIterator $keyedIds, $ttl, callable $callback, array $opts=[])
Method to fetch/regenerate multiple cache keys at once.
Definition: WANObjectCache.php:2145
WANObjectCache\getCollectionFromSisterKey
static getCollectionFromSisterKey(string $sisterKey)
Definition: WANObjectCache.php:1813
WANObjectCache\PASS_BY_REF
const PASS_BY_REF
Idiom for get()/getMulti() to return extra information by reference.
Definition: WANObjectCache.php:212
WANObjectCache\FLD_VALUE_VERSION
const FLD_VALUE_VERSION
Key to collection cache version number; stored in blobs.
Definition: WANObjectCache.php:295
WANObjectCache\worthRefreshExpiring
worthRefreshExpiring( $curTTL, $logicalTTL, $lowTTL)
Check if a key is nearing expiration and thus due for randomized regeneration.
Definition: WANObjectCache.php:2832
$res
$res
Definition: testCompression.php:57
WANObjectCache\RES_CHECK_AS_OF
const RES_CHECK_AS_OF
Highest "check" key timestamp for a key.
Definition: WANObjectCache.php:278
WANObjectCache\TTL_LAGGED
const TTL_LAGGED
Max TTL, in seconds, to store keys when a data source has high replication lag.
Definition: WANObjectCache.php:188
WANObjectCache\HOLDOFF_TTL_NONE
const HOLDOFF_TTL_NONE
Idiom for delete()/touchCheckKey() meaning "no hold-off period".
Definition: WANObjectCache.php:203
WANObjectCache\reapCheckKey
reapCheckKey( $key, $purgeTimestamp, &$isStale=false)
Set a "check" key to soon expire in the local cluster if it pre-dates $purgeTimestamp.
Definition: WANObjectCache.php:2264
Wikimedia\LightweightObjectStore\StorageAwareness\ERR_NONE
const ERR_NONE
No storage medium error.
Definition: StorageAwareness.php:34
WANObjectCache\SCHEME_HASH_STOP
const SCHEME_HASH_STOP
Use mcrouter-style Hash Stop key scheme (e.g.
Definition: WANObjectCache.php:217
WANObjectCache\RES_TOUCH_AS_OF
const RES_TOUCH_AS_OF
Highest "touched" timestamp for a key.
Definition: WANObjectCache.php:280
WANObjectCache\fetchKeys
fetchKeys(array $keys, array $checkKeys, $touchedCb=null)
Fetch the value and key metadata of several keys from cache.
Definition: WANObjectCache.php:552
WANObjectCache\getCheckKeyTime
getCheckKeyTime( $key)
Fetch the value of a timestamp "check" key.
Definition: WANObjectCache.php:1023
WANObjectCache\resolveBusyValue
resolveBusyValue( $busyValue)
Definition: WANObjectCache.php:1976
WANObjectCache\$logger
LoggerInterface $logger
Definition: WANObjectCache.php:137
WANObjectCache\makeCheckPurgeValue
makeCheckPurgeValue(float $timestamp, int $holdoff, array &$purge=null)
Definition: WANObjectCache.php:3011
WANObjectCache\isAcceptablyFreshValue
isAcceptablyFreshValue( $res, $graceTTL, $minAsOf)
Check if a key value is non-false, new enough, and either fresh or "gracefully" stale.
Definition: WANObjectCache.php:2728
WANObjectCache\$warmupKeyMisses
int $warmupKeyMisses
Key fetched.
Definition: WANObjectCache.php:173
WANObjectCache\newEmpty
static newEmpty()
Get an instance that wraps EmptyBagOStuff.
Definition: WANObjectCache.php:388
WANObjectCache\LOW_TTL
const LOW_TTL
Consider regeneration if the key will expire within this many seconds.
Definition: WANObjectCache.php:186
WANObjectCache\FLD_TIME
const FLD_TIME
Key to the cache timestamp; stored in blobs.
Definition: WANObjectCache.php:291
WANObjectCache\clearProcessCache
clearProcessCache()
Clear the in-process caches; useful for testing.
Definition: WANObjectCache.php:2497
WANObjectCache\$broadcastRoute
string null $broadcastRoute
Routing prefix for operations that should be broadcasted to all data centers.
Definition: WANObjectCache.php:150
WANObjectCache\$secret
string $secret
Stable secret used for hasing long strings into key components.
Definition: WANObjectCache.php:156
WANObjectCache\clearLastError
clearLastError()
Clear the "last error" registry.
Definition: WANObjectCache.php:2488
WANObjectCache\getCurrentTime
getCurrentTime()
Definition: WANObjectCache.php:3108
WANObjectCache\INTERIM_KEY_TTL
const INTERIM_KEY_TTL
Seconds to keep interim value keys for tombstoned keys around.
Definition: WANObjectCache.php:222
WANObjectCache\unwrap
unwrap( $wrapped, $now)
Definition: WANObjectCache.php:2905
WANObjectCache\fetchWrappedValuesForWarmupCache
fetchWrappedValuesForWarmupCache(array $keys, array $checkKeys)
Definition: WANObjectCache.php:3062
WANObjectCache\relayNonVolatilePurge
relayNonVolatilePurge(string $sisterKey)
Remove a sister key from all datacenters.
Definition: WANObjectCache.php:2659
WANObjectCache\makeMultiKeys
makeMultiKeys(array $ids, $keyCallback)
Get an iterator of (cache key => entity ID) for a list of entity IDs.
Definition: WANObjectCache.php:2374
WANObjectCache\RES_TTL
const RES_TTL
Logical TTL attribute for a key.
Definition: WANObjectCache.php:274
WANObjectCache\getMulti
getMulti(array $keys, &$curTTLs=[], array $checkKeys=[], &$info=[])
Fetch the value of several keys from cache.
Definition: WANObjectCache.php:500
WANObjectCache\__construct
__construct(array $params)
Definition: WANObjectCache.php:345
WANObjectCache\RES_AS_OF
const RES_AS_OF
Generation completion timestamp attribute for a key.
Definition: WANObjectCache.php:272
WANObjectCache\RES_CUR_TTL
const RES_CUR_TTL
Remaining TTL attribute for a key.
Definition: WANObjectCache.php:282
WANObjectCache\useInterimHoldOffCaching
useInterimHoldOffCaching( $enabled)
Enable or disable the use of brief caching for tombstoned keys.
Definition: WANObjectCache.php:2521
WANObjectCache\reap
reap( $key, $purgeTimestamp, &$isStale=false)
Set a key to soon expire in the local cluster if it pre-dates $purgeTimestamp.
Definition: WANObjectCache.php:2233
WANObjectCache\$keyHighQps
int $keyHighQps
Reads/second assumed during a hypothetical cache write stampede for a key.
Definition: WANObjectCache.php:161
WANObjectCache\touchCheckKey
touchCheckKey( $key, $holdoff=self::HOLDOFF_TTL)
Increase the last-purge timestamp of a "check" key in all datacenters.
Definition: WANObjectCache.php:1145
WANObjectCache\TYPE_COOLOFF
const TYPE_COOLOFF
Single character component for cool-off bounce keys.
Definition: WANObjectCache.php:308
MapCacheLRU
Handles a simple LRU key/value map with a maximum number of entries.
Definition: MapCacheLRU.php:36
WANObjectCache\getInterimValue
getInterimValue( $key, $minAsOf, $now, $touchedCb)
Definition: WANObjectCache.php:1928
WANObjectCache\makeSisterKey
makeSisterKey(string $baseKey, string $typeChar, string $route=null)
Get a sister key that should be collocated with a base cache key.
Definition: WANObjectCache.php:1791
WANObjectCache\wrap
wrap( $value, $ttl, $version, $now, $walltime)
Definition: WANObjectCache.php:2872
WANObjectCache\adaptiveTTL
adaptiveTTL( $mtime, $maxTTL, $minTTL=30, $factor=0.2)
Get a TTL that is higher for objects that have not changed recently.
Definition: WANObjectCache.php:2597
WANObjectCache\getNonProcessCachedMultiKeys
getNonProcessCachedMultiKeys(ArrayIterator $keys, array $opts)
Definition: WANObjectCache.php:3040
WANObjectCache\$epoch
float $epoch
Unix timestamp of the oldest possible valid values.
Definition: WANObjectCache.php:154
WANObjectCache\KEY_TOMB_AS_OF
const KEY_TOMB_AS_OF
Tomstone timestamp attribute for a key; keep value for b/c (< 1.36)
Definition: WANObjectCache.php:263
WANObjectCache\$wallClockOverride
float null $wallClockOverride
Definition: WANObjectCache.php:176
WANObjectCache\HOLDOFF_TTL
const HOLDOFF_TTL
Seconds to tombstone keys on delete() and to treat keys as volatile after purges.
Definition: WANObjectCache.php:183
Wikimedia\LightweightObjectStore\StorageAwareness
Generic interface providing error code and quality-of-service constants for object stores.
Definition: StorageAwareness.php:32
WANObjectCache\makeGlobalKey
makeGlobalKey( $collection,... $components)
Make a cache key for the global keyspace and given components.
Definition: WANObjectCache.php:2295
WANObjectCache\getMultiCheckKeyTime
getMultiCheckKeyTime(array $keys)
Fetch the values of each timestamp "check" key.
Definition: WANObjectCache.php:1088
WANObjectCache\worthRefreshPopular
worthRefreshPopular( $asOf, $ageNew, $timeTillRefresh, $now)
Check if a key is due for randomized regeneration due to its popularity.
Definition: WANObjectCache.php:2786
WANObjectCache\watchErrors
watchErrors()
Get a "watch point" token that can be used to get the "last error" to occur after now.
Definition: WANObjectCache.php:2449
WANObjectCache\TYPE_TIMESTAMP
const TYPE_TIMESTAMP
Single character component for timestamp check keys.
Definition: WANObjectCache.php:302
WANObjectCache\RAMPUP_TTL
const RAMPUP_TTL
Seconds to ramp up the chance of regeneration due to expected time-till-refresh.
Definition: WANObjectCache.php:229
WANObjectCache\$useInterimHoldOffCaching
bool $useInterimHoldOffCaching
Whether to use "interim" caching while keys are tombstoned.
Definition: WANObjectCache.php:152
WANObjectCache\PC_PRIMARY
const PC_PRIMARY
Default process cache name and max key count.
Definition: WANObjectCache.php:209
WANObjectCache
Multi-datacenter aware caching interface.
Definition: WANObjectCache.php:131
WANObjectCache\PURGE_TIME
const PURGE_TIME
Key to the tombstone entry timestamp.
Definition: WANObjectCache.php:247
WANObjectCache\VERSION
const VERSION
Cache format version number.
Definition: WANObjectCache.php:252
WANObjectCache\makeTombstonePurgeValue
makeTombstonePurgeValue(float $timestamp)
Definition: WANObjectCache.php:3001
WANObjectCache\timeSinceLoggedMiss
timeSinceLoggedMiss( $key, $now)
Definition: WANObjectCache.php:3094
WANObjectCache\getProcessCache
getProcessCache( $group)
Definition: WANObjectCache.php:3023
WANObjectCache\KEY_AS_OF
const KEY_AS_OF
Generation completion timestamp attribute for a key; keep value for b/c (< 1.36)
Definition: WANObjectCache.php:257
WANObjectCache\multiRemap
multiRemap(array $ids, array $res)
Get an (ID => value) map from (i) a non-unique list of entity IDs, and (ii) the list of corresponding...
Definition: WANObjectCache.php:2430
WANObjectCache\$keyHighUplinkBps
float $keyHighUplinkBps
Max tolerable bytes/second to spend on a cache write stampede for a key.
Definition: WANObjectCache.php:163
Wikimedia\LightweightObjectStore\StorageAwareness\ERR_UNEXPECTED
const ERR_UNEXPECTED
Storage medium operation failed due to usage limitations or an I/O error.
Definition: StorageAwareness.php:40
WANObjectCache\checkAndSetCooloff
checkAndSetCooloff( $key, $kClass, $value, $elapsed, $hasLock)
Check whether set() is rate-limited to avoid concurrent I/O spikes.
Definition: WANObjectCache.php:1870
WANObjectCache\processCheckKeys
processCheckKeys(array $checkSisterKeys, array $wrappedBySisterKey, float $now)
Definition: WANObjectCache.php:680
WANObjectCache\resetCheckKey
resetCheckKey( $key)
Clear the last-purge timestamp of a "check" key in all datacenters.
Definition: WANObjectCache.php:1185
WANObjectCache\TSE_NONE
const TSE_NONE
Idiom for getWithSetCallback() meaning "no cache stampede mutex".
Definition: WANObjectCache.php:196
WANObjectCache\claimStampedeLock
claimStampedeLock( $key)
Definition: WANObjectCache.php:1745
WANObjectCache\makeSisterKeys
makeSisterKeys(array $baseKeys, string $type, string $route=null)
Get sister keys that should be collocated with their corresponding base cache keys.
Definition: WANObjectCache.php:1772
WANObjectCache\parsePurgeValue
parsePurgeValue( $value)
Extract purge metadata from cached value if it is a valid purge value.
Definition: WANObjectCache.php:2973
WANObjectCache\HOT_TTR
const HOT_TTR
Expected time-till-refresh, in seconds, if the key is accessed once per second.
Definition: WANObjectCache.php:191
WANObjectCache\LOCK_TTL
const LOCK_TTL
Seconds to keep lock keys around.
Definition: WANObjectCache.php:225
WANObjectCache\TYPE_MUTEX
const TYPE_MUTEX
Single character component for mutex lock keys.
Definition: WANObjectCache.php:304
Wikimedia\LightweightObjectStore\StorageAwareness\ERR_UNREACHABLE
const ERR_UNREACHABLE
Storage medium could not be reached to establish a connection.
Definition: StorageAwareness.php:38
WANObjectCache\prependRoute
prependRoute(string $sisterKey, string $route)
Definition: WANObjectCache.php:2674
WANObjectCache\yieldStampedeLock
yieldStampedeLock( $key, $hasLock)
Definition: WANObjectCache.php:1755
WANObjectCache\RES_VERSION
const RES_VERSION
Version number attribute for a key.
Definition: WANObjectCache.php:270
WANObjectCache\PURGE_VAL_PREFIX
const PURGE_VAL_PREFIX
Value prefix of purge values.
Definition: WANObjectCache.php:311
WANObjectCache\$asyncHandler
callable null $asyncHandler
Function that takes a WAN cache callback and runs it later.
Definition: WANObjectCache.php:141
WANObjectCache\getWarmupKeyMisses
getWarmupKeyMisses()
Definition: WANObjectCache.php:2612
WANObjectCache\RES_TOMB_AS_OF
const RES_TOMB_AS_OF
Tomstone timestamp attribute for a key.
Definition: WANObjectCache.php:276
$keys
$keys
Definition: testCompression.php:72
WANObjectCache\KEY_CHECK_AS_OF
const KEY_CHECK_AS_OF
Highest "check" key timestamp for a key; keep value for b/c (< 1.36)
Definition: WANObjectCache.php:265
WANObjectCache\$callbackDepth
int $callbackDepth
Callback stack depth for getWithSetCallback()
Definition: WANObjectCache.php:169
WANObjectCache\FLD_FLAGS
const FLD_FLAGS
Key to the flags bit field (reserved number)
Definition: WANObjectCache.php:293
WANObjectCache\$coalesceScheme
int $coalesceScheme
Scheme to use for key coalescing (Hash Tags or Hash Stops)
Definition: WANObjectCache.php:158
WANObjectCache\setInterimValue
setInterimValue( $key, $value, $ttl, $version, $now, $walltime)
Definition: WANObjectCache.php:1958
WANObjectCache\AGE_NEW
const AGE_NEW
Minimum key age, in seconds, for expected time-till-refresh to be considered.
Definition: WANObjectCache.php:193
IStoreKeyEncoder
Generic interface for object stores with key encoding methods.
Definition: IStoreKeyEncoder.php:9
WANObjectCache\setLogger
setLogger(LoggerInterface $logger)
Definition: WANObjectCache.php:379
WANObjectCache\TYPE_VALUE
const TYPE_VALUE
Single character component for value keys.
Definition: WANObjectCache.php:300
WANObjectCache\makeKey
makeKey( $collection,... $components)
Make a cache key using the "global" keyspace for the given components.
Definition: WANObjectCache.php:2309
WANObjectCache\CHECK_KEY_TTL
const CHECK_KEY_TTL
Seconds to keep dependency purge keys around.
Definition: WANObjectCache.php:220
WANObjectCache\getLastError
getLastError( $watchPoint=0)
Get the "last error" registry.
Definition: WANObjectCache.php:2470
WANObjectCache\GRACE_TTL_NONE
const GRACE_TTL_NONE
Idiom for set()/getWithSetCallback() meaning "no post-expiration grace period".
Definition: WANObjectCache.php:201
WANObjectCache\RECENT_SET_HIGH_MS
const RECENT_SET_HIGH_MS
Max millisecond set() backoff during hold-off (far less than INTERIM_KEY_TTL)
Definition: WANObjectCache.php:239
WANObjectCache\GENERATION_HIGH_SEC
const GENERATION_HIGH_SEC
Consider value generation somewhat high if it takes this many seconds or more.
Definition: WANObjectCache.php:242
WANObjectCache\FLD_TTL
const FLD_TTL
Key to the original TTL; stored in blobs.
Definition: WANObjectCache.php:289
WANObjectCache\isValid
isValid( $value, $asOf, $minAsOf)
Check that a wrapper value exists and has an acceptable age.
Definition: WANObjectCache.php:2860
WANObjectCache\PURGE_HOLDOFF
const PURGE_HOLDOFF
Key to the tombstone entry hold-off TTL.
Definition: WANObjectCache.php:249
WANObjectCache\STALE_TTL_NONE
const STALE_TTL_NONE
Idiom for set()/getWithSetCallback() meaning "no post-expiration persistence".
Definition: WANObjectCache.php:199
WANObjectCache\$stats
StatsdDataFactoryInterface $stats
Definition: WANObjectCache.php:139
WANObjectCache\$processCaches
MapCacheLRU[] $processCaches
Map of group PHP instance caches.
Definition: WANObjectCache.php:135
WANObjectCache\scheduleAsyncRefresh
scheduleAsyncRefresh( $key, $ttl, $callback, array $opts, array $cbParams)
Schedule a deferred cache regeneration if possible.
Definition: WANObjectCache.php:2693
WANObjectCache\$missLog
array< int, array > $missLog
List of (key, UNIX timestamp) tuples for get() cache misses.
Definition: WANObjectCache.php:166
WANObjectCache\COOLOFF_TTL
const COOLOFF_TTL
Seconds to no-op key set() calls to avoid large blob I/O stampedes.
Definition: WANObjectCache.php:227
$type
$type
Definition: testCompression.php:52