MediaWiki  master
WANObjectCache.php
Go to the documentation of this file.
1 <?php
26 
116 class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInterface {
118  protected $cache;
120  protected $processCaches = [];
122  protected $logger;
124  protected $stats;
126  protected $asyncHandler;
127 
129  protected $mcrouterAware;
131  protected $region;
133  protected $cluster;
135  protected $useInterimHoldOffCaching = true;
137  protected $epoch;
139  protected $secret;
140 
142  private $callbackDepth = 0;
144  private $warmupCache = [];
146  private $warmupKeyMisses = 0;
147 
150 
152  const MAX_COMMIT_DELAY = 3;
154  const MAX_READ_LAG = 7;
156  const HOLDOFF_TTL = self::MAX_COMMIT_DELAY + self::MAX_READ_LAG + 1;
157 
159  const TTL_UNCACHEABLE = -1;
160 
162  const LOW_TTL = 30;
164  const TTL_LAGGED = 30;
165 
167  const HOT_TTR = 900;
169  const AGE_NEW = 60;
170 
172  const TSE_NONE = -1;
173 
175  const STALE_TTL_NONE = 0;
177  const GRACE_TTL_NONE = 0;
179  const HOLDOFF_TTL_NONE = 0;
181  const HOLDOFF_NONE = self::HOLDOFF_TTL_NONE;
182 
184  const MIN_TIMESTAMP_NONE = 0.0;
185 
187  const PC_PRIMARY = 'primary:1000';
188 
190  const PASS_BY_REF = -1;
191 
193  private static $CHECK_KEY_TTL = self::TTL_YEAR;
195  private static $INTERIM_KEY_TTL = 1;
196 
198  private static $LOCK_TTL = 10;
200  private static $COOLOFF_TTL = 1;
202  private static $RAMPUP_TTL = 30;
203 
205  private static $TINY_NEGATIVE = -0.000001;
207  private static $TINY_POSTIVE = 0.000001;
208 
210  private static $SET_DELAY_HIGH_MS = 50;
212  private static $RECENT_SET_LOW_MS = 50;
214  private static $RECENT_SET_HIGH_MS = 100;
215 
217  private static $GENERATION_SLOW_SEC = 3;
218 
220  private static $PURGE_TIME = 0;
222  private static $PURGE_HOLDOFF = 1;
223 
225  private static $VERSION = 1;
226 
228  private static $FLD_FORMAT_VERSION = 0;
230  private static $FLD_VALUE = 1;
232  private static $FLD_TTL = 2;
234  private static $FLD_TIME = 3;
236  private static $FLD_FLAGS = 4;
238  private static $FLD_VALUE_VERSION = 5;
240  private static $FLD_GENERATION_TIME = 6;
241 
242  private static $VALUE_KEY_PREFIX = 'WANCache:v:';
243  private static $INTERIM_KEY_PREFIX = 'WANCache:i:';
244  private static $TIME_KEY_PREFIX = 'WANCache:t:';
245  private static $MUTEX_KEY_PREFIX = 'WANCache:m:';
246  private static $COOLOFF_KEY_PREFIX = 'WANCache:c:';
247 
248  private static $PURGE_VAL_PREFIX = 'PURGED:';
249 
275  public function __construct( array $params ) {
276  $this->cache = $params['cache'];
277  $this->region = $params['region'] ?? 'main';
278  $this->cluster = $params['cluster'] ?? 'wan-main';
279  $this->mcrouterAware = !empty( $params['mcrouterAware'] );
280  $this->epoch = $params['epoch'] ?? 0;
281  $this->secret = $params['secret'] ?? (string)$this->epoch;
282 
283  $this->setLogger( $params['logger'] ?? new NullLogger() );
284  $this->stats = $params['stats'] ?? new NullStatsdDataFactory();
285  $this->asyncHandler = $params['asyncHandler'] ?? null;
286  }
287 
291  public function setLogger( LoggerInterface $logger ) {
292  $this->logger = $logger;
293  }
294 
300  public static function newEmpty() {
301  return new static( [ 'cache' => new EmptyBagOStuff() ] );
302  }
303 
354  final public function get(
355  $key, &$curTTL = null, array $checkKeys = [], &$info = null
356  ) {
357  $curTTLs = self::PASS_BY_REF;
358  $infoByKey = self::PASS_BY_REF;
359  $values = $this->getMulti( [ $key ], $curTTLs, $checkKeys, $infoByKey );
360  $curTTL = $curTTLs[$key] ?? null;
361  if ( $info === self::PASS_BY_REF ) {
362  $info = [
363  'asOf' => $infoByKey[$key]['asOf'] ?? null,
364  'tombAsOf' => $infoByKey[$key]['tombAsOf'] ?? null,
365  'lastCKPurge' => $infoByKey[$key]['lastCKPurge'] ?? null,
366  'version' => $infoByKey[$key]['version'] ?? null
367  ];
368  } else {
369  $info = $infoByKey[$key]['asOf'] ?? null; // b/c
370  }
371 
372  return $values[$key] ?? false;
373  }
374 
397  final public function getMulti(
398  array $keys,
399  &$curTTLs = [],
400  array $checkKeys = [],
401  &$info = null
402  ) {
403  $result = [];
404  $curTTLs = [];
405  $infoByKey = [];
406 
407  $vPrefixLen = strlen( self::$VALUE_KEY_PREFIX );
408  $valueKeys = self::prefixCacheKeys( $keys, self::$VALUE_KEY_PREFIX );
409 
410  $checkKeysForAll = [];
411  $checkKeysByKey = [];
412  $checkKeysFlat = [];
413  foreach ( $checkKeys as $i => $checkKeyGroup ) {
414  $prefixed = self::prefixCacheKeys( (array)$checkKeyGroup, self::$TIME_KEY_PREFIX );
415  $checkKeysFlat = array_merge( $checkKeysFlat, $prefixed );
416  // Are these check keys for a specific cache key, or for all keys being fetched?
417  if ( is_int( $i ) ) {
418  $checkKeysForAll = array_merge( $checkKeysForAll, $prefixed );
419  } else {
420  $checkKeysByKey[$i] = $prefixed;
421  }
422  }
423 
424  // Fetch all of the raw values
425  $keysGet = array_merge( $valueKeys, $checkKeysFlat );
426  if ( $this->warmupCache ) {
427  $wrappedValues = array_intersect_key( $this->warmupCache, array_flip( $keysGet ) );
428  $keysGet = array_diff( $keysGet, array_keys( $wrappedValues ) ); // keys left to fetch
429  $this->warmupKeyMisses += count( $keysGet );
430  } else {
431  $wrappedValues = [];
432  }
433  if ( $keysGet ) {
434  $wrappedValues += $this->cache->getMulti( $keysGet );
435  }
436  // Time used to compare/init "check" keys (derived after getMulti() to be pessimistic)
437  $now = $this->getCurrentTime();
438 
439  // Collect timestamps from all "check" keys
440  $purgeValuesForAll = $this->processCheckKeys( $checkKeysForAll, $wrappedValues, $now );
441  $purgeValuesByKey = [];
442  foreach ( $checkKeysByKey as $cacheKey => $checks ) {
443  $purgeValuesByKey[$cacheKey] =
444  $this->processCheckKeys( $checks, $wrappedValues, $now );
445  }
446 
447  // Get the main cache value for each key and validate them
448  foreach ( $valueKeys as $vKey ) {
449  $key = substr( $vKey, $vPrefixLen ); // unprefix
450  list( $value, $keyInfo ) = $this->unwrap( $wrappedValues[$vKey] ?? false, $now );
451  // Force dependent keys to be seen as stale for a while after purging
452  // to reduce race conditions involving stale data getting cached
453  $purgeValues = $purgeValuesForAll;
454  if ( isset( $purgeValuesByKey[$key] ) ) {
455  $purgeValues = array_merge( $purgeValues, $purgeValuesByKey[$key] );
456  }
457 
458  $lastCKPurge = null; // timestamp of the highest check key
459  foreach ( $purgeValues as $purge ) {
460  $lastCKPurge = max( $purge[self::$PURGE_TIME], $lastCKPurge );
461  $safeTimestamp = $purge[self::$PURGE_TIME] + $purge[self::$PURGE_HOLDOFF];
462  if ( $value !== false && $safeTimestamp >= $keyInfo['asOf'] ) {
463  // How long ago this value was invalidated by *this* check key
464  $ago = min( $purge[self::$PURGE_TIME] - $now, self::$TINY_NEGATIVE );
465  // How long ago this value was invalidated by *any* known check key
466  $keyInfo['curTTL'] = min( $keyInfo['curTTL'], $ago );
467  }
468  }
469  $keyInfo[ 'lastCKPurge'] = $lastCKPurge;
470 
471  if ( $value !== false ) {
472  $result[$key] = $value;
473  }
474  if ( $keyInfo['curTTL'] !== null ) {
475  $curTTLs[$key] = $keyInfo['curTTL'];
476  }
477 
478  $infoByKey[$key] = ( $info === self::PASS_BY_REF )
479  ? $keyInfo
480  : $keyInfo['asOf']; // b/c
481  }
482 
483  $info = $infoByKey;
484 
485  return $result;
486  }
487 
495  private function processCheckKeys( array $timeKeys, array $wrappedValues, $now ) {
496  $purgeValues = [];
497  foreach ( $timeKeys as $timeKey ) {
498  $purge = isset( $wrappedValues[$timeKey] )
499  ? $this->parsePurgeValue( $wrappedValues[$timeKey] )
500  : false;
501  if ( $purge === false ) {
502  // Key is not set or malformed; regenerate
503  $newVal = $this->makePurgeValue( $now, self::HOLDOFF_TTL );
504  $this->cache->add( $timeKey, $newVal, self::$CHECK_KEY_TTL );
505  $purge = $this->parsePurgeValue( $newVal );
506  }
507  $purgeValues[] = $purge;
508  }
509 
510  return $purgeValues;
511  }
512 
586  final public function set( $key, $value, $ttl = self::TTL_INDEFINITE, array $opts = [] ) {
587  $now = $this->getCurrentTime();
588  $lag = $opts['lag'] ?? 0;
589  $age = isset( $opts['since'] ) ? max( 0, $now - $opts['since'] ) : 0;
590  $pending = $opts['pending'] ?? false;
591  $lockTSE = $opts['lockTSE'] ?? self::TSE_NONE;
592  $staleTTL = $opts['staleTTL'] ?? self::STALE_TTL_NONE;
593  $creating = $opts['creating'] ?? false;
594  $version = $opts['version'] ?? null;
595  $walltime = $opts['walltime'] ?? 0.0;
596 
597  if ( $ttl < 0 ) {
598  return true;
599  }
600 
601  // Do not cache potentially uncommitted data as it might get rolled back
602  if ( $pending ) {
603  $this->logger->info(
604  'Rejected set() for {cachekey} due to pending writes.',
605  [ 'cachekey' => $key ]
606  );
607 
608  return true; // no-op the write for being unsafe
609  }
610 
611  $logicalTTL = null; // logical TTL override
612  // Check if there's a risk of writing stale data after the purge tombstone expired
613  if ( $lag === false || ( $lag + $age ) > self::MAX_READ_LAG ) {
614  // Case A: any long-running transaction
615  if ( $age > self::MAX_READ_LAG ) {
616  if ( $lockTSE >= 0 ) {
617  // Store value as *almost* stale to avoid cache and mutex stampedes
618  $logicalTTL = self::TTL_SECOND;
619  $this->logger->info(
620  'Lowered set() TTL for {cachekey} due to snapshot lag.',
621  [ 'cachekey' => $key, 'lag' => $lag, 'age' => $age ]
622  );
623  } else {
624  $this->logger->info(
625  'Rejected set() for {cachekey} due to snapshot lag.',
626  [ 'cachekey' => $key, 'lag' => $lag, 'age' => $age ]
627  );
628 
629  return true; // no-op the write for being unsafe
630  }
631  // Case B: high replication lag; lower TTL instead of ignoring all set()s
632  } elseif ( $lag === false || $lag > self::MAX_READ_LAG ) {
633  if ( $lockTSE >= 0 ) {
634  $logicalTTL = min( $ttl ?: INF, self::TTL_LAGGED );
635  } else {
636  $ttl = min( $ttl ?: INF, self::TTL_LAGGED );
637  }
638  $this->logger->warning(
639  'Lowered set() TTL for {cachekey} due to replication lag.',
640  [ 'cachekey' => $key, 'lag' => $lag, 'age' => $age ]
641  );
642  // Case C: medium length request with medium replication lag
643  } elseif ( $lockTSE >= 0 ) {
644  // Store value as *almost* stale to avoid cache and mutex stampedes
645  $logicalTTL = self::TTL_SECOND;
646  $this->logger->info(
647  'Lowered set() TTL for {cachekey} due to high read lag.',
648  [ 'cachekey' => $key, 'lag' => $lag, 'age' => $age ]
649  );
650  } else {
651  $this->logger->info(
652  'Rejected set() for {cachekey} due to high read lag.',
653  [ 'cachekey' => $key, 'lag' => $lag, 'age' => $age ]
654  );
655 
656  return true; // no-op the write for being unsafe
657  }
658  }
659 
660  // Wrap that value with time/TTL/version metadata
661  $wrapped = $this->wrap( $value, $logicalTTL ?: $ttl, $version, $now, $walltime );
662  $storeTTL = $ttl + $staleTTL;
663 
664  if ( $creating ) {
665  $ok = $this->cache->add( self::$VALUE_KEY_PREFIX . $key, $wrapped, $storeTTL );
666  } else {
667  $ok = $this->cache->merge(
668  self::$VALUE_KEY_PREFIX . $key,
669  function ( $cache, $key, $cWrapped ) use ( $wrapped ) {
670  // A string value means that it is a tombstone; do nothing in that case
671  return ( is_string( $cWrapped ) ) ? false : $wrapped;
672  },
673  $storeTTL,
674  1 // 1 attempt
675  );
676  }
677 
678  return $ok;
679  }
680 
742  final public function delete( $key, $ttl = self::HOLDOFF_TTL ) {
743  if ( $ttl <= 0 ) {
744  // Publish the purge to all datacenters
745  $ok = $this->relayDelete( self::$VALUE_KEY_PREFIX . $key );
746  } else {
747  // Publish the purge to all datacenters
748  $ok = $this->relayPurge( self::$VALUE_KEY_PREFIX . $key, $ttl, self::HOLDOFF_TTL_NONE );
749  }
750 
751  $kClass = $this->determineKeyClassForStats( $key );
752  $this->stats->increment( "wanobjectcache.$kClass.delete." . ( $ok ? 'ok' : 'error' ) );
753 
754  return $ok;
755  }
756 
776  final public function getCheckKeyTime( $key ) {
777  return $this->getMultiCheckKeyTime( [ $key ] )[$key];
778  }
779 
841  final public function getMultiCheckKeyTime( array $keys ) {
842  $rawKeys = [];
843  foreach ( $keys as $key ) {
844  $rawKeys[$key] = self::$TIME_KEY_PREFIX . $key;
845  }
846 
847  $rawValues = $this->cache->getMulti( $rawKeys );
848  $rawValues += array_fill_keys( $rawKeys, false );
849 
850  $times = [];
851  foreach ( $rawKeys as $key => $rawKey ) {
852  $purge = $this->parsePurgeValue( $rawValues[$rawKey] );
853  if ( $purge !== false ) {
854  $time = $purge[self::$PURGE_TIME];
855  } else {
856  // Casting assures identical floats for the next getCheckKeyTime() calls
857  $now = (string)$this->getCurrentTime();
858  $this->cache->add(
859  $rawKey,
860  $this->makePurgeValue( $now, self::HOLDOFF_TTL ),
861  self::$CHECK_KEY_TTL
862  );
863  $time = (float)$now;
864  }
865 
866  $times[$key] = $time;
867  }
868 
869  return $times;
870  }
871 
906  final public function touchCheckKey( $key, $holdoff = self::HOLDOFF_TTL ) {
907  // Publish the purge to all datacenters
908  $ok = $this->relayPurge( self::$TIME_KEY_PREFIX . $key, self::$CHECK_KEY_TTL, $holdoff );
909 
910  $kClass = $this->determineKeyClassForStats( $key );
911  $this->stats->increment( "wanobjectcache.$kClass.ck_touch." . ( $ok ? 'ok' : 'error' ) );
912 
913  return $ok;
914  }
915 
943  final public function resetCheckKey( $key ) {
944  // Publish the purge to all datacenters
945  $ok = $this->relayDelete( self::$TIME_KEY_PREFIX . $key );
946 
947  $kClass = $this->determineKeyClassForStats( $key );
948  $this->stats->increment( "wanobjectcache.$kClass.ck_reset." . ( $ok ? 'ok' : 'error' ) );
949 
950  return $ok;
951  }
952 
1255  final public function getWithSetCallback( $key, $ttl, $callback, array $opts = [] ) {
1256  $version = $opts['version'] ?? null;
1257  $pcTTL = $opts['pcTTL'] ?? self::TTL_UNCACHEABLE;
1258  $pCache = ( $pcTTL >= 0 )
1259  ? $this->getProcessCache( $opts['pcGroup'] ?? self::PC_PRIMARY )
1260  : null;
1261 
1262  // Use the process cache if requested as long as no outer cache callback is running.
1263  // Nested callback process cache use is not lag-safe with regard to HOLDOFF_TTL since
1264  // process cached values are more lagged than persistent ones as they are not purged.
1265  if ( $pCache && $this->callbackDepth == 0 ) {
1266  $cached = $pCache->get( $this->getProcessCacheKey( $key, $version ), INF, false );
1267  if ( $cached !== false ) {
1268  return $cached;
1269  }
1270  }
1271 
1272  $res = $this->fetchOrRegenerate( $key, $ttl, $callback, $opts );
1273  list( $value, $valueVersion, $curAsOf ) = $res;
1274  if ( $valueVersion !== $version ) {
1275  // Current value has a different version; use the variant key for this version.
1276  // Regenerate the variant value if it is not newer than the main value at $key
1277  // so that purges to the main key propagate to the variant value.
1278  list( $value ) = $this->fetchOrRegenerate(
1279  $this->makeGlobalKey( 'WANCache-key-variant', md5( $key ), $version ),
1280  $ttl,
1281  $callback,
1282  [ 'version' => null, 'minAsOf' => $curAsOf ] + $opts
1283  );
1284  }
1285 
1286  // Update the process cache if enabled
1287  if ( $pCache && $value !== false ) {
1288  $pCache->set( $this->getProcessCacheKey( $key, $version ), $value );
1289  }
1290 
1291  return $value;
1292  }
1293 
1309  private function fetchOrRegenerate( $key, $ttl, $callback, array $opts ) {
1310  $checkKeys = $opts['checkKeys'] ?? [];
1311  $graceTTL = $opts['graceTTL'] ?? self::GRACE_TTL_NONE;
1312  $minAsOf = $opts['minAsOf'] ?? self::MIN_TIMESTAMP_NONE;
1313  $hotTTR = $opts['hotTTR'] ?? self::HOT_TTR;
1314  $lowTTL = $opts['lowTTL'] ?? min( self::LOW_TTL, $ttl );
1315  $ageNew = $opts['ageNew'] ?? self::AGE_NEW;
1316  $touchedCb = $opts['touchedCallback'] ?? null;
1317  $initialTime = $this->getCurrentTime();
1318 
1319  $kClass = $this->determineKeyClassForStats( $key );
1320 
1321  // Get the current key value and its metadata
1322  $curTTL = self::PASS_BY_REF;
1323  $curInfo = self::PASS_BY_REF;
1324  $curValue = $this->get( $key, $curTTL, $checkKeys, $curInfo );
1325  // Apply any $touchedCb invalidation timestamp to get the "last purge timestamp"
1326  list( $curTTL, $LPT ) = $this->resolveCTL( $curValue, $curTTL, $curInfo, $touchedCb );
1327  // Use the cached value if it exists and is not due for synchronous regeneration
1328  if (
1329  $this->isValid( $curValue, $curInfo['asOf'], $minAsOf ) &&
1330  $this->isAliveOrInGracePeriod( $curTTL, $graceTTL )
1331  ) {
1332  $preemptiveRefresh = (
1333  $this->worthRefreshExpiring( $curTTL, $lowTTL ) ||
1334  $this->worthRefreshPopular( $curInfo['asOf'], $ageNew, $hotTTR, $initialTime )
1335  );
1336  if ( !$preemptiveRefresh ) {
1337  $this->stats->increment( "wanobjectcache.$kClass.hit.good" );
1338 
1339  return [ $curValue, $curInfo['version'], $curInfo['asOf'] ];
1340  } elseif ( $this->scheduleAsyncRefresh( $key, $ttl, $callback, $opts ) ) {
1341  $this->stats->increment( "wanobjectcache.$kClass.hit.refresh" );
1342 
1343  return [ $curValue, $curInfo['version'], $curInfo['asOf'] ];
1344  }
1345  }
1346 
1347  // Determine if there is stale or volatile cached value that is still usable
1348  $isKeyTombstoned = ( $curInfo['tombAsOf'] !== null );
1349  if ( $isKeyTombstoned ) {
1350  // Key is write-holed; use the (volatile) interim key as an alternative
1351  list( $possValue, $possInfo ) = $this->getInterimValue( $key, $minAsOf );
1352  // Update the "last purge time" since the $touchedCb timestamp depends on $value
1353  $LPT = $this->resolveTouched( $possValue, $LPT, $touchedCb );
1354  } else {
1355  $possValue = $curValue;
1356  $possInfo = $curInfo;
1357  }
1358 
1359  // Avoid overhead from callback runs, regeneration locks, and cache sets during
1360  // hold-off periods for the key by reusing very recently generated cached values
1361  if (
1362  $this->isValid( $possValue, $possInfo['asOf'], $minAsOf, $LPT ) &&
1363  $this->isVolatileValueAgeNegligible( $initialTime - $possInfo['asOf'] )
1364  ) {
1365  $this->stats->increment( "wanobjectcache.$kClass.hit.volatile" );
1366 
1367  return [ $possValue, $possInfo['version'], $curInfo['asOf'] ];
1368  }
1369 
1370  $lockTSE = $opts['lockTSE'] ?? self::TSE_NONE;
1371  $busyValue = $opts['busyValue'] ?? null;
1372  $staleTTL = $opts['staleTTL'] ?? self::STALE_TTL_NONE;
1373  $version = $opts['version'] ?? null;
1374 
1375  // Determine whether one thread per datacenter should handle regeneration at a time
1376  $useRegenerationLock =
1377  // Note that since tombstones no-op set(), $lockTSE and $curTTL cannot be used to
1378  // deduce the key hotness because |$curTTL| will always keep increasing until the
1379  // tombstone expires or is overwritten by a new tombstone. Also, even if $lockTSE
1380  // is not set, constant regeneration of a key for the tombstone lifetime might be
1381  // very expensive. Assume tombstoned keys are possibly hot in order to reduce
1382  // the risk of high regeneration load after the delete() method is called.
1383  $isKeyTombstoned ||
1384  // Assume a key is hot if requested soon ($lockTSE seconds) after invalidation.
1385  // This avoids stampedes when timestamps from $checkKeys/$touchedCb bump.
1386  ( $curTTL !== null && $curTTL <= 0 && abs( $curTTL ) <= $lockTSE ) ||
1387  // Assume a key is hot if there is no value and a busy fallback is given.
1388  // This avoids stampedes on eviction or preemptive regeneration taking too long.
1389  ( $busyValue !== null && $possValue === false );
1390 
1391  // If a regeneration lock is required, threads that do not get the lock will try to use
1392  // the stale value, the interim value, or the $busyValue placeholder, in that order. If
1393  // none of those are set then all threads will bypass the lock and regenerate the value.
1394  $hasLock = $useRegenerationLock && $this->claimStampedeLock( $key );
1395  if ( $useRegenerationLock && !$hasLock ) {
1396  if ( $this->isValid( $possValue, $possInfo['asOf'], $minAsOf ) ) {
1397  $this->stats->increment( "wanobjectcache.$kClass.hit.stale" );
1398 
1399  return [ $possValue, $possInfo['version'], $curInfo['asOf'] ];
1400  } elseif ( $busyValue !== null ) {
1401  $miss = is_infinite( $minAsOf ) ? 'renew' : 'miss';
1402  $this->stats->increment( "wanobjectcache.$kClass.$miss.busy" );
1403 
1404  return [ $this->resolveBusyValue( $busyValue ), $version, $curInfo['asOf'] ];
1405  }
1406  }
1407 
1408  // Generate the new value given any prior value with a matching version
1409  $setOpts = [];
1410  $preCallbackTime = $this->getCurrentTime();
1412  try {
1413  $value = $callback(
1414  ( $curInfo['version'] === $version ) ? $curValue : false,
1415  $ttl,
1416  $setOpts,
1417  ( $curInfo['version'] === $version ) ? $curInfo['asOf'] : null
1418  );
1419  } finally {
1421  }
1422  $postCallbackTime = $this->getCurrentTime();
1423 
1424  // How long it took to fetch, validate, and generate the value
1425  $elapsed = max( $postCallbackTime - $initialTime, 0.0 );
1426 
1427  // Attempt to save the newly generated value if applicable
1428  if (
1429  // Callback yielded a cacheable value
1430  ( $value !== false && $ttl >= 0 ) &&
1431  // Current thread was not raced out of a regeneration lock or key is tombstoned
1432  ( !$useRegenerationLock || $hasLock || $isKeyTombstoned ) &&
1433  // Key does not appear to be undergoing a set() stampede
1434  $this->checkAndSetCooloff( $key, $kClass, $elapsed, $lockTSE, $hasLock )
1435  ) {
1436  // How long it took to generate the value
1437  $walltime = max( $postCallbackTime - $preCallbackTime, 0.0 );
1438  $this->stats->timing( "wanobjectcache.$kClass.regen_walltime", 1e3 * $walltime );
1439  // If the key is write-holed then use the (volatile) interim key as an alternative
1440  if ( $isKeyTombstoned ) {
1441  $this->setInterimValue( $key, $value, $lockTSE, $version, $walltime );
1442  } else {
1443  $finalSetOpts = [
1444  'since' => $setOpts['since'] ?? $preCallbackTime,
1445  'version' => $version,
1446  'staleTTL' => $staleTTL,
1447  'lockTSE' => $lockTSE, // informs lag vs performance trade-offs
1448  'creating' => ( $curValue === false ), // optimization
1449  'walltime' => $walltime
1450  ] + $setOpts;
1451  $this->set( $key, $value, $ttl, $finalSetOpts );
1452  }
1453  }
1454 
1455  $this->yieldStampedeLock( $key, $hasLock );
1456 
1457  $miss = is_infinite( $minAsOf ) ? 'renew' : 'miss';
1458  $this->stats->increment( "wanobjectcache.$kClass.$miss.compute" );
1459 
1460  return [ $value, $version, $curInfo['asOf'] ];
1461  }
1462 
1467  private function claimStampedeLock( $key ) {
1468  // Note that locking is not bypassed due to I/O errors; this avoids stampedes
1469  return $this->cache->add( self::$MUTEX_KEY_PREFIX . $key, 1, self::$LOCK_TTL );
1470  }
1471 
1476  private function yieldStampedeLock( $key, $hasLock ) {
1477  if ( $hasLock ) {
1478  // The backend might be a mcrouter proxy set to broadcast DELETE to *all* the local
1479  // datacenter cache servers via OperationSelectorRoute (for increased consistency).
1480  // Since that would be excessive for these locks, use TOUCH to expire the key.
1481  $this->cache->changeTTL( self::$MUTEX_KEY_PREFIX . $key, $this->getCurrentTime() - 60 );
1482  }
1483  }
1484 
1489  private function isVolatileValueAgeNegligible( $age ) {
1490  return ( $age < mt_rand( self::$RECENT_SET_LOW_MS, self::$RECENT_SET_HIGH_MS ) / 1e3 );
1491  }
1492 
1501  private function checkAndSetCooloff( $key, $kClass, $elapsed, $lockTSE, $hasLock ) {
1502  $this->stats->timing( "wanobjectcache.$kClass.regen_set_delay", 1e3 * $elapsed );
1503 
1504  // If $lockTSE is set, the lock was bypassed because there was no stale/interim value,
1505  // and $elapsed indicates that regeration is slow, then there is a risk of set()
1506  // stampedes with large blobs. With a typical scale-out infrastructure, CPU and query
1507  // load from $callback invocations is distributed among appservers and replica DBs,
1508  // but cache operations for a given key route to a single cache server (e.g. striped
1509  // consistent hashing).
1510  if ( $lockTSE < 0 || $hasLock ) {
1511  return true; // either not a priori hot or thread has the lock
1512  } elseif ( $elapsed <= self::$SET_DELAY_HIGH_MS * 1e3 ) {
1513  return true; // not enough time for threads to pile up
1514  }
1515 
1516  $this->cache->clearLastError();
1517  if (
1518  !$this->cache->add( self::$COOLOFF_KEY_PREFIX . $key, 1, self::$COOLOFF_TTL ) &&
1519  // Don't treat failures due to I/O errors as the key being in cooloff
1520  $this->cache->getLastError() === BagOStuff::ERR_NONE
1521  ) {
1522  $this->stats->increment( "wanobjectcache.$kClass.cooloff_bounce" );
1523 
1524  return false;
1525  }
1526 
1527  return true;
1528  }
1529 
1538  private function resolveCTL( $value, $curTTL, $curInfo, $touchedCallback ) {
1539  if ( $touchedCallback === null || $value === false ) {
1540  return [ $curTTL, max( $curInfo['tombAsOf'], $curInfo['lastCKPurge'] ) ];
1541  }
1542 
1543  $touched = $touchedCallback( $value );
1544  if ( $touched !== null && $touched >= $curInfo['asOf'] ) {
1545  $curTTL = min( $curTTL, self::$TINY_NEGATIVE, $curInfo['asOf'] - $touched );
1546  }
1547 
1548  return [ $curTTL, max( $curInfo['tombAsOf'], $curInfo['lastCKPurge'], $touched ) ];
1549  }
1550 
1558  private function resolveTouched( $value, $lastPurge, $touchedCallback ) {
1559  return ( $touchedCallback === null || $value === false )
1560  ? $lastPurge // nothing to derive the "touched timestamp" from
1561  : max( $touchedCallback( $value ), $lastPurge );
1562  }
1563 
1569  private function getInterimValue( $key, $minAsOf ) {
1570  $now = $this->getCurrentTime();
1571 
1572  if ( $this->useInterimHoldOffCaching ) {
1573  $wrapped = $this->cache->get( self::$INTERIM_KEY_PREFIX . $key );
1574 
1575  list( $value, $keyInfo ) = $this->unwrap( $wrapped, $now );
1576  if ( $this->isValid( $value, $keyInfo['asOf'], $minAsOf ) ) {
1577  return [ $value, $keyInfo ];
1578  }
1579  }
1580 
1581  return $this->unwrap( false, $now );
1582  }
1583 
1591  private function setInterimValue( $key, $value, $ttl, $version, $walltime ) {
1592  $ttl = max( self::$INTERIM_KEY_TTL, (int)$ttl );
1593 
1594  $wrapped = $this->wrap( $value, $ttl, $version, $this->getCurrentTime(), $walltime );
1595  $this->cache->merge(
1596  self::$INTERIM_KEY_PREFIX . $key,
1597  function () use ( $wrapped ) {
1598  return $wrapped;
1599  },
1600  $ttl,
1601  1
1602  );
1603  }
1604 
1609  private function resolveBusyValue( $busyValue ) {
1610  return ( $busyValue instanceof Closure ) ? $busyValue() : $busyValue;
1611  }
1612 
1679  final public function getMultiWithSetCallback(
1680  ArrayIterator $keyedIds, $ttl, callable $callback, array $opts = []
1681  ) {
1682  // Load required keys into process cache in one go
1683  $this->warmupCache = $this->getRawKeysForWarmup(
1684  $this->getNonProcessCachedMultiKeys( $keyedIds, $opts ),
1685  $opts['checkKeys'] ?? []
1686  );
1687  $this->warmupKeyMisses = 0;
1688 
1689  // Wrap $callback to match the getWithSetCallback() format while passing $id to $callback
1690  $id = null; // current entity ID
1691  $func = function ( $oldValue, &$ttl, &$setOpts, $oldAsOf ) use ( $callback, &$id ) {
1692  return $callback( $id, $oldValue, $ttl, $setOpts, $oldAsOf );
1693  };
1694 
1695  $values = [];
1696  foreach ( $keyedIds as $key => $id ) { // preserve order
1697  $values[$key] = $this->getWithSetCallback( $key, $ttl, $func, $opts );
1698  }
1699 
1700  $this->warmupCache = [];
1701 
1702  return $values;
1703  }
1704 
1770  final public function getMultiWithUnionSetCallback(
1771  ArrayIterator $keyedIds, $ttl, callable $callback, array $opts = []
1772  ) {
1773  $checkKeys = $opts['checkKeys'] ?? [];
1774  unset( $opts['lockTSE'] ); // incompatible
1775  unset( $opts['busyValue'] ); // incompatible
1776 
1777  // Load required keys into process cache in one go
1778  $keysByIdGet = $this->getNonProcessCachedMultiKeys( $keyedIds, $opts );
1779  $this->warmupCache = $this->getRawKeysForWarmup( $keysByIdGet, $checkKeys );
1780  $this->warmupKeyMisses = 0;
1781 
1782  // IDs of entities known to be in need of regeneration
1783  $idsRegen = [];
1784 
1785  // Find out which keys are missing/deleted/stale
1786  $curTTLs = [];
1787  $asOfs = [];
1788  $curByKey = $this->getMulti( $keysByIdGet, $curTTLs, $checkKeys, $asOfs );
1789  foreach ( $keysByIdGet as $id => $key ) {
1790  if ( !array_key_exists( $key, $curByKey ) || $curTTLs[$key] < 0 ) {
1791  $idsRegen[] = $id;
1792  }
1793  }
1794 
1795  // Run the callback to populate the regeneration value map for all required IDs
1796  $newSetOpts = [];
1797  $newTTLsById = array_fill_keys( $idsRegen, $ttl );
1798  $newValsById = $idsRegen ? $callback( $idsRegen, $newTTLsById, $newSetOpts ) : [];
1799 
1800  // Wrap $callback to match the getWithSetCallback() format while passing $id to $callback
1801  $id = null; // current entity ID
1802  $func = function ( $oldValue, &$ttl, &$setOpts, $oldAsOf )
1803  use ( $callback, &$id, $newValsById, $newTTLsById, $newSetOpts )
1804  {
1805  if ( array_key_exists( $id, $newValsById ) ) {
1806  // Value was already regerated as expected, so use the value in $newValsById
1807  $newValue = $newValsById[$id];
1808  $ttl = $newTTLsById[$id];
1809  $setOpts = $newSetOpts;
1810  } else {
1811  // Pre-emptive/popularity refresh and version mismatch cases are not detected
1812  // above and thus $newValsById has no entry. Run $callback on this single entity.
1813  $ttls = [ $id => $ttl ];
1814  $newValue = $callback( [ $id ], $ttls, $setOpts )[$id];
1815  $ttl = $ttls[$id];
1816  }
1817 
1818  return $newValue;
1819  };
1820 
1821  // Run the cache-aside logic using warmupCache instead of persistent cache queries
1822  $values = [];
1823  foreach ( $keyedIds as $key => $id ) { // preserve order
1824  $values[$key] = $this->getWithSetCallback( $key, $ttl, $func, $opts );
1825  }
1826 
1827  $this->warmupCache = [];
1828 
1829  return $values;
1830  }
1831 
1844  final public function reap( $key, $purgeTimestamp, &$isStale = false ) {
1845  $minAsOf = $purgeTimestamp + self::HOLDOFF_TTL;
1846  $wrapped = $this->cache->get( self::$VALUE_KEY_PREFIX . $key );
1847  if ( is_array( $wrapped ) && $wrapped[self::$FLD_TIME] < $minAsOf ) {
1848  $isStale = true;
1849  $this->logger->warning( "Reaping stale value key '$key'." );
1850  $ttlReap = self::HOLDOFF_TTL; // avoids races with tombstone creation
1851  $ok = $this->cache->changeTTL( self::$VALUE_KEY_PREFIX . $key, $ttlReap );
1852  if ( !$ok ) {
1853  $this->logger->error( "Could not complete reap of key '$key'." );
1854  }
1855 
1856  return $ok;
1857  }
1858 
1859  $isStale = false;
1860 
1861  return true;
1862  }
1863 
1873  final public function reapCheckKey( $key, $purgeTimestamp, &$isStale = false ) {
1874  $purge = $this->parsePurgeValue( $this->cache->get( self::$TIME_KEY_PREFIX . $key ) );
1875  if ( $purge && $purge[self::$PURGE_TIME] < $purgeTimestamp ) {
1876  $isStale = true;
1877  $this->logger->warning( "Reaping stale check key '$key'." );
1878  $ok = $this->cache->changeTTL( self::$TIME_KEY_PREFIX . $key, self::TTL_SECOND );
1879  if ( !$ok ) {
1880  $this->logger->error( "Could not complete reap of check key '$key'." );
1881  }
1882 
1883  return $ok;
1884  }
1885 
1886  $isStale = false;
1887 
1888  return false;
1889  }
1890 
1898  public function makeKey( $class, ...$components ) {
1899  return $this->cache->makeKey( ...func_get_args() );
1900  }
1901 
1909  public function makeGlobalKey( $class, ...$components ) {
1910  return $this->cache->makeGlobalKey( ...func_get_args() );
1911  }
1912 
1920  public function hash256( $component ) {
1921  return hash_hmac( 'sha256', $component, $this->secret );
1922  }
1923 
1974  final public function makeMultiKeys( array $ids, $keyCallback ) {
1975  $idByKey = [];
1976  foreach ( $ids as $id ) {
1977  // Discourage triggering of automatic makeKey() hashing in some backends
1978  if ( strlen( $id ) > 64 ) {
1979  $this->logger->warning( __METHOD__ . ": long ID '$id'; use hash256()" );
1980  }
1981  $key = $keyCallback( $id, $this );
1982  // Edge case: ignore key collisions due to duplicate $ids like "42" and 42
1983  if ( !isset( $idByKey[$key] ) ) {
1984  $idByKey[$key] = $id;
1985  } elseif ( (string)$id !== (string)$idByKey[$key] ) {
1986  throw new UnexpectedValueException(
1987  "Cache key collision; IDs ('$id','{$idByKey[$key]}') map to '$key'"
1988  );
1989  }
1990  }
1991 
1992  return new ArrayIterator( $idByKey );
1993  }
1994 
2030  final public function multiRemap( array $ids, array $res ) {
2031  if ( count( $ids ) !== count( $res ) ) {
2032  // If makeMultiKeys() is called on a list of non-unique IDs, then the resulting
2033  // ArrayIterator will have less entries due to "first appearance" de-duplication
2034  $ids = array_keys( array_flip( $ids ) );
2035  if ( count( $ids ) !== count( $res ) ) {
2036  throw new UnexpectedValueException( "Multi-key result does not match ID list" );
2037  }
2038  }
2039 
2040  return array_combine( $ids, $res );
2041  }
2042 
2047  final public function getLastError() {
2048  $code = $this->cache->getLastError();
2049  switch ( $code ) {
2050  case BagOStuff::ERR_NONE:
2051  return self::ERR_NONE;
2053  return self::ERR_NO_RESPONSE;
2055  return self::ERR_UNREACHABLE;
2056  default:
2057  return self::ERR_UNEXPECTED;
2058  }
2059  }
2060 
2064  final public function clearLastError() {
2065  $this->cache->clearLastError();
2066  }
2067 
2073  public function clearProcessCache() {
2074  $this->processCaches = [];
2075  }
2076 
2097  final public function useInterimHoldOffCaching( $enabled ) {
2098  $this->useInterimHoldOffCaching = $enabled;
2099  }
2100 
2106  public function getQoS( $flag ) {
2107  return $this->cache->getQoS( $flag );
2108  }
2109 
2173  public function adaptiveTTL( $mtime, $maxTTL, $minTTL = 30, $factor = 0.2 ) {
2174  if ( is_float( $mtime ) || ctype_digit( $mtime ) ) {
2175  $mtime = (int)$mtime; // handle fractional seconds and string integers
2176  }
2177 
2178  if ( !is_int( $mtime ) || $mtime <= 0 ) {
2179  return $minTTL; // no last-modified time provided
2180  }
2181 
2182  $age = $this->getCurrentTime() - $mtime;
2183 
2184  return (int)min( $maxTTL, max( $minTTL, $factor * $age ) );
2185  }
2186 
2191  final public function getWarmupKeyMisses() {
2192  return $this->warmupKeyMisses;
2193  }
2194 
2205  protected function relayPurge( $key, $ttl, $holdoff ) {
2206  if ( $this->mcrouterAware ) {
2207  // See https://github.com/facebook/mcrouter/wiki/Multi-cluster-broadcast-setup
2208  // Wildcards select all matching routes, e.g. the WAN cluster on all DCs
2209  $ok = $this->cache->set(
2210  "/*/{$this->cluster}/{$key}",
2211  $this->makePurgeValue( $this->getCurrentTime(), $holdoff ),
2212  $ttl
2213  );
2214  } else {
2215  // Some other proxy handles broadcasting or there is only one datacenter
2216  $ok = $this->cache->set(
2217  $key,
2218  $this->makePurgeValue( $this->getCurrentTime(), $holdoff ),
2219  $ttl
2220  );
2221  }
2222 
2223  return $ok;
2224  }
2225 
2232  protected function relayDelete( $key ) {
2233  if ( $this->mcrouterAware ) {
2234  // See https://github.com/facebook/mcrouter/wiki/Multi-cluster-broadcast-setup
2235  // Wildcards select all matching routes, e.g. the WAN cluster on all DCs
2236  $ok = $this->cache->delete( "/*/{$this->cluster}/{$key}" );
2237  } else {
2238  // Some other proxy handles broadcasting or there is only one datacenter
2239  $ok = $this->cache->delete( $key );
2240  }
2241 
2242  return $ok;
2243  }
2244 
2253  private function scheduleAsyncRefresh( $key, $ttl, $callback, $opts ) {
2254  if ( !$this->asyncHandler ) {
2255  return false;
2256  }
2257  // Update the cache value later, such during post-send of an HTTP request
2258  $func = $this->asyncHandler;
2259  $func( function () use ( $key, $ttl, $callback, $opts ) {
2260  $opts['minAsOf'] = INF; // force a refresh
2261  $this->fetchOrRegenerate( $key, $ttl, $callback, $opts );
2262  } );
2263 
2264  return true;
2265  }
2266 
2280  private function isAliveOrInGracePeriod( $curTTL, $graceTTL ) {
2281  if ( $curTTL > 0 ) {
2282  return true;
2283  } elseif ( $graceTTL <= 0 ) {
2284  return false;
2285  }
2286 
2287  $ageStale = abs( $curTTL ); // seconds of staleness
2288  $curGTTL = ( $graceTTL - $ageStale ); // current grace-time-to-live
2289  if ( $curGTTL <= 0 ) {
2290  return false; // already out of grace period
2291  }
2292 
2293  // Chance of using a stale value is the complement of the chance of refreshing it
2294  return !$this->worthRefreshExpiring( $curGTTL, $graceTTL );
2295  }
2296 
2310  protected function worthRefreshExpiring( $curTTL, $lowTTL ) {
2311  if ( $lowTTL <= 0 ) {
2312  return false;
2313  } elseif ( $curTTL >= $lowTTL ) {
2314  return false;
2315  } elseif ( $curTTL <= 0 ) {
2316  return false;
2317  }
2318 
2319  $chance = ( 1 - $curTTL / $lowTTL );
2320 
2321  return mt_rand( 1, 1e9 ) <= 1e9 * $chance;
2322  }
2323 
2339  protected function worthRefreshPopular( $asOf, $ageNew, $timeTillRefresh, $now ) {
2340  if ( $ageNew < 0 || $timeTillRefresh <= 0 ) {
2341  return false;
2342  }
2343 
2344  $age = $now - $asOf;
2345  $timeOld = $age - $ageNew;
2346  if ( $timeOld <= 0 ) {
2347  return false;
2348  }
2349 
2350  $popularHitsPerSec = 1;
2351  // Lifecycle is: new, ramp-up refresh chance, full refresh chance.
2352  // Note that the "expected # of refreshes" for the ramp-up time range is half
2353  // of what it would be if P(refresh) was at its full value during that time range.
2354  $refreshWindowSec = max( $timeTillRefresh - $ageNew - self::$RAMPUP_TTL / 2, 1 );
2355  // P(refresh) * (# hits in $refreshWindowSec) = (expected # of refreshes)
2356  // P(refresh) * ($refreshWindowSec * $popularHitsPerSec) = 1 (by definition)
2357  // P(refresh) = 1/($refreshWindowSec * $popularHitsPerSec)
2358  $chance = 1 / ( $popularHitsPerSec * $refreshWindowSec );
2359 
2360  // Ramp up $chance from 0 to its nominal value over RAMPUP_TTL seconds to avoid stampedes
2361  $chance *= ( $timeOld <= self::$RAMPUP_TTL ) ? $timeOld / self::$RAMPUP_TTL : 1;
2362 
2363  return mt_rand( 1, 1e9 ) <= 1e9 * $chance;
2364  }
2365 
2375  protected function isValid( $value, $asOf, $minAsOf, $purgeTime = null ) {
2376  // Avoid reading any key not generated after the latest delete() or touch
2377  $safeMinAsOf = max( $minAsOf, $purgeTime + self::$TINY_POSTIVE );
2378 
2379  if ( $value === false ) {
2380  return false;
2381  } elseif ( $safeMinAsOf > 0 && $asOf < $minAsOf ) {
2382  return false;
2383  }
2384 
2385  return true;
2386  }
2387 
2396  private function wrap( $value, $ttl, $version, $now, $walltime ) {
2397  // Returns keys in ascending integer order for PHP7 array packing:
2398  // https://nikic.github.io/2014/12/22/PHPs-new-hashtable-implementation.html
2399  $wrapped = [
2400  self::$FLD_FORMAT_VERSION => self::$VERSION,
2401  self::$FLD_VALUE => $value,
2402  self::$FLD_TTL => $ttl,
2403  self::$FLD_TIME => $now
2404  ];
2405  if ( $version !== null ) {
2406  $wrapped[self::$FLD_VALUE_VERSION] = $version;
2407  }
2408  if ( $walltime >= self::$GENERATION_SLOW_SEC ) {
2409  $wrapped[self::$FLD_GENERATION_TIME] = $walltime;
2410  }
2411 
2412  return $wrapped;
2413  }
2414 
2425  private function unwrap( $wrapped, $now ) {
2426  $value = false;
2427  $info = [ 'asOf' => null, 'curTTL' => null, 'version' => null, 'tombAsOf' => null ];
2428 
2429  if ( is_array( $wrapped ) ) {
2430  // Entry expected to be a cached value; validate it
2431  if (
2432  ( $wrapped[self::$FLD_FORMAT_VERSION] ?? null ) === self::$VERSION &&
2433  $wrapped[self::$FLD_TIME] >= $this->epoch
2434  ) {
2435  if ( $wrapped[self::$FLD_TTL] > 0 ) {
2436  // Get the approximate time left on the key
2437  $age = $now - $wrapped[self::$FLD_TIME];
2438  $curTTL = max( $wrapped[self::$FLD_TTL] - $age, 0.0 );
2439  } else {
2440  // Key had no TTL, so the time left is unbounded
2441  $curTTL = INF;
2442  }
2443  $value = $wrapped[self::$FLD_VALUE];
2444  $info['version'] = $wrapped[self::$FLD_VALUE_VERSION] ?? null;
2445  $info['asOf'] = $wrapped[self::$FLD_TIME];
2446  $info['curTTL'] = $curTTL;
2447  }
2448  } else {
2449  // Entry expected to be a tombstone; parse it
2450  $purge = $this->parsePurgeValue( $wrapped );
2451  if ( $purge !== false ) {
2452  // Tombstoned keys should always have a negative current $ttl
2453  $info['curTTL'] = min( $purge[self::$PURGE_TIME] - $now, self::$TINY_NEGATIVE );
2454  $info['tombAsOf'] = $purge[self::$PURGE_TIME];
2455  }
2456  }
2457 
2458  return [ $value, $info ];
2459  }
2460 
2466  protected static function prefixCacheKeys( array $keys, $prefix ) {
2467  $res = [];
2468  foreach ( $keys as $key ) {
2469  $res[] = $prefix . $key;
2470  }
2471 
2472  return $res;
2473  }
2474 
2479  private function determineKeyClassForStats( $key ) {
2480  $parts = explode( ':', $key, 3 );
2481 
2482  return $parts[1] ?? $parts[0]; // sanity
2483  }
2484 
2490  private function parsePurgeValue( $value ) {
2491  if ( !is_string( $value ) ) {
2492  return false;
2493  }
2494 
2495  $segments = explode( ':', $value, 3 );
2496  if (
2497  !isset( $segments[0] ) ||
2498  !isset( $segments[1] ) ||
2499  "{$segments[0]}:" !== self::$PURGE_VAL_PREFIX
2500  ) {
2501  return false;
2502  }
2503 
2504  if ( !isset( $segments[2] ) ) {
2505  // Back-compat with old purge values without holdoff
2506  $segments[2] = self::HOLDOFF_TTL;
2507  }
2508 
2509  if ( $segments[1] < $this->epoch ) {
2510  // Values this old are ignored
2511  return false;
2512  }
2513 
2514  return [
2515  self::$PURGE_TIME => (float)$segments[1],
2516  self::$PURGE_HOLDOFF => (int)$segments[2],
2517  ];
2518  }
2519 
2525  private function makePurgeValue( $timestamp, $holdoff ) {
2526  return self::$PURGE_VAL_PREFIX . (float)$timestamp . ':' . (int)$holdoff;
2527  }
2528 
2533  private function getProcessCache( $group ) {
2534  if ( !isset( $this->processCaches[$group] ) ) {
2535  list( , $size ) = explode( ':', $group );
2536  $this->processCaches[$group] = new MapCacheLRU( (int)$size );
2537  }
2538 
2539  return $this->processCaches[$group];
2540  }
2541 
2547  private function getProcessCacheKey( $key, $version ) {
2548  return $key . ' ' . (int)$version;
2549  }
2550 
2556  private function getNonProcessCachedMultiKeys( ArrayIterator $keys, array $opts ) {
2557  $pcTTL = $opts['pcTTL'] ?? self::TTL_UNCACHEABLE;
2558 
2559  $keysMissing = [];
2560  if ( $pcTTL > 0 && $this->callbackDepth == 0 ) {
2561  $version = $opts['version'] ?? null;
2562  $pCache = $this->getProcessCache( $opts['pcGroup'] ?? self::PC_PRIMARY );
2563  foreach ( $keys as $key => $id ) {
2564  if ( !$pCache->has( $this->getProcessCacheKey( $key, $version ), $pcTTL ) ) {
2565  $keysMissing[$id] = $key;
2566  }
2567  }
2568  }
2569 
2570  return $keysMissing;
2571  }
2572 
2578  private function getRawKeysForWarmup( array $keys, array $checkKeys ) {
2579  if ( !$keys ) {
2580  return [];
2581  }
2582 
2583  $keysWarmUp = [];
2584  // Get all the value keys to fetch...
2585  foreach ( $keys as $key ) {
2586  $keysWarmUp[] = self::$VALUE_KEY_PREFIX . $key;
2587  }
2588  // Get all the check keys to fetch...
2589  foreach ( $checkKeys as $i => $checkKeyOrKeys ) {
2590  if ( is_int( $i ) ) {
2591  // Single check key that applies to all value keys
2592  $keysWarmUp[] = self::$TIME_KEY_PREFIX . $checkKeyOrKeys;
2593  } else {
2594  // List of check keys that apply to value key $i
2595  $keysWarmUp = array_merge(
2596  $keysWarmUp,
2597  self::prefixCacheKeys( $checkKeyOrKeys, self::$TIME_KEY_PREFIX )
2598  );
2599  }
2600  }
2601 
2602  $warmupCache = $this->cache->getMulti( $keysWarmUp );
2603  $warmupCache += array_fill_keys( $keysWarmUp, false );
2604 
2605  return $warmupCache;
2606  }
2607 
2612  protected function getCurrentTime() {
2613  if ( $this->wallClockOverride ) {
2614  return $this->wallClockOverride;
2615  }
2616 
2617  $clockTime = (float)time(); // call this first
2618  // microtime() uses an initial gettimeofday() call added to usage clocks.
2619  // This can severely drift from time() and the microtime() value of other threads
2620  // due to undercounting of the amount of time elapsed. Instead of seeing the current
2621  // time as being in the past, use the value of time(). This avoids setting cache values
2622  // that will immediately be seen as expired and possibly cause stampedes.
2623  return max( microtime( true ), $clockTime );
2624  }
2625 
2630  public function setMockTime( &$time ) {
2631  $this->wallClockOverride =& $time;
2632  $this->cache->setMockTime( $time );
2633  }
2634 }
processCheckKeys(array $timeKeys, array $wrappedValues, $now)
static int $VERSION
Cache format version number.
string $region
Physical region for mcrouter use.
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition: deferred.txt:11
makePurgeValue( $timestamp, $holdoff)
reapCheckKey( $key, $purgeTimestamp, &$isStale=false)
Set a "check" key to soon expire in the local cluster if it pre-dates $purgeTimestamp.
static $COOLOFF_KEY_PREFIX
static int $INTERIM_KEY_TTL
Seconds to keep interim value keys for tombstoned keys around.
processing should stop and the error should be shown to the user * false
Definition: hooks.txt:193
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
clearProcessCache()
Clear the in-process caches; useful for testing.
static prefixCacheKeys(array $keys, $prefix)
static int $FLD_VALUE_VERSION
Key to collection cache version number.
determineKeyClassForStats( $key)
relayDelete( $key)
Do the actual async bus delete of a key.
makeMultiKeys(array $ids, $keyCallback)
Get an iterator of (cache key => entity ID) for a list of entity IDs.
LoggerInterface $logger
mixed [] $warmupCache
Temporary warm-up cache.
static int $FLD_GENERATION_TIME
Key to how long it took to generate the value.
getMultiWithSetCallback(ArrayIterator $keyedIds, $ttl, callable $callback, array $opts=[])
Method to fetch multiple cache keys at once with regeneration.
This code would result in ircNotify being run twice when an article is and once for brion Hooks can return three possible true was required This is the default since MediaWiki *some string
Definition: hooks.txt:181
static float $TINY_NEGATIVE
Tiny negative float to use when CTL comes up >= 0 due to clock skew.
$value
useInterimHoldOffCaching( $enabled)
Enable or disable the use of brief caching for tombstoned keys.
string $secret
Stable secret used for hasing long strings into key components.
scheduleAsyncRefresh( $key, $ttl, $callback, $opts)
getWithSetCallback( $key, $ttl, $callback, array $opts=[])
Method to fetch/regenerate cache keys.
see documentation in includes Linker php for Linker::makeImageLink & $time
Definition: hooks.txt:1792
hash256( $component)
Hash a possibly long string into a suitable component for makeKey()/makeGlobalKey() ...
worthRefreshPopular( $asOf, $ageNew, $timeTillRefresh, $now)
Check if a key is due for randomized regeneration due to its popularity.
The index of the header message $result[1]=The index of the body text message $result[2 through n]=Parameters passed to body text message. Please note the header message cannot receive/use parameters. 'ImgAuthModifyHeaders':Executed just before a file is streamed to a user via img_auth.php, allowing headers to be modified beforehand. $title:LinkTarget object & $headers:HTTP headers(name=> value, names are case insensitive). Two headers get special handling:If-Modified-Since(value must be a valid HTTP date) and Range(must be of the form "bytes=(\*-\*)") will be honored when streaming the file. 'ImportHandleLogItemXMLTag':When parsing a XML tag in a log item. Return false to stop further processing of the tag $reader:XMLReader object $logInfo:Array of information 'ImportHandlePageXMLTag':When parsing a XML tag in a page. Return false to stop further processing of the tag $reader:XMLReader object & $pageInfo:Array of information 'ImportHandleRevisionXMLTag':When parsing a XML tag in a page revision. Return false to stop further processing of the tag $reader:XMLReader object $pageInfo:Array of page information $revisionInfo:Array of revision information 'ImportHandleToplevelXMLTag':When parsing a top level XML tag. Return false to stop further processing of the tag $reader:XMLReader object 'ImportHandleUnknownUser':When a user doesn 't exist locally, this hook is called to give extensions an opportunity to auto-create it. If the auto-creation is successful, return false. $name:User name 'ImportHandleUploadXMLTag':When parsing a XML tag in a file upload. Return false to stop further processing of the tag $reader:XMLReader object $revisionInfo:Array of information 'ImportLogInterwikiLink':Hook to change the interwiki link used in log entries and edit summaries for transwiki imports. & $fullInterwikiPrefix:Interwiki prefix, may contain colons. & $pageTitle:String that contains page title. 'ImportSources':Called when reading from the $wgImportSources configuration variable. Can be used to lazy-load the import sources list. & $importSources:The value of $wgImportSources. Modify as necessary. See the comment in DefaultSettings.php for the detail of how to structure this array. 'InfoAction':When building information to display on the action=info page. $context:IContextSource object & $pageInfo:Array of information 'InitializeArticleMaybeRedirect':MediaWiki check to see if title is a redirect. & $title:Title object for the current page & $request:WebRequest & $ignoreRedirect:boolean to skip redirect check & $target:Title/string of redirect target & $article:Article object 'InternalParseBeforeLinks':during Parser 's internalParse method before links but after nowiki/noinclude/includeonly/onlyinclude and other processings. & $parser:Parser object & $text:string containing partially parsed text & $stripState:Parser 's internal StripState object 'InternalParseBeforeSanitize':during Parser 's internalParse method just before the parser removes unwanted/dangerous HTML tags and after nowiki/noinclude/includeonly/onlyinclude and other processings. Ideal for syntax-extensions after template/parser function execution which respect nowiki and HTML-comments. & $parser:Parser object & $text:string containing partially parsed text & $stripState:Parser 's internal StripState object 'InterwikiLoadPrefix':When resolving if a given prefix is an interwiki or not. Return true without providing an interwiki to continue interwiki search. $prefix:interwiki prefix we are looking for. & $iwData:output array describing the interwiki with keys iw_url, iw_local, iw_trans and optionally iw_api and iw_wikiid. 'InvalidateEmailComplete':Called after a user 's email has been invalidated successfully. $user:user(object) whose email is being invalidated 'IRCLineURL':When constructing the URL to use in an IRC notification. Callee may modify $url and $query, URL will be constructed as $url . $query & $url:URL to index.php & $query:Query string $rc:RecentChange object that triggered url generation 'IsFileCacheable':Override the result of Article::isFileCacheable()(if true) & $article:article(object) being checked 'IsTrustedProxy':Override the result of IP::isTrustedProxy() & $ip:IP being check & $result:Change this value to override the result of IP::isTrustedProxy() 'IsUploadAllowedFromUrl':Override the result of UploadFromUrl::isAllowedUrl() $url:URL used to upload from & $allowed:Boolean indicating if uploading is allowed for given URL 'isValidEmailAddr':Override the result of Sanitizer::validateEmail(), for instance to return false if the domain name doesn 't match your organization. $addr:The e-mail address entered by the user & $result:Set this and return false to override the internal checks 'isValidPassword':Override the result of User::isValidPassword() $password:The password entered by the user & $result:Set this and return false to override the internal checks $user:User the password is being validated for 'Language::getMessagesFileName':$code:The language code or the language we 're looking for a messages file for & $file:The messages file path, you can override this to change the location. 'LanguageGetNamespaces':Provide custom ordering for namespaces or remove namespaces. Do not use this hook to add namespaces. Use CanonicalNamespaces for that. & $namespaces:Array of namespaces indexed by their numbers 'LanguageGetTranslatedLanguageNames':Provide translated language names. & $names:array of language code=> language name $code:language of the preferred translations 'LanguageLinks':Manipulate a page 's language links. This is called in various places to allow extensions to define the effective language links for a page. $title:The page 's Title. & $links:Array with elements of the form "language:title" in the order that they will be output. & $linkFlags:Associative array mapping prefixed links to arrays of flags. Currently unused, but planned to provide support for marking individual language links in the UI, e.g. for featured articles. 'LanguageSelector':Hook to change the language selector available on a page. $out:The output page. $cssClassName:CSS class name of the language selector. 'LinkBegin':DEPRECATED since 1.28! Use HtmlPageLinkRendererBegin instead. Used when generating internal and interwiki links in Linker::link(), before processing starts. Return false to skip default processing and return $ret. See documentation for Linker::link() for details on the expected meanings of parameters. $skin:the Skin object $target:the Title that the link is pointing to & $html:the contents that the< a > tag should have(raw HTML) $result
Definition: hooks.txt:1981
static int $RECENT_SET_LOW_MS
Min millisecond set() backoff during hold-off (far less than INTERIM_KEY_TTL)
checkAndSetCooloff( $key, $kClass, $elapsed, $lockTSE, $hasLock)
you have access to all of the normal MediaWiki so you can get a DB use the cache
Definition: maintenance.txt:52
resolveBusyValue( $busyValue)
getMultiWithUnionSetCallback(ArrayIterator $keyedIds, $ttl, callable $callback, array $opts=[])
Method to fetch/regenerate multiple cache keys at once.
float $epoch
Unix timestamp of the oldest possible valid values.
int $warmupKeyMisses
Key fetched.
static newEmpty()
Get an instance that wraps EmptyBagOStuff.
static $VALUE_KEY_PREFIX
string $cluster
Cache cluster name for mcrouter use.
makeKey( $class,... $components)
worthRefreshExpiring( $curTTL, $lowTTL)
Check if a key is nearing expiration and thus due for randomized regeneration.
getInterimValue( $key, $minAsOf)
getLastError()
Get the "last error" registered; clearLastError() should be called manually.
static int $FLD_FORMAT_VERSION
Key to WAN cache version number.
getMultiCheckKeyTime(array $keys)
Fetch the values of each timestamp "check" key.
$res
Definition: database.txt:21
__construct(array $params)
reap( $key, $purgeTimestamp, &$isStale=false)
Set a key to soon expire in the local cluster if it pre-dates $purgeTimestamp.
wrap( $value, $ttl, $version, $now, $walltime)
static int $SET_DELAY_HIGH_MS
Milliseconds of key fetch/validate/regenerate delay prone to set() stampedes.
$params
getMulti(array $keys, &$curTTLs=[], array $checkKeys=[], &$info=null)
Fetch the value of several keys from cache.
static int $CHECK_KEY_TTL
Seconds to keep dependency purge keys around.
unwrap( $wrapped, $now)
adaptiveTTL( $mtime, $maxTTL, $minTTL=30, $factor=0.2)
Get a TTL that is higher for objects that have not changed recently.
static $MUTEX_KEY_PREFIX
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that When $user is not null
Definition: hooks.txt:773
relayPurge( $key, $ttl, $holdoff)
Do the actual async bus purge of a key.
float null $wallClockOverride
getProcessCache( $group)
multiRemap(array $ids, array $res)
Get an (ID => value) map from (i) a non-unique list of entity IDs, and (ii) the list of corresponding...
static int $COOLOFF_TTL
Seconds to no-op key set() calls to avoid large blob I/O stampedes.
getCheckKeyTime( $key)
Fetch the value of a timestamp "check" key.
int $callbackDepth
Callback stack depth for getWithSetCallback()
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that When $user is not it can be in the form of< username >< more info > e g for bot passwords intended to be added to log contexts Fields it might only if the login was with a bot password it is not rendered in wiki pages or galleries in category pages allow injecting custom HTML after the section Any uses of the hook need to handle escaping see BaseTemplate::getToolbox and BaseTemplate::makeListItem for details on the format of individual items inside of this array or by returning and letting standard HTTP rendering take place modifiable or by returning false and taking over the output modifiable & $code
Definition: hooks.txt:773
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
static int $FLD_VALUE
Key to the cached value.
resetCheckKey( $key)
Delete a "check" key from all datacenters, invalidating keys that use it.
touchCheckKey( $key, $holdoff=self::HOLDOFF_TTL)
Purge a "check" key from all datacenters, invalidating keys that use it.
getNonProcessCachedMultiKeys(ArrayIterator $keys, array $opts)
static $PURGE_VAL_PREFIX
clearLastError()
Clear the "last error" registry.
callable null $asyncHandler
Function that takes a WAN cache callback and runs it later.
yieldStampedeLock( $key, $hasLock)
isVolatileValueAgeNegligible( $age)
getRawKeysForWarmup(array $keys, array $checkKeys)
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
resolveCTL( $value, $curTTL, $curInfo, $touchedCallback)
isAliveOrInGracePeriod( $curTTL, $graceTTL)
Check if a key is fresh or in the grace window and thus due for randomized reuse. ...
static int $FLD_TTL
Key to the original TTL.
bool $useInterimHoldOffCaching
Whether to use "interim" caching while keys are tombstoned.
static int $FLD_TIME
Key to the cache timestamp.
static int $RECENT_SET_HIGH_MS
Max millisecond set() backoff during hold-off (far less than INTERIM_KEY_TTL)
static float $TINY_POSTIVE
Tiny positive float to use when using "minTime" to assert an inequality.
$mcrouterAware
bool Whether to use mcrouter key prefixing for routing
static int $LOCK_TTL
Seconds to keep lock keys around.
static int $PURGE_HOLDOFF
Key to the tombstone entry hold-off TTL.
StatsdDataFactoryInterface $stats
setInterimValue( $key, $value, $ttl, $version, $walltime)
makeGlobalKey( $class,... $components)
Generic interface for object stores with key encoding methods.
MapCacheLRU [] $processCaches
Map of group PHP instance caches.
setLogger(LoggerInterface $logger)
parsePurgeValue( $value)
getProcessCacheKey( $key, $version)
static $INTERIM_KEY_PREFIX
static int $GENERATION_SLOW_SEC
Consider value generation slow if it takes more than this many seconds.
isValid( $value, $asOf, $minAsOf, $purgeTime=null)
Check if $value is not false, versioned (if needed), and not older than $minTime (if set) ...
static int $PURGE_TIME
Key to the tombstone entry timestamp.
fetchOrRegenerate( $key, $ttl, $callback, array $opts)
Do the actual I/O for getWithSetCallback() when needed.
static int $FLD_FLAGS
PhpUnusedPrivateFieldInspection
BagOStuff $cache
The local datacenter cache.
resolveTouched( $value, $lastPurge, $touchedCallback)
static int $RAMPUP_TTL
Seconds to ramp up the chance of regeneration due to expected time-till-refresh.