MediaWiki  master
WANObjectCache.php
Go to the documentation of this file.
1 <?php
26 
116 class WANObjectCache implements IExpiringStore, LoggerAwareInterface {
118  protected $cache;
120  protected $processCaches = [];
122  protected $mcrouterAware;
124  protected $region;
126  protected $cluster;
128  protected $logger;
130  protected $stats;
132  protected $useInterimHoldOffCaching = true;
134  protected $asyncHandler;
136  protected $epoch;
137 
139  private $callbackDepth = 0;
141  private $warmupCache = [];
143  private $warmupKeyMisses = 0;
144 
147 
149  const MAX_COMMIT_DELAY = 3;
151  const MAX_READ_LAG = 7;
153  const HOLDOFF_TTL = 11; // MAX_COMMIT_DELAY + MAX_READ_LAG + 1
154 
156  const CHECK_KEY_TTL = self::TTL_YEAR;
158  const INTERIM_KEY_TTL = 1;
159 
161  const LOCK_TTL = 10;
163  const COOLOFF_TTL = 1;
165  const LOW_TTL = 30;
166 
168  const AGE_NEW = 60;
170  const HOT_TTR = 900;
172  const HIT_RATE_HIGH = 1;
174  const RAMPUP_TTL = 30;
175 
177  const TTL_UNCACHEABLE = -1;
179  const TSE_NONE = -1;
181  const TTL_LAGGED = 30;
183  const HOLDOFF_NONE = 0;
185  const STALE_TTL_NONE = 0;
187  const GRACE_TTL_NONE = 0;
188 
190  const MIN_TIMESTAMP_NONE = 0.0;
191 
193  const TINY_NEGATIVE = -0.000001;
195  const TINY_POSTIVE = 0.000001;
196 
198  const SET_DELAY_HIGH_MS = 50;
200  const RECENT_SET_LOW_MS = 50;
202  const RECENT_SET_HIGH_MS = 100;
203 
205  const PASS_BY_REF = -1;
206 
208  const VERSION = 1;
209 
210  const FLD_VERSION = 0; // key to cache version number
211  const FLD_VALUE = 1; // key to the cached value
212  const FLD_TTL = 2; // key to the original TTL
213  const FLD_TIME = 3; // key to the cache time
214  const FLD_FLAGS = 4; // key to the flags bitfield (reserved number)
215  const FLD_HOLDOFF = 5; // key to any hold-off TTL
216 
217  const VALUE_KEY_PREFIX = 'WANCache:v:';
218  const INTERIM_KEY_PREFIX = 'WANCache:i:';
219  const TIME_KEY_PREFIX = 'WANCache:t:';
220  const MUTEX_KEY_PREFIX = 'WANCache:m:';
221  const COOLOFF_KEY_PREFIX = 'WANCache:c:';
222 
223  const PURGE_VAL_PREFIX = 'PURGED:';
224 
225  const VFLD_DATA = 'WOC:d'; // key to the value of versioned data
226  const VFLD_VERSION = 'WOC:v'; // key to the version of the value present
227 
228  const PC_PRIMARY = 'primary:1000'; // process cache name and max key count
229 
254  public function __construct( array $params ) {
255  $this->cache = $params['cache'];
256  $this->region = $params['region'] ?? 'main';
257  $this->cluster = $params['cluster'] ?? 'wan-main';
258  $this->mcrouterAware = !empty( $params['mcrouterAware'] );
259  $this->epoch = $params['epoch'] ?? 1.0;
260 
261  $this->setLogger( $params['logger'] ?? new NullLogger() );
262  $this->stats = $params['stats'] ?? new NullStatsdDataFactory();
263  $this->asyncHandler = $params['asyncHandler'] ?? null;
264  }
265 
269  public function setLogger( LoggerInterface $logger ) {
270  $this->logger = $logger;
271  }
272 
278  public static function newEmpty() {
279  return new static( [ 'cache' => new EmptyBagOStuff() ] );
280  }
281 
331  final public function get(
332  $key, &$curTTL = null, array $checkKeys = [], &$info = null
333  ) {
334  $curTTLs = self::PASS_BY_REF;
335  $infoByKey = self::PASS_BY_REF;
336  $values = $this->getMulti( [ $key ], $curTTLs, $checkKeys, $infoByKey );
337  $curTTL = $curTTLs[$key] ?? null;
338  if ( $info === self::PASS_BY_REF ) {
339  $info = [
340  'asOf' => $infoByKey[$key]['asOf'] ?? null,
341  'tombAsOf' => $infoByKey[$key]['tombAsOf'] ?? null,
342  'lastCKPurge' => $infoByKey[$key]['lastCKPurge'] ?? null
343  ];
344  } else {
345  $info = $infoByKey[$key]['asOf'] ?? null; // b/c
346  }
347 
348  return $values[$key] ?? false;
349  }
350 
370  final public function getMulti(
371  array $keys,
372  &$curTTLs = [],
373  array $checkKeys = [],
374  &$info = null
375  ) {
376  $result = [];
377  $curTTLs = [];
378  $infoByKey = [];
379 
380  $vPrefixLen = strlen( self::VALUE_KEY_PREFIX );
381  $valueKeys = self::prefixCacheKeys( $keys, self::VALUE_KEY_PREFIX );
382 
383  $checkKeysForAll = [];
384  $checkKeysByKey = [];
385  $checkKeysFlat = [];
386  foreach ( $checkKeys as $i => $checkKeyGroup ) {
387  $prefixed = self::prefixCacheKeys( (array)$checkKeyGroup, self::TIME_KEY_PREFIX );
388  $checkKeysFlat = array_merge( $checkKeysFlat, $prefixed );
389  // Are these check keys for a specific cache key, or for all keys being fetched?
390  if ( is_int( $i ) ) {
391  $checkKeysForAll = array_merge( $checkKeysForAll, $prefixed );
392  } else {
393  $checkKeysByKey[$i] = $prefixed;
394  }
395  }
396 
397  // Fetch all of the raw values
398  $keysGet = array_merge( $valueKeys, $checkKeysFlat );
399  if ( $this->warmupCache ) {
400  $wrappedValues = array_intersect_key( $this->warmupCache, array_flip( $keysGet ) );
401  $keysGet = array_diff( $keysGet, array_keys( $wrappedValues ) ); // keys left to fetch
402  $this->warmupKeyMisses += count( $keysGet );
403  } else {
404  $wrappedValues = [];
405  }
406  if ( $keysGet ) {
407  $wrappedValues += $this->cache->getMulti( $keysGet );
408  }
409  // Time used to compare/init "check" keys (derived after getMulti() to be pessimistic)
410  $now = $this->getCurrentTime();
411 
412  // Collect timestamps from all "check" keys
413  $purgeValuesForAll = $this->processCheckKeys( $checkKeysForAll, $wrappedValues, $now );
414  $purgeValuesByKey = [];
415  foreach ( $checkKeysByKey as $cacheKey => $checks ) {
416  $purgeValuesByKey[$cacheKey] =
417  $this->processCheckKeys( $checks, $wrappedValues, $now );
418  }
419 
420  // Get the main cache value for each key and validate them
421  foreach ( $valueKeys as $vKey ) {
422  $key = substr( $vKey, $vPrefixLen ); // unprefix
423  list( $value, $curTTL, $asOf, $tombAsOf ) = isset( $wrappedValues[$vKey] )
424  ? $this->unwrap( $wrappedValues[$vKey], $now )
425  : [ false, null, null, null ]; // not found
426  // Force dependent keys to be seen as stale for a while after purging
427  // to reduce race conditions involving stale data getting cached
428  $purgeValues = $purgeValuesForAll;
429  if ( isset( $purgeValuesByKey[$key] ) ) {
430  $purgeValues = array_merge( $purgeValues, $purgeValuesByKey[$key] );
431  }
432 
433  $lastCKPurge = null; // timestamp of the highest check key
434  foreach ( $purgeValues as $purge ) {
435  $lastCKPurge = max( $purge[self::FLD_TIME], $lastCKPurge );
436  $safeTimestamp = $purge[self::FLD_TIME] + $purge[self::FLD_HOLDOFF];
437  if ( $value !== false && $safeTimestamp >= $asOf ) {
438  // How long ago this value was invalidated by *this* check key
439  $ago = min( $purge[self::FLD_TIME] - $now, self::TINY_NEGATIVE );
440  // How long ago this value was invalidated by *any* known check key
441  $curTTL = min( $curTTL, $ago );
442  }
443  }
444 
445  if ( $value !== false ) {
446  $result[$key] = $value;
447  }
448  if ( $curTTL !== null ) {
449  $curTTLs[$key] = $curTTL;
450  }
451 
452  $infoByKey[$key] = ( $info === self::PASS_BY_REF )
453  ? [ 'asOf' => $asOf, 'tombAsOf' => $tombAsOf, 'lastCKPurge' => $lastCKPurge ]
454  : $asOf; // b/c
455  }
456 
457  $info = $infoByKey;
458 
459  return $result;
460  }
461 
469  private function processCheckKeys( array $timeKeys, array $wrappedValues, $now ) {
470  $purgeValues = [];
471  foreach ( $timeKeys as $timeKey ) {
472  $purge = isset( $wrappedValues[$timeKey] )
473  ? $this->parsePurgeValue( $wrappedValues[$timeKey] )
474  : false;
475  if ( $purge === false ) {
476  // Key is not set or malformed; regenerate
477  $newVal = $this->makePurgeValue( $now, self::HOLDOFF_TTL );
478  $this->cache->add( $timeKey, $newVal, self::CHECK_KEY_TTL );
479  $purge = $this->parsePurgeValue( $newVal );
480  }
481  $purgeValues[] = $purge;
482  }
483  return $purgeValues;
484  }
485 
554  final public function set( $key, $value, $ttl = self::TTL_INDEFINITE, array $opts = [] ) {
555  $now = $this->getCurrentTime();
556  $lockTSE = $opts['lockTSE'] ?? self::TSE_NONE;
557  $staleTTL = $opts['staleTTL'] ?? self::STALE_TTL_NONE;
558  $age = isset( $opts['since'] ) ? max( 0, $now - $opts['since'] ) : 0;
559  $creating = $opts['creating'] ?? false;
560  $lag = $opts['lag'] ?? 0;
561 
562  // Do not cache potentially uncommitted data as it might get rolled back
563  if ( !empty( $opts['pending'] ) ) {
564  $this->logger->info(
565  'Rejected set() for {cachekey} due to pending writes.',
566  [ 'cachekey' => $key ]
567  );
568 
569  return true; // no-op the write for being unsafe
570  }
571 
572  $logicalTTL = null; // logical TTL override
573  // Check if there's a risk of writing stale data after the purge tombstone expired
574  if ( $lag === false || ( $lag + $age ) > self::MAX_READ_LAG ) {
575  // Case A: any long-running transaction
576  if ( $age > self::MAX_READ_LAG ) {
577  if ( $lockTSE >= 0 ) {
578  // Store value as *almost* stale to avoid cache and mutex stampedes
579  $logicalTTL = self::TTL_SECOND;
580  $this->logger->info(
581  'Lowered set() TTL for {cachekey} due to snapshot lag.',
582  [ 'cachekey' => $key, 'lag' => $lag, 'age' => $age ]
583  );
584  } else {
585  $this->logger->info(
586  'Rejected set() for {cachekey} due to snapshot lag.',
587  [ 'cachekey' => $key, 'lag' => $lag, 'age' => $age ]
588  );
589 
590  return true; // no-op the write for being unsafe
591  }
592  // Case B: high replication lag; lower TTL instead of ignoring all set()s
593  } elseif ( $lag === false || $lag > self::MAX_READ_LAG ) {
594  if ( $lockTSE >= 0 ) {
595  $logicalTTL = min( $ttl ?: INF, self::TTL_LAGGED );
596  } else {
597  $ttl = min( $ttl ?: INF, self::TTL_LAGGED );
598  }
599  $this->logger->warning(
600  'Lowered set() TTL for {cachekey} due to replication lag.',
601  [ 'cachekey' => $key, 'lag' => $lag, 'age' => $age ]
602  );
603  // Case C: medium length request with medium replication lag
604  } elseif ( $lockTSE >= 0 ) {
605  // Store value as *almost* stale to avoid cache and mutex stampedes
606  $logicalTTL = self::TTL_SECOND;
607  $this->logger->info(
608  'Lowered set() TTL for {cachekey} due to high read lag.',
609  [ 'cachekey' => $key, 'lag' => $lag, 'age' => $age ]
610  );
611  } else {
612  $this->logger->info(
613  'Rejected set() for {cachekey} due to high read lag.',
614  [ 'cachekey' => $key, 'lag' => $lag, 'age' => $age ]
615  );
616 
617  return true; // no-op the write for being unsafe
618  }
619  }
620 
621  // Wrap that value with time/TTL/version metadata
622  $wrapped = $this->wrap( $value, $logicalTTL ?: $ttl, $now );
623  $storeTTL = $ttl + $staleTTL;
624 
625  if ( $creating ) {
626  $ok = $this->cache->add( self::VALUE_KEY_PREFIX . $key, $wrapped, $storeTTL );
627  } else {
628  $ok = $this->cache->merge(
629  self::VALUE_KEY_PREFIX . $key,
630  function ( $cache, $key, $cWrapped ) use ( $wrapped ) {
631  // A string value means that it is a tombstone; do nothing in that case
632  return ( is_string( $cWrapped ) ) ? false : $wrapped;
633  },
634  $storeTTL,
635  1 // 1 attempt
636  );
637  }
638 
639  return $ok;
640  }
641 
703  final public function delete( $key, $ttl = self::HOLDOFF_TTL ) {
704  if ( $ttl <= 0 ) {
705  // Publish the purge to all datacenters
706  $ok = $this->relayDelete( self::VALUE_KEY_PREFIX . $key );
707  } else {
708  // Publish the purge to all datacenters
709  $ok = $this->relayPurge( self::VALUE_KEY_PREFIX . $key, $ttl, self::HOLDOFF_NONE );
710  }
711 
712  $kClass = $this->determineKeyClassForStats( $key );
713  $this->stats->increment( "wanobjectcache.$kClass.delete." . ( $ok ? 'ok' : 'error' ) );
714 
715  return $ok;
716  }
717 
737  final public function getCheckKeyTime( $key ) {
738  return $this->getMultiCheckKeyTime( [ $key ] )[$key];
739  }
740 
802  final public function getMultiCheckKeyTime( array $keys ) {
803  $rawKeys = [];
804  foreach ( $keys as $key ) {
805  $rawKeys[$key] = self::TIME_KEY_PREFIX . $key;
806  }
807 
808  $rawValues = $this->cache->getMulti( $rawKeys );
809  $rawValues += array_fill_keys( $rawKeys, false );
810 
811  $times = [];
812  foreach ( $rawKeys as $key => $rawKey ) {
813  $purge = $this->parsePurgeValue( $rawValues[$rawKey] );
814  if ( $purge !== false ) {
815  $time = $purge[self::FLD_TIME];
816  } else {
817  // Casting assures identical floats for the next getCheckKeyTime() calls
818  $now = (string)$this->getCurrentTime();
819  $this->cache->add(
820  $rawKey,
821  $this->makePurgeValue( $now, self::HOLDOFF_TTL ),
822  self::CHECK_KEY_TTL
823  );
824  $time = (float)$now;
825  }
826 
827  $times[$key] = $time;
828  }
829 
830  return $times;
831  }
832 
867  final public function touchCheckKey( $key, $holdoff = self::HOLDOFF_TTL ) {
868  // Publish the purge to all datacenters
869  $ok = $this->relayPurge( self::TIME_KEY_PREFIX . $key, self::CHECK_KEY_TTL, $holdoff );
870 
871  $kClass = $this->determineKeyClassForStats( $key );
872  $this->stats->increment( "wanobjectcache.$kClass.ck_touch." . ( $ok ? 'ok' : 'error' ) );
873 
874  return $ok;
875  }
876 
904  final public function resetCheckKey( $key ) {
905  // Publish the purge to all datacenters
906  $ok = $this->relayDelete( self::TIME_KEY_PREFIX . $key );
907 
908  $kClass = $this->determineKeyClassForStats( $key );
909  $this->stats->increment( "wanobjectcache.$kClass.ck_reset." . ( $ok ? 'ok' : 'error' ) );
910 
911  return $ok;
912  }
913 
1215  final public function getWithSetCallback( $key, $ttl, $callback, array $opts = [] ) {
1216  $pcTTL = $opts['pcTTL'] ?? self::TTL_UNCACHEABLE;
1217 
1218  // Try the process cache if enabled and the cache callback is not within a cache callback.
1219  // Process cache use in nested callbacks is not lag-safe with regard to HOLDOFF_TTL since
1220  // the in-memory value is further lagged than the shared one since it uses a blind TTL.
1221  if ( $pcTTL >= 0 && $this->callbackDepth == 0 ) {
1222  $group = $opts['pcGroup'] ?? self::PC_PRIMARY;
1223  $procCache = $this->getProcessCache( $group );
1224  $value = $procCache->has( $key, $pcTTL ) ? $procCache->get( $key ) : false;
1225  } else {
1226  $procCache = false;
1227  $value = false;
1228  }
1229 
1230  if ( $value === false ) {
1231  // Fetch the value over the network
1232  if ( isset( $opts['version'] ) ) {
1233  $version = $opts['version'];
1234  $asOf = null;
1235  $cur = $this->doGetWithSetCallback(
1236  $key,
1237  $ttl,
1238  function ( $oldValue, &$ttl, &$setOpts, $oldAsOf )
1239  use ( $callback, $version ) {
1240  if ( is_array( $oldValue )
1241  && array_key_exists( self::VFLD_DATA, $oldValue )
1242  && array_key_exists( self::VFLD_VERSION, $oldValue )
1243  && $oldValue[self::VFLD_VERSION] === $version
1244  ) {
1245  $oldData = $oldValue[self::VFLD_DATA];
1246  } else {
1247  // VFLD_DATA is not set if an old, unversioned, key is present
1248  $oldData = false;
1249  $oldAsOf = null;
1250  }
1251 
1252  return [
1253  self::VFLD_DATA => $callback( $oldData, $ttl, $setOpts, $oldAsOf ),
1254  self::VFLD_VERSION => $version
1255  ];
1256  },
1257  $opts,
1258  $asOf
1259  );
1260  if ( $cur[self::VFLD_VERSION] === $version ) {
1261  // Value created or existed before with version; use it
1262  $value = $cur[self::VFLD_DATA];
1263  } else {
1264  // Value existed before with a different version; use variant key.
1265  // Reflect purges to $key by requiring that this key value be newer.
1266  $value = $this->doGetWithSetCallback(
1267  $this->makeGlobalKey( 'WANCache-key-variant', md5( $key ), $version ),
1268  $ttl,
1269  $callback,
1270  // Regenerate value if not newer than $key
1271  [ 'version' => null, 'minAsOf' => $asOf ] + $opts
1272  );
1273  }
1274  } else {
1275  $value = $this->doGetWithSetCallback( $key, $ttl, $callback, $opts );
1276  }
1277 
1278  // Update the process cache if enabled
1279  if ( $procCache && $value !== false ) {
1280  $procCache->set( $key, $value );
1281  }
1282  }
1283 
1284  return $value;
1285  }
1286 
1300  protected function doGetWithSetCallback( $key, $ttl, $callback, array $opts, &$asOf = null ) {
1301  $lowTTL = $opts['lowTTL'] ?? min( self::LOW_TTL, $ttl );
1302  $lockTSE = $opts['lockTSE'] ?? self::TSE_NONE;
1303  $staleTTL = $opts['staleTTL'] ?? self::STALE_TTL_NONE;
1304  $graceTTL = $opts['graceTTL'] ?? self::GRACE_TTL_NONE;
1305  $checkKeys = $opts['checkKeys'] ?? [];
1306  $busyValue = $opts['busyValue'] ?? null;
1307  $popWindow = $opts['hotTTR'] ?? self::HOT_TTR;
1308  $ageNew = $opts['ageNew'] ?? self::AGE_NEW;
1309  $minTime = $opts['minAsOf'] ?? self::MIN_TIMESTAMP_NONE;
1310  $needsVersion = isset( $opts['version'] );
1311  $touchedCb = $opts['touchedCallback'] ?? null;
1312  $initialTime = $this->getCurrentTime();
1313 
1314  $kClass = $this->determineKeyClassForStats( $key );
1315 
1316  // Get the current key value
1317  $curTTL = self::PASS_BY_REF;
1318  $curInfo = self::PASS_BY_REF;
1319  $curValue = $this->get( $key, $curTTL, $checkKeys, $curInfo );
1320  // Apply any $touchedCb invalidation timestamp to get the "last purge timestamp"
1321  list( $curTTL, $LPT ) = $this->resolveCTL( $curValue, $curTTL, $curInfo, $touchedCb );
1322  // Keep track of the best candidate value and its timestamp
1323  $value = $curValue; // return value
1324  $asOf = $curInfo['asOf']; // return value timestamp
1325 
1326  // Determine if a cached value regeneration is needed or desired
1327  if (
1328  $this->isValid( $value, $needsVersion, $asOf, $minTime ) &&
1329  $this->isAliveOrInGracePeriod( $curTTL, $graceTTL )
1330  ) {
1331  $preemptiveRefresh = (
1332  $this->worthRefreshExpiring( $curTTL, $lowTTL ) ||
1333  $this->worthRefreshPopular( $asOf, $ageNew, $popWindow, $initialTime )
1334  );
1335 
1336  if ( !$preemptiveRefresh ) {
1337  $this->stats->increment( "wanobjectcache.$kClass.hit.good" );
1338 
1339  return $value;
1340  } elseif ( $this->scheduleAsyncRefresh( $key, $ttl, $callback, $opts ) ) {
1341  $this->stats->increment( "wanobjectcache.$kClass.hit.refresh" );
1342 
1343  return $value;
1344  }
1345  }
1346 
1347  $isKeyTombstoned = ( $curInfo['tombAsOf'] !== null );
1348  if ( $isKeyTombstoned ) {
1349  // Get the interim key value since the key is tombstoned (write-holed)
1350  list( $value, $asOf ) = $this->getInterimValue( $key, $needsVersion, $minTime );
1351  // Update the "last purge time" since the $touchedCb timestamp depends on $value
1352  $LPT = $this->resolveTouched( $value, $LPT, $touchedCb );
1353  }
1354 
1355  // Reduce mutex and cache set spam while keys are in the tombstone/holdoff period by
1356  // checking if $value was genereated by a recent thread much less than a second ago.
1357  if (
1358  $this->isValid( $value, $needsVersion, $asOf, $minTime, $LPT ) &&
1359  $this->isVolatileValueAgeNegligible( $initialTime - $asOf )
1360  ) {
1361  $this->stats->increment( "wanobjectcache.$kClass.hit.volatile" );
1362 
1363  return $value;
1364  }
1365 
1366  // Decide if only one thread should handle regeneration at a time
1367  $useMutex =
1368  // Note that since tombstones no-op set(), $lockTSE and $curTTL cannot be used to
1369  // deduce the key hotness because |$curTTL| will always keep increasing until the
1370  // tombstone expires or is overwritten by a new tombstone. Also, even if $lockTSE
1371  // is not set, constant regeneration of a key for the tombstone lifetime might be
1372  // very expensive. Assume tombstoned keys are possibly hot in order to reduce
1373  // the risk of high regeneration load after the delete() method is called.
1374  $isKeyTombstoned ||
1375  // Assume a key is hot if requested soon ($lockTSE seconds) after invalidation.
1376  // This avoids stampedes when timestamps from $checkKeys/$touchedCb bump.
1377  ( $curTTL !== null && $curTTL <= 0 && abs( $curTTL ) <= $lockTSE ) ||
1378  // Assume a key is hot if there is no value and a busy fallback is given.
1379  // This avoids stampedes on eviction or preemptive regeneration taking too long.
1380  ( $busyValue !== null && $value === false );
1381 
1382  $hasLock = false;
1383  if ( $useMutex ) {
1384  // Acquire a datacenter-local non-blocking lock
1385  if ( $this->cache->add( self::MUTEX_KEY_PREFIX . $key, 1, self::LOCK_TTL ) ) {
1386  // Lock acquired; this thread will recompute the value and update cache
1387  $hasLock = true;
1388  } elseif ( $this->isValid( $value, $needsVersion, $asOf, $minTime ) ) {
1389  // Lock not acquired and a stale value exists; use the stale value
1390  $this->stats->increment( "wanobjectcache.$kClass.hit.stale" );
1391 
1392  return $value;
1393  } else {
1394  // Lock not acquired and no stale value exists
1395  if ( $busyValue !== null ) {
1396  // Use the busy fallback value if nothing else
1397  $miss = is_infinite( $minTime ) ? 'renew' : 'miss';
1398  $this->stats->increment( "wanobjectcache.$kClass.$miss.busy" );
1399 
1400  return is_callable( $busyValue ) ? $busyValue() : $busyValue;
1401  }
1402  }
1403  }
1404 
1405  if ( !is_callable( $callback ) ) {
1406  throw new InvalidArgumentException( "Invalid cache miss callback provided." );
1407  }
1408 
1409  $preCallbackTime = $this->getCurrentTime();
1410  // Generate the new value from the callback...
1411  $setOpts = [];
1413  try {
1414  $value = call_user_func_array( $callback, [ $curValue, &$ttl, &$setOpts, $asOf ] );
1415  } finally {
1417  }
1418  $valueIsCacheable = ( $value !== false && $ttl >= 0 );
1419 
1420  if ( $valueIsCacheable ) {
1421  $ago = max( $this->getCurrentTime() - $initialTime, 0.0 );
1422  $this->stats->timing( "wanobjectcache.$kClass.regen_set_delay", 1000 * $ago );
1423 
1424  if ( $isKeyTombstoned ) {
1425  if ( $this->checkAndSetCooloff( $key, $kClass, $ago, $lockTSE, $hasLock ) ) {
1426  // Use the interim key value since the key is tombstoned (write-holed)
1427  $tempTTL = max( self::INTERIM_KEY_TTL, (int)$lockTSE );
1428  $this->setInterimValue( $key, $value, $tempTTL, $this->getCurrentTime() );
1429  }
1430  } elseif ( !$useMutex || $hasLock ) {
1431  if ( $this->checkAndSetCooloff( $key, $kClass, $ago, $lockTSE, $hasLock ) ) {
1432  $setOpts['creating'] = ( $curValue === false );
1433  // Save the value unless a lock-winning thread is already expected to do that
1434  $setOpts['lockTSE'] = $lockTSE;
1435  $setOpts['staleTTL'] = $staleTTL;
1436  // Use best known "since" timestamp if not provided
1437  $setOpts += [ 'since' => $preCallbackTime ];
1438  // Update the cache; this will fail if the key is tombstoned
1439  $this->set( $key, $value, $ttl, $setOpts );
1440  }
1441  }
1442  }
1443 
1444  if ( $hasLock ) {
1445  $this->cache->changeTTL( self::MUTEX_KEY_PREFIX . $key, (int)$initialTime - 60 );
1446  }
1447 
1448  $miss = is_infinite( $minTime ) ? 'renew' : 'miss';
1449  $this->stats->increment( "wanobjectcache.$kClass.$miss.compute" );
1450 
1451  return $value;
1452  }
1453 
1458  private function isVolatileValueAgeNegligible( $age ) {
1459  return ( $age < mt_rand( self::RECENT_SET_LOW_MS, self::RECENT_SET_HIGH_MS ) / 1e3 );
1460  }
1461 
1470  private function checkAndSetCooloff( $key, $kClass, $elapsed, $lockTSE, $hasLock ) {
1471  // If $lockTSE is set, the lock was bypassed because there was no stale/interim value,
1472  // and $elapsed indicates that regeration is slow, then there is a risk of set()
1473  // stampedes with large blobs. With a typical scale-out infrastructure, CPU and query
1474  // load from $callback invocations is distributed among appservers and replica DBs,
1475  // but cache operations for a given key route to a single cache server (e.g. striped
1476  // consistent hashing).
1477  if ( $lockTSE < 0 || $hasLock ) {
1478  return true; // either not a priori hot or thread has the lock
1479  } elseif ( $elapsed <= self::SET_DELAY_HIGH_MS * 1e3 ) {
1480  return true; // not enough time for threads to pile up
1481  }
1482 
1483  $this->cache->clearLastError();
1484  if (
1485  !$this->cache->add( self::COOLOFF_KEY_PREFIX . $key, 1, self::COOLOFF_TTL ) &&
1486  // Don't treat failures due to I/O errors as the key being in cooloff
1487  $this->cache->getLastError() === BagOStuff::ERR_NONE
1488  ) {
1489  $this->stats->increment( "wanobjectcache.$kClass.cooloff_bounce" );
1490 
1491  return false;
1492  }
1493 
1494  return true;
1495  }
1496 
1505  protected function resolveCTL( $value, $curTTL, $curInfo, $touchedCallback ) {
1506  if ( $touchedCallback === null || $value === false ) {
1507  return [ $curTTL, max( $curInfo['tombAsOf'], $curInfo['lastCKPurge'] ) ];
1508  }
1509 
1510  if ( !is_callable( $touchedCallback ) ) {
1511  throw new InvalidArgumentException( "Invalid expiration callback provided." );
1512  }
1513 
1514  $touched = $touchedCallback( $value );
1515  if ( $touched !== null && $touched >= $curInfo['asOf'] ) {
1516  $curTTL = min( $curTTL, self::TINY_NEGATIVE, $curInfo['asOf'] - $touched );
1517  }
1518 
1519  return [ $curTTL, max( $curInfo['tombAsOf'], $curInfo['lastCKPurge'], $touched ) ];
1520  }
1521 
1529  protected function resolveTouched( $value, $lastPurge, $touchedCallback ) {
1530  if ( $touchedCallback === null || $value === false ) {
1531  return $lastPurge;
1532  }
1533 
1534  if ( !is_callable( $touchedCallback ) ) {
1535  throw new InvalidArgumentException( "Invalid expiration callback provided." );
1536  }
1537 
1538  return max( $touchedCallback( $value ), $lastPurge );
1539  }
1540 
1547  protected function getInterimValue( $key, $versioned, $minTime ) {
1548  if ( !$this->useInterimHoldOffCaching ) {
1549  return [ false, null ]; // disabled
1550  }
1551 
1552  $wrapped = $this->cache->get( self::INTERIM_KEY_PREFIX . $key );
1553  list( $value ) = $this->unwrap( $wrapped, $this->getCurrentTime() );
1554  $valueAsOf = $wrapped[self::FLD_TIME] ?? null;
1555  if ( $this->isValid( $value, $versioned, $valueAsOf, $minTime ) ) {
1556  return [ $value, $valueAsOf ];
1557  }
1558 
1559  return [ false, null ];
1560  }
1561 
1568  protected function setInterimValue( $key, $value, $tempTTL, $newAsOf ) {
1569  $wrapped = $this->wrap( $value, $tempTTL, $newAsOf );
1570 
1571  $this->cache->merge(
1572  self::INTERIM_KEY_PREFIX . $key,
1573  function () use ( $wrapped ) {
1574  return $wrapped;
1575  },
1576  $tempTTL,
1577  1
1578  );
1579  }
1580 
1647  final public function getMultiWithSetCallback(
1648  ArrayIterator $keyedIds, $ttl, callable $callback, array $opts = []
1649  ) {
1650  $valueKeys = array_keys( $keyedIds->getArrayCopy() );
1651  $checkKeys = $opts['checkKeys'] ?? [];
1652  $pcTTL = $opts['pcTTL'] ?? self::TTL_UNCACHEABLE;
1653 
1654  // Load required keys into process cache in one go
1655  $this->warmupCache = $this->getRawKeysForWarmup(
1656  $this->getNonProcessCachedKeys( $valueKeys, $opts, $pcTTL ),
1657  $checkKeys
1658  );
1659  $this->warmupKeyMisses = 0;
1660 
1661  // Wrap $callback to match the getWithSetCallback() format while passing $id to $callback
1662  $id = null; // current entity ID
1663  $func = function ( $oldValue, &$ttl, &$setOpts, $oldAsOf ) use ( $callback, &$id ) {
1664  return $callback( $id, $oldValue, $ttl, $setOpts, $oldAsOf );
1665  };
1666 
1667  $values = [];
1668  foreach ( $keyedIds as $key => $id ) { // preserve order
1669  $values[$key] = $this->getWithSetCallback( $key, $ttl, $func, $opts );
1670  }
1671 
1672  $this->warmupCache = [];
1673 
1674  return $values;
1675  }
1676 
1742  final public function getMultiWithUnionSetCallback(
1743  ArrayIterator $keyedIds, $ttl, callable $callback, array $opts = []
1744  ) {
1745  $idsByValueKey = $keyedIds->getArrayCopy();
1746  $valueKeys = array_keys( $idsByValueKey );
1747  $checkKeys = $opts['checkKeys'] ?? [];
1748  $pcTTL = $opts['pcTTL'] ?? self::TTL_UNCACHEABLE;
1749  unset( $opts['lockTSE'] ); // incompatible
1750  unset( $opts['busyValue'] ); // incompatible
1751 
1752  // Load required keys into process cache in one go
1753  $keysGet = $this->getNonProcessCachedKeys( $valueKeys, $opts, $pcTTL );
1754  $this->warmupCache = $this->getRawKeysForWarmup( $keysGet, $checkKeys );
1755  $this->warmupKeyMisses = 0;
1756 
1757  // IDs of entities known to be in need of regeneration
1758  $idsRegen = [];
1759 
1760  // Find out which keys are missing/deleted/stale
1761  $curTTLs = [];
1762  $asOfs = [];
1763  $curByKey = $this->getMulti( $keysGet, $curTTLs, $checkKeys, $asOfs );
1764  foreach ( $keysGet as $key ) {
1765  if ( !array_key_exists( $key, $curByKey ) || $curTTLs[$key] < 0 ) {
1766  $idsRegen[] = $idsByValueKey[$key];
1767  }
1768  }
1769 
1770  // Run the callback to populate the regeneration value map for all required IDs
1771  $newSetOpts = [];
1772  $newTTLsById = array_fill_keys( $idsRegen, $ttl );
1773  $newValsById = $idsRegen ? $callback( $idsRegen, $newTTLsById, $newSetOpts ) : [];
1774 
1775  // Wrap $callback to match the getWithSetCallback() format while passing $id to $callback
1776  $id = null; // current entity ID
1777  $func = function ( $oldValue, &$ttl, &$setOpts, $oldAsOf )
1778  use ( $callback, &$id, $newValsById, $newTTLsById, $newSetOpts )
1779  {
1780  if ( array_key_exists( $id, $newValsById ) ) {
1781  // Value was already regerated as expected, so use the value in $newValsById
1782  $newValue = $newValsById[$id];
1783  $ttl = $newTTLsById[$id];
1784  $setOpts = $newSetOpts;
1785  } else {
1786  // Pre-emptive/popularity refresh and version mismatch cases are not detected
1787  // above and thus $newValsById has no entry. Run $callback on this single entity.
1788  $ttls = [ $id => $ttl ];
1789  $newValue = $callback( [ $id ], $ttls, $setOpts )[$id];
1790  $ttl = $ttls[$id];
1791  }
1792 
1793  return $newValue;
1794  };
1795 
1796  // Run the cache-aside logic using warmupCache instead of persistent cache queries
1797  $values = [];
1798  foreach ( $idsByValueKey as $key => $id ) { // preserve order
1799  $values[$key] = $this->getWithSetCallback( $key, $ttl, $func, $opts );
1800  }
1801 
1802  $this->warmupCache = [];
1803 
1804  return $values;
1805  }
1806 
1819  final public function reap( $key, $purgeTimestamp, &$isStale = false ) {
1820  $minAsOf = $purgeTimestamp + self::HOLDOFF_TTL;
1821  $wrapped = $this->cache->get( self::VALUE_KEY_PREFIX . $key );
1822  if ( is_array( $wrapped ) && $wrapped[self::FLD_TIME] < $minAsOf ) {
1823  $isStale = true;
1824  $this->logger->warning( "Reaping stale value key '$key'." );
1825  $ttlReap = self::HOLDOFF_TTL; // avoids races with tombstone creation
1826  $ok = $this->cache->changeTTL( self::VALUE_KEY_PREFIX . $key, $ttlReap );
1827  if ( !$ok ) {
1828  $this->logger->error( "Could not complete reap of key '$key'." );
1829  }
1830 
1831  return $ok;
1832  }
1833 
1834  $isStale = false;
1835 
1836  return true;
1837  }
1838 
1848  final public function reapCheckKey( $key, $purgeTimestamp, &$isStale = false ) {
1849  $purge = $this->parsePurgeValue( $this->cache->get( self::TIME_KEY_PREFIX . $key ) );
1850  if ( $purge && $purge[self::FLD_TIME] < $purgeTimestamp ) {
1851  $isStale = true;
1852  $this->logger->warning( "Reaping stale check key '$key'." );
1853  $ok = $this->cache->changeTTL( self::TIME_KEY_PREFIX . $key, self::TTL_SECOND );
1854  if ( !$ok ) {
1855  $this->logger->error( "Could not complete reap of check key '$key'." );
1856  }
1857 
1858  return $ok;
1859  }
1860 
1861  $isStale = false;
1862 
1863  return false;
1864  }
1865 
1873  public function makeKey( $class, $component = null ) {
1874  return $this->cache->makeKey( ...func_get_args() );
1875  }
1876 
1884  public function makeGlobalKey( $class, $component = null ) {
1885  return $this->cache->makeGlobalKey( ...func_get_args() );
1886  }
1887 
1894  final public function makeMultiKeys( array $entities, callable $keyFunc ) {
1895  $map = [];
1896  foreach ( $entities as $entity ) {
1897  $map[$keyFunc( $entity, $this )] = $entity;
1898  }
1899 
1900  return new ArrayIterator( $map );
1901  }
1902 
1907  final public function getLastError() {
1908  $code = $this->cache->getLastError();
1909  switch ( $code ) {
1910  case BagOStuff::ERR_NONE:
1911  return self::ERR_NONE;
1913  return self::ERR_NO_RESPONSE;
1915  return self::ERR_UNREACHABLE;
1916  default:
1917  return self::ERR_UNEXPECTED;
1918  }
1919  }
1920 
1924  final public function clearLastError() {
1925  $this->cache->clearLastError();
1926  }
1927 
1933  public function clearProcessCache() {
1934  $this->processCaches = [];
1935  }
1936 
1957  final public function useInterimHoldOffCaching( $enabled ) {
1958  $this->useInterimHoldOffCaching = $enabled;
1959  }
1960 
1966  public function getQoS( $flag ) {
1967  return $this->cache->getQoS( $flag );
1968  }
1969 
2033  public function adaptiveTTL( $mtime, $maxTTL, $minTTL = 30, $factor = 0.2 ) {
2034  if ( is_float( $mtime ) || ctype_digit( $mtime ) ) {
2035  $mtime = (int)$mtime; // handle fractional seconds and string integers
2036  }
2037 
2038  if ( !is_int( $mtime ) || $mtime <= 0 ) {
2039  return $minTTL; // no last-modified time provided
2040  }
2041 
2042  $age = $this->getCurrentTime() - $mtime;
2043 
2044  return (int)min( $maxTTL, max( $minTTL, $factor * $age ) );
2045  }
2046 
2051  final public function getWarmupKeyMisses() {
2052  return $this->warmupKeyMisses;
2053  }
2054 
2065  protected function relayPurge( $key, $ttl, $holdoff ) {
2066  if ( $this->mcrouterAware ) {
2067  // See https://github.com/facebook/mcrouter/wiki/Multi-cluster-broadcast-setup
2068  // Wildcards select all matching routes, e.g. the WAN cluster on all DCs
2069  $ok = $this->cache->set(
2070  "/*/{$this->cluster}/{$key}",
2071  $this->makePurgeValue( $this->getCurrentTime(), self::HOLDOFF_NONE ),
2072  $ttl
2073  );
2074  } else {
2075  // This handles the mcrouter and the single-DC case
2076  $ok = $this->cache->set(
2077  $key,
2078  $this->makePurgeValue( $this->getCurrentTime(), self::HOLDOFF_NONE ),
2079  $ttl
2080  );
2081  }
2082 
2083  return $ok;
2084  }
2085 
2092  protected function relayDelete( $key ) {
2093  if ( $this->mcrouterAware ) {
2094  // See https://github.com/facebook/mcrouter/wiki/Multi-cluster-broadcast-setup
2095  // Wildcards select all matching routes, e.g. the WAN cluster on all DCs
2096  $ok = $this->cache->delete( "/*/{$this->cluster}/{$key}" );
2097  } else {
2098  // Some other proxy handles broadcasting or there is only one datacenter
2099  $ok = $this->cache->delete( $key );
2100  }
2101 
2102  return $ok;
2103  }
2104 
2112  private function scheduleAsyncRefresh( $key, $ttl, $callback, $opts ) {
2113  if ( !$this->asyncHandler ) {
2114  return false;
2115  }
2116  // Update the cache value later, such during post-send of an HTTP request
2117  $func = $this->asyncHandler;
2118  $func( function () use ( $key, $ttl, $callback, $opts ) {
2119  $asOf = null; // unused
2120  $opts['minAsOf'] = INF; // force a refresh
2121  $this->doGetWithSetCallback( $key, $ttl, $callback, $opts, $asOf );
2122  } );
2123 
2124  return true;
2125  }
2126 
2140  protected function isAliveOrInGracePeriod( $curTTL, $graceTTL ) {
2141  if ( $curTTL > 0 ) {
2142  return true;
2143  } elseif ( $graceTTL <= 0 ) {
2144  return false;
2145  }
2146 
2147  $ageStale = abs( $curTTL ); // seconds of staleness
2148  $curGTTL = ( $graceTTL - $ageStale ); // current grace-time-to-live
2149  if ( $curGTTL <= 0 ) {
2150  return false; // already out of grace period
2151  }
2152 
2153  // Chance of using a stale value is the complement of the chance of refreshing it
2154  return !$this->worthRefreshExpiring( $curGTTL, $graceTTL );
2155  }
2156 
2170  protected function worthRefreshExpiring( $curTTL, $lowTTL ) {
2171  if ( $lowTTL <= 0 ) {
2172  return false;
2173  } elseif ( $curTTL >= $lowTTL ) {
2174  return false;
2175  } elseif ( $curTTL <= 0 ) {
2176  return false;
2177  }
2178 
2179  $chance = ( 1 - $curTTL / $lowTTL );
2180 
2181  return mt_rand( 1, 1e9 ) <= 1e9 * $chance;
2182  }
2183 
2199  protected function worthRefreshPopular( $asOf, $ageNew, $timeTillRefresh, $now ) {
2200  if ( $ageNew < 0 || $timeTillRefresh <= 0 ) {
2201  return false;
2202  }
2203 
2204  $age = $now - $asOf;
2205  $timeOld = $age - $ageNew;
2206  if ( $timeOld <= 0 ) {
2207  return false;
2208  }
2209 
2210  // Lifecycle is: new, ramp-up refresh chance, full refresh chance.
2211  // Note that the "expected # of refreshes" for the ramp-up time range is half of what it
2212  // would be if P(refresh) was at its full value during that time range.
2213  $refreshWindowSec = max( $timeTillRefresh - $ageNew - self::RAMPUP_TTL / 2, 1 );
2214  // P(refresh) * (# hits in $refreshWindowSec) = (expected # of refreshes)
2215  // P(refresh) * ($refreshWindowSec * $popularHitsPerSec) = 1
2216  // P(refresh) = 1/($refreshWindowSec * $popularHitsPerSec)
2217  $chance = 1 / ( self::HIT_RATE_HIGH * $refreshWindowSec );
2218 
2219  // Ramp up $chance from 0 to its nominal value over RAMPUP_TTL seconds to avoid stampedes
2220  $chance *= ( $timeOld <= self::RAMPUP_TTL ) ? $timeOld / self::RAMPUP_TTL : 1;
2221 
2222  return mt_rand( 1, 1e9 ) <= 1e9 * $chance;
2223  }
2224 
2235  protected function isValid( $value, $versioned, $asOf, $minTime, $purgeTime = null ) {
2236  // Avoid reading any key not generated after the latest delete() or touch
2237  $safeMinTime = max( $minTime, $purgeTime + self::TINY_POSTIVE );
2238 
2239  if ( $value === false ) {
2240  return false;
2241  } elseif ( $versioned && !isset( $value[self::VFLD_VERSION] ) ) {
2242  return false;
2243  } elseif ( $safeMinTime > 0 && $asOf < $minTime ) {
2244  return false;
2245  }
2246 
2247  return true;
2248  }
2249 
2258  protected function wrap( $value, $ttl, $now ) {
2259  return [
2260  self::FLD_VERSION => self::VERSION,
2261  self::FLD_VALUE => $value,
2262  self::FLD_TTL => $ttl,
2263  self::FLD_TIME => $now
2264  ];
2265  }
2266 
2276  protected function unwrap( $wrapped, $now ) {
2277  // Check if the value is a tombstone
2278  $purge = $this->parsePurgeValue( $wrapped );
2279  if ( $purge !== false ) {
2280  // Purged values should always have a negative current $ttl
2281  $curTTL = min( $purge[self::FLD_TIME] - $now, self::TINY_NEGATIVE );
2282  return [ false, $curTTL, null, $purge[self::FLD_TIME] ];
2283  }
2284 
2285  if ( !is_array( $wrapped ) // not found
2286  || !isset( $wrapped[self::FLD_VERSION] ) // wrong format
2287  || $wrapped[self::FLD_VERSION] !== self::VERSION // wrong version
2288  ) {
2289  return [ false, null, null, null ];
2290  }
2291 
2292  if ( $wrapped[self::FLD_TTL] > 0 ) {
2293  // Get the approximate time left on the key
2294  $age = $now - $wrapped[self::FLD_TIME];
2295  $curTTL = max( $wrapped[self::FLD_TTL] - $age, 0.0 );
2296  } else {
2297  // Key had no TTL, so the time left is unbounded
2298  $curTTL = INF;
2299  }
2300 
2301  if ( $wrapped[self::FLD_TIME] < $this->epoch ) {
2302  // Values this old are ignored
2303  return [ false, null, null, null ];
2304  }
2305 
2306  return [ $wrapped[self::FLD_VALUE], $curTTL, $wrapped[self::FLD_TIME], null ];
2307  }
2308 
2314  protected static function prefixCacheKeys( array $keys, $prefix ) {
2315  $res = [];
2316  foreach ( $keys as $key ) {
2317  $res[] = $prefix . $key;
2318  }
2319 
2320  return $res;
2321  }
2322 
2327  protected function determineKeyClassForStats( $key ) {
2328  $parts = explode( ':', $key, 3 );
2329 
2330  return $parts[1] ?? $parts[0]; // sanity
2331  }
2332 
2338  protected function parsePurgeValue( $value ) {
2339  if ( !is_string( $value ) ) {
2340  return false;
2341  }
2342 
2343  $segments = explode( ':', $value, 3 );
2344  if ( !isset( $segments[0] ) || !isset( $segments[1] )
2345  || "{$segments[0]}:" !== self::PURGE_VAL_PREFIX
2346  ) {
2347  return false;
2348  }
2349 
2350  if ( !isset( $segments[2] ) ) {
2351  // Back-compat with old purge values without holdoff
2352  $segments[2] = self::HOLDOFF_TTL;
2353  }
2354 
2355  if ( $segments[1] < $this->epoch ) {
2356  // Values this old are ignored
2357  return false;
2358  }
2359 
2360  return [
2361  self::FLD_TIME => (float)$segments[1],
2362  self::FLD_HOLDOFF => (int)$segments[2],
2363  ];
2364  }
2365 
2371  protected function makePurgeValue( $timestamp, $holdoff ) {
2372  return self::PURGE_VAL_PREFIX . (float)$timestamp . ':' . (int)$holdoff;
2373  }
2374 
2379  protected function getProcessCache( $group ) {
2380  if ( !isset( $this->processCaches[$group] ) ) {
2381  list( , $n ) = explode( ':', $group );
2382  $this->processCaches[$group] = new MapCacheLRU( (int)$n );
2383  }
2384 
2385  return $this->processCaches[$group];
2386  }
2387 
2394  private function getNonProcessCachedKeys( array $keys, array $opts, $pcTTL ) {
2395  $keysFound = [];
2396  if ( isset( $opts['pcTTL'] ) && $opts['pcTTL'] > 0 && $this->callbackDepth == 0 ) {
2397  $pcGroup = $opts['pcGroup'] ?? self::PC_PRIMARY;
2398  $procCache = $this->getProcessCache( $pcGroup );
2399  foreach ( $keys as $key ) {
2400  if ( $procCache->has( $key, $pcTTL ) ) {
2401  $keysFound[] = $key;
2402  }
2403  }
2404  }
2405 
2406  return array_diff( $keys, $keysFound );
2407  }
2408 
2414  private function getRawKeysForWarmup( array $keys, array $checkKeys ) {
2415  if ( !$keys ) {
2416  return [];
2417  }
2418 
2419  $keysWarmUp = [];
2420  // Get all the value keys to fetch...
2421  foreach ( $keys as $key ) {
2422  $keysWarmUp[] = self::VALUE_KEY_PREFIX . $key;
2423  }
2424  // Get all the check keys to fetch...
2425  foreach ( $checkKeys as $i => $checkKeyOrKeys ) {
2426  if ( is_int( $i ) ) {
2427  // Single check key that applies to all value keys
2428  $keysWarmUp[] = self::TIME_KEY_PREFIX . $checkKeyOrKeys;
2429  } else {
2430  // List of check keys that apply to value key $i
2431  $keysWarmUp = array_merge(
2432  $keysWarmUp,
2433  self::prefixCacheKeys( $checkKeyOrKeys, self::TIME_KEY_PREFIX )
2434  );
2435  }
2436  }
2437 
2438  $warmupCache = $this->cache->getMulti( $keysWarmUp );
2439  $warmupCache += array_fill_keys( $keysWarmUp, false );
2440 
2441  return $warmupCache;
2442  }
2443 
2448  protected function getCurrentTime() {
2449  if ( $this->wallClockOverride ) {
2450  return $this->wallClockOverride;
2451  }
2452 
2453  $clockTime = (float)time(); // call this first
2454  // microtime() uses an initial gettimeofday() call added to usage clocks.
2455  // This can severely drift from time() and the microtime() value of other threads
2456  // due to undercounting of the amount of time elapsed. Instead of seeing the current
2457  // time as being in the past, use the value of time(). This avoids setting cache values
2458  // that will immediately be seen as expired and possibly cause stampedes.
2459  return max( microtime( true ), $clockTime );
2460  }
2461 
2466  public function setMockTime( &$time ) {
2467  $this->wallClockOverride =& $time;
2468  $this->cache->setMockTime( $time );
2469  }
2470 }
processCheckKeys(array $timeKeys, array $wrappedValues, $now)
string $region
Physical region for mcrouter use.
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition: deferred.txt:11
makePurgeValue( $timestamp, $holdoff)
const TTL_LAGGED
Max TTL to store keys when a data sourced is lagged.
const TINY_NEGATIVE
Tiny negative float to use when CTL comes up >= 0 due to clock skew.
reapCheckKey( $key, $purgeTimestamp, &$isStale=false)
Set a "check" key to soon expire in the local cluster if it pre-dates $purgeTimestamp.
const CHECK_KEY_TTL
Seconds to keep dependency purge keys around.
const STALE_TTL_NONE
Idiom for set()/getWithSetCallback() for "do not augment the storage medium TTL". ...
const RECENT_SET_HIGH_MS
Max millisecond set() backoff for keys in hold-off (far less than INTERIM_KEY_TTL) ...
processing should stop and the error should be shown to the user * false
Definition: hooks.txt:187
const COOLOFF_TTL
Seconds to no-op key set() calls to avoid large blob I/O stampedes.
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
clearProcessCache()
Clear the in-process caches; useful for testing.
const PASS_BY_REF
Parameter to get()/getMulti() to return extra information by reference.
static prefixCacheKeys(array $keys, $prefix)
determineKeyClassForStats( $key)
relayDelete( $key)
Do the actual async bus delete of a key.
const MAX_COMMIT_DELAY
Max time expected to pass between delete() and DB commit finishing.
const TTL_UNCACHEABLE
Idiom for getWithSetCallback() callbacks to avoid calling set()
LoggerInterface $logger
mixed [] $warmupCache
Temporary warm-up cache.
getMultiWithSetCallback(ArrayIterator $keyedIds, $ttl, callable $callback, array $opts=[])
Method to fetch multiple cache keys at once with regeneration.
This code would result in ircNotify being run twice when an article is and once for brion Hooks can return three possible true was required This is the default since MediaWiki *some string
Definition: hooks.txt:175
$value
useInterimHoldOffCaching( $enabled)
Enable or disable the use of brief caching for tombstoned keys.
scheduleAsyncRefresh( $key, $ttl, $callback, $opts)
getWithSetCallback( $key, $ttl, $callback, array $opts=[])
Method to fetch/regenerate cache keys.
see documentation in includes Linker php for Linker::makeImageLink & $time
Definition: hooks.txt:1799
setInterimValue( $key, $value, $tempTTL, $newAsOf)
worthRefreshPopular( $asOf, $ageNew, $timeTillRefresh, $now)
Check if a key is due for randomized regeneration due to its popularity.
The index of the header message $result[1]=The index of the body text message $result[2 through n]=Parameters passed to body text message. Please note the header message cannot receive/use parameters. 'ImportHandleLogItemXMLTag':When parsing a XML tag in a log item. Return false to stop further processing of the tag $reader:XMLReader object $logInfo:Array of information 'ImportHandlePageXMLTag':When parsing a XML tag in a page. Return false to stop further processing of the tag $reader:XMLReader object & $pageInfo:Array of information 'ImportHandleRevisionXMLTag':When parsing a XML tag in a page revision. Return false to stop further processing of the tag $reader:XMLReader object $pageInfo:Array of page information $revisionInfo:Array of revision information 'ImportHandleToplevelXMLTag':When parsing a top level XML tag. Return false to stop further processing of the tag $reader:XMLReader object 'ImportHandleUnknownUser':When a user doesn 't exist locally, this hook is called to give extensions an opportunity to auto-create it. If the auto-creation is successful, return false. $name:User name 'ImportHandleUploadXMLTag':When parsing a XML tag in a file upload. Return false to stop further processing of the tag $reader:XMLReader object $revisionInfo:Array of information 'ImportLogInterwikiLink':Hook to change the interwiki link used in log entries and edit summaries for transwiki imports. & $fullInterwikiPrefix:Interwiki prefix, may contain colons. & $pageTitle:String that contains page title. 'ImportSources':Called when reading from the $wgImportSources configuration variable. Can be used to lazy-load the import sources list. & $importSources:The value of $wgImportSources. Modify as necessary. See the comment in DefaultSettings.php for the detail of how to structure this array. 'InfoAction':When building information to display on the action=info page. $context:IContextSource object & $pageInfo:Array of information 'InitializeArticleMaybeRedirect':MediaWiki check to see if title is a redirect. & $title:Title object for the current page & $request:WebRequest & $ignoreRedirect:boolean to skip redirect check & $target:Title/string of redirect target & $article:Article object 'InternalParseBeforeLinks':during Parser 's internalParse method before links but after nowiki/noinclude/includeonly/onlyinclude and other processings. & $parser:Parser object & $text:string containing partially parsed text & $stripState:Parser 's internal StripState object 'InternalParseBeforeSanitize':during Parser 's internalParse method just before the parser removes unwanted/dangerous HTML tags and after nowiki/noinclude/includeonly/onlyinclude and other processings. Ideal for syntax-extensions after template/parser function execution which respect nowiki and HTML-comments. & $parser:Parser object & $text:string containing partially parsed text & $stripState:Parser 's internal StripState object 'InterwikiLoadPrefix':When resolving if a given prefix is an interwiki or not. Return true without providing an interwiki to continue interwiki search. $prefix:interwiki prefix we are looking for. & $iwData:output array describing the interwiki with keys iw_url, iw_local, iw_trans and optionally iw_api and iw_wikiid. 'InvalidateEmailComplete':Called after a user 's email has been invalidated successfully. $user:user(object) whose email is being invalidated 'IRCLineURL':When constructing the URL to use in an IRC notification. Callee may modify $url and $query, URL will be constructed as $url . $query & $url:URL to index.php & $query:Query string $rc:RecentChange object that triggered url generation 'IsFileCacheable':Override the result of Article::isFileCacheable()(if true) & $article:article(object) being checked 'IsTrustedProxy':Override the result of IP::isTrustedProxy() & $ip:IP being check & $result:Change this value to override the result of IP::isTrustedProxy() 'IsUploadAllowedFromUrl':Override the result of UploadFromUrl::isAllowedUrl() $url:URL used to upload from & $allowed:Boolean indicating if uploading is allowed for given URL 'isValidEmailAddr':Override the result of Sanitizer::validateEmail(), for instance to return false if the domain name doesn 't match your organization. $addr:The e-mail address entered by the user & $result:Set this and return false to override the internal checks 'isValidPassword':Override the result of User::isValidPassword() $password:The password entered by the user & $result:Set this and return false to override the internal checks $user:User the password is being validated for 'Language::getMessagesFileName':$code:The language code or the language we 're looking for a messages file for & $file:The messages file path, you can override this to change the location. 'LanguageGetNamespaces':Provide custom ordering for namespaces or remove namespaces. Do not use this hook to add namespaces. Use CanonicalNamespaces for that. & $namespaces:Array of namespaces indexed by their numbers 'LanguageGetTranslatedLanguageNames':Provide translated language names. & $names:array of language code=> language name $code:language of the preferred translations 'LanguageLinks':Manipulate a page 's language links. This is called in various places to allow extensions to define the effective language links for a page. $title:The page 's Title. & $links:Array with elements of the form "language:title" in the order that they will be output. & $linkFlags:Associative array mapping prefixed links to arrays of flags. Currently unused, but planned to provide support for marking individual language links in the UI, e.g. for featured articles. 'LanguageSelector':Hook to change the language selector available on a page. $out:The output page. $cssClassName:CSS class name of the language selector. 'LinkBegin':DEPRECATED since 1.28! Use HtmlPageLinkRendererBegin instead. Used when generating internal and interwiki links in Linker::link(), before processing starts. Return false to skip default processing and return $ret. See documentation for Linker::link() for details on the expected meanings of parameters. $skin:the Skin object $target:the Title that the link is pointing to & $html:the contents that the< a > tag should have(raw HTML) $result
Definition: hooks.txt:1980
checkAndSetCooloff( $key, $kClass, $elapsed, $lockTSE, $hasLock)
you have access to all of the normal MediaWiki so you can get a DB use the cache
Definition: maintenance.txt:52
const RECENT_SET_LOW_MS
Min millisecond set() backoff for keys in hold-off (far less than INTERIM_KEY_TTL) ...
getMultiWithUnionSetCallback(ArrayIterator $keyedIds, $ttl, callable $callback, array $opts=[])
Method to fetch/regenerate multiple cache keys at once.
float $epoch
Unix timestamp of the oldest possible valid values.
int $warmupKeyMisses
Key fetched.
static newEmpty()
Get an instance that wraps EmptyBagOStuff.
makeGlobalKey( $class, $component=null)
string $cluster
Cache cluster name for mcrouter use.
const LOW_TTL
Default remaining TTL at which to consider pre-emptive regeneration.
worthRefreshExpiring( $curTTL, $lowTTL)
Check if a key is nearing expiration and thus due for randomized regeneration.
getLastError()
Get the "last error" registered; clearLastError() should be called manually.
getMultiCheckKeyTime(array $keys)
Fetch the values of each timestamp "check" key.
doGetWithSetCallback( $key, $ttl, $callback, array $opts, &$asOf=null)
Do the actual I/O for getWithSetCallback() when needed.
$res
Definition: database.txt:21
__construct(array $params)
const RAMPUP_TTL
Seconds to ramp up to the "popularity" refresh chance after a key is no longer new.
reap( $key, $purgeTimestamp, &$isStale=false)
Set a key to soon expire in the local cluster if it pre-dates $purgeTimestamp.
$params
getMulti(array $keys, &$curTTLs=[], array $checkKeys=[], &$info=null)
Fetch the value of several keys from cache.
unwrap( $wrapped, $now)
Do not use this method outside WANObjectCache.
adaptiveTTL( $mtime, $maxTTL, $minTTL=30, $factor=0.2)
Get a TTL that is higher for objects that have not changed recently.
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that When $user is not null
Definition: hooks.txt:780
relayPurge( $key, $ttl, $holdoff)
Do the actual async bus purge of a key.
const INTERIM_KEY_TTL
Seconds to keep interim value keys for tombstoned keys around.
float null $wallClockOverride
getProcessCache( $group)
getCheckKeyTime( $key)
Fetch the value of a timestamp "check" key.
const HOLDOFF_NONE
Idiom for delete() for "no hold-off".
int $callbackDepth
Callback stack depth for getWithSetCallback()
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that When $user is not it can be in the form of< username >< more info > e g for bot passwords intended to be added to log contexts Fields it might only if the login was with a bot password it is not rendered in wiki pages or galleries in category pages allow injecting custom HTML after the section Any uses of the hook need to handle escaping see BaseTemplate::getToolbox and BaseTemplate::makeListItem for details on the format of individual items inside of this array or by returning and letting standard HTTP rendering take place modifiable or by returning false and taking over the output modifiable & $code
Definition: hooks.txt:780
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
resetCheckKey( $key)
Delete a "check" key from all datacenters, invalidating keys that use it.
const TINY_POSTIVE
Tiny positive float to use when using "minTime" to assert an inequality.
touchCheckKey( $key, $holdoff=self::HOLDOFF_TTL)
Purge a "check" key from all datacenters, invalidating keys that use it.
const VERSION
Cache format version number.
clearLastError()
Clear the "last error" registry.
callable null $asyncHandler
Function that takes a WAN cache callback and runs it later.
const HOLDOFF_TTL
Seconds to tombstone keys on delete()
isVolatileValueAgeNegligible( $age)
getRawKeysForWarmup(array $keys, array $checkKeys)
const MIN_TIMESTAMP_NONE
Idiom for getWithSetCallback() for "no minimum required as-of timestamp".
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
resolveCTL( $value, $curTTL, $curInfo, $touchedCallback)
const SET_DELAY_HIGH_MS
Milliseconds of delay after get() where set() storms are a consideration with &#39;lockTSE&#39;.
const HOT_TTR
The time length of the "popularity" refresh window for hot keys.
isAliveOrInGracePeriod( $curTTL, $graceTTL)
Check if a key is fresh or in the grace window and thus due for randomized reuse. ...
const AGE_NEW
Never consider performing "popularity" refreshes until a key reaches this age.
bool $useInterimHoldOffCaching
Whether to use "interim" caching while keys are tombstoned.
const LOCK_TTL
Seconds to keep lock keys around.
getNonProcessCachedKeys(array $keys, array $opts, $pcTTL)
getInterimValue( $key, $versioned, $minTime)
const GRACE_TTL_NONE
Idiom for set()/getWithSetCallback() for "no post-expired grace period".
$mcrouterAware
bool Whether to use mcrouter key prefixing for routing
StatsdDataFactoryInterface $stats
MapCacheLRU [] $processCaches
Map of group PHP instance caches.
wrap( $value, $ttl, $now)
Do not use this method outside WANObjectCache.
const MAX_READ_LAG
Max replication+snapshot lag before applying TTL_LAGGED or disallowing set()
setLogger(LoggerInterface $logger)
const TSE_NONE
Idiom for getWithSetCallback() callbacks to &#39;lockTSE&#39; logic.
parsePurgeValue( $value)
const HIT_RATE_HIGH
Hits/second for a refresh to be expected within the "popularity" window.
makeKey( $class, $component=null)
makeMultiKeys(array $entities, callable $keyFunc)
isValid( $value, $versioned, $asOf, $minTime, $purgeTime=null)
Check if $value is not false, versioned (if needed), and not older than $minTime (if set) ...
BagOStuff $cache
The local datacenter cache.
resolveTouched( $value, $lastPurge, $touchedCallback)