MediaWiki  1.28.0
WANObjectCache.php
Go to the documentation of this file.
1 <?php
26 
76 class WANObjectCache implements IExpiringStore, LoggerAwareInterface {
78  protected $cache;
80  protected $processCaches = [];
82  protected $purgeChannel;
84  protected $purgeRelayer;
86  protected $logger;
87 
89  protected $lastRelayError = self::ERR_NONE;
90 
92  private $callbackDepth = 0;
94  private $warmupCache = [];
95 
97  const MAX_COMMIT_DELAY = 3;
99  const MAX_READ_LAG = 7;
101  const HOLDOFF_TTL = 11; // MAX_COMMIT_DELAY + MAX_READ_LAG + 1
102 
104  const CHECK_KEY_TTL = self::TTL_YEAR;
106  const LOCK_TTL = 10;
108  const LOW_TTL = 30;
110  const LOCK_TSE = 1;
111 
113  const AGE_NEW = 60;
115  const HOT_TTR = 900;
117  const HIT_RATE_HIGH = 1;
119  const RAMPUP_TTL = 30;
120 
122  const TTL_UNCACHEABLE = -1;
124  const TSE_NONE = -1;
126  const TTL_LAGGED = 30;
128  const HOLDOFF_NONE = 0;
130  const MIN_TIMESTAMP_NONE = 0.0;
131 
133  const TINY_NEGATIVE = -0.000001;
134 
136  const VERSION = 1;
137 
138  const FLD_VERSION = 0; // key to cache version number
139  const FLD_VALUE = 1; // key to the cached value
140  const FLD_TTL = 2; // key to the original TTL
141  const FLD_TIME = 3; // key to the cache time
142  const FLD_FLAGS = 4; // key to the flags bitfield
143  const FLD_HOLDOFF = 5; // key to any hold-off TTL
144 
146  const FLG_STALE = 1;
147 
148  const ERR_NONE = 0; // no error
149  const ERR_NO_RESPONSE = 1; // no response
150  const ERR_UNREACHABLE = 2; // can't connect
151  const ERR_UNEXPECTED = 3; // response gave some error
152  const ERR_RELAY = 4; // relay broadcast failed
153 
154  const VALUE_KEY_PREFIX = 'WANCache:v:';
155  const INTERIM_KEY_PREFIX = 'WANCache:i:';
156  const TIME_KEY_PREFIX = 'WANCache:t:';
157  const MUTEX_KEY_PREFIX = 'WANCache:m:';
158 
159  const PURGE_VAL_PREFIX = 'PURGED:';
160 
161  const VFLD_DATA = 'WOC:d'; // key to the value of versioned data
162  const VFLD_VERSION = 'WOC:v'; // key to the version of the value present
163 
164  const PC_PRIMARY = 'primary:1000'; // process cache name and max key count
165 
166  const DEFAULT_PURGE_CHANNEL = 'wancache-purge';
167 
175  public function __construct( array $params ) {
176  $this->cache = $params['cache'];
177  $this->purgeChannel = isset( $params['channels']['purge'] )
178  ? $params['channels']['purge']
179  : self::DEFAULT_PURGE_CHANNEL;
180  $this->purgeRelayer = isset( $params['relayers']['purge'] )
181  ? $params['relayers']['purge']
182  : new EventRelayerNull( [] );
183  $this->setLogger( isset( $params['logger'] ) ? $params['logger'] : new NullLogger() );
184  }
185 
186  public function setLogger( LoggerInterface $logger ) {
187  $this->logger = $logger;
188  }
189 
195  public static function newEmpty() {
196  return new self( [
197  'cache' => new EmptyBagOStuff(),
198  'pool' => 'empty',
199  'relayer' => new EventRelayerNull( [] )
200  ] );
201  }
202 
243  final public function get( $key, &$curTTL = null, array $checkKeys = [], &$asOf = null ) {
244  $curTTLs = [];
245  $asOfs = [];
246  $values = $this->getMulti( [ $key ], $curTTLs, $checkKeys, $asOfs );
247  $curTTL = isset( $curTTLs[$key] ) ? $curTTLs[$key] : null;
248  $asOf = isset( $asOfs[$key] ) ? $asOfs[$key] : null;
249 
250  return isset( $values[$key] ) ? $values[$key] : false;
251  }
252 
265  final public function getMulti(
266  array $keys, &$curTTLs = [], array $checkKeys = [], array &$asOfs = []
267  ) {
268  $result = [];
269  $curTTLs = [];
270  $asOfs = [];
271 
272  $vPrefixLen = strlen( self::VALUE_KEY_PREFIX );
273  $valueKeys = self::prefixCacheKeys( $keys, self::VALUE_KEY_PREFIX );
274 
275  $checkKeysForAll = [];
276  $checkKeysByKey = [];
277  $checkKeysFlat = [];
278  foreach ( $checkKeys as $i => $keys ) {
279  $prefixed = self::prefixCacheKeys( (array)$keys, self::TIME_KEY_PREFIX );
280  $checkKeysFlat = array_merge( $checkKeysFlat, $prefixed );
281  // Is this check keys for a specific cache key, or for all keys being fetched?
282  if ( is_int( $i ) ) {
283  $checkKeysForAll = array_merge( $checkKeysForAll, $prefixed );
284  } else {
285  $checkKeysByKey[$i] = isset( $checkKeysByKey[$i] )
286  ? array_merge( $checkKeysByKey[$i], $prefixed )
287  : $prefixed;
288  }
289  }
290 
291  // Fetch all of the raw values
292  $keysGet = array_merge( $valueKeys, $checkKeysFlat );
293  if ( $this->warmupCache ) {
294  $wrappedValues = array_intersect_key( $this->warmupCache, array_flip( $keysGet ) );
295  $keysGet = array_diff( $keysGet, array_keys( $wrappedValues ) ); // keys left to fetch
296  } else {
297  $wrappedValues = [];
298  }
299  $wrappedValues += $this->cache->getMulti( $keysGet );
300  // Time used to compare/init "check" keys (derived after getMulti() to be pessimistic)
301  $now = microtime( true );
302 
303  // Collect timestamps from all "check" keys
304  $purgeValuesForAll = $this->processCheckKeys( $checkKeysForAll, $wrappedValues, $now );
305  $purgeValuesByKey = [];
306  foreach ( $checkKeysByKey as $cacheKey => $checks ) {
307  $purgeValuesByKey[$cacheKey] =
308  $this->processCheckKeys( $checks, $wrappedValues, $now );
309  }
310 
311  // Get the main cache value for each key and validate them
312  foreach ( $valueKeys as $vKey ) {
313  if ( !isset( $wrappedValues[$vKey] ) ) {
314  continue; // not found
315  }
316 
317  $key = substr( $vKey, $vPrefixLen ); // unprefix
318 
319  list( $value, $curTTL ) = $this->unwrap( $wrappedValues[$vKey], $now );
320  if ( $value !== false ) {
321  $result[$key] = $value;
322 
323  // Force dependant keys to be invalid for a while after purging
324  // to reduce race conditions involving stale data getting cached
325  $purgeValues = $purgeValuesForAll;
326  if ( isset( $purgeValuesByKey[$key] ) ) {
327  $purgeValues = array_merge( $purgeValues, $purgeValuesByKey[$key] );
328  }
329  foreach ( $purgeValues as $purge ) {
330  $safeTimestamp = $purge[self::FLD_TIME] + $purge[self::FLD_HOLDOFF];
331  if ( $safeTimestamp >= $wrappedValues[$vKey][self::FLD_TIME] ) {
332  // How long ago this value was expired by *this* check key
333  $ago = min( $purge[self::FLD_TIME] - $now, self::TINY_NEGATIVE );
334  // How long ago this value was expired by *any* known check key
335  $curTTL = min( $curTTL, $ago );
336  }
337  }
338  }
339  $curTTLs[$key] = $curTTL;
340  $asOfs[$key] = ( $value !== false ) ? $wrappedValues[$vKey][self::FLD_TIME] : null;
341  }
342 
343  return $result;
344  }
345 
353  private function processCheckKeys( array $timeKeys, array $wrappedValues, $now ) {
354  $purgeValues = [];
355  foreach ( $timeKeys as $timeKey ) {
356  $purge = isset( $wrappedValues[$timeKey] )
357  ? self::parsePurgeValue( $wrappedValues[$timeKey] )
358  : false;
359  if ( $purge === false ) {
360  // Key is not set or invalid; regenerate
361  $newVal = $this->makePurgeValue( $now, self::HOLDOFF_TTL );
362  $this->cache->add( $timeKey, $newVal, self::CHECK_KEY_TTL );
363  $purge = self::parsePurgeValue( $newVal );
364  }
365  $purgeValues[] = $purge;
366  }
367  return $purgeValues;
368  }
369 
428  final public function set( $key, $value, $ttl = 0, array $opts = [] ) {
429  $now = microtime( true );
430  $lockTSE = isset( $opts['lockTSE'] ) ? $opts['lockTSE'] : self::TSE_NONE;
431  $age = isset( $opts['since'] ) ? max( 0, $now - $opts['since'] ) : 0;
432  $lag = isset( $opts['lag'] ) ? $opts['lag'] : 0;
433  $staleTTL = isset( $opts['staleTTL'] ) ? $opts['staleTTL'] : 0;
434 
435  // Do not cache potentially uncommitted data as it might get rolled back
436  if ( !empty( $opts['pending'] ) ) {
437  $this->logger->info( "Rejected set() for $key due to pending writes." );
438 
439  return true; // no-op the write for being unsafe
440  }
441 
442  $wrapExtra = []; // additional wrapped value fields
443  // Check if there's a risk of writing stale data after the purge tombstone expired
444  if ( $lag === false || ( $lag + $age ) > self::MAX_READ_LAG ) {
445  // Case A: read lag with "lockTSE"; save but record value as stale
446  if ( $lockTSE >= 0 ) {
447  $ttl = max( 1, (int)$lockTSE ); // set() expects seconds
448  $wrapExtra[self::FLD_FLAGS] = self::FLG_STALE; // mark as stale
449  // Case B: any long-running transaction; ignore this set()
450  } elseif ( $age > self::MAX_READ_LAG ) {
451  $this->logger->warning( "Rejected set() for $key due to snapshot lag." );
452 
453  return true; // no-op the write for being unsafe
454  // Case C: high replication lag; lower TTL instead of ignoring all set()s
455  } elseif ( $lag === false || $lag > self::MAX_READ_LAG ) {
456  $ttl = $ttl ? min( $ttl, self::TTL_LAGGED ) : self::TTL_LAGGED;
457  $this->logger->warning( "Lowered set() TTL for $key due to replication lag." );
458  // Case D: medium length request with medium replication lag; ignore this set()
459  } else {
460  $this->logger->warning( "Rejected set() for $key due to high read lag." );
461 
462  return true; // no-op the write for being unsafe
463  }
464  }
465 
466  // Wrap that value with time/TTL/version metadata
467  $wrapped = $this->wrap( $value, $ttl, $now ) + $wrapExtra;
468 
469  $func = function ( $cache, $key, $cWrapped ) use ( $wrapped ) {
470  return ( is_string( $cWrapped ) )
471  ? false // key is tombstoned; do nothing
472  : $wrapped;
473  };
474 
475  return $this->cache->merge( self::VALUE_KEY_PREFIX . $key, $func, $ttl + $staleTTL, 1 );
476  }
477 
535  final public function delete( $key, $ttl = self::HOLDOFF_TTL ) {
536  $key = self::VALUE_KEY_PREFIX . $key;
537 
538  if ( $ttl <= 0 ) {
539  // Publish the purge to all datacenters
540  $ok = $this->relayDelete( $key );
541  } else {
542  // Publish the purge to all datacenters
543  $ok = $this->relayPurge( $key, $ttl, self::HOLDOFF_NONE );
544  }
545 
546  return $ok;
547  }
548 
568  final public function getCheckKeyTime( $key ) {
569  $key = self::TIME_KEY_PREFIX . $key;
570 
571  $purge = self::parsePurgeValue( $this->cache->get( $key ) );
572  if ( $purge !== false ) {
573  $time = $purge[self::FLD_TIME];
574  } else {
575  // Casting assures identical floats for the next getCheckKeyTime() calls
576  $now = (string)microtime( true );
577  $this->cache->add( $key,
578  $this->makePurgeValue( $now, self::HOLDOFF_TTL ),
579  self::CHECK_KEY_TTL
580  );
581  $time = (float)$now;
582  }
583 
584  return $time;
585  }
586 
620  final public function touchCheckKey( $key, $holdoff = self::HOLDOFF_TTL ) {
621  // Publish the purge to all datacenters
622  return $this->relayPurge( self::TIME_KEY_PREFIX . $key, self::CHECK_KEY_TTL, $holdoff );
623  }
624 
655  final public function resetCheckKey( $key ) {
656  // Publish the purge to all datacenters
657  return $this->relayDelete( self::TIME_KEY_PREFIX . $key );
658  }
659 
849  final public function getWithSetCallback( $key, $ttl, $callback, array $opts = [] ) {
850  $pcTTL = isset( $opts['pcTTL'] ) ? $opts['pcTTL'] : self::TTL_UNCACHEABLE;
851 
852  // Try the process cache if enabled and the cache callback is not within a cache callback.
853  // Process cache use in nested callbacks is not lag-safe with regard to HOLDOFF_TTL since
854  // the in-memory value is further lagged than the shared one since it uses a blind TTL.
855  if ( $pcTTL >= 0 && $this->callbackDepth == 0 ) {
856  $group = isset( $opts['pcGroup'] ) ? $opts['pcGroup'] : self::PC_PRIMARY;
857  $procCache = $this->getProcessCache( $group );
858  $value = $procCache->get( $key );
859  } else {
860  $procCache = false;
861  $value = false;
862  }
863 
864  if ( $value === false ) {
865  // Fetch the value over the network
866  if ( isset( $opts['version'] ) ) {
867  $version = $opts['version'];
868  $asOf = null;
869  $cur = $this->doGetWithSetCallback(
870  $key,
871  $ttl,
872  function ( $oldValue, &$ttl, &$setOpts, $oldAsOf )
873  use ( $callback, $version ) {
874  if ( is_array( $oldValue )
875  && array_key_exists( self::VFLD_DATA, $oldValue )
876  ) {
877  $oldData = $oldValue[self::VFLD_DATA];
878  } else {
879  // VFLD_DATA is not set if an old, unversioned, key is present
880  $oldData = false;
881  }
882 
883  return [
884  self::VFLD_DATA => $callback( $oldData, $ttl, $setOpts, $oldAsOf ),
885  self::VFLD_VERSION => $version
886  ];
887  },
888  $opts,
889  $asOf
890  );
891  if ( $cur[self::VFLD_VERSION] === $version ) {
892  // Value created or existed before with version; use it
893  $value = $cur[self::VFLD_DATA];
894  } else {
895  // Value existed before with a different version; use variant key.
896  // Reflect purges to $key by requiring that this key value be newer.
897  $value = $this->doGetWithSetCallback(
898  'cache-variant:' . md5( $key ) . ":$version",
899  $ttl,
900  $callback,
901  // Regenerate value if not newer than $key
902  [ 'version' => null, 'minAsOf' => $asOf ] + $opts
903  );
904  }
905  } else {
906  $value = $this->doGetWithSetCallback( $key, $ttl, $callback, $opts );
907  }
908 
909  // Update the process cache if enabled
910  if ( $procCache && $value !== false ) {
911  $procCache->set( $key, $value, $pcTTL );
912  }
913  }
914 
915  return $value;
916  }
917 
931  protected function doGetWithSetCallback( $key, $ttl, $callback, array $opts, &$asOf = null ) {
932  $lowTTL = isset( $opts['lowTTL'] ) ? $opts['lowTTL'] : min( self::LOW_TTL, $ttl );
933  $lockTSE = isset( $opts['lockTSE'] ) ? $opts['lockTSE'] : self::TSE_NONE;
934  $checkKeys = isset( $opts['checkKeys'] ) ? $opts['checkKeys'] : [];
935  $busyValue = isset( $opts['busyValue'] ) ? $opts['busyValue'] : null;
936  $popWindow = isset( $opts['hotTTR'] ) ? $opts['hotTTR'] : self::HOT_TTR;
937  $ageNew = isset( $opts['ageNew'] ) ? $opts['ageNew'] : self::AGE_NEW;
938  $minTime = isset( $opts['minAsOf'] ) ? $opts['minAsOf'] : self::MIN_TIMESTAMP_NONE;
939  $versioned = isset( $opts['version'] );
940 
941  // Get the current key value
942  $curTTL = null;
943  $cValue = $this->get( $key, $curTTL, $checkKeys, $asOf ); // current value
944  $value = $cValue; // return value
945 
946  $preCallbackTime = microtime( true );
947  // Determine if a cached value regeneration is needed or desired
948  if ( $value !== false
949  && $curTTL > 0
950  && $this->isValid( $value, $versioned, $asOf, $minTime )
951  && !$this->worthRefreshExpiring( $curTTL, $lowTTL )
952  && !$this->worthRefreshPopular( $asOf, $ageNew, $popWindow, $preCallbackTime )
953  ) {
954  return $value;
955  }
956 
957  // A deleted key with a negative TTL left must be tombstoned
958  $isTombstone = ( $curTTL !== null && $value === false );
959  // Assume a key is hot if requested soon after invalidation
960  $isHot = ( $curTTL !== null && $curTTL <= 0 && abs( $curTTL ) <= $lockTSE );
961  // Use the mutex if there is no value and a busy fallback is given
962  $checkBusy = ( $busyValue !== null && $value === false );
963  // Decide whether a single thread should handle regenerations.
964  // This avoids stampedes when $checkKeys are bumped and when preemptive
965  // renegerations take too long. It also reduces regenerations while $key
966  // is tombstoned. This balances cache freshness with avoiding DB load.
967  $useMutex = ( $isHot || ( $isTombstone && $lockTSE > 0 ) || $checkBusy );
968 
969  $lockAcquired = false;
970  if ( $useMutex ) {
971  // Acquire a datacenter-local non-blocking lock
972  if ( $this->cache->add( self::MUTEX_KEY_PREFIX . $key, 1, self::LOCK_TTL ) ) {
973  // Lock acquired; this thread should update the key
974  $lockAcquired = true;
975  } elseif ( $value !== false && $this->isValid( $value, $versioned, $asOf, $minTime ) ) {
976  // If it cannot be acquired; then the stale value can be used
977  return $value;
978  } else {
979  // Use the INTERIM value for tombstoned keys to reduce regeneration load.
980  // For hot keys, either another thread has the lock or the lock failed;
981  // use the INTERIM value from the last thread that regenerated it.
982  $wrapped = $this->cache->get( self::INTERIM_KEY_PREFIX . $key );
983  list( $value ) = $this->unwrap( $wrapped, microtime( true ) );
984  if ( $value !== false && $this->isValid( $value, $versioned, $asOf, $minTime ) ) {
985  $asOf = $wrapped[self::FLD_TIME];
986 
987  return $value;
988  }
989  // Use the busy fallback value if nothing else
990  if ( $busyValue !== null ) {
991  return is_callable( $busyValue ) ? $busyValue() : $busyValue;
992  }
993  }
994  }
995 
996  if ( !is_callable( $callback ) ) {
997  throw new InvalidArgumentException( "Invalid cache miss callback provided." );
998  }
999 
1000  // Generate the new value from the callback...
1001  $setOpts = [];
1003  try {
1004  $value = call_user_func_array( $callback, [ $cValue, &$ttl, &$setOpts, $asOf ] );
1005  } finally {
1007  }
1008  // When delete() is called, writes are write-holed by the tombstone,
1009  // so use a special INTERIM key to pass the new value around threads.
1010  if ( ( $isTombstone && $lockTSE > 0 ) && $value !== false && $ttl >= 0 ) {
1011  $tempTTL = max( 1, (int)$lockTSE ); // set() expects seconds
1012  $newAsOf = microtime( true );
1013  $wrapped = $this->wrap( $value, $tempTTL, $newAsOf );
1014  // Avoid using set() to avoid pointless mcrouter broadcasting
1015  $this->cache->merge(
1016  self::INTERIM_KEY_PREFIX . $key,
1017  function () use ( $wrapped ) {
1018  return $wrapped;
1019  },
1020  $tempTTL,
1021  1
1022  );
1023  }
1024 
1025  if ( $value !== false && $ttl >= 0 ) {
1026  $setOpts['lockTSE'] = $lockTSE;
1027  // Use best known "since" timestamp if not provided
1028  $setOpts += [ 'since' => $preCallbackTime ];
1029  // Update the cache; this will fail if the key is tombstoned
1030  $this->set( $key, $value, $ttl, $setOpts );
1031  }
1032 
1033  if ( $lockAcquired ) {
1034  // Avoid using delete() to avoid pointless mcrouter broadcasting
1035  $this->cache->changeTTL( self::MUTEX_KEY_PREFIX . $key, 1 );
1036  }
1037 
1038  return $value;
1039  }
1040 
1098  final public function getMultiWithSetCallback(
1099  ArrayIterator $keyedIds, $ttl, callable $callback, array $opts = []
1100  ) {
1101  $keysWarmUp = iterator_to_array( $keyedIds, true );
1102  $checkKeys = isset( $opts['checkKeys'] ) ? $opts['checkKeys'] : [];
1103  foreach ( $checkKeys as $i => $checkKeyOrKeys ) {
1104  if ( is_int( $i ) ) {
1105  $keysWarmUp[] = $checkKeyOrKeys;
1106  } else {
1107  $keysWarmUp = array_merge( $keysWarmUp, $checkKeyOrKeys );
1108  }
1109  }
1110 
1111  $this->warmupCache = $this->cache->getMulti( $keysWarmUp );
1112  $this->warmupCache += array_fill_keys( $keysWarmUp, false );
1113 
1114  // Wrap $callback to match the getWithSetCallback() format while passing $id to $callback
1115  $id = null;
1116  $func = function ( $oldValue, &$ttl, array $setOpts, $oldAsOf ) use ( $callback, &$id ) {
1117  return $callback( $id, $oldValue, $ttl, $setOpts, $oldAsOf );
1118  };
1119 
1120  $values = [];
1121  foreach ( $keyedIds as $key => $id ) {
1122  $values[$key] = $this->getWithSetCallback( $key, $ttl, $func, $opts );
1123  }
1124 
1125  $this->warmupCache = [];
1126 
1127  return $values;
1128  }
1129 
1136  public function makeKey() {
1137  return call_user_func_array( [ $this->cache, __FUNCTION__ ], func_get_args() );
1138  }
1139 
1146  public function makeGlobalKey() {
1147  return call_user_func_array( [ $this->cache, __FUNCTION__ ], func_get_args() );
1148  }
1149 
1156  public function makeMultiKeys( array $entities, callable $keyFunc ) {
1157  $map = [];
1158  foreach ( $entities as $entity ) {
1159  $map[$keyFunc( $entity, $this )] = $entity;
1160  }
1161 
1162  return new ArrayIterator( $map );
1163  }
1164 
1169  final public function getLastError() {
1170  if ( $this->lastRelayError ) {
1171  // If the cache and the relayer failed, focus on the latter.
1172  // An update not making it to the relayer means it won't show up
1173  // in other DCs (nor will consistent re-hashing see up-to-date values).
1174  // On the other hand, if just the cache update failed, then it should
1175  // eventually be applied by the relayer.
1176  return $this->lastRelayError;
1177  }
1178 
1179  $code = $this->cache->getLastError();
1180  switch ( $code ) {
1181  case BagOStuff::ERR_NONE:
1182  return self::ERR_NONE;
1184  return self::ERR_NO_RESPONSE;
1186  return self::ERR_UNREACHABLE;
1187  default:
1188  return self::ERR_UNEXPECTED;
1189  }
1190  }
1191 
1195  final public function clearLastError() {
1196  $this->cache->clearLastError();
1197  $this->lastRelayError = self::ERR_NONE;
1198  }
1199 
1205  public function clearProcessCache() {
1206  $this->processCaches = [];
1207  }
1208 
1214  public function getQoS( $flag ) {
1215  return $this->cache->getQoS( $flag );
1216  }
1217 
1241  public function adaptiveTTL( $mtime, $maxTTL, $minTTL = 30, $factor = .2 ) {
1242  if ( is_float( $mtime ) || ctype_digit( $mtime ) ) {
1243  $mtime = (int)$mtime; // handle fractional seconds and string integers
1244  }
1245 
1246  if ( !is_int( $mtime ) || $mtime <= 0 ) {
1247  return $minTTL; // no last-modified time provided
1248  }
1249 
1250  $age = time() - $mtime;
1251 
1252  return (int)min( $maxTTL, max( $minTTL, $factor * $age ) );
1253  }
1254 
1265  protected function relayPurge( $key, $ttl, $holdoff ) {
1266  if ( $this->purgeRelayer instanceof EventRelayerNull ) {
1267  // This handles the mcrouter and the single-DC case
1268  $ok = $this->cache->set( $key,
1269  $this->makePurgeValue( microtime( true ), self::HOLDOFF_NONE ),
1270  $ttl
1271  );
1272  } else {
1273  $event = $this->cache->modifySimpleRelayEvent( [
1274  'cmd' => 'set',
1275  'key' => $key,
1276  'val' => 'PURGED:$UNIXTIME$:' . (int)$holdoff,
1277  'ttl' => max( $ttl, 1 ),
1278  'sbt' => true, // substitute $UNIXTIME$ with actual microtime
1279  ] );
1280 
1281  $ok = $this->purgeRelayer->notify( $this->purgeChannel, $event );
1282  if ( !$ok ) {
1283  $this->lastRelayError = self::ERR_RELAY;
1284  }
1285  }
1286 
1287  return $ok;
1288  }
1289 
1296  protected function relayDelete( $key ) {
1297  if ( $this->purgeRelayer instanceof EventRelayerNull ) {
1298  // This handles the mcrouter and the single-DC case
1299  $ok = $this->cache->delete( $key );
1300  } else {
1301  $event = $this->cache->modifySimpleRelayEvent( [
1302  'cmd' => 'delete',
1303  'key' => $key,
1304  ] );
1305 
1306  $ok = $this->purgeRelayer->notify( $this->purgeChannel, $event );
1307  if ( !$ok ) {
1308  $this->lastRelayError = self::ERR_RELAY;
1309  }
1310  }
1311 
1312  return $ok;
1313  }
1314 
1327  protected function worthRefreshExpiring( $curTTL, $lowTTL ) {
1328  if ( $curTTL >= $lowTTL ) {
1329  return false;
1330  } elseif ( $curTTL <= 0 ) {
1331  return true;
1332  }
1333 
1334  $chance = ( 1 - $curTTL / $lowTTL );
1335 
1336  return mt_rand( 1, 1e9 ) <= 1e9 * $chance;
1337  }
1338 
1354  protected function worthRefreshPopular( $asOf, $ageNew, $timeTillRefresh, $now ) {
1355  $age = $now - $asOf;
1356  $timeOld = $age - $ageNew;
1357  if ( $timeOld <= 0 ) {
1358  return false;
1359  }
1360 
1361  // Lifecycle is: new, ramp-up refresh chance, full refresh chance
1362  $refreshWindowSec = max( $timeTillRefresh - $ageNew - self::RAMPUP_TTL / 2, 1 );
1363  // P(refresh) * (# hits in $refreshWindowSec) = (expected # of refreshes)
1364  // P(refresh) * ($refreshWindowSec * $popularHitsPerSec) = 1
1365  // P(refresh) = 1/($refreshWindowSec * $popularHitsPerSec)
1366  $chance = 1 / ( self::HIT_RATE_HIGH * $refreshWindowSec );
1367 
1368  // Ramp up $chance from 0 to its nominal value over RAMPUP_TTL seconds to avoid stampedes
1369  $chance *= ( $timeOld <= self::RAMPUP_TTL ) ? $timeOld / self::RAMPUP_TTL : 1;
1370 
1371  return mt_rand( 1, 1e9 ) <= 1e9 * $chance;
1372  }
1373 
1383  protected function isValid( $value, $versioned, $asOf, $minTime ) {
1384  if ( $versioned && !isset( $value[self::VFLD_VERSION] ) ) {
1385  return false;
1386  } elseif ( $minTime > 0 && $asOf < $minTime ) {
1387  return false;
1388  }
1389 
1390  return true;
1391  }
1392 
1401  protected function wrap( $value, $ttl, $now ) {
1402  return [
1403  self::FLD_VERSION => self::VERSION,
1404  self::FLD_VALUE => $value,
1405  self::FLD_TTL => $ttl,
1406  self::FLD_TIME => $now
1407  ];
1408  }
1409 
1417  protected function unwrap( $wrapped, $now ) {
1418  // Check if the value is a tombstone
1419  $purge = self::parsePurgeValue( $wrapped );
1420  if ( $purge !== false ) {
1421  // Purged values should always have a negative current $ttl
1422  $curTTL = min( $purge[self::FLD_TIME] - $now, self::TINY_NEGATIVE );
1423  return [ false, $curTTL ];
1424  }
1425 
1426  if ( !is_array( $wrapped ) // not found
1427  || !isset( $wrapped[self::FLD_VERSION] ) // wrong format
1428  || $wrapped[self::FLD_VERSION] !== self::VERSION // wrong version
1429  ) {
1430  return [ false, null ];
1431  }
1432 
1433  $flags = isset( $wrapped[self::FLD_FLAGS] ) ? $wrapped[self::FLD_FLAGS] : 0;
1434  if ( ( $flags & self::FLG_STALE ) == self::FLG_STALE ) {
1435  // Treat as expired, with the cache time as the expiration
1436  $age = $now - $wrapped[self::FLD_TIME];
1437  $curTTL = min( -$age, self::TINY_NEGATIVE );
1438  } elseif ( $wrapped[self::FLD_TTL] > 0 ) {
1439  // Get the approximate time left on the key
1440  $age = $now - $wrapped[self::FLD_TIME];
1441  $curTTL = max( $wrapped[self::FLD_TTL] - $age, 0.0 );
1442  } else {
1443  // Key had no TTL, so the time left is unbounded
1444  $curTTL = INF;
1445  }
1446 
1447  return [ $wrapped[self::FLD_VALUE], $curTTL ];
1448  }
1449 
1455  protected static function prefixCacheKeys( array $keys, $prefix ) {
1456  $res = [];
1457  foreach ( $keys as $key ) {
1458  $res[] = $prefix . $key;
1459  }
1460 
1461  return $res;
1462  }
1463 
1469  protected static function parsePurgeValue( $value ) {
1470  if ( !is_string( $value ) ) {
1471  return false;
1472  }
1473  $segments = explode( ':', $value, 3 );
1474  if ( !isset( $segments[0] ) || !isset( $segments[1] )
1475  || "{$segments[0]}:" !== self::PURGE_VAL_PREFIX
1476  ) {
1477  return false;
1478  }
1479  if ( !isset( $segments[2] ) ) {
1480  // Back-compat with old purge values without holdoff
1481  $segments[2] = self::HOLDOFF_TTL;
1482  }
1483  return [
1484  self::FLD_TIME => (float)$segments[1],
1485  self::FLD_HOLDOFF => (int)$segments[2],
1486  ];
1487  }
1488 
1494  protected function makePurgeValue( $timestamp, $holdoff ) {
1495  return self::PURGE_VAL_PREFIX . (float)$timestamp . ':' . (int)$holdoff;
1496  }
1497 
1502  protected function getProcessCache( $group ) {
1503  if ( !isset( $this->processCaches[$group] ) ) {
1504  list( , $n ) = explode( ':', $group );
1505  $this->processCaches[$group] = new HashBagOStuff( [ 'maxKeys' => (int)$n ] );
1506  }
1507 
1508  return $this->processCaches[$group];
1509  }
1510 }
processCheckKeys(array $timeKeys, array $wrappedValues, $now)
set($key, $value, $ttl=0, array $opts=[])
Set the value of a key in cache.
string $purgeChannel
Purge channel name.
doGetWithSetCallback($key, $ttl, $callback, array $opts, &$asOf=null)
Do the actual I/O for getWithSetCallback() when needed.
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition: deferred.txt:11
const TTL_LAGGED
Max TTL to store keys when a data sourced is lagged.
const TINY_NEGATIVE
Tiny negative float to use when CTL comes up >= 0 due to clock skew.
the array() calling protocol came about after MediaWiki 1.4rc1.
const CHECK_KEY_TTL
Seconds to keep dependency purge keys around.
processing should stop and the error should be shown to the user * false
Definition: hooks.txt:189
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
clearProcessCache()
Clear the in-process caches; useful for testing.
integer $callbackDepth
Callback stack depth for getWithSetCallback()
static prefixCacheKeys(array $keys, $prefix)
adaptiveTTL($mtime, $maxTTL, $minTTL=30, $factor=.2)
Get a TTL that is higher for objects that have not changed recently.
unwrap($wrapped, $now)
Do not use this method outside WANObjectCache.
const MAX_COMMIT_DELAY
Max time expected to pass between delete() and DB commit finishing.
const TTL_UNCACHEABLE
Idiom for getWithSetCallback() callbacks to avoid calling set()
LoggerInterface $logger
mixed[] $warmupCache
Temporary warm-up cache.
EventRelayer $purgeRelayer
Bus that handles purge broadcasts.
const ERR_NO_RESPONSE
Definition: BagOStuff.php:78
getMultiWithSetCallback(ArrayIterator $keyedIds, $ttl, callable $callback, array $opts=[])
Method to fetch/regenerate multiple cache keys at once.
touchCheckKey($key, $holdoff=self::HOLDOFF_TTL)
Purge a "check" key from all datacenters, invalidating keys that use it.
This code would result in ircNotify being run twice when an article is and once for brion Hooks can return three possible true was required This is the default since MediaWiki *some string
Definition: hooks.txt:177
$value
const DEFAULT_PURGE_CHANNEL
it s the revision text itself In either if gzip is the revision text is gzipped $flags
Definition: hooks.txt:2703
const ERR_UNREACHABLE
Definition: BagOStuff.php:79
No-op class for publishing messages into a PubSub system.
getWithSetCallback($key, $ttl, $callback, array $opts=[])
Method to fetch/regenerate cache keys.
The index of the header message $result[1]=The index of the body text message $result[2 through n]=Parameters passed to body text message.Please note the header message cannot receive/use parameters. 'ImportHandleLogItemXMLTag':When parsing a XML tag in a log item.Return false to stop further processing of the tag $reader:XMLReader object $logInfo:Array of information 'ImportHandlePageXMLTag':When parsing a XML tag in a page.Return false to stop further processing of the tag $reader:XMLReader object &$pageInfo:Array of information 'ImportHandleRevisionXMLTag':When parsing a XML tag in a page revision.Return false to stop further processing of the tag $reader:XMLReader object $pageInfo:Array of page information $revisionInfo:Array of revision information 'ImportHandleToplevelXMLTag':When parsing a top level XML tag.Return false to stop further processing of the tag $reader:XMLReader object 'ImportHandleUploadXMLTag':When parsing a XML tag in a file upload.Return false to stop further processing of the tag $reader:XMLReader object $revisionInfo:Array of information 'ImportLogInterwikiLink':Hook to change the interwiki link used in log entries and edit summaries for transwiki imports.&$fullInterwikiPrefix:Interwiki prefix, may contain colons.&$pageTitle:String that contains page title. 'ImportSources':Called when reading from the $wgImportSources configuration variable.Can be used to lazy-load the import sources list.&$importSources:The value of $wgImportSources.Modify as necessary.See the comment in DefaultSettings.php for the detail of how to structure this array. 'InfoAction':When building information to display on the action=info page.$context:IContextSource object &$pageInfo:Array of information 'InitializeArticleMaybeRedirect':MediaWiki check to see if title is a redirect.&$title:Title object for the current page &$request:WebRequest &$ignoreRedirect:boolean to skip redirect check &$target:Title/string of redirect target &$article:Article object 'InternalParseBeforeLinks':during Parser's internalParse method before links but after nowiki/noinclude/includeonly/onlyinclude and other processings.&$parser:Parser object &$text:string containing partially parsed text &$stripState:Parser's internal StripState object 'InternalParseBeforeSanitize':during Parser's internalParse method just before the parser removes unwanted/dangerous HTML tags and after nowiki/noinclude/includeonly/onlyinclude and other processings.Ideal for syntax-extensions after template/parser function execution which respect nowiki and HTML-comments.&$parser:Parser object &$text:string containing partially parsed text &$stripState:Parser's internal StripState object 'InterwikiLoadPrefix':When resolving if a given prefix is an interwiki or not.Return true without providing an interwiki to continue interwiki search.$prefix:interwiki prefix we are looking for.&$iwData:output array describing the interwiki with keys iw_url, iw_local, iw_trans and optionally iw_api and iw_wikiid. 'InvalidateEmailComplete':Called after a user's email has been invalidated successfully.$user:user(object) whose email is being invalidated 'IRCLineURL':When constructing the URL to use in an IRC notification.Callee may modify $url and $query, URL will be constructed as $url.$query &$url:URL to index.php &$query:Query string $rc:RecentChange object that triggered url generation 'IsFileCacheable':Override the result of Article::isFileCacheable()(if true) &$article:article(object) being checked 'IsTrustedProxy':Override the result of IP::isTrustedProxy() &$ip:IP being check &$result:Change this value to override the result of IP::isTrustedProxy() 'IsUploadAllowedFromUrl':Override the result of UploadFromUrl::isAllowedUrl() $url:URL used to upload from &$allowed:Boolean indicating if uploading is allowed for given URL 'isValidEmailAddr':Override the result of Sanitizer::validateEmail(), for instance to return false if the domain name doesn't match your organization.$addr:The e-mail address entered by the user &$result:Set this and return false to override the internal checks 'isValidPassword':Override the result of User::isValidPassword() $password:The password entered by the user &$result:Set this and return false to override the internal checks $user:User the password is being validated for 'Language::getMessagesFileName':$code:The language code or the language we're looking for a messages file for &$file:The messages file path, you can override this to change the location. 'LanguageGetMagic':DEPRECATED!Use $magicWords in a file listed in $wgExtensionMessagesFiles instead.Use this to define synonyms of magic words depending of the language &$magicExtensions:associative array of magic words synonyms $lang:language code(string) 'LanguageGetNamespaces':Provide custom ordering for namespaces or remove namespaces.Do not use this hook to add namespaces.Use CanonicalNamespaces for that.&$namespaces:Array of namespaces indexed by their numbers 'LanguageGetSpecialPageAliases':DEPRECATED!Use $specialPageAliases in a file listed in $wgExtensionMessagesFiles instead.Use to define aliases of special pages names depending of the language &$specialPageAliases:associative array of magic words synonyms $lang:language code(string) 'LanguageGetTranslatedLanguageNames':Provide translated language names.&$names:array of language code=> language name $code:language of the preferred translations 'LanguageLinks':Manipulate a page's language links.This is called in various places to allow extensions to define the effective language links for a page.$title:The page's Title.&$links:Associative array mapping language codes to prefixed links of the form"language:title".&$linkFlags:Associative array mapping prefixed links to arrays of flags.Currently unused, but planned to provide support for marking individual language links in the UI, e.g.for featured articles. 'LanguageSelector':Hook to change the language selector available on a page.$out:The output page.$cssClassName:CSS class name of the language selector. 'LinkBegin':DEPRECATED!Use HtmlPageLinkRendererBegin instead.Used when generating internal and interwiki links in Linker::link(), before processing starts.Return false to skip default processing and return $ret.See documentation for Linker::link() for details on the expected meanings of parameters.$skin:the Skin object $target:the Title that the link is pointing to &$html:the contents that the< a > tag should have(raw HTML) $result
Definition: hooks.txt:1934
you have access to all of the normal MediaWiki so you can get a DB use the cache
Definition: maintenance.txt:52
static newEmpty()
Get an instance that wraps EmptyBagOStuff.
const LOW_TTL
Default remaining TTL at which to consider pre-emptive regeneration.
getLastError()
Get the "last error" registered; clearLastError() should be called manually.
if($limit) $timestamp
$res
Definition: database.txt:21
__construct(array $params)
const RAMPUP_TTL
Seconds to ramp up to the "popularity" refresh chance after a key is no longer new.
$params
const ERR_NONE
Possible values for getLastError()
Definition: BagOStuff.php:77
getCheckKeyTime($key)
Fetch the value of a timestamp "check" key.
const LOCK_TSE
Default time-since-expiry on a miss that makes a key "hot".
A BagOStuff object with no objects in it.
const HOLDOFF_NONE
Idiom for delete() for "no hold-off".
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that probably a stub it is not rendered in wiki pages or galleries in category pages allow injecting custom HTML after the section Any uses of the hook need to handle escaping see BaseTemplate::getToolbox and BaseTemplate::makeListItem for details on the format of individual items inside of this array or by returning and letting standard HTTP rendering take place modifiable or by returning false and taking over the output modifiable & $code
Definition: hooks.txt:802
isValid($value, $versioned, $asOf, $minTime)
Check whether $value is appropriately versioned and not older than $minTime (if set) ...
const VERSION
Cache format version number.
clearLastError()
Clear the "last error" registry.
resetCheckKey($key)
Delete a "check" key from all datacenters, invalidating keys that use it.
const HOLDOFF_TTL
Seconds to tombstone keys on delete()
const MIN_TIMESTAMP_NONE
Idiom for getWithSetCallback() for "no minimum required as-of timestamp".
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
relayPurge($key, $ttl, $holdoff)
Do the actual async bus purge of a key.
int $lastRelayError
ERR_* constant for the "last error" registry.
worthRefreshPopular($asOf, $ageNew, $timeTillRefresh, $now)
Check if a key is due for randomized regeneration due to its popularity.
wrap($value, $ttl, $now)
Do not use this method outside WANObjectCache.
const HOT_TTR
The time length of the "popularity" refresh window for hot keys.
const AGE_NEW
Never consider performing "popularity" refreshes until a key reaches this age.
static parsePurgeValue($value)
const LOCK_TTL
Seconds to keep lock keys around.
worthRefreshExpiring($curTTL, $lowTTL)
Check if a key should be regenerated (using random probability)
makePurgeValue($timestamp, $holdoff)
getMulti(array $keys, &$curTTLs=[], array $checkKeys=[], array &$asOfs=[])
Fetch the value of several keys from cache.
relayDelete($key)
Do the actual async bus delete of a key.
Generic base class for storage interfaces.
const MAX_READ_LAG
Max replication+snapshot lag before applying TTL_LAGGED or disallowing set()
setLogger(LoggerInterface $logger)
const TSE_NONE
Idiom for getWithSetCallback() callbacks to 'lockTSE' logic.
const HIT_RATE_HIGH
Hits/second for a refresh to be expected within the "popularity" window.
makeMultiKeys(array $entities, callable $keyFunc)
see documentation in includes Linker php for Linker::makeImageLink & $time
Definition: hooks.txt:1749
BagOStuff $cache
The local datacenter cache.
HashBagOStuff[] $processCaches
Map of group PHP instance caches.