MediaWiki  1.33.1
BagOStuff.php
Go to the documentation of this file.
1 <?php
29 use Psr\Log\LoggerAwareInterface;
30 use Psr\Log\LoggerInterface;
31 use Psr\Log\NullLogger;
32 use Wikimedia\ScopedCallback;
33 use Wikimedia\WaitConditionLoop;
34 
58 abstract class BagOStuff implements IExpiringStore, LoggerAwareInterface {
60  protected $locks = [];
64  protected $keyspace = 'local';
66  protected $logger;
68  protected $asyncHandler;
70  protected $syncTimeout;
71 
73  private $debugMode = false;
75  private $duplicateKeyLookups = [];
77  private $reportDupes = false;
79  private $dupeTrackScheduled = false;
80 
82  protected $busyCallbacks = [];
83 
86 
88  protected $attrMap = [];
89 
91  const READ_LATEST = 1; // use latest data for replicated stores
92  const READ_VERIFIED = 2; // promise that caller can tell when keys are stale
94  const WRITE_SYNC = 4; // synchronously write to all locations for replicated stores
95  const WRITE_CACHE_ONLY = 8; // Only change state of the in-memory cache
96 
108  public function __construct( array $params = [] ) {
109  $this->setLogger( $params['logger'] ?? new NullLogger() );
110 
111  if ( isset( $params['keyspace'] ) ) {
112  $this->keyspace = $params['keyspace'];
113  }
114 
115  $this->asyncHandler = $params['asyncHandler'] ?? null;
116 
117  if ( !empty( $params['reportDupes'] ) && is_callable( $this->asyncHandler ) ) {
118  $this->reportDupes = true;
119  }
120 
121  $this->syncTimeout = $params['syncTimeout'] ?? 3;
122  }
123 
128  public function setLogger( LoggerInterface $logger ) {
129  $this->logger = $logger;
130  }
131 
135  public function setDebug( $bool ) {
136  $this->debugMode = $bool;
137  }
138 
151  final public function getWithSetCallback( $key, $ttl, $callback, $flags = 0 ) {
152  $value = $this->get( $key, $flags );
153 
154  if ( $value === false ) {
155  if ( !is_callable( $callback ) ) {
156  throw new InvalidArgumentException( "Invalid cache miss callback provided." );
157  }
158  $value = call_user_func( $callback );
159  if ( $value !== false ) {
160  $this->set( $key, $value, $ttl, $flags );
161  }
162  }
163 
164  return $value;
165  }
166 
180  public function get( $key, $flags = 0 ) {
181  $this->trackDuplicateKeys( $key );
182 
183  return $this->doGet( $key, $flags );
184  }
185 
190  private function trackDuplicateKeys( $key ) {
191  if ( !$this->reportDupes ) {
192  return;
193  }
194 
195  if ( !isset( $this->duplicateKeyLookups[$key] ) ) {
196  // Track that we have seen this key. This N-1 counting style allows
197  // easy filtering with array_filter() later.
198  $this->duplicateKeyLookups[$key] = 0;
199  } else {
200  $this->duplicateKeyLookups[$key] += 1;
201 
202  if ( $this->dupeTrackScheduled === false ) {
203  $this->dupeTrackScheduled = true;
204  // Schedule a callback that logs keys processed more than once by get().
205  call_user_func( $this->asyncHandler, function () {
206  $dups = array_filter( $this->duplicateKeyLookups );
207  foreach ( $dups as $key => $count ) {
208  $this->logger->warning(
209  'Duplicate get(): "{key}" fetched {count} times',
210  // Count is N-1 of the actual lookup count
211  [ 'key' => $key, 'count' => $count + 1, ]
212  );
213  }
214  } );
215  }
216  }
217  }
218 
225  abstract protected function doGet( $key, $flags = 0, &$casToken = null );
226 
236  abstract public function set( $key, $value, $exptime = 0, $flags = 0 );
237 
245  abstract public function delete( $key, $flags = 0 );
246 
256  abstract public function add( $key, $value, $exptime = 0, $flags = 0 );
257 
275  public function merge( $key, callable $callback, $exptime = 0, $attempts = 10, $flags = 0 ) {
276  return $this->mergeViaCas( $key, $callback, $exptime, $attempts, $flags );
277  }
278 
289  protected function mergeViaCas( $key, $callback, $exptime = 0, $attempts = 10, $flags = 0 ) {
290  do {
291  $casToken = null; // passed by reference
292  // Get the old value and CAS token from cache
293  $this->clearLastError();
294  $currentValue = $this->doGet( $key, self::READ_LATEST, $casToken );
295  if ( $this->getLastError() ) {
296  $this->logger->warning(
297  __METHOD__ . ' failed due to I/O error on get() for {key}.',
298  [ 'key' => $key ]
299  );
300 
301  return false; // don't spam retries (retry only on races)
302  }
303 
304  // Derive the new value from the old value
305  $value = call_user_func( $callback, $this, $key, $currentValue, $exptime );
306  $hadNoCurrentValue = ( $currentValue === false );
307  unset( $currentValue ); // free RAM in case the value is large
308 
309  $this->clearLastError();
310  if ( $value === false ) {
311  $success = true; // do nothing
312  } elseif ( $hadNoCurrentValue ) {
313  // Try to create the key, failing if it gets created in the meantime
314  $success = $this->add( $key, $value, $exptime, $flags );
315  } else {
316  // Try to update the key, failing if it gets changed in the meantime
317  $success = $this->cas( $casToken, $key, $value, $exptime, $flags );
318  }
319  if ( $this->getLastError() ) {
320  $this->logger->warning(
321  __METHOD__ . ' failed due to I/O error for {key}.',
322  [ 'key' => $key ]
323  );
324 
325  return false; // IO error; don't spam retries
326  }
327  } while ( !$success && --$attempts );
328 
329  return $success;
330  }
331 
343  protected function cas( $casToken, $key, $value, $exptime = 0, $flags = 0 ) {
344  if ( !$this->lock( $key, 0 ) ) {
345  return false; // non-blocking
346  }
347 
348  $curCasToken = null; // passed by reference
349  $this->doGet( $key, self::READ_LATEST, $curCasToken );
350  if ( $casToken === $curCasToken ) {
351  $success = $this->set( $key, $value, $exptime, $flags );
352  } else {
353  $this->logger->info(
354  __METHOD__ . ' failed due to race condition for {key}.',
355  [ 'key' => $key ]
356  );
357 
358  $success = false; // mismatched or failed
359  }
360 
361  $this->unlock( $key );
362 
363  return $success;
364  }
365 
377  public function changeTTL( $key, $expiry = 0, $flags = 0 ) {
378  $found = false;
379 
380  $ok = $this->merge(
381  $key,
382  function ( $cache, $ttl, $currentValue ) use ( &$found ) {
383  $found = ( $currentValue !== false );
384 
385  return $currentValue; // nothing is written if this is false
386  },
387  $expiry,
388  1, // 1 attempt
389  $flags
390  );
391 
392  return ( $ok && $found );
393  }
394 
406  public function lock( $key, $timeout = 6, $expiry = 6, $rclass = '' ) {
407  // Avoid deadlocks and allow lock reentry if specified
408  if ( isset( $this->locks[$key] ) ) {
409  if ( $rclass != '' && $this->locks[$key]['class'] === $rclass ) {
410  ++$this->locks[$key]['depth'];
411  return true;
412  } else {
413  return false;
414  }
415  }
416 
417  $fname = __METHOD__;
418  $expiry = min( $expiry ?: INF, self::TTL_DAY );
419  $loop = new WaitConditionLoop(
420  function () use ( $key, $expiry, $fname ) {
421  $this->clearLastError();
422  if ( $this->add( "{$key}:lock", 1, $expiry ) ) {
423  return WaitConditionLoop::CONDITION_REACHED; // locked!
424  } elseif ( $this->getLastError() ) {
425  $this->logger->warning(
426  $fname . ' failed due to I/O error for {key}.',
427  [ 'key' => $key ]
428  );
429 
430  return WaitConditionLoop::CONDITION_ABORTED; // network partition?
431  }
432 
433  return WaitConditionLoop::CONDITION_CONTINUE;
434  },
435  $timeout
436  );
437 
438  $code = $loop->invoke();
439  $locked = ( $code === $loop::CONDITION_REACHED );
440  if ( $locked ) {
441  $this->locks[$key] = [ 'class' => $rclass, 'depth' => 1 ];
442  } elseif ( $code === $loop::CONDITION_TIMED_OUT ) {
443  $this->logger->warning(
444  "$fname failed due to timeout for {key}.",
445  [ 'key' => $key, 'timeout' => $timeout ]
446  );
447  }
448 
449  return $locked;
450  }
451 
458  public function unlock( $key ) {
459  if ( isset( $this->locks[$key] ) && --$this->locks[$key]['depth'] <= 0 ) {
460  unset( $this->locks[$key] );
461 
462  $ok = $this->delete( "{$key}:lock" );
463  if ( !$ok ) {
464  $this->logger->warning(
465  __METHOD__ . ' failed to release lock for {key}.',
466  [ 'key' => $key ]
467  );
468  }
469 
470  return $ok;
471  }
472 
473  return true;
474  }
475 
492  final public function getScopedLock( $key, $timeout = 6, $expiry = 30, $rclass = '' ) {
493  $expiry = min( $expiry ?: INF, self::TTL_DAY );
494 
495  if ( !$this->lock( $key, $timeout, $expiry, $rclass ) ) {
496  return null;
497  }
498 
499  $lSince = $this->getCurrentTime(); // lock timestamp
500 
501  return new ScopedCallback( function () use ( $key, $lSince, $expiry ) {
502  $latency = 0.050; // latency skew (err towards keeping lock present)
503  $age = ( $this->getCurrentTime() - $lSince + $latency );
504  if ( ( $age + $latency ) >= $expiry ) {
505  $this->logger->warning(
506  "Lock for {key} held too long ({age} sec).",
507  [ 'key' => $key, 'age' => $age ]
508  );
509  return; // expired; it's not "safe" to delete the key
510  }
511  $this->unlock( $key );
512  } );
513  }
514 
524  public function deleteObjectsExpiringBefore( $date, $progressCallback = false ) {
525  // stub
526  return false;
527  }
528 
535  public function getMulti( array $keys, $flags = 0 ) {
536  $res = [];
537  foreach ( $keys as $key ) {
538  $val = $this->get( $key, $flags );
539  if ( $val !== false ) {
540  $res[$key] = $val;
541  }
542  }
543 
544  return $res;
545  }
546 
555  public function setMulti( array $data, $exptime = 0, $flags = 0 ) {
556  $res = true;
557  foreach ( $data as $key => $value ) {
558  if ( !$this->set( $key, $value, $exptime, $flags ) ) {
559  $res = false;
560  }
561  }
562 
563  return $res;
564  }
565 
573  public function deleteMulti( array $keys, $flags = 0 ) {
574  $res = true;
575  foreach ( $keys as $key ) {
576  $res = $this->delete( $key, $flags ) && $res;
577  }
578 
579  return $res;
580  }
581 
588  abstract public function incr( $key, $value = 1 );
589 
596  public function decr( $key, $value = 1 ) {
597  return $this->incr( $key, - $value );
598  }
599 
612  public function incrWithInit( $key, $ttl, $value = 1, $init = 1 ) {
613  $this->clearLastError();
614  $newValue = $this->incr( $key, $value );
615  if ( $newValue === false && !$this->getLastError() ) {
616  // No key set; initialize
617  $newValue = $this->add( $key, (int)$init, $ttl ) ? $init : false;
618  if ( $newValue === false && !$this->getLastError() ) {
619  // Raced out initializing; increment
620  $newValue = $this->incr( $key, $value );
621  }
622  }
623 
624  return $newValue;
625  }
626 
632  public function getLastError() {
633  return $this->lastError;
634  }
635 
640  public function clearLastError() {
641  $this->lastError = self::ERR_NONE;
642  }
643 
649  protected function setLastError( $err ) {
650  $this->lastError = $err;
651  }
652 
673  public function addBusyCallback( callable $workCallback ) {
674  $this->busyCallbacks[] = $workCallback;
675  }
676 
680  protected function debug( $text ) {
681  if ( $this->debugMode ) {
682  $this->logger->debug( "{class} debug: $text", [
683  'class' => static::class,
684  ] );
685  }
686  }
687 
692  protected function expiryIsRelative( $exptime ) {
693  return ( $exptime != 0 && $exptime < ( 10 * self::TTL_YEAR ) );
694  }
695 
701  protected function convertToExpiry( $exptime ) {
702  if ( $this->expiryIsRelative( $exptime ) ) {
703  return (int)$this->getCurrentTime() + $exptime;
704  } else {
705  return $exptime;
706  }
707  }
708 
716  protected function convertToRelative( $exptime ) {
717  if ( $exptime >= ( 10 * self::TTL_YEAR ) ) {
718  $exptime -= (int)$this->getCurrentTime();
719  if ( $exptime <= 0 ) {
720  $exptime = 1;
721  }
722  return $exptime;
723  } else {
724  return $exptime;
725  }
726  }
727 
734  protected function isInteger( $value ) {
735  return ( is_int( $value ) || ctype_digit( $value ) );
736  }
737 
746  public function makeKeyInternal( $keyspace, $args ) {
747  $key = $keyspace;
748  foreach ( $args as $arg ) {
749  $key .= ':' . str_replace( ':', '%3A', $arg );
750  }
751  return strtr( $key, ' ', '_' );
752  }
753 
762  public function makeGlobalKey( $class, $component = null ) {
763  return $this->makeKeyInternal( 'global', func_get_args() );
764  }
765 
774  public function makeKey( $class, $component = null ) {
775  return $this->makeKeyInternal( $this->keyspace, func_get_args() );
776  }
777 
783  public function getQoS( $flag ) {
784  return $this->attrMap[$flag] ?? self::QOS_UNKNOWN;
785  }
786 
793  protected function mergeFlagMaps( array $bags ) {
794  $map = [];
795  foreach ( $bags as $bag ) {
796  foreach ( $bag->attrMap as $attr => $rank ) {
797  if ( isset( $map[$attr] ) ) {
798  $map[$attr] = min( $map[$attr], $rank );
799  } else {
800  $map[$attr] = $rank;
801  }
802  }
803  }
804 
805  return $map;
806  }
807 
812  protected function getCurrentTime() {
813  return $this->wallClockOverride ?: microtime( true );
814  }
815 
820  public function setMockTime( &$time ) {
821  $this->wallClockOverride =& $time;
822  }
823 }
BagOStuff\isInteger
isInteger( $value)
Check if a value is an integer.
Definition: BagOStuff.php:734
BagOStuff\decr
decr( $key, $value=1)
Decrease stored value of $key by $value while preserving its TTL.
Definition: BagOStuff.php:596
false
processing should stop and the error should be shown to the user * false
Definition: hooks.txt:187
BagOStuff\getQoS
getQoS( $flag)
Definition: BagOStuff.php:783
BagOStuff\getLastError
getLastError()
Get the "last error" registered; clearLastError() should be called manually.
Definition: BagOStuff.php:632
BagOStuff\add
add( $key, $value, $exptime=0, $flags=0)
Insert an item if it does not already exist.
IExpiringStore\ERR_NONE
const ERR_NONE
Definition: IExpiringStore.php:63
BagOStuff\WRITE_SYNC
const WRITE_SYNC
Bitfield constants for set()/merge()
Definition: BagOStuff.php:94
BagOStuff\$reportDupes
bool $reportDupes
Definition: BagOStuff.php:77
BagOStuff\expiryIsRelative
expiryIsRelative( $exptime)
Definition: BagOStuff.php:692
BagOStuff\deleteObjectsExpiringBefore
deleteObjectsExpiringBefore( $date, $progressCallback=false)
Delete all objects expiring before a certain date.
Definition: BagOStuff.php:524
BagOStuff\debug
debug( $text)
Definition: BagOStuff.php:680
BagOStuff\mergeViaCas
mergeViaCas( $key, $callback, $exptime=0, $attempts=10, $flags=0)
Definition: BagOStuff.php:289
BagOStuff\$locks
array[] $locks
Lock tracking.
Definition: BagOStuff.php:60
$params
$params
Definition: styleTest.css.php:44
BagOStuff\trackDuplicateKeys
trackDuplicateKeys( $key)
Track the number of times that a given key has been used.
Definition: BagOStuff.php:190
BagOStuff
Class representing a cache/ephemeral data store.
Definition: BagOStuff.php:58
BagOStuff\$duplicateKeyLookups
array $duplicateKeyLookups
Definition: BagOStuff.php:75
BagOStuff\makeKey
makeKey( $class, $component=null)
Make a cache key, scoped to this instance's keyspace.
Definition: BagOStuff.php:774
$res
$res
Definition: database.txt:21
BagOStuff\setDebug
setDebug( $bool)
Definition: BagOStuff.php:135
$success
$success
Definition: NoLocalSettings.php:42
BagOStuff\doGet
doGet( $key, $flags=0, &$casToken=null)
BagOStuff\$logger
LoggerInterface $logger
Definition: BagOStuff.php:66
php
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
BagOStuff\setMockTime
setMockTime(&$time)
Definition: BagOStuff.php:820
BagOStuff\getWithSetCallback
getWithSetCallback( $key, $ttl, $callback, $flags=0)
Get an item with the given key, regenerating and setting it if not found.
Definition: BagOStuff.php:151
BagOStuff\unlock
unlock( $key)
Release an advisory lock on a key string.
Definition: BagOStuff.php:458
BagOStuff\clearLastError
clearLastError()
Clear the "last error" registry.
Definition: BagOStuff.php:640
$data
$data
Utility to generate mapping file used in mw.Title (phpCharToUpper.json)
Definition: generatePhpCharToUpperMappings.php:13
IExpiringStore
Generic interface for lightweight expiring object stores.
Definition: IExpiringStore.php:31
BagOStuff\$lastError
int $lastError
ERR_* class constant.
Definition: BagOStuff.php:62
BagOStuff\setLogger
setLogger(LoggerInterface $logger)
Definition: BagOStuff.php:128
BagOStuff\$asyncHandler
callable null $asyncHandler
Definition: BagOStuff.php:68
BagOStuff\incrWithInit
incrWithInit( $key, $ttl, $value=1, $init=1)
Increase stored value of $key by $value while preserving its TTL.
Definition: BagOStuff.php:612
BagOStuff\READ_LATEST
const READ_LATEST
Bitfield constants for get()/getMulti()
Definition: BagOStuff.php:91
use
as see the revision history and available at free of to any person obtaining a copy of this software and associated documentation to deal in the Software without including without limitation the rights to use
Definition: MIT-LICENSE.txt:10
$code
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that When $user is not it can be in the form of< username >< more info > e g for bot passwords intended to be added to log contexts Fields it might only if the login was with a bot password it is not rendered in wiki pages or galleries in category pages allow injecting custom HTML after the section Any uses of the hook need to handle escaping see BaseTemplate::getToolbox and BaseTemplate::makeListItem for details on the format of individual items inside of this array or by returning and letting standard HTTP rendering take place modifiable or by returning false and taking over the output modifiable & $code
Definition: hooks.txt:780
BagOStuff\addBusyCallback
addBusyCallback(callable $workCallback)
Let a callback be run to avoid wasting time on special blocking calls.
Definition: BagOStuff.php:673
BagOStuff\$busyCallbacks
callable[] $busyCallbacks
Definition: BagOStuff.php:82
BagOStuff\$wallClockOverride
float null $wallClockOverride
Definition: BagOStuff.php:85
array
The wiki should then use memcached to cache various data To use multiple just add more items to the array To increase the weight of a make its entry a array("192.168.0.1:11211", 2))
IExpiringStore\QOS_UNKNOWN
const QOS_UNKNOWN
Definition: IExpiringStore.php:61
$fname
if(defined( 'MW_SETUP_CALLBACK')) $fname
Customization point after all loading (constants, functions, classes, DefaultSettings,...
Definition: Setup.php:123
BagOStuff\mergeFlagMaps
mergeFlagMaps(array $bags)
Merge the flag maps of one or more BagOStuff objects into a "lowest common denominator" map.
Definition: BagOStuff.php:793
BagOStuff\__construct
__construct(array $params=[])
$params include:
Definition: BagOStuff.php:108
$value
$value
Definition: styleTest.css.php:49
BagOStuff\makeGlobalKey
makeGlobalKey( $class, $component=null)
Make a global cache key.
Definition: BagOStuff.php:762
BagOStuff\incr
incr( $key, $value=1)
Increase stored value of $key by $value while preserving its TTL.
BagOStuff\$syncTimeout
int $syncTimeout
Seconds.
Definition: BagOStuff.php:70
BagOStuff\convertToExpiry
convertToExpiry( $exptime)
Convert an optionally relative time to an absolute time.
Definition: BagOStuff.php:701
BagOStuff\merge
merge( $key, callable $callback, $exptime=0, $attempts=10, $flags=0)
Merge changes into the existing cache value (possibly creating a new one)
Definition: BagOStuff.php:275
BagOStuff\setMulti
setMulti(array $data, $exptime=0, $flags=0)
Batch insertion/replace.
Definition: BagOStuff.php:555
BagOStuff\deleteMulti
deleteMulti(array $keys, $flags=0)
Batch deletion.
Definition: BagOStuff.php:573
$args
if( $line===false) $args
Definition: cdb.php:64
BagOStuff\$debugMode
bool $debugMode
Definition: BagOStuff.php:73
$cache
$cache
Definition: mcc.php:33
BagOStuff\$attrMap
int[] $attrMap
Map of (ATTR_* class constant => QOS_* class constant)
Definition: BagOStuff.php:88
BagOStuff\READ_VERIFIED
const READ_VERIFIED
Definition: BagOStuff.php:92
as
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
BagOStuff\lock
lock( $key, $timeout=6, $expiry=6, $rclass='')
Acquire an advisory lock on a key string.
Definition: BagOStuff.php:406
$keys
$keys
Definition: testCompression.php:67
BagOStuff\getCurrentTime
getCurrentTime()
Definition: BagOStuff.php:812
BagOStuff\$keyspace
string $keyspace
Definition: BagOStuff.php:64
BagOStuff\cas
cas( $casToken, $key, $value, $exptime=0, $flags=0)
Check and set an item.
Definition: BagOStuff.php:343
class
you have access to all of the normal MediaWiki so you can get a DB use the etc For full docs on the Maintenance class
Definition: maintenance.txt:52
$time
see documentation in includes Linker php for Linker::makeImageLink & $time
Definition: hooks.txt:1802
BagOStuff\makeKeyInternal
makeKeyInternal( $keyspace, $args)
Construct a cache key.
Definition: BagOStuff.php:746
BagOStuff\getScopedLock
getScopedLock( $key, $timeout=6, $expiry=30, $rclass='')
Get a lightweight exclusive self-unlocking lock.
Definition: BagOStuff.php:492
BagOStuff\setLastError
setLastError( $err)
Set the "last error" registry.
Definition: BagOStuff.php:649
BagOStuff\changeTTL
changeTTL( $key, $expiry=0, $flags=0)
Change the expiration on a key if it exists.
Definition: BagOStuff.php:377
BagOStuff\getMulti
getMulti(array $keys, $flags=0)
Get an associative array containing the item for each of the keys that have items.
Definition: BagOStuff.php:535
BagOStuff\WRITE_CACHE_ONLY
const WRITE_CACHE_ONLY
Definition: BagOStuff.php:95
BagOStuff\convertToRelative
convertToRelative( $exptime)
Convert an optionally absolute expiry time to a relative time.
Definition: BagOStuff.php:716
BagOStuff\$dupeTrackScheduled
bool $dupeTrackScheduled
Definition: BagOStuff.php:79