MediaWiki  REL1_31
BagOStuff.php
Go to the documentation of this file.
1 <?php
29 use Psr\Log\LoggerAwareInterface;
30 use Psr\Log\LoggerInterface;
31 use Psr\Log\NullLogger;
32 use Wikimedia\ScopedCallback;
33 use Wikimedia\WaitConditionLoop;
34 
47 abstract class BagOStuff implements IExpiringStore, LoggerAwareInterface {
49  protected $locks = [];
53  protected $keyspace = 'local';
55  protected $logger;
57  protected $asyncHandler;
59  protected $syncTimeout;
60 
62  private $debugMode = false;
64  private $duplicateKeyLookups = [];
66  private $reportDupes = false;
68  private $dupeTrackScheduled = false;
69 
71  protected $busyCallbacks = [];
72 
75 
77  protected $attrMap = [];
78 
80  const ERR_NONE = 0; // no error
81  const ERR_NO_RESPONSE = 1; // no response
82  const ERR_UNREACHABLE = 2; // can't connect
83  const ERR_UNEXPECTED = 3; // response gave some error
84 
86  const READ_LATEST = 1; // use latest data for replicated stores
87  const READ_VERIFIED = 2; // promise that caller can tell when keys are stale
89  const WRITE_SYNC = 1; // synchronously write to all locations for replicated stores
90  const WRITE_CACHE_ONLY = 2; // Only change state of the in-memory cache
91 
103  public function __construct( array $params = [] ) {
104  if ( isset( $params['logger'] ) ) {
105  $this->setLogger( $params['logger'] );
106  } else {
107  $this->setLogger( new NullLogger() );
108  }
109 
110  if ( isset( $params['keyspace'] ) ) {
111  $this->keyspace = $params['keyspace'];
112  }
113 
114  $this->asyncHandler = isset( $params['asyncHandler'] )
115  ? $params['asyncHandler']
116  : null;
117 
118  if ( !empty( $params['reportDupes'] ) && is_callable( $this->asyncHandler ) ) {
119  $this->reportDupes = true;
120  }
121 
122  $this->syncTimeout = isset( $params['syncTimeout'] ) ? $params['syncTimeout'] : 3;
123  }
124 
129  public function setLogger( LoggerInterface $logger ) {
130  $this->logger = $logger;
131  }
132 
136  public function setDebug( $bool ) {
137  $this->debugMode = $bool;
138  }
139 
152  final public function getWithSetCallback( $key, $ttl, $callback, $flags = 0 ) {
153  $value = $this->get( $key, $flags );
154 
155  if ( $value === false ) {
156  if ( !is_callable( $callback ) ) {
157  throw new InvalidArgumentException( "Invalid cache miss callback provided." );
158  }
159  $value = call_user_func( $callback );
160  if ( $value !== false ) {
161  $this->set( $key, $value, $ttl );
162  }
163  }
164 
165  return $value;
166  }
167 
182  public function get( $key, $flags = 0, $oldFlags = null ) {
183  // B/C for ( $key, &$casToken = null, $flags = 0 )
184  $flags = is_int( $oldFlags ) ? $oldFlags : $flags;
185 
186  $this->trackDuplicateKeys( $key );
187 
188  return $this->doGet( $key, $flags );
189  }
190 
195  private function trackDuplicateKeys( $key ) {
196  if ( !$this->reportDupes ) {
197  return;
198  }
199 
200  if ( !isset( $this->duplicateKeyLookups[$key] ) ) {
201  // Track that we have seen this key. This N-1 counting style allows
202  // easy filtering with array_filter() later.
203  $this->duplicateKeyLookups[$key] = 0;
204  } else {
205  $this->duplicateKeyLookups[$key] += 1;
206 
207  if ( $this->dupeTrackScheduled === false ) {
208  $this->dupeTrackScheduled = true;
209  // Schedule a callback that logs keys processed more than once by get().
210  call_user_func( $this->asyncHandler, function () {
211  $dups = array_filter( $this->duplicateKeyLookups );
212  foreach ( $dups as $key => $count ) {
213  $this->logger->warning(
214  'Duplicate get(): "{key}" fetched {count} times',
215  // Count is N-1 of the actual lookup count
216  [ 'key' => $key, 'count' => $count + 1, ]
217  );
218  }
219  } );
220  }
221  }
222  }
223 
229  abstract protected function doGet( $key, $flags = 0 );
230 
240  protected function getWithToken( $key, &$casToken, $flags = 0 ) {
241  throw new Exception( __METHOD__ . ' not implemented.' );
242  }
243 
253  abstract public function set( $key, $value, $exptime = 0, $flags = 0 );
254 
261  abstract public function delete( $key );
262 
279  public function merge( $key, callable $callback, $exptime = 0, $attempts = 10, $flags = 0 ) {
280  return $this->mergeViaLock( $key, $callback, $exptime, $attempts, $flags );
281  }
282 
292  protected function mergeViaCas( $key, $callback, $exptime = 0, $attempts = 10 ) {
293  do {
294  $this->clearLastError();
296  $this->reportDupes = false;
297  $casToken = null; // passed by reference
298  $currentValue = $this->getWithToken( $key, $casToken, self::READ_LATEST );
299  $this->reportDupes = $reportDupes;
300 
301  if ( $this->getLastError() ) {
302  return false; // don't spam retries (retry only on races)
303  }
304 
305  // Derive the new value from the old value
306  $value = call_user_func( $callback, $this, $key, $currentValue, $exptime );
307 
308  $this->clearLastError();
309  if ( $value === false ) {
310  $success = true; // do nothing
311  } elseif ( $currentValue === false ) {
312  // Try to create the key, failing if it gets created in the meantime
313  $success = $this->add( $key, $value, $exptime );
314  } else {
315  // Try to update the key, failing if it gets changed in the meantime
316  $success = $this->cas( $casToken, $key, $value, $exptime );
317  }
318  if ( $this->getLastError() ) {
319  return false; // IO error; don't spam retries
320  }
321  } while ( !$success && --$attempts );
322 
323  return $success;
324  }
325 
336  protected function cas( $casToken, $key, $value, $exptime = 0 ) {
337  throw new Exception( "CAS is not implemented in " . __CLASS__ );
338  }
339 
350  protected function mergeViaLock( $key, $callback, $exptime = 0, $attempts = 10, $flags = 0 ) {
351  if ( !$this->lock( $key, 6 ) ) {
352  return false;
353  }
354 
355  $this->clearLastError();
357  $this->reportDupes = false;
358  $currentValue = $this->get( $key, self::READ_LATEST );
359  $this->reportDupes = $reportDupes;
360 
361  if ( $this->getLastError() ) {
362  $success = false;
363  } else {
364  // Derive the new value from the old value
365  $value = $callback( $this, $key, $currentValue, $exptime );
366  if ( $value === false ) {
367  $success = true; // do nothing
368  } else {
369  $success = $this->set( $key, $value, $exptime, $flags ); // set the new value
370  }
371  }
372 
373  if ( !$this->unlock( $key ) ) {
374  // this should never happen
375  trigger_error( "Could not release lock for key '$key'." );
376  }
377 
378  return $success;
379  }
380 
389  public function changeTTL( $key, $expiry = 0 ) {
390  $value = $this->get( $key );
391 
392  return ( $value === false ) ? false : $this->set( $key, $value, $expiry );
393  }
394 
406  public function lock( $key, $timeout = 6, $expiry = 6, $rclass = '' ) {
407  // Avoid deadlocks and allow lock reentry if specified
408  if ( isset( $this->locks[$key] ) ) {
409  if ( $rclass != '' && $this->locks[$key]['class'] === $rclass ) {
410  ++$this->locks[$key]['depth'];
411  return true;
412  } else {
413  return false;
414  }
415  }
416 
417  $expiry = min( $expiry ?: INF, self::TTL_DAY );
418  $loop = new WaitConditionLoop(
419  function () use ( $key, $timeout, $expiry ) {
420  $this->clearLastError();
421  if ( $this->add( "{$key}:lock", 1, $expiry ) ) {
422  return true; // locked!
423  } elseif ( $this->getLastError() ) {
424  return WaitConditionLoop::CONDITION_ABORTED; // network partition?
425  }
426 
427  return WaitConditionLoop::CONDITION_CONTINUE;
428  },
429  $timeout
430  );
431 
432  $locked = ( $loop->invoke() === $loop::CONDITION_REACHED );
433  if ( $locked ) {
434  $this->locks[$key] = [ 'class' => $rclass, 'depth' => 1 ];
435  }
436 
437  return $locked;
438  }
439 
446  public function unlock( $key ) {
447  if ( isset( $this->locks[$key] ) && --$this->locks[$key]['depth'] <= 0 ) {
448  unset( $this->locks[$key] );
449 
450  return $this->delete( "{$key}:lock" );
451  }
452 
453  return true;
454  }
455 
472  final public function getScopedLock( $key, $timeout = 6, $expiry = 30, $rclass = '' ) {
473  $expiry = min( $expiry ?: INF, self::TTL_DAY );
474 
475  if ( !$this->lock( $key, $timeout, $expiry, $rclass ) ) {
476  return null;
477  }
478 
479  $lSince = $this->getCurrentTime(); // lock timestamp
480 
481  return new ScopedCallback( function () use ( $key, $lSince, $expiry ) {
482  $latency = 0.050; // latency skew (err towards keeping lock present)
483  $age = ( $this->getCurrentTime() - $lSince + $latency );
484  if ( ( $age + $latency ) >= $expiry ) {
485  $this->logger->warning( "Lock for $key held too long ($age sec)." );
486  return; // expired; it's not "safe" to delete the key
487  }
488  $this->unlock( $key );
489  } );
490  }
491 
501  public function deleteObjectsExpiringBefore( $date, $progressCallback = false ) {
502  // stub
503  return false;
504  }
505 
512  public function getMulti( array $keys, $flags = 0 ) {
513  $res = [];
514  foreach ( $keys as $key ) {
515  $val = $this->get( $key );
516  if ( $val !== false ) {
517  $res[$key] = $val;
518  }
519  }
520  return $res;
521  }
522 
530  public function setMulti( array $data, $exptime = 0 ) {
531  $res = true;
532  foreach ( $data as $key => $value ) {
533  if ( !$this->set( $key, $value, $exptime ) ) {
534  $res = false;
535  }
536  }
537  return $res;
538  }
539 
546  public function add( $key, $value, $exptime = 0 ) {
547  if ( $this->get( $key ) === false ) {
548  return $this->set( $key, $value, $exptime );
549  }
550  return false; // key already set
551  }
552 
559  public function incr( $key, $value = 1 ) {
560  if ( !$this->lock( $key ) ) {
561  return false;
562  }
563  $n = $this->get( $key );
564  if ( $this->isInteger( $n ) ) { // key exists?
565  $n += intval( $value );
566  $this->set( $key, max( 0, $n ) ); // exptime?
567  } else {
568  $n = false;
569  }
570  $this->unlock( $key );
571 
572  return $n;
573  }
574 
581  public function decr( $key, $value = 1 ) {
582  return $this->incr( $key, - $value );
583  }
584 
597  public function incrWithInit( $key, $ttl, $value = 1, $init = 1 ) {
598  $newValue = $this->incr( $key, $value );
599  if ( $newValue === false ) {
600  // No key set; initialize
601  $newValue = $this->add( $key, (int)$init, $ttl ) ? $init : false;
602  }
603  if ( $newValue === false ) {
604  // Raced out initializing; increment
605  $newValue = $this->incr( $key, $value );
606  }
607 
608  return $newValue;
609  }
610 
616  public function getLastError() {
617  return $this->lastError;
618  }
619 
624  public function clearLastError() {
625  $this->lastError = self::ERR_NONE;
626  }
627 
633  protected function setLastError( $err ) {
634  $this->lastError = $err;
635  }
636 
657  public function addBusyCallback( callable $workCallback ) {
658  $this->busyCallbacks[] = $workCallback;
659  }
660 
675  public function modifySimpleRelayEvent( array $event ) {
676  return $event;
677  }
678 
682  protected function debug( $text ) {
683  if ( $this->debugMode ) {
684  $this->logger->debug( "{class} debug: $text", [
685  'class' => static::class,
686  ] );
687  }
688  }
689 
695  protected function convertExpiry( $exptime ) {
696  if ( $exptime != 0 && $exptime < ( 10 * self::TTL_YEAR ) ) {
697  return (int)$this->getCurrentTime() + $exptime;
698  } else {
699  return $exptime;
700  }
701  }
702 
710  protected function convertToRelative( $exptime ) {
711  if ( $exptime >= ( 10 * self::TTL_YEAR ) ) {
712  $exptime -= (int)$this->getCurrentTime();
713  if ( $exptime <= 0 ) {
714  $exptime = 1;
715  }
716  return $exptime;
717  } else {
718  return $exptime;
719  }
720  }
721 
728  protected function isInteger( $value ) {
729  return ( is_int( $value ) || ctype_digit( $value ) );
730  }
731 
740  public function makeKeyInternal( $keyspace, $args ) {
741  $key = $keyspace;
742  foreach ( $args as $arg ) {
743  $arg = str_replace( ':', '%3A', $arg );
744  $key = $key . ':' . $arg;
745  }
746  return strtr( $key, ' ', '_' );
747  }
748 
757  public function makeGlobalKey( $class, $component = null ) {
758  return $this->makeKeyInternal( 'global', func_get_args() );
759  }
760 
769  public function makeKey( $class, $component = null ) {
770  return $this->makeKeyInternal( $this->keyspace, func_get_args() );
771  }
772 
778  public function getQoS( $flag ) {
779  return isset( $this->attrMap[$flag] ) ? $this->attrMap[$flag] : self::QOS_UNKNOWN;
780  }
781 
788  protected function mergeFlagMaps( array $bags ) {
789  $map = [];
790  foreach ( $bags as $bag ) {
791  foreach ( $bag->attrMap as $attr => $rank ) {
792  if ( isset( $map[$attr] ) ) {
793  $map[$attr] = min( $map[$attr], $rank );
794  } else {
795  $map[$attr] = $rank;
796  }
797  }
798  }
799 
800  return $map;
801  }
802 
807  protected function getCurrentTime() {
808  return $this->wallClockOverride ?: microtime( true );
809  }
810 
815  public function setMockTime( &$time ) {
816  $this->wallClockOverride =& $time;
817  }
818 }
$time
see documentation in includes Linker php for Linker::makeImageLink & $time
Definition: hooks.txt:1795
BagOStuff\isInteger
isInteger( $value)
Check if a value is an integer.
Definition: BagOStuff.php:728
BagOStuff\add
add( $key, $value, $exptime=0)
Definition: BagOStuff.php:546
BagOStuff\setMulti
setMulti(array $data, $exptime=0)
Batch insertion.
Definition: BagOStuff.php:530
BagOStuff\$asyncHandler
callback null $asyncHandler
Definition: BagOStuff.php:57
BagOStuff\decr
decr( $key, $value=1)
Decrease stored value of $key by $value while preserving its TTL.
Definition: BagOStuff.php:581
use
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
Definition: APACHE-LICENSE-2.0.txt:10
BagOStuff\ERR_UNREACHABLE
const ERR_UNREACHABLE
Definition: BagOStuff.php:82
BagOStuff\getQoS
getQoS( $flag)
Definition: BagOStuff.php:778
array
the array() calling protocol came about after MediaWiki 1.4rc1.
BagOStuff\modifySimpleRelayEvent
modifySimpleRelayEvent(array $event)
Modify a cache update operation array for EventRelayer::notify()
Definition: BagOStuff.php:675
BagOStuff\getLastError
getLastError()
Get the "last error" registered; clearLastError() should be called manually.
Definition: BagOStuff.php:616
BagOStuff\WRITE_SYNC
const WRITE_SYNC
Bitfield constants for set()/merge()
Definition: BagOStuff.php:89
BagOStuff\$reportDupes
bool $reportDupes
Definition: BagOStuff.php:66
BagOStuff\deleteObjectsExpiringBefore
deleteObjectsExpiringBefore( $date, $progressCallback=false)
Delete all objects expiring before a certain date.
Definition: BagOStuff.php:501
BagOStuff\getWithToken
getWithToken( $key, &$casToken, $flags=0)
Definition: BagOStuff.php:240
BagOStuff\convertExpiry
convertExpiry( $exptime)
Convert an optionally relative time to an absolute time.
Definition: BagOStuff.php:695
BagOStuff\debug
debug( $text)
Definition: BagOStuff.php:682
BagOStuff\$locks
array[] $locks
Lock tracking.
Definition: BagOStuff.php:49
$params
$params
Definition: styleTest.css.php:40
BagOStuff\trackDuplicateKeys
trackDuplicateKeys( $key)
Track the number of times that a given key has been used.
Definition: BagOStuff.php:195
BagOStuff
interface is intended to be more or less compatible with the PHP memcached client.
Definition: BagOStuff.php:47
BagOStuff\$duplicateKeyLookups
array $duplicateKeyLookups
Definition: BagOStuff.php:64
BagOStuff\makeKey
makeKey( $class, $component=null)
Make a cache key, scoped to this instance's keyspace.
Definition: BagOStuff.php:769
$res
$res
Definition: database.txt:21
BagOStuff\setDebug
setDebug( $bool)
Definition: BagOStuff.php:136
$success
$success
Definition: NoLocalSettings.php:42
BagOStuff\$logger
LoggerInterface $logger
Definition: BagOStuff.php:55
BagOStuff\ERR_NONE
const ERR_NONE
Possible values for getLastError()
Definition: BagOStuff.php:80
php
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:37
BagOStuff\setMockTime
setMockTime(&$time)
Definition: BagOStuff.php:815
BagOStuff\getWithSetCallback
getWithSetCallback( $key, $ttl, $callback, $flags=0)
Get an item with the given key, regenerating and setting it if not found.
Definition: BagOStuff.php:152
BagOStuff\unlock
unlock( $key)
Release an advisory lock on a key string.
Definition: BagOStuff.php:446
BagOStuff\clearLastError
clearLastError()
Clear the "last error" registry.
Definition: BagOStuff.php:624
IExpiringStore
Generic base class for storage interfaces.
Definition: IExpiringStore.php:31
BagOStuff\$lastError
int $lastError
ERR_* class constant.
Definition: BagOStuff.php:51
BagOStuff\setLogger
setLogger(LoggerInterface $logger)
Definition: BagOStuff.php:129
BagOStuff\incrWithInit
incrWithInit( $key, $ttl, $value=1, $init=1)
Increase stored value of $key by $value while preserving its TTL.
Definition: BagOStuff.php:597
BagOStuff\READ_LATEST
const READ_LATEST
Bitfield constants for get()/getMulti()
Definition: BagOStuff.php:86
BagOStuff\addBusyCallback
addBusyCallback(callable $workCallback)
Let a callback be run to avoid wasting time on special blocking calls.
Definition: BagOStuff.php:657
BagOStuff\$busyCallbacks
callable[] $busyCallbacks
Definition: BagOStuff.php:71
BagOStuff\$wallClockOverride
float null $wallClockOverride
Definition: BagOStuff.php:74
IExpiringStore\QOS_UNKNOWN
const QOS_UNKNOWN
Definition: IExpiringStore.php:58
BagOStuff\mergeFlagMaps
mergeFlagMaps(array $bags)
Merge the flag maps of one or more BagOStuff objects into a "lowest common denominator" map.
Definition: BagOStuff.php:788
BagOStuff\__construct
__construct(array $params=[])
$params include:
Definition: BagOStuff.php:103
$value
$value
Definition: styleTest.css.php:45
BagOStuff\doGet
doGet( $key, $flags=0)
BagOStuff\makeGlobalKey
makeGlobalKey( $class, $component=null)
Make a global cache key.
Definition: BagOStuff.php:757
BagOStuff\incr
incr( $key, $value=1)
Increase stored value of $key by $value while preserving its TTL.
Definition: BagOStuff.php:559
BagOStuff\$syncTimeout
int $syncTimeout
Seconds.
Definition: BagOStuff.php:59
BagOStuff\changeTTL
changeTTL( $key, $expiry=0)
Reset the TTL on a key if it exists.
Definition: BagOStuff.php:389
BagOStuff\merge
merge( $key, callable $callback, $exptime=0, $attempts=10, $flags=0)
Merge changes into the existing cache value (possibly creating a new one)
Definition: BagOStuff.php:279
BagOStuff\mergeViaCas
mergeViaCas( $key, $callback, $exptime=0, $attempts=10)
Definition: BagOStuff.php:292
$args
if( $line===false) $args
Definition: cdb.php:64
BagOStuff\$debugMode
bool $debugMode
Definition: BagOStuff.php:62
BagOStuff\mergeViaLock
mergeViaLock( $key, $callback, $exptime=0, $attempts=10, $flags=0)
Definition: BagOStuff.php:350
BagOStuff\$attrMap
int[] $attrMap
Map of (ATTR_* class constant => QOS_* class constant)
Definition: BagOStuff.php:77
BagOStuff\READ_VERIFIED
const READ_VERIFIED
Definition: BagOStuff.php:87
as
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:22
BagOStuff\lock
lock( $key, $timeout=6, $expiry=6, $rclass='')
Acquire an advisory lock on a key string.
Definition: BagOStuff.php:406
$keys
$keys
Definition: testCompression.php:67
BagOStuff\getCurrentTime
getCurrentTime()
Definition: BagOStuff.php:807
BagOStuff\cas
cas( $casToken, $key, $value, $exptime=0)
Check and set an item.
Definition: BagOStuff.php:336
BagOStuff\$keyspace
string $keyspace
Definition: BagOStuff.php:53
class
you have access to all of the normal MediaWiki so you can get a DB use the etc For full docs on the Maintenance class
Definition: maintenance.txt:56
BagOStuff\makeKeyInternal
makeKeyInternal( $keyspace, $args)
Construct a cache key.
Definition: BagOStuff.php:740
BagOStuff\getScopedLock
getScopedLock( $key, $timeout=6, $expiry=30, $rclass='')
Get a lightweight exclusive self-unlocking lock.
Definition: BagOStuff.php:472
BagOStuff\ERR_NO_RESPONSE
const ERR_NO_RESPONSE
Definition: BagOStuff.php:81
BagOStuff\setLastError
setLastError( $err)
Set the "last error" registry.
Definition: BagOStuff.php:633
BagOStuff\ERR_UNEXPECTED
const ERR_UNEXPECTED
Definition: BagOStuff.php:83
false
processing should stop and the error should be shown to the user * false
Definition: hooks.txt:187
BagOStuff\getMulti
getMulti(array $keys, $flags=0)
Get an associative array containing the item for each of the keys that have items.
Definition: BagOStuff.php:512
BagOStuff\WRITE_CACHE_ONLY
const WRITE_CACHE_ONLY
Definition: BagOStuff.php:90
BagOStuff\convertToRelative
convertToRelative( $exptime)
Convert an optionally absolute expiry time to a relative time.
Definition: BagOStuff.php:710
BagOStuff\$dupeTrackScheduled
bool $dupeTrackScheduled
Definition: BagOStuff.php:68