MediaWiki  1.29.2
BagOStuff.php
Go to the documentation of this file.
1 <?php
29 use Psr\Log\LoggerAwareInterface;
30 use Psr\Log\LoggerInterface;
31 use Psr\Log\NullLogger;
32 use Wikimedia\ScopedCallback;
33 use Wikimedia\WaitConditionLoop;
34 
47 abstract class BagOStuff implements IExpiringStore, LoggerAwareInterface {
49  protected $locks = [];
53  protected $keyspace = 'local';
55  protected $logger;
57  protected $asyncHandler;
59  protected $syncTimeout;
60 
62  private $debugMode = false;
64  private $duplicateKeyLookups = [];
66  private $reportDupes = false;
68  private $dupeTrackScheduled = false;
69 
71  protected $busyCallbacks = [];
72 
74  protected $attrMap = [];
75 
77  const ERR_NONE = 0; // no error
78  const ERR_NO_RESPONSE = 1; // no response
79  const ERR_UNREACHABLE = 2; // can't connect
80  const ERR_UNEXPECTED = 3; // response gave some error
81 
83  const READ_LATEST = 1; // use latest data for replicated stores
84  const READ_VERIFIED = 2; // promise that caller can tell when keys are stale
86  const WRITE_SYNC = 1; // synchronously write to all locations for replicated stores
87  const WRITE_CACHE_ONLY = 2; // Only change state of the in-memory cache
88 
100  public function __construct( array $params = [] ) {
101  if ( isset( $params['logger'] ) ) {
102  $this->setLogger( $params['logger'] );
103  } else {
104  $this->setLogger( new NullLogger() );
105  }
106 
107  if ( isset( $params['keyspace'] ) ) {
108  $this->keyspace = $params['keyspace'];
109  }
110 
111  $this->asyncHandler = isset( $params['asyncHandler'] )
112  ? $params['asyncHandler']
113  : null;
114 
115  if ( !empty( $params['reportDupes'] ) && is_callable( $this->asyncHandler ) ) {
116  $this->reportDupes = true;
117  }
118 
119  $this->syncTimeout = isset( $params['syncTimeout'] ) ? $params['syncTimeout'] : 3;
120  }
121 
126  public function setLogger( LoggerInterface $logger ) {
127  $this->logger = $logger;
128  }
129 
133  public function setDebug( $bool ) {
134  $this->debugMode = $bool;
135  }
136 
149  final public function getWithSetCallback( $key, $ttl, $callback, $flags = 0 ) {
150  $value = $this->get( $key, $flags );
151 
152  if ( $value === false ) {
153  if ( !is_callable( $callback ) ) {
154  throw new InvalidArgumentException( "Invalid cache miss callback provided." );
155  }
156  $value = call_user_func( $callback );
157  if ( $value !== false ) {
158  $this->set( $key, $value, $ttl );
159  }
160  }
161 
162  return $value;
163  }
164 
179  public function get( $key, $flags = 0, $oldFlags = null ) {
180  // B/C for ( $key, &$casToken = null, $flags = 0 )
181  $flags = is_int( $oldFlags ) ? $oldFlags : $flags;
182 
183  $this->trackDuplicateKeys( $key );
184 
185  return $this->doGet( $key, $flags );
186  }
187 
192  private function trackDuplicateKeys( $key ) {
193  if ( !$this->reportDupes ) {
194  return;
195  }
196 
197  if ( !isset( $this->duplicateKeyLookups[$key] ) ) {
198  // Track that we have seen this key. This N-1 counting style allows
199  // easy filtering with array_filter() later.
200  $this->duplicateKeyLookups[$key] = 0;
201  } else {
202  $this->duplicateKeyLookups[$key] += 1;
203 
204  if ( $this->dupeTrackScheduled === false ) {
205  $this->dupeTrackScheduled = true;
206  // Schedule a callback that logs keys processed more than once by get().
207  call_user_func( $this->asyncHandler, function () {
208  $dups = array_filter( $this->duplicateKeyLookups );
209  foreach ( $dups as $key => $count ) {
210  $this->logger->warning(
211  'Duplicate get(): "{key}" fetched {count} times',
212  // Count is N-1 of the actual lookup count
213  [ 'key' => $key, 'count' => $count + 1, ]
214  );
215  }
216  } );
217  }
218  }
219  }
220 
226  abstract protected function doGet( $key, $flags = 0 );
227 
237  protected function getWithToken( $key, &$casToken, $flags = 0 ) {
238  throw new Exception( __METHOD__ . ' not implemented.' );
239  }
240 
250  abstract public function set( $key, $value, $exptime = 0, $flags = 0 );
251 
258  abstract public function delete( $key );
259 
276  public function merge( $key, callable $callback, $exptime = 0, $attempts = 10, $flags = 0 ) {
277  return $this->mergeViaLock( $key, $callback, $exptime, $attempts, $flags );
278  }
279 
289  protected function mergeViaCas( $key, $callback, $exptime = 0, $attempts = 10 ) {
290  do {
291  $this->clearLastError();
293  $this->reportDupes = false;
294  $casToken = null; // passed by reference
295  $currentValue = $this->getWithToken( $key, $casToken, self::READ_LATEST );
296  $this->reportDupes = $reportDupes;
297 
298  if ( $this->getLastError() ) {
299  return false; // don't spam retries (retry only on races)
300  }
301 
302  // Derive the new value from the old value
303  $value = call_user_func( $callback, $this, $key, $currentValue, $exptime );
304 
305  $this->clearLastError();
306  if ( $value === false ) {
307  $success = true; // do nothing
308  } elseif ( $currentValue === false ) {
309  // Try to create the key, failing if it gets created in the meantime
310  $success = $this->add( $key, $value, $exptime );
311  } else {
312  // Try to update the key, failing if it gets changed in the meantime
313  $success = $this->cas( $casToken, $key, $value, $exptime );
314  }
315  if ( $this->getLastError() ) {
316  return false; // IO error; don't spam retries
317  }
318  } while ( !$success && --$attempts );
319 
320  return $success;
321  }
322 
333  protected function cas( $casToken, $key, $value, $exptime = 0 ) {
334  throw new Exception( "CAS is not implemented in " . __CLASS__ );
335  }
336 
347  protected function mergeViaLock( $key, $callback, $exptime = 0, $attempts = 10, $flags = 0 ) {
348  if ( !$this->lock( $key, 6 ) ) {
349  return false;
350  }
351 
352  $this->clearLastError();
354  $this->reportDupes = false;
355  $currentValue = $this->get( $key, self::READ_LATEST );
356  $this->reportDupes = $reportDupes;
357 
358  if ( $this->getLastError() ) {
359  $success = false;
360  } else {
361  // Derive the new value from the old value
362  $value = call_user_func( $callback, $this, $key, $currentValue, $exptime );
363  if ( $value === false ) {
364  $success = true; // do nothing
365  } else {
366  $success = $this->set( $key, $value, $exptime, $flags ); // set the new value
367  }
368  }
369 
370  if ( !$this->unlock( $key ) ) {
371  // this should never happen
372  trigger_error( "Could not release lock for key '$key'." );
373  }
374 
375  return $success;
376  }
377 
386  public function changeTTL( $key, $expiry = 0 ) {
387  $value = $this->get( $key );
388 
389  return ( $value === false ) ? false : $this->set( $key, $value, $expiry );
390  }
391 
403  public function lock( $key, $timeout = 6, $expiry = 6, $rclass = '' ) {
404  // Avoid deadlocks and allow lock reentry if specified
405  if ( isset( $this->locks[$key] ) ) {
406  if ( $rclass != '' && $this->locks[$key]['class'] === $rclass ) {
407  ++$this->locks[$key]['depth'];
408  return true;
409  } else {
410  return false;
411  }
412  }
413 
414  $expiry = min( $expiry ?: INF, self::TTL_DAY );
415  $loop = new WaitConditionLoop(
416  function () use ( $key, $timeout, $expiry ) {
417  $this->clearLastError();
418  if ( $this->add( "{$key}:lock", 1, $expiry ) ) {
419  return true; // locked!
420  } elseif ( $this->getLastError() ) {
421  return WaitConditionLoop::CONDITION_ABORTED; // network partition?
422  }
423 
424  return WaitConditionLoop::CONDITION_CONTINUE;
425  },
426  $timeout
427  );
428 
429  $locked = ( $loop->invoke() === $loop::CONDITION_REACHED );
430  if ( $locked ) {
431  $this->locks[$key] = [ 'class' => $rclass, 'depth' => 1 ];
432  }
433 
434  return $locked;
435  }
436 
443  public function unlock( $key ) {
444  if ( isset( $this->locks[$key] ) && --$this->locks[$key]['depth'] <= 0 ) {
445  unset( $this->locks[$key] );
446 
447  return $this->delete( "{$key}:lock" );
448  }
449 
450  return true;
451  }
452 
469  final public function getScopedLock( $key, $timeout = 6, $expiry = 30, $rclass = '' ) {
470  $expiry = min( $expiry ?: INF, self::TTL_DAY );
471 
472  if ( !$this->lock( $key, $timeout, $expiry, $rclass ) ) {
473  return null;
474  }
475 
476  $lSince = microtime( true ); // lock timestamp
477 
478  return new ScopedCallback( function() use ( $key, $lSince, $expiry ) {
479  $latency = .050; // latency skew (err towards keeping lock present)
480  $age = ( microtime( true ) - $lSince + $latency );
481  if ( ( $age + $latency ) >= $expiry ) {
482  $this->logger->warning( "Lock for $key held too long ($age sec)." );
483  return; // expired; it's not "safe" to delete the key
484  }
485  $this->unlock( $key );
486  } );
487  }
488 
498  public function deleteObjectsExpiringBefore( $date, $progressCallback = false ) {
499  // stub
500  return false;
501  }
502 
509  public function getMulti( array $keys, $flags = 0 ) {
510  $res = [];
511  foreach ( $keys as $key ) {
512  $val = $this->get( $key );
513  if ( $val !== false ) {
514  $res[$key] = $val;
515  }
516  }
517  return $res;
518  }
519 
527  public function setMulti( array $data, $exptime = 0 ) {
528  $res = true;
529  foreach ( $data as $key => $value ) {
530  if ( !$this->set( $key, $value, $exptime ) ) {
531  $res = false;
532  }
533  }
534  return $res;
535  }
536 
543  public function add( $key, $value, $exptime = 0 ) {
544  if ( $this->get( $key ) === false ) {
545  return $this->set( $key, $value, $exptime );
546  }
547  return false; // key already set
548  }
549 
556  public function incr( $key, $value = 1 ) {
557  if ( !$this->lock( $key ) ) {
558  return false;
559  }
560  $n = $this->get( $key );
561  if ( $this->isInteger( $n ) ) { // key exists?
562  $n += intval( $value );
563  $this->set( $key, max( 0, $n ) ); // exptime?
564  } else {
565  $n = false;
566  }
567  $this->unlock( $key );
568 
569  return $n;
570  }
571 
578  public function decr( $key, $value = 1 ) {
579  return $this->incr( $key, - $value );
580  }
581 
594  public function incrWithInit( $key, $ttl, $value = 1, $init = 1 ) {
595  $newValue = $this->incr( $key, $value );
596  if ( $newValue === false ) {
597  // No key set; initialize
598  $newValue = $this->add( $key, (int)$init, $ttl ) ? $init : false;
599  }
600  if ( $newValue === false ) {
601  // Raced out initializing; increment
602  $newValue = $this->incr( $key, $value );
603  }
604 
605  return $newValue;
606  }
607 
613  public function getLastError() {
614  return $this->lastError;
615  }
616 
621  public function clearLastError() {
622  $this->lastError = self::ERR_NONE;
623  }
624 
630  protected function setLastError( $err ) {
631  $this->lastError = $err;
632  }
633 
654  public function addBusyCallback( callable $workCallback ) {
655  $this->busyCallbacks[] = $workCallback;
656  }
657 
672  public function modifySimpleRelayEvent( array $event ) {
673  return $event;
674  }
675 
679  protected function debug( $text ) {
680  if ( $this->debugMode ) {
681  $this->logger->debug( "{class} debug: $text", [
682  'class' => static::class,
683  ] );
684  }
685  }
686 
692  protected function convertExpiry( $exptime ) {
693  if ( $exptime != 0 && $exptime < ( 10 * self::TTL_YEAR ) ) {
694  return time() + $exptime;
695  } else {
696  return $exptime;
697  }
698  }
699 
707  protected function convertToRelative( $exptime ) {
708  if ( $exptime >= ( 10 * self::TTL_YEAR ) ) {
709  $exptime -= time();
710  if ( $exptime <= 0 ) {
711  $exptime = 1;
712  }
713  return $exptime;
714  } else {
715  return $exptime;
716  }
717  }
718 
725  protected function isInteger( $value ) {
726  return ( is_int( $value ) || ctype_digit( $value ) );
727  }
728 
737  public function makeKeyInternal( $keyspace, $args ) {
738  $key = $keyspace;
739  foreach ( $args as $arg ) {
740  $arg = str_replace( ':', '%3A', $arg );
741  $key = $key . ':' . $arg;
742  }
743  return strtr( $key, ' ', '_' );
744  }
745 
753  public function makeGlobalKey() {
754  return $this->makeKeyInternal( 'global', func_get_args() );
755  }
756 
764  public function makeKey() {
765  return $this->makeKeyInternal( $this->keyspace, func_get_args() );
766  }
767 
773  public function getQoS( $flag ) {
774  return isset( $this->attrMap[$flag] ) ? $this->attrMap[$flag] : self::QOS_UNKNOWN;
775  }
776 
783  protected function mergeFlagMaps( array $bags ) {
784  $map = [];
785  foreach ( $bags as $bag ) {
786  foreach ( $bag->attrMap as $attr => $rank ) {
787  if ( isset( $map[$attr] ) ) {
788  $map[$attr] = min( $map[$attr], $rank );
789  } else {
790  $map[$attr] = $rank;
791  }
792  }
793  }
794 
795  return $map;
796  }
797 }
BagOStuff\$lastError
integer $lastError
ERR_* class constant.
Definition: BagOStuff.php:51
BagOStuff\$syncTimeout
integer $syncTimeout
Seconds.
Definition: BagOStuff.php:59
BagOStuff\isInteger
isInteger( $value)
Check if a value is an integer.
Definition: BagOStuff.php:725
BagOStuff\add
add( $key, $value, $exptime=0)
Definition: BagOStuff.php:543
BagOStuff\setMulti
setMulti(array $data, $exptime=0)
Batch insertion.
Definition: BagOStuff.php:527
BagOStuff\$asyncHandler
callback null $asyncHandler
Definition: BagOStuff.php:57
BagOStuff\decr
decr( $key, $value=1)
Decrease stored value of $key by $value while preserving its TTL.
Definition: BagOStuff.php:578
false
processing should stop and the error should be shown to the user * false
Definition: hooks.txt:189
BagOStuff\ERR_UNREACHABLE
const ERR_UNREACHABLE
Definition: BagOStuff.php:79
BagOStuff\getQoS
getQoS( $flag)
Definition: BagOStuff.php:773
BagOStuff\modifySimpleRelayEvent
modifySimpleRelayEvent(array $event)
Modify a cache update operation array for EventRelayer::notify()
Definition: BagOStuff.php:672
BagOStuff\getLastError
getLastError()
Get the "last error" registered; clearLastError() should be called manually.
Definition: BagOStuff.php:613
BagOStuff\WRITE_SYNC
const WRITE_SYNC
Bitfield constants for set()/merge()
Definition: BagOStuff.php:86
BagOStuff\$reportDupes
bool $reportDupes
Definition: BagOStuff.php:66
BagOStuff\deleteObjectsExpiringBefore
deleteObjectsExpiringBefore( $date, $progressCallback=false)
Delete all objects expiring before a certain date.
Definition: BagOStuff.php:498
BagOStuff\getWithToken
getWithToken( $key, &$casToken, $flags=0)
Definition: BagOStuff.php:237
BagOStuff\convertExpiry
convertExpiry( $exptime)
Convert an optionally relative time to an absolute time.
Definition: BagOStuff.php:692
use
as see the revision history and available at free of to any person obtaining a copy of this software and associated documentation to deal in the Software without including without limitation the rights to use
Definition: MIT-LICENSE.txt:10
BagOStuff\debug
debug( $text)
Definition: BagOStuff.php:679
BagOStuff\$locks
array[] $locks
Lock tracking.
Definition: BagOStuff.php:49
$params
$params
Definition: styleTest.css.php:40
BagOStuff\trackDuplicateKeys
trackDuplicateKeys( $key)
Track the number of times that a given key has been used.
Definition: BagOStuff.php:192
BagOStuff
interface is intended to be more or less compatible with the PHP memcached client.
Definition: BagOStuff.php:47
BagOStuff\$duplicateKeyLookups
array $duplicateKeyLookups
Definition: BagOStuff.php:64
$res
$res
Definition: database.txt:21
BagOStuff\setDebug
setDebug( $bool)
Definition: BagOStuff.php:133
$success
$success
Definition: NoLocalSettings.php:44
BagOStuff\$logger
LoggerInterface $logger
Definition: BagOStuff.php:55
BagOStuff\ERR_NONE
const ERR_NONE
Possible values for getLastError()
Definition: BagOStuff.php:77
php
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
BagOStuff\getWithSetCallback
getWithSetCallback( $key, $ttl, $callback, $flags=0)
Get an item with the given key, regenerating and setting it if not found.
Definition: BagOStuff.php:149
BagOStuff\unlock
unlock( $key)
Release an advisory lock on a key string.
Definition: BagOStuff.php:443
BagOStuff\clearLastError
clearLastError()
Clear the "last error" registry.
Definition: BagOStuff.php:621
IExpiringStore
Generic base class for storage interfaces.
Definition: IExpiringStore.php:31
BagOStuff\setLogger
setLogger(LoggerInterface $logger)
Definition: BagOStuff.php:126
BagOStuff\incrWithInit
incrWithInit( $key, $ttl, $value=1, $init=1)
Increase stored value of $key by $value while preserving its TTL.
Definition: BagOStuff.php:594
BagOStuff\READ_LATEST
const READ_LATEST
Bitfield constants for get()/getMulti()
Definition: BagOStuff.php:83
BagOStuff\addBusyCallback
addBusyCallback(callable $workCallback)
Let a callback be run to avoid wasting time on special blocking calls.
Definition: BagOStuff.php:654
BagOStuff\$busyCallbacks
callable[] $busyCallbacks
Definition: BagOStuff.php:71
IExpiringStore\QOS_UNKNOWN
const QOS_UNKNOWN
Definition: IExpiringStore.php:57
BagOStuff\makeKey
makeKey()
Make a cache key, scoped to this instance's keyspace.
Definition: BagOStuff.php:764
BagOStuff\mergeFlagMaps
mergeFlagMaps(array $bags)
Merge the flag maps of one or more BagOStuff objects into a "lowest common denominator" map.
Definition: BagOStuff.php:783
BagOStuff\__construct
__construct(array $params=[])
$params include:
Definition: BagOStuff.php:100
$value
$value
Definition: styleTest.css.php:45
BagOStuff\doGet
doGet( $key, $flags=0)
BagOStuff\incr
incr( $key, $value=1)
Increase stored value of $key by $value while preserving its TTL.
Definition: BagOStuff.php:556
BagOStuff\changeTTL
changeTTL( $key, $expiry=0)
Reset the TTL on a key if it exists.
Definition: BagOStuff.php:386
BagOStuff\merge
merge( $key, callable $callback, $exptime=0, $attempts=10, $flags=0)
Merge changes into the existing cache value (possibly creating a new one)
Definition: BagOStuff.php:276
BagOStuff\mergeViaCas
mergeViaCas( $key, $callback, $exptime=0, $attempts=10)
Definition: BagOStuff.php:289
$args
if( $line===false) $args
Definition: cdb.php:63
BagOStuff\$debugMode
bool $debugMode
Definition: BagOStuff.php:62
BagOStuff\$attrMap
integer[] $attrMap
Map of (ATTR_* class constant => QOS_* class constant)
Definition: BagOStuff.php:74
BagOStuff\mergeViaLock
mergeViaLock( $key, $callback, $exptime=0, $attempts=10, $flags=0)
Definition: BagOStuff.php:347
BagOStuff\makeGlobalKey
makeGlobalKey()
Make a global cache key.
Definition: BagOStuff.php:753
BagOStuff\READ_VERIFIED
const READ_VERIFIED
Definition: BagOStuff.php:84
as
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
BagOStuff\lock
lock( $key, $timeout=6, $expiry=6, $rclass='')
Acquire an advisory lock on a key string.
Definition: BagOStuff.php:403
$keys
$keys
Definition: testCompression.php:65
BagOStuff\cas
cas( $casToken, $key, $value, $exptime=0)
Check and set an item.
Definition: BagOStuff.php:333
BagOStuff\$keyspace
string $keyspace
Definition: BagOStuff.php:53
class
you have access to all of the normal MediaWiki so you can get a DB use the etc For full docs on the Maintenance class
Definition: maintenance.txt:52
BagOStuff\makeKeyInternal
makeKeyInternal( $keyspace, $args)
Construct a cache key.
Definition: BagOStuff.php:737
BagOStuff\getScopedLock
getScopedLock( $key, $timeout=6, $expiry=30, $rclass='')
Get a lightweight exclusive self-unlocking lock.
Definition: BagOStuff.php:469
BagOStuff\ERR_NO_RESPONSE
const ERR_NO_RESPONSE
Definition: BagOStuff.php:78
BagOStuff\setLastError
setLastError( $err)
Set the "last error" registry.
Definition: BagOStuff.php:630
BagOStuff\ERR_UNEXPECTED
const ERR_UNEXPECTED
Definition: BagOStuff.php:80
BagOStuff\getMulti
getMulti(array $keys, $flags=0)
Get an associative array containing the item for each of the keys that have items.
Definition: BagOStuff.php:509
$flags
it s the revision text itself In either if gzip is the revision text is gzipped $flags
Definition: hooks.txt:2749
array
the array() calling protocol came about after MediaWiki 1.4rc1.
BagOStuff\WRITE_CACHE_ONLY
const WRITE_CACHE_ONLY
Definition: BagOStuff.php:87
BagOStuff\convertToRelative
convertToRelative( $exptime)
Convert an optionally absolute expiry time to a relative time.
Definition: BagOStuff.php:707
BagOStuff\$dupeTrackScheduled
bool $dupeTrackScheduled
Definition: BagOStuff.php:68