MediaWiki  1.28.0
BagOStuff.php
Go to the documentation of this file.
1 <?php
34 
47 abstract class BagOStuff implements IExpiringStore, LoggerAwareInterface {
49  protected $locks = [];
51  protected $lastError = self::ERR_NONE;
53  protected $keyspace = 'local';
55  protected $logger;
57  protected $asyncHandler;
59  protected $syncTimeout;
60 
62  private $debugMode = false;
64  private $duplicateKeyLookups = [];
66  private $reportDupes = false;
68  private $dupeTrackScheduled = false;
69 
71  protected $busyCallbacks = [];
72 
74  protected $attrMap = [];
75 
77  const ERR_NONE = 0; // no error
78  const ERR_NO_RESPONSE = 1; // no response
79  const ERR_UNREACHABLE = 2; // can't connect
80  const ERR_UNEXPECTED = 3; // response gave some error
81 
83  const READ_LATEST = 1; // use latest data for replicated stores
84  const READ_VERIFIED = 2; // promise that caller can tell when keys are stale
86  const WRITE_SYNC = 1; // synchronously write to all locations for replicated stores
87  const WRITE_CACHE_ONLY = 2; // Only change state of the in-memory cache
88 
100  public function __construct( array $params = [] ) {
101  if ( isset( $params['logger'] ) ) {
102  $this->setLogger( $params['logger'] );
103  } else {
104  $this->setLogger( new NullLogger() );
105  }
106 
107  if ( isset( $params['keyspace'] ) ) {
108  $this->keyspace = $params['keyspace'];
109  }
110 
111  $this->asyncHandler = isset( $params['asyncHandler'] )
112  ? $params['asyncHandler']
113  : null;
114 
115  if ( !empty( $params['reportDupes'] ) && is_callable( $this->asyncHandler ) ) {
116  $this->reportDupes = true;
117  }
118 
119  $this->syncTimeout = isset( $params['syncTimeout'] ) ? $params['syncTimeout'] : 3;
120  }
121 
126  public function setLogger( LoggerInterface $logger ) {
127  $this->logger = $logger;
128  }
129 
133  public function setDebug( $bool ) {
134  $this->debugMode = $bool;
135  }
136 
149  final public function getWithSetCallback( $key, $ttl, $callback, $flags = 0 ) {
150  $value = $this->get( $key, $flags );
151 
152  if ( $value === false ) {
153  if ( !is_callable( $callback ) ) {
154  throw new InvalidArgumentException( "Invalid cache miss callback provided." );
155  }
156  $value = call_user_func( $callback );
157  if ( $value !== false ) {
158  $this->set( $key, $value, $ttl );
159  }
160  }
161 
162  return $value;
163  }
164 
179  public function get( $key, $flags = 0, $oldFlags = null ) {
180  // B/C for ( $key, &$casToken = null, $flags = 0 )
181  $flags = is_int( $oldFlags ) ? $oldFlags : $flags;
182 
183  $this->trackDuplicateKeys( $key );
184 
185  return $this->doGet( $key, $flags );
186  }
187 
192  private function trackDuplicateKeys( $key ) {
193  if ( !$this->reportDupes ) {
194  return;
195  }
196 
197  if ( !isset( $this->duplicateKeyLookups[$key] ) ) {
198  // Track that we have seen this key. This N-1 counting style allows
199  // easy filtering with array_filter() later.
200  $this->duplicateKeyLookups[$key] = 0;
201  } else {
202  $this->duplicateKeyLookups[$key] += 1;
203 
204  if ( $this->dupeTrackScheduled === false ) {
205  $this->dupeTrackScheduled = true;
206  // Schedule a callback that logs keys processed more than once by get().
207  call_user_func( $this->asyncHandler, function () {
208  $dups = array_filter( $this->duplicateKeyLookups );
209  foreach ( $dups as $key => $count ) {
210  $this->logger->warning(
211  'Duplicate get(): "{key}" fetched {count} times',
212  // Count is N-1 of the actual lookup count
213  [ 'key' => $key, 'count' => $count + 1, ]
214  );
215  }
216  } );
217  }
218  }
219  }
220 
226  abstract protected function doGet( $key, $flags = 0 );
227 
237  protected function getWithToken( $key, &$casToken, $flags = 0 ) {
238  throw new Exception( __METHOD__ . ' not implemented.' );
239  }
240 
250  abstract public function set( $key, $value, $exptime = 0, $flags = 0 );
251 
258  abstract public function delete( $key );
259 
276  public function merge( $key, callable $callback, $exptime = 0, $attempts = 10, $flags = 0 ) {
277  return $this->mergeViaLock( $key, $callback, $exptime, $attempts, $flags );
278  }
279 
289  protected function mergeViaCas( $key, $callback, $exptime = 0, $attempts = 10 ) {
290  do {
291  $this->clearLastError();
293  $this->reportDupes = false;
294  $casToken = null; // passed by reference
295  $currentValue = $this->getWithToken( $key, $casToken, self::READ_LATEST );
296  $this->reportDupes = $reportDupes;
297 
298  if ( $this->getLastError() ) {
299  return false; // don't spam retries (retry only on races)
300  }
301 
302  // Derive the new value from the old value
303  $value = call_user_func( $callback, $this, $key, $currentValue, $exptime );
304 
305  $this->clearLastError();
306  if ( $value === false ) {
307  $success = true; // do nothing
308  } elseif ( $currentValue === false ) {
309  // Try to create the key, failing if it gets created in the meantime
310  $success = $this->add( $key, $value, $exptime );
311  } else {
312  // Try to update the key, failing if it gets changed in the meantime
313  $success = $this->cas( $casToken, $key, $value, $exptime );
314  }
315  if ( $this->getLastError() ) {
316  return false; // IO error; don't spam retries
317  }
318  } while ( !$success && --$attempts );
319 
320  return $success;
321  }
322 
333  protected function cas( $casToken, $key, $value, $exptime = 0 ) {
334  throw new Exception( "CAS is not implemented in " . __CLASS__ );
335  }
336 
347  protected function mergeViaLock( $key, $callback, $exptime = 0, $attempts = 10, $flags = 0 ) {
348  if ( !$this->lock( $key, 6 ) ) {
349  return false;
350  }
351 
352  $this->clearLastError();
354  $this->reportDupes = false;
355  $currentValue = $this->get( $key, self::READ_LATEST );
356  $this->reportDupes = $reportDupes;
357 
358  if ( $this->getLastError() ) {
359  $success = false;
360  } else {
361  // Derive the new value from the old value
362  $value = call_user_func( $callback, $this, $key, $currentValue, $exptime );
363  if ( $value === false ) {
364  $success = true; // do nothing
365  } else {
366  $success = $this->set( $key, $value, $exptime, $flags ); // set the new value
367  }
368  }
369 
370  if ( !$this->unlock( $key ) ) {
371  // this should never happen
372  trigger_error( "Could not release lock for key '$key'." );
373  }
374 
375  return $success;
376  }
377 
386  public function changeTTL( $key, $expiry = 0 ) {
387  $value = $this->get( $key );
388 
389  return ( $value === false ) ? false : $this->set( $key, $value, $expiry );
390  }
391 
403  public function lock( $key, $timeout = 6, $expiry = 6, $rclass = '' ) {
404  // Avoid deadlocks and allow lock reentry if specified
405  if ( isset( $this->locks[$key] ) ) {
406  if ( $rclass != '' && $this->locks[$key]['class'] === $rclass ) {
407  ++$this->locks[$key]['depth'];
408  return true;
409  } else {
410  return false;
411  }
412  }
413 
414  $expiry = min( $expiry ?: INF, self::TTL_DAY );
415  $loop = new WaitConditionLoop(
416  function () use ( $key, $timeout, $expiry ) {
417  $this->clearLastError();
418  if ( $this->add( "{$key}:lock", 1, $expiry ) ) {
419  return true; // locked!
420  } elseif ( $this->getLastError() ) {
421  return WaitConditionLoop::CONDITION_ABORTED; // network partition?
422  }
423 
424  return WaitConditionLoop::CONDITION_CONTINUE;
425  },
426  $timeout
427  );
428 
429  $locked = ( $loop->invoke() === $loop::CONDITION_REACHED );
430  if ( $locked ) {
431  $this->locks[$key] = [ 'class' => $rclass, 'depth' => 1 ];
432  }
433 
434  return $locked;
435  }
436 
443  public function unlock( $key ) {
444  if ( isset( $this->locks[$key] ) && --$this->locks[$key]['depth'] <= 0 ) {
445  unset( $this->locks[$key] );
446 
447  return $this->delete( "{$key}:lock" );
448  }
449 
450  return true;
451  }
452 
469  final public function getScopedLock( $key, $timeout = 6, $expiry = 30, $rclass = '' ) {
470  $expiry = min( $expiry ?: INF, self::TTL_DAY );
471 
472  if ( !$this->lock( $key, $timeout, $expiry, $rclass ) ) {
473  return null;
474  }
475 
476  $lSince = microtime( true ); // lock timestamp
477 
478  return new ScopedCallback( function() use ( $key, $lSince, $expiry ) {
479  $latency = .050; // latency skew (err towards keeping lock present)
480  $age = ( microtime( true ) - $lSince + $latency );
481  if ( ( $age + $latency ) >= $expiry ) {
482  $this->logger->warning( "Lock for $key held too long ($age sec)." );
483  return; // expired; it's not "safe" to delete the key
484  }
485  $this->unlock( $key );
486  } );
487  }
488 
498  public function deleteObjectsExpiringBefore( $date, $progressCallback = false ) {
499  // stub
500  return false;
501  }
502 
509  public function getMulti( array $keys, $flags = 0 ) {
510  $res = [];
511  foreach ( $keys as $key ) {
512  $val = $this->get( $key );
513  if ( $val !== false ) {
514  $res[$key] = $val;
515  }
516  }
517  return $res;
518  }
519 
527  public function setMulti( array $data, $exptime = 0 ) {
528  $res = true;
529  foreach ( $data as $key => $value ) {
530  if ( !$this->set( $key, $value, $exptime ) ) {
531  $res = false;
532  }
533  }
534  return $res;
535  }
536 
543  public function add( $key, $value, $exptime = 0 ) {
544  if ( $this->get( $key ) === false ) {
545  return $this->set( $key, $value, $exptime );
546  }
547  return false; // key already set
548  }
549 
556  public function incr( $key, $value = 1 ) {
557  if ( !$this->lock( $key ) ) {
558  return false;
559  }
560  $n = $this->get( $key );
561  if ( $this->isInteger( $n ) ) { // key exists?
562  $n += intval( $value );
563  $this->set( $key, max( 0, $n ) ); // exptime?
564  } else {
565  $n = false;
566  }
567  $this->unlock( $key );
568 
569  return $n;
570  }
571 
578  public function decr( $key, $value = 1 ) {
579  return $this->incr( $key, - $value );
580  }
581 
594  public function incrWithInit( $key, $ttl, $value = 1, $init = 1 ) {
595  $newValue = $this->incr( $key, $value );
596  if ( $newValue === false ) {
597  // No key set; initialize
598  $newValue = $this->add( $key, (int)$init, $ttl ) ? $init : false;
599  }
600  if ( $newValue === false ) {
601  // Raced out initializing; increment
602  $newValue = $this->incr( $key, $value );
603  }
604 
605  return $newValue;
606  }
607 
613  public function getLastError() {
614  return $this->lastError;
615  }
616 
621  public function clearLastError() {
622  $this->lastError = self::ERR_NONE;
623  }
624 
630  protected function setLastError( $err ) {
631  $this->lastError = $err;
632  }
633 
654  public function addBusyCallback( callable $workCallback ) {
655  $this->busyCallbacks[] = $workCallback;
656  }
657 
672  public function modifySimpleRelayEvent( array $event ) {
673  return $event;
674  }
675 
679  protected function debug( $text ) {
680  if ( $this->debugMode ) {
681  $this->logger->debug( "{class} debug: $text", [
682  'class' => get_class( $this ),
683  ] );
684  }
685  }
686 
692  protected function convertExpiry( $exptime ) {
693  if ( $exptime != 0 && $exptime < ( 10 * self::TTL_YEAR ) ) {
694  return time() + $exptime;
695  } else {
696  return $exptime;
697  }
698  }
699 
707  protected function convertToRelative( $exptime ) {
708  if ( $exptime >= ( 10 * self::TTL_YEAR ) ) {
709  $exptime -= time();
710  if ( $exptime <= 0 ) {
711  $exptime = 1;
712  }
713  return $exptime;
714  } else {
715  return $exptime;
716  }
717  }
718 
725  protected function isInteger( $value ) {
726  return ( is_int( $value ) || ctype_digit( $value ) );
727  }
728 
737  public function makeKeyInternal( $keyspace, $args ) {
738  $key = $keyspace;
739  foreach ( $args as $arg ) {
740  $arg = str_replace( ':', '%3A', $arg );
741  $key = $key . ':' . $arg;
742  }
743  return strtr( $key, ' ', '_' );
744  }
745 
753  public function makeGlobalKey() {
754  return $this->makeKeyInternal( 'global', func_get_args() );
755  }
756 
764  public function makeKey() {
765  return $this->makeKeyInternal( $this->keyspace, func_get_args() );
766  }
767 
773  public function getQoS( $flag ) {
774  return isset( $this->attrMap[$flag] ) ? $this->attrMap[$flag] : self::QOS_UNKNOWN;
775  }
776 
783  protected function mergeFlagMaps( array $bags ) {
784  $map = [];
785  foreach ( $bags as $bag ) {
786  foreach ( $bag->attrMap as $attr => $rank ) {
787  if ( isset( $map[$attr] ) ) {
788  $map[$attr] = min( $map[$attr], $rank );
789  } else {
790  $map[$attr] = $rank;
791  }
792  }
793  }
794 
795  return $map;
796  }
797 }
clearLastError()
Clear the "last error" registry.
Definition: BagOStuff.php:621
const ERR_UNEXPECTED
Definition: BagOStuff.php:80
the array() calling protocol came about after MediaWiki 1.4rc1.
getWithToken($key, &$casToken, $flags=0)
Definition: BagOStuff.php:237
trackDuplicateKeys($key)
Track the number of times that a given key has been used.
Definition: BagOStuff.php:192
bool $reportDupes
Definition: BagOStuff.php:66
$success
bool $dupeTrackScheduled
Definition: BagOStuff.php:68
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
callback null $asyncHandler
Definition: BagOStuff.php:57
getScopedLock($key, $timeout=6, $expiry=30, $rclass= '')
Get a lightweight exclusive self-unlocking lock.
Definition: BagOStuff.php:469
lock($key, $timeout=6, $expiry=6, $rclass= '')
Acquire an advisory lock on a key string.
Definition: BagOStuff.php:403
getQoS($flag)
Definition: BagOStuff.php:773
const ERR_NO_RESPONSE
Definition: BagOStuff.php:78
$value
it s the revision text itself In either if gzip is the revision text is gzipped $flags
Definition: hooks.txt:2703
incrWithInit($key, $ttl, $value=1, $init=1)
Increase stored value of $key by $value while preserving its TTL.
Definition: BagOStuff.php:594
const ERR_UNREACHABLE
Definition: BagOStuff.php:79
doGet($key, $flags=0)
set($key, $value, $exptime=0, $flags=0)
Set an item.
getWithSetCallback($key, $ttl, $callback, $flags=0)
Get an item with the given key, regenerating and setting it if not found.
Definition: BagOStuff.php:149
deleteObjectsExpiringBefore($date, $progressCallback=false)
Delete all objects expiring before a certain date.
Definition: BagOStuff.php:498
string $keyspace
Definition: BagOStuff.php:53
modifySimpleRelayEvent(array $event)
Modify a cache update operation array for EventRelayer::notify()
Definition: BagOStuff.php:672
if($line===false) $args
Definition: cdb.php:64
getLastError()
Get the "last error" registered; clearLastError() should be called manually.
Definition: BagOStuff.php:613
mergeViaCas($key, $callback, $exptime=0, $attempts=10)
Definition: BagOStuff.php:289
__construct(array $params=[])
$params include:
Definition: BagOStuff.php:100
bool $debugMode
Definition: BagOStuff.php:62
callable[] $busyCallbacks
Definition: BagOStuff.php:71
convertExpiry($exptime)
Convert an optionally relative time to an absolute time.
Definition: BagOStuff.php:692
const READ_VERIFIED
Definition: BagOStuff.php:84
array $duplicateKeyLookups
Definition: BagOStuff.php:64
$res
Definition: database.txt:21
add($key, $value, $exptime=0)
Definition: BagOStuff.php:543
getMulti(array $keys, $flags=0)
Get an associative array containing the item for each of the keys that have items.
Definition: BagOStuff.php:509
merge($key, callable $callback, $exptime=0, $attempts=10, $flags=0)
Merge changes into the existing cache value (possibly creating a new one)
Definition: BagOStuff.php:276
incr($key, $value=1)
Increase stored value of $key by $value while preserving its TTL.
Definition: BagOStuff.php:556
setDebug($bool)
Definition: BagOStuff.php:133
integer $syncTimeout
Seconds.
Definition: BagOStuff.php:59
const WRITE_SYNC
Bitfield constants for set()/merge()
Definition: BagOStuff.php:86
$params
const ERR_NONE
Possible values for getLastError()
Definition: BagOStuff.php:77
const READ_LATEST
Bitfield constants for get()/getMulti()
Definition: BagOStuff.php:83
mergeFlagMaps(array $bags)
Merge the flag maps of one or more BagOStuff objects into a "lowest common denominator" map...
Definition: BagOStuff.php:783
integer[] $attrMap
Map of (ATTR_* class constant => QOS_* class constant)
Definition: BagOStuff.php:74
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
setMulti(array $data, $exptime=0)
Batch insertion.
Definition: BagOStuff.php:527
mergeViaLock($key, $callback, $exptime=0, $attempts=10, $flags=0)
Definition: BagOStuff.php:347
const WRITE_CACHE_ONLY
Definition: BagOStuff.php:87
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
decr($key, $value=1)
Decrease stored value of $key by $value while preserving its TTL.
Definition: BagOStuff.php:578
changeTTL($key, $expiry=0)
Reset the TTL on a key if it exists.
Definition: BagOStuff.php:386
LoggerInterface $logger
Definition: BagOStuff.php:55
setLogger(LoggerInterface $logger)
Definition: BagOStuff.php:126
convertToRelative($exptime)
Convert an optionally absolute expiry time to a relative time.
Definition: BagOStuff.php:707
makeKey()
Make a cache key, scoped to this instance's keyspace.
Definition: BagOStuff.php:764
addBusyCallback(callable $workCallback)
Let a callback be run to avoid wasting time on special blocking calls.
Definition: BagOStuff.php:654
$count
cas($casToken, $key, $value, $exptime=0)
Check and set an item.
Definition: BagOStuff.php:333
Generic base class for storage interfaces.
array[] $locks
Lock tracking.
Definition: BagOStuff.php:49
makeKeyInternal($keyspace, $args)
Construct a cache key.
Definition: BagOStuff.php:737
debug($text)
Definition: BagOStuff.php:679
setLastError($err)
Set the "last error" registry.
Definition: BagOStuff.php:630
unlock($key)
Release an advisory lock on a key string.
Definition: BagOStuff.php:443
isInteger($value)
Check if a value is an integer.
Definition: BagOStuff.php:725
makeGlobalKey()
Make a global cache key.
Definition: BagOStuff.php:753
integer $lastError
ERR_* class constant.
Definition: BagOStuff.php:51