MediaWiki REL1_30
BagOStuff.php
Go to the documentation of this file.
1<?php
29use Psr\Log\LoggerAwareInterface;
30use Psr\Log\LoggerInterface;
31use Psr\Log\NullLogger;
32use Wikimedia\ScopedCallback;
33use Wikimedia\WaitConditionLoop;
34
47abstract class BagOStuff implements IExpiringStore, LoggerAwareInterface {
49 protected $locks = [];
51 protected $lastError = self::ERR_NONE;
53 protected $keyspace = 'local';
55 protected $logger;
57 protected $asyncHandler;
59 protected $syncTimeout;
60
62 private $debugMode = false;
66 private $reportDupes = false;
68 private $dupeTrackScheduled = false;
69
71 protected $busyCallbacks = [];
72
74 protected $attrMap = [];
75
77 const ERR_NONE = 0; // no error
78 const ERR_NO_RESPONSE = 1; // no response
79 const ERR_UNREACHABLE = 2; // can't connect
80 const ERR_UNEXPECTED = 3; // response gave some error
81
83 const READ_LATEST = 1; // use latest data for replicated stores
84 const READ_VERIFIED = 2; // promise that caller can tell when keys are stale
86 const WRITE_SYNC = 1; // synchronously write to all locations for replicated stores
87 const WRITE_CACHE_ONLY = 2; // Only change state of the in-memory cache
88
100 public function __construct( array $params = [] ) {
101 if ( isset( $params['logger'] ) ) {
102 $this->setLogger( $params['logger'] );
103 } else {
104 $this->setLogger( new NullLogger() );
105 }
106
107 if ( isset( $params['keyspace'] ) ) {
108 $this->keyspace = $params['keyspace'];
109 }
110
111 $this->asyncHandler = isset( $params['asyncHandler'] )
112 ? $params['asyncHandler']
113 : null;
114
115 if ( !empty( $params['reportDupes'] ) && is_callable( $this->asyncHandler ) ) {
116 $this->reportDupes = true;
117 }
118
119 $this->syncTimeout = isset( $params['syncTimeout'] ) ? $params['syncTimeout'] : 3;
120 }
121
126 public function setLogger( LoggerInterface $logger ) {
127 $this->logger = $logger;
128 }
129
133 public function setDebug( $bool ) {
134 $this->debugMode = $bool;
135 }
136
149 final public function getWithSetCallback( $key, $ttl, $callback, $flags = 0 ) {
150 $value = $this->get( $key, $flags );
151
152 if ( $value === false ) {
153 if ( !is_callable( $callback ) ) {
154 throw new InvalidArgumentException( "Invalid cache miss callback provided." );
155 }
156 $value = call_user_func( $callback );
157 if ( $value !== false ) {
158 $this->set( $key, $value, $ttl );
159 }
160 }
161
162 return $value;
163 }
164
179 public function get( $key, $flags = 0, $oldFlags = null ) {
180 // B/C for ( $key, &$casToken = null, $flags = 0 )
181 $flags = is_int( $oldFlags ) ? $oldFlags : $flags;
182
183 $this->trackDuplicateKeys( $key );
184
185 return $this->doGet( $key, $flags );
186 }
187
192 private function trackDuplicateKeys( $key ) {
193 if ( !$this->reportDupes ) {
194 return;
195 }
196
197 if ( !isset( $this->duplicateKeyLookups[$key] ) ) {
198 // Track that we have seen this key. This N-1 counting style allows
199 // easy filtering with array_filter() later.
200 $this->duplicateKeyLookups[$key] = 0;
201 } else {
202 $this->duplicateKeyLookups[$key] += 1;
203
204 if ( $this->dupeTrackScheduled === false ) {
205 $this->dupeTrackScheduled = true;
206 // Schedule a callback that logs keys processed more than once by get().
207 call_user_func( $this->asyncHandler, function () {
208 $dups = array_filter( $this->duplicateKeyLookups );
209 foreach ( $dups as $key => $count ) {
210 $this->logger->warning(
211 'Duplicate get(): "{key}" fetched {count} times',
212 // Count is N-1 of the actual lookup count
213 [ 'key' => $key, 'count' => $count + 1, ]
214 );
215 }
216 } );
217 }
218 }
219 }
220
226 abstract protected function doGet( $key, $flags = 0 );
227
237 protected function getWithToken( $key, &$casToken, $flags = 0 ) {
238 throw new Exception( __METHOD__ . ' not implemented.' );
239 }
240
250 abstract public function set( $key, $value, $exptime = 0, $flags = 0 );
251
258 abstract public function delete( $key );
259
276 public function merge( $key, callable $callback, $exptime = 0, $attempts = 10, $flags = 0 ) {
277 return $this->mergeViaLock( $key, $callback, $exptime, $attempts, $flags );
278 }
279
289 protected function mergeViaCas( $key, $callback, $exptime = 0, $attempts = 10 ) {
290 do {
291 $this->clearLastError();
292 $reportDupes = $this->reportDupes;
293 $this->reportDupes = false;
294 $casToken = null; // passed by reference
295 $currentValue = $this->getWithToken( $key, $casToken, self::READ_LATEST );
296 $this->reportDupes = $reportDupes;
297
298 if ( $this->getLastError() ) {
299 return false; // don't spam retries (retry only on races)
300 }
301
302 // Derive the new value from the old value
303 $value = call_user_func( $callback, $this, $key, $currentValue, $exptime );
304
305 $this->clearLastError();
306 if ( $value === false ) {
307 $success = true; // do nothing
308 } elseif ( $currentValue === false ) {
309 // Try to create the key, failing if it gets created in the meantime
310 $success = $this->add( $key, $value, $exptime );
311 } else {
312 // Try to update the key, failing if it gets changed in the meantime
313 $success = $this->cas( $casToken, $key, $value, $exptime );
314 }
315 if ( $this->getLastError() ) {
316 return false; // IO error; don't spam retries
317 }
318 } while ( !$success && --$attempts );
319
320 return $success;
321 }
322
333 protected function cas( $casToken, $key, $value, $exptime = 0 ) {
334 throw new Exception( "CAS is not implemented in " . __CLASS__ );
335 }
336
347 protected function mergeViaLock( $key, $callback, $exptime = 0, $attempts = 10, $flags = 0 ) {
348 if ( !$this->lock( $key, 6 ) ) {
349 return false;
350 }
351
352 $this->clearLastError();
353 $reportDupes = $this->reportDupes;
354 $this->reportDupes = false;
355 $currentValue = $this->get( $key, self::READ_LATEST );
356 $this->reportDupes = $reportDupes;
357
358 if ( $this->getLastError() ) {
359 $success = false;
360 } else {
361 // Derive the new value from the old value
362 $value = call_user_func( $callback, $this, $key, $currentValue, $exptime );
363 if ( $value === false ) {
364 $success = true; // do nothing
365 } else {
366 $success = $this->set( $key, $value, $exptime, $flags ); // set the new value
367 }
368 }
369
370 if ( !$this->unlock( $key ) ) {
371 // this should never happen
372 trigger_error( "Could not release lock for key '$key'." );
373 }
374
375 return $success;
376 }
377
386 public function changeTTL( $key, $expiry = 0 ) {
387 $value = $this->get( $key );
388
389 return ( $value === false ) ? false : $this->set( $key, $value, $expiry );
390 }
391
403 public function lock( $key, $timeout = 6, $expiry = 6, $rclass = '' ) {
404 // Avoid deadlocks and allow lock reentry if specified
405 if ( isset( $this->locks[$key] ) ) {
406 if ( $rclass != '' && $this->locks[$key]['class'] === $rclass ) {
407 ++$this->locks[$key]['depth'];
408 return true;
409 } else {
410 return false;
411 }
412 }
413
414 $expiry = min( $expiry ?: INF, self::TTL_DAY );
415 $loop = new WaitConditionLoop(
416 function () use ( $key, $timeout, $expiry ) {
417 $this->clearLastError();
418 if ( $this->add( "{$key}:lock", 1, $expiry ) ) {
419 return true; // locked!
420 } elseif ( $this->getLastError() ) {
421 return WaitConditionLoop::CONDITION_ABORTED; // network partition?
422 }
423
424 return WaitConditionLoop::CONDITION_CONTINUE;
425 },
426 $timeout
427 );
428
429 $locked = ( $loop->invoke() === $loop::CONDITION_REACHED );
430 if ( $locked ) {
431 $this->locks[$key] = [ 'class' => $rclass, 'depth' => 1 ];
432 }
433
434 return $locked;
435 }
436
443 public function unlock( $key ) {
444 if ( isset( $this->locks[$key] ) && --$this->locks[$key]['depth'] <= 0 ) {
445 unset( $this->locks[$key] );
446
447 return $this->delete( "{$key}:lock" );
448 }
449
450 return true;
451 }
452
469 final public function getScopedLock( $key, $timeout = 6, $expiry = 30, $rclass = '' ) {
470 $expiry = min( $expiry ?: INF, self::TTL_DAY );
471
472 if ( !$this->lock( $key, $timeout, $expiry, $rclass ) ) {
473 return null;
474 }
475
476 $lSince = microtime( true ); // lock timestamp
477
478 return new ScopedCallback( function () use ( $key, $lSince, $expiry ) {
479 $latency = 0.050; // latency skew (err towards keeping lock present)
480 $age = ( microtime( true ) - $lSince + $latency );
481 if ( ( $age + $latency ) >= $expiry ) {
482 $this->logger->warning( "Lock for $key held too long ($age sec)." );
483 return; // expired; it's not "safe" to delete the key
484 }
485 $this->unlock( $key );
486 } );
487 }
488
498 public function deleteObjectsExpiringBefore( $date, $progressCallback = false ) {
499 // stub
500 return false;
501 }
502
509 public function getMulti( array $keys, $flags = 0 ) {
510 $res = [];
511 foreach ( $keys as $key ) {
512 $val = $this->get( $key );
513 if ( $val !== false ) {
514 $res[$key] = $val;
515 }
516 }
517 return $res;
518 }
519
527 public function setMulti( array $data, $exptime = 0 ) {
528 $res = true;
529 foreach ( $data as $key => $value ) {
530 if ( !$this->set( $key, $value, $exptime ) ) {
531 $res = false;
532 }
533 }
534 return $res;
535 }
536
543 public function add( $key, $value, $exptime = 0 ) {
544 if ( $this->get( $key ) === false ) {
545 return $this->set( $key, $value, $exptime );
546 }
547 return false; // key already set
548 }
549
556 public function incr( $key, $value = 1 ) {
557 if ( !$this->lock( $key ) ) {
558 return false;
559 }
560 $n = $this->get( $key );
561 if ( $this->isInteger( $n ) ) { // key exists?
562 $n += intval( $value );
563 $this->set( $key, max( 0, $n ) ); // exptime?
564 } else {
565 $n = false;
566 }
567 $this->unlock( $key );
568
569 return $n;
570 }
571
578 public function decr( $key, $value = 1 ) {
579 return $this->incr( $key, - $value );
580 }
581
594 public function incrWithInit( $key, $ttl, $value = 1, $init = 1 ) {
595 $newValue = $this->incr( $key, $value );
596 if ( $newValue === false ) {
597 // No key set; initialize
598 $newValue = $this->add( $key, (int)$init, $ttl ) ? $init : false;
599 }
600 if ( $newValue === false ) {
601 // Raced out initializing; increment
602 $newValue = $this->incr( $key, $value );
603 }
604
605 return $newValue;
606 }
607
613 public function getLastError() {
614 return $this->lastError;
615 }
616
621 public function clearLastError() {
622 $this->lastError = self::ERR_NONE;
623 }
624
630 protected function setLastError( $err ) {
631 $this->lastError = $err;
632 }
633
654 public function addBusyCallback( callable $workCallback ) {
655 $this->busyCallbacks[] = $workCallback;
656 }
657
672 public function modifySimpleRelayEvent( array $event ) {
673 return $event;
674 }
675
679 protected function debug( $text ) {
680 if ( $this->debugMode ) {
681 $this->logger->debug( "{class} debug: $text", [
682 'class' => static::class,
683 ] );
684 }
685 }
686
692 protected function convertExpiry( $exptime ) {
693 if ( $exptime != 0 && $exptime < ( 10 * self::TTL_YEAR ) ) {
694 return time() + $exptime;
695 } else {
696 return $exptime;
697 }
698 }
699
707 protected function convertToRelative( $exptime ) {
708 if ( $exptime >= ( 10 * self::TTL_YEAR ) ) {
709 $exptime -= time();
710 if ( $exptime <= 0 ) {
711 $exptime = 1;
712 }
713 return $exptime;
714 } else {
715 return $exptime;
716 }
717 }
718
725 protected function isInteger( $value ) {
726 return ( is_int( $value ) || ctype_digit( $value ) );
727 }
728
737 public function makeKeyInternal( $keyspace, $args ) {
738 $key = $keyspace;
739 foreach ( $args as $arg ) {
740 $arg = str_replace( ':', '%3A', $arg );
741 $key = $key . ':' . $arg;
742 }
743 return strtr( $key, ' ', '_' );
744 }
745
753 public function makeGlobalKey() {
754 return $this->makeKeyInternal( 'global', func_get_args() );
755 }
756
764 public function makeKey() {
765 return $this->makeKeyInternal( $this->keyspace, func_get_args() );
766 }
767
773 public function getQoS( $flag ) {
774 return isset( $this->attrMap[$flag] ) ? $this->attrMap[$flag] : self::QOS_UNKNOWN;
775 }
776
783 protected function mergeFlagMaps( array $bags ) {
784 $map = [];
785 foreach ( $bags as $bag ) {
786 foreach ( $bag->attrMap as $attr => $rank ) {
787 if ( isset( $map[$attr] ) ) {
788 $map[$attr] = min( $map[$attr], $rank );
789 } else {
790 $map[$attr] = $rank;
791 }
792 }
793 }
794
795 return $map;
796 }
797}
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
if( $line===false) $args
Definition cdb.php:63
interface is intended to be more or less compatible with the PHP memcached client.
Definition BagOStuff.php:47
const ERR_UNEXPECTED
Definition BagOStuff.php:80
int[] $attrMap
Map of (ATTR_* class constant => QOS_* class constant)
Definition BagOStuff.php:74
getWithSetCallback( $key, $ttl, $callback, $flags=0)
Get an item with the given key, regenerating and setting it if not found.
__construct(array $params=[])
$params include:
getScopedLock( $key, $timeout=6, $expiry=30, $rclass='')
Get a lightweight exclusive self-unlocking lock.
unlock( $key)
Release an advisory lock on a key string.
incrWithInit( $key, $ttl, $value=1, $init=1)
Increase stored value of $key by $value while preserving its TTL.
isInteger( $value)
Check if a value is an integer.
lock( $key, $timeout=6, $expiry=6, $rclass='')
Acquire an advisory lock on a key string.
bool $dupeTrackScheduled
Definition BagOStuff.php:68
array $duplicateKeyLookups
Definition BagOStuff.php:64
add( $key, $value, $exptime=0)
getMulti(array $keys, $flags=0)
Get an associative array containing the item for each of the keys that have items.
decr( $key, $value=1)
Decrease stored value of $key by $value while preserving its TTL.
setDebug( $bool)
modifySimpleRelayEvent(array $event)
Modify a cache update operation array for EventRelayer::notify()
getQoS( $flag)
const READ_LATEST
Bitfield constants for get()/getMulti()
Definition BagOStuff.php:83
deleteObjectsExpiringBefore( $date, $progressCallback=false)
Delete all objects expiring before a certain date.
const READ_VERIFIED
Definition BagOStuff.php:84
trackDuplicateKeys( $key)
Track the number of times that a given key has been used.
convertExpiry( $exptime)
Convert an optionally relative time to an absolute time.
callable[] $busyCallbacks
Definition BagOStuff.php:71
callback null $asyncHandler
Definition BagOStuff.php:57
getWithToken( $key, &$casToken, $flags=0)
setMulti(array $data, $exptime=0)
Batch insertion.
const ERR_NO_RESPONSE
Definition BagOStuff.php:78
const ERR_NONE
Possible values for getLastError()
Definition BagOStuff.php:77
debug( $text)
doGet( $key, $flags=0)
convertToRelative( $exptime)
Convert an optionally absolute expiry time to a relative time.
setLogger(LoggerInterface $logger)
makeGlobalKey()
Make a global cache key.
int $syncTimeout
Seconds.
Definition BagOStuff.php:59
const WRITE_SYNC
Bitfield constants for set()/merge()
Definition BagOStuff.php:86
mergeFlagMaps(array $bags)
Merge the flag maps of one or more BagOStuff objects into a "lowest common denominator" map.
bool $reportDupes
Definition BagOStuff.php:66
getLastError()
Get the "last error" registered; clearLastError() should be called manually.
bool $debugMode
Definition BagOStuff.php:62
LoggerInterface $logger
Definition BagOStuff.php:55
makeKey()
Make a cache key, scoped to this instance's keyspace.
setLastError( $err)
Set the "last error" registry.
changeTTL( $key, $expiry=0)
Reset the TTL on a key if it exists.
mergeViaLock( $key, $callback, $exptime=0, $attempts=10, $flags=0)
clearLastError()
Clear the "last error" registry.
addBusyCallback(callable $workCallback)
Let a callback be run to avoid wasting time on special blocking calls.
cas( $casToken, $key, $value, $exptime=0)
Check and set an item.
makeKeyInternal( $keyspace, $args)
Construct a cache key.
const WRITE_CACHE_ONLY
Definition BagOStuff.php:87
string $keyspace
Definition BagOStuff.php:53
int $lastError
ERR_* class constant.
Definition BagOStuff.php:51
array[] $locks
Lock tracking.
Definition BagOStuff.php:49
mergeViaCas( $key, $callback, $exptime=0, $attempts=10)
incr( $key, $value=1)
Increase stored value of $key by $value while preserving its TTL.
merge( $key, callable $callback, $exptime=0, $attempts=10, $flags=0)
Merge changes into the existing cache value (possibly creating a new one)
const ERR_UNREACHABLE
Definition BagOStuff.php:79
$res
Definition database.txt:21
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
the array() calling protocol came about after MediaWiki 1.4rc1.
it s the revision text itself In either if gzip is the revision text is gzipped $flags
Definition hooks.txt:2805
processing should stop and the error should be shown to the user * false
Definition hooks.txt:187
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition injection.txt:37
Generic base class for storage interfaces.
$params