MediaWiki  master
RedisBagOStuff.php
Go to the documentation of this file.
1 <?php
34  protected $redisPool;
36  protected $servers;
38  protected $serverTagMap;
40  protected $automaticFailover;
41 
70  function __construct( $params ) {
71  parent::__construct( $params );
72  $redisConf = [ 'serializer' => 'none' ]; // manage that in this class
73  foreach ( [ 'connectTimeout', 'persistent', 'password' ] as $opt ) {
74  if ( isset( $params[$opt] ) ) {
75  $redisConf[$opt] = $params[$opt];
76  }
77  }
78  $this->redisPool = RedisConnectionPool::singleton( $redisConf );
79 
80  $this->servers = $params['servers'];
81  foreach ( $this->servers as $key => $server ) {
82  $this->serverTagMap[is_int( $key ) ? $server : $key] = $server;
83  }
84 
85  $this->automaticFailover = $params['automaticFailover'] ?? true;
86 
87  $this->attrMap[self::ATTR_SYNCWRITES] = self::QOS_SYNCWRITES_NONE;
88  }
89 
90  protected function doGet( $key, $flags = 0, &$casToken = null ) {
91  $casToken = null;
92 
93  $conn = $this->getConnection( $key );
94  if ( !$conn ) {
95  return false;
96  }
97 
98  $e = null;
99  try {
100  $value = $conn->get( $key );
101  $casToken = $value;
102  $result = $this->unserialize( $value );
103  } catch ( RedisException $e ) {
104  $result = false;
105  $this->handleException( $conn, $e );
106  }
107 
108  $this->logRequest( 'get', $key, $conn->getServer(), $e );
109 
110  return $result;
111  }
112 
113  protected function doSet( $key, $value, $exptime = 0, $flags = 0 ) {
114  $conn = $this->getConnection( $key );
115  if ( !$conn ) {
116  return false;
117  }
118 
119  $ttl = $this->getExpirationAsTTL( $exptime );
120 
121  $e = null;
122  try {
123  if ( $ttl ) {
124  $result = $conn->setex( $key, $ttl, $this->serialize( $value ) );
125  } else {
126  $result = $conn->set( $key, $this->serialize( $value ) );
127  }
128  } catch ( RedisException $e ) {
129  $result = false;
130  $this->handleException( $conn, $e );
131  }
132 
133  $this->logRequest( 'set', $key, $conn->getServer(), $e );
134 
135  return $result;
136  }
137 
138  protected function doDelete( $key, $flags = 0 ) {
139  $conn = $this->getConnection( $key );
140  if ( !$conn ) {
141  return false;
142  }
143 
144  $e = null;
145  try {
146  // Note that redis does not return false if the key was not there
147  $result = ( $conn->del( $key ) !== false );
148  } catch ( RedisException $e ) {
149  $result = false;
150  $this->handleException( $conn, $e );
151  }
152 
153  $this->logRequest( 'delete', $key, $conn->getServer(), $e );
154 
155  return $result;
156  }
157 
158  protected function doGetMulti( array $keys, $flags = 0 ) {
160  $conns = [];
161  $batches = [];
162  foreach ( $keys as $key ) {
163  $conn = $this->getConnection( $key );
164  if ( $conn ) {
165  $server = $conn->getServer();
166  $conns[$server] = $conn;
167  $batches[$server][] = $key;
168  }
169  }
170 
171  $result = [];
172  foreach ( $batches as $server => $batchKeys ) {
173  $conn = $conns[$server];
174 
175  $e = null;
176  try {
177  // Avoid mget() to reduce CPU hogging from a single request
178  $conn->multi( Redis::PIPELINE );
179  foreach ( $batchKeys as $key ) {
180  $conn->get( $key );
181  }
182  $batchResult = $conn->exec();
183  if ( $batchResult === false ) {
184  $this->logRequest( 'get', implode( ',', $batchKeys ), $server, true );
185  continue;
186  }
187 
188  foreach ( $batchResult as $i => $value ) {
189  if ( $value !== false ) {
190  $result[$batchKeys[$i]] = $this->unserialize( $value );
191  }
192  }
193  } catch ( RedisException $e ) {
194  $this->handleException( $conn, $e );
195  }
196 
197  $this->logRequest( 'get', implode( ',', $batchKeys ), $server, $e );
198  }
199 
200  return $result;
201  }
202 
203  protected function doSetMulti( array $data, $exptime = 0, $flags = 0 ) {
205  $conns = [];
206  $batches = [];
207  foreach ( $data as $key => $value ) {
208  $conn = $this->getConnection( $key );
209  if ( $conn ) {
210  $server = $conn->getServer();
211  $conns[$server] = $conn;
212  $batches[$server][] = $key;
213  }
214  }
215 
216  $ttl = $this->getExpirationAsTTL( $exptime );
217  $op = $ttl ? 'setex' : 'set';
218 
219  $result = true;
220  foreach ( $batches as $server => $batchKeys ) {
221  $conn = $conns[$server];
222 
223  $e = null;
224  try {
225  // Avoid mset() to reduce CPU hogging from a single request
226  $conn->multi( Redis::PIPELINE );
227  foreach ( $batchKeys as $key ) {
228  if ( $ttl ) {
229  $conn->setex( $key, $ttl, $this->serialize( $data[$key] ) );
230  } else {
231  $conn->set( $key, $this->serialize( $data[$key] ) );
232  }
233  }
234  $batchResult = $conn->exec();
235  if ( $batchResult === false ) {
236  $this->logRequest( $op, implode( ',', $batchKeys ), $server, true );
237  continue;
238  }
239  $result = $result && !in_array( false, $batchResult, true );
240  } catch ( RedisException $e ) {
241  $this->handleException( $conn, $e );
242  $result = false;
243  }
244 
245  $this->logRequest( $op, implode( ',', $batchKeys ), $server, $e );
246  }
247 
248  return $result;
249  }
250 
251  protected function doDeleteMulti( array $keys, $flags = 0 ) {
253  $conns = [];
254  $batches = [];
255  foreach ( $keys as $key ) {
256  $conn = $this->getConnection( $key );
257  if ( $conn ) {
258  $server = $conn->getServer();
259  $conns[$server] = $conn;
260  $batches[$server][] = $key;
261  }
262  }
263 
264  $result = true;
265  foreach ( $batches as $server => $batchKeys ) {
266  $conn = $conns[$server];
267 
268  $e = null;
269  try {
270  // Avoid delete() with array to reduce CPU hogging from a single request
271  $conn->multi( Redis::PIPELINE );
272  foreach ( $batchKeys as $key ) {
273  $conn->del( $key );
274  }
275  $batchResult = $conn->exec();
276  if ( $batchResult === false ) {
277  $this->logRequest( 'delete', implode( ',', $batchKeys ), $server, true );
278  continue;
279  }
280  // Note that redis does not return false if the key was not there
281  $result = $result && !in_array( false, $batchResult, true );
282  } catch ( RedisException $e ) {
283  $this->handleException( $conn, $e );
284  $result = false;
285  }
286 
287  $this->logRequest( 'delete', implode( ',', $batchKeys ), $server, $e );
288  }
289 
290  return $result;
291  }
292 
293  public function changeTTLMulti( array $keys, $exptime, $flags = 0 ) {
295  $conns = [];
296  $batches = [];
297  foreach ( $keys as $key ) {
298  $conn = $this->getConnection( $key );
299  if ( $conn ) {
300  $server = $conn->getServer();
301  $conns[$server] = $conn;
302  $batches[$server][] = $key;
303  }
304  }
305 
306  $relative = $this->isRelativeExpiration( $exptime );
307  $op = ( $exptime == self::TTL_INDEFINITE )
308  ? 'persist'
309  : ( $relative ? 'expire' : 'expireAt' );
310 
311  $result = true;
312  foreach ( $batches as $server => $batchKeys ) {
313  $conn = $conns[$server];
314 
315  $e = null;
316  try {
317  $conn->multi( Redis::PIPELINE );
318  foreach ( $batchKeys as $key ) {
319  if ( $exptime == self::TTL_INDEFINITE ) {
320  $conn->persist( $key );
321  } elseif ( $relative ) {
322  $conn->expire( $key, $this->getExpirationAsTTL( $exptime ) );
323  } else {
324  $conn->expireAt( $key, $this->getExpirationAsTimestamp( $exptime ) );
325  }
326  }
327  $batchResult = $conn->exec();
328  if ( $batchResult === false ) {
329  $this->logRequest( $op, implode( ',', $batchKeys ), $server, true );
330  continue;
331  }
332  $result = in_array( false, $batchResult, true ) ? false : $result;
333  } catch ( RedisException $e ) {
334  $this->handleException( $conn, $e );
335  $result = false;
336  }
337 
338  $this->logRequest( $op, implode( ',', $batchKeys ), $server, $e );
339  }
340 
341  return $result;
342  }
343 
344  protected function doAdd( $key, $value, $expiry = 0, $flags = 0 ) {
345  $conn = $this->getConnection( $key );
346  if ( !$conn ) {
347  return false;
348  }
349 
350  $ttl = $this->getExpirationAsTTL( $expiry );
351  try {
352  $result = $conn->set(
353  $key,
354  $this->serialize( $value ),
355  $ttl ? [ 'nx', 'ex' => $ttl ] : [ 'nx' ]
356  );
357  } catch ( RedisException $e ) {
358  $result = false;
359  $this->handleException( $conn, $e );
360  }
361 
362  $this->logRequest( 'add', $key, $conn->getServer(), $result );
363 
364  return $result;
365  }
366 
367  public function incr( $key, $value = 1, $flags = 0 ) {
368  $conn = $this->getConnection( $key );
369  if ( !$conn ) {
370  return false;
371  }
372 
373  try {
374  if ( !$conn->exists( $key ) ) {
375  return false;
376  }
377  // @FIXME: on races, the key may have a 0 TTL
378  $result = $conn->incrBy( $key, $value );
379  } catch ( RedisException $e ) {
380  $result = false;
381  $this->handleException( $conn, $e );
382  }
383 
384  $this->logRequest( 'incr', $key, $conn->getServer(), $result );
385 
386  return $result;
387  }
388 
389  public function decr( $key, $value = 1, $flags = 0 ) {
390  $conn = $this->getConnection( $key );
391  if ( !$conn ) {
392  return false;
393  }
394 
395  try {
396  if ( !$conn->exists( $key ) ) {
397  return false;
398  }
399  // @FIXME: on races, the key may have a 0 TTL
400  $result = $conn->decrBy( $key, $value );
401  } catch ( RedisException $e ) {
402  $result = false;
403  $this->handleException( $conn, $e );
404  }
405 
406  $this->logRequest( 'decr', $key, $conn->getServer(), $result );
407 
408  return $result;
409  }
410 
411  protected function doChangeTTL( $key, $exptime, $flags ) {
412  $conn = $this->getConnection( $key );
413  if ( !$conn ) {
414  return false;
415  }
416 
417  $relative = $this->isRelativeExpiration( $exptime );
418  try {
419  if ( $exptime == self::TTL_INDEFINITE ) {
420  $result = $conn->persist( $key );
421  $this->logRequest( 'persist', $key, $conn->getServer(), $result );
422  } elseif ( $relative ) {
423  $result = $conn->expire( $key, $this->getExpirationAsTTL( $exptime ) );
424  $this->logRequest( 'expire', $key, $conn->getServer(), $result );
425  } else {
426  $result = $conn->expireAt( $key, $this->getExpirationAsTimestamp( $exptime ) );
427  $this->logRequest( 'expireAt', $key, $conn->getServer(), $result );
428  }
429  } catch ( RedisException $e ) {
430  $result = false;
431  $this->handleException( $conn, $e );
432  }
433 
434  return $result;
435  }
436 
441  protected function getConnection( $key ) {
442  $candidates = array_keys( $this->serverTagMap );
443 
444  if ( count( $this->servers ) > 1 ) {
445  ArrayUtils::consistentHashSort( $candidates, $key, '/' );
446  if ( !$this->automaticFailover ) {
447  $candidates = array_slice( $candidates, 0, 1 );
448  }
449  }
450 
451  while ( ( $tag = array_shift( $candidates ) ) !== null ) {
452  $server = $this->serverTagMap[$tag];
453  $conn = $this->redisPool->getConnection( $server, $this->logger );
454  if ( !$conn ) {
455  continue;
456  }
457 
458  // If automatic failover is enabled, check that the server's link
459  // to its master (if any) is up -- but only if there are other
460  // viable candidates left to consider. Also, getMasterLinkStatus()
461  // does not work with twemproxy, though $candidates will be empty
462  // by now in such cases.
463  if ( $this->automaticFailover && $candidates ) {
464  try {
466  $info = $conn->info();
467  if ( ( $info['master_link_status'] ?? null ) === 'down' ) {
468  // If the master cannot be reached, fail-over to the next server.
469  // If masters are in data-center A, and replica DBs in data-center B,
470  // this helps avoid the case were fail-over happens in A but not
471  // to the corresponding server in B (e.g. read/write mismatch).
472  continue;
473  }
474  } catch ( RedisException $e ) {
475  // Server is not accepting commands
476  $this->redisPool->handleError( $conn, $e );
477  continue;
478  }
479  }
480 
481  return $conn;
482  }
483 
485 
486  return null;
487  }
488 
493  protected function logError( $msg ) {
494  $this->logger->error( "Redis error: $msg" );
495  }
496 
505  protected function handleException( RedisConnRef $conn, RedisException $e ) {
507  $this->redisPool->handleError( $conn, $e );
508  }
509 
517  public function logRequest( $op, $keys, $server, $e = null ) {
518  $this->debug( "$op($keys) on $server: " . ( $e ? "failure" : "success" ) );
519  }
520 }
array $serverTagMap
Map of (tag => server name)
changeTTLMulti(array $keys, $exptime, $flags=0)
doGet( $key, $flags=0, &$casToken=null)
processing should stop and the error should be shown to the user * false
Definition: hooks.txt:193
div flags Integer display flags(NO_ACTION_LINK, NO_EXTRA_USER_LINKS) 'LogException' returning false will NOT prevent logging $e
Definition: hooks.txt:2158
doAdd( $key, $value, $expiry=0, $flags=0)
$value
incr( $key, $value=1, $flags=0)
The index of the header message $result[1]=The index of the body text message $result[2 through n]=Parameters passed to body text message. Please note the header message cannot receive/use parameters. 'ImgAuthModifyHeaders':Executed just before a file is streamed to a user via img_auth.php, allowing headers to be modified beforehand. $title:LinkTarget object & $headers:HTTP headers(name=> value, names are case insensitive). Two headers get special handling:If-Modified-Since(value must be a valid HTTP date) and Range(must be of the form "bytes=(\*-\*)") will be honored when streaming the file. 'ImportHandleLogItemXMLTag':When parsing a XML tag in a log item. Return false to stop further processing of the tag $reader:XMLReader object $logInfo:Array of information 'ImportHandlePageXMLTag':When parsing a XML tag in a page. Return false to stop further processing of the tag $reader:XMLReader object & $pageInfo:Array of information 'ImportHandleRevisionXMLTag':When parsing a XML tag in a page revision. Return false to stop further processing of the tag $reader:XMLReader object $pageInfo:Array of page information $revisionInfo:Array of revision information 'ImportHandleToplevelXMLTag':When parsing a top level XML tag. Return false to stop further processing of the tag $reader:XMLReader object 'ImportHandleUnknownUser':When a user doesn 't exist locally, this hook is called to give extensions an opportunity to auto-create it. If the auto-creation is successful, return false. $name:User name 'ImportHandleUploadXMLTag':When parsing a XML tag in a file upload. Return false to stop further processing of the tag $reader:XMLReader object $revisionInfo:Array of information 'ImportLogInterwikiLink':Hook to change the interwiki link used in log entries and edit summaries for transwiki imports. & $fullInterwikiPrefix:Interwiki prefix, may contain colons. & $pageTitle:String that contains page title. 'ImportSources':Called when reading from the $wgImportSources configuration variable. Can be used to lazy-load the import sources list. & $importSources:The value of $wgImportSources. Modify as necessary. See the comment in DefaultSettings.php for the detail of how to structure this array. 'InfoAction':When building information to display on the action=info page. $context:IContextSource object & $pageInfo:Array of information 'InitializeArticleMaybeRedirect':MediaWiki check to see if title is a redirect. & $title:Title object for the current page & $request:WebRequest & $ignoreRedirect:boolean to skip redirect check & $target:Title/string of redirect target & $article:Article object 'InternalParseBeforeLinks':during Parser 's internalParse method before links but after nowiki/noinclude/includeonly/onlyinclude and other processings. & $parser:Parser object & $text:string containing partially parsed text & $stripState:Parser 's internal StripState object 'InternalParseBeforeSanitize':during Parser 's internalParse method just before the parser removes unwanted/dangerous HTML tags and after nowiki/noinclude/includeonly/onlyinclude and other processings. Ideal for syntax-extensions after template/parser function execution which respect nowiki and HTML-comments. & $parser:Parser object & $text:string containing partially parsed text & $stripState:Parser 's internal StripState object 'InterwikiLoadPrefix':When resolving if a given prefix is an interwiki or not. Return true without providing an interwiki to continue interwiki search. $prefix:interwiki prefix we are looking for. & $iwData:output array describing the interwiki with keys iw_url, iw_local, iw_trans and optionally iw_api and iw_wikiid. 'InvalidateEmailComplete':Called after a user 's email has been invalidated successfully. $user:user(object) whose email is being invalidated 'IRCLineURL':When constructing the URL to use in an IRC notification. Callee may modify $url and $query, URL will be constructed as $url . $query & $url:URL to index.php & $query:Query string $rc:RecentChange object that triggered url generation 'IsFileCacheable':Override the result of Article::isFileCacheable()(if true) & $article:article(object) being checked 'IsTrustedProxy':Override the result of IP::isTrustedProxy() & $ip:IP being check & $result:Change this value to override the result of IP::isTrustedProxy() 'IsUploadAllowedFromUrl':Override the result of UploadFromUrl::isAllowedUrl() $url:URL used to upload from & $allowed:Boolean indicating if uploading is allowed for given URL 'isValidEmailAddr':Override the result of Sanitizer::validateEmail(), for instance to return false if the domain name doesn 't match your organization. $addr:The e-mail address entered by the user & $result:Set this and return false to override the internal checks 'isValidPassword':Override the result of User::isValidPassword() $password:The password entered by the user & $result:Set this and return false to override the internal checks $user:User the password is being validated for 'Language::getMessagesFileName':$code:The language code or the language we 're looking for a messages file for & $file:The messages file path, you can override this to change the location. 'LanguageGetNamespaces':Provide custom ordering for namespaces or remove namespaces. Do not use this hook to add namespaces. Use CanonicalNamespaces for that. & $namespaces:Array of namespaces indexed by their numbers 'LanguageGetTranslatedLanguageNames':Provide translated language names. & $names:array of language code=> language name $code:language of the preferred translations 'LanguageLinks':Manipulate a page 's language links. This is called in various places to allow extensions to define the effective language links for a page. $title:The page 's Title. & $links:Array with elements of the form "language:title" in the order that they will be output. & $linkFlags:Associative array mapping prefixed links to arrays of flags. Currently unused, but planned to provide support for marking individual language links in the UI, e.g. for featured articles. 'LanguageSelector':Hook to change the language selector available on a page. $out:The output page. $cssClassName:CSS class name of the language selector. 'LinkBegin':DEPRECATED since 1.28! Use HtmlPageLinkRendererBegin instead. Used when generating internal and interwiki links in Linker::link(), before processing starts. Return false to skip default processing and return $ret. See documentation for Linker::link() for details on the expected meanings of parameters. $skin:the Skin object $target:the Title that the link is pointing to & $html:the contents that the< a > tag should have(raw HTML) $result
Definition: hooks.txt:1981
static singleton(array $options)
array $servers
List of server names.
logError( $msg)
Log a fatal error.
doDelete( $key, $flags=0)
doGetMulti(array $keys, $flags=0)
$params
decr( $key, $value=1, $flags=0)
logRequest( $op, $keys, $server, $e=null)
Send information about a single request to the debug log.
doSetMulti(array $data, $exptime=0, $flags=0)
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that When $user is not null
Definition: hooks.txt:773
RedisConnectionPool $redisPool
Redis-based caching module for redis server >= 2.6.12 and phpredis >= 2.2.4.
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
Storage medium specific cache for storing items (e.g.
getExpirationAsTimestamp( $exptime)
Convert an optionally relative timestamp to an absolute time.
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
doSet( $key, $value, $exptime=0, $flags=0)
doDeleteMulti(array $keys, $flags=0)
getExpirationAsTTL( $exptime)
Convert an optionally absolute expiry time to a relative time.
static consistentHashSort(&$array, $key, $separator="\0")
Sort the given array in a pseudo-random order which depends only on the given key and each element va...
Definition: ArrayUtils.php:49
setLastError( $err)
Set the "last error" registry.
Helper class to handle automatically marking connectons as reusable (via RAII pattern) ...
storage can be distributed across multiple servers
Definition: memcached.txt:33
doChangeTTL( $key, $exptime, $flags)
handleException(RedisConnRef $conn, RedisException $e)
The redis extension throws an exception in response to various read, write and protocol errors...
__construct( $params)
Construct a RedisBagOStuff object.