MediaWiki  master
RedisBagOStuff.php
Go to the documentation of this file.
1 <?php
35  protected $redisPool;
37  protected $servers;
39  protected $serverTagMap;
41  protected $automaticFailover;
42 
71  public function __construct( $params ) {
72  parent::__construct( $params );
73  $redisConf = [ 'serializer' => 'none' ]; // manage that in this class
74  foreach ( [ 'connectTimeout', 'persistent', 'password' ] as $opt ) {
75  if ( isset( $params[$opt] ) ) {
76  $redisConf[$opt] = $params[$opt];
77  }
78  }
79  $this->redisPool = RedisConnectionPool::singleton( $redisConf );
80 
81  $this->servers = $params['servers'];
82  foreach ( $this->servers as $key => $server ) {
83  $this->serverTagMap[is_int( $key ) ? $server : $key] = $server;
84  }
85 
86  $this->automaticFailover = $params['automaticFailover'] ?? true;
87 
88  // ...and uses rdb snapshots (redis.conf default)
90  }
91 
92  protected function doGet( $key, $flags = 0, &$casToken = null ) {
93  $getToken = ( $casToken === self::PASS_BY_REF );
94  $casToken = null;
95 
96  $conn = $this->getConnection( $key );
97  if ( !$conn ) {
98  return false;
99  }
100 
101  $e = null;
102  try {
103  $blob = $conn->get( $key );
104  if ( $blob !== false ) {
105  $value = $this->unserialize( $blob );
106  $valueSize = strlen( $blob );
107  } else {
108  $value = false;
109  $valueSize = false;
110  }
111  if ( $getToken && $value !== false ) {
112  $casToken = $blob;
113  }
114  } catch ( RedisException $e ) {
115  $value = false;
116  $valueSize = false;
117  $this->handleException( $conn, $e );
118  }
119 
120  $this->logRequest( 'get', $key, $conn->getServer(), $e );
121 
122  $this->updateOpStats( self::METRIC_OP_GET, [ $key => [ 0, $valueSize ] ] );
123 
124  return $value;
125  }
126 
127  protected function doSet( $key, $value, $exptime = 0, $flags = 0 ) {
128  $conn = $this->getConnection( $key );
129  if ( !$conn ) {
130  return false;
131  }
132 
133  $ttl = $this->getExpirationAsTTL( $exptime );
134  $serialized = $this->getSerialized( $value, $key );
135  $valueSize = strlen( $serialized );
136 
137  $e = null;
138  try {
139  if ( $ttl ) {
140  $result = $conn->setex( $key, $ttl, $serialized );
141  } else {
142  $result = $conn->set( $key, $serialized );
143  }
144  } catch ( RedisException $e ) {
145  $result = false;
146  $this->handleException( $conn, $e );
147  }
148 
149  $this->logRequest( 'set', $key, $conn->getServer(), $e );
150 
151  $this->updateOpStats( self::METRIC_OP_SET, [ $key => [ $valueSize, 0 ] ] );
152 
153  return $result;
154  }
155 
156  protected function doDelete( $key, $flags = 0 ) {
157  $conn = $this->getConnection( $key );
158  if ( !$conn ) {
159  return false;
160  }
161 
162  $e = null;
163  try {
164  // Note that redis does not return false if the key was not there
165  $result = ( $conn->del( $key ) !== false );
166  } catch ( RedisException $e ) {
167  $result = false;
168  $this->handleException( $conn, $e );
169  }
170 
171  $this->logRequest( 'delete', $key, $conn->getServer(), $e );
172 
173  $this->updateOpStats( self::METRIC_OP_DELETE, [ $key ] );
174 
175  return $result;
176  }
177 
178  protected function doGetMulti( array $keys, $flags = 0 ) {
180  $conns = [];
181  $batches = [];
182  foreach ( $keys as $key ) {
183  $conn = $this->getConnection( $key );
184  if ( $conn ) {
185  $server = $conn->getServer();
186  $conns[$server] = $conn;
187  $batches[$server][] = $key;
188  }
189  }
190 
191  $blobsFound = [];
192  foreach ( $batches as $server => $batchKeys ) {
193  $conn = $conns[$server];
194 
195  $e = null;
196  try {
197  // Avoid mget() to reduce CPU hogging from a single request
198  $conn->multi( Redis::PIPELINE );
199  foreach ( $batchKeys as $key ) {
200  $conn->get( $key );
201  }
202  $batchResult = $conn->exec();
203  if ( $batchResult === false ) {
204  $this->logRequest( 'get', implode( ',', $batchKeys ), $server, true );
205  continue;
206  }
207 
208  foreach ( $batchResult as $i => $blob ) {
209  if ( $blob !== false ) {
210  $blobsFound[$batchKeys[$i]] = $blob;
211  }
212  }
213  } catch ( RedisException $e ) {
214  $this->handleException( $conn, $e );
215  }
216 
217  $this->logRequest( 'get', implode( ',', $batchKeys ), $server, $e );
218  }
219 
220  // Preserve the order of $keys
221  $result = [];
222  $valueSizesByKey = [];
223  foreach ( $keys as $key ) {
224  if ( array_key_exists( $key, $blobsFound ) ) {
225  $blob = $blobsFound[$key];
226  $value = $this->unserialize( $blob );
227  if ( $value !== false ) {
228  $result[$key] = $value;
229  }
230  $valueSize = strlen( $blob );
231  } else {
232  $valueSize = false;
233  }
234  $valueSizesByKey[$key] = [ 0, $valueSize ];
235  }
236 
237  $this->updateOpStats( self::METRIC_OP_GET, $valueSizesByKey );
238 
239  return $result;
240  }
241 
242  protected function doSetMulti( array $data, $exptime = 0, $flags = 0 ) {
243  $result = true;
244 
246  $conns = [];
247  $batches = [];
248  foreach ( $data as $key => $value ) {
249  $conn = $this->getConnection( $key );
250  if ( $conn ) {
251  $server = $conn->getServer();
252  $conns[$server] = $conn;
253  $batches[$server][] = $key;
254  } else {
255  $result = false;
256  }
257  }
258 
259  $ttl = $this->getExpirationAsTTL( $exptime );
260  $op = $ttl ? 'setex' : 'set';
261 
262  $valueSizesByKey = [];
263  foreach ( $batches as $server => $batchKeys ) {
264  $conn = $conns[$server];
265 
266  $e = null;
267  try {
268  // Avoid mset() to reduce CPU hogging from a single request
269  $conn->multi( Redis::PIPELINE );
270  foreach ( $batchKeys as $key ) {
271  $serialized = $this->getSerialized( $data[$key], $key );
272  if ( $ttl ) {
273  $conn->setex( $key, $ttl, $serialized );
274  } else {
275  $conn->set( $key, $serialized );
276  }
277  $valueSizesByKey[$key] = [ strlen( $serialized ), 0 ];
278  }
279  $batchResult = $conn->exec();
280  if ( $batchResult === false ) {
281  $result = false;
282  $this->logRequest( $op, implode( ',', $batchKeys ), $server, true );
283  continue;
284  }
285 
286  $result = $result && !in_array( false, $batchResult, true );
287  } catch ( RedisException $e ) {
288  $this->handleException( $conn, $e );
289  $result = false;
290  }
291 
292  $this->logRequest( $op, implode( ',', $batchKeys ), $server, $e );
293  }
294 
295  $this->updateOpStats( self::METRIC_OP_SET, $valueSizesByKey );
296 
297  return $result;
298  }
299 
300  protected function doDeleteMulti( array $keys, $flags = 0 ) {
301  $result = true;
302 
304  $conns = [];
305  $batches = [];
306  foreach ( $keys as $key ) {
307  $conn = $this->getConnection( $key );
308  if ( $conn ) {
309  $server = $conn->getServer();
310  $conns[$server] = $conn;
311  $batches[$server][] = $key;
312  } else {
313  $result = false;
314  }
315  }
316 
317  foreach ( $batches as $server => $batchKeys ) {
318  $conn = $conns[$server];
319 
320  $e = null;
321  try {
322  // Avoid delete() with array to reduce CPU hogging from a single request
323  $conn->multi( Redis::PIPELINE );
324  foreach ( $batchKeys as $key ) {
325  $conn->del( $key );
326  }
327  $batchResult = $conn->exec();
328  if ( $batchResult === false ) {
329  $result = false;
330  $this->logRequest( 'delete', implode( ',', $batchKeys ), $server, true );
331  continue;
332  }
333  // Note that redis does not return false if the key was not there
334  $result = $result && !in_array( false, $batchResult, true );
335  } catch ( RedisException $e ) {
336  $this->handleException( $conn, $e );
337  $result = false;
338  }
339 
340  $this->logRequest( 'delete', implode( ',', $batchKeys ), $server, $e );
341  }
342 
343  $this->updateOpStats( self::METRIC_OP_DELETE, array_values( $keys ) );
344 
345  return $result;
346  }
347 
348  public function doChangeTTLMulti( array $keys, $exptime, $flags = 0 ) {
349  $result = true;
350 
352  $conns = [];
353  $batches = [];
354  foreach ( $keys as $key ) {
355  $conn = $this->getConnection( $key );
356  if ( $conn ) {
357  $server = $conn->getServer();
358  $conns[$server] = $conn;
359  $batches[$server][] = $key;
360  } else {
361  $result = false;
362  }
363  }
364 
365  $relative = $this->isRelativeExpiration( $exptime );
366  $op = ( $exptime == self::TTL_INDEFINITE )
367  ? 'persist'
368  : ( $relative ? 'expire' : 'expireAt' );
369 
370  foreach ( $batches as $server => $batchKeys ) {
371  $conn = $conns[$server];
372 
373  $e = null;
374  try {
375  $conn->multi( Redis::PIPELINE );
376  foreach ( $batchKeys as $key ) {
377  if ( $exptime == self::TTL_INDEFINITE ) {
378  $conn->persist( $key );
379  } elseif ( $relative ) {
380  $conn->expire( $key, $this->getExpirationAsTTL( $exptime ) );
381  } else {
382  $conn->expireAt( $key, $this->getExpirationAsTimestamp( $exptime ) );
383  }
384  }
385  $batchResult = $conn->exec();
386  if ( $batchResult === false ) {
387  $result = false;
388  $this->logRequest( $op, implode( ',', $batchKeys ), $server, true );
389  continue;
390  }
391  $result = in_array( false, $batchResult, true ) ? false : $result;
392  } catch ( RedisException $e ) {
393  $this->handleException( $conn, $e );
394  $result = false;
395  }
396 
397  $this->logRequest( $op, implode( ',', $batchKeys ), $server, $e );
398  }
399 
400  $this->updateOpStats( self::METRIC_OP_CHANGE_TTL, array_values( $keys ) );
401 
402  return $result;
403  }
404 
405  protected function doAdd( $key, $value, $exptime = 0, $flags = 0 ) {
406  $conn = $this->getConnection( $key );
407  if ( !$conn ) {
408  return false;
409  }
410 
411  $ttl = $this->getExpirationAsTTL( $exptime );
412  $serialized = $this->getSerialized( $value, $key );
413  $valueSize = strlen( $serialized );
414 
415  try {
416  $result = $conn->set(
417  $key,
418  $serialized,
419  $ttl ? [ 'nx', 'ex' => $ttl ] : [ 'nx' ]
420  );
421  } catch ( RedisException $e ) {
422  $result = false;
423  $this->handleException( $conn, $e );
424  }
425 
426  $this->logRequest( 'add', $key, $conn->getServer(), $result );
427 
428  $this->updateOpStats( self::METRIC_OP_ADD, [ $key => [ $valueSize, 0 ] ] );
429 
430  return $result;
431  }
432 
433  public function incr( $key, $value = 1, $flags = 0 ) {
434  $conn = $this->getConnection( $key );
435  if ( !$conn ) {
436  return false;
437  }
438 
439  try {
440  if ( !$conn->exists( $key ) ) {
441  return false;
442  }
443  // @FIXME: on races, the key may have a 0 TTL
444  $result = $conn->incrBy( $key, $value );
445  } catch ( RedisException $e ) {
446  $result = false;
447  $this->handleException( $conn, $e );
448  }
449 
450  $this->logRequest( 'incr', $key, $conn->getServer(), $result );
451 
452  $this->updateOpStats( self::METRIC_OP_INCR, [ $key ] );
453 
454  return $result;
455  }
456 
457  public function decr( $key, $value = 1, $flags = 0 ) {
458  $conn = $this->getConnection( $key );
459  if ( !$conn ) {
460  return false;
461  }
462 
463  try {
464  if ( !$conn->exists( $key ) ) {
465  return false;
466  }
467  // @FIXME: on races, the key may have a 0 TTL
468  $result = $conn->decrBy( $key, $value );
469  } catch ( RedisException $e ) {
470  $result = false;
471  $this->handleException( $conn, $e );
472  }
473 
474  $this->logRequest( 'decr', $key, $conn->getServer(), $result );
475 
476  $this->updateOpStats( self::METRIC_OP_DECR, [ $key ] );
477 
478  return $result;
479  }
480 
481  protected function doIncrWithInit( $key, $exptime, $step, $init, $flags ) {
482  $conn = $this->getConnection( $key );
483  if ( !$conn ) {
484  return false;
485  }
486 
487  $ttl = $this->getExpirationAsTTL( $exptime );
488 
489  try {
490  if ( $init === $step && $exptime == self::TTL_INDEFINITE ) {
491  $newValue = $conn->incrBy( $key, $step );
492  } else {
493  $conn->multi( Redis::PIPELINE );
494  $conn->set(
495  $key,
496  (string)( $init - $step ),
497  $ttl ? [ 'nx', 'ex' => $ttl ] : [ 'nx' ]
498  );
499  $conn->incrBy( $key, $step );
500  $batchResult = $conn->exec();
501  $newValue = ( $batchResult === false ) ? false : $batchResult[1];
502  $this->logRequest( 'incrWithInit', $key, $conn->getServer(), $newValue === false );
503  }
504  } catch ( RedisException $e ) {
505  $newValue = false;
506  $this->handleException( $conn, $e );
507  }
508 
509  return $newValue;
510  }
511 
512  protected function doChangeTTL( $key, $exptime, $flags ) {
513  $conn = $this->getConnection( $key );
514  if ( !$conn ) {
515  return false;
516  }
517 
518  $relative = $this->isRelativeExpiration( $exptime );
519  try {
520  if ( $exptime == self::TTL_INDEFINITE ) {
521  $result = $conn->persist( $key );
522  $this->logRequest( 'persist', $key, $conn->getServer(), $result );
523  } elseif ( $relative ) {
524  $result = $conn->expire( $key, $this->getExpirationAsTTL( $exptime ) );
525  $this->logRequest( 'expire', $key, $conn->getServer(), $result );
526  } else {
527  $result = $conn->expireAt( $key, $this->getExpirationAsTimestamp( $exptime ) );
528  $this->logRequest( 'expireAt', $key, $conn->getServer(), $result );
529  }
530  } catch ( RedisException $e ) {
531  $result = false;
532  $this->handleException( $conn, $e );
533  }
534 
535  $this->updateOpStats( self::METRIC_OP_CHANGE_TTL, [ $key ] );
536 
537  return $result;
538  }
539 
544  protected function getConnection( $key ) {
545  $candidates = array_keys( $this->serverTagMap );
546 
547  if ( count( $this->servers ) > 1 ) {
548  ArrayUtils::consistentHashSort( $candidates, $key, '/' );
549  if ( !$this->automaticFailover ) {
550  $candidates = array_slice( $candidates, 0, 1 );
551  }
552  }
553 
554  while ( ( $tag = array_shift( $candidates ) ) !== null ) {
555  $server = $this->serverTagMap[$tag];
556  $conn = $this->redisPool->getConnection( $server, $this->logger );
557  if ( !$conn ) {
558  continue;
559  }
560 
561  // If automatic failover is enabled, check that the server's link
562  // to its master (if any) is up -- but only if there are other
563  // viable candidates left to consider. Also, getMasterLinkStatus()
564  // does not work with twemproxy, though $candidates will be empty
565  // by now in such cases.
566  if ( $this->automaticFailover && $candidates ) {
567  try {
569  $info = $conn->info();
570  if ( ( $info['master_link_status'] ?? null ) === 'down' ) {
571  // If the master cannot be reached, fail-over to the next server.
572  // If masters are in data-center A, and replica DBs in data-center B,
573  // this helps avoid the case were fail-over happens in A but not
574  // to the corresponding server in B (e.g. read/write mismatch).
575  continue;
576  }
577  } catch ( RedisException $e ) {
578  // Server is not accepting commands
579  $this->redisPool->handleError( $conn, $e );
580  continue;
581  }
582  }
583 
584  return $conn;
585  }
586 
588 
589  return null;
590  }
591 
596  protected function logError( $msg ) {
597  $this->logger->error( "Redis error: $msg" );
598  }
599 
608  protected function handleException( RedisConnRef $conn, RedisException $e ) {
610  $this->redisPool->handleError( $conn, $e );
611  }
612 
620  public function logRequest( $op, $keys, $server, $e = null ) {
621  $this->debug( "$op($keys) on $server: " . ( $e ? "failure" : "success" ) );
622  }
623 
624  public function makeKeyInternal( $keyspace, $components ) {
625  return $this->genericKeyFromComponents( $keyspace, ...$components );
626  }
627 
628  protected function convertGenericKey( $key ) {
629  // short-circuit; already uses "generic" keys
630  return $key;
631  }
632 }
static consistentHashSort(&$array, $key, $separator="\000")
Sort the given array in a pseudo-random order which depends only on the given key and each element va...
Definition: ArrayUtils.php:49
setLastError( $error)
Set the "last error" registry due to a problem encountered during an attempted operation.
Definition: BagOStuff.php:529
genericKeyFromComponents(... $components)
At a minimum, there must be a keyspace and collection name component.
Definition: BagOStuff.php:703
string $keyspace
Default keyspace; used by makeKey()
Definition: BagOStuff.php:101
Storage medium specific cache for storing items (e.g.
const PASS_BY_REF
Idiom for doGet() to return extra information by reference.
getExpirationAsTimestamp( $exptime)
Convert an optionally relative timestamp to an absolute time.
getSerialized( $value, $key)
Get the serialized form a value, logging a warning if it involves custom classes.
updateOpStats(string $op, array $keyInfo)
getExpirationAsTTL( $exptime)
Convert an optionally absolute expiry time to a relative time.
Redis-based caching module for redis server >= 2.6.12 and phpredis >= 2.2.4.
RedisConnectionPool $redisPool
incr( $key, $value=1, $flags=0)
Increase stored value of $key by $value while preserving its TTL.
handleException(RedisConnRef $conn, RedisException $e)
The redis extension throws an exception in response to various read, write and protocol errors.
convertGenericKey( $key)
Convert a "generic" reversible cache key into one for this cache.
array $servers
List of server names.
doSetMulti(array $data, $exptime=0, $flags=0)
makeKeyInternal( $keyspace, $components)
Make a cache key for the given keyspace and components.
doChangeTTL( $key, $exptime, $flags)
logRequest( $op, $keys, $server, $e=null)
Send information about a single request to the debug log.
doIncrWithInit( $key, $exptime, $step, $init, $flags)
doDelete( $key, $flags=0)
Delete an item.
logError( $msg)
Log a fatal error.
__construct( $params)
Construct a RedisBagOStuff object.
doGet( $key, $flags=0, &$casToken=null)
Get an item.
array $serverTagMap
Map of (tag => server name)
doGetMulti(array $keys, $flags=0)
Get an associative array containing the item for each of the keys that have items.
doChangeTTLMulti(array $keys, $exptime, $flags=0)
doAdd( $key, $value, $exptime=0, $flags=0)
Insert an item if it does not already exist.
doSet( $key, $value, $exptime=0, $flags=0)
Set an item.
doDeleteMulti(array $keys, $flags=0)
decr( $key, $value=1, $flags=0)
Decrease stored value of $key by $value while preserving its TTL.
Helper class to handle automatically marking connections as reusable (via RAII pattern)
static singleton(array $options)
const ERR_UNREACHABLE
Storage medium could not be reached to establish a connection.
const ATTR_DURABILITY
Durability of writes; see QOS_DURABILITY_* (higher means stronger)
const ERR_UNEXPECTED
Storage medium operation failed due to usage limitations or an I/O error.
const QOS_DURABILITY_DISK
Data is saved to disk and writes do not usually block on fsync()
foreach( $res as $row) $serialized