MediaWiki  master
RedisBagOStuff.php
Go to the documentation of this file.
1 <?php
35  protected $redisPool;
37  protected $servers;
39  protected $serverTagMap;
41  protected $automaticFailover;
42 
71  public function __construct( $params ) {
72  parent::__construct( $params );
73  $redisConf = [ 'serializer' => 'none' ]; // manage that in this class
74  foreach ( [ 'connectTimeout', 'persistent', 'password' ] as $opt ) {
75  if ( isset( $params[$opt] ) ) {
76  $redisConf[$opt] = $params[$opt];
77  }
78  }
79  $this->redisPool = RedisConnectionPool::singleton( $redisConf );
80 
81  $this->servers = $params['servers'];
82  foreach ( $this->servers as $key => $server ) {
83  $this->serverTagMap[is_int( $key ) ? $server : $key] = $server;
84  }
85 
86  $this->automaticFailover = $params['automaticFailover'] ?? true;
87 
88  $this->attrMap[self::ATTR_SYNCWRITES] = self::QOS_SYNCWRITES_NONE;
89  }
90 
91  protected function doGet( $key, $flags = 0, &$casToken = null ) {
92  $getToken = ( $casToken === self::PASS_BY_REF );
93  $casToken = null;
94 
95  $conn = $this->getConnection( $key );
96  if ( !$conn ) {
97  return false;
98  }
99 
100  $e = null;
101  try {
102  $blob = $conn->get( $key );
103  if ( $getToken && $blob !== false ) {
104  $casToken = $blob;
105  }
106  $result = $this->unserialize( $blob );
107  $valueSize = strlen( $blob );
108  } catch ( RedisException $e ) {
109  $result = false;
110  $valueSize = false;
111  $this->handleException( $conn, $e );
112  }
113 
114  $this->logRequest( 'get', $key, $conn->getServer(), $e );
115 
116  $this->updateOpStats( self::METRIC_OP_GET, [ $key => [ null, $valueSize ] ] );
117 
118  return $result;
119  }
120 
121  protected function doSet( $key, $value, $exptime = 0, $flags = 0 ) {
122  $conn = $this->getConnection( $key );
123  if ( !$conn ) {
124  return false;
125  }
126 
127  $ttl = $this->getExpirationAsTTL( $exptime );
128  $serialized = $this->getSerialized( $value, $key );
129  $valueSize = strlen( $serialized );
130 
131  $e = null;
132  try {
133  if ( $ttl ) {
134  $result = $conn->setex( $key, $ttl, $serialized );
135  } else {
136  $result = $conn->set( $key, $serialized );
137  }
138  } catch ( RedisException $e ) {
139  $result = false;
140  $this->handleException( $conn, $e );
141  }
142 
143  $this->logRequest( 'set', $key, $conn->getServer(), $e );
144 
145  $this->updateOpStats( self::METRIC_OP_SET, [ $key => [ $valueSize, null ] ] );
146 
147  return $result;
148  }
149 
150  protected function doDelete( $key, $flags = 0 ) {
151  $conn = $this->getConnection( $key );
152  if ( !$conn ) {
153  return false;
154  }
155 
156  $e = null;
157  try {
158  // Note that redis does not return false if the key was not there
159  $result = ( $conn->del( $key ) !== false );
160  } catch ( RedisException $e ) {
161  $result = false;
162  $this->handleException( $conn, $e );
163  }
164 
165  $this->logRequest( 'delete', $key, $conn->getServer(), $e );
166 
167  $this->updateOpStats( self::METRIC_OP_DELETE, [ $key ] );
168 
169  return $result;
170  }
171 
172  protected function doGetMulti( array $keys, $flags = 0 ) {
174  $conns = [];
175  $batches = [];
176  foreach ( $keys as $key ) {
177  $conn = $this->getConnection( $key );
178  if ( $conn ) {
179  $server = $conn->getServer();
180  $conns[$server] = $conn;
181  $batches[$server][] = $key;
182  }
183  }
184 
185  $blobsFound = [];
186  foreach ( $batches as $server => $batchKeys ) {
187  $conn = $conns[$server];
188 
189  $e = null;
190  try {
191  // Avoid mget() to reduce CPU hogging from a single request
192  $conn->multi( Redis::PIPELINE );
193  foreach ( $batchKeys as $key ) {
194  $conn->get( $key );
195  }
196  $batchResult = $conn->exec();
197  if ( $batchResult === false ) {
198  $this->logRequest( 'get', implode( ',', $batchKeys ), $server, true );
199  continue;
200  }
201 
202  foreach ( $batchResult as $i => $blob ) {
203  if ( $blob !== false ) {
204  $blobsFound[$batchKeys[$i]] = $blob;
205  }
206  }
207  } catch ( RedisException $e ) {
208  $this->handleException( $conn, $e );
209  }
210 
211  $this->logRequest( 'get', implode( ',', $batchKeys ), $server, $e );
212  }
213 
214  // Preserve the order of $keys
215  $result = [];
216  $valueSizesByKey = [];
217  foreach ( $keys as $key ) {
218  if ( array_key_exists( $key, $blobsFound ) ) {
219  $blob = $blobsFound[$key];
220  $value = $this->unserialize( $blob );
221  if ( $value !== false ) {
222  $result[$key] = $value;
223  }
224  $valueSize = strlen( $blob );
225  } else {
226  $valueSize = false;
227  }
228  $valueSizesByKey[$key] = [ null, $valueSize ];
229  }
230 
231  $this->updateOpStats( self::METRIC_OP_GET, $valueSizesByKey );
232 
233  return $result;
234  }
235 
236  protected function doSetMulti( array $data, $exptime = 0, $flags = 0 ) {
237  $result = true;
238 
240  $conns = [];
241  $batches = [];
242  foreach ( $data as $key => $value ) {
243  $conn = $this->getConnection( $key );
244  if ( $conn ) {
245  $server = $conn->getServer();
246  $conns[$server] = $conn;
247  $batches[$server][] = $key;
248  } else {
249  $result = false;
250  }
251  }
252 
253  $ttl = $this->getExpirationAsTTL( $exptime );
254  $op = $ttl ? 'setex' : 'set';
255 
256  $valueSizesByKey = [];
257  foreach ( $batches as $server => $batchKeys ) {
258  $conn = $conns[$server];
259 
260  $e = null;
261  try {
262  // Avoid mset() to reduce CPU hogging from a single request
263  $conn->multi( Redis::PIPELINE );
264  foreach ( $batchKeys as $key ) {
265  $serialized = $this->getSerialized( $data[$key], $key );
266  if ( $ttl ) {
267  $conn->setex( $key, $ttl, $serialized );
268  } else {
269  $conn->set( $key, $serialized );
270  }
271  $valueSizesByKey[$key] = [ strlen( $serialized ), null ];
272  }
273  $batchResult = $conn->exec();
274  if ( $batchResult === false ) {
275  $result = false;
276  $this->logRequest( $op, implode( ',', $batchKeys ), $server, true );
277  continue;
278  }
279 
280  $result = $result && !in_array( false, $batchResult, true );
281  } catch ( RedisException $e ) {
282  $this->handleException( $conn, $e );
283  $result = false;
284  }
285 
286  $this->logRequest( $op, implode( ',', $batchKeys ), $server, $e );
287  }
288 
289  $this->updateOpStats( self::METRIC_OP_SET, $valueSizesByKey );
290 
291  return $result;
292  }
293 
294  protected function doDeleteMulti( array $keys, $flags = 0 ) {
295  $result = true;
296 
298  $conns = [];
299  $batches = [];
300  foreach ( $keys as $key ) {
301  $conn = $this->getConnection( $key );
302  if ( $conn ) {
303  $server = $conn->getServer();
304  $conns[$server] = $conn;
305  $batches[$server][] = $key;
306  } else {
307  $result = false;
308  }
309  }
310 
311  foreach ( $batches as $server => $batchKeys ) {
312  $conn = $conns[$server];
313 
314  $e = null;
315  try {
316  // Avoid delete() with array to reduce CPU hogging from a single request
317  $conn->multi( Redis::PIPELINE );
318  foreach ( $batchKeys as $key ) {
319  $conn->del( $key );
320  }
321  $batchResult = $conn->exec();
322  if ( $batchResult === false ) {
323  $result = false;
324  $this->logRequest( 'delete', implode( ',', $batchKeys ), $server, true );
325  continue;
326  }
327  // Note that redis does not return false if the key was not there
328  $result = $result && !in_array( false, $batchResult, true );
329  } catch ( RedisException $e ) {
330  $this->handleException( $conn, $e );
331  $result = false;
332  }
333 
334  $this->logRequest( 'delete', implode( ',', $batchKeys ), $server, $e );
335  }
336 
337  $this->updateOpStats( self::METRIC_OP_DELETE, array_values( $keys ) );
338 
339  return $result;
340  }
341 
342  public function doChangeTTLMulti( array $keys, $exptime, $flags = 0 ) {
343  $result = true;
344 
346  $conns = [];
347  $batches = [];
348  foreach ( $keys as $key ) {
349  $conn = $this->getConnection( $key );
350  if ( $conn ) {
351  $server = $conn->getServer();
352  $conns[$server] = $conn;
353  $batches[$server][] = $key;
354  } else {
355  $result = false;
356  }
357  }
358 
359  $relative = $this->isRelativeExpiration( $exptime );
360  $op = ( $exptime == self::TTL_INDEFINITE )
361  ? 'persist'
362  : ( $relative ? 'expire' : 'expireAt' );
363 
364  foreach ( $batches as $server => $batchKeys ) {
365  $conn = $conns[$server];
366 
367  $e = null;
368  try {
369  $conn->multi( Redis::PIPELINE );
370  foreach ( $batchKeys as $key ) {
371  if ( $exptime == self::TTL_INDEFINITE ) {
372  $conn->persist( $key );
373  } elseif ( $relative ) {
374  $conn->expire( $key, $this->getExpirationAsTTL( $exptime ) );
375  } else {
376  $conn->expireAt( $key, $this->getExpirationAsTimestamp( $exptime ) );
377  }
378  }
379  $batchResult = $conn->exec();
380  if ( $batchResult === false ) {
381  $result = false;
382  $this->logRequest( $op, implode( ',', $batchKeys ), $server, true );
383  continue;
384  }
385  $result = in_array( false, $batchResult, true ) ? false : $result;
386  } catch ( RedisException $e ) {
387  $this->handleException( $conn, $e );
388  $result = false;
389  }
390 
391  $this->logRequest( $op, implode( ',', $batchKeys ), $server, $e );
392  }
393 
394  $this->updateOpStats( self::METRIC_OP_CHANGE_TTL, array_values( $keys ) );
395 
396  return $result;
397  }
398 
399  protected function doAdd( $key, $value, $expiry = 0, $flags = 0 ) {
400  $conn = $this->getConnection( $key );
401  if ( !$conn ) {
402  return false;
403  }
404 
405  $ttl = $this->getExpirationAsTTL( $expiry );
406  $serialized = $this->getSerialized( $value, $key );
407  $valueSize = strlen( $serialized );
408 
409  try {
410  $result = $conn->set(
411  $key,
412  $serialized,
413  $ttl ? [ 'nx', 'ex' => $ttl ] : [ 'nx' ]
414  );
415  } catch ( RedisException $e ) {
416  $result = false;
417  $this->handleException( $conn, $e );
418  }
419 
420  $this->logRequest( 'add', $key, $conn->getServer(), $result );
421 
422  $this->updateOpStats( self::METRIC_OP_ADD, [ $key => [ $valueSize, null ] ] );
423 
424  return $result;
425  }
426 
427  public function incr( $key, $value = 1, $flags = 0 ) {
428  $conn = $this->getConnection( $key );
429  if ( !$conn ) {
430  return false;
431  }
432 
433  try {
434  if ( !$conn->exists( $key ) ) {
435  return false;
436  }
437  // @FIXME: on races, the key may have a 0 TTL
438  $result = $conn->incrBy( $key, $value );
439  } catch ( RedisException $e ) {
440  $result = false;
441  $this->handleException( $conn, $e );
442  }
443 
444  $this->logRequest( 'incr', $key, $conn->getServer(), $result );
445 
446  $this->updateOpStats( self::METRIC_OP_INCR, [ $key ] );
447 
448  return $result;
449  }
450 
451  public function decr( $key, $value = 1, $flags = 0 ) {
452  $conn = $this->getConnection( $key );
453  if ( !$conn ) {
454  return false;
455  }
456 
457  try {
458  if ( !$conn->exists( $key ) ) {
459  return false;
460  }
461  // @FIXME: on races, the key may have a 0 TTL
462  $result = $conn->decrBy( $key, $value );
463  } catch ( RedisException $e ) {
464  $result = false;
465  $this->handleException( $conn, $e );
466  }
467 
468  $this->logRequest( 'decr', $key, $conn->getServer(), $result );
469 
470  $this->updateOpStats( self::METRIC_OP_DECR, [ $key ] );
471 
472  return $result;
473  }
474 
475  protected function doChangeTTL( $key, $exptime, $flags ) {
476  $conn = $this->getConnection( $key );
477  if ( !$conn ) {
478  return false;
479  }
480 
481  $relative = $this->isRelativeExpiration( $exptime );
482  try {
483  if ( $exptime == self::TTL_INDEFINITE ) {
484  $result = $conn->persist( $key );
485  $this->logRequest( 'persist', $key, $conn->getServer(), $result );
486  } elseif ( $relative ) {
487  $result = $conn->expire( $key, $this->getExpirationAsTTL( $exptime ) );
488  $this->logRequest( 'expire', $key, $conn->getServer(), $result );
489  } else {
490  $result = $conn->expireAt( $key, $this->getExpirationAsTimestamp( $exptime ) );
491  $this->logRequest( 'expireAt', $key, $conn->getServer(), $result );
492  }
493  } catch ( RedisException $e ) {
494  $result = false;
495  $this->handleException( $conn, $e );
496  }
497 
498  $this->updateOpStats( self::METRIC_OP_CHANGE_TTL, [ $key ] );
499 
500  return $result;
501  }
502 
507  protected function getConnection( $key ) {
508  $candidates = array_keys( $this->serverTagMap );
509 
510  if ( count( $this->servers ) > 1 ) {
511  ArrayUtils::consistentHashSort( $candidates, $key, '/' );
512  if ( !$this->automaticFailover ) {
513  $candidates = array_slice( $candidates, 0, 1 );
514  }
515  }
516 
517  while ( ( $tag = array_shift( $candidates ) ) !== null ) {
518  $server = $this->serverTagMap[$tag];
519  $conn = $this->redisPool->getConnection( $server, $this->logger );
520  if ( !$conn ) {
521  continue;
522  }
523 
524  // If automatic failover is enabled, check that the server's link
525  // to its master (if any) is up -- but only if there are other
526  // viable candidates left to consider. Also, getMasterLinkStatus()
527  // does not work with twemproxy, though $candidates will be empty
528  // by now in such cases.
529  if ( $this->automaticFailover && $candidates ) {
530  try {
532  $info = $conn->info();
533  if ( ( $info['master_link_status'] ?? null ) === 'down' ) {
534  // If the master cannot be reached, fail-over to the next server.
535  // If masters are in data-center A, and replica DBs in data-center B,
536  // this helps avoid the case were fail-over happens in A but not
537  // to the corresponding server in B (e.g. read/write mismatch).
538  continue;
539  }
540  } catch ( RedisException $e ) {
541  // Server is not accepting commands
542  $this->redisPool->handleError( $conn, $e );
543  continue;
544  }
545  }
546 
547  return $conn;
548  }
549 
550  $this->setLastError( BagOStuff::ERR_UNREACHABLE );
551 
552  return null;
553  }
554 
559  protected function logError( $msg ) {
560  $this->logger->error( "Redis error: $msg" );
561  }
562 
571  protected function handleException( RedisConnRef $conn, RedisException $e ) {
572  $this->setLastError( BagOStuff::ERR_UNEXPECTED );
573  $this->redisPool->handleError( $conn, $e );
574  }
575 
583  public function logRequest( $op, $keys, $server, $e = null ) {
584  $this->debug( "$op($keys) on $server: " . ( $e ? "failure" : "success" ) );
585  }
586 
587  public function makeKeyInternal( $keyspace, $components ) {
588  return $this->genericKeyFromComponents( $keyspace, ...$components );
589  }
590 
591  protected function convertGenericKey( $key ) {
592  return $key; // short-circuit; already uses "generic" keys
593  }
594 }
RedisBagOStuff\doGet
doGet( $key, $flags=0, &$casToken=null)
Definition: RedisBagOStuff.php:91
MediumSpecificBagOStuff\setLastError
setLastError( $err)
Set the "last error" registry.
Definition: MediumSpecificBagOStuff.php:830
RedisBagOStuff\handleException
handleException(RedisConnRef $conn, RedisException $e)
The redis extension throws an exception in response to various read, write and protocol errors.
Definition: RedisBagOStuff.php:571
ArrayUtils\consistentHashSort
static consistentHashSort(&$array, $key, $separator="\000")
Sort the given array in a pseudo-random order which depends only on the given key and each element va...
Definition: ArrayUtils.php:49
RedisBagOStuff
Redis-based caching module for redis server >= 2.6.12 and phpredis >= 2.2.4.
Definition: RedisBagOStuff.php:33
RedisConnectionPool\singleton
static singleton(array $options)
Definition: RedisConnectionPool.php:149
RedisBagOStuff\convertGenericKey
convertGenericKey( $key)
Convert a "generic" reversible cache key into one for this cache.
Definition: RedisBagOStuff.php:591
MediumSpecificBagOStuff\debug
debug( $text)
Definition: MediumSpecificBagOStuff.php:1180
RedisBagOStuff\makeKeyInternal
makeKeyInternal( $keyspace, $components)
Make a cache key for the given keyspace and components.
Definition: RedisBagOStuff.php:587
RedisBagOStuff\doDelete
doDelete( $key, $flags=0)
Delete an item.
Definition: RedisBagOStuff.php:150
$serialized
foreach( $res as $row) $serialized
Definition: testCompression.php:88
RedisBagOStuff\$redisPool
RedisConnectionPool $redisPool
Definition: RedisBagOStuff.php:35
RedisBagOStuff\doAdd
doAdd( $key, $value, $expiry=0, $flags=0)
Insert an item if it does not already exist.
Definition: RedisBagOStuff.php:399
BagOStuff\genericKeyFromComponents
genericKeyFromComponents(... $components)
At a minimum, there must be a keyspace and collection name component.
Definition: BagOStuff.php:676
RedisBagOStuff\$automaticFailover
bool $automaticFailover
Definition: RedisBagOStuff.php:41
RedisBagOStuff\$servers
array $servers
List of server names.
Definition: RedisBagOStuff.php:37
RedisBagOStuff\decr
decr( $key, $value=1, $flags=0)
Decrease stored value of $key by $value while preserving its TTL.
Definition: RedisBagOStuff.php:451
RedisBagOStuff\doGetMulti
doGetMulti(array $keys, $flags=0)
Get an associative array containing the item for each of the keys that have items.
Definition: RedisBagOStuff.php:172
RedisBagOStuff\getConnection
getConnection( $key)
Definition: RedisBagOStuff.php:507
RedisBagOStuff\logError
logError( $msg)
Log a fatal error.
Definition: RedisBagOStuff.php:559
RedisBagOStuff\doDeleteMulti
doDeleteMulti(array $keys, $flags=0)
Definition: RedisBagOStuff.php:294
MediumSpecificBagOStuff\getExpirationAsTimestamp
getExpirationAsTimestamp( $exptime)
Convert an optionally relative timestamp to an absolute time.
Definition: MediumSpecificBagOStuff.php:913
$blob
$blob
Definition: testCompression.php:70
MediumSpecificBagOStuff\getExpirationAsTTL
getExpirationAsTTL( $exptime)
Convert an optionally absolute expiry time to a relative time.
Definition: MediumSpecificBagOStuff.php:937
MediumSpecificBagOStuff
Storage medium specific cache for storing items (e.g.
Definition: MediumSpecificBagOStuff.php:34
MediumSpecificBagOStuff\updateOpStats
updateOpStats(string $op, array $keyInfo)
Definition: MediumSpecificBagOStuff.php:1191
RedisBagOStuff\doSet
doSet( $key, $value, $exptime=0, $flags=0)
Set an item.
Definition: RedisBagOStuff.php:121
RedisBagOStuff\logRequest
logRequest( $op, $keys, $server, $e=null)
Send information about a single request to the debug log.
Definition: RedisBagOStuff.php:583
RedisBagOStuff\incr
incr( $key, $value=1, $flags=0)
Increase stored value of $key by $value while preserving its TTL.
Definition: RedisBagOStuff.php:427
RedisConnectionPool
Helper class to manage Redis connections.
Definition: RedisConnectionPool.php:41
RedisBagOStuff\doChangeTTLMulti
doChangeTTLMulti(array $keys, $exptime, $flags=0)
Definition: RedisBagOStuff.php:342
RedisBagOStuff\$serverTagMap
array $serverTagMap
Map of (tag => server name)
Definition: RedisBagOStuff.php:39
MediumSpecificBagOStuff\getSerialized
getSerialized( $value, $key)
Get the serialized form a value, using any applicable prepared value.
Definition: MediumSpecificBagOStuff.php:1026
MediumSpecificBagOStuff\isRelativeExpiration
isRelativeExpiration( $exptime)
Definition: MediumSpecificBagOStuff.php:896
RedisBagOStuff\doSetMulti
doSetMulti(array $data, $exptime=0, $flags=0)
Definition: RedisBagOStuff.php:236
MediumSpecificBagOStuff\unserialize
unserialize( $value)
Definition: MediumSpecificBagOStuff.php:1173
RedisBagOStuff\__construct
__construct( $params)
Construct a RedisBagOStuff object.
Definition: RedisBagOStuff.php:71
RedisConnRef
Helper class to handle automatically marking connectons as reusable (via RAII pattern)
Definition: RedisConnRef.php:31
$keys
$keys
Definition: testCompression.php:72
BagOStuff\$keyspace
string $keyspace
Default keyspace; used by makeKey()
Definition: BagOStuff.php:103
RedisBagOStuff\doChangeTTL
doChangeTTL( $key, $exptime, $flags)
Definition: RedisBagOStuff.php:475