MediaWiki  master
RedisBagOStuff.php
Go to the documentation of this file.
1 <?php
35  protected $redisPool;
37  protected $servers;
39  protected $serverTagMap;
41  protected $automaticFailover;
42 
71  public function __construct( $params ) {
72  parent::__construct( $params );
73  $redisConf = [ 'serializer' => 'none' ]; // manage that in this class
74  foreach ( [ 'connectTimeout', 'persistent', 'password' ] as $opt ) {
75  if ( isset( $params[$opt] ) ) {
76  $redisConf[$opt] = $params[$opt];
77  }
78  }
79  $this->redisPool = RedisConnectionPool::singleton( $redisConf );
80 
81  $this->servers = $params['servers'];
82  foreach ( $this->servers as $key => $server ) {
83  $this->serverTagMap[is_int( $key ) ? $server : $key] = $server;
84  }
85 
86  $this->automaticFailover = $params['automaticFailover'] ?? true;
87 
88  $this->attrMap[self::ATTR_SYNCWRITES] = self::QOS_SYNCWRITES_NONE;
89  }
90 
91  protected function doGet( $key, $flags = 0, &$casToken = null ) {
92  $getToken = ( $casToken === self::PASS_BY_REF );
93  $casToken = null;
94 
95  $conn = $this->getConnection( $key );
96  if ( !$conn ) {
97  return false;
98  }
99 
100  $e = null;
101  try {
102  $value = $conn->get( $key );
103  if ( $getToken && $value !== false ) {
104  $casToken = $value;
105  }
106  $result = $this->unserialize( $value );
107  } catch ( RedisException $e ) {
108  $result = false;
109  $this->handleException( $conn, $e );
110  }
111 
112  $this->logRequest( 'get', $key, $conn->getServer(), $e );
113 
114  return $result;
115  }
116 
117  protected function doSet( $key, $value, $exptime = 0, $flags = 0 ) {
118  $conn = $this->getConnection( $key );
119  if ( !$conn ) {
120  return false;
121  }
122 
123  $ttl = $this->getExpirationAsTTL( $exptime );
124 
125  $e = null;
126  try {
127  if ( $ttl ) {
128  $result = $conn->setex( $key, $ttl, $this->getSerialized( $value, $key ) );
129  } else {
130  $result = $conn->set( $key, $this->getSerialized( $value, $key ) );
131  }
132  } catch ( RedisException $e ) {
133  $result = false;
134  $this->handleException( $conn, $e );
135  }
136 
137  $this->logRequest( 'set', $key, $conn->getServer(), $e );
138 
139  return $result;
140  }
141 
142  protected function doDelete( $key, $flags = 0 ) {
143  $conn = $this->getConnection( $key );
144  if ( !$conn ) {
145  return false;
146  }
147 
148  $e = null;
149  try {
150  // Note that redis does not return false if the key was not there
151  $result = ( $conn->del( $key ) !== false );
152  } catch ( RedisException $e ) {
153  $result = false;
154  $this->handleException( $conn, $e );
155  }
156 
157  $this->logRequest( 'delete', $key, $conn->getServer(), $e );
158 
159  return $result;
160  }
161 
162  protected function doGetMulti( array $keys, $flags = 0 ) {
164  $conns = [];
165  $batches = [];
166  foreach ( $keys as $key ) {
167  $conn = $this->getConnection( $key );
168  if ( $conn ) {
169  $server = $conn->getServer();
170  $conns[$server] = $conn;
171  $batches[$server][] = $key;
172  }
173  }
174 
175  $result = [];
176  foreach ( $batches as $server => $batchKeys ) {
177  $conn = $conns[$server];
178 
179  $e = null;
180  try {
181  // Avoid mget() to reduce CPU hogging from a single request
182  $conn->multi( Redis::PIPELINE );
183  foreach ( $batchKeys as $key ) {
184  $conn->get( $key );
185  }
186  $batchResult = $conn->exec();
187  if ( $batchResult === false ) {
188  $this->logRequest( 'get', implode( ',', $batchKeys ), $server, true );
189  continue;
190  }
191 
192  foreach ( $batchResult as $i => $value ) {
193  if ( $value !== false ) {
194  $result[$batchKeys[$i]] = $this->unserialize( $value );
195  }
196  }
197  } catch ( RedisException $e ) {
198  $this->handleException( $conn, $e );
199  }
200 
201  $this->logRequest( 'get', implode( ',', $batchKeys ), $server, $e );
202  }
203 
204  return $result;
205  }
206 
207  protected function doSetMulti( array $data, $exptime = 0, $flags = 0 ) {
209  $conns = [];
210  $batches = [];
211  foreach ( $data as $key => $value ) {
212  $conn = $this->getConnection( $key );
213  if ( $conn ) {
214  $server = $conn->getServer();
215  $conns[$server] = $conn;
216  $batches[$server][] = $key;
217  }
218  }
219 
220  $ttl = $this->getExpirationAsTTL( $exptime );
221  $op = $ttl ? 'setex' : 'set';
222 
223  $result = true;
224  foreach ( $batches as $server => $batchKeys ) {
225  $conn = $conns[$server];
226 
227  $e = null;
228  try {
229  // Avoid mset() to reduce CPU hogging from a single request
230  $conn->multi( Redis::PIPELINE );
231  foreach ( $batchKeys as $key ) {
232  if ( $ttl ) {
233  $conn->setex( $key, $ttl, $this->getSerialized( $data[$key], $key ) );
234  } else {
235  $conn->set( $key, $this->getSerialized( $data[$key], $key ) );
236  }
237  }
238  $batchResult = $conn->exec();
239  if ( $batchResult === false ) {
240  $this->logRequest( $op, implode( ',', $batchKeys ), $server, true );
241  continue;
242  }
243  $result = $result && !in_array( false, $batchResult, true );
244  } catch ( RedisException $e ) {
245  $this->handleException( $conn, $e );
246  $result = false;
247  }
248 
249  $this->logRequest( $op, implode( ',', $batchKeys ), $server, $e );
250  }
251 
252  return $result;
253  }
254 
255  protected function doDeleteMulti( array $keys, $flags = 0 ) {
257  $conns = [];
258  $batches = [];
259  foreach ( $keys as $key ) {
260  $conn = $this->getConnection( $key );
261  if ( $conn ) {
262  $server = $conn->getServer();
263  $conns[$server] = $conn;
264  $batches[$server][] = $key;
265  }
266  }
267 
268  $result = true;
269  foreach ( $batches as $server => $batchKeys ) {
270  $conn = $conns[$server];
271 
272  $e = null;
273  try {
274  // Avoid delete() with array to reduce CPU hogging from a single request
275  $conn->multi( Redis::PIPELINE );
276  foreach ( $batchKeys as $key ) {
277  $conn->del( $key );
278  }
279  $batchResult = $conn->exec();
280  if ( $batchResult === false ) {
281  $this->logRequest( 'delete', implode( ',', $batchKeys ), $server, true );
282  continue;
283  }
284  // Note that redis does not return false if the key was not there
285  $result = $result && !in_array( false, $batchResult, true );
286  } catch ( RedisException $e ) {
287  $this->handleException( $conn, $e );
288  $result = false;
289  }
290 
291  $this->logRequest( 'delete', implode( ',', $batchKeys ), $server, $e );
292  }
293 
294  return $result;
295  }
296 
297  public function changeTTLMulti( array $keys, $exptime, $flags = 0 ) {
299  $conns = [];
300  $batches = [];
301  foreach ( $keys as $key ) {
302  $conn = $this->getConnection( $key );
303  if ( $conn ) {
304  $server = $conn->getServer();
305  $conns[$server] = $conn;
306  $batches[$server][] = $key;
307  }
308  }
309 
310  $relative = $this->isRelativeExpiration( $exptime );
311  $op = ( $exptime == self::TTL_INDEFINITE )
312  ? 'persist'
313  : ( $relative ? 'expire' : 'expireAt' );
314 
315  $result = true;
316  foreach ( $batches as $server => $batchKeys ) {
317  $conn = $conns[$server];
318 
319  $e = null;
320  try {
321  $conn->multi( Redis::PIPELINE );
322  foreach ( $batchKeys as $key ) {
323  if ( $exptime == self::TTL_INDEFINITE ) {
324  $conn->persist( $key );
325  } elseif ( $relative ) {
326  $conn->expire( $key, $this->getExpirationAsTTL( $exptime ) );
327  } else {
328  $conn->expireAt( $key, $this->getExpirationAsTimestamp( $exptime ) );
329  }
330  }
331  $batchResult = $conn->exec();
332  if ( $batchResult === false ) {
333  $this->logRequest( $op, implode( ',', $batchKeys ), $server, true );
334  continue;
335  }
336  $result = in_array( false, $batchResult, true ) ? false : $result;
337  } catch ( RedisException $e ) {
338  $this->handleException( $conn, $e );
339  $result = false;
340  }
341 
342  $this->logRequest( $op, implode( ',', $batchKeys ), $server, $e );
343  }
344 
345  return $result;
346  }
347 
348  protected function doAdd( $key, $value, $expiry = 0, $flags = 0 ) {
349  $conn = $this->getConnection( $key );
350  if ( !$conn ) {
351  return false;
352  }
353 
354  $ttl = $this->getExpirationAsTTL( $expiry );
355  try {
356  $result = $conn->set(
357  $key,
358  $this->getSerialized( $value, $key ),
359  $ttl ? [ 'nx', 'ex' => $ttl ] : [ 'nx' ]
360  );
361  } catch ( RedisException $e ) {
362  $result = false;
363  $this->handleException( $conn, $e );
364  }
365 
366  $this->logRequest( 'add', $key, $conn->getServer(), $result );
367 
368  return $result;
369  }
370 
371  public function incr( $key, $value = 1, $flags = 0 ) {
372  $conn = $this->getConnection( $key );
373  if ( !$conn ) {
374  return false;
375  }
376 
377  try {
378  if ( !$conn->exists( $key ) ) {
379  return false;
380  }
381  // @FIXME: on races, the key may have a 0 TTL
382  $result = $conn->incrBy( $key, $value );
383  } catch ( RedisException $e ) {
384  $result = false;
385  $this->handleException( $conn, $e );
386  }
387 
388  $this->logRequest( 'incr', $key, $conn->getServer(), $result );
389 
390  return $result;
391  }
392 
393  public function decr( $key, $value = 1, $flags = 0 ) {
394  $conn = $this->getConnection( $key );
395  if ( !$conn ) {
396  return false;
397  }
398 
399  try {
400  if ( !$conn->exists( $key ) ) {
401  return false;
402  }
403  // @FIXME: on races, the key may have a 0 TTL
404  $result = $conn->decrBy( $key, $value );
405  } catch ( RedisException $e ) {
406  $result = false;
407  $this->handleException( $conn, $e );
408  }
409 
410  $this->logRequest( 'decr', $key, $conn->getServer(), $result );
411 
412  return $result;
413  }
414 
415  protected function doChangeTTL( $key, $exptime, $flags ) {
416  $conn = $this->getConnection( $key );
417  if ( !$conn ) {
418  return false;
419  }
420 
421  $relative = $this->isRelativeExpiration( $exptime );
422  try {
423  if ( $exptime == self::TTL_INDEFINITE ) {
424  $result = $conn->persist( $key );
425  $this->logRequest( 'persist', $key, $conn->getServer(), $result );
426  } elseif ( $relative ) {
427  $result = $conn->expire( $key, $this->getExpirationAsTTL( $exptime ) );
428  $this->logRequest( 'expire', $key, $conn->getServer(), $result );
429  } else {
430  $result = $conn->expireAt( $key, $this->getExpirationAsTimestamp( $exptime ) );
431  $this->logRequest( 'expireAt', $key, $conn->getServer(), $result );
432  }
433  } catch ( RedisException $e ) {
434  $result = false;
435  $this->handleException( $conn, $e );
436  }
437 
438  return $result;
439  }
440 
445  protected function getConnection( $key ) {
446  $candidates = array_keys( $this->serverTagMap );
447 
448  if ( count( $this->servers ) > 1 ) {
449  ArrayUtils::consistentHashSort( $candidates, $key, '/' );
450  if ( !$this->automaticFailover ) {
451  $candidates = array_slice( $candidates, 0, 1 );
452  }
453  }
454 
455  while ( ( $tag = array_shift( $candidates ) ) !== null ) {
456  $server = $this->serverTagMap[$tag];
457  $conn = $this->redisPool->getConnection( $server, $this->logger );
458  if ( !$conn ) {
459  continue;
460  }
461 
462  // If automatic failover is enabled, check that the server's link
463  // to its master (if any) is up -- but only if there are other
464  // viable candidates left to consider. Also, getMasterLinkStatus()
465  // does not work with twemproxy, though $candidates will be empty
466  // by now in such cases.
467  if ( $this->automaticFailover && $candidates ) {
468  try {
470  $info = $conn->info();
471  if ( ( $info['master_link_status'] ?? null ) === 'down' ) {
472  // If the master cannot be reached, fail-over to the next server.
473  // If masters are in data-center A, and replica DBs in data-center B,
474  // this helps avoid the case were fail-over happens in A but not
475  // to the corresponding server in B (e.g. read/write mismatch).
476  continue;
477  }
478  } catch ( RedisException $e ) {
479  // Server is not accepting commands
480  $this->redisPool->handleError( $conn, $e );
481  continue;
482  }
483  }
484 
485  return $conn;
486  }
487 
488  $this->setLastError( BagOStuff::ERR_UNREACHABLE );
489 
490  return null;
491  }
492 
497  protected function logError( $msg ) {
498  $this->logger->error( "Redis error: $msg" );
499  }
500 
509  protected function handleException( RedisConnRef $conn, RedisException $e ) {
510  $this->setLastError( BagOStuff::ERR_UNEXPECTED );
511  $this->redisPool->handleError( $conn, $e );
512  }
513 
521  public function logRequest( $op, $keys, $server, $e = null ) {
522  $this->debug( "$op($keys) on $server: " . ( $e ? "failure" : "success" ) );
523  }
524 }
RedisBagOStuff\doGet
doGet( $key, $flags=0, &$casToken=null)
Definition: RedisBagOStuff.php:91
MediumSpecificBagOStuff\setLastError
setLastError( $err)
Set the "last error" registry.
Definition: MediumSpecificBagOStuff.php:758
RedisBagOStuff\handleException
handleException(RedisConnRef $conn, RedisException $e)
The redis extension throws an exception in response to various read, write and protocol errors.
Definition: RedisBagOStuff.php:509
ArrayUtils\consistentHashSort
static consistentHashSort(&$array, $key, $separator="\000")
Sort the given array in a pseudo-random order which depends only on the given key and each element va...
Definition: ArrayUtils.php:49
RedisBagOStuff
Redis-based caching module for redis server >= 2.6.12 and phpredis >= 2.2.4.
Definition: RedisBagOStuff.php:33
RedisConnectionPool\singleton
static singleton(array $options)
Definition: RedisConnectionPool.php:149
MediumSpecificBagOStuff\debug
debug( $text)
Definition: MediumSpecificBagOStuff.php:1064
RedisBagOStuff\doDelete
doDelete( $key, $flags=0)
Delete an item.
Definition: RedisBagOStuff.php:142
RedisBagOStuff\$redisPool
RedisConnectionPool $redisPool
Definition: RedisBagOStuff.php:35
RedisBagOStuff\doAdd
doAdd( $key, $value, $expiry=0, $flags=0)
Insert an item if it does not already exist.
Definition: RedisBagOStuff.php:348
RedisBagOStuff\$automaticFailover
bool $automaticFailover
Definition: RedisBagOStuff.php:41
RedisBagOStuff\$servers
array $servers
List of server names.
Definition: RedisBagOStuff.php:37
RedisBagOStuff\decr
decr( $key, $value=1, $flags=0)
Decrease stored value of $key by $value while preserving its TTL.
Definition: RedisBagOStuff.php:393
RedisBagOStuff\doGetMulti
doGetMulti(array $keys, $flags=0)
Get an associative array containing the item for each of the keys that have items.
Definition: RedisBagOStuff.php:162
RedisBagOStuff\getConnection
getConnection( $key)
Definition: RedisBagOStuff.php:445
RedisBagOStuff\logError
logError( $msg)
Log a fatal error.
Definition: RedisBagOStuff.php:497
RedisBagOStuff\doDeleteMulti
doDeleteMulti(array $keys, $flags=0)
Definition: RedisBagOStuff.php:255
MediumSpecificBagOStuff\getExpirationAsTimestamp
getExpirationAsTimestamp( $exptime)
Convert an optionally relative timestamp to an absolute time.
Definition: MediumSpecificBagOStuff.php:841
MediumSpecificBagOStuff\getExpirationAsTTL
getExpirationAsTTL( $exptime)
Convert an optionally absolute expiry time to a relative time.
Definition: MediumSpecificBagOStuff.php:865
MediumSpecificBagOStuff
Storage medium specific cache for storing items (e.g.
Definition: MediumSpecificBagOStuff.php:34
RedisBagOStuff\doSet
doSet( $key, $value, $exptime=0, $flags=0)
Set an item.
Definition: RedisBagOStuff.php:117
RedisBagOStuff\logRequest
logRequest( $op, $keys, $server, $e=null)
Send information about a single request to the debug log.
Definition: RedisBagOStuff.php:521
RedisBagOStuff\incr
incr( $key, $value=1, $flags=0)
Increase stored value of $key by $value while preserving its TTL.
Definition: RedisBagOStuff.php:371
RedisConnectionPool
Helper class to manage Redis connections.
Definition: RedisConnectionPool.php:41
RedisBagOStuff\$serverTagMap
array $serverTagMap
Map of (tag => server name)
Definition: RedisBagOStuff.php:39
MediumSpecificBagOStuff\getSerialized
getSerialized( $value, $key)
Get the serialized form a value, using any applicable prepared value.
Definition: MediumSpecificBagOStuff.php:971
MediumSpecificBagOStuff\isRelativeExpiration
isRelativeExpiration( $exptime)
Definition: MediumSpecificBagOStuff.php:824
RedisBagOStuff\doSetMulti
doSetMulti(array $data, $exptime=0, $flags=0)
Definition: RedisBagOStuff.php:207
MediumSpecificBagOStuff\unserialize
unserialize( $value)
Definition: MediumSpecificBagOStuff.php:1057
RedisBagOStuff\__construct
__construct( $params)
Construct a RedisBagOStuff object.
Definition: RedisBagOStuff.php:71
RedisConnRef
Helper class to handle automatically marking connectons as reusable (via RAII pattern)
Definition: RedisConnRef.php:31
$keys
$keys
Definition: testCompression.php:72
RedisBagOStuff\changeTTLMulti
changeTTLMulti(array $keys, $exptime, $flags=0)
Change the expiration of multiple keys that exist.
Definition: RedisBagOStuff.php:297
RedisBagOStuff\doChangeTTL
doChangeTTL( $key, $exptime, $flags)
Definition: RedisBagOStuff.php:415