MediaWiki  master
RedisBagOStuff.php
Go to the documentation of this file.
1 <?php
35  protected $redisPool;
37  protected $servers;
39  protected $serverTagMap;
41  protected $automaticFailover;
42 
71  public function __construct( $params ) {
72  parent::__construct( $params );
73  $redisConf = [ 'serializer' => 'none' ]; // manage that in this class
74  foreach ( [ 'connectTimeout', 'persistent', 'password' ] as $opt ) {
75  if ( isset( $params[$opt] ) ) {
76  $redisConf[$opt] = $params[$opt];
77  }
78  }
79  $this->redisPool = RedisConnectionPool::singleton( $redisConf );
80 
81  $this->servers = $params['servers'];
82  foreach ( $this->servers as $key => $server ) {
83  $this->serverTagMap[is_int( $key ) ? $server : $key] = $server;
84  }
85 
86  $this->automaticFailover = $params['automaticFailover'] ?? true;
87 
88  $this->attrMap[self::ATTR_SYNCWRITES] = self::QOS_SYNCWRITES_NONE;
89  }
90 
91  protected function doGet( $key, $flags = 0, &$casToken = null ) {
92  $casToken = null;
93 
94  $conn = $this->getConnection( $key );
95  if ( !$conn ) {
96  return false;
97  }
98 
99  $e = null;
100  try {
101  $value = $conn->get( $key );
102  $casToken = $value;
103  $result = $this->unserialize( $value );
104  } catch ( RedisException $e ) {
105  $result = false;
106  $this->handleException( $conn, $e );
107  }
108 
109  $this->logRequest( 'get', $key, $conn->getServer(), $e );
110 
111  return $result;
112  }
113 
114  protected function doSet( $key, $value, $exptime = 0, $flags = 0 ) {
115  $conn = $this->getConnection( $key );
116  if ( !$conn ) {
117  return false;
118  }
119 
120  $ttl = $this->getExpirationAsTTL( $exptime );
121 
122  $e = null;
123  try {
124  if ( $ttl ) {
125  $result = $conn->setex( $key, $ttl, $this->getSerialized( $value, $key ) );
126  } else {
127  $result = $conn->set( $key, $this->getSerialized( $value, $key ) );
128  }
129  } catch ( RedisException $e ) {
130  $result = false;
131  $this->handleException( $conn, $e );
132  }
133 
134  $this->logRequest( 'set', $key, $conn->getServer(), $e );
135 
136  return $result;
137  }
138 
139  protected function doDelete( $key, $flags = 0 ) {
140  $conn = $this->getConnection( $key );
141  if ( !$conn ) {
142  return false;
143  }
144 
145  $e = null;
146  try {
147  // Note that redis does not return false if the key was not there
148  $result = ( $conn->del( $key ) !== false );
149  } catch ( RedisException $e ) {
150  $result = false;
151  $this->handleException( $conn, $e );
152  }
153 
154  $this->logRequest( 'delete', $key, $conn->getServer(), $e );
155 
156  return $result;
157  }
158 
159  protected function doGetMulti( array $keys, $flags = 0 ) {
161  $conns = [];
162  $batches = [];
163  foreach ( $keys as $key ) {
164  $conn = $this->getConnection( $key );
165  if ( $conn ) {
166  $server = $conn->getServer();
167  $conns[$server] = $conn;
168  $batches[$server][] = $key;
169  }
170  }
171 
172  $result = [];
173  foreach ( $batches as $server => $batchKeys ) {
174  $conn = $conns[$server];
175 
176  $e = null;
177  try {
178  // Avoid mget() to reduce CPU hogging from a single request
179  $conn->multi( Redis::PIPELINE );
180  foreach ( $batchKeys as $key ) {
181  $conn->get( $key );
182  }
183  $batchResult = $conn->exec();
184  if ( $batchResult === false ) {
185  $this->logRequest( 'get', implode( ',', $batchKeys ), $server, true );
186  continue;
187  }
188 
189  foreach ( $batchResult as $i => $value ) {
190  if ( $value !== false ) {
191  $result[$batchKeys[$i]] = $this->unserialize( $value );
192  }
193  }
194  } catch ( RedisException $e ) {
195  $this->handleException( $conn, $e );
196  }
197 
198  $this->logRequest( 'get', implode( ',', $batchKeys ), $server, $e );
199  }
200 
201  return $result;
202  }
203 
204  protected function doSetMulti( array $data, $exptime = 0, $flags = 0 ) {
206  $conns = [];
207  $batches = [];
208  foreach ( $data as $key => $value ) {
209  $conn = $this->getConnection( $key );
210  if ( $conn ) {
211  $server = $conn->getServer();
212  $conns[$server] = $conn;
213  $batches[$server][] = $key;
214  }
215  }
216 
217  $ttl = $this->getExpirationAsTTL( $exptime );
218  $op = $ttl ? 'setex' : 'set';
219 
220  $result = true;
221  foreach ( $batches as $server => $batchKeys ) {
222  $conn = $conns[$server];
223 
224  $e = null;
225  try {
226  // Avoid mset() to reduce CPU hogging from a single request
227  $conn->multi( Redis::PIPELINE );
228  foreach ( $batchKeys as $key ) {
229  if ( $ttl ) {
230  $conn->setex( $key, $ttl, $this->getSerialized( $data[$key], $key ) );
231  } else {
232  $conn->set( $key, $this->getSerialized( $data[$key], $key ) );
233  }
234  }
235  $batchResult = $conn->exec();
236  if ( $batchResult === false ) {
237  $this->logRequest( $op, implode( ',', $batchKeys ), $server, true );
238  continue;
239  }
240  $result = $result && !in_array( false, $batchResult, true );
241  } catch ( RedisException $e ) {
242  $this->handleException( $conn, $e );
243  $result = false;
244  }
245 
246  $this->logRequest( $op, implode( ',', $batchKeys ), $server, $e );
247  }
248 
249  return $result;
250  }
251 
252  protected function doDeleteMulti( array $keys, $flags = 0 ) {
254  $conns = [];
255  $batches = [];
256  foreach ( $keys as $key ) {
257  $conn = $this->getConnection( $key );
258  if ( $conn ) {
259  $server = $conn->getServer();
260  $conns[$server] = $conn;
261  $batches[$server][] = $key;
262  }
263  }
264 
265  $result = true;
266  foreach ( $batches as $server => $batchKeys ) {
267  $conn = $conns[$server];
268 
269  $e = null;
270  try {
271  // Avoid delete() with array to reduce CPU hogging from a single request
272  $conn->multi( Redis::PIPELINE );
273  foreach ( $batchKeys as $key ) {
274  $conn->del( $key );
275  }
276  $batchResult = $conn->exec();
277  if ( $batchResult === false ) {
278  $this->logRequest( 'delete', implode( ',', $batchKeys ), $server, true );
279  continue;
280  }
281  // Note that redis does not return false if the key was not there
282  $result = $result && !in_array( false, $batchResult, true );
283  } catch ( RedisException $e ) {
284  $this->handleException( $conn, $e );
285  $result = false;
286  }
287 
288  $this->logRequest( 'delete', implode( ',', $batchKeys ), $server, $e );
289  }
290 
291  return $result;
292  }
293 
294  public function changeTTLMulti( array $keys, $exptime, $flags = 0 ) {
296  $conns = [];
297  $batches = [];
298  foreach ( $keys as $key ) {
299  $conn = $this->getConnection( $key );
300  if ( $conn ) {
301  $server = $conn->getServer();
302  $conns[$server] = $conn;
303  $batches[$server][] = $key;
304  }
305  }
306 
307  $relative = $this->isRelativeExpiration( $exptime );
308  $op = ( $exptime == self::TTL_INDEFINITE )
309  ? 'persist'
310  : ( $relative ? 'expire' : 'expireAt' );
311 
312  $result = true;
313  foreach ( $batches as $server => $batchKeys ) {
314  $conn = $conns[$server];
315 
316  $e = null;
317  try {
318  $conn->multi( Redis::PIPELINE );
319  foreach ( $batchKeys as $key ) {
320  if ( $exptime == self::TTL_INDEFINITE ) {
321  $conn->persist( $key );
322  } elseif ( $relative ) {
323  $conn->expire( $key, $this->getExpirationAsTTL( $exptime ) );
324  } else {
325  $conn->expireAt( $key, $this->getExpirationAsTimestamp( $exptime ) );
326  }
327  }
328  $batchResult = $conn->exec();
329  if ( $batchResult === false ) {
330  $this->logRequest( $op, implode( ',', $batchKeys ), $server, true );
331  continue;
332  }
333  $result = in_array( false, $batchResult, true ) ? false : $result;
334  } catch ( RedisException $e ) {
335  $this->handleException( $conn, $e );
336  $result = false;
337  }
338 
339  $this->logRequest( $op, implode( ',', $batchKeys ), $server, $e );
340  }
341 
342  return $result;
343  }
344 
345  protected function doAdd( $key, $value, $expiry = 0, $flags = 0 ) {
346  $conn = $this->getConnection( $key );
347  if ( !$conn ) {
348  return false;
349  }
350 
351  $ttl = $this->getExpirationAsTTL( $expiry );
352  try {
353  $result = $conn->set(
354  $key,
355  $this->getSerialized( $value, $key ),
356  $ttl ? [ 'nx', 'ex' => $ttl ] : [ 'nx' ]
357  );
358  } catch ( RedisException $e ) {
359  $result = false;
360  $this->handleException( $conn, $e );
361  }
362 
363  $this->logRequest( 'add', $key, $conn->getServer(), $result );
364 
365  return $result;
366  }
367 
368  public function incr( $key, $value = 1, $flags = 0 ) {
369  $conn = $this->getConnection( $key );
370  if ( !$conn ) {
371  return false;
372  }
373 
374  try {
375  if ( !$conn->exists( $key ) ) {
376  return false;
377  }
378  // @FIXME: on races, the key may have a 0 TTL
379  $result = $conn->incrBy( $key, $value );
380  } catch ( RedisException $e ) {
381  $result = false;
382  $this->handleException( $conn, $e );
383  }
384 
385  $this->logRequest( 'incr', $key, $conn->getServer(), $result );
386 
387  return $result;
388  }
389 
390  public function decr( $key, $value = 1, $flags = 0 ) {
391  $conn = $this->getConnection( $key );
392  if ( !$conn ) {
393  return false;
394  }
395 
396  try {
397  if ( !$conn->exists( $key ) ) {
398  return false;
399  }
400  // @FIXME: on races, the key may have a 0 TTL
401  $result = $conn->decrBy( $key, $value );
402  } catch ( RedisException $e ) {
403  $result = false;
404  $this->handleException( $conn, $e );
405  }
406 
407  $this->logRequest( 'decr', $key, $conn->getServer(), $result );
408 
409  return $result;
410  }
411 
412  protected function doChangeTTL( $key, $exptime, $flags ) {
413  $conn = $this->getConnection( $key );
414  if ( !$conn ) {
415  return false;
416  }
417 
418  $relative = $this->isRelativeExpiration( $exptime );
419  try {
420  if ( $exptime == self::TTL_INDEFINITE ) {
421  $result = $conn->persist( $key );
422  $this->logRequest( 'persist', $key, $conn->getServer(), $result );
423  } elseif ( $relative ) {
424  $result = $conn->expire( $key, $this->getExpirationAsTTL( $exptime ) );
425  $this->logRequest( 'expire', $key, $conn->getServer(), $result );
426  } else {
427  $result = $conn->expireAt( $key, $this->getExpirationAsTimestamp( $exptime ) );
428  $this->logRequest( 'expireAt', $key, $conn->getServer(), $result );
429  }
430  } catch ( RedisException $e ) {
431  $result = false;
432  $this->handleException( $conn, $e );
433  }
434 
435  return $result;
436  }
437 
442  protected function getConnection( $key ) {
443  $candidates = array_keys( $this->serverTagMap );
444 
445  if ( count( $this->servers ) > 1 ) {
446  ArrayUtils::consistentHashSort( $candidates, $key, '/' );
447  if ( !$this->automaticFailover ) {
448  $candidates = array_slice( $candidates, 0, 1 );
449  }
450  }
451 
452  while ( ( $tag = array_shift( $candidates ) ) !== null ) {
453  $server = $this->serverTagMap[$tag];
454  $conn = $this->redisPool->getConnection( $server, $this->logger );
455  if ( !$conn ) {
456  continue;
457  }
458 
459  // If automatic failover is enabled, check that the server's link
460  // to its master (if any) is up -- but only if there are other
461  // viable candidates left to consider. Also, getMasterLinkStatus()
462  // does not work with twemproxy, though $candidates will be empty
463  // by now in such cases.
464  if ( $this->automaticFailover && $candidates ) {
465  try {
467  $info = $conn->info();
468  if ( ( $info['master_link_status'] ?? null ) === 'down' ) {
469  // If the master cannot be reached, fail-over to the next server.
470  // If masters are in data-center A, and replica DBs in data-center B,
471  // this helps avoid the case were fail-over happens in A but not
472  // to the corresponding server in B (e.g. read/write mismatch).
473  continue;
474  }
475  } catch ( RedisException $e ) {
476  // Server is not accepting commands
477  $this->redisPool->handleError( $conn, $e );
478  continue;
479  }
480  }
481 
482  return $conn;
483  }
484 
485  $this->setLastError( BagOStuff::ERR_UNREACHABLE );
486 
487  return null;
488  }
489 
494  protected function logError( $msg ) {
495  $this->logger->error( "Redis error: $msg" );
496  }
497 
506  protected function handleException( RedisConnRef $conn, RedisException $e ) {
507  $this->setLastError( BagOStuff::ERR_UNEXPECTED );
508  $this->redisPool->handleError( $conn, $e );
509  }
510 
518  public function logRequest( $op, $keys, $server, $e = null ) {
519  $this->debug( "$op($keys) on $server: " . ( $e ? "failure" : "success" ) );
520  }
521 }
RedisBagOStuff\doGet
doGet( $key, $flags=0, &$casToken=null)
Definition: RedisBagOStuff.php:91
MediumSpecificBagOStuff\setLastError
setLastError( $err)
Set the "last error" registry.
Definition: MediumSpecificBagOStuff.php:755
RedisBagOStuff\handleException
handleException(RedisConnRef $conn, RedisException $e)
The redis extension throws an exception in response to various read, write and protocol errors.
Definition: RedisBagOStuff.php:506
ArrayUtils\consistentHashSort
static consistentHashSort(&$array, $key, $separator="\000")
Sort the given array in a pseudo-random order which depends only on the given key and each element va...
Definition: ArrayUtils.php:49
RedisBagOStuff
Redis-based caching module for redis server >= 2.6.12 and phpredis >= 2.2.4.
Definition: RedisBagOStuff.php:33
RedisConnectionPool\singleton
static singleton(array $options)
Definition: RedisConnectionPool.php:149
MediumSpecificBagOStuff\debug
debug( $text)
Definition: MediumSpecificBagOStuff.php:1061
RedisBagOStuff\doDelete
doDelete( $key, $flags=0)
Delete an item.
Definition: RedisBagOStuff.php:139
RedisBagOStuff\$redisPool
RedisConnectionPool $redisPool
Definition: RedisBagOStuff.php:35
RedisBagOStuff\doAdd
doAdd( $key, $value, $expiry=0, $flags=0)
Insert an item if it does not already exist.
Definition: RedisBagOStuff.php:345
RedisBagOStuff\$automaticFailover
bool $automaticFailover
Definition: RedisBagOStuff.php:41
RedisBagOStuff\$servers
array $servers
List of server names.
Definition: RedisBagOStuff.php:37
RedisBagOStuff\decr
decr( $key, $value=1, $flags=0)
Decrease stored value of $key by $value while preserving its TTL.
Definition: RedisBagOStuff.php:390
RedisBagOStuff\doGetMulti
doGetMulti(array $keys, $flags=0)
Get an associative array containing the item for each of the keys that have items.
Definition: RedisBagOStuff.php:159
RedisBagOStuff\getConnection
getConnection( $key)
Definition: RedisBagOStuff.php:442
RedisBagOStuff\logError
logError( $msg)
Log a fatal error.
Definition: RedisBagOStuff.php:494
RedisBagOStuff\doDeleteMulti
doDeleteMulti(array $keys, $flags=0)
Definition: RedisBagOStuff.php:252
MediumSpecificBagOStuff\getExpirationAsTimestamp
getExpirationAsTimestamp( $exptime)
Convert an optionally relative timestamp to an absolute time.
Definition: MediumSpecificBagOStuff.php:838
MediumSpecificBagOStuff\getExpirationAsTTL
getExpirationAsTTL( $exptime)
Convert an optionally absolute expiry time to a relative time.
Definition: MediumSpecificBagOStuff.php:862
MediumSpecificBagOStuff
Storage medium specific cache for storing items (e.g.
Definition: MediumSpecificBagOStuff.php:34
RedisBagOStuff\doSet
doSet( $key, $value, $exptime=0, $flags=0)
Set an item.
Definition: RedisBagOStuff.php:114
RedisBagOStuff\logRequest
logRequest( $op, $keys, $server, $e=null)
Send information about a single request to the debug log.
Definition: RedisBagOStuff.php:518
RedisBagOStuff\incr
incr( $key, $value=1, $flags=0)
Increase stored value of $key by $value while preserving its TTL.
Definition: RedisBagOStuff.php:368
RedisConnectionPool
Helper class to manage Redis connections.
Definition: RedisConnectionPool.php:41
RedisBagOStuff\$serverTagMap
array $serverTagMap
Map of (tag => server name)
Definition: RedisBagOStuff.php:39
MediumSpecificBagOStuff\getSerialized
getSerialized( $value, $key)
Get the serialized form a value, using any applicable prepared value.
Definition: MediumSpecificBagOStuff.php:968
MediumSpecificBagOStuff\isRelativeExpiration
isRelativeExpiration( $exptime)
Definition: MediumSpecificBagOStuff.php:821
RedisBagOStuff\doSetMulti
doSetMulti(array $data, $exptime=0, $flags=0)
Definition: RedisBagOStuff.php:204
MediumSpecificBagOStuff\unserialize
unserialize( $value)
Definition: MediumSpecificBagOStuff.php:1054
RedisBagOStuff\__construct
__construct( $params)
Construct a RedisBagOStuff object.
Definition: RedisBagOStuff.php:71
RedisConnRef
Helper class to handle automatically marking connectons as reusable (via RAII pattern)
Definition: RedisConnRef.php:31
$keys
$keys
Definition: testCompression.php:72
RedisBagOStuff\changeTTLMulti
changeTTLMulti(array $keys, $exptime, $flags=0)
Change the expiration of multiple keys that exist.
Definition: RedisBagOStuff.php:294
RedisBagOStuff\doChangeTTL
doChangeTTL( $key, $exptime, $flags)
Definition: RedisBagOStuff.php:412