MediaWiki  master
MemcachedPeclBagOStuff.php
Go to the documentation of this file.
1 <?php
31  protected $syncClient;
33  protected $asyncClient;
34 
36  protected $syncClientIsBuffering = false;
38  protected $hasUnflushedChanges = false;
39 
41  private static $OPTS_SYNC_WRITES = [
42  Memcached::OPT_NO_BLOCK => false, // async I/O (using TCP buffers)
43  Memcached::OPT_BUFFER_WRITES => false // libmemcached buffers
44  ];
46  private static $OPTS_ASYNC_WRITES = [
47  Memcached::OPT_NO_BLOCK => true, // async I/O (using TCP buffers)
48  Memcached::OPT_BUFFER_WRITES => true // libmemcached buffers
49  ];
50 
67  public function __construct( $params ) {
68  parent::__construct( $params );
69 
70  // Default class-specific parameters
71  $params += [
72  'compress_threshold' => 1500,
73  'connect_timeout' => 0.5,
74  'serializer' => 'php',
75  'use_binary_protocol' => false,
76  'allow_tcp_nagle_delay' => true
77  ];
78 
79  if ( $params['persistent'] ) {
80  // The pool ID must be unique to the server/option combination.
81  // The Memcached object is essentially shared for each pool ID.
82  // We can only reuse a pool ID if we keep the config consistent.
83  $connectionPoolId = md5( serialize( $params ) );
84  $syncClient = new Memcached( "$connectionPoolId-sync" );
85  // Avoid clobbering the main thread-shared Memcached instance
86  $asyncClient = new Memcached( "$connectionPoolId-async" );
87  } else {
88  $syncClient = new Memcached();
89  $asyncClient = null;
90  }
91 
92  $this->initializeClient( $syncClient, $params, self::$OPTS_SYNC_WRITES );
93  if ( $asyncClient ) {
94  $this->initializeClient( $asyncClient, $params, self::$OPTS_ASYNC_WRITES );
95  }
96 
97  // Set the main client and any dedicated one for buffered writes
98  $this->syncClient = $syncClient;
99  $this->asyncClient = $asyncClient;
100  // The compression threshold is an undocumented php.ini option for some
101  // reason. There's probably not much harm in setting it globally, for
102  // compatibility with the settings for the PHP client.
103  ini_set( 'memcached.compression_threshold', $params['compress_threshold'] );
104  }
105 
115  private function initializeClient( Memcached $client, array $params, array $options ) {
116  if ( $client->getServerList() ) {
117  $this->logger->debug( __METHOD__ . ": pre-initialized client instance." );
118 
119  return; // preserve persistent handle
120  }
121 
122  $this->logger->debug( __METHOD__ . ": initializing new client instance." );
123 
124  $options += [
125  Memcached::OPT_NO_BLOCK => false,
126  Memcached::OPT_BUFFER_WRITES => false,
127  // Network protocol (ASCII or binary)
128  Memcached::OPT_BINARY_PROTOCOL => $params['use_binary_protocol'],
129  // Set various network timeouts
130  Memcached::OPT_CONNECT_TIMEOUT => $params['connect_timeout'] * 1000,
131  Memcached::OPT_SEND_TIMEOUT => $params['timeout'],
132  Memcached::OPT_RECV_TIMEOUT => $params['timeout'],
133  Memcached::OPT_POLL_TIMEOUT => $params['timeout'] / 1000,
134  // Avoid pointless delay when sending/fetching large blobs
135  Memcached::OPT_TCP_NODELAY => !$params['allow_tcp_nagle_delay'],
136  // Set libketama mode since it's recommended by the documentation
137  Memcached::OPT_LIBKETAMA_COMPATIBLE => true
138  ];
139  if ( isset( $params['retry_timeout'] ) ) {
140  $options[Memcached::OPT_RETRY_TIMEOUT] = $params['retry_timeout'];
141  }
142  if ( isset( $params['server_failure_limit'] ) ) {
143  $options[Memcached::OPT_SERVER_FAILURE_LIMIT] = $params['server_failure_limit'];
144  }
145  if ( $params['serializer'] === 'php' ) {
146  $options[Memcached::OPT_SERIALIZER] = Memcached::SERIALIZER_PHP;
147  } elseif ( $params['serializer'] === 'igbinary' ) {
148  // @phan-suppress-next-line PhanImpossibleCondition
149  if ( !Memcached::HAVE_IGBINARY ) {
150  throw new RuntimeException(
151  __CLASS__ . ': the igbinary extension is not available ' .
152  'but igbinary serialization was requested.'
153  );
154  }
155  $options[Memcached::OPT_SERIALIZER] = Memcached::SERIALIZER_IGBINARY;
156  }
157 
158  if ( !$client->setOptions( $options ) ) {
159  throw new RuntimeException(
160  "Invalid options: " . json_encode( $options, JSON_PRETTY_PRINT )
161  );
162  }
163 
164  $servers = [];
165  foreach ( $params['servers'] as $host ) {
166  if ( preg_match( '/^\[(.+)\]:(\d+)$/', $host, $m ) ) {
167  $servers[] = [ $m[1], (int)$m[2] ]; // (ip, port)
168  } elseif ( preg_match( '/^([^:]+):(\d+)$/', $host, $m ) ) {
169  $servers[] = [ $m[1], (int)$m[2] ]; // (ip or path, port)
170  } else {
171  $servers[] = [ $host, false ]; // (ip or path, port)
172  }
173  }
174 
175  if ( !$client->addServers( $servers ) ) {
176  throw new RuntimeException( "Failed to inject server address list" );
177  }
178  }
179 
180  protected function doGet( $key, $flags = 0, &$casToken = null ) {
181  $getToken = ( $casToken === self::PASS_BY_REF );
182  $casToken = null;
183 
184  $this->debug( "get($key)" );
185 
186  $routeKey = $this->validateKeyAndPrependRoute( $key );
187 
188  $client = $this->acquireSyncClient();
189  // T257003: only require "gets" (instead of "get") when a CAS token is needed
190  if ( $getToken ) {
192  $flags = Memcached::GET_EXTENDED;
193  $res = $client->get( $routeKey, null, $flags );
194  if ( is_array( $res ) ) {
195  $result = $res['value'];
196  $casToken = $res['cas'];
197  } else {
198  $result = false;
199  }
200  } else {
201  $result = $client->get( $routeKey );
202  }
203 
204  return $this->checkResult( $key, $result );
205  }
206 
207  protected function doSet( $key, $value, $exptime = 0, $flags = 0 ) {
208  $this->debug( "set($key)" );
209 
210  $routeKey = $this->validateKeyAndPrependRoute( $key );
211  $client = $this->acquireSyncClient();
212  $result = $client->set( $routeKey, $value, $this->fixExpiry( $exptime ) );
213 
214  return ( $result === false && $client->getResultCode() === Memcached::RES_NOTSTORED )
215  // "Not stored" is always used as the mcrouter response with AllAsyncRoute
216  ? true
217  : $this->checkResult( $key, $result );
218  }
219 
220  protected function doCas( $casToken, $key, $value, $exptime = 0, $flags = 0 ) {
221  $this->debug( "cas($key)" );
222 
223  $routeKey = $this->validateKeyAndPrependRoute( $key );
224  $result = $this->acquireSyncClient()->cas(
225  $casToken,
226  $routeKey,
227  $value, $this->fixExpiry( $exptime )
228  );
229 
230  return $this->checkResult( $key, $result );
231  }
232 
233  protected function doDelete( $key, $flags = 0 ) {
234  $this->debug( "delete($key)" );
235 
236  $routeKey = $this->validateKeyAndPrependRoute( $key );
237  $client = $this->acquireSyncClient();
238  $result = $client->delete( $routeKey );
239 
240  return ( $result === false && $client->getResultCode() === Memcached::RES_NOTFOUND )
241  // "Not found" is counted as success in our interface
242  ? true
243  : $this->checkResult( $key, $result );
244  }
245 
246  protected function doAdd( $key, $value, $exptime = 0, $flags = 0 ) {
247  $this->debug( "add($key)" );
248 
249  $routeKey = $this->validateKeyAndPrependRoute( $key );
250  $result = $this->acquireSyncClient()->add(
251  $routeKey,
252  $value,
253  $this->fixExpiry( $exptime )
254  );
255 
256  return $this->checkResult( $key, $result );
257  }
258 
259  public function incr( $key, $value = 1, $flags = 0 ) {
260  $this->debug( "incr($key)" );
261 
262  $routeKey = $this->validateKeyAndPrependRoute( $key );
263  $result = $this->acquireSyncClient()->increment( $routeKey, $value );
264 
265  return $this->checkResult( $key, $result );
266  }
267 
268  public function decr( $key, $value = 1, $flags = 0 ) {
269  $this->debug( "decr($key)" );
270 
271  $routeKey = $this->validateKeyAndPrependRoute( $key );
272  $result = $this->acquireSyncClient()->decrement( $routeKey, $value );
273 
274  return $this->checkResult( $key, $result );
275  }
276 
277  public function setNewPreparedValues( array $valueByKey ) {
278  // The PECL driver does the serializing and will not reuse anything from here
279  $sizes = [];
280  foreach ( $valueByKey as $value ) {
281  $sizes[] = $this->guessSerialValueSize( $value );
282  }
283 
284  return $sizes;
285  }
286 
298  protected function checkResult( $key, $result ) {
299  if ( $result !== false ) {
300  return $result;
301  }
302 
303  $client = $this->syncClient;
304  switch ( $client->getResultCode() ) {
305  case Memcached::RES_SUCCESS:
306  break;
307  case Memcached::RES_DATA_EXISTS:
308  case Memcached::RES_NOTSTORED:
309  case Memcached::RES_NOTFOUND:
310  $this->debug( "result: " . $client->getResultMessage() );
311  break;
312  default:
313  $msg = $client->getResultMessage();
314  $logCtx = [];
315  if ( $key !== false ) {
316  $server = $client->getServerByKey( $key );
317  $logCtx['memcached-server'] = "{$server['host']}:{$server['port']}";
318  $logCtx['memcached-key'] = $key;
319  $msg = "Memcached error for key \"{memcached-key}\" " .
320  "on server \"{memcached-server}\": $msg";
321  } else {
322  $msg = "Memcached error: $msg";
323  }
324  $this->logger->error( $msg, $logCtx );
326  }
327  return $result;
328  }
329 
330  protected function doGetMulti( array $keys, $flags = 0 ) {
331  $this->debug( 'getMulti(' . implode( ', ', $keys ) . ')' );
332 
333  $routeKeys = [];
334  foreach ( $keys as $key ) {
335  $routeKeys[] = $this->validateKeyAndPrependRoute( $key );
336  }
337 
338  // The PECL implementation uses multi-key "get"/"gets"; no need to pipeline.
339  // T257003: avoid Memcached::GET_EXTENDED; no tokens are needed and that requires "gets"
340  // https://github.com/libmemcached/libmemcached/blob/eda2becbec24363f56115fa5d16d38a2d1f54775/libmemcached/get.cc#L272
341  $resByRouteKey = $this->acquireSyncClient()->getMulti( $routeKeys ) ?: [];
342 
343  if ( is_array( $resByRouteKey ) ) {
344  $res = [];
345  foreach ( $resByRouteKey as $routeKey => $value ) {
346  $res[$this->stripRouteFromKey( $routeKey )] = $value;
347  }
348  } else {
349  $res = false;
350  }
351 
352  return $this->checkResult( false, $res );
353  }
354 
355  protected function doSetMulti( array $data, $exptime = 0, $flags = 0 ) {
356  $this->debug( 'setMulti(' . implode( ', ', array_keys( $data ) ) . ')' );
357 
358  $exptime = $this->fixExpiry( $exptime );
359  $dataByRouteKey = [];
360  foreach ( $data as $key => $value ) {
361  $dataByRouteKey[$this->validateKeyAndPrependRoute( $key )] = $value;
362  }
363 
364  // The PECL implementation is a naïve for-loop so use async I/O to pipeline;
365  // https://github.com/php-memcached-dev/php-memcached/blob/master/php_memcached.c#L1852
366  if ( $this->fieldHasFlags( $flags, self::WRITE_BACKGROUND ) ) {
367  $client = $this->acquireAsyncClient();
368  // Ignore "failed to set" warning from php-memcached 3.x (T251450)
369  // phpcs:ignore Generic.PHP.NoSilencedErrors.Discouraged
370  $result = @$client->setMulti( $dataByRouteKey, $exptime );
371  $this->releaseAsyncClient( $client );
372  } else {
373  $client = $this->acquireSyncClient();
374  // Ignore "failed to set" warning from php-memcached 3.x (T251450)
375  // phpcs:ignore Generic.PHP.NoSilencedErrors.Discouraged
376  $result = @$client->setMulti( $dataByRouteKey, $exptime );
377  }
378 
379  return $this->checkResult( false, $result );
380  }
381 
382  protected function doDeleteMulti( array $keys, $flags = 0 ) {
383  $this->debug( 'deleteMulti(' . implode( ', ', $keys ) . ')' );
384 
385  $routeKeys = [];
386  foreach ( $keys as $key ) {
387  $routeKeys[] = $this->validateKeyAndPrependRoute( $key );
388  }
389 
390  // The PECL implementation is a naïve for-loop so use async I/O to pipeline;
391  // https://github.com/php-memcached-dev/php-memcached/blob/7443d16d02fb73cdba2e90ae282446f80969229c/php_memcached.c#L1852
392  if ( $this->fieldHasFlags( $flags, self::WRITE_BACKGROUND ) ) {
393  $client = $this->acquireAsyncClient();
394  $resultArray = $client->deleteMulti( $routeKeys ) ?: [];
395  $this->releaseAsyncClient( $client );
396  } else {
397  $resultArray = $this->acquireSyncClient()->deleteMulti( $routeKeys ) ?: [];
398  }
399 
400  $result = true;
401  foreach ( $resultArray as $code ) {
402  if ( !in_array( $code, [ true, Memcached::RES_NOTFOUND ], true ) ) {
403  // "Not found" is counted as success in our interface
404  $result = false;
405  }
406  }
407 
408  return $this->checkResult( false, $result );
409  }
410 
411  protected function doChangeTTL( $key, $exptime, $flags ) {
412  $this->debug( "touch($key)" );
413 
414  $routeKey = $this->validateKeyAndPrependRoute( $key );
415  $result = $this->acquireSyncClient()->touch( $routeKey, $this->fixExpiry( $exptime ) );
416 
417  return $this->checkResult( $key, $result );
418  }
419 
420  protected function serialize( $value ) {
421  if ( is_int( $value ) ) {
422  return $value;
423  }
424 
425  $serializer = $this->syncClient->getOption( Memcached::OPT_SERIALIZER );
426  if ( $serializer === Memcached::SERIALIZER_PHP ) {
427  return serialize( $value );
428  } elseif ( $serializer === Memcached::SERIALIZER_IGBINARY ) {
429  return igbinary_serialize( $value );
430  }
431 
432  throw new UnexpectedValueException( __METHOD__ . ": got serializer '$serializer'." );
433  }
434 
435  protected function unserialize( $value ) {
436  if ( $this->isInteger( $value ) ) {
437  return (int)$value;
438  }
439 
440  $serializer = $this->syncClient->getOption( Memcached::OPT_SERIALIZER );
441  if ( $serializer === Memcached::SERIALIZER_PHP ) {
442  return unserialize( $value );
443  } elseif ( $serializer === Memcached::SERIALIZER_IGBINARY ) {
444  return igbinary_unserialize( $value );
445  }
446 
447  throw new UnexpectedValueException( __METHOD__ . ": got serializer '$serializer'." );
448  }
449 
453  private function acquireSyncClient() {
454  if ( $this->syncClientIsBuffering ) {
455  throw new RuntimeException( "The main (unbuffered I/O) client is locked" );
456  }
457 
458  if ( $this->hasUnflushedChanges ) {
459  // Force a synchronous flush of async writes so that their changes are visible
460  $this->syncClient->fetch();
461  if ( $this->asyncClient ) {
462  $this->asyncClient->fetch();
463  }
464  $this->hasUnflushedChanges = false;
465  }
466 
467  return $this->syncClient;
468  }
469 
473  private function acquireAsyncClient() {
474  if ( $this->asyncClient ) {
475  return $this->asyncClient; // dedicated buffering instance
476  }
477 
478  // Modify the main instance to temporarily buffer writes
479  $this->syncClientIsBuffering = true;
480  $this->syncClient->setOptions( self::$OPTS_ASYNC_WRITES );
481 
482  return $this->syncClient;
483  }
484 
488  private function releaseAsyncClient( $client ) {
489  $this->hasUnflushedChanges = true;
490 
491  if ( !$this->asyncClient ) {
492  // This is the main instance; make it stop buffering writes again
493  $client->setOptions( self::$OPTS_SYNC_WRITES );
494  $this->syncClientIsBuffering = false;
495  }
496  }
497 }
MemcachedPeclBagOStuff\$hasUnflushedChanges
bool $hasUnflushedChanges
Whether the non-buffering client should be flushed before use.
Definition: MemcachedPeclBagOStuff.php:38
MediumSpecificBagOStuff\setLastError
setLastError( $err)
Set the "last error" registry.
Definition: MediumSpecificBagOStuff.php:830
MediumSpecificBagOStuff\isInteger
isInteger( $value)
Check if a value is an integer.
Definition: MediumSpecificBagOStuff.php:953
MemcachedPeclBagOStuff\doDeleteMulti
doDeleteMulti(array $keys, $flags=0)
Definition: MemcachedPeclBagOStuff.php:382
MemcachedPeclBagOStuff\incr
incr( $key, $value=1, $flags=0)
Increase stored value of $key by $value while preserving its TTL.
Definition: MemcachedPeclBagOStuff.php:259
MemcachedPeclBagOStuff\initializeClient
initializeClient(Memcached $client, array $params, array $options)
Initialize the client only if needed and reuse it otherwise.
Definition: MemcachedPeclBagOStuff.php:115
MemcachedPeclBagOStuff\acquireSyncClient
acquireSyncClient()
Definition: MemcachedPeclBagOStuff.php:453
MemcachedPeclBagOStuff\acquireAsyncClient
acquireAsyncClient()
Definition: MemcachedPeclBagOStuff.php:473
MediumSpecificBagOStuff\guessSerialValueSize
guessSerialValueSize( $value, $depth=0, &$loops=0)
Estimate the size of a variable once serialized.
Definition: MediumSpecificBagOStuff.php:1056
MemcachedBagOStuff
Base class for memcached clients.
Definition: MemcachedBagOStuff.php:29
MediumSpecificBagOStuff\debug
debug( $text)
Definition: MediumSpecificBagOStuff.php:1180
MemcachedPeclBagOStuff\doAdd
doAdd( $key, $value, $exptime=0, $flags=0)
Insert an item if it does not already exist.
Definition: MemcachedPeclBagOStuff.php:246
MemcachedBagOStuff\stripRouteFromKey
stripRouteFromKey( $key)
Definition: MemcachedBagOStuff.php:132
MemcachedPeclBagOStuff\doChangeTTL
doChangeTTL( $key, $exptime, $flags)
Definition: MemcachedPeclBagOStuff.php:411
true
return true
Definition: router.php:90
MemcachedPeclBagOStuff\doSet
doSet( $key, $value, $exptime=0, $flags=0)
Set an item.
Definition: MemcachedPeclBagOStuff.php:207
MemcachedPeclBagOStuff\__construct
__construct( $params)
Available parameters are:
Definition: MemcachedPeclBagOStuff.php:67
MemcachedPeclBagOStuff\$OPTS_SYNC_WRITES
static array $OPTS_SYNC_WRITES
Memcached options.
Definition: MemcachedPeclBagOStuff.php:41
MemcachedPeclBagOStuff\checkResult
checkResult( $key, $result)
Check the return value from a client method call and take any necessary action.
Definition: MemcachedPeclBagOStuff.php:298
$res
$res
Definition: testCompression.php:57
MemcachedPeclBagOStuff\$asyncClient
Memcached null $asyncClient
Definition: MemcachedPeclBagOStuff.php:33
MemcachedPeclBagOStuff\doGetMulti
doGetMulti(array $keys, $flags=0)
Get an associative array containing the item for each of the keys that have items.
Definition: MemcachedPeclBagOStuff.php:330
MemcachedPeclBagOStuff\unserialize
unserialize( $value)
Definition: MemcachedPeclBagOStuff.php:435
MemcachedPeclBagOStuff\doGet
doGet( $key, $flags=0, &$casToken=null)
Definition: MemcachedPeclBagOStuff.php:180
MemcachedPeclBagOStuff\serialize
serialize( $value)
Definition: MemcachedPeclBagOStuff.php:420
MemcachedBagOStuff\fixExpiry
fixExpiry( $exptime)
Definition: MemcachedBagOStuff.php:149
MemcachedPeclBagOStuff\decr
decr( $key, $value=1, $flags=0)
Decrease stored value of $key by $value while preserving its TTL.
Definition: MemcachedPeclBagOStuff.php:268
MemcachedPeclBagOStuff\doSetMulti
doSetMulti(array $data, $exptime=0, $flags=0)
Definition: MemcachedPeclBagOStuff.php:355
Wikimedia\LightweightObjectStore\StorageAwareness\ERR_UNEXPECTED
const ERR_UNEXPECTED
Storage medium operation failed due to usage limitations or an I/O error.
Definition: StorageAwareness.php:40
MemcachedBagOStuff\validateKeyAndPrependRoute
validateKeyAndPrependRoute( $key)
Definition: MemcachedBagOStuff.php:114
MemcachedPeclBagOStuff\$OPTS_ASYNC_WRITES
static array $OPTS_ASYNC_WRITES
Memcached options.
Definition: MemcachedPeclBagOStuff.php:46
MemcachedPeclBagOStuff
A wrapper class for the PECL memcached client.
Definition: MemcachedPeclBagOStuff.php:29
BagOStuff\fieldHasFlags
fieldHasFlags( $field, $flags)
Definition: BagOStuff.php:584
$keys
$keys
Definition: testCompression.php:72
MemcachedPeclBagOStuff\$syncClient
Memcached $syncClient
Definition: MemcachedPeclBagOStuff.php:31
MemcachedPeclBagOStuff\releaseAsyncClient
releaseAsyncClient( $client)
Definition: MemcachedPeclBagOStuff.php:488
MemcachedPeclBagOStuff\doDelete
doDelete( $key, $flags=0)
Delete an item.
Definition: MemcachedPeclBagOStuff.php:233
MemcachedPeclBagOStuff\doCas
doCas( $casToken, $key, $value, $exptime=0, $flags=0)
Check and set an item.
Definition: MemcachedPeclBagOStuff.php:220
MemcachedPeclBagOStuff\setNewPreparedValues
setNewPreparedValues(array $valueByKey)
Make a "generic" reversible cache key from the given components.
Definition: MemcachedPeclBagOStuff.php:277
MemcachedPeclBagOStuff\$syncClientIsBuffering
bool $syncClientIsBuffering
Whether the non-buffering client is locked from use.
Definition: MemcachedPeclBagOStuff.php:36