MediaWiki  master
MemcachedPeclBagOStuff.php
Go to the documentation of this file.
1 <?php
31  protected $syncClient;
33  protected $asyncClient;
34 
36  protected $syncClientIsBuffering = false;
38  protected $hasUnflushedChanges = false;
39 
41  private static $OPTS_SYNC_WRITES = [
42  Memcached::OPT_NO_BLOCK => false, // async I/O (using TCP buffers)
43  Memcached::OPT_BUFFER_WRITES => false // libmemcached buffers
44  ];
46  private static $OPTS_ASYNC_WRITES = [
47  Memcached::OPT_NO_BLOCK => true, // async I/O (using TCP buffers)
48  Memcached::OPT_BUFFER_WRITES => true // libmemcached buffers
49  ];
50 
67  public function __construct( $params ) {
68  parent::__construct( $params );
69 
70  // Default class-specific parameters
71  $params += [
72  'compress_threshold' => 1500,
73  'connect_timeout' => 0.5,
74  'serializer' => 'php',
75  'use_binary_protocol' => false,
76  'allow_tcp_nagle_delay' => true
77  ];
78 
79  if ( $params['persistent'] ) {
80  // The pool ID must be unique to the server/option combination.
81  // The Memcached object is essentially shared for each pool ID.
82  // We can only reuse a pool ID if we keep the config consistent.
83  $connectionPoolId = md5( serialize( $params ) );
84  $syncClient = new Memcached( "$connectionPoolId-sync" );
85  // Avoid clobbering the main thread-shared Memcached instance
86  $asyncClient = new Memcached( "$connectionPoolId-async" );
87  } else {
88  $syncClient = new Memcached();
89  $asyncClient = null;
90  }
91 
92  $this->initializeClient( $syncClient, $params, self::$OPTS_SYNC_WRITES );
93  if ( $asyncClient ) {
94  $this->initializeClient( $asyncClient, $params, self::$OPTS_ASYNC_WRITES );
95  }
96 
97  // Set the main client and any dedicated one for buffered writes
98  $this->syncClient = $syncClient;
99  $this->asyncClient = $asyncClient;
100  // The compression threshold is an undocumented php.ini option for some
101  // reason. There's probably not much harm in setting it globally, for
102  // compatibility with the settings for the PHP client.
103  ini_set( 'memcached.compression_threshold', $params['compress_threshold'] );
104  }
105 
115  private function initializeClient( Memcached $client, array $params, array $options ) {
116  if ( $client->getServerList() ) {
117  $this->logger->debug( __METHOD__ . ": pre-initialized client instance." );
118 
119  return; // preserve persistent handle
120  }
121 
122  $this->logger->debug( __METHOD__ . ": initializing new client instance." );
123 
124  $options += [
125  Memcached::OPT_NO_BLOCK => false,
126  Memcached::OPT_BUFFER_WRITES => false,
127  // Network protocol (ASCII or binary)
128  Memcached::OPT_BINARY_PROTOCOL => $params['use_binary_protocol'],
129  // Set various network timeouts
130  Memcached::OPT_CONNECT_TIMEOUT => $params['connect_timeout'] * 1000,
131  Memcached::OPT_SEND_TIMEOUT => $params['timeout'],
132  Memcached::OPT_RECV_TIMEOUT => $params['timeout'],
133  Memcached::OPT_POLL_TIMEOUT => $params['timeout'] / 1000,
134  // Avoid pointless delay when sending/fetching large blobs
135  Memcached::OPT_TCP_NODELAY => !$params['allow_tcp_nagle_delay'],
136  // Set libketama mode since it's recommended by the documentation
137  Memcached::OPT_LIBKETAMA_COMPATIBLE => true
138  ];
139  if ( isset( $params['retry_timeout'] ) ) {
140  $options[Memcached::OPT_RETRY_TIMEOUT] = $params['retry_timeout'];
141  }
142  if ( isset( $params['server_failure_limit'] ) ) {
143  $options[Memcached::OPT_SERVER_FAILURE_LIMIT] = $params['server_failure_limit'];
144  }
145  if ( $params['serializer'] === 'php' ) {
146  $options[Memcached::OPT_SERIALIZER] = Memcached::SERIALIZER_PHP;
147  } elseif ( $params['serializer'] === 'igbinary' ) {
148  // @phan-suppress-next-line PhanImpossibleCondition
149  if ( !Memcached::HAVE_IGBINARY ) {
150  throw new RuntimeException(
151  __CLASS__ . ': the igbinary extension is not available ' .
152  'but igbinary serialization was requested.'
153  );
154  }
155  $options[Memcached::OPT_SERIALIZER] = Memcached::SERIALIZER_IGBINARY;
156  }
157 
158  if ( !$client->setOptions( $options ) ) {
159  throw new RuntimeException(
160  "Invalid options: " . json_encode( $options, JSON_PRETTY_PRINT )
161  );
162  }
163 
164  $servers = [];
165  foreach ( $params['servers'] as $host ) {
166  if ( preg_match( '/^\[(.+)\]:(\d+)$/', $host, $m ) ) {
167  $servers[] = [ $m[1], (int)$m[2] ]; // (ip, port)
168  } elseif ( preg_match( '/^([^:]+):(\d+)$/', $host, $m ) ) {
169  $servers[] = [ $m[1], (int)$m[2] ]; // (ip or path, port)
170  } else {
171  $servers[] = [ $host, false ]; // (ip or path, port)
172  }
173  }
174 
175  if ( !$client->addServers( $servers ) ) {
176  throw new RuntimeException( "Failed to inject server address list" );
177  }
178  }
179 
180  protected function doGet( $key, $flags = 0, &$casToken = null ) {
181  $this->debug( "get($key)" );
182 
183  $client = $this->acquireSyncClient();
184  if ( defined( Memcached::class . '::GET_EXTENDED' ) ) { // v3.0.0
186  $flags = Memcached::GET_EXTENDED;
187  $res = $client->get( $this->validateKeyEncoding( $key ), null, $flags );
188  if ( is_array( $res ) ) {
189  $result = $res['value'];
190  $casToken = $res['cas'];
191  } else {
192  $result = false;
193  $casToken = null;
194  }
195  } else {
196  $result = $client->get( $this->validateKeyEncoding( $key ), null, $casToken );
197  }
198 
199  return $this->checkResult( $key, $result );
200  }
201 
202  protected function doSet( $key, $value, $exptime = 0, $flags = 0 ) {
203  $this->debug( "set($key)" );
204 
205  $client = $this->acquireSyncClient();
206  $result = $client->set(
207  $this->validateKeyEncoding( $key ),
208  $value,
209  $this->fixExpiry( $exptime )
210  );
211 
212  return ( $result === false && $client->getResultCode() === Memcached::RES_NOTSTORED )
213  // "Not stored" is always used as the mcrouter response with AllAsyncRoute
214  ? true
215  : $this->checkResult( $key, $result );
216  }
217 
218  protected function doCas( $casToken, $key, $value, $exptime = 0, $flags = 0 ) {
219  $this->debug( "cas($key)" );
220 
221  $result = $this->acquireSyncClient()->cas(
222  $casToken,
223  $this->validateKeyEncoding( $key ),
224  $value, $this->fixExpiry( $exptime )
225  );
226 
227  return $this->checkResult( $key, $result );
228  }
229 
230  protected function doDelete( $key, $flags = 0 ) {
231  $this->debug( "delete($key)" );
232 
233  $client = $this->acquireSyncClient();
234  $result = $client->delete( $this->validateKeyEncoding( $key ) );
235 
236  return ( $result === false && $client->getResultCode() === Memcached::RES_NOTFOUND )
237  // "Not found" is counted as success in our interface
238  ? true
239  : $this->checkResult( $key, $result );
240  }
241 
242  protected function doAdd( $key, $value, $exptime = 0, $flags = 0 ) {
243  $this->debug( "add($key)" );
244 
245  $result = $this->acquireSyncClient()->add(
246  $this->validateKeyEncoding( $key ),
247  $value,
248  $this->fixExpiry( $exptime )
249  );
250 
251  return $this->checkResult( $key, $result );
252  }
253 
254  public function incr( $key, $value = 1, $flags = 0 ) {
255  $this->debug( "incr($key)" );
256 
257  $result = $this->acquireSyncClient()->increment( $key, $value );
258 
259  return $this->checkResult( $key, $result );
260  }
261 
262  public function decr( $key, $value = 1, $flags = 0 ) {
263  $this->debug( "decr($key)" );
264 
265  $result = $this->acquireSyncClient()->decrement( $key, $value );
266 
267  return $this->checkResult( $key, $result );
268  }
269 
270  public function setNewPreparedValues( array $valueByKey ) {
271  // The PECL driver does the serializing and will not reuse anything from here
272  $sizes = [];
273  foreach ( $valueByKey as $value ) {
274  $sizes[] = $this->guessSerialValueSize( $value );
275  }
276 
277  return $sizes;
278  }
279 
291  protected function checkResult( $key, $result ) {
292  if ( $result !== false ) {
293  return $result;
294  }
295 
296  $client = $this->syncClient;
297  switch ( $client->getResultCode() ) {
298  case Memcached::RES_SUCCESS:
299  break;
300  case Memcached::RES_DATA_EXISTS:
301  case Memcached::RES_NOTSTORED:
302  case Memcached::RES_NOTFOUND:
303  $this->debug( "result: " . $client->getResultMessage() );
304  break;
305  default:
306  $msg = $client->getResultMessage();
307  $logCtx = [];
308  if ( $key !== false ) {
309  $server = $client->getServerByKey( $key );
310  $logCtx['memcached-server'] = "{$server['host']}:{$server['port']}";
311  $logCtx['memcached-key'] = $key;
312  $msg = "Memcached error for key \"{memcached-key}\" " .
313  "on server \"{memcached-server}\": $msg";
314  } else {
315  $msg = "Memcached error: $msg";
316  }
317  $this->logger->error( $msg, $logCtx );
318  $this->setLastError( BagOStuff::ERR_UNEXPECTED );
319  }
320  return $result;
321  }
322 
323  protected function doGetMulti( array $keys, $flags = 0 ) {
324  $this->debug( 'getMulti(' . implode( ', ', $keys ) . ')' );
325 
326  foreach ( $keys as $key ) {
327  $this->validateKeyEncoding( $key );
328  }
329 
330  // The PECL implementation uses "gets" which works as well as a pipeline
331  $result = $this->acquireSyncClient()->getMulti( $keys ) ?: [];
332 
333  return $this->checkResult( false, $result );
334  }
335 
336  protected function doSetMulti( array $data, $exptime = 0, $flags = 0 ) {
337  $this->debug( 'setMulti(' . implode( ', ', array_keys( $data ) ) . ')' );
338 
339  $exptime = $this->fixExpiry( $exptime );
340  foreach ( array_keys( $data ) as $key ) {
341  $this->validateKeyEncoding( $key );
342  }
343 
344  // The PECL implementation is a naïve for-loop so use async I/O to pipeline;
345  // https://github.com/php-memcached-dev/php-memcached/blob/master/php_memcached.c#L1852
346  if ( $this->fieldHasFlags( $flags, self::WRITE_BACKGROUND ) ) {
347  $client = $this->acquireAsyncClient();
348  $result = $client->setMulti( $data, $exptime );
349  $this->releaseAsyncClient( $client );
350  } else {
351  $result = $this->acquireSyncClient()->setMulti( $data, $exptime );
352  }
353 
354  return $this->checkResult( false, $result );
355  }
356 
357  protected function doDeleteMulti( array $keys, $flags = 0 ) {
358  $this->debug( 'deleteMulti(' . implode( ', ', $keys ) . ')' );
359 
360  foreach ( $keys as $key ) {
361  $this->validateKeyEncoding( $key );
362  }
363 
364  // The PECL implementation is a naïve for-loop so use async I/O to pipeline;
365  // https://github.com/php-memcached-dev/php-memcached/blob/7443d16d02fb73cdba2e90ae282446f80969229c/php_memcached.c#L1852
366  if ( $this->fieldHasFlags( $flags, self::WRITE_BACKGROUND ) ) {
367  $client = $this->acquireAsyncClient();
368  $resultArray = $client->deleteMulti( $keys ) ?: [];
369  $this->releaseAsyncClient( $client );
370  } else {
371  $resultArray = $this->acquireSyncClient()->deleteMulti( $keys ) ?: [];
372  }
373 
374  $result = true;
375  foreach ( $resultArray as $code ) {
376  if ( !in_array( $code, [ true, Memcached::RES_NOTFOUND ], true ) ) {
377  // "Not found" is counted as success in our interface
378  $result = false;
379  }
380  }
381 
382  return $this->checkResult( false, $result );
383  }
384 
385  protected function doChangeTTL( $key, $exptime, $flags ) {
386  $this->debug( "touch($key)" );
387 
388  $result = $this->acquireSyncClient()->touch( $key, $this->fixExpiry( $exptime ) );
389 
390  return $this->checkResult( $key, $result );
391  }
392 
393  protected function serialize( $value ) {
394  if ( is_int( $value ) ) {
395  return $value;
396  }
397 
398  $serializer = $this->syncClient->getOption( Memcached::OPT_SERIALIZER );
399  if ( $serializer === Memcached::SERIALIZER_PHP ) {
400  return serialize( $value );
401  } elseif ( $serializer === Memcached::SERIALIZER_IGBINARY ) {
402  return igbinary_serialize( $value );
403  }
404 
405  throw new UnexpectedValueException( __METHOD__ . ": got serializer '$serializer'." );
406  }
407 
408  protected function unserialize( $value ) {
409  if ( $this->isInteger( $value ) ) {
410  return (int)$value;
411  }
412 
413  $serializer = $this->syncClient->getOption( Memcached::OPT_SERIALIZER );
414  if ( $serializer === Memcached::SERIALIZER_PHP ) {
415  return unserialize( $value );
416  } elseif ( $serializer === Memcached::SERIALIZER_IGBINARY ) {
417  return igbinary_unserialize( $value );
418  }
419 
420  throw new UnexpectedValueException( __METHOD__ . ": got serializer '$serializer'." );
421  }
422 
426  private function acquireSyncClient() {
427  if ( $this->syncClientIsBuffering ) {
428  throw new RuntimeException( "The main (unbuffered I/O) client is locked" );
429  }
430 
431  if ( $this->hasUnflushedChanges ) {
432  // Force a synchronous flush of async writes so that their changes are visible
433  $this->syncClient->fetch();
434  if ( $this->asyncClient ) {
435  $this->asyncClient->fetch();
436  }
437  $this->hasUnflushedChanges = false;
438  }
439 
440  return $this->syncClient;
441  }
442 
446  private function acquireAsyncClient() {
447  if ( $this->asyncClient ) {
448  return $this->asyncClient; // dedicated buffering instance
449  }
450 
451  // Modify the main instance to temporarily buffer writes
452  $this->syncClientIsBuffering = true;
453  $this->syncClient->setOptions( self::$OPTS_ASYNC_WRITES );
454 
455  return $this->syncClient;
456  }
457 
461  private function releaseAsyncClient( $client ) {
462  $this->hasUnflushedChanges = true;
463 
464  if ( !$this->asyncClient ) {
465  // This is the main instance; make it stop buffering writes again
466  $client->setOptions( self::$OPTS_SYNC_WRITES );
467  $this->syncClientIsBuffering = false;
468  }
469  }
470 }
MemcachedPeclBagOStuff\$hasUnflushedChanges
bool $hasUnflushedChanges
Whether the non-buffering client should be flushed before use.
Definition: MemcachedPeclBagOStuff.php:38
MediumSpecificBagOStuff\setLastError
setLastError( $err)
Set the "last error" registry.
Definition: MediumSpecificBagOStuff.php:755
MediumSpecificBagOStuff\isInteger
isInteger( $value)
Check if a value is an integer.
Definition: MediumSpecificBagOStuff.php:878
MemcachedPeclBagOStuff\doDeleteMulti
doDeleteMulti(array $keys, $flags=0)
Definition: MemcachedPeclBagOStuff.php:357
MemcachedPeclBagOStuff\incr
incr( $key, $value=1, $flags=0)
Increase stored value of $key by $value while preserving its TTL.
Definition: MemcachedPeclBagOStuff.php:254
MemcachedPeclBagOStuff\initializeClient
initializeClient(Memcached $client, array $params, array $options)
Initialize the client only if needed and reuse it otherwise.
Definition: MemcachedPeclBagOStuff.php:115
MemcachedPeclBagOStuff\acquireSyncClient
acquireSyncClient()
Definition: MemcachedPeclBagOStuff.php:426
MemcachedPeclBagOStuff\acquireAsyncClient
acquireAsyncClient()
Definition: MemcachedPeclBagOStuff.php:446
MediumSpecificBagOStuff\guessSerialValueSize
guessSerialValueSize( $value, $depth=0, &$loops=0)
Estimate the size of a variable once serialized.
Definition: MediumSpecificBagOStuff.php:996
MemcachedBagOStuff
Base class for memcached clients.
Definition: MemcachedBagOStuff.php:29
MediumSpecificBagOStuff\debug
debug( $text)
Definition: MediumSpecificBagOStuff.php:1061
MemcachedPeclBagOStuff\doAdd
doAdd( $key, $value, $exptime=0, $flags=0)
Insert an item if it does not already exist.
Definition: MemcachedPeclBagOStuff.php:242
MemcachedPeclBagOStuff\doChangeTTL
doChangeTTL( $key, $exptime, $flags)
Definition: MemcachedPeclBagOStuff.php:385
true
return true
Definition: router.php:90
MemcachedPeclBagOStuff\doSet
doSet( $key, $value, $exptime=0, $flags=0)
Set an item.
Definition: MemcachedPeclBagOStuff.php:202
MemcachedPeclBagOStuff\__construct
__construct( $params)
Available parameters are:
Definition: MemcachedPeclBagOStuff.php:67
MemcachedPeclBagOStuff\$OPTS_SYNC_WRITES
static array $OPTS_SYNC_WRITES
Memcached options.
Definition: MemcachedPeclBagOStuff.php:41
MemcachedPeclBagOStuff\checkResult
checkResult( $key, $result)
Check the return value from a client method call and take any necessary action.
Definition: MemcachedPeclBagOStuff.php:291
$res
$res
Definition: testCompression.php:57
MemcachedPeclBagOStuff\$asyncClient
Memcached null $asyncClient
Definition: MemcachedPeclBagOStuff.php:33
MemcachedPeclBagOStuff\doGetMulti
doGetMulti(array $keys, $flags=0)
Get an associative array containing the item for each of the keys that have items.
Definition: MemcachedPeclBagOStuff.php:323
MemcachedPeclBagOStuff\unserialize
unserialize( $value)
Definition: MemcachedPeclBagOStuff.php:408
MemcachedPeclBagOStuff\doGet
doGet( $key, $flags=0, &$casToken=null)
Definition: MemcachedPeclBagOStuff.php:180
MemcachedPeclBagOStuff\serialize
serialize( $value)
Definition: MemcachedPeclBagOStuff.php:393
MemcachedBagOStuff\fixExpiry
fixExpiry( $exptime)
Definition: MemcachedBagOStuff.php:98
MemcachedPeclBagOStuff\decr
decr( $key, $value=1, $flags=0)
Decrease stored value of $key by $value while preserving its TTL.
Definition: MemcachedPeclBagOStuff.php:262
MemcachedPeclBagOStuff\doSetMulti
doSetMulti(array $data, $exptime=0, $flags=0)
Definition: MemcachedPeclBagOStuff.php:336
MemcachedPeclBagOStuff\$OPTS_ASYNC_WRITES
static array $OPTS_ASYNC_WRITES
Memcached options.
Definition: MemcachedPeclBagOStuff.php:46
MemcachedPeclBagOStuff
A wrapper class for the PECL memcached client.
Definition: MemcachedPeclBagOStuff.php:29
BagOStuff\fieldHasFlags
fieldHasFlags( $field, $flags)
Definition: BagOStuff.php:509
$keys
$keys
Definition: testCompression.php:72
MemcachedPeclBagOStuff\$syncClient
Memcached $syncClient
Definition: MemcachedPeclBagOStuff.php:31
MemcachedPeclBagOStuff\releaseAsyncClient
releaseAsyncClient( $client)
Definition: MemcachedPeclBagOStuff.php:461
MemcachedBagOStuff\validateKeyEncoding
validateKeyEncoding( $key)
Ensure that a key is safe to use (contains no control characters and no characters above the ASCII ra...
Definition: MemcachedBagOStuff.php:87
MemcachedPeclBagOStuff\doDelete
doDelete( $key, $flags=0)
Delete an item.
Definition: MemcachedPeclBagOStuff.php:230
MemcachedPeclBagOStuff\doCas
doCas( $casToken, $key, $value, $exptime=0, $flags=0)
Check and set an item.
Definition: MemcachedPeclBagOStuff.php:218
MemcachedPeclBagOStuff\setNewPreparedValues
setNewPreparedValues(array $valueByKey)
Prepare values for storage and get their serialized sizes, or, estimate those sizes.
Definition: MemcachedPeclBagOStuff.php:270
MemcachedPeclBagOStuff\$syncClientIsBuffering
bool $syncClientIsBuffering
Whether the non-buffering client is locked from use.
Definition: MemcachedPeclBagOStuff.php:36