MediaWiki master
MemcachedPeclBagOStuff.php
Go to the documentation of this file.
1<?php
20namespace Wikimedia\ObjectCache;
21
22use Memcached;
23use RuntimeException;
24use UnexpectedValueException;
25use Wikimedia\ScopedCallback;
26
37 protected $client;
38
56 public function __construct( $params ) {
57 parent::__construct( $params );
58
59 // Default class-specific parameters
60 $params += [
61 'compress_threshold' => 1500,
62 'connect_timeout' => 0.5,
63 'timeout' => 500_000,
64 'serializer' => 'php',
65 'use_binary_protocol' => false,
66 'allow_tcp_nagle_delay' => true
67 ];
68
69 if ( $params['persistent'] ) {
70 // The pool ID must be unique to the server/option combination.
71 // The Memcached object is essentially shared for each pool ID.
72 // We can only reuse a pool ID if we keep the config consistent.
73 $connectionPoolId = md5( serialize( $params ) );
74 $client = new Memcached( $connectionPoolId );
75 } else {
76 $client = new Memcached();
77 }
78
79 $this->initializeClient( $client, $params );
80
81 $this->client = $client;
82 // The compression threshold is an undocumented php.ini option for some
83 // reason. There's probably not much harm in setting it globally, for
84 // compatibility with the settings for the PHP client.
85 ini_set( 'memcached.compression_threshold', $params['compress_threshold'] );
86 }
87
97 private function initializeClient( Memcached $client, array $params ) {
98 if ( $client->getServerList() ) {
99 $this->logger->debug( __METHOD__ . ": pre-initialized client instance." );
100
101 return; // preserve persistent handle
102 }
103
104 $this->logger->debug( __METHOD__ . ": initializing new client instance." );
105
106 $options = [
107 Memcached::OPT_NO_BLOCK => false,
108 Memcached::OPT_BUFFER_WRITES => false,
109 Memcached::OPT_NOREPLY => false,
110 // Network protocol (ASCII or binary)
111 Memcached::OPT_BINARY_PROTOCOL => $params['use_binary_protocol'],
112 // Set various network timeouts
113 Memcached::OPT_CONNECT_TIMEOUT => $params['connect_timeout'] * 1000,
114 Memcached::OPT_SEND_TIMEOUT => $params['timeout'],
115 Memcached::OPT_RECV_TIMEOUT => $params['timeout'],
116 Memcached::OPT_POLL_TIMEOUT => $params['timeout'] / 1000,
117 // Avoid pointless delay when sending/fetching large blobs
118 Memcached::OPT_TCP_NODELAY => !$params['allow_tcp_nagle_delay'],
119 // Set libketama mode since it's recommended by the documentation
120 Memcached::OPT_LIBKETAMA_COMPATIBLE => true
121 ];
122 if ( isset( $params['retry_timeout'] ) ) {
123 $options[Memcached::OPT_RETRY_TIMEOUT] = $params['retry_timeout'];
124 }
125 if ( isset( $params['server_failure_limit'] ) ) {
126 $options[Memcached::OPT_SERVER_FAILURE_LIMIT] = $params['server_failure_limit'];
127 }
128 if ( $params['serializer'] === 'php' ) {
129 $options[Memcached::OPT_SERIALIZER] = Memcached::SERIALIZER_PHP;
130 } elseif ( $params['serializer'] === 'igbinary' ) {
131 // @phan-suppress-next-line PhanImpossibleCondition
132 if ( !Memcached::HAVE_IGBINARY ) {
133 throw new RuntimeException(
134 __CLASS__ . ': the igbinary extension is not available ' .
135 'but igbinary serialization was requested.'
136 );
137 }
138 $options[Memcached::OPT_SERIALIZER] = Memcached::SERIALIZER_IGBINARY;
139 }
140
141 if ( !$client->setOptions( $options ) ) {
142 throw new RuntimeException(
143 "Invalid options: " . json_encode( $options, JSON_PRETTY_PRINT )
144 );
145 }
146
147 $servers = [];
148 foreach ( $params['servers'] as $host ) {
149 if ( preg_match( '/^\[(.+)\]:(\d+)$/', $host, $m ) ) {
150 $servers[] = [ $m[1], (int)$m[2] ]; // (ip, port)
151 } elseif ( preg_match( '/^([^:]+):(\d+)$/', $host, $m ) ) {
152 $servers[] = [ $m[1], (int)$m[2] ]; // (ip or path, port)
153 } else {
154 $servers[] = [ $host, false ]; // (ip or path, port)
155 }
156 }
157
158 if ( !$client->addServers( $servers ) ) {
159 throw new RuntimeException( "Failed to inject server address list" );
160 }
161 }
162
172 private function noReplyScope( $flags ) {
173 if ( $flags !== true && !( $flags & self::WRITE_BACKGROUND ) ) {
174 return null;
175 }
177 $client->setOption( Memcached::OPT_NOREPLY, true );
178
179 return new ScopedCallback( static function () use ( $client ) {
180 $client->setOption( Memcached::OPT_NOREPLY, false );
181 } );
182 }
183
184 protected function doGet( $key, $flags = 0, &$casToken = null ) {
185 $getToken = ( $casToken === self::PASS_BY_REF );
186 $casToken = null;
187
188 $this->debug( "get($key)" );
189
190 $routeKey = $this->validateKeyAndPrependRoute( $key );
191
192 // T257003: only require "gets" (instead of "get") when a CAS token is needed
193 if ( $getToken ) {
195 $flags = Memcached::GET_EXTENDED;
196 $res = $this->client->get( $routeKey, null, $flags );
197 if ( is_array( $res ) ) {
198 $result = $res['value'];
199 $casToken = $res['cas'];
200 } else {
201 $result = false;
202 }
203 } else {
204 $result = $this->client->get( $routeKey );
205 }
206
207 return $this->checkResult( $key, $result );
208 }
209
210 protected function doSet( $key, $value, $exptime = 0, $flags = 0 ) {
211 $this->debug( "set($key)" );
212
213 $routeKey = $this->validateKeyAndPrependRoute( $key );
214
215 $noReplyScope = $this->noReplyScope( $flags );
216 $result = $this->client->set( $routeKey, $value, $this->fixExpiry( $exptime ) );
217 ScopedCallback::consume( $noReplyScope );
218
219 return ( !$result && $this->client->getResultCode() === Memcached::RES_NOTSTORED )
220 // "Not stored" is always used as the mcrouter response with AllAsyncRoute
221 ? true
222 : $this->checkResult( $key, $result );
223 }
224
225 protected function doCas( $casToken, $key, $value, $exptime = 0, $flags = 0 ) {
226 $this->debug( "cas($key)" );
227
228 $routeKey = $this->validateKeyAndPrependRoute( $key );
229 $result = $this->client->cas(
230 $casToken,
231 $routeKey,
232 $value, $this->fixExpiry( $exptime )
233 );
234
235 return $this->checkResult( $key, $result );
236 }
237
238 protected function doDelete( $key, $flags = 0 ) {
239 $this->debug( "delete($key)" );
240
241 $routeKey = $this->validateKeyAndPrependRoute( $key );
242 $noReplyScope = $this->noReplyScope( $flags );
243 $result = $this->client->delete( $routeKey );
244 ScopedCallback::consume( $noReplyScope );
245
246 return ( !$result && $this->client->getResultCode() === Memcached::RES_NOTFOUND )
247 // "Not found" is counted as success in our interface
248 ? true
249 : $this->checkResult( $key, $result );
250 }
251
252 protected function doAdd( $key, $value, $exptime = 0, $flags = 0 ) {
253 $this->debug( "add($key)" );
254
255 $routeKey = $this->validateKeyAndPrependRoute( $key );
256 $noReplyScope = $this->noReplyScope( $flags );
257 $result = $this->client->add(
258 $routeKey,
259 $value,
260 $this->fixExpiry( $exptime )
261 );
262 ScopedCallback::consume( $noReplyScope );
263
264 return $this->checkResult( $key, $result );
265 }
266
267 protected function doIncrWithInitAsync( $key, $exptime, $step, $init ) {
268 $this->debug( "incrWithInit($key)" );
269 $routeKey = $this->validateKeyAndPrependRoute( $key );
270 $watchPoint = $this->watchErrors();
271 $scope = $this->noReplyScope( true );
272 $this->checkResult( $key, $this->client->add( $routeKey, $init - $step, $this->fixExpiry( $exptime ) ) );
273 $this->checkResult( $key, $this->client->increment( $routeKey, $step ) );
274 ScopedCallback::consume( $scope );
275 $lastError = $this->getLastError( $watchPoint );
276
277 return !$lastError;
278 }
279
280 protected function doIncrWithInitSync( $key, $exptime, $step, $init ) {
281 $this->debug( "incrWithInit($key)" );
282 $routeKey = $this->validateKeyAndPrependRoute( $key );
283 $watchPoint = $this->watchErrors();
284 $result = $this->client->increment( $routeKey, $step );
285 $newValue = $this->checkResult( $key, $result );
286 if ( $newValue === false && !$this->getLastError( $watchPoint ) ) {
287 // No key set; initialize
288 $result = $this->client->add( $routeKey, $init, $this->fixExpiry( $exptime ) );
289 $newValue = $this->checkResult( $key, $result ) ? $init : false;
290 if ( $newValue === false && !$this->getLastError( $watchPoint ) ) {
291 // Raced out initializing; increment
292 $result = $this->client->increment( $routeKey, $step );
293 $newValue = $this->checkResult( $key, $result );
294 }
295 }
296
297 return $newValue;
298 }
299
312 protected function checkResult( $key, $result ) {
313 static $statusByCode = [
314 Memcached::RES_HOST_LOOKUP_FAILURE => self::ERR_UNREACHABLE,
315 Memcached::RES_SERVER_MARKED_DEAD => self::ERR_UNREACHABLE,
316 Memcached::RES_SERVER_TEMPORARILY_DISABLED => self::ERR_UNREACHABLE,
317 Memcached::RES_UNKNOWN_READ_FAILURE => self::ERR_NO_RESPONSE,
318 Memcached::RES_WRITE_FAILURE => self::ERR_NO_RESPONSE,
319 Memcached::RES_PARTIAL_READ => self::ERR_NO_RESPONSE,
320 // Hard-code values that only exist in recent versions of the PECL extension.
321 // https://github.com/JetBrains/phpstorm-stubs/blob/master/memcached/memcached.php
322 3 /* Memcached::RES_CONNECTION_FAILURE */ => self::ERR_UNREACHABLE,
323 27 /* Memcached::RES_FAIL_UNIX_SOCKET */ => self::ERR_UNREACHABLE,
324 6 /* Memcached::RES_READ_FAILURE */ => self::ERR_NO_RESPONSE
325 ];
326
327 if ( $result !== false ) {
328 return $result;
329 }
330
332 $code = $client->getResultCode();
333 switch ( $code ) {
334 case Memcached::RES_SUCCESS:
335 break;
336 case Memcached::RES_DATA_EXISTS:
337 case Memcached::RES_NOTSTORED:
338 case Memcached::RES_NOTFOUND:
339 $this->debug( "result: " . $client->getResultMessage() );
340 break;
341 default:
342 $msg = $client->getResultMessage();
343 $logCtx = [];
344 if ( $key !== false ) {
345 $server = $client->getServerByKey( $key );
346 $logCtx['memcached-server'] = "{$server['host']}:{$server['port']}";
347 $logCtx['memcached-key'] = $key;
348 $msg = "Memcached error for key \"{memcached-key}\" " .
349 "on server \"{memcached-server}\": $msg";
350 } else {
351 $msg = "Memcached error: $msg";
352 }
353 $this->logger->error( $msg, $logCtx );
354 $this->setLastError( $statusByCode[$code] ?? self::ERR_UNEXPECTED );
355 }
356
357 return $result;
358 }
359
360 protected function doGetMulti( array $keys, $flags = 0 ) {
361 $this->debug( 'getMulti(' . implode( ', ', $keys ) . ')' );
362
363 $routeKeys = [];
364 foreach ( $keys as $key ) {
365 $routeKeys[] = $this->validateKeyAndPrependRoute( $key );
366 }
367
368 // The PECL implementation uses multi-key "get"/"gets"; no need to pipeline.
369 // T257003: avoid Memcached::GET_EXTENDED; no tokens are needed and that requires "gets"
370 // https://github.com/libmemcached/libmemcached/blob/eda2becbec24363f56115fa5d16d38a2d1f54775/libmemcached/get.cc#L272
371 $resByRouteKey = $this->client->getMulti( $routeKeys );
372
373 if ( is_array( $resByRouteKey ) ) {
374 $res = [];
375 foreach ( $resByRouteKey as $routeKey => $value ) {
376 $res[$this->stripRouteFromKey( $routeKey )] = $value;
377 }
378 } else {
379 $res = false;
380 }
381
382 $res = $this->checkResult( false, $res );
383
384 return $res !== false ? $res : [];
385 }
386
387 protected function doSetMulti( array $data, $exptime = 0, $flags = 0 ) {
388 $this->debug( 'setMulti(' . implode( ', ', array_keys( $data ) ) . ')' );
389
390 $exptime = $this->fixExpiry( $exptime );
391 $dataByRouteKey = [];
392 foreach ( $data as $key => $value ) {
393 $dataByRouteKey[$this->validateKeyAndPrependRoute( $key )] = $value;
394 }
395
396 $noReplyScope = $this->noReplyScope( $flags );
397
398 // Ignore "failed to set" warning from php-memcached 3.x (T251450)
399 // phpcs:ignore Generic.PHP.NoSilencedErrors.Discouraged
400 $result = @$this->client->setMulti( $dataByRouteKey, $exptime );
401 ScopedCallback::consume( $noReplyScope );
402
403 return $this->checkResult( false, $result );
404 }
405
406 protected function doDeleteMulti( array $keys, $flags = 0 ) {
407 $this->debug( 'deleteMulti(' . implode( ', ', $keys ) . ')' );
408
409 $routeKeys = [];
410 foreach ( $keys as $key ) {
411 $routeKeys[] = $this->validateKeyAndPrependRoute( $key );
412 }
413
414 $noReplyScope = $this->noReplyScope( $flags );
415 $resultArray = $this->client->deleteMulti( $routeKeys ) ?: [];
416 ScopedCallback::consume( $noReplyScope );
417
418 $result = true;
419 foreach ( $resultArray as $code ) {
420 if ( !in_array( $code, [ true, Memcached::RES_NOTFOUND ], true ) ) {
421 // "Not found" is counted as success in our interface
422 $result = false;
423 }
424 }
425
426 return $this->checkResult( false, $result );
427 }
428
429 protected function doChangeTTL( $key, $exptime, $flags ) {
430 $this->debug( "touch($key)" );
431
432 $routeKey = $this->validateKeyAndPrependRoute( $key );
433 // Avoid NO_REPLY due to libmemcached hang
434 // https://phabricator.wikimedia.org/T310662#8031692
435 $result = $this->client->touch( $routeKey, $this->fixExpiry( $exptime ) );
436
437 return $this->checkResult( $key, $result );
438 }
439
440 protected function serialize( $value ) {
441 if ( is_int( $value ) ) {
442 return $value;
443 }
444
445 $serializer = $this->client->getOption( Memcached::OPT_SERIALIZER );
446 if ( $serializer === Memcached::SERIALIZER_PHP ) {
447 return serialize( $value );
448 } elseif ( $serializer === Memcached::SERIALIZER_IGBINARY ) {
449 return igbinary_serialize( $value );
450 }
451
452 throw new UnexpectedValueException( __METHOD__ . ": got serializer '$serializer'." );
453 }
454
455 protected function unserialize( $value ) {
456 if ( $this->isInteger( $value ) ) {
457 return (int)$value;
458 }
459
460 $serializer = $this->client->getOption( Memcached::OPT_SERIALIZER );
461 if ( $serializer === Memcached::SERIALIZER_PHP ) {
462 return unserialize( $value );
463 } elseif ( $serializer === Memcached::SERIALIZER_IGBINARY ) {
464 return igbinary_unserialize( $value );
465 }
466
467 throw new UnexpectedValueException( __METHOD__ . ": got serializer '$serializer'." );
468 }
469}
470
472class_alias( MemcachedPeclBagOStuff::class, 'MemcachedPeclBagOStuff' );
const ERR_NO_RESPONSE
Storage operation failed to yield a complete response.
setLastError( $error)
Set the "last error" registry due to a problem encountered during an attempted operation.
getLastError( $watchPoint=0)
Get the "last error" registry.
watchErrors()
Get a "watch point" token that can be used to get the "last error" to occur after now.
const ERR_UNREACHABLE
Storage operation could not establish a connection.
int $lastError
BagOStuff:ERR_* constant of the last error that occurred.
isInteger( $value)
Check if a value is an integer.
const PASS_BY_REF
Idiom for doGet() to return extra information by reference.
Store data in a memcached server or memcached cluster.
Store data on memcached server(s) via the php-memcached PECL extension.
doAdd( $key, $value, $exptime=0, $flags=0)
Insert an item if it does not already exist.
doSet( $key, $value, $exptime=0, $flags=0)
Set an item.
doCas( $casToken, $key, $value, $exptime=0, $flags=0)
Set an item if the current CAS token matches the provided CAS token.
checkResult( $key, $result)
Check the return value from a client method call and take any necessary action.
__construct( $params)
Available parameters are:
doGetMulti(array $keys, $flags=0)
Get an associative array containing the item for each of the keys that have items.
doGet( $key, $flags=0, &$casToken=null)
Get an item.