Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
44.71% |
76 / 170 |
|
33.33% |
6 / 18 |
CRAP | |
0.00% |
0 / 1 |
| MultiWriteBagOStuff | |
44.97% |
76 / 169 |
|
33.33% |
6 / 18 |
382.45 | |
0.00% |
0 / 1 |
| __construct | |
76.47% |
13 / 17 |
|
0.00% |
0 / 1 |
7.64 | |||
| get | |
0.00% |
0 / 32 |
|
0.00% |
0 / 1 |
56 | |||
| set | |
100.00% |
7 / 7 |
|
100.00% |
1 / 1 |
1 | |||
| delete | |
100.00% |
7 / 7 |
|
100.00% |
1 / 1 |
1 | |||
| add | |
94.44% |
17 / 18 |
|
0.00% |
0 / 1 |
3.00 | |||
| merge | |
100.00% |
7 / 7 |
|
100.00% |
1 / 1 |
1 | |||
| changeTTL | |
0.00% |
0 / 7 |
|
0.00% |
0 / 1 |
2 | |||
| lock | |
0.00% |
0 / 7 |
|
0.00% |
0 / 1 |
2 | |||
| unlock | |
0.00% |
0 / 7 |
|
0.00% |
0 / 1 |
2 | |||
| deleteObjectsExpiringBefore | |
0.00% |
0 / 5 |
|
0.00% |
0 / 1 |
12 | |||
| getMulti | |
0.00% |
0 / 6 |
|
0.00% |
0 / 1 |
12 | |||
| setMulti | |
0.00% |
0 / 7 |
|
0.00% |
0 / 1 |
2 | |||
| deleteMulti | |
0.00% |
0 / 7 |
|
0.00% |
0 / 1 |
2 | |||
| changeTTLMulti | |
0.00% |
0 / 7 |
|
0.00% |
0 / 1 |
2 | |||
| incrWithInit | |
100.00% |
7 / 7 |
|
100.00% |
1 / 1 |
1 | |||
| setMockTime | |
0.00% |
0 / 3 |
|
0.00% |
0 / 1 |
6 | |||
| callKeyMethodOnTierCache | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| callKeyWriteMethodOnTierCaches | |
100.00% |
17 / 17 |
|
100.00% |
1 / 1 |
9 | |||
| 1 | <?php |
| 2 | /** |
| 3 | * This program is free software; you can redistribute it and/or modify |
| 4 | * it under the terms of the GNU General Public License as published by |
| 5 | * the Free Software Foundation; either version 2 of the License, or |
| 6 | * (at your option) any later version. |
| 7 | * |
| 8 | * This program is distributed in the hope that it will be useful, |
| 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 11 | * GNU General Public License for more details. |
| 12 | * |
| 13 | * You should have received a copy of the GNU General Public License along |
| 14 | * with this program; if not, write to the Free Software Foundation, Inc., |
| 15 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
| 16 | * http://www.gnu.org/copyleft/gpl.html |
| 17 | * |
| 18 | * @file |
| 19 | */ |
| 20 | namespace Wikimedia\ObjectCache; |
| 21 | |
| 22 | use InvalidArgumentException; |
| 23 | use Wikimedia\ObjectFactory\ObjectFactory; |
| 24 | |
| 25 | /** |
| 26 | * Wrap multiple BagOStuff objects, to implement different caching tiers. |
| 27 | * |
| 28 | * The order of the caches is important. The first tier is considered the primary |
| 29 | * and highest tier which must handle the majority of the load for reads, |
| 30 | * and is generally less persistent, smaller, and faster (e.g. evicts data |
| 31 | * regularly based on demand, keeping fewer keys at a given time). |
| 32 | * The other caches are consider secondary and lower tiers, which should |
| 33 | * hold more data and retain it for longer than the primary tier. |
| 34 | * |
| 35 | * Data writes ("set") go to all given BagOStuff caches. |
| 36 | * If the `replication => async` option is set, then only the primary write |
| 37 | * is blocking during the web request, with other writes deferred until |
| 38 | * after the web response is sent. |
| 39 | * |
| 40 | * Data reads try each cache in the order they are given, until a value is found. |
| 41 | * When a value is found at a secondary tier, it is automatically copied (back) |
| 42 | * to the primary tier. |
| 43 | * |
| 44 | * **Example**: Keep popular data in memcached, with a fallback to a MySQL database. |
| 45 | * This is how ParserCache is used at Wikimedia Foundation (as of 2024). |
| 46 | * |
| 47 | * ``` |
| 48 | * $wgObjectCaches['parsercache-multiwrite'] = [ |
| 49 | * 'class' => 'MultiWriteBagOStuff', |
| 50 | * 'caches' => [ |
| 51 | * 0 => [ |
| 52 | * 'class' => 'MemcachedPeclBagOStuff', |
| 53 | * 'servers' => [ '127.0.0.1:11212' ], |
| 54 | * ], |
| 55 | * 1 => [ |
| 56 | * 'class' => 'SqlBagOStuff', |
| 57 | * 'servers' => $parserCacheDbServers, |
| 58 | * 'purgePeriod' => 0, |
| 59 | * 'tableName' => 'pc', |
| 60 | * 'shards' => 256, |
| 61 | * 'reportDupes' => false |
| 62 | * ], |
| 63 | * ] |
| 64 | * ]; |
| 65 | * ``` |
| 66 | * |
| 67 | * If you configure a memcached server for MultiWriteBagOStuff that is the same |
| 68 | * as the one used for MediaWiki more generally, it is recommended to specify |
| 69 | * the tier via ObjectCache::getInstance() so that the same object and Memcached |
| 70 | * connection can be re-used. |
| 71 | * |
| 72 | * ``` |
| 73 | * $wgObjectCaches['my-memcached'] = [ .. ]; |
| 74 | * $wgMainCacheType = 'my-memcached'; |
| 75 | * |
| 76 | * $wgObjectCaches['parsercache-multiwrite'] = [ |
| 77 | * 'class' => 'MultiWriteBagOStuff', |
| 78 | * 'caches' => [ |
| 79 | * 0 => [ |
| 80 | * 'factory' => [ 'ObjectCache', 'getInstance' ], |
| 81 | * 'args' => [ 'my-memcached' ], |
| 82 | * ], |
| 83 | * 1 => [ |
| 84 | * 'class' => 'SqlBagOStuff', |
| 85 | * 'servers' => $parserCacheDbServers, |
| 86 | * 'purgePeriod' => 0, |
| 87 | * 'tableName' => 'pc', |
| 88 | * 'shards' => 256, |
| 89 | * 'reportDupes' => false |
| 90 | * ], |
| 91 | * ] |
| 92 | * ]; |
| 93 | * ``` |
| 94 | * |
| 95 | * The makeKey() method of this class uses an implementation-agnostic encoding. |
| 96 | * When it forward gets and sets to the other BagOStuff objects, keys are |
| 97 | * automatically re-encoded. For example, to satisfy the character and length |
| 98 | * constraints of MemcachedBagOStuff. |
| 99 | * |
| 100 | * @newable |
| 101 | * @ingroup Cache |
| 102 | */ |
| 103 | class MultiWriteBagOStuff extends BagOStuff { |
| 104 | /** @var BagOStuff[] Backing cache stores in order of highest to lowest tier */ |
| 105 | protected $caches; |
| 106 | |
| 107 | /** @var bool Use async secondary writes */ |
| 108 | protected $asyncWrites = false; |
| 109 | /** @var int[] List of all backing cache indexes */ |
| 110 | protected $cacheIndexes = []; |
| 111 | |
| 112 | /** @var int TTL when a key is copied to a higher cache tier */ |
| 113 | private static $UPGRADE_TTL = 3600; |
| 114 | |
| 115 | /** |
| 116 | * @stable to call |
| 117 | * |
| 118 | * @param array $params |
| 119 | * - caches: A numbered array of either ObjectFactory::getObjectFromSpec |
| 120 | * arrays yielding BagOStuff objects or direct BagOStuff objects. |
| 121 | * If using the former, the 'args' field *must* be set. |
| 122 | * The first cache is the primary one, being the first to |
| 123 | * be read in the fallback chain. Writes happen to all stores |
| 124 | * in the order they are defined. However, lock()/unlock() calls |
| 125 | * only use the primary store. |
| 126 | * - replication: Either 'sync' or 'async'. This controls whether writes |
| 127 | * to secondary stores are deferred when possible. To use 'async' writes |
| 128 | * requires the 'asyncHandler' option to be set as well. |
| 129 | * Async writes can increase the chance of some race conditions |
| 130 | * or cause keys to expire seconds later than expected. It is |
| 131 | * safe to use for modules when cached values: are immutable, |
| 132 | * invalidation uses logical TTLs, invalidation uses etag/timestamp |
| 133 | * validation against the DB, or merge() is used to handle races. |
| 134 | * |
| 135 | * @phan-param array{caches:array<int,array|BagOStuff>,replication:string} $params |
| 136 | */ |
| 137 | public function __construct( $params ) { |
| 138 | parent::__construct( $params ); |
| 139 | |
| 140 | if ( empty( $params['caches'] ) || !is_array( $params['caches'] ) ) { |
| 141 | throw new InvalidArgumentException( |
| 142 | __METHOD__ . ': "caches" parameter must be an array of caches' |
| 143 | ); |
| 144 | } |
| 145 | |
| 146 | $this->caches = []; |
| 147 | foreach ( $params['caches'] as $cacheInfo ) { |
| 148 | if ( $cacheInfo instanceof BagOStuff ) { |
| 149 | $this->caches[] = $cacheInfo; |
| 150 | } else { |
| 151 | $this->caches[] = ObjectFactory::getObjectFromSpec( $cacheInfo ); |
| 152 | } |
| 153 | } |
| 154 | |
| 155 | $this->attrMap = $this->mergeFlagMaps( $this->caches ); |
| 156 | |
| 157 | $this->asyncWrites = ( |
| 158 | isset( $params['replication'] ) && |
| 159 | $params['replication'] === 'async' && |
| 160 | is_callable( $this->asyncHandler ) |
| 161 | ); |
| 162 | |
| 163 | $this->cacheIndexes = array_keys( $this->caches ); |
| 164 | } |
| 165 | |
| 166 | public function get( $key, $flags = 0 ) { |
| 167 | $args = func_get_args(); |
| 168 | |
| 169 | if ( $this->fieldHasFlags( $flags, self::READ_LATEST ) ) { |
| 170 | // If the latest write was a delete(), we do NOT want to fallback |
| 171 | // to the other tiers and possibly see the old value. Also, this |
| 172 | // is used by merge(), which only needs to hit the primary. |
| 173 | return $this->callKeyMethodOnTierCache( |
| 174 | 0, |
| 175 | __FUNCTION__, |
| 176 | self::ARG0_KEY, |
| 177 | self::RES_NONKEY, |
| 178 | $args |
| 179 | ); |
| 180 | } |
| 181 | |
| 182 | $value = false; |
| 183 | // backends checked |
| 184 | $missIndexes = []; |
| 185 | foreach ( $this->cacheIndexes as $i ) { |
| 186 | $value = $this->callKeyMethodOnTierCache( |
| 187 | $i, |
| 188 | __FUNCTION__, |
| 189 | self::ARG0_KEY, |
| 190 | self::RES_NONKEY, |
| 191 | $args |
| 192 | ); |
| 193 | if ( $value !== false ) { |
| 194 | break; |
| 195 | } |
| 196 | $missIndexes[] = $i; |
| 197 | } |
| 198 | |
| 199 | if ( |
| 200 | $value !== false && |
| 201 | $this->fieldHasFlags( $flags, self::READ_VERIFIED ) && |
| 202 | $missIndexes |
| 203 | ) { |
| 204 | // Backfill the value to the higher (and often faster/smaller) cache tiers |
| 205 | $this->callKeyWriteMethodOnTierCaches( |
| 206 | $missIndexes, |
| 207 | 'set', |
| 208 | self::ARG0_KEY, |
| 209 | self::RES_NONKEY, |
| 210 | [ $key, $value, self::$UPGRADE_TTL ] |
| 211 | ); |
| 212 | } |
| 213 | |
| 214 | return $value; |
| 215 | } |
| 216 | |
| 217 | public function set( $key, $value, $exptime = 0, $flags = 0 ) { |
| 218 | return $this->callKeyWriteMethodOnTierCaches( |
| 219 | $this->cacheIndexes, |
| 220 | __FUNCTION__, |
| 221 | self::ARG0_KEY, |
| 222 | self::RES_NONKEY, |
| 223 | func_get_args() |
| 224 | ); |
| 225 | } |
| 226 | |
| 227 | public function delete( $key, $flags = 0 ) { |
| 228 | return $this->callKeyWriteMethodOnTierCaches( |
| 229 | $this->cacheIndexes, |
| 230 | __FUNCTION__, |
| 231 | self::ARG0_KEY, |
| 232 | self::RES_NONKEY, |
| 233 | func_get_args() |
| 234 | ); |
| 235 | } |
| 236 | |
| 237 | public function add( $key, $value, $exptime = 0, $flags = 0 ) { |
| 238 | // Try the write to the top-tier cache |
| 239 | $ok = $this->callKeyMethodOnTierCache( |
| 240 | 0, |
| 241 | __FUNCTION__, |
| 242 | self::ARG0_KEY, |
| 243 | self::RES_NONKEY, |
| 244 | func_get_args() |
| 245 | ); |
| 246 | |
| 247 | if ( $ok ) { |
| 248 | // Relay the add() using set() if it succeeded. This is meant to handle certain |
| 249 | // migration scenarios where the same store might get written to twice for certain |
| 250 | // keys. In that case, it makes no sense to return false due to "self-conflicts". |
| 251 | $okSecondaries = $this->callKeyWriteMethodOnTierCaches( |
| 252 | array_slice( $this->cacheIndexes, 1 ), |
| 253 | 'set', |
| 254 | self::ARG0_KEY, |
| 255 | self::RES_NONKEY, |
| 256 | [ $key, $value, $exptime, $flags ] |
| 257 | ); |
| 258 | if ( $okSecondaries === false ) { |
| 259 | $ok = false; |
| 260 | } |
| 261 | } |
| 262 | |
| 263 | return $ok; |
| 264 | } |
| 265 | |
| 266 | public function merge( $key, callable $callback, $exptime = 0, $attempts = 10, $flags = 0 ) { |
| 267 | return $this->callKeyWriteMethodOnTierCaches( |
| 268 | $this->cacheIndexes, |
| 269 | __FUNCTION__, |
| 270 | self::ARG0_KEY, |
| 271 | self::RES_NONKEY, |
| 272 | func_get_args() |
| 273 | ); |
| 274 | } |
| 275 | |
| 276 | public function changeTTL( $key, $exptime = 0, $flags = 0 ) { |
| 277 | return $this->callKeyWriteMethodOnTierCaches( |
| 278 | $this->cacheIndexes, |
| 279 | __FUNCTION__, |
| 280 | self::ARG0_KEY, |
| 281 | self::RES_NONKEY, |
| 282 | func_get_args() |
| 283 | ); |
| 284 | } |
| 285 | |
| 286 | public function lock( $key, $timeout = 6, $exptime = 6, $rclass = '' ) { |
| 287 | // Only need to lock the first cache; also avoids deadlocks |
| 288 | return $this->callKeyMethodOnTierCache( |
| 289 | 0, |
| 290 | __FUNCTION__, |
| 291 | self::ARG0_KEY, |
| 292 | self::RES_NONKEY, |
| 293 | func_get_args() |
| 294 | ); |
| 295 | } |
| 296 | |
| 297 | public function unlock( $key ) { |
| 298 | // Only the first cache is locked |
| 299 | return $this->callKeyMethodOnTierCache( |
| 300 | 0, |
| 301 | __FUNCTION__, |
| 302 | self::ARG0_KEY, |
| 303 | self::RES_NONKEY, |
| 304 | func_get_args() |
| 305 | ); |
| 306 | } |
| 307 | |
| 308 | public function deleteObjectsExpiringBefore( |
| 309 | $timestamp, |
| 310 | ?callable $progress = null, |
| 311 | $limit = INF, |
| 312 | ?string $tag = null |
| 313 | ) { |
| 314 | $ret = false; |
| 315 | foreach ( $this->caches as $cache ) { |
| 316 | if ( $cache->deleteObjectsExpiringBefore( $timestamp, $progress, $limit, $tag ) ) { |
| 317 | $ret = true; |
| 318 | } |
| 319 | } |
| 320 | |
| 321 | return $ret; |
| 322 | } |
| 323 | |
| 324 | public function getMulti( array $keys, $flags = 0 ) { |
| 325 | // Just iterate over each key in order to handle all the backfill logic |
| 326 | $res = []; |
| 327 | foreach ( $keys as $key ) { |
| 328 | $val = $this->get( $key, $flags ); |
| 329 | if ( $val !== false ) { |
| 330 | $res[$key] = $val; |
| 331 | } |
| 332 | } |
| 333 | |
| 334 | return $res; |
| 335 | } |
| 336 | |
| 337 | public function setMulti( array $valueByKey, $exptime = 0, $flags = 0 ) { |
| 338 | return $this->callKeyWriteMethodOnTierCaches( |
| 339 | $this->cacheIndexes, |
| 340 | __FUNCTION__, |
| 341 | self::ARG0_KEYMAP, |
| 342 | self::RES_NONKEY, |
| 343 | func_get_args() |
| 344 | ); |
| 345 | } |
| 346 | |
| 347 | public function deleteMulti( array $keys, $flags = 0 ) { |
| 348 | return $this->callKeyWriteMethodOnTierCaches( |
| 349 | $this->cacheIndexes, |
| 350 | __FUNCTION__, |
| 351 | self::ARG0_KEYARR, |
| 352 | self::RES_NONKEY, |
| 353 | func_get_args() |
| 354 | ); |
| 355 | } |
| 356 | |
| 357 | public function changeTTLMulti( array $keys, $exptime, $flags = 0 ) { |
| 358 | return $this->callKeyWriteMethodOnTierCaches( |
| 359 | $this->cacheIndexes, |
| 360 | __FUNCTION__, |
| 361 | self::ARG0_KEYARR, |
| 362 | self::RES_NONKEY, |
| 363 | func_get_args() |
| 364 | ); |
| 365 | } |
| 366 | |
| 367 | public function incrWithInit( $key, $exptime, $step = 1, $init = null, $flags = 0 ) { |
| 368 | return $this->callKeyWriteMethodOnTierCaches( |
| 369 | $this->cacheIndexes, |
| 370 | __FUNCTION__, |
| 371 | self::ARG0_KEY, |
| 372 | self::RES_NONKEY, |
| 373 | func_get_args() |
| 374 | ); |
| 375 | } |
| 376 | |
| 377 | public function setMockTime( &$time ) { |
| 378 | parent::setMockTime( $time ); |
| 379 | foreach ( $this->caches as $cache ) { |
| 380 | $cache->setMockTime( $time ); |
| 381 | } |
| 382 | } |
| 383 | |
| 384 | /** |
| 385 | * Call a method on the cache instance for the given cache tier (index) |
| 386 | * |
| 387 | * @param int $index Cache tier |
| 388 | * @param string $method Method name |
| 389 | * @param int $arg0Sig BagOStuff::A0_* constant describing argument 0 |
| 390 | * @param int $rvSig BagOStuff::RV_* constant describing the return value |
| 391 | * @param array $args Method arguments |
| 392 | * |
| 393 | * @return mixed The result of calling the given method |
| 394 | */ |
| 395 | private function callKeyMethodOnTierCache( $index, $method, $arg0Sig, $rvSig, array $args ) { |
| 396 | return $this->caches[$index]->proxyCall( $method, $arg0Sig, $rvSig, $args, $this ); |
| 397 | } |
| 398 | |
| 399 | /** |
| 400 | * Call a write method on the cache instances, in order, for the given tiers (indexes) |
| 401 | * |
| 402 | * @param int[] $indexes List of cache tiers |
| 403 | * @param string $method Method name |
| 404 | * @param int $arg0Sig BagOStuff::ARG0_* constant describing argument 0 |
| 405 | * @param int $resSig BagOStuff::RES_* constant describing the return value |
| 406 | * @param array $args Method arguments |
| 407 | * |
| 408 | * @return mixed First synchronous result or false if any failed; null if all asynchronous |
| 409 | */ |
| 410 | private function callKeyWriteMethodOnTierCaches( |
| 411 | array $indexes, |
| 412 | $method, |
| 413 | $arg0Sig, |
| 414 | $resSig, |
| 415 | array $args |
| 416 | ) { |
| 417 | $res = null; |
| 418 | |
| 419 | if ( $this->asyncWrites && array_diff( $indexes, [ 0 ] ) && $method !== 'merge' ) { |
| 420 | // Deep-clone $args to prevent misbehavior when something writes an |
| 421 | // object to the BagOStuff then modifies it afterwards, e.g. T168040. |
| 422 | $args = unserialize( serialize( $args ) ); |
| 423 | } |
| 424 | |
| 425 | foreach ( $indexes as $i ) { |
| 426 | $cache = $this->caches[$i]; |
| 427 | |
| 428 | if ( $i == 0 || !$this->asyncWrites ) { |
| 429 | // Tier 0 store or in sync mode: write synchronously and get result |
| 430 | $storeRes = $cache->proxyCall( $method, $arg0Sig, $resSig, $args, $this ); |
| 431 | if ( $storeRes === false ) { |
| 432 | $res = false; |
| 433 | } elseif ( $res === null ) { |
| 434 | // first synchronous result |
| 435 | $res = $storeRes; |
| 436 | } |
| 437 | } else { |
| 438 | // Secondary write in async mode: do not block this HTTP request |
| 439 | ( $this->asyncHandler )( |
| 440 | function () use ( $cache, $method, $arg0Sig, $resSig, $args ) { |
| 441 | $cache->proxyCall( $method, $arg0Sig, $resSig, $args, $this ); |
| 442 | } |
| 443 | ); |
| 444 | } |
| 445 | } |
| 446 | |
| 447 | return $res; |
| 448 | } |
| 449 | } |
| 450 | |
| 451 | /** @deprecated class alias since 1.43 */ |
| 452 | class_alias( MultiWriteBagOStuff::class, 'MultiWriteBagOStuff' ); |