WANObjectCache.php

Go to the documentation of this file.

<?php
use Liuggio\StatsdClient\Factory\StatsdDataFactoryInterface;
use Psr\Log\LoggerAwareInterface;
use Psr\Log\LoggerInterface;
use Psr\Log\NullLogger;
use Wikimedia\LightweightObjectStore\ExpirationAwareness;
use Wikimedia\LightweightObjectStore\StorageAwareness;
 
class WANObjectCache implements
    ExpirationAwareness,
    StorageAwareness,
    IStoreKeyEncoder,
    LoggerAwareInterface
{
    protected $cache;
    protected $processCaches = [];
    protected $logger;
    protected $stats;
    protected $asyncHandler;
 
    protected $mcrouterAware;
    protected $region;
    protected $cluster;
    protected $useInterimHoldOffCaching = true;
    protected $epoch;
    protected $secret;
    protected $coalesceKeys;
    protected $coalesceScheme;
 
    private $keyHighQps;
    private $keyHighUplinkBps;
 
    private $callbackDepth = 0;
    private $warmupCache = [];
    private $warmupKeyMisses = 0;
 
    private $wallClockOverride;
 
    private const MAX_COMMIT_DELAY = 3;
    private const MAX_READ_LAG = 7;
    public const HOLDOFF_TTL = self::MAX_COMMIT_DELAY + self::MAX_READ_LAG + 1;
 
    private const LOW_TTL = 30;
    public const TTL_LAGGED = 30;
 
    private const HOT_TTR = 900;
    private const AGE_NEW = 60;
 
    private const TSE_NONE = -1;
 
    public const STALE_TTL_NONE = 0;
    public const GRACE_TTL_NONE = 0;
    public const HOLDOFF_TTL_NONE = 0;
    public const HOLDOFF_NONE = self::HOLDOFF_TTL_NONE;
 
    public const MIN_TIMESTAMP_NONE = 0.0;
 
    private const PC_PRIMARY = 'primary:1000';
 
    public const PASS_BY_REF = [];
 
    private const SCHEME_HASH_TAG = 1;
    private const SCHEME_HASH_STOP = 2;
 
    private static $CHECK_KEY_TTL = self::TTL_YEAR;
    private static $INTERIM_KEY_TTL = 1;
 
    private static $LOCK_TTL = 10;
    private static $COOLOFF_TTL = 1;
    private static $RAMPUP_TTL = 30;
 
    private static $TINY_NEGATIVE = -0.000001;
    private static $TINY_POSTIVE = 0.000001;
 
    private static $RECENT_SET_LOW_MS = 50;
    private static $RECENT_SET_HIGH_MS = 100;
 
    private static $GENERATION_SLOW_SEC = 3;
 
    private static $PURGE_TIME = 0;
    private static $PURGE_HOLDOFF = 1;
 
    private static $VERSION = 1;
 
    private const RES_VALUE = 0;
    private const RES_METADATA = 1;
 
    public const KEY_VERSION = 'version';
    public const KEY_AS_OF = 'asOf';
    public const KEY_TTL = 'ttl';
    public const KEY_CUR_TTL = 'curTTL';
    public const KEY_TOMB_AS_OF = 'tombAsOf';
    public const KEY_CHECK_AS_OF = 'lastCKPurge';
 
    private static $FLD_FORMAT_VERSION = 0;
    private static $FLD_VALUE = 1;
    private static $FLD_TTL = 2;
    private static $FLD_TIME = 3;
    private static  $FLD_FLAGS = 4;
    private static $FLD_VALUE_VERSION = 5;
    private static $FLD_GENERATION_TIME = 6;
 
    private const TYPE_VALUE = 'v';
    private const TYPE_TIMESTAMP = 't';
    private const TYPE_MUTEX = 'm';
    private const TYPE_INTERIM = 'i';
    private const TYPE_COOLOFF = 'c';
 
    private const PURGE_VAL_PREFIX = 'PURGED:';
 
    public function __construct( array $params ) {
        $this->cache = $params['cache'];
        $this->region = $params['region'] ?? 'main';
        $this->cluster = $params['cluster'] ?? 'wan-main';
        $this->mcrouterAware = !empty( $params['mcrouterAware'] );
        $this->epoch = $params['epoch'] ?? 0;
        $this->secret = $params['secret'] ?? (string)$this->epoch;
        $this->coalesceKeys = $params['coalesceKeys'] ?? false;
        if ( !empty( $params['mcrouterAware'] ) ) {
            // https://github.com/facebook/mcrouter/wiki/Key-syntax
            $this->coalesceScheme = self::SCHEME_HASH_STOP;
        } else {
            // https://redis.io/topics/cluster-spec
            // https://github.com/twitter/twemproxy/blob/v0.4.1/notes/recommendation.md#hash-tags
            // https://github.com/Netflix/dynomite/blob/v0.7.0/notes/recommendation.md#hash-tags
            $this->coalesceScheme = self::SCHEME_HASH_TAG;
        }
 
        $this->keyHighQps = $params['keyHighQps'] ?? 100;
        $this->keyHighUplinkBps = $params['keyHighUplinkBps'] ?? ( 1e9 / 8 / 100 );
 
        $this->setLogger( $params['logger'] ?? new NullLogger() );
        $this->stats = $params['stats'] ?? new NullStatsdDataFactory();
        $this->asyncHandler = $params['asyncHandler'] ?? null;
 
        $this->cache->registerWrapperInfoForStats(
            'WANCache',
            'wanobjectcache',
            [ __CLASS__, 'getCollectionFromKey' ]
        );
    }
 
    public function setLogger( LoggerInterface $logger ) {
        $this->logger = $logger;
    }
 
    public static function newEmpty() {
        return new static( [ 'cache' => new EmptyBagOStuff() ] );
    }
 
    final public function get( $key, &$curTTL = null, array $checkKeys = [], &$info = [] ) {
        // Note that an undeclared variable passed as $info starts as null (not the default).
        // Also, if no $info parameter is provided, then it doesn't matter how it changes here.
        $legacyInfo = ( $info !== self::PASS_BY_REF );
 
        $res = $this->fetchKeys( [ $key ], $checkKeys )[$key];
        $value = $res[self::RES_VALUE];
        $metadata = $res[self::RES_METADATA];
 
        $curTTL = $metadata[self::KEY_CUR_TTL];
        $info = $legacyInfo ? $metadata[self::KEY_AS_OF] : $metadata;
 
        return $value;
    }
 
    final public function getMulti(
        array $keys,
        &$curTTLs = [],
        array $checkKeys = [],
        &$info = []
    ) {
        // Note that an undeclared variable passed as $info starts as null (not the default).
        // Also, if no $info parameter is provided, then it doesn't matter how it changes here.
        $legacyInfo = ( $info !== self::PASS_BY_REF );
 
        $curTTLs = [];
        $info = [];
        $valuesByKey = [];
 
        $resByKey = $this->fetchKeys( $keys, $checkKeys );
        foreach ( $resByKey as $key => $res ) {
            $value = $res[self::RES_VALUE];
            $metadata = $res[self::RES_METADATA];
 
            if ( $value !== false ) {
                $valuesByKey[$key] = $value;
            }
 
            if ( $metadata[self::KEY_CUR_TTL] !== null ) {
                $curTTLs[$key] = $metadata[self::KEY_CUR_TTL];
            }
 
            $info[$key] = $legacyInfo ? $metadata[self::KEY_AS_OF] : $metadata;
        }
 
        return $valuesByKey;
    }
 
    protected function fetchKeys( array $keys, array $checkKeys ) {
        $resByKey = [];
 
        // Order-corresponding list of value keys for the provided base keys
        $valueKeys = $this->makeSisterKeys( $keys, self::TYPE_VALUE );
 
        $fullKeysNeeded = $valueKeys;
        $checkKeysForAll = [];
        $checkKeysByKey = [];
        foreach ( $checkKeys as $i => $checkKeyOrKeyGroup ) {
            // Note: avoid array_merge() inside loop in case there are many keys
            if ( is_int( $i ) ) {
                // Single check key that applies to all value keys
                $fullKey = $this->makeSisterKey( $checkKeyOrKeyGroup, self::TYPE_TIMESTAMP );
                $fullKeysNeeded[] = $fullKey;
                $checkKeysForAll[] = $fullKey;
            } else {
                // List of check keys that apply to a specific value key
                foreach ( (array)$checkKeyOrKeyGroup as $checkKey ) {
                    $fullKey = $this->makeSisterKey( $checkKey, self::TYPE_TIMESTAMP );
                    $fullKeysNeeded[] = $fullKey;
                    $checkKeysByKey[$i][] = $fullKey;
                }
            }
        }
 
        if ( $this->warmupCache ) {
            // Get the raw values of the keys from the warmup cache
            $wrappedValues = $this->warmupCache;
            $fullKeysMissing = array_diff( $fullKeysNeeded, array_keys( $wrappedValues ) );
            if ( $fullKeysMissing ) { // sanity
                $this->warmupKeyMisses += count( $fullKeysMissing );
                $wrappedValues += $this->cache->getMulti( $fullKeysMissing );
            }
        } else {
            // Fetch the raw values of the keys from the backend
            $wrappedValues = $this->cache->getMulti( $fullKeysNeeded );
        }
 
        // Time used to compare/init "check" keys (derived after getMulti() to be pessimistic)
        $now = $this->getCurrentTime();
 
        // Collect timestamps from all "check" keys
        $purgeValuesForAll = $this->processCheckKeys( $checkKeysForAll, $wrappedValues, $now );
        $purgeValuesByKey = [];
        foreach ( $checkKeysByKey as $cacheKey => $checks ) {
            $purgeValuesByKey[$cacheKey] = $this->processCheckKeys( $checks, $wrappedValues, $now );
        }
 
        // Get the main cache value for each key and validate them
        reset( $keys );
        foreach ( $valueKeys as $i => $vKey ) {
            // Get the corresponding base key for this value key
            $key = current( $keys );
            next( $keys );
 
            list( $value, $metadata ) = $this->unwrap(
                array_key_exists( $vKey, $wrappedValues ) ? $wrappedValues[$vKey] : false,
                $now
            );
            // Force dependent keys to be seen as stale for a while after purging
            // to reduce race conditions involving stale data getting cached
            $purgeValues = $purgeValuesForAll;
            if ( isset( $purgeValuesByKey[$key] ) ) {
                $purgeValues = array_merge( $purgeValues, $purgeValuesByKey[$key] );
            }
 
            $lastCKPurge = null; // timestamp of the highest check key
            foreach ( $purgeValues as $purge ) {
                $lastCKPurge = max( $purge[self::$PURGE_TIME], $lastCKPurge );
                $safeTimestamp = $purge[self::$PURGE_TIME] + $purge[self::$PURGE_HOLDOFF];
                if ( $value !== false && $safeTimestamp >= $metadata[self::KEY_AS_OF] ) {
                    // How long ago this value was invalidated by *this* check key
                    $ago = min( $purge[self::$PURGE_TIME] - $now, self::$TINY_NEGATIVE );
                    // How long ago this value was invalidated by *any* known check key
                    $metadata[self::KEY_CUR_TTL] = min( $metadata[self::KEY_CUR_TTL], $ago );
                }
            }
            $metadata[self::KEY_CHECK_AS_OF] = $lastCKPurge;
 
            $resByKey[$key] = [
                self::RES_VALUE => $value,
                self::RES_METADATA => $metadata
            ];
        }
 
        return $resByKey;
    }
 
    private function processCheckKeys( array $timeKeys, array $wrappedValues, $now ) {
        $purgeValues = [];
        foreach ( $timeKeys as $timeKey ) {
            $purge = isset( $wrappedValues[$timeKey] )
                ? $this->parsePurgeValue( $wrappedValues[$timeKey] )
                : false;
            if ( $purge === false ) {
                // Key is not set or malformed; regenerate
                $newVal = $this->makePurgeValue( $now, self::HOLDOFF_TTL );
                $this->cache->add( $timeKey, $newVal, self::$CHECK_KEY_TTL );
                $purge = $this->parsePurgeValue( $newVal );
            }
            $purgeValues[] = $purge;
        }
 
        return $purgeValues;
    }
 
    final public function set( $key, $value, $ttl = self::TTL_INDEFINITE, array $opts = [] ) {
        $now = $this->getCurrentTime();
        $lag = $opts['lag'] ?? 0;
        $age = isset( $opts['since'] ) ? max( 0, $now - $opts['since'] ) : 0;
        $pending = $opts['pending'] ?? false;
        $lockTSE = $opts['lockTSE'] ?? self::TSE_NONE;
        $staleTTL = $opts['staleTTL'] ?? self::STALE_TTL_NONE;
        $creating = $opts['creating'] ?? false;
        $version = $opts['version'] ?? null;
        $walltime = $opts['walltime'] ?? null;
 
        if ( $ttl < 0 ) {
            return true; // not cacheable
        }
 
        // Do not cache potentially uncommitted data as it might get rolled back
        if ( $pending ) {
            $this->logger->info(
                'Rejected set() for {cachekey} due to pending writes.',
                [ 'cachekey' => $key ]
            );
 
            return true; // no-op the write for being unsafe
        }
 
        // Check if there is a risk of caching (stale) data that predates the last delete()
        // tombstone due to the tombstone having expired. If so, then the behavior should depend
        // on whether the problem is specific to this regeneration attempt or systemically affects
        // attempts to regenerate this key. For systemic cases, the cache writes should set a low
        // TTL so that the value at least remains cacheable. For non-systemic cases, the cache
        // write can simply be rejected.
        if ( $age > self::MAX_READ_LAG ) {
            // Case A: high snapshot lag
            if ( $walltime === null ) {
                // Case A0: high snapshot lag without regeneration wall time info.
                // Probably systemic; use a low TTL to avoid stampedes/uncacheability.
                $mitigated = 'snapshot lag';
                $mitigationTTL = self::TTL_SECOND;
            } elseif ( ( $age - $walltime ) > self::MAX_READ_LAG ) {
                // Case A1: value regeneration during an already long-running transaction.
                // Probably non-systemic; rely on a less problematic regeneration attempt.
                $mitigated = 'snapshot lag (late regeneration)';
                $mitigationTTL = self::TTL_UNCACHEABLE;
            } else {
                // Case A2: value regeneration takes a long time.
                // Probably systemic; use a low TTL to avoid stampedes/uncacheability.
                $mitigated = 'snapshot lag (high regeneration time)';
                $mitigationTTL = self::TTL_SECOND;
            }
        } elseif ( $lag === false || $lag > self::MAX_READ_LAG ) {
            // Case B: high replication lag without high snapshot lag
            // Probably systemic; use a low TTL to avoid stampedes/uncacheability
            $mitigated = 'replication lag';
            $mitigationTTL = self::TTL_LAGGED;
        } elseif ( ( $lag + $age ) > self::MAX_READ_LAG ) {
            // Case C: medium length request with medium replication lag
            // Probably non-systemic; rely on a less problematic regeneration attempt
            $mitigated = 'read lag';
            $mitigationTTL = self::TTL_UNCACHEABLE;
        } else {
            // New value generated with recent enough data
            $mitigated = null;
            $mitigationTTL = null;
        }
 
        if ( $mitigationTTL === self::TTL_UNCACHEABLE ) {
            $this->logger->warning(
                "Rejected set() for {cachekey} due to $mitigated.",
                [ 'cachekey' => $key, 'lag' => $lag, 'age' => $age, 'walltime' => $walltime ]
            );
 
            return true; // no-op the write for being unsafe
        }
 
        // TTL to use in staleness checks (does not effect persistence layer TTL)
        $logicalTTL = null;
 
        if ( $mitigationTTL !== null ) {
            // New value generated from data that is old enough to be risky
            if ( $lockTSE >= 0 ) {
                // Value will have the normal expiry but will be seen as stale sooner
                $logicalTTL = min( $ttl ?: INF, $mitigationTTL );
            } else {
                // Value expires sooner (leaving enough TTL for preemptive refresh)
                $ttl = min( $ttl ?: INF, max( $mitigationTTL, self::LOW_TTL ) );
            }
 
            $this->logger->warning(
                "Lowered set() TTL for {cachekey} due to $mitigated.",
                [ 'cachekey' => $key, 'lag' => $lag, 'age' => $age, 'walltime' => $walltime ]
            );
        }
 
        // Wrap that value with time/TTL/version metadata
        $wrapped = $this->wrap( $value, $logicalTTL ?: $ttl, $version, $now, $walltime );
        $storeTTL = $ttl + $staleTTL;
 
        if ( $creating ) {
            $ok = $this->cache->add(
                $this->makeSisterKey( $key, self::TYPE_VALUE ),
                $wrapped,
                $storeTTL
            );
        } else {
            $ok = $this->cache->merge(
                $this->makeSisterKey( $key, self::TYPE_VALUE ),
                static function ( $cache, $key, $cWrapped ) use ( $wrapped ) {
                    // A string value means that it is a tombstone; do nothing in that case
                    return ( is_string( $cWrapped ) ) ? false : $wrapped;
                },
                $storeTTL,
                1 // 1 attempt
            );
        }
 
        return $ok;
    }
 
    final public function delete( $key, $ttl = self::HOLDOFF_TTL ) {
        if ( $ttl <= 0 ) {
            // Publish the purge to all datacenters
            $ok = $this->relayDelete( $this->makeSisterKey( $key, self::TYPE_VALUE ) );
        } else {
            // Publish the purge to all datacenters
            $ok = $this->relayPurge(
                $this->makeSisterKey( $key, self::TYPE_VALUE ),
                $ttl,
                self::HOLDOFF_TTL_NONE
            );
        }
 
        $kClass = $this->determineKeyClassForStats( $key );
        $this->stats->increment( "wanobjectcache.$kClass.delete." . ( $ok ? 'ok' : 'error' ) );
 
        return $ok;
    }
 
    final public function getCheckKeyTime( $key ) {
        return $this->getMultiCheckKeyTime( [ $key ] )[$key];
    }
 
    final public function getMultiCheckKeyTime( array $keys ) {
        $rawKeys = [];
        foreach ( $keys as $key ) {
            $rawKeys[$key] = $this->makeSisterKey( $key, self::TYPE_TIMESTAMP );
        }
 
        $rawValues = $this->cache->getMulti( $rawKeys );
        $rawValues += array_fill_keys( $rawKeys, false );
 
        $times = [];
        foreach ( $rawKeys as $key => $rawKey ) {
            $purge = $this->parsePurgeValue( $rawValues[$rawKey] );
            if ( $purge !== false ) {
                $time = $purge[self::$PURGE_TIME];
            } else {
                // Casting assures identical floats for the next getCheckKeyTime() calls
                $now = (string)$this->getCurrentTime();
                $this->cache->add(
                    $rawKey,
                    $this->makePurgeValue( $now, self::HOLDOFF_TTL ),
                    self::$CHECK_KEY_TTL
                );
                $time = (float)$now;
            }
 
            $times[$key] = $time;
        }
 
        return $times;
    }
 
    final public function touchCheckKey( $key, $holdoff = self::HOLDOFF_TTL ) {
        // Publish the purge to all datacenters
        $ok = $this->relayPurge(
            $this->makeSisterKey( $key, self::TYPE_TIMESTAMP ),
            self::$CHECK_KEY_TTL,
            $holdoff
        );
 
        $kClass = $this->determineKeyClassForStats( $key );
        $this->stats->increment( "wanobjectcache.$kClass.ck_touch." . ( $ok ? 'ok' : 'error' ) );
 
        return $ok;
    }
 
    final public function resetCheckKey( $key ) {
        // Publish the purge to all datacenters
        $ok = $this->relayDelete( $this->makeSisterKey( $key, self::TYPE_TIMESTAMP ) );
 
        $kClass = $this->determineKeyClassForStats( $key );
        $this->stats->increment( "wanobjectcache.$kClass.ck_reset." . ( $ok ? 'ok' : 'error' ) );
 
        return $ok;
    }
 
    final public function getWithSetCallback(
        $key, $ttl, $callback, array $opts = [], array $cbParams = []
    ) {
        $version = $opts['version'] ?? null;
        $pcTTL = $opts['pcTTL'] ?? self::TTL_UNCACHEABLE;
        $pCache = ( $pcTTL >= 0 )
            ? $this->getProcessCache( $opts['pcGroup'] ?? self::PC_PRIMARY )
            : null;
 
        // Use the process cache if requested as long as no outer cache callback is running.
        // Nested callback process cache use is not lag-safe with regard to HOLDOFF_TTL since
        // process cached values are more lagged than persistent ones as they are not purged.
        if ( $pCache && $this->callbackDepth == 0 ) {
            $cached = $pCache->get( $this->getProcessCacheKey( $key, $version ), $pcTTL, false );
            if ( $cached !== false ) {
                $this->logger->debug( "getWithSetCallback($key): process cache hit" );
                return $cached;
            }
        }
 
        $res = $this->fetchOrRegenerate( $key, $ttl, $callback, $opts, $cbParams );
        list( $value, $valueVersion, $curAsOf ) = $res;
        if ( $valueVersion !== $version ) {
            // Current value has a different version; use the variant key for this version.
            // Regenerate the variant value if it is not newer than the main value at $key
            // so that purges to the main key propagate to the variant value.
            $this->logger->debug( "getWithSetCallback($key): using variant key" );
            list( $value ) = $this->fetchOrRegenerate(
                $this->makeGlobalKey( 'WANCache-key-variant', md5( $key ), $version ),
                $ttl,
                $callback,
                [ 'version' => null, 'minAsOf' => $curAsOf ] + $opts,
                $cbParams
            );
        }
 
        // Update the process cache if enabled
        if ( $pCache && $value !== false ) {
            $pCache->set( $this->getProcessCacheKey( $key, $version ), $value );
        }
 
        return $value;
    }
 
    private function fetchOrRegenerate( $key, $ttl, $callback, array $opts, array $cbParams ) {
        $checkKeys = $opts['checkKeys'] ?? [];
        $graceTTL = $opts['graceTTL'] ?? self::GRACE_TTL_NONE;
        $minAsOf = $opts['minAsOf'] ?? self::MIN_TIMESTAMP_NONE;
        $hotTTR = $opts['hotTTR'] ?? self::HOT_TTR;
        $lowTTL = $opts['lowTTL'] ?? min( self::LOW_TTL, $ttl );
        $ageNew = $opts['ageNew'] ?? self::AGE_NEW;
        $touchedCb = $opts['touchedCallback'] ?? null;
        $initialTime = $this->getCurrentTime();
 
        $kClass = $this->determineKeyClassForStats( $key );
 
        // Get the current key value and its metadata
        $res = $this->fetchKeys( [ $key ], $checkKeys )[$key];
        $curValue = $res[self::RES_VALUE];
        $curInfo = $res[self::RES_METADATA];
        $curTTL = $curInfo[self::KEY_CUR_TTL];
        // Apply any $touchedCb invalidation timestamp to get the "last purge timestamp"
        list( $curTTL, $LPT ) = $this->resolveCTL( $curValue, $curTTL, $curInfo, $touchedCb );
        // Use the cached value if it exists and is not due for synchronous regeneration
        if (
            $this->isValid( $curValue, $curInfo[self::KEY_AS_OF], $minAsOf ) &&
            $this->isAliveOrInGracePeriod( $curTTL, $graceTTL )
        ) {
            $preemptiveRefresh = (
                $this->worthRefreshExpiring( $curTTL, $curInfo[self::KEY_TTL], $lowTTL ) ||
                $this->worthRefreshPopular( $curInfo['asOf'], $ageNew, $hotTTR, $initialTime )
            );
            if ( !$preemptiveRefresh ) {
                $this->stats->timing(
                    "wanobjectcache.$kClass.hit.good",
                    1e3 * ( $this->getCurrentTime() - $initialTime )
                );
 
                return [ $curValue, $curInfo[self::KEY_VERSION], $curInfo[self::KEY_AS_OF] ];
            } elseif ( $this->scheduleAsyncRefresh( $key, $ttl, $callback, $opts, $cbParams ) ) {
                $this->logger->debug( "fetchOrRegenerate($key): hit with async refresh" );
                $this->stats->timing(
                    "wanobjectcache.$kClass.hit.refresh",
                    1e3 * ( $this->getCurrentTime() - $initialTime )
                );
 
                return [ $curValue, $curInfo[self::KEY_VERSION], $curInfo[self::KEY_AS_OF] ];
            } else {
                $this->logger->debug( "fetchOrRegenerate($key): hit with sync refresh" );
            }
        }
 
        // Determine if there is stale or volatile cached value that is still usable
        $isKeyTombstoned = ( $curInfo[self::KEY_TOMB_AS_OF] !== null );
        if ( $isKeyTombstoned ) {
            // Key is write-holed; use the (volatile) interim key as an alternative
            list( $possValue, $possInfo ) = $this->getInterimValue( $key, $minAsOf );
            // Update the "last purge time" since the $touchedCb timestamp depends on $value
            $LPT = $this->resolveTouched( $possValue, $LPT, $touchedCb );
        } else {
            $possValue = $curValue;
            $possInfo = $curInfo;
        }
 
        // Avoid overhead from callback runs, regeneration locks, and cache sets during
        // hold-off periods for the key by reusing very recently generated cached values
        if (
            $this->isValid( $possValue, $possInfo[self::KEY_AS_OF], $minAsOf, $LPT ) &&
            $this->isVolatileValueAgeNegligible( $initialTime - $possInfo[self::KEY_AS_OF] )
        ) {
            $this->logger->debug( "fetchOrRegenerate($key): volatile hit" );
            $this->stats->timing(
                "wanobjectcache.$kClass.hit.volatile",
                1e3 * ( $this->getCurrentTime() - $initialTime )
            );
 
            return [ $possValue, $possInfo[self::KEY_VERSION], $curInfo[self::KEY_AS_OF] ];
        }
 
        $lockTSE = $opts['lockTSE'] ?? self::TSE_NONE;
        $busyValue = $opts['busyValue'] ?? null;
        $staleTTL = $opts['staleTTL'] ?? self::STALE_TTL_NONE;
        $version = $opts['version'] ?? null;
 
        // Determine whether one thread per datacenter should handle regeneration at a time
        $useRegenerationLock =
            // Note that since tombstones no-op set(), $lockTSE and $curTTL cannot be used to
            // deduce the key hotness because |$curTTL| will always keep increasing until the
            // tombstone expires or is overwritten by a new tombstone. Also, even if $lockTSE
            // is not set, constant regeneration of a key for the tombstone lifetime might be
            // very expensive. Assume tombstoned keys are possibly hot in order to reduce
            // the risk of high regeneration load after the delete() method is called.
            $isKeyTombstoned ||
            // Assume a key is hot if requested soon ($lockTSE seconds) after invalidation.
            // This avoids stampedes when timestamps from $checkKeys/$touchedCb bump.
            ( $curTTL !== null && $curTTL <= 0 && abs( $curTTL ) <= $lockTSE ) ||
            // Assume a key is hot if there is no value and a busy fallback is given.
            // This avoids stampedes on eviction or preemptive regeneration taking too long.
            ( $busyValue !== null && $possValue === false );
 
        // If a regeneration lock is required, threads that do not get the lock will try to use
        // the stale value, the interim value, or the $busyValue placeholder, in that order. If
        // none of those are set then all threads will bypass the lock and regenerate the value.
        $hasLock = $useRegenerationLock && $this->claimStampedeLock( $key );
        if ( $useRegenerationLock && !$hasLock ) {
            if ( $this->isValid( $possValue, $possInfo[self::KEY_AS_OF], $minAsOf ) ) {
                $this->logger->debug( "fetchOrRegenerate($key): returning stale value" );
                $this->stats->timing(
                    "wanobjectcache.$kClass.hit.stale",
                    1e3 * ( $this->getCurrentTime() - $initialTime )
                );
 
                return [ $possValue, $possInfo[self::KEY_VERSION], $curInfo[self::KEY_AS_OF] ];
            } elseif ( $busyValue !== null ) {
                $miss = is_infinite( $minAsOf ) ? 'renew' : 'miss';
                $this->logger->debug( "fetchOrRegenerate($key): busy $miss" );
                $this->stats->timing(
                    "wanobjectcache.$kClass.$miss.busy",
                    1e3 * ( $this->getCurrentTime() - $initialTime )
                );
                $placeholderValue = $this->resolveBusyValue( $busyValue );
 
                return [ $placeholderValue, $version, $curInfo[self::KEY_AS_OF] ];
            }
        }
 
        // Generate the new value given any prior value with a matching version
        $setOpts = [];
        $preCallbackTime = $this->getCurrentTime();
        ++$this->callbackDepth;
        try {
            $value = $callback(
                ( $curInfo[self::KEY_VERSION] === $version ) ? $curValue : false,
                $ttl,
                $setOpts,
                ( $curInfo[self::KEY_VERSION] === $version ) ? $curInfo[self::KEY_AS_OF] : null,
                $cbParams
            );
        } finally {
            --$this->callbackDepth;
        }
        $postCallbackTime = $this->getCurrentTime();
 
        // How long it took to fetch, validate, and generate the value
        $elapsed = max( $postCallbackTime - $initialTime, 0.0 );
 
        // Attempt to save the newly generated value if applicable
        if (
            // Callback yielded a cacheable value
            ( $value !== false && $ttl >= 0 ) &&
            // Current thread was not raced out of a regeneration lock or key is tombstoned
            ( !$useRegenerationLock || $hasLock || $isKeyTombstoned ) &&
            // Key does not appear to be undergoing a set() stampede
            $this->checkAndSetCooloff( $key, $kClass, $value, $elapsed, $hasLock )
        ) {
            // How long it took to generate the value
            $walltime = max( $postCallbackTime - $preCallbackTime, 0.0 );
            $this->stats->timing( "wanobjectcache.$kClass.regen_walltime", 1e3 * $walltime );
            // If the key is write-holed then use the (volatile) interim key as an alternative
            if ( $isKeyTombstoned ) {
                $this->setInterimValue( $key, $value, $lockTSE, $version, $walltime );
            } else {
                $finalSetOpts = [
                    // @phan-suppress-next-line PhanUselessBinaryAddRight
                    'since' => $setOpts['since'] ?? $preCallbackTime,
                    'version' => $version,
                    'staleTTL' => $staleTTL,
                    'lockTSE' => $lockTSE, // informs lag vs performance trade-offs
                    'creating' => ( $curValue === false ), // optimization
                    'walltime' => $walltime
                ] + $setOpts;
                $this->set( $key, $value, $ttl, $finalSetOpts );
            }
        }
 
        $this->yieldStampedeLock( $key, $hasLock );
 
        $miss = is_infinite( $minAsOf ) ? 'renew' : 'miss';
        $this->logger->debug( "fetchOrRegenerate($key): $miss, new value computed" );
        $this->stats->timing(
            "wanobjectcache.$kClass.$miss.compute",
            1e3 * ( $this->getCurrentTime() - $initialTime )
        );
 
        return [ $value, $version, $curInfo[self::KEY_AS_OF] ];
    }
 
    private function claimStampedeLock( $key ) {
        // Note that locking is not bypassed due to I/O errors; this avoids stampedes
        return $this->cache->add(
            $this->makeSisterKey( $key, self::TYPE_MUTEX ),
            1,
            self::$LOCK_TTL
        );
    }
 
    private function yieldStampedeLock( $key, $hasLock ) {
        if ( $hasLock ) {
            // The backend might be a mcrouter proxy set to broadcast DELETE to *all* the local
            // datacenter cache servers via OperationSelectorRoute (for increased consistency).
            // Since that would be excessive for these locks, use TOUCH to expire the key.
            $this->cache->changeTTL(
                $this->makeSisterKey( $key, self::TYPE_MUTEX ),
                $this->getCurrentTime() - 60
            );
        }
    }
 
    private function makeSisterKeys( array $baseKeys, $type ) {
        $keys = [];
        foreach ( $baseKeys as $baseKey ) {
            $keys[] = $this->makeSisterKey( $baseKey, $type );
        }
 
        return $keys;
    }
 
    private function makeSisterKey( $baseKey, $typeChar ) {
        if ( $this->coalesceKeys === 'non-global' ) {
            $useColocationScheme = ( strncmp( $baseKey, "global:", 7 ) !== 0 );
        } else {
            $useColocationScheme = ( $this->coalesceKeys === true );
        }
 
        if ( !$useColocationScheme ) {
            // Old key style: "WANCache:<character>:<base key>"
            $fullKey = 'WANCache:' . $typeChar . ':' . $baseKey;
        } elseif ( $this->coalesceScheme === self::SCHEME_HASH_STOP ) {
            // Key style: "WANCache:<base key>|#|<character>"
            $fullKey = 'WANCache:' . $baseKey . '|#|' . $typeChar;
        } else {
            // Key style: "WANCache:{<base key>}:<character>"
            $fullKey = 'WANCache:{' . $baseKey . '}:' . $typeChar;
        }
 
        return $fullKey;
    }
 
    public static function getCollectionFromKey( string $sisterKey ) {
        if ( substr( $sisterKey, -4 ) === '|#|v' ) {
            // Key style: "WANCache:<base key>|#|<character>"
            $collection = substr( $sisterKey, 9, strcspn( $sisterKey, ':|', 9 ) );
        } elseif ( substr( $sisterKey, -3 ) === '}:v' ) {
            // Key style: "WANCache:{<base key>}:<character>"
            $collection = substr( $sisterKey, 10, strcspn( $sisterKey, ':}', 10 ) );
        } elseif ( substr( $sisterKey, 9, 2 ) === 'v:' ) {
            // Old key style: "WANCache:<character>:<base key>"
            $collection = substr( $sisterKey, 11, strcspn( $sisterKey, ':', 11 ) );
        } else {
            $collection = 'internal';
        }
 
        return $collection;
    }
 
    private function isVolatileValueAgeNegligible( $age ) {
        return ( $age < mt_rand( self::$RECENT_SET_LOW_MS, self::$RECENT_SET_HIGH_MS ) / 1e3 );
    }
 
    private function checkAndSetCooloff( $key, $kClass, $value, $elapsed, $hasLock ) {
        $valueKey = $this->makeSisterKey( $key, self::TYPE_VALUE );
        list( $estimatedSize ) = $this->cache->setNewPreparedValues( [ $valueKey => $value ] );
 
        if ( !$hasLock ) {
            // Suppose that this cache key is very popular (KEY_HIGH_QPS reads/second).
            // After eviction, there will be cache misses until it gets regenerated and saved.
            // If the time window when the key is missing lasts less than one second, then the
            // number of misses will not reach KEY_HIGH_QPS. This window largely corresponds to
            // the key regeneration time. Estimate the count/rate of cache misses, e.g.:
            //  - 100 QPS, 20ms regeneration => ~2 misses (< 1s)
            //  - 100 QPS, 100ms regeneration => ~10 misses (< 1s)
            //  - 100 QPS, 3000ms regeneration => ~300 misses (100/s for 3s)
            $missesPerSecForHighQPS = ( min( $elapsed, 1 ) * $this->keyHighQps );
 
            // Determine whether there is enough I/O stampede risk to justify throttling set().
            // Estimate unthrottled set() overhead, as bps, from miss count/rate and value size,
            // comparing it to the per-key uplink bps limit (KEY_HIGH_UPLINK_BPS), e.g.:
            //  - 2 misses (< 1s), 10KB value, 1250000 bps limit => 160000 bits (low risk)
            //  - 2 misses (< 1s), 100KB value, 1250000 bps limit => 1600000 bits (high risk)
            //  - 10 misses (< 1s), 10KB value, 1250000 bps limit => 800000 bits (low risk)
            //  - 10 misses (< 1s), 100KB value, 1250000 bps limit => 8000000 bits (high risk)
            //  - 300 misses (100/s), 1KB value, 1250000 bps limit => 800000 bps (low risk)
            //  - 300 misses (100/s), 10KB value, 1250000 bps limit => 8000000 bps (high risk)
            //  - 300 misses (100/s), 100KB value, 1250000 bps limit => 80000000 bps (high risk)
            if ( ( $missesPerSecForHighQPS * $estimatedSize ) >= $this->keyHighUplinkBps ) {
                $this->cache->clearLastError();
                if (
                    !$this->cache->add(
                        $this->makeSisterKey( $key, self::TYPE_COOLOFF ),
                        1,
                        self::$COOLOFF_TTL
                    ) &&
                    // Don't treat failures due to I/O errors as the key being in cooloff
                    $this->cache->getLastError() === BagOStuff::ERR_NONE
                ) {
                    $this->stats->increment( "wanobjectcache.$kClass.cooloff_bounce" );
 
                    return false;
                }
            }
        }
 
        // Corresponding metrics for cache writes that actually get sent over the write
        $this->stats->timing( "wanobjectcache.$kClass.regen_set_delay", 1e3 * $elapsed );
        $this->stats->updateCount( "wanobjectcache.$kClass.regen_set_bytes", $estimatedSize );
 
        return true;
    }
 
    private function resolveCTL( $value, $curTTL, $curInfo, $touchedCallback ) {
        if ( $touchedCallback === null || $value === false ) {
            return [
                $curTTL,
                max( $curInfo[self::KEY_TOMB_AS_OF], $curInfo[self::KEY_CHECK_AS_OF] )
            ];
        }
 
        $touched = $touchedCallback( $value );
        if ( $touched !== null && $touched >= $curInfo[self::KEY_AS_OF] ) {
            $curTTL = min( $curTTL, self::$TINY_NEGATIVE, $curInfo[self::KEY_AS_OF] - $touched );
        }
 
        return [
            $curTTL,
            max(
                $curInfo[self::KEY_TOMB_AS_OF],
                $curInfo[self::KEY_CHECK_AS_OF],
                $touched
            )
         ];
    }
 
    private function resolveTouched( $value, $lastPurge, $touchedCallback ) {
        return ( $touchedCallback === null || $value === false )
            ? $lastPurge // nothing to derive the "touched timestamp" from
            : max( $touchedCallback( $value ), $lastPurge );
    }
 
    private function getInterimValue( $key, $minAsOf ) {
        $now = $this->getCurrentTime();
 
        if ( $this->useInterimHoldOffCaching ) {
            $wrapped = $this->cache->get(
                $this->makeSisterKey( $key, self::TYPE_INTERIM )
            );
 
            list( $value, $keyInfo ) = $this->unwrap( $wrapped, $now );
            if ( $this->isValid( $value, $keyInfo[self::KEY_AS_OF], $minAsOf ) ) {
                return [ $value, $keyInfo ];
            }
        }
 
        return $this->unwrap( false, $now );
    }
 
    private function setInterimValue( $key, $value, $ttl, $version, $walltime ) {
        $ttl = max( self::$INTERIM_KEY_TTL, (int)$ttl );
 
        $wrapped = $this->wrap( $value, $ttl, $version, $this->getCurrentTime(), $walltime );
        $this->cache->merge(
            $this->makeSisterKey( $key, self::TYPE_INTERIM ),
            static function () use ( $wrapped ) {
                return $wrapped;
            },
            $ttl,
            1
        );
    }
 
    private function resolveBusyValue( $busyValue ) {
        return ( $busyValue instanceof Closure ) ? $busyValue() : $busyValue;
    }
 
    final public function getMultiWithSetCallback(
        ArrayIterator $keyedIds, $ttl, callable $callback, array $opts = []
    ) {
        // Load required keys into process cache in one go
        $this->warmupCache = $this->getRawKeysForWarmup(
            $this->getNonProcessCachedMultiKeys( $keyedIds, $opts ),
            $opts['checkKeys'] ?? []
        );
        $this->warmupKeyMisses = 0;
 
        // The required callback signature includes $id as the first argument for convenience
        // to distinguish different items. To reuse the code in getWithSetCallback(), wrap the
        // callback with a proxy callback that has the standard getWithSetCallback() signature.
        // This is defined only once per batch to avoid closure creation overhead.
        $proxyCb = static function ( $oldValue, &$ttl, &$setOpts, $oldAsOf, $params ) use ( $callback ) {
            return $callback( $params['id'], $oldValue, $ttl, $setOpts, $oldAsOf );
        };
 
        $values = [];
        foreach ( $keyedIds as $key => $id ) { // preserve order
            $values[$key] = $this->getWithSetCallback(
                $key,
                $ttl,
                $proxyCb,
                $opts,
                [ 'id' => $id ]
            );
        }
 
        $this->warmupCache = [];
 
        return $values;
    }
 
    final public function getMultiWithUnionSetCallback(
        ArrayIterator $keyedIds, $ttl, callable $callback, array $opts = []
    ) {
        $checkKeys = $opts['checkKeys'] ?? [];
        unset( $opts['lockTSE'] ); // incompatible
        unset( $opts['busyValue'] ); // incompatible
 
        // Load required keys into process cache in one go
        $keysByIdGet = $this->getNonProcessCachedMultiKeys( $keyedIds, $opts );
        $this->warmupCache = $this->getRawKeysForWarmup( $keysByIdGet, $checkKeys );
        $this->warmupKeyMisses = 0;
 
        // IDs of entities known to be in need of regeneration
        $idsRegen = [];
 
        // Find out which keys are missing/deleted/stale
        $resByKey = $this->fetchKeys( $keysByIdGet, $checkKeys );
        foreach ( $keysByIdGet as $id => $key ) {
            $res = $resByKey[$key];
            $value = $res[self::RES_VALUE];
            $metadata = $res[self::RES_METADATA];
            if ( $value === false || $metadata[self::KEY_CUR_TTL] < 0 ) {
                $idsRegen[] = $id;
            }
        }
 
        // Run the callback to populate the regeneration value map for all required IDs
        $newSetOpts = [];
        $newTTLsById = array_fill_keys( $idsRegen, $ttl );
        $newValsById = $idsRegen ? $callback( $idsRegen, $newTTLsById, $newSetOpts ) : [];
 
        // The required callback signature includes $id as the first argument for convenience
        // to distinguish different items. To reuse the code in getWithSetCallback(), wrap the
        // callback with a proxy callback that has the standard getWithSetCallback() signature.
        // This is defined only once per batch to avoid closure creation overhead.
        $proxyCb = static function ( $oldValue, &$ttl, &$setOpts, $oldAsOf, $params )
            use ( $callback, $newValsById, $newTTLsById, $newSetOpts )
        {
            $id = $params['id'];
 
            if ( array_key_exists( $id, $newValsById ) ) {
                // Value was already regerated as expected, so use the value in $newValsById
                $newValue = $newValsById[$id];
                $ttl = $newTTLsById[$id];
                $setOpts = $newSetOpts;
            } else {
                // Pre-emptive/popularity refresh and version mismatch cases are not detected
                // above and thus $newValsById has no entry. Run $callback on this single entity.
                $ttls = [ $id => $ttl ];
                $newValue = $callback( [ $id ], $ttls, $setOpts )[$id];
                $ttl = $ttls[$id];
            }
 
            return $newValue;
        };
 
        // Run the cache-aside logic using warmupCache instead of persistent cache queries
        $values = [];
        foreach ( $keyedIds as $key => $id ) { // preserve order
            $values[$key] = $this->getWithSetCallback(
                $key,
                $ttl,
                $proxyCb,
                $opts,
                [ 'id' => $id ]
            );
        }
 
        $this->warmupCache = [];
 
        return $values;
    }
 
    final public function reap( $key, $purgeTimestamp, &$isStale = false ) {
        $minAsOf = $purgeTimestamp + self::HOLDOFF_TTL;
        $wrapped = $this->cache->get( $this->makeSisterKey( $key, self::TYPE_VALUE ) );
        if ( is_array( $wrapped ) && $wrapped[self::$FLD_TIME] < $minAsOf ) {
            $isStale = true;
            $this->logger->warning( "Reaping stale value key '$key'." );
            $ttlReap = self::HOLDOFF_TTL; // avoids races with tombstone creation
            $ok = $this->cache->changeTTL(
                $this->makeSisterKey( $key, self::TYPE_VALUE ),
                $ttlReap
            );
            if ( !$ok ) {
                $this->logger->error( "Could not complete reap of key '$key'." );
            }
 
            return $ok;
        }
 
        $isStale = false;
 
        return true;
    }
 
    final public function reapCheckKey( $key, $purgeTimestamp, &$isStale = false ) {
        $purge = $this->parsePurgeValue(
            $this->cache->get( $this->makeSisterKey( $key, self::TYPE_TIMESTAMP ) )
        );
        if ( $purge && $purge[self::$PURGE_TIME] < $purgeTimestamp ) {
            $isStale = true;
            $this->logger->warning( "Reaping stale check key '$key'." );
            $ok = $this->cache->changeTTL(
                $this->makeSisterKey( $key, self::TYPE_TIMESTAMP ),
                self::TTL_SECOND
            );
            if ( !$ok ) {
                $this->logger->error( "Could not complete reap of check key '$key'." );
            }
 
            return $ok;
        }
 
        $isStale = false;
 
        return false;
    }
 
    public function makeGlobalKey( $collection, ...$components ) {
        return $this->cache->makeGlobalKey( ...func_get_args() );
    }
 
    public function makeKey( $collection, ...$components ) {
        return $this->cache->makeKey( ...func_get_args() );
    }
 
    public function hash256( $component ) {
        return hash_hmac( 'sha256', $component, $this->secret );
    }
 
    final public function makeMultiKeys( array $ids, $keyCallback ) {
        $idByKey = [];
        foreach ( $ids as $id ) {
            // Discourage triggering of automatic makeKey() hashing in some backends
            if ( strlen( $id ) > 64 ) {
                $this->logger->warning( __METHOD__ . ": long ID '$id'; use hash256()" );
            }
            $key = $keyCallback( $id, $this );
            // Edge case: ignore key collisions due to duplicate $ids like "42" and 42
            if ( !isset( $idByKey[$key] ) ) {
                $idByKey[$key] = $id;
            } elseif ( (string)$id !== (string)$idByKey[$key] ) {
                throw new UnexpectedValueException(
                    "Cache key collision; IDs ('$id','{$idByKey[$key]}') map to '$key'"
                );
            }
        }
 
        return new ArrayIterator( $idByKey );
    }
 
    final public function multiRemap( array $ids, array $res ) {
        if ( count( $ids ) !== count( $res ) ) {
            // If makeMultiKeys() is called on a list of non-unique IDs, then the resulting
            // ArrayIterator will have less entries due to "first appearance" de-duplication
            $ids = array_keys( array_flip( $ids ) );
            if ( count( $ids ) !== count( $res ) ) {
                throw new UnexpectedValueException( "Multi-key result does not match ID list" );
            }
        }
 
        return array_combine( $ids, $res );
    }
 
    final public function getLastError() {
        $code = $this->cache->getLastError();
        switch ( $code ) {
            case BagOStuff::ERR_NONE:
                return self::ERR_NONE;
            case BagOStuff::ERR_NO_RESPONSE:
                return self::ERR_NO_RESPONSE;
            case BagOStuff::ERR_UNREACHABLE:
                return self::ERR_UNREACHABLE;
            default:
                return self::ERR_UNEXPECTED;
        }
    }
 
    final public function clearLastError() {
        $this->cache->clearLastError();
    }
 
    public function clearProcessCache() {
        $this->processCaches = [];
    }
 
    final public function useInterimHoldOffCaching( $enabled ) {
        $this->useInterimHoldOffCaching = $enabled;
    }
 
    public function getQoS( $flag ) {
        return $this->cache->getQoS( $flag );
    }
 
    public function adaptiveTTL( $mtime, $maxTTL, $minTTL = 30, $factor = 0.2 ) {
        if ( is_float( $mtime ) || ctype_digit( $mtime ) ) {
            $mtime = (int)$mtime; // handle fractional seconds and string integers
        }
 
        if ( !is_int( $mtime ) || $mtime <= 0 ) {
            return $minTTL; // no last-modified time provided
        }
 
        $age = $this->getCurrentTime() - $mtime;
 
        return (int)min( $maxTTL, max( $minTTL, $factor * $age ) );
    }
 
    final public function getWarmupKeyMisses() {
        return $this->warmupKeyMisses;
    }
 
    protected function relayPurge( $key, $ttl, $holdoff ) {
        if ( $this->mcrouterAware ) {
            // See https://github.com/facebook/mcrouter/wiki/Multi-cluster-broadcast-setup
            // Wildcards select all matching routes, e.g. the WAN cluster on all DCs
            $ok = $this->cache->set(
                "/*/{$this->cluster}/{$key}",
                $this->makePurgeValue( $this->getCurrentTime(), $holdoff ),
                $ttl
            );
        } else {
            // Some other proxy handles broadcasting or there is only one datacenter
            $ok = $this->cache->set(
                $key,
                $this->makePurgeValue( $this->getCurrentTime(), $holdoff ),
                $ttl
            );
        }
 
        return $ok;
    }
 
    protected function relayDelete( $key ) {
        if ( $this->mcrouterAware ) {
            // See https://github.com/facebook/mcrouter/wiki/Multi-cluster-broadcast-setup
            // Wildcards select all matching routes, e.g. the WAN cluster on all DCs
            $ok = $this->cache->delete( "/*/{$this->cluster}/{$key}" );
        } else {
            // Some other proxy handles broadcasting or there is only one datacenter
            $ok = $this->cache->delete( $key );
        }
 
        return $ok;
    }
 
    private function scheduleAsyncRefresh( $key, $ttl, $callback, array $opts, array $cbParams ) {
        if ( !$this->asyncHandler ) {
            return false;
        }
        // Update the cache value later, such during post-send of an HTTP request. This forces
        // cache regeneration by setting "minAsOf" to infinity, meaning that no existing value
        // is considered valid. Furthermore, note that preemptive regeneration is not applicable
        // to invalid values, so there is no risk of infinite preemptive regeneration loops.
        $func = $this->asyncHandler;
        $func( function () use ( $key, $ttl, $callback, $opts, $cbParams ) {
            $opts['minAsOf'] = INF;
            try {
                $this->fetchOrRegenerate( $key, $ttl, $callback, $opts, $cbParams );
            } catch ( Exception $e ) {
                // Log some context for easier debugging
                $this->logger->error( 'Async refresh failed for {key}', [
                    'key' => $key,
                    'ttl' => $ttl,
                    'exception' => $e
                ] );
                throw $e;
            }
        } );
 
        return true;
    }
 
    private function isAliveOrInGracePeriod( $curTTL, $graceTTL ) {
        if ( $curTTL > 0 ) {
            return true;
        } elseif ( $graceTTL <= 0 ) {
            return false;
        }
 
        $ageStale = abs( $curTTL ); // seconds of staleness
        $curGraceTTL = ( $graceTTL - $ageStale ); // current grace-time-to-live
        if ( $curGraceTTL <= 0 ) {
            return false; // already out of grace period
        }
 
        // Chance of using a stale value is the complement of the chance of refreshing it
        return !$this->worthRefreshExpiring( $curGraceTTL, $graceTTL, $graceTTL );
    }
 
    protected function worthRefreshExpiring( $curTTL, $logicalTTL, $lowTTL ) {
        if ( $lowTTL <= 0 ) {
            return false;
        }
 
        // T264787: avoid having keys start off with a high chance of being refreshed;
        // the point where refreshing becomes possible cannot precede the key lifetime.
        $effectiveLowTTL = min( $lowTTL, $logicalTTL ?: INF );
 
        if ( $curTTL >= $effectiveLowTTL || $curTTL <= 0 ) {
            return false;
        }
 
        $chance = ( 1 - $curTTL / $effectiveLowTTL );
 
        // @phan-suppress-next-line PhanTypeMismatchArgumentInternal
        $decision = ( mt_rand( 1, 1e9 ) <= 1e9 * $chance );
 
        $this->logger->debug(
            "worthRefreshExpiring($curTTL, $logicalTTL, $lowTTL): " .
            "p = $chance; refresh = " . ( $decision ? 'Y' : 'N' )
        );
 
        return $decision;
    }
 
    protected function worthRefreshPopular( $asOf, $ageNew, $timeTillRefresh, $now ) {
        if ( $ageNew < 0 || $timeTillRefresh <= 0 ) {
            return false;
        }
 
        $age = $now - $asOf;
        $timeOld = $age - $ageNew;
        if ( $timeOld <= 0 ) {
            return false;
        }
 
        $popularHitsPerSec = 1;
        // Lifecycle is: new, ramp-up refresh chance, full refresh chance.
        // Note that the "expected # of refreshes" for the ramp-up time range is half
        // of what it would be if P(refresh) was at its full value during that time range.
        $refreshWindowSec = max( $timeTillRefresh - $ageNew - self::$RAMPUP_TTL / 2, 1 );
        // P(refresh) * (# hits in $refreshWindowSec) = (expected # of refreshes)
        // P(refresh) * ($refreshWindowSec * $popularHitsPerSec) = 1 (by definition)
        // P(refresh) = 1/($refreshWindowSec * $popularHitsPerSec)
        $chance = 1 / ( $popularHitsPerSec * $refreshWindowSec );
        // Ramp up $chance from 0 to its nominal value over RAMPUP_TTL seconds to avoid stampedes
        $chance *= ( $timeOld <= self::$RAMPUP_TTL ) ? $timeOld / self::$RAMPUP_TTL : 1;
 
        // @phan-suppress-next-line PhanTypeMismatchArgumentInternal
        $decision = ( mt_rand( 1, 1e9 ) <= 1e9 * $chance );
 
        $this->logger->debug(
            "worthRefreshPopular($asOf, $ageNew, $timeTillRefresh, $now): " .
            "p = $chance; refresh = " . ( $decision ? 'Y' : 'N' )
        );
 
        return $decision;
    }
 
    protected function isValid( $value, $asOf, $minAsOf, $purgeTime = null ) {
        // Avoid reading any key not generated after the latest delete() or touch
        $safeMinAsOf = max( $minAsOf, $purgeTime + self::$TINY_POSTIVE );
 
        if ( $value === false ) {
            return false;
        } elseif ( $safeMinAsOf > 0 && $asOf < $minAsOf ) {
            return false;
        }
 
        return true;
    }
 
    private function wrap( $value, $ttl, $version, $now, $walltime ) {
        // Returns keys in ascending integer order for PHP7 array packing:
        // https://nikic.github.io/2014/12/22/PHPs-new-hashtable-implementation.html
        $wrapped = [
            self::$FLD_FORMAT_VERSION => self::$VERSION,
            self::$FLD_VALUE => $value,
            self::$FLD_TTL => $ttl,
            self::$FLD_TIME => $now
        ];
        if ( $version !== null ) {
            $wrapped[self::$FLD_VALUE_VERSION] = $version;
        }
        if ( $walltime >= self::$GENERATION_SLOW_SEC ) {
            $wrapped[self::$FLD_GENERATION_TIME] = $walltime;
        }
 
        return $wrapped;
    }
 
    private function unwrap( $wrapped, $now ) {
        $value = false;
        $info = $this->newKeyInfoPlaceholder();
 
        if ( is_array( $wrapped ) ) {
            // Entry expected to be a cached value; validate it
            if (
                ( $wrapped[self::$FLD_FORMAT_VERSION] ?? null ) === self::$VERSION &&
                $wrapped[self::$FLD_TIME] >= $this->epoch
            ) {
                if ( $wrapped[self::$FLD_TTL] > 0 ) {
                    // Get the approximate time left on the key
                    $age = $now - $wrapped[self::$FLD_TIME];
                    $curTTL = max( $wrapped[self::$FLD_TTL] - $age, 0.0 );
                } else {
                    // Key had no TTL, so the time left is unbounded
                    $curTTL = INF;
                }
                $value = $wrapped[self::$FLD_VALUE];
                $info[self::KEY_VERSION] = $wrapped[self::$FLD_VALUE_VERSION] ?? null;
                $info[self::KEY_AS_OF] = $wrapped[self::$FLD_TIME];
                $info[self::KEY_CUR_TTL] = $curTTL;
                $info[self::KEY_TTL] = $wrapped[self::$FLD_TTL];
            }
        } else {
            // Entry expected to be a tombstone; parse it
            $purge = $this->parsePurgeValue( $wrapped );
            if ( $purge !== false ) {
                // Tombstoned keys should always have a negative current $ttl
                $info[self::KEY_CUR_TTL] =
                    min( $purge[self::$PURGE_TIME] - $now, self::$TINY_NEGATIVE );
                $info[self::KEY_TOMB_AS_OF] = $purge[self::$PURGE_TIME];
            }
        }
 
        return [ $value, $info ];
    }
 
    private function newKeyInfoPlaceholder() {
        return [
            self::KEY_VERSION => null,
            self::KEY_AS_OF => null,
            self::KEY_TTL => null,
            self::KEY_CUR_TTL => null,
            self::KEY_TOMB_AS_OF => null
        ];
    }
 
    private function determineKeyClassForStats( $key ) {
        $parts = explode( ':', $key, 3 );
        // Sanity fallback in case the key was not made by makeKey.
        // Replace dots because they are special in StatsD (T232907)
        return strtr( $parts[1] ?? $parts[0], '.', '_' );
    }
 
    private function parsePurgeValue( $value ) {
        if ( !is_string( $value ) ) {
            return false;
        }
 
        $segments = explode( ':', $value, 3 );
        if (
            !isset( $segments[0] ) ||
            !isset( $segments[1] ) ||
            "{$segments[0]}:" !== self::PURGE_VAL_PREFIX
        ) {
            return false;
        }
 
        if ( !isset( $segments[2] ) ) {
            // Back-compat with old purge values without holdoff
            $segments[2] = self::HOLDOFF_TTL;
        }
 
        if ( $segments[1] < $this->epoch ) {
            // Values this old are ignored
            return false;
        }
 
        return [
            self::$PURGE_TIME => (float)$segments[1],
            self::$PURGE_HOLDOFF => (int)$segments[2],
        ];
    }
 
    private function makePurgeValue( $timestamp, $holdoff ) {
        return self::PURGE_VAL_PREFIX . (float)$timestamp . ':' . (int)$holdoff;
    }
 
    private function getProcessCache( $group ) {
        if ( !isset( $this->processCaches[$group] ) ) {
            list( , $size ) = explode( ':', $group );
            $this->processCaches[$group] = new MapCacheLRU( (int)$size );
            if ( $this->wallClockOverride !== null ) {
                $this->processCaches[$group]->setMockTime( $this->wallClockOverride );
            }
        }
 
        return $this->processCaches[$group];
    }
 
    private function getProcessCacheKey( $key, $version ) {
        return $key . ' ' . (int)$version;
    }
 
    private function getNonProcessCachedMultiKeys( ArrayIterator $keys, array $opts ) {
        $pcTTL = $opts['pcTTL'] ?? self::TTL_UNCACHEABLE;
 
        $keysMissing = [];
        if ( $pcTTL > 0 && $this->callbackDepth == 0 ) {
            $version = $opts['version'] ?? null;
            $pCache = $this->getProcessCache( $opts['pcGroup'] ?? self::PC_PRIMARY );
            foreach ( $keys as $key => $id ) {
                if ( !$pCache->has( $this->getProcessCacheKey( $key, $version ), $pcTTL ) ) {
                    $keysMissing[$id] = $key;
                }
            }
        }
 
        return $keysMissing;
    }
 
    private function getRawKeysForWarmup( array $keys, array $checkKeys ) {
        if ( !$keys ) {
            return [];
        }
 
        // Get all the value keys to fetch...
        $keysWarmup = $this->makeSisterKeys( $keys, self::TYPE_VALUE );
        // Get all the check keys to fetch...
        foreach ( $checkKeys as $i => $checkKeyOrKeyGroup ) {
            // Note: avoid array_merge() inside loop in case there are many keys
            if ( is_int( $i ) ) {
                // Single check key that applies to all value keys
                $keysWarmup[] = $this->makeSisterKey( $checkKeyOrKeyGroup, self::TYPE_TIMESTAMP );
            } else {
                // List of check keys that apply to a specific value key
                foreach ( (array)$checkKeyOrKeyGroup as $checkKey ) {
                    $keysWarmup[] = $this->makeSisterKey( $checkKey, self::TYPE_TIMESTAMP );
                }
            }
        }
 
        $warmupCache = $this->cache->getMulti( $keysWarmup );
        $warmupCache += array_fill_keys( $keysWarmup, false );
 
        return $warmupCache;
    }
 
    protected function getCurrentTime() {
        if ( $this->wallClockOverride ) {
            return $this->wallClockOverride;
        }
 
        $clockTime = (float)time(); // call this first
        // microtime() uses an initial gettimeofday() call added to usage clocks.
        // This can severely drift from time() and the microtime() value of other threads
        // due to undercounting of the amount of time elapsed. Instead of seeing the current
        // time as being in the past, use the value of time(). This avoids setting cache values
        // that will immediately be seen as expired and possibly cause stampedes.
        return max( microtime( true ), $clockTime );
    }
 
    public function setMockTime( &$time ) {
        $this->wallClockOverride =& $time;
        $this->cache->setMockTime( $time );
        foreach ( $this->processCaches as $pCache ) {
            $pCache->setMockTime( $time );
        }
    }
}