WANObjectCache.php

Go to the documentation of this file.

<?php
namespace Wikimedia\ObjectCache;
 
use ArrayIterator;
use Closure;
use Exception;
use Psr\Log\LoggerAwareInterface;
use Psr\Log\LoggerInterface;
use Psr\Log\NullLogger;
use RuntimeException;
use UnexpectedValueException;
use Wikimedia\LightweightObjectStore\ExpirationAwareness;
use Wikimedia\MapCacheLRU\MapCacheLRU;
use Wikimedia\Stats\StatsFactory;
use Wikimedia\Telemetry\NoopTracer;
use Wikimedia\Telemetry\SpanInterface;
use Wikimedia\Telemetry\TracerInterface;
 
class WANObjectCache implements
    ExpirationAwareness,
    IStoreKeyEncoder,
    LoggerAwareInterface
{
    protected $cache;
    protected $processCaches = [];
    protected $logger;
    protected $stats;
    protected $asyncHandler;
 
    protected $broadcastRoute;
    protected $useInterimHoldOffCaching = true;
    protected $epoch;
    protected $secret;
    protected $coalesceScheme;
 
    private $tracer;
 
    private $missLog;
 
    private $callbackDepth = 0;
    private $warmupCache = [];
    private $warmupKeyMisses = 0;
 
    private $wallClockOverride;
 
    private const MAX_COMMIT_DELAY = 3;
    private const MAX_READ_LAG = 7;
    public const HOLDOFF_TTL = self::MAX_COMMIT_DELAY + self::MAX_READ_LAG + 1;
 
    private const LOW_TTL = 60;
    public const TTL_LAGGED = 30;
 
    private const HOT_TTR = 900;
    private const AGE_NEW = 60;
 
    private const TSE_NONE = -1;
 
    public const STALE_TTL_NONE = 0;
    public const GRACE_TTL_NONE = 0;
    public const HOLDOFF_TTL_NONE = 0;
 
    public const MIN_TIMESTAMP_NONE = 0.0;
 
    private const PC_PRIMARY = 'primary:1000';
 
    public const PASS_BY_REF = [];
 
    private const SCHEME_HASH_TAG = 1;
    private const SCHEME_HASH_STOP = 2;
 
    private const CHECK_KEY_TTL = self::TTL_YEAR;
    private const INTERIM_KEY_TTL = 2;
 
    private const LOCK_TTL = 10;
    private const RAMPUP_TTL = 30;
 
    private const TINY_NEGATIVE = -0.000001;
    private const TINY_POSITIVE = 0.000001;
 
    private const RECENT_SET_LOW_MS = 50;
    private const RECENT_SET_HIGH_MS = 100;
 
    private const GENERATION_HIGH_SEC = 0.2;
 
    private const PURGE_TIME = 0;
    private const PURGE_HOLDOFF = 1;
 
    private const VERSION = 1;
 
    public const KEY_VERSION = 'version';
    public const KEY_AS_OF = 'asOf';
    public const KEY_TTL = 'ttl';
    public const KEY_CUR_TTL = 'curTTL';
    public const KEY_TOMB_AS_OF = 'tombAsOf';
    public const KEY_CHECK_AS_OF = 'lastCKPurge';
 
    private const RES_VALUE = 0;
    private const RES_VERSION = 1;
    private const RES_AS_OF = 2;
    private const RES_TTL = 3;
    private const RES_TOMB_AS_OF = 4;
    private const RES_CHECK_AS_OF = 5;
    private const RES_TOUCH_AS_OF = 6;
    private const RES_CUR_TTL = 7;
 
    private const FLD_FORMAT_VERSION = 0;
    private const FLD_VALUE = 1;
    private const FLD_TTL = 2;
    private const FLD_TIME = 3;
    private const  FLD_FLAGS = 4;
    private const FLD_VALUE_VERSION = 5;
    private const  FLD_GENERATION_TIME = 6;
 
    private const TYPE_VALUE = 'v';
    private const TYPE_TIMESTAMP = 't';
    private const TYPE_MUTEX = 'm';
    private const TYPE_INTERIM = 'i';
 
    private const PURGE_VAL_PREFIX = 'PURGED';
 
    public function __construct( array $params ) {
        $this->cache = $params['cache'];
        $this->broadcastRoute = $params['broadcastRoutingPrefix'] ?? null;
        $this->epoch = $params['epoch'] ?? 0;
        $this->secret = $params['secret'] ?? (string)$this->epoch;
        if ( ( $params['coalesceScheme'] ?? '' ) === 'hash_tag' ) {
            // https://redis.io/topics/cluster-spec
            // https://github.com/twitter/twemproxy/blob/v0.4.1/notes/recommendation.md#hash-tags
            // https://github.com/Netflix/dynomite/blob/v0.7.0/notes/recommendation.md#hash-tags
            $this->coalesceScheme = self::SCHEME_HASH_TAG;
        } else {
            // https://github.com/facebook/mcrouter/wiki/Key-syntax
            $this->coalesceScheme = self::SCHEME_HASH_STOP;
        }
 
        $this->setLogger( $params['logger'] ?? new NullLogger() );
        $this->tracer = $params['tracer'] ?? new NoopTracer();
        $this->stats = $params['stats'] ?? StatsFactory::newNull();
 
        $this->asyncHandler = $params['asyncHandler'] ?? null;
        $this->missLog = array_fill( 0, 10, [ '', 0.0 ] );
    }
 
    public function setLogger( LoggerInterface $logger ) {
        $this->logger = $logger;
    }
 
    public static function newEmpty() {
        return new static( [ 'cache' => new EmptyBagOStuff() ] );
    }
 
    final public function get( $key, &$curTTL = null, array $checkKeys = [], &$info = [] ) {
        // Note that an undeclared variable passed as $info starts as null (not the default).
        // Also, if no $info parameter is provided, then it doesn't matter how it changes here.
        $legacyInfo = ( $info !== self::PASS_BY_REF );
 
        $span = $this->startOperationSpan( __FUNCTION__, $key, $checkKeys );
 
        $now = $this->getCurrentTime();
        $res = $this->fetchKeys( [ $key ], $checkKeys, $now )[$key];
 
        $curTTL = $res[self::RES_CUR_TTL];
        $info = $legacyInfo
            ? $res[self::RES_AS_OF]
            : [
                self::KEY_VERSION => $res[self::RES_VERSION],
                self::KEY_AS_OF => $res[self::RES_AS_OF],
                self::KEY_TTL => $res[self::RES_TTL],
                self::KEY_CUR_TTL => $res[self::RES_CUR_TTL],
                self::KEY_TOMB_AS_OF => $res[self::RES_TOMB_AS_OF],
                self::KEY_CHECK_AS_OF => $res[self::RES_CHECK_AS_OF]
            ];
 
        if ( $curTTL === null || $curTTL <= 0 ) {
            // Log the timestamp in case a corresponding set() call does not provide "walltime"
            unset( $this->missLog[array_key_first( $this->missLog )] );
            $this->missLog[] = [ $key, $this->getCurrentTime() ];
        }
 
        return $res[self::RES_VALUE];
    }
 
    final public function getMulti(
        array $keys,
        &$curTTLs = [],
        array $checkKeys = [],
        &$info = []
    ) {
        // Note that an undeclared variable passed as $info starts as null (not the default).
        // Also, if no $info parameter is provided, then it doesn't matter how it changes here.
        $legacyInfo = ( $info !== self::PASS_BY_REF );
 
        $span = $this->startOperationSpan( __FUNCTION__, $keys, $checkKeys );
 
        $curTTLs = [];
        $info = [];
        $valuesByKey = [];
 
        $now = $this->getCurrentTime();
        $resByKey = $this->fetchKeys( $keys, $checkKeys, $now );
        foreach ( $resByKey as $key => $res ) {
            if ( $res[self::RES_VALUE] !== false ) {
                $valuesByKey[$key] = $res[self::RES_VALUE];
            }
 
            if ( $res[self::RES_CUR_TTL] !== null ) {
                $curTTLs[$key] = $res[self::RES_CUR_TTL];
            }
            $info[$key] = $legacyInfo
                ? $res[self::RES_AS_OF]
                : [
                    self::KEY_VERSION => $res[self::RES_VERSION],
                    self::KEY_AS_OF => $res[self::RES_AS_OF],
                    self::KEY_TTL => $res[self::RES_TTL],
                    self::KEY_CUR_TTL => $res[self::RES_CUR_TTL],
                    self::KEY_TOMB_AS_OF => $res[self::RES_TOMB_AS_OF],
                    self::KEY_CHECK_AS_OF => $res[self::RES_CHECK_AS_OF]
                ];
        }
 
        return $valuesByKey;
    }
 
    protected function fetchKeys( array $keys, array $checkKeys, float $now, $touchedCb = null ) {
        $resByKey = [];
 
        // List of all sister keys that need to be fetched from cache
        $allSisterKeys = [];
        // Order-corresponding value sister key list for the base key list ($keys)
        $valueSisterKeys = [];
        // List of "check" sister keys to compare all value sister keys against
        $checkSisterKeysForAll = [];
        // Map of (base key => additional "check" sister key(s) to compare against)
        $checkSisterKeysByKey = [];
 
        foreach ( $keys as $key ) {
            $sisterKey = $this->makeSisterKey( $key, self::TYPE_VALUE );
            $allSisterKeys[] = $sisterKey;
            $valueSisterKeys[] = $sisterKey;
        }
 
        foreach ( $checkKeys as $i => $checkKeyOrKeyGroup ) {
            // Note: avoid array_merge() inside loop in case there are many keys
            if ( is_int( $i ) ) {
                // Single "check" key that applies to all base keys
                $sisterKey = $this->makeSisterKey( $checkKeyOrKeyGroup, self::TYPE_TIMESTAMP );
                $allSisterKeys[] = $sisterKey;
                $checkSisterKeysForAll[] = $sisterKey;
            } else {
                // List of "check" keys that apply to a specific base key
                foreach ( (array)$checkKeyOrKeyGroup as $checkKey ) {
                    $sisterKey = $this->makeSisterKey( $checkKey, self::TYPE_TIMESTAMP );
                    $allSisterKeys[] = $sisterKey;
                    $checkSisterKeysByKey[$i][] = $sisterKey;
                }
            }
        }
 
        if ( $this->warmupCache ) {
            // Get the wrapped values of the sister keys from the warmup cache
            $wrappedBySisterKey = $this->warmupCache;
            $sisterKeysMissing = array_diff( $allSisterKeys, array_keys( $wrappedBySisterKey ) );
            if ( $sisterKeysMissing ) {
                $this->warmupKeyMisses += count( $sisterKeysMissing );
                $wrappedBySisterKey += $this->cache->getMulti( $sisterKeysMissing );
            }
        } else {
            // Fetch the wrapped values of the sister keys from the backend
            $wrappedBySisterKey = $this->cache->getMulti( $allSisterKeys );
        }
 
        // List of "check" sister key purge timestamps to compare all value sister keys against
        $ckPurgesForAll = $this->processCheckKeys(
            $checkSisterKeysForAll,
            $wrappedBySisterKey,
            $now
        );
        // Map of (base key => extra "check" sister key purge timestamp(s) to compare against)
        $ckPurgesByKey = [];
        foreach ( $checkSisterKeysByKey as $keyWithCheckKeys => $checkKeysForKey ) {
            $ckPurgesByKey[$keyWithCheckKeys] = $this->processCheckKeys(
                $checkKeysForKey,
                $wrappedBySisterKey,
                $now
            );
        }
 
        // Unwrap and validate any value found for each base key (under the value sister key)
        foreach (
            array_map( null, $valueSisterKeys, $keys )
                as [ $valueSisterKey, $key ]
        ) {
            if ( array_key_exists( $valueSisterKey, $wrappedBySisterKey ) ) {
                // Key exists as either a live value or tombstone value
                $wrapped = $wrappedBySisterKey[$valueSisterKey];
            } else {
                // Key does not exist
                $wrapped = false;
            }
 
            $res = $this->unwrap( $wrapped, $now );
            $value = $res[self::RES_VALUE];
 
            foreach ( array_merge( $ckPurgesForAll, $ckPurgesByKey[$key] ?? [] ) as $ckPurge ) {
                $res[self::RES_CHECK_AS_OF] = max(
                    $ckPurge[self::PURGE_TIME],
                    $res[self::RES_CHECK_AS_OF]
                );
                // Timestamp marking the end of the hold-off period for this purge
                $holdoffDeadline = $ckPurge[self::PURGE_TIME] + $ckPurge[self::PURGE_HOLDOFF];
                // Check if the value was generated during the hold-off period
                if ( $value !== false && $holdoffDeadline >= $res[self::RES_AS_OF] ) {
                    // How long ago this value was purged by *this* "check" key
                    $ago = min( $ckPurge[self::PURGE_TIME] - $now, self::TINY_NEGATIVE );
                    // How long ago this value was purged by *any* known "check" key
                    $res[self::RES_CUR_TTL] = min( $res[self::RES_CUR_TTL], $ago );
                }
            }
 
            if ( $touchedCb !== null && $value !== false ) {
                $touched = $touchedCb( $value );
                if ( $touched !== null && $touched >= $res[self::RES_AS_OF] ) {
                    $res[self::RES_CUR_TTL] = min(
                        $res[self::RES_CUR_TTL],
                        $res[self::RES_AS_OF] - $touched,
                        self::TINY_NEGATIVE
                    );
                }
            } else {
                $touched = null;
            }
 
            $res[self::RES_TOUCH_AS_OF] = max( $res[self::RES_TOUCH_AS_OF], $touched );
 
            $resByKey[$key] = $res;
        }
 
        return $resByKey;
    }
 
    private function processCheckKeys(
        array $checkSisterKeys,
        array $wrappedBySisterKey,
        float $now
    ) {
        $purges = [];
 
        foreach ( $checkSisterKeys as $timeKey ) {
            $purge = isset( $wrappedBySisterKey[$timeKey] )
                ? $this->parsePurgeValue( $wrappedBySisterKey[$timeKey] )
                : null;
 
            if ( $purge === null ) {
                // No holdoff when lazy creating a check key, use cache right away (T344191)
                $wrapped = $this->makeCheckPurgeValue( $now, self::HOLDOFF_TTL_NONE, $purge );
                $this->cache->add(
                    $timeKey,
                    $wrapped,
                    self::CHECK_KEY_TTL,
                    $this->cache::WRITE_BACKGROUND
                );
            }
 
            $purges[] = $purge;
        }
 
        return $purges;
    }
 
    final public function set( $key, $value, $ttl = self::TTL_INDEFINITE, array $opts = [] ) {
        $span = $this->startOperationSpan( __FUNCTION__, $key );
 
        $keygroup = $this->determineKeyGroupForStats( $key );
 
        $ok = $this->setMainValue(
            $key,
            $value,
            $ttl,
            $opts['version'] ?? null,
            $opts['walltime'] ?? null,
            $opts['lag'] ?? 0,
            $opts['since'] ?? null,
            $opts['pending'] ?? false,
            $opts['lockTSE'] ?? self::TSE_NONE,
            $opts['staleTTL'] ?? self::STALE_TTL_NONE,
            $opts['segmentable'] ?? false,
            $opts['creating'] ?? false
        );
 
        $this->stats->getCounter( 'wanobjectcache_set_total' )
            ->setLabel( 'keygroup', $keygroup )
            ->setLabel( 'result', ( $ok ? 'ok' : 'error' ) )
            ->copyToStatsdAt( "wanobjectcache.$keygroup.set." . ( $ok ? 'ok' : 'error' ) )
            ->increment();
 
        return $ok;
    }
 
    private function setMainValue(
        $key,
        $value,
        $ttl,
        ?int $version,
        ?float $walltime,
        $dataReplicaLag,
        $dataReadSince,
        bool $dataPendingCommit,
        int $lockTSE,
        int $staleTTL,
        bool $segmentable,
        bool $creating
    ) {
        if ( $ttl < 0 ) {
            // not cacheable
            return true;
        }
 
        $now = $this->getCurrentTime();
        $ttl = (int)$ttl;
        $walltime ??= $this->timeSinceLoggedMiss( $key, $now );
        $dataSnapshotLag = ( $dataReadSince !== null ) ? max( 0, $now - $dataReadSince ) : 0;
        $dataCombinedLag = $dataReplicaLag + $dataSnapshotLag;
 
        // Forbid caching data that only exists within an uncommitted transaction. Also, lower
        // the TTL when the data has a "since" time so far in the past that a delete() tombstone,
        // made after that time, could have already expired (the key is no longer write-holed).
        // The mitigation TTL depends on whether this data lag is assumed to systemically effect
        // regeneration attempts in the near future. The TTL also reflects regeneration wall time.
        if ( $dataPendingCommit ) {
            // Case A: data comes from an uncommitted write transaction
            $mitigated = 'pending writes';
            // Data might never be committed; rely on a less problematic regeneration attempt
            $mitigationTTL = self::TTL_UNCACHEABLE;
        } elseif ( $dataSnapshotLag > self::MAX_READ_LAG ) {
            // Case B: high snapshot lag
            $pregenSnapshotLag = ( $walltime !== null ) ? ( $dataSnapshotLag - $walltime ) : 0;
            if ( ( $pregenSnapshotLag + self::GENERATION_HIGH_SEC ) > self::MAX_READ_LAG ) {
                // Case B1: generation started when transaction duration was already long
                $mitigated = 'snapshot lag (late generation)';
                // Probably non-systemic; rely on a less problematic regeneration attempt
                $mitigationTTL = self::TTL_UNCACHEABLE;
            } else {
                // Case B2: slow generation made transaction duration long
                $mitigated = 'snapshot lag (high generation time)';
                // Probably systemic; use a low TTL to avoid stampedes/uncacheability
                $mitigationTTL = self::TTL_LAGGED;
            }
        } elseif ( $dataReplicaLag === false || $dataReplicaLag > self::MAX_READ_LAG ) {
            // Case C: low/medium snapshot lag with high replication lag
            $mitigated = 'replication lag';
            // Probably systemic; use a low TTL to avoid stampedes/uncacheability
            $mitigationTTL = self::TTL_LAGGED;
        } elseif ( $dataCombinedLag > self::MAX_READ_LAG ) {
            $pregenCombinedLag = ( $walltime !== null ) ? ( $dataCombinedLag - $walltime ) : 0;
            // Case D: medium snapshot lag with medium replication lag
            if ( ( $pregenCombinedLag + self::GENERATION_HIGH_SEC ) > self::MAX_READ_LAG ) {
                // Case D1: generation started when read lag was too high
                $mitigated = 'read lag (late generation)';
                // Probably non-systemic; rely on a less problematic regeneration attempt
                $mitigationTTL = self::TTL_UNCACHEABLE;
            } else {
                // Case D2: slow generation made read lag too high
                $mitigated = 'read lag (high generation time)';
                // Probably systemic; use a low TTL to avoid stampedes/uncacheability
                $mitigationTTL = self::TTL_LAGGED;
            }
        } else {
            // Case E: new value generated with recent data
            $mitigated = null;
            // Nothing to mitigate
            $mitigationTTL = null;
        }
 
        if ( $mitigationTTL === self::TTL_UNCACHEABLE ) {
            $this->logger->warning(
                "Rejected set() for {cachekey} due to $mitigated.",
                [
                    'cachekey' => $key,
                    'lag' => $dataReplicaLag,
                    'age' => $dataSnapshotLag,
                    'walltime' => $walltime
                ]
            );
 
            // no-op the write for being unsafe
            return true;
        }
 
        // TTL to use in staleness checks (does not effect persistence layer TTL)
        $logicalTTL = null;
 
        if ( $mitigationTTL !== null ) {
            // New value was generated from data that is old enough to be risky
            if ( $lockTSE >= 0 ) {
                // Persist the value as long as normal, but make it count as stale sooner
                $logicalTTL = min( $ttl ?: INF, $mitigationTTL );
            } else {
                // Persist the value for a shorter duration
                $ttl = min( $ttl ?: INF, $mitigationTTL );
            }
 
            $this->logger->warning(
                "Lowered set() TTL for {cachekey} due to $mitigated.",
                [
                    'cachekey' => $key,
                    'lag' => $dataReplicaLag,
                    'age' => $dataSnapshotLag,
                    'walltime' => $walltime
                ]
            );
        }
 
        // Wrap that value with time/TTL/version metadata
        $wrapped = $this->wrap( $value, $logicalTTL ?: $ttl, $version, $now );
        $storeTTL = $ttl + $staleTTL;
 
        $flags = $this->cache::WRITE_BACKGROUND;
        if ( $segmentable ) {
            $flags |= $this->cache::WRITE_ALLOW_SEGMENTS;
        }
 
        if ( $creating ) {
            $ok = $this->cache->add(
                $this->makeSisterKey( $key, self::TYPE_VALUE ),
                $wrapped,
                $storeTTL,
                $flags
            );
        } else {
            $ok = $this->cache->merge(
                $this->makeSisterKey( $key, self::TYPE_VALUE ),
                static function ( $cache, $key, $cWrapped ) use ( $wrapped ) {
                    // A string value means that it is a tombstone; do nothing in that case
                    return ( is_string( $cWrapped ) ) ? false : $wrapped;
                },
                $storeTTL,
                $this->cache::MAX_CONFLICTS_ONE,
                $flags
            );
        }
 
        return $ok;
    }
 
    final public function delete( $key, $ttl = self::HOLDOFF_TTL ) {
        $span = $this->startOperationSpan( __FUNCTION__, $key );
 
        // Purge values must be stored under the value key so that WANObjectCache::set()
        // can atomically merge values without accidentally undoing a recent purge and thus
        // violating the holdoff TTL restriction.
        $valueSisterKey = $this->makeSisterKey( $key, self::TYPE_VALUE );
 
        if ( $ttl <= 0 ) {
            // A client or cache cleanup script is requesting a cache purge, so there is no
            // volatility period due to replica DB lag. Any recent change to an entity cached
            // in this key should have triggered an appropriate purge event.
            $ok = $this->cache->delete( $this->getRouteKey( $valueSisterKey ), $this->cache::WRITE_BACKGROUND );
        } else {
            // A cacheable entity recently changed, so there might be a volatility period due
            // to replica DB lag. Clients usually expect their actions to be reflected in any
            // of their subsequent web request. This is attainable if (a) purge relay lag is
            // lower than the time it takes for subsequent request by the client to arrive,
            // and, (b) DB replica queries have "read-your-writes" consistency due to DB lag
            // mitigation systems.
            $now = $this->getCurrentTime();
            // Set the key to the purge value in all datacenters
            $purge = self::PURGE_VAL_PREFIX . ':' . (int)$now;
            $ok = $this->cache->set(
                $this->getRouteKey( $valueSisterKey ),
                $purge,
                $ttl,
                $this->cache::WRITE_BACKGROUND
            );
        }
 
        $keygroup = $this->determineKeyGroupForStats( $key );
 
        $this->stats->getCounter( 'wanobjectcache_delete_total' )
            ->setLabel( 'keygroup', $keygroup )
            ->setLabel( 'result', ( $ok ? 'ok' : 'error' ) )
            ->copyToStatsdAt( "wanobjectcache.$keygroup.delete." . ( $ok ? 'ok' : 'error' ) )
            ->increment();
 
        return $ok;
    }
 
    final public function getCheckKeyTime( $key ) {
        $span = $this->startOperationSpan( __FUNCTION__, $key );
 
        return $this->getMultiCheckKeyTime( [ $key ] )[$key];
    }
 
    final public function getMultiCheckKeyTime( array $keys ) {
        $span = $this->startOperationSpan( __FUNCTION__, $keys );
 
        $checkSisterKeysByKey = [];
        foreach ( $keys as $key ) {
            $checkSisterKeysByKey[$key] = $this->makeSisterKey( $key, self::TYPE_TIMESTAMP );
        }
 
        $wrappedBySisterKey = $this->cache->getMulti( $checkSisterKeysByKey );
        $wrappedBySisterKey += array_fill_keys( $checkSisterKeysByKey, false );
 
        $now = $this->getCurrentTime();
        $times = [];
        foreach ( $checkSisterKeysByKey as $key => $checkSisterKey ) {
            $purge = $this->parsePurgeValue( $wrappedBySisterKey[$checkSisterKey] );
            if ( $purge === null ) {
                $wrapped = $this->makeCheckPurgeValue( $now, self::HOLDOFF_TTL_NONE, $purge );
                $this->cache->add(
                    $checkSisterKey,
                    $wrapped,
                    self::CHECK_KEY_TTL,
                    $this->cache::WRITE_BACKGROUND
                );
            }
 
            $times[$key] = $purge[self::PURGE_TIME];
        }
 
        return $times;
    }
 
    public function touchCheckKey( $key, $holdoff = self::HOLDOFF_TTL ) {
        $span = $this->startOperationSpan( __FUNCTION__, $key );
 
        $checkSisterKey = $this->makeSisterKey( $key, self::TYPE_TIMESTAMP );
 
        $now = $this->getCurrentTime();
        $purge = $this->makeCheckPurgeValue( $now, $holdoff );
        $ok = $this->cache->set(
            $this->getRouteKey( $checkSisterKey ),
            $purge,
            self::CHECK_KEY_TTL,
            $this->cache::WRITE_BACKGROUND
        );
 
        $keygroup = $this->determineKeyGroupForStats( $key );
 
        $this->stats->getCounter( 'wanobjectcache_check_total' )
            ->setLabel( 'keygroup', $keygroup )
            ->setLabel( 'result', ( $ok ? 'ok' : 'error' ) )
            ->copyToStatsdAt( "wanobjectcache.$keygroup.ck_touch." . ( $ok ? 'ok' : 'error' ) )
            ->increment();
 
        return $ok;
    }
 
    public function resetCheckKey( $key ) {
        $span = $this->startOperationSpan( __FUNCTION__, $key );
 
        $checkSisterKey = $this->makeSisterKey( $key, self::TYPE_TIMESTAMP );
        $ok = $this->cache->delete( $this->getRouteKey( $checkSisterKey ), $this->cache::WRITE_BACKGROUND );
 
        $keygroup = $this->determineKeyGroupForStats( $key );
 
        $this->stats->getCounter( 'wanobjectcache_reset_total' )
            ->setLabel( 'keygroup', $keygroup )
            ->setLabel( 'result', ( $ok ? 'ok' : 'error' ) )
            ->copyToStatsdAt( "wanobjectcache.$keygroup.ck_reset." . ( $ok ? 'ok' : 'error' ) )
            ->increment();
 
        return $ok;
    }
 
    final public function getWithSetCallback(
        $key, $ttl, $callback, array $opts = [], array $cbParams = []
    ) {
        $span = $this->startOperationSpan( __FUNCTION__, $key );
 
        $version = $opts['version'] ?? null;
        $pcTTL = $opts['pcTTL'] ?? self::TTL_UNCACHEABLE;
        $pCache = ( $pcTTL >= 0 )
            ? $this->getProcessCache( $opts['pcGroup'] ?? self::PC_PRIMARY )
            : null;
 
        // Use the process cache if requested as long as no outer cache callback is running.
        // Nested callback process cache use is not lag-safe with regard to HOLDOFF_TTL since
        // process cached values are more lagged than persistent ones as they are not purged.
        if ( $pCache && $this->callbackDepth == 0 ) {
            $cached = $pCache->get( $key, $pcTTL, false );
            if ( $cached !== false ) {
                $this->logger->debug( "getWithSetCallback($key): process cache hit" );
                return $cached;
            }
        }
 
        [ $value, $valueVersion, $curAsOf ] = $this->fetchOrRegenerate( $key, $ttl, $callback, $opts, $cbParams );
        if ( $valueVersion !== $version ) {
            // Current value has a different version; use the variant key for this version.
            // Regenerate the variant value if it is not newer than the main value at $key
            // so that purges to the main key propagate to the variant value.
            $this->logger->debug( "getWithSetCallback($key): using variant key" );
            [ $value ] = $this->fetchOrRegenerate(
                $this->makeGlobalKey( 'WANCache-key-variant', md5( $key ), (string)$version ),
                $ttl,
                $callback,
                [ 'version' => null, 'minAsOf' => $curAsOf ] + $opts,
                $cbParams
            );
        }
 
        // Update the process cache if enabled
        if ( $pCache && $value !== false ) {
            $pCache->set( $key, $value );
        }
 
        return $value;
    }
 
    private function fetchOrRegenerate( $key, $ttl, $callback, array $opts, array $cbParams ) {
        $checkKeys = $opts['checkKeys'] ?? [];
        $graceTTL = $opts['graceTTL'] ?? self::GRACE_TTL_NONE;
        $minAsOf = $opts['minAsOf'] ?? self::MIN_TIMESTAMP_NONE;
        $hotTTR = $opts['hotTTR'] ?? self::HOT_TTR;
        $lowTTL = $opts['lowTTL'] ?? min( self::LOW_TTL, $ttl );
        $ageNew = $opts['ageNew'] ?? self::AGE_NEW;
        $touchedCb = $opts['touchedCallback'] ?? null;
        $startTime = $this->getCurrentTime();
 
        $keygroup = $this->determineKeyGroupForStats( $key );
 
        // Get the current key value and its metadata
        $curState = $this->fetchKeys( [ $key ], $checkKeys, $startTime, $touchedCb )[$key];
        $curValue = $curState[self::RES_VALUE];
 
        // Use the cached value if it exists and is not due for synchronous regeneration
        if ( $this->isAcceptablyFreshValue( $curState, $graceTTL, $minAsOf ) ) {
            if ( !$this->isLotteryRefreshDue( $curState, $lowTTL, $ageNew, $hotTTR, $startTime ) ) {
                $this->stats->getTiming( 'wanobjectcache_getwithset_seconds' )
                    ->setLabel( 'keygroup', $keygroup )
                    ->setLabel( 'result', 'hit' )
                    ->setLabel( 'reason', 'good' )
                    ->copyToStatsdAt( "wanobjectcache.$keygroup.hit.good" )
                    ->observe( 1e3 * ( $this->getCurrentTime() - $startTime ) );
 
                return [ $curValue, $curState[self::RES_VERSION], $curState[self::RES_AS_OF] ];
            } elseif ( $this->scheduleAsyncRefresh( $key, $ttl, $callback, $opts, $cbParams ) ) {
                $this->logger->debug( "fetchOrRegenerate($key): hit with async refresh" );
 
                $this->stats->getTiming( 'wanobjectcache_getwithset_seconds' )
                    ->setLabel( 'keygroup', $keygroup )
                    ->setLabel( 'result', 'hit' )
                    ->setLabel( 'reason', 'refresh' )
                    ->copyToStatsdAt( "wanobjectcache.$keygroup.hit.refresh" )
                    ->observe( 1e3 * ( $this->getCurrentTime() - $startTime ) );
 
                return [ $curValue, $curState[self::RES_VERSION], $curState[self::RES_AS_OF] ];
            } else {
                $this->logger->debug( "fetchOrRegenerate($key): hit with sync refresh" );
            }
        }
 
        $isKeyTombstoned = ( $curState[self::RES_TOMB_AS_OF] !== null );
        // Use the interim key as a temporary alternative if the key is tombstoned
        if ( $isKeyTombstoned ) {
            $volState = $this->getInterimValue( $key, $minAsOf, $startTime, $touchedCb );
            $volValue = $volState[self::RES_VALUE];
        } else {
            $volState = $curState;
            $volValue = $curValue;
        }
 
        // During the volatile "hold-off" period that follows a purge of the key, the value
        // will be regenerated many times if frequently accessed. This is done to mitigate
        // the effects of backend replication lag as soon as possible. However, throttle the
        // overhead of locking and regeneration by reusing values recently written to cache
        // tens of milliseconds ago. Verify the "as of" time against the last purge event.
        $lastPurgeTime = max(
            // RES_TOUCH_AS_OF depends on the value (possibly from the interim key)
            $volState[self::RES_TOUCH_AS_OF],
            $curState[self::RES_TOMB_AS_OF],
            $curState[self::RES_CHECK_AS_OF]
        );
        $safeMinAsOf = max( $minAsOf, $lastPurgeTime + self::TINY_POSITIVE );
 
        if ( $volState[self::RES_VALUE] === false || $volState[self::RES_AS_OF] < $safeMinAsOf ) {
            $isExtremelyNewValue = false;
        } else {
            $age = $startTime - $volState[self::RES_AS_OF];
            $isExtremelyNewValue = ( $age < mt_rand( self::RECENT_SET_LOW_MS, self::RECENT_SET_HIGH_MS ) / 1e3 );
        }
        if ( $isExtremelyNewValue ) {
            $this->logger->debug( "fetchOrRegenerate($key): volatile hit" );
 
            $this->stats->getTiming( 'wanobjectcache_getwithset_seconds' )
                ->setLabel( 'keygroup', $keygroup )
                ->setLabel( 'result', 'hit' )
                ->setLabel( 'reason', 'volatile' )
                ->copyToStatsdAt( "wanobjectcache.$keygroup.hit.volatile" )
                ->observe( 1e3 * ( $this->getCurrentTime() - $startTime ) );
 
            return [ $volValue, $volState[self::RES_VERSION], $curState[self::RES_AS_OF] ];
        }
 
        $lockTSE = $opts['lockTSE'] ?? self::TSE_NONE;
        $busyValue = $opts['busyValue'] ?? null;
        $staleTTL = $opts['staleTTL'] ?? self::STALE_TTL_NONE;
        $segmentable = $opts['segmentable'] ?? false;
        $version = $opts['version'] ?? null;
 
        // Determine whether one thread per datacenter should handle regeneration at a time
        $useRegenerationLock =
            // Note that since tombstones no-op set(), $lockTSE and $curTTL cannot be used to
            // deduce the key hotness because |$curTTL| will always keep increasing until the
            // tombstone expires or is overwritten by a new tombstone. Also, even if $lockTSE
            // is not set, constant regeneration of a key for the tombstone lifetime might be
            // very expensive. Assume tombstoned keys are possibly hot in order to reduce
            // the risk of high regeneration load after the delete() method is called.
            $isKeyTombstoned ||
            // Assume a key is hot if requested soon ($lockTSE seconds) after purge.
            // This avoids stampedes when timestamps from $checkKeys/$touchedCb bump.
            (
                $curState[self::RES_CUR_TTL] !== null &&
                $curState[self::RES_CUR_TTL] <= 0 &&
                abs( $curState[self::RES_CUR_TTL] ) <= $lockTSE
            ) ||
            // Assume a key is hot if there is no value and a busy fallback is given.
            // This avoids stampedes on eviction or preemptive regeneration taking too long.
            ( $busyValue !== null && $volValue === false );
 
        // If a regeneration lock is required, threads that do not get the lock will try to use
        // the stale value, the interim value, or the $busyValue placeholder, in that order. If
        // none of those are set then all threads will bypass the lock and regenerate the value.
        $mutexKey = $this->makeSisterKey( $key, self::TYPE_MUTEX );
        // Note that locking is not bypassed due to I/O errors; this avoids stampedes
        $hasLock = $useRegenerationLock && $this->cache->add( $mutexKey, 1, self::LOCK_TTL );
        if ( $useRegenerationLock && !$hasLock ) {
            // Determine if there is stale or volatile cached value that is still usable
            // @phan-suppress-next-line PhanTypeMismatchArgumentNullable False positive
            if ( $this->isValid( $volValue, $volState[self::RES_AS_OF], $minAsOf ) ) {
                $this->logger->debug( "fetchOrRegenerate($key): returning stale value" );
 
                $this->stats->getTiming( 'wanobjectcache_getwithset_seconds' )
                    ->setLabel( 'keygroup', $keygroup )
                    ->setLabel( 'result', 'hit' )
                    ->setLabel( 'reason', 'stale' )
                    ->copyToStatsdAt( "wanobjectcache.$keygroup.hit.stale" )
                    ->observe( 1e3 * ( $this->getCurrentTime() - $startTime ) );
 
                return [ $volValue, $volState[self::RES_VERSION], $curState[self::RES_AS_OF] ];
            } elseif ( $busyValue !== null ) {
                $miss = is_infinite( $minAsOf ) ? 'renew' : 'miss';
                $this->logger->debug( "fetchOrRegenerate($key): busy $miss" );
 
                $this->stats->getTiming( 'wanobjectcache_getwithset_seconds' )
                    ->setLabel( 'keygroup', $keygroup )
                    ->setLabel( 'result', $miss )
                    ->setLabel( 'reason', 'busy' )
                    ->copyToStatsdAt( "wanobjectcache.$keygroup.$miss.busy" )
                    ->observe( 1e3 * ( $this->getCurrentTime() - $startTime ) );
 
                $placeholderValue = ( $busyValue instanceof Closure ) ? $busyValue() : $busyValue;
 
                return [ $placeholderValue, $version, $curState[self::RES_AS_OF] ];
            }
        }
 
        // Generate the new value given any prior value with a matching version
        $setOpts = [];
        $preCallbackTime = $this->getCurrentTime();
        ++$this->callbackDepth;
        // https://github.com/phan/phan/issues/4419
        $value = null;
        try {
            $value = $callback(
                ( $curState[self::RES_VERSION] === $version ) ? $curValue : false,
                $ttl,
                $setOpts,
                ( $curState[self::RES_VERSION] === $version ) ? $curState[self::RES_AS_OF] : null,
                $cbParams
            );
        } finally {
            --$this->callbackDepth;
        }
        $postCallbackTime = $this->getCurrentTime();
 
        // How long it took to generate the value
        $walltime = max( $postCallbackTime - $preCallbackTime, 0.0 );
 
        $this->stats->getTiming( 'wanobjectcache_regen_seconds' )
            ->setLabel( 'keygroup', $keygroup )
            ->copyToStatsdAt( "wanobjectcache.$keygroup.regen_walltime" )
            ->observe( 1e3 * $walltime );
 
        // Attempt to save the newly generated value if applicable
        if (
            // Callback yielded a cacheable value
            ( $value !== false && $ttl >= 0 ) &&
            // Current thread was not raced out of a regeneration lock or key is tombstoned
            ( !$useRegenerationLock || $hasLock || $isKeyTombstoned )
        ) {
            // If the key is write-holed then use the (volatile) interim key as an alternative
            if ( $isKeyTombstoned ) {
                $this->setInterimValue(
                    $key,
                    $value,
                    $lockTSE,
                    $version,
                    $segmentable
                );
            } else {
                $this->setMainValue(
                    $key,
                    $value,
                    $ttl,
                    $version,
                    $walltime,
                    // @phan-suppress-next-line PhanCoalescingAlwaysNull
                    $setOpts['lag'] ?? 0,
                    // @phan-suppress-next-line PhanCoalescingAlwaysNull
                    $setOpts['since'] ?? $preCallbackTime,
                    // @phan-suppress-next-line PhanCoalescingAlwaysNull
                    $setOpts['pending'] ?? false,
                    $lockTSE,
                    $staleTTL,
                    $segmentable,
                    ( $curValue === false )
                );
            }
        }
 
        if ( $hasLock ) {
            $this->cache->delete( $mutexKey, $this->cache::WRITE_BACKGROUND );
        }
 
        $miss = is_infinite( $minAsOf ) ? 'renew' : 'miss';
        $this->logger->debug( "fetchOrRegenerate($key): $miss, new value computed" );
 
        $this->stats->getTiming( 'wanobjectcache_getwithset_seconds' )
            ->setLabel( 'keygroup', $keygroup )
            ->setLabel( 'result', $miss )
            ->setLabel( 'reason', 'compute' )
            ->copyToStatsdAt( "wanobjectcache.$keygroup.$miss.compute" )
            ->observe( 1e3 * ( $this->getCurrentTime() - $startTime ) );
 
        return [ $value, $version, $curState[self::RES_AS_OF] ];
    }
 
    private function makeSisterKey( string $baseKey, string $typeChar ) {
        if ( $this->coalesceScheme === self::SCHEME_HASH_STOP ) {
            // Key style: "WANCache:<base key>|#|<character>"
            $sisterKey = 'WANCache:' . $baseKey . '|#|' . $typeChar;
        } else {
            // Key style: "WANCache:{<base key>}:<character>"
            $sisterKey = 'WANCache:{' . $baseKey . '}:' . $typeChar;
        }
        return $sisterKey;
    }
 
    private function getInterimValue( $key, $minAsOf, $now, $touchedCb ) {
        if ( $this->useInterimHoldOffCaching ) {
            $interimSisterKey = $this->makeSisterKey( $key, self::TYPE_INTERIM );
            $wrapped = $this->cache->get( $interimSisterKey );
            $res = $this->unwrap( $wrapped, $now );
            if ( $res[self::RES_VALUE] !== false && $res[self::RES_AS_OF] >= $minAsOf ) {
                if ( $touchedCb !== null ) {
                    // Update "last purge time" since the $touchedCb timestamp depends on $value
                    // Get the new "touched timestamp", accounting for callback-checked dependencies
                    $res[self::RES_TOUCH_AS_OF] = max(
                        $touchedCb( $res[self::RES_VALUE] ),
                        $res[self::RES_TOUCH_AS_OF]
                    );
                }
 
                return $res;
            }
        }
 
        return $this->unwrap( false, $now );
    }
 
    private function setInterimValue(
        $key,
        $value,
        $ttl,
        ?int $version,
        bool $segmentable
    ) {
        $now = $this->getCurrentTime();
        $ttl = max( self::INTERIM_KEY_TTL, (int)$ttl );
 
        // Wrap that value with time/TTL/version metadata
        $wrapped = $this->wrap( $value, $ttl, $version, $now );
 
        $flags = $this->cache::WRITE_BACKGROUND;
        if ( $segmentable ) {
            $flags |= $this->cache::WRITE_ALLOW_SEGMENTS;
        }
 
        return $this->cache->set(
            $this->makeSisterKey( $key, self::TYPE_INTERIM ),
            $wrapped,
            $ttl,
            $flags
        );
    }
 
    final public function getMultiWithSetCallback(
        ArrayIterator $keyedIds, $ttl, callable $callback, array $opts = []
    ) {
        $span = $this->startOperationSpan( __FUNCTION__, '' );
        if ( $span->getContext()->isSampled() ) {
            $span->setAttributes( [
                'org.wikimedia.wancache.multi_count' => $keyedIds->count(),
                'org.wikimedia.wancache.ttl' => $ttl,
            ] );
        }
        // Batch load required keys into the in-process warmup cache
        $this->warmupCache = $this->fetchWrappedValuesForWarmupCache(
            $this->getNonProcessCachedMultiKeys( $keyedIds, $opts ),
            $opts['checkKeys'] ?? []
        );
        $this->warmupKeyMisses = 0;
 
        // The required callback signature includes $id as the first argument for convenience
        // to distinguish different items. To reuse the code in getWithSetCallback(), wrap the
        // callback with a proxy callback that has the standard getWithSetCallback() signature.
        // This is defined only once per batch to avoid closure creation overhead.
        $proxyCb = static function ( $oldValue, &$ttl, &$setOpts, $oldAsOf, $params )
            use ( $callback )
        {
            return $callback( $params['id'], $oldValue, $ttl, $setOpts, $oldAsOf );
        };
 
        // Get the order-preserved result map using the warm-up cache
        $values = [];
        foreach ( $keyedIds as $key => $id ) {
            $values[$key] = $this->getWithSetCallback(
                $key,
                $ttl,
                $proxyCb,
                $opts,
                [ 'id' => $id ]
            );
        }
 
        $this->warmupCache = [];
 
        return $values;
    }
 
    final public function getMultiWithUnionSetCallback(
        ArrayIterator $keyedIds, $ttl, callable $callback, array $opts = []
    ) {
        $span = $this->startOperationSpan( __FUNCTION__, '' );
        if ( $span->getContext()->isSampled() ) {
            $span->setAttributes( [
                'org.wikimedia.wancache.multi_count' => $keyedIds->count(),
                'org.wikimedia.wancache.ttl' => $ttl,
            ] );
        }
        $checkKeys = $opts['checkKeys'] ?? [];  // TODO: ???
        $minAsOf = $opts['minAsOf'] ?? self::MIN_TIMESTAMP_NONE;
 
        // unset incompatible keys
        unset( $opts['lockTSE'] );
        unset( $opts['busyValue'] );
 
        // Batch load required keys into the in-process warmup cache
        $keysByIdGet = $this->getNonProcessCachedMultiKeys( $keyedIds, $opts );
        $this->warmupCache = $this->fetchWrappedValuesForWarmupCache( $keysByIdGet, $checkKeys );
        $this->warmupKeyMisses = 0;
 
        // IDs of entities known to be in need of generation
        $idsRegen = [];
 
        // Find out which keys are missing/deleted/stale
        $now = $this->getCurrentTime();
        $resByKey = $this->fetchKeys( $keysByIdGet, $checkKeys, $now );
        foreach ( $keysByIdGet as $id => $key ) {
            $res = $resByKey[$key];
            if (
                $res[self::RES_VALUE] === false ||
                $res[self::RES_CUR_TTL] < 0 ||
                $res[self::RES_AS_OF] < $minAsOf
            ) {
                $idsRegen[] = $id;
            }
        }
 
        // Run the callback to populate the generation value map for all required IDs
        $newSetOpts = [];
        $newTTLsById = array_fill_keys( $idsRegen, $ttl );
        $newValsById = $idsRegen ? $callback( $idsRegen, $newTTLsById, $newSetOpts ) : [];
 
        $method = __METHOD__;
        // The required callback signature includes $id as the first argument for convenience
        // to distinguish different items. To reuse the code in getWithSetCallback(), wrap the
        // callback with a proxy callback that has the standard getWithSetCallback() signature.
        // This is defined only once per batch to avoid closure creation overhead.
        $proxyCb = function ( $oldValue, &$ttl, &$setOpts, $oldAsOf, $params )
            use ( $callback, $newValsById, $newTTLsById, $newSetOpts, $method )
        {
            $id = $params['id'];
 
            if ( array_key_exists( $id, $newValsById ) ) {
                // Value was already regenerated as expected, so use the value in $newValsById
                $newValue = $newValsById[$id];
                $ttl = $newTTLsById[$id];
                $setOpts = $newSetOpts;
            } else {
                // Pre-emptive/popularity refresh and version mismatch cases are not detected
                // above and thus $newValsById has no entry. Run $callback on this single entity.
                $ttls = [ $id => $ttl ];
                $result = $callback( [ $id ], $ttls, $setOpts );
                if ( !isset( $result[$id] ) ) {
                    // T303092
                    $this->logger->warning(
                        $method . ' failed due to {id} not set in result {result}', [
                            'id' => $id,
                            'result' => json_encode( $result )
                        ] );
                }
                $newValue = $result[$id];
                $ttl = $ttls[$id];
            }
 
            return $newValue;
        };
 
        // Get the order-preserved result map using the warm-up cache
        $values = [];
        foreach ( $keyedIds as $key => $id ) {
            $values[$key] = $this->getWithSetCallback(
                $key,
                $ttl,
                $proxyCb,
                $opts,
                [ 'id' => $id ]
            );
        }
 
        $this->warmupCache = [];
 
        return $values;
    }
 
    public function makeGlobalKey( $keygroup, ...$components ) {
        return $this->cache->makeGlobalKey( $keygroup, ...$components );
    }
 
    public function makeKey( $keygroup, ...$components ) {
        return $this->cache->makeKey( $keygroup, ...$components );
    }
 
    public function hash256( $component ) {
        return hash_hmac( 'sha256', $component, $this->secret );
    }
 
    final public function makeMultiKeys( array $ids, $keyCallback ) {
        $idByKey = [];
        foreach ( $ids as $id ) {
            // Discourage triggering of automatic makeKey() hashing in some backends
            if ( strlen( $id ) > 64 ) {
                $this->logger->warning( __METHOD__ . ": long ID '$id'; use hash256()" );
            }
            $key = $keyCallback( $id, $this );
            // Edge case: ignore key collisions due to duplicate $ids like "42" and 42
            if ( !isset( $idByKey[$key] ) ) {
                $idByKey[$key] = $id;
            } elseif ( (string)$id !== (string)$idByKey[$key] ) {
                throw new UnexpectedValueException(
                    "Cache key collision; IDs ('$id','{$idByKey[$key]}') map to '$key'"
                );
            }
        }
 
        return new ArrayIterator( $idByKey );
    }
 
    final public function multiRemap( array $ids, array $res ) {
        if ( count( $ids ) !== count( $res ) ) {
            // If makeMultiKeys() is called on a list of non-unique IDs, then the resulting
            // ArrayIterator will have less entries due to "first appearance" de-duplication
            $ids = array_keys( array_fill_keys( $ids, true ) );
            if ( count( $ids ) !== count( $res ) ) {
                throw new UnexpectedValueException( "Multi-key result does not match ID list" );
            }
        }
 
        return array_combine( $ids, $res );
    }
 
    public function watchErrors() {
        return $this->cache->watchErrors();
    }
 
    final public function getLastError( $watchPoint = 0 ) {
        $code = $this->cache->getLastError( $watchPoint );
        switch ( $code ) {
            case BagOStuff::ERR_NONE:
                return BagOStuff::ERR_NONE;
            case BagOStuff::ERR_NO_RESPONSE:
                return BagOStuff::ERR_NO_RESPONSE;
            case BagOStuff::ERR_UNREACHABLE:
                return BagOStuff::ERR_UNREACHABLE;
            default:
                return BagOStuff::ERR_UNEXPECTED;
        }
    }
 
    public function clearProcessCache() {
        $this->processCaches = [];
    }
 
    final public function useInterimHoldOffCaching( $enabled ) {
        $this->useInterimHoldOffCaching = $enabled;
    }
 
    public function getQoS( $flag ) {
        return $this->cache->getQoS( $flag );
    }
 
    public function adaptiveTTL( $mtime, $maxTTL, $minTTL = 30, $factor = 0.2 ) {
        // handle fractional seconds and string integers
        $mtime = (int)$mtime;
        if ( $mtime <= 0 ) {
            // no last-modified time provided
            return $minTTL;
        }
 
        $age = (int)$this->getCurrentTime() - $mtime;
 
        return (int)min( $maxTTL, max( $minTTL, $factor * $age ) );
    }
 
    final public function getWarmupKeyMisses() {
        // Number of misses in $this->warmupCache during the last call to certain methods
        return $this->warmupKeyMisses;
    }
 
    protected function getRouteKey( string $sisterKey ) {
        if ( $this->broadcastRoute !== null ) {
            if ( $sisterKey[0] === '/' ) {
                throw new RuntimeException( "Sister key '$sisterKey' already contains a route." );
            }
            return $this->broadcastRoute . $sisterKey;
        }
        return $sisterKey;
    }
 
    private function scheduleAsyncRefresh( $key, $ttl, $callback, array $opts, array $cbParams ) {
        if ( !$this->asyncHandler ) {
            return false;
        }
        // Update the cache value later, such during post-send of an HTTP request. This forces
        // cache regeneration by setting "minAsOf" to infinity, meaning that no existing value
        // is considered valid. Furthermore, note that preemptive regeneration is not applicable
        // to invalid values, so there is no risk of infinite preemptive regeneration loops.
        $func = $this->asyncHandler;
        $func( function () use ( $key, $ttl, $callback, $opts, $cbParams ) {
            $opts['minAsOf'] = INF;
            try {
                $this->fetchOrRegenerate( $key, $ttl, $callback, $opts, $cbParams );
            } catch ( Exception $e ) {
                // Log some context for easier debugging
                $this->logger->error( 'Async refresh failed for {key}', [
                    'key' => $key,
                    'ttl' => $ttl,
                    'exception' => $e
                ] );
                throw $e;
            }
        } );
 
        return true;
    }
 
    private function isAcceptablyFreshValue( $res, $graceTTL, $minAsOf ) {
        if ( !$this->isValid( $res[self::RES_VALUE], $res[self::RES_AS_OF], $minAsOf ) ) {
            // Value does not exists or is too old
            return false;
        }
 
        $curTTL = $res[self::RES_CUR_TTL];
        if ( $curTTL > 0 ) {
            // Value is definitely still fresh
            return true;
        }
 
        // Remaining seconds during which this stale value can be used
        $curGraceTTL = $graceTTL + $curTTL;
 
        return ( $curGraceTTL > 0 )
            // Chance of using the value decreases as $curTTL goes from 0 to -$graceTTL
            ? !$this->worthRefreshExpiring( $curGraceTTL, $graceTTL, $graceTTL )
            // Value is too stale to fall in the grace period
            : false;
    }
 
    protected function isLotteryRefreshDue( $res, $lowTTL, $ageNew, $hotTTR, $now ) {
        $curTTL = $res[self::RES_CUR_TTL];
        $logicalTTL = $res[self::RES_TTL];
        $asOf = $res[self::RES_AS_OF];
 
        return (
            $this->worthRefreshExpiring( $curTTL, $logicalTTL, $lowTTL ) ||
            $this->worthRefreshPopular( $asOf, $ageNew, $hotTTR, $now )
        );
    }
 
    protected function worthRefreshPopular( $asOf, $ageNew, $timeTillRefresh, $now ) {
        if ( $ageNew < 0 || $timeTillRefresh <= 0 ) {
            return false;
        }
 
        $age = $now - $asOf;
        $timeOld = $age - $ageNew;
        if ( $timeOld <= 0 ) {
            return false;
        }
 
        $popularHitsPerSec = 1;
        // Lifecycle is: new, ramp-up refresh chance, full refresh chance.
        // Note that the "expected # of refreshes" for the ramp-up time range is half
        // of what it would be if P(refresh) was at its full value during that time range.
        $refreshWindowSec = max( $timeTillRefresh - $ageNew - self::RAMPUP_TTL / 2, 1 );
        // P(refresh) * (# hits in $refreshWindowSec) = (expected # of refreshes)
        // P(refresh) * ($refreshWindowSec * $popularHitsPerSec) = 1 (by definition)
        // P(refresh) = 1/($refreshWindowSec * $popularHitsPerSec)
        $chance = 1 / ( $popularHitsPerSec * $refreshWindowSec );
        // Ramp up $chance from 0 to its nominal value over RAMPUP_TTL seconds to avoid stampedes
        $chance *= ( $timeOld <= self::RAMPUP_TTL ) ? $timeOld / self::RAMPUP_TTL : 1;
 
        return ( mt_rand( 1, 1_000_000_000 ) <= 1_000_000_000 * $chance );
    }
 
    protected function worthRefreshExpiring( $curTTL, $logicalTTL, $lowTTL ) {
        if ( $lowTTL <= 0 ) {
            return false;
        }
        // T264787: avoid having keys start off with a high chance of being refreshed;
        // the point where refreshing becomes possible cannot precede the key lifetime.
        $effectiveLowTTL = min( $lowTTL, $logicalTTL ?: INF );
 
        // How long the value was in the "low TTL" phase
        $timeOld = $effectiveLowTTL - $curTTL;
        if ( $timeOld <= 0 || $timeOld >= $effectiveLowTTL ) {
            return false;
        }
 
        // Ratio of the low TTL phase that has elapsed (r)
        $ttrRatio = $timeOld / $effectiveLowTTL;
        // Use p(r) as the monotonically increasing "chance of refresh" function,
        // having p(0)=0 and p(1)=1. The value expires at the nominal expiry.
        $chance = $ttrRatio ** 4;
 
        return ( mt_rand( 1, 1_000_000_000 ) <= 1_000_000_000 * $chance );
    }
 
    protected function isValid( $value, $asOf, $minAsOf ) {
        return ( $value !== false && $asOf >= $minAsOf );
    }
 
    private function wrap( $value, $ttl, $version, $now ) {
        // Returns keys in ascending integer order for PHP7 array packing:
        // https://nikic.github.io/2014/12/22/PHPs-new-hashtable-implementation.html
        $wrapped = [
            self::FLD_FORMAT_VERSION => self::VERSION,
            self::FLD_VALUE => $value,
            self::FLD_TTL => $ttl,
            self::FLD_TIME => $now
        ];
        if ( $version !== null ) {
            $wrapped[self::FLD_VALUE_VERSION] = $version;
        }
 
        return $wrapped;
    }
 
    private function unwrap( $wrapped, $now ) {
        // https://nikic.github.io/2014/12/22/PHPs-new-hashtable-implementation.html
        $res = [
            // Attributes that only depend on the fetched key value
            self::RES_VALUE => false,
            self::RES_VERSION => null,
            self::RES_AS_OF => null,
            self::RES_TTL => null,
            self::RES_TOMB_AS_OF => null,
            // Attributes that depend on caller-specific "check" keys or "touched callbacks"
            self::RES_CHECK_AS_OF => null,
            self::RES_TOUCH_AS_OF => null,
            self::RES_CUR_TTL => null
        ];
 
        if ( is_array( $wrapped ) ) {
            // Entry expected to be a cached value; validate it
            if (
                ( $wrapped[self::FLD_FORMAT_VERSION] ?? null ) === self::VERSION &&
                $wrapped[self::FLD_TIME] >= $this->epoch
            ) {
                if ( $wrapped[self::FLD_TTL] > 0 ) {
                    // Get the approximate time left on the key
                    $age = $now - $wrapped[self::FLD_TIME];
                    $curTTL = max( $wrapped[self::FLD_TTL] - $age, 0.0 );
                } else {
                    // Key had no TTL, so the time left is unbounded
                    $curTTL = INF;
                }
                $res[self::RES_VALUE] = $wrapped[self::FLD_VALUE];
                $res[self::RES_VERSION] = $wrapped[self::FLD_VALUE_VERSION] ?? null;
                $res[self::RES_AS_OF] = $wrapped[self::FLD_TIME];
                $res[self::RES_CUR_TTL] = $curTTL;
                $res[self::RES_TTL] = $wrapped[self::FLD_TTL];
            }
        } else {
            // Entry expected to be a tombstone; parse it
            $purge = $this->parsePurgeValue( $wrapped );
            if ( $purge !== null ) {
                // Tombstoned keys should always have a negative "current TTL"
                $curTTL = min( $purge[self::PURGE_TIME] - $now, self::TINY_NEGATIVE );
                $res[self::RES_CUR_TTL] = $curTTL;
                $res[self::RES_TOMB_AS_OF] = $purge[self::PURGE_TIME];
            }
        }
 
        return $res;
    }
 
    private function determineKeyGroupForStats( $key ) {
        $parts = explode( ':', $key, 3 );
        // Fallback in case the key was not made by makeKey.
        // Replace dots because they are special in StatsD (T232907)
        return strtr( $parts[1] ?? $parts[0], '.', '_' );
    }
 
    private function parsePurgeValue( $value ) {
        if ( !is_string( $value ) ) {
            return null;
        }
 
        $segments = explode( ':', $value, 3 );
        $prefix = $segments[0];
        if ( $prefix !== self::PURGE_VAL_PREFIX ) {
            // Not a purge value
            return null;
        }
 
        $timestamp = (float)$segments[1];
        // makeTombstonePurgeValue() doesn't store hold-off TTLs
        $holdoff = isset( $segments[2] ) ? (int)$segments[2] : self::HOLDOFF_TTL;
 
        if ( $timestamp < $this->epoch ) {
            // Purge value is too old
            return null;
        }
 
        return [ self::PURGE_TIME => $timestamp, self::PURGE_HOLDOFF => $holdoff ];
    }
 
    private function makeCheckPurgeValue( float $timestamp, int $holdoff, ?array &$purge = null ) {
        $normalizedTime = (int)$timestamp;
        // Purge array that matches what parsePurgeValue() would have returned
        $purge = [ self::PURGE_TIME => (float)$normalizedTime, self::PURGE_HOLDOFF => $holdoff ];
 
        return self::PURGE_VAL_PREFIX . ":$normalizedTime:$holdoff";
    }
 
    private function getProcessCache( $group ) {
        if ( !isset( $this->processCaches[$group] ) ) {
            [ , $size ] = explode( ':', $group );
            $this->processCaches[$group] = new MapCacheLRU( (int)$size );
            if ( $this->wallClockOverride !== null ) {
                $this->processCaches[$group]->setMockTime( $this->wallClockOverride );
            }
        }
 
        return $this->processCaches[$group];
    }
 
    private function getNonProcessCachedMultiKeys( ArrayIterator $keys, array $opts ) {
        $pcTTL = $opts['pcTTL'] ?? self::TTL_UNCACHEABLE;
 
        $keysMissing = [];
        if ( $pcTTL > 0 && $this->callbackDepth == 0 ) {
            $pCache = $this->getProcessCache( $opts['pcGroup'] ?? self::PC_PRIMARY );
            foreach ( $keys as $key => $id ) {
                if ( !$pCache->has( $key, $pcTTL ) ) {
                    $keysMissing[$id] = $key;
                }
            }
        }
 
        return $keysMissing;
    }
 
    private function fetchWrappedValuesForWarmupCache( array $keys, array $checkKeys ) {
        if ( !$keys ) {
            return [];
        }
 
        // Get all the value keys to fetch...
        $sisterKeys = [];
        foreach ( $keys as $baseKey ) {
            $sisterKeys[] = $this->makeSisterKey( $baseKey, self::TYPE_VALUE );
        }
        // Get all the "check" keys to fetch...
        foreach ( $checkKeys as $i => $checkKeyOrKeyGroup ) {
            // Note: avoid array_merge() inside loop in case there are many keys
            if ( is_int( $i ) ) {
                // Single "check" key that applies to all value keys
                $sisterKeys[] = $this->makeSisterKey( $checkKeyOrKeyGroup, self::TYPE_TIMESTAMP );
            } else {
                // List of "check" keys that apply to a specific value key
                foreach ( (array)$checkKeyOrKeyGroup as $checkKey ) {
                    $sisterKeys[] = $this->makeSisterKey( $checkKey, self::TYPE_TIMESTAMP );
                }
            }
        }
 
        $wrappedBySisterKey = $this->cache->getMulti( $sisterKeys );
        $wrappedBySisterKey += array_fill_keys( $sisterKeys, false );
 
        return $wrappedBySisterKey;
    }
 
    private function timeSinceLoggedMiss( $key, $now ) {
        // phpcs:ignore Generic.CodeAnalysis.AssignmentInCondition.Found
        for ( end( $this->missLog ); $miss = current( $this->missLog ); prev( $this->missLog ) ) {
            if ( $miss[0] === $key ) {
                return ( $now - $miss[1] );
            }
        }
 
        return null;
    }
 
    protected function getCurrentTime() {
        return $this->wallClockOverride ?: microtime( true );
    }
 
    public function setMockTime( &$time ) {
        $this->wallClockOverride =& $time;
        $this->cache->setMockTime( $time );
        foreach ( $this->processCaches as $pCache ) {
            $pCache->setMockTime( $time );
        }
    }
 
    private function startOperationSpan( $opName, $keys, $checkKeys = [] ) {
        $span = $this->tracer->createSpan( "WANObjectCache::$opName" )
            ->setSpanKind( SpanInterface::SPAN_KIND_CLIENT )
            ->start();
 
        if ( !$span->getContext()->isSampled() ) {
            return $span;
        }
 
        $keys = is_array( $keys ) ? implode( ' ', $keys ) : $keys;
 
        if ( count( $checkKeys ) > 0 ) {
            $checkKeys = array_map(
                static fn ( $checkKeyOrKeyGroup ) =>
                    is_array( $checkKeyOrKeyGroup )
                        ? implode( ' ', $checkKeyOrKeyGroup )
                        : $checkKeyOrKeyGroup,
                $checkKeys );
 
            $checkKeys = implode( ' ', $checkKeys );
            $span->setAttributes( [ 'org.wikimedia.wancache.check_keys' => $checkKeys ] );
        }
 
        $span->setAttributes( [ 'org.wikimedia.wancache.keys' => $keys ] );
 
        $span->activate();
        return $span;
    }
}
 
class_alias( WANObjectCache::class, 'WANObjectCache' );