LocalisationCache.php

Go to the documentation of this file.

<?php
namespace MediaWiki\Language;
 
use CLDRPluralRuleParser\Error as CLDRPluralRuleError;
use CLDRPluralRuleParser\Evaluator;
use DOMDocument;
use InvalidArgumentException;
use LogicException;
use MediaWiki\Config\ConfigException;
use MediaWiki\Config\ServiceOptions;
use MediaWiki\HookContainer\HookContainer;
use MediaWiki\HookContainer\HookRunner;
use MediaWiki\Json\FormatJson;
use MediaWiki\Language\Dependency\CacheDependency;
use MediaWiki\Language\Dependency\ConstantDependency;
use MediaWiki\Language\Dependency\FileDependency;
use MediaWiki\Language\Dependency\MainConfigDependency;
use MediaWiki\MainConfigNames;
use MediaWiki\MediaWikiServices;
use Psr\Log\LoggerInterface;
use RuntimeException;
use UnexpectedValueException;
use Wikimedia\Leximorph\Provider as LeximorphProvider;
use Wikimedia\Leximorph\Provider\PluralRules as LeximorphPluralRulesProvider;
 
class LocalisationCache {
    public const VERSION = 7;
 
    private $manualRecache;
 
    protected $data = [];
 
    protected $sourceLanguage = [];
 
    private HookRunner $hookRunner;
    private $clearStoreCallbacks;
 
    private $loadedItems = [];
 
    private $loadedSubitems = [];
 
    private $initialisedLangs = [];
 
    private $shallowFallbacks = [];
 
    private $fallbackCodes = [];
 
    private $recachedLangs = [];
 
    private $coreDataLoaded = [];
 
    public const ALL_KEYS = [
        'fallback', 'namespaceNames', 'bookstoreList',
        'magicWords', 'messages', 'rtl',
        'digitTransformTable', 'separatorTransformTable',
        'minimumGroupingDigits', 'numberingSystem', 'fallback8bitEncoding',
        'linkPrefixExtension', 'linkTrail', 'linkPrefixCharset',
        'namespaceAliases', 'dateFormats', 'jsDateFormats', 'datePreferences',
        'datePreferenceMigrationMap', 'defaultDateFormat',
        'specialPageAliases', 'imageFiles', 'preloadedMessages',
        'namespaceGenderAliases', 'digitGroupingPattern', 'pluralRules',
        'pluralRuleTypes', 'compiledPluralRules', 'formalityIndex'
    ];
 
    private const CORE_ONLY_KEYS = [
        'fallback', 'rtl', 'digitTransformTable', 'separatorTransformTable',
        'minimumGroupingDigits', 'numberingSystem',
        'fallback8bitEncoding', 'linkPrefixExtension',
        'linkTrail', 'linkPrefixCharset', 'datePreferences',
        'datePreferenceMigrationMap', 'defaultDateFormat', 'digitGroupingPattern',
        'formalityIndex',
    ];
 
    private const ALL_EXCEPT_CORE_ONLY_KEYS = [
        'namespaceNames', 'bookstoreList', 'magicWords', 'messages',
        'namespaceAliases', 'dateFormats', 'jsDateFormats', 'specialPageAliases',
        'imageFiles', 'preloadedMessages', 'namespaceGenderAliases',
        'pluralRules', 'pluralRuleTypes', 'compiledPluralRules',
    ];
 
    public const ALL_ALIAS_KEYS = [ 'specialPageAliases' ];
 
    private const MERGEABLE_MAP_KEYS = [ 'messages', 'namespaceNames',
        'namespaceAliases', 'dateFormats', 'jsDateFormats', 'imageFiles', 'preloadedMessages'
    ];
 
    private const MERGEABLE_ALIAS_LIST_KEYS = [ 'specialPageAliases' ];
 
    private const OPTIONAL_MERGE_KEYS = [ 'bookstoreList' ];
 
    private const MAGIC_WORD_KEYS = [ 'magicWords' ];
 
    private const SPLIT_KEYS = [ 'messages' ];
 
    private const SOURCE_PREFIX_KEYS = [ 'messages' ];
 
    private const SOURCEPREFIX_SEPARATOR = ':';
 
    private const PRELOADED_KEYS = [ 'dateFormats', 'namespaceNames' ];
 
    private const META_KEYS = [ 'deps', 'list', 'preload' ];
 
    private const PLURAL_FILES = [
        // Load CLDR plural rules
        MW_INSTALL_PATH . '/languages/data/plurals.xml',
        // Override or extend with MW-specific rules
        MW_INSTALL_PATH . '/languages/data/plurals-mediawiki.xml',
    ];
 
    private static $pluralRules = null;
 
    private static $pluralRuleTypes = null;
 
    private $expiredReason = [];
 
    public static function getStoreFromConf( array $conf, $fallbackCacheDir ): LCStore {
        $storeArg = [
            'directory' => $conf['storeDirectory'] ?: $fallbackCacheDir,
        ];
 
        // Custom classes are supported. For core builtins, short names are stable (since 1.23).
        $storeClass = match ( $conf['storeClass'] ) {
            'LCStoreCDB' => LCStoreCDB::class,
            'LCStoreDB' => LCStoreDB::class,
            'LCStoreNull' => LCStoreNull::class,
            'LCStoreStaticArray' => LCStoreStaticArray::class,
            default => $conf['storeClass'],
        };
        if ( !$storeClass ) {
            if (
                $conf['store'] === 'files'
                || $conf['store'] === 'file'
                || ( $conf['store'] === 'detect' && $storeArg['directory'] )
            ) {
                $storeClass = LCStoreCDB::class;
            } elseif ( $conf['store'] === 'db' || $conf['store'] === 'detect' ) {
                $storeClass = LCStoreDB::class;
                $storeArg['server'] = $conf['storeServer'] ?? [];
            } elseif ( $conf['store'] === 'array' ) {
                $storeClass = LCStoreStaticArray::class;
            } else {
                throw new ConfigException(
                    'Please set $wgLocalisationCacheConf[\'store\'] to something sensible.'
                );
            }
        }
 
        return new $storeClass( $storeArg );
    }
 
    public const CONSTRUCTOR_OPTIONS = [
        // True to treat all files as expired until they are regenerated by this object.
        'forceRecache',
        'manualRecache',
        MainConfigNames::ExtensionMessagesFiles,
        MainConfigNames::MessagesDirs,
        MainConfigNames::TranslationAliasesDirs,
        MainConfigNames::UseLeximorph,
    ];
 
    public function __construct(
        private readonly ServiceOptions $options,
        private LCStore $store,
        private readonly LoggerInterface $logger,
        array $clearStoreCallbacks,
        private readonly LanguageNameUtils $langNameUtils,
        HookContainer $hookContainer,
    ) {
        $options->assertRequiredOptions( self::CONSTRUCTOR_OPTIONS );
 
        $this->clearStoreCallbacks = $clearStoreCallbacks;
        $this->hookRunner = new HookRunner( $hookContainer );
 
        // Keep this separate from $this->options so that it can be mutable
        $this->manualRecache = $options->get( 'manualRecache' );
    }
 
    private static function isMergeableKey( string $key ): bool {
        static $mergeableKeys;
        $mergeableKeys ??= array_fill_keys( [
            ...self::MERGEABLE_MAP_KEYS,
            ...self::MERGEABLE_ALIAS_LIST_KEYS,
            ...self::OPTIONAL_MERGE_KEYS,
            ...self::MAGIC_WORD_KEYS,
        ], true );
        return isset( $mergeableKeys[$key] );
    }
 
    public function getItem( $code, $key ) {
        if ( !isset( $this->loadedItems[$code][$key] ) ) {
            $this->loadItem( $code, $key );
        }
 
        if ( $key === 'fallback' && isset( $this->shallowFallbacks[$code] ) ) {
            return $this->shallowFallbacks[$code];
        }
 
        // @phan-suppress-next-line PhanTypeArraySuspiciousNullable
        return $this->data[$code][$key];
    }
 
    public function getSubitem( $code, $key, $subkey ) {
        if ( !isset( $this->loadedSubitems[$code][$key][$subkey] ) &&
            !isset( $this->loadedItems[$code][$key] )
        ) {
            $this->loadSubitem( $code, $key, $subkey );
        }
 
        return $this->data[$code][$key][$subkey] ?? null;
    }
 
    public function getSubitemWithSource( $code, $key, $subkey ) {
        $subitem = $this->getSubitem( $code, $key, $subkey );
        // Undefined in the backend.
        if ( $subitem === null ) {
            return null;
        }
 
        // The source language should have been set, but to avoid a Phan error and to be double sure.
        return [ $subitem, $this->sourceLanguage[$code][$key][$subkey] ?? $code ];
    }
 
    public function getSubitemList( $code, $key ) {
        if ( in_array( $key, self::SPLIT_KEYS ) ) {
            return $this->getSubitem( $code, 'list', $key );
        } else {
            $item = $this->getItem( $code, $key );
            if ( is_array( $item ) ) {
                return array_keys( $item );
            } else {
                return false;
            }
        }
    }
 
    private function loadItem( $code, $key ) {
        if ( isset( $this->loadedItems[$code][$key] ) ) {
            return;
        }
 
        if (
            in_array( $key, self::CORE_ONLY_KEYS, true ) ||
            // "synthetic" keys added by loadCoreData based on "fallback"
            $key === 'fallbackSequence' ||
            $key === 'originalFallbackSequence'
        ) {
            if ( $this->langNameUtils->isValidBuiltInCode( $code ) ) {
                $this->loadCoreData( $code );
                return;
            }
        }
 
        if ( !isset( $this->initialisedLangs[$code] ) ) {
            $this->initLanguage( $code );
 
            // Check to see if initLanguage() loaded it for us
            if ( isset( $this->loadedItems[$code][$key] ) ) {
                return;
            }
        }
 
        if ( isset( $this->shallowFallbacks[$code] ) ) {
            $this->loadItem( $this->shallowFallbacks[$code], $key );
 
            return;
        }
 
        if ( in_array( $key, self::SPLIT_KEYS ) ) {
            $subkeyList = $this->getSubitem( $code, 'list', $key );
            foreach ( $subkeyList as $subkey ) {
                if ( isset( $this->data[$code][$key][$subkey] ) ) {
                    continue;
                }
                $this->loadSubitem( $code, $key, $subkey );
            }
        } else {
            $this->data[$code][$key] = $this->getFromStore( $code, $key );
        }
 
        $this->loadedItems[$code][$key] = true;
    }
 
    private function getFromStore( string $code, string $key ) {
        if ( $this->store->lateFallback() ) {
            $result = null;
            foreach ( $this->getFallbackCodes( $code ) as $langCode ) {
                $value = $this->store->get( $langCode, $key );
                $this->mergeItem( $key, $result, $value );
                if ( in_array( $key, self::META_KEYS ) ) {
                    break;
                }
                if ( is_string( $result ) ) {
                    // No need to do merges or look further
                    break;
                }
            }
            return $result;
        }
        return $this->store->get( $code, $key );
    }
 
    protected function getFallbackCodes( string $code ): array {
        if ( !array_key_exists( $code, $this->fallbackCodes ) ) {
            $this->fallbackCodes[$code] = [
                $code,
                ...MediaWikiServices::getInstance()->getLanguageFallback()->getAll( $code )
            ];
        }
        return $this->fallbackCodes[$code];
    }
 
    private function loadSubitem( $code, $key, $subkey ) {
        if ( !in_array( $key, self::SPLIT_KEYS ) ) {
            $this->loadItem( $code, $key );
 
            return;
        }
 
        if ( !isset( $this->initialisedLangs[$code] ) ) {
            $this->initLanguage( $code );
        }
 
        // Check to see if initLanguage() loaded it for us
        if ( isset( $this->loadedItems[$code][$key] ) ||
            isset( $this->loadedSubitems[$code][$key][$subkey] )
        ) {
            return;
        }
 
        if ( isset( $this->shallowFallbacks[$code] ) ) {
            $this->loadSubitem( $this->shallowFallbacks[$code], $key, $subkey );
 
            return;
        }
 
        $value = $this->getFromStore( $code, "$key:$subkey" );
        if ( $value !== null && in_array( $key, self::SOURCE_PREFIX_KEYS ) ) {
            [
                $this->sourceLanguage[$code][$key][$subkey],
                $this->data[$code][$key][$subkey]
            ] = explode( self::SOURCEPREFIX_SEPARATOR, $value, 2 );
        } else {
            $this->data[$code][$key][$subkey] = $value;
        }
 
        $this->loadedSubitems[$code][$key][$subkey] = true;
    }
 
    public function isExpired( $code ) {
        if ( $this->options->get( 'forceRecache' ) && !isset( $this->recachedLangs[$code] ) ) {
            $this->logger->debug( __METHOD__ . "($code): forced reload" );
 
            $this->expiredReason[$code] = "Forced rebuild requested";
            return true;
        }
 
        $deps = $this->getFromStore( $code, 'deps' );
        $keys = $this->getFromStore( $code, 'list' );
        $preload = $this->getFromStore( $code, 'preload' );
        // Different keys may expire separately for some stores
        if ( $deps === null || $keys === null || $preload === null ) {
            $this->logger->debug( __METHOD__ . "($code): cache missing, need to make one" );
 
            $this->expiredReason[$code] = "No existing cache";
            return true;
        }
 
        $expiredReason = '';
 
        $collectExpiredReason = static function ( $reason ) use ( &$expiredReason ) {
            $expiredReason = $reason;
        };
 
        foreach ( $deps as $dep ) {
            // Because we're unserializing stuff from cache, we
            // could receive objects of classes that don't exist
            // anymore (e.g., uninstalled extensions)
            // When this happens, always expire the cache
            if ( !$dep instanceof CacheDependency ) {
                $this->expiredReason[$code] = get_class( $dep ) . " is not a subtype of CacheDependency";
                return true;
            }
 
            if ( $dep->isExpired( $collectExpiredReason ) ) {
                $this->logger->debug( __METHOD__ . "($code): cache for $code expired due to " .
                    get_class( $dep ) );
                $this->expiredReason[$code] = $expiredReason;
                return true;
            }
        }
 
        return false;
    }
 
    public function getExpiredReason( $code ) {
        return $this->expiredReason[$code] ?? null;
    }
 
    private function initLanguage( $langCode ) {
        foreach ( array_reverse( $this->getFallbackCodes( $langCode ) ) as $code ) {
            if ( isset( $this->initialisedLangs[$code] ) ) {
                continue;
            }
 
            $this->initialisedLangs[$code] = true;
 
            # If the code is of the wrong form for a Messages*.php file, do a shallow fallback
            if ( !$this->langNameUtils->isValidBuiltInCode( $code ) ) {
                $this->initShallowFallback( $code, 'en' );
 
                continue;
            }
 
            # Re-cache the data if necessary
            if ( !$this->manualRecache && $this->isExpired( $code ) ) {
                if ( $this->langNameUtils->isSupportedLanguage( $code ) ) {
                    $this->recache( $code );
                } elseif ( $code === 'en' ) {
                    throw new RuntimeException( 'MessagesEn.php is missing.' );
                } else {
                    $this->initShallowFallback( $code, 'en' );
                }
 
                continue;
            }
 
            # Preload some stuff
            $preload = $this->getItem( $code, 'preload' );
            if ( $preload === null ) {
                if ( $this->manualRecache ) {
                    // No Messages*.php file. Do shallow fallback to en.
                    if ( $code === 'en' ) {
                        throw new RuntimeException( 'No localisation cache found for English. ' .
                            'Please run maintenance/rebuildLocalisationCache.php.' );
                    }
                    $this->initShallowFallback( $code, 'en' );
 
                    break;
                } else {
                    throw new RuntimeException( 'Invalid or missing localisation cache.' );
                }
            }
 
            foreach ( self::SOURCE_PREFIX_KEYS as $key ) {
                if ( !isset( $preload[$key] ) ) {
                    continue;
                }
                foreach ( $preload[$key] as $subkey => $value ) {
                    if ( $value !== null ) {
                        [
                            $this->sourceLanguage[$code][$key][$subkey],
                            $preload[$key][$subkey]
                        ] = explode( self::SOURCEPREFIX_SEPARATOR, $value, 2 );
                    } else {
                        $preload[$key][$subkey] = null;
                    }
                }
            }
 
            if ( isset( $this->data[$code] ) ) {
                foreach ( $preload as $key => $value ) {
                    // @phan-suppress-next-line PhanTypeArraySuspiciousNullable -- see isset() above
                    $this->mergeItem( $key, $this->data[$code][$key], $value );
                }
            } else {
                $this->data[$code] = $preload;
            }
            foreach ( $preload as $key => $item ) {
                if ( in_array( $key, self::SPLIT_KEYS ) ) {
                    foreach ( $item as $subkey => $subitem ) {
                        $this->loadedSubitems[$code][$key][$subkey] = true;
                    }
                } else {
                    $this->loadedItems[$code][$key] = true;
                }
            }
        }
    }
 
    private function initShallowFallback( $primaryCode, $fallbackCode ) {
        $this->data[$primaryCode] =& $this->data[$fallbackCode];
        $this->loadedItems[$primaryCode] =& $this->loadedItems[$fallbackCode];
        $this->loadedSubitems[$primaryCode] =& $this->loadedSubitems[$fallbackCode];
        $this->shallowFallbacks[$primaryCode] = $fallbackCode;
        $this->coreDataLoaded[$primaryCode] =& $this->coreDataLoaded[$fallbackCode];
    }
 
    protected function readPHPFile( $_fileName, $_fileType ) {
        include $_fileName;
 
        $data = [];
        if ( $_fileType == 'core' ) {
            foreach ( self::ALL_KEYS as $key ) {
                // Not all keys are set in language files, so
                // check they exist first
                // @phan-suppress-next-line MediaWikiNoIssetIfDefined  May be set in the included file
                if ( isset( $$key ) ) {
                    $data[$key] = $$key;
                }
            }
        } elseif ( $_fileType == 'extension' ) {
            foreach ( self::ALL_EXCEPT_CORE_ONLY_KEYS as $key ) {
                // @phan-suppress-next-line MediaWikiNoIssetIfDefined  May be set in the included file
                if ( isset( $$key ) ) {
                    $data[$key] = $$key;
                }
            }
        } elseif ( $_fileType == 'aliases' ) {
            // @phan-suppress-next-line PhanImpossibleCondition May be set in the included file
            if ( isset( $aliases ) ) {
                $data['aliases'] = $aliases;
            }
        } else {
            throw new InvalidArgumentException( __METHOD__ . ": Invalid file type: $_fileType" );
        }
 
        return $data;
    }
 
    private function readJSONFile( $fileName ) {
        if ( !is_readable( $fileName ) ) {
            return [];
        }
 
        $json = file_get_contents( $fileName );
        if ( $json === false ) {
            return [];
        }
 
        $data = FormatJson::decode( $json, true );
        if ( $data === null ) {
            throw new RuntimeException( __METHOD__ . ": Invalid JSON file: $fileName" );
        }
 
        // Remove keys starting with '@'; they are reserved for metadata and non-message data
        foreach ( $data as $key => $unused ) {
            if ( $key === '' || $key[0] === '@' ) {
                unset( $data[$key] );
            }
        }
 
        return $data;
    }
 
    private function getCompiledPluralRules( $code ) {
        $rules = $this->getPluralRules( $code );
        if ( $rules === null ) {
            return null;
        }
        try {
            $compiledRules = Evaluator::compile( $rules );
        } catch ( CLDRPluralRuleError $e ) {
            $this->logger->debug( $e->getMessage() );
 
            return [];
        }
 
        return $compiledRules;
    }
 
    private function getPluralRules( $code ) {
        if ( self::$pluralRules === null ) {
            self::loadPluralFiles();
        }
        return self::$pluralRules[$code] ?? null;
    }
 
    private function getPluralRuleTypes( $code ) {
        if ( self::$pluralRuleTypes === null ) {
            self::loadPluralFiles();
        }
        return self::$pluralRuleTypes[$code] ?? null;
    }
 
    private static function loadPluralFiles() {
        foreach ( self::PLURAL_FILES as $fileName ) {
            self::loadPluralFile( $fileName );
        }
    }
 
    private static function loadPluralFile( $fileName ) {
        // Use file_get_contents instead of DOMDocument::load (T58439)
        $xml = file_get_contents( $fileName );
        if ( !$xml ) {
            throw new RuntimeException( "Unable to read plurals file $fileName" );
        }
        $doc = new DOMDocument;
        $doc->loadXML( $xml );
        $rulesets = $doc->getElementsByTagName( "pluralRules" );
        foreach ( $rulesets as $ruleset ) {
            $codes = $ruleset->getAttribute( 'locales' );
            $rules = [];
            $ruleTypes = [];
            $ruleElements = $ruleset->getElementsByTagName( "pluralRule" );
            foreach ( $ruleElements as $elt ) {
                $ruleType = $elt->getAttribute( 'count' );
                if ( $ruleType === 'other' ) {
                    // Don't record "other" rules, which have an empty condition
                    continue;
                }
                $rules[] = $elt->nodeValue;
                $ruleTypes[] = $ruleType;
            }
            foreach ( explode( ' ', $codes ) as $code ) {
                self::$pluralRules[$code] = $rules;
                self::$pluralRuleTypes[$code] = $ruleTypes;
            }
        }
    }
 
    private function readSourceFilesAndRegisterDeps( $code, &$deps ) {
        // This reads in the PHP i18n file with non-messages l10n data
        $fileName = $this->langNameUtils->getMessagesFileName( $code );
        if ( !is_file( $fileName ) ) {
            $data = [];
        } else {
            $deps[] = new FileDependency( $fileName );
            $data = $this->readPHPFile( $fileName, 'core' );
        }
 
        return $data;
    }
 
    private function readPluralFilesAndRegisterDeps( $code, &$deps ) {
        if ( $this->isLeximorphEnabled() ) {
            $provider = ( new LeximorphProvider( $code, $this->logger ) )->getPluralProvider();
            $pluralRules = $provider->getPluralRules();
            $compiledPluralRules = $provider->getCompiledPluralRules();
            $pluralRuleTypes = $provider->getPluralRuleTypes();
            $depFiles = LeximorphPluralRulesProvider::PLURAL_FILES;
        } else {
            $pluralRules = $this->getPluralRules( $code );
            $compiledPluralRules = $this->getCompiledPluralRules( $code );
            $pluralRuleTypes = $this->getPluralRuleTypes( $code );
            $depFiles = self::PLURAL_FILES;
        }
 
        foreach ( $depFiles as $fileName ) {
            $deps[] = new FileDependency( $fileName );
        }
 
        return [
            // Load CLDR plural rules for JavaScript
            'pluralRules' => $pluralRules,
            // And for PHP
            'compiledPluralRules' => $compiledPluralRules,
            // Load plural rule types
            'pluralRuleTypes' => $pluralRuleTypes,
        ];
    }
 
    private function mergeItem( $key, &$value, $fallbackValue ) {
        if ( $value !== null ) {
            if ( $fallbackValue !== null ) {
                if ( in_array( $key, self::MERGEABLE_MAP_KEYS ) ) {
                    $value += $fallbackValue;
                } elseif ( in_array( $key, self::MERGEABLE_ALIAS_LIST_KEYS ) ) {
                    $value = array_merge_recursive( $value, $fallbackValue );
                } elseif ( in_array( $key, self::OPTIONAL_MERGE_KEYS ) ) {
                    if ( !empty( $value['inherit'] ) ) {
                        $value = array_merge( $fallbackValue, $value );
                    }
 
                    unset( $value['inherit'] );
                } elseif ( in_array( $key, self::MAGIC_WORD_KEYS ) ) {
                    $this->mergeMagicWords( $value, $fallbackValue );
                }
            }
        } else {
            $value = $fallbackValue;
        }
    }
 
    private function mergeMagicWords( array &$value, array $fallbackValue ): void {
        foreach ( $fallbackValue as $magicName => $fallbackInfo ) {
            if ( !isset( $value[$magicName] ) ) {
                $value[$magicName] = $fallbackInfo;
            } else {
                $value[$magicName] = [
                    $fallbackInfo[0],
                    ...array_unique( [
                        // First value is 1 if the magic word is case-sensitive, 0 if not
                        ...array_slice( $value[$magicName], 1 ),
                        ...array_slice( $fallbackInfo, 1 ),
                    ] )
                ];
            }
        }
    }
 
    public function getMessagesDirs() {
        global $IP;
 
        return [
            'core' => "$IP/languages/i18n",
            'botpasswords' => "$IP/languages/i18n/botpasswords",
            'codex' => "$IP/languages/i18n/codex",
            'datetime' => "$IP/languages/i18n/datetime",
            'exif' => "$IP/languages/i18n/exif",
            'languageconverter' => "$IP/languages/i18n/languageconverter",
            'interwiki' => "$IP/languages/i18n/interwiki",
            'preferences' => "$IP/languages/i18n/preferences",
            'userrights' => "$IP/languages/i18n/userrights",
 
            'nontranslatable' => "$IP/languages/i18n/nontranslatable",
 
            'api' => "$IP/includes/Api/i18n",
            'rest' => "$IP/includes/Rest/i18n",
            'oojs-ui' => "$IP/resources/lib/ooui/i18n",
            'paramvalidator' => "$IP/includes/libs/ParamValidator/i18n",
            'installer' => "$IP/includes/Installer/i18n",
        ] + $this->options->get( MainConfigNames::MessagesDirs );
    }
 
    private function loadCoreData( string $code ) {
        if ( !$code ) {
            throw new InvalidArgumentException( "Invalid language code requested" );
        }
        if ( $this->coreDataLoaded[$code] ?? false ) {
            return;
        }
 
        $coreData = array_fill_keys( self::CORE_ONLY_KEYS, null );
        $deps = [];
 
        # Load the primary localisation from the source file
        $data = $this->readSourceFilesAndRegisterDeps( $code, $deps );
        $this->logger->debug( __METHOD__ . ": got localisation for $code from source" );
 
        # Merge primary localisation
        foreach ( $data as $key => $value ) {
            $this->mergeItem( $key, $coreData[ $key ], $value );
        }
 
        # Fill in the fallback if it's not there already
        // @phan-suppress-next-line PhanRedundantValueComparison
        if ( ( $coreData['fallback'] === null || $coreData['fallback'] === false ) && $code === 'en' ) {
            $coreData['fallback'] = false;
            $coreData['originalFallbackSequence'] = $coreData['fallbackSequence'] = [];
        } else {
            if ( $coreData['fallback'] !== null ) {
                $coreData['fallbackSequence'] = array_map( 'trim', explode( ',', $coreData['fallback'] ) );
            } else {
                $coreData['fallbackSequence'] = [];
            }
            $len = count( $coreData['fallbackSequence'] );
 
            # Before we add the 'en' fallback for messages, keep a copy of
            # the original fallback sequence
            $coreData['originalFallbackSequence'] = $coreData['fallbackSequence'];
 
            # Ensure that the sequence ends at 'en' for messages
            if ( !$len || $coreData['fallbackSequence'][$len - 1] !== 'en' ) {
                $coreData['fallbackSequence'][] = 'en';
            }
        }
 
        foreach ( $coreData['fallbackSequence'] as $fbCode ) {
            // load core fallback data
            $fbData = $this->readSourceFilesAndRegisterDeps( $fbCode, $deps );
            foreach ( self::CORE_ONLY_KEYS as $key ) {
                // core-only keys are not mergeable, only set if not present in core data yet
                if ( isset( $fbData[$key] ) && !isset( $coreData[$key] ) ) {
                    $coreData[$key] = $fbData[$key];
                }
            }
        }
 
        $coreData['deps'] = $deps;
        foreach ( $coreData as $key => $item ) {
            $this->data[$code][$key] ??= null;
            // @phan-suppress-next-line PhanTypeArraySuspiciousNullable -- we just set a default null
            $this->mergeItem( $key, $this->data[$code][$key], $item );
            if (
                in_array( $key, self::CORE_ONLY_KEYS, true ) ||
                // "synthetic" keys based on "fallback" (see above)
                $key === 'fallbackSequence' ||
                $key === 'originalFallbackSequence'
            ) {
                // only mark core-only keys as loaded;
                // we may have loaded additional ones from the source file,
                // but they are not fully loaded yet, since recache()
                // may have to merge in additional values from fallback languages
                $this->loadedItems[$code][$key] = true;
            }
        }
 
        $this->coreDataLoaded[$code] = true;
    }
 
    public function recache( $code ) {
        if ( !$code ) {
            throw new InvalidArgumentException( "Invalid language code requested" );
        }
        $this->recachedLangs[ $code ] = true;
 
        # Initial values
        $initialData = array_fill_keys( self::ALL_KEYS, null );
        $this->data[$code] = [];
        $this->loadedItems[$code] = [];
        $this->loadedSubitems[$code] = [];
        $this->coreDataLoaded[$code] = false;
        $this->loadCoreData( $code );
        $coreData = $this->data[$code];
        $deps = $coreData['deps'];
        $coreData += $this->readPluralFilesAndRegisterDeps( $code, $deps );
 
        if ( $this->store->lateFallback() ) {
            // This LCStore can handle multiple queries efficiently and
            // requests to merge fallback languages at read time.
            $codeSequence = [ $code ];
        } else {
            // Our LCStore prefers to cache pre-combined data with all
            // the fallback paths filled out to reduce query count.
            $codeSequence = [ $code, ...$coreData['fallbackSequence'] ];
        }
        $messageDirs = $this->getMessagesDirs();
        $translationAliasesDirs = $this->options->get( MainConfigNames::TranslationAliasesDirs );
 
        # Load non-JSON localisation data for extensions
        $extensionData = array_fill_keys( $codeSequence, $initialData );
        foreach ( $this->options->get( MainConfigNames::ExtensionMessagesFiles ) as $extension => $fileName ) {
            if ( isset( $messageDirs[$extension] ) || isset( $translationAliasesDirs[$extension] ) ) {
                # This extension has JSON message data; skip the PHP shim
                continue;
            }
 
            $data = $this->readPHPFile( $fileName, 'extension' );
            $used = false;
 
            foreach ( $data as $key => $item ) {
                foreach ( $codeSequence as $csCode ) {
                    if ( isset( $item[$csCode] ) ) {
                        // Keep the behaviour the same as for json messages.
                        // TODO: Consider deprecating using a PHP file for messages.
                        if ( in_array( $key, self::SOURCE_PREFIX_KEYS ) ) {
                            foreach ( $item[$csCode] as $subkey => $_ ) {
                                $this->sourceLanguage[$code][$key][$subkey] ??= $csCode;
                            }
                        }
                        $this->mergeItem( $key, $extensionData[$csCode][$key], $item[$csCode] );
                        $used = true;
                    }
                }
            }
 
            if ( $used ) {
                $deps[] = new FileDependency( $fileName );
            }
        }
 
        # Load the localisation data for each fallback, then merge it into the full array
        $allData = $initialData;
        foreach ( $codeSequence as $csCode ) {
            $csData = $initialData;
 
            # Load core messages and the extension localisations.
            foreach ( $messageDirs as $dirs ) {
                foreach ( (array)$dirs as $dir ) {
                    $fileName = "$dir/$csCode.json";
                    $messages = $this->readJSONFile( $fileName );
 
                    foreach ( $messages as $subkey => $_ ) {
                        $this->sourceLanguage[$code]['messages'][$subkey] ??= $csCode;
                    }
                    $this->mergeItem( 'messages', $csData['messages'], $messages );
 
                    $deps[] = new FileDependency( $fileName );
                }
            }
 
            foreach ( $translationAliasesDirs as $dirs ) {
                foreach ( (array)$dirs as $dir ) {
                    $fileName = "$dir/$csCode.json";
                    $data = $this->readJSONFile( $fileName );
 
                    foreach ( $data as $key => $item ) {
                        // We allow the key in the JSON to be specified in PascalCase similar to key definitions in
                        // extension.json, but eventually they are stored in camelCase
                        $normalizedKey = lcfirst( $key );
 
                        if ( $normalizedKey === '@metadata' ) {
                            // Don't store @metadata information in extension data.
                            continue;
                        }
 
                        if ( !in_array( $normalizedKey, self::ALL_ALIAS_KEYS ) ) {
                            throw new UnexpectedValueException(
                                "Invalid key: \"$key\" for " . MainConfigNames::TranslationAliasesDirs . ". " .
                                'Valid keys: ' . implode( ', ', self::ALL_ALIAS_KEYS )
                            );
                        }
 
                        $this->mergeItem( $normalizedKey, $extensionData[$csCode][$normalizedKey], $item );
                    }
 
                    $deps[] = new FileDependency( $fileName );
                }
            }
 
            # Merge non-JSON extension data
            if ( isset( $extensionData[$csCode] ) ) {
                foreach ( $extensionData[$csCode] as $key => $item ) {
                    $this->mergeItem( $key, $csData[$key], $item );
                }
            }
 
            if ( $csCode === $code ) {
                # Merge core data into extension data
                foreach ( $coreData as $key => $item ) {
                    $this->mergeItem( $key, $csData[$key], $item );
                }
            } else {
                # Load the secondary localisation from the source file to
                # avoid infinite cycles on cyclic fallbacks
                $fbData = $this->readSourceFilesAndRegisterDeps( $csCode, $deps );
                $fbData += $this->readPluralFilesAndRegisterDeps( $csCode, $deps );
                # Only merge the keys that make sense to merge
                foreach ( self::ALL_KEYS as $key ) {
                    if ( !isset( $fbData[ $key ] ) ) {
                        continue;
                    }
 
                    if ( !isset( $coreData[ $key ] ) || self::isMergeableKey( $key ) ) {
                        $this->mergeItem( $key, $csData[ $key ], $fbData[ $key ] );
                    }
                }
            }
 
            # Allow extensions an opportunity to adjust the data for this fallback
            $this->hookRunner->onLocalisationCacheRecacheFallback( $this, $csCode, $csData );
 
            # Merge the data for this fallback into the final array
            if ( $csCode === $code ) {
                $allData = $csData;
            } else {
                foreach ( self::ALL_KEYS as $key ) {
                    if ( !isset( $csData[$key] ) ) {
                        continue;
                    }
 
                    // @phan-suppress-next-line PhanTypeArraySuspiciousNullable
                    if ( $allData[$key] === null || self::isMergeableKey( $key ) ) {
                        $this->mergeItem( $key, $allData[$key], $csData[$key] );
                    }
                }
            }
        }
 
        if ( !isset( $allData['rtl'] ) ) {
            throw new RuntimeException( __METHOD__ . ': Localisation data failed validation check! ' .
                'Check that your languages/messages/MessagesEn.php file is intact.' );
        }
 
        // Add cache dependencies for any referenced configs
        // We use the keys prefixed with 'wg' for historical reasons.
        $deps['wgExtensionMessagesFiles'] =
            new MainConfigDependency( MainConfigNames::ExtensionMessagesFiles );
        $deps['wgMessagesDirs'] =
            new MainConfigDependency( MainConfigNames::MessagesDirs );
        $deps['wgUseLeximorph'] =
            new MainConfigDependency( MainConfigNames::UseLeximorph );
        $deps['version'] = new ConstantDependency( self::class . '::VERSION' );
 
        # Add dependencies to the cache entry
        $allData['deps'] = $deps;
 
        # Replace spaces with underscores in namespace names
        if ( isset( $allData['namespaceNames'] ) ) {
            $allData['namespaceNames'] = str_replace( ' ', '_', $allData['namespaceNames'] );
        }
 
        # And do the same for special page aliases. $page is an array.
        if ( isset( $allData['specialPageAliases'] ) ) {
            foreach ( $allData['specialPageAliases'] as &$page ) {
                $page = str_replace( ' ', '_', $page );
            }
        }
        # Decouple the reference to prevent accidental damage
        unset( $page );
 
        # If there were no plural rules, return an empty array
        $allData['pluralRules'] ??= [];
        $allData['compiledPluralRules'] ??= [];
        # If there were no plural rule types, return an empty array
        $allData['pluralRuleTypes'] ??= [];
 
        # Set the list keys
        $allData['list'] = [];
        foreach ( self::SPLIT_KEYS as $key ) {
            $allData['list'][$key] = array_keys( $allData[$key] );
        }
        # Run hooks
        $unused = true; // Used to be $purgeBlobs, removed in 1.34
        $this->hookRunner->onLocalisationCacheRecache( $this, $code, $allData, $unused );
 
        if ( $this->store->lateFallback() ) {
            // Our in-process cache stores merged data, so let it be reloaded
            // from the new cache as backend declares it's cheap to do so.
        } else {
            // Save to the process cache and register the items loaded
            $this->data[$code] = $allData;
            $this->loadedItems[$code] = [];
            $this->loadedSubitems[$code] = [];
            foreach ( $allData as $key => $item ) {
                $this->loadedItems[$code][$key] = true;
            }
        }
 
        # Prefix each item with its source language code before save
        foreach ( self::SOURCE_PREFIX_KEYS as $key ) {
            // @phan-suppress-next-line PhanTypeArraySuspiciousNullable
            foreach ( $allData[$key] as $subKey => $value ) {
                // The source language should have been set, but to avoid Phan error and be double sure.
                $allData[$key][$subKey] = ( $this->sourceLanguage[$code][$key][$subKey] ?? $code ) .
                    self::SOURCEPREFIX_SEPARATOR . $value;
            }
        }
 
        # Set the preload key
        $allData['preload'] = $this->buildPreload( $allData );
 
        # Save to the persistent cache
        $this->store->startWrite( $code );
        foreach ( $allData as $key => $value ) {
            if ( in_array( $key, self::SPLIT_KEYS ) ) {
                foreach ( $value as $subkey => $subvalue ) {
                    $this->store->set( "$key:$subkey", $subvalue );
                }
            } else {
                $this->store->set( $key, $value );
            }
        }
        $this->store->finishWrite();
 
        # Clear out the MessageBlobStore
        # HACK: If using a null (i.e., disabled) storage backend, we
        # can't write to the MessageBlobStore either
        if ( !$this->store instanceof LCStoreNull ) {
            foreach ( $this->clearStoreCallbacks as $callback ) {
                $callback();
            }
        }
    }
 
    private function buildPreload( $data ) {
        $preload = [ 'messages' => [] ];
        foreach ( self::PRELOADED_KEYS as $key ) {
            if ( isset( $data[$key] ) ) {
                $preload[$key] = $data[$key];
            }
        }
 
        foreach ( $data['preloadedMessages'] ?? [] as $subkey ) {
            if ( isset( $data['messages'][$subkey] ) ) {
                $preload['messages'][$subkey] = $data['messages'][$subkey];
            }
        }
 
        return $preload;
    }
 
    public function unload( $code ) {
        unset( $this->data[$code] );
        unset( $this->loadedItems[$code] );
        unset( $this->loadedSubitems[$code] );
        unset( $this->initialisedLangs[$code] );
        unset( $this->shallowFallbacks[$code] );
        unset( $this->sourceLanguage[$code] );
        unset( $this->coreDataLoaded[$code] );
 
        foreach ( $this->shallowFallbacks as $shallowCode => $fbCode ) {
            if ( $fbCode === $code ) {
                $this->unload( $shallowCode );
            }
        }
    }
 
    public function unloadAll() {
        foreach ( $this->initialisedLangs as $lang => $unused ) {
            $this->unload( $lang );
        }
    }
 
    public function disableBackend() {
        $this->store = new LCStoreNull;
        $this->manualRecache = false;
    }
 
    private function isLeximorphEnabled(): bool {
        return (bool)$this->options->get( MainConfigNames::UseLeximorph );
    }
 
    public function setSubitemForTesting(
        string $code, string $key, string $subkey, mixed $value
    ): void {
        if ( !defined( 'MW_PHPUNIT_TEST' ) ) {
            throw new LogicException( __METHOD__ . '() should be called only from tests!' );
        }
        if ( !isset( $this->initialisedLangs[$code] ) ) {
            $this->initLanguage( $code );
        }
        $this->data[$code][$key][$subkey] = $value;
        $this->loadedSubitems[$code][$key][$subkey] = true;
    }
}
 
class_alias( LocalisationCache::class, 'LocalisationCache' );