21 use CLDRPluralRuleParser\Error as CLDRPluralRuleError;
22 use CLDRPluralRuleParser\Evaluator;
28 use Psr\Log\LoggerInterface;
57 private $manualRecache;
83 private $clearStoreCallbacks;
86 private $langNameUtils;
96 private $loadedItems = [];
102 private $loadedSubitems = [];
109 private $initialisedLangs = [];
116 private $shallowFallbacks = [];
121 private $recachedLangs = [];
127 'fallback',
'namespaceNames',
'bookstoreList',
128 'magicWords',
'messages',
'rtl',
129 'digitTransformTable',
'separatorTransformTable',
130 'minimumGroupingDigits',
'fallback8bitEncoding',
131 'linkPrefixExtension',
'linkTrail',
'linkPrefixCharset',
132 'namespaceAliases',
'dateFormats',
'datePreferences',
133 'datePreferenceMigrationMap',
'defaultDateFormat',
134 'specialPageAliases',
'imageFiles',
'preloadedMessages',
135 'namespaceGenderAliases',
'digitGroupingPattern',
'pluralRules',
136 'pluralRuleTypes',
'compiledPluralRules',
144 'namespaceAliases',
'dateFormats',
'imageFiles',
'preloadedMessages'
184 private $pluralRules =
null;
198 private $pluralRuleTypes =
null;
200 private $mergeableKeys =
null;
212 $storeArg[
'directory'] =
213 $conf[
'storeDirectory'] ?: $fallbackCacheDir;
215 if ( !empty( $conf[
'storeClass'] ) ) {
216 $storeClass = $conf[
'storeClass'];
217 } elseif ( $conf[
'store'] ===
'files' || $conf[
'store'] ===
'file' ||
218 ( $conf[
'store'] ===
'detect' && $storeArg[
'directory'] )
220 $storeClass = LCStoreCDB::class;
221 } elseif ( $conf[
'store'] ===
'db' || $conf[
'store'] ===
'detect' ) {
222 $storeClass = LCStoreDB::class;
223 $storeArg[
'server'] = $conf[
'storeServer'] ?? [];
224 } elseif ( $conf[
'store'] ===
'array' ) {
225 $storeClass = LCStoreStaticArray::class;
228 'Please set $wgLocalisationCacheConf[\'store\'] to something sensible.'
232 return new $storeClass( $storeArg );
238 public const CONSTRUCTOR_OPTIONS = [
242 MainConfigNames::ExtensionMessagesFiles,
243 MainConfigNames::MessagesDirs,
262 LoggerInterface $logger,
263 array $clearStoreCallbacks,
269 $this->options = $options;
270 $this->store = $store;
271 $this->logger = $logger;
272 $this->clearStoreCallbacks = $clearStoreCallbacks;
273 $this->langNameUtils = $langNameUtils;
274 $this->hookRunner =
new HookRunner( $hookContainer );
277 $this->manualRecache = $options->
get(
'manualRecache' );
287 if ( $this->mergeableKeys ===
null ) {
288 $this->mergeableKeys = array_fill_keys( array_merge(
289 self::$mergeableMapKeys,
290 self::$mergeableListKeys,
291 self::$mergeableAliasListKeys,
292 self::$optionalMergeKeys,
297 return isset( $this->mergeableKeys[$key] );
310 if ( !isset( $this->loadedItems[$code][$key] ) ) {
311 $this->loadItem( $code, $key );
314 if ( $key ===
'fallback' && isset( $this->shallowFallbacks[$code] ) ) {
315 return $this->shallowFallbacks[$code];
318 return $this->data[$code][$key];
329 if ( !isset( $this->loadedSubitems[$code][$key][$subkey] ) &&
330 !isset( $this->loadedItems[$code][$key] )
332 $this->loadSubitem( $code, $key, $subkey );
335 return $this->data[$code][$key][$subkey] ??
null;
351 if ( in_array( $key, self::$splitKeys ) ) {
352 return $this->getSubitem( $code,
'list', $key );
354 $item = $this->getItem( $code, $key );
355 if ( is_array( $item ) ) {
356 return array_keys( $item );
369 if ( !isset( $this->initialisedLangs[$code] ) ) {
370 $this->initLanguage( $code );
374 if ( isset( $this->loadedItems[$code][$key] ) ) {
378 if ( isset( $this->shallowFallbacks[$code] ) ) {
379 $this->loadItem( $this->shallowFallbacks[$code], $key );
384 if ( in_array( $key, self::$splitKeys ) ) {
385 $subkeyList = $this->getSubitem( $code,
'list', $key );
386 foreach ( $subkeyList as $subkey ) {
387 if ( isset( $this->data[$code][$key][$subkey] ) ) {
390 $this->data[$code][$key][$subkey] = $this->getSubitem( $code, $key, $subkey );
393 $this->data[$code][$key] = $this->store->get( $code, $key );
396 $this->loadedItems[$code][$key] =
true;
406 if ( !in_array( $key, self::$splitKeys ) ) {
407 $this->loadItem( $code, $key );
412 if ( !isset( $this->initialisedLangs[$code] ) ) {
413 $this->initLanguage( $code );
417 if ( isset( $this->loadedItems[$code][$key] ) ||
418 isset( $this->loadedSubitems[$code][$key][$subkey] )
423 if ( isset( $this->shallowFallbacks[$code] ) ) {
424 $this->loadSubitem( $this->shallowFallbacks[$code], $key, $subkey );
429 $value = $this->store->get( $code,
"$key:$subkey" );
430 $this->data[$code][$key][$subkey] = $value;
431 $this->loadedSubitems[$code][$key][$subkey] =
true;
442 if ( $this->options->get(
'forceRecache' ) && !isset( $this->recachedLangs[$code] ) ) {
443 $this->logger->debug( __METHOD__ .
"($code): forced reload" );
448 $deps = $this->store->get( $code,
'deps' );
449 $keys = $this->store->get( $code,
'list' );
450 $preload = $this->store->get( $code,
'preload' );
452 if ( $deps ===
null ||
$keys ===
null || $preload ===
null ) {
453 $this->logger->debug( __METHOD__ .
"($code): cache missing, need to make one" );
458 foreach ( $deps as $dep ) {
464 $this->logger->debug( __METHOD__ .
"($code): cache for $code expired due to " .
480 if ( isset( $this->initialisedLangs[$code] ) ) {
484 $this->initialisedLangs[$code] =
true;
486 # If the code is of the wrong form for a Messages*.php file, do a shallow fallback
487 if ( !$this->langNameUtils->isValidBuiltInCode( $code ) ) {
488 $this->initShallowFallback( $code,
'en' );
493 # Recache the data if necessary
494 if ( !$this->manualRecache && $this->isExpired( $code ) ) {
495 if ( $this->langNameUtils->isSupportedLanguage( $code ) ) {
496 $this->recache( $code );
497 } elseif ( $code ===
'en' ) {
498 throw new MWException(
'MessagesEn.php is missing.' );
500 $this->initShallowFallback( $code,
'en' );
507 $preload = $this->getItem( $code,
'preload' );
508 if ( $preload ===
null ) {
509 if ( $this->manualRecache ) {
511 if ( $code ===
'en' ) {
512 throw new MWException(
'No localisation cache found for English. ' .
513 'Please run maintenance/rebuildLocalisationCache.php.' );
515 $this->initShallowFallback( $code,
'en' );
519 throw new MWException(
'Invalid or missing localisation cache.' );
522 $this->data[$code] = $preload;
523 foreach ( $preload as $key => $item ) {
524 if ( in_array( $key, self::$splitKeys ) ) {
525 foreach ( $item as $subkey => $subitem ) {
526 $this->loadedSubitems[$code][$key][$subkey] =
true;
529 $this->loadedItems[$code][$key] =
true;
541 $this->data[$primaryCode] =& $this->data[$fallbackCode];
542 $this->loadedItems[$primaryCode] =& $this->loadedItems[$fallbackCode];
543 $this->loadedSubitems[$primaryCode] =& $this->loadedSubitems[$fallbackCode];
544 $this->shallowFallbacks[$primaryCode] = $fallbackCode;
558 if ( $_fileType ==
'core' || $_fileType ==
'extension' ) {
559 foreach ( self::$allKeys as $key ) {
562 if ( isset( $$key ) ) {
566 } elseif ( $_fileType ==
'aliases' ) {
568 if ( isset( $aliases ) ) {
569 $data[
'aliases'] = $aliases;
572 throw new MWException( __METHOD__ .
": Invalid file type: $_fileType" );
585 if ( !is_readable( $fileName ) ) {
589 $json = file_get_contents( $fileName );
590 if ( $json ===
false ) {
595 if ( $data ===
null ) {
596 throw new MWException( __METHOD__ .
": Invalid JSON file: $fileName" );
600 foreach ( $data as $key => $unused ) {
601 if ( $key ===
'' || $key[0] ===
'@' ) {
602 unset( $data[$key] );
607 return [
'messages' => $data ];
617 $rules = $this->getPluralRules( $code );
618 if ( $rules ===
null ) {
622 $compiledRules = Evaluator::compile( $rules );
623 }
catch ( CLDRPluralRuleError $e ) {
624 $this->logger->debug( $e->getMessage() );
629 return $compiledRules;
640 if ( $this->pluralRules ===
null ) {
641 $this->loadPluralFiles();
643 return $this->pluralRules[$code] ??
null;
654 if ( $this->pluralRuleTypes ===
null ) {
655 $this->loadPluralFiles();
657 return $this->pluralRuleTypes[$code] ??
null;
664 foreach ( $this->getPluralFiles() as $fileName ) {
665 $this->loadPluralFile( $fileName );
669 private function getPluralFiles(): array {
673 "$IP/languages/data/plurals.xml",
675 "$IP/languages/data/plurals-mediawiki.xml",
688 $xml = file_get_contents( $fileName );
690 throw new MWException(
"Unable to read plurals file $fileName" );
692 $doc =
new DOMDocument;
693 $doc->loadXML( $xml );
694 $rulesets = $doc->getElementsByTagName(
"pluralRules" );
695 foreach ( $rulesets as $ruleset ) {
696 $codes = $ruleset->getAttribute(
'locales' );
699 $ruleElements = $ruleset->getElementsByTagName(
"pluralRule" );
700 foreach ( $ruleElements as $elt ) {
701 $ruleType = $elt->getAttribute(
'count' );
702 if ( $ruleType ===
'other' ) {
706 $rules[] = $elt->nodeValue;
707 $ruleTypes[] = $ruleType;
709 foreach ( explode(
' ', $codes ) as $code ) {
710 $this->pluralRules[$code] = $rules;
711 $this->pluralRuleTypes[$code] = $ruleTypes;
727 $fileName = $this->langNameUtils->getMessagesFileName( $code );
728 if ( !is_file( $fileName ) ) {
732 $data = $this->readPHPFile( $fileName,
'core' );
736 $data[
'pluralRules'] = $this->getPluralRules( $code );
738 $data[
'compiledPluralRules'] = $this->getCompiledPluralRules( $code );
740 $data[
'pluralRuleTypes'] = $this->getPluralRuleTypes( $code );
742 foreach ( $this->getPluralFiles() as $fileName ) {
756 protected function mergeItem( $key, &$value, $fallbackValue ) {
757 if ( $value !==
null ) {
758 if ( $fallbackValue !==
null ) {
759 if ( in_array( $key, self::$mergeableMapKeys ) ) {
760 $value += $fallbackValue;
761 } elseif ( in_array( $key, self::$mergeableListKeys ) ) {
762 $value = array_unique( array_merge( $fallbackValue, $value ) );
763 } elseif ( in_array( $key, self::$mergeableAliasListKeys ) ) {
764 $value = array_merge_recursive( $value, $fallbackValue );
765 } elseif ( in_array( $key, self::$optionalMergeKeys ) ) {
766 if ( !empty( $value[
'inherit'] ) ) {
767 $value = array_merge( $fallbackValue, $value );
770 unset( $value[
'inherit'] );
771 } elseif ( in_array( $key, self::$magicWordKeys ) ) {
772 $this->mergeMagicWords( $value, $fallbackValue );
776 $value = $fallbackValue;
785 foreach ( $fallbackValue as $magicName => $fallbackInfo ) {
786 if ( !isset( $value[$magicName] ) ) {
787 $value[$magicName] = $fallbackInfo;
789 $oldSynonyms = array_slice( $fallbackInfo, 1 );
790 $newSynonyms = array_slice( $value[$magicName], 1 );
791 $synonyms = array_values( array_unique( array_merge(
792 $newSynonyms, $oldSynonyms ) ) );
793 $value[$magicName] = array_merge( [ $fallbackInfo[0] ], $synonyms );
809 'core' =>
"$IP/languages/i18n",
810 'exif' =>
"$IP/languages/i18n/exif",
811 'api' =>
"$IP/includes/api/i18n",
812 'rest' =>
"$IP/includes/Rest/i18n",
813 'oojs-ui' =>
"$IP/resources/lib/ooui/i18n",
814 'paramvalidator' =>
"$IP/includes/libs/ParamValidator/i18n",
815 ] + $this->options->get( MainConfigNames::MessagesDirs );
826 throw new MWException(
"Invalid language code requested" );
828 $this->recachedLangs[ $code ] =
true;
831 $initialData = array_fill_keys( self::$allKeys,
null );
832 $coreData = $initialData;
835 # Load the primary localisation from the source file
836 $data = $this->readSourceFilesAndRegisterDeps( $code, $deps );
837 $this->logger->debug( __METHOD__ .
": got localisation for $code from source" );
839 # Merge primary localisation
840 foreach ( $data as $key => $value ) {
841 $this->mergeItem( $key, $coreData[ $key ], $value );
844 # Fill in the fallback if it's not there already
846 if ( ( $coreData[
'fallback'] ===
null || $coreData[
'fallback'] ===
false ) && $code ===
'en' ) {
847 $coreData[
'fallback'] =
false;
848 $coreData[
'originalFallbackSequence'] = $coreData[
'fallbackSequence'] = [];
850 if ( $coreData[
'fallback'] !==
null ) {
851 $coreData[
'fallbackSequence'] = array_map(
'trim', explode(
',', $coreData[
'fallback'] ) );
853 $coreData[
'fallbackSequence'] = [];
855 $len = count( $coreData[
'fallbackSequence'] );
857 # Before we add the 'en' fallback for messages, keep a copy of
858 # the original fallback sequence
859 $coreData[
'originalFallbackSequence'] = $coreData[
'fallbackSequence'];
861 # Ensure that the sequence ends at 'en' for messages
862 if ( !$len || $coreData[
'fallbackSequence'][$len - 1] !==
'en' ) {
863 $coreData[
'fallbackSequence'][] =
'en';
867 $codeSequence = array_merge( [ $code ], $coreData[
'fallbackSequence'] );
868 $messageDirs = $this->getMessagesDirs();
870 # Load non-JSON localisation data for extensions
871 $extensionData = array_fill_keys( $codeSequence, $initialData );
872 foreach ( $this->options->get( MainConfigNames::ExtensionMessagesFiles ) as $extension => $fileName ) {
873 if ( isset( $messageDirs[$extension] ) ) {
874 # This extension has JSON message data; skip the PHP shim
878 $data = $this->readPHPFile( $fileName,
'extension' );
881 foreach ( $data as $key => $item ) {
882 foreach ( $codeSequence as $csCode ) {
883 if ( isset( $item[$csCode] ) ) {
884 $this->mergeItem( $key, $extensionData[$csCode][$key], $item[$csCode] );
895 # Load the localisation data for each fallback, then merge it into the full array
896 $allData = $initialData;
897 foreach ( $codeSequence as $csCode ) {
898 $csData = $initialData;
900 # Load core messages and the extension localisations.
901 foreach ( $messageDirs as
$dirs ) {
902 foreach ( (array)
$dirs as $dir ) {
903 $fileName =
"$dir/$csCode.json";
904 $data = $this->readJSONFile( $fileName );
906 foreach ( $data as $key => $item ) {
907 $this->mergeItem( $key, $csData[$key], $item );
914 # Merge non-JSON extension data
915 if ( isset( $extensionData[$csCode] ) ) {
916 foreach ( $extensionData[$csCode] as $key => $item ) {
917 $this->mergeItem( $key, $csData[$key], $item );
921 if ( $csCode === $code ) {
922 # Merge core data into extension data
923 foreach ( $coreData as $key => $item ) {
924 $this->mergeItem( $key, $csData[$key], $item );
927 # Load the secondary localisation from the source file to
928 # avoid infinite cycles on cyclic fallbacks
929 $fbData = $this->readSourceFilesAndRegisterDeps( $csCode, $deps );
930 # Only merge the keys that make sense to merge
931 foreach ( self::$allKeys as $key ) {
932 if ( !isset( $fbData[ $key ] ) ) {
936 if ( ( $coreData[ $key ] ) ===
null || $this->isMergeableKey( $key ) ) {
937 $this->mergeItem( $key, $csData[ $key ], $fbData[ $key ] );
942 # Allow extensions an opportunity to adjust the data for this
944 $this->hookRunner->onLocalisationCacheRecacheFallback( $this, $csCode, $csData );
946 # Merge the data for this fallback into the final array
947 if ( $csCode === $code ) {
950 foreach ( self::$allKeys as $key ) {
951 if ( !isset( $csData[$key] ) ) {
956 if ( $allData[$key] ===
null || $this->isMergeableKey( $key ) ) {
957 $this->mergeItem( $key, $allData[$key], $csData[$key] );
963 if ( !isset( $allData[
'rtl'] ) ) {
964 throw new MWException( __METHOD__ .
': Localisation data failed validation check! ' .
965 'Check that your languages/messages/MessagesEn.php file is intact.' );
968 # Add cache dependencies for any referenced globals
969 $deps[
'wgExtensionMessagesFiles'] =
new GlobalDependency(
'wgExtensionMessagesFiles' );
975 # Add dependencies to the cache entry
976 $allData[
'deps'] = $deps;
978 # Replace spaces with underscores in namespace names
979 $allData[
'namespaceNames'] = str_replace(
' ',
'_', $allData[
'namespaceNames'] );
981 # And do the same for special page aliases. $page is an array.
982 foreach ( $allData[
'specialPageAliases'] as &$page ) {
983 $page = str_replace(
' ',
'_', $page );
985 # Decouple the reference to prevent accidental damage
988 # If there were no plural rules, return an empty array
989 $allData[
'pluralRules'] ??= [];
990 $allData[
'compiledPluralRules'] ??= [];
991 # If there were no plural rule types, return an empty array
992 $allData[
'pluralRuleTypes'] ??= [];
995 $allData[
'list'] = [];
996 foreach ( self::$splitKeys as $key ) {
997 $allData[
'list'][$key] = array_keys( $allData[$key] );
1001 $this->hookRunner->onLocalisationCacheRecache( $this, $code, $allData, $unused );
1003 # Set the preload key
1004 $allData[
'preload'] = $this->buildPreload( $allData );
1006 # Save to the process cache and register the items loaded
1007 $this->data[$code] = $allData;
1008 foreach ( $allData as $key => $item ) {
1009 $this->loadedItems[$code][$key] =
true;
1012 # Save to the persistent cache
1013 $this->store->startWrite( $code );
1014 foreach ( $allData as $key => $value ) {
1015 if ( in_array( $key, self::$splitKeys ) ) {
1016 foreach ( $value as $subkey => $subvalue ) {
1017 $this->store->set(
"$key:$subkey", $subvalue );
1020 $this->store->set( $key, $value );
1023 $this->store->finishWrite();
1025 # Clear out the MessageBlobStore
1026 # HACK: If using a null (i.e. disabled) storage backend, we
1027 # can't write to the MessageBlobStore either
1029 foreach ( $this->clearStoreCallbacks as $callback ) {
1044 $preload = [
'messages' => [] ];
1045 foreach ( self::$preloadedKeys as $key ) {
1046 $preload[$key] = $data[$key];
1049 foreach ( $data[
'preloadedMessages'] as $subkey ) {
1050 $subitem = $data[
'messages'][$subkey] ??
null;
1051 $preload[
'messages'][$subkey] = $subitem;
1063 unset( $this->data[$code] );
1064 unset( $this->loadedItems[$code] );
1065 unset( $this->loadedSubitems[$code] );
1066 unset( $this->initialisedLangs[$code] );
1067 unset( $this->shallowFallbacks[$code] );
1069 foreach ( $this->shallowFallbacks as $shallowCode => $fbCode ) {
1070 if ( $fbCode === $code ) {
1071 $this->unload( $shallowCode );
1080 foreach ( $this->initialisedLangs as
$lang => $unused ) {
1081 $this->unload(
$lang );
1090 $this->manualRecache =
false;
if(!defined( 'MEDIAWIKI')) if(ini_get( 'mbstring.func_overload')) if(!defined( 'MW_ENTRY_POINT')) global $IP
Environment checks.
Base class to represent dependencies for LocalisationCache entries.
isExpired()
Returns true if the dependency is expired, false otherwise.
Depend on a PHP constant.
Depend on a PHP global variable.
Null store backend, used to avoid DB errors during install.
Caching for the contents of localisation files.
static $magicWordKeys
Keys for items that are formatted like $magicWords.
buildPreload( $data)
Build the preload item from the given pre-cache data.
initLanguage( $code)
Initialise a language in this object.
static $mergeableMapKeys
Keys for items which consist of associative arrays, which may be merged by a fallback sequence.
readSourceFilesAndRegisterDeps( $code, &$deps)
Read the data from the source files for a given language, and register the relevant dependencies in t...
isMergeableKey( $key)
Returns true if the given key is mergeable, that is, if it is an associative array which can be merge...
loadPluralFile( $fileName)
Load a plural XML file with the given filename, compile the relevant rules, and save the compiled rul...
getPluralRules( $code)
Get the plural rules for a given language from the XML files.
readPHPFile( $_fileName, $_fileType)
Read a PHP file containing localisation data.
readJSONFile( $fileName)
Read a JSON file containing localisation messages.
loadPluralFiles()
Load the plural XML files.
unload( $code)
Unload the data for a given language from the object cache.
unloadAll()
Unload all data.
disableBackend()
Disable the storage backend.
static $mergeableAliasListKeys
Keys for items which contain an array of arrays of equivalent aliases for each subitem.
getSubitemList( $code, $key)
Get the list of subitem keys for a given item.
__construct(ServiceOptions $options, LCStore $store, LoggerInterface $logger, array $clearStoreCallbacks, LanguageNameUtils $langNameUtils, HookContainer $hookContainer)
For constructor parameters, \MediaWiki\MainConfigSchema::LocalisationCacheConf.
static $mergeableListKeys
Keys for items which are a numbered array.
recache( $code)
Load localisation data for a given language for both core and extensions and save it to the persisten...
isExpired( $code)
Returns true if the cache identified by $code is missing or expired.
mergeItem( $key, &$value, $fallbackValue)
Merge two localisation values, a primary and a fallback, overwriting the primary value in place.
getSubitem( $code, $key, $subkey)
Get a subitem, for instance a single message for a given language.
mergeMagicWords(&$value, $fallbackValue)
initShallowFallback( $primaryCode, $fallbackCode)
Create a fallback from one language to another, without creating a complete persistent cache.
static getStoreFromConf(array $conf, $fallbackCacheDir)
Return a suitable LCStore as specified by the given configuration.
static $allKeys
All item keys.
getCompiledPluralRules( $code)
Get the compiled plural rules for a given language from the XML files.
static $optionalMergeKeys
Keys for items which contain an associative array, and may be merged if the primary value contains th...
getMessagesDirs()
Gets the combined list of messages dirs from core and extensions.
static $preloadedKeys
Keys which are loaded automatically by initLanguage()
getItem( $code, $key)
Get a cache item.
static $splitKeys
Keys for items where the subitems are stored in the backend separately.
loadItem( $code, $key)
Load an item into the cache.
getPluralRuleTypes( $code)
Get the plural rule types for a given language from the XML files.
loadSubitem( $code, $key, $subkey)
Load a subitem into the cache.
Depend on a MW configuration variable.
A class containing constants representing the names of configuration variables.
Interface for the persistence layer of LocalisationCache.
if(!isset( $args[0])) $lang