16use MediaWiki\Logger\LoggerFactory;
17use MediaWiki\MediaWikiServices;
28 private const CACHEKEY =
'Translate-MessageIndex-interim';
30 private const READ_LATEST =
true;
33 protected static $instance;
35 private static $keysCache;
37 protected $interimCache;
41 private $jobQueueGroup;
43 public function __construct() {
45 $mwInstance = MediaWikiServices::getInstance();
46 $this->statusCache = $mwInstance->getMainWANObjectCache();
47 $this->jobQueueGroup = $mwInstance->getJobQueueGroup();
55 if ( self::$instance === null ) {
56 self::$instance = Services::getInstance()->getMessageIndex();
59 return self::$instance;
69 self::$instance = $instance;
79 global $wgTranslateMessageNamespaces;
83 if ( !$title->inNamespaces( $wgTranslateMessageNamespaces ) ) {
87 $namespace = $title->getNamespace();
89 $normkey = Utilities::normaliseKey( $namespace, $key );
91 $cache = self::getCache();
92 $value = $cache->get( $normkey );
93 if ( $value ===
null ) {
94 $value = (array)self::singleton()->getWithCache( $normkey );
95 $cache->set( $normkey, $value );
102 private static function getCache() {
103 if ( self::$keysCache ===
null ) {
104 self::$keysCache =
new MapCacheLRU( 30 );
106 return self::$keysCache;
115 $groups = self::getGroupIds( $handle );
117 return count( $groups ) ? array_shift( $groups ) : null;
120 private function getWithCache( $key ) {
121 $interimCacheValue = $this->getInterimCache()->get( self::CACHEKEY );
122 if ( $interimCacheValue && isset( $interimCacheValue[
'newKeys'][$key] ) ) {
123 return $interimCacheValue[
'newKeys'][$key];
126 return $this->
get( $key );
135 protected function get( $key ) {
137 $mi = $this->retrieve();
138 return $mi[$key] ??
null;
141 abstract public function retrieve(
bool $readLatest =
false ): array;
148 return array_keys( $this->retrieve() );
151 abstract protected function store( array $array, array $diff );
153 protected function lock() {
157 protected function unlock() {
168 public function rebuild(
float $timestamp =
null ): array {
169 $logger = LoggerFactory::getInstance(
'Translate' );
171 static $recursion = 0;
173 if ( $recursion > 0 ) {
174 $msg = __METHOD__ .
': trying to recurse - building the index first time?';
183 '[MessageIndex] Started rebuild. Initiated by {callers}',
184 [
'callers' => wfGetAllCallers( 20 ) ]
187 $groups = MessageGroups::singleton()->getGroups();
189 $tsStart = microtime(
true );
190 if ( !$this->lock() ) {
194 $lockWaitDuration = microtime(
true ) - $tsStart;
196 '[MessageIndex] Got lock in {duration}',
197 [
'duration' => $lockWaitDuration ]
200 self::getCache()->clear();
203 $old = $this->retrieve( self::READ_LATEST );
207 foreach ( $groups as $g ) {
208 if ( !$g->exists() ) {
210 wfWarn( __METHOD__ .
": group '$id' is registered but does not exist" );
215 if ( $g->isMeta() ) {
220 $this->checkAndAdd( $new, $g );
223 foreach ( $postponed as $g ) {
224 $this->checkAndAdd( $new, $g,
true );
227 $diff = self::getArrayDiff( $old, $new );
228 $this->store( $new, $diff[
'keys'] );
231 $criticalSectionDuration = microtime(
true ) - $tsStart - $lockWaitDuration;
233 '[MessageIndex] Finished critical section in {duration}',
234 [
'duration' => $criticalSectionDuration ]
237 $cache = $this->getInterimCache();
238 $interimCacheValue = $cache->get( self::CACHEKEY );
239 $timestamp = $timestamp ?? microtime(
true );
240 if ( $interimCacheValue ) {
241 if ( $interimCacheValue[
'timestamp'] <= $timestamp ) {
242 $cache->delete( self::CACHEKEY );
247 $this->jobQueueGroup->push( $job );
252 $this->statusCache->touchCheckKey( $this->getStatusCacheKey() );
254 $this->clearMessageGroupStats( $diff );
266 return $this->statusCache->makeKey(
'Translate',
'MessageIndex',
'status' );
269 private function getInterimCache(): BagOStuff {
270 return ObjectCache::getInstance( CACHE_ANYTHING );
273 public function storeInterim(
MessageGroup $group, array $newKeys ): void {
274 $namespace = $group->getNamespace();
275 $id = $group->
getId();
277 $normalizedNewKeys = [];
278 foreach ( $newKeys as $key ) {
279 $normalizedNewKeys[Utilities::normaliseKey( $namespace, $key )] = $id;
282 $cache = $this->getInterimCache();
284 $interimCacheValue = $cache->get( self::CACHEKEY, $cache::READ_LATEST );
285 if ( $interimCacheValue ) {
286 $normalizedNewKeys = array_merge( $interimCacheValue[
'newKeys'], $normalizedNewKeys );
290 'timestamp' => microtime(
true ),
291 'newKeys' => $normalizedNewKeys,
294 $cache->set( self::CACHEKEY, $value, $cache::TTL_DAY );
327 $record =
static function ( $groups ) use ( &$values ) {
328 foreach ( $groups as $group ) {
329 $values[$group] =
true;
339 foreach ( $new as $key => $groups ) {
340 if ( !isset( $old[$key] ) ) {
341 $keys[
'add'][$key] = [ [], (array)$groups ];
342 $record( (array)$groups );
344 } elseif ( $groups != $old[$key] ) {
345 $keys[
'mod'][$key] = [ (array)$old[$key], (array)$groups ];
346 $record( array_diff( (array)$old[$key], (array)$groups ) );
347 $record( array_diff( (array)$groups, (array)$old[$key] ) );
351 foreach ( $old as $key => $groups ) {
352 if ( !isset( $new[$key] ) ) {
353 $keys[
'del'][$key] = [ (array)$groups, [] ];
354 $record( (array)$groups );
361 'values' => array_keys( $values ),
371 $job = MessageGroupStatsRebuildJob::newRefreshGroupsJob( $diff[
'values'] );
372 $this->jobQueueGroup->push( $job );
374 foreach ( $diff[
'keys'] as $keys ) {
375 foreach ( $keys as $key => $data ) {
376 [ $ns, $pagename ] = explode(
':', $key, 2 );
377 $title = Title::makeTitle( (
int)$ns, $pagename );
379 [ $oldGroups, $newGroups ] = $data;
380 Hooks::run(
'TranslateEventMessageMembershipChange',
381 [ $handle, $oldGroups, $newGroups ] );
396 foreach ( $keys as $key ) {
397 # Force all keys to lower case, because the case doesn't matter and it is
398 # easier to do comparing when the case of first letter is unknown, because
399 # mediawiki forces it to upper case
400 $key = Utilities::normaliseKey( $namespace, $key );
401 if ( isset( $hugearray[$key] ) ) {
403 $to = implode(
', ', (array)$hugearray[$key] );
404 wfWarn(
"Key $key already belongs to $to, conflict with $id" );
407 if ( is_array( $hugearray[$key] ) ) {
409 $hugearray[$key][] = & $id;
414 $value = & $hugearray[$key];
415 unset( $hugearray[$key] );
416 $hugearray[$key] = [ &$value, &$id ];
419 $hugearray[$key] = & $id;
433 if ( is_array( $data ) ) {
434 return implode(
'|', $data );
440 protected function unserialize( $data ) {
441 if ( strpos( $data,
'|' ) !==
false ) {
442 return explode(
'|', $data );
466 protected $filename =
'translate_messageindex.ser';
468 public function retrieve(
bool $readLatest =
false ): array {
469 if ( $this->index !==
null ) {
473 $file = Utilities::cacheFile( $this->filename );
474 if ( file_exists( $file ) ) {
475 $this->index = unserialize( file_get_contents( $file ) );
477 $this->index = $this->
rebuild();
483 protected function store( array $array, array $diff ) {
484 $file = Utilities::cacheFile( $this->filename );
485 file_put_contents( $file,
serialize( $array ) );
486 $this->index = $array;
505 protected function lock() {
506 $dbw = wfGetDB( DB_PRIMARY );
510 $ok = $dbw->lock(
'translate-messageindex', __METHOD__, 30 );
512 $dbw->commit( __METHOD__,
'flush' );
518 protected function unlock() {
520 $dbw = wfGetDB( DB_PRIMARY );
522 if ( !$dbw->trxLevel() ) {
523 $dbw->unlock(
'translate-messageindex', $fname );
524 } elseif ( is_callable( [ $dbw,
'onTransactionResolution' ] ) ) {
525 $dbw->onTransactionResolution(
static function () use ( $dbw, $fname ) {
526 $dbw->unlock(
'translate-messageindex', $fname );
529 $dbw->onTransactionCommitOrIdle(
static function () use ( $dbw, $fname ) {
530 $dbw->unlock(
'translate-messageindex', $fname );
537 public function retrieve(
bool $readLatest =
false ): array {
538 if ( $this->index !==
null && !$readLatest ) {
542 $dbr = wfGetDB( $readLatest ? DB_PRIMARY : DB_REPLICA );
543 $res = $dbr->select(
'translate_messageindex',
'*', [], __METHOD__ );
545 foreach ( $res as $row ) {
546 $this->index[$row->tmi_key] = $this->unserialize( $row->tmi_value );
552 protected function get( $key ) {
553 $dbr = wfGetDB( DB_REPLICA );
554 $value = $dbr->selectField(
555 'translate_messageindex',
557 [
'tmi_key' => $key ],
561 if ( is_string( $value ) ) {
562 $value = $this->unserialize( $value );
570 protected function store( array $array, array $diff ) {
573 foreach ( [ $diff[
'add'], $diff[
'mod'] ] as $changes ) {
574 foreach ( $changes as $key => $data ) {
583 $index = [
'tmi_key' ];
584 $deletions = array_keys( $diff[
'del'] );
586 $dbw = wfGetDB( DB_PRIMARY );
587 $dbw->startAtomic( __METHOD__ );
589 if ( $updates !== [] ) {
590 $dbw->replace(
'translate_messageindex', [ $index ], $updates, __METHOD__ );
593 if ( $deletions !== [] ) {
594 $dbw->delete(
'translate_messageindex', [
'tmi_key' => $deletions ], __METHOD__ );
597 $dbw->endAtomic( __METHOD__ );
599 $this->index = $array;
612 protected $key =
'translate-messageindex';
617 protected function __construct() {
618 parent::__construct();
619 $this->cache = ObjectCache::getInstance( CACHE_ANYTHING );
622 public function retrieve(
bool $readLatest =
false ): array {
623 if ( $this->index !==
null ) {
627 $key = $this->cache->makeKey( $this->key );
628 $data = $this->cache->get( $key );
629 if ( is_array( $data ) ) {
630 $this->index = $data;
632 $this->index = $this->
rebuild();
638 protected function store( array $array, array $diff ) {
639 $key = $this->cache->makeKey( $this->key );
640 $this->cache->set( $key, $array );
642 $this->index = $array;
665 protected $filename =
'translate_messageindex.cdb';
671 public function retrieve(
bool $readLatest =
false ): array {
672 $reader = $this->getReader();
674 if ( $this->index !==
null ) {
679 foreach ( $this->
getKeys() as $key ) {
680 $this->index[$key] = $this->unserialize( $reader->get( $key ) );
687 $reader = $this->getReader();
689 $key = $reader->firstkey();
690 while ( $key !==
false ) {
692 $key = $reader->nextkey();
698 protected function get( $key ) {
699 $reader = $this->getReader();
701 if ( $this->index !==
null ) {
702 return $this->index[$key] ??
null;
705 $value = $reader->get( $key );
706 if ( !is_string( $value ) ) {
709 $value = $this->unserialize( $value );
715 protected function store( array $array, array $diff ) {
716 $this->reader =
null;
718 $file = Utilities::cacheFile( $this->filename );
719 $cache = Writer::open( $file );
721 foreach ( $array as $key => $value ) {
722 $value = $this->serialize( $value );
723 $cache->set( $key, $value );
728 $this->index = $array;
731 protected function getReader() {
732 if ( $this->reader ) {
733 return $this->reader;
736 $file = Utilities::cacheFile( $this->filename );
737 if ( !file_exists( $file ) ) {
739 $this->store( [], [] );
740 $this->index = $this->rebuild();
743 $this->reader = Reader::open( $file );
744 return $this->reader;
757 protected $index = [];
759 public function retrieve(
bool $readLatest =
false ): array {
767 protected function get( $key ) {
768 return $this->index[$key] ??
null;
771 protected function store( array $array, array $diff ) {
772 $this->index = $array;
return[ 'Translate:ConfigHelper'=> static function():ConfigHelper { return new ConfigHelper();}, 'Translate:CsvTranslationImporter'=> static function(MediaWikiServices $services):CsvTranslationImporter { return new CsvTranslationImporter( $services->getWikiPageFactory());}, 'Translate:EntitySearch'=> static function(MediaWikiServices $services):EntitySearch { return new EntitySearch($services->getMainWANObjectCache(), $services->getCollationFactory() ->makeCollation( 'uca-default-u-kn'), MessageGroups::singleton(), $services->getNamespaceInfo(), $services->get( 'Translate:MessageIndex'), $services->getTitleParser(), $services->getTitleFormatter());}, 'Translate:ExternalMessageSourceStateImporter'=> static function(MediaWikiServices $services):ExternalMessageSourceStateImporter { return new ExternalMessageSourceStateImporter($services->getMainConfig(), $services->get( 'Translate:GroupSynchronizationCache'), $services->getJobQueueGroup(), LoggerFactory::getInstance( 'Translate.GroupSynchronization'), $services->get( 'Translate:MessageIndex'));}, 'Translate:GroupSynchronizationCache'=> static function(MediaWikiServices $services):GroupSynchronizationCache { return new GroupSynchronizationCache( $services->get( 'Translate:PersistentCache'));}, 'Translate:MessageBundleStore'=> static function(MediaWikiServices $services):MessageBundleStore { return new MessageBundleStore(new RevTagStore(), $services->getJobQueueGroup(), $services->getLanguageNameUtils(), $services->get( 'Translate:MessageIndex'));}, 'Translate:MessageGroupReview'=> static function(MediaWikiServices $services):MessageGroupReview { return new MessageGroupReview($services->getDBLoadBalancer(), $services->getHookContainer());}, 'Translate:MessageGroupStatsTableFactory'=> static function(MediaWikiServices $services):MessageGroupStatsTableFactory { return new MessageGroupStatsTableFactory($services->get( 'Translate:ProgressStatsTableFactory'), $services->getDBLoadBalancer(), $services->getLinkRenderer(), $services->getMainConfig() ->get( 'TranslateWorkflowStates') !==false);}, 'Translate:MessageIndex'=> static function(MediaWikiServices $services):MessageIndex { $params=$services->getMainConfig() ->get( 'TranslateMessageIndex');if(is_string( $params)) { $params=(array) $params;} $class=array_shift( $params);return new $class( $params);}, 'Translate:MessagePrefixStats'=> static function(MediaWikiServices $services):MessagePrefixStats { return new MessagePrefixStats( $services->getTitleParser());}, 'Translate:ParsingPlaceholderFactory'=> static function():ParsingPlaceholderFactory { return new ParsingPlaceholderFactory();}, 'Translate:PersistentCache'=> static function(MediaWikiServices $services):PersistentCache { return new PersistentDatabaseCache($services->getDBLoadBalancer(), $services->getJsonCodec());}, 'Translate:ProgressStatsTableFactory'=> static function(MediaWikiServices $services):ProgressStatsTableFactory { return new ProgressStatsTableFactory($services->getLinkRenderer(), $services->get( 'Translate:ConfigHelper'));}, 'Translate:SubpageListBuilder'=> static function(MediaWikiServices $services):SubpageListBuilder { return new SubpageListBuilder($services->get( 'Translate:TranslatableBundleFactory'), $services->getLinkBatchFactory());}, 'Translate:TranslatableBundleFactory'=> static function(MediaWikiServices $services):TranslatableBundleFactory { return new TranslatableBundleFactory($services->get( 'Translate:TranslatablePageStore'), $services->get( 'Translate:MessageBundleStore'));}, 'Translate:TranslatableBundleMover'=> static function(MediaWikiServices $services):TranslatableBundleMover { return new TranslatableBundleMover($services->getMovePageFactory(), $services->getJobQueueGroup(), $services->getLinkBatchFactory(), $services->get( 'Translate:TranslatableBundleFactory'), $services->get( 'Translate:SubpageListBuilder'), $services->getMainConfig() ->get( 'TranslatePageMoveLimit'));}, 'Translate:TranslatableBundleStatusStore'=> static function(MediaWikiServices $services):TranslatableBundleStatusStore { return new TranslatableBundleStatusStore($services->getDBLoadBalancer() ->getConnection(DB_PRIMARY), $services->getCollationFactory() ->makeCollation( 'uca-default-u-kn'), $services->getDBLoadBalancer() ->getMaintenanceConnectionRef(DB_PRIMARY));}, 'Translate:TranslatablePageParser'=> static function(MediaWikiServices $services):TranslatablePageParser { return new TranslatablePageParser($services->get( 'Translate:ParsingPlaceholderFactory'));}, 'Translate:TranslatablePageStore'=> static function(MediaWikiServices $services):TranslatablePageStore { return new TranslatablePageStore($services->get( 'Translate:MessageIndex'), $services->getJobQueueGroup(), new RevTagStore(), $services->getDBLoadBalancer(), $services->get( 'Translate:TranslatableBundleStatusStore'));}, 'Translate:TranslationStashReader'=> static function(MediaWikiServices $services):TranslationStashReader { $db=$services->getDBLoadBalancer() ->getConnectionRef(DB_REPLICA);return new TranslationStashStorage( $db);}, 'Translate:TranslationStatsDataProvider'=> static function(MediaWikiServices $services):TranslationStatsDataProvider { return new TranslationStatsDataProvider(new ServiceOptions(TranslationStatsDataProvider::CONSTRUCTOR_OPTIONS, $services->getMainConfig()), $services->getObjectFactory());}, 'Translate:TranslationUnitStoreFactory'=> static function(MediaWikiServices $services):TranslationUnitStoreFactory { return new TranslationUnitStoreFactory( $services->getDBLoadBalancer());}, 'Translate:TranslatorActivity'=> static function(MediaWikiServices $services):TranslatorActivity { $query=new TranslatorActivityQuery($services->getMainConfig(), $services->getDBLoadBalancer());return new TranslatorActivity($services->getMainObjectStash(), $query, $services->getJobQueueGroup());}, 'Translate:TtmServerFactory'=> static function(MediaWikiServices $services):TtmServerFactory { $config=$services->getMainConfig();$default=$config->get( 'TranslateTranslationDefaultService');if( $default===false) { $default=null;} return new TtmServerFactory( $config->get( 'TranslateTranslationServices'), $default);}]
@phpcs-require-sorted-array
retrieve(bool $readLatest=false)
Storage on the object cache.
Storage on the database itself.
clearMessageGroupStats(array $diff)
Purge stuff when set of keys have changed.
Class for pointing to messages, like Title class is for titles.
getTitle()
Get the original title.
getKey()
Returns the identified or guessed message key.
Creates a database of keys in all groups, so that namespace and key can be used to get the groups the...
clearMessageGroupStats(array $diff)
Purge stuff when set of keys have changed.
checkAndAdd(&$hugearray, MessageGroup $g, $ignore=false)
static getPrimaryGroupId(MessageHandle $handle)
serialize( $data)
These are probably slower than serialize and unserialize, but they are more space efficient because w...
rebuild(float $timestamp=null)
Creates the index from scratch.
static setInstance(self $instance)
Override the global instance, for testing.
static getArrayDiff(array $old, array $new)
Compares two associative arrays.
static getGroupIds(MessageHandle $handle)
Retrieves a list of groups given MessageHandle belongs to.
Storage on serialized file.
Interface for message groups.
getNamespace()
Returns the namespace where messages are placed.
getId()
Returns the unique identifier for this group.
getKeys()
Shortcut for array_keys( getDefinitions() ) that can be optimized by the implementing classes.