MediaWiki  master
MessageCache.php
Go to the documentation of this file.
1 <?php
36 use Psr\Log\LoggerAwareInterface;
37 use Psr\Log\LoggerInterface;
40 use Wikimedia\RequestTimeout\TimeoutException;
41 use Wikimedia\ScopedCallback;
42 
47 define( 'MSG_CACHE_VERSION', 2 );
48 
54 class MessageCache implements LoggerAwareInterface {
58  public const CONSTRUCTOR_OPTIONS = [
59  MainConfigNames::UseDatabaseMessages,
60  MainConfigNames::MaxMsgCacheEntrySize,
61  MainConfigNames::AdaptiveMessageCache,
62  ];
63 
68  public const MAX_REQUEST_LANGUAGES = 10;
69 
70  private const FOR_UPDATE = 1; // force message reload
71 
73  private const WAIT_SEC = 15;
75  private const LOCK_TTL = 30;
76 
81  private const WAN_TTL = IExpiringStore::TTL_DAY;
82 
84  private $logger;
85 
91  private $cache;
92 
98  private $systemMessageNames;
99 
103  private $cacheVolatile = [];
104 
109  private $disable;
110 
112  private $maxEntrySize;
113 
115  private $adaptive;
116 
121  private $parserOptions;
123  private $parser;
124 
128  private $inParser = false;
129 
131  private $wanCache;
133  private $clusterCache;
135  private $srvCache;
137  private $contLang;
139  private $contLangCode;
141  private $contLangConverter;
143  private $langFactory;
145  private $localisationCache;
147  private $languageNameUtils;
149  private $languageFallback;
151  private $hookRunner;
152 
159  public static function normalizeKey( $key ) {
160  $lckey = strtr( $key, ' ', '_' );
161  if ( $lckey === '' ) {
162  // T300792
163  return $lckey;
164  }
165 
166  if ( ord( $lckey ) < 128 ) {
167  $lckey[0] = strtolower( $lckey[0] );
168  } else {
169  $lckey = MediaWikiServices::getInstance()->getContentLanguage()->lcfirst( $lckey );
170  }
171 
172  return $lckey;
173  }
174 
190  public function __construct(
191  WANObjectCache $wanCache,
192  BagOStuff $clusterCache,
193  BagOStuff $serverCache,
194  Language $contLang,
195  LanguageConverterFactory $langConverterFactory,
196  LoggerInterface $logger,
197  ServiceOptions $options,
198  LanguageFactory $langFactory,
199  LocalisationCache $localisationCache,
200  LanguageNameUtils $languageNameUtils,
201  LanguageFallback $languageFallback,
202  HookContainer $hookContainer
203  ) {
204  $this->wanCache = $wanCache;
205  $this->clusterCache = $clusterCache;
206  $this->srvCache = $serverCache;
207  $this->contLang = $contLang;
208  $this->contLangConverter = $langConverterFactory->getLanguageConverter( $contLang );
209  $this->contLangCode = $contLang->getCode();
210  $this->logger = $logger;
211  $this->langFactory = $langFactory;
212  $this->localisationCache = $localisationCache;
213  $this->languageNameUtils = $languageNameUtils;
214  $this->languageFallback = $languageFallback;
215  $this->hookRunner = new HookRunner( $hookContainer );
216 
217  // limit size
218  $this->cache = new MapCacheLRU( self::MAX_REQUEST_LANGUAGES );
219 
220  $options->assertRequiredOptions( self::CONSTRUCTOR_OPTIONS );
221  $this->disable = !$options->get( MainConfigNames::UseDatabaseMessages );
222  $this->maxEntrySize = $options->get( MainConfigNames::MaxMsgCacheEntrySize );
223  $this->adaptive = $options->get( MainConfigNames::AdaptiveMessageCache );
224  }
225 
226  public function setLogger( LoggerInterface $logger ) {
227  $this->logger = $logger;
228  }
229 
235  private function getParserOptions() {
236  if ( !$this->parserOptions ) {
237  $context = RequestContext::getMain();
238  $user = $context->getUser();
239  if ( !$user->isSafeToLoad() ) {
240  // It isn't safe to use the context user yet, so don't try to get a
241  // ParserOptions for it. And don't cache this ParserOptions
242  // either.
244  $po->setAllowUnsafeRawHtml( false );
245  return $po;
246  }
247 
248  $this->parserOptions = ParserOptions::newFromContext( $context );
249  // Messages may take parameters that could come
250  // from malicious sources. As a precaution, disable
251  // the <html> parser tag when parsing messages.
252  $this->parserOptions->setAllowUnsafeRawHtml( false );
253  }
254 
255  return $this->parserOptions;
256  }
257 
264  private function getLocalCache( $code ) {
265  $cacheKey = $this->srvCache->makeKey( __CLASS__, $code );
266 
267  return $this->srvCache->get( $cacheKey );
268  }
269 
276  private function saveToLocalCache( $code, $cache ) {
277  $cacheKey = $this->srvCache->makeKey( __CLASS__, $code );
278  $this->srvCache->set( $cacheKey, $cache );
279  }
280 
301  private function load( string $code, $mode = null ) {
302  // Don't do double loading...
303  if ( $this->isLanguageLoaded( $code ) && $mode !== self::FOR_UPDATE ) {
304  return true;
305  }
306 
307  // Show a log message (once) if loading is disabled
308  if ( $this->disable ) {
309  static $shownDisabled = false;
310  if ( !$shownDisabled ) {
311  $this->logger->debug( __METHOD__ . ': disabled' );
312  $shownDisabled = true;
313  }
314 
315  return true;
316  }
317 
318  try {
319  return $this->loadUnguarded( $code, $mode );
320  } catch ( Throwable $e ) {
321  // Don't try to load again during the exception handler
322  $this->disable = true;
323  throw $e;
324  }
325  }
326 
334  private function loadUnguarded( $code, $mode ) {
335  $success = false; // Keep track of success
336  $staleCache = false; // a cache array with expired data, or false if none has been loaded
337  $where = []; // Debug info, delayed to avoid spamming debug log too much
338 
339  // Hash of the contents is stored in memcache, to detect if data-center cache
340  // or local cache goes out of date (e.g. due to replace() on some other server)
341  [ $hash, $hashVolatile ] = $this->getValidationHash( $code );
342  $this->cacheVolatile[$code] = $hashVolatile;
343 
344  // Try the local cache and check against the cluster hash key...
345  $cache = $this->getLocalCache( $code );
346  if ( !$cache ) {
347  $where[] = 'local cache is empty';
348  } elseif ( !isset( $cache['HASH'] ) || $cache['HASH'] !== $hash ) {
349  $where[] = 'local cache has the wrong hash';
350  $staleCache = $cache;
351  } elseif ( $this->isCacheExpired( $cache ) ) {
352  $where[] = 'local cache is expired';
353  $staleCache = $cache;
354  } elseif ( $hashVolatile ) {
355  $where[] = 'local cache validation key is expired/volatile';
356  $staleCache = $cache;
357  } else {
358  $where[] = 'got from local cache';
359  $this->cache->set( $code, $cache );
360  $success = true;
361  }
362 
363  if ( !$success ) {
364  $cacheKey = $this->clusterCache->makeKey( 'messages', $code );
365  // Try the global cache. If it is empty, try to acquire a lock. If
366  // the lock can't be acquired, wait for the other thread to finish
367  // and then try the global cache a second time.
368  for ( $failedAttempts = 0; $failedAttempts <= 1; $failedAttempts++ ) {
369  if ( $hashVolatile && $staleCache ) {
370  // Do not bother fetching the whole cache blob to avoid I/O.
371  // Instead, just try to get the non-blocking $statusKey lock
372  // below, and use the local stale value if it was not acquired.
373  $where[] = 'global cache is presumed expired';
374  } else {
375  $cache = $this->clusterCache->get( $cacheKey );
376  if ( !$cache ) {
377  $where[] = 'global cache is empty';
378  } elseif ( $this->isCacheExpired( $cache ) ) {
379  $where[] = 'global cache is expired';
380  $staleCache = $cache;
381  } elseif ( $hashVolatile ) {
382  // DB results are replica DB lag prone until the holdoff TTL passes.
383  // By then, updates should be reflected in loadFromDBWithLock().
384  // One thread regenerates the cache while others use old values.
385  $where[] = 'global cache is expired/volatile';
386  $staleCache = $cache;
387  } else {
388  $where[] = 'got from global cache';
389  $this->cache->set( $code, $cache );
390  $this->saveToCaches( $cache, 'local-only', $code );
391  $success = true;
392  }
393  }
394 
395  if ( $success ) {
396  // Done, no need to retry
397  break;
398  }
399 
400  // We need to call loadFromDB. Limit the concurrency to one process.
401  // This prevents the site from going down when the cache expires.
402  // Note that the DB slam protection lock here is non-blocking.
403  $loadStatus = $this->loadFromDBWithLock( $code, $where, $mode );
404  if ( $loadStatus === true ) {
405  $success = true;
406  break;
407  } elseif ( $staleCache ) {
408  // Use the stale cache while some other thread constructs the new one
409  $where[] = 'using stale cache';
410  $this->cache->set( $code, $staleCache );
411  $success = true;
412  break;
413  } elseif ( $failedAttempts > 0 ) {
414  // Already blocked once, so avoid another lock/unlock cycle.
415  // This case will typically be hit if memcached is down, or if
416  // loadFromDB() takes longer than LOCK_WAIT.
417  $where[] = "could not acquire status key.";
418  break;
419  } elseif ( $loadStatus === 'cantacquire' ) {
420  // Wait for the other thread to finish, then retry. Normally,
421  // the memcached get() will then yield the other thread's result.
422  $where[] = 'waited for other thread to complete';
423  $this->getReentrantScopedLock( $cacheKey );
424  } else {
425  // Disable cache; $loadStatus is 'disabled'
426  break;
427  }
428  }
429  }
430 
431  if ( !$success ) {
432  $where[] = 'loading FAILED - cache is disabled';
433  $this->disable = true;
434  $this->cache->set( $code, [] );
435  $this->logger->error( __METHOD__ . ": Failed to load $code" );
436  // This used to throw an exception, but that led to nasty side effects like
437  // the whole wiki being instantly down if the memcached server died
438  }
439 
440  if ( !$this->isLanguageLoaded( $code ) ) {
441  throw new LogicException( "Process cache for '$code' should be set by now." );
442  }
443 
444  $info = implode( ', ', $where );
445  $this->logger->debug( __METHOD__ . ": Loading $code... $info" );
446 
447  return $success;
448  }
449 
456  private function loadFromDBWithLock( $code, array &$where, $mode = null ) {
457  // If cache updates on all levels fail, give up on message overrides.
458  // This is to avoid easy site outages; see $saveSuccess comments below.
459  $statusKey = $this->clusterCache->makeKey( 'messages', $code, 'status' );
460  $status = $this->clusterCache->get( $statusKey );
461  if ( $status === 'error' ) {
462  $where[] = "could not load; method is still globally disabled";
463  return 'disabled';
464  }
465 
466  // Now let's regenerate
467  $where[] = 'loading from database';
468 
469  // Lock the cache to prevent conflicting writes.
470  // This lock is non-blocking so stale cache can quickly be used.
471  // Note that load() will call a blocking getReentrantScopedLock()
472  // after this if it really need to wait for any current thread.
473  $cacheKey = $this->clusterCache->makeKey( 'messages', $code );
474  $scopedLock = $this->getReentrantScopedLock( $cacheKey, 0 );
475  if ( !$scopedLock ) {
476  $where[] = 'could not acquire main lock';
477  return 'cantacquire';
478  }
479 
480  $cache = $this->loadFromDB( $code, $mode );
481  $this->cache->set( $code, $cache );
482  $saveSuccess = $this->saveToCaches( $cache, 'all', $code );
483 
484  if ( !$saveSuccess ) {
498  if ( $this->srvCache instanceof EmptyBagOStuff ) {
499  $this->clusterCache->set( $statusKey, 'error', 60 * 5 );
500  $where[] = 'could not save cache, disabled globally for 5 minutes';
501  } else {
502  $where[] = "could not save global cache";
503  }
504  }
505 
506  return true;
507  }
508 
518  private function loadFromDB( $code, $mode = null ) {
519  $dbr = wfGetDB( ( $mode === self::FOR_UPDATE ) ? DB_PRIMARY : DB_REPLICA );
520 
521  $cache = [];
522 
523  $mostused = []; // list of "<cased message key>/<code>"
524  if ( $this->adaptive && $code !== $this->contLangCode ) {
525  if ( !$this->cache->has( $this->contLangCode ) ) {
526  $this->load( $this->contLangCode );
527  }
528  $mostused = array_keys( $this->cache->get( $this->contLangCode ) );
529  foreach ( $mostused as $key => $value ) {
530  $mostused[$key] = "$value/$code";
531  }
532  }
533 
534  // Common conditions
535  $conds = [
536  'page_is_redirect' => 0,
537  'page_namespace' => NS_MEDIAWIKI,
538  ];
539  if ( count( $mostused ) ) {
540  $conds['page_title'] = $mostused;
541  } elseif ( $code !== $this->contLangCode ) {
542  $conds[] = 'page_title' . $dbr->buildLike( $dbr->anyString(), '/', $code );
543  } else {
544  // Effectively disallows use of '/' character in NS_MEDIAWIKI for uses
545  // other than language code.
546  $conds[] = 'page_title NOT' .
547  $dbr->buildLike( $dbr->anyString(), '/', $dbr->anyString() );
548  }
549 
550  // Set the stubs for oversized software-defined messages in the main cache map
551  $res = $dbr->select(
552  'page',
553  [ 'page_title', 'page_latest' ],
554  array_merge( $conds, [ 'page_len > ' . intval( $this->maxEntrySize ) ] ),
555  __METHOD__ . "($code)-big"
556  );
557  foreach ( $res as $row ) {
558  // Include entries/stubs for all keys in $mostused in adaptive mode
559  if ( $this->adaptive || $this->isMainCacheable( $row->page_title ) ) {
560  $cache[$row->page_title] = '!TOO BIG';
561  }
562  // At least include revision ID so page changes are reflected in the hash
563  $cache['EXCESSIVE'][$row->page_title] = $row->page_latest;
564  }
565 
566  // Can not inject the RevisionStore as it would break the installer since
567  // it instantiates MessageCache before the DB.
568  $revisionStore = MediaWikiServices::getInstance()->getRevisionStore();
569  // Set the text for small software-defined messages in the main cache map
570  $revQuery = $revisionStore->getQueryInfo( [ 'page' ] );
571 
572  // T231196: MySQL/MariaDB (10.1.37) can sometimes irrationally decide that querying `actor` then
573  // `revision` then `page` is somehow better than starting with `page`. Tell it not to reorder the
574  // query (and also reorder it ourselves because as generated by RevisionStore it'll have
575  // `revision` first rather than `page`).
576  $revQuery['joins']['revision'] = $revQuery['joins']['page'];
577  unset( $revQuery['joins']['page'] );
578  // It isn't actually necessary to reorder $revQuery['tables'] as Database does the right thing
579  // when join conditions are given for all joins, but GergÅ‘ is wary of relying on that so pull
580  // `page` to the start.
581  $revQuery['tables'] = array_merge(
582  [ 'page' ],
583  array_diff( $revQuery['tables'], [ 'page' ] )
584  );
585 
586  $res = $dbr->select(
587  $revQuery['tables'],
588  $revQuery['fields'],
589  array_merge( $conds, [
590  'page_len <= ' . intval( $this->maxEntrySize ),
591  'page_latest = rev_id' // get the latest revision only
592  ] ),
593  __METHOD__ . "($code)-small",
594  [ 'STRAIGHT_JOIN' ],
595  $revQuery['joins']
596  );
597 
598  // Don't load content from uncacheable rows (T313004)
599  [ $cacheableRows, $uncacheableRows ] = $this->separateCacheableRows( $res );
600  $result = $revisionStore->newRevisionsFromBatch( $cacheableRows, [
601  'slots' => [ SlotRecord::MAIN ],
602  'content' => true
603  ] );
604  $revisions = $result->isOK() ? $result->getValue() : [];
605 
606  foreach ( $cacheableRows as $row ) {
607  try {
608  $rev = $revisions[$row->rev_id] ?? null;
609  $content = $rev ? $rev->getContent( SlotRecord::MAIN ) : null;
610  $text = $this->getMessageTextFromContent( $content );
611  } catch ( TimeoutException $e ) {
612  throw $e;
613  } catch ( Exception $ex ) {
614  $text = false;
615  }
616 
617  if ( !is_string( $text ) ) {
618  $entry = '!ERROR';
619  $this->logger->error(
620  __METHOD__
621  . ": failed to load message page text for {$row->page_title} ($code)"
622  );
623  } else {
624  $entry = ' ' . $text;
625  }
626  $cache[$row->page_title] = $entry;
627  }
628 
629  foreach ( $uncacheableRows as $row ) {
630  // T193271: cache object gets too big and slow to generate.
631  // At least include revision ID so page changes are reflected in the hash.
632  $cache['EXCESSIVE'][$row->page_title] = $row->page_latest;
633  }
634 
635  $cache['VERSION'] = MSG_CACHE_VERSION;
636  ksort( $cache );
637 
638  // Hash for validating local cache (APC). No need to take into account
639  // messages larger than $wgMaxMsgCacheEntrySize, since those are only
640  // stored and fetched from memcache.
641  $cache['HASH'] = md5( serialize( $cache ) );
642  $cache['EXPIRY'] = wfTimestamp( TS_MW, time() + self::WAN_TTL );
643  unset( $cache['EXCESSIVE'] ); // only needed for hash
644 
645  return $cache;
646  }
647 
654  private function isLanguageLoaded( $lang ) {
655  // It is important that this only returns true if the cache was fully
656  // populated by load(), so that callers can assume all cache keys exist.
657  // It is possible for $this->cache to be only partially populated through
658  // methods like MessageCache::replace(), which must not make this method
659  // return true (T208897). And this method must cease to return true
660  // if the language was evicted by MapCacheLRU (T230690).
661  return $this->cache->hasField( $lang, 'VERSION' );
662  }
663 
675  private function isMainCacheable( $name, $code = null ) {
676  // Convert first letter to lowercase, and strip /code suffix
677  $name = $this->contLang->lcfirst( $name );
678  // Include common conversion table pages. This also avoids problems with
679  // Installer::parse() bailing out due to disallowed DB queries (T207979).
680  if ( strpos( $name, 'conversiontable/' ) === 0 ) {
681  return true;
682  }
683  $msg = preg_replace( '/\/[a-z0-9-]{2,}$/', '', $name );
684 
685  if ( $code === null ) {
686  // Bulk load
687  if ( $this->systemMessageNames === null ) {
688  $this->systemMessageNames = array_fill_keys(
689  $this->localisationCache->getSubitemList( $this->contLangCode, 'messages' ),
690  true );
691  }
692  return isset( $this->systemMessageNames[$msg] );
693  } else {
694  // Use individual subitem
695  return $this->localisationCache->getSubitem( $code, 'messages', $msg ) !== null;
696  }
697  }
698 
706  private function separateCacheableRows( $res ) {
707  if ( $this->adaptive ) {
708  // Include entries/stubs for all keys in $mostused in adaptive mode
709  return [ $res, [] ];
710  }
711  $cacheableRows = [];
712  $uncacheableRows = [];
713  foreach ( $res as $row ) {
714  if ( $this->isMainCacheable( $row->page_title ) ) {
715  $cacheableRows[] = $row;
716  } else {
717  $uncacheableRows[] = $row;
718  }
719  }
720  return [ $cacheableRows, $uncacheableRows ];
721  }
722 
729  public function replace( $title, $text ) {
730  if ( $this->disable ) {
731  return;
732  }
733 
734  [ $msg, $code ] = $this->figureMessage( $title );
735  if ( strpos( $title, '/' ) !== false && $code === $this->contLangCode ) {
736  // Content language overrides do not use the /<code> suffix
737  return;
738  }
739 
740  // (a) Update the process cache with the new message text
741  if ( $text === false ) {
742  // Page deleted
743  $this->cache->setField( $code, $title, '!NONEXISTENT' );
744  } else {
745  // Ignore $wgMaxMsgCacheEntrySize so the process cache is up to date
746  $this->cache->setField( $code, $title, ' ' . $text );
747  }
748 
749  // (b) Update the shared caches in a deferred update with a fresh DB snapshot
751  new MessageCacheUpdate( $code, $title, $msg ),
752  DeferredUpdates::PRESEND
753  );
754  }
755 
760  public function refreshAndReplaceInternal( string $code, array $replacements ) {
761  // Allow one caller at a time to avoid race conditions
762  $scopedLock = $this->getReentrantScopedLock(
763  $this->clusterCache->makeKey( 'messages', $code )
764  );
765  if ( !$scopedLock ) {
766  foreach ( $replacements as [ $title ] ) {
767  $this->logger->error(
768  __METHOD__ . ': could not acquire lock to update {title} ({code})',
769  [ 'title' => $title, 'code' => $code ] );
770  }
771 
772  return;
773  }
774 
775  // Load the existing cache to update it in the local DC cache.
776  // The other DCs will see a hash mismatch.
777  if ( $this->load( $code, self::FOR_UPDATE ) ) {
778  $cache = $this->cache->get( $code );
779  } else {
780  // Err? Fall back to loading from the database.
781  $cache = $this->loadFromDB( $code, self::FOR_UPDATE );
782  }
783  // Check if individual cache keys should exist and update cache accordingly
784  $newTextByTitle = []; // map of (title => content)
785  $newBigTitles = []; // map of (title => latest revision ID), like EXCESSIVE in loadFromDB()
786  // Can not inject the WikiPageFactory as it would break the installer since
787  // it instantiates MessageCache before the DB.
788  $wikiPageFactory = MediaWikiServices::getInstance()->getWikiPageFactory();
789  foreach ( $replacements as [ $title ] ) {
790  $page = $wikiPageFactory->newFromTitle( Title::makeTitle( NS_MEDIAWIKI, $title ) );
791  $page->loadPageData( $page::READ_LATEST );
792  $text = $this->getMessageTextFromContent( $page->getContent() );
793  // Remember the text for the blob store update later on
794  $newTextByTitle[$title] = $text ?? '';
795  // Note that if $text is false, then $cache should have a !NONEXISTANT entry
796  if ( !is_string( $text ) ) {
797  $cache[$title] = '!NONEXISTENT';
798  } elseif ( strlen( $text ) > $this->maxEntrySize ) {
799  $cache[$title] = '!TOO BIG';
800  $newBigTitles[$title] = $page->getLatest();
801  } else {
802  $cache[$title] = ' ' . $text;
803  }
804  }
805  // Update HASH for the new key. Incorporates various administrative keys,
806  // including the old HASH (and thereby the EXCESSIVE value from loadFromDB()
807  // and previous replace() calls), but that doesn't really matter since we
808  // only ever compare it for equality with a copy saved by saveToCaches().
809  $cache['HASH'] = md5( serialize( $cache + [ 'EXCESSIVE' => $newBigTitles ] ) );
810  // Update the too-big WAN cache entries now that we have the new HASH
811  foreach ( $newBigTitles as $title => $id ) {
812  // Match logic of loadCachedMessagePageEntry()
813  $this->wanCache->set(
814  $this->bigMessageCacheKey( $cache['HASH'], $title ),
815  ' ' . $newTextByTitle[$title],
816  self::WAN_TTL
817  );
818  }
819  // Mark this cache as definitely being "latest" (non-volatile) so
820  // load() calls do not try to refresh the cache with replica DB data
821  $cache['LATEST'] = time();
822  // Update the process cache
823  $this->cache->set( $code, $cache );
824  // Pre-emptively update the local datacenter cache so things like edit filter and
825  // prevented changes are reflected immediately; these often use MediaWiki: pages.
826  // The datacenter handling replace() calls should be the same one handling edits
827  // as they require HTTP POST.
828  $this->saveToCaches( $cache, 'all', $code );
829  // Release the lock now that the cache is saved
830  ScopedCallback::consume( $scopedLock );
831 
832  // Relay the purge. Touching this check key expires cache contents
833  // and local cache (APC) validation hash across all datacenters.
834  $this->wanCache->touchCheckKey( $this->getCheckKey( $code ) );
835 
836  // Purge the messages in the message blob store and fire any hook handlers
837  $blobStore = MediaWikiServices::getInstance()->getResourceLoader()->getMessageBlobStore();
838  foreach ( $replacements as [ $title, $msg ] ) {
839  $blobStore->updateMessage( $this->contLang->lcfirst( $msg ) );
840  $this->hookRunner->onMessageCacheReplace( $title, $newTextByTitle[$title] );
841  }
842  }
843 
850  private function isCacheExpired( $cache ) {
851  if ( !isset( $cache['VERSION'] ) || !isset( $cache['EXPIRY'] ) ) {
852  return true;
853  }
854  if ( $cache['VERSION'] !== MSG_CACHE_VERSION ) {
855  return true;
856  }
857  if ( wfTimestampNow() >= $cache['EXPIRY'] ) {
858  return true;
859  }
860 
861  return false;
862  }
863 
873  private function saveToCaches( array $cache, $dest, $code = false ) {
874  if ( $dest === 'all' ) {
875  $cacheKey = $this->clusterCache->makeKey( 'messages', $code );
876  $success = $this->clusterCache->set( $cacheKey, $cache );
877  $this->setValidationHash( $code, $cache );
878  } else {
879  $success = true;
880  }
881 
882  $this->saveToLocalCache( $code, $cache );
883 
884  return $success;
885  }
886 
893  private function getValidationHash( $code ) {
894  $curTTL = null;
895  $value = $this->wanCache->get(
896  $this->wanCache->makeKey( 'messages', $code, 'hash', 'v1' ),
897  $curTTL,
898  [ $this->getCheckKey( $code ) ]
899  );
900 
901  if ( $value ) {
902  $hash = $value['hash'];
903  if ( ( time() - $value['latest'] ) < WANObjectCache::TTL_MINUTE ) {
904  // Cache was recently updated via replace() and should be up-to-date.
905  // That method is only called in the primary datacenter and uses FOR_UPDATE.
906  // Also, it is unlikely that the current datacenter is *now* secondary one.
907  $expired = false;
908  } else {
909  // See if the "check" key was bumped after the hash was generated
910  $expired = ( $curTTL < 0 );
911  }
912  } else {
913  // No hash found at all; cache must regenerate to be safe
914  $hash = false;
915  $expired = true;
916  }
917 
918  return [ $hash, $expired ];
919  }
920 
931  private function setValidationHash( $code, array $cache ) {
932  $this->wanCache->set(
933  $this->wanCache->makeKey( 'messages', $code, 'hash', 'v1' ),
934  [
935  'hash' => $cache['HASH'],
936  'latest' => $cache['LATEST'] ?? 0
937  ],
938  WANObjectCache::TTL_INDEFINITE
939  );
940  }
941 
947  private function getReentrantScopedLock( $key, $timeout = self::WAIT_SEC ) {
948  return $this->clusterCache->getScopedLock( $key, $timeout, self::LOCK_TTL, __METHOD__ );
949  }
950 
983  public function get( $key, $useDB = true, $langcode = true ) {
984  if ( is_int( $key ) ) {
985  // Fix numerical strings that somehow become ints on their way here
986  $key = (string)$key;
987  } elseif ( !is_string( $key ) ) {
988  throw new TypeError( 'Message key must be a string' );
989  } elseif ( $key === '' ) {
990  // Shortcut: the empty key is always missing
991  return false;
992  }
993 
994  // Normalise title-case input (with some inlining)
995  $lckey = self::normalizeKey( $key );
996 
997  $this->hookRunner->onMessageCache__get( $lckey );
998 
999  // Loop through each language in the fallback list until we find something useful
1000  $message = $this->getMessageFromFallbackChain(
1001  wfGetLangObj( $langcode ),
1002  $lckey,
1003  !$this->disable && $useDB
1004  );
1005 
1006  // If we still have no message, maybe the key was in fact a full key so try that
1007  if ( $message === false ) {
1008  $parts = explode( '/', $lckey );
1009  // We may get calls for things that are http-urls from sidebar
1010  // Let's not load nonexistent languages for those
1011  // They usually have more than one slash.
1012  if ( count( $parts ) === 2 && $parts[1] !== '' ) {
1013  $message = $this->localisationCache->getSubitem( $parts[1], 'messages', $parts[0] );
1014  if ( $message === null ) {
1015  $message = false;
1016  }
1017  }
1018  }
1019 
1020  // Post-processing if the message exists
1021  if ( $message !== false ) {
1022  // Fix whitespace
1023  $message = str_replace(
1024  [
1025  // Fix for trailing whitespace, removed by textarea
1026  '&#32;',
1027  // Fix for NBSP, converted to space by firefox
1028  '&nbsp;',
1029  '&#160;',
1030  '&shy;'
1031  ],
1032  [
1033  ' ',
1034  "\u{00A0}",
1035  "\u{00A0}",
1036  "\u{00AD}"
1037  ],
1038  $message
1039  );
1040  }
1041 
1042  return $message;
1043  }
1044 
1057  private function getMessageFromFallbackChain( $lang, $lckey, $useDB ) {
1058  $alreadyTried = [];
1059 
1060  // First try the requested language.
1061  $message = $this->getMessageForLang( $lang, $lckey, $useDB, $alreadyTried );
1062  if ( $message !== false ) {
1063  return $message;
1064  }
1065 
1066  // Now try checking the site language.
1067  $message = $this->getMessageForLang( $this->contLang, $lckey, $useDB, $alreadyTried );
1068  return $message;
1069  }
1070 
1081  private function getMessageForLang( $lang, $lckey, $useDB, &$alreadyTried ) {
1082  $langcode = $lang->getCode();
1083 
1084  // Try checking the database for the requested language
1085  if ( $useDB ) {
1086  $uckey = $this->contLang->ucfirst( $lckey );
1087 
1088  if ( !isset( $alreadyTried[$langcode] ) ) {
1089  $message = $this->getMsgFromNamespace(
1090  $this->getMessagePageName( $langcode, $uckey ),
1091  $langcode
1092  );
1093  if ( $message !== false ) {
1094  return $message;
1095  }
1096  $alreadyTried[$langcode] = true;
1097  }
1098  } else {
1099  $uckey = null;
1100  }
1101 
1102  // Check the CDB cache
1103  $message = $lang->getMessage( $lckey );
1104  if ( $message !== null ) {
1105  return $message;
1106  }
1107 
1108  // Try checking the database for all of the fallback languages
1109  if ( $useDB ) {
1110  $fallbackChain = $this->languageFallback->getAll( $langcode );
1111 
1112  foreach ( $fallbackChain as $code ) {
1113  if ( isset( $alreadyTried[$code] ) ) {
1114  continue;
1115  }
1116 
1117  $message = $this->getMsgFromNamespace(
1118  // @phan-suppress-next-line PhanTypeMismatchArgumentNullable uckey is set when used
1119  $this->getMessagePageName( $code, $uckey ), $code );
1120 
1121  if ( $message !== false ) {
1122  return $message;
1123  }
1124  $alreadyTried[$code] = true;
1125  }
1126  }
1127 
1128  return false;
1129  }
1130 
1138  private function getMessagePageName( $langcode, $uckey ) {
1139  if ( $langcode === $this->contLangCode ) {
1140  // Messages created in the content language will not have the /lang extension
1141  return $uckey;
1142  } else {
1143  return "$uckey/$langcode";
1144  }
1145  }
1146 
1159  public function getMsgFromNamespace( $title, $code ) {
1160  // Load all MediaWiki page definitions into cache. Note that individual keys
1161  // already loaded into cache during this request remain in the cache, which
1162  // includes the value of hook-defined messages.
1163  $this->load( $code );
1164 
1165  $entry = $this->cache->getField( $code, $title );
1166 
1167  if ( $entry !== null ) {
1168  // Message page exists as an override of a software messages
1169  if ( substr( $entry, 0, 1 ) === ' ' ) {
1170  // The message exists and is not '!TOO BIG' or '!ERROR'
1171  return (string)substr( $entry, 1 );
1172  } elseif ( $entry === '!NONEXISTENT' ) {
1173  // The text might be '-' or missing due to some data loss
1174  return false;
1175  }
1176  // Load the message page, utilizing the individual message cache.
1177  // If the page does not exist, there will be no hook handler fallbacks.
1178  $entry = $this->loadCachedMessagePageEntry(
1179  $title,
1180  $code,
1181  $this->cache->getField( $code, 'HASH' )
1182  );
1183  } else {
1184  // Message page either does not exist or does not override a software message
1185  if ( !$this->isMainCacheable( $title, $code ) ) {
1186  // Message page does not override any software-defined message. A custom
1187  // message might be defined to have content or settings specific to the wiki.
1188  // Load the message page, utilizing the individual message cache as needed.
1189  $entry = $this->loadCachedMessagePageEntry(
1190  $title,
1191  $code,
1192  $this->cache->getField( $code, 'HASH' )
1193  );
1194  }
1195  if ( $entry === null || substr( $entry, 0, 1 ) !== ' ' ) {
1196  // Message does not have a MediaWiki page definition; try hook handlers
1197  $message = false;
1198  // @phan-suppress-next-line PhanTypeMismatchArgument Type mismatch on pass-by-ref args
1199  $this->hookRunner->onMessagesPreLoad( $title, $message, $code );
1200  if ( $message !== false ) {
1201  $this->cache->setField( $code, $title, ' ' . $message );
1202  } else {
1203  $this->cache->setField( $code, $title, '!NONEXISTENT' );
1204  }
1205 
1206  return $message;
1207  }
1208  }
1209 
1210  if ( $entry !== false && substr( $entry, 0, 1 ) === ' ' ) {
1211  if ( $this->cacheVolatile[$code] ) {
1212  // Make sure that individual keys respect the WAN cache holdoff period too
1213  $this->logger->debug(
1214  __METHOD__ . ': loading volatile key \'{titleKey}\'',
1215  [ 'titleKey' => $title, 'code' => $code ] );
1216  } else {
1217  $this->cache->setField( $code, $title, $entry );
1218  }
1219  // The message exists, so make sure a string is returned
1220  return (string)substr( $entry, 1 );
1221  }
1222 
1223  $this->cache->setField( $code, $title, '!NONEXISTENT' );
1224 
1225  return false;
1226  }
1227 
1234  private function loadCachedMessagePageEntry( $dbKey, $code, $hash ) {
1235  $fname = __METHOD__;
1236  return $this->srvCache->getWithSetCallback(
1237  $this->srvCache->makeKey( 'messages-big', $hash, $dbKey ),
1238  BagOStuff::TTL_HOUR,
1239  function () use ( $code, $dbKey, $hash, $fname ) {
1240  return $this->wanCache->getWithSetCallback(
1241  $this->bigMessageCacheKey( $hash, $dbKey ),
1242  self::WAN_TTL,
1243  function ( $oldValue, &$ttl, &$setOpts ) use ( $dbKey, $code, $fname ) {
1244  // Try loading the message from the database
1245  $setOpts += Database::getCacheSetOptions( wfGetDB( DB_REPLICA ) );
1246  // Use newKnownCurrent() to avoid querying revision/user tables
1247  $title = Title::makeTitle( NS_MEDIAWIKI, $dbKey );
1248  // Injecting RevisionStore breaks installer since it
1249  // instantiates MessageCache before DB.
1250  $revision = MediaWikiServices::getInstance()
1251  ->getRevisionLookup()
1252  ->getKnownCurrentRevision( $title );
1253  if ( !$revision ) {
1254  // The wiki doesn't have a local override page. Cache absence with normal TTL.
1255  // When overrides are created, self::replace() takes care of the cache.
1256  return '!NONEXISTENT';
1257  }
1258  $content = $revision->getContent( SlotRecord::MAIN );
1259  if ( $content ) {
1260  $message = $this->getMessageTextFromContent( $content );
1261  } else {
1262  $this->logger->warning(
1263  $fname . ': failed to load page text for \'{titleKey}\'',
1264  [ 'titleKey' => $dbKey, 'code' => $code ]
1265  );
1266  $message = null;
1267  }
1268 
1269  if ( !is_string( $message ) ) {
1270  // Revision failed to load Content, or Content is incompatible with wikitext.
1271  // Possibly a temporary loading failure.
1272  $ttl = 5;
1273 
1274  return '!NONEXISTENT';
1275  }
1276 
1277  return ' ' . $message;
1278  }
1279  );
1280  }
1281  );
1282  }
1283 
1291  public function transform( $message, $interface = false, $language = null, PageReference $page = null ) {
1292  // Avoid creating parser if nothing to transform
1293  if ( strpos( $message, '{{' ) === false ) {
1294  return $message;
1295  }
1296 
1297  if ( $this->inParser ) {
1298  return $message;
1299  }
1300 
1301  $parser = $this->getParser();
1302  if ( $parser ) {
1303  $popts = $this->getParserOptions();
1304  $popts->setInterfaceMessage( $interface );
1305  $popts->setTargetLanguage( $language );
1306 
1307  $userlang = $popts->setUserLang( $language );
1308  $this->inParser = true;
1309  $message = $parser->transformMsg( $message, $popts, $page );
1310  $this->inParser = false;
1311  $popts->setUserLang( $userlang );
1312  }
1313 
1314  return $message;
1315  }
1316 
1320  public function getParser() {
1321  if ( !$this->parser ) {
1322  $parser = MediaWikiServices::getInstance()->getParser();
1323  // Clone it and store it
1324  $this->parser = clone $parser;
1325  }
1326 
1327  return $this->parser;
1328  }
1329 
1338  public function parse( $text, PageReference $page = null, $linestart = true,
1339  $interface = false, $language = null
1340  ) {
1341  global $wgTitle;
1342 
1343  if ( $this->inParser ) {
1344  return htmlspecialchars( $text );
1345  }
1346 
1347  $parser = $this->getParser();
1348  $popts = $this->getParserOptions();
1349  $popts->setInterfaceMessage( $interface );
1350 
1351  if ( is_string( $language ) ) {
1352  $language = $this->langFactory->getLanguage( $language );
1353  }
1354  $popts->setTargetLanguage( $language );
1355 
1356  if ( !$page ) {
1357  $logger = LoggerFactory::getInstance( 'GlobalTitleFail' );
1358  $logger->info(
1359  __METHOD__ . ' called with no title set.',
1360  [ 'exception' => new Exception ]
1361  );
1362  $page = $wgTitle;
1363  }
1364  // Sometimes $wgTitle isn't set either...
1365  if ( !$page ) {
1366  // It's not uncommon having a null $wgTitle in scripts. See r80898
1367  // Create a ghost title in such case
1368  $page = PageReferenceValue::localReference(
1369  NS_SPECIAL,
1370  'Badtitle/title not set in ' . __METHOD__
1371  );
1372  }
1373 
1374  $this->inParser = true;
1375  $res = $parser->parse( $text, $page, $popts, $linestart );
1376  $this->inParser = false;
1377 
1378  return $res;
1379  }
1380 
1381  public function disable() {
1382  $this->disable = true;
1383  }
1384 
1385  public function enable() {
1386  $this->disable = false;
1387  }
1388 
1401  public function isDisabled() {
1402  return $this->disable;
1403  }
1404 
1410  public function clear() {
1411  $langs = $this->languageNameUtils->getLanguageNames();
1412  foreach ( array_keys( $langs ) as $code ) {
1413  $this->wanCache->touchCheckKey( $this->getCheckKey( $code ) );
1414  }
1415  $this->cache->clear();
1416  }
1417 
1422  public function figureMessage( $key ) {
1423  $pieces = explode( '/', $key );
1424  if ( count( $pieces ) < 2 ) {
1425  return [ $key, $this->contLangCode ];
1426  }
1427 
1428  $lang = array_pop( $pieces );
1429  if ( !$this->languageNameUtils->getLanguageName(
1430  $lang,
1431  LanguageNameUtils::AUTONYMS,
1432  LanguageNameUtils::DEFINED
1433  ) ) {
1434  return [ $key, $this->contLangCode ];
1435  }
1436 
1437  $message = implode( '/', $pieces );
1438 
1439  return [ $message, $lang ];
1440  }
1441 
1450  public function getAllMessageKeys( $code ) {
1451  $this->load( $code );
1452  if ( !$this->cache->has( $code ) ) {
1453  // Apparently load() failed
1454  return null;
1455  }
1456  // Remove administrative keys
1457  $cache = $this->cache->get( $code );
1458  unset( $cache['VERSION'] );
1459  unset( $cache['EXPIRY'] );
1460  unset( $cache['EXCESSIVE'] );
1461  // Remove any !NONEXISTENT keys
1462  $cache = array_diff( $cache, [ '!NONEXISTENT' ] );
1463 
1464  // Keys may appear with a capital first letter. lcfirst them.
1465  return array_map( [ $this->contLang, 'lcfirst' ], array_keys( $cache ) );
1466  }
1467 
1475  public function updateMessageOverride( LinkTarget $linkTarget, Content $content = null ) {
1476  $msgText = $this->getMessageTextFromContent( $content );
1477  if ( $msgText === null ) {
1478  $msgText = false; // treat as not existing
1479  }
1480 
1481  $this->replace( $linkTarget->getDBkey(), $msgText );
1482 
1483  if ( $this->contLangConverter->hasVariants() ) {
1484  $this->contLangConverter->updateConversionTable( $linkTarget );
1485  }
1486  }
1487 
1492  public function getCheckKey( $code ) {
1493  return $this->wanCache->makeKey( 'messages', $code );
1494  }
1495 
1500  private function getMessageTextFromContent( Content $content = null ) {
1501  // @TODO: could skip pseudo-messages like js/css here, based on content model
1502  if ( $content ) {
1503  // Message page exists...
1504  // XXX: Is this the right way to turn a Content object into a message?
1505  // NOTE: $content is typically either WikitextContent, JavaScriptContent or
1506  // CssContent. MessageContent is *not* used for storing messages, it's
1507  // only used for wrapping them when needed.
1508  $msgText = $content->getWikitextForTransclusion();
1509  if ( $msgText === false || $msgText === null ) {
1510  // This might be due to some kind of misconfiguration...
1511  $msgText = null;
1512  $this->logger->warning(
1513  __METHOD__ . ": message content doesn't provide wikitext "
1514  . "(content model: " . $content->getModel() . ")" );
1515  }
1516  } else {
1517  // Message page does not exist...
1518  $msgText = false;
1519  }
1520 
1521  return $msgText;
1522  }
1523 
1529  private function bigMessageCacheKey( $hash, $title ) {
1530  return $this->wanCache->makeKey( 'messages-big', $hash, $title );
1531  }
1532 }
const NS_MEDIAWIKI
Definition: Defines.php:72
const NS_SPECIAL
Definition: Defines.php:53
wfGetLangObj( $langcode=false)
Return a Language object from $langcode.
wfTimestampNow()
Convenience function; returns MediaWiki timestamp for the present time.
wfGetDB( $db, $groups=[], $wiki=false)
Get a Database object.
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
const MSG_CACHE_VERSION
MediaWiki message cache structure version.
$success
if(!defined( 'MW_NO_SESSION') &&! $wgCommandLineMode) $wgTitle
Definition: Setup.php:493
Class representing a cache/ephemeral data store.
Definition: BagOStuff.php:85
static addUpdate(DeferrableUpdate $update, $stage=self::POSTSEND)
Add an update to the pending update queue for execution at the appropriate time.
A BagOStuff object with no objects in it.
Base class for language-specific code.
Definition: Language.php:54
getCode()
Get the internal language code for this language object.
Definition: Language.php:4372
Caching for the contents of localisation files.
Handles a simple LRU key/value map with a maximum number of entries.
Definition: MapCacheLRU.php:36
A class for passing options to services.
assertRequiredOptions(array $expectedKeys)
Assert that the list of options provided in this instance exactly match $expectedKeys,...
This class provides an implementation of the core hook interfaces, forwarding hook calls to HookConta...
Definition: HookRunner.php:563
An interface for creating language converters.
getLanguageConverter( $language=null)
Provide a LanguageConverter for given language.
Internationalisation code See https://www.mediawiki.org/wiki/Special:MyLanguage/Localisation for more...
A service that provides utilities to do with language names and codes.
PSR-3 logger instance factory.
A class containing constants representing the names of configuration variables.
Service locator for MediaWiki core services.
Immutable value object representing a page reference.
Value object representing a content slot associated with a page revision.
Definition: SlotRecord.php:40
Class to implement stub globals, which are globals that delay loading the their associated module cod...
Definition: StubObject.php:55
Message cache purging and in-place update handler for specific message page changes.
Cache messages that are defined by MediaWiki-namespace pages or by hooks.
refreshAndReplaceInternal(string $code, array $replacements)
const MAX_REQUEST_LANGUAGES
The size of the MapCacheLRU which stores message data.
getCheckKey( $code)
__construct(WANObjectCache $wanCache, BagOStuff $clusterCache, BagOStuff $serverCache, Language $contLang, LanguageConverterFactory $langConverterFactory, LoggerInterface $logger, ServiceOptions $options, LanguageFactory $langFactory, LocalisationCache $localisationCache, LanguageNameUtils $languageNameUtils, LanguageFallback $languageFallback, HookContainer $hookContainer)
getMsgFromNamespace( $title, $code)
Get a message from the MediaWiki namespace, with caching.
parse( $text, PageReference $page=null, $linestart=true, $interface=false, $language=null)
transform( $message, $interface=false, $language=null, PageReference $page=null)
updateMessageOverride(LinkTarget $linkTarget, Content $content=null)
Purge message caches when a MediaWiki: page is created, updated, or deleted.
const CONSTRUCTOR_OPTIONS
Options to be included in the ServiceOptions.
isDisabled()
Whether DB/cache usage is disabled for determining messages.
setLogger(LoggerInterface $logger)
clear()
Clear all stored messages in global and local cache.
getAllMessageKeys( $code)
Get all message keys stored in the message cache for a given language.
static normalizeKey( $key)
Normalize message key input.
figureMessage( $key)
replace( $title, $text)
Updates cache as necessary when message page is changed.
static newFromContext(IContextSource $context)
Get a ParserOptions object from a IContextSource object.
static newFromAnon()
Get a ParserOptions object for an anonymous user.
parse( $text, PageReference $page, ParserOptions $options, $linestart=true, $clearState=true, $revid=null)
Convert wikitext to HTML Do not call this function recursively.
Definition: Parser.php:682
transformMsg( $text, ParserOptions $options, ?PageReference $page=null)
Wrapper for preprocess()
Definition: Parser.php:4892
static getMain()
Get the RequestContext object associated with the main request.
static makeTitle( $ns, $title, $fragment='', $interwiki='')
Create a new Title from a namespace index and a DB key.
Definition: Title.php:641
Multi-datacenter aware caching interface.
Base interface for representing page content.
Definition: Content.php:34
getDBkey()
Get the main part of the link target, in canonical database form.
Interface for objects (potentially) representing a page that can be viewable and linked to on a wiki.
Result wrapper for grabbing data queried from an IDatabase object.
$cache
Definition: mcc.php:33
const DB_REPLICA
Definition: defines.php:26
const DB_PRIMARY
Definition: defines.php:28
$content
Definition: router.php:76
return true
Definition: router.php:90
if(!isset( $args[0])) $lang
$revQuery