MediaWiki  master
LanguageConverter.php
Go to the documentation of this file.
1 <?php
24 
34 class LanguageConverter {
40  public static $languagesWithVariants = [
41  'en',
42  'crh',
43  'gan',
44  'iu',
45  'kk',
46  'ku',
47  'shi',
48  'sr',
49  'tg',
50  'uz',
51  'zh',
52  ];
53 
54  public $mMainLanguageCode;
55 
59  public $mVariants;
60  public $mVariantFallbacks;
61  public $mVariantNames;
62  public $mTablesLoaded = false;
63 
67  public $mTables;
68 
69  // 'bidirectional' 'unidirectional' 'disable' for each variant
70  public $mManualLevel;
71 
72  public $mLangObj;
73  public $mFlags;
74  public $mDescCodeSep = ':', $mDescVarSep = ';';
75  public $mUcfirst = false;
76  public $mConvRuleTitle = false;
77  public $mURLVariant;
78  public $mUserVariant;
79  public $mHeaderVariant;
80  public $mMaxDepth = 10;
81  public $mVarSeparatorPattern;
82 
83  const CACHE_VERSION_KEY = 'VERSION 7';
84 
93  public function __construct( Language $langobj, $maincode, $variants = [],
94  $variantfallbacks = [], $flags = [],
95  $manualLevel = [] ) {
96  global $wgDisabledVariants;
97  $this->mLangObj = $langobj;
98  $this->mMainLanguageCode = $maincode;
99  $this->mVariants = array_diff( $variants, $wgDisabledVariants );
100  $this->mVariantFallbacks = $variantfallbacks;
101  $this->mVariantNames = Language::fetchLanguageNames();
102  $defaultflags = [
103  // 'S' show converted text
104  // '+' add rules for alltext
105  // 'E' the gave flags is error
106  // these flags above are reserved for program
107  'A' => 'A', // add rule for convert code (all text convert)
108  'T' => 'T', // title convert
109  'R' => 'R', // raw content
110  'D' => 'D', // convert description (subclass implement)
111  '-' => '-', // remove convert (not implement)
112  'H' => 'H', // add rule for convert code (but no display in placed code)
113  'N' => 'N', // current variant name
114  ];
115  $this->mFlags = array_merge( $defaultflags, $flags );
116  foreach ( $this->mVariants as $v ) {
117  if ( array_key_exists( $v, $manualLevel ) ) {
118  $this->mManualLevel[$v] = $manualLevel[$v];
119  } else {
120  $this->mManualLevel[$v] = 'bidirectional';
121  }
122  $this->mFlags[$v] = $v;
123  }
124  }
125 
132  public function getVariants() {
133  return $this->mVariants;
134  }
135 
147  public function getVariantFallbacks( $variant ) {
148  return $this->mVariantFallbacks[$variant] ?? $this->mMainLanguageCode;
149  }
150 
155  public function getConvRuleTitle() {
156  return $this->mConvRuleTitle;
157  }
158 
163  public function getPreferredVariant() {
164  global $wgDefaultLanguageVariant, $wgUser;
165 
166  $req = $this->getURLVariant();
167 
168  Hooks::run( 'GetLangPreferredVariant', [ &$req ] );
169 
170  if ( $wgUser->isSafeToLoad() && $wgUser->isLoggedIn() && !$req ) {
171  $req = $this->getUserVariant();
172  } elseif ( !$req ) {
173  $req = $this->getHeaderVariant();
174  }
175 
176  if ( $wgDefaultLanguageVariant && !$req ) {
177  $req = $this->validateVariant( $wgDefaultLanguageVariant );
178  }
179 
180  $req = $this->validateVariant( $req );
181 
182  // This function, unlike the other get*Variant functions, is
183  // not memoized (i.e. there return value is not cached) since
184  // new information might appear during processing after this
185  // is first called.
186  if ( $req ) {
187  return $req;
188  }
189  return $this->mMainLanguageCode;
190  }
191 
197  public function getDefaultVariant() {
199 
200  $req = $this->getURLVariant();
201 
202  if ( !$req ) {
203  $req = $this->getHeaderVariant();
204  }
205 
206  if ( $wgDefaultLanguageVariant && !$req ) {
207  $req = $this->validateVariant( $wgDefaultLanguageVariant );
208  }
209 
210  if ( $req ) {
211  return $req;
212  }
213  return $this->mMainLanguageCode;
214  }
215 
225  public function validateVariant( $variant = null ) {
226  if ( $variant === null ) {
227  return null;
228  }
229  // Our internal variants are always lower-case; the variant we
230  // are validating may have mixed case.
231  $variant = LanguageCode::replaceDeprecatedCodes( strtolower( $variant ) );
232  if ( in_array( $variant, $this->mVariants ) ) {
233  return $variant;
234  }
235  // Browsers are supposed to use BCP 47 standard in the
236  // Accept-Language header, but not all of our internal
237  // mediawiki variant codes are BCP 47. Map BCP 47 code
238  // to our internal code.
239  foreach ( $this->mVariants as $v ) {
240  // Case-insensitive match (BCP 47 is mixed case)
241  if ( strtolower( LanguageCode::bcp47( $v ) ) === $variant ) {
242  return $v;
243  }
244  }
245  return null;
246  }
247 
253  public function getURLVariant() {
254  global $wgRequest;
255 
256  if ( $this->mURLVariant ) {
257  return $this->mURLVariant;
258  }
259 
260  // see if the preference is set in the request
261  $ret = $wgRequest->getText( 'variant' );
262 
263  if ( !$ret ) {
264  $ret = $wgRequest->getVal( 'uselang' );
265  }
266 
267  $this->mURLVariant = $this->validateVariant( $ret );
268  return $this->mURLVariant;
269  }
270 
276  protected function getUserVariant() {
277  global $wgUser;
278 
279  // memoizing this function wreaks havoc on parserTest.php
280  /*
281  if ( $this->mUserVariant ) {
282  return $this->mUserVariant;
283  }
284  */
285 
286  // Get language variant preference from logged in users
287  // Don't call this on stub objects because that causes infinite
288  // recursion during initialisation
289  if ( !$wgUser->isSafeToLoad() ) {
290  return false;
291  }
292  if ( $wgUser->isLoggedIn() ) {
293  if (
294  $this->mMainLanguageCode ==
295  MediaWikiServices::getInstance()->getContentLanguage()->getCode()
296  ) {
297  $ret = $wgUser->getOption( 'variant' );
298  } else {
299  $ret = $wgUser->getOption( 'variant-' . $this->mMainLanguageCode );
300  }
301  } else {
302  // figure out user lang without constructing wgLang to avoid
303  // infinite recursion
304  $ret = $wgUser->getOption( 'language' );
305  }
306 
307  $this->mUserVariant = $this->validateVariant( $ret );
308  return $this->mUserVariant;
309  }
310 
316  protected function getHeaderVariant() {
317  global $wgRequest;
318 
319  if ( $this->mHeaderVariant ) {
320  return $this->mHeaderVariant;
321  }
322 
323  // See if some supported language variant is set in the
324  // HTTP header.
325  $languages = array_keys( $wgRequest->getAcceptLang() );
326  if ( empty( $languages ) ) {
327  return null;
328  }
329 
330  $fallbackLanguages = [];
331  foreach ( $languages as $language ) {
332  $this->mHeaderVariant = $this->validateVariant( $language );
333  if ( $this->mHeaderVariant ) {
334  break;
335  }
336 
337  // To see if there are fallbacks of current language.
338  // We record these fallback variants, and process
339  // them later.
340  $fallbacks = $this->getVariantFallbacks( $language );
341  if ( is_string( $fallbacks ) && $fallbacks !== $this->mMainLanguageCode ) {
342  $fallbackLanguages[] = $fallbacks;
343  } elseif ( is_array( $fallbacks ) ) {
344  $fallbackLanguages =
345  array_merge( $fallbackLanguages, $fallbacks );
346  }
347  }
348 
349  if ( !$this->mHeaderVariant ) {
350  // process fallback languages now
351  $fallback_languages = array_unique( $fallbackLanguages );
352  foreach ( $fallback_languages as $language ) {
353  $this->mHeaderVariant = $this->validateVariant( $language );
354  if ( $this->mHeaderVariant ) {
355  break;
356  }
357  }
358  }
359 
360  return $this->mHeaderVariant;
361  }
362 
373  public function autoConvert( $text, $toVariant = false ) {
374  $this->loadTables();
375 
376  if ( !$toVariant ) {
377  $toVariant = $this->getPreferredVariant();
378  if ( !$toVariant ) {
379  return $text;
380  }
381  }
382 
383  if ( $this->guessVariant( $text, $toVariant ) ) {
384  return $text;
385  }
386  /* we convert everything except:
387  1. HTML markups (anything between < and >)
388  2. HTML entities
389  3. placeholders created by the parser
390  IMPORTANT: Beware of failure from pcre.backtrack_limit (T124404).
391  Minimize use of backtracking where possible.
392  */
393  static $reg;
394  if ( $reg === null ) {
395  $marker = '|' . Parser::MARKER_PREFIX . '[^\x7f]++\x7f';
396 
397  // this one is needed when the text is inside an HTML markup
398  $htmlfix = '|<[^>\004]++(?=\004$)|^[^<>]*+>';
399 
400  // Optimize for the common case where these tags have
401  // few or no children. Thus try and possesively get as much as
402  // possible, and only engage in backtracking when we hit a '<'.
403 
404  // disable convert to variants between <code> tags
405  $codefix = '<code>[^<]*+(?:(?:(?!<\/code>).)[^<]*+)*+<\/code>|';
406  // disable conversion of <script> tags
407  $scriptfix = '<script[^>]*+>[^<]*+(?:(?:(?!<\/script>).)[^<]*+)*+<\/script>|';
408  // disable conversion of <pre> tags
409  $prefix = '<pre[^>]*+>[^<]*+(?:(?:(?!<\/pre>).)[^<]*+)*+<\/pre>|';
410  // The "|.*+)" at the end, is in case we missed some part of html syntax,
411  // we will fail securely (hopefully) by matching the rest of the string.
412  $htmlFullTag = '<(?:[^>=]*+(?>[^>=]*+=\s*+(?:"[^"]*"|\'[^\']*\'|[^\'">\s]*+))*+[^>=]*+>|.*+)|';
413 
414  $reg = '/' . $codefix . $scriptfix . $prefix . $htmlFullTag .
415  '&[a-zA-Z#][a-z0-9]++;' . $marker . $htmlfix . '|\004$/s';
416  }
417  $startPos = 0;
418  $sourceBlob = '';
419  $literalBlob = '';
420 
421  // Guard against delimiter nulls in the input
422  // (should never happen: see T159174)
423  $text = str_replace( "\000", '', $text );
424  $text = str_replace( "\004", '', $text );
425 
426  $markupMatches = null;
427  $elementMatches = null;
428 
429  // We add a marker (\004) at the end of text, to ensure we always match the
430  // entire text (Otherwise, pcre.backtrack_limit might cause silent failure)
431  $textWithMarker = $text . "\004";
432  while ( $startPos < strlen( $text ) ) {
433  if ( preg_match( $reg, $textWithMarker, $markupMatches, PREG_OFFSET_CAPTURE, $startPos ) ) {
434  $elementPos = $markupMatches[0][1];
435  $element = $markupMatches[0][0];
436  if ( $element === "\004" ) {
437  // We hit the end.
438  $elementPos = strlen( $text );
439  $element = '';
440  } elseif ( substr( $element, -1 ) === "\004" ) {
441  // This can sometimes happen if we have
442  // unclosed html tags (For example
443  // when converting a title attribute
444  // during a recursive call that contains
445  // a &lt; e.g. <div title="&lt;">.
446  $element = substr( $element, 0, -1 );
447  }
448  } else {
449  // If we hit here, then Language Converter could be tricked
450  // into doing an XSS, so we refuse to translate.
451  // If non-crazy input manages to reach this code path,
452  // we should consider it a bug.
453  $log = LoggerFactory::getInstance( 'languageconverter' );
454  $log->error( "Hit pcre.backtrack_limit in " . __METHOD__
455  . ". Disabling language conversion for this page.",
456  [
457  "method" => __METHOD__,
458  "variant" => $toVariant,
459  "startOfText" => substr( $text, 0, 500 )
460  ]
461  );
462  return $text;
463  }
464  // Queue the part before the markup for translation in a batch
465  $sourceBlob .= substr( $text, $startPos, $elementPos - $startPos ) . "\000";
466 
467  // Advance to the next position
468  $startPos = $elementPos + strlen( $element );
469 
470  // Translate any alt or title attributes inside the matched element
471  if ( $element !== ''
472  && preg_match( '/^(<[^>\s]*+)\s([^>]*+)(.*+)$/', $element, $elementMatches )
473  ) {
474  // FIXME, this decodes entities, so if you have something
475  // like <div title="foo&lt;bar"> the bar won't get
476  // translated since after entity decoding it looks like
477  // unclosed html and we call this method recursively
478  // on attributes.
479  $attrs = Sanitizer::decodeTagAttributes( $elementMatches[2] );
480  // Ensure self-closing tags stay self-closing.
481  $close = substr( $elementMatches[2], -1 ) === '/' ? ' /' : '';
482  $changed = false;
483  foreach ( [ 'title', 'alt' ] as $attrName ) {
484  if ( !isset( $attrs[$attrName] ) ) {
485  continue;
486  }
487  $attr = $attrs[$attrName];
488  // Don't convert URLs
489  if ( !strpos( $attr, '://' ) ) {
490  $attr = $this->recursiveConvertTopLevel( $attr, $toVariant );
491  }
492 
493  if ( $attr !== $attrs[$attrName] ) {
494  $attrs[$attrName] = $attr;
495  $changed = true;
496  }
497  }
498  if ( $changed ) {
499  $element = $elementMatches[1] . Html::expandAttributes( $attrs ) .
500  $close . $elementMatches[3];
501  }
502  }
503  $literalBlob .= $element . "\000";
504  }
505 
506  // Do the main translation batch
507  $translatedBlob = $this->translate( $sourceBlob, $toVariant );
508 
509  // Put the output back together
510  $translatedIter = StringUtils::explode( "\000", $translatedBlob );
511  $literalIter = StringUtils::explode( "\000", $literalBlob );
512  $output = '';
513  while ( $translatedIter->valid() && $literalIter->valid() ) {
514  $output .= $translatedIter->current();
515  $output .= $literalIter->current();
516  $translatedIter->next();
517  $literalIter->next();
518  }
519 
520  return $output;
521  }
522 
532  public function translate( $text, $variant ) {
533  // If $text is empty or only includes spaces, do nothing
534  // Otherwise translate it
535  if ( trim( $text ) ) {
536  $this->loadTables();
537  $text = $this->mTables[$variant]->replace( $text );
538  }
539  return $text;
540  }
541 
548  public function autoConvertToAllVariants( $text ) {
549  $this->loadTables();
550 
551  $ret = [];
552  foreach ( $this->mVariants as $variant ) {
553  $ret[$variant] = $this->translate( $text, $variant );
554  }
555 
556  return $ret;
557  }
558 
564  protected function applyManualConv( $convRule ) {
565  // Use syntax -{T|zh-cn:TitleCN; zh-tw:TitleTw}- to custom
566  // title conversion.
567  // T26072: $mConvRuleTitle was overwritten by other manual
568  // rule(s) not for title, this breaks the title conversion.
569  $newConvRuleTitle = $convRule->getTitle();
570  if ( $newConvRuleTitle ) {
571  // So I add an empty check for getTitle()
572  $this->mConvRuleTitle = $newConvRuleTitle;
573  }
574 
575  // merge/remove manual conversion rules to/from global table
576  $convTable = $convRule->getConvTable();
577  $action = $convRule->getRulesAction();
578  foreach ( $convTable as $variant => $pair ) {
579  $v = $this->validateVariant( $variant );
580  if ( !$v ) {
581  continue;
582  }
583 
584  if ( $action == 'add' ) {
585  // More efficient than array_merge(), about 2.5 times.
586  foreach ( $pair as $from => $to ) {
587  $this->mTables[$v]->setPair( $from, $to );
588  }
589  } elseif ( $action == 'remove' ) {
590  $this->mTables[$v]->removeArray( $pair );
591  }
592  }
593  }
594 
602  public function convertTitle( $title ) {
603  $variant = $this->getPreferredVariant();
604  $index = $title->getNamespace();
605  if ( $index !== NS_MAIN ) {
606  $text = $this->convertNamespace( $index, $variant ) . ':';
607  } else {
608  $text = '';
609  }
610  $text .= $this->translate( $title->getText(), $variant );
611  return $text;
612  }
613 
621  public function convertNamespace( $index, $variant = null ) {
622  if ( $index === NS_MAIN ) {
623  return '';
624  }
625 
626  if ( $variant === null ) {
627  $variant = $this->getPreferredVariant();
628  }
629 
630  $cache = MediaWikiServices::getInstance()->getLocalServerObjectCache();
631  $key = $cache->makeKey( 'languageconverter', 'namespace-text', $index, $variant );
632  $nsVariantText = $cache->get( $key );
633  if ( $nsVariantText !== false ) {
634  return $nsVariantText;
635  }
636 
637  // First check if a message gives a converted name in the target variant.
638  $nsConvMsg = wfMessage( 'conversion-ns' . $index )->inLanguage( $variant );
639  if ( $nsConvMsg->exists() ) {
640  $nsVariantText = $nsConvMsg->plain();
641  }
642 
643  // Then check if a message gives a converted name in content language
644  // which needs extra translation to the target variant.
645  if ( $nsVariantText === false ) {
646  $nsConvMsg = wfMessage( 'conversion-ns' . $index )->inContentLanguage();
647  if ( $nsConvMsg->exists() ) {
648  $nsVariantText = $this->translate( $nsConvMsg->plain(), $variant );
649  }
650  }
651 
652  if ( $nsVariantText === false ) {
653  // No message exists, retrieve it from the target variant's namespace names.
654  $langObj = $this->mLangObj->factory( $variant );
655  $nsVariantText = $langObj->getFormattedNsText( $index );
656  }
657 
658  $cache->set( $key, $nsVariantText, 60 );
659 
660  return $nsVariantText;
661  }
662 
681  public function convert( $text ) {
682  $variant = $this->getPreferredVariant();
683  return $this->convertTo( $text, $variant );
684  }
685 
695  public function convertTo( $text, $variant ) {
697  if ( $wgDisableLangConversion ) {
698  return $text;
699  }
700  // Reset converter state for a new converter run.
701  $this->mConvRuleTitle = false;
702  return $this->recursiveConvertTopLevel( $text, $variant );
703  }
704 
714  protected function recursiveConvertTopLevel( $text, $variant, $depth = 0 ) {
715  $startPos = 0;
716  $out = '';
717  $length = strlen( $text );
718  $shouldConvert = !$this->guessVariant( $text, $variant );
719  $continue = 1;
720 
721  $noScript = '<script.*?>.*?<\/script>(*SKIP)(*FAIL)';
722  $noStyle = '<style.*?>.*?<\/style>(*SKIP)(*FAIL)';
723  // phpcs:ignore Generic.Files.LineLength
724  $noHtml = '<(?:[^>=]*+(?>[^>=]*+=\s*+(?:"[^"]*"|\'[^\']*\'|[^\'">\s]*+))*+[^>=]*+>|.*+)(*SKIP)(*FAIL)';
725  while ( $startPos < $length && $continue ) {
726  $continue = preg_match(
727  // Only match -{ outside of html.
728  "/$noScript|$noStyle|$noHtml|-\{/",
729  $text,
730  $m,
731  PREG_OFFSET_CAPTURE,
732  $startPos
733  );
734 
735  if ( !$continue ) {
736  // No more markup, append final segment
737  $fragment = substr( $text, $startPos );
738  $out .= $shouldConvert ? $this->autoConvert( $fragment, $variant ) : $fragment;
739  return $out;
740  }
741 
742  // Offset of the match of the regex pattern.
743  $pos = $m[0][1];
744 
745  // Append initial segment
746  $fragment = substr( $text, $startPos, $pos - $startPos );
747  $out .= $shouldConvert ? $this->autoConvert( $fragment, $variant ) : $fragment;
748  // -{ marker found, not in attribute
749  // Advance position up to -{ marker.
750  $startPos = $pos;
751  // Do recursive conversion
752  // Note: This passes $startPos by reference, and advances it.
753  $out .= $this->recursiveConvertRule( $text, $variant, $startPos, $depth + 1 );
754  }
755  return $out;
756  }
757 
769  protected function recursiveConvertRule( $text, $variant, &$startPos, $depth = 0 ) {
770  // Quick sanity check (no function calls)
771  if ( $text[$startPos] !== '-' || $text[$startPos + 1] !== '{' ) {
772  throw new MWException( __METHOD__ . ': invalid input string' );
773  }
774 
775  $startPos += 2;
776  $inner = '';
777  $warningDone = false;
778  $length = strlen( $text );
779 
780  while ( $startPos < $length ) {
781  $m = false;
782  preg_match( '/-\{|\}-/', $text, $m, PREG_OFFSET_CAPTURE, $startPos );
783  if ( !$m ) {
784  // Unclosed rule
785  break;
786  }
787 
788  $token = $m[0][0];
789  $pos = $m[0][1];
790 
791  // Markup found
792  // Append initial segment
793  $inner .= substr( $text, $startPos, $pos - $startPos );
794 
795  // Advance position
796  $startPos = $pos;
797 
798  switch ( $token ) {
799  case '-{':
800  // Check max depth
801  if ( $depth >= $this->mMaxDepth ) {
802  $inner .= '-{';
803  if ( !$warningDone ) {
804  $inner .= '<span class="error">' .
805  wfMessage( 'language-converter-depth-warning' )
806  ->numParams( $this->mMaxDepth )->inContentLanguage()->text() .
807  '</span>';
808  $warningDone = true;
809  }
810  $startPos += 2;
811  break;
812  }
813  // Recursively parse another rule
814  $inner .= $this->recursiveConvertRule( $text, $variant, $startPos, $depth + 1 );
815  break;
816  case '}-':
817  // Apply the rule
818  $startPos += 2;
819  $rule = new ConverterRule( $inner, $this );
820  $rule->parse( $variant );
821  $this->applyManualConv( $rule );
822  return $rule->getDisplay();
823  default:
824  throw new MWException( __METHOD__ . ': invalid regex match' );
825  }
826  }
827 
828  // Unclosed rule
829  if ( $startPos < $length ) {
830  $inner .= substr( $text, $startPos );
831  }
832  $startPos = $length;
833  return '-{' . $this->autoConvert( $inner, $variant );
834  }
835 
847  public function findVariantLink( &$link, &$nt, $ignoreOtherCond = false ) {
848  # If the article has already existed, there is no need to
849  # check it again, otherwise it may cause a fault.
850  if ( is_object( $nt ) && $nt->exists() ) {
851  return;
852  }
853 
855  $isredir = $wgRequest->getText( 'redirect', 'yes' );
856  $action = $wgRequest->getText( 'action' );
857  if ( $action == 'edit' && $wgRequest->getBool( 'redlink' ) ) {
858  $action = 'view';
859  }
860  $linkconvert = $wgRequest->getText( 'linkconvert', 'yes' );
861  $disableLinkConversion = $wgDisableLangConversion
863  $linkBatch = new LinkBatch();
864 
865  $ns = NS_MAIN;
866 
867  if ( $disableLinkConversion ||
868  ( !$ignoreOtherCond &&
869  ( $isredir == 'no'
870  || $action == 'edit'
871  || $action == 'submit'
872  || $linkconvert == 'no' ) ) ) {
873  return;
874  }
875 
876  if ( is_object( $nt ) ) {
877  $ns = $nt->getNamespace();
878  }
879 
880  $variants = $this->autoConvertToAllVariants( $link );
881  if ( !$variants ) { // give up
882  return;
883  }
884 
885  $titles = [];
886 
887  foreach ( $variants as $v ) {
888  if ( $v != $link ) {
889  $varnt = Title::newFromText( $v, $ns );
890  if ( $varnt !== null ) {
891  $linkBatch->addObj( $varnt );
892  $titles[] = $varnt;
893  }
894  }
895  }
896 
897  // fetch all variants in single query
898  $linkBatch->execute();
899 
900  foreach ( $titles as $varnt ) {
901  if ( $varnt->getArticleID() > 0 ) {
902  $nt = $varnt;
903  $link = $varnt->getText();
904  break;
905  }
906  }
907  }
908 
914  public function getExtraHashOptions() {
915  $variant = $this->getPreferredVariant();
916 
917  return '!' . $variant;
918  }
919 
930  public function guessVariant( $text, $variant ) {
931  return false;
932  }
933 
940  protected function loadDefaultTables() {
941  $class = static::class;
942  throw new MWException( "Must implement loadDefaultTables() method in class $class" );
943  }
944 
950  protected function loadTables( $fromCache = true ) {
952 
953  if ( $this->mTablesLoaded ) {
954  return;
955  }
956 
957  $this->mTablesLoaded = true;
958  $this->mTables = null;
960  $cacheKey = $cache->makeKey( 'conversiontables', $this->mMainLanguageCode );
961  if ( $fromCache ) {
962  $this->mTables = $cache->get( $cacheKey );
963  }
964  if ( !$this->mTables || !array_key_exists( self::CACHE_VERSION_KEY, $this->mTables ) ) {
965  // not in cache, or we need a fresh reload.
966  // We will first load the default tables
967  // then update them using things in MediaWiki:Conversiontable/*
968  $this->loadDefaultTables();
969  foreach ( $this->mVariants as $var ) {
970  $cached = $this->parseCachedTable( $var );
971  // @phan-suppress-next-next-line PhanTypeArraySuspiciousNullable
972  // FIXME: $this->mTables could theoretically be null here
973  $this->mTables[$var]->mergeArray( $cached );
974  }
975 
976  $this->postLoadTables();
977  $this->mTables[self::CACHE_VERSION_KEY] = true;
978 
979  $cache->set( $cacheKey, $this->mTables, 43200 );
980  }
981  }
982 
986  protected function postLoadTables() {
987  }
988 
996  private function reloadTables() {
997  if ( $this->mTables ) {
998  // @phan-suppress-next-line PhanTypeObjectUnsetDeclaredProperty
999  unset( $this->mTables );
1000  }
1001 
1002  $this->mTablesLoaded = false;
1003  $this->loadTables( false );
1004  }
1005 
1025  private function parseCachedTable( $code, $subpage = '', $recursive = true ) {
1026  static $parsed = [];
1027 
1028  $key = 'Conversiontable/' . $code;
1029  if ( $subpage ) {
1030  $key .= '/' . $subpage;
1031  }
1032  if ( array_key_exists( $key, $parsed ) ) {
1033  return [];
1034  }
1035 
1036  $parsed[$key] = true;
1037 
1038  if ( $subpage === '' ) {
1039  $txt = MessageCache::singleton()->getMsgFromNamespace( $key, $code );
1040  } else {
1041  $txt = false;
1043  if ( $title && $title->exists() ) {
1044  $revision = Revision::newFromTitle( $title );
1045  if ( $revision ) {
1046  if ( $revision->getContentModel() == CONTENT_MODEL_WIKITEXT ) {
1047  // @phan-suppress-next-line PhanUndeclaredMethod
1048  $txt = $revision->getContent( RevisionRecord::RAW )->getText();
1049  }
1050 
1051  // @todo in the future, use a specialized content model, perhaps based on json!
1052  }
1053  }
1054  }
1055 
1056  # Nothing to parse if there's no text
1057  if ( $txt === false || $txt === null || $txt === '' ) {
1058  return [];
1059  }
1060 
1061  // get all subpage links of the form
1062  // [[MediaWiki:Conversiontable/zh-xx/...|...]]
1063  $linkhead = $this->mLangObj->getNsText( NS_MEDIAWIKI ) .
1064  ':Conversiontable';
1065  $subs = StringUtils::explode( '[[', $txt );
1066  $sublinks = [];
1067  foreach ( $subs as $sub ) {
1068  $link = explode( ']]', $sub, 2 );
1069  if ( count( $link ) != 2 ) {
1070  continue;
1071  }
1072  $b = explode( '|', $link[0], 2 );
1073  $b = explode( '/', trim( $b[0] ), 3 );
1074  if ( count( $b ) == 3 ) {
1075  $sublink = $b[2];
1076  } else {
1077  $sublink = '';
1078  }
1079 
1080  if ( $b[0] == $linkhead && $b[1] == $code ) {
1081  $sublinks[] = $sublink;
1082  }
1083  }
1084 
1085  // parse the mappings in this page
1086  $blocks = StringUtils::explode( '-{', $txt );
1087  $ret = [];
1088  $first = true;
1089  foreach ( $blocks as $block ) {
1090  if ( $first ) {
1091  // Skip the part before the first -{
1092  $first = false;
1093  continue;
1094  }
1095  $mappings = explode( '}-', $block, 2 )[0];
1096  $stripped = str_replace( [ "'", '"', '*', '#' ], '', $mappings );
1097  $table = StringUtils::explode( ';', $stripped );
1098  foreach ( $table as $t ) {
1099  $m = explode( '=>', $t, 3 );
1100  if ( count( $m ) != 2 ) {
1101  continue;
1102  }
1103  // trim any trailling comments starting with '//'
1104  $tt = explode( '//', $m[1], 2 );
1105  $ret[trim( $m[0] )] = trim( $tt[0] );
1106  }
1107  }
1108 
1109  // recursively parse the subpages
1110  if ( $recursive ) {
1111  foreach ( $sublinks as $link ) {
1112  $s = $this->parseCachedTable( $code, $link, $recursive );
1113  $ret = $s + $ret;
1114  }
1115  }
1116 
1117  if ( $this->mUcfirst ) {
1118  foreach ( $ret as $k => $v ) {
1119  $ret[$this->mLangObj->ucfirst( $k )] = $this->mLangObj->ucfirst( $v );
1120  }
1121  }
1122  return $ret;
1123  }
1124 
1133  public function markNoConversion( $text, $noParse = false ) {
1134  # don't mark if already marked
1135  if ( strpos( $text, '-{' ) || strpos( $text, '}-' ) ) {
1136  return $text;
1137  }
1138 
1139  $ret = "-{R|$text}-";
1140  return $ret;
1141  }
1142 
1151  public function convertCategoryKey( $key ) {
1152  return $key;
1153  }
1154 
1161  public function updateConversionTable( Title $titleobj ) {
1162  if ( $titleobj->getNamespace() == NS_MEDIAWIKI ) {
1163  $title = $titleobj->getDBkey();
1164  $t = explode( '/', $title, 3 );
1165  $c = count( $t );
1166  if ( $c > 1 && $t[0] == 'Conversiontable' ) {
1167  if ( $this->validateVariant( $t[1] ) ) {
1168  $this->reloadTables();
1169  }
1170  }
1171  }
1172  }
1173 
1178  public function getVarSeparatorPattern() {
1179  if ( $this->mVarSeparatorPattern === null ) {
1180  // varsep_pattern for preg_split:
1181  // text should be splited by ";" only if a valid variant
1182  // name exist after the markup, for example:
1183  // -{zh-hans:<span style="font-size:120%;">xxx</span>;zh-hant:\
1184  // <span style="font-size:120%;">yyy</span>;}-
1185  // we should split it as:
1186  // [
1187  // [0] => 'zh-hans:<span style="font-size:120%;">xxx</span>'
1188  // [1] => 'zh-hant:<span style="font-size:120%;">yyy</span>'
1189  // [2] => ''
1190  // ]
1191  $expandedVariants = [];
1192  foreach ( $this->mVariants as $variant ) {
1193  $expandedVariants[ $variant ] = 1;
1194  // Accept standard BCP 47 names for variants as well.
1195  $expandedVariants[ LanguageCode::bcp47( $variant ) ] = 1;
1196  }
1197  // Accept old deprecated names for variants
1198  foreach ( LanguageCode::getDeprecatedCodeMapping() as $old => $new ) {
1199  if ( isset( $expandedVariants[ $new ] ) ) {
1200  $expandedVariants[ $old ] = 1;
1201  }
1202  }
1203 
1204  $pat = '/;\s*(?=';
1205  foreach ( $expandedVariants as $variant => $ignore ) {
1206  // zh-hans:xxx;zh-hant:yyy
1207  $pat .= $variant . '\s*:|';
1208  // xxx=>zh-hans:yyy; xxx=>zh-hant:zzz
1209  $pat .= '[^;]*?=>\s*' . $variant . '\s*:|';
1210  }
1211  $pat .= '\s*$)/';
1212  $this->mVarSeparatorPattern = $pat;
1213  }
1214  return $this->mVarSeparatorPattern;
1215  }
1216 }
LanguageCode\replaceDeprecatedCodes
static replaceDeprecatedCodes( $code)
Replace deprecated language codes that were used in previous versions of MediaWiki to up-to-date,...
Definition: LanguageCode.php:162
ConverterRule
Parser for rules of language conversion, parse rules in -{ }- tag.
Definition: ConverterRule.php:27
LanguageCode\getDeprecatedCodeMapping
static getDeprecatedCodeMapping()
Returns a mapping of deprecated language codes that were used in previous versions of MediaWiki to up...
Definition: LanguageCode.php:125
Title\newFromText
static newFromText( $text, $defaultNamespace=NS_MAIN)
Create a new Title from text, such as what one would find in a link.
Definition: Title.php:317
Revision\RevisionRecord
Page revision base class.
Definition: RevisionRecord.php:46
$wgDisabledVariants
$wgDisabledVariants
Disabled variants array of language variant conversion.
Definition: DefaultSettings.php:3153
LinkBatch
Class representing a list of titles The execute() method checks them all for existence and adds them ...
Definition: LinkBatch.php:35
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:130
$languages
switch( $options['output']) $languages
Definition: transstat.php:76
$wgDisableTitleConversion
$wgDisableTitleConversion
Whether to enable language variant conversion for links.
Definition: DefaultSettings.php:3131
wfMessage
wfMessage( $key,... $params)
This is the function for getting translated interface messages.
Definition: GlobalFunctions.php:1263
$s
$s
Definition: mergeMessageFileList.php:185
CONTENT_MODEL_WIKITEXT
const CONTENT_MODEL_WIKITEXT
Definition: Defines.php:215
NS_MAIN
const NS_MAIN
Definition: Defines.php:60
Revision\newFromTitle
static newFromTitle(LinkTarget $linkTarget, $id=0, $flags=0)
Load either the current, or a specified, revision that's attached to a given link target.
Definition: Revision.php:138
Title\getDBkey
getDBkey()
Get the main part with underscores.
Definition: Title.php:1014
MWException
MediaWiki exception.
Definition: MWException.php:26
MediaWiki\Logger\LoggerFactory
PSR-3 logger instance factory.
Definition: LoggerFactory.php:45
Title\getNamespace
getNamespace()
Get the namespace index, i.e.
Definition: Title.php:1035
$wgLanguageConverterCacheType
$wgLanguageConverterCacheType
The cache type for storing language conversion tables, which are used when parsing certain text and i...
Definition: DefaultSettings.php:2373
$wgDefaultLanguageVariant
$wgDefaultLanguageVariant
Default variant code, if false, the default will be the language code.
Definition: DefaultSettings.php:3136
StringUtils\explode
static explode( $separator, $subject)
Workalike for explode() with limited memory usage.
Definition: StringUtils.php:356
ObjectCache\getInstance
static getInstance( $id)
Get a cached instance of the specified type of cache object.
Definition: ObjectCache.php:78
$title
$title
Definition: testCompression.php:36
$wgDisableLangConversion
$wgDisableLangConversion
Whether to enable language variant conversion.
Definition: DefaultSettings.php:3126
MessageCache\singleton
static singleton()
Get the singleton instance of this class.
Definition: MessageCache.php:118
Title\makeTitleSafe
static makeTitleSafe( $ns, $title, $fragment='', $interwiki='')
Create a new Title from a namespace index and a DB key.
Definition: Title.php:610
ReplacementArray
Wrapper around strtr() that holds replacements.
Definition: ReplacementArray.php:24
Title
Represents a title within MediaWiki.
Definition: Title.php:42
$cache
$cache
Definition: mcc.php:33
LanguageCode\bcp47
static bcp47( $code)
Get the normalised IETF language tag See unit test for examples.
Definition: LanguageCode.php:176
NS_MEDIAWIKI
const NS_MEDIAWIKI
Definition: Defines.php:68
$t
$t
Definition: testCompression.php:71
$wgRequest
if(! $wgDBerrorLogTZ) $wgRequest
Definition: Setup.php:713
Language\fetchLanguageNames
static fetchLanguageNames( $inLanguage=self::AS_AUTONYMS, $include='mw')
Get an array of language names, indexed by code.
Definition: Language.php:854
Hooks\run
static run( $event, array $args=[], $deprecatedVersion=null)
Call hook functions defined in Hooks::register and $wgHooks.
Definition: Hooks.php:200
Language
Internationalisation code.
Definition: Language.php:39