MediaWiki  1.28.0
IcuCollation.php
Go to the documentation of this file.
1 <?php
24 class IcuCollation extends Collation {
26 
29 
31  private $mainCollator;
32 
34  private $locale;
35 
38 
40  private $useNumericCollation = false;
41 
44 
54  private static $cjkBlocks = [
55  [ 0x2E80, 0x2EFF ], // CJK Radicals Supplement
56  [ 0x2F00, 0x2FDF ], // Kangxi Radicals
57  [ 0x2FF0, 0x2FFF ], // Ideographic Description Characters
58  [ 0x3000, 0x303F ], // CJK Symbols and Punctuation
59  [ 0x31C0, 0x31EF ], // CJK Strokes
60  [ 0x3200, 0x32FF ], // Enclosed CJK Letters and Months
61  [ 0x3300, 0x33FF ], // CJK Compatibility
62  [ 0x3400, 0x4DBF ], // CJK Unified Ideographs Extension A
63  [ 0x4E00, 0x9FFF ], // CJK Unified Ideographs
64  [ 0xF900, 0xFAFF ], // CJK Compatibility Ideographs
65  [ 0xFE30, 0xFE4F ], // CJK Compatibility Forms
66  [ 0x20000, 0x2A6DF ], // CJK Unified Ideographs Extension B
67  [ 0x2A700, 0x2B73F ], // CJK Unified Ideographs Extension C
68  [ 0x2B740, 0x2B81F ], // CJK Unified Ideographs Extension D
69  [ 0x2F800, 0x2FA1F ], // CJK Compatibility Ideographs Supplement
70  ];
71 
93  private static $tailoringFirstLetters = [
94  // Verified by native speakers
95  'be' => [ "Ё" ],
96  'be-tarask' => [ "Ё" ],
97  'bs' => [ "Č", "Ć", "Dž", "Đ", "Lj", "Nj", "Š", "Ž" ],
98  'cs' => [ "Č", "Ch", "Ř", "Š", "Ž" ],
99  'cy' => [ "Ch", "Dd", "Ff", "Ng", "Ll", "Ph", "Rh", "Th" ],
100  'en' => [],
101  'fa' => [
102  // RTL, let's put each letter on a new line
103  "آ",
104  "ء",
105  "ه",
106  "ا",
107  "و"
108  ],
109  'fi' => [ "Å", "Ä", "Ö" ],
110  'fr' => [],
111  'hr' => [ "Č", "Ć", "Dž", "Đ", "Lj", "Nj", "Š", "Ž" ],
112  'hsb' => [ "Č", "Dź", "Ě", "Ch", "Ł", "Ń", "Ř", "Š", "Ć", "Ž" ],
113  'hu' => [ "Cs", "Dz", "Dzs", "Gy", "Ly", "Ny", "Ö", "Sz", "Ty", "Ü", "Zs" ],
114  'is' => [ "Á", "Ð", "É", "Í", "Ó", "Ú", "Ý", "Þ", "Æ", "Ö", "Å" ],
115  'it' => [],
116  'lt' => [ "Č", "Š", "Ž" ],
117  'lv' => [ "Č", "Ģ", "Ķ", "Ļ", "Ņ", "Š", "Ž" ],
118  'mk' => [ "Ѓ", "Ќ" ],
119  'nl' => [],
120  'pl' => [ "Ą", "Ć", "Ę", "Ł", "Ń", "Ó", "Ś", "Ź", "Ż" ],
121  'pt' => [],
122  'ru' => [],
123  'sk' => [ "Ä", "Č", "Ch", "Ô", "Š", "Ž" ],
124  'sr' => [],
125  'sv' => [ "Å", "Ä", "Ö" ],
126  'sv@collation=standard' => [ "Å", "Ä", "Ö" ],
127  'ta' => [
128  "\xE0\xAE\x82", "ஃ", "க்ஷ", "க்", "ங்", "ச்", "ஞ்", "ட்", "ண்", "த்", "ந்",
129  "ப்", "ம்", "ய்", "ர்", "ல்", "வ்", "ழ்", "ள்", "ற்", "ன்", "ஜ்", "ஶ்", "ஷ்",
130  "ஸ்", "ஹ்", "க்ஷ்"
131  ],
132  'uk' => [ "Ґ", "Ь" ],
133  'vi' => [ "Ă", "Â", "Đ", "Ê", "Ô", "Ơ", "Ư" ],
134  // Not verified, but likely correct
135  'af' => [],
136  'ast' => [ "Ch", "Ll", "Ñ" ],
137  'az' => [ "Ç", "Ə", "Ğ", "İ", "Ö", "Ş", "Ü" ],
138  'bg' => [],
139  'br' => [ "Ch", "C'h" ],
140  'ca' => [],
141  'co' => [],
142  'da' => [ "Æ", "Ø", "Å" ],
143  'de' => [],
144  'dsb' => [ "Č", "Ć", "Dź", "Ě", "Ch", "Ł", "Ń", "Ŕ", "Š", "Ś", "Ž", "Ź" ],
145  'el' => [],
146  'eo' => [ "Ĉ", "Ĝ", "Ĥ", "Ĵ", "Ŝ", "Ŭ" ],
147  'es' => [ "Ñ" ],
148  'et' => [ "Š", "Ž", "Õ", "Ä", "Ö", "Ü", "W" ], // added W for CollationEt (xx-uca-et)
149  'eu' => [ "Ñ" ],
150  'fo' => [ "Á", "Ð", "Í", "Ó", "Ú", "Ý", "Æ", "Ø", "Å" ],
151  'fur' => [ "À", "Á", "Â", "È", "Ì", "Ò", "Ù" ],
152  'fy' => [],
153  'ga' => [],
154  'gd' => [],
155  'gl' => [ "Ch", "Ll", "Ñ" ],
156  'kk' => [ "Ү", "І" ],
157  'kl' => [ "Æ", "Ø", "Å" ],
158  'ku' => [ "Ç", "Ê", "Î", "Ş", "Û" ],
159  'ky' => [ "Ё" ],
160  'la' => [],
161  'lb' => [],
162  'mo' => [ "Ă", "Â", "Î", "Ş", "Ţ" ],
163  'mt' => [ "Ċ", "Ġ", "Għ", "Ħ", "Ż" ],
164  'no' => [ "Æ", "Ø", "Å" ],
165  'oc' => [],
166  'rm' => [],
167  'ro' => [ "Ă", "Â", "Î", "Ş", "Ţ" ],
168  'rup' => [ "Ă", "Â", "Î", "Ľ", "Ń", "Ş", "Ţ" ],
169  'sco' => [],
170  'sl' => [ "Č", "Š", "Ž" ],
171  'smn' => [ "Á", "Č", "Đ", "Ŋ", "Š", "Ŧ", "Ž", "Æ", "Ø", "Å", "Ä", "Ö" ],
172  'sq' => [ "Ç", "Dh", "Ë", "Gj", "Ll", "Nj", "Rr", "Sh", "Th", "Xh", "Zh" ],
173  'tk' => [ "Ç", "Ä", "Ž", "Ň", "Ö", "Ş", "Ü", "Ý" ],
174  'tl' => [ "Ñ", "Ng" ],
175  'tr' => [ "Ç", "Ğ", "İ", "Ö", "Ş", "Ü" ],
176  'tt' => [ "Ә", "Ө", "Ү", "Җ", "Ң", "Һ" ],
177  'uz' => [ "Ch", "G'", "Ng", "O'", "Sh" ],
178  ];
179 
183  const RECORD_LENGTH = 14;
184 
185  public function __construct( $locale ) {
186  if ( !extension_loaded( 'intl' ) ) {
187  throw new MWException( 'An ICU collation was requested, ' .
188  'but the intl extension is not available.' );
189  }
190 
191  $this->locale = $locale;
192  // Drop everything after the '@' in locale's name
193  $localeParts = explode( '@', $locale );
194  $this->digitTransformLanguage = Language::factory( $locale === 'root' ? 'en' : $localeParts[0] );
195 
196  $this->mainCollator = Collator::create( $locale );
197  if ( !$this->mainCollator ) {
198  throw new MWException( "Invalid ICU locale specified for collation: $locale" );
199  }
200 
201  $this->primaryCollator = Collator::create( $locale );
202  $this->primaryCollator->setStrength( Collator::PRIMARY );
203 
204  // If the special suffix for numeric collation is present, turn on numeric collation.
205  if ( substr( $locale, -5, 5 ) === '-u-kn' ) {
206  $this->useNumericCollation = true;
207  // Strip off the special suffix so it doesn't trip up fetchFirstLetterData().
208  $this->locale = substr( $this->locale, 0, -5 );
209  $this->mainCollator->setAttribute( Collator::NUMERIC_COLLATION, Collator::ON );
210  $this->primaryCollator->setAttribute( Collator::NUMERIC_COLLATION, Collator::ON );
211  }
212  }
213 
214  public function getSortKey( $string ) {
215  return $this->mainCollator->getSortKey( $string );
216  }
217 
218  public function getPrimarySortKey( $string ) {
219  return $this->primaryCollator->getSortKey( $string );
220  }
221 
222  public function getFirstLetter( $string ) {
223  $string = strval( $string );
224  if ( $string === '' ) {
225  return '';
226  }
227 
228  $firstChar = mb_substr( $string, 0, 1, 'UTF-8' );
229 
230  // If the first character is a CJK character, just return that character.
231  if ( ord( $firstChar ) > 0x7f && self::isCjk( UtfNormal\Utils::utf8ToCodepoint( $firstChar ) ) ) {
232  return $firstChar;
233  }
234 
235  $sortKey = $this->getPrimarySortKey( $string );
236 
237  // Do a binary search to find the correct letter to sort under
239  [ $this, 'getSortKeyByLetterIndex' ],
240  $this->getFirstLetterCount(),
241  'strcmp',
242  $sortKey );
243 
244  if ( $min === false ) {
245  // Before the first letter
246  return '';
247  }
248 
249  $sortLetter = $this->getLetterByIndex( $min );
250 
251  if ( $this->useNumericCollation ) {
252  // If the sort letter is a number, return '0–9' (or localized equivalent).
253  // ASCII value of 0 is 48. ASCII value of 9 is 57.
254  // Note that this also applies to non-Arabic numerals since they are
255  // mapped to Arabic numeral sort letters. For example, ২ sorts as 2.
256  if ( ord( $sortLetter ) >= 48 && ord( $sortLetter ) <= 57 ) {
257  $sortLetter = wfMessage( 'category-header-numerals' )->numParams( 0, 9 )->text();
258  }
259  }
260  return $sortLetter;
261  }
262 
267  public function getFirstLetterData() {
268  if ( $this->firstLetterData === null ) {
270  $cacheKey = $cache->makeKey(
271  'first-letters',
272  $this->locale,
273  $this->digitTransformLanguage->getCode(),
274  self::getICUVersion(),
275  self::FIRST_LETTER_VERSION
276  );
277  $this->firstLetterData = $cache->getWithSetCallback( $cacheKey, $cache::TTL_WEEK, function () {
278  return $this->fetchFirstLetterData();
279  } );
280  }
281  return $this->firstLetterData;
282  }
283 
288  private function fetchFirstLetterData() {
289  // Generate data from serialized data file
290  if ( isset( self::$tailoringFirstLetters[$this->locale] ) ) {
291  $letters = wfGetPrecompiledData( 'first-letters-root.ser' );
292  // Append additional characters
293  $letters = array_merge( $letters, self::$tailoringFirstLetters[$this->locale] );
294  // Remove unnecessary ones, if any
295  if ( isset( self::$tailoringFirstLetters['-' . $this->locale] ) ) {
296  $letters = array_diff( $letters, self::$tailoringFirstLetters['-' . $this->locale] );
297  }
298  // Apply digit transforms
299  $digits = [ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' ];
300  $letters = array_diff( $letters, $digits );
301  foreach ( $digits as $digit ) {
302  $letters[] = $this->digitTransformLanguage->formatNum( $digit, true );
303  }
304  } else {
305  $letters = wfGetPrecompiledData( "first-letters-{$this->locale}.ser" );
306  if ( $letters === false ) {
307  throw new MWException( "MediaWiki does not support ICU locale " .
308  "\"{$this->locale}\"" );
309  }
310  }
311 
312  /* Sort the letters.
313  *
314  * It's impossible to have the precompiled data file properly sorted,
315  * because the sort order changes depending on ICU version. If the
316  * array is not properly sorted, the binary search will return random
317  * results.
318  *
319  * We also take this opportunity to remove primary collisions.
320  */
321  $letterMap = [];
322  foreach ( $letters as $letter ) {
323  $key = $this->getPrimarySortKey( $letter );
324  if ( isset( $letterMap[$key] ) ) {
325  // Primary collision
326  // Keep whichever one sorts first in the main collator
327  if ( $this->mainCollator->compare( $letter, $letterMap[$key] ) < 0 ) {
328  $letterMap[$key] = $letter;
329  }
330  } else {
331  $letterMap[$key] = $letter;
332  }
333  }
334  ksort( $letterMap, SORT_STRING );
335 
336  /* Remove duplicate prefixes. Basically if something has a sortkey
337  * which is a prefix of some other sortkey, then it is an
338  * expansion and probably should not be considered a section
339  * header.
340  *
341  * For example 'þ' is sometimes sorted as if it is the letters
342  * 'th'. Other times it is its own primary element. Another
343  * example is '₨'. Sometimes its a currency symbol. Sometimes it
344  * is an 'R' followed by an 's'.
345  *
346  * Additionally an expanded element should always sort directly
347  * after its first element due to they way sortkeys work.
348  *
349  * UCA sortkey elements are of variable length but no collation
350  * element should be a prefix of some other element, so I think
351  * this is safe. See:
352  * - https://ssl.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm
353  * - http://site.icu-project.org/design/collation/uca-weight-allocation
354  *
355  * Additionally, there is something called primary compression to
356  * worry about. Basically, if you have two primary elements that
357  * are more than one byte and both start with the same byte then
358  * the first byte is dropped on the second primary. Additionally
359  * either \x03 or \xFF may be added to mean that the next primary
360  * does not start with the first byte of the first primary.
361  *
362  * This shouldn't matter much, as the first primary is not
363  * changed, and that is what we are comparing against.
364  *
365  * tl;dr: This makes some assumptions about how icu implements
366  * collations. It seems incredibly unlikely these assumptions
367  * will change, but nonetheless they are assumptions.
368  */
369 
370  $prev = false;
371  $duplicatePrefixes = [];
372  foreach ( $letterMap as $key => $value ) {
373  // Remove terminator byte. Otherwise the prefix
374  // comparison will get hung up on that.
375  $trimmedKey = rtrim( $key, "\0" );
376  if ( $prev === false || $prev === '' ) {
377  $prev = $trimmedKey;
378  // We don't yet have a collation element
379  // to compare against, so continue.
380  continue;
381  }
382 
383  // Due to the fact the array is sorted, we only have
384  // to compare with the element directly previous
385  // to the current element (skipping expansions).
386  // An element "X" will always sort directly
387  // before "XZ" (Unless we have "XY", but we
388  // do not update $prev in that case).
389  if ( substr( $trimmedKey, 0, strlen( $prev ) ) === $prev ) {
390  $duplicatePrefixes[] = $key;
391  // If this is an expansion, we don't want to
392  // compare the next element to this element,
393  // but to what is currently $prev
394  continue;
395  }
396  $prev = $trimmedKey;
397  }
398  foreach ( $duplicatePrefixes as $badKey ) {
399  wfDebug( "Removing '{$letterMap[$badKey]}' from first letters.\n" );
400  unset( $letterMap[$badKey] );
401  // This code assumes that unsetting does not change sort order.
402  }
403  $data = [
404  'chars' => array_values( $letterMap ),
405  'keys' => array_keys( $letterMap ),
406  ];
407 
408  // Reduce memory usage before caching
409  unset( $letterMap );
410 
411  return $data;
412  }
413 
417  public function getLetterByIndex( $index ) {
418  return $this->getFirstLetterData()['chars'][$index];
419  }
420 
424  public function getSortKeyByLetterIndex( $index ) {
425  return $this->getFirstLetterData()['keys'][$index];
426  }
427 
431  public function getFirstLetterCount() {
432  return count( $this->getFirstLetterData()['chars'] );
433  }
434 
439  public static function isCjk( $codepoint ) {
440  foreach ( self::$cjkBlocks as $block ) {
441  if ( $codepoint >= $block[0] && $codepoint <= $block[1] ) {
442  return true;
443  }
444  }
445  return false;
446  }
447 
460  static function getICUVersion() {
461  return defined( 'INTL_ICU_VERSION' ) ? INTL_ICU_VERSION : false;
462  }
463 
471  static function getUnicodeVersionForICU() {
472  $icuVersion = IcuCollation::getICUVersion();
473  if ( !$icuVersion ) {
474  return false;
475  }
476 
477  $versionPrefix = substr( $icuVersion, 0, 3 );
478  // Source: http://site.icu-project.org/download
479  $map = [
480  '57.' => '8.0',
481  '56.' => '8.0',
482  '55.' => '7.0',
483  '54.' => '7.0',
484  '53.' => '6.3',
485  '52.' => '6.3',
486  '51.' => '6.2',
487  '50.' => '6.2',
488  '49.' => '6.1',
489  '4.8' => '6.0',
490  '4.6' => '6.0',
491  '4.4' => '5.2',
492  '4.2' => '5.1',
493  '4.0' => '5.1',
494  '3.8' => '5.0',
495  '3.6' => '5.0',
496  '3.4' => '4.1',
497  ];
498 
499  if ( isset( $map[$versionPrefix] ) ) {
500  return $map[$versionPrefix];
501  } else {
502  return false;
503  }
504  }
505 }
utf8ToCodepoint($char)
Determine the Unicode codepoint of a single-character UTF-8 sequence.
getLetterByIndex($index)
Collator $mainCollator
getSortKey($string)
$value
__construct($locale)
getFirstLetter($string)
do that in ParserLimitReportFormat instead use this to modify the parameters of the image and a DIV can begin in one section and end in another Make sure your code can handle that case gracefully See the EditSectionClearerLink extension for an example zero but section is usually empty its values are the globals values before the output is cached one of create
Definition: hooks.txt:2491
wfDebug($text, $dest= 'all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
getSortKeyByLetterIndex($index)
Unicode normalization routines for working with UTF-8 strings.
Definition: UtfNormal.php:48
static getICUVersion()
Return the version of ICU library used by PHP's intl extension, or false when the extension is not in...
getPrimarySortKey($string)
either a unescaped string or a HtmlArmor object after in associative array form externallinks including delete and has completed for all link tables whether this was an auto creation default is conds Array Extra conditions for the No matching items in log is displayed if loglist is empty msgKey Array If you want a nice box with a set this to the key of the message First element is the message additional optional elements are parameters for the key that are processed with wfMessage() -> params() ->parseAsBlock()-offset Set to overwrite offset parameter in $wgRequest set to ''to unsetoffset-wrap String Wrap the message in html(usually something like"&lt
array $firstLetterData
static isCjk($codepoint)
Test if a code point is a CJK (Chinese, Japanese, Korean) character.
$cache
Definition: mcc.php:33
static findLowerBound($valueCallback, $valueCount, $comparisonCallback, $target)
Do a binary search, and return the index of the largest item that sorts less than or equal to the tar...
Definition: ArrayUtils.php:112
boolean $useNumericCollation
const FIRST_LETTER_VERSION
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
Collator $primaryCollator
Language $digitTransformLanguage
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
static getLocalServerInstance($fallback=CACHE_NONE)
Factory function for CACHE_ACCEL (referenced from DefaultSettings.php)
const CACHE_ANYTHING
Definition: Defines.php:93
string $locale
static $tailoringFirstLetters
Additional characters (or character groups) to be considered separate letters for given languages...
static const RECORD_LENGTH
static factory($code)
Get a cached or new language object for a given language code.
Definition: Language.php:181
wfGetPrecompiledData($name)
Get an object from the precompiled serialized directory.
static getUnicodeVersionForICU()
Return the version of Unicode appropriate for the version of ICU library currently in use...
static $cjkBlocks
Unified CJK blocks.