MediaWiki  1.27.2
IcuCollation.php
Go to the documentation of this file.
1 <?php
24 class IcuCollation extends Collation {
26 
29 
31  private $mainCollator;
32 
34  private $locale;
35 
38 
41 
51  private static $cjkBlocks = [
52  [ 0x2E80, 0x2EFF ], // CJK Radicals Supplement
53  [ 0x2F00, 0x2FDF ], // Kangxi Radicals
54  [ 0x2FF0, 0x2FFF ], // Ideographic Description Characters
55  [ 0x3000, 0x303F ], // CJK Symbols and Punctuation
56  [ 0x31C0, 0x31EF ], // CJK Strokes
57  [ 0x3200, 0x32FF ], // Enclosed CJK Letters and Months
58  [ 0x3300, 0x33FF ], // CJK Compatibility
59  [ 0x3400, 0x4DBF ], // CJK Unified Ideographs Extension A
60  [ 0x4E00, 0x9FFF ], // CJK Unified Ideographs
61  [ 0xF900, 0xFAFF ], // CJK Compatibility Ideographs
62  [ 0xFE30, 0xFE4F ], // CJK Compatibility Forms
63  [ 0x20000, 0x2A6DF ], // CJK Unified Ideographs Extension B
64  [ 0x2A700, 0x2B73F ], // CJK Unified Ideographs Extension C
65  [ 0x2B740, 0x2B81F ], // CJK Unified Ideographs Extension D
66  [ 0x2F800, 0x2FA1F ], // CJK Compatibility Ideographs Supplement
67  ];
68 
90  private static $tailoringFirstLetters = [
91  // Verified by native speakers
92  'be' => [ "Ё" ],
93  'be-tarask' => [ "Ё" ],
94  'cy' => [ "Ch", "Dd", "Ff", "Ng", "Ll", "Ph", "Rh", "Th" ],
95  'en' => [],
96  'fa' => [ "آ", "ء", "ه" ],
97  'fi' => [ "Å", "Ä", "Ö" ],
98  'fr' => [],
99  'hu' => [ "Cs", "Dz", "Dzs", "Gy", "Ly", "Ny", "Ö", "Sz", "Ty", "Ü", "Zs" ],
100  'is' => [ "Á", "Ð", "É", "Í", "Ó", "Ú", "Ý", "Þ", "Æ", "Ö", "Å" ],
101  'it' => [],
102  'lv' => [ "Č", "Ģ", "Ķ", "Ļ", "Ņ", "Š", "Ž" ],
103  'pl' => [ "Ą", "Ć", "Ę", "Ł", "Ń", "Ó", "Ś", "Ź", "Ż" ],
104  'pt' => [],
105  'ru' => [],
106  'sv' => [ "Å", "Ä", "Ö" ],
107  'sv@collation=standard' => [ "Å", "Ä", "Ö" ],
108  'uk' => [ "Ґ", "Ь" ],
109  'vi' => [ "Ă", "Â", "Đ", "Ê", "Ô", "Ơ", "Ư" ],
110  // Not verified, but likely correct
111  'af' => [],
112  'ast' => [ "Ch", "Ll", "Ñ" ],
113  'az' => [ "Ç", "Ə", "Ğ", "İ", "Ö", "Ş", "Ü" ],
114  'bg' => [],
115  'br' => [ "Ch", "C'h" ],
116  'bs' => [ "Č", "Ć", "Dž", "Đ", "Lj", "Nj", "Š", "Ž" ],
117  'ca' => [],
118  'co' => [],
119  'cs' => [ "Č", "Ch", "Ř", "Š", "Ž" ],
120  'da' => [ "Æ", "Ø", "Å" ],
121  'de' => [],
122  'dsb' => [ "Č", "Ć", "Dź", "Ě", "Ch", "Ł", "Ń", "Ŕ", "Š", "Ś", "Ž", "Ź" ],
123  'el' => [],
124  'eo' => [ "Ĉ", "Ĝ", "Ĥ", "Ĵ", "Ŝ", "Ŭ" ],
125  'es' => [ "Ñ" ],
126  'et' => [ "Š", "Ž", "Õ", "Ä", "Ö", "Ü", "W" ], // added W for CollationEt (xx-uca-et)
127  'eu' => [ "Ñ" ],
128  'fo' => [ "Á", "Ð", "Í", "Ó", "Ú", "Ý", "Æ", "Ø", "Å" ],
129  'fur' => [ "À", "Á", "Â", "È", "Ì", "Ò", "Ù" ],
130  'fy' => [],
131  'ga' => [],
132  'gd' => [],
133  'gl' => [ "Ch", "Ll", "Ñ" ],
134  'hr' => [ "Č", "Ć", "Dž", "Đ", "Lj", "Nj", "Š", "Ž" ],
135  'hsb' => [ "Č", "Dź", "Ě", "Ch", "Ł", "Ń", "Ř", "Š", "Ć", "Ž" ],
136  'kk' => [ "Ү", "І" ],
137  'kl' => [ "Æ", "Ø", "Å" ],
138  'ku' => [ "Ç", "Ê", "Î", "Ş", "Û" ],
139  'ky' => [ "Ё" ],
140  'la' => [],
141  'lb' => [],
142  'lt' => [ "Č", "Š", "Ž" ],
143  'mk' => [],
144  'mo' => [ "Ă", "Â", "Î", "Ş", "Ţ" ],
145  'mt' => [ "Ċ", "Ġ", "Għ", "Ħ", "Ż" ],
146  'nl' => [],
147  'no' => [ "Æ", "Ø", "Å" ],
148  'oc' => [],
149  'rm' => [],
150  'ro' => [ "Ă", "Â", "Î", "Ş", "Ţ" ],
151  'rup' => [ "Ă", "Â", "Î", "Ľ", "Ń", "Ş", "Ţ" ],
152  'sco' => [],
153  'sk' => [ "Ä", "Č", "Ch", "Ô", "Š", "Ž" ],
154  'sl' => [ "Č", "Š", "Ž" ],
155  'smn' => [ "Á", "Č", "Đ", "Ŋ", "Š", "Ŧ", "Ž", "Æ", "Ø", "Å", "Ä", "Ö" ],
156  'sq' => [ "Ç", "Dh", "Ë", "Gj", "Ll", "Nj", "Rr", "Sh", "Th", "Xh", "Zh" ],
157  'sr' => [],
158  'tk' => [ "Ç", "Ä", "Ž", "Ň", "Ö", "Ş", "Ü", "Ý" ],
159  'tl' => [ "Ñ", "Ng" ],
160  'tr' => [ "Ç", "Ğ", "İ", "Ö", "Ş", "Ü" ],
161  'tt' => [ "Ә", "Ө", "Ү", "Җ", "Ң", "Һ" ],
162  'uz' => [ "Ch", "G'", "Ng", "O'", "Sh" ],
163  ];
164 
168  const RECORD_LENGTH = 14;
169 
170  public function __construct( $locale ) {
171  if ( !extension_loaded( 'intl' ) ) {
172  throw new MWException( 'An ICU collation was requested, ' .
173  'but the intl extension is not available.' );
174  }
175 
176  $this->locale = $locale;
177  // Drop everything after the '@' in locale's name
178  $localeParts = explode( '@', $locale );
179  $this->digitTransformLanguage = Language::factory( $locale === 'root' ? 'en' : $localeParts[0] );
180 
181  $this->mainCollator = Collator::create( $locale );
182  if ( !$this->mainCollator ) {
183  throw new MWException( "Invalid ICU locale specified for collation: $locale" );
184  }
185 
186  $this->primaryCollator = Collator::create( $locale );
187  $this->primaryCollator->setStrength( Collator::PRIMARY );
188  }
189 
190  public function getSortKey( $string ) {
191  // intl extension produces non null-terminated
192  // strings. Appending '' fixes it so that it doesn't generate
193  // a warning on each access in debug php.
194  MediaWiki\suppressWarnings();
195  $key = $this->mainCollator->getSortKey( $string ) . '';
196  MediaWiki\restoreWarnings();
197  return $key;
198  }
199 
200  public function getPrimarySortKey( $string ) {
201  MediaWiki\suppressWarnings();
202  $key = $this->primaryCollator->getSortKey( $string ) . '';
203  MediaWiki\restoreWarnings();
204  return $key;
205  }
206 
207  public function getFirstLetter( $string ) {
208  $string = strval( $string );
209  if ( $string === '' ) {
210  return '';
211  }
212 
213  // Check for CJK
214  $firstChar = mb_substr( $string, 0, 1, 'UTF-8' );
215  if ( ord( $firstChar ) > 0x7f && self::isCjk( UtfNormal\Utils::utf8ToCodepoint( $firstChar ) ) ) {
216  return $firstChar;
217  }
218 
219  $sortKey = $this->getPrimarySortKey( $string );
220 
221  // Do a binary search to find the correct letter to sort under
223  [ $this, 'getSortKeyByLetterIndex' ],
224  $this->getFirstLetterCount(),
225  'strcmp',
226  $sortKey );
227 
228  if ( $min === false ) {
229  // Before the first letter
230  return '';
231  }
232  return $this->getLetterByIndex( $min );
233  }
234 
238  public function getFirstLetterData() {
239  if ( $this->firstLetterData !== null ) {
240  return $this->firstLetterData;
241  }
242 
244  $cacheKey = $cache->makeKey(
245  'first-letters',
246  $this->locale,
247  $this->digitTransformLanguage->getCode(),
248  self::getICUVersion()
249  );
250  $cacheEntry = $cache->get( $cacheKey );
251 
252  if ( $cacheEntry && isset( $cacheEntry['version'] )
253  && $cacheEntry['version'] == self::FIRST_LETTER_VERSION
254  ) {
255  $this->firstLetterData = $cacheEntry;
256  return $this->firstLetterData;
257  }
258 
259  // Generate data from serialized data file
260 
261  if ( isset( self::$tailoringFirstLetters[$this->locale] ) ) {
262  $letters = wfGetPrecompiledData( "first-letters-root.ser" );
263  // Append additional characters
264  $letters = array_merge( $letters, self::$tailoringFirstLetters[$this->locale] );
265  // Remove unnecessary ones, if any
266  if ( isset( self::$tailoringFirstLetters['-' . $this->locale] ) ) {
267  $letters = array_diff( $letters, self::$tailoringFirstLetters['-' . $this->locale] );
268  }
269  // Apply digit transforms
270  $digits = [ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' ];
271  $letters = array_diff( $letters, $digits );
272  foreach ( $digits as $digit ) {
273  $letters[] = $this->digitTransformLanguage->formatNum( $digit, true );
274  }
275  } else {
276  $letters = wfGetPrecompiledData( "first-letters-{$this->locale}.ser" );
277  if ( $letters === false ) {
278  throw new MWException( "MediaWiki does not support ICU locale " .
279  "\"{$this->locale}\"" );
280  }
281  }
282 
283  /* Sort the letters.
284  *
285  * It's impossible to have the precompiled data file properly sorted,
286  * because the sort order changes depending on ICU version. If the
287  * array is not properly sorted, the binary search will return random
288  * results.
289  *
290  * We also take this opportunity to remove primary collisions.
291  */
292  $letterMap = [];
293  foreach ( $letters as $letter ) {
294  $key = $this->getPrimarySortKey( $letter );
295  if ( isset( $letterMap[$key] ) ) {
296  // Primary collision
297  // Keep whichever one sorts first in the main collator
298  if ( $this->mainCollator->compare( $letter, $letterMap[$key] ) < 0 ) {
299  $letterMap[$key] = $letter;
300  }
301  } else {
302  $letterMap[$key] = $letter;
303  }
304  }
305  ksort( $letterMap, SORT_STRING );
306 
307  /* Remove duplicate prefixes. Basically if something has a sortkey
308  * which is a prefix of some other sortkey, then it is an
309  * expansion and probably should not be considered a section
310  * header.
311  *
312  * For example 'þ' is sometimes sorted as if it is the letters
313  * 'th'. Other times it is its own primary element. Another
314  * example is '₨'. Sometimes its a currency symbol. Sometimes it
315  * is an 'R' followed by an 's'.
316  *
317  * Additionally an expanded element should always sort directly
318  * after its first element due to they way sortkeys work.
319  *
320  * UCA sortkey elements are of variable length but no collation
321  * element should be a prefix of some other element, so I think
322  * this is safe. See:
323  * - https://ssl.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm
324  * - http://site.icu-project.org/design/collation/uca-weight-allocation
325  *
326  * Additionally, there is something called primary compression to
327  * worry about. Basically, if you have two primary elements that
328  * are more than one byte and both start with the same byte then
329  * the first byte is dropped on the second primary. Additionally
330  * either \x03 or \xFF may be added to mean that the next primary
331  * does not start with the first byte of the first primary.
332  *
333  * This shouldn't matter much, as the first primary is not
334  * changed, and that is what we are comparing against.
335  *
336  * tl;dr: This makes some assumptions about how icu implements
337  * collations. It seems incredibly unlikely these assumptions
338  * will change, but nonetheless they are assumptions.
339  */
340 
341  $prev = false;
342  $duplicatePrefixes = [];
343  foreach ( $letterMap as $key => $value ) {
344  // Remove terminator byte. Otherwise the prefix
345  // comparison will get hung up on that.
346  $trimmedKey = rtrim( $key, "\0" );
347  if ( $prev === false || $prev === '' ) {
348  $prev = $trimmedKey;
349  // We don't yet have a collation element
350  // to compare against, so continue.
351  continue;
352  }
353 
354  // Due to the fact the array is sorted, we only have
355  // to compare with the element directly previous
356  // to the current element (skipping expansions).
357  // An element "X" will always sort directly
358  // before "XZ" (Unless we have "XY", but we
359  // do not update $prev in that case).
360  if ( substr( $trimmedKey, 0, strlen( $prev ) ) === $prev ) {
361  $duplicatePrefixes[] = $key;
362  // If this is an expansion, we don't want to
363  // compare the next element to this element,
364  // but to what is currently $prev
365  continue;
366  }
367  $prev = $trimmedKey;
368  }
369  foreach ( $duplicatePrefixes as $badKey ) {
370  wfDebug( "Removing '{$letterMap[$badKey]}' from first letters.\n" );
371  unset( $letterMap[$badKey] );
372  // This code assumes that unsetting does not change sort order.
373  }
374  $data = [
375  'chars' => array_values( $letterMap ),
376  'keys' => array_keys( $letterMap ),
377  'version' => self::FIRST_LETTER_VERSION,
378  ];
379 
380  // Reduce memory usage before caching
381  unset( $letterMap );
382 
383  // Save to cache
384  $this->firstLetterData = $data;
385  $cache->set( $cacheKey, $data, $cache::TTL_WEEK );
386  return $data;
387  }
388 
392  public function getLetterByIndex( $index ) {
393  if ( $this->firstLetterData === null ) {
394  $this->getFirstLetterData();
395  }
396  return $this->firstLetterData['chars'][$index];
397  }
398 
402  public function getSortKeyByLetterIndex( $index ) {
403  if ( $this->firstLetterData === null ) {
404  $this->getFirstLetterData();
405  }
406  return $this->firstLetterData['keys'][$index];
407  }
408 
412  public function getFirstLetterCount() {
413  if ( $this->firstLetterData === null ) {
414  $this->getFirstLetterData();
415  }
416  return count( $this->firstLetterData['chars'] );
417  }
418 
422  public static function isCjk( $codepoint ) {
423  foreach ( self::$cjkBlocks as $block ) {
424  if ( $codepoint >= $block[0] && $codepoint <= $block[1] ) {
425  return true;
426  }
427  }
428  return false;
429  }
430 
443  static function getICUVersion() {
444  return defined( 'INTL_ICU_VERSION' ) ? INTL_ICU_VERSION : false;
445  }
446 
454  static function getUnicodeVersionForICU() {
455  $icuVersion = IcuCollation::getICUVersion();
456  if ( !$icuVersion ) {
457  return false;
458  }
459 
460  $versionPrefix = substr( $icuVersion, 0, 3 );
461  // Source: http://site.icu-project.org/download
462  $map = [
463  '50.' => '6.2',
464  '49.' => '6.1',
465  '4.8' => '6.0',
466  '4.6' => '6.0',
467  '4.4' => '5.2',
468  '4.2' => '5.1',
469  '4.0' => '5.1',
470  '3.8' => '5.0',
471  '3.6' => '5.0',
472  '3.4' => '4.1',
473  ];
474 
475  if ( isset( $map[$versionPrefix] ) ) {
476  return $map[$versionPrefix];
477  } else {
478  return false;
479  }
480  }
481 }
utf8ToCodepoint($char)
Determine the Unicode codepoint of a single-character UTF-8 sequence.
magic word the default is to use $key to get the and $key value or $key value text $key value html to format the value $key
Definition: hooks.txt:2321
getLetterByIndex($index)
Collator $mainCollator
getSortKey($string)
$value
__construct($locale)
getFirstLetter($string)
do that in ParserLimitReportFormat instead use this to modify the parameters of the image and a DIV can begin in one section and end in another Make sure your code can handle that case gracefully See the EditSectionClearerLink extension for an example zero but section is usually empty its values are the globals values before the output is cached one of create
Definition: hooks.txt:2338
wfDebug($text, $dest= 'all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
getSortKeyByLetterIndex($index)
Unicode normalization routines for working with UTF-8 strings.
Definition: UtfNormal.php:48
static getICUVersion()
Return the version of ICU library used by PHP's intl extension, or false when the extension is not in...
getPrimarySortKey($string)
array $firstLetterData
static isCjk($codepoint)
$cache
Definition: mcc.php:33
static findLowerBound($valueCallback, $valueCount, $comparisonCallback, $target)
Do a binary search, and return the index of the largest item that sorts less than or equal to the tar...
Definition: ArrayUtils.php:112
const FIRST_LETTER_VERSION
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
Collator $primaryCollator
Language $digitTransformLanguage
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
static getLocalServerInstance($fallback=CACHE_NONE)
Factory function for CACHE_ACCEL (referenced from DefaultSettings.php)
const CACHE_ANYTHING
Definition: Defines.php:101
string $locale
static $tailoringFirstLetters
Additional characters (or character groups) to be considered separate letters for given languages...
static const RECORD_LENGTH
static factory($code)
Get a cached or new language object for a given language code.
Definition: Language.php:179
wfGetPrecompiledData($name)
Get an object from the precompiled serialized directory.
static getUnicodeVersionForICU()
Return the version of Unicode appropriate for the version of ICU library currently in use...
static $cjkBlocks
Unified CJK blocks.