MediaWiki REL1_35
generateCollationData.php
Go to the documentation of this file.
1<?php
24require_once __DIR__ . '/../Maintenance.php';
25
27
35 public $dataDir;
36
38 public $weights;
39
46
48
52 private const NORMAL_UPPERCASE = 0x08;
53 private const NORMAL_HIRAGANA = 0x0E;
54
55 public function __construct() {
56 parent::__construct();
57 $this->addOption( 'data-dir', 'A directory on the local filesystem ' .
58 'containing allkeys.txt and ucd.all.grouped.xml from unicode.org',
59 false, true );
60 $this->addOption( 'debug-output', 'Filename for sending debug output to',
61 false, true );
62 }
63
64 public function execute() {
65 $this->dataDir = $this->getOption( 'data-dir', '.' );
66
67 $allkeysPresent = file_exists( "{$this->dataDir}/allkeys.txt" );
68 $ucdallPresent = file_exists( "{$this->dataDir}/ucd.all.grouped.xml" );
69
70 // As of January 2013, these links work for all versions of Unicode
71 // between 5.1 and 6.2, inclusive.
72 $allkeysURL = "https://www.unicode.org/Public/UCA/<Unicode version>/allkeys.txt";
73 $ucdallURL = "https://www.unicode.org/Public/<Unicode version>/ucdxml/ucd.all.grouped.zip";
74
75 if ( !$allkeysPresent || !$ucdallPresent ) {
76 $icuVersion = INTL_ICU_VERSION;
77 $unicodeVersion = IcuCollation::getUnicodeVersionForICU();
78
79 $error = "";
80
81 if ( !$allkeysPresent ) {
82 $error .= "Unable to find allkeys.txt. "
83 . "Download it and specify its location with --data-dir=<DIR>. "
84 . "\n\n";
85 }
86 if ( !$ucdallPresent ) {
87 $error .= "Unable to find ucd.all.grouped.xml. "
88 . "Download it, unzip, and specify its location with --data-dir=<DIR>. "
89 . "\n\n";
90 }
91
92 $versionKnown = false;
93 if ( version_compare( $icuVersion, "4.0", "<" ) ) {
94 // Extra old version
95 $error .= "You are using outdated version of ICU ($icuVersion), intended for "
96 . ( $unicodeVersion ? "Unicode $unicodeVersion" : "an unknown version of Unicode" )
97 . "; this file might not be available for it, and it's not supported by MediaWiki. "
98 . " You are on your own; consider upgrading PHP's intl extension or try "
99 . "one of the files available at:";
100 } elseif ( version_compare( $icuVersion, "51.0", ">=" ) ) {
101 // Extra recent version
102 $error .= "You are using ICU $icuVersion, released after this script was last updated. "
103 . "Check what is the Unicode version it is using at http://site.icu-project.org/download . "
104 . "It can't be guaranteed everything will work, but appropriate file(s) should "
105 . "be available at:";
106 } else {
107 // ICU 4.0 to 50.x
108 $versionKnown = true;
109 $error .= "You are using ICU $icuVersion, intended for "
110 . ( $unicodeVersion ? "Unicode $unicodeVersion" : "an unknown version of Unicode" )
111 . ". Appropriate file(s) should be available at:";
112 }
113 $error .= "\n";
114
115 if ( $versionKnown && $unicodeVersion ) {
116 $allkeysURL = str_replace( "<Unicode version>", "$unicodeVersion.0", $allkeysURL );
117 $ucdallURL = str_replace( "<Unicode version>", "$unicodeVersion.0", $ucdallURL );
118 }
119
120 if ( !$allkeysPresent ) {
121 $error .= "* $allkeysURL\n";
122 }
123 if ( !$ucdallPresent ) {
124 $error .= "* $ucdallURL\n";
125 }
126
127 $this->fatalError( $error );
128 }
129
130 $debugOutFileName = $this->getOption( 'debug-output' );
131 if ( $debugOutFileName ) {
132 $this->debugOutFile = fopen( $debugOutFileName, 'w' );
133 if ( !$this->debugOutFile ) {
134 $this->fatalError( "Unable to open debug output file for writing" );
135 }
136 }
137 $this->loadUcd();
138 $this->generateFirstChars();
139 }
140
141 private function loadUcd() {
142 $uxr = new UcdXmlReader( "{$this->dataDir}/ucd.all.grouped.xml" );
143 $uxr->readChars( [ $this, 'charCallback' ] );
144 }
145
146 private function charCallback( $data ) {
147 // Skip non-printable characters,
148 // but do not skip a normal space (U+0020) since
149 // people like to use that as a fake no header symbol.
150 $category = substr( $data['gc'], 0, 1 );
151 if ( strpos( 'LNPS', $category ) === false
152 && $data['cp'] !== '0020'
153 ) {
154 return;
155 }
156 $cp = hexdec( $data['cp'] );
157
158 // Skip the CJK ideograph blocks, as an optimisation measure.
159 // UCA doesn't sort them properly anyway, without tailoring.
160 if ( IcuCollation::isCjk( $cp ) ) {
161 return;
162 }
163
164 // Skip the composed Hangul syllables, we will use the bare Jamo
165 // as first letters
166 if ( $data['block'] == 'Hangul Syllables' ) {
167 return;
168 }
169
170 // Calculate implicit weight per UTS #10 v6.0.0, sec 7.1.3
171 if ( $data['UIdeo'] === 'Y' ) {
172 if ( $data['block'] == 'CJK Unified Ideographs'
173 || $data['block'] == 'CJK Compatibility Ideographs'
174 ) {
175 $base = 0xFB40;
176 } else {
177 $base = 0xFB80;
178 }
179 } else {
180 $base = 0xFBC0;
181 }
182 $a = $base + ( $cp >> 15 );
183 $b = ( $cp & 0x7fff ) | 0x8000;
184
185 $this->weights[$cp] = sprintf( ".%04X.%04X", $a, $b );
186
187 if ( $data['dm'] !== '#' ) {
188 $this->mappedChars[$cp] = true;
189 }
190
191 if ( $cp % 4096 == 0 ) {
192 print "{$data['cp']}\n";
193 }
194 }
195
196 private function generateFirstChars() {
197 $file = fopen( "{$this->dataDir}/allkeys.txt", 'r' );
198 if ( !$file ) {
199 $this->fatalError( "Unable to open allkeys.txt" );
200 }
201
202 $goodTertiaryChars = [];
203
204 // For each character with an entry in allkeys.txt, overwrite the implicit
205 // entry in $this->weights that came from the UCD.
206 // Also gather a list of tertiary weights, for use in selecting the group header
207 while ( ( $line = fgets( $file ) ) !== false ) {
208 // We're only interested in single-character weights, pick them out with a regex
209 $line = trim( $line );
210 if ( !preg_match( '/^([0-9A-F]+)\s*;\s*([^#]*)/', $line, $m ) ) {
211 continue;
212 }
213
214 $cp = hexdec( $m[1] );
215 $allWeights = trim( $m[2] );
216 $primary = '';
217 $tertiary = '';
218
219 if ( !isset( $this->weights[$cp] ) ) {
220 // Non-printable, ignore
221 continue;
222 }
223 foreach ( StringUtils::explode( '[', $allWeights ) as $weightStr ) {
224 preg_match_all( '/[*.]([0-9A-F]+)/', $weightStr, $m );
225 if ( !empty( $m[1] ) ) {
226 if ( $m[1][0] !== '0000' ) {
227 $primary .= '.' . $m[1][0];
228 }
229 if ( $m[1][2] !== '0000' ) {
230 $tertiary .= '.' . $m[1][2];
231 }
232 }
233 }
234 $this->weights[$cp] = $primary;
235 if ( $tertiary === '.0008'
236 || $tertiary === '.000E'
237 ) {
238 $goodTertiaryChars[$cp] = true;
239 }
240 }
241 fclose( $file );
242
243 // Identify groups of characters with the same primary weight
244 $this->groups = [];
245 asort( $this->weights, SORT_STRING );
246 $prevWeight = reset( $this->weights );
247 $group = [];
248 foreach ( $this->weights as $cp => $weight ) {
249 if ( $weight !== $prevWeight ) {
250 $this->groups[$prevWeight] = $group;
251 $prevWeight = $weight;
252 $group = $this->groups[$weight] ?? [];
253 }
254 $group[] = $cp;
255 }
256 if ( $group ) {
257 $this->groups[$prevWeight] = $group;
258 }
259
260 // If one character has a given primary weight sequence, and a second
261 // character has a longer primary weight sequence with an initial
262 // portion equal to the first character, then remove the second
263 // character. This avoids having characters like U+A732 (double A)
264 // polluting the basic Latin sort area.
265
266 foreach ( $this->groups as $weight => $group ) {
267 if ( preg_match( '/(\.[0-9A-F]*)\./', $weight, $m ) ) {
268 if ( isset( $this->groups[$m[1]] ) ) {
269 unset( $this->groups[$weight] );
270 }
271 }
272 }
273
274 ksort( $this->groups, SORT_STRING );
275
276 // Identify the header character in each group
277 $headerChars = [];
278 $prevChar = "\000";
279 $tertiaryCollator = new Collator( 'root' );
280 $primaryCollator = new Collator( 'root' );
281 $primaryCollator->setStrength( Collator::PRIMARY );
282 $numOutOfOrder = 0;
283 foreach ( $this->groups as $weight => $group ) {
284 $uncomposedChars = [];
285 $goodChars = [];
286 foreach ( $group as $cp ) {
287 if ( isset( $goodTertiaryChars[$cp] ) ) {
288 $goodChars[] = $cp;
289 }
290 if ( !isset( $this->mappedChars[$cp] ) ) {
291 $uncomposedChars[] = $cp;
292 }
293 }
294 $x = array_intersect( $goodChars, $uncomposedChars );
295 if ( !$x ) {
296 $x = $uncomposedChars;
297 if ( !$x ) {
298 $x = $group;
299 }
300 }
301
302 // Use ICU to pick the lowest sorting character in the selection
303 $tertiaryCollator->sort( $x );
304 $cp = $x[0];
305
306 $char = UtfNormal\Utils::codepointToUtf8( $cp );
307 $headerChars[] = $char;
308 if ( $primaryCollator->compare( $char, $prevChar ) <= 0 ) {
309 $numOutOfOrder++;
310 }
311 $prevChar = $char;
312
313 if ( $this->debugOutFile ) {
314 fwrite( $this->debugOutFile, sprintf( "%05X %s %s (%s)\n", $cp, $weight, $char,
315 implode( ' ', array_map( 'UtfNormal\Utils::codepointToUtf8', $group ) ) ) );
316 }
317 }
318
319 print "Out of order: $numOutOfOrder / " . count( $headerChars ) . "\n";
320
321 global $IP;
322 $writer = new StaticArrayWriter();
323 file_put_contents(
324 "$IP/includes/collation/data/first-letters-root.php",
325 $writer->create( $headerChars, 'File created by generateCollationData.php' )
326 );
327 echo "first-letters-root: file written.\n";
328 }
329}
330
332 public $fileName;
333 public $callback;
335 public $xml;
336 public $blocks = [];
338
339 public function __construct( $fileName ) {
340 $this->fileName = $fileName;
341 }
342
343 public function readChars( $callback ) {
344 $this->getBlocks();
345 $this->currentBlock = reset( $this->blocks );
346 $xml = $this->open();
347 $this->callback = $callback;
348
349 while ( $xml->name !== 'repertoire' && $xml->next() );
350
351 while ( $xml->read() ) {
352 if ( $xml->nodeType == XMLReader::ELEMENT ) {
353 if ( $xml->name === 'group' ) {
354 $this->groupAttrs = $this->readAttributes();
355 } elseif ( $xml->name === 'char' ) {
356 $this->handleChar();
357 }
358 } elseif ( $xml->nodeType === XMLReader::END_ELEMENT ) {
359 if ( $xml->name === 'group' ) {
360 $this->groupAttrs = [];
361 }
362 }
363 }
364 $xml->close();
365 }
366
367 protected function open() {
368 $this->xml = new XMLReader;
369 $this->xml->open( $this->fileName );
370 if ( !$this->xml ) {
371 throw new MWException( __METHOD__ . ": unable to open {$this->fileName}" );
372 }
373 while ( $this->xml->name !== 'ucd' && $this->xml->read() );
374 $this->xml->read();
375
376 return $this->xml;
377 }
378
384 protected function readAttributes() {
385 $attrs = [];
386 while ( $this->xml->moveToNextAttribute() ) {
387 $attrs[$this->xml->name] = $this->xml->value;
388 }
389
390 return $attrs;
391 }
392
393 protected function handleChar() {
394 $attrs = $this->readAttributes() + $this->groupAttrs;
395 if ( isset( $attrs['cp'] ) ) {
396 $first = $last = hexdec( $attrs['cp'] );
397 } else {
398 $first = hexdec( $attrs['first-cp'] );
399 $last = hexdec( $attrs['last-cp'] );
400 unset( $attrs['first-cp'] );
401 unset( $attrs['last-cp'] );
402 }
403
404 for ( $cp = $first; $cp <= $last; $cp++ ) {
405 $hexCp = sprintf( "%04X", $cp );
406 foreach ( [ 'na', 'na1' ] as $nameProp ) {
407 if ( isset( $attrs[$nameProp] ) ) {
408 $attrs[$nameProp] = str_replace( '#', $hexCp, $attrs[$nameProp] );
409 }
410 }
411
412 while ( $this->currentBlock ) {
413 if ( $cp < $this->currentBlock[0] ) {
414 break;
415 } elseif ( $cp <= $this->currentBlock[1] ) {
416 $attrs['block'] = key( $this->blocks );
417 break;
418 } else {
419 $this->currentBlock = next( $this->blocks );
420 }
421 }
422
423 $attrs['cp'] = $hexCp;
424 call_user_func( $this->callback, $attrs );
425 }
426 }
427
428 public function getBlocks() {
429 if ( $this->blocks ) {
430 return $this->blocks;
431 }
432
433 $xml = $this->open();
434 while ( $xml->name !== 'blocks' && $xml->read() );
435
436 while ( $xml->read() ) {
437 if ( $xml->nodeType == XMLReader::ELEMENT ) {
438 if ( $xml->name === 'block' ) {
439 $attrs = $this->readAttributes();
440 $first = hexdec( $attrs['first-cp'] );
441 $last = hexdec( $attrs['last-cp'] );
442 $this->blocks[$attrs['name']] = [ $first, $last ];
443 }
444 }
445 }
446 $xml->close();
447
448 return $this->blocks;
449 }
450}
451
452$maintClass = GenerateCollationData::class;
453require_once RUN_MAINTENANCE_IF_MAIN;
const RUN_MAINTENANCE_IF_MAIN
$IP
Definition WebStart.php:49
Generate first letter data files for Collation.php.
__construct()
Default constructor.
$weights
The primary weights, indexed by codepoint.
execute()
Do the actual work.
$dataDir
The directory with source data files in it.
$mappedChars
A hashtable keyed by codepoint, where presence indicates that a character has a decomposition mapping...
const NORMAL_UPPERCASE
Important tertiary weights from UTS #10 section 7.2.
static isCjk( $codepoint)
Test if a code point is a CJK (Chinese, Japanese, Korean) character.
static getUnicodeVersionForICU()
Return the version of Unicode appropriate for the version of ICU library currently in use,...
MediaWiki exception.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
static explode( $separator, $subject)
Workalike for explode() with limited memory usage.
readAttributes()
Read the attributes of the current element node and return them as an array.
Format a static PHP array to be written to a file.
while(( $__line=Maintenance::readconsole()) !==false) print
Definition eval.php:64
$line
Definition mcc.php:119
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition router.php:42