MediaWiki master
generateCollationData.php
Go to the documentation of this file.
1<?php
24// @codeCoverageIgnoreStart
25require_once __DIR__ . '/../Maintenance.php';
26// @codeCoverageIgnoreEnd
27
31
39 public $dataDir;
40
42 public $weights;
43
51
54
56 private $groups;
57
58 public function __construct() {
59 parent::__construct();
60 $this->addOption( 'data-dir', 'A directory on the local filesystem ' .
61 'containing allkeys.txt and ucd.all.grouped.xml from unicode.org',
62 false, true );
63 $this->addOption( 'debug-output', 'Filename for sending debug output to',
64 false, true );
65 }
66
67 public function execute() {
68 $this->dataDir = $this->getOption( 'data-dir', '.' );
69
70 $allkeysPresent = file_exists( "{$this->dataDir}/allkeys.txt" );
71 $ucdallPresent = file_exists( "{$this->dataDir}/ucd.all.grouped.xml" );
72
73 if ( !$allkeysPresent || !$ucdallPresent ) {
74 $icuVersion = INTL_ICU_VERSION;
75 $unicodeVersion = implode( '.', array_slice( IntlChar::getUnicodeVersion(), 0, 3 ) );
76
77 $error = "";
78
79 if ( !$allkeysPresent ) {
80 $error .= "Unable to find allkeys.txt. "
81 . "Download it and specify its location with --data-dir=<DIR>. "
82 . "\n\n";
83 }
84 if ( !$ucdallPresent ) {
85 $error .= "Unable to find ucd.all.grouped.xml. "
86 . "Download it, unzip, and specify its location with --data-dir=<DIR>. "
87 . "\n\n";
88 }
89
90 $error .= "You are using ICU $icuVersion, intended for Unicode $unicodeVersion. "
91 . "Appropriate file(s) should be available at:\n";
92
93 $allkeysURL = "https://www.unicode.org/Public/UCA/$unicodeVersion/allkeys.txt";
94 $ucdallURL = "https://www.unicode.org/Public/$unicodeVersion/ucdxml/ucd.all.grouped.zip";
95
96 if ( !$allkeysPresent ) {
97 $error .= "* $allkeysURL\n";
98 }
99 if ( !$ucdallPresent ) {
100 $error .= "* $ucdallURL\n";
101 }
102
103 $this->fatalError( $error );
104 }
105
106 $debugOutFileName = $this->getOption( 'debug-output' );
107 if ( $debugOutFileName ) {
108 $this->debugOutFile = fopen( $debugOutFileName, 'w' );
109 if ( !$this->debugOutFile ) {
110 $this->fatalError( "Unable to open debug output file for writing" );
111 }
112 }
113 $this->loadUcd();
114 $this->generateFirstChars();
115 }
116
117 private function loadUcd() {
118 $uxr = new UcdXmlReader( "{$this->dataDir}/ucd.all.grouped.xml" );
119 $uxr->readChars( [ $this, 'charCallback' ] );
120 }
121
122 private function charCallback( array $data ) {
123 // Skip non-printable characters,
124 // but do not skip a normal space (U+0020) since
125 // people like to use that as a fake no header symbol.
126 $category = substr( $data['gc'], 0, 1 );
127 if ( strpos( 'LNPS', $category ) === false
128 && $data['cp'] !== '0020'
129 ) {
130 return;
131 }
132 $cp = hexdec( $data['cp'] );
133
134 // Skip the CJK ideograph blocks, as an optimisation measure.
135 // UCA doesn't sort them properly anyway, without tailoring.
136 if ( IcuCollation::isCjk( $cp ) ) {
137 return;
138 }
139
140 // Skip the composed Hangul syllables, we will use the bare Jamo
141 // as first letters
142 if ( $data['block'] == 'Hangul Syllables' ) {
143 return;
144 }
145
146 // Calculate implicit weight per UTS #10 v6.0.0, sec 7.1.3
147 if ( $data['UIdeo'] === 'Y' ) {
148 if ( $data['block'] == 'CJK Unified Ideographs'
149 || $data['block'] == 'CJK Compatibility Ideographs'
150 ) {
151 $base = 0xFB40;
152 } else {
153 $base = 0xFB80;
154 }
155 } else {
156 $base = 0xFBC0;
157 }
158 $a = $base + ( $cp >> 15 );
159 $b = ( $cp & 0x7fff ) | 0x8000;
160
161 $this->weights[$cp] = sprintf( ".%04X.%04X", $a, $b );
162
163 if ( $data['dm'] !== '#' ) {
164 $this->mappedChars[$cp] = true;
165 }
166
167 if ( $cp % 4096 == 0 ) {
168 print "{$data['cp']}\n";
169 }
170 }
171
172 private function generateFirstChars() {
173 $file = fopen( "{$this->dataDir}/allkeys.txt", 'r' );
174 if ( !$file ) {
175 $this->fatalError( "Unable to open allkeys.txt" );
176 }
177
178 $goodTertiaryChars = [];
179
180 // For each character with an entry in allkeys.txt, overwrite the implicit
181 // entry in $this->weights that came from the UCD.
182 // Also gather a list of tertiary weights, for use in selecting the group header
183 // phpcs:ignore Generic.CodeAnalysis.AssignmentInCondition.FoundInWhileCondition
184 while ( ( $line = fgets( $file ) ) !== false ) {
185 // We're only interested in single-character weights, pick them out with a regex
186 $line = trim( $line );
187 if ( !preg_match( '/^([0-9A-F]+)\s*;\s*([^#]*)/', $line, $m ) ) {
188 continue;
189 }
190
191 $cp = hexdec( $m[1] );
192 $allWeights = trim( $m[2] );
193 $primary = '';
194 $tertiary = '';
195
196 if ( !isset( $this->weights[$cp] ) ) {
197 // Non-printable, ignore
198 continue;
199 }
200 foreach ( StringUtils::explode( '[', $allWeights ) as $weightStr ) {
201 if ( preg_match_all( '/[*.]([0-9A-F]+)/', $weightStr, $m ) ) {
202 if ( $m[1][0] !== '0000' ) {
203 $primary .= '.' . $m[1][0];
204 }
205 if ( $m[1][2] !== '0000' ) {
206 $tertiary .= '.' . $m[1][2];
207 }
208 }
209 }
210 $this->weights[$cp] = $primary;
211 if ( $tertiary === '.0008'
212 || $tertiary === '.000E'
213 ) {
214 $goodTertiaryChars[$cp] = true;
215 }
216 }
217 fclose( $file );
218
219 // Identify groups of characters with the same primary weight
220 $this->groups = [];
221 asort( $this->weights, SORT_STRING );
222 $prevWeight = reset( $this->weights );
223 $group = [];
224 foreach ( $this->weights as $cp => $weight ) {
225 if ( $weight !== $prevWeight ) {
226 $this->groups[$prevWeight] = $group;
227 $prevWeight = $weight;
228 $group = $this->groups[$weight] ?? [];
229 }
230 $group[] = $cp;
231 }
232 if ( $group ) {
233 $this->groups[$prevWeight] = $group;
234 }
235
236 // If one character has a given primary weight sequence, and a second
237 // character has a longer primary weight sequence with an initial
238 // portion equal to the first character, then remove the second
239 // character. This avoids having characters like U+A732 (double A)
240 // polluting the basic Latin sort area.
241
242 foreach ( $this->groups as $weight => $group ) {
243 if ( preg_match( '/(\.[0-9A-F]*)\./', $weight, $m ) ) {
244 if ( isset( $this->groups[$m[1]] ) ) {
245 unset( $this->groups[$weight] );
246 }
247 }
248 }
249
250 ksort( $this->groups, SORT_STRING );
251
252 // Identify the header character in each group
253 $headerChars = [];
254 $prevChar = "\000";
255 $tertiaryCollator = new Collator( 'root' );
256 $primaryCollator = new Collator( 'root' );
257 $primaryCollator->setStrength( Collator::PRIMARY );
258 $numOutOfOrder = 0;
259 foreach ( $this->groups as $weight => $group ) {
260 $uncomposedChars = [];
261 $goodChars = [];
262 foreach ( $group as $cp ) {
263 if ( isset( $goodTertiaryChars[$cp] ) ) {
264 $goodChars[] = $cp;
265 }
266 if ( !isset( $this->mappedChars[$cp] ) ) {
267 $uncomposedChars[] = $cp;
268 }
269 }
270 $x = array_intersect( $goodChars, $uncomposedChars );
271 if ( !$x ) {
272 $x = $uncomposedChars;
273 if ( !$x ) {
274 $x = $group;
275 }
276 }
277
278 // Use ICU to pick the lowest sorting character in the selection
279 $tertiaryCollator->sort( $x );
280 $cp = $x[0];
281
282 $char = UtfNormal\Utils::codepointToUtf8( $cp );
283 $headerChars[] = $char;
284 if ( $primaryCollator->compare( $char, $prevChar ) <= 0 ) {
285 $numOutOfOrder++;
286 }
287 $prevChar = $char;
288
289 if ( $this->debugOutFile ) {
290 fwrite( $this->debugOutFile, sprintf( "%05X %s %s (%s)\n", $cp, $weight, $char,
291 implode( ' ', array_map( [ UtfNormal\Utils::class, 'codepointToUtf8' ], $group ) ) ) );
292 }
293 }
294
295 print "Out of order: $numOutOfOrder / " . count( $headerChars ) . "\n";
296
297 global $IP;
298 $writer = new StaticArrayWriter();
299 file_put_contents(
300 "$IP/includes/collation/data/first-letters-root.php",
301 $writer->create( $headerChars, 'File created by generateCollationData.php' )
302 );
303 echo "first-letters-root: file written.\n";
304 }
305}
306
309 public $fileName;
311 public $callback;
315 public $xml;
317 public $blocks = [];
320
321 public function __construct( string $fileName ) {
322 $this->fileName = $fileName;
323 }
324
325 public function readChars( callable $callback ) {
326 $this->getBlocks();
327 $this->currentBlock = reset( $this->blocks );
328 $xml = $this->open();
329 $this->callback = $callback;
330
331 while ( $xml->name !== 'repertoire' && $xml->next() );
332
333 while ( $xml->read() ) {
334 if ( $xml->nodeType == XMLReader::ELEMENT ) {
335 if ( $xml->name === 'group' ) {
336 $this->groupAttrs = $this->readAttributes();
337 } elseif ( $xml->name === 'char' ) {
338 $this->handleChar();
339 }
340 } elseif ( $xml->nodeType === XMLReader::END_ELEMENT ) {
341 if ( $xml->name === 'group' ) {
342 $this->groupAttrs = [];
343 }
344 }
345 }
346 $xml->close();
347 }
348
349 protected function open(): XMLReader {
350 $this->xml = new XMLReader;
351 if ( !$this->xml->open( $this->fileName ) ) {
352 throw new RuntimeException( __METHOD__ . ": unable to open {$this->fileName}" );
353 }
354 while ( $this->xml->name !== 'ucd' && $this->xml->read() );
355 $this->xml->read();
356
357 return $this->xml;
358 }
359
365 protected function readAttributes() {
366 $attrs = [];
367 while ( $this->xml->moveToNextAttribute() ) {
368 $attrs[$this->xml->name] = $this->xml->value;
369 }
370
371 return $attrs;
372 }
373
374 protected function handleChar() {
375 $attrs = $this->readAttributes() + $this->groupAttrs;
376 if ( isset( $attrs['cp'] ) ) {
377 $first = $last = hexdec( $attrs['cp'] );
378 } else {
379 $first = hexdec( $attrs['first-cp'] );
380 $last = hexdec( $attrs['last-cp'] );
381 unset( $attrs['first-cp'] );
382 unset( $attrs['last-cp'] );
383 }
384
385 for ( $cp = $first; $cp <= $last; $cp++ ) {
386 $hexCp = sprintf( "%04X", $cp );
387 foreach ( [ 'na', 'na1' ] as $nameProp ) {
388 if ( isset( $attrs[$nameProp] ) ) {
389 $attrs[$nameProp] = str_replace( '#', $hexCp, $attrs[$nameProp] );
390 }
391 }
392
393 while ( $this->currentBlock ) {
394 if ( $cp < $this->currentBlock[0] ) {
395 break;
396 } elseif ( $cp <= $this->currentBlock[1] ) {
397 $attrs['block'] = key( $this->blocks );
398 break;
399 } else {
400 $this->currentBlock = next( $this->blocks );
401 }
402 }
403
404 $attrs['cp'] = $hexCp;
405 ( $this->callback )( $attrs );
406 }
407 }
408
409 public function getBlocks(): array {
410 if ( $this->blocks ) {
411 return $this->blocks;
412 }
413
414 $xml = $this->open();
415 while ( $xml->name !== 'blocks' && $xml->read() );
416
417 while ( $xml->read() ) {
418 if ( $xml->nodeType == XMLReader::ELEMENT ) {
419 if ( $xml->name === 'block' ) {
420 $attrs = $this->readAttributes();
421 $first = hexdec( $attrs['first-cp'] );
422 $last = hexdec( $attrs['last-cp'] );
423 $this->blocks[$attrs['name']] = [ $first, $last ];
424 }
425 }
426 }
427 $xml->close();
428
429 return $this->blocks;
430 }
431}
432
433// @codeCoverageIgnoreStart
434$maintClass = GenerateCollationData::class;
435require_once RUN_MAINTENANCE_IF_MAIN;
436// @codeCoverageIgnoreEnd
if(!defined('MEDIAWIKI')) if(!defined( 'MW_ENTRY_POINT')) global $IP
Environment checks.
Definition Setup.php:103
if(!defined('MW_SETUP_CALLBACK'))
Definition WebStart.php:82
Generate first letter data files for Collation.php.
__construct()
Default constructor.
string[] $mappedChars
A hashtable keyed by codepoint, where presence indicates that a character has a decomposition mapping...
execute()
Do the actual work.
int $weights
The primary weights, indexed by codepoint.
string $dataDir
The directory with source data files in it.
static isCjk( $codepoint)
Test if a code point is a CJK (Chinese, Japanese, Korean) character.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
__construct(string $fileName)
readChars(callable $callback)
readAttributes()
Read the attributes of the current element node and return them as an array.
Format a static PHP array to be written to a file.
A collection of static methods to play with strings.