24 require_once __DIR__ .
'/../Maintenance.php';
50 parent::__construct();
51 $this->
addOption(
'data-dir',
'A directory on the local filesystem ' .
52 'containing allkeys.txt and ucd.all.grouped.xml from unicode.org',
54 $this->
addOption(
'debug-output',
'Filename for sending debug output to',
59 $this->dataDir = $this->
getOption(
'data-dir',
'.' );
61 $allkeysPresent = file_exists(
"{$this->dataDir}/allkeys.txt" );
62 $ucdallPresent = file_exists(
"{$this->dataDir}/ucd.all.grouped.xml" );
64 if ( !$allkeysPresent || !$ucdallPresent ) {
65 $icuVersion = INTL_ICU_VERSION;
66 $unicodeVersion = implode(
'.', array_slice( IntlChar::getUnicodeVersion(), 0, 3 ) );
70 if ( !$allkeysPresent ) {
71 $error .=
"Unable to find allkeys.txt. "
72 .
"Download it and specify its location with --data-dir=<DIR>. "
75 if ( !$ucdallPresent ) {
76 $error .=
"Unable to find ucd.all.grouped.xml. "
77 .
"Download it, unzip, and specify its location with --data-dir=<DIR>. "
81 $error .=
"You are using ICU $icuVersion, intended for Unicode $unicodeVersion. "
82 .
"Appropriate file(s) should be available at:\n";
84 $allkeysURL =
"https://www.unicode.org/Public/UCA/$unicodeVersion/allkeys.txt";
85 $ucdallURL =
"https://www.unicode.org/Public/$unicodeVersion/ucdxml/ucd.all.grouped.zip";
87 if ( !$allkeysPresent ) {
88 $error .=
"* $allkeysURL\n";
90 if ( !$ucdallPresent ) {
91 $error .=
"* $ucdallURL\n";
97 $debugOutFileName = $this->
getOption(
'debug-output' );
98 if ( $debugOutFileName ) {
99 $this->debugOutFile = fopen( $debugOutFileName,
'w' );
100 if ( !$this->debugOutFile ) {
101 $this->
fatalError(
"Unable to open debug output file for writing" );
105 $this->generateFirstChars();
108 private function loadUcd() {
109 $uxr =
new UcdXmlReader(
"{$this->dataDir}/ucd.all.grouped.xml" );
110 $uxr->readChars( [ $this,
'charCallback' ] );
113 private function charCallback( $data ) {
117 $category = substr( $data[
'gc'], 0, 1 );
118 if ( strpos(
'LNPS', $category ) ===
false
119 && $data[
'cp'] !==
'0020'
123 $cp = hexdec( $data[
'cp'] );
133 if ( $data[
'block'] ==
'Hangul Syllables' ) {
138 if ( $data[
'UIdeo'] ===
'Y' ) {
139 if ( $data[
'block'] ==
'CJK Unified Ideographs'
140 || $data[
'block'] ==
'CJK Compatibility Ideographs'
149 $a =
$base + ( $cp >> 15 );
150 $b = ( $cp & 0x7fff ) | 0x8000;
152 $this->weights[$cp] = sprintf(
".%04X.%04X", $a, $b );
154 if ( $data[
'dm'] !==
'#' ) {
155 $this->mappedChars[$cp] =
true;
158 if ( $cp % 4096 == 0 ) {
159 print
"{$data['cp']}\n";
163 private function generateFirstChars() {
164 $file = fopen(
"{$this->dataDir}/allkeys.txt",
'r' );
166 $this->
fatalError(
"Unable to open allkeys.txt" );
169 $goodTertiaryChars = [];
174 while ( ( $line = fgets(
$file ) ) !==
false ) {
176 $line = trim( $line );
177 if ( !preg_match(
'/^([0-9A-F]+)\s*;\s*([^#]*)/', $line, $m ) ) {
181 $cp = hexdec( $m[1] );
182 $allWeights = trim( $m[2] );
186 if ( !isset( $this->weights[$cp] ) ) {
191 if ( preg_match_all(
'/[*.]([0-9A-F]+)/', $weightStr, $m ) ) {
192 if ( $m[1][0] !==
'0000' ) {
193 $primary .=
'.' . $m[1][0];
195 if ( $m[1][2] !==
'0000' ) {
196 $tertiary .=
'.' . $m[1][2];
200 $this->weights[$cp] = $primary;
201 if ( $tertiary ===
'.0008'
202 || $tertiary ===
'.000E'
204 $goodTertiaryChars[$cp] =
true;
211 asort( $this->weights, SORT_STRING );
212 $prevWeight = reset( $this->weights );
214 foreach ( $this->weights as $cp => $weight ) {
215 if ( $weight !== $prevWeight ) {
216 $this->groups[$prevWeight] = $group;
217 $prevWeight = $weight;
218 $group = $this->groups[$weight] ?? [];
223 $this->groups[$prevWeight] = $group;
232 foreach ( $this->groups as $weight => $group ) {
233 if ( preg_match(
'/(\.[0-9A-F]*)\./', $weight, $m ) ) {
234 if ( isset( $this->groups[$m[1]] ) ) {
235 unset( $this->groups[$weight] );
240 ksort( $this->groups, SORT_STRING );
245 $tertiaryCollator =
new Collator(
'root' );
246 $primaryCollator =
new Collator(
'root' );
247 $primaryCollator->setStrength( Collator::PRIMARY );
249 foreach ( $this->groups as $weight => $group ) {
250 $uncomposedChars = [];
252 foreach ( $group as $cp ) {
253 if ( isset( $goodTertiaryChars[$cp] ) ) {
256 if ( !isset( $this->mappedChars[$cp] ) ) {
257 $uncomposedChars[] = $cp;
260 $x = array_intersect( $goodChars, $uncomposedChars );
262 $x = $uncomposedChars;
269 $tertiaryCollator->sort( $x );
272 $char = UtfNormal\Utils::codepointToUtf8( $cp );
273 $headerChars[] = $char;
274 if ( $primaryCollator->compare( $char, $prevChar ) <= 0 ) {
279 if ( $this->debugOutFile ) {
280 fwrite( $this->debugOutFile, sprintf(
"%05X %s %s (%s)\n", $cp, $weight, $char,
281 implode(
' ', array_map( [ UtfNormal\Utils::class,
'codepointToUtf8' ], $group ) ) ) );
285 print
"Out of order: $numOutOfOrder / " . count( $headerChars ) .
"\n";
290 "$IP/includes/collation/data/first-letters-root.php",
291 $writer->create( $headerChars,
'File created by generateCollationData.php' )
293 echo
"first-letters-root: file written.\n";
311 $this->currentBlock = reset( $this->blocks );
315 while (
$xml->name !==
'repertoire' &&
$xml->next() );
317 while (
$xml->read() ) {
318 if (
$xml->nodeType == XMLReader::ELEMENT ) {
319 if (
$xml->name ===
'group' ) {
321 } elseif (
$xml->name ===
'char' ) {
324 } elseif (
$xml->nodeType === XMLReader::END_ELEMENT ) {
325 if (
$xml->name ===
'group' ) {
326 $this->groupAttrs = [];
334 $this->xml =
new XMLReader;
335 $this->xml->open( $this->fileName );
337 throw new MWException( __METHOD__ .
": unable to open {$this->fileName}" );
339 while ( $this->xml->name !==
'ucd' && $this->xml->read() );
352 while ( $this->xml->moveToNextAttribute() ) {
353 $attrs[$this->xml->name] = $this->xml->value;
361 if ( isset( $attrs[
'cp'] ) ) {
362 $first = $last = hexdec( $attrs[
'cp'] );
364 $first = hexdec( $attrs[
'first-cp'] );
365 $last = hexdec( $attrs[
'last-cp'] );
366 unset( $attrs[
'first-cp'] );
367 unset( $attrs[
'last-cp'] );
370 for ( $cp = $first; $cp <= $last; $cp++ ) {
371 $hexCp = sprintf(
"%04X", $cp );
372 foreach ( [
'na',
'na1' ] as $nameProp ) {
373 if ( isset( $attrs[$nameProp] ) ) {
374 $attrs[$nameProp] = str_replace(
'#', $hexCp, $attrs[$nameProp] );
378 while ( $this->currentBlock ) {
379 if ( $cp < $this->currentBlock[0] ) {
381 } elseif ( $cp <= $this->currentBlock[1] ) {
382 $attrs[
'block'] = key( $this->blocks );
385 $this->currentBlock = next( $this->blocks );
389 $attrs[
'cp'] = $hexCp;
390 call_user_func( $this->callback, $attrs );
395 if ( $this->blocks ) {
400 while (
$xml->name !==
'blocks' &&
$xml->read() );
402 while (
$xml->read() ) {
403 if (
$xml->nodeType == XMLReader::ELEMENT ) {
404 if (
$xml->name ===
'block' ) {
406 $first = hexdec( $attrs[
'first-cp'] );
407 $last = hexdec( $attrs[
'last-cp'] );
408 $this->blocks[$attrs[
'name']] = [ $first, $last ];
419 require_once RUN_MAINTENANCE_IF_MAIN;
if(!defined( 'MEDIAWIKI')) if(ini_get( 'mbstring.func_overload')) if(!defined( 'MW_ENTRY_POINT')) global $IP
Environment checks.
Generate first letter data files for Collation.php.
__construct()
Default constructor.
$weights
The primary weights, indexed by codepoint.
execute()
Do the actual work.
$dataDir
The directory with source data files in it.
$mappedChars
A hashtable keyed by codepoint, where presence indicates that a character has a decomposition mapping...
static isCjk( $codepoint)
Test if a code point is a CJK (Chinese, Japanese, Korean) character.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
static explode( $separator, $subject)
Workalike for explode() with limited memory usage.
readAttributes()
Read the attributes of the current element node and return them as an array.
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.