52 parent::__construct();
53 $this->
addOption(
'data-dir',
'A directory on the local filesystem ' .
54 'containing allkeys.txt and ucd.all.grouped.xml from unicode.org',
56 $this->
addOption(
'debug-output',
'Filename for sending debug output to',
61 $this->dataDir = $this->
getOption(
'data-dir',
'.' );
63 $allkeysPresent = file_exists(
"{$this->dataDir}/allkeys.txt" );
64 $ucdallPresent = file_exists(
"{$this->dataDir}/ucd.all.grouped.xml" );
66 if ( !$allkeysPresent || !$ucdallPresent ) {
67 $icuVersion = INTL_ICU_VERSION;
68 $unicodeVersion = implode(
'.', array_slice( IntlChar::getUnicodeVersion(), 0, 3 ) );
72 if ( !$allkeysPresent ) {
73 $error .=
"Unable to find allkeys.txt. "
74 .
"Download it and specify its location with --data-dir=<DIR>. "
77 if ( !$ucdallPresent ) {
78 $error .=
"Unable to find ucd.all.grouped.xml. "
79 .
"Download it, unzip, and specify its location with --data-dir=<DIR>. "
83 $error .=
"You are using ICU $icuVersion, intended for Unicode $unicodeVersion. "
84 .
"Appropriate file(s) should be available at:\n";
86 $allkeysURL =
"https://www.unicode.org/Public/UCA/$unicodeVersion/allkeys.txt";
87 $ucdallURL =
"https://www.unicode.org/Public/$unicodeVersion/ucdxml/ucd.all.grouped.zip";
89 if ( !$allkeysPresent ) {
90 $error .=
"* $allkeysURL\n";
92 if ( !$ucdallPresent ) {
93 $error .=
"* $ucdallURL\n";
99 $debugOutFileName = $this->
getOption(
'debug-output' );
100 if ( $debugOutFileName ) {
101 $this->debugOutFile = fopen( $debugOutFileName,
'w' );
102 if ( !$this->debugOutFile ) {
103 $this->
fatalError(
"Unable to open debug output file for writing" );
107 $this->generateFirstChars();
110 private function loadUcd() {
111 $uxr =
new UcdXmlReader(
"{$this->dataDir}/ucd.all.grouped.xml" );
112 $uxr->readChars( [ $this,
'charCallback' ] );
115 private function charCallback( $data ) {
119 $category = substr( $data[
'gc'], 0, 1 );
120 if ( strpos(
'LNPS', $category ) ===
false
121 && $data[
'cp'] !==
'0020'
125 $cp = hexdec( $data[
'cp'] );
135 if ( $data[
'block'] ==
'Hangul Syllables' ) {
140 if ( $data[
'UIdeo'] ===
'Y' ) {
141 if ( $data[
'block'] ==
'CJK Unified Ideographs'
142 || $data[
'block'] ==
'CJK Compatibility Ideographs'
151 $a = $base + ( $cp >> 15 );
152 $b = ( $cp & 0x7fff ) | 0x8000;
154 $this->weights[$cp] = sprintf(
".%04X.%04X", $a, $b );
156 if ( $data[
'dm'] !==
'#' ) {
157 $this->mappedChars[$cp] =
true;
160 if ( $cp % 4096 == 0 ) {
161 print
"{$data['cp']}\n";
165 private function generateFirstChars() {
166 $file = fopen(
"{$this->dataDir}/allkeys.txt",
'r' );
168 $this->
fatalError(
"Unable to open allkeys.txt" );
171 $goodTertiaryChars = [];
176 while ( ( $line = fgets( $file ) ) !==
false ) {
178 $line = trim( $line );
179 if ( !preg_match(
'/^([0-9A-F]+)\s*;\s*([^#]*)/', $line, $m ) ) {
183 $cp = hexdec( $m[1] );
184 $allWeights = trim( $m[2] );
188 if ( !isset( $this->weights[$cp] ) ) {
192 foreach ( StringUtils::explode(
'[', $allWeights ) as $weightStr ) {
193 if ( preg_match_all(
'/[*.]([0-9A-F]+)/', $weightStr, $m ) ) {
194 if ( $m[1][0] !==
'0000' ) {
195 $primary .=
'.' . $m[1][0];
197 if ( $m[1][2] !==
'0000' ) {
198 $tertiary .=
'.' . $m[1][2];
202 $this->weights[$cp] = $primary;
203 if ( $tertiary ===
'.0008'
204 || $tertiary ===
'.000E'
206 $goodTertiaryChars[$cp] =
true;
213 asort( $this->weights, SORT_STRING );
214 $prevWeight = reset( $this->weights );
216 foreach ( $this->weights as $cp => $weight ) {
217 if ( $weight !== $prevWeight ) {
218 $this->groups[$prevWeight] = $group;
219 $prevWeight = $weight;
220 $group = $this->groups[$weight] ?? [];
225 $this->groups[$prevWeight] = $group;
234 foreach ( $this->groups as $weight => $group ) {
235 if ( preg_match(
'/(\.[0-9A-F]*)\./', $weight, $m ) ) {
236 if ( isset( $this->groups[$m[1]] ) ) {
237 unset( $this->groups[$weight] );
242 ksort( $this->groups, SORT_STRING );
247 $tertiaryCollator =
new Collator(
'root' );
248 $primaryCollator =
new Collator(
'root' );
249 $primaryCollator->setStrength( Collator::PRIMARY );
251 foreach ( $this->groups as $weight => $group ) {
252 $uncomposedChars = [];
254 foreach ( $group as $cp ) {
255 if ( isset( $goodTertiaryChars[$cp] ) ) {
258 if ( !isset( $this->mappedChars[$cp] ) ) {
259 $uncomposedChars[] = $cp;
262 $x = array_intersect( $goodChars, $uncomposedChars );
264 $x = $uncomposedChars;
271 $tertiaryCollator->sort( $x );
274 $char = UtfNormal\Utils::codepointToUtf8( $cp );
275 $headerChars[] = $char;
276 if ( $primaryCollator->compare( $char, $prevChar ) <= 0 ) {
281 if ( $this->debugOutFile ) {
282 fwrite( $this->debugOutFile, sprintf(
"%05X %s %s (%s)\n", $cp, $weight, $char,
283 implode(
' ', array_map( [ UtfNormal\Utils::class,
'codepointToUtf8' ], $group ) ) ) );
287 print
"Out of order: $numOutOfOrder / " . count( $headerChars ) .
"\n";
292 "$IP/includes/collation/data/first-letters-root.php",
293 $writer->create( $headerChars,
'File created by generateCollationData.php' )
295 echo
"first-letters-root: file written.\n";
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.