59 parent::__construct();
60 $this->
addOption(
'data-dir',
'A directory on the local filesystem ' .
61 'containing allkeys.txt and ucd.all.grouped.xml from unicode.org',
63 $this->
addOption(
'debug-output',
'Filename for sending debug output to',
68 $this->dataDir = $this->
getOption(
'data-dir',
'.' );
70 $allkeysPresent = file_exists(
"{$this->dataDir}/allkeys.txt" );
71 $ucdallPresent = file_exists(
"{$this->dataDir}/ucd.all.grouped.xml" );
73 if ( !$allkeysPresent || !$ucdallPresent ) {
74 $icuVersion = INTL_ICU_VERSION;
75 $unicodeVersion = implode(
'.', array_slice( IntlChar::getUnicodeVersion(), 0, 3 ) );
79 if ( !$allkeysPresent ) {
80 $error .=
"Unable to find allkeys.txt. "
81 .
"Download it and specify its location with --data-dir=<DIR>. "
84 if ( !$ucdallPresent ) {
85 $error .=
"Unable to find ucd.all.grouped.xml. "
86 .
"Download it, unzip, and specify its location with --data-dir=<DIR>. "
90 $error .=
"You are using ICU $icuVersion, intended for Unicode $unicodeVersion. "
91 .
"Appropriate file(s) should be available at:\n";
93 $allkeysURL =
"https://www.unicode.org/Public/UCA/$unicodeVersion/allkeys.txt";
94 $ucdallURL =
"https://www.unicode.org/Public/$unicodeVersion/ucdxml/ucd.all.grouped.zip";
96 if ( !$allkeysPresent ) {
97 $error .=
"* $allkeysURL\n";
99 if ( !$ucdallPresent ) {
100 $error .=
"* $ucdallURL\n";
106 $debugOutFileName = $this->
getOption(
'debug-output' );
107 if ( $debugOutFileName ) {
108 $this->debugOutFile = fopen( $debugOutFileName,
'w' );
109 if ( !$this->debugOutFile ) {
110 $this->
fatalError(
"Unable to open debug output file for writing" );
114 $this->generateFirstChars();
117 private function loadUcd() {
118 $uxr =
new UcdXmlReader(
"{$this->dataDir}/ucd.all.grouped.xml" );
119 $uxr->readChars( [ $this,
'charCallback' ] );
122 private function charCallback( array $data ) {
126 $category = substr( $data[
'gc'], 0, 1 );
127 if ( strpos(
'LNPS', $category ) ===
false
128 && $data[
'cp'] !==
'0020'
132 $cp = hexdec( $data[
'cp'] );
142 if ( $data[
'block'] ==
'Hangul Syllables' ) {
147 if ( $data[
'UIdeo'] ===
'Y' ) {
148 if ( $data[
'block'] ==
'CJK Unified Ideographs'
149 || $data[
'block'] ==
'CJK Compatibility Ideographs'
158 $a = $base + ( $cp >> 15 );
159 $b = ( $cp & 0x7fff ) | 0x8000;
161 $this->weights[$cp] = sprintf(
".%04X.%04X", $a, $b );
163 if ( $data[
'dm'] !==
'#' ) {
164 $this->mappedChars[$cp] =
true;
167 if ( $cp % 4096 == 0 ) {
168 print
"{$data['cp']}\n";
172 private function generateFirstChars() {
173 $file = fopen(
"{$this->dataDir}/allkeys.txt",
'r' );
175 $this->
fatalError(
"Unable to open allkeys.txt" );
178 $goodTertiaryChars = [];
184 while ( ( $line = fgets( $file ) ) !==
false ) {
186 $line = trim( $line );
187 if ( !preg_match(
'/^([0-9A-F]+)\s*;\s*([^#]*)/', $line, $m ) ) {
191 $cp = hexdec( $m[1] );
192 $allWeights = trim( $m[2] );
196 if ( !isset( $this->weights[$cp] ) ) {
200 foreach ( StringUtils::explode(
'[', $allWeights ) as $weightStr ) {
201 if ( preg_match_all(
'/[*.]([0-9A-F]+)/', $weightStr, $m ) ) {
202 if ( $m[1][0] !==
'0000' ) {
203 $primary .=
'.' . $m[1][0];
205 if ( $m[1][2] !==
'0000' ) {
206 $tertiary .=
'.' . $m[1][2];
210 $this->weights[$cp] = $primary;
211 if ( $tertiary ===
'.0008'
212 || $tertiary ===
'.000E'
214 $goodTertiaryChars[$cp] =
true;
221 asort( $this->weights, SORT_STRING );
222 $prevWeight = reset( $this->weights );
224 foreach ( $this->weights as $cp => $weight ) {
225 if ( $weight !== $prevWeight ) {
226 $this->groups[$prevWeight] = $group;
227 $prevWeight = $weight;
228 $group = $this->groups[$weight] ?? [];
233 $this->groups[$prevWeight] = $group;
242 foreach ( $this->groups as $weight => $group ) {
243 if ( preg_match(
'/(\.[0-9A-F]*)\./', $weight, $m ) ) {
244 if ( isset( $this->groups[$m[1]] ) ) {
245 unset( $this->groups[$weight] );
250 ksort( $this->groups, SORT_STRING );
255 $tertiaryCollator =
new Collator(
'root' );
256 $primaryCollator =
new Collator(
'root' );
257 $primaryCollator->setStrength( Collator::PRIMARY );
259 foreach ( $this->groups as $weight => $group ) {
260 $uncomposedChars = [];
262 foreach ( $group as $cp ) {
263 if ( isset( $goodTertiaryChars[$cp] ) ) {
266 if ( !isset( $this->mappedChars[$cp] ) ) {
267 $uncomposedChars[] = $cp;
270 $x = array_intersect( $goodChars, $uncomposedChars );
272 $x = $uncomposedChars;
279 $tertiaryCollator->sort( $x );
282 $char = UtfNormal\Utils::codepointToUtf8( $cp );
283 $headerChars[] = $char;
284 if ( $primaryCollator->compare( $char, $prevChar ) <= 0 ) {
289 if ( $this->debugOutFile ) {
290 fwrite( $this->debugOutFile, sprintf(
"%05X %s %s (%s)\n", $cp, $weight, $char,
291 implode(
' ', array_map( [ UtfNormal\Utils::class,
'codepointToUtf8' ], $group ) ) ) );
295 print
"Out of order: $numOutOfOrder / " . count( $headerChars ) .
"\n";
300 "$IP/includes/collation/data/first-letters-root.php",
301 $writer->create( $headerChars,
'File created by generateCollationData.php' )
303 echo
"first-letters-root: file written.\n";
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.