Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
Total | |
77.94% |
53 / 68 |
|
66.67% |
6 / 9 |
CRAP | |
0.00% |
0 / 1 |
LanguageCode | |
79.10% |
53 / 67 |
|
66.67% |
6 / 9 |
32.17 | |
0.00% |
0 / 1 |
__construct | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
toString | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
getDeprecatedCodeMapping | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
getNonstandardLanguageCodeMapping | |
0.00% |
0 / 8 |
|
0.00% |
0 / 1 |
20 | |||
replaceDeprecatedCodes | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
bcp47 | |
100.00% |
14 / 14 |
|
100.00% |
1 / 1 |
9 | |||
bcp47ToInternal | |
80.00% |
8 / 10 |
|
0.00% |
0 / 1 |
5.20 | |||
normalizeNonstandardCodeAndWarn | |
0.00% |
0 / 4 |
|
0.00% |
0 / 1 |
6 | |||
isWellFormedLanguageTag | |
100.00% |
27 / 27 |
|
100.00% |
1 / 1 |
2 |
1 | <?php |
2 | /** |
3 | * This program is free software; you can redistribute it and/or modify |
4 | * it under the terms of the GNU General Public License as published by |
5 | * the Free Software Foundation; either version 2 of the License, or |
6 | * (at your option) any later version. |
7 | * |
8 | * This program is distributed in the hope that it will be useful, |
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
11 | * GNU General Public License for more details. |
12 | * |
13 | * You should have received a copy of the GNU General Public License along |
14 | * with this program; if not, write to the Free Software Foundation, Inc., |
15 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
16 | * http://www.gnu.org/copyleft/gpl.html |
17 | * |
18 | * @file |
19 | */ |
20 | |
21 | namespace MediaWiki\Language; |
22 | |
23 | use Wikimedia\Bcp47Code\Bcp47Code; |
24 | use Wikimedia\Bcp47Code\Bcp47CodeValue; |
25 | |
26 | /** |
27 | * Methods for dealing with language codes. |
28 | * |
29 | * @since 1.29 |
30 | * @ingroup Language |
31 | */ |
32 | class LanguageCode { |
33 | |
34 | private string $code; |
35 | |
36 | /** |
37 | * @param string $code |
38 | * @unstable |
39 | * @since 1.43 |
40 | */ |
41 | public function __construct( string $code ) { |
42 | $this->code = $code; |
43 | } |
44 | |
45 | /** |
46 | * @return string |
47 | * @since 1.43 |
48 | */ |
49 | public function toString(): string { |
50 | return $this->code; |
51 | } |
52 | |
53 | /** |
54 | * Mapping of deprecated language codes that were used in previous |
55 | * versions of MediaWiki to up-to-date, current language codes. |
56 | * These may or may not be valid BCP 47 codes; they are included here |
57 | * because MediaWiki renamed these particular codes at some point. |
58 | * |
59 | * @var array Mapping from deprecated MediaWiki-internal language code |
60 | * to replacement MediaWiki-internal language code. |
61 | * |
62 | * @see https://meta.wikimedia.org/wiki/Special_language_codes |
63 | * @phpcs-require-sorted-array |
64 | */ |
65 | private const DEPRECATED_LANGUAGE_CODE_MAPPING = [ |
66 | // Note that als is actually a valid ISO 639 code (Tosk Albanian), but it |
67 | // was previously used in MediaWiki for Alsatian, which comes under gsw |
68 | 'als' => 'gsw', // T25215 |
69 | 'bat-smg' => 'sgs', // T27522 |
70 | 'be-x-old' => 'be-tarask', // T11823 |
71 | 'fiu-vro' => 'vro', // T31186 |
72 | 'roa-rup' => 'rup', // T17988 |
73 | 'zh-classical' => 'lzh', // T30443 |
74 | 'zh-min-nan' => 'nan', // T30442 |
75 | 'zh-yue' => 'yue', // T30441 |
76 | ]; |
77 | |
78 | /** |
79 | * Mapping of non-standard language codes used in MediaWiki to |
80 | * standardized BCP 47 codes. These are not deprecated (yet?): |
81 | * IANA may eventually recognize the subtag, in which case the `-x-` |
82 | * infix could be removed, or else we could rename the code in |
83 | * MediaWiki, in which case they'd move up to the above mapping |
84 | * of deprecated codes. |
85 | * |
86 | * As a rule, we preserve all distinctions made by MediaWiki |
87 | * internally. For example, `de-formal` becomes `de-x-formal` |
88 | * instead of just `de` because MediaWiki distinguishes `de-formal` |
89 | * from `de` (for example, for interface translations). Similarly, |
90 | * BCP 47 indicates that `kk-Cyrl` SHOULD not be used because it |
91 | * "typically does not add information", but in our case MediaWiki |
92 | * LanguageConverter distinguishes `kk` (render content in a mix of |
93 | * Kurdish variants) from `kk-Cyrl` (convert content to be uniformly |
94 | * Cyrillic). As the BCP 47 requirement is a SHOULD not a MUST, |
95 | * `kk-Cyrl` is a valid code, although some validators may emit |
96 | * a warning note. |
97 | * |
98 | * @var array Mapping from nonstandard MediaWiki-internal codes to |
99 | * BCP 47 codes |
100 | * |
101 | * @see https://meta.wikimedia.org/wiki/Special_language_codes |
102 | * @see https://phabricator.wikimedia.org/T125073 |
103 | */ |
104 | private const NON_STANDARD_LANGUAGE_CODE_MAPPING = [ |
105 | // All codes returned by LanguageNameUtils::getLanguageNames() validated |
106 | // against IANA registry at |
107 | // https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry |
108 | // with help of validator at |
109 | // http://schneegans.de/lv/ |
110 | 'cbk-zam' => 'cbk', // T124657 |
111 | 'de-formal' => 'de-x-formal', |
112 | 'eml' => 'egl', // T36217 |
113 | 'en-rtl' => 'en-x-rtl', |
114 | 'es-formal' => 'es-x-formal', |
115 | 'hu-formal' => 'hu-x-formal', |
116 | 'map-bms' => 'jv-x-bms', // [[en:Banyumasan_dialect]] T125073 |
117 | 'mo' => 'ro-Cyrl-MD', // T125073 |
118 | 'nrm' => 'nrf', // [[en:Norman_language]] T25216 |
119 | 'nl-informal' => 'nl-x-informal', |
120 | 'roa-tara' => 'nap-x-tara', // [[en:Tarantino_dialect]] |
121 | 'simple' => 'en-simple', |
122 | 'sr-ec' => 'sr-Cyrl', // T117845 |
123 | 'sr-el' => 'sr-Latn', // T117845 |
124 | |
125 | // Although these next codes aren't *wrong* per se, including |
126 | // both the script and the country code helps compatibility with |
127 | // other BCP 47 users. Note that MW also uses |
128 | // `kk-Arab`/`kk-Cyrl`/`kk-Latn`, `zh-Hans`/`zh-Hant`, |
129 | // without a country code, and those should be left alone. |
130 | // `kk` has the Suppress-Script: Cyrl field, so `kk-KZ` won't be mapped |
131 | // to `kk-Cyrl-KZ`. |
132 | // (See getVariantsFallbacks() in KkConverter.php for Arab/Cyrl/Latn id.) |
133 | // (See getVariantsFallbacks() in ZhConverter.php for Hans/Hant id.) |
134 | 'crh-ro' => 'crh-Latn-RO', |
135 | 'kk-cn' => 'kk-Arab-CN', |
136 | 'kk-tr' => 'kk-Latn-TR', |
137 | 'zh-cn' => 'zh-Hans-CN', |
138 | 'zh-sg' => 'zh-Hans-SG', |
139 | 'zh-my' => 'zh-Hans-MY', |
140 | 'zh-tw' => 'zh-Hant-TW', |
141 | 'zh-hk' => 'zh-Hant-HK', |
142 | 'zh-mo' => 'zh-Hant-MO', |
143 | ]; |
144 | |
145 | /** |
146 | * Returns a mapping of deprecated language codes that were used in previous |
147 | * versions of MediaWiki to up-to-date, current language codes. |
148 | * |
149 | * This array is merged into $wgDummyLanguageCodes in |
150 | * SetupDynamicConfig.php, along with the fake language codes |
151 | * 'qqq' and 'qqx', which are used internally by MediaWiki's |
152 | * localisation system. |
153 | * |
154 | * @return string[] |
155 | * |
156 | * @since 1.29 |
157 | */ |
158 | public static function getDeprecatedCodeMapping() { |
159 | return self::DEPRECATED_LANGUAGE_CODE_MAPPING; |
160 | } |
161 | |
162 | /** |
163 | * Returns a mapping of non-standard language codes used by |
164 | * (current and previous version of) MediaWiki, mapped to standard |
165 | * BCP 47 names. |
166 | * |
167 | * This array is exported to JavaScript to ensure |
168 | * mediawiki.language.bcp47 stays in sync with LanguageCode::bcp47(). |
169 | * |
170 | * @return string[] |
171 | * |
172 | * @since 1.32 |
173 | */ |
174 | public static function getNonstandardLanguageCodeMapping() { |
175 | static $result = []; |
176 | if ( $result ) { |
177 | return $result; |
178 | } |
179 | foreach ( self::DEPRECATED_LANGUAGE_CODE_MAPPING as $code => $ignore ) { |
180 | $result[$code] = self::bcp47( $code ); |
181 | } |
182 | foreach ( self::NON_STANDARD_LANGUAGE_CODE_MAPPING as $code => $ignore ) { |
183 | $result[$code] = self::bcp47( $code ); |
184 | } |
185 | return $result; |
186 | } |
187 | |
188 | /** |
189 | * Replace deprecated language codes that were used in previous |
190 | * versions of MediaWiki to up-to-date, current language codes. |
191 | * Other values will be returned unchanged. |
192 | * |
193 | * @param string $code Old language code |
194 | * @return string New language code |
195 | * |
196 | * @since 1.30 |
197 | */ |
198 | public static function replaceDeprecatedCodes( $code ) { |
199 | return self::DEPRECATED_LANGUAGE_CODE_MAPPING[$code] ?? $code; |
200 | } |
201 | |
202 | /** |
203 | * Get the normalised IANA language tag |
204 | * See unit test for examples. |
205 | * See mediawiki.language.bcp47 for the JavaScript implementation. |
206 | * |
207 | * @param string $code The language code. |
208 | * @return string A language code complying with BCP 47 standards. |
209 | * |
210 | * @since 1.31 |
211 | */ |
212 | public static function bcp47( $code ) { |
213 | $code = self::replaceDeprecatedCodes( strtolower( $code ) ); |
214 | if ( isset( self::NON_STANDARD_LANGUAGE_CODE_MAPPING[$code] ) ) { |
215 | $code = self::NON_STANDARD_LANGUAGE_CODE_MAPPING[$code]; |
216 | } |
217 | $codeSegment = explode( '-', $code ); |
218 | $codeBCP = []; |
219 | foreach ( $codeSegment as $segNo => $seg ) { |
220 | // when the previous segment is x, it is a private segment and should be lc |
221 | if ( $segNo > 0 && strtolower( $codeSegment[( $segNo - 1 )] ) == 'x' ) { |
222 | $codeBCP[$segNo] = strtolower( $seg ); |
223 | // ISO 3166 country code |
224 | } elseif ( ( strlen( $seg ) == 2 ) && ( $segNo > 0 ) ) { |
225 | $codeBCP[$segNo] = strtoupper( $seg ); |
226 | // ISO 15924 script code |
227 | } elseif ( ( strlen( $seg ) == 4 ) && ( $segNo > 0 ) ) { |
228 | $codeBCP[$segNo] = ucfirst( strtolower( $seg ) ); |
229 | // Use lowercase for other cases |
230 | } else { |
231 | $codeBCP[$segNo] = strtolower( $seg ); |
232 | } |
233 | } |
234 | return implode( '-', $codeBCP ); |
235 | } |
236 | |
237 | /** |
238 | * Convert standardized BCP 47 codes to the internal names used |
239 | * by MediaWiki and returned by Language::getCode(). This function |
240 | * should be the inverse of LanguageCode::bcp47(). Note that BCP 47 |
241 | * explicitly states that language codes are case-insensitive. |
242 | * |
243 | * Since LanguageFactory::getLanguage() is pretty generous about |
244 | * accepting aliases (as long as they are lowercased), this function |
245 | * should be equivalent to: |
246 | * LanguageFactory::getLanguage(strtolower($code))->getCode() |
247 | * but (a) better describes the caller's intention, and (b) should |
248 | * be much more efficient in practice. |
249 | * |
250 | * @param string|Bcp47Code $code The standard BCP-47 language code |
251 | * @return string A MediaWiki-internal code, as returned, for example, by |
252 | * Language::getCode() |
253 | * @since 1.40 |
254 | */ |
255 | public static function bcp47ToInternal( $code ): string { |
256 | if ( $code instanceof Language ) { |
257 | return $code->getCode(); |
258 | } |
259 | if ( $code instanceof Bcp47Code ) { |
260 | $code = $code->toBcp47Code(); |
261 | } |
262 | static $invertedLookup = []; |
263 | if ( !$invertedLookup ) { |
264 | // There should never be two different entries in |
265 | // NON_STANDARD_LANGUAGE_CODE_MAPPING that map *different* |
266 | // internal codes to the same external BCP-47 code. That is, |
267 | // BCP-47 should preserve all the information from the internal |
268 | // code (discussed further above)[*]. But note the converse isn't |
269 | // true: multiple BCP-47 codes can alias to the same internal code: |
270 | // BCP-47 internal |
271 | // zh-Hans-CN => zh-cn (in NON_STANDARD_LANGUAGE_CODE_MAPPING) |
272 | // zh-Hans => zh-hans (not in " ) |
273 | // zh-CN => zh-cn (not in " ) |
274 | // |
275 | // [*] eml/egl are the "exception that proves the rule": `egl` *is* |
276 | // (prematurely?) defined as an internal code, but only |
277 | // eml.wikipedia.org exists, and it defines its language as `eml`; |
278 | // for internal purposes `egl` should map back into `eml` until |
279 | // `eml` is deprecated (aka an `eml => egl` entry is added to |
280 | // DEPRECATED_LANGUAGE_CODE_MAPPING): T36217. |
281 | foreach ( self::NON_STANDARD_LANGUAGE_CODE_MAPPING as $internal => $bcp47 ) { |
282 | $invertedLookup[strtolower( $bcp47 )] = $internal; |
283 | } |
284 | // We deliberately do *not* use DEPRECATED_LANGUAGE_CODE_MAPPING |
285 | // here: deprecated codes are no longer valid mediawiki internal |
286 | // codes, and we should never return them. |
287 | } |
288 | // Internal codes are all lowercase. This also achieves |
289 | // case-insensitivity in the lookup. |
290 | $code = strtolower( $code ); |
291 | return $invertedLookup[$code] ?? $code; |
292 | } |
293 | |
294 | /** |
295 | * We want to eventually require valid BCP-47 codes on HTTP and HTML |
296 | * APIs (where the standards require it). This will "prefer" to |
297 | * interpret the given $code as BCP-47, but if a mediawiki internal |
298 | * code is provided, it will map it to the proper BCP-47 code. We |
299 | * don't emit a logged warning on this path yet, but we intend to |
300 | * in the future. |
301 | * |
302 | * @param string $code A "language code" provided from an HTTP or HTML |
303 | * API, presumed to be BCP-47 |
304 | * @return Bcp47Code An "actual" BCP-47 code |
305 | * @internal |
306 | */ |
307 | public static function normalizeNonstandardCodeAndWarn( string $code ): Bcp47Code { |
308 | $compatMap = self::getNonstandardLanguageCodeMapping(); |
309 | if ( isset( $compatMap[strtolower( $code )] ) ) { |
310 | // Backward compatibility, since clients may have been |
311 | // sending us non-standards-compliant |
312 | // "mediawiki internal language codes"; eventually we'll |
313 | // emit a logged warning here. |
314 | $code = $compatMap[strtolower( $code )]; |
315 | } |
316 | return new Bcp47CodeValue( $code ); |
317 | } |
318 | |
319 | /** |
320 | * Returns true if a language code string is a well-formed language tag |
321 | * according to RFC 5646. |
322 | * This function only checks well-formedness; it doesn't check that |
323 | * language, script or variant codes actually exist in the repositories. |
324 | * |
325 | * Based on regexes by Mark Davis of the Unicode Consortium: |
326 | * https://github.com/unicode-org/icu/blob/37e295627156bc334e1f1e88807025fac984da0e/icu4j/main/tests/translit/src/com/ibm/icu/dev/test/translit/langtagRegex.txt |
327 | * |
328 | * @param string $code |
329 | * @param bool $lenient Whether to allow '_' as separator. The default is only '-'. |
330 | * |
331 | * @return bool |
332 | * @since 1.39 |
333 | */ |
334 | public static function isWellFormedLanguageTag( string $code, bool $lenient = false ): bool { |
335 | $alpha = '[a-z]'; |
336 | $digit = '[0-9]'; |
337 | $alphanum = '[a-z0-9]'; |
338 | $x = 'x'; # private use singleton |
339 | $singleton = '[a-wy-z]'; # other singleton |
340 | $s = $lenient ? '[-_]' : '-'; |
341 | |
342 | $language = "$alpha{2,8}|$alpha{2,3}$s$alpha{3}"; |
343 | $script = "$alpha{4}"; # ISO 15924 |
344 | $region = "(?:$alpha{2}|$digit{3})"; # ISO 3166-1 alpha-2 or UN M.49 |
345 | $variant = "(?:$alphanum{5,8}|$digit$alphanum{3})"; |
346 | $extension = "$singleton(?:$s$alphanum{2,8})+"; |
347 | $privateUse = "$x(?:$s$alphanum{1,8})+"; |
348 | |
349 | # Define certain legacy language tags (marked as “Type: grandfathered” in BCP 47), |
350 | # since otherwise the regex is pretty useless. |
351 | # Since these are limited, this is safe even later changes to the registry -- |
352 | # the only oddity is that it might change the type of the tag, and thus |
353 | # the results from the capturing groups. |
354 | # https://www.iana.org/assignments/language-subtag-registry |
355 | |
356 | $legacy = "en{$s}gb{$s}oed" |
357 | . "|i{$s}(?:ami|bnn|default|enochian|hak|klingon|lux|mingo|navajo|pwn|tao|tay|tsu)" |
358 | . "|no{$s}(?:bok|nyn)" |
359 | . "|sgn{$s}(?:be{$s}(?:fr|nl)|ch{$s}de)" |
360 | . "|zh{$s}min{$s}nan"; |
361 | |
362 | $variantList = "$variant(?:$s$variant)*"; |
363 | $extensionList = "$extension(?:$s$extension)*"; |
364 | |
365 | $langtag = "(?:($language)" |
366 | . "(?:$s$script)?" |
367 | . "(?:$s$region)?" |
368 | . "(?:$s$variantList)?" |
369 | . "(?:$s$extensionList)?" |
370 | . "(?:$s$privateUse)?)"; |
371 | |
372 | # Here is the final breakdown, with capturing groups for each of these components |
373 | # The variants, extensions, legacy, and private-use may have interior '-' |
374 | |
375 | $root = "^(?:$langtag|$privateUse|$legacy)$"; |
376 | |
377 | return preg_match( "/$root/i", $code ); |
378 | } |
379 | } |
380 | |
381 | /** @deprecated class alias since 1.43 */ |
382 | class_alias( LanguageCode::class, 'LanguageCode' ); |