Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
Total | |
78.46% |
51 / 65 |
|
57.14% |
4 / 7 |
CRAP | |
0.00% |
0 / 1 |
LanguageCode | |
78.46% |
51 / 65 |
|
57.14% |
4 / 7 |
29.76 | |
0.00% |
0 / 1 |
getDeprecatedCodeMapping | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
getNonstandardLanguageCodeMapping | |
0.00% |
0 / 8 |
|
0.00% |
0 / 1 |
20 | |||
replaceDeprecatedCodes | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
bcp47 | |
100.00% |
14 / 14 |
|
100.00% |
1 / 1 |
9 | |||
bcp47ToInternal | |
80.00% |
8 / 10 |
|
0.00% |
0 / 1 |
5.20 | |||
normalizeNonstandardCodeAndWarn | |
0.00% |
0 / 4 |
|
0.00% |
0 / 1 |
6 | |||
isWellFormedLanguageTag | |
100.00% |
27 / 27 |
|
100.00% |
1 / 1 |
2 |
1 | <?php |
2 | /** |
3 | * This program is free software; you can redistribute it and/or modify |
4 | * it under the terms of the GNU General Public License as published by |
5 | * the Free Software Foundation; either version 2 of the License, or |
6 | * (at your option) any later version. |
7 | * |
8 | * This program is distributed in the hope that it will be useful, |
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
11 | * GNU General Public License for more details. |
12 | * |
13 | * You should have received a copy of the GNU General Public License along |
14 | * with this program; if not, write to the Free Software Foundation, Inc., |
15 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
16 | * http://www.gnu.org/copyleft/gpl.html |
17 | * |
18 | * @file |
19 | */ |
20 | |
21 | use Wikimedia\Bcp47Code\Bcp47Code; |
22 | use Wikimedia\Bcp47Code\Bcp47CodeValue; |
23 | |
24 | /** |
25 | * Methods for dealing with language codes. |
26 | * |
27 | * @since 1.29 |
28 | * @ingroup Language |
29 | */ |
30 | class LanguageCode { |
31 | /** |
32 | * Mapping of deprecated language codes that were used in previous |
33 | * versions of MediaWiki to up-to-date, current language codes. |
34 | * These may or may not be valid BCP 47 codes; they are included here |
35 | * because MediaWiki renamed these particular codes at some point. |
36 | * |
37 | * @var array Mapping from deprecated MediaWiki-internal language code |
38 | * to replacement MediaWiki-internal language code. |
39 | * |
40 | * @see https://meta.wikimedia.org/wiki/Special_language_codes |
41 | */ |
42 | private const DEPRECATED_LANGUAGE_CODE_MAPPING = [ |
43 | // Note that als is actually a valid ISO 639 code (Tosk Albanian), but it |
44 | // was previously used in MediaWiki for Alsatian, which comes under gsw |
45 | 'als' => 'gsw', // T25215 |
46 | 'bat-smg' => 'sgs', // T27522 |
47 | 'be-x-old' => 'be-tarask', // T11823 |
48 | 'fiu-vro' => 'vro', // T31186 |
49 | 'roa-rup' => 'rup', // T17988 |
50 | 'zh-classical' => 'lzh', // T30443 |
51 | 'zh-min-nan' => 'nan', // T30442 |
52 | 'zh-yue' => 'yue', // T30441 |
53 | ]; |
54 | |
55 | /** |
56 | * Mapping of non-standard language codes used in MediaWiki to |
57 | * standardized BCP 47 codes. These are not deprecated (yet?): |
58 | * IANA may eventually recognize the subtag, in which case the `-x-` |
59 | * infix could be removed, or else we could rename the code in |
60 | * MediaWiki, in which case they'd move up to the above mapping |
61 | * of deprecated codes. |
62 | * |
63 | * As a rule, we preserve all distinctions made by MediaWiki |
64 | * internally. For example, `de-formal` becomes `de-x-formal` |
65 | * instead of just `de` because MediaWiki distinguishes `de-formal` |
66 | * from `de` (for example, for interface translations). Similarly, |
67 | * BCP 47 indicates that `kk-Cyrl` SHOULD not be used because it |
68 | * "typically does not add information", but in our case MediaWiki |
69 | * LanguageConverter distinguishes `kk` (render content in a mix of |
70 | * Kurdish variants) from `kk-Cyrl` (convert content to be uniformly |
71 | * Cyrillic). As the BCP 47 requirement is a SHOULD not a MUST, |
72 | * `kk-Cyrl` is a valid code, although some validators may emit |
73 | * a warning note. |
74 | * |
75 | * @var array Mapping from nonstandard MediaWiki-internal codes to |
76 | * BCP 47 codes |
77 | * |
78 | * @see https://meta.wikimedia.org/wiki/Special_language_codes |
79 | * @see https://phabricator.wikimedia.org/T125073 |
80 | */ |
81 | private const NON_STANDARD_LANGUAGE_CODE_MAPPING = [ |
82 | // All codes returned by LanguageNameUtils::getLanguageNames() validated |
83 | // against IANA registry at |
84 | // https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry |
85 | // with help of validator at |
86 | // http://schneegans.de/lv/ |
87 | 'cbk-zam' => 'cbk', // T124657 |
88 | 'de-formal' => 'de-x-formal', |
89 | 'eml' => 'egl', // T36217 |
90 | 'en-rtl' => 'en-x-rtl', |
91 | 'es-formal' => 'es-x-formal', |
92 | 'hu-formal' => 'hu-x-formal', |
93 | 'map-bms' => 'jv-x-bms', // [[en:Banyumasan_dialect]] T125073 |
94 | 'mo' => 'ro-Cyrl-MD', // T125073 |
95 | 'nrm' => 'nrf', // [[en:Norman_language]] T25216 |
96 | 'nl-informal' => 'nl-x-informal', |
97 | 'roa-tara' => 'nap-x-tara', // [[en:Tarantino_dialect]] |
98 | 'simple' => 'en-simple', |
99 | 'sr-ec' => 'sr-Cyrl', // T117845 |
100 | 'sr-el' => 'sr-Latn', // T117845 |
101 | |
102 | // Although these next codes aren't *wrong* per se, including |
103 | // both the script and the country code helps compatibility with |
104 | // other BCP 47 users. Note that MW also uses |
105 | // `kk-Arab`/`kk-Cyrl`/`kk-Latn`, `zh-Hans`/`zh-Hant`, |
106 | // without a country code, and those should be left alone. |
107 | // `kk` has the Suppress-Script: Cyrl field, so `kk-KZ` won't be mapped |
108 | // to `kk-Cyrl-KZ`. |
109 | // (See getVariantsFallbacks() in KkConverter.php for Arab/Cyrl/Latn id.) |
110 | // (See getVariantsFallbacks() in ZhConverter.php for Hans/Hant id.) |
111 | 'crh-ro' => 'crh-Latn-RO', |
112 | 'kk-cn' => 'kk-Arab-CN', |
113 | 'kk-tr' => 'kk-Latn-TR', |
114 | 'zh-cn' => 'zh-Hans-CN', |
115 | 'zh-sg' => 'zh-Hans-SG', |
116 | 'zh-my' => 'zh-Hans-MY', |
117 | 'zh-tw' => 'zh-Hant-TW', |
118 | 'zh-hk' => 'zh-Hant-HK', |
119 | 'zh-mo' => 'zh-Hant-MO', |
120 | ]; |
121 | |
122 | /** |
123 | * Returns a mapping of deprecated language codes that were used in previous |
124 | * versions of MediaWiki to up-to-date, current language codes. |
125 | * |
126 | * This array is merged into $wgDummyLanguageCodes in Setup.php, along with |
127 | * the fake language codes 'qqq' and 'qqx', which are used internally by |
128 | * MediaWiki's localisation system. |
129 | * |
130 | * @return string[] |
131 | * |
132 | * @since 1.29 |
133 | */ |
134 | public static function getDeprecatedCodeMapping() { |
135 | return self::DEPRECATED_LANGUAGE_CODE_MAPPING; |
136 | } |
137 | |
138 | /** |
139 | * Returns a mapping of non-standard language codes used by |
140 | * (current and previous version of) MediaWiki, mapped to standard |
141 | * BCP 47 names. |
142 | * |
143 | * This array is exported to JavaScript to ensure |
144 | * mediawiki.language.bcp47 stays in sync with LanguageCode::bcp47(). |
145 | * |
146 | * @return string[] |
147 | * |
148 | * @since 1.32 |
149 | */ |
150 | public static function getNonstandardLanguageCodeMapping() { |
151 | static $result = []; |
152 | if ( $result ) { |
153 | return $result; |
154 | } |
155 | foreach ( self::DEPRECATED_LANGUAGE_CODE_MAPPING as $code => $ignore ) { |
156 | $result[$code] = self::bcp47( $code ); |
157 | } |
158 | foreach ( self::NON_STANDARD_LANGUAGE_CODE_MAPPING as $code => $ignore ) { |
159 | $result[$code] = self::bcp47( $code ); |
160 | } |
161 | return $result; |
162 | } |
163 | |
164 | /** |
165 | * Replace deprecated language codes that were used in previous |
166 | * versions of MediaWiki to up-to-date, current language codes. |
167 | * Other values will be returned unchanged. |
168 | * |
169 | * @param string $code Old language code |
170 | * @return string New language code |
171 | * |
172 | * @since 1.30 |
173 | */ |
174 | public static function replaceDeprecatedCodes( $code ) { |
175 | return self::DEPRECATED_LANGUAGE_CODE_MAPPING[$code] ?? $code; |
176 | } |
177 | |
178 | /** |
179 | * Get the normalised IANA language tag |
180 | * See unit test for examples. |
181 | * See mediawiki.language.bcp47 for the JavaScript implementation. |
182 | * |
183 | * @param string $code The language code. |
184 | * @return string A language code complying with BCP 47 standards. |
185 | * |
186 | * @since 1.31 |
187 | */ |
188 | public static function bcp47( $code ) { |
189 | $code = self::replaceDeprecatedCodes( strtolower( $code ) ); |
190 | if ( isset( self::NON_STANDARD_LANGUAGE_CODE_MAPPING[$code] ) ) { |
191 | $code = self::NON_STANDARD_LANGUAGE_CODE_MAPPING[$code]; |
192 | } |
193 | $codeSegment = explode( '-', $code ); |
194 | $codeBCP = []; |
195 | foreach ( $codeSegment as $segNo => $seg ) { |
196 | // when the previous segment is x, it is a private segment and should be lc |
197 | if ( $segNo > 0 && strtolower( $codeSegment[( $segNo - 1 )] ) == 'x' ) { |
198 | $codeBCP[$segNo] = strtolower( $seg ); |
199 | // ISO 3166 country code |
200 | } elseif ( ( strlen( $seg ) == 2 ) && ( $segNo > 0 ) ) { |
201 | $codeBCP[$segNo] = strtoupper( $seg ); |
202 | // ISO 15924 script code |
203 | } elseif ( ( strlen( $seg ) == 4 ) && ( $segNo > 0 ) ) { |
204 | $codeBCP[$segNo] = ucfirst( strtolower( $seg ) ); |
205 | // Use lowercase for other cases |
206 | } else { |
207 | $codeBCP[$segNo] = strtolower( $seg ); |
208 | } |
209 | } |
210 | return implode( '-', $codeBCP ); |
211 | } |
212 | |
213 | /** |
214 | * Convert standardized BCP 47 codes to the internal names used |
215 | * by MediaWiki and returned by Language::getCode(). This function |
216 | * should be the inverse of LanguageCode::bcp47(). Note that BCP 47 |
217 | * explicitly states that language codes are case-insensitive. |
218 | * |
219 | * Since LanguageFactory::getLanguage() is pretty generous about |
220 | * accepting aliases (as long as they are lowercased), this function |
221 | * should be equivalent to: |
222 | * LanguageFactory::getLanguage(strtolower($code))->getCode() |
223 | * but (a) better describes the caller's intention, and (b) should |
224 | * be much more efficient in practice. |
225 | * |
226 | * @param string|Bcp47Code $code The standard BCP-47 language code |
227 | * @return string A MediaWiki-internal code, as returned, for example, by |
228 | * Language::getCode() |
229 | * @since 1.40 |
230 | */ |
231 | public static function bcp47ToInternal( $code ): string { |
232 | if ( $code instanceof Language ) { |
233 | return $code->getCode(); |
234 | } |
235 | if ( $code instanceof Bcp47Code ) { |
236 | $code = $code->toBcp47Code(); |
237 | } |
238 | static $invertedLookup = []; |
239 | if ( !$invertedLookup ) { |
240 | // There should never be two different entries in |
241 | // NON_STANDARD_LANGUAGE_CODE_MAPPING that map *different* |
242 | // internal codes to the same external BCP-47 code. That is, |
243 | // BCP-47 should preserve all the information from the internal |
244 | // code (discussed further above)[*]. But note the converse isn't |
245 | // true: multiple BCP-47 codes can alias to the same internal code: |
246 | // BCP-47 internal |
247 | // zh-Hans-CN => zh-cn (in NON_STANDARD_LANGUAGE_CODE_MAPPING) |
248 | // zh-Hans => zh-hans (not in " ) |
249 | // zh-CN => zh-cn (not in " ) |
250 | // |
251 | // [*] eml/egl are the "exception that proves the rule": `egl` *is* |
252 | // (prematurely?) defined as an internal code, but only |
253 | // eml.wikipedia.org exists, and it defines its language as `eml`; |
254 | // for internal purposes `egl` should map back into `eml` until |
255 | // `eml` is deprecated (aka an `eml => egl` entry is added to |
256 | // DEPRECATED_LANGUAGE_CODE_MAPPING): T36217. |
257 | foreach ( self::NON_STANDARD_LANGUAGE_CODE_MAPPING as $internal => $bcp47 ) { |
258 | $invertedLookup[strtolower( $bcp47 )] = $internal; |
259 | } |
260 | // We deliberately do *not* use DEPRECATED_LANGUAGE_CODE_MAPPING |
261 | // here: deprecated codes are no longer valid mediawiki internal |
262 | // codes, and we should never return them. |
263 | } |
264 | // Internal codes are all lowercase. This also achieves |
265 | // case-insensitivity in the lookup. |
266 | $code = strtolower( $code ); |
267 | return $invertedLookup[$code] ?? $code; |
268 | } |
269 | |
270 | /** |
271 | * We want to eventually require valid BCP-47 codes on HTTP and HTML |
272 | * APIs (where the standards require it). This will "prefer" to |
273 | * interpret the given $code as BCP-47, but if a mediawiki internal |
274 | * code is provided, it will map it to the proper BCP-47 code. We |
275 | * don't emit a logged warning on this path yet, but we intend to |
276 | * in the future. |
277 | * |
278 | * @param string $code A "language code" provided from an HTTP or HTML |
279 | * API, presumed to be BCP-47 |
280 | * @return Bcp47Code An "actual" BCP-47 code |
281 | * @internal |
282 | */ |
283 | public static function normalizeNonstandardCodeAndWarn( string $code ): Bcp47Code { |
284 | $compatMap = self::getNonstandardLanguageCodeMapping(); |
285 | if ( isset( $compatMap[strtolower( $code )] ) ) { |
286 | // Backward compatibility, since clients may have been |
287 | // sending us non-standards-compliant |
288 | // "mediawiki internal language codes"; eventually we'll |
289 | // emit a logged warning here. |
290 | $code = $compatMap[strtolower( $code )]; |
291 | } |
292 | return new Bcp47CodeValue( $code ); |
293 | } |
294 | |
295 | /** |
296 | * Returns true if a language code string is a well-formed language tag |
297 | * according to RFC 5646. |
298 | * This function only checks well-formedness; it doesn't check that |
299 | * language, script or variant codes actually exist in the repositories. |
300 | * |
301 | * Based on regexes by Mark Davis of the Unicode Consortium: |
302 | * https://github.com/unicode-org/icu/blob/37e295627156bc334e1f1e88807025fac984da0e/icu4j/main/tests/translit/src/com/ibm/icu/dev/test/translit/langtagRegex.txt |
303 | * |
304 | * @param string $code |
305 | * @param bool $lenient Whether to allow '_' as separator. The default is only '-'. |
306 | * |
307 | * @return bool |
308 | * @since 1.39 |
309 | */ |
310 | public static function isWellFormedLanguageTag( string $code, bool $lenient = false ): bool { |
311 | $alpha = '[a-z]'; |
312 | $digit = '[0-9]'; |
313 | $alphanum = '[a-z0-9]'; |
314 | $x = 'x'; # private use singleton |
315 | $singleton = '[a-wy-z]'; # other singleton |
316 | $s = $lenient ? '[-_]' : '-'; |
317 | |
318 | $language = "$alpha{2,8}|$alpha{2,3}$s$alpha{3}"; |
319 | $script = "$alpha{4}"; # ISO 15924 |
320 | $region = "(?:$alpha{2}|$digit{3})"; # ISO 3166-1 alpha-2 or UN M.49 |
321 | $variant = "(?:$alphanum{5,8}|$digit$alphanum{3})"; |
322 | $extension = "$singleton(?:$s$alphanum{2,8})+"; |
323 | $privateUse = "$x(?:$s$alphanum{1,8})+"; |
324 | |
325 | # Define certain legacy language tags (marked as “Type: grandfathered” in BCP 47), |
326 | # since otherwise the regex is pretty useless. |
327 | # Since these are limited, this is safe even later changes to the registry -- |
328 | # the only oddity is that it might change the type of the tag, and thus |
329 | # the results from the capturing groups. |
330 | # https://www.iana.org/assignments/language-subtag-registry |
331 | |
332 | $legacy = "en{$s}gb{$s}oed" |
333 | . "|i{$s}(?:ami|bnn|default|enochian|hak|klingon|lux|mingo|navajo|pwn|tao|tay|tsu)" |
334 | . "|no{$s}(?:bok|nyn)" |
335 | . "|sgn{$s}(?:be{$s}(?:fr|nl)|ch{$s}de)" |
336 | . "|zh{$s}min{$s}nan"; |
337 | |
338 | $variantList = "$variant(?:$s$variant)*"; |
339 | $extensionList = "$extension(?:$s$extension)*"; |
340 | |
341 | $langtag = "(?:($language)" |
342 | . "(?:$s$script)?" |
343 | . "(?:$s$region)?" |
344 | . "(?:$s$variantList)?" |
345 | . "(?:$s$extensionList)?" |
346 | . "(?:$s$privateUse)?)"; |
347 | |
348 | # Here is the final breakdown, with capturing groups for each of these components |
349 | # The variants, extensions, legacy, and private-use may have interior '-' |
350 | |
351 | $root = "^(?:$langtag|$privateUse|$legacy)$"; |
352 | |
353 | return preg_match( "/$root/i", $code ); |
354 | } |
355 | } |