Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
77.94% |
53 / 68 |
|
66.67% |
6 / 9 |
CRAP | |
0.00% |
0 / 1 |
| LanguageCode | |
79.10% |
53 / 67 |
|
66.67% |
6 / 9 |
32.17 | |
0.00% |
0 / 1 |
| __construct | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| toString | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| getDeprecatedCodeMapping | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| getNonstandardLanguageCodeMapping | |
0.00% |
0 / 8 |
|
0.00% |
0 / 1 |
20 | |||
| replaceDeprecatedCodes | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| bcp47 | |
100.00% |
14 / 14 |
|
100.00% |
1 / 1 |
9 | |||
| bcp47ToInternal | |
80.00% |
8 / 10 |
|
0.00% |
0 / 1 |
5.20 | |||
| normalizeNonstandardCodeAndWarn | |
0.00% |
0 / 4 |
|
0.00% |
0 / 1 |
6 | |||
| isWellFormedLanguageTag | |
100.00% |
27 / 27 |
|
100.00% |
1 / 1 |
2 | |||
| 1 | <?php |
| 2 | /** |
| 3 | * This program is free software; you can redistribute it and/or modify |
| 4 | * it under the terms of the GNU General Public License as published by |
| 5 | * the Free Software Foundation; either version 2 of the License, or |
| 6 | * (at your option) any later version. |
| 7 | * |
| 8 | * This program is distributed in the hope that it will be useful, |
| 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 11 | * GNU General Public License for more details. |
| 12 | * |
| 13 | * You should have received a copy of the GNU General Public License along |
| 14 | * with this program; if not, write to the Free Software Foundation, Inc., |
| 15 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
| 16 | * http://www.gnu.org/copyleft/gpl.html |
| 17 | * |
| 18 | * @file |
| 19 | */ |
| 20 | |
| 21 | namespace MediaWiki\Language; |
| 22 | |
| 23 | use Wikimedia\Bcp47Code\Bcp47Code; |
| 24 | use Wikimedia\Bcp47Code\Bcp47CodeValue; |
| 25 | |
| 26 | /** |
| 27 | * Methods for dealing with language codes. |
| 28 | * |
| 29 | * @since 1.29 |
| 30 | * @ingroup Language |
| 31 | */ |
| 32 | class LanguageCode { |
| 33 | |
| 34 | private string $code; |
| 35 | |
| 36 | /** |
| 37 | * @param string $code |
| 38 | * @unstable |
| 39 | * @since 1.43 |
| 40 | */ |
| 41 | public function __construct( string $code ) { |
| 42 | $this->code = $code; |
| 43 | } |
| 44 | |
| 45 | /** |
| 46 | * @return string |
| 47 | * @since 1.43 |
| 48 | */ |
| 49 | public function toString(): string { |
| 50 | return $this->code; |
| 51 | } |
| 52 | |
| 53 | /** |
| 54 | * Mapping of deprecated language codes that were used in previous |
| 55 | * versions of MediaWiki to up-to-date, current language codes. |
| 56 | * These may or may not be valid BCP 47 codes; they are included here |
| 57 | * because MediaWiki renamed these particular codes at some point. |
| 58 | * |
| 59 | * @var array Mapping from deprecated MediaWiki-internal language code |
| 60 | * to replacement MediaWiki-internal language code. |
| 61 | * |
| 62 | * @see https://meta.wikimedia.org/wiki/Special_language_codes |
| 63 | * @phpcs-require-sorted-array |
| 64 | */ |
| 65 | private const DEPRECATED_LANGUAGE_CODE_MAPPING = [ |
| 66 | // Note that als is actually a valid ISO 639 code (Tosk Albanian), but it |
| 67 | // was previously used in MediaWiki for Alsatian, which comes under gsw |
| 68 | 'als' => 'gsw', // T25215 |
| 69 | 'bat-smg' => 'sgs', // T27522 |
| 70 | 'be-x-old' => 'be-tarask', // T11823 |
| 71 | 'fiu-vro' => 'vro', // T31186 |
| 72 | 'roa-rup' => 'rup', // T17988 |
| 73 | 'zh-classical' => 'lzh', // T30443 |
| 74 | 'zh-min-nan' => 'nan', // T30442 |
| 75 | 'zh-yue' => 'yue', // T30441 |
| 76 | ]; |
| 77 | |
| 78 | /** |
| 79 | * Mapping of non-standard language codes used in MediaWiki to |
| 80 | * standardized BCP 47 codes. These are not deprecated (yet?): |
| 81 | * IANA may eventually recognize the subtag, in which case the `-x-` |
| 82 | * infix could be removed, or else we could rename the code in |
| 83 | * MediaWiki, in which case they'd move up to the above mapping |
| 84 | * of deprecated codes. |
| 85 | * |
| 86 | * As a rule, we preserve all distinctions made by MediaWiki |
| 87 | * internally. For example, `de-formal` becomes `de-x-formal` |
| 88 | * instead of just `de` because MediaWiki distinguishes `de-formal` |
| 89 | * from `de` (for example, for interface translations). Similarly, |
| 90 | * BCP 47 indicates that `kk-Cyrl` SHOULD not be used because it |
| 91 | * "typically does not add information", but in our case MediaWiki |
| 92 | * LanguageConverter distinguishes `kk` (render content in a mix of |
| 93 | * Kurdish variants) from `kk-Cyrl` (convert content to be uniformly |
| 94 | * Cyrillic). As the BCP 47 requirement is a SHOULD not a MUST, |
| 95 | * `kk-Cyrl` is a valid code, although some validators may emit |
| 96 | * a warning note. |
| 97 | * |
| 98 | * @var array Mapping from nonstandard MediaWiki-internal codes to |
| 99 | * BCP 47 codes |
| 100 | * |
| 101 | * @see https://meta.wikimedia.org/wiki/Special_language_codes |
| 102 | * @see https://phabricator.wikimedia.org/T125073 |
| 103 | */ |
| 104 | private const NON_STANDARD_LANGUAGE_CODE_MAPPING = [ |
| 105 | // All codes returned by LanguageNameUtils::getLanguageNames() validated |
| 106 | // against IANA registry at |
| 107 | // https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry |
| 108 | // with help of validator at |
| 109 | // http://schneegans.de/lv/ |
| 110 | 'cbk-zam' => 'cbk', // T124657 |
| 111 | 'de-formal' => 'de-x-formal', |
| 112 | 'eml' => 'egl', // T36217 |
| 113 | 'en-rtl' => 'en-x-rtl', |
| 114 | 'es-formal' => 'es-x-formal', |
| 115 | 'hu-formal' => 'hu-x-formal', |
| 116 | 'map-bms' => 'jv-x-bms', // [[en:Banyumasan_dialect]] T125073 |
| 117 | 'mo' => 'ro-Cyrl-MD', // T125073 |
| 118 | 'nrm' => 'nrf', // [[en:Norman_language]] T25216 |
| 119 | 'nl-informal' => 'nl-x-informal', |
| 120 | 'roa-tara' => 'nap-x-tara', // [[en:Tarantino_dialect]] |
| 121 | 'simple' => 'en-simple', |
| 122 | 'sr-ec' => 'sr-Cyrl', // T117845 |
| 123 | 'sr-el' => 'sr-Latn', // T117845 |
| 124 | |
| 125 | // Although these next codes aren't *wrong* per se, including |
| 126 | // both the script and the country code helps compatibility with |
| 127 | // other BCP 47 users. Note that MW also uses |
| 128 | // `kk-Arab`/`kk-Cyrl`/`kk-Latn`, `zh-Hans`/`zh-Hant`, |
| 129 | // without a country code, and those should be left alone. |
| 130 | // `kk` has the Suppress-Script: Cyrl field, so `kk-KZ` won't be mapped |
| 131 | // to `kk-Cyrl-KZ`. |
| 132 | // (See getVariantsFallbacks() in KkConverter.php for Arab/Cyrl/Latn id.) |
| 133 | // (See getVariantsFallbacks() in ZhConverter.php for Hans/Hant id.) |
| 134 | 'crh-ro' => 'crh-Latn-RO', |
| 135 | 'kk-cn' => 'kk-Arab-CN', |
| 136 | 'kk-tr' => 'kk-Latn-TR', |
| 137 | 'zh-cn' => 'zh-Hans-CN', |
| 138 | 'zh-sg' => 'zh-Hans-SG', |
| 139 | 'zh-my' => 'zh-Hans-MY', |
| 140 | 'zh-tw' => 'zh-Hant-TW', |
| 141 | 'zh-hk' => 'zh-Hant-HK', |
| 142 | 'zh-mo' => 'zh-Hant-MO', |
| 143 | ]; |
| 144 | |
| 145 | /** |
| 146 | * Returns a mapping of deprecated language codes that were used in previous |
| 147 | * versions of MediaWiki to up-to-date, current language codes. |
| 148 | * |
| 149 | * This array is merged into $wgDummyLanguageCodes in |
| 150 | * SetupDynamicConfig.php, along with the fake language codes |
| 151 | * 'qqq' and 'qqx', which are used internally by MediaWiki's |
| 152 | * localisation system. |
| 153 | * |
| 154 | * @return string[] |
| 155 | * |
| 156 | * @since 1.29 |
| 157 | */ |
| 158 | public static function getDeprecatedCodeMapping() { |
| 159 | return self::DEPRECATED_LANGUAGE_CODE_MAPPING; |
| 160 | } |
| 161 | |
| 162 | /** |
| 163 | * Returns a mapping of non-standard language codes used by |
| 164 | * (current and previous version of) MediaWiki, mapped to standard |
| 165 | * BCP 47 names. |
| 166 | * |
| 167 | * This array is exported to JavaScript to ensure |
| 168 | * mediawiki.language.bcp47 stays in sync with LanguageCode::bcp47(). |
| 169 | * |
| 170 | * @return string[] |
| 171 | * |
| 172 | * @since 1.32 |
| 173 | */ |
| 174 | public static function getNonstandardLanguageCodeMapping() { |
| 175 | static $result = []; |
| 176 | if ( $result ) { |
| 177 | return $result; |
| 178 | } |
| 179 | foreach ( self::DEPRECATED_LANGUAGE_CODE_MAPPING as $code => $ignore ) { |
| 180 | $result[$code] = self::bcp47( $code ); |
| 181 | } |
| 182 | foreach ( self::NON_STANDARD_LANGUAGE_CODE_MAPPING as $code => $ignore ) { |
| 183 | $result[$code] = self::bcp47( $code ); |
| 184 | } |
| 185 | return $result; |
| 186 | } |
| 187 | |
| 188 | /** |
| 189 | * Replace deprecated language codes that were used in previous |
| 190 | * versions of MediaWiki to up-to-date, current language codes. |
| 191 | * Other values will be returned unchanged. |
| 192 | * |
| 193 | * @param string $code Old language code |
| 194 | * @return string New language code |
| 195 | * |
| 196 | * @since 1.30 |
| 197 | */ |
| 198 | public static function replaceDeprecatedCodes( $code ) { |
| 199 | return self::DEPRECATED_LANGUAGE_CODE_MAPPING[$code] ?? $code; |
| 200 | } |
| 201 | |
| 202 | /** |
| 203 | * Get the normalised IANA language tag |
| 204 | * See unit test for examples. |
| 205 | * See mediawiki.language.bcp47 for the JavaScript implementation. |
| 206 | * |
| 207 | * @param string $code The language code. |
| 208 | * @return string A language code complying with BCP 47 standards. |
| 209 | * |
| 210 | * @since 1.31 |
| 211 | */ |
| 212 | public static function bcp47( $code ) { |
| 213 | $code = self::replaceDeprecatedCodes( strtolower( $code ) ); |
| 214 | if ( isset( self::NON_STANDARD_LANGUAGE_CODE_MAPPING[$code] ) ) { |
| 215 | $code = self::NON_STANDARD_LANGUAGE_CODE_MAPPING[$code]; |
| 216 | } |
| 217 | $codeSegment = explode( '-', $code ); |
| 218 | $codeBCP = []; |
| 219 | foreach ( $codeSegment as $segNo => $seg ) { |
| 220 | // when the previous segment is x, it is a private segment and should be lc |
| 221 | if ( $segNo > 0 && strtolower( $codeSegment[( $segNo - 1 )] ) == 'x' ) { |
| 222 | $codeBCP[$segNo] = strtolower( $seg ); |
| 223 | // ISO 3166 country code |
| 224 | } elseif ( ( strlen( $seg ) == 2 ) && ( $segNo > 0 ) ) { |
| 225 | $codeBCP[$segNo] = strtoupper( $seg ); |
| 226 | // ISO 15924 script code |
| 227 | } elseif ( ( strlen( $seg ) == 4 ) && ( $segNo > 0 ) ) { |
| 228 | $codeBCP[$segNo] = ucfirst( strtolower( $seg ) ); |
| 229 | // Use lowercase for other cases |
| 230 | } else { |
| 231 | $codeBCP[$segNo] = strtolower( $seg ); |
| 232 | } |
| 233 | } |
| 234 | return implode( '-', $codeBCP ); |
| 235 | } |
| 236 | |
| 237 | /** |
| 238 | * Convert standardized BCP 47 codes to the internal names used |
| 239 | * by MediaWiki and returned by Language::getCode(). This function |
| 240 | * should be the inverse of LanguageCode::bcp47(). Note that BCP 47 |
| 241 | * explicitly states that language codes are case-insensitive. |
| 242 | * |
| 243 | * Since LanguageFactory::getLanguage() is pretty generous about |
| 244 | * accepting aliases (as long as they are lowercased), this function |
| 245 | * should be equivalent to: |
| 246 | * LanguageFactory::getLanguage(strtolower($code))->getCode() |
| 247 | * but (a) better describes the caller's intention, and (b) should |
| 248 | * be much more efficient in practice. |
| 249 | * |
| 250 | * @param string|Bcp47Code $code The standard BCP-47 language code |
| 251 | * @return string A MediaWiki-internal code, as returned, for example, by |
| 252 | * Language::getCode() |
| 253 | * @since 1.40 |
| 254 | */ |
| 255 | public static function bcp47ToInternal( $code ): string { |
| 256 | if ( $code instanceof Language ) { |
| 257 | return $code->getCode(); |
| 258 | } |
| 259 | if ( $code instanceof Bcp47Code ) { |
| 260 | $code = $code->toBcp47Code(); |
| 261 | } |
| 262 | static $invertedLookup = []; |
| 263 | if ( !$invertedLookup ) { |
| 264 | // There should never be two different entries in |
| 265 | // NON_STANDARD_LANGUAGE_CODE_MAPPING that map *different* |
| 266 | // internal codes to the same external BCP-47 code. That is, |
| 267 | // BCP-47 should preserve all the information from the internal |
| 268 | // code (discussed further above)[*]. But note the converse isn't |
| 269 | // true: multiple BCP-47 codes can alias to the same internal code: |
| 270 | // BCP-47 internal |
| 271 | // zh-Hans-CN => zh-cn (in NON_STANDARD_LANGUAGE_CODE_MAPPING) |
| 272 | // zh-Hans => zh-hans (not in " ) |
| 273 | // zh-CN => zh-cn (not in " ) |
| 274 | // |
| 275 | // [*] eml/egl are the "exception that proves the rule": `egl` *is* |
| 276 | // (prematurely?) defined as an internal code, but only |
| 277 | // eml.wikipedia.org exists, and it defines its language as `eml`; |
| 278 | // for internal purposes `egl` should map back into `eml` until |
| 279 | // `eml` is deprecated (aka an `eml => egl` entry is added to |
| 280 | // DEPRECATED_LANGUAGE_CODE_MAPPING): T36217. |
| 281 | foreach ( self::NON_STANDARD_LANGUAGE_CODE_MAPPING as $internal => $bcp47 ) { |
| 282 | $invertedLookup[strtolower( $bcp47 )] = $internal; |
| 283 | } |
| 284 | // We deliberately do *not* use DEPRECATED_LANGUAGE_CODE_MAPPING |
| 285 | // here: deprecated codes are no longer valid mediawiki internal |
| 286 | // codes, and we should never return them. |
| 287 | } |
| 288 | // Internal codes are all lowercase. This also achieves |
| 289 | // case-insensitivity in the lookup. |
| 290 | $code = strtolower( $code ); |
| 291 | return $invertedLookup[$code] ?? $code; |
| 292 | } |
| 293 | |
| 294 | /** |
| 295 | * We want to eventually require valid BCP-47 codes on HTTP and HTML |
| 296 | * APIs (where the standards require it). This will "prefer" to |
| 297 | * interpret the given $code as BCP-47, but if a mediawiki internal |
| 298 | * code is provided, it will map it to the proper BCP-47 code. We |
| 299 | * don't emit a logged warning on this path yet, but we intend to |
| 300 | * in the future. |
| 301 | * |
| 302 | * @param string $code A "language code" provided from an HTTP or HTML |
| 303 | * API, presumed to be BCP-47 |
| 304 | * @return Bcp47Code An "actual" BCP-47 code |
| 305 | * @internal |
| 306 | */ |
| 307 | public static function normalizeNonstandardCodeAndWarn( string $code ): Bcp47Code { |
| 308 | $compatMap = self::getNonstandardLanguageCodeMapping(); |
| 309 | if ( isset( $compatMap[strtolower( $code )] ) ) { |
| 310 | // Backward compatibility, since clients may have been |
| 311 | // sending us non-standards-compliant |
| 312 | // "mediawiki internal language codes"; eventually we'll |
| 313 | // emit a logged warning here. |
| 314 | $code = $compatMap[strtolower( $code )]; |
| 315 | } |
| 316 | return new Bcp47CodeValue( $code ); |
| 317 | } |
| 318 | |
| 319 | /** |
| 320 | * Returns true if a language code string is a well-formed language tag |
| 321 | * according to RFC 5646. |
| 322 | * This function only checks well-formedness; it doesn't check that |
| 323 | * language, script or variant codes actually exist in the repositories. |
| 324 | * |
| 325 | * Based on regexes by Mark Davis of the Unicode Consortium: |
| 326 | * https://github.com/unicode-org/icu/blob/37e295627156bc334e1f1e88807025fac984da0e/icu4j/main/tests/translit/src/com/ibm/icu/dev/test/translit/langtagRegex.txt |
| 327 | * |
| 328 | * @param string $code |
| 329 | * @param bool $lenient Whether to allow '_' as separator. The default is only '-'. |
| 330 | * |
| 331 | * @return bool |
| 332 | * @since 1.39 |
| 333 | */ |
| 334 | public static function isWellFormedLanguageTag( string $code, bool $lenient = false ): bool { |
| 335 | $alpha = '[a-z]'; |
| 336 | $digit = '[0-9]'; |
| 337 | $alphanum = '[a-z0-9]'; |
| 338 | $x = 'x'; # private use singleton |
| 339 | $singleton = '[a-wy-z]'; # other singleton |
| 340 | $s = $lenient ? '[-_]' : '-'; |
| 341 | |
| 342 | $language = "$alpha{2,8}|$alpha{2,3}$s$alpha{3}"; |
| 343 | $script = "$alpha{4}"; # ISO 15924 |
| 344 | $region = "(?:$alpha{2}|$digit{3})"; # ISO 3166-1 alpha-2 or UN M.49 |
| 345 | $variant = "(?:$alphanum{5,8}|$digit$alphanum{3})"; |
| 346 | $extension = "$singleton(?:$s$alphanum{2,8})+"; |
| 347 | $privateUse = "$x(?:$s$alphanum{1,8})+"; |
| 348 | |
| 349 | # Define certain legacy language tags (marked as “Type: grandfathered” in BCP 47), |
| 350 | # since otherwise the regex is pretty useless. |
| 351 | # Since these are limited, this is safe even later changes to the registry -- |
| 352 | # the only oddity is that it might change the type of the tag, and thus |
| 353 | # the results from the capturing groups. |
| 354 | # https://www.iana.org/assignments/language-subtag-registry |
| 355 | |
| 356 | $legacy = "en{$s}gb{$s}oed" |
| 357 | . "|i{$s}(?:ami|bnn|default|enochian|hak|klingon|lux|mingo|navajo|pwn|tao|tay|tsu)" |
| 358 | . "|no{$s}(?:bok|nyn)" |
| 359 | . "|sgn{$s}(?:be{$s}(?:fr|nl)|ch{$s}de)" |
| 360 | . "|zh{$s}min{$s}nan"; |
| 361 | |
| 362 | $variantList = "$variant(?:$s$variant)*"; |
| 363 | $extensionList = "$extension(?:$s$extension)*"; |
| 364 | |
| 365 | $langtag = "(?:($language)" |
| 366 | . "(?:$s$script)?" |
| 367 | . "(?:$s$region)?" |
| 368 | . "(?:$s$variantList)?" |
| 369 | . "(?:$s$extensionList)?" |
| 370 | . "(?:$s$privateUse)?)"; |
| 371 | |
| 372 | # Here is the final breakdown, with capturing groups for each of these components |
| 373 | # The variants, extensions, legacy, and private-use may have interior '-' |
| 374 | |
| 375 | $root = "^(?:$langtag|$privateUse|$legacy)$"; |
| 376 | |
| 377 | return preg_match( "/$root/i", $code ); |
| 378 | } |
| 379 | } |
| 380 | |
| 381 | /** @deprecated class alias since 1.43 */ |
| 382 | class_alias( LanguageCode::class, 'LanguageCode' ); |