Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
Total | |
0.00% |
0 / 275 |
|
0.00% |
0 / 7 |
CRAP | |
0.00% |
0 / 1 |
ExportTranslationsMaintenanceScript | |
0.00% |
0 / 275 |
|
0.00% |
0 / 7 |
2652 | |
0.00% |
0 / 1 |
__construct | |
0.00% |
0 / 80 |
|
0.00% |
0 / 1 |
2 | |||
execute | |
0.00% |
0 / 138 |
|
0.00% |
0 / 1 |
756 | |||
getMessageGroups | |
0.00% |
0 / 17 |
|
0.00% |
0 / 1 |
56 | |||
getLanguageExportActions | |
0.00% |
0 / 18 |
|
0.00% |
0 / 1 |
90 | |||
canGroupBeExported | |
0.00% |
0 / 10 |
|
0.00% |
0 / 1 |
20 | |||
csv2array | |
0.00% |
0 / 6 |
|
0.00% |
0 / 1 |
2 | |||
parseLanguageCodes | |
0.00% |
0 / 6 |
|
0.00% |
0 / 1 |
6 |
1 | <?php |
2 | |
3 | namespace MediaWiki\Extension\Translate\Synchronization; |
4 | |
5 | use FileBasedMessageGroup; |
6 | use MediaWiki\Extension\Translate\FileFormatSupport\GettextFormat; |
7 | use MediaWiki\Extension\Translate\MessageGroupProcessing\MessageGroups; |
8 | use MediaWiki\Extension\Translate\Services; |
9 | use MediaWiki\Extension\Translate\Utilities\BaseMaintenanceScript; |
10 | use MediaWiki\Logger\LoggerFactory; |
11 | use MediaWiki\MediaWikiServices; |
12 | use MessageGroup; |
13 | use MessageGroupStats; |
14 | |
15 | /** |
16 | * Script to export translations of message groups to files. |
17 | * |
18 | * @author Niklas Laxström |
19 | * @author Siebrand Mazeland |
20 | * @copyright Copyright © 2008-2013, Niklas Laxström, Siebrand Mazeland |
21 | * @license GPL-2.0-or-later |
22 | */ |
23 | class ExportTranslationsMaintenanceScript extends BaseMaintenanceScript { |
24 | /// The translation file should be deleted if it exists |
25 | private const ACTION_DELETE = 'delete'; |
26 | /// The translation file should be created or updated |
27 | private const ACTION_CREATE = 'create'; |
28 | /// The translation file should be updated if exists, but not created as a new |
29 | private const ACTION_UPDATE = 'update'; |
30 | |
31 | public function __construct() { |
32 | parent::__construct(); |
33 | $this->addDescription( 'Export translations to files.' ); |
34 | |
35 | $this->addOption( |
36 | 'group', |
37 | 'Comma separated list of message group IDs (supports * wildcard) to export', |
38 | self::REQUIRED, |
39 | self::HAS_ARG |
40 | ); |
41 | $this->addOption( |
42 | 'lang', |
43 | 'Comma separated list of language codes to export or * for all languages', |
44 | self::REQUIRED, |
45 | self::HAS_ARG |
46 | ); |
47 | $this->addOption( |
48 | 'always-export-languages', |
49 | '(optional) Comma separated list of languages to export ignoring export threshold', |
50 | self::OPTIONAL, |
51 | self::HAS_ARG |
52 | ); |
53 | $this->addOption( |
54 | 'never-export-languages', |
55 | '(optional) Comma separated list of languages to never export (overrides everything else)', |
56 | self::OPTIONAL, |
57 | self::HAS_ARG |
58 | ); |
59 | $this->addOption( |
60 | 'skip-source-language', |
61 | '(optional) Do not export the source language of each message group', |
62 | self::OPTIONAL, |
63 | self::NO_ARG |
64 | ); |
65 | $this->addOption( |
66 | 'target', |
67 | 'Target directory for exported files', |
68 | self::REQUIRED, |
69 | self::HAS_ARG |
70 | ); |
71 | $this->addOption( |
72 | 'skip', |
73 | '(deprecated) See --never-export-languages', |
74 | self::OPTIONAL, |
75 | self::HAS_ARG |
76 | ); |
77 | $this->addOption( |
78 | 'skipgroup', |
79 | '(optional) Comma separated list of message group IDs (supports * wildcard) to not export', |
80 | self::OPTIONAL, |
81 | self::HAS_ARG |
82 | ); |
83 | $this->addOption( |
84 | 'threshold', |
85 | '(optional) Threshold for translation completion percentage that must be exceeded for initial export', |
86 | self::OPTIONAL, |
87 | self::HAS_ARG |
88 | ); |
89 | $this->addOption( |
90 | 'removal-threshold', |
91 | '(optional) Threshold for translation completion percentage that must be exceeded to keep the file', |
92 | self::OPTIONAL, |
93 | self::HAS_ARG |
94 | ); |
95 | $this->addOption( |
96 | 'no-fuzzy', |
97 | '(optional) Do not include any messages marked as fuzzy/outdated' |
98 | ); |
99 | $this->addOption( |
100 | 'offline-gettext-format', |
101 | '(optional) Export languages in offline Gettext format. Give a file pattern with ' |
102 | . '%GROUPID% and %CODE%. Empty pattern defaults to %GROUPID%/%CODE%.po.', |
103 | self::OPTIONAL, |
104 | self::HAS_ARG |
105 | ); |
106 | $this->addOption( |
107 | 'skip-group-sync-check', |
108 | '(optional) Skip exporting group if synchronization is still in progress or if there ' . |
109 | 'was an error during synchronization. See: ' . |
110 | 'https://www.mediawiki.org/wiki/Help:Extension:Translate/Group_management#Strong_synchronization' |
111 | ); |
112 | |
113 | $this->requireExtension( 'Translate' ); |
114 | } |
115 | |
116 | public function execute() { |
117 | $logger = LoggerFactory::getInstance( 'Translate.GroupSynchronization' ); |
118 | $groupPattern = $this->getOption( 'group' ) ?? ''; |
119 | $groupSkipPattern = $this->getOption( 'skipgroup' ) ?? ''; |
120 | $skipGroupSyncCheck = $this->hasOption( 'skip-group-sync-check' ); |
121 | |
122 | $logger->info( |
123 | 'Starting exports for groups {groups}', |
124 | [ 'groups' => $groupPattern ] |
125 | ); |
126 | $exportStartTime = microtime( true ); |
127 | |
128 | $target = $this->getOption( 'target' ); |
129 | if ( !is_writable( $target ) ) { |
130 | $this->fatalError( "Target directory is not writable ($target)." ); |
131 | } |
132 | |
133 | $exportThreshold = $this->getOption( 'threshold' ); |
134 | $removalThreshold = $this->getOption( 'removal-threshold' ); |
135 | $noFuzzy = $this->hasOption( 'no-fuzzy' ); |
136 | $requestedLanguages = $this->parseLanguageCodes( $this->getOption( 'lang' ) ); |
137 | $alwaysExportLanguages = $this->csv2array( |
138 | $this->getOption( 'always-export-languages' ) ?? '' |
139 | ); |
140 | $neverExportLanguages = $this->csv2array( |
141 | $this->getOption( 'never-export-languages' ) ?? |
142 | $this->getOption( 'skip' ) ?? |
143 | '' |
144 | ); |
145 | $skipSourceLanguage = $this->hasOption( 'skip-source-language' ); |
146 | |
147 | $forOffline = $this->hasOption( 'offline-gettext-format' ); |
148 | $offlineTargetPattern = $this->getOption( 'offline-gettext-format' ) ?: "%GROUPID%/%CODE%.po"; |
149 | |
150 | $groups = $this->getMessageGroups( $groupPattern, $groupSkipPattern, $forOffline ); |
151 | if ( $groups === [] ) { |
152 | $this->fatalError( 'EE1: No valid message groups identified.' ); |
153 | } |
154 | |
155 | $groupSyncCacheEnabled = MediaWikiServices::getInstance()->getMainConfig() |
156 | ->get( 'TranslateGroupSynchronizationCache' ); |
157 | $groupSyncCache = Services::getInstance()->getGroupSynchronizationCache(); |
158 | |
159 | foreach ( $groups as $groupId => $group ) { |
160 | if ( $groupSyncCacheEnabled && !$skipGroupSyncCheck ) { |
161 | if ( !$this->canGroupBeExported( $groupSyncCache, $groupId ) ) { |
162 | continue; |
163 | } |
164 | } |
165 | |
166 | if ( $exportThreshold !== null || $removalThreshold !== null ) { |
167 | $logger->info( 'Calculating stats for group {groupId}', [ 'groupId' => $groupId ] ); |
168 | $tStartTime = microtime( true ); |
169 | |
170 | $languageExportActions = $this->getLanguageExportActions( |
171 | $groupId, |
172 | $requestedLanguages, |
173 | $alwaysExportLanguages, |
174 | (int)$exportThreshold, |
175 | (int)$removalThreshold |
176 | ); |
177 | |
178 | $tEndTime = microtime( true ); |
179 | $logger->info( |
180 | 'Finished calculating stats for group {groupId}. Time: {duration} secs', |
181 | [ |
182 | 'groupId' => $groupId, |
183 | 'duration' => round( $tEndTime - $tStartTime, 3 ), |
184 | ] |
185 | ); |
186 | } else { |
187 | // Convert list to an associative array |
188 | $languageExportActions = array_fill_keys( $requestedLanguages, self::ACTION_CREATE ); |
189 | |
190 | foreach ( $alwaysExportLanguages as $code ) { |
191 | $languageExportActions[ $code ] = self::ACTION_CREATE; |
192 | } |
193 | } |
194 | |
195 | foreach ( $neverExportLanguages as $code ) { |
196 | unset( $languageExportActions[ $code ] ); |
197 | } |
198 | |
199 | if ( $skipSourceLanguage ) { |
200 | unset( $languageExportActions[ $group->getSourceLanguage() ] ); |
201 | } |
202 | |
203 | if ( $languageExportActions === [] ) { |
204 | continue; |
205 | } |
206 | |
207 | $this->output( "Exporting group $groupId\n" ); |
208 | $logger->info( 'Exporting group {groupId}', [ 'groupId' => $groupId ] ); |
209 | |
210 | if ( $forOffline ) { |
211 | $fileBasedGroup = FileBasedMessageGroup::newFromMessageGroup( $group, $offlineTargetPattern ); |
212 | $fileFormat = new GettextFormat( $fileBasedGroup ); |
213 | $fileFormat->setOfflineMode( true ); |
214 | } else { |
215 | $fileBasedGroup = $group; |
216 | // At this point $group should be an instance of FileBasedMessageGroup |
217 | // This is primarily to keep linting tools / IDE happy. |
218 | if ( !$fileBasedGroup instanceof FileBasedMessageGroup ) { |
219 | $this->fatalError( "EE2: Unexportable message group $groupId" ); |
220 | } |
221 | $fileFormat = $fileBasedGroup->getFFS(); |
222 | } |
223 | |
224 | $fileFormat->setWritePath( $target ); |
225 | $sourceLanguage = $group->getSourceLanguage(); |
226 | $collection = $group->initCollection( $sourceLanguage ); |
227 | |
228 | $inclusionList = $group->getTranslatableLanguages(); |
229 | |
230 | $langExportTimes = [ |
231 | 'collection' => 0, |
232 | 'ffs' => 0, |
233 | ]; |
234 | |
235 | $languagesExportedCount = 0; |
236 | |
237 | $langStartTime = microtime( true ); |
238 | foreach ( $languageExportActions as $lang => $action ) { |
239 | // Do not export languages that are excluded (or not included). |
240 | // Also check that inclusion list is not null, which means that all |
241 | // languages are allowed for translation and export. |
242 | if ( is_array( $inclusionList ) && !isset( $inclusionList[$lang] ) ) { |
243 | continue; |
244 | } |
245 | |
246 | $targetFilePath = $target . '/' . $fileBasedGroup->getTargetFilename( $lang ); |
247 | if ( $action === self::ACTION_DELETE ) { |
248 | // phpcs:ignore Generic.PHP.NoSilencedErrors.Discouraged |
249 | @$ok = unlink( $targetFilePath ); |
250 | if ( $ok ) { |
251 | $logger->info( "Removed $targetFilePath due to removal threshold" ); |
252 | } |
253 | continue; |
254 | } elseif ( $action === self::ACTION_UPDATE && !file_exists( $targetFilePath ) ) { |
255 | // Language is under export threshold, do not export yet |
256 | $logger->info( "Not creating $targetFilePath due to export threshold" ); |
257 | continue; |
258 | } |
259 | |
260 | $startTime = microtime( true ); |
261 | $collection->resetForNewLanguage( $lang ); |
262 | $collection->loadTranslations(); |
263 | // Don't export ignored, unless it is the source language |
264 | // or message documentation |
265 | global $wgTranslateDocumentationLanguageCode; |
266 | if ( $lang !== $wgTranslateDocumentationLanguageCode |
267 | && $lang !== $sourceLanguage |
268 | ) { |
269 | $collection->filter( 'ignored' ); |
270 | } |
271 | |
272 | if ( $noFuzzy ) { |
273 | $collection->filter( 'fuzzy' ); |
274 | } |
275 | |
276 | $languagesExportedCount++; |
277 | |
278 | $endTime = microtime( true ); |
279 | $langExportTimes['collection'] += ( $endTime - $startTime ); |
280 | |
281 | $startTime = microtime( true ); |
282 | $fileFormat->write( $collection ); |
283 | $endTime = microtime( true ); |
284 | $langExportTimes['ffs'] += ( $endTime - $startTime ); |
285 | } |
286 | $langEndTime = microtime( true ); |
287 | |
288 | $logger->info( |
289 | 'Done exporting {count} languages for group {groupId}. Time taken {duration} secs.', |
290 | [ |
291 | 'count' => $languagesExportedCount, |
292 | 'groupId' => $groupId, |
293 | 'duration' => round( $langEndTime - $langStartTime, 3 ), |
294 | ] |
295 | ); |
296 | |
297 | foreach ( $langExportTimes as $type => $time ) { |
298 | $logger->info( |
299 | 'Time taken by "{type}" for group {groupId} – {duration} secs.', |
300 | [ |
301 | 'groupId' => $groupId, |
302 | 'type' => $type, |
303 | 'duration' => round( $time, 3 ), |
304 | ] |
305 | ); |
306 | } |
307 | } |
308 | |
309 | $exportEndTime = microtime( true ); |
310 | $logger->info( |
311 | 'Finished export process for groups {groups}. Time: {duration} secs.', |
312 | [ |
313 | 'groups' => $groupPattern, |
314 | 'duration' => round( $exportEndTime - $exportStartTime, 3 ), |
315 | ] |
316 | ); |
317 | } |
318 | |
319 | /** @return MessageGroup[] */ |
320 | private function getMessageGroups( |
321 | string $groupPattern, |
322 | string $excludePattern, |
323 | bool $forOffline |
324 | ): array { |
325 | $groupIds = MessageGroups::expandWildcards( explode( ',', trim( $groupPattern ) ) ); |
326 | $groups = MessageGroups::getGroupsById( $groupIds ); |
327 | if ( !$forOffline ) { |
328 | foreach ( $groups as $groupId => $group ) { |
329 | if ( $group->isMeta() ) { |
330 | $this->output( "Skipping meta message group $groupId.\n" ); |
331 | unset( $groups[$groupId] ); |
332 | continue; |
333 | } |
334 | |
335 | if ( !$group instanceof FileBasedMessageGroup ) { |
336 | $this->output( "EE2: Unexportable message group $groupId.\n" ); |
337 | unset( $groups[$groupId] ); |
338 | } |
339 | } |
340 | } |
341 | |
342 | $skipIds = MessageGroups::expandWildcards( explode( ',', trim( $excludePattern ) ) ); |
343 | foreach ( $skipIds as $groupId ) { |
344 | if ( isset( $groups[$groupId] ) ) { |
345 | unset( $groups[$groupId] ); |
346 | $this->output( "Group $groupId is in skipgroup.\n" ); |
347 | } |
348 | } |
349 | |
350 | return $groups; |
351 | } |
352 | |
353 | /** @return string[] */ |
354 | private function getLanguageExportActions( |
355 | string $groupId, |
356 | array $requestedLanguages, |
357 | array $alwaysExportLanguages, |
358 | int $exportThreshold = 0, |
359 | int $removalThreshold = 0 |
360 | ): array { |
361 | $stats = MessageGroupStats::forGroup( $groupId ); |
362 | |
363 | $languages = []; |
364 | |
365 | foreach ( $requestedLanguages as $code ) { |
366 | // Statistics unavailable. This should only happen if unknown language code requested. |
367 | if ( !isset( $stats[$code] ) ) { |
368 | continue; |
369 | } |
370 | |
371 | $total = $stats[$code][MessageGroupStats::TOTAL]; |
372 | $translated = $stats[$code][MessageGroupStats::TRANSLATED]; |
373 | $percentage = $total ? $translated / $total * 100 : 0; |
374 | |
375 | if ( $percentage === 0 || $percentage < $removalThreshold ) { |
376 | $languages[$code] = self::ACTION_DELETE; |
377 | } elseif ( $percentage > $exportThreshold ) { |
378 | $languages[$code] = self::ACTION_CREATE; |
379 | } else { |
380 | $languages[$code] = self::ACTION_UPDATE; |
381 | } |
382 | } |
383 | |
384 | foreach ( $alwaysExportLanguages as $code ) { |
385 | $languages[$code] = self::ACTION_CREATE; |
386 | // DWIM: Do not export languages with zero translations, even if requested |
387 | if ( ( $stats[$code][MessageGroupStats::TRANSLATED] ?? null ) === 0 ) { |
388 | $languages[$code] = self::ACTION_DELETE; |
389 | } |
390 | } |
391 | |
392 | return $languages; |
393 | } |
394 | |
395 | private function canGroupBeExported( GroupSynchronizationCache $groupSyncCache, string $groupId ): bool { |
396 | if ( $groupSyncCache->isGroupBeingProcessed( $groupId ) ) { |
397 | $this->error( "Group $groupId is currently being synchronized; skipping exports\n" ); |
398 | return false; |
399 | } |
400 | |
401 | if ( $groupSyncCache->groupHasErrors( $groupId ) ) { |
402 | $this->error( "Skipping $groupId due to synchronization error\n" ); |
403 | return false; |
404 | } |
405 | |
406 | if ( $groupSyncCache->isGroupInReview( $groupId ) ) { |
407 | $this->error( "Group $groupId is currently in review. Review changes on Special:ManageMessageGroups\n" ); |
408 | return false; |
409 | } |
410 | return true; |
411 | } |
412 | |
413 | /** @return string[] */ |
414 | private function csv2array( string $input ): array { |
415 | return array_filter( |
416 | array_map( 'trim', explode( ',', $input ) ), |
417 | static function ( $v ) { |
418 | return $v !== ''; |
419 | } |
420 | ); |
421 | } |
422 | |
423 | /** @return string[] */ |
424 | private function parseLanguageCodes( string $input ): array { |
425 | if ( $input === '*' ) { |
426 | $languageNameUtils = MediaWikiServices::getInstance()->getLanguageNameUtils(); |
427 | $languages = $languageNameUtils->getLanguageNames(); |
428 | ksort( $languages ); |
429 | return array_keys( $languages ); |
430 | } |
431 | |
432 | return $this->csv2array( $input ); |
433 | } |
434 | } |