Translate extension for MediaWiki
 
Loading...
Searching...
No Matches
ExportTranslationsMaintenanceScript.php
1<?php
2
4
6use GettextFFS;
9use MediaWiki\Logger\LoggerFactory;
10use MediaWiki\MediaWikiServices;
11use MessageGroup;
14
25 private const ACTION_DELETE = 'delete';
27 private const ACTION_CREATE = 'create';
29 private const ACTION_UPDATE = 'update';
30
31 public function __construct() {
32 parent::__construct();
33 $this->addDescription( 'Export translations to files.' );
34
35 $this->addOption(
36 'group',
37 'Comma separated list of message group IDs (supports * wildcard) to export',
38 self::REQUIRED,
39 self::HAS_ARG
40 );
41 $this->addOption(
42 'lang',
43 'Comma separated list of language codes to export or * for all languages',
44 self::REQUIRED,
45 self::HAS_ARG
46 );
47 $this->addOption(
48 'always-export-languages',
49 '(optional) Comma separated list of languages to export ignoring export threshold',
50 self::OPTIONAL,
51 self::HAS_ARG
52 );
53 $this->addOption(
54 'never-export-languages',
55 '(optional) Comma separated list of languages to never export (overrides everything else)',
56 self::OPTIONAL,
57 self::HAS_ARG
58 );
59 $this->addOption(
60 'skip-source-language',
61 '(optional) Do not export the source language of each message group',
62 self::OPTIONAL,
63 self::NO_ARG
64 );
65 $this->addOption(
66 'target',
67 'Target directory for exported files',
68 self::REQUIRED,
69 self::HAS_ARG
70 );
71 $this->addOption(
72 'skip',
73 '(deprecated) See --never-export-languages',
74 self::OPTIONAL,
75 self::HAS_ARG
76 );
77 $this->addOption(
78 'skipgroup',
79 '(optional) Comma separated list of message group IDs (supports * wildcard) to not export',
80 self::OPTIONAL,
81 self::HAS_ARG
82 );
83 $this->addOption(
84 'threshold',
85 '(optional) Threshold for translation completion percentage that must be exceeded for initial export',
86 self::OPTIONAL,
87 self::HAS_ARG
88 );
89 $this->addOption(
90 'removal-threshold',
91 '(optional) Threshold for translation completion percentage that must be exceeded to keep the file',
92 self::OPTIONAL,
93 self::HAS_ARG
94 );
95 $this->addOption(
96 'no-fuzzy',
97 '(optional) Do not include any messages marked as fuzzy/outdated'
98 );
99 $this->addOption(
100 'offline-gettext-format',
101 '(optional) Export languages in offline Gettext format. Give a file pattern with '
102 . '%GROUPID% and %CODE%. Empty pattern defaults to %GROUPID%/%CODE%.po.',
103 self::OPTIONAL,
104 self::HAS_ARG
105 );
106 $this->addOption(
107 'skip-group-sync-check',
108 '(optional) Skip exporting group if synchronization is still in progress or if there ' .
109 'was an error during synchronization. See: ' .
110 'https://www.mediawiki.org/wiki/Help:Extension:Translate/Group_management#Strong_synchronization'
111 );
112
113 $this->requireExtension( 'Translate' );
114 }
115
116 public function execute() {
117 $logger = LoggerFactory::getInstance( 'Translate.GroupSynchronization' );
118 $groupPattern = $this->getOption( 'group' ) ?? '';
119 $groupSkipPattern = $this->getOption( 'skipgroup' ) ?? '';
120 $skipGroupSyncCheck = $this->hasOption( 'skip-group-sync-check' );
121
122 $logger->info(
123 'Starting exports for groups {groups}',
124 [ 'groups' => $groupPattern ]
125 );
126 $exportStartTime = microtime( true );
127
128 $target = $this->getOption( 'target' );
129 if ( !is_writable( $target ) ) {
130 $this->fatalError( "Target directory is not writable ($target)." );
131 }
132
133 $exportThreshold = $this->getOption( 'threshold' );
134 $removalThreshold = $this->getOption( 'removal-threshold' );
135 $noFuzzy = $this->hasOption( 'no-fuzzy' );
136 $requestedLanguages = $this->parseLanguageCodes( $this->getOption( 'lang' ) );
137 $alwaysExportLanguages = $this->csv2array(
138 $this->getOption( 'always-export-languages' ) ?? ''
139 );
140 $neverExportLanguages = $this->csv2array(
141 $this->getOption( 'never-export-languages' ) ??
142 $this->getOption( 'skip' ) ??
143 ''
144 );
145 $skipSourceLanguage = $this->hasOption( 'skip-source-language' );
146
147 $forOffline = $this->hasOption( 'offline-gettext-format' );
148 $offlineTargetPattern = $this->getOption( 'offline-gettext-format' ) ?: "%GROUPID%/%CODE%.po";
149
150 $groups = $this->getMessageGroups( $groupPattern, $groupSkipPattern, $forOffline );
151 if ( $groups === [] ) {
152 $this->fatalError( 'EE1: No valid message groups identified.' );
153 }
154
155 $groupSyncCacheEnabled = MediaWikiServices::getInstance()->getMainConfig()
156 ->get( 'TranslateGroupSynchronizationCache' );
157 $groupSyncCache = Services::getInstance()->getGroupSynchronizationCache();
158
159 foreach ( $groups as $groupId => $group ) {
160 if ( $groupSyncCacheEnabled && !$skipGroupSyncCheck ) {
161 if ( !$this->canGroupBeExported( $groupSyncCache, $groupId ) ) {
162 continue;
163 }
164 }
165
166 if ( $exportThreshold !== null || $removalThreshold !== null ) {
167 $logger->info( 'Calculating stats for group {groupId}', [ 'groupId' => $groupId ] );
168 $tStartTime = microtime( true );
169
170 $languageExportActions = $this->getLanguageExportActions(
171 $groupId,
172 $requestedLanguages,
173 $alwaysExportLanguages,
174 (int)$exportThreshold,
175 (int)$removalThreshold
176 );
177
178 $tEndTime = microtime( true );
179 $logger->info(
180 'Finished calculating stats for group {groupId}. Time: {duration} secs',
181 [
182 'groupId' => $groupId,
183 'duration' => round( $tEndTime - $tStartTime, 3 ),
184 ]
185 );
186 } else {
187 // Convert list to an associative array
188 $languageExportActions = array_fill_keys( $requestedLanguages, self::ACTION_CREATE );
189
190 foreach ( $alwaysExportLanguages as $code ) {
191 $languageExportActions[ $code ] = self::ACTION_CREATE;
192 }
193 }
194
195 foreach ( $neverExportLanguages as $code ) {
196 unset( $languageExportActions[ $code ] );
197 }
198
199 if ( $skipSourceLanguage ) {
200 unset( $languageExportActions[ $group->getSourceLanguage() ] );
201 }
202
203 if ( $languageExportActions === [] ) {
204 continue;
205 }
206
207 $this->output( "Exporting group $groupId\n" );
208 $logger->info( 'Exporting group {groupId}', [ 'groupId' => $groupId ] );
209
210 if ( $forOffline ) {
211 $fileBasedGroup = FileBasedMessageGroup::newFromMessageGroup( $group, $offlineTargetPattern );
212 $ffs = new GettextFFS( $fileBasedGroup );
213 $ffs->setOfflineMode( true );
214 } else {
215 $fileBasedGroup = $group;
216 // At this point $group should be an instance of FileBasedMessageGroup
217 // This is primarily to keep linting tools / IDE happy.
218 if ( !$fileBasedGroup instanceof FileBasedMessageGroup ) {
219 $this->fatalError( "EE2: Unexportable message group $groupId" );
220 }
221 $ffs = $fileBasedGroup->getFFS();
222 }
223
224 $ffs->setWritePath( $target );
225 $sourceLanguage = $group->getSourceLanguage();
226 $collection = $group->initCollection( $sourceLanguage );
227
228 $inclusionList = $group->getTranslatableLanguages();
229
230 $langExportTimes = [
231 'collection' => 0,
232 'ffs' => 0,
233 ];
234
235 $languagesExportedCount = 0;
236
237 $langStartTime = microtime( true );
238 foreach ( $languageExportActions as $lang => $action ) {
239 // Do not export languages that are excluded (or not included).
240 // Also check that inclusion list is not null, which means that all
241 // languages are allowed for translation and export.
242 if ( is_array( $inclusionList ) && !isset( $inclusionList[$lang] ) ) {
243 continue;
244 }
245
246 $targetFilePath = $target . '/' . $fileBasedGroup->getTargetFilename( $lang );
247 if ( $action === self::ACTION_DELETE ) {
248 // phpcs:ignore Generic.PHP.NoSilencedErrors.Discouraged
249 @$ok = unlink( $targetFilePath );
250 if ( $ok ) {
251 $logger->info( "Removed $targetFilePath due to removal threshold" );
252 }
253 continue;
254 } elseif ( $action === self::ACTION_UPDATE && !file_exists( $targetFilePath ) ) {
255 // Language is under export threshold, do not export yet
256 $logger->info( "Not creating $targetFilePath due to export threshold" );
257 continue;
258 }
259
260 $startTime = microtime( true );
261 $collection->resetForNewLanguage( $lang );
262 $collection->loadTranslations();
263 // Don't export ignored, unless it is the source language
264 // or message documentation
265 global $wgTranslateDocumentationLanguageCode;
266 if ( $lang !== $wgTranslateDocumentationLanguageCode
267 && $lang !== $sourceLanguage
268 ) {
269 $collection->filter( 'ignored' );
270 }
271
272 if ( $noFuzzy ) {
273 $collection->filter( 'fuzzy' );
274 }
275
276 $languagesExportedCount++;
277
278 $endTime = microtime( true );
279 $langExportTimes['collection'] += ( $endTime - $startTime );
280
281 $startTime = microtime( true );
282 $ffs->write( $collection );
283 $endTime = microtime( true );
284 $langExportTimes['ffs'] += ( $endTime - $startTime );
285 }
286 $langEndTime = microtime( true );
287
288 $logger->info(
289 'Done exporting {count} languages for group {groupId}. Time taken {duration} secs.',
290 [
291 'count' => $languagesExportedCount,
292 'groupId' => $groupId,
293 'duration' => round( $langEndTime - $langStartTime, 3 ),
294 ]
295 );
296
297 foreach ( $langExportTimes as $type => $time ) {
298 $logger->info(
299 'Time taken by "{type}" for group {groupId} – {duration} secs.',
300 [
301 'groupId' => $groupId,
302 'type' => $type,
303 'duration' => round( $time, 3 ),
304 ]
305 );
306 }
307 }
308
309 $exportEndTime = microtime( true );
310 $logger->info(
311 'Finished export process for groups {groups}. Time: {duration} secs.',
312 [
313 'groups' => $groupPattern,
314 'duration' => round( $exportEndTime - $exportStartTime, 3 ),
315 ]
316 );
317 }
318
320 private function getMessageGroups(
321 string $groupPattern,
322 string $excludePattern,
323 bool $forOffline
324 ): array {
325 $groupIds = MessageGroups::expandWildcards( explode( ',', trim( $groupPattern ) ) );
326 $groups = MessageGroups::getGroupsById( $groupIds );
327 if ( !$forOffline ) {
328 foreach ( $groups as $groupId => $group ) {
329 if ( $group->isMeta() ) {
330 $this->output( "Skipping meta message group $groupId.\n" );
331 unset( $groups[$groupId] );
332 continue;
333 }
334
335 if ( !$group instanceof FileBasedMessageGroup ) {
336 $this->output( "EE2: Unexportable message group $groupId.\n" );
337 unset( $groups[$groupId] );
338 }
339 }
340 }
341
342 $skipIds = MessageGroups::expandWildcards( explode( ',', trim( $excludePattern ) ) );
343 foreach ( $skipIds as $groupId ) {
344 if ( isset( $groups[$groupId] ) ) {
345 unset( $groups[$groupId] );
346 $this->output( "Group $groupId is in skipgroup.\n" );
347 }
348 }
349
350 return $groups;
351 }
352
354 private function getLanguageExportActions(
355 string $groupId,
356 array $requestedLanguages,
357 array $alwaysExportLanguages,
358 int $exportThreshold = 0,
359 int $removalThreshold = 0
360 ): array {
361 $stats = MessageGroupStats::forGroup( $groupId );
362
363 $languages = [];
364
365 foreach ( $requestedLanguages as $code ) {
366 // Statistics unavailable. This should only happen if unknown language code requested.
367 if ( !isset( $stats[$code] ) ) {
368 continue;
369 }
370
371 $total = $stats[$code][MessageGroupStats::TOTAL];
372 $translated = $stats[$code][MessageGroupStats::TRANSLATED];
373 $percentage = $total === 0 ? 0 : $translated / $total * 100;
374
375 if ( $percentage === 0 || $percentage < $removalThreshold ) {
376 $languages[$code] = self::ACTION_DELETE;
377 } elseif ( $percentage > $exportThreshold ) {
378 $languages[$code] = self::ACTION_CREATE;
379 } else {
380 $languages[$code] = self::ACTION_UPDATE;
381 }
382 }
383
384 foreach ( $alwaysExportLanguages as $code ) {
385 $languages[$code] = self::ACTION_CREATE;
386 // DWIM: Do not export languages with zero translations, even if requested
387 if ( ( $stats[$code][MessageGroupStats::TRANSLATED] ?? null ) === 0 ) {
388 $languages[$code] = self::ACTION_DELETE;
389 }
390 }
391
392 return $languages;
393 }
394
395 private function canGroupBeExported( GroupSynchronizationCache $groupSyncCache, string $groupId ): bool {
396 if ( $groupSyncCache->isGroupBeingProcessed( $groupId ) ) {
397 $this->error( "Group $groupId is currently being synchronized; skipping exports\n" );
398 return false;
399 }
400
401 if ( $groupSyncCache->groupHasErrors( $groupId ) ) {
402 $this->error( "Skipping $groupId due to synchronization error\n" );
403 return false;
404 }
405
406 if ( $groupSyncCache->isGroupInReview( $groupId ) ) {
407 $this->error( "Group $groupId is currently in review. Review changes on Special:ManageMessageGroups\n" );
408 return false;
409 }
410 return true;
411 }
412
414 private function csv2array( string $input ): array {
415 return array_filter(
416 array_map( 'trim', explode( ',', $input ) ),
417 static function ( $v ) {
418 return $v !== '';
419 }
420 );
421 }
422
424 private function parseLanguageCodes( string $input ): array {
425 if ( $input === '*' ) {
426 $languageNameUtils = MediaWikiServices::getInstance()->getLanguageNameUtils();
427 $languages = $languageNameUtils->getLanguageNames();
428 ksort( $languages );
429 return array_keys( $languages );
430 }
431
432 return $this->csv2array( $input );
433 }
434}
This class implements default behavior for file based message groups.
New-style FFS class that implements support for gettext file format.
Minimal service container.
Definition Services.php:38
isGroupBeingProcessed(string $groupId)
Check if the group is in synchronization.
Constants for making code for maintenance scripts more readable.
This class abstract MessageGroup statistics calculation and storing.
Factory class for accessing message groups individually by id or all of them as an list.
Interface for message groups.
Finds external changes for file based message groups.