Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
0.00% |
0 / 174 |
|
0.00% |
0 / 7 |
CRAP | |
0.00% |
0 / 1 |
| BackportTranslationsMaintenanceScript | |
0.00% |
0 / 174 |
|
0.00% |
0 / 7 |
1260 | |
0.00% |
0 / 1 |
| __construct | |
0.00% |
0 / 33 |
|
0.00% |
0 / 1 |
2 | |||
| execute | |
0.00% |
0 / 78 |
|
0.00% |
0 / 1 |
240 | |||
| csv2array | |
0.00% |
0 / 6 |
|
0.00% |
0 / 1 |
2 | |||
| matchPath | |
0.00% |
0 / 3 |
|
0.00% |
0 / 1 |
2 | |||
| loadDefinitions | |
0.00% |
0 / 5 |
|
0.00% |
0 / 1 |
6 | |||
| getKeyCompatibilityMap | |
0.00% |
0 / 4 |
|
0.00% |
0 / 1 |
12 | |||
| backport | |
0.00% |
0 / 45 |
|
0.00% |
0 / 1 |
156 | |||
| 1 | <?php |
| 2 | declare( strict_types = 1 ); |
| 3 | |
| 4 | namespace MediaWiki\Extension\Translate\Synchronization; |
| 5 | |
| 6 | use FileBasedMessageGroup; |
| 7 | use MediaWiki\Extension\Translate\FileFormatSupport\JsonFormat; |
| 8 | use MediaWiki\Extension\Translate\FileFormatSupport\SimpleFormat; |
| 9 | use MediaWiki\Extension\Translate\MessageGroupProcessing\MessageGroups; |
| 10 | use MediaWiki\Extension\Translate\Utilities\BaseMaintenanceScript; |
| 11 | use MediaWiki\Extension\Translate\Utilities\Utilities; |
| 12 | use MediaWiki\Logger\LoggerFactory; |
| 13 | use RuntimeException; |
| 14 | |
| 15 | /** |
| 16 | * Script to backport translations from one branch to another. |
| 17 | * |
| 18 | * @since 2021.05 |
| 19 | * @license GPL-2.0-or-later |
| 20 | * @author Niklas Laxström |
| 21 | */ |
| 22 | class BackportTranslationsMaintenanceScript extends BaseMaintenanceScript { |
| 23 | public function __construct() { |
| 24 | parent::__construct(); |
| 25 | $this->addDescription( 'Backport translations from one branch to another.' ); |
| 26 | |
| 27 | $this->addOption( |
| 28 | 'group', |
| 29 | 'Comma separated list of message group IDs (supports * wildcard) to backport', |
| 30 | self::REQUIRED, |
| 31 | self::HAS_ARG |
| 32 | ); |
| 33 | $this->addOption( |
| 34 | 'source-path', |
| 35 | 'Source path for reading updated translations. Defaults to $wgTranslateGroupRoot.', |
| 36 | self::OPTIONAL, |
| 37 | self::HAS_ARG |
| 38 | ); |
| 39 | $this->addOption( |
| 40 | 'target-path', |
| 41 | 'Target path for writing backported translations', |
| 42 | self::REQUIRED, |
| 43 | self::HAS_ARG |
| 44 | ); |
| 45 | $this->addOption( |
| 46 | 'filter-path', |
| 47 | 'Only export a group if its export path matches this prefix (relative to target-path)', |
| 48 | self::OPTIONAL, |
| 49 | self::HAS_ARG |
| 50 | ); |
| 51 | $this->addOption( |
| 52 | 'never-export-languages', |
| 53 | 'Languages to not export', |
| 54 | self::OPTIONAL, |
| 55 | self::HAS_ARG |
| 56 | ); |
| 57 | $this->requireExtension( 'Translate' ); |
| 58 | } |
| 59 | |
| 60 | /** @inheritDoc */ |
| 61 | public function execute() { |
| 62 | $config = $this->getConfig(); |
| 63 | $logger = LoggerFactory::getInstance( 'Translate.GroupSynchronization' ); |
| 64 | $groupPattern = $this->getOption( 'group' ) ?? ''; |
| 65 | $logger->info( |
| 66 | 'Starting backports for groups {groups}', |
| 67 | [ 'groups' => $groupPattern ] |
| 68 | ); |
| 69 | |
| 70 | $sourcePath = $this->getOption( 'source-path' ) ?: $config->get( 'TranslateGroupRoot' ); |
| 71 | if ( !is_readable( $sourcePath ) ) { |
| 72 | $this->fatalError( "Source directory is not readable ($sourcePath)." ); |
| 73 | } |
| 74 | |
| 75 | $targetPath = $this->getOption( 'target-path' ); |
| 76 | if ( !is_writable( $targetPath ) ) { |
| 77 | $this->fatalError( "Target directory is not writable ($targetPath)." ); |
| 78 | } |
| 79 | |
| 80 | $groupIds = MessageGroups::expandWildcards( explode( ',', trim( $groupPattern ) ) ); |
| 81 | $groups = MessageGroups::getGroupsById( $groupIds ); |
| 82 | if ( $groups === [] ) { |
| 83 | $this->fatalError( "Pattern $groupPattern did not match any message groups." ); |
| 84 | } |
| 85 | |
| 86 | $neverExportLanguages = $this->csv2array( |
| 87 | $this->getOption( 'never-export-languages' ) ?? '' |
| 88 | ); |
| 89 | $supportedLanguages = Utilities::getLanguageNames( 'en' ); |
| 90 | |
| 91 | foreach ( $groups as $group ) { |
| 92 | $groupId = $group->getId(); |
| 93 | if ( !$group instanceof FileBasedMessageGroup ) { |
| 94 | if ( !$this->hasOption( 'filter-path' ) ) { |
| 95 | $this->error( "Skipping $groupId: Not instance of FileBasedMessageGroup" ); |
| 96 | } |
| 97 | continue; |
| 98 | } |
| 99 | |
| 100 | if ( !$group->getFFS() instanceof JsonFormat ) { |
| 101 | $this->error( "Skipping $groupId: Only JSON format is supported" ); |
| 102 | continue; |
| 103 | } |
| 104 | |
| 105 | if ( $this->hasOption( 'filter-path' ) ) { |
| 106 | $filter = $this->getOption( 'filter-path' ); |
| 107 | $exportPath = $group->getTargetFilename( '*' ); |
| 108 | if ( !$this->matchPath( $filter, $exportPath ) ) { |
| 109 | continue; |
| 110 | } |
| 111 | } |
| 112 | |
| 113 | $sourceLanguage = $group->getSourceLanguage(); |
| 114 | try { |
| 115 | $sourceDefinitions = $this->loadDefinitions( $group, $sourcePath, $sourceLanguage ); |
| 116 | $targetDefinitions = $this->loadDefinitions( $group, $targetPath, $sourceLanguage ); |
| 117 | } catch ( RuntimeException $e ) { |
| 118 | $this->output( |
| 119 | "Skipping $groupId: Error while loading definitions: {$e->getMessage()}\n" |
| 120 | ); |
| 121 | continue; |
| 122 | } |
| 123 | |
| 124 | $keyCompatibilityMap = $this->getKeyCompatibilityMap( |
| 125 | $sourceDefinitions['MESSAGES'], |
| 126 | $targetDefinitions['MESSAGES'], |
| 127 | $group->getFFS() |
| 128 | ); |
| 129 | |
| 130 | if ( array_filter( $keyCompatibilityMap ) === [] ) { |
| 131 | $this->output( "Skipping $groupId: No compatible keys found\n" ); |
| 132 | continue; |
| 133 | } |
| 134 | |
| 135 | $summary = []; |
| 136 | $languages = array_keys( $group->getTranslatableLanguages() ?? $supportedLanguages ); |
| 137 | $languagesToSkip = $neverExportLanguages; |
| 138 | $languagesToSkip[] = $sourceLanguage; |
| 139 | $languages = array_diff( $languages, $languagesToSkip ); |
| 140 | |
| 141 | foreach ( $languages as $language ) { |
| 142 | $status = $this->backport( |
| 143 | $group, |
| 144 | $sourcePath, |
| 145 | $targetPath, |
| 146 | $keyCompatibilityMap, |
| 147 | $language |
| 148 | ); |
| 149 | |
| 150 | $summary[$status][] = $language; |
| 151 | } |
| 152 | |
| 153 | $numUpdated = count( $summary[ 'updated' ] ?? [] ); |
| 154 | $numAdded = count( $summary[ 'new' ] ?? [] ); |
| 155 | if ( ( $numUpdated + $numAdded ) > 0 ) { |
| 156 | $this->output( |
| 157 | sprintf( |
| 158 | "%s: Compatible keys: %d. Updated %d languages, %d new (%s)\n", |
| 159 | $group->getId(), |
| 160 | count( $keyCompatibilityMap ), |
| 161 | $numUpdated, |
| 162 | $numAdded, |
| 163 | implode( ', ', $summary[ 'new' ] ?? [] ) |
| 164 | ) |
| 165 | ); |
| 166 | } |
| 167 | } |
| 168 | } |
| 169 | |
| 170 | private function csv2array( string $input ): array { |
| 171 | return array_filter( |
| 172 | array_map( 'trim', explode( ',', $input ) ), |
| 173 | static function ( $v ) { |
| 174 | return $v !== ''; |
| 175 | } |
| 176 | ); |
| 177 | } |
| 178 | |
| 179 | private function matchPath( string $prefix, string $full ): bool { |
| 180 | $prefix = "./$prefix"; |
| 181 | $length = strlen( $prefix ); |
| 182 | return substr( $full, 0, $length ) === $prefix; |
| 183 | } |
| 184 | |
| 185 | private function loadDefinitions( |
| 186 | FileBasedMessageGroup $group, |
| 187 | string $path, |
| 188 | string $language |
| 189 | ): array { |
| 190 | $file = $path . '/' . $group->getTargetFilename( $language ); |
| 191 | |
| 192 | if ( !file_exists( $file ) ) { |
| 193 | throw new RuntimeException( "File $file does not exist" ); |
| 194 | } |
| 195 | |
| 196 | $contents = file_get_contents( $file ); |
| 197 | return $group->getFFS()->readFromVariable( $contents ); |
| 198 | } |
| 199 | |
| 200 | /** |
| 201 | * Compares two arrays and returns a new array with keys from the target array with associated values |
| 202 | * being a boolean indicating whether the source array value is compatible with the target array value. |
| 203 | * |
| 204 | * Target array key order was chosen because in backporting we want to use the order of keys in the |
| 205 | * backport target (stable branch). Comparison is done with SimpleFormat::isContentEqual. |
| 206 | * |
| 207 | * @return array<string,bool> Keys in target order |
| 208 | */ |
| 209 | private function getKeyCompatibilityMap( array $source, array $target, SimpleFormat $fileFormat ): array { |
| 210 | $keys = []; |
| 211 | foreach ( $target as $key => $value ) { |
| 212 | $keys[$key] = isset( $source[ $key ] ) && $fileFormat->isContentEqual( $source[ $key ], $value ); |
| 213 | } |
| 214 | return $keys; |
| 215 | } |
| 216 | |
| 217 | private function backport( |
| 218 | FileBasedMessageGroup $group, |
| 219 | string $source, |
| 220 | string $targetPath, |
| 221 | array $keyCompatibilityMap, |
| 222 | string $language |
| 223 | ): string { |
| 224 | try { |
| 225 | $sourceTemplate = $this->loadDefinitions( $group, $source, $language ); |
| 226 | } catch ( RuntimeException $e ) { |
| 227 | return 'no definitions'; |
| 228 | } |
| 229 | |
| 230 | try { |
| 231 | $targetTemplate = $this->loadDefinitions( $group, $targetPath, $language ); |
| 232 | } catch ( RuntimeException $e ) { |
| 233 | $targetTemplate = [ |
| 234 | 'MESSAGES' => [], |
| 235 | 'AUTHORS' => [], |
| 236 | ]; |
| 237 | } |
| 238 | |
| 239 | // Amend the target with compatible things from the source |
| 240 | $hasUpdates = false; |
| 241 | |
| 242 | $fileFormat = $group->getFFS(); |
| 243 | |
| 244 | // This has been checked before, but checking again to keep Phan and IDEs happy. |
| 245 | // Remove once support for other file formats are added. |
| 246 | if ( !$fileFormat instanceof JsonFormat ) { |
| 247 | throw new RuntimeException( |
| 248 | "Expected file format type: " . JsonFormat::class . '; got: ' . get_class( $fileFormat ) |
| 249 | ); |
| 250 | } |
| 251 | |
| 252 | $combinedMessages = []; |
| 253 | // $keyCompatibilityMap has the target (stable branch) source language key order |
| 254 | foreach ( $keyCompatibilityMap as $key => $isCompatible ) { |
| 255 | $sourceValue = $sourceTemplate['MESSAGES'][$key] ?? null; |
| 256 | $targetValue = $targetTemplate['MESSAGES'][$key] ?? null; |
| 257 | |
| 258 | // Use existing translation value from the target (stable branch) as the default |
| 259 | if ( $targetValue !== null ) { |
| 260 | $combinedMessages[$key] = $targetValue; |
| 261 | } |
| 262 | |
| 263 | // If the source (development branch) has a different translation for a compatible key |
| 264 | // replace the target (stable branch) translation with it. |
| 265 | if ( !$isCompatible ) { |
| 266 | continue; |
| 267 | } |
| 268 | if ( $sourceValue !== null && !$fileFormat->isContentEqual( $sourceValue, $targetValue ) ) { |
| 269 | // Keep track if we actually overwrote any values, so we can report back stats |
| 270 | $hasUpdates = true; |
| 271 | $combinedMessages[$key] = $sourceValue; |
| 272 | } |
| 273 | } |
| 274 | |
| 275 | if ( !$hasUpdates ) { |
| 276 | return 'no updates'; |
| 277 | } |
| 278 | |
| 279 | // Copy over all authors (we do not know per-message level) |
| 280 | $combinedAuthors = array_merge( |
| 281 | $targetTemplate[ 'AUTHORS' ] ?? [], |
| 282 | $sourceTemplate[ 'AUTHORS' ] ?? [] |
| 283 | ); |
| 284 | $combinedAuthors = array_unique( $combinedAuthors ); |
| 285 | $combinedAuthors = $fileFormat->filterAuthors( $combinedAuthors, $language ); |
| 286 | |
| 287 | $targetTemplate['AUTHORS'] = $combinedAuthors; |
| 288 | $targetTemplate['MESSAGES'] = $combinedMessages; |
| 289 | |
| 290 | $backportedContent = $fileFormat->generateFile( $targetTemplate ); |
| 291 | |
| 292 | $targetFilename = $targetPath . '/' . $group->getTargetFilename( $language ); |
| 293 | if ( file_exists( $targetFilename ) ) { |
| 294 | $currentContent = file_get_contents( $targetFilename ); |
| 295 | |
| 296 | if ( $fileFormat->shouldOverwrite( $currentContent, $backportedContent ) ) { |
| 297 | file_put_contents( $targetFilename, $backportedContent ); |
| 298 | } |
| 299 | return 'updated'; |
| 300 | } else { |
| 301 | file_put_contents( $targetFilename, $backportedContent ); |
| 302 | return 'new'; |
| 303 | } |
| 304 | } |
| 305 | } |