Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
0.00% |
0 / 138 |
|
0.00% |
0 / 7 |
CRAP | |
0.00% |
0 / 1 |
| ExternalMessageSourceStateImporter | |
0.00% |
0 / 138 |
|
0.00% |
0 / 7 |
1560 | |
0.00% |
0 / 1 |
| __construct | |
0.00% |
0 / 7 |
|
0.00% |
0 / 1 |
2 | |||
| import | |
0.00% |
0 / 55 |
|
0.00% |
0 / 1 |
132 | |||
| canImportGroup | |
0.00% |
0 / 12 |
|
0.00% |
0 / 1 |
42 | |||
| createMessageUpdateJobs | |
0.00% |
0 / 13 |
|
0.00% |
0 / 1 |
12 | |||
| updateGroupSyncInfo | |
0.00% |
0 / 15 |
|
0.00% |
0 / 1 |
12 | |||
| identifySafeLanguages | |
0.00% |
0 / 31 |
|
0.00% |
0 / 1 |
156 | |||
| isLanguageSafe | |
0.00% |
0 / 5 |
|
0.00% |
0 / 1 |
12 | |||
| 1 | <?php |
| 2 | declare( strict_types = 1 ); |
| 3 | |
| 4 | /** |
| 5 | * Finds external changes for file based message groups. |
| 6 | * @author Niklas Laxström |
| 7 | * @license GPL-2.0-or-later |
| 8 | * @since 2016.02 |
| 9 | */ |
| 10 | namespace MediaWiki\Extension\Translate\Synchronization; |
| 11 | |
| 12 | use FileBasedMessageGroup; |
| 13 | use JobQueueGroup; |
| 14 | use MediaWiki\Config\ServiceOptions; |
| 15 | use MediaWiki\Extension\Translate\MessageGroupProcessing\MessageGroups; |
| 16 | use MediaWiki\Extension\Translate\MessageLoading\MessageHandle; |
| 17 | use MediaWiki\Extension\Translate\MessageLoading\MessageIndex; |
| 18 | use MediaWiki\Extension\Translate\MessageSync\MessageSourceChange; |
| 19 | use MediaWiki\Language\RawMessage; |
| 20 | use MessageGroup; |
| 21 | use MessageUpdateJob; |
| 22 | use Psr\Log\LoggerInterface; |
| 23 | use RuntimeException; |
| 24 | use Status; |
| 25 | use TitleFactory; |
| 26 | use function wfWarn; |
| 27 | |
| 28 | class ExternalMessageSourceStateImporter { |
| 29 | private GroupSynchronizationCache $groupSynchronizationCache; |
| 30 | private JobQueueGroup $jobQueueGroup; |
| 31 | private LoggerInterface $logger; |
| 32 | private MessageIndex $messageIndex; |
| 33 | private TitleFactory $titleFactory; |
| 34 | private bool $isGroupSyncCacheEnabled; |
| 35 | // Do not perform any import |
| 36 | public const IMPORT_NONE = 1; |
| 37 | // Import changes in a language for a group if it only has additions |
| 38 | public const IMPORT_SAFE = 2; |
| 39 | // Import changes in a language for a group if it only has additions or changes, but |
| 40 | // not deletions as it may be a rename of an addition |
| 41 | public const IMPORT_NON_RENAMES = 3; |
| 42 | public const CONSTRUCTOR_OPTIONS = [ 'TranslateGroupSynchronizationCache' ]; |
| 43 | |
| 44 | public function __construct( |
| 45 | GroupSynchronizationCache $groupSynchronizationCache, |
| 46 | JobQueueGroup $jobQueueGroup, |
| 47 | LoggerInterface $logger, |
| 48 | MessageIndex $messageIndex, |
| 49 | TitleFactory $titleFactory, |
| 50 | ServiceOptions $options |
| 51 | ) { |
| 52 | $this->groupSynchronizationCache = $groupSynchronizationCache; |
| 53 | $this->jobQueueGroup = $jobQueueGroup; |
| 54 | $this->logger = $logger; |
| 55 | $this->messageIndex = $messageIndex; |
| 56 | $this->titleFactory = $titleFactory; |
| 57 | $options->assertRequiredOptions( self::CONSTRUCTOR_OPTIONS ); |
| 58 | $this->isGroupSyncCacheEnabled = $options->get( 'TranslateGroupSynchronizationCache' ); |
| 59 | } |
| 60 | |
| 61 | /** |
| 62 | * @param MessageSourceChange[] $changeData |
| 63 | * @param string $name |
| 64 | * @param int $importStrategy |
| 65 | * @return array |
| 66 | */ |
| 67 | public function import( array $changeData, string $name, int $importStrategy ): array { |
| 68 | $processed = []; |
| 69 | $skipped = []; |
| 70 | $jobs = []; |
| 71 | |
| 72 | foreach ( $changeData as $groupId => $changesForGroup ) { |
| 73 | $group = MessageGroups::getGroup( $groupId ); |
| 74 | if ( !$group ) { |
| 75 | unset( $changeData[$groupId] ); |
| 76 | continue; |
| 77 | } |
| 78 | |
| 79 | if ( !$group instanceof FileBasedMessageGroup ) { |
| 80 | $this->logger->warning( |
| 81 | '[ExternalMessageSourceStateImporter] Expected FileBasedMessageGroup, ' . |
| 82 | 'but got {class} for group {groupId}', |
| 83 | [ |
| 84 | 'class' => get_class( $group ), |
| 85 | 'groupId' => $groupId |
| 86 | ] |
| 87 | ); |
| 88 | unset( $changeData[$groupId] ); |
| 89 | continue; |
| 90 | } |
| 91 | |
| 92 | $processed[$groupId] = []; |
| 93 | $languages = $changesForGroup->getLanguages(); |
| 94 | $groupJobs = []; |
| 95 | |
| 96 | $groupSafeLanguages = $this->identifySafeLanguages( $group, $changesForGroup, $importStrategy ); |
| 97 | |
| 98 | foreach ( $languages as $language ) { |
| 99 | if ( !$groupSafeLanguages[ $language ] ) { |
| 100 | $skipped[$groupId] = true; |
| 101 | continue; |
| 102 | } |
| 103 | |
| 104 | $additions = $changesForGroup->getAdditions( $language ); |
| 105 | if ( $additions === [] ) { |
| 106 | continue; |
| 107 | } |
| 108 | |
| 109 | [ $groupLanguageJobs, $groupProcessed ] = $this->createMessageUpdateJobs( |
| 110 | $group, $additions, $language |
| 111 | ); |
| 112 | |
| 113 | $groupJobs = array_merge( $groupJobs, $groupLanguageJobs ); |
| 114 | $processed[$groupId][$language] = $groupProcessed; |
| 115 | |
| 116 | // We only remove additions since if less-safe-import option is used, then |
| 117 | // changes to existing messages might still need to be processed manually. |
| 118 | $changesForGroup->removeAdditions( $language, null ); |
| 119 | $group->getMessageGroupCache( $language )->create(); |
| 120 | } |
| 121 | |
| 122 | // Mark the skipped group as in review |
| 123 | if ( $this->isGroupSyncCacheEnabled && isset( $skipped[$groupId] ) ) { |
| 124 | $this->groupSynchronizationCache->markGroupAsInReview( $groupId ); |
| 125 | } |
| 126 | |
| 127 | if ( $groupJobs !== [] ) { |
| 128 | if ( $this->isGroupSyncCacheEnabled ) { |
| 129 | $this->updateGroupSyncInfo( $groupId, $groupJobs ); |
| 130 | } |
| 131 | $jobs = array_merge( $jobs, $groupJobs ); |
| 132 | } |
| 133 | } |
| 134 | |
| 135 | // Remove groups where everything was imported |
| 136 | $changeData = array_filter( $changeData, static function ( MessageSourceChange $change ) { |
| 137 | return $change->getAllModifications() !== []; |
| 138 | } ); |
| 139 | |
| 140 | // Remove groups with no imports |
| 141 | $processed = array_filter( $processed ); |
| 142 | |
| 143 | $file = MessageChangeStorage::getCdbPath( $name ); |
| 144 | MessageChangeStorage::writeChanges( $changeData, $file ); |
| 145 | $this->jobQueueGroup->push( $jobs ); |
| 146 | |
| 147 | return [ |
| 148 | 'processed' => $processed, |
| 149 | 'skipped' => $skipped, |
| 150 | 'name' => $name, |
| 151 | ]; |
| 152 | } |
| 153 | |
| 154 | public function canImportGroup( MessageGroup $group, bool $skipGroupSyncCache ): Status { |
| 155 | $groupId = $group->getId(); |
| 156 | if ( !$group instanceof FileBasedMessageGroup ) { |
| 157 | $error = "Group $groupId expected to be FileBasedMessageGroup, got " . get_class( $group ) . " instead."; |
| 158 | return Status::newFatal( new RawMessage( $error ) ); |
| 159 | } |
| 160 | |
| 161 | if ( $this->isGroupSyncCacheEnabled && !$skipGroupSyncCache ) { |
| 162 | if ( $this->groupSynchronizationCache->isGroupBeingProcessed( $groupId ) ) { |
| 163 | $error = "Group $groupId is currently being synchronized; skipping processing of changes\n"; |
| 164 | return Status::newFatal( new RawMessage( $error ) ); |
| 165 | } |
| 166 | |
| 167 | if ( $this->groupSynchronizationCache->groupHasErrors( $groupId ) ) { |
| 168 | $error = "Skipping $groupId due to an error during synchronization\n"; |
| 169 | return Status::newFatal( new RawMessage( $error ) ); |
| 170 | } |
| 171 | } |
| 172 | |
| 173 | return Status::newGood(); |
| 174 | } |
| 175 | |
| 176 | /** Creates MessageUpdateJobs additions for a language under a group */ |
| 177 | private function createMessageUpdateJobs( |
| 178 | FileBasedMessageGroup $group, |
| 179 | array $additions, |
| 180 | string $language |
| 181 | ): array { |
| 182 | $groupId = $group->getId(); |
| 183 | $jobs = []; |
| 184 | $processed = 0; |
| 185 | foreach ( $additions as $addition ) { |
| 186 | $namespace = $group->getNamespace(); |
| 187 | $name = "{$addition['key']}/$language"; |
| 188 | |
| 189 | $title = $this->titleFactory->makeTitleSafe( $namespace, $name ); |
| 190 | if ( !$title ) { |
| 191 | wfWarn( "Invalid title for group $groupId key {$addition['key']}" ); |
| 192 | continue; |
| 193 | } |
| 194 | |
| 195 | $jobs[] = MessageUpdateJob::newJob( $title, $addition['content'] ); |
| 196 | $processed++; |
| 197 | } |
| 198 | |
| 199 | return [ $jobs, $processed ]; |
| 200 | } |
| 201 | |
| 202 | /** |
| 203 | * @param string $groupId |
| 204 | * @param MessageUpdateJob[] $groupJobs |
| 205 | */ |
| 206 | private function updateGroupSyncInfo( string $groupId, array $groupJobs ): void { |
| 207 | $messageParams = []; |
| 208 | $groupMessageKeys = []; |
| 209 | foreach ( $groupJobs as $job ) { |
| 210 | $messageParams[] = MessageUpdateParameter::createFromJob( $job ); |
| 211 | // Ensure there are no duplicates as the same key may be present in |
| 212 | // multiple languages |
| 213 | $groupMessageKeys[( new MessageHandle( $job->getTitle() ) )->getKey()] = true; |
| 214 | } |
| 215 | |
| 216 | $group = MessageGroups::getGroup( $groupId ); |
| 217 | if ( $group === null ) { |
| 218 | // How did we get here? This should never happen. |
| 219 | throw new RuntimeException( "Did not find group $groupId" ); |
| 220 | } |
| 221 | |
| 222 | $this->messageIndex->storeInterim( $group, array_keys( $groupMessageKeys ) ); |
| 223 | |
| 224 | $this->groupSynchronizationCache->addMessages( $groupId, ...$messageParams ); |
| 225 | $this->groupSynchronizationCache->markGroupForSync( $groupId ); |
| 226 | |
| 227 | $this->logger->info( |
| 228 | '[ExternalMessageSourceStateImporter] Synchronization started for {groupId}', |
| 229 | [ 'groupId' => $groupId ] |
| 230 | ); |
| 231 | } |
| 232 | |
| 233 | /** |
| 234 | * Identifies languages in a message group that are safe to import |
| 235 | * @return array<string,bool> |
| 236 | */ |
| 237 | private function identifySafeLanguages( |
| 238 | FileBasedMessageGroup $group, |
| 239 | MessageSourceChange $changesForGroup, |
| 240 | int $importStrategy |
| 241 | ): array { |
| 242 | $sourceLanguage = $group->getSourceLanguage(); |
| 243 | $safeLanguagesMap = []; |
| 244 | $modifiedLanguages = $changesForGroup->getLanguages(); |
| 245 | |
| 246 | // Set all languages to not safe to start with. |
| 247 | $safeLanguagesMap[ $sourceLanguage ] = false; |
| 248 | foreach ( $modifiedLanguages as $language ) { |
| 249 | $safeLanguagesMap[ $language ] = false; |
| 250 | } |
| 251 | |
| 252 | if ( !self::isLanguageSafe( $changesForGroup, $sourceLanguage, $importStrategy ) ) { |
| 253 | return $safeLanguagesMap; |
| 254 | } |
| 255 | |
| 256 | $sourceLanguageKeyCache = []; |
| 257 | foreach ( $changesForGroup->getAdditions( $sourceLanguage ) as $change ) { |
| 258 | if ( $change['content'] === '' ) { |
| 259 | return $safeLanguagesMap; |
| 260 | } |
| 261 | |
| 262 | $sourceLanguageKeyCache[ $change['key'] ] = true; |
| 263 | } |
| 264 | |
| 265 | $safeLanguagesMap[ $sourceLanguage ] = true; |
| 266 | |
| 267 | $groupNamespace = $group->getNamespace(); |
| 268 | |
| 269 | // Remove source language from the modifiedLanguage list if present since it's already processed. |
| 270 | // The $sourceLanguageKeyCache will only have values if sourceLanguage has safe changes. |
| 271 | if ( $sourceLanguageKeyCache ) { |
| 272 | array_splice( $modifiedLanguages, array_search( $sourceLanguage, $modifiedLanguages ), 1 ); |
| 273 | } |
| 274 | |
| 275 | foreach ( $modifiedLanguages as $language ) { |
| 276 | if ( !self::isLanguageSafe( $changesForGroup, $sourceLanguage, $importStrategy ) ) { |
| 277 | continue; |
| 278 | } |
| 279 | |
| 280 | foreach ( $changesForGroup->getAdditions( $language ) as $change ) { |
| 281 | if ( $change['content'] === '' ) { |
| 282 | continue 2; |
| 283 | } |
| 284 | |
| 285 | $msgKey = $change['key']; |
| 286 | |
| 287 | if ( !isset( $sourceLanguageKeyCache[ $msgKey ] ) ) { |
| 288 | // This is either a new external translation which is not added in the same sync |
| 289 | // as the source language key, or this translation does not have a corresponding |
| 290 | // definition. We will check the message index to determine which of the two. |
| 291 | $sourceHandle = new MessageHandle( $this->titleFactory->makeTitle( $groupNamespace, $msgKey ) ); |
| 292 | $sourceLanguageKeyCache[ $msgKey ] = $sourceHandle->isValid(); |
| 293 | } |
| 294 | |
| 295 | if ( !$sourceLanguageKeyCache[ $msgKey ] ) { |
| 296 | continue 2; |
| 297 | } |
| 298 | } |
| 299 | |
| 300 | $safeLanguagesMap[ $language ] = true; |
| 301 | } |
| 302 | |
| 303 | return $safeLanguagesMap; |
| 304 | } |
| 305 | |
| 306 | private static function isLanguageSafe( |
| 307 | MessageSourceChange $changesForGroup, |
| 308 | string $languageCode, |
| 309 | int $importStrategy |
| 310 | ): bool { |
| 311 | if ( $importStrategy === self::IMPORT_NONE ) { |
| 312 | // If import strategy is none, every change needs to be reviewed. |
| 313 | return false; |
| 314 | } |
| 315 | |
| 316 | if ( $importStrategy === self::IMPORT_SAFE ) { |
| 317 | return $changesForGroup->hasOnly( $languageCode, MessageSourceChange::ADDITION ); |
| 318 | } |
| 319 | |
| 320 | // If language has deletions, we consider additions also unsafe because deletions |
| 321 | // maybe renames of messages that have been added, so they have to be reviewed. |
| 322 | return $changesForGroup->getDeletions( $languageCode ) === []; |
| 323 | } |
| 324 | } |