Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
0.00% |
0 / 147 |
|
0.00% |
0 / 7 |
CRAP | |
0.00% |
0 / 1 |
| ExternalMessageSourceStateImporter | |
0.00% |
0 / 147 |
|
0.00% |
0 / 7 |
1640 | |
0.00% |
0 / 1 |
| __construct | |
0.00% |
0 / 8 |
|
0.00% |
0 / 1 |
2 | |||
| import | |
0.00% |
0 / 56 |
|
0.00% |
0 / 1 |
132 | |||
| canImportGroup | |
0.00% |
0 / 12 |
|
0.00% |
0 / 1 |
42 | |||
| createUpdateMessageJobs | |
0.00% |
0 / 20 |
|
0.00% |
0 / 1 |
20 | |||
| updateGroupSyncInfo | |
0.00% |
0 / 15 |
|
0.00% |
0 / 1 |
12 | |||
| identifySafeLanguages | |
0.00% |
0 / 31 |
|
0.00% |
0 / 1 |
156 | |||
| isLanguageSafe | |
0.00% |
0 / 5 |
|
0.00% |
0 / 1 |
12 | |||
| 1 | <?php |
| 2 | declare( strict_types = 1 ); |
| 3 | |
| 4 | /** |
| 5 | * Finds external changes for file based message groups. |
| 6 | * @author Niklas Laxström |
| 7 | * @license GPL-2.0-or-later |
| 8 | * @since 2016.02 |
| 9 | */ |
| 10 | namespace MediaWiki\Extension\Translate\Synchronization; |
| 11 | |
| 12 | use FileBasedMessageGroup; |
| 13 | use JobQueueGroup; |
| 14 | use MediaWiki\Config\ServiceOptions; |
| 15 | use MediaWiki\Extension\Translate\MessageGroupProcessing\MessageGroups; |
| 16 | use MediaWiki\Extension\Translate\MessageGroupProcessing\MessageGroupSubscription; |
| 17 | use MediaWiki\Extension\Translate\MessageLoading\MessageHandle; |
| 18 | use MediaWiki\Extension\Translate\MessageLoading\MessageIndex; |
| 19 | use MediaWiki\Extension\Translate\MessageSync\MessageSourceChange; |
| 20 | use MediaWiki\Language\RawMessage; |
| 21 | use MediaWiki\Status\Status; |
| 22 | use MediaWiki\Title\TitleFactory; |
| 23 | use MessageGroup; |
| 24 | use Psr\Log\LoggerInterface; |
| 25 | use RuntimeException; |
| 26 | use function wfWarn; |
| 27 | |
| 28 | class ExternalMessageSourceStateImporter { |
| 29 | private GroupSynchronizationCache $groupSynchronizationCache; |
| 30 | private JobQueueGroup $jobQueueGroup; |
| 31 | private LoggerInterface $logger; |
| 32 | private MessageIndex $messageIndex; |
| 33 | private TitleFactory $titleFactory; |
| 34 | private MessageGroupSubscription $messageGroupSubscription; |
| 35 | private bool $isGroupSyncCacheEnabled; |
| 36 | // Do not perform any import |
| 37 | public const IMPORT_NONE = 1; |
| 38 | // Import changes in a language for a group if it only has additions |
| 39 | public const IMPORT_SAFE = 2; |
| 40 | // Import changes in a language for a group if it only has additions or changes, but |
| 41 | // not deletions as it may be a rename of an addition |
| 42 | public const IMPORT_NON_RENAMES = 3; |
| 43 | public const CONSTRUCTOR_OPTIONS = [ 'TranslateGroupSynchronizationCache' ]; |
| 44 | |
| 45 | public function __construct( |
| 46 | GroupSynchronizationCache $groupSynchronizationCache, |
| 47 | JobQueueGroup $jobQueueGroup, |
| 48 | LoggerInterface $logger, |
| 49 | MessageIndex $messageIndex, |
| 50 | TitleFactory $titleFactory, |
| 51 | MessageGroupSubscription $messageGroupSubscription, |
| 52 | ServiceOptions $options |
| 53 | ) { |
| 54 | $this->groupSynchronizationCache = $groupSynchronizationCache; |
| 55 | $this->jobQueueGroup = $jobQueueGroup; |
| 56 | $this->logger = $logger; |
| 57 | $this->messageIndex = $messageIndex; |
| 58 | $this->titleFactory = $titleFactory; |
| 59 | $this->messageGroupSubscription = $messageGroupSubscription; |
| 60 | $options->assertRequiredOptions( self::CONSTRUCTOR_OPTIONS ); |
| 61 | $this->isGroupSyncCacheEnabled = $options->get( 'TranslateGroupSynchronizationCache' ); |
| 62 | } |
| 63 | |
| 64 | /** |
| 65 | * @param MessageSourceChange[] $changeData |
| 66 | * @param string $name |
| 67 | * @param int $importStrategy |
| 68 | * @return array |
| 69 | */ |
| 70 | public function import( array $changeData, string $name, int $importStrategy ): array { |
| 71 | $processed = []; |
| 72 | $skipped = []; |
| 73 | $jobs = []; |
| 74 | |
| 75 | foreach ( $changeData as $groupId => $changesForGroup ) { |
| 76 | $group = MessageGroups::getGroup( $groupId ); |
| 77 | if ( !$group ) { |
| 78 | unset( $changeData[$groupId] ); |
| 79 | continue; |
| 80 | } |
| 81 | |
| 82 | if ( !$group instanceof FileBasedMessageGroup ) { |
| 83 | $this->logger->warning( |
| 84 | '[ExternalMessageSourceStateImporter] Expected FileBasedMessageGroup, ' . |
| 85 | 'but got {class} for group {groupId}', |
| 86 | [ |
| 87 | 'class' => get_class( $group ), |
| 88 | 'groupId' => $groupId |
| 89 | ] |
| 90 | ); |
| 91 | unset( $changeData[$groupId] ); |
| 92 | continue; |
| 93 | } |
| 94 | |
| 95 | $processed[$groupId] = []; |
| 96 | $languages = $changesForGroup->getLanguages(); |
| 97 | $groupJobs = []; |
| 98 | |
| 99 | $groupSafeLanguages = $this->identifySafeLanguages( $group, $changesForGroup, $importStrategy ); |
| 100 | |
| 101 | foreach ( $languages as $language ) { |
| 102 | if ( !$groupSafeLanguages[ $language ] ) { |
| 103 | $skipped[$groupId] = true; |
| 104 | continue; |
| 105 | } |
| 106 | |
| 107 | $additions = $changesForGroup->getAdditions( $language ); |
| 108 | if ( $additions === [] ) { |
| 109 | continue; |
| 110 | } |
| 111 | |
| 112 | [ $groupLanguageJobs, $groupProcessed ] = $this->createUpdateMessageJobs( |
| 113 | $group, $additions, $language |
| 114 | ); |
| 115 | |
| 116 | $groupJobs = array_merge( $groupJobs, $groupLanguageJobs ); |
| 117 | $processed[$groupId][$language] = $groupProcessed; |
| 118 | |
| 119 | // We only remove additions since if less-safe-import option is used, then |
| 120 | // changes to existing messages might still need to be processed manually. |
| 121 | $changesForGroup->removeAdditions( $language, null ); |
| 122 | $group->getMessageGroupCache( $language )->create(); |
| 123 | } |
| 124 | |
| 125 | // Mark the skipped group as in review |
| 126 | if ( $this->isGroupSyncCacheEnabled && isset( $skipped[$groupId] ) ) { |
| 127 | $this->groupSynchronizationCache->markGroupAsInReview( $groupId ); |
| 128 | } |
| 129 | |
| 130 | if ( $groupJobs !== [] ) { |
| 131 | if ( $this->isGroupSyncCacheEnabled ) { |
| 132 | $this->updateGroupSyncInfo( $groupId, $groupJobs ); |
| 133 | } |
| 134 | $jobs = array_merge( $jobs, $groupJobs ); |
| 135 | } |
| 136 | } |
| 137 | |
| 138 | $this->messageGroupSubscription->queueNotificationJob(); |
| 139 | |
| 140 | // Remove groups where everything was imported |
| 141 | $changeData = array_filter( $changeData, static function ( MessageSourceChange $change ) { |
| 142 | return $change->getAllModifications() !== []; |
| 143 | } ); |
| 144 | |
| 145 | // Remove groups with no imports |
| 146 | $processed = array_filter( $processed ); |
| 147 | |
| 148 | $file = MessageChangeStorage::getCdbPath( $name ); |
| 149 | MessageChangeStorage::writeChanges( $changeData, $file ); |
| 150 | $this->jobQueueGroup->push( $jobs ); |
| 151 | |
| 152 | return [ |
| 153 | 'processed' => $processed, |
| 154 | 'skipped' => $skipped, |
| 155 | 'name' => $name, |
| 156 | ]; |
| 157 | } |
| 158 | |
| 159 | public function canImportGroup( MessageGroup $group, bool $skipGroupSyncCache ): Status { |
| 160 | $groupId = $group->getId(); |
| 161 | if ( !$group instanceof FileBasedMessageGroup ) { |
| 162 | $error = "Group $groupId expected to be FileBasedMessageGroup, got " . get_class( $group ) . " instead."; |
| 163 | return Status::newFatal( new RawMessage( $error ) ); |
| 164 | } |
| 165 | |
| 166 | if ( $this->isGroupSyncCacheEnabled && !$skipGroupSyncCache ) { |
| 167 | if ( $this->groupSynchronizationCache->isGroupBeingProcessed( $groupId ) ) { |
| 168 | $error = "Group $groupId is currently being synchronized; skipping processing of changes\n"; |
| 169 | return Status::newFatal( new RawMessage( $error ) ); |
| 170 | } |
| 171 | |
| 172 | if ( $this->groupSynchronizationCache->groupHasErrors( $groupId ) ) { |
| 173 | $error = "Skipping $groupId due to an error during synchronization\n"; |
| 174 | return Status::newFatal( new RawMessage( $error ) ); |
| 175 | } |
| 176 | } |
| 177 | |
| 178 | return Status::newGood(); |
| 179 | } |
| 180 | |
| 181 | /** |
| 182 | * Creates UpdateMessageJobs additions for a language under a group. Also queues a message |
| 183 | * for notification if the addition is in the source language |
| 184 | */ |
| 185 | private function createUpdateMessageJobs( |
| 186 | FileBasedMessageGroup $group, |
| 187 | array $additions, |
| 188 | string $language |
| 189 | ): array { |
| 190 | $groupId = $group->getId(); |
| 191 | $isSourceLanguage = $group->getSourceLanguage() === $language; |
| 192 | $jobs = []; |
| 193 | $processed = 0; |
| 194 | foreach ( $additions as $addition ) { |
| 195 | $namespace = $group->getNamespace(); |
| 196 | $name = "{$addition['key']}/$language"; |
| 197 | |
| 198 | $title = $this->titleFactory->makeTitleSafe( $namespace, $name ); |
| 199 | if ( !$title ) { |
| 200 | wfWarn( "Invalid title for group $groupId key {$addition['key']}" ); |
| 201 | continue; |
| 202 | } |
| 203 | |
| 204 | $jobs[] = UpdateMessageJob::newJob( $title, $addition['content'] ); |
| 205 | $processed++; |
| 206 | |
| 207 | if ( $isSourceLanguage ) { |
| 208 | $this->messageGroupSubscription->queueMessage( |
| 209 | $title, |
| 210 | MessageGroupSubscription::STATE_ADDED, |
| 211 | $groupId |
| 212 | ); |
| 213 | } |
| 214 | } |
| 215 | |
| 216 | return [ $jobs, $processed ]; |
| 217 | } |
| 218 | |
| 219 | /** |
| 220 | * @param string $groupId |
| 221 | * @param UpdateMessageJob[] $groupJobs |
| 222 | */ |
| 223 | private function updateGroupSyncInfo( string $groupId, array $groupJobs ): void { |
| 224 | $messageParams = []; |
| 225 | $groupMessageKeys = []; |
| 226 | foreach ( $groupJobs as $job ) { |
| 227 | $messageParams[] = MessageUpdateParameter::createFromJob( $job ); |
| 228 | // Ensure there are no duplicates as the same key may be present in |
| 229 | // multiple languages |
| 230 | $groupMessageKeys[( new MessageHandle( $job->getTitle() ) )->getKey()] = true; |
| 231 | } |
| 232 | |
| 233 | $group = MessageGroups::getGroup( $groupId ); |
| 234 | if ( $group === null ) { |
| 235 | // How did we get here? This should never happen. |
| 236 | throw new RuntimeException( "Did not find group $groupId" ); |
| 237 | } |
| 238 | |
| 239 | $this->messageIndex->storeInterim( $group, array_keys( $groupMessageKeys ) ); |
| 240 | |
| 241 | $this->groupSynchronizationCache->addMessages( $groupId, ...$messageParams ); |
| 242 | $this->groupSynchronizationCache->markGroupForSync( $groupId ); |
| 243 | |
| 244 | $this->logger->info( |
| 245 | '[ExternalMessageSourceStateImporter] Synchronization started for {groupId}', |
| 246 | [ 'groupId' => $groupId ] |
| 247 | ); |
| 248 | } |
| 249 | |
| 250 | /** |
| 251 | * Identifies languages in a message group that are safe to import |
| 252 | * @return array<string,bool> |
| 253 | */ |
| 254 | private function identifySafeLanguages( |
| 255 | FileBasedMessageGroup $group, |
| 256 | MessageSourceChange $changesForGroup, |
| 257 | int $importStrategy |
| 258 | ): array { |
| 259 | $sourceLanguage = $group->getSourceLanguage(); |
| 260 | $safeLanguagesMap = []; |
| 261 | $modifiedLanguages = $changesForGroup->getLanguages(); |
| 262 | |
| 263 | // Set all languages to not safe to start with. |
| 264 | $safeLanguagesMap[ $sourceLanguage ] = false; |
| 265 | foreach ( $modifiedLanguages as $language ) { |
| 266 | $safeLanguagesMap[ $language ] = false; |
| 267 | } |
| 268 | |
| 269 | if ( !self::isLanguageSafe( $changesForGroup, $sourceLanguage, $importStrategy ) ) { |
| 270 | return $safeLanguagesMap; |
| 271 | } |
| 272 | |
| 273 | $sourceLanguageKeyCache = []; |
| 274 | foreach ( $changesForGroup->getAdditions( $sourceLanguage ) as $change ) { |
| 275 | if ( $change['content'] === '' ) { |
| 276 | return $safeLanguagesMap; |
| 277 | } |
| 278 | |
| 279 | $sourceLanguageKeyCache[ $change['key'] ] = true; |
| 280 | } |
| 281 | |
| 282 | $safeLanguagesMap[ $sourceLanguage ] = true; |
| 283 | |
| 284 | $groupNamespace = $group->getNamespace(); |
| 285 | |
| 286 | // Remove source language from the modifiedLanguage list if present since it's already processed. |
| 287 | // The $sourceLanguageKeyCache will only have values if sourceLanguage has safe changes. |
| 288 | if ( $sourceLanguageKeyCache ) { |
| 289 | array_splice( $modifiedLanguages, array_search( $sourceLanguage, $modifiedLanguages ), 1 ); |
| 290 | } |
| 291 | |
| 292 | foreach ( $modifiedLanguages as $language ) { |
| 293 | if ( !self::isLanguageSafe( $changesForGroup, $sourceLanguage, $importStrategy ) ) { |
| 294 | continue; |
| 295 | } |
| 296 | |
| 297 | foreach ( $changesForGroup->getAdditions( $language ) as $change ) { |
| 298 | if ( $change['content'] === '' ) { |
| 299 | continue 2; |
| 300 | } |
| 301 | |
| 302 | $msgKey = $change['key']; |
| 303 | |
| 304 | if ( !isset( $sourceLanguageKeyCache[ $msgKey ] ) ) { |
| 305 | // This is either a new external translation which is not added in the same sync |
| 306 | // as the source language key, or this translation does not have a corresponding |
| 307 | // definition. We will check the message index to determine which of the two. |
| 308 | $sourceHandle = new MessageHandle( $this->titleFactory->makeTitle( $groupNamespace, $msgKey ) ); |
| 309 | $sourceLanguageKeyCache[ $msgKey ] = $sourceHandle->isValid(); |
| 310 | } |
| 311 | |
| 312 | if ( !$sourceLanguageKeyCache[ $msgKey ] ) { |
| 313 | continue 2; |
| 314 | } |
| 315 | } |
| 316 | |
| 317 | $safeLanguagesMap[ $language ] = true; |
| 318 | } |
| 319 | |
| 320 | return $safeLanguagesMap; |
| 321 | } |
| 322 | |
| 323 | private static function isLanguageSafe( |
| 324 | MessageSourceChange $changesForGroup, |
| 325 | string $languageCode, |
| 326 | int $importStrategy |
| 327 | ): bool { |
| 328 | if ( $importStrategy === self::IMPORT_NONE ) { |
| 329 | // If import strategy is none, every change needs to be reviewed. |
| 330 | return false; |
| 331 | } |
| 332 | |
| 333 | if ( $importStrategy === self::IMPORT_SAFE ) { |
| 334 | return $changesForGroup->hasOnly( $languageCode, MessageSourceChange::ADDITION ); |
| 335 | } |
| 336 | |
| 337 | // If language has deletions, we consider additions also unsafe because deletions |
| 338 | // maybe renames of messages that have been added, so they have to be reviewed. |
| 339 | return $changesForGroup->getDeletions( $languageCode ) === []; |
| 340 | } |
| 341 | } |