27 private RevisionLookup $revisionLookup;
28 private PageStore $pageStore;
30 public function __construct(
32 RevisionLookup $revisionLookup,
35 $this->stringComparator = $stringComparator;
36 $this->revisionLookup = $revisionLookup;
37 $this->pageStore = $pageStore;
66 $this->processLanguage( $group, $sourceLanguage, $changes );
67 unset( $languages[ $sourceLanguage] );
69 foreach ( array_keys( $languages ) as $language ) {
70 $this->processLanguage( $group, $language, $changes );
76 private function processLanguage(
79 MessageSourceChange $changes
81 $cache = $group->getMessageGroupCache( $language );
83 if ( !$cache->isValid( $reason ) ) {
84 $this->addMessageUpdateChanges( $group, $language, $changes, $reason, $cache );
86 if ( $changes->getModificationsForLanguage( $language ) === [] ) {
119 $wiki = $group->initCollection( $language );
120 $wiki->filter(
'hastranslation',
false );
121 $wiki->loadTranslations();
122 $wikiKeys = $wiki->getMessageKeys();
126 $ffs = $group->getFFS();
127 if ( $language === $sourceLanguage && !$ffs->exists( $language ) ) {
129 throw new RuntimeException(
"Source message file for {$group->getId()} does not exist: $path" );
132 $file = $ffs->read( $language );
135 if ( $file ===
false ) {
140 if ( !isset( $file[
'MESSAGES'] ) ) {
141 $id = $group->
getId();
142 $ffsClass = get_class( $ffs );
144 error_log(
"$id has an FFS ($ffsClass) - it didn't return cake for $language" );
149 $fileKeys = array_keys( $file[
'MESSAGES'] );
151 $common = array_intersect( $fileKeys, $wikiKeys );
153 $supportsFuzzy = $ffs->supportsFuzzy();
154 $changesToRemove = [];
156 foreach ( $common as $key ) {
157 $sourceContent = $file[
'MESSAGES'][$key];
159 $wikiMessage = $wiki[$key];
160 $wikiContent = $wikiMessage->translation();
164 $wikiContent = str_replace( TRANSLATE_FUZZY,
'', $wikiContent );
167 if ( $supportsFuzzy ===
'yes' && $wikiMessage->hasTag(
'fuzzy' ) ) {
168 $wikiContent = TRANSLATE_FUZZY . $wikiContent;
171 if ( $ffs->isContentEqual( $sourceContent, $wikiContent ) ) {
179 if ( $reason !== MessageGroupCache::NO_CACHE ) {
180 $cacheContent = $cache->
get( $key );
189 $this->hasCacheEntry( $cache, $wiki, $key ) &&
190 !$ffs->isContentEqual( $wikiContent, $cacheContent ) &&
191 $ffs->isContentEqual( $sourceContent, $cacheContent )
197 if ( $language !== $sourceLanguage ) {
202 if ( $renameMsg !==
null ) {
207 $this->addNonSourceRenames(
208 $changes, $key, $renameMsg[
'key'], $sourceContent, $wikiContent, $language
210 $changesToRemove[] = $key;
214 $changes->
addChange( $language, $key, $sourceContent );
219 $added = array_diff( $fileKeys, $wikiKeys );
220 foreach ( $added as $key ) {
221 $sourceContent = $file[
'MESSAGES'][$key];
222 $changes->
addAddition( $language, $key, $sourceContent );
229 if ( $reason !== MessageGroupCache::NO_CACHE ) {
230 $deleted = array_diff( $wikiKeys, $fileKeys );
231 foreach ( $deleted as $key ) {
232 if ( $cache->
get( $key ) ===
false ) {
237 $changes->
addDeletion( $language, $key, $wiki[$key]->translation() );
241 if ( $language === $sourceLanguage ) {
242 $this->findAndMarkSourceRenames( $changes, $language );
245 $this->checkNonSourceAdditionsForRename(
246 $changes, $sourceLanguage, $language, $wiki, $wikiKeys
260 private function checkNonSourceAdditionsForRename(
261 MessageSourceChange $changes,
262 string $sourceLanguage,
263 string $targetLanguage,
264 MessageCollection $wiki,
267 $additions = $changes->getAdditions( $targetLanguage );
268 if ( $additions === [] ) {
272 $additionsToRemove = [];
273 $deletionsToRemove = [];
274 foreach ( $additions as $addedMsg ) {
275 $addedMsgKey = $addedMsg[
'key'];
278 $renamedSourceMsg = $changes->findMessage(
279 $sourceLanguage, $addedMsgKey, [ MessageSourceChange::RENAME ]
282 if ( $renamedSourceMsg ===
null ) {
288 $deletedSource = $changes->getMatchedMessage( $sourceLanguage, $renamedSourceMsg[
'key'] );
289 if ( $deletedSource ===
null ) {
292 $deletedMsgKey = $deletedSource[
'key'];
293 $deletedMsg = $changes->findMessage(
294 $targetLanguage, $deletedMsgKey, [ MessageSourceChange::DELETION ]
300 if ( $deletedMsg ===
null ) {
302 if ( in_array( $deletedMsgKey, $wikiKeys ) ) {
303 $content = $wiki[ $deletedMsgKey ]->translation();
306 'key' => $deletedMsgKey,
307 'content' => $content
311 $similarityPercent = $this->stringComparator->getSimilarity(
312 $addedMsg[
'content'], $deletedMsg[
'content']
315 $changes->addRename( $targetLanguage, [
316 'key' => $addedMsgKey,
317 'content' => $addedMsg[
'content']
319 'key' => $deletedMsgKey,
320 'content' => $deletedMsg[
'content']
321 ], $similarityPercent );
323 $deletionsToRemove[] = $deletedMsgKey;
324 $additionsToRemove[] = $addedMsgKey;
327 $changes->removeAdditions( $targetLanguage, $additionsToRemove );
328 $changes->removeDeletions( $targetLanguage, $deletionsToRemove );
336 private function findAndMarkSourceRenames( MessageSourceChange $changes,
string $sourceLanguage ): void {
340 $deletions = $changes->getDeletions( $sourceLanguage );
341 $additions = $changes->getAdditions( $sourceLanguage );
342 if ( $deletions === [] || $additions === [] ) {
348 $potentialRenames = [];
349 foreach ( $additions as $addedMsg ) {
350 $addedMsgKey = $addedMsg[
'key'];
352 foreach ( $deletions as $deletedMsg ) {
353 $similarityPercent = $this->stringComparator->getSimilarity(
354 $addedMsg[
'content'], $deletedMsg[
'content']
357 if ( $changes->areStringsSimilar( $similarityPercent ) ) {
358 $potentialRenames[ $addedMsgKey .
'|' . $deletedMsg[
'key'] ] = $similarityPercent;
363 $this->matchRenames( $changes, $potentialRenames, $sourceLanguage );
367 private function addNonSourceRenames(
368 MessageSourceChange $changes,
371 string $sourceContent,
377 'content' => $sourceContent
382 'content' => $wikiContent
385 $similarityPercent = $this->stringComparator->getSimilarity(
386 $sourceContent, $wikiContent
388 $changes->addRename( $language, $addedMsg, $removedMsg, $similarityPercent );
398 private function matchRenames( MessageSourceChange $changes, array $trackRename,
string $language ): void {
399 arsort( $trackRename, SORT_NUMERIC );
401 $alreadyRenamed = $additionsToRemove = $deletionsToRemove = [];
402 foreach ( $trackRename as $key => $similarityPercent ) {
403 [ $addKey, $deleteKey ] = explode(
'|', $key, 2 );
404 if ( isset( $alreadyRenamed[ $addKey ] ) || isset( $alreadyRenamed[ $deleteKey ] ) ) {
410 $alreadyRenamed[ $addKey ] = 1;
411 $alreadyRenamed[ $deleteKey ] = 1;
413 $addMsg = $changes->findMessage( $language, $addKey, [ MessageSourceChange::ADDITION ] );
414 $deleteMsg = $changes->findMessage( $language, $deleteKey, [ MessageSourceChange::DELETION ] );
416 $changes->addRename( $language, $addMsg, $deleteMsg, $similarityPercent );
419 $additionsToRemove[] = $addMsg[
'key'];
421 $deletionsToRemove[] = $deleteMsg[
'key'];
424 $changes->removeAdditions( $language, $additionsToRemove );
425 $changes->removeDeletions( $language, $deletionsToRemove );
433 private function hasCacheEntry(
434 MessageGroupCache $cache,
435 MessageCollection $collection,
438 $cacheContent = $cache->get( $messageKey );
439 if ( $cacheContent !==
false ) {
443 $cacheUpdateTime = $cache->getUpdateTimestamp();
444 $cacheUpdateTime = $cacheUpdateTime !==
false ? MWTimestamp::convert( TS_MW, $cacheUpdateTime ) : false;
446 $pageIdentity = $this->pageStore->getPageForLink( $collection->keys()[ $messageKey ] );
447 $oldestRevision = $this->revisionLookup->getFirstRevision( $pageIdentity );
448 $latestRevision = $this->revisionLookup->getRevisionByTitle( $pageIdentity );
450 $logger = LoggerFactory::getInstance(
'Translate' );
457 $cacheUpdateTime !==
false &&
458 ( $oldestRevision && $oldestRevision->getTimestamp() < $cacheUpdateTime ) &&
459 ( $latestRevision && $cacheUpdateTime < $latestRevision->getTimestamp() )
462 'Expected cache miss for {messageKey} in language: {language}. Cache update time: {cacheUpdateTime}',
464 'messageKey' => $messageKey,
465 'language' => $collection->getLanguage(),
466 'cacheUpdateTime' => $cacheUpdateTime,
467 'oldestRevisionTs' => $oldestRevision->getTimestamp(),
468 'latestRevisionTs' => $latestRevision->getTimestamp()
475 'Unexpected cache miss for {messageKey} in language: {language}. Cache update time: {cacheUpdateTime}',
477 'messageKey' => $messageKey,
478 'language' => $collection->getLanguage(),
479 'cacheUpdateTime' => $cacheUpdateTime,
480 'oldestRevisionTs' => $oldestRevision ? $oldestRevision->getTimestamp() :
'N/A',
481 'latestRevisionTs' => $latestRevision ? $latestRevision->getTimestamp() :
'N/A'