Code Coverage
 
Lines
Functions and Methods
Classes and Traits
Total
88.79% covered (warning)
88.79%
198 / 223
74.29% covered (warning)
74.29%
26 / 35
CRAP
0.00% covered (danger)
0.00%
0 / 1
GroupSynchronizationCache
88.79% covered (warning)
88.79%
198 / 223
74.29% covered (warning)
74.29%
26 / 35
70.95
0.00% covered (danger)
0.00%
0 / 1
 __construct
100.00% covered (success)
100.00%
4 / 4
100.00% covered (success)
100.00%
1 / 1
1
 getGroupsInSync
100.00% covered (success)
100.00%
5 / 5
100.00% covered (success)
100.00%
1 / 1
2
 markGroupForSync
100.00% covered (success)
100.00%
10 / 10
100.00% covered (success)
100.00%
1 / 1
1
 getSyncEndTime
100.00% covered (success)
100.00%
2 / 2
100.00% covered (success)
100.00%
1 / 1
2
 endSync
100.00% covered (success)
100.00%
7 / 7
100.00% covered (success)
100.00%
1 / 1
2
 forceEndSync
100.00% covered (success)
100.00%
2 / 2
100.00% covered (success)
100.00%
1 / 1
1
 addMessages
100.00% covered (success)
100.00%
11 / 11
100.00% covered (success)
100.00%
1 / 1
2
 isGroupBeingProcessed
100.00% covered (success)
100.00%
2 / 2
100.00% covered (success)
100.00%
1 / 1
1
 getGroupMessages
87.50% covered (warning)
87.50%
7 / 8
0.00% covered (danger)
0.00%
0 / 1
3.02
 isMessageBeingProcessed
100.00% covered (success)
100.00%
2 / 2
100.00% covered (success)
100.00%
1 / 1
1
 getSynchronizationStatus
63.16% covered (warning)
63.16%
12 / 19
0.00% covered (danger)
0.00%
0 / 1
4.80
 removeMessages
100.00% covered (success)
100.00%
2 / 2
100.00% covered (success)
100.00%
1 / 1
1
 addGroupErrors
100.00% covered (success)
100.00%
31 / 31
100.00% covered (success)
100.00%
1 / 1
4
 getGroupsWithErrors
87.50% covered (warning)
87.50%
7 / 8
0.00% covered (danger)
0.00%
0 / 1
3.02
 getGroupErrorInfo
90.00% covered (success)
90.00%
18 / 20
0.00% covered (danger)
0.00%
0 / 1
6.04
 markGroupAsResolved
100.00% covered (success)
100.00%
7 / 7
100.00% covered (success)
100.00%
1 / 1
2
 markMessageAsResolved
57.14% covered (warning)
57.14%
4 / 7
0.00% covered (danger)
0.00%
0 / 1
2.31
 groupHasErrors
100.00% covered (success)
100.00%
2 / 2
100.00% covered (success)
100.00%
1 / 1
1
 syncGroupErrors
100.00% covered (success)
100.00%
6 / 6
100.00% covered (success)
100.00%
1 / 1
2
 getGroupsInReview
0.00% covered (danger)
0.00%
0 / 5
0.00% covered (danger)
0.00%
0 / 1
6
 markGroupAsInReview
100.00% covered (success)
100.00%
10 / 10
100.00% covered (success)
100.00%
1 / 1
1
 markGroupAsReviewed
0.00% covered (danger)
0.00%
0 / 3
0.00% covered (danger)
0.00%
0 / 1
2
 isGroupInReview
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 extendGroupExpiryTime
100.00% covered (success)
100.00%
15 / 15
100.00% covered (success)
100.00%
1 / 1
4
 getGroupExpiryTime
80.00% covered (warning)
80.00%
4 / 5
0.00% covered (danger)
0.00%
0 / 1
2.03
 hasGroupTimedOut
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 getExpireTime
100.00% covered (success)
100.00%
4 / 4
100.00% covered (success)
100.00%
1 / 1
1
 invalidArgument
0.00% covered (danger)
0.00%
0 / 2
0.00% covered (danger)
0.00%
0 / 1
6
 getGroupTag
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 getGroupKey
100.00% covered (success)
100.00%
2 / 2
100.00% covered (success)
100.00%
1 / 1
1
 getMessageKeys
100.00% covered (success)
100.00%
7 / 7
100.00% covered (success)
100.00%
1 / 1
2
 getGroupErrorKey
100.00% covered (success)
100.00%
2 / 2
100.00% covered (success)
100.00%
1 / 1
1
 getMessageErrorKey
100.00% covered (success)
100.00%
7 / 7
100.00% covered (success)
100.00%
1 / 1
2
 getGroupMessageErrorTag
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 getGroupReviewKey
100.00% covered (success)
100.00%
2 / 2
100.00% covered (success)
100.00%
1 / 1
1
1<?php
2declare( strict_types = 1 );
3
4namespace MediaWiki\Extension\Translate\Synchronization;
5
6use DateTime;
7use InvalidArgumentException;
8use LogicException;
9use MediaWiki\Extension\Translate\Cache\PersistentCache;
10use MediaWiki\Extension\Translate\Cache\PersistentCacheEntry;
11use MediaWiki\Logger\LoggerFactory;
12use Psr\Log\LoggerInterface;
13use RuntimeException;
14
15/**
16 * Message group synchronization cache. Handles storage of data in the cache
17 * to track which groups are currently being synchronized.
18 * Stores:
19 *
20 * 1. Groups in sync:
21 *   - Key: {hash($groupId)}_$groupId
22 *   - Value: $groupId
23 *   - Tag: See GroupSynchronizationCache::getGroupsTag()
24 *   - Exptime: Set when startSyncTimer is called
25 *
26 * 2. Message under each group being modified:
27 *   - Key: {hash($groupId_$messageKey)}_$messageKey
28 *   - Value: MessageUpdateParameter
29 *   - Tag: gsc_$groupId
30 *   - Exptime: none
31 *
32 * @author Abijeet Patro
33 * @license GPL-2.0-or-later
34 * @since 2020.06
35 */
36class GroupSynchronizationCache {
37    private PersistentCache $cache;
38    private int $initialTimeoutSeconds;
39    private int $incrementalTimeoutSeconds;
40    /** @var string Cache tag used for groups */
41    private const GROUP_LIST_TAG = 'gsc_%group_in_sync%';
42    /** @var string Cache tag used for tracking groups that have errors */
43    private const GROUP_ERROR_TAG = 'gsc_%group_with_error%';
44    /** @var string Cache tag used for tracking groups that are in review */
45    private const GROUP_IN_REVIEW_TAG = 'gsc_%group_in_review%';
46    private LoggerInterface $logger;
47
48    public function __construct(
49        PersistentCache $cache,
50        int $initialTimeoutSeconds = 2400,
51        int $incrementalTimeoutSeconds = 600
52
53    ) {
54        $this->cache = $cache;
55        // The timeout is set to 40 minutes initially, and then incremented by 10 minutes
56        // each time a message is marked as processed if group is about to expire.
57        $this->initialTimeoutSeconds = $initialTimeoutSeconds;
58        $this->incrementalTimeoutSeconds = $incrementalTimeoutSeconds;
59        $this->logger = LoggerFactory::getInstance( 'Translate.GroupSynchronization' );
60    }
61
62    /**
63     * Get the groups currently in sync
64     * @return string[]
65     */
66    public function getGroupsInSync(): array {
67        $groupsInSyncEntries = $this->cache->getByTag( self::GROUP_LIST_TAG );
68        $groups = [];
69        foreach ( $groupsInSyncEntries as $entry ) {
70            $groups[] = $entry->value();
71        }
72
73        return $groups;
74    }
75
76    /** Start synchronization process for a group and starts the expiry time */
77    public function markGroupForSync( string $groupId ): void {
78        $expTime = $this->getExpireTime( $this->initialTimeoutSeconds );
79        $this->cache->set(
80            new PersistentCacheEntry(
81                $this->getGroupKey( $groupId ),
82                $groupId,
83                $expTime,
84                self::GROUP_LIST_TAG
85            )
86        );
87        $this->logger->debug( 'Started sync for group {groupId}', [ 'groupId' => $groupId ] );
88    }
89
90    public function getSyncEndTime( string $groupId ): ?int {
91        $cacheEntry = $this->cache->get( $this->getGroupKey( $groupId ) );
92        return $cacheEntry ? $cacheEntry[0]->exptime() : null;
93    }
94
95    /** End synchronization for a group. Deletes the group key */
96    public function endSync( string $groupId ): void {
97        if ( $this->cache->hasEntryWithTag( $this->getGroupTag( $groupId ) ) ) {
98            throw new InvalidArgumentException(
99                'Cannot end synchronization for a group that still has messages to be processed.'
100            );
101        }
102
103        $groupKey = $this->getGroupKey( $groupId );
104        $this->cache->delete( $groupKey );
105        $this->logger->debug( 'Ended sync for group {groupId}', [ 'groupId' => $groupId ] );
106    }
107
108    /** End synchronization for a group. Deletes the group key and messages */
109    public function forceEndSync( string $groupId ): void {
110        $this->cache->deleteEntriesWithTag( $this->getGroupTag( $groupId ) );
111        $this->endSync( $groupId );
112    }
113
114    /** Add messages for a group to the cache */
115    public function addMessages( string $groupId, MessageUpdateParameter ...$messageParams ): void {
116        $messagesToAdd = [];
117        $groupTag = $this->getGroupTag( $groupId );
118        foreach ( $messageParams as $messageParam ) {
119            $titleKey = $this->getMessageKeys( $groupId, $messageParam->getPageName() )[0];
120            $messagesToAdd[] = new PersistentCacheEntry(
121                $titleKey,
122                $messageParam,
123                null,
124                $groupTag
125            );
126        }
127
128        $this->cache->set( ...$messagesToAdd );
129    }
130
131    /** Check if the group is in synchronization */
132    public function isGroupBeingProcessed( string $groupId ): bool {
133        $groupEntry = $this->cache->get( $this->getGroupKey( $groupId ) );
134        return $groupEntry !== [];
135    }
136
137    /**
138     * Return all messages in a group
139     * @param string $groupId
140     * @return MessageUpdateParameter[] Returns a key value pair, with the key being the
141     * messageKey and value being MessageUpdateParameter
142     */
143    public function getGroupMessages( string $groupId ): array {
144        $messageEntries = $this->cache->getByTag( $this->getGroupTag( $groupId ) );
145
146        $allMessageParams = [];
147        foreach ( $messageEntries as $entry ) {
148            $message = $entry->value();
149            if ( $message instanceof MessageUpdateParameter ) {
150                $allMessageParams[$message->getPageName()] = $message;
151            } else {
152                // Should not happen, but handle primarily to keep phan happy.
153                throw $this->invalidArgument( $message, MessageUpdateParameter::class );
154            }
155        }
156
157        return $allMessageParams;
158    }
159
160    /** Check if a message is being processed */
161    public function isMessageBeingProcessed( string $groupId, string $messageKey ): bool {
162        $messageCacheKey = $this->getMessageKeys( $groupId, $messageKey );
163        return $this->cache->has( $messageCacheKey[0] );
164    }
165
166    /** Get the current synchronization status of the group. Does not perform any updates. */
167    public function getSynchronizationStatus( string $groupId ): GroupSynchronizationResponse {
168        if ( !$this->isGroupBeingProcessed( $groupId ) ) {
169            // Group is currently not being processed.
170            throw new LogicException(
171                'Sync requested for a group currently not being processed. Check if ' .
172                'group is being processed by calling isGroupBeingProcessed() first'
173            );
174        }
175
176        $remainingMessages = $this->getGroupMessages( $groupId );
177
178        // No messages are present
179        if ( !$remainingMessages ) {
180            return new GroupSynchronizationResponse( $groupId, [], false );
181        }
182
183        $syncExpTime = $this->getSyncEndTime( $groupId );
184        if ( $syncExpTime === null ) {
185            // This should not happen
186            throw new RuntimeException(
187                "Unexpected condition. Group: $groupId; Messages present, but group key not found."
188            );
189        }
190
191        $hasTimedOut = $this->hasGroupTimedOut( $syncExpTime );
192
193        return new GroupSynchronizationResponse(
194            $groupId,
195            $remainingMessages,
196            $hasTimedOut
197        );
198    }
199
200    /** Remove messages from the cache. */
201    public function removeMessages( string $groupId, string ...$messageKeys ): void {
202        $messageCacheKeys = $this->getMessageKeys( $groupId, ...$messageKeys );
203
204        $this->cache->delete( ...$messageCacheKeys );
205    }
206
207    public function addGroupErrors( GroupSynchronizationResponse $response ): void {
208        $groupId = $response->getGroupId();
209        $remainingMessages = $response->getRemainingMessages();
210
211        if ( !$remainingMessages ) {
212            throw new LogicException( 'Cannot add a group without any remaining messages to the errors list' );
213        }
214
215        $groupMessageErrorTag = $this->getGroupMessageErrorTag( $groupId );
216
217        $entriesToSave = [];
218        foreach ( $remainingMessages as $messageParam ) {
219            $titleErrorKey = $this->getMessageErrorKey( $groupId, $messageParam->getPageName() )[0];
220            $entriesToSave[] = new PersistentCacheEntry(
221                $titleErrorKey,
222                $messageParam,
223                null,
224                $groupMessageErrorTag
225            );
226        }
227
228        $this->cache->set( ...$entriesToSave );
229
230        $groupErrorKey = $this->getGroupErrorKey( $groupId );
231
232        // Check if the group already has errors
233        $groupInfo = $this->cache->get( $groupErrorKey );
234        if ( $groupInfo ) {
235            return;
236        }
237
238        // Group did not have an error previously, add it now. When adding,
239        // remove the remaining messages from the GroupSynchronizationResponse to
240        // avoid the value in the cache becoming too big. The remaining messages
241        // are stored as separate items in the cache.
242        $trimmedGroupSyncResponse = new GroupSynchronizationResponse(
243            $groupId,
244            [],
245            $response->hasTimedOut()
246        );
247
248        $entriesToSave[] = new PersistentCacheEntry(
249            $groupErrorKey,
250            $trimmedGroupSyncResponse,
251            null,
252            self::GROUP_ERROR_TAG
253        );
254
255        $this->cache->set( ...$entriesToSave );
256    }
257
258    /**
259     * Return the groups that have errors
260     * @return string[]
261     */
262    public function getGroupsWithErrors(): array {
263        $groupsInSyncEntries = $this->cache->getByTag( self::GROUP_ERROR_TAG );
264        $groupIds = [];
265        foreach ( $groupsInSyncEntries as $entry ) {
266            $groupResponse = $entry->value();
267            if ( $groupResponse instanceof GroupSynchronizationResponse ) {
268                $groupIds[] = $groupResponse->getGroupId();
269            } else {
270                // Should not happen, but handle primarily to keep phan happy.
271                throw $this->invalidArgument( $groupResponse, GroupSynchronizationResponse::class );
272            }
273        }
274
275        return $groupIds;
276    }
277
278    /** Fetch information about a particular group that has errors including messages that failed */
279    public function getGroupErrorInfo( string $groupId ): GroupSynchronizationResponse {
280        $groupMessageErrorTag = $this->getGroupMessageErrorTag( $groupId );
281        $groupMessageEntries = $this->cache->getByTag( $groupMessageErrorTag );
282
283        $groupErrorKey = $this->getGroupErrorKey( $groupId );
284        $groupResponseEntry = $this->cache->get( $groupErrorKey );
285        $groupResponse = $groupResponseEntry[0] ? $groupResponseEntry[0]->value() : null;
286        if ( $groupResponse ) {
287            if ( !$groupResponse instanceof GroupSynchronizationResponse ) {
288                // Should not happen, but handle primarily to keep phan happy.
289                throw $this->invalidArgument( $groupResponse, GroupSynchronizationResponse::class );
290            }
291        } else {
292            throw new LogicException( 'Requested to fetch errors for a group that has no errors.' );
293        }
294
295        $messageParams = [];
296        foreach ( $groupMessageEntries as $messageEntries ) {
297            $messageParam = $messageEntries->value();
298            if ( $messageParam instanceof MessageUpdateParameter ) {
299                $messageParams[] = $messageParam;
300            } else {
301                // Should not happen, but handle primarily to keep phan happy.
302                throw $this->invalidArgument( $messageParam, MessageUpdateParameter::class );
303            }
304        }
305
306        return new GroupSynchronizationResponse(
307            $groupId,
308            $messageParams,
309            $groupResponse->hasTimedOut()
310        );
311    }
312
313    /** Marks all messages in a group and the group itself as resolved */
314    public function markGroupAsResolved( string $groupId ): GroupSynchronizationResponse {
315        $groupSyncResponse = $this->getGroupErrorInfo( $groupId );
316        $errorMessages = $groupSyncResponse->getRemainingMessages();
317
318        $errorMessageKeys = [];
319        foreach ( $errorMessages as $message ) {
320            $errorMessageKeys[] = $this->getMessageErrorKey( $groupId, $message->getPageName() )[0];
321        }
322
323        $this->cache->delete( ...$errorMessageKeys );
324        return $this->syncGroupErrors( $groupId );
325    }
326
327    /** Marks errors for a message as resolved */
328    public function markMessageAsResolved( string $groupId, string $messagePageName ): void {
329        $messageErrorKey = $this->getMessageErrorKey( $groupId, $messagePageName )[0];
330        $messageInCache = $this->cache->get( $messageErrorKey );
331        if ( !$messageInCache ) {
332            throw new InvalidArgumentException(
333                'Message does not appear to have synchronization errors'
334            );
335        }
336
337        $this->cache->delete( $messageErrorKey );
338    }
339
340    /** Checks if the group has errors */
341    public function groupHasErrors( string $groupId ): bool {
342        $groupErrorKey = $this->getGroupErrorKey( $groupId );
343        return $this->cache->has( $groupErrorKey );
344    }
345
346    /** Checks if group has unresolved error messages. If not clears the group from error list */
347    public function syncGroupErrors( string $groupId ): GroupSynchronizationResponse {
348        $groupSyncResponse = $this->getGroupErrorInfo( $groupId );
349        if ( $groupSyncResponse->getRemainingMessages() ) {
350            return $groupSyncResponse;
351        }
352
353        // No remaining messages left, remove group from errors list.
354        $groupErrorKey = $this->getGroupErrorKey( $groupId );
355        $this->cache->delete( $groupErrorKey );
356
357        return $groupSyncResponse;
358    }
359
360    /**
361     * Return groups that are in review
362     * @return string[]
363     */
364    public function getGroupsInReview(): array {
365        $groupsInReviewEntries = $this->cache->getByTag( self::GROUP_IN_REVIEW_TAG );
366        $groups = [];
367        foreach ( $groupsInReviewEntries as $entry ) {
368            $groups[] = $entry->value();
369        }
370
371        return $groups;
372    }
373
374    public function markGroupAsInReview( string $groupId ): void {
375        $groupReviewKey = $this->getGroupReviewKey( $groupId );
376        $this->cache->set(
377            new PersistentCacheEntry(
378                $groupReviewKey,
379                $groupId,
380                null,
381                self::GROUP_IN_REVIEW_TAG
382            )
383        );
384        $this->logger->debug( 'Group {groupId} marked for review', [ 'groupId' => $groupId ] );
385    }
386
387    public function markGroupAsReviewed( string $groupId ): void {
388        $groupReviewKey = $this->getGroupReviewKey( $groupId );
389        $this->cache->delete( $groupReviewKey );
390        $this->logger->debug( 'Group {groupId} removed from review', [ 'groupId' => $groupId ] );
391    }
392
393    public function isGroupInReview( string $groupId ): bool {
394        return $this->cache->has( $this->getGroupReviewKey( $groupId ) );
395    }
396
397    public function extendGroupExpiryTime( string $groupId ): void {
398        $groupKey = $this->getGroupKey( $groupId );
399        $groupEntry = $this->cache->get( $groupKey );
400
401        if ( $groupEntry === [] ) {
402            // Group is currently not being processed.
403            throw new LogicException(
404                'Requested extension of expiry time for a group that is not being processed. ' .
405                'Check if group is being processed by calling isGroupBeingProcessed() first'
406            );
407        }
408
409        if ( $groupEntry[0]->hasExpired() ) {
410            throw new InvalidArgumentException(
411                'Cannot extend expiry time for a group that has already expired.'
412            );
413        }
414
415        $newExpiryTime = $this->getExpireTime( $this->incrementalTimeoutSeconds );
416
417        // We start with the initial timeout minutes, we only change the timeout if the group
418        // is actually about to expire.
419        if ( $newExpiryTime < $groupEntry[0]->exptime() ) {
420            return;
421        }
422
423        $this->cache->setExpiry( $groupKey, $newExpiryTime );
424    }
425
426    /** @internal - Internal; For testing use only */
427    public function getGroupExpiryTime( string $groupId ): int {
428        $groupKey = $this->getGroupKey( $groupId );
429        $groupEntry = $this->cache->get( $groupKey );
430        if ( $groupEntry === [] ) {
431            throw new InvalidArgumentException( "$groupId currently not in processing!" );
432        }
433
434        return $groupEntry[0]->exptime();
435    }
436
437    private function hasGroupTimedOut( int $syncExpTime ): bool {
438        return ( new DateTime() )->getTimestamp() > $syncExpTime;
439    }
440
441    private function getExpireTime( int $timeoutSeconds ): int {
442        $currentTime = ( new DateTime() )->getTimestamp();
443        return ( new DateTime() )
444            ->setTimestamp( $currentTime + $timeoutSeconds )
445            ->getTimestamp();
446    }
447
448    private function invalidArgument( $value, string $expectedType ): RuntimeException {
449        $valueType = $value ? get_class( $value ) : gettype( $value );
450        return new RuntimeException( "Expected $expectedType, got $valueType" );
451    }
452
453    // Cache keys / tag related functions start here.
454
455    private function getGroupTag( string $groupId ): string {
456        return 'gsc_' . $groupId;
457    }
458
459    private function getGroupKey( string $groupId ): string {
460        $hash = substr( hash( 'sha256', $groupId ), 0, 40 );
461        return substr( "{$hash}_$groupId", 0, 255 );
462    }
463
464    /** @return string[] */
465    private function getMessageKeys( string $groupId, string ...$messages ): array {
466        $messageKeys = [];
467        foreach ( $messages as $message ) {
468            $key = $groupId . '_' . $message;
469            $hash = substr( hash( 'sha256', $key ), 0, 40 );
470            $finalKey = substr( $hash . '_' . $key, 0, 255 );
471            $messageKeys[] = $finalKey;
472        }
473
474        return $messageKeys;
475    }
476
477    private function getGroupErrorKey( string $groupId ): string {
478        $hash = substr( hash( 'sha256', $groupId ), 0, 40 );
479        return substr( "{$hash}_gsc_error_$groupId", 0, 255 );
480    }
481
482    /** @return string[] */
483    private function getMessageErrorKey( string $groupId, string ...$messages ): array {
484        $messageKeys = [];
485        foreach ( $messages as $message ) {
486            $key = $groupId . '_' . $message;
487            $hash = substr( hash( 'sha256', $key ), 0, 40 );
488            $finalKey = substr( $hash . '_gsc_error_' . $key, 0, 255 );
489            $messageKeys[] = $finalKey;
490        }
491
492        return $messageKeys;
493    }
494
495    private function getGroupMessageErrorTag( string $groupId ): string {
496        return "gsc_%error%_$groupId";
497    }
498
499    private function getGroupReviewKey( string $groupId ): string {
500        $hash = substr( hash( 'sha256', $groupId ), 0, 40 );
501        return substr( "{$hash}_gsc_%review%_$groupId", 0, 255 );
502    }
503}