MessageIndex.php

Go to the documentation of this file.

<?php
use Cdb\Reader;
use Cdb\Writer;
use MediaWiki\Extension\Translate\HookRunner;
use MediaWiki\Extension\Translate\MessageGroupProcessing\MessageGroups;
use MediaWiki\Extension\Translate\Services;
use MediaWiki\Extension\Translate\Utilities\Utilities;
use MediaWiki\Logger\LoggerFactory;
use MediaWiki\MediaWikiServices;
 
abstract class MessageIndex {
    private const CACHEKEY = 'Translate-MessageIndex-interim';
 
    private const READ_LATEST = true;
 
    protected static $instance;
    private static $keysCache;
    protected $interimCache;
    private $statusCache;
    private $jobQueueGroup;
    private $hookRunner;
 
    public function __construct() {
        // TODO: Use dependency injection
        $mwInstance = MediaWikiServices::getInstance();
        $this->statusCache = $mwInstance->getMainWANObjectCache();
        $this->jobQueueGroup = $mwInstance->getJobQueueGroup();
        $this->hookRunner = Services::getInstance()->getHookRunner();
    }
 
    public static function singleton(): self {
        if ( self::$instance === null ) {
            self::$instance = Services::getInstance()->getMessageIndex();
        }
 
        return self::$instance;
    }
 
    public static function setInstance( self $instance ) {
        self::$instance = $instance;
    }
 
    public static function getGroupIds( MessageHandle $handle ): array {
        global $wgTranslateMessageNamespaces;
 
        $title = $handle->getTitle();
 
        if ( !$title->inNamespaces( $wgTranslateMessageNamespaces ) ) {
            return [];
        }
 
        $namespace = $title->getNamespace();
        $key = $handle->getKey();
        $normkey = Utilities::normaliseKey( $namespace, $key );
 
        $cache = self::getCache();
        $value = $cache->get( $normkey );
        if ( $value === null ) {
            $value = (array)self::singleton()->getWithCache( $normkey );
            $cache->set( $normkey, $value );
        }
 
        return $value;
    }
 
    private static function getCache() {
        if ( self::$keysCache === null ) {
            self::$keysCache = new MapCacheLRU( 30 );
        }
        return self::$keysCache;
    }
 
    public static function getPrimaryGroupId( MessageHandle $handle ): ?string {
        $groups = self::getGroupIds( $handle );
 
        return count( $groups ) ? array_shift( $groups ) : null;
    }
 
    private function getWithCache( $key ) {
        $interimCacheValue = $this->getInterimCache()->get( self::CACHEKEY );
        if ( $interimCacheValue && isset( $interimCacheValue['newKeys'][$key] ) ) {
            return $interimCacheValue['newKeys'][$key];
        }
 
        return $this->get( $key );
    }
 
    protected function get( $key ) {
        // Default implementation
        $mi = $this->retrieve();
        return $mi[$key] ?? null;
    }
 
    abstract public function retrieve( bool $readLatest = false ): array;
 
    public function getKeys() {
        return array_keys( $this->retrieve() );
    }
 
    abstract protected function store( array $array, array $diff );
 
    protected function lock() {
        return true;
    }
 
    protected function unlock() {
        return true;
    }
 
    public function rebuild( float $timestamp = null ): array {
        $logger = LoggerFactory::getInstance( 'Translate' );
 
        static $recursion = 0;
 
        if ( $recursion > 0 ) {
            $msg = __METHOD__ . ': trying to recurse - building the index first time?';
            wfWarn( $msg );
 
            $recursion--;
            return [];
        }
        $recursion++;
 
        $logger->info(
            '[MessageIndex] Started rebuild. Initiated by {callers}',
            [ 'callers' => wfGetAllCallers( 20 ) ]
        );
 
        $groups = MessageGroups::singleton()->getGroups();
 
        $tsStart = microtime( true );
        if ( !$this->lock() ) {
            throw new MessageIndexException( __CLASS__ . ': unable to acquire lock' );
        }
 
        $lockWaitDuration = microtime( true ) - $tsStart;
        $logger->info(
            '[MessageIndex] Got lock in {duration}',
            [ 'duration' => $lockWaitDuration ]
        );
 
        self::getCache()->clear();
 
        $new = [];
        $old = $this->retrieve( self::READ_LATEST );
        $postponed = [];
 
        foreach ( $groups as $g ) {
            if ( !$g->exists() ) {
                $id = $g->getId();
                wfWarn( __METHOD__ . ": group '$id' is registered but does not exist" );
                continue;
            }
 
            # Skip meta thingies
            if ( $g->isMeta() ) {
                $postponed[] = $g;
                continue;
            }
 
            $this->checkAndAdd( $new, $g );
        }
 
        foreach ( $postponed as $g ) {
            $this->checkAndAdd( $new, $g, true );
        }
 
        $diff = self::getArrayDiff( $old, $new );
        $this->store( $new, $diff['keys'] );
        $this->unlock();
 
        $criticalSectionDuration = microtime( true ) - $tsStart - $lockWaitDuration;
        $logger->info(
            '[MessageIndex] Finished critical section in {duration}',
            [ 'duration' => $criticalSectionDuration ]
        );
 
        $cache = $this->getInterimCache();
        $interimCacheValue = $cache->get( self::CACHEKEY );
        if ( $interimCacheValue ) {
            $timestamp ??= microtime( true );
            if ( $interimCacheValue['timestamp'] <= $timestamp ) {
                $cache->delete( self::CACHEKEY );
            } else {
                // Cache has a later timestamp. This may be caused due to
                // job deduplication. Just in case, spin off a new job to clean up the cache.
                $job = MessageIndexRebuildJob::newJob();
                $this->jobQueueGroup->push( $job );
            }
        }
 
        // Other caches can check this key to know when they need to refresh
        $this->statusCache->touchCheckKey( $this->getStatusCacheKey() );
 
        $this->clearMessageGroupStats( $diff );
 
        $recursion--;
 
        return $new;
    }
 
    public function getStatusCacheKey(): string {
        return $this->statusCache->makeKey( 'Translate', 'MessageIndex', 'status' );
    }
 
    private function getInterimCache(): BagOStuff {
        return ObjectCache::getInstance( CACHE_ANYTHING );
    }
 
    public function storeInterim( MessageGroup $group, array $newKeys ): void {
        $namespace = $group->getNamespace();
        $id = $group->getId();
 
        $normalizedNewKeys = [];
        foreach ( $newKeys as $key ) {
            $normalizedNewKeys[Utilities::normaliseKey( $namespace, $key )] = $id;
        }
 
        $cache = $this->getInterimCache();
        // Merge existing with existing keys
        $interimCacheValue = $cache->get( self::CACHEKEY, $cache::READ_LATEST );
        if ( $interimCacheValue ) {
            $normalizedNewKeys = array_merge( $interimCacheValue['newKeys'], $normalizedNewKeys );
        }
 
        $value = [
            'timestamp' => microtime( true ),
            'newKeys' => $normalizedNewKeys,
        ];
 
        $cache->set( self::CACHEKEY, $value, $cache::TTL_DAY );
    }
 
    public static function getArrayDiff( array $old, array $new ) {
        $values = [];
        $record = static function ( $groups ) use ( &$values ) {
            foreach ( $groups as $group ) {
                $values[$group] = true;
            }
        };
 
        $keys = [
            'add' => [],
            'del' => [],
            'mod' => [],
        ];
 
        foreach ( $new as $key => $groups ) {
            if ( !isset( $old[$key] ) ) {
                $keys['add'][$key] = [ [], (array)$groups ];
                $record( (array)$groups );
            // Using != here on purpose to ignore the order of items
            } elseif ( $groups != $old[$key] ) {
                $keys['mod'][$key] = [ (array)$old[$key], (array)$groups ];
                $record( array_diff( (array)$old[$key], (array)$groups ) );
                $record( array_diff( (array)$groups, (array)$old[$key] ) );
            }
        }
 
        foreach ( $old as $key => $groups ) {
            if ( !isset( $new[$key] ) ) {
                $keys['del'][$key] = [ (array)$groups, [] ];
                $record( (array)$groups );
            }
            // We already checked for diffs above
        }
 
        return [
            'keys' => $keys,
            'values' => array_keys( $values ),
        ];
    }
 
    protected function clearMessageGroupStats( array $diff ) {
        $job = MessageGroupStatsRebuildJob::newRefreshGroupsJob( $diff['values'] );
        $this->jobQueueGroup->push( $job );
 
        foreach ( $diff['keys'] as $keys ) {
            foreach ( $keys as $key => $data ) {
                [ $ns, $pagename ] = explode( ':', $key, 2 );
                $title = Title::makeTitle( (int)$ns, $pagename );
                $handle = new MessageHandle( $title );
                [ $oldGroups, $newGroups ] = $data;
                $this->hookRunner->onTranslateEventMessageMembershipChange(
                    $handle, $oldGroups, $newGroups );
            }
        }
    }
 
    protected function checkAndAdd( &$hugearray, MessageGroup $g, $ignore = false ) {
        $keys = $g->getKeys();
        $id = $g->getId();
        $namespace = $g->getNamespace();
 
        foreach ( $keys as $key ) {
            # Force all keys to lower case, because the case doesn't matter and it is
            # easier to do comparing when the case of first letter is unknown, because
            # mediawiki forces it to upper case
            $key = Utilities::normaliseKey( $namespace, $key );
            if ( isset( $hugearray[$key] ) ) {
                if ( !$ignore ) {
                    $to = implode( ', ', (array)$hugearray[$key] );
                    wfWarn( "Key $key already belongs to $to, conflict with $id" );
                }
 
                if ( is_array( $hugearray[$key] ) ) {
                    // Hard work is already done, just add a new reference
                    $hugearray[$key][] = & $id;
                } else {
                    // Store the actual reference, then remove it from array, to not
                    // replace the references value, but to store an array of new
                    // references instead. References are hard!
                    $value = & $hugearray[$key];
                    unset( $hugearray[$key] );
                    $hugearray[$key] = [ &$value, &$id ];
                }
            } else {
                $hugearray[$key] = & $id;
            }
        }
        unset( $id ); // Disconnect the previous references to this $id
    }
 
    protected function serialize( $data ) {
        return is_array( $data ) ? implode( '|', $data ) : $data;
    }
 
    protected function unserialize( $data ) {
        $array = explode( '|', $data );
        return count( $array ) > 1 ? $array : $data;
    }
}
 
class SerializedMessageIndex extends MessageIndex {
    protected $index;
    protected $filename = 'translate_messageindex.ser';
 
    public function retrieve( bool $readLatest = false ): array {
        if ( $this->index !== null ) {
            return $this->index;
        }
 
        $file = Utilities::cacheFile( $this->filename );
        if ( file_exists( $file ) ) {
            $this->index = unserialize( file_get_contents( $file ) );
        } else {
            $this->index = $this->rebuild();
        }
 
        return $this->index;
    }
 
    protected function store( array $array, array $diff ) {
        $file = Utilities::cacheFile( $this->filename );
        file_put_contents( $file, serialize( $array ) );
        $this->index = $array;
    }
}
 
class DatabaseMessageIndex extends MessageIndex {
    protected $index;
 
    protected function lock() {
        $dbw = wfGetDB( DB_PRIMARY );
 
        // Any transaction should be flushed after getting the lock to avoid
        // stale pre-lock REPEATABLE-READ snapshot data.
        $ok = $dbw->lock( 'translate-messageindex', __METHOD__, 30 );
        if ( $ok ) {
            $dbw->commit( __METHOD__, 'flush' );
        }
 
        return $ok;
    }
 
    protected function unlock() {
        $fname = __METHOD__;
        $dbw = wfGetDB( DB_PRIMARY );
        // Unlock once the rows are actually unlocked to avoid deadlocks
        if ( !$dbw->trxLevel() ) {
            $dbw->unlock( 'translate-messageindex', $fname );
        } elseif ( is_callable( [ $dbw, 'onTransactionResolution' ] ) ) { // 1.28
            $dbw->onTransactionResolution( static function () use ( $dbw, $fname ) {
                $dbw->unlock( 'translate-messageindex', $fname );
            }, $fname );
        } else {
            $dbw->onTransactionCommitOrIdle( static function () use ( $dbw, $fname ) {
                $dbw->unlock( 'translate-messageindex', $fname );
            }, $fname );
        }
 
        return true;
    }
 
    public function retrieve( bool $readLatest = false ): array {
        if ( $this->index !== null && !$readLatest ) {
            return $this->index;
        }
 
        $dbr = wfGetDB( $readLatest ? DB_PRIMARY : DB_REPLICA );
        $res = $dbr->select( 'translate_messageindex', '*', [], __METHOD__ );
        $this->index = [];
        foreach ( $res as $row ) {
            $this->index[$row->tmi_key] = $this->unserialize( $row->tmi_value );
        }
 
        return $this->index;
    }
 
    protected function get( $key ) {
        $dbr = wfGetDB( DB_REPLICA );
        $value = $dbr->selectField(
            'translate_messageindex',
            'tmi_value',
            [ 'tmi_key' => $key ],
            __METHOD__
        );
 
        return is_string( $value ) ? $this->unserialize( $value ) : null;
    }
 
    protected function store( array $array, array $diff ) {
        $updates = [];
 
        foreach ( [ $diff['add'], $diff['mod'] ] as $changes ) {
            foreach ( $changes as $key => $data ) {
                [ , $new ] = $data;
                $updates[] = [
                    'tmi_key' => $key,
                    'tmi_value' => $this->serialize( $new ),
                ];
            }
        }
 
        $index = [ 'tmi_key' ];
        $deletions = array_keys( $diff['del'] );
 
        $dbw = wfGetDB( DB_PRIMARY );
        $dbw->startAtomic( __METHOD__ );
 
        if ( $updates !== [] ) {
            $dbw->replace( 'translate_messageindex', [ $index ], $updates, __METHOD__ );
        }
 
        if ( $deletions !== [] ) {
            $dbw->delete( 'translate_messageindex', [ 'tmi_key' => $deletions ], __METHOD__ );
        }
 
        $dbw->endAtomic( __METHOD__ );
 
        $this->index = $array;
    }
}
 
class CachedMessageIndex extends MessageIndex {
    protected $key = 'translate-messageindex';
    protected $cache;
    protected $index;
 
    protected function __construct() {
        parent::__construct();
        $this->cache = ObjectCache::getInstance( CACHE_ANYTHING );
    }
 
    public function retrieve( bool $readLatest = false ): array {
        if ( $this->index !== null ) {
            return $this->index;
        }
 
        $key = $this->cache->makeKey( $this->key );
        $data = $this->cache->get( $key );
        if ( is_array( $data ) ) {
            $this->index = $data;
        } else {
            $this->index = $this->rebuild();
        }
 
        return $this->index;
    }
 
    protected function store( array $array, array $diff ) {
        $key = $this->cache->makeKey( $this->key );
        $this->cache->set( $key, $array );
 
        $this->index = $array;
    }
}
 
class CDBMessageIndex extends MessageIndex {
    protected $index;
    protected $reader;
    protected $filename = 'translate_messageindex.cdb';
 
    public function retrieve( bool $readLatest = false ): array {
        $reader = $this->getReader();
        // This must be below the line above, which may fill the index
        if ( $this->index !== null ) {
            return $this->index;
        }
 
        $this->index = [];
        foreach ( $this->getKeys() as $key ) {
            $this->index[$key] = $this->unserialize( $reader->get( $key ) );
        }
 
        return $this->index;
    }
 
    public function getKeys() {
        $reader = $this->getReader();
        $keys = [];
        $key = $reader->firstkey();
        while ( $key !== false ) {
            $keys[] = $key;
            $key = $reader->nextkey();
        }
 
        return $keys;
    }
 
    protected function get( $key ) {
        $reader = $this->getReader();
        // We might have the full cache loaded
        if ( $this->index !== null ) {
            return $this->index[$key] ?? null;
        }
 
        $value = $reader->get( $key );
        return is_string( $value ) ? $this->unserialize( $value ) : null;
    }
 
    protected function store( array $array, array $diff ) {
        $this->reader = null;
 
        $file = Utilities::cacheFile( $this->filename );
        $cache = Writer::open( $file );
 
        foreach ( $array as $key => $value ) {
            $value = $this->serialize( $value );
            $cache->set( $key, $value );
        }
 
        $cache->close();
 
        $this->index = $array;
    }
 
    protected function getReader() {
        if ( $this->reader ) {
            return $this->reader;
        }
 
        $file = Utilities::cacheFile( $this->filename );
        if ( !file_exists( $file ) ) {
            // Create an empty index to allow rebuild
            $this->store( [], [] );
            $this->index = $this->rebuild();
        }
 
        $this->reader = Reader::open( $file );
        return $this->reader;
    }
}
 
class HashMessageIndex extends MessageIndex {
    protected $index = [];
 
    public function retrieve( bool $readLatest = false ): array {
        return $this->index;
    }
 
    protected function get( $key ) {
        return $this->index[$key] ?? null;
    }
 
    protected function store( array $array, array $diff ) {
        $this->index = $array;
    }
 
    protected function clearMessageGroupStats( array $diff ) {
    }
}