MessageGroupStats.php

Go to the documentation of this file.

<?php
use MediaWiki\Extension\Translate\MessageGroupProcessing\MessageGroups;
use MediaWiki\Extension\Translate\MessageLoading\MessageCollection;
use MediaWiki\Extension\Translate\Utilities\Utilities;
use MediaWiki\Logger\LoggerFactory;
use MediaWiki\MediaWikiServices;
use Wikimedia\Rdbms\Database;
use Wikimedia\Rdbms\IDatabase;
 
class MessageGroupStats {
    private const TABLE = 'translate_groupstats';
    private const LANGUAGE_STATS_KEY = 'translate-all-language-stats';
 
    public const TOTAL = 0; 
    public const TRANSLATED = 1; 
    public const FUZZY = 2; 
    public const PROOFREAD = 3; 
 
    public const FLAG_CACHE_ONLY = 1;
    public const FLAG_NO_CACHE = 2;
    public const FLAG_IMMEDIATE_WRITES = 4;
 
    protected static $updates = [];
    private static $languages;
 
    public static function getEmptyStats() {
        return [ 0, 0, 0, 0 ];
    }
 
    protected static function getUnknownStats() {
        return [ null, null, null, null ];
    }
 
    private static function isValidLanguage( $code ) {
        $languages = self::getLanguages();
        return in_array( $code, $languages );
    }
 
    private static function isValidMessageGroup( MessageGroup $group = null ) {
        /* In case some code calls stats for dynamic groups. Calculating these numbers
         * don't make sense for dynamic groups, and would just throw an exception. */
        return $group && !MessageGroups::isDynamic( $group );
    }
 
    public static function forItem( $id, $code, $flags = 0 ) {
        $group = MessageGroups::getGroup( $id );
        if ( !self::isValidMessageGroup( $group ) || !self::isValidLanguage( $code ) ) {
            return self::getUnknownStats();
        }
 
        $res = self::selectRowsIdLang( [ $id ], [ $code ], $flags );
        $stats = self::extractResults( $res, [ $id ] );
 
        if ( !isset( $stats[$id][$code] ) ) {
            $stats[$id][$code] = self::forItemInternal( $stats, $group, $code, $flags );
        }
 
        self::queueUpdates( $flags );
 
        return $stats[$id][$code];
    }
 
    public static function forLanguage( $code, $flags = 0 ) {
        if ( !self::isValidLanguage( $code ) ) {
            $stats = [];
            $groups = MessageGroups::singleton()->getGroups();
            $ids = array_keys( $groups );
            foreach ( $ids as $id ) {
                $stats[$id] = self::getUnknownStats();
            }
 
            return $stats;
        }
 
        $stats = self::forLanguageInternal( $code, [], $flags );
        $flattened = [];
        foreach ( $stats as $group => $languages ) {
            $flattened[$group] = $languages[$code];
        }
 
        self::queueUpdates( $flags );
 
        return $flattened;
    }
 
    public static function forGroup( $id, $flags = 0 ) {
        $group = MessageGroups::getGroup( $id );
        if ( !self::isValidMessageGroup( $group ) ) {
            $languages = self::getLanguages();
            $stats = [];
            foreach ( $languages as $code ) {
                $stats[$code] = self::getUnknownStats();
            }
 
            return $stats;
        }
 
        $stats = self::forGroupInternal( $group, [], $flags );
 
        self::queueUpdates( $flags );
 
        return $stats[$id];
    }
 
    public static function forEverything( $flags = 0 ) {
        $groups = MessageGroups::singleton()->getGroups();
        $groupIds = array_keys( $groups );
        $languages = self::getLanguages();
 
        // Pre-load cached values
        $res = self::selectRowsIdLang( $groupIds, $languages, $flags );
 
        $stats = [];
        // Go over each group and language filling missing entries
        foreach ( $groups as $groupId => $group ) {
            $stats = self::extractResults( $res, $groupIds, $stats );
            foreach ( $languages as $code ) {
                $stats[$groupId][$code] ??= self::forItemInternal( $stats, $group, $code, $flags );
            }
            // This is for sorting the values added later in correct order
            ksort( $stats[$groupId] );
        }
 
        self::queueUpdates( $flags );
 
        return $stats;
    }
 
    public static function clear( MessageHandle $handle ): void {
        $code = $handle->getCode();
        if ( !self::isValidLanguage( $code ) ) {
            return;
        }
        $groups = self::getSortedGroupsForClearing( $handle->getGroupIds() );
        self::internalClearGroups( $code, $groups, 0 );
    }
 
    public static function clearGroup( $id, int $flags = 0 ): void {
        $languages = self::getLanguages();
        $groups = self::getSortedGroupsForClearing( (array)$id );
 
        // Do one language at a time, to save memory
        foreach ( $languages as $code ) {
            self::internalClearGroups( $code, $groups, $flags );
        }
    }
 
    public static function getApproximateLanguageStats(): array {
        $cache = MediaWikiServices::getInstance()->getMainWANObjectCache();
        return $cache->getWithSetCallback(
            self::LANGUAGE_STATS_KEY,
            WANObjectCache::TTL_INDEFINITE,
            function ( $oldValue, &$ttl, array &$setOpts ) {
                $dbr = Utilities::getSafeReadDB();
                $setOpts += Database::getCacheSetOptions( $dbr );
 
                return self::getAllLanguageStats();
            },
            [
                'checkKeys' => [ self::LANGUAGE_STATS_KEY ],
                'pcTTL' => $cache::TTL_PROC_SHORT,
            ]
        );
    }
 
    private static function getAllLanguageStats(): array {
        $dbr = Utilities::getSafeReadDB();
        $res = $dbr->newSelectQueryBuilder()
            ->table( self::TABLE )
            ->select( [
                'tgs_lang',
                'SUM(tgs_translated) AS tgs_translated',
                'SUM(tgs_fuzzy) AS tgs_fuzzy',
                'SUM(tgs_total) AS tgs_total',
                'SUM(tgs_proofread) AS tgs_proofread'
            ] )
            ->groupBy( 'tgs_lang' )
            ->fetchResultSet();
 
        $allLanguages = self::getLanguages();
        $languagesCodes = array_flip( $allLanguages );
 
        $allStats = [];
        foreach ( $res as $row ) {
            $allStats[ $row->tgs_lang ] = self::extractNumbers( $row );
            unset( $languagesCodes[ $row->tgs_lang ] );
        }
 
        // Fill empty stats for missing language codes
        foreach ( array_keys( $languagesCodes ) as $code ) {
            $allStats[ $code ] = self::getEmptyStats();
        }
 
        return $allStats;
    }
 
    private static function internalClearGroups( $code, array $groups, int $flags ): void {
        $stats = [];
        foreach ( $groups as $group ) {
            // $stats is modified by reference
            self::forItemInternal( $stats, $group, $code, $flags );
        }
        self::queueUpdates( 0 );
    }
 
    private static function getSortedGroupsForClearing( array $ids ) {
        $groups = array_map( [ MessageGroups::class, 'getGroup' ], $ids );
        // Sanity: Remove any invalid groups
        $groups = array_filter( $groups );
 
        $sorted = [];
        $aggs = [];
        foreach ( $groups as $group ) {
            if ( $group instanceof AggregateMessageGroup ) {
                $aggs[$group->getId()] = $group;
            } else {
                $sorted[$group->getId()] = $group;
            }
        }
 
        return array_merge( $sorted, $aggs );
    }
 
    public static function getLanguages() {
        if ( self::$languages === null ) {
            $languages = array_keys( Utilities::getLanguageNames( 'en' ) );
            sort( $languages );
            self::$languages = $languages;
        }
 
        return self::$languages;
    }
 
    public static function clearLanguage( $code ) {
        if ( !count( $code ) ) {
            return;
        }
        $dbw = wfGetDB( DB_PRIMARY );
        $conds = [ 'tgs_lang' => $code ];
        $dbw->delete( self::TABLE, $conds, __METHOD__ );
        wfDebugLog( 'messagegroupstats', 'Cleared ' . serialize( $conds ) );
    }
 
    public static function clearAll() {
        $dbw = wfGetDB( DB_PRIMARY );
        $dbw->delete( self::TABLE, '*', __METHOD__ );
        wfDebugLog( 'messagegroupstats', 'Cleared everything :(' );
    }
 
    protected static function extractResults( $res, array $ids, array $stats = [] ) {
        // Map the internal ids back to real ids
        $idmap = array_combine( array_map( [ self::class, 'getDatabaseIdForGroupId' ], $ids ), $ids );
 
        foreach ( $res as $row ) {
            if ( !isset( $idmap[$row->tgs_group] ) ) {
                // Stale entry, ignore for now
                // TODO: Schedule for purge
                continue;
            }
 
            $realId = $idmap[$row->tgs_group];
            $stats[$realId][$row->tgs_lang] = self::extractNumbers( $row );
        }
 
        return $stats;
    }
 
    protected static function extractNumbers( $row ) {
        return [
            self::TOTAL => (int)$row->tgs_total,
            self::TRANSLATED => (int)$row->tgs_translated,
            self::FUZZY => (int)$row->tgs_fuzzy,
            self::PROOFREAD => (int)$row->tgs_proofread,
        ];
    }
 
    protected static function forLanguageInternal( $code, array $stats, $flags ) {
        $groups = MessageGroups::singleton()->getGroups();
 
        $ids = array_keys( $groups );
        $res = self::selectRowsIdLang( null, [ $code ], $flags );
        $stats = self::extractResults( $res, $ids, $stats );
 
        foreach ( $groups as $id => $group ) {
            if ( isset( $stats[$id][$code] ) ) {
                continue;
            }
            $stats[$id][$code] = self::forItemInternal( $stats, $group, $code, $flags );
        }
 
        return $stats;
    }
 
    protected static function expandAggregates( AggregateMessageGroup $agg ) {
        $flattened = [];
 
        foreach ( $agg->getGroups() as $group ) {
            if ( $group instanceof AggregateMessageGroup ) {
                $flattened += self::expandAggregates( $group );
            } else {
                $flattened[$group->getId()] = $group;
            }
        }
 
        return $flattened;
    }
 
    protected static function forGroupInternal( MessageGroup $group, array $stats, $flags ) {
        $id = $group->getId();
 
        $res = self::selectRowsIdLang( [ $id ], null, $flags );
        $stats = self::extractResults( $res, [ $id ], $stats );
 
        // Go over each language filling missing entries
        $languages = self::getLanguages();
        foreach ( $languages as $code ) {
            if ( isset( $stats[$id][$code] ) ) {
                continue;
            }
            $stats[$id][$code] = self::forItemInternal( $stats, $group, $code, $flags );
        }
 
        // This is for sorting the values added later in correct order
        foreach ( array_keys( $stats ) as $key ) {
            ksort( $stats[$key] );
        }
 
        return $stats;
    }
 
    protected static function selectRowsIdLang( ?array $ids, ?array $codes, $flags ) {
        if ( $flags & self::FLAG_NO_CACHE ) {
            return [];
        }
 
        $conds = [];
        if ( $ids !== null ) {
            $dbids = array_map( [ self::class, 'getDatabaseIdForGroupId' ], $ids );
            $conds['tgs_group'] = $dbids;
        }
 
        if ( $codes !== null ) {
            $conds['tgs_lang'] = $codes;
        }
 
        $dbr = Utilities::getSafeReadDB();
        $res = $dbr->select( self::TABLE, '*', $conds, __METHOD__ );
 
        return $res;
    }
 
    protected static function forItemInternal( &$stats, MessageGroup $group, $code, $flags ) {
        $id = $group->getId();
 
        if ( $flags & self::FLAG_CACHE_ONLY ) {
            $stats[$id][$code] = self::getUnknownStats();
            return $stats[$id][$code];
        }
 
        // It may happen that caches are requested repeatedly for a group before we get a chance
        // to write the values to the database. Check for queued updates first. This has the
        // benefit of avoiding duplicate rows for inserts. Ideally this would be checked before we
        // query the database for missing values. This code is somewhat ugly as it needs to
        // reverse engineer the values from the row format.
        $databaseGroupId = self::getDatabaseIdForGroupId( $id );
        $uniqueKey = "$databaseGroupId|$code";
        $queuedValue = self::$updates[$uniqueKey] ?? null;
        if ( $queuedValue && !( $flags & self::FLAG_NO_CACHE ) ) {
            return [
                self::TOTAL => $queuedValue['tgs_total'],
                self::TRANSLATED => $queuedValue['tgs_translated'],
                self::FUZZY => $queuedValue['tgs_fuzzy'],
                self::PROOFREAD => $queuedValue['tgs_proofread'],
            ];
        }
 
        if ( $group instanceof AggregateMessageGroup ) {
            $aggregates = self::calculateAggregageGroup( $stats, $group, $code, $flags );
        } else {
            $aggregates = self::calculateGroup( $group, $code );
        }
        // Cache for use in subsequent forItemInternal calls
        $stats[$id][$code] = $aggregates;
 
        // Don't add nulls to the database, causes annoying warnings
        if ( $aggregates[self::TOTAL] === null ) {
            return $aggregates;
        }
 
        self::$updates[$uniqueKey] = [
            'tgs_group' => $databaseGroupId,
            'tgs_lang' => $code,
            'tgs_total' => $aggregates[self::TOTAL],
            'tgs_translated' => $aggregates[self::TRANSLATED],
            'tgs_fuzzy' => $aggregates[self::FUZZY],
            'tgs_proofread' => $aggregates[self::PROOFREAD],
        ];
 
        // For big and lengthy updates, attempt some interim saves. This might not have
        // any effect, because writes to the database may be deferred.
        if ( count( self::$updates ) % 100 === 0 ) {
            self::queueUpdates( $flags );
        }
 
        return $aggregates;
    }
 
    private static function calculateAggregageGroup( &$stats, $group, $code, $flags ) {
        $aggregates = self::getEmptyStats();
 
        $expanded = self::expandAggregates( $group );
        $subGroupIds = array_keys( $expanded );
 
        // Performance: if we have per-call cache of stats, do not query them again.
        foreach ( $subGroupIds as $index => $sid ) {
            if ( isset( $stats[$sid][$code] ) ) {
                unset( $subGroupIds[ $index ] );
            }
        }
 
        if ( $subGroupIds !== [] ) {
            $res = self::selectRowsIdLang( $subGroupIds, [ $code ], $flags );
            $stats = self::extractResults( $res, $subGroupIds, $stats );
        }
 
        foreach ( $expanded as $sid => $subgroup ) {
            // Discouraged groups may belong to another group, usually if there
            // is an aggregate group for all translatable pages. In that case
            // calculate and store the statistics, but don't count them as part of
            // the aggregate group, so that the numbers in Special:LanguageStats
            // add up. The statistics for discouraged groups can still be viewed
            // through Special:MessageGroupStats.
            if ( !isset( $stats[$sid][$code] ) ) {
                $stats[$sid][$code] = self::forItemInternal( $stats, $subgroup, $code, $flags );
            }
 
            if ( !TranslateMetadata::isExcluded( $sid, $code ) ) {
                $aggregates = self::multiAdd( $aggregates, $stats[$sid][$code] );
            }
        }
 
        return $aggregates;
    }
 
    public static function multiAdd( &$a, $b ) {
        if ( $a[0] === null || $b[0] === null ) {
            return array_fill( 0, count( $a ), null );
        }
        foreach ( $a as $i => &$v ) {
            $v += $b[$i];
        }
 
        return $a;
    }
 
    protected static function calculateGroup( MessageGroup $group, $code ) {
        global $wgTranslateDocumentationLanguageCode;
        // Calculate if missing and store in the db
        $collection = $group->initCollection( $code );
 
        if (
            $code === $wgTranslateDocumentationLanguageCode
            && $group instanceof FileBasedMessageGroup
        ) {
            $cache = $group->getMessageGroupCache( $group->getSourceLanguage() );
            if ( $cache->exists() ) {
                $template = $cache->getExtra()['TEMPLATE'] ?? [];
                $infile = [];
                foreach ( $template as $key => $data ) {
                    if ( isset( $data['comments']['.'] ) ) {
                        $infile[$key] = '1';
                    }
                }
                $collection->setInFile( $infile );
            }
        }
 
        return self::getStatsForCollection( $collection );
    }
 
    protected static function queueUpdates( $flags ) {
        $mwInstance = MediaWikiServices::getInstance();
        if ( self::$updates === [] || $mwInstance->getReadOnlyMode()->isReadOnly() ) {
            return;
        }
 
        $lb = $mwInstance->getDBLoadBalancer();
        $dbw = $lb->getConnection( DB_PRIMARY ); // avoid connecting yet
        $table = self::TABLE;
        $callers = wfGetAllCallers( 50 );
 
        $updateOp = self::withLock(
            $dbw,
            'updates',
            __METHOD__,
            static function ( IDatabase $dbw, $method ) use ( $table, $callers, $mwInstance ) {
                // Maybe another deferred update already processed these
                if ( self::$updates === [] ) {
                    return;
                }
 
                // This path should only be hit during web requests
                if ( count( self::$updates ) > 100 ) {
                    $groups = array_unique( array_column( self::$updates, 'tgs_group' ) );
                    LoggerFactory::getInstance( 'Translate' )->warning(
                        "Huge translation update of {count} rows for group(s) {groups}",
                        [
                            'count' => count( self::$updates ),
                            'groups' => implode( ', ', $groups ),
                            'callers' => $callers,
                        ]
                    );
                }
 
                $primaryKey = [ 'tgs_group', 'tgs_lang' ];
                $dbw->replace( $table, [ $primaryKey ], array_values( self::$updates ), $method );
                self::$updates = [];
 
                $mwInstance->getMainWANObjectCache()->touchCheckKey( self::LANGUAGE_STATS_KEY );
            }
        );
 
        if ( $flags & self::FLAG_IMMEDIATE_WRITES ) {
            call_user_func( $updateOp );
        } else {
            DeferredUpdates::addCallableUpdate( $updateOp );
        }
    }
 
    protected static function withLock( IDatabase $dbw, $key, $method, $callback ) {
        $fname = __METHOD__;
        return static function () use ( $dbw, $key, $method, $callback, $fname ) {
            $lockName = 'MessageGroupStats:' . $key;
            if ( !$dbw->lock( $lockName, $fname, 1 ) ) {
                return; // raced out
            }
 
            $dbw->commit( $fname, 'flush' );
            call_user_func( $callback, $dbw, $method );
            $dbw->commit( $fname, 'flush' );
 
            $dbw->unlock( $lockName, $fname );
        };
    }
 
    public static function getDatabaseIdForGroupId( $id ) {
        // The column is 100 bytes long, but we don't need to use it all
        if ( strlen( $id ) <= 72 ) {
            return $id;
        }
 
        $hash = hash( 'sha256', $id, /*asHex*/false );
        $dbid = substr( $id, 0, 50 ) . '||' . substr( $hash, 0, 20 );
        return $dbid;
    }
 
    public static function getStatsForCollection( MessageCollection $collection ): array {
        $collection->filter( 'ignored' );
        $collection->filterUntranslatedOptional();
        // Store the count of real messages for later calculation.
        $total = count( $collection );
 
        // Count fuzzy first.
        $collection->filter( 'fuzzy' );
        $fuzzy = $total - count( $collection );
 
        // Count the completed translations.
        $collection->filter( 'hastranslation', false );
        $translated = count( $collection );
 
        // Count how many of the completed translations
        // have been proofread
        $collection->filter( 'reviewer', false );
        $proofread = count( $collection );
 
        return [
            self::TOTAL => $total,
            self::TRANSLATED => $translated,
            self::FUZZY => $fuzzy,
            self::PROOFREAD => $proofread,
        ];
    }
}