28 private const TABLE =
'translate_groupstats';
30 private const LANGUAGE_STATS_KEY =
'translate-all-language-stats';
45 protected static $updates = [];
47 private static $languages;
56 return [ 0, 0, 0, 0 ];
66 return [
null,
null,
null, null ];
69 private static function isValidLanguage( $code ) {
70 $languages = self::getLanguages();
71 return in_array( $code, $languages );
74 private static function isValidMessageGroup(
MessageGroup $group =
null ) {
77 return $group && !MessageGroups::isDynamic( $group );
87 public static function forItem( $id, $code, $flags = 0 ) {
88 $group = MessageGroups::getGroup( $id );
89 if ( !self::isValidMessageGroup( $group ) || !self::isValidLanguage( $code ) ) {
90 return self::getUnknownStats();
93 $res = self::selectRowsIdLang( [ $id ], [ $code ], $flags );
94 $stats = self::extractResults( $res, [ $id ] );
96 if ( !isset( $stats[$id][$code] ) ) {
97 $stats[$id][$code] = self::forItemInternal( $stats, $group, $code, $flags );
100 self::queueUpdates( $flags );
102 return $stats[$id][$code];
112 if ( !self::isValidLanguage( $code ) ) {
114 $groups = MessageGroups::singleton()->getGroups();
115 $ids = array_keys( $groups );
116 foreach ( $ids as $id ) {
117 $stats[$id] = self::getUnknownStats();
123 $stats = self::forLanguageInternal( $code, [], $flags );
125 foreach ( $stats as $group => $languages ) {
126 $flattened[$group] = $languages[$code];
129 self::queueUpdates( $flags );
140 public static function forGroup( $id, $flags = 0 ) {
141 $group = MessageGroups::getGroup( $id );
142 if ( !self::isValidMessageGroup( $group ) ) {
143 $languages = self::getLanguages();
145 foreach ( $languages as $code ) {
146 $stats[$code] = self::getUnknownStats();
152 $stats = self::forGroupInternal( $group, [], $flags );
154 self::queueUpdates( $flags );
167 $groups = MessageGroups::singleton()->getGroups();
168 $groupIds = array_keys( $groups );
169 $languages = self::getLanguages();
172 $res = self::selectRowsIdLang( $groupIds, $languages, $flags );
176 foreach ( $groups as $groupId => $group ) {
177 $stats = self::extractResults( $res, $groupIds, $stats );
178 foreach ( $languages as $code ) {
179 $stats[$groupId][$code] ??= self::forItemInternal( $stats, $group, $code, $flags );
182 ksort( $stats[$groupId] );
185 self::queueUpdates( $flags );
197 $code = $handle->getCode();
198 if ( !self::isValidLanguage( $code ) ) {
201 $groups = self::getSortedGroupsForClearing( $handle->
getGroupIds() );
202 self::internalClearGroups( $code, $groups, 0 );
211 public static function clearGroup( $id,
int $flags = 0 ): void {
212 $languages = self::getLanguages();
213 $groups = self::getSortedGroupsForClearing( (array)$id );
216 foreach ( $languages as $code ) {
217 self::internalClearGroups( $code, $groups, $flags );
228 $cache = MediaWikiServices::getInstance()->getMainWANObjectCache();
229 return $cache->getWithSetCallback(
230 self::LANGUAGE_STATS_KEY,
231 WANObjectCache::TTL_INDEFINITE,
232 function ( $oldValue, &$ttl, array &$setOpts ) {
233 $dbr = Utilities::getSafeReadDB();
234 $setOpts += Database::getCacheSetOptions( $dbr );
236 return self::getAllLanguageStats();
239 'checkKeys' => [ self::LANGUAGE_STATS_KEY ],
240 'pcTTL' => $cache::TTL_PROC_SHORT,
245 private static function getAllLanguageStats(): array {
247 $res = $dbr->newSelectQueryBuilder()
248 ->table( self::TABLE )
251 'SUM(tgs_translated) AS tgs_translated',
252 'SUM(tgs_fuzzy) AS tgs_fuzzy',
253 'SUM(tgs_total) AS tgs_total',
254 'SUM(tgs_proofread) AS tgs_proofread'
256 ->groupBy(
'tgs_lang' )
259 $allLanguages = self::getLanguages();
260 $languagesCodes = array_flip( $allLanguages );
263 foreach ( $res as $row ) {
264 $allStats[ $row->tgs_lang ] = self::extractNumbers( $row );
265 unset( $languagesCodes[ $row->tgs_lang ] );
269 foreach ( array_keys( $languagesCodes ) as $code ) {
270 $allStats[ $code ] = self::getEmptyStats();
283 private static function internalClearGroups( $code, array $groups,
int $flags ): void {
285 foreach ( $groups as $group ) {
287 self::forItemInternal( $stats, $group, $code, $flags );
289 self::queueUpdates( 0 );
303 private static function getSortedGroupsForClearing( array $ids ) {
304 $groups = array_map( [ MessageGroups::class,
'getGroup' ], $ids );
306 $groups = array_filter( $groups );
310 foreach ( $groups as $group ) {
312 $aggs[$group->getId()] = $group;
314 $sorted[$group->getId()] = $group;
318 return array_merge( $sorted, $aggs );
327 if ( self::$languages ===
null ) {
328 $languages = array_keys( Utilities::getLanguageNames(
'en' ) );
330 self::$languages = $languages;
333 return self::$languages;
336 public static function clearLanguage( $code ) {
337 if ( !count( $code ) ) {
340 $dbw = wfGetDB( DB_PRIMARY );
341 $conds = [
'tgs_lang' => $code ];
342 $dbw->delete( self::TABLE, $conds, __METHOD__ );
343 wfDebugLog(
'messagegroupstats',
'Cleared ' . serialize( $conds ) );
352 $dbw = wfGetDB( DB_PRIMARY );
353 $dbw->delete( self::TABLE,
'*', __METHOD__ );
354 wfDebugLog(
'messagegroupstats',
'Cleared everything :(' );
366 protected static function extractResults( $res, array $ids, array $stats = [] ) {
368 $idmap = array_combine( array_map( [ self::class,
'getDatabaseIdForGroupId' ], $ids ), $ids );
370 foreach ( $res as $row ) {
371 if ( !isset( $idmap[$row->tgs_group] ) ) {
377 $realId = $idmap[$row->tgs_group];
378 $stats[$realId][$row->tgs_lang] = self::extractNumbers( $row );
391 self::TOTAL => (int)$row->tgs_total,
392 self::TRANSLATED => (
int)$row->tgs_translated,
393 self::FUZZY => (int)$row->tgs_fuzzy,
394 self::PROOFREAD => (
int)$row->tgs_proofread,
405 $groups = MessageGroups::singleton()->getGroups();
407 $ids = array_keys( $groups );
408 $res = self::selectRowsIdLang(
null, [ $code ], $flags );
409 $stats = self::extractResults( $res, $ids, $stats );
411 foreach ( $groups as $id => $group ) {
412 if ( isset( $stats[$id][$code] ) ) {
415 $stats[$id][$code] = self::forItemInternal( $stats, $group, $code, $flags );
429 foreach ( $agg->
getGroups() as $group ) {
431 $flattened += self::expandAggregates( $group );
433 $flattened[$group->getId()] = $group;
447 $id = $group->
getId();
449 $res = self::selectRowsIdLang( [ $id ],
null, $flags );
450 $stats = self::extractResults( $res, [ $id ], $stats );
453 $languages = self::getLanguages();
454 foreach ( $languages as $code ) {
455 if ( isset( $stats[$id][$code] ) ) {
458 $stats[$id][$code] = self::forItemInternal( $stats, $group, $code, $flags );
462 foreach ( array_keys( $stats ) as $key ) {
463 ksort( $stats[$key] );
478 if ( $flags & self::FLAG_NO_CACHE ) {
483 if ( $ids !==
null ) {
484 $dbids = array_map( [ self::class,
'getDatabaseIdForGroupId' ], $ids );
485 $conds[
'tgs_group'] = $dbids;
488 if ( $codes !==
null ) {
489 $conds[
'tgs_lang'] = $codes;
492 $dbr = Utilities::getSafeReadDB();
493 $res = $dbr->select( self::TABLE,
'*', $conds, __METHOD__ );
506 $id = $group->
getId();
508 if ( $flags & self::FLAG_CACHE_ONLY ) {
509 $stats[$id][$code] = self::getUnknownStats();
510 return $stats[$id][$code];
518 $databaseGroupId = self::getDatabaseIdForGroupId( $id );
519 $uniqueKey =
"$databaseGroupId|$code";
520 $queuedValue = self::$updates[$uniqueKey] ??
null;
521 if ( $queuedValue && !( $flags & self::FLAG_NO_CACHE ) ) {
523 self::TOTAL => $queuedValue[
'tgs_total'],
524 self::TRANSLATED => $queuedValue[
'tgs_translated'],
525 self::FUZZY => $queuedValue[
'tgs_fuzzy'],
526 self::PROOFREAD => $queuedValue[
'tgs_proofread'],
531 $aggregates = self::calculateAggregageGroup( $stats, $group, $code, $flags );
533 $aggregates = self::calculateGroup( $group, $code );
536 $stats[$id][$code] = $aggregates;
539 if ( $aggregates[self::TOTAL] ===
null ) {
543 self::$updates[$uniqueKey] = [
544 'tgs_group' => $databaseGroupId,
546 'tgs_total' => $aggregates[self::TOTAL],
547 'tgs_translated' => $aggregates[self::TRANSLATED],
548 'tgs_fuzzy' => $aggregates[self::FUZZY],
549 'tgs_proofread' => $aggregates[self::PROOFREAD],
554 if ( count( self::$updates ) % 100 === 0 ) {
555 self::queueUpdates( $flags );
561 private static function calculateAggregageGroup( &$stats, $group, $code, $flags ) {
562 $aggregates = self::getEmptyStats();
564 $expanded = self::expandAggregates( $group );
565 $subGroupIds = array_keys( $expanded );
568 foreach ( $subGroupIds as $index => $sid ) {
569 if ( isset( $stats[$sid][$code] ) ) {
570 unset( $subGroupIds[ $index ] );
574 if ( $subGroupIds !== [] ) {
575 $res = self::selectRowsIdLang( $subGroupIds, [ $code ], $flags );
576 $stats = self::extractResults( $res, $subGroupIds, $stats );
579 foreach ( $expanded as $sid => $subgroup ) {
586 if ( !isset( $stats[$sid][$code] ) ) {
587 $stats[$sid][$code] = self::forItemInternal( $stats, $subgroup, $code, $flags );
590 if ( !TranslateMetadata::isExcluded( $sid, $code ) ) {
591 $aggregates = self::multiAdd( $aggregates, $stats[$sid][$code] );
598 public static function multiAdd( &$a, $b ) {
599 if ( $a[0] ===
null || $b[0] ===
null ) {
600 return array_fill( 0, count( $a ),
null );
602 foreach ( $a as $i => &$v ) {
615 global $wgTranslateDocumentationLanguageCode;
620 $code === $wgTranslateDocumentationLanguageCode
624 if ( $cache->exists() ) {
625 $template = $cache->getExtra()[
'TEMPLATE'] ?? [];
627 foreach ( $template as $key => $data ) {
628 if ( isset( $data[
'comments'][
'.'] ) ) {
632 $collection->setInFile( $infile );
636 return self::getStatsForCollection( $collection );
639 protected static function queueUpdates( $flags ) {
640 $mwInstance = MediaWikiServices::getInstance();
641 if ( $mwInstance->getReadOnlyMode()->isReadOnly() ) {
645 if ( self::$updates === [] ) {
649 $lb = $mwInstance->getDBLoadBalancer();
650 $dbw = $lb->getConnectionRef( DB_PRIMARY );
651 $table = self::TABLE;
652 $callers = wfGetAllCallers( 50 );
654 $updateOp = self::withLock(
658 static function ( IDatabase $dbw, $method ) use ( $table, $callers, $mwInstance ) {
660 if ( self::$updates === [] ) {
665 if ( count( self::$updates ) > 100 ) {
666 $groups = array_unique( array_column( self::$updates,
'tgs_group' ) );
667 LoggerFactory::getInstance(
'Translate' )->warning(
668 "Huge translation update of {count} rows for group(s) {groups}",
670 'count' => count( self::$updates ),
671 'groups' => implode(
', ', $groups ),
672 'callers' => $callers,
677 $primaryKey = [
'tgs_group',
'tgs_lang' ];
678 $dbw->replace( $table, [ $primaryKey ], array_values( self::$updates ), $method );
681 $mwInstance->getMainWANObjectCache()->touchCheckKey( self::LANGUAGE_STATS_KEY );
685 if ( $flags & self::FLAG_IMMEDIATE_WRITES ) {
686 call_user_func( $updateOp );
688 DeferredUpdates::addCallableUpdate( $updateOp );
692 protected static function withLock( IDatabase $dbw, $key, $method, $callback ) {
694 return static function () use ( $dbw, $key, $method, $callback, $fname ) {
695 $lockName =
'MessageGroupStats:' . $key;
696 if ( !$dbw->lock( $lockName, $fname, 1 ) ) {
700 $dbw->commit( $fname,
'flush' );
701 call_user_func( $callback, $dbw, $method );
702 $dbw->commit( $fname,
'flush' );
704 $dbw->unlock( $lockName, $fname );
708 public static function getDatabaseIdForGroupId( $id ) {
710 if ( strlen( $id ) <= 72 ) {
714 $hash = hash(
'sha256', $id,
false );
715 $dbid = substr( $id, 0, 50 ) .
'||' . substr( $hash, 0, 20 );
719 public static function getStatsForCollection(
MessageCollection $collection ): array {
720 $collection->filter(
'ignored' );
721 $collection->filterUntranslatedOptional();
723 $total = count( $collection );
726 $collection->
filter(
'fuzzy' );
727 $fuzzy = $total - count( $collection );
730 $collection->
filter(
'hastranslation',
false );
731 $translated = count( $collection );
735 $collection->
filter(
'reviewer',
false );
736 $proofread = count( $collection );
739 self::TOTAL => $total,
740 self::TRANSLATED => $translated,
741 self::FUZZY => $fuzzy,
742 self::PROOFREAD => $proofread,