ttmserver-export.php

Go to the documentation of this file.

<?php
// Standard boilerplate to define $IP
if ( getenv( 'MW_INSTALL_PATH' ) !== false ) {
    $IP = getenv( 'MW_INSTALL_PATH' );
} else {
    $dir = __DIR__;
    $IP = "$dir/../../..";
}
require_once "$IP/maintenance/Maintenance.php";
 
class TTMServerBootstrap extends Maintenance {
    private $start;
 
    public function __construct() {
        parent::__construct();
        $this->addDescription( 'Script to bootstrap TTMServer.' );
        $this->addOption(
            'threads',
            '(optional) Number of threads',
            /*required*/false,
            /*has arg*/true
        );
        $this->addOption(
            'ttmserver',
            '(optional) Server configuration identifier',
            /*required*/false,
            /*has arg*/true
        );
        // This option erases all data, empties the index and rebuilds it.
        $this->addOption(
            'reindex',
            'Update the index mapping. Warning: Clears all existing data in the index.'
        );
        $this->addOption(
            'dry-run',
            'Do not make any changes to the index.'
        );
        $this->addOption(
            'verbose',
            'Output more status information.'
        );
        $this->setBatchSize( 500 );
        $this->requireExtension( 'Translate' );
        $this->start = microtime( true );
    }
 
    public function statusLine( $text, $channel = null ) {
        $pid = sprintf( '%5s', getmypid() );
        $prefix = sprintf( '%6.2f', microtime( true ) - $this->start );
        $mem = sprintf( '%5.1fM', memory_get_usage( true ) / ( 1024 * 1024 ) );
        $this->output( "$pid $prefix $mem  $text", $channel );
    }
 
    public function execute() {
        global $wgTranslateTranslationServices,
            $wgTranslateTranslationDefaultService;
 
        $configKey = $this->getOption( 'ttmserver', $wgTranslateTranslationDefaultService );
        if ( !isset( $wgTranslateTranslationServices[$configKey] ) ) {
            $this->fatalError( 'Translation memory is not configured properly' );
        }
 
        $dryRun = $this->getOption( 'dry-run' );
        if ( $dryRun ) {
            $config = [ 'class' => FakeTTMServer::class ];
        } else {
            $config = $wgTranslateTranslationServices[$configKey];
        }
 
        $server = $this->getServer( $config );
        $this->logInfo( "Implementation: " . get_class( $server ) . "\n" );
 
        // Do as little as possible in the main thread, to not clobber forked processes.
        // See also #resetStateForFork.
        $pid = pcntl_fork();
        if ( $pid === 0 ) {
            $this->resetStateForFork();
            $server = $this->getServer( $config );
            $this->beginBootstrap( $server );
            exit();
        } elseif ( $pid === -1 ) {
            // Fork failed do it serialized
            $this->beginBootstrap( $server );
        } else {
            // Main thread
            $this->statusLine( "Forked thread $pid to handle bootstrapping\n" );
            $status = 0;
            pcntl_waitpid( $pid, $status );
            // beginBootstrap probably failed, give up.
            if ( !$this->verifyChildStatus( $pid, $status ) ) {
                $this->fatalError( 'Bootstrap failed.' );
            }
        }
 
        $hasErrors = false;
        $threads = $this->getOption( 'threads', 1 );
        $pids = [];
 
        $groups = MessageGroups::singleton()->getGroups();
        foreach ( $groups as $id => $group ) {
            if ( $group->isMeta() ) {
                continue;
            }
 
            // Fork to increase speed with parallelism. Also helps with memory usage if there are leaks.
            $pid = pcntl_fork();
 
            if ( $pid === 0 ) {
                $this->resetStateForFork();
                $server = $this->getServer( $config );
                $this->exportGroup( $group, $server );
                exit();
            } elseif ( $pid === -1 ) {
                // Fork failed do it serialized
                $this->exportGroup( $group, $server );
            } else {
                // Main thread
                $this->statusLine( "Forked thread $pid to handle $id\n" );
                $pids[$pid] = true;
 
                // If we hit the thread limit, wait for any child to finish.
                if ( count( $pids ) >= $threads ) {
                    $status = 0;
                    $pid = pcntl_wait( $status );
                    $hasErrors = $hasErrors || !$this->verifyChildStatus( $pid, $status );
                    unset( $pids[$pid] );
                }
            }
        }
 
        // Return control after all threads have finished.
        foreach ( array_keys( $pids ) as $pid ) {
            $status = 0;
            pcntl_waitpid( $pid, $status );
            $hasErrors = $hasErrors || !$this->verifyChildStatus( $pid, $status );
        }
 
        // It's okay to do this in the main thread as it is the last thing
        $this->endBootstrap( $server );
 
        if ( $hasErrors ) {
            $this->fatalError( '!!! Some threads failed. Review the script output !!!' );
        }
    }
 
    private function getServer( array $config ): WritableTTMServer {
        $server = TTMServer::factory( $config );
        if ( !$server instanceof WritableTTMServer ) {
            $this->fatalError( "Service must implement WritableTTMServer" );
        }
 
        if ( is_callable( [ $server, 'setLogger' ] ) ) {
            // Phan, why you so strict?
            // @phan-suppress-next-line PhanUndeclaredMethod
            $server->setLogger( $this );
        }
 
        if ( $this->getOption( 'reindex', false ) ) {
            // This doesn't do the update, just sets a flag to do it
            $server->setDoReIndex();
        }
 
        return $server;
    }
 
    protected function beginBootstrap( WritableTTMServer $server ) {
        $this->statusLine( "Cleaning up old entries...\n" );
        $server->beginBootstrap();
    }
 
    protected function endBootstrap( WritableTTMServer $server ) {
        $this->statusLine( "Optimizing...\n" );
        $server->endBootstrap();
    }
 
    protected function exportGroup( MessageGroup $group, WritableTTMServer $server ) {
        $times = [
            'total' => -microtime( true ),
            'stats' => 0,
            'init' => 0,
            'trans' => 0,
        ];
        $countItems = 0;
 
        $id = $group->getId();
        $sourceLanguage = $group->getSourceLanguage();
 
        $times[ 'stats' ] -= microtime( true );
        $stats = MessageGroupStats::forGroup( $id );
        $times[ 'stats' ] += microtime( true );
 
        $times[ 'init' ] -= microtime( true );
        $collection = $group->initCollection( $sourceLanguage );
        $collection->filter( 'ignored' );
        $collection->initMessages();
 
        $server->beginBatch();
        $inserts = [];
        foreach ( $collection->keys() as $mkey => $titleValue ) {
            $title = Title::newFromLinkTarget( $titleValue );
            $handle = new MessageHandle( $title );
            $inserts[] = [ $handle, $sourceLanguage, $collection[$mkey]->definition() ];
            $countItems++;
        }
 
        while ( $inserts !== [] ) {
            $batch = array_splice( $inserts, 0, $this->mBatchSize );
            $server->batchInsertDefinitions( $batch );
        }
        $inserts = [];
        $times[ 'init' ] += microtime( true );
 
        $times[ 'trans' ] -= microtime( true );
        foreach ( $stats as $targetLanguage => $numbers ) {
            if ( $targetLanguage === $sourceLanguage ) {
                continue;
            }
            if ( $numbers[MessageGroupStats::TRANSLATED] === 0 ) {
                continue;
            }
 
            $collection->resetForNewLanguage( $targetLanguage );
            $collection->filter( 'ignored' );
            $collection->filter( 'translated', false );
            $collection->loadTranslations();
 
            foreach ( $collection->keys() as $mkey => $titleValue ) {
                $title = Title::newFromLinkTarget( $titleValue );
                $handle = new MessageHandle( $title );
                $inserts[] = [ $handle, $sourceLanguage, $collection[$mkey]->translation() ];
                $countItems++;
            }
 
            while ( count( $inserts ) >= $this->mBatchSize ) {
                $batch = array_splice( $inserts, 0, $this->mBatchSize );
                $server->batchInsertTranslations( $batch );
            }
        }
 
        while ( $inserts !== [] ) {
            $batch = array_splice( $inserts, 0, $this->mBatchSize );
            $server->batchInsertTranslations( $batch );
        }
 
        $server->endBatch();
        $times[ 'trans' ] += microtime( true );
        $times[ 'total' ] += microtime( true );
 
        if ( $countItems !== 0 ) {
            $debug = sprintf(
                "Total %.1f s for %d items >> stats/init/trans %%: %d/%d/%d >> %.1f ms/item",
                $times[ 'total' ],
                $countItems,
                $times[ 'stats'] / $times[ 'total' ] * 100,
                $times[ 'init'] / $times[ 'total' ] * 100,
                $times[ 'trans'] / $times[ 'total' ] * 100,
                $times[ 'total' ] / $countItems * 1000
            );
            $this->logInfo( "Finished exporting $id. $debug\n" );
        }
    }
 
    private function logInfo( string $text ) {
        if ( $this->getOption( 'verbose', false ) ) {
            $this->statusLine( $text );
        }
    }
 
    protected function resetStateForFork() {
        // Make sure all existing connections are dead,
        // we can't use them in forked children.
        MediaWiki\MediaWikiServices::resetChildProcessServices();
        // Temporary workaround for https://phabricator.wikimedia.org/T258860.
        // This script just moves data around, so skipping the message cache should not
        // cause any major issues. Things like message documentation language name and
        // main page name were being checked from the message cache and sometimes failing.
        MediaWiki\MediaWikiServices::getInstance()->getMessageCache()->disable();
    }
 
    private function verifyChildStatus( int $pid, int $status ): bool {
        if ( pcntl_wifexited( $status ) ) {
            $code = pcntl_wexitstatus( $status );
            if ( $code ) {
                $this->output( "Pid $pid exited with status $code !!\n" );
                return false;
            }
        } elseif ( pcntl_wifsignaled( $status ) ) {
            $signum = pcntl_wtermsig( $status );
            $this->output( "Pid $pid terminated by signal $signum !!\n" );
            return false;
        }
 
        return true;
    }
}
 
$maintClass = TTMServerBootstrap::class;
require_once RUN_MAINTENANCE_IF_MAIN;