29 public function __construct() {
30 parent::__construct();
31 $this->addDescription(
'Script to bootstrap TTMServer.' );
34 '(optional) Number of threads',
40 '(optional) Server configuration identifier',
47 'Update the index mapping. Warning: Clears all existing data in the index.'
51 'Do not make any changes to the index.'
55 'Output more status information.'
59 'Only run setup and and cleanup. Skip inserting content.'
61 $this->setBatchSize( 500 );
62 $this->requireExtension(
'Translate' );
63 $this->start = microtime(
true );
66 public function statusLine( $text, $channel =
null ) {
67 $pid = sprintf(
'%5s', getmypid() );
68 $prefix = sprintf(
'%6.2f', microtime(
true ) - $this->start );
69 $mem = sprintf(
'%5.1fM', memory_get_usage(
true ) / ( 1024 * 1024 ) );
70 $this->output(
"$pid $prefix $mem $text", $channel );
73 public function execute() {
74 global $wgTranslateTranslationServices,
75 $wgTranslateTranslationDefaultService;
77 $configKey = $this->getOption(
'ttmserver', $wgTranslateTranslationDefaultService );
78 if ( !isset( $wgTranslateTranslationServices[$configKey] ) ) {
79 $this->fatalError(
'Translation memory is not configured properly' );
82 $dryRun = $this->getOption(
'dry-run' );
84 $config = [
'class' => FakeTTMServer::class ];
86 $config = $wgTranslateTranslationServices[$configKey];
89 $server = $this->getServer( $config );
90 $this->logInfo(
"Implementation: " . get_class( $server ) .
"\n" );
96 $this->resetStateForFork();
97 $server = $this->getServer( $config );
98 $this->beginBootstrap( $server );
100 } elseif ( $pid === -1 ) {
102 $this->beginBootstrap( $server );
105 $this->statusLine(
"Forked thread $pid to handle bootstrapping\n" );
107 pcntl_waitpid( $pid, $status );
109 if ( !$this->verifyChildStatus( $pid, $status ) ) {
110 $this->fatalError(
'Bootstrap failed.' );
115 $threads = $this->getOption(
'threads', 1 );
118 if ( $this->hasOption(
'clean' ) ) {
121 $groups = MessageGroups::singleton()->getGroups();
123 foreach ( $groups as $id => $group ) {
125 if ( $group->isMeta() ) {
133 $this->resetStateForFork();
134 $server = $this->getServer( $config );
135 $this->exportGroup( $group, $server );
137 } elseif ( $pid === -1 ) {
139 $this->exportGroup( $group, $server );
142 $this->statusLine(
"Forked thread $pid to handle $id\n" );
146 if ( count( $pids ) >= $threads ) {
148 $pid = pcntl_wait( $status );
149 $hasErrors = $hasErrors || !$this->verifyChildStatus( $pid, $status );
150 unset( $pids[$pid] );
156 foreach ( array_keys( $pids ) as $pid ) {
158 pcntl_waitpid( $pid, $status );
159 $hasErrors = $hasErrors || !$this->verifyChildStatus( $pid, $status );
163 $this->endBootstrap( $server );
166 $this->fatalError(
'!!! Some threads failed. Review the script output !!!' );
171 $server = TTMServer::factory( $config );
173 $this->fatalError(
"Service must implement WritableTTMServer" );
176 if ( method_exists( $server,
'setLogger' ) ) {
178 $server->setLogger( $this );
181 if ( $this->getOption(
'reindex',
false ) ) {
183 $server->setDoReIndex();
190 $this->statusLine(
"Cleaning up old entries...\n" );
195 $this->statusLine(
"Optimizing...\n" );
201 'total' => -microtime(
true ),
208 $id = $group->
getId();
211 $times[
'stats' ] -= microtime(
true );
212 $stats = MessageGroupStats::forGroup( $id );
213 $times[
'stats' ] += microtime(
true );
215 $times[
'init' ] -= microtime(
true );
217 $collection->filter(
'ignored' );
218 $collection->initMessages();
222 foreach ( $collection->keys() as $mkey => $titleValue ) {
223 $title = Title::newFromLinkTarget( $titleValue );
225 $inserts[] = [ $handle, $sourceLanguage, $collection[$mkey]->definition() ];
229 while ( $inserts !== [] ) {
230 $batch = array_splice( $inserts, 0, $this->mBatchSize );
234 $times[
'init' ] += microtime(
true );
236 $times[
'trans' ] -= microtime(
true );
237 foreach ( $stats as $targetLanguage => $numbers ) {
238 if ( $targetLanguage === $sourceLanguage ) {
241 if ( $numbers[MessageGroupStats::TRANSLATED] === 0 ) {
245 $collection->resetForNewLanguage( $targetLanguage );
246 $collection->filter(
'ignored' );
247 $collection->filter(
'translated',
false );
248 $collection->loadTranslations();
250 foreach ( $collection->keys() as $mkey => $titleValue ) {
251 $title = Title::newFromLinkTarget( $titleValue );
253 $inserts[] = [ $handle, $sourceLanguage, $collection[$mkey]->translation() ];
257 while ( count( $inserts ) >= $this->mBatchSize ) {
258 $batch = array_splice( $inserts, 0, $this->mBatchSize );
263 while ( $inserts !== [] ) {
264 $batch = array_splice( $inserts, 0, $this->mBatchSize );
269 $times[
'trans' ] += microtime(
true );
270 $times[
'total' ] += microtime(
true );
272 if ( $countItems !== 0 ) {
274 "Total %.1f s for %d items >> stats/init/trans %%: %d/%d/%d >> %.1f ms/item",
277 $times[
'stats'] / $times[
'total' ] * 100,
278 $times[
'init'] / $times[
'total' ] * 100,
279 $times[
'trans'] / $times[
'total' ] * 100,
280 $times[
'total' ] / $countItems * 1000
282 $this->logInfo(
"Finished exporting $id. $debug\n" );
286 private function logInfo(
string $text ) {
287 if ( $this->getOption(
'verbose',
false ) ) {
288 $this->statusLine( $text );
292 protected function resetStateForFork() {
295 MediaWiki\MediaWikiServices::resetChildProcessServices();
300 MediaWiki\MediaWikiServices::getInstance()->getMessageCache()->disable();
303 private function verifyChildStatus(
int $pid,
int $status ):
bool {
304 if ( pcntl_wifexited( $status ) ) {
305 $code = pcntl_wexitstatus( $status );
307 $this->output(
"Pid $pid exited with status $code !!\n" );
310 } elseif ( pcntl_wifsignaled( $status ) ) {
311 $signum = pcntl_wtermsig( $status );
312 $this->output(
"Pid $pid terminated by signal $signum !!\n" );