32 private const FAKE_TTM =
'dry-run';
34 public function __construct() {
35 parent::__construct();
36 $this->addDescription(
'Script to bootstrap TTMServer.' );
39 '(optional) Number of threads',
45 '(optional) Server configuration identifier',
52 'Update the index mapping. Warning: Clears all existing data in the index.'
56 'Do not make any changes to the index.'
60 'Output more status information.'
64 'Only run setup and and cleanup. Skip inserting content.'
66 $this->setBatchSize( 500 );
67 $this->requireExtension(
'Translate' );
68 $this->start = microtime(
true );
71 public function statusLine( $text, $channel =
null ) {
72 $pid = sprintf(
'%5s', getmypid() );
73 $prefix = sprintf(
'%6.2f', microtime(
true ) - $this->start );
74 $mem = sprintf(
'%5.1fM', memory_get_usage(
true ) / ( 1024 * 1024 ) );
75 $this->output(
"$pid $prefix $mem $text", $channel );
78 public function execute() {
79 $dryRun = $this->hasOption(
'dry-run' );
80 $ttmServerId = $this->getOption(
'ttmserver' );
81 $shouldReindex = $this->getOption(
'reindex',
false );
83 if ( $this->mBatchSize !==
null && $this->mBatchSize < 1 ) {
84 $this->fatalError(
'Invalid value for option: "batch-size"' );
87 $servers = $this->getServers( $dryRun, $shouldReindex, $ttmServerId );
91 foreach ( array_keys( $servers ) as $serverId ) {
95 $server = $this->getWritableServer( $serverId );
96 $this->resetStateForFork();
97 $this->beginBootstrap( $server, $serverId );
99 } elseif ( $pid === -1 ) {
101 $server = $this->getWritableServer( $serverId );
102 $this->beginBootstrap( $server, $serverId );
105 $this->statusLine(
"Forked thread $pid to handle bootstrapping for '$serverId'\n" );
107 pcntl_waitpid( $pid, $status );
109 if ( !$this->verifyChildStatus( $pid, $status ) ) {
110 $this->fatalError(
"Bootstrap failed for '$serverId'." );
116 $threads = $this->getOption(
'threads', 1 );
119 if ( $this->hasOption(
'clean' ) ) {
122 $groups = MessageGroups::singleton()->getGroups();
125 foreach ( $groups as $id => $group ) {
127 if ( $group->isMeta() ) {
134 $this->resetStateForFork();
135 $this->exportGroup( $group, $servers );
137 } elseif ( $pid === -1 ) {
138 $this->exportGroup( $group, $servers );
141 $this->statusLine(
"Forked thread $pid to handle $id\n" );
145 if ( count( $pids ) >= $threads ) {
147 $pid = pcntl_wait( $status );
148 $hasErrors = $hasErrors || !$this->verifyChildStatus( $pid, $status );
149 unset( $pids[$pid] );
155 foreach ( array_keys( $pids ) as $pid ) {
157 pcntl_waitpid( $pid, $status );
158 $hasErrors = $hasErrors || !$this->verifyChildStatus( $pid, $status );
162 $this->endBootstrap( $servers );
165 $this->fatalError(
'!!! Some threads failed. Review the script output !!!' );
175 private function getServers(
178 ?
string $ttmServerId =
null
181 $ttmServerFactory = Services::getInstance()->getTtmServerFactory();
185 if ( $ttmServerId !==
null ) {
187 $servers[ $ttmServerId ] = $ttmServerFactory->create( $ttmServerId );
189 $this->fatalError(
"Error while creating TtmServer $ttmServerId: " . $e->getMessage() );
192 $servers = $ttmServerFactory->getWritable();
197 $this->fatalError(
"No writable TtmServers found." );
200 foreach ( $servers as $server ) {
201 Assert::parameterType( WritableTtmServer::class, $server,
'$server' );
203 if ( method_exists( $server,
'setLogger' ) ) {
205 $server->setLogger( $this );
208 if ( $shouldReindex ) {
210 $server->setDoReIndex();
217 protected function beginBootstrap(
WritableTtmServer $server,
string $serverId ) {
218 $this->statusLine(
"Cleaning up old entries in '$serverId'...\n" );
222 protected function endBootstrap( array $servers ) {
223 foreach ( $servers as $serverId => $server ) {
224 $this->statusLine(
"Optimizing '$serverId'...\n" );
234 private function exportGroup(
MessageGroup $group, array $servers ):
void {
236 'total' => -microtime(
true ),
246 $times[
'init' ] -= microtime(
true );
247 $collection = $this->getCollection( $group, $sourceLanguage );
248 $times[
'init' ] += microtime(
true );
250 $times[
'stats' ] -= microtime(
true );
251 $stats = MessageGroupStats::forGroup( $group->
getId() );
252 $times[
'stats' ] += microtime(
true );
253 unset( $stats[ $sourceLanguage ] );
255 $translationCount = $definitionCount = 0;
257 foreach ( $servers as $server ) {
258 $server->beginBatch();
261 foreach ( $this->getDefinitions( $collection, $sourceLanguage ) as $batch ) {
262 $definitionCount += count( $batch );
263 foreach ( $servers as $server ) {
264 $times[
'writes' ] -= microtime(
true );
265 $server->batchInsertDefinitions( $batch );
266 $times[
'writes' ] += microtime(
true );
270 $times[
'trans' ] -= microtime(
true );
271 foreach ( $stats as $targetLanguage => $numbers ) {
272 if ( $numbers[MessageGroupStats::TRANSLATED] === 0 ) {
276 foreach ( $this->getTranslations( $collection, $targetLanguage ) as $batch ) {
277 $translationCount += count( $batch );
278 foreach ( $servers as $server ) {
279 $transWrites -= microtime(
true );
280 $server->batchInsertTranslations( $batch );
281 $transWrites += microtime(
true );
286 $times[
'trans' ] += ( microtime(
true ) - $transWrites );
287 $times[
'writes' ] += $transWrites;
289 foreach ( $servers as $server ) {
293 $times[
'total' ] += microtime(
true );
294 $countItems = $translationCount + $definitionCount;
296 if ( $countItems !== 0 ) {
298 "Total %.1f s for %d items on %d server(s) >> stats/init/trans/writes %%: %d/%d/%d/%d >> %.1f ms/item",
302 $times[
'stats'] / $times[
'total'] * 100,
303 $times[
'init'] / $times[
'total'] * 100,
304 $times[
'trans'] / $times[
'total'] * 100,
305 $times[
'writes'] / $times[
'total'] * 100,
306 $times[
'total'] / $countItems * 1000
308 $this->logInfo(
"Finished exporting {$group->getId()}. $debug\n" );
312 private function getDefinitions(
MessageCollection $collection,
string $sourceLanguage ): Generator {
314 foreach ( $collection->
keys() as $mKey => $titleValue ) {
315 $title = Title::newFromLinkTarget( $titleValue );
317 $definition = [ $handle, $sourceLanguage, $collection[$mKey]->definition() ];
318 $definitions[] = $definition;
319 if ( $this->mBatchSize && count( $definitions ) === $this->mBatchSize ) {
325 if ( $definitions ) {
330 private function getTranslations(
MessageCollection $collection,
string $targetLanguage ): Generator {
332 $collection->
filter(
'ignored' );
333 $collection->
filter(
'translated',
false );
337 foreach ( $collection->
keys() as $mkey => $titleValue ) {
338 $title = Title::newFromLinkTarget( $titleValue );
340 $translations[] = [ $handle, $targetLanguage, $collection[$mkey]->translation() ];
341 if ( $this->mBatchSize && count( $translations ) === $this->mBatchSize ) {
347 if ( $translations ) {
352 private function logInfo(
string $text ) {
353 if ( $this->hasOption(
'verbose' ) ) {
354 $this->statusLine( $text );
358 protected function resetStateForFork() {
361 MediaWiki\MediaWikiServices::resetChildProcessServices();
366 MediaWiki\MediaWikiServices::getInstance()->getMessageCache()->disable();
369 ObjectCache::clear();
372 private function verifyChildStatus(
int $pid,
int $status ):
bool {
373 if ( pcntl_wifexited( $status ) ) {
374 $code = pcntl_wexitstatus( $status );
376 $this->output(
"Pid $pid exited with status $code !!\n" );
379 } elseif ( pcntl_wifsignaled( $status ) ) {
380 $signum = pcntl_wtermsig( $status );
381 $this->output(
"Pid $pid terminated by signal $signum !!\n" );
389 if ( $serverId === self::FAKE_TTM ) {
393 $server = Services::getInstance()->getTtmServerFactory()->create( $serverId );
395 throw new InvalidArgumentException(
396 "$serverId TTM server does not implement WritableTtmServer interface "
405 $collection->
filter(
'ignored' );