MediaWiki  master
recompressTracked.php
Go to the documentation of this file.
1 <?php
29 
31 require __DIR__ . '/../CommandLineInc.php';
32 
33 if ( count( $args ) < 1 ) {
34  echo "Usage: php recompressTracked.php [options] <cluster> [... <cluster>...]
35 Moves blobs indexed by trackBlobs.php to a specified list of destination clusters,
36 and recompresses them in the process. Restartable.
37 
38 Options:
39  --procs <procs> Set the number of child processes (default 1)
40  --copy-only Copy only, do not update the text table. Restart
41  without this option to complete.
42  --debug-log <file> Log debugging data to the specified file
43  --info-log <file> Log progress messages to the specified file
44  --critical-log <file> Log error messages to the specified file
45 ";
46  exit( 1 );
47 }
48 
50 $job->execute();
51 
59  public $destClusters;
60  public $batchSize = 1000;
61  public $orphanBatchSize = 1000;
62  public $reportingInterval = 10;
63  public $numProcs = 1;
64  public $numBatches = 0;
67  public $copyOnly = false;
68  public $isChild = false;
69  public $childId = false;
70  public $noCount = false;
73  public $store;
75  private $blobStore;
76 
77  private static $optionsWithArgs = [
78  'procs',
79  'child-id',
80  'debug-log',
81  'info-log',
82  'critical-log'
83  ];
84 
85  private static $cmdLineOptionMap = [
86  'no-count' => 'noCount',
87  'procs' => 'numProcs',
88  'copy-only' => 'copyOnly',
89  'child' => 'isChild',
90  'child-id' => 'childId',
91  'debug-log' => 'debugLog',
92  'info-log' => 'infoLog',
93  'critical-log' => 'criticalLog',
94  ];
95 
96  public static function getOptionsWithArgs() {
98  }
99 
100  public static function newFromCommandLine( $args, $options ) {
101  $jobOptions = [ 'destClusters' => $args ];
102  foreach ( self::$cmdLineOptionMap as $cmdOption => $classOption ) {
103  if ( isset( $options[$cmdOption] ) ) {
104  $jobOptions[$classOption] = $options[$cmdOption];
105  }
106  }
107 
108  return new self( $jobOptions );
109  }
110 
111  public function __construct( $options ) {
112  foreach ( $options as $name => $value ) {
113  $this->$name = $value;
114  }
115  $esFactory = MediaWikiServices::getInstance()->getExternalStoreFactory();
116  $this->store = $esFactory->getStore( 'DB' );
117  if ( !$this->isChild ) {
118  $GLOBALS['wgDebugLogPrefix'] = "RCT M: ";
119  } elseif ( $this->childId !== false ) {
120  $GLOBALS['wgDebugLogPrefix'] = "RCT {$this->childId}: ";
121  }
122  $this->pageBlobClass = function_exists( 'xdiff_string_bdiff' ) ?
123  DiffHistoryBlob::class : ConcatenatedGzipHistoryBlob::class;
124  $this->orphanBlobClass = ConcatenatedGzipHistoryBlob::class;
125 
126  $this->blobStore = MediaWikiServices::getInstance()
127  ->getBlobStoreFactory()
128  ->newSqlBlobStore();
129  }
130 
131  public function debug( $msg ) {
132  wfDebug( "$msg" );
133  if ( $this->debugLog ) {
134  $this->logToFile( $msg, $this->debugLog );
135  }
136  }
137 
138  public function info( $msg ) {
139  echo "$msg\n";
140  if ( $this->infoLog ) {
141  $this->logToFile( $msg, $this->infoLog );
142  }
143  }
144 
145  public function critical( $msg ) {
146  echo "$msg\n";
147  if ( $this->criticalLog ) {
148  $this->logToFile( $msg, $this->criticalLog );
149  }
150  }
151 
152  private function logToFile( $msg, $file ) {
153  $header = '[' . date( 'd\TH:i:s' ) . '] ' . wfHostname() . ' ' . posix_getpid();
154  if ( $this->childId !== false ) {
155  $header .= "({$this->childId})";
156  }
157  $header .= ' ' . WikiMap::getCurrentWikiDbDomain()->getId();
158  LegacyLogger::emit( sprintf( "%-50s %s\n", $header, $msg ), $file );
159  }
160 
166  private function syncDBs() {
167  $dbw = wfGetDB( DB_PRIMARY );
168  $dbr = wfGetDB( DB_REPLICA );
169  $pos = $dbw->getPrimaryPos();
170  $dbr->primaryPosWait( $pos, 100000 );
171  }
172 
176  public function execute() {
177  if ( $this->isChild ) {
178  $this->executeChild();
179  } else {
180  $this->executeParent();
181  }
182  }
183 
187  public function executeParent() {
188  if ( !$this->checkTrackingTable() ) {
189  return;
190  }
191 
192  $this->syncDBs();
193  $this->startChildProcs();
194  $this->doAllPages();
195  $this->doAllOrphans();
196  $this->killChildProcs();
197  }
198 
203  private function checkTrackingTable() {
204  $dbr = wfGetDB( DB_REPLICA );
205  if ( !$dbr->tableExists( 'blob_tracking', __METHOD__ ) ) {
206  $this->critical( "Error: blob_tracking table does not exist" );
207 
208  return false;
209  }
210  $row = $dbr->selectRow( 'blob_tracking', '*', '', __METHOD__ );
211  if ( !$row ) {
212  $this->info( "Warning: blob_tracking table contains no rows, skipping this wiki." );
213 
214  return false;
215  }
216 
217  return true;
218  }
219 
226  private function startChildProcs() {
227  $wiki = WikiMap::getCurrentWikiId();
228 
229  $cmd = 'php ' . Shell::escape( __FILE__ );
230  foreach ( self::$cmdLineOptionMap as $cmdOption => $classOption ) {
231  if ( $cmdOption == 'child-id' ) {
232  continue;
233  }
234  if ( in_array( $cmdOption, self::$optionsWithArgs ) && isset( $this->$classOption ) ) {
235  $cmd .= " --$cmdOption " . Shell::escape( $this->$classOption );
236  } elseif ( $this->$classOption ) {
237  $cmd .= " --$cmdOption";
238  }
239  }
240  $cmd .= ' --child' .
241  ' --wiki ' . Shell::escape( $wiki ) .
242  ' ' . Shell::escape( ...$this->destClusters );
243 
244  $this->childPipes = $this->childProcs = [];
245  for ( $i = 0; $i < $this->numProcs; $i++ ) {
246  $pipes = [];
247  $spec = [
248  [ 'pipe', 'r' ],
249  [ 'file', 'php://stdout', 'w' ],
250  [ 'file', 'php://stderr', 'w' ]
251  ];
252  Wikimedia\suppressWarnings();
253  $proc = proc_open( "$cmd --child-id $i", $spec, $pipes );
254  Wikimedia\restoreWarnings();
255  if ( !$proc ) {
256  $this->critical( "Error opening child process: $cmd" );
257  exit( 1 );
258  }
259  $this->childProcs[$i] = $proc;
260  $this->childPipes[$i] = $pipes[0];
261  }
262  $this->prevChildId = -1;
263  }
264 
268  private function killChildProcs() {
269  $this->info( "Waiting for child processes to finish..." );
270  for ( $i = 0; $i < $this->numProcs; $i++ ) {
271  $this->dispatchToChild( $i, 'quit' );
272  }
273  for ( $i = 0; $i < $this->numProcs; $i++ ) {
274  $status = proc_close( $this->childProcs[$i] );
275  if ( $status ) {
276  $this->critical( "Warning: child #$i exited with status $status" );
277  }
278  }
279  $this->info( "Done." );
280  }
281 
287  private function dispatch( ...$args ) {
288  $pipes = $this->childPipes;
289  $x = [];
290  $y = [];
291  $numPipes = stream_select( $x, $pipes, $y, 3600 );
292  if ( !$numPipes ) {
293  $this->critical( "Error waiting to write to child process. Aborting" );
294  exit( 1 );
295  }
296  for ( $i = 0; $i < $this->numProcs; $i++ ) {
297  $childId = ( $i + $this->prevChildId + 1 ) % $this->numProcs;
298  if ( isset( $pipes[$childId] ) ) {
299  $this->prevChildId = $childId;
300  $this->dispatchToChild( $childId, $args );
301 
302  return;
303  }
304  }
305  $this->critical( "Unreachable" );
306  exit( 1 );
307  }
308 
314  private function dispatchToChild( $childId, $args ) {
315  $args = (array)$args;
316  $cmd = implode( ' ', $args );
317  fwrite( $this->childPipes[$childId], "$cmd\n" );
318  }
319 
323  private function doAllPages() {
324  $dbr = wfGetDB( DB_REPLICA );
325  $i = 0;
326  $startId = 0;
327  if ( $this->noCount ) {
328  $numPages = '[unknown]';
329  } else {
330  $numPages = $dbr->selectField( 'blob_tracking',
331  'COUNT(DISTINCT bt_page)',
332  # A condition is required so that this query uses the index
333  [ 'bt_moved' => 0 ],
334  __METHOD__
335  );
336  }
337  if ( $this->copyOnly ) {
338  $this->info( "Copying pages..." );
339  } else {
340  $this->info( "Moving pages..." );
341  }
342  while ( true ) {
343  $res = $dbr->select( 'blob_tracking',
344  [ 'bt_page' ],
345  [
346  'bt_moved' => 0,
347  'bt_page > ' . $dbr->addQuotes( $startId )
348  ],
349  __METHOD__,
350  [
351  'DISTINCT',
352  'ORDER BY' => 'bt_page',
353  'LIMIT' => $this->batchSize,
354  ]
355  );
356  if ( !$res->numRows() ) {
357  break;
358  }
359  foreach ( $res as $row ) {
360  $startId = $row->bt_page;
361  $this->dispatch( 'doPage', $row->bt_page );
362  $i++;
363  }
364  $this->report( 'pages', $i, $numPages );
365  }
366  $this->report( 'pages', $i, $numPages );
367  if ( $this->copyOnly ) {
368  $this->info( "All page copies queued." );
369  } else {
370  $this->info( "All page moves queued." );
371  }
372  }
373 
380  private function report( $label, $current, $end ) {
381  $this->numBatches++;
382  if ( $current == $end || $this->numBatches >= $this->reportingInterval ) {
383  $this->numBatches = 0;
384  $this->info( "$label: $current / $end" );
385  MediaWikiServices::getInstance()->getDBLoadBalancerFactory()->waitForReplication();
386  }
387  }
388 
392  private function doAllOrphans() {
393  $dbr = wfGetDB( DB_REPLICA );
394  $startId = 0;
395  $i = 0;
396  if ( $this->noCount ) {
397  $numOrphans = '[unknown]';
398  } else {
399  $numOrphans = $dbr->selectField( 'blob_tracking',
400  'COUNT(DISTINCT bt_text_id)',
401  [ 'bt_moved' => 0, 'bt_page' => 0 ],
402  __METHOD__ );
403  if ( !$numOrphans ) {
404  return;
405  }
406  }
407  if ( $this->copyOnly ) {
408  $this->info( "Copying orphans..." );
409  } else {
410  $this->info( "Moving orphans..." );
411  }
412 
413  while ( true ) {
414  $res = $dbr->select( 'blob_tracking',
415  [ 'bt_text_id' ],
416  [
417  'bt_moved' => 0,
418  'bt_page' => 0,
419  'bt_text_id > ' . $dbr->addQuotes( $startId )
420  ],
421  __METHOD__,
422  [
423  'DISTINCT',
424  'ORDER BY' => 'bt_text_id',
425  'LIMIT' => $this->batchSize
426  ]
427  );
428  if ( !$res->numRows() ) {
429  break;
430  }
431  $ids = [];
432  foreach ( $res as $row ) {
433  $startId = $row->bt_text_id;
434  $ids[] = $row->bt_text_id;
435  $i++;
436  }
437  // Need to send enough orphan IDs to the child at a time to fill a blob,
438  // so orphanBatchSize needs to be at least ~100.
439  // batchSize can be smaller or larger.
440  while ( count( $ids ) > $this->orphanBatchSize ) {
441  $args = array_slice( $ids, 0, $this->orphanBatchSize );
442  $ids = array_slice( $ids, $this->orphanBatchSize );
443  array_unshift( $args, 'doOrphanList' );
444  $this->dispatch( ...$args );
445  }
446  if ( count( $ids ) ) {
447  $args = $ids;
448  array_unshift( $args, 'doOrphanList' );
449  $this->dispatch( ...$args );
450  }
451 
452  $this->report( 'orphans', $i, $numOrphans );
453  }
454  $this->report( 'orphans', $i, $numOrphans );
455  $this->info( "All orphans queued." );
456  }
457 
461  public function executeChild() {
462  $this->debug( 'starting' );
463  $this->syncDBs();
464 
465  while ( !feof( STDIN ) ) {
466  $line = rtrim( fgets( STDIN ) );
467  if ( $line == '' ) {
468  continue;
469  }
470  $this->debug( $line );
471  $args = explode( ' ', $line );
472  $cmd = array_shift( $args );
473  switch ( $cmd ) {
474  case 'doPage':
475  $this->doPage( intval( $args[0] ) );
476  break;
477  case 'doOrphanList':
478  $this->doOrphanList( array_map( 'intval', $args ) );
479  break;
480  case 'quit':
481  return;
482  }
483  MediaWikiServices::getInstance()->getDBLoadBalancerFactory()->waitForReplication();
484  }
485  }
486 
492  private function doPage( $pageId ) {
493  $title = Title::newFromID( $pageId );
494  if ( $title ) {
495  $titleText = $title->getPrefixedText();
496  } else {
497  $titleText = '[deleted]';
498  }
499  $dbr = wfGetDB( DB_REPLICA );
500 
501  // Finish any incomplete transactions
502  if ( !$this->copyOnly ) {
503  $this->finishIncompleteMoves( [ 'bt_page' => $pageId ] );
504  $this->syncDBs();
505  }
506 
507  $startId = 0;
508  $trx = new CgzCopyTransaction( $this, $this->pageBlobClass );
509 
510  $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
511  while ( true ) {
512  $res = $dbr->select(
513  [ 'blob_tracking', 'text' ],
514  '*',
515  [
516  'bt_page' => $pageId,
517  'bt_text_id > ' . $dbr->addQuotes( $startId ),
518  'bt_moved' => 0,
519  'bt_new_url IS NULL',
520  'bt_text_id=old_id',
521  ],
522  __METHOD__,
523  [
524  'ORDER BY' => 'bt_text_id',
525  'LIMIT' => $this->batchSize
526  ]
527  );
528  if ( !$res->numRows() ) {
529  break;
530  }
531 
532  $lastTextId = 0;
533  foreach ( $res as $row ) {
534  $startId = $row->bt_text_id;
535  if ( $lastTextId == $row->bt_text_id ) {
536  // Duplicate (null edit)
537  continue;
538  }
539  $lastTextId = $row->bt_text_id;
540  // Load the text
541  $text = $this->blobStore->expandBlob( $row->old_text, $row->old_flags );
542  if ( $text === false ) {
543  $this->critical( "Error loading {$row->bt_rev_id}/{$row->bt_text_id}" );
544  continue;
545  }
546 
547  // Queue it
548  if ( !$trx->addItem( $text, $row->bt_text_id ) ) {
549  $this->debug( "$titleText: committing blob with " . $trx->getSize() . " items" );
550  $trx->commit();
551  $trx = new CgzCopyTransaction( $this, $this->pageBlobClass );
552  $lbFactory->waitForReplication();
553  }
554  }
555  }
556 
557  $this->debug( "$titleText: committing blob with " . $trx->getSize() . " items" );
558  $trx->commit();
559  }
560 
574  public function moveTextRow( $textId, $url ) {
575  if ( $this->copyOnly ) {
576  $this->critical( "Internal error: can't call moveTextRow() in --copy-only mode" );
577  exit( 1 );
578  }
579  $dbw = wfGetDB( DB_PRIMARY );
580  $dbw->begin( __METHOD__ );
581  $dbw->update( 'text',
582  [ // set
583  'old_text' => $url,
584  'old_flags' => 'external,utf-8',
585  ],
586  [ // where
587  'old_id' => $textId
588  ],
589  __METHOD__
590  );
591  $dbw->update( 'blob_tracking',
592  [ 'bt_moved' => 1 ],
593  [ 'bt_text_id' => $textId ],
594  __METHOD__
595  );
596  $dbw->commit( __METHOD__ );
597  }
598 
609  private function finishIncompleteMoves( $conds ) {
610  $dbr = wfGetDB( DB_REPLICA );
611  $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
612 
613  $startId = 0;
614  $conds = array_merge( $conds, [
615  'bt_moved' => 0,
616  'bt_new_url IS NOT NULL'
617  ] );
618  while ( true ) {
619  $res = $dbr->select( 'blob_tracking',
620  '*',
621  array_merge( $conds, [ 'bt_text_id > ' . $dbr->addQuotes( $startId ) ] ),
622  __METHOD__,
623  [
624  'ORDER BY' => 'bt_text_id',
625  'LIMIT' => $this->batchSize,
626  ]
627  );
628  if ( !$res->numRows() ) {
629  break;
630  }
631  $this->debug( 'Incomplete: ' . $res->numRows() . ' rows' );
632  foreach ( $res as $row ) {
633  $startId = $row->bt_text_id;
634  $this->moveTextRow( $row->bt_text_id, $row->bt_new_url );
635  if ( $row->bt_text_id % 10 == 0 ) {
636  $lbFactory->waitForReplication();
637  }
638  }
639  }
640  }
641 
646  public function getTargetCluster() {
647  $cluster = next( $this->destClusters );
648  if ( $cluster === false ) {
649  $cluster = reset( $this->destClusters );
650  }
651 
652  return $cluster;
653  }
654 
660  private function doOrphanList( $textIds ) {
661  // Finish incomplete moves
662  if ( !$this->copyOnly ) {
663  $this->finishIncompleteMoves( [ 'bt_text_id' => $textIds ] );
664  $this->syncDBs();
665  }
666 
667  $trx = new CgzCopyTransaction( $this, $this->orphanBlobClass );
668 
669  $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
670  $res = wfGetDB( DB_REPLICA )->select(
671  [ 'text', 'blob_tracking' ],
672  [ 'old_id', 'old_text', 'old_flags' ],
673  [
674  'old_id' => $textIds,
675  'bt_text_id=old_id',
676  'bt_moved' => 0,
677  ],
678  __METHOD__,
679  [ 'DISTINCT' ]
680  );
681 
682  foreach ( $res as $row ) {
683  $text = $this->blobStore->expandBlob( $row->old_text, $row->old_flags );
684  if ( $text === false ) {
685  $this->critical( "Error: cannot load revision text for old_id={$row->old_id}" );
686  continue;
687  }
688 
689  if ( !$trx->addItem( $text, $row->old_id ) ) {
690  $this->debug( "[orphan]: committing blob with " . $trx->getSize() . " rows" );
691  $trx->commit();
692  $trx = new CgzCopyTransaction( $this, $this->orphanBlobClass );
693  $lbFactory->waitForReplication();
694  }
695  }
696  $this->debug( "[orphan]: committing blob with " . $trx->getSize() . " rows" );
697  $trx->commit();
698  }
699 }
700 
706  public $parent;
707  public $blobClass;
709  public $cgz;
710  public $referrers;
712  private $texts;
713 
719  public function __construct( $parent, $blobClass ) {
720  $this->blobClass = $blobClass;
721  $this->cgz = false;
722  $this->texts = [];
723  $this->parent = $parent;
724  }
725 
733  public function addItem( $text, $textId ) {
734  if ( !$this->cgz ) {
735  $class = $this->blobClass;
736  $this->cgz = new $class;
737  }
738  $hash = $this->cgz->addItem( $text );
739  $this->referrers[$textId] = $hash;
740  $this->texts[$textId] = $text;
741 
742  return $this->cgz->isHappy();
743  }
744 
745  public function getSize() {
746  return count( $this->texts );
747  }
748 
752  public function recompress() {
753  $class = $this->blobClass;
754  $this->cgz = new $class;
755  $this->referrers = [];
756  foreach ( $this->texts as $textId => $text ) {
757  $hash = $this->cgz->addItem( $text );
758  $this->referrers[$textId] = $hash;
759  }
760  }
761 
767  public function commit() {
768  $originalCount = count( $this->texts );
769  if ( !$originalCount ) {
770  return;
771  }
772 
773  /* Check to see if the target text_ids have been moved already.
774  *
775  * We originally read from the replica DB, so this can happen when a single
776  * text_id is shared between multiple pages. It's rare, but possible
777  * if a delete/move/undelete cycle splits up a null edit.
778  *
779  * We do a locking read to prevent closer-run race conditions.
780  */
781  $dbw = wfGetDB( DB_PRIMARY );
782  $dbw->begin( __METHOD__ );
783  $res = $dbw->select( 'blob_tracking',
784  [ 'bt_text_id', 'bt_moved' ],
785  [ 'bt_text_id' => array_keys( $this->referrers ) ],
786  __METHOD__, [ 'FOR UPDATE' ] );
787  $dirty = false;
788  foreach ( $res as $row ) {
789  if ( $row->bt_moved ) {
790  # This row has already been moved, remove it
791  $this->parent->debug( "TRX: conflict detected in old_id={$row->bt_text_id}" );
792  unset( $this->texts[$row->bt_text_id] );
793  $dirty = true;
794  }
795  }
796 
797  // Recompress the blob if necessary
798  if ( $dirty ) {
799  if ( !count( $this->texts ) ) {
800  // All have been moved already
801  if ( $originalCount > 1 ) {
802  // This is suspcious, make noise
803  $this->parent->critical(
804  "Warning: concurrent operation detected, are there two conflicting " .
805  "processes running, doing the same job?" );
806  }
807 
808  return;
809  }
810  $this->recompress();
811  }
812 
813  // Insert the data into the destination cluster
814  $targetCluster = $this->parent->getTargetCluster();
815  $store = $this->parent->store;
816  $targetDB = $store->getPrimary( $targetCluster );
817  $targetDB->clearFlag( DBO_TRX ); // we manage the transactions
818  $targetDB->begin( __METHOD__ );
819  $baseUrl = $this->parent->store->store( $targetCluster, serialize( $this->cgz ) );
820 
821  // Write the new URLs to the blob_tracking table
822  foreach ( $this->referrers as $textId => $hash ) {
823  $url = $baseUrl . '/' . $hash;
824  $dbw->update( 'blob_tracking',
825  [ 'bt_new_url' => $url ],
826  [
827  'bt_text_id' => $textId,
828  'bt_moved' => 0, # Check for concurrent conflicting update
829  ],
830  __METHOD__
831  );
832  }
833 
834  $targetDB->commit( __METHOD__ );
835  // Critical section here: interruption at this point causes blob duplication
836  // Reversing the order of the commits would cause data loss instead
837  $dbw->commit( __METHOD__ );
838 
839  // Write the new URLs to the text table and set the moved flag
840  if ( !$this->parent->copyOnly ) {
841  foreach ( $this->referrers as $textId => $hash ) {
842  $url = $baseUrl . '/' . $hash;
843  $this->parent->moveTextRow( $textId, $url );
844  }
845  }
846  }
847 }
RecompressTracked\$cmdLineOptionMap
static $cmdLineOptionMap
Definition: recompressTracked.php:85
MediaWiki\Shell\Shell
Executes shell commands.
Definition: Shell.php:45
WikiMap\getCurrentWikiDbDomain
static getCurrentWikiDbDomain()
Definition: WikiMap.php:293
RecompressTracked\info
info( $msg)
Definition: recompressTracked.php:138
RecompressTracked
Maintenance script that moves blobs indexed by trackBlobs.php to a specified list of destination clus...
Definition: recompressTracked.php:58
ExternalStoreDB
DB accessible external objects.
Definition: ExternalStoreDB.php:40
RecompressTracked\logToFile
logToFile( $msg, $file)
Definition: recompressTracked.php:152
CgzCopyTransaction\addItem
addItem( $text, $textId)
Add text.
Definition: recompressTracked.php:733
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:193
RecompressTracked\$copyOnly
$copyOnly
Definition: recompressTracked.php:67
MediaWiki\Storage\SqlBlobStore
Service for storing and loading Content objects.
Definition: SqlBlobStore.php:52
ConcatenatedGzipHistoryBlob
Concatenated gzip (CGZ) storage Improves compression ratio by concatenating like objects before gzipp...
Definition: ConcatenatedGzipHistoryBlob.php:27
RecompressTracked\$optionsWithArgs
static $optionsWithArgs
Definition: recompressTracked.php:77
RecompressTracked\$debugLog
$debugLog
Definition: recompressTracked.php:71
RecompressTracked\getTargetCluster
getTargetCluster()
Returns the name of the next target cluster.
Definition: recompressTracked.php:646
RecompressTracked\$childPipes
$childPipes
Definition: recompressTracked.php:66
RecompressTracked\debug
debug( $msg)
Definition: recompressTracked.php:131
RecompressTracked\killChildProcs
killChildProcs()
Gracefully terminate the child processes.
Definition: recompressTracked.php:268
$file
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition: router.php:42
wfHostname
wfHostname()
Get host name of the current machine, for use in error reporting.
Definition: GlobalFunctions.php:1249
RecompressTracked\$isChild
$isChild
Definition: recompressTracked.php:68
CgzCopyTransaction\recompress
recompress()
Recompress text after some aberrant modification.
Definition: recompressTracked.php:752
RecompressTracked\$prevChildId
$prevChildId
Definition: recompressTracked.php:66
$res
$res
Definition: testCompression.php:57
serialize
serialize()
Definition: ApiMessageTrait.php:138
WikiMap\getCurrentWikiId
static getCurrentWikiId()
Definition: WikiMap.php:303
RecompressTracked\executeParent
executeParent()
Execute the parent process.
Definition: recompressTracked.php:187
DBO_TRX
const DBO_TRX
Definition: defines.php:12
CgzCopyTransaction\getSize
getSize()
Definition: recompressTracked.php:745
$dbr
$dbr
Definition: testCompression.php:54
RecompressTracked\$destClusters
$destClusters
Definition: recompressTracked.php:59
RecompressTracked\$orphanBlobClass
$orphanBlobClass
Definition: recompressTracked.php:65
RecompressTracked\$pageBlobClass
$pageBlobClass
Definition: recompressTracked.php:65
RecompressTracked\finishIncompleteMoves
finishIncompleteMoves( $conds)
Moves are done in two phases: bt_new_url and then bt_moved.
Definition: recompressTracked.php:609
wfGetDB
wfGetDB( $db, $groups=[], $wiki=false)
Get a Database object.
Definition: GlobalFunctions.php:2202
RecompressTracked\$reportingInterval
$reportingInterval
Definition: recompressTracked.php:62
RecompressTracked\critical
critical( $msg)
Definition: recompressTracked.php:145
CgzCopyTransaction\commit
commit()
Commit the blob.
Definition: recompressTracked.php:767
CgzCopyTransaction\$parent
RecompressTracked $parent
Definition: recompressTracked.php:706
RecompressTracked\$criticalLog
$criticalLog
Definition: recompressTracked.php:71
$args
if( $line===false) $args
Definition: mcc.php:124
CgzCopyTransaction\$cgz
ConcatenatedGzipHistoryBlob false $cgz
Definition: recompressTracked.php:709
RecompressTracked\doPage
doPage( $pageId)
Move tracked text in a given page.
Definition: recompressTracked.php:492
RecompressTracked\$orphanBatchSize
$orphanBatchSize
Definition: recompressTracked.php:61
RecompressTracked\newFromCommandLine
static newFromCommandLine( $args, $options)
Definition: recompressTracked.php:100
$title
$title
Definition: testCompression.php:38
DB_REPLICA
const DB_REPLICA
Definition: defines.php:25
wfDebug
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
Definition: GlobalFunctions.php:894
RecompressTracked\$blobStore
SqlBlobStore $blobStore
Definition: recompressTracked.php:75
RecompressTracked\dispatch
dispatch(... $args)
Dispatch a command to the next available child process.
Definition: recompressTracked.php:287
RecompressTracked\dispatchToChild
dispatchToChild( $childId, $args)
Dispatch a command to a specified child process.
Definition: recompressTracked.php:314
RecompressTracked\$batchSize
$batchSize
Definition: recompressTracked.php:60
RecompressTracked\doAllOrphans
doAllOrphans()
Move all orphan text to the new clusters.
Definition: recompressTracked.php:392
CgzCopyTransaction\$referrers
$referrers
Definition: recompressTracked.php:710
RecompressTracked\$childId
$childId
Definition: recompressTracked.php:69
CgzCopyTransaction\$blobClass
$blobClass
Definition: recompressTracked.php:707
$header
$header
Definition: updateCredits.php:37
DB_PRIMARY
const DB_PRIMARY
Definition: defines.php:27
$line
$line
Definition: mcc.php:119
CgzCopyTransaction
Class to represent a recompression operation for a single CGZ blob.
Definition: recompressTracked.php:704
RecompressTracked\report
report( $label, $current, $end)
Display a progress report.
Definition: recompressTracked.php:380
RecompressTracked\moveTextRow
moveTextRow( $textId, $url)
Atomic move operation.
Definition: recompressTracked.php:574
CgzCopyTransaction\__construct
__construct( $parent, $blobClass)
Create a transaction from a RecompressTracked object.
Definition: recompressTracked.php:719
$optionsWithArgs
$optionsWithArgs
Definition: recompressTracked.php:30
RecompressTracked\executeChild
executeChild()
Main entry point for worker processes.
Definition: recompressTracked.php:461
RecompressTracked\checkTrackingTable
checkTrackingTable()
Make sure the tracking table exists and isn't empty.
Definition: recompressTracked.php:203
CgzCopyTransaction\$texts
array $texts
Definition: recompressTracked.php:712
RecompressTracked\execute
execute()
Execute parent or child depending on the isChild option.
Definition: recompressTracked.php:176
RecompressTracked\$childProcs
$childProcs
Definition: recompressTracked.php:66
$job
if(count( $args)< 1) $job
Definition: recompressTracked.php:49
RecompressTracked\$numProcs
$numProcs
Definition: recompressTracked.php:63
RecompressTracked\$numBatches
$numBatches
Definition: recompressTracked.php:64
RecompressTracked\$infoLog
$infoLog
Definition: recompressTracked.php:71
RecompressTracked\doAllPages
doAllPages()
Move all tracked pages to the new clusters.
Definition: recompressTracked.php:323
MediaWiki\Logger\LegacyLogger
PSR-3 logger that mimics the historic implementation of MediaWiki's former wfErrorLog logging impleme...
Definition: LegacyLogger.php:51
RecompressTracked\startChildProcs
startChildProcs()
Start the worker processes.
Definition: recompressTracked.php:226
RecompressTracked\__construct
__construct( $options)
Definition: recompressTracked.php:111
RecompressTracked\syncDBs
syncDBs()
Wait until the selected replica DB has caught up to the master.
Definition: recompressTracked.php:166
Title\newFromID
static newFromID( $id, $flags=0)
Create a new Title from an article ID.
Definition: Title.php:529
RecompressTracked\$store
ExternalStoreDB $store
Definition: recompressTracked.php:73
RecompressTracked\$noCount
$noCount
Definition: recompressTracked.php:70
RecompressTracked\doOrphanList
doOrphanList( $textIds)
Move an orphan text_id to the new cluster.
Definition: recompressTracked.php:660
RecompressTracked\getOptionsWithArgs
static getOptionsWithArgs()
Definition: recompressTracked.php:96