MediaWiki  1.29.1
recompressTracked.php
Go to the documentation of this file.
1 <?php
27 
29 require __DIR__ . '/../commandLine.inc';
30 
31 if ( count( $args ) < 1 ) {
32  echo "Usage: php recompressTracked.php [options] <cluster> [... <cluster>...]
33 Moves blobs indexed by trackBlobs.php to a specified list of destination clusters,
34 and recompresses them in the process. Restartable.
35 
36 Options:
37  --procs <procs> Set the number of child processes (default 1)
38  --copy-only Copy only, do not update the text table. Restart
39  without this option to complete.
40  --debug-log <file> Log debugging data to the specified file
41  --info-log <file> Log progress messages to the specified file
42  --critical-log <file> Log error messages to the specified file
43 ";
44  exit( 1 );
45 }
46 
48 $job->execute();
49 
57  public $destClusters;
58  public $batchSize = 1000;
59  public $orphanBatchSize = 1000;
60  public $reportingInterval = 10;
61  public $numProcs = 1;
62  public $numBatches = 0;
65  public $copyOnly = false;
66  public $isChild = false;
67  public $replicaId = false;
68  public $noCount = false;
70  public $store;
71 
72  private static $optionsWithArgs = [
73  'procs',
74  'replica-id',
75  'debug-log',
76  'info-log',
77  'critical-log'
78  ];
79 
80  private static $cmdLineOptionMap = [
81  'no-count' => 'noCount',
82  'procs' => 'numProcs',
83  'copy-only' => 'copyOnly',
84  'child' => 'isChild',
85  'replica-id' => 'replicaId',
86  'debug-log' => 'debugLog',
87  'info-log' => 'infoLog',
88  'critical-log' => 'criticalLog',
89  ];
90 
91  static function getOptionsWithArgs() {
93  }
94 
95  static function newFromCommandLine( $args, $options ) {
96  $jobOptions = [ 'destClusters' => $args ];
97  foreach ( self::$cmdLineOptionMap as $cmdOption => $classOption ) {
98  if ( isset( $options[$cmdOption] ) ) {
99  $jobOptions[$classOption] = $options[$cmdOption];
100  }
101  }
102 
103  return new self( $jobOptions );
104  }
105 
106  function __construct( $options ) {
107  foreach ( $options as $name => $value ) {
108  $this->$name = $value;
109  }
110  $this->store = new ExternalStoreDB;
111  if ( !$this->isChild ) {
112  $GLOBALS['wgDebugLogPrefix'] = "RCT M: ";
113  } elseif ( $this->replicaId !== false ) {
114  $GLOBALS['wgDebugLogPrefix'] = "RCT {$this->replicaId}: ";
115  }
116  $this->pageBlobClass = function_exists( 'xdiff_string_bdiff' ) ?
117  'DiffHistoryBlob' : 'ConcatenatedGzipHistoryBlob';
118  $this->orphanBlobClass = 'ConcatenatedGzipHistoryBlob';
119  }
120 
121  function debug( $msg ) {
122  wfDebug( "$msg\n" );
123  if ( $this->debugLog ) {
124  $this->logToFile( $msg, $this->debugLog );
125  }
126  }
127 
128  function info( $msg ) {
129  echo "$msg\n";
130  if ( $this->infoLog ) {
131  $this->logToFile( $msg, $this->infoLog );
132  }
133  }
134 
135  function critical( $msg ) {
136  echo "$msg\n";
137  if ( $this->criticalLog ) {
138  $this->logToFile( $msg, $this->criticalLog );
139  }
140  }
141 
142  function logToFile( $msg, $file ) {
143  $header = '[' . date( 'd\TH:i:s' ) . '] ' . wfHostname() . ' ' . posix_getpid();
144  if ( $this->replicaId !== false ) {
145  $header .= "({$this->replicaId})";
146  }
147  $header .= ' ' . wfWikiID();
148  LegacyLogger::emit( sprintf( "%-50s %s\n", $header, $msg ), $file );
149  }
150 
156  function syncDBs() {
157  $dbw = wfGetDB( DB_MASTER );
158  $dbr = wfGetDB( DB_REPLICA );
159  $pos = $dbw->getMasterPos();
160  $dbr->masterPosWait( $pos, 100000 );
161  }
162 
166  function execute() {
167  if ( $this->isChild ) {
168  $this->executeChild();
169  } else {
170  $this->executeParent();
171  }
172  }
173 
177  function executeParent() {
178  if ( !$this->checkTrackingTable() ) {
179  return;
180  }
181 
182  $this->syncDBs();
183  $this->startReplicaProcs();
184  $this->doAllPages();
185  $this->doAllOrphans();
186  $this->killReplicaProcs();
187  }
188 
193  function checkTrackingTable() {
194  $dbr = wfGetDB( DB_REPLICA );
195  if ( !$dbr->tableExists( 'blob_tracking' ) ) {
196  $this->critical( "Error: blob_tracking table does not exist" );
197 
198  return false;
199  }
200  $row = $dbr->selectRow( 'blob_tracking', '*', '', __METHOD__ );
201  if ( !$row ) {
202  $this->info( "Warning: blob_tracking table contains no rows, skipping this wiki." );
203 
204  return false;
205  }
206 
207  return true;
208  }
209 
216  function startReplicaProcs() {
217  $cmd = 'php ' . wfEscapeShellArg( __FILE__ );
218  foreach ( self::$cmdLineOptionMap as $cmdOption => $classOption ) {
219  if ( $cmdOption == 'replica-id' ) {
220  continue;
221  } elseif ( in_array( $cmdOption, self::$optionsWithArgs ) && isset( $this->$classOption ) ) {
222  $cmd .= " --$cmdOption " . wfEscapeShellArg( $this->$classOption );
223  } elseif ( $this->$classOption ) {
224  $cmd .= " --$cmdOption";
225  }
226  }
227  $cmd .= ' --child' .
228  ' --wiki ' . wfEscapeShellArg( wfWikiID() ) .
229  ' ' . call_user_func_array( 'wfEscapeShellArg', $this->destClusters );
230 
231  $this->replicaPipes = $this->replicaProcs = [];
232  for ( $i = 0; $i < $this->numProcs; $i++ ) {
233  $pipes = [];
234  $spec = [
235  [ 'pipe', 'r' ],
236  [ 'file', 'php://stdout', 'w' ],
237  [ 'file', 'php://stderr', 'w' ]
238  ];
239  MediaWiki\suppressWarnings();
240  $proc = proc_open( "$cmd --replica-id $i", $spec, $pipes );
241  MediaWiki\restoreWarnings();
242  if ( !$proc ) {
243  $this->critical( "Error opening replica DB process: $cmd" );
244  exit( 1 );
245  }
246  $this->replicaProcs[$i] = $proc;
247  $this->replicaPipes[$i] = $pipes[0];
248  }
249  $this->prevReplicaId = -1;
250  }
251 
255  function killReplicaProcs() {
256  $this->info( "Waiting for replica DB processes to finish..." );
257  for ( $i = 0; $i < $this->numProcs; $i++ ) {
258  $this->dispatchToReplica( $i, 'quit' );
259  }
260  for ( $i = 0; $i < $this->numProcs; $i++ ) {
261  $status = proc_close( $this->replicaProcs[$i] );
262  if ( $status ) {
263  $this->critical( "Warning: child #$i exited with status $status" );
264  }
265  }
266  $this->info( "Done." );
267  }
268 
273  function dispatch( /*...*/ ) {
274  $args = func_get_args();
275  $pipes = $this->replicaPipes;
276  $numPipes = stream_select( $x = [], $pipes, $y = [], 3600 );
277  if ( !$numPipes ) {
278  $this->critical( "Error waiting to write to replica DBs. Aborting" );
279  exit( 1 );
280  }
281  for ( $i = 0; $i < $this->numProcs; $i++ ) {
282  $replicaId = ( $i + $this->prevReplicaId + 1 ) % $this->numProcs;
283  if ( isset( $pipes[$replicaId] ) ) {
284  $this->prevReplicaId = $replicaId;
285  $this->dispatchToReplica( $replicaId, $args );
286 
287  return;
288  }
289  }
290  $this->critical( "Unreachable" );
291  exit( 1 );
292  }
293 
300  $args = (array)$args;
301  $cmd = implode( ' ', $args );
302  fwrite( $this->replicaPipes[$replicaId], "$cmd\n" );
303  }
304 
308  function doAllPages() {
309  $dbr = wfGetDB( DB_REPLICA );
310  $i = 0;
311  $startId = 0;
312  if ( $this->noCount ) {
313  $numPages = '[unknown]';
314  } else {
315  $numPages = $dbr->selectField( 'blob_tracking',
316  'COUNT(DISTINCT bt_page)',
317  # A condition is required so that this query uses the index
318  [ 'bt_moved' => 0 ],
319  __METHOD__
320  );
321  }
322  if ( $this->copyOnly ) {
323  $this->info( "Copying pages..." );
324  } else {
325  $this->info( "Moving pages..." );
326  }
327  while ( true ) {
328  $res = $dbr->select( 'blob_tracking',
329  [ 'bt_page' ],
330  [
331  'bt_moved' => 0,
332  'bt_page > ' . $dbr->addQuotes( $startId )
333  ],
334  __METHOD__,
335  [
336  'DISTINCT',
337  'ORDER BY' => 'bt_page',
338  'LIMIT' => $this->batchSize,
339  ]
340  );
341  if ( !$res->numRows() ) {
342  break;
343  }
344  foreach ( $res as $row ) {
345  $startId = $row->bt_page;
346  $this->dispatch( 'doPage', $row->bt_page );
347  $i++;
348  }
349  $this->report( 'pages', $i, $numPages );
350  }
351  $this->report( 'pages', $i, $numPages );
352  if ( $this->copyOnly ) {
353  $this->info( "All page copies queued." );
354  } else {
355  $this->info( "All page moves queued." );
356  }
357  }
358 
365  function report( $label, $current, $end ) {
366  $this->numBatches++;
367  if ( $current == $end || $this->numBatches >= $this->reportingInterval ) {
368  $this->numBatches = 0;
369  $this->info( "$label: $current / $end" );
370  MediaWikiServices::getInstance()->getDBLoadBalancerFactory()->waitForReplication();
371  }
372  }
373 
377  function doAllOrphans() {
378  $dbr = wfGetDB( DB_REPLICA );
379  $startId = 0;
380  $i = 0;
381  if ( $this->noCount ) {
382  $numOrphans = '[unknown]';
383  } else {
384  $numOrphans = $dbr->selectField( 'blob_tracking',
385  'COUNT(DISTINCT bt_text_id)',
386  [ 'bt_moved' => 0, 'bt_page' => 0 ],
387  __METHOD__ );
388  if ( !$numOrphans ) {
389  return;
390  }
391  }
392  if ( $this->copyOnly ) {
393  $this->info( "Copying orphans..." );
394  } else {
395  $this->info( "Moving orphans..." );
396  }
397 
398  while ( true ) {
399  $res = $dbr->select( 'blob_tracking',
400  [ 'bt_text_id' ],
401  [
402  'bt_moved' => 0,
403  'bt_page' => 0,
404  'bt_text_id > ' . $dbr->addQuotes( $startId )
405  ],
406  __METHOD__,
407  [
408  'DISTINCT',
409  'ORDER BY' => 'bt_text_id',
410  'LIMIT' => $this->batchSize
411  ]
412  );
413  if ( !$res->numRows() ) {
414  break;
415  }
416  $ids = [];
417  foreach ( $res as $row ) {
418  $startId = $row->bt_text_id;
419  $ids[] = $row->bt_text_id;
420  $i++;
421  }
422  // Need to send enough orphan IDs to the child at a time to fill a blob,
423  // so orphanBatchSize needs to be at least ~100.
424  // batchSize can be smaller or larger.
425  while ( count( $ids ) > $this->orphanBatchSize ) {
426  $args = array_slice( $ids, 0, $this->orphanBatchSize );
427  $ids = array_slice( $ids, $this->orphanBatchSize );
428  array_unshift( $args, 'doOrphanList' );
429  call_user_func_array( [ $this, 'dispatch' ], $args );
430  }
431  if ( count( $ids ) ) {
432  $args = $ids;
433  array_unshift( $args, 'doOrphanList' );
434  call_user_func_array( [ $this, 'dispatch' ], $args );
435  }
436 
437  $this->report( 'orphans', $i, $numOrphans );
438  }
439  $this->report( 'orphans', $i, $numOrphans );
440  $this->info( "All orphans queued." );
441  }
442 
446  function executeChild() {
447  $this->debug( 'starting' );
448  $this->syncDBs();
449 
450  while ( !feof( STDIN ) ) {
451  $line = rtrim( fgets( STDIN ) );
452  if ( $line == '' ) {
453  continue;
454  }
455  $this->debug( $line );
456  $args = explode( ' ', $line );
457  $cmd = array_shift( $args );
458  switch ( $cmd ) {
459  case 'doPage':
460  $this->doPage( intval( $args[0] ) );
461  break;
462  case 'doOrphanList':
463  $this->doOrphanList( array_map( 'intval', $args ) );
464  break;
465  case 'quit':
466  return;
467  }
468  MediaWikiServices::getInstance()->getDBLoadBalancerFactory()->waitForReplication();
469  }
470  }
471 
477  function doPage( $pageId ) {
478  $title = Title::newFromID( $pageId );
479  if ( $title ) {
480  $titleText = $title->getPrefixedText();
481  } else {
482  $titleText = '[deleted]';
483  }
484  $dbr = wfGetDB( DB_REPLICA );
485 
486  // Finish any incomplete transactions
487  if ( !$this->copyOnly ) {
488  $this->finishIncompleteMoves( [ 'bt_page' => $pageId ] );
489  $this->syncDBs();
490  }
491 
492  $startId = 0;
493  $trx = new CgzCopyTransaction( $this, $this->pageBlobClass );
494 
495  $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
496  while ( true ) {
497  $res = $dbr->select(
498  [ 'blob_tracking', 'text' ],
499  '*',
500  [
501  'bt_page' => $pageId,
502  'bt_text_id > ' . $dbr->addQuotes( $startId ),
503  'bt_moved' => 0,
504  'bt_new_url IS NULL',
505  'bt_text_id=old_id',
506  ],
507  __METHOD__,
508  [
509  'ORDER BY' => 'bt_text_id',
510  'LIMIT' => $this->batchSize
511  ]
512  );
513  if ( !$res->numRows() ) {
514  break;
515  }
516 
517  $lastTextId = 0;
518  foreach ( $res as $row ) {
519  $startId = $row->bt_text_id;
520  if ( $lastTextId == $row->bt_text_id ) {
521  // Duplicate (null edit)
522  continue;
523  }
524  $lastTextId = $row->bt_text_id;
525  // Load the text
526  $text = Revision::getRevisionText( $row );
527  if ( $text === false ) {
528  $this->critical( "Error loading {$row->bt_rev_id}/{$row->bt_text_id}" );
529  continue;
530  }
531 
532  // Queue it
533  if ( !$trx->addItem( $text, $row->bt_text_id ) ) {
534  $this->debug( "$titleText: committing blob with " . $trx->getSize() . " items" );
535  $trx->commit();
536  $trx = new CgzCopyTransaction( $this, $this->pageBlobClass );
537  $lbFactory->waitForReplication();
538  }
539  }
540  }
541 
542  $this->debug( "$titleText: committing blob with " . $trx->getSize() . " items" );
543  $trx->commit();
544  }
545 
559  function moveTextRow( $textId, $url ) {
560  if ( $this->copyOnly ) {
561  $this->critical( "Internal error: can't call moveTextRow() in --copy-only mode" );
562  exit( 1 );
563  }
564  $dbw = wfGetDB( DB_MASTER );
565  $dbw->begin( __METHOD__ );
566  $dbw->update( 'text',
567  [ // set
568  'old_text' => $url,
569  'old_flags' => 'external,utf-8',
570  ],
571  [ // where
572  'old_id' => $textId
573  ],
574  __METHOD__
575  );
576  $dbw->update( 'blob_tracking',
577  [ 'bt_moved' => 1 ],
578  [ 'bt_text_id' => $textId ],
579  __METHOD__
580  );
581  $dbw->commit( __METHOD__ );
582  }
583 
594  function finishIncompleteMoves( $conds ) {
595  $dbr = wfGetDB( DB_REPLICA );
596  $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
597 
598  $startId = 0;
599  $conds = array_merge( $conds, [
600  'bt_moved' => 0,
601  'bt_new_url IS NOT NULL'
602  ] );
603  while ( true ) {
604  $res = $dbr->select( 'blob_tracking',
605  '*',
606  array_merge( $conds, [ 'bt_text_id > ' . $dbr->addQuotes( $startId ) ] ),
607  __METHOD__,
608  [
609  'ORDER BY' => 'bt_text_id',
610  'LIMIT' => $this->batchSize,
611  ]
612  );
613  if ( !$res->numRows() ) {
614  break;
615  }
616  $this->debug( 'Incomplete: ' . $res->numRows() . ' rows' );
617  foreach ( $res as $row ) {
618  $startId = $row->bt_text_id;
619  $this->moveTextRow( $row->bt_text_id, $row->bt_new_url );
620  if ( $row->bt_text_id % 10 == 0 ) {
621  $lbFactory->waitForReplication();
622  }
623  }
624  }
625  }
626 
631  function getTargetCluster() {
632  $cluster = next( $this->destClusters );
633  if ( $cluster === false ) {
634  $cluster = reset( $this->destClusters );
635  }
636 
637  return $cluster;
638  }
639 
645  function getExtDB( $cluster ) {
646  $lb = wfGetLBFactory()->getExternalLB( $cluster );
647 
648  return $lb->getConnection( DB_MASTER );
649  }
650 
656  function doOrphanList( $textIds ) {
657  // Finish incomplete moves
658  if ( !$this->copyOnly ) {
659  $this->finishIncompleteMoves( [ 'bt_text_id' => $textIds ] );
660  $this->syncDBs();
661  }
662 
663  $trx = new CgzCopyTransaction( $this, $this->orphanBlobClass );
664 
665  $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
666  $res = wfGetDB( DB_REPLICA )->select(
667  [ 'text', 'blob_tracking' ],
668  [ 'old_id', 'old_text', 'old_flags' ],
669  [
670  'old_id' => $textIds,
671  'bt_text_id=old_id',
672  'bt_moved' => 0,
673  ],
674  __METHOD__,
675  [ 'DISTINCT' ]
676  );
677 
678  foreach ( $res as $row ) {
679  $text = Revision::getRevisionText( $row );
680  if ( $text === false ) {
681  $this->critical( "Error: cannot load revision text for old_id={$row->old_id}" );
682  continue;
683  }
684 
685  if ( !$trx->addItem( $text, $row->old_id ) ) {
686  $this->debug( "[orphan]: committing blob with " . $trx->getSize() . " rows" );
687  $trx->commit();
688  $trx = new CgzCopyTransaction( $this, $this->orphanBlobClass );
689  $lbFactory->waitForReplication();
690  }
691  }
692  $this->debug( "[orphan]: committing blob with " . $trx->getSize() . " rows" );
693  $trx->commit();
694  }
695 }
696 
702  public $parent;
703  public $blobClass;
705  public $cgz;
706  public $referrers;
707 
714  $this->blobClass = $blobClass;
715  $this->cgz = false;
716  $this->texts = [];
717  $this->parent = $parent;
718  }
719 
727  function addItem( $text, $textId ) {
728  if ( !$this->cgz ) {
729  $class = $this->blobClass;
730  $this->cgz = new $class;
731  }
732  $hash = $this->cgz->addItem( $text );
733  $this->referrers[$textId] = $hash;
734  $this->texts[$textId] = $text;
735 
736  return $this->cgz->isHappy();
737  }
738 
739  function getSize() {
740  return count( $this->texts );
741  }
742 
746  function recompress() {
747  $class = $this->blobClass;
748  $this->cgz = new $class;
749  $this->referrers = [];
750  foreach ( $this->texts as $textId => $text ) {
751  $hash = $this->cgz->addItem( $text );
752  $this->referrers[$textId] = $hash;
753  }
754  }
755 
761  function commit() {
762  $originalCount = count( $this->texts );
763  if ( !$originalCount ) {
764  return;
765  }
766 
767  /* Check to see if the target text_ids have been moved already.
768  *
769  * We originally read from the replica DB, so this can happen when a single
770  * text_id is shared between multiple pages. It's rare, but possible
771  * if a delete/move/undelete cycle splits up a null edit.
772  *
773  * We do a locking read to prevent closer-run race conditions.
774  */
775  $dbw = wfGetDB( DB_MASTER );
776  $dbw->begin( __METHOD__ );
777  $res = $dbw->select( 'blob_tracking',
778  [ 'bt_text_id', 'bt_moved' ],
779  [ 'bt_text_id' => array_keys( $this->referrers ) ],
780  __METHOD__, [ 'FOR UPDATE' ] );
781  $dirty = false;
782  foreach ( $res as $row ) {
783  if ( $row->bt_moved ) {
784  # This row has already been moved, remove it
785  $this->parent->debug( "TRX: conflict detected in old_id={$row->bt_text_id}" );
786  unset( $this->texts[$row->bt_text_id] );
787  $dirty = true;
788  }
789  }
790 
791  // Recompress the blob if necessary
792  if ( $dirty ) {
793  if ( !count( $this->texts ) ) {
794  // All have been moved already
795  if ( $originalCount > 1 ) {
796  // This is suspcious, make noise
797  $this->parent->critical(
798  "Warning: concurrent operation detected, are there two conflicting " .
799  "processes running, doing the same job?" );
800  }
801 
802  return;
803  }
804  $this->recompress();
805  }
806 
807  // Insert the data into the destination cluster
808  $targetCluster = $this->parent->getTargetCluster();
809  $store = $this->parent->store;
810  $targetDB = $store->getMaster( $targetCluster );
811  $targetDB->clearFlag( DBO_TRX ); // we manage the transactions
812  $targetDB->begin( __METHOD__ );
813  $baseUrl = $this->parent->store->store( $targetCluster, serialize( $this->cgz ) );
814 
815  // Write the new URLs to the blob_tracking table
816  foreach ( $this->referrers as $textId => $hash ) {
817  $url = $baseUrl . '/' . $hash;
818  $dbw->update( 'blob_tracking',
819  [ 'bt_new_url' => $url ],
820  [
821  'bt_text_id' => $textId,
822  'bt_moved' => 0, # Check for concurrent conflicting update
823  ],
824  __METHOD__
825  );
826  }
827 
828  $targetDB->commit( __METHOD__ );
829  // Critical section here: interruption at this point causes blob duplication
830  // Reversing the order of the commits would cause data loss instead
831  $dbw->commit( __METHOD__ );
832 
833  // Write the new URLs to the text table and set the moved flag
834  if ( !$this->parent->copyOnly ) {
835  foreach ( $this->referrers as $textId => $hash ) {
836  $url = $baseUrl . '/' . $hash;
837  $this->parent->moveTextRow( $textId, $url );
838  }
839  }
840  }
841 }
RecompressTracked\$cmdLineOptionMap
static $cmdLineOptionMap
Definition: recompressTracked.php:80
query
For a write query
Definition: database.txt:26
RecompressTracked\info
info( $msg)
Definition: recompressTracked.php:128
RecompressTracked
Maintenance script that moves blobs indexed by trackBlobs.php to a specified list of destination clus...
Definition: recompressTracked.php:56
ExternalStoreDB
DB accessable external objects.
Definition: ExternalStoreDB.php:36
RecompressTracked\logToFile
logToFile( $msg, $file)
Definition: recompressTracked.php:142
CgzCopyTransaction\addItem
addItem( $text, $textId)
Add text.
Definition: recompressTracked.php:727
is
We use the convention $dbr for read and $dbw for write to help you keep track of whether the database object is a the world will explode Or to be a subsequent write query which succeeded on the master may fail when replicated to the slave due to a unique key collision Replication on the slave will stop and it may take hours to repair the database and get it back online Setting read_only in my cnf on the slave will avoid this but given the dire we prefer to have as many checks as possible We provide a but the wrapper functions like please read the documentation for except in special pages derived from QueryPage It s a common pitfall for new developers to submit code containing SQL queries which examine huge numbers of rows Remember that COUNT * is(N), counting rows in atable is like counting beans in a bucket.------------------------------------------------------------------------ Replication------------------------------------------------------------------------The largest installation of MediaWiki, Wikimedia, uses a large set ofslave MySQL servers replicating writes made to a master MySQL server. Itis important to understand the issues associated with this setup if youwant to write code destined for Wikipedia.It 's often the case that the best algorithm to use for a given taskdepends on whether or not replication is in use. Due to our unabashedWikipedia-centrism, we often just use the replication-friendly version, but if you like, you can use wfGetLB() ->getServerCount() > 1 tocheck to see if replication is in use.===Lag===Lag primarily occurs when large write queries are sent to the master.Writes on the master are executed in parallel, but they are executed inserial when they are replicated to the slaves. The master writes thequery to the binlog when the transaction is committed. The slaves pollthe binlog and start executing the query as soon as it appears. They canservice reads while they are performing a write query, but will not readanything more from the binlog and thus will perform no more writes. Thismeans that if the write query runs for a long time, the slaves will lagbehind the master for the time it takes for the write query to complete.Lag can be exacerbated by high read load. MediaWiki 's load balancer willstop sending reads to a slave when it is lagged by more than 30 seconds.If the load ratios are set incorrectly, or if there is too much loadgenerally, this may lead to a slave permanently hovering around 30seconds lag.If all slaves are lagged by more than 30 seconds, MediaWiki will stopwriting to the database. All edits and other write operations will berefused, with an error returned to the user. This gives the slaves achance to catch up. Before we had this mechanism, the slaves wouldregularly lag by several minutes, making review of recent editsdifficult.In addition to this, MediaWiki attempts to ensure that the user seesevents occurring on the wiki in chronological order. A few seconds of lagcan be tolerated, as long as the user sees a consistent picture fromsubsequent requests. This is done by saving the master binlog positionin the session, and then at the start of each request, waiting for theslave to catch up to that position before doing any reads from it. Ifthis wait times out, reads are allowed anyway, but the request isconsidered to be in "lagged slave mode". Lagged slave mode can bechecked by calling wfGetLB() ->getLaggedSlaveMode(). The onlypractical consequence at present is a warning displayed in the pagefooter.===Lag avoidance===To avoid excessive lag, queries which write large numbers of rows shouldbe split up, generally to write one row at a time. Multi-row INSERT ...SELECT queries are the worst offenders should be avoided altogether.Instead do the select first and then the insert.===Working with lag===Despite our best efforts, it 's not practical to guarantee a low-lagenvironment. Lag will usually be less than one second, but mayoccasionally be up to 30 seconds. For scalability, it 's very importantto keep load on the master low, so simply sending all your queries tothe master is not the answer. So when you have a genuine need forup-to-date data, the following approach is advised:1) Do a quick query to the master for a sequence number or timestamp 2) Run the full query on the slave and check if it matches the data you gotfrom the master 3) If it doesn 't, run the full query on the masterTo avoid swamping the master every time the slaves lag, use of thisapproach should be kept to a minimum. In most cases you should just readfrom the slave and let the user deal with the delay.------------------------------------------------------------------------ Lock contention------------------------------------------------------------------------Due to the high write rate on Wikipedia(and some other wikis), MediaWiki developers need to be very careful to structure their writesto avoid long-lasting locks. By default, MediaWiki opens a transactionat the first query, and commits it before the output is sent. Locks willbe held from the time when the query is done until the commit. So youcan reduce lock time by doing as much processing as possible before youdo your write queries.Often this approach is not good enough, and it becomes necessary toenclose small groups of queries in their own transaction. Use thefollowing syntax:$dbw=wfGetDB(DB_MASTER
RecompressTracked\$copyOnly
$copyOnly
Definition: recompressTracked.php:65
captcha-old.count
count
Definition: captcha-old.py:225
ConcatenatedGzipHistoryBlob
Concatenated gzip (CGZ) storage Improves compression ratio by concatenating like objects before gzipp...
Definition: HistoryBlob.php:73
RecompressTracked\$optionsWithArgs
static $optionsWithArgs
Definition: recompressTracked.php:72
RecompressTracked\$debugLog
$debugLog
Definition: recompressTracked.php:69
RecompressTracked\getTargetCluster
getTargetCluster()
Returns the name of the next target cluster.
Definition: recompressTracked.php:631
$status
this hook is for auditing only RecentChangesLinked and Watchlist RecentChangesLinked and Watchlist Do not use this to implement individual filters if they are compatible with the ChangesListFilter and ChangesListFilterGroup structure use sub classes of those in conjunction with the ChangesListSpecialPageStructuredFilters hook This hook can be used to implement filters that do not implement that or custom behavior that is not an individual filter e g Watchlist and Watchlist you will want to construct new ChangesListBooleanFilter or ChangesListStringOptionsFilter objects When constructing you specify which group they belong to You can reuse existing or create your you must register them with $special registerFilterGroup removed from all revisions and log entries to which it was applied This gives extensions a chance to take it off their books as the deletion has already been partly carried out by this point or something similar the user will be unable to create the tag set $status
Definition: hooks.txt:1049
use
as see the revision history and available at free of to any person obtaining a copy of this software and associated documentation to deal in the Software without including without limitation the rights to use
Definition: MIT-LICENSE.txt:10
RecompressTracked\debug
debug( $msg)
Definition: recompressTracked.php:121
wfHostname
wfHostname()
Fetch server name for use in error reporting etc.
Definition: GlobalFunctions.php:1435
serialize
serialize()
Definition: ApiMessage.php:177
RecompressTracked\$isChild
$isChild
Definition: recompressTracked.php:66
Revision\getRevisionText
static getRevisionText( $row, $prefix='old_', $wiki=false)
Get revision text associated with an old or archive row.
Definition: Revision.php:1258
$res
$res
Definition: database.txt:21
$name
Allows to change the fields on the form that will be generated $name
Definition: hooks.txt:304
so
c Accompany it with the information you received as to the offer to distribute corresponding source complete source code means all the source code for all modules it plus any associated interface definition plus the scripts used to control compilation and installation of the executable as a special the source code distributed need not include anything that is normally and so on of the operating system on which the executable unless that component itself accompanies the executable If distribution of executable or object code is made by offering access to copy from a designated then offering equivalent access to copy the source code from the same place counts as distribution of the source even though third parties are not compelled to copy the source along with the object code You may not or distribute the Program except as expressly provided under this License Any attempt otherwise to sublicense or distribute the Program is and will automatically terminate your rights under this License parties who have received or from you under this License will not have their licenses terminated so long as such parties remain in full compliance You are not required to accept this since you have not signed it nothing else grants you permission to modify or distribute the Program or its derivative works These actions are prohibited by law if you do not accept this License by modifying or distributing the you indicate your acceptance of this License to do so
Definition: COPYING.txt:185
RecompressTracked\$replicaProcs
$replicaProcs
Definition: recompressTracked.php:64
CgzCopyTransaction\recompress
recompress()
Recompress text after some aberrant modification.
Definition: recompressTracked.php:746
$lbFactory
$lbFactory
Definition: doMaintenance.php:117
RecompressTracked\getExtDB
getExtDB( $cluster)
Gets a DB master connection for the given external cluster name.
Definition: recompressTracked.php:645
RecompressTracked\executeParent
executeParent()
Execute the parent process.
Definition: recompressTracked.php:177
DBO_TRX
const DBO_TRX
Definition: defines.php:12
php
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
CgzCopyTransaction\getSize
getSize()
Definition: recompressTracked.php:739
RecompressTracked\$replicaId
$replicaId
Definition: recompressTracked.php:67
RecompressTracked\$destClusters
$destClusters
Definition: recompressTracked.php:57
$title
namespace and then decline to actually register it file or subcat img or subcat $title
Definition: hooks.txt:934
RecompressTracked\$orphanBlobClass
$orphanBlobClass
Definition: recompressTracked.php:63
RecompressTracked\$pageBlobClass
$pageBlobClass
Definition: recompressTracked.php:63
RecompressTracked\$prevReplicaId
$prevReplicaId
Definition: recompressTracked.php:64
RecompressTracked\finishIncompleteMoves
finishIncompleteMoves( $conds)
Moves are done in two phases: bt_new_url and then bt_moved.
Definition: recompressTracked.php:594
wfGetDB
wfGetDB( $db, $groups=[], $wiki=false)
Get a Database object.
Definition: GlobalFunctions.php:3060
RecompressTracked\$reportingInterval
$reportingInterval
Definition: recompressTracked.php:60
RecompressTracked\critical
critical( $msg)
Definition: recompressTracked.php:135
CgzCopyTransaction\commit
commit()
Commit the blob.
Definition: recompressTracked.php:761
CgzCopyTransaction\$parent
RecompressTracked $parent
Definition: recompressTracked.php:702
RecompressTracked\$criticalLog
$criticalLog
Definition: recompressTracked.php:69
store
MediaWiki s SiteStore can be cached and stored in a flat in a json format If the SiteStore is frequently the file cache may provide a performance benefit over a database store
Definition: sitescache.txt:1
RecompressTracked\killReplicaProcs
killReplicaProcs()
Gracefully terminate the child processes.
Definition: recompressTracked.php:255
RecompressTracked\doPage
doPage( $pageId)
Move tracked text in a given page.
Definition: recompressTracked.php:477
RecompressTracked\$orphanBatchSize
$orphanBatchSize
Definition: recompressTracked.php:59
RecompressTracked\newFromCommandLine
static newFromCommandLine( $args, $options)
Definition: recompressTracked.php:95
DB_REPLICA
const DB_REPLICA
Definition: defines.php:25
$GLOBALS
$GLOBALS['wgAutoloadClasses']['LocalisationUpdate']
Definition: Autoload.php:10
DB_MASTER
const DB_MASTER
Definition: defines.php:26
wfDebug
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
Definition: GlobalFunctions.php:999
RecompressTracked\$batchSize
$batchSize
Definition: recompressTracked.php:58
RecompressTracked\doAllOrphans
doAllOrphans()
Move all orphan text to the new clusters.
Definition: recompressTracked.php:377
CgzCopyTransaction\$referrers
$referrers
Definition: recompressTracked.php:706
$line
$line
Definition: cdb.php:58
RecompressTracked\$store
$store
Definition: recompressTracked.php:70
wfWikiID
wfWikiID()
Get an ASCII string identifying this wiki This is used as a prefix in memcached keys.
Definition: GlobalFunctions.php:3011
CgzCopyTransaction\$blobClass
$blobClass
Definition: recompressTracked.php:703
$value
$value
Definition: styleTest.css.php:45
$header
$header
Definition: updateCredits.php:35
wfEscapeShellArg
wfEscapeShellArg()
Version of escapeshellarg() that works better on Windows.
Definition: GlobalFunctions.php:2195
CgzCopyTransaction
Class to represent a recompression operation for a single CGZ blob.
Definition: recompressTracked.php:700
RecompressTracked\$replicaPipes
$replicaPipes
Definition: recompressTracked.php:64
RecompressTracked\report
report( $label, $current, $end)
Display a progress report.
Definition: recompressTracked.php:365
RecompressTracked\moveTextRow
moveTextRow( $textId, $url)
Atomic move operation.
Definition: recompressTracked.php:559
CgzCopyTransaction\__construct
__construct( $parent, $blobClass)
Create a transaction from a RecompressTracked object.
Definition: recompressTracked.php:713
$optionsWithArgs
$optionsWithArgs
Definition: recompressTracked.php:28
wfGetLBFactory
wfGetLBFactory()
Get the load balancer factory object.
Definition: GlobalFunctions.php:3089
RecompressTracked\dispatchToReplica
dispatchToReplica( $replicaId, $args)
Dispatch a command to a specified replica DB.
Definition: recompressTracked.php:299
RecompressTracked\executeChild
executeChild()
Main entry point for worker processes.
Definition: recompressTracked.php:446
RecompressTracked\checkTrackingTable
checkTrackingTable()
Make sure the tracking table exists and isn't empty.
Definition: recompressTracked.php:193
$args
if( $line===false) $args
Definition: cdb.php:63
RecompressTracked\execute
execute()
Execute parent or child depending on the isChild option.
Definition: recompressTracked.php:166
$dbr
if(! $regexes) $dbr
Definition: cleanup.php:94
$job
if(count( $args)< 1) $job
Definition: recompressTracked.php:47
as
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
RecompressTracked\startReplicaProcs
startReplicaProcs()
Start the worker processes.
Definition: recompressTracked.php:216
RecompressTracked\$numProcs
$numProcs
Definition: recompressTracked.php:61
RecompressTracked\$numBatches
$numBatches
Definition: recompressTracked.php:62
RecompressTracked\$infoLog
$infoLog
Definition: recompressTracked.php:69
RecompressTracked\doAllPages
doAllPages()
Move all tracked pages to the new clusters.
Definition: recompressTracked.php:308
that
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global then executing the whole list after the page is displayed We don t do anything smart like collating updates to the same table or such because the list is almost always going to have just one item on if that
Definition: deferred.txt:11
RecompressTracked\dispatch
dispatch()
Dispatch a command to the next available replica DB.
Definition: recompressTracked.php:273
MediaWikiServices
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency MediaWikiServices
Definition: injection.txt:23
RecompressTracked\__construct
__construct( $options)
Definition: recompressTracked.php:106
RecompressTracked\syncDBs
syncDBs()
Wait until the selected replica DB has caught up to the master.
Definition: recompressTracked.php:156
Title\newFromID
static newFromID( $id, $flags=0)
Create a new Title from an article ID.
Definition: Title.php:405
RecompressTracked\$noCount
$noCount
Definition: recompressTracked.php:68
$options
this hook is for auditing only RecentChangesLinked and Watchlist RecentChangesLinked and Watchlist Do not use this to implement individual filters if they are compatible with the ChangesListFilter and ChangesListFilterGroup structure use sub classes of those in conjunction with the ChangesListSpecialPageStructuredFilters hook This hook can be used to implement filters that do not implement that or custom behavior that is not an individual filter e g Watchlist and Watchlist you will want to construct new ChangesListBooleanFilter or ChangesListStringOptionsFilter objects When constructing you specify which group they belong to You can reuse existing or create your you must register them with $special registerFilterGroup removed from all revisions and log entries to which it was applied This gives extensions a chance to take it off their books as the deletion has already been partly carried out by this point or something similar the user will be unable to create the tag set and then return false from the hook function Ensure you consume the ChangeTagAfterDelete hook to carry out custom deletion actions as context called by AbstractContent::getParserOutput May be used to override the normal model specific rendering of page content as context as context $options
Definition: hooks.txt:1049
LegacyLogger
MediaWiki Logger LegacyLogger
Definition: logger.txt:54
CgzCopyTransaction\$cgz
ConcatenatedGzipHistoryBlob $cgz
Definition: recompressTracked.php:705
RecompressTracked\doOrphanList
doOrphanList( $textIds)
Move an orphan text_id to the new cluster.
Definition: recompressTracked.php:656
array
the array() calling protocol came about after MediaWiki 1.4rc1.
RecompressTracked\getOptionsWithArgs
static getOptionsWithArgs()
Definition: recompressTracked.php:91