MediaWiki  master
dumpTextPass.php
Go to the documentation of this file.
1 <?php
27 require_once __DIR__ . '/backup.inc';
28 require_once __DIR__ . '/../includes/export/WikiExporter.php';
29 
31 
37  public $prefetch = null;
39  private $thisPage;
41  private $thisRev;
42 
43  // when we spend more than maxTimeAllowed seconds on this run, we continue
44  // processing until we write out the next complete page, then save output file(s),
45  // rename it/them and open new one(s)
46  public $maxTimeAllowed = 0; // 0 = no limit
47 
48  protected $input = "php://stdin";
50  protected $fetchCount = 0;
51  protected $prefetchCount = 0;
52  protected $prefetchCountLast = 0;
53  protected $fetchCountLast = 0;
54 
55  protected $maxFailures = 5;
57  protected $failureTimeout = 5; // Seconds to sleep after db failure
58 
59  protected $bufferSize = 524288; // In bytes. Maximum size to read from the stub in on go.
60 
61  protected $php = "php";
62  protected $spawn = false;
63 
67  protected $spawnProc = false;
68 
72  protected $spawnWrite = false;
73 
77  protected $spawnRead = false;
78 
82  protected $spawnErr = false;
83 
87  protected $xmlwriterobj = false;
88 
89  protected $timeExceeded = false;
90  protected $firstPageWritten = false;
91  protected $lastPageWritten = false;
92  protected $checkpointJustWritten = false;
93  protected $checkpointFiles = [];
94 
98  protected $db;
99 
103  function __construct( $args = null ) {
104  parent::__construct();
105 
106  $this->addDescription( <<<TEXT
107 This script postprocesses XML dumps from dumpBackup.php to add
108 page text which was stubbed out (using --stub).
109 
110 XML input is accepted on stdin.
111 XML output is sent to stdout; progress reports are sent to stderr.
112 TEXT
113  );
114  $this->stderr = fopen( "php://stderr", "wt" );
115 
116  $this->addOption( 'stub', 'To load a compressed stub dump instead of stdin. ' .
117  'Specify as --stub=<type>:<file>.', false, true );
118  $this->addOption( 'prefetch', 'Use a prior dump file as a text source, to savepressure on the ' .
119  'database. (Requires the XMLReader extension). Specify as --prefetch=<type>:<file>',
120  false, true );
121  $this->addOption( 'maxtime', 'Write out checkpoint file after this many minutes (writing' .
122  'out complete page, closing xml file properly, and opening new one' .
123  'with header). This option requires the checkpointfile option.', false, true );
124  $this->addOption( 'checkpointfile', 'Use this string for checkpoint filenames,substituting ' .
125  'first pageid written for the first %s (required) and the last pageid written for the ' .
126  'second %s if it exists.', false, true, false, true ); // This can be specified multiple times
127  $this->addOption( 'quiet', 'Don\'t dump status reports to stderr.' );
128  $this->addOption( 'current', 'Base ETA on number of pages in database instead of all revisions' );
129  $this->addOption( 'spawn', 'Spawn a subprocess for loading text records' );
130  $this->addOption( 'buffersize', 'Buffer size in bytes to use for reading the stub. ' .
131  '(Default: 512KB, Minimum: 4KB)', false, true );
132 
133  if ( $args ) {
134  $this->loadWithArgv( $args );
135  $this->processOptions();
136  }
137  }
138 
139  function execute() {
140  $this->processOptions();
141  $this->dump( true );
142  }
143 
144  function processOptions() {
145  global $IP;
146 
147  parent::processOptions();
148 
149  if ( $this->hasOption( 'buffersize' ) ) {
150  $this->bufferSize = max( intval( $this->getOption( 'buffersize' ) ), 4 * 1024 );
151  }
152 
153  if ( $this->hasOption( 'prefetch' ) ) {
154  require_once "$IP/maintenance/backupPrefetch.inc";
155  $url = $this->processFileOpt( $this->getOption( 'prefetch' ) );
156  $this->prefetch = new BaseDump( $url );
157  }
158 
159  if ( $this->hasOption( 'stub' ) ) {
160  $this->input = $this->processFileOpt( $this->getOption( 'stub' ) );
161  }
162 
163  if ( $this->hasOption( 'maxtime' ) ) {
164  $this->maxTimeAllowed = intval( $this->getOption( 'maxtime' ) ) * 60;
165  }
166 
167  if ( $this->hasOption( 'checkpointfile' ) ) {
168  $this->checkpointFiles = $this->getOption( 'checkpointfile' );
169  }
170 
171  if ( $this->hasOption( 'current' ) ) {
173  }
174 
175  if ( $this->hasOption( 'full' ) ) {
176  $this->history = WikiExporter::FULL;
177  }
178 
179  if ( $this->hasOption( 'spawn' ) ) {
180  $this->spawn = true;
181  $val = $this->getOption( 'spawn' );
182  if ( $val !== 1 ) {
183  $this->php = $val;
184  }
185  }
186  }
187 
199  function rotateDb() {
200  // Cleaning up old connections
201  if ( isset( $this->lb ) ) {
202  $this->lb->closeAll();
203  unset( $this->lb );
204  }
205 
206  if ( $this->forcedDb !== null ) {
207  $this->db = $this->forcedDb;
208 
209  return;
210  }
211 
212  if ( isset( $this->db ) && $this->db->isOpen() ) {
213  throw new MWException( 'DB is set and has not been closed by the Load Balancer' );
214  }
215 
216  unset( $this->db );
217 
218  // Trying to set up new connection.
219  // We do /not/ retry upon failure, but delegate to encapsulating logic, to avoid
220  // individually retrying at different layers of code.
221 
222  try {
223  $this->lb = wfGetLBFactory()->newMainLB();
224  } catch ( Exception $e ) {
225  throw new MWException( __METHOD__
226  . " rotating DB failed to obtain new load balancer (" . $e->getMessage() . ")" );
227  }
228 
229  try {
230  $this->db = $this->lb->getConnection( DB_REPLICA, 'dump' );
231  } catch ( Exception $e ) {
232  throw new MWException( __METHOD__
233  . " rotating DB failed to obtain new database (" . $e->getMessage() . ")" );
234  }
235  }
236 
238  parent::initProgress();
239  $this->timeOfCheckpoint = $this->startTime;
240  }
241 
242  function dump( $history, $text = WikiExporter::TEXT ) {
243  // Notice messages will foul up your XML output even if they're
244  // relatively harmless.
245  if ( ini_get( 'display_errors' ) ) {
246  ini_set( 'display_errors', 'stderr' );
247  }
248 
249  $this->initProgress( $this->history );
250 
251  // We are trying to get an initial database connection to avoid that the
252  // first try of this request's first call to getText fails. However, if
253  // obtaining a good DB connection fails it's not a serious issue, as
254  // getText does retry upon failure and can start without having a working
255  // DB connection.
256  try {
257  $this->rotateDb();
258  } catch ( Exception $e ) {
259  // We do not even count this as failure. Just let eventual
260  // watchdogs know.
261  $this->progress( "Getting initial DB connection failed (" .
262  $e->getMessage() . ")" );
263  }
264 
265  $this->egress = new ExportProgressFilter( $this->sink, $this );
266 
267  // it would be nice to do it in the constructor, oh well. need egress set
268  $this->finalOptionCheck();
269 
270  // we only want this so we know how to close a stream :-P
271  $this->xmlwriterobj = new XmlDumpWriter();
272 
273  $input = fopen( $this->input, "rt" );
274  $this->readDump( $input );
275 
276  if ( $this->spawnProc ) {
277  $this->closeSpawn();
278  }
279 
280  $this->report( true );
281  }
282 
283  function processFileOpt( $opt ) {
284  $split = explode( ':', $opt, 2 );
285  $val = $split[0];
286  $param = '';
287  if ( count( $split ) === 2 ) {
288  $param = $split[1];
289  }
290  $fileURIs = explode( ';', $param );
291  foreach ( $fileURIs as $URI ) {
292  switch ( $val ) {
293  case "file":
294  $newURI = $URI;
295  break;
296  case "gzip":
297  $newURI = "compress.zlib://$URI";
298  break;
299  case "bzip2":
300  $newURI = "compress.bzip2://$URI";
301  break;
302  case "7zip":
303  $newURI = "mediawiki.compress.7z://$URI";
304  break;
305  default:
306  $newURI = $URI;
307  }
308  $newFileURIs[] = $newURI;
309  }
310  $val = implode( ';', $newFileURIs );
311 
312  return $val;
313  }
314 
318  function showReport() {
319  if ( !$this->prefetch ) {
320  parent::showReport();
321 
322  return;
323  }
324 
325  if ( $this->reporting ) {
326  $now = wfTimestamp( TS_DB );
327  $nowts = microtime( true );
328  $deltaAll = $nowts - $this->startTime;
329  $deltaPart = $nowts - $this->lastTime;
330  $this->pageCountPart = $this->pageCount - $this->pageCountLast;
331  $this->revCountPart = $this->revCount - $this->revCountLast;
332 
333  if ( $deltaAll ) {
334  $portion = $this->revCount / $this->maxCount;
335  $eta = $this->startTime + $deltaAll / $portion;
336  $etats = wfTimestamp( TS_DB, intval( $eta ) );
337  if ( $this->fetchCount ) {
338  $fetchRate = 100.0 * $this->prefetchCount / $this->fetchCount;
339  } else {
340  $fetchRate = '-';
341  }
342  $pageRate = $this->pageCount / $deltaAll;
343  $revRate = $this->revCount / $deltaAll;
344  } else {
345  $pageRate = '-';
346  $revRate = '-';
347  $etats = '-';
348  $fetchRate = '-';
349  }
350  if ( $deltaPart ) {
351  if ( $this->fetchCountLast ) {
352  $fetchRatePart = 100.0 * $this->prefetchCountLast / $this->fetchCountLast;
353  } else {
354  $fetchRatePart = '-';
355  }
356  $pageRatePart = $this->pageCountPart / $deltaPart;
357  $revRatePart = $this->revCountPart / $deltaPart;
358  } else {
359  $fetchRatePart = '-';
360  $pageRatePart = '-';
361  $revRatePart = '-';
362  }
363  $this->progress( sprintf(
364  "%s: %s (ID %d) %d pages (%0.1f|%0.1f/sec all|curr), "
365  . "%d revs (%0.1f|%0.1f/sec all|curr), %0.1f%%|%0.1f%% "
366  . "prefetched (all|curr), ETA %s [max %d]",
367  $now, wfWikiID(), $this->ID, $this->pageCount, $pageRate,
368  $pageRatePart, $this->revCount, $revRate, $revRatePart,
369  $fetchRate, $fetchRatePart, $etats, $this->maxCount
370  ) );
371  $this->lastTime = $nowts;
372  $this->revCountLast = $this->revCount;
373  $this->prefetchCountLast = $this->prefetchCount;
374  $this->fetchCountLast = $this->fetchCount;
375  }
376  }
377 
378  function setTimeExceeded() {
379  $this->timeExceeded = true;
380  }
381 
382  function checkIfTimeExceeded() {
383  if ( $this->maxTimeAllowed
384  && ( $this->lastTime - $this->timeOfCheckpoint > $this->maxTimeAllowed )
385  ) {
386  return true;
387  }
388 
389  return false;
390  }
391 
392  function finalOptionCheck() {
393  if ( ( $this->checkpointFiles && !$this->maxTimeAllowed )
394  || ( $this->maxTimeAllowed && !$this->checkpointFiles )
395  ) {
396  throw new MWException( "Options checkpointfile and maxtime must be specified together.\n" );
397  }
398  foreach ( $this->checkpointFiles as $checkpointFile ) {
399  $count = substr_count( $checkpointFile, "%s" );
400  if ( $count != 2 ) {
401  throw new MWException( "Option checkpointfile must contain two '%s' "
402  . "for substitution of first and last pageids, count is $count instead, "
403  . "file is $checkpointFile.\n" );
404  }
405  }
406 
407  if ( $this->checkpointFiles ) {
408  $filenameList = (array)$this->egress->getFilenames();
409  if ( count( $filenameList ) != count( $this->checkpointFiles ) ) {
410  throw new MWException( "One checkpointfile must be specified "
411  . "for each output option, if maxtime is used.\n" );
412  }
413  }
414  }
415 
421  function readDump( $input ) {
422  $this->buffer = "";
423  $this->openElement = false;
424  $this->atStart = true;
425  $this->state = "";
426  $this->lastName = "";
427  $this->thisPage = 0;
428  $this->thisRev = 0;
429  $this->thisRevModel = null;
430  $this->thisRevFormat = null;
431 
432  $parser = xml_parser_create( "UTF-8" );
433  xml_parser_set_option( $parser, XML_OPTION_CASE_FOLDING, false );
434 
435  xml_set_element_handler(
436  $parser,
437  [ $this, 'startElement' ],
438  [ $this, 'endElement' ]
439  );
440  xml_set_character_data_handler( $parser, [ $this, 'characterData' ] );
441 
442  $offset = 0; // for context extraction on error reporting
443  do {
444  if ( $this->checkIfTimeExceeded() ) {
445  $this->setTimeExceeded();
446  }
447  $chunk = fread( $input, $this->bufferSize );
448  if ( !xml_parse( $parser, $chunk, feof( $input ) ) ) {
449  wfDebug( "TextDumpPass::readDump encountered XML parsing error\n" );
450 
451  $byte = xml_get_current_byte_index( $parser );
452  $msg = wfMessage( 'xml-error-string',
453  'XML import parse failure',
454  xml_get_current_line_number( $parser ),
455  xml_get_current_column_number( $parser ),
456  $byte . ( is_null( $chunk ) ? null : ( '; "' . substr( $chunk, $byte - $offset, 16 ) . '"' ) ),
457  xml_error_string( xml_get_error_code( $parser ) ) )->escaped();
458 
459  xml_parser_free( $parser );
460 
461  throw new MWException( $msg );
462  }
463  $offset += strlen( $chunk );
464  } while ( $chunk !== false && !feof( $input ) );
465  if ( $this->maxTimeAllowed ) {
466  $filenameList = (array)$this->egress->getFilenames();
467  // we wrote some stuff after last checkpoint that needs renamed
468  if ( file_exists( $filenameList[0] ) ) {
469  $newFilenames = [];
470  # we might have just written the header and footer and had no
471  # pages or revisions written... perhaps they were all deleted
472  # there's no pageID 0 so we use that. the caller is responsible
473  # for deciding what to do with a file containing only the
474  # siteinfo information and the mw tags.
475  if ( !$this->firstPageWritten ) {
476  $firstPageID = str_pad( 0, 9, "0", STR_PAD_LEFT );
477  $lastPageID = str_pad( 0, 9, "0", STR_PAD_LEFT );
478  } else {
479  $firstPageID = str_pad( $this->firstPageWritten, 9, "0", STR_PAD_LEFT );
480  $lastPageID = str_pad( $this->lastPageWritten, 9, "0", STR_PAD_LEFT );
481  }
482 
483  $filenameCount = count( $filenameList );
484  for ( $i = 0; $i < $filenameCount; $i++ ) {
485  $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID );
486  $fileinfo = pathinfo( $filenameList[$i] );
487  $newFilenames[] = $fileinfo['dirname'] . '/' . $checkpointNameFilledIn;
488  }
489  $this->egress->closeAndRename( $newFilenames );
490  }
491  }
492  xml_parser_free( $parser );
493 
494  return true;
495  }
496 
506  private function exportTransform( $text, $model, $format = null ) {
507  try {
509  $text = $handler->exportTransform( $text, $format );
510  }
511  catch ( MWException $ex ) {
512  $this->progress(
513  "Unable to apply export transformation for content model '$model': " .
514  $ex->getMessage()
515  );
516  }
517 
518  return $text;
519  }
520 
541  function getText( $id, $model = null, $format = null ) {
542  global $wgContentHandlerUseDB;
543 
544  $prefetchNotTried = true; // Whether or not we already tried to get the text via prefetch.
545  $text = false; // The candidate for a good text. false if no proper value.
546  $failures = 0; // The number of times, this invocation of getText already failed.
547 
548  // The number of times getText failed without yielding a good text in between.
549  static $consecutiveFailedTextRetrievals = 0;
550 
551  $this->fetchCount++;
552 
553  // To allow to simply return on success and do not have to worry about book keeping,
554  // we assume, this fetch works (possible after some retries). Nevertheless, we koop
555  // the old value, so we can restore it, if problems occur (See after the while loop).
556  $oldConsecutiveFailedTextRetrievals = $consecutiveFailedTextRetrievals;
557  $consecutiveFailedTextRetrievals = 0;
558 
559  if ( $model === null && $wgContentHandlerUseDB ) {
560  $row = $this->db->selectRow(
561  'revision',
562  [ 'rev_content_model', 'rev_content_format' ],
563  [ 'rev_id' => $this->thisRev ],
564  __METHOD__
565  );
566 
567  if ( $row ) {
568  $model = $row->rev_content_model;
569  $format = $row->rev_content_format;
570  }
571  }
572 
573  if ( $model === null || $model === '' ) {
574  $model = false;
575  }
576 
577  while ( $failures < $this->maxFailures ) {
578  // As soon as we found a good text for the $id, we will return immediately.
579  // Hence, if we make it past the try catch block, we know that we did not
580  // find a good text.
581 
582  try {
583  // Step 1: Get some text (or reuse from previous iteratuon if checking
584  // for plausibility failed)
585 
586  // Trying to get prefetch, if it has not been tried before
587  if ( $text === false && isset( $this->prefetch ) && $prefetchNotTried ) {
588  $prefetchNotTried = false;
589  $tryIsPrefetch = true;
590  $text = $this->prefetch->prefetch( (int)$this->thisPage, (int)$this->thisRev );
591 
592  if ( $text === null ) {
593  $text = false;
594  }
595 
596  if ( is_string( $text ) && $model !== false ) {
597  // Apply export transformation to text coming from an old dump.
598  // The purpose of this transformation is to convert up from legacy
599  // formats, which may still be used in the older dump that is used
600  // for pre-fetching. Applying the transformation again should not
601  // interfere with content that is already in the correct form.
602  $text = $this->exportTransform( $text, $model, $format );
603  }
604  }
605 
606  if ( $text === false ) {
607  // Fallback to asking the database
608  $tryIsPrefetch = false;
609  if ( $this->spawn ) {
610  $text = $this->getTextSpawned( $id );
611  } else {
612  $text = $this->getTextDb( $id );
613  }
614 
615  if ( $text !== false && $model !== false ) {
616  // Apply export transformation to text coming from the database.
617  // Prefetched text should already have transformations applied.
618  $text = $this->exportTransform( $text, $model, $format );
619  }
620 
621  // No more checks for texts from DB for now.
622  // If we received something that is not false,
623  // We treat it as good text, regardless of whether it actually is or is not
624  if ( $text !== false ) {
625  return $text;
626  }
627  }
628 
629  if ( $text === false ) {
630  throw new MWException( "Generic error while obtaining text for id " . $id );
631  }
632 
633  // We received a good candidate for the text of $id via some method
634 
635  // Step 2: Checking for plausibility and return the text if it is
636  // plausible
637  $revID = intval( $this->thisRev );
638  if ( !isset( $this->db ) ) {
639  throw new MWException( "No database available" );
640  }
641 
642  if ( $model !== CONTENT_MODEL_WIKITEXT ) {
643  $revLength = strlen( $text );
644  } else {
645  $revLength = $this->db->selectField( 'revision', 'rev_len', [ 'rev_id' => $revID ] );
646  }
647 
648  if ( strlen( $text ) == $revLength ) {
649  if ( $tryIsPrefetch ) {
650  $this->prefetchCount++;
651  }
652 
653  return $text;
654  }
655 
656  $text = false;
657  throw new MWException( "Received text is unplausible for id " . $id );
658  } catch ( Exception $e ) {
659  $msg = "getting/checking text " . $id . " failed (" . $e->getMessage() . ")";
660  if ( $failures + 1 < $this->maxFailures ) {
661  $msg .= " (Will retry " . ( $this->maxFailures - $failures - 1 ) . " more times)";
662  }
663  $this->progress( $msg );
664  }
665 
666  // Something went wrong; we did not a text that was plausible :(
667  $failures++;
668 
669  // A failure in a prefetch hit does not warrant resetting db connection etc.
670  if ( !$tryIsPrefetch ) {
671  // After backing off for some time, we try to reboot the whole process as
672  // much as possible to not carry over failures from one part to the other
673  // parts
674  sleep( $this->failureTimeout );
675  try {
676  $this->rotateDb();
677  if ( $this->spawn ) {
678  $this->closeSpawn();
679  $this->openSpawn();
680  }
681  } catch ( Exception $e ) {
682  $this->progress( "Rebooting getText infrastructure failed (" . $e->getMessage() . ")" .
683  " Trying to continue anyways" );
684  }
685  }
686  }
687 
688  // Retirieving a good text for $id failed (at least) maxFailures times.
689  // We abort for this $id.
690 
691  // Restoring the consecutive failures, and maybe aborting, if the dump
692  // is too broken.
693  $consecutiveFailedTextRetrievals = $oldConsecutiveFailedTextRetrievals + 1;
694  if ( $consecutiveFailedTextRetrievals > $this->maxConsecutiveFailedTextRetrievals ) {
695  throw new MWException( "Graceful storage failure" );
696  }
697 
698  return "";
699  }
700 
707  private function getTextDb( $id ) {
709  if ( !isset( $this->db ) ) {
710  throw new MWException( __METHOD__ . "No database available" );
711  }
712  $row = $this->db->selectRow( 'text',
713  [ 'old_text', 'old_flags' ],
714  [ 'old_id' => $id ],
715  __METHOD__ );
716  $text = Revision::getRevisionText( $row );
717  if ( $text === false ) {
718  return false;
719  }
720  $stripped = str_replace( "\r", "", $text );
721  $normalized = $wgContLang->normalize( $stripped );
722 
723  return $normalized;
724  }
725 
726  private function getTextSpawned( $id ) {
727  MediaWiki\suppressWarnings();
728  if ( !$this->spawnProc ) {
729  // First time?
730  $this->openSpawn();
731  }
732  $text = $this->getTextSpawnedOnce( $id );
733  MediaWiki\restoreWarnings();
734 
735  return $text;
736  }
737 
738  function openSpawn() {
739  global $IP;
740 
741  if ( file_exists( "$IP/../multiversion/MWScript.php" ) ) {
742  $cmd = implode( " ",
743  array_map( 'wfEscapeShellArg',
744  [
745  $this->php,
746  "$IP/../multiversion/MWScript.php",
747  "fetchText.php",
748  '--wiki', wfWikiID() ] ) );
749  } else {
750  $cmd = implode( " ",
751  array_map( 'wfEscapeShellArg',
752  [
753  $this->php,
754  "$IP/maintenance/fetchText.php",
755  '--wiki', wfWikiID() ] ) );
756  }
757  $spec = [
758  0 => [ "pipe", "r" ],
759  1 => [ "pipe", "w" ],
760  2 => [ "file", "/dev/null", "a" ] ];
761  $pipes = [];
762 
763  $this->progress( "Spawning database subprocess: $cmd" );
764  $this->spawnProc = proc_open( $cmd, $spec, $pipes );
765  if ( !$this->spawnProc ) {
766  $this->progress( "Subprocess spawn failed." );
767 
768  return false;
769  }
770  list(
771  $this->spawnWrite, // -> stdin
772  $this->spawnRead, // <- stdout
773  ) = $pipes;
774 
775  return true;
776  }
777 
778  private function closeSpawn() {
779  MediaWiki\suppressWarnings();
780  if ( $this->spawnRead ) {
781  fclose( $this->spawnRead );
782  }
783  $this->spawnRead = false;
784  if ( $this->spawnWrite ) {
785  fclose( $this->spawnWrite );
786  }
787  $this->spawnWrite = false;
788  if ( $this->spawnErr ) {
789  fclose( $this->spawnErr );
790  }
791  $this->spawnErr = false;
792  if ( $this->spawnProc ) {
793  pclose( $this->spawnProc );
794  }
795  $this->spawnProc = false;
796  MediaWiki\restoreWarnings();
797  }
798 
799  private function getTextSpawnedOnce( $id ) {
801 
802  $ok = fwrite( $this->spawnWrite, "$id\n" );
803  // $this->progress( ">> $id" );
804  if ( !$ok ) {
805  return false;
806  }
807 
808  $ok = fflush( $this->spawnWrite );
809  // $this->progress( ">> [flush]" );
810  if ( !$ok ) {
811  return false;
812  }
813 
814  // check that the text id they are sending is the one we asked for
815  // this avoids out of sync revision text errors we have encountered in the past
816  $newId = fgets( $this->spawnRead );
817  if ( $newId === false ) {
818  return false;
819  }
820  if ( $id != intval( $newId ) ) {
821  return false;
822  }
823 
824  $len = fgets( $this->spawnRead );
825  // $this->progress( "<< " . trim( $len ) );
826  if ( $len === false ) {
827  return false;
828  }
829 
830  $nbytes = intval( $len );
831  // actual error, not zero-length text
832  if ( $nbytes < 0 ) {
833  return false;
834  }
835 
836  $text = "";
837 
838  // Subprocess may not send everything at once, we have to loop.
839  while ( $nbytes > strlen( $text ) ) {
840  $buffer = fread( $this->spawnRead, $nbytes - strlen( $text ) );
841  if ( $buffer === false ) {
842  break;
843  }
844  $text .= $buffer;
845  }
846 
847  $gotbytes = strlen( $text );
848  if ( $gotbytes != $nbytes ) {
849  $this->progress( "Expected $nbytes bytes from database subprocess, got $gotbytes " );
850 
851  return false;
852  }
853 
854  // Do normalization in the dump thread...
855  $stripped = str_replace( "\r", "", $text );
856  $normalized = $wgContLang->normalize( $stripped );
857 
858  return $normalized;
859  }
860 
862  $this->checkpointJustWritten = false;
863 
864  $this->clearOpenElement( null );
865  $this->lastName = $name;
866 
867  if ( $name == 'revision' ) {
868  $this->state = $name;
869  $this->egress->writeOpenPage( null, $this->buffer );
870  $this->buffer = "";
871  } elseif ( $name == 'page' ) {
872  $this->state = $name;
873  if ( $this->atStart ) {
874  $this->egress->writeOpenStream( $this->buffer );
875  $this->buffer = "";
876  $this->atStart = false;
877  }
878  }
879 
880  if ( $name == "text" && isset( $attribs['id'] ) ) {
881  $id = $attribs['id'];
882  $model = trim( $this->thisRevModel );
883  $format = trim( $this->thisRevFormat );
884 
885  $model = $model === '' ? null : $model;
886  $format = $format === '' ? null : $format;
887 
888  $text = $this->getText( $id, $model, $format );
889  $this->openElement = [ $name, [ 'xml:space' => 'preserve' ] ];
890  if ( strlen( $text ) > 0 ) {
891  $this->characterData( $parser, $text );
892  }
893  } else {
894  $this->openElement = [ $name, $attribs ];
895  }
896  }
897 
898  function endElement( $parser, $name ) {
899  $this->checkpointJustWritten = false;
900 
901  if ( $this->openElement ) {
902  $this->clearOpenElement( "" );
903  } else {
904  $this->buffer .= "</$name>";
905  }
906 
907  if ( $name == 'revision' ) {
908  $this->egress->writeRevision( null, $this->buffer );
909  $this->buffer = "";
910  $this->thisRev = "";
911  $this->thisRevModel = null;
912  $this->thisRevFormat = null;
913  } elseif ( $name == 'page' ) {
914  if ( !$this->firstPageWritten ) {
915  $this->firstPageWritten = trim( $this->thisPage );
916  }
917  $this->lastPageWritten = trim( $this->thisPage );
918  if ( $this->timeExceeded ) {
919  $this->egress->writeClosePage( $this->buffer );
920  // nasty hack, we can't just write the chardata after the
921  // page tag, it will include leading blanks from the next line
922  $this->egress->sink->write( "\n" );
923 
924  $this->buffer = $this->xmlwriterobj->closeStream();
925  $this->egress->writeCloseStream( $this->buffer );
926 
927  $this->buffer = "";
928  $this->thisPage = "";
929  // this could be more than one file if we had more than one output arg
930 
931  $filenameList = (array)$this->egress->getFilenames();
932  $newFilenames = [];
933  $firstPageID = str_pad( $this->firstPageWritten, 9, "0", STR_PAD_LEFT );
934  $lastPageID = str_pad( $this->lastPageWritten, 9, "0", STR_PAD_LEFT );
935  $filenamesCount = count( $filenameList );
936  for ( $i = 0; $i < $filenamesCount; $i++ ) {
937  $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID );
938  $fileinfo = pathinfo( $filenameList[$i] );
939  $newFilenames[] = $fileinfo['dirname'] . '/' . $checkpointNameFilledIn;
940  }
941  $this->egress->closeRenameAndReopen( $newFilenames );
942  $this->buffer = $this->xmlwriterobj->openStream();
943  $this->timeExceeded = false;
944  $this->timeOfCheckpoint = $this->lastTime;
945  $this->firstPageWritten = false;
946  $this->checkpointJustWritten = true;
947  } else {
948  $this->egress->writeClosePage( $this->buffer );
949  $this->buffer = "";
950  $this->thisPage = "";
951  }
952  } elseif ( $name == 'mediawiki' ) {
953  $this->egress->writeCloseStream( $this->buffer );
954  $this->buffer = "";
955  }
956  }
957 
958  function characterData( $parser, $data ) {
959  $this->clearOpenElement( null );
960  if ( $this->lastName == "id" ) {
961  if ( $this->state == "revision" ) {
962  $this->thisRev .= $data;
963  } elseif ( $this->state == "page" ) {
964  $this->thisPage .= $data;
965  }
966  } elseif ( $this->lastName == "model" ) {
967  $this->thisRevModel .= $data;
968  } elseif ( $this->lastName == "format" ) {
969  $this->thisRevFormat .= $data;
970  }
971 
972  // have to skip the newline left over from closepagetag line of
973  // end of checkpoint files. nasty hack!!
974  if ( $this->checkpointJustWritten ) {
975  if ( $data[0] == "\n" ) {
976  $data = substr( $data, 1 );
977  }
978  $this->checkpointJustWritten = false;
979  }
980  $this->buffer .= htmlspecialchars( $data );
981  }
982 
983  function clearOpenElement( $style ) {
984  if ( $this->openElement ) {
985  $this->buffer .= Xml::element( $this->openElement[0], $this->openElement[1], $style );
986  $this->openElement = false;
987  }
988  }
989 }
990 
991 $maintClass = 'TextPassDumper';
992 require_once RUN_MAINTENANCE_IF_MAIN;
bool resource $spawnWrite
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition: deferred.txt:11
IMaintainableDatabase $db
the array() calling protocol came about after MediaWiki 1.4rc1.
static getRevisionText($row, $prefix= 'old_', $wiki=false)
Get revision text associated with an old or archive row.
Definition: Revision.php:1282
const CONTENT_MODEL_WIKITEXT
Definition: Defines.php:236
static element($element, $attribs=null, $contents= '', $allowShortTag=true)
Format an XML element with given attributes and, optionally, text content.
Definition: Xml.php:39
$IP
Definition: WebStart.php:57
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
static getForModelID($modelId)
Returns the ContentHandler singleton for the given model ID.
div flags Integer display flags(NO_ACTION_LINK, NO_EXTRA_USER_LINKS) 'LogException'returning false will NOT prevent logging $e
Definition: hooks.txt:2142
We use the convention $dbr for read and $dbw for write to help you keep track of whether the database object is a the world will explode Or to be a subsequent write query which succeeded on the master may fail when replicated to the slave due to a unique key collision Replication on the slave will stop and it may take hours to repair the database and get it back online Setting read_only in my cnf on the slave will avoid this but given the dire we prefer to have as many checks as possible We provide a but the wrapper functions like please read the documentation for except in special pages derived from QueryPage It s a common pitfall for new developers to submit code containing SQL queries which examine huge numbers of rows Remember that COUNT * is(N), counting rows in atable is like counting beans in a bucket.------------------------------------------------------------------------Replication------------------------------------------------------------------------The largest installation of MediaWiki, Wikimedia, uses a large set ofslave MySQL servers replicating writes made to a master MySQL server.Itis important to understand the issues associated with this setup if youwant to write code destined for Wikipedia.It's often the case that the best algorithm to use for a given taskdepends on whether or not replication is in use.Due to our unabashedWikipedia-centrism, we often just use the replication-friendly version, but if you like, you can use wfGetLB() ->getServerCount() > 1 tocheck to see if replication is in use.===Lag===Lag primarily occurs when large write queries are sent to the master.Writes on the master are executed in parallel, but they are executed inserial when they are replicated to the slaves.The master writes thequery to the binlog when the transaction is committed.The slaves pollthe binlog and start executing the query as soon as it appears.They canservice reads while they are performing a write query, but will not readanything more from the binlog and thus will perform no more writes.Thismeans that if the write query runs for a long time, the slaves will lagbehind the master for the time it takes for the write query to complete.Lag can be exacerbated by high read load.MediaWiki's load balancer willstop sending reads to a slave when it is lagged by more than 30 seconds.If the load ratios are set incorrectly, or if there is too much loadgenerally, this may lead to a slave permanently hovering around 30seconds lag.If all slaves are lagged by more than 30 seconds, MediaWiki will stopwriting to the database.All edits and other write operations will berefused, with an error returned to the user.This gives the slaves achance to catch up.Before we had this mechanism, the slaves wouldregularly lag by several minutes, making review of recent editsdifficult.In addition to this, MediaWiki attempts to ensure that the user seesevents occurring on the wiki in chronological order.A few seconds of lagcan be tolerated, as long as the user sees a consistent picture fromsubsequent requests.This is done by saving the master binlog positionin the session, and then at the start of each request, waiting for theslave to catch up to that position before doing any reads from it.Ifthis wait times out, reads are allowed anyway, but the request isconsidered to be in"lagged slave mode".Lagged slave mode can bechecked by calling wfGetLB() ->getLaggedSlaveMode().The onlypractical consequence at present is a warning displayed in the pagefooter.===Lag avoidance===To avoid excessive lag, queries which write large numbers of rows shouldbe split up, generally to write one row at a time.Multi-row INSERT...SELECT queries are the worst offenders should be avoided altogether.Instead do the select first and then the insert.===Working with lag===Despite our best efforts, it's not practical to guarantee a low-lagenvironment.Lag will usually be less than one second, but mayoccasionally be up to 30 seconds.For scalability, it's very importantto keep load on the master low, so simply sending all your queries tothe master is not the answer.So when you have a genuine need forup-to-date data, the following approach is advised:1) Do a quick query to the master for a sequence number or timestamp 2) Run the full query on the slave and check if it matches the data you gotfrom the master 3) If it doesn't, run the full query on the masterTo avoid swamping the master every time the slaves lag, use of thisapproach should be kept to a minimum.In most cases you should just readfrom the slave and let the user deal with the delay.------------------------------------------------------------------------Lock contention------------------------------------------------------------------------Due to the high write rate on Wikipedia(and some other wikis), MediaWiki developers need to be very careful to structure their writesto avoid long-lasting locks.By default, MediaWiki opens a transactionat the first query, and commits it before the output is sent.Locks willbe held from the time when the query is done until the commit.So youcan reduce lock time by doing as much processing as possible before youdo your write queries.Often this approach is not good enough, and it becomes necessary toenclose small groups of queries in their own transaction.Use thefollowing syntax:$dbw=wfGetDB(DB_MASTER
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for and distribution as defined by Sections through of this document Licensor shall mean the copyright owner or entity authorized by the copyright owner that is granting the License Legal Entity shall mean the union of the acting entity and all other entities that control are controlled by or are under common control with that entity For the purposes of this definition control direct or to cause the direction or management of such whether by contract or including but not limited to software source documentation and configuration files Object form shall mean any form resulting from mechanical transformation or translation of a Source including but not limited to compiled object generated and conversions to other media types Work shall mean the work of whether in Source or Object made available under the as indicated by a copyright notice that is included in or attached to the whether in Source or Object that is based or other modifications as a an original work of authorship For the purposes of this Derivative Works shall not include works that remain separable from
string bool $thisRev
getTextDb($id)
May throw a database error if, say, the server dies during query.
hasOption($name)
Checks to see if a particular param exists.
require_once RUN_MAINTENANCE_IF_MAIN
Definition: maintenance.txt:50
showReport()
Overridden to include prefetch ratio if enabled.
target page
when a variable name is used in a it is silently declared as a new local masking the global
Definition: design.txt:93
bool XmlDumpWriter $xmlwriterobj
report($final=false)
Definition: backup.inc:372
magic word & $parser
Definition: hooks.txt:2556
exportTransform($text, $model, $format=null)
Applies applicable export transformations to $text.
wfDebug($text, $dest= 'all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
if($line===false) $args
Definition: cdb.php:63
The ContentHandler facility adds support for arbitrary content types on wiki instead of relying on wikitext for everything It was introduced in MediaWiki Each kind of and so on Built in content types are
An extension or a local will often add custom code to the function with or without a global variable For someone wanting email notification when an article is shown may add
Definition: hooks.txt:51
$maintClass
wfTimestamp($outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
addOption($name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
either a unescaped string or a HtmlArmor object after in associative array form externallinks including delete and has completed for all link tables whether this was an auto creation default is conds Array Extra conditions for the No matching items in log is displayed if loglist is empty msgKey Array If you want a nice box with a set this to the key of the message First element is the message additional optional elements are parameters for the key that are processed with wfMessage() -> params() ->parseAsBlock()-offset Set to overwrite offset parameter in $wgRequest set to ''to unsetoffset-wrap String Wrap the message in html(usually something like"&lt
dump($history, $text=WikiExporter::TEXT)
Readahead helper for making large MediaWiki data dumps; reads in a previous XML dump to sequentially ...
bool resource $spawnProc
$maxConsecutiveFailedTextRetrievals
endElement($parser, $name)
null means default in associative array with keys and values unescaped Should be merged with default with a value of false meaning to suppress the attribute in associative array with keys and values unescaped noclasses just before the function returns a value If you return an< a > element with HTML attributes $attribs and contents $html will be returned If you return $ret will be returned and may include noclasses after processing & $attribs
Definition: hooks.txt:1966
$buffer
design txt This is a brief overview of the new design More thorough and up to date information is available on the documentation wiki at etc Handles the details of getting and saving to the user table of the and dealing with sessions and cookies OutputPage Encapsulates the entire HTML page that will be sent in response to any server request It is used by calling its functions to add text
Definition: design.txt:12
wfWikiID()
Get an ASCII string identifying this wiki This is used as a prefix in memcached keys.
addDescription($text)
Set the description text.
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
getOption($name, $default=null)
Get an option, or return the default.
output($out, $channel=null)
Throw some output to the user.
clearOpenElement($style)
loadWithArgv($argv)
Load params and arguments from a given array of command-line arguments.
string bool $thisPage
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
presenting them properly to the user as errors is done by the caller return true use this to change the list i e etc next in line in page history
Definition: hooks.txt:1751
wfGetLBFactory()
Get the load balancer factory object.
IDatabase null $forcedDb
The dependency-injected database to use.
Definition: backup.inc:70
startElement($parser, $name, $attribs)
bool resource $spawnRead
this class mediates it Skin Encapsulates a look and feel for the wiki All of the functions that render HTML and make choices about how to render it are here and are called from various other places when and is meant to be subclassed with other skins that may override some of its functions The User object contains a reference to a and so rather than having a global skin object we just rely on the global User and get the skin with $wgUser and also has some character encoding functions and other locale stuff The current user interface language is instantiated as and the local content language as $wgContLang
Definition: design.txt:56
Allows to change the fields on the form that will be generated $name
Definition: hooks.txt:302
We ve cleaned up the code here by removing clumps of infrequently used code and moving them off somewhere else It s much easier for someone working with this code to see what s _really_ going on
Definition: hooks.txt:84
const DB_REPLICA
Definition: defines.php:25
progress($string)
Definition: backup.inc:417
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that probably a stub it is not rendered in wiki pages or galleries in category pages allow injecting custom HTML after the section Any uses of the hook need to handle escaping see BaseTemplate::getToolbox and BaseTemplate::makeListItem for details on the format of individual items inside of this array or by returning and letting standard HTTP rendering take place modifiable or by returning false and taking over the output modifiable modifiable after all normalizations have been except for the $wgMaxImageArea check set to true or false to override the $wgMaxImageArea check result gives extension the possibility to transform it themselves $handler
Definition: hooks.txt:782
BaseDump $prefetch
initProgress($history=WikiExporter::FULL)
bool resource $spawnErr
characterData($parser, $data)
__construct($args=null)
rotateDb()
Drop the database connection $this->db and try to get a new one.
getText($id, $model=null, $format=null)
Tries to get the revision text for a revision id.