MediaWiki  master
dumpTextPass.php
Go to the documentation of this file.
1 <?php
27 require_once __DIR__ . '/backup.inc';
28 require_once __DIR__ . '/7zip.inc';
29 require_once __DIR__ . '/../includes/export/WikiExporter.php';
30 
32 
38  public $prefetch = null;
40  private $thisPage;
42  private $thisRev;
43 
44  // when we spend more than maxTimeAllowed seconds on this run, we continue
45  // processing until we write out the next complete page, then save output file(s),
46  // rename it/them and open new one(s)
47  public $maxTimeAllowed = 0; // 0 = no limit
48 
49  protected $input = "php://stdin";
51  protected $fetchCount = 0;
52  protected $prefetchCount = 0;
53  protected $prefetchCountLast = 0;
54  protected $fetchCountLast = 0;
55 
56  protected $maxFailures = 5;
58  protected $failureTimeout = 5; // Seconds to sleep after db failure
59 
60  protected $bufferSize = 524288; // In bytes. Maximum size to read from the stub in on go.
61 
62  protected $php = "php";
63  protected $spawn = false;
64 
68  protected $spawnProc = false;
69 
73  protected $spawnWrite = false;
74 
78  protected $spawnRead = false;
79 
83  protected $spawnErr = false;
84 
88  protected $xmlwriterobj = false;
89 
90  protected $timeExceeded = false;
91  protected $firstPageWritten = false;
92  protected $lastPageWritten = false;
93  protected $checkpointJustWritten = false;
94  protected $checkpointFiles = [];
95 
99  protected $db;
100 
104  function __construct( $args = null ) {
105  parent::__construct();
106 
107  $this->addDescription( <<<TEXT
108 This script postprocesses XML dumps from dumpBackup.php to add
109 page text which was stubbed out (using --stub).
110 
111 XML input is accepted on stdin.
112 XML output is sent to stdout; progress reports are sent to stderr.
113 TEXT
114  );
115  $this->stderr = fopen( "php://stderr", "wt" );
116 
117  $this->addOption( 'stub', 'To load a compressed stub dump instead of stdin. ' .
118  'Specify as --stub=<type>:<file>.', false, true );
119  $this->addOption( 'prefetch', 'Use a prior dump file as a text source, to savepressure on the ' .
120  'database. (Requires the XMLReader extension). Specify as --prefetch=<type>:<file>',
121  false, true );
122  $this->addOption( 'maxtime', 'Write out checkpoint file after this many minutes (writing' .
123  'out complete page, closing xml file properly, and opening new one' .
124  'with header). This option requires the checkpointfile option.', false, true );
125  $this->addOption( 'checkpointfile', 'Use this string for checkpoint filenames,substituting ' .
126  'first pageid written for the first %s (required) and the last pageid written for the ' .
127  'second %s if it exists.', false, true, false, true ); // This can be specified multiple times
128  $this->addOption( 'quiet', 'Don\'t dump status reports to stderr.' );
129  $this->addOption( 'current', 'Base ETA on number of pages in database instead of all revisions' );
130  $this->addOption( 'spawn', 'Spawn a subprocess for loading text records' );
131  $this->addOption( 'buffersize', 'Buffer size in bytes to use for reading the stub. ' .
132  '(Default: 512KB, Minimum: 4KB)', false, true );
133 
134  if ( $args ) {
135  $this->loadWithArgv( $args );
136  $this->processOptions();
137  }
138  }
139 
140  function execute() {
141  $this->processOptions();
142  $this->dump( true );
143  }
144 
145  function processOptions() {
146  parent::processOptions();
147 
148  if ( $this->hasOption( 'buffersize' ) ) {
149  $this->bufferSize = max( intval( $this->getOption( 'buffersize' ) ), 4 * 1024 );
150  }
151 
152  if ( $this->hasOption( 'prefetch' ) ) {
153  $url = $this->processFileOpt( $this->getOption( 'prefetch' ) );
154  $this->prefetch = new BaseDump( $url );
155  }
156 
157  if ( $this->hasOption( 'stub' ) ) {
158  $this->input = $this->processFileOpt( $this->getOption( 'stub' ) );
159  }
160 
161  if ( $this->hasOption( 'maxtime' ) ) {
162  $this->maxTimeAllowed = intval( $this->getOption( 'maxtime' ) ) * 60;
163  }
164 
165  if ( $this->hasOption( 'checkpointfile' ) ) {
166  $this->checkpointFiles = $this->getOption( 'checkpointfile' );
167  }
168 
169  if ( $this->hasOption( 'current' ) ) {
171  }
172 
173  if ( $this->hasOption( 'full' ) ) {
174  $this->history = WikiExporter::FULL;
175  }
176 
177  if ( $this->hasOption( 'spawn' ) ) {
178  $this->spawn = true;
179  $val = $this->getOption( 'spawn' );
180  if ( $val !== 1 ) {
181  $this->php = $val;
182  }
183  }
184  }
185 
197  function rotateDb() {
198  // Cleaning up old connections
199  if ( isset( $this->lb ) ) {
200  $this->lb->closeAll();
201  unset( $this->lb );
202  }
203 
204  if ( $this->forcedDb !== null ) {
205  $this->db = $this->forcedDb;
206 
207  return;
208  }
209 
210  if ( isset( $this->db ) && $this->db->isOpen() ) {
211  throw new MWException( 'DB is set and has not been closed by the Load Balancer' );
212  }
213 
214  unset( $this->db );
215 
216  // Trying to set up new connection.
217  // We do /not/ retry upon failure, but delegate to encapsulating logic, to avoid
218  // individually retrying at different layers of code.
219 
220  try {
221  $this->lb = wfGetLBFactory()->newMainLB();
222  } catch ( Exception $e ) {
223  throw new MWException( __METHOD__
224  . " rotating DB failed to obtain new load balancer (" . $e->getMessage() . ")" );
225  }
226 
227  try {
228  $this->db = $this->lb->getConnection( DB_REPLICA, 'dump' );
229  } catch ( Exception $e ) {
230  throw new MWException( __METHOD__
231  . " rotating DB failed to obtain new database (" . $e->getMessage() . ")" );
232  }
233  }
234 
236  parent::initProgress();
237  $this->timeOfCheckpoint = $this->startTime;
238  }
239 
240  function dump( $history, $text = WikiExporter::TEXT ) {
241  // Notice messages will foul up your XML output even if they're
242  // relatively harmless.
243  if ( ini_get( 'display_errors' ) ) {
244  ini_set( 'display_errors', 'stderr' );
245  }
246 
247  $this->initProgress( $this->history );
248 
249  // We are trying to get an initial database connection to avoid that the
250  // first try of this request's first call to getText fails. However, if
251  // obtaining a good DB connection fails it's not a serious issue, as
252  // getText does retry upon failure and can start without having a working
253  // DB connection.
254  try {
255  $this->rotateDb();
256  } catch ( Exception $e ) {
257  // We do not even count this as failure. Just let eventual
258  // watchdogs know.
259  $this->progress( "Getting initial DB connection failed (" .
260  $e->getMessage() . ")" );
261  }
262 
263  $this->egress = new ExportProgressFilter( $this->sink, $this );
264 
265  // it would be nice to do it in the constructor, oh well. need egress set
266  $this->finalOptionCheck();
267 
268  // we only want this so we know how to close a stream :-P
269  $this->xmlwriterobj = new XmlDumpWriter();
270 
271  $input = fopen( $this->input, "rt" );
272  $this->readDump( $input );
273 
274  if ( $this->spawnProc ) {
275  $this->closeSpawn();
276  }
277 
278  $this->report( true );
279  }
280 
281  function processFileOpt( $opt ) {
282  $split = explode( ':', $opt, 2 );
283  $val = $split[0];
284  $param = '';
285  if ( count( $split ) === 2 ) {
286  $param = $split[1];
287  }
288  $fileURIs = explode( ';', $param );
289  foreach ( $fileURIs as $URI ) {
290  switch ( $val ) {
291  case "file":
292  $newURI = $URI;
293  break;
294  case "gzip":
295  $newURI = "compress.zlib://$URI";
296  break;
297  case "bzip2":
298  $newURI = "compress.bzip2://$URI";
299  break;
300  case "7zip":
301  $newURI = "mediawiki.compress.7z://$URI";
302  break;
303  default:
304  $newURI = $URI;
305  }
306  $newFileURIs[] = $newURI;
307  }
308  $val = implode( ';', $newFileURIs );
309 
310  return $val;
311  }
312 
316  function showReport() {
317  if ( !$this->prefetch ) {
318  parent::showReport();
319 
320  return;
321  }
322 
323  if ( $this->reporting ) {
324  $now = wfTimestamp( TS_DB );
325  $nowts = microtime( true );
326  $deltaAll = $nowts - $this->startTime;
327  $deltaPart = $nowts - $this->lastTime;
328  $this->pageCountPart = $this->pageCount - $this->pageCountLast;
329  $this->revCountPart = $this->revCount - $this->revCountLast;
330 
331  if ( $deltaAll ) {
332  $portion = $this->revCount / $this->maxCount;
333  $eta = $this->startTime + $deltaAll / $portion;
334  $etats = wfTimestamp( TS_DB, intval( $eta ) );
335  if ( $this->fetchCount ) {
336  $fetchRate = 100.0 * $this->prefetchCount / $this->fetchCount;
337  } else {
338  $fetchRate = '-';
339  }
340  $pageRate = $this->pageCount / $deltaAll;
341  $revRate = $this->revCount / $deltaAll;
342  } else {
343  $pageRate = '-';
344  $revRate = '-';
345  $etats = '-';
346  $fetchRate = '-';
347  }
348  if ( $deltaPart ) {
349  if ( $this->fetchCountLast ) {
350  $fetchRatePart = 100.0 * $this->prefetchCountLast / $this->fetchCountLast;
351  } else {
352  $fetchRatePart = '-';
353  }
354  $pageRatePart = $this->pageCountPart / $deltaPart;
355  $revRatePart = $this->revCountPart / $deltaPart;
356  } else {
357  $fetchRatePart = '-';
358  $pageRatePart = '-';
359  $revRatePart = '-';
360  }
361  $this->progress( sprintf(
362  "%s: %s (ID %d) %d pages (%0.1f|%0.1f/sec all|curr), "
363  . "%d revs (%0.1f|%0.1f/sec all|curr), %0.1f%%|%0.1f%% "
364  . "prefetched (all|curr), ETA %s [max %d]",
365  $now, wfWikiID(), $this->ID, $this->pageCount, $pageRate,
366  $pageRatePart, $this->revCount, $revRate, $revRatePart,
367  $fetchRate, $fetchRatePart, $etats, $this->maxCount
368  ) );
369  $this->lastTime = $nowts;
370  $this->revCountLast = $this->revCount;
371  $this->prefetchCountLast = $this->prefetchCount;
372  $this->fetchCountLast = $this->fetchCount;
373  }
374  }
375 
376  function setTimeExceeded() {
377  $this->timeExceeded = true;
378  }
379 
380  function checkIfTimeExceeded() {
381  if ( $this->maxTimeAllowed
382  && ( $this->lastTime - $this->timeOfCheckpoint > $this->maxTimeAllowed )
383  ) {
384  return true;
385  }
386 
387  return false;
388  }
389 
390  function finalOptionCheck() {
391  if ( ( $this->checkpointFiles && !$this->maxTimeAllowed )
392  || ( $this->maxTimeAllowed && !$this->checkpointFiles )
393  ) {
394  throw new MWException( "Options checkpointfile and maxtime must be specified together.\n" );
395  }
396  foreach ( $this->checkpointFiles as $checkpointFile ) {
397  $count = substr_count( $checkpointFile, "%s" );
398  if ( $count != 2 ) {
399  throw new MWException( "Option checkpointfile must contain two '%s' "
400  . "for substitution of first and last pageids, count is $count instead, "
401  . "file is $checkpointFile.\n" );
402  }
403  }
404 
405  if ( $this->checkpointFiles ) {
406  $filenameList = (array)$this->egress->getFilenames();
407  if ( count( $filenameList ) != count( $this->checkpointFiles ) ) {
408  throw new MWException( "One checkpointfile must be specified "
409  . "for each output option, if maxtime is used.\n" );
410  }
411  }
412  }
413 
419  function readDump( $input ) {
420  $this->buffer = "";
421  $this->openElement = false;
422  $this->atStart = true;
423  $this->state = "";
424  $this->lastName = "";
425  $this->thisPage = 0;
426  $this->thisRev = 0;
427  $this->thisRevModel = null;
428  $this->thisRevFormat = null;
429 
430  $parser = xml_parser_create( "UTF-8" );
431  xml_parser_set_option( $parser, XML_OPTION_CASE_FOLDING, false );
432 
433  xml_set_element_handler(
434  $parser,
435  [ $this, 'startElement' ],
436  [ $this, 'endElement' ]
437  );
438  xml_set_character_data_handler( $parser, [ $this, 'characterData' ] );
439 
440  $offset = 0; // for context extraction on error reporting
441  do {
442  if ( $this->checkIfTimeExceeded() ) {
443  $this->setTimeExceeded();
444  }
445  $chunk = fread( $input, $this->bufferSize );
446  if ( !xml_parse( $parser, $chunk, feof( $input ) ) ) {
447  wfDebug( "TextDumpPass::readDump encountered XML parsing error\n" );
448 
449  $byte = xml_get_current_byte_index( $parser );
450  $msg = wfMessage( 'xml-error-string',
451  'XML import parse failure',
452  xml_get_current_line_number( $parser ),
453  xml_get_current_column_number( $parser ),
454  $byte . ( is_null( $chunk ) ? null : ( '; "' . substr( $chunk, $byte - $offset, 16 ) . '"' ) ),
455  xml_error_string( xml_get_error_code( $parser ) ) )->escaped();
456 
457  xml_parser_free( $parser );
458 
459  throw new MWException( $msg );
460  }
461  $offset += strlen( $chunk );
462  } while ( $chunk !== false && !feof( $input ) );
463  if ( $this->maxTimeAllowed ) {
464  $filenameList = (array)$this->egress->getFilenames();
465  // we wrote some stuff after last checkpoint that needs renamed
466  if ( file_exists( $filenameList[0] ) ) {
467  $newFilenames = [];
468  # we might have just written the header and footer and had no
469  # pages or revisions written... perhaps they were all deleted
470  # there's no pageID 0 so we use that. the caller is responsible
471  # for deciding what to do with a file containing only the
472  # siteinfo information and the mw tags.
473  if ( !$this->firstPageWritten ) {
474  $firstPageID = str_pad( 0, 9, "0", STR_PAD_LEFT );
475  $lastPageID = str_pad( 0, 9, "0", STR_PAD_LEFT );
476  } else {
477  $firstPageID = str_pad( $this->firstPageWritten, 9, "0", STR_PAD_LEFT );
478  $lastPageID = str_pad( $this->lastPageWritten, 9, "0", STR_PAD_LEFT );
479  }
480 
481  $filenameCount = count( $filenameList );
482  for ( $i = 0; $i < $filenameCount; $i++ ) {
483  $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID );
484  $fileinfo = pathinfo( $filenameList[$i] );
485  $newFilenames[] = $fileinfo['dirname'] . '/' . $checkpointNameFilledIn;
486  }
487  $this->egress->closeAndRename( $newFilenames );
488  }
489  }
490  xml_parser_free( $parser );
491 
492  return true;
493  }
494 
504  private function exportTransform( $text, $model, $format = null ) {
505  try {
507  $text = $handler->exportTransform( $text, $format );
508  }
509  catch ( MWException $ex ) {
510  $this->progress(
511  "Unable to apply export transformation for content model '$model': " .
512  $ex->getMessage()
513  );
514  }
515 
516  return $text;
517  }
518 
539  function getText( $id, $model = null, $format = null ) {
540  global $wgContentHandlerUseDB;
541 
542  $prefetchNotTried = true; // Whether or not we already tried to get the text via prefetch.
543  $text = false; // The candidate for a good text. false if no proper value.
544  $failures = 0; // The number of times, this invocation of getText already failed.
545 
546  // The number of times getText failed without yielding a good text in between.
547  static $consecutiveFailedTextRetrievals = 0;
548 
549  $this->fetchCount++;
550 
551  // To allow to simply return on success and do not have to worry about book keeping,
552  // we assume, this fetch works (possible after some retries). Nevertheless, we koop
553  // the old value, so we can restore it, if problems occur (See after the while loop).
554  $oldConsecutiveFailedTextRetrievals = $consecutiveFailedTextRetrievals;
555  $consecutiveFailedTextRetrievals = 0;
556 
557  if ( $model === null && $wgContentHandlerUseDB ) {
558  $row = $this->db->selectRow(
559  'revision',
560  [ 'rev_content_model', 'rev_content_format' ],
561  [ 'rev_id' => $this->thisRev ],
562  __METHOD__
563  );
564 
565  if ( $row ) {
566  $model = $row->rev_content_model;
567  $format = $row->rev_content_format;
568  }
569  }
570 
571  if ( $model === null || $model === '' ) {
572  $model = false;
573  }
574 
575  while ( $failures < $this->maxFailures ) {
576  // As soon as we found a good text for the $id, we will return immediately.
577  // Hence, if we make it past the try catch block, we know that we did not
578  // find a good text.
579 
580  try {
581  // Step 1: Get some text (or reuse from previous iteratuon if checking
582  // for plausibility failed)
583 
584  // Trying to get prefetch, if it has not been tried before
585  if ( $text === false && isset( $this->prefetch ) && $prefetchNotTried ) {
586  $prefetchNotTried = false;
587  $tryIsPrefetch = true;
588  $text = $this->prefetch->prefetch( (int)$this->thisPage, (int)$this->thisRev );
589 
590  if ( $text === null ) {
591  $text = false;
592  }
593 
594  if ( is_string( $text ) && $model !== false ) {
595  // Apply export transformation to text coming from an old dump.
596  // The purpose of this transformation is to convert up from legacy
597  // formats, which may still be used in the older dump that is used
598  // for pre-fetching. Applying the transformation again should not
599  // interfere with content that is already in the correct form.
600  $text = $this->exportTransform( $text, $model, $format );
601  }
602  }
603 
604  if ( $text === false ) {
605  // Fallback to asking the database
606  $tryIsPrefetch = false;
607  if ( $this->spawn ) {
608  $text = $this->getTextSpawned( $id );
609  } else {
610  $text = $this->getTextDb( $id );
611  }
612 
613  if ( $text !== false && $model !== false ) {
614  // Apply export transformation to text coming from the database.
615  // Prefetched text should already have transformations applied.
616  $text = $this->exportTransform( $text, $model, $format );
617  }
618 
619  // No more checks for texts from DB for now.
620  // If we received something that is not false,
621  // We treat it as good text, regardless of whether it actually is or is not
622  if ( $text !== false ) {
623  return $text;
624  }
625  }
626 
627  if ( $text === false ) {
628  throw new MWException( "Generic error while obtaining text for id " . $id );
629  }
630 
631  // We received a good candidate for the text of $id via some method
632 
633  // Step 2: Checking for plausibility and return the text if it is
634  // plausible
635  $revID = intval( $this->thisRev );
636  if ( !isset( $this->db ) ) {
637  throw new MWException( "No database available" );
638  }
639 
640  if ( $model !== CONTENT_MODEL_WIKITEXT ) {
641  $revLength = strlen( $text );
642  } else {
643  $revLength = $this->db->selectField( 'revision', 'rev_len', [ 'rev_id' => $revID ] );
644  }
645 
646  if ( strlen( $text ) == $revLength ) {
647  if ( $tryIsPrefetch ) {
648  $this->prefetchCount++;
649  }
650 
651  return $text;
652  }
653 
654  $text = false;
655  throw new MWException( "Received text is unplausible for id " . $id );
656  } catch ( Exception $e ) {
657  $msg = "getting/checking text " . $id . " failed (" . $e->getMessage() . ")";
658  if ( $failures + 1 < $this->maxFailures ) {
659  $msg .= " (Will retry " . ( $this->maxFailures - $failures - 1 ) . " more times)";
660  }
661  $this->progress( $msg );
662  }
663 
664  // Something went wrong; we did not a text that was plausible :(
665  $failures++;
666 
667  // A failure in a prefetch hit does not warrant resetting db connection etc.
668  if ( !$tryIsPrefetch ) {
669  // After backing off for some time, we try to reboot the whole process as
670  // much as possible to not carry over failures from one part to the other
671  // parts
672  sleep( $this->failureTimeout );
673  try {
674  $this->rotateDb();
675  if ( $this->spawn ) {
676  $this->closeSpawn();
677  $this->openSpawn();
678  }
679  } catch ( Exception $e ) {
680  $this->progress( "Rebooting getText infrastructure failed (" . $e->getMessage() . ")" .
681  " Trying to continue anyways" );
682  }
683  }
684  }
685 
686  // Retirieving a good text for $id failed (at least) maxFailures times.
687  // We abort for this $id.
688 
689  // Restoring the consecutive failures, and maybe aborting, if the dump
690  // is too broken.
691  $consecutiveFailedTextRetrievals = $oldConsecutiveFailedTextRetrievals + 1;
692  if ( $consecutiveFailedTextRetrievals > $this->maxConsecutiveFailedTextRetrievals ) {
693  throw new MWException( "Graceful storage failure" );
694  }
695 
696  return "";
697  }
698 
705  private function getTextDb( $id ) {
707  if ( !isset( $this->db ) ) {
708  throw new MWException( __METHOD__ . "No database available" );
709  }
710  $row = $this->db->selectRow( 'text',
711  [ 'old_text', 'old_flags' ],
712  [ 'old_id' => $id ],
713  __METHOD__ );
714  $text = Revision::getRevisionText( $row );
715  if ( $text === false ) {
716  return false;
717  }
718  $stripped = str_replace( "\r", "", $text );
719  $normalized = $wgContLang->normalize( $stripped );
720 
721  return $normalized;
722  }
723 
724  private function getTextSpawned( $id ) {
725  MediaWiki\suppressWarnings();
726  if ( !$this->spawnProc ) {
727  // First time?
728  $this->openSpawn();
729  }
730  $text = $this->getTextSpawnedOnce( $id );
731  MediaWiki\restoreWarnings();
732 
733  return $text;
734  }
735 
736  function openSpawn() {
737  global $IP;
738 
739  if ( file_exists( "$IP/../multiversion/MWScript.php" ) ) {
740  $cmd = implode( " ",
741  array_map( 'wfEscapeShellArg',
742  [
743  $this->php,
744  "$IP/../multiversion/MWScript.php",
745  "fetchText.php",
746  '--wiki', wfWikiID() ] ) );
747  } else {
748  $cmd = implode( " ",
749  array_map( 'wfEscapeShellArg',
750  [
751  $this->php,
752  "$IP/maintenance/fetchText.php",
753  '--wiki', wfWikiID() ] ) );
754  }
755  $spec = [
756  0 => [ "pipe", "r" ],
757  1 => [ "pipe", "w" ],
758  2 => [ "file", "/dev/null", "a" ] ];
759  $pipes = [];
760 
761  $this->progress( "Spawning database subprocess: $cmd" );
762  $this->spawnProc = proc_open( $cmd, $spec, $pipes );
763  if ( !$this->spawnProc ) {
764  $this->progress( "Subprocess spawn failed." );
765 
766  return false;
767  }
768  list(
769  $this->spawnWrite, // -> stdin
770  $this->spawnRead, // <- stdout
771  ) = $pipes;
772 
773  return true;
774  }
775 
776  private function closeSpawn() {
777  MediaWiki\suppressWarnings();
778  if ( $this->spawnRead ) {
779  fclose( $this->spawnRead );
780  }
781  $this->spawnRead = false;
782  if ( $this->spawnWrite ) {
783  fclose( $this->spawnWrite );
784  }
785  $this->spawnWrite = false;
786  if ( $this->spawnErr ) {
787  fclose( $this->spawnErr );
788  }
789  $this->spawnErr = false;
790  if ( $this->spawnProc ) {
791  pclose( $this->spawnProc );
792  }
793  $this->spawnProc = false;
794  MediaWiki\restoreWarnings();
795  }
796 
797  private function getTextSpawnedOnce( $id ) {
799 
800  $ok = fwrite( $this->spawnWrite, "$id\n" );
801  // $this->progress( ">> $id" );
802  if ( !$ok ) {
803  return false;
804  }
805 
806  $ok = fflush( $this->spawnWrite );
807  // $this->progress( ">> [flush]" );
808  if ( !$ok ) {
809  return false;
810  }
811 
812  // check that the text id they are sending is the one we asked for
813  // this avoids out of sync revision text errors we have encountered in the past
814  $newId = fgets( $this->spawnRead );
815  if ( $newId === false ) {
816  return false;
817  }
818  if ( $id != intval( $newId ) ) {
819  return false;
820  }
821 
822  $len = fgets( $this->spawnRead );
823  // $this->progress( "<< " . trim( $len ) );
824  if ( $len === false ) {
825  return false;
826  }
827 
828  $nbytes = intval( $len );
829  // actual error, not zero-length text
830  if ( $nbytes < 0 ) {
831  return false;
832  }
833 
834  $text = "";
835 
836  // Subprocess may not send everything at once, we have to loop.
837  while ( $nbytes > strlen( $text ) ) {
838  $buffer = fread( $this->spawnRead, $nbytes - strlen( $text ) );
839  if ( $buffer === false ) {
840  break;
841  }
842  $text .= $buffer;
843  }
844 
845  $gotbytes = strlen( $text );
846  if ( $gotbytes != $nbytes ) {
847  $this->progress( "Expected $nbytes bytes from database subprocess, got $gotbytes " );
848 
849  return false;
850  }
851 
852  // Do normalization in the dump thread...
853  $stripped = str_replace( "\r", "", $text );
854  $normalized = $wgContLang->normalize( $stripped );
855 
856  return $normalized;
857  }
858 
860  $this->checkpointJustWritten = false;
861 
862  $this->clearOpenElement( null );
863  $this->lastName = $name;
864 
865  if ( $name == 'revision' ) {
866  $this->state = $name;
867  $this->egress->writeOpenPage( null, $this->buffer );
868  $this->buffer = "";
869  } elseif ( $name == 'page' ) {
870  $this->state = $name;
871  if ( $this->atStart ) {
872  $this->egress->writeOpenStream( $this->buffer );
873  $this->buffer = "";
874  $this->atStart = false;
875  }
876  }
877 
878  if ( $name == "text" && isset( $attribs['id'] ) ) {
879  $id = $attribs['id'];
880  $model = trim( $this->thisRevModel );
881  $format = trim( $this->thisRevFormat );
882 
883  $model = $model === '' ? null : $model;
884  $format = $format === '' ? null : $format;
885 
886  $text = $this->getText( $id, $model, $format );
887  $this->openElement = [ $name, [ 'xml:space' => 'preserve' ] ];
888  if ( strlen( $text ) > 0 ) {
889  $this->characterData( $parser, $text );
890  }
891  } else {
892  $this->openElement = [ $name, $attribs ];
893  }
894  }
895 
896  function endElement( $parser, $name ) {
897  $this->checkpointJustWritten = false;
898 
899  if ( $this->openElement ) {
900  $this->clearOpenElement( "" );
901  } else {
902  $this->buffer .= "</$name>";
903  }
904 
905  if ( $name == 'revision' ) {
906  $this->egress->writeRevision( null, $this->buffer );
907  $this->buffer = "";
908  $this->thisRev = "";
909  $this->thisRevModel = null;
910  $this->thisRevFormat = null;
911  } elseif ( $name == 'page' ) {
912  if ( !$this->firstPageWritten ) {
913  $this->firstPageWritten = trim( $this->thisPage );
914  }
915  $this->lastPageWritten = trim( $this->thisPage );
916  if ( $this->timeExceeded ) {
917  $this->egress->writeClosePage( $this->buffer );
918  // nasty hack, we can't just write the chardata after the
919  // page tag, it will include leading blanks from the next line
920  $this->egress->sink->write( "\n" );
921 
922  $this->buffer = $this->xmlwriterobj->closeStream();
923  $this->egress->writeCloseStream( $this->buffer );
924 
925  $this->buffer = "";
926  $this->thisPage = "";
927  // this could be more than one file if we had more than one output arg
928 
929  $filenameList = (array)$this->egress->getFilenames();
930  $newFilenames = [];
931  $firstPageID = str_pad( $this->firstPageWritten, 9, "0", STR_PAD_LEFT );
932  $lastPageID = str_pad( $this->lastPageWritten, 9, "0", STR_PAD_LEFT );
933  $filenamesCount = count( $filenameList );
934  for ( $i = 0; $i < $filenamesCount; $i++ ) {
935  $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID );
936  $fileinfo = pathinfo( $filenameList[$i] );
937  $newFilenames[] = $fileinfo['dirname'] . '/' . $checkpointNameFilledIn;
938  }
939  $this->egress->closeRenameAndReopen( $newFilenames );
940  $this->buffer = $this->xmlwriterobj->openStream();
941  $this->timeExceeded = false;
942  $this->timeOfCheckpoint = $this->lastTime;
943  $this->firstPageWritten = false;
944  $this->checkpointJustWritten = true;
945  } else {
946  $this->egress->writeClosePage( $this->buffer );
947  $this->buffer = "";
948  $this->thisPage = "";
949  }
950  } elseif ( $name == 'mediawiki' ) {
951  $this->egress->writeCloseStream( $this->buffer );
952  $this->buffer = "";
953  }
954  }
955 
956  function characterData( $parser, $data ) {
957  $this->clearOpenElement( null );
958  if ( $this->lastName == "id" ) {
959  if ( $this->state == "revision" ) {
960  $this->thisRev .= $data;
961  } elseif ( $this->state == "page" ) {
962  $this->thisPage .= $data;
963  }
964  } elseif ( $this->lastName == "model" ) {
965  $this->thisRevModel .= $data;
966  } elseif ( $this->lastName == "format" ) {
967  $this->thisRevFormat .= $data;
968  }
969 
970  // have to skip the newline left over from closepagetag line of
971  // end of checkpoint files. nasty hack!!
972  if ( $this->checkpointJustWritten ) {
973  if ( $data[0] == "\n" ) {
974  $data = substr( $data, 1 );
975  }
976  $this->checkpointJustWritten = false;
977  }
978  $this->buffer .= htmlspecialchars( $data );
979  }
980 
981  function clearOpenElement( $style ) {
982  if ( $this->openElement ) {
983  $this->buffer .= Xml::element( $this->openElement[0], $this->openElement[1], $style );
984  $this->openElement = false;
985  }
986  }
987 }
988 
989 $maintClass = 'TextPassDumper';
990 require_once RUN_MAINTENANCE_IF_MAIN;
bool resource $spawnWrite
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition: deferred.txt:11
IMaintainableDatabase $db
the array() calling protocol came about after MediaWiki 1.4rc1.
static getRevisionText($row, $prefix= 'old_', $wiki=false)
Get revision text associated with an old or archive row.
Definition: Revision.php:1008
const CONTENT_MODEL_WIKITEXT
Definition: Defines.php:236
static element($element, $attribs=null, $contents= '', $allowShortTag=true)
Format an XML element with given attributes and, optionally, text content.
Definition: Xml.php:39
$IP
Definition: WebStart.php:54
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
static getForModelID($modelId)
Returns the ContentHandler singleton for the given model ID.
div flags Integer display flags(NO_ACTION_LINK, NO_EXTRA_USER_LINKS) 'LogException'returning false will NOT prevent logging $e
Definition: hooks.txt:2149
We use the convention $dbr for read and $dbw for write to help you keep track of whether the database object is a the world will explode Or to be a subsequent write query which succeeded on the master may fail when replicated to the slave due to a unique key collision Replication on the slave will stop and it may take hours to repair the database and get it back online Setting read_only in my cnf on the slave will avoid this but given the dire we prefer to have as many checks as possible We provide a but the wrapper functions like please read the documentation for except in special pages derived from QueryPage It s a common pitfall for new developers to submit code containing SQL queries which examine huge numbers of rows Remember that COUNT * is(N), counting rows in atable is like counting beans in a bucket.------------------------------------------------------------------------Replication------------------------------------------------------------------------The largest installation of MediaWiki, Wikimedia, uses a large set ofslave MySQL servers replicating writes made to a master MySQL server.Itis important to understand the issues associated with this setup if youwant to write code destined for Wikipedia.It's often the case that the best algorithm to use for a given taskdepends on whether or not replication is in use.Due to our unabashedWikipedia-centrism, we often just use the replication-friendly version, but if you like, you can use wfGetLB() ->getServerCount() > 1 tocheck to see if replication is in use.===Lag===Lag primarily occurs when large write queries are sent to the master.Writes on the master are executed in parallel, but they are executed inserial when they are replicated to the slaves.The master writes thequery to the binlog when the transaction is committed.The slaves pollthe binlog and start executing the query as soon as it appears.They canservice reads while they are performing a write query, but will not readanything more from the binlog and thus will perform no more writes.Thismeans that if the write query runs for a long time, the slaves will lagbehind the master for the time it takes for the write query to complete.Lag can be exacerbated by high read load.MediaWiki's load balancer willstop sending reads to a slave when it is lagged by more than 30 seconds.If the load ratios are set incorrectly, or if there is too much loadgenerally, this may lead to a slave permanently hovering around 30seconds lag.If all slaves are lagged by more than 30 seconds, MediaWiki will stopwriting to the database.All edits and other write operations will berefused, with an error returned to the user.This gives the slaves achance to catch up.Before we had this mechanism, the slaves wouldregularly lag by several minutes, making review of recent editsdifficult.In addition to this, MediaWiki attempts to ensure that the user seesevents occurring on the wiki in chronological order.A few seconds of lagcan be tolerated, as long as the user sees a consistent picture fromsubsequent requests.This is done by saving the master binlog positionin the session, and then at the start of each request, waiting for theslave to catch up to that position before doing any reads from it.Ifthis wait times out, reads are allowed anyway, but the request isconsidered to be in"lagged slave mode".Lagged slave mode can bechecked by calling wfGetLB() ->getLaggedSlaveMode().The onlypractical consequence at present is a warning displayed in the pagefooter.===Lag avoidance===To avoid excessive lag, queries which write large numbers of rows shouldbe split up, generally to write one row at a time.Multi-row INSERT...SELECT queries are the worst offenders should be avoided altogether.Instead do the select first and then the insert.===Working with lag===Despite our best efforts, it's not practical to guarantee a low-lagenvironment.Lag will usually be less than one second, but mayoccasionally be up to 30 seconds.For scalability, it's very importantto keep load on the master low, so simply sending all your queries tothe master is not the answer.So when you have a genuine need forup-to-date data, the following approach is advised:1) Do a quick query to the master for a sequence number or timestamp 2) Run the full query on the slave and check if it matches the data you gotfrom the master 3) If it doesn't, run the full query on the masterTo avoid swamping the master every time the slaves lag, use of thisapproach should be kept to a minimum.In most cases you should just readfrom the slave and let the user deal with the delay.------------------------------------------------------------------------Lock contention------------------------------------------------------------------------Due to the high write rate on Wikipedia(and some other wikis), MediaWiki developers need to be very careful to structure their writesto avoid long-lasting locks.By default, MediaWiki opens a transactionat the first query, and commits it before the output is sent.Locks willbe held from the time when the query is done until the commit.So youcan reduce lock time by doing as much processing as possible before youdo your write queries.Often this approach is not good enough, and it becomes necessary toenclose small groups of queries in their own transaction.Use thefollowing syntax:$dbw=wfGetDB(DB_MASTER
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for and distribution as defined by Sections through of this document Licensor shall mean the copyright owner or entity authorized by the copyright owner that is granting the License Legal Entity shall mean the union of the acting entity and all other entities that control are controlled by or are under common control with that entity For the purposes of this definition control direct or to cause the direction or management of such whether by contract or including but not limited to software source documentation and configuration files Object form shall mean any form resulting from mechanical transformation or translation of a Source including but not limited to compiled object generated and conversions to other media types Work shall mean the work of whether in Source or Object made available under the as indicated by a copyright notice that is included in or attached to the whether in Source or Object that is based or other modifications as a an original work of authorship For the purposes of this Derivative Works shall not include works that remain separable from
string bool $thisRev
getTextDb($id)
May throw a database error if, say, the server dies during query.
hasOption($name)
Checks to see if a particular param exists.
require_once RUN_MAINTENANCE_IF_MAIN
Definition: maintenance.txt:50
showReport()
Overridden to include prefetch ratio if enabled.
target page
when a variable name is used in a it is silently declared as a new local masking the global
Definition: design.txt:93
bool XmlDumpWriter $xmlwriterobj
report($final=false)
Definition: backup.inc:371
magic word & $parser
Definition: hooks.txt:2563
exportTransform($text, $model, $format=null)
Applies applicable export transformations to $text.
wfDebug($text, $dest= 'all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
if($line===false) $args
Definition: cdb.php:64
The ContentHandler facility adds support for arbitrary content types on wiki instead of relying on wikitext for everything It was introduced in MediaWiki Each kind of and so on Built in content types are
An extension or a local will often add custom code to the function with or without a global variable For someone wanting email notification when an article is shown may add
Definition: hooks.txt:51
$maintClass
wfTimestamp($outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
addOption($name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
either a unescaped string or a HtmlArmor object after in associative array form externallinks including delete and has completed for all link tables whether this was an auto creation default is conds Array Extra conditions for the No matching items in log is displayed if loglist is empty msgKey Array If you want a nice box with a set this to the key of the message First element is the message additional optional elements are parameters for the key that are processed with wfMessage() -> params() ->parseAsBlock()-offset Set to overwrite offset parameter in $wgRequest set to ''to unsetoffset-wrap String Wrap the message in html(usually something like"&lt
dump($history, $text=WikiExporter::TEXT)
bool resource $spawnProc
$maxConsecutiveFailedTextRetrievals
endElement($parser, $name)
null means default in associative array with keys and values unescaped Should be merged with default with a value of false meaning to suppress the attribute in associative array with keys and values unescaped noclasses just before the function returns a value If you return an< a > element with HTML attributes $attribs and contents $html will be returned If you return $ret will be returned and may include noclasses after processing & $attribs
Definition: hooks.txt:1973
$buffer
design txt This is a brief overview of the new design More thorough and up to date information is available on the documentation wiki at etc Handles the details of getting and saving to the user table of the and dealing with sessions and cookies OutputPage Encapsulates the entire HTML page that will be sent in response to any server request It is used by calling its functions to add text
Definition: design.txt:12
wfWikiID()
Get an ASCII string identifying this wiki This is used as a prefix in memcached keys.
addDescription($text)
Set the description text.
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
getOption($name, $default=null)
Get an option, or return the default.
output($out, $channel=null)
Throw some output to the user.
clearOpenElement($style)
loadWithArgv($argv)
Load params and arguments from a given array of command-line arguments.
string bool $thisPage
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
presenting them properly to the user as errors is done by the caller return true use this to change the list i e etc next in line in page history
Definition: hooks.txt:1753
wfGetLBFactory()
Get the load balancer factory object.
IDatabase null $forcedDb
The dependency-injected database to use.
Definition: backup.inc:69
startElement($parser, $name, $attribs)
bool resource $spawnRead
this class mediates it Skin Encapsulates a look and feel for the wiki All of the functions that render HTML and make choices about how to render it are here and are called from various other places when and is meant to be subclassed with other skins that may override some of its functions The User object contains a reference to a and so rather than having a global skin object we just rely on the global User and get the skin with $wgUser and also has some character encoding functions and other locale stuff The current user interface language is instantiated as and the local content language as $wgContLang
Definition: design.txt:56
Allows to change the fields on the form that will be generated $name
Definition: hooks.txt:302
We ve cleaned up the code here by removing clumps of infrequently used code and moving them off somewhere else It s much easier for someone working with this code to see what s _really_ going on
Definition: hooks.txt:84
const DB_REPLICA
Definition: defines.php:25
progress($string)
Definition: backup.inc:416
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that probably a stub it is not rendered in wiki pages or galleries in category pages allow injecting custom HTML after the section Any uses of the hook need to handle escaping see BaseTemplate::getToolbox and BaseTemplate::makeListItem for details on the format of individual items inside of this array or by returning and letting standard HTTP rendering take place modifiable or by returning false and taking over the output modifiable modifiable after all normalizations have been except for the $wgMaxImageArea check set to true or false to override the $wgMaxImageArea check result gives extension the possibility to transform it themselves $handler
Definition: hooks.txt:784
BaseDump $prefetch
initProgress($history=WikiExporter::FULL)
bool resource $spawnErr
characterData($parser, $data)
__construct($args=null)
rotateDb()
Drop the database connection $this->db and try to get a new one.
getText($id, $model=null, $format=null)
Tries to get the revision text for a revision id.