MediaWiki  1.27.2
dumpTextPass.php
Go to the documentation of this file.
1 <?php
27 require_once __DIR__ . '/backup.inc';
28 require_once __DIR__ . '/../includes/export/WikiExporter.php';
29 
34  public $prefetch = null;
35 
36  // when we spend more than maxTimeAllowed seconds on this run, we continue
37  // processing until we write out the next complete page, then save output file(s),
38  // rename it/them and open new one(s)
39  public $maxTimeAllowed = 0; // 0 = no limit
40 
41  protected $input = "php://stdin";
43  protected $fetchCount = 0;
44  protected $prefetchCount = 0;
45  protected $prefetchCountLast = 0;
46  protected $fetchCountLast = 0;
47 
48  protected $maxFailures = 5;
50  protected $failureTimeout = 5; // Seconds to sleep after db failure
51 
52  protected $bufferSize = 524288; // In bytes. Maximum size to read from the stub in on go.
53 
54  protected $php = "php";
55  protected $spawn = false;
56 
60  protected $spawnProc = false;
61 
65  protected $spawnWrite = false;
66 
70  protected $spawnRead = false;
71 
75  protected $spawnErr = false;
76 
77  protected $xmlwriterobj = false;
78 
79  protected $timeExceeded = false;
80  protected $firstPageWritten = false;
81  protected $lastPageWritten = false;
82  protected $checkpointJustWritten = false;
83  protected $checkpointFiles = [];
84 
88  protected $db;
89 
93  function __construct( $args = null ) {
94  parent::__construct();
95 
96  $this->addDescription( <<<TEXT
97 This script postprocesses XML dumps from dumpBackup.php to add
98 page text which was stubbed out (using --stub).
99 
100 XML input is accepted on stdin.
101 XML output is sent to stdout; progress reports are sent to stderr.
102 TEXT
103  );
104  $this->stderr = fopen( "php://stderr", "wt" );
105 
106  $this->addOption( 'stub', 'To load a compressed stub dump instead of stdin. ' .
107  'Specify as --stub=<type>:<file>.', false, true );
108  $this->addOption( 'prefetch', 'Use a prior dump file as a text source, to savepressure on the ' .
109  'database. (Requires the XMLReader extension). Specify as --prefetch=<type>:<file>',
110  false, true );
111  $this->addOption( 'maxtime', 'Write out checkpoint file after this many minutes (writing' .
112  'out complete page, closing xml file properly, and opening new one' .
113  'with header). This option requires the checkpointfile option.', false, true );
114  $this->addOption( 'checkpointfile', 'Use this string for checkpoint filenames,substituting ' .
115  'first pageid written for the first %s (required) and the last pageid written for the ' .
116  'second %s if it exists.', false, true, false, true ); // This can be specified multiple times
117  $this->addOption( 'quiet', 'Don\'t dump status reports to stderr.' );
118  $this->addOption( 'current', 'Base ETA on number of pages in database instead of all revisions' );
119  $this->addOption( 'spawn', 'Spawn a subprocess for loading text records' );
120  $this->addOption( 'buffersize', 'Buffer size in bytes to use for reading the stub. ' .
121  '(Default: 512KB, Minimum: 4KB)', false, true );
122 
123  if ( $args ) {
124  $this->loadWithArgv( $args );
125  $this->processOptions();
126  }
127  }
128 
129  function execute() {
130  $this->processOptions();
131  $this->dump( true );
132  }
133 
134  function processOptions() {
135  global $IP;
136 
137  parent::processOptions();
138 
139  if ( $this->hasOption( 'buffersize' ) ) {
140  $this->bufferSize = max( intval( $this->getOption( 'buffersize' ) ), 4 * 1024 );
141  }
142 
143  if ( $this->hasOption( 'prefetch' ) ) {
144  require_once "$IP/maintenance/backupPrefetch.inc";
145  $url = $this->processFileOpt( $this->getOption( 'prefetch' ) );
146  $this->prefetch = new BaseDump( $url );
147  }
148 
149  if ( $this->hasOption( 'stub' ) ) {
150  $this->input = $this->processFileOpt( $this->getOption( 'stub' ) );
151  }
152 
153  if ( $this->hasOption( 'maxtime' ) ) {
154  $this->maxTimeAllowed = intval( $this->getOption( 'maxtime' ) ) * 60;
155  }
156 
157  if ( $this->hasOption( 'checkpointfile' ) ) {
158  $this->checkpointFiles = $this->getOption( 'checkpointfile' );
159  }
160 
161  if ( $this->hasOption( 'current' ) ) {
163  }
164 
165  if ( $this->hasOption( 'full' ) ) {
166  $this->history = WikiExporter::FULL;
167  }
168 
169  if ( $this->hasOption( 'spawn' ) ) {
170  $this->spawn = true;
171  $val = $this->getOption( 'spawn' );
172  if ( $val !== 1 ) {
173  $this->php = $val;
174  }
175  }
176  }
177 
189  function rotateDb() {
190  // Cleaning up old connections
191  if ( isset( $this->lb ) ) {
192  $this->lb->closeAll();
193  unset( $this->lb );
194  }
195 
196  if ( $this->forcedDb !== null ) {
197  $this->db = $this->forcedDb;
198 
199  return;
200  }
201 
202  if ( isset( $this->db ) && $this->db->isOpen() ) {
203  throw new MWException( 'DB is set and has not been closed by the Load Balancer' );
204  }
205 
206  unset( $this->db );
207 
208  // Trying to set up new connection.
209  // We do /not/ retry upon failure, but delegate to encapsulating logic, to avoid
210  // individually retrying at different layers of code.
211 
212  // 1. The LoadBalancer.
213  try {
214  $this->lb = wfGetLBFactory()->newMainLB();
215  } catch ( Exception $e ) {
216  throw new MWException( __METHOD__
217  . " rotating DB failed to obtain new load balancer (" . $e->getMessage() . ")" );
218  }
219 
220  // 2. The Connection, through the load balancer.
221  try {
222  $this->db = $this->lb->getConnection( DB_SLAVE, 'dump' );
223  } catch ( Exception $e ) {
224  throw new MWException( __METHOD__
225  . " rotating DB failed to obtain new database (" . $e->getMessage() . ")" );
226  }
227  }
228 
230  parent::initProgress();
231  $this->timeOfCheckpoint = $this->startTime;
232  }
233 
234  function dump( $history, $text = WikiExporter::TEXT ) {
235  // Notice messages will foul up your XML output even if they're
236  // relatively harmless.
237  if ( ini_get( 'display_errors' ) ) {
238  ini_set( 'display_errors', 'stderr' );
239  }
240 
241  $this->initProgress( $this->history );
242 
243  // We are trying to get an initial database connection to avoid that the
244  // first try of this request's first call to getText fails. However, if
245  // obtaining a good DB connection fails it's not a serious issue, as
246  // getText does retry upon failure and can start without having a working
247  // DB connection.
248  try {
249  $this->rotateDb();
250  } catch ( Exception $e ) {
251  // We do not even count this as failure. Just let eventual
252  // watchdogs know.
253  $this->progress( "Getting initial DB connection failed (" .
254  $e->getMessage() . ")" );
255  }
256 
257  $this->egress = new ExportProgressFilter( $this->sink, $this );
258 
259  // it would be nice to do it in the constructor, oh well. need egress set
260  $this->finalOptionCheck();
261 
262  // we only want this so we know how to close a stream :-P
263  $this->xmlwriterobj = new XmlDumpWriter();
264 
265  $input = fopen( $this->input, "rt" );
266  $this->readDump( $input );
267 
268  if ( $this->spawnProc ) {
269  $this->closeSpawn();
270  }
271 
272  $this->report( true );
273  }
274 
275  function processFileOpt( $opt ) {
276  $split = explode( ':', $opt, 2 );
277  $val = $split[0];
278  $param = '';
279  if ( count( $split ) === 2 ) {
280  $param = $split[1];
281  }
282  $fileURIs = explode( ';', $param );
283  foreach ( $fileURIs as $URI ) {
284  switch ( $val ) {
285  case "file":
286  $newURI = $URI;
287  break;
288  case "gzip":
289  $newURI = "compress.zlib://$URI";
290  break;
291  case "bzip2":
292  $newURI = "compress.bzip2://$URI";
293  break;
294  case "7zip":
295  $newURI = "mediawiki.compress.7z://$URI";
296  break;
297  default:
298  $newURI = $URI;
299  }
300  $newFileURIs[] = $newURI;
301  }
302  $val = implode( ';', $newFileURIs );
303 
304  return $val;
305  }
306 
310  function showReport() {
311  if ( !$this->prefetch ) {
312  parent::showReport();
313 
314  return;
315  }
316 
317  if ( $this->reporting ) {
318  $now = wfTimestamp( TS_DB );
319  $nowts = microtime( true );
320  $deltaAll = $nowts - $this->startTime;
321  $deltaPart = $nowts - $this->lastTime;
322  $this->pageCountPart = $this->pageCount - $this->pageCountLast;
323  $this->revCountPart = $this->revCount - $this->revCountLast;
324 
325  if ( $deltaAll ) {
326  $portion = $this->revCount / $this->maxCount;
327  $eta = $this->startTime + $deltaAll / $portion;
328  $etats = wfTimestamp( TS_DB, intval( $eta ) );
329  if ( $this->fetchCount ) {
330  $fetchRate = 100.0 * $this->prefetchCount / $this->fetchCount;
331  } else {
332  $fetchRate = '-';
333  }
334  $pageRate = $this->pageCount / $deltaAll;
335  $revRate = $this->revCount / $deltaAll;
336  } else {
337  $pageRate = '-';
338  $revRate = '-';
339  $etats = '-';
340  $fetchRate = '-';
341  }
342  if ( $deltaPart ) {
343  if ( $this->fetchCountLast ) {
344  $fetchRatePart = 100.0 * $this->prefetchCountLast / $this->fetchCountLast;
345  } else {
346  $fetchRatePart = '-';
347  }
348  $pageRatePart = $this->pageCountPart / $deltaPart;
349  $revRatePart = $this->revCountPart / $deltaPart;
350  } else {
351  $fetchRatePart = '-';
352  $pageRatePart = '-';
353  $revRatePart = '-';
354  }
355  $this->progress( sprintf(
356  "%s: %s (ID %d) %d pages (%0.1f|%0.1f/sec all|curr), "
357  . "%d revs (%0.1f|%0.1f/sec all|curr), %0.1f%%|%0.1f%% "
358  . "prefetched (all|curr), ETA %s [max %d]",
359  $now, wfWikiID(), $this->ID, $this->pageCount, $pageRate,
360  $pageRatePart, $this->revCount, $revRate, $revRatePart,
361  $fetchRate, $fetchRatePart, $etats, $this->maxCount
362  ) );
363  $this->lastTime = $nowts;
364  $this->revCountLast = $this->revCount;
365  $this->prefetchCountLast = $this->prefetchCount;
366  $this->fetchCountLast = $this->fetchCount;
367  }
368  }
369 
370  function setTimeExceeded() {
371  $this->timeExceeded = true;
372  }
373 
374  function checkIfTimeExceeded() {
375  if ( $this->maxTimeAllowed
376  && ( $this->lastTime - $this->timeOfCheckpoint > $this->maxTimeAllowed )
377  ) {
378  return true;
379  }
380 
381  return false;
382  }
383 
384  function finalOptionCheck() {
385  if ( ( $this->checkpointFiles && !$this->maxTimeAllowed )
386  || ( $this->maxTimeAllowed && !$this->checkpointFiles )
387  ) {
388  throw new MWException( "Options checkpointfile and maxtime must be specified together.\n" );
389  }
390  foreach ( $this->checkpointFiles as $checkpointFile ) {
391  $count = substr_count( $checkpointFile, "%s" );
392  if ( $count != 2 ) {
393  throw new MWException( "Option checkpointfile must contain two '%s' "
394  . "for substitution of first and last pageids, count is $count instead, "
395  . "file is $checkpointFile.\n" );
396  }
397  }
398 
399  if ( $this->checkpointFiles ) {
400  $filenameList = (array)$this->egress->getFilenames();
401  if ( count( $filenameList ) != count( $this->checkpointFiles ) ) {
402  throw new MWException( "One checkpointfile must be specified "
403  . "for each output option, if maxtime is used.\n" );
404  }
405  }
406  }
407 
413  function readDump( $input ) {
414  $this->buffer = "";
415  $this->openElement = false;
416  $this->atStart = true;
417  $this->state = "";
418  $this->lastName = "";
419  $this->thisPage = 0;
420  $this->thisRev = 0;
421  $this->thisRevModel = null;
422  $this->thisRevFormat = null;
423 
424  $parser = xml_parser_create( "UTF-8" );
425  xml_parser_set_option( $parser, XML_OPTION_CASE_FOLDING, false );
426 
427  xml_set_element_handler(
428  $parser,
429  [ $this, 'startElement' ],
430  [ $this, 'endElement' ]
431  );
432  xml_set_character_data_handler( $parser, [ $this, 'characterData' ] );
433 
434  $offset = 0; // for context extraction on error reporting
435  do {
436  if ( $this->checkIfTimeExceeded() ) {
437  $this->setTimeExceeded();
438  }
439  $chunk = fread( $input, $this->bufferSize );
440  if ( !xml_parse( $parser, $chunk, feof( $input ) ) ) {
441  wfDebug( "TextDumpPass::readDump encountered XML parsing error\n" );
442 
443  $byte = xml_get_current_byte_index( $parser );
444  $msg = wfMessage( 'xml-error-string',
445  'XML import parse failure',
446  xml_get_current_line_number( $parser ),
447  xml_get_current_column_number( $parser ),
448  $byte . ( is_null( $chunk ) ? null : ( '; "' . substr( $chunk, $byte - $offset, 16 ) . '"' ) ),
449  xml_error_string( xml_get_error_code( $parser ) ) )->escaped();
450 
451  xml_parser_free( $parser );
452 
453  throw new MWException( $msg );
454  }
455  $offset += strlen( $chunk );
456  } while ( $chunk !== false && !feof( $input ) );
457  if ( $this->maxTimeAllowed ) {
458  $filenameList = (array)$this->egress->getFilenames();
459  // we wrote some stuff after last checkpoint that needs renamed
460  if ( file_exists( $filenameList[0] ) ) {
461  $newFilenames = [];
462  # we might have just written the header and footer and had no
463  # pages or revisions written... perhaps they were all deleted
464  # there's no pageID 0 so we use that. the caller is responsible
465  # for deciding what to do with a file containing only the
466  # siteinfo information and the mw tags.
467  if ( !$this->firstPageWritten ) {
468  $firstPageID = str_pad( 0, 9, "0", STR_PAD_LEFT );
469  $lastPageID = str_pad( 0, 9, "0", STR_PAD_LEFT );
470  } else {
471  $firstPageID = str_pad( $this->firstPageWritten, 9, "0", STR_PAD_LEFT );
472  $lastPageID = str_pad( $this->lastPageWritten, 9, "0", STR_PAD_LEFT );
473  }
474 
475  $filenameCount = count( $filenameList );
476  for ( $i = 0; $i < $filenameCount; $i++ ) {
477  $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID );
478  $fileinfo = pathinfo( $filenameList[$i] );
479  $newFilenames[] = $fileinfo['dirname'] . '/' . $checkpointNameFilledIn;
480  }
481  $this->egress->closeAndRename( $newFilenames );
482  }
483  }
484  xml_parser_free( $parser );
485 
486  return true;
487  }
488 
498  private function exportTransform( $text, $model, $format = null ) {
499  try {
501  $text = $handler->exportTransform( $text, $format );
502  }
503  catch ( MWException $ex ) {
504  $this->progress(
505  "Unable to apply export transformation for content model '$model': " .
506  $ex->getMessage()
507  );
508  }
509 
510  return $text;
511  }
512 
533  function getText( $id, $model = null, $format = null ) {
534  global $wgContentHandlerUseDB;
535 
536  $prefetchNotTried = true; // Whether or not we already tried to get the text via prefetch.
537  $text = false; // The candidate for a good text. false if no proper value.
538  $failures = 0; // The number of times, this invocation of getText already failed.
539 
540  // The number of times getText failed without yielding a good text in between.
541  static $consecutiveFailedTextRetrievals = 0;
542 
543  $this->fetchCount++;
544 
545  // To allow to simply return on success and do not have to worry about book keeping,
546  // we assume, this fetch works (possible after some retries). Nevertheless, we koop
547  // the old value, so we can restore it, if problems occur (See after the while loop).
548  $oldConsecutiveFailedTextRetrievals = $consecutiveFailedTextRetrievals;
549  $consecutiveFailedTextRetrievals = 0;
550 
551  if ( $model === null && $wgContentHandlerUseDB ) {
552  $row = $this->db->selectRow(
553  'revision',
554  [ 'rev_content_model', 'rev_content_format' ],
555  [ 'rev_id' => $this->thisRev ],
556  __METHOD__
557  );
558 
559  if ( $row ) {
560  $model = $row->rev_content_model;
561  $format = $row->rev_content_format;
562  }
563  }
564 
565  if ( $model === null || $model === '' ) {
566  $model = false;
567  }
568 
569  while ( $failures < $this->maxFailures ) {
570 
571  // As soon as we found a good text for the $id, we will return immediately.
572  // Hence, if we make it past the try catch block, we know that we did not
573  // find a good text.
574 
575  try {
576  // Step 1: Get some text (or reuse from previous iteratuon if checking
577  // for plausibility failed)
578 
579  // Trying to get prefetch, if it has not been tried before
580  if ( $text === false && isset( $this->prefetch ) && $prefetchNotTried ) {
581  $prefetchNotTried = false;
582  $tryIsPrefetch = true;
583  $text = $this->prefetch->prefetch( intval( $this->thisPage ),
584  intval( $this->thisRev ) );
585 
586  if ( $text === null ) {
587  $text = false;
588  }
589 
590  if ( is_string( $text ) && $model !== false ) {
591  // Apply export transformation to text coming from an old dump.
592  // The purpose of this transformation is to convert up from legacy
593  // formats, which may still be used in the older dump that is used
594  // for pre-fetching. Applying the transformation again should not
595  // interfere with content that is already in the correct form.
596  $text = $this->exportTransform( $text, $model, $format );
597  }
598  }
599 
600  if ( $text === false ) {
601  // Fallback to asking the database
602  $tryIsPrefetch = false;
603  if ( $this->spawn ) {
604  $text = $this->getTextSpawned( $id );
605  } else {
606  $text = $this->getTextDb( $id );
607  }
608 
609  if ( $text !== false && $model !== false ) {
610  // Apply export transformation to text coming from the database.
611  // Prefetched text should already have transformations applied.
612  $text = $this->exportTransform( $text, $model, $format );
613  }
614 
615  // No more checks for texts from DB for now.
616  // If we received something that is not false,
617  // We treat it as good text, regardless of whether it actually is or is not
618  if ( $text !== false ) {
619  return $text;
620  }
621  }
622 
623  if ( $text === false ) {
624  throw new MWException( "Generic error while obtaining text for id " . $id );
625  }
626 
627  // We received a good candidate for the text of $id via some method
628 
629  // Step 2: Checking for plausibility and return the text if it is
630  // plausible
631  $revID = intval( $this->thisRev );
632  if ( !isset( $this->db ) ) {
633  throw new MWException( "No database available" );
634  }
635 
636  if ( $model !== CONTENT_MODEL_WIKITEXT ) {
637  $revLength = strlen( $text );
638  } else {
639  $revLength = $this->db->selectField( 'revision', 'rev_len', [ 'rev_id' => $revID ] );
640  }
641 
642  if ( strlen( $text ) == $revLength ) {
643  if ( $tryIsPrefetch ) {
644  $this->prefetchCount++;
645  }
646 
647  return $text;
648  }
649 
650  $text = false;
651  throw new MWException( "Received text is unplausible for id " . $id );
652  } catch ( Exception $e ) {
653  $msg = "getting/checking text " . $id . " failed (" . $e->getMessage() . ")";
654  if ( $failures + 1 < $this->maxFailures ) {
655  $msg .= " (Will retry " . ( $this->maxFailures - $failures - 1 ) . " more times)";
656  }
657  $this->progress( $msg );
658  }
659 
660  // Something went wrong; we did not a text that was plausible :(
661  $failures++;
662 
663  // A failure in a prefetch hit does not warrant resetting db connection etc.
664  if ( !$tryIsPrefetch ) {
665  // After backing off for some time, we try to reboot the whole process as
666  // much as possible to not carry over failures from one part to the other
667  // parts
668  sleep( $this->failureTimeout );
669  try {
670  $this->rotateDb();
671  if ( $this->spawn ) {
672  $this->closeSpawn();
673  $this->openSpawn();
674  }
675  } catch ( Exception $e ) {
676  $this->progress( "Rebooting getText infrastructure failed (" . $e->getMessage() . ")" .
677  " Trying to continue anyways" );
678  }
679  }
680  }
681 
682  // Retirieving a good text for $id failed (at least) maxFailures times.
683  // We abort for this $id.
684 
685  // Restoring the consecutive failures, and maybe aborting, if the dump
686  // is too broken.
687  $consecutiveFailedTextRetrievals = $oldConsecutiveFailedTextRetrievals + 1;
688  if ( $consecutiveFailedTextRetrievals > $this->maxConsecutiveFailedTextRetrievals ) {
689  throw new MWException( "Graceful storage failure" );
690  }
691 
692  return "";
693  }
694 
701  private function getTextDb( $id ) {
703  if ( !isset( $this->db ) ) {
704  throw new MWException( __METHOD__ . "No database available" );
705  }
706  $row = $this->db->selectRow( 'text',
707  [ 'old_text', 'old_flags' ],
708  [ 'old_id' => $id ],
709  __METHOD__ );
710  $text = Revision::getRevisionText( $row );
711  if ( $text === false ) {
712  return false;
713  }
714  $stripped = str_replace( "\r", "", $text );
715  $normalized = $wgContLang->normalize( $stripped );
716 
717  return $normalized;
718  }
719 
720  private function getTextSpawned( $id ) {
721  MediaWiki\suppressWarnings();
722  if ( !$this->spawnProc ) {
723  // First time?
724  $this->openSpawn();
725  }
726  $text = $this->getTextSpawnedOnce( $id );
727  MediaWiki\restoreWarnings();
728 
729  return $text;
730  }
731 
732  function openSpawn() {
733  global $IP;
734 
735  if ( file_exists( "$IP/../multiversion/MWScript.php" ) ) {
736  $cmd = implode( " ",
737  array_map( 'wfEscapeShellArg',
738  [
739  $this->php,
740  "$IP/../multiversion/MWScript.php",
741  "fetchText.php",
742  '--wiki', wfWikiID() ] ) );
743  } else {
744  $cmd = implode( " ",
745  array_map( 'wfEscapeShellArg',
746  [
747  $this->php,
748  "$IP/maintenance/fetchText.php",
749  '--wiki', wfWikiID() ] ) );
750  }
751  $spec = [
752  0 => [ "pipe", "r" ],
753  1 => [ "pipe", "w" ],
754  2 => [ "file", "/dev/null", "a" ] ];
755  $pipes = [];
756 
757  $this->progress( "Spawning database subprocess: $cmd" );
758  $this->spawnProc = proc_open( $cmd, $spec, $pipes );
759  if ( !$this->spawnProc ) {
760  $this->progress( "Subprocess spawn failed." );
761 
762  return false;
763  }
764  list(
765  $this->spawnWrite, // -> stdin
766  $this->spawnRead, // <- stdout
767  ) = $pipes;
768 
769  return true;
770  }
771 
772  private function closeSpawn() {
773  MediaWiki\suppressWarnings();
774  if ( $this->spawnRead ) {
775  fclose( $this->spawnRead );
776  }
777  $this->spawnRead = false;
778  if ( $this->spawnWrite ) {
779  fclose( $this->spawnWrite );
780  }
781  $this->spawnWrite = false;
782  if ( $this->spawnErr ) {
783  fclose( $this->spawnErr );
784  }
785  $this->spawnErr = false;
786  if ( $this->spawnProc ) {
787  pclose( $this->spawnProc );
788  }
789  $this->spawnProc = false;
790  MediaWiki\restoreWarnings();
791  }
792 
793  private function getTextSpawnedOnce( $id ) {
795 
796  $ok = fwrite( $this->spawnWrite, "$id\n" );
797  // $this->progress( ">> $id" );
798  if ( !$ok ) {
799  return false;
800  }
801 
802  $ok = fflush( $this->spawnWrite );
803  // $this->progress( ">> [flush]" );
804  if ( !$ok ) {
805  return false;
806  }
807 
808  // check that the text id they are sending is the one we asked for
809  // this avoids out of sync revision text errors we have encountered in the past
810  $newId = fgets( $this->spawnRead );
811  if ( $newId === false ) {
812  return false;
813  }
814  if ( $id != intval( $newId ) ) {
815  return false;
816  }
817 
818  $len = fgets( $this->spawnRead );
819  // $this->progress( "<< " . trim( $len ) );
820  if ( $len === false ) {
821  return false;
822  }
823 
824  $nbytes = intval( $len );
825  // actual error, not zero-length text
826  if ( $nbytes < 0 ) {
827  return false;
828  }
829 
830  $text = "";
831 
832  // Subprocess may not send everything at once, we have to loop.
833  while ( $nbytes > strlen( $text ) ) {
834  $buffer = fread( $this->spawnRead, $nbytes - strlen( $text ) );
835  if ( $buffer === false ) {
836  break;
837  }
838  $text .= $buffer;
839  }
840 
841  $gotbytes = strlen( $text );
842  if ( $gotbytes != $nbytes ) {
843  $this->progress( "Expected $nbytes bytes from database subprocess, got $gotbytes " );
844 
845  return false;
846  }
847 
848  // Do normalization in the dump thread...
849  $stripped = str_replace( "\r", "", $text );
850  $normalized = $wgContLang->normalize( $stripped );
851 
852  return $normalized;
853  }
854 
856  $this->checkpointJustWritten = false;
857 
858  $this->clearOpenElement( null );
859  $this->lastName = $name;
860 
861  if ( $name == 'revision' ) {
862  $this->state = $name;
863  $this->egress->writeOpenPage( null, $this->buffer );
864  $this->buffer = "";
865  } elseif ( $name == 'page' ) {
866  $this->state = $name;
867  if ( $this->atStart ) {
868  $this->egress->writeOpenStream( $this->buffer );
869  $this->buffer = "";
870  $this->atStart = false;
871  }
872  }
873 
874  if ( $name == "text" && isset( $attribs['id'] ) ) {
875  $id = $attribs['id'];
876  $model = trim( $this->thisRevModel );
877  $format = trim( $this->thisRevFormat );
878 
879  $model = $model === '' ? null : $model;
880  $format = $format === '' ? null : $format;
881 
882  $text = $this->getText( $id, $model, $format );
883  $this->openElement = [ $name, [ 'xml:space' => 'preserve' ] ];
884  if ( strlen( $text ) > 0 ) {
885  $this->characterData( $parser, $text );
886  }
887  } else {
888  $this->openElement = [ $name, $attribs ];
889  }
890  }
891 
892  function endElement( $parser, $name ) {
893  $this->checkpointJustWritten = false;
894 
895  if ( $this->openElement ) {
896  $this->clearOpenElement( "" );
897  } else {
898  $this->buffer .= "</$name>";
899  }
900 
901  if ( $name == 'revision' ) {
902  $this->egress->writeRevision( null, $this->buffer );
903  $this->buffer = "";
904  $this->thisRev = "";
905  $this->thisRevModel = null;
906  $this->thisRevFormat = null;
907  } elseif ( $name == 'page' ) {
908  if ( !$this->firstPageWritten ) {
909  $this->firstPageWritten = trim( $this->thisPage );
910  }
911  $this->lastPageWritten = trim( $this->thisPage );
912  if ( $this->timeExceeded ) {
913  $this->egress->writeClosePage( $this->buffer );
914  // nasty hack, we can't just write the chardata after the
915  // page tag, it will include leading blanks from the next line
916  $this->egress->sink->write( "\n" );
917 
918  $this->buffer = $this->xmlwriterobj->closeStream();
919  $this->egress->writeCloseStream( $this->buffer );
920 
921  $this->buffer = "";
922  $this->thisPage = "";
923  // this could be more than one file if we had more than one output arg
924 
925  $filenameList = (array)$this->egress->getFilenames();
926  $newFilenames = [];
927  $firstPageID = str_pad( $this->firstPageWritten, 9, "0", STR_PAD_LEFT );
928  $lastPageID = str_pad( $this->lastPageWritten, 9, "0", STR_PAD_LEFT );
929  $filenamesCount = count( $filenameList );
930  for ( $i = 0; $i < $filenamesCount; $i++ ) {
931  $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID );
932  $fileinfo = pathinfo( $filenameList[$i] );
933  $newFilenames[] = $fileinfo['dirname'] . '/' . $checkpointNameFilledIn;
934  }
935  $this->egress->closeRenameAndReopen( $newFilenames );
936  $this->buffer = $this->xmlwriterobj->openStream();
937  $this->timeExceeded = false;
938  $this->timeOfCheckpoint = $this->lastTime;
939  $this->firstPageWritten = false;
940  $this->checkpointJustWritten = true;
941  } else {
942  $this->egress->writeClosePage( $this->buffer );
943  $this->buffer = "";
944  $this->thisPage = "";
945  }
946  } elseif ( $name == 'mediawiki' ) {
947  $this->egress->writeCloseStream( $this->buffer );
948  $this->buffer = "";
949  }
950  }
951 
952  function characterData( $parser, $data ) {
953  $this->clearOpenElement( null );
954  if ( $this->lastName == "id" ) {
955  if ( $this->state == "revision" ) {
956  $this->thisRev .= $data;
957  } elseif ( $this->state == "page" ) {
958  $this->thisPage .= $data;
959  }
960  } elseif ( $this->lastName == "model" ) {
961  $this->thisRevModel .= $data;
962  } elseif ( $this->lastName == "format" ) {
963  $this->thisRevFormat .= $data;
964  }
965 
966  // have to skip the newline left over from closepagetag line of
967  // end of checkpoint files. nasty hack!!
968  if ( $this->checkpointJustWritten ) {
969  if ( $data[0] == "\n" ) {
970  $data = substr( $data, 1 );
971  }
972  $this->checkpointJustWritten = false;
973  }
974  $this->buffer .= htmlspecialchars( $data );
975  }
976 
977  function clearOpenElement( $style ) {
978  if ( $this->openElement ) {
979  $this->buffer .= Xml::element( $this->openElement[0], $this->openElement[1], $style );
980  $this->openElement = false;
981  }
982  }
983 }
984 
985 $maintClass = 'TextPassDumper';
986 require_once RUN_MAINTENANCE_IF_MAIN;
bool resource $spawnWrite
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition: deferred.txt:11
the array() calling protocol came about after MediaWiki 1.4rc1.
static getRevisionText($row, $prefix= 'old_', $wiki=false)
Get revision text associated with an old or archive row $row is usually an object from wfFetchRow()...
Definition: Revision.php:1231
const CONTENT_MODEL_WIKITEXT
Definition: Defines.php:277
static element($element, $attribs=null, $contents= '', $allowShortTag=true)
Format an XML element with given attributes and, optionally, text content.
Definition: Xml.php:39
$IP
Definition: WebStart.php:58
static getForModelID($modelId)
Returns the ContentHandler singleton for the given model ID.
div flags Integer display flags(NO_ACTION_LINK, NO_EXTRA_USER_LINKS) 'LogException'returning false will NOT prevent logging $e
Definition: hooks.txt:1932
We use the convention $dbr for read and $dbw for write to help you keep track of whether the database object is a the world will explode Or to be a subsequent write query which succeeded on the master may fail when replicated to the slave due to a unique key collision Replication on the slave will stop and it may take hours to repair the database and get it back online Setting read_only in my cnf on the slave will avoid this but given the dire we prefer to have as many checks as possible We provide a but the wrapper functions like please read the documentation for except in special pages derived from QueryPage It s a common pitfall for new developers to submit code containing SQL queries which examine huge numbers of rows Remember that COUNT * is(N), counting rows in atable is like counting beans in a bucket.------------------------------------------------------------------------Replication------------------------------------------------------------------------The largest installation of MediaWiki, Wikimedia, uses a large set ofslave MySQL servers replicating writes made to a master MySQL server.Itis important to understand the issues associated with this setup if youwant to write code destined for Wikipedia.It's often the case that the best algorithm to use for a given taskdepends on whether or not replication is in use.Due to our unabashedWikipedia-centrism, we often just use the replication-friendly version, but if you like, you can use wfGetLB() ->getServerCount() > 1 tocheck to see if replication is in use.===Lag===Lag primarily occurs when large write queries are sent to the master.Writes on the master are executed in parallel, but they are executed inserial when they are replicated to the slaves.The master writes thequery to the binlog when the transaction is committed.The slaves pollthe binlog and start executing the query as soon as it appears.They canservice reads while they are performing a write query, but will not readanything more from the binlog and thus will perform no more writes.Thismeans that if the write query runs for a long time, the slaves will lagbehind the master for the time it takes for the write query to complete.Lag can be exacerbated by high read load.MediaWiki's load balancer willstop sending reads to a slave when it is lagged by more than 30 seconds.If the load ratios are set incorrectly, or if there is too much loadgenerally, this may lead to a slave permanently hovering around 30seconds lag.If all slaves are lagged by more than 30 seconds, MediaWiki will stopwriting to the database.All edits and other write operations will berefused, with an error returned to the user.This gives the slaves achance to catch up.Before we had this mechanism, the slaves wouldregularly lag by several minutes, making review of recent editsdifficult.In addition to this, MediaWiki attempts to ensure that the user seesevents occurring on the wiki in chronological order.A few seconds of lagcan be tolerated, as long as the user sees a consistent picture fromsubsequent requests.This is done by saving the master binlog positionin the session, and then at the start of each request, waiting for theslave to catch up to that position before doing any reads from it.Ifthis wait times out, reads are allowed anyway, but the request isconsidered to be in"lagged slave mode".Lagged slave mode can bechecked by calling wfGetLB() ->getLaggedSlaveMode().The onlypractical consequence at present is a warning displayed in the pagefooter.===Lag avoidance===To avoid excessive lag, queries which write large numbers of rows shouldbe split up, generally to write one row at a time.Multi-row INSERT...SELECT queries are the worst offenders should be avoided altogether.Instead do the select first and then the insert.===Working with lag===Despite our best efforts, it's not practical to guarantee a low-lagenvironment.Lag will usually be less than one second, but mayoccasionally be up to 30 seconds.For scalability, it's very importantto keep load on the master low, so simply sending all your queries tothe master is not the answer.So when you have a genuine need forup-to-date data, the following approach is advised:1) Do a quick query to the master for a sequence number or timestamp 2) Run the full query on the slave and check if it matches the data you gotfrom the master 3) If it doesn't, run the full query on the masterTo avoid swamping the master every time the slaves lag, use of thisapproach should be kept to a minimum.In most cases you should just readfrom the slave and let the user deal with the delay.------------------------------------------------------------------------Lock contention------------------------------------------------------------------------Due to the high write rate on Wikipedia(and some other wikis), MediaWiki developers need to be very careful to structure their writesto avoid long-lasting locks.By default, MediaWiki opens a transactionat the first query, and commits it before the output is sent.Locks willbe held from the time when the query is done until the commit.So youcan reduce lock time by doing as much processing as possible before youdo your write queries.Often this approach is not good enough, and it becomes necessary toenclose small groups of queries in their own transaction.Use thefollowing syntax:$dbw=wfGetDB(DB_MASTER
getTextDb($id)
May throw a database error if, say, the server dies during query.
hasOption($name)
Checks to see if a particular param exists.
require_once RUN_MAINTENANCE_IF_MAIN
Definition: maintenance.txt:50
showReport()
Overridden to include prefetch ratio if enabled.
when a variable name is used in a it is silently declared as a new local masking the global
Definition: design.txt:93
report($final=false)
Definition: backup.inc:368
magic word & $parser
Definition: hooks.txt:2321
exportTransform($text, $model, $format=null)
Applies applicable export transformations to $text.
wfDebug($text, $dest= 'all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for and distribution as defined by Sections through of this document Licensor shall mean the copyright owner or entity authorized by the copyright owner that is granting the License Legal Entity shall mean the union of the acting entity and all other entities that control are controlled by or are under common control with that entity For the purposes of this definition control direct or to cause the direction or management of such whether by contract or including but not limited to software source documentation and configuration files Object form shall mean any form resulting from mechanical transformation or translation of a Source including but not limited to compiled object generated and conversions to other media types Work shall mean the work of whether in Source or Object made available under the as indicated by a copyright notice that is included in or attached to the whether in Source or Object that is based or other modifications as a an original work of authorship For the purposes of this Derivative Works shall not include works that remain separable from
if($line===false) $args
Definition: cdb.php:64
The ContentHandler facility adds support for arbitrary content types on wiki instead of relying on wikitext for everything It was introduced in MediaWiki Each kind of and so on Built in content types are
An extension or a local will often add custom code to the function with or without a global variable For someone wanting email notification when an article is shown may add
Definition: hooks.txt:51
$maintClass
wfTimestamp($outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
addOption($name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
dump($history, $text=WikiExporter::TEXT)
Readahead helper for making large MediaWiki data dumps; reads in a previous XML dump to sequentially ...
bool resource $spawnProc
$maxConsecutiveFailedTextRetrievals
null means default in associative array with keys and values unescaped Should be merged with default with a value of false meaning to suppress the attribute in associative array with keys and values unescaped noclasses just before the function returns a value If you return an< a > element with HTML attributes $attribs and contents $html will be returned If you return $ret will be returned and may include noclasses after processing after in associative array form externallinks including delete and has completed for all link tables whether this was an auto creation default is conds Array Extra conditions for the No matching items in log is displayed if loglist is empty msgKey Array If you want a nice box with a set this to the key of the message First element is the message additional optional elements are parameters for the key that are processed with wfMessage() -> params() ->parseAsBlock()-offset Set to overwrite offset parameter in $wgRequest set to ''to unsetoffset-wrap String Wrap the message in html(usually something like"&lt
endElement($parser, $name)
const DB_SLAVE
Definition: Defines.php:46
$buffer
design txt This is a brief overview of the new design More thorough and up to date information is available on the documentation wiki at etc Handles the details of getting and saving to the user table of the and dealing with sessions and cookies OutputPage Encapsulates the entire HTML page that will be sent in response to any server request It is used by calling its functions to add text
Definition: design.txt:12
wfWikiID()
Get an ASCII string identifying this wiki This is used as a prefix in memcached keys.
addDescription($text)
Set the description text.
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
const TS_DB
MySQL DATETIME (YYYY-MM-DD HH:MM:SS)
getOption($name, $default=null)
Get an option, or return the default.
output($out, $channel=null)
Throw some output to the user.
clearOpenElement($style)
DatabaseBase $db
loadWithArgv($argv)
Load params and arguments from a given array of command-line arguments.
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
presenting them properly to the user as errors is done by the caller return true use this to change the list i e etc next in line in page history
Definition: hooks.txt:1584
wfGetLBFactory()
Get the load balancer factory object.
startElement($parser, $name, $attribs)
bool resource $spawnRead
this class mediates it Skin Encapsulates a look and feel for the wiki All of the functions that render HTML and make choices about how to render it are here and are called from various other places when and is meant to be subclassed with other skins that may override some of its functions The User object contains a reference to a and so rather than having a global skin object we just rely on the global User and get the skin with $wgUser and also has some character encoding functions and other locale stuff The current user interface language is instantiated as and the local content language as $wgContLang
Definition: design.txt:56
$count
We ve cleaned up the code here by removing clumps of infrequently used code and moving them off somewhere else It s much easier for someone working with this code to see what s _really_ going on
Definition: hooks.txt:86
progress($string)
Definition: backup.inc:413
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that probably a stub it is not rendered in wiki pages or galleries in category pages allow injecting custom HTML after the section Any uses of the hook need to handle escaping see BaseTemplate::getToolbox and BaseTemplate::makeListItem for details on the format of individual items inside of this array or by returning and letting standard HTTP rendering take place modifiable or by returning false and taking over the output modifiable modifiable after all normalizations have been except for the $wgMaxImageArea check set to true or false to override the $wgMaxImageArea check result gives extension the possibility to transform it themselves $handler
Definition: hooks.txt:762
initProgress($history=WikiExporter::FULL)
bool resource $spawnErr
characterData($parser, $data)
__construct($args=null)
rotateDb()
Drop the database connection $this->db and try to get a new one.
do that in ParserLimitReportFormat instead use this to modify the parameters of the image and a DIV can begin in one section and end in another Make sure your code can handle that case gracefully See the EditSectionClearerLink extension for an example zero but section is usually empty its values are the globals values before the output is cached one of or reset my talk page
Definition: hooks.txt:2338
null means default in associative array with keys and values unescaped Should be merged with default with a value of false meaning to suppress the attribute in associative array with keys and values unescaped noclasses just before the function returns a value If you return an< a > element with HTML attributes $attribs and contents $html will be returned If you return $ret will be returned and may include noclasses after processing & $attribs
Definition: hooks.txt:1798
DatabaseBase null $forcedDb
The dependency-injected database to use.
Definition: backup.inc:66
getText($id, $model=null, $format=null)
Tries to get the revision text for a revision id.
Allows to change the fields on the form that will be generated $name
Definition: hooks.txt:310