MediaWiki  1.28.0
dumpTextPass.php
Go to the documentation of this file.
1 <?php
27 require_once __DIR__ . '/backup.inc';
28 require_once __DIR__ . '/../includes/export/WikiExporter.php';
29 
34  public $prefetch = null;
35 
36  // when we spend more than maxTimeAllowed seconds on this run, we continue
37  // processing until we write out the next complete page, then save output file(s),
38  // rename it/them and open new one(s)
39  public $maxTimeAllowed = 0; // 0 = no limit
40 
41  protected $input = "php://stdin";
43  protected $fetchCount = 0;
44  protected $prefetchCount = 0;
45  protected $prefetchCountLast = 0;
46  protected $fetchCountLast = 0;
47 
48  protected $maxFailures = 5;
50  protected $failureTimeout = 5; // Seconds to sleep after db failure
51 
52  protected $bufferSize = 524288; // In bytes. Maximum size to read from the stub in on go.
53 
54  protected $php = "php";
55  protected $spawn = false;
56 
60  protected $spawnProc = false;
61 
65  protected $spawnWrite = false;
66 
70  protected $spawnRead = false;
71 
75  protected $spawnErr = false;
76 
80  protected $xmlwriterobj = false;
81 
82  protected $timeExceeded = false;
83  protected $firstPageWritten = false;
84  protected $lastPageWritten = false;
85  protected $checkpointJustWritten = false;
86  protected $checkpointFiles = [];
87 
91  protected $db;
92 
96  function __construct( $args = null ) {
97  parent::__construct();
98 
99  $this->addDescription( <<<TEXT
100 This script postprocesses XML dumps from dumpBackup.php to add
101 page text which was stubbed out (using --stub).
102 
103 XML input is accepted on stdin.
104 XML output is sent to stdout; progress reports are sent to stderr.
105 TEXT
106  );
107  $this->stderr = fopen( "php://stderr", "wt" );
108 
109  $this->addOption( 'stub', 'To load a compressed stub dump instead of stdin. ' .
110  'Specify as --stub=<type>:<file>.', false, true );
111  $this->addOption( 'prefetch', 'Use a prior dump file as a text source, to savepressure on the ' .
112  'database. (Requires the XMLReader extension). Specify as --prefetch=<type>:<file>',
113  false, true );
114  $this->addOption( 'maxtime', 'Write out checkpoint file after this many minutes (writing' .
115  'out complete page, closing xml file properly, and opening new one' .
116  'with header). This option requires the checkpointfile option.', false, true );
117  $this->addOption( 'checkpointfile', 'Use this string for checkpoint filenames,substituting ' .
118  'first pageid written for the first %s (required) and the last pageid written for the ' .
119  'second %s if it exists.', false, true, false, true ); // This can be specified multiple times
120  $this->addOption( 'quiet', 'Don\'t dump status reports to stderr.' );
121  $this->addOption( 'current', 'Base ETA on number of pages in database instead of all revisions' );
122  $this->addOption( 'spawn', 'Spawn a subprocess for loading text records' );
123  $this->addOption( 'buffersize', 'Buffer size in bytes to use for reading the stub. ' .
124  '(Default: 512KB, Minimum: 4KB)', false, true );
125 
126  if ( $args ) {
127  $this->loadWithArgv( $args );
128  $this->processOptions();
129  }
130  }
131 
132  function execute() {
133  $this->processOptions();
134  $this->dump( true );
135  }
136 
137  function processOptions() {
138  global $IP;
139 
140  parent::processOptions();
141 
142  if ( $this->hasOption( 'buffersize' ) ) {
143  $this->bufferSize = max( intval( $this->getOption( 'buffersize' ) ), 4 * 1024 );
144  }
145 
146  if ( $this->hasOption( 'prefetch' ) ) {
147  require_once "$IP/maintenance/backupPrefetch.inc";
148  $url = $this->processFileOpt( $this->getOption( 'prefetch' ) );
149  $this->prefetch = new BaseDump( $url );
150  }
151 
152  if ( $this->hasOption( 'stub' ) ) {
153  $this->input = $this->processFileOpt( $this->getOption( 'stub' ) );
154  }
155 
156  if ( $this->hasOption( 'maxtime' ) ) {
157  $this->maxTimeAllowed = intval( $this->getOption( 'maxtime' ) ) * 60;
158  }
159 
160  if ( $this->hasOption( 'checkpointfile' ) ) {
161  $this->checkpointFiles = $this->getOption( 'checkpointfile' );
162  }
163 
164  if ( $this->hasOption( 'current' ) ) {
166  }
167 
168  if ( $this->hasOption( 'full' ) ) {
169  $this->history = WikiExporter::FULL;
170  }
171 
172  if ( $this->hasOption( 'spawn' ) ) {
173  $this->spawn = true;
174  $val = $this->getOption( 'spawn' );
175  if ( $val !== 1 ) {
176  $this->php = $val;
177  }
178  }
179  }
180 
192  function rotateDb() {
193  // Cleaning up old connections
194  if ( isset( $this->lb ) ) {
195  $this->lb->closeAll();
196  unset( $this->lb );
197  }
198 
199  if ( $this->forcedDb !== null ) {
200  $this->db = $this->forcedDb;
201 
202  return;
203  }
204 
205  if ( isset( $this->db ) && $this->db->isOpen() ) {
206  throw new MWException( 'DB is set and has not been closed by the Load Balancer' );
207  }
208 
209  unset( $this->db );
210 
211  // Trying to set up new connection.
212  // We do /not/ retry upon failure, but delegate to encapsulating logic, to avoid
213  // individually retrying at different layers of code.
214 
215  // 1. The LoadBalancer.
216  try {
217  $this->lb = wfGetLBFactory()->newMainLB();
218  } catch ( Exception $e ) {
219  throw new MWException( __METHOD__
220  . " rotating DB failed to obtain new load balancer (" . $e->getMessage() . ")" );
221  }
222 
223  // 2. The Connection, through the load balancer.
224  try {
225  $this->db = $this->lb->getConnection( DB_REPLICA, 'dump' );
226  } catch ( Exception $e ) {
227  throw new MWException( __METHOD__
228  . " rotating DB failed to obtain new database (" . $e->getMessage() . ")" );
229  }
230  }
231 
233  parent::initProgress();
234  $this->timeOfCheckpoint = $this->startTime;
235  }
236 
237  function dump( $history, $text = WikiExporter::TEXT ) {
238  // Notice messages will foul up your XML output even if they're
239  // relatively harmless.
240  if ( ini_get( 'display_errors' ) ) {
241  ini_set( 'display_errors', 'stderr' );
242  }
243 
244  $this->initProgress( $this->history );
245 
246  // We are trying to get an initial database connection to avoid that the
247  // first try of this request's first call to getText fails. However, if
248  // obtaining a good DB connection fails it's not a serious issue, as
249  // getText does retry upon failure and can start without having a working
250  // DB connection.
251  try {
252  $this->rotateDb();
253  } catch ( Exception $e ) {
254  // We do not even count this as failure. Just let eventual
255  // watchdogs know.
256  $this->progress( "Getting initial DB connection failed (" .
257  $e->getMessage() . ")" );
258  }
259 
260  $this->egress = new ExportProgressFilter( $this->sink, $this );
261 
262  // it would be nice to do it in the constructor, oh well. need egress set
263  $this->finalOptionCheck();
264 
265  // we only want this so we know how to close a stream :-P
266  $this->xmlwriterobj = new XmlDumpWriter();
267 
268  $input = fopen( $this->input, "rt" );
269  $this->readDump( $input );
270 
271  if ( $this->spawnProc ) {
272  $this->closeSpawn();
273  }
274 
275  $this->report( true );
276  }
277 
278  function processFileOpt( $opt ) {
279  $split = explode( ':', $opt, 2 );
280  $val = $split[0];
281  $param = '';
282  if ( count( $split ) === 2 ) {
283  $param = $split[1];
284  }
285  $fileURIs = explode( ';', $param );
286  foreach ( $fileURIs as $URI ) {
287  switch ( $val ) {
288  case "file":
289  $newURI = $URI;
290  break;
291  case "gzip":
292  $newURI = "compress.zlib://$URI";
293  break;
294  case "bzip2":
295  $newURI = "compress.bzip2://$URI";
296  break;
297  case "7zip":
298  $newURI = "mediawiki.compress.7z://$URI";
299  break;
300  default:
301  $newURI = $URI;
302  }
303  $newFileURIs[] = $newURI;
304  }
305  $val = implode( ';', $newFileURIs );
306 
307  return $val;
308  }
309 
313  function showReport() {
314  if ( !$this->prefetch ) {
315  parent::showReport();
316 
317  return;
318  }
319 
320  if ( $this->reporting ) {
321  $now = wfTimestamp( TS_DB );
322  $nowts = microtime( true );
323  $deltaAll = $nowts - $this->startTime;
324  $deltaPart = $nowts - $this->lastTime;
325  $this->pageCountPart = $this->pageCount - $this->pageCountLast;
326  $this->revCountPart = $this->revCount - $this->revCountLast;
327 
328  if ( $deltaAll ) {
329  $portion = $this->revCount / $this->maxCount;
330  $eta = $this->startTime + $deltaAll / $portion;
331  $etats = wfTimestamp( TS_DB, intval( $eta ) );
332  if ( $this->fetchCount ) {
333  $fetchRate = 100.0 * $this->prefetchCount / $this->fetchCount;
334  } else {
335  $fetchRate = '-';
336  }
337  $pageRate = $this->pageCount / $deltaAll;
338  $revRate = $this->revCount / $deltaAll;
339  } else {
340  $pageRate = '-';
341  $revRate = '-';
342  $etats = '-';
343  $fetchRate = '-';
344  }
345  if ( $deltaPart ) {
346  if ( $this->fetchCountLast ) {
347  $fetchRatePart = 100.0 * $this->prefetchCountLast / $this->fetchCountLast;
348  } else {
349  $fetchRatePart = '-';
350  }
351  $pageRatePart = $this->pageCountPart / $deltaPart;
352  $revRatePart = $this->revCountPart / $deltaPart;
353  } else {
354  $fetchRatePart = '-';
355  $pageRatePart = '-';
356  $revRatePart = '-';
357  }
358  $this->progress( sprintf(
359  "%s: %s (ID %d) %d pages (%0.1f|%0.1f/sec all|curr), "
360  . "%d revs (%0.1f|%0.1f/sec all|curr), %0.1f%%|%0.1f%% "
361  . "prefetched (all|curr), ETA %s [max %d]",
362  $now, wfWikiID(), $this->ID, $this->pageCount, $pageRate,
363  $pageRatePart, $this->revCount, $revRate, $revRatePart,
364  $fetchRate, $fetchRatePart, $etats, $this->maxCount
365  ) );
366  $this->lastTime = $nowts;
367  $this->revCountLast = $this->revCount;
368  $this->prefetchCountLast = $this->prefetchCount;
369  $this->fetchCountLast = $this->fetchCount;
370  }
371  }
372 
373  function setTimeExceeded() {
374  $this->timeExceeded = true;
375  }
376 
377  function checkIfTimeExceeded() {
378  if ( $this->maxTimeAllowed
379  && ( $this->lastTime - $this->timeOfCheckpoint > $this->maxTimeAllowed )
380  ) {
381  return true;
382  }
383 
384  return false;
385  }
386 
387  function finalOptionCheck() {
388  if ( ( $this->checkpointFiles && !$this->maxTimeAllowed )
389  || ( $this->maxTimeAllowed && !$this->checkpointFiles )
390  ) {
391  throw new MWException( "Options checkpointfile and maxtime must be specified together.\n" );
392  }
393  foreach ( $this->checkpointFiles as $checkpointFile ) {
394  $count = substr_count( $checkpointFile, "%s" );
395  if ( $count != 2 ) {
396  throw new MWException( "Option checkpointfile must contain two '%s' "
397  . "for substitution of first and last pageids, count is $count instead, "
398  . "file is $checkpointFile.\n" );
399  }
400  }
401 
402  if ( $this->checkpointFiles ) {
403  $filenameList = (array)$this->egress->getFilenames();
404  if ( count( $filenameList ) != count( $this->checkpointFiles ) ) {
405  throw new MWException( "One checkpointfile must be specified "
406  . "for each output option, if maxtime is used.\n" );
407  }
408  }
409  }
410 
416  function readDump( $input ) {
417  $this->buffer = "";
418  $this->openElement = false;
419  $this->atStart = true;
420  $this->state = "";
421  $this->lastName = "";
422  $this->thisPage = 0;
423  $this->thisRev = 0;
424  $this->thisRevModel = null;
425  $this->thisRevFormat = null;
426 
427  $parser = xml_parser_create( "UTF-8" );
428  xml_parser_set_option( $parser, XML_OPTION_CASE_FOLDING, false );
429 
430  xml_set_element_handler(
431  $parser,
432  [ $this, 'startElement' ],
433  [ $this, 'endElement' ]
434  );
435  xml_set_character_data_handler( $parser, [ $this, 'characterData' ] );
436 
437  $offset = 0; // for context extraction on error reporting
438  do {
439  if ( $this->checkIfTimeExceeded() ) {
440  $this->setTimeExceeded();
441  }
442  $chunk = fread( $input, $this->bufferSize );
443  if ( !xml_parse( $parser, $chunk, feof( $input ) ) ) {
444  wfDebug( "TextDumpPass::readDump encountered XML parsing error\n" );
445 
446  $byte = xml_get_current_byte_index( $parser );
447  $msg = wfMessage( 'xml-error-string',
448  'XML import parse failure',
449  xml_get_current_line_number( $parser ),
450  xml_get_current_column_number( $parser ),
451  $byte . ( is_null( $chunk ) ? null : ( '; "' . substr( $chunk, $byte - $offset, 16 ) . '"' ) ),
452  xml_error_string( xml_get_error_code( $parser ) ) )->escaped();
453 
454  xml_parser_free( $parser );
455 
456  throw new MWException( $msg );
457  }
458  $offset += strlen( $chunk );
459  } while ( $chunk !== false && !feof( $input ) );
460  if ( $this->maxTimeAllowed ) {
461  $filenameList = (array)$this->egress->getFilenames();
462  // we wrote some stuff after last checkpoint that needs renamed
463  if ( file_exists( $filenameList[0] ) ) {
464  $newFilenames = [];
465  # we might have just written the header and footer and had no
466  # pages or revisions written... perhaps they were all deleted
467  # there's no pageID 0 so we use that. the caller is responsible
468  # for deciding what to do with a file containing only the
469  # siteinfo information and the mw tags.
470  if ( !$this->firstPageWritten ) {
471  $firstPageID = str_pad( 0, 9, "0", STR_PAD_LEFT );
472  $lastPageID = str_pad( 0, 9, "0", STR_PAD_LEFT );
473  } else {
474  $firstPageID = str_pad( $this->firstPageWritten, 9, "0", STR_PAD_LEFT );
475  $lastPageID = str_pad( $this->lastPageWritten, 9, "0", STR_PAD_LEFT );
476  }
477 
478  $filenameCount = count( $filenameList );
479  for ( $i = 0; $i < $filenameCount; $i++ ) {
480  $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID );
481  $fileinfo = pathinfo( $filenameList[$i] );
482  $newFilenames[] = $fileinfo['dirname'] . '/' . $checkpointNameFilledIn;
483  }
484  $this->egress->closeAndRename( $newFilenames );
485  }
486  }
487  xml_parser_free( $parser );
488 
489  return true;
490  }
491 
501  private function exportTransform( $text, $model, $format = null ) {
502  try {
504  $text = $handler->exportTransform( $text, $format );
505  }
506  catch ( MWException $ex ) {
507  $this->progress(
508  "Unable to apply export transformation for content model '$model': " .
509  $ex->getMessage()
510  );
511  }
512 
513  return $text;
514  }
515 
536  function getText( $id, $model = null, $format = null ) {
537  global $wgContentHandlerUseDB;
538 
539  $prefetchNotTried = true; // Whether or not we already tried to get the text via prefetch.
540  $text = false; // The candidate for a good text. false if no proper value.
541  $failures = 0; // The number of times, this invocation of getText already failed.
542 
543  // The number of times getText failed without yielding a good text in between.
544  static $consecutiveFailedTextRetrievals = 0;
545 
546  $this->fetchCount++;
547 
548  // To allow to simply return on success and do not have to worry about book keeping,
549  // we assume, this fetch works (possible after some retries). Nevertheless, we koop
550  // the old value, so we can restore it, if problems occur (See after the while loop).
551  $oldConsecutiveFailedTextRetrievals = $consecutiveFailedTextRetrievals;
552  $consecutiveFailedTextRetrievals = 0;
553 
554  if ( $model === null && $wgContentHandlerUseDB ) {
555  $row = $this->db->selectRow(
556  'revision',
557  [ 'rev_content_model', 'rev_content_format' ],
558  [ 'rev_id' => $this->thisRev ],
559  __METHOD__
560  );
561 
562  if ( $row ) {
563  $model = $row->rev_content_model;
564  $format = $row->rev_content_format;
565  }
566  }
567 
568  if ( $model === null || $model === '' ) {
569  $model = false;
570  }
571 
572  while ( $failures < $this->maxFailures ) {
573 
574  // As soon as we found a good text for the $id, we will return immediately.
575  // Hence, if we make it past the try catch block, we know that we did not
576  // find a good text.
577 
578  try {
579  // Step 1: Get some text (or reuse from previous iteratuon if checking
580  // for plausibility failed)
581 
582  // Trying to get prefetch, if it has not been tried before
583  if ( $text === false && isset( $this->prefetch ) && $prefetchNotTried ) {
584  $prefetchNotTried = false;
585  $tryIsPrefetch = true;
586  $text = $this->prefetch->prefetch( intval( $this->thisPage ),
587  intval( $this->thisRev ) );
588 
589  if ( $text === null ) {
590  $text = false;
591  }
592 
593  if ( is_string( $text ) && $model !== false ) {
594  // Apply export transformation to text coming from an old dump.
595  // The purpose of this transformation is to convert up from legacy
596  // formats, which may still be used in the older dump that is used
597  // for pre-fetching. Applying the transformation again should not
598  // interfere with content that is already in the correct form.
599  $text = $this->exportTransform( $text, $model, $format );
600  }
601  }
602 
603  if ( $text === false ) {
604  // Fallback to asking the database
605  $tryIsPrefetch = false;
606  if ( $this->spawn ) {
607  $text = $this->getTextSpawned( $id );
608  } else {
609  $text = $this->getTextDb( $id );
610  }
611 
612  if ( $text !== false && $model !== false ) {
613  // Apply export transformation to text coming from the database.
614  // Prefetched text should already have transformations applied.
615  $text = $this->exportTransform( $text, $model, $format );
616  }
617 
618  // No more checks for texts from DB for now.
619  // If we received something that is not false,
620  // We treat it as good text, regardless of whether it actually is or is not
621  if ( $text !== false ) {
622  return $text;
623  }
624  }
625 
626  if ( $text === false ) {
627  throw new MWException( "Generic error while obtaining text for id " . $id );
628  }
629 
630  // We received a good candidate for the text of $id via some method
631 
632  // Step 2: Checking for plausibility and return the text if it is
633  // plausible
634  $revID = intval( $this->thisRev );
635  if ( !isset( $this->db ) ) {
636  throw new MWException( "No database available" );
637  }
638 
639  if ( $model !== CONTENT_MODEL_WIKITEXT ) {
640  $revLength = strlen( $text );
641  } else {
642  $revLength = $this->db->selectField( 'revision', 'rev_len', [ 'rev_id' => $revID ] );
643  }
644 
645  if ( strlen( $text ) == $revLength ) {
646  if ( $tryIsPrefetch ) {
647  $this->prefetchCount++;
648  }
649 
650  return $text;
651  }
652 
653  $text = false;
654  throw new MWException( "Received text is unplausible for id " . $id );
655  } catch ( Exception $e ) {
656  $msg = "getting/checking text " . $id . " failed (" . $e->getMessage() . ")";
657  if ( $failures + 1 < $this->maxFailures ) {
658  $msg .= " (Will retry " . ( $this->maxFailures - $failures - 1 ) . " more times)";
659  }
660  $this->progress( $msg );
661  }
662 
663  // Something went wrong; we did not a text that was plausible :(
664  $failures++;
665 
666  // A failure in a prefetch hit does not warrant resetting db connection etc.
667  if ( !$tryIsPrefetch ) {
668  // After backing off for some time, we try to reboot the whole process as
669  // much as possible to not carry over failures from one part to the other
670  // parts
671  sleep( $this->failureTimeout );
672  try {
673  $this->rotateDb();
674  if ( $this->spawn ) {
675  $this->closeSpawn();
676  $this->openSpawn();
677  }
678  } catch ( Exception $e ) {
679  $this->progress( "Rebooting getText infrastructure failed (" . $e->getMessage() . ")" .
680  " Trying to continue anyways" );
681  }
682  }
683  }
684 
685  // Retirieving a good text for $id failed (at least) maxFailures times.
686  // We abort for this $id.
687 
688  // Restoring the consecutive failures, and maybe aborting, if the dump
689  // is too broken.
690  $consecutiveFailedTextRetrievals = $oldConsecutiveFailedTextRetrievals + 1;
691  if ( $consecutiveFailedTextRetrievals > $this->maxConsecutiveFailedTextRetrievals ) {
692  throw new MWException( "Graceful storage failure" );
693  }
694 
695  return "";
696  }
697 
704  private function getTextDb( $id ) {
706  if ( !isset( $this->db ) ) {
707  throw new MWException( __METHOD__ . "No database available" );
708  }
709  $row = $this->db->selectRow( 'text',
710  [ 'old_text', 'old_flags' ],
711  [ 'old_id' => $id ],
712  __METHOD__ );
713  $text = Revision::getRevisionText( $row );
714  if ( $text === false ) {
715  return false;
716  }
717  $stripped = str_replace( "\r", "", $text );
718  $normalized = $wgContLang->normalize( $stripped );
719 
720  return $normalized;
721  }
722 
723  private function getTextSpawned( $id ) {
724  MediaWiki\suppressWarnings();
725  if ( !$this->spawnProc ) {
726  // First time?
727  $this->openSpawn();
728  }
729  $text = $this->getTextSpawnedOnce( $id );
730  MediaWiki\restoreWarnings();
731 
732  return $text;
733  }
734 
735  function openSpawn() {
736  global $IP;
737 
738  if ( file_exists( "$IP/../multiversion/MWScript.php" ) ) {
739  $cmd = implode( " ",
740  array_map( 'wfEscapeShellArg',
741  [
742  $this->php,
743  "$IP/../multiversion/MWScript.php",
744  "fetchText.php",
745  '--wiki', wfWikiID() ] ) );
746  } else {
747  $cmd = implode( " ",
748  array_map( 'wfEscapeShellArg',
749  [
750  $this->php,
751  "$IP/maintenance/fetchText.php",
752  '--wiki', wfWikiID() ] ) );
753  }
754  $spec = [
755  0 => [ "pipe", "r" ],
756  1 => [ "pipe", "w" ],
757  2 => [ "file", "/dev/null", "a" ] ];
758  $pipes = [];
759 
760  $this->progress( "Spawning database subprocess: $cmd" );
761  $this->spawnProc = proc_open( $cmd, $spec, $pipes );
762  if ( !$this->spawnProc ) {
763  $this->progress( "Subprocess spawn failed." );
764 
765  return false;
766  }
767  list(
768  $this->spawnWrite, // -> stdin
769  $this->spawnRead, // <- stdout
770  ) = $pipes;
771 
772  return true;
773  }
774 
775  private function closeSpawn() {
776  MediaWiki\suppressWarnings();
777  if ( $this->spawnRead ) {
778  fclose( $this->spawnRead );
779  }
780  $this->spawnRead = false;
781  if ( $this->spawnWrite ) {
782  fclose( $this->spawnWrite );
783  }
784  $this->spawnWrite = false;
785  if ( $this->spawnErr ) {
786  fclose( $this->spawnErr );
787  }
788  $this->spawnErr = false;
789  if ( $this->spawnProc ) {
790  pclose( $this->spawnProc );
791  }
792  $this->spawnProc = false;
793  MediaWiki\restoreWarnings();
794  }
795 
796  private function getTextSpawnedOnce( $id ) {
798 
799  $ok = fwrite( $this->spawnWrite, "$id\n" );
800  // $this->progress( ">> $id" );
801  if ( !$ok ) {
802  return false;
803  }
804 
805  $ok = fflush( $this->spawnWrite );
806  // $this->progress( ">> [flush]" );
807  if ( !$ok ) {
808  return false;
809  }
810 
811  // check that the text id they are sending is the one we asked for
812  // this avoids out of sync revision text errors we have encountered in the past
813  $newId = fgets( $this->spawnRead );
814  if ( $newId === false ) {
815  return false;
816  }
817  if ( $id != intval( $newId ) ) {
818  return false;
819  }
820 
821  $len = fgets( $this->spawnRead );
822  // $this->progress( "<< " . trim( $len ) );
823  if ( $len === false ) {
824  return false;
825  }
826 
827  $nbytes = intval( $len );
828  // actual error, not zero-length text
829  if ( $nbytes < 0 ) {
830  return false;
831  }
832 
833  $text = "";
834 
835  // Subprocess may not send everything at once, we have to loop.
836  while ( $nbytes > strlen( $text ) ) {
837  $buffer = fread( $this->spawnRead, $nbytes - strlen( $text ) );
838  if ( $buffer === false ) {
839  break;
840  }
841  $text .= $buffer;
842  }
843 
844  $gotbytes = strlen( $text );
845  if ( $gotbytes != $nbytes ) {
846  $this->progress( "Expected $nbytes bytes from database subprocess, got $gotbytes " );
847 
848  return false;
849  }
850 
851  // Do normalization in the dump thread...
852  $stripped = str_replace( "\r", "", $text );
853  $normalized = $wgContLang->normalize( $stripped );
854 
855  return $normalized;
856  }
857 
859  $this->checkpointJustWritten = false;
860 
861  $this->clearOpenElement( null );
862  $this->lastName = $name;
863 
864  if ( $name == 'revision' ) {
865  $this->state = $name;
866  $this->egress->writeOpenPage( null, $this->buffer );
867  $this->buffer = "";
868  } elseif ( $name == 'page' ) {
869  $this->state = $name;
870  if ( $this->atStart ) {
871  $this->egress->writeOpenStream( $this->buffer );
872  $this->buffer = "";
873  $this->atStart = false;
874  }
875  }
876 
877  if ( $name == "text" && isset( $attribs['id'] ) ) {
878  $id = $attribs['id'];
879  $model = trim( $this->thisRevModel );
880  $format = trim( $this->thisRevFormat );
881 
882  $model = $model === '' ? null : $model;
883  $format = $format === '' ? null : $format;
884 
885  $text = $this->getText( $id, $model, $format );
886  $this->openElement = [ $name, [ 'xml:space' => 'preserve' ] ];
887  if ( strlen( $text ) > 0 ) {
888  $this->characterData( $parser, $text );
889  }
890  } else {
891  $this->openElement = [ $name, $attribs ];
892  }
893  }
894 
895  function endElement( $parser, $name ) {
896  $this->checkpointJustWritten = false;
897 
898  if ( $this->openElement ) {
899  $this->clearOpenElement( "" );
900  } else {
901  $this->buffer .= "</$name>";
902  }
903 
904  if ( $name == 'revision' ) {
905  $this->egress->writeRevision( null, $this->buffer );
906  $this->buffer = "";
907  $this->thisRev = "";
908  $this->thisRevModel = null;
909  $this->thisRevFormat = null;
910  } elseif ( $name == 'page' ) {
911  if ( !$this->firstPageWritten ) {
912  $this->firstPageWritten = trim( $this->thisPage );
913  }
914  $this->lastPageWritten = trim( $this->thisPage );
915  if ( $this->timeExceeded ) {
916  $this->egress->writeClosePage( $this->buffer );
917  // nasty hack, we can't just write the chardata after the
918  // page tag, it will include leading blanks from the next line
919  $this->egress->sink->write( "\n" );
920 
921  $this->buffer = $this->xmlwriterobj->closeStream();
922  $this->egress->writeCloseStream( $this->buffer );
923 
924  $this->buffer = "";
925  $this->thisPage = "";
926  // this could be more than one file if we had more than one output arg
927 
928  $filenameList = (array)$this->egress->getFilenames();
929  $newFilenames = [];
930  $firstPageID = str_pad( $this->firstPageWritten, 9, "0", STR_PAD_LEFT );
931  $lastPageID = str_pad( $this->lastPageWritten, 9, "0", STR_PAD_LEFT );
932  $filenamesCount = count( $filenameList );
933  for ( $i = 0; $i < $filenamesCount; $i++ ) {
934  $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID );
935  $fileinfo = pathinfo( $filenameList[$i] );
936  $newFilenames[] = $fileinfo['dirname'] . '/' . $checkpointNameFilledIn;
937  }
938  $this->egress->closeRenameAndReopen( $newFilenames );
939  $this->buffer = $this->xmlwriterobj->openStream();
940  $this->timeExceeded = false;
941  $this->timeOfCheckpoint = $this->lastTime;
942  $this->firstPageWritten = false;
943  $this->checkpointJustWritten = true;
944  } else {
945  $this->egress->writeClosePage( $this->buffer );
946  $this->buffer = "";
947  $this->thisPage = "";
948  }
949  } elseif ( $name == 'mediawiki' ) {
950  $this->egress->writeCloseStream( $this->buffer );
951  $this->buffer = "";
952  }
953  }
954 
955  function characterData( $parser, $data ) {
956  $this->clearOpenElement( null );
957  if ( $this->lastName == "id" ) {
958  if ( $this->state == "revision" ) {
959  $this->thisRev .= $data;
960  } elseif ( $this->state == "page" ) {
961  $this->thisPage .= $data;
962  }
963  } elseif ( $this->lastName == "model" ) {
964  $this->thisRevModel .= $data;
965  } elseif ( $this->lastName == "format" ) {
966  $this->thisRevFormat .= $data;
967  }
968 
969  // have to skip the newline left over from closepagetag line of
970  // end of checkpoint files. nasty hack!!
971  if ( $this->checkpointJustWritten ) {
972  if ( $data[0] == "\n" ) {
973  $data = substr( $data, 1 );
974  }
975  $this->checkpointJustWritten = false;
976  }
977  $this->buffer .= htmlspecialchars( $data );
978  }
979 
980  function clearOpenElement( $style ) {
981  if ( $this->openElement ) {
982  $this->buffer .= Xml::element( $this->openElement[0], $this->openElement[1], $style );
983  $this->openElement = false;
984  }
985  }
986 }
987 
988 $maintClass = 'TextPassDumper';
989 require_once RUN_MAINTENANCE_IF_MAIN;
bool resource $spawnWrite
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition: deferred.txt:11
the array() calling protocol came about after MediaWiki 1.4rc1.
static getRevisionText($row, $prefix= 'old_', $wiki=false)
Get revision text associated with an old or archive row $row is usually an object from wfFetchRow()...
Definition: Revision.php:1273
const CONTENT_MODEL_WIKITEXT
Definition: Defines.php:239
static element($element, $attribs=null, $contents= '', $allowShortTag=true)
Format an XML element with given attributes and, optionally, text content.
Definition: Xml.php:39
$IP
Definition: WebStart.php:58
static getForModelID($modelId)
Returns the ContentHandler singleton for the given model ID.
div flags Integer display flags(NO_ACTION_LINK, NO_EXTRA_USER_LINKS) 'LogException'returning false will NOT prevent logging $e
Definition: hooks.txt:2102
We use the convention $dbr for read and $dbw for write to help you keep track of whether the database object is a the world will explode Or to be a subsequent write query which succeeded on the master may fail when replicated to the slave due to a unique key collision Replication on the slave will stop and it may take hours to repair the database and get it back online Setting read_only in my cnf on the slave will avoid this but given the dire we prefer to have as many checks as possible We provide a but the wrapper functions like please read the documentation for except in special pages derived from QueryPage It s a common pitfall for new developers to submit code containing SQL queries which examine huge numbers of rows Remember that COUNT * is(N), counting rows in atable is like counting beans in a bucket.------------------------------------------------------------------------Replication------------------------------------------------------------------------The largest installation of MediaWiki, Wikimedia, uses a large set ofslave MySQL servers replicating writes made to a master MySQL server.Itis important to understand the issues associated with this setup if youwant to write code destined for Wikipedia.It's often the case that the best algorithm to use for a given taskdepends on whether or not replication is in use.Due to our unabashedWikipedia-centrism, we often just use the replication-friendly version, but if you like, you can use wfGetLB() ->getServerCount() > 1 tocheck to see if replication is in use.===Lag===Lag primarily occurs when large write queries are sent to the master.Writes on the master are executed in parallel, but they are executed inserial when they are replicated to the slaves.The master writes thequery to the binlog when the transaction is committed.The slaves pollthe binlog and start executing the query as soon as it appears.They canservice reads while they are performing a write query, but will not readanything more from the binlog and thus will perform no more writes.Thismeans that if the write query runs for a long time, the slaves will lagbehind the master for the time it takes for the write query to complete.Lag can be exacerbated by high read load.MediaWiki's load balancer willstop sending reads to a slave when it is lagged by more than 30 seconds.If the load ratios are set incorrectly, or if there is too much loadgenerally, this may lead to a slave permanently hovering around 30seconds lag.If all slaves are lagged by more than 30 seconds, MediaWiki will stopwriting to the database.All edits and other write operations will berefused, with an error returned to the user.This gives the slaves achance to catch up.Before we had this mechanism, the slaves wouldregularly lag by several minutes, making review of recent editsdifficult.In addition to this, MediaWiki attempts to ensure that the user seesevents occurring on the wiki in chronological order.A few seconds of lagcan be tolerated, as long as the user sees a consistent picture fromsubsequent requests.This is done by saving the master binlog positionin the session, and then at the start of each request, waiting for theslave to catch up to that position before doing any reads from it.Ifthis wait times out, reads are allowed anyway, but the request isconsidered to be in"lagged slave mode".Lagged slave mode can bechecked by calling wfGetLB() ->getLaggedSlaveMode().The onlypractical consequence at present is a warning displayed in the pagefooter.===Lag avoidance===To avoid excessive lag, queries which write large numbers of rows shouldbe split up, generally to write one row at a time.Multi-row INSERT...SELECT queries are the worst offenders should be avoided altogether.Instead do the select first and then the insert.===Working with lag===Despite our best efforts, it's not practical to guarantee a low-lagenvironment.Lag will usually be less than one second, but mayoccasionally be up to 30 seconds.For scalability, it's very importantto keep load on the master low, so simply sending all your queries tothe master is not the answer.So when you have a genuine need forup-to-date data, the following approach is advised:1) Do a quick query to the master for a sequence number or timestamp 2) Run the full query on the slave and check if it matches the data you gotfrom the master 3) If it doesn't, run the full query on the masterTo avoid swamping the master every time the slaves lag, use of thisapproach should be kept to a minimum.In most cases you should just readfrom the slave and let the user deal with the delay.------------------------------------------------------------------------Lock contention------------------------------------------------------------------------Due to the high write rate on Wikipedia(and some other wikis), MediaWiki developers need to be very careful to structure their writesto avoid long-lasting locks.By default, MediaWiki opens a transactionat the first query, and commits it before the output is sent.Locks willbe held from the time when the query is done until the commit.So youcan reduce lock time by doing as much processing as possible before youdo your write queries.Often this approach is not good enough, and it becomes necessary toenclose small groups of queries in their own transaction.Use thefollowing syntax:$dbw=wfGetDB(DB_MASTER
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for and distribution as defined by Sections through of this document Licensor shall mean the copyright owner or entity authorized by the copyright owner that is granting the License Legal Entity shall mean the union of the acting entity and all other entities that control are controlled by or are under common control with that entity For the purposes of this definition control direct or to cause the direction or management of such whether by contract or including but not limited to software source documentation and configuration files Object form shall mean any form resulting from mechanical transformation or translation of a Source including but not limited to compiled object generated and conversions to other media types Work shall mean the work of whether in Source or Object made available under the as indicated by a copyright notice that is included in or attached to the whether in Source or Object that is based or other modifications as a an original work of authorship For the purposes of this Derivative Works shall not include works that remain separable from
getTextDb($id)
May throw a database error if, say, the server dies during query.
hasOption($name)
Checks to see if a particular param exists.
require_once RUN_MAINTENANCE_IF_MAIN
Definition: maintenance.txt:50
showReport()
Overridden to include prefetch ratio if enabled.
when a variable name is used in a it is silently declared as a new local masking the global
Definition: design.txt:93
bool XmlDumpWriter $xmlwriterobj
report($final=false)
Definition: backup.inc:369
magic word & $parser
Definition: hooks.txt:2487
exportTransform($text, $model, $format=null)
Applies applicable export transformations to $text.
wfDebug($text, $dest= 'all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
if($line===false) $args
Definition: cdb.php:64
The ContentHandler facility adds support for arbitrary content types on wiki instead of relying on wikitext for everything It was introduced in MediaWiki Each kind of and so on Built in content types are
An extension or a local will often add custom code to the function with or without a global variable For someone wanting email notification when an article is shown may add
Definition: hooks.txt:51
$maintClass
wfTimestamp($outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
addOption($name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
either a unescaped string or a HtmlArmor object after in associative array form externallinks including delete and has completed for all link tables whether this was an auto creation default is conds Array Extra conditions for the No matching items in log is displayed if loglist is empty msgKey Array If you want a nice box with a set this to the key of the message First element is the message additional optional elements are parameters for the key that are processed with wfMessage() -> params() ->parseAsBlock()-offset Set to overwrite offset parameter in $wgRequest set to ''to unsetoffset-wrap String Wrap the message in html(usually something like"&lt
dump($history, $text=WikiExporter::TEXT)
Readahead helper for making large MediaWiki data dumps; reads in a previous XML dump to sequentially ...
bool resource $spawnProc
$maxConsecutiveFailedTextRetrievals
endElement($parser, $name)
null means default in associative array with keys and values unescaped Should be merged with default with a value of false meaning to suppress the attribute in associative array with keys and values unescaped noclasses just before the function returns a value If you return an< a > element with HTML attributes $attribs and contents $html will be returned If you return $ret will be returned and may include noclasses after processing & $attribs
Definition: hooks.txt:1936
$buffer
design txt This is a brief overview of the new design More thorough and up to date information is available on the documentation wiki at etc Handles the details of getting and saving to the user table of the and dealing with sessions and cookies OutputPage Encapsulates the entire HTML page that will be sent in response to any server request It is used by calling its functions to add text
Definition: design.txt:12
wfWikiID()
Get an ASCII string identifying this wiki This is used as a prefix in memcached keys.
addDescription($text)
Set the description text.
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
getOption($name, $default=null)
Get an option, or return the default.
output($out, $channel=null)
Throw some output to the user.
clearOpenElement($style)
loadWithArgv($argv)
Load params and arguments from a given array of command-line arguments.
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
presenting them properly to the user as errors is done by the caller return true use this to change the list i e etc next in line in page history
Definition: hooks.txt:1721
wfGetLBFactory()
Get the load balancer factory object.
startElement($parser, $name, $attribs)
bool resource $spawnRead
this class mediates it Skin Encapsulates a look and feel for the wiki All of the functions that render HTML and make choices about how to render it are here and are called from various other places when and is meant to be subclassed with other skins that may override some of its functions The User object contains a reference to a and so rather than having a global skin object we just rely on the global User and get the skin with $wgUser and also has some character encoding functions and other locale stuff The current user interface language is instantiated as and the local content language as $wgContLang
Definition: design.txt:56
$count
We ve cleaned up the code here by removing clumps of infrequently used code and moving them off somewhere else It s much easier for someone working with this code to see what s _really_ going on
Definition: hooks.txt:86
const DB_REPLICA
Definition: defines.php:22
progress($string)
Definition: backup.inc:414
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that probably a stub it is not rendered in wiki pages or galleries in category pages allow injecting custom HTML after the section Any uses of the hook need to handle escaping see BaseTemplate::getToolbox and BaseTemplate::makeListItem for details on the format of individual items inside of this array or by returning and letting standard HTTP rendering take place modifiable or by returning false and taking over the output modifiable modifiable after all normalizations have been except for the $wgMaxImageArea check set to true or false to override the $wgMaxImageArea check result gives extension the possibility to transform it themselves $handler
Definition: hooks.txt:802
initProgress($history=WikiExporter::FULL)
bool resource $spawnErr
characterData($parser, $data)
__construct($args=null)
rotateDb()
Drop the database connection $this->db and try to get a new one.
do that in ParserLimitReportFormat instead use this to modify the parameters of the image and a DIV can begin in one section and end in another Make sure your code can handle that case gracefully See the EditSectionClearerLink extension for an example zero but section is usually empty its values are the globals values before the output is cached one of or reset my talk page
Definition: hooks.txt:2491
const TS_DB
MySQL DATETIME (YYYY-MM-DD HH:MM:SS)
Definition: defines.php:16
DatabaseBase null $forcedDb
The dependency-injected database to use.
Definition: backup.inc:67
getText($id, $model=null, $format=null)
Tries to get the revision text for a revision id.
Allows to change the fields on the form that will be generated $name
Definition: hooks.txt:300