MediaWiki  master
dumpTextPass.php
Go to the documentation of this file.
1 <?php
27 require_once __DIR__ . '/includes/BackupDumper.php';
28 require_once __DIR__ . '/7zip.inc';
29 require_once __DIR__ . '/../includes/export/WikiExporter.php';
30 
36 
42  public $prefetch = null;
44  private $thisPage;
46  private $thisRev;
47 
48  // when we spend more than maxTimeAllowed seconds on this run, we continue
49  // processing until we write out the next complete page, then save output file(s),
50  // rename it/them and open new one(s)
51  public $maxTimeAllowed = 0; // 0 = no limit
52 
53  protected $input = "php://stdin";
55  protected $fetchCount = 0;
56  protected $prefetchCount = 0;
57  protected $prefetchCountLast = 0;
58  protected $fetchCountLast = 0;
59 
60  protected $maxFailures = 5;
62  protected $failureTimeout = 5; // Seconds to sleep after db failure
63 
64  protected $bufferSize = 524288; // In bytes. Maximum size to read from the stub in on go.
65 
66  protected $php = "php";
67  protected $spawn = false;
68 
72  protected $spawnProc = false;
73 
77  protected $spawnWrite = false;
78 
82  protected $spawnRead = false;
83 
87  protected $spawnErr = false;
88 
92  protected $xmlwriterobj = false;
93 
94  protected $timeExceeded = false;
95  protected $firstPageWritten = false;
96  protected $lastPageWritten = false;
97  protected $checkpointJustWritten = false;
98  protected $checkpointFiles = [];
99 
103  protected $db;
104 
108  function __construct( $args = null ) {
109  parent::__construct();
110 
111  $this->addDescription( <<<TEXT
112 This script postprocesses XML dumps from dumpBackup.php to add
113 page text which was stubbed out (using --stub).
114 
115 XML input is accepted on stdin.
116 XML output is sent to stdout; progress reports are sent to stderr.
117 TEXT
118  );
119  $this->stderr = fopen( "php://stderr", "wt" );
120 
121  $this->addOption( 'stub', 'To load a compressed stub dump instead of stdin. ' .
122  'Specify as --stub=<type>:<file>.', false, true );
123  $this->addOption( 'prefetch', 'Use a prior dump file as a text source, to savepressure on the ' .
124  'database. (Requires the XMLReader extension). Specify as --prefetch=<type>:<file>',
125  false, true );
126  $this->addOption( 'maxtime', 'Write out checkpoint file after this many minutes (writing' .
127  'out complete page, closing xml file properly, and opening new one' .
128  'with header). This option requires the checkpointfile option.', false, true );
129  $this->addOption( 'checkpointfile', 'Use this string for checkpoint filenames,substituting ' .
130  'first pageid written for the first %s (required) and the last pageid written for the ' .
131  'second %s if it exists.', false, true, false, true ); // This can be specified multiple times
132  $this->addOption( 'quiet', 'Don\'t dump status reports to stderr.' );
133  $this->addOption( 'full', 'Dump all revisions of every page' );
134  $this->addOption( 'current', 'Base ETA on number of pages in database instead of all revisions' );
135  $this->addOption( 'spawn', 'Spawn a subprocess for loading text records' );
136  $this->addOption( 'buffersize', 'Buffer size in bytes to use for reading the stub. ' .
137  '(Default: 512KB, Minimum: 4KB)', false, true );
138 
139  if ( $args ) {
140  $this->loadWithArgv( $args );
141  $this->processOptions();
142  }
143  }
144 
148  private function getBlobStore() {
149  return MediaWikiServices::getInstance()->getBlobStore();
150  }
151 
152  function execute() {
153  $this->processOptions();
154  $this->dump( true );
155  }
156 
157  function processOptions() {
158  parent::processOptions();
159 
160  if ( $this->hasOption( 'buffersize' ) ) {
161  $this->bufferSize = max( intval( $this->getOption( 'buffersize' ) ), 4 * 1024 );
162  }
163 
164  if ( $this->hasOption( 'prefetch' ) ) {
165  $url = $this->processFileOpt( $this->getOption( 'prefetch' ) );
166  $this->prefetch = new BaseDump( $url );
167  }
168 
169  if ( $this->hasOption( 'stub' ) ) {
170  $this->input = $this->processFileOpt( $this->getOption( 'stub' ) );
171  }
172 
173  if ( $this->hasOption( 'maxtime' ) ) {
174  $this->maxTimeAllowed = intval( $this->getOption( 'maxtime' ) ) * 60;
175  }
176 
177  if ( $this->hasOption( 'checkpointfile' ) ) {
178  $this->checkpointFiles = $this->getOption( 'checkpointfile' );
179  }
180 
181  if ( $this->hasOption( 'current' ) ) {
183  }
184 
185  if ( $this->hasOption( 'full' ) ) {
186  $this->history = WikiExporter::FULL;
187  }
188 
189  if ( $this->hasOption( 'spawn' ) ) {
190  $this->spawn = true;
191  $val = $this->getOption( 'spawn' );
192  if ( $val !== 1 ) {
193  $this->php = $val;
194  }
195  }
196  }
197 
209  function rotateDb() {
210  // Cleaning up old connections
211  if ( isset( $this->lb ) ) {
212  $this->lb->closeAll();
213  unset( $this->lb );
214  }
215 
216  if ( $this->forcedDb !== null ) {
217  $this->db = $this->forcedDb;
218 
219  return;
220  }
221 
222  if ( isset( $this->db ) && $this->db->isOpen() ) {
223  throw new MWException( 'DB is set and has not been closed by the Load Balancer' );
224  }
225 
226  unset( $this->db );
227 
228  // Trying to set up new connection.
229  // We do /not/ retry upon failure, but delegate to encapsulating logic, to avoid
230  // individually retrying at different layers of code.
231 
232  try {
233  $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
234  $this->lb = $lbFactory->newMainLB();
235  } catch ( Exception $e ) {
236  throw new MWException( __METHOD__
237  . " rotating DB failed to obtain new load balancer (" . $e->getMessage() . ")" );
238  }
239 
240  try {
241  $this->db = $this->lb->getConnection( DB_REPLICA, 'dump' );
242  } catch ( Exception $e ) {
243  throw new MWException( __METHOD__
244  . " rotating DB failed to obtain new database (" . $e->getMessage() . ")" );
245  }
246  }
247 
249  parent::initProgress();
250  $this->timeOfCheckpoint = $this->startTime;
251  }
252 
253  function dump( $history, $text = WikiExporter::TEXT ) {
254  // Notice messages will foul up your XML output even if they're
255  // relatively harmless.
256  if ( ini_get( 'display_errors' ) ) {
257  ini_set( 'display_errors', 'stderr' );
258  }
259 
260  $this->initProgress( $this->history );
261 
262  // We are trying to get an initial database connection to avoid that the
263  // first try of this request's first call to getText fails. However, if
264  // obtaining a good DB connection fails it's not a serious issue, as
265  // getText does retry upon failure and can start without having a working
266  // DB connection.
267  try {
268  $this->rotateDb();
269  } catch ( Exception $e ) {
270  // We do not even count this as failure. Just let eventual
271  // watchdogs know.
272  $this->progress( "Getting initial DB connection failed (" .
273  $e->getMessage() . ")" );
274  }
275 
276  $this->egress = new ExportProgressFilter( $this->sink, $this );
277 
278  // it would be nice to do it in the constructor, oh well. need egress set
279  $this->finalOptionCheck();
280 
281  // we only want this so we know how to close a stream :-P
282  $this->xmlwriterobj = new XmlDumpWriter();
283 
284  $input = fopen( $this->input, "rt" );
285  $this->readDump( $input );
286 
287  if ( $this->spawnProc ) {
288  $this->closeSpawn();
289  }
290 
291  $this->report( true );
292  }
293 
294  function processFileOpt( $opt ) {
295  $split = explode( ':', $opt, 2 );
296  $val = $split[0];
297  $param = '';
298  if ( count( $split ) === 2 ) {
299  $param = $split[1];
300  }
301  $fileURIs = explode( ';', $param );
302  foreach ( $fileURIs as $URI ) {
303  switch ( $val ) {
304  case "file":
305  $newURI = $URI;
306  break;
307  case "gzip":
308  $newURI = "compress.zlib://$URI";
309  break;
310  case "bzip2":
311  $newURI = "compress.bzip2://$URI";
312  break;
313  case "7zip":
314  $newURI = "mediawiki.compress.7z://$URI";
315  break;
316  default:
317  $newURI = $URI;
318  }
319  $newFileURIs[] = $newURI;
320  }
321  $val = implode( ';', $newFileURIs );
322 
323  return $val;
324  }
325 
329  function showReport() {
330  if ( !$this->prefetch ) {
331  parent::showReport();
332 
333  return;
334  }
335 
336  if ( $this->reporting ) {
337  $now = wfTimestamp( TS_DB );
338  $nowts = microtime( true );
339  $deltaAll = $nowts - $this->startTime;
340  $deltaPart = $nowts - $this->lastTime;
341  $this->pageCountPart = $this->pageCount - $this->pageCountLast;
342  $this->revCountPart = $this->revCount - $this->revCountLast;
343 
344  if ( $deltaAll ) {
345  $portion = $this->revCount / $this->maxCount;
346  $eta = $this->startTime + $deltaAll / $portion;
347  $etats = wfTimestamp( TS_DB, intval( $eta ) );
348  if ( $this->fetchCount ) {
349  $fetchRate = 100.0 * $this->prefetchCount / $this->fetchCount;
350  } else {
351  $fetchRate = '-';
352  }
353  $pageRate = $this->pageCount / $deltaAll;
354  $revRate = $this->revCount / $deltaAll;
355  } else {
356  $pageRate = '-';
357  $revRate = '-';
358  $etats = '-';
359  $fetchRate = '-';
360  }
361  if ( $deltaPart ) {
362  if ( $this->fetchCountLast ) {
363  $fetchRatePart = 100.0 * $this->prefetchCountLast / $this->fetchCountLast;
364  } else {
365  $fetchRatePart = '-';
366  }
367  $pageRatePart = $this->pageCountPart / $deltaPart;
368  $revRatePart = $this->revCountPart / $deltaPart;
369  } else {
370  $fetchRatePart = '-';
371  $pageRatePart = '-';
372  $revRatePart = '-';
373  }
374  $this->progress( sprintf(
375  "%s: %s (ID %d) %d pages (%0.1f|%0.1f/sec all|curr), "
376  . "%d revs (%0.1f|%0.1f/sec all|curr), %0.1f%%|%0.1f%% "
377  . "prefetched (all|curr), ETA %s [max %d]",
378  $now, wfWikiID(), $this->ID, $this->pageCount, $pageRate,
379  $pageRatePart, $this->revCount, $revRate, $revRatePart,
380  $fetchRate, $fetchRatePart, $etats, $this->maxCount
381  ) );
382  $this->lastTime = $nowts;
383  $this->revCountLast = $this->revCount;
384  $this->prefetchCountLast = $this->prefetchCount;
385  $this->fetchCountLast = $this->fetchCount;
386  }
387  }
388 
389  function setTimeExceeded() {
390  $this->timeExceeded = true;
391  }
392 
393  function checkIfTimeExceeded() {
394  if ( $this->maxTimeAllowed
395  && ( $this->lastTime - $this->timeOfCheckpoint > $this->maxTimeAllowed )
396  ) {
397  return true;
398  }
399 
400  return false;
401  }
402 
403  function finalOptionCheck() {
404  if ( ( $this->checkpointFiles && !$this->maxTimeAllowed )
405  || ( $this->maxTimeAllowed && !$this->checkpointFiles )
406  ) {
407  throw new MWException( "Options checkpointfile and maxtime must be specified together.\n" );
408  }
409  foreach ( $this->checkpointFiles as $checkpointFile ) {
410  $count = substr_count( $checkpointFile, "%s" );
411  if ( $count != 2 ) {
412  throw new MWException( "Option checkpointfile must contain two '%s' "
413  . "for substitution of first and last pageids, count is $count instead, "
414  . "file is $checkpointFile.\n" );
415  }
416  }
417 
418  if ( $this->checkpointFiles ) {
419  $filenameList = (array)$this->egress->getFilenames();
420  if ( count( $filenameList ) != count( $this->checkpointFiles ) ) {
421  throw new MWException( "One checkpointfile must be specified "
422  . "for each output option, if maxtime is used.\n" );
423  }
424  }
425  }
426 
432  function readDump( $input ) {
433  $this->buffer = "";
434  $this->openElement = false;
435  $this->atStart = true;
436  $this->state = "";
437  $this->lastName = "";
438  $this->thisPage = 0;
439  $this->thisRev = 0;
440  $this->thisRevModel = null;
441  $this->thisRevFormat = null;
442 
443  $parser = xml_parser_create( "UTF-8" );
444  xml_parser_set_option( $parser, XML_OPTION_CASE_FOLDING, false );
445 
446  xml_set_element_handler(
447  $parser,
448  [ $this, 'startElement' ],
449  [ $this, 'endElement' ]
450  );
451  xml_set_character_data_handler( $parser, [ $this, 'characterData' ] );
452 
453  $offset = 0; // for context extraction on error reporting
454  do {
455  if ( $this->checkIfTimeExceeded() ) {
456  $this->setTimeExceeded();
457  }
458  $chunk = fread( $input, $this->bufferSize );
459  if ( !xml_parse( $parser, $chunk, feof( $input ) ) ) {
460  wfDebug( "TextDumpPass::readDump encountered XML parsing error\n" );
461 
462  $byte = xml_get_current_byte_index( $parser );
463  $msg = wfMessage( 'xml-error-string',
464  'XML import parse failure',
465  xml_get_current_line_number( $parser ),
466  xml_get_current_column_number( $parser ),
467  $byte . ( is_null( $chunk ) ? null : ( '; "' . substr( $chunk, $byte - $offset, 16 ) . '"' ) ),
468  xml_error_string( xml_get_error_code( $parser ) ) )->escaped();
469 
470  xml_parser_free( $parser );
471 
472  throw new MWException( $msg );
473  }
474  $offset += strlen( $chunk );
475  } while ( $chunk !== false && !feof( $input ) );
476  if ( $this->maxTimeAllowed ) {
477  $filenameList = (array)$this->egress->getFilenames();
478  // we wrote some stuff after last checkpoint that needs renamed
479  if ( file_exists( $filenameList[0] ) ) {
480  $newFilenames = [];
481  # we might have just written the header and footer and had no
482  # pages or revisions written... perhaps they were all deleted
483  # there's no pageID 0 so we use that. the caller is responsible
484  # for deciding what to do with a file containing only the
485  # siteinfo information and the mw tags.
486  if ( !$this->firstPageWritten ) {
487  $firstPageID = str_pad( 0, 9, "0", STR_PAD_LEFT );
488  $lastPageID = str_pad( 0, 9, "0", STR_PAD_LEFT );
489  } else {
490  $firstPageID = str_pad( $this->firstPageWritten, 9, "0", STR_PAD_LEFT );
491  $lastPageID = str_pad( $this->lastPageWritten, 9, "0", STR_PAD_LEFT );
492  }
493 
494  $filenameCount = count( $filenameList );
495  for ( $i = 0; $i < $filenameCount; $i++ ) {
496  $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID );
497  $fileinfo = pathinfo( $filenameList[$i] );
498  $newFilenames[] = $fileinfo['dirname'] . '/' . $checkpointNameFilledIn;
499  }
500  $this->egress->closeAndRename( $newFilenames );
501  }
502  }
503  xml_parser_free( $parser );
504 
505  return true;
506  }
507 
517  private function exportTransform( $text, $model, $format = null ) {
518  try {
520  $text = $handler->exportTransform( $text, $format );
521  }
522  catch ( MWException $ex ) {
523  $this->progress(
524  "Unable to apply export transformation for content model '$model': " .
525  $ex->getMessage()
526  );
527  }
528 
529  return $text;
530  }
531 
552  function getText( $id, $model = null, $format = null ) {
553  global $wgContentHandlerUseDB;
554 
555  $prefetchNotTried = true; // Whether or not we already tried to get the text via prefetch.
556  $text = false; // The candidate for a good text. false if no proper value.
557  $failures = 0; // The number of times, this invocation of getText already failed.
558 
559  // The number of times getText failed without yielding a good text in between.
560  static $consecutiveFailedTextRetrievals = 0;
561 
562  $this->fetchCount++;
563 
564  // To allow to simply return on success and do not have to worry about book keeping,
565  // we assume, this fetch works (possible after some retries). Nevertheless, we koop
566  // the old value, so we can restore it, if problems occur (See after the while loop).
567  $oldConsecutiveFailedTextRetrievals = $consecutiveFailedTextRetrievals;
568  $consecutiveFailedTextRetrievals = 0;
569 
570  if ( $model === null && $wgContentHandlerUseDB ) {
571  // TODO: MCR: use content table
572  $row = $this->db->selectRow(
573  'revision',
574  [ 'rev_content_model', 'rev_content_format' ],
575  [ 'rev_id' => $this->thisRev ],
576  __METHOD__
577  );
578 
579  if ( $row ) {
580  $model = $row->rev_content_model;
581  $format = $row->rev_content_format;
582  }
583  }
584 
585  if ( $model === null || $model === '' ) {
586  $model = false;
587  }
588 
589  while ( $failures < $this->maxFailures ) {
590  // As soon as we found a good text for the $id, we will return immediately.
591  // Hence, if we make it past the try catch block, we know that we did not
592  // find a good text.
593 
594  try {
595  // Step 1: Get some text (or reuse from previous iteratuon if checking
596  // for plausibility failed)
597 
598  // Trying to get prefetch, if it has not been tried before
599  if ( $text === false && isset( $this->prefetch ) && $prefetchNotTried ) {
600  $prefetchNotTried = false;
601  $tryIsPrefetch = true;
602  $text = $this->prefetch->prefetch( (int)$this->thisPage, (int)$this->thisRev );
603 
604  if ( $text === null ) {
605  $text = false;
606  }
607 
608  if ( is_string( $text ) && $model !== false ) {
609  // Apply export transformation to text coming from an old dump.
610  // The purpose of this transformation is to convert up from legacy
611  // formats, which may still be used in the older dump that is used
612  // for pre-fetching. Applying the transformation again should not
613  // interfere with content that is already in the correct form.
614  $text = $this->exportTransform( $text, $model, $format );
615  }
616  }
617 
618  if ( $text === false ) {
619  // Fallback to asking the database
620  $tryIsPrefetch = false;
621  if ( $this->spawn ) {
622  $text = $this->getTextSpawned( $id );
623  } else {
624  $text = $this->getTextDb( $id );
625  }
626 
627  if ( $text !== false && $model !== false ) {
628  // Apply export transformation to text coming from the database.
629  // Prefetched text should already have transformations applied.
630  $text = $this->exportTransform( $text, $model, $format );
631  }
632 
633  // No more checks for texts from DB for now.
634  // If we received something that is not false,
635  // We treat it as good text, regardless of whether it actually is or is not
636  if ( $text !== false ) {
637  return $text;
638  }
639  }
640 
641  if ( $text === false ) {
642  throw new MWException( "Generic error while obtaining text for id " . $id );
643  }
644 
645  // We received a good candidate for the text of $id via some method
646 
647  // Step 2: Checking for plausibility and return the text if it is
648  // plausible
649  $revID = intval( $this->thisRev );
650  if ( !isset( $this->db ) ) {
651  throw new MWException( "No database available" );
652  }
653 
654  if ( $model !== CONTENT_MODEL_WIKITEXT ) {
655  $revLength = strlen( $text );
656  } else {
657  $revLength = $this->db->selectField( 'revision', 'rev_len', [ 'rev_id' => $revID ] );
658  }
659 
660  if ( strlen( $text ) == $revLength ) {
661  if ( $tryIsPrefetch ) {
662  $this->prefetchCount++;
663  }
664 
665  return $text;
666  }
667 
668  $text = false;
669  throw new MWException( "Received text is unplausible for id " . $id );
670  } catch ( Exception $e ) {
671  $msg = "getting/checking text " . $id . " failed (" . $e->getMessage() . ")";
672  if ( $failures + 1 < $this->maxFailures ) {
673  $msg .= " (Will retry " . ( $this->maxFailures - $failures - 1 ) . " more times)";
674  }
675  $this->progress( $msg );
676  }
677 
678  // Something went wrong; we did not a text that was plausible :(
679  $failures++;
680 
681  // A failure in a prefetch hit does not warrant resetting db connection etc.
682  if ( !$tryIsPrefetch ) {
683  // After backing off for some time, we try to reboot the whole process as
684  // much as possible to not carry over failures from one part to the other
685  // parts
686  sleep( $this->failureTimeout );
687  try {
688  $this->rotateDb();
689  if ( $this->spawn ) {
690  $this->closeSpawn();
691  $this->openSpawn();
692  }
693  } catch ( Exception $e ) {
694  $this->progress( "Rebooting getText infrastructure failed (" . $e->getMessage() . ")" .
695  " Trying to continue anyways" );
696  }
697  }
698  }
699 
700  // Retirieving a good text for $id failed (at least) maxFailures times.
701  // We abort for this $id.
702 
703  // Restoring the consecutive failures, and maybe aborting, if the dump
704  // is too broken.
705  $consecutiveFailedTextRetrievals = $oldConsecutiveFailedTextRetrievals + 1;
706  if ( $consecutiveFailedTextRetrievals > $this->maxConsecutiveFailedTextRetrievals ) {
707  throw new MWException( "Graceful storage failure" );
708  }
709 
710  return "";
711  }
712 
719  private function getTextDb( $id ) {
720  $store = $this->getBlobStore();
721  $address = ( is_int( $id ) || strpos( $id, ':' ) === false )
722  ? SqlBlobStore::makeAddressFromTextId( (int)$id )
723  : $id;
724 
725  try {
726  $text = $store->getBlob( $address );
727 
728  $stripped = str_replace( "\r", "", $text );
729  $normalized = MediaWikiServices::getInstance()->getContentLanguage()
730  ->normalize( $stripped );
731 
732  return $normalized;
733  } catch ( BlobAccessException $ex ) {
734  // XXX: log a warning?
735  return false;
736  }
737  }
738 
743  private function getTextSpawned( $address ) {
744  Wikimedia\suppressWarnings();
745  if ( !$this->spawnProc ) {
746  // First time?
747  $this->openSpawn();
748  }
749  $text = $this->getTextSpawnedOnce( $address );
750  Wikimedia\restoreWarnings();
751 
752  return $text;
753  }
754 
755  function openSpawn() {
756  global $IP;
757 
758  if ( file_exists( "$IP/../multiversion/MWScript.php" ) ) {
759  $cmd = implode( " ",
760  array_map( [ Shell::class, 'escape' ],
761  [
762  $this->php,
763  "$IP/../multiversion/MWScript.php",
764  "fetchText.php",
765  '--wiki', wfWikiID() ] ) );
766  } else {
767  $cmd = implode( " ",
768  array_map( [ Shell::class, 'escape' ],
769  [
770  $this->php,
771  "$IP/maintenance/fetchText.php",
772  '--wiki', wfWikiID() ] ) );
773  }
774  $spec = [
775  0 => [ "pipe", "r" ],
776  1 => [ "pipe", "w" ],
777  2 => [ "file", "/dev/null", "a" ] ];
778  $pipes = [];
779 
780  $this->progress( "Spawning database subprocess: $cmd" );
781  $this->spawnProc = proc_open( $cmd, $spec, $pipes );
782  if ( !$this->spawnProc ) {
783  $this->progress( "Subprocess spawn failed." );
784 
785  return false;
786  }
787  list(
788  $this->spawnWrite, // -> stdin
789  $this->spawnRead, // <- stdout
790  ) = $pipes;
791 
792  return true;
793  }
794 
795  private function closeSpawn() {
796  Wikimedia\suppressWarnings();
797  if ( $this->spawnRead ) {
798  fclose( $this->spawnRead );
799  }
800  $this->spawnRead = false;
801  if ( $this->spawnWrite ) {
802  fclose( $this->spawnWrite );
803  }
804  $this->spawnWrite = false;
805  if ( $this->spawnErr ) {
806  fclose( $this->spawnErr );
807  }
808  $this->spawnErr = false;
809  if ( $this->spawnProc ) {
810  pclose( $this->spawnProc );
811  }
812  $this->spawnProc = false;
813  Wikimedia\restoreWarnings();
814  }
815 
820  private function getTextSpawnedOnce( $address ) {
821  if ( is_int( $address ) || intval( $address ) ) {
822  $address = SqlBlobStore::makeAddressFromTextId( (int)$address );
823  }
824 
825  $ok = fwrite( $this->spawnWrite, "$address\n" );
826  // $this->progress( ">> $id" );
827  if ( !$ok ) {
828  return false;
829  }
830 
831  $ok = fflush( $this->spawnWrite );
832  // $this->progress( ">> [flush]" );
833  if ( !$ok ) {
834  return false;
835  }
836 
837  // check that the text address they are sending is the one we asked for
838  // this avoids out of sync revision text errors we have encountered in the past
839  $newAddress = fgets( $this->spawnRead );
840  if ( $newAddress === false ) {
841  return false;
842  }
843  $newAddress = trim( $newAddress );
844  if ( strpos( $newAddress, ':' ) === false ) {
845  $newAddress = SqlBlobStore::makeAddressFromTextId( intval( $newAddress ) );
846  }
847 
848  if ( $newAddress !== $address ) {
849  return false;
850  }
851 
852  $len = fgets( $this->spawnRead );
853  // $this->progress( "<< " . trim( $len ) );
854  if ( $len === false ) {
855  return false;
856  }
857 
858  $nbytes = intval( $len );
859  // actual error, not zero-length text
860  if ( $nbytes < 0 ) {
861  return false;
862  }
863 
864  $text = "";
865 
866  // Subprocess may not send everything at once, we have to loop.
867  while ( $nbytes > strlen( $text ) ) {
868  $buffer = fread( $this->spawnRead, $nbytes - strlen( $text ) );
869  if ( $buffer === false ) {
870  break;
871  }
872  $text .= $buffer;
873  }
874 
875  $gotbytes = strlen( $text );
876  if ( $gotbytes != $nbytes ) {
877  $this->progress( "Expected $nbytes bytes from database subprocess, got $gotbytes " );
878 
879  return false;
880  }
881 
882  // Do normalization in the dump thread...
883  $stripped = str_replace( "\r", "", $text );
884  $normalized = MediaWikiServices::getInstance()->getContentLanguage()->
885  normalize( $stripped );
886 
887  return $normalized;
888  }
889 
891  $this->checkpointJustWritten = false;
892 
893  $this->clearOpenElement( null );
894  $this->lastName = $name;
895 
896  if ( $name == 'revision' ) {
897  $this->state = $name;
898  $this->egress->writeOpenPage( null, $this->buffer );
899  $this->buffer = "";
900  } elseif ( $name == 'page' ) {
901  $this->state = $name;
902  if ( $this->atStart ) {
903  $this->egress->writeOpenStream( $this->buffer );
904  $this->buffer = "";
905  $this->atStart = false;
906  }
907  }
908 
909  if ( $name == "text" && isset( $attribs['id'] ) ) {
910  $id = $attribs['id'];
911  $model = trim( $this->thisRevModel );
912  $format = trim( $this->thisRevFormat );
913 
914  $model = $model === '' ? null : $model;
915  $format = $format === '' ? null : $format;
916 
917  $text = $this->getText( $id, $model, $format );
918  $this->openElement = [ $name, [ 'xml:space' => 'preserve' ] ];
919  if ( strlen( $text ) > 0 ) {
920  $this->characterData( $parser, $text );
921  }
922  } else {
923  $this->openElement = [ $name, $attribs ];
924  }
925  }
926 
927  function endElement( $parser, $name ) {
928  $this->checkpointJustWritten = false;
929 
930  if ( $this->openElement ) {
931  $this->clearOpenElement( "" );
932  } else {
933  $this->buffer .= "</$name>";
934  }
935 
936  if ( $name == 'revision' ) {
937  $this->egress->writeRevision( null, $this->buffer );
938  $this->buffer = "";
939  $this->thisRev = "";
940  $this->thisRevModel = null;
941  $this->thisRevFormat = null;
942  } elseif ( $name == 'page' ) {
943  if ( !$this->firstPageWritten ) {
944  $this->firstPageWritten = trim( $this->thisPage );
945  }
946  $this->lastPageWritten = trim( $this->thisPage );
947  if ( $this->timeExceeded ) {
948  $this->egress->writeClosePage( $this->buffer );
949  // nasty hack, we can't just write the chardata after the
950  // page tag, it will include leading blanks from the next line
951  $this->egress->sink->write( "\n" );
952 
953  $this->buffer = $this->xmlwriterobj->closeStream();
954  $this->egress->writeCloseStream( $this->buffer );
955 
956  $this->buffer = "";
957  $this->thisPage = "";
958  // this could be more than one file if we had more than one output arg
959 
960  $filenameList = (array)$this->egress->getFilenames();
961  $newFilenames = [];
962  $firstPageID = str_pad( $this->firstPageWritten, 9, "0", STR_PAD_LEFT );
963  $lastPageID = str_pad( $this->lastPageWritten, 9, "0", STR_PAD_LEFT );
964  $filenamesCount = count( $filenameList );
965  for ( $i = 0; $i < $filenamesCount; $i++ ) {
966  $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID );
967  $fileinfo = pathinfo( $filenameList[$i] );
968  $newFilenames[] = $fileinfo['dirname'] . '/' . $checkpointNameFilledIn;
969  }
970  $this->egress->closeRenameAndReopen( $newFilenames );
971  $this->buffer = $this->xmlwriterobj->openStream();
972  $this->timeExceeded = false;
973  $this->timeOfCheckpoint = $this->lastTime;
974  $this->firstPageWritten = false;
975  $this->checkpointJustWritten = true;
976  } else {
977  $this->egress->writeClosePage( $this->buffer );
978  $this->buffer = "";
979  $this->thisPage = "";
980  }
981  } elseif ( $name == 'mediawiki' ) {
982  $this->egress->writeCloseStream( $this->buffer );
983  $this->buffer = "";
984  }
985  }
986 
987  function characterData( $parser, $data ) {
988  $this->clearOpenElement( null );
989  if ( $this->lastName == "id" ) {
990  if ( $this->state == "revision" ) {
991  $this->thisRev .= $data;
992  } elseif ( $this->state == "page" ) {
993  $this->thisPage .= $data;
994  }
995  } elseif ( $this->lastName == "model" ) {
996  $this->thisRevModel .= $data;
997  } elseif ( $this->lastName == "format" ) {
998  $this->thisRevFormat .= $data;
999  }
1000 
1001  // have to skip the newline left over from closepagetag line of
1002  // end of checkpoint files. nasty hack!!
1003  if ( $this->checkpointJustWritten ) {
1004  if ( $data[0] == "\n" ) {
1005  $data = substr( $data, 1 );
1006  }
1007  $this->checkpointJustWritten = false;
1008  }
1009  $this->buffer .= htmlspecialchars( $data );
1010  }
1011 
1012  function clearOpenElement( $style ) {
1013  if ( $this->openElement ) {
1014  $this->buffer .= Xml::element( $this->openElement[0], $this->openElement[1], $style );
1015  $this->openElement = false;
1016  }
1017  }
1018 }
1019 
1021 require_once RUN_MAINTENANCE_IF_MAIN;
The wiki should then use memcached to cache various data To use multiple just add more items to the array To increase the weight of a make its entry a array("192.168.0.1:11211", 2))
bool resource $spawnWrite
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition: deferred.txt:11
IMaintainableDatabase $db
$data
Utility to generate mapping file used in mw.Title (phpCharToUpper.json)
const CONTENT_MODEL_WIKITEXT
Definition: Defines.php:235
processing should stop and the error should be shown to the user * false
Definition: hooks.txt:187
$IP
Definition: WebStart.php:41
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
getOption( $name, $default=null)
Get an option, or return the default.
progress( $string)
div flags Integer display flags(NO_ACTION_LINK, NO_EXTRA_USER_LINKS) 'LogException' returning false will NOT prevent logging $e
Definition: hooks.txt:2159
characterData( $parser, $data)
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for and distribution as defined by Sections through of this document Licensor shall mean the copyright owner or entity authorized by the copyright owner that is granting the License Legal Entity shall mean the union of the acting entity and all other entities that control are controlled by or are under common control with that entity For the purposes of this definition control direct or to cause the direction or management of such whether by contract or including but not limited to software source documentation and configuration files Object form shall mean any form resulting from mechanical transformation or translation of a Source including but not limited to compiled object generated and conversions to other media types Work shall mean the work of whether in Source or Object made available under the as indicated by a copyright notice that is included in or attached to the whether in Source or Object that is based or other modifications as a an original work of authorship For the purposes of this Derivative Works shall not include works that remain separable from
string bool $thisRev
exportTransform( $text, $model, $format=null)
Applies applicable export transformations to $text.
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency MediaWikiServices
Definition: injection.txt:23
__construct( $args=null)
getText( $id, $model=null, $format=null)
Tries to load revision text.
hasOption( $name)
Checks to see if a particular option exists.
require_once RUN_MAINTENANCE_IF_MAIN
Definition: maintenance.txt:50
showReport()
Overridden to include prefetch ratio if enabled.
target page
bool XmlDumpWriter $xmlwriterobj
loadWithArgv( $argv)
Load params and arguments from a given array of command-line arguments.
Exception representing a failure to access a data blob.
startElement( $parser, $name, $attribs)
see documentation in includes Linker php for Linker::makeImageLink or false for current used if you return false $parser
Definition: hooks.txt:1799
This list may contain false positives That usually means there is additional text with links below the first Each row contains links to the first and second as well as the first line of the second redirect text
clearOpenElement( $style)
This document provides an overview of the usage of PageUpdater and that is
Definition: pageupdater.txt:3
if( $line===false) $args
Definition: cdb.php:64
The ContentHandler facility adds support for arbitrary content types on wiki instead of relying on wikitext for everything It was introduced in MediaWiki Each kind of and so on Built in content types are
$wgContentHandlerUseDB
Set to false to disable use of the database fields introduced by the ContentHandler facility...
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that When $user is not it can be in the form of< username >< more info > e g for bot passwords intended to be added to log contexts Fields it might only if the login was with a bot password it is not rendered in wiki pages or galleries in category pages allow injecting custom HTML after the section Any uses of the hook need to handle escaping see BaseTemplate::getToolbox and BaseTemplate::makeListItem for details on the format of individual items inside of this array or by returning and letting standard HTTP rendering take place modifiable or by returning false and taking over the output modifiable modifiable after all normalizations have been except for the $wgMaxImageArea check set to true or false to override the $wgMaxImageArea check result gives extension the possibility to transform it themselves $handler
Definition: hooks.txt:780
An extension or a local will often add custom code to the function with or without a global variable For someone wanting email notification when an article is shown may add
Definition: hooks.txt:51
$maintClass
getTextSpawned( $address)
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
readDump( $input)
dump( $history, $text=WikiExporter::TEXT)
either a unescaped string or a HtmlArmor object after in associative array form externallinks including delete and has completed for all link tables whether this was an auto creation use $formDescriptor instead default is conds Array Extra conditions for the No matching items in log is displayed if loglist is empty msgKey Array If you want a nice box with a set this to the key of the message First element is the message additional optional elements are parameters for the key that are processed with wfMessage() -> params() ->parseAsBlock() - offset Set to overwrite offset parameter in $wgRequest set to '' to unset offset - wrap String Wrap the message in html(usually something like "&lt
processFileOpt( $opt)
static getForModelID( $modelId)
Returns the ContentHandler singleton for the given model ID.
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
addDescription( $text)
Set the description text.
bool resource $spawnProc
$maxConsecutiveFailedTextRetrievals
getTextDb( $id)
Loads the serialized content from storage.
null means default in associative array with keys and values unescaped Should be merged with default with a value of false meaning to suppress the attribute in associative array with keys and values unescaped noclasses just before the function returns a value If you return an< a > element with HTML attributes $attribs and contents $html will be returned If you return $ret will be returned and may include noclasses after processing & $attribs
Definition: hooks.txt:1982
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that When $user is not null
Definition: hooks.txt:780
$buffer
wfWikiID()
Get an ASCII string identifying this wiki This is used as a prefix in memcached keys.
output( $out, $channel=null)
Throw some output to the user.
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
string bool $thisPage
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
presenting them properly to the user as errors is done by the caller return true use this to change the list i e etc next in line in page history
Definition: hooks.txt:1766
you have access to all of the normal MediaWiki so you can get a DB use the etc For full docs on the Maintenance class
Definition: maintenance.txt:52
IDatabase null $forcedDb
The dependency-injected database to use.
report( $final=false)
initProgress( $history=WikiExporter::FULL)
bool resource $spawnRead
static element( $element, $attribs=null, $contents='', $allowShortTag=true)
Format an XML element with given attributes and, optionally, text content.
Definition: Xml.php:41
Using a hook running we can avoid having all this option specific stuff in our mainline code Using the function We ve cleaned up the code here by removing clumps of infrequently used code and moving them off somewhere else It s much easier for someone working with this code to see what s _really_ going on
Definition: hooks.txt:77
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
Allows to change the fields on the form that will be generated $name
Definition: hooks.txt:271
const DB_REPLICA
Definition: defines.php:25
BaseDump $prefetch
bool resource $spawnErr
rotateDb()
Drop the database connection $this->db and try to get a new one.
getTextSpawnedOnce( $address)
endElement( $parser, $name)