MediaWiki  master
TextPassDumper.php
Go to the documentation of this file.
1 <?php
28 require_once __DIR__ . '/BackupDumper.php';
29 require_once __DIR__ . '/../../includes/export/WikiExporter.php';
30 
39 use Wikimedia\AtEase\AtEase;
41 
47  public $prefetch = null;
49  private $thisPage;
51  private $thisRev;
53  private $thisRole = null;
54 
60  public $maxTimeAllowed = 0;
61 
63  protected $input = "php://stdin";
67  protected $fetchCount = 0;
69  protected $prefetchCount = 0;
71  protected $prefetchCountLast = 0;
73  protected $fetchCountLast = 0;
74 
76  protected $maxFailures = 5;
80  protected $failureTimeout = 5;
81 
83  protected $bufferSize = 524288;
84 
86  protected $php = [ PHP_BINARY ];
88  protected $spawn = false;
89 
93  protected $spawnProc = false;
94 
98  protected $spawnWrite;
99 
103  protected $spawnRead;
104 
108  protected $spawnErr = false;
109 
113  protected $xmlwriterobj = false;
114 
116  protected $timeExceeded = false;
118  protected $firstPageWritten = false;
120  protected $lastPageWritten = false;
122  protected $checkpointJustWritten = false;
124  protected $checkpointFiles = [];
125 
129  protected $db;
130 
134  public function __construct( $args = null ) {
135  parent::__construct();
136 
137  $this->addDescription( <<<TEXT
138 This script postprocesses XML dumps from dumpBackup.php to add
139 page text which was stubbed out (using --stub).
140 
141 XML input is accepted on stdin.
142 XML output is sent to stdout; progress reports are sent to stderr.
143 TEXT
144  );
145  $this->stderr = fopen( "php://stderr", "wt" );
146 
147  $this->addOption( 'stub', 'To load a compressed stub dump instead of stdin. ' .
148  'Specify as --stub=<type>:<file>.', false, true );
149  $this->addOption( 'prefetch', 'Use a prior dump file as a text source, to savepressure on the ' .
150  'database. (Requires the XMLReader extension). Specify as --prefetch=<type>:<file>',
151  false, true );
152  $this->addOption( 'maxtime', 'Write out checkpoint file after this many minutes (writing' .
153  'out complete page, closing xml file properly, and opening new one' .
154  'with header). This option requires the checkpointfile option.', false, true );
155  $this->addOption( 'checkpointfile', 'Use this string for checkpoint filenames,substituting ' .
156  'first pageid written for the first %s (required) and the last pageid written for the ' .
157  'second %s if it exists.', false, true, false, true ); // This can be specified multiple times
158  $this->addOption( 'quiet', 'Don\'t dump status reports to stderr.' );
159  $this->addOption( 'full', 'Dump all revisions of every page' );
160  $this->addOption( 'current', 'Base ETA on number of pages in database instead of all revisions' );
161  $this->addOption( 'spawn', 'Spawn a subprocess for loading text records, optionally specify ' .
162  'php[,mwscript] paths' );
163  $this->addOption( 'buffersize', 'Buffer size in bytes to use for reading the stub. ' .
164  '(Default: 512 KiB, Minimum: 4 KiB)', false, true );
165 
166  if ( $args ) {
167  $this->loadWithArgv( $args );
168  $this->processOptions();
169  }
170  }
171 
172  public function finalSetup( SettingsBuilder $settingsBuilder = null ) {
173  parent::finalSetup( $settingsBuilder );
174 
176  }
177 
181  private function getBlobStore() {
182  return MediaWikiServices::getInstance()->getBlobStore();
183  }
184 
185  public function execute() {
186  $this->processOptions();
187  $this->dump( true );
188  }
189 
190  protected function processOptions() {
191  parent::processOptions();
192 
193  if ( $this->hasOption( 'buffersize' ) ) {
194  $this->bufferSize = max( intval( $this->getOption( 'buffersize' ) ), 4 * 1024 );
195  }
196 
197  if ( $this->hasOption( 'prefetch' ) ) {
198  $url = $this->processFileOpt( $this->getOption( 'prefetch' ) );
199  $this->prefetch = new BaseDump( $url );
200  }
201 
202  if ( $this->hasOption( 'stub' ) ) {
203  $this->input = $this->processFileOpt( $this->getOption( 'stub' ) );
204  }
205 
206  if ( $this->hasOption( 'maxtime' ) ) {
207  $this->maxTimeAllowed = intval( $this->getOption( 'maxtime' ) ) * 60;
208  }
209 
210  if ( $this->hasOption( 'checkpointfile' ) ) {
211  $this->checkpointFiles = $this->getOption( 'checkpointfile' );
212  }
213 
214  if ( $this->hasOption( 'current' ) ) {
215  $this->history = WikiExporter::CURRENT;
216  }
217 
218  if ( $this->hasOption( 'full' ) ) {
219  $this->history = WikiExporter::FULL;
220  }
221 
222  if ( $this->hasOption( 'spawn' ) ) {
223  $this->spawn = true;
224  $val = $this->getOption( 'spawn' );
225  if ( $val !== 1 ) {
226  $this->php = explode( ',', $val, 2 );
227  }
228  }
229  }
230 
243  protected function rotateDb() {
244  // Cleaning up old connections
245  if ( isset( $this->lb ) ) {
246  $this->lb->closeAll( __METHOD__ );
247  unset( $this->lb );
248  }
249 
250  if ( $this->forcedDb !== null ) {
251  $this->db = $this->forcedDb;
252 
253  return;
254  }
255 
256  if ( isset( $this->db ) && $this->db->isOpen() ) {
257  throw new MWException( 'DB is set and has not been closed by the Load Balancer' );
258  }
259 
260  unset( $this->db );
261 
262  // Trying to set up new connection.
263  // We do /not/ retry upon failure, but delegate to encapsulating logic, to avoid
264  // individually retrying at different layers of code.
265 
266  try {
267  $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
268  $this->lb = $lbFactory->newMainLB();
269  } catch ( Exception $e ) {
270  throw new MWException( __METHOD__
271  . " rotating DB failed to obtain new load balancer (" . $e->getMessage() . ")" );
272  }
273 
274  try {
275  $this->db = $this->lb->getMaintenanceConnectionRef( DB_REPLICA, 'dump' );
276  } catch ( Exception $e ) {
277  throw new MWException( __METHOD__
278  . " rotating DB failed to obtain new database (" . $e->getMessage() . ")" );
279  }
280  }
281 
282  public function initProgress( $history = WikiExporter::FULL ) {
283  parent::initProgress();
284  $this->timeOfCheckpoint = $this->startTime;
285  }
286 
287  public function dump( $history, $text = WikiExporter::TEXT ) {
288  // Notice messages will foul up your XML output even if they're
289  // relatively harmless.
290  if ( ini_get( 'display_errors' ) ) {
291  ini_set( 'display_errors', 'stderr' );
292  }
293 
294  $this->initProgress( $this->history );
295 
296  // We are trying to get an initial database connection to avoid that the
297  // first try of this request's first call to getText fails. However, if
298  // obtaining a good DB connection fails it's not a serious issue, as
299  // getText does retry upon failure and can start without having a working
300  // DB connection.
301  try {
302  $this->rotateDb();
303  } catch ( Exception $e ) {
304  // We do not even count this as failure. Just let eventual
305  // watchdogs know.
306  $this->progress( "Getting initial DB connection failed (" .
307  $e->getMessage() . ")" );
308  }
309 
310  $this->egress = new ExportProgressFilter( $this->sink, $this );
311 
312  // it would be nice to do it in the constructor, oh well. need egress set
313  $this->finalOptionCheck();
314 
315  // we only want this so we know how to close a stream :-P
316  $this->xmlwriterobj = new XmlDumpWriter( XmlDumpWriter::WRITE_CONTENT, $this->schemaVersion );
317 
318  $input = fopen( $this->input, "rt" );
319  $this->readDump( $input );
320 
321  if ( $this->spawnProc ) {
322  $this->closeSpawn();
323  }
324 
325  $this->report( true );
326  }
327 
328  protected function processFileOpt( $opt ) {
329  $split = explode( ':', $opt, 2 );
330  $val = $split[0];
331  $param = '';
332  if ( count( $split ) === 2 ) {
333  $param = $split[1];
334  }
335  $fileURIs = explode( ';', $param );
336  $newFileURIs = [];
337  foreach ( $fileURIs as $URI ) {
338  switch ( $val ) {
339  case "file":
340  $newURI = $URI;
341  break;
342  case "gzip":
343  $newURI = "compress.zlib://$URI";
344  break;
345  case "bzip2":
346  $newURI = "compress.bzip2://$URI";
347  break;
348  case "7zip":
349  $newURI = "mediawiki.compress.7z://$URI";
350  break;
351  default:
352  $newURI = $URI;
353  }
354  $newFileURIs[] = $newURI;
355  }
356  $val = implode( ';', $newFileURIs );
357 
358  return $val;
359  }
360 
364  public function showReport() {
365  if ( !$this->prefetch ) {
366  parent::showReport();
367 
368  return;
369  }
370 
371  if ( $this->reporting ) {
372  $now = wfTimestamp( TS_DB );
373  $nowts = microtime( true );
374  $deltaAll = $nowts - $this->startTime;
375  $deltaPart = $nowts - $this->lastTime;
376  $this->pageCountPart = $this->pageCount - $this->pageCountLast;
377  $this->revCountPart = $this->revCount - $this->revCountLast;
378 
379  if ( $deltaAll ) {
380  $portion = $this->revCount / $this->maxCount;
381  $eta = $this->startTime + $deltaAll / $portion;
382  $etats = wfTimestamp( TS_DB, intval( $eta ) );
383  if ( $this->fetchCount ) {
384  $fetchRate = 100.0 * $this->prefetchCount / $this->fetchCount;
385  } else {
386  $fetchRate = '-';
387  }
388  $pageRate = $this->pageCount / $deltaAll;
389  $revRate = $this->revCount / $deltaAll;
390  } else {
391  $pageRate = '-';
392  $revRate = '-';
393  $etats = '-';
394  $fetchRate = '-';
395  }
396  if ( $deltaPart ) {
397  if ( $this->fetchCountLast ) {
398  $fetchRatePart = 100.0 * $this->prefetchCountLast / $this->fetchCountLast;
399  } else {
400  $fetchRatePart = '-';
401  }
402  $pageRatePart = $this->pageCountPart / $deltaPart;
403  $revRatePart = $this->revCountPart / $deltaPart;
404  } else {
405  $fetchRatePart = '-';
406  $pageRatePart = '-';
407  $revRatePart = '-';
408  }
409 
410  $dbDomain = WikiMap::getCurrentWikiDbDomain()->getId();
411  $this->progress( sprintf(
412  "%s: %s (ID %d) %d pages (%0.1f|%0.1f/sec all|curr), "
413  . "%d revs (%0.1f|%0.1f/sec all|curr), %0.1f%%|%0.1f%% "
414  . "prefetched (all|curr), ETA %s [max %d]",
415  $now, $dbDomain, $this->ID, $this->pageCount, $pageRate,
416  $pageRatePart, $this->revCount, $revRate, $revRatePart,
417  $fetchRate, $fetchRatePart, $etats, $this->maxCount
418  ) );
419  $this->lastTime = $nowts;
420  $this->revCountLast = $this->revCount;
421  $this->prefetchCountLast = $this->prefetchCount;
422  $this->fetchCountLast = $this->fetchCount;
423  }
424  }
425 
426  private function setTimeExceeded() {
427  $this->timeExceeded = true;
428  }
429 
430  private function checkIfTimeExceeded() {
431  if ( $this->maxTimeAllowed
432  && ( $this->lastTime - $this->timeOfCheckpoint > $this->maxTimeAllowed )
433  ) {
434  return true;
435  }
436 
437  return false;
438  }
439 
440  private function finalOptionCheck() {
441  if ( ( $this->checkpointFiles && !$this->maxTimeAllowed )
442  || ( $this->maxTimeAllowed && !$this->checkpointFiles )
443  ) {
444  throw new MWException( "Options checkpointfile and maxtime must be specified together.\n" );
445  }
446  foreach ( $this->checkpointFiles as $checkpointFile ) {
447  $count = substr_count( $checkpointFile, "%s" );
448  if ( $count != 2 ) {
449  throw new MWException( "Option checkpointfile must contain two '%s' "
450  . "for substitution of first and last pageids, count is $count instead, "
451  . "file is $checkpointFile.\n" );
452  }
453  }
454 
455  if ( $this->checkpointFiles ) {
456  $filenameList = (array)$this->egress->getFilenames();
457  if ( count( $filenameList ) != count( $this->checkpointFiles ) ) {
458  throw new MWException( "One checkpointfile must be specified "
459  . "for each output option, if maxtime is used.\n" );
460  }
461  }
462  }
463 
469  protected function readDump( $input ) {
470  $this->buffer = "";
471  $this->openElement = false;
472  $this->atStart = true;
473  $this->state = "";
474  $this->lastName = "";
475  $this->thisPage = "";
476  $this->thisRev = "";
477  $this->thisRole = null;
478  $this->thisRevModel = null;
479  $this->thisRevFormat = null;
480 
481  $parser = xml_parser_create( "UTF-8" );
482  xml_parser_set_option( $parser, XML_OPTION_CASE_FOLDING, 0 );
483 
484  xml_set_element_handler(
485  $parser,
486  [ $this, 'startElement' ],
487  [ $this, 'endElement' ]
488  );
489  xml_set_character_data_handler( $parser, [ $this, 'characterData' ] );
490 
491  $offset = 0; // for context extraction on error reporting
492  do {
493  if ( $this->checkIfTimeExceeded() ) {
494  $this->setTimeExceeded();
495  }
496  $chunk = fread( $input, $this->bufferSize );
497  if ( !xml_parse( $parser, $chunk, feof( $input ) ) ) {
498  wfDebug( "TextDumpPass::readDump encountered XML parsing error" );
499 
500  $byte = xml_get_current_byte_index( $parser );
501  $msg = wfMessage( 'xml-error-string',
502  'XML import parse failure',
503  xml_get_current_line_number( $parser ),
504  xml_get_current_column_number( $parser ),
505  $byte . ( $chunk === false ? '' : ( '; "' . substr( $chunk, $byte - $offset, 16 ) . '"' ) ),
506  xml_error_string( xml_get_error_code( $parser ) ) )->escaped();
507 
508  xml_parser_free( $parser );
509 
510  throw new MWException( $msg );
511  }
512  $offset += strlen( $chunk );
513  } while ( $chunk !== false && !feof( $input ) );
514  if ( $this->maxTimeAllowed ) {
515  $filenameList = (array)$this->egress->getFilenames();
516  // we wrote some stuff after last checkpoint that needs renamed
517  if ( file_exists( $filenameList[0] ) ) {
518  $newFilenames = [];
519  # we might have just written the header and footer and had no
520  # pages or revisions written... perhaps they were all deleted
521  # there's no pageID 0 so we use that. the caller is responsible
522  # for deciding what to do with a file containing only the
523  # siteinfo information and the mw tags.
524  if ( !$this->firstPageWritten ) {
525  $firstPageID = str_pad( '0', 9, "0", STR_PAD_LEFT );
526  $lastPageID = str_pad( '0', 9, "0", STR_PAD_LEFT );
527  } else {
528  $firstPageID = str_pad( $this->firstPageWritten, 9, "0", STR_PAD_LEFT );
529  $lastPageID = str_pad( $this->lastPageWritten, 9, "0", STR_PAD_LEFT );
530  }
531 
532  $filenameCount = count( $filenameList );
533  for ( $i = 0; $i < $filenameCount; $i++ ) {
534  $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID );
535  $fileinfo = pathinfo( $filenameList[$i] );
536  $newFilenames[] = $fileinfo['dirname'] . '/' . $checkpointNameFilledIn;
537  }
538  $this->egress->closeAndRename( $newFilenames );
539  }
540  }
541  xml_parser_free( $parser );
542 
543  return true;
544  }
545 
555  private function exportTransform( $text, $model, $format = null ) {
556  try {
557  $text = MediaWikiServices::getInstance()
558  ->getContentHandlerFactory()
559  ->getContentHandler( $model )
560  ->exportTransform( $text, $format );
561  } catch ( MWException $ex ) {
562  wfWarn( "Unable to apply export transformation for content model '$model': " .
563  $ex->getMessage() );
564 
565  $this->progress(
566  "Unable to apply export transformation for content model '$model': " .
567  $ex->getMessage()
568  );
569  }
570 
571  return $text;
572  }
573 
594  protected function getText( $id, $model = null, $format = null, $expSize = null ) {
595  if ( !$this->isValidTextId( $id ) ) {
596  $msg = "Skipping bad text id " . $id . " of revision " . $this->thisRev;
597  $this->progress( $msg );
598  return '';
599  }
600 
601  $model = $model ?: null;
602  $prefetchNotTried = true; // Whether or not we already tried to get the text via prefetch.
603  $text = false; // The candidate for a good text. false if no proper value.
604  $failures = 0; // The number of times, this invocation of getText already failed.
605 
606  // The number of times getText failed without yielding a good text in between.
607  static $consecutiveFailedTextRetrievals = 0;
608 
609  $this->fetchCount++;
610 
611  // To allow to simply return on success and do not have to worry about book keeping,
612  // we assume, this fetch works (possible after some retries). Nevertheless, we koop
613  // the old value, so we can restore it, if problems occur (See after the while loop).
614  $oldConsecutiveFailedTextRetrievals = $consecutiveFailedTextRetrievals;
615  $consecutiveFailedTextRetrievals = 0;
616 
617  while ( $failures < $this->maxFailures ) {
618  // As soon as we found a good text for the $id, we will return immediately.
619  // Hence, if we make it past the try catch block, we know that we did not
620  // find a good text.
621 
622  try {
623  // Step 1: Get some text (or reuse from previous iteratuon if checking
624  // for plausibility failed)
625 
626  // Trying to get prefetch, if it has not been tried before
627  // @phan-suppress-next-line PhanSuspiciousValueComparisonInLoop
628  if ( $text === false && isset( $this->prefetch ) && $prefetchNotTried ) {
629  $prefetchNotTried = false;
630  $tryIsPrefetch = true;
631  $text = $this->prefetch->prefetch(
632  (int)$this->thisPage,
633  (int)$this->thisRev,
634  trim( $this->thisRole )
635  );
636 
637  if ( $text === null ) {
638  $text = false;
639  }
640 
641  if ( is_string( $text ) && $model !== null ) {
642  // Apply export transformation to text coming from an old dump.
643  // The purpose of this transformation is to convert up from legacy
644  // formats, which may still be used in the older dump that is used
645  // for pre-fetching. Applying the transformation again should not
646  // interfere with content that is already in the correct form.
647  $text = $this->exportTransform( $text, $model, $format );
648  }
649  }
650 
651  if ( $text === false ) {
652  // Fallback to asking the database
653  $tryIsPrefetch = false;
654  if ( $this->spawn ) {
655  $text = $this->getTextSpawned( $id );
656  } else {
657  $text = $this->getTextDb( $id );
658  }
659 
660  if ( $text !== false && $model !== null ) {
661  // Apply export transformation to text coming from the database.
662  // Prefetched text should already have transformations applied.
663  $text = $this->exportTransform( $text, $model, $format );
664  }
665 
666  // No more checks for texts from DB for now.
667  // If we received something that is not false,
668  // We treat it as good text, regardless of whether it actually is or is not
669  if ( $text !== false ) {
670  return $text;
671  }
672  }
673 
674  if ( $text === false ) {
675  throw new MWException( "Generic error while obtaining text for id " . $id );
676  }
677 
678  // We received a good candidate for the text of $id via some method
679 
680  // Step 2: Checking for plausibility and return the text if it is
681  // plausible
682 
683  if ( $expSize === null || strlen( $text ) == $expSize ) {
684  // @phan-suppress-next-line PhanPossiblyUndeclaredVariable Set when text is not false
685  if ( $tryIsPrefetch ) {
686  $this->prefetchCount++;
687  }
688 
689  return $text;
690  }
691 
692  $text = false;
693  throw new MWException( "Received text is unplausible for id " . $id );
694  } catch ( Exception $e ) {
695  $msg = "getting/checking text " . $id . " failed (" . $e->getMessage()
696  . ") for revision " . $this->thisRev;
697  if ( $failures + 1 < $this->maxFailures ) {
698  $msg .= " (Will retry " . ( $this->maxFailures - $failures - 1 ) . " more times)";
699  }
700  $this->progress( $msg );
701  }
702 
703  // Something went wrong; we did not a text that was plausible :(
704  $failures++;
705 
706  // A failure in a prefetch hit does not warrant resetting db connection etc.
707  // @phan-suppress-next-line PhanPossiblyUndeclaredVariable Control flow is hard to understand here.
708  if ( !$tryIsPrefetch ) {
709  // After backing off for some time, we try to reboot the whole process as
710  // much as possible to not carry over failures from one part to the other
711  // parts
712  sleep( $this->failureTimeout );
713  try {
714  $this->rotateDb();
715  if ( $this->spawn ) {
716  $this->closeSpawn();
717  $this->openSpawn();
718  }
719  } catch ( Exception $e ) {
720  $this->progress( "Rebooting getText infrastructure failed (" . $e->getMessage() . ")" .
721  " Trying to continue anyways" );
722  }
723  }
724  }
725 
726  // Retrieving a good text for $id failed (at least) maxFailures times.
727  // We abort for this $id.
728 
729  // Restoring the consecutive failures, and maybe aborting, if the dump
730  // is too broken.
731  $consecutiveFailedTextRetrievals = $oldConsecutiveFailedTextRetrievals + 1;
732  if ( $consecutiveFailedTextRetrievals > $this->maxConsecutiveFailedTextRetrievals ) {
733  throw new MWException( "Graceful storage failure" );
734  }
735 
736  return "";
737  }
738 
745  private function getTextDb( $id ) {
746  $store = $this->getBlobStore();
747  $address = ( is_int( $id ) || strpos( $id, ':' ) === false )
748  ? SqlBlobStore::makeAddressFromTextId( (int)$id )
749  : $id;
750 
751  try {
752  $text = $store->getBlob( $address );
753 
754  $stripped = str_replace( "\r", "", $text );
755  $normalized = MediaWikiServices::getInstance()->getContentLanguage()
756  ->normalize( $stripped );
757 
758  return $normalized;
759  } catch ( BlobAccessException $ex ) {
760  // XXX: log a warning?
761  return false;
762  }
763  }
764 
769  private function getTextSpawned( $address ) {
770  AtEase::suppressWarnings();
771  if ( !$this->spawnProc ) {
772  // First time?
773  $this->openSpawn();
774  }
775  $text = $this->getTextSpawnedOnce( $address );
776  AtEase::restoreWarnings();
777 
778  return $text;
779  }
780 
781  protected function openSpawn() {
782  global $IP;
783 
784  $wiki = WikiMap::getCurrentWikiId();
785  if ( count( $this->php ) == 2 ) {
786  $mwscriptpath = $this->php[1];
787  } else {
788  $mwscriptpath = "$IP/../multiversion/MWScript.php";
789  }
790  if ( file_exists( $mwscriptpath ) ) {
791  $cmd = implode( " ",
792  array_map( [ Shell::class, 'escape' ],
793  [
794  $this->php[0],
795  $mwscriptpath,
796  "fetchText.php",
797  '--wiki', $wiki ] ) );
798  } else {
799  $cmd = implode( " ",
800  array_map( [ Shell::class, 'escape' ],
801  [
802  $this->php[0],
803  "$IP/maintenance/fetchText.php",
804  '--wiki', $wiki ] ) );
805  }
806  $spec = [
807  0 => [ "pipe", "r" ],
808  1 => [ "pipe", "w" ],
809  2 => [ "file", "/dev/null", "a" ] ];
810  $pipes = [];
811 
812  $this->progress( "Spawning database subprocess: $cmd" );
813  $this->spawnProc = proc_open( $cmd, $spec, $pipes );
814  if ( !$this->spawnProc ) {
815  $this->progress( "Subprocess spawn failed." );
816 
817  return false;
818  }
819  [
820  $this->spawnWrite, // -> stdin
821  $this->spawnRead, // <- stdout
822  ] = $pipes;
823 
824  return true;
825  }
826 
827  private function closeSpawn() {
828  AtEase::suppressWarnings();
829  if ( $this->spawnRead ) {
830  fclose( $this->spawnRead );
831  }
832  $this->spawnRead = null;
833  if ( $this->spawnWrite ) {
834  fclose( $this->spawnWrite );
835  }
836  $this->spawnWrite = null;
837  if ( $this->spawnErr ) {
838  fclose( $this->spawnErr );
839  }
840  $this->spawnErr = false;
841  if ( $this->spawnProc ) {
842  pclose( $this->spawnProc );
843  }
844  $this->spawnProc = false;
845  AtEase::restoreWarnings();
846  }
847 
852  private function getTextSpawnedOnce( $address ) {
853  if ( is_int( $address ) || intval( $address ) ) {
854  $address = SqlBlobStore::makeAddressFromTextId( (int)$address );
855  }
856 
857  $ok = fwrite( $this->spawnWrite, "$address\n" );
858  // $this->progress( ">> $id" );
859  if ( !$ok ) {
860  return false;
861  }
862 
863  $ok = fflush( $this->spawnWrite );
864  // $this->progress( ">> [flush]" );
865  if ( !$ok ) {
866  return false;
867  }
868 
869  // check that the text address they are sending is the one we asked for
870  // this avoids out of sync revision text errors we have encountered in the past
871  $newAddress = fgets( $this->spawnRead );
872  if ( $newAddress === false ) {
873  return false;
874  }
875  $newAddress = trim( $newAddress );
876  if ( strpos( $newAddress, ':' ) === false ) {
877  $newAddress = SqlBlobStore::makeAddressFromTextId( intval( $newAddress ) );
878  }
879 
880  if ( $newAddress !== $address ) {
881  return false;
882  }
883 
884  $len = fgets( $this->spawnRead );
885  // $this->progress( "<< " . trim( $len ) );
886  if ( $len === false ) {
887  return false;
888  }
889 
890  $nbytes = intval( $len );
891  // actual error, not zero-length text
892  if ( $nbytes < 0 ) {
893  return false;
894  }
895 
896  $text = "";
897 
898  // Subprocess may not send everything at once, we have to loop.
899  while ( $nbytes > strlen( $text ) ) {
900  $buffer = fread( $this->spawnRead, $nbytes - strlen( $text ) );
901  if ( $buffer === false ) {
902  break;
903  }
904  $text .= $buffer;
905  }
906 
907  $gotbytes = strlen( $text );
908  if ( $gotbytes != $nbytes ) {
909  $this->progress( "Expected $nbytes bytes from database subprocess, got $gotbytes " );
910 
911  return false;
912  }
913 
914  // Do normalization in the dump thread...
915  $stripped = str_replace( "\r", "", $text );
916  $normalized = MediaWikiServices::getInstance()->getContentLanguage()->
917  normalize( $stripped );
918 
919  return $normalized;
920  }
921 
922  protected function startElement( $parser, $name, $attribs ) {
923  $this->checkpointJustWritten = false;
924 
925  $this->clearOpenElement( null );
926  $this->lastName = $name;
927 
928  if ( $name == 'revision' ) {
929  $this->state = $name;
930  $this->egress->writeOpenPage( null, $this->buffer );
931  $this->buffer = "";
932  } elseif ( $name == 'page' ) {
933  $this->state = $name;
934  if ( $this->atStart ) {
935  $this->egress->writeOpenStream( $this->buffer );
936  $this->buffer = "";
937  $this->atStart = false;
938  }
939  } elseif ( $name === 'mediawiki' ) {
940  if ( isset( $attribs['version'] ) ) {
941  if ( $attribs['version'] !== $this->schemaVersion ) {
942  throw new MWException( 'Mismatching schema version. '
943  . 'Use the --schema-version option to set the output schema version to '
944  . 'the version declared by the stub file, namely ' . $attribs['version'] );
945  }
946  }
947  }
948 
949  if ( $name == "text" && ( isset( $attribs['id'] ) || isset( $attribs['location'] ) ) ) {
950  $id = $attribs['location'] ?? $attribs['id'];
951  $model = trim( $this->thisRevModel );
952  $format = trim( $this->thisRevFormat );
953 
954  $model = $model === '' ? null : $model;
955  $format = $format === '' ? null : $format;
956  $expSize = !empty( $attribs['bytes'] ) && $model === CONTENT_MODEL_WIKITEXT
957  ? (int)$attribs['bytes'] : null;
958 
959  $text = $this->getText( $id, $model, $format, $expSize );
960 
961  unset( $attribs['id'] );
962  unset( $attribs['location'] );
963  if ( strlen( $text ) > 0 ) {
964  $attribs['xml:space'] = 'preserve';
965  }
966 
967  $this->openElement = [ $name, $attribs ];
968  if ( strlen( $text ) > 0 ) {
969  $this->characterData( $parser, $text );
970  }
971  } else {
972  $this->openElement = [ $name, $attribs ];
973  }
974  }
975 
976  protected function endElement( $parser, $name ) {
977  $this->checkpointJustWritten = false;
978 
979  if ( $this->openElement ) {
980  $this->clearOpenElement( "" );
981  } else {
982  $this->buffer .= "</$name>";
983  }
984 
985  if ( $name == 'revision' ) {
986  $this->egress->writeRevision( null, $this->buffer );
987  $this->buffer = "";
988  $this->thisRev = "";
989  $this->thisRole = null;
990  $this->thisRevModel = null;
991  $this->thisRevFormat = null;
992  } elseif ( $name == 'page' ) {
993  if ( !$this->firstPageWritten ) {
994  $this->firstPageWritten = trim( $this->thisPage );
995  }
996  $this->lastPageWritten = trim( $this->thisPage );
997  if ( $this->timeExceeded ) {
998  $this->egress->writeClosePage( $this->buffer );
999  // nasty hack, we can't just write the chardata after the
1000  // page tag, it will include leading blanks from the next line
1001  $this->egress->sink->write( "\n" );
1002 
1003  $this->buffer = $this->xmlwriterobj->closeStream();
1004  $this->egress->writeCloseStream( $this->buffer );
1005 
1006  $this->buffer = "";
1007  $this->thisPage = "";
1008  // this could be more than one file if we had more than one output arg
1009 
1010  $filenameList = (array)$this->egress->getFilenames();
1011  $newFilenames = [];
1012  $firstPageID = str_pad( $this->firstPageWritten, 9, "0", STR_PAD_LEFT );
1013  $lastPageID = str_pad( $this->lastPageWritten, 9, "0", STR_PAD_LEFT );
1014  $filenamesCount = count( $filenameList );
1015  for ( $i = 0; $i < $filenamesCount; $i++ ) {
1016  $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID );
1017  $fileinfo = pathinfo( $filenameList[$i] );
1018  $newFilenames[] = $fileinfo['dirname'] . '/' . $checkpointNameFilledIn;
1019  }
1020  $this->egress->closeRenameAndReopen( $newFilenames );
1021  $this->buffer = $this->xmlwriterobj->openStream();
1022  $this->timeExceeded = false;
1023  $this->timeOfCheckpoint = $this->lastTime;
1024  $this->firstPageWritten = false;
1025  $this->checkpointJustWritten = true;
1026  } else {
1027  $this->egress->writeClosePage( $this->buffer );
1028  $this->buffer = "";
1029  $this->thisPage = "";
1030  }
1031  } elseif ( $name == 'mediawiki' ) {
1032  $this->egress->writeCloseStream( $this->buffer );
1033  $this->buffer = "";
1034  }
1035  }
1036 
1037  protected function characterData( $parser, $data ) {
1038  $this->clearOpenElement( null );
1039  if ( $this->lastName == "id" ) {
1040  if ( $this->state == "revision" ) {
1041  $this->thisRev .= $data;
1042  $this->thisRole = SlotRecord::MAIN;
1043  } elseif ( $this->state == "page" ) {
1044  $this->thisPage .= $data;
1045  }
1046  } elseif ( $this->lastName == "model" ) {
1047  $this->thisRevModel .= $data;
1048  } elseif ( $this->lastName == "format" ) {
1049  $this->thisRevFormat .= $data;
1050  } elseif ( $this->lastName == "content" ) {
1051  $this->thisRole = "";
1052  $this->thisRevModel = "";
1053  $this->thisRevFormat = "";
1054  } elseif ( $this->lastName == "role" ) {
1055  $this->thisRole .= $data;
1056  }
1057 
1058  // have to skip the newline left over from closepagetag line of
1059  // end of checkpoint files. nasty hack!!
1060  if ( $this->checkpointJustWritten ) {
1061  if ( $data[0] == "\n" ) {
1062  $data = substr( $data, 1 );
1063  }
1064  $this->checkpointJustWritten = false;
1065  }
1066  $this->buffer .= htmlspecialchars( $data, ENT_COMPAT );
1067  }
1068 
1069  protected function clearOpenElement( $style ) {
1070  if ( $this->openElement ) {
1071  $this->buffer .= Xml::element( $this->openElement[0], $this->openElement[1], $style );
1072  $this->openElement = false;
1073  }
1074  }
1075 
1076  private function isValidTextId( $id ) {
1077  if ( preg_match( '/:/', $id ) ) {
1078  return $id !== 'tt:0';
1079  } elseif ( preg_match( '/^\d+$/', $id ) ) {
1080  return intval( $id ) > 0;
1081  }
1082 
1083  return false;
1084  }
1085 
1086 }
const CONTENT_MODEL_WIKITEXT
Definition: Defines.php:211
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
wfWarn( $msg, $callerOffset=1, $level=E_USER_NOTICE)
Send a warning either to the debug log or in a PHP error depending on $wgDevelopmentWarnings.
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
wfMessage( $key,... $params)
This is the function for getting translated interface messages.
if(!defined( 'MEDIAWIKI')) if(ini_get( 'mbstring.func_overload')) if(!defined( 'MW_ENTRY_POINT')) global $IP
Environment checks.
Definition: Setup.php:93
float $startTime
progress( $string)
IMaintainableDatabase null $forcedDb
The dependency-injected database to use.
report( $final=false)
Readahead helper for making large MediaWiki data dumps; reads in a previous XML dump to sequentially ...
Definition: BaseDump.php:44
MediaWiki exception.
Definition: MWException.php:32
output( $out, $channel=null)
Throw some output to the user.
hasOption( $name)
Checks to see if a particular option was set.
addDescription( $text)
Set the description text.
loadWithArgv( $argv)
Load params and arguments from a given array of command-line arguments.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
Service locator for MediaWiki core services.
Value object representing a content slot associated with a page revision.
Definition: SlotRecord.php:40
Builder class for constructing a Config object from a set of sources during bootstrap.
Executes shell commands.
Definition: Shell.php:46
Exception representing a failure to access a data blob.
Service for storing and loading Content objects representing revision data blobs.
Helper tools for dealing with other locally-hosted wikis.
Definition: WikiMap.php:33
showReport()
Overridden to include prefetch ratio if enabled.
int $bufferSize
In bytes.
string false $lastPageWritten
endElement( $parser, $name)
clearOpenElement( $style)
string false $firstPageWritten
startElement( $parser, $name, $attribs)
finalSetup(SettingsBuilder $settingsBuilder=null)
Handle some last-minute setup here.
BaseDump null $prefetch
int $maxConsecutiveFailedTextRetrievals
int $failureTimeout
Seconds to sleep after db failure.
processOptions()
Processes arguments and sets $this->$sink accordingly.
getText( $id, $model=null, $format=null, $expSize=null)
Tries to load revision text.
characterData( $parser, $data)
resource false $spawnProc
resource null $spawnRead
initProgress( $history=WikiExporter::FULL)
Initialise starting time and maximum revision count.
execute()
Do the actual work.
__construct( $args=null)
resource false $spawnErr
dump( $history, $text=WikiExporter::TEXT)
string[] $checkpointFiles
XmlDumpWriter false $xmlwriterobj
int $maxTimeAllowed
when we spend more than maxTimeAllowed seconds on this run, we continue processing until we write out...
IMaintainableDatabase $db
resource null $spawnWrite
rotateDb()
Drop the database connection $this->db and try to get a new one.
const WRITE_CONTENT
Output serialized revision content.
static element( $element, $attribs=null, $contents='', $allowShortTag=true)
Format an XML element with given attributes and, optionally, text content.
Definition: Xml.php:44
Service for loading and storing data blobs.
Definition: BlobStore.php:33
Advanced database interface for IDatabase handles that include maintenance methods.
const DB_REPLICA
Definition: defines.php:26