28require_once __DIR__ .
'/BackupDumper.php';
29require_once __DIR__ .
'/../../includes/export/WikiExporter.php';
73 protected $php = [ PHP_BINARY ];
117 parent::__construct();
120This script postprocesses XML dumps from dumpBackup.php to add
121page text which was stubbed out (
using --stub).
123XML input is accepted on stdin.
127 $this->stderr = fopen(
"php://stderr",
"wt" );
129 $this->
addOption(
'stub',
'To load a compressed stub dump instead of stdin. ' .
130 'Specify as --stub=<type>:<file>.',
false,
true );
131 $this->
addOption(
'prefetch',
'Use a prior dump file as a text source, to savepressure on the ' .
132 'database. (Requires the XMLReader extension). Specify as --prefetch=<type>:<file>',
134 $this->
addOption(
'maxtime',
'Write out checkpoint file after this many minutes (writing' .
135 'out complete page, closing xml file properly, and opening new one' .
136 'with header). This option requires the checkpointfile option.',
false,
true );
137 $this->
addOption(
'checkpointfile',
'Use this string for checkpoint filenames,substituting ' .
138 'first pageid written for the first %s (required) and the last pageid written for the ' .
139 'second %s if it exists.',
false,
true,
false,
true );
140 $this->
addOption(
'quiet',
'Don\'t dump status reports to stderr.' );
141 $this->
addOption(
'full',
'Dump all revisions of every page' );
142 $this->
addOption(
'current',
'Base ETA on number of pages in database instead of all revisions' );
143 $this->
addOption(
'spawn',
'Spawn a subprocess for loading text records, optionally specify ' .
144 'php[,mwscript] paths' );
145 $this->
addOption(
'buffersize',
'Buffer size in bytes to use for reading the stub. ' .
146 '(Default: 512 KiB, Minimum: 4 KiB)',
false,
true );
155 parent::finalSetup();
164 return MediaWikiServices::getInstance()->getBlobStore();
173 parent::processOptions();
175 if ( $this->
hasOption(
'buffersize' ) ) {
176 $this->bufferSize = max( intval( $this->
getOption(
'buffersize' ) ), 4 * 1024 );
181 $this->prefetch =
new BaseDump( $url );
189 $this->maxTimeAllowed = intval( $this->
getOption(
'maxtime' ) ) * 60;
192 if ( $this->
hasOption(
'checkpointfile' ) ) {
193 $this->checkpointFiles = $this->
getOption(
'checkpointfile' );
208 $this->php = explode(
',', $val, 2 );
227 if ( isset( $this->lb ) ) {
228 $this->lb->closeAll( __METHOD__ );
232 if ( $this->forcedDb !==
null ) {
238 if ( isset( $this->db ) && $this->db->isOpen() ) {
239 throw new MWException(
'DB is set and has not been closed by the Load Balancer' );
249 $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
250 $this->lb = $lbFactory->newMainLB();
251 }
catch ( Exception $e ) {
253 .
" rotating DB failed to obtain new load balancer (" . $e->getMessage() .
")" );
257 $this->db = $this->lb->getMaintenanceConnectionRef(
DB_REPLICA,
'dump' );
258 }
catch ( Exception $e ) {
260 .
" rotating DB failed to obtain new database (" . $e->getMessage() .
")" );
265 parent::initProgress();
272 if ( ini_get(
'display_errors' ) ) {
273 ini_set(
'display_errors',
'stderr' );
285 }
catch ( Exception $e ) {
288 $this->
progress(
"Getting initial DB connection failed (" .
289 $e->getMessage() .
")" );
298 $this->xmlwriterobj =
new XmlDumpWriter( XmlDumpWriter::WRITE_CONTENT, $this->schemaVersion );
300 $input = fopen( $this->input,
"rt" );
303 if ( $this->spawnProc ) {
311 $split = explode(
':', $opt, 2 );
314 if ( count( $split ) === 2 ) {
317 $fileURIs = explode(
';', $param );
319 foreach ( $fileURIs as $URI ) {
325 $newURI =
"compress.zlib://$URI";
328 $newURI =
"compress.bzip2://$URI";
331 $newURI =
"mediawiki.compress.7z://$URI";
336 $newFileURIs[] = $newURI;
338 $val = implode(
';', $newFileURIs );
347 if ( !$this->prefetch ) {
348 parent::showReport();
353 if ( $this->reporting ) {
355 $nowts = microtime(
true );
363 $eta = $this->startTime + $deltaAll / $portion;
365 if ( $this->fetchCount ) {
370 $pageRate = $this->pageCount / $deltaAll;
371 $revRate = $this->
revCount / $deltaAll;
379 if ( $this->fetchCountLast ) {
382 $fetchRatePart =
'-';
384 $pageRatePart = $this->pageCountPart / $deltaPart;
385 $revRatePart = $this->revCountPart / $deltaPart;
387 $fetchRatePart =
'-';
392 $dbDomain = WikiMap::getCurrentWikiDbDomain()->getId();
394 "%s: %s (ID %d) %d pages (%0.1f|%0.1f/sec all|curr), "
395 .
"%d revs (%0.1f|%0.1f/sec all|curr), %0.1f%%|%0.1f%% "
396 .
"prefetched (all|curr), ETA %s [max %d]",
397 $now, $dbDomain, $this->ID, $this->pageCount, $pageRate,
398 $pageRatePart, $this->
revCount, $revRate, $revRatePart,
399 $fetchRate, $fetchRatePart, $etats, $this->maxCount
401 $this->lastTime = $nowts;
409 $this->timeExceeded =
true;
413 if ( $this->maxTimeAllowed
414 && ( $this->lastTime - $this->timeOfCheckpoint > $this->maxTimeAllowed )
423 if ( ( $this->checkpointFiles && !$this->maxTimeAllowed )
424 || ( $this->maxTimeAllowed && !$this->checkpointFiles )
426 throw new MWException(
"Options checkpointfile and maxtime must be specified together.\n" );
428 foreach ( $this->checkpointFiles as $checkpointFile ) {
429 $count = substr_count( $checkpointFile,
"%s" );
431 throw new MWException(
"Option checkpointfile must contain two '%s' "
432 .
"for substitution of first and last pageids, count is $count instead, "
433 .
"file is $checkpointFile.\n" );
437 if ( $this->checkpointFiles ) {
438 $filenameList = (array)$this->egress->getFilenames();
439 if ( count( $filenameList ) != count( $this->checkpointFiles ) ) {
440 throw new MWException(
"One checkpointfile must be specified "
441 .
"for each output option, if maxtime is used.\n" );
453 $this->openElement =
false;
454 $this->atStart =
true;
456 $this->lastName =
"";
459 $this->thisRole =
null;
460 $this->thisRevModel =
null;
461 $this->thisRevFormat =
null;
463 $parser = xml_parser_create(
"UTF-8" );
464 xml_parser_set_option( $parser, XML_OPTION_CASE_FOLDING,
false );
466 xml_set_element_handler(
468 [ $this,
'startElement' ],
469 [ $this,
'endElement' ]
471 xml_set_character_data_handler( $parser, [ $this,
'characterData' ] );
478 $chunk = fread(
$input, $this->bufferSize );
479 if ( !xml_parse( $parser, $chunk, feof(
$input ) ) ) {
480 wfDebug(
"TextDumpPass::readDump encountered XML parsing error" );
482 $byte = xml_get_current_byte_index( $parser );
484 'XML import parse failure',
485 xml_get_current_line_number( $parser ),
486 xml_get_current_column_number( $parser ),
487 $byte . ( $chunk ===
false ?
'' : (
'; "' . substr( $chunk, $byte - $offset, 16 ) .
'"' ) ),
488 xml_error_string( xml_get_error_code( $parser ) ) )->escaped();
490 xml_parser_free( $parser );
494 $offset += strlen( $chunk );
495 }
while ( $chunk !==
false && !feof(
$input ) );
496 if ( $this->maxTimeAllowed ) {
497 $filenameList = (array)$this->egress->getFilenames();
499 if ( file_exists( $filenameList[0] ) ) {
501 # we might have just written the header and footer and had no
502 # pages or revisions written... perhaps they were all deleted
503 # there's no pageID 0 so we use that. the caller is responsible
504 # for deciding what to do with a file containing only the
505 # siteinfo information and the mw tags.
506 if ( !$this->firstPageWritten ) {
507 $firstPageID = str_pad( 0, 9,
"0", STR_PAD_LEFT );
508 $lastPageID = str_pad( 0, 9,
"0", STR_PAD_LEFT );
510 $firstPageID = str_pad( $this->firstPageWritten, 9,
"0", STR_PAD_LEFT );
511 $lastPageID = str_pad( $this->lastPageWritten, 9,
"0", STR_PAD_LEFT );
514 $filenameCount = count( $filenameList );
515 for ( $i = 0; $i < $filenameCount; $i++ ) {
516 $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID );
517 $fileinfo = pathinfo( $filenameList[$i] );
518 $newFilenames[] =
$fileinfo[
'dirname'] .
'/' . $checkpointNameFilledIn;
520 $this->egress->closeAndRename( $newFilenames );
523 xml_parser_free( $parser );
539 $text = MediaWikiServices::getInstance()
540 ->getContentHandlerFactory()
541 ->getContentHandler( $model )
542 ->exportTransform( $text, $format );
545 wfWarn(
"Unable to apply export transformation for content model '$model': " .
549 "Unable to apply export transformation for content model '$model': " .
577 protected function getText( $id, $model =
null, $format =
null, $expSize =
null ) {
579 $msg =
"Skipping bad text id " . $id .
" of revision " .
$this->thisRev;
584 $model = $model ?:
null;
585 $prefetchNotTried =
true;
590 static $consecutiveFailedTextRetrievals = 0;
597 $oldConsecutiveFailedTextRetrievals = $consecutiveFailedTextRetrievals;
598 $consecutiveFailedTextRetrievals = 0;
600 while ( $failures < $this->maxFailures ) {
611 if ( $text ===
false && isset( $this->prefetch ) && $prefetchNotTried ) {
612 $prefetchNotTried =
false;
613 $tryIsPrefetch =
true;
614 $text = $this->prefetch->prefetch(
615 (
int)$this->thisPage,
617 trim( $this->thisRole )
620 if ( $text ===
null ) {
624 if ( is_string( $text ) && $model !==
null ) {
634 if ( $text ===
false ) {
636 $tryIsPrefetch =
false;
637 if ( $this->spawn ) {
643 if ( $text !==
false && $model !==
null ) {
652 if ( $text !==
false ) {
657 if ( $text ===
false ) {
658 throw new MWException(
"Generic error while obtaining text for id " . $id );
666 if ( $expSize ===
null || strlen( $text ) == $expSize ) {
667 if ( $tryIsPrefetch ) {
668 $this->prefetchCount++;
675 throw new MWException(
"Received text is unplausible for id " . $id );
676 }
catch ( Exception $e ) {
677 $msg =
"getting/checking text " . $id .
" failed (" . $e->getMessage()
679 if ( $failures + 1 < $this->maxFailures ) {
680 $msg .=
" (Will retry " . ( $this->maxFailures - $failures - 1 ) .
" more times)";
689 if ( !$tryIsPrefetch ) {
693 sleep( $this->failureTimeout );
696 if ( $this->spawn ) {
700 }
catch ( Exception $e ) {
701 $this->
progress(
"Rebooting getText infrastructure failed (" . $e->getMessage() .
")" .
702 " Trying to continue anyways" );
712 $consecutiveFailedTextRetrievals = $oldConsecutiveFailedTextRetrievals + 1;
713 if ( $consecutiveFailedTextRetrievals > $this->maxConsecutiveFailedTextRetrievals ) {
714 throw new MWException(
"Graceful storage failure" );
728 $address = ( is_int( $id ) || strpos( $id,
':' ) === false )
729 ? SqlBlobStore::makeAddressFromTextId( (
int)$id )
733 $text = $store->getBlob( $address );
735 $stripped = str_replace(
"\r",
"", $text );
736 $normalized = MediaWikiServices::getInstance()->getContentLanguage()
737 ->normalize( $stripped );
751 Wikimedia\suppressWarnings();
752 if ( !$this->spawnProc ) {
757 Wikimedia\restoreWarnings();
765 $wiki = WikiMap::getCurrentWikiId();
766 if ( count( $this->php ) == 2 ) {
767 $mwscriptpath = $this->php[1];
769 $mwscriptpath =
"$IP/../multiversion/MWScript.php";
771 if ( file_exists( $mwscriptpath ) ) {
773 array_map( [ Shell::class,
'escape' ],
778 '--wiki', $wiki ] ) );
781 array_map( [ Shell::class,
'escape' ],
784 "$IP/maintenance/fetchText.php",
785 '--wiki', $wiki ] ) );
788 0 => [
"pipe",
"r" ],
789 1 => [
"pipe",
"w" ],
790 2 => [
"file",
"/dev/null",
"a" ] ];
793 $this->
progress(
"Spawning database subprocess: $cmd" );
794 $this->spawnProc = proc_open( $cmd, $spec, $pipes );
795 if ( !$this->spawnProc ) {
796 $this->
progress(
"Subprocess spawn failed." );
809 Wikimedia\suppressWarnings();
810 if ( $this->spawnRead ) {
811 fclose( $this->spawnRead );
813 $this->spawnRead =
null;
814 if ( $this->spawnWrite ) {
815 fclose( $this->spawnWrite );
817 $this->spawnWrite =
null;
818 if ( $this->spawnErr ) {
819 fclose( $this->spawnErr );
821 $this->spawnErr =
false;
822 if ( $this->spawnProc ) {
823 pclose( $this->spawnProc );
825 $this->spawnProc =
false;
826 Wikimedia\restoreWarnings();
834 if ( is_int( $address ) || intval( $address ) ) {
835 $address = SqlBlobStore::makeAddressFromTextId( (
int)$address );
838 $ok = fwrite( $this->spawnWrite,
"$address\n" );
844 $ok = fflush( $this->spawnWrite );
852 $newAddress = fgets( $this->spawnRead );
853 if ( $newAddress ===
false ) {
856 $newAddress = trim( $newAddress );
857 if ( strpos( $newAddress,
':' ) ===
false ) {
858 $newAddress = SqlBlobStore::makeAddressFromTextId( intval( $newAddress ) );
861 if ( $newAddress !== $address ) {
865 $len = fgets( $this->spawnRead );
867 if ( $len ===
false ) {
871 $nbytes = intval( $len );
880 while ( $nbytes > strlen( $text ) ) {
881 $buffer = fread( $this->spawnRead, $nbytes - strlen( $text ) );
888 $gotbytes = strlen( $text );
889 if ( $gotbytes != $nbytes ) {
890 $this->
progress(
"Expected $nbytes bytes from database subprocess, got $gotbytes " );
896 $stripped = str_replace(
"\r",
"", $text );
897 $normalized = MediaWikiServices::getInstance()->getContentLanguage()->
898 normalize( $stripped );
904 $this->checkpointJustWritten =
false;
907 $this->lastName = $name;
909 if ( $name ==
'revision' ) {
910 $this->state = $name;
911 $this->egress->writeOpenPage(
null, $this->buffer );
913 } elseif ( $name ==
'page' ) {
914 $this->state = $name;
915 if ( $this->atStart ) {
916 $this->egress->writeOpenStream( $this->buffer );
918 $this->atStart =
false;
920 } elseif ( $name ===
'mediawiki' ) {
921 if ( isset( $attribs[
'version'] ) ) {
922 if ( $attribs[
'version'] !== $this->schemaVersion ) {
923 throw new MWException(
'Mismatching schema version. '
924 .
'Use the --schema-version option to set the output schema version to '
925 .
'the version declared by the stub file, namely ' . $attribs[
'version'] );
930 if ( $name ==
"text" && ( isset( $attribs[
'id'] ) || isset( $attribs[
'location'] ) ) ) {
931 $id = $attribs[
'location'] ?? $attribs[
'id'];
932 $model = trim( $this->thisRevModel );
933 $format = trim( $this->thisRevFormat );
935 $model = $model ===
'' ? null : $model;
936 $format = $format ===
'' ? null : $format;
938 ? (int)$attribs[
'bytes'] :
null;
940 $text = $this->
getText( $id, $model, $format, $expSize );
942 unset( $attribs[
'id'] );
943 unset( $attribs[
'location'] );
944 if ( strlen( $text ) > 0 ) {
945 $attribs[
'xml:space'] =
'preserve';
948 $this->openElement = [ $name, $attribs ];
949 if ( strlen( $text ) > 0 ) {
953 $this->openElement = [ $name, $attribs ];
958 $this->checkpointJustWritten =
false;
960 if ( $this->openElement ) {
963 $this->buffer .=
"</$name>";
966 if ( $name ==
'revision' ) {
967 $this->egress->writeRevision(
null, $this->buffer );
970 $this->thisRole =
null;
971 $this->thisRevModel =
null;
972 $this->thisRevFormat =
null;
973 } elseif ( $name ==
'page' ) {
974 if ( !$this->firstPageWritten ) {
975 $this->firstPageWritten = trim( $this->thisPage );
977 $this->lastPageWritten = trim( $this->thisPage );
978 if ( $this->timeExceeded ) {
979 $this->egress->writeClosePage( $this->buffer );
982 $this->egress->sink->write(
"\n" );
984 $this->buffer = $this->xmlwriterobj->closeStream();
985 $this->egress->writeCloseStream( $this->buffer );
988 $this->thisPage =
"";
991 $filenameList = (array)$this->egress->getFilenames();
993 $firstPageID = str_pad( $this->firstPageWritten, 9,
"0", STR_PAD_LEFT );
994 $lastPageID = str_pad( $this->lastPageWritten, 9,
"0", STR_PAD_LEFT );
995 $filenamesCount = count( $filenameList );
996 for ( $i = 0; $i < $filenamesCount; $i++ ) {
997 $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID );
998 $fileinfo = pathinfo( $filenameList[$i] );
999 $newFilenames[] =
$fileinfo[
'dirname'] .
'/' . $checkpointNameFilledIn;
1001 $this->egress->closeRenameAndReopen( $newFilenames );
1002 $this->buffer = $this->xmlwriterobj->openStream();
1003 $this->timeExceeded =
false;
1005 $this->firstPageWritten =
false;
1006 $this->checkpointJustWritten =
true;
1008 $this->egress->writeClosePage( $this->buffer );
1010 $this->thisPage =
"";
1012 } elseif ( $name ==
'mediawiki' ) {
1013 $this->egress->writeCloseStream( $this->buffer );
1020 if ( $this->lastName ==
"id" ) {
1021 if ( $this->state ==
"revision" ) {
1022 $this->thisRev .= $data;
1023 $this->thisRole = SlotRecord::MAIN;
1024 } elseif ( $this->state ==
"page" ) {
1025 $this->thisPage .= $data;
1027 } elseif ( $this->lastName ==
"model" ) {
1028 $this->thisRevModel .= $data;
1029 } elseif ( $this->lastName ==
"format" ) {
1030 $this->thisRevFormat .= $data;
1031 } elseif ( $this->lastName ==
"content" ) {
1032 $this->thisRole =
"";
1033 $this->thisRevModel =
"";
1034 $this->thisRevFormat =
"";
1035 } elseif ( $this->lastName ==
"role" ) {
1036 $this->thisRole .= $data;
1041 if ( $this->checkpointJustWritten ) {
1042 if ( $data[0] ==
"\n" ) {
1043 $data = substr( $data, 1 );
1045 $this->checkpointJustWritten =
false;
1047 $this->buffer .= htmlspecialchars( $data );
1051 if ( $this->openElement ) {
1052 $this->buffer .= Xml::element( $this->openElement[0], $this->openElement[1], $style );
1053 $this->openElement =
false;
1058 if ( preg_match(
'/:/', $id ) ) {
1059 return $id !==
'tt:0';
1060 } elseif ( preg_match(
'/^\d+$/', $id ) ) {
1061 return intval( $id ) > 0;
const CONTENT_MODEL_WIKITEXT
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
wfWarn( $msg, $callerOffset=1, $level=E_USER_NOTICE)
Send a warning either to the debug log or in a PHP error depending on $wgDevelopmentWarnings.
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
wfMessage( $key,... $params)
This is the function for getting translated interface messages.
IMaintainableDatabase null $forcedDb
The dependency-injected database to use.
Readahead helper for making large MediaWiki data dumps; reads in a previous XML dump to sequentially ...
output( $out, $channel=null)
Throw some output to the user.
hasOption( $name)
Checks to see if a particular option was set.
addDescription( $text)
Set the description text.
loadWithArgv( $argv)
Load params and arguments from a given array of command-line arguments.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
$maxConsecutiveFailedTextRetrievals
showReport()
Overridden to include prefetch ratio if enabled.
getTextDb( $id)
Loads the serialized content from storage.
endElement( $parser, $name)
clearOpenElement( $style)
getTextSpawned( $address)
exportTransform( $text, $model, $format=null)
Applies applicable export transformations to $text.
startElement( $parser, $name, $attribs)
getTextSpawnedOnce( $address)
processOptions()
Processes arguments and sets $this->$sink accordingly.
getText( $id, $model=null, $format=null, $expSize=null)
Tries to load revision text.
characterData( $parser, $data)
initProgress( $history=WikiExporter::FULL)
Initialise starting time and maximum revision count.
execute()
Do the actual work.
dump( $history, $text=WikiExporter::TEXT)
string[] $checkpointFiles
int $maxTimeAllowed
when we spend more than maxTimeAllowed seconds on this run, we continue processing until we write out...
bool XmlDumpWriter $xmlwriterobj
IMaintainableDatabase $db
finalSetup()
Handle some last-minute setup here.
rotateDb()
Drop the database connection $this->db and try to get a new one.
Advanced database interface for IDatabase handles that include maintenance methods.