28require_once __DIR__ .
'/BackupDumper.php';
29require_once __DIR__ .
'/../../includes/export/WikiExporter.php';
38use Wikimedia\AtEase\AtEase;
52 private $thisRole =
null;
85 protected $php = [ PHP_BINARY ];
134 parent::__construct();
137This script postprocesses XML dumps from dumpBackup.php to add
138page text which was stubbed out (
using --stub).
140XML input is accepted on stdin.
144 $this->stderr = fopen(
"php://stderr",
"wt" );
146 $this->
addOption(
'stub',
'To load a compressed stub dump instead of stdin. ' .
147 'Specify as --stub=<type>:<file>.',
false,
true );
148 $this->
addOption(
'prefetch',
'Use a prior dump file as a text source, to savepressure on the ' .
149 'database. (Requires the XMLReader extension). Specify as --prefetch=<type>:<file>',
151 $this->
addOption(
'maxtime',
'Write out checkpoint file after this many minutes (writing' .
152 'out complete page, closing xml file properly, and opening new one' .
153 'with header). This option requires the checkpointfile option.',
false,
true );
154 $this->
addOption(
'checkpointfile',
'Use this string for checkpoint filenames,substituting ' .
155 'first pageid written for the first %s (required) and the last pageid written for the ' .
156 'second %s if it exists.',
false,
true,
false,
true );
157 $this->
addOption(
'quiet',
'Don\'t dump status reports to stderr.' );
158 $this->
addOption(
'full',
'Dump all revisions of every page' );
159 $this->
addOption(
'current',
'Base ETA on number of pages in database instead of all revisions' );
160 $this->
addOption(
'spawn',
'Spawn a subprocess for loading text records, optionally specify ' .
161 'php[,mwscript] paths' );
162 $this->
addOption(
'buffersize',
'Buffer size in bytes to use for reading the stub. ' .
163 '(Default: 512 KiB, Minimum: 4 KiB)',
false,
true );
172 parent::finalSetup( $settingsBuilder );
180 private function getBlobStore() {
181 return MediaWikiServices::getInstance()->getBlobStore();
190 parent::processOptions();
192 if ( $this->
hasOption(
'buffersize' ) ) {
193 $this->bufferSize = max( intval( $this->
getOption(
'buffersize' ) ), 4 * 1024 );
198 $this->prefetch =
new BaseDump( $url );
206 $this->maxTimeAllowed = intval( $this->
getOption(
'maxtime' ) ) * 60;
209 if ( $this->
hasOption(
'checkpointfile' ) ) {
210 $this->checkpointFiles = $this->
getOption(
'checkpointfile' );
214 $this->history = WikiExporter::CURRENT;
218 $this->history = WikiExporter::FULL;
225 $this->php = explode(
',', $val, 2 );
244 if ( isset( $this->lb ) ) {
245 $this->lb->closeAll( __METHOD__ );
249 if ( $this->forcedDb !==
null ) {
255 if ( isset( $this->db ) && $this->db->isOpen() ) {
256 throw new MWException(
'DB is set and has not been closed by the Load Balancer' );
266 $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
267 $this->lb = $lbFactory->newMainLB();
268 }
catch ( Exception $e ) {
270 .
" rotating DB failed to obtain new load balancer (" . $e->getMessage() .
")" );
274 $this->db = $this->lb->getMaintenanceConnectionRef(
DB_REPLICA,
'dump' );
275 }
catch ( Exception $e ) {
277 .
" rotating DB failed to obtain new database (" . $e->getMessage() .
")" );
282 parent::initProgress();
289 if ( ini_get(
'display_errors' ) ) {
290 ini_set(
'display_errors',
'stderr' );
302 }
catch ( Exception $e ) {
305 $this->
progress(
"Getting initial DB connection failed (" .
306 $e->getMessage() .
")" );
312 $this->finalOptionCheck();
315 $this->xmlwriterobj =
new XmlDumpWriter( XmlDumpWriter::WRITE_CONTENT, $this->schemaVersion );
317 $input = fopen( $this->input,
"rt" );
320 if ( $this->spawnProc ) {
328 $split = explode(
':', $opt, 2 );
331 if ( count( $split ) === 2 ) {
334 $fileURIs = explode(
';', $param );
336 foreach ( $fileURIs as $URI ) {
342 $newURI =
"compress.zlib://$URI";
345 $newURI =
"compress.bzip2://$URI";
348 $newURI =
"mediawiki.compress.7z://$URI";
353 $newFileURIs[] = $newURI;
355 $val = implode(
';', $newFileURIs );
364 if ( !$this->prefetch ) {
365 parent::showReport();
370 if ( $this->reporting ) {
372 $nowts = microtime(
true );
380 $eta = $this->startTime + $deltaAll / $portion;
382 if ( $this->fetchCount ) {
387 $pageRate = $this->pageCount / $deltaAll;
388 $revRate = $this->
revCount / $deltaAll;
396 if ( $this->fetchCountLast ) {
399 $fetchRatePart =
'-';
401 $pageRatePart = $this->pageCountPart / $deltaPart;
402 $revRatePart = $this->revCountPart / $deltaPart;
404 $fetchRatePart =
'-';
409 $dbDomain = WikiMap::getCurrentWikiDbDomain()->getId();
411 "%s: %s (ID %d) %d pages (%0.1f|%0.1f/sec all|curr), "
412 .
"%d revs (%0.1f|%0.1f/sec all|curr), %0.1f%%|%0.1f%% "
413 .
"prefetched (all|curr), ETA %s [max %d]",
414 $now, $dbDomain, $this->ID, $this->pageCount, $pageRate,
415 $pageRatePart, $this->
revCount, $revRate, $revRatePart,
416 $fetchRate, $fetchRatePart, $etats, $this->maxCount
418 $this->lastTime = $nowts;
425 private function setTimeExceeded() {
426 $this->timeExceeded =
true;
429 private function checkIfTimeExceeded() {
430 if ( $this->maxTimeAllowed
431 && ( $this->lastTime - $this->timeOfCheckpoint > $this->maxTimeAllowed )
439 private function finalOptionCheck() {
440 if ( ( $this->checkpointFiles && !$this->maxTimeAllowed )
441 || ( $this->maxTimeAllowed && !$this->checkpointFiles )
443 throw new MWException(
"Options checkpointfile and maxtime must be specified together.\n" );
445 foreach ( $this->checkpointFiles as $checkpointFile ) {
446 $count = substr_count( $checkpointFile,
"%s" );
448 throw new MWException(
"Option checkpointfile must contain two '%s' "
449 .
"for substitution of first and last pageids, count is $count instead, "
450 .
"file is $checkpointFile.\n" );
454 if ( $this->checkpointFiles ) {
455 $filenameList = (array)$this->egress->getFilenames();
456 if ( count( $filenameList ) != count( $this->checkpointFiles ) ) {
457 throw new MWException(
"One checkpointfile must be specified "
458 .
"for each output option, if maxtime is used.\n" );
470 $this->openElement =
false;
471 $this->atStart =
true;
473 $this->lastName =
"";
474 $this->thisPage =
"";
476 $this->thisRole =
null;
477 $this->thisRevModel =
null;
478 $this->thisRevFormat =
null;
480 $parser = xml_parser_create(
"UTF-8" );
481 xml_parser_set_option( $parser, XML_OPTION_CASE_FOLDING, 0 );
483 xml_set_element_handler(
485 [ $this,
'startElement' ],
486 [ $this,
'endElement' ]
488 xml_set_character_data_handler( $parser, [ $this,
'characterData' ] );
492 if ( $this->checkIfTimeExceeded() ) {
493 $this->setTimeExceeded();
495 $chunk = fread(
$input, $this->bufferSize );
496 if ( !xml_parse( $parser, $chunk, feof(
$input ) ) ) {
497 wfDebug(
"TextDumpPass::readDump encountered XML parsing error" );
499 $byte = xml_get_current_byte_index( $parser );
501 'XML import parse failure',
502 xml_get_current_line_number( $parser ),
503 xml_get_current_column_number( $parser ),
504 $byte . ( $chunk ===
false ?
'' : (
'; "' . substr( $chunk, $byte - $offset, 16 ) .
'"' ) ),
505 xml_error_string( xml_get_error_code( $parser ) ) )->escaped();
507 xml_parser_free( $parser );
511 $offset += strlen( $chunk );
512 }
while ( $chunk !==
false && !feof(
$input ) );
513 if ( $this->maxTimeAllowed ) {
514 $filenameList = (array)$this->egress->getFilenames();
516 if ( file_exists( $filenameList[0] ) ) {
518 # we might have just written the header and footer and had no
519 # pages or revisions written... perhaps they were all deleted
520 # there's no pageID 0 so we use that. the caller is responsible
521 # for deciding what to do with a file containing only the
522 # siteinfo information and the mw tags.
523 if ( !$this->firstPageWritten ) {
524 $firstPageID = str_pad(
'0', 9,
"0", STR_PAD_LEFT );
525 $lastPageID = str_pad(
'0', 9,
"0", STR_PAD_LEFT );
527 $firstPageID = str_pad( $this->firstPageWritten, 9,
"0", STR_PAD_LEFT );
528 $lastPageID = str_pad( $this->lastPageWritten, 9,
"0", STR_PAD_LEFT );
531 $filenameCount = count( $filenameList );
532 for ( $i = 0; $i < $filenameCount; $i++ ) {
533 $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID );
534 $fileinfo = pathinfo( $filenameList[$i] );
535 $newFilenames[] =
$fileinfo[
'dirname'] .
'/' . $checkpointNameFilledIn;
537 $this->egress->closeAndRename( $newFilenames );
540 xml_parser_free( $parser );
554 private function exportTransform( $text, $model, $format =
null ) {
556 $text = MediaWikiServices::getInstance()
557 ->getContentHandlerFactory()
558 ->getContentHandler( $model )
559 ->exportTransform( $text, $format );
561 wfWarn(
"Unable to apply export transformation for content model '$model': " .
565 "Unable to apply export transformation for content model '$model': " .
593 protected function getText( $id, $model =
null, $format =
null, $expSize =
null ) {
594 if ( !$this->isValidTextId( $id ) ) {
595 $msg =
"Skipping bad text id " . $id .
" of revision " . $this->thisRev;
600 $model = $model ?:
null;
601 $prefetchNotTried =
true;
606 static $consecutiveFailedTextRetrievals = 0;
613 $oldConsecutiveFailedTextRetrievals = $consecutiveFailedTextRetrievals;
614 $consecutiveFailedTextRetrievals = 0;
616 while ( $failures < $this->maxFailures ) {
627 if ( $text ===
false && isset( $this->prefetch ) && $prefetchNotTried ) {
628 $prefetchNotTried =
false;
629 $tryIsPrefetch =
true;
630 $text = $this->prefetch->prefetch(
631 (
int)$this->thisPage,
633 trim( $this->thisRole )
636 if ( $text ===
null ) {
640 if ( is_string( $text ) && $model !==
null ) {
646 $text = $this->exportTransform( $text, $model, $format );
650 if ( $text ===
false ) {
652 $tryIsPrefetch =
false;
653 if ( $this->spawn ) {
654 $text = $this->getTextSpawned( $id );
656 $text = $this->getTextDb( $id );
659 if ( $text !==
false && $model !==
null ) {
662 $text = $this->exportTransform( $text, $model, $format );
668 if ( $text !==
false ) {
673 if ( $text ===
false ) {
674 throw new MWException(
"Generic error while obtaining text for id " . $id );
682 if ( $expSize ===
null || strlen( $text ) == $expSize ) {
684 if ( $tryIsPrefetch ) {
685 $this->prefetchCount++;
692 throw new MWException(
"Received text is unplausible for id " . $id );
693 }
catch ( Exception $e ) {
694 $msg =
"getting/checking text " . $id .
" failed (" . $e->getMessage()
695 .
") for revision " . $this->thisRev;
696 if ( $failures + 1 < $this->maxFailures ) {
697 $msg .=
" (Will retry " . ( $this->maxFailures - $failures - 1 ) .
" more times)";
707 if ( !$tryIsPrefetch ) {
711 sleep( $this->failureTimeout );
714 if ( $this->spawn ) {
718 }
catch ( Exception $e ) {
719 $this->
progress(
"Rebooting getText infrastructure failed (" . $e->getMessage() .
")" .
720 " Trying to continue anyways" );
730 $consecutiveFailedTextRetrievals = $oldConsecutiveFailedTextRetrievals + 1;
731 if ( $consecutiveFailedTextRetrievals > $this->maxConsecutiveFailedTextRetrievals ) {
732 throw new MWException(
"Graceful storage failure" );
744 private function getTextDb( $id ) {
745 $store = $this->getBlobStore();
746 $address = ( is_int( $id ) || strpos( $id,
':' ) === false )
747 ? SqlBlobStore::makeAddressFromTextId( (
int)$id )
751 $text = $store->getBlob( $address );
753 $stripped = str_replace(
"\r",
"", $text );
754 $normalized = MediaWikiServices::getInstance()->getContentLanguage()
755 ->normalize( $stripped );
768 private function getTextSpawned( $address ) {
769 AtEase::suppressWarnings();
770 if ( !$this->spawnProc ) {
774 $text = $this->getTextSpawnedOnce( $address );
775 AtEase::restoreWarnings();
783 $wiki = WikiMap::getCurrentWikiId();
784 if ( count( $this->php ) == 2 ) {
785 $mwscriptpath = $this->php[1];
787 $mwscriptpath =
"$IP/../multiversion/MWScript.php";
789 if ( file_exists( $mwscriptpath ) ) {
791 array_map( [ Shell::class,
'escape' ],
796 '--wiki', $wiki ] ) );
799 array_map( [ Shell::class,
'escape' ],
802 "$IP/maintenance/fetchText.php",
803 '--wiki', $wiki ] ) );
806 0 => [
"pipe",
"r" ],
807 1 => [
"pipe",
"w" ],
808 2 => [
"file",
"/dev/null",
"a" ] ];
811 $this->
progress(
"Spawning database subprocess: $cmd" );
812 $this->spawnProc = proc_open( $cmd, $spec, $pipes );
813 if ( !$this->spawnProc ) {
814 $this->
progress(
"Subprocess spawn failed." );
826 private function closeSpawn() {
827 AtEase::suppressWarnings();
828 if ( $this->spawnRead ) {
829 fclose( $this->spawnRead );
831 $this->spawnRead =
null;
832 if ( $this->spawnWrite ) {
833 fclose( $this->spawnWrite );
835 $this->spawnWrite =
null;
836 if ( $this->spawnErr ) {
837 fclose( $this->spawnErr );
839 $this->spawnErr =
false;
840 if ( $this->spawnProc ) {
841 proc_close( $this->spawnProc );
843 $this->spawnProc =
false;
844 AtEase::restoreWarnings();
851 private function getTextSpawnedOnce( $address ) {
852 if ( is_int( $address ) || intval( $address ) ) {
853 $address = SqlBlobStore::makeAddressFromTextId( (
int)$address );
856 $ok = fwrite( $this->spawnWrite,
"$address\n" );
862 $ok = fflush( $this->spawnWrite );
870 $newAddress = fgets( $this->spawnRead );
871 if ( $newAddress ===
false ) {
874 $newAddress = trim( $newAddress );
875 if ( strpos( $newAddress,
':' ) ===
false ) {
876 $newAddress = SqlBlobStore::makeAddressFromTextId( intval( $newAddress ) );
879 if ( $newAddress !== $address ) {
883 $len = fgets( $this->spawnRead );
885 if ( $len ===
false ) {
889 $nbytes = intval( $len );
898 while ( $nbytes > strlen( $text ) ) {
899 $buffer = fread( $this->spawnRead, $nbytes - strlen( $text ) );
906 $gotbytes = strlen( $text );
907 if ( $gotbytes != $nbytes ) {
908 $this->
progress(
"Expected $nbytes bytes from database subprocess, got $gotbytes " );
914 $stripped = str_replace(
"\r",
"", $text );
915 $normalized = MediaWikiServices::getInstance()->getContentLanguage()->
916 normalize( $stripped );
922 $this->checkpointJustWritten =
false;
925 $this->lastName = $name;
927 if ( $name ==
'revision' ) {
928 $this->state = $name;
929 $this->egress->writeOpenPage(
null, $this->buffer );
931 } elseif ( $name ==
'page' ) {
932 $this->state = $name;
933 if ( $this->atStart ) {
934 $this->egress->writeOpenStream( $this->buffer );
936 $this->atStart =
false;
938 } elseif ( $name ===
'mediawiki' ) {
939 if ( isset( $attribs[
'version'] ) ) {
940 if ( $attribs[
'version'] !== $this->schemaVersion ) {
941 throw new MWException(
'Mismatching schema version. '
942 .
'Use the --schema-version option to set the output schema version to '
943 .
'the version declared by the stub file, namely ' . $attribs[
'version'] );
948 if ( $name ==
"text" && ( isset( $attribs[
'id'] ) || isset( $attribs[
'location'] ) ) ) {
949 $id = $attribs[
'location'] ?? $attribs[
'id'];
950 $model = trim( $this->thisRevModel );
951 $format = trim( $this->thisRevFormat );
953 $model = $model ===
'' ? null : $model;
954 $format = $format ===
'' ? null : $format;
956 ? (int)$attribs[
'bytes'] :
null;
958 $text = $this->
getText( $id, $model, $format, $expSize );
960 unset( $attribs[
'id'] );
961 unset( $attribs[
'location'] );
962 if ( strlen( $text ) > 0 ) {
963 $attribs[
'xml:space'] =
'preserve';
966 $this->openElement = [ $name, $attribs ];
967 if ( strlen( $text ) > 0 ) {
971 $this->openElement = [ $name, $attribs ];
976 $this->checkpointJustWritten =
false;
978 if ( $this->openElement ) {
981 $this->buffer .=
"</$name>";
984 if ( $name ==
'revision' ) {
985 $this->egress->writeRevision(
null, $this->buffer );
988 $this->thisRole =
null;
989 $this->thisRevModel =
null;
990 $this->thisRevFormat =
null;
991 } elseif ( $name ==
'page' ) {
992 if ( !$this->firstPageWritten ) {
993 $this->firstPageWritten = trim( $this->thisPage );
995 $this->lastPageWritten = trim( $this->thisPage );
996 if ( $this->timeExceeded ) {
997 $this->egress->writeClosePage( $this->buffer );
1000 $this->egress->sink->write(
"\n" );
1002 $this->buffer = $this->xmlwriterobj->closeStream();
1003 $this->egress->writeCloseStream( $this->buffer );
1006 $this->thisPage =
"";
1009 $filenameList = (array)$this->egress->getFilenames();
1011 $firstPageID = str_pad( $this->firstPageWritten, 9,
"0", STR_PAD_LEFT );
1012 $lastPageID = str_pad( $this->lastPageWritten, 9,
"0", STR_PAD_LEFT );
1013 $filenamesCount = count( $filenameList );
1014 for ( $i = 0; $i < $filenamesCount; $i++ ) {
1015 $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID );
1016 $fileinfo = pathinfo( $filenameList[$i] );
1017 $newFilenames[] =
$fileinfo[
'dirname'] .
'/' . $checkpointNameFilledIn;
1019 $this->egress->closeRenameAndReopen( $newFilenames );
1020 $this->buffer = $this->xmlwriterobj->openStream();
1021 $this->timeExceeded =
false;
1023 $this->firstPageWritten =
false;
1024 $this->checkpointJustWritten =
true;
1026 $this->egress->writeClosePage( $this->buffer );
1028 $this->thisPage =
"";
1030 } elseif ( $name ==
'mediawiki' ) {
1031 $this->egress->writeCloseStream( $this->buffer );
1038 if ( $this->lastName ==
"id" ) {
1039 if ( $this->state ==
"revision" ) {
1040 $this->thisRev .= $data;
1041 $this->thisRole = SlotRecord::MAIN;
1042 } elseif ( $this->state ==
"page" ) {
1043 $this->thisPage .= $data;
1045 } elseif ( $this->lastName ==
"model" ) {
1046 $this->thisRevModel .= $data;
1047 } elseif ( $this->lastName ==
"format" ) {
1048 $this->thisRevFormat .= $data;
1049 } elseif ( $this->lastName ==
"content" ) {
1050 $this->thisRole =
"";
1051 $this->thisRevModel =
"";
1052 $this->thisRevFormat =
"";
1053 } elseif ( $this->lastName ==
"role" ) {
1054 $this->thisRole .= $data;
1059 if ( $this->checkpointJustWritten ) {
1060 if ( $data[0] ==
"\n" ) {
1061 $data = substr( $data, 1 );
1063 $this->checkpointJustWritten =
false;
1065 $this->buffer .= htmlspecialchars( $data, ENT_COMPAT );
1069 if ( $this->openElement ) {
1070 $this->buffer .= Xml::element( $this->openElement[0], $this->openElement[1], $style );
1071 $this->openElement =
false;
1075 private function isValidTextId( $id ) {
1076 if ( preg_match(
'/:/', $id ) ) {
1077 return $id !==
'tt:0';
1078 } elseif ( preg_match(
'/^\d+$/', $id ) ) {
1079 return intval( $id ) > 0;
const CONTENT_MODEL_WIKITEXT
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
wfWarn( $msg, $callerOffset=1, $level=E_USER_NOTICE)
Send a warning either to the debug log or in a PHP error depending on $wgDevelopmentWarnings.
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
wfMessage( $key,... $params)
This is the function for getting translated interface messages.
if(!defined( 'MEDIAWIKI')) if(ini_get('mbstring.func_overload')) if(!defined( 'MW_ENTRY_POINT')) global $IP
Environment checks.
IMaintainableDatabase null $forcedDb
The dependency-injected database to use.
Readahead helper for making large MediaWiki data dumps; reads in a previous XML dump to sequentially ...
output( $out, $channel=null)
Throw some output to the user.
hasOption( $name)
Checks to see if a particular option was set.
addDescription( $text)
Set the description text.
loadWithArgv( $argv)
Load params and arguments from a given array of command-line arguments.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
showReport()
Overridden to include prefetch ratio if enabled.
string false $lastPageWritten
endElement( $parser, $name)
clearOpenElement( $style)
string false $firstPageWritten
bool $checkpointJustWritten
startElement( $parser, $name, $attribs)
finalSetup(SettingsBuilder $settingsBuilder=null)
Handle some last-minute setup here.
int $maxConsecutiveFailedTextRetrievals
int $failureTimeout
Seconds to sleep after db failure.
processOptions()
Processes arguments and sets $this->$sink accordingly.
getText( $id, $model=null, $format=null, $expSize=null)
Tries to load revision text.
characterData( $parser, $data)
resource false $spawnProc
initProgress( $history=WikiExporter::FULL)
Initialise starting time and maximum revision count.
execute()
Do the actual work.
dump( $history, $text=WikiExporter::TEXT)
string[] $checkpointFiles
XmlDumpWriter false $xmlwriterobj
int $maxTimeAllowed
when we spend more than maxTimeAllowed seconds on this run, we continue processing until we write out...
IMaintainableDatabase $db
resource null $spawnWrite
rotateDb()
Drop the database connection $this->db and try to get a new one.
Advanced database interface for IDatabase handles that include maintenance methods.