27 require_once __DIR__ .
'/backup.inc';
28 require_once __DIR__ .
'/../includes/export/WikiExporter.php';
99 parent::__construct();
102 This
script postprocesses XML dumps
from dumpBackup.php to
add
103 page text which was stubbed out (
using --stub).
105 XML input
is accepted
on stdin.
109 $this->stderr = fopen(
"php://stderr",
"wt" );
111 $this->
addOption(
'stub',
'To load a compressed stub dump instead of stdin. ' .
112 'Specify as --stub=<type>:<file>.',
false,
true );
113 $this->
addOption(
'prefetch',
'Use a prior dump file as a text source, to savepressure on the ' .
114 'database. (Requires the XMLReader extension). Specify as --prefetch=<type>:<file>',
116 $this->
addOption(
'maxtime',
'Write out checkpoint file after this many minutes (writing' .
117 'out complete page, closing xml file properly, and opening new one' .
118 'with header). This option requires the checkpointfile option.',
false,
true );
119 $this->
addOption(
'checkpointfile',
'Use this string for checkpoint filenames,substituting ' .
120 'first pageid written for the first %s (required) and the last pageid written for the ' .
121 'second %s if it exists.',
false,
true,
false,
true );
122 $this->
addOption(
'quiet',
'Don\'t dump status reports to stderr.' );
123 $this->
addOption(
'current',
'Base ETA on number of pages in database instead of all revisions' );
124 $this->
addOption(
'spawn',
'Spawn a subprocess for loading text records' );
125 $this->
addOption(
'buffersize',
'Buffer size in bytes to use for reading the stub. ' .
126 '(Default: 512KB, Minimum: 4KB)',
false,
true );
142 parent::processOptions();
144 if ( $this->
hasOption(
'buffersize' ) ) {
145 $this->bufferSize = max( intval( $this->
getOption(
'buffersize' ) ), 4 * 1024 );
149 require_once
"$IP/maintenance/backupPrefetch.inc";
151 $this->prefetch =
new BaseDump( $url );
159 $this->maxTimeAllowed = intval( $this->
getOption(
'maxtime' ) ) * 60;
162 if ( $this->
hasOption(
'checkpointfile' ) ) {
163 $this->checkpointFiles = $this->
getOption(
'checkpointfile' );
196 if ( isset( $this->lb ) ) {
197 $this->lb->closeAll();
201 if ( $this->forcedDb !==
null ) {
207 if ( isset( $this->db ) && $this->db->isOpen() ) {
208 throw new MWException(
'DB is set and has not been closed by the Load Balancer' );
219 }
catch ( Exception
$e ) {
221 .
" rotating DB failed to obtain new load balancer (" .
$e->getMessage() .
")" );
225 $this->db = $this->lb->getConnection(
DB_REPLICA,
'dump' );
226 }
catch ( Exception
$e ) {
228 .
" rotating DB failed to obtain new database (" .
$e->getMessage() .
")" );
233 parent::initProgress();
234 $this->timeOfCheckpoint = $this->startTime;
240 if ( ini_get(
'display_errors' ) ) {
241 ini_set(
'display_errors',
'stderr' );
253 }
catch ( Exception
$e ) {
256 $this->
progress(
"Getting initial DB connection failed (" .
257 $e->getMessage() .
")" );
268 $input = fopen( $this->input,
"rt" );
271 if ( $this->spawnProc ) {
279 $split = explode(
':',
$opt, 2 );
282 if (
count( $split ) === 2 ) {
285 $fileURIs = explode(
';', $param );
286 foreach ( $fileURIs
as $URI ) {
292 $newURI =
"compress.zlib://$URI";
295 $newURI =
"compress.bzip2://$URI";
298 $newURI =
"mediawiki.compress.7z://$URI";
303 $newFileURIs[] = $newURI;
305 $val = implode(
';', $newFileURIs );
314 if ( !$this->prefetch ) {
315 parent::showReport();
320 if ( $this->reporting ) {
322 $nowts = microtime(
true );
323 $deltaAll = $nowts - $this->startTime;
329 $portion = $this->
revCount / $this->maxCount;
330 $eta = $this->startTime + $deltaAll / $portion;
332 if ( $this->fetchCount ) {
337 $pageRate = $this->pageCount / $deltaAll;
338 $revRate = $this->
revCount / $deltaAll;
346 if ( $this->fetchCountLast ) {
349 $fetchRatePart =
'-';
351 $pageRatePart = $this->pageCountPart / $deltaPart;
352 $revRatePart = $this->revCountPart / $deltaPart;
354 $fetchRatePart =
'-';
359 "%s: %s (ID %d) %d pages (%0.1f|%0.1f/sec all|curr), "
360 .
"%d revs (%0.1f|%0.1f/sec all|curr), %0.1f%%|%0.1f%% "
361 .
"prefetched (all|curr), ETA %s [max %d]",
362 $now,
wfWikiID(), $this->ID, $this->pageCount, $pageRate,
363 $pageRatePart, $this->
revCount, $revRate, $revRatePart,
364 $fetchRate, $fetchRatePart, $etats, $this->maxCount
366 $this->lastTime = $nowts;
374 $this->timeExceeded =
true;
378 if ( $this->maxTimeAllowed
379 && ( $this->lastTime - $this->timeOfCheckpoint > $this->maxTimeAllowed )
388 if ( ( $this->checkpointFiles && !$this->maxTimeAllowed )
389 || ( $this->maxTimeAllowed && !$this->checkpointFiles )
391 throw new MWException(
"Options checkpointfile and maxtime must be specified together.\n" );
393 foreach ( $this->checkpointFiles
as $checkpointFile ) {
394 $count = substr_count( $checkpointFile,
"%s" );
396 throw new MWException(
"Option checkpointfile must contain two '%s' "
397 .
"for substitution of first and last pageids, count is $count instead, "
398 .
"file is $checkpointFile.\n" );
402 if ( $this->checkpointFiles ) {
403 $filenameList = (
array)$this->egress->getFilenames();
404 if (
count( $filenameList ) !=
count( $this->checkpointFiles ) ) {
405 throw new MWException(
"One checkpointfile must be specified "
406 .
"for each output option, if maxtime is used.\n" );
418 $this->openElement =
false;
419 $this->atStart =
true;
421 $this->lastName =
"";
424 $this->thisRevModel =
null;
425 $this->thisRevFormat =
null;
427 $parser = xml_parser_create(
"UTF-8" );
428 xml_parser_set_option(
$parser, XML_OPTION_CASE_FOLDING,
false );
430 xml_set_element_handler(
432 [ $this,
'startElement' ],
433 [ $this,
'endElement' ]
435 xml_set_character_data_handler(
$parser, [ $this,
'characterData' ] );
442 $chunk = fread(
$input, $this->bufferSize );
444 wfDebug(
"TextDumpPass::readDump encountered XML parsing error\n" );
446 $byte = xml_get_current_byte_index(
$parser );
448 'XML import parse failure',
449 xml_get_current_line_number(
$parser ),
450 xml_get_current_column_number(
$parser ),
451 $byte . ( is_null( $chunk ) ?
null : (
'; "' . substr( $chunk, $byte - $offset, 16 ) .
'"' ) ),
452 xml_error_string( xml_get_error_code(
$parser ) ) )->escaped();
458 $offset += strlen( $chunk );
459 }
while ( $chunk !==
false && !feof(
$input ) );
460 if ( $this->maxTimeAllowed ) {
461 $filenameList = (
array)$this->egress->getFilenames();
463 if ( file_exists( $filenameList[0] ) ) {
465 # we might have just written the header and footer and had no
466 # pages or revisions written... perhaps they were all deleted
467 # there's no pageID 0 so we use that. the caller is responsible
468 # for deciding what to do with a file containing only the
469 # siteinfo information and the mw tags.
470 if ( !$this->firstPageWritten ) {
471 $firstPageID = str_pad( 0, 9,
"0", STR_PAD_LEFT );
472 $lastPageID = str_pad( 0, 9,
"0", STR_PAD_LEFT );
474 $firstPageID = str_pad( $this->firstPageWritten, 9,
"0", STR_PAD_LEFT );
475 $lastPageID = str_pad( $this->lastPageWritten, 9,
"0", STR_PAD_LEFT );
478 $filenameCount =
count( $filenameList );
479 for ( $i = 0; $i < $filenameCount; $i++ ) {
480 $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID );
481 $fileinfo = pathinfo( $filenameList[$i] );
482 $newFilenames[] =
$fileinfo[
'dirname'] .
'/' . $checkpointNameFilledIn;
484 $this->egress->closeAndRename( $newFilenames );
504 $text =
$handler->exportTransform( $text, $format );
508 "Unable to apply export transformation for content model '$model': " .
536 function getText( $id, $model =
null, $format =
null ) {
537 global $wgContentHandlerUseDB;
539 $prefetchNotTried =
true;
544 static $consecutiveFailedTextRetrievals = 0;
551 $oldConsecutiveFailedTextRetrievals = $consecutiveFailedTextRetrievals;
552 $consecutiveFailedTextRetrievals = 0;
554 if ( $model ===
null && $wgContentHandlerUseDB ) {
555 $row = $this->db->selectRow(
557 [
'rev_content_model',
'rev_content_format' ],
558 [
'rev_id' => $this->thisRev ],
563 $model = $row->rev_content_model;
564 $format = $row->rev_content_format;
568 if ( $model ===
null || $model ===
'' ) {
572 while ( $failures < $this->maxFailures ) {
583 if ( $text ===
false && isset( $this->prefetch ) && $prefetchNotTried ) {
584 $prefetchNotTried =
false;
585 $tryIsPrefetch =
true;
586 $text = $this->prefetch->prefetch( intval( $this->thisPage ),
587 intval( $this->thisRev ) );
589 if ( $text ===
null ) {
593 if ( is_string( $text ) && $model !==
false ) {
603 if ( $text ===
false ) {
605 $tryIsPrefetch =
false;
606 if ( $this->spawn ) {
612 if ( $text !==
false && $model !==
false ) {
621 if ( $text !==
false ) {
626 if ( $text ===
false ) {
627 throw new MWException(
"Generic error while obtaining text for id " . $id );
634 $revID = intval( $this->thisRev );
635 if ( !isset( $this->db ) ) {
640 $revLength = strlen( $text );
642 $revLength = $this->db->selectField(
'revision',
'rev_len', [
'rev_id' => $revID ] );
645 if ( strlen( $text ) == $revLength ) {
646 if ( $tryIsPrefetch ) {
647 $this->prefetchCount++;
654 throw new MWException(
"Received text is unplausible for id " . $id );
655 }
catch ( Exception
$e ) {
656 $msg =
"getting/checking text " . $id .
" failed (" .
$e->getMessage() .
")";
657 if ( $failures + 1 < $this->maxFailures ) {
658 $msg .=
" (Will retry " . ( $this->maxFailures - $failures - 1 ) .
" more times)";
667 if ( !$tryIsPrefetch ) {
671 sleep( $this->failureTimeout );
674 if ( $this->spawn ) {
678 }
catch ( Exception
$e ) {
679 $this->
progress(
"Rebooting getText infrastructure failed (" . $e->getMessage() .
")" .
680 " Trying to continue anyways" );
690 $consecutiveFailedTextRetrievals = $oldConsecutiveFailedTextRetrievals + 1;
691 if ( $consecutiveFailedTextRetrievals > $this->maxConsecutiveFailedTextRetrievals ) {
692 throw new MWException(
"Graceful storage failure" );
706 if ( !isset( $this->db ) ) {
707 throw new MWException( __METHOD__ .
"No database available" );
709 $row = $this->db->selectRow(
'text',
710 [
'old_text',
'old_flags' ],
714 if ( $text ===
false ) {
717 $stripped = str_replace(
"\r",
"", $text );
724 MediaWiki\suppressWarnings();
725 if ( !$this->spawnProc ) {
730 MediaWiki\restoreWarnings();
738 if ( file_exists(
"$IP/../multiversion/MWScript.php" ) ) {
740 array_map(
'wfEscapeShellArg',
743 "$IP/../multiversion/MWScript.php",
748 array_map(
'wfEscapeShellArg',
751 "$IP/maintenance/fetchText.php",
755 0 => [
"pipe",
"r" ],
756 1 => [
"pipe",
"w" ],
757 2 => [
"file",
"/dev/null",
"a" ] ];
760 $this->
progress(
"Spawning database subprocess: $cmd" );
761 $this->spawnProc = proc_open( $cmd, $spec, $pipes );
762 if ( !$this->spawnProc ) {
763 $this->
progress(
"Subprocess spawn failed." );
776 MediaWiki\suppressWarnings();
777 if ( $this->spawnRead ) {
778 fclose( $this->spawnRead );
780 $this->spawnRead =
false;
781 if ( $this->spawnWrite ) {
782 fclose( $this->spawnWrite );
784 $this->spawnWrite =
false;
785 if ( $this->spawnErr ) {
786 fclose( $this->spawnErr );
788 $this->spawnErr =
false;
789 if ( $this->spawnProc ) {
790 pclose( $this->spawnProc );
792 $this->spawnProc =
false;
793 MediaWiki\restoreWarnings();
799 $ok = fwrite( $this->spawnWrite,
"$id\n" );
805 $ok = fflush( $this->spawnWrite );
813 $newId = fgets( $this->spawnRead );
814 if ( $newId ===
false ) {
817 if ( $id != intval( $newId ) ) {
821 $len = fgets( $this->spawnRead );
823 if ( $len ===
false ) {
827 $nbytes = intval( $len );
836 while ( $nbytes > strlen( $text ) ) {
837 $buffer = fread( $this->spawnRead, $nbytes - strlen( $text ) );
844 $gotbytes = strlen( $text );
845 if ( $gotbytes != $nbytes ) {
846 $this->
progress(
"Expected $nbytes bytes from database subprocess, got $gotbytes " );
852 $stripped = str_replace(
"\r",
"", $text );
859 $this->checkpointJustWritten =
false;
862 $this->lastName =
$name;
864 if (
$name ==
'revision' ) {
865 $this->state =
$name;
866 $this->egress->writeOpenPage(
null, $this->buffer );
868 } elseif (
$name ==
'page' ) {
869 $this->state =
$name;
870 if ( $this->atStart ) {
871 $this->egress->writeOpenStream( $this->buffer );
873 $this->atStart =
false;
879 $model = trim( $this->thisRevModel );
880 $format = trim( $this->thisRevFormat );
882 $model = $model ===
'' ? null : $model;
883 $format = $format ===
'' ? null : $format;
885 $text = $this->
getText( $id, $model, $format );
886 $this->openElement = [
$name, [
'xml:space' =>
'preserve' ] ];
887 if ( strlen( $text ) > 0 ) {
896 $this->checkpointJustWritten =
false;
898 if ( $this->openElement ) {
901 $this->buffer .=
"</$name>";
904 if (
$name ==
'revision' ) {
905 $this->egress->writeRevision(
null, $this->buffer );
908 $this->thisRevModel =
null;
909 $this->thisRevFormat =
null;
910 } elseif (
$name ==
'page' ) {
911 if ( !$this->firstPageWritten ) {
912 $this->firstPageWritten = trim( $this->thisPage );
914 $this->lastPageWritten = trim( $this->thisPage );
915 if ( $this->timeExceeded ) {
916 $this->egress->writeClosePage( $this->buffer );
919 $this->egress->sink->write(
"\n" );
921 $this->buffer = $this->xmlwriterobj->closeStream();
922 $this->egress->writeCloseStream( $this->buffer );
925 $this->thisPage =
"";
928 $filenameList = (
array)$this->egress->getFilenames();
930 $firstPageID = str_pad( $this->firstPageWritten, 9,
"0", STR_PAD_LEFT );
931 $lastPageID = str_pad( $this->lastPageWritten, 9,
"0", STR_PAD_LEFT );
932 $filenamesCount =
count( $filenameList );
933 for ( $i = 0; $i < $filenamesCount; $i++ ) {
934 $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID );
935 $fileinfo = pathinfo( $filenameList[$i] );
936 $newFilenames[] =
$fileinfo[
'dirname'] .
'/' . $checkpointNameFilledIn;
938 $this->egress->closeRenameAndReopen( $newFilenames );
939 $this->buffer = $this->xmlwriterobj->openStream();
940 $this->timeExceeded =
false;
942 $this->firstPageWritten =
false;
943 $this->checkpointJustWritten =
true;
945 $this->egress->writeClosePage( $this->buffer );
947 $this->thisPage =
"";
949 } elseif (
$name ==
'mediawiki' ) {
950 $this->egress->writeCloseStream( $this->buffer );
957 if ( $this->lastName ==
"id" ) {
958 if ( $this->state ==
"revision" ) {
959 $this->thisRev .= $data;
960 } elseif ( $this->state ==
"page" ) {
961 $this->thisPage .= $data;
963 } elseif ( $this->lastName ==
"model" ) {
964 $this->thisRevModel .= $data;
965 } elseif ( $this->lastName ==
"format" ) {
966 $this->thisRevFormat .= $data;
971 if ( $this->checkpointJustWritten ) {
972 if ( $data[0] ==
"\n" ) {
973 $data = substr( $data, 1 );
975 $this->checkpointJustWritten =
false;
977 $this->buffer .= htmlspecialchars( $data );
981 if ( $this->openElement ) {
982 $this->buffer .=
Xml::element( $this->openElement[0], $this->openElement[1], $style );
983 $this->openElement =
false;