27 require_once __DIR__ .
'/includes/BackupDumper.php';
28 require_once __DIR__ .
'/7zip.inc';
29 require_once __DIR__ .
'/../includes/export/WikiExporter.php';
109 parent::__construct();
112 This
script postprocesses XML dumps
from dumpBackup.php to
add
113 page text which was stubbed out (
using --stub).
115 XML input
is accepted
on stdin.
119 $this->stderr = fopen(
"php://stderr",
"wt" );
121 $this->
addOption(
'stub',
'To load a compressed stub dump instead of stdin. ' .
122 'Specify as --stub=<type>:<file>.',
false,
true );
123 $this->
addOption(
'prefetch',
'Use a prior dump file as a text source, to savepressure on the ' .
124 'database. (Requires the XMLReader extension). Specify as --prefetch=<type>:<file>',
126 $this->
addOption(
'maxtime',
'Write out checkpoint file after this many minutes (writing' .
127 'out complete page, closing xml file properly, and opening new one' .
128 'with header). This option requires the checkpointfile option.',
false,
true );
129 $this->
addOption(
'checkpointfile',
'Use this string for checkpoint filenames,substituting ' .
130 'first pageid written for the first %s (required) and the last pageid written for the ' .
131 'second %s if it exists.',
false,
true,
false,
true );
132 $this->
addOption(
'quiet',
'Don\'t dump status reports to stderr.' );
133 $this->
addOption(
'full',
'Dump all revisions of every page' );
134 $this->
addOption(
'current',
'Base ETA on number of pages in database instead of all revisions' );
135 $this->
addOption(
'spawn',
'Spawn a subprocess for loading text records' );
136 $this->
addOption(
'buffersize',
'Buffer size in bytes to use for reading the stub. ' .
137 '(Default: 512KB, Minimum: 4KB)',
false,
true );
149 return MediaWikiServices::getInstance()->getBlobStore();
158 parent::processOptions();
160 if ( $this->
hasOption(
'buffersize' ) ) {
161 $this->bufferSize = max( intval( $this->
getOption(
'buffersize' ) ), 4 * 1024 );
166 $this->prefetch =
new BaseDump( $url );
174 $this->maxTimeAllowed = intval( $this->
getOption(
'maxtime' ) ) * 60;
177 if ( $this->
hasOption(
'checkpointfile' ) ) {
178 $this->checkpointFiles = $this->
getOption(
'checkpointfile' );
211 if ( isset( $this->lb ) ) {
212 $this->lb->closeAll();
216 if ( $this->forcedDb !==
null ) {
222 if ( isset( $this->db ) && $this->db->isOpen() ) {
223 throw new MWException(
'DB is set and has not been closed by the Load Balancer' );
233 $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
234 $this->lb = $lbFactory->newMainLB();
235 }
catch ( Exception
$e ) {
237 .
" rotating DB failed to obtain new load balancer (" .
$e->getMessage() .
")" );
241 $this->db = $this->lb->getConnection(
DB_REPLICA,
'dump' );
242 }
catch ( Exception
$e ) {
244 .
" rotating DB failed to obtain new database (" .
$e->getMessage() .
")" );
249 parent::initProgress();
250 $this->timeOfCheckpoint = $this->startTime;
256 if ( ini_get(
'display_errors' ) ) {
257 ini_set(
'display_errors',
'stderr' );
269 }
catch ( Exception
$e ) {
272 $this->
progress(
"Getting initial DB connection failed (" .
273 $e->getMessage() .
")" );
284 $input = fopen( $this->input,
"rt" );
287 if ( $this->spawnProc ) {
295 $split = explode(
':',
$opt, 2 );
298 if (
count( $split ) === 2 ) {
301 $fileURIs = explode(
';', $param );
302 foreach ( $fileURIs
as $URI ) {
308 $newURI =
"compress.zlib://$URI";
311 $newURI =
"compress.bzip2://$URI";
314 $newURI =
"mediawiki.compress.7z://$URI";
319 $newFileURIs[] = $newURI;
321 $val = implode(
';', $newFileURIs );
330 if ( !$this->prefetch ) {
331 parent::showReport();
336 if ( $this->reporting ) {
338 $nowts = microtime(
true );
339 $deltaAll = $nowts - $this->startTime;
345 $portion = $this->
revCount / $this->maxCount;
346 $eta = $this->startTime + $deltaAll / $portion;
348 if ( $this->fetchCount ) {
353 $pageRate = $this->pageCount / $deltaAll;
354 $revRate = $this->
revCount / $deltaAll;
362 if ( $this->fetchCountLast ) {
365 $fetchRatePart =
'-';
367 $pageRatePart = $this->pageCountPart / $deltaPart;
368 $revRatePart = $this->revCountPart / $deltaPart;
370 $fetchRatePart =
'-';
375 "%s: %s (ID %d) %d pages (%0.1f|%0.1f/sec all|curr), "
376 .
"%d revs (%0.1f|%0.1f/sec all|curr), %0.1f%%|%0.1f%% "
377 .
"prefetched (all|curr), ETA %s [max %d]",
378 $now,
wfWikiID(), $this->ID, $this->pageCount, $pageRate,
379 $pageRatePart, $this->
revCount, $revRate, $revRatePart,
380 $fetchRate, $fetchRatePart, $etats, $this->maxCount
382 $this->lastTime = $nowts;
390 $this->timeExceeded =
true;
394 if ( $this->maxTimeAllowed
395 && ( $this->lastTime - $this->timeOfCheckpoint > $this->maxTimeAllowed )
404 if ( ( $this->checkpointFiles && !$this->maxTimeAllowed )
405 || ( $this->maxTimeAllowed && !$this->checkpointFiles )
407 throw new MWException(
"Options checkpointfile and maxtime must be specified together.\n" );
409 foreach ( $this->checkpointFiles
as $checkpointFile ) {
410 $count = substr_count( $checkpointFile,
"%s" );
412 throw new MWException(
"Option checkpointfile must contain two '%s' "
413 .
"for substitution of first and last pageids, count is $count instead, "
414 .
"file is $checkpointFile.\n" );
418 if ( $this->checkpointFiles ) {
419 $filenameList = (
array)$this->egress->getFilenames();
420 if (
count( $filenameList ) !=
count( $this->checkpointFiles ) ) {
421 throw new MWException(
"One checkpointfile must be specified "
422 .
"for each output option, if maxtime is used.\n" );
434 $this->openElement =
false;
435 $this->atStart =
true;
437 $this->lastName =
"";
440 $this->thisRevModel =
null;
441 $this->thisRevFormat =
null;
443 $parser = xml_parser_create(
"UTF-8" );
444 xml_parser_set_option(
$parser, XML_OPTION_CASE_FOLDING,
false );
446 xml_set_element_handler(
448 [ $this,
'startElement' ],
449 [ $this,
'endElement' ]
451 xml_set_character_data_handler(
$parser, [ $this,
'characterData' ] );
458 $chunk = fread(
$input, $this->bufferSize );
460 wfDebug(
"TextDumpPass::readDump encountered XML parsing error\n" );
462 $byte = xml_get_current_byte_index(
$parser );
464 'XML import parse failure',
465 xml_get_current_line_number(
$parser ),
466 xml_get_current_column_number(
$parser ),
467 $byte . ( is_null( $chunk ) ?
null : (
'; "' . substr( $chunk, $byte - $offset, 16 ) .
'"' ) ),
468 xml_error_string( xml_get_error_code(
$parser ) ) )->escaped();
474 $offset += strlen( $chunk );
475 }
while ( $chunk !==
false && !feof(
$input ) );
476 if ( $this->maxTimeAllowed ) {
477 $filenameList = (
array)$this->egress->getFilenames();
479 if ( file_exists( $filenameList[0] ) ) {
481 # we might have just written the header and footer and had no
482 # pages or revisions written... perhaps they were all deleted
483 # there's no pageID 0 so we use that. the caller is responsible
484 # for deciding what to do with a file containing only the
485 # siteinfo information and the mw tags.
486 if ( !$this->firstPageWritten ) {
487 $firstPageID = str_pad( 0, 9,
"0", STR_PAD_LEFT );
488 $lastPageID = str_pad( 0, 9,
"0", STR_PAD_LEFT );
490 $firstPageID = str_pad( $this->firstPageWritten, 9,
"0", STR_PAD_LEFT );
491 $lastPageID = str_pad( $this->lastPageWritten, 9,
"0", STR_PAD_LEFT );
494 $filenameCount =
count( $filenameList );
495 for ( $i = 0; $i < $filenameCount; $i++ ) {
496 $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID );
497 $fileinfo = pathinfo( $filenameList[$i] );
498 $newFilenames[] =
$fileinfo[
'dirname'] .
'/' . $checkpointNameFilledIn;
500 $this->egress->closeAndRename( $newFilenames );
520 $text =
$handler->exportTransform( $text, $format );
524 "Unable to apply export transformation for content model '$model': " .
552 function getText( $id, $model =
null, $format =
null ) {
555 $prefetchNotTried =
true;
560 static $consecutiveFailedTextRetrievals = 0;
567 $oldConsecutiveFailedTextRetrievals = $consecutiveFailedTextRetrievals;
568 $consecutiveFailedTextRetrievals = 0;
572 $row = $this->db->selectRow(
574 [
'rev_content_model',
'rev_content_format' ],
575 [
'rev_id' => $this->thisRev ],
580 $model = $row->rev_content_model;
581 $format = $row->rev_content_format;
585 if ( $model ===
null || $model ===
'' ) {
589 while ( $failures < $this->maxFailures ) {
599 if ( $text ===
false && isset( $this->prefetch ) && $prefetchNotTried ) {
600 $prefetchNotTried =
false;
601 $tryIsPrefetch =
true;
602 $text = $this->prefetch->prefetch( (
int)$this->thisPage, (
int)$this->thisRev );
604 if ( $text ===
null ) {
608 if ( is_string( $text ) && $model !==
false ) {
618 if ( $text ===
false ) {
620 $tryIsPrefetch =
false;
621 if ( $this->spawn ) {
627 if ( $text !==
false && $model !==
false ) {
636 if ( $text !==
false ) {
641 if ( $text ===
false ) {
642 throw new MWException(
"Generic error while obtaining text for id " . $id );
649 $revID = intval( $this->thisRev );
650 if ( !isset( $this->db ) ) {
655 $revLength = strlen( $text );
657 $revLength = $this->db->selectField(
'revision',
'rev_len', [
'rev_id' => $revID ] );
660 if ( strlen( $text ) == $revLength ) {
661 if ( $tryIsPrefetch ) {
662 $this->prefetchCount++;
669 throw new MWException(
"Received text is unplausible for id " . $id );
670 }
catch ( Exception
$e ) {
671 $msg =
"getting/checking text " . $id .
" failed (" .
$e->getMessage() .
")";
672 if ( $failures + 1 < $this->maxFailures ) {
673 $msg .=
" (Will retry " . ( $this->maxFailures - $failures - 1 ) .
" more times)";
682 if ( !$tryIsPrefetch ) {
686 sleep( $this->failureTimeout );
689 if ( $this->spawn ) {
693 }
catch ( Exception
$e ) {
694 $this->
progress(
"Rebooting getText infrastructure failed (" . $e->getMessage() .
")" .
695 " Trying to continue anyways" );
705 $consecutiveFailedTextRetrievals = $oldConsecutiveFailedTextRetrievals + 1;
706 if ( $consecutiveFailedTextRetrievals > $this->maxConsecutiveFailedTextRetrievals ) {
707 throw new MWException(
"Graceful storage failure" );
721 $address = ( is_int( $id ) || strpos( $id,
':' ) ===
false )
722 ? SqlBlobStore::makeAddressFromTextId( (
int)$id )
726 $text = $store->getBlob( $address );
728 $stripped = str_replace(
"\r",
"", $text );
729 $normalized = MediaWikiServices::getInstance()->getContentLanguage()
730 ->normalize( $stripped );
744 Wikimedia\suppressWarnings();
745 if ( !$this->spawnProc ) {
750 Wikimedia\restoreWarnings();
758 if ( file_exists(
"$IP/../multiversion/MWScript.php" ) ) {
763 "$IP/../multiversion/MWScript.php",
771 "$IP/maintenance/fetchText.php",
775 0 => [
"pipe",
"r" ],
776 1 => [
"pipe",
"w" ],
777 2 => [
"file",
"/dev/null",
"a" ] ];
780 $this->
progress(
"Spawning database subprocess: $cmd" );
781 $this->spawnProc = proc_open( $cmd, $spec, $pipes );
782 if ( !$this->spawnProc ) {
783 $this->
progress(
"Subprocess spawn failed." );
796 Wikimedia\suppressWarnings();
797 if ( $this->spawnRead ) {
798 fclose( $this->spawnRead );
800 $this->spawnRead =
false;
801 if ( $this->spawnWrite ) {
802 fclose( $this->spawnWrite );
804 $this->spawnWrite =
false;
805 if ( $this->spawnErr ) {
806 fclose( $this->spawnErr );
808 $this->spawnErr =
false;
809 if ( $this->spawnProc ) {
810 pclose( $this->spawnProc );
812 $this->spawnProc =
false;
813 Wikimedia\restoreWarnings();
821 if ( is_int( $address ) || intval( $address ) ) {
822 $address = SqlBlobStore::makeAddressFromTextId( (
int)$address );
825 $ok = fwrite( $this->spawnWrite,
"$address\n" );
831 $ok = fflush( $this->spawnWrite );
839 $newAddress = fgets( $this->spawnRead );
840 if ( $newAddress ===
false ) {
843 $newAddress = trim( $newAddress );
844 if ( strpos( $newAddress,
':' ) ===
false ) {
845 $newAddress = SqlBlobStore::makeAddressFromTextId( intval( $newAddress ) );
848 if ( $newAddress !== $address ) {
852 $len = fgets( $this->spawnRead );
854 if ( $len ===
false ) {
858 $nbytes = intval( $len );
867 while ( $nbytes > strlen( $text ) ) {
868 $buffer = fread( $this->spawnRead, $nbytes - strlen( $text ) );
875 $gotbytes = strlen( $text );
876 if ( $gotbytes != $nbytes ) {
877 $this->
progress(
"Expected $nbytes bytes from database subprocess, got $gotbytes " );
883 $stripped = str_replace(
"\r",
"", $text );
884 $normalized = MediaWikiServices::getInstance()->getContentLanguage()->
885 normalize( $stripped );
891 $this->checkpointJustWritten =
false;
894 $this->lastName =
$name;
896 if (
$name ==
'revision' ) {
897 $this->state =
$name;
898 $this->egress->writeOpenPage(
null, $this->buffer );
900 } elseif (
$name ==
'page' ) {
901 $this->state =
$name;
902 if ( $this->atStart ) {
903 $this->egress->writeOpenStream( $this->buffer );
905 $this->atStart =
false;
911 $model = trim( $this->thisRevModel );
912 $format = trim( $this->thisRevFormat );
914 $model = $model ===
'' ?
null : $model;
915 $format = $format ===
'' ?
null : $format;
917 $text = $this->
getText( $id, $model, $format );
918 $this->openElement = [
$name, [
'xml:space' =>
'preserve' ] ];
919 if ( strlen( $text ) > 0 ) {
928 $this->checkpointJustWritten =
false;
930 if ( $this->openElement ) {
933 $this->buffer .=
"</$name>";
936 if (
$name ==
'revision' ) {
937 $this->egress->writeRevision(
null, $this->buffer );
940 $this->thisRevModel =
null;
941 $this->thisRevFormat =
null;
942 } elseif (
$name ==
'page' ) {
943 if ( !$this->firstPageWritten ) {
944 $this->firstPageWritten = trim( $this->thisPage );
946 $this->lastPageWritten = trim( $this->thisPage );
947 if ( $this->timeExceeded ) {
948 $this->egress->writeClosePage( $this->buffer );
951 $this->egress->sink->write(
"\n" );
953 $this->buffer = $this->xmlwriterobj->closeStream();
954 $this->egress->writeCloseStream( $this->buffer );
957 $this->thisPage =
"";
960 $filenameList = (
array)$this->egress->getFilenames();
962 $firstPageID = str_pad( $this->firstPageWritten, 9,
"0", STR_PAD_LEFT );
963 $lastPageID = str_pad( $this->lastPageWritten, 9,
"0", STR_PAD_LEFT );
964 $filenamesCount =
count( $filenameList );
965 for ( $i = 0; $i < $filenamesCount; $i++ ) {
966 $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID );
967 $fileinfo = pathinfo( $filenameList[$i] );
968 $newFilenames[] =
$fileinfo[
'dirname'] .
'/' . $checkpointNameFilledIn;
970 $this->egress->closeRenameAndReopen( $newFilenames );
971 $this->buffer = $this->xmlwriterobj->openStream();
972 $this->timeExceeded =
false;
974 $this->firstPageWritten =
false;
975 $this->checkpointJustWritten =
true;
977 $this->egress->writeClosePage( $this->buffer );
979 $this->thisPage =
"";
981 } elseif (
$name ==
'mediawiki' ) {
982 $this->egress->writeCloseStream( $this->buffer );
989 if ( $this->lastName ==
"id" ) {
990 if ( $this->state ==
"revision" ) {
991 $this->thisRev .=
$data;
992 } elseif ( $this->state ==
"page" ) {
993 $this->thisPage .=
$data;
995 } elseif ( $this->lastName ==
"model" ) {
996 $this->thisRevModel .=
$data;
997 } elseif ( $this->lastName ==
"format" ) {
998 $this->thisRevFormat .=
$data;
1003 if ( $this->checkpointJustWritten ) {
1004 if (
$data[0] ==
"\n" ) {
1007 $this->checkpointJustWritten =
false;
1009 $this->buffer .= htmlspecialchars(
$data );
1013 if ( $this->openElement ) {
1014 $this->buffer .=
Xml::element( $this->openElement[0], $this->openElement[1], $style );
1015 $this->openElement =
false;