27require_once __DIR__ .
'/backup.inc';
28require_once __DIR__ .
'/../includes/export/WikiExporter.php';
104 parent::__construct();
107This script postprocesses XML dumps from dumpBackup.php to add
108page text which was stubbed out (
using --stub).
110XML input is accepted on stdin.
114 $this->stderr = fopen(
"php://stderr",
"wt" );
116 $this->
addOption(
'stub',
'To load a compressed stub dump instead of stdin. ' .
117 'Specify as --stub=<type>:<file>.',
false,
true );
118 $this->
addOption(
'prefetch',
'Use a prior dump file as a text source, to savepressure on the ' .
119 'database. (Requires the XMLReader extension). Specify as --prefetch=<type>:<file>',
121 $this->
addOption(
'maxtime',
'Write out checkpoint file after this many minutes (writing' .
122 'out complete page, closing xml file properly, and opening new one' .
123 'with header). This option requires the checkpointfile option.',
false,
true );
124 $this->
addOption(
'checkpointfile',
'Use this string for checkpoint filenames,substituting ' .
125 'first pageid written for the first %s (required) and the last pageid written for the ' .
126 'second %s if it exists.',
false,
true,
false,
true );
127 $this->
addOption(
'quiet',
'Don\'t dump status reports to stderr.' );
128 $this->
addOption(
'current',
'Base ETA on number of pages in database instead of all revisions' );
129 $this->
addOption(
'spawn',
'Spawn a subprocess for loading text records' );
130 $this->
addOption(
'buffersize',
'Buffer size in bytes to use for reading the stub. ' .
131 '(Default: 512KB, Minimum: 4KB)',
false,
true );
147 parent::processOptions();
149 if ( $this->
hasOption(
'buffersize' ) ) {
150 $this->bufferSize = max( intval( $this->
getOption(
'buffersize' ) ), 4 * 1024 );
154 require_once
"$IP/maintenance/backupPrefetch.inc";
156 $this->prefetch =
new BaseDump( $url );
164 $this->maxTimeAllowed = intval( $this->
getOption(
'maxtime' ) ) * 60;
167 if ( $this->
hasOption(
'checkpointfile' ) ) {
168 $this->checkpointFiles = $this->
getOption(
'checkpointfile' );
201 if ( isset( $this->lb ) ) {
202 $this->lb->closeAll();
206 if ( $this->forcedDb !==
null ) {
212 if ( isset( $this->db ) && $this->db->isOpen() ) {
213 throw new MWException(
'DB is set and has not been closed by the Load Balancer' );
224 }
catch ( Exception
$e ) {
226 .
" rotating DB failed to obtain new load balancer (" .
$e->getMessage() .
")" );
230 $this->db = $this->lb->getConnection(
DB_REPLICA,
'dump' );
231 }
catch ( Exception
$e ) {
233 .
" rotating DB failed to obtain new database (" .
$e->getMessage() .
")" );
238 parent::initProgress();
239 $this->timeOfCheckpoint = $this->startTime;
245 if ( ini_get(
'display_errors' ) ) {
246 ini_set(
'display_errors',
'stderr' );
258 }
catch ( Exception
$e ) {
261 $this->
progress(
"Getting initial DB connection failed (" .
262 $e->getMessage() .
")" );
273 $input = fopen( $this->input,
"rt" );
276 if ( $this->spawnProc ) {
284 $split = explode(
':',
$opt, 2 );
287 if ( count( $split ) === 2 ) {
290 $fileURIs = explode(
';', $param );
291 foreach ( $fileURIs as $URI ) {
297 $newURI =
"compress.zlib://$URI";
300 $newURI =
"compress.bzip2://$URI";
303 $newURI =
"mediawiki.compress.7z://$URI";
308 $newFileURIs[] = $newURI;
310 $val = implode(
';', $newFileURIs );
319 if ( !$this->prefetch ) {
320 parent::showReport();
325 if ( $this->reporting ) {
327 $nowts = microtime(
true );
328 $deltaAll = $nowts - $this->startTime;
334 $portion = $this->
revCount / $this->maxCount;
335 $eta = $this->startTime + $deltaAll / $portion;
337 if ( $this->fetchCount ) {
342 $pageRate = $this->pageCount / $deltaAll;
343 $revRate = $this->
revCount / $deltaAll;
351 if ( $this->fetchCountLast ) {
354 $fetchRatePart =
'-';
356 $pageRatePart = $this->pageCountPart / $deltaPart;
357 $revRatePart = $this->revCountPart / $deltaPart;
359 $fetchRatePart =
'-';
364 "%s: %s (ID %d) %d pages (%0.1f|%0.1f/sec all|curr), "
365 .
"%d revs (%0.1f|%0.1f/sec all|curr), %0.1f%%|%0.1f%% "
366 .
"prefetched (all|curr), ETA %s [max %d]",
367 $now,
wfWikiID(), $this->ID, $this->pageCount, $pageRate,
368 $pageRatePart, $this->
revCount, $revRate, $revRatePart,
369 $fetchRate, $fetchRatePart, $etats, $this->maxCount
371 $this->lastTime = $nowts;
379 $this->timeExceeded =
true;
383 if ( $this->maxTimeAllowed
384 && ( $this->lastTime - $this->timeOfCheckpoint > $this->maxTimeAllowed )
393 if ( ( $this->checkpointFiles && !$this->maxTimeAllowed )
394 || ( $this->maxTimeAllowed && !$this->checkpointFiles )
396 throw new MWException(
"Options checkpointfile and maxtime must be specified together.\n" );
398 foreach ( $this->checkpointFiles as $checkpointFile ) {
399 $count = substr_count( $checkpointFile,
"%s" );
401 throw new MWException(
"Option checkpointfile must contain two '%s' "
402 .
"for substitution of first and last pageids, count is $count instead, "
403 .
"file is $checkpointFile.\n" );
407 if ( $this->checkpointFiles ) {
408 $filenameList = (
array)$this->egress->getFilenames();
409 if ( count( $filenameList ) != count( $this->checkpointFiles ) ) {
410 throw new MWException(
"One checkpointfile must be specified "
411 .
"for each output option, if maxtime is used.\n" );
423 $this->openElement =
false;
424 $this->atStart =
true;
426 $this->lastName =
"";
429 $this->thisRevModel =
null;
430 $this->thisRevFormat =
null;
432 $parser = xml_parser_create(
"UTF-8" );
433 xml_parser_set_option(
$parser, XML_OPTION_CASE_FOLDING,
false );
435 xml_set_element_handler(
437 [ $this,
'startElement' ],
438 [ $this,
'endElement' ]
440 xml_set_character_data_handler(
$parser, [ $this,
'characterData' ] );
447 $chunk = fread(
$input, $this->bufferSize );
449 wfDebug(
"TextDumpPass::readDump encountered XML parsing error\n" );
451 $byte = xml_get_current_byte_index(
$parser );
453 'XML import parse failure',
454 xml_get_current_line_number(
$parser ),
455 xml_get_current_column_number(
$parser ),
456 $byte . ( is_null( $chunk ) ?
null : (
'; "' . substr( $chunk, $byte - $offset, 16 ) .
'"' ) ),
457 xml_error_string( xml_get_error_code(
$parser ) ) )->escaped();
463 $offset += strlen( $chunk );
464 }
while ( $chunk !==
false && !feof(
$input ) );
465 if ( $this->maxTimeAllowed ) {
466 $filenameList = (
array)$this->egress->getFilenames();
468 if ( file_exists( $filenameList[0] ) ) {
470 # we might have just written the header and footer and had no
471 # pages or revisions written... perhaps they were all deleted
472 # there's no pageID 0 so we use that. the caller is responsible
473 # for deciding what to do with a file containing only the
474 # siteinfo information and the mw tags.
475 if ( !$this->firstPageWritten ) {
476 $firstPageID = str_pad( 0, 9,
"0", STR_PAD_LEFT );
477 $lastPageID = str_pad( 0, 9,
"0", STR_PAD_LEFT );
479 $firstPageID = str_pad( $this->firstPageWritten, 9,
"0", STR_PAD_LEFT );
480 $lastPageID = str_pad( $this->lastPageWritten, 9,
"0", STR_PAD_LEFT );
483 $filenameCount = count( $filenameList );
484 for ( $i = 0; $i < $filenameCount; $i++ ) {
485 $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID );
486 $fileinfo = pathinfo( $filenameList[$i] );
487 $newFilenames[] =
$fileinfo[
'dirname'] .
'/' . $checkpointNameFilledIn;
489 $this->egress->closeAndRename( $newFilenames );
509 $text =
$handler->exportTransform( $text, $format );
513 "Unable to apply export transformation for content model '$model': " .
541 function getText( $id, $model =
null, $format =
null ) {
544 $prefetchNotTried =
true;
549 static $consecutiveFailedTextRetrievals = 0;
556 $oldConsecutiveFailedTextRetrievals = $consecutiveFailedTextRetrievals;
557 $consecutiveFailedTextRetrievals = 0;
560 $row = $this->db->selectRow(
562 [
'rev_content_model',
'rev_content_format' ],
563 [
'rev_id' => $this->thisRev ],
568 $model = $row->rev_content_model;
569 $format = $row->rev_content_format;
573 if ( $model ===
null || $model ===
'' ) {
577 while ( $failures < $this->maxFailures ) {
587 if ( $text ===
false && isset( $this->prefetch ) && $prefetchNotTried ) {
588 $prefetchNotTried =
false;
589 $tryIsPrefetch =
true;
590 $text = $this->prefetch->prefetch( (
int)$this->thisPage, (
int)$this->thisRev );
592 if ( $text ===
null ) {
596 if ( is_string( $text ) && $model !==
false ) {
606 if ( $text ===
false ) {
608 $tryIsPrefetch =
false;
609 if ( $this->spawn ) {
615 if ( $text !==
false && $model !==
false ) {
624 if ( $text !==
false ) {
629 if ( $text ===
false ) {
630 throw new MWException(
"Generic error while obtaining text for id " . $id );
637 $revID = intval( $this->thisRev );
638 if ( !isset( $this->db ) ) {
643 $revLength = strlen( $text );
645 $revLength = $this->db->selectField(
'revision',
'rev_len', [
'rev_id' => $revID ] );
648 if ( strlen( $text ) == $revLength ) {
649 if ( $tryIsPrefetch ) {
650 $this->prefetchCount++;
657 throw new MWException(
"Received text is unplausible for id " . $id );
658 }
catch ( Exception
$e ) {
659 $msg =
"getting/checking text " . $id .
" failed (" .
$e->getMessage() .
")";
660 if ( $failures + 1 < $this->maxFailures ) {
661 $msg .=
" (Will retry " . ( $this->maxFailures - $failures - 1 ) .
" more times)";
670 if ( !$tryIsPrefetch ) {
674 sleep( $this->failureTimeout );
677 if ( $this->spawn ) {
681 }
catch ( Exception
$e ) {
682 $this->
progress(
"Rebooting getText infrastructure failed (" . $e->getMessage() .
")" .
683 " Trying to continue anyways" );
693 $consecutiveFailedTextRetrievals = $oldConsecutiveFailedTextRetrievals + 1;
694 if ( $consecutiveFailedTextRetrievals > $this->maxConsecutiveFailedTextRetrievals ) {
695 throw new MWException(
"Graceful storage failure" );
709 if ( !isset( $this->db ) ) {
710 throw new MWException( __METHOD__ .
"No database available" );
712 $row = $this->db->selectRow(
'text',
713 [
'old_text',
'old_flags' ],
717 if ( $text ===
false ) {
720 $stripped = str_replace(
"\r",
"", $text );
727 MediaWiki\suppressWarnings();
728 if ( !$this->spawnProc ) {
733 MediaWiki\restoreWarnings();
741 if ( file_exists(
"$IP/../multiversion/MWScript.php" ) ) {
743 array_map(
'wfEscapeShellArg',
746 "$IP/../multiversion/MWScript.php",
751 array_map(
'wfEscapeShellArg',
754 "$IP/maintenance/fetchText.php",
758 0 => [
"pipe",
"r" ],
759 1 => [
"pipe",
"w" ],
760 2 => [
"file",
"/dev/null",
"a" ] ];
763 $this->
progress(
"Spawning database subprocess: $cmd" );
764 $this->spawnProc = proc_open( $cmd, $spec, $pipes );
765 if ( !$this->spawnProc ) {
766 $this->
progress(
"Subprocess spawn failed." );
779 MediaWiki\suppressWarnings();
780 if ( $this->spawnRead ) {
781 fclose( $this->spawnRead );
783 $this->spawnRead =
false;
784 if ( $this->spawnWrite ) {
785 fclose( $this->spawnWrite );
787 $this->spawnWrite =
false;
788 if ( $this->spawnErr ) {
789 fclose( $this->spawnErr );
791 $this->spawnErr =
false;
792 if ( $this->spawnProc ) {
793 pclose( $this->spawnProc );
795 $this->spawnProc =
false;
796 MediaWiki\restoreWarnings();
802 $ok = fwrite( $this->spawnWrite,
"$id\n" );
808 $ok = fflush( $this->spawnWrite );
816 $newId = fgets( $this->spawnRead );
817 if ( $newId ===
false ) {
820 if ( $id != intval( $newId ) ) {
824 $len = fgets( $this->spawnRead );
826 if ( $len ===
false ) {
830 $nbytes = intval( $len );
839 while ( $nbytes > strlen( $text ) ) {
840 $buffer = fread( $this->spawnRead, $nbytes - strlen( $text ) );
847 $gotbytes = strlen( $text );
848 if ( $gotbytes != $nbytes ) {
849 $this->
progress(
"Expected $nbytes bytes from database subprocess, got $gotbytes " );
855 $stripped = str_replace(
"\r",
"", $text );
862 $this->checkpointJustWritten =
false;
865 $this->lastName =
$name;
867 if ( $name ==
'revision' ) {
868 $this->state =
$name;
869 $this->egress->writeOpenPage(
null, $this->buffer );
871 } elseif ( $name ==
'page' ) {
872 $this->state =
$name;
873 if ( $this->atStart ) {
874 $this->egress->writeOpenStream( $this->buffer );
876 $this->atStart =
false;
880 if ( $name ==
"text" && isset(
$attribs[
'id'] ) ) {
882 $model = trim( $this->thisRevModel );
883 $format = trim( $this->thisRevFormat );
885 $model = $model ===
'' ? null : $model;
886 $format = $format ===
'' ? null : $format;
888 $text = $this->
getText( $id, $model, $format );
889 $this->openElement = [
$name, [
'xml:space' =>
'preserve' ] ];
890 if ( strlen( $text ) > 0 ) {
899 $this->checkpointJustWritten =
false;
901 if ( $this->openElement ) {
904 $this->buffer .=
"</$name>";
907 if ( $name ==
'revision' ) {
908 $this->egress->writeRevision(
null, $this->buffer );
911 $this->thisRevModel =
null;
912 $this->thisRevFormat =
null;
913 } elseif ( $name ==
'page' ) {
914 if ( !$this->firstPageWritten ) {
915 $this->firstPageWritten = trim( $this->thisPage );
917 $this->lastPageWritten = trim( $this->thisPage );
918 if ( $this->timeExceeded ) {
919 $this->egress->writeClosePage( $this->buffer );
922 $this->egress->sink->write(
"\n" );
924 $this->buffer = $this->xmlwriterobj->closeStream();
925 $this->egress->writeCloseStream( $this->buffer );
928 $this->thisPage =
"";
931 $filenameList = (
array)$this->egress->getFilenames();
933 $firstPageID = str_pad( $this->firstPageWritten, 9,
"0", STR_PAD_LEFT );
934 $lastPageID = str_pad( $this->lastPageWritten, 9,
"0", STR_PAD_LEFT );
935 $filenamesCount = count( $filenameList );
936 for ( $i = 0; $i < $filenamesCount; $i++ ) {
937 $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID );
938 $fileinfo = pathinfo( $filenameList[$i] );
939 $newFilenames[] =
$fileinfo[
'dirname'] .
'/' . $checkpointNameFilledIn;
941 $this->egress->closeRenameAndReopen( $newFilenames );
942 $this->buffer = $this->xmlwriterobj->openStream();
943 $this->timeExceeded =
false;
945 $this->firstPageWritten =
false;
946 $this->checkpointJustWritten =
true;
948 $this->egress->writeClosePage( $this->buffer );
950 $this->thisPage =
"";
952 } elseif ( $name ==
'mediawiki' ) {
953 $this->egress->writeCloseStream( $this->buffer );
960 if ( $this->lastName ==
"id" ) {
961 if ( $this->state ==
"revision" ) {
962 $this->thisRev .= $data;
963 } elseif ( $this->state ==
"page" ) {
964 $this->thisPage .= $data;
966 } elseif ( $this->lastName ==
"model" ) {
967 $this->thisRevModel .= $data;
968 } elseif ( $this->lastName ==
"format" ) {
969 $this->thisRevFormat .= $data;
974 if ( $this->checkpointJustWritten ) {
975 if ( $data[0] ==
"\n" ) {
976 $data = substr( $data, 1 );
978 $this->checkpointJustWritten =
false;
980 $this->buffer .= htmlspecialchars( $data );
984 if ( $this->openElement ) {
985 $this->buffer .= Xml::element( $this->openElement[0], $this->openElement[1], $style );
986 $this->openElement =
false;
$wgContentHandlerUseDB
Set to false to disable use of the database fields introduced by the ContentHandler facility.
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
wfGetLBFactory()
Get the load balancer factory object.
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
wfWikiID()
Get an ASCII string identifying this wiki This is used as a prefix in memcached keys.
IDatabase null $forcedDb
The dependency-injected database to use.
Readahead helper for making large MediaWiki data dumps; reads in a previous XML dump to sequentially ...
static getForModelID( $modelId)
Returns the ContentHandler singleton for the given model ID.
hasOption( $name)
Checks to see if a particular param exists.
addDescription( $text)
Set the description text.
loadWithArgv( $argv)
Load params and arguments from a given array of command-line arguments.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
static getRevisionText( $row, $prefix='old_', $wiki=false)
Get revision text associated with an old or archive row.
$maxConsecutiveFailedTextRetrievals
showReport()
Overridden to include prefetch ratio if enabled.
getTextDb( $id)
May throw a database error if, say, the server dies during query.
endElement( $parser, $name)
clearOpenElement( $style)
getText( $id, $model=null, $format=null)
Tries to get the revision text for a revision id.
exportTransform( $text, $model, $format=null)
Applies applicable export transformations to $text.
startElement( $parser, $name, $attribs)
processOptions()
Processes arguments and sets $this->$sink accordingly.
characterData( $parser, $data)
initProgress( $history=WikiExporter::FULL)
Initialise starting time and maximum revision count.
execute()
Do the actual work.
dump( $history, $text=WikiExporter::TEXT)
bool XmlDumpWriter $xmlwriterobj
IMaintainableDatabase $db
bool resource $spawnWrite
rotateDb()
Drop the database connection $this->db and try to get a new one.
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
this class mediates it Skin Encapsulates a look and feel for the wiki All of the functions that render HTML and make choices about how to render it are here and are called from various other places when and is meant to be subclassed with other skins that may override some of its functions The User object contains a reference to a and so rather than having a global skin object we just rely on the global User and get the skin with $wgUser and also has some character encoding functions and other locale stuff The current user interface language is instantiated as and the local content language as $wgContLang
design txt This is a brief overview of the new design More thorough and up to date information is available on the documentation wiki at etc Handles the details of getting and saving to the user table of the and dealing with sessions and cookies OutputPage Encapsulates the entire HTML page that will be sent in response to any server request It is used by calling its functions to add in any and then calling output() to send it all. It could be easily changed to send incrementally if that becomes useful
the array() calling protocol came about after MediaWiki 1.4rc1.
do that in ParserLimitReportFormat instead $parser
presenting them properly to the user as errors is done by the caller return true use this to change the list i e etc next in line in page history
either a unescaped string or a HtmlArmor object after in associative array form externallinks including delete and has completed for all link tables whether this was an auto creation default is conds Array Extra conditions for the No matching items in log is displayed if loglist is empty msgKey Array If you want a nice box with a set this to the key of the message First element is the message additional optional elements are parameters for the key that are processed with wfMessage() -> params() ->parseAsBlock() - offset Set to overwrite offset parameter in $wgRequest set to '' to unset offset - wrap String Wrap the message in html(usually something like "<div ...>$1</div>"). - flags Integer display flags(NO_ACTION_LINK, NO_EXTRA_USER_LINKS) 'LogException':Called before an exception(or PHP error) is logged. This is meant for integration with external error aggregation services
Allows to change the fields on the form that will be generated $name
null means default in associative array with keys and values unescaped Should be merged with default with a value of false meaning to suppress the attribute in associative array with keys and values unescaped noclasses just before the function returns a value If you return an< a > element with HTML attributes $attribs and contents $html will be returned If you return $ret will be returned and may include noclasses after processing & $attribs
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that probably a stub it is not rendered in wiki pages or galleries in category pages allow injecting custom HTML after the section Any uses of the hook need to handle escaping see BaseTemplate::getToolbox and BaseTemplate::makeListItem for details on the format of individual items inside of this array or by returning and letting standard HTTP rendering take place modifiable or by returning false and taking over the output modifiable modifiable after all normalizations have been except for the $wgMaxImageArea check set to true or false to override the $wgMaxImageArea check result gives extension the possibility to transform it themselves $handler
returning false will NOT prevent logging $e
const CONTENT_MODEL_WIKITEXT
Advanced database interface for IDatabase handles that include maintenance methods.
require_once RUN_MAINTENANCE_IF_MAIN