MediaWiki REL1_30
dumpTextPass.php
Go to the documentation of this file.
1<?php
27require_once __DIR__ . '/backup.inc';
28require_once __DIR__ . '/../includes/export/WikiExporter.php';
29
31
37 public $prefetch = null;
39 private $thisPage;
41 private $thisRev;
42
43 // when we spend more than maxTimeAllowed seconds on this run, we continue
44 // processing until we write out the next complete page, then save output file(s),
45 // rename it/them and open new one(s)
46 public $maxTimeAllowed = 0; // 0 = no limit
47
48 protected $input = "php://stdin";
50 protected $fetchCount = 0;
51 protected $prefetchCount = 0;
52 protected $prefetchCountLast = 0;
53 protected $fetchCountLast = 0;
54
55 protected $maxFailures = 5;
57 protected $failureTimeout = 5; // Seconds to sleep after db failure
58
59 protected $bufferSize = 524288; // In bytes. Maximum size to read from the stub in on go.
60
61 protected $php = "php";
62 protected $spawn = false;
63
67 protected $spawnProc = false;
68
72 protected $spawnWrite = false;
73
77 protected $spawnRead = false;
78
82 protected $spawnErr = false;
83
87 protected $xmlwriterobj = false;
88
89 protected $timeExceeded = false;
90 protected $firstPageWritten = false;
91 protected $lastPageWritten = false;
92 protected $checkpointJustWritten = false;
93 protected $checkpointFiles = [];
94
98 protected $db;
99
103 function __construct( $args = null ) {
104 parent::__construct();
105
106 $this->addDescription( <<<TEXT
107This script postprocesses XML dumps from dumpBackup.php to add
108page text which was stubbed out (using --stub).
109
110XML input is accepted on stdin.
111XML output is sent to stdout; progress reports are sent to stderr.
112TEXT
113 );
114 $this->stderr = fopen( "php://stderr", "wt" );
115
116 $this->addOption( 'stub', 'To load a compressed stub dump instead of stdin. ' .
117 'Specify as --stub=<type>:<file>.', false, true );
118 $this->addOption( 'prefetch', 'Use a prior dump file as a text source, to savepressure on the ' .
119 'database. (Requires the XMLReader extension). Specify as --prefetch=<type>:<file>',
120 false, true );
121 $this->addOption( 'maxtime', 'Write out checkpoint file after this many minutes (writing' .
122 'out complete page, closing xml file properly, and opening new one' .
123 'with header). This option requires the checkpointfile option.', false, true );
124 $this->addOption( 'checkpointfile', 'Use this string for checkpoint filenames,substituting ' .
125 'first pageid written for the first %s (required) and the last pageid written for the ' .
126 'second %s if it exists.', false, true, false, true ); // This can be specified multiple times
127 $this->addOption( 'quiet', 'Don\'t dump status reports to stderr.' );
128 $this->addOption( 'current', 'Base ETA on number of pages in database instead of all revisions' );
129 $this->addOption( 'spawn', 'Spawn a subprocess for loading text records' );
130 $this->addOption( 'buffersize', 'Buffer size in bytes to use for reading the stub. ' .
131 '(Default: 512KB, Minimum: 4KB)', false, true );
132
133 if ( $args ) {
134 $this->loadWithArgv( $args );
135 $this->processOptions();
136 }
137 }
138
139 function execute() {
140 $this->processOptions();
141 $this->dump( true );
142 }
143
144 function processOptions() {
145 global $IP;
146
147 parent::processOptions();
148
149 if ( $this->hasOption( 'buffersize' ) ) {
150 $this->bufferSize = max( intval( $this->getOption( 'buffersize' ) ), 4 * 1024 );
151 }
152
153 if ( $this->hasOption( 'prefetch' ) ) {
154 require_once "$IP/maintenance/backupPrefetch.inc";
155 $url = $this->processFileOpt( $this->getOption( 'prefetch' ) );
156 $this->prefetch = new BaseDump( $url );
157 }
158
159 if ( $this->hasOption( 'stub' ) ) {
160 $this->input = $this->processFileOpt( $this->getOption( 'stub' ) );
161 }
162
163 if ( $this->hasOption( 'maxtime' ) ) {
164 $this->maxTimeAllowed = intval( $this->getOption( 'maxtime' ) ) * 60;
165 }
166
167 if ( $this->hasOption( 'checkpointfile' ) ) {
168 $this->checkpointFiles = $this->getOption( 'checkpointfile' );
169 }
170
171 if ( $this->hasOption( 'current' ) ) {
173 }
174
175 if ( $this->hasOption( 'full' ) ) {
177 }
178
179 if ( $this->hasOption( 'spawn' ) ) {
180 $this->spawn = true;
181 $val = $this->getOption( 'spawn' );
182 if ( $val !== 1 ) {
183 $this->php = $val;
184 }
185 }
186 }
187
199 function rotateDb() {
200 // Cleaning up old connections
201 if ( isset( $this->lb ) ) {
202 $this->lb->closeAll();
203 unset( $this->lb );
204 }
205
206 if ( $this->forcedDb !== null ) {
207 $this->db = $this->forcedDb;
208
209 return;
210 }
211
212 if ( isset( $this->db ) && $this->db->isOpen() ) {
213 throw new MWException( 'DB is set and has not been closed by the Load Balancer' );
214 }
215
216 unset( $this->db );
217
218 // Trying to set up new connection.
219 // We do /not/ retry upon failure, but delegate to encapsulating logic, to avoid
220 // individually retrying at different layers of code.
221
222 try {
223 $this->lb = wfGetLBFactory()->newMainLB();
224 } catch ( Exception $e ) {
225 throw new MWException( __METHOD__
226 . " rotating DB failed to obtain new load balancer (" . $e->getMessage() . ")" );
227 }
228
229 try {
230 $this->db = $this->lb->getConnection( DB_REPLICA, 'dump' );
231 } catch ( Exception $e ) {
232 throw new MWException( __METHOD__
233 . " rotating DB failed to obtain new database (" . $e->getMessage() . ")" );
234 }
235 }
236
238 parent::initProgress();
239 $this->timeOfCheckpoint = $this->startTime;
240 }
241
242 function dump( $history, $text = WikiExporter::TEXT ) {
243 // Notice messages will foul up your XML output even if they're
244 // relatively harmless.
245 if ( ini_get( 'display_errors' ) ) {
246 ini_set( 'display_errors', 'stderr' );
247 }
248
249 $this->initProgress( $this->history );
250
251 // We are trying to get an initial database connection to avoid that the
252 // first try of this request's first call to getText fails. However, if
253 // obtaining a good DB connection fails it's not a serious issue, as
254 // getText does retry upon failure and can start without having a working
255 // DB connection.
256 try {
257 $this->rotateDb();
258 } catch ( Exception $e ) {
259 // We do not even count this as failure. Just let eventual
260 // watchdogs know.
261 $this->progress( "Getting initial DB connection failed (" .
262 $e->getMessage() . ")" );
263 }
264
265 $this->egress = new ExportProgressFilter( $this->sink, $this );
266
267 // it would be nice to do it in the constructor, oh well. need egress set
268 $this->finalOptionCheck();
269
270 // we only want this so we know how to close a stream :-P
271 $this->xmlwriterobj = new XmlDumpWriter();
272
273 $input = fopen( $this->input, "rt" );
274 $this->readDump( $input );
275
276 if ( $this->spawnProc ) {
277 $this->closeSpawn();
278 }
279
280 $this->report( true );
281 }
282
283 function processFileOpt( $opt ) {
284 $split = explode( ':', $opt, 2 );
285 $val = $split[0];
286 $param = '';
287 if ( count( $split ) === 2 ) {
288 $param = $split[1];
289 }
290 $fileURIs = explode( ';', $param );
291 foreach ( $fileURIs as $URI ) {
292 switch ( $val ) {
293 case "file":
294 $newURI = $URI;
295 break;
296 case "gzip":
297 $newURI = "compress.zlib://$URI";
298 break;
299 case "bzip2":
300 $newURI = "compress.bzip2://$URI";
301 break;
302 case "7zip":
303 $newURI = "mediawiki.compress.7z://$URI";
304 break;
305 default:
306 $newURI = $URI;
307 }
308 $newFileURIs[] = $newURI;
309 }
310 $val = implode( ';', $newFileURIs );
311
312 return $val;
313 }
314
318 function showReport() {
319 if ( !$this->prefetch ) {
320 parent::showReport();
321
322 return;
323 }
324
325 if ( $this->reporting ) {
326 $now = wfTimestamp( TS_DB );
327 $nowts = microtime( true );
328 $deltaAll = $nowts - $this->startTime;
329 $deltaPart = $nowts - $this->lastTime;
330 $this->pageCountPart = $this->pageCount - $this->pageCountLast;
331 $this->revCountPart = $this->revCount - $this->revCountLast;
332
333 if ( $deltaAll ) {
334 $portion = $this->revCount / $this->maxCount;
335 $eta = $this->startTime + $deltaAll / $portion;
336 $etats = wfTimestamp( TS_DB, intval( $eta ) );
337 if ( $this->fetchCount ) {
338 $fetchRate = 100.0 * $this->prefetchCount / $this->fetchCount;
339 } else {
340 $fetchRate = '-';
341 }
342 $pageRate = $this->pageCount / $deltaAll;
343 $revRate = $this->revCount / $deltaAll;
344 } else {
345 $pageRate = '-';
346 $revRate = '-';
347 $etats = '-';
348 $fetchRate = '-';
349 }
350 if ( $deltaPart ) {
351 if ( $this->fetchCountLast ) {
352 $fetchRatePart = 100.0 * $this->prefetchCountLast / $this->fetchCountLast;
353 } else {
354 $fetchRatePart = '-';
355 }
356 $pageRatePart = $this->pageCountPart / $deltaPart;
357 $revRatePart = $this->revCountPart / $deltaPart;
358 } else {
359 $fetchRatePart = '-';
360 $pageRatePart = '-';
361 $revRatePart = '-';
362 }
363 $this->progress( sprintf(
364 "%s: %s (ID %d) %d pages (%0.1f|%0.1f/sec all|curr), "
365 . "%d revs (%0.1f|%0.1f/sec all|curr), %0.1f%%|%0.1f%% "
366 . "prefetched (all|curr), ETA %s [max %d]",
367 $now, wfWikiID(), $this->ID, $this->pageCount, $pageRate,
368 $pageRatePart, $this->revCount, $revRate, $revRatePart,
369 $fetchRate, $fetchRatePart, $etats, $this->maxCount
370 ) );
371 $this->lastTime = $nowts;
372 $this->revCountLast = $this->revCount;
373 $this->prefetchCountLast = $this->prefetchCount;
374 $this->fetchCountLast = $this->fetchCount;
375 }
376 }
377
378 function setTimeExceeded() {
379 $this->timeExceeded = true;
380 }
381
383 if ( $this->maxTimeAllowed
384 && ( $this->lastTime - $this->timeOfCheckpoint > $this->maxTimeAllowed )
385 ) {
386 return true;
387 }
388
389 return false;
390 }
391
392 function finalOptionCheck() {
393 if ( ( $this->checkpointFiles && !$this->maxTimeAllowed )
394 || ( $this->maxTimeAllowed && !$this->checkpointFiles )
395 ) {
396 throw new MWException( "Options checkpointfile and maxtime must be specified together.\n" );
397 }
398 foreach ( $this->checkpointFiles as $checkpointFile ) {
399 $count = substr_count( $checkpointFile, "%s" );
400 if ( $count != 2 ) {
401 throw new MWException( "Option checkpointfile must contain two '%s' "
402 . "for substitution of first and last pageids, count is $count instead, "
403 . "file is $checkpointFile.\n" );
404 }
405 }
406
407 if ( $this->checkpointFiles ) {
408 $filenameList = (array)$this->egress->getFilenames();
409 if ( count( $filenameList ) != count( $this->checkpointFiles ) ) {
410 throw new MWException( "One checkpointfile must be specified "
411 . "for each output option, if maxtime is used.\n" );
412 }
413 }
414 }
415
421 function readDump( $input ) {
422 $this->buffer = "";
423 $this->openElement = false;
424 $this->atStart = true;
425 $this->state = "";
426 $this->lastName = "";
427 $this->thisPage = 0;
428 $this->thisRev = 0;
429 $this->thisRevModel = null;
430 $this->thisRevFormat = null;
431
432 $parser = xml_parser_create( "UTF-8" );
433 xml_parser_set_option( $parser, XML_OPTION_CASE_FOLDING, false );
434
435 xml_set_element_handler(
436 $parser,
437 [ $this, 'startElement' ],
438 [ $this, 'endElement' ]
439 );
440 xml_set_character_data_handler( $parser, [ $this, 'characterData' ] );
441
442 $offset = 0; // for context extraction on error reporting
443 do {
444 if ( $this->checkIfTimeExceeded() ) {
445 $this->setTimeExceeded();
446 }
447 $chunk = fread( $input, $this->bufferSize );
448 if ( !xml_parse( $parser, $chunk, feof( $input ) ) ) {
449 wfDebug( "TextDumpPass::readDump encountered XML parsing error\n" );
450
451 $byte = xml_get_current_byte_index( $parser );
452 $msg = wfMessage( 'xml-error-string',
453 'XML import parse failure',
454 xml_get_current_line_number( $parser ),
455 xml_get_current_column_number( $parser ),
456 $byte . ( is_null( $chunk ) ? null : ( '; "' . substr( $chunk, $byte - $offset, 16 ) . '"' ) ),
457 xml_error_string( xml_get_error_code( $parser ) ) )->escaped();
458
459 xml_parser_free( $parser );
460
461 throw new MWException( $msg );
462 }
463 $offset += strlen( $chunk );
464 } while ( $chunk !== false && !feof( $input ) );
465 if ( $this->maxTimeAllowed ) {
466 $filenameList = (array)$this->egress->getFilenames();
467 // we wrote some stuff after last checkpoint that needs renamed
468 if ( file_exists( $filenameList[0] ) ) {
469 $newFilenames = [];
470 # we might have just written the header and footer and had no
471 # pages or revisions written... perhaps they were all deleted
472 # there's no pageID 0 so we use that. the caller is responsible
473 # for deciding what to do with a file containing only the
474 # siteinfo information and the mw tags.
475 if ( !$this->firstPageWritten ) {
476 $firstPageID = str_pad( 0, 9, "0", STR_PAD_LEFT );
477 $lastPageID = str_pad( 0, 9, "0", STR_PAD_LEFT );
478 } else {
479 $firstPageID = str_pad( $this->firstPageWritten, 9, "0", STR_PAD_LEFT );
480 $lastPageID = str_pad( $this->lastPageWritten, 9, "0", STR_PAD_LEFT );
481 }
482
483 $filenameCount = count( $filenameList );
484 for ( $i = 0; $i < $filenameCount; $i++ ) {
485 $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID );
486 $fileinfo = pathinfo( $filenameList[$i] );
487 $newFilenames[] = $fileinfo['dirname'] . '/' . $checkpointNameFilledIn;
488 }
489 $this->egress->closeAndRename( $newFilenames );
490 }
491 }
492 xml_parser_free( $parser );
493
494 return true;
495 }
496
506 private function exportTransform( $text, $model, $format = null ) {
507 try {
509 $text = $handler->exportTransform( $text, $format );
510 }
511 catch ( MWException $ex ) {
512 $this->progress(
513 "Unable to apply export transformation for content model '$model': " .
514 $ex->getMessage()
515 );
516 }
517
518 return $text;
519 }
520
541 function getText( $id, $model = null, $format = null ) {
543
544 $prefetchNotTried = true; // Whether or not we already tried to get the text via prefetch.
545 $text = false; // The candidate for a good text. false if no proper value.
546 $failures = 0; // The number of times, this invocation of getText already failed.
547
548 // The number of times getText failed without yielding a good text in between.
549 static $consecutiveFailedTextRetrievals = 0;
550
551 $this->fetchCount++;
552
553 // To allow to simply return on success and do not have to worry about book keeping,
554 // we assume, this fetch works (possible after some retries). Nevertheless, we koop
555 // the old value, so we can restore it, if problems occur (See after the while loop).
556 $oldConsecutiveFailedTextRetrievals = $consecutiveFailedTextRetrievals;
557 $consecutiveFailedTextRetrievals = 0;
558
559 if ( $model === null && $wgContentHandlerUseDB ) {
560 $row = $this->db->selectRow(
561 'revision',
562 [ 'rev_content_model', 'rev_content_format' ],
563 [ 'rev_id' => $this->thisRev ],
564 __METHOD__
565 );
566
567 if ( $row ) {
568 $model = $row->rev_content_model;
569 $format = $row->rev_content_format;
570 }
571 }
572
573 if ( $model === null || $model === '' ) {
574 $model = false;
575 }
576
577 while ( $failures < $this->maxFailures ) {
578 // As soon as we found a good text for the $id, we will return immediately.
579 // Hence, if we make it past the try catch block, we know that we did not
580 // find a good text.
581
582 try {
583 // Step 1: Get some text (or reuse from previous iteratuon if checking
584 // for plausibility failed)
585
586 // Trying to get prefetch, if it has not been tried before
587 if ( $text === false && isset( $this->prefetch ) && $prefetchNotTried ) {
588 $prefetchNotTried = false;
589 $tryIsPrefetch = true;
590 $text = $this->prefetch->prefetch( (int)$this->thisPage, (int)$this->thisRev );
591
592 if ( $text === null ) {
593 $text = false;
594 }
595
596 if ( is_string( $text ) && $model !== false ) {
597 // Apply export transformation to text coming from an old dump.
598 // The purpose of this transformation is to convert up from legacy
599 // formats, which may still be used in the older dump that is used
600 // for pre-fetching. Applying the transformation again should not
601 // interfere with content that is already in the correct form.
602 $text = $this->exportTransform( $text, $model, $format );
603 }
604 }
605
606 if ( $text === false ) {
607 // Fallback to asking the database
608 $tryIsPrefetch = false;
609 if ( $this->spawn ) {
610 $text = $this->getTextSpawned( $id );
611 } else {
612 $text = $this->getTextDb( $id );
613 }
614
615 if ( $text !== false && $model !== false ) {
616 // Apply export transformation to text coming from the database.
617 // Prefetched text should already have transformations applied.
618 $text = $this->exportTransform( $text, $model, $format );
619 }
620
621 // No more checks for texts from DB for now.
622 // If we received something that is not false,
623 // We treat it as good text, regardless of whether it actually is or is not
624 if ( $text !== false ) {
625 return $text;
626 }
627 }
628
629 if ( $text === false ) {
630 throw new MWException( "Generic error while obtaining text for id " . $id );
631 }
632
633 // We received a good candidate for the text of $id via some method
634
635 // Step 2: Checking for plausibility and return the text if it is
636 // plausible
637 $revID = intval( $this->thisRev );
638 if ( !isset( $this->db ) ) {
639 throw new MWException( "No database available" );
640 }
641
642 if ( $model !== CONTENT_MODEL_WIKITEXT ) {
643 $revLength = strlen( $text );
644 } else {
645 $revLength = $this->db->selectField( 'revision', 'rev_len', [ 'rev_id' => $revID ] );
646 }
647
648 if ( strlen( $text ) == $revLength ) {
649 if ( $tryIsPrefetch ) {
650 $this->prefetchCount++;
651 }
652
653 return $text;
654 }
655
656 $text = false;
657 throw new MWException( "Received text is unplausible for id " . $id );
658 } catch ( Exception $e ) {
659 $msg = "getting/checking text " . $id . " failed (" . $e->getMessage() . ")";
660 if ( $failures + 1 < $this->maxFailures ) {
661 $msg .= " (Will retry " . ( $this->maxFailures - $failures - 1 ) . " more times)";
662 }
663 $this->progress( $msg );
664 }
665
666 // Something went wrong; we did not a text that was plausible :(
667 $failures++;
668
669 // A failure in a prefetch hit does not warrant resetting db connection etc.
670 if ( !$tryIsPrefetch ) {
671 // After backing off for some time, we try to reboot the whole process as
672 // much as possible to not carry over failures from one part to the other
673 // parts
674 sleep( $this->failureTimeout );
675 try {
676 $this->rotateDb();
677 if ( $this->spawn ) {
678 $this->closeSpawn();
679 $this->openSpawn();
680 }
681 } catch ( Exception $e ) {
682 $this->progress( "Rebooting getText infrastructure failed (" . $e->getMessage() . ")" .
683 " Trying to continue anyways" );
684 }
685 }
686 }
687
688 // Retirieving a good text for $id failed (at least) maxFailures times.
689 // We abort for this $id.
690
691 // Restoring the consecutive failures, and maybe aborting, if the dump
692 // is too broken.
693 $consecutiveFailedTextRetrievals = $oldConsecutiveFailedTextRetrievals + 1;
694 if ( $consecutiveFailedTextRetrievals > $this->maxConsecutiveFailedTextRetrievals ) {
695 throw new MWException( "Graceful storage failure" );
696 }
697
698 return "";
699 }
700
707 private function getTextDb( $id ) {
708 global $wgContLang;
709 if ( !isset( $this->db ) ) {
710 throw new MWException( __METHOD__ . "No database available" );
711 }
712 $row = $this->db->selectRow( 'text',
713 [ 'old_text', 'old_flags' ],
714 [ 'old_id' => $id ],
715 __METHOD__ );
716 $text = Revision::getRevisionText( $row );
717 if ( $text === false ) {
718 return false;
719 }
720 $stripped = str_replace( "\r", "", $text );
721 $normalized = $wgContLang->normalize( $stripped );
722
723 return $normalized;
724 }
725
726 private function getTextSpawned( $id ) {
727 MediaWiki\suppressWarnings();
728 if ( !$this->spawnProc ) {
729 // First time?
730 $this->openSpawn();
731 }
732 $text = $this->getTextSpawnedOnce( $id );
733 MediaWiki\restoreWarnings();
734
735 return $text;
736 }
737
738 function openSpawn() {
739 global $IP;
740
741 if ( file_exists( "$IP/../multiversion/MWScript.php" ) ) {
742 $cmd = implode( " ",
743 array_map( 'wfEscapeShellArg',
744 [
745 $this->php,
746 "$IP/../multiversion/MWScript.php",
747 "fetchText.php",
748 '--wiki', wfWikiID() ] ) );
749 } else {
750 $cmd = implode( " ",
751 array_map( 'wfEscapeShellArg',
752 [
753 $this->php,
754 "$IP/maintenance/fetchText.php",
755 '--wiki', wfWikiID() ] ) );
756 }
757 $spec = [
758 0 => [ "pipe", "r" ],
759 1 => [ "pipe", "w" ],
760 2 => [ "file", "/dev/null", "a" ] ];
761 $pipes = [];
762
763 $this->progress( "Spawning database subprocess: $cmd" );
764 $this->spawnProc = proc_open( $cmd, $spec, $pipes );
765 if ( !$this->spawnProc ) {
766 $this->progress( "Subprocess spawn failed." );
767
768 return false;
769 }
770 list(
771 $this->spawnWrite, // -> stdin
772 $this->spawnRead, // <- stdout
773 ) = $pipes;
774
775 return true;
776 }
777
778 private function closeSpawn() {
779 MediaWiki\suppressWarnings();
780 if ( $this->spawnRead ) {
781 fclose( $this->spawnRead );
782 }
783 $this->spawnRead = false;
784 if ( $this->spawnWrite ) {
785 fclose( $this->spawnWrite );
786 }
787 $this->spawnWrite = false;
788 if ( $this->spawnErr ) {
789 fclose( $this->spawnErr );
790 }
791 $this->spawnErr = false;
792 if ( $this->spawnProc ) {
793 pclose( $this->spawnProc );
794 }
795 $this->spawnProc = false;
796 MediaWiki\restoreWarnings();
797 }
798
799 private function getTextSpawnedOnce( $id ) {
800 global $wgContLang;
801
802 $ok = fwrite( $this->spawnWrite, "$id\n" );
803 // $this->progress( ">> $id" );
804 if ( !$ok ) {
805 return false;
806 }
807
808 $ok = fflush( $this->spawnWrite );
809 // $this->progress( ">> [flush]" );
810 if ( !$ok ) {
811 return false;
812 }
813
814 // check that the text id they are sending is the one we asked for
815 // this avoids out of sync revision text errors we have encountered in the past
816 $newId = fgets( $this->spawnRead );
817 if ( $newId === false ) {
818 return false;
819 }
820 if ( $id != intval( $newId ) ) {
821 return false;
822 }
823
824 $len = fgets( $this->spawnRead );
825 // $this->progress( "<< " . trim( $len ) );
826 if ( $len === false ) {
827 return false;
828 }
829
830 $nbytes = intval( $len );
831 // actual error, not zero-length text
832 if ( $nbytes < 0 ) {
833 return false;
834 }
835
836 $text = "";
837
838 // Subprocess may not send everything at once, we have to loop.
839 while ( $nbytes > strlen( $text ) ) {
840 $buffer = fread( $this->spawnRead, $nbytes - strlen( $text ) );
841 if ( $buffer === false ) {
842 break;
843 }
844 $text .= $buffer;
845 }
846
847 $gotbytes = strlen( $text );
848 if ( $gotbytes != $nbytes ) {
849 $this->progress( "Expected $nbytes bytes from database subprocess, got $gotbytes " );
850
851 return false;
852 }
853
854 // Do normalization in the dump thread...
855 $stripped = str_replace( "\r", "", $text );
856 $normalized = $wgContLang->normalize( $stripped );
857
858 return $normalized;
859 }
860
861 function startElement( $parser, $name, $attribs ) {
862 $this->checkpointJustWritten = false;
863
864 $this->clearOpenElement( null );
865 $this->lastName = $name;
866
867 if ( $name == 'revision' ) {
868 $this->state = $name;
869 $this->egress->writeOpenPage( null, $this->buffer );
870 $this->buffer = "";
871 } elseif ( $name == 'page' ) {
872 $this->state = $name;
873 if ( $this->atStart ) {
874 $this->egress->writeOpenStream( $this->buffer );
875 $this->buffer = "";
876 $this->atStart = false;
877 }
878 }
879
880 if ( $name == "text" && isset( $attribs['id'] ) ) {
881 $id = $attribs['id'];
882 $model = trim( $this->thisRevModel );
883 $format = trim( $this->thisRevFormat );
884
885 $model = $model === '' ? null : $model;
886 $format = $format === '' ? null : $format;
887
888 $text = $this->getText( $id, $model, $format );
889 $this->openElement = [ $name, [ 'xml:space' => 'preserve' ] ];
890 if ( strlen( $text ) > 0 ) {
891 $this->characterData( $parser, $text );
892 }
893 } else {
894 $this->openElement = [ $name, $attribs ];
895 }
896 }
897
898 function endElement( $parser, $name ) {
899 $this->checkpointJustWritten = false;
900
901 if ( $this->openElement ) {
902 $this->clearOpenElement( "" );
903 } else {
904 $this->buffer .= "</$name>";
905 }
906
907 if ( $name == 'revision' ) {
908 $this->egress->writeRevision( null, $this->buffer );
909 $this->buffer = "";
910 $this->thisRev = "";
911 $this->thisRevModel = null;
912 $this->thisRevFormat = null;
913 } elseif ( $name == 'page' ) {
914 if ( !$this->firstPageWritten ) {
915 $this->firstPageWritten = trim( $this->thisPage );
916 }
917 $this->lastPageWritten = trim( $this->thisPage );
918 if ( $this->timeExceeded ) {
919 $this->egress->writeClosePage( $this->buffer );
920 // nasty hack, we can't just write the chardata after the
921 // page tag, it will include leading blanks from the next line
922 $this->egress->sink->write( "\n" );
923
924 $this->buffer = $this->xmlwriterobj->closeStream();
925 $this->egress->writeCloseStream( $this->buffer );
926
927 $this->buffer = "";
928 $this->thisPage = "";
929 // this could be more than one file if we had more than one output arg
930
931 $filenameList = (array)$this->egress->getFilenames();
932 $newFilenames = [];
933 $firstPageID = str_pad( $this->firstPageWritten, 9, "0", STR_PAD_LEFT );
934 $lastPageID = str_pad( $this->lastPageWritten, 9, "0", STR_PAD_LEFT );
935 $filenamesCount = count( $filenameList );
936 for ( $i = 0; $i < $filenamesCount; $i++ ) {
937 $checkpointNameFilledIn = sprintf( $this->checkpointFiles[$i], $firstPageID, $lastPageID );
938 $fileinfo = pathinfo( $filenameList[$i] );
939 $newFilenames[] = $fileinfo['dirname'] . '/' . $checkpointNameFilledIn;
940 }
941 $this->egress->closeRenameAndReopen( $newFilenames );
942 $this->buffer = $this->xmlwriterobj->openStream();
943 $this->timeExceeded = false;
944 $this->timeOfCheckpoint = $this->lastTime;
945 $this->firstPageWritten = false;
946 $this->checkpointJustWritten = true;
947 } else {
948 $this->egress->writeClosePage( $this->buffer );
949 $this->buffer = "";
950 $this->thisPage = "";
951 }
952 } elseif ( $name == 'mediawiki' ) {
953 $this->egress->writeCloseStream( $this->buffer );
954 $this->buffer = "";
955 }
956 }
957
958 function characterData( $parser, $data ) {
959 $this->clearOpenElement( null );
960 if ( $this->lastName == "id" ) {
961 if ( $this->state == "revision" ) {
962 $this->thisRev .= $data;
963 } elseif ( $this->state == "page" ) {
964 $this->thisPage .= $data;
965 }
966 } elseif ( $this->lastName == "model" ) {
967 $this->thisRevModel .= $data;
968 } elseif ( $this->lastName == "format" ) {
969 $this->thisRevFormat .= $data;
970 }
971
972 // have to skip the newline left over from closepagetag line of
973 // end of checkpoint files. nasty hack!!
974 if ( $this->checkpointJustWritten ) {
975 if ( $data[0] == "\n" ) {
976 $data = substr( $data, 1 );
977 }
978 $this->checkpointJustWritten = false;
979 }
980 $this->buffer .= htmlspecialchars( $data );
981 }
982
983 function clearOpenElement( $style ) {
984 if ( $this->openElement ) {
985 $this->buffer .= Xml::element( $this->openElement[0], $this->openElement[1], $style );
986 $this->openElement = false;
987 }
988 }
989}
990
991$maintClass = 'TextPassDumper';
992require_once RUN_MAINTENANCE_IF_MAIN;
$wgContentHandlerUseDB
Set to false to disable use of the database fields introduced by the ContentHandler facility.
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
wfGetLBFactory()
Get the load balancer factory object.
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
wfWikiID()
Get an ASCII string identifying this wiki This is used as a prefix in memcached keys.
if( $line===false) $args
Definition cdb.php:63
progress( $string)
Definition backup.inc:417
IDatabase null $forcedDb
The dependency-injected database to use.
Definition backup.inc:70
report( $final=false)
Definition backup.inc:372
Readahead helper for making large MediaWiki data dumps; reads in a previous XML dump to sequentially ...
static getForModelID( $modelId)
Returns the ContentHandler singleton for the given model ID.
MediaWiki exception.
hasOption( $name)
Checks to see if a particular param exists.
addDescription( $text)
Set the description text.
loadWithArgv( $argv)
Load params and arguments from a given array of command-line arguments.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
static getRevisionText( $row, $prefix='old_', $wiki=false)
Get revision text associated with an old or archive row.
$maxConsecutiveFailedTextRetrievals
showReport()
Overridden to include prefetch ratio if enabled.
bool resource $spawnProc
getTextDb( $id)
May throw a database error if, say, the server dies during query.
endElement( $parser, $name)
clearOpenElement( $style)
bool resource $spawnErr
getText( $id, $model=null, $format=null)
Tries to get the revision text for a revision id.
exportTransform( $text, $model, $format=null)
Applies applicable export transformations to $text.
BaseDump $prefetch
startElement( $parser, $name, $attribs)
processOptions()
Processes arguments and sets $this->$sink accordingly.
bool resource $spawnRead
characterData( $parser, $data)
initProgress( $history=WikiExporter::FULL)
Initialise starting time and maximum revision count.
execute()
Do the actual work.
__construct( $args=null)
dump( $history, $text=WikiExporter::TEXT)
string bool $thisRev
string bool $thisPage
bool XmlDumpWriter $xmlwriterobj
IMaintainableDatabase $db
bool resource $spawnWrite
rotateDb()
Drop the database connection $this->db and try to get a new one.
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition deferred.txt:11
this class mediates it Skin Encapsulates a look and feel for the wiki All of the functions that render HTML and make choices about how to render it are here and are called from various other places when and is meant to be subclassed with other skins that may override some of its functions The User object contains a reference to a and so rather than having a global skin object we just rely on the global User and get the skin with $wgUser and also has some character encoding functions and other locale stuff The current user interface language is instantiated as and the local content language as $wgContLang
Definition design.txt:57
design txt This is a brief overview of the new design More thorough and up to date information is available on the documentation wiki at etc Handles the details of getting and saving to the user table of the and dealing with sessions and cookies OutputPage Encapsulates the entire HTML page that will be sent in response to any server request It is used by calling its functions to add in any and then calling output() to send it all. It could be easily changed to send incrementally if that becomes useful
the array() calling protocol came about after MediaWiki 1.4rc1.
do that in ParserLimitReportFormat instead $parser
Definition hooks.txt:2572
presenting them properly to the user as errors is done by the caller return true use this to change the list i e etc next in line in page history
Definition hooks.txt:1760
either a unescaped string or a HtmlArmor object after in associative array form externallinks including delete and has completed for all link tables whether this was an auto creation default is conds Array Extra conditions for the No matching items in log is displayed if loglist is empty msgKey Array If you want a nice box with a set this to the key of the message First element is the message additional optional elements are parameters for the key that are processed with wfMessage() -> params() ->parseAsBlock() - offset Set to overwrite offset parameter in $wgRequest set to '' to unset offset - wrap String Wrap the message in html(usually something like "&lt;div ...>$1&lt;/div>"). - flags Integer display flags(NO_ACTION_LINK, NO_EXTRA_USER_LINKS) 'LogException':Called before an exception(or PHP error) is logged. This is meant for integration with external error aggregation services
Allows to change the fields on the form that will be generated $name
Definition hooks.txt:302
null means default in associative array with keys and values unescaped Should be merged with default with a value of false meaning to suppress the attribute in associative array with keys and values unescaped noclasses just before the function returns a value If you return an< a > element with HTML attributes $attribs and contents $html will be returned If you return $ret will be returned and may include noclasses after processing & $attribs
Definition hooks.txt:1984
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that probably a stub it is not rendered in wiki pages or galleries in category pages allow injecting custom HTML after the section Any uses of the hook need to handle escaping see BaseTemplate::getToolbox and BaseTemplate::makeListItem for details on the format of individual items inside of this array or by returning and letting standard HTTP rendering take place modifiable or by returning false and taking over the output modifiable modifiable after all normalizations have been except for the $wgMaxImageArea check set to true or false to override the $wgMaxImageArea check result gives extension the possibility to transform it themselves $handler
Definition hooks.txt:901
returning false will NOT prevent logging $e
Definition hooks.txt:2146
$maintClass
$IP
Definition update.php:3
const CONTENT_MODEL_WIKITEXT
Definition Defines.php:236
Advanced database interface for IDatabase handles that include maintenance methods.
require_once RUN_MAINTENANCE_IF_MAIN
$buffer
const DB_REPLICA
Definition defines.php:25