14require_once __DIR__ .
"/../../../maintenance/dumpTextPass.php";
42 $this->tablesUsed[] =
'page';
43 $this->tablesUsed[] =
'revision';
44 $this->tablesUsed[] =
'ip_changes';
45 $this->tablesUsed[] =
'text';
48 "BackupTextPassTestModel" => BackupTextPassTestModelHandler::class,
55 $title = Title::newFromText(
'BackupDumperTestP1', $ns );
56 $page = WikiPage::factory( $title );
58 "BackupDumperTestP1Text1",
"BackupDumperTestP1Summary1" );
59 $this->pageId1 = $page->getId();
62 $title = Title::newFromText(
'BackupDumperTestP2', $ns );
63 $page = WikiPage::factory( $title );
65 "BackupDumperTestP2Text1",
"BackupDumperTestP2Summary1" );
67 "BackupDumperTestP2Text2",
"BackupDumperTestP2Summary2" );
69 "BackupDumperTestP2Text3",
"BackupDumperTestP2Summary3" );
71 "BackupDumperTestP2Text4 some additional Text ",
72 "BackupDumperTestP2Summary4 extra " );
73 $this->pageId2 = $page->getId();
76 $title = Title::newFromText(
'BackupDumperTestP3', $ns );
77 $page = WikiPage::factory( $title );
79 "BackupDumperTestP3Text1",
"BackupDumperTestP2Summary1" );
81 "BackupDumperTestP3Text2",
"BackupDumperTestP2Summary2" );
82 $this->pageId3 = $page->getId();
83 $page->doDeleteArticle(
"Testing ;)" );
90 throw new MWException(
"The default wikitext namespace is the talk namespace. "
91 .
" We can't currently deal with that." );
94 $title = Title::newFromText(
'BackupDumperTestP1',
NS_TALK );
95 $page = WikiPage::factory( $title );
97 "Talk about BackupDumperTestP1 Text1",
98 "Talk BackupDumperTestP1 Summary1",
99 "BackupTextPassTestModel" );
100 $this->pageId4 = $page->getId();
101 }
catch ( Exception
$e ) {
105 $this->exceptionFromAddDBData =
$e;
117 [ $this->pageId2, $this->pageId3, $this->pageId4 ],
118 [ $this->pageId1 + 1, $this->pageId1 + 2, $this->pageId1 + 3 ],
119 "Page ids increasing without holes" );
127 "--output=file:" . $nameFull ] );
128 $dumper->reporting =
false;
129 $dumper->setDB( $this->db );
132 $dumper->dump( WikiExporter::FULL, WikiExporter::TEXT );
139 $this->
assertRevision( $this->revId1_1,
"BackupDumperTestP1Summary1",
140 $this->textId1_1,
false,
"0bolhl6ol7i6x0e7yq91gxgaan39j87",
141 "BackupDumperTestP1Text1" );
146 $this->
assertRevision( $this->revId2_1,
"BackupDumperTestP2Summary1",
147 $this->textId2_1,
false,
"jprywrymfhysqllua29tj3sc7z39dl2",
148 "BackupDumperTestP2Text1" );
149 $this->
assertRevision( $this->revId2_2,
"BackupDumperTestP2Summary2",
150 $this->textId2_2,
false,
"b7vj5ks32po5m1z1t1br4o7scdwwy95",
151 "BackupDumperTestP2Text2", $this->revId2_1 );
152 $this->
assertRevision( $this->revId2_3,
"BackupDumperTestP2Summary3",
153 $this->textId2_3,
false,
"jfunqmh1ssfb8rs43r19w98k28gg56r",
154 "BackupDumperTestP2Text3", $this->revId2_2 );
155 $this->
assertRevision( $this->revId2_4,
"BackupDumperTestP2Summary4 extra",
156 $this->textId2_4,
false,
"6o1ciaxa6pybnqprmungwofc4lv00wv",
157 "BackupDumperTestP2Text4 some additional Text", $this->revId2_3 );
165 $this->
assertRevision( $this->revId4_1,
"Talk BackupDumperTestP1 Summary1",
166 $this->textId4_1,
false,
"nktofwzd0tl192k3zfepmlzxoax1lpe",
167 "TALK ABOUT BACKUPDUMPERTESTP1 TEXT1",
169 "BackupTextPassTestModel",
184 $prefetchMock = $this->getMockBuilder( BaseDump::class )
185 ->setMethods( [
'prefetch' ] )
186 ->disableOriginalConstructor()
188 $prefetchMock->expects( $this->exactly( 6 ) )
189 ->method(
'prefetch' )
190 ->will( $this->returnValueMap( $prefetchMap ) );
197 "--output=file:" . $nameFull ] );
199 $dumper->prefetch = $prefetchMock;
200 $dumper->reporting =
false;
201 $dumper->setDB( $this->db );
204 $dumper->dump( WikiExporter::FULL, WikiExporter::TEXT );
213 $this->
assertRevision( $this->revId1_1,
"BackupDumperTestP1Summary1",
214 $this->textId1_1,
false,
"0bolhl6ol7i6x0e7yq91gxgaan39j87",
215 "Prefetch_________1Text1" );
220 $this->
assertRevision( $this->revId2_1,
"BackupDumperTestP2Summary1",
221 $this->textId2_1,
false,
"jprywrymfhysqllua29tj3sc7z39dl2",
222 "BackupDumperTestP2Text1" );
223 $this->
assertRevision( $this->revId2_2,
"BackupDumperTestP2Summary2",
224 $this->textId2_2,
false,
"b7vj5ks32po5m1z1t1br4o7scdwwy95",
225 "BackupDumperTestP2Text2", $this->revId2_1 );
228 $this->
assertRevision( $this->revId2_3,
"BackupDumperTestP2Summary3",
229 $this->textId2_3,
false,
"jfunqmh1ssfb8rs43r19w98k28gg56r",
230 "Prefetch_________2Text3", $this->revId2_2 );
231 $this->
assertRevision( $this->revId2_4,
"BackupDumperTestP2Summary4 extra",
232 $this->textId2_4,
false,
"6o1ciaxa6pybnqprmungwofc4lv00wv",
233 "BackupDumperTestP2Text4 some additional Text", $this->revId2_3 );
241 $this->
assertRevision( $this->revId4_1,
"Talk BackupDumperTestP1 Summary1",
242 $this->textId4_1,
false,
"nktofwzd0tl192k3zfepmlzxoax1lpe",
243 "TALK ABOUT BACKUPDUMPERTESTP1 TEXT1",
245 "BackupTextPassTestModel",
264 $stderr = fopen(
'php://output',
'a' );
265 if ( $stderr ===
false ) {
266 $this->fail(
"Could not open stream for stderr" );
274 $checkpointAfter = 0.5;
279 while ( $lastDuration < $minDuration ) {
283 "Creating temporary output directory " );
284 $this->
setUpStub( $nameStub, $iterations );
286 $dumper->loadWithArgv( [
"--stub=file:" . $nameStub,
287 "--output=" . $checkpointFormat .
":" . $nameOutputDir .
"/full",
289 "--buffersize=32768",
290 "--checkpointfile=checkpoint-%s-%s.xml.gz" ] );
291 $dumper->setDB( $this->db );
292 $dumper->maxTimeAllowed = $checkpointAfter;
293 $dumper->stderr = $stderr;
296 $ts_before = microtime(
true );
298 $ts_after = microtime(
true );
299 $lastDuration = $ts_after - $ts_before;
302 if ( $lastDuration < $minDuration ) {
303 $old_iterations = $iterations;
304 if ( $lastDuration > 0.2 ) {
306 $factor = ( $minDuration + 0.5 ) / $lastDuration;
307 if ( ( $factor > 1.1 ) && ( $factor < 100 ) ) {
309 $iterations = (int)( $iterations * $factor );
313 if ( $old_iterations == $iterations ) {
318 $this->assertLessThan( 50000, $iterations,
319 "Emergency stop against infinitely increasing iteration "
320 .
"count ( last duration: $lastDuration )" );
328 $files = scandir( $nameOutputDir );
329 $this->assertTrue( asort( $files ),
"Sorting files in temporary directory" );
332 $checkpointFiles = 0;
336 for ( $i = 0; $i < $iterations; ) {
339 if ( !$fileOpened ) {
340 $this->assertNotEmpty( $files,
"No more existing dump files, "
341 .
"but not yet all pages found" );
342 $fname = array_shift( $files );
344 $this->assertNotEmpty( $files,
"No more existing dump"
345 .
" files, but not yet all pages found" );
346 $fname = array_shift( $files );
348 if ( $checkpointFormat ==
"gzip" ) {
357 switch ( $lookingForPage ) {
361 "BackupDumperTestP1" );
362 $this->
assertRevision( $this->revId1_1 + $i * self::$numOfRevs,
"BackupDumperTestP1Summary1",
363 $this->textId1_1,
false,
"0bolhl6ol7i6x0e7yq91gxgaan39j87",
364 "BackupDumperTestP1Text1" );
373 "BackupDumperTestP2" );
374 $this->
assertRevision( $this->revId2_1 + $i * self::$numOfRevs,
"BackupDumperTestP2Summary1",
375 $this->textId2_1,
false,
"jprywrymfhysqllua29tj3sc7z39dl2",
376 "BackupDumperTestP2Text1" );
377 $this->
assertRevision( $this->revId2_2 + $i * self::$numOfRevs,
"BackupDumperTestP2Summary2",
378 $this->textId2_2,
false,
"b7vj5ks32po5m1z1t1br4o7scdwwy95",
379 "BackupDumperTestP2Text2", $this->revId2_1 + $i * self::$numOfRevs );
380 $this->
assertRevision( $this->revId2_3 + $i * self::$numOfRevs,
"BackupDumperTestP2Summary3",
381 $this->textId2_3,
false,
"jfunqmh1ssfb8rs43r19w98k28gg56r",
382 "BackupDumperTestP2Text3", $this->revId2_2 + $i * self::$numOfRevs );
384 "BackupDumperTestP2Summary4 extra",
385 $this->textId2_4,
false,
"6o1ciaxa6pybnqprmungwofc4lv00wv",
386 "BackupDumperTestP2Text4 some additional Text",
387 $this->revId2_3 + $i * self::$numOfRevs );
396 "Talk:BackupDumperTestP1" );
398 "Talk BackupDumperTestP1 Summary1",
399 $this->textId4_1,
false,
"nktofwzd0tl192k3zfepmlzxoax1lpe",
400 "TALK ABOUT BACKUPDUMPERTESTP1 TEXT1",
402 "BackupTextPassTestModel",
413 $this->fail(
"Bad setting for lookingForPage ($lookingForPage)" );
417 if ( $this->xml->nodeType == XMLReader::END_ELEMENT
418 && $this->xml->name ==
"mediawiki"
426 $this->assertFalse( $fileOpened,
"Currently read file still open?" );
427 $this->assertEmpty( $files,
"Remaining unchecked files" );
430 $this->assertGreaterThan(
433 "expected more than 1 checkpoint to have been created. "
434 .
"Checkpoint interval is $checkpointAfter seconds, maybe your computer is too fast?"
487 $header =
'<mediawiki xmlns="http://www.mediawiki.org/xml/export-0.10/" '
488 .
'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" '
489 .
'xsi:schemaLocation="http://www.mediawiki.org/xml/export-0.10/ '
490 .
'http://www.mediawiki.org/xml/export-0.10.xsd" version="0.10" xml:lang="en">
492 <sitename>wikisvn</sitename>
493 <base>http://localhost/wiki-svn/index.php/Main_Page</base>
494 <generator>MediaWiki 1.21alpha</generator>
495 <case>first-letter</case>
497 <namespace key="-2" case="first-letter">Media</namespace>
498 <namespace key="-1" case="first-letter">Special</namespace>
499 <namespace key="0" case="first-letter" />
500 <namespace key="1" case="first-letter">Talk</namespace>
501 <namespace key="2" case="first-letter">User</namespace>
502 <namespace key="3" case="first-letter">User talk</namespace>
503 <namespace key="4" case="first-letter">Wikisvn</namespace>
504 <namespace key="5" case="first-letter">Wikisvn talk</namespace>
505 <namespace key="6" case="first-letter">File</namespace>
506 <namespace key="7" case="first-letter">File talk</namespace>
507 <namespace key="8" case="first-letter">MediaWiki</namespace>
508 <namespace key="9" case="first-letter">MediaWiki talk</namespace>
509 <namespace key="10" case="first-letter">Template</namespace>
510 <namespace key="11" case="first-letter">Template talk</namespace>
511 <namespace key="12" case="first-letter">Help</namespace>
512 <namespace key="13" case="first-letter">Help talk</namespace>
513 <namespace key="14" case="first-letter">Category</namespace>
514 <namespace key="15" case="first-letter">Category talk</namespace>
518 $tail =
'</mediawiki>
522 $iterations = intval( $iterations );
523 for ( $i = 0; $i < $iterations; $i++ ) {
525 <title>BackupDumperTestP1</title>
529 <id>' . ( $this->revId1_1 + $i * self::$numOfRevs ) .
'</id>
530 <timestamp>2012-04-01T16:46:05Z</timestamp>
534 <comment>BackupDumperTestP1Summary1</comment>
535 <model>wikitext</model>
536 <format>text/x-wiki</format>
537 <text id="' . $this->textId1_1 .
'" bytes="23" />
538 <sha1>0bolhl6ol7i6x0e7yq91gxgaan39j87</sha1>
543 <title>BackupDumperTestP2</title>
547 <id>' . ( $this->revId2_1 + $i * self::$numOfRevs ) .
'</id>
548 <timestamp>2012-04-01T16:46:05Z</timestamp>
552 <comment>BackupDumperTestP2Summary1</comment>
553 <model>wikitext</model>
554 <format>text/x-wiki</format>
555 <text id="' . $this->textId2_1 .
'" bytes="23" />
556 <sha1>jprywrymfhysqllua29tj3sc7z39dl2</sha1>
560 <parentid>' . ( $this->revId2_1 + $i * self::$numOfRevs ) .
'</parentid>
561 <timestamp>2012-04-01T16:46:05Z</timestamp>
565 <comment>BackupDumperTestP2Summary2</comment>
566 <model>wikitext</model>
567 <format>text/x-wiki</format>
568 <text id="' . $this->textId2_2 .
'" bytes="23" />
569 <sha1>b7vj5ks32po5m1z1t1br4o7scdwwy95</sha1>
573 <parentid>' . ( $this->revId2_2 + $i * self::$numOfRevs ) .
'</parentid>
574 <timestamp>2012-04-01T16:46:05Z</timestamp>
578 <comment>BackupDumperTestP2Summary3</comment>
579 <model>wikitext</model>
580 <format>text/x-wiki</format>
581 <text id="' . $this->textId2_3 .
'" bytes="23" />
582 <sha1>jfunqmh1ssfb8rs43r19w98k28gg56r</sha1>
586 <parentid>' . ( $this->revId2_3 + $i * self::$numOfRevs ) .
'</parentid>
587 <timestamp>2012-04-01T16:46:05Z</timestamp>
591 <comment>BackupDumperTestP2Summary4 extra</comment>
592 <model>wikitext</model>
593 <format>text/x-wiki</format>
594 <text id="' . $this->textId2_4 .
'" bytes="44" />
595 <sha1>6o1ciaxa6pybnqprmungwofc4lv00wv</sha1>
602 <title>Talk:BackupDumperTestP1</title>
606 <id>' . ( $this->revId4_1 + $i * self::$numOfRevs ) .
'</id>
607 <timestamp>2012-04-01T16:46:05Z</timestamp>
611 <comment>Talk BackupDumperTestP1 Summary1</comment>
612 <model>BackupTextPassTestModel</model>
613 <format>text/plain</format>
614 <text id="' . $this->textId4_1 .
'" bytes="35" />
615 <sha1>nktofwzd0tl192k3zfepmlzxoax1lpe</sha1>
619 $content .= $page1 . $page2 . $page4;
622 $this->assertEquals( strlen(
$content ), file_put_contents(
632 parent::__construct(
'BackupTextPassTestModel' );
636 return strtoupper( $text );
658 $dumper->loadWithArgv( [
"--buffersize=" . $size ] );
660 $this->assertEquals( $expected, $dumper->getBufferSize(), $msg );
671 [ 512 * 1024, 512 * 1024,
"Setting 512KB is not effective" ],
672 [ 8192, 8192,
"Setting 8KB is not effective" ],
673 [ 4096, 2048,
"Could set buffer size below lower bound" ]
wfRecursiveRemoveDir( $dir)
Remove a directory and all its content.
wfMkdirParents( $dir, $mode=null, $caller=null)
Make directory, and make all parent directories if they don't exist.
if(defined( 'MW_SETUP_CALLBACK')) $fname
Customization point after all loading (constants, functions, classes, DefaultSettings,...
exportTransform( $text, $format=null)
Applies transformations on export (returns the blob unchanged per default).
gunzip( $fname)
gunzips the given file and stores the result in the original file name
assertDumpStart( $fname, $skip_siteinfo=true)
Opens an XML file to analyze and optionally skips past siteinfo.
assertPageEnd()
Asserts that the xml reader is at the page's closing element and skips to the next element.
addRevision(Page $page, $text, $summary, $model=CONTENT_MODEL_WIKITEXT)
Adds a revision to a page, while returning the resuting revision's id.
assertDumpEnd( $name="mediawiki")
Asserts that the xml reader is at the final closing tag of an xml file and closes the reader.
expectETAOutput()
Checks for test output consisting only of lines containing ETA announcements.
checkHasGzip()
Skip the test if 'gzip' is not in $PATH.
assertRevision( $id, $summary, $text_id, $text_bytes, $text_sha1, $text=false, $parentid=false, $model=CONTENT_MODEL_WIKITEXT, $format=CONTENT_FORMAT_WIKITEXT)
Asserts that the xml reader is at a revision and checks its representation before skipping over it.
assertPageStart( $id, $ns, $name)
Asserts that the xml reader is at the start of a page element and skips over the first tags,...
Accessor for internal state of TextPassDumper.
dump( $history, $text=null)
getBufferSize()
Gets the bufferSize.
Tests for TextPassDumper that rely on the database.
setUp()
Default set up function.
setUpStub( $fname=null, $iterations=1)
Creates a stub file that is used for testing the text pass of dumps.
checkpointHelper( $checkpointFormat="file")
Ensures that checkpoint dumps are used and written, by successively increasing the stub size and dump...
testCheckpointPlain()
Broken per T70653.
testCheckpointGzip()
tests for working checkpoint generation in gzip format work.
Tests for TextPassDumper that do not rely on the database.
bufferSizeProvider()
Ensures that setting the buffer size is effective.
testBufferSizeSetting( $expected, $size, $msg)
Ensures that setting the buffer size is effective.
Base content handler implementation for flat text contents.
Represents a title within MediaWiki.
Class representing a MediaWiki article and history.
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
returning false will NOT prevent logging $e