MediaWiki master
checkStorage.php
Go to the documentation of this file.
1<?php
26
27require_once __DIR__ . '/../Maintenance.php';
28
29// ----------------------------------------------------------------------------------
30
38 private const CONCAT_HEADER = 'O:27:"concatenatedgziphistoryblob"';
39
41
43 public $dbStore = null;
44
45 public function __construct() {
46 parent::__construct();
47
48 $this->addOption( 'fix', 'Fix errors if possible' );
49 $this->addArg( 'xml', 'Path to an XML dump', false );
50 }
51
52 public function execute() {
53 $fix = $this->hasOption( 'fix' );
54 $xml = $this->getArg( 'xml', false );
55 $this->check( $fix, $xml );
56 }
57
59 'restore text' => 'Damaged text, need to be restored from a backup',
60 'restore revision' => 'Damaged revision row, need to be restored from a backup',
61 'unfixable' => 'Unexpected errors with no automated fixing method',
62 'fixed' => 'Errors already fixed',
63 'fixable' => 'Errors which would already be fixed if --fix was specified',
64 ];
65
66 public function check( $fix = false, $xml = '' ) {
67 $dbr = $this->getReplicaDB();
68 if ( $fix ) {
69 print "Checking, will fix errors if possible...\n";
70 } else {
71 print "Checking...\n";
72 }
73 $maxRevId = $dbr->newSelectQueryBuilder()
74 ->select( 'MAX(rev_id)' )
75 ->from( 'revision' )
76 ->caller( __METHOD__ )->fetchField();
77 $chunkSize = 1000;
78 $flagStats = [];
79 $objectStats = [];
80 $knownFlags = [ 'external', 'gzip', 'object', 'utf-8' ];
81 $this->errors = [
82 'restore text' => [],
83 'restore revision' => [],
84 'unfixable' => [],
85 'fixed' => [],
86 'fixable' => [],
87 ];
88
89 for ( $chunkStart = 1; $chunkStart < $maxRevId; $chunkStart += $chunkSize ) {
90 $chunkEnd = $chunkStart + $chunkSize - 1;
91 // print "$chunkStart of $maxRevId\n";
92
93 $this->oldIdMap = [];
94 $dbr->ping();
95
96 // Fetch revision rows
97 $res = $dbr->newSelectQueryBuilder()
98 ->select( [ 'slot_revision_id', 'content_address' ] )
99 ->from( 'slots' )
100 ->join( 'content', null, 'content_id = slot_content_id' )
101 ->where( [ "slot_revision_id BETWEEN $chunkStart AND $chunkEnd" ] )
102 ->caller( __METHOD__ )->fetchResultSet();
104 $blobStore = $this->getServiceContainer()->getBlobStore();
105 '@phan-var \MediaWiki\Storage\SqlBlobStore $blobStore';
106 foreach ( $res as $row ) {
107 $textId = $blobStore->getTextIdFromAddress( $row->content_address );
108 if ( $textId ) {
109 if ( !isset( $this->oldIdMap[$textId] ) ) {
110 $this->oldIdMap[ $textId ] = [ $row->slot_revision_id ];
111 } elseif ( !in_array( $row->slot_revision_id, $this->oldIdMap[$textId] ) ) {
112 $this->oldIdMap[ $textId ][] = $row->slot_revision_id;
113 }
114 }
115 }
116
117 if ( !count( $this->oldIdMap ) ) {
118 continue;
119 }
120
121 // Fetch old_flags
122 $missingTextRows = $this->oldIdMap;
123 $externalRevs = [];
124 $objectRevs = [];
125 $res = $dbr->newSelectQueryBuilder()
126 ->select( [ 'old_id', 'old_flags' ] )
127 ->from( 'text' )
128 ->where( [ 'old_id' => array_keys( $this->oldIdMap ) ] )
129 ->caller( __METHOD__ )->fetchResultSet();
130 foreach ( $res as $row ) {
134 $flags = $row->old_flags;
135 $id = $row->old_id;
136
137 // Create flagStats row if it doesn't exist
138 $flagStats += [ $flags => 0 ];
139 // Increment counter
140 $flagStats[$flags]++;
141
142 // Not missing
143 unset( $missingTextRows[$row->old_id] );
144
145 // Check for external or object
146 if ( $flags == '' ) {
147 $flagArray = [];
148 } else {
149 $flagArray = explode( ',', $flags );
150 }
151 if ( in_array( 'external', $flagArray ) ) {
152 $externalRevs[] = $id;
153 } elseif ( in_array( 'object', $flagArray ) ) {
154 $objectRevs[] = $id;
155 }
156
157 // Check for unrecognised flags
158 if ( $flags == '0' ) {
159 // This is a known bug from 2004
160 // It's safe to just erase the old_flags field
161 if ( $fix ) {
162 $this->addError( 'fixed', "Warning: old_flags set to 0", $id );
163 $dbw = $this->getPrimaryDB();
164 $dbw->ping();
165 $dbw->update( 'text', [ 'old_flags' => '' ],
166 [ 'old_id' => $id ], __METHOD__ );
167 echo "Fixed\n";
168 } else {
169 $this->addError( 'fixable', "Warning: old_flags set to 0", $id );
170 }
171 } elseif ( count( array_diff( $flagArray, $knownFlags ) ) ) {
172 $this->addError( 'unfixable', "Error: invalid flags field \"$flags\"", $id );
173 }
174 }
175
176 // Output errors for any missing text rows
177 foreach ( $missingTextRows as $oldId => $revIds ) {
178 $this->addError( 'restore revision', "Error: missing text row", $oldId );
179 }
180
181 // Verify external revisions
182 $externalConcatBlobs = [];
183 $externalNormalBlobs = [];
184 if ( count( $externalRevs ) ) {
185 $res = $dbr->newSelectQueryBuilder()
186 ->select( [ 'old_id', 'old_flags', 'old_text' ] )
187 ->from( 'text' )
188 ->where( [ 'old_id' => $externalRevs ] )
189 ->caller( __METHOD__ )->fetchResultSet();
190 foreach ( $res as $row ) {
191 $urlParts = explode( '://', $row->old_text, 2 );
192 if ( count( $urlParts ) !== 2 || $urlParts[1] == '' ) {
193 $this->addError( 'restore text', "Error: invalid URL \"{$row->old_text}\"", $row->old_id );
194 continue;
195 }
196 [ $proto, ] = $urlParts;
197 if ( $proto != 'DB' ) {
198 $this->addError(
199 'restore text',
200 "Error: invalid external protocol \"$proto\"",
201 $row->old_id );
202 continue;
203 }
204 $path = explode( '/', $row->old_text );
205 $cluster = $path[2];
206 $id = $path[3];
207 if ( isset( $path[4] ) ) {
208 $externalConcatBlobs[$cluster][$id][] = $row->old_id;
209 } else {
210 $externalNormalBlobs[$cluster][$id][] = $row->old_id;
211 }
212 }
213 }
214
215 // Check external concat blobs for the right header
216 $this->checkExternalConcatBlobs( $externalConcatBlobs );
217
218 // Check external normal blobs for existence
219 if ( count( $externalNormalBlobs ) ) {
220 if ( $this->dbStore === null ) {
221 $esFactory = $this->getServiceContainer()->getExternalStoreFactory();
222 $this->dbStore = $esFactory->getStore( 'DB' );
223 }
224 foreach ( $externalConcatBlobs as $cluster => $xBlobIds ) {
225 $blobIds = array_keys( $xBlobIds );
226 $extDb = $this->dbStore->getReplica( $cluster );
227 $blobsTable = $this->dbStore->getTable( $extDb );
228 $res = $extDb->newSelectQueryBuilder()
229 ->select( [ 'blob_id' ] )
230 ->from( $blobsTable )
231 ->where( [ 'blob_id' => $blobIds ] )
232 ->caller( __METHOD__ )->fetchResultSet();
233 foreach ( $res as $row ) {
234 unset( $xBlobIds[$row->blob_id] );
235 }
236 // Print errors for missing blobs rows
237 foreach ( $xBlobIds as $blobId => $oldId ) {
238 $this->addError(
239 'restore text',
240 "Error: missing target $blobId for one-part ES URL",
241 $oldId );
242 }
243 }
244 }
245
246 // Check local objects
247 $dbr->ping();
248 $concatBlobs = [];
249 $curIds = [];
250 if ( count( $objectRevs ) ) {
251 $headerLength = 300;
252 $res = $dbr->newSelectQueryBuilder()
253 ->select( [ 'old_id', 'old_flags', "LEFT(old_text, $headerLength) AS header" ] )
254 ->from( 'text' )
255 ->where( [ 'old_id' => $objectRevs ] )
256 ->caller( __METHOD__ )->fetchResultSet();
257 foreach ( $res as $row ) {
258 $oldId = $row->old_id;
259 $matches = [];
260 if ( !preg_match( '/^O:(\d+):"(\w+)"/', $row->header, $matches ) ) {
261 $this->addError( 'restore text', "Error: invalid object header", $oldId );
262 continue;
263 }
264
265 $className = strtolower( $matches[2] );
266 if ( strlen( $className ) != $matches[1] ) {
267 $this->addError(
268 'restore text',
269 "Error: invalid object header, wrong class name length",
270 $oldId
271 );
272 continue;
273 }
274
275 $objectStats += [ $className => 0 ];
276 $objectStats[$className]++;
277
278 switch ( $className ) {
279 case 'concatenatedgziphistoryblob':
280 // Good
281 break;
282 case 'historyblobstub':
283 case 'historyblobcurstub':
284 if ( strlen( $row->header ) == $headerLength ) {
285 $this->addError( 'unfixable', "Error: overlong stub header", $oldId );
286 break;
287 }
288 $stubObj = unserialize( $row->header );
289 if ( !is_object( $stubObj ) ) {
290 $this->addError( 'restore text', "Error: unable to unserialize stub object", $oldId );
291 break;
292 }
293 if ( $className == 'historyblobstub' ) {
294 $concatBlobs[$stubObj->getLocation()][] = $oldId;
295 } else {
296 $curIds[$stubObj->mCurId][] = $oldId;
297 }
298 break;
299 default:
300 $this->addError( 'unfixable', "Error: unrecognised object class \"$className\"", $oldId );
301 }
302 }
303 }
304
305 // Check local concat blob validity
306 $externalConcatBlobs = [];
307 if ( count( $concatBlobs ) ) {
308 $headerLength = 300;
309 $res = $dbr->newSelectQueryBuilder()
310 ->select( [ 'old_id', 'old_flags', "LEFT(old_text, $headerLength) AS header" ] )
311 ->from( 'text' )
312 ->where( [ 'old_id' => array_keys( $concatBlobs ) ] )
313 ->caller( __METHOD__ )->fetchResultSet();
314 foreach ( $res as $row ) {
315 $flags = explode( ',', $row->old_flags );
316 if ( in_array( 'external', $flags ) ) {
317 // Concat blob is in external storage?
318 if ( in_array( 'object', $flags ) ) {
319 $urlParts = explode( '/', $row->header );
320 if ( $urlParts[0] != 'DB:' ) {
321 $this->addError(
322 'unfixable',
323 "Error: unrecognised external storage type \"{$urlParts[0]}",
324 $row->old_id
325 );
326 } else {
327 $cluster = $urlParts[2];
328 $id = $urlParts[3];
329 if ( !isset( $externalConcatBlobs[$cluster][$id] ) ) {
330 $externalConcatBlobs[$cluster][$id] = [];
331 }
332 $externalConcatBlobs[$cluster][$id] = array_merge(
333 $externalConcatBlobs[$cluster][$id], $concatBlobs[$row->old_id]
334 );
335 }
336 } else {
337 $this->addError(
338 'unfixable',
339 "Error: invalid flags \"{$row->old_flags}\" on concat bulk row {$row->old_id}",
340 $concatBlobs[$row->old_id] );
341 }
342 } elseif ( strcasecmp(
343 substr( $row->header, 0, strlen( self::CONCAT_HEADER ) ),
344 self::CONCAT_HEADER
345 ) ) {
346 $this->addError(
347 'restore text',
348 "Error: Incorrect object header for concat bulk row {$row->old_id}",
349 $concatBlobs[$row->old_id]
350 );
351 }
352
353 unset( $concatBlobs[$row->old_id] );
354 }
355 }
356
357 // Check targets of unresolved stubs
358 $this->checkExternalConcatBlobs( $externalConcatBlobs );
359 // next chunk
360 }
361
362 print "\n\nErrors:\n";
363 foreach ( $this->errors as $name => $errors ) {
364 if ( count( $errors ) ) {
365 $description = $this->errorDescriptions[$name];
366 echo "$description: " . implode( ',', array_keys( $errors ) ) . "\n";
367 }
368 }
369
370 if ( count( $this->errors['restore text'] ) && $fix ) {
371 if ( (string)$xml !== '' ) {
372 $this->restoreText( array_keys( $this->errors['restore text'] ), $xml );
373 } else {
374 echo "Can't fix text, no XML backup specified\n";
375 }
376 }
377
378 print "\nFlag statistics:\n";
379 $total = array_sum( $flagStats );
380 foreach ( $flagStats as $flag => $count ) {
381 printf( "%-30s %10d %5.2f%%\n", $flag, $count, $count / $total * 100 );
382 }
383 print "\nLocal object statistics:\n";
384 $total = array_sum( $objectStats );
385 foreach ( $objectStats as $className => $count ) {
386 printf( "%-30s %10d %5.2f%%\n", $className, $count, $count / $total * 100 );
387 }
388 }
389
390 private function addError( $type, $msg, $ids ) {
391 if ( is_array( $ids ) && count( $ids ) == 1 ) {
392 $ids = reset( $ids );
393 }
394 if ( is_array( $ids ) ) {
395 $revIds = [];
396 foreach ( $ids as $id ) {
397 $revIds = array_unique( array_merge( $revIds, $this->oldIdMap[$id] ) );
398 }
399 print "$msg in text rows " . implode( ', ', $ids ) .
400 ", revisions " . implode( ', ', $revIds ) . "\n";
401 } else {
402 $id = $ids;
403 $revIds = $this->oldIdMap[$id];
404 if ( count( $revIds ) == 1 ) {
405 print "$msg in old_id $id, rev_id {$revIds[0]}\n";
406 } else {
407 print "$msg in old_id $id, revisions " . implode( ', ', $revIds ) . "\n";
408 }
409 }
410 $this->errors[$type] += array_fill_keys( $revIds, true );
411 }
412
413 private function checkExternalConcatBlobs( $externalConcatBlobs ) {
414 if ( !count( $externalConcatBlobs ) ) {
415 return;
416 }
417
418 if ( $this->dbStore === null ) {
419 $esFactory = $this->getServiceContainer()->getExternalStoreFactory();
420 $this->dbStore = $esFactory->getStore( 'DB' );
421 }
422
423 foreach ( $externalConcatBlobs as $cluster => $oldIds ) {
424 $blobIds = array_keys( $oldIds );
425 $extDb = $this->dbStore->getReplica( $cluster );
426 $blobsTable = $this->dbStore->getTable( $extDb );
427 $headerLength = strlen( self::CONCAT_HEADER );
428 $res = $extDb->newSelectQueryBuilder()
429 ->select( [ 'blob_id', "LEFT(blob_text, $headerLength) AS header" ] )
430 ->from( $blobsTable )
431 ->where( [ 'blob_id' => $blobIds ] )
432 ->caller( __METHOD__ )->fetchResultSet();
433 foreach ( $res as $row ) {
434 if ( strcasecmp( $row->header, self::CONCAT_HEADER ) ) {
435 $this->addError(
436 'restore text',
437 "Error: invalid header on target $cluster/{$row->blob_id} of two-part ES URL",
438 $oldIds[$row->blob_id]
439 );
440 }
441 unset( $oldIds[$row->blob_id] );
442 }
443
444 // Print errors for missing blobs rows
445 foreach ( $oldIds as $blobId => $oldIds2 ) {
446 $this->addError(
447 'restore text',
448 "Error: missing target $cluster/$blobId for two-part ES URL",
449 $oldIds2
450 );
451 }
452 }
453 }
454
455 private function restoreText( $revIds, $xml ) {
456 global $wgDBname;
457 $tmpDir = wfTempDir();
458
459 if ( !count( $revIds ) ) {
460 return;
461 }
462
463 print "Restoring text from XML backup...\n";
464
465 $revFileName = "$tmpDir/broken-revlist-$wgDBname";
466 $filteredXmlFileName = "$tmpDir/filtered-$wgDBname.xml";
467
468 // Write revision list
469 if ( !file_put_contents( $revFileName, implode( "\n", $revIds ) ) ) {
470 echo "Error writing revision list, can't restore text\n";
471
472 return;
473 }
474
475 // Run mwdumper
476 echo "Filtering XML dump...\n";
477 $exitStatus = 0;
478 // phpcs:ignore MediaWiki.Usage.ForbiddenFunctions.passthru
479 passthru( 'mwdumper ' .
480 Shell::escape(
481 "--output=file:$filteredXmlFileName",
482 "--filter=revlist:$revFileName",
483 $xml
484 ), $exitStatus
485 );
486
487 if ( $exitStatus ) {
488 echo "mwdumper died with exit status $exitStatus\n";
489
490 return;
491 }
492
493 $file = fopen( $filteredXmlFileName, 'r' );
494 if ( !$file ) {
495 echo "Unable to open filtered XML file\n";
496
497 return;
498 }
499
500 $dbr = $this->getReplicaDB();
501 $dbw = $this->getPrimaryDB();
502 $dbr->ping();
503 $dbw->ping();
504
505 $source = new ImportStreamSource( $file );
506 $user = User::newSystemUser( User::MAINTENANCE_SCRIPT_USER, [ 'steal' => true ] );
507 $importer = $this->getServiceContainer()
508 ->getWikiImporterFactory()
509 ->getWikiImporter( $source, new UltimateAuthority( $user ) );
510 $importer->setRevisionCallback( [ $this, 'importRevision' ] );
511 $importer->setNoticeCallback( static function ( $msg, $params ) {
512 echo wfMessage( $msg, $params )->text() . "\n";
513 } );
514 $importer->doImport();
515 }
516
520 public function importRevision( $revision ) {
521 $id = $revision->getID();
522 $content = $revision->getContent();
523 $id = $id ?: '';
524
525 if ( $content === null ) {
526 echo "Revision $id is broken, we have no content available\n";
527
528 return;
529 }
530
531 $text = $content->serialize();
532 if ( $text === '' ) {
533 // This is what happens if the revision was broken at the time the
534 // dump was made. Unfortunately, it also happens if the revision was
535 // legitimately blank, so there's no way to tell the difference. To
536 // be safe, we'll skip it and leave it broken
537
538 echo "Revision $id is blank in the dump, may have been broken before export\n";
539
540 return;
541 }
542
543 if ( !$id ) {
544 // No ID, can't import
545 echo "No id tag in revision, can't import\n";
546
547 return;
548 }
549
550 // Find text row again
551 $dbr = $this->getReplicaDB();
552 $res = $dbr->newSelectQueryBuilder()
553 ->select( [ 'content_address' ] )
554 ->from( 'slots' )
555 ->join( 'content', null, 'content_id = slot_content_id' )
556 ->where( [ 'slot_revision_id' => $id ] )
557 ->caller( __METHOD__ )->fetchRow();
558
559 $blobStore = $this->getServiceContainer()
560 ->getBlobStoreFactory()
561 ->newSqlBlobStore();
562 $oldId = $blobStore->getTextIdFromAddress( $res->content_address );
563
564 if ( !$oldId ) {
565 echo "Missing revision row for rev_id $id\n";
566 return;
567 }
568
569 // Compress the text
570 $flags = $blobStore->compressData( $text );
571
572 // Update the text row
573 $dbw = $this->getPrimaryDB();
574 $dbw->update( 'text',
575 [ 'old_flags' => $flags, 'old_text' => $text ],
576 [ 'old_id' => $oldId ],
577 __METHOD__, [ 'LIMIT' => 1 ]
578 );
579
580 // Remove it from the unfixed list and add it to the fixed list
581 unset( $this->errors['restore text'][$id] );
582 $this->errors['fixed'][$id] = true;
583 }
584
585}
586
587$maintClass = CheckStorage::class;
588require_once RUN_MAINTENANCE_IF_MAIN;
wfTempDir()
Tries to get the system directory for temporary files.
wfMessage( $key,... $params)
This is the function for getting translated interface messages.
$maintClass
Maintenance script to do various checks on external storage.
importRevision( $revision)
check( $fix=false, $xml='')
__construct()
Default constructor.
execute()
Do the actual work.
ExternalStoreDB $dbStore
External storage in a SQL database.
Imports a XML dump from a file (either from file upload, files on disk, or HTTP)
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
addArg( $arg, $description, $required=true, $multi=false)
Add some args that are needed.
hasOption( $name)
Checks to see if a particular option was set.
getServiceContainer()
Returns the main service container.
getArg( $argId=0, $default=null)
Get an argument.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
Represents an authority that has all permissions.
Executes shell commands.
Definition Shell.php:46
$wgDBname
Config variable stub for the DBname setting, for use by phpdoc and IDEs.
$source