MediaWiki master
checkStorage.php
Go to the documentation of this file.
1<?php
26
27require_once __DIR__ . '/../Maintenance.php';
28
29// ----------------------------------------------------------------------------------
30
38 private const CONCAT_HEADER = 'O:27:"concatenatedgziphistoryblob"';
39
40 public array $oldIdMap;
41 public array $errors;
42
44 public $dbStore = null;
45
46 public function __construct() {
47 parent::__construct();
48
49 $this->addOption( 'fix', 'Fix errors if possible' );
50 $this->addArg( 'xml', 'Path to an XML dump', false );
51 }
52
53 public function execute() {
54 $fix = $this->hasOption( 'fix' );
55 $xml = $this->getArg( 'xml', false );
56 $this->check( $fix, $xml );
57 }
58
60 'restore text' => 'Damaged text, need to be restored from a backup',
61 'restore revision' => 'Damaged revision row, need to be restored from a backup',
62 'unfixable' => 'Unexpected errors with no automated fixing method',
63 'fixed' => 'Errors already fixed',
64 'fixable' => 'Errors which would already be fixed if --fix was specified',
65 ];
66
67 public function check( $fix = false, $xml = '' ) {
68 $dbr = $this->getReplicaDB();
69 if ( $fix ) {
70 print "Checking, will fix errors if possible...\n";
71 } else {
72 print "Checking...\n";
73 }
74 $maxRevId = $dbr->newSelectQueryBuilder()
75 ->select( 'MAX(rev_id)' )
76 ->from( 'revision' )
77 ->caller( __METHOD__ )->fetchField();
78 $chunkSize = 1000;
79 $flagStats = [];
80 $objectStats = [];
81 $knownFlags = [ 'external', 'gzip', 'object', 'utf-8' ];
82 $this->errors = [
83 'restore text' => [],
84 'restore revision' => [],
85 'unfixable' => [],
86 'fixed' => [],
87 'fixable' => [],
88 ];
89
90 for ( $chunkStart = 1; $chunkStart < $maxRevId; $chunkStart += $chunkSize ) {
91 $chunkEnd = $chunkStart + $chunkSize - 1;
92 // print "$chunkStart of $maxRevId\n";
93
94 $this->oldIdMap = [];
95 $dbr->ping();
96
97 // Fetch revision rows
98 $res = $dbr->newSelectQueryBuilder()
99 ->select( [ 'slot_revision_id', 'content_address' ] )
100 ->from( 'slots' )
101 ->join( 'content', null, 'content_id = slot_content_id' )
102 ->where( [
103 $dbr->expr( 'slot_revision_id', '>=', $chunkStart ),
104 $dbr->expr( 'slot_revision_id', '<=', $chunkEnd ),
105 ] )
106 ->caller( __METHOD__ )->fetchResultSet();
108 $blobStore = $this->getServiceContainer()->getBlobStore();
109 '@phan-var \MediaWiki\Storage\SqlBlobStore $blobStore';
110 foreach ( $res as $row ) {
111 $textId = $blobStore->getTextIdFromAddress( $row->content_address );
112 if ( $textId ) {
113 if ( !isset( $this->oldIdMap[$textId] ) ) {
114 $this->oldIdMap[ $textId ] = [ $row->slot_revision_id ];
115 } elseif ( !in_array( $row->slot_revision_id, $this->oldIdMap[$textId] ) ) {
116 $this->oldIdMap[ $textId ][] = $row->slot_revision_id;
117 }
118 }
119 }
120
121 if ( !count( $this->oldIdMap ) ) {
122 continue;
123 }
124
125 // Fetch old_flags
126 $missingTextRows = $this->oldIdMap;
127 $externalRevs = [];
128 $objectRevs = [];
129 $res = $dbr->newSelectQueryBuilder()
130 ->select( [ 'old_id', 'old_flags' ] )
131 ->from( 'text' )
132 ->where( [ 'old_id' => array_keys( $this->oldIdMap ) ] )
133 ->caller( __METHOD__ )->fetchResultSet();
134 foreach ( $res as $row ) {
138 $flags = $row->old_flags;
139 $id = $row->old_id;
140
141 // Create flagStats row if it doesn't exist
142 $flagStats += [ $flags => 0 ];
143 // Increment counter
144 $flagStats[$flags]++;
145
146 // Not missing
147 unset( $missingTextRows[$row->old_id] );
148
149 // Check for external or object
150 if ( $flags == '' ) {
151 $flagArray = [];
152 } else {
153 $flagArray = explode( ',', $flags );
154 }
155 if ( in_array( 'external', $flagArray ) ) {
156 $externalRevs[] = $id;
157 } elseif ( in_array( 'object', $flagArray ) ) {
158 $objectRevs[] = $id;
159 }
160
161 // Check for unrecognised flags
162 if ( $flags == '0' ) {
163 // This is a known bug from 2004
164 // It's safe to just erase the old_flags field
165 if ( $fix ) {
166 $this->addError( 'fixed', "Warning: old_flags set to 0", $id );
167 $dbw = $this->getPrimaryDB();
168 $dbw->ping();
169 $dbw->newUpdateQueryBuilder()
170 ->update( 'text' )
171 ->set( [ 'old_flags' => '' ] )
172 ->where( [ 'old_id' => $id ] )
173 ->caller( __METHOD__ )
174 ->execute();
175 echo "Fixed\n";
176 } else {
177 $this->addError( 'fixable', "Warning: old_flags set to 0", $id );
178 }
179 } elseif ( count( array_diff( $flagArray, $knownFlags ) ) ) {
180 $this->addError( 'unfixable', "Error: invalid flags field \"$flags\"", $id );
181 }
182 }
183
184 // Output errors for any missing text rows
185 foreach ( $missingTextRows as $oldId => $revIds ) {
186 $this->addError( 'restore revision', "Error: missing text row", $oldId );
187 }
188
189 // Verify external revisions
190 $externalConcatBlobs = [];
191 $externalNormalBlobs = [];
192 if ( count( $externalRevs ) ) {
193 $res = $dbr->newSelectQueryBuilder()
194 ->select( [ 'old_id', 'old_flags', 'old_text' ] )
195 ->from( 'text' )
196 ->where( [ 'old_id' => $externalRevs ] )
197 ->caller( __METHOD__ )->fetchResultSet();
198 foreach ( $res as $row ) {
199 $urlParts = explode( '://', $row->old_text, 2 );
200 if ( count( $urlParts ) !== 2 || $urlParts[1] == '' ) {
201 $this->addError( 'restore text', "Error: invalid URL \"{$row->old_text}\"", $row->old_id );
202 continue;
203 }
204 [ $proto, ] = $urlParts;
205 if ( $proto != 'DB' ) {
206 $this->addError(
207 'restore text',
208 "Error: invalid external protocol \"$proto\"",
209 $row->old_id );
210 continue;
211 }
212 $path = explode( '/', $row->old_text );
213 $cluster = $path[2];
214 $id = $path[3];
215 if ( isset( $path[4] ) ) {
216 $externalConcatBlobs[$cluster][$id][] = $row->old_id;
217 } else {
218 $externalNormalBlobs[$cluster][$id][] = $row->old_id;
219 }
220 }
221 }
222
223 // Check external concat blobs for the right header
224 $this->checkExternalConcatBlobs( $externalConcatBlobs );
225
226 // Check external normal blobs for existence
227 if ( count( $externalNormalBlobs ) ) {
228 if ( $this->dbStore === null ) {
229 $esFactory = $this->getServiceContainer()->getExternalStoreFactory();
230 $this->dbStore = $esFactory->getStore( 'DB' );
231 }
232 foreach ( $externalConcatBlobs as $cluster => $xBlobIds ) {
233 $blobIds = array_keys( $xBlobIds );
234 $extDb = $this->dbStore->getReplica( $cluster );
235 $blobsTable = $this->dbStore->getTable( $extDb );
236 $res = $extDb->newSelectQueryBuilder()
237 ->select( [ 'blob_id' ] )
238 ->from( $blobsTable )
239 ->where( [ 'blob_id' => $blobIds ] )
240 ->caller( __METHOD__ )->fetchResultSet();
241 foreach ( $res as $row ) {
242 unset( $xBlobIds[$row->blob_id] );
243 }
244 // Print errors for missing blobs rows
245 foreach ( $xBlobIds as $blobId => $oldId ) {
246 $this->addError(
247 'restore text',
248 "Error: missing target $blobId for one-part ES URL",
249 $oldId );
250 }
251 }
252 }
253
254 // Check local objects
255 $dbr->ping();
256 $concatBlobs = [];
257 $curIds = [];
258 if ( count( $objectRevs ) ) {
259 $headerLength = 300;
260 $res = $dbr->newSelectQueryBuilder()
261 ->select( [ 'old_id', 'old_flags', "LEFT(old_text, $headerLength) AS header" ] )
262 ->from( 'text' )
263 ->where( [ 'old_id' => $objectRevs ] )
264 ->caller( __METHOD__ )->fetchResultSet();
265 foreach ( $res as $row ) {
266 $oldId = $row->old_id;
267 $matches = [];
268 if ( !preg_match( '/^O:(\d+):"(\w+)"/', $row->header, $matches ) ) {
269 $this->addError( 'restore text', "Error: invalid object header", $oldId );
270 continue;
271 }
272
273 $className = strtolower( $matches[2] );
274 if ( strlen( $className ) != $matches[1] ) {
275 $this->addError(
276 'restore text',
277 "Error: invalid object header, wrong class name length",
278 $oldId
279 );
280 continue;
281 }
282
283 $objectStats += [ $className => 0 ];
284 $objectStats[$className]++;
285
286 switch ( $className ) {
287 case 'concatenatedgziphistoryblob':
288 // Good
289 break;
290 case 'historyblobstub':
291 case 'historyblobcurstub':
292 if ( strlen( $row->header ) == $headerLength ) {
293 $this->addError( 'unfixable', "Error: overlong stub header", $oldId );
294 break;
295 }
296 $stubObj = unserialize( $row->header );
297 if ( !is_object( $stubObj ) ) {
298 $this->addError( 'restore text', "Error: unable to unserialize stub object", $oldId );
299 break;
300 }
301 if ( $className == 'historyblobstub' ) {
302 $concatBlobs[$stubObj->getLocation()][] = $oldId;
303 } else {
304 $curIds[$stubObj->mCurId][] = $oldId;
305 }
306 break;
307 default:
308 $this->addError( 'unfixable', "Error: unrecognised object class \"$className\"", $oldId );
309 }
310 }
311 }
312
313 // Check local concat blob validity
314 $externalConcatBlobs = [];
315 if ( count( $concatBlobs ) ) {
316 $headerLength = 300;
317 $res = $dbr->newSelectQueryBuilder()
318 ->select( [ 'old_id', 'old_flags', "LEFT(old_text, $headerLength) AS header" ] )
319 ->from( 'text' )
320 ->where( [ 'old_id' => array_keys( $concatBlobs ) ] )
321 ->caller( __METHOD__ )->fetchResultSet();
322 foreach ( $res as $row ) {
323 $flags = explode( ',', $row->old_flags );
324 if ( in_array( 'external', $flags ) ) {
325 // Concat blob is in external storage?
326 if ( in_array( 'object', $flags ) ) {
327 $urlParts = explode( '/', $row->header );
328 if ( $urlParts[0] != 'DB:' ) {
329 $this->addError(
330 'unfixable',
331 "Error: unrecognised external storage type \"{$urlParts[0]}",
332 $row->old_id
333 );
334 } else {
335 $cluster = $urlParts[2];
336 $id = $urlParts[3];
337 if ( !isset( $externalConcatBlobs[$cluster][$id] ) ) {
338 $externalConcatBlobs[$cluster][$id] = [];
339 }
340 $externalConcatBlobs[$cluster][$id] = array_merge(
341 $externalConcatBlobs[$cluster][$id], $concatBlobs[$row->old_id]
342 );
343 }
344 } else {
345 $this->addError(
346 'unfixable',
347 "Error: invalid flags \"{$row->old_flags}\" on concat bulk row {$row->old_id}",
348 $concatBlobs[$row->old_id] );
349 }
350 } elseif ( strcasecmp(
351 substr( $row->header, 0, strlen( self::CONCAT_HEADER ) ),
352 self::CONCAT_HEADER
353 ) ) {
354 $this->addError(
355 'restore text',
356 "Error: Incorrect object header for concat bulk row {$row->old_id}",
357 $concatBlobs[$row->old_id]
358 );
359 }
360
361 unset( $concatBlobs[$row->old_id] );
362 }
363 }
364
365 // Check targets of unresolved stubs
366 $this->checkExternalConcatBlobs( $externalConcatBlobs );
367 // next chunk
368 }
369
370 print "\n\nErrors:\n";
371 foreach ( $this->errors as $name => $errors ) {
372 if ( count( $errors ) ) {
373 $description = $this->errorDescriptions[$name];
374 echo "$description: " . implode( ',', array_keys( $errors ) ) . "\n";
375 }
376 }
377
378 if ( count( $this->errors['restore text'] ) && $fix ) {
379 if ( (string)$xml !== '' ) {
380 $this->restoreText( array_keys( $this->errors['restore text'] ), $xml );
381 } else {
382 echo "Can't fix text, no XML backup specified\n";
383 }
384 }
385
386 print "\nFlag statistics:\n";
387 $total = array_sum( $flagStats );
388 foreach ( $flagStats as $flag => $count ) {
389 printf( "%-30s %10d %5.2f%%\n", $flag, $count, $count / $total * 100 );
390 }
391 print "\nLocal object statistics:\n";
392 $total = array_sum( $objectStats );
393 foreach ( $objectStats as $className => $count ) {
394 printf( "%-30s %10d %5.2f%%\n", $className, $count, $count / $total * 100 );
395 }
396 }
397
398 private function addError( $type, $msg, $ids ) {
399 if ( is_array( $ids ) && count( $ids ) == 1 ) {
400 $ids = reset( $ids );
401 }
402 if ( is_array( $ids ) ) {
403 $revIds = [];
404 foreach ( $ids as $id ) {
405 $revIds = array_unique( array_merge( $revIds, $this->oldIdMap[$id] ) );
406 }
407 print "$msg in text rows " . implode( ', ', $ids ) .
408 ", revisions " . implode( ', ', $revIds ) . "\n";
409 } else {
410 $id = $ids;
411 $revIds = $this->oldIdMap[$id];
412 if ( count( $revIds ) == 1 ) {
413 print "$msg in old_id $id, rev_id {$revIds[0]}\n";
414 } else {
415 print "$msg in old_id $id, revisions " . implode( ', ', $revIds ) . "\n";
416 }
417 }
418 $this->errors[$type] += array_fill_keys( $revIds, true );
419 }
420
421 private function checkExternalConcatBlobs( $externalConcatBlobs ) {
422 if ( !count( $externalConcatBlobs ) ) {
423 return;
424 }
425
426 if ( $this->dbStore === null ) {
427 $esFactory = $this->getServiceContainer()->getExternalStoreFactory();
428 $this->dbStore = $esFactory->getStore( 'DB' );
429 }
430
431 foreach ( $externalConcatBlobs as $cluster => $oldIds ) {
432 $blobIds = array_keys( $oldIds );
433 $extDb = $this->dbStore->getReplica( $cluster );
434 $blobsTable = $this->dbStore->getTable( $extDb );
435 $headerLength = strlen( self::CONCAT_HEADER );
436 $res = $extDb->newSelectQueryBuilder()
437 ->select( [ 'blob_id', "LEFT(blob_text, $headerLength) AS header" ] )
438 ->from( $blobsTable )
439 ->where( [ 'blob_id' => $blobIds ] )
440 ->caller( __METHOD__ )->fetchResultSet();
441 foreach ( $res as $row ) {
442 if ( strcasecmp( $row->header, self::CONCAT_HEADER ) ) {
443 $this->addError(
444 'restore text',
445 "Error: invalid header on target $cluster/{$row->blob_id} of two-part ES URL",
446 $oldIds[$row->blob_id]
447 );
448 }
449 unset( $oldIds[$row->blob_id] );
450 }
451
452 // Print errors for missing blobs rows
453 foreach ( $oldIds as $blobId => $oldIds2 ) {
454 $this->addError(
455 'restore text',
456 "Error: missing target $cluster/$blobId for two-part ES URL",
457 $oldIds2
458 );
459 }
460 }
461 }
462
463 private function restoreText( $revIds, $xml ) {
464 global $wgDBname;
465 $tmpDir = wfTempDir();
466
467 if ( !count( $revIds ) ) {
468 return;
469 }
470
471 print "Restoring text from XML backup...\n";
472
473 $revFileName = "$tmpDir/broken-revlist-$wgDBname";
474 $filteredXmlFileName = "$tmpDir/filtered-$wgDBname.xml";
475
476 // Write revision list
477 if ( !file_put_contents( $revFileName, implode( "\n", $revIds ) ) ) {
478 echo "Error writing revision list, can't restore text\n";
479
480 return;
481 }
482
483 // Run mwdumper
484 echo "Filtering XML dump...\n";
485 $exitStatus = 0;
486 // phpcs:ignore MediaWiki.Usage.ForbiddenFunctions.passthru
487 passthru( 'mwdumper ' .
488 Shell::escape(
489 "--output=file:$filteredXmlFileName",
490 "--filter=revlist:$revFileName",
491 $xml
492 ), $exitStatus
493 );
494
495 if ( $exitStatus ) {
496 echo "mwdumper died with exit status $exitStatus\n";
497
498 return;
499 }
500
501 $file = fopen( $filteredXmlFileName, 'r' );
502 if ( !$file ) {
503 echo "Unable to open filtered XML file\n";
504
505 return;
506 }
507
508 $dbr = $this->getReplicaDB();
509 $dbw = $this->getPrimaryDB();
510 $dbr->ping();
511 $dbw->ping();
512
513 $source = new ImportStreamSource( $file );
514 $user = User::newSystemUser( User::MAINTENANCE_SCRIPT_USER, [ 'steal' => true ] );
515 $importer = $this->getServiceContainer()
516 ->getWikiImporterFactory()
517 ->getWikiImporter( $source, new UltimateAuthority( $user ) );
518 $importer->setRevisionCallback( [ $this, 'importRevision' ] );
519 $importer->setNoticeCallback( static function ( $msg, $params ) {
520 echo wfMessage( $msg, $params )->text() . "\n";
521 } );
522 $importer->doImport();
523 }
524
528 public function importRevision( $revision ) {
529 $id = $revision->getID();
530 $content = $revision->getContent();
531 $id = $id ?: '';
532
533 if ( $content === null ) {
534 echo "Revision $id is broken, we have no content available\n";
535
536 return;
537 }
538
539 $text = $content->serialize();
540 if ( $text === '' ) {
541 // This is what happens if the revision was broken at the time the
542 // dump was made. Unfortunately, it also happens if the revision was
543 // legitimately blank, so there's no way to tell the difference. To
544 // be safe, we'll skip it and leave it broken
545
546 echo "Revision $id is blank in the dump, may have been broken before export\n";
547
548 return;
549 }
550
551 if ( !$id ) {
552 // No ID, can't import
553 echo "No id tag in revision, can't import\n";
554
555 return;
556 }
557
558 // Find text row again
559 $dbr = $this->getReplicaDB();
560 $res = $dbr->newSelectQueryBuilder()
561 ->select( [ 'content_address' ] )
562 ->from( 'slots' )
563 ->join( 'content', null, 'content_id = slot_content_id' )
564 ->where( [ 'slot_revision_id' => $id ] )
565 ->caller( __METHOD__ )->fetchRow();
566
567 $blobStore = $this->getServiceContainer()
568 ->getBlobStoreFactory()
569 ->newSqlBlobStore();
570 $oldId = $blobStore->getTextIdFromAddress( $res->content_address );
571
572 if ( !$oldId ) {
573 echo "Missing revision row for rev_id $id\n";
574 return;
575 }
576
577 // Compress the text
578 $flags = $blobStore->compressData( $text );
579
580 // Update the text row
581 $dbw = $this->getPrimaryDB();
582 $dbw->newUpdateQueryBuilder()
583 ->update( 'text' )
584 ->set( [ 'old_flags' => $flags, 'old_text' => $text ] )
585 ->where( [ 'old_id' => $oldId ] )
586 ->caller( __METHOD__ )
587 ->execute();
588
589 // Remove it from the unfixed list and add it to the fixed list
590 unset( $this->errors['restore text'][$id] );
591 $this->errors['fixed'][$id] = true;
592 }
593
594}
595
596$maintClass = CheckStorage::class;
597require_once RUN_MAINTENANCE_IF_MAIN;
wfTempDir()
Tries to get the system directory for temporary files.
wfMessage( $key,... $params)
This is the function for getting translated interface messages.
array $params
The job parameters.
$maintClass
Maintenance script to do various checks on external storage.
importRevision( $revision)
check( $fix=false, $xml='')
__construct()
Default constructor.
execute()
Do the actual work.
ExternalStoreDB $dbStore
External storage in a SQL database.
Imports a XML dump from a file (either from file upload, files on disk, or HTTP)
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
addArg( $arg, $description, $required=true, $multi=false)
Add some args that are needed.
hasOption( $name)
Checks to see if a particular option was set.
getServiceContainer()
Returns the main service container.
getArg( $argId=0, $default=null)
Get an argument.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
Represents an authority that has all permissions.
Executes shell commands.
Definition Shell.php:46
$wgDBname
Config variable stub for the DBname setting, for use by phpdoc and IDEs.
$source