MediaWiki  master
checkStorage.php
Go to the documentation of this file.
1 <?php
26 
27 if ( !defined( 'MEDIAWIKI' ) ) {
28  $optionsWithoutArgs = [ 'fix' ];
29  require_once __DIR__ . '/../CommandLineInc.php';
30 
31  $cs = new CheckStorage;
32  $fix = isset( $options['fix'] );
33  $xml = $args[0] ?? false;
34  $cs->check( $fix, $xml );
35 }
36 
37 // ----------------------------------------------------------------------------------
38 
45 class CheckStorage {
46  private const CONCAT_HEADER = 'O:27:"concatenatedgziphistoryblob"';
47  public $oldIdMap, $errors;
49  public $dbStore = null;
50 
51  public $errorDescriptions = [
52  'restore text' => 'Damaged text, need to be restored from a backup',
53  'restore revision' => 'Damaged revision row, need to be restored from a backup',
54  'unfixable' => 'Unexpected errors with no automated fixing method',
55  'fixed' => 'Errors already fixed',
56  'fixable' => 'Errors which would already be fixed if --fix was specified',
57  ];
58 
59  public function check( $fix = false, $xml = '' ) {
60  $dbr = wfGetDB( DB_REPLICA );
61  if ( $fix ) {
62  print "Checking, will fix errors if possible...\n";
63  } else {
64  print "Checking...\n";
65  }
66  $maxRevId = $dbr->selectField( 'revision', 'MAX(rev_id)', '', __METHOD__ );
67  $chunkSize = 1000;
68  $flagStats = [];
69  $objectStats = [];
70  $knownFlags = [ 'external', 'gzip', 'object', 'utf-8' ];
71  $this->errors = [
72  'restore text' => [],
73  'restore revision' => [],
74  'unfixable' => [],
75  'fixed' => [],
76  'fixable' => [],
77  ];
78 
79  for ( $chunkStart = 1; $chunkStart < $maxRevId; $chunkStart += $chunkSize ) {
80  $chunkEnd = $chunkStart + $chunkSize - 1;
81  // print "$chunkStart of $maxRevId\n";
82 
83  $this->oldIdMap = [];
84  $dbr->ping();
85 
86  // Fetch revision rows
87  $res = $dbr->select(
88  [ 'slots', 'content' ],
89  [ 'slot_revision_id', 'content_address' ],
90  [ "slot_revision_id BETWEEN $chunkStart AND $chunkEnd" ],
91  __METHOD__,
92  [],
93  [ 'content' => [ 'INNER JOIN', [ 'content_id = slot_content_id' ] ] ]
94  );
96  $blobStore = MediaWikiServices::getInstance()->getBlobStore();
97  '@phan-var \MediaWiki\Storage\SqlBlobStore $blobStore';
98  foreach ( $res as $row ) {
99  $textId = $blobStore->getTextIdFromAddress( $row->content_address );
100  if ( $textId ) {
101  if ( !isset( $this->oldIdMap[$textId] ) ) {
102  $this->oldIdMap[ $textId ] = [ $row->slot_revision_id ];
103  } elseif ( !in_array( $row->slot_revision_id, $this->oldIdMap[$textId] ) ) {
104  $this->oldIdMap[ $textId ][] = $row->slot_revision_id;
105  }
106  }
107  }
108 
109  if ( !count( $this->oldIdMap ) ) {
110  continue;
111  }
112 
113  // Fetch old_flags
114  $missingTextRows = $this->oldIdMap;
115  $externalRevs = [];
116  $objectRevs = [];
117  $res = $dbr->select(
118  'text',
119  [ 'old_id', 'old_flags' ],
120  [ 'old_id' => array_keys( $this->oldIdMap ) ],
121  __METHOD__
122  );
123  foreach ( $res as $row ) {
127  $flags = $row->old_flags;
128  $id = $row->old_id;
129 
130  // Create flagStats row if it doesn't exist
131  $flagStats += [ $flags => 0 ];
132  // Increment counter
133  $flagStats[$flags]++;
134 
135  // Not missing
136  unset( $missingTextRows[$row->old_id] );
137 
138  // Check for external or object
139  if ( $flags == '' ) {
140  $flagArray = [];
141  } else {
142  $flagArray = explode( ',', $flags );
143  }
144  if ( in_array( 'external', $flagArray ) ) {
145  $externalRevs[] = $id;
146  } elseif ( in_array( 'object', $flagArray ) ) {
147  $objectRevs[] = $id;
148  }
149 
150  // Check for unrecognised flags
151  if ( $flags == '0' ) {
152  // This is a known bug from 2004
153  // It's safe to just erase the old_flags field
154  if ( $fix ) {
155  $this->addError( 'fixed', "Warning: old_flags set to 0", $id );
156  $dbw = wfGetDB( DB_PRIMARY );
157  $dbw->ping();
158  $dbw->update( 'text', [ 'old_flags' => '' ],
159  [ 'old_id' => $id ], __METHOD__ );
160  echo "Fixed\n";
161  } else {
162  $this->addError( 'fixable', "Warning: old_flags set to 0", $id );
163  }
164  } elseif ( count( array_diff( $flagArray, $knownFlags ) ) ) {
165  $this->addError( 'unfixable', "Error: invalid flags field \"$flags\"", $id );
166  }
167  }
168 
169  // Output errors for any missing text rows
170  foreach ( $missingTextRows as $oldId => $revIds ) {
171  $this->addError( 'restore revision', "Error: missing text row", $oldId );
172  }
173 
174  // Verify external revisions
175  $externalConcatBlobs = [];
176  $externalNormalBlobs = [];
177  if ( count( $externalRevs ) ) {
178  $res = $dbr->select(
179  'text',
180  [ 'old_id', 'old_flags', 'old_text' ],
181  [ 'old_id' => $externalRevs ],
182  __METHOD__
183  );
184  foreach ( $res as $row ) {
185  $urlParts = explode( '://', $row->old_text, 2 );
186  if ( count( $urlParts ) !== 2 || $urlParts[1] == '' ) {
187  $this->addError( 'restore text', "Error: invalid URL \"{$row->old_text}\"", $row->old_id );
188  continue;
189  }
190  list( $proto, ) = $urlParts;
191  if ( $proto != 'DB' ) {
192  $this->addError(
193  'restore text',
194  "Error: invalid external protocol \"$proto\"",
195  $row->old_id );
196  continue;
197  }
198  $path = explode( '/', $row->old_text );
199  $cluster = $path[2];
200  $id = $path[3];
201  if ( isset( $path[4] ) ) {
202  $externalConcatBlobs[$cluster][$id][] = $row->old_id;
203  } else {
204  $externalNormalBlobs[$cluster][$id][] = $row->old_id;
205  }
206  }
207  }
208 
209  // Check external concat blobs for the right header
210  $this->checkExternalConcatBlobs( $externalConcatBlobs );
211 
212  // Check external normal blobs for existence
213  if ( count( $externalNormalBlobs ) ) {
214  if ( $this->dbStore === null ) {
215  $esFactory = MediaWikiServices::getInstance()->getExternalStoreFactory();
216  $this->dbStore = $esFactory->getStore( 'DB' );
217  }
218  foreach ( $externalConcatBlobs as $cluster => $xBlobIds ) {
219  $blobIds = array_keys( $xBlobIds );
220  $extDb =& $this->dbStore->getReplica( $cluster );
221  $blobsTable = $this->dbStore->getTable( $extDb );
222  $res = $extDb->select( $blobsTable,
223  [ 'blob_id' ],
224  [ 'blob_id' => $blobIds ],
225  __METHOD__
226  );
227  foreach ( $res as $row ) {
228  unset( $xBlobIds[$row->blob_id] );
229  }
230  // Print errors for missing blobs rows
231  foreach ( $xBlobIds as $blobId => $oldId ) {
232  $this->addError(
233  'restore text',
234  "Error: missing target $blobId for one-part ES URL",
235  $oldId );
236  }
237  }
238  }
239 
240  // Check local objects
241  $dbr->ping();
242  $concatBlobs = [];
243  $curIds = [];
244  if ( count( $objectRevs ) ) {
245  $headerLength = 300;
246  $res = $dbr->select(
247  'text',
248  [ 'old_id', 'old_flags', "LEFT(old_text, $headerLength) AS header" ],
249  [ 'old_id' => $objectRevs ],
250  __METHOD__
251  );
252  foreach ( $res as $row ) {
253  $oldId = $row->old_id;
254  $matches = [];
255  if ( !preg_match( '/^O:(\d+):"(\w+)"/', $row->header, $matches ) ) {
256  $this->addError( 'restore text', "Error: invalid object header", $oldId );
257  continue;
258  }
259 
260  $className = strtolower( $matches[2] );
261  if ( strlen( $className ) != $matches[1] ) {
262  $this->addError(
263  'restore text',
264  "Error: invalid object header, wrong class name length",
265  $oldId
266  );
267  continue;
268  }
269 
270  $objectStats += [ $className => 0 ];
271  $objectStats[$className]++;
272 
273  switch ( $className ) {
274  case 'concatenatedgziphistoryblob':
275  // Good
276  break;
277  case 'historyblobstub':
278  case 'historyblobcurstub':
279  if ( strlen( $row->header ) == $headerLength ) {
280  $this->addError( 'unfixable', "Error: overlong stub header", $oldId );
281  break;
282  }
283  $stubObj = unserialize( $row->header );
284  if ( !is_object( $stubObj ) ) {
285  $this->addError( 'restore text', "Error: unable to unserialize stub object", $oldId );
286  break;
287  }
288  if ( $className == 'historyblobstub' ) {
289  $concatBlobs[$stubObj->getLocation()][] = $oldId;
290  } else {
291  $curIds[$stubObj->mCurId][] = $oldId;
292  }
293  break;
294  default:
295  $this->addError( 'unfixable', "Error: unrecognised object class \"$className\"", $oldId );
296  }
297  }
298  }
299 
300  // Check local concat blob validity
301  $externalConcatBlobs = [];
302  if ( count( $concatBlobs ) ) {
303  $headerLength = 300;
304  $res = $dbr->select(
305  'text',
306  [ 'old_id', 'old_flags', "LEFT(old_text, $headerLength) AS header" ],
307  [ 'old_id' => array_keys( $concatBlobs ) ],
308  __METHOD__
309  );
310  foreach ( $res as $row ) {
311  $flags = explode( ',', $row->old_flags );
312  if ( in_array( 'external', $flags ) ) {
313  // Concat blob is in external storage?
314  if ( in_array( 'object', $flags ) ) {
315  $urlParts = explode( '/', $row->header );
316  if ( $urlParts[0] != 'DB:' ) {
317  $this->addError(
318  'unfixable',
319  "Error: unrecognised external storage type \"{$urlParts[0]}",
320  $row->old_id
321  );
322  } else {
323  $cluster = $urlParts[2];
324  $id = $urlParts[3];
325  if ( !isset( $externalConcatBlobs[$cluster][$id] ) ) {
326  $externalConcatBlobs[$cluster][$id] = [];
327  }
328  $externalConcatBlobs[$cluster][$id] = array_merge(
329  $externalConcatBlobs[$cluster][$id], $concatBlobs[$row->old_id]
330  );
331  }
332  } else {
333  $this->addError(
334  'unfixable',
335  "Error: invalid flags \"{$row->old_flags}\" on concat bulk row {$row->old_id}",
336  $concatBlobs[$row->old_id] );
337  }
338  } elseif ( strcasecmp(
339  substr( $row->header, 0, strlen( self::CONCAT_HEADER ) ),
340  self::CONCAT_HEADER
341  ) ) {
342  $this->addError(
343  'restore text',
344  "Error: Incorrect object header for concat bulk row {$row->old_id}",
345  $concatBlobs[$row->old_id]
346  );
347  }
348 
349  unset( $concatBlobs[$row->old_id] );
350  }
351  }
352 
353  // Check targets of unresolved stubs
354  $this->checkExternalConcatBlobs( $externalConcatBlobs );
355  // next chunk
356  }
357 
358  print "\n\nErrors:\n";
359  foreach ( $this->errors as $name => $errors ) {
360  if ( count( $errors ) ) {
361  $description = $this->errorDescriptions[$name];
362  echo "$description: " . implode( ',', array_keys( $errors ) ) . "\n";
363  }
364  }
365 
366  if ( count( $this->errors['restore text'] ) && $fix ) {
367  if ( (string)$xml !== '' ) {
368  $this->restoreText( array_keys( $this->errors['restore text'] ), $xml );
369  } else {
370  echo "Can't fix text, no XML backup specified\n";
371  }
372  }
373 
374  print "\nFlag statistics:\n";
375  $total = array_sum( $flagStats );
376  foreach ( $flagStats as $flag => $count ) {
377  printf( "%-30s %10d %5.2f%%\n", $flag, $count, $count / $total * 100 );
378  }
379  print "\nLocal object statistics:\n";
380  $total = array_sum( $objectStats );
381  foreach ( $objectStats as $className => $count ) {
382  printf( "%-30s %10d %5.2f%%\n", $className, $count, $count / $total * 100 );
383  }
384  }
385 
386  private function addError( $type, $msg, $ids ) {
387  if ( is_array( $ids ) && count( $ids ) == 1 ) {
388  $ids = reset( $ids );
389  }
390  if ( is_array( $ids ) ) {
391  $revIds = [];
392  foreach ( $ids as $id ) {
393  $revIds = array_unique( array_merge( $revIds, $this->oldIdMap[$id] ) );
394  }
395  print "$msg in text rows " . implode( ', ', $ids ) .
396  ", revisions " . implode( ', ', $revIds ) . "\n";
397  } else {
398  $id = $ids;
399  $revIds = $this->oldIdMap[$id];
400  if ( count( $revIds ) == 1 ) {
401  print "$msg in old_id $id, rev_id {$revIds[0]}\n";
402  } else {
403  print "$msg in old_id $id, revisions " . implode( ', ', $revIds ) . "\n";
404  }
405  }
406  $this->errors[$type] += array_fill_keys( $revIds, true );
407  }
408 
409  private function checkExternalConcatBlobs( $externalConcatBlobs ) {
410  if ( !count( $externalConcatBlobs ) ) {
411  return;
412  }
413 
414  if ( $this->dbStore === null ) {
415  $esFactory = MediaWikiServices::getInstance()->getExternalStoreFactory();
416  $this->dbStore = $esFactory->getStore( 'DB' );
417  }
418 
419  foreach ( $externalConcatBlobs as $cluster => $oldIds ) {
420  $blobIds = array_keys( $oldIds );
421  $extDb =& $this->dbStore->getReplica( $cluster );
422  $blobsTable = $this->dbStore->getTable( $extDb );
423  $headerLength = strlen( self::CONCAT_HEADER );
424  $res = $extDb->select( $blobsTable,
425  [ 'blob_id', "LEFT(blob_text, $headerLength) AS header" ],
426  [ 'blob_id' => $blobIds ],
427  __METHOD__
428  );
429  foreach ( $res as $row ) {
430  if ( strcasecmp( $row->header, self::CONCAT_HEADER ) ) {
431  $this->addError(
432  'restore text',
433  "Error: invalid header on target $cluster/{$row->blob_id} of two-part ES URL",
434  $oldIds[$row->blob_id]
435  );
436  }
437  unset( $oldIds[$row->blob_id] );
438  }
439 
440  // Print errors for missing blobs rows
441  foreach ( $oldIds as $blobId => $oldIds2 ) {
442  $this->addError(
443  'restore text',
444  "Error: missing target $cluster/$blobId for two-part ES URL",
445  $oldIds2
446  );
447  }
448  }
449  }
450 
451  private function restoreText( $revIds, $xml ) {
452  global $wgDBname;
453  $tmpDir = wfTempDir();
454 
455  if ( !count( $revIds ) ) {
456  return;
457  }
458 
459  print "Restoring text from XML backup...\n";
460 
461  $revFileName = "$tmpDir/broken-revlist-$wgDBname";
462  $filteredXmlFileName = "$tmpDir/filtered-$wgDBname.xml";
463 
464  // Write revision list
465  if ( !file_put_contents( $revFileName, implode( "\n", $revIds ) ) ) {
466  echo "Error writing revision list, can't restore text\n";
467 
468  return;
469  }
470 
471  // Run mwdumper
472  echo "Filtering XML dump...\n";
473  $exitStatus = 0;
474  passthru( 'mwdumper ' .
475  Shell::escape(
476  "--output=file:$filteredXmlFileName",
477  "--filter=revlist:$revFileName",
478  $xml
479  ), $exitStatus
480  );
481 
482  if ( $exitStatus ) {
483  echo "mwdumper died with exit status $exitStatus\n";
484 
485  return;
486  }
487 
488  $file = fopen( $filteredXmlFileName, 'r' );
489  if ( !$file ) {
490  echo "Unable to open filtered XML file\n";
491 
492  return;
493  }
494 
495  $dbr = wfGetDB( DB_REPLICA );
496  $dbw = wfGetDB( DB_PRIMARY );
497  $dbr->ping();
498  $dbw->ping();
499 
501  $importer = MediaWikiServices::getInstance()
502  ->getWikiImporterFactory()
503  ->getWikiImporter( $source );
504  $importer->setRevisionCallback( [ $this, 'importRevision' ] );
505  $importer->setNoticeCallback( static function ( $msg, $params ) {
506  echo wfMessage( $msg, $params )->text() . "\n";
507  } );
508  $importer->doImport();
509  }
510 
514  public function importRevision( $revision ) {
515  $id = $revision->getID();
516  $content = $revision->getContent();
517  $id = $id ?: '';
518 
519  if ( $content === null ) {
520  echo "Revision $id is broken, we have no content available\n";
521 
522  return;
523  }
524 
525  $text = $content->serialize();
526  if ( $text === '' ) {
527  // This is what happens if the revision was broken at the time the
528  // dump was made. Unfortunately, it also happens if the revision was
529  // legitimately blank, so there's no way to tell the difference. To
530  // be safe, we'll skip it and leave it broken
531 
532  echo "Revision $id is blank in the dump, may have been broken before export\n";
533 
534  return;
535  }
536 
537  if ( !$id ) {
538  // No ID, can't import
539  echo "No id tag in revision, can't import\n";
540 
541  return;
542  }
543 
544  // Find text row again
545  $dbr = wfGetDB( DB_REPLICA );
546  $res = $dbr->selectRow(
547  [ 'slots', 'content' ],
548  [ 'content_address' ],
549  [ 'slot_revision_id' => $id ],
550  __METHOD__,
551  [],
552  [ 'content' => [ 'INNER JOIN', [ 'content_id = slot_content_id' ] ] ]
553  );
554 
555  $blobStore = MediaWikiServices::getInstance()
556  ->getBlobStoreFactory()
557  ->newSqlBlobStore();
558  $oldId = $blobStore->getTextIdFromAddress( $res->content_address );
559 
560  if ( !$oldId ) {
561  echo "Missing revision row for rev_id $id\n";
562  return;
563  }
564 
565  // Compress the text
566  $flags = $blobStore->compressData( $text );
567 
568  // Update the text row
569  $dbw = wfGetDB( DB_PRIMARY );
570  $dbw->update( 'text',
571  [ 'old_flags' => $flags, 'old_text' => $text ],
572  [ 'old_id' => $oldId ],
573  __METHOD__, [ 'LIMIT' => 1 ]
574  );
575 
576  // Remove it from the unfixed list and add it to the fixed list
577  unset( $this->errors['restore text'][$id] );
578  $this->errors['fixed'][$id] = true;
579  }
580 }
MediaWiki\Shell\Shell
Executes shell commands.
Definition: Shell.php:45
CheckStorage
Maintenance script to do various checks on external storage.
Definition: checkStorage.php:45
ExternalStoreDB
External storage in a SQL database.
Definition: ExternalStoreDB.php:39
$wgDBname
$wgDBname
Current wiki database name.
Definition: DefaultSettings.php:2065
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:203
CheckStorage\check
check( $fix=false, $xml='')
Definition: checkStorage.php:59
$file
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition: router.php:42
wfMessage
wfMessage( $key,... $params)
This is the function for getting translated interface messages.
Definition: GlobalFunctions.php:1167
$res
$res
Definition: testCompression.php:57
CheckStorage\importRevision
importRevision( $revision)
Definition: checkStorage.php:514
ImportStreamSource
Imports a XML dump from a file (either from file upload, files on disk, or HTTP)
Definition: ImportStreamSource.php:33
CheckStorage\$errorDescriptions
$errorDescriptions
Definition: checkStorage.php:51
$dbr
$dbr
Definition: testCompression.php:54
CheckStorage\$oldIdMap
$oldIdMap
Definition: checkStorage.php:47
wfGetDB
wfGetDB( $db, $groups=[], $wiki=false)
Get a Database object.
Definition: GlobalFunctions.php:2186
$matches
$matches
Definition: NoLocalSettings.php:24
$args
if( $line===false) $args
Definition: mcc.php:124
CheckStorage\addError
addError( $type, $msg, $ids)
Definition: checkStorage.php:386
DB_REPLICA
const DB_REPLICA
Definition: defines.php:25
CheckStorage\CONCAT_HEADER
const CONCAT_HEADER
Definition: checkStorage.php:46
$content
$content
Definition: router.php:76
$optionsWithoutArgs
global $optionsWithoutArgs
Definition: CommandLineInc.php:28
DB_PRIMARY
const DB_PRIMARY
Definition: defines.php:27
CheckStorage\restoreText
restoreText( $revIds, $xml)
Definition: checkStorage.php:451
unserialize
unserialize( $serialized)
Definition: ApiMessageTrait.php:146
wfTempDir
wfTempDir()
Tries to get the system directory for temporary files.
Definition: GlobalFunctions.php:1712
CheckStorage\$errors
$errors
Definition: checkStorage.php:47
$path
$path
Definition: NoLocalSettings.php:25
$source
$source
Definition: mwdoc-filter.php:34
CheckStorage\$dbStore
ExternalStoreDB $dbStore
Definition: checkStorage.php:49
CheckStorage\checkExternalConcatBlobs
checkExternalConcatBlobs( $externalConcatBlobs)
Definition: checkStorage.php:409
$type
$type
Definition: testCompression.php:52