MediaWiki  master
findBadBlobs.php
Go to the documentation of this file.
1 <?php
31 
32 require_once __DIR__ . '/Maintenance.php';
33 
39 class FindBadBlobs extends Maintenance {
40 
44  private $revisionStore;
45 
49  private $blobStore;
50 
54  private $loadBalancer;
55 
59  private $lbFactory;
60 
61  public function __construct() {
62  parent::__construct();
63 
64  $this->setBatchSize( 1000 );
65  $this->addDescription( 'Find and mark bad content blobs. Marked blobs will be read as empty. '
66  . 'Use --scan-from to find revisions with bad blobs, use --mark to mark them.' );
67  $this->addOption( 'scan-from', 'Start scanning revisions at the given date. '
68  . 'Format: Anything supported by MediaWiki, e.g. YYYYMMDDHHMMSS or YYYY-MM-DDTHH:MM:SS',
69  false, true );
70  $this->addOption( 'revisions', 'A list of revision IDs to process, separated by comma or '
71  . 'colon or whitespace. Revisions belonging to deleted pages will work. '
72  . 'If set to "-" IDs are read from stdin, one per line.', false, true );
73  $this->addOption( 'limit', 'Maximum number of revisions for --scan-from to scan. '
74  . 'Default: 1000', false, true );
75  $this->addOption( 'mark', 'Mark the blob as "known bad", to avoid errors when '
76  . 'attempting to read it. The value given is the reason for marking the blob as bad, '
77  . 'typically a ticket ID. Requires --revisions to also be set.', false, true );
78  }
79 
80  public function initializeServices(
81  ?RevisionStore $revisionStore = null,
82  ?BlobStore $blobStore = null,
83  ?LoadBalancer $loadBalancer = null,
84  ?LBFactory $lbFactory = null
85  ) {
86  $services = $this->getServiceContainer();
87 
88  $this->revisionStore = $revisionStore ?? $this->revisionStore ?? $services->getRevisionStore();
89  $this->blobStore = $blobStore ?? $this->blobStore ?? $services->getBlobStore();
90  $this->loadBalancer = $loadBalancer ?? $this->loadBalancer ?? $services->getDBLoadBalancer();
91  $this->lbFactory = $lbFactory ?? $this->lbFactory ?? $services->getDBLoadBalancerFactory();
92  }
93 
97  private function getStartTimestamp() {
98  $tsOpt = $this->getOption( 'scan-from' );
99  if ( strlen( $tsOpt ) < 14 ) {
100  $this->fatalError( 'Bad timestamp: ' . $tsOpt
101  . ', please provide time and date down to the second.' );
102  }
103 
104  $ts = wfTimestamp( TS_MW, $tsOpt );
105  if ( !$ts ) {
106  $this->fatalError( 'Bad timestamp: ' . $tsOpt );
107  }
108 
109  return $ts;
110  }
111 
115  private function getRevisionIds() {
116  $opt = $this->getOption( 'revisions' );
117 
118  if ( $opt === '-' ) {
119  $opt = stream_get_contents( STDIN );
120 
121  if ( !$opt ) {
122  return [];
123  }
124  }
125 
126  return $this->parseIntList( $opt );
127  }
128 
132  public function execute() {
133  $this->initializeServices();
134 
135  if ( $this->hasOption( 'revisions' ) ) {
136  if ( $this->hasOption( 'scan-from' ) ) {
137  $this->fatalError( 'Cannot use --revisions together with --scan-from' );
138  }
139 
140  $ids = $this->getRevisionIds();
141 
142  $count = $this->scanRevisionsById( $ids );
143  } elseif ( $this->hasOption( 'scan-from' ) ) {
144  if ( $this->hasOption( 'mark' ) ) {
145  $this->fatalError( 'Cannot use --mark with --scan-from, '
146  . 'use --revisions to specify revisions to mark.' );
147  }
148 
149  $fromTimestamp = $this->getStartTimestamp();
150  $total = $this->getOption( 'limit', 1000 );
151 
152  $count = $this->scanRevisionsByTimestamp( $fromTimestamp, $total );
153 
154  $this->output( "The range of archive rows scanned is based on the range of revision IDs "
155  . "scanned in the revision table.\n" );
156  } else {
157  if ( $this->hasOption( 'mark' ) ) {
158  $this->fatalError( 'The --mark must be used together with --revisions' );
159  } else {
160  $this->fatalError( 'Must specify one of --revisions or --scan-from' );
161  }
162  }
163 
164  if ( $this->hasOption( 'mark' ) ) {
165  $this->output( "Marked $count bad revisions.\n" );
166  } else {
167  $this->output( "Found $count bad revisions.\n" );
168 
169  if ( $count > 0 ) {
170  $this->output( "On a unix/linux environment, you can use grep and cut to list of IDs\n" );
171  $this->output( "that can then be used with the --revisions option. E.g.\n" );
172  $this->output( " grep '! Found bad blob' | cut -s -f 3\n" );
173  }
174  }
175  }
176 
183  private function scanRevisionsByTimestamp( $fromTimestamp, $total ) {
184  $count = 0;
185  $lastRevId = 0;
186  $firstRevId = 0;
187  $lastTimestamp = $fromTimestamp;
188  $revisionRowsScanned = 0;
189  $archiveRowsScanned = 0;
190 
191  $this->output( "Scanning revisions table, "
192  . "$total rows starting at rev_timestamp $fromTimestamp\n" );
193 
194  while ( $revisionRowsScanned < $total ) {
195  $batchSize = min( $total - $revisionRowsScanned, $this->getBatchSize() );
196  $revisions = $this->loadRevisionsByTimestamp( $lastRevId, $lastTimestamp, $batchSize );
197  if ( !$revisions ) {
198  break;
199  }
200 
201  foreach ( $revisions as $rev ) {
202  // we are sorting by timestamp, so we may encounter revision IDs out of sequence
203  $firstRevId = $firstRevId ? min( $firstRevId, $rev->getId() ) : $rev->getId();
204  $lastRevId = max( $lastRevId, $rev->getId() );
205 
206  $count += $this->checkRevision( $rev );
207  }
208 
209  $lastTimestamp = $rev->getTimestamp();
210  $batchSize = count( $revisions );
211  $revisionRowsScanned += $batchSize;
212  $this->output(
213  "\t- Scanned a batch of $batchSize revisions, "
214  . "up to revision $lastRevId ($lastTimestamp)\n"
215  );
216 
217  $this->waitForReplication();
218  }
219 
220  // NOTE: the archive table isn't indexed by timestamp, so the best we can do is use the
221  // revision ID just before the first revision ID we found above as the starting point
222  // of the scan, and scan up to on revision after the last revision ID we found above.
223  // If $firstRevId is 0, the loop body above didn't execute,
224  // so we should skip the one below as well.
225  $fromArchived = $this->getNextRevision( $firstRevId, '<', 'DESC' );
226  $maxArchived = $this->getNextRevision( $lastRevId, '>', 'ASC' );
227  $maxArchived = $maxArchived ?: PHP_INT_MAX;
228 
229  $this->output( "Scanning archive table by ar_rev_id, $fromArchived to $maxArchived\n" );
230  while ( $firstRevId > 0 && $fromArchived < $maxArchived ) {
231  $batchSize = min( $total - $archiveRowsScanned, $this->getBatchSize() );
232  $revisions = $this->loadArchiveByRevisionId( $fromArchived, $maxArchived, $batchSize );
233  if ( !$revisions ) {
234  break;
235  }
237  foreach ( $revisions as $rev ) {
238  $count += $this->checkRevision( $rev );
239  }
240  $fromArchived = $rev->getId();
241  $batchSize = count( $revisions );
242  $archiveRowsScanned += $batchSize;
243  $this->output(
244  "\t- Scanned a batch of $batchSize archived revisions, "
245  . "up to revision $fromArchived ($lastTimestamp)\n"
246  );
247 
248  $this->waitForReplication();
249  }
250 
251  return $count;
252  }
253 
261  private function loadRevisionsByTimestamp( int $afterId, string $fromTimestamp, $batchSize ) {
262  $db = $this->lbFactory->getReplicaDatabase();
263  $queryBuilder = $this->revisionStore->newSelectQueryBuilder( $db );
264  $rows = $queryBuilder->joinComment()
265  ->where( $db->buildComparison( '>', [
266  'rev_timestamp' => $fromTimestamp,
267  'rev_id' => $afterId,
268  ] ) )
269  ->useIndex( [ 'revision' => 'rev_timestamp' ] )
270  ->orderBy( [ 'rev_timestamp', 'rev_id' ] )
271  ->limit( $batchSize )
272  ->caller( __METHOD__ )->fetchResultSet();
273  $result = $this->revisionStore->newRevisionsFromBatch( $rows, [ 'slots' => true ] );
274  $this->handleStatus( $result );
275 
276  $records = array_filter( $result->value );
277 
278  '@phan-var RevisionStoreRecord[] $records';
279  return $records;
280  }
281 
289  private function loadArchiveByRevisionId( int $afterId, int $uptoId, $batchSize ) {
290  $db = $this->lbFactory->getReplicaDatabase();
291  $rows = $this->revisionStore->newArchiveSelectQueryBuilder( $db )
292  ->joinComment()
293  ->where( [ "ar_rev_id > $afterId", "ar_rev_id <= $uptoId" ] )
294  ->orderBy( 'ar_rev_id' )
295  ->limit( $batchSize )
296  ->caller( __METHOD__ )->fetchResultSet();
297  $result = $this->revisionStore->newRevisionsFromBatch(
298  $rows,
299  [ 'archive' => true, 'slots' => true ]
300  );
301  $this->handleStatus( $result );
302 
303  $records = array_filter( $result->value );
304 
305  '@phan-var RevisionArchiveRecord[] $records';
306  return $records;
307  }
308 
318  private function getNextRevision( int $revId, string $comp, string $dir ) {
319  $db = $this->loadBalancer->getConnectionRef( DB_REPLICA );
320  $next = $db->newSelectQueryBuilder()
321  ->select( 'rev_id' )
322  ->from( 'revision' )
323  ->where( "rev_id $comp $revId" )
324  ->orderBy( [ "rev_id" ], $dir )
325  ->caller( __METHOD__ )
326  ->fetchField();
327  return (int)$next;
328  }
329 
335  private function scanRevisionsById( array $ids ) {
336  $count = 0;
337  $total = count( $ids );
338 
339  $this->output( "Scanning $total ids\n" );
340 
341  foreach ( array_chunk( $ids, $this->getBatchSize() ) as $batch ) {
342  $revisions = $this->loadRevisionsById( $batch );
343 
344  if ( !$revisions ) {
345  continue;
346  }
347 
349  foreach ( $revisions as $rev ) {
350  $count += $this->checkRevision( $rev );
351  }
352 
353  $batchSize = count( $revisions );
354  $this->output( "\t- Scanned a batch of $batchSize revisions\n" );
355  }
356 
357  return $count;
358  }
359 
365  private function loadRevisionsById( array $ids ) {
366  $db = $this->lbFactory->getReplicaDatabase();
367  $queryBuilder = $this->revisionStore->newSelectQueryBuilder( $db );
368 
369  $rows = $queryBuilder
370  ->joinComment()
371  ->where( [ 'rev_id' => $ids ] )
372  ->caller( __METHOD__ )->fetchResultSet();
373 
374  $result = $this->revisionStore->newRevisionsFromBatch( $rows, [ 'slots' => true ] );
375 
376  $this->handleStatus( $result );
377 
378  $revisions = array_filter( $result->value );
379  '@phan-var RevisionArchiveRecord[] $revisions';
380 
381  // if not all revisions were found, check the archive table.
382  if ( count( $revisions ) < count( $ids ) ) {
383  $rows = $this->revisionStore->newArchiveSelectQueryBuilder( $db )
384  ->joinComment()
385  ->where( [ 'ar_rev_id' => array_diff( $ids, array_keys( $revisions ) ) ] )
386  ->caller( __METHOD__ )->fetchResultSet();
387 
388  $archiveResult = $this->revisionStore->newRevisionsFromBatch(
389  $rows,
390  [ 'slots' => true, 'archive' => true ]
391  );
392 
393  $this->handleStatus( $archiveResult );
394 
395  // don't use array_merge, since it will re-index
396  $revisions += array_filter( $archiveResult->value );
397  }
398 
399  return $revisions;
400  }
401 
407  private function checkRevision( RevisionRecord $rev ) {
408  $count = 0;
409  foreach ( $rev->getSlots()->getSlots() as $slot ) {
410  $count += $this->checkSlot( $rev, $slot );
411  }
412 
413  if ( $count === 0 && $this->hasOption( 'mark' ) ) {
414  $this->output( "\t# No bad blob found on revision {$rev->getId()}, skipped!\n" );
415  }
416 
417  return $count;
418  }
419 
426  private function checkSlot( RevisionRecord $rev, SlotRecord $slot ) {
427  $address = $slot->getAddress();
428 
429  try {
430  $this->blobStore->getBlob( $address );
431  // nothing to do
432  return 0;
433  } catch ( Exception $ex ) {
434  $error = $ex->getMessage();
435  $type = get_class( $ex );
436  }
437 
438  // NOTE: output the revision ID again at the end in a separate column for easy processing
439  // via the "cut" shell command.
440  $this->output( "\t! Found bad blob on revision {$rev->getId()} "
441  . "from {$rev->getTimestamp()} ({$slot->getRole()} slot): "
442  . "content_id={$slot->getContentId()}, address=<{$slot->getAddress()}>, "
443  . "error='$error', type='$type'. ID:\t{$rev->getId()}\n" );
444 
445  if ( $this->hasOption( 'mark' ) ) {
446  $newAddress = $this->markBlob( $slot, $error );
447  $this->output( "\tChanged address to <$newAddress>\n" );
448  }
449 
450  return 1;
451  }
452 
459  private function markBlob( SlotRecord $slot, string $error = null ) {
460  $args = [];
461 
462  if ( $this->hasOption( 'mark' ) ) {
463  $args['reason'] = $this->getOption( 'mark' );
464  }
465 
466  if ( $error ) {
467  $args['error'] = $error;
468  }
469 
470  $address = $slot->getAddress() ?: 'empty';
471  $badAddress = 'bad:' . urlencode( $address );
472 
473  if ( $args ) {
474  $badAddress .= '?' . wfArrayToCgi( $args );
475  }
476 
477  $badAddress = substr( $badAddress, 0, 255 );
478 
479  $dbw = $this->loadBalancer->getConnectionRef( DB_PRIMARY );
480  $dbw->update(
481  'content',
482  [ 'content_address' => $badAddress ],
483  [ 'content_id' => $slot->getContentId() ],
484  __METHOD__
485  );
486 
487  return $badAddress;
488  }
489 
490  private function handleStatus( StatusValue $status ) {
491  if ( !$status->isOK() ) {
492  $this->fatalError(
493  Status::wrap( $status )->getMessage( false, false, 'en' )->text()
494  );
495  }
496  if ( !$status->isGood() ) {
497  $this->error(
498  "\t! " . Status::wrap( $status )->getMessage( false, false, 'en' )->text()
499  );
500  }
501  }
502 
503 }
504 
505 $maintClass = FindBadBlobs::class;
506 require_once RUN_MAINTENANCE_IF_MAIN;
wfArrayToCgi( $array1, $array2=null, $prefix='')
This function takes one or two arrays as input, and returns a CGI-style string, e....
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Maintenance script for finding and marking bad content blobs.
__construct()
Default constructor.
initializeServices(?RevisionStore $revisionStore=null, ?BlobStore $blobStore=null, ?LoadBalancer $loadBalancer=null, ?LBFactory $lbFactory=null)
execute()
Do the actual work.All child classes will need to implement thisbool|null|void True for success,...
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:66
error( $err, $die=0)
Throw an error to the user.
output( $out, $channel=null)
Throw some output to the user.
waitForReplication()
Wait for replica DBs to catch up.
hasOption( $name)
Checks to see if a particular option was set.
getServiceContainer()
Returns the main service container.
getBatchSize()
Returns batch size.
parseIntList( $text)
Utility function to parse a string (perhaps from a command line option) into a list of integers (perh...
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
setBatchSize( $s=0)
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
A RevisionRecord representing a revision of a deleted page persisted in the archive table.
Page revision base class.
getSlots()
Returns the slots defined for this revision.
A RevisionRecord representing an existing revision persisted in the revision table.
Service for looking up page revisions.
Value object representing a content slot associated with a page revision.
Definition: SlotRecord.php:40
getAddress()
Returns the address of this slot's content.
Definition: SlotRecord.php:519
getContentId()
Returns the ID of the content meta data row associated with the slot.
Definition: SlotRecord.php:533
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Definition: Status.php:58
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Definition: StatusValue.php:46
isOK()
Returns whether the operation completed.
isGood()
Returns whether the operation completed and didn't have any error or warnings.
$maintClass
Service for loading and storing data blobs.
Definition: BlobStore.php:33
const DB_REPLICA
Definition: defines.php:26
const DB_PRIMARY
Definition: defines.php:28