MediaWiki master
findBadBlobs.php
Go to the documentation of this file.
1<?php
28
29require_once __DIR__ . '/Maintenance.php';
30
37
38 private RevisionStore $revisionStore;
39 private BlobStore $blobStore;
40
41 public function __construct() {
42 parent::__construct();
43
44 $this->setBatchSize( 1000 );
45 $this->addDescription( 'Find and mark bad content blobs. Marked blobs will be read as empty. '
46 . 'Use --scan-from to find revisions with bad blobs, use --mark to mark them.' );
47 $this->addOption( 'scan-from', 'Start scanning revisions at the given date. '
48 . 'Format: Anything supported by MediaWiki, e.g. YYYYMMDDHHMMSS or YYYY-MM-DDTHH:MM:SS',
49 false, true );
50 $this->addOption( 'revisions', 'A list of revision IDs to process, separated by comma or '
51 . 'colon or whitespace. Revisions belonging to deleted pages will work. '
52 . 'If set to "-" IDs are read from stdin, one per line.', false, true );
53 $this->addOption( 'limit', 'Maximum number of revisions for --scan-from to scan. '
54 . 'Default: 1000', false, true );
55 $this->addOption( 'mark', 'Mark the blob as "known bad", to avoid errors when '
56 . 'attempting to read it. The value given is the reason for marking the blob as bad, '
57 . 'typically a ticket ID. Requires --revisions to also be set.', false, true );
58 }
59
63 private function getStartTimestamp() {
64 $tsOpt = $this->getOption( 'scan-from' );
65 if ( strlen( $tsOpt ) < 14 ) {
66 $this->fatalError( 'Bad timestamp: ' . $tsOpt
67 . ', please provide time and date down to the second.' );
68 }
69
70 $ts = wfTimestamp( TS_MW, $tsOpt );
71 if ( !$ts ) {
72 $this->fatalError( 'Bad timestamp: ' . $tsOpt );
73 }
74
75 return $ts;
76 }
77
81 private function getRevisionIds() {
82 $opt = $this->getOption( 'revisions' );
83
84 if ( $opt === '-' ) {
85 $opt = stream_get_contents( STDIN );
86
87 if ( !$opt ) {
88 return [];
89 }
90 }
91
92 return $this->parseIntList( $opt );
93 }
94
98 public function execute() {
99 $services = $this->getServiceContainer();
100 $this->revisionStore = $services->getRevisionStore();
101 $this->blobStore = $services->getBlobStore();
102 $this->setDBProvider( $services->getConnectionProvider() );
103
104 if ( $this->hasOption( 'revisions' ) ) {
105 if ( $this->hasOption( 'scan-from' ) ) {
106 $this->fatalError( 'Cannot use --revisions together with --scan-from' );
107 }
108
109 $ids = $this->getRevisionIds();
110
111 $count = $this->scanRevisionsById( $ids );
112 } elseif ( $this->hasOption( 'scan-from' ) ) {
113 if ( $this->hasOption( 'mark' ) ) {
114 $this->fatalError( 'Cannot use --mark with --scan-from, '
115 . 'use --revisions to specify revisions to mark.' );
116 }
117
118 $fromTimestamp = $this->getStartTimestamp();
119 $total = $this->getOption( 'limit', 1000 );
120
121 $count = $this->scanRevisionsByTimestamp( $fromTimestamp, $total );
122
123 $this->output( "The range of archive rows scanned is based on the range of revision IDs "
124 . "scanned in the revision table.\n" );
125 } else {
126 if ( $this->hasOption( 'mark' ) ) {
127 $this->fatalError( 'The --mark must be used together with --revisions' );
128 } else {
129 $this->fatalError( 'Must specify one of --revisions or --scan-from' );
130 }
131 }
132
133 if ( $this->hasOption( 'mark' ) ) {
134 $this->output( "Marked $count bad revisions.\n" );
135 } else {
136 $this->output( "Found $count bad revisions.\n" );
137
138 if ( $count > 0 ) {
139 $this->output( "On a unix/linux environment, you can use grep and cut to list of IDs\n" );
140 $this->output( "that can then be used with the --revisions option. E.g.\n" );
141 $this->output( " grep '! Found bad blob' | cut -s -f 3\n" );
142 }
143 }
144 }
145
152 private function scanRevisionsByTimestamp( $fromTimestamp, $total ) {
153 $count = 0;
154 $lastRevId = 0;
155 $firstRevId = 0;
156 $lastTimestamp = $fromTimestamp;
157 $revisionRowsScanned = 0;
158 $archiveRowsScanned = 0;
159
160 $this->output( "Scanning revisions table, "
161 . "$total rows starting at rev_timestamp $fromTimestamp\n" );
162
163 while ( $revisionRowsScanned < $total ) {
164 $batchSize = min( $total - $revisionRowsScanned, $this->getBatchSize() );
165 $revisions = $this->loadRevisionsByTimestamp( $lastRevId, $lastTimestamp, $batchSize );
166 if ( !$revisions ) {
167 break;
168 }
169
170 foreach ( $revisions as $rev ) {
171 // we are sorting by timestamp, so we may encounter revision IDs out of sequence
172 $firstRevId = $firstRevId ? min( $firstRevId, $rev->getId() ) : $rev->getId();
173 $lastRevId = max( $lastRevId, $rev->getId() );
174
175 $count += $this->checkRevision( $rev );
176 }
177
178 $lastTimestamp = $rev->getTimestamp();
179 $batchSize = count( $revisions );
180 $revisionRowsScanned += $batchSize;
181 $this->output(
182 "\t- Scanned a batch of $batchSize revisions, "
183 . "up to revision $lastRevId ($lastTimestamp)\n"
184 );
185
186 $this->waitForReplication();
187 }
188
189 // NOTE: the archive table isn't indexed by timestamp, so the best we can do is use the
190 // revision ID just before the first revision ID we found above as the starting point
191 // of the scan, and scan up to on revision after the last revision ID we found above.
192 // If $firstRevId is 0, the loop body above didn't execute,
193 // so we should skip the one below as well.
194 $fromArchived = $this->getNextRevision( $firstRevId, '<', 'DESC' );
195 $maxArchived = $this->getNextRevision( $lastRevId, '>', 'ASC' );
196 $maxArchived = $maxArchived ?: PHP_INT_MAX;
197
198 $this->output( "Scanning archive table by ar_rev_id, $fromArchived to $maxArchived\n" );
199 while ( $firstRevId > 0 && $fromArchived < $maxArchived ) {
200 $batchSize = min( $total - $archiveRowsScanned, $this->getBatchSize() );
201 $revisions = $this->loadArchiveByRevisionId( $fromArchived, $maxArchived, $batchSize );
202 if ( !$revisions ) {
203 break;
204 }
206 foreach ( $revisions as $rev ) {
207 $count += $this->checkRevision( $rev );
208 }
209 $fromArchived = $rev->getId();
210 $batchSize = count( $revisions );
211 $archiveRowsScanned += $batchSize;
212 $this->output(
213 "\t- Scanned a batch of $batchSize archived revisions, "
214 . "up to revision $fromArchived ($lastTimestamp)\n"
215 );
216
217 $this->waitForReplication();
218 }
219
220 return $count;
221 }
222
230 private function loadRevisionsByTimestamp( int $afterId, string $fromTimestamp, $batchSize ) {
231 $db = $this->getReplicaDB();
232 $queryBuilder = $this->revisionStore->newSelectQueryBuilder( $db );
233 $rows = $queryBuilder->joinComment()
234 ->where( $db->buildComparison( '>', [
235 'rev_timestamp' => $fromTimestamp,
236 'rev_id' => $afterId,
237 ] ) )
238 ->useIndex( [ 'revision' => 'rev_timestamp' ] )
239 ->orderBy( [ 'rev_timestamp', 'rev_id' ] )
240 ->limit( $batchSize )
241 ->caller( __METHOD__ )->fetchResultSet();
242 $result = $this->revisionStore->newRevisionsFromBatch( $rows, [ 'slots' => true ] );
243 $this->handleStatus( $result );
244
245 $records = array_filter( $result->value );
246
247 '@phan-var RevisionStoreRecord[] $records';
248 return $records;
249 }
250
258 private function loadArchiveByRevisionId( int $afterId, int $uptoId, $batchSize ) {
259 $db = $this->getReplicaDB();
260 $rows = $this->revisionStore->newArchiveSelectQueryBuilder( $db )
261 ->joinComment()
262 ->where( [ "ar_rev_id > $afterId", "ar_rev_id <= $uptoId" ] )
263 ->orderBy( 'ar_rev_id' )
264 ->limit( $batchSize )
265 ->caller( __METHOD__ )->fetchResultSet();
266 $result = $this->revisionStore->newRevisionsFromBatch(
267 $rows,
268 [ 'archive' => true, 'slots' => true ]
269 );
270 $this->handleStatus( $result );
271
272 $records = array_filter( $result->value );
273
274 '@phan-var RevisionArchiveRecord[] $records';
275 return $records;
276 }
277
287 private function getNextRevision( int $revId, string $comp, string $dir ) {
288 $db = $this->getReplicaDB();
289 $next = $db->newSelectQueryBuilder()
290 ->select( 'rev_id' )
291 ->from( 'revision' )
292 ->where( "rev_id $comp $revId" )
293 ->orderBy( [ "rev_id" ], $dir )
294 ->caller( __METHOD__ )
295 ->fetchField();
296 return (int)$next;
297 }
298
304 private function scanRevisionsById( array $ids ) {
305 $count = 0;
306 $total = count( $ids );
307
308 $this->output( "Scanning $total ids\n" );
309
310 foreach ( array_chunk( $ids, $this->getBatchSize() ) as $batch ) {
311 $revisions = $this->loadRevisionsById( $batch );
312
313 if ( !$revisions ) {
314 continue;
315 }
316
318 foreach ( $revisions as $rev ) {
319 $count += $this->checkRevision( $rev );
320 }
321
322 $batchSize = count( $revisions );
323 $this->output( "\t- Scanned a batch of $batchSize revisions\n" );
324 }
325
326 return $count;
327 }
328
334 private function loadRevisionsById( array $ids ) {
335 $db = $this->getReplicaDB();
336 $queryBuilder = $this->revisionStore->newSelectQueryBuilder( $db );
337
338 $rows = $queryBuilder
339 ->joinComment()
340 ->where( [ 'rev_id' => $ids ] )
341 ->caller( __METHOD__ )->fetchResultSet();
342
343 $result = $this->revisionStore->newRevisionsFromBatch( $rows, [ 'slots' => true ] );
344
345 $this->handleStatus( $result );
346
347 $revisions = array_filter( $result->value );
348 '@phan-var RevisionArchiveRecord[] $revisions';
349
350 // if not all revisions were found, check the archive table.
351 if ( count( $revisions ) < count( $ids ) ) {
352 $rows = $this->revisionStore->newArchiveSelectQueryBuilder( $db )
353 ->joinComment()
354 ->where( [ 'ar_rev_id' => array_diff( $ids, array_keys( $revisions ) ) ] )
355 ->caller( __METHOD__ )->fetchResultSet();
356
357 $archiveResult = $this->revisionStore->newRevisionsFromBatch(
358 $rows,
359 [ 'slots' => true, 'archive' => true ]
360 );
361
362 $this->handleStatus( $archiveResult );
363
364 // don't use array_merge, since it will re-index
365 $revisions += array_filter( $archiveResult->value );
366 }
367
368 return $revisions;
369 }
370
376 private function checkRevision( RevisionRecord $rev ) {
377 $count = 0;
378 foreach ( $rev->getSlots()->getSlots() as $slot ) {
379 $count += $this->checkSlot( $rev, $slot );
380 }
381
382 if ( $count === 0 && $this->hasOption( 'mark' ) ) {
383 $this->output( "\t# No bad blob found on revision {$rev->getId()}, skipped!\n" );
384 }
385
386 return $count;
387 }
388
395 private function checkSlot( RevisionRecord $rev, SlotRecord $slot ) {
396 $address = $slot->getAddress();
397
398 try {
399 $this->blobStore->getBlob( $address );
400 // nothing to do
401 return 0;
402 } catch ( Exception $ex ) {
403 $error = $ex->getMessage();
404 $type = get_class( $ex );
405 }
406
407 // NOTE: output the revision ID again at the end in a separate column for easy processing
408 // via the "cut" shell command.
409 $this->output( "\t! Found bad blob on revision {$rev->getId()} "
410 . "from {$rev->getTimestamp()} ({$slot->getRole()} slot): "
411 . "content_id={$slot->getContentId()}, address=<{$slot->getAddress()}>, "
412 . "error='$error', type='$type'. ID:\t{$rev->getId()}\n" );
413
414 if ( $this->hasOption( 'mark' ) ) {
415 $newAddress = $this->markBlob( $slot, $error );
416 $this->output( "\tChanged address to <$newAddress>\n" );
417 }
418
419 return 1;
420 }
421
428 private function markBlob( SlotRecord $slot, string $error = null ) {
429 $args = [];
430
431 if ( $this->hasOption( 'mark' ) ) {
432 $args['reason'] = $this->getOption( 'mark' );
433 }
434
435 if ( $error ) {
436 $args['error'] = $error;
437 }
438
439 $address = $slot->getAddress() ?: 'empty';
440 $badAddress = 'bad:' . urlencode( $address );
441
442 if ( $args ) {
443 $badAddress .= '?' . wfArrayToCgi( $args );
444 }
445
446 $badAddress = substr( $badAddress, 0, 255 );
447
448 $dbw = $this->getPrimaryDB();
449 $dbw->newUpdateQueryBuilder()
450 ->update( 'content' )
451 ->set( [ 'content_address' => $badAddress ] )
452 ->where( [ 'content_id' => $slot->getContentId() ] )
453 ->caller( __METHOD__ )->execute();
454
455 return $badAddress;
456 }
457
458 private function handleStatus( StatusValue $status ) {
459 if ( !$status->isOK() ) {
460 $this->fatalError( $status );
461 }
462 if ( !$status->isGood() ) {
463 $this->error( $status );
464 }
465 }
466
467}
468
469$maintClass = FindBadBlobs::class;
470require_once RUN_MAINTENANCE_IF_MAIN;
wfArrayToCgi( $array1, $array2=null, $prefix='')
This function takes one or two arrays as input, and returns a CGI-style string, e....
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Maintenance script for finding and marking bad content blobs.
__construct()
Default constructor.
execute()
Do the actual work.All child classes will need to implement thisbool|null|void True for success,...
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
error( $err, $die=0)
Throw an error to the user.
output( $out, $channel=null)
Throw some output to the user.
waitForReplication()
Wait for replica DBs to catch up.
hasOption( $name)
Checks to see if a particular option was set.
getServiceContainer()
Returns the main service container.
getBatchSize()
Returns batch size.
parseIntList( $text)
Utility function to parse a string (perhaps from a command line option) into a list of integers (perh...
addDescription( $text)
Set the description text.
setDBProvider(IConnectionProvider $dbProvider)
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
setBatchSize( $s=0)
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
A RevisionRecord representing a revision of a deleted page persisted in the archive table.
Page revision base class.
getSlots()
Returns the slots defined for this revision.
A RevisionRecord representing an existing revision persisted in the revision table.
Service for looking up page revisions.
Value object representing a content slot associated with a page revision.
getAddress()
Returns the address of this slot's content.
getContentId()
Returns the ID of the content meta data row associated with the slot.
Generic operation result class Has warning/error list, boolean status and arbitrary value.
isOK()
Returns whether the operation completed.
isGood()
Returns whether the operation completed and didn't have any error or warnings.
$maintClass
Service for loading and storing data blobs.
Definition BlobStore.php:33