MediaWiki master
findBadBlobs.php
Go to the documentation of this file.
1<?php
29
30// @codeCoverageIgnoreStart
31require_once __DIR__ . '/Maintenance.php';
32// @codeCoverageIgnoreEnd
33
40
41 private RevisionStore $revisionStore;
42 private BlobStore $blobStore;
43
44 public function __construct() {
45 parent::__construct();
46
47 $this->setBatchSize( 1000 );
48 $this->addDescription( 'Find and mark bad content blobs. Marked blobs will be read as empty. '
49 . 'Use --scan-from to find revisions with bad blobs, use --mark to mark them.' );
50 $this->addOption( 'scan-from', 'Start scanning revisions at the given date. '
51 . 'Format: Anything supported by MediaWiki, e.g. YYYYMMDDHHMMSS or YYYY-MM-DDTHH:MM:SS',
52 false, true );
53 $this->addOption( 'scan-to', 'End of scan date range. '
54 . 'Format: Anything supported by MediaWiki, e.g. YYYYMMDDHHMMSS or YYYY-MM-DDTHH:MM:SS',
55 false, true );
56 $this->addOption( 'revisions', 'A list of revision IDs to process, separated by comma or '
57 . 'colon or whitespace. Revisions belonging to deleted pages will work. '
58 . 'If set to "-" IDs are read from stdin, one per line.', false, true );
59 $this->addOption( 'limit', 'Maximum number of revisions for --scan-from to scan. '
60 . 'Default: 1000', false, true );
61 $this->addOption( 'mark', 'Mark the blob as "known bad", to avoid errors when '
62 . 'attempting to read it. The value given is the reason for marking the blob as bad, '
63 . 'typically a ticket ID. Requires --revisions to also be set.', false, true );
64 }
65
69 private function getStartTimestamp() {
70 $tsOpt = $this->getOption( 'scan-from' );
71 if ( strlen( $tsOpt ) < 14 ) {
72 $this->fatalError( 'Bad timestamp: ' . $tsOpt
73 . ', please provide time and date down to the second.' );
74 }
75
76 $ts = wfTimestamp( TS_MW, $tsOpt );
77 if ( !$ts ) {
78 $this->fatalError( 'Bad timestamp: ' . $tsOpt );
79 }
80
81 return $ts;
82 }
83
84 private function getEndTimestamp(): string {
85 $tsOpt = $this->getOption( 'scan-to' );
86 if ( strlen( $tsOpt ) < 14 ) {
87 $this->fatalError( 'Bad timestamp: ' . $tsOpt
88 . ', please provide time and date down to the second.' );
89 }
90
91 $ts = wfTimestamp( TS_MW, $tsOpt );
92 if ( !$ts ) {
93 $this->fatalError( 'Bad timestamp: ' . $tsOpt );
94 }
95
96 return $ts;
97 }
98
102 private function getRevisionIds() {
103 $opt = $this->getOption( 'revisions' );
104
105 if ( $opt === '-' ) {
106 $opt = stream_get_contents( STDIN );
107
108 if ( !$opt ) {
109 return [];
110 }
111 }
112
113 return $this->parseIntList( $opt );
114 }
115
119 public function execute() {
120 $services = $this->getServiceContainer();
121 $this->revisionStore = $services->getRevisionStore();
122 $this->blobStore = $services->getBlobStore();
123
124 if ( $this->hasOption( 'revisions' ) ) {
125 if ( $this->hasOption( 'scan-from' ) || $this->hasOption( 'scan-to' ) ) {
126 $this->fatalError( 'Cannot use --revisions together with --scan-from or --scan-to' );
127 }
128
129 $ids = $this->getRevisionIds();
130
131 $count = $this->scanRevisionsById( $ids );
132 } elseif ( $this->hasOption( 'scan-from' ) ) {
133 if ( $this->hasOption( 'mark' ) ) {
134 $this->fatalError( 'Cannot use --mark with --scan-from, '
135 . 'use --revisions to specify revisions to mark.' );
136 }
137
138 if ( $this->hasOption( 'scan-to' ) && $this->hasOption( 'limit' ) ) {
139 $this->fatalError( 'Cannot use --limit with --scan-to' );
140 }
141
142 $count = $this->scanRevisionsByTimestamp();
143 $this->output( "The range of archive rows scanned is based on the range of revision IDs "
144 . "scanned in the revision table.\n" );
145 } else {
146 if ( $this->hasOption( 'mark' ) ) {
147 $this->fatalError( 'The --mark must be used together with --revisions' );
148 } else {
149 $this->fatalError( 'Must specify one of --revisions or --scan-from' );
150 }
151 }
152
153 if ( $this->hasOption( 'mark' ) ) {
154 $this->output( "Marked $count bad revisions.\n" );
155 } else {
156 $this->output( "Found $count bad revisions.\n" );
157
158 if ( $count > 0 ) {
159 $this->output( "On a unix/linux environment, you can use grep and cut to list of IDs\n" );
160 $this->output( "that can then be used with the --revisions option. E.g.\n" );
161 $this->output( " grep '! Found bad blob' | cut -s -f 3\n" );
162 }
163 }
164 }
165
169 private function scanRevisionsByTimestamp() {
170 $fromTimestamp = $this->getStartTimestamp();
171 if ( $this->getOption( 'scan-to' ) ) {
172 $toTimestamp = $this->getEndTimestamp();
173 } else {
174 $toTimestamp = null;
175 }
176
177 $total = $this->getOption( 'limit', 1000 );
178 $count = 0;
179 $lastRevId = 0;
180 $firstRevId = 0;
181 $lastTimestamp = $fromTimestamp;
182 $revisionRowsScanned = 0;
183 $archiveRowsScanned = 0;
184
185 $this->output( "Scanning revisions table, "
186 . "$total rows starting at rev_timestamp $fromTimestamp\n" );
187
188 while ( $toTimestamp === null ? $revisionRowsScanned < $total : true ) {
189 $batchSize = min( $total - $revisionRowsScanned, $this->getBatchSize() );
190 $revisions = $this->loadRevisionsByTimestamp( $lastRevId, $lastTimestamp, $batchSize, $toTimestamp );
191 if ( !$revisions ) {
192 break;
193 }
194
195 foreach ( $revisions as $rev ) {
196 // we are sorting by timestamp, so we may encounter revision IDs out of sequence
197 $firstRevId = $firstRevId ? min( $firstRevId, $rev->getId() ) : $rev->getId();
198 $lastRevId = max( $lastRevId, $rev->getId() );
199
200 $count += $this->checkRevision( $rev );
201 }
202
203 $lastTimestamp = $rev->getTimestamp();
204 $batchSize = count( $revisions );
205 $revisionRowsScanned += $batchSize;
206 $this->output(
207 "\t- Scanned a batch of $batchSize revisions, "
208 . "up to revision $lastRevId ($lastTimestamp)\n"
209 );
210
211 $this->waitForReplication();
212 }
213
214 // NOTE: the archive table isn't indexed by timestamp, so the best we can do is use the
215 // revision ID just before the first revision ID we found above as the starting point
216 // of the scan, and scan up to on revision after the last revision ID we found above.
217 // If $firstRevId is 0, the loop body above didn't execute,
218 // so we should skip the one below as well.
219 $fromArchived = $this->getNextRevision( $firstRevId, '<', 'DESC' );
220 $maxArchived = $this->getNextRevision( $lastRevId, '>', 'ASC' );
221 $maxArchived = $maxArchived ?: PHP_INT_MAX;
222
223 $this->output( "Scanning archive table by ar_rev_id, $fromArchived to $maxArchived\n" );
224 while ( $firstRevId > 0 && $fromArchived < $maxArchived ) {
225 $batchSize = min( $total - $archiveRowsScanned, $this->getBatchSize() );
226 $revisions = $this->loadArchiveByRevisionId( $fromArchived, $maxArchived, $batchSize );
227 if ( !$revisions ) {
228 break;
229 }
231 foreach ( $revisions as $rev ) {
232 $count += $this->checkRevision( $rev );
233 }
234 $fromArchived = $rev->getId();
235 $batchSize = count( $revisions );
236 $archiveRowsScanned += $batchSize;
237 $this->output(
238 "\t- Scanned a batch of $batchSize archived revisions, "
239 . "up to revision $fromArchived ($lastTimestamp)\n"
240 );
241
242 $this->waitForReplication();
243 }
244
245 return $count;
246 }
247
256 private function loadRevisionsByTimestamp( int $afterId, string $fromTimestamp, $batchSize, $toTimestamp ) {
257 $db = $this->getReplicaDB();
258 $queryBuilder = $this->revisionStore->newSelectQueryBuilder( $db )
259 ->joinComment()
260 ->where( $db->buildComparison( '>', [
261 'rev_timestamp' => $fromTimestamp,
262 'rev_id' => $afterId,
263 ] ) )
264 ->useIndex( [ 'revision' => 'rev_timestamp' ] )
265 ->orderBy( [ 'rev_timestamp', 'rev_id' ] )
266 ->limit( $batchSize );
267
268 if ( $toTimestamp ) {
269 $queryBuilder->where( $db->expr( 'rev_timestamp', '<', $toTimestamp ) );
270 }
271
272 $rows = $queryBuilder->caller( __METHOD__ )->fetchResultSet();
273 $result = $this->revisionStore->newRevisionsFromBatch( $rows, [ 'slots' => true ] );
274 $this->handleStatus( $result );
275
276 $records = array_filter( $result->value );
277
278 '@phan-var RevisionStoreRecord[] $records';
279 return $records;
280 }
281
289 private function loadArchiveByRevisionId( int $afterId, int $uptoId, $batchSize ) {
290 $db = $this->getReplicaDB();
291 $rows = $this->revisionStore->newArchiveSelectQueryBuilder( $db )
292 ->joinComment()
293 ->where( [ $db->expr( 'ar_rev_id', '>', $afterId ), $db->expr( 'ar_rev_id', '<=', $uptoId ) ] )
294 ->orderBy( 'ar_rev_id' )
295 ->limit( $batchSize )
296 ->caller( __METHOD__ )->fetchResultSet();
297 $result = $this->revisionStore->newRevisionsFromBatch(
298 $rows,
299 [ 'archive' => true, 'slots' => true ]
300 );
301 $this->handleStatus( $result );
302
303 $records = array_filter( $result->value );
304
305 '@phan-var RevisionArchiveRecord[] $records';
306 return $records;
307 }
308
318 private function getNextRevision( int $revId, string $comp, string $dir ) {
319 $db = $this->getReplicaDB();
320 $next = $db->newSelectQueryBuilder()
321 ->select( 'rev_id' )
322 ->from( 'revision' )
323 ->where( "rev_id $comp $revId" )
324 ->orderBy( [ "rev_id" ], $dir )
325 ->caller( __METHOD__ )
326 ->fetchField();
327 return (int)$next;
328 }
329
335 private function scanRevisionsById( array $ids ) {
336 $count = 0;
337 $total = count( $ids );
338
339 $this->output( "Scanning $total ids\n" );
340
341 foreach ( array_chunk( $ids, $this->getBatchSize() ) as $batch ) {
342 $revisions = $this->loadRevisionsById( $batch );
343
344 if ( !$revisions ) {
345 continue;
346 }
347
349 foreach ( $revisions as $rev ) {
350 $count += $this->checkRevision( $rev );
351 }
352
353 $batchSize = count( $revisions );
354 $this->output( "\t- Scanned a batch of $batchSize revisions\n" );
355 }
356
357 return $count;
358 }
359
365 private function loadRevisionsById( array $ids ) {
366 $db = $this->getReplicaDB();
367 $queryBuilder = $this->revisionStore->newSelectQueryBuilder( $db );
368
369 $rows = $queryBuilder
370 ->joinComment()
371 ->where( [ 'rev_id' => $ids ] )
372 ->caller( __METHOD__ )->fetchResultSet();
373
374 $result = $this->revisionStore->newRevisionsFromBatch( $rows, [ 'slots' => true ] );
375
376 $this->handleStatus( $result );
377
378 $revisions = array_filter( $result->value );
379 '@phan-var RevisionArchiveRecord[] $revisions';
380
381 // if not all revisions were found, check the archive table.
382 if ( count( $revisions ) < count( $ids ) ) {
383 $rows = $this->revisionStore->newArchiveSelectQueryBuilder( $db )
384 ->joinComment()
385 ->where( [ 'ar_rev_id' => array_diff( $ids, array_keys( $revisions ) ) ] )
386 ->caller( __METHOD__ )->fetchResultSet();
387
388 $archiveResult = $this->revisionStore->newRevisionsFromBatch(
389 $rows,
390 [ 'slots' => true, 'archive' => true ]
391 );
392
393 $this->handleStatus( $archiveResult );
394
395 // don't use array_merge, since it will re-index
396 $revisions += array_filter( $archiveResult->value );
397 }
398
399 return $revisions;
400 }
401
407 private function checkRevision( RevisionRecord $rev ) {
408 $count = 0;
409 foreach ( $rev->getSlots()->getSlots() as $slot ) {
410 $count += $this->checkSlot( $rev, $slot );
411 }
412
413 if ( $count === 0 && $this->hasOption( 'mark' ) ) {
414 $this->output( "\t# No bad blob found on revision {$rev->getId()}, skipped!\n" );
415 }
416
417 return $count;
418 }
419
426 private function checkSlot( RevisionRecord $rev, SlotRecord $slot ) {
427 $address = $slot->getAddress();
428
429 try {
430 $blob = $this->blobStore->getBlob( $address );
431 if ( mb_check_encoding( $blob ) ) {
432 // nothing to do
433 return 0;
434 } else {
435 $type = 'invalid-utf-8';
436 $error = 'Invalid UTF-8';
437 }
438 } catch ( Exception $ex ) {
439 $error = $ex->getMessage();
440 $type = get_class( $ex );
441 }
442
443 // NOTE: output the revision ID again at the end in a separate column for easy processing
444 // via the "cut" shell command.
445 $this->output( "\t! Found bad blob on revision {$rev->getId()} "
446 . "from {$rev->getTimestamp()} ({$slot->getRole()} slot): "
447 . "content_id={$slot->getContentId()}, address=<{$slot->getAddress()}>, "
448 . "error='$error', type='$type'. ID:\t{$rev->getId()}\n" );
449
450 if ( $this->hasOption( 'mark' ) ) {
451 $newAddress = $this->markBlob( $slot, $error );
452 $this->output( "\tChanged address to <$newAddress>\n" );
453 }
454
455 return 1;
456 }
457
464 private function markBlob( SlotRecord $slot, ?string $error = null ) {
465 $args = [];
466
467 if ( $this->hasOption( 'mark' ) ) {
468 $args['reason'] = $this->getOption( 'mark' );
469 }
470
471 if ( $error ) {
472 $args['error'] = $error;
473 }
474
475 $address = $slot->getAddress() ?: 'empty';
476 $badAddress = 'bad:' . urlencode( $address );
477
478 if ( $args ) {
479 $badAddress .= '?' . wfArrayToCgi( $args );
480 }
481
482 $badAddress = substr( $badAddress, 0, 255 );
483
484 $dbw = $this->getPrimaryDB();
485 $dbw->newUpdateQueryBuilder()
486 ->update( 'content' )
487 ->set( [ 'content_address' => $badAddress ] )
488 ->where( [ 'content_id' => $slot->getContentId() ] )
489 ->caller( __METHOD__ )->execute();
490
491 return $badAddress;
492 }
493
494 private function handleStatus( StatusValue $status ) {
495 if ( !$status->isOK() ) {
496 $this->fatalError( $status );
497 }
498 if ( !$status->isGood() ) {
499 $this->error( $status );
500 }
501 }
502
503}
504
505// @codeCoverageIgnoreStart
506$maintClass = FindBadBlobs::class;
507require_once RUN_MAINTENANCE_IF_MAIN;
508// @codeCoverageIgnoreEnd
wfArrayToCgi( $array1, $array2=null, $prefix='')
This function takes one or two arrays as input, and returns a CGI-style string, e....
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Maintenance script for finding and marking bad content blobs.
__construct()
Default constructor.
execute()
Do the actual work.All child classes will need to implement thisbool|null|void True for success,...
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
addDescription( $text)
Set the description text.
A RevisionRecord representing a revision of a deleted page persisted in the archive table.
Page revision base class.
getSlots()
Returns the slots defined for this revision.
A RevisionRecord representing an existing revision persisted in the revision table.
Service for looking up page revisions.
Value object representing a content slot associated with a page revision.
getAddress()
Returns the address of this slot's content.
getContentId()
Returns the ID of the content meta data row associated with the slot.
Generic operation result class Has warning/error list, boolean status and arbitrary value.
isOK()
Returns whether the operation completed.
isGood()
Returns whether the operation completed and didn't have any error or warnings.
$maintClass
Service for loading and storing data blobs.
Definition BlobStore.php:33