45 parent::__construct();
48 $this->
addDescription(
'Find and mark bad content blobs. Marked blobs will be read as empty. '
49 .
'Use --scan-from to find revisions with bad blobs, use --mark to mark them.' );
50 $this->
addOption(
'scan-from',
'Start scanning revisions at the given date. '
51 .
'Format: Anything supported by MediaWiki, e.g. YYYYMMDDHHMMSS or YYYY-MM-DDTHH:MM:SS',
53 $this->
addOption(
'scan-to',
'End of scan date range. '
54 .
'Format: Anything supported by MediaWiki, e.g. YYYYMMDDHHMMSS or YYYY-MM-DDTHH:MM:SS',
56 $this->
addOption(
'revisions',
'A list of revision IDs to process, separated by comma or '
57 .
'colon or whitespace. Revisions belonging to deleted pages will work. '
58 .
'If set to "-" IDs are read from stdin, one per line.',
false,
true );
59 $this->
addOption(
'limit',
'Maximum number of revisions for --scan-from to scan. '
60 .
'Default: 1000',
false,
true );
61 $this->
addOption(
'mark',
'Mark the blob as "known bad", to avoid errors when '
62 .
'attempting to read it. The value given is the reason for marking the blob as bad, '
63 .
'typically a ticket ID. Requires --revisions to also be set.',
false,
true );
69 private function getStartTimestamp() {
71 if ( strlen( $tsOpt ) < 14 ) {
73 .
', please provide time and date down to the second.' );
78 $this->
fatalError(
'Bad timestamp: ' . $tsOpt );
84 private function getEndTimestamp(): string {
86 if ( strlen( $tsOpt ) < 14 ) {
88 .
', please provide time and date down to the second.' );
93 $this->
fatalError(
'Bad timestamp: ' . $tsOpt );
102 private function getRevisionIds() {
103 $opt = $this->getOption(
'revisions' );
105 if ( $opt ===
'-' ) {
106 $opt = stream_get_contents( STDIN );
113 return $this->parseIntList( $opt );
120 $services = $this->getServiceContainer();
121 $this->revisionStore = $services->getRevisionStore();
122 $this->blobStore = $services->getBlobStore();
124 if ( $this->hasOption(
'revisions' ) ) {
125 if ( $this->hasOption(
'scan-from' ) || $this->hasOption(
'scan-to' ) ) {
126 $this->fatalError(
'Cannot use --revisions together with --scan-from or --scan-to' );
129 $ids = $this->getRevisionIds();
131 $count = $this->scanRevisionsById( $ids );
132 } elseif ( $this->hasOption(
'scan-from' ) ) {
133 if ( $this->hasOption(
'mark' ) ) {
134 $this->fatalError(
'Cannot use --mark with --scan-from, '
135 .
'use --revisions to specify revisions to mark.' );
138 if ( $this->hasOption(
'scan-to' ) && $this->hasOption(
'limit' ) ) {
139 $this->fatalError(
'Cannot use --limit with --scan-to' );
142 $count = $this->scanRevisionsByTimestamp();
143 $this->output(
"The range of archive rows scanned is based on the range of revision IDs "
144 .
"scanned in the revision table.\n" );
146 if ( $this->hasOption(
'mark' ) ) {
147 $this->fatalError(
'The --mark must be used together with --revisions' );
149 $this->fatalError(
'Must specify one of --revisions or --scan-from' );
153 if ( $this->hasOption(
'mark' ) ) {
154 $this->output(
"Marked $count bad revisions.\n" );
156 $this->output(
"Found $count bad revisions.\n" );
159 $this->output(
"On a unix/linux environment, you can use grep and cut to list of IDs\n" );
160 $this->output(
"that can then be used with the --revisions option. E.g.\n" );
161 $this->output(
" grep '! Found bad blob' | cut -s -f 3\n" );
169 private function scanRevisionsByTimestamp() {
170 $fromTimestamp = $this->getStartTimestamp();
171 if ( $this->getOption(
'scan-to' ) ) {
172 $toTimestamp = $this->getEndTimestamp();
177 $total = $this->getOption(
'limit', 1000 );
181 $lastTimestamp = $fromTimestamp;
182 $revisionRowsScanned = 0;
183 $archiveRowsScanned = 0;
185 $this->output(
"Scanning revisions table, "
186 .
"$total rows starting at rev_timestamp $fromTimestamp\n" );
188 while ( $toTimestamp ===
null ? $revisionRowsScanned < $total : true ) {
189 $batchSize = min( $total - $revisionRowsScanned, $this->getBatchSize() );
190 $revisions = $this->loadRevisionsByTimestamp( $lastRevId, $lastTimestamp, $batchSize, $toTimestamp );
195 foreach ( $revisions as $rev ) {
197 $firstRevId = $firstRevId ? min( $firstRevId, $rev->getId() ) : $rev->getId();
198 $lastRevId = max( $lastRevId, $rev->getId() );
200 $count += $this->checkRevision( $rev );
203 $lastTimestamp = $rev->getTimestamp();
204 $batchSize = count( $revisions );
205 $revisionRowsScanned += $batchSize;
207 "\t- Scanned a batch of $batchSize revisions, "
208 .
"up to revision $lastRevId ($lastTimestamp)\n"
211 $this->waitForReplication();
219 $fromArchived = $this->getNextRevision( $firstRevId,
'<',
'DESC' );
220 $maxArchived = $this->getNextRevision( $lastRevId,
'>',
'ASC' );
221 $maxArchived = $maxArchived ?: PHP_INT_MAX;
223 $this->output(
"Scanning archive table by ar_rev_id, $fromArchived to $maxArchived\n" );
224 while ( $firstRevId > 0 && $fromArchived < $maxArchived ) {
225 $batchSize = min( $total - $archiveRowsScanned, $this->getBatchSize() );
226 $revisions = $this->loadArchiveByRevisionId( $fromArchived, $maxArchived, $batchSize );
231 foreach ( $revisions as $rev ) {
232 $count += $this->checkRevision( $rev );
234 $fromArchived = $rev->getId();
235 $batchSize = count( $revisions );
236 $archiveRowsScanned += $batchSize;
238 "\t- Scanned a batch of $batchSize archived revisions, "
239 .
"up to revision $fromArchived ($lastTimestamp)\n"
242 $this->waitForReplication();
256 private function loadRevisionsByTimestamp(
int $afterId,
string $fromTimestamp, $batchSize, $toTimestamp ) {
257 $db = $this->getReplicaDB();
258 $queryBuilder = $this->revisionStore->newSelectQueryBuilder( $db )
260 ->where( $db->buildComparison(
'>', [
261 'rev_timestamp' => $fromTimestamp,
262 'rev_id' => $afterId,
264 ->useIndex( [
'revision' =>
'rev_timestamp' ] )
265 ->orderBy( [
'rev_timestamp',
'rev_id' ] )
266 ->limit( $batchSize );
268 if ( $toTimestamp ) {
269 $queryBuilder->where( $db->expr(
'rev_timestamp',
'<', $toTimestamp ) );
272 $rows = $queryBuilder->caller( __METHOD__ )->fetchResultSet();
273 $result = $this->revisionStore->newRevisionsFromBatch( $rows, [
'slots' =>
true ] );
274 $this->handleStatus( $result );
276 $records = array_filter( $result->value );
278 '@phan-var RevisionStoreRecord[] $records';
289 private function loadArchiveByRevisionId(
int $afterId,
int $uptoId, $batchSize ) {
290 $db = $this->getReplicaDB();
291 $rows = $this->revisionStore->newArchiveSelectQueryBuilder( $db )
293 ->where( [ $db->expr(
'ar_rev_id',
'>', $afterId ), $db->expr(
'ar_rev_id',
'<=', $uptoId ) ] )
294 ->orderBy(
'ar_rev_id' )
295 ->limit( $batchSize )
296 ->caller( __METHOD__ )->fetchResultSet();
297 $result = $this->revisionStore->newRevisionsFromBatch(
299 [
'archive' =>
true,
'slots' =>
true ]
301 $this->handleStatus( $result );
303 $records = array_filter( $result->value );
305 '@phan-var RevisionArchiveRecord[] $records';
318 private function getNextRevision(
int $revId,
string $comp,
string $dir ) {
319 $db = $this->getReplicaDB();
320 $next = $db->newSelectQueryBuilder()
323 ->where(
"rev_id $comp $revId" )
324 ->orderBy( [
"rev_id" ], $dir )
325 ->caller( __METHOD__ )
335 private function scanRevisionsById( array $ids ) {
337 $total = count( $ids );
339 $this->output(
"Scanning $total ids\n" );
341 foreach ( array_chunk( $ids, $this->getBatchSize() ) as $batch ) {
342 $revisions = $this->loadRevisionsById( $batch );
349 foreach ( $revisions as $rev ) {
350 $count += $this->checkRevision( $rev );
353 $batchSize = count( $revisions );
354 $this->output(
"\t- Scanned a batch of $batchSize revisions\n" );
365 private function loadRevisionsById( array $ids ) {
366 $db = $this->getReplicaDB();
367 $queryBuilder = $this->revisionStore->newSelectQueryBuilder( $db );
369 $rows = $queryBuilder
371 ->where( [
'rev_id' => $ids ] )
372 ->caller( __METHOD__ )->fetchResultSet();
374 $result = $this->revisionStore->newRevisionsFromBatch( $rows, [
'slots' =>
true ] );
376 $this->handleStatus( $result );
378 $revisions = array_filter( $result->value );
379 '@phan-var RevisionArchiveRecord[] $revisions';
382 if ( count( $revisions ) < count( $ids ) ) {
383 $rows = $this->revisionStore->newArchiveSelectQueryBuilder( $db )
385 ->where( [
'ar_rev_id' => array_diff( $ids, array_keys( $revisions ) ) ] )
386 ->caller( __METHOD__ )->fetchResultSet();
388 $archiveResult = $this->revisionStore->newRevisionsFromBatch(
390 [
'slots' =>
true,
'archive' =>
true ]
393 $this->handleStatus( $archiveResult );
396 $revisions += array_filter( $archiveResult->value );
409 foreach ( $rev->
getSlots()->getSlots() as $slot ) {
410 $count += $this->checkSlot( $rev, $slot );
413 if ( $count === 0 && $this->hasOption(
'mark' ) ) {
414 $this->output(
"\t# No bad blob found on revision {$rev->getId()}, skipped!\n" );
430 $blob = $this->blobStore->getBlob( $address );
431 if ( mb_check_encoding( $blob ) ) {
435 $type =
'invalid-utf-8';
436 $error =
'Invalid UTF-8';
438 }
catch ( Exception $ex ) {
439 $error = $ex->getMessage();
440 $type = get_class( $ex );
445 $this->output(
"\t! Found bad blob on revision {$rev->getId()} "
446 .
"from {$rev->getTimestamp()} ({$slot->getRole()} slot): "
447 .
"content_id={$slot->getContentId()}, address=<{$slot->getAddress()}>, "
448 .
"error='$error', type='$type'. ID:\t{$rev->getId()}\n" );
450 if ( $this->hasOption(
'mark' ) ) {
451 $newAddress = $this->markBlob( $slot, $error );
452 $this->output(
"\tChanged address to <$newAddress>\n" );
464 private function markBlob(
SlotRecord $slot, ?
string $error =
null ) {
467 if ( $this->hasOption(
'mark' ) ) {
468 $args[
'reason'] = $this->getOption(
'mark' );
472 $args[
'error'] = $error;
476 $badAddress =
'bad:' . urlencode( $address );
482 $badAddress = substr( $badAddress, 0, 255 );
484 $dbw = $this->getPrimaryDB();
485 $dbw->newUpdateQueryBuilder()
486 ->update(
'content' )
487 ->set( [
'content_address' => $badAddress ] )
489 ->caller( __METHOD__ )->execute();
494 private function handleStatus(
StatusValue $status ) {
495 if ( !$status->
isOK() ) {
496 $this->fatalError( $status );
498 if ( !$status->
isGood() ) {
499 $this->error( $status );