37 private const REPORTING_INTERVAL = 100;
40 parent::__construct();
42 $this->
addOption(
'verbose',
'Output information about link refresh progress',
false,
false,
'v' );
43 $this->
addOption(
'dfn-only',
'Delete links from nonexistent articles only' );
44 $this->
addOption(
'new-only',
'Only affect articles with just a single edit' );
45 $this->
addOption(
'redirects-only',
'Only fix redirects, not all links' );
46 $this->
addOption(
'touched-only',
'Only fix pages that have been touched after last update' );
47 $this->
addOption(
'e',
'Last page id to refresh',
false,
true );
48 $this->
addOption(
'dfn-chunk-size',
'Maximum number of existent IDs to check per ' .
49 'query, default 100,000',
false,
true );
50 $this->
addOption(
'namespace',
'Only fix pages in this namespace',
false,
true );
51 $this->
addOption(
'category',
'Only fix pages in this category',
false,
true );
52 $this->
addOption(
'tracking-category',
'Only fix pages in this tracking category',
false,
true );
53 $this->
addOption(
'before-timestamp',
'Only fix pages that were last updated before this timestamp',
55 $this->
addArg(
'start',
'Page_id to start from, default 1',
false );
64 $start = (int)$this->
getArg( 0 ) ?:
null;
65 $end = (int)$this->
getOption(
'e' ) ?:
null;
66 $dfnChunkSize = (int)$this->
getOption(
'dfn-chunk-size', 100_000 );
69 $this->deleteLinksFromNonexistent( $start, $end, $this->
getBatchSize(), $dfnChunkSize );
74 $builder = $dbr->newSelectQueryBuilder()
76 ->where( self::intervalCond( $dbr,
'page_id', $start, $end ) )
80 $builder->
andWhere( [
'page_namespace' => (
int)$this->
getOption(
'namespace' ) ] );
83 if ( $this->
hasOption(
'before-timestamp' ) ) {
85 $dbr->expr(
'page_links_updated',
'<', $this->getOption(
'before-timestamp' ) )
86 ->or(
'page_links_updated',
'=',
null )
91 $category = $this->
getOption(
'category' );
92 $title = Title::makeTitleSafe(
NS_CATEGORY, $category );
94 $this->
fatalError(
"'$category' is an invalid category name!\n" );
96 $this->refreshCategory( $builder, $title );
97 } elseif ( $this->
hasOption(
'tracking-category' ) ) {
99 $this->refreshTrackingCategory( $builder, $this->
getOption(
'tracking-category' ) );
102 $redir = $this->
hasOption(
'redirects-only' );
103 $touched = $this->
hasOption(
'touched-only' );
104 $what = $redir ?
'redirects' :
'links';
106 $builder->
andWhere( [
'page_is_new' => 1 ] );
107 $this->
output(
"Refreshing $what from new pages...\n" );
111 'page_touched > page_links_updated OR page_links_updated IS NULL',
114 $this->
output(
"Refreshing $what from pages...\n" );
116 $this->doRefreshLinks( $builder, $redir );
117 if ( !$this->
hasOption(
'namespace' ) ) {
118 $this->deleteLinksFromNonexistent( $start, $end, $this->
getBatchSize(), $dfnChunkSize );
129 private function doRefreshLinks(
131 bool $redirectsOnly =
false,
132 array $indexFields = [
'page_id' ]
135 $this->getHookRunner()->onMaintenanceRefreshLinksInit( $this );
138 $this->output(
"Estimated page count: $estimateCount\n" );
141 $lastIndexes = array_fill_keys( $indexFields, 0 );
142 $selectFields = in_array(
'page_id', $indexFields )
143 ? $indexFields : [
'page_id', ...$indexFields ];
144 $verbose = $this->hasOption(
'verbose' );
147 $batchCond = $dbr->buildComparison(
'>', $lastIndexes );
148 $res = ( clone $builder )->select( $selectFields )
149 ->andWhere( [ $batchCond ] )
150 ->orderBy( $indexFields )
151 ->caller( __METHOD__ )->fetchResultSet();
154 $this->output(
"Refreshing links for {$res->numRows()} pages\n" );
157 foreach ( $res as $row ) {
158 if ( !( ++$i % self::REPORTING_INTERVAL ) ) {
163 $this->
output(
"Refreshing links for page ID {$row->page_id}\n" );
165 self::fixRedirect( $this, $row->page_id );
166 if ( !$redirectsOnly ) {
167 self::fixLinksFromArticle( $row->page_id );
170 if ( $res->numRows() ) {
171 $res->seek( $res->numRows() - 1 );
172 foreach ( $indexFields as $field ) {
173 $lastIndexes[$field] = $res->current()->$field;
177 }
while ( $res->numRows() == $this->getBatchSize() );
198 if ( $page ===
null ) {
203 $content = $page->getContent( RevisionRecord::RAW );
204 if ( $content !==
null ) {
205 $rt = $content->getRedirectTarget();
209 if ( $rt ===
null ) {
212 $dbw->newDeleteQueryBuilder()
213 ->deleteFrom(
'redirect' )
214 ->where( [
'rd_from' => $id ] )
215 ->caller( __METHOD__ )->execute();
218 $page->insertRedirectEntry( $rt );
223 $dbw->newUpdateQueryBuilder()
225 ->set( [
'page_is_redirect' => $fieldValue ] )
226 ->where( [
'page_id' => $id ] )
227 ->caller( __METHOD__ )
236 $services = MediaWikiServices::getInstance();
237 $page = $services->getWikiPageFactory()->newFromID( $id );
240 if ( $page ===
null ) {
247 $page->doSecondaryDataUpdates( [
248 'defer' => DeferredUpdates::POSTSEND,
249 'causeAction' =>
'refresh-links-maintenance',
250 'recursive' =>
false,
252 DeferredUpdates::doUpdates();
266 private function deleteLinksFromNonexistent( $start =
null, $end =
null, $batchSize = 100,
269 $this->waitForReplication();
270 $this->output(
"Deleting illegal entries from the links tables...\n" );
275 $nextStart = $dbr->newSelectQueryBuilder()
276 ->select(
'page_id' )
278 ->where( [ self::intervalCond( $dbr,
'page_id', $start, $end ) ] )
279 ->orderBy(
'page_id' )
280 ->offset( $chunkSize )
281 ->caller( __METHOD__ )->fetchField();
283 if ( $nextStart !==
false ) {
288 $chunkEnd = $nextStart - 1;
294 $fmtStart = $start !==
null ?
"[$start" :
'(-INF';
295 $fmtChunkEnd = $chunkEnd !==
null ?
"$chunkEnd]" :
'INF)';
296 $this->
output(
" Checking interval $fmtStart, $fmtChunkEnd\n" );
297 $this->dfnCheckInterval( $start, $chunkEnd, $batchSize );
301 }
while ( $nextStart !==
false );
310 private function dfnCheckInterval( $start =
null, $end =
null, $batchSize = 100 ) {
316 'pagelinks' =>
'pl_from',
317 'imagelinks' =>
'il_from',
318 'categorylinks' =>
'cl_from',
319 'templatelinks' =>
'tl_from',
320 'externallinks' =>
'el_from',
321 'iwlinks' =>
'iwl_from',
322 'langlinks' =>
'll_from',
323 'redirect' =>
'rd_from',
324 'page_props' =>
'pp_page',
327 foreach ( $linksTables as $table => $field ) {
328 $this->
output(
" $table: 0" );
329 $tableStart = $start;
332 $ids = $dbr->newSelectQueryBuilder()
336 ->leftJoin(
'page',
null,
"$field = page_id" )
337 ->where( self::intervalCond( $dbr, $field, $tableStart, $end ) )
338 ->andWhere( [
'page_id' =>
null ] )
340 ->limit( $batchSize )
341 ->caller( __METHOD__ )->fetchFieldValues();
343 $numIds = count( $ids );
346 $dbw->newDeleteQueryBuilder()
347 ->deleteFrom( $table )
348 ->where( [ $field => $ids ] )
349 ->caller( __METHOD__ )->execute();
350 $this->
output(
", $counter" );
351 $tableStart = $ids[$numIds - 1] + 1;
355 }
while ( $numIds >= $batchSize && ( $end ===
null || $tableStart <= $end ) );
357 $this->
output(
" deleted.\n" );
373 private static function intervalCond(
IReadableDatabase $db, $var, $start, $end ) {
374 if ( $start ===
null && $end ===
null ) {
375 return "$var IS NOT NULL";
376 } elseif ( $end ===
null ) {
377 return "$var >= " . $db->
addQuotes( $start );
378 } elseif ( $start ===
null ) {
379 return "$var <= " . $db->
addQuotes( $end );
392 $cats = $this->getPossibleCategories( $category );
395 $this->
error(
"Tracking category '$category' is disabled\n" );
399 foreach ( $cats as $cat ) {
400 $this->refreshCategory( clone $builder, $cat );
411 $this->
output(
"Refreshing pages in category '{$category->getText()}'...\n" );
413 $builder->
join(
'categorylinks',
null,
'page_id=cl_from' )
414 ->andWhere( [
'cl_to' => $category->
getDBkey() ] );
415 $this->doRefreshLinks( $builder,
false, [
'cl_timestamp',
'cl_from' ] );
424 private function getPossibleCategories( $categoryKey ) {
426 if ( isset( $cats[$categoryKey] ) ) {
427 return $cats[$categoryKey][
'cats'];
429 $this->
fatalError(
"Unknown tracking category {$categoryKey}\n" );
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
error( $err, $die=0)
Throw an error to the user.
addArg( $arg, $description, $required=true, $multi=false)
Add some args that are needed.
output( $out, $channel=null)
Throw some output to the user.
waitForReplication()
Wait for replica DBs to catch up.
hasOption( $name)
Checks to see if a particular option was set.
getServiceContainer()
Returns the main service container.
getArg( $argId=0, $default=null)
Get an argument.
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.