38 private const REPORTING_INTERVAL = 100;
41 parent::__construct();
43 $this->
addOption(
'verbose',
'Output information about link refresh progress',
false,
false,
'v' );
44 $this->
addOption(
'dfn-only',
'Delete links from nonexistent articles only' );
45 $this->
addOption(
'new-only',
'Only affect articles with just a single edit' );
46 $this->
addOption(
'redirects-only',
'Only fix redirects, not all links' );
47 $this->
addOption(
'touched-only',
'Only fix pages that have been touched after last update' );
48 $this->
addOption(
'e',
'Last page id to refresh',
false,
true );
49 $this->
addOption(
'dfn-chunk-size',
'Maximum number of existent IDs to check per ' .
50 'query, default 100,000',
false,
true );
51 $this->
addOption(
'namespace',
'Only fix pages in this namespace',
false,
true );
52 $this->
addOption(
'category',
'Only fix pages in this category',
false,
true );
53 $this->
addOption(
'tracking-category',
'Only fix pages in this tracking category',
false,
true );
54 $this->
addOption(
'before-timestamp',
'Only fix pages that were last updated before this timestamp',
56 $this->
addArg(
'start',
'Page_id to start from, default 1',
false );
65 $start = (int)$this->
getArg( 0 ) ?:
null;
66 $end = (int)$this->
getOption(
'e' ) ?:
null;
67 $dfnChunkSize = (int)$this->
getOption(
'dfn-chunk-size', 100_000 );
70 $this->deleteLinksFromNonexistent( $start, $end, $this->
getBatchSize(), $dfnChunkSize );
75 $builder = $dbr->newSelectQueryBuilder()
77 ->where( self::intervalCond( $dbr,
'page_id', $start, $end ) )
81 $builder->
andWhere( [
'page_namespace' => (
int)$this->
getOption(
'namespace' ) ] );
84 if ( $this->
hasOption(
'before-timestamp' ) ) {
86 $dbr->expr(
'page_links_updated',
'<', $this->getOption(
'before-timestamp' ) )
87 ->or(
'page_links_updated',
'=',
null )
92 $category = $this->
getOption(
'category' );
93 $title = Title::makeTitleSafe(
NS_CATEGORY, $category );
95 $this->
fatalError(
"'$category' is an invalid category name!\n" );
97 $this->refreshCategory( $builder, $title );
98 } elseif ( $this->
hasOption(
'tracking-category' ) ) {
100 $this->refreshTrackingCategory( $builder, $this->
getOption(
'tracking-category' ) );
103 $redir = $this->
hasOption(
'redirects-only' );
104 $touched = $this->
hasOption(
'touched-only' );
105 $what = $redir ?
'redirects' :
'links';
107 $builder->
andWhere( [
'page_is_new' => 1 ] );
108 $this->
output(
"Refreshing $what from new pages...\n" );
112 $dbr->expr(
'page_touched',
'>',
'page_links_updated' )
113 ->or(
'page_links_updated',
'=',
null ),
116 $this->
output(
"Refreshing $what from pages...\n" );
118 $this->doRefreshLinks( $builder, $redir );
119 if ( !$this->
hasOption(
'namespace' ) ) {
120 $this->deleteLinksFromNonexistent( $start, $end, $this->
getBatchSize(), $dfnChunkSize );
131 private function doRefreshLinks(
133 bool $redirectsOnly =
false,
134 array $indexFields = [
'page_id' ]
137 $this->getHookRunner()->onMaintenanceRefreshLinksInit( $this );
140 $this->output(
"Estimated page count: $estimateCount\n" );
143 $lastIndexes = array_fill_keys( $indexFields, 0 );
144 $selectFields = in_array(
'page_id', $indexFields )
145 ? $indexFields : [
'page_id', ...$indexFields ];
146 $verbose = $this->hasOption(
'verbose' );
149 $batchCond = $dbr->buildComparison(
'>', $lastIndexes );
150 $res = ( clone $builder )->select( $selectFields )
151 ->andWhere( [ $batchCond ] )
152 ->orderBy( $indexFields )
153 ->caller( __METHOD__ )->fetchResultSet();
156 $this->output(
"Refreshing links for {$res->numRows()} pages\n" );
159 foreach ( $res as $row ) {
160 if ( !( ++$i % self::REPORTING_INTERVAL ) ) {
165 $this->
output(
"Refreshing links for page ID {$row->page_id}\n" );
167 self::fixRedirect( $this, $row->page_id );
168 if ( !$redirectsOnly ) {
169 self::fixLinksFromArticle( $row->page_id );
172 if ( $res->numRows() ) {
173 $res->seek( $res->numRows() - 1 );
174 foreach ( $indexFields as $field ) {
175 $lastIndexes[$field] = $res->current()->$field;
179 }
while ( $res->numRows() == $this->getBatchSize() );
200 if ( $page ===
null ) {
205 $content = $page->getContent( RevisionRecord::RAW );
206 if ( $content !==
null ) {
207 $rt = $content->getRedirectTarget();
211 if ( $rt ===
null ) {
214 $dbw->newDeleteQueryBuilder()
215 ->deleteFrom(
'redirect' )
216 ->where( [
'rd_from' => $id ] )
217 ->caller( __METHOD__ )->execute();
220 $page->insertRedirectEntry( $rt );
225 $dbw->newUpdateQueryBuilder()
227 ->set( [
'page_is_redirect' => $fieldValue ] )
228 ->where( [
'page_id' => $id ] )
229 ->caller( __METHOD__ )
238 $services = MediaWikiServices::getInstance();
239 $page = $services->getWikiPageFactory()->newFromID( $id );
242 if ( $page ===
null ) {
249 $page->doSecondaryDataUpdates( [
250 'defer' => DeferredUpdates::POSTSEND,
251 'causeAction' =>
'refresh-links-maintenance',
252 'recursive' =>
false,
254 DeferredUpdates::doUpdates();
268 private function deleteLinksFromNonexistent( $start =
null, $end =
null, $batchSize = 100,
271 $this->waitForReplication();
272 $this->output(
"Deleting illegal entries from the links tables...\n" );
277 $nextStart = $dbr->newSelectQueryBuilder()
278 ->select(
'page_id' )
280 ->where( [ self::intervalCond( $dbr,
'page_id', $start, $end ) ] )
281 ->orderBy(
'page_id' )
282 ->offset( $chunkSize )
283 ->caller( __METHOD__ )->fetchField();
285 if ( $nextStart !==
false ) {
290 $chunkEnd = $nextStart - 1;
296 $fmtStart = $start !==
null ?
"[$start" :
'(-INF';
297 $fmtChunkEnd = $chunkEnd !==
null ?
"$chunkEnd]" :
'INF)';
298 $this->
output(
" Checking interval $fmtStart, $fmtChunkEnd\n" );
299 $this->dfnCheckInterval( $start, $chunkEnd, $batchSize );
303 }
while ( $nextStart !==
false );
312 private function dfnCheckInterval( $start =
null, $end =
null, $batchSize = 100 ) {
318 'pagelinks' =>
'pl_from',
319 'imagelinks' =>
'il_from',
320 'categorylinks' =>
'cl_from',
321 'templatelinks' =>
'tl_from',
322 'externallinks' =>
'el_from',
323 'iwlinks' =>
'iwl_from',
324 'langlinks' =>
'll_from',
325 'redirect' =>
'rd_from',
326 'page_props' =>
'pp_page',
329 foreach ( $linksTables as $table => $field ) {
330 $this->
output(
" $table: 0" );
331 $tableStart = $start;
334 $ids = $dbr->newSelectQueryBuilder()
338 ->leftJoin(
'page',
null,
"$field = page_id" )
339 ->where( self::intervalCond( $dbr, $field, $tableStart, $end ) )
340 ->andWhere( [
'page_id' =>
null ] )
342 ->limit( $batchSize )
343 ->caller( __METHOD__ )->fetchFieldValues();
345 $numIds = count( $ids );
348 $dbw->newDeleteQueryBuilder()
349 ->deleteFrom( $table )
350 ->where( [ $field => $ids ] )
351 ->caller( __METHOD__ )->execute();
352 $this->
output(
", $counter" );
353 $tableStart = $ids[$numIds - 1] + 1;
357 }
while ( $numIds >= $batchSize && ( $end ===
null || $tableStart <= $end ) );
359 $this->
output(
" deleted.\n" );
375 private static function intervalCond(
IReadableDatabase $db, $var, $start, $end ) {
376 if ( $start ===
null && $end ===
null ) {
377 return $db->
expr( $var,
'!=',
null );
378 } elseif ( $end ===
null ) {
379 return $db->
expr( $var,
'>=', $start );
380 } elseif ( $start ===
null ) {
381 return $db->
expr( $var,
'<=', $end );
383 return $db->
expr( $var,
'>=', $start )->and( $var,
'<=', $end );
394 $cats = $this->getPossibleCategories( $category );
397 $this->
error(
"Tracking category '$category' is disabled\n" );
401 foreach ( $cats as $cat ) {
402 $this->refreshCategory( clone $builder, $cat );
413 $this->
output(
"Refreshing pages in category '{$category->getText()}'...\n" );
415 $builder->
join(
'categorylinks',
null,
'page_id=cl_from' )
416 ->andWhere( [
'cl_to' => $category->
getDBkey() ] );
417 $this->doRefreshLinks( $builder,
false, [
'cl_timestamp',
'cl_from' ] );
426 private function getPossibleCategories( $categoryKey ) {
428 if ( isset( $cats[$categoryKey] ) ) {
429 return $cats[$categoryKey][
'cats'];
431 $this->
fatalError(
"Unknown tracking category {$categoryKey}\n" );
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
error( $err, $die=0)
Throw an error to the user.
addArg( $arg, $description, $required=true, $multi=false)
Add some args that are needed.
output( $out, $channel=null)
Throw some output to the user.
waitForReplication()
Wait for replica DBs to catch up.
hasOption( $name)
Checks to see if a particular option was set.
getServiceContainer()
Returns the main service container.
getArg( $argId=0, $default=null)
Get an argument.
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.