38 private const REPORTING_INTERVAL = 100;
41 parent::__construct();
43 $this->
addOption(
'verbose',
'Output information about link refresh progress',
false,
false,
'v' );
44 $this->
addOption(
'dfn-only',
'Delete links from nonexistent articles only' );
45 $this->
addOption(
'new-only',
'Only affect articles with just a single edit' );
46 $this->
addOption(
'redirects-only',
'Only fix redirects, not all links' );
47 $this->
addOption(
'touched-only',
'Only fix pages that have been touched after last update' );
48 $this->
addOption(
'e',
'Last page id to refresh',
false,
true );
49 $this->
addOption(
'dfn-chunk-size',
'Maximum number of existent IDs to check per ' .
50 'query, default 100,000',
false,
true );
51 $this->
addOption(
'namespace',
'Only fix pages in this namespace',
false,
true );
52 $this->
addOption(
'category',
'Only fix pages in this category',
false,
true );
53 $this->
addOption(
'tracking-category',
'Only fix pages in this tracking category',
false,
true );
54 $this->
addOption(
'before-timestamp',
'Only fix pages that were last updated before this timestamp',
56 $this->
addArg(
'start',
'Page_id to start from, default 1',
false );
65 $start = (int)$this->
getArg( 0 ) ?:
null;
66 $end = (int)$this->
getOption(
'e' ) ?:
null;
67 $dfnChunkSize = (int)$this->
getOption(
'dfn-chunk-size', 100_000 );
70 $this->deleteLinksFromNonexistent( $start, $end, $this->
getBatchSize(), $dfnChunkSize );
75 $builder = $dbr->newSelectQueryBuilder()
77 ->where( self::intervalCond( $dbr,
'page_id', $start, $end ) )
81 $builder->
andWhere( [
'page_namespace' => (
int)$this->
getOption(
'namespace' ) ] );
84 if ( $this->
hasOption(
'before-timestamp' ) ) {
86 $dbr->expr(
'page_links_updated',
'<', $this->getOption(
'before-timestamp' ) )
87 ->or(
'page_links_updated',
'=',
null )
92 $category = $this->
getOption(
'category' );
93 $title = Title::makeTitleSafe(
NS_CATEGORY, $category );
95 $this->
fatalError(
"'$category' is an invalid category name!\n" );
97 $this->refreshCategory( $builder, $title );
98 } elseif ( $this->
hasOption(
'tracking-category' ) ) {
100 $this->refreshTrackingCategory( $builder, $this->
getOption(
'tracking-category' ) );
103 $redir = $this->
hasOption(
'redirects-only' );
104 $touched = $this->
hasOption(
'touched-only' );
105 $what = $redir ?
'redirects' :
'links';
107 $builder->
andWhere( [
'page_is_new' => 1 ] );
108 $this->
output(
"Refreshing $what from new pages...\n" );
112 'page_touched > page_links_updated OR page_links_updated IS NULL',
115 $this->
output(
"Refreshing $what from pages...\n" );
117 $this->doRefreshLinks( $builder, $redir );
118 if ( !$this->
hasOption(
'namespace' ) ) {
119 $this->deleteLinksFromNonexistent( $start, $end, $this->
getBatchSize(), $dfnChunkSize );
130 private function doRefreshLinks(
132 bool $redirectsOnly =
false,
133 array $indexFields = [
'page_id' ]
136 $this->getHookRunner()->onMaintenanceRefreshLinksInit( $this );
139 $this->output(
"Estimated page count: $estimateCount\n" );
142 $lastIndexes = array_fill_keys( $indexFields, 0 );
143 $selectFields = in_array(
'page_id', $indexFields )
144 ? $indexFields : [
'page_id', ...$indexFields ];
145 $verbose = $this->hasOption(
'verbose' );
148 $batchCond = $dbr->buildComparison(
'>', $lastIndexes );
149 $res = ( clone $builder )->select( $selectFields )
150 ->andWhere( [ $batchCond ] )
151 ->orderBy( $indexFields )
152 ->caller( __METHOD__ )->fetchResultSet();
155 $this->output(
"Refreshing links for {$res->numRows()} pages\n" );
158 foreach ( $res as $row ) {
159 if ( !( ++$i % self::REPORTING_INTERVAL ) ) {
164 $this->
output(
"Refreshing links for page ID {$row->page_id}\n" );
166 self::fixRedirect( $this, $row->page_id );
167 if ( !$redirectsOnly ) {
168 self::fixLinksFromArticle( $row->page_id );
171 if ( $res->numRows() ) {
172 $res->seek( $res->numRows() - 1 );
173 foreach ( $indexFields as $field ) {
174 $lastIndexes[$field] = $res->current()->$field;
178 }
while ( $res->numRows() == $this->getBatchSize() );
199 if ( $page ===
null ) {
204 $content = $page->getContent( RevisionRecord::RAW );
205 if ( $content !==
null ) {
206 $rt = $content->getRedirectTarget();
210 if ( $rt ===
null ) {
213 $dbw->newDeleteQueryBuilder()
214 ->deleteFrom(
'redirect' )
215 ->where( [
'rd_from' => $id ] )
216 ->caller( __METHOD__ )->execute();
219 $page->insertRedirectEntry( $rt );
224 $dbw->newUpdateQueryBuilder()
226 ->set( [
'page_is_redirect' => $fieldValue ] )
227 ->where( [
'page_id' => $id ] )
228 ->caller( __METHOD__ )
237 $services = MediaWikiServices::getInstance();
238 $page = $services->getWikiPageFactory()->newFromID( $id );
241 if ( $page ===
null ) {
248 $page->doSecondaryDataUpdates( [
249 'defer' => DeferredUpdates::POSTSEND,
250 'causeAction' =>
'refresh-links-maintenance',
251 'recursive' =>
false,
253 DeferredUpdates::doUpdates();
267 private function deleteLinksFromNonexistent( $start =
null, $end =
null, $batchSize = 100,
270 $this->waitForReplication();
271 $this->output(
"Deleting illegal entries from the links tables...\n" );
276 $nextStart = $dbr->newSelectQueryBuilder()
277 ->select(
'page_id' )
279 ->where( [ self::intervalCond( $dbr,
'page_id', $start, $end ) ] )
280 ->orderBy(
'page_id' )
281 ->offset( $chunkSize )
282 ->caller( __METHOD__ )->fetchField();
284 if ( $nextStart !==
false ) {
289 $chunkEnd = $nextStart - 1;
295 $fmtStart = $start !==
null ?
"[$start" :
'(-INF';
296 $fmtChunkEnd = $chunkEnd !==
null ?
"$chunkEnd]" :
'INF)';
297 $this->
output(
" Checking interval $fmtStart, $fmtChunkEnd\n" );
298 $this->dfnCheckInterval( $start, $chunkEnd, $batchSize );
302 }
while ( $nextStart !==
false );
311 private function dfnCheckInterval( $start =
null, $end =
null, $batchSize = 100 ) {
317 'pagelinks' =>
'pl_from',
318 'imagelinks' =>
'il_from',
319 'categorylinks' =>
'cl_from',
320 'templatelinks' =>
'tl_from',
321 'externallinks' =>
'el_from',
322 'iwlinks' =>
'iwl_from',
323 'langlinks' =>
'll_from',
324 'redirect' =>
'rd_from',
325 'page_props' =>
'pp_page',
328 foreach ( $linksTables as $table => $field ) {
329 $this->
output(
" $table: 0" );
330 $tableStart = $start;
333 $ids = $dbr->newSelectQueryBuilder()
337 ->leftJoin(
'page',
null,
"$field = page_id" )
338 ->where( self::intervalCond( $dbr, $field, $tableStart, $end ) )
339 ->andWhere( [
'page_id' =>
null ] )
341 ->limit( $batchSize )
342 ->caller( __METHOD__ )->fetchFieldValues();
344 $numIds = count( $ids );
347 $dbw->newDeleteQueryBuilder()
348 ->deleteFrom( $table )
349 ->where( [ $field => $ids ] )
350 ->caller( __METHOD__ )->execute();
351 $this->
output(
", $counter" );
352 $tableStart = $ids[$numIds - 1] + 1;
356 }
while ( $numIds >= $batchSize && ( $end ===
null || $tableStart <= $end ) );
358 $this->
output(
" deleted.\n" );
374 private static function intervalCond(
IReadableDatabase $db, $var, $start, $end ) {
375 if ( $start ===
null && $end ===
null ) {
376 return $db->
expr( $var,
'!=',
null );
377 } elseif ( $end ===
null ) {
378 return $db->
expr( $var,
'>=', $start );
379 } elseif ( $start ===
null ) {
380 return $db->
expr( $var,
'<=', $end );
382 return $db->
expr( $var,
'>=', $start )->and( $var,
'<=', $end );
393 $cats = $this->getPossibleCategories( $category );
396 $this->
error(
"Tracking category '$category' is disabled\n" );
400 foreach ( $cats as $cat ) {
401 $this->refreshCategory( clone $builder, $cat );
412 $this->
output(
"Refreshing pages in category '{$category->getText()}'...\n" );
414 $builder->
join(
'categorylinks',
null,
'page_id=cl_from' )
415 ->andWhere( [
'cl_to' => $category->
getDBkey() ] );
416 $this->doRefreshLinks( $builder,
false, [
'cl_timestamp',
'cl_from' ] );
425 private function getPossibleCategories( $categoryKey ) {
427 if ( isset( $cats[$categoryKey] ) ) {
428 return $cats[$categoryKey][
'cats'];
430 $this->
fatalError(
"Unknown tracking category {$categoryKey}\n" );
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
error( $err, $die=0)
Throw an error to the user.
addArg( $arg, $description, $required=true, $multi=false)
Add some args that are needed.
output( $out, $channel=null)
Throw some output to the user.
waitForReplication()
Wait for replica DBs to catch up.
hasOption( $name)
Checks to see if a particular option was set.
getServiceContainer()
Returns the main service container.
getArg( $argId=0, $default=null)
Get an argument.
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.