MediaWiki master
refreshLinks.php
Go to the documentation of this file.
1<?php
29
30require_once __DIR__ . '/Maintenance.php';
31
38 private const REPORTING_INTERVAL = 100;
39
40 public function __construct() {
41 parent::__construct();
42 $this->addDescription( 'Refresh link tables' );
43 $this->addOption( 'verbose', 'Output information about link refresh progress', false, false, 'v' );
44 $this->addOption( 'dfn-only', 'Delete links from nonexistent articles only' );
45 $this->addOption( 'new-only', 'Only affect articles with just a single edit' );
46 $this->addOption( 'redirects-only', 'Only fix redirects, not all links' );
47 $this->addOption( 'touched-only', 'Only fix pages that have been touched after last update' );
48 $this->addOption( 'e', 'Last page id to refresh', false, true );
49 $this->addOption( 'dfn-chunk-size', 'Maximum number of existent IDs to check per ' .
50 'query, default 100,000', false, true );
51 $this->addOption( 'namespace', 'Only fix pages in this namespace', false, true );
52 $this->addOption( 'category', 'Only fix pages in this category', false, true );
53 $this->addOption( 'tracking-category', 'Only fix pages in this tracking category', false, true );
54 $this->addOption( 'before-timestamp', 'Only fix pages that were last updated before this timestamp',
55 false, true );
56 $this->addArg( 'start', 'Page_id to start from, default 1', false );
57 $this->setBatchSize( 100 );
58 }
59
60 public function execute() {
61 // Note that there is a difference between not specifying the start
62 // and end IDs and using the minimum and maximum values from the page
63 // table. In the latter case, deleteLinksFromNonexistent() will not
64 // delete entries for nonexistent IDs that fall outside the range.
65 $start = (int)$this->getArg( 0 ) ?: null;
66 $end = (int)$this->getOption( 'e' ) ?: null;
67 $dfnChunkSize = (int)$this->getOption( 'dfn-chunk-size', 100_000 );
68
69 if ( $this->hasOption( 'dfn-only' ) ) {
70 $this->deleteLinksFromNonexistent( $start, $end, $this->getBatchSize(), $dfnChunkSize );
71 return;
72 }
73
74 $dbr = $this->getDB( DB_REPLICA, [ 'vslow' ] );
75 $builder = $dbr->newSelectQueryBuilder()
76 ->from( 'page' )
77 ->where( self::intervalCond( $dbr, 'page_id', $start, $end ) )
78 ->limit( $this->getBatchSize() );
79
80 if ( $this->hasOption( 'namespace' ) ) {
81 $builder->andWhere( [ 'page_namespace' => (int)$this->getOption( 'namespace' ) ] );
82 }
83
84 if ( $this->hasOption( 'before-timestamp' ) ) {
85 $builder->andWhere(
86 $dbr->expr( 'page_links_updated', '<', $this->getOption( 'before-timestamp' ) )
87 ->or( 'page_links_updated', '=', null )
88 );
89 }
90
91 if ( $this->hasOption( 'category' ) ) {
92 $category = $this->getOption( 'category' );
93 $title = Title::makeTitleSafe( NS_CATEGORY, $category );
94 if ( !$title ) {
95 $this->fatalError( "'$category' is an invalid category name!\n" );
96 }
97 $this->refreshCategory( $builder, $title );
98 } elseif ( $this->hasOption( 'tracking-category' ) ) {
99 // See TrackingCategories::CORE_TRACKING_CATEGORIES for tracking category keys defined by core
100 $this->refreshTrackingCategory( $builder, $this->getOption( 'tracking-category' ) );
101 } else {
102 $new = $this->hasOption( 'new-only' );
103 $redir = $this->hasOption( 'redirects-only' );
104 $touched = $this->hasOption( 'touched-only' );
105 $what = $redir ? 'redirects' : 'links';
106 if ( $new ) {
107 $builder->andWhere( [ 'page_is_new' => 1 ] );
108 $this->output( "Refreshing $what from new pages...\n" );
109 } else {
110 if ( $touched ) {
111 $builder->andWhere( [
112 'page_touched > page_links_updated OR page_links_updated IS NULL',
113 ] );
114 }
115 $this->output( "Refreshing $what from pages...\n" );
116 }
117 $this->doRefreshLinks( $builder, $redir );
118 if ( !$this->hasOption( 'namespace' ) ) {
119 $this->deleteLinksFromNonexistent( $start, $end, $this->getBatchSize(), $dfnChunkSize );
120 }
121 }
122 }
123
130 private function doRefreshLinks(
131 SelectQueryBuilder $builder,
132 bool $redirectsOnly = false,
133 array $indexFields = [ 'page_id' ]
134 ) {
135 // Give extensions a chance to optimize settings
136 $this->getHookRunner()->onMaintenanceRefreshLinksInit( $this );
137
138 $estimateCount = $builder->estimateRowCount();
139 $this->output( "Estimated page count: $estimateCount\n" );
140
141 $i = 0;
142 $lastIndexes = array_fill_keys( $indexFields, 0 );
143 $selectFields = in_array( 'page_id', $indexFields )
144 ? $indexFields : [ 'page_id', ...$indexFields ];
145 $verbose = $this->hasOption( 'verbose' );
146 $dbr = $this->getDB( DB_REPLICA, [ 'vslow' ] );
147 do {
148 $batchCond = $dbr->buildComparison( '>', $lastIndexes );
149 $res = ( clone $builder )->select( $selectFields )
150 ->andWhere( [ $batchCond ] )
151 ->orderBy( $indexFields )
152 ->caller( __METHOD__ )->fetchResultSet();
153
154 if ( $verbose ) {
155 $this->output( "Refreshing links for {$res->numRows()} pages\n" );
156 }
157
158 foreach ( $res as $row ) {
159 if ( !( ++$i % self::REPORTING_INTERVAL ) ) {
160 $this->output( "$i\n" );
161 $this->waitForReplication();
162 }
163 if ( $verbose ) {
164 $this->output( "Refreshing links for page ID {$row->page_id}\n" );
165 }
166 self::fixRedirect( $this, $row->page_id );
167 if ( !$redirectsOnly ) {
168 self::fixLinksFromArticle( $row->page_id );
169 }
170 }
171 if ( $res->numRows() ) {
172 $res->seek( $res->numRows() - 1 );
173 foreach ( $indexFields as $field ) {
174 $lastIndexes[$field] = $res->current()->$field;
175 }
176 }
177
178 } while ( $res->numRows() == $this->getBatchSize() );
179 }
180
195 public static function fixRedirect( Maintenance $maint, $id ) {
196 $page = $maint->getServiceContainer()->getWikiPageFactory()->newFromID( $id );
197
198 // In case the page just got deleted.
199 if ( $page === null ) {
200 return;
201 }
202
203 $rt = null;
204 $content = $page->getContent( RevisionRecord::RAW );
205 if ( $content !== null ) {
206 $rt = $content->getRedirectTarget();
207 }
208
209 $dbw = $maint->getDB( DB_PRIMARY );
210 if ( $rt === null ) {
211 // The page is not a redirect
212 // Delete any redirect table entry for it
213 $dbw->newDeleteQueryBuilder()
214 ->deleteFrom( 'redirect' )
215 ->where( [ 'rd_from' => $id ] )
216 ->caller( __METHOD__ )->execute();
217 $fieldValue = 0;
218 } else {
219 $page->insertRedirectEntry( $rt );
220 $fieldValue = 1;
221 }
222
223 // Update the page table to be sure it is an a consistent state
224 $dbw->newUpdateQueryBuilder()
225 ->update( 'page' )
226 ->set( [ 'page_is_redirect' => $fieldValue ] )
227 ->where( [ 'page_id' => $id ] )
228 ->caller( __METHOD__ )
229 ->execute();
230 }
231
236 public static function fixLinksFromArticle( $id ) {
237 $services = MediaWikiServices::getInstance();
238 $page = $services->getWikiPageFactory()->newFromID( $id );
239
240 // In case the page just got deleted.
241 if ( $page === null ) {
242 return;
243 }
244
245 // Defer updates to post-send but then immediately execute deferred updates;
246 // this is the simplest way to run all updates immediately (including updates
247 // scheduled by other updates).
248 $page->doSecondaryDataUpdates( [
249 'defer' => DeferredUpdates::POSTSEND,
250 'causeAction' => 'refresh-links-maintenance',
251 'recursive' => false,
252 ] );
253 DeferredUpdates::doUpdates();
254 }
255
267 private function deleteLinksFromNonexistent( $start = null, $end = null, $batchSize = 100,
268 $chunkSize = 100_000
269 ) {
270 $this->waitForReplication();
271 $this->output( "Deleting illegal entries from the links tables...\n" );
272 $dbr = $this->getDB( DB_REPLICA, [ 'vslow' ] );
273 do {
274 // Find the start of the next chunk. This is based only
275 // on existent page_ids.
276 $nextStart = $dbr->newSelectQueryBuilder()
277 ->select( 'page_id' )
278 ->from( 'page' )
279 ->where( [ self::intervalCond( $dbr, 'page_id', $start, $end ) ] )
280 ->orderBy( 'page_id' )
281 ->offset( $chunkSize )
282 ->caller( __METHOD__ )->fetchField();
283
284 if ( $nextStart !== false ) {
285 // To find the end of the current chunk, subtract one.
286 // This will serve to limit the number of rows scanned in
287 // dfnCheckInterval(), per query, to at most the sum of
288 // the chunk size and deletion batch size.
289 $chunkEnd = $nextStart - 1;
290 } else {
291 // This is the last chunk. Check all page_ids up to $end.
292 $chunkEnd = $end;
293 }
294
295 $fmtStart = $start !== null ? "[$start" : '(-INF';
296 $fmtChunkEnd = $chunkEnd !== null ? "$chunkEnd]" : 'INF)';
297 $this->output( " Checking interval $fmtStart, $fmtChunkEnd\n" );
298 $this->dfnCheckInterval( $start, $chunkEnd, $batchSize );
299
300 $start = $nextStart;
301
302 } while ( $nextStart !== false );
303 }
304
311 private function dfnCheckInterval( $start = null, $end = null, $batchSize = 100 ) {
312 $dbw = $this->getPrimaryDB();
313 $dbr = $this->getDB( DB_REPLICA, [ 'vslow' ] );
314
315 $linksTables = [
316 // table name => page_id field
317 'pagelinks' => 'pl_from',
318 'imagelinks' => 'il_from',
319 'categorylinks' => 'cl_from',
320 'templatelinks' => 'tl_from',
321 'externallinks' => 'el_from',
322 'iwlinks' => 'iwl_from',
323 'langlinks' => 'll_from',
324 'redirect' => 'rd_from',
325 'page_props' => 'pp_page',
326 ];
327
328 foreach ( $linksTables as $table => $field ) {
329 $this->output( " $table: 0" );
330 $tableStart = $start;
331 $counter = 0;
332 do {
333 $ids = $dbr->newSelectQueryBuilder()
334 ->select( $field )
335 ->distinct()
336 ->from( $table )
337 ->leftJoin( 'page', null, "$field = page_id" )
338 ->where( self::intervalCond( $dbr, $field, $tableStart, $end ) )
339 ->andWhere( [ 'page_id' => null ] )
340 ->orderBy( $field )
341 ->limit( $batchSize )
342 ->caller( __METHOD__ )->fetchFieldValues();
343
344 $numIds = count( $ids );
345 if ( $numIds ) {
346 $counter += $numIds;
347 $dbw->newDeleteQueryBuilder()
348 ->deleteFrom( $table )
349 ->where( [ $field => $ids ] )
350 ->caller( __METHOD__ )->execute();
351 $this->output( ", $counter" );
352 $tableStart = $ids[$numIds - 1] + 1;
353 $this->waitForReplication();
354 }
355
356 } while ( $numIds >= $batchSize && ( $end === null || $tableStart <= $end ) );
357
358 $this->output( " deleted.\n" );
359 }
360 }
361
374 private static function intervalCond( IReadableDatabase $db, $var, $start, $end ) {
375 if ( $start === null && $end === null ) {
376 return $db->expr( $var, '!=', null );
377 } elseif ( $end === null ) {
378 return $db->expr( $var, '>=', $start );
379 } elseif ( $start === null ) {
380 return $db->expr( $var, '<=', $end );
381 } else {
382 return $db->expr( $var, '>=', $start )->and( $var, '<=', $end );
383 }
384 }
385
392 private function refreshTrackingCategory( SelectQueryBuilder $builder, $category ) {
393 $cats = $this->getPossibleCategories( $category );
394
395 if ( !$cats ) {
396 $this->error( "Tracking category '$category' is disabled\n" );
397 // Output to stderr but don't bail out.
398 }
399
400 foreach ( $cats as $cat ) {
401 $this->refreshCategory( clone $builder, $cat );
402 }
403 }
404
411 private function refreshCategory( SelectQueryBuilder $builder, LinkTarget $category ) {
412 $this->output( "Refreshing pages in category '{$category->getText()}'...\n" );
413
414 $builder->join( 'categorylinks', null, 'page_id=cl_from' )
415 ->andWhere( [ 'cl_to' => $category->getDBkey() ] );
416 $this->doRefreshLinks( $builder, false, [ 'cl_timestamp', 'cl_from' ] );
417 }
418
425 private function getPossibleCategories( $categoryKey ) {
426 $cats = $this->getServiceContainer()->getTrackingCategories()->getTrackingCategories();
427 if ( isset( $cats[$categoryKey] ) ) {
428 return $cats[$categoryKey]['cats'];
429 }
430 $this->fatalError( "Unknown tracking category {$categoryKey}\n" );
431 }
432}
433
434$maintClass = RefreshLinks::class;
435require_once RUN_MAINTENANCE_IF_MAIN;
getDB()
const NS_CATEGORY
Definition Defines.php:79
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
error( $err, $die=0)
Throw an error to the user.
addArg( $arg, $description, $required=true, $multi=false)
Add some args that are needed.
output( $out, $channel=null)
Throw some output to the user.
waitForReplication()
Wait for replica DBs to catch up.
hasOption( $name)
Checks to see if a particular option was set.
getServiceContainer()
Returns the main service container.
getBatchSize()
Returns batch size.
getArg( $argId=0, $default=null)
Get an argument.
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
setBatchSize( $s=0)
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
Defer callable updates to run later in the PHP process.
Service locator for MediaWiki core services.
Page revision base class.
Represents a title within MediaWiki.
Definition Title.php:78
join( $table, $alias=null, $conds=[])
Inner join a table or group of tables.
Build SELECT queries with a fluent interface.
estimateRowCount()
Estimate the number of rows in dataset.
andWhere( $conds)
Add conditions to the query.
from( $table, $alias=null)
Add a single table to the SELECT query.
Represents the target of a wiki link.
getDBkey()
Get the main part of the link target, in canonical database form.
A database connection without write operations.
expr(string $field, string $op, $value)
See Expression::__construct()
const DB_REPLICA
Definition defines.php:26
const DB_PRIMARY
Definition defines.php:28