MediaWiki  master
rebuildrecentchanges.php
Go to the documentation of this file.
1 <?php
26 require_once __DIR__ . '/Maintenance.php';
27 
30 
38  private $cutoffFrom;
40  private $cutoffTo;
41 
42  public function __construct() {
43  parent::__construct();
44  $this->addDescription( 'Rebuild recent changes' );
45 
46  $this->addOption(
47  'from',
48  "Only rebuild rows in requested time range (in YYYYMMDDHHMMSS format)",
49  false,
50  true
51  );
52  $this->addOption(
53  'to',
54  "Only rebuild rows in requested time range (in YYYYMMDDHHMMSS format)",
55  false,
56  true
57  );
58  $this->setBatchSize( 200 );
59  }
60 
61  public function execute() {
62  if (
63  ( $this->hasOption( 'from' ) && !$this->hasOption( 'to' ) ) ||
64  ( !$this->hasOption( 'from' ) && $this->hasOption( 'to' ) )
65  ) {
66  $this->fatalError( "Both 'from' and 'to' must be given, or neither" );
67  }
68 
69  $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
70  $this->rebuildRecentChangesTablePass1( $lbFactory );
71  $this->rebuildRecentChangesTablePass2( $lbFactory );
72  $this->rebuildRecentChangesTablePass3( $lbFactory );
73  $this->rebuildRecentChangesTablePass4( $lbFactory );
74  $this->rebuildRecentChangesTablePass5( $lbFactory );
75  if ( !( $this->hasOption( 'from' ) && $this->hasOption( 'to' ) ) ) {
76  $this->purgeFeeds();
77  }
78  $this->output( "Done.\n" );
79  }
80 
84  private function rebuildRecentChangesTablePass1( ILBFactory $lbFactory ) {
85  $dbw = $this->getDB( DB_MASTER );
86  $commentStore = CommentStore::getStore();
87 
88  if ( $this->hasOption( 'from' ) && $this->hasOption( 'to' ) ) {
89  $this->cutoffFrom = wfTimestamp( TS_UNIX, $this->getOption( 'from' ) );
90  $this->cutoffTo = wfTimestamp( TS_UNIX, $this->getOption( 'to' ) );
91 
92  $sec = $this->cutoffTo - $this->cutoffFrom;
93  $days = $sec / 24 / 3600;
94  $this->output( "Rebuilding range of $sec seconds ($days days)\n" );
95  } else {
96  global $wgRCMaxAge;
97 
98  $days = $wgRCMaxAge / 24 / 3600;
99  $this->output( "Rebuilding \$wgRCMaxAge=$wgRCMaxAge seconds ($days days)\n" );
100 
101  $this->cutoffFrom = time() - $wgRCMaxAge;
102  $this->cutoffTo = time();
103  }
104 
105  $this->output( "Clearing recentchanges table for time range...\n" );
106  $rcids = $dbw->selectFieldValues(
107  'recentchanges',
108  'rc_id',
109  [
110  'rc_timestamp > ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ),
111  'rc_timestamp < ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) )
112  ]
113  );
114  foreach ( array_chunk( $rcids, $this->getBatchSize() ) as $rcidBatch ) {
115  $dbw->delete( 'recentchanges', [ 'rc_id' => $rcidBatch ], __METHOD__ );
116  $lbFactory->waitForReplication();
117  }
118 
119  $this->output( "Loading from page and revision tables...\n" );
120 
121  $commentQuery = $commentStore->getJoin( 'rev_comment' );
122  $actorQuery = ActorMigration::newMigration()->getJoin( 'rev_user' );
123  $res = $dbw->select(
124  [ 'revision', 'page' ] + $commentQuery['tables'] + $actorQuery['tables'],
125  [
126  'rev_timestamp',
127  'rev_minor_edit',
128  'rev_id',
129  'rev_deleted',
130  'page_namespace',
131  'page_title',
132  'page_is_new',
133  'page_id'
134  ] + $commentQuery['fields'] + $actorQuery['fields'],
135  [
136  'rev_timestamp > ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ),
137  'rev_timestamp < ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) )
138  ],
139  __METHOD__,
140  [ 'ORDER BY' => 'rev_timestamp DESC' ],
141  [
142  'page' => [ 'JOIN', 'rev_page=page_id' ],
143  ] + $commentQuery['joins'] + $actorQuery['joins']
144  );
145 
146  $this->output( "Inserting from page and revision tables...\n" );
147  $inserted = 0;
148  $actorMigration = ActorMigration::newMigration();
149  foreach ( $res as $row ) {
150  $comment = $commentStore->getComment( 'rev_comment', $row );
151  $user = User::newFromAnyId( $row->rev_user, $row->rev_user_text, $row->rev_actor );
152  $dbw->insert(
153  'recentchanges',
154  [
155  'rc_timestamp' => $row->rev_timestamp,
156  'rc_namespace' => $row->page_namespace,
157  'rc_title' => $row->page_title,
158  'rc_minor' => $row->rev_minor_edit,
159  'rc_bot' => 0,
160  'rc_new' => $row->page_is_new,
161  'rc_cur_id' => $row->page_id,
162  'rc_this_oldid' => $row->rev_id,
163  'rc_last_oldid' => 0, // is this ok?
164  'rc_type' => $row->page_is_new ? RC_NEW : RC_EDIT,
165  'rc_source' => $row->page_is_new ? RecentChange::SRC_NEW : RecentChange::SRC_EDIT,
166  'rc_deleted' => $row->rev_deleted
167  ] + $commentStore->insert( $dbw, 'rc_comment', $comment )
168  + $actorMigration->getInsertValues( $dbw, 'rc_user', $user ),
169  __METHOD__
170  );
171  if ( ( ++$inserted % $this->getBatchSize() ) == 0 ) {
172  $lbFactory->waitForReplication();
173  }
174  }
175  }
176 
181  private function rebuildRecentChangesTablePass2( ILBFactory $lbFactory ) {
182  $dbw = $this->getDB( DB_MASTER );
183 
184  $this->output( "Updating links and size differences...\n" );
185 
186  # Fill in the rc_last_oldid field, which points to the previous edit
187  $res = $dbw->select(
188  'recentchanges',
189  [ 'rc_cur_id', 'rc_this_oldid', 'rc_timestamp' ],
190  [
191  "rc_timestamp > " . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ),
192  "rc_timestamp < " . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) )
193  ],
194  __METHOD__,
195  [ 'ORDER BY' => 'rc_cur_id,rc_timestamp' ]
196  );
197 
198  $lastCurId = 0;
199  $lastOldId = 0;
200  $lastSize = null;
201  $updated = 0;
202  foreach ( $res as $obj ) {
203  $new = 0;
204 
205  if ( $obj->rc_cur_id != $lastCurId ) {
206  # Switch! Look up the previous last edit, if any
207  $lastCurId = intval( $obj->rc_cur_id );
208  $emit = $obj->rc_timestamp;
209 
210  $row = $dbw->selectRow(
211  'revision',
212  [ 'rev_id', 'rev_len' ],
213  [ 'rev_page' => $lastCurId, "rev_timestamp < " . $dbw->addQuotes( $emit ) ],
214  __METHOD__,
215  [ 'ORDER BY' => 'rev_timestamp DESC' ]
216  );
217  if ( $row ) {
218  $lastOldId = intval( $row->rev_id );
219  # Grab the last text size if available
220  $lastSize = !is_null( $row->rev_len ) ? intval( $row->rev_len ) : null;
221  } else {
222  # No previous edit
223  $lastOldId = 0;
224  $lastSize = 0;
225  $new = 1; // probably true
226  }
227  }
228 
229  if ( $lastCurId == 0 ) {
230  $this->output( "Uhhh, something wrong? No curid\n" );
231  } else {
232  # Grab the entry's text size
233  $size = (int)$dbw->selectField(
234  'revision',
235  'rev_len',
236  [ 'rev_id' => $obj->rc_this_oldid ],
237  __METHOD__
238  );
239 
240  $dbw->update(
241  'recentchanges',
242  [
243  'rc_last_oldid' => $lastOldId,
244  'rc_new' => $new,
245  'rc_type' => $new ? RC_NEW : RC_EDIT,
246  'rc_source' => $new === 1 ? RecentChange::SRC_NEW : RecentChange::SRC_EDIT,
247  'rc_old_len' => $lastSize,
248  'rc_new_len' => $size,
249  ],
250  [
251  'rc_cur_id' => $lastCurId,
252  'rc_this_oldid' => $obj->rc_this_oldid,
253  'rc_timestamp' => $obj->rc_timestamp // index usage
254  ],
255  __METHOD__
256  );
257 
258  $lastOldId = intval( $obj->rc_this_oldid );
259  $lastSize = $size;
260 
261  if ( ( ++$updated % $this->getBatchSize() ) == 0 ) {
262  $lbFactory->waitForReplication();
263  }
264  }
265  }
266  }
267 
271  private function rebuildRecentChangesTablePass3( ILBFactory $lbFactory ) {
273 
274  $dbw = $this->getDB( DB_MASTER );
275  $commentStore = CommentStore::getStore();
276  $nonRCLogs = array_merge( array_keys( $wgLogRestrictions ),
277  array_keys( $wgFilterLogTypes ),
278  [ 'create' ] );
279 
280  $this->output( "Loading from user and logging tables...\n" );
281 
282  $commentQuery = $commentStore->getJoin( 'log_comment' );
283  $actorQuery = ActorMigration::newMigration()->getJoin( 'log_user' );
284  $res = $dbw->select(
285  [ 'logging' ] + $commentQuery['tables'] + $actorQuery['tables'],
286  [
287  'log_timestamp',
288  'log_namespace',
289  'log_title',
290  'log_page',
291  'log_type',
292  'log_action',
293  'log_id',
294  'log_params',
295  'log_deleted'
296  ] + $commentQuery['fields'] + $actorQuery['fields'],
297  [
298  'log_timestamp > ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ),
299  'log_timestamp < ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ),
300  // Some logs don't go in RC since they are private, or are included in the filterable log types.
301  'log_type' => array_diff( LogPage::validTypes(), $nonRCLogs ),
302  ],
303  __METHOD__,
304  [ 'ORDER BY' => 'log_timestamp DESC' ],
305  $commentQuery['joins'] + $actorQuery['joins']
306  );
307 
308  $field = $dbw->fieldInfo( 'recentchanges', 'rc_cur_id' );
309 
310  $inserted = 0;
311  $actorMigration = ActorMigration::newMigration();
312  foreach ( $res as $row ) {
313  $comment = $commentStore->getComment( 'log_comment', $row );
314  $user = User::newFromAnyId( $row->log_user, $row->log_user_text, $row->log_actor );
315  $dbw->insert(
316  'recentchanges',
317  [
318  'rc_timestamp' => $row->log_timestamp,
319  'rc_namespace' => $row->log_namespace,
320  'rc_title' => $row->log_title,
321  'rc_minor' => 0,
322  'rc_bot' => 0,
323  'rc_patrolled' => 1,
324  'rc_new' => 0,
325  'rc_this_oldid' => 0,
326  'rc_last_oldid' => 0,
327  'rc_type' => RC_LOG,
328  'rc_source' => RecentChange::SRC_LOG,
329  'rc_cur_id' => $field->isNullable()
330  ? $row->log_page
331  : (int)$row->log_page, // NULL => 0,
332  'rc_log_type' => $row->log_type,
333  'rc_log_action' => $row->log_action,
334  'rc_logid' => $row->log_id,
335  'rc_params' => $row->log_params,
336  'rc_deleted' => $row->log_deleted
337  ] + $commentStore->insert( $dbw, 'rc_comment', $comment )
338  + $actorMigration->getInsertValues( $dbw, 'rc_user', $user ),
339  __METHOD__
340  );
341 
342  if ( ( ++$inserted % $this->getBatchSize() ) == 0 ) {
343  $lbFactory->waitForReplication();
344  }
345  }
346  }
347 
351  private function rebuildRecentChangesTablePass4( ILBFactory $lbFactory ) {
353 
354  $dbw = $this->getDB( DB_MASTER );
355 
356  $userQuery = User::getQueryInfo();
357 
358  # @FIXME: recognize other bot account groups (not the same as users with 'bot' rights)
359  # @NOTE: users with 'bot' rights choose when edits are bot edits or not. That information
360  # may be lost at this point (aside from joining on the patrol log table entries).
361  $botgroups = [ 'bot' ];
362  $autopatrolgroups = $wgUseRCPatrol ? User::getGroupsWithPermission( 'autopatrol' ) : [];
363 
364  # Flag our recent bot edits
365  if ( $botgroups ) {
366  $this->output( "Flagging bot account edits...\n" );
367 
368  # Find all users that are bots
369  $res = $dbw->select(
370  array_merge( [ 'user_groups' ], $userQuery['tables'] ),
371  $userQuery['fields'],
372  [ 'ug_group' => $botgroups ],
373  __METHOD__,
374  [ 'DISTINCT' ],
375  [ 'user_groups' => [ 'JOIN', 'user_id = ug_user' ] ] + $userQuery['joins']
376  );
377 
378  $botusers = [];
379  foreach ( $res as $obj ) {
380  $botusers[] = User::newFromRow( $obj );
381  }
382 
383  # Fill in the rc_bot field
384  if ( $botusers ) {
385  $actorQuery = ActorMigration::newMigration()->getWhere( $dbw, 'rc_user', $botusers, false );
386  $rcids = [];
387  foreach ( $actorQuery['orconds'] as $cond ) {
388  $rcids = array_merge( $rcids, $dbw->selectFieldValues(
389  [ 'recentchanges' ] + $actorQuery['tables'],
390  'rc_id',
391  [
392  "rc_timestamp > " . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ),
393  "rc_timestamp < " . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ),
394  $cond,
395  ],
396  __METHOD__,
397  [],
398  $actorQuery['joins']
399  ) );
400  }
401  $rcids = array_values( array_unique( $rcids ) );
402 
403  foreach ( array_chunk( $rcids, $this->getBatchSize() ) as $rcidBatch ) {
404  $dbw->update(
405  'recentchanges',
406  [ 'rc_bot' => 1 ],
407  [ 'rc_id' => $rcidBatch ],
408  __METHOD__
409  );
410  $lbFactory->waitForReplication();
411  }
412  }
413  }
414 
415  # Flag our recent autopatrolled edits
416  if ( !$wgMiserMode && $autopatrolgroups ) {
417  $patrolusers = [];
418 
419  $this->output( "Flagging auto-patrolled edits...\n" );
420 
421  # Find all users in RC with autopatrol rights
422  $res = $dbw->select(
423  array_merge( [ 'user_groups' ], $userQuery['tables'] ),
424  $userQuery['fields'],
425  [ 'ug_group' => $autopatrolgroups ],
426  __METHOD__,
427  [ 'DISTINCT' ],
428  [ 'user_groups' => [ 'JOIN', 'user_id = ug_user' ] ] + $userQuery['joins']
429  );
430 
431  foreach ( $res as $obj ) {
432  $patrolusers[] = User::newFromRow( $obj );
433  }
434 
435  # Fill in the rc_patrolled field
436  if ( $patrolusers ) {
437  $actorQuery = ActorMigration::newMigration()->getWhere( $dbw, 'rc_user', $patrolusers, false );
438  foreach ( $actorQuery['orconds'] as $cond ) {
439  $dbw->update(
440  'recentchanges',
441  [ 'rc_patrolled' => 1 ],
442  [
443  $cond,
444  'rc_timestamp > ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ),
445  'rc_timestamp < ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ),
446  ],
447  __METHOD__
448  );
449  $lbFactory->waitForReplication();
450  }
451  }
452  }
453  }
454 
459  private function rebuildRecentChangesTablePass5( ILBFactory $lbFactory ) {
460  $dbw = wfGetDB( DB_MASTER );
461 
462  $this->output( "Removing duplicate revision and logging entries...\n" );
463 
464  $res = $dbw->select(
465  [ 'logging', 'log_search' ],
466  [ 'ls_value', 'ls_log_id' ],
467  [
468  'ls_log_id = log_id',
469  'ls_field' => 'associated_rev_id',
470  'log_type' => 'upload',
471  'log_timestamp > ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ),
472  'log_timestamp < ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ),
473  ],
474  __METHOD__
475  );
476 
477  $updates = 0;
478  foreach ( $res as $obj ) {
479  $rev_id = $obj->ls_value;
480  $log_id = $obj->ls_log_id;
481 
482  // Mark the logging row as having an associated rev id
483  $dbw->update(
484  'recentchanges',
485  /*SET*/ [ 'rc_this_oldid' => $rev_id ],
486  /*WHERE*/ [ 'rc_logid' => $log_id ],
487  __METHOD__
488  );
489 
490  // Delete the revision row
491  $dbw->delete(
492  'recentchanges',
493  /*WHERE*/ [ 'rc_this_oldid' => $rev_id, 'rc_logid' => 0 ],
494  __METHOD__
495  );
496 
497  if ( ( ++$updates % $this->getBatchSize() ) == 0 ) {
498  $lbFactory->waitForReplication();
499  }
500  }
501  }
502 
506  private function purgeFeeds() {
507  global $wgFeedClasses;
508 
509  $this->output( "Deleting feed timestamps.\n" );
510 
511  $wanCache = MediaWikiServices::getInstance()->getMainWANObjectCache();
512  foreach ( $wgFeedClasses as $feed => $className ) {
513  $wanCache->delete( $wanCache->makeKey( 'rcfeed', $feed, 'timestamp' ) ); # Good enough for now.
514  }
515  }
516 }
517 
519 require_once RUN_MAINTENANCE_IF_MAIN;
Maintenance script that rebuilds recent changes from scratch.
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
getOption( $name, $default=null)
Get an option, or return the default.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: maintenance.txt:39
wfGetDB( $db, $groups=[], $wiki=false)
Get a Database object.
rebuildRecentChangesTablePass2(ILBFactory $lbFactory)
Rebuild pass 2: Enhance entries for page revisions with references to the previous revision (rc_last_...
int $cutoffTo
UNIX timestamp.
waitForReplication(array $opts=[])
Waits for the replica DBs to catch up to the current master position.
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency MediaWikiServices
Definition: injection.txt:23
setBatchSize( $s=0)
Set the batch size.
static getQueryInfo()
Return the tables, fields, and join conditions to be selected to create a new user object...
Definition: User.php:5648
hasOption( $name)
Checks to see if a particular option exists.
require_once RUN_MAINTENANCE_IF_MAIN
Definition: maintenance.txt:50
$wgLogRestrictions
This restricts log access to those who have a certain right Users without this will not see it in the...
const DB_MASTER
Definition: defines.php:26
$wgRCMaxAge
Recentchanges items are periodically purged; entries older than this many seconds will go...
purgeFeeds()
Purge cached feeds in $wanCache.
rebuildRecentChangesTablePass1(ILBFactory $lbFactory)
Rebuild pass 1: Insert recentchanges entries for page revisions.
rebuildRecentChangesTablePass5(ILBFactory $lbFactory)
Rebuild pass 5: Delete duplicate entries where we generate both a page revision and a log entry for a...
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
static newMigration()
Static constructor.
static getGroupsWithPermission( $role)
Get all the groups who have a given permission.
Definition: User.php:5044
$res
Definition: database.txt:21
$wgMiserMode
Disable database-intensive features.
addDescription( $text)
Set the description text.
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that When $user is not null
Definition: hooks.txt:780
rebuildRecentChangesTablePass4(ILBFactory $lbFactory)
Rebuild pass 4: Mark bot and autopatrolled entries.
getDB( $db, $groups=[], $wiki=false)
Returns a database to be used by current maintenance script.
$wgUseRCPatrol
Use RC Patrolling to check for vandalism (from recent changes and watchlists) New pages and new files...
output( $out, $channel=null)
Throw some output to the user.
int $cutoffFrom
UNIX timestamp.
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
static newFromAnyId( $userId, $userName, $actorId)
Static factory method for creation from an ID, name, and/or actor ID.
Definition: User.php:678
An interface for generating database load balancers.
Definition: ILBFactory.php:33
static getStore()
you have access to all of the normal MediaWiki so you can get a DB use the etc For full docs on the Maintenance class
Definition: maintenance.txt:52
static newFromRow( $row, $data=null)
Create a new user object from a user row.
Definition: User.php:774
getBatchSize()
Returns batch size.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
const RC_NEW
Definition: Defines.php:143
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
rebuildRecentChangesTablePass3(ILBFactory $lbFactory)
Rebuild pass 3: Insert recentchanges entries for action logs.
static validTypes()
Get the list of valid log types.
Definition: LogPage.php:198
return true to allow those checks to and false if checking is done & $user
Definition: hooks.txt:1473
$wgFilterLogTypes
Show/hide links on Special:Log will be shown for these log types.
const RC_EDIT
Definition: Defines.php:142
const RC_LOG
Definition: Defines.php:144
$wgFeedClasses
Available feeds objects.