MediaWiki  master
CategoryMembershipChangeJob.php
Go to the documentation of this file.
1 <?php
27 
43  private $ticket;
44 
45  private const ENQUEUE_FUDGE_SEC = 60;
46 
52  public static function newSpec( Title $title, $revisionTimestamp ) {
53  return new JobSpecification(
54  'categoryMembershipChange',
55  [
56  'pageId' => $title->getArticleID(),
57  'revTimestamp' => $revisionTimestamp,
58  ],
59  [
60  'removeDuplicates' => true,
61  'removeDuplicatesIgnoreParams' => [ 'revTimestamp' ]
62  ],
63  $title
64  );
65  }
66 
73  public function __construct( Title $title, array $params ) {
74  parent::__construct( 'categoryMembershipChange', $title, $params );
75  // Only need one job per page. Note that ENQUEUE_FUDGE_SEC handles races where an
76  // older revision job gets inserted while the newer revision job is de-duplicated.
77  $this->removeDuplicates = true;
78  }
79 
80  public function run() {
81  $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
82  $lb = $lbFactory->getMainLB();
83  $dbw = $lb->getConnectionRef( DB_MASTER );
84 
85  $this->ticket = $lbFactory->getEmptyTransactionTicket( __METHOD__ );
86 
87  $page = WikiPage::newFromID( $this->params['pageId'], WikiPage::READ_LATEST );
88  if ( !$page ) {
89  $this->setLastError( "Could not find page #{$this->params['pageId']}" );
90  return false; // deleted?
91  }
92 
93  // Cut down on the time spent in waitForMasterPos() in the critical section
94  $dbr = $lb->getConnectionRef( DB_REPLICA, [ 'recentchanges' ] );
95  if ( !$lb->waitForMasterPos( $dbr ) ) {
96  $this->setLastError( "Timed out while pre-waiting for replica DB to catch up" );
97  return false;
98  }
99 
100  // Use a named lock so that jobs for this page see each others' changes
101  $lockKey = "{$dbw->getDomainID()}:CategoryMembershipChange:{$page->getId()}"; // per-wiki
102  $scopedLock = $dbw->getScopedLockAndFlush( $lockKey, __METHOD__, 3 );
103  if ( !$scopedLock ) {
104  $this->setLastError( "Could not acquire lock '$lockKey'" );
105  return false;
106  }
107 
108  // Wait till replica DB is caught up so that jobs for this page see each others' changes
109  if ( !$lb->waitForMasterPos( $dbr ) ) {
110  $this->setLastError( "Timed out while waiting for replica DB to catch up" );
111  return false;
112  }
113  // Clear any stale REPEATABLE-READ snapshot
114  $dbr->flushSnapshot( __METHOD__ );
115 
116  $cutoffUnix = wfTimestamp( TS_UNIX, $this->params['revTimestamp'] );
117  // Using ENQUEUE_FUDGE_SEC handles jobs inserted out of revision order due to the delay
118  // between COMMIT and actual enqueueing of the CategoryMembershipChangeJob job.
119  $cutoffUnix -= self::ENQUEUE_FUDGE_SEC;
120 
121  // Get the newest page revision that has a SRC_CATEGORIZE row.
122  // Assume that category changes before it were already handled.
123  $row = $dbr->selectRow(
124  'revision',
125  [ 'rev_timestamp', 'rev_id' ],
126  [
127  'rev_page' => $page->getId(),
128  'rev_timestamp >= ' . $dbr->addQuotes( $dbr->timestamp( $cutoffUnix ) ),
129  'EXISTS (' . $dbr->selectSQLText(
130  'recentchanges',
131  '1',
132  [
133  'rc_this_oldid = rev_id',
134  'rc_source' => RecentChange::SRC_CATEGORIZE,
135  ]
136  ) . ')'
137  ],
138  __METHOD__,
139  [ 'ORDER BY' => [ 'rev_timestamp DESC', 'rev_id DESC' ] ]
140  );
141  // Only consider revisions newer than any such revision
142  if ( $row ) {
143  $cutoffUnix = wfTimestamp( TS_UNIX, $row->rev_timestamp );
144  $lastRevId = (int)$row->rev_id;
145  } else {
146  $lastRevId = 0;
147  }
148 
149  // Find revisions to this page made around and after this revision which lack category
150  // notifications in recent changes. This lets jobs pick up were the last one left off.
151  $encCutoff = $dbr->addQuotes( $dbr->timestamp( $cutoffUnix ) );
152  $revisionStore = MediaWikiServices::getInstance()->getRevisionStore();
153  $revQuery = $revisionStore->getQueryInfo();
154  $res = $dbr->select(
155  $revQuery['tables'],
156  $revQuery['fields'],
157  [
158  'rev_page' => $page->getId(),
159  "rev_timestamp > $encCutoff" .
160  " OR (rev_timestamp = $encCutoff AND rev_id > $lastRevId)"
161  ],
162  __METHOD__,
163  [ 'ORDER BY' => [ 'rev_timestamp ASC', 'rev_id ASC' ] ],
164  $revQuery['joins']
165  );
166 
167  // Apply all category updates in revision timestamp order
168  foreach ( $res as $row ) {
169  $this->notifyUpdatesForRevision( $lbFactory, $page, $revisionStore->newRevisionFromRow( $row ) );
170  }
171 
172  return true;
173  }
174 
181  protected function notifyUpdatesForRevision(
182  LBFactory $lbFactory, WikiPage $page, RevisionRecord $newRev
183  ) {
184  $config = RequestContext::getMain()->getConfig();
185  $title = $page->getTitle();
186 
187  // Get the new revision
188  if ( $newRev->isDeleted( RevisionRecord::DELETED_TEXT ) ) {
189  return;
190  }
191 
192  // Get the prior revision (the same for null edits)
193  if ( $newRev->getParentId() ) {
194  $oldRev = MediaWikiServices::getInstance()
195  ->getRevisionLookup()
196  ->getRevisionById( $newRev->getParentId(), RevisionLookup::READ_LATEST );
197  if ( !$oldRev || $oldRev->isDeleted( RevisionRecord::DELETED_TEXT ) ) {
198  return;
199  }
200  } else {
201  $oldRev = null;
202  }
203 
204  // Parse the new revision and get the categories
205  $categoryChanges = $this->getExplicitCategoriesChanges( $page, $newRev, $oldRev );
206  list( $categoryInserts, $categoryDeletes ) = $categoryChanges;
207  if ( !$categoryInserts && !$categoryDeletes ) {
208  return; // nothing to do
209  }
210 
211  $catMembChange = new CategoryMembershipChange( $title, $newRev );
212  $catMembChange->checkTemplateLinks();
213 
214  $batchSize = $config->get( 'UpdateRowsPerQuery' );
215  $insertCount = 0;
216 
217  foreach ( $categoryInserts as $categoryName ) {
218  $categoryTitle = Title::makeTitle( NS_CATEGORY, $categoryName );
219  $catMembChange->triggerCategoryAddedNotification( $categoryTitle );
220  if ( $insertCount++ && ( $insertCount % $batchSize ) == 0 ) {
221  $lbFactory->commitAndWaitForReplication( __METHOD__, $this->ticket );
222  }
223  }
224 
225  foreach ( $categoryDeletes as $categoryName ) {
226  $categoryTitle = Title::makeTitle( NS_CATEGORY, $categoryName );
227  $catMembChange->triggerCategoryRemovedNotification( $categoryTitle );
228  if ( $insertCount++ && ( $insertCount++ % $batchSize ) == 0 ) {
229  $lbFactory->commitAndWaitForReplication( __METHOD__, $this->ticket );
230  }
231  }
232  }
233 
235  WikiPage $page, RevisionRecord $newRev, RevisionRecord $oldRev = null
236  ) {
237  // Inject the same timestamp for both revision parses to avoid seeing category changes
238  // due to time-based parser functions. Inject the same page title for the parses too.
239  // Note that REPEATABLE-READ makes template/file pages appear unchanged between parses.
240  $parseTimestamp = $newRev->getTimestamp();
241  // Parse the old rev and get the categories. Do not use link tables as that
242  // assumes these updates are perfectly FIFO and that link tables are always
243  // up to date, neither of which are true.
244  $oldCategories = $oldRev
245  ? $this->getCategoriesAtRev( $page, $oldRev, $parseTimestamp )
246  : [];
247  // Parse the new revision and get the categories
248  $newCategories = $this->getCategoriesAtRev( $page, $newRev, $parseTimestamp );
249 
250  $categoryInserts = array_values( array_diff( $newCategories, $oldCategories ) );
251  $categoryDeletes = array_values( array_diff( $oldCategories, $newCategories ) );
252 
253  return [ $categoryInserts, $categoryDeletes ];
254  }
255 
263  private function getCategoriesAtRev( WikiPage $page, RevisionRecord $rev, $parseTimestamp ) {
264  $services = MediaWikiServices::getInstance();
265  $options = $page->makeParserOptions( 'canonical' );
266  $options->setTimestamp( $parseTimestamp );
267 
268  $output = $rev instanceof RevisionStoreRecord && $rev->isCurrent()
269  ? $services->getParserCache()->get( $page, $options )
270  : null;
271 
272  if ( !$output || $output->getCacheRevisionId() !== $rev->getId() ) {
273  $output = $services->getRevisionRenderer()->getRenderedRevision( $rev, $options )
274  ->getRevisionParserOutput();
275  }
276 
277  // array keys will cast numeric category names to ints
278  // so we need to cast them back to strings to avoid breaking things!
279  return array_map( 'strval', array_keys( $output->getCategories() ) );
280  }
281 
282  public function getDeduplicationInfo() {
283  $info = parent::getDeduplicationInfo();
284  unset( $info['params']['revTimestamp'] ); // first job wins
285 
286  return $info;
287  }
288 }
Revision\RevisionRecord
Page revision base class.
Definition: RevisionRecord.php:46
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:144
Revision\RevisionRecord\isDeleted
isDeleted( $field)
MCR migration note: this replaces Revision::isDeleted.
Definition: RevisionRecord.php:420
CategoryMembershipChangeJob\notifyUpdatesForRevision
notifyUpdatesForRevision(LBFactory $lbFactory, WikiPage $page, RevisionRecord $newRev)
Definition: CategoryMembershipChangeJob.php:181
Wikimedia\Rdbms\LBFactory\commitAndWaitForReplication
commitAndWaitForReplication( $fname, $ticket, array $opts=[])
Convenience method for safely running commitMasterChanges()/waitForReplication()
Definition: LBFactory.php:503
wfTimestamp
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Definition: GlobalFunctions.php:1806
Job\$title
Title $title
Definition: Job.php:41
Revision\RevisionRecord\getTimestamp
getTimestamp()
MCR migration note: this replaces Revision::getTimestamp.
Definition: RevisionRecord.php:442
CategoryMembershipChangeJob\getCategoriesAtRev
getCategoriesAtRev(WikiPage $page, RevisionRecord $rev, $parseTimestamp)
Definition: CategoryMembershipChangeJob.php:263
WikiPage
Class representing a MediaWiki article and history.
Definition: WikiPage.php:48
Title\getArticleID
getArticleID( $flags=0)
Get the article ID for this Title from the link cache, adding it if necessary.
Definition: Title.php:3171
WikiPage\makeParserOptions
makeParserOptions( $context)
Get parser options suitable for rendering the primary article wikitext.
Definition: WikiPage.php:1995
RecentChange\SRC_CATEGORIZE
const SRC_CATEGORIZE
Definition: RecentChange.php:78
$res
$res
Definition: testCompression.php:57
CategoryMembershipChangeJob\getDeduplicationInfo
getDeduplicationInfo()
Subclasses may need to override this to make duplication detection work.
Definition: CategoryMembershipChangeJob.php:282
Job\$params
array $params
Array of job parameters.
Definition: Job.php:35
$revQuery
$revQuery
Definition: testCompression.php:56
Job\setLastError
setLastError( $error)
Definition: Job.php:418
Revision\RevisionLookup
Service for looking up page revisions.
Definition: RevisionLookup.php:38
$dbr
$dbr
Definition: testCompression.php:54
Job
Class to both describe a background job and handle jobs.
Definition: Job.php:30
WikiPage\getTitle
getTitle()
Get the title object of the article.
Definition: WikiPage.php:307
CategoryMembershipChangeJob\run
run()
Run the job.
Definition: CategoryMembershipChangeJob.php:80
Title\makeTitle
static makeTitle( $ns, $title, $fragment='', $interwiki='')
Create a new Title from a namespace index and a DB key.
Definition: Title.php:595
DB_REPLICA
const DB_REPLICA
Definition: defines.php:25
NS_CATEGORY
const NS_CATEGORY
Definition: Defines.php:83
DB_MASTER
const DB_MASTER
Definition: defines.php:26
CategoryMembershipChangeJob
Job to add recent change entries mentioning category membership changes.
Definition: CategoryMembershipChangeJob.php:41
Revision\RevisionRecord\getId
getId()
Get revision ID.
Definition: RevisionRecord.php:279
Revision\RevisionRecord\getParentId
getParentId()
Get parent revision ID (the original previous page revision).
Definition: RevisionRecord.php:295
WikiPage\newFromID
static newFromID( $id, $from='fromdb')
Constructor from a page id.
Definition: WikiPage.php:181
CategoryMembershipChangeJob\$ticket
int null $ticket
Definition: CategoryMembershipChangeJob.php:43
Revision\RevisionStoreRecord
A RevisionRecord representing an existing revision persisted in the revision table.
Definition: RevisionStoreRecord.php:40
RequestContext\getMain
static getMain()
Get the RequestContext object associated with the main request.
Definition: RequestContext.php:451
Title
Represents a title within MediaWiki.
Definition: Title.php:42
JobSpecification
Job queue task description base code.
Definition: JobSpecification.php:39
Wikimedia\Rdbms\LBFactory
An interface for generating database load balancers.
Definition: LBFactory.php:41
CategoryMembershipChangeJob\ENQUEUE_FUDGE_SEC
const ENQUEUE_FUDGE_SEC
Definition: CategoryMembershipChangeJob.php:45
CategoryMembershipChangeJob\getExplicitCategoriesChanges
getExplicitCategoriesChanges(WikiPage $page, RevisionRecord $newRev, RevisionRecord $oldRev=null)
Definition: CategoryMembershipChangeJob.php:234
CategoryMembershipChangeJob\newSpec
static newSpec(Title $title, $revisionTimestamp)
Definition: CategoryMembershipChangeJob.php:52
Revision\RevisionRecord\isCurrent
isCurrent()
Checks whether the revision record is a stored current revision.
Definition: RevisionRecord.php:582
CategoryMembershipChange
Definition: CategoryMembershipChange.php:31
CategoryMembershipChangeJob\__construct
__construct(Title $title, array $params)
Constructor for use by the Job Queue infrastructure.
Definition: CategoryMembershipChangeJob.php:73