MediaWiki  master
CategoryMembershipChangeJob.php
Go to the documentation of this file.
1 <?php
24 
40  private $ticket;
41 
42  const ENQUEUE_FUDGE_SEC = 60;
43 
47  private $parserCache;
48 
54  public static function newSpec( Title $title, $revisionTimestamp ) {
55  return new JobSpecification(
56  'categoryMembershipChange',
57  [
58  'pageId' => $title->getArticleID(),
59  'revTimestamp' => $revisionTimestamp,
60  ],
61  [
62  'removeDuplicates' => true,
63  'removeDuplicatesIgnoreParams' => [ 'revTimestamp' ]
64  ],
65  $title
66  );
67  }
68 
76  public function __construct( ParserCache $parserCache, Title $title, array $params ) {
77  parent::__construct( 'categoryMembershipChange', $title, $params );
78  // Only need one job per page. Note that ENQUEUE_FUDGE_SEC handles races where an
79  // older revision job gets inserted while the newer revision job is de-duplicated.
80  $this->removeDuplicates = true;
81  $this->parserCache = $parserCache;
82  }
83 
84  public function run() {
85  $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
86  $lb = $lbFactory->getMainLB();
87  $dbw = $lb->getConnectionRef( DB_MASTER );
88 
89  $this->ticket = $lbFactory->getEmptyTransactionTicket( __METHOD__ );
90 
91  $page = WikiPage::newFromID( $this->params['pageId'], WikiPage::READ_LATEST );
92  if ( !$page ) {
93  $this->setLastError( "Could not find page #{$this->params['pageId']}" );
94  return false; // deleted?
95  }
96 
97  // Cut down on the time spent in waitForMasterPos() in the critical section
98  $dbr = $lb->getConnectionRef( DB_REPLICA, [ 'recentchanges' ] );
99  if ( !$lb->waitForMasterPos( $dbr ) ) {
100  $this->setLastError( "Timed out while pre-waiting for replica DB to catch up" );
101  return false;
102  }
103 
104  // Use a named lock so that jobs for this page see each others' changes
105  $lockKey = "{$dbw->getDomainID()}:CategoryMembershipChange:{$page->getId()}"; // per-wiki
106  $scopedLock = $dbw->getScopedLockAndFlush( $lockKey, __METHOD__, 3 );
107  if ( !$scopedLock ) {
108  $this->setLastError( "Could not acquire lock '$lockKey'" );
109  return false;
110  }
111 
112  // Wait till replica DB is caught up so that jobs for this page see each others' changes
113  if ( !$lb->waitForMasterPos( $dbr ) ) {
114  $this->setLastError( "Timed out while waiting for replica DB to catch up" );
115  return false;
116  }
117  // Clear any stale REPEATABLE-READ snapshot
118  $dbr->flushSnapshot( __METHOD__ );
119 
120  $cutoffUnix = wfTimestamp( TS_UNIX, $this->params['revTimestamp'] );
121  // Using ENQUEUE_FUDGE_SEC handles jobs inserted out of revision order due to the delay
122  // between COMMIT and actual enqueueing of the CategoryMembershipChangeJob job.
123  $cutoffUnix -= self::ENQUEUE_FUDGE_SEC;
124 
125  // Get the newest page revision that has a SRC_CATEGORIZE row.
126  // Assume that category changes before it were already handled.
127  $row = $dbr->selectRow(
128  'revision',
129  [ 'rev_timestamp', 'rev_id' ],
130  [
131  'rev_page' => $page->getId(),
132  'rev_timestamp >= ' . $dbr->addQuotes( $dbr->timestamp( $cutoffUnix ) ),
133  'EXISTS (' . $dbr->selectSQLText(
134  'recentchanges',
135  '1',
136  [
137  'rc_this_oldid = rev_id',
138  'rc_source' => RecentChange::SRC_CATEGORIZE,
139  ]
140  ) . ')'
141  ],
142  __METHOD__,
143  [ 'ORDER BY' => [ 'rev_timestamp DESC', 'rev_id DESC' ] ]
144  );
145  // Only consider revisions newer than any such revision
146  if ( $row ) {
147  $cutoffUnix = wfTimestamp( TS_UNIX, $row->rev_timestamp );
148  $lastRevId = (int)$row->rev_id;
149  } else {
150  $lastRevId = 0;
151  }
152 
153  // Find revisions to this page made around and after this revision which lack category
154  // notifications in recent changes. This lets jobs pick up were the last one left off.
155  $encCutoff = $dbr->addQuotes( $dbr->timestamp( $cutoffUnix ) );
157  $res = $dbr->select(
158  $revQuery['tables'],
159  $revQuery['fields'],
160  [
161  'rev_page' => $page->getId(),
162  "rev_timestamp > $encCutoff" .
163  " OR (rev_timestamp = $encCutoff AND rev_id > $lastRevId)"
164  ],
165  __METHOD__,
166  [ 'ORDER BY' => [ 'rev_timestamp ASC', 'rev_id ASC' ] ],
167  $revQuery['joins']
168  );
169 
170  // Apply all category updates in revision timestamp order
171  foreach ( $res as $row ) {
172  $this->notifyUpdatesForRevision( $lbFactory, $page, Revision::newFromRow( $row ) );
173  }
174 
175  return true;
176  }
177 
184  protected function notifyUpdatesForRevision(
185  LBFactory $lbFactory, WikiPage $page, Revision $newRev
186  ) {
187  $config = RequestContext::getMain()->getConfig();
188  $title = $page->getTitle();
189 
190  // Get the new revision
191  if ( !$newRev->getContent() ) {
192  return; // deleted?
193  }
194 
195  // Get the prior revision (the same for null edits)
196  if ( $newRev->getParentId() ) {
197  $oldRev = Revision::newFromId( $newRev->getParentId(), Revision::READ_LATEST );
198  if ( !$oldRev || !$oldRev->getContent() ) {
199  return; // deleted?
200  }
201  } else {
202  $oldRev = null;
203  }
204 
205  // Parse the new revision and get the categories
206  $categoryChanges = $this->getExplicitCategoriesChanges( $page, $newRev, $oldRev );
207  list( $categoryInserts, $categoryDeletes ) = $categoryChanges;
208  if ( !$categoryInserts && !$categoryDeletes ) {
209  return; // nothing to do
210  }
211 
212  $catMembChange = new CategoryMembershipChange( $title, $newRev );
213  $catMembChange->checkTemplateLinks();
214 
215  $batchSize = $config->get( 'UpdateRowsPerQuery' );
216  $insertCount = 0;
217 
218  foreach ( $categoryInserts as $categoryName ) {
219  $categoryTitle = Title::makeTitle( NS_CATEGORY, $categoryName );
220  $catMembChange->triggerCategoryAddedNotification( $categoryTitle );
221  if ( $insertCount++ && ( $insertCount % $batchSize ) == 0 ) {
222  $lbFactory->commitAndWaitForReplication( __METHOD__, $this->ticket );
223  }
224  }
225 
226  foreach ( $categoryDeletes as $categoryName ) {
227  $categoryTitle = Title::makeTitle( NS_CATEGORY, $categoryName );
228  $catMembChange->triggerCategoryRemovedNotification( $categoryTitle );
229  if ( $insertCount++ && ( $insertCount++ % $batchSize ) == 0 ) {
230  $lbFactory->commitAndWaitForReplication( __METHOD__, $this->ticket );
231  }
232  }
233  }
234 
236  WikiPage $page, Revision $newRev, Revision $oldRev = null
237  ) {
238  // Inject the same timestamp for both revision parses to avoid seeing category changes
239  // due to time-based parser functions. Inject the same page title for the parses too.
240  // Note that REPEATABLE-READ makes template/file pages appear unchanged between parses.
241  $parseTimestamp = $newRev->getTimestamp();
242  // Parse the old rev and get the categories. Do not use link tables as that
243  // assumes these updates are perfectly FIFO and that link tables are always
244  // up to date, neither of which are true.
245  $oldCategories = $oldRev
246  ? $this->getCategoriesAtRev( $page, $oldRev, $parseTimestamp )
247  : [];
248  // Parse the new revision and get the categories
249  $newCategories = $this->getCategoriesAtRev( $page, $newRev, $parseTimestamp );
250 
251  $categoryInserts = array_values( array_diff( $newCategories, $oldCategories ) );
252  $categoryDeletes = array_values( array_diff( $oldCategories, $newCategories ) );
253 
254  return [ $categoryInserts, $categoryDeletes ];
255  }
256 
264  private function getCategoriesAtRev( WikiPage $page, Revision $rev, $parseTimestamp ) {
265  $renderer = MediaWikiServices::getInstance()->getRevisionRenderer();
266  $options = $page->makeParserOptions( 'canonical' );
267  $options->setTimestamp( $parseTimestamp );
268 
269  $output = $rev->isCurrent() ? $this->parserCache->get( $page, $options ) : null;
270 
271  if ( !$output || $output->getCacheRevisionId() !== $rev->getId() ) {
272  $output = $renderer->getRenderedRevision( $rev->getRevisionRecord(), $options )
273  ->getRevisionParserOutput();
274  }
275 
276  // array keys will cast numeric category names to ints
277  // so we need to cast them back to strings to avoid breaking things!
278  return array_map( 'strval', array_keys( $output->getCategories() ) );
279  }
280 
281  public function getDeduplicationInfo() {
282  $info = parent::getDeduplicationInfo();
283  unset( $info['params']['revTimestamp'] ); // first job wins
284 
285  return $info;
286  }
287 }
getArticleID( $flags=0)
Get the article ID for this Title from the link cache, adding it if necessary.
Definition: Title.php:3161
const SRC_CATEGORIZE
getRevisionRecord()
Definition: Revision.php:434
An interface for generating database load balancers.
Definition: LBFactory.php:40
getTimestamp()
Definition: Revision.php:799
notifyUpdatesForRevision(LBFactory $lbFactory, WikiPage $page, Revision $newRev)
getDeduplicationInfo()
Subclasses may need to override this to make duplication detection work.
Class to both describe a background job and handle jobs.
Definition: Job.php:30
commitAndWaitForReplication( $fname, $ticket, array $opts=[])
Convenience method for safely running commitMasterChanges()/waitForReplication()
Definition: LBFactory.php:470
getContent( $audience=self::FOR_PUBLIC, User $user=null)
Fetch revision content if it&#39;s available to the specified audience.
Definition: Revision.php:720
const DB_MASTER
Definition: defines.php:26
setLastError( $error)
Definition: Job.php:418
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
static getMain()
Get the RequestContext object associated with the main request.
getParentId()
Get parent revision ID (the original previous page revision)
Definition: Revision.php:522
getId()
Get revision ID.
Definition: Revision.php:443
getTitle()
Get the title object of the article.
Definition: WikiPage.php:298
const NS_CATEGORY
Definition: Defines.php:74
static getQueryInfo( $options=[])
Return the tables, fields, and join conditions to be selected to create a new revision object...
Definition: Revision.php:316
__construct(ParserCache $parserCache, Title $title, array $params)
Constructor for use by the Job Queue infrastructure.
getExplicitCategoriesChanges(WikiPage $page, Revision $newRev, Revision $oldRev=null)
makeParserOptions( $context)
Get parser options suitable for rendering the primary article wikitext.
Definition: WikiPage.php:1961
getCategoriesAtRev(WikiPage $page, Revision $rev, $parseTimestamp)
static makeTitle( $ns, $title, $fragment='', $interwiki='')
Create a new Title from a namespace index and a DB key.
Definition: Title.php:584
static newFromID( $id, $from='fromdb')
Constructor from a page id.
Definition: WikiPage.php:180
static newFromRow( $row)
Definition: Revision.php:224
static newSpec(Title $title, $revisionTimestamp)
$revQuery
array $params
Array of job parameters.
Definition: Job.php:35
const DB_REPLICA
Definition: defines.php:25
static newFromId( $id, $flags=0)
Load a page revision from a given revision ID number.
Definition: Revision.php:120
Title $title
Definition: Job.php:41
Job queue task description base code.
isCurrent()
Definition: Revision.php:806