MediaWiki  master
CategoryMembershipChangeJob.php
Go to the documentation of this file.
1 <?php
29 
46  private $ticket;
47 
48  private const ENQUEUE_FUDGE_SEC = 60;
49 
55  public static function newSpec( PageIdentity $page, $revisionTimestamp ) {
56  return new JobSpecification(
57  'categoryMembershipChange',
58  [
59  'pageId' => $page->getId(),
60  'revTimestamp' => $revisionTimestamp,
61  ],
62  [
63  'removeDuplicates' => true,
64  'removeDuplicatesIgnoreParams' => [ 'revTimestamp' ]
65  ],
66  $page
67  );
68  }
69 
76  public function __construct( PageIdentity $page, array $params ) {
77  parent::__construct( 'categoryMembershipChange', $page, $params );
78  // Only need one job per page. Note that ENQUEUE_FUDGE_SEC handles races where an
79  // older revision job gets inserted while the newer revision job is de-duplicated.
80  $this->removeDuplicates = true;
81  }
82 
83  public function run() {
84  $services = MediaWikiServices::getInstance();
85  $lbFactory = $services->getDBLoadBalancerFactory();
86  $lb = $lbFactory->getMainLB();
87  $dbw = $lb->getConnectionRef( DB_PRIMARY );
88 
89  $this->ticket = $lbFactory->getEmptyTransactionTicket( __METHOD__ );
90 
91  $page = $services->getWikiPageFactory()->newFromID( $this->params['pageId'], WikiPage::READ_LATEST );
92  if ( !$page ) {
93  $this->setLastError( "Could not find page #{$this->params['pageId']}" );
94  return false; // deleted?
95  }
96 
97  // Cut down on the time spent in waitForPrimaryPos() in the critical section
98  $dbr = $lb->getConnectionRef( DB_REPLICA );
99  if ( !$lb->waitForPrimaryPos( $dbr ) ) {
100  $this->setLastError( "Timed out while pre-waiting for replica DB to catch up" );
101  return false;
102  }
103 
104  // Use a named lock so that jobs for this page see each others' changes
105  $lockKey = "{$dbw->getDomainID()}:CategoryMembershipChange:{$page->getId()}"; // per-wiki
106  $scopedLock = $dbw->getScopedLockAndFlush( $lockKey, __METHOD__, 3 );
107  if ( !$scopedLock ) {
108  $this->setLastError( "Could not acquire lock '$lockKey'" );
109  return false;
110  }
111 
112  // Wait till replica DB is caught up so that jobs for this page see each others' changes
113  if ( !$lb->waitForPrimaryPos( $dbr ) ) {
114  $this->setLastError( "Timed out while waiting for replica DB to catch up" );
115  return false;
116  }
117  // Clear any stale REPEATABLE-READ snapshot
118  $dbr->flushSnapshot( __METHOD__ );
119 
120  $cutoffUnix = wfTimestamp( TS_UNIX, $this->params['revTimestamp'] );
121  // Using ENQUEUE_FUDGE_SEC handles jobs inserted out of revision order due to the delay
122  // between COMMIT and actual enqueueing of the CategoryMembershipChangeJob job.
123  $cutoffUnix -= self::ENQUEUE_FUDGE_SEC;
124 
125  // Get the newest page revision that has a SRC_CATEGORIZE row.
126  // Assume that category changes before it were already handled.
127  $row = $dbr->selectRow(
128  'revision',
129  [ 'rev_timestamp', 'rev_id' ],
130  [
131  'rev_page' => $page->getId(),
132  'rev_timestamp >= ' . $dbr->addQuotes( $dbr->timestamp( $cutoffUnix ) ),
133  'EXISTS (' . $dbr->selectSQLText(
134  'recentchanges',
135  '1',
136  [
137  'rc_this_oldid = rev_id',
138  'rc_source' => RecentChange::SRC_CATEGORIZE,
139  ],
140  __METHOD__
141  ) . ')'
142  ],
143  __METHOD__,
144  [ 'ORDER BY' => [ 'rev_timestamp DESC', 'rev_id DESC' ] ]
145  );
146  // Only consider revisions newer than any such revision
147  if ( $row ) {
148  $cutoffUnix = wfTimestamp( TS_UNIX, $row->rev_timestamp );
149  $lastRevId = (int)$row->rev_id;
150  } else {
151  $lastRevId = 0;
152  }
153 
154  // Find revisions to this page made around and after this revision which lack category
155  // notifications in recent changes. This lets jobs pick up were the last one left off.
156  $revisionStore = $services->getRevisionStore();
157  $revQuery = $revisionStore->getQueryInfo();
158  $res = $dbr->select(
159  $revQuery['tables'],
160  $revQuery['fields'],
161  [
162  'rev_page' => $page->getId(),
163  $dbr->buildComparison( '>', [
164  'rev_timestamp' => $dbr->timestamp( $cutoffUnix ),
165  'rev_id' => $lastRevId,
166  ] )
167  ],
168  __METHOD__,
169  [ 'ORDER BY' => [ 'rev_timestamp ASC', 'rev_id ASC' ] ],
170  $revQuery['joins']
171  );
172 
173  // Apply all category updates in revision timestamp order
174  foreach ( $res as $row ) {
175  $this->notifyUpdatesForRevision( $lbFactory, $page, $revisionStore->newRevisionFromRow( $row ) );
176  }
177 
178  return true;
179  }
180 
187  protected function notifyUpdatesForRevision(
188  LBFactory $lbFactory, WikiPage $page, RevisionRecord $newRev
189  ) {
190  $title = $page->getTitle();
191 
192  // Get the new revision
193  if ( $newRev->isDeleted( RevisionRecord::DELETED_TEXT ) ) {
194  return;
195  }
196 
197  $services = MediaWikiServices::getInstance();
198  // Get the prior revision (the same for null edits)
199  if ( $newRev->getParentId() ) {
200  $oldRev = $services->getRevisionLookup()
201  ->getRevisionById( $newRev->getParentId(), RevisionLookup::READ_LATEST );
202  if ( !$oldRev || $oldRev->isDeleted( RevisionRecord::DELETED_TEXT ) ) {
203  return;
204  }
205  } else {
206  $oldRev = null;
207  }
208 
209  // Parse the new revision and get the categories
210  $categoryChanges = $this->getExplicitCategoriesChanges( $page, $newRev, $oldRev );
211  [ $categoryInserts, $categoryDeletes ] = $categoryChanges;
212  if ( !$categoryInserts && !$categoryDeletes ) {
213  return; // nothing to do
214  }
215 
216  $blc = $services->getBacklinkCacheFactory()->getBacklinkCache( $title );
217  $catMembChange = new CategoryMembershipChange( $title, $blc, $newRev );
218  $catMembChange->checkTemplateLinks();
219 
220  $batchSize = $services->getMainConfig()->get( MainConfigNames::UpdateRowsPerQuery );
221  $insertCount = 0;
222 
223  foreach ( $categoryInserts as $categoryName ) {
224  $categoryTitle = Title::makeTitle( NS_CATEGORY, $categoryName );
225  $catMembChange->triggerCategoryAddedNotification( $categoryTitle );
226  if ( $insertCount++ && ( $insertCount % $batchSize ) == 0 ) {
227  $lbFactory->commitAndWaitForReplication( __METHOD__, $this->ticket );
228  }
229  }
230 
231  foreach ( $categoryDeletes as $categoryName ) {
232  $categoryTitle = Title::makeTitle( NS_CATEGORY, $categoryName );
233  $catMembChange->triggerCategoryRemovedNotification( $categoryTitle );
234  if ( $insertCount++ && ( $insertCount++ % $batchSize ) == 0 ) {
235  $lbFactory->commitAndWaitForReplication( __METHOD__, $this->ticket );
236  }
237  }
238  }
239 
240  private function getExplicitCategoriesChanges(
241  WikiPage $page, RevisionRecord $newRev, RevisionRecord $oldRev = null
242  ) {
243  // Inject the same timestamp for both revision parses to avoid seeing category changes
244  // due to time-based parser functions. Inject the same page title for the parses too.
245  // Note that REPEATABLE-READ makes template/file pages appear unchanged between parses.
246  $parseTimestamp = $newRev->getTimestamp();
247  // Parse the old rev and get the categories. Do not use link tables as that
248  // assumes these updates are perfectly FIFO and that link tables are always
249  // up to date, neither of which are true.
250  $oldCategories = $oldRev
251  ? $this->getCategoriesAtRev( $page, $oldRev, $parseTimestamp )
252  : [];
253  // Parse the new revision and get the categories
254  $newCategories = $this->getCategoriesAtRev( $page, $newRev, $parseTimestamp );
255 
256  $categoryInserts = array_values( array_diff( $newCategories, $oldCategories ) );
257  $categoryDeletes = array_values( array_diff( $oldCategories, $newCategories ) );
258 
259  return [ $categoryInserts, $categoryDeletes ];
260  }
261 
269  private function getCategoriesAtRev( WikiPage $page, RevisionRecord $rev, $parseTimestamp ) {
270  $services = MediaWikiServices::getInstance();
271  $options = $page->makeParserOptions( 'canonical' );
272  $options->setTimestamp( $parseTimestamp );
273  $options->setRenderReason( 'CategoryMembershipChangeJob' );
274 
275  $output = $rev instanceof RevisionStoreRecord && $rev->isCurrent()
276  ? $services->getParserCache()->get( $page, $options )
277  : null;
278 
279  if ( !$output || $output->getCacheRevisionId() !== $rev->getId() ) {
280  $output = $services->getRevisionRenderer()->getRenderedRevision( $rev, $options )
281  ->getRevisionParserOutput();
282  }
283 
284  // array keys will cast numeric category names to ints;
285  // ::getCategoryNames() is careful to cast them back to strings
286  // to avoid breaking things!
287  return $output->getCategoryNames();
288  }
289 
290  public function getDeduplicationInfo() {
291  $info = parent::getDeduplicationInfo();
292  unset( $info['params']['revTimestamp'] ); // first job wins
293 
294  return $info;
295  }
296 }
const NS_CATEGORY
Definition: Defines.php:78
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Job to add recent change entries mentioning category membership changes.
static newSpec(PageIdentity $page, $revisionTimestamp)
__construct(PageIdentity $page, array $params)
Constructor for use by the Job Queue infrastructure.
getDeduplicationInfo()
Subclasses may need to override this to make duplication detection work.
notifyUpdatesForRevision(LBFactory $lbFactory, WikiPage $page, RevisionRecord $newRev)
Job queue task description base code.
Class to both describe a background job and handle jobs.
Definition: Job.php:39
Title $title
Definition: Job.php:50
setLastError( $error)
Definition: Job.php:431
array $params
Array of job parameters.
Definition: Job.php:44
A class containing constants representing the names of configuration variables.
Service locator for MediaWiki core services.
Page revision base class.
getParentId( $wikiId=self::LOCAL)
Get parent revision ID (the original previous page revision).
isCurrent()
Checks whether the revision record is a stored current revision.
getTimestamp()
MCR migration note: this replaced Revision::getTimestamp.
isDeleted( $field)
MCR migration note: this replaced Revision::isDeleted.
getId( $wikiId=self::LOCAL)
Get revision ID.
A RevisionRecord representing an existing revision persisted in the revision table.
Represents a title within MediaWiki.
Definition: Title.php:82
const SRC_CATEGORIZE
Base representation for an editable wiki page.
Definition: WikiPage.php:75
static newFromID( $id, $from='fromdb')
Constructor from a page id.
Definition: WikiPage.php:227
makeParserOptions( $context)
Get parser options suitable for rendering the primary article wikitext.
Definition: WikiPage.php:1946
getId( $wikiId=self::LOCAL)
Definition: WikiPage.php:588
getTitle()
Get the title object of the article.
Definition: WikiPage.php:318
commitAndWaitForReplication( $fname, $ticket, array $opts=[])
Commit primary DB transactions and wait for replication (if $ticket indicates it is safe).
Definition: LBFactory.php:597
Interface for objects (potentially) representing an editable wiki page.
getId( $wikiId=self::LOCAL)
Returns the page ID.
Service for looking up page revisions.
const DB_REPLICA
Definition: defines.php:26
const DB_PRIMARY
Definition: defines.php:28
$revQuery