MediaWiki  master
CategoryMembershipChangeJob.php
Go to the documentation of this file.
1 <?php
28 
45  private $ticket;
46 
47  private const ENQUEUE_FUDGE_SEC = 60;
48 
54  public static function newSpec( PageIdentity $page, $revisionTimestamp ) {
55  return new JobSpecification(
56  'categoryMembershipChange',
57  [
58  'pageId' => $page->getId(),
59  'revTimestamp' => $revisionTimestamp,
60  ],
61  [
62  'removeDuplicates' => true,
63  'removeDuplicatesIgnoreParams' => [ 'revTimestamp' ]
64  ],
65  $page
66  );
67  }
68 
75  public function __construct( PageIdentity $page, array $params ) {
76  parent::__construct( 'categoryMembershipChange', $page, $params );
77  // Only need one job per page. Note that ENQUEUE_FUDGE_SEC handles races where an
78  // older revision job gets inserted while the newer revision job is de-duplicated.
79  $this->removeDuplicates = true;
80  }
81 
82  public function run() {
83  $services = MediaWikiServices::getInstance();
84  $lbFactory = $services->getDBLoadBalancerFactory();
85  $lb = $lbFactory->getMainLB();
86  $dbw = $lb->getConnectionRef( DB_PRIMARY );
87 
88  $this->ticket = $lbFactory->getEmptyTransactionTicket( __METHOD__ );
89 
90  $page = $services->getWikiPageFactory()->newFromID( $this->params['pageId'], WikiPage::READ_LATEST );
91  if ( !$page ) {
92  $this->setLastError( "Could not find page #{$this->params['pageId']}" );
93  return false; // deleted?
94  }
95 
96  // Cut down on the time spent in waitForPrimaryPos() in the critical section
97  $dbr = $lb->getConnectionRef( DB_REPLICA, [ 'recentchanges' ] );
98  if ( !$lb->waitForPrimaryPos( $dbr ) ) {
99  $this->setLastError( "Timed out while pre-waiting for replica DB to catch up" );
100  return false;
101  }
102 
103  // Use a named lock so that jobs for this page see each others' changes
104  $lockKey = "{$dbw->getDomainID()}:CategoryMembershipChange:{$page->getId()}"; // per-wiki
105  $scopedLock = $dbw->getScopedLockAndFlush( $lockKey, __METHOD__, 3 );
106  if ( !$scopedLock ) {
107  $this->setLastError( "Could not acquire lock '$lockKey'" );
108  return false;
109  }
110 
111  // Wait till replica DB is caught up so that jobs for this page see each others' changes
112  if ( !$lb->waitForPrimaryPos( $dbr ) ) {
113  $this->setLastError( "Timed out while waiting for replica DB to catch up" );
114  return false;
115  }
116  // Clear any stale REPEATABLE-READ snapshot
117  $dbr->flushSnapshot( __METHOD__ );
118 
119  $cutoffUnix = wfTimestamp( TS_UNIX, $this->params['revTimestamp'] );
120  // Using ENQUEUE_FUDGE_SEC handles jobs inserted out of revision order due to the delay
121  // between COMMIT and actual enqueueing of the CategoryMembershipChangeJob job.
122  $cutoffUnix -= self::ENQUEUE_FUDGE_SEC;
123 
124  // Get the newest page revision that has a SRC_CATEGORIZE row.
125  // Assume that category changes before it were already handled.
126  $row = $dbr->selectRow(
127  'revision',
128  [ 'rev_timestamp', 'rev_id' ],
129  [
130  'rev_page' => $page->getId(),
131  'rev_timestamp >= ' . $dbr->addQuotes( $dbr->timestamp( $cutoffUnix ) ),
132  'EXISTS (' . $dbr->selectSQLText(
133  'recentchanges',
134  '1',
135  [
136  'rc_this_oldid = rev_id',
137  'rc_source' => RecentChange::SRC_CATEGORIZE,
138  ],
139  __METHOD__
140  ) . ')'
141  ],
142  __METHOD__,
143  [ 'ORDER BY' => [ 'rev_timestamp DESC', 'rev_id DESC' ] ]
144  );
145  // Only consider revisions newer than any such revision
146  if ( $row ) {
147  $cutoffUnix = wfTimestamp( TS_UNIX, $row->rev_timestamp );
148  $lastRevId = (int)$row->rev_id;
149  } else {
150  $lastRevId = 0;
151  }
152 
153  // Find revisions to this page made around and after this revision which lack category
154  // notifications in recent changes. This lets jobs pick up were the last one left off.
155  $revisionStore = $services->getRevisionStore();
156  $revQuery = $revisionStore->getQueryInfo();
157  $res = $dbr->select(
158  $revQuery['tables'],
159  $revQuery['fields'],
160  [
161  'rev_page' => $page->getId(),
162  $dbr->buildComparison( '>', [
163  'rev_timestamp' => $dbr->timestamp( $cutoffUnix ),
164  'rev_id' => $lastRevId,
165  ] )
166  ],
167  __METHOD__,
168  [ 'ORDER BY' => [ 'rev_timestamp ASC', 'rev_id ASC' ] ],
169  $revQuery['joins']
170  );
171 
172  // Apply all category updates in revision timestamp order
173  foreach ( $res as $row ) {
174  $this->notifyUpdatesForRevision( $lbFactory, $page, $revisionStore->newRevisionFromRow( $row ) );
175  }
176 
177  return true;
178  }
179 
186  protected function notifyUpdatesForRevision(
187  LBFactory $lbFactory, WikiPage $page, RevisionRecord $newRev
188  ) {
189  $title = $page->getTitle();
190 
191  // Get the new revision
192  if ( $newRev->isDeleted( RevisionRecord::DELETED_TEXT ) ) {
193  return;
194  }
195 
196  $services = MediaWikiServices::getInstance();
197  // Get the prior revision (the same for null edits)
198  if ( $newRev->getParentId() ) {
199  $oldRev = $services->getRevisionLookup()
200  ->getRevisionById( $newRev->getParentId(), RevisionLookup::READ_LATEST );
201  if ( !$oldRev || $oldRev->isDeleted( RevisionRecord::DELETED_TEXT ) ) {
202  return;
203  }
204  } else {
205  $oldRev = null;
206  }
207 
208  // Parse the new revision and get the categories
209  $categoryChanges = $this->getExplicitCategoriesChanges( $page, $newRev, $oldRev );
210  [ $categoryInserts, $categoryDeletes ] = $categoryChanges;
211  if ( !$categoryInserts && !$categoryDeletes ) {
212  return; // nothing to do
213  }
214 
215  $blc = $services->getBacklinkCacheFactory()->getBacklinkCache( $title );
216  $catMembChange = new CategoryMembershipChange( $title, $blc, $newRev );
217  $catMembChange->checkTemplateLinks();
218 
219  $batchSize = $services->getMainConfig()->get( MainConfigNames::UpdateRowsPerQuery );
220  $insertCount = 0;
221 
222  foreach ( $categoryInserts as $categoryName ) {
223  $categoryTitle = Title::makeTitle( NS_CATEGORY, $categoryName );
224  $catMembChange->triggerCategoryAddedNotification( $categoryTitle );
225  if ( $insertCount++ && ( $insertCount % $batchSize ) == 0 ) {
226  $lbFactory->commitAndWaitForReplication( __METHOD__, $this->ticket );
227  }
228  }
229 
230  foreach ( $categoryDeletes as $categoryName ) {
231  $categoryTitle = Title::makeTitle( NS_CATEGORY, $categoryName );
232  $catMembChange->triggerCategoryRemovedNotification( $categoryTitle );
233  if ( $insertCount++ && ( $insertCount++ % $batchSize ) == 0 ) {
234  $lbFactory->commitAndWaitForReplication( __METHOD__, $this->ticket );
235  }
236  }
237  }
238 
239  private function getExplicitCategoriesChanges(
240  WikiPage $page, RevisionRecord $newRev, RevisionRecord $oldRev = null
241  ) {
242  // Inject the same timestamp for both revision parses to avoid seeing category changes
243  // due to time-based parser functions. Inject the same page title for the parses too.
244  // Note that REPEATABLE-READ makes template/file pages appear unchanged between parses.
245  $parseTimestamp = $newRev->getTimestamp();
246  // Parse the old rev and get the categories. Do not use link tables as that
247  // assumes these updates are perfectly FIFO and that link tables are always
248  // up to date, neither of which are true.
249  $oldCategories = $oldRev
250  ? $this->getCategoriesAtRev( $page, $oldRev, $parseTimestamp )
251  : [];
252  // Parse the new revision and get the categories
253  $newCategories = $this->getCategoriesAtRev( $page, $newRev, $parseTimestamp );
254 
255  $categoryInserts = array_values( array_diff( $newCategories, $oldCategories ) );
256  $categoryDeletes = array_values( array_diff( $oldCategories, $newCategories ) );
257 
258  return [ $categoryInserts, $categoryDeletes ];
259  }
260 
268  private function getCategoriesAtRev( WikiPage $page, RevisionRecord $rev, $parseTimestamp ) {
269  $services = MediaWikiServices::getInstance();
270  $options = $page->makeParserOptions( 'canonical' );
271  $options->setTimestamp( $parseTimestamp );
272  $options->setRenderReason( 'CategoryMembershipChangeJob' );
273 
274  $output = $rev instanceof RevisionStoreRecord && $rev->isCurrent()
275  ? $services->getParserCache()->get( $page, $options )
276  : null;
277 
278  if ( !$output || $output->getCacheRevisionId() !== $rev->getId() ) {
279  $output = $services->getRevisionRenderer()->getRenderedRevision( $rev, $options )
280  ->getRevisionParserOutput();
281  }
282 
283  // array keys will cast numeric category names to ints
284  // so we need to cast them back to strings to avoid breaking things!
285  return array_map( 'strval', array_keys( $output->getCategories() ) );
286  }
287 
288  public function getDeduplicationInfo() {
289  $info = parent::getDeduplicationInfo();
290  unset( $info['params']['revTimestamp'] ); // first job wins
291 
292  return $info;
293  }
294 }
const NS_CATEGORY
Definition: Defines.php:78
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Job to add recent change entries mentioning category membership changes.
static newSpec(PageIdentity $page, $revisionTimestamp)
__construct(PageIdentity $page, array $params)
Constructor for use by the Job Queue infrastructure.
getDeduplicationInfo()
Subclasses may need to override this to make duplication detection work.
notifyUpdatesForRevision(LBFactory $lbFactory, WikiPage $page, RevisionRecord $newRev)
Job queue task description base code.
Class to both describe a background job and handle jobs.
Definition: Job.php:39
Title $title
Definition: Job.php:50
setLastError( $error)
Definition: Job.php:469
array $params
Array of job parameters.
Definition: Job.php:44
A class containing constants representing the names of configuration variables.
Service locator for MediaWiki core services.
Page revision base class.
getParentId( $wikiId=self::LOCAL)
Get parent revision ID (the original previous page revision).
isCurrent()
Checks whether the revision record is a stored current revision.
getTimestamp()
MCR migration note: this replaced Revision::getTimestamp.
isDeleted( $field)
MCR migration note: this replaced Revision::isDeleted.
getId( $wikiId=self::LOCAL)
Get revision ID.
A RevisionRecord representing an existing revision persisted in the revision table.
const SRC_CATEGORIZE
static makeTitle( $ns, $title, $fragment='', $interwiki='')
Create a new Title from a namespace index and a DB key.
Definition: Title.php:641
Base representation for an editable wiki page.
Definition: WikiPage.php:70
static newFromID( $id, $from='fromdb')
Constructor from a page id.
Definition: WikiPage.php:222
makeParserOptions( $context)
Get parser options suitable for rendering the primary article wikitext.
Definition: WikiPage.php:1943
getId( $wikiId=self::LOCAL)
Definition: WikiPage.php:581
getTitle()
Get the title object of the article.
Definition: WikiPage.php:311
commitAndWaitForReplication( $fname, $ticket, array $opts=[])
Call commitPrimaryChanges() and waitForReplication() if $ticket indicates it is safe.
Definition: LBFactory.php:618
Interface for objects (potentially) representing an editable wiki page.
getId( $wikiId=self::LOCAL)
Returns the page ID.
Service for looking up page revisions.
const DB_REPLICA
Definition: defines.php:26
const DB_PRIMARY
Definition: defines.php:28
$revQuery