MediaWiki  master
CategoryMembershipChangeJob.php
Go to the documentation of this file.
1 <?php
28 
44  private $ticket;
45 
46  private const ENQUEUE_FUDGE_SEC = 60;
47 
53  public static function newSpec( PageIdentity $page, $revisionTimestamp ) {
54  return new JobSpecification(
55  'categoryMembershipChange',
56  [
57  'pageId' => $page->getId(),
58  'revTimestamp' => $revisionTimestamp,
59  ],
60  [
61  'removeDuplicates' => true,
62  'removeDuplicatesIgnoreParams' => [ 'revTimestamp' ]
63  ],
64  $page
65  );
66  }
67 
74  public function __construct( PageIdentity $page, array $params ) {
75  parent::__construct( 'categoryMembershipChange', $page, $params );
76  // Only need one job per page. Note that ENQUEUE_FUDGE_SEC handles races where an
77  // older revision job gets inserted while the newer revision job is de-duplicated.
78  $this->removeDuplicates = true;
79  }
80 
81  public function run() {
82  $services = MediaWikiServices::getInstance();
83  $lbFactory = $services->getDBLoadBalancerFactory();
84  $lb = $lbFactory->getMainLB();
85  $dbw = $lb->getConnectionRef( DB_PRIMARY );
86 
87  $this->ticket = $lbFactory->getEmptyTransactionTicket( __METHOD__ );
88 
89  $page = $services->getWikiPageFactory()->newFromID( $this->params['pageId'], WikiPage::READ_LATEST );
90  if ( !$page ) {
91  $this->setLastError( "Could not find page #{$this->params['pageId']}" );
92  return false; // deleted?
93  }
94 
95  // Cut down on the time spent in waitForPrimaryPos() in the critical section
96  $dbr = $lb->getConnectionRef( DB_REPLICA, [ 'recentchanges' ] );
97  if ( !$lb->waitForPrimaryPos( $dbr ) ) {
98  $this->setLastError( "Timed out while pre-waiting for replica DB to catch up" );
99  return false;
100  }
101 
102  // Use a named lock so that jobs for this page see each others' changes
103  $lockKey = "{$dbw->getDomainID()}:CategoryMembershipChange:{$page->getId()}"; // per-wiki
104  $scopedLock = $dbw->getScopedLockAndFlush( $lockKey, __METHOD__, 3 );
105  if ( !$scopedLock ) {
106  $this->setLastError( "Could not acquire lock '$lockKey'" );
107  return false;
108  }
109 
110  // Wait till replica DB is caught up so that jobs for this page see each others' changes
111  if ( !$lb->waitForPrimaryPos( $dbr ) ) {
112  $this->setLastError( "Timed out while waiting for replica DB to catch up" );
113  return false;
114  }
115  // Clear any stale REPEATABLE-READ snapshot
116  $dbr->flushSnapshot( __METHOD__ );
117 
118  $cutoffUnix = wfTimestamp( TS_UNIX, $this->params['revTimestamp'] );
119  // Using ENQUEUE_FUDGE_SEC handles jobs inserted out of revision order due to the delay
120  // between COMMIT and actual enqueueing of the CategoryMembershipChangeJob job.
121  $cutoffUnix -= self::ENQUEUE_FUDGE_SEC;
122 
123  // Get the newest page revision that has a SRC_CATEGORIZE row.
124  // Assume that category changes before it were already handled.
125  $row = $dbr->selectRow(
126  'revision',
127  [ 'rev_timestamp', 'rev_id' ],
128  [
129  'rev_page' => $page->getId(),
130  'rev_timestamp >= ' . $dbr->addQuotes( $dbr->timestamp( $cutoffUnix ) ),
131  'EXISTS (' . $dbr->selectSQLText(
132  'recentchanges',
133  '1',
134  [
135  'rc_this_oldid = rev_id',
136  'rc_source' => RecentChange::SRC_CATEGORIZE,
137  ],
138  __METHOD__
139  ) . ')'
140  ],
141  __METHOD__,
142  [ 'ORDER BY' => [ 'rev_timestamp DESC', 'rev_id DESC' ] ]
143  );
144  // Only consider revisions newer than any such revision
145  if ( $row ) {
146  $cutoffUnix = wfTimestamp( TS_UNIX, $row->rev_timestamp );
147  $lastRevId = (int)$row->rev_id;
148  } else {
149  $lastRevId = 0;
150  }
151 
152  // Find revisions to this page made around and after this revision which lack category
153  // notifications in recent changes. This lets jobs pick up were the last one left off.
154  $encCutoff = $dbr->addQuotes( $dbr->timestamp( $cutoffUnix ) );
155  $revisionStore = $services->getRevisionStore();
156  $revQuery = $revisionStore->getQueryInfo();
157  $res = $dbr->select(
158  $revQuery['tables'],
159  $revQuery['fields'],
160  [
161  'rev_page' => $page->getId(),
162  "rev_timestamp > $encCutoff" .
163  " OR (rev_timestamp = $encCutoff AND rev_id > $lastRevId)"
164  ],
165  __METHOD__,
166  [ 'ORDER BY' => [ 'rev_timestamp ASC', 'rev_id ASC' ] ],
167  $revQuery['joins']
168  );
169 
170  // Apply all category updates in revision timestamp order
171  foreach ( $res as $row ) {
172  $this->notifyUpdatesForRevision( $lbFactory, $page, $revisionStore->newRevisionFromRow( $row ) );
173  }
174 
175  return true;
176  }
177 
184  protected function notifyUpdatesForRevision(
185  LBFactory $lbFactory, WikiPage $page, RevisionRecord $newRev
186  ) {
187  $config = RequestContext::getMain()->getConfig();
188  $title = $page->getTitle();
189 
190  // Get the new revision
191  if ( $newRev->isDeleted( RevisionRecord::DELETED_TEXT ) ) {
192  return;
193  }
194 
195  $services = MediaWikiServices::getInstance();
196  // Get the prior revision (the same for null edits)
197  if ( $newRev->getParentId() ) {
198  $oldRev = $services->getRevisionLookup()
199  ->getRevisionById( $newRev->getParentId(), RevisionLookup::READ_LATEST );
200  if ( !$oldRev || $oldRev->isDeleted( RevisionRecord::DELETED_TEXT ) ) {
201  return;
202  }
203  } else {
204  $oldRev = null;
205  }
206 
207  // Parse the new revision and get the categories
208  $categoryChanges = $this->getExplicitCategoriesChanges( $page, $newRev, $oldRev );
209  list( $categoryInserts, $categoryDeletes ) = $categoryChanges;
210  if ( !$categoryInserts && !$categoryDeletes ) {
211  return; // nothing to do
212  }
213 
214  $blc = $services->getBacklinkCacheFactory()->getBacklinkCache( $title );
215  $catMembChange = new CategoryMembershipChange( $title, $blc, $newRev );
216  $catMembChange->checkTemplateLinks();
217 
218  $batchSize = $config->get( 'UpdateRowsPerQuery' );
219  $insertCount = 0;
220 
221  foreach ( $categoryInserts as $categoryName ) {
222  $categoryTitle = Title::makeTitle( NS_CATEGORY, $categoryName );
223  $catMembChange->triggerCategoryAddedNotification( $categoryTitle );
224  if ( $insertCount++ && ( $insertCount % $batchSize ) == 0 ) {
225  $lbFactory->commitAndWaitForReplication( __METHOD__, $this->ticket );
226  }
227  }
228 
229  foreach ( $categoryDeletes as $categoryName ) {
230  $categoryTitle = Title::makeTitle( NS_CATEGORY, $categoryName );
231  $catMembChange->triggerCategoryRemovedNotification( $categoryTitle );
232  if ( $insertCount++ && ( $insertCount++ % $batchSize ) == 0 ) {
233  $lbFactory->commitAndWaitForReplication( __METHOD__, $this->ticket );
234  }
235  }
236  }
237 
239  WikiPage $page, RevisionRecord $newRev, RevisionRecord $oldRev = null
240  ) {
241  // Inject the same timestamp for both revision parses to avoid seeing category changes
242  // due to time-based parser functions. Inject the same page title for the parses too.
243  // Note that REPEATABLE-READ makes template/file pages appear unchanged between parses.
244  $parseTimestamp = $newRev->getTimestamp();
245  // Parse the old rev and get the categories. Do not use link tables as that
246  // assumes these updates are perfectly FIFO and that link tables are always
247  // up to date, neither of which are true.
248  $oldCategories = $oldRev
249  ? $this->getCategoriesAtRev( $page, $oldRev, $parseTimestamp )
250  : [];
251  // Parse the new revision and get the categories
252  $newCategories = $this->getCategoriesAtRev( $page, $newRev, $parseTimestamp );
253 
254  $categoryInserts = array_values( array_diff( $newCategories, $oldCategories ) );
255  $categoryDeletes = array_values( array_diff( $oldCategories, $newCategories ) );
256 
257  return [ $categoryInserts, $categoryDeletes ];
258  }
259 
267  private function getCategoriesAtRev( WikiPage $page, RevisionRecord $rev, $parseTimestamp ) {
268  $services = MediaWikiServices::getInstance();
269  $options = $page->makeParserOptions( 'canonical' );
270  $options->setTimestamp( $parseTimestamp );
271 
272  $output = $rev instanceof RevisionStoreRecord && $rev->isCurrent()
273  ? $services->getParserCache()->get( $page, $options )
274  : null;
275 
276  if ( !$output || $output->getCacheRevisionId() !== $rev->getId() ) {
277  $output = $services->getRevisionRenderer()->getRenderedRevision( $rev, $options )
278  ->getRevisionParserOutput();
279  }
280 
281  // array keys will cast numeric category names to ints
282  // so we need to cast them back to strings to avoid breaking things!
283  return array_map( 'strval', array_keys( $output->getCategories() ) );
284  }
285 
286  public function getDeduplicationInfo() {
287  $info = parent::getDeduplicationInfo();
288  unset( $info['params']['revTimestamp'] ); // first job wins
289 
290  return $info;
291  }
292 }
Page\PageIdentity
Interface for objects (potentially) representing an editable wiki page.
Definition: PageIdentity.php:64
MediaWiki\Revision\RevisionRecord
Page revision base class.
Definition: RevisionRecord.php:47
MediaWiki\Revision\RevisionRecord\isCurrent
isCurrent()
Checks whether the revision record is a stored current revision.
Definition: RevisionRecord.php:587
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:193
CategoryMembershipChangeJob\notifyUpdatesForRevision
notifyUpdatesForRevision(LBFactory $lbFactory, WikiPage $page, RevisionRecord $newRev)
Definition: CategoryMembershipChangeJob.php:184
Wikimedia\Rdbms\LBFactory\commitAndWaitForReplication
commitAndWaitForReplication( $fname, $ticket, array $opts=[])
Call commitPrimaryChanges() and waitForReplication() if $ticket indicates it is safe.
Definition: LBFactory.php:533
wfTimestamp
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Definition: GlobalFunctions.php:1664
Job\$title
Title $title
Definition: Job.php:48
CategoryMembershipChangeJob\getCategoriesAtRev
getCategoriesAtRev(WikiPage $page, RevisionRecord $rev, $parseTimestamp)
Definition: CategoryMembershipChangeJob.php:267
WikiPage
Class representing a MediaWiki article and history.
Definition: WikiPage.php:60
Page\PageIdentity\getId
getId( $wikiId=self::LOCAL)
Returns the page ID.
WikiPage\makeParserOptions
makeParserOptions( $context)
Get parser options suitable for rendering the primary article wikitext.
Definition: WikiPage.php:1996
RecentChange\SRC_CATEGORIZE
const SRC_CATEGORIZE
Definition: RecentChange.php:89
$res
$res
Definition: testCompression.php:57
CategoryMembershipChangeJob\getDeduplicationInfo
getDeduplicationInfo()
Subclasses may need to override this to make duplication detection work.
Definition: CategoryMembershipChangeJob.php:286
Job\$params
array $params
Array of job parameters.
Definition: Job.php:42
$revQuery
$revQuery
Definition: testCompression.php:56
Job\setLastError
setLastError( $error)
Definition: Job.php:466
MediaWiki\Revision\RevisionLookup
Service for looking up page revisions.
Definition: RevisionLookup.php:38
$dbr
$dbr
Definition: testCompression.php:54
Job
Class to both describe a background job and handle jobs.
Definition: Job.php:37
WikiPage\getTitle
getTitle()
Get the title object of the article.
Definition: WikiPage.php:311
CategoryMembershipChangeJob\run
run()
Run the job.
Definition: CategoryMembershipChangeJob.php:81
Title\makeTitle
static makeTitle( $ns, $title, $fragment='', $interwiki='')
Create a new Title from a namespace index and a DB key.
Definition: Title.php:651
DB_REPLICA
const DB_REPLICA
Definition: defines.php:25
CategoryMembershipChangeJob
Job to add recent change entries mentioning category membership changes.
Definition: CategoryMembershipChangeJob.php:42
MediaWiki\Revision\RevisionRecord\getParentId
getParentId( $wikiId=self::LOCAL)
Get parent revision ID (the original previous page revision).
Definition: RevisionRecord.php:297
CategoryMembershipChangeJob\newSpec
static newSpec(PageIdentity $page, $revisionTimestamp)
Definition: CategoryMembershipChangeJob.php:53
DB_PRIMARY
const DB_PRIMARY
Definition: defines.php:27
CategoryMembershipChangeJob\$ticket
int null $ticket
Definition: CategoryMembershipChangeJob.php:44
MediaWiki\Revision\RevisionStoreRecord
A RevisionRecord representing an existing revision persisted in the revision table.
Definition: RevisionStoreRecord.php:39
RequestContext\getMain
static getMain()
Get the RequestContext object associated with the main request.
Definition: RequestContext.php:484
MediaWiki\Revision\RevisionRecord\getId
getId( $wikiId=self::LOCAL)
Get revision ID.
Definition: RevisionRecord.php:279
MediaWiki\Revision\RevisionRecord\isDeleted
isDeleted( $field)
MCR migration note: this replaced Revision::isDeleted.
Definition: RevisionRecord.php:437
JobSpecification
Job queue task description base code.
Definition: JobSpecification.php:42
Wikimedia\Rdbms\LBFactory
An interface for generating database load balancers.
Definition: LBFactory.php:42
CategoryMembershipChangeJob\ENQUEUE_FUDGE_SEC
const ENQUEUE_FUDGE_SEC
Definition: CategoryMembershipChangeJob.php:46
NS_CATEGORY
const NS_CATEGORY
Definition: Defines.php:78
MediaWiki\Revision\RevisionRecord\getTimestamp
getTimestamp()
MCR migration note: this replaced Revision::getTimestamp.
Definition: RevisionRecord.php:459
CategoryMembershipChangeJob\getExplicitCategoriesChanges
getExplicitCategoriesChanges(WikiPage $page, RevisionRecord $newRev, RevisionRecord $oldRev=null)
Definition: CategoryMembershipChangeJob.php:238
CategoryMembershipChange
Definition: CategoryMembershipChange.php:32
CategoryMembershipChangeJob\__construct
__construct(PageIdentity $page, array $params)
Constructor for use by the Job Queue infrastructure.
Definition: CategoryMembershipChangeJob.php:74