MediaWiki  master
CategoryMembershipChangeJob.php
Go to the documentation of this file.
1 <?php
24 
40  private $ticket;
41 
42  const ENQUEUE_FUDGE_SEC = 60;
43 
47  private $parserCache;
48 
54  public static function newSpec( Title $title, $revisionTimestamp ) {
55  return new JobSpecification(
56  'categoryMembershipChange',
57  [
58  'pageId' => $title->getArticleID(),
59  'revTimestamp' => $revisionTimestamp,
60  ],
61  [],
62  $title
63  );
64  }
65 
74  parent::__construct( 'categoryMembershipChange', $title, $params );
75  // Only need one job per page. Note that ENQUEUE_FUDGE_SEC handles races where an
76  // older revision job gets inserted while the newer revision job is de-duplicated.
77  $this->removeDuplicates = true;
78  $this->parserCache = $parserCache;
79  }
80 
81  public function run() {
82  $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
83  $lb = $lbFactory->getMainLB();
84  $dbw = $lb->getConnection( DB_MASTER );
85 
86  $this->ticket = $lbFactory->getEmptyTransactionTicket( __METHOD__ );
87 
88  $page = WikiPage::newFromID( $this->params['pageId'], WikiPage::READ_LATEST );
89  if ( !$page ) {
90  $this->setLastError( "Could not find page #{$this->params['pageId']}" );
91  return false; // deleted?
92  }
93 
94  // Cut down on the time spent in safeWaitForMasterPos() in the critical section
95  $dbr = $lb->getConnection( DB_REPLICA, [ 'recentchanges' ] );
96  if ( !$lb->safeWaitForMasterPos( $dbr ) ) {
97  $this->setLastError( "Timed out while pre-waiting for replica DB to catch up" );
98  return false;
99  }
100 
101  // Use a named lock so that jobs for this page see each others' changes
102  $lockKey = "CategoryMembershipUpdates:{$page->getId()}";
103  $scopedLock = $dbw->getScopedLockAndFlush( $lockKey, __METHOD__, 3 );
104  if ( !$scopedLock ) {
105  $this->setLastError( "Could not acquire lock '$lockKey'" );
106  return false;
107  }
108 
109  // Wait till replica DB is caught up so that jobs for this page see each others' changes
110  if ( !$lb->safeWaitForMasterPos( $dbr ) ) {
111  $this->setLastError( "Timed out while waiting for replica DB to catch up" );
112  return false;
113  }
114  // Clear any stale REPEATABLE-READ snapshot
115  $dbr->flushSnapshot( __METHOD__ );
116 
117  $cutoffUnix = wfTimestamp( TS_UNIX, $this->params['revTimestamp'] );
118  // Using ENQUEUE_FUDGE_SEC handles jobs inserted out of revision order due to the delay
119  // between COMMIT and actual enqueueing of the CategoryMembershipChangeJob job.
120  $cutoffUnix -= self::ENQUEUE_FUDGE_SEC;
121 
122  // Get the newest page revision that has a SRC_CATEGORIZE row.
123  // Assume that category changes before it were already handled.
124  $row = $dbr->selectRow(
125  'revision',
126  [ 'rev_timestamp', 'rev_id' ],
127  [
128  'rev_page' => $page->getId(),
129  'rev_timestamp >= ' . $dbr->addQuotes( $dbr->timestamp( $cutoffUnix ) ),
130  'EXISTS (' . $dbr->selectSQLText(
131  'recentchanges',
132  '1',
133  [
134  'rc_this_oldid = rev_id',
135  'rc_source' => RecentChange::SRC_CATEGORIZE,
136  // Allow rc_cur_id or rc_timestamp index usage
137  'rc_cur_id = rev_page',
138  'rc_timestamp = rev_timestamp'
139  ]
140  ) . ')'
141  ],
142  __METHOD__,
143  [ 'ORDER BY' => 'rev_timestamp DESC, rev_id DESC' ]
144  );
145  // Only consider revisions newer than any such revision
146  if ( $row ) {
147  $cutoffUnix = wfTimestamp( TS_UNIX, $row->rev_timestamp );
148  $lastRevId = (int)$row->rev_id;
149  } else {
150  $lastRevId = 0;
151  }
152 
153  // Find revisions to this page made around and after this revision which lack category
154  // notifications in recent changes. This lets jobs pick up were the last one left off.
155  $encCutoff = $dbr->addQuotes( $dbr->timestamp( $cutoffUnix ) );
157  $res = $dbr->select(
158  $revQuery['tables'],
159  $revQuery['fields'],
160  [
161  'rev_page' => $page->getId(),
162  "rev_timestamp > $encCutoff" .
163  " OR (rev_timestamp = $encCutoff AND rev_id > $lastRevId)"
164  ],
165  __METHOD__,
166  [ 'ORDER BY' => 'rev_timestamp ASC, rev_id ASC' ],
167  $revQuery['joins']
168  );
169 
170  // Apply all category updates in revision timestamp order
171  foreach ( $res as $row ) {
172  $this->notifyUpdatesForRevision( $lbFactory, $page, Revision::newFromRow( $row ) );
173  }
174 
175  return true;
176  }
177 
184  protected function notifyUpdatesForRevision(
185  LBFactory $lbFactory, WikiPage $page, Revision $newRev
186  ) {
187  $config = RequestContext::getMain()->getConfig();
188  $title = $page->getTitle();
189 
190  // Get the new revision
191  if ( !$newRev->getContent() ) {
192  return; // deleted?
193  }
194 
195  // Get the prior revision (the same for null edits)
196  if ( $newRev->getParentId() ) {
197  $oldRev = Revision::newFromId( $newRev->getParentId(), Revision::READ_LATEST );
198  if ( !$oldRev || !$oldRev->getContent() ) {
199  return; // deleted?
200  }
201  } else {
202  $oldRev = null;
203  }
204 
205  // Parse the new revision and get the categories
206  $categoryChanges = $this->getExplicitCategoriesChanges( $page, $newRev, $oldRev );
207  list( $categoryInserts, $categoryDeletes ) = $categoryChanges;
208  if ( !$categoryInserts && !$categoryDeletes ) {
209  return; // nothing to do
210  }
211 
212  $catMembChange = new CategoryMembershipChange( $title, $newRev );
213  $catMembChange->checkTemplateLinks();
214 
215  $batchSize = $config->get( 'UpdateRowsPerQuery' );
216  $insertCount = 0;
217 
218  foreach ( $categoryInserts as $categoryName ) {
219  $categoryTitle = Title::makeTitle( NS_CATEGORY, $categoryName );
220  $catMembChange->triggerCategoryAddedNotification( $categoryTitle );
221  if ( $insertCount++ && ( $insertCount % $batchSize ) == 0 ) {
222  $lbFactory->commitAndWaitForReplication( __METHOD__, $this->ticket );
223  }
224  }
225 
226  foreach ( $categoryDeletes as $categoryName ) {
227  $categoryTitle = Title::makeTitle( NS_CATEGORY, $categoryName );
228  $catMembChange->triggerCategoryRemovedNotification( $categoryTitle );
229  if ( $insertCount++ && ( $insertCount++ % $batchSize ) == 0 ) {
230  $lbFactory->commitAndWaitForReplication( __METHOD__, $this->ticket );
231  }
232  }
233  }
234 
236  WikiPage $page, Revision $newRev, Revision $oldRev = null
237  ) {
238  // Inject the same timestamp for both revision parses to avoid seeing category changes
239  // due to time-based parser functions. Inject the same page title for the parses too.
240  // Note that REPEATABLE-READ makes template/file pages appear unchanged between parses.
241  $parseTimestamp = $newRev->getTimestamp();
242  // Parse the old rev and get the categories. Do not use link tables as that
243  // assumes these updates are perfectly FIFO and that link tables are always
244  // up to date, neither of which are true.
245  $oldCategories = $oldRev
246  ? $this->getCategoriesAtRev( $page, $oldRev, $parseTimestamp )
247  : [];
248  // Parse the new revision and get the categories
249  $newCategories = $this->getCategoriesAtRev( $page, $newRev, $parseTimestamp );
250 
251  $categoryInserts = array_values( array_diff( $newCategories, $oldCategories ) );
252  $categoryDeletes = array_values( array_diff( $oldCategories, $newCategories ) );
253 
254  return [ $categoryInserts, $categoryDeletes ];
255  }
256 
264  private function getCategoriesAtRev( WikiPage $page, Revision $rev, $parseTimestamp ) {
265  $renderer = MediaWikiServices::getInstance()->getRevisionRenderer();
266  $options = $page->makeParserOptions( 'canonical' );
267  $options->setTimestamp( $parseTimestamp );
268 
269  $output = $rev->isCurrent() ? $this->parserCache->get( $page, $options ) : null;
270 
271  if ( !$output || $output->getCacheRevisionId() !== $rev->getId() ) {
272  $output = $renderer->getRenderedRevision( $rev->getRevisionRecord(), $options )
273  ->getRevisionParserOutput();
274  }
275 
276  // array keys will cast numeric category names to ints
277  // so we need to cast them back to strings to avoid breaking things!
278  return array_map( 'strval', array_keys( $output->getCategories() ) );
279  }
280 
281  public function getDeduplicationInfo() {
282  $info = parent::getDeduplicationInfo();
283  unset( $info['params']['revTimestamp'] ); // first job wins
284 
285  return $info;
286  }
287 }
The wiki should then use memcached to cache various data To use multiple just add more items to the array To increase the weight of a make its entry a array("192.168.0.1:11211", 2))
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition: deferred.txt:11
getArticleID( $flags=0)
Get the article ID for this Title from the link cache, adding it if necessary.
Definition: Title.php:2985
const SRC_CATEGORIZE
getRevisionRecord()
Definition: Revision.php:629
An interface for generating database load balancers.
Definition: LBFactory.php:39
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
getTimestamp()
Definition: Revision.php:994
notifyUpdatesForRevision(LBFactory $lbFactory, WikiPage $page, Revision $newRev)
getDeduplicationInfo()
Subclasses may need to override this to make duplication detection work.
Class to both describe a background job and handle jobs.
Definition: Job.php:30
commitAndWaitForReplication( $fname, $ticket, array $opts=[])
Convenience method for safely running commitMasterChanges()/waitForReplication()
Definition: LBFactory.php:461
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency MediaWikiServices
Definition: injection.txt:23
getContent( $audience=self::FOR_PUBLIC, User $user=null)
Fetch revision content if it&#39;s available to the specified audience.
Definition: Revision.php:915
const DB_MASTER
Definition: defines.php:26
setLastError( $error)
Definition: Job.php:436
static configuration should be added through ResourceLoaderGetConfigVars instead can be used to get the real title e g db for database replication lag or jobqueue for job queue size converted to pseudo seconds It is possible to add more fields and they will be returned to the user in the API response after the basic globals have been set but before ordinary actions take place $output
Definition: hooks.txt:2217
$newRev
Definition: pageupdater.txt:66
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
static getMain()
Get the RequestContext object associated with the main request.
getParentId()
Get parent revision ID (the original previous page revision)
Definition: Revision.php:717
getId()
Get revision ID.
Definition: Revision.php:638
$res
Definition: database.txt:21
getTitle()
Get the title object of the article.
Definition: WikiPage.php:294
const NS_CATEGORY
Definition: Defines.php:78
null means default in associative array with keys and values unescaped Should be merged with default with a value of false meaning to suppress the attribute in associative array with keys and values unescaped & $options
Definition: hooks.txt:1982
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that When $user is not null
Definition: hooks.txt:780
static getQueryInfo( $options=[])
Return the tables, fields, and join conditions to be selected to create a new revision object...
Definition: Revision.php:511
__construct(ParserCache $parserCache, Title $title, array $params)
Constructor for use by the Job Queue infrastructure.
presenting them properly to the user as errors is done by the caller return true use this to change the list i e etc $rev
Definition: hooks.txt:1766
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
getExplicitCategoriesChanges(WikiPage $page, Revision $newRev, Revision $oldRev=null)
makeParserOptions( $context)
Get parser options suitable for rendering the primary article wikitext.
Definition: WikiPage.php:1945
getCategoriesAtRev(WikiPage $page, Revision $rev, $parseTimestamp)
static makeTitle( $ns, $title, $fragment='', $interwiki='')
Create a new Title from a namespace index and a DB key.
Definition: Title.php:589
static newFromID( $id, $from='fromdb')
Constructor from a page id.
Definition: WikiPage.php:176
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
static newFromRow( $row)
Definition: Revision.php:222
static newSpec(Title $title, $revisionTimestamp)
$revQuery
array $params
Array of job parameters.
Definition: Job.php:35
const DB_REPLICA
Definition: defines.php:25
static newFromId( $id, $flags=0)
Load a page revision from a given revision ID number.
Definition: Revision.php:118
Title $title
Definition: Job.php:41
Job queue task description base code.