MediaWiki  master
RefreshLinksJob.php
Go to the documentation of this file.
1 <?php
25 
39 class RefreshLinksJob extends Job {
41  const PARSE_THRESHOLD_SEC = 1.0;
43  const CLOCK_FUDGE = 10;
45  const LAG_WAIT_TIMEOUT = 15;
46 
47  function __construct( Title $title, array $params ) {
48  parent::__construct( 'refreshLinks', $title, $params );
49  // Avoid the overhead of de-duplication when it would be pointless
50  $this->removeDuplicates = (
51  // Ranges rarely will line up
52  !isset( $params['range'] ) &&
53  // Multiple pages per job make matches unlikely
54  !( isset( $params['pages'] ) && count( $params['pages'] ) != 1 )
55  );
56  $this->params += [ 'causeAction' => 'unknown', 'causeAgent' => 'unknown' ];
57  // This will control transaction rounds in order to run DataUpdates
58  $this->executionFlags |= self::JOB_NO_EXPLICIT_TRX_ROUND;
59  }
60 
66  public static function newPrioritized( Title $title, array $params ) {
67  $job = new self( $title, $params );
68  $job->command = 'refreshLinksPrioritized';
69 
70  return $job;
71  }
72 
78  public static function newDynamic( Title $title, array $params ) {
79  $job = new self( $title, $params );
80  $job->command = 'refreshLinksDynamic';
81 
82  return $job;
83  }
84 
85  function run() {
86  global $wgUpdateRowsPerJob;
87 
88  $ok = true;
89  // Job to update all (or a range of) backlink pages for a page
90  if ( !empty( $this->params['recursive'] ) ) {
91  // When the base job branches, wait for the replica DBs to catch up to the master.
92  // From then on, we know that any template changes at the time the base job was
93  // enqueued will be reflected in backlink page parses when the leaf jobs run.
94  if ( !isset( $this->params['range'] ) ) {
95  $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
96  if ( !$lbFactory->waitForReplication( [
97  'domain' => $lbFactory->getLocalDomainID(),
98  'timeout' => self::LAG_WAIT_TIMEOUT
99  ] ) ) { // only try so hard
100  $stats = MediaWikiServices::getInstance()->getStatsdDataFactory();
101  $stats->increment( 'refreshlinks.lag_wait_failed' );
102  }
103  }
104  // Carry over information for de-duplication
105  $extraParams = $this->getRootJobParams();
106  $extraParams['triggeredRecursive'] = true;
107  // Carry over cause information for logging
108  $extraParams['causeAction'] = $this->params['causeAction'];
109  $extraParams['causeAgent'] = $this->params['causeAgent'];
110  // Convert this into no more than $wgUpdateRowsPerJob RefreshLinks per-title
111  // jobs and possibly a recursive RefreshLinks job for the rest of the backlinks
113  $this,
114  $wgUpdateRowsPerJob,
115  1, // job-per-title
116  [ 'params' => $extraParams ]
117  );
118  JobQueueGroup::singleton()->push( $jobs );
119  // Job to update link tables for a set of titles
120  } elseif ( isset( $this->params['pages'] ) ) {
121  foreach ( $this->params['pages'] as list( $ns, $dbKey ) ) {
122  $title = Title::makeTitleSafe( $ns, $dbKey );
123  if ( $title ) {
124  $this->runForTitle( $title );
125  } else {
126  $ok = false;
127  $this->setLastError( "Invalid title ($ns,$dbKey)." );
128  }
129  }
130  // Job to update link tables for a given title
131  } else {
132  $this->runForTitle( $this->title );
133  }
134 
135  return $ok;
136  }
137 
142  protected function runForTitle( Title $title ) {
143  $services = MediaWikiServices::getInstance();
144  $stats = $services->getStatsdDataFactory();
145  $lbFactory = $services->getDBLoadBalancerFactory();
146  $revisionStore = $services->getRevisionStore();
147  $renderer = $services->getRevisionRenderer();
148  $ticket = $lbFactory->getEmptyTransactionTicket( __METHOD__ );
149 
150  $lbFactory->beginMasterChanges( __METHOD__ );
151 
152  $page = WikiPage::factory( $title );
153  $page->loadPageData( WikiPage::READ_LATEST );
154 
155  // Serialize links updates by page ID so they see each others' changes
156  $dbw = $lbFactory->getMainLB()->getConnection( DB_MASTER );
158  $scopedLock = LinksUpdate::acquirePageLock( $dbw, $page->getId(), 'job' );
159  if ( $scopedLock === null ) {
160  $lbFactory->commitMasterChanges( __METHOD__ );
161  // Another job is already updating the page, likely for an older revision (T170596).
162  $this->setLastError( 'LinksUpdate already running for this page, try again later.' );
163  return false;
164  }
165  // Get the latest ID *after* acquirePageLock() flushed the transaction.
166  // This is used to detect edits/moves after loadPageData() but before the scope lock.
167  // The works around the chicken/egg problem of determining the scope lock key.
168  $latest = $title->getLatestRevID( Title::GAID_FOR_UPDATE );
169 
170  if ( !empty( $this->params['triggeringRevisionId'] ) ) {
171  // Fetch the specified revision; lockAndGetLatest() below detects if the page
172  // was edited since and aborts in order to avoid corrupting the link tables
173  $revision = $revisionStore->getRevisionById(
174  (int)$this->params['triggeringRevisionId'],
175  Revision::READ_LATEST
176  );
177  } else {
178  // Fetch current revision; READ_LATEST reduces lockAndGetLatest() check failures
179  $revision = $revisionStore->getRevisionByTitle( $title, 0, Revision::READ_LATEST );
180  }
181 
182  if ( !$revision ) {
183  $lbFactory->commitMasterChanges( __METHOD__ );
184  $stats->increment( 'refreshlinks.rev_not_found' );
185  $this->setLastError( "Revision not found for {$title->getPrefixedDBkey()}" );
186  return false; // just deleted?
187  } elseif ( $revision->getId() != $latest || $revision->getPageId() !== $page->getId() ) {
188  $lbFactory->commitMasterChanges( __METHOD__ );
189  // Do not clobber over newer updates with older ones. If all jobs where FIFO and
190  // serialized, it would be OK to update links based on older revisions since it
191  // would eventually get to the latest. Since that is not the case (by design),
192  // only update the link tables to a state matching the current revision's output.
193  $stats->increment( 'refreshlinks.rev_not_current' );
194  $this->setLastError( "Revision {$revision->getId()} is not current" );
195  return false;
196  }
197 
198  $parserOutput = false;
199  $parserOptions = $page->makeParserOptions( 'canonical' );
200  // If page_touched changed after this root job, then it is likely that
201  // any views of the pages already resulted in re-parses which are now in
202  // cache. The cache can be reused to avoid expensive parsing in some cases.
203  if ( isset( $this->params['rootJobTimestamp'] ) ) {
204  $opportunistic = !empty( $this->params['isOpportunistic'] );
205 
206  $skewedTimestamp = $this->params['rootJobTimestamp'];
207  if ( $opportunistic ) {
208  // Neither clock skew nor DB snapshot/replica DB lag matter much for such
209  // updates; focus on reusing the (often recently updated) cache
210  } else {
211  // For transclusion updates, the template changes must be reflected
212  $skewedTimestamp = wfTimestamp( TS_MW,
213  wfTimestamp( TS_UNIX, $skewedTimestamp ) + self::CLOCK_FUDGE
214  );
215  }
216 
217  if ( $page->getLinksTimestamp() > $skewedTimestamp ) {
218  $lbFactory->commitMasterChanges( __METHOD__ );
219  // Something already updated the backlinks since this job was made
220  $stats->increment( 'refreshlinks.update_skipped' );
221  return true;
222  }
223 
224  if ( $page->getTouched() >= $this->params['rootJobTimestamp'] || $opportunistic ) {
225  // Cache is suspected to be up-to-date. As long as the cache rev ID matches
226  // and it reflects the job's triggering change, then it is usable.
227  $parserOutput = $services->getParserCache()->getDirty( $page, $parserOptions );
228  if ( !$parserOutput
229  || $parserOutput->getCacheRevisionId() != $revision->getId()
230  || $parserOutput->getCacheTime() < $skewedTimestamp
231  ) {
232  $parserOutput = false; // too stale
233  }
234  }
235  }
236 
237  // Fetch the current revision and parse it if necessary...
238  if ( $parserOutput ) {
239  $stats->increment( 'refreshlinks.parser_cached' );
240  } else {
241  $start = microtime( true );
242 
243  $checkCache = $page->shouldCheckParserCache( $parserOptions, $revision->getId() );
244 
245  // Revision ID must be passed to the parser output to get revision variables correct
246  $renderedRevision = $renderer->getRenderedRevision(
247  $revision,
248  $parserOptions,
249  null,
250  [
251  // use master, for consistency with the getRevisionByTitle call above.
252  'use-master' => true,
253  // bypass audience checks, since we know that this is the current revision.
254  'audience' => RevisionRecord::RAW
255  ]
256  );
257  $parserOutput = $renderedRevision->getRevisionParserOutput(
258  // HTML is only needed if the output is to be placed in the parser cache
259  [ 'generate-html' => $checkCache ]
260  );
261 
262  // If it took a long time to render, then save this back to the cache to avoid
263  // wasted CPU by other apaches or job runners. We don't want to always save to
264  // cache as this can cause high cache I/O and LRU churn when a template changes.
265  $elapsed = microtime( true ) - $start;
266 
267  $parseThreshold = $this->params['parseThreshold'] ?? self::PARSE_THRESHOLD_SEC;
268 
269  if ( $checkCache && $elapsed >= $parseThreshold && $parserOutput->isCacheable() ) {
270  $ctime = wfTimestamp( TS_MW, (int)$start ); // cache time
271  $services->getParserCache()->save(
272  $parserOutput, $page, $parserOptions, $ctime, $revision->getId()
273  );
274  }
275  $stats->increment( 'refreshlinks.parser_uncached' );
276  }
277 
278  $options = [
279  'recursive' => !empty( $this->params['useRecursiveLinksUpdate'] ),
280  // Carry over cause so the update can do extra logging
281  'causeAction' => $this->params['causeAction'],
282  'causeAgent' => $this->params['causeAgent']
283  ];
284  if ( !empty( $this->params['triggeringUser'] ) ) {
285  $userInfo = $this->params['triggeringUser'];
286  if ( $userInfo['userId'] ) {
287  $options['triggeringUser'] = User::newFromId( $userInfo['userId'] );
288  } else {
289  // Anonymous, use the username
290  $options['triggeringUser'] = User::newFromName( $userInfo['userName'], false );
291  }
292  }
293 
294  $lbFactory->commitMasterChanges( __METHOD__ );
295 
296  $page->doSecondaryDataUpdates( $options );
297 
298  InfoAction::invalidateCache( $title );
299 
300  // Commit any writes here in case this method is called in a loop.
301  // In that case, the scoped lock will fail to be acquired.
302  $lbFactory->commitAndWaitForReplication( __METHOD__, $ticket );
303 
304  return true;
305  }
306 
307  public function getDeduplicationInfo() {
308  $info = parent::getDeduplicationInfo();
309  unset( $info['causeAction'] );
310  unset( $info['causeAgent'] );
311  if ( is_array( $info['params'] ) ) {
312  // For per-pages jobs, the job title is that of the template that changed
313  // (or similar), so remove that since it ruins duplicate detection
314  if ( isset( $info['params']['pages'] ) ) {
315  unset( $info['namespace'] );
316  unset( $info['title'] );
317  }
318  }
319 
320  return $info;
321  }
322 
323  public function workItemCount() {
324  if ( !empty( $this->params['recursive'] ) ) {
325  return 0; // nothing actually refreshed
326  } elseif ( isset( $this->params['pages'] ) ) {
327  return count( $this->params['pages'] );
328  }
329 
330  return 1; // one title
331  }
332 }
static factory(Title $title)
Create a WikiPage object of the appropriate class for the given title.
Definition: WikiPage.php:138
__construct(Title $title, array $params)
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition: deferred.txt:11
run()
Run the job.
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
$wgUpdateRowsPerJob
Number of rows to update per job.
Class to both describe a background job and handle jobs.
Definition: Job.php:30
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency MediaWikiServices
Definition: injection.txt:23
title
const DB_MASTER
Definition: defines.php:26
setLastError( $error)
Definition: Job.php:453
static acquirePageLock(IDatabase $dbw, $pageId, $why='atomicity')
Acquire a lock for performing link table updates for a page on a DB.
runForTitle(Title $title)
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
static invalidateCache(Title $title, $revid=null)
Clear the info cache for a given Title.
Definition: InfoAction.php:70
static configuration should be added through ResourceLoaderGetConfigVars instead can be used to get the real title e g db for database replication lag or jobqueue for job queue size converted to pseudo seconds It is possible to add more fields and they will be returned to the user in the API response after the basic globals have been set but before ordinary actions take place or wrap services the preferred way to define a new service is the $wgServiceWiringFiles array $services
Definition: hooks.txt:2217
getRootJobParams()
Definition: Job.php:348
const GAID_FOR_UPDATE
Used to be GAID_FOR_UPDATE define.
Definition: Title.php:55
null means default in associative array with keys and values unescaped Should be merged with default with a value of false meaning to suppress the attribute in associative array with keys and values unescaped & $options
Definition: hooks.txt:1982
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that When $user is not null
Definition: hooks.txt:780
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
Job to update link tables for pages.
static makeTitleSafe( $ns, $title, $fragment='', $interwiki='')
Create a new Title from a namespace index and a DB key.
Definition: Title.php:617
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
static newDynamic(Title $title, array $params)
static newFromId( $id)
Static factory method for creation from a given user ID.
Definition: User.php:618
if(count( $args)< 1) $job
getDeduplicationInfo()
Subclasses may need to override this to make duplication detection work.
static newPrioritized(Title $title, array $params)
getLatestRevID( $flags=0)
What is the page_latest field for this page?
Definition: Title.php:3092
static singleton( $domain=false)
array $params
Array of job parameters.
Definition: Job.php:35
static partitionBacklinkJob(Job $job, $bSize, $cSize, $opts=[])
Break down $job into approximately ($bSize/$cSize) leaf jobs and a single partition job that covers t...
static newFromName( $name, $validate='valid')
Static factory method for creation from username.
Definition: User.php:594
Title $title
Definition: Job.php:41