MediaWiki  master
DeferredUpdates.php
Go to the documentation of this file.
1 <?php
32 
64  private static $preSendUpdates = [];
66  private static $postSendUpdates = [];
67 
68  const ALL = 0; // all updates; in web requests, use only after flushing the output buffer
69  const PRESEND = 1; // for updates that should run before flushing output buffer
70  const POSTSEND = 2; // for updates that should run after flushing output buffer
71 
72  const BIG_QUEUE_SIZE = 100;
73 
75  private static $executeContext;
76 
85  public static function addUpdate( DeferrableUpdate $update, $stage = self::POSTSEND ) {
86  global $wgCommandLineMode;
87 
88  if (
89  self::$executeContext &&
90  self::$executeContext['stage'] >= $stage &&
91  !( $update instanceof MergeableUpdate )
92  ) {
93  // This is a sub-DeferredUpdate; run it right after its parent update.
94  // Also, while post-send updates are running, push any "pre-send" jobs to the
95  // active post-send queue to make sure they get run this round (or at all).
96  self::$executeContext['subqueue'][] = $update;
97 
98  return;
99  }
100 
101  if ( $stage === self::PRESEND ) {
102  self::push( self::$preSendUpdates, $update );
103  } else {
104  self::push( self::$postSendUpdates, $update );
105  }
106 
107  // Try to run the updates now if in CLI mode and no transaction is active.
108  // This covers scripts that don't/barely use the DB but make updates to other stores.
109  if ( $wgCommandLineMode ) {
110  self::tryOpportunisticExecute( 'run' );
111  }
112  }
113 
124  public static function addCallableUpdate(
125  $callable, $stage = self::POSTSEND, $dbw = null
126  ) {
127  self::addUpdate( new MWCallableUpdate( $callable, wfGetCaller(), $dbw ), $stage );
128  }
129 
139  public static function doUpdates( $mode = 'run', $stage = self::ALL ) {
140  $stageEffective = ( $stage === self::ALL ) ? self::POSTSEND : $stage;
141  // For ALL mode, make sure that any PRESEND updates added along the way get run.
142  // Normally, these use the subqueue, but that isn't true for MergeableUpdate items.
143  do {
144  if ( $stage === self::ALL || $stage === self::PRESEND ) {
145  self::handleUpdateQueue( self::$preSendUpdates, $mode, $stageEffective );
146  }
147 
148  if ( $stage === self::ALL || $stage == self::POSTSEND ) {
149  self::handleUpdateQueue( self::$postSendUpdates, $mode, $stageEffective );
150  }
151  } while ( $stage === self::ALL && self::$preSendUpdates );
152  }
153 
158  private static function push( array &$queue, DeferrableUpdate $update ) {
159  if ( $update instanceof MergeableUpdate ) {
160  $class = get_class( $update ); // fully-qualified class
161  if ( isset( $queue[$class] ) ) {
163  $existingUpdate = $queue[$class];
164  $existingUpdate->merge( $update );
165  // Move the update to the end to handle things like mergeable purge
166  // updates that might depend on the prior updates in the queue running
167  unset( $queue[$class] );
168  $queue[$class] = $existingUpdate;
169  } else {
170  $queue[$class] = $update;
171  }
172  } else {
173  $queue[] = $update;
174  }
175  }
176 
186  protected static function handleUpdateQueue( array &$queue, $mode, $stage ) {
187  $services = MediaWikiServices::getInstance();
188  $stats = $services->getStatsdDataFactory();
189  $lbf = $services->getDBLoadBalancerFactory();
190  $logger = LoggerFactory::getInstance( 'DeferredUpdates' );
191  $httpMethod = $services->getMainConfig()->get( 'CommandLineMode' )
192  ? 'cli'
193  : strtolower( RequestContext::getMain()->getRequest()->getMethod() );
194 
196  $guiEx = null;
198  $updates = $queue;
199 
200  // Keep doing rounds of updates until none get enqueued...
201  while ( $updates ) {
202  $queue = []; // clear the queue
203 
204  // Segregate the queue into one for DataUpdate and one for everything else
205  $dataUpdateQueue = [];
206  $genericUpdateQueue = [];
207  foreach ( $updates as $update ) {
208  if ( $update instanceof DataUpdate ) {
209  $dataUpdateQueue[] = $update;
210  } else {
211  $genericUpdateQueue[] = $update;
212  }
213  }
214  // Execute all DataUpdate queue followed by the DeferrableUpdate queue...
215  foreach ( [ $dataUpdateQueue, $genericUpdateQueue ] as $updateQueue ) {
216  foreach ( $updateQueue as $du ) {
217  // Enqueue the task into the job queue system instead if applicable
218  if ( $mode === 'enqueue' && $du instanceof EnqueueableDataUpdate ) {
219  self::jobify( $du, $lbf, $logger, $stats, $httpMethod );
220  continue;
221  }
222  // Otherwise, execute the task and any subtasks that it spawns
223  self::$executeContext = [ 'stage' => $stage, 'subqueue' => [] ];
224  try {
225  $e = self::run( $du, $lbf, $logger, $stats, $httpMethod );
226  $guiEx = $guiEx ?: ( $e instanceof ErrorPageError ? $e : null );
227  // Do the subqueue updates for $update until there are none
228  while ( self::$executeContext['subqueue'] ) {
229  $duChild = reset( self::$executeContext['subqueue'] );
230  $firstKey = key( self::$executeContext['subqueue'] );
231  unset( self::$executeContext['subqueue'][$firstKey] );
232 
233  $e = self::run( $duChild, $lbf, $logger, $stats, $httpMethod );
234  $guiEx = $guiEx ?: ( $e instanceof ErrorPageError ? $e : null );
235  }
236  } finally {
237  // Make sure we always clean up the context.
238  // Losing updates while rewinding the stack is acceptable,
239  // losing updates that are added later is not.
240  self::$executeContext = null;
241  }
242  }
243  }
244 
245  $updates = $queue; // new snapshot of queue (check for new entries)
246  }
247 
248  // Throw the first of any GUI errors as long as the context is HTTP pre-send. However,
249  // callers should check permissions *before* enqueueing updates. If the main transaction
250  // round actions succeed but some deferred updates fail due to permissions errors then
251  // there is a risk that some secondary data was not properly updated.
252  if ( $guiEx && $stage === self::PRESEND && !headers_sent() ) {
253  throw $guiEx;
254  }
255  }
256 
267  private static function run(
268  DeferrableUpdate $update,
269  LBFactory $lbFactory,
270  LoggerInterface $logger,
271  StatsdDataFactoryInterface $stats,
272  $httpMethod
273  ) {
274  $name = get_class( $update );
275  $suffix = ( $update instanceof DeferrableCallback ) ? "_{$update->getOrigin()}" : '';
276  $stats->increment( "deferred_updates.$httpMethod.{$name}{$suffix}" );
277 
278  $e = null;
279  try {
280  self::attemptUpdate( $update, $lbFactory );
281  } catch ( Exception $e ) {
282  } catch ( Throwable $e ) {
283  }
284 
285  if ( $e ) {
286  $logger->error(
287  "Deferred update {type} failed: {message}",
288  [
289  'type' => $name . $suffix,
290  'message' => $e->getMessage(),
291  'trace' => $e->getTraceAsString()
292  ]
293  );
294  $lbFactory->rollbackMasterChanges( __METHOD__ );
295  // VW-style hack to work around T190178, so we can make sure
296  // PageMetaDataUpdater doesn't throw exceptions.
297  if ( defined( 'MW_PHPUNIT_TEST' ) ) {
298  throw $e;
299  }
300  }
301 
302  return $e;
303  }
304 
314  private static function jobify(
315  EnqueueableDataUpdate $update,
316  LBFactory $lbFactory,
317  LoggerInterface $logger,
318  StatsdDataFactoryInterface $stats,
319  $httpMethod
320  ) {
321  $stats->increment( "deferred_updates.$httpMethod." . get_class( $update ) );
322 
323  $e = null;
324  try {
325  $spec = $update->getAsJobSpecification();
326  JobQueueGroup::singleton( $spec['domain'] ?? $spec['wiki'] )->push( $spec['job'] );
327  } catch ( Exception $e ) {
328  } catch ( Throwable $e ) {
329  }
330 
331  if ( $e ) {
332  $logger->error(
333  "Job insertion of deferred update {type} failed: {message}",
334  [
335  'type' => get_class( $update ),
336  'message' => $e->getMessage(),
337  'trace' => $e->getTraceAsString()
338  ]
339  );
340  $lbFactory->rollbackMasterChanges( __METHOD__ );
341  }
342  }
343 
355  public static function attemptUpdate( DeferrableUpdate $update, ILBFactory $lbFactory ) {
356  $ticket = $lbFactory->getEmptyTransactionTicket( __METHOD__ );
357  if ( !$ticket || $lbFactory->hasTransactionRound() ) {
358  throw new DBTransactionError( null, "A database transaction round is pending." );
359  }
360 
361  if ( $update instanceof DataUpdate ) {
362  $update->setTransactionTicket( $ticket );
363  }
364 
365  $fnameTrxOwner = get_class( $update ) . '::doUpdate';
366  $useExplicitTrxRound = !(
367  $update instanceof TransactionRoundAwareUpdate &&
368  $update->getTransactionRoundRequirement() == $update::TRX_ROUND_ABSENT
369  );
370  // Flush any pending changes left over from an implicit transaction round
371  if ( $useExplicitTrxRound ) {
372  $lbFactory->beginMasterChanges( $fnameTrxOwner ); // new explicit round
373  } else {
374  $lbFactory->commitMasterChanges( $fnameTrxOwner ); // new implicit round
375  }
376  // Run the update after any stale master view snapshots have been flushed
377  $update->doUpdate();
378  // Commit any pending changes from the explicit or implicit transaction round
379  $lbFactory->commitMasterChanges( $fnameTrxOwner );
380  }
381 
393  public static function tryOpportunisticExecute( $mode = 'run' ) {
394  // execute() loop is already running
395  if ( self::$executeContext ) {
396  return false;
397  }
398 
399  // Avoiding running updates without them having outer scope
400  if ( !self::areDatabaseTransactionsActive() ) {
401  self::doUpdates( $mode );
402  return true;
403  }
404 
405  if ( self::pendingUpdatesCount() >= self::BIG_QUEUE_SIZE ) {
406  // If we cannot run the updates with outer transaction context, try to
407  // at least enqueue all the updates that support queueing to job queue
408  self::$preSendUpdates = self::enqueueUpdates( self::$preSendUpdates );
409  self::$postSendUpdates = self::enqueueUpdates( self::$postSendUpdates );
410  }
411 
412  return !self::pendingUpdatesCount();
413  }
414 
421  private static function enqueueUpdates( array $updates ) {
422  $remaining = [];
423 
424  foreach ( $updates as $update ) {
425  if ( $update instanceof EnqueueableDataUpdate ) {
426  $spec = $update->getAsJobSpecification();
427  $domain = $spec['domain'] ?? $spec['wiki'];
428  JobQueueGroup::singleton( $domain )->push( $spec['job'] );
429  } else {
430  $remaining[] = $update;
431  }
432  }
433 
434  return $remaining;
435  }
436 
441  public static function pendingUpdatesCount() {
442  return count( self::$preSendUpdates ) + count( self::$postSendUpdates );
443  }
444 
450  public static function getPendingUpdates( $stage = self::ALL ) {
451  $updates = [];
452  if ( $stage === self::ALL || $stage === self::PRESEND ) {
453  $updates = array_merge( $updates, self::$preSendUpdates );
454  }
455  if ( $stage === self::ALL || $stage === self::POSTSEND ) {
456  $updates = array_merge( $updates, self::$postSendUpdates );
457  }
458  return $updates;
459  }
460 
465  public static function clearPendingUpdates() {
466  self::$preSendUpdates = [];
467  self::$postSendUpdates = [];
468  }
469 
473  private static function areDatabaseTransactionsActive() {
474  $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
475  if ( $lbFactory->hasTransactionRound() || !$lbFactory->isReadyForRoundOperations() ) {
476  return true;
477  }
478 
479  $connsBusy = false;
480  $lbFactory->forEachLB( function ( LoadBalancer $lb ) use ( &$connsBusy ) {
481  $lb->forEachOpenMasterConnection( function ( IDatabase $conn ) use ( &$connsBusy ) {
482  if ( $conn->writesOrCallbacksPending() || $conn->explicitTrxActive() ) {
483  $connsBusy = true;
484  }
485  } );
486  } );
487 
488  return $connsBusy;
489  }
490 }
beginMasterChanges( $fname=__METHOD__)
Flush any master transaction snapshots and set DBO_TRX (if DBO_DEFAULT is set)
static enqueueUpdates(array $updates)
Enqueue a job for each EnqueueableDataUpdate item and return the other items.
static DeferrableUpdate [] $postSendUpdates
Updates to be deferred until after request end.
Deferrable update that specifies whether it must run outside of any explicit LBFactory transaction ro...
static clearPendingUpdates()
Clear all pending updates without performing them.
An interface for generating database load balancers.
Definition: LBFactory.php:40
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
div flags Integer display flags(NO_ACTION_LINK, NO_EXTRA_USER_LINKS) 'LogException' returning false will NOT prevent logging $e
Definition: hooks.txt:2147
static attemptUpdate(DeferrableUpdate $update, ILBFactory $lbFactory)
Attempt to run an update with the appropriate transaction round state it expects. ...
Interface that marks a DataUpdate as enqueuable via the JobQueue.
Interface that deferrable updates can implement to signal that updates can be combined.
either a unescaped string or a HtmlArmor object after in associative array form externallinks including delete and has completed for all link tables whether this was an auto creation use $formDescriptor instead default is conds Array Extra conditions for the No matching items in log is displayed if loglist is empty msgKey Array If you want a nice box with a set this to the key of the message First element is the message key
Definition: hooks.txt:2139
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency MediaWikiServices
Definition: injection.txt:23
getEmptyTransactionTicket( $fname)
Get a token asserting that no transaction writes are active.
static array null $executeContext
Information about the current execute() call or null if not running.
static areDatabaseTransactionsActive()
static run(DeferrableUpdate $update, LBFactory $lbFactory, LoggerInterface $logger, StatsdDataFactoryInterface $stats, $httpMethod)
Run a task and catch/log any exceptions.
writesOrCallbacksPending()
Whether there is a transaction open with either possible write queries or unresolved pre-commit/commi...
static pendingUpdatesCount()
static addCallableUpdate( $callable, $stage=self::POSTSEND, $dbw=null)
Add a callable update.
static getMain()
Get the RequestContext object associated with the main request.
static configuration should be added through ResourceLoaderGetConfigVars instead can be used to get the real title e g db for database replication lag or jobqueue for job queue size converted to pseudo seconds It is possible to add more fields and they will be returned to the user in the API response after the basic globals have been set but before ordinary actions take place or wrap services the preferred way to define a new service is the $wgServiceWiringFiles array $services
Definition: hooks.txt:2205
An error page which can definitely be safely rendered using the OutputPage.
Callback wrapper that has an originating method.
forEachOpenMasterConnection( $callback, array $params=[])
Call a function with each open connection object to a master.
hasTransactionRound()
Check if an explicit transaction round is active.
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that When $user is not null
Definition: hooks.txt:767
commitMasterChanges( $fname=__METHOD__, array $options=[])
Commit changes and clear view snapshots on all master connections.
wfGetCaller( $level=2)
Get the name of the function which called this function wfGetCaller( 1 ) is the function with the wfG...
static getPendingUpdates( $stage=self::ALL)
static handleUpdateQueue(array &$queue, $mode, $stage)
Immediately run or enqueue a list of updates.
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
static tryOpportunisticExecute( $mode='run')
Run all deferred updates immediately if there are no DB writes active.
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
static doUpdates( $mode='run', $stage=self::ALL)
Do any deferred updates and clear the list.
static addUpdate(DeferrableUpdate $update, $stage=self::POSTSEND)
Add an update to the deferred list to be run later by execute()
An interface for generating database load balancers.
Definition: ILBFactory.php:33
Basic database interface for live and lazy-loaded relation database handles.
Definition: IDatabase.php:38
global $wgCommandLineMode
rollbackMasterChanges( $fname=__METHOD__)
Rollback changes on all master connections.
Definition: LBFactory.php:302
static push(array &$queue, DeferrableUpdate $update)
doUpdate()
Perform the actual work.
static jobify(EnqueueableDataUpdate $update, LBFactory $lbFactory, LoggerInterface $logger, StatsdDataFactoryInterface $stats, $httpMethod)
Push a task into the job queue system and catch/log any exceptions.
MediaWiki Logger LoggerFactory implements a PSR [0] compatible message logging system Named Psr Log LoggerInterface instances can be obtained from the MediaWiki Logger LoggerFactory::getInstance() static method. MediaWiki\Logger\LoggerFactory expects a class implementing the MediaWiki\Logger\Spi interface to act as a factory for new Psr\Log\LoggerInterface instances. The "Spi" in MediaWiki\Logger\Spi stands for "service provider interface". An SPI is an API intended to be implemented or extended by a third party. This software design pattern is intended to enable framework extension and replaceable components. It is specifically used in the MediaWiki\Logger\LoggerFactory service to allow alternate PSR-3 logging implementations to be easily integrated with MediaWiki. The service provider interface allows the backend logging library to be implemented in multiple ways. The $wgMWLoggerDefaultSpi global provides the classname of the default MediaWiki\Logger\Spi implementation to be loaded at runtime. This can either be the name of a class implementing the MediaWiki\Logger\Spi with a zero argument const ructor or a callable that will return an MediaWiki\Logger\Spi instance. Alternately the MediaWiki\Logger\LoggerFactory MediaWiki Logger LoggerFactory
Definition: logger.txt:5
static singleton( $domain=false)
Allows to change the fields on the form that will be generated $name
Definition: hooks.txt:271
Database connection, tracking, load balancing, and transaction manager for a cluster.
static DeferrableUpdate [] $preSendUpdates
Updates to be deferred until before request end.