MediaWiki  master
JobQueueGroup.php
Go to the documentation of this file.
1 <?php
23 
32  protected static $instances = [];
33 
35  protected $cache;
36 
38  protected $domain;
40  protected $readOnlyReason;
42  protected $invalidDomain = false;
43 
45  protected $coalescedQueues;
46 
47  const TYPE_DEFAULT = 1; // integer; jobs popped by default
48  const TYPE_ANY = 2; // integer; any job
49 
50  const USE_CACHE = 1; // integer; use process or persistent cache
51 
52  const PROC_CACHE_TTL = 15; // integer; seconds
53 
54  const CACHE_VERSION = 1; // integer; cache version
55 
60  protected function __construct( $domain, $readOnlyReason ) {
61  $this->domain = $domain;
62  $this->readOnlyReason = $readOnlyReason;
63  $this->cache = new MapCacheLRU( 10 );
64  }
65 
70  public static function singleton( $domain = false ) {
71  global $wgLocalDatabases;
72 
73  if ( $domain === false ) {
75  }
76 
77  if ( !isset( self::$instances[$domain] ) ) {
78  self::$instances[$domain] = new self( $domain, wfConfiguredReadOnlyReason() );
79  // Make sure jobs are not getting pushed to bogus wikis. This can confuse
80  // the job runner system into spawning endless RPC requests that fail (T171371).
81  $wikiId = WikiMap::getWikiIdFromDomain( $domain );
82  if (
83  !WikiMap::isCurrentWikiDomain( $domain ) &&
84  !in_array( $wikiId, $wgLocalDatabases )
85  ) {
86  self::$instances[$domain]->invalidDomain = true;
87  }
88  }
89 
90  return self::$instances[$domain];
91  }
92 
98  public static function destroySingletons() {
99  self::$instances = [];
100  }
101 
108  public function get( $type ) {
109  global $wgJobTypeConf;
110 
111  $conf = [ 'domain' => $this->domain, 'type' => $type ];
112  if ( isset( $wgJobTypeConf[$type] ) ) {
113  $conf = $conf + $wgJobTypeConf[$type];
114  } else {
115  $conf = $conf + $wgJobTypeConf['default'];
116  }
117  $conf['aggregator'] = JobQueueAggregator::singleton();
118  if ( !isset( $conf['readOnlyReason'] ) ) {
119  $conf['readOnlyReason'] = $this->readOnlyReason;
120  }
121 
122  return JobQueue::factory( $conf );
123  }
124 
135  public function push( $jobs ) {
137 
138  if ( $this->invalidDomain ) {
139  // Do not enqueue job that cannot be run (T171371)
140  $e = new LogicException( "Domain '{$this->domain}' is not recognized." );
142  return;
143  }
144 
145  $jobs = is_array( $jobs ) ? $jobs : [ $jobs ];
146  if ( !count( $jobs ) ) {
147  return;
148  }
149 
150  $this->assertValidJobs( $jobs );
151 
152  $jobsByType = []; // (job type => list of jobs)
153  foreach ( $jobs as $job ) {
154  $jobsByType[$job->getType()][] = $job;
155  }
156 
157  foreach ( $jobsByType as $type => $jobs ) {
158  $this->get( $type )->push( $jobs );
159  }
160 
161  if ( $this->cache->hasField( 'queues-ready', 'list' ) ) {
162  $list = $this->cache->getField( 'queues-ready', 'list' );
163  if ( count( array_diff( array_keys( $jobsByType ), $list ) ) ) {
164  $this->cache->clear( 'queues-ready' );
165  }
166  }
167 
169  $cache->set(
170  $cache->makeGlobalKey( 'jobqueue', $this->domain, 'hasjobs', self::TYPE_ANY ),
171  'true',
172  15
173  );
174  if ( array_diff( array_keys( $jobsByType ), $wgJobTypesExcludedFromDefaultQueue ) ) {
175  $cache->set(
176  $cache->makeGlobalKey( 'jobqueue', $this->domain, 'hasjobs', self::TYPE_DEFAULT ),
177  'true',
178  15
179  );
180  }
181  }
182 
194  public function lazyPush( $jobs ) {
195  if ( $this->invalidDomain ) {
196  // Do not enqueue job that cannot be run (T171371)
197  throw new LogicException( "Domain '{$this->domain}' is not recognized." );
198  }
199 
200  if ( PHP_SAPI === 'cli' || PHP_SAPI === 'phpdbg' ) {
201  $this->push( $jobs );
202  return;
203  }
204 
205  $jobs = is_array( $jobs ) ? $jobs : [ $jobs ];
206 
207  // Throw errors now instead of on push(), when other jobs may be buffered
208  $this->assertValidJobs( $jobs );
209 
210  DeferredUpdates::addUpdate( new JobQueueEnqueueUpdate( $this->domain, $jobs ) );
211  }
212 
220  public static function pushLazyJobs() {
221  wfDeprecated( __METHOD__, '1.33' );
222  }
223 
235  public function pop( $qtype = self::TYPE_DEFAULT, $flags = 0, array $blacklist = [] ) {
236  $job = false;
237 
238  if ( is_string( $qtype ) ) { // specific job type
239  if ( !in_array( $qtype, $blacklist ) ) {
240  $job = $this->get( $qtype )->pop();
241  }
242  } else { // any job in the "default" jobs types
243  if ( $flags & self::USE_CACHE ) {
244  if ( !$this->cache->hasField( 'queues-ready', 'list', self::PROC_CACHE_TTL ) ) {
245  $this->cache->setField( 'queues-ready', 'list', $this->getQueuesWithJobs() );
246  }
247  $types = $this->cache->getField( 'queues-ready', 'list' );
248  } else {
249  $types = $this->getQueuesWithJobs();
250  }
251 
252  if ( $qtype == self::TYPE_DEFAULT ) {
253  $types = array_intersect( $types, $this->getDefaultQueueTypes() );
254  }
255 
256  $types = array_diff( $types, $blacklist ); // avoid selected types
257  shuffle( $types ); // avoid starvation
258 
259  foreach ( $types as $type ) { // for each queue...
260  $job = $this->get( $type )->pop();
261  if ( $job ) { // found
262  break;
263  } else { // not found
264  $this->cache->clear( 'queues-ready' );
265  }
266  }
267  }
268 
269  return $job;
270  }
271 
278  public function ack( Job $job ) {
279  $this->get( $job->getType() )->ack( $job );
280  }
281 
289  public function deduplicateRootJob( Job $job ) {
290  return $this->get( $job->getType() )->deduplicateRootJob( $job );
291  }
292 
300  public function waitForBackups() {
301  global $wgJobTypeConf;
302 
303  // Try to avoid doing this more than once per queue storage medium
304  foreach ( $wgJobTypeConf as $type => $conf ) {
305  $this->get( $type )->waitForBackups();
306  }
307  }
308 
314  public function getQueueTypes() {
315  return array_keys( $this->getCachedConfigVar( 'wgJobClasses' ) );
316  }
317 
323  public function getDefaultQueueTypes() {
325 
326  return array_diff( $this->getQueueTypes(), $wgJobTypesExcludedFromDefaultQueue );
327  }
328 
336  public function queuesHaveJobs( $type = self::TYPE_ANY ) {
338  $key = $cache->makeGlobalKey( 'jobqueue', $this->domain, 'hasjobs', $type );
339 
340  $value = $cache->get( $key );
341  if ( $value === false ) {
342  $queues = $this->getQueuesWithJobs();
343  if ( $type == self::TYPE_DEFAULT ) {
344  $queues = array_intersect( $queues, $this->getDefaultQueueTypes() );
345  }
346  $value = count( $queues ) ? 'true' : 'false';
347  $cache->add( $key, $value, 15 );
348  }
349 
350  return ( $value === 'true' );
351  }
352 
358  public function getQueuesWithJobs() {
359  $types = [];
360  foreach ( $this->getCoalescedQueues() as $info ) {
361  $nonEmpty = $info['queue']->getSiblingQueuesWithJobs( $this->getQueueTypes() );
362  if ( is_array( $nonEmpty ) ) { // batching features supported
363  $types = array_merge( $types, $nonEmpty );
364  } else { // we have to go through the queues in the bucket one-by-one
365  foreach ( $info['types'] as $type ) {
366  if ( !$this->get( $type )->isEmpty() ) {
367  $types[] = $type;
368  }
369  }
370  }
371  }
372 
373  return $types;
374  }
375 
381  public function getQueueSizes() {
382  $sizeMap = [];
383  foreach ( $this->getCoalescedQueues() as $info ) {
384  $sizes = $info['queue']->getSiblingQueueSizes( $this->getQueueTypes() );
385  if ( is_array( $sizes ) ) { // batching features supported
386  $sizeMap = $sizeMap + $sizes;
387  } else { // we have to go through the queues in the bucket one-by-one
388  foreach ( $info['types'] as $type ) {
389  $sizeMap[$type] = $this->get( $type )->getSize();
390  }
391  }
392  }
393 
394  return $sizeMap;
395  }
396 
400  protected function getCoalescedQueues() {
401  global $wgJobTypeConf;
402 
403  if ( $this->coalescedQueues === null ) {
404  $this->coalescedQueues = [];
405  foreach ( $wgJobTypeConf as $type => $conf ) {
407  [ 'wiki' => $this->domain, 'type' => 'null' ] + $conf );
408  $loc = $queue->getCoalesceLocationInternal();
409  if ( !isset( $this->coalescedQueues[$loc] ) ) {
410  $this->coalescedQueues[$loc]['queue'] = $queue;
411  $this->coalescedQueues[$loc]['types'] = [];
412  }
413  if ( $type === 'default' ) {
414  $this->coalescedQueues[$loc]['types'] = array_merge(
415  $this->coalescedQueues[$loc]['types'],
416  array_diff( $this->getQueueTypes(), array_keys( $wgJobTypeConf ) )
417  );
418  } else {
419  $this->coalescedQueues[$loc]['types'][] = $type;
420  }
421  }
422  }
423 
424  return $this->coalescedQueues;
425  }
426 
431  private function getCachedConfigVar( $name ) {
432  // @TODO: cleanup this whole method with a proper config system
433  if ( WikiMap::isCurrentWikiDomain( $this->domain ) ) {
434  return $GLOBALS[$name]; // common case
435  } else {
436  $wiki = WikiMap::getWikiIdFromDomain( $this->domain );
437  $cache = MediaWikiServices::getInstance()->getMainWANObjectCache();
438  $value = $cache->getWithSetCallback(
439  $cache->makeGlobalKey( 'jobqueue', 'configvalue', $this->domain, $name ),
440  $cache::TTL_DAY + mt_rand( 0, $cache::TTL_DAY ),
441  function () use ( $wiki, $name ) {
442  global $wgConf;
443  // @TODO: use the full domain ID here
444  return [ 'v' => $wgConf->getConfig( $wiki, $name ) ];
445  },
446  [ 'pcTTL' => WANObjectCache::TTL_PROC_LONG ]
447  );
448 
449  return $value['v'];
450  }
451  }
452 
457  private function assertValidJobs( array $jobs ) {
458  foreach ( $jobs as $job ) { // sanity checks
459  if ( !( $job instanceof IJobSpecification ) ) {
460  throw new InvalidArgumentException( "Expected IJobSpecification objects" );
461  }
462  }
463  }
464 }
The wiki should then use memcached to cache various data To use multiple just add more items to the array To increase the weight of a make its entry a array("192.168.0.1:11211", 2))
get( $key, $prop)
Get a property field for a cache entry.
ProcessCacheLRU $cache
getType()
Definition: Job.php:128
assertValidJobs(array $jobs)
$wgJobTypesExcludedFromDefaultQueue
Jobs that must be explicitly requested, i.e.
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
div flags Integer display flags(NO_ACTION_LINK, NO_EXTRA_USER_LINKS) 'LogException' returning false will NOT prevent logging $e
Definition: hooks.txt:2173
static getWikiIdFromDomain( $domain)
Get the wiki ID of a database domain.
Definition: WikiMap.php:254
bool $invalidDomain
Whether the wiki is not recognized in configuration.
__construct( $domain, $readOnlyReason)
Class to both describe a background job and handle jobs.
Definition: Job.php:30
$value
static getCurrentWikiDomain()
Definition: WikiMap.php:276
queuesHaveJobs( $type=self::TYPE_ANY)
Check if there are any queues with jobs (this is cached)
static getLocalClusterInstance()
Get the main cluster-local cache object.
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency MediaWikiServices
Definition: injection.txt:23
waitForBackups()
Wait for any replica DBs or backup queue servers to catch up.
deduplicateRootJob(Job $job)
Register the "root job" of a given job into the queue for de-duplication.
static pushLazyJobs()
Push all jobs buffered via lazyPush() into their respective queues.
ack(Job $job)
Acknowledge that a job was completed.
pop( $qtype=self::TYPE_DEFAULT, $flags=0, array $blacklist=[])
Pop a job off one of the job queues.
getQueuesWithJobs()
Get the list of job types that have non-empty queues.
you have access to all of the normal MediaWiki so you can get a DB use the cache
Definition: maintenance.txt:52
getDefaultQueueTypes()
Get the list of default queue types.
wfConfiguredReadOnlyReason()
Get the value of $wgReadOnly or the contents of $wgReadOnlyFile.
Enqueue lazy-pushed jobs that have accumulated from JobQueueGroup.
getQueueTypes()
Get the list of queue types.
static isCurrentWikiDomain( $domain)
Definition: WikiMap.php:287
const PROC_CACHE_TTL
string bool $readOnlyReason
Read only rationale (or false if r/w)
static logException( $e, $catcher=self::CAUGHT_BY_OTHER)
Log an exception to the exception log (if enabled).
lazyPush( $jobs)
Buffer jobs for insertion via push() or call it now if in CLI mode.
$GLOBALS['IP']
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
getQueueSizes()
Get the size of the queus for a list of job types.
$wgConf
wgConf hold the site configuration.
getCachedConfigVar( $name)
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
static JobQueueGroup [] $instances
static addUpdate(DeferrableUpdate $update, $stage=self::POSTSEND)
Add an update to the deferred list to be run later by execute()
set( $key, $prop, $value)
Set a property field for a cache entry.
static factory(array $params)
Get a job queue object of the specified type.
Definition: JobQueue.php:104
$wgJobTypeConf
Map of job types to configuration arrays.
if(count( $args)< 1) $job
string $domain
Wiki domain ID.
wfDeprecated( $function, $version=false, $component=false, $callerOffset=2)
Throws a warning that $function is deprecated.
push( $jobs)
Insert jobs into the respective queues of which they belong.
array $coalescedQueues
Map of (bucket => (queue => JobQueue, types => list of types)
static singleton( $domain=false)
Job queue task description interface.
Allows to change the fields on the form that will be generated $name
Definition: hooks.txt:276
string [] $wgLocalDatabases
Other wikis on this site, can be administered from a single developer account.
static destroySingletons()
Destroy the singleton instances.