MediaWiki  REL1_31
JobQueueTest.php
Go to the documentation of this file.
1 <?php
2 
4 
11  protected $key;
13 
14  function __construct( $name = null, array $data = [], $dataName = '' ) {
15  parent::__construct( $name, $data, $dataName );
16 
17  $this->tablesUsed[] = 'job';
18  }
19 
20  protected function setUp() {
22  parent::setUp();
23 
24  if ( $this->getCliArg( 'use-jobqueue' ) ) {
25  $name = $this->getCliArg( 'use-jobqueue' );
26  if ( !isset( $wgJobTypeConf[$name] ) ) {
27  throw new MWException( "No \$wgJobTypeConf entry for '$name'." );
28  }
29  $baseConfig = $wgJobTypeConf[$name];
30  } else {
31  $baseConfig = [ 'class' => JobQueueDBSingle::class ];
32  }
33  $baseConfig['type'] = 'null';
34  $baseConfig['wiki'] = wfWikiID();
35  $variants = [
36  'queueRand' => [ 'order' => 'random', 'claimTTL' => 0 ],
37  'queueRandTTL' => [ 'order' => 'random', 'claimTTL' => 10 ],
38  'queueTimestamp' => [ 'order' => 'timestamp', 'claimTTL' => 0 ],
39  'queueTimestampTTL' => [ 'order' => 'timestamp', 'claimTTL' => 10 ],
40  'queueFifo' => [ 'order' => 'fifo', 'claimTTL' => 0 ],
41  'queueFifoTTL' => [ 'order' => 'fifo', 'claimTTL' => 10 ],
42  ];
43  foreach ( $variants as $q => $settings ) {
44  try {
45  $this->$q = JobQueue::factory( $settings + $baseConfig );
46  } catch ( MWException $e ) {
47  // unsupported?
48  // @todo What if it was another error?
49  };
50  }
51  }
52 
53  protected function tearDown() {
54  parent::tearDown();
55  foreach (
56  [
57  'queueRand', 'queueRandTTL', 'queueTimestamp', 'queueTimestampTTL',
58  'queueFifo', 'queueFifoTTL'
59  ] as $q
60  ) {
61  if ( $this->$q ) {
62  $this->$q->delete();
63  }
64  $this->$q = null;
65  }
66  }
67 
72  public function testGetWiki( $queue, $recycles, $desc ) {
73  $queue = $this->$queue;
74  if ( !$queue ) {
75  $this->markTestSkipped( $desc );
76  }
77  $this->assertEquals( wfWikiID(), $queue->getWiki(), "Proper wiki ID ($desc)" );
78  }
79 
84  public function testGetType( $queue, $recycles, $desc ) {
85  $queue = $this->$queue;
86  if ( !$queue ) {
87  $this->markTestSkipped( $desc );
88  }
89  $this->assertEquals( 'null', $queue->getType(), "Proper job type ($desc)" );
90  }
91 
96  public function testBasicOperations( $queue, $recycles, $desc ) {
97  $queue = $this->$queue;
98  if ( !$queue ) {
99  $this->markTestSkipped( $desc );
100  }
101 
102  $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
103 
104  $queue->flushCaches();
105  $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
106  $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
107 
108  $this->assertNull( $queue->push( $this->newJob() ), "Push worked ($desc)" );
109  $this->assertNull( $queue->batchPush( [ $this->newJob() ] ), "Push worked ($desc)" );
110 
111  $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
112 
113  $queue->flushCaches();
114  $this->assertEquals( 2, $queue->getSize(), "Queue size is correct ($desc)" );
115  $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
116  $jobs = iterator_to_array( $queue->getAllQueuedJobs() );
117  $this->assertEquals( 2, count( $jobs ), "Queue iterator size is correct ($desc)" );
118 
119  $job1 = $queue->pop();
120  $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
121 
122  $queue->flushCaches();
123  $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
124 
125  $queue->flushCaches();
126  if ( $recycles ) {
127  $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
128  }
129 
130  $job2 = $queue->pop();
131  $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
132  $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
133 
134  $queue->flushCaches();
135  if ( $recycles ) {
136  $this->assertEquals( 2, $queue->getAcquiredCount(), "Active job count ($desc)" );
137  }
138 
139  $queue->ack( $job1 );
140 
141  $queue->flushCaches();
142  if ( $recycles ) {
143  $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
144  }
145 
146  $queue->ack( $job2 );
147 
148  $queue->flushCaches();
149  $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
150 
151  $this->assertNull( $queue->batchPush( [ $this->newJob(), $this->newJob() ] ),
152  "Push worked ($desc)" );
153  $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
154 
155  $queue->delete();
156  $queue->flushCaches();
157  $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
158  $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
159  }
160 
165  public function testBasicDeduplication( $queue, $recycles, $desc ) {
166  $queue = $this->$queue;
167  if ( !$queue ) {
168  $this->markTestSkipped( $desc );
169  }
170 
171  $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
172 
173  $queue->flushCaches();
174  $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
175  $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
176 
177  $this->assertNull(
178  $queue->batchPush(
179  [ $this->newDedupedJob(), $this->newDedupedJob(), $this->newDedupedJob() ]
180  ),
181  "Push worked ($desc)" );
182 
183  $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
184 
185  $queue->flushCaches();
186  $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
187  $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
188 
189  $this->assertNull(
190  $queue->batchPush(
191  [ $this->newDedupedJob(), $this->newDedupedJob(), $this->newDedupedJob() ]
192  ),
193  "Push worked ($desc)"
194  );
195 
196  $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
197 
198  $queue->flushCaches();
199  $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
200  $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
201 
202  $job1 = $queue->pop();
203  $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
204 
205  $queue->flushCaches();
206  $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
207  if ( $recycles ) {
208  $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
209  }
210 
211  $queue->ack( $job1 );
212 
213  $queue->flushCaches();
214  $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
215  }
216 
221  public function testDeduplicationWhileClaimed( $queue, $recycles, $desc ) {
222  $queue = $this->$queue;
223  if ( !$queue ) {
224  $this->markTestSkipped( $desc );
225  }
226 
227  $job = $this->newDedupedJob();
228  $queue->push( $job );
229 
230  // De-duplication does not apply to already-claimed jobs
231  $j = $queue->pop();
232  $queue->push( $job );
233  $queue->ack( $j );
234 
235  $j = $queue->pop();
236  // Make sure ack() of the twin did not delete the sibling data
237  $this->assertType( NullJob::class, $j );
238  }
239 
244  public function testRootDeduplication( $queue, $recycles, $desc ) {
245  $queue = $this->$queue;
246  if ( !$queue ) {
247  $this->markTestSkipped( $desc );
248  }
249 
250  $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
251 
252  $queue->flushCaches();
253  $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
254  $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
255 
256  $id = wfRandomString( 32 );
257  $root1 = Job::newRootJobParams( "nulljobspam:$id" ); // task ID/timestamp
258  for ( $i = 0; $i < 5; ++$i ) {
259  $this->assertNull( $queue->push( $this->newJob( 0, $root1 ) ), "Push worked ($desc)" );
260  }
261  $queue->deduplicateRootJob( $this->newJob( 0, $root1 ) );
262 
263  $root2 = $root1;
264  # Add a second to UNIX epoch and format back to TS_MW
265  $root2_ts = strtotime( $root2['rootJobTimestamp'] );
266  $root2_ts++;
267  $root2['rootJobTimestamp'] = wfTimestamp( TS_MW, $root2_ts );
268 
269  $this->assertNotEquals( $root1['rootJobTimestamp'], $root2['rootJobTimestamp'],
270  "Root job signatures have different timestamps." );
271  for ( $i = 0; $i < 5; ++$i ) {
272  $this->assertNull( $queue->push( $this->newJob( 0, $root2 ) ), "Push worked ($desc)" );
273  }
274  $queue->deduplicateRootJob( $this->newJob( 0, $root2 ) );
275 
276  $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
277 
278  $queue->flushCaches();
279  $this->assertEquals( 10, $queue->getSize(), "Queue size is correct ($desc)" );
280  $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
281 
282  $dupcount = 0;
283  $jobs = [];
284  do {
285  $job = $queue->pop();
286  if ( $job ) {
287  $jobs[] = $job;
288  $queue->ack( $job );
289  }
290  if ( $job instanceof DuplicateJob ) {
291  ++$dupcount;
292  }
293  } while ( $job );
294 
295  $this->assertEquals( 10, count( $jobs ), "Correct number of jobs popped ($desc)" );
296  $this->assertEquals( 5, $dupcount, "Correct number of duplicate jobs popped ($desc)" );
297  }
298 
303  public function testJobOrder( $queue, $recycles, $desc ) {
304  $queue = $this->$queue;
305  if ( !$queue ) {
306  $this->markTestSkipped( $desc );
307  }
308 
309  $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
310 
311  $queue->flushCaches();
312  $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
313  $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
314 
315  for ( $i = 0; $i < 10; ++$i ) {
316  $this->assertNull( $queue->push( $this->newJob( $i ) ), "Push worked ($desc)" );
317  }
318 
319  for ( $i = 0; $i < 10; ++$i ) {
320  $job = $queue->pop();
321  $this->assertTrue( $job instanceof Job, "Jobs popped from queue ($desc)" );
322  $params = $job->getParams();
323  $this->assertEquals( $i, $params['i'], "Job popped from queue is FIFO ($desc)" );
324  $queue->ack( $job );
325  }
326 
327  $this->assertFalse( $queue->pop(), "Queue is not empty ($desc)" );
328 
329  $queue->flushCaches();
330  $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
331  $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
332  }
333 
337  public function testQueueAggregateTable() {
339  if ( !$queue || !method_exists( $queue, 'getServerQueuesWithJobs' ) ) {
340  $this->markTestSkipped();
341  }
342 
343  $this->assertNotContains(
344  [ $queue->getType(), $queue->getWiki() ],
345  $queue->getServerQueuesWithJobs(),
346  "Null queue not in listing"
347  );
348 
349  $queue->push( $this->newJob( 0 ) );
350 
351  $this->assertContains(
352  [ $queue->getType(), $queue->getWiki() ],
353  $queue->getServerQueuesWithJobs(),
354  "Null queue in listing"
355  );
356  }
357 
358  public static function provider_queueLists() {
359  return [
360  [ 'queueRand', false, 'Random queue without ack()' ],
361  [ 'queueRandTTL', true, 'Random queue with ack()' ],
362  [ 'queueTimestamp', false, 'Time ordered queue without ack()' ],
363  [ 'queueTimestampTTL', true, 'Time ordered queue with ack()' ],
364  [ 'queueFifo', false, 'FIFO ordered queue without ack()' ],
365  [ 'queueFifoTTL', true, 'FIFO ordered queue with ack()' ]
366  ];
367  }
368 
369  public static function provider_fifoQueueLists() {
370  return [
371  [ 'queueFifo', false, 'Ordered queue without ack()' ],
372  [ 'queueFifoTTL', true, 'Ordered queue with ack()' ]
373  ];
374  }
375 
376  function newJob( $i = 0, $rootJob = [] ) {
377  return new NullJob( Title::newMainPage(),
378  [ 'lives' => 0, 'usleep' => 0, 'removeDuplicates' => 0, 'i' => $i ] + $rootJob );
379  }
380 
381  function newDedupedJob( $i = 0, $rootJob = [] ) {
382  return new NullJob( Title::newMainPage(),
383  [ 'lives' => 0, 'usleep' => 0, 'removeDuplicates' => 1, 'i' => $i ] + $rootJob );
384  }
385 }
386 
388  protected function getDB( $index ) {
389  $lb = MediaWikiServices::getInstance()->getDBLoadBalancer();
390  // Override to not use CONN_TRX_AUTOCOMMIT so that we see the same temporary `job` table
391  return $lb->getConnection( $index, [], $this->wiki );
392  }
393 }
JobQueueTest\$key
$key
Definition: JobQueueTest.php:11
JobQueueDBSingle
Definition: JobQueueTest.php:387
JobQueueTest\provider_fifoQueueLists
static provider_fifoQueueLists()
Definition: JobQueueTest.php:369
use
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
Definition: APACHE-LICENSE-2.0.txt:10
$wgJobTypeConf
$wgJobTypeConf
Map of job types to configuration arrays.
Definition: DefaultSettings.php:7545
array
the array() calling protocol came about after MediaWiki 1.4rc1.
JobQueueTest\__construct
__construct( $name=null, array $data=[], $dataName='')
Definition: JobQueueTest.php:14
JobQueueTest\$queueRandTTL
$queueRandTTL
Definition: JobQueueTest.php:12
Title\newMainPage
static newMainPage()
Create a new Title for the Main Page.
Definition: Title.php:586
wfTimestamp
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Definition: GlobalFunctions.php:1980
wiki
Prior to maintenance scripts were a hodgepodge of code that had no cohesion or formal method of action Beginning maintenance scripts have been cleaned up to use a unified class Directory structure How to run a script How to write your own DIRECTORY STRUCTURE The maintenance directory of a MediaWiki installation contains several all of which have unique purposes HOW TO RUN A SCRIPT Ridiculously just call php someScript php that s in the top level maintenance directory if not default wiki
Definition: maintenance.txt:27
JobQueueTest\testGetWiki
testGetWiki( $queue, $recycles, $desc)
provider_queueLists JobQueue::getWiki
Definition: JobQueueTest.php:72
JobQueueTest\newJob
newJob( $i=0, $rootJob=[])
Definition: JobQueueTest.php:376
$params
$params
Definition: styleTest.css.php:40
JobQueueTest\newDedupedJob
newDedupedJob( $i=0, $rootJob=[])
Definition: JobQueueTest.php:381
MediaWikiTestCase\getCliArg
getCliArg( $offset)
Definition: MediaWikiTestCase.php:1651
NullJob
Degenerate job that does nothing, but can optionally replace itself in the queue and/or sleep for a b...
Definition: NullJob.php:47
JobQueueTest\testBasicDeduplication
testBasicDeduplication( $queue, $recycles, $desc)
provider_queueLists JobQueue
Definition: JobQueueTest.php:165
JobQueueDB
Class to handle job queues stored in the DB.
Definition: JobQueueDB.php:35
JobQueueTest\$queueFifoTTL
$queueFifoTTL
Definition: JobQueueTest.php:12
JobQueueTest\testQueueAggregateTable
testQueueAggregateTable()
JobQueue.
Definition: JobQueueTest.php:337
php
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:37
Job
Class to both describe a background job and handle jobs.
Definition: Job.php:31
JobQueueTest\testJobOrder
testJobOrder( $queue, $recycles, $desc)
provider_fifoQueueLists JobQueue
Definition: JobQueueTest.php:303
MWException
MediaWiki exception.
Definition: MWException.php:26
JobQueueTest\testDeduplicationWhileClaimed
testDeduplicationWhileClaimed( $queue, $recycles, $desc)
provider_queueLists JobQueue
Definition: JobQueueTest.php:221
MediaWikiTestCase
Definition: MediaWikiTestCase.php:17
$queue
$queue
Definition: mergeMessageFileList.php:160
JobQueueTest\setUp
setUp()
Definition: JobQueueTest.php:20
global
when a variable name is used in a it is silently declared as a new masking the global
Definition: design.txt:95
Job\newRootJobParams
static newRootJobParams( $key)
Get "root job" parameters for a task.
Definition: Job.php:286
DuplicateJob
No-op job that does nothing.
Definition: DuplicateJob.php:29
JobQueueTest\testBasicOperations
testBasicOperations( $queue, $recycles, $desc)
provider_queueLists JobQueue
Definition: JobQueueTest.php:96
JobQueueDBSingle\getDB
getDB( $index)
Definition: JobQueueTest.php:388
JobQueueTest\$queueFifo
$queueFifo
Definition: JobQueueTest.php:12
wfWikiID
wfWikiID()
Get an ASCII string identifying this wiki This is used as a prefix in memcached keys.
Definition: GlobalFunctions.php:2763
JobQueueTest\provider_queueLists
static provider_queueLists()
Definition: JobQueueTest.php:358
JobQueueTest\testGetType
testGetType( $queue, $recycles, $desc)
provider_queueLists JobQueue::getType
Definition: JobQueueTest.php:84
JobQueue\factory
static factory(array $params)
Get a job queue object of the specified type.
Definition: JobQueue.php:108
JobQueueTest\testRootDeduplication
testRootDeduplication( $queue, $recycles, $desc)
provider_queueLists JobQueue
Definition: JobQueueTest.php:244
JobQueueTest\$queueRand
$queueRand
Definition: JobQueueTest.php:12
$name
Allows to change the fields on the form that will be generated $name
Definition: hooks.txt:302
JobQueueTest\tearDown
tearDown()
Definition: JobQueueTest.php:53
$job
if(count( $args)< 1) $job
Definition: recompressTracked.php:47
as
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:22
JobQueueTest
JobQueue medium Database.
Definition: JobQueueTest.php:10
class
you have access to all of the normal MediaWiki so you can get a DB use the etc For full docs on the Maintenance class
Definition: maintenance.txt:56
MediaWikiServices
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency MediaWikiServices
Definition: injection.txt:25
MediaWikiTestCase\assertType
assertType( $type, $actual, $message='')
Asserts the type of the provided value.
Definition: MediaWikiTestCase.php:1878
$e
div flags Integer display flags(NO_ACTION_LINK, NO_EXTRA_USER_LINKS) 'LogException' returning false will NOT prevent logging $e
Definition: hooks.txt:2171
wfRandomString
wfRandomString( $length=32)
Get a random string containing a number of pseudo-random hex characters.
Definition: GlobalFunctions.php:305