MediaWiki  1.23.13
runJobs.php
Go to the documentation of this file.
1 <?php
24 require_once __DIR__ . '/Maintenance.php';
25 
31 class RunJobs extends Maintenance {
32  public function __construct() {
33  parent::__construct();
34  $this->mDescription = "Run pending jobs";
35  $this->addOption( 'maxjobs', 'Maximum number of jobs to run', false, true );
36  $this->addOption( 'maxtime', 'Maximum amount of wall-clock time', false, true );
37  $this->addOption( 'type', 'Type of job to run', false, true );
38  $this->addOption( 'procs', 'Number of processes to use', false, true );
39  $this->addOption( 'nothrottle', 'Ignore job throttling configuration', false, false );
40  }
41 
42  public function memoryLimit() {
43  if ( $this->hasOption( 'memory-limit' ) ) {
44  return parent::memoryLimit();
45  }
46  // Don't eat all memory on the machine if we get a bad job.
47  return "150M";
48  }
49 
50  public function execute() {
51  if ( wfReadOnly() ) {
52  $this->error( "Unable to run jobs; the wiki is in read-only mode.", 1 ); // die
53  }
54 
55  if ( $this->hasOption( 'procs' ) ) {
56  $procs = intval( $this->getOption( 'procs' ) );
57  if ( $procs < 1 || $procs > 1000 ) {
58  $this->error( "Invalid argument to --procs", true );
59  } elseif ( $procs != 1 ) {
60  $fc = new ForkController( $procs );
61  if ( $fc->start() != 'child' ) {
62  exit( 0 );
63  }
64  }
65  }
66 
67  $type = $this->getOption( 'type', false );
68  $maxJobs = $this->getOption( 'maxjobs', false );
69  $maxTime = $this->getOption( 'maxtime', false );
70  $noThrottle = $this->hasOption( 'nothrottle' );
71  $startTime = time();
72 
73  $group = JobQueueGroup::singleton();
74  // Handle any required periodic queue maintenance
75  $count = $group->executeReadyPeriodicTasks();
76  if ( $count > 0 ) {
77  $this->runJobsLog( "Executed $count periodic queue task(s)." );
78  }
79 
80  $backoffs = $this->loadBackoffs(); // map of (type => UNIX expiry)
81  $startingBackoffs = $backoffs; // avoid unnecessary writes
82  $backoffExpireFunc = function( $t ) { return $t > time(); };
83 
84  $jobsRun = 0; // counter
86  $lastTime = time(); // time since last slave check
87  do {
88  $backoffs = array_filter( $backoffs, $backoffExpireFunc );
89  $blacklist = $noThrottle ? array() : array_keys( $backoffs );
90  if ( $type === false ) {
91  $job = $group->pop( JobQueueGroup::TYPE_DEFAULT, $flags, $blacklist );
92  } elseif ( in_array( $type, $blacklist ) ) {
93  $job = false; // requested queue in backoff state
94  } else {
95  $job = $group->pop( $type ); // job from a single queue
96  }
97  if ( $job ) { // found a job
98  ++$jobsRun;
99  $this->runJobsLog( $job->toString() . " STARTING" );
100 
101  // Set timer to stop the job if too much CPU time is used
102  set_time_limit( $maxTime ?: 0 );
103  // Run the job...
104  wfProfileIn( __METHOD__ . '-' . get_class( $job ) );
105  $t = microtime( true );
106  try {
107  $status = $job->run();
108  $error = $job->getLastError();
109  } catch ( MWException $e ) {
111  $status = false;
112  $error = get_class( $e ) . ': ' . $e->getMessage();
113  $e->report(); // write error to STDERR and the log
114  }
115  $timeMs = intval( ( microtime( true ) - $t ) * 1000 );
116  wfProfileOut( __METHOD__ . '-' . get_class( $job ) );
117  // Disable the timer
118  set_time_limit( 0 );
119 
120  // Mark the job as done on success or when the job cannot be retried
121  if ( $status !== false || !$job->allowRetries() ) {
122  $group->ack( $job ); // done
123  }
124 
125  if ( $status === false ) {
126  $this->runJobsLog( $job->toString() . " t=$timeMs error={$error}" );
127  } else {
128  $this->runJobsLog( $job->toString() . " t=$timeMs good" );
129  }
130 
131  // Back off of certain jobs for a while
132  $ttw = $this->getBackoffTimeToWait( $job );
133  if ( $ttw > 0 ) {
134  $jType = $job->getType();
135  $backoffs[$jType] = isset( $backoffs[$jType] ) ? $backoffs[$jType] : 0;
136  $backoffs[$jType] = max( $backoffs[$jType], time() + $ttw );
137  }
138 
139  // Break out if we hit the job count or wall time limits...
140  if ( $maxJobs && $jobsRun >= $maxJobs ) {
141  break;
142  } elseif ( $maxTime && ( time() - $startTime ) > $maxTime ) {
143  break;
144  }
145 
146  // Don't let any of the main DB slaves get backed up
147  $timePassed = time() - $lastTime;
148  if ( $timePassed >= 5 || $timePassed < 0 ) {
149  wfWaitForSlaves();
150  $lastTime = time();
151  }
152  // Don't let any queue slaves/backups fall behind
153  if ( $jobsRun > 0 && ( $jobsRun % 100 ) == 0 ) {
154  $group->waitForBackups();
155  }
156 
157  // Bail if near-OOM instead of in a job
158  $this->assertMemoryOK();
159  }
160  } while ( $job ); // stop when there are no jobs
161  // Sync the persistent backoffs for the next runJobs.php pass
162  $backoffs = array_filter( $backoffs, $backoffExpireFunc );
163  if ( $backoffs !== $startingBackoffs ) {
164  $this->syncBackoffs( $backoffs );
165  }
166  }
167 
173  private function getBackoffTimeToWait( Job $job ) {
174  global $wgJobBackoffThrottling;
175 
176  if ( !isset( $wgJobBackoffThrottling[$job->getType()] ) ) {
177  return 0; // not throttled
178  }
179  $itemsPerSecond = $wgJobBackoffThrottling[$job->getType()];
180  if ( $itemsPerSecond <= 0 ) {
181  return 0; // not throttled
182  }
183 
184  $seconds = 0;
185  if ( $job->workItemCount() > 0 ) {
186  $seconds = floor( $job->workItemCount() / $itemsPerSecond );
187  $remainder = $job->workItemCount() % $itemsPerSecond;
188  $seconds += ( mt_rand( 1, $itemsPerSecond ) <= $remainder ) ? 1 : 0;
189  }
190 
191  return (int)$seconds;
192  }
193 
199  private function loadBackoffs() {
200  $section = new ProfileSection( __METHOD__ );
201 
202  $backoffs = array();
203  $file = wfTempDir() . '/mw-runJobs-backoffs.json';
204  if ( is_file( $file ) ) {
205  $handle = fopen( $file, 'rb' );
206  flock( $handle, LOCK_SH );
207  $content = stream_get_contents( $handle );
208  flock( $handle, LOCK_UN );
209  fclose( $handle );
210  $backoffs = json_decode( $content, true ) ?: array();
211  }
212 
213  return $backoffs;
214  }
215 
221  private function syncBackoffs( array $backoffs ) {
222  $section = new ProfileSection( __METHOD__ );
223 
224  $file = wfTempDir() . '/mw-runJobs-backoffs.json';
225  $handle = fopen( $file, 'wb+' );
226  flock( $handle, LOCK_EX );
227  $content = stream_get_contents( $handle );
228  $cBackoffs = json_decode( $content, true ) ?: array();
229  foreach ( $backoffs as $type => $timestamp ) {
230  $cBackoffs[$type] = isset( $cBackoffs[$type] ) ? $cBackoffs[$type] : 0;
231  $cBackoffs[$type] = max( $cBackoffs[$type], $backoffs[$type] );
232  }
233  ftruncate( $handle, 0 );
234  fwrite( $handle, json_encode( $backoffs ) );
235  flock( $handle, LOCK_UN );
236  fclose( $handle );
237  }
238 
244  private function assertMemoryOK() {
245  static $maxBytes = null;
246  if ( $maxBytes === null ) {
247  $m = array();
248  if ( preg_match( '!^(\d+)(k|m|g|)$!i', ini_get( 'memory_limit' ), $m ) ) {
249  list( , $num, $unit ) = $m;
250  $conv = array( 'g' => 1073741824, 'm' => 1048576, 'k' => 1024, '' => 1 );
251  $maxBytes = $num * $conv[strtolower( $unit )];
252  } else {
253  $maxBytes = 0;
254  }
255  }
256  $usedBytes = memory_get_usage();
257  if ( $maxBytes && $usedBytes >= 0.95 * $maxBytes ) {
258  throw new MWException( "Detected excessive memory usage ($usedBytes/$maxBytes)." );
259  }
260  }
261 
266  private function runJobsLog( $msg ) {
267  $this->output( wfTimestamp( TS_DB ) . " $msg\n" );
268  wfDebugLog( 'runJobs', $msg );
269  }
270 }
271 
272 $maintClass = "RunJobs";
273 require_once RUN_MAINTENANCE_IF_MAIN;
JobQueueGroup\USE_CACHE
const USE_CACHE
Definition: JobQueueGroup.php:43
RunJobs
Maintenance script that runs pending jobs.
Definition: runJobs.php:31
RunJobs\execute
execute()
Do the actual work.
Definition: runJobs.php:50
php
skin txt MediaWiki includes four core it has been set as the default in MediaWiki since the replacing Monobook it had been been the default skin since before being replaced by Vector largely rewritten in while keeping its appearance Several legacy skins were removed in the as the burden of supporting them became too heavy to bear Those in etc for skin dependent CSS etc for skin dependent JavaScript These can also be customised on a per user by etc This feature has led to a wide variety of user styles becoming that gallery is a good place to ending in php
Definition: skin.txt:62
RunJobs\assertMemoryOK
assertMemoryOK()
Make sure that this script is not too close to the memory usage limit.
Definition: runJobs.php:244
ForkController
Class for managing forking command line scripts.
Definition: ForkController.php:32
RunJobs\memoryLimit
memoryLimit()
Normally we disable the memory_limit when running admin scripts.
Definition: runJobs.php:42
JobQueueGroup\TYPE_DEFAULT
const TYPE_DEFAULT
Definition: JobQueueGroup.php:40
$timestamp
if( $limit) $timestamp
Definition: importImages.php:104
wfTimestamp
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Definition: GlobalFunctions.php:2530
wfDebugLog
wfDebugLog( $logGroup, $text, $dest='all')
Send a line to a supplementary debug log file, if configured, or main debug log if not.
Definition: GlobalFunctions.php:1087
wfProfileIn
wfProfileIn( $functionname)
Begin profiling of a function.
Definition: Profiler.php:33
Maintenance\addOption
addOption( $name, $description, $required=false, $withArg=false, $shortName=false)
Add a parameter to the script.
Definition: Maintenance.php:169
RUN_MAINTENANCE_IF_MAIN
require_once RUN_MAINTENANCE_IF_MAIN
Definition: maintenance.txt:50
wfReadOnly
wfReadOnly()
Check whether the wiki is in read-only mode.
Definition: GlobalFunctions.php:1360
TS_DB
const TS_DB
MySQL DATETIME (YYYY-MM-DD HH:MM:SS)
Definition: GlobalFunctions.php:2483
Maintenance
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: maintenance.txt:39
$flags
it s the revision text itself In either if gzip is the revision text is gzipped $flags
Definition: hooks.txt:2118
Job
Class to both describe a background job and handle jobs.
Definition: Job.php:31
ProfileSection
Class for handling function-scope profiling.
Definition: Profiler.php:60
MWException
MediaWiki exception.
Definition: MWException.php:26
RunJobs\__construct
__construct()
Default constructor.
Definition: runJobs.php:32
wfProfileOut
wfProfileOut( $functionname='missing')
Stop profiling of a function.
Definition: Profiler.php:46
array
the array() calling protocol came about after MediaWiki 1.4rc1.
List of Api Query prop modules.
global
when a variable name is used in a it is silently declared as a new masking the global
Definition: design.txt:93
wfWaitForSlaves
wfWaitForSlaves( $maxLag=false, $wiki=false, $cluster=false)
Modern version of wfWaitForSlaves().
Definition: GlobalFunctions.php:3851
list
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition: deferred.txt:11
$section
$section
Definition: Utf8Test.php:88
$file
if(PHP_SAPI !='cli') $file
Definition: UtfNormalTest2.php:30
$count
$count
Definition: UtfNormalTest2.php:96
$maintClass
$maintClass
Definition: runJobs.php:272
RunJobs\getBackoffTimeToWait
getBackoffTimeToWait(Job $job)
Definition: runJobs.php:173
wfTempDir
wfTempDir()
Tries to get the system directory for temporary files.
Definition: GlobalFunctions.php:2611
Maintenance\getOption
getOption( $name, $default=null)
Get an option, or return the default.
Definition: Maintenance.php:191
$job
if(count( $args)< 1) $job
Definition: recompressTracked.php:42
MWExceptionHandler\rollbackMasterChangesAndLog
static rollbackMasterChangesAndLog(Exception $e)
If there are any open database transactions, roll them back and log the stack trace of the exception ...
Definition: MWExceptionHandler.php:111
JobQueueGroup\singleton
static singleton( $wiki=false)
Definition: JobQueueGroup.php:61
as
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
Maintenance\error
error( $err, $die=0)
Throw an error to the user.
Definition: Maintenance.php:333
Maintenance\output
output( $out, $channel=null)
Throw some output to the user.
Definition: Maintenance.php:314
RunJobs\runJobsLog
runJobsLog( $msg)
Log the job message.
Definition: runJobs.php:266
$t
$t
Definition: testCompression.php:65
RunJobs\syncBackoffs
syncBackoffs(array $backoffs)
Merge the current backoff expiries from persistent storage.
Definition: runJobs.php:221
$error
usually copyright or history_copyright This message must be in HTML not wikitext $subpages will be ignored and the rest of subPageSubtitle() will run. 'SkinTemplateBuildNavUrlsNav_urlsAfterPermalink' whether MediaWiki currently thinks this is a CSS JS page Hooks may change this value to override the return value of Title::isCssOrJsPage(). 'TitleIsAlwaysKnown' whether MediaWiki currently thinks this page is known isMovable() always returns false. $title whether MediaWiki currently thinks this page is movable Hooks may change this value to override the return value of Title::isMovable(). 'TitleIsWikitextPage' whether MediaWiki currently thinks this is a wikitext page Hooks may change this value to override the return value of Title::isWikitextPage() 'TitleMove' use UploadVerification and UploadVerifyFile instead where the first element is the message key and the remaining elements are used as parameters to the message based on mime etc Preferred in most cases over UploadVerification object with all info about the upload string as detected by MediaWiki Handlers will typically only apply for specific mime types object & $error
Definition: hooks.txt:2578
Maintenance\hasOption
hasOption( $name)
Checks to see if a particular param exists.
Definition: Maintenance.php:181
RunJobs\loadBackoffs
loadBackoffs()
Get the previous backoff expiries from persistent storage.
Definition: runJobs.php:199
$e
div flags Integer display flags(NO_ACTION_LINK, NO_EXTRA_USER_LINKS) 'LogException' returning false will NOT prevent logging $e
Definition: hooks.txt:1632
$type
$type
Definition: testCompression.php:46