JobQueue.php

Go to the documentation of this file.

<?php
use MediaWiki\MediaWikiServices;
 
abstract class JobQueue {
    protected $wiki;
    protected $type;
    protected $order;
    protected $claimTTL;
    protected $maxTries;
    protected $readOnlyReason;
 
    protected $dupCache;
    protected $aggr;
 
    const QOS_ATOMIC = 1; // integer; "all-or-nothing" job insertions
 
    const ROOTJOB_TTL = 2419200; // integer; seconds to remember root jobs (28 days)
 
    protected function __construct( array $params ) {
        $this->wiki = $params['wiki'];
        $this->type = $params['type'];
        $this->claimTTL = $params['claimTTL'] ?? 0;
        $this->maxTries = $params['maxTries'] ?? 3;
        if ( isset( $params['order'] ) && $params['order'] !== 'any' ) {
            $this->order = $params['order'];
        } else {
            $this->order = $this->optimalOrder();
        }
        if ( !in_array( $this->order, $this->supportedOrders() ) ) {
            throw new MWException( __CLASS__ . " does not support '{$this->order}' order." );
        }
        $this->dupCache = wfGetCache( CACHE_ANYTHING );
        $this->aggr = $params['aggregator'] ?? new JobQueueAggregatorNull( [] );
        $this->readOnlyReason = $params['readOnlyReason'] ?? false;
    }
 
    final public static function factory( array $params ) {
        $class = $params['class'];
        if ( !class_exists( $class ) ) {
            throw new MWException( "Invalid job queue class '$class'." );
        }
        $obj = new $class( $params );
        if ( !( $obj instanceof self ) ) {
            throw new MWException( "Class '$class' is not a " . __CLASS__ . " class." );
        }
 
        return $obj;
    }
 
    final public function getWiki() {
        return $this->wiki;
    }
 
    final public function getType() {
        return $this->type;
    }
 
    final public function getOrder() {
        return $this->order;
    }
 
    abstract protected function supportedOrders();
 
    abstract protected function optimalOrder();
 
    protected function supportsDelayedJobs() {
        return false; // not implemented
    }
 
    final public function delayedJobsEnabled() {
        return $this->supportsDelayedJobs();
    }
 
    public function getReadOnlyReason() {
        return $this->readOnlyReason;
    }
 
    final public function isEmpty() {
        $res = $this->doIsEmpty();
 
        return $res;
    }
 
    abstract protected function doIsEmpty();
 
    final public function getSize() {
        $res = $this->doGetSize();
 
        return $res;
    }
 
    abstract protected function doGetSize();
 
    final public function getAcquiredCount() {
        $res = $this->doGetAcquiredCount();
 
        return $res;
    }
 
    abstract protected function doGetAcquiredCount();
 
    final public function getDelayedCount() {
        $res = $this->doGetDelayedCount();
 
        return $res;
    }
 
    protected function doGetDelayedCount() {
        return 0; // not implemented
    }
 
    final public function getAbandonedCount() {
        $res = $this->doGetAbandonedCount();
 
        return $res;
    }
 
    protected function doGetAbandonedCount() {
        return 0; // not implemented
    }
 
    final public function push( $jobs, $flags = 0 ) {
        $jobs = is_array( $jobs ) ? $jobs : [ $jobs ];
        $this->batchPush( $jobs, $flags );
    }
 
    final public function batchPush( array $jobs, $flags = 0 ) {
        $this->assertNotReadOnly();
 
        if ( !count( $jobs ) ) {
            return; // nothing to do
        }
 
        foreach ( $jobs as $job ) {
            if ( $job->getType() !== $this->type ) {
                throw new MWException(
                    "Got '{$job->getType()}' job; expected a '{$this->type}' job." );
            } elseif ( $job->getReleaseTimestamp() && !$this->supportsDelayedJobs() ) {
                throw new MWException(
                    "Got delayed '{$job->getType()}' job; delays are not supported." );
            }
        }
 
        $this->doBatchPush( $jobs, $flags );
        $this->aggr->notifyQueueNonEmpty( $this->wiki, $this->type );
 
        foreach ( $jobs as $job ) {
            if ( $job->isRootJob() ) {
                $this->deduplicateRootJob( $job );
            }
        }
    }
 
    abstract protected function doBatchPush( array $jobs, $flags );
 
    final public function pop() {
        global $wgJobClasses;
 
        $this->assertNotReadOnly();
        if ( !WikiMap::isCurrentWikiDbDomain( $this->wiki ) ) {
            throw new MWException( "Cannot pop '{$this->type}' job off foreign wiki queue." );
        } elseif ( !isset( $wgJobClasses[$this->type] ) ) {
            // Do not pop jobs if there is no class for the queue type
            throw new MWException( "Unrecognized job type '{$this->type}'." );
        }
 
        $job = $this->doPop();
 
        if ( !$job ) {
            $this->aggr->notifyQueueEmpty( $this->wiki, $this->type );
        }
 
        // Flag this job as an old duplicate based on its "root" job...
        try {
            if ( $job && $this->isRootJobOldDuplicate( $job ) ) {
                self::incrStats( 'dupe_pops', $this->type );
                $job = DuplicateJob::newFromJob( $job ); // convert to a no-op
            }
        } catch ( Exception $e ) {
            // don't lose jobs over this
        }
 
        return $job;
    }
 
    abstract protected function doPop();
 
    final public function ack( Job $job ) {
        $this->assertNotReadOnly();
        if ( $job->getType() !== $this->type ) {
            throw new MWException( "Got '{$job->getType()}' job; expected '{$this->type}'." );
        }
 
        $this->doAck( $job );
    }
 
    abstract protected function doAck( Job $job );
 
    final public function deduplicateRootJob( IJobSpecification $job ) {
        $this->assertNotReadOnly();
        if ( $job->getType() !== $this->type ) {
            throw new MWException( "Got '{$job->getType()}' job; expected '{$this->type}'." );
        }
 
        return $this->doDeduplicateRootJob( $job );
    }
 
    protected function doDeduplicateRootJob( IJobSpecification $job ) {
        if ( !$job->hasRootJobParams() ) {
            throw new MWException( "Cannot register root job; missing parameters." );
        }
        $params = $job->getRootJobParams();
 
        $key = $this->getRootJobCacheKey( $params['rootJobSignature'] );
        // Callers should call JobQueueGroup::push() before this method so that if the insert
        // fails, the de-duplication registration will be aborted. Since the insert is
        // deferred till "transaction idle", do the same here, so that the ordering is
        // maintained. Having only the de-duplication registration succeed would cause
        // jobs to become no-ops without any actual jobs that made them redundant.
        $timestamp = $this->dupCache->get( $key ); // current last timestamp of this job
        if ( $timestamp && $timestamp >= $params['rootJobTimestamp'] ) {
            return true; // a newer version of this root job was enqueued
        }
 
        // Update the timestamp of the last root job started at the location...
        return $this->dupCache->set( $key, $params['rootJobTimestamp'], JobQueueDB::ROOTJOB_TTL );
    }
 
    final protected function isRootJobOldDuplicate( Job $job ) {
        if ( $job->getType() !== $this->type ) {
            throw new MWException( "Got '{$job->getType()}' job; expected '{$this->type}'." );
        }
        $isDuplicate = $this->doIsRootJobOldDuplicate( $job );
 
        return $isDuplicate;
    }
 
    protected function doIsRootJobOldDuplicate( Job $job ) {
        if ( !$job->hasRootJobParams() ) {
            return false; // job has no de-deplication info
        }
        $params = $job->getRootJobParams();
 
        $key = $this->getRootJobCacheKey( $params['rootJobSignature'] );
        // Get the last time this root job was enqueued
        $timestamp = $this->dupCache->get( $key );
 
        // Check if a new root job was started at the location after this one's...
        return ( $timestamp && $timestamp > $params['rootJobTimestamp'] );
    }
 
    protected function getRootJobCacheKey( $signature ) {
        list( $db, $prefix ) = wfSplitWikiID( $this->wiki );
 
        return wfForeignMemcKey( $db, $prefix, 'jobqueue', $this->type, 'rootjob', $signature );
    }
 
    final public function delete() {
        $this->assertNotReadOnly();
 
        $this->doDelete();
    }
 
    protected function doDelete() {
        throw new MWException( "This method is not implemented." );
    }
 
    final public function waitForBackups() {
        $this->doWaitForBackups();
    }
 
    protected function doWaitForBackups() {
    }
 
    final public function flushCaches() {
        $this->doFlushCaches();
    }
 
    protected function doFlushCaches() {
    }
 
    abstract public function getAllQueuedJobs();
 
    public function getAllDelayedJobs() {
        return new ArrayIterator( [] ); // not implemented
    }
 
    public function getAllAcquiredJobs() {
        return new ArrayIterator( [] ); // not implemented
    }
 
    public function getAllAbandonedJobs() {
        return new ArrayIterator( [] ); // not implemented
    }
 
    public function getCoalesceLocationInternal() {
        return null;
    }
 
    final public function getSiblingQueuesWithJobs( array $types ) {
        return $this->doGetSiblingQueuesWithJobs( $types );
    }
 
    protected function doGetSiblingQueuesWithJobs( array $types ) {
        return null; // not supported
    }
 
    final public function getSiblingQueueSizes( array $types ) {
        return $this->doGetSiblingQueueSizes( $types );
    }
 
    protected function doGetSiblingQueueSizes( array $types ) {
        return null; // not supported
    }
 
    protected function assertNotReadOnly() {
        if ( $this->readOnlyReason !== false ) {
            throw new JobQueueReadOnlyError( "Job queue is read-only: {$this->readOnlyReason}" );
        }
    }
 
    public static function incrStats( $key, $type, $delta = 1 ) {
        static $stats;
        if ( !$stats ) {
            $stats = MediaWikiServices::getInstance()->getStatsdDataFactory();
        }
        $stats->updateCount( "jobqueue.{$key}.all", $delta );
        $stats->updateCount( "jobqueue.{$key}.{$type}", $delta );
    }
}
 
class JobQueueError extends MWException {
}
 
class JobQueueConnectionError extends JobQueueError {
}
 
class JobQueueReadOnlyError extends JobQueueError {
 
}