Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
0.00% |
0 / 80 |
|
0.00% |
0 / 3 |
CRAP | |
0.00% |
0 / 1 |
| InitImageData | |
0.00% |
0 / 74 |
|
0.00% |
0 / 3 |
110 | |
0.00% |
0 / 1 |
| __construct | |
0.00% |
0 / 14 |
|
0.00% |
0 / 1 |
2 | |||
| execute | |
0.00% |
0 / 49 |
|
0.00% |
0 / 1 |
42 | |||
| waitForMaxPressure | |
0.00% |
0 / 11 |
|
0.00% |
0 / 1 |
12 | |||
| 1 | <?php |
| 2 | |
| 3 | $IP = getenv( 'MW_INSTALL_PATH' ); |
| 4 | if ( $IP === false ) { |
| 5 | $IP = __DIR__ . '/../../..'; |
| 6 | } |
| 7 | require_once "$IP/maintenance/Maintenance.php"; |
| 8 | |
| 9 | use MediaWiki\Maintenance\Maintenance; |
| 10 | use MediaWiki\Title\Title; |
| 11 | use PageImages\Job\InitImageDataJob; |
| 12 | |
| 13 | /** |
| 14 | * @license WTFPL |
| 15 | * @author Max Semenik |
| 16 | */ |
| 17 | class InitImageData extends Maintenance { |
| 18 | public function __construct() { |
| 19 | parent::__construct(); |
| 20 | $this->addDescription( 'Initializes PageImages data' ); |
| 21 | $this->addOption( 'namespaces', |
| 22 | 'Comma-separated list of namespace(s) to refresh', false, true ); |
| 23 | $this->addOption( 'earlier-than', |
| 24 | 'Run only on pages touched earlier than this timestamp', false, true ); |
| 25 | $this->addOption( 'later-than', |
| 26 | 'Run only on pages touched later than this timestamp', false, true ); |
| 27 | $this->addOption( 'start', 'Starting page ID', false, true ); |
| 28 | $this->addOption( 'queue-pressure', 'Maximum number of jobs to enqueue at a time. ' . |
| 29 | 'If not provided or 0 will be run in-process.', false, true ); |
| 30 | $this->addOption( 'quiet', "Don't report on job queue pressure" ); |
| 31 | $this->setBatchSize( 100 ); |
| 32 | |
| 33 | $this->requireExtension( 'PageImages' ); |
| 34 | } |
| 35 | |
| 36 | /** |
| 37 | * Do the actual work of filling out page images |
| 38 | */ |
| 39 | public function execute() { |
| 40 | $lastId = $this->getOption( 'start', 0 ); |
| 41 | $isQuiet = $this->getOption( 'quiet', false ); |
| 42 | $queue = null; |
| 43 | $maxPressure = $this->getOption( 'queue-pressure', 0 ); |
| 44 | if ( $maxPressure > 0 ) { |
| 45 | $queue = $this->getServiceContainer()->getJobQueueGroup(); |
| 46 | } |
| 47 | |
| 48 | do { |
| 49 | $dbr = $this->getServiceContainer()->getDBLoadBalancerFactory() |
| 50 | ->getReplicaDatabase(); |
| 51 | $queryBuilder = $dbr->newSelectQueryBuilder() |
| 52 | ->select( 'page_id' ) |
| 53 | ->from( 'page' ) |
| 54 | ->leftJoin( 'imagelinks', null, 'page_id = il_from' ) |
| 55 | ->where( [ |
| 56 | $dbr->expr( 'page_id', '>', (int)$lastId ), |
| 57 | $dbr->expr( 'il_from', '!=', null ), |
| 58 | 'page_is_redirect' => 0, |
| 59 | ] ) |
| 60 | ->orderBy( 'page_id' ) |
| 61 | ->groupBy( 'page_id' ) |
| 62 | ->limit( $this->mBatchSize ) |
| 63 | ->caller( __METHOD__ ); |
| 64 | if ( $this->hasOption( 'namespaces' ) ) { |
| 65 | $ns = explode( ',', $this->getOption( 'namespaces' ) ); |
| 66 | $queryBuilder->andWhere( [ 'page_namespace' => $ns ] ); |
| 67 | } else { |
| 68 | $queryBuilder->andWhere( [ |
| 69 | 'page_namespace' => $this->getServiceContainer()->getMainConfig()->get( 'PageImagesNamespaces' ) |
| 70 | ] ); |
| 71 | } |
| 72 | if ( $this->hasOption( 'earlier-than' ) ) { |
| 73 | $queryBuilder->andWhere( |
| 74 | $dbr->expr( 'page_touched', '<', $dbr->timestamp( $this->getOption( 'earlier-than' ) ) ) |
| 75 | ); |
| 76 | } |
| 77 | if ( $this->hasOption( 'later-than' ) ) { |
| 78 | $queryBuilder->andWhere( |
| 79 | $dbr->expr( 'page_touched', '>', $dbr->timestamp( $this->getOption( 'later-than' ) ) ) |
| 80 | ); |
| 81 | } |
| 82 | $pageIds = $queryBuilder->fetchFieldValues(); |
| 83 | $job = new InitImageDataJob( |
| 84 | Title::newMainPage(), |
| 85 | [ 'page_ids' => $pageIds ], |
| 86 | $this->getServiceContainer()->getDBLoadBalancerFactory() |
| 87 | ); |
| 88 | if ( $queue === null ) { |
| 89 | $job->run(); |
| 90 | } else { |
| 91 | $queue->push( $job ); |
| 92 | $this->waitForMaxPressure( $queue, $maxPressure, $isQuiet ); |
| 93 | } |
| 94 | $lastId = end( $pageIds ); |
| 95 | $this->output( "$lastId\n" ); |
| 96 | } while ( $pageIds ); |
| 97 | $this->output( "done\n" ); |
| 98 | } |
| 99 | |
| 100 | /** |
| 101 | * @param JobQueueGroup $queue The job queue to fetch pressure from |
| 102 | * @param int $maxPressure The maximum number of queued + active |
| 103 | * jobs that can exist when returning |
| 104 | * @param bool $isQuiet When false report on job queue pressure every 10s |
| 105 | */ |
| 106 | private function waitForMaxPressure( JobQueueGroup $queue, $maxPressure, $isQuiet ) { |
| 107 | $group = $queue->get( 'InitImageDataJob' ); |
| 108 | $i = 0; |
| 109 | do { |
| 110 | sleep( 1 ); |
| 111 | $queued = $group->getSize(); |
| 112 | $running = $group->getAcquiredCount(); |
| 113 | $abandoned = $group->getAbandonedCount(); |
| 114 | |
| 115 | if ( !$isQuiet && ++$i % 10 === 0 ) { |
| 116 | $now = date( 'Y-m-d H:i:s T' ); |
| 117 | $this->output( "[$now] Queued: $queued Running: $running " . |
| 118 | "Abandoned: $abandoned Max: $maxPressure\n" ); |
| 119 | } |
| 120 | } while ( $queued + $running - $abandoned >= $maxPressure ); |
| 121 | } |
| 122 | } |
| 123 | |
| 124 | $maintClass = InitImageData::class; |
| 125 | require_once RUN_MAINTENANCE_IF_MAIN; |