Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
Total | |
0.00% |
0 / 80 |
|
0.00% |
0 / 3 |
CRAP | |
0.00% |
0 / 1 |
InitImageData | |
0.00% |
0 / 74 |
|
0.00% |
0 / 3 |
110 | |
0.00% |
0 / 1 |
__construct | |
0.00% |
0 / 14 |
|
0.00% |
0 / 1 |
2 | |||
execute | |
0.00% |
0 / 49 |
|
0.00% |
0 / 1 |
42 | |||
waitForMaxPressure | |
0.00% |
0 / 11 |
|
0.00% |
0 / 1 |
12 |
1 | <?php |
2 | |
3 | $IP = getenv( 'MW_INSTALL_PATH' ); |
4 | if ( $IP === false ) { |
5 | $IP = __DIR__ . '/../../..'; |
6 | } |
7 | require_once "$IP/maintenance/Maintenance.php"; |
8 | |
9 | use MediaWiki\Title\Title; |
10 | use PageImages\Job\InitImageDataJob; |
11 | |
12 | /** |
13 | * @license WTFPL |
14 | * @author Max Semenik |
15 | */ |
16 | class InitImageData extends Maintenance { |
17 | public function __construct() { |
18 | parent::__construct(); |
19 | $this->addDescription( 'Initializes PageImages data' ); |
20 | $this->addOption( 'namespaces', |
21 | 'Comma-separated list of namespace(s) to refresh', false, true ); |
22 | $this->addOption( 'earlier-than', |
23 | 'Run only on pages touched earlier than this timestamp', false, true ); |
24 | $this->addOption( 'later-than', |
25 | 'Run only on pages touched later than this timestamp', false, true ); |
26 | $this->addOption( 'start', 'Starting page ID', false, true ); |
27 | $this->addOption( 'queue-pressure', 'Maximum number of jobs to enqueue at a time. ' . |
28 | 'If not provided or 0 will be run in-process.', false, true ); |
29 | $this->addOption( 'quiet', "Don't report on job queue pressure" ); |
30 | $this->setBatchSize( 100 ); |
31 | |
32 | $this->requireExtension( 'PageImages' ); |
33 | } |
34 | |
35 | /** |
36 | * Do the actual work of filling out page images |
37 | */ |
38 | public function execute() { |
39 | $lastId = $this->getOption( 'start', 0 ); |
40 | $isQuiet = $this->getOption( 'quiet', false ); |
41 | $queue = null; |
42 | $maxPressure = $this->getOption( 'queue-pressure', 0 ); |
43 | if ( $maxPressure > 0 ) { |
44 | $queue = $this->getServiceContainer()->getJobQueueGroup(); |
45 | } |
46 | |
47 | do { |
48 | $dbr = $this->getServiceContainer()->getDBLoadBalancerFactory() |
49 | ->getReplicaDatabase(); |
50 | $queryBuilder = $dbr->newSelectQueryBuilder() |
51 | ->select( 'page_id' ) |
52 | ->from( 'page' ) |
53 | ->leftJoin( 'imagelinks', null, 'page_id = il_from' ) |
54 | ->where( [ |
55 | $dbr->expr( 'page_id', '>', (int)$lastId ), |
56 | $dbr->expr( 'il_from', '!=', null ), |
57 | 'page_is_redirect' => 0, |
58 | ] ) |
59 | ->orderBy( 'page_id' ) |
60 | ->groupBy( 'page_id' ) |
61 | ->limit( $this->mBatchSize ) |
62 | ->caller( __METHOD__ ); |
63 | if ( $this->hasOption( 'namespaces' ) ) { |
64 | $ns = explode( ',', $this->getOption( 'namespaces' ) ); |
65 | $queryBuilder->andWhere( [ 'page_namespace' => $ns ] ); |
66 | } else { |
67 | $queryBuilder->andWhere( [ |
68 | 'page_namespace' => $this->getServiceContainer()->getMainConfig()->get( 'PageImagesNamespaces' ) |
69 | ] ); |
70 | } |
71 | if ( $this->hasOption( 'earlier-than' ) ) { |
72 | $queryBuilder->andWhere( |
73 | $dbr->expr( 'page_touched', '<', $dbr->timestamp( $this->getOption( 'earlier-than' ) ) ) |
74 | ); |
75 | } |
76 | if ( $this->hasOption( 'later-than' ) ) { |
77 | $queryBuilder->andWhere( |
78 | $dbr->expr( 'page_touched', '>', $dbr->timestamp( $this->getOption( 'later-than' ) ) ) |
79 | ); |
80 | } |
81 | $pageIds = $queryBuilder->fetchFieldValues(); |
82 | $job = new InitImageDataJob( |
83 | Title::newMainPage(), |
84 | [ 'page_ids' => $pageIds ], |
85 | $this->getServiceContainer()->getDBLoadBalancerFactory() |
86 | ); |
87 | if ( $queue === null ) { |
88 | $job->run(); |
89 | } else { |
90 | $queue->push( $job ); |
91 | $this->waitForMaxPressure( $queue, $maxPressure, $isQuiet ); |
92 | } |
93 | $lastId = end( $pageIds ); |
94 | $this->output( "$lastId\n" ); |
95 | } while ( $pageIds ); |
96 | $this->output( "done\n" ); |
97 | } |
98 | |
99 | /** |
100 | * @param JobQueueGroup $queue The job queue to fetch pressure from |
101 | * @param int $maxPressure The maximum number of queued + active |
102 | * jobs that can exist when returning |
103 | * @param bool $isQuiet When false report on job queue pressure every 10s |
104 | */ |
105 | private function waitForMaxPressure( JobQueueGroup $queue, $maxPressure, $isQuiet ) { |
106 | $group = $queue->get( 'InitImageDataJob' ); |
107 | $i = 0; |
108 | do { |
109 | sleep( 1 ); |
110 | $queued = $group->getSize(); |
111 | $running = $group->getAcquiredCount(); |
112 | $abandoned = $group->getAbandonedCount(); |
113 | |
114 | if ( !$isQuiet && ++$i % 10 === 0 ) { |
115 | $now = date( 'Y-m-d H:i:s T' ); |
116 | $this->output( "[$now] Queued: $queued Running: $running " . |
117 | "Abandoned: $abandoned Max: $maxPressure\n" ); |
118 | } |
119 | } while ( $queued + $running - $abandoned >= $maxPressure ); |
120 | } |
121 | } |
122 | |
123 | $maintClass = InitImageData::class; |
124 | require_once RUN_MAINTENANCE_IF_MAIN; |