Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
Total | |
0.00% |
0 / 80 |
|
0.00% |
0 / 3 |
CRAP | |
0.00% |
0 / 1 |
InitImageData | |
0.00% |
0 / 74 |
|
0.00% |
0 / 3 |
132 | |
0.00% |
0 / 1 |
__construct | |
0.00% |
0 / 14 |
|
0.00% |
0 / 1 |
2 | |||
execute | |
0.00% |
0 / 49 |
|
0.00% |
0 / 1 |
56 | |||
waitForMaxPressure | |
0.00% |
0 / 11 |
|
0.00% |
0 / 1 |
12 |
1 | <?php |
2 | |
3 | $IP = getenv( 'MW_INSTALL_PATH' ); |
4 | if ( $IP === false ) { |
5 | $IP = __DIR__ . '/../../..'; |
6 | } |
7 | require_once "$IP/maintenance/Maintenance.php"; |
8 | |
9 | use MediaWiki\Title\Title; |
10 | use PageImages\Job\InitImageDataJob; |
11 | |
12 | /** |
13 | * @license WTFPL |
14 | * @author Max Semenik |
15 | */ |
16 | class InitImageData extends Maintenance { |
17 | public function __construct() { |
18 | parent::__construct(); |
19 | $this->addDescription( 'Initializes PageImages data' ); |
20 | $this->addOption( 'namespaces', |
21 | 'Comma-separated list of namespace(s) to refresh', false, true ); |
22 | $this->addOption( 'earlier-than', |
23 | 'Run only on pages touched earlier than this timestamp', false, true ); |
24 | $this->addOption( 'later-than', |
25 | 'Run only on pages touched later than this timestamp', false, true ); |
26 | $this->addOption( 'start', 'Starting page ID', false, true ); |
27 | $this->addOption( 'queue-pressure', 'Maximum number of jobs to enqueue at a time. ' . |
28 | 'If not provided or 0 will be run in-process.', false, true ); |
29 | $this->addOption( 'quiet', "Don't report on job queue pressure" ); |
30 | $this->setBatchSize( 100 ); |
31 | |
32 | $this->requireExtension( 'PageImages' ); |
33 | } |
34 | |
35 | /** |
36 | * Do the actual work of filling out page images |
37 | */ |
38 | public function execute() { |
39 | global $wgPageImagesNamespaces; |
40 | |
41 | $lastId = $this->getOption( 'start', 0 ); |
42 | $isQuiet = $this->getOption( 'quiet', false ); |
43 | $queue = null; |
44 | $maxPressure = $this->getOption( 'queue-pressure', 0 ); |
45 | if ( $maxPressure > 0 ) { |
46 | $queue = $this->getServiceContainer()->getJobQueueGroup(); |
47 | } |
48 | |
49 | do { |
50 | $tables = [ 'page', 'imagelinks' ]; |
51 | $conds = [ |
52 | 'page_id > ' . (int)$lastId, |
53 | 'il_from IS NOT NULL', |
54 | 'page_is_redirect' => 0, |
55 | ]; |
56 | $fields = [ 'page_id' ]; |
57 | $joinConds = [ 'imagelinks' => [ |
58 | 'LEFT JOIN', 'page_id = il_from', |
59 | ] ]; |
60 | |
61 | $dbr = $this->getServiceContainer()->getDBLoadBalancerFactory() |
62 | ->getReplicaDatabase(); |
63 | if ( $this->hasOption( 'namespaces' ) ) { |
64 | $ns = explode( ',', $this->getOption( 'namespaces' ) ); |
65 | $conds['page_namespace'] = $ns; |
66 | } else { |
67 | $conds['page_namespace'] = $wgPageImagesNamespaces; |
68 | } |
69 | if ( $this->hasOption( 'earlier-than' ) ) { |
70 | $conds[] = 'page_touched < ' |
71 | . $dbr->addQuotes( $dbr->timestamp( $this->getOption( 'earlier-than' ) ) ); |
72 | } |
73 | if ( $this->hasOption( 'later-than' ) ) { |
74 | $conds[] = 'page_touched > ' |
75 | . $dbr->addQuotes( $dbr->timestamp( $this->getOption( 'later-than' ) ) ); |
76 | } |
77 | $res = $dbr->select( $tables, $fields, $conds, __METHOD__, |
78 | [ 'LIMIT' => $this->mBatchSize, 'ORDER_BY' => 'page_id', 'GROUP BY' => 'page_id' ], |
79 | $joinConds |
80 | ); |
81 | $pageIds = []; |
82 | foreach ( $res as $row ) { |
83 | $pageIds[] = $row->page_id; |
84 | } |
85 | $job = new InitImageDataJob( |
86 | Title::newMainPage(), |
87 | [ 'page_ids' => $pageIds ], |
88 | $this->getServiceContainer()->getDBLoadBalancerFactory() |
89 | ); |
90 | if ( $queue === null ) { |
91 | $job->run(); |
92 | } else { |
93 | $queue->push( $job ); |
94 | $this->waitForMaxPressure( $queue, $maxPressure, $isQuiet ); |
95 | } |
96 | $lastId = end( $pageIds ); |
97 | $this->output( "$lastId\n" ); |
98 | } while ( $res->numRows() ); |
99 | $this->output( "done\n" ); |
100 | } |
101 | |
102 | /** |
103 | * @param JobQueueGroup $queue The job queue to fetch pressure from |
104 | * @param int $maxPressure The maximum number of queued + active |
105 | * jobs that can exist when returning |
106 | * @param bool $isQuiet When false report on job queue pressure every 10s |
107 | */ |
108 | private function waitForMaxPressure( JobQueueGroup $queue, $maxPressure, $isQuiet ) { |
109 | $group = $queue->get( 'InitImageDataJob' ); |
110 | $i = 0; |
111 | do { |
112 | sleep( 1 ); |
113 | $queued = $group->getSize(); |
114 | $running = $group->getAcquiredCount(); |
115 | $abandoned = $group->getAbandonedCount(); |
116 | |
117 | if ( !$isQuiet && ++$i % 10 === 0 ) { |
118 | $now = date( 'Y-m-d H:i:s T' ); |
119 | $this->output( "[$now] Queued: $queued Running: $running " . |
120 | "Abandoned: $abandoned Max: $maxPressure\n" ); |
121 | } |
122 | } while ( $queued + $running - $abandoned >= $maxPressure ); |
123 | } |
124 | } |
125 | |
126 | $maintClass = InitImageData::class; |
127 | require_once RUN_MAINTENANCE_IF_MAIN; |