Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
Total | |
0.00% |
0 / 81 |
|
0.00% |
0 / 3 |
CRAP | |
0.00% |
0 / 1 |
PurgeChangedPages | |
0.00% |
0 / 78 |
|
0.00% |
0 / 3 |
420 | |
0.00% |
0 / 1 |
__construct | |
0.00% |
0 / 9 |
|
0.00% |
0 / 1 |
2 | |||
execute | |
0.00% |
0 / 56 |
|
0.00% |
0 / 1 |
182 | |||
pageableSortedRows | |
0.00% |
0 / 13 |
|
0.00% |
0 / 1 |
42 |
1 | <?php |
2 | /** |
3 | * Send purge requests for pages edited in date range to squid/varnish. |
4 | * |
5 | * This program is free software; you can redistribute it and/or modify |
6 | * it under the terms of the GNU General Public License as published by |
7 | * the Free Software Foundation; either version 2 of the License, or |
8 | * (at your option) any later version. |
9 | * |
10 | * This program is distributed in the hope that it will be useful, |
11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
13 | * GNU General Public License for more details. |
14 | * |
15 | * You should have received a copy of the GNU General Public License along |
16 | * with this program; if not, write to the Free Software Foundation, Inc., |
17 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
18 | * http://www.gnu.org/copyleft/gpl.html |
19 | * |
20 | * @file |
21 | * @ingroup Maintenance |
22 | */ |
23 | |
24 | require_once __DIR__ . '/Maintenance.php'; |
25 | |
26 | use MediaWiki\Title\Title; |
27 | use Wikimedia\Rdbms\IResultWrapper; |
28 | |
29 | /** |
30 | * Maintenance script that sends purge requests for pages edited in a date |
31 | * range to squid/varnish. |
32 | * |
33 | * Can be used to recover from an HTCP message partition or other major cache |
34 | * layer interruption. |
35 | * |
36 | * @ingroup Maintenance |
37 | */ |
38 | class PurgeChangedPages extends Maintenance { |
39 | |
40 | public function __construct() { |
41 | parent::__construct(); |
42 | $this->addDescription( 'Send purge requests for edits in date range to squid/varnish' ); |
43 | $this->addOption( 'starttime', 'Starting timestamp', true, true ); |
44 | $this->addOption( 'endtime', 'Ending timestamp', true, true ); |
45 | $this->addOption( 'htcp-dest', 'HTCP announcement destination (IP:port)', false, true ); |
46 | $this->addOption( 'sleep-per-batch', 'Milliseconds to sleep between batches', false, true ); |
47 | $this->addOption( 'dry-run', 'Do not send purge requests' ); |
48 | $this->addOption( 'verbose', 'Show more output', false, false, 'v' ); |
49 | $this->setBatchSize( 100 ); |
50 | } |
51 | |
52 | public function execute() { |
53 | global $wgHTCPRouting; |
54 | |
55 | if ( $this->hasOption( 'htcp-dest' ) ) { |
56 | $parts = explode( ':', $this->getOption( 'htcp-dest' ), 2 ); |
57 | if ( count( $parts ) < 2 ) { |
58 | // Add default htcp port |
59 | $parts[] = '4827'; |
60 | } |
61 | |
62 | // Route all HTCP messages to provided host:port |
63 | $wgHTCPRouting = [ |
64 | '' => [ 'host' => $parts[0], 'port' => $parts[1] ], |
65 | ]; |
66 | if ( $this->hasOption( 'verbose' ) ) { |
67 | $this->output( "HTCP broadcasts to {$parts[0]}:{$parts[1]}\n" ); |
68 | } |
69 | } |
70 | |
71 | $dbr = $this->getReplicaDB(); |
72 | $minTime = $dbr->timestamp( $this->getOption( 'starttime' ) ); |
73 | $maxTime = $dbr->timestamp( $this->getOption( 'endtime' ) ); |
74 | |
75 | if ( $maxTime < $minTime ) { |
76 | $this->error( "\nERROR: starttime after endtime\n" ); |
77 | $this->maybeHelp( true ); |
78 | } |
79 | |
80 | $stuckCount = 0; |
81 | while ( true ) { |
82 | // Adjust bach size if we are stuck in a second that had many changes |
83 | $bSize = ( $stuckCount + 1 ) * $this->getBatchSize(); |
84 | |
85 | $res = $dbr->newSelectQueryBuilder() |
86 | ->select( [ 'rev_timestamp', 'page_namespace', 'page_title', ] ) |
87 | ->from( 'revision' ) |
88 | ->join( 'page', null, 'rev_page=page_id' ) |
89 | ->where( [ |
90 | $dbr->expr( 'rev_timestamp', '>', $minTime ), |
91 | $dbr->expr( 'rev_timestamp', '<=', $maxTime ), |
92 | ] ) |
93 | // Only get rows where the revision is the latest for the page. |
94 | // Other revisions would be duplicate and we don't need to purge if |
95 | // there has been an edit after the interesting time window. |
96 | ->andWhere( "page_latest = rev_id" ) |
97 | ->orderBy( 'rev_timestamp' ) |
98 | ->limit( $bSize ) |
99 | ->caller( __METHOD__ )->fetchResultSet(); |
100 | |
101 | if ( !$res->numRows() ) { |
102 | // nothing more found so we are done |
103 | break; |
104 | } |
105 | |
106 | // Kludge to not get stuck in loops for batches with the same timestamp |
107 | [ $rows, $lastTime ] = $this->pageableSortedRows( $res, 'rev_timestamp', $bSize ); |
108 | if ( !count( $rows ) ) { |
109 | ++$stuckCount; |
110 | continue; |
111 | } |
112 | // Reset suck counter |
113 | $stuckCount = 0; |
114 | |
115 | $this->output( "Processing changes from {$minTime} to {$lastTime}.\n" ); |
116 | |
117 | // Advance past the last row next time |
118 | $minTime = $lastTime; |
119 | |
120 | // Create list of URLs from page_namespace + page_title |
121 | $urls = []; |
122 | foreach ( $rows as $row ) { |
123 | $title = Title::makeTitle( $row->page_namespace, $row->page_title ); |
124 | $urls[] = $title->getInternalURL(); |
125 | } |
126 | |
127 | if ( $this->hasOption( 'dry-run' ) || $this->hasOption( 'verbose' ) ) { |
128 | $this->output( implode( "\n", $urls ) . "\n" ); |
129 | if ( $this->hasOption( 'dry-run' ) ) { |
130 | continue; |
131 | } |
132 | } |
133 | |
134 | // Send batch of purge requests out to CDN servers |
135 | $hcu = $this->getServiceContainer()->getHtmlCacheUpdater(); |
136 | $hcu->purgeUrls( $urls, $hcu::PURGE_NAIVE ); |
137 | |
138 | if ( $this->hasOption( 'sleep-per-batch' ) ) { |
139 | // sleep-per-batch is milliseconds, usleep wants micro seconds. |
140 | usleep( 1000 * (int)$this->getOption( 'sleep-per-batch' ) ); |
141 | } |
142 | } |
143 | |
144 | $this->output( "Done!\n" ); |
145 | } |
146 | |
147 | /** |
148 | * Remove all the rows in a result set with the highest value for column |
149 | * $column unless the number of rows is less $limit. This returns the new |
150 | * array of rows and the highest value of column $column for the rows left. |
151 | * The ordering of rows is maintained. |
152 | * |
153 | * This is useful for paging on mostly-unique values that may sometimes |
154 | * have large clumps of identical values. It should be safe to do the next |
155 | * query on items with a value higher than the highest of the rows returned here. |
156 | * If this returns an empty array for a non-empty query result, then all the rows |
157 | * had the same column value and the query should be repeated with a higher LIMIT. |
158 | * |
159 | * @todo move this elsewhere |
160 | * |
161 | * @param IResultWrapper $res Query result sorted by $column (ascending) |
162 | * @param string $column |
163 | * @param int $limit |
164 | * @return array (array of rows, string column value) |
165 | */ |
166 | protected function pageableSortedRows( IResultWrapper $res, $column, $limit ) { |
167 | $rows = iterator_to_array( $res, false ); |
168 | |
169 | // Nothing to do |
170 | if ( !$rows ) { |
171 | return [ [], null ]; |
172 | } |
173 | |
174 | $lastValue = end( $rows )->$column; |
175 | if ( count( $rows ) < $limit ) { |
176 | return [ $rows, $lastValue ]; |
177 | } |
178 | |
179 | for ( $i = count( $rows ) - 1; $i >= 0; --$i ) { |
180 | if ( $rows[$i]->$column !== $lastValue ) { |
181 | break; |
182 | } |
183 | |
184 | unset( $rows[$i] ); |
185 | } |
186 | |
187 | // No more rows left |
188 | if ( !$rows ) { |
189 | return [ [], null ]; |
190 | } |
191 | |
192 | return [ $rows, end( $rows )->$column ]; |
193 | } |
194 | } |
195 | |
196 | $maintClass = PurgeChangedPages::class; |
197 | require_once RUN_MAINTENANCE_IF_MAIN; |