MediaWiki master
purgeChangedPages.php
Go to the documentation of this file.
1<?php
24// @codeCoverageIgnoreStart
25require_once __DIR__ . '/Maintenance.php';
26// @codeCoverageIgnoreEnd
27
31
42
43 public function __construct() {
44 parent::__construct();
45 $this->addDescription( 'Send purge requests for edits in date range to squid/varnish' );
46 $this->addOption( 'starttime', 'Starting timestamp', true, true );
47 $this->addOption( 'endtime', 'Ending timestamp', true, true );
48 $this->addOption( 'htcp-dest', 'HTCP announcement destination (IP:port)', false, true );
49 $this->addOption( 'sleep-per-batch', 'Milliseconds to sleep between batches', false, true );
50 $this->addOption( 'dry-run', 'Do not send purge requests' );
51 $this->addOption( 'verbose', 'Show more output', false, false, 'v' );
52 $this->setBatchSize( 100 );
53 }
54
55 public function execute() {
56 global $wgHTCPRouting;
57
58 if ( $this->hasOption( 'htcp-dest' ) ) {
59 $parts = explode( ':', $this->getOption( 'htcp-dest' ), 2 );
60 if ( count( $parts ) < 2 ) {
61 // Add default htcp port
62 $parts[] = '4827';
63 }
64
65 // Route all HTCP messages to provided host:port
67 '' => [ 'host' => $parts[0], 'port' => $parts[1] ],
68 ];
69 if ( $this->hasOption( 'verbose' ) ) {
70 $this->output( "HTCP broadcasts to {$parts[0]}:{$parts[1]}\n" );
71 }
72 }
73
74 $dbr = $this->getReplicaDB();
75 $minTime = $dbr->timestamp( $this->getOption( 'starttime' ) );
76 $maxTime = $dbr->timestamp( $this->getOption( 'endtime' ) );
77
78 if ( $maxTime < $minTime ) {
79 $this->error( "\nERROR: starttime after endtime\n" );
80 $this->maybeHelp( true );
81 }
82
83 $stuckCount = 0;
84 while ( true ) {
85 // Adjust bach size if we are stuck in a second that had many changes
86 $bSize = ( $stuckCount + 1 ) * $this->getBatchSize();
87
88 $res = $dbr->newSelectQueryBuilder()
89 ->select( [ 'rev_timestamp', 'page_namespace', 'page_title', ] )
90 ->from( 'revision' )
91 ->join( 'page', null, 'rev_page=page_id' )
92 ->where( [
93 $dbr->expr( 'rev_timestamp', '>', $minTime ),
94 $dbr->expr( 'rev_timestamp', '<=', $maxTime ),
95 ] )
96 // Only get rows where the revision is the latest for the page.
97 // Other revisions would be duplicate and we don't need to purge if
98 // there has been an edit after the interesting time window.
99 ->andWhere( "page_latest = rev_id" )
100 ->orderBy( 'rev_timestamp' )
101 ->limit( $bSize )
102 ->caller( __METHOD__ )->fetchResultSet();
103
104 if ( !$res->numRows() ) {
105 // nothing more found so we are done
106 break;
107 }
108
109 // Kludge to not get stuck in loops for batches with the same timestamp
110 [ $rows, $lastTime ] = $this->pageableSortedRows( $res, 'rev_timestamp', $bSize );
111 if ( !count( $rows ) ) {
112 ++$stuckCount;
113 continue;
114 }
115 // Reset suck counter
116 $stuckCount = 0;
117
118 $this->output( "Processing changes from {$minTime} to {$lastTime}.\n" );
119
120 // Advance past the last row next time
121 $minTime = $lastTime;
122
123 // Create list of URLs from page_namespace + page_title
124 $urls = [];
125 foreach ( $rows as $row ) {
126 $title = Title::makeTitle( $row->page_namespace, $row->page_title );
127 $urls[] = $title->getInternalURL();
128 }
129
130 if ( $this->hasOption( 'dry-run' ) || $this->hasOption( 'verbose' ) ) {
131 $this->output( implode( "\n", $urls ) . "\n" );
132 if ( $this->hasOption( 'dry-run' ) ) {
133 continue;
134 }
135 }
136
137 // Send batch of purge requests out to CDN servers
138 $hcu = $this->getServiceContainer()->getHtmlCacheUpdater();
139 $hcu->purgeUrls( $urls, $hcu::PURGE_NAIVE );
140
141 if ( $this->hasOption( 'sleep-per-batch' ) ) {
142 // sleep-per-batch is milliseconds, usleep wants micro seconds.
143 usleep( 1000 * (int)$this->getOption( 'sleep-per-batch' ) );
144 }
145 }
146
147 $this->output( "Done!\n" );
148 }
149
169 protected function pageableSortedRows( IResultWrapper $res, $column, $limit ) {
170 $rows = iterator_to_array( $res, false );
171
172 // Nothing to do
173 if ( !$rows ) {
174 return [ [], null ];
175 }
176
177 $lastValue = end( $rows )->$column;
178 if ( count( $rows ) < $limit ) {
179 return [ $rows, $lastValue ];
180 }
181
182 for ( $i = count( $rows ) - 1; $i >= 0; --$i ) {
183 if ( $rows[$i]->$column !== $lastValue ) {
184 break;
185 }
186
187 unset( $rows[$i] );
188 }
189
190 // No more rows left
191 if ( !$rows ) {
192 return [ [], null ];
193 }
194
195 return [ $rows, end( $rows )->$column ];
196 }
197}
198
199// @codeCoverageIgnoreStart
200$maintClass = PurgeChangedPages::class;
201require_once RUN_MAINTENANCE_IF_MAIN;
202// @codeCoverageIgnoreEnd
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
getBatchSize()
Returns batch size.
output( $out, $channel=null)
Throw some output to the user.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
hasOption( $name)
Checks to see if a particular option was set.
getOption( $name, $default=null)
Get an option, or return the default.
error( $err, $die=0)
Throw an error to the user.
getServiceContainer()
Returns the main service container.
maybeHelp( $force=false)
Maybe show the help.
addDescription( $text)
Set the description text.
Represents a title within MediaWiki.
Definition Title.php:78
Maintenance script that sends purge requests for pages edited in a date range to squid/varnish.
execute()
Do the actual work.
__construct()
Default constructor.
pageableSortedRows(IResultWrapper $res, $column, $limit)
Remove all the rows in a result set with the highest value for column $column unless the number of ro...
$wgHTCPRouting
Config variable stub for the HTCPRouting setting, for use by phpdoc and IDEs.
Result wrapper for grabbing data queried from an IDatabase object.