MediaWiki  master
purgeChangedPages.php
Go to the documentation of this file.
1 <?php
24 require_once __DIR__ . '/Maintenance.php';
25 
28 
39 
40  public function __construct() {
41  parent::__construct();
42  $this->addDescription( 'Send purge requests for edits in date range to squid/varnish' );
43  $this->addOption( 'starttime', 'Starting timestamp', true, true );
44  $this->addOption( 'endtime', 'Ending timestamp', true, true );
45  $this->addOption( 'htcp-dest', 'HTCP announcement destination (IP:port)', false, true );
46  $this->addOption( 'sleep-per-batch', 'Milliseconds to sleep between batches', false, true );
47  $this->addOption( 'dry-run', 'Do not send purge requests' );
48  $this->addOption( 'verbose', 'Show more output', false, false, 'v' );
49  $this->setBatchSize( 100 );
50  }
51 
52  public function execute() {
53  global $wgHTCPRouting;
54 
55  if ( $this->hasOption( 'htcp-dest' ) ) {
56  $parts = explode( ':', $this->getOption( 'htcp-dest' ), 2 );
57  if ( count( $parts ) < 2 ) {
58  // Add default htcp port
59  $parts[] = '4827';
60  }
61 
62  // Route all HTCP messages to provided host:port
63  $wgHTCPRouting = [
64  '' => [ 'host' => $parts[0], 'port' => $parts[1] ],
65  ];
66  if ( $this->hasOption( 'verbose' ) ) {
67  $this->output( "HTCP broadcasts to {$parts[0]}:{$parts[1]}\n" );
68  }
69  }
70 
71  $dbr = $this->getDB( DB_REPLICA );
72  $minTime = $dbr->timestamp( $this->getOption( 'starttime' ) );
73  $maxTime = $dbr->timestamp( $this->getOption( 'endtime' ) );
74 
75  if ( $maxTime < $minTime ) {
76  $this->error( "\nERROR: starttime after endtime\n" );
77  $this->maybeHelp( true );
78  }
79 
80  $stuckCount = 0;
81  while ( true ) {
82  // Adjust bach size if we are stuck in a second that had many changes
83  $bSize = ( $stuckCount + 1 ) * $this->getBatchSize();
84 
85  $res = $dbr->select(
86  [ 'page', 'revision' ],
87  [
88  'rev_timestamp',
89  'page_namespace',
90  'page_title',
91  ],
92  [
93  "rev_timestamp > " . $dbr->addQuotes( $minTime ),
94  "rev_timestamp <= " . $dbr->addQuotes( $maxTime ),
95  // Only get rows where the revision is the latest for the page.
96  // Other revisions would be duplicate and we don't need to purge if
97  // there has been an edit after the interesting time window.
98  "page_latest = rev_id",
99  ],
100  __METHOD__,
101  [ 'ORDER BY' => 'rev_timestamp', 'LIMIT' => $bSize ],
102  [
103  'page' => [ 'JOIN', 'rev_page=page_id' ],
104  ]
105  );
106 
107  if ( !$res->numRows() ) {
108  // nothing more found so we are done
109  break;
110  }
111 
112  // Kludge to not get stuck in loops for batches with the same timestamp
113  list( $rows, $lastTime ) = $this->pageableSortedRows( $res, 'rev_timestamp', $bSize );
114  if ( !count( $rows ) ) {
115  ++$stuckCount;
116  continue;
117  }
118  // Reset suck counter
119  $stuckCount = 0;
120 
121  $this->output( "Processing changes from {$minTime} to {$lastTime}.\n" );
122 
123  // Advance past the last row next time
124  $minTime = $lastTime;
125 
126  // Create list of URLs from page_namespace + page_title
127  $urls = [];
128  foreach ( $rows as $row ) {
129  $title = Title::makeTitle( $row->page_namespace, $row->page_title );
130  $urls[] = $title->getInternalURL();
131  }
132 
133  if ( $this->hasOption( 'dry-run' ) || $this->hasOption( 'verbose' ) ) {
134  $this->output( implode( "\n", $urls ) . "\n" );
135  if ( $this->hasOption( 'dry-run' ) ) {
136  continue;
137  }
138  }
139 
140  // Send batch of purge requests out to CDN servers
141  $hcu = MediaWikiServices::getInstance()->getHtmlCacheUpdater();
142  $hcu->purgeUrls( $urls, $hcu::PURGE_NAIVE );
143 
144  if ( $this->hasOption( 'sleep-per-batch' ) ) {
145  // sleep-per-batch is milliseconds, usleep wants micro seconds.
146  usleep( 1000 * (int)$this->getOption( 'sleep-per-batch' ) );
147  }
148  }
149 
150  $this->output( "Done!\n" );
151  }
152 
172  protected function pageableSortedRows( IResultWrapper $res, $column, $limit ) {
173  $rows = iterator_to_array( $res, false );
174 
175  // Nothing to do
176  if ( !$rows ) {
177  return [ [], null ];
178  }
179 
180  $lastValue = end( $rows )->$column;
181  if ( count( $rows ) < $limit ) {
182  return [ $rows, $lastValue ];
183  }
184 
185  for ( $i = count( $rows ) - 1; $i >= 0; --$i ) {
186  if ( $rows[$i]->$column !== $lastValue ) {
187  break;
188  }
189 
190  unset( $rows[$i] );
191  }
192 
193  // No more rows left
194  if ( !$rows ) {
195  return [ [], null ];
196  }
197 
198  return [ $rows, end( $rows )->$column ];
199  }
200 }
201 
202 $maintClass = PurgeChangedPages::class;
203 require_once RUN_MAINTENANCE_IF_MAIN;
RUN_MAINTENANCE_IF_MAIN
const RUN_MAINTENANCE_IF_MAIN
Definition: Maintenance.php:38
PurgeChangedPages\pageableSortedRows
pageableSortedRows(IResultWrapper $res, $column, $limit)
Remove all the rows in a result set with the highest value for column $column unless the number of ro...
Definition: purgeChangedPages.php:172
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:163
Maintenance\maybeHelp
maybeHelp( $force=false)
Maybe show the help.
Definition: Maintenance.php:1050
Maintenance\addDescription
addDescription( $text)
Set the description text.
Definition: Maintenance.php:327
PurgeChangedPages\execute
execute()
Do the actual work.
Definition: purgeChangedPages.php:52
PurgeChangedPages\__construct
__construct()
Default constructor.
Definition: purgeChangedPages.php:40
Maintenance
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:55
$res
$res
Definition: testCompression.php:57
PurgeChangedPages
Maintenance script that sends purge requests for pages edited in a date range to squid/varnish.
Definition: purgeChangedPages.php:38
$dbr
$dbr
Definition: testCompression.php:54
Wikimedia\Rdbms\IResultWrapper
Result wrapper for grabbing data queried from an IDatabase object.
Definition: IResultWrapper.php:24
Maintenance\addOption
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
Definition: Maintenance.php:245
$wgHTCPRouting
$wgHTCPRouting
Routing configuration for HTCP multicast purging.
Definition: DefaultSettings.php:3067
$title
$title
Definition: testCompression.php:38
Title\makeTitle
static makeTitle( $ns, $title, $fragment='', $interwiki='')
Create a new Title from a namespace index and a DB key.
Definition: Title.php:591
DB_REPLICA
const DB_REPLICA
Definition: defines.php:25
$maintClass
$maintClass
Definition: purgeChangedPages.php:202
Maintenance\getDB
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
Definition: Maintenance.php:1370
Maintenance\getOption
getOption( $name, $default=null)
Get an option, or return the default.
Definition: Maintenance.php:281
Maintenance\getBatchSize
getBatchSize()
Returns batch size.
Definition: Maintenance.php:366
Maintenance\error
error( $err, $die=0)
Throw an error to the user.
Definition: Maintenance.php:463
Maintenance\output
output( $out, $channel=null)
Throw some output to the user.
Definition: Maintenance.php:434
Maintenance\hasOption
hasOption( $name)
Checks to see if a particular option exists.
Definition: Maintenance.php:266
Maintenance\setBatchSize
setBatchSize( $s=0)
Set the batch size.
Definition: Maintenance.php:374