MediaWiki  master
CdnCacheUpdate.php
Go to the documentation of this file.
1 <?php
25 
32  private $urls = [];
33 
37  public function __construct( array $urlArr ) {
38  $this->urls = $urlArr;
39  }
40 
41  public function merge( MergeableUpdate $update ) {
43  Assert::parameterType( __CLASS__, $update, '$update' );
44  '@phan-var self $update';
45 
46  $this->urls = array_merge( $this->urls, $update->urls );
47  }
48 
56  public static function newFromTitles( $titles, $urlArr = [] ) {
57  ( new LinkBatch( $titles ) )->execute();
59  foreach ( $titles as $title ) {
60  $urlArr = array_merge( $urlArr, $title->getCdnUrls() );
61  }
62 
63  return new CdnCacheUpdate( $urlArr );
64  }
65 
69  public function doUpdate() {
71 
72  self::purge( $this->urls );
73 
74  if ( $wgCdnReboundPurgeDelay > 0 ) {
75  JobQueueGroup::singleton()->lazyPush( new CdnPurgeJob( [
76  'urls' => $this->urls,
77  'jobReleaseTimestamp' => time() + $wgCdnReboundPurgeDelay
78  ] ) );
79  }
80  }
81 
89  public static function purge( array $urlArr ) {
91 
92  if ( !$urlArr ) {
93  return;
94  }
95 
96  // Remove duplicate URLs from list
97  $urlArr = array_unique( $urlArr );
98 
99  wfDebugLog( 'squid', __METHOD__ . ': ' . implode( ' ', $urlArr ) );
100 
101  // Reliably broadcast the purge to all edge nodes
102  $ts = microtime( true );
103  $relayerGroup = MediaWikiServices::getInstance()->getEventRelayerGroup();
104  $relayerGroup->getRelayer( 'cdn-url-purges' )->notifyMulti(
105  'cdn-url-purges',
106  array_map(
107  function ( $url ) use ( $ts ) {
108  return [
109  'url' => $url,
110  'timestamp' => $ts,
111  ];
112  },
113  $urlArr
114  )
115  );
116 
117  // Send lossy UDP broadcasting if enabled
118  if ( $wgHTCPRouting ) {
119  self::HTCPPurge( $urlArr );
120  }
121 
122  // Do direct server purges if enabled (this does not scale very well)
123  if ( $wgCdnServers ) {
124  // Maximum number of parallel connections per CDN
125  $maxSocketsPerCdn = 8;
126  // Number of requests to send per socket
127  // 400 seems to be a good tradeoff, opening a socket takes a while
128  $urlsPerSocket = 400;
129  $socketsPerCdn = ceil( count( $urlArr ) / $urlsPerSocket );
130  if ( $socketsPerCdn > $maxSocketsPerCdn ) {
131  $socketsPerCdn = $maxSocketsPerCdn;
132  }
133 
134  $pool = new SquidPurgeClientPool;
135  $chunks = array_chunk( $urlArr, ceil( count( $urlArr ) / $socketsPerCdn ) );
136  foreach ( $wgCdnServers as $server ) {
137  foreach ( $chunks as $chunk ) {
138  $client = new SquidPurgeClient( $server );
139  foreach ( $chunk as $url ) {
140  $client->queuePurge( self::expand( $url ) );
141  }
142  $pool->addClient( $client );
143  }
144  }
145 
146  $pool->run();
147  }
148  }
149 
156  private static function HTCPPurge( array $urlArr ) {
158 
159  // HTCP CLR operation
160  $htcpOpCLR = 4;
161 
162  // @todo FIXME: PHP doesn't support these socket constants (include/linux/in.h)
163  if ( !defined( "IPPROTO_IP" ) ) {
164  define( "IPPROTO_IP", 0 );
165  define( "IP_MULTICAST_LOOP", 34 );
166  define( "IP_MULTICAST_TTL", 33 );
167  }
168 
169  // pfsockopen doesn't work because we need set_sock_opt
170  $conn = socket_create( AF_INET, SOCK_DGRAM, SOL_UDP );
171  if ( !$conn ) {
172  $errstr = socket_strerror( socket_last_error() );
173  wfDebugLog( 'squid', __METHOD__ .
174  ": Error opening UDP socket: $errstr" );
175 
176  return;
177  }
178 
179  // Set socket options
180  socket_set_option( $conn, IPPROTO_IP, IP_MULTICAST_LOOP, 0 );
181  if ( $wgHTCPMulticastTTL != 1 ) {
182  // Set multicast time to live (hop count) option on socket
183  socket_set_option( $conn, IPPROTO_IP, IP_MULTICAST_TTL,
184  $wgHTCPMulticastTTL );
185  }
186 
187  // Get sequential trx IDs for packet loss counting
189  'squidhtcppurge', 32, count( $urlArr ), UIDGenerator::QUICK_VOLATILE
190  );
191 
192  foreach ( $urlArr as $url ) {
193  if ( !is_string( $url ) ) {
194  throw new MWException( 'Bad purge URL' );
195  }
196  $url = self::expand( $url );
197  $conf = self::getRuleForURL( $url, $wgHTCPRouting );
198  if ( !$conf ) {
199  wfDebugLog( 'squid', __METHOD__ .
200  "No HTCP rule configured for URL {$url} , skipping" );
201  continue;
202  }
203 
204  if ( isset( $conf['host'] ) && isset( $conf['port'] ) ) {
205  // Normalize single entries
206  $conf = [ $conf ];
207  }
208  foreach ( $conf as $subconf ) {
209  if ( !isset( $subconf['host'] ) || !isset( $subconf['port'] ) ) {
210  throw new MWException( "Invalid HTCP rule for URL $url\n" );
211  }
212  }
213 
214  // Construct a minimal HTCP request diagram
215  // as per RFC 2756
216  // Opcode 'CLR', no response desired, no auth
217  $htcpTransID = current( $ids );
218  next( $ids );
219 
220  $htcpSpecifier = pack( 'na4na*na8n',
221  4, 'HEAD', strlen( $url ), $url,
222  8, 'HTTP/1.0', 0 );
223 
224  $htcpDataLen = 8 + 2 + strlen( $htcpSpecifier );
225  $htcpLen = 4 + $htcpDataLen + 2;
226 
227  // Note! Squid gets the bit order of the first
228  // word wrong, wrt the RFC. Apparently no other
229  // implementation exists, so adapt to Squid
230  $htcpPacket = pack( 'nxxnCxNxxa*n',
231  $htcpLen, $htcpDataLen, $htcpOpCLR,
232  $htcpTransID, $htcpSpecifier, 2 );
233 
234  wfDebugLog( 'squid', __METHOD__ .
235  "Purging URL $url via HTCP" );
236  foreach ( $conf as $subconf ) {
237  socket_sendto( $conn, $htcpPacket, $htcpLen, 0,
238  $subconf['host'], $subconf['port'] );
239  }
240  }
241  }
242 
257  private static function expand( $url ) {
258  return wfExpandUrl( $url, PROTO_INTERNAL );
259  }
260 
267  private static function getRuleForURL( $url, $rules ) {
268  foreach ( $rules as $regex => $routing ) {
269  if ( $regex === '' || preg_match( $regex, $url ) ) {
270  return $routing;
271  }
272  }
273 
274  return false;
275  }
276 }
const QUICK_VOLATILE
static HTCPPurge(array $urlArr)
Send Hyper Text Caching Protocol (HTCP) CLR requests.
static purge(array $urlArr)
Purges a list of CDN nodes defined in $wgCdnServers.
wfExpandUrl( $url, $defaultProto=PROTO_CURRENT)
Expand a potentially local URL to a fully-qualified URL.
static expand( $url)
Expand local URLs to fully-qualified URLs using the internal protocol and host defined in $wgInternal...
Handles purging the appropriate CDN objects given a list of URLs or Title instances.
Interface that deferrable updates can implement to signal that updates can be combined.
$wgCdnServers
List of proxy servers to purge on changes; default port is 80.
static getRuleForURL( $url, $rules)
Find the HTCP routing rule to use for a given URL.
Class representing a list of titles The execute() method checks them all for existence and adds them ...
Definition: LinkBatch.php:34
static newFromTitles( $titles, $urlArr=[])
Create an update object from an array of Title objects, or a TitleArray object.
const PROTO_INTERNAL
Definition: Defines.php:204
string [] $urls
Collection of URLs to purge.
An HTTP 1.0 client built for the purposes of purging Squid and Varnish.
doUpdate()
Purges the list of URLs passed to the constructor.
merge(MergeableUpdate $update)
Merge this update with $update.
__construct(array $urlArr)
$wgHTCPMulticastTTL
HTCP multicast TTL.
static newSequentialPerNodeIDs( $bucket, $bits, $count, $flags=0)
Return IDs that are sequential only for this node and bucket.
wfDebugLog( $logGroup, $text, $dest='all', array $context=[])
Send a line to a supplementary debug log file, if configured, or main debug log if not...
static singleton( $domain=false)
$wgHTCPRouting
Routing configuration for HTCP multicast purging.
Job to purge a set of URLs from CDN.
Definition: CdnPurgeJob.php:30
$wgCdnReboundPurgeDelay
If set, any SquidPurge call on a URL or URLs will send a second purge no less than this many seconds ...