MediaWiki  REL1_31
CdnCacheUpdate.php
Go to the documentation of this file.
1 <?php
23 use Wikimedia\Assert\Assert;
25 
32  protected $urls = [];
33 
37  public function __construct( array $urlArr ) {
38  $this->urls = $urlArr;
39  }
40 
41  public function merge( MergeableUpdate $update ) {
43  Assert::parameterType( __CLASS__, $update, '$update' );
44 
45  $this->urls = array_merge( $this->urls, $update->urls );
46  }
47 
55  public static function newFromTitles( $titles, $urlArr = [] ) {
56  ( new LinkBatch( $titles ) )->execute();
58  foreach ( $titles as $title ) {
59  $urlArr = array_merge( $urlArr, $title->getCdnUrls() );
60  }
61 
62  return new CdnCacheUpdate( $urlArr );
63  }
64 
70  public static function newSimplePurge( Title $title ) {
71  return new CdnCacheUpdate( $title->getCdnUrls() );
72  }
73 
77  public function doUpdate() {
79 
80  self::purge( $this->urls );
81 
82  if ( $wgCdnReboundPurgeDelay > 0 ) {
83  JobQueueGroup::singleton()->lazyPush( new CdnPurgeJob(
84  Title::makeTitle( NS_SPECIAL, 'Badtitle/' . __CLASS__ ),
85  [
86  'urls' => $this->urls,
87  'jobReleaseTimestamp' => time() + $wgCdnReboundPurgeDelay
88  ]
89  ) );
90  }
91  }
92 
100  public static function purge( array $urlArr ) {
102 
103  if ( !$urlArr ) {
104  return;
105  }
106 
107  // Remove duplicate URLs from list
108  $urlArr = array_unique( $urlArr );
109 
110  wfDebugLog( 'squid', __METHOD__ . ': ' . implode( ' ', $urlArr ) );
111 
112  // Reliably broadcast the purge to all edge nodes
113  $relayer = MediaWikiServices::getInstance()->getEventRelayerGroup()
114  ->getRelayer( 'cdn-url-purges' );
115  $ts = microtime( true );
116  $relayer->notifyMulti(
117  'cdn-url-purges',
118  array_map(
119  function ( $url ) use ( $ts ) {
120  return [
121  'url' => $url,
122  'timestamp' => $ts,
123  ];
124  },
125  $urlArr
126  )
127  );
128 
129  // Send lossy UDP broadcasting if enabled
130  if ( $wgHTCPRouting ) {
131  self::HTCPPurge( $urlArr );
132  }
133 
134  // Do direct server purges if enabled (this does not scale very well)
135  if ( $wgSquidServers ) {
136  // Maximum number of parallel connections per squid
137  $maxSocketsPerSquid = 8;
138  // Number of requests to send per socket
139  // 400 seems to be a good tradeoff, opening a socket takes a while
140  $urlsPerSocket = 400;
141  $socketsPerSquid = ceil( count( $urlArr ) / $urlsPerSocket );
142  if ( $socketsPerSquid > $maxSocketsPerSquid ) {
143  $socketsPerSquid = $maxSocketsPerSquid;
144  }
145 
146  $pool = new SquidPurgeClientPool;
147  $chunks = array_chunk( $urlArr, ceil( count( $urlArr ) / $socketsPerSquid ) );
148  foreach ( $wgSquidServers as $server ) {
149  foreach ( $chunks as $chunk ) {
150  $client = new SquidPurgeClient( $server );
151  foreach ( $chunk as $url ) {
152  $client->queuePurge( $url );
153  }
154  $pool->addClient( $client );
155  }
156  }
157 
158  $pool->run();
159  }
160  }
161 
168  private static function HTCPPurge( array $urlArr ) {
170 
171  // HTCP CLR operation
172  $htcpOpCLR = 4;
173 
174  // @todo FIXME: PHP doesn't support these socket constants (include/linux/in.h)
175  if ( !defined( "IPPROTO_IP" ) ) {
176  define( "IPPROTO_IP", 0 );
177  define( "IP_MULTICAST_LOOP", 34 );
178  define( "IP_MULTICAST_TTL", 33 );
179  }
180 
181  // pfsockopen doesn't work because we need set_sock_opt
182  $conn = socket_create( AF_INET, SOCK_DGRAM, SOL_UDP );
183  if ( !$conn ) {
184  $errstr = socket_strerror( socket_last_error() );
185  wfDebugLog( 'squid', __METHOD__ .
186  ": Error opening UDP socket: $errstr" );
187 
188  return;
189  }
190 
191  // Set socket options
192  socket_set_option( $conn, IPPROTO_IP, IP_MULTICAST_LOOP, 0 );
193  if ( $wgHTCPMulticastTTL != 1 ) {
194  // Set multicast time to live (hop count) option on socket
195  socket_set_option( $conn, IPPROTO_IP, IP_MULTICAST_TTL,
197  }
198 
199  // Get sequential trx IDs for packet loss counting
201  'squidhtcppurge', 32, count( $urlArr ), UIDGenerator::QUICK_VOLATILE
202  );
203 
204  foreach ( $urlArr as $url ) {
205  if ( !is_string( $url ) ) {
206  throw new MWException( 'Bad purge URL' );
207  }
208  $url = self::expand( $url );
209  $conf = self::getRuleForURL( $url, $wgHTCPRouting );
210  if ( !$conf ) {
211  wfDebugLog( 'squid', __METHOD__ .
212  "No HTCP rule configured for URL {$url} , skipping" );
213  continue;
214  }
215 
216  if ( isset( $conf['host'] ) && isset( $conf['port'] ) ) {
217  // Normalize single entries
218  $conf = [ $conf ];
219  }
220  foreach ( $conf as $subconf ) {
221  if ( !isset( $subconf['host'] ) || !isset( $subconf['port'] ) ) {
222  throw new MWException( "Invalid HTCP rule for URL $url\n" );
223  }
224  }
225 
226  // Construct a minimal HTCP request diagram
227  // as per RFC 2756
228  // Opcode 'CLR', no response desired, no auth
229  $htcpTransID = current( $ids );
230  next( $ids );
231 
232  $htcpSpecifier = pack( 'na4na*na8n',
233  4, 'HEAD', strlen( $url ), $url,
234  8, 'HTTP/1.0', 0 );
235 
236  $htcpDataLen = 8 + 2 + strlen( $htcpSpecifier );
237  $htcpLen = 4 + $htcpDataLen + 2;
238 
239  // Note! Squid gets the bit order of the first
240  // word wrong, wrt the RFC. Apparently no other
241  // implementation exists, so adapt to Squid
242  $htcpPacket = pack( 'nxxnCxNxxa*n',
243  $htcpLen, $htcpDataLen, $htcpOpCLR,
244  $htcpTransID, $htcpSpecifier, 2 );
245 
246  wfDebugLog( 'squid', __METHOD__ .
247  "Purging URL $url via HTCP" );
248  foreach ( $conf as $subconf ) {
249  socket_sendto( $conn, $htcpPacket, $htcpLen, 0,
250  $subconf['host'], $subconf['port'] );
251  }
252  }
253  }
254 
269  public static function expand( $url ) {
270  return wfExpandUrl( $url, PROTO_INTERNAL );
271  }
272 
279  private static function getRuleForURL( $url, $rules ) {
280  foreach ( $rules as $regex => $routing ) {
281  if ( $regex === '' || preg_match( $regex, $url ) ) {
282  return $routing;
283  }
284  }
285 
286  return false;
287  }
288 }
289 
294  // Keep class name for b/c
295 }
use
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
Definition: APACHE-LICENSE-2.0.txt:10
CdnCacheUpdate\purge
static purge(array $urlArr)
Purges a list of CDN nodes defined in $wgSquidServers.
Definition: CdnCacheUpdate.php:100
PROTO_INTERNAL
const PROTO_INTERNAL
Definition: Defines.php:234
LinkBatch
Class representing a list of titles The execute() method checks them all for existence and adds them ...
Definition: LinkBatch.php:34
array
the array() calling protocol came about after MediaWiki 1.4rc1.
$wgHTCPMulticastTTL
$wgHTCPMulticastTTL
HTCP multicast TTL.
Definition: DefaultSettings.php:2857
MergeableUpdate
Interface that deferrable updates can implement.
Definition: MergeableUpdate.php:9
CdnCacheUpdate\$urls
string[] $urls
Collection of URLs to purge.
Definition: CdnCacheUpdate.php:32
CdnCacheUpdate\doUpdate
doUpdate()
Purges the list of URLs passed to the constructor.
Definition: CdnCacheUpdate.php:77
SquidPurgeClient
An HTTP 1.0 client built for the purposes of purging Squid and Varnish.
Definition: SquidPurgeClient.php:30
CdnCacheUpdate\HTCPPurge
static HTCPPurge(array $urlArr)
Send Hyper Text Caching Protocol (HTCP) CLR requests.
Definition: CdnCacheUpdate.php:168
wfDebugLog
wfDebugLog( $logGroup, $text, $dest='all', array $context=[])
Send a line to a supplementary debug log file, if configured, or main debug log if not.
Definition: GlobalFunctions.php:1087
CdnCacheUpdate\getRuleForURL
static getRuleForURL( $url, $rules)
Find the HTCP routing rule to use for a given URL.
Definition: CdnCacheUpdate.php:279
php
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:37
SquidPurgeClientPool
Definition: SquidPurgeClientPool.php:22
CdnCacheUpdate\newSimplePurge
static newSimplePurge(Title $title)
Definition: CdnCacheUpdate.php:70
NS_SPECIAL
const NS_SPECIAL
Definition: Defines.php:63
CdnCacheUpdate\merge
merge(MergeableUpdate $update)
Merge this update with $update.
Definition: CdnCacheUpdate.php:41
MWException
MediaWiki exception.
Definition: MWException.php:26
$titles
linkcache txt The LinkCache class maintains a list of article titles and the information about whether or not the article exists in the database This is used to mark up links when displaying a page If the same link appears more than once on any page then it only has to be looked up once In most cases link lookups are done in batches with the LinkBatch class or the equivalent in so the link cache is mostly useful for short snippets of parsed and for links in the navigation areas of the skin The link cache was formerly used to track links used in a document for the purposes of updating the link tables This application is now deprecated To create a you can use the following $titles
Definition: linkcache.txt:17
CdnCacheUpdate\expand
static expand( $url)
Expand local URLs to fully-qualified URLs using the internal protocol and host defined in $wgInternal...
Definition: CdnCacheUpdate.php:269
$wgHTCPRouting
$wgHTCPRouting
Routing configuration for HTCP multicast purging.
Definition: DefaultSettings.php:2851
$title
namespace and then decline to actually register it file or subcat img or subcat $title
Definition: hooks.txt:964
Title\makeTitle
static makeTitle( $ns, $title, $fragment='', $interwiki='')
Create a new Title from a namespace index and a DB key.
Definition: Title.php:534
global
when a variable name is used in a it is silently declared as a new masking the global
Definition: design.txt:95
SquidUpdate
Definition: CdnCacheUpdate.php:293
CdnCacheUpdate\__construct
__construct(array $urlArr)
Definition: CdnCacheUpdate.php:37
CdnCacheUpdate\newFromTitles
static newFromTitles( $titles, $urlArr=[])
Create an update object from an array of Title objects, or a TitleArray object.
Definition: CdnCacheUpdate.php:55
execute
$batch execute()
CdnCacheUpdate
Handles purging appropriate CDN URLs given a title (or titles)
Definition: CdnCacheUpdate.php:30
CdnPurgeJob
Job to purge a set of URLs from CDN.
Definition: CdnPurgeJob.php:30
$wgSquidServers
$wgSquidServers
List of proxy servers to purge on changes; default port is 80.
Definition: DefaultSettings.php:2769
Title
Represents a title within MediaWiki.
Definition: Title.php:39
JobQueueGroup\singleton
static singleton( $domain=false)
Definition: JobQueueGroup.php:72
as
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:22
UIDGenerator\QUICK_VOLATILE
const QUICK_VOLATILE
Definition: UIDGenerator.php:46
DeferrableUpdate
Interface that deferrable updates should implement.
Definition: DeferrableUpdate.php:9
UIDGenerator\newSequentialPerNodeIDs
static newSequentialPerNodeIDs( $bucket, $bits, $count, $flags=0)
Return IDs that are sequential only for this node and bucket.
Definition: UIDGenerator.php:342
MediaWikiServices
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency MediaWikiServices
Definition: injection.txt:25
$wgCdnReboundPurgeDelay
$wgCdnReboundPurgeDelay
If set, any SquidPurge call on a URL or URLs will send a second purge no less than this many seconds ...
Definition: DefaultSettings.php:2745
wfExpandUrl
wfExpandUrl( $url, $defaultProto=PROTO_CURRENT)
Expand a potentially local URL to a fully-qualified URL.
Definition: GlobalFunctions.php:521