MediaWiki  1.28.0
CdnCacheUpdate.php
Go to the documentation of this file.
1 <?php
26 
33  protected $urls = [];
34 
38  public function __construct( array $urlArr ) {
39  $this->urls = $urlArr;
40  }
41 
42  public function merge( MergeableUpdate $update ) {
44  Assert::parameterType( __CLASS__, $update, '$update' );
45 
46  $this->urls = array_merge( $this->urls, $update->urls );
47  }
48 
56  public static function newFromTitles( $titles, $urlArr = [] ) {
58  foreach ( $titles as $title ) {
59  $urlArr = array_merge( $urlArr, $title->getCdnUrls() );
60  }
61 
62  return new CdnCacheUpdate( $urlArr );
63  }
64 
70  public static function newSimplePurge( Title $title ) {
71  return new CdnCacheUpdate( $title->getCdnUrls() );
72  }
73 
77  public function doUpdate() {
79 
80  self::purge( $this->urls );
81 
82  if ( $wgCdnReboundPurgeDelay > 0 ) {
83  JobQueueGroup::singleton()->lazyPush( new CdnPurgeJob(
84  Title::makeTitle( NS_SPECIAL, 'Badtitle/' . __CLASS__ ),
85  [
86  'urls' => $this->urls,
87  'jobReleaseTimestamp' => time() + $wgCdnReboundPurgeDelay
88  ]
89  ) );
90  }
91  }
92 
100  public static function purge( array $urlArr ) {
102 
103  if ( !$urlArr ) {
104  return;
105  }
106 
107  // Remove duplicate URLs from list
108  $urlArr = array_unique( $urlArr );
109 
110  wfDebugLog( 'squid', __METHOD__ . ': ' . implode( ' ', $urlArr ) );
111 
112  // Reliably broadcast the purge to all edge nodes
113  $relayer = MediaWikiServices::getInstance()->getEventRelayerGroup()
114  ->getRelayer( 'cdn-url-purges' );
115  $ts = microtime( true );
116  $relayer->notifyMulti(
117  'cdn-url-purges',
118  array_map(
119  function ( $url ) use ( $ts ) {
120  return [
121  'url' => $url,
122  'timestamp' => $ts,
123  ];
124  },
125  $urlArr
126  )
127  );
128 
129  // Send lossy UDP broadcasting if enabled
130  if ( $wgHTCPRouting ) {
131  self::HTCPPurge( $urlArr );
132  }
133 
134  // Do direct server purges if enabled (this does not scale very well)
135  if ( $wgSquidServers ) {
136  // Maximum number of parallel connections per squid
137  $maxSocketsPerSquid = 8;
138  // Number of requests to send per socket
139  // 400 seems to be a good tradeoff, opening a socket takes a while
140  $urlsPerSocket = 400;
141  $socketsPerSquid = ceil( count( $urlArr ) / $urlsPerSocket );
142  if ( $socketsPerSquid > $maxSocketsPerSquid ) {
143  $socketsPerSquid = $maxSocketsPerSquid;
144  }
145 
146  $pool = new SquidPurgeClientPool;
147  $chunks = array_chunk( $urlArr, ceil( count( $urlArr ) / $socketsPerSquid ) );
148  foreach ( $wgSquidServers as $server ) {
149  foreach ( $chunks as $chunk ) {
150  $client = new SquidPurgeClient( $server );
151  foreach ( $chunk as $url ) {
152  $client->queuePurge( $url );
153  }
154  $pool->addClient( $client );
155  }
156  }
157 
158  $pool->run();
159  }
160  }
161 
168  private static function HTCPPurge( array $urlArr ) {
170 
171  // HTCP CLR operation
172  $htcpOpCLR = 4;
173 
174  // @todo FIXME: PHP doesn't support these socket constants (include/linux/in.h)
175  if ( !defined( "IPPROTO_IP" ) ) {
176  define( "IPPROTO_IP", 0 );
177  define( "IP_MULTICAST_LOOP", 34 );
178  define( "IP_MULTICAST_TTL", 33 );
179  }
180 
181  // pfsockopen doesn't work because we need set_sock_opt
182  $conn = socket_create( AF_INET, SOCK_DGRAM, SOL_UDP );
183  if ( !$conn ) {
184  $errstr = socket_strerror( socket_last_error() );
185  wfDebugLog( 'squid', __METHOD__ .
186  ": Error opening UDP socket: $errstr" );
187 
188  return;
189  }
190 
191  // Set socket options
192  socket_set_option( $conn, IPPROTO_IP, IP_MULTICAST_LOOP, 0 );
193  if ( $wgHTCPMulticastTTL != 1 ) {
194  // Set multicast time to live (hop count) option on socket
195  socket_set_option( $conn, IPPROTO_IP, IP_MULTICAST_TTL,
196  $wgHTCPMulticastTTL );
197  }
198 
199  // Get sequential trx IDs for packet loss counting
201  'squidhtcppurge', 32, count( $urlArr ), UIDGenerator::QUICK_VOLATILE
202  );
203 
204  foreach ( $urlArr as $url ) {
205  if ( !is_string( $url ) ) {
206  throw new MWException( 'Bad purge URL' );
207  }
208  $url = self::expand( $url );
209  $conf = self::getRuleForURL( $url, $wgHTCPRouting );
210  if ( !$conf ) {
211  wfDebugLog( 'squid', __METHOD__ .
212  "No HTCP rule configured for URL {$url} , skipping" );
213  continue;
214  }
215 
216  if ( isset( $conf['host'] ) && isset( $conf['port'] ) ) {
217  // Normalize single entries
218  $conf = [ $conf ];
219  }
220  foreach ( $conf as $subconf ) {
221  if ( !isset( $subconf['host'] ) || !isset( $subconf['port'] ) ) {
222  throw new MWException( "Invalid HTCP rule for URL $url\n" );
223  }
224  }
225 
226  // Construct a minimal HTCP request diagram
227  // as per RFC 2756
228  // Opcode 'CLR', no response desired, no auth
229  $htcpTransID = current( $ids );
230  next( $ids );
231 
232  $htcpSpecifier = pack( 'na4na*na8n',
233  4, 'HEAD', strlen( $url ), $url,
234  8, 'HTTP/1.0', 0 );
235 
236  $htcpDataLen = 8 + 2 + strlen( $htcpSpecifier );
237  $htcpLen = 4 + $htcpDataLen + 2;
238 
239  // Note! Squid gets the bit order of the first
240  // word wrong, wrt the RFC. Apparently no other
241  // implementation exists, so adapt to Squid
242  $htcpPacket = pack( 'nxxnCxNxxa*n',
243  $htcpLen, $htcpDataLen, $htcpOpCLR,
244  $htcpTransID, $htcpSpecifier, 2 );
245 
246  wfDebugLog( 'squid', __METHOD__ .
247  "Purging URL $url via HTCP" );
248  foreach ( $conf as $subconf ) {
249  socket_sendto( $conn, $htcpPacket, $htcpLen, 0,
250  $subconf['host'], $subconf['port'] );
251  }
252  }
253  }
254 
269  public static function expand( $url ) {
270  return wfExpandUrl( $url, PROTO_INTERNAL );
271  }
272 
279  private static function getRuleForURL( $url, $rules ) {
280  foreach ( $rules as $regex => $routing ) {
281  if ( $regex === '' || preg_match( $regex, $url ) ) {
282  return $routing;
283  }
284  }
285 
286  return false;
287  }
288 }
289 
294  // Keep class name for b/c
295 }
const QUICK_VOLATILE
the array() calling protocol came about after MediaWiki 1.4rc1.
static HTCPPurge(array $urlArr)
Send Hyper Text Caching Protocol (HTCP) CLR requests.
$wgSquidServers
List of proxy servers to purge on changes; default port is 80.
static purge(array $urlArr)
Purges a list of CDN nodes defined in $wgSquidServers.
Interface that deferrable updates should implement.
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
static newSequentialPerNodeIDs($bucket, $bits, $count, $flags=0)
Return IDs that are sequential only for this node and bucket.
static newSimplePurge(Title $title)
Handles purging appropriate CDN URLs given a title (or titles)
Interface that deferrable updates can implement.
const NS_SPECIAL
Definition: Defines.php:45
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency MediaWikiServices
Definition: injection.txt:23
when a variable name is used in a it is silently declared as a new local masking the global
Definition: design.txt:93
wfExpandUrl($url, $defaultProto=PROTO_CURRENT)
Expand a potentially local URL to a fully-qualified URL.
static getRuleForURL($url, $rules)
Find the HTCP routing rule to use for a given URL.
wfDebugLog($logGroup, $text, $dest= 'all', array $context=[])
Send a line to a supplementary debug log file, if configured, or main debug log if not...
const PROTO_INTERNAL
Definition: Defines.php:228
string[] $urls
Collection of URLs to purge.
An HTTP 1.0 client built for the purposes of purging Squid and Varnish.
doUpdate()
Purges the list of URLs passed to the constructor.
getCdnUrls()
Get a list of URLs to purge from the CDN cache when this page changes.
Definition: Title.php:3558
merge(MergeableUpdate $update)
Merge this update with $update.
__construct(array $urlArr)
namespace and then decline to actually register it file or subcat img or subcat $title
Definition: hooks.txt:953
static expand($url)
Expand local URLs to fully-qualified URLs using the internal protocol and host defined in $wgInternal...
$wgHTCPMulticastTTL
HTCP multicast TTL.
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
static singleton($wiki=false)
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
linkcache txt The LinkCache class maintains a list of article titles and the information about whether or not the article exists in the database This is used to mark up links when displaying a page If the same link appears more than once on any page then it only has to be looked up once In most cases link lookups are done in batches with the LinkBatch class or the equivalent in so the link cache is mostly useful for short snippets of parsed and for links in the navigation areas of the skin The link cache was formerly used to track links used in a document for the purposes of updating the link tables This application is now deprecated To create a you can use the following $titles
Definition: linkcache.txt:17
$wgHTCPRouting
Routing configuration for HTCP multicast purging.
static makeTitle($ns, $title, $fragment= '', $interwiki= '')
Create a new Title from a namespace index and a DB key.
Definition: Title.php:511
Job to purge a set of URLs from CDN.
Definition: CdnPurgeJob.php:30
static newFromTitles($titles, $urlArr=[])
Create an update object from an array of Title objects, or a TitleArray object.
$wgCdnReboundPurgeDelay
If set, any SquidPurge call on a URL or URLs will send a second purge no less than this many seconds ...