Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
82.57% |
308 / 373 |
|
41.18% |
7 / 17 |
CRAP | |
0.00% |
0 / 1 |
| MultiHttpClient | |
82.80% |
308 / 372 |
|
41.18% |
7 / 17 |
159.08 | |
0.00% |
0 / 1 |
| __construct | |
76.92% |
10 / 13 |
|
0.00% |
0 / 1 |
5.31 | |||
| run | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| runMulti | |
100.00% |
16 / 16 |
|
100.00% |
1 / 1 |
6 | |||
| isCurlEnabled | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
2 | |||
| runMultiCurl | |
78.21% |
61 / 78 |
|
0.00% |
0 / 1 |
12.25 | |||
| getCurlHandle | |
64.77% |
57 / 88 |
|
0.00% |
0 / 1 |
43.16 | |||
| getCurlMulti | |
87.50% |
14 / 16 |
|
0.00% |
0 / 1 |
6.07 | |||
| getCurlTime | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| runMultiHttp | |
98.00% |
49 / 50 |
|
0.00% |
0 / 1 |
10 | |||
| normalizeHeaders | |
100.00% |
4 / 4 |
|
100.00% |
1 / 1 |
2 | |||
| normalizeRequests | |
93.02% |
40 / 43 |
|
0.00% |
0 / 1 |
12.05 | |||
| useReverseProxy | |
78.57% |
11 / 14 |
|
0.00% |
0 / 1 |
4.16 | |||
| assembleUrl | |
100.00% |
17 / 17 |
|
100.00% |
1 / 1 |
10 | |||
| isLocalURL | |
94.12% |
16 / 17 |
|
0.00% |
0 / 1 |
6.01 | |||
| getSelectTimeout | |
77.78% |
7 / 9 |
|
0.00% |
0 / 1 |
3.10 | |||
| setLogger | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| __destruct | |
100.00% |
3 / 3 |
|
100.00% |
1 / 1 |
2 | |||
| 1 | <?php |
| 2 | /** |
| 3 | * HTTP service client |
| 4 | * |
| 5 | * @license GPL-2.0-or-later |
| 6 | * @file |
| 7 | */ |
| 8 | |
| 9 | namespace Wikimedia\Http; |
| 10 | |
| 11 | use CurlHandle; |
| 12 | use CurlMultiHandle; |
| 13 | use InvalidArgumentException; |
| 14 | use MediaWiki\MediaWikiServices; |
| 15 | use Psr\Log\LoggerAwareInterface; |
| 16 | use Psr\Log\LoggerInterface; |
| 17 | use Psr\Log\NullLogger; |
| 18 | use RuntimeException; |
| 19 | |
| 20 | /** |
| 21 | * Class to handle multiple HTTP requests |
| 22 | * |
| 23 | * If curl is available, requests will be made concurrently. |
| 24 | * Otherwise, they will be made serially. |
| 25 | * |
| 26 | * HTTP request maps are arrays that use the following format: |
| 27 | * - method : GET/HEAD/PUT/POST/DELETE |
| 28 | * - url : HTTP/HTTPS URL |
| 29 | * - query : <query parameter field/value associative array> (uses RFC 3986) |
| 30 | * - headers : <header name/value associative array> |
| 31 | * - body : source to get the HTTP request body from; |
| 32 | * this can simply be a string (always), a resource for |
| 33 | * PUT requests, and a field/value array for POST request; |
| 34 | * array bodies are encoded as multipart/form-data and strings |
| 35 | * use application/x-www-form-urlencoded (headers sent automatically) |
| 36 | * - stream : resource to stream the HTTP response body to |
| 37 | * - proxy : HTTP proxy to use |
| 38 | * - flags : map of boolean flags which supports: |
| 39 | * - relayResponseHeaders : write out header via header() |
| 40 | * Request maps can use integer index 0 instead of 'method' and 1 instead of 'url'. |
| 41 | * |
| 42 | * Since 1.35, callers should use HttpRequestFactory::createMultiClient() to get |
| 43 | * a client object with appropriately configured timeouts. |
| 44 | * |
| 45 | * @since 1.23 |
| 46 | */ |
| 47 | class MultiHttpClient implements LoggerAwareInterface { |
| 48 | /** Regex for headers likely to contain tokens, etc. that we want to redact from logs */ |
| 49 | private const SENSITIVE_HEADERS = '/(^|-|_)(authorization|auth|password|cookie)($|-|_)/'; |
| 50 | /** |
| 51 | * curl_multi_init() handle, initialized in getCurlMulti() |
| 52 | */ |
| 53 | protected ?CurlMultiHandle $cmh = null; |
| 54 | /** SSL certificates path */ |
| 55 | protected ?string $caBundlePath = null; |
| 56 | protected float $connTimeout = 10; |
| 57 | protected float $maxConnTimeout = INF; |
| 58 | protected float $reqTimeout = 30; |
| 59 | protected float $maxReqTimeout = INF; |
| 60 | protected bool $usePipelining = false; |
| 61 | protected int $maxConnsPerHost = 50; |
| 62 | protected ?string $proxy = null; |
| 63 | protected string|false $localProxy = false; |
| 64 | /** @var string[] */ |
| 65 | protected array $localVirtualHosts = []; |
| 66 | protected string $userAgent = 'wikimedia/multi-http-client v1.1'; |
| 67 | protected LoggerInterface $logger; |
| 68 | protected array $headers = []; |
| 69 | |
| 70 | private ?TelemetryHeadersInterface $telemetry = null; |
| 71 | |
| 72 | /** |
| 73 | * Since MW 1.35, callers should use HttpRequestFactory::createMultiClient() to get |
| 74 | * a client object with appropriately configured timeouts instead of constructing |
| 75 | * a MultiHttpClient directly. |
| 76 | * |
| 77 | * @param array $options |
| 78 | * - connTimeout : default connection timeout (seconds) |
| 79 | * - reqTimeout : default request timeout (seconds) |
| 80 | * - maxConnTimeout : maximum connection timeout (seconds) |
| 81 | * - maxReqTimeout : maximum request timeout (seconds) |
| 82 | * - proxy : HTTP proxy to use |
| 83 | * - localProxy : Reverse proxy to use for domains in localVirtualHosts |
| 84 | * - localVirtualHosts : Domains that are configured as virtual hosts on the same machine |
| 85 | * - usePipelining : whether to use HTTP pipelining if possible (for all hosts) |
| 86 | * - maxConnsPerHost : maximum number of concurrent connections (per host) |
| 87 | * - userAgent : The User-Agent header value to send |
| 88 | * - logger : a {@see LoggerInterface} instance for debug logging |
| 89 | * - caBundlePath : path to specific Certificate Authority bundle (if any) |
| 90 | * - headers : an array of default headers to send with every request |
| 91 | * - telemetry : a {@link \Wikimedia\Http\RequestTelemetry] instance to track telemetry data |
| 92 | */ |
| 93 | public function __construct( array $options ) { |
| 94 | if ( isset( $options['caBundlePath'] ) ) { |
| 95 | $this->caBundlePath = $options['caBundlePath']; |
| 96 | if ( !file_exists( $this->caBundlePath ) ) { |
| 97 | throw new InvalidArgumentException( "Cannot find CA bundle: " . $this->caBundlePath ); |
| 98 | } |
| 99 | } |
| 100 | static $opts = [ |
| 101 | 'connTimeout', 'maxConnTimeout', 'reqTimeout', 'maxReqTimeout', |
| 102 | 'usePipelining', 'maxConnsPerHost', 'proxy', 'userAgent', 'logger', |
| 103 | 'localProxy', 'localVirtualHosts', 'headers', 'telemetry' |
| 104 | ]; |
| 105 | foreach ( $opts as $key ) { |
| 106 | if ( isset( $options[$key] ) ) { |
| 107 | $this->$key = $options[$key]; |
| 108 | } |
| 109 | } |
| 110 | $this->logger ??= new NullLogger; |
| 111 | } |
| 112 | |
| 113 | /** |
| 114 | * Execute an HTTP(S) request |
| 115 | * |
| 116 | * This method returns a response map of: |
| 117 | * - code : HTTP response code or 0 if there was a serious error |
| 118 | * - reason : HTTP response reason (empty if there was a serious error) |
| 119 | * - headers : <header name/value associative array> |
| 120 | * - body : HTTP response body or resource (if "stream" was set) |
| 121 | * - error : Any error string |
| 122 | * The map also stores integer-indexed copies of these values. This lets callers do: |
| 123 | * @code |
| 124 | * [ $rcode, $rdesc, $rhdrs, $rbody, $rerr ] = $http->run( $req ); |
| 125 | * @endcode |
| 126 | * @param array $req HTTP request array |
| 127 | * @param array $opts |
| 128 | * - connTimeout : connection timeout per request (seconds) |
| 129 | * - reqTimeout : post-connection timeout per request (seconds) |
| 130 | * - usePipelining : whether to use HTTP pipelining if possible (for all hosts) |
| 131 | * - maxConnsPerHost : maximum number of concurrent connections (per host) |
| 132 | * - httpVersion : One of 'v1.0', 'v1.1', 'v2', 'v2.0', 'v3' or 'v3.0'. Leave empty to use |
| 133 | * PHP/curl's default |
| 134 | * @param string $caller The method making this request, for attribution in logs |
| 135 | * @return array Response array for request |
| 136 | */ |
| 137 | public function run( array $req, array $opts = [], string $caller = __METHOD__ ): array { |
| 138 | return $this->runMulti( [ $req ], $opts, $caller )[0]['response']; |
| 139 | } |
| 140 | |
| 141 | /** |
| 142 | * Execute a set of HTTP(S) requests. |
| 143 | * |
| 144 | * If curl is available, requests will be made concurrently. |
| 145 | * Otherwise, they will be made serially. |
| 146 | * |
| 147 | * The maps are returned by this method with the 'response' field set to a map of: |
| 148 | * - code : HTTP response code or 0 if there was a serious error |
| 149 | * - reason : HTTP response reason (empty if there was a serious error) |
| 150 | * - headers : <header name/value associative array> |
| 151 | * - body : HTTP response body or resource (if "stream" was set) |
| 152 | * - error : Any error string |
| 153 | * The map also stores integer-indexed copies of these values. This lets callers do: |
| 154 | * @code |
| 155 | * [ $rcode, $rdesc, $rhdrs, $rbody, $rerr ] = $req['response']; |
| 156 | * @endcode |
| 157 | * All headers in the 'headers' field are normalized to use lower case names. |
| 158 | * This is true for the request headers and the response headers. Integer-indexed |
| 159 | * method/URL entries will also be changed to use the corresponding string keys. |
| 160 | * |
| 161 | * @param array[] $reqs Map of HTTP request arrays |
| 162 | * @param array $opts Options |
| 163 | * - connTimeout : connection timeout per request (seconds) |
| 164 | * - reqTimeout : post-connection timeout per request (seconds) |
| 165 | * - usePipelining : whether to use HTTP pipelining if possible (for all hosts) |
| 166 | * - maxConnsPerHost : maximum number of concurrent connections (per host) |
| 167 | * - httpVersion : One of 'v1.0', 'v1.1', 'v2', 'v2.0', 'v3' or 'v3.0'. Leave empty to use |
| 168 | * PHP/curl's default |
| 169 | * @param string $caller The method making these requests, for attribution in logs |
| 170 | * @return array[] $reqs With response array populated for each |
| 171 | */ |
| 172 | public function runMulti( array $reqs, array $opts = [], string $caller = __METHOD__ ): array { |
| 173 | $this->normalizeRequests( $reqs ); |
| 174 | $opts += [ 'connTimeout' => $this->connTimeout, 'reqTimeout' => $this->reqTimeout ]; |
| 175 | |
| 176 | if ( $this->maxConnTimeout && $opts['connTimeout'] > $this->maxConnTimeout ) { |
| 177 | $opts['connTimeout'] = $this->maxConnTimeout; |
| 178 | } |
| 179 | if ( $this->maxReqTimeout && $opts['reqTimeout'] > $this->maxReqTimeout ) { |
| 180 | $opts['reqTimeout'] = $this->maxReqTimeout; |
| 181 | } |
| 182 | |
| 183 | if ( $this->isCurlEnabled() ) { |
| 184 | $opts['httpVersion'] = match ( $opts['httpVersion'] ?? null ) { |
| 185 | 'v1.0' => CURL_HTTP_VERSION_1_0, |
| 186 | 'v1.1' => CURL_HTTP_VERSION_1_1, |
| 187 | 'v2', 'v2.0' => CURL_HTTP_VERSION_2_0, |
| 188 | 'v3', 'v3.0' => CURL_HTTP_VERSION_3, |
| 189 | default => CURL_HTTP_VERSION_NONE, |
| 190 | }; |
| 191 | return $this->runMultiCurl( $reqs, $opts, $caller ); |
| 192 | } else { |
| 193 | # TODO: Add handling for httpVersion option |
| 194 | return $this->runMultiHttp( $reqs, $opts ); |
| 195 | } |
| 196 | } |
| 197 | |
| 198 | /** |
| 199 | * Determines if the curl extension is available |
| 200 | * |
| 201 | * @return bool true if curl is available, false otherwise. |
| 202 | */ |
| 203 | protected function isCurlEnabled(): bool { |
| 204 | // Explicitly test if curl_multi* is blocked, as some users' hosts provide |
| 205 | // them with a modified curl with the multi-threaded parts removed(!) |
| 206 | return extension_loaded( 'curl' ) && function_exists( 'curl_multi_init' ); |
| 207 | } |
| 208 | |
| 209 | /** |
| 210 | * Execute a set of HTTP(S) requests concurrently |
| 211 | * |
| 212 | * @see MultiHttpClient::runMulti() |
| 213 | * |
| 214 | * @param array[] $reqs Map of HTTP request arrays |
| 215 | * @param array $opts |
| 216 | * - connTimeout : connection timeout per request (seconds) |
| 217 | * - reqTimeout : post-connection timeout per request (seconds) |
| 218 | * - usePipelining : whether to use HTTP pipelining if possible |
| 219 | * - maxConnsPerHost : maximum number of concurrent connections (per host) |
| 220 | * - httpVersion: : HTTP version to use |
| 221 | * @phan-param array{connTimeout?:int,reqTimeout?:int,usePipelining?:bool,maxConnsPerHost?:int} $opts |
| 222 | * @param string $caller The method making these requests, for attribution in logs |
| 223 | * @return array $reqs With response array populated for each |
| 224 | * @suppress PhanTypeInvalidDimOffset |
| 225 | */ |
| 226 | private function runMultiCurl( array $reqs, array $opts, string $caller = __METHOD__ ): array { |
| 227 | $chm = $this->getCurlMulti( $opts ); |
| 228 | |
| 229 | $selectTimeout = $this->getSelectTimeout( $opts ); |
| 230 | |
| 231 | // Add all of the required cURL handles... |
| 232 | $handles = []; |
| 233 | foreach ( $reqs as $index => &$req ) { |
| 234 | $handles[$index] = $this->getCurlHandle( $req, $opts ); |
| 235 | curl_multi_add_handle( $chm, $handles[$index] ); |
| 236 | } |
| 237 | // don't assign over this by accident |
| 238 | unset( $req ); |
| 239 | |
| 240 | $infos = []; |
| 241 | // Execute the cURL handles concurrently... |
| 242 | // handles still being processed |
| 243 | $active = null; |
| 244 | do { |
| 245 | // Do any available work... |
| 246 | $mrc = curl_multi_exec( $chm, $active ); |
| 247 | |
| 248 | if ( $mrc !== CURLM_OK ) { |
| 249 | $error = curl_multi_strerror( $mrc ); |
| 250 | $this->logger->error( 'curl_multi_exec() failed: {error}', [ |
| 251 | 'error' => $error, |
| 252 | 'exception' => new RuntimeException(), |
| 253 | 'method' => $caller, |
| 254 | ] ); |
| 255 | break; |
| 256 | } |
| 257 | |
| 258 | // Wait (if possible) for available work... |
| 259 | if ( $active > 0 && curl_multi_select( $chm, $selectTimeout ) === -1 ) { |
| 260 | $errno = curl_multi_errno( $chm ); |
| 261 | $error = curl_multi_strerror( $errno ); |
| 262 | $this->logger->error( 'curl_multi_select() failed: {error}', [ |
| 263 | 'error' => $error, |
| 264 | 'exception' => new RuntimeException(), |
| 265 | 'method' => $caller, |
| 266 | ] ); |
| 267 | } |
| 268 | } while ( $active > 0 ); |
| 269 | |
| 270 | $queuedMessages = null; |
| 271 | do { |
| 272 | $info = curl_multi_info_read( $chm, $queuedMessages ); |
| 273 | if ( $info !== false && $info['msg'] === CURLMSG_DONE ) { |
| 274 | // Note: cast to integer even works on PHP 8.0+ despite the |
| 275 | // handle being an object not a resource, because CurlHandle |
| 276 | // has a backwards-compatible cast_object handler. |
| 277 | $infos[(int)$info['handle']] = $info; |
| 278 | } |
| 279 | } while ( $queuedMessages > 0 ); |
| 280 | |
| 281 | // Remove all the cURL handles and check for errors... |
| 282 | foreach ( $reqs as $index => &$req ) { |
| 283 | $ch = $handles[$index]; |
| 284 | curl_multi_remove_handle( $chm, $ch ); |
| 285 | |
| 286 | if ( isset( $infos[(int)$ch] ) ) { |
| 287 | $info = $infos[(int)$ch]; |
| 288 | $errno = $info['result']; |
| 289 | if ( $errno !== 0 ) { |
| 290 | $req['response']['error'] = "(curl error: $errno) " . |
| 291 | curl_strerror( $errno ) . " " . curl_error( $ch ); |
| 292 | $this->logger->error( 'Error fetching URL "{url}": {error}', [ |
| 293 | 'url' => $req['url'], |
| 294 | 'error' => $req['response']['error'], |
| 295 | 'exception' => new RuntimeException(), |
| 296 | 'method' => $caller, |
| 297 | ] ); |
| 298 | } else { |
| 299 | $this->logger->debug( |
| 300 | "HTTP complete: {method} {url} code={response_code} size={size} " . |
| 301 | "total={total_time} connect={connect_time}", |
| 302 | [ |
| 303 | 'method' => $req['method'], |
| 304 | 'url' => $req['url'], |
| 305 | 'response_code' => $req['response']['code'], |
| 306 | 'size' => curl_getinfo( $ch, CURLINFO_SIZE_DOWNLOAD ), |
| 307 | 'total_time' => $this->getCurlTime( |
| 308 | $ch, |
| 309 | CURLINFO_TOTAL_TIME_T |
| 310 | ), |
| 311 | 'connect_time' => $this->getCurlTime( |
| 312 | $ch, |
| 313 | CURLINFO_CONNECT_TIME_T |
| 314 | ), |
| 315 | ] |
| 316 | ); |
| 317 | } |
| 318 | } else { |
| 319 | $req['response']['error'] = "(curl error: no status set)"; |
| 320 | } |
| 321 | |
| 322 | // For convenience with array destructuring |
| 323 | $req['response'] += [ |
| 324 | 0 => $req['response']['code'], |
| 325 | 1 => $req['response']['reason'], |
| 326 | 2 => $req['response']['headers'], |
| 327 | 3 => $req['response']['body'], |
| 328 | 4 => $req['response']['error'], |
| 329 | ]; |
| 330 | // Close any string wrapper file handles |
| 331 | if ( isset( $req['_closeHandle'] ) ) { |
| 332 | fclose( $req['_closeHandle'] ); |
| 333 | unset( $req['_closeHandle'] ); |
| 334 | } |
| 335 | } |
| 336 | // don't assign over this by accident |
| 337 | unset( $req ); |
| 338 | |
| 339 | return $reqs; |
| 340 | } |
| 341 | |
| 342 | /** |
| 343 | * @param array &$req HTTP request map |
| 344 | * @phpcs:ignore Generic.Files.LineLength |
| 345 | * @phan-param array{url:string,proxy?:?string,query:mixed,method:string,body:string|CurlHandle,headers:array<string,string>,stream?:resource,flags:array} $req |
| 346 | * @param array $opts |
| 347 | * - connTimeout : default connection timeout |
| 348 | * - reqTimeout : default request timeout |
| 349 | * - httpVersion: default HTTP version |
| 350 | */ |
| 351 | protected function getCurlHandle( array &$req, array $opts ): CurlHandle { |
| 352 | $ch = curl_init(); |
| 353 | |
| 354 | curl_setopt( $ch, CURLOPT_PROXY, $req['proxy'] ?? $this->proxy ); |
| 355 | curl_setopt( $ch, CURLOPT_CONNECTTIMEOUT_MS, intval( $opts['connTimeout'] * 1e3 ) ); |
| 356 | curl_setopt( $ch, CURLOPT_TIMEOUT_MS, intval( $opts['reqTimeout'] * 1e3 ) ); |
| 357 | curl_setopt( $ch, CURLOPT_FOLLOWLOCATION, 1 ); |
| 358 | curl_setopt( $ch, CURLOPT_MAXREDIRS, 4 ); |
| 359 | curl_setopt( $ch, CURLOPT_HEADER, 0 ); |
| 360 | if ( $this->caBundlePath !== null ) { |
| 361 | curl_setopt( $ch, CURLOPT_SSL_VERIFYPEER, true ); |
| 362 | curl_setopt( $ch, CURLOPT_CAINFO, $this->caBundlePath ); |
| 363 | } |
| 364 | curl_setopt( $ch, CURLOPT_RETURNTRANSFER, 1 ); |
| 365 | |
| 366 | $url = $req['url']; |
| 367 | $query = http_build_query( $req['query'], '', '&', PHP_QUERY_RFC3986 ); |
| 368 | if ( $query != '' ) { |
| 369 | $url .= !str_contains( $req['url'], '?' ) ? "?$query" : "&$query"; |
| 370 | } |
| 371 | curl_setopt( $ch, CURLOPT_URL, $url ); |
| 372 | curl_setopt( $ch, CURLOPT_CUSTOMREQUEST, $req['method'] ); |
| 373 | curl_setopt( $ch, CURLOPT_NOBODY, ( $req['method'] === 'HEAD' ) ); |
| 374 | curl_setopt( $ch, CURLOPT_HTTP_VERSION, $opts['httpVersion'] ?? CURL_HTTP_VERSION_NONE ); |
| 375 | |
| 376 | if ( $req['method'] === 'PUT' ) { |
| 377 | curl_setopt( $ch, CURLOPT_PUT, 1 ); |
| 378 | // phpcs:ignore MediaWiki.Usage.ForbiddenFunctions.is_resource |
| 379 | if ( is_resource( $req['body'] ) ) { |
| 380 | curl_setopt( $ch, CURLOPT_INFILE, $req['body'] ); |
| 381 | if ( isset( $req['headers']['content-length'] ) ) { |
| 382 | curl_setopt( $ch, CURLOPT_INFILESIZE, $req['headers']['content-length'] ); |
| 383 | } elseif ( isset( $req['headers']['transfer-encoding'] ) && |
| 384 | $req['headers']['transfer-encoding'] === 'chunks' |
| 385 | ) { |
| 386 | curl_setopt( $ch, CURLOPT_UPLOAD, true ); |
| 387 | } else { |
| 388 | throw new InvalidArgumentException( "Missing 'Content-Length' or 'Transfer-Encoding' header." ); |
| 389 | } |
| 390 | } elseif ( $req['body'] !== '' ) { |
| 391 | $fp = fopen( "php://temp", "wb+" ); |
| 392 | fwrite( $fp, $req['body'], strlen( $req['body'] ) ); |
| 393 | rewind( $fp ); |
| 394 | curl_setopt( $ch, CURLOPT_INFILE, $fp ); |
| 395 | curl_setopt( $ch, CURLOPT_INFILESIZE, strlen( $req['body'] ) ); |
| 396 | // remember to close this later |
| 397 | $req['_closeHandle'] = $fp; |
| 398 | } else { |
| 399 | curl_setopt( $ch, CURLOPT_INFILESIZE, 0 ); |
| 400 | } |
| 401 | curl_setopt( $ch, CURLOPT_READFUNCTION, |
| 402 | static function ( $ch, $fd, $length ) { |
| 403 | return (string)fread( $fd, $length ); |
| 404 | } |
| 405 | ); |
| 406 | } elseif ( $req['method'] === 'POST' ) { |
| 407 | curl_setopt( $ch, CURLOPT_POST, 1 ); |
| 408 | curl_setopt( $ch, CURLOPT_POSTFIELDS, $req['body'] ); |
| 409 | } else { |
| 410 | // phpcs:ignore MediaWiki.Usage.ForbiddenFunctions.is_resource |
| 411 | if ( is_resource( $req['body'] ) || $req['body'] !== '' ) { |
| 412 | throw new InvalidArgumentException( "HTTP body specified for a non PUT/POST request." ); |
| 413 | } |
| 414 | $req['headers']['content-length'] = 0; |
| 415 | } |
| 416 | |
| 417 | if ( !isset( $req['headers']['user-agent'] ) ) { |
| 418 | $req['headers']['user-agent'] = $this->userAgent; |
| 419 | } |
| 420 | |
| 421 | $headers = []; |
| 422 | foreach ( $req['headers'] as $name => $value ) { |
| 423 | if ( str_contains( $name, ':' ) ) { |
| 424 | throw new InvalidArgumentException( "Header name must not contain colon-space." ); |
| 425 | } |
| 426 | $headers[] = $name . ': ' . trim( $value ); |
| 427 | } |
| 428 | curl_setopt( $ch, CURLOPT_HTTPHEADER, $headers ); |
| 429 | |
| 430 | curl_setopt( $ch, CURLOPT_HEADERFUNCTION, |
| 431 | static function ( $ch, $header ) use ( &$req ) { |
| 432 | if ( !empty( $req['flags']['relayResponseHeaders'] ) && trim( $header ) !== '' ) { |
| 433 | header( $header ); |
| 434 | } |
| 435 | $length = strlen( $header ); |
| 436 | $matches = []; |
| 437 | if ( preg_match( "/^(HTTP\/(?:1\.[01]|2|3)) (\d{3}) (.*)/", $header, $matches ) ) { |
| 438 | $req['response']['code'] = (int)$matches[2]; |
| 439 | $req['response']['reason'] = trim( $matches[3] ); |
| 440 | // After a redirect we will receive this again, but we already stored headers |
| 441 | // that belonged to a redirect response. Start over. |
| 442 | $req['response']['headers'] = []; |
| 443 | return $length; |
| 444 | } |
| 445 | if ( !str_contains( $header, ":" ) ) { |
| 446 | return $length; |
| 447 | } |
| 448 | [ $name, $value ] = explode( ":", $header, 2 ); |
| 449 | $name = strtolower( $name ); |
| 450 | $value = trim( $value ); |
| 451 | if ( isset( $req['response']['headers'][$name] ) ) { |
| 452 | $req['response']['headers'][$name] .= ', ' . $value; |
| 453 | } else { |
| 454 | $req['response']['headers'][$name] = $value; |
| 455 | } |
| 456 | return $length; |
| 457 | } |
| 458 | ); |
| 459 | |
| 460 | // This works with both file and php://temp handles (unlike CURLOPT_FILE) |
| 461 | $hasOutputStream = isset( $req['stream'] ); |
| 462 | curl_setopt( $ch, CURLOPT_WRITEFUNCTION, |
| 463 | static function ( $ch, $data ) use ( &$req, $hasOutputStream ) { |
| 464 | if ( $hasOutputStream ) { |
| 465 | return (int)fwrite( $req['stream'], $data ); |
| 466 | } else { |
| 467 | $req['response']['body'] .= $data; |
| 468 | |
| 469 | return strlen( $data ); |
| 470 | } |
| 471 | } |
| 472 | ); |
| 473 | |
| 474 | return $ch; |
| 475 | } |
| 476 | |
| 477 | protected function getCurlMulti( array $opts ): CurlMultiHandle { |
| 478 | if ( !$this->cmh ) { |
| 479 | $cmh = curl_multi_init(); |
| 480 | // Limit the size of the idle connection cache such that consecutive parallel |
| 481 | // request batches to the same host can avoid having to keep making connections |
| 482 | curl_multi_setopt( $cmh, CURLMOPT_MAXCONNECTS, $this->maxConnsPerHost ); |
| 483 | $this->cmh = $cmh; |
| 484 | } |
| 485 | |
| 486 | $curlVersion = curl_version()['version']; |
| 487 | |
| 488 | // CURLMOPT_MAX_HOST_CONNECTIONS is available since PHP 7.0.7 and cURL 7.30.0 |
| 489 | if ( version_compare( $curlVersion, '7.30.0', '>=' ) ) { |
| 490 | // Limit the number of in-flight requests for any given host |
| 491 | $maxHostConns = $opts['maxConnsPerHost'] ?? $this->maxConnsPerHost; |
| 492 | curl_multi_setopt( $this->cmh, CURLMOPT_MAX_HOST_CONNECTIONS, (int)$maxHostConns ); |
| 493 | } |
| 494 | |
| 495 | if ( $opts['usePipelining'] ?? $this->usePipelining ) { |
| 496 | if ( version_compare( $curlVersion, '7.43', '<' ) ) { |
| 497 | // The option is a boolean |
| 498 | $pipelining = 1; |
| 499 | } elseif ( version_compare( $curlVersion, '7.62', '<' ) ) { |
| 500 | // The option is a bitfield and HTTP/1.x pipelining is supported |
| 501 | $pipelining = CURLPIPE_HTTP1 | CURLPIPE_MULTIPLEX; |
| 502 | } else { |
| 503 | // The option is a bitfield but HTTP/1.x pipelining has been removed |
| 504 | $pipelining = CURLPIPE_MULTIPLEX; |
| 505 | } |
| 506 | // Suppress deprecation, we know already (T264735) |
| 507 | // phpcs:ignore Generic.PHP.NoSilencedErrors |
| 508 | @curl_multi_setopt( $this->cmh, CURLMOPT_PIPELINING, $pipelining ); |
| 509 | } |
| 510 | |
| 511 | return $this->cmh; |
| 512 | } |
| 513 | |
| 514 | /** |
| 515 | * Get a time in seconds, formatted with microsecond resolution. |
| 516 | */ |
| 517 | private function getCurlTime( CurlHandle $ch, int $option ): string { |
| 518 | return sprintf( "%.6F", curl_getinfo( $ch, $option ) / 1e6 ); |
| 519 | } |
| 520 | |
| 521 | /** |
| 522 | * Execute a set of HTTP(S) requests sequentially. |
| 523 | * |
| 524 | * @see MultiHttpClient::runMulti() |
| 525 | * @todo Remove dependency on MediaWikiServices: rewrite using Guzzle T202352 |
| 526 | * @param array $reqs Map of HTTP request arrays |
| 527 | * @phpcs:ignore Generic.Files.LineLength |
| 528 | * @phan-param array<int,array{url:string,query:array,method:string,body:string,headers:array<string,string>,proxy?:?string}> $reqs |
| 529 | * @param array $opts |
| 530 | * - connTimeout : connection timeout per request (seconds) |
| 531 | * - reqTimeout : post-connection timeout per request (seconds) |
| 532 | * @phan-param array{connTimeout:int,reqTimeout:int} $opts |
| 533 | * @return array $reqs With response array populated for each |
| 534 | */ |
| 535 | private function runMultiHttp( array $reqs, array $opts = [] ): array { |
| 536 | $httpOptions = [ |
| 537 | 'timeout' => $opts['reqTimeout'] ?? $this->reqTimeout, |
| 538 | 'connectTimeout' => $opts['connTimeout'] ?? $this->connTimeout, |
| 539 | 'logger' => $this->logger, |
| 540 | 'caInfo' => $this->caBundlePath, |
| 541 | ]; |
| 542 | foreach ( $reqs as &$req ) { |
| 543 | $reqOptions = $httpOptions + [ |
| 544 | 'method' => $req['method'], |
| 545 | 'proxy' => $req['proxy'] ?? $this->proxy, |
| 546 | 'userAgent' => $req['headers']['user-agent'] ?? $this->userAgent, |
| 547 | 'postData' => $req['body'], |
| 548 | ]; |
| 549 | |
| 550 | $url = $req['url']; |
| 551 | $query = http_build_query( $req['query'], '', '&', PHP_QUERY_RFC3986 ); |
| 552 | if ( $query != '' ) { |
| 553 | $url .= !str_contains( $req['url'], '?' ) ? "?$query" : "&$query"; |
| 554 | } |
| 555 | |
| 556 | $httpRequest = MediaWikiServices::getInstance()->getHttpRequestFactory()->create( |
| 557 | $url, $reqOptions, __METHOD__ ); |
| 558 | $httpRequest->setLogger( $this->logger ); |
| 559 | foreach ( $req['headers'] as $header => $value ) { |
| 560 | $httpRequest->setHeader( $header, $value ); |
| 561 | } |
| 562 | $sv = $httpRequest->execute()->getStatusValue(); |
| 563 | |
| 564 | $respHeaders = array_map( |
| 565 | static fn ( $v ) => implode( ', ', $v ), |
| 566 | $httpRequest->getResponseHeaders() ); |
| 567 | |
| 568 | $req['response'] = [ |
| 569 | 'code' => $httpRequest->getStatus(), |
| 570 | 'reason' => '', |
| 571 | 'headers' => $respHeaders, |
| 572 | 'body' => $httpRequest->getContent(), |
| 573 | 'error' => '', |
| 574 | ]; |
| 575 | |
| 576 | if ( !$sv->isOK() ) { |
| 577 | $svErrors = $sv->getErrors(); |
| 578 | if ( isset( $svErrors[0] ) ) { |
| 579 | $req['response']['error'] = $svErrors[0]['message']; |
| 580 | |
| 581 | // param values vary per failure type (ex. unknown host vs unknown page) |
| 582 | if ( isset( $svErrors[0]['params'][0] ) ) { |
| 583 | if ( is_numeric( $svErrors[0]['params'][0] ) ) { |
| 584 | if ( isset( $svErrors[0]['params'][1] ) ) { |
| 585 | // @phan-suppress-next-line PhanTypeInvalidDimOffset |
| 586 | $req['response']['reason'] = $svErrors[0]['params'][1]; |
| 587 | } |
| 588 | } else { |
| 589 | $req['response']['reason'] = $svErrors[0]['params'][0]; |
| 590 | } |
| 591 | } |
| 592 | } |
| 593 | } |
| 594 | |
| 595 | $req['response'] += [ |
| 596 | 0 => $req['response']['code'], |
| 597 | 1 => $req['response']['reason'], |
| 598 | 2 => $req['response']['headers'], |
| 599 | 3 => $req['response']['body'], |
| 600 | 4 => $req['response']['error'], |
| 601 | ]; |
| 602 | } |
| 603 | |
| 604 | return $reqs; |
| 605 | } |
| 606 | |
| 607 | /** |
| 608 | * Normalize the headers array |
| 609 | */ |
| 610 | private function normalizeHeaders( array $headers ): array { |
| 611 | $normalized = []; |
| 612 | foreach ( $headers as $name => $value ) { |
| 613 | $normalized[strtolower( $name )] = $value; |
| 614 | } |
| 615 | return $normalized; |
| 616 | } |
| 617 | |
| 618 | /** |
| 619 | * Normalize request information |
| 620 | * |
| 621 | * @param array[] &$reqs the requests to normalize |
| 622 | */ |
| 623 | private function normalizeRequests( array &$reqs ) { |
| 624 | foreach ( $reqs as &$req ) { |
| 625 | $req['response'] = [ |
| 626 | 'code' => 0, |
| 627 | 'reason' => '', |
| 628 | 'headers' => [], |
| 629 | 'body' => '', |
| 630 | 'error' => '', |
| 631 | ]; |
| 632 | if ( isset( $req[0] ) ) { |
| 633 | // short-form |
| 634 | $req['method'] = $req[0]; |
| 635 | unset( $req[0] ); |
| 636 | } |
| 637 | if ( isset( $req[1] ) ) { |
| 638 | // short-form |
| 639 | $req['url'] = $req[1]; |
| 640 | unset( $req[1] ); |
| 641 | } |
| 642 | if ( !isset( $req['method'] ) ) { |
| 643 | throw new InvalidArgumentException( "Request has no 'method' field set." ); |
| 644 | } elseif ( !isset( $req['url'] ) ) { |
| 645 | throw new InvalidArgumentException( "Request has no 'url' field set." ); |
| 646 | } |
| 647 | if ( $this->localProxy !== false && $this->isLocalURL( $req['url'] ) ) { |
| 648 | $this->useReverseProxy( $req, $this->localProxy ); |
| 649 | } |
| 650 | $req['query'] ??= []; |
| 651 | $req['headers'] = $this->normalizeHeaders( |
| 652 | array_merge( |
| 653 | $this->headers, |
| 654 | $this->telemetry ? $this->telemetry->getRequestHeaders() : [], |
| 655 | $req['headers'] ?? [] |
| 656 | ) |
| 657 | ); |
| 658 | |
| 659 | if ( !isset( $req['body'] ) ) { |
| 660 | $req['body'] = ''; |
| 661 | $req['headers']['content-length'] = 0; |
| 662 | } |
| 663 | // Redact some headers we know to have tokens before logging them |
| 664 | $logHeaders = $req['headers']; |
| 665 | foreach ( $logHeaders as $header => $value ) { |
| 666 | if ( preg_match( self::SENSITIVE_HEADERS, $header ) === 1 ) { |
| 667 | $logHeaders[$header] = '[redacted]'; |
| 668 | } |
| 669 | } |
| 670 | $this->logger->debug( "HTTP start: {method} {url}", |
| 671 | [ |
| 672 | 'method' => $req['method'], |
| 673 | 'url' => $req['url'], |
| 674 | 'headers' => $logHeaders, |
| 675 | ] |
| 676 | ); |
| 677 | $req['flags'] ??= []; |
| 678 | } |
| 679 | } |
| 680 | |
| 681 | private function useReverseProxy( array &$req, string $proxy ): void { |
| 682 | $parsedProxy = parse_url( $proxy ); |
| 683 | if ( $parsedProxy === false ) { |
| 684 | throw new InvalidArgumentException( "Invalid reverseProxy configured: $proxy" ); |
| 685 | } |
| 686 | $parsedUrl = parse_url( $req['url'] ); |
| 687 | if ( $parsedUrl === false ) { |
| 688 | throw new InvalidArgumentException( "Invalid url specified: {$req['url']}" ); |
| 689 | } |
| 690 | // Set the current host in the Host header |
| 691 | // @phan-suppress-next-line PhanTypePossiblyInvalidDimOffset |
| 692 | $req['headers']['Host'] = $parsedUrl['host']; |
| 693 | // Replace scheme, host and port in the request |
| 694 | // @phan-suppress-next-line PhanTypePossiblyInvalidDimOffset |
| 695 | $parsedUrl['scheme'] = $parsedProxy['scheme']; |
| 696 | // @phan-suppress-next-line PhanTypePossiblyInvalidDimOffset |
| 697 | $parsedUrl['host'] = $parsedProxy['host']; |
| 698 | if ( isset( $parsedProxy['port'] ) ) { |
| 699 | $parsedUrl['port'] = $parsedProxy['port']; |
| 700 | } else { |
| 701 | unset( $parsedUrl['port'] ); |
| 702 | } |
| 703 | $req['url'] = self::assembleUrl( $parsedUrl ); |
| 704 | // Explicitly disable use of another proxy by setting to false, |
| 705 | // since null will fall back to $this->proxy |
| 706 | $req['proxy'] = false; |
| 707 | } |
| 708 | |
| 709 | /** |
| 710 | * This is derived from MediaWiki\Utils\UrlUtils::assemble but changed to work |
| 711 | * with parse_url's result so the delimiter is hardcoded. |
| 712 | * |
| 713 | * The basic structure used: |
| 714 | * [scheme://][[user][:pass]@][host][:port][path][?query][#fragment] |
| 715 | * |
| 716 | * @param array $urlParts URL parts, as output from parse_url() |
| 717 | * @return string URL assembled from its component parts |
| 718 | */ |
| 719 | private static function assembleUrl( array $urlParts ): string { |
| 720 | $result = isset( $urlParts['scheme'] ) ? $urlParts['scheme'] . '://' : ''; |
| 721 | |
| 722 | if ( isset( $urlParts['host'] ) ) { |
| 723 | if ( isset( $urlParts['user'] ) ) { |
| 724 | $result .= $urlParts['user']; |
| 725 | if ( isset( $urlParts['pass'] ) ) { |
| 726 | $result .= ':' . $urlParts['pass']; |
| 727 | } |
| 728 | $result .= '@'; |
| 729 | } |
| 730 | |
| 731 | $result .= $urlParts['host']; |
| 732 | |
| 733 | if ( isset( $urlParts['port'] ) ) { |
| 734 | $result .= ':' . $urlParts['port']; |
| 735 | } |
| 736 | } |
| 737 | |
| 738 | if ( isset( $urlParts['path'] ) ) { |
| 739 | $result .= $urlParts['path']; |
| 740 | } |
| 741 | |
| 742 | if ( isset( $urlParts['query'] ) && $urlParts['query'] !== '' ) { |
| 743 | $result .= '?' . $urlParts['query']; |
| 744 | } |
| 745 | |
| 746 | if ( isset( $urlParts['fragment'] ) ) { |
| 747 | $result .= '#' . $urlParts['fragment']; |
| 748 | } |
| 749 | |
| 750 | return $result; |
| 751 | } |
| 752 | |
| 753 | /** |
| 754 | * Check if the URL can be served by localhost |
| 755 | * |
| 756 | * @note this is mostly a copy of MWHttpRequest::isLocalURL() |
| 757 | * @param string $url Full url to check |
| 758 | */ |
| 759 | private function isLocalURL( string $url ): bool { |
| 760 | if ( !$this->localVirtualHosts ) { |
| 761 | // Shortcut |
| 762 | return false; |
| 763 | } |
| 764 | |
| 765 | // Extract host part |
| 766 | $matches = []; |
| 767 | if ( preg_match( '!^https?://([\w.-]+)[/:].*$!', $url, $matches ) ) { |
| 768 | $host = $matches[1]; |
| 769 | // Split up dotwise |
| 770 | $domainParts = explode( '.', $host ); |
| 771 | // Check if this domain or any superdomain is listed as a local virtual host |
| 772 | $domainParts = array_reverse( $domainParts ); |
| 773 | |
| 774 | $domain = ''; |
| 775 | $countParts = count( $domainParts ); |
| 776 | for ( $i = 0; $i < $countParts; $i++ ) { |
| 777 | $domainPart = $domainParts[$i]; |
| 778 | if ( $i == 0 ) { |
| 779 | $domain = $domainPart; |
| 780 | } else { |
| 781 | $domain = $domainPart . '.' . $domain; |
| 782 | } |
| 783 | |
| 784 | if ( in_array( $domain, $this->localVirtualHosts ) ) { |
| 785 | return true; |
| 786 | } |
| 787 | } |
| 788 | } |
| 789 | |
| 790 | return false; |
| 791 | } |
| 792 | |
| 793 | /** |
| 794 | * Get a suitable select timeout for the given options. |
| 795 | */ |
| 796 | private function getSelectTimeout( array $opts ): float { |
| 797 | $connTimeout = $opts['connTimeout'] ?? $this->connTimeout; |
| 798 | $reqTimeout = $opts['reqTimeout'] ?? $this->reqTimeout; |
| 799 | $timeouts = array_filter( [ $connTimeout, $reqTimeout ] ); |
| 800 | if ( count( $timeouts ) === 0 ) { |
| 801 | return 1; |
| 802 | } |
| 803 | |
| 804 | $selectTimeout = min( $timeouts ); |
| 805 | // Minimum 10us |
| 806 | if ( $selectTimeout < 10e-6 ) { |
| 807 | $selectTimeout = 10e-6; |
| 808 | } |
| 809 | return $selectTimeout; |
| 810 | } |
| 811 | |
| 812 | /** |
| 813 | * Register a logger |
| 814 | */ |
| 815 | public function setLogger( LoggerInterface $logger ): void { |
| 816 | $this->logger = $logger; |
| 817 | } |
| 818 | |
| 819 | public function __destruct() { |
| 820 | if ( $this->cmh ) { |
| 821 | curl_multi_close( $this->cmh ); |
| 822 | $this->cmh = null; |
| 823 | } |
| 824 | } |
| 825 | |
| 826 | } |
| 827 | /** @deprecated class alias since 1.43 */ |
| 828 | class_alias( MultiHttpClient::class, 'MultiHttpClient' ); |