60 private const SENSITIVE_HEADERS =
'/(^|-|_)(authorization|auth|password|cookie)($|-|_)/';
87 protected $userAgent =
'wikimedia/multi-http-client v1.1';
96 private const TIMEOUT_ACCURACY_FACTOR = 0.1;
123 if ( isset( $options[
'caBundlePath'] ) ) {
124 $this->caBundlePath = $options[
'caBundlePath'];
125 if ( !file_exists( $this->caBundlePath ) ) {
126 throw new InvalidArgumentException(
"Cannot find CA bundle: " . $this->caBundlePath );
130 'connTimeout',
'maxConnTimeout',
'reqTimeout',
'maxReqTimeout',
131 'usePipelining',
'maxConnsPerHost',
'proxy',
'userAgent',
'logger',
132 'localProxy',
'localVirtualHosts',
'headers',
'telemetry'
134 foreach ( $opts as $key ) {
135 if ( isset( $options[$key] ) ) {
136 $this->$key = $options[$key];
139 $this->logger ??=
new NullLogger;
165 public function run( array $req, array $opts = [] ) {
166 return $this->
runMulti( [ $req ], $opts )[0][
'response'];
200 public function runMulti( array $reqs, array $opts = [] ) {
201 $this->normalizeRequests( $reqs );
204 if ( $this->maxConnTimeout && $opts[
'connTimeout'] > $this->maxConnTimeout ) {
207 if ( $this->maxReqTimeout && $opts[
'reqTimeout'] > $this->maxReqTimeout ) {
212 switch ( $opts[
'httpVersion'] ??
null ) {
214 $opts[
'httpVersion'] = CURL_HTTP_VERSION_1_0;
217 $opts[
'httpVersion'] = CURL_HTTP_VERSION_1_1;
221 $opts[
'httpVersion'] = CURL_HTTP_VERSION_2_0;
224 $opts[
'httpVersion'] = CURL_HTTP_VERSION_NONE;
226 return $this->runMultiCurl( $reqs, $opts );
228 # TODO: Add handling for httpVersion option
229 return $this->runMultiHttp( $reqs, $opts );
241 return extension_loaded(
'curl' ) && function_exists(
'curl_multi_init' );
261 private function runMultiCurl( array $reqs, array $opts ) {
264 $selectTimeout = $this->getSelectTimeout( $opts );
268 foreach ( $reqs as $index => &$req ) {
270 curl_multi_add_handle( $chm, $handles[$index] );
280 $mrc = curl_multi_exec( $chm, $active );
281 $info = curl_multi_info_read( $chm );
282 if ( $info !==
false ) {
286 $infos[(int)$info[
'handle']] = $info;
288 }
while ( $mrc == CURLM_CALL_MULTI_PERFORM );
290 if ( $active > 0 && $mrc == CURLM_OK && curl_multi_select( $chm, $selectTimeout ) == -1 ) {
294 }
while ( $active > 0 && $mrc == CURLM_OK );
297 foreach ( $reqs as $index => &$req ) {
298 $ch = $handles[$index];
299 curl_multi_remove_handle( $chm, $ch );
301 if ( isset( $infos[(
int)$ch] ) ) {
302 $info = $infos[(int)$ch];
303 $errno = $info[
'result'];
304 if ( $errno !== 0 ) {
305 $req[
'response'][
'error'] =
"(curl error: $errno)";
306 if ( function_exists(
'curl_strerror' ) ) {
307 $req[
'response'][
'error'] .=
" " . curl_strerror( $errno );
309 $this->logger->warning(
"Error fetching URL \"{$req['url']}\": " .
310 $req[
'response'][
'error'] );
312 $this->logger->debug(
313 "HTTP complete: {method} {url} code={response_code} size={size} " .
314 "total={total_time} connect={connect_time}",
316 'method' => $req[
'method'],
317 'url' => $req[
'url'],
318 'response_code' => $req[
'response'][
'code'],
319 'size' => curl_getinfo( $ch, CURLINFO_SIZE_DOWNLOAD ),
320 'total_time' => $this->getCurlTime(
321 $ch, CURLINFO_TOTAL_TIME,
'CURLINFO_TOTAL_TIME_T'
323 'connect_time' => $this->getCurlTime(
324 $ch, CURLINFO_CONNECT_TIME,
'CURLINFO_CONNECT_TIME_T'
330 $req[
'response'][
'error'] =
"(curl error: no status set)";
334 $req[
'response'][0] = $req[
'response'][
'code'];
335 $req[
'response'][1] = $req[
'response'][
'reason'];
336 $req[
'response'][2] = $req[
'response'][
'headers'];
337 $req[
'response'][3] = $req[
'response'][
'body'];
338 $req[
'response'][4] = $req[
'response'][
'error'];
341 if ( isset( $req[
'_closeHandle'] ) ) {
342 fclose( $req[
'_closeHandle'] );
343 unset( $req[
'_closeHandle'] );
366 curl_setopt( $ch, CURLOPT_PROXY, $req[
'proxy'] ?? $this->proxy );
367 curl_setopt( $ch, CURLOPT_CONNECTTIMEOUT_MS, intval( $opts[
'connTimeout'] * 1e3 ) );
368 curl_setopt( $ch, CURLOPT_TIMEOUT_MS, intval( $opts[
'reqTimeout'] * 1e3 ) );
369 curl_setopt( $ch, CURLOPT_FOLLOWLOCATION, 1 );
370 curl_setopt( $ch, CURLOPT_MAXREDIRS, 4 );
371 curl_setopt( $ch, CURLOPT_HEADER, 0 );
372 if ( $this->caBundlePath !==
null ) {
373 curl_setopt( $ch, CURLOPT_SSL_VERIFYPEER,
true );
374 curl_setopt( $ch, CURLOPT_CAINFO, $this->caBundlePath );
376 curl_setopt( $ch, CURLOPT_RETURNTRANSFER, 1 );
379 $query = http_build_query( $req[
'query'],
'',
'&', PHP_QUERY_RFC3986 );
380 if ( $query !=
'' ) {
381 $url .= strpos( $req[
'url'],
'?' ) ===
false ?
"?$query" :
"&$query";
383 curl_setopt( $ch, CURLOPT_URL,
$url );
384 curl_setopt( $ch, CURLOPT_CUSTOMREQUEST, $req[
'method'] );
385 curl_setopt( $ch, CURLOPT_NOBODY, ( $req[
'method'] ===
'HEAD' ) );
386 curl_setopt( $ch, CURLOPT_HTTP_VERSION, $opts[
'httpVersion'] ?? CURL_HTTP_VERSION_NONE );
388 if ( $req[
'method'] ===
'PUT' ) {
389 curl_setopt( $ch, CURLOPT_PUT, 1 );
391 if ( is_resource( $req[
'body'] ) ) {
392 curl_setopt( $ch, CURLOPT_INFILE, $req[
'body'] );
393 if ( isset( $req[
'headers'][
'content-length'] ) ) {
394 curl_setopt( $ch, CURLOPT_INFILESIZE, $req[
'headers'][
'content-length'] );
395 } elseif ( isset( $req[
'headers'][
'transfer-encoding'] ) &&
396 $req[
'headers'][
'transfer-encoding'] ===
'chunks'
398 curl_setopt( $ch, CURLOPT_UPLOAD,
true );
400 throw new InvalidArgumentException(
"Missing 'Content-Length' or 'Transfer-Encoding' header." );
402 } elseif ( $req[
'body'] !==
'' ) {
403 $fp = fopen(
"php://temp",
"wb+" );
404 fwrite( $fp, $req[
'body'], strlen( $req[
'body'] ) );
406 curl_setopt( $ch, CURLOPT_INFILE, $fp );
407 curl_setopt( $ch, CURLOPT_INFILESIZE, strlen( $req[
'body'] ) );
408 $req[
'_closeHandle'] = $fp;
410 curl_setopt( $ch, CURLOPT_INFILESIZE, 0 );
412 curl_setopt( $ch, CURLOPT_READFUNCTION,
413 static function ( $ch, $fd, $length ) {
414 return (
string)fread( $fd, $length );
417 } elseif ( $req[
'method'] ===
'POST' ) {
418 curl_setopt( $ch, CURLOPT_POST, 1 );
419 curl_setopt( $ch, CURLOPT_POSTFIELDS, $req[
'body'] );
422 if ( is_resource( $req[
'body'] ) || $req[
'body'] !==
'' ) {
423 throw new InvalidArgumentException(
"HTTP body specified for a non PUT/POST request." );
425 $req[
'headers'][
'content-length'] = 0;
428 if ( !isset( $req[
'headers'][
'user-agent'] ) ) {
433 foreach ( $req[
'headers'] as $name => $value ) {
434 if ( strpos( $name,
':' ) !==
false ) {
435 throw new InvalidArgumentException(
"Header name must not contain colon-space." );
437 $headers[] = $name .
': ' . trim( $value );
439 curl_setopt( $ch, CURLOPT_HTTPHEADER,
$headers );
441 curl_setopt( $ch, CURLOPT_HEADERFUNCTION,
442 static function ( $ch,
$header ) use ( &$req ) {
443 if ( !empty( $req[
'flags'][
'relayResponseHeaders'] ) && trim(
$header ) !==
'' ) {
448 if ( preg_match(
"/^(HTTP\/(?:1\.[01]|2)) (\d{3}) (.*)/",
$header,
$matches ) ) {
449 $req[
'response'][
'code'] = (int)
$matches[2];
450 $req[
'response'][
'reason'] = trim(
$matches[3] );
453 $req[
'response'][
'headers'] = [];
456 if ( strpos(
$header,
":" ) ===
false ) {
459 [ $name, $value ] = explode(
":",
$header, 2 );
460 $name = strtolower( $name );
461 $value = trim( $value );
462 if ( isset( $req[
'response'][
'headers'][$name] ) ) {
463 $req[
'response'][
'headers'][$name] .=
', ' . $value;
465 $req[
'response'][
'headers'][$name] = $value;
472 $hasOutputStream = isset( $req[
'stream'] );
473 curl_setopt( $ch, CURLOPT_WRITEFUNCTION,
474 static function ( $ch, $data ) use ( &$req, $hasOutputStream ) {
475 if ( $hasOutputStream ) {
477 return fwrite( $req[
'stream'], $data );
480 $req[
'response'][
'body'] .= $data;
482 return strlen( $data );
498 $cmh = curl_multi_init();
501 curl_multi_setopt(
$cmh, CURLMOPT_MAXCONNECTS, (
int)$this->maxConnsPerHost );
505 $curlVersion = curl_version()[
'version'];
508 if ( version_compare( $curlVersion,
'7.30.0',
'>=' ) ) {
511 curl_multi_setopt( $this->cmh, CURLMOPT_MAX_HOST_CONNECTIONS, (
int)$maxHostConns );
514 if ( $opts[
'usePipelining'] ?? $this->usePipelining ) {
515 if ( version_compare( $curlVersion,
'7.43',
'<' ) ) {
518 } elseif ( version_compare( $curlVersion,
'7.62',
'<' ) ) {
520 $pipelining = CURLPIPE_HTTP1 | CURLPIPE_MULTIPLEX;
523 $pipelining = CURLPIPE_MULTIPLEX;
527 @curl_multi_setopt( $this->cmh, CURLMOPT_PIPELINING, $pipelining );
543 private function getCurlTime( $ch, $oldOption, $newConstName ): string {
544 if ( defined( $newConstName ) ) {
545 return sprintf(
"%.6F", curl_getinfo( $ch, constant( $newConstName ) ) / 1e6 );
547 return (
string)curl_getinfo( $ch, $oldOption );
566 private function runMultiHttp( array $reqs, array $opts = [] ) {
568 'timeout' => $opts[
'reqTimeout'] ?? $this->reqTimeout,
569 'connectTimeout' => $opts[
'connTimeout'] ?? $this->connTimeout,
570 'logger' => $this->logger,
571 'caInfo' => $this->caBundlePath,
573 foreach ( $reqs as &$req ) {
574 $reqOptions = $httpOptions + [
575 'method' => $req[
'method'],
576 'proxy' => $req[
'proxy'] ?? $this->proxy,
577 'userAgent' => $req[
'headers'][
'user-agent'] ?? $this->userAgent,
578 'postData' => $req[
'body'],
582 $query = http_build_query( $req[
'query'],
'',
'&', PHP_QUERY_RFC3986 );
583 if ( $query !=
'' ) {
584 $url .= strpos( $req[
'url'],
'?' ) ===
false ?
"?$query" :
"&$query";
587 $httpRequest = MediaWikiServices::getInstance()->getHttpRequestFactory()->create(
588 $url, $reqOptions, __METHOD__ );
589 $httpRequest->setLogger( $this->logger );
590 foreach ( $req[
'headers'] as
$header => $value ) {
591 $httpRequest->setHeader(
$header, $value );
593 $sv = $httpRequest->execute()->getStatusValue();
595 $respHeaders = array_map(
596 static function ( $v ) {
597 return implode(
', ', $v );
599 $httpRequest->getResponseHeaders() );
602 'code' => $httpRequest->getStatus(),
604 'headers' => $respHeaders,
605 'body' => $httpRequest->getContent(),
609 if ( !$sv->isOK() ) {
610 $svErrors = $sv->getErrors();
611 if ( isset( $svErrors[0] ) ) {
612 $req[
'response'][
'error'] = $svErrors[0][
'message'];
615 if ( isset( $svErrors[0][
'params'][0] ) ) {
616 if ( is_numeric( $svErrors[0][
'params'][0] ) ) {
617 if ( isset( $svErrors[0][
'params'][1] ) ) {
619 $req[
'response'][
'reason'] = $svErrors[0][
'params'][1];
622 $req[
'response'][
'reason'] = $svErrors[0][
'params'][0];
628 $req[
'response'][0] = $req[
'response'][
'code'];
629 $req[
'response'][1] = $req[
'response'][
'reason'];
630 $req[
'response'][2] = $req[
'response'][
'headers'];
631 $req[
'response'][3] = $req[
'response'][
'body'];
632 $req[
'response'][4] = $req[
'response'][
'error'];
643 private function normalizeHeaders( array $headers ): array {
645 foreach ( $headers as $name => $value ) {
646 $normalized[strtolower( $name )] = $value;
656 private function normalizeRequests( array &$reqs ) {
657 foreach ( $reqs as &$req ) {
665 if ( isset( $req[0] ) ) {
666 $req[
'method'] = $req[0];
669 if ( isset( $req[1] ) ) {
670 $req[
'url'] = $req[1];
673 if ( !isset( $req[
'method'] ) ) {
674 throw new InvalidArgumentException(
"Request has no 'method' field set." );
675 } elseif ( !isset( $req[
'url'] ) ) {
676 throw new InvalidArgumentException(
"Request has no 'url' field set." );
678 if ( $this->localProxy !==
false && $this->isLocalURL( $req[
'url'] ) ) {
679 $this->useReverseProxy( $req, $this->localProxy );
681 $req[
'query'] ??= [];
682 $req[
'headers'] = $this->normalizeHeaders(
685 $this->telemetry ? $this->telemetry->getRequestHeaders() : [],
686 $req[
'headers'] ?? []
690 if ( !isset( $req[
'body'] ) ) {
692 $req[
'headers'][
'content-length'] = 0;
695 $logHeaders = $req[
'headers'];
696 foreach ( $logHeaders as
$header => $value ) {
697 if ( preg_match( self::SENSITIVE_HEADERS,
$header ) === 1 ) {
698 $logHeaders[
$header] =
'[redacted]';
701 $this->logger->debug(
"HTTP start: {method} {url}",
703 'method' => $req[
'method'],
704 'url' => $req[
'url'],
705 'headers' => $logHeaders,
708 $req[
'flags'] ??= [];
712 private function useReverseProxy( array &$req, $proxy ) {
714 if ( $parsedProxy ===
false ) {
715 throw new InvalidArgumentException(
"Invalid reverseProxy configured: $proxy" );
718 if ( $parsedUrl ===
false ) {
719 throw new InvalidArgumentException(
"Invalid url specified: {$req['url']}" );
722 $req[
'headers'][
'Host'] = $parsedUrl[
'host'];
724 $parsedUrl[
'scheme'] = $parsedProxy[
'scheme'];
725 $parsedUrl[
'host'] = $parsedProxy[
'host'];
726 if ( isset( $parsedProxy[
'port'] ) ) {
727 $parsedUrl[
'port'] = $parsedProxy[
'port'];
729 unset( $parsedUrl[
'port'] );
734 $req[
'proxy'] =
false;
744 private function isLocalURL(
$url ) {
745 if ( !$this->localVirtualHosts ) {
752 if ( preg_match(
'!^https?://([\w.-]+)[/:].*$!',
$url,
$matches ) ) {
755 $domainParts = explode(
'.', $host );
757 $domainParts = array_reverse( $domainParts );
760 $countParts = count( $domainParts );
761 for ( $i = 0; $i < $countParts; $i++ ) {
762 $domainPart = $domainParts[$i];
764 $domain = $domainPart;
766 $domain = $domainPart .
'.' . $domain;
769 if ( in_array( $domain, $this->localVirtualHosts ) ) {
784 private function getSelectTimeout( $opts ) {
785 $connTimeout = $opts[
'connTimeout'] ?? $this->connTimeout;
786 $reqTimeout = $opts[
'reqTimeout'] ?? $this->reqTimeout;
787 $timeouts = array_filter( [ $connTimeout, $reqTimeout ] );
788 if ( count( $timeouts ) === 0 ) {
792 $selectTimeout = min( $timeouts ) * self::TIMEOUT_ACCURACY_FACTOR;
794 if ( $selectTimeout < 10e-6 ) {
795 $selectTimeout = 10e-6;
797 return $selectTimeout;
806 $this->logger = $logger;
811 curl_multi_close( $this->cmh );