25 use Wikimedia\AtEase\AtEase;
119 parent::__construct( $config );
121 $this->swiftAuthUrl = $config[
'swiftAuthUrl'];
122 $this->swiftUser = $config[
'swiftUser'];
123 $this->swiftKey = $config[
'swiftKey'];
125 $this->authTTL = $config[
'swiftAuthTTL'] ?? 15 * 60;
126 $this->swiftTempUrlKey = $config[
'swiftTempUrlKey'] ??
'';
127 $this->swiftStorageUrl = $config[
'swiftStorageUrl'] ??
null;
128 $this->shardViaHashLevels = $config[
'shardViaHashLevels'] ??
'';
129 $this->rgwS3AccessKey = $config[
'rgwS3AccessKey'] ??
'';
130 $this->rgwS3SecretKey = $config[
'rgwS3SecretKey'] ??
'';
134 foreach ( [
'connTimeout',
'reqTimeout' ] as $optionName ) {
135 if ( isset( $config[$optionName] ) ) {
136 $httpOptions[$optionName] = $config[$optionName];
142 if ( isset( $config[
'wanCache'] ) && $config[
'wanCache'] instanceof
WANObjectCache ) {
143 $this->memCache = $config[
'wanCache'];
146 $this->containerStatCache =
new MapCacheLRU( 300 );
148 if ( !empty( $config[
'cacheAuthInfo'] ) && isset( $config[
'srvCache'] ) ) {
149 $this->srvCache = $config[
'srvCache'];
153 $this->readUsers = $config[
'readUsers'] ?? [];
154 $this->writeUsers = $config[
'writeUsers'] ?? [];
155 $this->secureReadUsers = $config[
'secureReadUsers'] ?? [];
156 $this->secureWriteUsers = $config[
'secureWriteUsers'] ?? [];
161 self::ATTR_UNICODE_PATHS |
168 if ( !mb_check_encoding( $relStoragePath,
'UTF-8' ) ) {
170 } elseif ( strlen( rawurlencode( $relStoragePath ) ) > 1024 ) {
174 return $relStoragePath;
179 if ( $rel ===
null ) {
196 $contentHeaders = [];
198 foreach ( $headers as
$name => $value ) {
200 if ( !preg_match(
'/^(x-)?content-(?!length$)/',
$name ) ) {
203 } elseif (
$name ===
'content-type' && !strlen( $value ) ) {
207 $contentHeaders[
$name] = $value;
210 if ( isset( $contentHeaders[
'content-disposition'] ) ) {
213 $offset = $maxLength - strlen( $contentHeaders[
'content-disposition'] );
215 $pos = strrpos( $contentHeaders[
'content-disposition'],
';', $offset );
216 $contentHeaders[
'content-disposition'] = $pos ===
false
218 : trim( substr( $contentHeaders[
'content-disposition'], 0, $pos ) );
222 return $contentHeaders;
231 $metadataHeaders = [];
232 foreach ( $headers as
$name => $value ) {
234 if ( strpos(
$name,
'x-object-meta-' ) === 0 ) {
235 $metadataHeaders[
$name] = $value;
239 return $metadataHeaders;
248 $prefixLen = strlen(
'x-object-meta-' );
252 $metadata[substr(
$name, $prefixLen )] = $value;
262 if ( $dstRel ===
null ) {
263 $status->fatal(
'backend-fail-invalidpath', $params[
'dst'] );
271 $mutableHeaders[
'content-type'] = $mutableHeaders[
'content-type']
272 ?? $this->
getContentType( $params[
'dst'], $params[
'content'],
null );
276 'url' => [ $dstCont, $dstRel ],
277 'headers' => array_merge(
280 'etag' => md5( $params[
'content'] ),
281 'content-length' => strlen( $params[
'content'] ),
282 'x-object-meta-sha1base36' =>
283 Wikimedia\base_convert( sha1( $params[
'content'] ), 16, 36, 31 )
286 'body' => $params[
'content']
289 $method = __METHOD__;
290 $handler =
function ( array $request,
StatusValue $status ) use ( $method, $params ) {
291 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request[
'response'];
292 if ( $rcode === 201 || $rcode === 202 ) {
294 } elseif ( $rcode === 412 ) {
295 $status->fatal(
'backend-fail-contenttype', $params[
'dst'] );
297 $this->
onError( $status, $method, $params, $rerr, $rcode, $rdesc );
300 return SwiftFileOpHandle::CONTINUE_IF_OK;
304 if ( !empty( $params[
'async'] ) ) {
305 $status->value = $opHandle;
317 if ( $dstRel ===
null ) {
318 $status->fatal(
'backend-fail-invalidpath', $params[
'dst'] );
326 AtEase::suppressWarnings();
327 $srcHandle = fopen( $params[
'src'],
'rb' );
328 AtEase::restoreWarnings();
329 if ( $srcHandle ===
false ) {
330 $status->fatal(
'backend-fail-notexists', $params[
'src'] );
336 $srcSize = fstat( $srcHandle )[
'size'];
337 $md5Context = hash_init(
'md5' );
338 $sha1Context = hash_init(
'sha1' );
340 while ( !feof( $srcHandle ) ) {
341 $buffer = (string)fread( $srcHandle, 131072 );
342 hash_update( $md5Context, $buffer );
343 hash_update( $sha1Context, $buffer );
344 $hashDigestSize += strlen( $buffer );
347 rewind( $srcHandle );
349 if ( $hashDigestSize !== $srcSize ) {
350 $status->fatal(
'backend-fail-hash', $params[
'src'] );
358 $mutableHeaders[
'content-type'] = $mutableHeaders[
'content-type']
363 'url' => [ $dstCont, $dstRel ],
364 'headers' => array_merge(
367 'content-length' => $srcSize,
368 'etag' => hash_final( $md5Context ),
369 'x-object-meta-sha1base36' =>
370 Wikimedia\base_convert( hash_final( $sha1Context ), 16, 36, 31 )
376 $method = __METHOD__;
377 $handler =
function ( array $request,
StatusValue $status ) use ( $method, $params ) {
378 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request[
'response'];
379 if ( $rcode === 201 || $rcode === 202 ) {
381 } elseif ( $rcode === 412 ) {
382 $status->fatal(
'backend-fail-contenttype', $params[
'dst'] );
384 $this->
onError( $status, $method, $params, $rerr, $rcode, $rdesc );
387 return SwiftFileOpHandle::CONTINUE_IF_OK;
391 $opHandle->resourcesToClose[] = $srcHandle;
393 if ( !empty( $params[
'async'] ) ) {
394 $status->value = $opHandle;
406 if ( $srcRel ===
null ) {
407 $status->fatal(
'backend-fail-invalidpath', $params[
'src'] );
413 if ( $dstRel ===
null ) {
414 $status->fatal(
'backend-fail-invalidpath', $params[
'dst'] );
421 'url' => [ $dstCont, $dstRel ],
422 'headers' => array_merge(
425 'x-copy-from' =>
'/' . rawurlencode( $srcCont ) .
'/' .
426 str_replace(
"%2F",
"/", rawurlencode( $srcRel ) )
431 $method = __METHOD__;
432 $handler =
function ( array $request,
StatusValue $status ) use ( $method, $params ) {
433 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request[
'response'];
434 if ( $rcode === 201 ) {
436 } elseif ( $rcode === 404 ) {
437 if ( empty( $params[
'ignoreMissingSource'] ) ) {
438 $status->fatal(
'backend-fail-copy', $params[
'src'], $params[
'dst'] );
441 $this->
onError( $status, $method, $params, $rerr, $rcode, $rdesc );
444 return SwiftFileOpHandle::CONTINUE_IF_OK;
448 if ( !empty( $params[
'async'] ) ) {
449 $status->value = $opHandle;
461 if ( $srcRel ===
null ) {
462 $status->fatal(
'backend-fail-invalidpath', $params[
'src'] );
468 if ( $dstRel ===
null ) {
469 $status->fatal(
'backend-fail-invalidpath', $params[
'dst'] );
476 'url' => [ $dstCont, $dstRel ],
477 'headers' => array_merge(
480 'x-copy-from' =>
'/' . rawurlencode( $srcCont ) .
'/' .
481 str_replace(
"%2F",
"/", rawurlencode( $srcRel ) )
485 if (
"{$srcCont}/{$srcRel}" !==
"{$dstCont}/{$dstRel}" ) {
487 'method' =>
'DELETE',
488 'url' => [ $srcCont, $srcRel ],
493 $method = __METHOD__;
494 $handler =
function ( array $request,
StatusValue $status ) use ( $method, $params ) {
495 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request[
'response'];
496 if ( $request[
'method'] ===
'PUT' && $rcode === 201 ) {
498 } elseif ( $request[
'method'] ===
'DELETE' && $rcode === 204 ) {
500 } elseif ( $rcode === 404 ) {
501 if ( empty( $params[
'ignoreMissingSource'] ) ) {
502 $status->fatal(
'backend-fail-move', $params[
'src'], $params[
'dst'] );
505 return SwiftFileOpHandle::CONTINUE_NO;
508 $this->
onError( $status, $method, $params, $rerr, $rcode, $rdesc );
511 return SwiftFileOpHandle::CONTINUE_IF_OK;
515 if ( !empty( $params[
'async'] ) ) {
516 $status->value = $opHandle;
528 if ( $srcRel ===
null ) {
529 $status->fatal(
'backend-fail-invalidpath', $params[
'src'] );
535 'method' =>
'DELETE',
536 'url' => [ $srcCont, $srcRel ],
540 $method = __METHOD__;
541 $handler =
function ( array $request,
StatusValue $status ) use ( $method, $params ) {
542 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request[
'response'];
543 if ( $rcode === 204 ) {
545 } elseif ( $rcode === 404 ) {
546 if ( empty( $params[
'ignoreMissingSource'] ) ) {
547 $status->fatal(
'backend-fail-delete', $params[
'src'] );
550 $this->
onError( $status, $method, $params, $rerr, $rcode, $rdesc );
553 return SwiftFileOpHandle::CONTINUE_IF_OK;
557 if ( !empty( $params[
'async'] ) ) {
558 $status->value = $opHandle;
570 if ( $srcRel ===
null ) {
571 $status->fatal(
'backend-fail-invalidpath', $params[
'src'] );
577 $stat = $this->
getFileStat( [
'src' => $params[
'src'],
'latest' => 1 ] );
578 if ( $stat && !isset( $stat[
'xattr'] ) ) {
579 $stat = $this->
doGetFileStat( [
'src' => $params[
'src'],
'latest' => 1 ] );
582 $status->fatal(
'backend-fail-describe', $params[
'src'] );
590 $oldMetadataHeaders = [];
591 foreach ( $stat[
'xattr'][
'metadata'] as
$name => $value ) {
592 $oldMetadataHeaders[
"x-object-meta-$name"] = $value;
595 $oldContentHeaders = $stat[
'xattr'][
'headers'];
599 'url' => [ $srcCont, $srcRel ],
600 'headers' => $oldMetadataHeaders + $newContentHeaders + $oldContentHeaders
603 $method = __METHOD__;
604 $handler =
function ( array $request,
StatusValue $status ) use ( $method, $params ) {
605 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request[
'response'];
606 if ( $rcode === 202 ) {
608 } elseif ( $rcode === 404 ) {
609 $status->fatal(
'backend-fail-describe', $params[
'src'] );
611 $this->
onError( $status, $method, $params, $rerr, $rcode, $rdesc );
616 if ( !empty( $params[
'async'] ) ) {
617 $status->value = $opHandle;
630 if ( is_array( $stat ) ) {
632 } elseif ( $stat === self::$RES_ERROR ) {
633 $status->fatal(
'backend-fail-internal', $this->name );
634 $this->logger->error( __METHOD__ .
': cannot get container stat' );
640 if ( $stat ===
false ) {
641 $params[
'op'] =
'prepare';
650 if ( empty( $params[
'noAccess'] ) ) {
655 if ( is_array( $stat ) ) {
656 $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
657 $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
664 } elseif ( $stat ===
false ) {
665 $status->fatal(
'backend-fail-usable', $params[
'dir'] );
667 $status->fatal(
'backend-fail-internal', $this->name );
668 $this->logger->error( __METHOD__ .
': cannot get container stat' );
678 if ( is_array( $stat ) ) {
679 $readUsers = array_merge( $this->readUsers, [ $this->swiftUser,
'.r:*' ] );
680 $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
688 } elseif ( $stat ===
false ) {
689 $status->fatal(
'backend-fail-usable', $params[
'dir'] );
691 $status->fatal(
'backend-fail-internal', $this->name );
692 $this->logger->error( __METHOD__ .
': cannot get container stat' );
708 if ( $stat ===
false ) {
710 } elseif ( !is_array( $stat ) ) {
711 $status->fatal(
'backend-fail-internal', $this->name );
712 $this->logger->error( __METHOD__ .
': cannot get container stat' );
718 if ( $stat[
'count'] == 0 ) {
719 $params[
'op'] =
'clean';
727 $params = [
'srcs' => [ $params[
'src'] ],
'concurrency' => 1 ] + $params;
728 unset( $params[
'src'] );
731 return reset( $stats );
748 return $timestamp->getTimestamp( $format );
749 }
catch ( Exception $e ) {
762 if ( isset( $objHdrs[
'x-object-meta-sha1base36'] ) ) {
768 $this->logger->error( __METHOD__ .
": {path} was not stored with SHA-1 metadata.",
769 [
'path' =>
$path ] );
771 $objHdrs[
'x-object-meta-sha1base36'] =
false;
786 if ( $status->isOK() ) {
789 $hash = $tmpFile->getSha1Base36();
790 if ( $hash !==
false ) {
791 $objHdrs[
'x-object-meta-sha1base36'] = $hash;
793 $postHeaders[
'x-object-meta-sha1base36'] = $hash;
795 list( $rcode ) = $this->http->run( [
797 'url' => $this->
storageUrl( $auth, $srcCont, $srcRel ),
800 if ( $rcode >= 200 && $rcode <= 299 ) {
809 $this->logger->error( __METHOD__ .
': unable to set SHA-1 metadata for {path}',
810 [
'path' =>
$path ] );
818 $ep = array_diff_key( $params, [
'srcs' => 1 ] );
824 $contents = array_fill_keys( $params[
'srcs'], self::$RES_ERROR );
825 foreach ( $params[
'srcs'] as
$path ) {
827 if ( $srcRel ===
null || !$auth ) {
831 $handle = fopen(
'php://temp',
'wb' );
835 'url' => $this->
storageUrl( $auth, $srcCont, $srcRel ),
843 $opts = [
'maxConnsPerHost' => $params[
'concurrency'] ];
844 $reqs = $this->http->runMulti( $reqs, $opts );
845 foreach ( $reqs as
$path => $op ) {
846 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op[
'response'];
847 if ( $rcode >= 200 && $rcode <= 299 ) {
848 rewind( $op[
'stream'] );
849 $content = (string)stream_get_contents( $op[
'stream'] );
852 if ( $size === (
int)$rhdrs[
'content-length'] ) {
856 $rerr =
"Got {$size}/{$rhdrs['content-length']} bytes";
857 $this->
onError(
null, __METHOD__,
858 [
'src' =>
$path ] + $ep, $rerr, $rcode, $rdesc );
860 } elseif ( $rcode === 404 ) {
864 $this->
onError(
null, __METHOD__,
865 [
'src' =>
$path ] + $ep, $rerr, $rcode, $rdesc );
867 fclose( $op[
'stream'] );
874 $prefix = ( $dir ==
'' ) ?
null :
"{$dir}/";
875 $status = $this->
objectListing( $fullCont,
'names', 1,
null, $prefix );
876 if ( $status->isOK() ) {
877 return ( count( $status->value ) ) > 0;
918 if ( $after === INF ) {
925 $prefix = ( $dir ==
'' ) ?
null :
"{$dir}/";
927 if ( !empty( $params[
'topOnly'] ) ) {
928 $status = $this->
objectListing( $fullCont,
'names', $limit, $after, $prefix,
'/' );
929 if ( !$status->isOK() ) {
932 $objects = $status->value;
934 foreach ( $objects as $object ) {
935 if ( substr( $object, -1 ) ===
'/' ) {
941 $getParentDir =
function (
$path ) {
942 return ( strpos(
$path,
'/' ) !== false ) ? dirname(
$path ) :
false;
946 $lastDir = $getParentDir( $after );
947 $status = $this->
objectListing( $fullCont,
'names', $limit, $after, $prefix );
949 if ( !$status->isOK() ) {
953 $objects = $status->value;
956 foreach ( $objects as $object ) {
957 $objectDir = $getParentDir( $object );
959 if ( $objectDir !==
false && $objectDir !== $dir ) {
964 if ( strcmp( $objectDir, $lastDir ) > 0 ) {
967 $dirs[] =
"{$pDir}/";
968 $pDir = $getParentDir( $pDir );
969 }
while ( $pDir !==
false
970 && strcmp( $pDir, $lastDir ) > 0
971 && strlen( $pDir ) > strlen( $dir )
974 $lastDir = $objectDir;
979 if ( count( $objects ) < $limit ) {
982 $after = end( $objects );
1001 if ( $after === INF ) {
1008 $prefix = ( $dir ==
'' ) ?
null :
"{$dir}/";
1011 if ( !empty( $params[
'topOnly'] ) ) {
1012 if ( !empty( $params[
'adviseStat'] ) ) {
1013 $status = $this->
objectListing( $fullCont,
'info', $limit, $after, $prefix,
'/' );
1015 $status = $this->
objectListing( $fullCont,
'names', $limit, $after, $prefix,
'/' );
1019 if ( !empty( $params[
'adviseStat'] ) ) {
1020 $status = $this->
objectListing( $fullCont,
'info', $limit, $after, $prefix );
1022 $status = $this->
objectListing( $fullCont,
'names', $limit, $after, $prefix );
1027 if ( !$status->isOK() ) {
1031 $objects = $status->value;
1035 if ( count( $objects ) < $limit ) {
1038 $after = end( $objects );
1039 $after = is_object( $after ) ? $after->name : $after;
1054 foreach ( $objects as $object ) {
1055 if ( is_object( $object ) ) {
1056 if ( isset( $object->subdir ) || !isset( $object->name ) ) {
1062 'size' => (int)$object->bytes,
1065 'md5' => ctype_xdigit( $object->hash ) ? $object->hash :
null,
1068 $names[] = [ $object->name, $stat ];
1069 } elseif ( substr( $object, -1 ) !==
'/' ) {
1071 $names[] = [ $object, null ];
1085 $this->cheapCache->setField(
$path,
'stat', $val );
1091 if ( is_array( $stat ) && !isset( $stat[
'xattr'] ) ) {
1096 if ( is_array( $stat ) ) {
1097 return $stat[
'xattr'];
1106 $params[
'requireSHA1'] =
true;
1109 if ( is_array( $stat ) ) {
1110 return $stat[
'sha1'];
1122 if ( $srcRel ===
null ) {
1124 $status->fatal(
'backend-fail-invalidpath', $params[
'src'] );
1132 $status->fatal(
'backend-fail-stream', $params[
'src'] );
1139 if ( $params[
'headers'] && !$this->
fileExists( $params ) ) {
1141 $status->fatal(
'backend-fail-stream', $params[
'src'] );
1147 foreach ( $params[
'headers'] as
$header ) {
1151 if ( empty( $params[
'allowOB'] ) ) {
1156 $handle = fopen(
'php://output',
'wb' );
1157 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1159 'url' => $this->
storageUrl( $auth, $srcCont, $srcRel ),
1162 'stream' => $handle,
1163 'flags' => [
'relayResponseHeaders' => empty( $params[
'headless'] ) ]
1166 if ( $rcode >= 200 && $rcode <= 299 ) {
1168 } elseif ( $rcode === 404 ) {
1169 $status->fatal(
'backend-fail-stream', $params[
'src'] );
1176 $this->
onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1185 $ep = array_diff_key( $params, [
'srcs' => 1 ] );
1191 $tmpFiles = array_fill_keys( $params[
'srcs'], self::$RES_ERROR );
1192 foreach ( $params[
'srcs'] as
$path ) {
1194 if ( $srcRel ===
null || !$auth ) {
1200 $tmpFile = $this->tmpFileFactory->newTempFSFile(
'localcopy_',
$ext );
1201 $handle = $tmpFile ? fopen( $tmpFile->getPath(),
'wb' ) :
false;
1205 'url' => $this->
storageUrl( $auth, $srcCont, $srcRel ),
1208 'stream' => $handle,
1210 $tmpFiles[
$path] = $tmpFile;
1215 $latest = ( $this->isRGW || !empty( $params[
'latest'] ) );
1217 $opts = [
'maxConnsPerHost' => $params[
'concurrency'] ];
1218 $reqs = $this->http->runMulti( $reqs, $opts );
1219 foreach ( $reqs as
$path => $op ) {
1220 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op[
'response'];
1221 fclose( $op[
'stream'] );
1222 if ( $rcode >= 200 && $rcode <= 299 ) {
1224 $tmpFile = $tmpFiles[
$path];
1226 $size = $tmpFile->getSize();
1227 if ( $size !== (
int)$rhdrs[
'content-length'] ) {
1229 $rerr =
"Got {$size}/{$rhdrs['content-length']} bytes";
1230 $this->
onError(
null, __METHOD__,
1231 [
'src' =>
$path ] + $ep, $rerr, $rcode, $rdesc );
1235 $stat[
'latest'] = $latest;
1236 $this->cheapCache->setField(
$path,
'stat', $stat );
1237 } elseif ( $rcode === 404 ) {
1239 $this->cheapCache->setField(
1242 $latest ? self::$ABSENT_LATEST : self::$ABSENT_NORMAL
1246 $this->
onError(
null, __METHOD__,
1247 [
'src' =>
$path ] + $ep, $rerr, $rcode, $rdesc );
1255 if ( $this->swiftTempUrlKey !=
'' ||
1256 ( $this->rgwS3AccessKey !=
'' && $this->rgwS3SecretKey !=
'' )
1259 if ( $srcRel ===
null ) {
1260 return self::TEMPURL_ERROR;
1265 return self::TEMPURL_ERROR;
1268 $ttl = $params[
'ttl'] ?? 86400;
1269 $expires = time() + $ttl;
1271 if ( $this->swiftTempUrlKey !=
'' ) {
1272 $url = $this->
storageUrl( $auth, $srcCont, $srcRel );
1274 $contPath = parse_url( $this->
storageUrl( $auth, $srcCont ), PHP_URL_PATH );
1275 $signature = hash_hmac(
'sha1',
1276 "GET\n{$expires}\n{$contPath}/{$srcRel}",
1277 $this->swiftTempUrlKey
1280 return "{$url}?temp_url_sig={$signature}&temp_url_expires={$expires}";
1283 $spath =
'/' . rawurlencode( $srcCont ) .
'/' .
1284 str_replace(
'%2F',
'/', rawurlencode( $srcRel ) );
1286 $signature = base64_encode( hash_hmac(
1288 "GET\n\n\n{$expires}\n{$spath}",
1289 $this->rgwS3SecretKey,
1295 return str_replace(
'/swift/v1',
'', $this->
storageUrl( $auth ) . $spath ) .
1298 'Signature' => $signature,
1299 'Expires' => $expires,
1300 'AWSAccessKeyId' => $this->rgwS3AccessKey
1305 return self::TEMPURL_ERROR;
1322 if ( !empty( $params[
'latest'] ) ) {
1323 $hdrs[
'x-newest'] =
'true';
1331 '@phan-var SwiftFileOpHandle[] $fileOpHandles';
1338 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1339 $statuses[$index] = $this->
newStatus(
'backend-fail-connect', $this->name );
1346 $httpReqsByStage = [];
1347 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1348 $reqs = $fileOpHandle->httpOp;
1350 foreach ( $reqs as $stage => &$req ) {
1351 list( $container, $relPath ) = $req[
'url'];
1352 $req[
'url'] = $this->
storageUrl( $auth, $container, $relPath );
1353 $req[
'headers'] = $req[
'headers'] ?? [];
1355 $httpReqsByStage[$stage][$index] = $req;
1361 $reqCount = count( $httpReqsByStage );
1362 for ( $stage = 0; $stage < $reqCount; ++$stage ) {
1363 $httpReqs = $this->http->runMulti( $httpReqsByStage[$stage] );
1364 foreach ( $httpReqs as $index => $httpReq ) {
1366 $fileOpHandle = $fileOpHandles[$index];
1368 $status = $statuses[$index];
1369 ( $fileOpHandle->callback )( $httpReq, $status );
1374 $fileOpHandle->state === $fileOpHandle::CONTINUE_NO
1376 $stages = count( $fileOpHandle->httpOp );
1377 for (
$s = ( $stage + 1 );
$s < $stages; ++
$s ) {
1378 unset( $httpReqsByStage[
$s][$index] );
1414 $status->fatal(
'backend-fail-connect', $this->name );
1419 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1421 'url' => $this->
storageUrl( $auth, $container ),
1423 'x-container-read' => implode(
',',
$readUsers ),
1424 'x-container-write' => implode(
',',
$writeUsers )
1428 if ( $rcode != 204 && $rcode !== 202 ) {
1429 $status->fatal(
'backend-fail-internal', $this->name );
1430 $this->logger->error( __METHOD__ .
': unexpected rcode value ({rcode})',
1431 [
'rcode' => $rcode ] );
1449 if ( $bypassCache ) {
1450 $this->containerStatCache->clear( $container );
1451 } elseif ( !$this->containerStatCache->hasField( $container,
'stat' ) ) {
1454 if ( !$this->containerStatCache->hasField( $container,
'stat' ) ) {
1460 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1462 'url' => $this->
storageUrl( $auth, $container ),
1466 if ( $rcode === 204 ) {
1468 'count' => $rhdrs[
'x-container-object-count'],
1469 'bytes' => $rhdrs[
'x-container-bytes-used']
1471 if ( $bypassCache ) {
1474 $this->containerStatCache->setField( $container,
'stat', $stat );
1477 } elseif ( $rcode === 404 ) {
1480 $this->
onError(
null, __METHOD__,
1481 [
'cont' => $container ], $rerr, $rcode, $rdesc );
1487 return $this->containerStatCache->getField( $container,
'stat' );
1502 $status->fatal(
'backend-fail-connect', $this->name );
1508 if ( empty( $params[
'noAccess'] ) ) {
1510 $readUsers = array_merge( $this->readUsers, [
'.r:*', $this->swiftUser ] );
1511 $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
1514 $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
1515 $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
1518 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1520 'url' => $this->
storageUrl( $auth, $container ),
1522 'x-container-read' => implode(
',',
$readUsers ),
1523 'x-container-write' => implode(
',',
$writeUsers )
1527 if ( $rcode === 201 ) {
1529 } elseif ( $rcode === 202 ) {
1532 $this->
onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1550 $status->fatal(
'backend-fail-connect', $this->name );
1555 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1556 'method' =>
'DELETE',
1557 'url' => $this->
storageUrl( $auth, $container ),
1561 if ( $rcode >= 200 && $rcode <= 299 ) {
1562 $this->containerStatCache->clear( $container );
1563 } elseif ( $rcode === 404 ) {
1565 } elseif ( $rcode === 409 ) {
1566 $this->
onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1569 $this->
onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1588 $fullCont,
$type, $limit, $after =
null, $prefix =
null, $delim =
null
1594 $status->fatal(
'backend-fail-connect', $this->name );
1599 $query = [
'limit' => $limit ];
1600 if (
$type ===
'info' ) {
1601 $query[
'format'] =
'json';
1603 if ( $after !==
null ) {
1604 $query[
'marker'] = $after;
1606 if ( $prefix !==
null ) {
1607 $query[
'prefix'] = $prefix;
1609 if ( $delim !==
null ) {
1610 $query[
'delimiter'] = $delim;
1613 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1615 'url' => $this->
storageUrl( $auth, $fullCont ),
1620 $params = [
'cont' => $fullCont,
'prefix' => $prefix,
'delim' => $delim ];
1621 if ( $rcode === 200 ) {
1622 if (
$type ===
'info' ) {
1625 $status->value = explode(
"\n", trim( $rbody ) );
1627 } elseif ( $rcode === 204 ) {
1628 $status->value = [];
1629 } elseif ( $rcode === 404 ) {
1630 $status->value = [];
1632 $this->
onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1639 foreach ( $containerInfo as $container => $info ) {
1640 $this->containerStatCache->setField( $container,
'stat', $info );
1651 foreach ( $params[
'srcs'] as
$path ) {
1653 if ( $srcRel ===
null || !$auth ) {
1659 if ( $cstat === self::$RES_ABSENT ) {
1662 } elseif ( !is_array( $cstat ) ) {
1669 'url' => $this->
storageUrl( $auth, $srcCont, $srcRel ),
1675 $opts = [
'maxConnsPerHost' => $params[
'concurrency'] ];
1676 $reqs = $this->http->runMulti( $reqs, $opts );
1677 foreach ( $reqs as
$path => $op ) {
1678 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op[
'response'];
1679 if ( $rcode === 200 || $rcode === 204 ) {
1681 if ( !empty( $params[
'requireSHA1'] ) ) {
1686 if ( $this->isRGW ) {
1687 $stat[
'latest'] =
true;
1689 } elseif ( $rcode === 404 ) {
1693 $this->
onError(
null, __METHOD__, $params, $rerr, $rcode, $rdesc );
1695 $stats[
$path] = $stat;
1715 'size' => isset( $rhdrs[
'content-length'] ) ? (int)$rhdrs[
'content-length'] : 0,
1716 'sha1' => $metadata[
'sha1base36'] ??
null,
1718 'md5' => ctype_xdigit( $rhdrs[
'etag'] ) ? $rhdrs[
'etag'] :
null,
1719 'xattr' => [
'metadata' => $metadata,
'headers' => $headers ]
1727 if ( $this->authErrorTimestamp !==
null ) {
1728 if ( ( time() - $this->authErrorTimestamp ) < 60 ) {
1731 $this->authErrorTimestamp =
null;
1737 if ( !$this->authCreds || $reAuth ) {
1738 $this->authSessionTimestamp = 0;
1740 $creds = $this->srvCache->get( $cacheKey );
1742 if ( isset( $creds[
'auth_token'] ) && isset( $creds[
'storage_url'] ) ) {
1743 $this->authCreds = $creds;
1745 $this->authSessionTimestamp = time() - ceil( $this->authTTL / 2 );
1747 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1749 'url' =>
"{$this->swiftAuthUrl}/v1.0",
1751 'x-auth-user' => $this->swiftUser,
1752 'x-auth-key' => $this->swiftKey
1756 if ( $rcode >= 200 && $rcode <= 299 ) {
1757 $this->authCreds = [
1758 'auth_token' => $rhdrs[
'x-auth-token'],
1759 'storage_url' => $this->swiftStorageUrl ?? $rhdrs[
'x-storage-url']
1762 $this->srvCache->set( $cacheKey, $this->authCreds, ceil( $this->authTTL / 2 ) );
1763 $this->authSessionTimestamp = time();
1764 } elseif ( $rcode === 401 ) {
1765 $this->
onError(
null, __METHOD__, [],
"Authentication failed.", $rcode );
1766 $this->authErrorTimestamp = time();
1770 $this->
onError(
null, __METHOD__, [],
"HTTP return code: $rcode", $rcode );
1771 $this->authErrorTimestamp = time();
1777 if ( substr( $this->authCreds[
'storage_url'], -3 ) ===
'/v1' ) {
1778 $this->isRGW =
true;
1791 protected function storageUrl( array $creds, $container =
null, $object =
null ) {
1792 $parts = [ $creds[
'storage_url'] ];
1793 if ( strlen( $container ) ) {
1794 $parts[] = rawurlencode( $container );
1796 if ( strlen( $object ) ) {
1797 $parts[] = str_replace(
"%2F",
"/", rawurlencode( $object ) );
1800 return implode(
'/', $parts );
1808 return [
'x-auth-token' => $creds[
'auth_token'] ];
1818 return 'swiftcredentials:' . md5( $username .
':' . $this->swiftAuthUrl );
1832 public function onError( $status, $func, array $params, $err =
'', $code = 0, $desc =
'' ) {
1834 $status->fatal(
'backend-fail-internal', $this->name );
1836 if ( $code == 401 ) {
1839 $msg =
"HTTP {code} ({desc}) in '{func}' (given '{req_params}')";
1848 $msgParams[
'err'] = $err;
1850 $this->logger->error( $msg, $msgParams );