26 use Psr\Log\LoggerInterface;
27 use Wikimedia\AtEase\AtEase;
28 use Wikimedia\RequestTimeout\TimeoutException;
40 private const DEFAULT_HTTP_OPTIONS = [
'httpVersion' =>
'v1.1' ];
124 parent::__construct( $config );
126 $this->swiftAuthUrl = $config[
'swiftAuthUrl'];
127 $this->swiftUser = $config[
'swiftUser'];
128 $this->swiftKey = $config[
'swiftKey'];
130 $this->authTTL = $config[
'swiftAuthTTL'] ?? 15 * 60;
131 $this->swiftTempUrlKey = $config[
'swiftTempUrlKey'] ??
'';
132 $this->swiftStorageUrl = $config[
'swiftStorageUrl'] ??
null;
133 $this->shardViaHashLevels = $config[
'shardViaHashLevels'] ??
'';
134 $this->rgwS3AccessKey = $config[
'rgwS3AccessKey'] ??
'';
135 $this->rgwS3SecretKey = $config[
'rgwS3SecretKey'] ??
'';
139 foreach ( [
'connTimeout',
'reqTimeout' ] as $optionName ) {
140 if ( isset( $config[$optionName] ) ) {
141 $httpOptions[$optionName] = $config[$optionName];
145 $this->http->setLogger( $this->logger );
148 if ( isset( $config[
'wanCache'] ) && $config[
'wanCache'] instanceof
WANObjectCache ) {
149 $this->memCache = $config[
'wanCache'];
152 $this->containerStatCache =
new MapCacheLRU( 300 );
154 if ( !empty( $config[
'cacheAuthInfo'] ) && isset( $config[
'srvCache'] ) ) {
155 $this->srvCache = $config[
'srvCache'];
159 $this->readUsers = $config[
'readUsers'] ?? [];
160 $this->writeUsers = $config[
'writeUsers'] ?? [];
161 $this->secureReadUsers = $config[
'secureReadUsers'] ?? [];
162 $this->secureWriteUsers = $config[
'secureWriteUsers'] ?? [];
167 $this->http->setLogger(
$logger );
172 self::ATTR_UNICODE_PATHS |
179 if ( !mb_check_encoding( $relStoragePath,
'UTF-8' ) ) {
181 } elseif ( strlen( rawurlencode( $relStoragePath ) ) > 1024 ) {
185 return $relStoragePath;
190 if ( $rel ===
null ) {
207 $contentHeaders = [];
209 foreach ( $headers as
$name => $value ) {
211 if (
$name ===
'x-delete-at' && is_numeric( $value ) ) {
213 $contentHeaders[
$name] = $value;
214 } elseif (
$name ===
'x-delete-after' && is_numeric( $value ) ) {
216 $contentHeaders[
$name] = $value;
217 } elseif ( preg_match(
'/^(x-)?content-(?!length$)/',
$name ) ) {
219 $contentHeaders[
$name] = $value;
220 } elseif (
$name ===
'content-type' && strlen( $value ) ) {
222 $contentHeaders[
$name] = $value;
226 if ( isset( $contentHeaders[
'content-disposition'] ) ) {
229 $offset = $maxLength - strlen( $contentHeaders[
'content-disposition'] );
231 $pos = strrpos( $contentHeaders[
'content-disposition'],
';', $offset );
232 $contentHeaders[
'content-disposition'] = $pos ===
false
234 : trim( substr( $contentHeaders[
'content-disposition'], 0, $pos ) );
238 return $contentHeaders;
247 $metadataHeaders = [];
248 foreach ( $headers as
$name => $value ) {
250 if ( strpos(
$name,
'x-object-meta-' ) === 0 ) {
251 $metadataHeaders[
$name] = $value;
255 return $metadataHeaders;
264 $prefixLen = strlen(
'x-object-meta-' );
268 $metadata[substr(
$name, $prefixLen )] = $value;
278 if ( $dstRel ===
null ) {
279 $status->fatal(
'backend-fail-invalidpath', $params[
'dst'] );
287 $mutableHeaders[
'content-type']
288 ??= $this->
getContentType( $params[
'dst'], $params[
'content'],
null );
292 'url' => [ $dstCont, $dstRel ],
293 'headers' => array_merge(
296 'etag' => md5( $params[
'content'] ),
297 'content-length' => strlen( $params[
'content'] ),
298 'x-object-meta-sha1base36' =>
299 Wikimedia\base_convert( sha1( $params[
'content'] ), 16, 36, 31 )
302 'body' => $params[
'content']
305 $method = __METHOD__;
306 $handler =
function ( array $request,
StatusValue $status ) use ( $method, $params ) {
307 [ $rcode, $rdesc, , $rbody, $rerr ] = $request[
'response'];
308 if ( $rcode === 201 || $rcode === 202 ) {
310 } elseif ( $rcode === 412 ) {
311 $status->fatal(
'backend-fail-contenttype', $params[
'dst'] );
313 $this->
onError( $status, $method, $params, $rerr, $rcode, $rdesc, $rbody );
316 return SwiftFileOpHandle::CONTINUE_IF_OK;
320 if ( !empty( $params[
'async'] ) ) {
321 $status->value = $opHandle;
333 if ( $dstRel ===
null ) {
334 $status->fatal(
'backend-fail-invalidpath', $params[
'dst'] );
342 AtEase::suppressWarnings();
343 $srcHandle = fopen( $params[
'src'],
'rb' );
344 AtEase::restoreWarnings();
345 if ( $srcHandle ===
false ) {
346 $status->fatal(
'backend-fail-notexists', $params[
'src'] );
352 $srcSize = fstat( $srcHandle )[
'size'];
353 $md5Context = hash_init(
'md5' );
354 $sha1Context = hash_init(
'sha1' );
356 while ( !feof( $srcHandle ) ) {
357 $buffer = (string)fread( $srcHandle, 131072 );
358 hash_update( $md5Context, $buffer );
359 hash_update( $sha1Context, $buffer );
360 $hashDigestSize += strlen( $buffer );
363 rewind( $srcHandle );
365 if ( $hashDigestSize !== $srcSize ) {
366 $status->fatal(
'backend-fail-hash', $params[
'src'] );
374 $mutableHeaders[
'content-type']
375 ??= $this->
getContentType( $params[
'dst'],
null, $params[
'src'] );
379 'url' => [ $dstCont, $dstRel ],
380 'headers' => array_merge(
383 'content-length' => $srcSize,
384 'etag' => hash_final( $md5Context ),
385 'x-object-meta-sha1base36' =>
386 Wikimedia\base_convert( hash_final( $sha1Context ), 16, 36, 31 )
392 $method = __METHOD__;
393 $handler =
function ( array $request,
StatusValue $status ) use ( $method, $params ) {
394 [ $rcode, $rdesc, , $rbody, $rerr ] = $request[
'response'];
395 if ( $rcode === 201 || $rcode === 202 ) {
397 } elseif ( $rcode === 412 ) {
398 $status->fatal(
'backend-fail-contenttype', $params[
'dst'] );
400 $this->
onError( $status, $method, $params, $rerr, $rcode, $rdesc, $rbody );
403 return SwiftFileOpHandle::CONTINUE_IF_OK;
407 $opHandle->resourcesToClose[] = $srcHandle;
409 if ( !empty( $params[
'async'] ) ) {
410 $status->value = $opHandle;
422 if ( $srcRel ===
null ) {
423 $status->fatal(
'backend-fail-invalidpath', $params[
'src'] );
429 if ( $dstRel ===
null ) {
430 $status->fatal(
'backend-fail-invalidpath', $params[
'dst'] );
437 'url' => [ $dstCont, $dstRel ],
438 'headers' => array_merge(
441 'x-copy-from' =>
'/' . rawurlencode( $srcCont ) .
'/' .
442 str_replace(
"%2F",
"/", rawurlencode( $srcRel ) )
447 $method = __METHOD__;
448 $handler =
function ( array $request,
StatusValue $status ) use ( $method, $params ) {
449 [ $rcode, $rdesc, , $rbody, $rerr ] = $request[
'response'];
450 if ( $rcode === 201 ) {
452 } elseif ( $rcode === 404 ) {
453 if ( empty( $params[
'ignoreMissingSource'] ) ) {
454 $status->fatal(
'backend-fail-copy', $params[
'src'], $params[
'dst'] );
457 $this->
onError( $status, $method, $params, $rerr, $rcode, $rdesc, $rbody );
460 return SwiftFileOpHandle::CONTINUE_IF_OK;
464 if ( !empty( $params[
'async'] ) ) {
465 $status->value = $opHandle;
477 if ( $srcRel ===
null ) {
478 $status->fatal(
'backend-fail-invalidpath', $params[
'src'] );
484 if ( $dstRel ===
null ) {
485 $status->fatal(
'backend-fail-invalidpath', $params[
'dst'] );
492 'url' => [ $dstCont, $dstRel ],
493 'headers' => array_merge(
496 'x-copy-from' =>
'/' . rawurlencode( $srcCont ) .
'/' .
497 str_replace(
"%2F",
"/", rawurlencode( $srcRel ) )
501 if (
"{$srcCont}/{$srcRel}" !==
"{$dstCont}/{$dstRel}" ) {
503 'method' =>
'DELETE',
504 'url' => [ $srcCont, $srcRel ],
509 $method = __METHOD__;
510 $handler =
function ( array $request,
StatusValue $status ) use ( $method, $params ) {
511 [ $rcode, $rdesc, , $rbody, $rerr ] = $request[
'response'];
512 if ( $request[
'method'] ===
'PUT' && $rcode === 201 ) {
514 } elseif ( $request[
'method'] ===
'DELETE' && $rcode === 204 ) {
516 } elseif ( $rcode === 404 ) {
517 if ( empty( $params[
'ignoreMissingSource'] ) ) {
518 $status->fatal(
'backend-fail-move', $params[
'src'], $params[
'dst'] );
521 return SwiftFileOpHandle::CONTINUE_NO;
524 $this->
onError( $status, $method, $params, $rerr, $rcode, $rdesc, $rbody );
527 return SwiftFileOpHandle::CONTINUE_IF_OK;
531 if ( !empty( $params[
'async'] ) ) {
532 $status->value = $opHandle;
544 if ( $srcRel ===
null ) {
545 $status->fatal(
'backend-fail-invalidpath', $params[
'src'] );
551 'method' =>
'DELETE',
552 'url' => [ $srcCont, $srcRel ],
556 $method = __METHOD__;
557 $handler =
function ( array $request,
StatusValue $status ) use ( $method, $params ) {
558 [ $rcode, $rdesc, , $rbody, $rerr ] = $request[
'response'];
559 if ( $rcode === 204 ) {
561 } elseif ( $rcode === 404 ) {
562 if ( empty( $params[
'ignoreMissingSource'] ) ) {
563 $status->fatal(
'backend-fail-delete', $params[
'src'] );
566 $this->
onError( $status, $method, $params, $rerr, $rcode, $rdesc, $rbody );
569 return SwiftFileOpHandle::CONTINUE_IF_OK;
573 if ( !empty( $params[
'async'] ) ) {
574 $status->value = $opHandle;
586 if ( $srcRel ===
null ) {
587 $status->fatal(
'backend-fail-invalidpath', $params[
'src'] );
593 $stat = $this->
getFileStat( [
'src' => $params[
'src'],
'latest' => 1 ] );
594 if ( $stat && !isset( $stat[
'xattr'] ) ) {
595 $stat = $this->
doGetFileStat( [
'src' => $params[
'src'],
'latest' => 1 ] );
598 $status->fatal(
'backend-fail-describe', $params[
'src'] );
606 $oldMetadataHeaders = [];
607 foreach ( $stat[
'xattr'][
'metadata'] as
$name => $value ) {
608 $oldMetadataHeaders[
"x-object-meta-$name"] = $value;
611 $oldContentHeaders = $stat[
'xattr'][
'headers'];
615 'url' => [ $srcCont, $srcRel ],
616 'headers' => $oldMetadataHeaders + $newContentHeaders + $oldContentHeaders
619 $method = __METHOD__;
620 $handler =
function ( array $request,
StatusValue $status ) use ( $method, $params ) {
621 [ $rcode, $rdesc, , $rbody, $rerr ] = $request[
'response'];
622 if ( $rcode === 202 ) {
624 } elseif ( $rcode === 404 ) {
625 $status->fatal(
'backend-fail-describe', $params[
'src'] );
627 $this->
onError( $status, $method, $params, $rerr, $rcode, $rdesc, $rbody );
632 if ( !empty( $params[
'async'] ) ) {
633 $status->value = $opHandle;
649 if ( is_array( $stat ) ) {
651 } elseif ( $stat === self::$RES_ERROR ) {
652 $status->fatal(
'backend-fail-internal', $this->name );
653 $this->logger->error( __METHOD__ .
': cannot get container stat' );
656 $params[
'op'] =
'prepare';
665 if ( empty( $params[
'noAccess'] ) ) {
670 if ( is_array( $stat ) ) {
671 $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
672 $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
679 } elseif ( $stat === self::$RES_ABSENT ) {
680 $status->fatal(
'backend-fail-usable', $params[
'dir'] );
682 $status->fatal(
'backend-fail-internal', $this->name );
683 $this->logger->error( __METHOD__ .
': cannot get container stat' );
693 if ( is_array( $stat ) ) {
694 $readUsers = array_merge( $this->readUsers, [ $this->swiftUser,
'.r:*' ] );
695 $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
703 } elseif ( $stat === self::$RES_ABSENT ) {
704 $status->fatal(
'backend-fail-usable', $params[
'dir'] );
706 $status->fatal(
'backend-fail-internal', $this->name );
707 $this->logger->error( __METHOD__ .
': cannot get container stat' );
723 if ( $stat === self::$RES_ABSENT ) {
725 } elseif ( $stat === self::$RES_ERROR ) {
726 $status->fatal(
'backend-fail-internal', $this->name );
727 $this->logger->error( __METHOD__ .
': cannot get container stat' );
728 } elseif ( is_array( $stat ) && $stat[
'count'] == 0 ) {
730 $params[
'op'] =
'clean';
738 $params = [
'srcs' => [ $params[
'src'] ],
'concurrency' => 1 ] + $params;
739 unset( $params[
'src'] );
742 return reset( $stats );
759 return $timestamp->getTimestamp( $format );
760 }
catch ( TimeoutException $e ) {
762 }
catch ( Exception $e ) {
775 if ( isset( $objHdrs[
'x-object-meta-sha1base36'] ) ) {
781 $this->logger->error( __METHOD__ .
": {path} was not stored with SHA-1 metadata.",
782 [
'path' =>
$path ] );
784 $objHdrs[
'x-object-meta-sha1base36'] =
false;
799 if ( $status->isOK() ) {
802 $hash = $tmpFile->getSha1Base36();
803 if ( $hash !==
false ) {
804 $objHdrs[
'x-object-meta-sha1base36'] = $hash;
806 $postHeaders[
'x-object-meta-sha1base36'] = $hash;
808 [ $rcode ] = $this->http->run( [
810 'url' => $this->
storageUrl( $auth, $srcCont, $srcRel ),
812 ], self::DEFAULT_HTTP_OPTIONS );
813 if ( $rcode >= 200 && $rcode <= 299 ) {
822 $this->logger->error( __METHOD__ .
': unable to set SHA-1 metadata for {path}',
823 [
'path' =>
$path ] );
831 $ep = array_diff_key( $params, [
'srcs' => 1 ] );
837 $contents = array_fill_keys( $params[
'srcs'], self::$RES_ERROR );
838 foreach ( $params[
'srcs'] as
$path ) {
840 if ( $srcRel ===
null || !$auth ) {
844 $handle = fopen(
'php://temp',
'wb' );
848 'url' => $this->
storageUrl( $auth, $srcCont, $srcRel ),
857 'maxConnsPerHost' => $params[
'concurrency'],
858 ] + self::DEFAULT_HTTP_OPTIONS;
859 $reqs = $this->http->runMulti( $reqs, $opts );
860 foreach ( $reqs as
$path => $op ) {
861 [ $rcode, $rdesc, $rhdrs, $rbody, $rerr ] = $op[
'response'];
862 if ( $rcode >= 200 && $rcode <= 299 ) {
863 rewind( $op[
'stream'] );
864 $content = (string)stream_get_contents( $op[
'stream'] );
867 if ( $size === (
int)$rhdrs[
'content-length'] ) {
871 $rerr =
"Got {$size}/{$rhdrs['content-length']} bytes";
872 $this->
onError(
null, __METHOD__,
873 [
'src' =>
$path ] + $ep, $rerr, $rcode, $rdesc );
875 } elseif ( $rcode === 404 ) {
879 $this->
onError(
null, __METHOD__,
880 [
'src' =>
$path ] + $ep, $rerr, $rcode, $rdesc, $rbody );
882 fclose( $op[
'stream'] );
889 $prefix = ( $dir ==
'' ) ?
null :
"{$dir}/";
890 $status = $this->objectListing( $fullCont,
'names', 1,
null, $prefix );
891 if ( $status->isOK() ) {
892 return ( count( $status->value ) ) > 0;
933 if ( $after === INF ) {
940 $prefix = ( $dir ==
'' ) ?
null :
"{$dir}/";
942 if ( !empty( $params[
'topOnly'] ) ) {
943 $status = $this->objectListing( $fullCont,
'names', $limit, $after, $prefix,
'/' );
944 if ( !$status->isOK() ) {
947 $objects = $status->value;
949 foreach ( $objects as $object ) {
950 if ( substr( $object, -1 ) ===
'/' ) {
956 $getParentDir =
static function (
$path ) {
957 return ( strpos(
$path,
'/' ) !== false ) ? dirname(
$path ) :
false;
961 $lastDir = $getParentDir( $after );
962 $status = $this->objectListing( $fullCont,
'names', $limit, $after, $prefix );
964 if ( !$status->isOK() ) {
968 $objects = $status->value;
971 foreach ( $objects as $object ) {
972 $objectDir = $getParentDir( $object );
974 if ( $objectDir !==
false && $objectDir !== $dir ) {
979 if ( strcmp( $objectDir, $lastDir ) > 0 ) {
982 $dirs[] =
"{$pDir}/";
983 $pDir = $getParentDir( $pDir );
984 }
while ( $pDir !==
false
985 && strcmp( $pDir, $lastDir ) > 0
986 && strlen( $pDir ) > strlen( $dir )
989 $lastDir = $objectDir;
994 if ( count( $objects ) < $limit ) {
997 $after = end( $objects );
1016 if ( $after === INF ) {
1023 $prefix = ( $dir ==
'' ) ?
null :
"{$dir}/";
1026 if ( !empty( $params[
'topOnly'] ) ) {
1027 if ( !empty( $params[
'adviseStat'] ) ) {
1028 $status = $this->objectListing( $fullCont,
'info', $limit, $after, $prefix,
'/' );
1030 $status = $this->objectListing( $fullCont,
'names', $limit, $after, $prefix,
'/' );
1034 if ( !empty( $params[
'adviseStat'] ) ) {
1035 $status = $this->objectListing( $fullCont,
'info', $limit, $after, $prefix );
1037 $status = $this->objectListing( $fullCont,
'names', $limit, $after, $prefix );
1042 if ( !$status->isOK() ) {
1046 $objects = $status->value;
1047 $files = $this->buildFileObjectListing( $objects );
1050 if ( count( $objects ) < $limit ) {
1053 $after = end( $objects );
1054 $after = is_object( $after ) ? $after->name : $after;
1067 private function buildFileObjectListing( array $objects ) {
1069 foreach ( $objects as $object ) {
1070 if ( is_object( $object ) ) {
1071 if ( isset( $object->subdir ) || !isset( $object->name ) ) {
1077 'size' => (int)$object->bytes,
1080 'md5' => ctype_xdigit( $object->hash ) ? $object->hash :
null,
1083 $names[] = [ $object->name, $stat ];
1084 } elseif ( substr( $object, -1 ) !==
'/' ) {
1086 $names[] = [ $object, null ];
1100 $this->cheapCache->setField(
$path,
'stat', $val );
1106 if ( is_array( $stat ) && !isset( $stat[
'xattr'] ) ) {
1111 if ( is_array( $stat ) ) {
1112 return $stat[
'xattr'];
1121 $params[
'requireSHA1'] =
true;
1124 if ( is_array( $stat ) ) {
1125 return $stat[
'sha1'];
1137 if ( $srcRel ===
null ) {
1139 $status->fatal(
'backend-fail-invalidpath', $params[
'src'] );
1147 $status->fatal(
'backend-fail-stream', $params[
'src'] );
1154 if ( $params[
'headers'] && !$this->
fileExists( $params ) ) {
1156 $status->fatal(
'backend-fail-stream', $params[
'src'] );
1162 foreach ( $params[
'headers'] as
$header ) {
1166 if ( empty( $params[
'allowOB'] ) ) {
1171 $handle = fopen(
'php://output',
'wb' );
1172 [ $rcode, $rdesc, , $rbody, $rerr ] = $this->http->run( [
1174 'url' => $this->
storageUrl( $auth, $srcCont, $srcRel ),
1177 'stream' => $handle,
1178 'flags' => [
'relayResponseHeaders' => empty( $params[
'headless'] ) ]
1179 ], self::DEFAULT_HTTP_OPTIONS );
1181 if ( $rcode >= 200 && $rcode <= 299 ) {
1183 } elseif ( $rcode === 404 ) {
1184 $status->fatal(
'backend-fail-stream', $params[
'src'] );
1191 $this->
onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc, $rbody );
1200 $ep = array_diff_key( $params, [
'srcs' => 1 ] );
1206 $tmpFiles = array_fill_keys( $params[
'srcs'], self::$RES_ERROR );
1207 foreach ( $params[
'srcs'] as
$path ) {
1209 if ( $srcRel ===
null || !$auth ) {
1215 $tmpFile = $this->tmpFileFactory->newTempFSFile(
'localcopy_',
$ext );
1216 $handle = $tmpFile ? fopen( $tmpFile->getPath(),
'wb' ) :
false;
1220 'url' => $this->
storageUrl( $auth, $srcCont, $srcRel ),
1223 'stream' => $handle,
1225 $tmpFiles[
$path] = $tmpFile;
1230 $latest = ( $this->isRGW || !empty( $params[
'latest'] ) );
1233 'maxConnsPerHost' => $params[
'concurrency'],
1234 ] + self::DEFAULT_HTTP_OPTIONS;
1235 $reqs = $this->http->runMulti( $reqs, $opts );
1236 foreach ( $reqs as
$path => $op ) {
1237 [ $rcode, $rdesc, $rhdrs, $rbody, $rerr ] = $op[
'response'];
1238 fclose( $op[
'stream'] );
1239 if ( $rcode >= 200 && $rcode <= 299 ) {
1241 $tmpFile = $tmpFiles[
$path];
1243 $size = $tmpFile->getSize();
1244 if ( $size !== (
int)$rhdrs[
'content-length'] ) {
1246 $rerr =
"Got {$size}/{$rhdrs['content-length']} bytes";
1247 $this->
onError(
null, __METHOD__,
1248 [
'src' =>
$path ] + $ep, $rerr, $rcode, $rdesc );
1252 $stat[
'latest'] = $latest;
1253 $this->cheapCache->setField(
$path,
'stat', $stat );
1254 } elseif ( $rcode === 404 ) {
1256 $this->cheapCache->setField(
1259 $latest ? self::$ABSENT_LATEST : self::$ABSENT_NORMAL
1263 $this->
onError(
null, __METHOD__,
1264 [
'src' =>
$path ] + $ep, $rerr, $rcode, $rdesc, $rbody );
1272 if ( $this->swiftTempUrlKey !=
'' ||
1273 ( $this->rgwS3AccessKey !=
'' && $this->rgwS3SecretKey !=
'' )
1276 if ( $srcRel ===
null ) {
1277 return self::TEMPURL_ERROR;
1282 return self::TEMPURL_ERROR;
1285 $ttl = $params[
'ttl'] ?? 86400;
1286 $expires = time() + $ttl;
1288 if ( $this->swiftTempUrlKey !=
'' ) {
1289 $url = $this->
storageUrl( $auth, $srcCont, $srcRel );
1291 $contPath = parse_url( $this->
storageUrl( $auth, $srcCont ), PHP_URL_PATH );
1292 $signature = hash_hmac(
'sha1',
1293 "GET\n{$expires}\n{$contPath}/{$srcRel}",
1294 $this->swiftTempUrlKey
1297 return "{$url}?temp_url_sig={$signature}&temp_url_expires={$expires}";
1300 $spath =
'/' . rawurlencode( $srcCont ) .
'/' .
1301 str_replace(
'%2F',
'/', rawurlencode( $srcRel ) );
1303 $signature = base64_encode( hash_hmac(
1305 "GET\n\n\n{$expires}\n{$spath}",
1306 $this->rgwS3SecretKey,
1312 return str_replace(
'/swift/v1',
'', $this->
storageUrl( $auth ) . $spath ) .
1315 'Signature' => $signature,
1316 'Expires' => $expires,
1317 'AWSAccessKeyId' => $this->rgwS3AccessKey
1322 return self::TEMPURL_ERROR;
1339 if ( !empty( $params[
'latest'] ) ) {
1340 $hdrs[
'x-newest'] =
'true';
1348 '@phan-var SwiftFileOpHandle[] $fileOpHandles';
1355 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1356 $statuses[$index] = $this->
newStatus(
'backend-fail-connect', $this->name );
1363 $httpReqsByStage = [];
1364 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1365 $reqs = $fileOpHandle->httpOp;
1367 foreach ( $reqs as $stage => &$req ) {
1368 [ $container, $relPath ] = $req[
'url'];
1369 $req[
'url'] = $this->
storageUrl( $auth, $container, $relPath );
1370 $req[
'headers'] ??= [];
1372 $httpReqsByStage[$stage][$index] = $req;
1378 $reqCount = count( $httpReqsByStage );
1379 for ( $stage = 0; $stage < $reqCount; ++$stage ) {
1380 $httpReqs = $this->http->runMulti( $httpReqsByStage[$stage], self::DEFAULT_HTTP_OPTIONS );
1381 foreach ( $httpReqs as $index => $httpReq ) {
1383 $fileOpHandle = $fileOpHandles[$index];
1385 $status = $statuses[$index];
1386 ( $fileOpHandle->callback )( $httpReq, $status );
1391 $fileOpHandle->state === $fileOpHandle::CONTINUE_NO
1393 $stages = count( $fileOpHandle->httpOp );
1394 for ( $s = ( $stage + 1 ); $s < $stages; ++$s ) {
1395 unset( $httpReqsByStage[$s][$index] );
1431 $status->fatal(
'backend-fail-connect', $this->name );
1436 [ $rcode, , , , ] = $this->http->run( [
1438 'url' => $this->
storageUrl( $auth, $container ),
1440 'x-container-read' => implode(
',',
$readUsers ),
1441 'x-container-write' => implode(
',',
$writeUsers )
1443 ], self::DEFAULT_HTTP_OPTIONS );
1445 if ( $rcode != 204 && $rcode !== 202 ) {
1446 $status->fatal(
'backend-fail-internal', $this->name );
1447 $this->logger->error( __METHOD__ .
': unexpected rcode value ({rcode})',
1448 [
'rcode' => $rcode ] );
1466 if ( $bypassCache ) {
1467 $this->containerStatCache->clear( $container );
1468 } elseif ( !$this->containerStatCache->hasField( $container,
'stat' ) ) {
1471 if ( !$this->containerStatCache->hasField( $container,
'stat' ) ) {
1477 [ $rcode, $rdesc, $rhdrs, $rbody, $rerr ] = $this->http->run( [
1479 'url' => $this->
storageUrl( $auth, $container ),
1481 ], self::DEFAULT_HTTP_OPTIONS );
1483 if ( $rcode === 204 ) {
1485 'count' => $rhdrs[
'x-container-object-count'],
1486 'bytes' => $rhdrs[
'x-container-bytes-used']
1488 if ( $bypassCache ) {
1491 $this->containerStatCache->setField( $container,
'stat', $stat );
1494 } elseif ( $rcode === 404 ) {
1497 $this->
onError(
null, __METHOD__,
1498 [
'cont' => $container ], $rerr, $rcode, $rdesc, $rbody );
1504 return $this->containerStatCache->getField( $container,
'stat' );
1519 $status->fatal(
'backend-fail-connect', $this->name );
1525 if ( empty( $params[
'noAccess'] ) ) {
1527 $readUsers = array_merge( $this->readUsers, [
'.r:*', $this->swiftUser ] );
1528 $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
1531 $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
1532 $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
1535 [ $rcode, $rdesc, , $rbody, $rerr ] = $this->http->run( [
1537 'url' => $this->
storageUrl( $auth, $container ),
1539 'x-container-read' => implode(
',',
$readUsers ),
1540 'x-container-write' => implode(
',',
$writeUsers )
1542 ], self::DEFAULT_HTTP_OPTIONS );
1544 if ( $rcode === 201 ) {
1546 } elseif ( $rcode === 202 ) {
1549 $this->
onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc, $rbody );
1567 $status->fatal(
'backend-fail-connect', $this->name );
1572 [ $rcode, $rdesc, , $rbody, $rerr ] = $this->http->run( [
1573 'method' =>
'DELETE',
1574 'url' => $this->
storageUrl( $auth, $container ),
1576 ], self::DEFAULT_HTTP_OPTIONS );
1578 if ( $rcode >= 200 && $rcode <= 299 ) {
1579 $this->containerStatCache->clear( $container );
1580 } elseif ( $rcode === 404 ) {
1582 } elseif ( $rcode === 409 ) {
1583 $this->
onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1585 $this->
onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc, $rbody );
1603 private function objectListing(
1604 $fullCont, $type, $limit, $after =
null, $prefix =
null, $delim =
null
1610 $status->fatal(
'backend-fail-connect', $this->name );
1615 $query = [
'limit' => $limit ];
1616 if ( $type ===
'info' ) {
1617 $query[
'format'] =
'json';
1619 if ( $after !==
null ) {
1620 $query[
'marker'] = $after;
1622 if ( $prefix !==
null ) {
1623 $query[
'prefix'] = $prefix;
1625 if ( $delim !==
null ) {
1626 $query[
'delimiter'] = $delim;
1629 [ $rcode, $rdesc, , $rbody, $rerr ] = $this->http->run( [
1631 'url' => $this->
storageUrl( $auth, $fullCont ),
1634 ], self::DEFAULT_HTTP_OPTIONS );
1636 $params = [
'cont' => $fullCont,
'prefix' => $prefix,
'delim' => $delim ];
1637 if ( $rcode === 200 ) {
1638 if ( $type ===
'info' ) {
1641 $status->value = explode(
"\n", trim( $rbody ) );
1643 } elseif ( $rcode === 204 ) {
1644 $status->value = [];
1645 } elseif ( $rcode === 404 ) {
1646 $status->value = [];
1648 $this->
onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc, $rbody );
1655 foreach ( $containerInfo as $container => $info ) {
1656 $this->containerStatCache->setField( $container,
'stat', $info );
1667 foreach ( $params[
'srcs'] as
$path ) {
1669 if ( $srcRel ===
null || !$auth ) {
1675 if ( $cstat === self::$RES_ABSENT ) {
1678 } elseif ( $cstat === self::$RES_ERROR ) {
1685 'url' => $this->
storageUrl( $auth, $srcCont, $srcRel ),
1692 'maxConnsPerHost' => $params[
'concurrency'],
1693 ] + self::DEFAULT_HTTP_OPTIONS;
1694 $reqs = $this->http->runMulti( $reqs, $opts );
1695 foreach ( $reqs as
$path => $op ) {
1696 [ $rcode, $rdesc, $rhdrs, $rbody, $rerr ] = $op[
'response'];
1697 if ( $rcode === 200 || $rcode === 204 ) {
1699 if ( !empty( $params[
'requireSHA1'] ) ) {
1704 if ( $this->isRGW ) {
1705 $stat[
'latest'] =
true;
1707 } elseif ( $rcode === 404 ) {
1711 $this->
onError(
null, __METHOD__, $params, $rerr, $rcode, $rdesc, $rbody );
1713 $stats[
$path] = $stat;
1733 'size' => isset( $rhdrs[
'content-length'] ) ? (int)$rhdrs[
'content-length'] : 0,
1734 'sha1' => $metadata[
'sha1base36'] ??
null,
1736 'md5' => ctype_xdigit( $rhdrs[
'etag'] ) ? $rhdrs[
'etag'] :
null,
1737 'xattr' => [
'metadata' => $metadata,
'headers' => $headers ]
1745 if ( $this->authErrorTimestamp !==
null ) {
1746 if ( ( time() - $this->authErrorTimestamp ) < 60 ) {
1749 $this->authErrorTimestamp =
null;
1755 if ( !$this->authCreds || $reAuth ) {
1756 $this->authSessionTimestamp = 0;
1757 $cacheKey = $this->getCredsCacheKey( $this->swiftUser );
1758 $creds = $this->srvCache->get( $cacheKey );
1760 if ( isset( $creds[
'auth_token'] ) && isset( $creds[
'storage_url'] ) ) {
1761 $this->authCreds = $creds;
1763 $this->authSessionTimestamp = time() - (int)ceil( $this->authTTL / 2 );
1765 [ $rcode, , $rhdrs, $rbody, ] = $this->http->run( [
1767 'url' =>
"{$this->swiftAuthUrl}/v1.0",
1769 'x-auth-user' => $this->swiftUser,
1770 'x-auth-key' => $this->swiftKey
1772 ], self::DEFAULT_HTTP_OPTIONS );
1774 if ( $rcode >= 200 && $rcode <= 299 ) {
1775 $this->authCreds = [
1776 'auth_token' => $rhdrs[
'x-auth-token'],
1777 'storage_url' => $this->swiftStorageUrl ?? $rhdrs[
'x-storage-url']
1780 $this->srvCache->set( $cacheKey, $this->authCreds, ceil( $this->authTTL / 2 ) );
1781 $this->authSessionTimestamp = time();
1782 } elseif ( $rcode === 401 ) {
1783 $this->
onError(
null, __METHOD__, [],
"Authentication failed.", $rcode );
1784 $this->authErrorTimestamp = time();
1788 $this->
onError(
null, __METHOD__, [],
"HTTP return code: $rcode", $rcode, $rbody );
1789 $this->authErrorTimestamp = time();
1795 if ( str_ends_with( $this->authCreds[
'storage_url'],
'/v1' ) ) {
1796 $this->isRGW =
true;
1809 protected function storageUrl( array $creds, $container =
null, $object =
null ) {
1810 $parts = [ $creds[
'storage_url'] ];
1811 if ( strlen( $container ) ) {
1812 $parts[] = rawurlencode( $container );
1814 if ( strlen( $object ) ) {
1815 $parts[] = str_replace(
"%2F",
"/", rawurlencode( $object ) );
1818 return implode(
'/', $parts );
1826 return [
'x-auth-token' => $creds[
'auth_token'] ];
1835 private function getCredsCacheKey( $username ) {
1836 return 'swiftcredentials:' . md5( $username .
':' . $this->swiftAuthUrl );
1851 public function onError( $status, $func, array $params, $err =
'', $code = 0, $desc =
'', $body =
'' ) {
1853 $status->fatal(
'backend-fail-internal', $this->name );
1855 if ( $code == 401 ) {
1856 $this->srvCache->delete( $this->getCredsCacheKey( $this->swiftUser ) );
1858 $msg =
"HTTP {code} ({desc}) in '{func}' (given '{req_params}')";
1867 $msgParams[
'err'] = $err;
1869 if ( $code == 502 ) {
1870 $msg .=
' ({truncatedBody})';
1871 $msgParams[
'truncatedBody'] = substr( strip_tags( $body ), 0, 100 );
1873 $this->logger->error( $msg, $msgParams );
A BagOStuff object with no objects in it.
File backend exception for checked exceptions (e.g.
Base class for all backends using particular storage medium.
setContainerCache( $container, array $val)
Set the cached info for a container.
executeOpHandlesInternal(array $fileOpHandles)
Execute a list of FileBackendStoreOpHandle handles in parallel.
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
deleteFileCache( $path)
Delete the cached stat info for a file path.
getContentType( $storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
static false $RES_ABSENT
Idiom for "no result due to missing file" (since 1.34)
static null $RES_ERROR
Idiom for "no result due to I/O errors" (since 1.34)
fileExists(array $params)
Check if a file exists at a storage path in the backend.
getLocalCopy(array $params)
Get a local copy on disk of the file at a storage path in the backend.
string $name
Unique backend name.
static extensionFromPath( $path, $case='lowercase')
Get the final extension from a storage or FS path.
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
scopedProfileSection( $section)
newStatus(... $args)
Yields the result of the status wrapper callback on either:
static send404Message( $fname, $flags=0)
Send out a standard 404 message for a file.
Store key-value entries in a size-limited in-memory LRU cache.
Class to handle multiple HTTP requests.
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Iterator for listing directories.
Iterator for listing regular files.
Class for an OpenStack Swift (or Ceph RGW) based file backend.
string $swiftUser
Swift user (account:user) to authenticate as.
string $swiftAuthUrl
Authentication base URL (without version)
string $swiftTempUrlKey
Shared secret value for making temp URLs.
MapCacheLRU $containerStatCache
Container stat cache.
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path in the backend.
getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
doPublishInternal( $fullCont, $dir, array $params)
doCreateInternal(array $params)
doGetFileStatMulti(array $params)
Get file stat information (concurrently if possible) for several files.
doGetFileSha1base36(array $params)
int null $authErrorTimestamp
UNIX timestamp.
array $writeUsers
Additional users (account:user) with write permissions on public containers.
__construct(array $config)
doGetFileXAttributes(array $params)
authTokenHeaders(array $creds)
getStatFromHeaders(array $rhdrs)
string $swiftStorageUrl
Override of storage base URL.
createContainer( $container, array $params)
Create a Swift container.
doCopyInternal(array $params)
getDirectoryListInternal( $fullCont, $dir, array $params)
string $rgwS3AccessKey
S3 access key (RADOS Gateway)
setContainerAccess( $container, array $readUsers, array $writeUsers)
Set read/write permissions for a Swift container.
getFileHttpUrl(array $params)
array $secureWriteUsers
Additional users (account:user) with write permissions on private containers.
extractMetadataHeaders(array $headers)
int $authTTL
TTL in seconds.
headersFromParams(array $params)
Get headers to send to Swift when reading a file based on a FileBackend params array,...
bool $isRGW
Whether the server is an Ceph RGW.
doStoreInternal(array $params)
int $authSessionTimestamp
UNIX timestamp.
onError( $status, $func, array $params, $err='', $code=0, $desc='', $body='')
Log an unexpected exception for this backend.
loadListingStatInternal( $path, array $val)
Do not call this function outside of SwiftFileBackendFileList.
doPrepareInternal( $fullCont, $dir, array $params)
FileBackendStore::doPrepare() Stability: stableto override StatusValue Good status without value for ...
setLogger(LoggerInterface $logger)
doSecureInternal( $fullCont, $dir, array $params)
getFileListInternal( $fullCont, $dir, array $params)
getMetadataFromHeaders(array $headers)
doMoveInternal(array $params)
addMissingHashMetadata(array $objHdrs, $path)
Fill in any missing object metadata and save it to Swift.
getFeatures()
Get the a bitfield of extra features supported by the backend medium.
deleteContainer( $container, array $params)
Delete a Swift container.
doGetFileStat(array $params)
doGetLocalCopyMulti(array $params)
string $rgwS3SecretKey
S3 authentication key (RADOS Gateway)
doGetFileContentsMulti(array $params)
storageUrl(array $creds, $container=null, $object=null)
convertSwiftDate( $ts, $format=TS_MW)
Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT"/"2013-05-11T07:37:27.678360Z".
doStreamFile(array $params)
doPrimeContainerCache(array $containerInfo)
Fill the backend-specific process cache given an array of resolved container names and their correspo...
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
array $readUsers
Additional users (account:user) with read permissions on public containers.
array $secureReadUsers
Additional users (account:user) with read permissions on private containers.
doCleanInternal( $fullCont, $dir, array $params)
getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
string $swiftKey
Secret key for user.
doDirectoryExists( $fullCont, $dir, array $params)
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
doExecuteOpHandlesInternal(array $fileOpHandles)
doDeleteInternal(array $params)
doDescribeInternal(array $params)
extractMutableContentHeaders(array $headers)
Filter/normalize a header map to only include mutable "content-"/"x-content-" headers.
getContainerStat( $container, $bypassCache=false)
Get a Swift container stat map, possibly from process cache.
Multi-datacenter aware caching interface.
if(!is_readable( $file)) $ext