26use Psr\Log\LoggerInterface;
27use Wikimedia\AtEase\AtEase;
28use Wikimedia\RequestTimeout\TimeoutException;
40 private const DEFAULT_HTTP_OPTIONS = [
'httpVersion' =>
'v1.1' ];
41 private const AUTH_FAILURE_ERROR =
'Could not connect due to prior authentication failure';
123 parent::__construct( $config );
125 $this->swiftAuthUrl = $config[
'swiftAuthUrl'];
126 $this->swiftUser = $config[
'swiftUser'];
127 $this->swiftKey = $config[
'swiftKey'];
129 $this->authTTL = $config[
'swiftAuthTTL'] ?? 15 * 60;
130 $this->swiftTempUrlKey = $config[
'swiftTempUrlKey'] ??
'';
131 $this->swiftStorageUrl = $config[
'swiftStorageUrl'] ??
null;
132 $this->shardViaHashLevels = $config[
'shardViaHashLevels'] ??
'';
133 $this->rgwS3AccessKey = $config[
'rgwS3AccessKey'] ??
'';
134 $this->rgwS3SecretKey = $config[
'rgwS3SecretKey'] ??
'';
138 foreach ( [
'connTimeout',
'reqTimeout' ] as $optionName ) {
139 if ( isset( $config[$optionName] ) ) {
140 $httpOptions[$optionName] = $config[$optionName];
144 $this->http->setLogger( $this->logger );
147 if ( isset( $config[
'wanCache'] ) && $config[
'wanCache'] instanceof
WANObjectCache ) {
148 $this->memCache = $config[
'wanCache'];
151 $this->containerStatCache =
new MapCacheLRU( 300 );
153 if ( !empty( $config[
'cacheAuthInfo'] ) && isset( $config[
'srvCache'] ) ) {
154 $this->srvCache = $config[
'srvCache'];
158 $this->readUsers = $config[
'readUsers'] ?? [];
159 $this->writeUsers = $config[
'writeUsers'] ?? [];
160 $this->secureReadUsers = $config[
'secureReadUsers'] ?? [];
161 $this->secureWriteUsers = $config[
'secureWriteUsers'] ?? [];
166 $this->maxFileSize = 5 * 1024 * 1024 * 1024;
171 $this->http->setLogger(
$logger );
176 self::ATTR_UNICODE_PATHS |
183 if ( !mb_check_encoding( $relStoragePath,
'UTF-8' ) ) {
185 } elseif ( strlen( rawurlencode( $relStoragePath ) ) > 1024 ) {
189 return $relStoragePath;
194 if ( $rel ===
null ) {
211 $contentHeaders = [];
213 foreach ( $headers as
$name => $value ) {
215 if (
$name ===
'x-delete-at' && is_numeric( $value ) ) {
217 $contentHeaders[
$name] = $value;
218 } elseif (
$name ===
'x-delete-after' && is_numeric( $value ) ) {
220 $contentHeaders[
$name] = $value;
221 } elseif ( preg_match(
'/^(x-)?content-(?!length$)/',
$name ) ) {
223 $contentHeaders[
$name] = $value;
224 } elseif (
$name ===
'content-type' && strlen( $value ) ) {
226 $contentHeaders[
$name] = $value;
230 if ( isset( $contentHeaders[
'content-disposition'] ) ) {
233 $offset = $maxLength - strlen( $contentHeaders[
'content-disposition'] );
235 $pos = strrpos( $contentHeaders[
'content-disposition'],
';', $offset );
236 $contentHeaders[
'content-disposition'] = $pos ===
false
238 : trim( substr( $contentHeaders[
'content-disposition'], 0, $pos ) );
242 return $contentHeaders;
251 $metadataHeaders = [];
252 foreach ( $headers as
$name => $value ) {
254 if ( strpos(
$name,
'x-object-meta-' ) === 0 ) {
255 $metadataHeaders[
$name] = $value;
259 return $metadataHeaders;
268 $prefixLen = strlen(
'x-object-meta-' );
272 $metadata[substr(
$name, $prefixLen )] = $value;
282 if ( $dstRel ===
null ) {
283 $status->fatal(
'backend-fail-invalidpath',
$params[
'dst'] );
291 $mutableHeaders[
'content-type']
296 'container' => $dstCont,
297 'relPath' => $dstRel,
298 'headers' => array_merge(
301 'etag' => md5(
$params[
'content'] ),
302 'content-length' => strlen(
$params[
'content'] ),
303 'x-object-meta-sha1base36' =>
310 $method = __METHOD__;
311 $handler =
function ( array $request,
StatusValue $status ) use ( $method,
$params ) {
312 [ $rcode, $rdesc, , $rbody, $rerr ] = $request[
'response'];
313 if ( $rcode === 201 || $rcode === 202 ) {
315 } elseif ( $rcode === 412 ) {
316 $status->fatal(
'backend-fail-contenttype',
$params[
'dst'] );
318 $this->
onError( $status, $method,
$params, $rerr, $rcode, $rdesc, $rbody );
321 return SwiftFileOpHandle::CONTINUE_IF_OK;
325 if ( !empty(
$params[
'async'] ) ) {
326 $status->value = $opHandle;
338 if ( $dstRel ===
null ) {
339 $status->fatal(
'backend-fail-invalidpath',
$params[
'dst'] );
347 AtEase::suppressWarnings();
348 $srcHandle = fopen(
$params[
'src'],
'rb' );
349 AtEase::restoreWarnings();
350 if ( $srcHandle ===
false ) {
351 $status->fatal(
'backend-fail-notexists',
$params[
'src'] );
357 $srcSize = fstat( $srcHandle )[
'size'];
358 $md5Context = hash_init(
'md5' );
359 $sha1Context = hash_init(
'sha1' );
361 while ( !feof( $srcHandle ) ) {
362 $buffer = (string)fread( $srcHandle, 131_072 );
363 hash_update( $md5Context, $buffer );
364 hash_update( $sha1Context, $buffer );
365 $hashDigestSize += strlen( $buffer );
368 rewind( $srcHandle );
370 if ( $hashDigestSize !== $srcSize ) {
371 $status->fatal(
'backend-fail-hash',
$params[
'src'] );
379 $mutableHeaders[
'content-type']
384 'container' => $dstCont,
385 'relPath' => $dstRel,
386 'headers' => array_merge(
389 'content-length' => $srcSize,
390 'etag' => hash_final( $md5Context ),
391 'x-object-meta-sha1base36' =>
392 Wikimedia\base_convert( hash_final( $sha1Context ), 16, 36, 31 )
398 $method = __METHOD__;
399 $handler =
function ( array $request,
StatusValue $status ) use ( $method,
$params ) {
400 [ $rcode, $rdesc, , $rbody, $rerr ] = $request[
'response'];
401 if ( $rcode === 201 || $rcode === 202 ) {
403 } elseif ( $rcode === 412 ) {
404 $status->fatal(
'backend-fail-contenttype',
$params[
'dst'] );
406 $this->
onError( $status, $method,
$params, $rerr, $rcode, $rdesc, $rbody );
409 return SwiftFileOpHandle::CONTINUE_IF_OK;
413 $opHandle->resourcesToClose[] = $srcHandle;
415 if ( !empty(
$params[
'async'] ) ) {
416 $status->value = $opHandle;
428 if ( $srcRel ===
null ) {
429 $status->fatal(
'backend-fail-invalidpath',
$params[
'src'] );
435 if ( $dstRel ===
null ) {
436 $status->fatal(
'backend-fail-invalidpath',
$params[
'dst'] );
443 'container' => $dstCont,
444 'relPath' => $dstRel,
445 'headers' => array_merge(
448 'x-copy-from' =>
'/' . rawurlencode( $srcCont ) .
'/' .
449 str_replace(
"%2F",
"/", rawurlencode( $srcRel ) )
454 $method = __METHOD__;
455 $handler =
function ( array $request,
StatusValue $status ) use ( $method,
$params ) {
456 [ $rcode, $rdesc, , $rbody, $rerr ] = $request[
'response'];
457 if ( $rcode === 201 ) {
459 } elseif ( $rcode === 404 ) {
460 if ( empty(
$params[
'ignoreMissingSource'] ) ) {
461 $status->fatal(
'backend-fail-copy',
$params[
'src'],
$params[
'dst'] );
464 $this->
onError( $status, $method,
$params, $rerr, $rcode, $rdesc, $rbody );
467 return SwiftFileOpHandle::CONTINUE_IF_OK;
471 if ( !empty(
$params[
'async'] ) ) {
472 $status->value = $opHandle;
484 if ( $srcRel ===
null ) {
485 $status->fatal(
'backend-fail-invalidpath',
$params[
'src'] );
491 if ( $dstRel ===
null ) {
492 $status->fatal(
'backend-fail-invalidpath',
$params[
'dst'] );
499 'container' => $dstCont,
500 'relPath' => $dstRel,
501 'headers' => array_merge(
504 'x-copy-from' =>
'/' . rawurlencode( $srcCont ) .
'/' .
505 str_replace(
"%2F",
"/", rawurlencode( $srcRel ) )
509 if (
"{$srcCont}/{$srcRel}" !==
"{$dstCont}/{$dstRel}" ) {
511 'method' =>
'DELETE',
512 'container' => $srcCont,
513 'relPath' => $srcRel,
518 $method = __METHOD__;
519 $handler =
function ( array $request,
StatusValue $status ) use ( $method,
$params ) {
520 [ $rcode, $rdesc, , $rbody, $rerr ] = $request[
'response'];
521 if ( $request[
'method'] ===
'PUT' && $rcode === 201 ) {
523 } elseif ( $request[
'method'] ===
'DELETE' && $rcode === 204 ) {
525 } elseif ( $rcode === 404 ) {
526 if ( empty(
$params[
'ignoreMissingSource'] ) ) {
527 $status->fatal(
'backend-fail-move',
$params[
'src'],
$params[
'dst'] );
530 return SwiftFileOpHandle::CONTINUE_NO;
533 $this->
onError( $status, $method,
$params, $rerr, $rcode, $rdesc, $rbody );
536 return SwiftFileOpHandle::CONTINUE_IF_OK;
540 if ( !empty(
$params[
'async'] ) ) {
541 $status->value = $opHandle;
553 if ( $srcRel ===
null ) {
554 $status->fatal(
'backend-fail-invalidpath',
$params[
'src'] );
560 'method' =>
'DELETE',
561 'container' => $srcCont,
562 'relPath' => $srcRel,
566 $method = __METHOD__;
567 $handler =
function ( array $request,
StatusValue $status ) use ( $method,
$params ) {
568 [ $rcode, $rdesc, , $rbody, $rerr ] = $request[
'response'];
569 if ( $rcode === 204 ) {
571 } elseif ( $rcode === 404 ) {
572 if ( empty(
$params[
'ignoreMissingSource'] ) ) {
573 $status->fatal(
'backend-fail-delete',
$params[
'src'] );
576 $this->
onError( $status, $method,
$params, $rerr, $rcode, $rdesc, $rbody );
579 return SwiftFileOpHandle::CONTINUE_IF_OK;
583 if ( !empty(
$params[
'async'] ) ) {
584 $status->value = $opHandle;
596 if ( $srcRel ===
null ) {
597 $status->fatal(
'backend-fail-invalidpath',
$params[
'src'] );
603 $stat = $this->
getFileStat( [
'src' => $params[
'src'],
'latest' => 1 ] );
604 if ( $stat && !isset( $stat[
'xattr'] ) ) {
605 $stat = $this->
doGetFileStat( [
'src' => $params[
'src'],
'latest' => 1 ] );
608 $status->fatal(
'backend-fail-describe',
$params[
'src'] );
616 $oldMetadataHeaders = [];
617 foreach ( $stat[
'xattr'][
'metadata'] as
$name => $value ) {
618 $oldMetadataHeaders[
"x-object-meta-$name"] = $value;
621 $oldContentHeaders = $stat[
'xattr'][
'headers'];
625 'container' => $srcCont,
626 'relPath' => $srcRel,
627 'headers' => $oldMetadataHeaders + $newContentHeaders + $oldContentHeaders
630 $method = __METHOD__;
631 $handler =
function ( array $request,
StatusValue $status ) use ( $method,
$params ) {
632 [ $rcode, $rdesc, , $rbody, $rerr ] = $request[
'response'];
633 if ( $rcode === 202 ) {
635 } elseif ( $rcode === 404 ) {
636 $status->fatal(
'backend-fail-describe',
$params[
'src'] );
638 $this->
onError( $status, $method,
$params, $rerr, $rcode, $rdesc, $rbody );
643 if ( !empty(
$params[
'async'] ) ) {
644 $status->value = $opHandle;
660 if ( is_array( $stat ) ) {
662 } elseif ( $stat === self::RES_ERROR ) {
663 $status->fatal(
'backend-fail-internal', $this->name );
664 $this->logger->error( __METHOD__ .
': cannot get container stat' );
676 if ( empty(
$params[
'noAccess'] ) ) {
681 if ( is_array( $stat ) ) {
682 $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
683 $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
690 } elseif ( $stat === self::RES_ABSENT ) {
691 $status->fatal(
'backend-fail-usable',
$params[
'dir'] );
693 $status->fatal(
'backend-fail-internal', $this->name );
694 $this->logger->error( __METHOD__ .
': cannot get container stat' );
702 if ( empty(
$params[
'access'] ) ) {
707 if ( is_array( $stat ) ) {
708 $readUsers = array_merge( $this->readUsers, [ $this->swiftUser,
'.r:*' ] );
709 if ( !empty(
$params[
'listing'] ) ) {
712 $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
720 } elseif ( $stat === self::RES_ABSENT ) {
721 $status->fatal(
'backend-fail-usable',
$params[
'dir'] );
723 $status->fatal(
'backend-fail-internal', $this->name );
724 $this->logger->error( __METHOD__ .
': cannot get container stat' );
740 if ( $stat === self::RES_ABSENT ) {
742 } elseif ( $stat === self::RES_ERROR ) {
743 $status->fatal(
'backend-fail-internal', $this->name );
744 $this->logger->error( __METHOD__ .
': cannot get container stat' );
745 } elseif ( is_array( $stat ) && $stat[
'count'] == 0 ) {
759 return reset( $stats );
776 return $timestamp->getTimestamp( $format );
777 }
catch ( TimeoutException $e ) {
779 }
catch ( Exception $e ) {
792 if ( isset( $objHdrs[
'x-object-meta-sha1base36'] ) ) {
798 $this->logger->error( __METHOD__ .
": {path} was not stored with SHA-1 metadata.",
799 [
'path' =>
$path ] );
801 $objHdrs[
'x-object-meta-sha1base36'] =
false;
811 if ( $status->isOK() ) {
814 $hash = $tmpFile->getSha1Base36();
815 if ( $hash !==
false ) {
816 $objHdrs[
'x-object-meta-sha1base36'] = $hash;
818 $postHeaders[
'x-object-meta-sha1base36'] = $hash;
820 [ $rcode ] = $this->requestWithAuth( [
822 'container' => $srcCont,
823 'relPath' => $srcRel,
824 'headers' => $postHeaders
826 if ( $rcode >= 200 && $rcode <= 299 ) {
835 $this->logger->error( __METHOD__ .
': unable to set SHA-1 metadata for {path}',
836 [
'path' =>
$path ] );
842 $ep = array_diff_key(
$params, [
'srcs' => 1 ] );
848 $contents = array_fill_keys(
$params[
'srcs'], self::RES_ERROR );
851 if ( $srcRel ===
null ) {
855 $handle = fopen(
'php://temp',
'wb' );
859 'container' => $srcCont,
860 'relPath' => $srcRel,
867 $reqs = $this->requestMultiWithAuth(
869 [
'maxConnsPerHost' =>
$params[
'concurrency'] ]
871 foreach ( $reqs as
$path => $op ) {
872 [ $rcode, $rdesc, $rhdrs, $rbody, $rerr ] = $op[
'response'];
873 if ( $rcode >= 200 && $rcode <= 299 ) {
874 rewind( $op[
'stream'] );
875 $content = (string)stream_get_contents( $op[
'stream'] );
876 $size = strlen( $content );
878 if ( $size === (
int)$rhdrs[
'content-length'] ) {
879 $contents[
$path] = $content;
881 $contents[
$path] = self::RES_ERROR;
882 $rerr =
"Got {$size}/{$rhdrs['content-length']} bytes";
883 $this->
onError(
null, __METHOD__,
884 [
'src' =>
$path ] + $ep, $rerr, $rcode, $rdesc );
886 } elseif ( $rcode === 404 ) {
887 $contents[
$path] = self::RES_ABSENT;
889 $contents[
$path] = self::RES_ERROR;
890 $this->
onError(
null, __METHOD__,
891 [
'src' =>
$path ] + $ep, $rerr, $rcode, $rdesc, $rbody );
893 fclose( $op[
'stream'] );
900 $prefix = ( $dir ==
'' ) ?
null :
"{$dir}/";
901 $status = $this->objectListing( $fullCont,
'names', 1,
null, $prefix );
902 if ( $status->isOK() ) {
903 return ( count( $status->value ) ) > 0;
906 return self::RES_ERROR;
944 if ( $after === INF ) {
951 $prefix = ( $dir ==
'' ) ?
null :
"{$dir}/";
953 if ( !empty(
$params[
'topOnly'] ) ) {
954 $status = $this->objectListing( $fullCont,
'names', $limit, $after, $prefix,
'/' );
955 if ( !$status->isOK() ) {
958 $objects = $status->value;
960 foreach ( $objects as $object ) {
961 if ( substr( $object, -1 ) ===
'/' ) {
967 $getParentDir =
static function (
$path ) {
968 return (
$path !==
null && strpos(
$path,
'/' ) !== false ) ? dirname(
$path ) :
false;
972 $lastDir = $getParentDir( $after );
973 $status = $this->objectListing( $fullCont,
'names', $limit, $after, $prefix );
975 if ( !$status->isOK() ) {
979 $objects = $status->value;
982 foreach ( $objects as $object ) {
983 $objectDir = $getParentDir( $object );
985 if ( $objectDir !==
false && $objectDir !== $dir ) {
990 if ( strcmp( $objectDir, $lastDir ) > 0 ) {
993 $dirs[] =
"{$pDir}/";
994 $pDir = $getParentDir( $pDir );
995 }
while ( $pDir !==
false
996 && strcmp( $pDir, $lastDir ) > 0
997 && strlen( $pDir ) > strlen( $dir )
1000 $lastDir = $objectDir;
1005 if ( count( $objects ) < $limit ) {
1008 $after = end( $objects );
1027 if ( $after === INF ) {
1034 $prefix = ( $dir ==
'' ) ?
null :
"{$dir}/";
1037 if ( !empty(
$params[
'topOnly'] ) ) {
1038 if ( !empty(
$params[
'adviseStat'] ) ) {
1039 $status = $this->objectListing( $fullCont,
'info', $limit, $after, $prefix,
'/' );
1041 $status = $this->objectListing( $fullCont,
'names', $limit, $after, $prefix,
'/' );
1045 if ( !empty(
$params[
'adviseStat'] ) ) {
1046 $status = $this->objectListing( $fullCont,
'info', $limit, $after, $prefix );
1048 $status = $this->objectListing( $fullCont,
'names', $limit, $after, $prefix );
1053 if ( !$status->isOK() ) {
1057 $objects = $status->value;
1058 $files = $this->buildFileObjectListing( $objects );
1061 if ( count( $objects ) < $limit ) {
1064 $after = end( $objects );
1065 $after = is_object( $after ) ? $after->name : $after;
1078 private function buildFileObjectListing( array $objects ) {
1080 foreach ( $objects as $object ) {
1081 if ( is_object( $object ) ) {
1082 if ( isset( $object->subdir ) || !isset( $object->name ) ) {
1088 'size' => (int)$object->bytes,
1091 'md5' => ctype_xdigit( $object->hash ) ? $object->hash :
null,
1094 $names[] = [ $object->name, $stat ];
1095 } elseif ( substr( $object, -1 ) !==
'/' ) {
1097 $names[] = [ $object, null ];
1111 $this->cheapCache->setField(
$path,
'stat', $val );
1117 if ( is_array( $stat ) && !isset( $stat[
'xattr'] ) ) {
1122 if ( is_array( $stat ) ) {
1123 return $stat[
'xattr'];
1126 return $stat === self::RES_ERROR ? self::RES_ERROR : self::RES_ABSENT;
1132 $params[
'requireSHA1'] =
true;
1135 if ( is_array( $stat ) ) {
1136 return $stat[
'sha1'];
1139 return $stat === self::RES_ERROR ? self::RES_ERROR : self::RES_ABSENT;
1145 $flags = !empty(
$params[
'headless'] ) ? HTTPFileStreamer::STREAM_HEADLESS : 0;
1148 if ( $srcRel ===
null ) {
1149 HTTPFileStreamer::send404Message(
$params[
'src'], $flags );
1150 $status->fatal(
'backend-fail-invalidpath',
$params[
'src'] );
1156 HTTPFileStreamer::send404Message(
$params[
'src'], $flags );
1157 $status->fatal(
'backend-fail-stream',
$params[
'src'] );
1165 HTTPFileStreamer::send404Message(
$params[
'src'], $flags );
1166 $status->fatal(
'backend-fail-stream',
$params[
'src'] );
1172 if ( empty(
$params[
'headless'] ) ) {
1178 if ( empty(
$params[
'allowOB'] ) ) {
1183 $handle = fopen(
'php://output',
'wb' );
1184 [ $rcode, $rdesc, , $rbody, $rerr ] = $this->requestWithAuth( [
1186 'container' => $srcCont,
1187 'relPath' => $srcRel,
1189 'stream' => $handle,
1190 'flags' => [
'relayResponseHeaders' => empty(
$params[
'headless'] ) ]
1193 if ( $rcode >= 200 && $rcode <= 299 ) {
1195 } elseif ( $rcode === 404 ) {
1196 $status->fatal(
'backend-fail-stream',
$params[
'src'] );
1203 $this->
onError( $status, __METHOD__,
$params, $rerr, $rcode, $rdesc, $rbody );
1210 $ep = array_diff_key(
$params, [
'srcs' => 1 ] );
1216 $tmpFiles = array_fill_keys(
$params[
'srcs'], self::RES_ERROR );
1219 if ( $srcRel ===
null ) {
1225 $tmpFile = $this->tmpFileFactory->newTempFSFile(
'localcopy_', $ext );
1226 $handle = $tmpFile ? fopen( $tmpFile->getPath(),
'wb' ) :
false;
1230 'container' => $srcCont,
1231 'relPath' => $srcRel,
1233 'stream' => $handle,
1235 $tmpFiles[
$path] = $tmpFile;
1240 $latest = ( $this->isRGW || !empty(
$params[
'latest'] ) );
1242 $reqs = $this->requestMultiWithAuth(
1244 [
'maxConnsPerHost' =>
$params[
'concurrency'] ]
1246 foreach ( $reqs as
$path => $op ) {
1247 [ $rcode, $rdesc, $rhdrs, $rbody, $rerr ] = $op[
'response'];
1248 fclose( $op[
'stream'] );
1249 if ( $rcode >= 200 && $rcode <= 299 ) {
1251 $tmpFile = $tmpFiles[
$path];
1253 $size = $tmpFile->getSize();
1254 if ( $size !== (
int)$rhdrs[
'content-length'] ) {
1255 $tmpFiles[
$path] = self::RES_ERROR;
1256 $rerr =
"Got {$size}/{$rhdrs['content-length']} bytes";
1257 $this->
onError(
null, __METHOD__,
1258 [
'src' =>
$path ] + $ep, $rerr, $rcode, $rdesc );
1262 $stat[
'latest'] = $latest;
1263 $this->cheapCache->setField(
$path,
'stat', $stat );
1264 } elseif ( $rcode === 404 ) {
1265 $tmpFiles[
$path] = self::RES_ABSENT;
1266 $this->cheapCache->setField(
1269 $latest ? self::ABSENT_LATEST : self::ABSENT_NORMAL
1272 $tmpFiles[
$path] = self::RES_ERROR;
1273 $this->
onError(
null, __METHOD__,
1274 [
'src' =>
$path ] + $ep, $rerr, $rcode, $rdesc, $rbody );
1282 if ( $this->swiftTempUrlKey !=
'' ||
1283 ( $this->rgwS3AccessKey !=
'' && $this->rgwS3SecretKey !=
'' )
1286 if ( $srcRel ===
null ) {
1287 return self::TEMPURL_ERROR;
1292 return self::TEMPURL_ERROR;
1295 $ttl =
$params[
'ttl'] ?? 86400;
1296 $expires = time() + $ttl;
1298 if ( $this->swiftTempUrlKey !=
'' ) {
1299 $url = $this->
storageUrl( $auth, $srcCont, $srcRel );
1301 $contPath = parse_url( $this->
storageUrl( $auth, $srcCont ), PHP_URL_PATH );
1302 $signature = hash_hmac(
'sha1',
1303 "GET\n{$expires}\n{$contPath}/{$srcRel}",
1304 $this->swiftTempUrlKey
1307 return "{$url}?temp_url_sig={$signature}&temp_url_expires={$expires}";
1310 $spath =
'/' . rawurlencode( $srcCont ) .
'/' .
1311 str_replace(
'%2F',
'/', rawurlencode( $srcRel ) );
1313 $signature = base64_encode( hash_hmac(
1315 "GET\n\n\n{$expires}\n{$spath}",
1316 $this->rgwS3SecretKey,
1322 return str_replace(
'/swift/v1',
'', $this->
storageUrl( $auth ) . $spath ) .
1325 'Signature' => $signature,
1326 'Expires' => $expires,
1327 'AWSAccessKeyId' => $this->rgwS3AccessKey
1332 return self::TEMPURL_ERROR;
1349 if ( !empty(
$params[
'latest'] ) ) {
1350 $hdrs[
'x-newest'] =
'true';
1358 '@phan-var SwiftFileOpHandle[] $fileOpHandles';
1364 $httpReqsByStage = [];
1365 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1366 $reqs = $fileOpHandle->httpOp;
1367 foreach ( $reqs as $stage => $req ) {
1368 $httpReqsByStage[$stage][$index] = $req;
1374 $reqCount = count( $httpReqsByStage );
1375 for ( $stage = 0; $stage < $reqCount; ++$stage ) {
1376 $httpReqs = $this->requestMultiWithAuth( $httpReqsByStage[$stage] );
1377 foreach ( $httpReqs as $index => $httpReq ) {
1379 $fileOpHandle = $fileOpHandles[$index];
1381 $status = $statuses[$index];
1382 ( $fileOpHandle->callback )( $httpReq, $status );
1387 $fileOpHandle->state === $fileOpHandle::CONTINUE_NO
1389 $stages = count( $fileOpHandle->httpOp );
1390 for ( $s = ( $stage + 1 ); $s < $stages; ++$s ) {
1391 unset( $httpReqsByStage[$s][$index] );
1425 [ $rcode, , , , ] = $this->requestWithAuth( [
1427 'container' => $container,
1429 'x-container-read' => implode(
',',
$readUsers ),
1430 'x-container-write' => implode(
',',
$writeUsers )
1434 if ( $rcode != 204 && $rcode !== 202 ) {
1435 $status->fatal(
'backend-fail-internal', $this->name );
1436 $this->logger->error( __METHOD__ .
': unexpected rcode value ({rcode})',
1437 [
'rcode' => $rcode ] );
1455 if ( $bypassCache ) {
1456 $this->containerStatCache->clear( $container );
1457 } elseif ( !$this->containerStatCache->hasField( $container,
'stat' ) ) {
1460 if ( !$this->containerStatCache->hasField( $container,
'stat' ) ) {
1461 [ $rcode, $rdesc, $rhdrs, $rbody, $rerr ] = $this->requestWithAuth( [
1463 'container' => $container
1466 if ( $rcode === 204 ) {
1468 'count' => $rhdrs[
'x-container-object-count'],
1469 'bytes' => $rhdrs[
'x-container-bytes-used']
1471 if ( $bypassCache ) {
1474 $this->containerStatCache->setField( $container,
'stat', $stat );
1477 } elseif ( $rcode === 404 ) {
1478 return self::RES_ABSENT;
1480 $this->
onError(
null, __METHOD__,
1481 [
'cont' => $container ], $rerr, $rcode, $rdesc, $rbody );
1483 return self::RES_ERROR;
1487 return $this->containerStatCache->getField( $container,
'stat' );
1501 if ( empty(
$params[
'noAccess'] ) ) {
1503 $readUsers = array_merge( $this->readUsers, [
'.r:*', $this->swiftUser ] );
1504 if ( empty(
$params[
'noListing'] ) ) {
1507 $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
1510 $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
1511 $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
1514 [ $rcode, $rdesc, , $rbody, $rerr ] = $this->requestWithAuth( [
1516 'container' => $container,
1518 'x-container-read' => implode(
',',
$readUsers ),
1519 'x-container-write' => implode(
',',
$writeUsers )
1523 if ( $rcode === 201 ) {
1525 } elseif ( $rcode === 202 ) {
1528 $this->
onError( $status, __METHOD__,
$params, $rerr, $rcode, $rdesc, $rbody );
1544 [ $rcode, $rdesc, , $rbody, $rerr ] = $this->requestWithAuth( [
1545 'method' =>
'DELETE',
1546 'container' => $container
1549 if ( $rcode >= 200 && $rcode <= 299 ) {
1550 $this->containerStatCache->clear( $container );
1551 } elseif ( $rcode === 404 ) {
1553 } elseif ( $rcode === 409 ) {
1554 $this->
onError( $status, __METHOD__,
$params, $rerr, $rcode, $rdesc );
1556 $this->
onError( $status, __METHOD__,
$params, $rerr, $rcode, $rdesc, $rbody );
1574 private function objectListing(
1575 $fullCont, $type, $limit, $after =
null, $prefix =
null, $delim =
null
1579 $query = [
'limit' => $limit ];
1580 if ( $type ===
'info' ) {
1581 $query[
'format'] =
'json';
1583 if ( $after !==
null ) {
1584 $query[
'marker'] = $after;
1586 if ( $prefix !==
null ) {
1587 $query[
'prefix'] = $prefix;
1589 if ( $delim !==
null ) {
1590 $query[
'delimiter'] = $delim;
1593 [ $rcode, $rdesc, , $rbody, $rerr ] = $this->requestWithAuth( [
1595 'container' => $fullCont,
1599 $params = [
'cont' => $fullCont,
'prefix' => $prefix,
'delim' => $delim ];
1600 if ( $rcode === 200 ) {
1601 if ( $type ===
'info' ) {
1604 $status->value = explode(
"\n", trim( $rbody ) );
1606 } elseif ( $rcode === 204 ) {
1607 $status->value = [];
1608 } elseif ( $rcode === 404 ) {
1609 $status->value = [];
1611 $this->
onError( $status, __METHOD__,
$params, $rerr, $rcode, $rdesc, $rbody );
1618 foreach ( $containerInfo as $container => $info ) {
1619 $this->containerStatCache->setField( $container,
'stat', $info );
1630 if ( $srcRel ===
null ) {
1632 $stats[
$path] = self::RES_ERROR;
1637 if ( $cstat === self::RES_ABSENT ) {
1638 $stats[
$path] = self::RES_ABSENT;
1640 } elseif ( $cstat === self::RES_ERROR ) {
1641 $stats[
$path] = self::RES_ERROR;
1647 'container' => $srcCont,
1648 'relPath' => $srcRel,
1654 $reqs = $this->requestMultiWithAuth(
1656 [
'maxConnsPerHost' =>
$params[
'concurrency'] ]
1658 foreach ( $reqs as
$path => $op ) {
1659 [ $rcode, $rdesc, $rhdrs, $rbody, $rerr ] = $op[
'response'];
1660 if ( $rcode === 200 || $rcode === 204 ) {
1662 if ( !empty(
$params[
'requireSHA1'] ) ) {
1667 if ( $this->isRGW ) {
1668 $stat[
'latest'] =
true;
1670 } elseif ( $rcode === 404 ) {
1671 $stat = self::RES_ABSENT;
1673 $stat = self::RES_ERROR;
1674 $this->
onError(
null, __METHOD__,
$params, $rerr, $rcode, $rdesc, $rbody );
1676 $stats[
$path] = $stat;
1696 'size' => isset( $rhdrs[
'content-length'] ) ? (int)$rhdrs[
'content-length'] : 0,
1697 'sha1' => $metadata[
'sha1base36'] ??
null,
1699 'md5' => ctype_xdigit( $rhdrs[
'etag'] ) ? $rhdrs[
'etag'] :
null,
1700 'xattr' => [
'metadata' => $metadata,
'headers' => $headers ]
1710 if ( $this->authErrorTimestamp !==
null ) {
1712 if ( $interval < 60 ) {
1713 $this->logger->debug(
1714 'rejecting request since auth failure occurred {interval} seconds ago',
1715 [
'interval' => $interval ]
1719 $this->authErrorTimestamp =
null;
1723 if ( !$this->authCreds ) {
1724 $cacheKey = $this->getCredsCacheKey( $this->swiftUser );
1725 $creds = $this->srvCache->get( $cacheKey );
1727 if ( isset( $creds[
'auth_token'] )
1728 && isset( $creds[
'storage_url'] )
1729 && isset( $creds[
'expiry_time'] )
1730 && $creds[
'expiry_time'] > time()
1732 $this->setAuthCreds( $creds );
1734 $this->refreshAuthentication();
1746 private function setAuthCreds( ?array $creds ) {
1747 $this->logger->debug(
'Using auth token with expiry_time={expiry_time}',
1749 'expiry_time' => isset( $creds[
'expiry_time'] )
1750 ? gmdate(
'c', $creds[
'expiry_time'] ) :
'null'
1753 $this->authCreds = $creds;
1755 if ( $creds && str_ends_with( $creds[
'storage_url'],
'/v1' ) ) {
1756 $this->isRGW =
true;
1765 private function refreshAuthentication() {
1766 [ $rcode, , $rhdrs, $rbody, ] = $this->http->run( [
1768 'url' =>
"{$this->swiftAuthUrl}/v1.0",
1770 'x-auth-user' => $this->swiftUser,
1771 'x-auth-key' => $this->swiftKey
1773 ], self::DEFAULT_HTTP_OPTIONS );
1775 if ( $rcode >= 200 && $rcode <= 299 ) {
1776 if ( isset( $rhdrs[
'x-auth-token-expires'] ) ) {
1777 $ttl = intval( $rhdrs[
'x-auth-token-expires'] );
1781 $expiryTime = time() + $ttl;
1783 'auth_token' => $rhdrs[
'x-auth-token'],
1784 'storage_url' => $this->swiftStorageUrl ?? $rhdrs[
'x-storage-url'],
1785 'expiry_time' => $expiryTime,
1787 $this->srvCache->set( $this->getCredsCacheKey( $this->swiftUser ), $creds, $expiryTime );
1788 } elseif ( $rcode === 401 ) {
1789 $this->
onError(
null, __METHOD__, [],
"Authentication failed.", $rcode );
1790 $this->authErrorTimestamp = time();
1793 $this->
onError(
null, __METHOD__, [],
"HTTP return code: $rcode", $rcode, $rbody );
1794 $this->authErrorTimestamp = time();
1797 $this->setAuthCreds( $creds );
1807 protected function storageUrl( array $creds, $container =
null, $object =
null ) {
1808 $parts = [ $creds[
'storage_url'] ];
1809 if ( strlen( $container ??
'' ) ) {
1810 $parts[] = rawurlencode( $container );
1812 if ( strlen( $object ??
'' ) ) {
1813 $parts[] = str_replace(
"%2F",
"/", rawurlencode( $object ) );
1816 return implode(
'/', $parts );
1824 return [
'x-auth-token' => $creds[
'auth_token'] ];
1833 private function getCredsCacheKey( $username ) {
1834 return 'swiftcredentials:' . md5( $username .
':' . $this->swiftAuthUrl );
1851 private function requestWithAuth( array $req, array $options = [] ) {
1852 return $this->requestMultiWithAuth( [ $req ], $options )[0][
'response'];
1864 private function requestMultiWithAuth( array $reqs, $options = [] ) {
1865 $remainingTries = 2;
1869 foreach ( $reqs as &$req ) {
1870 if ( !isset( $req[
'response'] ) ) {
1871 $req[
'response'] = $this->getAuthFailureResponse();
1876 foreach ( $reqs as &$req ) {
1877 '@phan-var array $req';
1878 if ( isset( $req[
'response'] ) ) {
1881 if ( $req[
'response'][
'code'] !== 401 ) {
1885 $req[
'headers'] = $this->
authTokenHeaders( $auth ) + ( $req[
'headers'] ?? [] );
1886 $req[
'url'] = $this->
storageUrl( $auth, $req[
'container'], $req[
'relPath'] ??
null );
1889 $reqs = $this->http->runMulti( $reqs, $options + self::DEFAULT_HTTP_OPTIONS );
1890 if ( --$remainingTries > 0 ) {
1892 foreach ( $reqs as $req ) {
1893 if ( $req[
'response'][
'code'] === 401 ) {
1894 $auth = $this->refreshAuthentication();
1912 private function getAuthFailureResponse() {
1922 'error' => self::AUTH_FAILURE_ERROR,
1923 4 => self::AUTH_FAILURE_ERROR
1934 private function isAuthFailureResponse( $code, $error ) {
1935 return $code === 0 && $error === self::AUTH_FAILURE_ERROR;
1950 public function onError( $status, $func, array
$params, $err =
'', $code = 0, $desc =
'', $body =
'' ) {
1951 if ( $this->isAuthFailureResponse( $code, $err ) ) {
1953 $status->fatal(
'backend-fail-connect', $this->name );
1959 $status->fatal(
'backend-fail-internal', $this->name );
1961 $msg =
"HTTP {code} ({desc}) in '{func}' (given '{req_params}')";
1966 'req_params' => FormatJson::encode(
$params ),
1970 $msgParams[
'err'] = $err;
1972 if ( $code == 502 ) {
1973 $msg .=
' ({truncatedBody})';
1974 $msgParams[
'truncatedBody'] = substr( strip_tags( $body ), 0, 100 );
1976 $this->logger->error( $msg, $msgParams );
array $params
The job parameters.
Class representing a cache/ephemeral data store.
A BagOStuff object with no objects in it.
File backend exception for checked exceptions (e.g.
Base class for all backends using particular storage medium.
setContainerCache( $container, array $val)
Set the cached info for a container.
executeOpHandlesInternal(array $fileOpHandles)
Execute a list of FileBackendStoreOpHandle handles in parallel.
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
deleteFileCache( $path)
Delete the cached stat info for a file path.
getContentType( $storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
fileExists(array $params)
Check if a file exists at a storage path in the backend.
getLocalCopy(array $params)
Get a local copy on disk of the file at a storage path in the backend.
string $name
Unique backend name.
static extensionFromPath( $path, $case='lowercase')
Get the final extension from a storage or FS path.
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
scopedProfileSection( $section)
newStatus(... $args)
Yields the result of the status wrapper callback on either:
Store key-value entries in a size-limited in-memory LRU cache.
Class to handle multiple HTTP requests.
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Iterator for listing directories.
Iterator for listing regular files.
Class for an OpenStack Swift (or Ceph RGW) based file backend.
string $swiftUser
Swift user (account:user) to authenticate as.
string $swiftAuthUrl
Authentication base URL (without version)
string $swiftTempUrlKey
Shared secret value for making temp URLs.
MapCacheLRU $containerStatCache
Container stat cache.
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path in the backend.
getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
doPublishInternal( $fullCont, $dir, array $params)
doCreateInternal(array $params)
doGetFileStatMulti(array $params)
Get file stat information (concurrently if possible) for several files.
doGetFileSha1base36(array $params)
int null $authErrorTimestamp
UNIX timestamp.
array $writeUsers
Additional users (account:user) with write permissions on public containers.
__construct(array $config)
doGetFileXAttributes(array $params)
authTokenHeaders(array $creds)
getStatFromHeaders(array $rhdrs)
string $swiftStorageUrl
Override of storage base URL.
createContainer( $container, array $params)
Create a Swift container.
doCopyInternal(array $params)
getDirectoryListInternal( $fullCont, $dir, array $params)
string $rgwS3AccessKey
S3 access key (RADOS Gateway)
setContainerAccess( $container, array $readUsers, array $writeUsers)
Set read/write permissions for a Swift container.
getFileHttpUrl(array $params)
array $secureWriteUsers
Additional users (account:user) with write permissions on private containers.
extractMetadataHeaders(array $headers)
int $authTTL
TTL in seconds.
headersFromParams(array $params)
Get headers to send to Swift when reading a file based on a FileBackend params array,...
bool $isRGW
Whether the server is an Ceph RGW.
doStoreInternal(array $params)
onError( $status, $func, array $params, $err='', $code=0, $desc='', $body='')
Log an unexpected exception for this backend.
loadListingStatInternal( $path, array $val)
Do not call this function outside of SwiftFileBackendFileList.
doPrepareInternal( $fullCont, $dir, array $params)
FileBackendStore::doPrepare() to override StatusValue Good status without value for success,...
setLogger(LoggerInterface $logger)
doSecureInternal( $fullCont, $dir, array $params)
getFileListInternal( $fullCont, $dir, array $params)
getMetadataFromHeaders(array $headers)
doMoveInternal(array $params)
addMissingHashMetadata(array $objHdrs, $path)
Fill in any missing object metadata and save it to Swift.
getFeatures()
Get the a bitfield of extra features supported by the backend medium.
deleteContainer( $container, array $params)
Delete a Swift container.
doGetFileStat(array $params)
getAuthentication()
Get the cached auth token.
doGetLocalCopyMulti(array $params)
string $rgwS3SecretKey
S3 authentication key (RADOS Gateway)
doGetFileContentsMulti(array $params)
storageUrl(array $creds, $container=null, $object=null)
convertSwiftDate( $ts, $format=TS_MW)
Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT"/"2013-05-11T07:37:27.678360Z".
doStreamFile(array $params)
doPrimeContainerCache(array $containerInfo)
Fill the backend-specific process cache given an array of resolved container names and their correspo...
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
array $readUsers
Additional users (account:user) with read permissions on public containers.
array $secureReadUsers
Additional users (account:user) with read permissions on private containers.
doCleanInternal( $fullCont, $dir, array $params)
getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
string $swiftKey
Secret key for user.
doDirectoryExists( $fullCont, $dir, array $params)
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
doExecuteOpHandlesInternal(array $fileOpHandles)
doDeleteInternal(array $params)
doDescribeInternal(array $params)
extractMutableContentHeaders(array $headers)
Filter/normalize a header map to only include mutable "content-"/"x-content-" headers.
getContainerStat( $container, $bypassCache=false)
Get a Swift container stat map, possibly from process cache.
Multi-datacenter aware caching interface.