27use Psr\Log\LoggerInterface;
28use Wikimedia\AtEase\AtEase;
33use Wikimedia\RequestTimeout\TimeoutException;
45 private const DEFAULT_HTTP_OPTIONS = [
'httpVersion' =>
'v1.1' ];
46 private const AUTH_FAILURE_ERROR =
'Could not connect due to prior authentication failure';
128 parent::__construct( $config );
130 $this->swiftAuthUrl = $config[
'swiftAuthUrl'];
131 $this->swiftUser = $config[
'swiftUser'];
132 $this->swiftKey = $config[
'swiftKey'];
134 $this->authTTL = $config[
'swiftAuthTTL'] ?? 15 * 60;
135 $this->swiftTempUrlKey = $config[
'swiftTempUrlKey'] ??
'';
136 $this->swiftStorageUrl = $config[
'swiftStorageUrl'] ??
null;
137 $this->shardViaHashLevels = $config[
'shardViaHashLevels'] ??
'';
138 $this->rgwS3AccessKey = $config[
'rgwS3AccessKey'] ??
'';
139 $this->rgwS3SecretKey = $config[
'rgwS3SecretKey'] ??
'';
143 foreach ( [
'connTimeout',
'reqTimeout' ] as $optionName ) {
144 if ( isset( $config[$optionName] ) ) {
145 $httpOptions[$optionName] = $config[$optionName];
149 $this->http->setLogger( $this->logger );
152 if ( isset( $config[
'wanCache'] ) && $config[
'wanCache'] instanceof
WANObjectCache ) {
153 $this->memCache = $config[
'wanCache'];
156 $this->containerStatCache =
new MapCacheLRU( 300 );
158 if ( !empty( $config[
'cacheAuthInfo'] ) && isset( $config[
'srvCache'] ) ) {
159 $this->srvCache = $config[
'srvCache'];
163 $this->readUsers = $config[
'readUsers'] ?? [];
164 $this->writeUsers = $config[
'writeUsers'] ?? [];
165 $this->secureReadUsers = $config[
'secureReadUsers'] ?? [];
166 $this->secureWriteUsers = $config[
'secureWriteUsers'] ?? [];
171 $this->maxFileSize = 5 * 1024 * 1024 * 1024;
176 $this->http->setLogger(
$logger );
181 self::ATTR_UNICODE_PATHS |
188 if ( !mb_check_encoding( $relStoragePath,
'UTF-8' ) ) {
190 } elseif ( strlen( rawurlencode( $relStoragePath ) ) > 1024 ) {
194 return $relStoragePath;
199 if ( $rel ===
null ) {
216 $contentHeaders = [];
218 foreach ( $headers as
$name => $value ) {
220 if (
$name ===
'x-delete-at' && is_numeric( $value ) ) {
222 $contentHeaders[
$name] = $value;
223 } elseif (
$name ===
'x-delete-after' && is_numeric( $value ) ) {
225 $contentHeaders[
$name] = $value;
226 } elseif ( preg_match(
'/^(x-)?content-(?!length$)/',
$name ) ) {
228 $contentHeaders[
$name] = $value;
229 } elseif (
$name ===
'content-type' && strlen( $value ) ) {
231 $contentHeaders[
$name] = $value;
235 if ( isset( $contentHeaders[
'content-disposition'] ) ) {
238 $offset = $maxLength - strlen( $contentHeaders[
'content-disposition'] );
240 $pos = strrpos( $contentHeaders[
'content-disposition'],
';', $offset );
241 $contentHeaders[
'content-disposition'] = $pos ===
false
243 : trim( substr( $contentHeaders[
'content-disposition'], 0, $pos ) );
247 return $contentHeaders;
256 $metadataHeaders = [];
257 foreach ( $headers as
$name => $value ) {
259 if ( strpos(
$name,
'x-object-meta-' ) === 0 ) {
260 $metadataHeaders[
$name] = $value;
264 return $metadataHeaders;
273 $prefixLen = strlen(
'x-object-meta-' );
277 $metadata[substr(
$name, $prefixLen )] = $value;
287 if ( $dstRel ===
null ) {
288 $status->fatal(
'backend-fail-invalidpath',
$params[
'dst'] );
296 $mutableHeaders[
'content-type']
301 'container' => $dstCont,
302 'relPath' => $dstRel,
303 'headers' => array_merge(
306 'etag' => md5(
$params[
'content'] ),
307 'content-length' => strlen(
$params[
'content'] ),
308 'x-object-meta-sha1base36' =>
315 $method = __METHOD__;
316 $handler =
function ( array $request,
StatusValue $status ) use ( $method,
$params ) {
317 [ $rcode, $rdesc, , $rbody, $rerr ] = $request[
'response'];
318 if ( $rcode === 201 || $rcode === 202 ) {
320 } elseif ( $rcode === 412 ) {
321 $status->fatal(
'backend-fail-contenttype',
$params[
'dst'] );
323 $this->
onError( $status, $method,
$params, $rerr, $rcode, $rdesc, $rbody );
326 return SwiftFileOpHandle::CONTINUE_IF_OK;
330 if ( !empty(
$params[
'async'] ) ) {
331 $status->value = $opHandle;
343 if ( $dstRel ===
null ) {
344 $status->fatal(
'backend-fail-invalidpath',
$params[
'dst'] );
352 AtEase::suppressWarnings();
353 $srcHandle = fopen(
$params[
'src'],
'rb' );
354 AtEase::restoreWarnings();
355 if ( $srcHandle ===
false ) {
356 $status->fatal(
'backend-fail-notexists',
$params[
'src'] );
362 $srcSize = fstat( $srcHandle )[
'size'];
363 $md5Context = hash_init(
'md5' );
364 $sha1Context = hash_init(
'sha1' );
366 while ( !feof( $srcHandle ) ) {
367 $buffer = (string)fread( $srcHandle, 131_072 );
368 hash_update( $md5Context, $buffer );
369 hash_update( $sha1Context, $buffer );
370 $hashDigestSize += strlen( $buffer );
373 rewind( $srcHandle );
375 if ( $hashDigestSize !== $srcSize ) {
376 $status->fatal(
'backend-fail-hash',
$params[
'src'] );
384 $mutableHeaders[
'content-type']
389 'container' => $dstCont,
390 'relPath' => $dstRel,
391 'headers' => array_merge(
394 'content-length' => $srcSize,
395 'etag' => hash_final( $md5Context ),
396 'x-object-meta-sha1base36' =>
397 Wikimedia\base_convert( hash_final( $sha1Context ), 16, 36, 31 )
403 $method = __METHOD__;
404 $handler =
function ( array $request,
StatusValue $status ) use ( $method,
$params ) {
405 [ $rcode, $rdesc, , $rbody, $rerr ] = $request[
'response'];
406 if ( $rcode === 201 || $rcode === 202 ) {
408 } elseif ( $rcode === 412 ) {
409 $status->fatal(
'backend-fail-contenttype',
$params[
'dst'] );
411 $this->
onError( $status, $method,
$params, $rerr, $rcode, $rdesc, $rbody );
414 return SwiftFileOpHandle::CONTINUE_IF_OK;
418 $opHandle->resourcesToClose[] = $srcHandle;
420 if ( !empty(
$params[
'async'] ) ) {
421 $status->value = $opHandle;
433 if ( $srcRel ===
null ) {
434 $status->fatal(
'backend-fail-invalidpath',
$params[
'src'] );
440 if ( $dstRel ===
null ) {
441 $status->fatal(
'backend-fail-invalidpath',
$params[
'dst'] );
448 'container' => $dstCont,
449 'relPath' => $dstRel,
450 'headers' => array_merge(
453 'x-copy-from' =>
'/' . rawurlencode( $srcCont ) .
'/' .
454 str_replace(
"%2F",
"/", rawurlencode( $srcRel ) )
459 $method = __METHOD__;
460 $handler =
function ( array $request,
StatusValue $status ) use ( $method,
$params ) {
461 [ $rcode, $rdesc, , $rbody, $rerr ] = $request[
'response'];
462 if ( $rcode === 201 ) {
464 } elseif ( $rcode === 404 ) {
465 if ( empty(
$params[
'ignoreMissingSource'] ) ) {
466 $status->fatal(
'backend-fail-copy',
$params[
'src'],
$params[
'dst'] );
469 $this->
onError( $status, $method,
$params, $rerr, $rcode, $rdesc, $rbody );
472 return SwiftFileOpHandle::CONTINUE_IF_OK;
476 if ( !empty(
$params[
'async'] ) ) {
477 $status->value = $opHandle;
489 if ( $srcRel ===
null ) {
490 $status->fatal(
'backend-fail-invalidpath',
$params[
'src'] );
496 if ( $dstRel ===
null ) {
497 $status->fatal(
'backend-fail-invalidpath',
$params[
'dst'] );
504 'container' => $dstCont,
505 'relPath' => $dstRel,
506 'headers' => array_merge(
509 'x-copy-from' =>
'/' . rawurlencode( $srcCont ) .
'/' .
510 str_replace(
"%2F",
"/", rawurlencode( $srcRel ) )
514 if (
"{$srcCont}/{$srcRel}" !==
"{$dstCont}/{$dstRel}" ) {
516 'method' =>
'DELETE',
517 'container' => $srcCont,
518 'relPath' => $srcRel,
523 $method = __METHOD__;
524 $handler =
function ( array $request,
StatusValue $status ) use ( $method,
$params ) {
525 [ $rcode, $rdesc, , $rbody, $rerr ] = $request[
'response'];
526 if ( $request[
'method'] ===
'PUT' && $rcode === 201 ) {
528 } elseif ( $request[
'method'] ===
'DELETE' && $rcode === 204 ) {
530 } elseif ( $rcode === 404 ) {
531 if ( empty(
$params[
'ignoreMissingSource'] ) ) {
532 $status->fatal(
'backend-fail-move',
$params[
'src'],
$params[
'dst'] );
535 return SwiftFileOpHandle::CONTINUE_NO;
538 $this->
onError( $status, $method,
$params, $rerr, $rcode, $rdesc, $rbody );
541 return SwiftFileOpHandle::CONTINUE_IF_OK;
545 if ( !empty(
$params[
'async'] ) ) {
546 $status->value = $opHandle;
558 if ( $srcRel ===
null ) {
559 $status->fatal(
'backend-fail-invalidpath',
$params[
'src'] );
565 'method' =>
'DELETE',
566 'container' => $srcCont,
567 'relPath' => $srcRel,
571 $method = __METHOD__;
572 $handler =
function ( array $request,
StatusValue $status ) use ( $method,
$params ) {
573 [ $rcode, $rdesc, , $rbody, $rerr ] = $request[
'response'];
574 if ( $rcode === 204 ) {
576 } elseif ( $rcode === 404 ) {
577 if ( empty(
$params[
'ignoreMissingSource'] ) ) {
578 $status->fatal(
'backend-fail-delete',
$params[
'src'] );
581 $this->
onError( $status, $method,
$params, $rerr, $rcode, $rdesc, $rbody );
584 return SwiftFileOpHandle::CONTINUE_IF_OK;
588 if ( !empty(
$params[
'async'] ) ) {
589 $status->value = $opHandle;
601 if ( $srcRel ===
null ) {
602 $status->fatal(
'backend-fail-invalidpath',
$params[
'src'] );
608 $stat = $this->
getFileStat( [
'src' => $params[
'src'],
'latest' => 1 ] );
609 if ( $stat && !isset( $stat[
'xattr'] ) ) {
610 $stat = $this->
doGetFileStat( [
'src' => $params[
'src'],
'latest' => 1 ] );
613 $status->fatal(
'backend-fail-describe',
$params[
'src'] );
621 $oldMetadataHeaders = [];
622 foreach ( $stat[
'xattr'][
'metadata'] as
$name => $value ) {
623 $oldMetadataHeaders[
"x-object-meta-$name"] = $value;
626 $oldContentHeaders = $stat[
'xattr'][
'headers'];
630 'container' => $srcCont,
631 'relPath' => $srcRel,
632 'headers' => $oldMetadataHeaders + $newContentHeaders + $oldContentHeaders
635 $method = __METHOD__;
636 $handler =
function ( array $request,
StatusValue $status ) use ( $method,
$params ) {
637 [ $rcode, $rdesc, , $rbody, $rerr ] = $request[
'response'];
638 if ( $rcode === 202 ) {
640 } elseif ( $rcode === 404 ) {
641 $status->fatal(
'backend-fail-describe',
$params[
'src'] );
643 $this->
onError( $status, $method,
$params, $rerr, $rcode, $rdesc, $rbody );
648 if ( !empty(
$params[
'async'] ) ) {
649 $status->value = $opHandle;
665 if ( is_array( $stat ) ) {
667 } elseif ( $stat === self::RES_ERROR ) {
668 $status->fatal(
'backend-fail-internal', $this->name );
669 $this->logger->error( __METHOD__ .
': cannot get container stat' );
681 if ( empty(
$params[
'noAccess'] ) ) {
686 if ( is_array( $stat ) ) {
687 $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
688 $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
695 } elseif ( $stat === self::RES_ABSENT ) {
696 $status->fatal(
'backend-fail-usable',
$params[
'dir'] );
698 $status->fatal(
'backend-fail-internal', $this->name );
699 $this->logger->error( __METHOD__ .
': cannot get container stat' );
707 if ( empty(
$params[
'access'] ) ) {
712 if ( is_array( $stat ) ) {
713 $readUsers = array_merge( $this->readUsers, [ $this->swiftUser,
'.r:*' ] );
714 if ( !empty(
$params[
'listing'] ) ) {
717 $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
725 } elseif ( $stat === self::RES_ABSENT ) {
726 $status->fatal(
'backend-fail-usable',
$params[
'dir'] );
728 $status->fatal(
'backend-fail-internal', $this->name );
729 $this->logger->error( __METHOD__ .
': cannot get container stat' );
745 if ( $stat === self::RES_ABSENT ) {
747 } elseif ( $stat === self::RES_ERROR ) {
748 $status->fatal(
'backend-fail-internal', $this->name );
749 $this->logger->error( __METHOD__ .
': cannot get container stat' );
750 } elseif ( is_array( $stat ) && $stat[
'count'] == 0 ) {
764 return reset( $stats );
781 return $timestamp->getTimestamp( $format );
782 }
catch ( TimeoutException $e ) {
784 }
catch ( Exception $e ) {
797 if ( isset( $objHdrs[
'x-object-meta-sha1base36'] ) ) {
803 $this->logger->error( __METHOD__ .
": {path} was not stored with SHA-1 metadata.",
804 [
'path' =>
$path ] );
806 $objHdrs[
'x-object-meta-sha1base36'] =
false;
816 if ( $status->isOK() ) {
819 $hash = $tmpFile->getSha1Base36();
820 if ( $hash !==
false ) {
821 $objHdrs[
'x-object-meta-sha1base36'] = $hash;
823 $postHeaders[
'x-object-meta-sha1base36'] = $hash;
825 [ $rcode ] = $this->requestWithAuth( [
827 'container' => $srcCont,
828 'relPath' => $srcRel,
829 'headers' => $postHeaders
831 if ( $rcode >= 200 && $rcode <= 299 ) {
840 $this->logger->error( __METHOD__ .
': unable to set SHA-1 metadata for {path}',
841 [
'path' =>
$path ] );
847 $ep = array_diff_key(
$params, [
'srcs' => 1 ] );
853 $contents = array_fill_keys(
$params[
'srcs'], self::RES_ERROR );
856 if ( $srcRel ===
null ) {
860 $handle = fopen(
'php://temp',
'wb' );
864 'container' => $srcCont,
865 'relPath' => $srcRel,
872 $reqs = $this->requestMultiWithAuth(
874 [
'maxConnsPerHost' =>
$params[
'concurrency'] ]
876 foreach ( $reqs as
$path => $op ) {
877 [ $rcode, $rdesc, $rhdrs, $rbody, $rerr ] = $op[
'response'];
878 if ( $rcode >= 200 && $rcode <= 299 ) {
879 rewind( $op[
'stream'] );
880 $content = (string)stream_get_contents( $op[
'stream'] );
881 $size = strlen( $content );
883 if ( $size === (
int)$rhdrs[
'content-length'] ) {
884 $contents[
$path] = $content;
886 $contents[
$path] = self::RES_ERROR;
887 $rerr =
"Got {$size}/{$rhdrs['content-length']} bytes";
888 $this->
onError(
null, __METHOD__,
889 [
'src' =>
$path ] + $ep, $rerr, $rcode, $rdesc );
891 } elseif ( $rcode === 404 ) {
892 $contents[
$path] = self::RES_ABSENT;
894 $contents[
$path] = self::RES_ERROR;
895 $this->
onError(
null, __METHOD__,
896 [
'src' =>
$path ] + $ep, $rerr, $rcode, $rdesc, $rbody );
898 fclose( $op[
'stream'] );
905 $prefix = ( $dir ==
'' ) ?
null :
"{$dir}/";
906 $status = $this->objectListing( $fullCont,
'names', 1,
null, $prefix );
907 if ( $status->isOK() ) {
908 return ( count( $status->value ) ) > 0;
911 return self::RES_ERROR;
949 if ( $after === INF ) {
956 $prefix = ( $dir ==
'' ) ?
null :
"{$dir}/";
958 if ( !empty(
$params[
'topOnly'] ) ) {
959 $status = $this->objectListing( $fullCont,
'names', $limit, $after, $prefix,
'/' );
960 if ( !$status->isOK() ) {
963 $objects = $status->value;
965 foreach ( $objects as $object ) {
966 if ( substr( $object, -1 ) ===
'/' ) {
972 $getParentDir =
static function (
$path ) {
973 return (
$path !==
null && strpos(
$path,
'/' ) !== false ) ? dirname(
$path ) :
false;
977 $lastDir = $getParentDir( $after );
978 $status = $this->objectListing( $fullCont,
'names', $limit, $after, $prefix );
980 if ( !$status->isOK() ) {
984 $objects = $status->value;
987 foreach ( $objects as $object ) {
988 $objectDir = $getParentDir( $object );
990 if ( $objectDir !==
false && $objectDir !== $dir ) {
995 if ( strcmp( $objectDir, $lastDir ) > 0 ) {
998 $dirs[] =
"{$pDir}/";
999 $pDir = $getParentDir( $pDir );
1000 }
while ( $pDir !==
false
1001 && strcmp( $pDir, $lastDir ) > 0
1002 && strlen( $pDir ) > strlen( $dir )
1005 $lastDir = $objectDir;
1010 if ( count( $objects ) < $limit ) {
1013 $after = end( $objects );
1032 if ( $after === INF ) {
1039 $prefix = ( $dir ==
'' ) ?
null :
"{$dir}/";
1042 if ( !empty(
$params[
'topOnly'] ) ) {
1043 if ( !empty(
$params[
'adviseStat'] ) ) {
1044 $status = $this->objectListing( $fullCont,
'info', $limit, $after, $prefix,
'/' );
1046 $status = $this->objectListing( $fullCont,
'names', $limit, $after, $prefix,
'/' );
1050 if ( !empty(
$params[
'adviseStat'] ) ) {
1051 $status = $this->objectListing( $fullCont,
'info', $limit, $after, $prefix );
1053 $status = $this->objectListing( $fullCont,
'names', $limit, $after, $prefix );
1058 if ( !$status->isOK() ) {
1062 $objects = $status->value;
1063 $files = $this->buildFileObjectListing( $objects );
1066 if ( count( $objects ) < $limit ) {
1069 $after = end( $objects );
1070 $after = is_object( $after ) ? $after->name : $after;
1083 private function buildFileObjectListing( array $objects ) {
1085 foreach ( $objects as $object ) {
1086 if ( is_object( $object ) ) {
1087 if ( isset( $object->subdir ) || !isset( $object->name ) ) {
1093 'size' => (int)$object->bytes,
1096 'md5' => ctype_xdigit( $object->hash ) ? $object->hash :
null,
1099 $names[] = [ $object->name, $stat ];
1100 } elseif ( substr( $object, -1 ) !==
'/' ) {
1102 $names[] = [ $object, null ];
1116 $this->cheapCache->setField(
$path,
'stat', $val );
1122 if ( is_array( $stat ) && !isset( $stat[
'xattr'] ) ) {
1127 if ( is_array( $stat ) ) {
1128 return $stat[
'xattr'];
1131 return $stat === self::RES_ERROR ? self::RES_ERROR : self::RES_ABSENT;
1137 $params[
'requireSHA1'] =
true;
1140 if ( is_array( $stat ) ) {
1141 return $stat[
'sha1'];
1144 return $stat === self::RES_ERROR ? self::RES_ERROR : self::RES_ABSENT;
1150 $flags = !empty(
$params[
'headless'] ) ? HTTPFileStreamer::STREAM_HEADLESS : 0;
1153 if ( $srcRel ===
null ) {
1154 HTTPFileStreamer::send404Message(
$params[
'src'], $flags );
1155 $status->fatal(
'backend-fail-invalidpath',
$params[
'src'] );
1161 HTTPFileStreamer::send404Message(
$params[
'src'], $flags );
1162 $status->fatal(
'backend-fail-stream',
$params[
'src'] );
1170 HTTPFileStreamer::send404Message(
$params[
'src'], $flags );
1171 $status->fatal(
'backend-fail-stream',
$params[
'src'] );
1177 if ( empty(
$params[
'headless'] ) ) {
1179 $this->
header( $header );
1183 if ( empty(
$params[
'allowOB'] ) ) {
1188 $handle = fopen(
'php://output',
'wb' );
1189 [ $rcode, $rdesc, , $rbody, $rerr ] = $this->requestWithAuth( [
1191 'container' => $srcCont,
1192 'relPath' => $srcRel,
1194 'stream' => $handle,
1195 'flags' => [
'relayResponseHeaders' => empty(
$params[
'headless'] ) ]
1198 if ( $rcode >= 200 && $rcode <= 299 ) {
1200 } elseif ( $rcode === 404 ) {
1201 $status->fatal(
'backend-fail-stream',
$params[
'src'] );
1208 $this->
onError( $status, __METHOD__,
$params, $rerr, $rcode, $rdesc, $rbody );
1215 $ep = array_diff_key(
$params, [
'srcs' => 1 ] );
1221 $tmpFiles = array_fill_keys(
$params[
'srcs'], self::RES_ERROR );
1224 if ( $srcRel ===
null ) {
1228 $ext = FileBackend::extensionFromPath(
$path );
1230 $tmpFile = $this->tmpFileFactory->newTempFSFile(
'localcopy_', $ext );
1231 $handle = $tmpFile ? fopen( $tmpFile->getPath(),
'wb' ) :
false;
1235 'container' => $srcCont,
1236 'relPath' => $srcRel,
1238 'stream' => $handle,
1240 $tmpFiles[
$path] = $tmpFile;
1245 $latest = ( $this->isRGW || !empty(
$params[
'latest'] ) );
1247 $reqs = $this->requestMultiWithAuth(
1249 [
'maxConnsPerHost' =>
$params[
'concurrency'] ]
1251 foreach ( $reqs as
$path => $op ) {
1252 [ $rcode, $rdesc, $rhdrs, $rbody, $rerr ] = $op[
'response'];
1253 fclose( $op[
'stream'] );
1254 if ( $rcode >= 200 && $rcode <= 299 ) {
1256 $tmpFile = $tmpFiles[
$path];
1258 $size = $tmpFile->getSize();
1259 if ( $size !== (
int)$rhdrs[
'content-length'] ) {
1260 $tmpFiles[
$path] = self::RES_ERROR;
1261 $rerr =
"Got {$size}/{$rhdrs['content-length']} bytes";
1262 $this->
onError(
null, __METHOD__,
1263 [
'src' =>
$path ] + $ep, $rerr, $rcode, $rdesc );
1267 $stat[
'latest'] = $latest;
1268 $this->cheapCache->setField(
$path,
'stat', $stat );
1269 } elseif ( $rcode === 404 ) {
1270 $tmpFiles[
$path] = self::RES_ABSENT;
1271 $this->cheapCache->setField(
1274 $latest ? self::ABSENT_LATEST : self::ABSENT_NORMAL
1277 $tmpFiles[
$path] = self::RES_ERROR;
1278 $this->
onError(
null, __METHOD__,
1279 [
'src' =>
$path ] + $ep, $rerr, $rcode, $rdesc, $rbody );
1287 if ( $this->swiftTempUrlKey !=
'' ||
1288 ( $this->rgwS3AccessKey !=
'' && $this->rgwS3SecretKey !=
'' )
1291 if ( $srcRel ===
null ) {
1292 return self::TEMPURL_ERROR;
1297 return self::TEMPURL_ERROR;
1300 $ttl =
$params[
'ttl'] ?? 86400;
1301 $expires = time() + $ttl;
1303 if ( $this->swiftTempUrlKey !=
'' ) {
1306 $contPath = parse_url( $this->
storageUrl( $auth, $srcCont ), PHP_URL_PATH );
1307 $signature = hash_hmac(
'sha1',
1308 "GET\n{$expires}\n{$contPath}/{$srcRel}",
1309 $this->swiftTempUrlKey
1312 return "{$url}?temp_url_sig={$signature}&temp_url_expires={$expires}";
1315 $spath =
'/' . rawurlencode( $srcCont ) .
'/' .
1316 str_replace(
'%2F',
'/', rawurlencode( $srcRel ) );
1318 $signature = base64_encode( hash_hmac(
1320 "GET\n\n\n{$expires}\n{$spath}",
1321 $this->rgwS3SecretKey,
1327 return str_replace(
'/swift/v1',
'', $this->
storageUrl( $auth ) . $spath ) .
1330 'Signature' => $signature,
1331 'Expires' => $expires,
1332 'AWSAccessKeyId' => $this->rgwS3AccessKey
1337 return self::TEMPURL_ERROR;
1354 if ( !empty(
$params[
'latest'] ) ) {
1355 $hdrs[
'x-newest'] =
'true';
1363 '@phan-var SwiftFileOpHandle[] $fileOpHandles';
1369 $httpReqsByStage = [];
1370 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1371 $reqs = $fileOpHandle->httpOp;
1372 foreach ( $reqs as $stage => $req ) {
1373 $httpReqsByStage[$stage][$index] = $req;
1379 $reqCount = count( $httpReqsByStage );
1380 for ( $stage = 0; $stage < $reqCount; ++$stage ) {
1381 $httpReqs = $this->requestMultiWithAuth( $httpReqsByStage[$stage] );
1382 foreach ( $httpReqs as $index => $httpReq ) {
1384 $fileOpHandle = $fileOpHandles[$index];
1386 $status = $statuses[$index];
1387 ( $fileOpHandle->callback )( $httpReq, $status );
1392 $fileOpHandle->state === $fileOpHandle::CONTINUE_NO
1394 $stages = count( $fileOpHandle->httpOp );
1395 for ( $s = ( $stage + 1 ); $s < $stages; ++$s ) {
1396 unset( $httpReqsByStage[$s][$index] );
1430 [ $rcode, , , , ] = $this->requestWithAuth( [
1432 'container' => $container,
1434 'x-container-read' => implode(
',',
$readUsers ),
1435 'x-container-write' => implode(
',',
$writeUsers )
1439 if ( $rcode != 204 && $rcode !== 202 ) {
1440 $status->fatal(
'backend-fail-internal', $this->name );
1441 $this->logger->error( __METHOD__ .
': unexpected rcode value ({rcode})',
1442 [
'rcode' => $rcode ] );
1460 if ( $bypassCache ) {
1461 $this->containerStatCache->clear( $container );
1462 } elseif ( !$this->containerStatCache->hasField( $container,
'stat' ) ) {
1465 if ( !$this->containerStatCache->hasField( $container,
'stat' ) ) {
1466 [ $rcode, $rdesc, $rhdrs, $rbody, $rerr ] = $this->requestWithAuth( [
1468 'container' => $container
1471 if ( $rcode === 204 ) {
1473 'count' => $rhdrs[
'x-container-object-count'],
1474 'bytes' => $rhdrs[
'x-container-bytes-used']
1476 if ( $bypassCache ) {
1479 $this->containerStatCache->setField( $container,
'stat', $stat );
1482 } elseif ( $rcode === 404 ) {
1483 return self::RES_ABSENT;
1485 $this->
onError(
null, __METHOD__,
1486 [
'cont' => $container ], $rerr, $rcode, $rdesc, $rbody );
1488 return self::RES_ERROR;
1492 return $this->containerStatCache->getField( $container,
'stat' );
1506 if ( empty(
$params[
'noAccess'] ) ) {
1508 $readUsers = array_merge( $this->readUsers, [
'.r:*', $this->swiftUser ] );
1509 if ( empty(
$params[
'noListing'] ) ) {
1512 $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
1515 $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
1516 $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
1519 [ $rcode, $rdesc, , $rbody, $rerr ] = $this->requestWithAuth( [
1521 'container' => $container,
1523 'x-container-read' => implode(
',',
$readUsers ),
1524 'x-container-write' => implode(
',',
$writeUsers )
1528 if ( $rcode === 201 ) {
1530 } elseif ( $rcode === 202 ) {
1533 $this->
onError( $status, __METHOD__,
$params, $rerr, $rcode, $rdesc, $rbody );
1549 [ $rcode, $rdesc, , $rbody, $rerr ] = $this->requestWithAuth( [
1550 'method' =>
'DELETE',
1551 'container' => $container
1554 if ( $rcode >= 200 && $rcode <= 299 ) {
1555 $this->containerStatCache->clear( $container );
1556 } elseif ( $rcode === 404 ) {
1558 } elseif ( $rcode === 409 ) {
1559 $this->
onError( $status, __METHOD__,
$params, $rerr, $rcode, $rdesc );
1561 $this->
onError( $status, __METHOD__,
$params, $rerr, $rcode, $rdesc, $rbody );
1579 private function objectListing(
1580 $fullCont, $type, $limit, $after =
null, $prefix =
null, $delim =
null
1584 $query = [
'limit' => $limit ];
1585 if ( $type ===
'info' ) {
1586 $query[
'format'] =
'json';
1588 if ( $after !==
null ) {
1589 $query[
'marker'] = $after;
1591 if ( $prefix !==
null ) {
1592 $query[
'prefix'] = $prefix;
1594 if ( $delim !==
null ) {
1595 $query[
'delimiter'] = $delim;
1598 [ $rcode, $rdesc, , $rbody, $rerr ] = $this->requestWithAuth( [
1600 'container' => $fullCont,
1604 $params = [
'cont' => $fullCont,
'prefix' => $prefix,
'delim' => $delim ];
1605 if ( $rcode === 200 ) {
1606 if ( $type ===
'info' ) {
1607 $status->value = FormatJson::decode( trim( $rbody ) );
1609 $status->value = explode(
"\n", trim( $rbody ) );
1611 } elseif ( $rcode === 204 ) {
1612 $status->value = [];
1613 } elseif ( $rcode === 404 ) {
1614 $status->value = [];
1616 $this->
onError( $status, __METHOD__,
$params, $rerr, $rcode, $rdesc, $rbody );
1623 foreach ( $containerInfo as $container => $info ) {
1624 $this->containerStatCache->setField( $container,
'stat', $info );
1635 if ( $srcRel ===
null ) {
1637 $stats[
$path] = self::RES_ERROR;
1642 if ( $cstat === self::RES_ABSENT ) {
1643 $stats[
$path] = self::RES_ABSENT;
1645 } elseif ( $cstat === self::RES_ERROR ) {
1646 $stats[
$path] = self::RES_ERROR;
1652 'container' => $srcCont,
1653 'relPath' => $srcRel,
1659 $reqs = $this->requestMultiWithAuth(
1661 [
'maxConnsPerHost' =>
$params[
'concurrency'] ]
1663 foreach ( $reqs as
$path => $op ) {
1664 [ $rcode, $rdesc, $rhdrs, $rbody, $rerr ] = $op[
'response'];
1665 if ( $rcode === 200 || $rcode === 204 ) {
1667 if ( !empty(
$params[
'requireSHA1'] ) ) {
1672 if ( $this->isRGW ) {
1673 $stat[
'latest'] =
true;
1675 } elseif ( $rcode === 404 ) {
1676 $stat = self::RES_ABSENT;
1678 $stat = self::RES_ERROR;
1679 $this->
onError(
null, __METHOD__,
$params, $rerr, $rcode, $rdesc, $rbody );
1681 $stats[
$path] = $stat;
1701 'size' => isset( $rhdrs[
'content-length'] ) ? (int)$rhdrs[
'content-length'] : 0,
1702 'sha1' => $metadata[
'sha1base36'] ??
null,
1704 'md5' => ctype_xdigit( $rhdrs[
'etag'] ) ? $rhdrs[
'etag'] :
null,
1705 'xattr' => [
'metadata' => $metadata,
'headers' => $headers ]
1715 if ( $this->authErrorTimestamp !==
null ) {
1717 if ( $interval < 60 ) {
1718 $this->logger->debug(
1719 'rejecting request since auth failure occurred {interval} seconds ago',
1720 [
'interval' => $interval ]
1724 $this->authErrorTimestamp =
null;
1728 if ( !$this->authCreds ) {
1729 $cacheKey = $this->getCredsCacheKey( $this->swiftUser );
1730 $creds = $this->srvCache->get( $cacheKey );
1732 if ( isset( $creds[
'auth_token'] )
1733 && isset( $creds[
'storage_url'] )
1734 && isset( $creds[
'expiry_time'] )
1735 && $creds[
'expiry_time'] > time()
1737 $this->setAuthCreds( $creds );
1739 $this->refreshAuthentication();
1751 private function setAuthCreds( ?array $creds ) {
1752 $this->logger->debug(
'Using auth token with expiry_time={expiry_time}',
1754 'expiry_time' => isset( $creds[
'expiry_time'] )
1755 ? gmdate(
'c', $creds[
'expiry_time'] ) :
'null'
1758 $this->authCreds = $creds;
1760 if ( $creds && str_ends_with( $creds[
'storage_url'],
'/v1' ) ) {
1761 $this->isRGW =
true;
1770 private function refreshAuthentication() {
1771 [ $rcode, , $rhdrs, $rbody, ] = $this->http->run( [
1773 'url' =>
"{$this->swiftAuthUrl}/v1.0",
1775 'x-auth-user' => $this->swiftUser,
1776 'x-auth-key' => $this->swiftKey
1778 ], self::DEFAULT_HTTP_OPTIONS );
1780 if ( $rcode >= 200 && $rcode <= 299 ) {
1781 if ( isset( $rhdrs[
'x-auth-token-expires'] ) ) {
1782 $ttl = intval( $rhdrs[
'x-auth-token-expires'] );
1786 $expiryTime = time() + $ttl;
1788 'auth_token' => $rhdrs[
'x-auth-token'],
1789 'storage_url' => $this->swiftStorageUrl ?? $rhdrs[
'x-storage-url'],
1790 'expiry_time' => $expiryTime,
1792 $this->srvCache->set( $this->getCredsCacheKey( $this->swiftUser ), $creds, $expiryTime );
1793 } elseif ( $rcode === 401 ) {
1794 $this->
onError(
null, __METHOD__, [],
"Authentication failed.", $rcode );
1795 $this->authErrorTimestamp = time();
1798 $this->
onError(
null, __METHOD__, [],
"HTTP return code: $rcode", $rcode, $rbody );
1799 $this->authErrorTimestamp = time();
1802 $this->setAuthCreds( $creds );
1812 protected function storageUrl( array $creds, $container =
null, $object =
null ) {
1813 $parts = [ $creds[
'storage_url'] ];
1814 if ( strlen( $container ??
'' ) ) {
1815 $parts[] = rawurlencode( $container );
1817 if ( strlen( $object ??
'' ) ) {
1818 $parts[] = str_replace(
"%2F",
"/", rawurlencode( $object ) );
1821 return implode(
'/', $parts );
1829 return [
'x-auth-token' => $creds[
'auth_token'] ];
1838 private function getCredsCacheKey( $username ) {
1839 return 'swiftcredentials:' . md5( $username .
':' . $this->swiftAuthUrl );
1856 private function requestWithAuth( array $req, array $options = [] ) {
1857 return $this->requestMultiWithAuth( [ $req ], $options )[0][
'response'];
1869 private function requestMultiWithAuth( array $reqs, $options = [] ) {
1870 $remainingTries = 2;
1874 foreach ( $reqs as &$req ) {
1875 if ( !isset( $req[
'response'] ) ) {
1876 $req[
'response'] = $this->getAuthFailureResponse();
1881 foreach ( $reqs as &$req ) {
1882 '@phan-var array $req';
1883 if ( isset( $req[
'response'] ) ) {
1886 if ( $req[
'response'][
'code'] !== 401 ) {
1890 $req[
'headers'] = $this->
authTokenHeaders( $auth ) + ( $req[
'headers'] ?? [] );
1891 $req[
'url'] = $this->
storageUrl( $auth, $req[
'container'], $req[
'relPath'] ??
null );
1894 $reqs = $this->http->runMulti( $reqs, $options + self::DEFAULT_HTTP_OPTIONS );
1895 if ( --$remainingTries > 0 ) {
1897 foreach ( $reqs as $req ) {
1898 if ( $req[
'response'][
'code'] === 401 ) {
1899 $auth = $this->refreshAuthentication();
1917 private function getAuthFailureResponse() {
1927 'error' => self::AUTH_FAILURE_ERROR,
1928 4 => self::AUTH_FAILURE_ERROR
1939 private function isAuthFailureResponse( $code, $error ) {
1940 return $code === 0 && $error === self::AUTH_FAILURE_ERROR;
1955 public function onError( $status, $func, array
$params, $err =
'', $code = 0, $desc =
'', $body =
'' ) {
1956 if ( $this->isAuthFailureResponse( $code, $err ) ) {
1958 $status->fatal(
'backend-fail-connect', $this->name );
1964 $status->fatal(
'backend-fail-internal', $this->name );
1966 $msg =
"HTTP {code} ({desc}) in '{func}' (given '{req_params}')";
1971 'req_params' => FormatJson::encode(
$params ),
1975 $msgParams[
'err'] = $err;
1977 if ( $code == 502 ) {
1978 $msg .=
' ({truncatedBody})';
1979 $msgParams[
'truncatedBody'] = substr( strip_tags( $body ), 0, 100 );
1981 $this->logger->error( $msg, $msgParams );
array $params
The job parameters.
File backend exception for checked exceptions (e.g.
Base class for all backends using particular storage medium.
setContainerCache( $container, array $val)
Set the cached info for a container.
executeOpHandlesInternal(array $fileOpHandles)
Execute a list of FileBackendStoreOpHandle handles in parallel.
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
deleteFileCache( $path)
Delete the cached stat info for a file path.
getContentType( $storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
fileExists(array $params)
Check if a file exists at a storage path in the backend.
Store key-value entries in a size-limited in-memory LRU cache.
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Iterator for listing directories.
Iterator for listing regular files.
Class for an OpenStack Swift (or Ceph RGW) based file backend.
string $swiftUser
Swift user (account:user) to authenticate as.
string $swiftAuthUrl
Authentication base URL (without version)
string $swiftTempUrlKey
Shared secret value for making temp URLs.
MapCacheLRU $containerStatCache
Container stat cache.
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path in the backend.
getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
doPublishInternal( $fullCont, $dir, array $params)
doCreateInternal(array $params)
doGetFileStatMulti(array $params)
Get file stat information (concurrently if possible) for several files.
doGetFileSha1base36(array $params)
int null $authErrorTimestamp
UNIX timestamp.
array $writeUsers
Additional users (account:user) with write permissions on public containers.
__construct(array $config)
doGetFileXAttributes(array $params)
authTokenHeaders(array $creds)
getStatFromHeaders(array $rhdrs)
string $swiftStorageUrl
Override of storage base URL.
createContainer( $container, array $params)
Create a Swift container.
doCopyInternal(array $params)
getDirectoryListInternal( $fullCont, $dir, array $params)
string $rgwS3AccessKey
S3 access key (RADOS Gateway)
setContainerAccess( $container, array $readUsers, array $writeUsers)
Set read/write permissions for a Swift container.
getFileHttpUrl(array $params)
array $secureWriteUsers
Additional users (account:user) with write permissions on private containers.
extractMetadataHeaders(array $headers)
int $authTTL
TTL in seconds.
headersFromParams(array $params)
Get headers to send to Swift when reading a file based on a FileBackend params array,...
bool $isRGW
Whether the server is an Ceph RGW.
doStoreInternal(array $params)
onError( $status, $func, array $params, $err='', $code=0, $desc='', $body='')
Log an unexpected exception for this backend.
loadListingStatInternal( $path, array $val)
Do not call this function outside of SwiftFileBackendFileList.
doPrepareInternal( $fullCont, $dir, array $params)
FileBackendStore::doPrepare() to override StatusValue Good status without value for success,...
setLogger(LoggerInterface $logger)
doSecureInternal( $fullCont, $dir, array $params)
getFileListInternal( $fullCont, $dir, array $params)
getMetadataFromHeaders(array $headers)
doMoveInternal(array $params)
addMissingHashMetadata(array $objHdrs, $path)
Fill in any missing object metadata and save it to Swift.
getFeatures()
Get the a bitfield of extra features supported by the backend medium.
deleteContainer( $container, array $params)
Delete a Swift container.
doGetFileStat(array $params)
getAuthentication()
Get the cached auth token.
doGetLocalCopyMulti(array $params)
string $rgwS3SecretKey
S3 authentication key (RADOS Gateway)
doGetFileContentsMulti(array $params)
storageUrl(array $creds, $container=null, $object=null)
convertSwiftDate( $ts, $format=TS_MW)
Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT"/"2013-05-11T07:37:27.678360Z".
doStreamFile(array $params)
doPrimeContainerCache(array $containerInfo)
Fill the backend-specific process cache given an array of resolved container names and their correspo...
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
array $readUsers
Additional users (account:user) with read permissions on public containers.
array $secureReadUsers
Additional users (account:user) with read permissions on private containers.
doCleanInternal( $fullCont, $dir, array $params)
getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
string $swiftKey
Secret key for user.
doDirectoryExists( $fullCont, $dir, array $params)
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
doExecuteOpHandlesInternal(array $fileOpHandles)
doDeleteInternal(array $params)
doDescribeInternal(array $params)
extractMutableContentHeaders(array $headers)
Filter/normalize a header map to only include mutable "content-"/"x-content-" headers.
getContainerStat( $container, $bypassCache=false)
Get a Swift container stat map, possibly from process cache.
Multi-datacenter aware caching interface.