26use InvalidArgumentException;
28use Shellbox\Command\BoxedCommand;
31use Wikimedia\AtEase\AtEase;
48use Wikimedia\Timestamp\ConvertibleTimestamp;
88 protected const RES_ABSENT =
false;
90 protected const RES_ERROR =
null;
93 protected const ABSENT_NORMAL =
'FNE-N';
95 protected const ABSENT_LATEST =
'FNE-L';
111 parent::__construct( $config );
112 $this->mimeCallback = $config[
'mimeCallback'] ??
null;
114 $this->memCache = WANObjectCache::newEmpty();
115 $this->cheapCache =
new MapCacheLRU( self::CACHE_CHEAP_SIZE );
116 $this->expensiveCache =
new MapCacheLRU( self::CACHE_EXPENSIVE_SIZE );
127 return min( $this->maxFileSize, PHP_INT_MAX );
165 $status = $this->
newStatus(
'backend-fail-maxsize',
170 if ( $params[
'dstExists'] ??
true ) {
208 $status = $this->
newStatus(
'backend-fail-maxsize',
213 if ( $params[
'dstExists'] ??
true ) {
253 if ( $params[
'dstExists'] ??
true ) {
322 $this->
clearCache( [ $params[
'src'], $params[
'dst'] ] );
324 if ( $params[
'dstExists'] ??
true ) {
356 if ( count( $params[
'headers'] ) ) {
395 $scopeLockS = $this->
getScopedFileLocks( $params[
'srcs'], LockManager::LOCK_UW, $status );
396 if ( $status->isOK() ) {
398 $hrStart = hrtime(
true );
400 $sec = ( hrtime(
true ) - $hrStart ) / 1e9;
401 if ( !$status->isOK() ) {
402 $this->logger->error( static::class .
"-{$this->name}" .
403 " failed to concatenate " . count( $params[
'srcs'] ) .
" file(s) [$sec sec]" );
418 $tmpPath = $params[
'dst'];
419 unset( $params[
'latest'] );
422 AtEase::suppressWarnings();
423 $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 );
424 AtEase::restoreWarnings();
426 $status->fatal(
'backend-fail-opentemp', $tmpPath );
433 foreach ( $fsFiles as
$path => &$fsFile ) {
438 $fsFile === self::RES_ERROR ?
'backend-fail-read' :
'backend-fail-notexists',
449 $tmpHandle = fopen( $tmpPath,
'ab' );
450 if ( $tmpHandle ===
false ) {
451 $status->fatal(
'backend-fail-opentemp', $tmpPath );
457 foreach ( $fsFiles as $virtualSource => $fsFile ) {
459 $sourceHandle = fopen( $fsFile->getPath(),
'rb' );
460 if ( $sourceHandle ===
false ) {
461 fclose( $tmpHandle );
462 $status->fatal(
'backend-fail-read', $virtualSource );
467 if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
468 fclose( $sourceHandle );
469 fclose( $tmpHandle );
470 $status->fatal(
'backend-fail-writetemp', $tmpPath );
474 fclose( $sourceHandle );
476 if ( !fclose( $tmpHandle ) ) {
477 $status->fatal(
'backend-fail-closetemp', $tmpPath );
496 if ( $dir ===
null ) {
497 $status->fatal(
'backend-fail-invalidpath', $params[
'dir'] );
502 if ( $shard !==
null ) {
505 $this->logger->debug( __METHOD__ .
": iterating over all container shards." );
508 $status->merge( $this->
doPrepareInternal(
"{$fullCont}{$suffix}", $dir, $params ) );
527 final protected function doSecure( array $params ) {
533 if ( $dir ===
null ) {
534 $status->fatal(
'backend-fail-invalidpath', $params[
'dir'] );
539 if ( $shard !==
null ) {
542 $this->logger->debug( __METHOD__ .
": iterating over all container shards." );
545 $status->merge( $this->
doSecureInternal(
"{$fullCont}{$suffix}", $dir, $params ) );
570 if ( $dir ===
null ) {
571 $status->fatal(
'backend-fail-invalidpath', $params[
'dir'] );
576 if ( $shard !==
null ) {
579 $this->logger->debug( __METHOD__ .
": iterating over all container shards." );
582 $status->merge( $this->
doPublishInternal(
"{$fullCont}{$suffix}", $dir, $params ) );
601 final protected function doClean( array $params ) {
609 if ( $subDirsRel !==
null ) {
610 foreach ( $subDirsRel as $subDirRel ) {
611 $subDir = $params[
'dir'] .
"/{$subDirRel}";
612 $status->merge( $this->
doClean( [
'dir' => $subDir ] + $params ) );
614 unset( $subDirsRel );
619 if ( $dir ===
null ) {
620 $status->fatal(
'backend-fail-invalidpath', $params[
'dir'] );
626 $filesLockEx = [ $params[
'dir'] ];
628 $scopedLockE = $this->
getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
629 if ( !$status->isOK() ) {
633 if ( $shard !==
null ) {
637 $this->logger->debug( __METHOD__ .
": iterating over all container shards." );
640 $status->merge( $this->
doCleanInternal(
"{$fullCont}{$suffix}", $dir, $params ) );
665 if ( is_array( $stat ) ) {
669 return $stat === self::RES_ABSENT ? false : self::EXISTENCE_ERROR;
677 if ( is_array( $stat ) ) {
678 return $stat[
'mtime'];
681 return self::TIMESTAMP_FAIL;
689 if ( is_array( $stat ) ) {
690 return $stat[
'size'];
693 return self::SIZE_FAIL;
701 if (
$path ===
null ) {
702 return self::STAT_ERROR;
707 $latest = !empty( $params[
'latest'] );
709 $requireSHA1 = !empty( $params[
'requireSHA1'] );
711 $stat = $this->cheapCache->getField(
$path,
'stat', self::CACHE_TTL );
720 ( $requireSHA1 && is_array( $stat ) && !isset( $stat[
'sha1'] ) )
724 $stat = $this->cheapCache->getField(
$path,
'stat', self::CACHE_TTL );
728 if ( is_array( $stat ) ) {
730 ( !$latest || !empty( $stat[
'latest'] ) ) &&
731 ( !$requireSHA1 || isset( $stat[
'sha1'] ) )
735 } elseif ( $stat === self::ABSENT_LATEST ) {
736 return self::STAT_ABSENT;
737 } elseif ( $stat === self::ABSENT_NORMAL ) {
739 return self::STAT_ABSENT;
747 if ( is_array( $stat ) ) {
751 return $stat === self::RES_ERROR ? self::STAT_ERROR : self::STAT_ABSENT;
764 foreach ( $stats as
$path => $stat ) {
765 if ( is_array( $stat ) ) {
767 $stat[
'latest'] ??= $latest;
769 $this->cheapCache->setField(
$path,
'stat', $stat );
770 if ( isset( $stat[
'sha1'] ) ) {
772 $this->cheapCache->setField(
775 [
'hash' => $stat[
'sha1'],
'latest' => $latest ]
778 if ( isset( $stat[
'xattr'] ) ) {
781 $this->cheapCache->setField(
784 [
'map' => $stat[
'xattr'],
'latest' => $latest ]
789 } elseif ( $stat === self::RES_ABSENT ) {
790 $this->cheapCache->setField(
793 $latest ? self::ABSENT_LATEST : self::ABSENT_NORMAL
795 $this->cheapCache->setField(
798 [
'map' => self::XATTRS_FAIL,
'latest' => $latest ]
800 $this->cheapCache->setField(
803 [
'hash' => self::SHA1_FAIL,
'latest' => $latest ]
805 $this->logger->debug(
806 __METHOD__ .
': File {path} does not exist',
811 $this->logger->error(
812 __METHOD__ .
': Could not stat file {path}',
834 foreach ( $contents as
$path => $content ) {
835 if ( !is_string( $content ) ) {
836 $contents[
$path] = self::CONTENT_FAIL;
852 if ( $fsFile instanceof
FSFile ) {
853 AtEase::suppressWarnings();
854 $content = file_get_contents( $fsFile->getPath() );
855 AtEase::restoreWarnings();
856 $contents[
$path] = is_string( $content ) ? $content : self::RES_ERROR;
859 $contents[
$path] = $fsFile;
871 if (
$path ===
null ) {
872 return self::XATTRS_FAIL;
874 $latest = !empty( $params[
'latest'] );
875 if ( $this->cheapCache->hasField(
$path,
'xattr', self::CACHE_TTL ) ) {
876 $stat = $this->cheapCache->getField(
$path,
'xattr' );
879 if ( !$latest || $stat[
'latest'] ) {
884 if ( is_array( $fields ) ) {
886 $this->cheapCache->setField(
889 [
'map' => $fields,
'latest' => $latest ]
891 } elseif ( $fields === self::RES_ABSENT ) {
892 $this->cheapCache->setField(
895 [
'map' => self::XATTRS_FAIL,
'latest' => $latest ]
898 $fields = self::XATTRS_FAIL;
911 return [
'headers' => [],
'metadata' => [] ];
919 if (
$path ===
null ) {
920 return self::SHA1_FAIL;
922 $latest = !empty( $params[
'latest'] );
923 if ( $this->cheapCache->hasField(
$path,
'sha1', self::CACHE_TTL ) ) {
924 $stat = $this->cheapCache->getField(
$path,
'sha1' );
927 if ( !$latest || $stat[
'latest'] ) {
928 return $stat[
'hash'];
932 if ( is_string( $sha1 ) ) {
933 $this->cheapCache->setField(
936 [
'hash' => $sha1,
'latest' => $latest ]
938 } elseif ( $sha1 === self::RES_ABSENT ) {
939 $this->cheapCache->setField(
942 [
'hash' => self::SHA1_FAIL,
'latest' => $latest ]
945 $sha1 = self::SHA1_FAIL;
959 if ( $fsFile instanceof
FSFile ) {
960 $sha1 = $fsFile->getSha1Base36();
962 return is_string( $sha1 ) ? $sha1 : self::RES_ERROR;
965 return $fsFile === self::RES_ERROR ? self::RES_ERROR : self::RES_ABSENT;
974 return $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
984 $latest = !empty( $params[
'latest'] );
986 foreach ( $params[
'srcs'] as $src ) {
988 if (
$path ===
null ) {
989 $fsFiles[$src] = self::RES_ERROR;
990 } elseif ( $this->expensiveCache->hasField(
$path,
'localRef' ) ) {
991 $val = $this->expensiveCache->getField(
$path,
'localRef' );
994 if ( !$latest || $val[
'latest'] ) {
995 $fsFiles[$src] = $val[
'object'];
1000 $params[
'srcs'] = array_diff( $params[
'srcs'], array_keys( $fsFiles ) );
1002 $fsFiles[
$path] = $fsFile;
1003 if ( $fsFile instanceof
FSFile ) {
1004 $this->expensiveCache->setField(
1007 [
'object' => $fsFile,
'latest' => $latest ]
1048 return self::TEMPURL_ERROR;
1055 if ( $ref ===
false ) {
1056 return $this->
newStatus(
'backend-fail-notexists', $params[
'src'] );
1057 } elseif ( $ref ===
null ) {
1058 return $this->
newStatus(
'backend-fail-read', $params[
'src'] );
1060 $file = $command->newInputFileFromFile( $ref->getPath() )
1061 ->userData( __CLASS__, $ref );
1062 $command->inputFile( $boxedName, $file );
1073 $params[
'options'] ??= [];
1074 $params[
'headers'] ??= [];
1077 if ( ( empty( $params[
'headless'] ) || $params[
'headers'] ) && headers_sent() ) {
1078 print
"Headers already sent, terminating.\n";
1079 $status->fatal(
'backend-fail-stream', $params[
'src'] );
1105 $this->getStreamerOptions()
1107 $res = $streamer->stream( $params[
'headers'],
true, $params[
'options'], $flags );
1114 $status->fatal(
'backend-fail-stream', $params[
'src'] );
1122 if ( $dir ===
null ) {
1123 return self::EXISTENCE_ERROR;
1125 if ( $shard !==
null ) {
1128 $this->logger->debug( __METHOD__ .
": iterating over all container shards." );
1133 if ( $exists ===
true ) {
1136 } elseif ( $exists === self::RES_ERROR ) {
1137 $res = self::EXISTENCE_ERROR;
1157 if ( $dir ===
null ) {
1158 return self::EXISTENCE_ERROR;
1160 if ( $shard !==
null ) {
1164 $this->logger->debug( __METHOD__ .
": iterating over all container shards." );
1187 if ( $dir ===
null ) {
1188 return self::LIST_ERROR;
1190 if ( $shard !==
null ) {
1194 $this->logger->debug( __METHOD__ .
": iterating over all container shards." );
1228 'store' => StoreFileOp::class,
1229 'copy' => CopyFileOp::class,
1230 'move' => MoveFileOp::class,
1231 'delete' => DeleteFileOp::class,
1232 'create' => CreateFileOp::class,
1233 'describe' => DescribeFileOp::class,
1234 'null' => NullFileOp::class
1239 foreach ( $ops as $operation ) {
1240 $opName = $operation[
'op'];
1241 if ( isset( $supportedOps[$opName] ) ) {
1242 $class = $supportedOps[$opName];
1244 $params = $operation;
1246 $performOps[] =
new $class( $this, $params, $this->logger );
1267 $paths = [
'sh' => [],
'ex' => [] ];
1268 foreach ( $performOps as $fileOp ) {
1269 $paths[
'sh'] = array_merge( $paths[
'sh'], $fileOp->storagePathsRead() );
1270 $paths[
'ex'] = array_merge( $paths[
'ex'], $fileOp->storagePathsChanged() );
1273 $paths[
'sh'] = array_diff( $paths[
'sh'], $paths[
'ex'] );
1275 $paths[
'sh'] = array_merge( $paths[
'sh'], array_map(
'dirname', $paths[
'ex'] ) );
1278 LockManager::LOCK_UW => $paths[
'sh'],
1279 LockManager::LOCK_EX => $paths[
'ex']
1299 foreach ( $fileOps as $fileOp ) {
1300 $pathsUsed = array_merge( $pathsUsed, $fileOp->storagePathsReadOrChanged() );
1304 if ( empty( $opts[
'nonLocking'] ) ) {
1308 if ( !$status->isOK() ) {
1314 if ( empty( $opts[
'preserveCache'] ) ) {
1319 $this->cheapCache->setMaxSize( max( 2 * count( $pathsUsed ), self::CACHE_CHEAP_SIZE ) );
1324 $ok = $this->
preloadFileStat( [
'srcs' => $pathsUsed,
'latest' =>
true ] );
1332 $subStatus = $this->
newStatus(
'backend-fail-internal', $this->name );
1333 foreach ( $ops as $i => $op ) {
1334 $subStatus->success[$i] =
false;
1335 ++$subStatus->failCount;
1337 $this->logger->error( static::class .
"-{$this->name} stat failure",
1338 [
'aborted_operations' => $ops ]
1343 $status->merge( $subStatus );
1344 $status->success = $subStatus->success;
1347 $this->cheapCache->setMaxSize( self::CACHE_CHEAP_SIZE );
1362 foreach ( $fileOps as $fileOp ) {
1363 $pathsUsed = array_merge( $pathsUsed, $fileOp->storagePathsReadOrChanged() );
1370 $async = ( $this->parallelize ===
'implicit' && count( $ops ) > 1 );
1376 foreach ( $fileOps as $index => $fileOp ) {
1378 ? $fileOp->attemptAsyncQuick()
1379 : $fileOp->attemptQuick();
1381 if ( count( $batch ) >= $maxConcurrency ) {
1387 $batch[$index] = $subStatus->value;
1389 $statuses[$index] = $subStatus;
1392 if ( count( $batch ) ) {
1396 foreach ( $statuses as $index => $subStatus ) {
1397 $status->merge( $subStatus );
1398 if ( $subStatus->isOK() ) {
1399 $status->success[$index] =
true;
1400 ++$status->successCount;
1402 $status->success[$index] =
false;
1403 ++$status->failCount;
1425 foreach ( $fileOpHandles as $fileOpHandle ) {
1427 throw new InvalidArgumentException(
"Expected FileBackendStoreOpHandle object." );
1428 } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) {
1429 throw new InvalidArgumentException(
"Expected handle for this file backend." );
1434 foreach ( $fileOpHandles as $fileOpHandle ) {
1435 $fileOpHandle->closeResources();
1451 if ( count( $fileOpHandles ) ) {
1452 throw new FileBackendError(
"Backend does not support asynchronous operations." );
1470 static $longs = [
'content-disposition' ];
1472 if ( isset( $op[
'headers'] ) ) {
1474 foreach ( $op[
'headers'] as
$name => $value ) {
1476 $maxHVLen = in_array(
$name, $longs ) ? INF : 255;
1477 if ( strlen(
$name ) > 255 || strlen( $value ) > $maxHVLen ) {
1478 $this->logger->error(
"Header '{header}' is too long.", [
1479 'filebackend' => $this->name,
1480 'header' =>
"$name: $value",
1483 $newHeaders[
$name] = strlen( $value ) ? $value :
'';
1486 $op[
'headers'] = $newHeaders;
1494 foreach ( $paths as
$path ) {
1496 $fullConts[] = $fullCont;
1504 if ( is_array( $paths ) ) {
1505 $paths = array_map( [ FileBackend::class,
'normalizeStoragePath' ], $paths );
1506 $paths = array_filter( $paths,
'strlen' );
1508 if ( $paths ===
null ) {
1509 $this->cheapCache->clear();
1510 $this->expensiveCache->clear();
1512 foreach ( $paths as
$path ) {
1513 $this->cheapCache->clear(
$path );
1514 $this->expensiveCache->clear(
$path );
1535 $params[
'concurrency'] = ( $this->parallelize !==
'off' ) ? $this->concurrency : 1;
1537 if ( $stats ===
null ) {
1542 $latest = !empty( $params[
'latest'] );
1605 return (
bool)preg_match(
'/^[a-z0-9][a-z0-9-_.]{0,199}$/i', $container );
1623 if ( $backend === $this->name && $relPath !==
null ) {
1625 if ( $relPath !==
null && self::isValidShortContainerName( $shortCont ) ) {
1630 if ( $relPath !==
null ) {
1633 if ( self::isValidContainerName( $container ) ) {
1636 if ( $container !==
null ) {
1637 return [ $container, $relPath, $cShard ];
1644 return [
null,
null, null ];
1664 if ( $cShard !==
null && substr( $relPath, -1 ) !==
'/' ) {
1665 return [ $container, $relPath ];
1668 return [
null, null ];
1681 if ( $levels == 1 || $levels == 2 ) {
1683 $char = ( $base == 36 ) ?
'[0-9a-z]' :
'[0-9a-f]';
1686 if ( $levels === 1 ) {
1687 $hashDirRegex =
'(' . $char .
')';
1690 $hashDirRegex = $char .
'/(' . $char .
'{2})';
1692 $hashDirRegex =
'(' . $char .
')/(' . $char .
')';
1699 if ( preg_match(
"!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1700 return '.' . implode(
'', array_slice( $m, 1 ) );
1720 return ( $shard !==
null );
1732 if ( isset( $this->shardViaHashLevels[$container] ) ) {
1733 $config = $this->shardViaHashLevels[$container];
1734 $hashLevels = (int)$config[
'levels'];
1735 if ( $hashLevels == 1 || $hashLevels == 2 ) {
1736 $hashBase = (int)$config[
'base'];
1737 if ( $hashBase == 16 || $hashBase == 36 ) {
1738 return [ $hashLevels, $hashBase, $config[
'repeat'] ];
1743 return [ 0, 0, false ];
1755 if ( $digits > 0 ) {
1756 $numShards = $base ** $digits;
1757 for ( $index = 0; $index < $numShards; $index++ ) {
1758 $shards[] =
'.' . \Wikimedia\base_convert( (
string)$index, 10, $base, $digits );
1772 if ( $this->domainId !=
'' ) {
1773 return "{$this->domainId}-$container";
1804 return $relStoragePath;
1813 private function containerCacheKey( $container ) {
1814 return "filebackend:{$this->name}:{$this->domainId}:container:{$container}";
1824 if ( !$this->memCache->set( $this->containerCacheKey( $container ), $val, 14 * 86400 ) ) {
1825 $this->logger->warning(
"Unable to set stat cache for container {container}.",
1826 [
'filebackend' => $this->name,
'container' => $container ]
1838 if ( !$this->memCache->delete( $this->containerCacheKey( $container ), 300 ) ) {
1839 $this->logger->warning(
"Unable to delete stat cache for container {container}.",
1840 [
'filebackend' => $this->name,
'container' => $container ]
1857 foreach ( $items as $item ) {
1858 if ( self::isStoragePath( $item ) ) {
1860 } elseif ( is_string( $item ) ) {
1861 $contNames[$this->containerCacheKey( $item )] = $item;
1865 foreach ( $paths as
$path ) {
1867 if ( $fullCont !==
null ) {
1868 $contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
1874 $values = $this->memCache->getMulti( array_keys( $contNames ) );
1875 foreach ( $values as $cacheKey => $val ) {
1876 $contInfo[$contNames[$cacheKey]] = $val;
1900 private function fileCacheKey(
$path ) {
1901 return "filebackend:{$this->name}:{$this->domainId}:file:" . sha1(
$path );
1914 if (
$path ===
null ) {
1917 $mtime = (int)ConvertibleTimestamp::convert( TS_UNIX, $val[
'mtime'] );
1918 $ttl = $this->memCache->adaptiveTTL( $mtime, 7 * 86400, 300, 0.1 );
1919 $key = $this->fileCacheKey(
$path );
1921 if ( !$this->memCache->set( $key, $val, $ttl ) ) {
1922 $this->logger->warning(
"Unable to set stat cache for file {path}.",
1923 [
'filebackend' => $this->name,
'path' =>
$path ]
1938 if (
$path ===
null ) {
1941 if ( !$this->memCache->delete( $this->fileCacheKey(
$path ), 300 ) ) {
1942 $this->logger->warning(
"Unable to delete stat cache for file {path}.",
1943 [
'filebackend' => $this->name,
'path' =>
$path ]
1962 foreach ( $items as $item ) {
1963 if ( self::isStoragePath( $item ) ) {
1965 if (
$path !==
null ) {
1971 foreach ( $paths as
$path ) {
1973 if ( $rel !==
null ) {
1974 $pathNames[$this->fileCacheKey(
$path )] =
$path;
1979 $values = $this->memCache->getMulti( array_keys( $pathNames ) );
1981 foreach ( array_filter( $values,
'is_array' ) as $cacheKey => $stat ) {
1982 $path = $pathNames[$cacheKey];
1985 unset( $stat[
'latest'] );
1987 $this->cheapCache->setField(
$path,
'stat', $stat );
1988 if ( isset( $stat[
'sha1'] ) && strlen( $stat[
'sha1'] ) == 31 ) {
1990 $this->cheapCache->setField(
1993 [
'hash' => $stat[
'sha1'],
'latest' =>
false ]
1996 if ( isset( $stat[
'xattr'] ) && is_array( $stat[
'xattr'] ) ) {
1999 $this->cheapCache->setField(
2002 [
'map' => $stat[
'xattr'],
'latest' =>
false ]
2016 $newXAttr = [
'headers' => [],
'metadata' => [] ];
2018 foreach ( $xattr[
'headers'] as
$name => $value ) {
2019 $newXAttr[
'headers'][strtolower(
$name )] = $value;
2022 foreach ( $xattr[
'metadata'] as
$name => $value ) {
2023 $newXAttr[
'metadata'][strtolower(
$name )] = $value;
2036 $opts[
'concurrency'] = 1;
2037 if ( $this->parallelize ===
'implicit' ) {
2038 if ( $opts[
'parallelize'] ??
true ) {
2041 } elseif ( $this->parallelize ===
'explicit' ) {
2042 if ( !empty( $opts[
'parallelize'] ) ) {
2060 if ( $this->mimeCallback ) {
2061 return ( $this->mimeCallback )( $storagePath, $content, $fsPath );
2064 $mime = ( $fsPath !== null ) ? mime_content_type( $fsPath ) :
false;
2065 return $mime ?:
'unknown/unknown';
2070class_alias( FileBackendStore::class,
'FileBackendStore' );
Resource locking handling.
Generic operation result class Has warning/error list, boolean status and arbitrary value.