25use Wikimedia\AtEase\AtEase;
29use Wikimedia\Timestamp\ConvertibleTimestamp;
68 protected const RES_ABSENT =
false;
70 protected const RES_ERROR =
null;
73 protected const ABSENT_NORMAL =
'FNE-N';
75 protected const ABSENT_LATEST =
'FNE-L';
91 parent::__construct( $config );
92 $this->mimeCallback = $config[
'mimeCallback'] ??
null;
94 $this->memCache = WANObjectCache::newEmpty();
95 $this->cheapCache =
new MapCacheLRU( self::CACHE_CHEAP_SIZE );
96 $this->expensiveCache =
new MapCacheLRU( self::CACHE_EXPENSIVE_SIZE );
107 return min( $this->maxFileSize, PHP_INT_MAX );
145 $status = $this->
newStatus(
'backend-fail-maxsize',
150 if (
$params[
'dstExists'] ??
true ) {
188 $status = $this->
newStatus(
'backend-fail-maxsize',
193 if (
$params[
'dstExists'] ??
true ) {
233 if (
$params[
'dstExists'] ??
true ) {
304 if (
$params[
'dstExists'] ??
true ) {
336 if ( count(
$params[
'headers'] ) ) {
375 $scopeLockS = $this->
getScopedFileLocks( $params[
'srcs'], LockManager::LOCK_UW, $status );
376 if ( $status->isOK() ) {
378 $start_time = microtime(
true );
380 $sec = microtime(
true ) - $start_time;
381 if ( !$status->isOK() ) {
382 $this->logger->error( static::class .
"-{$this->name}" .
383 " failed to concatenate " . count(
$params[
'srcs'] ) .
" file(s) [$sec sec]" );
402 AtEase::suppressWarnings();
403 $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 );
404 AtEase::restoreWarnings();
406 $status->fatal(
'backend-fail-opentemp', $tmpPath );
413 foreach ( $fsFiles as
$path => &$fsFile ) {
418 $fsFile === self::RES_ERROR ?
'backend-fail-read' :
'backend-fail-notexists',
429 $tmpHandle = fopen( $tmpPath,
'ab' );
430 if ( $tmpHandle ===
false ) {
431 $status->fatal(
'backend-fail-opentemp', $tmpPath );
437 foreach ( $fsFiles as $virtualSource => $fsFile ) {
439 $sourceHandle = fopen( $fsFile->getPath(),
'rb' );
440 if ( $sourceHandle ===
false ) {
441 fclose( $tmpHandle );
442 $status->fatal(
'backend-fail-read', $virtualSource );
447 if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
448 fclose( $sourceHandle );
449 fclose( $tmpHandle );
450 $status->fatal(
'backend-fail-writetemp', $tmpPath );
454 fclose( $sourceHandle );
456 if ( !fclose( $tmpHandle ) ) {
457 $status->fatal(
'backend-fail-closetemp', $tmpPath );
476 if ( $dir ===
null ) {
477 $status->fatal(
'backend-fail-invalidpath',
$params[
'dir'] );
482 if ( $shard !==
null ) {
485 $this->logger->debug( __METHOD__ .
": iterating over all container shards." );
513 if ( $dir ===
null ) {
514 $status->fatal(
'backend-fail-invalidpath',
$params[
'dir'] );
519 if ( $shard !==
null ) {
522 $this->logger->debug( __METHOD__ .
": iterating over all container shards." );
550 if ( $dir ===
null ) {
551 $status->fatal(
'backend-fail-invalidpath',
$params[
'dir'] );
556 if ( $shard !==
null ) {
559 $this->logger->debug( __METHOD__ .
": iterating over all container shards." );
589 if ( $subDirsRel !==
null ) {
590 foreach ( $subDirsRel as $subDirRel ) {
591 $subDir =
$params[
'dir'] .
"/{$subDirRel}";
592 $status->merge( $this->
doClean( [
'dir' => $subDir ] +
$params ) );
594 unset( $subDirsRel );
599 if ( $dir ===
null ) {
600 $status->fatal(
'backend-fail-invalidpath',
$params[
'dir'] );
606 $filesLockEx = [
$params[
'dir'] ];
608 $scopedLockE = $this->
getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
609 if ( !$status->isOK() ) {
613 if ( $shard !==
null ) {
617 $this->logger->debug( __METHOD__ .
": iterating over all container shards." );
645 if ( is_array( $stat ) ) {
649 return $stat === self::RES_ABSENT ? false : self::EXISTENCE_ERROR;
657 if ( is_array( $stat ) ) {
658 return $stat[
'mtime'];
661 return self::TIMESTAMP_FAIL;
669 if ( is_array( $stat ) ) {
670 return $stat[
'size'];
673 return self::SIZE_FAIL;
681 if (
$path ===
null ) {
682 return self::STAT_ERROR;
687 $latest = !empty(
$params[
'latest'] );
689 $requireSHA1 = !empty(
$params[
'requireSHA1'] );
691 $stat = $this->cheapCache->getField(
$path,
'stat', self::CACHE_TTL );
700 ( $requireSHA1 && is_array( $stat ) && !isset( $stat[
'sha1'] ) )
704 $stat = $this->cheapCache->getField(
$path,
'stat', self::CACHE_TTL );
708 if ( is_array( $stat ) ) {
710 ( !$latest || !empty( $stat[
'latest'] ) ) &&
711 ( !$requireSHA1 || isset( $stat[
'sha1'] ) )
715 } elseif ( $stat === self::ABSENT_LATEST ) {
716 return self::STAT_ABSENT;
717 } elseif ( $stat === self::ABSENT_NORMAL ) {
719 return self::STAT_ABSENT;
727 if ( is_array( $stat ) ) {
731 return $stat === self::RES_ERROR ? self::STAT_ERROR : self::STAT_ABSENT;
744 foreach ( $stats as
$path => $stat ) {
745 if ( is_array( $stat ) ) {
747 $stat[
'latest'] ??= $latest;
749 $this->cheapCache->setField(
$path,
'stat', $stat );
750 if ( isset( $stat[
'sha1'] ) ) {
752 $this->cheapCache->setField(
755 [
'hash' => $stat[
'sha1'],
'latest' => $latest ]
758 if ( isset( $stat[
'xattr'] ) ) {
761 $this->cheapCache->setField(
764 [
'map' => $stat[
'xattr'],
'latest' => $latest ]
769 } elseif ( $stat === self::RES_ABSENT ) {
770 $this->cheapCache->setField(
773 $latest ? self::ABSENT_LATEST : self::ABSENT_NORMAL
775 $this->cheapCache->setField(
778 [
'map' => self::XATTRS_FAIL,
'latest' => $latest ]
780 $this->cheapCache->setField(
783 [
'hash' => self::SHA1_FAIL,
'latest' => $latest ]
785 $this->logger->debug(
786 __METHOD__ .
': File {path} does not exist',
791 $this->logger->error(
792 __METHOD__ .
': Could not stat file {path}',
814 foreach ( $contents as
$path => $content ) {
815 if ( !is_string( $content ) ) {
816 $contents[
$path] = self::CONTENT_FAIL;
832 if ( $fsFile instanceof
FSFile ) {
833 AtEase::suppressWarnings();
834 $content = file_get_contents( $fsFile->getPath() );
835 AtEase::restoreWarnings();
836 $contents[
$path] = is_string( $content ) ? $content : self::RES_ERROR;
839 $contents[
$path] = $fsFile;
851 if (
$path ===
null ) {
852 return self::XATTRS_FAIL;
854 $latest = !empty(
$params[
'latest'] );
855 if ( $this->cheapCache->hasField(
$path,
'xattr', self::CACHE_TTL ) ) {
856 $stat = $this->cheapCache->getField(
$path,
'xattr' );
859 if ( !$latest || $stat[
'latest'] ) {
864 if ( is_array( $fields ) ) {
866 $this->cheapCache->setField(
869 [
'map' => $fields,
'latest' => $latest ]
871 } elseif ( $fields === self::RES_ABSENT ) {
872 $this->cheapCache->setField(
875 [
'map' => self::XATTRS_FAIL,
'latest' => $latest ]
878 $fields = self::XATTRS_FAIL;
891 return [
'headers' => [],
'metadata' => [] ];
899 if (
$path ===
null ) {
900 return self::SHA1_FAIL;
902 $latest = !empty(
$params[
'latest'] );
903 if ( $this->cheapCache->hasField(
$path,
'sha1', self::CACHE_TTL ) ) {
904 $stat = $this->cheapCache->getField(
$path,
'sha1' );
907 if ( !$latest || $stat[
'latest'] ) {
908 return $stat[
'hash'];
912 if ( is_string( $sha1 ) ) {
913 $this->cheapCache->setField(
916 [
'hash' => $sha1,
'latest' => $latest ]
918 } elseif ( $sha1 === self::RES_ABSENT ) {
919 $this->cheapCache->setField(
922 [
'hash' => self::SHA1_FAIL,
'latest' => $latest ]
925 $sha1 = self::SHA1_FAIL;
939 if ( $fsFile instanceof
FSFile ) {
940 $sha1 = $fsFile->getSha1Base36();
942 return is_string( $sha1 ) ? $sha1 : self::RES_ERROR;
945 return $fsFile === self::RES_ERROR ? self::RES_ERROR : self::RES_ABSENT;
954 return $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
964 $latest = !empty(
$params[
'latest'] );
966 foreach (
$params[
'srcs'] as $src ) {
968 if (
$path ===
null ) {
969 $fsFiles[$src] = self::RES_ERROR;
970 } elseif ( $this->expensiveCache->hasField(
$path,
'localRef' ) ) {
971 $val = $this->expensiveCache->getField(
$path,
'localRef' );
974 if ( !$latest || $val[
'latest'] ) {
975 $fsFiles[$src] = $val[
'object'];
980 $params[
'srcs'] = array_diff(
$params[
'srcs'], array_keys( $fsFiles ) );
982 if ( $fsFile instanceof
FSFile ) {
983 $fsFiles[
$path] = $fsFile;
984 $this->expensiveCache->setField(
987 [
'object' => $fsFile,
'latest' => $latest ]
991 $fsFiles[
$path] = $fsFile;
1031 return self::TEMPURL_ERROR;
1040 $params[
'options'] ??= [];
1044 if ( ( empty(
$params[
'headless'] ) ||
$params[
'headers'] ) && headers_sent() ) {
1045 print
"Headers already sent, terminating.\n";
1046 $status->fatal(
'backend-fail-stream',
$params[
'src'] );
1065 $flags |= !empty(
$params[
'headless'] ) ? HTTPFileStreamer::STREAM_HEADLESS : 0;
1066 $flags |= !empty(
$params[
'allowOB'] ) ? HTTPFileStreamer::STREAM_ALLOW_OB : 0;
1072 $this->getStreamerOptions()
1074 $res = $streamer->stream(
$params[
'headers'],
true,
$params[
'options'], $flags );
1077 HTTPFileStreamer::send404Message(
$params[
'src'], $flags );
1081 $status->fatal(
'backend-fail-stream',
$params[
'src'] );
1089 if ( $dir ===
null ) {
1090 return self::EXISTENCE_ERROR;
1092 if ( $shard !==
null ) {
1095 $this->logger->debug( __METHOD__ .
": iterating over all container shards." );
1100 if ( $exists ===
true ) {
1103 } elseif ( $exists === self::RES_ERROR ) {
1104 $res = self::EXISTENCE_ERROR;
1124 if ( $dir ===
null ) {
1125 return self::EXISTENCE_ERROR;
1127 if ( $shard !==
null ) {
1131 $this->logger->debug( __METHOD__ .
": iterating over all container shards." );
1154 if ( $dir ===
null ) {
1155 return self::LIST_ERROR;
1157 if ( $shard !==
null ) {
1161 $this->logger->debug( __METHOD__ .
": iterating over all container shards." );
1195 'store' => StoreFileOp::class,
1196 'copy' => CopyFileOp::class,
1197 'move' => MoveFileOp::class,
1198 'delete' => DeleteFileOp::class,
1199 'create' => CreateFileOp::class,
1200 'describe' => DescribeFileOp::class,
1201 'null' => NullFileOp::class
1206 foreach ( $ops as $operation ) {
1207 $opName = $operation[
'op'];
1208 if ( isset( $supportedOps[$opName] ) ) {
1209 $class = $supportedOps[$opName];
1213 $performOps[] =
new $class( $this,
$params, $this->logger );
1234 $paths = [
'sh' => [],
'ex' => [] ];
1235 foreach ( $performOps as $fileOp ) {
1236 $paths[
'sh'] = array_merge( $paths[
'sh'], $fileOp->storagePathsRead() );
1237 $paths[
'ex'] = array_merge( $paths[
'ex'], $fileOp->storagePathsChanged() );
1240 $paths[
'sh'] = array_diff( $paths[
'sh'], $paths[
'ex'] );
1242 $paths[
'sh'] = array_merge( $paths[
'sh'], array_map(
'dirname', $paths[
'ex'] ) );
1245 LockManager::LOCK_UW => $paths[
'sh'],
1246 LockManager::LOCK_EX => $paths[
'ex']
1262 $ops = array_map( [ $this,
'sanitizeOpHeaders' ], $ops );
1266 foreach ( $fileOps as $fileOp ) {
1267 $pathsUsed = array_merge( $pathsUsed, $fileOp->storagePathsReadOrChanged() );
1271 if ( empty( $opts[
'nonLocking'] ) ) {
1275 if ( !$status->isOK() ) {
1281 if ( empty( $opts[
'preserveCache'] ) ) {
1286 $this->cheapCache->setMaxSize( max( 2 * count( $pathsUsed ), self::CACHE_CHEAP_SIZE ) );
1291 $ok = $this->
preloadFileStat( [
'srcs' => $pathsUsed,
'latest' =>
true ] );
1299 $subStatus = $this->
newStatus(
'backend-fail-internal', $this->name );
1300 foreach ( $ops as $i => $op ) {
1301 $subStatus->success[$i] =
false;
1302 ++$subStatus->failCount;
1304 $this->logger->error( static::class .
"-{$this->name} " .
1305 " stat failure; aborted operations: " . FormatJson::encode( $ops ) );
1309 $status->merge( $subStatus );
1310 $status->success = $subStatus->success;
1313 $this->cheapCache->setMaxSize( self::CACHE_CHEAP_SIZE );
1324 $ops = array_map( [ $this,
'sanitizeOpHeaders' ], $ops );
1328 foreach ( $fileOps as $fileOp ) {
1329 $pathsUsed = array_merge( $pathsUsed, $fileOp->storagePathsReadOrChanged() );
1336 $async = ( $this->parallelize ===
'implicit' && count( $ops ) > 1 );
1342 foreach ( $fileOps as $index => $fileOp ) {
1344 ? $fileOp->attemptAsyncQuick()
1345 : $fileOp->attemptQuick();
1347 if ( count( $batch ) >= $maxConcurrency ) {
1353 $batch[$index] = $subStatus->value;
1355 $statuses[$index] = $subStatus;
1358 if ( count( $batch ) ) {
1362 foreach ( $statuses as $index => $subStatus ) {
1363 $status->merge( $subStatus );
1364 if ( $subStatus->isOK() ) {
1365 $status->success[$index] =
true;
1366 ++$status->successCount;
1368 $status->success[$index] =
false;
1369 ++$status->failCount;
1391 foreach ( $fileOpHandles as $fileOpHandle ) {
1393 throw new InvalidArgumentException(
"Expected FileBackendStoreOpHandle object." );
1394 } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) {
1395 throw new InvalidArgumentException(
"Expected handle for this file backend." );
1400 foreach ( $fileOpHandles as $fileOpHandle ) {
1401 $fileOpHandle->closeResources();
1417 if ( count( $fileOpHandles ) ) {
1418 throw new FileBackendError(
"Backend does not support asynchronous operations." );
1436 static $longs = [
'content-disposition' ];
1438 if ( isset( $op[
'headers'] ) ) {
1440 foreach ( $op[
'headers'] as
$name => $value ) {
1442 $maxHVLen = in_array(
$name, $longs ) ? INF : 255;
1443 if ( strlen(
$name ) > 255 || strlen( $value ) > $maxHVLen ) {
1444 $this->logger->error(
"Header '{header}' is too long.", [
1445 'filebackend' => $this->name,
1446 'header' =>
"$name: $value",
1449 $newHeaders[
$name] = strlen( $value ) ? $value :
'';
1452 $op[
'headers'] = $newHeaders;
1460 foreach ( $paths as
$path ) {
1462 $fullConts[] = $fullCont;
1470 if ( is_array( $paths ) ) {
1471 $paths = array_map( [ FileBackend::class,
'normalizeStoragePath' ], $paths );
1472 $paths = array_filter( $paths,
'strlen' );
1474 if ( $paths ===
null ) {
1475 $this->cheapCache->clear();
1476 $this->expensiveCache->clear();
1478 foreach ( $paths as
$path ) {
1479 $this->cheapCache->clear(
$path );
1480 $this->expensiveCache->clear(
$path );
1501 $params[
'concurrency'] = ( $this->parallelize !==
'off' ) ? $this->concurrency : 1;
1503 if ( $stats ===
null ) {
1508 $latest = !empty(
$params[
'latest'] );
1571 return (
bool)preg_match(
'/^[a-z0-9][a-z0-9-_.]{0,199}$/i', $container );
1589 if ( $backend === $this->name ) {
1591 if ( $relPath !==
null && self::isValidShortContainerName( $shortCont ) ) {
1596 if ( $relPath !==
null ) {
1599 if ( self::isValidContainerName( $container ) ) {
1602 if ( $container !==
null ) {
1603 return [ $container, $relPath, $cShard ];
1610 return [
null,
null, null ];
1630 if ( $cShard !==
null && substr( $relPath, -1 ) !==
'/' ) {
1631 return [ $container, $relPath ];
1634 return [
null, null ];
1647 if ( $levels == 1 || $levels == 2 ) {
1649 $char = ( $base == 36 ) ?
'[0-9a-z]' :
'[0-9a-f]';
1652 if ( $levels === 1 ) {
1653 $hashDirRegex =
'(' . $char .
')';
1656 $hashDirRegex = $char .
'/(' . $char .
'{2})';
1658 $hashDirRegex =
'(' . $char .
')/(' . $char .
')';
1665 if ( preg_match(
"!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1666 return '.' . implode(
'', array_slice( $m, 1 ) );
1686 return ( $shard !==
null );
1698 if ( isset( $this->shardViaHashLevels[$container] ) ) {
1699 $config = $this->shardViaHashLevels[$container];
1700 $hashLevels = (int)$config[
'levels'];
1701 if ( $hashLevels == 1 || $hashLevels == 2 ) {
1702 $hashBase = (int)$config[
'base'];
1703 if ( $hashBase == 16 || $hashBase == 36 ) {
1704 return [ $hashLevels, $hashBase, $config[
'repeat'] ];
1709 return [ 0, 0, false ];
1721 if ( $digits > 0 ) {
1722 $numShards = $base ** $digits;
1723 for ( $index = 0; $index < $numShards; $index++ ) {
1724 $shards[] =
'.' . Wikimedia\base_convert( (
string)$index, 10, $base, $digits );
1738 if ( $this->domainId !=
'' ) {
1739 return "{$this->domainId}-$container";
1770 return $relStoragePath;
1779 private function containerCacheKey( $container ) {
1780 return "filebackend:{$this->name}:{$this->domainId}:container:{$container}";
1790 if ( !$this->memCache->set( $this->containerCacheKey( $container ), $val, 14 * 86400 ) ) {
1791 $this->logger->warning(
"Unable to set stat cache for container {container}.",
1792 [
'filebackend' => $this->name,
'container' => $container ]
1804 if ( !$this->memCache->delete( $this->containerCacheKey( $container ), 300 ) ) {
1805 $this->logger->warning(
"Unable to delete stat cache for container {container}.",
1806 [
'filebackend' => $this->name,
'container' => $container ]
1825 foreach ( $items as $item ) {
1826 if ( self::isStoragePath( $item ) ) {
1828 } elseif ( is_string( $item ) ) {
1829 $contNames[$this->containerCacheKey( $item )] = $item;
1833 foreach ( $paths as
$path ) {
1835 if ( $fullCont !==
null ) {
1836 $contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
1842 $values = $this->memCache->getMulti( array_keys( $contNames ) );
1843 foreach ( $values as $cacheKey => $val ) {
1844 $contInfo[$contNames[$cacheKey]] = $val;
1868 private function fileCacheKey(
$path ) {
1869 return "filebackend:{$this->name}:{$this->domainId}:file:" . sha1(
$path );
1881 $path = FileBackend::normalizeStoragePath(
$path );
1882 if (
$path ===
null ) {
1885 $mtime = (int)ConvertibleTimestamp::convert( TS_UNIX, $val[
'mtime'] );
1886 $ttl = $this->memCache->adaptiveTTL( $mtime, 7 * 86400, 300, 0.1 );
1887 $key = $this->fileCacheKey(
$path );
1889 if ( !$this->memCache->set( $key, $val, $ttl ) ) {
1890 $this->logger->warning(
"Unable to set stat cache for file {path}.",
1891 [
'filebackend' => $this->name,
'path' =>
$path ]
1905 $path = FileBackend::normalizeStoragePath(
$path );
1906 if (
$path ===
null ) {
1909 if ( !$this->memCache->delete( $this->fileCacheKey(
$path ), 300 ) ) {
1910 $this->logger->warning(
"Unable to delete stat cache for file {path}.",
1911 [
'filebackend' => $this->name,
'path' =>
$path ]
1930 foreach ( $items as $item ) {
1931 if ( self::isStoragePath( $item ) ) {
1932 $path = FileBackend::normalizeStoragePath( $item );
1933 if (
$path !==
null ) {
1939 foreach ( $paths as
$path ) {
1941 if ( $rel !==
null ) {
1942 $pathNames[$this->fileCacheKey(
$path )] =
$path;
1947 $values = $this->memCache->getMulti( array_keys( $pathNames ) );
1949 foreach ( array_filter( $values,
'is_array' ) as $cacheKey => $stat ) {
1950 $path = $pathNames[$cacheKey];
1953 unset( $stat[
'latest'] );
1955 $this->cheapCache->setField(
$path,
'stat', $stat );
1956 if ( isset( $stat[
'sha1'] ) && strlen( $stat[
'sha1'] ) == 31 ) {
1958 $this->cheapCache->setField(
1961 [
'hash' => $stat[
'sha1'],
'latest' =>
false ]
1964 if ( isset( $stat[
'xattr'] ) && is_array( $stat[
'xattr'] ) ) {
1967 $this->cheapCache->setField(
1970 [
'map' => $stat[
'xattr'],
'latest' =>
false ]
1984 $newXAttr = [
'headers' => [],
'metadata' => [] ];
1986 foreach ( $xattr[
'headers'] as
$name => $value ) {
1987 $newXAttr[
'headers'][strtolower(
$name )] = $value;
1990 foreach ( $xattr[
'metadata'] as
$name => $value ) {
1991 $newXAttr[
'metadata'][strtolower(
$name )] = $value;
2004 $opts[
'concurrency'] = 1;
2005 if ( $this->parallelize ===
'implicit' ) {
2006 if ( $opts[
'parallelize'] ??
true ) {
2009 } elseif ( $this->parallelize ===
'explicit' ) {
2010 if ( !empty( $opts[
'parallelize'] ) ) {
2028 if ( $this->mimeCallback ) {
2029 return call_user_func_array( $this->mimeCallback, func_get_args() );
2032 $mime = ( $fsPath !== null ) ? mime_content_type( $fsPath ) :
false;
2033 return $mime ?:
'unknown/unknown';
array $params
The job parameters.
Class representing a non-directory file on the file system.
File backend exception for checked exceptions (e.g.
FileBackendStore helper class for performing asynchronous file operations.
Iterator for listing directories.
Iterator for listing regular files.
Base class for all backends using particular storage medium.
preloadFileStat(array $params)
Preload file stat information (concurrently if possible) into in-process cache.
doGetLocalReferenceMulti(array $params)
resolveContainerName( $container)
Resolve a container name, checking if it's allowed by the backend.
static normalizeXAttributes(array $xattr)
Normalize file headers/metadata to the FileBackend::getFileXAttributes() format.
getLocalReferenceMulti(array $params)
Like getLocalReference() except it takes an array of storage paths and yields an order-preserved map ...
setFileCache( $path, array $val)
Set the cached stat info for a file path.
moveInternal(array $params)
Move a file from one storage path to another in the backend.
setContainerCache( $container, array $val)
Set the cached info for a container.
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path in the backend.
doGetFileContentsMulti(array $params)
doStreamFile(array $params)
const CACHE_EXPENSIVE_SIZE
MapCacheLRU $expensiveCache
Map of paths to large (RAM/disk) cache items.
getFileTimestamp(array $params)
Get the last-modified timestamp of the file at a storage path.
doGetLocalCopyMulti(array $params)
getScopedLocksForOps(array $ops, StatusValue $status)
Get an array of scoped locks needed for a batch of file operations.
doCleanInternal( $container, $dir, array $params)
getPathsToLockForOpsInternal(array $performOps)
Get a list of storage paths to lock for a list of operations Returns an array with LockManager::LOCK_...
executeOpHandlesInternal(array $fileOpHandles)
Execute a list of FileBackendStoreOpHandle handles in parallel.
doPrepareInternal( $container, $dir, array $params)
concatenate(array $params)
Concatenate a list of storage files into a single file system file.
storeInternal(array $params)
Store a file into the backend from a file on disk.
getFileProps(array $params)
Get the properties of the content of the file at a storage path in the backend.
getDirectoryList(array $params)
Get an iterator to list all directories under a storage directory.
doQuickOperationsInternal(array $ops, array $opts)
resolveStoragePath( $storagePath)
Splits a storage path into an internal container name, an internal relative file name,...
copyInternal(array $params)
Copy a file from one storage path to another in the backend.
doStoreInternal(array $params)
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
doGetFileStat(array $params)
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
MapCacheLRU $cheapCache
Map of paths to small (RAM/disk) cache items.
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
doPublishInternal( $container, $dir, array $params)
doGetFileStatMulti(array $params)
Get file stat information (concurrently if possible) for several files.
getFileSha1Base36(array $params)
Get a SHA-1 hash of the content of the file at a storage path in the backend.
static isValidContainerName( $container)
Check if a full container name is valid.
ingestFreshFileStats(array $stats, $latest)
Ingest file stat entries that just came from querying the backend (not cache)
getFileListInternal( $container, $dir, array $params)
Do not call this function from places outside FileBackend.
doCopyInternal(array $params)
getLocalCopyMulti(array $params)
Like getLocalCopy() except it takes an array of storage paths and yields an order preserved-map of st...
getFileContentsMulti(array $params)
Like getFileContents() except it takes an array of storage paths and returns an order preserved map o...
doClearCache(array $paths=null)
Clears any additional stat caches for storage paths.
fullContainerName( $container)
Get the full container name, including the domain ID prefix.
primeFileCache(array $items)
Do a batch lookup from cache for file stats for all paths used in a list of storage paths or FileOp o...
doOperationsInternal(array $ops, array $opts)
isSingleShardPathInternal( $storagePath)
Check if a storage path maps to a single shard.
directoryExists(array $params)
Check if a directory exists at a given storage path.
doDeleteInternal(array $params)
getContainerSuffixes( $container)
Get a list of full container shard suffixes for a container.
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
sanitizeOpHeaders(array $op)
Normalize and filter HTTP headers from a file operation.
doDescribeInternal(array $params)
doGetFileSha1Base36(array $params)
deleteInternal(array $params)
Delete a file at the storage path.
doPrimeContainerCache(array $containerInfo)
Fill the backend-specific process cache given an array of resolved container names and their correspo...
doMoveInternal(array $params)
maxFileSizeInternal()
Get the maximum allowable file size given backend medium restrictions and basic performance constrain...
describeInternal(array $params)
Alter metadata for a file at the storage path.
streamFile(array $params)
Stream the content of the file at a storage path in the backend.
getOperationsInternal(array $ops)
Return a list of FileOp objects from a list of operations.
deleteFileCache( $path)
Delete the cached stat info for a file path.
setConcurrencyFlags(array $opts)
Set the 'concurrency' option from a list of operation options.
getFileXAttributes(array $params)
Get metadata about a file at a storage path in the backend.
getContentType( $storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
preloadCache(array $paths)
Preload persistent file stat cache and property cache into in-process cache.
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
doDirectoryExists( $container, $dir, array $params)
doPrepare(array $params)
FileBackend::prepare() StatusValue Good status without value for success, fatal otherwise.
getFileHttpUrl(array $params)
__construct(array $config)
doGetFileXAttributes(array $params)
doExecuteOpHandlesInternal(array $fileOpHandles)
nullInternal(array $params)
No-op file operation that does nothing.
array< string, array > $shardViaHashLevels
Map of container names to sharding config.
getFileSize(array $params)
Get the size (bytes) of a file at a storage path in the backend.
callable null $mimeCallback
Method to get the MIME type of files.
doConcatenate(array $params)
doSecureInternal( $container, $dir, array $params)
doCreateInternal(array $params)
fileExists(array $params)
Check if a file exists at a storage path in the backend.
getDirectoryListInternal( $container, $dir, array $params)
Do not call this function from places outside FileBackend.
static isValidShortContainerName( $container)
Check if a short container name is valid.
getContainerHashLevels( $container)
Get the sharding config for a container.
getContainerShard( $container, $relPath)
Get the container name shard suffix for a given path.
deleteContainerCache( $container)
Delete the cached info for a container.
createInternal(array $params)
Create a file in the backend with the given contents.
getFileList(array $params)
Get an iterator to list all stored files under a storage directory.
static attempt(array $performOps, array $opts)
Attempt to perform a series of file operations.
Functions related to the output of file content.
Store key-value entries in a size-limited in-memory LRU cache.
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Multi-datacenter aware caching interface.