24use Wikimedia\AtEase\AtEase;
25use Wikimedia\Timestamp\ConvertibleTimestamp;
64 protected const RES_ABSENT =
false;
66 protected const RES_ERROR =
null;
69 protected const ABSENT_NORMAL =
'FNE-N';
71 protected const ABSENT_LATEST =
'FNE-L';
87 parent::__construct( $config );
88 $this->mimeCallback = $config[
'mimeCallback'] ??
null;
90 $this->memCache = WANObjectCache::newEmpty();
91 $this->cheapCache =
new MapCacheLRU( self::CACHE_CHEAP_SIZE );
92 $this->expensiveCache =
new MapCacheLRU( self::CACHE_EXPENSIVE_SIZE );
103 return min( $this->maxFileSize, PHP_INT_MAX );
141 $status = $this->
newStatus(
'backend-fail-maxsize',
146 if (
$params[
'dstExists'] ??
true ) {
184 $status = $this->
newStatus(
'backend-fail-maxsize',
189 if (
$params[
'dstExists'] ??
true ) {
229 if (
$params[
'dstExists'] ??
true ) {
300 if (
$params[
'dstExists'] ??
true ) {
332 if ( count(
$params[
'headers'] ) ) {
371 $scopeLockS = $this->
getScopedFileLocks( $params[
'srcs'], LockManager::LOCK_UW, $status );
372 if ( $status->isOK() ) {
374 $start_time = microtime(
true );
376 $sec = microtime(
true ) - $start_time;
377 if ( !$status->isOK() ) {
378 $this->logger->error( static::class .
"-{$this->name}" .
379 " failed to concatenate " . count(
$params[
'srcs'] ) .
" file(s) [$sec sec]" );
398 AtEase::suppressWarnings();
399 $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 );
400 AtEase::restoreWarnings();
402 $status->fatal(
'backend-fail-opentemp', $tmpPath );
409 foreach ( $fsFiles as
$path => &$fsFile ) {
414 $fsFile === self::RES_ERROR ?
'backend-fail-read' :
'backend-fail-notexists',
425 $tmpHandle = fopen( $tmpPath,
'ab' );
426 if ( $tmpHandle ===
false ) {
427 $status->fatal(
'backend-fail-opentemp', $tmpPath );
433 foreach ( $fsFiles as $virtualSource => $fsFile ) {
435 $sourceHandle = fopen( $fsFile->getPath(),
'rb' );
436 if ( $sourceHandle ===
false ) {
437 fclose( $tmpHandle );
438 $status->fatal(
'backend-fail-read', $virtualSource );
443 if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
444 fclose( $sourceHandle );
445 fclose( $tmpHandle );
446 $status->fatal(
'backend-fail-writetemp', $tmpPath );
450 fclose( $sourceHandle );
452 if ( !fclose( $tmpHandle ) ) {
453 $status->fatal(
'backend-fail-closetemp', $tmpPath );
472 if ( $dir ===
null ) {
473 $status->fatal(
'backend-fail-invalidpath',
$params[
'dir'] );
478 if ( $shard !==
null ) {
481 $this->logger->debug( __METHOD__ .
": iterating over all container shards." );
509 if ( $dir ===
null ) {
510 $status->fatal(
'backend-fail-invalidpath',
$params[
'dir'] );
515 if ( $shard !==
null ) {
518 $this->logger->debug( __METHOD__ .
": iterating over all container shards." );
546 if ( $dir ===
null ) {
547 $status->fatal(
'backend-fail-invalidpath',
$params[
'dir'] );
552 if ( $shard !==
null ) {
555 $this->logger->debug( __METHOD__ .
": iterating over all container shards." );
585 if ( $subDirsRel !==
null ) {
586 foreach ( $subDirsRel as $subDirRel ) {
587 $subDir =
$params[
'dir'] .
"/{$subDirRel}";
588 $status->merge( $this->
doClean( [
'dir' => $subDir ] +
$params ) );
590 unset( $subDirsRel );
595 if ( $dir ===
null ) {
596 $status->fatal(
'backend-fail-invalidpath',
$params[
'dir'] );
602 $filesLockEx = [
$params[
'dir'] ];
604 $scopedLockE = $this->
getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
605 if ( !$status->isOK() ) {
609 if ( $shard !==
null ) {
613 $this->logger->debug( __METHOD__ .
": iterating over all container shards." );
641 if ( is_array( $stat ) ) {
645 return $stat === self::RES_ABSENT ? false : self::EXISTENCE_ERROR;
653 if ( is_array( $stat ) ) {
654 return $stat[
'mtime'];
657 return self::TIMESTAMP_FAIL;
665 if ( is_array( $stat ) ) {
666 return $stat[
'size'];
669 return self::SIZE_FAIL;
677 if (
$path ===
null ) {
678 return self::STAT_ERROR;
683 $latest = !empty(
$params[
'latest'] );
685 $requireSHA1 = !empty(
$params[
'requireSHA1'] );
687 $stat = $this->cheapCache->getField(
$path,
'stat', self::CACHE_TTL );
696 ( $requireSHA1 && is_array( $stat ) && !isset( $stat[
'sha1'] ) )
700 $stat = $this->cheapCache->getField(
$path,
'stat', self::CACHE_TTL );
704 if ( is_array( $stat ) ) {
706 ( !$latest || !empty( $stat[
'latest'] ) ) &&
707 ( !$requireSHA1 || isset( $stat[
'sha1'] ) )
711 } elseif ( $stat === self::ABSENT_LATEST ) {
712 return self::STAT_ABSENT;
713 } elseif ( $stat === self::ABSENT_NORMAL ) {
715 return self::STAT_ABSENT;
723 if ( is_array( $stat ) ) {
727 return $stat === self::RES_ERROR ? self::STAT_ERROR : self::STAT_ABSENT;
740 foreach ( $stats as
$path => $stat ) {
741 if ( is_array( $stat ) ) {
743 $stat[
'latest'] ??= $latest;
745 $this->cheapCache->setField(
$path,
'stat', $stat );
746 if ( isset( $stat[
'sha1'] ) ) {
748 $this->cheapCache->setField(
751 [
'hash' => $stat[
'sha1'],
'latest' => $latest ]
754 if ( isset( $stat[
'xattr'] ) ) {
757 $this->cheapCache->setField(
760 [
'map' => $stat[
'xattr'],
'latest' => $latest ]
765 } elseif ( $stat === self::RES_ABSENT ) {
766 $this->cheapCache->setField(
769 $latest ? self::ABSENT_LATEST : self::ABSENT_NORMAL
771 $this->cheapCache->setField(
774 [
'map' => self::XATTRS_FAIL,
'latest' => $latest ]
776 $this->cheapCache->setField(
779 [
'hash' => self::SHA1_FAIL,
'latest' => $latest ]
781 $this->logger->debug(
782 __METHOD__ .
': File {path} does not exist',
787 $this->logger->error(
788 __METHOD__ .
': Could not stat file {path}',
810 foreach ( $contents as
$path => $content ) {
811 if ( !is_string( $content ) ) {
812 $contents[
$path] = self::CONTENT_FAIL;
828 if ( $fsFile instanceof
FSFile ) {
829 AtEase::suppressWarnings();
830 $content = file_get_contents( $fsFile->getPath() );
831 AtEase::restoreWarnings();
832 $contents[
$path] = is_string( $content ) ? $content : self::RES_ERROR;
835 $contents[
$path] = $fsFile;
847 if (
$path ===
null ) {
848 return self::XATTRS_FAIL;
850 $latest = !empty(
$params[
'latest'] );
851 if ( $this->cheapCache->hasField(
$path,
'xattr', self::CACHE_TTL ) ) {
852 $stat = $this->cheapCache->getField(
$path,
'xattr' );
855 if ( !$latest || $stat[
'latest'] ) {
860 if ( is_array( $fields ) ) {
862 $this->cheapCache->setField(
865 [
'map' => $fields,
'latest' => $latest ]
867 } elseif ( $fields === self::RES_ABSENT ) {
868 $this->cheapCache->setField(
871 [
'map' => self::XATTRS_FAIL,
'latest' => $latest ]
874 $fields = self::XATTRS_FAIL;
887 return [
'headers' => [],
'metadata' => [] ];
895 if (
$path ===
null ) {
896 return self::SHA1_FAIL;
898 $latest = !empty(
$params[
'latest'] );
899 if ( $this->cheapCache->hasField(
$path,
'sha1', self::CACHE_TTL ) ) {
900 $stat = $this->cheapCache->getField(
$path,
'sha1' );
903 if ( !$latest || $stat[
'latest'] ) {
904 return $stat[
'hash'];
908 if ( is_string( $sha1 ) ) {
909 $this->cheapCache->setField(
912 [
'hash' => $sha1,
'latest' => $latest ]
914 } elseif ( $sha1 === self::RES_ABSENT ) {
915 $this->cheapCache->setField(
918 [
'hash' => self::SHA1_FAIL,
'latest' => $latest ]
921 $sha1 = self::SHA1_FAIL;
935 if ( $fsFile instanceof
FSFile ) {
936 $sha1 = $fsFile->getSha1Base36();
938 return is_string( $sha1 ) ? $sha1 : self::RES_ERROR;
941 return $fsFile === self::RES_ERROR ? self::RES_ERROR : self::RES_ABSENT;
960 $latest = !empty(
$params[
'latest'] );
962 foreach (
$params[
'srcs'] as $src ) {
964 if (
$path ===
null ) {
965 $fsFiles[$src] = self::RES_ERROR;
966 } elseif ( $this->expensiveCache->hasField(
$path,
'localRef' ) ) {
967 $val = $this->expensiveCache->getField(
$path,
'localRef' );
970 if ( !$latest || $val[
'latest'] ) {
971 $fsFiles[$src] = $val[
'object'];
976 $params[
'srcs'] = array_diff(
$params[
'srcs'], array_keys( $fsFiles ) );
978 if ( $fsFile instanceof
FSFile ) {
979 $fsFiles[
$path] = $fsFile;
980 $this->expensiveCache->setField(
983 [
'object' => $fsFile,
'latest' => $latest ]
987 $fsFiles[
$path] = $fsFile;
1027 return self::TEMPURL_ERROR;
1036 $params[
'options'] ??= [];
1040 if ( ( empty(
$params[
'headless'] ) ||
$params[
'headers'] ) && headers_sent() ) {
1041 print
"Headers already sent, terminating.\n";
1042 $status->fatal(
'backend-fail-stream',
$params[
'src'] );
1061 $flags |= !empty(
$params[
'headless'] ) ? HTTPFileStreamer::STREAM_HEADLESS : 0;
1062 $flags |= !empty(
$params[
'allowOB'] ) ? HTTPFileStreamer::STREAM_ALLOW_OB : 0;
1068 $this->getStreamerOptions()
1070 $res = $streamer->stream(
$params[
'headers'],
true,
$params[
'options'], $flags );
1073 HTTPFileStreamer::send404Message(
$params[
'src'], $flags );
1077 $status->fatal(
'backend-fail-stream',
$params[
'src'] );
1085 if ( $dir ===
null ) {
1086 return self::EXISTENCE_ERROR;
1088 if ( $shard !==
null ) {
1091 $this->logger->debug( __METHOD__ .
": iterating over all container shards." );
1096 if ( $exists ===
true ) {
1099 } elseif ( $exists === self::RES_ERROR ) {
1100 $res = self::EXISTENCE_ERROR;
1120 if ( $dir ===
null ) {
1121 return self::EXISTENCE_ERROR;
1123 if ( $shard !==
null ) {
1127 $this->logger->debug( __METHOD__ .
": iterating over all container shards." );
1150 if ( $dir ===
null ) {
1151 return self::LIST_ERROR;
1153 if ( $shard !==
null ) {
1157 $this->logger->debug( __METHOD__ .
": iterating over all container shards." );
1191 'store' => StoreFileOp::class,
1192 'copy' => CopyFileOp::class,
1193 'move' => MoveFileOp::class,
1194 'delete' => DeleteFileOp::class,
1195 'create' => CreateFileOp::class,
1196 'describe' => DescribeFileOp::class,
1197 'null' => NullFileOp::class
1202 foreach ( $ops as $operation ) {
1203 $opName = $operation[
'op'];
1204 if ( isset( $supportedOps[$opName] ) ) {
1205 $class = $supportedOps[$opName];
1209 $performOps[] =
new $class( $this,
$params, $this->logger );
1230 $paths = [
'sh' => [],
'ex' => [] ];
1231 foreach ( $performOps as $fileOp ) {
1232 $paths[
'sh'] = array_merge( $paths[
'sh'], $fileOp->storagePathsRead() );
1233 $paths[
'ex'] = array_merge( $paths[
'ex'], $fileOp->storagePathsChanged() );
1236 $paths[
'sh'] = array_diff( $paths[
'sh'], $paths[
'ex'] );
1238 $paths[
'sh'] = array_merge( $paths[
'sh'], array_map(
'dirname', $paths[
'ex'] ) );
1241 LockManager::LOCK_UW => $paths[
'sh'],
1242 LockManager::LOCK_EX => $paths[
'ex']
1258 $ops = array_map( [ $this,
'sanitizeOpHeaders' ], $ops );
1262 foreach ( $fileOps as $fileOp ) {
1263 $pathsUsed = array_merge( $pathsUsed, $fileOp->storagePathsReadOrChanged() );
1267 if ( empty( $opts[
'nonLocking'] ) ) {
1271 if ( !$status->isOK() ) {
1277 if ( empty( $opts[
'preserveCache'] ) ) {
1282 $this->cheapCache->setMaxSize( max( 2 * count( $pathsUsed ), self::CACHE_CHEAP_SIZE ) );
1287 $ok = $this->
preloadFileStat( [
'srcs' => $pathsUsed,
'latest' =>
true ] );
1295 $subStatus = $this->
newStatus(
'backend-fail-internal', $this->name );
1296 foreach ( $ops as $i => $op ) {
1297 $subStatus->success[$i] =
false;
1298 ++$subStatus->failCount;
1300 $this->logger->error( static::class .
"-{$this->name} " .
1301 " stat failure; aborted operations: " . FormatJson::encode( $ops ) );
1305 $status->merge( $subStatus );
1306 $status->success = $subStatus->success;
1309 $this->cheapCache->setMaxSize( self::CACHE_CHEAP_SIZE );
1320 $ops = array_map( [ $this,
'sanitizeOpHeaders' ], $ops );
1324 foreach ( $fileOps as $fileOp ) {
1325 $pathsUsed = array_merge( $pathsUsed, $fileOp->storagePathsReadOrChanged() );
1332 $async = ( $this->parallelize ===
'implicit' && count( $ops ) > 1 );
1338 foreach ( $fileOps as $index => $fileOp ) {
1340 ? $fileOp->attemptAsyncQuick()
1341 : $fileOp->attemptQuick();
1343 if ( count( $batch ) >= $maxConcurrency ) {
1349 $batch[$index] = $subStatus->value;
1351 $statuses[$index] = $subStatus;
1354 if ( count( $batch ) ) {
1358 foreach ( $statuses as $index => $subStatus ) {
1359 $status->merge( $subStatus );
1360 if ( $subStatus->isOK() ) {
1361 $status->success[$index] =
true;
1362 ++$status->successCount;
1364 $status->success[$index] =
false;
1365 ++$status->failCount;
1387 foreach ( $fileOpHandles as $fileOpHandle ) {
1389 throw new InvalidArgumentException(
"Expected FileBackendStoreOpHandle object." );
1390 } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) {
1391 throw new InvalidArgumentException(
"Expected handle for this file backend." );
1396 foreach ( $fileOpHandles as $fileOpHandle ) {
1397 $fileOpHandle->closeResources();
1413 if ( count( $fileOpHandles ) ) {
1414 throw new FileBackendError(
"Backend does not support asynchronous operations." );
1432 static $longs = [
'content-disposition' ];
1434 if ( isset( $op[
'headers'] ) ) {
1436 foreach ( $op[
'headers'] as
$name => $value ) {
1438 $maxHVLen = in_array(
$name, $longs ) ? INF : 255;
1439 if ( strlen(
$name ) > 255 || strlen( $value ) > $maxHVLen ) {
1440 $this->logger->error(
"Header '{header}' is too long.", [
1441 'filebackend' => $this->name,
1442 'header' =>
"$name: $value",
1445 $newHeaders[
$name] = strlen( $value ) ? $value :
'';
1448 $op[
'headers'] = $newHeaders;
1456 foreach ( $paths as
$path ) {
1458 $fullConts[] = $fullCont;
1466 if ( is_array( $paths ) ) {
1467 $paths = array_map( [ FileBackend::class,
'normalizeStoragePath' ], $paths );
1468 $paths = array_filter( $paths,
'strlen' );
1470 if ( $paths ===
null ) {
1471 $this->cheapCache->clear();
1472 $this->expensiveCache->clear();
1474 foreach ( $paths as
$path ) {
1475 $this->cheapCache->clear(
$path );
1476 $this->expensiveCache->clear(
$path );
1497 $params[
'concurrency'] = ( $this->parallelize !==
'off' ) ? $this->concurrency : 1;
1499 if ( $stats ===
null ) {
1504 $latest = !empty(
$params[
'latest'] );
1567 return (
bool)preg_match(
'/^[a-z0-9][a-z0-9-_.]{0,199}$/i', $container );
1585 if ( $backend === $this->name ) {
1587 if ( $relPath !==
null && self::isValidShortContainerName( $shortCont ) ) {
1592 if ( $relPath !==
null ) {
1595 if ( self::isValidContainerName( $container ) ) {
1598 if ( $container !==
null ) {
1599 return [ $container, $relPath, $cShard ];
1606 return [
null,
null, null ];
1626 if ( $cShard !==
null && substr( $relPath, -1 ) !==
'/' ) {
1627 return [ $container, $relPath ];
1630 return [
null, null ];
1643 if ( $levels == 1 || $levels == 2 ) {
1645 $char = ( $base == 36 ) ?
'[0-9a-z]' :
'[0-9a-f]';
1648 if ( $levels === 1 ) {
1649 $hashDirRegex =
'(' . $char .
')';
1652 $hashDirRegex = $char .
'/(' . $char .
'{2})';
1654 $hashDirRegex =
'(' . $char .
')/(' . $char .
')';
1661 if ( preg_match(
"!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1662 return '.' . implode(
'', array_slice( $m, 1 ) );
1682 return ( $shard !==
null );
1694 if ( isset( $this->shardViaHashLevels[$container] ) ) {
1695 $config = $this->shardViaHashLevels[$container];
1696 $hashLevels = (int)$config[
'levels'];
1697 if ( $hashLevels == 1 || $hashLevels == 2 ) {
1698 $hashBase = (int)$config[
'base'];
1699 if ( $hashBase == 16 || $hashBase == 36 ) {
1700 return [ $hashLevels, $hashBase, $config[
'repeat'] ];
1705 return [ 0, 0, false ];
1717 if ( $digits > 0 ) {
1718 $numShards = $base ** $digits;
1719 for ( $index = 0; $index < $numShards; $index++ ) {
1720 $shards[] =
'.' . Wikimedia\base_convert( (
string)$index, 10, $base, $digits );
1734 if ( $this->domainId !=
'' ) {
1735 return "{$this->domainId}-$container";
1766 return $relStoragePath;
1775 private function containerCacheKey( $container ) {
1776 return "filebackend:{$this->name}:{$this->domainId}:container:{$container}";
1786 if ( !$this->memCache->set( $this->containerCacheKey( $container ), $val, 14 * 86400 ) ) {
1787 $this->logger->warning(
"Unable to set stat cache for container {container}.",
1788 [
'filebackend' => $this->name,
'container' => $container ]
1800 if ( !$this->memCache->delete( $this->containerCacheKey( $container ), 300 ) ) {
1801 $this->logger->warning(
"Unable to delete stat cache for container {container}.",
1802 [
'filebackend' => $this->name,
'container' => $container ]
1821 foreach ( $items as $item ) {
1822 if ( self::isStoragePath( $item ) ) {
1824 } elseif ( is_string( $item ) ) {
1825 $contNames[$this->containerCacheKey( $item )] = $item;
1829 foreach ( $paths as
$path ) {
1831 if ( $fullCont !==
null ) {
1832 $contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
1838 $values = $this->memCache->getMulti( array_keys( $contNames ) );
1839 foreach ( $values as $cacheKey => $val ) {
1840 $contInfo[$contNames[$cacheKey]] = $val;
1864 private function fileCacheKey(
$path ) {
1865 return "filebackend:{$this->name}:{$this->domainId}:file:" . sha1(
$path );
1878 if (
$path ===
null ) {
1881 $mtime = (int)ConvertibleTimestamp::convert( TS_UNIX, $val[
'mtime'] );
1882 $ttl = $this->memCache->adaptiveTTL( $mtime, 7 * 86400, 300, 0.1 );
1883 $key = $this->fileCacheKey(
$path );
1885 if ( !$this->memCache->set( $key, $val, $ttl ) ) {
1886 $this->logger->warning(
"Unable to set stat cache for file {path}.",
1887 [
'filebackend' => $this->name,
'path' =>
$path ]
1902 if (
$path ===
null ) {
1905 if ( !$this->memCache->delete( $this->fileCacheKey(
$path ), 300 ) ) {
1906 $this->logger->warning(
"Unable to delete stat cache for file {path}.",
1907 [
'filebackend' => $this->name,
'path' =>
$path ]
1926 foreach ( $items as $item ) {
1927 if ( self::isStoragePath( $item ) ) {
1929 if (
$path !==
null ) {
1935 foreach ( $paths as
$path ) {
1937 if ( $rel !==
null ) {
1938 $pathNames[$this->fileCacheKey(
$path )] =
$path;
1943 $values = $this->memCache->getMulti( array_keys( $pathNames ) );
1945 foreach ( array_filter( $values,
'is_array' ) as $cacheKey => $stat ) {
1946 $path = $pathNames[$cacheKey];
1949 unset( $stat[
'latest'] );
1951 $this->cheapCache->setField(
$path,
'stat', $stat );
1952 if ( isset( $stat[
'sha1'] ) && strlen( $stat[
'sha1'] ) == 31 ) {
1954 $this->cheapCache->setField(
1957 [
'hash' => $stat[
'sha1'],
'latest' =>
false ]
1960 if ( isset( $stat[
'xattr'] ) && is_array( $stat[
'xattr'] ) ) {
1963 $this->cheapCache->setField(
1966 [
'map' => $stat[
'xattr'],
'latest' =>
false ]
1980 $newXAttr = [
'headers' => [],
'metadata' => [] ];
1982 foreach ( $xattr[
'headers'] as
$name => $value ) {
1983 $newXAttr[
'headers'][strtolower(
$name )] = $value;
1986 foreach ( $xattr[
'metadata'] as
$name => $value ) {
1987 $newXAttr[
'metadata'][strtolower(
$name )] = $value;
2000 $opts[
'concurrency'] = 1;
2001 if ( $this->parallelize ===
'implicit' ) {
2002 if ( $opts[
'parallelize'] ??
true ) {
2005 } elseif ( $this->parallelize ===
'explicit' ) {
2006 if ( !empty( $opts[
'parallelize'] ) ) {
2024 if ( $this->mimeCallback ) {
2025 return call_user_func_array( $this->mimeCallback, func_get_args() );
2028 $mime = ( $fsPath !== null ) ? mime_content_type( $fsPath ) :
false;
2029 return $mime ?:
'unknown/unknown';
array $params
The job parameters.
Class representing a cache/ephemeral data store.
A BagOStuff object with no objects in it.
Class representing a non-directory file on the file system.
static placeholderProps()
Placeholder file properties to use for files that don't exist.
File backend exception for checked exceptions (e.g.
FileBackendStore helper class for performing asynchronous file operations.
Iterator for listing directories.
Iterator for listing regular files.
Base class for all backends using particular storage medium.
preloadFileStat(array $params)
Preload file stat information (concurrently if possible) into in-process cache.
doGetLocalReferenceMulti(array $params)
resolveContainerName( $container)
Resolve a container name, checking if it's allowed by the backend.
static normalizeXAttributes(array $xattr)
Normalize file headers/metadata to the FileBackend::getFileXAttributes() format.
getLocalReferenceMulti(array $params)
Like getLocalReference() except it takes an array of storage paths and yields an order-preserved map ...
setFileCache( $path, array $val)
Set the cached stat info for a file path.
moveInternal(array $params)
Move a file from one storage path to another in the backend.
setContainerCache( $container, array $val)
Set the cached info for a container.
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path in the backend.
doGetFileContentsMulti(array $params)
doStreamFile(array $params)
const CACHE_EXPENSIVE_SIZE
MapCacheLRU $expensiveCache
Map of paths to large (RAM/disk) cache items.
getFileTimestamp(array $params)
Get the last-modified timestamp of the file at a storage path.
doGetLocalCopyMulti(array $params)
getScopedLocksForOps(array $ops, StatusValue $status)
Get an array of scoped locks needed for a batch of file operations.
doCleanInternal( $container, $dir, array $params)
getPathsToLockForOpsInternal(array $performOps)
Get a list of storage paths to lock for a list of operations Returns an array with LockManager::LOCK_...
executeOpHandlesInternal(array $fileOpHandles)
Execute a list of FileBackendStoreOpHandle handles in parallel.
doPrepareInternal( $container, $dir, array $params)
concatenate(array $params)
Concatenate a list of storage files into a single file system file.
storeInternal(array $params)
Store a file into the backend from a file on disk.
getFileProps(array $params)
Get the properties of the content of the file at a storage path in the backend.
getDirectoryList(array $params)
Get an iterator to list all directories under a storage directory.
doQuickOperationsInternal(array $ops, array $opts)
resolveStoragePath( $storagePath)
Splits a storage path into an internal container name, an internal relative file name,...
copyInternal(array $params)
Copy a file from one storage path to another in the backend.
doStoreInternal(array $params)
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
doGetFileStat(array $params)
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
MapCacheLRU $cheapCache
Map of paths to small (RAM/disk) cache items.
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
doPublishInternal( $container, $dir, array $params)
doGetFileStatMulti(array $params)
Get file stat information (concurrently if possible) for several files.
getFileSha1Base36(array $params)
Get a SHA-1 hash of the content of the file at a storage path in the backend.
static isValidContainerName( $container)
Check if a full container name is valid.
ingestFreshFileStats(array $stats, $latest)
Ingest file stat entries that just came from querying the backend (not cache)
getFileListInternal( $container, $dir, array $params)
Do not call this function from places outside FileBackend.
doCopyInternal(array $params)
getLocalCopyMulti(array $params)
Like getLocalCopy() except it takes an array of storage paths and yields an order preserved-map of st...
getFileContentsMulti(array $params)
Like getFileContents() except it takes an array of storage paths and returns an order preserved map o...
doClearCache(array $paths=null)
Clears any additional stat caches for storage paths.
fullContainerName( $container)
Get the full container name, including the domain ID prefix.
primeFileCache(array $items)
Do a batch lookup from cache for file stats for all paths used in a list of storage paths or FileOp o...
doOperationsInternal(array $ops, array $opts)
isSingleShardPathInternal( $storagePath)
Check if a storage path maps to a single shard.
directoryExists(array $params)
Check if a directory exists at a given storage path.
doDeleteInternal(array $params)
getContainerSuffixes( $container)
Get a list of full container shard suffixes for a container.
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
sanitizeOpHeaders(array $op)
Normalize and filter HTTP headers from a file operation.
doDescribeInternal(array $params)
doGetFileSha1Base36(array $params)
deleteInternal(array $params)
Delete a file at the storage path.
doPrimeContainerCache(array $containerInfo)
Fill the backend-specific process cache given an array of resolved container names and their correspo...
doMoveInternal(array $params)
maxFileSizeInternal()
Get the maximum allowable file size given backend medium restrictions and basic performance constrain...
describeInternal(array $params)
Alter metadata for a file at the storage path.
streamFile(array $params)
Stream the content of the file at a storage path in the backend.
getOperationsInternal(array $ops)
Return a list of FileOp objects from a list of operations.
deleteFileCache( $path)
Delete the cached stat info for a file path.
setConcurrencyFlags(array $opts)
Set the 'concurrency' option from a list of operation options.
getFileXAttributes(array $params)
Get metadata about a file at a storage path in the backend.
getContentType( $storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
preloadCache(array $paths)
Preload persistent file stat cache and property cache into in-process cache.
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
doDirectoryExists( $container, $dir, array $params)
doPrepare(array $params)
FileBackend::prepare() StatusValue Good status without value for success, fatal otherwise.
getFileHttpUrl(array $params)
__construct(array $config)
doGetFileXAttributes(array $params)
doExecuteOpHandlesInternal(array $fileOpHandles)
nullInternal(array $params)
No-op file operation that does nothing.
array< string, array > $shardViaHashLevels
Map of container names to sharding config.
getFileSize(array $params)
Get the size (bytes) of a file at a storage path in the backend.
callable null $mimeCallback
Method to get the MIME type of files.
doConcatenate(array $params)
doSecureInternal( $container, $dir, array $params)
doCreateInternal(array $params)
fileExists(array $params)
Check if a file exists at a storage path in the backend.
getDirectoryListInternal( $container, $dir, array $params)
Do not call this function from places outside FileBackend.
static isValidShortContainerName( $container)
Check if a short container name is valid.
getContainerHashLevels( $container)
Get the sharding config for a container.
getContainerShard( $container, $relPath)
Get the container name shard suffix for a given path.
deleteContainerCache( $container)
Delete the cached info for a container.
createInternal(array $params)
Create a file in the backend with the given contents.
getFileList(array $params)
Get an iterator to list all stored files under a storage directory.
Base class for all file backend classes (including multi-write backends).
string $name
Unique backend name.
static splitStoragePath( $storagePath)
Split a storage path into a backend name, a container name, and a relative file path.
static normalizeContainerPath( $path)
Validate and normalize a relative storage path.
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
getTopDirectoryList(array $params)
Same as FileBackend::getDirectoryList() except only lists directories that are immediately under the ...
scopedProfileSection( $section)
newStatus(... $args)
Yields the result of the status wrapper callback on either:
static normalizeStoragePath( $storagePath)
Normalize a storage path by cleaning up directory separators.
int $concurrency
How many operations can be done in parallel.
getLocalReference(array $params)
Returns a file system file, identical in content to the file at a storage path.
static attempt(array $performOps, array $opts)
Attempt to perform a series of file operations.
Functions related to the output of file content.
Store key-value entries in a size-limited in-memory LRU cache.
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Multi-datacenter aware caching interface.