FileBackend.php

Go to the documentation of this file.

<?php
namespace Wikimedia\FileBackend;
 
use InvalidArgumentException;
use LockManager;
use NullLockManager;
use Psr\Log\LoggerAwareInterface;
use Psr\Log\LoggerInterface;
use Psr\Log\NullLogger;
use ScopedLock;
use Shellbox\Command\BoxedCommand;
use StatusValue;
use Wikimedia\FileBackend\FSFile\FSFile;
use Wikimedia\FileBackend\FSFile\TempFSFile;
use Wikimedia\FileBackend\FSFile\TempFSFileFactory;
use Wikimedia\Message\MessageParam;
use Wikimedia\Message\MessageSpecifier;
use Wikimedia\ScopedCallback;
 
abstract class FileBackend implements LoggerAwareInterface {
    protected $name;
 
    protected $domainId;
 
    protected $readOnly;
 
    protected $parallelize;
 
    protected $concurrency;
 
    protected $tmpFileFactory;
 
    protected $lockManager;
    protected $logger;
    protected $profiler;
 
    private $obResetFunc;
    private $headerFunc;
    private $asyncHandler;
    protected $streamerOptions;
    protected $statusWrapper;
 
    public const ATTR_HEADERS = 1; // files can be tagged with standard HTTP headers
    public const ATTR_METADATA = 2; // files can be stored with metadata key/values
    public const ATTR_UNICODE_PATHS = 4; // files can have Unicode paths (not just ASCII)
 
    protected const STAT_ABSENT = false;
 
    public const STAT_ERROR = null;
    public const LIST_ERROR = null;
    public const TEMPURL_ERROR = null;
    public const EXISTENCE_ERROR = null;
 
    public const TIMESTAMP_FAIL = false;
    public const CONTENT_FAIL = false;
    public const XATTRS_FAIL = false;
    public const SIZE_FAIL = false;
    public const SHA1_FAIL = false;
 
    public function __construct( array $config ) {
        if ( !array_key_exists( 'name', $config ) ) {
            throw new InvalidArgumentException( 'Backend name not specified.' );
        }
        $this->name = $config['name'];
        $this->domainId = $config['domainId'] // e.g. "my_wiki-en_"
            ?? $config['wikiId'] // b/c alias
            ?? null;
        if ( !is_string( $this->name ) || !preg_match( '!^[a-zA-Z0-9-_]{1,255}$!', $this->name ) ) {
            throw new InvalidArgumentException( "Backend name '{$this->name}' is invalid." );
        }
        if ( !is_string( $this->domainId ) ) {
            throw new InvalidArgumentException(
                "Backend domain ID not provided for '{$this->name}'." );
        }
        $this->lockManager = $config['lockManager'] ?? new NullLockManager( [] );
        $this->readOnly = isset( $config['readOnly'] )
            ? (string)$config['readOnly']
            : '';
        $this->parallelize = isset( $config['parallelize'] )
            ? (string)$config['parallelize']
            : 'off';
        $this->concurrency = isset( $config['concurrency'] )
            ? (int)$config['concurrency']
            : 50;
        $this->obResetFunc = $config['obResetFunc']
            ?? [ self::class, 'resetOutputBufferTheDefaultWay' ];
        $this->headerFunc = $config['headerFunc'] ?? 'header';
        $this->asyncHandler = $config['asyncHandler'] ?? null;
        $this->streamerOptions = [
            'obResetFunc' => $this->obResetFunc,
            'headerFunc' => $this->headerFunc,
            'streamMimeFunc' => $config['streamMimeFunc'] ?? null,
        ];
 
        $this->profiler = $config['profiler'] ?? null;
        if ( !is_callable( $this->profiler ) ) {
            $this->profiler = null;
        }
        $this->logger = $config['logger'] ?? new NullLogger();
        $this->statusWrapper = $config['statusWrapper'] ?? null;
        // tmpDirectory gets precedence for backward compatibility
        if ( isset( $config['tmpDirectory'] ) ) {
            $this->tmpFileFactory = new TempFSFileFactory( $config['tmpDirectory'] );
        } else {
            $this->tmpFileFactory = $config['tmpFileFactory'] ?? new TempFSFileFactory();
        }
    }
 
    protected function header( $header ) {
        ( $this->headerFunc )( $header );
    }
 
    protected function callNowOrLater( callable $update ) {
        if ( $this->asyncHandler ) {
            ( $this->asyncHandler )( $update );
        } else {
            $update();
        }
    }
 
    protected function resetOutputBuffer() {
        // By default, this ends up calling $this->defaultOutputBufferReset
        ( $this->obResetFunc )();
    }
 
    public function setLogger( LoggerInterface $logger ) {
        $this->logger = $logger;
    }
 
    final public function getName() {
        return $this->name;
    }
 
    final public function getDomainId() {
        return $this->domainId;
    }
 
    final public function getWikiId() {
        return $this->getDomainId();
    }
 
    final public function isReadOnly() {
        return ( $this->readOnly != '' );
    }
 
    final public function getReadOnlyReason() {
        return ( $this->readOnly != '' ) ? $this->readOnly : false;
    }
 
    public function getFeatures() {
        return self::ATTR_UNICODE_PATHS;
    }
 
    final public function hasFeatures( $bitfield ) {
        return ( $this->getFeatures() & $bitfield ) === $bitfield;
    }
 
    final public function doOperations( array $ops, array $opts = [] ) {
        if ( empty( $opts['bypassReadOnly'] ) && $this->isReadOnly() ) {
            return $this->newStatus( 'backend-fail-readonly', $this->name, $this->readOnly );
        }
        if ( $ops === [] ) {
            return $this->newStatus(); // nothing to do
        }
 
        $ops = $this->resolveFSFileObjects( $ops );
        if ( empty( $opts['force'] ) ) {
            unset( $opts['nonLocking'] );
        }
 
        $scope = ScopedCallback::newScopedIgnoreUserAbort(); // try to ignore client aborts
 
        return $this->doOperationsInternal( $ops, $opts );
    }
 
    abstract protected function doOperationsInternal( array $ops, array $opts );
 
    final public function doOperation( array $op, array $opts = [] ) {
        return $this->doOperations( [ $op ], $opts );
    }
 
    final public function create( array $params, array $opts = [] ) {
        return $this->doOperation( [ 'op' => 'create' ] + $params, $opts );
    }
 
    final public function store( array $params, array $opts = [] ) {
        return $this->doOperation( [ 'op' => 'store' ] + $params, $opts );
    }
 
    final public function copy( array $params, array $opts = [] ) {
        return $this->doOperation( [ 'op' => 'copy' ] + $params, $opts );
    }
 
    final public function move( array $params, array $opts = [] ) {
        return $this->doOperation( [ 'op' => 'move' ] + $params, $opts );
    }
 
    final public function delete( array $params, array $opts = [] ) {
        return $this->doOperation( [ 'op' => 'delete' ] + $params, $opts );
    }
 
    final public function describe( array $params, array $opts = [] ) {
        return $this->doOperation( [ 'op' => 'describe' ] + $params, $opts );
    }
 
    final public function doQuickOperations( array $ops, array $opts = [] ) {
        if ( empty( $opts['bypassReadOnly'] ) && $this->isReadOnly() ) {
            return $this->newStatus( 'backend-fail-readonly', $this->name, $this->readOnly );
        }
        if ( $ops === [] ) {
            return $this->newStatus(); // nothing to do
        }
 
        $ops = $this->resolveFSFileObjects( $ops );
        foreach ( $ops as &$op ) {
            $op['overwrite'] = true; // avoids RTTs in key/value stores
        }
 
        $scope = ScopedCallback::newScopedIgnoreUserAbort(); // try to ignore client aborts
 
        return $this->doQuickOperationsInternal( $ops, $opts );
    }
 
    abstract protected function doQuickOperationsInternal( array $ops, array $opts );
 
    final public function doQuickOperation( array $op, array $opts = [] ) {
        return $this->doQuickOperations( [ $op ], $opts );
    }
 
    final public function quickCreate( array $params, array $opts = [] ) {
        return $this->doQuickOperation( [ 'op' => 'create' ] + $params, $opts );
    }
 
    final public function quickStore( array $params, array $opts = [] ) {
        return $this->doQuickOperation( [ 'op' => 'store' ] + $params, $opts );
    }
 
    final public function quickCopy( array $params, array $opts = [] ) {
        return $this->doQuickOperation( [ 'op' => 'copy' ] + $params, $opts );
    }
 
    final public function quickMove( array $params, array $opts = [] ) {
        return $this->doQuickOperation( [ 'op' => 'move' ] + $params, $opts );
    }
 
    final public function quickDelete( array $params, array $opts = [] ) {
        return $this->doQuickOperation( [ 'op' => 'delete' ] + $params, $opts );
    }
 
    final public function quickDescribe( array $params, array $opts = [] ) {
        return $this->doQuickOperation( [ 'op' => 'describe' ] + $params, $opts );
    }
 
    abstract public function concatenate( array $params );
 
    final public function prepare( array $params ) {
        if ( empty( $params['bypassReadOnly'] ) && $this->isReadOnly() ) {
            return $this->newStatus( 'backend-fail-readonly', $this->name, $this->readOnly );
        }
        $scope = ScopedCallback::newScopedIgnoreUserAbort(); // try to ignore client aborts
        return $this->doPrepare( $params );
    }
 
    abstract protected function doPrepare( array $params );
 
    final public function secure( array $params ) {
        if ( empty( $params['bypassReadOnly'] ) && $this->isReadOnly() ) {
            return $this->newStatus( 'backend-fail-readonly', $this->name, $this->readOnly );
        }
        $scope = ScopedCallback::newScopedIgnoreUserAbort(); // try to ignore client aborts
        return $this->doSecure( $params );
    }
 
    abstract protected function doSecure( array $params );
 
    final public function publish( array $params ) {
        if ( empty( $params['bypassReadOnly'] ) && $this->isReadOnly() ) {
            return $this->newStatus( 'backend-fail-readonly', $this->name, $this->readOnly );
        }
        $scope = ScopedCallback::newScopedIgnoreUserAbort(); // try to ignore client aborts
        return $this->doPublish( $params );
    }
 
    abstract protected function doPublish( array $params );
 
    final public function clean( array $params ) {
        if ( empty( $params['bypassReadOnly'] ) && $this->isReadOnly() ) {
            return $this->newStatus( 'backend-fail-readonly', $this->name, $this->readOnly );
        }
        $scope = ScopedCallback::newScopedIgnoreUserAbort(); // try to ignore client aborts
        return $this->doClean( $params );
    }
 
    abstract protected function doClean( array $params );
 
    abstract public function fileExists( array $params );
 
    abstract public function getFileTimestamp( array $params );
 
    final public function getFileContents( array $params ) {
        $contents = $this->getFileContentsMulti( [ 'srcs' => [ $params['src'] ] ] + $params );
 
        return $contents[$params['src']];
    }
 
    abstract public function getFileContentsMulti( array $params );
 
    abstract public function getFileXAttributes( array $params );
 
    abstract public function getFileSize( array $params );
 
    abstract public function getFileStat( array $params );
 
    abstract public function getFileSha1Base36( array $params );
 
    abstract public function getFileProps( array $params );
 
    abstract public function streamFile( array $params );
 
    final public function getLocalReference( array $params ) {
        $fsFiles = $this->getLocalReferenceMulti( [ 'srcs' => [ $params['src'] ] ] + $params );
 
        return $fsFiles[$params['src']];
    }
 
    abstract public function getLocalReferenceMulti( array $params );
 
    final public function getLocalCopy( array $params ) {
        $tmpFiles = $this->getLocalCopyMulti( [ 'srcs' => [ $params['src'] ] ] + $params );
 
        return $tmpFiles[$params['src']];
    }
 
    abstract public function getLocalCopyMulti( array $params );
 
    abstract public function getFileHttpUrl( array $params );
 
    abstract public function addShellboxInputFile( BoxedCommand $command, string $boxedName,
        array $params );
 
    abstract public function directoryExists( array $params );
 
    abstract public function getDirectoryList( array $params );
 
    final public function getTopDirectoryList( array $params ) {
        return $this->getDirectoryList( [ 'topOnly' => true ] + $params );
    }
 
    abstract public function getFileList( array $params );
 
    final public function getTopFileList( array $params ) {
        return $this->getFileList( [ 'topOnly' => true ] + $params );
    }
 
    abstract public function preloadCache( array $paths );
 
    abstract public function clearCache( ?array $paths = null );
 
    abstract public function preloadFileStat( array $params );
 
    final public function lockFiles( array $paths, $type, $timeout = 0 ) {
        $paths = array_map( [ __CLASS__, 'normalizeStoragePath' ], $paths );
 
        return $this->wrapStatus( $this->lockManager->lock( $paths, $type, $timeout ) );
    }
 
    final public function unlockFiles( array $paths, $type ) {
        $paths = array_map( [ __CLASS__, 'normalizeStoragePath' ], $paths );
 
        return $this->wrapStatus( $this->lockManager->unlock( $paths, $type ) );
    }
 
    final public function getScopedFileLocks(
        array $paths, $type, StatusValue $status, $timeout = 0
    ) {
        if ( $type === 'mixed' ) {
            foreach ( $paths as &$typePaths ) {
                $typePaths = array_map( [ __CLASS__, 'normalizeStoragePath' ], $typePaths );
            }
        } else {
            $paths = array_map( [ __CLASS__, 'normalizeStoragePath' ], $paths );
        }
 
        return ScopedLock::factory( $this->lockManager, $paths, $type, $status, $timeout );
    }
 
    abstract public function getScopedLocksForOps( array $ops, StatusValue $status );
 
    final public function getRootStoragePath() {
        return "mwstore://{$this->name}";
    }
 
    final public function getContainerStoragePath( $container ) {
        return $this->getRootStoragePath() . "/{$container}";
    }
 
    protected function resolveFSFileObjects( array $ops ) {
        foreach ( $ops as &$op ) {
            $src = $op['src'] ?? null;
            if ( $src instanceof FSFile ) {
                $op['srcRef'] = $src;
                $op['src'] = $src->getPath();
            }
        }
        unset( $op );
 
        return $ops;
    }
 
    final public static function isStoragePath( $path ) {
        return ( strpos( $path ?? '', 'mwstore://' ) === 0 );
    }
 
    final public static function splitStoragePath( $storagePath ) {
        if ( self::isStoragePath( $storagePath ) ) {
            // Remove the "mwstore://" prefix and split the path
            $parts = explode( '/', substr( $storagePath, 10 ), 3 );
            if ( count( $parts ) >= 2 && $parts[0] != '' && $parts[1] != '' ) {
                if ( count( $parts ) == 3 ) {
                    return $parts; // e.g. "backend/container/path"
                } else {
                    return [ $parts[0], $parts[1], '' ]; // e.g. "backend/container"
                }
            }
        }
 
        return [ null, null, null ];
    }
 
    final public static function normalizeStoragePath( $storagePath ) {
        [ $backend, $container, $relPath ] = self::splitStoragePath( $storagePath );
        if ( $relPath !== null ) { // must be for this backend
            $relPath = self::normalizeContainerPath( $relPath );
            if ( $relPath !== null ) {
                return ( $relPath != '' )
                    ? "mwstore://{$backend}/{$container}/{$relPath}"
                    : "mwstore://{$backend}/{$container}";
            }
        }
 
        return null;
    }
 
    final public static function parentStoragePath( $storagePath ) {
        // XXX dirname() depends on platform and locale! If nothing enforces that the storage path
        // doesn't contain characters like '\', behavior can vary by platform. We should use
        // explode() instead.
        $storagePath = dirname( $storagePath );
        [ , , $rel ] = self::splitStoragePath( $storagePath );
 
        return ( $rel === null ) ? null : $storagePath;
    }
 
    final public static function extensionFromPath( $path, $case = 'lowercase' ) {
        // This will treat a string starting with . as not having an extension, but store paths have
        // to start with 'mwstore://', so "garbage in, garbage out".
        $i = strrpos( $path, '.' );
        $ext = $i ? substr( $path, $i + 1 ) : '';
 
        if ( $case === 'lowercase' ) {
            $ext = strtolower( $ext );
        } elseif ( $case === 'uppercase' ) {
            $ext = strtoupper( $ext );
        }
 
        return $ext;
    }
 
    final public static function isPathTraversalFree( $path ) {
        return ( self::normalizeContainerPath( $path ) !== null );
    }
 
    final public static function makeContentDisposition( $type, $filename = '' ) {
        $parts = [];
 
        $type = strtolower( $type );
        if ( !in_array( $type, [ 'inline', 'attachment' ] ) ) {
            throw new InvalidArgumentException( "Invalid Content-Disposition type '$type'." );
        }
        $parts[] = $type;
 
        if ( $filename !== '' ) {
            $parts[] = "filename*=UTF-8''" . rawurlencode( basename( $filename ) );
        }
 
        return implode( ';', $parts );
    }
 
    final protected static function normalizeContainerPath( $path ) {
        // Normalize directory separators
        $path = strtr( $path, '\\', '/' );
        // Collapse any consecutive directory separators
        $path = preg_replace( '![/]{2,}!', '/', $path );
        // Remove any leading directory separator
        $path = ltrim( $path, '/' );
        // Use the same traversal protection as Title::secureAndSplit()
        if ( strpos( $path, '.' ) !== false ) {
            if (
                $path === '.' ||
                $path === '..' ||
                strpos( $path, './' ) === 0 ||
                strpos( $path, '../' ) === 0 ||
                strpos( $path, '/./' ) !== false ||
                strpos( $path, '/../' ) !== false
            ) {
                return null;
            }
        }
 
        return $path;
    }
 
    final protected function newStatus( $message = null, ...$params ) {
        if ( $message !== null ) {
            $sv = StatusValue::newFatal( $message, ...$params );
        } else {
            $sv = StatusValue::newGood();
        }
 
        return $this->wrapStatus( $sv );
    }
 
    final protected function wrapStatus( StatusValue $sv ) {
        return $this->statusWrapper ? ( $this->statusWrapper )( $sv ) : $sv;
    }
 
    protected function scopedProfileSection( $section ) {
        return $this->profiler ? ( $this->profiler )( $section ) : null;
    }
 
    public static function resetOutputBufferTheDefaultWay() {
        // XXX According to documentation, ob_get_status() always returns a non-empty array and this
        // condition will always be true
        while ( ob_get_status() ) {
            if ( !ob_end_clean() ) {
                // Could not remove output buffer handler; abort now
                // to avoid getting in some kind of infinite loop.
                break;
            }
        }
    }
 
    public function getStreamerOptions(): array {
        return $this->streamerOptions;
    }
}
class_alias( FileBackend::class, 'FileBackend' );