LocalRepo.php

Go to the documentation of this file.

<?php
namespace MediaWiki\FileRepo;
 
use Closure;
use InvalidArgumentException;
use MediaWiki\Context\RequestContext;
use MediaWiki\FileRepo\File\File;
use MediaWiki\FileRepo\File\FileSelectQueryBuilder;
use MediaWiki\FileRepo\File\LocalFile;
use MediaWiki\FileRepo\File\OldLocalFile;
use MediaWiki\Linker\LinkTarget;
use MediaWiki\MediaWikiServices;
use MediaWiki\Page\PageIdentity;
use MediaWiki\Permissions\Authority;
use MediaWiki\Status\Status;
use MediaWiki\Storage\BlobStore;
use MediaWiki\Title\Title;
use MediaWiki\WikiMap\WikiMap;
use stdClass;
use Wikimedia\ObjectCache\WANObjectCache;
use Wikimedia\Rdbms\Database;
use Wikimedia\Rdbms\IConnectionProvider;
use Wikimedia\Rdbms\IDatabase;
use Wikimedia\Rdbms\IExpression;
use Wikimedia\Rdbms\IReadableDatabase;
use Wikimedia\Rdbms\IResultWrapper;
use Wikimedia\Rdbms\LikeValue;
 
class LocalRepo extends FileRepo {
    protected $fileFactory = [ LocalFile::class, 'newFromTitle' ];
    protected $fileFactoryKey = [ LocalFile::class, 'newFromKey' ];
    protected $fileFromRowFactory = [ LocalFile::class, 'newFromRow' ];
    protected $oldFileFromRowFactory = [ OldLocalFile::class, 'newFromRow' ];
    protected $oldFileFactory = [ OldLocalFile::class, 'newFromTitle' ];
    protected $oldFileFactoryKey = [ OldLocalFile::class, 'newFromKey' ];
 
    protected $dbDomain;
    protected IConnectionProvider $dbProvider;
    protected $hasAccessibleSharedCache;
 
    protected $blobStore;
 
    protected $useJsonMetadata = true;
 
    protected $useSplitMetadata = false;
 
    protected $splitMetadataThreshold = 1000;
 
    protected $updateCompatibleMetadata = false;
 
    protected $reserializeMetadata = false;
 
    public function __construct( ?array $info = null ) {
        parent::__construct( $info );
 
        $this->dbDomain = WikiMap::getCurrentWikiDbDomain();
        $this->hasAccessibleSharedCache = true;
 
        $this->hasSha1Storage = ( $info['storageLayout'] ?? null ) === 'sha1';
        $this->dbProvider = MediaWikiServices::getInstance()->getConnectionProvider();
 
        if ( $this->hasSha1Storage() ) {
            $this->backend = new FileBackendDBRepoWrapper( [
                'backend'         => $this->backend,
                'repoName'        => $this->name,
                'dbHandleFactory' => $this->getDBFactory()
            ] );
        }
 
        foreach (
            [
                'useJsonMetadata',
                'useSplitMetadata',
                'splitMetadataThreshold',
                'updateCompatibleMetadata',
                'reserializeMetadata',
            ] as $option
        ) {
            if ( isset( $info[$option] ) ) {
                $this->$option = $info[$option];
            }
        }
    }
 
    public function newFileFromRow( $row ) {
        if ( isset( $row->img_name ) ) {
            return ( $this->fileFromRowFactory )( $row, $this );
        } elseif ( isset( $row->oi_name ) ) {
            return ( $this->oldFileFromRowFactory )( $row, $this );
        } else {
            throw new InvalidArgumentException( __METHOD__ . ': invalid row' );
        }
    }
 
    public function newFromArchiveName( $title, $archiveName ) {
        $title = File::normalizeTitle( $title );
        return OldLocalFile::newFromArchiveName( $title, $this, $archiveName );
    }
 
    public function cleanupDeletedBatch( array $storageKeys ) {
        if ( $this->hasSha1Storage() ) {
            wfDebug( __METHOD__ . ": skipped because storage uses sha1 paths" );
            return Status::newGood();
        }
 
        $backend = $this->backend; // convenience
        $root = $this->getZonePath( 'deleted' );
        $dbw = $this->getPrimaryDB();
        $status = $this->newGood();
        $storageKeys = array_unique( $storageKeys );
        foreach ( $storageKeys as $key ) {
            $hashPath = $this->getDeletedHashPath( $key );
            $path = "$root/$hashPath$key";
            $dbw->startAtomic( __METHOD__ );
            // Check for usage in deleted/hidden files and preemptively
            // lock the key to avoid any future use until we are finished.
            $deleted = $this->deletedFileHasKey( $key, 'lock' );
            $hidden = $this->hiddenFileHasKey( $key, 'lock' );
            if ( !$deleted && !$hidden ) { // not in use now
                wfDebug( __METHOD__ . ": deleting $key" );
                $op = [ 'op' => 'delete', 'src' => $path ];
                if ( !$backend->doOperation( $op )->isOK() ) {
                    $status->error( 'undelete-cleanup-error', $path );
                    $status->failCount++;
                }
            } else {
                wfDebug( __METHOD__ . ": $key still in use" );
                $status->successCount++;
            }
            $dbw->endAtomic( __METHOD__ );
        }
 
        return $status;
    }
 
    protected function deletedFileHasKey( $key, $lock = null ) {
        $queryBuilder = $this->getPrimaryDB()->newSelectQueryBuilder()
            ->select( '1' )
            ->from( 'filearchive' )
            ->where( [ 'fa_storage_group' => 'deleted', 'fa_storage_key' => $key ] );
        if ( $lock === 'lock' ) {
            $queryBuilder->forUpdate();
        }
        return (bool)$queryBuilder->caller( __METHOD__ )->fetchField();
    }
 
    protected function hiddenFileHasKey( $key, $lock = null ) {
        $sha1 = self::getHashFromKey( $key );
        $ext = File::normalizeExtension( substr( $key, strcspn( $key, '.' ) + 1 ) );
 
        $dbw = $this->getPrimaryDB();
        $queryBuilder = $dbw->newSelectQueryBuilder()
            ->select( '1' )
            ->from( 'oldimage' )
            ->where( [
                'oi_sha1' => $sha1,
                $dbw->expr( 'oi_archive_name', IExpression::LIKE, new LikeValue( $dbw->anyString(), ".$ext" ) ),
                $dbw->bitAnd( 'oi_deleted', File::DELETED_FILE ) => File::DELETED_FILE,
            ] );
        if ( $lock === 'lock' ) {
            $queryBuilder->forUpdate();
        }
 
        return (bool)$queryBuilder->caller( __METHOD__ )->fetchField();
    }
 
    public static function getHashFromKey( $key ) {
        $sha1 = strtok( $key, '.' );
        if ( is_string( $sha1 ) && strlen( $sha1 ) === 32 && $sha1[0] === '0' ) {
            $sha1 = substr( $sha1, 1 );
        }
        return $sha1;
    }
 
    public function checkRedirect( $title ) {
        $title = File::normalizeTitle( $title, 'exception' );
 
        $memcKey = $this->getSharedCacheKey( 'file-redirect', md5( $title->getDBkey() ) );
        if ( $memcKey === false ) {
            $memcKey = $this->getLocalCacheKey( 'file-redirect', md5( $title->getDBkey() ) );
            $expiry = 300; // no invalidation, 5 minutes
        } else {
            $expiry = 86400; // has invalidation, 1 day
        }
 
        $method = __METHOD__;
        $redirDbKey = $this->wanCache->getWithSetCallback(
            $memcKey,
            $expiry,
            function ( $oldValue, &$ttl, array &$setOpts ) use ( $method, $title ) {
                $dbr = $this->getReplicaDB(); // possibly remote DB
 
                $setOpts += Database::getCacheSetOptions( $dbr );
 
                $row = $dbr->newSelectQueryBuilder()
                    ->select( [ 'rd_namespace', 'rd_title' ] )
                    ->from( 'page' )
                    ->join( 'redirect', null, 'rd_from = page_id' )
                    ->where( [ 'page_namespace' => $title->getNamespace(), 'page_title' => $title->getDBkey() ] )
                    ->caller( $method )->fetchRow();
 
                return ( $row && $row->rd_namespace == NS_FILE )
                    ? Title::makeTitle( $row->rd_namespace, $row->rd_title )->getDBkey()
                    : ''; // negative cache
            },
            [ 'pcTTL' => WANObjectCache::TTL_PROC_LONG ]
        );
 
        // @note: also checks " " for b/c
        if ( $redirDbKey !== ' ' && strval( $redirDbKey ) !== '' ) {
            // Page is a redirect to another file
            return Title::newFromText( $redirDbKey, NS_FILE );
        }
 
        return false; // no redirect
    }
 
    public function findFiles( array $items, $flags = 0 ) {
        $finalFiles = []; // map of (DB key => corresponding File) for matches
 
        $searchSet = []; // map of (normalized DB key => search params)
        foreach ( $items as $item ) {
            if ( is_array( $item ) ) {
                $title = File::normalizeTitle( $item['title'] );
                if ( $title ) {
                    $searchSet[$title->getDBkey()] = $item;
                }
            } else {
                $title = File::normalizeTitle( $item );
                if ( $title ) {
                    $searchSet[$title->getDBkey()] = [];
                }
            }
        }
 
        $fileMatchesSearch = static function ( File $file, array $search ) {
            // Note: file name comparison done elsewhere (to handle redirects)
 
            // Fallback to RequestContext::getMain should be replaced with a better
            // way of setting the user that should be used; currently it needs to be
            // set for each file individually. See T263033#6477586
            $contextPerformer = RequestContext::getMain()->getAuthority();
            $performer = ( !empty( $search['private'] ) && $search['private'] instanceof Authority )
                ? $search['private']
                : $contextPerformer;
 
            return (
                $file->exists() &&
                (
                    ( empty( $search['time'] ) && !$file->isOld() ) ||
                    ( !empty( $search['time'] ) && $search['time'] === $file->getTimestamp() )
                ) &&
                ( !empty( $search['private'] ) || !$file->isDeleted( File::DELETED_FILE ) ) &&
                $file->userCan( File::DELETED_FILE, $performer )
            );
        };
 
        $applyMatchingFiles = function ( IResultWrapper $res, &$searchSet, &$finalFiles )
            use ( $fileMatchesSearch, $flags )
        {
            $contLang = MediaWikiServices::getInstance()->getContentLanguage();
            $info = $this->getInfo();
            foreach ( $res as $row ) {
                $file = $this->newFileFromRow( $row );
                // There must have been a search for this DB key, but this has to handle the
                // cases were title capitalization is different on the client and repo wikis.
                $dbKeysLook = [ strtr( $file->getName(), ' ', '_' ) ];
                if ( !empty( $info['initialCapital'] ) ) {
                    // Search keys for "hi.png" and "Hi.png" should use the "Hi.png file"
                    $dbKeysLook[] = $contLang->lcfirst( $file->getName() );
                }
                foreach ( $dbKeysLook as $dbKey ) {
                    if ( isset( $searchSet[$dbKey] )
                        && $fileMatchesSearch( $file, $searchSet[$dbKey] )
                    ) {
                        $finalFiles[$dbKey] = ( $flags & FileRepo::NAME_AND_TIME_ONLY )
                            ? [ 'title' => $dbKey, 'timestamp' => $file->getTimestamp() ]
                            : $file;
                        unset( $searchSet[$dbKey] );
                    }
                }
            }
        };
 
        $dbr = $this->getReplicaDB();
 
        // Query image table
        $imgNames = [];
        foreach ( $searchSet as $dbKey => $_ ) {
            $imgNames[] = $this->getNameFromTitle( File::normalizeTitle( $dbKey ) );
        }
 
        if ( count( $imgNames ) ) {
            $queryBuilder = FileSelectQueryBuilder::newForFile( $dbr );
            $res = $queryBuilder->where( [ 'img_name' => $imgNames ] )->caller( __METHOD__ )->fetchResultSet();
            $applyMatchingFiles( $res, $searchSet, $finalFiles );
        }
 
        // Query old image table
        $oiConds = []; // WHERE clause array for each file
        foreach ( $searchSet as $dbKey => $search ) {
            if ( isset( $search['time'] ) ) {
                $oiConds[] = $dbr
                    ->expr( 'oi_name', '=', $this->getNameFromTitle( File::normalizeTitle( $dbKey ) ) )
                    ->and( 'oi_timestamp', '=', $dbr->timestamp( $search['time'] ) );
            }
        }
 
        if ( count( $oiConds ) ) {
            $queryBuilder = FileSelectQueryBuilder::newForOldFile( $dbr );
 
            $res = $queryBuilder->where( $dbr->orExpr( $oiConds ) )
                ->caller( __METHOD__ )->fetchResultSet();
            $applyMatchingFiles( $res, $searchSet, $finalFiles );
        }
 
        // Check for redirects...
        foreach ( $searchSet as $dbKey => $search ) {
            if ( !empty( $search['ignoreRedirect'] ) ) {
                continue;
            }
 
            $title = File::normalizeTitle( $dbKey );
            $redir = $this->checkRedirect( $title ); // hopefully hits memcached
 
            if ( $redir && $redir->getNamespace() === NS_FILE ) {
                $file = $this->newFile( $redir );
                if ( $file && $fileMatchesSearch( $file, $search ) ) {
                    $file->redirectedFrom( $title->getDBkey() );
                    if ( $flags & FileRepo::NAME_AND_TIME_ONLY ) {
                        $finalFiles[$dbKey] = [
                            'title' => $file->getTitle()->getDBkey(),
                            'timestamp' => $file->getTimestamp()
                        ];
                    } else {
                        $finalFiles[$dbKey] = $file;
                    }
                }
            }
        }
 
        return $finalFiles;
    }
 
    public function findBySha1( $hash ) {
        $queryBuilder = FileSelectQueryBuilder::newForFile( $this->getReplicaDB() );
        $res = $queryBuilder->where( [ 'img_sha1' => $hash ] )
            ->orderBy( 'img_name' )
            ->caller( __METHOD__ )->fetchResultSet();
 
        $result = [];
        foreach ( $res as $row ) {
            $result[] = $this->newFileFromRow( $row );
        }
        $res->free();
 
        return $result;
    }
 
    public function findBySha1s( array $hashes ) {
        if ( $hashes === [] ) {
            return []; // empty parameter
        }
 
        $dbr = $this->getReplicaDB();
        $queryBuilder = FileSelectQueryBuilder::newForFile( $dbr );
 
        $queryBuilder->where( [ 'img_sha1' => $hashes ] )
            ->orderBy( 'img_name' );
        $res = $queryBuilder->caller( __METHOD__ )->fetchResultSet();
 
        $result = [];
        foreach ( $res as $row ) {
            $file = $this->newFileFromRow( $row );
            $result[$file->getSha1()][] = $file;
        }
        $res->free();
 
        return $result;
    }
 
    public function findFilesByPrefix( $prefix, $limit ) {
        $dbr = $this->getReplicaDB();
        $queryBuilder = FileSelectQueryBuilder::newForFile( $dbr );
 
        $queryBuilder
            ->where( $dbr->expr( 'img_name', IExpression::LIKE, new LikeValue( $prefix, $dbr->anyString() ) ) )
            ->orderBy( 'img_name' )
            ->limit( intval( $limit ) );
        $res = $queryBuilder->caller( __METHOD__ )->fetchResultSet();
 
        // Build file objects
        $files = [];
        foreach ( $res as $row ) {
            $files[] = $this->newFileFromRow( $row );
        }
 
        return $files;
    }
 
    public function getReplicaDB() {
        return $this->dbProvider->getReplicaDatabase();
    }
 
    public function getPrimaryDB() {
        return $this->dbProvider->getPrimaryDatabase();
    }
 
    protected function getDBFactory() {
        // TODO: DB_REPLICA/DB_PRIMARY shouldn't be passed around
        return static function ( $index ) {
            if ( $index === DB_PRIMARY ) {
                return MediaWikiServices::getInstance()->getConnectionProvider()->getPrimaryDatabase();
            } else {
                return MediaWikiServices::getInstance()->getConnectionProvider()->getReplicaDatabase();
            }
        };
    }
 
    protected function hasAcessibleSharedCache() {
        return $this->hasAccessibleSharedCache;
    }
 
    public function getSharedCacheKey( $kClassSuffix, ...$components ) {
        // T267668: do not include the repo name in the key
        return $this->hasAcessibleSharedCache()
            ? $this->wanCache->makeGlobalKey(
                'filerepo-' . $kClassSuffix,
                $this->dbDomain,
                ...$components
            )
            : false;
    }
 
    public function invalidateImageRedirect( $title ) {
        $key = $this->getSharedCacheKey( 'file-redirect', md5( $title->getDBkey() ) );
        if ( $key ) {
            $this->getPrimaryDB()->onTransactionPreCommitOrIdle(
                function () use ( $key ) {
                    $this->wanCache->delete( $key );
                },
                __METHOD__
            );
        }
    }
 
    public function store( $srcPath, $dstZone, $dstRel, $flags = 0 ) {
        return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
    }
 
    public function storeBatch( array $triplets, $flags = 0 ) {
        return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
    }
 
    public function cleanupBatch( array $files, $flags = 0 ) {
        return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
    }
 
    public function publish(
        $src,
        $dstRel,
        $archiveRel,
        $flags = 0,
        array $options = []
    ) {
        return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
    }
 
    public function publishBatch( array $ntuples, $flags = 0 ) {
        return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
    }
 
    public function delete( $srcRel, $archiveRel ) {
        return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
    }
 
    public function deleteBatch( array $sourceDestPairs ) {
        return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
    }
 
    protected function skipWriteOperationIfSha1( $function, array $args ) {
        $this->assertWritableRepo(); // fail out if read-only
 
        if ( $this->hasSha1Storage() ) {
            wfDebug( __METHOD__ . ": skipped because storage uses sha1 paths" );
            return Status::newGood();
        } else {
            return parent::$function( ...$args );
        }
    }
 
    public function isJsonMetadataEnabled() {
        return $this->useJsonMetadata;
    }
 
    public function isSplitMetadataEnabled() {
        return $this->isJsonMetadataEnabled() && $this->useSplitMetadata;
    }
 
    public function getSplitMetadataThreshold() {
        return $this->splitMetadataThreshold;
    }
 
    public function isMetadataUpdateEnabled() {
        return $this->updateCompatibleMetadata;
    }
 
    public function isMetadataReserializeEnabled() {
        return $this->reserializeMetadata;
    }
 
    public function getBlobStore(): ?BlobStore {
        if ( !$this->blobStore ) {
            $this->blobStore = MediaWikiServices::getInstance()->getBlobStoreFactory()
                ->newBlobStore( $this->dbDomain );
        }
        return $this->blobStore;
    }
}
 
class_alias( LocalRepo::class, 'LocalRepo' );