LocalRepo.php

Go to the documentation of this file.

<?php
use MediaWiki\MediaWikiServices;
use Wikimedia\Rdbms\IResultWrapper;
use Wikimedia\Rdbms\Database;
use Wikimedia\Rdbms\IDatabase;

class LocalRepo extends FileRepo {
    protected $fileFactory = [ LocalFile::class, 'newFromTitle' ];
    protected $fileFactoryKey = [ LocalFile::class, 'newFromKey' ];
    protected $fileFromRowFactory = [ LocalFile::class, 'newFromRow' ];
    protected $oldFileFromRowFactory = [ OldLocalFile::class, 'newFromRow' ];
    protected $oldFileFactory = [ OldLocalFile::class, 'newFromTitle' ];
    protected $oldFileFactoryKey = [ OldLocalFile::class, 'newFromKey' ];

    function __construct( array $info = null ) {
        parent::__construct( $info );

        $this->hasSha1Storage = isset( $info['storageLayout'] )
            && $info['storageLayout'] === 'sha1';

        if ( $this->hasSha1Storage() ) {
            $this->backend = new FileBackendDBRepoWrapper( [
                'backend'         => $this->backend,
                'repoName'        => $this->name,
                'dbHandleFactory' => $this->getDBFactory()
            ] );
        }
    }

    function newFileFromRow( $row ) {
        if ( isset( $row->img_name ) ) {
            return call_user_func( $this->fileFromRowFactory, $row, $this );
        } elseif ( isset( $row->oi_name ) ) {
            return call_user_func( $this->oldFileFromRowFactory, $row, $this );
        } else {
            throw new MWException( __METHOD__ . ': invalid row' );
        }
    }

    function newFromArchiveName( $title, $archiveName ) {
        return OldLocalFile::newFromArchiveName( $title, $this, $archiveName );
    }

    function cleanupDeletedBatch( array $storageKeys ) {
        if ( $this->hasSha1Storage() ) {
            wfDebug( __METHOD__ . ": skipped because storage uses sha1 paths\n" );
            return Status::newGood();
        }

        $backend = $this->backend; // convenience
        $root = $this->getZonePath( 'deleted' );
        $dbw = $this->getMasterDB();
        $status = $this->newGood();
        $storageKeys = array_unique( $storageKeys );
        foreach ( $storageKeys as $key ) {
            $hashPath = $this->getDeletedHashPath( $key );
            $path = "$root/$hashPath$key";
            $dbw->startAtomic( __METHOD__ );
            // Check for usage in deleted/hidden files and preemptively
            // lock the key to avoid any future use until we are finished.
            $deleted = $this->deletedFileHasKey( $key, 'lock' );
            $hidden = $this->hiddenFileHasKey( $key, 'lock' );
            if ( !$deleted && !$hidden ) { // not in use now
                wfDebug( __METHOD__ . ": deleting $key\n" );
                $op = [ 'op' => 'delete', 'src' => $path ];
                if ( !$backend->doOperation( $op )->isOK() ) {
                    $status->error( 'undelete-cleanup-error', $path );
                    $status->failCount++;
                }
            } else {
                wfDebug( __METHOD__ . ": $key still in use\n" );
                $status->successCount++;
            }
            $dbw->endAtomic( __METHOD__ );
        }

        return $status;
    }

    protected function deletedFileHasKey( $key, $lock = null ) {
        $options = ( $lock === 'lock' ) ? [ 'FOR UPDATE' ] : [];

        $dbw = $this->getMasterDB();

        return (bool)$dbw->selectField( 'filearchive', '1',
            [ 'fa_storage_group' => 'deleted', 'fa_storage_key' => $key ],
            __METHOD__, $options
        );
    }

    protected function hiddenFileHasKey( $key, $lock = null ) {
        $options = ( $lock === 'lock' ) ? [ 'FOR UPDATE' ] : [];

        $sha1 = self::getHashFromKey( $key );
        $ext = File::normalizeExtension( substr( $key, strcspn( $key, '.' ) + 1 ) );

        $dbw = $this->getMasterDB();

        return (bool)$dbw->selectField( 'oldimage', '1',
            [ 'oi_sha1' => $sha1,
                'oi_archive_name ' . $dbw->buildLike( $dbw->anyString(), ".$ext" ),
                $dbw->bitAnd( 'oi_deleted', File::DELETED_FILE ) => File::DELETED_FILE ],
            __METHOD__, $options
        );
    }

    public static function getHashFromKey( $key ) {
        $sha1 = strtok( $key, '.' );
        if ( is_string( $sha1 ) && strlen( $sha1 ) === 32 && $sha1[0] === '0' ) {
            $sha1 = substr( $sha1, 1 );
        }
        return $sha1;
    }

    function checkRedirect( Title $title ) {
        $title = File::normalizeTitle( $title, 'exception' );

        $memcKey = $this->getSharedCacheKey( 'image_redirect', md5( $title->getDBkey() ) );
        if ( $memcKey === false ) {
            $memcKey = $this->getLocalCacheKey( 'image_redirect', md5( $title->getDBkey() ) );
            $expiry = 300; // no invalidation, 5 minutes
        } else {
            $expiry = 86400; // has invalidation, 1 day
        }

        $method = __METHOD__;
        $redirDbKey = $this->wanCache->getWithSetCallback(
            $memcKey,
            $expiry,
            function ( $oldValue, &$ttl, array &$setOpts ) use ( $method, $title ) {
                $dbr = $this->getReplicaDB(); // possibly remote DB

                $setOpts += Database::getCacheSetOptions( $dbr );

                $row = $dbr->selectRow(
                    [ 'page', 'redirect' ],
                    [ 'rd_namespace', 'rd_title' ],
                    [
                        'page_namespace' => $title->getNamespace(),
                        'page_title' => $title->getDBkey(),
                        'rd_from = page_id'
                    ],
                    $method
                );

                return ( $row && $row->rd_namespace == NS_FILE )
                    ? Title::makeTitle( $row->rd_namespace, $row->rd_title )->getDBkey()
                    : ''; // negative cache
            },
            [ 'pcTTL' => WANObjectCache::TTL_PROC_LONG ]
        );

        // @note: also checks " " for b/c
        if ( $redirDbKey !== ' ' && strval( $redirDbKey ) !== '' ) {
            // Page is a redirect to another file
            return Title::newFromText( $redirDbKey, NS_FILE );
        }

        return false; // no redirect
    }

    public function findFiles( array $items, $flags = 0 ) {
        $finalFiles = []; // map of (DB key => corresponding File) for matches

        $searchSet = []; // map of (normalized DB key => search params)
        foreach ( $items as $item ) {
            if ( is_array( $item ) ) {
                $title = File::normalizeTitle( $item['title'] );
                if ( $title ) {
                    $searchSet[$title->getDBkey()] = $item;
                }
            } else {
                $title = File::normalizeTitle( $item );
                if ( $title ) {
                    $searchSet[$title->getDBkey()] = [];
                }
            }
        }

        $fileMatchesSearch = function ( File $file, array $search ) {
            // Note: file name comparison done elsewhere (to handle redirects)
            $user = ( !empty( $search['private'] ) && $search['private'] instanceof User )
                ? $search['private']
                : null;

            return (
                $file->exists() &&
                (
                    ( empty( $search['time'] ) && !$file->isOld() ) ||
                    ( !empty( $search['time'] ) && $search['time'] === $file->getTimestamp() )
                ) &&
                ( !empty( $search['private'] ) || !$file->isDeleted( File::DELETED_FILE ) ) &&
                $file->userCan( File::DELETED_FILE, $user )
            );
        };

        $applyMatchingFiles = function ( IResultWrapper $res, &$searchSet, &$finalFiles )
            use ( $fileMatchesSearch, $flags )
        {
            $contLang = MediaWikiServices::getInstance()->getContentLanguage();
            $info = $this->getInfo();
            foreach ( $res as $row ) {
                $file = $this->newFileFromRow( $row );
                // There must have been a search for this DB key, but this has to handle the
                // cases were title capitalization is different on the client and repo wikis.
                $dbKeysLook = [ strtr( $file->getName(), ' ', '_' ) ];
                if ( !empty( $info['initialCapital'] ) ) {
                    // Search keys for "hi.png" and "Hi.png" should use the "Hi.png file"
                    $dbKeysLook[] = $contLang->lcfirst( $file->getName() );
                }
                foreach ( $dbKeysLook as $dbKey ) {
                    if ( isset( $searchSet[$dbKey] )
                        && $fileMatchesSearch( $file, $searchSet[$dbKey] )
                    ) {
                        $finalFiles[$dbKey] = ( $flags & FileRepo::NAME_AND_TIME_ONLY )
                            ? [ 'title' => $dbKey, 'timestamp' => $file->getTimestamp() ]
                            : $file;
                        unset( $searchSet[$dbKey] );
                    }
                }
            }
        };

        $dbr = $this->getReplicaDB();

        // Query image table
        $imgNames = [];
        foreach ( array_keys( $searchSet ) as $dbKey ) {
            $imgNames[] = $this->getNameFromTitle( File::normalizeTitle( $dbKey ) );
        }

        if ( count( $imgNames ) ) {
            $fileQuery = LocalFile::getQueryInfo();
            $res = $dbr->select( $fileQuery['tables'], $fileQuery['fields'], [ 'img_name' => $imgNames ],
                __METHOD__, [], $fileQuery['joins'] );
            $applyMatchingFiles( $res, $searchSet, $finalFiles );
        }

        // Query old image table
        $oiConds = []; // WHERE clause array for each file
        foreach ( $searchSet as $dbKey => $search ) {
            if ( isset( $search['time'] ) ) {
                $oiConds[] = $dbr->makeList(
                    [
                        'oi_name' => $this->getNameFromTitle( File::normalizeTitle( $dbKey ) ),
                        'oi_timestamp' => $dbr->timestamp( $search['time'] )
                    ],
                    LIST_AND
                );
            }
        }

        if ( count( $oiConds ) ) {
            $fileQuery = OldLocalFile::getQueryInfo();
            $res = $dbr->select( $fileQuery['tables'], $fileQuery['fields'],
                $dbr->makeList( $oiConds, LIST_OR ),
                __METHOD__, [], $fileQuery['joins'] );
            $applyMatchingFiles( $res, $searchSet, $finalFiles );
        }

        // Check for redirects...
        foreach ( $searchSet as $dbKey => $search ) {
            if ( !empty( $search['ignoreRedirect'] ) ) {
                continue;
            }

            $title = File::normalizeTitle( $dbKey );
            $redir = $this->checkRedirect( $title ); // hopefully hits memcached

            if ( $redir && $redir->getNamespace() == NS_FILE ) {
                $file = $this->newFile( $redir );
                if ( $file && $fileMatchesSearch( $file, $search ) ) {
                    $file->redirectedFrom( $title->getDBkey() );
                    if ( $flags & FileRepo::NAME_AND_TIME_ONLY ) {
                        $finalFiles[$dbKey] = [
                            'title' => $file->getTitle()->getDBkey(),
                            'timestamp' => $file->getTimestamp()
                        ];
                    } else {
                        $finalFiles[$dbKey] = $file;
                    }
                }
            }
        }

        return $finalFiles;
    }

    function findBySha1( $hash ) {
        $dbr = $this->getReplicaDB();
        $fileQuery = LocalFile::getQueryInfo();
        $res = $dbr->select(
            $fileQuery['tables'],
            $fileQuery['fields'],
            [ 'img_sha1' => $hash ],
            __METHOD__,
            [ 'ORDER BY' => 'img_name' ],
            $fileQuery['joins']
        );

        $result = [];
        foreach ( $res as $row ) {
            $result[] = $this->newFileFromRow( $row );
        }
        $res->free();

        return $result;
    }

    function findBySha1s( array $hashes ) {
        if ( $hashes === [] ) {
            return []; // empty parameter
        }

        $dbr = $this->getReplicaDB();
        $fileQuery = LocalFile::getQueryInfo();
        $res = $dbr->select(
            $fileQuery['tables'],
            $fileQuery['fields'],
            [ 'img_sha1' => $hashes ],
            __METHOD__,
            [ 'ORDER BY' => 'img_name' ],
            $fileQuery['joins']
        );

        $result = [];
        foreach ( $res as $row ) {
            $file = $this->newFileFromRow( $row );
            $result[$file->getSha1()][] = $file;
        }
        $res->free();

        return $result;
    }

    public function findFilesByPrefix( $prefix, $limit ) {
        $selectOptions = [ 'ORDER BY' => 'img_name', 'LIMIT' => intval( $limit ) ];

        // Query database
        $dbr = $this->getReplicaDB();
        $fileQuery = LocalFile::getQueryInfo();
        $res = $dbr->select(
            $fileQuery['tables'],
            $fileQuery['fields'],
            'img_name ' . $dbr->buildLike( $prefix, $dbr->anyString() ),
            __METHOD__,
            $selectOptions,
            $fileQuery['joins']
        );

        // Build file objects
        $files = [];
        foreach ( $res as $row ) {
            $files[] = $this->newFileFromRow( $row );
        }

        return $files;
    }

    function getReplicaDB() {
        return wfGetDB( DB_REPLICA );
    }

    function getSlaveDB() {
        return $this->getReplicaDB();
    }

    function getMasterDB() {
        return wfGetDB( DB_MASTER );
    }

    protected function getDBFactory() {
        return function ( $index ) {
            return wfGetDB( $index );
        };
    }

    function getSharedCacheKey( ...$args ) {
        return $this->wanCache->makeKey( ...$args );
    }

    function invalidateImageRedirect( Title $title ) {
        $key = $this->getSharedCacheKey( 'image_redirect', md5( $title->getDBkey() ) );
        if ( $key ) {
            $this->getMasterDB()->onTransactionPreCommitOrIdle(
                function () use ( $key ) {
                    $this->wanCache->delete( $key );
                },
                __METHOD__
            );
        }
    }

    function getInfo() {
        global $wgFavicon;

        return array_merge( parent::getInfo(), [
            'favicon' => wfExpandUrl( $wgFavicon ),
        ] );
    }

    public function store( $srcPath, $dstZone, $dstRel, $flags = 0 ) {
        return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
    }

    public function storeBatch( array $triplets, $flags = 0 ) {
        return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
    }

    public function cleanupBatch( array $files, $flags = 0 ) {
        return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
    }

    public function publish(
        $src,
        $dstRel,
        $archiveRel,
        $flags = 0,
        array $options = []
    ) {
        return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
    }

    public function publishBatch( array $ntuples, $flags = 0 ) {
        return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
    }

    public function delete( $srcRel, $archiveRel ) {
        return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
    }

    public function deleteBatch( array $sourceDestPairs ) {
        return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
    }

    protected function skipWriteOperationIfSha1( $function, array $args ) {
        $this->assertWritableRepo(); // fail out if read-only

        if ( $this->hasSha1Storage() ) {
            wfDebug( __METHOD__ . ": skipped because storage uses sha1 paths\n" );
            return Status::newGood();
        } else {
            return parent::$function( ...$args );
        }
    }
}