LocalFileMoveBatch.php

Go to the documentation of this file.

<?php
namespace MediaWiki\FileRepo\File;
 
use MediaWiki\FileRepo\FileRepo;
use MediaWiki\Logger\LoggerFactory;
use MediaWiki\MainConfigNames;
use MediaWiki\MediaWikiServices;
use MediaWiki\Status\Status;
use MediaWiki\Title\Title;
use Psr\Log\LoggerInterface;
use Wikimedia\Rdbms\IDatabase;
use Wikimedia\Rdbms\RawSQLValue;
use Wikimedia\ScopedCallback;
 
class LocalFileMoveBatch {
    protected $file;
 
    protected $target;
 
    protected $cur;
 
    protected $olds;
 
    protected $oldCount;
 
    protected $db;
 
    protected $oldHash;
 
    protected $newHash;
 
    protected $oldName;
 
    protected $newName;
 
    protected $oldRel;
 
    protected $newRel;
 
    private $logger;
 
    private $haveSourceLock = false;
 
    private $haveTargetLock = false;
 
    private $targetFile;
 
    public function __construct( LocalFile $file, Title $target ) {
        $this->file = $file;
        $this->target = $target;
        $this->oldHash = $this->file->repo->getHashPath( $this->file->getName() );
        $this->newHash = $this->file->repo->getHashPath( $this->target->getDBkey() );
        $this->oldName = $this->file->getName();
        $this->newName = $this->file->repo->getNameFromTitle( $this->target );
        $this->oldRel = $this->oldHash . $this->oldName;
        $this->newRel = $this->newHash . $this->newName;
        $this->db = $file->getRepo()->getPrimaryDB();
 
        $this->logger = LoggerFactory::getInstance( 'imagemove' );
    }
 
    public function addCurrent() {
        $status = $this->acquireSourceLock();
        if ( $status->isOK() ) {
            $this->cur = [ $this->oldRel, $this->newRel ];
        }
        return $status;
    }
 
    public function addOlds() {
        $archiveBase = 'archive';
        $this->olds = [];
        $this->oldCount = 0;
        $archiveNames = [];
 
        $migrationStage = MediaWikiServices::getInstance()->getMainConfig()->get(
            MainConfigNames::FileSchemaMigrationStage
        );
        if ( $migrationStage && SCHEMA_COMPAT_WRITE_OLD ) {
            $result = $this->db->newSelectQueryBuilder()
                ->select( [ 'oi_archive_name', 'oi_deleted' ] )
                ->forUpdate() // ignore snapshot
                ->from( 'oldimage' )
                ->where( [ 'oi_name' => $this->oldName ] )
                ->caller( __METHOD__ )->fetchResultSet();
        } else {
            $result = $this->db->newSelectQueryBuilder()
                ->select( [
                    'oi_archive_name' => 'fr_archive_name',
                    'oi_deleted' => 'fr_deleted',
                ] )
                ->forUpdate() // ignore snapshot
                ->from( 'filerevision' )
                ->join( 'file', null, 'fr_file = file_id' )
                ->where( [ 'file_name' => $this->oldName, 'file_deleted' => 0, 'file_latest != fr_id' ] )
                ->caller( __METHOD__ )->fetchResultSet();
        }
 
        foreach ( $result as $row ) {
            $archiveNames[] = $row->oi_archive_name;
            $oldName = $row->oi_archive_name;
            $bits = explode( '!', $oldName, 2 );
 
            if ( count( $bits ) != 2 ) {
                $this->logger->debug(
                    'Old file name missing !: {oldName}',
                    [ 'oldName' => $oldName ]
                );
                continue;
            }
 
            [ $timestamp, $filename ] = $bits;
 
            if ( $this->oldName != $filename ) {
                $this->logger->debug(
                    'Old file name does not match: {oldName}',
                    [ 'oldName' => $oldName ]
                );
                continue;
            }
 
            $this->oldCount++;
 
            // Do we want to add those to oldCount?
            if ( $row->oi_deleted & File::DELETED_FILE ) {
                continue;
            }
 
            $this->olds[] = [
                "{$archiveBase}/{$this->oldHash}{$oldName}",
                "{$archiveBase}/{$this->newHash}{$timestamp}!{$this->newName}"
            ];
        }
 
        return $archiveNames;
    }
 
    protected function acquireSourceLock() {
        if ( $this->haveSourceLock ) {
            return Status::newGood();
        }
        $status = $this->file->acquireFileLock();
        if ( $status->isOK() ) {
            $this->haveSourceLock = true;
        }
        return $status;
    }
 
    protected function acquireTargetLock() {
        if ( $this->haveTargetLock ) {
            return Status::newGood();
        }
        $status = $this->getTargetFile()->acquireFileLock();
        if ( $status->isOK() ) {
            $this->haveTargetLock = true;
        }
        return $status;
    }
 
    protected function releaseLocks() {
        if ( $this->haveSourceLock ) {
            $this->file->releaseFileLock();
            $this->haveSourceLock = false;
        }
        if ( $this->haveTargetLock ) {
            $this->getTargetFile()->releaseFileLock();
            $this->haveTargetLock = false;
        }
    }
 
    protected function getTargetFile() {
        if ( $this->targetFile === null ) {
            $this->targetFile = MediaWikiServices::getInstance()->getRepoGroup()->getLocalRepo()
                ->newFile( $this->target );
        }
        return $this->targetFile;
    }
 
    public function execute() {
        $repo = $this->file->repo;
        $status = $repo->newGood();
 
        $status->merge( $this->acquireSourceLock() );
        if ( !$status->isOK() ) {
            return $status;
        }
        $status->merge( $this->acquireTargetLock() );
        if ( !$status->isOK() ) {
            $this->releaseLocks();
            return $status;
        }
        $unlockScope = new ScopedCallback( function () {
            $this->releaseLocks();
        } );
 
        $triplets = $this->getMoveTriplets();
        $checkStatus = $this->removeNonexistentFiles( $triplets );
        if ( !$checkStatus->isGood() ) {
            $status->merge( $checkStatus ); // couldn't talk to file backend
            return $status;
        }
        $triplets = $checkStatus->value;
 
        // Verify the file versions metadata in the DB.
        $statusDb = $this->verifyDBUpdates();
        if ( !$statusDb->isGood() ) {
            $statusDb->setOK( false );
 
            return $statusDb;
        }
 
        if ( !$repo->hasSha1Storage() ) {
            // Copy the files into their new location.
            // If a prior process fataled copying or cleaning up files we tolerate any
            // of the existing files if they are identical to the ones being stored.
            $statusMove = $repo->storeBatch( $triplets, FileRepo::OVERWRITE_SAME );
 
            $this->logger->debug(
                'Moved files for {fileName}: {successCount} successes, {failCount} failures',
                [
                    'fileName' => $this->file->getName(),
                    'successCount' => $statusMove->successCount,
                    'failCount' => $statusMove->failCount,
                ]
            );
 
            if ( !$statusMove->isGood() ) {
                // Delete any files copied over (while the destination is still locked)
                $this->cleanupTarget( $triplets );
 
                $this->logger->debug(
                    'Error in moving files: {error}',
                    [ 'error' => $statusMove->getWikiText( false, false, 'en' ) ]
                );
 
                $statusMove->setOK( false );
 
                return $statusMove;
            }
            $status->merge( $statusMove );
        }
 
        // Rename the file versions metadata in the DB.
        $this->doDBUpdates();
 
        $this->logger->debug(
            'Renamed {fileName} in database: {successCount} successes, {failCount} failures',
            [
                'fileName' => $this->file->getName(),
                'successCount' => $statusDb->successCount,
                'failCount' => $statusDb->failCount,
            ]
        );
 
        // Everything went ok, remove the source files
        $this->cleanupSource( $triplets );
 
        // Defer lock release until the transaction is committed.
        if ( $this->db->trxLevel() ) {
            ScopedCallback::cancel( $unlockScope );
            $this->db->onTransactionResolution( function () {
                $this->releaseLocks();
            }, __METHOD__ );
        } else {
            ScopedCallback::consume( $unlockScope );
        }
 
        $status->merge( $statusDb );
 
        return $status;
    }
 
    protected function verifyDBUpdates() {
        $repo = $this->file->repo;
        $status = $repo->newGood();
        $dbw = $this->db;
 
        $migrationStage = MediaWikiServices::getInstance()->getMainConfig()->get(
            MainConfigNames::FileSchemaMigrationStage
        );
        if ( $migrationStage && SCHEMA_COMPAT_WRITE_OLD ) {
            // Lock the image row
            $hasCurrent = $dbw->newSelectQueryBuilder()
                ->from( 'image' )
                ->where( [ 'img_name' => $this->oldName ] )
                ->forUpdate()
                ->caller( __METHOD__ )
                ->fetchRowCount();
 
            // Lock the oldimage rows
            $oldRowCount = $dbw->newSelectQueryBuilder()
                ->from( 'oldimage' )
                ->where( [ 'oi_name' => $this->oldName ] )
                ->forUpdate()
                ->caller( __METHOD__ )
                ->fetchRowCount();
            if ( $hasCurrent ) {
                $status->successCount++;
            } else {
                $status->failCount++;
            }
            $status->successCount += $oldRowCount;
            // T36934: oldCount is based on files that actually exist.
            // There may be more DB rows than such files, in which case $affected
            // can be greater than $total. We use max() to avoid negatives here.
            $status->failCount += max( 0, $this->oldCount - $oldRowCount );
            if ( $status->failCount ) {
                $status->error( 'imageinvalidfilename' );
            }
        } else {
            $status->successCount += $this->db->newSelectQueryBuilder()
                ->forUpdate() // ignore snapshot
                ->from( 'filerevision' )
                ->join( 'file', null, 'fr_file = file_id' )
                ->where( [ 'file_name' => $this->oldName, 'file_deleted' => 0 ] )
                ->caller( __METHOD__ )->fetchRowCount();
        }
 
        return $status;
    }
 
    protected function doDBUpdates() {
        $dbw = $this->db;
 
        $migrationStage = MediaWikiServices::getInstance()->getMainConfig()->get(
            MainConfigNames::FileSchemaMigrationStage
        );
        if ( ( $migrationStage & SCHEMA_COMPAT_WRITE_NEW ) && $this->file->getFileIdFromName() ) {
            $deleted = $dbw->newSelectQueryBuilder()
                ->select( 'file_id' )
                ->from( 'file' )
                ->where( [ 'file_name' => $this->newName ] )
                ->andWhere( [ 'file_deleted' => 1 ] )
                ->caller( __METHOD__ )->fetchField();
            if ( $deleted ) {
                // Overwriting an existing file that was deleted.
                // Once the file deletion storage refactor starts,
                // this should change to update deleted revisions too.
                $dbw->newDeleteQueryBuilder()
                    ->deleteFrom( 'file' )
                    ->where( [ 'file_name' => $this->newName ] )
                    ->andWhere( [ 'file_deleted' => 1 ] )
                    ->caller( __METHOD__ )->execute();
                // Paranoia
                $dbw->newUpdateQueryBuilder()
                    ->update( 'filerevision' )
                    ->set( [ 'fr_file' => $this->file->getFileIdFromName() ] )
                    ->where( [ 'fr_file' => $deleted ] )
                    ->caller( __METHOD__ )->execute();
            }
            $dbw->newUpdateQueryBuilder()
                ->update( 'file' )
                ->set( [ 'file_name' => $this->newName ] )
                ->where( [ 'file_id' => $this->file->getFileIdFromName() ] )
                ->caller( __METHOD__ )->execute();
        }
        if ( $migrationStage & SCHEMA_COMPAT_WRITE_OLD ) {
            // Update current image
            $dbw->newUpdateQueryBuilder()
                ->update( 'image' )
                ->set( [ 'img_name' => $this->newName ] )
                ->where( [ 'img_name' => $this->oldName ] )
                ->caller( __METHOD__ )->execute();
 
            // Update old images
            $dbw->newUpdateQueryBuilder()
                ->update( 'oldimage' )
                ->set( [
                    'oi_name' => $this->newName,
                    'oi_archive_name' => new RawSQLValue( $dbw->strreplace(
                        'oi_archive_name',
                        $dbw->addQuotes( $this->oldName ),
                        $dbw->addQuotes( $this->newName )
                    ) ),
                ] )
                ->where( [ 'oi_name' => $this->oldName ] )
                ->caller( __METHOD__ )->execute();
        }
    }
 
    protected function getMoveTriplets() {
        $triplets = []; // The format is: (srcUrl, destZone, destUrl)
 
        foreach ( [ $this->cur, ...$this->olds ] as $move ) {
            // $move: (oldRelativePath, newRelativePath)
            $srcUrl = $this->file->repo->getVirtualUrl() . '/public/' . rawurlencode( $move[0] );
            $triplets[] = [ $srcUrl, 'public', $move[1] ];
 
            $this->logger->debug(
                'Generated move triplet for {fileName}: {srcUrl} :: public :: {move1}',
                [
                    'fileName' => $this->file->getName(),
                    'srcUrl' => $srcUrl,
                    'move1' => $move[1],
                ]
            );
        }
 
        return $triplets;
    }
 
    protected function removeNonexistentFiles( $triplets ) {
        $files = [];
 
        foreach ( $triplets as $file ) {
            $files[$file[0]] = $file[0];
        }
 
        $result = $this->file->repo->fileExistsBatch( $files );
        if ( in_array( null, $result, true ) ) {
            return Status::newFatal( 'backend-fail-internal',
                $this->file->repo->getBackend()->getName() );
        }
 
        $filteredTriplets = [];
        foreach ( $triplets as $file ) {
            if ( $result[$file[0]] ) {
                $filteredTriplets[] = $file;
            } else {
                $this->logger->debug(
                    'File {file} does not exist',
                    [ 'file' => $file[0] ]
                );
            }
        }
 
        return Status::newGood( $filteredTriplets );
    }
 
    protected function cleanupTarget( $triplets ) {
        // Create dest pairs from the triplets
        $pairs = [];
        foreach ( $triplets as $triplet ) {
            // $triplet: (old source virtual URL, dst zone, dest rel)
            $pairs[] = [ $triplet[1], $triplet[2] ];
        }
 
        $this->file->repo->cleanupBatch( $pairs );
    }
 
    protected function cleanupSource( $triplets ) {
        // Create source file names from the triplets
        $files = [];
        foreach ( $triplets as $triplet ) {
            $files[] = $triplet[0];
        }
 
        $this->file->repo->cleanupBatch( $files );
    }
}
 
class_alias( LocalFileMoveBatch::class, 'LocalFileMoveBatch' );