rebuildImages.php

Go to the documentation of this file.

<?php
// @codeCoverageIgnoreStart
require_once __DIR__ . '/Maintenance.php';
// @codeCoverageIgnoreEnd
 
use MediaWiki\FileRepo\File\FileSelectQueryBuilder;
use MediaWiki\FileRepo\LocalRepo;
use MediaWiki\MainConfigNames;
use MediaWiki\Maintenance\Maintenance;
use MediaWiki\Specials\SpecialUpload;
use MediaWiki\User\User;
use Wikimedia\Rdbms\IDatabase;
use Wikimedia\Rdbms\SelectQueryBuilder;
use Wikimedia\Timestamp\TimestampFormat as TS;
 
class ImageBuilder extends Maintenance {
    protected $dbw;
 
    private $dryrun;
 
    private $repo;
 
    private $updated;
 
    private $processed;
 
    private $count;
 
    private $startTime;
 
    private $table;
 
    public function __construct() {
        parent::__construct();
        $this->addDescription( 'Script to update image metadata records' );
 
        $this->addOption( 'missing', 'Check for files without associated database record' );
        $this->addOption( 'dry-run', 'Only report, don\'t update the database' );
    }
 
    public function execute() {
        $this->dbw = $this->getPrimaryDB();
        $this->dryrun = $this->hasOption( 'dry-run' );
        if ( $this->dryrun ) {
            $this->getServiceContainer()->getReadOnlyMode()
                ->setReason( 'Dry run mode, image upgrades are suppressed' );
        }
 
        if ( $this->hasOption( 'missing' ) ) {
            $this->crawlMissing();
        } else {
            $this->build();
        }
    }
 
    private function getRepo() {
        if ( $this->repo === null ) {
            $this->repo = $this->getServiceContainer()->getRepoGroup()
                ->newCustomLocalRepo( [
                    // make sure to update old, but compatible img_metadata fields.
                    'updateCompatibleMetadata' => true
                ] );
        }
 
        return $this->repo;
    }
 
    private function build() {
        $migrationStage = $this->getServiceContainer()->getMainConfig()->get(
            MainConfigNames::FileSchemaMigrationStage
        );
 
        if ( $migrationStage & SCHEMA_COMPAT_READ_OLD ) {
            $this->buildImage();
            $this->buildOldImage();
        } else {
            $this->buildFile();
        }
    }
 
    private function init( $count, $table ) {
        $this->processed = 0;
        $this->updated = 0;
        $this->count = $count;
        $this->startTime = microtime( true );
        $this->table = $table;
    }
 
    private function progress( int $updated ) {
        $this->updated += $updated;
        $this->processed++;
        if ( $this->processed % 100 != 0 ) {
            return;
        }
        $portion = $this->processed / $this->count;
        $updateRate = $this->updated / $this->processed;
 
        $now = microtime( true );
        $delta = $now - $this->startTime;
        $estimatedTotalTime = $delta / $portion;
        $eta = $this->startTime + $estimatedTotalTime;
        $rate = $this->processed / $delta;
 
        $this->output( sprintf( "%s: %6.2f%% done on %s; ETA %s [%d/%d] %.2f/sec <%.2f%% updated>\n",
            wfTimestamp( TS::DB, intval( $now ) ),
            $portion * 100.0,
            $this->table,
            wfTimestamp( TS::DB, intval( $eta ) ),
            $this->processed,
            $this->count,
            $rate,
            $updateRate * 100.0 ) );
        flush();
    }
 
    private function buildTable( string $table, SelectQueryBuilder $queryBuilder, callable $callback ) {
        $count = $this->dbw->newSelectQueryBuilder()
            ->select( 'count(*)' )
            ->from( $table )
            ->caller( __METHOD__ )->fetchField();
        $this->init( $count, $table );
        $this->output( "Processing $table...\n" );
 
        $result = $queryBuilder->caller( __METHOD__ )->fetchResultSet();
 
        foreach ( $result as $row ) {
            $update = $callback( $row );
            if ( $update ) {
                $this->progress( 1 );
            } else {
                $this->progress( 0 );
            }
        }
        $this->output( "Finished $table... $this->updated of $this->processed rows updated\n" );
    }
 
    private function buildImage() {
        $this->buildTable(
            'image',
            FileSelectQueryBuilder::newForFile( $this->getReplicaDB() ),
            $this->imageCallback( ... )
        );
    }
 
    private function imageCallback( \stdClass $row ): bool {
        // Create a File object from the row
        // This will also upgrade it
        $file = $this->getRepo()->newFileFromRow( $row );
 
        return $file->getUpgraded();
    }
 
    private function buildOldImage() {
        $this->buildTable(
            'oldimage',
            FileSelectQueryBuilder::newForOldFile( $this->getReplicaDB() ),
            $this->oldimageCallback( ... )
        );
    }
 
    private function oldimageCallback( \stdClass $row ): bool {
        // Create a File object from the row
        // This will also upgrade it
        if ( $row->oi_archive_name == '' ) {
            $this->output( "Empty oi_archive_name for oi_name={$row->oi_name}\n" );
 
            return false;
        }
        $file = $this->getRepo()->newFileFromRow( $row );
 
        return $file->getUpgraded();
    }
 
    private function buildFile() {
        $this->buildTable(
            'file',
            FileSelectQueryBuilder::newForFile( $this->getReplicaDB() ),
            $this->fileCallback( ... )
        );
    }
 
    private function fileCallback( \stdClass $row ): bool {
        // Create a File object from the row
        // This will also upgrade it
        $file = $this->getRepo()->newFile( $row->file_name );
 
        return $file->getUpgraded();
    }
 
    private function crawlMissing() {
        $this->getRepo()->enumFiles( $this->checkMissingImage( ... ) );
    }
 
    private function checkMissingImage( string $fullpath ) {
        $filename = wfBaseName( $fullpath );
 
        $row = FileSelectQueryBuilder::newForFile( $this->getReplicaDB() )
            ->where( [ 'img_name' => $filename ] )
            ->caller( __METHOD__ )->fetchRow();
 
        if ( !$row ) {
            // file not registered
            $this->addMissingImage( $filename, $fullpath );
        }
    }
 
    private function addMissingImage( string $filename, string $fullpath ) {
        $timestamp = $this->dbw->timestamp( $this->getRepo()->getFileTimestamp( $fullpath ) );
        $services = $this->getServiceContainer();
 
        $altname = $services->getContentLanguage()->checkTitleEncoding( $filename );
        if ( $altname != $filename ) {
            if ( $this->dryrun ) {
                $filename = $altname;
                $this->output( "Estimating transcoding... $altname\n" );
            } else {
                // @fixme create renameFile()
                // @phan-suppress-next-line PhanUndeclaredMethod See comment above...
                $filename = $this->renameFile( $filename );
            }
        }
 
        if ( $filename == '' ) {
            $this->output( "Empty filename for $fullpath\n" );
 
            return;
        }
        if ( !$this->dryrun ) {
            $file = $services->getRepoGroup()->getLocalRepo()->newFile( $filename );
            $pageText = SpecialUpload::getInitialPageText(
                '(recovered file, missing upload log entry)'
            );
            $user = User::newSystemUser( User::MAINTENANCE_SCRIPT_USER, [ 'steal' => true ] );
            $status = $file->recordUpload3(
                '',
                '(recovered file, missing upload log entry)',
                $pageText,
                $user,
                false,
                $timestamp
            );
            if ( !$status->isOK() ) {
                $this->output( "Error uploading file $fullpath\n" );
 
                return;
            }
        }
        $this->output( $fullpath . "\n" );
    }
}
 
// @codeCoverageIgnoreStart
$maintClass = ImageBuilder::class;
require_once RUN_MAINTENANCE_IF_MAIN;
// @codeCoverageIgnoreEnd