purgeChangedFiles.php

Go to the documentation of this file.

<?php
require_once __DIR__ . '/Maintenance.php';

class PurgeChangedFiles extends Maintenance {
    private static $typeMappings = [
        'created' => [
            'upload' => [ 'upload' ],
            'import' => [ 'upload', 'interwiki' ],
        ],
        'deleted' => [
            'delete' => [ 'delete', 'revision' ],
            'suppress' => [ 'delete', 'revision' ],
        ],
        'modified' => [
            'upload' => [ 'overwrite', 'revert' ],
            'move' => [ 'move', 'move_redir' ],
        ],
    ];

    private $startTimestamp;

    private $endTimestamp;

    public function __construct() {
        parent::__construct();
        $this->addDescription( 'Scan the logging table and purge files and thumbnails.' );
        $this->addOption( 'starttime', 'Starting timestamp', true, true );
        $this->addOption( 'endtime', 'Ending timestamp', true, true );
        $this->addOption( 'type', 'Comma-separated list of types of changes to send purges for (' .
            implode( ',', array_keys( self::$typeMappings ) ) . ',all)', false, true );
        $this->addOption( 'htcp-dest', 'HTCP announcement destination (IP:port)', false, true );
        $this->addOption( 'dry-run', 'Do not send purge requests' );
        $this->addOption( 'sleep-per-batch', 'Milliseconds to sleep between batches', false, true );
        $this->addOption( 'verbose', 'Show more output', false, false, 'v' );
        $this->setBatchSize( 100 );
    }

    public function execute() {
        global $wgHTCPRouting;

        if ( $this->hasOption( 'htcp-dest' ) ) {
            $parts = explode( ':', $this->getOption( 'htcp-dest' ) );
            if ( count( $parts ) < 2 ) {
                // Add default htcp port
                $parts[] = '4827';
            }

            // Route all HTCP messages to provided host:port
            $wgHTCPRouting = [
                '' => [ 'host' => $parts[0], 'port' => $parts[1] ],
            ];
            $this->verbose( "HTCP broadcasts to {$parts[0]}:{$parts[1]}\n" );
        }

        // Find out which actions we should be concerned with
        $typeOpt = $this->getOption( 'type', 'all' );
        $validTypes = array_keys( self::$typeMappings );
        if ( $typeOpt === 'all' ) {
            // Convert 'all' to all registered types
            $typeOpt = implode( ',', $validTypes );
        }
        $typeList = explode( ',', $typeOpt );
        foreach ( $typeList as $type ) {
            if ( !in_array( $type, $validTypes ) ) {
                $this->error( "\nERROR: Unknown type: {$type}\n" );
                $this->maybeHelp( true );
            }
        }

        // Validate the timestamps
        $dbr = $this->getDB( DB_REPLICA );
        $this->startTimestamp = $dbr->timestamp( $this->getOption( 'starttime' ) );
        $this->endTimestamp = $dbr->timestamp( $this->getOption( 'endtime' ) );

        if ( $this->startTimestamp > $this->endTimestamp ) {
            $this->error( "\nERROR: starttime after endtime\n" );
            $this->maybeHelp( true );
        }

        // Turn on verbose when dry-run is enabled
        if ( $this->hasOption( 'dry-run' ) ) {
            $this->mOptions['verbose'] = 1;
        }

        $this->verbose( 'Purging files that were: ' . implode( ', ', $typeList ) . "\n" );
        foreach ( $typeList as $type ) {
            $this->verbose( "Checking for {$type} files...\n" );
            $this->purgeFromLogType( $type );
            if ( !$this->hasOption( 'dry-run' ) ) {
                $this->verbose( "...{$type} files purged.\n\n" );
            }
        }
    }

    protected function purgeFromLogType( $type ) {
        $repo = RepoGroup::singleton()->getLocalRepo();
        $dbr = $this->getDB( DB_REPLICA );

        foreach ( self::$typeMappings[$type] as $logType => $logActions ) {
            $this->verbose( "Scanning for {$logType}/" . implode( ',', $logActions ) . "\n" );

            $res = $dbr->select(
                'logging',
                [ 'log_title', 'log_timestamp', 'log_params' ],
                [
                    'log_namespace' => NS_FILE,
                    'log_type' => $logType,
                    'log_action' => $logActions,
                    'log_timestamp >= ' . $dbr->addQuotes( $this->startTimestamp ),
                    'log_timestamp <= ' . $dbr->addQuotes( $this->endTimestamp ),
                ],
                __METHOD__
            );

            $bSize = 0;
            foreach ( $res as $row ) {
                $file = $repo->newFile( Title::makeTitle( NS_FILE, $row->log_title ) );

                if ( $this->hasOption( 'dry-run' ) ) {
                    $this->verbose( "{$type}[{$row->log_timestamp}]: {$row->log_title}\n" );
                    continue;
                }

                // Purge current version and its thumbnails
                $file->purgeCache();
                // Purge the old versions and their thumbnails
                foreach ( $file->getHistory() as $oldFile ) {
                    $oldFile->purgeCache();
                }

                if ( $logType === 'delete' ) {
                    // If there is an orphaned storage file... delete it
                    if ( !$file->exists() && $repo->fileExists( $file->getPath() ) ) {
                        $dpath = $this->getDeletedPath( $repo, $file );
                        if ( $repo->fileExists( $dpath ) ) {
                            // Sanity check to avoid data loss
                            $repo->getBackend()->delete( [ 'src' => $file->getPath() ] );
                            $this->verbose( "Deleted orphan file: {$file->getPath()}.\n" );
                        } else {
                            $this->error( "File was not deleted: {$file->getPath()}.\n" );
                        }
                    }

                    // Purge items from fileachive table (rows are likely here)
                    $this->purgeFromArchiveTable( $repo, $file );
                } elseif ( $logType === 'move' ) {
                    // Purge the target file as well

                    $params = unserialize( $row->log_params );
                    if ( isset( $params['4::target'] ) ) {
                        $target = $params['4::target'];
                        $targetFile = $repo->newFile( Title::makeTitle( NS_FILE, $target ) );
                        $targetFile->purgeCache();
                        $this->verbose( "Purged file {$target}; move target @{$row->log_timestamp}.\n" );
                    }
                }

                $this->verbose( "Purged file {$row->log_title}; {$type} @{$row->log_timestamp}.\n" );

                if ( $this->hasOption( 'sleep-per-batch' ) && ++$bSize > $this->mBatchSize ) {
                    $bSize = 0;
                    // sleep-per-batch is milliseconds, usleep wants micro seconds.
                    usleep( 1000 * (int)$this->getOption( 'sleep-per-batch' ) );
                }
            }
        }
    }

    protected function purgeFromArchiveTable( LocalRepo $repo, LocalFile $file ) {
        $dbr = $repo->getSlaveDB();
        $res = $dbr->select(
            'filearchive',
            [ 'fa_archive_name' ],
            [ 'fa_name' => $file->getName() ],
            __METHOD__
        );

        foreach ( $res as $row ) {
            if ( $row->fa_archive_name === null ) {
                // Was not an old version (current version names checked already)
                continue;
            }
            $ofile = $repo->newFromArchiveName( $file->getTitle(), $row->fa_archive_name );
            // If there is an orphaned storage file still there...delete it
            if ( !$file->exists() && $repo->fileExists( $ofile->getPath() ) ) {
                $dpath = $this->getDeletedPath( $repo, $ofile );
                if ( $repo->fileExists( $dpath ) ) {
                    // Sanity check to avoid data loss
                    $repo->getBackend()->delete( [ 'src' => $ofile->getPath() ] );
                    $this->output( "Deleted orphan file: {$ofile->getPath()}.\n" );
                } else {
                    $this->error( "File was not deleted: {$ofile->getPath()}.\n" );
                }
            }
            $file->purgeOldThumbnails( $row->fa_archive_name );
        }
    }

    protected function getDeletedPath( LocalRepo $repo, LocalFile $file ) {
        $hash = $repo->getFileSha1( $file->getPath() );
        $key = "{$hash}.{$file->getExtension()}";

        return $repo->getDeletedHashPath( $key ) . $key;
    }

    protected function verbose( $msg ) {
        if ( $this->hasOption( 'verbose' ) ) {
            $this->output( $msg );
        }
    }
}

$maintClass = "PurgeChangedFiles";
require_once RUN_MAINTENANCE_IF_MAIN;