Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
0.00% |
0 / 107 |
|
0.00% |
0 / 6 |
CRAP | |
0.00% |
0 / 1 |
| PurgeChangedFiles | |
0.00% |
0 / 107 |
|
0.00% |
0 / 6 |
1122 | |
0.00% |
0 / 1 |
| __construct | |
0.00% |
0 / 11 |
|
0.00% |
0 / 1 |
2 | |||
| execute | |
0.00% |
0 / 31 |
|
0.00% |
0 / 1 |
110 | |||
| purgeFromLogType | |
0.00% |
0 / 43 |
|
0.00% |
0 / 1 |
182 | |||
| purgeFromArchiveTable | |
0.00% |
0 / 17 |
|
0.00% |
0 / 1 |
42 | |||
| getDeletedPath | |
0.00% |
0 / 3 |
|
0.00% |
0 / 1 |
2 | |||
| verbose | |
0.00% |
0 / 2 |
|
0.00% |
0 / 1 |
6 | |||
| 1 | <?php |
| 2 | /** |
| 3 | * Scan the logging table and purge affected files within a timeframe. |
| 4 | * |
| 5 | * This program is free software; you can redistribute it and/or modify |
| 6 | * it under the terms of the GNU General Public License as published by |
| 7 | * the Free Software Foundation; either version 2 of the License, or |
| 8 | * (at your option) any later version. |
| 9 | * |
| 10 | * This program is distributed in the hope that it will be useful, |
| 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 13 | * GNU General Public License for more details. |
| 14 | * |
| 15 | * You should have received a copy of the GNU General Public License along |
| 16 | * with this program; if not, write to the Free Software Foundation, Inc., |
| 17 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
| 18 | * http://www.gnu.org/copyleft/gpl.html |
| 19 | * |
| 20 | * @file |
| 21 | * @ingroup Maintenance |
| 22 | */ |
| 23 | |
| 24 | use MediaWiki\Maintenance\Maintenance; |
| 25 | use MediaWiki\Title\Title; |
| 26 | |
| 27 | // @codeCoverageIgnoreStart |
| 28 | require_once __DIR__ . '/Maintenance.php'; |
| 29 | // @codeCoverageIgnoreEnd |
| 30 | |
| 31 | /** |
| 32 | * Maintenance script that scans the deletion log and purges affected files |
| 33 | * within a timeframe. |
| 34 | * |
| 35 | * @ingroup Maintenance |
| 36 | */ |
| 37 | class PurgeChangedFiles extends Maintenance { |
| 38 | /** |
| 39 | * Mapping from type option to log type and actions. |
| 40 | * @var array |
| 41 | */ |
| 42 | private static $typeMappings = [ |
| 43 | 'created' => [ |
| 44 | 'upload' => [ 'upload' ], |
| 45 | 'import' => [ 'upload', 'interwiki' ], |
| 46 | ], |
| 47 | 'deleted' => [ |
| 48 | 'delete' => [ 'delete', 'revision' ], |
| 49 | 'suppress' => [ 'delete', 'revision' ], |
| 50 | ], |
| 51 | 'modified' => [ |
| 52 | 'upload' => [ 'overwrite', 'revert' ], |
| 53 | 'move' => [ 'move', 'move_redir' ], |
| 54 | ], |
| 55 | ]; |
| 56 | |
| 57 | /** |
| 58 | * @var string |
| 59 | */ |
| 60 | private $startTimestamp; |
| 61 | |
| 62 | /** |
| 63 | * @var string |
| 64 | */ |
| 65 | private $endTimestamp; |
| 66 | |
| 67 | public function __construct() { |
| 68 | parent::__construct(); |
| 69 | $this->addDescription( 'Scan the logging table and purge files and thumbnails.' ); |
| 70 | $this->addOption( 'starttime', 'Starting timestamp', true, true ); |
| 71 | $this->addOption( 'endtime', 'Ending timestamp', true, true ); |
| 72 | $this->addOption( 'type', 'Comma-separated list of types of changes to send purges for (' . |
| 73 | implode( ',', array_keys( self::$typeMappings ) ) . ',all)', false, true ); |
| 74 | $this->addOption( 'htcp-dest', 'HTCP announcement destination (IP:port)', false, true ); |
| 75 | $this->addOption( 'dry-run', 'Do not send purge requests' ); |
| 76 | $this->addOption( 'sleep-per-batch', 'Milliseconds to sleep between batches', false, true ); |
| 77 | $this->addOption( 'verbose', 'Show more output', false, false, 'v' ); |
| 78 | $this->setBatchSize( 100 ); |
| 79 | } |
| 80 | |
| 81 | public function execute() { |
| 82 | global $wgHTCPRouting; |
| 83 | |
| 84 | if ( $this->hasOption( 'htcp-dest' ) ) { |
| 85 | $parts = explode( ':', $this->getOption( 'htcp-dest' ), 2 ); |
| 86 | if ( count( $parts ) < 2 ) { |
| 87 | // Add default htcp port |
| 88 | $parts[] = '4827'; |
| 89 | } |
| 90 | |
| 91 | // Route all HTCP messages to provided host:port |
| 92 | $wgHTCPRouting = [ |
| 93 | '' => [ 'host' => $parts[0], 'port' => $parts[1] ], |
| 94 | ]; |
| 95 | $this->verbose( "HTCP broadcasts to {$parts[0]}:{$parts[1]}\n" ); |
| 96 | } |
| 97 | |
| 98 | // Find out which actions we should be concerned with |
| 99 | $typeOpt = $this->getOption( 'type', 'all' ); |
| 100 | if ( $typeOpt === 'all' ) { |
| 101 | // Convert 'all' to all registered types |
| 102 | $typeOpt = implode( ',', array_keys( self::$typeMappings ) ); |
| 103 | } |
| 104 | $typeList = explode( ',', $typeOpt ); |
| 105 | foreach ( $typeList as $type ) { |
| 106 | if ( !isset( self::$typeMappings[$type] ) ) { |
| 107 | $this->error( "\nERROR: Unknown type: {$type}\n" ); |
| 108 | $this->maybeHelp( true ); |
| 109 | } |
| 110 | } |
| 111 | |
| 112 | // Validate the timestamps |
| 113 | $dbr = $this->getReplicaDB(); |
| 114 | $this->startTimestamp = $dbr->timestamp( $this->getOption( 'starttime' ) ); |
| 115 | $this->endTimestamp = $dbr->timestamp( $this->getOption( 'endtime' ) ); |
| 116 | |
| 117 | if ( $this->startTimestamp > $this->endTimestamp ) { |
| 118 | $this->error( "\nERROR: starttime after endtime\n" ); |
| 119 | $this->maybeHelp( true ); |
| 120 | } |
| 121 | |
| 122 | // Turn on verbose when dry-run is enabled |
| 123 | if ( $this->hasOption( 'dry-run' ) ) { |
| 124 | $this->setOption( 'verbose', 1 ); |
| 125 | } |
| 126 | |
| 127 | $this->verbose( 'Purging files that were: ' . implode( ', ', $typeList ) . "\n" ); |
| 128 | foreach ( $typeList as $type ) { |
| 129 | $this->verbose( "Checking for {$type} files...\n" ); |
| 130 | $this->purgeFromLogType( $type ); |
| 131 | if ( !$this->hasOption( 'dry-run' ) ) { |
| 132 | $this->verbose( "...{$type} files purged.\n\n" ); |
| 133 | } |
| 134 | } |
| 135 | } |
| 136 | |
| 137 | /** |
| 138 | * Purge cache and thumbnails for changes of the given type. |
| 139 | * |
| 140 | * @param string $type Type of change to find |
| 141 | */ |
| 142 | protected function purgeFromLogType( $type ) { |
| 143 | $repo = $this->getServiceContainer()->getRepoGroup()->getLocalRepo(); |
| 144 | $dbr = $this->getReplicaDB(); |
| 145 | |
| 146 | foreach ( self::$typeMappings[$type] as $logType => $logActions ) { |
| 147 | $this->verbose( "Scanning for {$logType}/" . implode( ',', $logActions ) . "\n" ); |
| 148 | |
| 149 | $res = $dbr->newSelectQueryBuilder() |
| 150 | ->select( [ 'log_title', 'log_timestamp', 'log_params' ] ) |
| 151 | ->from( 'logging' ) |
| 152 | ->where( [ |
| 153 | 'log_namespace' => NS_FILE, |
| 154 | 'log_type' => $logType, |
| 155 | 'log_action' => $logActions, |
| 156 | $dbr->expr( 'log_timestamp', '>=', $this->startTimestamp ), |
| 157 | $dbr->expr( 'log_timestamp', '<=', $this->endTimestamp ), |
| 158 | ] ) |
| 159 | ->caller( __METHOD__ )->fetchResultSet(); |
| 160 | |
| 161 | $bSize = 0; |
| 162 | foreach ( $res as $row ) { |
| 163 | $file = $repo->newFile( Title::makeTitle( NS_FILE, $row->log_title ) ); |
| 164 | |
| 165 | if ( $this->hasOption( 'dry-run' ) ) { |
| 166 | $this->verbose( "{$type}[{$row->log_timestamp}]: {$row->log_title}\n" ); |
| 167 | continue; |
| 168 | } |
| 169 | |
| 170 | // Purge current version and its thumbnails |
| 171 | $file->purgeCache(); |
| 172 | // Purge the old versions and their thumbnails |
| 173 | foreach ( $file->getHistory() as $oldFile ) { |
| 174 | $oldFile->purgeCache(); |
| 175 | } |
| 176 | |
| 177 | if ( $logType === 'delete' ) { |
| 178 | // If there is an orphaned storage file... delete it |
| 179 | if ( !$file->exists() && $repo->fileExists( $file->getPath() ) ) { |
| 180 | $dpath = $this->getDeletedPath( $repo, $file ); |
| 181 | if ( $repo->fileExists( $dpath ) ) { |
| 182 | // Check to avoid data loss |
| 183 | $repo->getBackend()->delete( [ 'src' => $file->getPath() ] ); |
| 184 | $this->verbose( "Deleted orphan file: {$file->getPath()}.\n" ); |
| 185 | } else { |
| 186 | $this->error( "File was not deleted: {$file->getPath()}.\n" ); |
| 187 | } |
| 188 | } |
| 189 | |
| 190 | // Purge items from fileachive table (rows are likely here) |
| 191 | $this->purgeFromArchiveTable( $repo, $file ); |
| 192 | } elseif ( $logType === 'move' ) { |
| 193 | // Purge the target file as well |
| 194 | |
| 195 | $params = unserialize( $row->log_params ); |
| 196 | if ( isset( $params['4::target'] ) ) { |
| 197 | $target = $params['4::target']; |
| 198 | $targetFile = $repo->newFile( Title::makeTitle( NS_FILE, $target ) ); |
| 199 | $targetFile->purgeCache(); |
| 200 | $this->verbose( "Purged file {$target}; move target @{$row->log_timestamp}.\n" ); |
| 201 | } |
| 202 | } |
| 203 | |
| 204 | $this->verbose( "Purged file {$row->log_title}; {$type} @{$row->log_timestamp}.\n" ); |
| 205 | |
| 206 | if ( $this->hasOption( 'sleep-per-batch' ) && ++$bSize > $this->getBatchSize() ) { |
| 207 | $bSize = 0; |
| 208 | // sleep-per-batch is milliseconds, usleep wants micro seconds. |
| 209 | usleep( 1000 * (int)$this->getOption( 'sleep-per-batch' ) ); |
| 210 | } |
| 211 | } |
| 212 | } |
| 213 | } |
| 214 | |
| 215 | protected function purgeFromArchiveTable( LocalRepo $repo, LocalFile $file ) { |
| 216 | $dbr = $repo->getReplicaDB(); |
| 217 | $res = $dbr->newSelectQueryBuilder() |
| 218 | ->select( [ 'fa_archive_name' ] ) |
| 219 | ->from( 'filearchive' ) |
| 220 | ->where( [ 'fa_name' => $file->getName() ] ) |
| 221 | ->caller( __METHOD__ )->fetchResultSet(); |
| 222 | |
| 223 | foreach ( $res as $row ) { |
| 224 | if ( $row->fa_archive_name === null ) { |
| 225 | // Was not an old version (current version names checked already) |
| 226 | continue; |
| 227 | } |
| 228 | $ofile = $repo->newFromArchiveName( $file->getTitle(), $row->fa_archive_name ); |
| 229 | // If there is an orphaned storage file still there...delete it |
| 230 | if ( !$file->exists() && $repo->fileExists( $ofile->getPath() ) ) { |
| 231 | $dpath = $this->getDeletedPath( $repo, $ofile ); |
| 232 | if ( $repo->fileExists( $dpath ) ) { |
| 233 | // Check to avoid data loss |
| 234 | $repo->getBackend()->delete( [ 'src' => $ofile->getPath() ] ); |
| 235 | $this->output( "Deleted orphan file: {$ofile->getPath()}.\n" ); |
| 236 | } else { |
| 237 | $this->error( "File was not deleted: {$ofile->getPath()}.\n" ); |
| 238 | } |
| 239 | } |
| 240 | $file->purgeOldThumbnails( $row->fa_archive_name ); |
| 241 | } |
| 242 | } |
| 243 | |
| 244 | protected function getDeletedPath( LocalRepo $repo, LocalFile $file ) { |
| 245 | $hash = $repo->getFileSha1( $file->getPath() ); |
| 246 | $key = "{$hash}.{$file->getExtension()}"; |
| 247 | |
| 248 | return $repo->getDeletedHashPath( $key ) . $key; |
| 249 | } |
| 250 | |
| 251 | /** |
| 252 | * Send an output message iff the 'verbose' option has been provided. |
| 253 | * |
| 254 | * @param string $msg Message to output |
| 255 | */ |
| 256 | protected function verbose( $msg ) { |
| 257 | if ( $this->hasOption( 'verbose' ) ) { |
| 258 | $this->output( $msg ); |
| 259 | } |
| 260 | } |
| 261 | } |
| 262 | |
| 263 | // @codeCoverageIgnoreStart |
| 264 | $maintClass = PurgeChangedFiles::class; |
| 265 | require_once RUN_MAINTENANCE_IF_MAIN; |
| 266 | // @codeCoverageIgnoreEnd |