MediaWiki master
purgeChangedFiles.php
Go to the documentation of this file.
1<?php
28
29// @codeCoverageIgnoreStart
30require_once __DIR__ . '/Maintenance.php';
31// @codeCoverageIgnoreEnd
32
44 private static $typeMappings = [
45 'created' => [
46 'upload' => [ 'upload' ],
47 'import' => [ 'upload', 'interwiki' ],
48 ],
49 'deleted' => [
50 'delete' => [ 'delete', 'revision' ],
51 'suppress' => [ 'delete', 'revision' ],
52 ],
53 'modified' => [
54 'upload' => [ 'overwrite', 'revert' ],
55 'move' => [ 'move', 'move_redir' ],
56 ],
57 ];
58
62 private $startTimestamp;
63
67 private $endTimestamp;
68
69 public function __construct() {
70 parent::__construct();
71 $this->addDescription( 'Scan the logging table and purge files and thumbnails.' );
72 $this->addOption( 'starttime', 'Starting timestamp', true, true );
73 $this->addOption( 'endtime', 'Ending timestamp', true, true );
74 $this->addOption( 'type', 'Comma-separated list of types of changes to send purges for (' .
75 implode( ',', array_keys( self::$typeMappings ) ) . ',all)', false, true );
76 $this->addOption( 'htcp-dest', 'HTCP announcement destination (IP:port)', false, true );
77 $this->addOption( 'dry-run', 'Do not send purge requests' );
78 $this->addOption( 'sleep-per-batch', 'Milliseconds to sleep between batches', false, true );
79 $this->addOption( 'verbose', 'Show more output', false, false, 'v' );
80 $this->setBatchSize( 100 );
81 }
82
83 public function execute() {
84 global $wgHTCPRouting;
85
86 if ( $this->hasOption( 'htcp-dest' ) ) {
87 $parts = explode( ':', $this->getOption( 'htcp-dest' ), 2 );
88 if ( count( $parts ) < 2 ) {
89 // Add default htcp port
90 $parts[] = '4827';
91 }
92
93 // Route all HTCP messages to provided host:port
95 '' => [ 'host' => $parts[0], 'port' => $parts[1] ],
96 ];
97 $this->verbose( "HTCP broadcasts to {$parts[0]}:{$parts[1]}\n" );
98 }
99
100 // Find out which actions we should be concerned with
101 $typeOpt = $this->getOption( 'type', 'all' );
102 if ( $typeOpt === 'all' ) {
103 // Convert 'all' to all registered types
104 $typeOpt = implode( ',', array_keys( self::$typeMappings ) );
105 }
106 $typeList = explode( ',', $typeOpt );
107 foreach ( $typeList as $type ) {
108 if ( !isset( self::$typeMappings[$type] ) ) {
109 $this->error( "\nERROR: Unknown type: {$type}\n" );
110 $this->maybeHelp( true );
111 }
112 }
113
114 // Validate the timestamps
115 $dbr = $this->getReplicaDB();
116 $this->startTimestamp = $dbr->timestamp( $this->getOption( 'starttime' ) );
117 $this->endTimestamp = $dbr->timestamp( $this->getOption( 'endtime' ) );
118
119 if ( $this->startTimestamp > $this->endTimestamp ) {
120 $this->error( "\nERROR: starttime after endtime\n" );
121 $this->maybeHelp( true );
122 }
123
124 // Turn on verbose when dry-run is enabled
125 if ( $this->hasOption( 'dry-run' ) ) {
126 $this->setOption( 'verbose', 1 );
127 }
128
129 $this->verbose( 'Purging files that were: ' . implode( ', ', $typeList ) . "\n" );
130 foreach ( $typeList as $type ) {
131 $this->verbose( "Checking for {$type} files...\n" );
132 $this->purgeFromLogType( $type );
133 if ( !$this->hasOption( 'dry-run' ) ) {
134 $this->verbose( "...{$type} files purged.\n\n" );
135 }
136 }
137 }
138
144 protected function purgeFromLogType( $type ) {
145 $repo = $this->getServiceContainer()->getRepoGroup()->getLocalRepo();
146 $dbr = $this->getReplicaDB();
147
148 foreach ( self::$typeMappings[$type] as $logType => $logActions ) {
149 $this->verbose( "Scanning for {$logType}/" . implode( ',', $logActions ) . "\n" );
150
151 $res = $dbr->newSelectQueryBuilder()
152 ->select( [ 'log_title', 'log_timestamp', 'log_params' ] )
153 ->from( 'logging' )
154 ->where( [
155 'log_namespace' => NS_FILE,
156 'log_type' => $logType,
157 'log_action' => $logActions,
158 $dbr->expr( 'log_timestamp', '>=', $this->startTimestamp ),
159 $dbr->expr( 'log_timestamp', '<=', $this->endTimestamp ),
160 ] )
161 ->caller( __METHOD__ )->fetchResultSet();
162
163 $bSize = 0;
164 foreach ( $res as $row ) {
165 $file = $repo->newFile( Title::makeTitle( NS_FILE, $row->log_title ) );
166
167 if ( $this->hasOption( 'dry-run' ) ) {
168 $this->verbose( "{$type}[{$row->log_timestamp}]: {$row->log_title}\n" );
169 continue;
170 }
171
172 // Purge current version and its thumbnails
173 $file->purgeCache();
174 // Purge the old versions and their thumbnails
175 foreach ( $file->getHistory() as $oldFile ) {
176 $oldFile->purgeCache();
177 }
178
179 if ( $logType === 'delete' ) {
180 // If there is an orphaned storage file... delete it
181 if ( !$file->exists() && $repo->fileExists( $file->getPath() ) ) {
182 $dpath = $this->getDeletedPath( $repo, $file );
183 if ( $repo->fileExists( $dpath ) ) {
184 // Check to avoid data loss
185 $repo->getBackend()->delete( [ 'src' => $file->getPath() ] );
186 $this->verbose( "Deleted orphan file: {$file->getPath()}.\n" );
187 } else {
188 $this->error( "File was not deleted: {$file->getPath()}.\n" );
189 }
190 }
191
192 // Purge items from fileachive table (rows are likely here)
193 $this->purgeFromArchiveTable( $repo, $file );
194 } elseif ( $logType === 'move' ) {
195 // Purge the target file as well
196
197 $params = unserialize( $row->log_params );
198 if ( isset( $params['4::target'] ) ) {
199 $target = $params['4::target'];
200 $targetFile = $repo->newFile( Title::makeTitle( NS_FILE, $target ) );
201 $targetFile->purgeCache();
202 $this->verbose( "Purged file {$target}; move target @{$row->log_timestamp}.\n" );
203 }
204 }
205
206 $this->verbose( "Purged file {$row->log_title}; {$type} @{$row->log_timestamp}.\n" );
207
208 if ( $this->hasOption( 'sleep-per-batch' ) && ++$bSize > $this->getBatchSize() ) {
209 $bSize = 0;
210 // sleep-per-batch is milliseconds, usleep wants micro seconds.
211 usleep( 1000 * (int)$this->getOption( 'sleep-per-batch' ) );
212 }
213 }
214 }
215 }
216
217 protected function purgeFromArchiveTable( LocalRepo $repo, LocalFile $file ) {
218 $dbr = $repo->getReplicaDB();
219 $res = $dbr->newSelectQueryBuilder()
220 ->select( [ 'fa_archive_name' ] )
221 ->from( 'filearchive' )
222 ->where( [ 'fa_name' => $file->getName() ] )
223 ->caller( __METHOD__ )->fetchResultSet();
224
225 foreach ( $res as $row ) {
226 if ( $row->fa_archive_name === null ) {
227 // Was not an old version (current version names checked already)
228 continue;
229 }
230 $ofile = $repo->newFromArchiveName( $file->getTitle(), $row->fa_archive_name );
231 // If there is an orphaned storage file still there...delete it
232 if ( !$file->exists() && $repo->fileExists( $ofile->getPath() ) ) {
233 $dpath = $this->getDeletedPath( $repo, $ofile );
234 if ( $repo->fileExists( $dpath ) ) {
235 // Check to avoid data loss
236 $repo->getBackend()->delete( [ 'src' => $ofile->getPath() ] );
237 $this->output( "Deleted orphan file: {$ofile->getPath()}.\n" );
238 } else {
239 $this->error( "File was not deleted: {$ofile->getPath()}.\n" );
240 }
241 }
242 $file->purgeOldThumbnails( $row->fa_archive_name );
243 }
244 }
245
246 protected function getDeletedPath( LocalRepo $repo, LocalFile $file ): string {
247 $hash = $repo->getFileSha1( $file->getPath() );
248 $key = "{$hash}.{$file->getExtension()}";
249
250 return $repo->getDeletedHashPath( $key ) . $key;
251 }
252
258 protected function verbose( $msg ) {
259 if ( $this->hasOption( 'verbose' ) ) {
260 $this->output( $msg );
261 }
262 }
263}
264
265// @codeCoverageIgnoreStart
266$maintClass = PurgeChangedFiles::class;
267require_once RUN_MAINTENANCE_IF_MAIN;
268// @codeCoverageIgnoreEnd
const NS_FILE
Definition Defines.php:71
getDeletedHashPath( $key)
Get a relative path for a deletion archive key, e.g.
fileExists( $file)
Checks existence of a file.
getBackend()
Get the file backend instance.
Definition FileRepo.php:270
getTitle()
Return the associated title object.
Definition File.php:391
getName()
Return the name of this file.
Definition File.php:361
Local file in the wiki's own database.
Definition LocalFile.php:93
purgeOldThumbnails( $archiveName)
Delete cached transformed files for an archived version only.
Local repository that stores files in the local filesystem and registers them in the wiki's own datab...
Definition LocalRepo.php:57
newFromArchiveName( $title, $archiveName)
getReplicaDB()
Get a connection to the replica DB.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
getBatchSize()
Returns batch size.
output( $out, $channel=null)
Throw some output to the user.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
hasOption( $name)
Checks to see if a particular option was set.
setOption(string $name, $value)
Programmatically set the value of the given option.
getOption( $name, $default=null)
Get an option, or return the default.
error( $err, $die=0)
Throw an error to the user.
getServiceContainer()
Returns the main service container.
maybeHelp( $force=false)
Maybe show the help.
addDescription( $text)
Set the description text.
Represents a title within MediaWiki.
Definition Title.php:78
Maintenance script that scans the deletion log and purges affected files within a timeframe.
execute()
Do the actual work.
__construct()
Default constructor.
purgeFromLogType( $type)
Purge cache and thumbnails for changes of the given type.
getDeletedPath(LocalRepo $repo, LocalFile $file)
purgeFromArchiveTable(LocalRepo $repo, LocalFile $file)
verbose( $msg)
Send an output message iff the 'verbose' option has been provided.
$wgHTCPRouting
Config variable stub for the HTCPRouting setting, for use by phpdoc and IDEs.