MediaWiki master
purgeChangedFiles.php
Go to the documentation of this file.
1<?php
26
27// @codeCoverageIgnoreStart
28require_once __DIR__ . '/Maintenance.php';
29// @codeCoverageIgnoreEnd
30
42 private static $typeMappings = [
43 'created' => [
44 'upload' => [ 'upload' ],
45 'import' => [ 'upload', 'interwiki' ],
46 ],
47 'deleted' => [
48 'delete' => [ 'delete', 'revision' ],
49 'suppress' => [ 'delete', 'revision' ],
50 ],
51 'modified' => [
52 'upload' => [ 'overwrite', 'revert' ],
53 'move' => [ 'move', 'move_redir' ],
54 ],
55 ];
56
60 private $startTimestamp;
61
65 private $endTimestamp;
66
67 public function __construct() {
68 parent::__construct();
69 $this->addDescription( 'Scan the logging table and purge files and thumbnails.' );
70 $this->addOption( 'starttime', 'Starting timestamp', true, true );
71 $this->addOption( 'endtime', 'Ending timestamp', true, true );
72 $this->addOption( 'type', 'Comma-separated list of types of changes to send purges for (' .
73 implode( ',', array_keys( self::$typeMappings ) ) . ',all)', false, true );
74 $this->addOption( 'htcp-dest', 'HTCP announcement destination (IP:port)', false, true );
75 $this->addOption( 'dry-run', 'Do not send purge requests' );
76 $this->addOption( 'sleep-per-batch', 'Milliseconds to sleep between batches', false, true );
77 $this->addOption( 'verbose', 'Show more output', false, false, 'v' );
78 $this->setBatchSize( 100 );
79 }
80
81 public function execute() {
82 global $wgHTCPRouting;
83
84 if ( $this->hasOption( 'htcp-dest' ) ) {
85 $parts = explode( ':', $this->getOption( 'htcp-dest' ), 2 );
86 if ( count( $parts ) < 2 ) {
87 // Add default htcp port
88 $parts[] = '4827';
89 }
90
91 // Route all HTCP messages to provided host:port
93 '' => [ 'host' => $parts[0], 'port' => $parts[1] ],
94 ];
95 $this->verbose( "HTCP broadcasts to {$parts[0]}:{$parts[1]}\n" );
96 }
97
98 // Find out which actions we should be concerned with
99 $typeOpt = $this->getOption( 'type', 'all' );
100 if ( $typeOpt === 'all' ) {
101 // Convert 'all' to all registered types
102 $typeOpt = implode( ',', array_keys( self::$typeMappings ) );
103 }
104 $typeList = explode( ',', $typeOpt );
105 foreach ( $typeList as $type ) {
106 if ( !isset( self::$typeMappings[$type] ) ) {
107 $this->error( "\nERROR: Unknown type: {$type}\n" );
108 $this->maybeHelp( true );
109 }
110 }
111
112 // Validate the timestamps
113 $dbr = $this->getReplicaDB();
114 $this->startTimestamp = $dbr->timestamp( $this->getOption( 'starttime' ) );
115 $this->endTimestamp = $dbr->timestamp( $this->getOption( 'endtime' ) );
116
117 if ( $this->startTimestamp > $this->endTimestamp ) {
118 $this->error( "\nERROR: starttime after endtime\n" );
119 $this->maybeHelp( true );
120 }
121
122 // Turn on verbose when dry-run is enabled
123 if ( $this->hasOption( 'dry-run' ) ) {
124 $this->setOption( 'verbose', 1 );
125 }
126
127 $this->verbose( 'Purging files that were: ' . implode( ', ', $typeList ) . "\n" );
128 foreach ( $typeList as $type ) {
129 $this->verbose( "Checking for {$type} files...\n" );
130 $this->purgeFromLogType( $type );
131 if ( !$this->hasOption( 'dry-run' ) ) {
132 $this->verbose( "...{$type} files purged.\n\n" );
133 }
134 }
135 }
136
142 protected function purgeFromLogType( $type ) {
143 $repo = $this->getServiceContainer()->getRepoGroup()->getLocalRepo();
144 $dbr = $this->getReplicaDB();
145
146 foreach ( self::$typeMappings[$type] as $logType => $logActions ) {
147 $this->verbose( "Scanning for {$logType}/" . implode( ',', $logActions ) . "\n" );
148
149 $res = $dbr->newSelectQueryBuilder()
150 ->select( [ 'log_title', 'log_timestamp', 'log_params' ] )
151 ->from( 'logging' )
152 ->where( [
153 'log_namespace' => NS_FILE,
154 'log_type' => $logType,
155 'log_action' => $logActions,
156 $dbr->expr( 'log_timestamp', '>=', $this->startTimestamp ),
157 $dbr->expr( 'log_timestamp', '<=', $this->endTimestamp ),
158 ] )
159 ->caller( __METHOD__ )->fetchResultSet();
160
161 $bSize = 0;
162 foreach ( $res as $row ) {
163 $file = $repo->newFile( Title::makeTitle( NS_FILE, $row->log_title ) );
164
165 if ( $this->hasOption( 'dry-run' ) ) {
166 $this->verbose( "{$type}[{$row->log_timestamp}]: {$row->log_title}\n" );
167 continue;
168 }
169
170 // Purge current version and its thumbnails
171 $file->purgeCache();
172 // Purge the old versions and their thumbnails
173 foreach ( $file->getHistory() as $oldFile ) {
174 $oldFile->purgeCache();
175 }
176
177 if ( $logType === 'delete' ) {
178 // If there is an orphaned storage file... delete it
179 if ( !$file->exists() && $repo->fileExists( $file->getPath() ) ) {
180 $dpath = $this->getDeletedPath( $repo, $file );
181 if ( $repo->fileExists( $dpath ) ) {
182 // Check to avoid data loss
183 $repo->getBackend()->delete( [ 'src' => $file->getPath() ] );
184 $this->verbose( "Deleted orphan file: {$file->getPath()}.\n" );
185 } else {
186 $this->error( "File was not deleted: {$file->getPath()}.\n" );
187 }
188 }
189
190 // Purge items from fileachive table (rows are likely here)
191 $this->purgeFromArchiveTable( $repo, $file );
192 } elseif ( $logType === 'move' ) {
193 // Purge the target file as well
194
195 $params = unserialize( $row->log_params );
196 if ( isset( $params['4::target'] ) ) {
197 $target = $params['4::target'];
198 $targetFile = $repo->newFile( Title::makeTitle( NS_FILE, $target ) );
199 $targetFile->purgeCache();
200 $this->verbose( "Purged file {$target}; move target @{$row->log_timestamp}.\n" );
201 }
202 }
203
204 $this->verbose( "Purged file {$row->log_title}; {$type} @{$row->log_timestamp}.\n" );
205
206 if ( $this->hasOption( 'sleep-per-batch' ) && ++$bSize > $this->getBatchSize() ) {
207 $bSize = 0;
208 // sleep-per-batch is milliseconds, usleep wants micro seconds.
209 usleep( 1000 * (int)$this->getOption( 'sleep-per-batch' ) );
210 }
211 }
212 }
213 }
214
215 protected function purgeFromArchiveTable( LocalRepo $repo, LocalFile $file ) {
216 $dbr = $repo->getReplicaDB();
217 $res = $dbr->newSelectQueryBuilder()
218 ->select( [ 'fa_archive_name' ] )
219 ->from( 'filearchive' )
220 ->where( [ 'fa_name' => $file->getName() ] )
221 ->caller( __METHOD__ )->fetchResultSet();
222
223 foreach ( $res as $row ) {
224 if ( $row->fa_archive_name === null ) {
225 // Was not an old version (current version names checked already)
226 continue;
227 }
228 $ofile = $repo->newFromArchiveName( $file->getTitle(), $row->fa_archive_name );
229 // If there is an orphaned storage file still there...delete it
230 if ( !$file->exists() && $repo->fileExists( $ofile->getPath() ) ) {
231 $dpath = $this->getDeletedPath( $repo, $ofile );
232 if ( $repo->fileExists( $dpath ) ) {
233 // Check to avoid data loss
234 $repo->getBackend()->delete( [ 'src' => $ofile->getPath() ] );
235 $this->output( "Deleted orphan file: {$ofile->getPath()}.\n" );
236 } else {
237 $this->error( "File was not deleted: {$ofile->getPath()}.\n" );
238 }
239 }
240 $file->purgeOldThumbnails( $row->fa_archive_name );
241 }
242 }
243
244 protected function getDeletedPath( LocalRepo $repo, LocalFile $file ) {
245 $hash = $repo->getFileSha1( $file->getPath() );
246 $key = "{$hash}.{$file->getExtension()}";
247
248 return $repo->getDeletedHashPath( $key ) . $key;
249 }
250
256 protected function verbose( $msg ) {
257 if ( $this->hasOption( 'verbose' ) ) {
258 $this->output( $msg );
259 }
260 }
261}
262
263// @codeCoverageIgnoreStart
264$maintClass = PurgeChangedFiles::class;
265require_once RUN_MAINTENANCE_IF_MAIN;
266// @codeCoverageIgnoreEnd
const NS_FILE
Definition Defines.php:71
array $params
The job parameters.
fileExists( $file)
Checks existence of a file.
getFileSha1( $virtualUrl)
Get the sha1 (base 36) of a file with a given virtual URL/storage path.
getDeletedHashPath( $key)
Get a relative path for a deletion archive key, e.g.
getBackend()
Get the file backend instance.
Definition FileRepo.php:259
getPath()
Return the storage path to the file.
Definition File.php:475
getName()
Return the name of this file.
Definition File.php:347
getTitle()
Return the associated title object.
Definition File.php:377
Local file in the wiki's own database.
Definition LocalFile.php:75
exists()
canRender inherited
purgeOldThumbnails( $archiveName)
Delete cached transformed files for an archived version only.
Local repository that stores files in the local filesystem and registers them in the wiki's own datab...
Definition LocalRepo.php:49
getReplicaDB()
Get a connection to the replica DB.
newFromArchiveName( $title, $archiveName)
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
getBatchSize()
Returns batch size.
output( $out, $channel=null)
Throw some output to the user.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
hasOption( $name)
Checks to see if a particular option was set.
setOption(string $name, $value)
Programmatically set the value of the given option.
getOption( $name, $default=null)
Get an option, or return the default.
error( $err, $die=0)
Throw an error to the user.
getServiceContainer()
Returns the main service container.
maybeHelp( $force=false)
Maybe show the help.
addDescription( $text)
Set the description text.
Represents a title within MediaWiki.
Definition Title.php:78
Maintenance script that scans the deletion log and purges affected files within a timeframe.
execute()
Do the actual work.
__construct()
Default constructor.
purgeFromLogType( $type)
Purge cache and thumbnails for changes of the given type.
getDeletedPath(LocalRepo $repo, LocalFile $file)
purgeFromArchiveTable(LocalRepo $repo, LocalFile $file)
verbose( $msg)
Send an output message iff the 'verbose' option has been provided.
$wgHTCPRouting
Config variable stub for the HTCPRouting setting, for use by phpdoc and IDEs.