Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
Total | |
0.00% |
0 / 110 |
|
0.00% |
0 / 6 |
CRAP | |
0.00% |
0 / 1 |
PurgeChangedFiles | |
0.00% |
0 / 107 |
|
0.00% |
0 / 6 |
1122 | |
0.00% |
0 / 1 |
__construct | |
0.00% |
0 / 11 |
|
0.00% |
0 / 1 |
2 | |||
execute | |
0.00% |
0 / 31 |
|
0.00% |
0 / 1 |
110 | |||
purgeFromLogType | |
0.00% |
0 / 43 |
|
0.00% |
0 / 1 |
182 | |||
purgeFromArchiveTable | |
0.00% |
0 / 17 |
|
0.00% |
0 / 1 |
42 | |||
getDeletedPath | |
0.00% |
0 / 3 |
|
0.00% |
0 / 1 |
2 | |||
verbose | |
0.00% |
0 / 2 |
|
0.00% |
0 / 1 |
6 |
1 | <?php |
2 | /** |
3 | * Scan the logging table and purge affected files within a timeframe. |
4 | * |
5 | * This program is free software; you can redistribute it and/or modify |
6 | * it under the terms of the GNU General Public License as published by |
7 | * the Free Software Foundation; either version 2 of the License, or |
8 | * (at your option) any later version. |
9 | * |
10 | * This program is distributed in the hope that it will be useful, |
11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
13 | * GNU General Public License for more details. |
14 | * |
15 | * You should have received a copy of the GNU General Public License along |
16 | * with this program; if not, write to the Free Software Foundation, Inc., |
17 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
18 | * http://www.gnu.org/copyleft/gpl.html |
19 | * |
20 | * @file |
21 | * @ingroup Maintenance |
22 | */ |
23 | |
24 | use MediaWiki\Title\Title; |
25 | |
26 | require_once __DIR__ . '/Maintenance.php'; |
27 | |
28 | /** |
29 | * Maintenance script that scans the deletion log and purges affected files |
30 | * within a timeframe. |
31 | * |
32 | * @ingroup Maintenance |
33 | */ |
34 | class PurgeChangedFiles extends Maintenance { |
35 | /** |
36 | * Mapping from type option to log type and actions. |
37 | * @var array |
38 | */ |
39 | private static $typeMappings = [ |
40 | 'created' => [ |
41 | 'upload' => [ 'upload' ], |
42 | 'import' => [ 'upload', 'interwiki' ], |
43 | ], |
44 | 'deleted' => [ |
45 | 'delete' => [ 'delete', 'revision' ], |
46 | 'suppress' => [ 'delete', 'revision' ], |
47 | ], |
48 | 'modified' => [ |
49 | 'upload' => [ 'overwrite', 'revert' ], |
50 | 'move' => [ 'move', 'move_redir' ], |
51 | ], |
52 | ]; |
53 | |
54 | /** |
55 | * @var string |
56 | */ |
57 | private $startTimestamp; |
58 | |
59 | /** |
60 | * @var string |
61 | */ |
62 | private $endTimestamp; |
63 | |
64 | public function __construct() { |
65 | parent::__construct(); |
66 | $this->addDescription( 'Scan the logging table and purge files and thumbnails.' ); |
67 | $this->addOption( 'starttime', 'Starting timestamp', true, true ); |
68 | $this->addOption( 'endtime', 'Ending timestamp', true, true ); |
69 | $this->addOption( 'type', 'Comma-separated list of types of changes to send purges for (' . |
70 | implode( ',', array_keys( self::$typeMappings ) ) . ',all)', false, true ); |
71 | $this->addOption( 'htcp-dest', 'HTCP announcement destination (IP:port)', false, true ); |
72 | $this->addOption( 'dry-run', 'Do not send purge requests' ); |
73 | $this->addOption( 'sleep-per-batch', 'Milliseconds to sleep between batches', false, true ); |
74 | $this->addOption( 'verbose', 'Show more output', false, false, 'v' ); |
75 | $this->setBatchSize( 100 ); |
76 | } |
77 | |
78 | public function execute() { |
79 | global $wgHTCPRouting; |
80 | |
81 | if ( $this->hasOption( 'htcp-dest' ) ) { |
82 | $parts = explode( ':', $this->getOption( 'htcp-dest' ), 2 ); |
83 | if ( count( $parts ) < 2 ) { |
84 | // Add default htcp port |
85 | $parts[] = '4827'; |
86 | } |
87 | |
88 | // Route all HTCP messages to provided host:port |
89 | $wgHTCPRouting = [ |
90 | '' => [ 'host' => $parts[0], 'port' => $parts[1] ], |
91 | ]; |
92 | $this->verbose( "HTCP broadcasts to {$parts[0]}:{$parts[1]}\n" ); |
93 | } |
94 | |
95 | // Find out which actions we should be concerned with |
96 | $typeOpt = $this->getOption( 'type', 'all' ); |
97 | if ( $typeOpt === 'all' ) { |
98 | // Convert 'all' to all registered types |
99 | $typeOpt = implode( ',', array_keys( self::$typeMappings ) ); |
100 | } |
101 | $typeList = explode( ',', $typeOpt ); |
102 | foreach ( $typeList as $type ) { |
103 | if ( !isset( self::$typeMappings[$type] ) ) { |
104 | $this->error( "\nERROR: Unknown type: {$type}\n" ); |
105 | $this->maybeHelp( true ); |
106 | } |
107 | } |
108 | |
109 | // Validate the timestamps |
110 | $dbr = $this->getReplicaDB(); |
111 | $this->startTimestamp = $dbr->timestamp( $this->getOption( 'starttime' ) ); |
112 | $this->endTimestamp = $dbr->timestamp( $this->getOption( 'endtime' ) ); |
113 | |
114 | if ( $this->startTimestamp > $this->endTimestamp ) { |
115 | $this->error( "\nERROR: starttime after endtime\n" ); |
116 | $this->maybeHelp( true ); |
117 | } |
118 | |
119 | // Turn on verbose when dry-run is enabled |
120 | if ( $this->hasOption( 'dry-run' ) ) { |
121 | $this->mOptions['verbose'] = 1; |
122 | } |
123 | |
124 | $this->verbose( 'Purging files that were: ' . implode( ', ', $typeList ) . "\n" ); |
125 | foreach ( $typeList as $type ) { |
126 | $this->verbose( "Checking for {$type} files...\n" ); |
127 | $this->purgeFromLogType( $type ); |
128 | if ( !$this->hasOption( 'dry-run' ) ) { |
129 | $this->verbose( "...{$type} files purged.\n\n" ); |
130 | } |
131 | } |
132 | } |
133 | |
134 | /** |
135 | * Purge cache and thumbnails for changes of the given type. |
136 | * |
137 | * @param string $type Type of change to find |
138 | */ |
139 | protected function purgeFromLogType( $type ) { |
140 | $repo = $this->getServiceContainer()->getRepoGroup()->getLocalRepo(); |
141 | $dbr = $this->getReplicaDB(); |
142 | |
143 | foreach ( self::$typeMappings[$type] as $logType => $logActions ) { |
144 | $this->verbose( "Scanning for {$logType}/" . implode( ',', $logActions ) . "\n" ); |
145 | |
146 | $res = $dbr->newSelectQueryBuilder() |
147 | ->select( [ 'log_title', 'log_timestamp', 'log_params' ] ) |
148 | ->from( 'logging' ) |
149 | ->where( [ |
150 | 'log_namespace' => NS_FILE, |
151 | 'log_type' => $logType, |
152 | 'log_action' => $logActions, |
153 | $dbr->expr( 'log_timestamp', '>=', $this->startTimestamp ), |
154 | $dbr->expr( 'log_timestamp', '<=', $this->endTimestamp ), |
155 | ] ) |
156 | ->caller( __METHOD__ )->fetchResultSet(); |
157 | |
158 | $bSize = 0; |
159 | foreach ( $res as $row ) { |
160 | $file = $repo->newFile( Title::makeTitle( NS_FILE, $row->log_title ) ); |
161 | |
162 | if ( $this->hasOption( 'dry-run' ) ) { |
163 | $this->verbose( "{$type}[{$row->log_timestamp}]: {$row->log_title}\n" ); |
164 | continue; |
165 | } |
166 | |
167 | // Purge current version and its thumbnails |
168 | $file->purgeCache(); |
169 | // Purge the old versions and their thumbnails |
170 | foreach ( $file->getHistory() as $oldFile ) { |
171 | $oldFile->purgeCache(); |
172 | } |
173 | |
174 | if ( $logType === 'delete' ) { |
175 | // If there is an orphaned storage file... delete it |
176 | if ( !$file->exists() && $repo->fileExists( $file->getPath() ) ) { |
177 | $dpath = $this->getDeletedPath( $repo, $file ); |
178 | if ( $repo->fileExists( $dpath ) ) { |
179 | // Check to avoid data loss |
180 | $repo->getBackend()->delete( [ 'src' => $file->getPath() ] ); |
181 | $this->verbose( "Deleted orphan file: {$file->getPath()}.\n" ); |
182 | } else { |
183 | $this->error( "File was not deleted: {$file->getPath()}.\n" ); |
184 | } |
185 | } |
186 | |
187 | // Purge items from fileachive table (rows are likely here) |
188 | $this->purgeFromArchiveTable( $repo, $file ); |
189 | } elseif ( $logType === 'move' ) { |
190 | // Purge the target file as well |
191 | |
192 | $params = unserialize( $row->log_params ); |
193 | if ( isset( $params['4::target'] ) ) { |
194 | $target = $params['4::target']; |
195 | $targetFile = $repo->newFile( Title::makeTitle( NS_FILE, $target ) ); |
196 | $targetFile->purgeCache(); |
197 | $this->verbose( "Purged file {$target}; move target @{$row->log_timestamp}.\n" ); |
198 | } |
199 | } |
200 | |
201 | $this->verbose( "Purged file {$row->log_title}; {$type} @{$row->log_timestamp}.\n" ); |
202 | |
203 | if ( $this->hasOption( 'sleep-per-batch' ) && ++$bSize > $this->getBatchSize() ) { |
204 | $bSize = 0; |
205 | // sleep-per-batch is milliseconds, usleep wants micro seconds. |
206 | usleep( 1000 * (int)$this->getOption( 'sleep-per-batch' ) ); |
207 | } |
208 | } |
209 | } |
210 | } |
211 | |
212 | protected function purgeFromArchiveTable( LocalRepo $repo, LocalFile $file ) { |
213 | $dbr = $repo->getReplicaDB(); |
214 | $res = $dbr->newSelectQueryBuilder() |
215 | ->select( [ 'fa_archive_name' ] ) |
216 | ->from( 'filearchive' ) |
217 | ->where( [ 'fa_name' => $file->getName() ] ) |
218 | ->caller( __METHOD__ )->fetchResultSet(); |
219 | |
220 | foreach ( $res as $row ) { |
221 | if ( $row->fa_archive_name === null ) { |
222 | // Was not an old version (current version names checked already) |
223 | continue; |
224 | } |
225 | $ofile = $repo->newFromArchiveName( $file->getTitle(), $row->fa_archive_name ); |
226 | // If there is an orphaned storage file still there...delete it |
227 | if ( !$file->exists() && $repo->fileExists( $ofile->getPath() ) ) { |
228 | $dpath = $this->getDeletedPath( $repo, $ofile ); |
229 | if ( $repo->fileExists( $dpath ) ) { |
230 | // Check to avoid data loss |
231 | $repo->getBackend()->delete( [ 'src' => $ofile->getPath() ] ); |
232 | $this->output( "Deleted orphan file: {$ofile->getPath()}.\n" ); |
233 | } else { |
234 | $this->error( "File was not deleted: {$ofile->getPath()}.\n" ); |
235 | } |
236 | } |
237 | $file->purgeOldThumbnails( $row->fa_archive_name ); |
238 | } |
239 | } |
240 | |
241 | protected function getDeletedPath( LocalRepo $repo, LocalFile $file ) { |
242 | $hash = $repo->getFileSha1( $file->getPath() ); |
243 | $key = "{$hash}.{$file->getExtension()}"; |
244 | |
245 | return $repo->getDeletedHashPath( $key ) . $key; |
246 | } |
247 | |
248 | /** |
249 | * Send an output message iff the 'verbose' option has been provided. |
250 | * |
251 | * @param string $msg Message to output |
252 | */ |
253 | protected function verbose( $msg ) { |
254 | if ( $this->hasOption( 'verbose' ) ) { |
255 | $this->output( $msg ); |
256 | } |
257 | } |
258 | } |
259 | |
260 | $maintClass = PurgeChangedFiles::class; |
261 | require_once RUN_MAINTENANCE_IF_MAIN; |