MediaWiki  master
purgeChangedFiles.php
Go to the documentation of this file.
1 <?php
26 
27 require_once __DIR__ . '/Maintenance.php';
28 
40  private static $typeMappings = [
41  'created' => [
42  'upload' => [ 'upload' ],
43  'import' => [ 'upload', 'interwiki' ],
44  ],
45  'deleted' => [
46  'delete' => [ 'delete', 'revision' ],
47  'suppress' => [ 'delete', 'revision' ],
48  ],
49  'modified' => [
50  'upload' => [ 'overwrite', 'revert' ],
51  'move' => [ 'move', 'move_redir' ],
52  ],
53  ];
54 
58  private $startTimestamp;
59 
63  private $endTimestamp;
64 
65  public function __construct() {
66  parent::__construct();
67  $this->addDescription( 'Scan the logging table and purge files and thumbnails.' );
68  $this->addOption( 'starttime', 'Starting timestamp', true, true );
69  $this->addOption( 'endtime', 'Ending timestamp', true, true );
70  $this->addOption( 'type', 'Comma-separated list of types of changes to send purges for (' .
71  implode( ',', array_keys( self::$typeMappings ) ) . ',all)', false, true );
72  $this->addOption( 'htcp-dest', 'HTCP announcement destination (IP:port)', false, true );
73  $this->addOption( 'dry-run', 'Do not send purge requests' );
74  $this->addOption( 'sleep-per-batch', 'Milliseconds to sleep between batches', false, true );
75  $this->addOption( 'verbose', 'Show more output', false, false, 'v' );
76  $this->setBatchSize( 100 );
77  }
78 
79  public function execute() {
80  global $wgHTCPRouting;
81 
82  if ( $this->hasOption( 'htcp-dest' ) ) {
83  $parts = explode( ':', $this->getOption( 'htcp-dest' ), 2 );
84  if ( count( $parts ) < 2 ) {
85  // Add default htcp port
86  $parts[] = '4827';
87  }
88 
89  // Route all HTCP messages to provided host:port
90  $wgHTCPRouting = [
91  '' => [ 'host' => $parts[0], 'port' => $parts[1] ],
92  ];
93  $this->verbose( "HTCP broadcasts to {$parts[0]}:{$parts[1]}\n" );
94  }
95 
96  // Find out which actions we should be concerned with
97  $typeOpt = $this->getOption( 'type', 'all' );
98  if ( $typeOpt === 'all' ) {
99  // Convert 'all' to all registered types
100  $typeOpt = implode( ',', array_keys( self::$typeMappings ) );
101  }
102  $typeList = explode( ',', $typeOpt );
103  foreach ( $typeList as $type ) {
104  if ( !isset( self::$typeMappings[$type] ) ) {
105  $this->error( "\nERROR: Unknown type: {$type}\n" );
106  $this->maybeHelp( true );
107  }
108  }
109 
110  // Validate the timestamps
111  $dbr = $this->getDB( DB_REPLICA );
112  $this->startTimestamp = $dbr->timestamp( $this->getOption( 'starttime' ) );
113  $this->endTimestamp = $dbr->timestamp( $this->getOption( 'endtime' ) );
114 
115  if ( $this->startTimestamp > $this->endTimestamp ) {
116  $this->error( "\nERROR: starttime after endtime\n" );
117  $this->maybeHelp( true );
118  }
119 
120  // Turn on verbose when dry-run is enabled
121  if ( $this->hasOption( 'dry-run' ) ) {
122  $this->mOptions['verbose'] = 1;
123  }
124 
125  $this->verbose( 'Purging files that were: ' . implode( ', ', $typeList ) . "\n" );
126  foreach ( $typeList as $type ) {
127  $this->verbose( "Checking for {$type} files...\n" );
128  $this->purgeFromLogType( $type );
129  if ( !$this->hasOption( 'dry-run' ) ) {
130  $this->verbose( "...{$type} files purged.\n\n" );
131  }
132  }
133  }
134 
140  protected function purgeFromLogType( $type ) {
141  $repo = MediaWikiServices::getInstance()->getRepoGroup()->getLocalRepo();
142  $dbr = $this->getDB( DB_REPLICA );
143 
144  foreach ( self::$typeMappings[$type] as $logType => $logActions ) {
145  $this->verbose( "Scanning for {$logType}/" . implode( ',', $logActions ) . "\n" );
146 
147  $res = $dbr->select(
148  'logging',
149  [ 'log_title', 'log_timestamp', 'log_params' ],
150  [
151  'log_namespace' => NS_FILE,
152  'log_type' => $logType,
153  'log_action' => $logActions,
154  'log_timestamp >= ' . $dbr->addQuotes( $this->startTimestamp ),
155  'log_timestamp <= ' . $dbr->addQuotes( $this->endTimestamp ),
156  ],
157  __METHOD__
158  );
159 
160  $bSize = 0;
161  foreach ( $res as $row ) {
162  $file = $repo->newFile( Title::makeTitle( NS_FILE, $row->log_title ) );
163 
164  if ( $this->hasOption( 'dry-run' ) ) {
165  $this->verbose( "{$type}[{$row->log_timestamp}]: {$row->log_title}\n" );
166  continue;
167  }
168 
169  // Purge current version and its thumbnails
170  $file->purgeCache();
171  // Purge the old versions and their thumbnails
172  foreach ( $file->getHistory() as $oldFile ) {
173  $oldFile->purgeCache();
174  }
175 
176  if ( $logType === 'delete' ) {
177  // If there is an orphaned storage file... delete it
178  if ( !$file->exists() && $repo->fileExists( $file->getPath() ) ) {
179  $dpath = $this->getDeletedPath( $repo, $file );
180  if ( $repo->fileExists( $dpath ) ) {
181  // Check to avoid data loss
182  $repo->getBackend()->delete( [ 'src' => $file->getPath() ] );
183  $this->verbose( "Deleted orphan file: {$file->getPath()}.\n" );
184  } else {
185  $this->error( "File was not deleted: {$file->getPath()}.\n" );
186  }
187  }
188 
189  // Purge items from fileachive table (rows are likely here)
190  $this->purgeFromArchiveTable( $repo, $file );
191  } elseif ( $logType === 'move' ) {
192  // Purge the target file as well
193 
194  $params = unserialize( $row->log_params );
195  if ( isset( $params['4::target'] ) ) {
196  $target = $params['4::target'];
197  $targetFile = $repo->newFile( Title::makeTitle( NS_FILE, $target ) );
198  $targetFile->purgeCache();
199  $this->verbose( "Purged file {$target}; move target @{$row->log_timestamp}.\n" );
200  }
201  }
202 
203  $this->verbose( "Purged file {$row->log_title}; {$type} @{$row->log_timestamp}.\n" );
204 
205  if ( $this->hasOption( 'sleep-per-batch' ) && ++$bSize > $this->getBatchSize() ) {
206  $bSize = 0;
207  // sleep-per-batch is milliseconds, usleep wants micro seconds.
208  usleep( 1000 * (int)$this->getOption( 'sleep-per-batch' ) );
209  }
210  }
211  }
212  }
213 
214  protected function purgeFromArchiveTable( LocalRepo $repo, LocalFile $file ) {
215  $dbr = $repo->getReplicaDB();
216  $res = $dbr->select(
217  'filearchive',
218  [ 'fa_archive_name' ],
219  [ 'fa_name' => $file->getName() ],
220  __METHOD__
221  );
222 
223  foreach ( $res as $row ) {
224  if ( $row->fa_archive_name === null ) {
225  // Was not an old version (current version names checked already)
226  continue;
227  }
228  $ofile = $repo->newFromArchiveName( $file->getTitle(), $row->fa_archive_name );
229  // If there is an orphaned storage file still there...delete it
230  if ( !$file->exists() && $repo->fileExists( $ofile->getPath() ) ) {
231  $dpath = $this->getDeletedPath( $repo, $ofile );
232  if ( $repo->fileExists( $dpath ) ) {
233  // Check to avoid data loss
234  $repo->getBackend()->delete( [ 'src' => $ofile->getPath() ] );
235  $this->output( "Deleted orphan file: {$ofile->getPath()}.\n" );
236  } else {
237  $this->error( "File was not deleted: {$ofile->getPath()}.\n" );
238  }
239  }
240  $file->purgeOldThumbnails( $row->fa_archive_name );
241  }
242  }
243 
244  protected function getDeletedPath( LocalRepo $repo, LocalFile $file ) {
245  $hash = $repo->getFileSha1( $file->getPath() );
246  $key = "{$hash}.{$file->getExtension()}";
247 
248  return $repo->getDeletedHashPath( $key ) . $key;
249  }
250 
256  protected function verbose( $msg ) {
257  if ( $this->hasOption( 'verbose' ) ) {
258  $this->output( $msg );
259  }
260  }
261 }
262 
263 $maintClass = PurgeChangedFiles::class;
264 require_once RUN_MAINTENANCE_IF_MAIN;
const NS_FILE
Definition: Defines.php:70
fileExists( $file)
Checks existence of a file.
Definition: FileRepo.php:1433
getFileSha1( $virtualUrl)
Get the sha1 (base 36) of a file with a given virtual URL/storage path.
Definition: FileRepo.php:1661
getDeletedHashPath( $key)
Get a relative path for a deletion archive key, e.g.
Definition: FileRepo.php:1557
getBackend()
Get the file backend instance.
Definition: FileRepo.php:251
Local file in the wiki's own database.
Definition: LocalFile.php:61
purgeOldThumbnails( $archiveName)
Delete cached transformed files for an archived version only.
Definition: LocalFile.php:1376
Local repository that stores files in the local filesystem and registers them in the wiki's own datab...
Definition: LocalRepo.php:41
getReplicaDB()
Get a connection to the replica DB.
Definition: LocalRepo.php:516
newFromArchiveName( $title, $archiveName)
Definition: LocalRepo.php:129
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:66
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
error( $err, $die=0)
Throw an error to the user.
output( $out, $channel=null)
Throw some output to the user.
hasOption( $name)
Checks to see if a particular option was set.
getBatchSize()
Returns batch size.
addDescription( $text)
Set the description text.
maybeHelp( $force=false)
Maybe show the help.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
setBatchSize( $s=0)
Service locator for MediaWiki core services.
Represents a title within MediaWiki.
Definition: Title.php:82
Maintenance script that scans the deletion log and purges affected files within a timeframe.
execute()
Do the actual work.
__construct()
Default constructor.
purgeFromLogType( $type)
Purge cache and thumbnails for changes of the given type.
getDeletedPath(LocalRepo $repo, LocalFile $file)
purgeFromArchiveTable(LocalRepo $repo, LocalFile $file)
verbose( $msg)
Send an output message iff the 'verbose' option has been provided.
$wgHTCPRouting
Config variable stub for the HTCPRouting setting, for use by phpdoc and IDEs.
const DB_REPLICA
Definition: defines.php:26
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition: router.php:42