MediaWiki  1.23.1
purgeChangedFiles.php
Go to the documentation of this file.
1 <?php
25 require_once __DIR__ . '/Maintenance.php';
26 
38  private static $typeMappings = array(
39  'created' => array(
40  'upload' => array( 'upload' ),
41  'import' => array( 'upload', 'interwiki' ),
42  ),
43  'deleted' => array(
44  'delete' => array( 'delete', 'revision' ),
45  'suppress' => array( 'delete', 'revision' ),
46  ),
47  'modified' => array(
48  'upload' => array( 'overwrite', 'revert' ),
49  'move' => array( 'move', 'move_redir' ),
50  ),
51  );
52 
56  private $startTimestamp;
57 
61  private $endTimestamp;
62 
63  public function __construct() {
64  parent::__construct();
65  $this->mDescription = "Scan the logging table and purge files and thumbnails.";
66  $this->addOption( 'starttime', 'Starting timestamp', true, true );
67  $this->addOption( 'endtime', 'Ending timestamp', true, true );
68  $this->addOption( 'type', 'Comma-separated list of types of changes to send purges for (' .
69  implode( ',', array_keys( self::$typeMappings ) ) . ',all)', false, true );
70  $this->addOption( 'htcp-dest', 'HTCP announcement destination (IP:port)', false, true );
71  $this->addOption( 'dry-run', 'Do not send purge requests' );
72  $this->addOption( 'sleep-per-batch', 'Milliseconds to sleep between batches', false, true );
73  $this->addOption( 'verbose', 'Show more output', false, false, 'v' );
74  $this->setBatchSize( 100 );
75  }
76 
77  public function execute() {
78  global $wgHTCPRouting;
79 
80  if ( $this->hasOption( 'htcp-dest' ) ) {
81  $parts = explode( ':', $this->getOption( 'htcp-dest' ) );
82  if ( count( $parts ) < 2 ) {
83  // Add default htcp port
84  $parts[] = '4827';
85  }
86 
87  // Route all HTCP messages to provided host:port
88  $wgHTCPRouting = array(
89  '' => array( 'host' => $parts[0], 'port' => $parts[1] ),
90  );
91  $this->verbose( "HTCP broadcasts to {$parts[0]}:{$parts[1]}\n" );
92  }
93 
94  // Find out which actions we should be concerned with
95  $typeOpt = $this->getOption( 'type', 'all' );
96  $validTypes = array_keys( self::$typeMappings );
97  if ( $typeOpt === 'all' ) {
98  // Convert 'all' to all registered types
99  $typeOpt = implode( ',', $validTypes );
100  }
101  $typeList = explode( ',', $typeOpt );
102  foreach ( $typeList as $type ) {
103  if ( !in_array( $type, $validTypes ) ) {
104  $this->error( "\nERROR: Unknown type: {$type}\n" );
105  $this->maybeHelp( true );
106  }
107  }
108 
109  // Validate the timestamps
110  $dbr = $this->getDB( DB_SLAVE );
111  $this->startTimestamp = $dbr->timestamp( $this->getOption( 'starttime' ) );
112  $this->endTimestamp = $dbr->timestamp( $this->getOption( 'endtime' ) );
113 
114  if ( $this->startTimestamp > $this->endTimestamp ) {
115  $this->error( "\nERROR: starttime after endtime\n" );
116  $this->maybeHelp( true );
117  }
118 
119  // Turn on verbose when dry-run is enabled
120  if ( $this->hasOption( 'dry-run' ) ) {
121  $this->mOptions['verbose'] = 1;
122  }
123 
124  $this->verbose( 'Purging files that were: ' . implode( ', ', $typeList ) . "\n" );
125  foreach ( $typeList as $type ) {
126  $this->verbose( "Checking for {$type} files...\n" );
127  $this->purgeFromLogType( $type );
128  if ( !$this->hasOption( 'dry-run' ) ) {
129  $this->verbose( "...{$type} files purged.\n\n" );
130  }
131  }
132  }
133 
139  protected function purgeFromLogType( $type ) {
140  $repo = RepoGroup::singleton()->getLocalRepo();
141  $dbr = $this->getDB( DB_SLAVE );
142 
143  foreach ( self::$typeMappings[$type] as $logType => $logActions ) {
144  $this->verbose( "Scanning for {$logType}/" . implode( ',', $logActions ) . "\n" );
145 
146  $res = $dbr->select(
147  'logging',
148  array( 'log_title', 'log_timestamp', 'log_params' ),
149  array(
150  'log_namespace' => NS_FILE,
151  'log_type' => $logType,
152  'log_action' => $logActions,
153  'log_timestamp >= ' . $dbr->addQuotes( $this->startTimestamp ),
154  'log_timestamp <= ' . $dbr->addQuotes( $this->endTimestamp ),
155  ),
156  __METHOD__
157  );
158 
159  $bSize = 0;
160  foreach ( $res as $row ) {
161  $file = $repo->newFile( Title::makeTitle( NS_FILE, $row->log_title ) );
162 
163  if ( $this->hasOption( 'dry-run' ) ) {
164  $this->verbose( "{$type}[{$row->log_timestamp}]: {$row->log_title}\n" );
165  continue;
166  }
167 
168  // Purge current version and any versions in oldimage table
169  $file->purgeCache();
170  $file->purgeHistory();
171 
172  if ( $logType === 'delete' ) {
173  // If there is an orphaned storage file... delete it
174  if ( !$file->exists() && $repo->fileExists( $file->getPath() ) ) {
175  $dpath = $this->getDeletedPath( $repo, $file );
176  if ( $repo->fileExists( $dpath ) ) {
177  // Sanity check to avoid data loss
178  $repo->getBackend()->delete( array( 'src' => $file->getPath() ) );
179  $this->verbose( "Deleted orphan file: {$file->getPath()}.\n" );
180 
181  } else {
182  $this->error( "File was not deleted: {$file->getPath()}.\n" );
183  }
184  }
185 
186  // Purge items from fileachive table (rows are likely here)
187  $this->purgeFromArchiveTable( $repo, $file );
188 
189  } elseif ( $logType === 'move' ) {
190  // Purge the target file as well
191 
192  $params = unserialize( $row->log_params );
193  if ( isset( $params['4::target'] ) ) {
194  $target = $params['4::target'];
195  $targetFile = $repo->newFile( Title::makeTitle( NS_FILE, $target ) );
196  $targetFile->purgeCache();
197  $targetFile->purgeHistory();
198  $this->verbose( "Purged file {$target}; move target @{$row->log_timestamp}.\n" );
199  }
200  }
201 
202  $this->verbose( "Purged file {$row->log_title}; {$type} @{$row->log_timestamp}.\n" );
203 
204  if ( $this->hasOption( 'sleep-per-batch' ) && ++$bSize > $this->mBatchSize ) {
205  $bSize = 0;
206  // sleep-per-batch is milliseconds, usleep wants micro seconds.
207  usleep( 1000 * (int)$this->getOption( 'sleep-per-batch' ) );
208  }
209  }
210  }
211  }
212 
213  protected function purgeFromArchiveTable( LocalRepo $repo, LocalFile $file ) {
214  $dbr = $repo->getSlaveDB();
215  $res = $dbr->select(
216  'filearchive',
217  array( 'fa_archive_name' ),
218  array( 'fa_name' => $file->getName() ),
219  __METHOD__
220  );
221 
222  foreach ( $res as $row ) {
223  if ( $row->fa_archive_name === null ) {
224  // Was not an old version (current version names checked already)
225  continue;
226  }
227  $ofile = $repo->newFromArchiveName( $file->getTitle(), $row->fa_archive_name );
228  // If there is an orphaned storage file still there...delete it
229  if ( !$file->exists() && $repo->fileExists( $ofile->getPath() ) ) {
230  $dpath = $this->getDeletedPath( $repo, $ofile );
231  if ( $repo->fileExists( $dpath ) ) {
232  // Sanity check to avoid data loss
233  $repo->getBackend()->delete( array( 'src' => $ofile->getPath() ) );
234  $this->output( "Deleted orphan file: {$ofile->getPath()}.\n" );
235 
236  } else {
237  $this->error( "File was not deleted: {$ofile->getPath()}.\n" );
238  }
239  }
240  $file->purgeOldThumbnails( $row->fa_archive_name );
241  }
242  }
243 
244  protected function getDeletedPath( LocalRepo $repo, LocalFile $file ) {
245  $hash = $repo->getFileSha1( $file->getPath() );
246  $key = "{$hash}.{$file->getExtension()}";
247  return $repo->getDeletedHashPath( $key ) . $key;
248  }
249 
255  protected function verbose( $msg ) {
256  if ( $this->hasOption( 'verbose' ) ) {
257  $this->output( $msg );
258  }
259  }
260 
261 }
262 
263 $maintClass = "PurgeChangedFiles";
264 require_once RUN_MAINTENANCE_IF_MAIN;
Title\makeTitle
static & makeTitle( $ns, $title, $fragment='', $interwiki='')
Create a new Title from a namespace index and a DB key.
Definition: Title.php:398
RepoGroup\singleton
static singleton()
Get a RepoGroup instance.
Definition: RepoGroup.php:53
php
skin txt MediaWiki includes four core it has been set as the default in MediaWiki since the replacing Monobook it had been been the default skin since before being replaced by Vector largely rewritten in while keeping its appearance Several legacy skins were removed in the as the burden of supporting them became too heavy to bear Those in etc for skin dependent CSS etc for skin dependent JavaScript These can also be customised on a per user by etc This feature has led to a wide variety of user styles becoming that gallery is a good place to ending in php
Definition: skin.txt:62
PurgeChangedFiles\$endTimestamp
string $endTimestamp
Definition: purgeChangedFiles.php:59
Maintenance\maybeHelp
maybeHelp( $force=false)
Maybe show the help.
Definition: Maintenance.php:710
$maintClass
$maintClass
Definition: purgeChangedFiles.php:261
Maintenance\addOption
addOption( $name, $description, $required=false, $withArg=false, $shortName=false)
Add a parameter to the script.
Definition: Maintenance.php:169
NS_FILE
const NS_FILE
Definition: Defines.php:85
RUN_MAINTENANCE_IF_MAIN
require_once RUN_MAINTENANCE_IF_MAIN
Definition: maintenance.txt:50
$params
$params
Definition: styleTest.css.php:40
LocalFile\purgeOldThumbnails
purgeOldThumbnails( $archiveName)
Delete cached transformed files for an archived version only.
Definition: LocalFile.php:877
PurgeChangedFiles\__construct
__construct()
Default constructor.
Definition: purgeChangedFiles.php:61
Maintenance
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: maintenance.txt:39
PurgeChangedFiles\verbose
verbose( $msg)
Send an output message iff the 'verbose' option has been provided.
Definition: purgeChangedFiles.php:253
$dbr
$dbr
Definition: testCompression.php:48
Maintenance\getDB
& getDB( $db, $groups=array(), $wiki=false)
Returns a database to be used by current maintenance script.
Definition: Maintenance.php:1007
LocalRepo\getSlaveDB
getSlaveDB()
Get a connection to the slave DB.
Definition: LocalRepo.php:454
PurgeChangedFiles\purgeFromArchiveTable
purgeFromArchiveTable(LocalRepo $repo, LocalFile $file)
Definition: purgeChangedFiles.php:211
array
the array() calling protocol came about after MediaWiki 1.4rc1.
List of Api Query prop modules.
PurgeChangedFiles\$startTimestamp
string $startTimestamp
Definition: purgeChangedFiles.php:55
global
when a variable name is used in a it is silently declared as a new masking the global
Definition: design.txt:93
PurgeChangedFiles\$typeMappings
static $typeMappings
Definition: purgeChangedFiles.php:38
FileRepo\fileExists
fileExists( $file)
Checks existence of a a file.
Definition: FileRepo.php:1340
FileRepo\getFileSha1
getFileSha1( $virtualUrl)
Get the sha1 (base 36) of a file with a given virtual URL/storage path.
Definition: FileRepo.php:1563
LocalFile
Class to represent a local file in the wiki's own database.
Definition: LocalFile.php:46
LocalRepo\newFromArchiveName
newFromArchiveName( $title, $archiveName)
Definition: LocalRepo.php:64
FileRepo\getBackend
getBackend()
Get the file backend instance.
Definition: FileRepo.php:190
PurgeChangedFiles\execute
execute()
Do the actual work.
Definition: purgeChangedFiles.php:75
PurgeChangedFiles\getDeletedPath
getDeletedPath(LocalRepo $repo, LocalFile $file)
Definition: purgeChangedFiles.php:242
$hash
return false to override stock group addition can be modified try getUserPermissionsErrors userCan checks are continued by internal code can override on output return false to not delete it return false to override the default password checks & $hash
Definition: hooks.txt:2697
$file
if(PHP_SAPI !='cli') $file
Definition: UtfNormalTest2.php:30
PurgeChangedFiles\purgeFromLogType
purgeFromLogType( $type)
Purge cache and thumbnails for changes of the given type.
Definition: purgeChangedFiles.php:137
DB_SLAVE
const DB_SLAVE
Definition: Defines.php:55
Maintenance\getOption
getOption( $name, $default=null)
Get an option, or return the default.
Definition: Maintenance.php:191
as
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
Maintenance\error
error( $err, $die=0)
Throw an error to the user.
Definition: Maintenance.php:333
Maintenance\output
output( $out, $channel=null)
Throw some output to the user.
Definition: Maintenance.php:314
Maintenance\hasOption
hasOption( $name)
Checks to see if a particular param exists.
Definition: Maintenance.php:181
$res
$res
Definition: database.txt:21
FileRepo\getDeletedHashPath
getDeletedHashPath( $key)
Get a relative path for a deletion archive key, e.g.
Definition: FileRepo.php:1465
PurgeChangedFiles
Maintenance script that scans the deletion log and purges affected files within a timeframe.
Definition: purgeChangedFiles.php:33
LocalRepo
A repository that stores files in the local filesystem and registers them in the wiki's own database.
Definition: LocalRepo.php:31
Maintenance\setBatchSize
setBatchSize( $s=0)
Set the batch size.
Definition: Maintenance.php:254
$type
$type
Definition: testCompression.php:46