MediaWiki  master
cleanupUploadStash.php
Go to the documentation of this file.
1 <?php
28 require_once __DIR__ . '/Maintenance.php';
29 
37 
38  public function __construct() {
39  parent::__construct();
40  $this->addDescription( 'Clean up abandoned files in temporary uploaded file stash' );
41  $this->setBatchSize( 50 );
42  }
43 
44  public function execute() {
45  global $wgUploadStashMaxAge;
46 
47  $repo = RepoGroup::singleton()->getLocalRepo();
48  $tempRepo = $repo->getTempRepo();
49 
50  $dbr = $repo->getReplicaDB();
51 
52  // how far back should this look for files to delete?
53  $cutoff = time() - $wgUploadStashMaxAge;
54 
55  $this->output( "Getting list of files to clean up...\n" );
56  $res = $dbr->select(
57  'uploadstash',
58  'us_key',
59  'us_timestamp < ' . $dbr->addQuotes( $dbr->timestamp( $cutoff ) ),
60  __METHOD__
61  );
62 
63  // Delete all registered stash files...
64  if ( $res->numRows() == 0 ) {
65  $this->output( "No stashed files to cleanup according to the DB.\n" );
66  } else {
67  // finish the read before starting writes.
68  $keys = [];
69  foreach ( $res as $row ) {
70  array_push( $keys, $row->us_key );
71  }
72 
73  $this->output( 'Removing ' . count( $keys ) . " file(s)...\n" );
74  // this could be done some other, more direct/efficient way, but using
75  // UploadStash's own methods means it's less likely to fall accidentally
76  // out-of-date someday
77  $stash = new UploadStash( $repo );
78 
79  $i = 0;
80  foreach ( $keys as $key ) {
81  $i++;
82  try {
83  $stash->getFile( $key, true );
84  $stash->removeFileNoAuth( $key );
85  } catch ( UploadStashException $ex ) {
86  $type = get_class( $ex );
87  $this->output( "Failed removing stashed upload with key: $key ($type)\n" );
88  }
89  if ( $i % 100 == 0 ) {
91  $this->output( "$i\n" );
92  }
93  }
94  $this->output( "$i done\n" );
95  }
96 
97  // Delete all the corresponding thumbnails...
98  $dir = $tempRepo->getZonePath( 'thumb' );
99  $iterator = $tempRepo->getBackend()->getFileList( [ 'dir' => $dir, 'adviseStat' => 1 ] );
100  $this->output( "Deleting old thumbnails...\n" );
101  $i = 0;
102  $batch = []; // operation batch
103  foreach ( $iterator as $file ) {
104  if ( wfTimestamp( TS_UNIX, $tempRepo->getFileTimestamp( "$dir/$file" ) ) < $cutoff ) {
105  $batch[] = [ 'op' => 'delete', 'src' => "$dir/$file" ];
106  if ( count( $batch ) >= $this->getBatchSize() ) {
107  $this->doOperations( $tempRepo, $batch );
108  $i += count( $batch );
109  $batch = [];
110  $this->output( "$i\n" );
111  }
112  }
113  }
114  if ( count( $batch ) ) {
115  $this->doOperations( $tempRepo, $batch );
116  $i += count( $batch );
117  }
118  $this->output( "$i done\n" );
119 
120  // Apparently lots of stash files are not registered in the DB...
121  $dir = $tempRepo->getZonePath( 'public' );
122  $iterator = $tempRepo->getBackend()->getFileList( [ 'dir' => $dir, 'adviseStat' => 1 ] );
123  $this->output( "Deleting orphaned temp files...\n" );
124  if ( strpos( $dir, '/local-temp' ) === false ) { // sanity check
125  $this->fatalError( "Temp repo is not using the temp container." );
126  }
127  $i = 0;
128  $batch = []; // operation batch
129  foreach ( $iterator as $file ) {
130  if ( wfTimestamp( TS_UNIX, $tempRepo->getFileTimestamp( "$dir/$file" ) ) < $cutoff ) {
131  $batch[] = [ 'op' => 'delete', 'src' => "$dir/$file" ];
132  if ( count( $batch ) >= $this->getBatchSize() ) {
133  $this->doOperations( $tempRepo, $batch );
134  $i += count( $batch );
135  $batch = [];
136  $this->output( "$i\n" );
137  }
138  }
139  }
140  if ( count( $batch ) ) {
141  $this->doOperations( $tempRepo, $batch );
142  $i += count( $batch );
143  }
144  $this->output( "$i done\n" );
145  }
146 
147  protected function doOperations( FileRepo $tempRepo, array $ops ) {
148  $status = $tempRepo->getBackend()->doQuickOperations( $ops );
149  if ( !$status->isOK() ) {
150  // @phan-suppress-next-line PhanUndeclaredMethod
151  $this->error( print_r( $status->getErrorsArray(), true ) );
152  }
153  }
154 }
155 
156 $maintClass = CleanupUploadStash::class;
157 require_once RUN_MAINTENANCE_IF_MAIN;
RUN_MAINTENANCE_IF_MAIN
const RUN_MAINTENANCE_IF_MAIN
Definition: Maintenance.php:39
RepoGroup\singleton
static singleton()
Definition: RepoGroup.php:60
CleanupUploadStash\execute
execute()
Do the actual work.
Definition: cleanupUploadStash.php:44
Maintenance\fatalError
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
Definition: Maintenance.php:508
Maintenance\addDescription
addDescription( $text)
Set the description text.
Definition: Maintenance.php:352
true
return true
Definition: router.php:92
$maintClass
$maintClass
Definition: cleanupUploadStash.php:156
wfTimestamp
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Definition: GlobalFunctions.php:1871
UploadStash
UploadStash is intended to accomplish a few things:
Definition: UploadStash.php:53
$file
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition: router.php:42
Maintenance
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:86
$res
$res
Definition: testCompression.php:54
wfWaitForSlaves
wfWaitForSlaves( $ifWritesSince=null, $wiki=false, $cluster=false, $timeout=null)
Waits for the replica DBs to catch up to the master position.
Definition: GlobalFunctions.php:2722
$dbr
$dbr
Definition: testCompression.php:52
$wgUploadStashMaxAge
$wgUploadStashMaxAge
The maximum age of temporary (incomplete) uploaded files.
Definition: DefaultSettings.php:407
FileRepo
Base class for file repositories.
Definition: FileRepo.php:41
CleanupUploadStash\doOperations
doOperations(FileRepo $tempRepo, array $ops)
Definition: cleanupUploadStash.php:147
FileRepo\getBackend
getBackend()
Get the file backend instance.
Definition: FileRepo.php:242
Maintenance\getBatchSize
getBatchSize()
Returns batch size.
Definition: Maintenance.php:390
$keys
$keys
Definition: testCompression.php:69
Maintenance\error
error( $err, $die=0)
Throw an error to the user.
Definition: Maintenance.php:485
Maintenance\output
output( $out, $channel=null)
Throw some output to the user.
Definition: Maintenance.php:457
CleanupUploadStash\__construct
__construct()
Default constructor.
Definition: cleanupUploadStash.php:38
Maintenance\setBatchSize
setBatchSize( $s=0)
Set the batch size.
Definition: Maintenance.php:398
UploadStashException
Definition: UploadStashException.php:26
CleanupUploadStash
Maintenance script to remove old or broken uploads from temporary uploaded file storage and clean up ...
Definition: cleanupUploadStash.php:36
$type
$type
Definition: testCompression.php:50