MediaWiki  master
cleanupUploadStash.php
Go to the documentation of this file.
1 <?php
29 
30 require_once __DIR__ . '/Maintenance.php';
31 
39 
40  public function __construct() {
41  parent::__construct();
42  $this->addDescription( 'Clean up abandoned files in temporary uploaded file stash' );
43  $this->setBatchSize( 50 );
44  }
45 
46  public function execute() {
47  global $wgUploadStashMaxAge;
48 
49  $repo = MediaWikiServices::getInstance()->getRepoGroup()->getLocalRepo();
50  $tempRepo = $repo->getTempRepo();
51 
52  $dbr = $repo->getReplicaDB();
53 
54  // how far back should this look for files to delete?
55  $cutoff = time() - $wgUploadStashMaxAge;
56 
57  $this->output( "Getting list of files to clean up...\n" );
58  $res = $dbr->select(
59  'uploadstash',
60  'us_key',
61  'us_timestamp < ' . $dbr->addQuotes( $dbr->timestamp( $cutoff ) ),
62  __METHOD__
63  );
64 
65  // Delete all registered stash files...
66  if ( $res->numRows() == 0 ) {
67  $this->output( "No stashed files to cleanup according to the DB.\n" );
68  } else {
69  // finish the read before starting writes.
70  $keys = [];
71  foreach ( $res as $row ) {
72  array_push( $keys, $row->us_key );
73  }
74 
75  $this->output( 'Removing ' . count( $keys ) . " file(s)...\n" );
76  // this could be done some other, more direct/efficient way, but using
77  // UploadStash's own methods means it's less likely to fall accidentally
78  // out-of-date someday
79  $stash = new UploadStash( $repo );
80 
81  $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
82 
83  $i = 0;
84  foreach ( $keys as $key ) {
85  $i++;
86  try {
87  $stash->getFile( $key, true );
88  $stash->removeFileNoAuth( $key );
89  } catch ( UploadStashException $ex ) {
90  $type = get_class( $ex );
91  $this->output( "Failed removing stashed upload with key: $key ($type)\n" );
92  }
93  if ( $i % 100 == 0 ) {
94  $lbFactory->waitForReplication();
95  $this->output( "$i\n" );
96  }
97  }
98  $this->output( "$i done\n" );
99  }
100 
101  // Delete all the corresponding thumbnails...
102  $dir = $tempRepo->getZonePath( 'thumb' );
103  $iterator = $tempRepo->getBackend()->getFileList( [ 'dir' => $dir, 'adviseStat' => 1 ] );
104  if ( $iterator === null ) {
105  $this->fatalError( "Could not get file listing." );
106  }
107  $this->output( "Deleting old thumbnails...\n" );
108  $i = 0;
109  $batch = [];
110  foreach ( $iterator as $file ) {
111  if ( wfTimestamp( TS_UNIX, $tempRepo->getFileTimestamp( "$dir/$file" ) ) < $cutoff ) {
112  $batch[] = [ 'op' => 'delete', 'src' => "$dir/$file" ];
113  if ( count( $batch ) >= $this->getBatchSize() ) {
114  $this->doOperations( $tempRepo, $batch );
115  $i += count( $batch );
116  $batch = [];
117  $this->output( "$i\n" );
118  }
119  }
120  }
121  if ( count( $batch ) ) {
122  $this->doOperations( $tempRepo, $batch );
123  $i += count( $batch );
124  }
125  $this->output( "$i done\n" );
126 
127  // Apparently lots of stash files are not registered in the DB...
128  $dir = $tempRepo->getZonePath( 'public' );
129  $iterator = $tempRepo->getBackend()->getFileList( [ 'dir' => $dir, 'adviseStat' => 1 ] );
130  if ( $iterator === null ) {
131  $this->fatalError( "Could not get file listing." );
132  }
133  $this->output( "Deleting orphaned temp files...\n" );
134  if ( strpos( $dir, '/local-temp' ) === false ) {
135  // sanity check
136  $this->fatalError( "Temp repo is not using the temp container." );
137  }
138 
139  $i = 0;
140  $batch = [];
141  foreach ( $iterator as $file ) {
142  if ( wfTimestamp( TS_UNIX, $tempRepo->getFileTimestamp( "$dir/$file" ) ) < $cutoff ) {
143  $batch[] = [ 'op' => 'delete', 'src' => "$dir/$file" ];
144  if ( count( $batch ) >= $this->getBatchSize() ) {
145  $this->doOperations( $tempRepo, $batch );
146  $i += count( $batch );
147  $batch = [];
148  $this->output( "$i\n" );
149  }
150  }
151  }
152  if ( count( $batch ) ) {
153  $this->doOperations( $tempRepo, $batch );
154  $i += count( $batch );
155  }
156  $this->output( "$i done\n" );
157  }
158 
159  protected function doOperations( FileRepo $tempRepo, array $ops ) {
160  $status = $tempRepo->getBackend()->doQuickOperations( $ops );
161  if ( !$status->isOK() ) {
162  $this->error( print_r( Status::wrap( $status )->getErrorsArray(), true ) );
163  }
164  }
165 }
166 
167 $maintClass = CleanupUploadStash::class;
168 require_once RUN_MAINTENANCE_IF_MAIN;
RUN_MAINTENANCE_IF_MAIN
const RUN_MAINTENANCE_IF_MAIN
Definition: Maintenance.php:38
CleanupUploadStash\execute
execute()
Do the actual work.
Definition: cleanupUploadStash.php:46
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:154
Maintenance\fatalError
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
Definition: Maintenance.php:487
Maintenance\addDescription
addDescription( $text)
Set the description text.
Definition: Maintenance.php:327
$maintClass
$maintClass
Definition: cleanupUploadStash.php:167
wfTimestamp
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Definition: GlobalFunctions.php:1808
UploadStash
UploadStash is intended to accomplish a few things:
Definition: UploadStash.php:53
$file
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition: router.php:42
Maintenance
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:55
$res
$res
Definition: testCompression.php:57
$dbr
$dbr
Definition: testCompression.php:54
$wgUploadStashMaxAge
$wgUploadStashMaxAge
The maximum age of temporary (incomplete) uploaded files.
Definition: DefaultSettings.php:469
FileRepo
Base class for file repositories.
Definition: FileRepo.php:41
Status\wrap
static wrap( $sv)
Succinct helper method to wrap a StatusValue.
Definition: Status.php:62
CleanupUploadStash\doOperations
doOperations(FileRepo $tempRepo, array $ops)
Definition: cleanupUploadStash.php:159
FileRepo\getBackend
getBackend()
Get the file backend instance.
Definition: FileRepo.php:244
Maintenance\getBatchSize
getBatchSize()
Returns batch size.
Definition: Maintenance.php:366
$keys
$keys
Definition: testCompression.php:72
Maintenance\error
error( $err, $die=0)
Throw an error to the user.
Definition: Maintenance.php:463
Maintenance\output
output( $out, $channel=null)
Throw some output to the user.
Definition: Maintenance.php:434
CleanupUploadStash\__construct
__construct()
Default constructor.
Definition: cleanupUploadStash.php:40
Maintenance\setBatchSize
setBatchSize( $s=0)
Set the batch size.
Definition: Maintenance.php:374
UploadStashException
@newable
Definition: UploadStashException.php:27
CleanupUploadStash
Maintenance script to remove old or broken uploads from temporary uploaded file storage and clean up ...
Definition: cleanupUploadStash.php:38
$type
$type
Definition: testCompression.php:52