MediaWiki  master
cleanupUploadStash.php
Go to the documentation of this file.
1 <?php
30 
31 require_once __DIR__ . '/Maintenance.php';
32 
40 
41  public function __construct() {
42  parent::__construct();
43  $this->addDescription( 'Clean up abandoned files in temporary uploaded file stash' );
44  $this->setBatchSize( 50 );
45  }
46 
47  public function execute() {
48  $repo = MediaWikiServices::getInstance()->getRepoGroup()->getLocalRepo();
49  $tempRepo = $repo->getTempRepo();
50 
51  $dbr = $repo->getReplicaDB();
52 
53  // how far back should this look for files to delete?
54  $cutoff = time() - (int)$this->getConfig()->get( MainConfigNames::UploadStashMaxAge );
55 
56  $this->output( "Getting list of files to clean up...\n" );
57  $res = $dbr->newSelectQueryBuilder()
58  ->select( 'us_key' )
59  ->from( 'uploadstash' )
60  ->where( 'us_timestamp < ' . $dbr->addQuotes( $dbr->timestamp( $cutoff ) ) )
61  ->caller( __METHOD__ )
62  ->fetchResultSet();
63 
64  // Delete all registered stash files...
65  if ( $res->numRows() == 0 ) {
66  $this->output( "No stashed files to cleanup according to the DB.\n" );
67  } else {
68  // finish the read before starting writes.
69  $keys = [];
70  foreach ( $res as $row ) {
71  $keys[] = $row->us_key;
72  }
73 
74  $this->output( 'Removing ' . count( $keys ) . " file(s)...\n" );
75  // this could be done some other, more direct/efficient way, but using
76  // UploadStash's own methods means it's less likely to fall accidentally
77  // out-of-date someday
78  $stash = new UploadStash( $repo );
79 
80  $i = 0;
81  foreach ( $keys as $key ) {
82  $i++;
83  try {
84  $stash->getFile( $key, true );
85  $stash->removeFileNoAuth( $key );
86  } catch ( UploadStashException $ex ) {
87  $type = get_class( $ex );
88  $this->output( "Failed removing stashed upload with key: $key ($type)\n" );
89  }
90  if ( $i % 100 == 0 ) {
91  $this->waitForReplication();
92  $this->output( "$i\n" );
93  }
94  }
95  $this->output( "$i done\n" );
96  }
97 
98  // Delete all the corresponding thumbnails...
99  $dir = $tempRepo->getZonePath( 'thumb' );
100  $iterator = $tempRepo->getBackend()->getFileList( [ 'dir' => $dir, 'adviseStat' => 1 ] );
101  if ( $iterator === null ) {
102  $this->fatalError( "Could not get file listing." );
103  }
104  $this->output( "Deleting old thumbnails...\n" );
105  $i = 0;
106  $batch = [];
107  foreach ( $iterator as $file ) {
108  if ( wfTimestamp( TS_UNIX, $tempRepo->getFileTimestamp( "$dir/$file" ) ) < $cutoff ) {
109  $batch[] = [ 'op' => 'delete', 'src' => "$dir/$file" ];
110  if ( count( $batch ) >= $this->getBatchSize() ) {
111  $this->doOperations( $tempRepo, $batch );
112  $i += count( $batch );
113  $batch = [];
114  $this->output( "$i\n" );
115  }
116  }
117  }
118  if ( count( $batch ) ) {
119  $this->doOperations( $tempRepo, $batch );
120  $i += count( $batch );
121  }
122  $this->output( "$i done\n" );
123 
124  // Apparently lots of stash files are not registered in the DB...
125  $dir = $tempRepo->getZonePath( 'public' );
126  $iterator = $tempRepo->getBackend()->getFileList( [ 'dir' => $dir, 'adviseStat' => 1 ] );
127  if ( $iterator === null ) {
128  $this->fatalError( "Could not get file listing." );
129  }
130  $this->output( "Deleting orphaned temp files...\n" );
131  if ( strpos( $dir, '/local-temp' ) === false ) {
132  $this->fatalError( "Temp repo is not using the temp container." );
133  }
134 
135  $i = 0;
136  $batch = [];
137  foreach ( $iterator as $file ) {
138  if ( wfTimestamp( TS_UNIX, $tempRepo->getFileTimestamp( "$dir/$file" ) ) < $cutoff ) {
139  $batch[] = [ 'op' => 'delete', 'src' => "$dir/$file" ];
140  if ( count( $batch ) >= $this->getBatchSize() ) {
141  $this->doOperations( $tempRepo, $batch );
142  $i += count( $batch );
143  $batch = [];
144  $this->output( "$i\n" );
145  }
146  }
147  }
148  if ( count( $batch ) ) {
149  $this->doOperations( $tempRepo, $batch );
150  $i += count( $batch );
151  }
152  $this->output( "$i done\n" );
153  }
154 
155  protected function doOperations( FileRepo $tempRepo, array $ops ) {
156  $status = $tempRepo->getBackend()->doQuickOperations( $ops );
157  if ( !$status->isOK() ) {
158  $this->error( print_r( Status::wrap( $status )->getErrorsArray(), true ) );
159  }
160  }
161 }
162 
163 $maintClass = CleanupUploadStash::class;
164 require_once RUN_MAINTENANCE_IF_MAIN;
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Maintenance script to remove old or broken uploads from temporary uploaded file storage and clean up ...
execute()
Do the actual work.
doOperations(FileRepo $tempRepo, array $ops)
__construct()
Default constructor.
Base class for file repositories.
Definition: FileRepo.php:48
getBackend()
Get the file backend instance.
Definition: FileRepo.php:251
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:66
error( $err, $die=0)
Throw an error to the user.
output( $out, $channel=null)
Throw some output to the user.
waitForReplication()
Wait for replica DBs to catch up.
getBatchSize()
Returns batch size.
addDescription( $text)
Set the description text.
setBatchSize( $s=0)
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
A class containing constants representing the names of configuration variables.
Service locator for MediaWiki core services.
static wrap( $sv)
Succinct helper method to wrap a StatusValue.
Definition: Status.php:64
UploadStash is intended to accomplish a few things:
Definition: UploadStash.php:57
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition: router.php:42