MediaWiki master
cleanupUploadStash.php
Go to the documentation of this file.
1<?php
30
31require_once __DIR__ . '/Maintenance.php';
32
40
41 public function __construct() {
42 parent::__construct();
43 $this->addDescription( 'Clean up abandoned files in temporary uploaded file stash' );
44 $this->setBatchSize( 50 );
45 }
46
47 public function execute() {
48 $repo = $this->getServiceContainer()->getRepoGroup()->getLocalRepo();
49 $tempRepo = $repo->getTempRepo();
50
51 $dbr = $repo->getReplicaDB();
52
53 // how far back should this look for files to delete?
54 $cutoff = time() - (int)$this->getConfig()->get( MainConfigNames::UploadStashMaxAge );
55
56 $this->output( "Getting list of files to clean up...\n" );
57 $res = $dbr->newSelectQueryBuilder()
58 ->select( 'us_key' )
59 ->from( 'uploadstash' )
60 ->where( $dbr->expr( 'us_timestamp', '<', $dbr->timestamp( $cutoff ) ) )
61 ->caller( __METHOD__ )
62 ->fetchResultSet();
63
64 // Delete all registered stash files...
65 if ( $res->numRows() == 0 ) {
66 $this->output( "No stashed files to cleanup according to the DB.\n" );
67 } else {
68 // finish the read before starting writes.
69 $keys = [];
70 foreach ( $res as $row ) {
71 $keys[] = $row->us_key;
72 }
73
74 $this->output( 'Removing ' . count( $keys ) . " file(s)...\n" );
75 // this could be done some other, more direct/efficient way, but using
76 // UploadStash's own methods means it's less likely to fall accidentally
77 // out-of-date someday
78 $stash = new UploadStash( $repo );
79
80 $i = 0;
81 foreach ( $keys as $key ) {
82 $i++;
83 try {
84 $stash->getFile( $key, true );
85 $stash->removeFileNoAuth( $key );
86 } catch ( UploadStashException $ex ) {
87 $type = get_class( $ex );
88 $this->output( "Failed removing stashed upload with key: $key ($type)\n" );
89 }
90 if ( $i % 100 == 0 ) {
91 $this->waitForReplication();
92 $this->output( "$i\n" );
93 }
94 }
95 $this->output( "$i done\n" );
96 }
97
98 // Delete all the corresponding thumbnails...
99 $dir = $tempRepo->getZonePath( 'thumb' );
100 $iterator = $tempRepo->getBackend()->getFileList( [ 'dir' => $dir, 'adviseStat' => 1 ] );
101 if ( $iterator === null ) {
102 $this->fatalError( "Could not get file listing." );
103 }
104 $this->output( "Deleting old thumbnails...\n" );
105 $i = 0;
106 $batch = [];
107 foreach ( $iterator as $file ) {
108 if ( wfTimestamp( TS_UNIX, $tempRepo->getFileTimestamp( "$dir/$file" ) ) < $cutoff ) {
109 $batch[] = [ 'op' => 'delete', 'src' => "$dir/$file" ];
110 if ( count( $batch ) >= $this->getBatchSize() ) {
111 $this->doOperations( $tempRepo, $batch );
112 $i += count( $batch );
113 $batch = [];
114 $this->output( "$i\n" );
115 }
116 }
117 }
118 if ( count( $batch ) ) {
119 $this->doOperations( $tempRepo, $batch );
120 $i += count( $batch );
121 }
122 $this->output( "$i done\n" );
123
124 // Apparently lots of stash files are not registered in the DB...
125 $dir = $tempRepo->getZonePath( 'public' );
126 $iterator = $tempRepo->getBackend()->getFileList( [ 'dir' => $dir, 'adviseStat' => 1 ] );
127 if ( $iterator === null ) {
128 $this->fatalError( "Could not get file listing." );
129 }
130 $this->output( "Deleting orphaned temp files...\n" );
131 if ( strpos( $dir, '/local-temp' ) === false ) {
132 $this->output( "Temp repo might be misconfigured. It points to directory: '$dir' \n" );
133 }
134
135 $i = 0;
136 $batch = [];
137 foreach ( $iterator as $file ) {
138 if ( wfTimestamp( TS_UNIX, $tempRepo->getFileTimestamp( "$dir/$file" ) ) < $cutoff ) {
139 $batch[] = [ 'op' => 'delete', 'src' => "$dir/$file" ];
140 if ( count( $batch ) >= $this->getBatchSize() ) {
141 $this->doOperations( $tempRepo, $batch );
142 $i += count( $batch );
143 $batch = [];
144 $this->output( "$i\n" );
145 }
146 }
147 }
148 if ( count( $batch ) ) {
149 $this->doOperations( $tempRepo, $batch );
150 $i += count( $batch );
151 }
152 $this->output( "$i done\n" );
153 }
154
155 protected function doOperations( FileRepo $tempRepo, array $ops ) {
156 $status = $tempRepo->getBackend()->doQuickOperations( $ops );
157 if ( !$status->isOK() ) {
158 $this->error( print_r( Status::wrap( $status )->getErrorsArray(), true ) );
159 }
160 }
161}
162
163$maintClass = CleanupUploadStash::class;
164require_once RUN_MAINTENANCE_IF_MAIN;
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Maintenance script to remove old or broken uploads from temporary uploaded file storage and clean up ...
execute()
Do the actual work.
doOperations(FileRepo $tempRepo, array $ops)
__construct()
Default constructor.
Base class for file repositories.
Definition FileRepo.php:52
getBackend()
Get the file backend instance.
Definition FileRepo.php:254
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
error( $err, $die=0)
Throw an error to the user.
output( $out, $channel=null)
Throw some output to the user.
waitForReplication()
Wait for replica DBs to catch up.
getServiceContainer()
Returns the main service container.
getBatchSize()
Returns batch size.
addDescription( $text)
Set the description text.
setBatchSize( $s=0)
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
A class containing constants representing the names of configuration variables.
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Definition Status.php:54
UploadStash is intended to accomplish a few things: