MediaWiki master
cleanupUploadStash.php
Go to the documentation of this file.
1<?php
31
32// @codeCoverageIgnoreStart
33require_once __DIR__ . '/Maintenance.php';
34// @codeCoverageIgnoreEnd
35
43
44 public function __construct() {
45 parent::__construct();
46 $this->addDescription( 'Clean up abandoned files in temporary uploaded file stash' );
47 $this->setBatchSize( 50 );
48 }
49
50 public function execute() {
51 $repo = $this->getServiceContainer()->getRepoGroup()->getLocalRepo();
52 $tempRepo = $repo->getTempRepo();
53
54 $dbr = $repo->getReplicaDB();
55
56 // how far back should this look for files to delete?
57 $cutoff = time() - (int)$this->getConfig()->get( MainConfigNames::UploadStashMaxAge );
58
59 $this->output( "Getting list of files to clean up...\n" );
60 $res = $dbr->newSelectQueryBuilder()
61 ->select( 'us_key' )
62 ->from( 'uploadstash' )
63 ->where( $dbr->expr( 'us_timestamp', '<', $dbr->timestamp( $cutoff ) ) )
64 ->caller( __METHOD__ )
65 ->fetchResultSet();
66
67 // Delete all registered stash files...
68 if ( $res->numRows() == 0 ) {
69 $this->output( "No stashed files to cleanup according to the DB.\n" );
70 } else {
71 // finish the read before starting writes.
72 $keys = [];
73 foreach ( $res as $row ) {
74 $keys[] = $row->us_key;
75 }
76
77 $this->output( 'Removing ' . count( $keys ) . " file(s)...\n" );
78 // this could be done some other, more direct/efficient way, but using
79 // UploadStash's own methods means it's less likely to fall accidentally
80 // out-of-date someday
81 $stash = new UploadStash( $repo );
82
83 $i = 0;
84 foreach ( $keys as $key ) {
85 $i++;
86 try {
87 $stash->getFile( $key, true );
88 $stash->removeFileNoAuth( $key );
89 } catch ( UploadStashException $ex ) {
90 $type = get_class( $ex );
91 $this->output( "Failed removing stashed upload with key: $key ($type)\n" );
92 }
93 if ( $i % 100 == 0 ) {
94 $this->waitForReplication();
95 $this->output( "$i\n" );
96 }
97 }
98 $this->output( "$i done\n" );
99 }
100
101 // Delete all the corresponding thumbnails...
102 $dir = $tempRepo->getZonePath( 'thumb' );
103 $iterator = $tempRepo->getBackend()->getFileList( [ 'dir' => $dir, 'adviseStat' => 1 ] );
104 if ( $iterator === null ) {
105 $this->fatalError( "Could not get file listing." );
106 }
107 $this->output( "Deleting old thumbnails...\n" );
108 $i = 0;
109 $batch = [];
110 foreach ( $iterator as $file ) {
111 if ( wfTimestamp( TS_UNIX, $tempRepo->getFileTimestamp( "$dir/$file" ) ) < $cutoff ) {
112 $batch[] = [ 'op' => 'delete', 'src' => "$dir/$file" ];
113 if ( count( $batch ) >= $this->getBatchSize() ) {
114 $this->doOperations( $tempRepo, $batch );
115 $i += count( $batch );
116 $batch = [];
117 $this->output( "$i\n" );
118 }
119 }
120 }
121 if ( count( $batch ) ) {
122 $this->doOperations( $tempRepo, $batch );
123 $i += count( $batch );
124 }
125 $this->output( "$i done\n" );
126
127 // Apparently lots of stash files are not registered in the DB...
128 $dir = $tempRepo->getZonePath( 'public' );
129 $iterator = $tempRepo->getBackend()->getFileList( [ 'dir' => $dir, 'adviseStat' => 1 ] );
130 if ( $iterator === null ) {
131 $this->fatalError( "Could not get file listing." );
132 }
133 $this->output( "Deleting orphaned temp files...\n" );
134 if ( strpos( $dir, '/local-temp' ) === false ) {
135 $this->output( "Temp repo might be misconfigured. It points to directory: '$dir' \n" );
136 }
137
138 $i = 0;
139 $batch = [];
140 foreach ( $iterator as $file ) {
141 if ( wfTimestamp( TS_UNIX, $tempRepo->getFileTimestamp( "$dir/$file" ) ) < $cutoff ) {
142 $batch[] = [ 'op' => 'delete', 'src' => "$dir/$file" ];
143 if ( count( $batch ) >= $this->getBatchSize() ) {
144 $this->doOperations( $tempRepo, $batch );
145 $i += count( $batch );
146 $batch = [];
147 $this->output( "$i\n" );
148 }
149 }
150 }
151 if ( count( $batch ) ) {
152 $this->doOperations( $tempRepo, $batch );
153 $i += count( $batch );
154 }
155 $this->output( "$i done\n" );
156 }
157
158 protected function doOperations( FileRepo $tempRepo, array $ops ) {
159 $status = $tempRepo->getBackend()->doQuickOperations( $ops );
160 if ( !$status->isOK() ) {
161 $this->error( $status );
162 }
163 }
164}
165
166// @codeCoverageIgnoreStart
167$maintClass = CleanupUploadStash::class;
168require_once RUN_MAINTENANCE_IF_MAIN;
169// @codeCoverageIgnoreEnd
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Maintenance script to remove old or broken uploads from temporary uploaded file storage and clean up ...
execute()
Do the actual work.
doOperations(FileRepo $tempRepo, array $ops)
__construct()
Default constructor.
Base class for file repositories.
Definition FileRepo.php:68
getBackend()
Get the file backend instance.
Definition FileRepo.php:270
A class containing constants representing the names of configuration variables.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
getBatchSize()
Returns batch size.
output( $out, $channel=null)
Throw some output to the user.
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
waitForReplication()
Wait for replica DB servers to catch up.
error( $err, $die=0)
Throw an error to the user.
getServiceContainer()
Returns the main service container.
addDescription( $text)
Set the description text.
UploadStash is intended to accomplish a few things: