Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
0.00% |
0 / 79 |
|
0.00% |
0 / 3 |
CRAP | |
0.00% |
0 / 1 |
| CleanupUploadStash | |
0.00% |
0 / 76 |
|
0.00% |
0 / 3 |
420 | |
0.00% |
0 / 1 |
| __construct | |
0.00% |
0 / 3 |
|
0.00% |
0 / 1 |
2 | |||
| execute | |
0.00% |
0 / 70 |
|
0.00% |
0 / 1 |
306 | |||
| doOperations | |
0.00% |
0 / 3 |
|
0.00% |
0 / 1 |
6 | |||
| 1 | <?php |
| 2 | /** |
| 3 | * Remove old or broken uploads from temporary uploaded file storage, |
| 4 | * clean up associated database records |
| 5 | * |
| 6 | * Copyright © 2011, Wikimedia Foundation |
| 7 | * |
| 8 | * This program is free software; you can redistribute it and/or modify |
| 9 | * it under the terms of the GNU General Public License as published by |
| 10 | * the Free Software Foundation; either version 2 of the License, or |
| 11 | * (at your option) any later version. |
| 12 | * |
| 13 | * This program is distributed in the hope that it will be useful, |
| 14 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 16 | * GNU General Public License for more details. |
| 17 | * |
| 18 | * You should have received a copy of the GNU General Public License along |
| 19 | * with this program; if not, write to the Free Software Foundation, Inc., |
| 20 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
| 21 | * http://www.gnu.org/copyleft/gpl.html |
| 22 | * |
| 23 | * @file |
| 24 | * @author Ian Baker <ibaker@wikimedia.org> |
| 25 | * @ingroup Maintenance |
| 26 | */ |
| 27 | |
| 28 | use MediaWiki\MainConfigNames; |
| 29 | use MediaWiki\Status\Status; |
| 30 | |
| 31 | require_once __DIR__ . '/Maintenance.php'; |
| 32 | |
| 33 | /** |
| 34 | * Maintenance script to remove old or broken uploads from temporary uploaded |
| 35 | * file storage and clean up associated database records. |
| 36 | * |
| 37 | * @ingroup Maintenance |
| 38 | */ |
| 39 | class CleanupUploadStash extends Maintenance { |
| 40 | |
| 41 | public function __construct() { |
| 42 | parent::__construct(); |
| 43 | $this->addDescription( 'Clean up abandoned files in temporary uploaded file stash' ); |
| 44 | $this->setBatchSize( 50 ); |
| 45 | } |
| 46 | |
| 47 | public function execute() { |
| 48 | $repo = $this->getServiceContainer()->getRepoGroup()->getLocalRepo(); |
| 49 | $tempRepo = $repo->getTempRepo(); |
| 50 | |
| 51 | $dbr = $repo->getReplicaDB(); |
| 52 | |
| 53 | // how far back should this look for files to delete? |
| 54 | $cutoff = time() - (int)$this->getConfig()->get( MainConfigNames::UploadStashMaxAge ); |
| 55 | |
| 56 | $this->output( "Getting list of files to clean up...\n" ); |
| 57 | $res = $dbr->newSelectQueryBuilder() |
| 58 | ->select( 'us_key' ) |
| 59 | ->from( 'uploadstash' ) |
| 60 | ->where( $dbr->expr( 'us_timestamp', '<', $dbr->timestamp( $cutoff ) ) ) |
| 61 | ->caller( __METHOD__ ) |
| 62 | ->fetchResultSet(); |
| 63 | |
| 64 | // Delete all registered stash files... |
| 65 | if ( $res->numRows() == 0 ) { |
| 66 | $this->output( "No stashed files to cleanup according to the DB.\n" ); |
| 67 | } else { |
| 68 | // finish the read before starting writes. |
| 69 | $keys = []; |
| 70 | foreach ( $res as $row ) { |
| 71 | $keys[] = $row->us_key; |
| 72 | } |
| 73 | |
| 74 | $this->output( 'Removing ' . count( $keys ) . " file(s)...\n" ); |
| 75 | // this could be done some other, more direct/efficient way, but using |
| 76 | // UploadStash's own methods means it's less likely to fall accidentally |
| 77 | // out-of-date someday |
| 78 | $stash = new UploadStash( $repo ); |
| 79 | |
| 80 | $i = 0; |
| 81 | foreach ( $keys as $key ) { |
| 82 | $i++; |
| 83 | try { |
| 84 | $stash->getFile( $key, true ); |
| 85 | $stash->removeFileNoAuth( $key ); |
| 86 | } catch ( UploadStashException $ex ) { |
| 87 | $type = get_class( $ex ); |
| 88 | $this->output( "Failed removing stashed upload with key: $key ($type)\n" ); |
| 89 | } |
| 90 | if ( $i % 100 == 0 ) { |
| 91 | $this->waitForReplication(); |
| 92 | $this->output( "$i\n" ); |
| 93 | } |
| 94 | } |
| 95 | $this->output( "$i done\n" ); |
| 96 | } |
| 97 | |
| 98 | // Delete all the corresponding thumbnails... |
| 99 | $dir = $tempRepo->getZonePath( 'thumb' ); |
| 100 | $iterator = $tempRepo->getBackend()->getFileList( [ 'dir' => $dir, 'adviseStat' => 1 ] ); |
| 101 | if ( $iterator === null ) { |
| 102 | $this->fatalError( "Could not get file listing." ); |
| 103 | } |
| 104 | $this->output( "Deleting old thumbnails...\n" ); |
| 105 | $i = 0; |
| 106 | $batch = []; |
| 107 | foreach ( $iterator as $file ) { |
| 108 | if ( wfTimestamp( TS_UNIX, $tempRepo->getFileTimestamp( "$dir/$file" ) ) < $cutoff ) { |
| 109 | $batch[] = [ 'op' => 'delete', 'src' => "$dir/$file" ]; |
| 110 | if ( count( $batch ) >= $this->getBatchSize() ) { |
| 111 | $this->doOperations( $tempRepo, $batch ); |
| 112 | $i += count( $batch ); |
| 113 | $batch = []; |
| 114 | $this->output( "$i\n" ); |
| 115 | } |
| 116 | } |
| 117 | } |
| 118 | if ( count( $batch ) ) { |
| 119 | $this->doOperations( $tempRepo, $batch ); |
| 120 | $i += count( $batch ); |
| 121 | } |
| 122 | $this->output( "$i done\n" ); |
| 123 | |
| 124 | // Apparently lots of stash files are not registered in the DB... |
| 125 | $dir = $tempRepo->getZonePath( 'public' ); |
| 126 | $iterator = $tempRepo->getBackend()->getFileList( [ 'dir' => $dir, 'adviseStat' => 1 ] ); |
| 127 | if ( $iterator === null ) { |
| 128 | $this->fatalError( "Could not get file listing." ); |
| 129 | } |
| 130 | $this->output( "Deleting orphaned temp files...\n" ); |
| 131 | if ( strpos( $dir, '/local-temp' ) === false ) { |
| 132 | $this->output( "Temp repo might be misconfigured. It points to directory: '$dir' \n" ); |
| 133 | } |
| 134 | |
| 135 | $i = 0; |
| 136 | $batch = []; |
| 137 | foreach ( $iterator as $file ) { |
| 138 | if ( wfTimestamp( TS_UNIX, $tempRepo->getFileTimestamp( "$dir/$file" ) ) < $cutoff ) { |
| 139 | $batch[] = [ 'op' => 'delete', 'src' => "$dir/$file" ]; |
| 140 | if ( count( $batch ) >= $this->getBatchSize() ) { |
| 141 | $this->doOperations( $tempRepo, $batch ); |
| 142 | $i += count( $batch ); |
| 143 | $batch = []; |
| 144 | $this->output( "$i\n" ); |
| 145 | } |
| 146 | } |
| 147 | } |
| 148 | if ( count( $batch ) ) { |
| 149 | $this->doOperations( $tempRepo, $batch ); |
| 150 | $i += count( $batch ); |
| 151 | } |
| 152 | $this->output( "$i done\n" ); |
| 153 | } |
| 154 | |
| 155 | protected function doOperations( FileRepo $tempRepo, array $ops ) { |
| 156 | $status = $tempRepo->getBackend()->doQuickOperations( $ops ); |
| 157 | if ( !$status->isOK() ) { |
| 158 | $this->error( print_r( Status::wrap( $status )->getErrorsArray(), true ) ); |
| 159 | } |
| 160 | } |
| 161 | } |
| 162 | |
| 163 | $maintClass = CleanupUploadStash::class; |
| 164 | require_once RUN_MAINTENANCE_IF_MAIN; |