Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
0.00% |
0 / 158 |
|
0.00% |
0 / 4 |
CRAP | |
0.00% |
0 / 1 |
| RefreshImageMetadata | |
0.00% |
0 / 158 |
|
0.00% |
0 / 4 |
870 | |
0.00% |
0 / 1 |
| __construct | |
0.00% |
0 / 59 |
|
0.00% |
0 / 1 |
2 | |||
| execute | |
0.00% |
0 / 65 |
|
0.00% |
0 / 1 |
210 | |||
| setConditions | |
0.00% |
0 / 18 |
|
0.00% |
0 / 1 |
42 | |||
| newLocalRepo | |
0.00% |
0 / 16 |
|
0.00% |
0 / 1 |
72 | |||
| 1 | <?php |
| 2 | /** |
| 3 | * Refresh image metadata fields. See also rebuildImages.php |
| 4 | * |
| 5 | * Usage: php refreshImageMetadata.php |
| 6 | * |
| 7 | * Copyright © 2011 Brian Wolff |
| 8 | * https://www.mediawiki.org/ |
| 9 | * |
| 10 | * This program is free software; you can redistribute it and/or modify |
| 11 | * it under the terms of the GNU General Public License as published by |
| 12 | * the Free Software Foundation; either version 2 of the License, or |
| 13 | * (at your option) any later version. |
| 14 | * |
| 15 | * This program is distributed in the hope that it will be useful, |
| 16 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 17 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 18 | * GNU General Public License for more details. |
| 19 | * |
| 20 | * You should have received a copy of the GNU General Public License along |
| 21 | * with this program; if not, write to the Free Software Foundation, Inc., |
| 22 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
| 23 | * http://www.gnu.org/copyleft/gpl.html |
| 24 | * |
| 25 | * @file |
| 26 | * @author Brian Wolff |
| 27 | * @ingroup Maintenance |
| 28 | */ |
| 29 | |
| 30 | // @codeCoverageIgnoreStart |
| 31 | require_once __DIR__ . '/Maintenance.php'; |
| 32 | // @codeCoverageIgnoreEnd |
| 33 | |
| 34 | use MediaWiki\FileRepo\File\FileSelectQueryBuilder; |
| 35 | use MediaWiki\Maintenance\Maintenance; |
| 36 | use Wikimedia\Rdbms\IExpression; |
| 37 | use Wikimedia\Rdbms\IMaintainableDatabase; |
| 38 | use Wikimedia\Rdbms\IReadableDatabase; |
| 39 | use Wikimedia\Rdbms\LikeValue; |
| 40 | use Wikimedia\Rdbms\SelectQueryBuilder; |
| 41 | |
| 42 | /** |
| 43 | * Maintenance script to refresh image metadata fields. |
| 44 | * |
| 45 | * @ingroup Maintenance |
| 46 | */ |
| 47 | class RefreshImageMetadata extends Maintenance { |
| 48 | |
| 49 | /** |
| 50 | * @var IMaintainableDatabase |
| 51 | */ |
| 52 | protected $dbw; |
| 53 | |
| 54 | public function __construct() { |
| 55 | parent::__construct(); |
| 56 | |
| 57 | $this->addDescription( 'Script to update image metadata records' ); |
| 58 | $this->setBatchSize( 200 ); |
| 59 | |
| 60 | $this->addOption( |
| 61 | 'force', |
| 62 | 'Reload metadata from file even if the metadata looks ok', |
| 63 | false, |
| 64 | false, |
| 65 | 'f' |
| 66 | ); |
| 67 | $this->addOption( |
| 68 | 'broken-only', |
| 69 | 'Only fix really broken records, leave old but still compatible records alone.' |
| 70 | ); |
| 71 | $this->addOption( |
| 72 | 'convert-to-json', |
| 73 | 'Fix records with an out of date serialization format.' |
| 74 | ); |
| 75 | $this->addOption( |
| 76 | 'split', |
| 77 | 'Enable splitting out large metadata items to the text table. Implies --convert-to-json.' |
| 78 | ); |
| 79 | $this->addOption( |
| 80 | 'verbose', |
| 81 | 'Output extra information about each upgraded/non-upgraded file.', |
| 82 | false, |
| 83 | false, |
| 84 | 'v' |
| 85 | ); |
| 86 | $this->addOption( 'start', 'Name of file to start with', false, true ); |
| 87 | $this->addOption( 'end', 'Name of file to end with', false, true ); |
| 88 | |
| 89 | $this->addOption( |
| 90 | 'mediatype', |
| 91 | 'Only refresh files with this media type, e.g. BITMAP, UNKNOWN etc.', |
| 92 | false, |
| 93 | true |
| 94 | ); |
| 95 | $this->addOption( |
| 96 | 'mime', |
| 97 | "Only refresh files with this MIME type. Can accept wild-card 'image/*'. " |
| 98 | . "Potentially inefficient unless 'mediatype' is also specified", |
| 99 | false, |
| 100 | true |
| 101 | ); |
| 102 | $this->addOption( |
| 103 | 'metadata-contains', |
| 104 | '(Inefficient!) Only refresh files where the img_metadata field ' |
| 105 | . 'contains this string. Can be used if its known a specific ' |
| 106 | . 'property was being extracted incorrectly.', |
| 107 | false, |
| 108 | true |
| 109 | ); |
| 110 | $this->addOption( |
| 111 | 'sleep', |
| 112 | 'Time to sleep between each batch (in seconds). Default: 0', |
| 113 | false, |
| 114 | true |
| 115 | ); |
| 116 | $this->addOption( 'oldimage', 'Run and refresh on oldimage table.' ); |
| 117 | } |
| 118 | |
| 119 | public function execute() { |
| 120 | $force = $this->hasOption( 'force' ); |
| 121 | $brokenOnly = $this->hasOption( 'broken-only' ); |
| 122 | $verbose = $this->hasOption( 'verbose' ); |
| 123 | $start = $this->getOption( 'start', false ); |
| 124 | $split = $this->hasOption( 'split' ); |
| 125 | $sleep = (int)$this->getOption( 'sleep', 0 ); |
| 126 | $reserialize = $this->hasOption( 'convert-to-json' ); |
| 127 | $oldimage = $this->hasOption( 'oldimage' ); |
| 128 | |
| 129 | $dbw = $this->getPrimaryDB(); |
| 130 | if ( $oldimage ) { |
| 131 | $fieldPrefix = 'oi_'; |
| 132 | $queryBuilderTemplate = FileSelectQueryBuilder::newForOldFile( $dbw ); |
| 133 | } else { |
| 134 | $fieldPrefix = 'img_'; |
| 135 | $queryBuilderTemplate = FileSelectQueryBuilder::newForFile( $dbw ); |
| 136 | } |
| 137 | |
| 138 | $upgraded = 0; |
| 139 | $leftAlone = 0; |
| 140 | $error = 0; |
| 141 | $batchSize = intval( $this->getBatchSize() ); |
| 142 | if ( $batchSize <= 0 ) { |
| 143 | $this->fatalError( "Batch size is too low...", 12 ); |
| 144 | } |
| 145 | $repo = $this->newLocalRepo( $force, $brokenOnly, $reserialize, $split ); |
| 146 | $this->setConditions( $dbw, $queryBuilderTemplate, $fieldPrefix ); |
| 147 | $queryBuilderTemplate |
| 148 | ->orderBy( $fieldPrefix . 'name', SelectQueryBuilder::SORT_ASC ) |
| 149 | ->limit( $batchSize ); |
| 150 | |
| 151 | $batchCondition = []; |
| 152 | // For the WHERE img_name > 'foo' condition that comes after doing a batch |
| 153 | if ( $start !== false ) { |
| 154 | $batchCondition[] = $dbw->expr( $fieldPrefix . 'name', '>=', $start ); |
| 155 | } |
| 156 | do { |
| 157 | $queryBuilder = clone $queryBuilderTemplate; |
| 158 | $res = $queryBuilder->andWhere( $batchCondition ) |
| 159 | ->caller( __METHOD__ )->fetchResultSet(); |
| 160 | $nameField = $fieldPrefix . 'name'; |
| 161 | if ( $res->numRows() > 0 ) { |
| 162 | $row1 = $res->current(); |
| 163 | $this->output( "Processing next {$res->numRows()} row(s) starting with {$row1->$nameField}.\n" ); |
| 164 | $res->rewind(); |
| 165 | } |
| 166 | |
| 167 | foreach ( $res as $row ) { |
| 168 | try { |
| 169 | // LocalFile will upgrade immediately here if obsolete |
| 170 | $file = $repo->newFileFromRow( $row ); |
| 171 | $file->maybeUpgradeRow(); |
| 172 | if ( $file->getUpgraded() ) { |
| 173 | // File was upgraded. |
| 174 | $upgraded++; |
| 175 | $this->output( "Refreshed File:{$row->$nameField}.\n" ); |
| 176 | } else { |
| 177 | $leftAlone++; |
| 178 | if ( $force ) { |
| 179 | $file->upgradeRow(); |
| 180 | if ( $verbose ) { |
| 181 | $this->output( "Forcibly refreshed File:{$row->$nameField}.\n" ); |
| 182 | } |
| 183 | } else { |
| 184 | if ( $verbose ) { |
| 185 | $this->output( "Skipping File:{$row->$nameField}.\n" ); |
| 186 | } |
| 187 | } |
| 188 | } |
| 189 | } catch ( Exception $e ) { |
| 190 | $this->output( "{$row->$nameField} failed. {$e->getMessage()}\n" ); |
| 191 | } |
| 192 | } |
| 193 | if ( $res->numRows() > 0 ) { |
| 194 | // @phan-suppress-next-line PhanPossiblyUndeclaredVariable rows contains at least one item |
| 195 | $batchCondition = [ $dbw->expr( $fieldPrefix . 'name', '>', $row->$nameField ) ]; |
| 196 | } |
| 197 | $this->waitForReplication(); |
| 198 | if ( $sleep ) { |
| 199 | sleep( $sleep ); |
| 200 | } |
| 201 | } while ( $res->numRows() === $batchSize ); |
| 202 | |
| 203 | $total = $upgraded + $leftAlone; |
| 204 | if ( $force ) { |
| 205 | $this->output( "\nFinished refreshing file metadata for $total files. " |
| 206 | . "$upgraded needed to be refreshed, $leftAlone did not need to " |
| 207 | . "be but were refreshed anyways, and $error refreshes were suspicious.\n" ); |
| 208 | } else { |
| 209 | $this->output( "\nFinished refreshing file metadata for $total files. " |
| 210 | . "$upgraded were refreshed, $leftAlone were already up to date, " |
| 211 | . "and $error refreshes were suspicious.\n" ); |
| 212 | } |
| 213 | } |
| 214 | |
| 215 | /** |
| 216 | * @param IReadableDatabase $dbw |
| 217 | * @param SelectQueryBuilder $queryBuilder |
| 218 | * @param string $fieldPrefix like img_ or oi_ |
| 219 | * @return void |
| 220 | */ |
| 221 | private function setConditions( IReadableDatabase $dbw, SelectQueryBuilder $queryBuilder, $fieldPrefix ) { |
| 222 | $end = $this->getOption( 'end', false ); |
| 223 | $mime = $this->getOption( 'mime', false ); |
| 224 | $mediatype = $this->getOption( 'mediatype', false ); |
| 225 | $like = $this->getOption( 'metadata-contains', false ); |
| 226 | |
| 227 | if ( $end !== false ) { |
| 228 | $queryBuilder->andWhere( $dbw->expr( $fieldPrefix . 'name', '<=', $end ) ); |
| 229 | } |
| 230 | if ( $mime !== false ) { |
| 231 | [ $major, $minor ] = File::splitMime( $mime ); |
| 232 | $queryBuilder->andWhere( [ $fieldPrefix . 'major_mime' => $major ] ); |
| 233 | if ( $minor !== '*' ) { |
| 234 | $queryBuilder->andWhere( [ $fieldPrefix . 'minor_mime' => $minor ] ); |
| 235 | } |
| 236 | } |
| 237 | if ( $mediatype !== false ) { |
| 238 | $queryBuilder->andWhere( [ $fieldPrefix . 'media_type' => $mediatype ] ); |
| 239 | } |
| 240 | if ( $like ) { |
| 241 | $queryBuilder->andWhere( |
| 242 | $dbw->expr( $fieldPrefix . 'metadata', IExpression::LIKE, |
| 243 | new LikeValue( $dbw->anyString(), $like, $dbw->anyString() ) ) |
| 244 | ); |
| 245 | } |
| 246 | } |
| 247 | |
| 248 | /** |
| 249 | * @param bool $force |
| 250 | * @param bool $brokenOnly |
| 251 | * @param bool $reserialize |
| 252 | * @param bool $split |
| 253 | * |
| 254 | * @return LocalRepo |
| 255 | */ |
| 256 | private function newLocalRepo( $force, $brokenOnly, $reserialize, $split ): LocalRepo { |
| 257 | if ( $brokenOnly && $force ) { |
| 258 | $this->fatalError( 'Cannot use --broken-only and --force together. ', 2 ); |
| 259 | } |
| 260 | $reserialize = $reserialize || $split; |
| 261 | if ( $brokenOnly && $reserialize ) { |
| 262 | $this->fatalError( 'Cannot use --broken-only with --convert-to-json or --split. ', |
| 263 | 2 ); |
| 264 | } |
| 265 | |
| 266 | $overrides = [ |
| 267 | 'updateCompatibleMetadata' => !$brokenOnly, |
| 268 | ]; |
| 269 | if ( $reserialize ) { |
| 270 | $overrides['reserializeMetadata'] = true; |
| 271 | $overrides['useJsonMetadata'] = true; |
| 272 | } |
| 273 | if ( $split ) { |
| 274 | $overrides['useSplitMetadata'] = true; |
| 275 | } |
| 276 | |
| 277 | return $this->getServiceContainer()->getRepoGroup() |
| 278 | ->newCustomLocalRepo( $overrides ); |
| 279 | } |
| 280 | } |
| 281 | |
| 282 | // @codeCoverageIgnoreStart |
| 283 | $maintClass = RefreshImageMetadata::class; |
| 284 | require_once RUN_MAINTENANCE_IF_MAIN; |
| 285 | // @codeCoverageIgnoreEnd |