MediaWiki  master
refreshImageMetadata.php
Go to the documentation of this file.
1 <?php
30 require_once __DIR__ . '/Maintenance.php';
31 
35 
42 
46  protected $dbw;
47 
48  public function __construct() {
49  parent::__construct();
50 
51  $this->addDescription( 'Script to update image metadata records' );
52  $this->setBatchSize( 200 );
53 
54  $this->addOption(
55  'force',
56  'Reload metadata from file even if the metadata looks ok',
57  false,
58  false,
59  'f'
60  );
61  $this->addOption(
62  'broken-only',
63  'Only fix really broken records, leave old but still compatible records alone.'
64  );
65  $this->addOption(
66  'convert-to-json',
67  'Fix records with an out of date serialization format.'
68  );
69  $this->addOption(
70  'split',
71  'Enable splitting out large metadata items to the text table. Implies --convert-to-json.'
72  );
73  $this->addOption(
74  'verbose',
75  'Output extra information about each upgraded/non-upgraded file.',
76  false,
77  false,
78  'v'
79  );
80  $this->addOption( 'start', 'Name of file to start with', false, true );
81  $this->addOption( 'end', 'Name of file to end with', false, true );
82 
83  $this->addOption(
84  'mediatype',
85  'Only refresh files with this media type, e.g. BITMAP, UNKNOWN etc.',
86  false,
87  true
88  );
89  $this->addOption(
90  'mime',
91  "Only refresh files with this MIME type. Can accept wild-card 'image/*'. "
92  . "Potentially inefficient unless 'mediatype' is also specified",
93  false,
94  true
95  );
96  $this->addOption(
97  'metadata-contains',
98  '(Inefficient!) Only refresh files where the img_metadata field '
99  . 'contains this string. Can be used if its known a specific '
100  . 'property was being extracted incorrectly.',
101  false,
102  true
103  );
104  $this->addOption(
105  'sleep',
106  'Time to sleep between each batch (in seconds). Default: 0',
107  false,
108  true
109  );
110  }
111 
112  public function execute() {
113  $force = $this->hasOption( 'force' );
114  $brokenOnly = $this->hasOption( 'broken-only' );
115  $verbose = $this->hasOption( 'verbose' );
116  $start = $this->getOption( 'start', false );
117  $split = $this->hasOption( 'split' );
118  $sleep = (int)$this->getOption( 'sleep', 0 );
119  $reserialize = $this->hasOption( 'convert-to-json' );
120 
121  $upgraded = 0;
122  $leftAlone = 0;
123  $error = 0;
124 
125  $dbw = $this->getDB( DB_PRIMARY );
126  $batchSize = $this->getBatchSize();
127  if ( $batchSize <= 0 ) {
128  $this->fatalError( "Batch size is too low...", 12 );
129  }
130 
131  $repo = $this->newLocalRepo( $force, $brokenOnly, $reserialize, $split );
132  $conds = $this->getConditions( $dbw );
133 
134  // For the WHERE img_name > 'foo' condition that comes after doing a batch
135  $conds2 = [];
136  if ( $start !== false ) {
137  $conds2[] = 'img_name >= ' . $dbw->addQuotes( $start );
138  }
139 
140  $options = [
141  'LIMIT' => $batchSize,
142  'ORDER BY' => 'img_name ASC',
143  ];
144 
145  $fileQuery = LocalFile::getQueryInfo();
146  $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
147 
148  do {
149  $res = $dbw->select(
150  $fileQuery['tables'],
151  $fileQuery['fields'],
152  array_merge( $conds, $conds2 ),
153  __METHOD__,
154  $options,
155  $fileQuery['joins']
156  );
157 
158  if ( $res->numRows() > 0 ) {
159  $row1 = $res->current();
160  $this->output( "Processing next {$res->numRows()} row(s) starting with {$row1->img_name}.\n" );
161  $res->rewind();
162  }
163 
164  foreach ( $res as $row ) {
165  try {
166  // LocalFile will upgrade immediately here if obsolete
167  $file = $repo->newFileFromRow( $row );
168  $file->maybeUpgradeRow();
169  if ( $file->getUpgraded() ) {
170  // File was upgraded.
171  $upgraded++;
172  $this->output( "Refreshed File:{$row->img_name}.\n" );
173  } else {
174  $leftAlone++;
175  if ( $force ) {
176  $file->upgradeRow();
177  if ( $verbose ) {
178  $this->output( "Forcibly refreshed File:{$row->img_name}.\n" );
179  }
180  } else {
181  if ( $verbose ) {
182  $this->output( "Skipping File:{$row->img_name}.\n" );
183  }
184  }
185  }
186  } catch ( Exception $e ) {
187  $this->output( "{$row->img_name} failed. {$e->getMessage()}\n" );
188  }
189  }
190  $conds2 = [ 'img_name > ' . $dbw->addQuotes( $row->img_name ) ];
191  $lbFactory->waitForReplication();
192  if ( $sleep ) {
193  sleep( $sleep );
194  }
195  } while ( $res->numRows() === $batchSize );
196 
197  $total = $upgraded + $leftAlone;
198  if ( $force ) {
199  $this->output( "\nFinished refreshing file metadata for $total files. "
200  . "$upgraded needed to be refreshed, $leftAlone did not need to "
201  . "be but were refreshed anyways, and $error refreshes were suspicious.\n" );
202  } else {
203  $this->output( "\nFinished refreshing file metadata for $total files. "
204  . "$upgraded were refreshed, $leftAlone were already up to date, "
205  . "and $error refreshes were suspicious.\n" );
206  }
207  }
208 
213  private function getConditions( $dbw ) {
214  $conds = [];
215 
216  $end = $this->getOption( 'end', false );
217  $mime = $this->getOption( 'mime', false );
218  $mediatype = $this->getOption( 'mediatype', false );
219  $like = $this->getOption( 'metadata-contains', false );
220 
221  if ( $end !== false ) {
222  $conds[] = 'img_name <= ' . $dbw->addQuotes( $end );
223  }
224  if ( $mime !== false ) {
225  list( $major, $minor ) = File::splitMime( $mime );
226  $conds['img_major_mime'] = $major;
227  if ( $minor !== '*' ) {
228  $conds['img_minor_mime'] = $minor;
229  }
230  }
231  if ( $mediatype !== false ) {
232  $conds['img_media_type'] = $mediatype;
233  }
234  if ( $like ) {
235  $conds[] = 'img_metadata ' . $dbw->buildLike( $dbw->anyString(), $like, $dbw->anyString() );
236  }
237 
238  return $conds;
239  }
240 
249  private function newLocalRepo( $force, $brokenOnly, $reserialize, $split ): LocalRepo {
250  if ( $brokenOnly && $force ) {
251  $this->fatalError( 'Cannot use --broken-only and --force together. ', 2 );
252  }
253  $reserialize = $reserialize || $split;
254  if ( $brokenOnly && $reserialize ) {
255  $this->fatalError( 'Cannot use --broken-only with --convert-to-json or --split. ',
256  2 );
257  }
258 
259  $overrides = [
260  'updateCompatibleMetadata' => !$brokenOnly,
261  ];
262  if ( $reserialize ) {
263  $overrides['reserializeMetadata'] = true;
264  $overrides['useJsonMetadata'] = true;
265  }
266  if ( $split ) {
267  $overrides['useSplitMetadata'] = true;
268  }
269 
270  return MediaWikiServices::getInstance()->getRepoGroup()
271  ->newCustomLocalRepo( $overrides );
272  }
273 }
274 
275 $maintClass = RefreshImageMetadata::class;
276 require_once RUN_MAINTENANCE_IF_MAIN;
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:193
Maintenance\fatalError
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
Definition: Maintenance.php:489
Maintenance\addDescription
addDescription( $text)
Set the description text.
Definition: Maintenance.php:329
$file
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition: router.php:42
Maintenance
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:59
File\splitMime
static splitMime( $mime)
Split an internet media type into its two components; if not a two-part name, set the minor type to '...
Definition: File.php:305
RefreshImageMetadata\__construct
__construct()
Default constructor.
Definition: refreshImageMetadata.php:48
$res
$res
Definition: testCompression.php:57
RefreshImageMetadata
Maintenance script to refresh image metadata fields.
Definition: refreshImageMetadata.php:41
Wikimedia\Rdbms\IDatabase
Basic database interface for live and lazy-loaded relation database handles.
Definition: IDatabase.php:38
RefreshImageMetadata\newLocalRepo
newLocalRepo( $force, $brokenOnly, $reserialize, $split)
Definition: refreshImageMetadata.php:249
Maintenance\addOption
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
Definition: Maintenance.php:249
RefreshImageMetadata\getConditions
getConditions( $dbw)
Definition: refreshImageMetadata.php:213
RefreshImageMetadata\$dbw
IMaintainableDatabase $dbw
Definition: refreshImageMetadata.php:46
DB_PRIMARY
const DB_PRIMARY
Definition: defines.php:27
Maintenance\getDB
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
Definition: Maintenance.php:1362
Wikimedia\Rdbms\IDatabase\anyString
anyString()
Returns a token for buildLike() that denotes a '' to be used in a LIKE query.
LocalFile\getQueryInfo
static getQueryInfo(array $options=[])
Return the tables, fields, and join conditions to be selected to create a new localfile object.
Definition: LocalFile.php:267
Maintenance\getOption
getOption( $name, $default=null)
Get an option, or return the default.
Definition: Maintenance.php:286
Wikimedia\Rdbms\IDatabase\addQuotes
addQuotes( $s)
Escape and quote a raw value string for use in a SQL query.
Maintenance\getBatchSize
getBatchSize()
Returns batch size.
Definition: Maintenance.php:368
Wikimedia\Rdbms\IDatabase\select
select( $table, $vars, $conds='', $fname=__METHOD__, $options=[], $join_conds=[])
Execute a SELECT query constructed using the various parameters provided.
Maintenance\output
output( $out, $channel=null)
Throw some output to the user.
Definition: Maintenance.php:435
$mime
$mime
Definition: router.php:60
Maintenance\hasOption
hasOption( $name)
Checks to see if a particular option was set.
Definition: Maintenance.php:271
$maintClass
$maintClass
Definition: refreshImageMetadata.php:275
Wikimedia\Rdbms\IDatabase\buildLike
buildLike( $param,... $params)
LIKE statement wrapper.
Wikimedia\Rdbms\IMaintainableDatabase
Advanced database interface for IDatabase handles that include maintenance methods.
Definition: IMaintainableDatabase.php:38
RefreshImageMetadata\execute
execute()
Do the actual work.
Definition: refreshImageMetadata.php:112
LocalRepo
A repository that stores files in the local filesystem and registers them in the wiki's own database.
Definition: LocalRepo.php:41
Maintenance\setBatchSize
setBatchSize( $s=0)
Definition: Maintenance.php:375