MediaWiki  master
refreshImageMetadata.php
Go to the documentation of this file.
1 <?php
30 require_once __DIR__ . '/Maintenance.php';
31 
35 
42 
46  protected $dbw;
47 
48  public function __construct() {
49  parent::__construct();
50 
51  $this->addDescription( 'Script to update image metadata records' );
52  $this->setBatchSize( 200 );
53 
54  $this->addOption(
55  'force',
56  'Reload metadata from file even if the metadata looks ok',
57  false,
58  false,
59  'f'
60  );
61  $this->addOption(
62  'broken-only',
63  'Only fix really broken records, leave old but still compatible records alone.'
64  );
65  $this->addOption(
66  'convert-to-json',
67  'Fix records with an out of date serialization format.'
68  );
69  $this->addOption(
70  'split',
71  'Enable splitting out large metadata items to the text table. Implies --convert-to-json.'
72  );
73  $this->addOption(
74  'verbose',
75  'Output extra information about each upgraded/non-upgraded file.',
76  false,
77  false,
78  'v'
79  );
80  $this->addOption( 'start', 'Name of file to start with', false, true );
81  $this->addOption( 'end', 'Name of file to end with', false, true );
82 
83  $this->addOption(
84  'mediatype',
85  'Only refresh files with this media type, e.g. BITMAP, UNKNOWN etc.',
86  false,
87  true
88  );
89  $this->addOption(
90  'mime',
91  "Only refresh files with this MIME type. Can accept wild-card 'image/*'. "
92  . "Potentially inefficient unless 'mediatype' is also specified",
93  false,
94  true
95  );
96  $this->addOption(
97  'metadata-contains',
98  '(Inefficient!) Only refresh files where the img_metadata field '
99  . 'contains this string. Can be used if its known a specific '
100  . 'property was being extracted incorrectly.',
101  false,
102  true
103  );
104  $this->addOption(
105  'sleep',
106  'Time to sleep between each batch (in seconds). Default: 0',
107  false,
108  true
109  );
110  $this->addOption( 'oldimage', 'Run and refresh on oldimage table.' );
111  }
112 
113  public function execute() {
114  $force = $this->hasOption( 'force' );
115  $brokenOnly = $this->hasOption( 'broken-only' );
116  $verbose = $this->hasOption( 'verbose' );
117  $start = $this->getOption( 'start', false );
118  $split = $this->hasOption( 'split' );
119  $sleep = (int)$this->getOption( 'sleep', 0 );
120  $reserialize = $this->hasOption( 'convert-to-json' );
121  $oldimage = $this->hasOption( 'oldimage' );
122  if ( $oldimage ) {
123  $fieldPrefix = 'oi_';
124  $fileQuery = OldLocalFile::getQueryInfo();
125  } else {
126  $fieldPrefix = 'img_';
127  $fileQuery = LocalFile::getQueryInfo();
128  }
129 
130  $upgraded = 0;
131  $leftAlone = 0;
132  $error = 0;
133 
134  $dbw = $this->getDB( DB_PRIMARY );
135  $batchSize = $this->getBatchSize();
136  if ( $batchSize <= 0 ) {
137  $this->fatalError( "Batch size is too low...", 12 );
138  }
139 
140  $repo = $this->newLocalRepo( $force, $brokenOnly, $reserialize, $split );
141  $conds = $this->getConditions( $dbw, $fieldPrefix );
142 
143  // For the WHERE img_name > 'foo' condition that comes after doing a batch
144  $conds2 = [];
145  if ( $start !== false ) {
146  $conds2[] = $fieldPrefix . 'name >= ' . $dbw->addQuotes( $start );
147  }
148 
149  $options = [
150  'LIMIT' => $batchSize,
151  'ORDER BY' => $fieldPrefix . 'name ASC',
152  ];
153 
154  $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
155 
156  do {
157  $res = $dbw->select(
158  $fileQuery['tables'],
159  $fileQuery['fields'],
160  array_merge( $conds, $conds2 ),
161  __METHOD__,
162  $options,
163  $fileQuery['joins']
164  );
165 
166  $nameField = $fieldPrefix . 'name';
167  if ( $res->numRows() > 0 ) {
168  $row1 = $res->current();
169  $this->output( "Processing next {$res->numRows()} row(s) starting with {$row1->$nameField}.\n" );
170  $res->rewind();
171  }
172 
173  foreach ( $res as $row ) {
174  try {
175  // LocalFile will upgrade immediately here if obsolete
176  $file = $repo->newFileFromRow( $row );
177  $file->maybeUpgradeRow();
178  if ( $file->getUpgraded() ) {
179  // File was upgraded.
180  $upgraded++;
181  $this->output( "Refreshed File:{$row->$nameField}.\n" );
182  } else {
183  $leftAlone++;
184  if ( $force ) {
185  $file->upgradeRow();
186  if ( $verbose ) {
187  $this->output( "Forcibly refreshed File:{$row->$nameField}.\n" );
188  }
189  } else {
190  if ( $verbose ) {
191  $this->output( "Skipping File:{$row->$nameField}.\n" );
192  }
193  }
194  }
195  } catch ( Exception $e ) {
196  $this->output( "{$row->$nameField} failed. {$e->getMessage()}\n" );
197  }
198  }
199  if ( $res->numRows() > 0 ) {
200  // @phan-suppress-next-line PhanPossiblyUndeclaredVariable rows contains at least one item
201  $conds2 = [ $fieldPrefix . 'name > ' . $dbw->addQuotes( $row->$nameField ) ];
202  }
203  $lbFactory->waitForReplication();
204  if ( $sleep ) {
205  sleep( $sleep );
206  }
207  } while ( $res->numRows() === $batchSize );
208 
209  $total = $upgraded + $leftAlone;
210  if ( $force ) {
211  $this->output( "\nFinished refreshing file metadata for $total files. "
212  . "$upgraded needed to be refreshed, $leftAlone did not need to "
213  . "be but were refreshed anyways, and $error refreshes were suspicious.\n" );
214  } else {
215  $this->output( "\nFinished refreshing file metadata for $total files. "
216  . "$upgraded were refreshed, $leftAlone were already up to date, "
217  . "and $error refreshes were suspicious.\n" );
218  }
219  }
220 
226  private function getConditions( $dbw, $fieldPrefix ) {
227  $conds = [];
228 
229  $end = $this->getOption( 'end', false );
230  $mime = $this->getOption( 'mime', false );
231  $mediatype = $this->getOption( 'mediatype', false );
232  $like = $this->getOption( 'metadata-contains', false );
233 
234  if ( $end !== false ) {
235  $conds[] = $fieldPrefix . 'name <= ' . $dbw->addQuotes( $end );
236  }
237  if ( $mime !== false ) {
238  list( $major, $minor ) = File::splitMime( $mime );
239  $conds[$fieldPrefix . 'major_mime'] = $major;
240  if ( $minor !== '*' ) {
241  $conds[$fieldPrefix . 'minor_mime'] = $minor;
242  }
243  }
244  if ( $mediatype !== false ) {
245  $conds[$fieldPrefix . 'media_type'] = $mediatype;
246  }
247  if ( $like ) {
248  $conds[] = $fieldPrefix . 'metadata ' . $dbw->buildLike( $dbw->anyString(), $like, $dbw->anyString() );
249  }
250 
251  return $conds;
252  }
253 
262  private function newLocalRepo( $force, $brokenOnly, $reserialize, $split ): LocalRepo {
263  if ( $brokenOnly && $force ) {
264  $this->fatalError( 'Cannot use --broken-only and --force together. ', 2 );
265  }
266  $reserialize = $reserialize || $split;
267  if ( $brokenOnly && $reserialize ) {
268  $this->fatalError( 'Cannot use --broken-only with --convert-to-json or --split. ',
269  2 );
270  }
271 
272  $overrides = [
273  'updateCompatibleMetadata' => !$brokenOnly,
274  ];
275  if ( $reserialize ) {
276  $overrides['reserializeMetadata'] = true;
277  $overrides['useJsonMetadata'] = true;
278  }
279  if ( $split ) {
280  $overrides['useSplitMetadata'] = true;
281  }
282 
283  return MediaWikiServices::getInstance()->getRepoGroup()
284  ->newCustomLocalRepo( $overrides );
285  }
286 }
287 
288 $maintClass = RefreshImageMetadata::class;
289 require_once RUN_MAINTENANCE_IF_MAIN;
if(!defined('MW_SETUP_CALLBACK'))
The persistent session ID (if any) loaded at startup.
Definition: WebStart.php:82
static splitMime( $mime)
Split an internet media type into its two components; if not a two-part name, set the minor type to '...
Definition: File.php:307
static getQueryInfo(array $options=[])
Return the tables, fields, and join conditions to be selected to create a new localfile object.
Definition: LocalFile.php:270
Local repository that stores files in the local filesystem and registers them in the wiki's own datab...
Definition: LocalRepo.php:39
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:66
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
output( $out, $channel=null)
Throw some output to the user.
hasOption( $name)
Checks to see if a particular option was set.
getBatchSize()
Returns batch size.
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
setBatchSize( $s=0)
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
Service locator for MediaWiki core services.
static getQueryInfo(array $options=[])
Return the tables, fields, and join conditions to be selected to create a new oldlocalfile object.
Maintenance script to refresh image metadata fields.
IMaintainableDatabase $dbw
__construct()
Default constructor.
execute()
Do the actual work.
newLocalRepo( $force, $brokenOnly, $reserialize, $split)
getConditions( $dbw, $fieldPrefix)
addQuotes( $s)
Escape and quote a raw value string for use in a SQL query.
Basic database interface for live and lazy-loaded relation database handles.
Definition: IDatabase.php:39
select( $table, $vars, $conds='', $fname=__METHOD__, $options=[], $join_conds=[])
Execute a SELECT query constructed using the various parameters provided.
Advanced database interface for IDatabase handles that include maintenance methods.
buildLike( $param,... $params)
LIKE statement wrapper.
anyString()
Returns a token for buildLike() that denotes a '' to be used in a LIKE query.
const DB_PRIMARY
Definition: defines.php:28
$mime
Definition: router.php:60
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition: router.php:42