MediaWiki  master
refreshImageMetadata.php
Go to the documentation of this file.
1 <?php
30 require_once __DIR__ . '/Maintenance.php';
31 
35 
42 
46  protected $dbw;
47 
48  public function __construct() {
49  parent::__construct();
50 
51  $this->addDescription( 'Script to update image metadata records' );
52  $this->setBatchSize( 200 );
53 
54  $this->addOption(
55  'force',
56  'Reload metadata from file even if the metadata looks ok',
57  false,
58  false,
59  'f'
60  );
61  $this->addOption(
62  'broken-only',
63  'Only fix really broken records, leave old but still compatible records alone.'
64  );
65  $this->addOption(
66  'verbose',
67  'Output extra information about each upgraded/non-upgraded file.',
68  false,
69  false,
70  'v'
71  );
72  $this->addOption( 'start', 'Name of file to start with', false, true );
73  $this->addOption( 'end', 'Name of file to end with', false, true );
74 
75  $this->addOption(
76  'mediatype',
77  'Only refresh files with this media type, e.g. BITMAP, UNKNOWN etc.',
78  false,
79  true
80  );
81  $this->addOption(
82  'mime',
83  "Only refresh files with this MIME type. Can accept wild-card 'image/*'. "
84  . "Potentially inefficient unless 'mediatype' is also specified",
85  false,
86  true
87  );
88  $this->addOption(
89  'metadata-contains',
90  '(Inefficient!) Only refresh files where the img_metadata field '
91  . 'contains this string. Can be used if its known a specific '
92  . 'property was being extracted incorrectly.',
93  false,
94  true
95  );
96  }
97 
98  public function execute() {
99  $force = $this->hasOption( 'force' );
100  $brokenOnly = $this->hasOption( 'broken-only' );
101  $verbose = $this->hasOption( 'verbose' );
102  $start = $this->getOption( 'start', false );
103  $this->setupParameters( $force, $brokenOnly );
104 
105  $upgraded = 0;
106  $leftAlone = 0;
107  $error = 0;
108 
109  $dbw = $this->getDB( DB_MASTER );
110  $batchSize = $this->getBatchSize();
111  if ( $batchSize <= 0 ) {
112  $this->fatalError( "Batch size is too low...", 12 );
113  }
114 
115  $repo = MediaWikiServices::getInstance()->getRepoGroup()->getLocalRepo();
116  $conds = $this->getConditions( $dbw );
117 
118  // For the WHERE img_name > 'foo' condition that comes after doing a batch
119  $conds2 = [];
120  if ( $start !== false ) {
121  $conds2[] = 'img_name >= ' . $dbw->addQuotes( $start );
122  }
123 
124  $options = [
125  'LIMIT' => $batchSize,
126  'ORDER BY' => 'img_name ASC',
127  ];
128 
129  $fileQuery = LocalFile::getQueryInfo();
130  $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
131 
132  do {
133  $res = $dbw->select(
134  $fileQuery['tables'],
135  $fileQuery['fields'],
136  array_merge( $conds, $conds2 ),
137  __METHOD__,
138  $options,
139  $fileQuery['joins']
140  );
141 
142  if ( $res->numRows() > 0 ) {
143  $row1 = $res->current();
144  $this->output( "Processing next {$res->numRows()} row(s) starting with {$row1->img_name}.\n" );
145  $res->rewind();
146  }
147 
148  foreach ( $res as $row ) {
149  try {
150  // LocalFile will upgrade immediately here if obsolete
151  $file = $repo->newFileFromRow( $row );
152  if ( $file->getUpgraded() ) {
153  // File was upgraded.
154  $upgraded++;
155  $newLength = strlen( $file->getMetadata() );
156  $oldLength = strlen( $row->img_metadata );
157  if ( $newLength < $oldLength - 5 ) {
158  // If after updating, the metadata is smaller then
159  // what it was before, that's probably not a good thing
160  // because we extract more data with time, not less.
161  // Thus this probably indicates an error of some sort,
162  // or at the very least is suspicious. Have the - 5 just
163  // to weed out any inconsequential changes.
164  $error++;
165  $this->output(
166  "Warning: File:{$row->img_name} used to have " .
167  "$oldLength bytes of metadata but now has $newLength bytes.\n"
168  );
169  } elseif ( $verbose ) {
170  $this->output( "Refreshed File:{$row->img_name}.\n" );
171  }
172  } else {
173  $leftAlone++;
174  if ( $force ) {
175  $file->upgradeRow();
176  $newLength = strlen( $file->getMetadata() );
177  $oldLength = strlen( $row->img_metadata );
178  if ( $newLength < $oldLength - 5 ) {
179  $error++;
180  $this->output(
181  "Warning: File:{$row->img_name} used to have " .
182  "$oldLength bytes of metadata but now has $newLength bytes. (forced)\n"
183  );
184  }
185  if ( $verbose ) {
186  $this->output( "Forcibly refreshed File:{$row->img_name}.\n" );
187  }
188  } else {
189  if ( $verbose ) {
190  $this->output( "Skipping File:{$row->img_name}.\n" );
191  }
192  }
193  }
194  } catch ( Exception $e ) {
195  $this->output( "{$row->img_name} failed. {$e->getMessage()}\n" );
196  }
197  }
198  $conds2 = [ 'img_name > ' . $dbw->addQuotes( $row->img_name ) ];
199  $lbFactory->waitForReplication();
200  } while ( $res->numRows() === $batchSize );
201 
202  $total = $upgraded + $leftAlone;
203  if ( $force ) {
204  $this->output( "\nFinished refreshing file metadata for $total files. "
205  . "$upgraded needed to be refreshed, $leftAlone did not need to "
206  . "be but were refreshed anyways, and $error refreshes were suspicious.\n" );
207  } else {
208  $this->output( "\nFinished refreshing file metadata for $total files. "
209  . "$upgraded were refreshed, $leftAlone were already up to date, "
210  . "and $error refreshes were suspicious.\n" );
211  }
212  }
213 
218  private function getConditions( $dbw ) {
219  $conds = [];
220 
221  $end = $this->getOption( 'end', false );
222  $mime = $this->getOption( 'mime', false );
223  $mediatype = $this->getOption( 'mediatype', false );
224  $like = $this->getOption( 'metadata-contains', false );
225 
226  if ( $end !== false ) {
227  $conds[] = 'img_name <= ' . $dbw->addQuotes( $end );
228  }
229  if ( $mime !== false ) {
230  list( $major, $minor ) = File::splitMime( $mime );
231  $conds['img_major_mime'] = $major;
232  if ( $minor !== '*' ) {
233  $conds['img_minor_mime'] = $minor;
234  }
235  }
236  if ( $mediatype !== false ) {
237  $conds['img_media_type'] = $mediatype;
238  }
239  if ( $like ) {
240  $conds[] = 'img_metadata ' . $dbw->buildLike( $dbw->anyString(), $like, $dbw->anyString() );
241  }
242 
243  return $conds;
244  }
245 
250  private function setupParameters( $force, $brokenOnly ) {
252 
253  if ( $brokenOnly ) {
255  } else {
257  }
258 
259  if ( $brokenOnly && $force ) {
260  $this->fatalError( 'Cannot use --broken-only and --force together. ', 2 );
261  }
262  }
263 }
264 
265 $maintClass = RefreshImageMetadata::class;
266 require_once RUN_MAINTENANCE_IF_MAIN;
RUN_MAINTENANCE_IF_MAIN
const RUN_MAINTENANCE_IF_MAIN
Definition: Maintenance.php:38
$wgUpdateCompatibleMetadata
$wgUpdateCompatibleMetadata
If to automatically update the img_metadata field if the metadata field is outdated but compatible wi...
Definition: DefaultSettings.php:865
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:146
Maintenance\fatalError
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
Definition: Maintenance.php:480
Maintenance\addDescription
addDescription( $text)
Set the description text.
Definition: Maintenance.php:323
$file
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition: router.php:42
Maintenance
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:53
File\splitMime
static splitMime( $mime)
Split an internet media type into its two components; if not a two-part name, set the minor type to '...
Definition: File.php:286
RefreshImageMetadata\__construct
__construct()
Default constructor.
Definition: refreshImageMetadata.php:48
$res
$res
Definition: testCompression.php:57
RefreshImageMetadata
Maintenance script to refresh image metadata fields.
Definition: refreshImageMetadata.php:41
Wikimedia\Rdbms\IDatabase
Basic database interface for live and lazy-loaded relation database handles.
Definition: IDatabase.php:38
Maintenance\addOption
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
Definition: Maintenance.php:241
DB_MASTER
const DB_MASTER
Definition: defines.php:26
RefreshImageMetadata\getConditions
getConditions( $dbw)
Definition: refreshImageMetadata.php:218
RefreshImageMetadata\$dbw
IMaintainableDatabase $dbw
Definition: refreshImageMetadata.php:46
Maintenance\getDB
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
Definition: Maintenance.php:1350
Wikimedia\Rdbms\IDatabase\anyString
anyString()
Returns a token for buildLike() that denotes a '' to be used in a LIKE query.
LocalFile\getQueryInfo
static getQueryInfo(array $options=[])
Return the tables, fields, and join conditions to be selected to create a new localfile object.
Definition: LocalFile.php:216
Maintenance\getOption
getOption( $name, $default=null)
Get an option, or return the default.
Definition: Maintenance.php:277
Wikimedia\Rdbms\IDatabase\addQuotes
addQuotes( $s)
Escape and quote a raw value string for use in a SQL query.
RefreshImageMetadata\setupParameters
setupParameters( $force, $brokenOnly)
Definition: refreshImageMetadata.php:250
Maintenance\getBatchSize
getBatchSize()
Returns batch size.
Definition: Maintenance.php:362
Wikimedia\Rdbms\IDatabase\select
select( $table, $vars, $conds='', $fname=__METHOD__, $options=[], $join_conds=[])
Execute a SELECT query constructed using the various parameters provided.
Maintenance\output
output( $out, $channel=null)
Throw some output to the user.
Definition: Maintenance.php:429
$mime
$mime
Definition: router.php:60
Maintenance\hasOption
hasOption( $name)
Checks to see if a particular option exists.
Definition: Maintenance.php:262
$maintClass
$maintClass
Definition: refreshImageMetadata.php:265
Wikimedia\Rdbms\IDatabase\buildLike
buildLike( $param,... $params)
LIKE statement wrapper.
Wikimedia\Rdbms\IMaintainableDatabase
Advanced database interface for IDatabase handles that include maintenance methods.
Definition: IMaintainableDatabase.php:38
RefreshImageMetadata\execute
execute()
Do the actual work.
Definition: refreshImageMetadata.php:98
Maintenance\setBatchSize
setBatchSize( $s=0)
Set the batch size.
Definition: Maintenance.php:370