MediaWiki  master
rebuildImages.php
Go to the documentation of this file.
1 <?php
33 require_once __DIR__ . '/Maintenance.php';
34 
38 
44 class ImageBuilder extends Maintenance {
48  protected $dbw;
49 
51  private $dryrun;
52 
54  private $repo;
55 
57  private $updated;
58 
60  private $processed;
61 
63  private $count;
64 
66  private $startTime;
67 
69  private $table;
70 
71  public function __construct() {
72  parent::__construct();
73  $this->addDescription( 'Script to update image metadata records' );
74 
75  $this->addOption( 'missing', 'Check for files without associated database record' );
76  $this->addOption( 'dry-run', 'Only report, don\'t update the database' );
77  }
78 
79  public function execute() {
80  $this->dbw = $this->getDB( DB_PRIMARY );
81  $this->dryrun = $this->hasOption( 'dry-run' );
82  if ( $this->dryrun ) {
83  $this->getServiceContainer()->getReadOnlyMode()
84  ->setReason( 'Dry run mode, image upgrades are suppressed' );
85  }
86 
87  if ( $this->hasOption( 'missing' ) ) {
88  $this->crawlMissing();
89  } else {
90  $this->build();
91  }
92  }
93 
97  private function getRepo() {
98  if ( $this->repo === null ) {
99  $this->repo = $this->getServiceContainer()->getRepoGroup()
100  ->newCustomLocalRepo( [
101  // make sure to update old, but compatible img_metadata fields.
102  'updateCompatibleMetadata' => true
103  ] );
104  }
105 
106  return $this->repo;
107  }
108 
109  private function build() {
110  $this->buildImage();
111  $this->buildOldImage();
112  }
113 
118  private function init( $count, $table ) {
119  $this->processed = 0;
120  $this->updated = 0;
121  $this->count = $count;
122  $this->startTime = microtime( true );
123  $this->table = $table;
124  }
125 
126  private function progress( $updated ) {
127  $this->updated += $updated;
128  $this->processed++;
129  if ( $this->processed % 100 != 0 ) {
130  return;
131  }
132  $portion = $this->processed / $this->count;
133  $updateRate = $this->updated / $this->processed;
134 
135  $now = microtime( true );
136  $delta = $now - $this->startTime;
137  $estimatedTotalTime = $delta / $portion;
138  $eta = $this->startTime + $estimatedTotalTime;
139  $rate = $this->processed / $delta;
140 
141  $this->output( sprintf( "%s: %6.2f%% done on %s; ETA %s [%d/%d] %.2f/sec <%.2f%% updated>\n",
142  wfTimestamp( TS_DB, intval( $now ) ),
143  $portion * 100.0,
144  $this->table,
145  wfTimestamp( TS_DB, intval( $eta ) ),
146  $this->processed,
147  $this->count,
148  $rate,
149  $updateRate * 100.0 ) );
150  flush();
151  }
152 
153  private function buildTable( $table, $queryInfo, $callback ) {
154  $count = $this->dbw->newSelectQueryBuilder()
155  ->select( 'count(*)' )
156  ->from( $table )
157  ->caller( __METHOD__ )->fetchField();
158  $this->init( $count, $table );
159  $this->output( "Processing $table...\n" );
160 
161  $result = $this->getDB( DB_REPLICA )->select(
162  $queryInfo['tables'], $queryInfo['fields'], [], __METHOD__, [], $queryInfo['joins']
163  );
164 
165  foreach ( $result as $row ) {
166  $update = call_user_func( $callback, $row );
167  if ( $update ) {
168  $this->progress( 1 );
169  } else {
170  $this->progress( 0 );
171  }
172  }
173  $this->output( "Finished $table... $this->updated of $this->processed rows updated\n" );
174  }
175 
176  private function buildImage() {
177  $callback = [ $this, 'imageCallback' ];
178  $this->buildTable( 'image', LocalFile::getQueryInfo(), $callback );
179  }
180 
181  private function imageCallback( $row ) {
182  // Create a File object from the row
183  // This will also upgrade it
184  $file = $this->getRepo()->newFileFromRow( $row );
185 
186  return $file->getUpgraded();
187  }
188 
189  private function buildOldImage() {
190  $this->buildTable( 'oldimage', OldLocalFile::getQueryInfo(),
191  [ $this, 'oldimageCallback' ] );
192  }
193 
194  private function oldimageCallback( $row ) {
195  // Create a File object from the row
196  // This will also upgrade it
197  if ( $row->oi_archive_name == '' ) {
198  $this->output( "Empty oi_archive_name for oi_name={$row->oi_name}\n" );
199 
200  return false;
201  }
202  $file = $this->getRepo()->newFileFromRow( $row );
203 
204  return $file->getUpgraded();
205  }
206 
207  private function crawlMissing() {
208  $this->getRepo()->enumFiles( [ $this, 'checkMissingImage' ] );
209  }
210 
211  public function checkMissingImage( $fullpath ) {
212  $filename = wfBaseName( $fullpath );
213  $row = $this->dbw->newSelectQueryBuilder()
214  ->select( [ 'img_name' ] )
215  ->from( 'image' )
216  ->where( [ 'img_name' => $filename ] )
217  ->caller( __METHOD__ )->fetchRow();
218 
219  if ( !$row ) {
220  // file not registered
221  $this->addMissingImage( $filename, $fullpath );
222  }
223  }
224 
225  private function addMissingImage( $filename, $fullpath ) {
226  $timestamp = $this->dbw->timestamp( $this->getRepo()->getFileTimestamp( $fullpath ) );
227  $services = $this->getServiceContainer();
228 
229  $altname = $services->getContentLanguage()->checkTitleEncoding( $filename );
230  if ( $altname != $filename ) {
231  if ( $this->dryrun ) {
232  $filename = $altname;
233  $this->output( "Estimating transcoding... $altname\n" );
234  } else {
235  // @fixme create renameFile()
236  // @phan-suppress-next-line PhanUndeclaredMethod See comment above...
237  $filename = $this->renameFile( $filename );
238  }
239  }
240 
241  if ( $filename == '' ) {
242  $this->output( "Empty filename for $fullpath\n" );
243 
244  return;
245  }
246  if ( !$this->dryrun ) {
247  $file = $services->getRepoGroup()->getLocalRepo()->newFile( $filename );
248  $pageText = SpecialUpload::getInitialPageText(
249  '(recovered file, missing upload log entry)'
250  );
251  $user = User::newSystemUser( User::MAINTENANCE_SCRIPT_USER, [ 'steal' => true ] );
252  $status = $file->recordUpload3(
253  '',
254  '(recovered file, missing upload log entry)',
255  $pageText,
256  $user,
257  false,
258  $timestamp
259  );
260  if ( !$status->isOK() ) {
261  $this->output( "Error uploading file $fullpath\n" );
262 
263  return;
264  }
265  }
266  $this->output( $fullpath . "\n" );
267  }
268 }
269 
270 $maintClass = ImageBuilder::class;
271 require_once RUN_MAINTENANCE_IF_MAIN;
wfBaseName( $path, $suffix='')
Return the final portion of a pathname.
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Maintenance script to update image metadata records.
__construct()
Default constructor.
checkMissingImage( $fullpath)
IMaintainableDatabase $dbw
execute()
Do the actual work.
static getQueryInfo(array $options=[])
Return the tables, fields, and join conditions to be selected to create a new localfile object.
Definition: LocalFile.php:275
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:66
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
output( $out, $channel=null)
Throw some output to the user.
hasOption( $name)
Checks to see if a particular option was set.
getServiceContainer()
Returns the main service container.
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
Form for handling uploads and special page.
internal since 1.36
Definition: User.php:98
static getQueryInfo(array $options=[])
Return the tables, fields, and join conditions to be selected to create a new oldlocalfile object.
Advanced database interface for IDatabase handles that include maintenance methods.
const DB_REPLICA
Definition: defines.php:26
const DB_PRIMARY
Definition: defines.php:28
$maintClass
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition: router.php:42