MediaWiki  master
rebuildImages.php
Go to the documentation of this file.
1 <?php
33 require_once __DIR__ . '/Maintenance.php';
34 
37 
43 class ImageBuilder extends Maintenance {
47  protected $dbw;
48 
50  private $dryrun;
51 
53  private $repo;
54 
56  private $updated;
57 
59  private $processed;
60 
62  private $count;
63 
65  private $startTime;
66 
68  private $table;
69 
70  public function __construct() {
71  parent::__construct();
72  $this->addDescription( 'Script to update image metadata records' );
73 
74  $this->addOption( 'missing', 'Check for files without associated database record' );
75  $this->addOption( 'dry-run', 'Only report, don\'t update the database' );
76  }
77 
78  public function execute() {
79  $this->dbw = $this->getDB( DB_PRIMARY );
80  $this->dryrun = $this->hasOption( 'dry-run' );
81  if ( $this->dryrun ) {
82  MediaWiki\MediaWikiServices::getInstance()->getReadOnlyMode()
83  ->setReason( 'Dry run mode, image upgrades are suppressed' );
84  }
85 
86  if ( $this->hasOption( 'missing' ) ) {
87  $this->crawlMissing();
88  } else {
89  $this->build();
90  }
91  }
92 
96  private function getRepo() {
97  if ( $this->repo === null ) {
98  $this->repo = MediaWikiServices::getInstance()->getRepoGroup()
99  ->newCustomLocalRepo( [
100  // make sure to update old, but compatible img_metadata fields.
101  'updateCompatibleMetadata' => true
102  ] );
103  }
104 
105  return $this->repo;
106  }
107 
108  private function build() {
109  $this->buildImage();
110  $this->buildOldImage();
111  }
112 
117  private function init( $count, $table ) {
118  $this->processed = 0;
119  $this->updated = 0;
120  $this->count = $count;
121  $this->startTime = microtime( true );
122  $this->table = $table;
123  }
124 
125  private function progress( $updated ) {
126  $this->updated += $updated;
127  $this->processed++;
128  if ( $this->processed % 100 != 0 ) {
129  return;
130  }
131  $portion = $this->processed / $this->count;
132  $updateRate = $this->updated / $this->processed;
133 
134  $now = microtime( true );
135  $delta = $now - $this->startTime;
136  $estimatedTotalTime = $delta / $portion;
137  $eta = $this->startTime + $estimatedTotalTime;
138  $rate = $this->processed / $delta;
139 
140  $this->output( sprintf( "%s: %6.2f%% done on %s; ETA %s [%d/%d] %.2f/sec <%.2f%% updated>\n",
141  wfTimestamp( TS_DB, intval( $now ) ),
142  $portion * 100.0,
143  $this->table,
144  wfTimestamp( TS_DB, intval( $eta ) ),
145  $this->processed,
146  $this->count,
147  $rate,
148  $updateRate * 100.0 ) );
149  flush();
150  }
151 
152  private function buildTable( $table, $key, $queryInfo, $callback ) {
153  $count = $this->dbw->selectField( $table, 'count(*)', '', __METHOD__ );
154  $this->init( $count, $table );
155  $this->output( "Processing $table...\n" );
156 
157  $result = $this->getDB( DB_REPLICA )->select(
158  $queryInfo['tables'], $queryInfo['fields'], [], __METHOD__, [], $queryInfo['joins']
159  );
160 
161  foreach ( $result as $row ) {
162  $update = call_user_func( $callback, $row, null );
163  if ( $update ) {
164  $this->progress( 1 );
165  } else {
166  $this->progress( 0 );
167  }
168  }
169  $this->output( "Finished $table... $this->updated of $this->processed rows updated\n" );
170  }
171 
172  private function buildImage() {
173  $callback = [ $this, 'imageCallback' ];
174  $this->buildTable( 'image', 'img_name', LocalFile::getQueryInfo(), $callback );
175  }
176 
177  private function imageCallback( $row, $copy ) {
178  // Create a File object from the row
179  // This will also upgrade it
180  $file = $this->getRepo()->newFileFromRow( $row );
181 
182  return $file->getUpgraded();
183  }
184 
185  private function buildOldImage() {
186  $this->buildTable( 'oldimage', 'oi_archive_name', OldLocalFile::getQueryInfo(),
187  [ $this, 'oldimageCallback' ] );
188  }
189 
190  private function oldimageCallback( $row, $copy ) {
191  // Create a File object from the row
192  // This will also upgrade it
193  if ( $row->oi_archive_name == '' ) {
194  $this->output( "Empty oi_archive_name for oi_name={$row->oi_name}\n" );
195 
196  return false;
197  }
198  $file = $this->getRepo()->newFileFromRow( $row );
199 
200  return $file->getUpgraded();
201  }
202 
203  private function crawlMissing() {
204  $this->getRepo()->enumFiles( [ $this, 'checkMissingImage' ] );
205  }
206 
207  public function checkMissingImage( $fullpath ) {
208  $filename = wfBaseName( $fullpath );
209  $row = $this->dbw->selectRow( 'image',
210  [ 'img_name' ],
211  [ 'img_name' => $filename ],
212  __METHOD__ );
213 
214  if ( !$row ) {
215  // file not registered
216  $this->addMissingImage( $filename, $fullpath );
217  }
218  }
219 
220  private function addMissingImage( $filename, $fullpath ) {
221  $timestamp = $this->dbw->timestamp( $this->getRepo()->getFileTimestamp( $fullpath ) );
222  $services = MediaWikiServices::getInstance();
223 
224  $altname = $services->getContentLanguage()->checkTitleEncoding( $filename );
225  if ( $altname != $filename ) {
226  if ( $this->dryrun ) {
227  $filename = $altname;
228  $this->output( "Estimating transcoding... $altname\n" );
229  } else {
230  // @fixme create renameFile()
231  // @phan-suppress-next-line PhanUndeclaredMethod See comment above...
232  $filename = $this->renameFile( $filename );
233  }
234  }
235 
236  if ( $filename == '' ) {
237  $this->output( "Empty filename for $fullpath\n" );
238 
239  return;
240  }
241  if ( !$this->dryrun ) {
242  $file = $services->getRepoGroup()->getLocalRepo()->newFile( $filename );
244  '(recovered file, missing upload log entry)'
245  );
246  $user = User::newSystemUser( User::MAINTENANCE_SCRIPT_USER, [ 'steal' => true ] );
247  $status = $file->recordUpload3(
248  '',
249  '(recovered file, missing upload log entry)',
250  $pageText,
251  $user,
252  false,
253  $timestamp
254  );
255  if ( !$status->isOK() ) {
256  $this->output( "Error uploading file $fullpath\n" );
257 
258  return;
259  }
260  }
261  $this->output( $fullpath . "\n" );
262  }
263 }
264 
265 $maintClass = ImageBuilder::class;
266 require_once RUN_MAINTENANCE_IF_MAIN;
ImageBuilder\init
init( $count, $table)
Definition: rebuildImages.php:117
SpecialUpload\getInitialPageText
static getInitialPageText( $comment='', $license='', $copyStatus='', $source='', Config $config=null)
Get the initial image page text based on a comment and optional file status information.
Definition: SpecialUpload.php:644
ImageBuilder\buildOldImage
buildOldImage()
Definition: rebuildImages.php:185
ImageBuilder\execute
execute()
Do the actual work.
Definition: rebuildImages.php:78
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:186
OldLocalFile\getQueryInfo
static getQueryInfo(array $options=[])
Return the tables, fields, and join conditions to be selected to create a new oldlocalfile object.
Definition: OldLocalFile.php:137
Maintenance\addDescription
addDescription( $text)
Set the description text.
Definition: Maintenance.php:329
wfTimestamp
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Definition: GlobalFunctions.php:1692
ImageBuilder
Maintenance script to update image metadata records.
Definition: rebuildImages.php:43
$maintClass
$maintClass
Definition: rebuildImages.php:265
wfBaseName
wfBaseName( $path, $suffix='')
Return the final portion of a pathname.
Definition: GlobalFunctions.php:2118
$file
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition: router.php:42
ImageBuilder\$processed
int $processed
Definition: rebuildImages.php:59
ImageBuilder\$updated
int $updated
Definition: rebuildImages.php:56
ImageBuilder\getRepo
getRepo()
Definition: rebuildImages.php:96
Maintenance
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:59
User\newSystemUser
static newSystemUser( $name, $options=[])
Static factory method for creation of a "system" user from username.
Definition: User.php:805
MediaWiki\MediaWikiServices\getInstance
static getInstance()
Returns the global default instance of the top level service locator.
Definition: MediaWikiServices.php:247
ImageBuilder\$count
int $count
Definition: rebuildImages.php:62
ImageBuilder\imageCallback
imageCallback( $row, $copy)
Definition: rebuildImages.php:177
Maintenance\addOption
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
Definition: Maintenance.php:249
DB_REPLICA
const DB_REPLICA
Definition: defines.php:25
ImageBuilder\$dryrun
bool $dryrun
Definition: rebuildImages.php:50
ImageBuilder\$repo
LocalRepo null $repo
Definition: rebuildImages.php:53
DB_PRIMARY
const DB_PRIMARY
Definition: defines.php:27
ImageBuilder\checkMissingImage
checkMissingImage( $fullpath)
Definition: rebuildImages.php:207
Maintenance\getDB
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
Definition: Maintenance.php:1361
ImageBuilder\__construct
__construct()
Default constructor.
Definition: rebuildImages.php:70
ImageBuilder\oldimageCallback
oldimageCallback( $row, $copy)
Definition: rebuildImages.php:190
ImageBuilder\crawlMissing
crawlMissing()
Definition: rebuildImages.php:203
LocalFile\getQueryInfo
static getQueryInfo(array $options=[])
Return the tables, fields, and join conditions to be selected to create a new localfile object.
Definition: LocalFile.php:267
ImageBuilder\$table
string $table
Definition: rebuildImages.php:68
ImageBuilder\buildTable
buildTable( $table, $key, $queryInfo, $callback)
Definition: rebuildImages.php:152
ImageBuilder\addMissingImage
addMissingImage( $filename, $fullpath)
Definition: rebuildImages.php:220
ImageBuilder\$dbw
IMaintainableDatabase $dbw
Definition: rebuildImages.php:47
Maintenance\output
output( $out, $channel=null)
Throw some output to the user.
Definition: Maintenance.php:435
ImageBuilder\$startTime
int $startTime
Definition: rebuildImages.php:65
Maintenance\hasOption
hasOption( $name)
Checks to see if a particular option was set.
Definition: Maintenance.php:271
ImageBuilder\build
build()
Definition: rebuildImages.php:108
ImageBuilder\progress
progress( $updated)
Definition: rebuildImages.php:125
Wikimedia\Rdbms\IMaintainableDatabase
Advanced database interface for IDatabase handles that include maintenance methods.
Definition: IMaintainableDatabase.php:38
ImageBuilder\buildImage
buildImage()
Definition: rebuildImages.php:172
LocalRepo
A repository that stores files in the local filesystem and registers them in the wiki's own database.
Definition: LocalRepo.php:41
User\MAINTENANCE_SCRIPT_USER
const MAINTENANCE_SCRIPT_USER
Username used for various maintenance scripts.
Definition: User.php:121