MediaWiki  master
rebuildImages.php
Go to the documentation of this file.
1 <?php
33 require_once __DIR__ . '/Maintenance.php';
34 
37 
43 class ImageBuilder extends Maintenance {
47  protected $dbw;
48 
50  private $dryrun;
51 
53  private $repo;
54 
56  private $updated;
57 
59  private $processed;
60 
62  private $count;
63 
65  private $startTime;
66 
68  private $table;
69 
70  public function __construct() {
71  parent::__construct();
72 
74  // make sure to update old, but compatible img_metadata fields.
76 
77  $this->addDescription( 'Script to update image metadata records' );
78 
79  $this->addOption( 'missing', 'Check for files without associated database record' );
80  $this->addOption( 'dry-run', 'Only report, don\'t update the database' );
81  }
82 
83  public function execute() {
84  $this->dbw = $this->getDB( DB_MASTER );
85  $this->dryrun = $this->hasOption( 'dry-run' );
86  if ( $this->dryrun ) {
87  MediaWiki\MediaWikiServices::getInstance()->getReadOnlyMode()
88  ->setReason( 'Dry run mode, image upgrades are suppressed' );
89  }
90 
91  if ( $this->hasOption( 'missing' ) ) {
92  $this->crawlMissing();
93  } else {
94  $this->build();
95  }
96  }
97 
101  private function getRepo() {
102  if ( $this->repo === null ) {
103  $this->repo = MediaWikiServices::getInstance()->getRepoGroup()->getLocalRepo();
104  }
105 
106  return $this->repo;
107  }
108 
109  private function build() {
110  $this->buildImage();
111  $this->buildOldImage();
112  }
113 
118  private function init( $count, $table ) {
119  $this->processed = 0;
120  $this->updated = 0;
121  $this->count = $count;
122  $this->startTime = microtime( true );
123  $this->table = $table;
124  }
125 
126  private function progress( $updated ) {
127  $this->updated += $updated;
128  $this->processed++;
129  if ( $this->processed % 100 != 0 ) {
130  return;
131  }
132  $portion = $this->processed / $this->count;
133  $updateRate = $this->updated / $this->processed;
134 
135  $now = microtime( true );
136  $delta = $now - $this->startTime;
137  $estimatedTotalTime = $delta / $portion;
138  $eta = $this->startTime + $estimatedTotalTime;
139  $rate = $this->processed / $delta;
140 
141  $this->output( sprintf( "%s: %6.2f%% done on %s; ETA %s [%d/%d] %.2f/sec <%.2f%% updated>\n",
142  wfTimestamp( TS_DB, intval( $now ) ),
143  $portion * 100.0,
144  $this->table,
145  wfTimestamp( TS_DB, intval( $eta ) ),
146  $this->processed,
147  $this->count,
148  $rate,
149  $updateRate * 100.0 ) );
150  flush();
151  }
152 
153  private function buildTable( $table, $key, $queryInfo, $callback ) {
154  $count = $this->dbw->selectField( $table, 'count(*)', '', __METHOD__ );
155  $this->init( $count, $table );
156  $this->output( "Processing $table...\n" );
157 
158  $result = $this->getDB( DB_REPLICA )->select(
159  $queryInfo['tables'], $queryInfo['fields'], [], __METHOD__, [], $queryInfo['joins']
160  );
161 
162  foreach ( $result as $row ) {
163  $update = call_user_func( $callback, $row, null );
164  if ( $update ) {
165  $this->progress( 1 );
166  } else {
167  $this->progress( 0 );
168  }
169  }
170  $this->output( "Finished $table... $this->updated of $this->processed rows updated\n" );
171  }
172 
173  private function buildImage() {
174  $callback = [ $this, 'imageCallback' ];
175  $this->buildTable( 'image', 'img_name', LocalFile::getQueryInfo(), $callback );
176  }
177 
178  private function imageCallback( $row, $copy ) {
179  // Create a File object from the row
180  // This will also upgrade it
181  $file = $this->getRepo()->newFileFromRow( $row );
182 
183  return $file->getUpgraded();
184  }
185 
186  private function buildOldImage() {
187  $this->buildTable( 'oldimage', 'oi_archive_name', OldLocalFile::getQueryInfo(),
188  [ $this, 'oldimageCallback' ] );
189  }
190 
191  private function oldimageCallback( $row, $copy ) {
192  // Create a File object from the row
193  // This will also upgrade it
194  if ( $row->oi_archive_name == '' ) {
195  $this->output( "Empty oi_archive_name for oi_name={$row->oi_name}\n" );
196 
197  return false;
198  }
199  $file = $this->getRepo()->newFileFromRow( $row );
200 
201  return $file->getUpgraded();
202  }
203 
204  private function crawlMissing() {
205  $this->getRepo()->enumFiles( [ $this, 'checkMissingImage' ] );
206  }
207 
208  private function checkMissingImage( $fullpath ) {
209  $filename = wfBaseName( $fullpath );
210  $row = $this->dbw->selectRow( 'image',
211  [ 'img_name' ],
212  [ 'img_name' => $filename ],
213  __METHOD__ );
214 
215  if ( !$row ) {
216  // file not registered
217  $this->addMissingImage( $filename, $fullpath );
218  }
219  }
220 
221  private function addMissingImage( $filename, $fullpath ) {
222  $timestamp = $this->dbw->timestamp( $this->getRepo()->getFileTimestamp( $fullpath ) );
223  $services = MediaWikiServices::getInstance();
224 
225  $altname = $services->getContentLanguage()->checkTitleEncoding( $filename );
226  if ( $altname != $filename ) {
227  if ( $this->dryrun ) {
228  $filename = $altname;
229  $this->output( "Estimating transcoding... $altname\n" );
230  } else {
231  // @fixme create renameFile()
232  // @phan-suppress-next-line PhanUndeclaredMethod See comment above...
233  $filename = $this->renameFile( $filename );
234  }
235  }
236 
237  if ( $filename == '' ) {
238  $this->output( "Empty filename for $fullpath\n" );
239 
240  return;
241  }
242  if ( !$this->dryrun ) {
243  $file = $services->getRepoGroup()->getLocalRepo()->newFile( $filename );
245  '(recovered file, missing upload log entry)'
246  );
247  $user = User::newSystemUser( 'Maintenance script', [ 'steal' => true ] );
248  $status = $file->recordUpload2(
249  '',
250  '(recovered file, missing upload log entry)',
251  $pageText,
252  false,
253  $timestamp,
254  $user
255  );
256  if ( !$status->isOK() ) {
257  $this->output( "Error uploading file $fullpath\n" );
258 
259  return;
260  }
261  }
262  $this->output( $fullpath . "\n" );
263  }
264 }
265 
266 $maintClass = ImageBuilder::class;
267 require_once RUN_MAINTENANCE_IF_MAIN;
RUN_MAINTENANCE_IF_MAIN
const RUN_MAINTENANCE_IF_MAIN
Definition: Maintenance.php:38
ImageBuilder\init
init( $count, $table)
Definition: rebuildImages.php:118
$wgUpdateCompatibleMetadata
$wgUpdateCompatibleMetadata
If to automatically update the img_metadata field if the metadata field is outdated but compatible wi...
Definition: DefaultSettings.php:891
SpecialUpload\getInitialPageText
static getInitialPageText( $comment='', $license='', $copyStatus='', $source='', Config $config=null)
Get the initial image page text based on a comment and optional file status information.
Definition: SpecialUpload.php:611
ImageBuilder\buildOldImage
buildOldImage()
Definition: rebuildImages.php:186
ImageBuilder\execute
execute()
Do the actual work.
Definition: rebuildImages.php:83
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:154
OldLocalFile\getQueryInfo
static getQueryInfo(array $options=[])
Return the tables, fields, and join conditions to be selected to create a new oldlocalfile object.
Definition: OldLocalFile.php:130
Maintenance\addDescription
addDescription( $text)
Set the description text.
Definition: Maintenance.php:327
wfTimestamp
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Definition: GlobalFunctions.php:1808
ImageBuilder
Maintenance script to update image metadata records.
Definition: rebuildImages.php:43
$maintClass
$maintClass
Definition: rebuildImages.php:266
wfBaseName
wfBaseName( $path, $suffix='')
Return the final portion of a pathname.
Definition: GlobalFunctions.php:2313
$file
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition: router.php:42
ImageBuilder\$processed
int $processed
Definition: rebuildImages.php:59
ImageBuilder\$updated
int $updated
Definition: rebuildImages.php:56
ImageBuilder\getRepo
getRepo()
Definition: rebuildImages.php:101
Maintenance
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:55
User\newSystemUser
static newSystemUser( $name, $options=[])
Static factory method for creation of a "system" user from username.
Definition: User.php:756
MediaWiki\MediaWikiServices\getInstance
static getInstance()
Returns the global default instance of the top level service locator.
Definition: MediaWikiServices.php:185
ImageBuilder\$count
int $count
Definition: rebuildImages.php:62
ImageBuilder\imageCallback
imageCallback( $row, $copy)
Definition: rebuildImages.php:178
Maintenance\addOption
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
Definition: Maintenance.php:245
DB_REPLICA
const DB_REPLICA
Definition: defines.php:25
DB_MASTER
const DB_MASTER
Definition: defines.php:26
ImageBuilder\$dryrun
bool $dryrun
Definition: rebuildImages.php:50
ImageBuilder\$repo
LocalRepo null $repo
Definition: rebuildImages.php:53
ImageBuilder\checkMissingImage
checkMissingImage( $fullpath)
Definition: rebuildImages.php:208
Maintenance\getDB
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
Definition: Maintenance.php:1366
ImageBuilder\__construct
__construct()
Default constructor.
Definition: rebuildImages.php:70
ImageBuilder\oldimageCallback
oldimageCallback( $row, $copy)
Definition: rebuildImages.php:191
ImageBuilder\crawlMissing
crawlMissing()
Definition: rebuildImages.php:204
LocalFile\getQueryInfo
static getQueryInfo(array $options=[])
Return the tables, fields, and join conditions to be selected to create a new localfile object.
Definition: LocalFile.php:227
ImageBuilder\$table
string $table
Definition: rebuildImages.php:68
ImageBuilder\buildTable
buildTable( $table, $key, $queryInfo, $callback)
Definition: rebuildImages.php:153
ImageBuilder\addMissingImage
addMissingImage( $filename, $fullpath)
Definition: rebuildImages.php:221
ImageBuilder\$dbw
IMaintainableDatabase $dbw
Definition: rebuildImages.php:47
Maintenance\output
output( $out, $channel=null)
Throw some output to the user.
Definition: Maintenance.php:434
ImageBuilder\$startTime
int $startTime
Definition: rebuildImages.php:65
Maintenance\hasOption
hasOption( $name)
Checks to see if a particular option exists.
Definition: Maintenance.php:266
ImageBuilder\build
build()
Definition: rebuildImages.php:109
ImageBuilder\progress
progress( $updated)
Definition: rebuildImages.php:126
Wikimedia\Rdbms\IMaintainableDatabase
Advanced database interface for IDatabase handles that include maintenance methods.
Definition: IMaintainableDatabase.php:38
ImageBuilder\buildImage
buildImage()
Definition: rebuildImages.php:173
LocalRepo
A repository that stores files in the local filesystem and registers them in the wiki's own database.
Definition: LocalRepo.php:37