MediaWiki master
rebuildImages.php
Go to the documentation of this file.
1<?php
18
19// @codeCoverageIgnoreStart
20require_once __DIR__ . '/Maintenance.php';
21// @codeCoverageIgnoreEnd
22
28use MediaWiki\User\User;
31use Wikimedia\Timestamp\ConvertibleTimestamp;
32use Wikimedia\Timestamp\TimestampFormat as TS;
33
43 protected $dbw;
44
46 private $dryrun;
47
49 private $repo;
50
52 private $updated;
53
55 private $processed;
56
58 private $count;
59
61 private $startTimer;
62
64 private $table;
65
66 public function __construct() {
67 parent::__construct();
68 $this->addDescription( 'Script to update image metadata records' );
69
70 $this->addOption( 'missing', 'Check for files without associated database record' );
71 $this->addOption( 'dry-run', 'Only report, don\'t update the database' );
72 }
73
74 public function execute() {
75 $this->dbw = $this->getPrimaryDB();
76 $this->dryrun = $this->hasOption( 'dry-run' );
77 if ( $this->dryrun ) {
78 $this->getServiceContainer()->getReadOnlyMode()
79 ->setReason( 'Dry run mode, image upgrades are suppressed' );
80 }
81
82 if ( $this->hasOption( 'missing' ) ) {
83 $this->crawlMissing();
84 } else {
85 $this->build();
86 }
87 }
88
92 private function getRepo() {
93 if ( $this->repo === null ) {
94 $this->repo = $this->getServiceContainer()->getRepoGroup()
95 ->newCustomLocalRepo( [
96 // make sure to update old, but compatible img_metadata fields.
97 'updateCompatibleMetadata' => true
98 ] );
99 }
100
101 return $this->repo;
102 }
103
104 private function build() {
105 $migrationStage = $this->getServiceContainer()->getMainConfig()->get(
107 );
108
109 if ( $migrationStage & SCHEMA_COMPAT_READ_OLD ) {
110 $this->buildImage();
111 $this->buildOldImage();
112 } else {
113 $this->buildFile();
114 }
115 }
116
121 private function init( $count, $table ) {
122 $this->processed = 0;
123 $this->updated = 0;
124 $this->count = $count;
125 $this->startTimer = ConvertibleTimestamp::hrtime();
126 $this->table = $table;
127 }
128
129 private function progress( int $updated ) {
130 $this->updated += $updated;
131 $this->processed++;
132 if ( $this->processed % 100 != 0 ) {
133 return;
134 }
135 $portion = $this->processed / $this->count;
136 $updateRate = $this->updated / $this->processed;
137
138 $now = microtime( true );
139 $delta = ( ConvertibleTimestamp::hrtime() - $this->startTimer ) / 1e9;
140 $estimatedTotalTime = $delta / $portion;
141 // The ETA is rendered as a date, so it is the current wall clock plus
142 // the time still to go. The remaining time comes from the timer, so a
143 // system clock adjustment cannot distort it.
144 $eta = $now + ( $estimatedTotalTime - $delta );
145 $rate = $this->processed / $delta;
146
147 $this->output( sprintf( "%s: %6.2f%% done on %s; ETA %s [%d/%d] %.2f/sec <%.2f%% updated>\n",
148 wfTimestamp( TS::DB, intval( $now ) ),
149 $portion * 100.0,
150 $this->table,
151 wfTimestamp( TS::DB, intval( $eta ) ),
152 $this->processed,
153 $this->count,
154 $rate,
155 $updateRate * 100.0 ) );
156 flush();
157 }
158
159 private function buildTable( string $table, SelectQueryBuilder $queryBuilder, callable $callback ) {
160 $count = $this->dbw->newSelectQueryBuilder()
161 ->select( 'count(*)' )
162 ->from( $table )
163 ->caller( __METHOD__ )->fetchField();
164 $this->init( $count, $table );
165 $this->output( "Processing $table...\n" );
166
167 $result = $queryBuilder->caller( __METHOD__ )->fetchResultSet();
168
169 foreach ( $result as $row ) {
170 $update = $callback( $row );
171 if ( $update ) {
172 $this->progress( 1 );
173 } else {
174 $this->progress( 0 );
175 }
176 }
177 $this->output( "Finished $table... $this->updated of $this->processed rows updated\n" );
178 }
179
180 private function buildImage() {
181 $this->buildTable(
182 'image',
183 FileSelectQueryBuilder::newForFile( $this->getReplicaDB() ),
184 $this->imageCallback( ... )
185 );
186 }
187
188 private function imageCallback( \stdClass $row ): bool {
189 // Create a File object from the row
190 // This will also upgrade it
191 $file = $this->getRepo()->newFileFromRow( $row );
192
193 return $file->getUpgraded();
194 }
195
196 private function buildOldImage() {
197 $this->buildTable(
198 'oldimage',
199 FileSelectQueryBuilder::newForOldFile( $this->getReplicaDB() ),
200 $this->oldimageCallback( ... )
201 );
202 }
203
204 private function oldimageCallback( \stdClass $row ): bool {
205 // Create a File object from the row
206 // This will also upgrade it
207 if ( $row->oi_archive_name == '' ) {
208 $this->output( "Empty oi_archive_name for oi_name={$row->oi_name}\n" );
209
210 return false;
211 }
212 $file = $this->getRepo()->newFileFromRow( $row );
213
214 return $file->getUpgraded();
215 }
216
217 private function buildFile() {
218 $this->buildTable(
219 'file',
220 FileSelectQueryBuilder::newForFile( $this->getReplicaDB() ),
221 $this->fileCallback( ... )
222 );
223 }
224
225 private function fileCallback( \stdClass $row ): bool {
226 // Create a File object from the row
227 // This will also upgrade it
228 $file = $this->getRepo()->newFileFromRow( $row );
229
230 return $file->getUpgraded();
231 }
232
233 private function crawlMissing() {
234 $this->getRepo()->enumFiles( $this->checkMissingImage( ... ) );
235 }
236
237 private function checkMissingImage( string $fullpath ) {
238 $filename = wfBaseName( $fullpath );
239
240 $row = FileSelectQueryBuilder::newForFile( $this->getReplicaDB() )
241 ->where( [ 'img_name' => $filename ] )
242 ->caller( __METHOD__ )->fetchRow();
243
244 if ( !$row ) {
245 // file not registered
246 $this->addMissingImage( $filename, $fullpath );
247 }
248 }
249
250 private function addMissingImage( string $filename, string $fullpath ) {
251 $timestamp = $this->dbw->timestamp( $this->getRepo()->getFileTimestamp( $fullpath ) );
252 $services = $this->getServiceContainer();
253
254 $altname = $services->getContentLanguage()->checkTitleEncoding( $filename );
255 if ( $altname != $filename ) {
256 if ( $this->dryrun ) {
257 $filename = $altname;
258 $this->output( "Estimating transcoding... $altname\n" );
259 } else {
260 // @fixme create renameFile()
261 // @phan-suppress-next-line PhanUndeclaredMethod See comment above...
262 $filename = $this->renameFile( $filename );
263 }
264 }
265
266 if ( $filename == '' ) {
267 $this->output( "Empty filename for $fullpath\n" );
268
269 return;
270 }
271 if ( !$this->dryrun ) {
272 $file = $services->getRepoGroup()->getLocalRepo()->newFile( $filename );
273 $pageText = SpecialUpload::getInitialPageText(
274 '(recovered file, missing upload log entry)'
275 );
276 $user = User::newSystemUser( User::MAINTENANCE_SCRIPT_USER, [ 'steal' => true ] );
277 $status = $file->recordUpload3(
278 '',
279 '(recovered file, missing upload log entry)',
280 $pageText,
281 $user,
282 false,
283 $timestamp
284 );
285 if ( !$status->isOK() ) {
286 $this->output( "Error uploading file $fullpath\n" );
287
288 return;
289 }
290 }
291 $this->output( $fullpath . "\n" );
292 }
293}
294
295// @codeCoverageIgnoreStart
296$maintClass = ImageBuilder::class;
297require_once RUN_MAINTENANCE_IF_MAIN;
298// @codeCoverageIgnoreEnd
const SCHEMA_COMPAT_READ_OLD
Definition Defines.php:294
wfBaseName( $path, $suffix='')
Return the final portion of a pathname.
wfTimestamp( $outputtype=TS::UNIX, $ts=0)
Get a timestamp string in one of various formats.
if(!defined('MW_SETUP_CALLBACK'))
Definition WebStart.php:72
$maintClass
Maintenance script to update image metadata records.
IDatabase $dbw
static newForFile(IReadableDatabase $db, array $options=[])
Local repository that stores files in the local filesystem and registers them in the wiki's own datab...
Definition LocalRepo.php:43
A class containing constants representing the names of configuration variables.
const FileSchemaMigrationStage
Name constant for the FileSchemaMigrationStage setting, for use with Config\get().
Form for uploading media files.
Build SELECT queries with a fluent interface.
caller( $fname)
Set the method name to be included in an SQL comment.
Interface to a relational database.
Definition IDatabase.php:31