MediaWiki  master
populateImageSha1.php
Go to the documentation of this file.
1 <?php
26 
27 require_once __DIR__ . '/Maintenance.php';
28 
35  public function __construct() {
36  parent::__construct();
37  $this->addDescription( 'Populate the img_sha1 field' );
38  $this->addOption( 'force', "Recalculate sha1 for rows that already have a value" );
39  $this->addOption( 'multiversiononly', "Calculate only for files with several versions" );
40  $this->addOption( 'method', "Use 'pipe' to pipe to mysql command line,\n" .
41  "\t\tdefault uses Database class", false, true );
42  $this->addOption(
43  'file',
44  'Fix for a specific file, without File: namespace prefixed',
45  false,
46  true
47  );
48  }
49 
50  protected function getUpdateKey() {
51  return 'populate img_sha1';
52  }
53 
54  protected function updateSkippedMessage() {
55  return 'img_sha1 column of image table already populated.';
56  }
57 
58  public function execute() {
59  if ( $this->getOption( 'file' ) || $this->hasOption( 'multiversiononly' ) ) {
60  $this->doDBUpdates(); // skip update log checks/saves
61  } else {
62  parent::execute();
63  }
64  }
65 
66  public function doDBUpdates() {
67  $method = $this->getOption( 'method', 'normal' );
68  $file = $this->getOption( 'file', '' );
69  $force = $this->getOption( 'force' );
70  $isRegen = ( $force || $file != '' ); // forced recalculation?
71 
72  $t = -microtime( true );
73  $dbw = $this->getDB( DB_MASTER );
74  if ( $file != '' ) {
75  $res = $dbw->select(
76  'image',
77  [ 'img_name' ],
78  [ 'img_name' => $file ],
79  __METHOD__
80  );
81  if ( !$res ) {
82  $this->fatalError( "No such file: $file" );
83  }
84  $this->output( "Populating img_sha1 field for specified files\n" );
85  } else {
86  if ( $this->hasOption( 'multiversiononly' ) ) {
87  $conds = [];
88  $this->output( "Populating and recalculating img_sha1 field for versioned files\n" );
89  } elseif ( $force ) {
90  $conds = [];
91  $this->output( "Populating and recalculating img_sha1 field\n" );
92  } else {
93  $conds = [ 'img_sha1' => '' ];
94  $this->output( "Populating img_sha1 field\n" );
95  }
96  if ( $this->hasOption( 'multiversiononly' ) ) {
97  $res = $dbw->select( 'oldimage',
98  [ 'img_name' => 'DISTINCT(oi_name)' ], $conds, __METHOD__ );
99  } else {
100  $res = $dbw->select( 'image', [ 'img_name' ], $conds, __METHOD__ );
101  }
102  }
103 
104  $imageTable = $dbw->tableName( 'image' );
105  $oldImageTable = $dbw->tableName( 'oldimage' );
106 
107  if ( $method == 'pipe' ) {
108  // Opening a pipe allows the SHA-1 operation to be done in parallel
109  // with the database write operation, because the writes are queued
110  // in the pipe buffer. This can improve performance by up to a
111  // factor of 2.
112  $config = $this->getConfig();
113  $cmd = 'mysql -u' . Shell::escape( $config->get( 'DBuser' ) ) .
114  ' -h' . Shell::escape( $config->get( 'DBserver' ) ) .
115  ' -p' . Shell::escape( $config->get( 'DBpassword' ), $config->get( 'DBname' ) );
116  $this->output( "Using pipe method\n" );
117  $pipe = popen( $cmd, 'w' );
118  }
119 
120  $numRows = $res->numRows();
121  $i = 0;
122  $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
123  foreach ( $res as $row ) {
124  if ( $i % $this->getBatchSize() == 0 ) {
125  $this->output( sprintf(
126  "Done %d of %d, %5.3f%% \r", $i, $numRows, $i / $numRows * 100 ) );
127  $lbFactory->waitForReplication();
128  }
129 
130  $file = MediaWikiServices::getInstance()->getRepoGroup()->getLocalRepo()
131  ->newFile( $row->img_name );
132  if ( !$file ) {
133  continue;
134  }
135 
136  // Upgrade the current file version...
137  $sha1 = $file->getRepo()->getFileSha1( $file->getPath() );
138  if ( strval( $sha1 ) !== '' ) { // file on disk and hashed properly
139  if ( $isRegen && $file->getSha1() !== $sha1 ) {
140  // The population was probably done already. If the old SHA1
141  // does not match, then both fix the SHA1 and the metadata.
142  $file->upgradeRow();
143  } else {
144  $sql = "UPDATE $imageTable SET img_sha1=" . $dbw->addQuotes( $sha1 ) .
145  " WHERE img_name=" . $dbw->addQuotes( $file->getName() );
146  if ( $method == 'pipe' ) {
147  fwrite( $pipe, "$sql;\n" );
148  } else {
149  $dbw->query( $sql, __METHOD__ );
150  }
151  }
152  }
153  // Upgrade the old file versions...
154  foreach ( $file->getHistory() as $oldFile ) {
156  '@phan-var OldLocalFile $oldFile';
157  $sha1 = $oldFile->getRepo()->getFileSha1( $oldFile->getPath() );
158  if ( strval( $sha1 ) !== '' ) { // file on disk and hashed properly
159  if ( $isRegen && $oldFile->getSha1() !== $sha1 ) {
160  // The population was probably done already. If the old SHA1
161  // does not match, then both fix the SHA1 and the metadata.
162  $oldFile->upgradeRow();
163  } else {
164  $sql = "UPDATE $oldImageTable SET oi_sha1=" . $dbw->addQuotes( $sha1 ) .
165  " WHERE (oi_name=" . $dbw->addQuotes( $oldFile->getName() ) . " AND" .
166  " oi_archive_name=" . $dbw->addQuotes( $oldFile->getArchiveName() ) . ")";
167  if ( $method == 'pipe' ) {
168  fwrite( $pipe, "$sql;\n" );
169  } else {
170  $dbw->query( $sql, __METHOD__ );
171  }
172  }
173  }
174  }
175  $i++;
176  }
177  if ( $method == 'pipe' ) {
178  fflush( $pipe );
179  pclose( $pipe );
180  }
181  $t += microtime( true );
182  $this->output( sprintf( "\nDone %d files in %.1f seconds\n", $numRows, $t ) );
183 
184  return !$file; // we only updated *some* files, don't log
185  }
186 }
187 
188 $maintClass = PopulateImageSha1::class;
189 require_once RUN_MAINTENANCE_IF_MAIN;
RUN_MAINTENANCE_IF_MAIN
const RUN_MAINTENANCE_IF_MAIN
Definition: Maintenance.php:38
MediaWiki\Shell\Shell
Executes shell commands.
Definition: Shell.php:44
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:154
Maintenance\fatalError
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
Definition: Maintenance.php:487
Maintenance\addDescription
addDescription( $text)
Set the description text.
Definition: Maintenance.php:327
$file
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition: router.php:42
$res
$res
Definition: testCompression.php:57
PopulateImageSha1\updateSkippedMessage
updateSkippedMessage()
Message to show that the update was done already and was just skipped.
Definition: populateImageSha1.php:54
Maintenance\getConfig
getConfig()
Definition: Maintenance.php:596
PopulateImageSha1\getUpdateKey
getUpdateKey()
Get the update key name to go in the update log table.
Definition: populateImageSha1.php:50
LoggedUpdateMaintenance
Class for scripts that perform database maintenance and want to log the update in updatelog so we can...
Definition: LoggedUpdateMaintenance.php:26
Config\get
get( $name)
Get a configuration variable such as "Sitename" or "UploadMaintenance.".
Maintenance\addOption
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
Definition: Maintenance.php:245
DB_MASTER
const DB_MASTER
Definition: defines.php:26
PopulateImageSha1\__construct
__construct()
Default constructor.
Definition: populateImageSha1.php:35
Maintenance\$config
Config $config
Accessible via getConfig()
Definition: Maintenance.php:149
PopulateImageSha1
Maintenance script to populate the img_sha1 field.
Definition: populateImageSha1.php:34
Maintenance\getDB
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
Definition: Maintenance.php:1366
PopulateImageSha1\doDBUpdates
doDBUpdates()
Do the actual work.
Definition: populateImageSha1.php:66
Maintenance\getOption
getOption( $name, $default=null)
Get an option, or return the default.
Definition: Maintenance.php:281
PopulateImageSha1\execute
execute()
Do the actual work.
Definition: populateImageSha1.php:58
$maintClass
$maintClass
Definition: populateImageSha1.php:188
Maintenance\getBatchSize
getBatchSize()
Returns batch size.
Definition: Maintenance.php:366
Maintenance\output
output( $out, $channel=null)
Throw some output to the user.
Definition: Maintenance.php:434
$t
$t
Definition: testCompression.php:74
Maintenance\hasOption
hasOption( $name)
Checks to see if a particular option exists.
Definition: Maintenance.php:266