MediaWiki REL1_37
populateImageSha1.php
Go to the documentation of this file.
1<?php
26
27require_once __DIR__ . '/Maintenance.php';
28
35 public function __construct() {
36 parent::__construct();
37 $this->addDescription( 'Populate the img_sha1 field' );
38 $this->addOption( 'force', "Recalculate sha1 for rows that already have a value" );
39 $this->addOption( 'multiversiononly', "Calculate only for files with several versions" );
40 $this->addOption( 'method', "Use 'pipe' to pipe to mysql command line,\n" .
41 "\t\tdefault uses Database class", false, true );
42 $this->addOption(
43 'file',
44 'Fix for a specific file, without File: namespace prefixed',
45 false,
46 true
47 );
48 }
49
50 protected function getUpdateKey() {
51 return 'populate img_sha1';
52 }
53
54 protected function updateSkippedMessage() {
55 return 'img_sha1 column of image table already populated.';
56 }
57
58 public function execute() {
59 if ( $this->getOption( 'file' ) || $this->hasOption( 'multiversiononly' ) ) {
60 $this->doDBUpdates(); // skip update log checks/saves
61 } else {
62 parent::execute();
63 }
64 }
65
66 public function doDBUpdates() {
67 $method = $this->getOption( 'method', 'normal' );
68 $file = $this->getOption( 'file', '' );
69 $force = $this->getOption( 'force' );
70 $isRegen = ( $force || $file != '' ); // forced recalculation?
71
72 $t = -microtime( true );
73 $dbw = $this->getDB( DB_PRIMARY );
74 if ( $file != '' ) {
75 $res = $dbw->select(
76 'image',
77 [ 'img_name' ],
78 [ 'img_name' => $file ],
79 __METHOD__
80 );
81 if ( !$res ) {
82 $this->fatalError( "No such file: $file" );
83 }
84 $this->output( "Populating img_sha1 field for specified files\n" );
85 } else {
86 if ( $this->hasOption( 'multiversiononly' ) ) {
87 $conds = [];
88 $this->output( "Populating and recalculating img_sha1 field for versioned files\n" );
89 } elseif ( $force ) {
90 $conds = [];
91 $this->output( "Populating and recalculating img_sha1 field\n" );
92 } else {
93 $conds = [ 'img_sha1' => '' ];
94 $this->output( "Populating img_sha1 field\n" );
95 }
96 if ( $this->hasOption( 'multiversiononly' ) ) {
97 $res = $dbw->select( 'oldimage',
98 [ 'img_name' => 'DISTINCT(oi_name)' ], $conds, __METHOD__ );
99 } else {
100 $res = $dbw->select( 'image', [ 'img_name' ], $conds, __METHOD__ );
101 }
102 }
103
104 $imageTable = $dbw->tableName( 'image' );
105 $oldImageTable = $dbw->tableName( 'oldimage' );
106
107 if ( $method == 'pipe' ) {
108 // Opening a pipe allows the SHA-1 operation to be done in parallel
109 // with the database write operation, because the writes are queued
110 // in the pipe buffer. This can improve performance by up to a
111 // factor of 2.
112 $config = $this->getConfig();
113 $cmd = 'mysql -u' . Shell::escape( $config->get( 'DBuser' ) ) .
114 ' -h' . Shell::escape( $config->get( 'DBserver' ) ) .
115 ' -p' . Shell::escape( $config->get( 'DBpassword' ), $config->get( 'DBname' ) );
116 $this->output( "Using pipe method\n" );
117 $pipe = popen( $cmd, 'w' );
118 }
119
120 $numRows = $res->numRows();
121 $i = 0;
122 $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
123 foreach ( $res as $row ) {
124 if ( $i % $this->getBatchSize() == 0 ) {
125 $this->output( sprintf(
126 "Done %d of %d, %5.3f%% \r", $i, $numRows, $i / $numRows * 100 ) );
127 $lbFactory->waitForReplication();
128 }
129
130 $file = MediaWikiServices::getInstance()->getRepoGroup()->getLocalRepo()
131 ->newFile( $row->img_name );
132 if ( !$file ) {
133 continue;
134 }
135
136 // Upgrade the current file version...
137 $sha1 = $file->getRepo()->getFileSha1( $file->getPath() );
138 if ( strval( $sha1 ) !== '' ) { // file on disk and hashed properly
139 if ( $isRegen && $file->getSha1() !== $sha1 ) {
140 // The population was probably done already. If the old SHA1
141 // does not match, then both fix the SHA1 and the metadata.
142 $file->upgradeRow();
143 } else {
144 $sql = "UPDATE $imageTable SET img_sha1=" . $dbw->addQuotes( $sha1 ) .
145 " WHERE img_name=" . $dbw->addQuotes( $file->getName() );
146 if ( $method == 'pipe' ) {
147 fwrite( $pipe, "$sql;\n" );
148 } else {
149 $dbw->query( $sql, __METHOD__ );
150 }
151 }
152 }
153 // Upgrade the old file versions...
154 foreach ( $file->getHistory() as $oldFile ) {
156 '@phan-var OldLocalFile $oldFile';
157 $sha1 = $oldFile->getRepo()->getFileSha1( $oldFile->getPath() );
158 if ( strval( $sha1 ) !== '' ) { // file on disk and hashed properly
159 if ( $isRegen && $oldFile->getSha1() !== $sha1 ) {
160 // The population was probably done already. If the old SHA1
161 // does not match, then both fix the SHA1 and the metadata.
162 $oldFile->upgradeRow();
163 } else {
164 $sql = "UPDATE $oldImageTable SET oi_sha1=" . $dbw->addQuotes( $sha1 ) .
165 " WHERE (oi_name=" . $dbw->addQuotes( $oldFile->getName() ) . " AND" .
166 " oi_archive_name=" . $dbw->addQuotes( $oldFile->getArchiveName() ) . ")";
167 if ( $method == 'pipe' ) {
168 fwrite( $pipe, "$sql;\n" );
169 } else {
170 $dbw->query( $sql, __METHOD__ );
171 }
172 }
173 }
174 }
175 $i++;
176 }
177 if ( $method == 'pipe' ) {
178 fflush( $pipe );
179 pclose( $pipe );
180 }
181 $t += microtime( true );
182 $this->output( sprintf( "\nDone %d files in %.1f seconds\n", $numRows, $t ) );
183
184 return !$file; // we only updated *some* files, don't log
185 }
186}
187
188$maintClass = PopulateImageSha1::class;
189require_once RUN_MAINTENANCE_IF_MAIN;
getDB()
Class for scripts that perform database maintenance and want to log the update in updatelog so we can...
output( $out, $channel=null)
Throw some output to the user.
Config null $config
Accessible via getConfig()
hasOption( $name)
Checks to see if a particular option was set.
getBatchSize()
Returns batch size.
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
MediaWikiServices is the service locator for the application scope of MediaWiki.
Executes shell commands.
Definition Shell.php:45
Maintenance script to populate the img_sha1 field.
execute()
Do the actual work.
doDBUpdates()
Do the actual work.
__construct()
Default constructor.
getUpdateKey()
Get the update key name to go in the update log table.
updateSkippedMessage()
Message to show that the update was done already and was just skipped.
get( $name)
Get a configuration variable such as "Sitename" or "UploadMaintenance.".
const DB_PRIMARY
Definition defines.php:27
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition router.php:42