MediaWiki  master
populateImageSha1.php
Go to the documentation of this file.
1 <?php
26 
27 require_once __DIR__ . '/Maintenance.php';
28 
35  public function __construct() {
36  parent::__construct();
37  $this->addDescription( 'Populate the img_sha1 field' );
38  $this->addOption( 'force', "Recalculate sha1 for rows that already have a value" );
39  $this->addOption( 'multiversiononly', "Calculate only for files with several versions" );
40  $this->addOption( 'method', "Use 'pipe' to pipe to mysql command line,\n" .
41  "\t\tdefault uses Database class", false, true );
42  $this->addOption(
43  'file',
44  'Fix for a specific file, without File: namespace prefixed',
45  false,
46  true
47  );
48  }
49 
50  protected function getUpdateKey() {
51  return 'populate img_sha1';
52  }
53 
54  protected function updateSkippedMessage() {
55  return 'img_sha1 column of image table already populated.';
56  }
57 
58  public function execute() {
59  if ( $this->getOption( 'file' ) || $this->hasOption( 'multiversiononly' ) ) {
60  $this->doDBUpdates(); // skip update log checks/saves
61  } else {
63  }
64  }
65 
66  public function doDBUpdates() {
67  $method = $this->getOption( 'method', 'normal' );
68  $file = $this->getOption( 'file', '' );
69  $force = $this->getOption( 'force' );
70  $isRegen = ( $force || $file != '' ); // forced recalculation?
71 
72  $t = -microtime( true );
73  $dbw = $this->getDB( DB_MASTER );
74  if ( $file != '' ) {
75  $res = $dbw->select(
76  'image',
77  [ 'img_name' ],
78  [ 'img_name' => $file ],
79  __METHOD__
80  );
81  if ( !$res ) {
82  $this->fatalError( "No such file: $file" );
83  }
84  $this->output( "Populating img_sha1 field for specified files\n" );
85  } else {
86  if ( $this->hasOption( 'multiversiononly' ) ) {
87  $conds = [];
88  $this->output( "Populating and recalculating img_sha1 field for versioned files\n" );
89  } elseif ( $force ) {
90  $conds = [];
91  $this->output( "Populating and recalculating img_sha1 field\n" );
92  } else {
93  $conds = [ 'img_sha1' => '' ];
94  $this->output( "Populating img_sha1 field\n" );
95  }
96  if ( $this->hasOption( 'multiversiononly' ) ) {
97  $res = $dbw->select( 'oldimage',
98  [ 'img_name' => 'DISTINCT(oi_name)' ], $conds, __METHOD__ );
99  } else {
100  $res = $dbw->select( 'image', [ 'img_name' ], $conds, __METHOD__ );
101  }
102  }
103 
104  $imageTable = $dbw->tableName( 'image' );
105  $oldImageTable = $dbw->tableName( 'oldimage' );
106 
107  if ( $method == 'pipe' ) {
108  // Opening a pipe allows the SHA-1 operation to be done in parallel
109  // with the database write operation, because the writes are queued
110  // in the pipe buffer. This can improve performance by up to a
111  // factor of 2.
113  $cmd = 'mysql -u' . Shell::escape( $wgDBuser ) .
114  ' -h' . Shell::escape( $wgDBserver ) .
115  ' -p' . Shell::escape( $wgDBpassword, $wgDBname );
116  $this->output( "Using pipe method\n" );
117  $pipe = popen( $cmd, 'w' );
118  }
119 
120  $numRows = $res->numRows();
121  $i = 0;
122  foreach ( $res as $row ) {
123  if ( $i % $this->getBatchSize() == 0 ) {
124  $this->output( sprintf(
125  "Done %d of %d, %5.3f%% \r", $i, $numRows, $i / $numRows * 100 ) );
126  wfWaitForSlaves();
127  }
128 
129  $file = MediaWikiServices::getInstance()->getRepoGroup()->getLocalRepo()
130  ->newFile( $row->img_name );
131  if ( !$file ) {
132  continue;
133  }
134 
135  // Upgrade the current file version...
136  $sha1 = $file->getRepo()->getFileSha1( $file->getPath() );
137  if ( strval( $sha1 ) !== '' ) { // file on disk and hashed properly
138  if ( $isRegen && $file->getSha1() !== $sha1 ) {
139  // The population was probably done already. If the old SHA1
140  // does not match, then both fix the SHA1 and the metadata.
141  $file->upgradeRow();
142  } else {
143  $sql = "UPDATE $imageTable SET img_sha1=" . $dbw->addQuotes( $sha1 ) .
144  " WHERE img_name=" . $dbw->addQuotes( $file->getName() );
145  if ( $method == 'pipe' ) {
146  fwrite( $pipe, "$sql;\n" );
147  } else {
148  $dbw->query( $sql, __METHOD__ );
149  }
150  }
151  }
152  // Upgrade the old file versions...
153  foreach ( $file->getHistory() as $oldFile ) {
154  $sha1 = $oldFile->getRepo()->getFileSha1( $oldFile->getPath() );
155  if ( strval( $sha1 ) !== '' ) { // file on disk and hashed properly
156  if ( $isRegen && $oldFile->getSha1() !== $sha1 ) {
157  // The population was probably done already. If the old SHA1
158  // does not match, then both fix the SHA1 and the metadata.
159  $oldFile->upgradeRow();
160  } else {
161  $sql = "UPDATE $oldImageTable SET oi_sha1=" . $dbw->addQuotes( $sha1 ) .
162  " WHERE (oi_name=" . $dbw->addQuotes( $oldFile->getName() ) . " AND" .
163  " oi_archive_name=" . $dbw->addQuotes( $oldFile->getArchiveName() ) . ")";
164  if ( $method == 'pipe' ) {
165  fwrite( $pipe, "$sql;\n" );
166  } else {
167  $dbw->query( $sql, __METHOD__ );
168  }
169  }
170  }
171  }
172  $i++;
173  }
174  if ( $method == 'pipe' ) {
175  fflush( $pipe );
176  pclose( $pipe );
177  }
178  $t += microtime( true );
179  $this->output( sprintf( "\nDone %d files in %.1f seconds\n", $numRows, $t ) );
180 
181  return !$file; // we only updated *some* files, don't log
182  }
183 }
184 
186 require_once RUN_MAINTENANCE_IF_MAIN;
$batch execute()
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
getOption( $name, $default=null)
Get an option, or return the default.
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Definition: router.php:42
$wgDBpassword
Database user&#39;s password.
$wgDBserver
Database host name or IP address.
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency MediaWikiServices
Definition: injection.txt:23
$wgDBuser
Database username.
hasOption( $name)
Checks to see if a particular option exists.
require_once RUN_MAINTENANCE_IF_MAIN
Definition: maintenance.txt:50
const DB_MASTER
Definition: defines.php:26
$res
Definition: database.txt:21
wfWaitForSlaves( $ifWritesSince=null, $wiki=false, $cluster=false, $timeout=null)
Waits for the replica DBs to catch up to the master position.
addDescription( $text)
Set the description text.
getDB( $db, $groups=[], $wiki=false)
Returns a database to be used by current maintenance script.
output( $out, $channel=null)
Throw some output to the user.
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
Class for scripts that perform database maintenance and want to log the update in updatelog so we can...
you have access to all of the normal MediaWiki so you can get a DB use the etc For full docs on the Maintenance class
Definition: maintenance.txt:52
getBatchSize()
Returns batch size.
Maintenance script to populate the img_sha1 field.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
The wiki should then use memcached to cache various data To use multiple just add more items to the array To increase the weight of a make its entry a controlled by the following MediaWiki still creates a BagOStuff but calls it to it are no ops If the cache daemon can t be it should also disable itself fairly $wgDBname
Definition: memcached.txt:93
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.