MediaWiki  master
migrateFileRepoLayout.php
Go to the documentation of this file.
1 <?php
24 require_once __DIR__ . '/Maintenance.php';
25 
34  public function __construct() {
35  parent::__construct();
36  $this->addDescription( 'Copy files in repo to a different layout.' );
37  $this->addOption( 'oldlayout', "Old layout; one of 'name' or 'sha1'", true, true );
38  $this->addOption( 'newlayout', "New layout; one of 'name' or 'sha1'", true, true );
39  $this->addOption( 'since', "Copy only files from after this timestamp", false, true );
40  $this->setBatchSize( 50 );
41  }
42 
43  public function execute() {
44  $oldLayout = $this->getOption( 'oldlayout' );
45  if ( !in_array( $oldLayout, [ 'name', 'sha1' ] ) ) {
46  $this->fatalError( "Invalid old layout." );
47  }
48  $newLayout = $this->getOption( 'newlayout' );
49  if ( !in_array( $newLayout, [ 'name', 'sha1' ] ) ) {
50  $this->fatalError( "Invalid new layout." );
51  }
52  $since = $this->getOption( 'since' );
53 
54  $repo = $this->getRepo();
55 
56  $be = $repo->getBackend();
57  if ( $be instanceof FileBackendDBRepoWrapper ) {
58  $be = $be->getInternalBackend(); // avoid path translations for this script
59  }
60 
61  $dbw = $repo->getMasterDB();
62 
63  $origBase = $be->getContainerStoragePath( "{$repo->getName()}-original" );
64  $startTime = wfTimestampNow();
65 
66  // Do current and archived versions...
67  $conds = [];
68  if ( $since ) {
69  $conds[] = 'img_timestamp >= ' . $dbw->addQuotes( $dbw->timestamp( $since ) );
70  }
71 
72  $batchSize = $this->getBatchSize();
73  $batch = [];
74  $lastName = '';
75  do {
76  $res = $dbw->select( 'image',
77  [ 'img_name', 'img_sha1' ],
78  array_merge( [ 'img_name > ' . $dbw->addQuotes( $lastName ) ], $conds ),
79  __METHOD__,
80  [ 'LIMIT' => $batchSize, 'ORDER BY' => 'img_name' ]
81  );
82 
83  foreach ( $res as $row ) {
84  $lastName = $row->img_name;
86  $file = $repo->newFile( $row->img_name );
87  // Check in case SHA1 rows are not populated for some files
88  $sha1 = strlen( $row->img_sha1 ) ? $row->img_sha1 : $file->getSha1();
89 
90  if ( !strlen( $sha1 ) ) {
91  $this->error( "Image SHA-1 not known for {$row->img_name}." );
92  } else {
93  if ( $oldLayout === 'sha1' ) {
94  $spath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
95  } else {
96  $spath = $file->getPath();
97  }
98 
99  if ( $newLayout === 'sha1' ) {
100  $dpath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
101  } else {
102  $dpath = $file->getPath();
103  }
104 
105  $status = $be->prepare( [
106  'dir' => dirname( $dpath ), 'bypassReadOnly' => 1 ] );
107  if ( !$status->isOK() ) {
108  $this->error( print_r( $status->getErrors(), true ) );
109  }
110 
111  $batch[] = [ 'op' => 'copy', 'overwrite' => true,
112  'src' => $spath, 'dst' => $dpath, 'img' => $row->img_name ];
113  }
114 
115  foreach ( $file->getHistory() as $ofile ) {
116  $sha1 = $ofile->getSha1();
117  if ( !strlen( $sha1 ) ) {
118  $this->error( "Image SHA-1 not set for {$ofile->getArchiveName()}." );
119  continue;
120  }
121 
122  if ( $oldLayout === 'sha1' ) {
123  $spath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
124  } elseif ( $ofile->isDeleted( File::DELETED_FILE ) ) {
125  $spath = $be->getContainerStoragePath( "{$repo->getName()}-deleted" ) .
126  '/' . $repo->getDeletedHashPath( $sha1 ) .
127  $sha1 . '.' . $ofile->getExtension();
128  } else {
129  $spath = $ofile->getPath();
130  }
131 
132  if ( $newLayout === 'sha1' ) {
133  $dpath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
134  } else {
135  $dpath = $ofile->getPath();
136  }
137 
138  $status = $be->prepare( [
139  'dir' => dirname( $dpath ), 'bypassReadOnly' => 1 ] );
140  if ( !$status->isOK() ) {
141  $this->error( print_r( $status->getErrors(), true ) );
142  }
143  $batch[] = [ 'op' => 'copy', 'overwrite' => true,
144  'src' => $spath, 'dst' => $dpath, 'img' => $ofile->getArchiveName() ];
145  }
146 
147  if ( count( $batch ) >= $batchSize ) {
148  $this->runBatch( $batch, $be );
149  $batch = [];
150  }
151  }
152  } while ( $res->numRows() );
153 
154  if ( count( $batch ) ) {
155  $this->runBatch( $batch, $be );
156  }
157 
158  // Do deleted versions...
159  $conds = [];
160  if ( $since ) {
161  $conds[] = 'fa_deleted_timestamp >= ' . $dbw->addQuotes( $dbw->timestamp( $since ) );
162  }
163 
164  $batch = [];
165  $lastId = 0;
166  do {
167  $res = $dbw->select( 'filearchive', [ 'fa_storage_key', 'fa_id', 'fa_name' ],
168  array_merge( [ 'fa_id > ' . $dbw->addQuotes( $lastId ) ], $conds ),
169  __METHOD__,
170  [ 'LIMIT' => $batchSize, 'ORDER BY' => 'fa_id' ]
171  );
172 
173  foreach ( $res as $row ) {
174  $lastId = $row->fa_id;
175  $sha1Key = $row->fa_storage_key;
176  if ( !strlen( $sha1Key ) ) {
177  $this->error( "Image SHA-1 not set for file #{$row->fa_id} (deleted)." );
178  continue;
179  }
180  $sha1 = substr( $sha1Key, 0, strpos( $sha1Key, '.' ) );
181 
182  if ( $oldLayout === 'sha1' ) {
183  $spath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
184  } else {
185  $spath = $be->getContainerStoragePath( "{$repo->getName()}-deleted" ) .
186  '/' . $repo->getDeletedHashPath( $sha1Key ) . $sha1Key;
187  }
188 
189  if ( $newLayout === 'sha1' ) {
190  $dpath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
191  } else {
192  $dpath = $be->getContainerStoragePath( "{$repo->getName()}-deleted" ) .
193  '/' . $repo->getDeletedHashPath( $sha1Key ) . $sha1Key;
194  }
195 
196  $status = $be->prepare( [
197  'dir' => dirname( $dpath ), 'bypassReadOnly' => 1 ] );
198  if ( !$status->isOK() ) {
199  $this->error( print_r( $status->getErrors(), true ) );
200  }
201 
202  $batch[] = [ 'op' => 'copy', 'src' => $spath, 'dst' => $dpath,
203  'overwriteSame' => true, 'img' => "(ID {$row->fa_id}) {$row->fa_name}" ];
204 
205  if ( count( $batch ) >= $batchSize ) {
206  $this->runBatch( $batch, $be );
207  $batch = [];
208  }
209  }
210  } while ( $res->numRows() );
211 
212  if ( count( $batch ) ) {
213  $this->runBatch( $batch, $be );
214  }
215 
216  $this->output( "Done (started $startTime)\n" );
217  }
218 
219  protected function getRepo() {
220  return RepoGroup::singleton()->getLocalRepo();
221  }
222 
227  protected function runBatch( array $ops, FileBackend $be ) {
228  $this->output( "Migrating file batch:\n" );
229  foreach ( $ops as $op ) {
230  $this->output( "\"{$op['img']}\" (dest: {$op['dst']})\n" );
231  }
232 
233  $status = $be->doOperations( $ops, [ 'bypassReadOnly' => 1 ] );
234  if ( !$status->isOK() ) {
235  $this->output( print_r( $status->getErrors(), true ) );
236  }
237 
238  $this->output( "Batch done\n\n" );
239  }
240 }
241 
242 $maintClass = MigrateFileRepoLayout::class;
243 require_once RUN_MAINTENANCE_IF_MAIN;
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition: router.php:42
const RUN_MAINTENANCE_IF_MAIN
Definition: Maintenance.php:39
doOperations(array $ops, array $opts=[])
This is the main entry point into the backend for write operations.
error( $err, $die=0)
Throw an error to the user.
getOption( $name, $default=null)
Get an option, or return the default.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:86
setBatchSize( $s=0)
Set the batch size.
const DELETED_FILE
Definition: File.php:63
runBatch(array $ops, FileBackend $be)
Copy all files in FileRepo to an originals container using SHA1 paths.
static singleton()
Definition: RepoGroup.php:60
addDescription( $text)
Set the description text.
wfTimestampNow()
Convenience function; returns MediaWiki timestamp for the present time.
output( $out, $channel=null)
Throw some output to the user.
getBatchSize()
Returns batch size.
Base class for all file backend classes (including multi-write backends).
Definition: FileBackend.php:96
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
return true
Definition: router.php:92
Proxy backend that manages file layout rewriting for FileRepo.