MediaWiki  master
migrateFileRepoLayout.php
Go to the documentation of this file.
1 <?php
25 
26 require_once __DIR__ . '/Maintenance.php';
27 
36  public function __construct() {
37  parent::__construct();
38  $this->addDescription( 'Copy files in repo to a different layout.' );
39  $this->addOption( 'oldlayout', "Old layout; one of 'name' or 'sha1'", true, true );
40  $this->addOption( 'newlayout', "New layout; one of 'name' or 'sha1'", true, true );
41  $this->addOption( 'since', "Copy only files from after this timestamp", false, true );
42  $this->setBatchSize( 50 );
43  }
44 
45  public function execute() {
46  $oldLayout = $this->getOption( 'oldlayout' );
47  if ( !in_array( $oldLayout, [ 'name', 'sha1' ] ) ) {
48  $this->fatalError( "Invalid old layout." );
49  }
50  $newLayout = $this->getOption( 'newlayout' );
51  if ( !in_array( $newLayout, [ 'name', 'sha1' ] ) ) {
52  $this->fatalError( "Invalid new layout." );
53  }
54  $since = $this->getOption( 'since' );
55 
56  $repo = $this->getRepo();
57 
58  $be = $repo->getBackend();
59  if ( $be instanceof FileBackendDBRepoWrapper ) {
60  // avoid path translations for this script
61  $be = $be->getInternalBackend();
62  }
63 
64  $dbw = $repo->getMasterDB();
65 
66  $origBase = $be->getContainerStoragePath( "{$repo->getName()}-original" );
67  $startTime = wfTimestampNow();
68 
69  // Do current and archived versions...
70  $conds = [];
71  if ( $since ) {
72  $conds[] = 'img_timestamp >= ' . $dbw->addQuotes( $dbw->timestamp( $since ) );
73  }
74 
75  $batchSize = $this->getBatchSize();
76  $batch = [];
77  $lastName = '';
78  do {
79  $res = $dbw->select( 'image',
80  [ 'img_name', 'img_sha1' ],
81  array_merge( [ 'img_name > ' . $dbw->addQuotes( $lastName ) ], $conds ),
82  __METHOD__,
83  [ 'LIMIT' => $batchSize, 'ORDER BY' => 'img_name' ]
84  );
85 
86  foreach ( $res as $row ) {
87  $lastName = $row->img_name;
89  $file = $repo->newFile( $row->img_name );
90  // Check in case SHA1 rows are not populated for some files
91  $sha1 = strlen( $row->img_sha1 ) ? $row->img_sha1 : $file->getSha1();
92 
93  if ( !strlen( $sha1 ) ) {
94  $this->error( "Image SHA-1 not known for {$row->img_name}." );
95  } else {
96  if ( $oldLayout === 'sha1' ) {
97  $spath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
98  } else {
99  $spath = $file->getPath();
100  }
101 
102  if ( $newLayout === 'sha1' ) {
103  $dpath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
104  } else {
105  $dpath = $file->getPath();
106  }
107 
108  $status = $be->prepare( [
109  'dir' => dirname( $dpath ), 'bypassReadOnly' => 1 ] );
110  if ( !$status->isOK() ) {
111  $this->error( print_r( $status->getErrors(), true ) );
112  }
113 
114  $batch[] = [ 'op' => 'copy', 'overwrite' => true,
115  'src' => $spath, 'dst' => $dpath, 'img' => $row->img_name ];
116  }
117 
118  foreach ( $file->getHistory() as $ofile ) {
119  $sha1 = $ofile->getSha1();
120  if ( !strlen( $sha1 ) ) {
121  $this->error( "Image SHA-1 not set for {$ofile->getArchiveName()}." );
122  continue;
123  }
124 
125  if ( $oldLayout === 'sha1' ) {
126  $spath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
127  } elseif ( $ofile->isDeleted( File::DELETED_FILE ) ) {
128  $spath = $be->getContainerStoragePath( "{$repo->getName()}-deleted" ) .
129  '/' . $repo->getDeletedHashPath( $sha1 ) .
130  $sha1 . '.' . $ofile->getExtension();
131  } else {
132  $spath = $ofile->getPath();
133  }
134 
135  if ( $newLayout === 'sha1' ) {
136  $dpath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
137  } else {
138  $dpath = $ofile->getPath();
139  }
140 
141  $status = $be->prepare( [
142  'dir' => dirname( $dpath ), 'bypassReadOnly' => 1 ] );
143  if ( !$status->isOK() ) {
144  $this->error( print_r( $status->getErrors(), true ) );
145  }
146  $batch[] = [ 'op' => 'copy', 'overwrite' => true,
147  'src' => $spath, 'dst' => $dpath, 'img' => $ofile->getArchiveName() ];
148  }
149 
150  if ( count( $batch ) >= $batchSize ) {
151  $this->runBatch( $batch, $be );
152  $batch = [];
153  }
154  }
155  } while ( $res->numRows() );
156 
157  if ( count( $batch ) ) {
158  $this->runBatch( $batch, $be );
159  }
160 
161  // Do deleted versions...
162  $conds = [];
163  if ( $since ) {
164  $conds[] = 'fa_deleted_timestamp >= ' . $dbw->addQuotes( $dbw->timestamp( $since ) );
165  }
166 
167  $batch = [];
168  $lastId = 0;
169  do {
170  $res = $dbw->select( 'filearchive', [ 'fa_storage_key', 'fa_id', 'fa_name' ],
171  array_merge( [ 'fa_id > ' . $dbw->addQuotes( $lastId ) ], $conds ),
172  __METHOD__,
173  [ 'LIMIT' => $batchSize, 'ORDER BY' => 'fa_id' ]
174  );
175 
176  foreach ( $res as $row ) {
177  $lastId = $row->fa_id;
178  $sha1Key = $row->fa_storage_key;
179  if ( !strlen( $sha1Key ) ) {
180  $this->error( "Image SHA-1 not set for file #{$row->fa_id} (deleted)." );
181  continue;
182  }
183  $sha1 = substr( $sha1Key, 0, strpos( $sha1Key, '.' ) );
184 
185  if ( $oldLayout === 'sha1' ) {
186  $spath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
187  } else {
188  $spath = $be->getContainerStoragePath( "{$repo->getName()}-deleted" ) .
189  '/' . $repo->getDeletedHashPath( $sha1Key ) . $sha1Key;
190  }
191 
192  if ( $newLayout === 'sha1' ) {
193  $dpath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
194  } else {
195  $dpath = $be->getContainerStoragePath( "{$repo->getName()}-deleted" ) .
196  '/' . $repo->getDeletedHashPath( $sha1Key ) . $sha1Key;
197  }
198 
199  $status = $be->prepare( [
200  'dir' => dirname( $dpath ), 'bypassReadOnly' => 1 ] );
201  if ( !$status->isOK() ) {
202  $this->error( print_r( $status->getErrors(), true ) );
203  }
204 
205  $batch[] = [ 'op' => 'copy', 'src' => $spath, 'dst' => $dpath,
206  'overwriteSame' => true, 'img' => "(ID {$row->fa_id}) {$row->fa_name}" ];
207 
208  if ( count( $batch ) >= $batchSize ) {
209  $this->runBatch( $batch, $be );
210  $batch = [];
211  }
212  }
213  } while ( $res->numRows() );
214 
215  if ( count( $batch ) ) {
216  $this->runBatch( $batch, $be );
217  }
218 
219  $this->output( "Done (started $startTime)\n" );
220  }
221 
222  protected function getRepo() {
223  return MediaWikiServices::getInstance()->getRepoGroup()->getLocalRepo();
224  }
225 
230  protected function runBatch( array $ops, FileBackend $be ) {
231  $this->output( "Migrating file batch:\n" );
232  foreach ( $ops as $op ) {
233  $this->output( "\"{$op['img']}\" (dest: {$op['dst']})\n" );
234  }
235 
236  $status = $be->doOperations( $ops, [ 'bypassReadOnly' => 1 ] );
237  if ( !$status->isOK() ) {
238  $this->output( print_r( $status->getErrors(), true ) );
239  }
240 
241  $this->output( "Batch done\n\n" );
242  }
243 }
244 
245 $maintClass = MigrateFileRepoLayout::class;
246 require_once RUN_MAINTENANCE_IF_MAIN;
RUN_MAINTENANCE_IF_MAIN
const RUN_MAINTENANCE_IF_MAIN
Definition: Maintenance.php:38
FileBackend\doOperations
doOperations(array $ops, array $opts=[])
This is the main entry point into the backend for write operations.
Definition: FileBackend.php:468
FileBackend
Base class for all file backend classes (including multi-write backends).
Definition: FileBackend.php:98
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:154
Maintenance\fatalError
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
Definition: Maintenance.php:487
Maintenance\addDescription
addDescription( $text)
Set the description text.
Definition: Maintenance.php:327
$file
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition: router.php:42
Maintenance
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:55
$res
$res
Definition: testCompression.php:57
MigrateFileRepoLayout\getRepo
getRepo()
Definition: migrateFileRepoLayout.php:222
Maintenance\addOption
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
Definition: Maintenance.php:245
wfTimestampNow
wfTimestampNow()
Convenience function; returns MediaWiki timestamp for the present time.
Definition: GlobalFunctions.php:1837
MigrateFileRepoLayout\__construct
__construct()
Default constructor.
Definition: migrateFileRepoLayout.php:36
MigrateFileRepoLayout\runBatch
runBatch(array $ops, FileBackend $be)
Definition: migrateFileRepoLayout.php:230
MigrateFileRepoLayout\execute
execute()
Do the actual work.
Definition: migrateFileRepoLayout.php:45
Maintenance\getOption
getOption( $name, $default=null)
Get an option, or return the default.
Definition: Maintenance.php:281
Maintenance\getBatchSize
getBatchSize()
Returns batch size.
Definition: Maintenance.php:366
Maintenance\error
error( $err, $die=0)
Throw an error to the user.
Definition: Maintenance.php:463
Maintenance\output
output( $out, $channel=null)
Throw some output to the user.
Definition: Maintenance.php:434
File\DELETED_FILE
const DELETED_FILE
Definition: File.php:67
$maintClass
$maintClass
Definition: migrateFileRepoLayout.php:245
MigrateFileRepoLayout
Copy all files in FileRepo to an originals container using SHA1 paths.
Definition: migrateFileRepoLayout.php:35
FileBackendDBRepoWrapper
Proxy backend that manages file layout rewriting for FileRepo.
Definition: FileBackendDBRepoWrapper.php:41
Maintenance\setBatchSize
setBatchSize( $s=0)
Set the batch size.
Definition: Maintenance.php:374