MediaWiki REL1_35
migrateFileRepoLayout.php
Go to the documentation of this file.
1<?php
25
26require_once __DIR__ . '/Maintenance.php';
27
36 public function __construct() {
37 parent::__construct();
38 $this->addDescription( 'Copy files in repo to a different layout.' );
39 $this->addOption( 'oldlayout', "Old layout; one of 'name' or 'sha1'", true, true );
40 $this->addOption( 'newlayout', "New layout; one of 'name' or 'sha1'", true, true );
41 $this->addOption( 'since', "Copy only files from after this timestamp", false, true );
42 $this->setBatchSize( 50 );
43 }
44
45 public function execute() {
46 $oldLayout = $this->getOption( 'oldlayout' );
47 if ( !in_array( $oldLayout, [ 'name', 'sha1' ] ) ) {
48 $this->fatalError( "Invalid old layout." );
49 }
50 $newLayout = $this->getOption( 'newlayout' );
51 if ( !in_array( $newLayout, [ 'name', 'sha1' ] ) ) {
52 $this->fatalError( "Invalid new layout." );
53 }
54 $since = $this->getOption( 'since' );
55
56 $repo = $this->getRepo();
57
58 $be = $repo->getBackend();
59 if ( $be instanceof FileBackendDBRepoWrapper ) {
60 // avoid path translations for this script
61 $be = $be->getInternalBackend();
62 }
63
64 $dbw = $repo->getMasterDB();
65
66 $origBase = $be->getContainerStoragePath( "{$repo->getName()}-original" );
67 $startTime = wfTimestampNow();
68
69 // Do current and archived versions...
70 $conds = [];
71 if ( $since ) {
72 $conds[] = 'img_timestamp >= ' . $dbw->addQuotes( $dbw->timestamp( $since ) );
73 }
74
75 $batchSize = $this->getBatchSize();
76 $batch = [];
77 $lastName = '';
78 do {
79 $res = $dbw->select( 'image',
80 [ 'img_name', 'img_sha1' ],
81 array_merge( [ 'img_name > ' . $dbw->addQuotes( $lastName ) ], $conds ),
82 __METHOD__,
83 [ 'LIMIT' => $batchSize, 'ORDER BY' => 'img_name' ]
84 );
85
86 foreach ( $res as $row ) {
87 $lastName = $row->img_name;
89 $file = $repo->newFile( $row->img_name );
90 // Check in case SHA1 rows are not populated for some files
91 $sha1 = strlen( $row->img_sha1 ) ? $row->img_sha1 : $file->getSha1();
92
93 if ( !strlen( $sha1 ) ) {
94 $this->error( "Image SHA-1 not known for {$row->img_name}." );
95 } else {
96 if ( $oldLayout === 'sha1' ) {
97 $spath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
98 } else {
99 $spath = $file->getPath();
100 }
101
102 if ( $newLayout === 'sha1' ) {
103 $dpath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
104 } else {
105 $dpath = $file->getPath();
106 }
107
108 $status = $be->prepare( [
109 'dir' => dirname( $dpath ), 'bypassReadOnly' => 1 ] );
110 if ( !$status->isOK() ) {
111 $this->error( print_r( $status->getErrors(), true ) );
112 }
113
114 $batch[] = [ 'op' => 'copy', 'overwrite' => true,
115 'src' => $spath, 'dst' => $dpath, 'img' => $row->img_name ];
116 }
117
118 foreach ( $file->getHistory() as $ofile ) {
119 $sha1 = $ofile->getSha1();
120 if ( !strlen( $sha1 ) ) {
121 $this->error( "Image SHA-1 not set for {$ofile->getArchiveName()}." );
122 continue;
123 }
124
125 if ( $oldLayout === 'sha1' ) {
126 $spath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
127 } elseif ( $ofile->isDeleted( File::DELETED_FILE ) ) {
128 $spath = $be->getContainerStoragePath( "{$repo->getName()}-deleted" ) .
129 '/' . $repo->getDeletedHashPath( $sha1 ) .
130 $sha1 . '.' . $ofile->getExtension();
131 } else {
132 $spath = $ofile->getPath();
133 }
134
135 if ( $newLayout === 'sha1' ) {
136 $dpath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
137 } else {
138 $dpath = $ofile->getPath();
139 }
140
141 $status = $be->prepare( [
142 'dir' => dirname( $dpath ), 'bypassReadOnly' => 1 ] );
143 if ( !$status->isOK() ) {
144 $this->error( print_r( $status->getErrors(), true ) );
145 }
146 $batch[] = [ 'op' => 'copy', 'overwrite' => true,
147 'src' => $spath, 'dst' => $dpath, 'img' => $ofile->getArchiveName() ];
148 }
149
150 if ( count( $batch ) >= $batchSize ) {
151 $this->runBatch( $batch, $be );
152 $batch = [];
153 }
154 }
155 } while ( $res->numRows() );
156
157 if ( count( $batch ) ) {
158 $this->runBatch( $batch, $be );
159 }
160
161 // Do deleted versions...
162 $conds = [];
163 if ( $since ) {
164 $conds[] = 'fa_deleted_timestamp >= ' . $dbw->addQuotes( $dbw->timestamp( $since ) );
165 }
166
167 $batch = [];
168 $lastId = 0;
169 do {
170 $res = $dbw->select( 'filearchive', [ 'fa_storage_key', 'fa_id', 'fa_name' ],
171 array_merge( [ 'fa_id > ' . $dbw->addQuotes( $lastId ) ], $conds ),
172 __METHOD__,
173 [ 'LIMIT' => $batchSize, 'ORDER BY' => 'fa_id' ]
174 );
175
176 foreach ( $res as $row ) {
177 $lastId = $row->fa_id;
178 $sha1Key = $row->fa_storage_key;
179 if ( !strlen( $sha1Key ) ) {
180 $this->error( "Image SHA-1 not set for file #{$row->fa_id} (deleted)." );
181 continue;
182 }
183 $sha1 = substr( $sha1Key, 0, strpos( $sha1Key, '.' ) );
184
185 if ( $oldLayout === 'sha1' ) {
186 $spath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
187 } else {
188 $spath = $be->getContainerStoragePath( "{$repo->getName()}-deleted" ) .
189 '/' . $repo->getDeletedHashPath( $sha1Key ) . $sha1Key;
190 }
191
192 if ( $newLayout === 'sha1' ) {
193 $dpath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
194 } else {
195 $dpath = $be->getContainerStoragePath( "{$repo->getName()}-deleted" ) .
196 '/' . $repo->getDeletedHashPath( $sha1Key ) . $sha1Key;
197 }
198
199 $status = $be->prepare( [
200 'dir' => dirname( $dpath ), 'bypassReadOnly' => 1 ] );
201 if ( !$status->isOK() ) {
202 $this->error( print_r( $status->getErrors(), true ) );
203 }
204
205 $batch[] = [ 'op' => 'copy', 'src' => $spath, 'dst' => $dpath,
206 'overwriteSame' => true, 'img' => "(ID {$row->fa_id}) {$row->fa_name}" ];
207
208 if ( count( $batch ) >= $batchSize ) {
209 $this->runBatch( $batch, $be );
210 $batch = [];
211 }
212 }
213 } while ( $res->numRows() );
214
215 if ( count( $batch ) ) {
216 $this->runBatch( $batch, $be );
217 }
218
219 $this->output( "Done (started $startTime)\n" );
220 }
221
222 protected function getRepo() {
223 return MediaWikiServices::getInstance()->getRepoGroup()->getLocalRepo();
224 }
225
230 protected function runBatch( array $ops, FileBackend $be ) {
231 $this->output( "Migrating file batch:\n" );
232 foreach ( $ops as $op ) {
233 $this->output( "\"{$op['img']}\" (dest: {$op['dst']})\n" );
234 }
235
236 $status = $be->doOperations( $ops, [ 'bypassReadOnly' => 1 ] );
237 if ( !$status->isOK() ) {
238 $this->output( print_r( $status->getErrors(), true ) );
239 }
240
241 $this->output( "Batch done\n\n" );
242 }
243}
244
245$maintClass = MigrateFileRepoLayout::class;
246require_once RUN_MAINTENANCE_IF_MAIN;
wfTimestampNow()
Convenience function; returns MediaWiki timestamp for the present time.
const RUN_MAINTENANCE_IF_MAIN
Proxy backend that manages file layout rewriting for FileRepo.
Base class for all file backend classes (including multi-write backends).
doOperations(array $ops, array $opts=[])
This is the main entry point into the backend for write operations.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
error( $err, $die=0)
Throw an error to the user.
output( $out, $channel=null)
Throw some output to the user.
getBatchSize()
Returns batch size.
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
setBatchSize( $s=0)
Set the batch size.
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
MediaWikiServices is the service locator for the application scope of MediaWiki.
Copy all files in FileRepo to an originals container using SHA1 paths.
runBatch(array $ops, FileBackend $be)
__construct()
Default constructor.
execute()
Do the actual work.
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition router.php:42