24 require_once __DIR__ .
'/Maintenance.php';
42 parent::__construct();
44 $this->
addOption(
'src',
'Backend containing the source files',
true,
true );
45 $this->
addOption(
'dst',
'Backend where files should be copied to',
true,
true );
46 $this->
addOption(
'containers',
'Pipe separated list of containers',
true,
true );
47 $this->
addOption(
'subdir',
'Only do items in this child directory',
false,
true );
48 $this->
addOption(
'ratefile',
'File to check periodically for batch size',
false,
true );
49 $this->
addOption(
'prestat',
'Stat the destination files first (try to use listings)' );
50 $this->
addOption(
'skiphash',
'Skip SHA-1 sync checks for files' );
51 $this->
addOption(
'missingonly',
'Only copy files missing from destination listing' );
52 $this->
addOption(
'syncviadelete',
'Delete destination files missing from source listing' );
53 $this->
addOption(
'utf8only',
'Skip source files that do not have valid UTF-8 names' );
60 $containers = explode(
'|', $this->
getOption(
'containers' ) );
61 $subDir = rtrim( $this->
getOption(
'subdir',
'' ),
'/' );
63 $rateFile = $this->
getOption(
'ratefile' );
65 if ( $this->
hasOption(
'utf8only' ) && !extension_loaded(
'mbstring' ) ) {
66 $this->
error(
"Cannot check for UTF-8, mbstring extension missing.", 1 );
69 foreach ( $containers
as $container ) {
70 if ( $subDir !=
'' ) {
71 $backendRel =
"$container/$subDir";
72 $this->
output(
"Doing container '$container', directory '$subDir'...\n" );
74 $backendRel = $container;
75 $this->
output(
"Doing container '$container'...\n" );
78 if ( $this->
hasOption(
'missingonly' ) ) {
79 $this->
output(
"\tBuilding list of missing files..." );
81 $this->
output( count( $srcPathsRel ) .
" file(s) need to be copied.\n" );
83 $srcPathsRel = $src->getFileList( [
84 'dir' => $src->getRootStoragePath() .
"/$backendRel",
87 if ( $srcPathsRel === null ) {
88 $this->
error(
"Could not list files in $container.", 1 );
94 $this->
output(
"\tBuilding destination stat cache..." );
95 $dstPathsRel = $dst->getFileList( [
96 'dir' => $dst->getRootStoragePath() .
"/$backendRel",
99 if ( $dstPathsRel === null ) {
100 $this->
error(
"Could not list files in $container.", 1 );
102 $this->statCache = [];
103 foreach ( $dstPathsRel
as $dstPathRel ) {
104 $path = $dst->getRootStoragePath() .
"/$backendRel/$dstPathRel";
105 $this->statCache[sha1(
$path )] = $dst->getFileStat( [
'src' =>
$path ] );
107 $this->
output(
"done [" . count( $this->statCache ) .
" file(s)]\n" );
110 $this->
output(
"\tCopying file(s)...\n" );
113 foreach ( $srcPathsRel
as $srcPathRel ) {
115 if ( $rateFile && ( !
$count || (
$count % 500 ) == 0 ) ) {
116 $this->mBatchSize = max( 1, (
int)file_get_contents( $rateFile ) );
117 $this->
output(
"\tBatch size is now {$this->mBatchSize}.\n" );
119 $batchPaths[$srcPathRel] = 1;
120 if ( count( $batchPaths ) >= $this->mBatchSize ) {
121 $this->
copyFileBatch( array_keys( $batchPaths ), $backendRel, $src, $dst );
126 if ( count( $batchPaths ) ) {
127 $this->
copyFileBatch( array_keys( $batchPaths ), $backendRel, $src, $dst );
130 $this->
output(
"\tCopied $count file(s).\n" );
132 if ( $this->
hasOption(
'syncviadelete' ) ) {
133 $this->
output(
"\tBuilding list of excess destination files..." );
135 $this->
output( count( $delPathsRel ) .
" file(s) need to be deleted.\n" );
137 $this->
output(
"\tDeleting file(s)...\n" );
140 foreach ( $delPathsRel
as $delPathRel ) {
142 if ( $rateFile && ( !
$count || (
$count % 500 ) == 0 ) ) {
143 $this->mBatchSize = max( 1, (
int)file_get_contents( $rateFile ) );
144 $this->
output(
"\tBatch size is now {$this->mBatchSize}.\n" );
146 $batchPaths[$delPathRel] = 1;
147 if ( count( $batchPaths ) >= $this->mBatchSize ) {
148 $this->
delFileBatch( array_keys( $batchPaths ), $backendRel, $dst );
153 if ( count( $batchPaths ) ) {
154 $this->
delFileBatch( array_keys( $batchPaths ), $backendRel, $dst );
158 $this->
output(
"\tDeleted $count file(s).\n" );
161 if ( $subDir !=
'' ) {
162 $this->
output(
"Finished container '$container', directory '$subDir'.\n" );
164 $this->
output(
"Finished container '$container'.\n" );
168 $this->
output(
"Done.\n" );
180 if ( $srcPathsRel === null ) {
181 $this->
error(
"Could not list files in source container.", 1 );
185 if ( $dstPathsRel === null ) {
186 $this->
error(
"Could not list files in destination container.", 1 );
189 $relFilesDstSha1 = [];
190 foreach ( $dstPathsRel
as $dstPathRel ) {
191 $relFilesDstSha1[sha1( $dstPathRel )] = 1;
193 unset( $dstPathsRel );
195 $missingPathsRel = [];
196 foreach ( $srcPathsRel
as $srcPathRel ) {
197 if ( !isset( $relFilesDstSha1[sha1( $srcPathRel )] ) ) {
198 $missingPathsRel[] = $srcPathRel;
201 unset( $srcPathsRel );
203 return $missingPathsRel;
222 if ( $this->
hasOption(
'missingonly' ) ) {
224 foreach ( $srcPathsRel
as $srcPathRel ) {
227 $t_start = microtime(
true );
229 $elapsed_ms = floor( ( microtime(
true ) - $t_start ) * 1000 );
230 $this->
output(
"\n\tDownloaded these file(s) [{$elapsed_ms}ms]:\n\t" .
231 implode(
"\n\t", $srcPaths ) .
"\n\n" );
235 foreach ( $srcPathsRel
as $srcPathRel ) {
238 if ( $this->
hasOption(
'utf8only' ) && !mb_check_encoding( $srcPath,
'UTF-8' ) ) {
239 $this->
error(
"$wikiId: Detected illegal (non-UTF8) path for $srcPath." );
241 } elseif ( !$this->
hasOption(
'missingonly' )
242 && $this->
filesAreSame( $src, $dst, $srcPath, $dstPath )
244 $this->
output(
"\tAlready have $srcPathRel.\n" );
247 $fsFile = array_key_exists( $srcPath, $fsFiles )
252 if ( $src->
fileExists( [
'src' => $srcPath,
'latest' => 1 ] ) ===
false ) {
253 $this->
error(
"$wikiId: File '$srcPath' was listed but does not exist." );
255 $this->
error(
"$wikiId: Could not get local copy of $srcPath." );
258 } elseif ( !$fsFile->exists() ) {
262 $this->
error(
"$wikiId: Detected possible illegal path for $srcPath." );
265 $fsFiles[] = $fsFile;
267 $status = $dst->
prepare( [
'dir' => dirname( $dstPath ),
'bypassReadOnly' => 1 ] );
270 $this->
error(
"$wikiId: Could not copy $srcPath to $dstPath.", 1 );
272 $ops[] = [
'op' =>
'store',
273 'src' => $fsFile->getPath(),
'dst' => $dstPath,
'overwrite' => 1 ];
274 $copiedRel[] = $srcPathRel;
278 $t_start = microtime(
true );
284 $elapsed_ms = floor( ( microtime(
true ) - $t_start ) * 1000 );
287 $this->
error(
"$wikiId: Could not copy file batch.", 1 );
288 } elseif ( count( $copiedRel ) ) {
289 $this->
output(
"\n\tCopied these file(s) [{$elapsed_ms}ms]:\n\t" .
290 implode(
"\n\t", $copiedRel ) .
"\n\n" );
308 foreach ( $dstPathsRel
as $dstPathRel ) {
310 $ops[] = [
'op' =>
'delete',
'src' => $dstPath ];
311 $deletedRel[] = $dstPathRel;
315 $t_start = microtime(
true );
321 $elapsed_ms = floor( ( microtime(
true ) - $t_start ) * 1000 );
324 $this->
error(
"$wikiId: Could not delete file batch.", 1 );
325 } elseif ( count( $deletedRel ) ) {
326 $this->
output(
"\n\tDeleted these file(s) [{$elapsed_ms}ms]:\n\t" .
327 implode(
"\n\t", $deletedRel ) .
"\n\n" );
339 $skipHash = $this->
hasOption(
'skiphash' );
340 $srcStat = $src->
getFileStat( [
'src' => $sPath ] );
341 $dPathSha1 = sha1( $dPath );
342 if ( $this->statCache !== null ) {
344 $dstStat = isset( $this->statCache[$dPathSha1] )
345 ? $this->statCache[$dPathSha1]
348 $dstStat = $dst->
getFileStat( [
'src' => $dPath ] );
353 && is_array( $dstStat )
354 && $srcStat[
'size'] === $dstStat[
'size']
359 } elseif ( isset( $srcStat[
'md5'] ) && isset( $dstStat[
'md5'] ) ) {
363 $same = ( $srcStat[
'md5'] === $dstStat[
'md5'] );
364 } elseif ( $skipHash ) {
369 $same = ( $srcStat[
'mtime'] <= $dstStat[
'mtime'] );
Copy all files in one container of one backend to another.
the array() calling protocol came about after MediaWiki 1.4rc1.
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
getWikiId()
Get the wiki identifier used for this backend (possibly empty).
processing should stop and the error should be shown to the user * false
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
hasOption($name)
Checks to see if a particular param exists.
require_once RUN_MAINTENANCE_IF_MAIN
getLocalReferenceMulti(array $params)
Like getLocalReference() except it takes an array of storage paths and returns a map of storage paths...
null means default in associative array with keys and values unescaped Should be merged with default with a value of false meaning to suppress the attribute in associative array with keys and values unescaped noclasses just before the function returns a value If you return true
addOption($name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
fileExists(array $params)
Check if a file exists at a storage path in the backend.
array null $statCache
(path sha1 => stat) Pre-computed dst stat entries from listings
getFileSha1Base36(array $params)
Get a SHA-1 hash of the file at a storage path in the backend.
addDescription($text)
Set the description text.
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
doQuickOperations(array $ops, array $opts=[])
Perform a set of independent file operations on some files.
prepare(array $params)
Prepare a storage directory for usage.
getListingDiffRel(FileBackend $src, FileBackend $dst, $backendRel)
getOption($name, $default=null)
Get an option, or return the default.
output($out, $channel=null)
Throw some output to the user.
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
getRootStoragePath()
Get the root storage path of this backend.
error($err, $die=0)
Throw an error to the user.
Base class for all file backend classes (including multi-write backends).
getFileList(array $params)
Get an iterator to list all stored files under a storage directory.
this hook is for auditing only RecentChangesLinked and Watchlist RecentChangesLinked and Watchlist e g Watchlist removed from all revisions and log entries to which it was applied This gives extensions a chance to take it off their books as the deletion has already been partly carried out by this point or something similar the user will be unable to create the tag set $status
filesAreSame(FileBackend $src, FileBackend $dst, $sPath, $dPath)
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
setBatchSize($s=0)
Set the batch size.
copyFileBatch(array $srcPathsRel, $backendRel, FileBackend $src, FileBackend $dst)
delFileBatch(array $dstPathsRel, $backendRel, FileBackend $dst)
getLocalReference(array $params)
Returns a file system file, identical to the file at a storage path.