24 require_once __DIR__ .
'/Maintenance.php';
42 parent::__construct();
44 $this->
addOption(
'src',
'Backend containing the source files',
true,
true );
45 $this->
addOption(
'dst',
'Backend where files should be copied to',
true,
true );
46 $this->
addOption(
'containers',
'Pipe separated list of containers',
true,
true );
47 $this->
addOption(
'subdir',
'Only do items in this child directory',
false,
true );
48 $this->
addOption(
'ratefile',
'File to check periodically for batch size',
false,
true );
49 $this->
addOption(
'prestat',
'Stat the destination files first (try to use listings)' );
50 $this->
addOption(
'skiphash',
'Skip SHA-1 sync checks for files' );
51 $this->
addOption(
'missingonly',
'Only copy files missing from destination listing' );
52 $this->
addOption(
'syncviadelete',
'Delete destination files missing from source listing' );
53 $this->
addOption(
'utf8only',
'Skip source files that do not have valid UTF-8 names' );
60 $containers = explode(
'|', $this->
getOption(
'containers' ) );
61 $subDir = rtrim( $this->
getOption(
'subdir',
'' ),
'/' );
63 $rateFile = $this->
getOption(
'ratefile' );
65 foreach ( $containers
as $container ) {
66 if ( $subDir !=
'' ) {
67 $backendRel =
"$container/$subDir";
68 $this->
output(
"Doing container '$container', directory '$subDir'...\n" );
70 $backendRel = $container;
71 $this->
output(
"Doing container '$container'...\n" );
74 if ( $this->
hasOption(
'missingonly' ) ) {
75 $this->
output(
"\tBuilding list of missing files..." );
77 $this->
output(
count( $srcPathsRel ) .
" file(s) need to be copied.\n" );
79 $srcPathsRel = $src->getFileList( [
80 'dir' => $src->getRootStoragePath() .
"/$backendRel",
83 if ( $srcPathsRel ===
null ) {
84 $this->
error(
"Could not list files in $container.", 1 );
90 $this->
output(
"\tBuilding destination stat cache..." );
91 $dstPathsRel = $dst->getFileList( [
92 'dir' => $dst->getRootStoragePath() .
"/$backendRel",
95 if ( $dstPathsRel ===
null ) {
96 $this->
error(
"Could not list files in $container.", 1 );
98 $this->statCache = [];
99 foreach ( $dstPathsRel
as $dstPathRel ) {
100 $path = $dst->getRootStoragePath() .
"/$backendRel/$dstPathRel";
101 $this->statCache[sha1(
$path )] = $dst->getFileStat( [
'src' =>
$path ] );
103 $this->
output(
"done [" .
count( $this->statCache ) .
" file(s)]\n" );
106 $this->
output(
"\tCopying file(s)...\n" );
109 foreach ( $srcPathsRel
as $srcPathRel ) {
111 if ( $rateFile && ( !$count || ( $count % 500 ) == 0 ) ) {
112 $this->mBatchSize = max( 1, (
int)file_get_contents( $rateFile ) );
113 $this->
output(
"\tBatch size is now {$this->mBatchSize}.\n" );
115 $batchPaths[$srcPathRel] = 1;
116 if (
count( $batchPaths ) >= $this->mBatchSize ) {
117 $this->
copyFileBatch( array_keys( $batchPaths ), $backendRel, $src, $dst );
122 if (
count( $batchPaths ) ) {
123 $this->
copyFileBatch( array_keys( $batchPaths ), $backendRel, $src, $dst );
126 $this->
output(
"\tCopied $count file(s).\n" );
128 if ( $this->
hasOption(
'syncviadelete' ) ) {
129 $this->
output(
"\tBuilding list of excess destination files..." );
131 $this->
output(
count( $delPathsRel ) .
" file(s) need to be deleted.\n" );
133 $this->
output(
"\tDeleting file(s)...\n" );
136 foreach ( $delPathsRel
as $delPathRel ) {
138 if ( $rateFile && ( !$count || ( $count % 500 ) == 0 ) ) {
139 $this->mBatchSize = max( 1, (
int)file_get_contents( $rateFile ) );
140 $this->
output(
"\tBatch size is now {$this->mBatchSize}.\n" );
142 $batchPaths[$delPathRel] = 1;
143 if (
count( $batchPaths ) >= $this->mBatchSize ) {
144 $this->
delFileBatch( array_keys( $batchPaths ), $backendRel, $dst );
149 if (
count( $batchPaths ) ) {
150 $this->
delFileBatch( array_keys( $batchPaths ), $backendRel, $dst );
154 $this->
output(
"\tDeleted $count file(s).\n" );
157 if ( $subDir !=
'' ) {
158 $this->
output(
"Finished container '$container', directory '$subDir'.\n" );
160 $this->
output(
"Finished container '$container'.\n" );
164 $this->
output(
"Done.\n" );
176 if ( $srcPathsRel ===
null ) {
177 $this->
error(
"Could not list files in source container.", 1 );
181 if ( $dstPathsRel ===
null ) {
182 $this->
error(
"Could not list files in destination container.", 1 );
185 $relFilesDstSha1 = [];
186 foreach ( $dstPathsRel
as $dstPathRel ) {
187 $relFilesDstSha1[sha1( $dstPathRel )] = 1;
189 unset( $dstPathsRel );
191 $missingPathsRel = [];
192 foreach ( $srcPathsRel
as $srcPathRel ) {
193 if ( !isset( $relFilesDstSha1[sha1( $srcPathRel )] ) ) {
194 $missingPathsRel[] = $srcPathRel;
197 unset( $srcPathsRel );
199 return $missingPathsRel;
218 if ( $this->
hasOption(
'missingonly' ) ) {
220 foreach ( $srcPathsRel
as $srcPathRel ) {
223 $t_start = microtime(
true );
225 $elapsed_ms = floor( ( microtime(
true ) - $t_start ) * 1000 );
226 $this->
output(
"\n\tDownloaded these file(s) [{$elapsed_ms}ms]:\n\t" .
227 implode(
"\n\t", $srcPaths ) .
"\n\n" );
231 foreach ( $srcPathsRel
as $srcPathRel ) {
234 if ( $this->
hasOption(
'utf8only' ) && !mb_check_encoding( $srcPath,
'UTF-8' ) ) {
235 $this->
error(
"$wikiId: Detected illegal (non-UTF8) path for $srcPath." );
237 } elseif ( !$this->
hasOption(
'missingonly' )
238 && $this->
filesAreSame( $src, $dst, $srcPath, $dstPath )
240 $this->
output(
"\tAlready have $srcPathRel.\n" );
243 $fsFile = array_key_exists( $srcPath, $fsFiles )
248 if ( $src->
fileExists( [
'src' => $srcPath,
'latest' => 1 ] ) ===
false ) {
249 $this->
error(
"$wikiId: File '$srcPath' was listed but does not exist." );
251 $this->
error(
"$wikiId: Could not get local copy of $srcPath." );
254 } elseif ( !$fsFile->exists() ) {
258 $this->
error(
"$wikiId: Detected possible illegal path for $srcPath." );
261 $fsFiles[] = $fsFile;
263 $status = $dst->
prepare( [
'dir' => dirname( $dstPath ),
'bypassReadOnly' => 1 ] );
266 $this->
error(
"$wikiId: Could not copy $srcPath to $dstPath.", 1 );
268 $ops[] = [
'op' =>
'store',
269 'src' => $fsFile->getPath(),
'dst' => $dstPath,
'overwrite' => 1 ];
270 $copiedRel[] = $srcPathRel;
274 $t_start = microtime(
true );
280 $elapsed_ms = floor( ( microtime(
true ) - $t_start ) * 1000 );
283 $this->
error(
"$wikiId: Could not copy file batch.", 1 );
284 } elseif (
count( $copiedRel ) ) {
285 $this->
output(
"\n\tCopied these file(s) [{$elapsed_ms}ms]:\n\t" .
286 implode(
"\n\t", $copiedRel ) .
"\n\n" );
304 foreach ( $dstPathsRel
as $dstPathRel ) {
306 $ops[] = [
'op' =>
'delete',
'src' => $dstPath ];
307 $deletedRel[] = $dstPathRel;
311 $t_start = microtime(
true );
317 $elapsed_ms = floor( ( microtime(
true ) - $t_start ) * 1000 );
320 $this->
error(
"$wikiId: Could not delete file batch.", 1 );
321 } elseif (
count( $deletedRel ) ) {
322 $this->
output(
"\n\tDeleted these file(s) [{$elapsed_ms}ms]:\n\t" .
323 implode(
"\n\t", $deletedRel ) .
"\n\n" );
335 $skipHash = $this->
hasOption(
'skiphash' );
336 $srcStat = $src->
getFileStat( [
'src' => $sPath ] );
337 $dPathSha1 = sha1( $dPath );
338 if ( $this->statCache !==
null ) {
340 $dstStat = isset( $this->statCache[$dPathSha1] )
341 ? $this->statCache[$dPathSha1]
344 $dstStat = $dst->
getFileStat( [
'src' => $dPath ] );
349 && is_array( $dstStat )
350 && $srcStat[
'size'] === $dstStat[
'size']
355 } elseif ( isset( $srcStat[
'md5'] ) && isset( $dstStat[
'md5'] ) ) {
359 $same = ( $srcStat[
'md5'] === $dstStat[
'md5'] );
360 } elseif ( $skipHash ) {
365 $same = ( $srcStat[
'mtime'] <= $dstStat[
'mtime'] );