MediaWiki  1.28.0
copyFileBackend.php
Go to the documentation of this file.
1 <?php
24 require_once __DIR__ . '/Maintenance.php';
25 
39  protected $statCache = null;
40 
41  public function __construct() {
42  parent::__construct();
43  $this->addDescription( 'Copy files in one backend to another.' );
44  $this->addOption( 'src', 'Backend containing the source files', true, true );
45  $this->addOption( 'dst', 'Backend where files should be copied to', true, true );
46  $this->addOption( 'containers', 'Pipe separated list of containers', true, true );
47  $this->addOption( 'subdir', 'Only do items in this child directory', false, true );
48  $this->addOption( 'ratefile', 'File to check periodically for batch size', false, true );
49  $this->addOption( 'prestat', 'Stat the destination files first (try to use listings)' );
50  $this->addOption( 'skiphash', 'Skip SHA-1 sync checks for files' );
51  $this->addOption( 'missingonly', 'Only copy files missing from destination listing' );
52  $this->addOption( 'syncviadelete', 'Delete destination files missing from source listing' );
53  $this->addOption( 'utf8only', 'Skip source files that do not have valid UTF-8 names' );
54  $this->setBatchSize( 50 );
55  }
56 
57  public function execute() {
58  $src = FileBackendGroup::singleton()->get( $this->getOption( 'src' ) );
59  $dst = FileBackendGroup::singleton()->get( $this->getOption( 'dst' ) );
60  $containers = explode( '|', $this->getOption( 'containers' ) );
61  $subDir = rtrim( $this->getOption( 'subdir', '' ), '/' );
62 
63  $rateFile = $this->getOption( 'ratefile' );
64 
65  foreach ( $containers as $container ) {
66  if ( $subDir != '' ) {
67  $backendRel = "$container/$subDir";
68  $this->output( "Doing container '$container', directory '$subDir'...\n" );
69  } else {
70  $backendRel = $container;
71  $this->output( "Doing container '$container'...\n" );
72  }
73 
74  if ( $this->hasOption( 'missingonly' ) ) {
75  $this->output( "\tBuilding list of missing files..." );
76  $srcPathsRel = $this->getListingDiffRel( $src, $dst, $backendRel );
77  $this->output( count( $srcPathsRel ) . " file(s) need to be copied.\n" );
78  } else {
79  $srcPathsRel = $src->getFileList( [
80  'dir' => $src->getRootStoragePath() . "/$backendRel",
81  'adviseStat' => true // avoid HEADs
82  ] );
83  if ( $srcPathsRel === null ) {
84  $this->error( "Could not list files in $container.", 1 ); // die
85  }
86  }
87 
88  if ( $this->getOption( 'prestat' ) && !$this->hasOption( 'missingonly' ) ) {
89  // Build the stat cache for the destination files
90  $this->output( "\tBuilding destination stat cache..." );
91  $dstPathsRel = $dst->getFileList( [
92  'dir' => $dst->getRootStoragePath() . "/$backendRel",
93  'adviseStat' => true // avoid HEADs
94  ] );
95  if ( $dstPathsRel === null ) {
96  $this->error( "Could not list files in $container.", 1 ); // die
97  }
98  $this->statCache = [];
99  foreach ( $dstPathsRel as $dstPathRel ) {
100  $path = $dst->getRootStoragePath() . "/$backendRel/$dstPathRel";
101  $this->statCache[sha1( $path )] = $dst->getFileStat( [ 'src' => $path ] );
102  }
103  $this->output( "done [" . count( $this->statCache ) . " file(s)]\n" );
104  }
105 
106  $this->output( "\tCopying file(s)...\n" );
107  $count = 0;
108  $batchPaths = [];
109  foreach ( $srcPathsRel as $srcPathRel ) {
110  // Check up on the rate file periodically to adjust the concurrency
111  if ( $rateFile && ( !$count || ( $count % 500 ) == 0 ) ) {
112  $this->mBatchSize = max( 1, (int)file_get_contents( $rateFile ) );
113  $this->output( "\tBatch size is now {$this->mBatchSize}.\n" );
114  }
115  $batchPaths[$srcPathRel] = 1; // remove duplicates
116  if ( count( $batchPaths ) >= $this->mBatchSize ) {
117  $this->copyFileBatch( array_keys( $batchPaths ), $backendRel, $src, $dst );
118  $batchPaths = []; // done
119  }
120  ++$count;
121  }
122  if ( count( $batchPaths ) ) { // left-overs
123  $this->copyFileBatch( array_keys( $batchPaths ), $backendRel, $src, $dst );
124  $batchPaths = []; // done
125  }
126  $this->output( "\tCopied $count file(s).\n" );
127 
128  if ( $this->hasOption( 'syncviadelete' ) ) {
129  $this->output( "\tBuilding list of excess destination files..." );
130  $delPathsRel = $this->getListingDiffRel( $dst, $src, $backendRel );
131  $this->output( count( $delPathsRel ) . " file(s) need to be deleted.\n" );
132 
133  $this->output( "\tDeleting file(s)...\n" );
134  $count = 0;
135  $batchPaths = [];
136  foreach ( $delPathsRel as $delPathRel ) {
137  // Check up on the rate file periodically to adjust the concurrency
138  if ( $rateFile && ( !$count || ( $count % 500 ) == 0 ) ) {
139  $this->mBatchSize = max( 1, (int)file_get_contents( $rateFile ) );
140  $this->output( "\tBatch size is now {$this->mBatchSize}.\n" );
141  }
142  $batchPaths[$delPathRel] = 1; // remove duplicates
143  if ( count( $batchPaths ) >= $this->mBatchSize ) {
144  $this->delFileBatch( array_keys( $batchPaths ), $backendRel, $dst );
145  $batchPaths = []; // done
146  }
147  ++$count;
148  }
149  if ( count( $batchPaths ) ) { // left-overs
150  $this->delFileBatch( array_keys( $batchPaths ), $backendRel, $dst );
151  $batchPaths = []; // done
152  }
153 
154  $this->output( "\tDeleted $count file(s).\n" );
155  }
156 
157  if ( $subDir != '' ) {
158  $this->output( "Finished container '$container', directory '$subDir'.\n" );
159  } else {
160  $this->output( "Finished container '$container'.\n" );
161  }
162  }
163 
164  $this->output( "Done.\n" );
165  }
166 
173  protected function getListingDiffRel( FileBackend $src, FileBackend $dst, $backendRel ) {
174  $srcPathsRel = $src->getFileList( [
175  'dir' => $src->getRootStoragePath() . "/$backendRel" ] );
176  if ( $srcPathsRel === null ) {
177  $this->error( "Could not list files in source container.", 1 ); // die
178  }
179  $dstPathsRel = $dst->getFileList( [
180  'dir' => $dst->getRootStoragePath() . "/$backendRel" ] );
181  if ( $dstPathsRel === null ) {
182  $this->error( "Could not list files in destination container.", 1 ); // die
183  }
184  // Get the list of destination files
185  $relFilesDstSha1 = [];
186  foreach ( $dstPathsRel as $dstPathRel ) {
187  $relFilesDstSha1[sha1( $dstPathRel )] = 1;
188  }
189  unset( $dstPathsRel ); // free
190  // Get the list of missing files
191  $missingPathsRel = [];
192  foreach ( $srcPathsRel as $srcPathRel ) {
193  if ( !isset( $relFilesDstSha1[sha1( $srcPathRel )] ) ) {
194  $missingPathsRel[] = $srcPathRel;
195  }
196  }
197  unset( $srcPathsRel ); // free
198 
199  return $missingPathsRel;
200  }
201 
209  protected function copyFileBatch(
210  array $srcPathsRel, $backendRel, FileBackend $src, FileBackend $dst
211  ) {
212  $ops = [];
213  $fsFiles = [];
214  $copiedRel = []; // for output message
215  $wikiId = $src->getWikiId();
216 
217  // Download the batch of source files into backend cache...
218  if ( $this->hasOption( 'missingonly' ) ) {
219  $srcPaths = [];
220  foreach ( $srcPathsRel as $srcPathRel ) {
221  $srcPaths[] = $src->getRootStoragePath() . "/$backendRel/$srcPathRel";
222  }
223  $t_start = microtime( true );
224  $fsFiles = $src->getLocalReferenceMulti( [ 'srcs' => $srcPaths, 'latest' => 1 ] );
225  $elapsed_ms = floor( ( microtime( true ) - $t_start ) * 1000 );
226  $this->output( "\n\tDownloaded these file(s) [{$elapsed_ms}ms]:\n\t" .
227  implode( "\n\t", $srcPaths ) . "\n\n" );
228  }
229 
230  // Determine what files need to be copied over...
231  foreach ( $srcPathsRel as $srcPathRel ) {
232  $srcPath = $src->getRootStoragePath() . "/$backendRel/$srcPathRel";
233  $dstPath = $dst->getRootStoragePath() . "/$backendRel/$srcPathRel";
234  if ( $this->hasOption( 'utf8only' ) && !mb_check_encoding( $srcPath, 'UTF-8' ) ) {
235  $this->error( "$wikiId: Detected illegal (non-UTF8) path for $srcPath." );
236  continue;
237  } elseif ( !$this->hasOption( 'missingonly' )
238  && $this->filesAreSame( $src, $dst, $srcPath, $dstPath )
239  ) {
240  $this->output( "\tAlready have $srcPathRel.\n" );
241  continue; // assume already copied...
242  }
243  $fsFile = array_key_exists( $srcPath, $fsFiles )
244  ? $fsFiles[$srcPath]
245  : $src->getLocalReference( [ 'src' => $srcPath, 'latest' => 1 ] );
246  if ( !$fsFile ) {
247  $src->clearCache( [ $srcPath ] );
248  if ( $src->fileExists( [ 'src' => $srcPath, 'latest' => 1 ] ) === false ) {
249  $this->error( "$wikiId: File '$srcPath' was listed but does not exist." );
250  } else {
251  $this->error( "$wikiId: Could not get local copy of $srcPath." );
252  }
253  continue;
254  } elseif ( !$fsFile->exists() ) {
255  // FSFileBackends just return the path for getLocalReference() and paths with
256  // illegal slashes may get normalized to a different path. This can cause the
257  // local reference to not exist...skip these broken files.
258  $this->error( "$wikiId: Detected possible illegal path for $srcPath." );
259  continue;
260  }
261  $fsFiles[] = $fsFile; // keep TempFSFile objects alive as needed
262  // Note: prepare() is usually fast for key/value backends
263  $status = $dst->prepare( [ 'dir' => dirname( $dstPath ), 'bypassReadOnly' => 1 ] );
264  if ( !$status->isOK() ) {
265  $this->error( print_r( $status->getErrorsArray(), true ) );
266  $this->error( "$wikiId: Could not copy $srcPath to $dstPath.", 1 ); // die
267  }
268  $ops[] = [ 'op' => 'store',
269  'src' => $fsFile->getPath(), 'dst' => $dstPath, 'overwrite' => 1 ];
270  $copiedRel[] = $srcPathRel;
271  }
272 
273  // Copy in the batch of source files...
274  $t_start = microtime( true );
275  $status = $dst->doQuickOperations( $ops, [ 'bypassReadOnly' => 1 ] );
276  if ( !$status->isOK() ) {
277  sleep( 10 ); // wait and retry copy again
278  $status = $dst->doQuickOperations( $ops, [ 'bypassReadOnly' => 1 ] );
279  }
280  $elapsed_ms = floor( ( microtime( true ) - $t_start ) * 1000 );
281  if ( !$status->isOK() ) {
282  $this->error( print_r( $status->getErrorsArray(), true ) );
283  $this->error( "$wikiId: Could not copy file batch.", 1 ); // die
284  } elseif ( count( $copiedRel ) ) {
285  $this->output( "\n\tCopied these file(s) [{$elapsed_ms}ms]:\n\t" .
286  implode( "\n\t", $copiedRel ) . "\n\n" );
287  }
288  }
289 
296  protected function delFileBatch(
297  array $dstPathsRel, $backendRel, FileBackend $dst
298  ) {
299  $ops = [];
300  $deletedRel = []; // for output message
301  $wikiId = $dst->getWikiId();
302 
303  // Determine what files need to be copied over...
304  foreach ( $dstPathsRel as $dstPathRel ) {
305  $dstPath = $dst->getRootStoragePath() . "/$backendRel/$dstPathRel";
306  $ops[] = [ 'op' => 'delete', 'src' => $dstPath ];
307  $deletedRel[] = $dstPathRel;
308  }
309 
310  // Delete the batch of source files...
311  $t_start = microtime( true );
312  $status = $dst->doQuickOperations( $ops, [ 'bypassReadOnly' => 1 ] );
313  if ( !$status->isOK() ) {
314  sleep( 10 ); // wait and retry copy again
315  $status = $dst->doQuickOperations( $ops, [ 'bypassReadOnly' => 1 ] );
316  }
317  $elapsed_ms = floor( ( microtime( true ) - $t_start ) * 1000 );
318  if ( !$status->isOK() ) {
319  $this->error( print_r( $status->getErrorsArray(), true ) );
320  $this->error( "$wikiId: Could not delete file batch.", 1 ); // die
321  } elseif ( count( $deletedRel ) ) {
322  $this->output( "\n\tDeleted these file(s) [{$elapsed_ms}ms]:\n\t" .
323  implode( "\n\t", $deletedRel ) . "\n\n" );
324  }
325  }
326 
334  protected function filesAreSame( FileBackend $src, FileBackend $dst, $sPath, $dPath ) {
335  $skipHash = $this->hasOption( 'skiphash' );
336  $srcStat = $src->getFileStat( [ 'src' => $sPath ] );
337  $dPathSha1 = sha1( $dPath );
338  if ( $this->statCache !== null ) {
339  // All dst files are already in stat cache
340  $dstStat = isset( $this->statCache[$dPathSha1] )
341  ? $this->statCache[$dPathSha1]
342  : false;
343  } else {
344  $dstStat = $dst->getFileStat( [ 'src' => $dPath ] );
345  }
346  // Initial fast checks to see if files are obviously different
347  $sameFast = (
348  is_array( $srcStat ) // sanity check that source exists
349  && is_array( $dstStat ) // dest exists
350  && $srcStat['size'] === $dstStat['size']
351  );
352  // More thorough checks against files
353  if ( !$sameFast ) {
354  $same = false; // no need to look farther
355  } elseif ( isset( $srcStat['md5'] ) && isset( $dstStat['md5'] ) ) {
356  // If MD5 was already in the stat info, just use it.
357  // This is useful as many objects stores can return this in object listing,
358  // so we can use it to avoid slow per-file HEADs.
359  $same = ( $srcStat['md5'] === $dstStat['md5'] );
360  } elseif ( $skipHash ) {
361  // This mode is good for copying to a backup location or resyncing clone
362  // backends in FileBackendMultiWrite (since they get writes second, they have
363  // higher timestamps). However, when copying the other way, this hits loads of
364  // false positives (possibly 100%) and wastes a bunch of time on GETs/PUTs.
365  $same = ( $srcStat['mtime'] <= $dstStat['mtime'] );
366  } else {
367  // This is the slowest method which does many per-file HEADs (unless an object
368  // store tracks SHA-1 in listings).
369  $same = ( $src->getFileSha1Base36( [ 'src' => $sPath, 'latest' => 1 ] )
370  === $dst->getFileSha1Base36( [ 'src' => $dPath, 'latest' => 1 ] ) );
371  }
372 
373  return $same;
374  }
375 }
376 
377 $maintClass = 'CopyFileBackend';
378 require_once RUN_MAINTENANCE_IF_MAIN;
Copy all files in one container of one backend to another.
the array() calling protocol came about after MediaWiki 1.4rc1.
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
getWikiId()
Alias to getDomainId()
processing should stop and the error should be shown to the user * false
Definition: hooks.txt:189
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: maintenance.txt:39
hasOption($name)
Checks to see if a particular param exists.
require_once RUN_MAINTENANCE_IF_MAIN
Definition: maintenance.txt:50
getLocalReferenceMulti(array $params)
Like getLocalReference() except it takes an array of storage paths and returns a map of storage paths...
null means default in associative array with keys and values unescaped Should be merged with default with a value of false meaning to suppress the attribute in associative array with keys and values unescaped noclasses just before the function returns a value If you return true
Definition: hooks.txt:1936
addOption($name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
fileExists(array $params)
Check if a file exists at a storage path in the backend.
array null $statCache
(path sha1 => stat) Pre-computed dst stat entries from listings
getFileSha1Base36(array $params)
Get a SHA-1 hash of the file at a storage path in the backend.
addDescription($text)
Set the description text.
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
doQuickOperations(array $ops, array $opts=[])
Perform a set of independent file operations on some files.
prepare(array $params)
Prepare a storage directory for usage.
getListingDiffRel(FileBackend $src, FileBackend $dst, $backendRel)
getOption($name, $default=null)
Get an option, or return the default.
output($out, $channel=null)
Throw some output to the user.
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
getRootStoragePath()
Get the root storage path of this backend.
error($err, $die=0)
Throw an error to the user.
Base class for all file backend classes (including multi-write backends).
Definition: FileBackend.php:93
getFileList(array $params)
Get an iterator to list all stored files under a storage directory.
this hook is for auditing only RecentChangesLinked and Watchlist RecentChangesLinked and Watchlist e g Watchlist removed from all revisions and log entries to which it was applied This gives extensions a chance to take it off their books as the deletion has already been partly carried out by this point or something similar the user will be unable to create the tag set $status
Definition: hooks.txt:1046
filesAreSame(FileBackend $src, FileBackend $dst, $sPath, $dPath)
$count
$maintClass
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
setBatchSize($s=0)
Set the batch size.
copyFileBatch(array $srcPathsRel, $backendRel, FileBackend $src, FileBackend $dst)
delFileBatch(array $dstPathsRel, $backendRel, FileBackend $dst)
getLocalReference(array $params)
Returns a file system file, identical to the file at a storage path.