MediaWiki  1.23.1
copyFileBackend.php
Go to the documentation of this file.
1 <?php
24 require_once __DIR__ . '/Maintenance.php';
25 
39  protected $statCache = null;
40 
41  public function __construct() {
42  parent::__construct();
43  $this->mDescription = "Copy files in one backend to another.";
44  $this->addOption( 'src', 'Backend containing the source files', true, true );
45  $this->addOption( 'dst', 'Backend where files should be copied to', true, true );
46  $this->addOption( 'containers', 'Pipe separated list of containers', true, true );
47  $this->addOption( 'subdir', 'Only do items in this child directory', false, true );
48  $this->addOption( 'ratefile', 'File to check periodically for batch size', false, true );
49  $this->addOption( 'prestat', 'Stat the destination files first (try to use listings)' );
50  $this->addOption( 'skiphash', 'Skip SHA-1 sync checks for files' );
51  $this->addOption( 'missingonly', 'Only copy files missing from destination listing' );
52  $this->addOption( 'syncviadelete', 'Delete destination files missing from source listing' );
53  $this->addOption( 'utf8only', 'Skip source files that do not have valid UTF-8 names' );
54  $this->setBatchSize( 50 );
55  }
56 
57  public function execute() {
58  $src = FileBackendGroup::singleton()->get( $this->getOption( 'src' ) );
59  $dst = FileBackendGroup::singleton()->get( $this->getOption( 'dst' ) );
60  $containers = explode( '|', $this->getOption( 'containers' ) );
61  $subDir = rtrim( $this->getOption( 'subdir', '' ), '/' );
62 
63  $rateFile = $this->getOption( 'ratefile' );
64 
65  if ( $this->hasOption( 'utf8only' ) && !extension_loaded( 'mbstring' ) ) {
66  $this->error( "Cannot check for UTF-8, mbstring extension missing.", 1 ); // die
67  }
68 
69  foreach ( $containers as $container ) {
70  if ( $subDir != '' ) {
71  $backendRel = "$container/$subDir";
72  $this->output( "Doing container '$container', directory '$subDir'...\n" );
73  } else {
74  $backendRel = $container;
75  $this->output( "Doing container '$container'...\n" );
76  }
77 
78  if ( $this->hasOption( 'missingonly' ) ) {
79  $this->output( "\tBuilding list of missing files..." );
80  $srcPathsRel = $this->getListingDiffRel( $src, $dst, $backendRel );
81  $this->output( count( $srcPathsRel ) . " file(s) need to be copied.\n" );
82  } else {
83  $srcPathsRel = $src->getFileList( array(
84  'dir' => $src->getRootStoragePath() . "/$backendRel",
85  'adviseStat' => true // avoid HEADs
86  ) );
87  if ( $srcPathsRel === null ) {
88  $this->error( "Could not list files in $container.", 1 ); // die
89  }
90  }
91 
92  if ( $this->getOption( 'prestat' ) && !$this->hasOption( 'missingonly' ) ) {
93  // Build the stat cache for the destination files
94  $this->output( "\tBuilding destination stat cache..." );
95  $dstPathsRel = $dst->getFileList( array(
96  'dir' => $dst->getRootStoragePath() . "/$backendRel",
97  'adviseStat' => true // avoid HEADs
98  ) );
99  if ( $dstPathsRel === null ) {
100  $this->error( "Could not list files in $container.", 1 ); // die
101  }
102  $this->statCache = array();
103  foreach ( $dstPathsRel as $dstPathRel ) {
104  $path = $dst->getRootStoragePath() . "/$backendRel/$dstPathRel";
105  $this->statCache[sha1( $path )] = $dst->getFileStat( array( 'src' => $path ) );
106  }
107  $this->output( "done [" . count( $this->statCache ) . " file(s)]\n" );
108  }
109 
110  $this->output( "\tCopying file(s)...\n" );
111  $count = 0;
112  $batchPaths = array();
113  foreach ( $srcPathsRel as $srcPathRel ) {
114  // Check up on the rate file periodically to adjust the concurrency
115  if ( $rateFile && ( !$count || ( $count % 500 ) == 0 ) ) {
116  $this->mBatchSize = max( 1, (int)file_get_contents( $rateFile ) );
117  $this->output( "\tBatch size is now {$this->mBatchSize}.\n" );
118  }
119  $batchPaths[$srcPathRel] = 1; // remove duplicates
120  if ( count( $batchPaths ) >= $this->mBatchSize ) {
121  $this->copyFileBatch( array_keys( $batchPaths ), $backendRel, $src, $dst );
122  $batchPaths = array(); // done
123  }
124  ++$count;
125  }
126  if ( count( $batchPaths ) ) { // left-overs
127  $this->copyFileBatch( array_keys( $batchPaths ), $backendRel, $src, $dst );
128  $batchPaths = array(); // done
129  }
130  $this->output( "\tCopied $count file(s).\n" );
131 
132  if ( $this->hasOption( 'syncviadelete' ) ) {
133  $this->output( "\tBuilding list of excess destination files..." );
134  $delPathsRel = $this->getListingDiffRel( $dst, $src, $backendRel );
135  $this->output( count( $delPathsRel ) . " file(s) need to be deleted.\n" );
136 
137  $this->output( "\tDeleting file(s)...\n" );
138  $count = 0;
139  $batchPaths = array();
140  foreach ( $delPathsRel as $delPathRel ) {
141  // Check up on the rate file periodically to adjust the concurrency
142  if ( $rateFile && ( !$count || ( $count % 500 ) == 0 ) ) {
143  $this->mBatchSize = max( 1, (int)file_get_contents( $rateFile ) );
144  $this->output( "\tBatch size is now {$this->mBatchSize}.\n" );
145  }
146  $batchPaths[$delPathRel] = 1; // remove duplicates
147  if ( count( $batchPaths ) >= $this->mBatchSize ) {
148  $this->delFileBatch( array_keys( $batchPaths ), $backendRel, $dst );
149  $batchPaths = array(); // done
150  }
151  ++$count;
152  }
153  if ( count( $batchPaths ) ) { // left-overs
154  $this->delFileBatch( array_keys( $batchPaths ), $backendRel, $dst );
155  $batchPaths = array(); // done
156  }
157 
158  $this->output( "\tDeleted $count file(s).\n" );
159  }
160 
161  if ( $subDir != '' ) {
162  $this->output( "Finished container '$container', directory '$subDir'.\n" );
163  } else {
164  $this->output( "Finished container '$container'.\n" );
165  }
166  }
167 
168  $this->output( "Done.\n" );
169  }
170 
177  protected function getListingDiffRel( FileBackend $src, FileBackend $dst, $backendRel ) {
178  $srcPathsRel = $src->getFileList( array(
179  'dir' => $src->getRootStoragePath() . "/$backendRel" ) );
180  if ( $srcPathsRel === null ) {
181  $this->error( "Could not list files in source container.", 1 ); // die
182  }
183  $dstPathsRel = $dst->getFileList( array(
184  'dir' => $dst->getRootStoragePath() . "/$backendRel" ) );
185  if ( $dstPathsRel === null ) {
186  $this->error( "Could not list files in destination container.", 1 ); // die
187  }
188  // Get the list of destination files
189  $relFilesDstSha1 = array();
190  foreach ( $dstPathsRel as $dstPathRel ) {
191  $relFilesDstSha1[sha1( $dstPathRel )] = 1;
192  }
193  unset( $dstPathsRel ); // free
194  // Get the list of missing files
195  $missingPathsRel = array();
196  foreach ( $srcPathsRel as $srcPathRel ) {
197  if ( !isset( $relFilesDstSha1[sha1( $srcPathRel )] ) ) {
198  $missingPathsRel[] = $srcPathRel;
199  }
200  }
201  unset( $srcPathsRel ); // free
202 
203  return $missingPathsRel;
204  }
205 
213  protected function copyFileBatch(
214  array $srcPathsRel, $backendRel, FileBackend $src, FileBackend $dst
215  ) {
216  $ops = array();
217  $fsFiles = array();
218  $copiedRel = array(); // for output message
219  $wikiId = $src->getWikiId();
220 
221  // Download the batch of source files into backend cache...
222  if ( $this->hasOption( 'missingonly' ) ) {
223  $srcPaths = array();
224  foreach ( $srcPathsRel as $srcPathRel ) {
225  $srcPaths[] = $src->getRootStoragePath() . "/$backendRel/$srcPathRel";
226  }
227  $t_start = microtime( true );
228  $fsFiles = $src->getLocalReferenceMulti( array( 'srcs' => $srcPaths, 'latest' => 1 ) );
229  $ellapsed_ms = floor( ( microtime( true ) - $t_start ) * 1000 );
230  $this->output( "\n\tDownloaded these file(s) [{$ellapsed_ms}ms]:\n\t" .
231  implode( "\n\t", $srcPaths ) . "\n\n" );
232  }
233 
234  // Determine what files need to be copied over...
235  foreach ( $srcPathsRel as $srcPathRel ) {
236  $srcPath = $src->getRootStoragePath() . "/$backendRel/$srcPathRel";
237  $dstPath = $dst->getRootStoragePath() . "/$backendRel/$srcPathRel";
238  if ( $this->hasOption( 'utf8only' ) && !mb_check_encoding( $srcPath, 'UTF-8' ) ) {
239  $this->error( "$wikiId: Detected illegal (non-UTF8) path for $srcPath." );
240  continue;
241  } elseif ( !$this->hasOption( 'missingonly' )
242  && $this->filesAreSame( $src, $dst, $srcPath, $dstPath )
243  ) {
244  $this->output( "\tAlready have $srcPathRel.\n" );
245  continue; // assume already copied...
246  }
247  $fsFile = array_key_exists( $srcPath, $fsFiles )
248  ? $fsFiles[$srcPath]
249  : $src->getLocalReference( array( 'src' => $srcPath, 'latest' => 1 ) );
250  if ( !$fsFile ) {
251  $src->clearCache( array( $srcPath ) );
252  if ( $src->fileExists( array( 'src' => $srcPath, 'latest' => 1 ) ) === false ) {
253  $this->error( "$wikiId: File '$srcPath' was listed but does not exist." );
254  } else {
255  $this->error( "$wikiId: Could not get local copy of $srcPath." );
256  }
257  continue;
258  } elseif ( !$fsFile->exists() ) {
259  // FSFileBackends just return the path for getLocalReference() and paths with
260  // illegal slashes may get normalized to a different path. This can cause the
261  // local reference to not exist...skip these broken files.
262  $this->error( "$wikiId: Detected possible illegal path for $srcPath." );
263  continue;
264  }
265  $fsFiles[] = $fsFile; // keep TempFSFile objects alive as needed
266  // Note: prepare() is usually fast for key/value backends
267  $status = $dst->prepare( array( 'dir' => dirname( $dstPath ), 'bypassReadOnly' => 1 ) );
268  if ( !$status->isOK() ) {
269  $this->error( print_r( $status->getErrorsArray(), true ) );
270  $this->error( "$wikiId: Could not copy $srcPath to $dstPath.", 1 ); // die
271  }
272  $ops[] = array( 'op' => 'store',
273  'src' => $fsFile->getPath(), 'dst' => $dstPath, 'overwrite' => 1 );
274  $copiedRel[] = $srcPathRel;
275  }
276 
277  // Copy in the batch of source files...
278  $t_start = microtime( true );
279  $status = $dst->doQuickOperations( $ops, array( 'bypassReadOnly' => 1 ) );
280  if ( !$status->isOK() ) {
281  sleep( 10 ); // wait and retry copy again
282  $status = $dst->doQuickOperations( $ops, array( 'bypassReadOnly' => 1 ) );
283  }
284  $ellapsed_ms = floor( ( microtime( true ) - $t_start ) * 1000 );
285  if ( !$status->isOK() ) {
286  $this->error( print_r( $status->getErrorsArray(), true ) );
287  $this->error( "$wikiId: Could not copy file batch.", 1 ); // die
288  } elseif ( count( $copiedRel ) ) {
289  $this->output( "\n\tCopied these file(s) [{$ellapsed_ms}ms]:\n\t" .
290  implode( "\n\t", $copiedRel ) . "\n\n" );
291  }
292  }
293 
300  protected function delFileBatch(
301  array $dstPathsRel, $backendRel, FileBackend $dst
302  ) {
303  $ops = array();
304  $deletedRel = array(); // for output message
305  $wikiId = $dst->getWikiId();
306 
307  // Determine what files need to be copied over...
308  foreach ( $dstPathsRel as $dstPathRel ) {
309  $dstPath = $dst->getRootStoragePath() . "/$backendRel/$dstPathRel";
310  $ops[] = array( 'op' => 'delete', 'src' => $dstPath );
311  $deletedRel[] = $dstPathRel;
312  }
313 
314  // Delete the batch of source files...
315  $t_start = microtime( true );
316  $status = $dst->doQuickOperations( $ops, array( 'bypassReadOnly' => 1 ) );
317  if ( !$status->isOK() ) {
318  sleep( 10 ); // wait and retry copy again
319  $status = $dst->doQuickOperations( $ops, array( 'bypassReadOnly' => 1 ) );
320  }
321  $ellapsed_ms = floor( ( microtime( true ) - $t_start ) * 1000 );
322  if ( !$status->isOK() ) {
323  $this->error( print_r( $status->getErrorsArray(), true ) );
324  $this->error( "$wikiId: Could not delete file batch.", 1 ); // die
325  } elseif ( count( $deletedRel ) ) {
326  $this->output( "\n\tDeleted these file(s) [{$ellapsed_ms}ms]:\n\t" .
327  implode( "\n\t", $deletedRel ) . "\n\n" );
328  }
329  }
330 
338  protected function filesAreSame( FileBackend $src, FileBackend $dst, $sPath, $dPath ) {
339  $skipHash = $this->hasOption( 'skiphash' );
340  $srcStat = $src->getFileStat( array( 'src' => $sPath ) );
341  $dPathSha1 = sha1( $dPath );
342  if ( $this->statCache !== null ) {
343  // All dst files are already in stat cache
344  $dstStat = isset( $this->statCache[$dPathSha1] )
345  ? $this->statCache[$dPathSha1]
346  : false;
347  } else {
348  $dstStat = $dst->getFileStat( array( 'src' => $dPath ) );
349  }
350  // Initial fast checks to see if files are obviously different
351  $sameFast = (
352  is_array( $srcStat ) // sanity check that source exists
353  && is_array( $dstStat ) // dest exists
354  && $srcStat['size'] === $dstStat['size']
355  );
356  // More thorough checks against files
357  if ( !$sameFast ) {
358  $same = false; // no need to look farther
359  } elseif ( isset( $srcStat['md5'] ) && isset( $dstStat['md5'] ) ) {
360  // If MD5 was already in the stat info, just use it.
361  // This is useful as many objects stores can return this in object listing,
362  // so we can use it to avoid slow per-file HEADs.
363  $same = ( $srcStat['md5'] === $dstStat['md5'] );
364  } elseif ( $skipHash ) {
365  // This mode is good for copying to a backup location or resyncing clone
366  // backends in FileBackendMultiWrite (since they get writes second, they have
367  // higher timestamps). However, when copying the other way, this hits loads of
368  // false positives (possibly 100%) and wastes a bunch of time on GETs/PUTs.
369  $same = ( $srcStat['mtime'] <= $dstStat['mtime'] );
370  } else {
371  // This is the slowest method which does many per-file HEADs (unless an object
372  // store tracks SHA-1 in listings).
373  $same = ( $src->getFileSha1Base36( array( 'src' => $sPath, 'latest' => 1 ) )
374  === $dst->getFileSha1Base36( array( 'src' => $dPath, 'latest' => 1 ) ) );
375  }
376  return $same;
377  }
378 }
379 
380 $maintClass = 'CopyFileBackend';
381 require_once RUN_MAINTENANCE_IF_MAIN;
CopyFileBackend
Copy all files in one container of one backend to another.
Definition: copyFileBackend.php:37
php
skin txt MediaWiki includes four core it has been set as the default in MediaWiki since the replacing Monobook it had been been the default skin since before being replaced by Vector largely rewritten in while keeping its appearance Several legacy skins were removed in the as the burden of supporting them became too heavy to bear Those in etc for skin dependent CSS etc for skin dependent JavaScript These can also be customised on a per user by etc This feature has led to a wide variety of user styles becoming that gallery is a good place to ending in php
Definition: skin.txt:62
FileBackend
Base class for all file backend classes (including multi-write backends).
Definition: FileBackend.php:85
FileBackend\getLocalReferenceMulti
getLocalReferenceMulti(array $params)
Like getLocalReference() except it takes an array of storage paths and returns a map of storage paths...
FileBackend\getFileStat
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
Maintenance\addOption
addOption( $name, $description, $required=false, $withArg=false, $shortName=false)
Add a parameter to the script.
Definition: Maintenance.php:169
RUN_MAINTENANCE_IF_MAIN
require_once RUN_MAINTENANCE_IF_MAIN
Definition: maintenance.txt:50
Maintenance
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: maintenance.txt:39
true
null means default in associative array with keys and values unescaped Should be merged with default with a value of false meaning to suppress the attribute in associative array with keys and values unescaped noclasses just before the function returns a value If you return true
Definition: hooks.txt:1530
FileBackendGroup\singleton
static singleton()
Definition: FileBackendGroup.php:43
CopyFileBackend\delFileBatch
delFileBatch(array $dstPathsRel, $backendRel, FileBackend $dst)
Definition: copyFileBackend.php:299
FileBackend\getFileSha1Base36
getFileSha1Base36(array $params)
Get a SHA-1 hash of the file at a storage path in the backend.
array
the array() calling protocol came about after MediaWiki 1.4rc1.
List of Api Query prop modules.
CopyFileBackend\execute
execute()
Do the actual work.
Definition: copyFileBackend.php:56
false
processing should stop and the error should be shown to the user * false
Definition: hooks.txt:188
FileBackend\fileExists
fileExists(array $params)
Check if a file exists at a storage path in the backend.
CopyFileBackend\copyFileBatch
copyFileBatch(array $srcPathsRel, $backendRel, FileBackend $src, FileBackend $dst)
Definition: copyFileBackend.php:212
FileBackend\doQuickOperations
doQuickOperations(array $ops, array $opts=array())
Perform a set of independent file operations on some files.
Definition: FileBackend.php:601
FileBackend\prepare
prepare(array $params)
Prepare a storage directory for usage.
Definition: FileBackend.php:754
FileBackend\clearCache
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
FileBackend\getFileList
getFileList(array $params)
Get an iterator to list all stored files under a storage directory.
$count
$count
Definition: UtfNormalTest2.php:96
FileBackend\getRootStoragePath
getRootStoragePath()
Get the root storage path of this backend.
Definition: FileBackend.php:1299
Maintenance\getOption
getOption( $name, $default=null)
Get an option, or return the default.
Definition: Maintenance.php:191
$path
$path
Definition: NoLocalSettings.php:35
as
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
CopyFileBackend\$statCache
Array null $statCache
(path sha1 => stat) Pre-computed dst stat entries from listings *
Definition: copyFileBackend.php:38
Maintenance\error
error( $err, $die=0)
Throw an error to the user.
Definition: Maintenance.php:333
Maintenance\output
output( $out, $channel=null)
Throw some output to the user.
Definition: Maintenance.php:314
CopyFileBackend\getListingDiffRel
getListingDiffRel(FileBackend $src, FileBackend $dst, $backendRel)
Definition: copyFileBackend.php:176
FileBackend\getLocalReference
getLocalReference(array $params)
Returns a file system file, identical to the file at a storage path.
Definition: FileBackend.php:1021
CopyFileBackend\__construct
__construct()
Default constructor.
Definition: copyFileBackend.php:40
CopyFileBackend\filesAreSame
filesAreSame(FileBackend $src, FileBackend $dst, $sPath, $dPath)
Definition: copyFileBackend.php:337
Maintenance\hasOption
hasOption( $name)
Checks to see if a particular param exists.
Definition: Maintenance.php:181
FileBackend\getWikiId
getWikiId()
Get the wiki identifier used for this backend (possibly empty).
Definition: FileBackend.php:178
$maintClass
$maintClass
Definition: copyFileBackend.php:379
Maintenance\setBatchSize
setBatchSize( $s=0)
Set the batch size.
Definition: Maintenance.php:254