24 use Wikimedia\Timestamp\ConvertibleTimestamp;
64 private const CHECK_SIZE = 1;
66 private const CHECK_TIME = 2;
68 private const CHECK_SHA1 = 4;
99 parent::__construct( $config );
100 $this->syncChecks = $config[
'syncChecks'] ?? self::CHECK_SIZE;
101 $this->autoResync = $config[
'autoResync'] ??
false;
102 $this->asyncWrites = isset( $config[
'replication'] ) && $config[
'replication'] ===
'async';
106 foreach ( $config[
'backends'] as $index => $beConfig ) {
107 $name = $beConfig[
'name'];
108 if ( isset( $namesUsed[
$name] ) ) {
109 throw new LogicException(
"Two or more backends defined with the name $name." );
111 $namesUsed[
$name] = 1;
113 unset( $beConfig[
'readOnly'] );
114 unset( $beConfig[
'lockManager'] );
117 if ( !empty( $beConfig[
'isMultiMaster'] ) ) {
118 if ( $this->masterIndex >= 0 ) {
119 throw new LogicException(
'More than one master backend defined.' );
121 $this->masterIndex = $index;
123 if ( !empty( $beConfig[
'readAffinity'] ) ) {
124 $this->readIndex = $index;
127 if ( !isset( $beConfig[
'class'] ) ) {
128 throw new InvalidArgumentException(
'No class given for a backend config.' );
130 $class = $beConfig[
'class'];
131 $this->backends[$index] =
new $class( $beConfig );
133 if ( $this->masterIndex < 0 ) {
134 throw new LogicException(
'No master backend defined.' );
136 if ( $this->readIndex < 0 ) {
149 if ( empty( $opts[
'nonLocking'] ) ) {
151 if ( !$status->isOK() ) {
159 $opts[
'preserveCache'] =
true;
162 if ( !$status->isOK() ) {
167 if ( !$syncStatus->isOK() ) {
168 $this->logger->error(
173 $this->autoResync ===
false ||
174 !$this->
resyncFiles( $relevantPaths, $this->autoResync )->isOK()
176 $status->merge( $syncStatus );
183 $masterStatus = $mbe->doOperations( $realOps, $opts );
184 $status->merge( $masterStatus );
188 if ( $masterStatus->isOK() && $masterStatus->successCount > 0 ) {
189 foreach ( $this->backends as $index => $backend ) {
190 if ( $index === $this->masterIndex ) {
199 $backend, $realOps, $opts, $scopeLock, $relevantPaths, $fname
201 $this->logger->debug(
202 "$fname: '{$backend->getName()}' async replication; paths: " .
205 $backend->doOperations( $realOps, $opts );
209 $this->logger->debug(
210 "$fname: '{$backend->getName()}' sync replication; paths: " .
213 $status->merge( $backend->doOperations( $realOps, $opts ) );
220 $status->success = $masterStatus->success;
221 $status->successCount = $masterStatus->successCount;
222 $status->failCount = $masterStatus->failCount;
238 if ( $this->syncChecks == 0 || count( $this->backends ) <= 1 ) {
243 foreach ( $this->backends as $backend ) {
244 $realPaths = $this->
substPaths( $paths, $backend );
245 $backend->preloadFileStat( [
'srcs' => $realPaths,
'latest' =>
true ] );
248 foreach ( $paths as
$path ) {
249 $params = [
'src' =>
$path,
'latest' =>
true ];
252 $masterParams = $this->
substOpPaths( $params, $masterBackend );
253 $masterStat = $masterBackend->getFileStat( $masterParams );
254 if ( $masterStat === self::STAT_ERROR ) {
255 $status->fatal(
'backend-fail-stat',
$path );
258 if ( $this->syncChecks & self::CHECK_SHA1 ) {
259 $masterSha1 = $masterBackend->getFileSha1Base36( $masterParams );
260 if ( ( $masterSha1 !==
false ) !== (
bool)$masterStat ) {
261 $status->fatal(
'backend-fail-hash',
$path );
269 foreach ( $this->backends as $index => $cloneBackend ) {
270 if ( $index === $this->masterIndex ) {
275 $cloneParams = $this->
substOpPaths( $params, $cloneBackend );
276 $cloneStat = $cloneBackend->getFileStat( $cloneParams );
282 $status->fatal(
'backend-fail-synced',
$path );
284 ( $this->syncChecks & self::CHECK_SIZE ) &&
285 $cloneStat[
'size'] !== $masterStat[
'size']
288 $status->fatal(
'backend-fail-synced',
$path );
290 ( $this->syncChecks & self::CHECK_TIME ) &&
292 (
int)ConvertibleTimestamp::convert( TS_UNIX, $masterStat[
'mtime'] ) -
293 (
int)ConvertibleTimestamp::convert( TS_UNIX, $cloneStat[
'mtime'] )
297 $status->fatal(
'backend-fail-synced',
$path );
299 ( $this->syncChecks & self::CHECK_SHA1 ) &&
300 $cloneBackend->getFileSha1Base36( $cloneParams ) !== $masterSha1
303 $status->fatal(
'backend-fail-synced',
$path );
309 $status->fatal(
'backend-fail-synced',
$path );
326 if ( count( $this->backends ) <= 1 ) {
330 foreach ( $paths as
$path ) {
331 foreach ( $this->backends as $backend ) {
332 $realPath = $this->
substPaths( $path, $backend );
333 if ( !$backend->isPathUsableInternal( $realPath ) ) {
334 $status->fatal(
'backend-fail-usable',
$path );
356 foreach ( $paths as
$path ) {
357 $params = [
'src' =>
$path,
'latest' =>
true ];
360 $masterParams = $this->
substOpPaths( $params, $masterBackend );
361 $masterPath = $masterParams[
'src'];
362 $masterStat = $masterBackend->getFileStat( $masterParams );
363 if ( $masterStat === self::STAT_ERROR ) {
364 $status->fatal(
'backend-fail-stat',
$path );
365 $this->logger->error(
"$fname: file '$masterPath' is not available" );
368 $masterSha1 = $masterBackend->getFileSha1Base36( $masterParams );
369 if ( ( $masterSha1 !==
false ) !== (
bool)$masterStat ) {
370 $status->fatal(
'backend-fail-hash',
$path );
371 $this->logger->error(
"$fname: file '$masterPath' hash does not match stat" );
376 foreach ( $this->backends as $index => $cloneBackend ) {
377 if ( $index === $this->masterIndex ) {
382 $cloneParams = $this->
substOpPaths( $params, $cloneBackend );
383 $clonePath = $cloneParams[
'src'];
384 $cloneStat = $cloneBackend->getFileStat( $cloneParams );
385 if ( $cloneStat === self::STAT_ERROR ) {
386 $status->fatal(
'backend-fail-stat',
$path );
387 $this->logger->error(
"$fname: file '$clonePath' is not available" );
390 $cloneSha1 = $cloneBackend->getFileSha1Base36( $cloneParams );
391 if ( ( $cloneSha1 !==
false ) !== (
bool)$cloneStat ) {
392 $status->fatal(
'backend-fail-hash',
$path );
393 $this->logger->error(
"$fname: file '$clonePath' hash does not match stat" );
397 if ( $masterSha1 === $cloneSha1 ) {
399 $this->logger->debug(
"$fname: file '$clonePath' matches '$masterPath'" );
400 } elseif ( $masterSha1 !==
false ) {
403 $resyncMode ===
'conservative' &&
406 $cloneStat[
'mtime'] > $masterStat[
'mtime']
409 $status->fatal(
'backend-fail-synced',
$path );
412 $fsFile = $masterBackend->getLocalReference( $masterParams );
413 $status->merge( $cloneBackend->quickStore( [
418 } elseif ( $masterStat ===
false ) {
420 if ( $resyncMode ===
'conservative' ) {
422 $status->fatal(
'backend-fail-synced',
$path );
423 $this->logger->error(
"$fname: not allowed to delete file '$clonePath'" );
426 $status->merge( $cloneBackend->quickDelete( [
'src' => $clonePath ] ) );
432 if ( !$status->isOK() ) {
447 foreach ( $ops as $op ) {
448 if ( isset( $op[
'src'] ) ) {
451 if ( empty( $op[
'ignoreMissingSource'] )
452 || $this->
fileExists( [
'src' => $op[
'src'] ] )
454 $paths[] = $op[
'src'];
457 if ( isset( $op[
'srcs'] ) ) {
458 $paths = array_merge( $paths, $op[
'srcs'] );
460 if ( isset( $op[
'dst'] ) ) {
461 $paths[] = $op[
'dst'];
465 return array_values( array_unique( array_filter( $paths, [ FileBackend::class,
'isStoragePath' ] ) ) );
478 foreach ( $ops as $op ) {
480 foreach ( [
'src',
'srcs',
'dst',
'dir' ] as $par ) {
481 if ( isset( $newOp[$par] ) ) {
482 $newOp[$par] = $this->
substPaths( $newOp[$par], $backend );
513 '!^mwstore://' . preg_quote( $this->name,
'!' ) .
'/!',
528 '!^mwstore://' . preg_quote( $backend->
getName(),
'!' ) .
'/!',
539 foreach ( $ops as $op ) {
540 if ( $op[
'op'] ===
'store' && !isset( $op[
'srcRef'] ) ) {
551 $realOps = $this->
substOpBatchPaths( $ops, $this->backends[$this->masterIndex] );
553 $status->merge( $masterStatus );
555 foreach ( $this->backends as $index => $backend ) {
556 if ( $index === $this->masterIndex ) {
563 static function () use ( $backend, $realOps ) {
564 $backend->doQuickOperations( $realOps );
568 $status->merge( $backend->doQuickOperations( $realOps ) );
574 $status->success = $masterStatus->success;
575 $status->successCount = $masterStatus->successCount;
576 $status->failCount = $masterStatus->failCount;
605 $realParams = $this->
substOpPaths( $params, $this->backends[$this->masterIndex] );
607 $status->merge( $masterStatus );
609 foreach ( $this->backends as $index => $backend ) {
610 if ( $index === $this->masterIndex ) {
615 if ( $this->asyncWrites ) {
617 static function () use ( $backend, $method, $realParams ) {
618 $backend->$method( $realParams );
622 $status->merge( $backend->$method( $realParams ) );
633 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
635 $status->merge( $this->backends[$index]->
concatenate( $realParams ) );
642 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
644 return $this->backends[$index]->fileExists( $realParams );
649 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
651 return $this->backends[$index]->getFileTimestamp( $realParams );
656 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
658 return $this->backends[$index]->getFileSize( $realParams );
663 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
665 return $this->backends[$index]->getFileStat( $realParams );
670 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
672 return $this->backends[$index]->getFileXAttributes( $realParams );
677 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
679 $contentsM = $this->backends[$index]->getFileContentsMulti( $realParams );
682 foreach ( $contentsM as
$path => $data ) {
691 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
693 return $this->backends[$index]->getFileSha1Base36( $realParams );
698 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
700 return $this->backends[$index]->getFileProps( $realParams );
705 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
707 return $this->backends[$index]->streamFile( $realParams );
712 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
714 $fsFilesM = $this->backends[$index]->getLocalReferenceMulti( $realParams );
717 foreach ( $fsFilesM as
$path => $fsFile ) {
726 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
728 $tempFilesM = $this->backends[$index]->getLocalCopyMulti( $realParams );
731 foreach ( $tempFilesM as
$path => $tempFile ) {
740 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
742 return $this->backends[$index]->getFileHttpUrl( $realParams );
746 $realParams = $this->
substOpPaths( $params, $this->backends[$this->masterIndex] );
752 $realParams = $this->
substOpPaths( $params, $this->backends[$this->masterIndex] );
758 if ( isset( $params[
'forWrite'] ) && $params[
'forWrite'] ) {
759 return $this->getFileListForWrite( $params );
762 $realParams = $this->
substOpPaths( $params, $this->backends[$this->masterIndex] );
766 private function getFileListForWrite( $params ) {
771 foreach ( $this->backends as $backend ) {
773 $iterator = $backend->getFileList( $realParams );
774 if ( $iterator !==
null ) {
775 foreach ( $iterator as
$file ) {
781 return array_unique( $files );
789 foreach ( $this->backends as $backend ) {
790 $realPaths = is_array( $paths ) ? $this->
substPaths( $paths, $backend ) :
null;
791 $backend->clearCache( $realPaths );
796 $realPaths = $this->
substPaths( $paths, $this->backends[$this->readIndex] );
802 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
804 return $this->backends[$index]->preloadFileStat( $realParams );
808 $realOps = $this->
substOpBatchPaths( $ops, $this->backends[$this->masterIndex] );
811 $paths = $this->backends[
$this->masterIndex]->getPathsToLockForOpsInternal( $fileOps );
816 $this->backends[$this->masterIndex]
820 $this->backends[$this->masterIndex]
static addCallableUpdate( $callable, $stage=self::POSTSEND, $dbw=null)
Add an update to the pending update queue that invokes the specified callback when run.
Proxy backend that mirrors writes to several internal backends.
FileBackendStore[] $backends
Prioritized list of FileBackendStore objects.
fileExists(array $params)
Check if a file exists at a storage path in the backend.
getLocalReferenceMulti(array $params)
Like getLocalReference() except it takes an array of storage paths and yields an order-preserved map ...
getLocalCopyMulti(array $params)
Like getLocalCopy() except it takes an array of storage paths and yields an order preserved-map of st...
consistencyCheck(array $paths)
Check that a set of files are consistent across all internal backends.
unsubstPaths( $paths, FileBackendStore $backend)
Substitute the backend of internal storage paths with the proxy backend's name.
getFileSha1Base36(array $params)
Get a SHA-1 hash of the content of the file at a storage path in the backend.
int $readIndex
Index of read affinity backend.
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
doQuickOperationsInternal(array $ops, array $opts)
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
getFileContentsMulti(array $params)
Like getFileContents() except it takes an array of storage paths and returns an order preserved map o...
preloadCache(array $paths)
Preload persistent file stat cache and property cache into in-process cache.
resyncFiles(array $paths, $resyncMode=true)
Check that a set of files are consistent across all internal backends and re-synchronize those files ...
getDirectoryList(array $params)
Get an iterator to list all directories under a storage directory.
__construct(array $config)
Construct a proxy backend that consists of several internal backends.
getFileList(array $params)
Get an iterator to list all stored files under a storage directory.
getReadIndexFromParams(array $params)
doOperationsInternal(array $ops, array $opts)
streamFile(array $params)
Stream the content of the file at a storage path in the backend.
int $masterIndex
Index of master backend.
accessibilityCheck(array $paths)
Check that a set of file paths are usable across all internal backends.
getFeatures()
Get the a bitfield of extra features supported by the backend medium.
hasVolatileSources(array $ops)
fileStoragePathsForOps(array $ops)
Get a list of file storage paths to read or write for a list of operations.
concatenate(array $params)
Concatenate a list of storage files into a single file system file.
substOpPaths(array $ops, FileBackendStore $backend)
Same as substOpBatchPaths() but for a single operation.
substOpBatchPaths(array $ops, FileBackendStore $backend)
Substitute the backend name in storage path parameters for a set of operations with that of a given i...
getFileProps(array $params)
Get the properties of the content of the file at a storage path in the backend.
doDirectoryOp( $method, array $params)
getScopedLocksForOps(array $ops, StatusValue $status)
Get an array of scoped locks needed for a batch of file operations.
getFileHttpUrl(array $params)
Return an HTTP URL to a given file that requires no authentication to use.
preloadFileStat(array $params)
Preload file stat information (concurrently if possible) into in-process cache.
getFileSize(array $params)
Get the size (bytes) of a file at a storage path in the backend.
getFileXAttributes(array $params)
Get metadata about a file at a storage path in the backend.
substPaths( $paths, FileBackendStore $backend)
Substitute the backend of storage paths with an internal backend's name.
directoryExists(array $params)
Check if a directory exists at a given storage path.
getFileTimestamp(array $params)
Get the last-modified timestamp of the file at a storage path.
Base class for all backends using particular storage medium.
Base class for all file backend classes (including multi-write backends).
string $name
Unique backend name.
string $domainId
Unique domain name.
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
newStatus(... $args)
Yields the result of the status wrapper callback on either:
getName()
Get the unique backend name.
Generic operation result class Has warning/error list, boolean status and arbitrary value.
static escapeRegexReplacement( $string)
Escape a string to make it suitable for inclusion in a preg_replace() replacement parameter.
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.