25use Wikimedia\Timestamp\ConvertibleTimestamp;
65 private const CHECK_SIZE = 1;
67 private const CHECK_TIME = 2;
69 private const CHECK_SHA1 = 4;
100 parent::__construct( $config );
101 $this->syncChecks = $config[
'syncChecks'] ?? self::CHECK_SIZE;
102 $this->autoResync = $config[
'autoResync'] ??
false;
103 $this->asyncWrites = isset( $config[
'replication'] ) && $config[
'replication'] ===
'async';
107 foreach ( $config[
'backends'] as $index => $beConfig ) {
108 $name = $beConfig[
'name'];
109 if ( isset( $namesUsed[
$name] ) ) {
110 throw new LogicException(
"Two or more backends defined with the name $name." );
112 $namesUsed[
$name] = 1;
114 unset( $beConfig[
'readOnly'] );
115 unset( $beConfig[
'lockManager'] );
118 if ( !empty( $beConfig[
'isMultiMaster'] ) ) {
119 if ( $this->masterIndex >= 0 ) {
120 throw new LogicException(
'More than one master backend defined.' );
122 $this->masterIndex = $index;
124 if ( !empty( $beConfig[
'readAffinity'] ) ) {
125 $this->readIndex = $index;
128 if ( !isset( $beConfig[
'class'] ) ) {
129 throw new InvalidArgumentException(
'No class given for a backend config.' );
131 $class = $beConfig[
'class'];
132 $this->backends[$index] =
new $class( $beConfig );
134 if ( $this->masterIndex < 0 ) {
135 throw new LogicException(
'No master backend defined.' );
137 if ( $this->readIndex < 0 ) {
150 if ( empty( $opts[
'nonLocking'] ) ) {
152 if ( !$status->isOK() ) {
160 $opts[
'preserveCache'] =
true;
163 if ( !$status->isOK() ) {
168 if ( !$syncStatus->isOK() ) {
169 $this->logger->error(
170 "$fname: failed sync check: " . FormatJson::encode( $relevantPaths )
174 $this->autoResync ===
false ||
175 !$this->
resyncFiles( $relevantPaths, $this->autoResync )->isOK()
177 $status->merge( $syncStatus );
184 $masterStatus = $mbe->doOperations( $realOps, $opts );
185 $status->merge( $masterStatus );
189 if ( $masterStatus->isOK() && $masterStatus->successCount > 0 ) {
190 foreach ( $this->backends as $index => $backend ) {
191 if ( $index === $this->masterIndex ) {
198 DeferredUpdates::addCallableUpdate(
200 $backend, $realOps, $opts, $scopeLock, $relevantPaths, $fname
202 $this->logger->debug(
203 "$fname: '{$backend->getName()}' async replication; paths: " .
204 FormatJson::encode( $relevantPaths )
206 $backend->doOperations( $realOps, $opts );
210 $this->logger->debug(
211 "$fname: '{$backend->getName()}' sync replication; paths: " .
212 FormatJson::encode( $relevantPaths )
214 $status->merge( $backend->doOperations( $realOps, $opts ) );
221 $status->success = $masterStatus->success;
222 $status->successCount = $masterStatus->successCount;
223 $status->failCount = $masterStatus->failCount;
239 if ( $this->syncChecks == 0 || count( $this->backends ) <= 1 ) {
244 foreach ( $this->backends as $backend ) {
245 $realPaths = $this->
substPaths( $paths, $backend );
246 $backend->preloadFileStat( [
'srcs' => $realPaths,
'latest' =>
true ] );
249 foreach ( $paths as
$path ) {
254 $masterStat = $masterBackend->getFileStat( $masterParams );
255 if ( $masterStat === self::STAT_ERROR ) {
256 $status->fatal(
'backend-fail-stat',
$path );
259 if ( $this->syncChecks & self::CHECK_SHA1 ) {
260 $masterSha1 = $masterBackend->getFileSha1Base36( $masterParams );
261 if ( ( $masterSha1 !==
false ) !== (
bool)$masterStat ) {
262 $status->fatal(
'backend-fail-hash',
$path );
270 foreach ( $this->backends as $index => $cloneBackend ) {
271 if ( $index === $this->masterIndex ) {
277 $cloneStat = $cloneBackend->getFileStat( $cloneParams );
283 $status->fatal(
'backend-fail-synced',
$path );
285 ( $this->syncChecks & self::CHECK_SIZE ) &&
286 $cloneStat[
'size'] !== $masterStat[
'size']
289 $status->fatal(
'backend-fail-synced',
$path );
291 ( $this->syncChecks & self::CHECK_TIME ) &&
293 (
int)ConvertibleTimestamp::convert( TS_UNIX, $masterStat[
'mtime'] ) -
294 (
int)ConvertibleTimestamp::convert( TS_UNIX, $cloneStat[
'mtime'] )
298 $status->fatal(
'backend-fail-synced',
$path );
300 ( $this->syncChecks & self::CHECK_SHA1 ) &&
301 $cloneBackend->getFileSha1Base36( $cloneParams ) !== $masterSha1
304 $status->fatal(
'backend-fail-synced',
$path );
310 $status->fatal(
'backend-fail-synced',
$path );
327 if ( count( $this->backends ) <= 1 ) {
331 foreach ( $paths as
$path ) {
332 foreach ( $this->backends as $backend ) {
333 $realPath = $this->
substPaths( $path, $backend );
334 if ( !$backend->isPathUsableInternal( $realPath ) ) {
335 $status->fatal(
'backend-fail-usable',
$path );
357 foreach ( $paths as
$path ) {
362 $masterPath = $masterParams[
'src'];
363 $masterStat = $masterBackend->getFileStat( $masterParams );
364 if ( $masterStat === self::STAT_ERROR ) {
365 $status->fatal(
'backend-fail-stat',
$path );
366 $this->logger->error(
"$fname: file '$masterPath' is not available" );
369 $masterSha1 = $masterBackend->getFileSha1Base36( $masterParams );
370 if ( ( $masterSha1 !==
false ) !== (
bool)$masterStat ) {
371 $status->fatal(
'backend-fail-hash',
$path );
372 $this->logger->error(
"$fname: file '$masterPath' hash does not match stat" );
377 foreach ( $this->backends as $index => $cloneBackend ) {
378 if ( $index === $this->masterIndex ) {
384 $clonePath = $cloneParams[
'src'];
385 $cloneStat = $cloneBackend->getFileStat( $cloneParams );
386 if ( $cloneStat === self::STAT_ERROR ) {
387 $status->fatal(
'backend-fail-stat',
$path );
388 $this->logger->error(
"$fname: file '$clonePath' is not available" );
391 $cloneSha1 = $cloneBackend->getFileSha1Base36( $cloneParams );
392 if ( ( $cloneSha1 !==
false ) !== (
bool)$cloneStat ) {
393 $status->fatal(
'backend-fail-hash',
$path );
394 $this->logger->error(
"$fname: file '$clonePath' hash does not match stat" );
398 if ( $masterSha1 === $cloneSha1 ) {
400 $this->logger->debug(
"$fname: file '$clonePath' matches '$masterPath'" );
401 } elseif ( $masterSha1 !==
false ) {
404 $resyncMode ===
'conservative' &&
407 $cloneStat[
'mtime'] > $masterStat[
'mtime']
410 $status->fatal(
'backend-fail-synced',
$path );
413 $fsFile = $masterBackend->getLocalReference( $masterParams );
414 $status->merge( $cloneBackend->quickStore( [
419 } elseif ( $masterStat ===
false ) {
421 if ( $resyncMode ===
'conservative' ) {
423 $status->fatal(
'backend-fail-synced',
$path );
424 $this->logger->error(
"$fname: not allowed to delete file '$clonePath'" );
427 $status->merge( $cloneBackend->quickDelete( [
'src' => $clonePath ] ) );
433 if ( !$status->isOK() ) {
434 $this->logger->error(
"$fname: failed to resync: " . FormatJson::encode( $paths ) );
448 foreach ( $ops as $op ) {
449 if ( isset( $op[
'src'] ) ) {
452 if ( empty( $op[
'ignoreMissingSource'] )
453 || $this->
fileExists( [
'src' => $op[
'src'] ] )
455 $paths[] = $op[
'src'];
458 if ( isset( $op[
'srcs'] ) ) {
459 $paths = array_merge( $paths, $op[
'srcs'] );
461 if ( isset( $op[
'dst'] ) ) {
462 $paths[] = $op[
'dst'];
466 return array_values( array_unique( array_filter( $paths, [ FileBackend::class,
'isStoragePath' ] ) ) );
479 foreach ( $ops as $op ) {
481 foreach ( [
'src',
'srcs',
'dst',
'dir' ] as $par ) {
482 if ( isset( $newOp[$par] ) ) {
483 $newOp[$par] = $this->
substPaths( $newOp[$par], $backend );
514 '!^mwstore://' . preg_quote( $this->name,
'!' ) .
'/!',
515 StringUtils::escapeRegexReplacement(
"mwstore://{$backend->getName()}/" ),
529 '!^mwstore://' . preg_quote( $backend->
getName(),
'!' ) .
'/!',
530 StringUtils::escapeRegexReplacement(
"mwstore://{$this->name}/" ),
540 foreach ( $ops as $op ) {
541 if ( $op[
'op'] ===
'store' && !isset( $op[
'srcRef'] ) ) {
552 $realOps = $this->
substOpBatchPaths( $ops, $this->backends[$this->masterIndex] );
554 $status->merge( $masterStatus );
556 foreach ( $this->backends as $index => $backend ) {
557 if ( $index === $this->masterIndex ) {
563 DeferredUpdates::addCallableUpdate(
564 static function () use ( $backend, $realOps ) {
565 $backend->doQuickOperations( $realOps );
569 $status->merge( $backend->doQuickOperations( $realOps ) );
575 $status->success = $masterStatus->success;
576 $status->successCount = $masterStatus->successCount;
577 $status->failCount = $masterStatus->failCount;
606 $realParams = $this->
substOpPaths( $params, $this->backends[$this->masterIndex] );
608 $status->merge( $masterStatus );
610 foreach ( $this->backends as $index => $backend ) {
611 if ( $index === $this->masterIndex ) {
616 if ( $this->asyncWrites ) {
617 DeferredUpdates::addCallableUpdate(
618 static function () use ( $backend, $method, $realParams ) {
619 $backend->$method( $realParams );
623 $status->merge( $backend->$method( $realParams ) );
634 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
636 $status->merge( $this->backends[$index]->
concatenate( $realParams ) );
643 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
645 return $this->backends[$index]->fileExists( $realParams );
650 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
652 return $this->backends[$index]->getFileTimestamp( $realParams );
657 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
659 return $this->backends[$index]->getFileSize( $realParams );
664 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
666 return $this->backends[$index]->getFileStat( $realParams );
671 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
673 return $this->backends[$index]->getFileXAttributes( $realParams );
678 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
680 $contentsM = $this->backends[$index]->getFileContentsMulti( $realParams );
683 foreach ( $contentsM as
$path => $data ) {
692 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
694 return $this->backends[$index]->getFileSha1Base36( $realParams );
699 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
701 return $this->backends[$index]->getFileProps( $realParams );
706 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
708 return $this->backends[$index]->streamFile( $realParams );
713 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
715 $fsFilesM = $this->backends[$index]->getLocalReferenceMulti( $realParams );
718 foreach ( $fsFilesM as
$path => $fsFile ) {
727 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
729 $tempFilesM = $this->backends[$index]->getLocalCopyMulti( $realParams );
732 foreach ( $tempFilesM as
$path => $tempFile ) {
741 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
743 return $this->backends[$index]->getFileHttpUrl( $realParams );
747 $realParams = $this->
substOpPaths( $params, $this->backends[$this->masterIndex] );
753 $realParams = $this->
substOpPaths( $params, $this->backends[$this->masterIndex] );
760 return $this->getFileListForWrite(
$params );
763 $realParams = $this->
substOpPaths( $params, $this->backends[$this->masterIndex] );
767 private function getFileListForWrite(
$params ) {
772 foreach ( $this->backends as $backend ) {
774 $iterator = $backend->getFileList( $realParams );
775 if ( $iterator !==
null ) {
776 foreach ( $iterator as $file ) {
782 return array_unique( $files );
790 foreach ( $this->backends as $backend ) {
791 $realPaths = is_array( $paths ) ? $this->
substPaths( $paths, $backend ) :
null;
792 $backend->clearCache( $realPaths );
797 $realPaths = $this->
substPaths( $paths, $this->backends[$this->readIndex] );
803 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
805 return $this->backends[$index]->preloadFileStat( $realParams );
809 $realOps = $this->
substOpBatchPaths( $ops, $this->backends[$this->masterIndex] );
812 $paths = $this->backends[
$this->masterIndex]->getPathsToLockForOpsInternal( $fileOps );
816 $paths[LockManager::LOCK_UW],
817 $this->backends[$this->masterIndex]
820 $paths[LockManager::LOCK_EX],
821 $this->backends[$this->masterIndex]
array $params
The job parameters.
Proxy backend that mirrors writes to several internal backends.
FileBackendStore[] $backends
Prioritized list of FileBackendStore objects.
fileExists(array $params)
Check if a file exists at a storage path in the backend.
getLocalReferenceMulti(array $params)
Like getLocalReference() except it takes an array of storage paths and yields an order-preserved map ...
getLocalCopyMulti(array $params)
Like getLocalCopy() except it takes an array of storage paths and yields an order preserved-map of st...
consistencyCheck(array $paths)
Check that a set of files are consistent across all internal backends.
unsubstPaths( $paths, FileBackendStore $backend)
Substitute the backend of internal storage paths with the proxy backend's name.
getFileSha1Base36(array $params)
Get a SHA-1 hash of the content of the file at a storage path in the backend.
int $readIndex
Index of read affinity backend.
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
doQuickOperationsInternal(array $ops, array $opts)
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
getFileContentsMulti(array $params)
Like getFileContents() except it takes an array of storage paths and returns an order preserved map o...
preloadCache(array $paths)
Preload persistent file stat cache and property cache into in-process cache.
resyncFiles(array $paths, $resyncMode=true)
Check that a set of files are consistent across all internal backends and re-synchronize those files ...
getDirectoryList(array $params)
Get an iterator to list all directories under a storage directory.
__construct(array $config)
Construct a proxy backend that consists of several internal backends.
getFileList(array $params)
Get an iterator to list all stored files under a storage directory.
getReadIndexFromParams(array $params)
doOperationsInternal(array $ops, array $opts)
streamFile(array $params)
Stream the content of the file at a storage path in the backend.
int $masterIndex
Index of master backend.
accessibilityCheck(array $paths)
Check that a set of file paths are usable across all internal backends.
getFeatures()
Get the a bitfield of extra features supported by the backend medium.
hasVolatileSources(array $ops)
fileStoragePathsForOps(array $ops)
Get a list of file storage paths to read or write for a list of operations.
concatenate(array $params)
Concatenate a list of storage files into a single file system file.
substOpPaths(array $ops, FileBackendStore $backend)
Same as substOpBatchPaths() but for a single operation.
substOpBatchPaths(array $ops, FileBackendStore $backend)
Substitute the backend name in storage path parameters for a set of operations with that of a given i...
getFileProps(array $params)
Get the properties of the content of the file at a storage path in the backend.
doDirectoryOp( $method, array $params)
getScopedLocksForOps(array $ops, StatusValue $status)
Get an array of scoped locks needed for a batch of file operations.
getFileHttpUrl(array $params)
Return an HTTP URL to a given file that requires no authentication to use.
preloadFileStat(array $params)
Preload file stat information (concurrently if possible) into in-process cache.
getFileSize(array $params)
Get the size (bytes) of a file at a storage path in the backend.
getFileXAttributes(array $params)
Get metadata about a file at a storage path in the backend.
substPaths( $paths, FileBackendStore $backend)
Substitute the backend of storage paths with an internal backend's name.
directoryExists(array $params)
Check if a directory exists at a given storage path.
getFileTimestamp(array $params)
Get the last-modified timestamp of the file at a storage path.
Base class for all backends using particular storage medium.
Base class for all file backend classes (including multi-write backends).
string $name
Unique backend name.
string $domainId
Unique domain name.
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
newStatus(... $args)
Yields the result of the status wrapper callback on either:
getName()
Get the unique backend name.
Generic operation result class Has warning/error list, boolean status and arbitrary value.