27use Wikimedia\Timestamp\ConvertibleTimestamp;
51 protected $backends = [];
54 protected $masterIndex = -1;
56 protected $readIndex = -1;
59 protected $syncChecks = 0;
61 protected $autoResync =
false;
64 protected $asyncWrites =
false;
67 private const CHECK_SIZE = 1;
69 private const CHECK_TIME = 2;
71 private const CHECK_SHA1 = 4;
102 parent::__construct( $config );
103 $this->syncChecks = $config[
'syncChecks'] ?? self::CHECK_SIZE;
104 $this->autoResync = $config[
'autoResync'] ??
false;
105 $this->asyncWrites = isset( $config[
'replication'] ) && $config[
'replication'] ===
'async';
109 foreach ( $config[
'backends'] as $index => $beConfig ) {
110 $name = $beConfig[
'name'];
111 if ( isset( $namesUsed[
$name] ) ) {
112 throw new LogicException(
"Two or more backends defined with the name $name." );
114 $namesUsed[$name] = 1;
116 unset( $beConfig[
'readOnly'] );
117 unset( $beConfig[
'lockManager'] );
118 $beConfig[
'domainId'] = $this->domainId;
119 $beConfig[
'logger'] = $this->logger;
120 if ( !empty( $beConfig[
'isMultiMaster'] ) ) {
121 if ( $this->masterIndex >= 0 ) {
122 throw new LogicException(
'More than one master backend defined.' );
124 $this->masterIndex = $index;
126 if ( !empty( $beConfig[
'readAffinity'] ) ) {
127 $this->readIndex = $index;
130 if ( !isset( $beConfig[
'class'] ) ) {
131 throw new InvalidArgumentException(
'No class given for a backend config.' );
133 $class = $beConfig[
'class'];
134 $this->backends[$index] =
new $class( $beConfig );
136 if ( $this->masterIndex < 0 ) {
137 throw new LogicException(
'No master backend defined.' );
139 if ( $this->readIndex < 0 ) {
140 $this->readIndex = $this->masterIndex;
148 $mbe = $this->backends[$this->masterIndex];
152 if ( empty( $opts[
'nonLocking'] ) ) {
154 if ( !$status->isOK() ) {
162 $opts[
'preserveCache'] =
true;
165 if ( !$status->isOK() ) {
170 if ( !$syncStatus->isOK() ) {
171 $this->logger->error(
172 "$fname: failed sync check: " . FormatJson::encode( $relevantPaths )
176 $this->autoResync ===
false ||
177 !$this->
resyncFiles( $relevantPaths, $this->autoResync )->isOK()
179 $status->merge( $syncStatus );
186 $masterStatus = $mbe->doOperations( $realOps, $opts );
187 $status->merge( $masterStatus );
191 if ( $masterStatus->isOK() && $masterStatus->successCount > 0 ) {
192 foreach ( $this->backends as $index => $backend ) {
193 if ( $index === $this->masterIndex ) {
200 DeferredUpdates::addCallableUpdate(
202 $backend, $realOps, $opts, $scopeLock, $relevantPaths, $fname
204 $this->logger->debug(
205 "$fname: '{$backend->getName()}' async replication; paths: " .
206 FormatJson::encode( $relevantPaths )
208 $backend->doOperations( $realOps, $opts );
212 $this->logger->debug(
213 "$fname: '{$backend->getName()}' sync replication; paths: " .
214 FormatJson::encode( $relevantPaths )
216 $status->merge( $backend->doOperations( $realOps, $opts ) );
223 $status->success = $masterStatus->success;
224 $status->successCount = $masterStatus->successCount;
225 $status->failCount = $masterStatus->failCount;
241 if ( $this->syncChecks == 0 || count( $this->backends ) <= 1 ) {
246 foreach ( $this->backends as $backend ) {
247 $realPaths = $this->
substPaths( $paths, $backend );
248 $backend->preloadFileStat( [
'srcs' => $realPaths,
'latest' =>
true ] );
251 foreach ( $paths as
$path ) {
254 $masterBackend = $this->backends[$this->masterIndex];
256 $masterStat = $masterBackend->getFileStat( $masterParams );
257 if ( $masterStat === self::STAT_ERROR ) {
258 $status->fatal(
'backend-fail-stat',
$path );
261 if ( $this->syncChecks & self::CHECK_SHA1 ) {
262 $masterSha1 = $masterBackend->getFileSha1Base36( $masterParams );
263 if ( ( $masterSha1 !==
false ) !== (
bool)$masterStat ) {
264 $status->fatal(
'backend-fail-hash',
$path );
272 foreach ( $this->backends as $index => $cloneBackend ) {
273 if ( $index === $this->masterIndex ) {
279 $cloneStat = $cloneBackend->getFileStat( $cloneParams );
285 $status->fatal(
'backend-fail-synced',
$path );
287 ( $this->syncChecks & self::CHECK_SIZE ) &&
288 $cloneStat[
'size'] !== $masterStat[
'size']
291 $status->fatal(
'backend-fail-synced',
$path );
293 ( $this->syncChecks & self::CHECK_TIME ) &&
295 (
int)ConvertibleTimestamp::convert( TS_UNIX, $masterStat[
'mtime'] ) -
296 (
int)ConvertibleTimestamp::convert( TS_UNIX, $cloneStat[
'mtime'] )
300 $status->fatal(
'backend-fail-synced',
$path );
302 ( $this->syncChecks & self::CHECK_SHA1 ) &&
303 $cloneBackend->getFileSha1Base36( $cloneParams ) !== $masterSha1
306 $status->fatal(
'backend-fail-synced',
$path );
312 $status->fatal(
'backend-fail-synced',
$path );
329 if ( count( $this->backends ) <= 1 ) {
333 foreach ( $paths as
$path ) {
334 foreach ( $this->backends as $backend ) {
336 if ( !$backend->isPathUsableInternal( $realPath ) ) {
337 $status->fatal(
'backend-fail-usable',
$path );
359 foreach ( $paths as
$path ) {
362 $masterBackend = $this->backends[$this->masterIndex];
364 $masterPath = $masterParams[
'src'];
365 $masterStat = $masterBackend->getFileStat( $masterParams );
366 if ( $masterStat === self::STAT_ERROR ) {
367 $status->fatal(
'backend-fail-stat',
$path );
368 $this->logger->error(
"$fname: file '$masterPath' is not available" );
371 $masterSha1 = $masterBackend->getFileSha1Base36( $masterParams );
372 if ( ( $masterSha1 !==
false ) !== (
bool)$masterStat ) {
373 $status->fatal(
'backend-fail-hash',
$path );
374 $this->logger->error(
"$fname: file '$masterPath' hash does not match stat" );
379 foreach ( $this->backends as $index => $cloneBackend ) {
380 if ( $index === $this->masterIndex ) {
386 $clonePath = $cloneParams[
'src'];
387 $cloneStat = $cloneBackend->getFileStat( $cloneParams );
388 if ( $cloneStat === self::STAT_ERROR ) {
389 $status->fatal(
'backend-fail-stat',
$path );
390 $this->logger->error(
"$fname: file '$clonePath' is not available" );
393 $cloneSha1 = $cloneBackend->getFileSha1Base36( $cloneParams );
394 if ( ( $cloneSha1 !==
false ) !== (
bool)$cloneStat ) {
395 $status->fatal(
'backend-fail-hash',
$path );
396 $this->logger->error(
"$fname: file '$clonePath' hash does not match stat" );
400 if ( $masterSha1 === $cloneSha1 ) {
402 $this->logger->debug(
"$fname: file '$clonePath' matches '$masterPath'" );
403 } elseif ( $masterSha1 !==
false ) {
406 $resyncMode ===
'conservative' &&
409 $cloneStat[
'mtime'] > $masterStat[
'mtime']
412 $status->fatal(
'backend-fail-synced',
$path );
415 $fsFile = $masterBackend->getLocalReference( $masterParams );
416 $status->merge( $cloneBackend->quickStore( [
421 } elseif ( $masterStat ===
false ) {
423 if ( $resyncMode ===
'conservative' ) {
425 $status->fatal(
'backend-fail-synced',
$path );
426 $this->logger->error(
"$fname: not allowed to delete file '$clonePath'" );
429 $status->merge( $cloneBackend->quickDelete( [
'src' => $clonePath ] ) );
435 if ( !$status->isOK() ) {
436 $this->logger->error(
"$fname: failed to resync: " . FormatJson::encode( $paths ) );
450 foreach ( $ops as $op ) {
451 if ( isset( $op[
'src'] ) ) {
454 if ( empty( $op[
'ignoreMissingSource'] )
455 || $this->
fileExists( [
'src' => $op[
'src'] ] )
457 $paths[] = $op[
'src'];
460 if ( isset( $op[
'srcs'] ) ) {
461 $paths = array_merge( $paths, $op[
'srcs'] );
463 if ( isset( $op[
'dst'] ) ) {
464 $paths[] = $op[
'dst'];
468 return array_values( array_unique( array_filter( $paths, [ FileBackend::class,
'isStoragePath' ] ) ) );
481 foreach ( $ops as $op ) {
483 foreach ( [
'src',
'srcs',
'dst',
'dir' ] as $par ) {
484 if ( isset( $newOp[$par] ) ) {
485 $newOp[$par] = $this->
substPaths( $newOp[$par], $backend );
516 '!^mwstore://' . preg_quote( $this->name,
'!' ) .
'/!',
517 StringUtils::escapeRegexReplacement(
"mwstore://{$backend->getName()}/" ),
531 '!^mwstore://' . preg_quote( $backend->
getName(),
'!' ) .
'/!',
532 StringUtils::escapeRegexReplacement(
"mwstore://{$this->name}/" ),
542 foreach ( $ops as $op ) {
543 if ( $op[
'op'] ===
'store' && !isset( $op[
'srcRef'] ) ) {
554 $realOps = $this->
substOpBatchPaths( $ops, $this->backends[$this->masterIndex] );
555 $masterStatus = $this->backends[$this->masterIndex]->doQuickOperations( $realOps );
556 $status->merge( $masterStatus );
558 foreach ( $this->backends as $index => $backend ) {
559 if ( $index === $this->masterIndex ) {
565 DeferredUpdates::addCallableUpdate(
566 static function () use ( $backend, $realOps ) {
567 $backend->doQuickOperations( $realOps );
571 $status->merge( $backend->doQuickOperations( $realOps ) );
577 $status->success = $masterStatus->success;
578 $status->successCount = $masterStatus->successCount;
579 $status->failCount = $masterStatus->failCount;
609 $masterStatus = $this->backends[$this->masterIndex]->$method( $realParams );
610 $status->merge( $masterStatus );
612 foreach ( $this->backends as $index => $backend ) {
613 if ( $index === $this->masterIndex ) {
618 if ( $this->asyncWrites ) {
619 DeferredUpdates::addCallableUpdate(
620 static function () use ( $backend, $method, $realParams ) {
621 $backend->$method( $realParams );
625 $status->merge( $backend->$method( $realParams ) );
638 $status->merge( $this->backends[$index]->
concatenate( $realParams ) );
647 return $this->backends[$index]->fileExists( $realParams );
654 return $this->backends[$index]->getFileTimestamp( $realParams );
661 return $this->backends[$index]->getFileSize( $realParams );
668 return $this->backends[$index]->getFileStat( $realParams );
675 return $this->backends[$index]->getFileXAttributes( $realParams );
682 $contentsM = $this->backends[$index]->getFileContentsMulti( $realParams );
685 foreach ( $contentsM as
$path => $data ) {
696 return $this->backends[$index]->getFileSha1Base36( $realParams );
703 return $this->backends[$index]->getFileProps( $realParams );
710 return $this->backends[$index]->streamFile( $realParams );
717 $fsFilesM = $this->backends[$index]->getLocalReferenceMulti( $realParams );
720 foreach ( $fsFilesM as
$path => $fsFile ) {
731 $tempFilesM = $this->backends[$index]->getLocalCopyMulti( $realParams );
734 foreach ( $tempFilesM as
$path => $tempFile ) {
745 return $this->backends[$index]->getFileHttpUrl( $realParams );
751 return $this->backends[$this->masterIndex]->directoryExists( $realParams );
757 return $this->backends[$this->masterIndex]->getDirectoryList( $realParams );
762 return $this->getFileListForWrite(
$params );
766 return $this->backends[$this->masterIndex]->getFileList( $realParams );
769 private function getFileListForWrite(
$params ) {
774 foreach ( $this->backends as $backend ) {
775 $realParams = $this->substOpPaths(
$params, $backend );
776 $iterator = $backend->getFileList( $realParams );
777 if ( $iterator !==
null ) {
778 foreach ( $iterator as $file ) {
784 return array_unique( $files );
788 return $this->backends[$this->masterIndex]->getFeatures();
792 foreach ( $this->backends as $backend ) {
793 $realPaths = is_array( $paths ) ? $this->
substPaths( $paths, $backend ) :
null;
794 $backend->clearCache( $realPaths );
799 $realPaths = $this->
substPaths( $paths, $this->backends[$this->readIndex] );
800 $this->backends[$this->readIndex]->preloadCache( $realPaths );
807 return $this->backends[$index]->preloadFileStat( $realParams );
811 $realOps = $this->
substOpBatchPaths( $ops, $this->backends[$this->masterIndex] );
812 $fileOps = $this->backends[$this->masterIndex]->getOperationsInternal( $realOps );
814 $paths = $this->backends[$this->masterIndex]->getPathsToLockForOpsInternal( $fileOps );
818 $paths[LockManager::LOCK_UW],
819 $this->backends[$this->masterIndex]
822 $paths[LockManager::LOCK_EX],
823 $this->backends[$this->masterIndex]
836 return !empty(
$params[
'latest'] ) ? $this->masterIndex : $this->readIndex;
array $params
The job parameters.
Proxy backend that mirrors writes to several internal backends.
FileBackendStore[] $backends
Prioritized list of FileBackendStore objects.
fileExists(array $params)
Check if a file exists at a storage path in the backend.
getLocalReferenceMulti(array $params)
Like getLocalReference() except it takes an array of storage paths and yields an order-preserved map ...
getLocalCopyMulti(array $params)
Like getLocalCopy() except it takes an array of storage paths and yields an order preserved-map of st...
consistencyCheck(array $paths)
Check that a set of files are consistent across all internal backends.
unsubstPaths( $paths, FileBackendStore $backend)
Substitute the backend of internal storage paths with the proxy backend's name.
getFileSha1Base36(array $params)
Get a SHA-1 hash of the content of the file at a storage path in the backend.
int $readIndex
Index of read affinity backend.
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
doQuickOperationsInternal(array $ops, array $opts)
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
getFileContentsMulti(array $params)
Like getFileContents() except it takes an array of storage paths and returns an order preserved map o...
preloadCache(array $paths)
Preload persistent file stat cache and property cache into in-process cache.
resyncFiles(array $paths, $resyncMode=true)
Check that a set of files are consistent across all internal backends and re-synchronize those files ...
getDirectoryList(array $params)
Get an iterator to list all directories under a storage directory.
__construct(array $config)
Construct a proxy backend that consists of several internal backends.
getFileList(array $params)
Get an iterator to list all stored files under a storage directory.
getReadIndexFromParams(array $params)
doOperationsInternal(array $ops, array $opts)
streamFile(array $params)
Stream the content of the file at a storage path in the backend.
int $masterIndex
Index of master backend.
accessibilityCheck(array $paths)
Check that a set of file paths are usable across all internal backends.
getFeatures()
Get the a bitfield of extra features supported by the backend medium.
hasVolatileSources(array $ops)
fileStoragePathsForOps(array $ops)
Get a list of file storage paths to read or write for a list of operations.
concatenate(array $params)
Concatenate a list of storage files into a single file system file.
substOpPaths(array $ops, FileBackendStore $backend)
Same as substOpBatchPaths() but for a single operation.
substOpBatchPaths(array $ops, FileBackendStore $backend)
Substitute the backend name in storage path parameters for a set of operations with that of a given i...
getFileProps(array $params)
Get the properties of the content of the file at a storage path in the backend.
doDirectoryOp( $method, array $params)
getScopedLocksForOps(array $ops, StatusValue $status)
Get an array of scoped locks needed for a batch of file operations.
getFileHttpUrl(array $params)
Return an HTTP URL to a given file that requires no authentication to use.
preloadFileStat(array $params)
Preload file stat information (concurrently if possible) into in-process cache.
getFileSize(array $params)
Get the size (bytes) of a file at a storage path in the backend.
getFileXAttributes(array $params)
Get metadata about a file at a storage path in the backend.
substPaths( $paths, FileBackendStore $backend)
Substitute the backend of storage paths with an internal backend's name.
directoryExists(array $params)
Check if a directory exists at a given storage path.
getFileTimestamp(array $params)
Get the last-modified timestamp of the file at a storage path.
Base class for all backends using particular storage medium.
Generic operation result class Has warning/error list, boolean status and arbitrary value.