26use InvalidArgumentException;
33use Wikimedia\Timestamp\ConvertibleTimestamp;
73 private const CHECK_SIZE = 1;
75 private const CHECK_TIME = 2;
77 private const CHECK_SHA1 = 4;
107 parent::__construct( $config );
108 $this->syncChecks = $config[
'syncChecks'] ?? self::CHECK_SIZE;
109 $this->autoResync = $config[
'autoResync'] ??
false;
110 $this->asyncWrites = isset( $config[
'replication'] ) && $config[
'replication'] ===
'async';
114 foreach ( $config[
'backends'] as $index => $beConfig ) {
115 $name = $beConfig[
'name'];
116 if ( isset( $namesUsed[
$name] ) ) {
117 throw new LogicException(
"Two or more backends defined with the name $name." );
119 $namesUsed[
$name] = 1;
121 unset( $beConfig[
'readOnly'] );
122 unset( $beConfig[
'lockManager'] );
125 if ( !empty( $beConfig[
'isMultiMaster'] ) ) {
126 if ( $this->masterIndex >= 0 ) {
127 throw new LogicException(
'More than one master backend defined.' );
129 $this->masterIndex = $index;
131 if ( !empty( $beConfig[
'readAffinity'] ) ) {
132 $this->readIndex = $index;
135 if ( !isset( $beConfig[
'class'] ) ) {
136 throw new InvalidArgumentException(
'No class given for a backend config.' );
138 $class = $beConfig[
'class'];
139 $this->backends[$index] =
new $class( $beConfig );
141 if ( $this->masterIndex < 0 ) {
142 throw new LogicException(
'No master backend defined.' );
144 if ( $this->readIndex < 0 ) {
157 if ( empty( $opts[
'nonLocking'] ) ) {
159 if ( !$status->isOK() ) {
167 $opts[
'preserveCache'] =
true;
170 if ( !$status->isOK() ) {
175 if ( !$syncStatus->isOK() ) {
176 $this->logger->error(
177 "$fname: failed sync check: " . FormatJson::encode( $relevantPaths )
181 $this->autoResync ===
false ||
182 !$this->
resyncFiles( $relevantPaths, $this->autoResync )->isOK()
184 $status->merge( $syncStatus );
191 $masterStatus = $mbe->doOperations( $realOps, $opts );
192 $status->merge( $masterStatus );
196 if ( $masterStatus->isOK() && $masterStatus->successCount > 0 ) {
197 foreach ( $this->backends as $index => $backend ) {
198 if ( $index === $this->masterIndex ) {
205 DeferredUpdates::addCallableUpdate(
207 $backend, $realOps, $opts, $scopeLock, $relevantPaths, $fname
209 $this->logger->debug(
210 "$fname: '{$backend->getName()}' async replication; paths: " .
211 FormatJson::encode( $relevantPaths )
213 $backend->doOperations( $realOps, $opts );
217 $this->logger->debug(
218 "$fname: '{$backend->getName()}' sync replication; paths: " .
219 FormatJson::encode( $relevantPaths )
221 $status->merge( $backend->doOperations( $realOps, $opts ) );
228 $status->success = $masterStatus->success;
229 $status->successCount = $masterStatus->successCount;
230 $status->failCount = $masterStatus->failCount;
246 if ( $this->syncChecks == 0 || count( $this->backends ) <= 1 ) {
251 foreach ( $this->backends as $backend ) {
252 $realPaths = $this->
substPaths( $paths, $backend );
253 $backend->preloadFileStat( [
'srcs' => $realPaths,
'latest' =>
true ] );
256 foreach ( $paths as
$path ) {
261 $masterStat = $masterBackend->getFileStat( $masterParams );
262 if ( $masterStat === self::STAT_ERROR ) {
263 $status->fatal(
'backend-fail-stat',
$path );
266 if ( $this->syncChecks & self::CHECK_SHA1 ) {
267 $masterSha1 = $masterBackend->getFileSha1Base36( $masterParams );
268 if ( ( $masterSha1 !==
false ) !== (
bool)$masterStat ) {
269 $status->fatal(
'backend-fail-hash',
$path );
277 foreach ( $this->backends as $index => $cloneBackend ) {
278 if ( $index === $this->masterIndex ) {
284 $cloneStat = $cloneBackend->getFileStat( $cloneParams );
290 $status->fatal(
'backend-fail-synced',
$path );
292 ( $this->syncChecks & self::CHECK_SIZE ) &&
293 $cloneStat[
'size'] !== $masterStat[
'size']
296 $status->fatal(
'backend-fail-synced',
$path );
298 ( $this->syncChecks & self::CHECK_TIME ) &&
300 (
int)ConvertibleTimestamp::convert( TS_UNIX, $masterStat[
'mtime'] ) -
301 (
int)ConvertibleTimestamp::convert( TS_UNIX, $cloneStat[
'mtime'] )
305 $status->fatal(
'backend-fail-synced',
$path );
307 ( $this->syncChecks & self::CHECK_SHA1 ) &&
308 $cloneBackend->getFileSha1Base36( $cloneParams ) !== $masterSha1
311 $status->fatal(
'backend-fail-synced',
$path );
317 $status->fatal(
'backend-fail-synced',
$path );
334 if ( count( $this->backends ) <= 1 ) {
338 foreach ( $paths as
$path ) {
339 foreach ( $this->backends as $backend ) {
340 $realPath = $this->
substPaths( $path, $backend );
341 if ( !$backend->isPathUsableInternal( $realPath ) ) {
342 $status->fatal(
'backend-fail-usable',
$path );
364 foreach ( $paths as
$path ) {
369 $masterPath = $masterParams[
'src'];
370 $masterStat = $masterBackend->getFileStat( $masterParams );
371 if ( $masterStat === self::STAT_ERROR ) {
372 $status->fatal(
'backend-fail-stat',
$path );
373 $this->logger->error(
"$fname: file '$masterPath' is not available" );
376 $masterSha1 = $masterBackend->getFileSha1Base36( $masterParams );
377 if ( ( $masterSha1 !==
false ) !== (
bool)$masterStat ) {
378 $status->fatal(
'backend-fail-hash',
$path );
379 $this->logger->error(
"$fname: file '$masterPath' hash does not match stat" );
384 foreach ( $this->backends as $index => $cloneBackend ) {
385 if ( $index === $this->masterIndex ) {
391 $clonePath = $cloneParams[
'src'];
392 $cloneStat = $cloneBackend->getFileStat( $cloneParams );
393 if ( $cloneStat === self::STAT_ERROR ) {
394 $status->fatal(
'backend-fail-stat',
$path );
395 $this->logger->error(
"$fname: file '$clonePath' is not available" );
398 $cloneSha1 = $cloneBackend->getFileSha1Base36( $cloneParams );
399 if ( ( $cloneSha1 !==
false ) !== (
bool)$cloneStat ) {
400 $status->fatal(
'backend-fail-hash',
$path );
401 $this->logger->error(
"$fname: file '$clonePath' hash does not match stat" );
405 if ( $masterSha1 === $cloneSha1 ) {
407 $this->logger->debug(
"$fname: file '$clonePath' matches '$masterPath'" );
408 } elseif ( $masterSha1 !==
false ) {
411 $resyncMode ===
'conservative' &&
414 $cloneStat[
'mtime'] > $masterStat[
'mtime']
417 $status->fatal(
'backend-fail-synced',
$path );
420 $fsFile = $masterBackend->getLocalReference( $masterParams );
421 $status->merge( $cloneBackend->quickStore( [
426 } elseif ( $masterStat ===
false ) {
428 if ( $resyncMode ===
'conservative' ) {
430 $status->fatal(
'backend-fail-synced',
$path );
431 $this->logger->error(
"$fname: not allowed to delete file '$clonePath'" );
434 $status->merge( $cloneBackend->quickDelete( [
'src' => $clonePath ] ) );
440 if ( !$status->isOK() ) {
441 $this->logger->error(
"$fname: failed to resync: " . FormatJson::encode( $paths ) );
455 foreach ( $ops as $op ) {
456 if ( isset( $op[
'src'] ) ) {
459 if ( empty( $op[
'ignoreMissingSource'] )
460 || $this->
fileExists( [
'src' => $op[
'src'] ] )
462 $paths[] = $op[
'src'];
465 if ( isset( $op[
'srcs'] ) ) {
466 $paths = array_merge( $paths, $op[
'srcs'] );
468 if ( isset( $op[
'dst'] ) ) {
469 $paths[] = $op[
'dst'];
473 return array_values( array_unique( array_filter( $paths, [ FileBackend::class,
'isStoragePath' ] ) ) );
486 foreach ( $ops as $op ) {
488 foreach ( [
'src',
'srcs',
'dst',
'dir' ] as $par ) {
489 if ( isset( $newOp[$par] ) ) {
490 $newOp[$par] = $this->
substPaths( $newOp[$par], $backend );
521 '!^mwstore://' . preg_quote( $this->name,
'!' ) .
'/!',
522 StringUtils::escapeRegexReplacement(
"mwstore://{$backend->getName()}/" ),
536 '!^mwstore://' . preg_quote( $backend->
getName(),
'!' ) .
'/!',
537 StringUtils::escapeRegexReplacement(
"mwstore://{$this->name}/" ),
547 foreach ( $ops as $op ) {
548 if ( $op[
'op'] ===
'store' && !isset( $op[
'srcRef'] ) ) {
559 $realOps = $this->
substOpBatchPaths( $ops, $this->backends[$this->masterIndex] );
561 $status->merge( $masterStatus );
563 foreach ( $this->backends as $index => $backend ) {
564 if ( $index === $this->masterIndex ) {
570 DeferredUpdates::addCallableUpdate(
571 static function () use ( $backend, $realOps ) {
572 $backend->doQuickOperations( $realOps );
576 $status->merge( $backend->doQuickOperations( $realOps ) );
582 $status->success = $masterStatus->success;
583 $status->successCount = $masterStatus->successCount;
584 $status->failCount = $masterStatus->failCount;
613 $realParams = $this->
substOpPaths( $params, $this->backends[$this->masterIndex] );
615 $status->merge( $masterStatus );
617 foreach ( $this->backends as $index => $backend ) {
618 if ( $index === $this->masterIndex ) {
623 if ( $this->asyncWrites ) {
624 DeferredUpdates::addCallableUpdate(
625 static function () use ( $backend, $method, $realParams ) {
626 $backend->$method( $realParams );
630 $status->merge( $backend->$method( $realParams ) );
641 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
643 $status->merge( $this->backends[$index]->
concatenate( $realParams ) );
650 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
652 return $this->backends[$index]->fileExists( $realParams );
657 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
659 return $this->backends[$index]->getFileTimestamp( $realParams );
664 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
666 return $this->backends[$index]->getFileSize( $realParams );
671 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
673 return $this->backends[$index]->getFileStat( $realParams );
678 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
680 return $this->backends[$index]->getFileXAttributes( $realParams );
685 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
687 $contentsM = $this->backends[$index]->getFileContentsMulti( $realParams );
690 foreach ( $contentsM as
$path => $data ) {
699 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
701 return $this->backends[$index]->getFileSha1Base36( $realParams );
706 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
708 return $this->backends[$index]->getFileProps( $realParams );
713 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
715 return $this->backends[$index]->streamFile( $realParams );
720 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
722 $fsFilesM = $this->backends[$index]->getLocalReferenceMulti( $realParams );
725 foreach ( $fsFilesM as
$path => $fsFile ) {
734 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
736 $tempFilesM = $this->backends[$index]->getLocalCopyMulti( $realParams );
739 foreach ( $tempFilesM as
$path => $tempFile ) {
748 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
750 return $this->backends[$index]->getFileHttpUrl( $realParams );
754 $realParams = $this->
substOpPaths( $params, $this->backends[$this->masterIndex] );
760 $realParams = $this->
substOpPaths( $params, $this->backends[$this->masterIndex] );
767 return $this->getFileListForWrite(
$params );
770 $realParams = $this->
substOpPaths( $params, $this->backends[$this->masterIndex] );
774 private function getFileListForWrite(
$params ) {
779 foreach ( $this->backends as $backend ) {
781 $iterator = $backend->getFileList( $realParams );
782 if ( $iterator !==
null ) {
783 foreach ( $iterator as $file ) {
789 return array_unique( $files );
797 foreach ( $this->backends as $backend ) {
798 $realPaths = is_array( $paths ) ? $this->
substPaths( $paths, $backend ) :
null;
799 $backend->clearCache( $realPaths );
804 $realPaths = $this->
substPaths( $paths, $this->backends[$this->readIndex] );
810 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
812 return $this->backends[$index]->preloadFileStat( $realParams );
816 $realOps = $this->
substOpBatchPaths( $ops, $this->backends[$this->masterIndex] );
819 $paths = $this->backends[
$this->masterIndex]->getPathsToLockForOpsInternal( $fileOps );
823 $paths[LockManager::LOCK_UW],
824 $this->backends[$this->masterIndex]
827 $paths[LockManager::LOCK_EX],
828 $this->backends[$this->masterIndex]
846class_alias( FileBackendMultiWrite::class,
'FileBackendMultiWrite' );
array $params
The job parameters.
Resource locking handling.
Generic operation result class Has warning/error list, boolean status and arbitrary value.
A collection of static methods to play with strings.