10namespace Wikimedia\FileBackend;
12use InvalidArgumentException;
14use Shellbox\Command\BoxedCommand;
16use Wikimedia\LockManager\LockManager;
17use Wikimedia\StringUtils\StringUtils;
64 parent::__construct( $config );
65 $this->asyncWrites = isset( $config[
'replication'] ) && $config[
'replication'] ===
'async';
69 foreach ( $config[
'backends'] as $index => $beConfig ) {
70 $name = $beConfig[
'name'];
71 if ( isset( $namesUsed[
$name] ) ) {
72 throw new LogicException(
"Two or more backends defined with the name $name." );
74 $namesUsed[
$name] = 1;
76 unset( $beConfig[
'readOnly'] );
77 unset( $beConfig[
'lockManager'] );
80 if ( !empty( $beConfig[
'isMultiMaster'] ) ) {
81 if ( $this->masterIndex >= 0 ) {
82 throw new LogicException(
'More than one master backend defined.' );
84 $this->masterIndex = $index;
86 if ( !empty( $beConfig[
'readAffinity'] ) ) {
87 $this->readIndex = $index;
90 if ( !isset( $beConfig[
'class'] ) ) {
91 throw new InvalidArgumentException(
'No class given for a backend config.' );
93 $class = $beConfig[
'class'];
94 $this->backends[$index] =
new $class( $beConfig );
96 if ( $this->masterIndex < 0 ) {
97 throw new LogicException(
'No master backend defined.' );
99 if ( $this->readIndex < 0 ) {
113 if ( empty( $opts[
'nonLocking'] ) ) {
115 if ( !$status->isOK() ) {
123 $opts[
'preserveCache'] =
true;
126 $masterStatus = $mbe->doOperations( $realOps, $opts );
127 $status->merge( $masterStatus );
131 if ( $masterStatus->isOK() && $masterStatus->successCount > 0 ) {
132 foreach ( $this->backends as $index => $backend ) {
133 if ( $index === $this->masterIndex ) {
143 $backend, $realOps, $opts, $scopeLock, $relevantPaths, $fname
145 $this->logger->debug(
146 "$fname: '{$backend->getName()}' async replication; paths: " .
147 implode(
', ', $relevantPaths )
149 $backend->doOperations( $realOps, $opts );
153 $this->logger->debug(
154 "$fname: '{$backend->getName()}' sync replication; paths: " .
155 implode(
', ', $relevantPaths )
157 $status->merge( $backend->doOperations( $realOps, $opts ) );
164 $status->success = $masterStatus->success;
165 $status->successCount = $masterStatus->successCount;
166 $status->failCount = $masterStatus->failCount;
179 foreach ( $ops as $op ) {
180 if ( isset( $op[
'src'] ) ) {
181 $paths[] = $op[
'src'];
183 if ( isset( $op[
'srcs'] ) ) {
184 $paths = array_merge( $paths, $op[
'srcs'] );
186 if ( isset( $op[
'dst'] ) ) {
187 $paths[] = $op[
'dst'];
191 return array_values( array_unique( array_filter(
207 foreach ( $ops as $op ) {
209 foreach ( [
'src',
'srcs',
'dst',
'dir' ] as $par ) {
210 if ( isset( $newOp[$par] ) ) {
211 $newOp[$par] = $this->
substPaths( $newOp[$par], $backend );
242 '!^mwstore://' . preg_quote( $this->name,
'!' ) .
'/!',
243 StringUtils::escapeRegexReplacement(
"mwstore://{$backend->getName()}/" ),
257 '!^mwstore://' . preg_quote( $backend->
getName(),
'!' ) .
'/!',
258 StringUtils::escapeRegexReplacement(
"mwstore://{$this->name}/" ),
268 foreach ( $ops as $op ) {
269 if ( $op[
'op'] ===
'store' && !isset( $op[
'srcRef'] ) ) {
281 $realOps = $this->
substOpBatchPaths( $ops, $this->backends[$this->masterIndex] );
283 $status->merge( $masterStatus );
285 foreach ( $this->backends as $index => $backend ) {
286 if ( $index === $this->masterIndex ) {
293 static function () use ( $backend, $realOps ) {
294 $backend->doQuickOperations( $realOps );
298 $status->merge( $backend->doQuickOperations( $realOps ) );
304 $status->success = $masterStatus->success;
305 $status->successCount = $masterStatus->successCount;
306 $status->failCount = $masterStatus->failCount;
339 $realParams = $this->
substOpPaths( $params, $this->backends[$this->masterIndex] );
341 $status->merge( $masterStatus );
343 foreach ( $this->backends as $index => $backend ) {
344 if ( $index === $this->masterIndex ) {
349 if ( $this->asyncWrites ) {
351 static function () use ( $backend, $method, $realParams ) {
352 $backend->$method( $realParams );
356 $status->merge( $backend->$method( $realParams ) );
368 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
370 $status->merge( $this->backends[$index]->
concatenate( $realParams ) );
378 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
380 return $this->backends[$index]->fileExists( $realParams );
386 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
388 return $this->backends[$index]->getFileTimestamp( $realParams );
394 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
396 return $this->backends[$index]->getFileSize( $realParams );
402 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
404 return $this->backends[$index]->getFileStat( $realParams );
410 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
412 return $this->backends[$index]->getFileXAttributes( $realParams );
418 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
420 $contentsM = $this->backends[$index]->getFileContentsMulti( $realParams );
423 foreach ( $contentsM as
$path => $data ) {
433 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
435 return $this->backends[$index]->getFileSha1Base36( $realParams );
441 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
443 return $this->backends[$index]->getFileProps( $realParams );
449 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
451 return $this->backends[$index]->streamFile( $realParams );
457 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
459 $fsFilesM = $this->backends[$index]->getLocalReferenceMulti( $realParams );
462 foreach ( $fsFilesM as
$path => $fsFile ) {
472 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
474 $tempFilesM = $this->backends[$index]->getLocalCopyMulti( $realParams );
477 foreach ( $tempFilesM as
$path => $tempFile ) {
487 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
489 return $this->backends[$index]->getFileHttpUrl( $realParams );
497 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
498 return $this->backends[$index]->addShellboxInputFile( $command, $boxedName, $realParams );
503 $realParams = $this->
substOpPaths( $params, $this->backends[$this->masterIndex] );
510 $realParams = $this->
substOpPaths( $params, $this->backends[$this->masterIndex] );
517 if ( isset( $params[
'forWrite'] ) && $params[
'forWrite'] ) {
518 return $this->getFileListForWrite( $params );
521 $realParams = $this->
substOpPaths( $params, $this->backends[$this->masterIndex] );
525 private function getFileListForWrite( array $params ): array {
530 foreach ( $this->backends as $backend ) {
532 $iterator = $backend->getFileList( $realParams );
533 if ( $iterator !==
null ) {
534 foreach ( $iterator as $file ) {
540 return array_unique( $files );
545 return $this->backends[$this->masterIndex]->getFeatures();
550 foreach ( $this->backends as $backend ) {
551 $realPaths = is_array( $paths ) ? $this->
substPaths( $paths, $backend ) :
null;
552 $backend->clearCache( $realPaths );
558 $realPaths = $this->
substPaths( $paths, $this->backends[$this->readIndex] );
559 $this->backends[$this->readIndex]->preloadCache( $realPaths );
565 $realParams = $this->
substOpPaths( $params, $this->backends[$index] );
567 return $this->backends[$index]->preloadFileStat( $realParams );
572 $realOps = $this->
substOpBatchPaths( $ops, $this->backends[$this->masterIndex] );
573 $fileOps = $this->backends[$this->masterIndex]->getOperationsInternal( $realOps );
575 $paths = $this->backends[$this->masterIndex]->getPathsToLockForOpsInternal( $fileOps );
579 $paths[LockManager::LOCK_UW],
580 $this->backends[$this->masterIndex]
583 $paths[LockManager::LOCK_EX],
584 $this->backends[$this->masterIndex]
597 return !empty( $params[
'latest'] ) ? $this->masterIndex : $this->readIndex;
602class_alias( FileBackendMultiWrite::class,
'FileBackendMultiWrite' );