MediaWiki master
FileBackendMultiWrite.php
Go to the documentation of this file.
1<?php
24namespace Wikimedia\FileBackend;
25
26use InvalidArgumentException;
27use LockManager;
28use LogicException;
31use StatusValue;
32use StringUtils;
33use Wikimedia\Timestamp\ConvertibleTimestamp;
34
57 protected $backends = [];
58
60 protected $masterIndex = -1;
62 protected $readIndex = -1;
63
65 protected $syncChecks = 0;
67 protected $autoResync = false;
68
70 protected $asyncWrites = false;
71
73 private const CHECK_SIZE = 1;
75 private const CHECK_TIME = 2;
77 private const CHECK_SHA1 = 4;
78
106 public function __construct( array $config ) {
107 parent::__construct( $config );
108 $this->syncChecks = $config['syncChecks'] ?? self::CHECK_SIZE;
109 $this->autoResync = $config['autoResync'] ?? false;
110 $this->asyncWrites = isset( $config['replication'] ) && $config['replication'] === 'async';
111 // Construct backends here rather than via registration
112 // to keep these backends hidden from outside the proxy.
113 $namesUsed = [];
114 foreach ( $config['backends'] as $index => $beConfig ) {
115 $name = $beConfig['name'];
116 if ( isset( $namesUsed[$name] ) ) { // don't break FileOp predicates
117 throw new LogicException( "Two or more backends defined with the name $name." );
118 }
119 $namesUsed[$name] = 1;
120 // Alter certain sub-backend settings
121 unset( $beConfig['readOnly'] ); // use proxy backend setting
122 unset( $beConfig['lockManager'] ); // lock under proxy backend
123 $beConfig['domainId'] = $this->domainId; // use the proxy backend wiki ID
124 $beConfig['logger'] = $this->logger; // use the proxy backend logger
125 if ( !empty( $beConfig['isMultiMaster'] ) ) {
126 if ( $this->masterIndex >= 0 ) {
127 throw new LogicException( 'More than one master backend defined.' );
128 }
129 $this->masterIndex = $index; // this is the "master"
130 }
131 if ( !empty( $beConfig['readAffinity'] ) ) {
132 $this->readIndex = $index; // prefer this for reads
133 }
134 // Create sub-backend object
135 if ( !isset( $beConfig['class'] ) ) {
136 throw new InvalidArgumentException( 'No class given for a backend config.' );
137 }
138 $class = $beConfig['class'];
139 $this->backends[$index] = new $class( $beConfig );
140 }
141 if ( $this->masterIndex < 0 ) { // need backends and must have a master
142 throw new LogicException( 'No master backend defined.' );
143 }
144 if ( $this->readIndex < 0 ) {
145 $this->readIndex = $this->masterIndex; // default
146 }
147 }
148
149 final protected function doOperationsInternal( array $ops, array $opts ) {
150 $status = $this->newStatus();
151
152 $fname = __METHOD__;
153 $mbe = $this->backends[$this->masterIndex]; // convenience
154
155 // Acquire any locks as needed
156 $scopeLock = null;
157 if ( empty( $opts['nonLocking'] ) ) {
158 $scopeLock = $this->getScopedLocksForOps( $ops, $status );
159 if ( !$status->isOK() ) {
160 return $status; // abort
161 }
162 }
163 // Get the list of paths to read/write
164 $relevantPaths = $this->fileStoragePathsForOps( $ops );
165 // Clear any cache entries (after locks acquired)
166 $this->clearCache( $relevantPaths );
167 $opts['preserveCache'] = true; // only locked files are cached
168 // Check if the paths are valid and accessible on all backends
169 $status->merge( $this->accessibilityCheck( $relevantPaths ) );
170 if ( !$status->isOK() ) {
171 return $status; // abort
172 }
173 // Do a consistency check to see if the backends are consistent
174 $syncStatus = $this->consistencyCheck( $relevantPaths );
175 if ( !$syncStatus->isOK() ) {
176 $this->logger->error(
177 "$fname: failed sync check: " . FormatJson::encode( $relevantPaths )
178 );
179 // Try to resync the clone backends to the master on the spot
180 if (
181 $this->autoResync === false ||
182 !$this->resyncFiles( $relevantPaths, $this->autoResync )->isOK()
183 ) {
184 $status->merge( $syncStatus );
185
186 return $status; // abort
187 }
188 }
189 // Actually attempt the operation batch on the master backend
190 $realOps = $this->substOpBatchPaths( $ops, $mbe );
191 $masterStatus = $mbe->doOperations( $realOps, $opts );
192 $status->merge( $masterStatus );
193 // Propagate the operations to the clone backends if there were no unexpected errors
194 // and everything didn't fail due to predicted errors. If $ops only had one operation,
195 // this might avoid backend sync inconsistencies.
196 if ( $masterStatus->isOK() && $masterStatus->successCount > 0 ) {
197 foreach ( $this->backends as $index => $backend ) {
198 if ( $index === $this->masterIndex ) {
199 continue; // done already
200 }
201
202 $realOps = $this->substOpBatchPaths( $ops, $backend );
203 if ( $this->asyncWrites && !$this->hasVolatileSources( $ops ) ) {
204 // Bind $scopeLock to the callback to preserve locks
205 DeferredUpdates::addCallableUpdate(
206 function () use (
207 $backend, $realOps, $opts, $scopeLock, $relevantPaths, $fname
208 ) {
209 $this->logger->debug(
210 "$fname: '{$backend->getName()}' async replication; paths: " .
211 FormatJson::encode( $relevantPaths )
212 );
213 $backend->doOperations( $realOps, $opts );
214 }
215 );
216 } else {
217 $this->logger->debug(
218 "$fname: '{$backend->getName()}' sync replication; paths: " .
219 FormatJson::encode( $relevantPaths )
220 );
221 $status->merge( $backend->doOperations( $realOps, $opts ) );
222 }
223 }
224 }
225 // Make 'success', 'successCount', and 'failCount' fields reflect
226 // the overall operation, rather than all the batches for each backend.
227 // Do this by only using success values from the master backend's batch.
228 $status->success = $masterStatus->success;
229 $status->successCount = $masterStatus->successCount;
230 $status->failCount = $masterStatus->failCount;
231
232 return $status;
233 }
234
244 public function consistencyCheck( array $paths ) {
245 $status = $this->newStatus();
246 if ( $this->syncChecks == 0 || count( $this->backends ) <= 1 ) {
247 return $status; // skip checks
248 }
249
250 // Preload all of the stat info in as few round trips as possible
251 foreach ( $this->backends as $backend ) {
252 $realPaths = $this->substPaths( $paths, $backend );
253 $backend->preloadFileStat( [ 'srcs' => $realPaths, 'latest' => true ] );
254 }
255
256 foreach ( $paths as $path ) {
257 $params = [ 'src' => $path, 'latest' => true ];
258 // Get the state of the file on the master backend
259 $masterBackend = $this->backends[$this->masterIndex];
260 $masterParams = $this->substOpPaths( $params, $masterBackend );
261 $masterStat = $masterBackend->getFileStat( $masterParams );
262 if ( $masterStat === self::STAT_ERROR ) {
263 $status->fatal( 'backend-fail-stat', $path );
264 continue;
265 }
266 if ( $this->syncChecks & self::CHECK_SHA1 ) {
267 $masterSha1 = $masterBackend->getFileSha1Base36( $masterParams );
268 if ( ( $masterSha1 !== false ) !== (bool)$masterStat ) {
269 $status->fatal( 'backend-fail-hash', $path );
270 continue;
271 }
272 } else {
273 $masterSha1 = null; // unused
274 }
275
276 // Check if all clone backends agree with the master...
277 foreach ( $this->backends as $index => $cloneBackend ) {
278 if ( $index === $this->masterIndex ) {
279 continue; // master
280 }
281
282 // Get the state of the file on the clone backend
283 $cloneParams = $this->substOpPaths( $params, $cloneBackend );
284 $cloneStat = $cloneBackend->getFileStat( $cloneParams );
285
286 if ( $masterStat ) {
287 // File exists in the master backend
288 if ( !$cloneStat ) {
289 // File is missing from the clone backend
290 $status->fatal( 'backend-fail-synced', $path );
291 } elseif (
292 ( $this->syncChecks & self::CHECK_SIZE ) &&
293 $cloneStat['size'] !== $masterStat['size']
294 ) {
295 // File in the clone backend is different
296 $status->fatal( 'backend-fail-synced', $path );
297 } elseif (
298 ( $this->syncChecks & self::CHECK_TIME ) &&
299 abs(
300 (int)ConvertibleTimestamp::convert( TS_UNIX, $masterStat['mtime'] ) -
301 (int)ConvertibleTimestamp::convert( TS_UNIX, $cloneStat['mtime'] )
302 ) > 30
303 ) {
304 // File in the clone backend is significantly newer or older
305 $status->fatal( 'backend-fail-synced', $path );
306 } elseif (
307 ( $this->syncChecks & self::CHECK_SHA1 ) &&
308 $cloneBackend->getFileSha1Base36( $cloneParams ) !== $masterSha1
309 ) {
310 // File in the clone backend is different
311 $status->fatal( 'backend-fail-synced', $path );
312 }
313 } else {
314 // File does not exist in the master backend
315 if ( $cloneStat ) {
316 // Stray file exists in the clone backend
317 $status->fatal( 'backend-fail-synced', $path );
318 }
319 }
320 }
321 }
322
323 return $status;
324 }
325
332 public function accessibilityCheck( array $paths ) {
333 $status = $this->newStatus();
334 if ( count( $this->backends ) <= 1 ) {
335 return $status; // skip checks
336 }
337
338 foreach ( $paths as $path ) {
339 foreach ( $this->backends as $backend ) {
340 $realPath = $this->substPaths( $path, $backend );
341 if ( !$backend->isPathUsableInternal( $realPath ) ) {
342 $status->fatal( 'backend-fail-usable', $path );
343 }
344 }
345 }
346
347 return $status;
348 }
349
360 public function resyncFiles( array $paths, $resyncMode = true ) {
361 $status = $this->newStatus();
362
363 $fname = __METHOD__;
364 foreach ( $paths as $path ) {
365 $params = [ 'src' => $path, 'latest' => true ];
366 // Get the state of the file on the master backend
367 $masterBackend = $this->backends[$this->masterIndex];
368 $masterParams = $this->substOpPaths( $params, $masterBackend );
369 $masterPath = $masterParams['src'];
370 $masterStat = $masterBackend->getFileStat( $masterParams );
371 if ( $masterStat === self::STAT_ERROR ) {
372 $status->fatal( 'backend-fail-stat', $path );
373 $this->logger->error( "$fname: file '$masterPath' is not available" );
374 continue;
375 }
376 $masterSha1 = $masterBackend->getFileSha1Base36( $masterParams );
377 if ( ( $masterSha1 !== false ) !== (bool)$masterStat ) {
378 $status->fatal( 'backend-fail-hash', $path );
379 $this->logger->error( "$fname: file '$masterPath' hash does not match stat" );
380 continue;
381 }
382
383 // Check of all clone backends agree with the master...
384 foreach ( $this->backends as $index => $cloneBackend ) {
385 if ( $index === $this->masterIndex ) {
386 continue; // master
387 }
388
389 // Get the state of the file on the clone backend
390 $cloneParams = $this->substOpPaths( $params, $cloneBackend );
391 $clonePath = $cloneParams['src'];
392 $cloneStat = $cloneBackend->getFileStat( $cloneParams );
393 if ( $cloneStat === self::STAT_ERROR ) {
394 $status->fatal( 'backend-fail-stat', $path );
395 $this->logger->error( "$fname: file '$clonePath' is not available" );
396 continue;
397 }
398 $cloneSha1 = $cloneBackend->getFileSha1Base36( $cloneParams );
399 if ( ( $cloneSha1 !== false ) !== (bool)$cloneStat ) {
400 $status->fatal( 'backend-fail-hash', $path );
401 $this->logger->error( "$fname: file '$clonePath' hash does not match stat" );
402 continue;
403 }
404
405 if ( $masterSha1 === $cloneSha1 ) {
406 // File is either the same in both backends or absent from both backends
407 $this->logger->debug( "$fname: file '$clonePath' matches '$masterPath'" );
408 } elseif ( $masterSha1 !== false ) {
409 // File is either missing from or different in the clone backend
410 if (
411 $resyncMode === 'conservative' &&
412 $cloneStat &&
413 // @phan-suppress-next-line PhanTypeArraySuspiciousNullable
414 $cloneStat['mtime'] > $masterStat['mtime']
415 ) {
416 // Do not replace files with older ones; reduces the risk of data loss
417 $status->fatal( 'backend-fail-synced', $path );
418 } else {
419 // Copy the master backend file to the clone backend in overwrite mode
420 $fsFile = $masterBackend->getLocalReference( $masterParams );
421 $status->merge( $cloneBackend->quickStore( [
422 'src' => $fsFile,
423 'dst' => $clonePath
424 ] ) );
425 }
426 } elseif ( $masterStat === false ) {
427 // Stray file exists in the clone backend
428 if ( $resyncMode === 'conservative' ) {
429 // Do not delete stray files; reduces the risk of data loss
430 $status->fatal( 'backend-fail-synced', $path );
431 $this->logger->error( "$fname: not allowed to delete file '$clonePath'" );
432 } else {
433 // Delete the stay file from the clone backend
434 $status->merge( $cloneBackend->quickDelete( [ 'src' => $clonePath ] ) );
435 }
436 }
437 }
438 }
439
440 if ( !$status->isOK() ) {
441 $this->logger->error( "$fname: failed to resync: " . FormatJson::encode( $paths ) );
442 }
443
444 return $status;
445 }
446
453 protected function fileStoragePathsForOps( array $ops ) {
454 $paths = [];
455 foreach ( $ops as $op ) {
456 if ( isset( $op['src'] ) ) {
457 // For things like copy/move/delete with "ignoreMissingSource" and there
458 // is no source file, nothing should happen and there should be no errors.
459 if ( empty( $op['ignoreMissingSource'] )
460 || $this->fileExists( [ 'src' => $op['src'] ] )
461 ) {
462 $paths[] = $op['src'];
463 }
464 }
465 if ( isset( $op['srcs'] ) ) {
466 $paths = array_merge( $paths, $op['srcs'] );
467 }
468 if ( isset( $op['dst'] ) ) {
469 $paths[] = $op['dst'];
470 }
471 }
472
473 return array_values( array_unique( array_filter( $paths, [ FileBackend::class, 'isStoragePath' ] ) ) );
474 }
475
484 protected function substOpBatchPaths( array $ops, FileBackendStore $backend ) {
485 $newOps = []; // operations
486 foreach ( $ops as $op ) {
487 $newOp = $op; // operation
488 foreach ( [ 'src', 'srcs', 'dst', 'dir' ] as $par ) {
489 if ( isset( $newOp[$par] ) ) { // string or array
490 $newOp[$par] = $this->substPaths( $newOp[$par], $backend );
491 }
492 }
493 $newOps[] = $newOp;
494 }
495
496 return $newOps;
497 }
498
506 protected function substOpPaths( array $ops, FileBackendStore $backend ) {
507 $newOps = $this->substOpBatchPaths( [ $ops ], $backend );
508
509 return $newOps[0];
510 }
511
519 protected function substPaths( $paths, FileBackendStore $backend ) {
520 return preg_replace(
521 '!^mwstore://' . preg_quote( $this->name, '!' ) . '/!',
522 StringUtils::escapeRegexReplacement( "mwstore://{$backend->getName()}/" ),
523 $paths // string or array
524 );
525 }
526
534 protected function unsubstPaths( $paths, FileBackendStore $backend ) {
535 return preg_replace(
536 '!^mwstore://' . preg_quote( $backend->getName(), '!' ) . '/!',
537 StringUtils::escapeRegexReplacement( "mwstore://{$this->name}/" ),
538 $paths // string or array
539 );
540 }
541
546 protected function hasVolatileSources( array $ops ) {
547 foreach ( $ops as $op ) {
548 if ( $op['op'] === 'store' && !isset( $op['srcRef'] ) ) {
549 return true; // source file might be deleted anytime after do*Operations()
550 }
551 }
552
553 return false;
554 }
555
556 protected function doQuickOperationsInternal( array $ops, array $opts ) {
557 $status = $this->newStatus();
558 // Do the operations on the master backend; setting StatusValue fields
559 $realOps = $this->substOpBatchPaths( $ops, $this->backends[$this->masterIndex] );
560 $masterStatus = $this->backends[$this->masterIndex]->doQuickOperations( $realOps );
561 $status->merge( $masterStatus );
562 // Propagate the operations to the clone backends...
563 foreach ( $this->backends as $index => $backend ) {
564 if ( $index === $this->masterIndex ) {
565 continue; // done already
566 }
567
568 $realOps = $this->substOpBatchPaths( $ops, $backend );
569 if ( $this->asyncWrites && !$this->hasVolatileSources( $ops ) ) {
570 DeferredUpdates::addCallableUpdate(
571 static function () use ( $backend, $realOps ) {
572 $backend->doQuickOperations( $realOps );
573 }
574 );
575 } else {
576 $status->merge( $backend->doQuickOperations( $realOps ) );
577 }
578 }
579 // Make 'success', 'successCount', and 'failCount' fields reflect
580 // the overall operation, rather than all the batches for each backend.
581 // Do this by only using success values from the master backend's batch.
582 $status->success = $masterStatus->success;
583 $status->successCount = $masterStatus->successCount;
584 $status->failCount = $masterStatus->failCount;
585
586 return $status;
587 }
588
589 protected function doPrepare( array $params ) {
590 return $this->doDirectoryOp( 'prepare', $params );
591 }
592
593 protected function doSecure( array $params ) {
594 return $this->doDirectoryOp( 'secure', $params );
595 }
596
597 protected function doPublish( array $params ) {
598 return $this->doDirectoryOp( 'publish', $params );
599 }
600
601 protected function doClean( array $params ) {
602 return $this->doDirectoryOp( 'clean', $params );
603 }
604
610 protected function doDirectoryOp( $method, array $params ) {
611 $status = $this->newStatus();
612
613 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
614 $masterStatus = $this->backends[$this->masterIndex]->$method( $realParams );
615 $status->merge( $masterStatus );
616
617 foreach ( $this->backends as $index => $backend ) {
618 if ( $index === $this->masterIndex ) {
619 continue; // already done
620 }
621
622 $realParams = $this->substOpPaths( $params, $backend );
623 if ( $this->asyncWrites ) {
624 DeferredUpdates::addCallableUpdate(
625 static function () use ( $backend, $method, $realParams ) {
626 $backend->$method( $realParams );
627 }
628 );
629 } else {
630 $status->merge( $backend->$method( $realParams ) );
631 }
632 }
633
634 return $status;
635 }
636
637 public function concatenate( array $params ) {
638 $status = $this->newStatus();
639 // We are writing to an FS file, so we don't need to do this per-backend
640 $index = $this->getReadIndexFromParams( $params );
641 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
642
643 $status->merge( $this->backends[$index]->concatenate( $realParams ) );
644
645 return $status;
646 }
647
648 public function fileExists( array $params ) {
649 $index = $this->getReadIndexFromParams( $params );
650 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
651
652 return $this->backends[$index]->fileExists( $realParams );
653 }
654
655 public function getFileTimestamp( array $params ) {
656 $index = $this->getReadIndexFromParams( $params );
657 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
658
659 return $this->backends[$index]->getFileTimestamp( $realParams );
660 }
661
662 public function getFileSize( array $params ) {
663 $index = $this->getReadIndexFromParams( $params );
664 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
665
666 return $this->backends[$index]->getFileSize( $realParams );
667 }
668
669 public function getFileStat( array $params ) {
670 $index = $this->getReadIndexFromParams( $params );
671 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
672
673 return $this->backends[$index]->getFileStat( $realParams );
674 }
675
676 public function getFileXAttributes( array $params ) {
677 $index = $this->getReadIndexFromParams( $params );
678 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
679
680 return $this->backends[$index]->getFileXAttributes( $realParams );
681 }
682
683 public function getFileContentsMulti( array $params ) {
684 $index = $this->getReadIndexFromParams( $params );
685 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
686
687 $contentsM = $this->backends[$index]->getFileContentsMulti( $realParams );
688
689 $contents = []; // (path => FSFile) mapping using the proxy backend's name
690 foreach ( $contentsM as $path => $data ) {
691 $contents[$this->unsubstPaths( $path, $this->backends[$index] )] = $data;
692 }
693
694 return $contents;
695 }
696
697 public function getFileSha1Base36( array $params ) {
698 $index = $this->getReadIndexFromParams( $params );
699 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
700
701 return $this->backends[$index]->getFileSha1Base36( $realParams );
702 }
703
704 public function getFileProps( array $params ) {
705 $index = $this->getReadIndexFromParams( $params );
706 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
707
708 return $this->backends[$index]->getFileProps( $realParams );
709 }
710
711 public function streamFile( array $params ) {
712 $index = $this->getReadIndexFromParams( $params );
713 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
714
715 return $this->backends[$index]->streamFile( $realParams );
716 }
717
718 public function getLocalReferenceMulti( array $params ) {
719 $index = $this->getReadIndexFromParams( $params );
720 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
721
722 $fsFilesM = $this->backends[$index]->getLocalReferenceMulti( $realParams );
723
724 $fsFiles = []; // (path => FSFile) mapping using the proxy backend's name
725 foreach ( $fsFilesM as $path => $fsFile ) {
726 $fsFiles[$this->unsubstPaths( $path, $this->backends[$index] )] = $fsFile;
727 }
728
729 return $fsFiles;
730 }
731
732 public function getLocalCopyMulti( array $params ) {
733 $index = $this->getReadIndexFromParams( $params );
734 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
735
736 $tempFilesM = $this->backends[$index]->getLocalCopyMulti( $realParams );
737
738 $tempFiles = []; // (path => TempFSFile) mapping using the proxy backend's name
739 foreach ( $tempFilesM as $path => $tempFile ) {
740 $tempFiles[$this->unsubstPaths( $path, $this->backends[$index] )] = $tempFile;
741 }
742
743 return $tempFiles;
744 }
745
746 public function getFileHttpUrl( array $params ) {
747 $index = $this->getReadIndexFromParams( $params );
748 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
749
750 return $this->backends[$index]->getFileHttpUrl( $realParams );
751 }
752
753 public function directoryExists( array $params ) {
754 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
755
756 return $this->backends[$this->masterIndex]->directoryExists( $realParams );
757 }
758
759 public function getDirectoryList( array $params ) {
760 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
761
762 return $this->backends[$this->masterIndex]->getDirectoryList( $realParams );
763 }
764
765 public function getFileList( array $params ) {
766 if ( isset( $params['forWrite'] ) && $params['forWrite'] ) {
767 return $this->getFileListForWrite( $params );
768 }
769
770 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
771 return $this->backends[$this->masterIndex]->getFileList( $realParams );
772 }
773
774 private function getFileListForWrite( $params ) {
775 $files = [];
776 // Get the list of thumbnails from all backends to allow
777 // deleting all of them. Otherwise, old thumbnails existing on
778 // one backend only won't get updated in reupload (T331138).
779 foreach ( $this->backends as $backend ) {
780 $realParams = $this->substOpPaths( $params, $backend );
781 $iterator = $backend->getFileList( $realParams );
782 if ( $iterator !== null ) {
783 foreach ( $iterator as $file ) {
784 $files[] = $file;
785 }
786 }
787 }
788
789 return array_unique( $files );
790 }
791
792 public function getFeatures() {
793 return $this->backends[$this->masterIndex]->getFeatures();
794 }
795
796 public function clearCache( array $paths = null ) {
797 foreach ( $this->backends as $backend ) {
798 $realPaths = is_array( $paths ) ? $this->substPaths( $paths, $backend ) : null;
799 $backend->clearCache( $realPaths );
800 }
801 }
802
803 public function preloadCache( array $paths ) {
804 $realPaths = $this->substPaths( $paths, $this->backends[$this->readIndex] );
805 $this->backends[$this->readIndex]->preloadCache( $realPaths );
806 }
807
808 public function preloadFileStat( array $params ) {
809 $index = $this->getReadIndexFromParams( $params );
810 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
811
812 return $this->backends[$index]->preloadFileStat( $realParams );
813 }
814
815 public function getScopedLocksForOps( array $ops, StatusValue $status ) {
816 $realOps = $this->substOpBatchPaths( $ops, $this->backends[$this->masterIndex] );
817 $fileOps = $this->backends[$this->masterIndex]->getOperationsInternal( $realOps );
818 // Get the paths to lock from the master backend
819 $paths = $this->backends[$this->masterIndex]->getPathsToLockForOpsInternal( $fileOps );
820 // Get the paths under the proxy backend's name
821 $pbPaths = [
822 LockManager::LOCK_UW => $this->unsubstPaths(
823 $paths[LockManager::LOCK_UW],
824 $this->backends[$this->masterIndex]
825 ),
826 LockManager::LOCK_EX => $this->unsubstPaths(
827 $paths[LockManager::LOCK_EX],
828 $this->backends[$this->masterIndex]
829 )
830 ];
831
832 // Actually acquire the locks
833 return $this->getScopedFileLocks( $pbPaths, 'mixed', $status );
834 }
835
840 protected function getReadIndexFromParams( array $params ) {
841 return !empty( $params['latest'] ) ? $this->masterIndex : $this->readIndex;
842 }
843}
844
846class_alias( FileBackendMultiWrite::class, 'FileBackendMultiWrite' );
array $params
The job parameters.
Resource locking handling.
Defer callable updates to run later in the PHP process.
JSON formatter wrapper class.
Generic operation result class Has warning/error list, boolean status and arbitrary value.
A collection of static methods to play with strings.
Proxy backend that mirrors writes to several internal backends.
getScopedLocksForOps(array $ops, StatusValue $status)
Get an array of scoped locks needed for a batch of file operations.
getFileXAttributes(array $params)
Get metadata about a file at a storage path in the backend.
streamFile(array $params)
Stream the content of the file at a storage path in the backend.
getFileSha1Base36(array $params)
Get a SHA-1 hash of the content of the file at a storage path in the backend.
fileStoragePathsForOps(array $ops)
Get a list of file storage paths to read or write for a list of operations.
substPaths( $paths, FileBackendStore $backend)
Substitute the backend of storage paths with an internal backend's name.
__construct(array $config)
Construct a proxy backend that consists of several internal backends.
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
consistencyCheck(array $paths)
Check that a set of files are consistent across all internal backends.
directoryExists(array $params)
Check if a directory exists at a given storage path.
FileBackendStore[] $backends
Prioritized list of FileBackendStore objects.
preloadFileStat(array $params)
Preload file stat information (concurrently if possible) into in-process cache.
getFileProps(array $params)
Get the properties of the content of the file at a storage path in the backend.
int $readIndex
Index of read affinity backend.
unsubstPaths( $paths, FileBackendStore $backend)
Substitute the backend of internal storage paths with the proxy backend's name.
getFeatures()
Get the a bitfield of extra features supported by the backend medium.
fileExists(array $params)
Check if a file exists at a storage path in the backend.
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
getFileHttpUrl(array $params)
Return an HTTP URL to a given file that requires no authentication to use.
substOpBatchPaths(array $ops, FileBackendStore $backend)
Substitute the backend name in storage path parameters for a set of operations with that of a given i...
concatenate(array $params)
Concatenate a list of storage files into a single file system file.
getLocalReferenceMulti(array $params)
Like getLocalReference() except it takes an array of storage paths and yields an order-preserved map ...
preloadCache(array $paths)
Preload persistent file stat cache and property cache into in-process cache.
getLocalCopyMulti(array $params)
Like getLocalCopy() except it takes an array of storage paths and yields an order preserved-map of st...
accessibilityCheck(array $paths)
Check that a set of file paths are usable across all internal backends.
getFileSize(array $params)
Get the size (bytes) of a file at a storage path in the backend.
getDirectoryList(array $params)
Get an iterator to list all directories under a storage directory.
resyncFiles(array $paths, $resyncMode=true)
Check that a set of files are consistent across all internal backends and re-synchronize those files ...
getFileTimestamp(array $params)
Get the last-modified timestamp of the file at a storage path.
substOpPaths(array $ops, FileBackendStore $backend)
Same as substOpBatchPaths() but for a single operation.
getFileList(array $params)
Get an iterator to list all stored files under a storage directory.
getFileContentsMulti(array $params)
Like getFileContents() except it takes an array of storage paths and returns an order preserved map o...
Base class for all backends using particular storage medium.
Base class for all file backend classes (including multi-write backends).
string $name
Unique backend name.
getName()
Get the unique backend name.
newStatus(... $args)
Yields the result of the status wrapper callback on either:
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
string $domainId
Unique domain name.