MediaWiki master
FileBackendMultiWrite.php
Go to the documentation of this file.
1<?php
9
10namespace Wikimedia\FileBackend;
11
12use InvalidArgumentException;
13use LogicException;
14use Shellbox\Command\BoxedCommand;
15use StatusValue;
16use Wikimedia\LockManager\LockManager;
17use Wikimedia\StringUtils\StringUtils;
18
39 protected $backends = [];
40
42 protected $masterIndex = -1;
44 protected $readIndex = -1;
45
47 protected $asyncWrites = false;
48
63 public function __construct( array $config ) {
64 parent::__construct( $config );
65 $this->asyncWrites = isset( $config['replication'] ) && $config['replication'] === 'async';
66 // Construct backends here rather than via registration
67 // to keep these backends hidden from outside the proxy.
68 $namesUsed = [];
69 foreach ( $config['backends'] as $index => $beConfig ) {
70 $name = $beConfig['name'];
71 if ( isset( $namesUsed[$name] ) ) { // don't break FileOp predicates
72 throw new LogicException( "Two or more backends defined with the name $name." );
73 }
74 $namesUsed[$name] = 1;
75 // Alter certain sub-backend settings
76 unset( $beConfig['readOnly'] ); // use proxy backend setting
77 unset( $beConfig['lockManager'] ); // lock under proxy backend
78 $beConfig['domainId'] = $this->domainId; // use the proxy backend wiki ID
79 $beConfig['logger'] = $this->logger; // use the proxy backend logger
80 if ( !empty( $beConfig['isMultiMaster'] ) ) {
81 if ( $this->masterIndex >= 0 ) {
82 throw new LogicException( 'More than one master backend defined.' );
83 }
84 $this->masterIndex = $index; // this is the "master"
85 }
86 if ( !empty( $beConfig['readAffinity'] ) ) {
87 $this->readIndex = $index; // prefer this for reads
88 }
89 // Create sub-backend object
90 if ( !isset( $beConfig['class'] ) ) {
91 throw new InvalidArgumentException( 'No class given for a backend config.' );
92 }
93 $class = $beConfig['class'];
94 $this->backends[$index] = new $class( $beConfig );
95 }
96 if ( $this->masterIndex < 0 ) { // need backends and must have a master
97 throw new LogicException( 'No master backend defined.' );
98 }
99 if ( $this->readIndex < 0 ) {
100 $this->readIndex = $this->masterIndex; // default
101 }
102 }
103
105 final protected function doOperationsInternal( array $ops, array $opts ) {
106 $status = $this->newStatus();
107
108 $fname = __METHOD__;
109 $mbe = $this->backends[$this->masterIndex]; // convenience
110
111 // Acquire any locks as needed
112 $scopeLock = null;
113 if ( empty( $opts['nonLocking'] ) ) {
114 $scopeLock = $this->getScopedLocksForOps( $ops, $status );
115 if ( !$status->isOK() ) {
116 return $status; // abort
117 }
118 }
119 // Get the list of paths to read/write
120 $relevantPaths = $this->fileStoragePathsForOps( $ops );
121 // Clear any cache entries (after locks acquired)
122 $this->clearCache( $relevantPaths );
123 $opts['preserveCache'] = true;
124 // Actually attempt the operation batch on the master backend
125 $realOps = $this->substOpBatchPaths( $ops, $mbe );
126 $masterStatus = $mbe->doOperations( $realOps, $opts );
127 $status->merge( $masterStatus );
128 // Propagate the operations to the clone backends if there were no unexpected errors
129 // and everything didn't fail due to predicted errors. If $ops only had one operation,
130 // this might avoid backend sync inconsistencies.
131 if ( $masterStatus->isOK() && $masterStatus->successCount > 0 ) {
132 foreach ( $this->backends as $index => $backend ) {
133 if ( $index === $this->masterIndex ) {
134 continue; // done already
135 }
136
137 $realOps = $this->substOpBatchPaths( $ops, $backend );
138 if ( $this->asyncWrites && !$this->hasVolatileSources( $ops ) ) {
139 // Bind $scopeLock to the callback to preserve locks
140 $this->callNowOrLater(
141 function () use (
142 // @phan-suppress-next-line PhanUnusedClosureUseVariable
143 $backend, $realOps, $opts, $scopeLock, $relevantPaths, $fname
144 ) {
145 $this->logger->debug(
146 "$fname: '{$backend->getName()}' async replication; paths: " .
147 implode( ', ', $relevantPaths )
148 );
149 $backend->doOperations( $realOps, $opts );
150 }
151 );
152 } else {
153 $this->logger->debug(
154 "$fname: '{$backend->getName()}' sync replication; paths: " .
155 implode( ', ', $relevantPaths )
156 );
157 $status->merge( $backend->doOperations( $realOps, $opts ) );
158 }
159 }
160 }
161 // Make 'success', 'successCount', and 'failCount' fields reflect
162 // the overall operation, rather than all the batches for each backend.
163 // Do this by only using success values from the master backend's batch.
164 $status->success = $masterStatus->success;
165 $status->successCount = $masterStatus->successCount;
166 $status->failCount = $masterStatus->failCount;
167
168 return $status;
169 }
170
177 protected function fileStoragePathsForOps( array $ops ) {
178 $paths = [];
179 foreach ( $ops as $op ) {
180 if ( isset( $op['src'] ) ) {
181 $paths[] = $op['src'];
182 }
183 if ( isset( $op['srcs'] ) ) {
184 $paths = array_merge( $paths, $op['srcs'] );
185 }
186 if ( isset( $op['dst'] ) ) {
187 $paths[] = $op['dst'];
188 }
189 }
190
191 return array_values( array_unique( array_filter(
192 $paths,
194 ) ) );
195 }
196
205 protected function substOpBatchPaths( array $ops, FileBackendStore $backend ) {
206 $newOps = []; // operations
207 foreach ( $ops as $op ) {
208 $newOp = $op; // operation
209 foreach ( [ 'src', 'srcs', 'dst', 'dir' ] as $par ) {
210 if ( isset( $newOp[$par] ) ) { // string or array
211 $newOp[$par] = $this->substPaths( $newOp[$par], $backend );
212 }
213 }
214 $newOps[] = $newOp;
215 }
216
217 return $newOps;
218 }
219
227 protected function substOpPaths( array $ops, FileBackendStore $backend ) {
228 $newOps = $this->substOpBatchPaths( [ $ops ], $backend );
229
230 return $newOps[0];
231 }
232
240 protected function substPaths( $paths, FileBackendStore $backend ) {
241 return preg_replace(
242 '!^mwstore://' . preg_quote( $this->name, '!' ) . '/!',
243 StringUtils::escapeRegexReplacement( "mwstore://{$backend->getName()}/" ),
244 $paths // string or array
245 );
246 }
247
255 protected function unsubstPaths( $paths, FileBackendStore $backend ) {
256 return preg_replace(
257 '!^mwstore://' . preg_quote( $backend->getName(), '!' ) . '/!',
258 StringUtils::escapeRegexReplacement( "mwstore://{$this->name}/" ),
259 $paths // string or array
260 );
261 }
262
267 protected function hasVolatileSources( array $ops ) {
268 foreach ( $ops as $op ) {
269 if ( $op['op'] === 'store' && !isset( $op['srcRef'] ) ) {
270 return true; // source file might be deleted anytime after do*Operations()
271 }
272 }
273
274 return false;
275 }
276
278 protected function doQuickOperationsInternal( array $ops, array $opts ) {
279 $status = $this->newStatus();
280 // Do the operations on the master backend; setting StatusValue fields
281 $realOps = $this->substOpBatchPaths( $ops, $this->backends[$this->masterIndex] );
282 $masterStatus = $this->backends[$this->masterIndex]->doQuickOperations( $realOps );
283 $status->merge( $masterStatus );
284 // Propagate the operations to the clone backends...
285 foreach ( $this->backends as $index => $backend ) {
286 if ( $index === $this->masterIndex ) {
287 continue; // done already
288 }
289
290 $realOps = $this->substOpBatchPaths( $ops, $backend );
291 if ( $this->asyncWrites && !$this->hasVolatileSources( $ops ) ) {
292 $this->callNowOrLater(
293 static function () use ( $backend, $realOps ) {
294 $backend->doQuickOperations( $realOps );
295 }
296 );
297 } else {
298 $status->merge( $backend->doQuickOperations( $realOps ) );
299 }
300 }
301 // Make 'success', 'successCount', and 'failCount' fields reflect
302 // the overall operation, rather than all the batches for each backend.
303 // Do this by only using success values from the master backend's batch.
304 $status->success = $masterStatus->success;
305 $status->successCount = $masterStatus->successCount;
306 $status->failCount = $masterStatus->failCount;
307
308 return $status;
309 }
310
312 protected function doPrepare( array $params ) {
313 return $this->doDirectoryOp( 'prepare', $params );
314 }
315
317 protected function doSecure( array $params ) {
318 return $this->doDirectoryOp( 'secure', $params );
319 }
320
322 protected function doPublish( array $params ) {
323 return $this->doDirectoryOp( 'publish', $params );
324 }
325
327 protected function doClean( array $params ) {
328 return $this->doDirectoryOp( 'clean', $params );
329 }
330
336 protected function doDirectoryOp( $method, array $params ) {
337 $status = $this->newStatus();
338
339 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
340 $masterStatus = $this->backends[$this->masterIndex]->$method( $realParams );
341 $status->merge( $masterStatus );
342
343 foreach ( $this->backends as $index => $backend ) {
344 if ( $index === $this->masterIndex ) {
345 continue; // already done
346 }
347
348 $realParams = $this->substOpPaths( $params, $backend );
349 if ( $this->asyncWrites ) {
350 $this->callNowOrLater(
351 static function () use ( $backend, $method, $realParams ) {
352 $backend->$method( $realParams );
353 }
354 );
355 } else {
356 $status->merge( $backend->$method( $realParams ) );
357 }
358 }
359
360 return $status;
361 }
362
364 public function concatenate( array $params ) {
365 $status = $this->newStatus();
366 // We are writing to an FS file, so we don't need to do this per-backend
367 $index = $this->getReadIndexFromParams( $params );
368 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
369
370 $status->merge( $this->backends[$index]->concatenate( $realParams ) );
371
372 return $status;
373 }
374
376 public function fileExists( array $params ) {
377 $index = $this->getReadIndexFromParams( $params );
378 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
379
380 return $this->backends[$index]->fileExists( $realParams );
381 }
382
384 public function getFileTimestamp( array $params ) {
385 $index = $this->getReadIndexFromParams( $params );
386 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
387
388 return $this->backends[$index]->getFileTimestamp( $realParams );
389 }
390
392 public function getFileSize( array $params ) {
393 $index = $this->getReadIndexFromParams( $params );
394 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
395
396 return $this->backends[$index]->getFileSize( $realParams );
397 }
398
400 public function getFileStat( array $params ) {
401 $index = $this->getReadIndexFromParams( $params );
402 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
403
404 return $this->backends[$index]->getFileStat( $realParams );
405 }
406
408 public function getFileXAttributes( array $params ) {
409 $index = $this->getReadIndexFromParams( $params );
410 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
411
412 return $this->backends[$index]->getFileXAttributes( $realParams );
413 }
414
416 public function getFileContentsMulti( array $params ) {
417 $index = $this->getReadIndexFromParams( $params );
418 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
419
420 $contentsM = $this->backends[$index]->getFileContentsMulti( $realParams );
421
422 $contents = []; // (path => FSFile) mapping using the proxy backend's name
423 foreach ( $contentsM as $path => $data ) {
424 $contents[$this->unsubstPaths( $path, $this->backends[$index] )] = $data;
425 }
426
427 return $contents;
428 }
429
431 public function getFileSha1Base36( array $params ) {
432 $index = $this->getReadIndexFromParams( $params );
433 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
434
435 return $this->backends[$index]->getFileSha1Base36( $realParams );
436 }
437
439 public function getFileProps( array $params ) {
440 $index = $this->getReadIndexFromParams( $params );
441 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
442
443 return $this->backends[$index]->getFileProps( $realParams );
444 }
445
447 public function streamFile( array $params ) {
448 $index = $this->getReadIndexFromParams( $params );
449 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
450
451 return $this->backends[$index]->streamFile( $realParams );
452 }
453
455 public function getLocalReferenceMulti( array $params ) {
456 $index = $this->getReadIndexFromParams( $params );
457 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
458
459 $fsFilesM = $this->backends[$index]->getLocalReferenceMulti( $realParams );
460
461 $fsFiles = []; // (path => FSFile) mapping using the proxy backend's name
462 foreach ( $fsFilesM as $path => $fsFile ) {
463 $fsFiles[$this->unsubstPaths( $path, $this->backends[$index] )] = $fsFile;
464 }
465
466 return $fsFiles;
467 }
468
470 public function getLocalCopyMulti( array $params ) {
471 $index = $this->getReadIndexFromParams( $params );
472 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
473
474 $tempFilesM = $this->backends[$index]->getLocalCopyMulti( $realParams );
475
476 $tempFiles = []; // (path => TempFSFile) mapping using the proxy backend's name
477 foreach ( $tempFilesM as $path => $tempFile ) {
478 $tempFiles[$this->unsubstPaths( $path, $this->backends[$index] )] = $tempFile;
479 }
480
481 return $tempFiles;
482 }
483
485 public function getFileHttpUrl( array $params ) {
486 $index = $this->getReadIndexFromParams( $params );
487 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
488
489 return $this->backends[$index]->getFileHttpUrl( $realParams );
490 }
491
493 public function addShellboxInputFile( BoxedCommand $command, string $boxedName,
494 array $params
495 ) {
496 $index = $this->getReadIndexFromParams( $params );
497 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
498 return $this->backends[$index]->addShellboxInputFile( $command, $boxedName, $realParams );
499 }
500
502 public function directoryExists( array $params ) {
503 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
504
505 return $this->backends[$this->masterIndex]->directoryExists( $realParams );
506 }
507
509 public function getDirectoryList( array $params ) {
510 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
511
512 return $this->backends[$this->masterIndex]->getDirectoryList( $realParams );
513 }
514
516 public function getFileList( array $params ) {
517 if ( isset( $params['forWrite'] ) && $params['forWrite'] ) {
518 return $this->getFileListForWrite( $params );
519 }
520
521 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
522 return $this->backends[$this->masterIndex]->getFileList( $realParams );
523 }
524
525 private function getFileListForWrite( array $params ): array {
526 $files = [];
527 // Get the list of thumbnails from all backends to allow
528 // deleting all of them. Otherwise, old thumbnails existing on
529 // one backend only won't get updated in reupload (T331138).
530 foreach ( $this->backends as $backend ) {
531 $realParams = $this->substOpPaths( $params, $backend );
532 $iterator = $backend->getFileList( $realParams );
533 if ( $iterator !== null ) {
534 foreach ( $iterator as $file ) {
535 $files[] = $file;
536 }
537 }
538 }
539
540 return array_unique( $files );
541 }
542
544 public function getFeatures() {
545 return $this->backends[$this->masterIndex]->getFeatures();
546 }
547
549 public function clearCache( ?array $paths = null ) {
550 foreach ( $this->backends as $backend ) {
551 $realPaths = is_array( $paths ) ? $this->substPaths( $paths, $backend ) : null;
552 $backend->clearCache( $realPaths );
553 }
554 }
555
557 public function preloadCache( array $paths ) {
558 $realPaths = $this->substPaths( $paths, $this->backends[$this->readIndex] );
559 $this->backends[$this->readIndex]->preloadCache( $realPaths );
560 }
561
563 public function preloadFileStat( array $params ) {
564 $index = $this->getReadIndexFromParams( $params );
565 $realParams = $this->substOpPaths( $params, $this->backends[$index] );
566
567 return $this->backends[$index]->preloadFileStat( $realParams );
568 }
569
571 public function getScopedLocksForOps( array $ops, StatusValue $status ) {
572 $realOps = $this->substOpBatchPaths( $ops, $this->backends[$this->masterIndex] );
573 $fileOps = $this->backends[$this->masterIndex]->getOperationsInternal( $realOps );
574 // Get the paths to lock from the master backend
575 $paths = $this->backends[$this->masterIndex]->getPathsToLockForOpsInternal( $fileOps );
576 // Get the paths under the proxy backend's name
577 $pbPaths = [
578 LockManager::LOCK_UW => $this->unsubstPaths(
579 $paths[LockManager::LOCK_UW],
580 $this->backends[$this->masterIndex]
581 ),
582 LockManager::LOCK_EX => $this->unsubstPaths(
583 $paths[LockManager::LOCK_EX],
584 $this->backends[$this->masterIndex]
585 )
586 ];
587
588 // Actually acquire the locks
589 return $this->getScopedFileLocks( $pbPaths, 'mixed', $status );
590 }
591
596 protected function getReadIndexFromParams( array $params ) {
597 return !empty( $params['latest'] ) ? $this->masterIndex : $this->readIndex;
598 }
599}
600
602class_alias( FileBackendMultiWrite::class, 'FileBackendMultiWrite' );
Proxy backend that mirrors writes to several internal backends.
getScopedLocksForOps(array $ops, StatusValue $status)
Get an array of scoped locks needed for a batch of file operations.Normally, FileBackend\doOperations...
getFileXAttributes(array $params)
Get metadata about a file at a storage path in the backend.If the file does not exist,...
streamFile(array $params)
Stream the content of the file at a storage path in the backend.If the file does not exists,...
getFileSha1Base36(array $params)
Get a SHA-1 hash of the content of the file at a storage path in the backend.FileBackend\SHA1_FAILstr...
doPublish(array $params)
FileBackend\publish() StatusValue
fileStoragePathsForOps(array $ops)
Get a list of file storage paths to read or write for a list of operations.
substPaths( $paths, FileBackendStore $backend)
Substitute the backend of storage paths with an internal backend's name.
doClean(array $params)
FileBackend\clean() StatusValue
__construct(array $config)
Construct a proxy backend that consists of several internal backends.
directoryExists(array $params)
Check if a directory exists at a given storage path.For backends using key/value stores,...
FileBackendStore[] $backends
Prioritized list of FileBackendStore objects.
preloadFileStat(array $params)
Preload file stat information (concurrently if possible) into in-process cache.This should be used wh...
getFileProps(array $params)
Get the properties of the content of the file at a storage path in the backend.This gives the result ...
doSecure(array $params)
FileBackend\secure() StatusValue
int $readIndex
Index of read affinity backend.
unsubstPaths( $paths, FileBackendStore $backend)
Substitute the backend of internal storage paths with the proxy backend's name.
getFeatures()
Get the a bitfield of extra features supported by the backend medium.to overrideint Bitfield of FileB...
fileExists(array $params)
Check if a file exists at a storage path in the backend.This returns false if only a directory exists...
doQuickOperationsInternal(array $ops, array $opts)
FileBackend\doQuickOperations() StatusValue 1.20
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.If the file does not exist,...
getFileHttpUrl(array $params)
Return an HTTP URL to a given file that requires no authentication to use.The URL may be pre-authenti...
substOpBatchPaths(array $ops, FileBackendStore $backend)
Substitute the backend name in storage path parameters for a set of operations with that of a given i...
concatenate(array $params)
Concatenate a list of storage files into a single file system file.The target path should refer to a ...
getLocalReferenceMulti(array $params)
Like getLocalReference() except it takes an array of storage paths and yields an order-preserved map ...
preloadCache(array $paths)
Preload persistent file stat cache and property cache into in-process cache.This should be used when ...
getLocalCopyMulti(array $params)
Like getLocalCopy() except it takes an array of storage paths and yields an order preserved-map of st...
doOperationsInternal(array $ops, array $opts)
FileBackend\doOperations() StatusValue
getFileSize(array $params)
Get the size (bytes) of a file at a storage path in the backend.FileBackend\SIZE_FAILint|false File s...
getDirectoryList(array $params)
Get an iterator to list all directories under a storage directory.If the directory is of the form "mw...
clearCache(?array $paths=null)
Invalidate any in-process file stat and property cache.If $paths is given, then only the cache for th...
getFileTimestamp(array $params)
Get the last-modified timestamp of the file at a storage path.FileBackend\TIMESTAMP_FAILstring|false ...
substOpPaths(array $ops, FileBackendStore $backend)
Same as substOpBatchPaths() but for a single operation.
getFileList(array $params)
Get an iterator to list all stored files under a storage directory.If the directory is of the form "m...
getFileContentsMulti(array $params)
Like getFileContents() except it takes an array of storage paths and returns an order preserved map o...
addShellboxInputFile(BoxedCommand $command, string $boxedName, array $params)
Add a file to a Shellbox command as an input file.StatusValue 1.43
doPrepare(array $params)
FileBackend\prepare() StatusValue Good status without value for success, fatal otherwise.
Base class for all backends using particular storage medium.
Base class for all file backend classes (including multi-write backends).
string $name
Unique backend name.
getName()
Get the unique backend name.
static isStoragePath( $path)
Check if a given path is a "mwstore://" path.
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
newStatus( $message=null,... $params)
Yields the result of the status wrapper callback on either:
string $domainId
Unique domain name.