MediaWiki master
FileBackendStore.php
Go to the documentation of this file.
1<?php
24use Wikimedia\AtEase\AtEase;
25use Wikimedia\Timestamp\ConvertibleTimestamp;
26
41abstract class FileBackendStore extends FileBackend {
43 protected $memCache;
45 protected $srvCache;
47 protected $cheapCache;
49 protected $expensiveCache;
50
52 protected $shardViaHashLevels = [];
53
55 protected $mimeCallback;
56
57 protected $maxFileSize = 32 * 1024 * 1024 * 1024; // integer bytes (32GiB)
58
59 protected const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries
60 protected const CACHE_CHEAP_SIZE = 500; // integer; max entries in "cheap cache"
61 protected const CACHE_EXPENSIVE_SIZE = 5; // integer; max entries in "expensive cache"
62
64 protected const RES_ABSENT = false;
66 protected const RES_ERROR = null;
67
69 protected const ABSENT_NORMAL = 'FNE-N';
71 protected const ABSENT_LATEST = 'FNE-L';
72
86 public function __construct( array $config ) {
87 parent::__construct( $config );
88 $this->mimeCallback = $config['mimeCallback'] ?? null;
89 $this->srvCache = new EmptyBagOStuff(); // disabled by default
90 $this->memCache = WANObjectCache::newEmpty(); // disabled by default
91 $this->cheapCache = new MapCacheLRU( self::CACHE_CHEAP_SIZE );
92 $this->expensiveCache = new MapCacheLRU( self::CACHE_EXPENSIVE_SIZE );
93 }
94
102 final public function maxFileSizeInternal() {
103 return min( $this->maxFileSize, PHP_INT_MAX );
104 }
105
116 abstract public function isPathUsableInternal( $storagePath );
117
136 final public function createInternal( array $params ) {
138 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
139
140 if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
141 $status = $this->newStatus( 'backend-fail-maxsize',
142 $params['dst'], $this->maxFileSizeInternal() );
143 } else {
144 $status = $this->doCreateInternal( $params );
145 $this->clearCache( [ $params['dst'] ] );
146 if ( $params['dstExists'] ?? true ) {
147 $this->deleteFileCache( $params['dst'] ); // persistent cache
148 }
149 }
150
151 return $status;
152 }
153
159 abstract protected function doCreateInternal( array $params );
160
179 final public function storeInternal( array $params ) {
181 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
182
183 if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
184 $status = $this->newStatus( 'backend-fail-maxsize',
185 $params['dst'], $this->maxFileSizeInternal() );
186 } else {
187 $status = $this->doStoreInternal( $params );
188 $this->clearCache( [ $params['dst'] ] );
189 if ( $params['dstExists'] ?? true ) {
190 $this->deleteFileCache( $params['dst'] ); // persistent cache
191 }
192 }
193
194 return $status;
195 }
196
202 abstract protected function doStoreInternal( array $params );
203
223 final public function copyInternal( array $params ) {
225 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
226
227 $status = $this->doCopyInternal( $params );
228 $this->clearCache( [ $params['dst'] ] );
229 if ( $params['dstExists'] ?? true ) {
230 $this->deleteFileCache( $params['dst'] ); // persistent cache
231 }
232
233 return $status;
234 }
235
241 abstract protected function doCopyInternal( array $params );
242
257 final public function deleteInternal( array $params ) {
259 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
260
261 $status = $this->doDeleteInternal( $params );
262 $this->clearCache( [ $params['src'] ] );
263 $this->deleteFileCache( $params['src'] ); // persistent cache
264 return $status;
265 }
266
272 abstract protected function doDeleteInternal( array $params );
273
293 final public function moveInternal( array $params ) {
295 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
296
297 $status = $this->doMoveInternal( $params );
298 $this->clearCache( [ $params['src'], $params['dst'] ] );
299 $this->deleteFileCache( $params['src'] ); // persistent cache
300 if ( $params['dstExists'] ?? true ) {
301 $this->deleteFileCache( $params['dst'] ); // persistent cache
302 }
303
304 return $status;
305 }
306
313 protected function doMoveInternal( array $params ) {
314 unset( $params['async'] ); // two steps, won't work here :)
315 $nsrc = FileBackend::normalizeStoragePath( $params['src'] );
316 $ndst = FileBackend::normalizeStoragePath( $params['dst'] );
317 // Copy source to dest
318 $status = $this->copyInternal( $params );
319 if ( $nsrc !== $ndst && $status->isOK() ) {
320 // Delete source (only fails due to races or network problems)
321 $status->merge( $this->deleteInternal( [ 'src' => $params['src'] ] ) );
322 $status->setResult( true, $status->value ); // ignore delete() errors
323 }
324
325 return $status;
326 }
327
342 final public function describeInternal( array $params ) {
344 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
345
346 if ( count( $params['headers'] ) ) {
347 $status = $this->doDescribeInternal( $params );
348 $this->clearCache( [ $params['src'] ] );
349 $this->deleteFileCache( $params['src'] ); // persistent cache
350 } else {
351 $status = $this->newStatus(); // nothing to do
352 }
353
354 return $status;
355 }
356
363 protected function doDescribeInternal( array $params ) {
364 return $this->newStatus();
365 }
366
374 final public function nullInternal( array $params ) {
375 return $this->newStatus();
376 }
377
378 final public function concatenate( array $params ) {
380 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
381 $status = $this->newStatus();
382
383 // Try to lock the source files for the scope of this function
385 $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
386 if ( $status->isOK() ) {
387 // Actually do the file concatenation...
388 $start_time = microtime( true );
389 $status->merge( $this->doConcatenate( $params ) );
390 $sec = microtime( true ) - $start_time;
391 if ( !$status->isOK() ) {
392 $this->logger->error( static::class . "-{$this->name}" .
393 " failed to concatenate " . count( $params['srcs'] ) . " file(s) [$sec sec]" );
394 }
395 }
396
397 return $status;
398 }
399
406 protected function doConcatenate( array $params ) {
407 $status = $this->newStatus();
408 $tmpPath = $params['dst'];
409 unset( $params['latest'] );
410
411 // Check that the specified temp file is valid...
412 AtEase::suppressWarnings();
413 $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 );
414 AtEase::restoreWarnings();
415 if ( !$ok ) { // not present or not empty
416 $status->fatal( 'backend-fail-opentemp', $tmpPath );
417
418 return $status;
419 }
420
421 // Get local FS versions of the chunks needed for the concatenation...
422 $fsFiles = $this->getLocalReferenceMulti( $params );
423 foreach ( $fsFiles as $path => &$fsFile ) {
424 if ( !$fsFile ) { // chunk failed to download?
425 $fsFile = $this->getLocalReference( [ 'src' => $path ] );
426 if ( !$fsFile ) { // retry failed?
427 $status->fatal(
428 $fsFile === self::RES_ERROR ? 'backend-fail-read' : 'backend-fail-notexists',
429 $path
430 );
431
432 return $status;
433 }
434 }
435 }
436 unset( $fsFile ); // unset reference so we can reuse $fsFile
437
438 // Get a handle for the destination temp file
439 $tmpHandle = fopen( $tmpPath, 'ab' );
440 if ( $tmpHandle === false ) {
441 $status->fatal( 'backend-fail-opentemp', $tmpPath );
442
443 return $status;
444 }
445
446 // Build up the temp file using the source chunks (in order)...
447 foreach ( $fsFiles as $virtualSource => $fsFile ) {
448 // Get a handle to the local FS version
449 $sourceHandle = fopen( $fsFile->getPath(), 'rb' );
450 if ( $sourceHandle === false ) {
451 fclose( $tmpHandle );
452 $status->fatal( 'backend-fail-read', $virtualSource );
453
454 return $status;
455 }
456 // Append chunk to file (pass chunk size to avoid magic quotes)
457 if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
458 fclose( $sourceHandle );
459 fclose( $tmpHandle );
460 $status->fatal( 'backend-fail-writetemp', $tmpPath );
461
462 return $status;
463 }
464 fclose( $sourceHandle );
465 }
466 if ( !fclose( $tmpHandle ) ) {
467 $status->fatal( 'backend-fail-closetemp', $tmpPath );
468
469 return $status;
470 }
471
472 clearstatcache(); // temp file changed
473
474 return $status;
475 }
476
480 final protected function doPrepare( array $params ) {
482 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
483 $status = $this->newStatus();
484
485 [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
486 if ( $dir === null ) {
487 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
488
489 return $status; // invalid storage path
490 }
491
492 if ( $shard !== null ) { // confined to a single container/shard
493 $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
494 } else { // directory is on several shards
495 $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
496 [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
497 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
498 $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
499 }
500 }
501
502 return $status;
503 }
504
513 protected function doPrepareInternal( $container, $dir, array $params ) {
514 return $this->newStatus();
515 }
516
517 final protected function doSecure( array $params ) {
519 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
520 $status = $this->newStatus();
521
522 [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
523 if ( $dir === null ) {
524 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
525
526 return $status; // invalid storage path
527 }
528
529 if ( $shard !== null ) { // confined to a single container/shard
530 $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
531 } else { // directory is on several shards
532 $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
533 [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
534 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
535 $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
536 }
537 }
538
539 return $status;
540 }
541
550 protected function doSecureInternal( $container, $dir, array $params ) {
551 return $this->newStatus();
552 }
553
554 final protected function doPublish( array $params ) {
556 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
557 $status = $this->newStatus();
558
559 [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
560 if ( $dir === null ) {
561 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
562
563 return $status; // invalid storage path
564 }
565
566 if ( $shard !== null ) { // confined to a single container/shard
567 $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
568 } else { // directory is on several shards
569 $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
570 [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
571 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
572 $status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) );
573 }
574 }
575
576 return $status;
577 }
578
587 protected function doPublishInternal( $container, $dir, array $params ) {
588 return $this->newStatus();
589 }
590
591 final protected function doClean( array $params ) {
593 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
594 $status = $this->newStatus();
595
596 // Recursive: first delete all empty subdirs recursively
597 if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) {
598 $subDirsRel = $this->getTopDirectoryList( [ 'dir' => $params['dir'] ] );
599 if ( $subDirsRel !== null ) { // no errors
600 foreach ( $subDirsRel as $subDirRel ) {
601 $subDir = $params['dir'] . "/{$subDirRel}"; // full path
602 $status->merge( $this->doClean( [ 'dir' => $subDir ] + $params ) );
603 }
604 unset( $subDirsRel ); // free directory for rmdir() on Windows (for FS backends)
605 }
606 }
607
608 [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
609 if ( $dir === null ) {
610 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
611
612 return $status; // invalid storage path
613 }
614
615 // Attempt to lock this directory...
616 $filesLockEx = [ $params['dir'] ];
618 $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
619 if ( !$status->isOK() ) {
620 return $status; // abort
621 }
622
623 if ( $shard !== null ) { // confined to a single container/shard
624 $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
625 $this->deleteContainerCache( $fullCont ); // purge cache
626 } else { // directory is on several shards
627 $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
628 [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
629 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
630 $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
631 $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache
632 }
633 }
634
635 return $status;
636 }
637
646 protected function doCleanInternal( $container, $dir, array $params ) {
647 return $this->newStatus();
648 }
649
650 final public function fileExists( array $params ) {
652 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
653
654 $stat = $this->getFileStat( $params );
655 if ( is_array( $stat ) ) {
656 return true;
657 }
658
659 return $stat === self::RES_ABSENT ? false : self::EXISTENCE_ERROR;
660 }
661
662 final public function getFileTimestamp( array $params ) {
664 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
665
666 $stat = $this->getFileStat( $params );
667 if ( is_array( $stat ) ) {
668 return $stat['mtime'];
669 }
670
671 return self::TIMESTAMP_FAIL; // all failure cases
672 }
673
674 final public function getFileSize( array $params ) {
676 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
677
678 $stat = $this->getFileStat( $params );
679 if ( is_array( $stat ) ) {
680 return $stat['size'];
681 }
682
683 return self::SIZE_FAIL; // all failure cases
684 }
685
686 final public function getFileStat( array $params ) {
688 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
689
690 $path = self::normalizeStoragePath( $params['src'] );
691 if ( $path === null ) {
692 return self::STAT_ERROR; // invalid storage path
693 }
694
695 // Whether to bypass cache except for process cache entries loaded directly from
696 // high consistency backend queries (caller handles any cache flushing and locking)
697 $latest = !empty( $params['latest'] );
698 // Whether to ignore cache entries missing the SHA-1 field for existing files
699 $requireSHA1 = !empty( $params['requireSHA1'] );
700
701 $stat = $this->cheapCache->getField( $path, 'stat', self::CACHE_TTL );
702 // Load the persistent stat cache into process cache if needed
703 if ( !$latest ) {
704 if (
705 // File stat is not in process cache
706 $stat === null ||
707 // Key/value store backends might opportunistically set file stat process
708 // cache entries from object listings that do not include the SHA-1. In that
709 // case, loading the persistent stat cache will likely yield the SHA-1.
710 ( $requireSHA1 && is_array( $stat ) && !isset( $stat['sha1'] ) )
711 ) {
712 $this->primeFileCache( [ $path ] );
713 // Get any newly process-cached entry
714 $stat = $this->cheapCache->getField( $path, 'stat', self::CACHE_TTL );
715 }
716 }
717
718 if ( is_array( $stat ) ) {
719 if (
720 ( !$latest || !empty( $stat['latest'] ) ) &&
721 ( !$requireSHA1 || isset( $stat['sha1'] ) )
722 ) {
723 return $stat;
724 }
725 } elseif ( $stat === self::ABSENT_LATEST ) {
726 return self::STAT_ABSENT;
727 } elseif ( $stat === self::ABSENT_NORMAL ) {
728 if ( !$latest ) {
729 return self::STAT_ABSENT;
730 }
731 }
732
733 // Load the file stat from the backend and update caches
734 $stat = $this->doGetFileStat( $params );
735 $this->ingestFreshFileStats( [ $path => $stat ], $latest );
736
737 if ( is_array( $stat ) ) {
738 return $stat;
739 }
740
741 return $stat === self::RES_ERROR ? self::STAT_ERROR : self::STAT_ABSENT;
742 }
743
751 final protected function ingestFreshFileStats( array $stats, $latest ) {
752 $success = true;
753
754 foreach ( $stats as $path => $stat ) {
755 if ( is_array( $stat ) ) {
756 // Strongly consistent backends might automatically set this flag
757 $stat['latest'] ??= $latest;
758
759 $this->cheapCache->setField( $path, 'stat', $stat );
760 if ( isset( $stat['sha1'] ) ) {
761 // Some backends store the SHA-1 hash as metadata
762 $this->cheapCache->setField(
763 $path,
764 'sha1',
765 [ 'hash' => $stat['sha1'], 'latest' => $latest ]
766 );
767 }
768 if ( isset( $stat['xattr'] ) ) {
769 // Some backends store custom headers/metadata
770 $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] );
771 $this->cheapCache->setField(
772 $path,
773 'xattr',
774 [ 'map' => $stat['xattr'], 'latest' => $latest ]
775 );
776 }
777 // Update persistent cache (@TODO: set all entries in one batch)
778 $this->setFileCache( $path, $stat );
779 } elseif ( $stat === self::RES_ABSENT ) {
780 $this->cheapCache->setField(
781 $path,
782 'stat',
783 $latest ? self::ABSENT_LATEST : self::ABSENT_NORMAL
784 );
785 $this->cheapCache->setField(
786 $path,
787 'xattr',
788 [ 'map' => self::XATTRS_FAIL, 'latest' => $latest ]
789 );
790 $this->cheapCache->setField(
791 $path,
792 'sha1',
793 [ 'hash' => self::SHA1_FAIL, 'latest' => $latest ]
794 );
795 $this->logger->debug(
796 __METHOD__ . ': File {path} does not exist',
797 [ 'path' => $path ]
798 );
799 } else {
800 $success = false;
801 $this->logger->error(
802 __METHOD__ . ': Could not stat file {path}',
803 [ 'path' => $path ]
804 );
805 }
806 }
807
808 return $success;
809 }
810
816 abstract protected function doGetFileStat( array $params );
817
818 public function getFileContentsMulti( array $params ) {
820 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
821
822 $params = $this->setConcurrencyFlags( $params );
823 $contents = $this->doGetFileContentsMulti( $params );
824 foreach ( $contents as $path => $content ) {
825 if ( !is_string( $content ) ) {
826 $contents[$path] = self::CONTENT_FAIL; // used for all failure cases
827 }
828 }
829
830 return $contents;
831 }
832
839 protected function doGetFileContentsMulti( array $params ) {
840 $contents = [];
841 foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
842 if ( $fsFile instanceof FSFile ) {
843 AtEase::suppressWarnings();
844 $content = file_get_contents( $fsFile->getPath() );
845 AtEase::restoreWarnings();
846 $contents[$path] = is_string( $content ) ? $content : self::RES_ERROR;
847 } else {
848 // self::RES_ERROR or self::RES_ABSENT
849 $contents[$path] = $fsFile;
850 }
851 }
852
853 return $contents;
854 }
855
856 final public function getFileXAttributes( array $params ) {
858 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
859
860 $path = self::normalizeStoragePath( $params['src'] );
861 if ( $path === null ) {
862 return self::XATTRS_FAIL; // invalid storage path
863 }
864 $latest = !empty( $params['latest'] ); // use latest data?
865 if ( $this->cheapCache->hasField( $path, 'xattr', self::CACHE_TTL ) ) {
866 $stat = $this->cheapCache->getField( $path, 'xattr' );
867 // If we want the latest data, check that this cached
868 // value was in fact fetched with the latest available data.
869 if ( !$latest || $stat['latest'] ) {
870 return $stat['map'];
871 }
872 }
873 $fields = $this->doGetFileXAttributes( $params );
874 if ( is_array( $fields ) ) {
875 $fields = self::normalizeXAttributes( $fields );
876 $this->cheapCache->setField(
877 $path,
878 'xattr',
879 [ 'map' => $fields, 'latest' => $latest ]
880 );
881 } elseif ( $fields === self::RES_ABSENT ) {
882 $this->cheapCache->setField(
883 $path,
884 'xattr',
885 [ 'map' => self::XATTRS_FAIL, 'latest' => $latest ]
886 );
887 } else {
888 $fields = self::XATTRS_FAIL; // used for all failure cases
889 }
890
891 return $fields;
892 }
893
900 protected function doGetFileXAttributes( array $params ) {
901 return [ 'headers' => [], 'metadata' => [] ]; // not supported
902 }
903
904 final public function getFileSha1Base36( array $params ) {
906 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
907
908 $path = self::normalizeStoragePath( $params['src'] );
909 if ( $path === null ) {
910 return self::SHA1_FAIL; // invalid storage path
911 }
912 $latest = !empty( $params['latest'] ); // use latest data?
913 if ( $this->cheapCache->hasField( $path, 'sha1', self::CACHE_TTL ) ) {
914 $stat = $this->cheapCache->getField( $path, 'sha1' );
915 // If we want the latest data, check that this cached
916 // value was in fact fetched with the latest available data.
917 if ( !$latest || $stat['latest'] ) {
918 return $stat['hash'];
919 }
920 }
921 $sha1 = $this->doGetFileSha1Base36( $params );
922 if ( is_string( $sha1 ) ) {
923 $this->cheapCache->setField(
924 $path,
925 'sha1',
926 [ 'hash' => $sha1, 'latest' => $latest ]
927 );
928 } elseif ( $sha1 === self::RES_ABSENT ) {
929 $this->cheapCache->setField(
930 $path,
931 'sha1',
932 [ 'hash' => self::SHA1_FAIL, 'latest' => $latest ]
933 );
934 } else {
935 $sha1 = self::SHA1_FAIL; // used for all failure cases
936 }
937
938 return $sha1;
939 }
940
947 protected function doGetFileSha1Base36( array $params ) {
948 $fsFile = $this->getLocalReference( $params );
949 if ( $fsFile instanceof FSFile ) {
950 $sha1 = $fsFile->getSha1Base36();
951
952 return is_string( $sha1 ) ? $sha1 : self::RES_ERROR;
953 }
954
955 return $fsFile === self::RES_ERROR ? self::RES_ERROR : self::RES_ABSENT;
956 }
957
958 final public function getFileProps( array $params ) {
960 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
961
962 $fsFile = $this->getLocalReference( $params );
963
964 return $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
965 }
966
967 final public function getLocalReferenceMulti( array $params ) {
969 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
970
971 $params = $this->setConcurrencyFlags( $params );
972
973 $fsFiles = []; // (path => FSFile)
974 $latest = !empty( $params['latest'] ); // use latest data?
975 // Reuse any files already in process cache...
976 foreach ( $params['srcs'] as $src ) {
978 if ( $path === null ) {
979 $fsFiles[$src] = self::RES_ERROR; // invalid storage path
980 } elseif ( $this->expensiveCache->hasField( $path, 'localRef' ) ) {
981 $val = $this->expensiveCache->getField( $path, 'localRef' );
982 // If we want the latest data, check that this cached
983 // value was in fact fetched with the latest available data.
984 if ( !$latest || $val['latest'] ) {
985 $fsFiles[$src] = $val['object'];
986 }
987 }
988 }
989 // Fetch local references of any remaining files...
990 $params['srcs'] = array_diff( $params['srcs'], array_keys( $fsFiles ) );
991 foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
992 if ( $fsFile instanceof FSFile ) {
993 $fsFiles[$path] = $fsFile;
994 $this->expensiveCache->setField(
995 $path,
996 'localRef',
997 [ 'object' => $fsFile, 'latest' => $latest ]
998 );
999 } else {
1000 // self::RES_ERROR or self::RES_ABSENT
1001 $fsFiles[$path] = $fsFile;
1002 }
1003 }
1004
1005 return $fsFiles;
1006 }
1007
1014 protected function doGetLocalReferenceMulti( array $params ) {
1015 return $this->doGetLocalCopyMulti( $params );
1016 }
1017
1018 final public function getLocalCopyMulti( array $params ) {
1020 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1021
1022 $params = $this->setConcurrencyFlags( $params );
1023
1024 return $this->doGetLocalCopyMulti( $params );
1025 }
1026
1032 abstract protected function doGetLocalCopyMulti( array $params );
1033
1040 public function getFileHttpUrl( array $params ) {
1041 return self::TEMPURL_ERROR; // not supported
1042 }
1043
1044 final public function streamFile( array $params ) {
1046 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1047 $status = $this->newStatus();
1048
1049 // Always set some fields for subclass convenience
1050 $params['options'] ??= [];
1051 $params['headers'] ??= [];
1052
1053 // Don't stream it out as text/html if there was a PHP error
1054 if ( ( empty( $params['headless'] ) || $params['headers'] ) && headers_sent() ) {
1055 print "Headers already sent, terminating.\n";
1056 $status->fatal( 'backend-fail-stream', $params['src'] );
1057 return $status;
1058 }
1059
1060 $status->merge( $this->doStreamFile( $params ) );
1061
1062 return $status;
1063 }
1064
1071 protected function doStreamFile( array $params ) {
1072 $status = $this->newStatus();
1073
1074 $flags = 0;
1075 $flags |= !empty( $params['headless'] ) ? HTTPFileStreamer::STREAM_HEADLESS : 0;
1076 $flags |= !empty( $params['allowOB'] ) ? HTTPFileStreamer::STREAM_ALLOW_OB : 0;
1077
1078 $fsFile = $this->getLocalReference( $params );
1079 if ( $fsFile ) {
1080 $streamer = new HTTPFileStreamer(
1081 $fsFile->getPath(),
1082 [
1083 'obResetFunc' => $this->obResetFunc,
1084 'streamMimeFunc' => $this->streamMimeFunc
1085 ]
1086 );
1087 $res = $streamer->stream( $params['headers'], true, $params['options'], $flags );
1088 } else {
1089 $res = false;
1090 HTTPFileStreamer::send404Message( $params['src'], $flags );
1091 }
1092
1093 if ( !$res ) {
1094 $status->fatal( 'backend-fail-stream', $params['src'] );
1095 }
1096
1097 return $status;
1098 }
1099
1100 final public function directoryExists( array $params ) {
1101 [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
1102 if ( $dir === null ) {
1103 return self::EXISTENCE_ERROR; // invalid storage path
1104 }
1105 if ( $shard !== null ) { // confined to a single container/shard
1106 return $this->doDirectoryExists( $fullCont, $dir, $params );
1107 } else { // directory is on several shards
1108 $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
1109 [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
1110 $res = false; // response
1111 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
1112 $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params );
1113 if ( $exists === true ) {
1114 $res = true;
1115 break; // found one!
1116 } elseif ( $exists === self::RES_ERROR ) {
1117 $res = self::EXISTENCE_ERROR;
1118 }
1119 }
1120
1121 return $res;
1122 }
1123 }
1124
1133 abstract protected function doDirectoryExists( $container, $dir, array $params );
1134
1135 final public function getDirectoryList( array $params ) {
1136 [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
1137 if ( $dir === null ) {
1138 return self::EXISTENCE_ERROR; // invalid storage path
1139 }
1140 if ( $shard !== null ) {
1141 // File listing is confined to a single container/shard
1142 return $this->getDirectoryListInternal( $fullCont, $dir, $params );
1143 } else {
1144 $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
1145 // File listing spans multiple containers/shards
1146 [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
1147
1148 return new FileBackendStoreShardDirIterator( $this,
1149 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
1150 }
1151 }
1152
1163 abstract public function getDirectoryListInternal( $container, $dir, array $params );
1164
1165 final public function getFileList( array $params ) {
1166 [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
1167 if ( $dir === null ) {
1168 return self::LIST_ERROR; // invalid storage path
1169 }
1170 if ( $shard !== null ) {
1171 // File listing is confined to a single container/shard
1172 return $this->getFileListInternal( $fullCont, $dir, $params );
1173 } else {
1174 $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
1175 // File listing spans multiple containers/shards
1176 [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
1177
1178 return new FileBackendStoreShardFileIterator( $this,
1179 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
1180 }
1181 }
1182
1193 abstract public function getFileListInternal( $container, $dir, array $params );
1194
1206 final public function getOperationsInternal( array $ops ) {
1207 $supportedOps = [
1208 'store' => StoreFileOp::class,
1209 'copy' => CopyFileOp::class,
1210 'move' => MoveFileOp::class,
1211 'delete' => DeleteFileOp::class,
1212 'create' => CreateFileOp::class,
1213 'describe' => DescribeFileOp::class,
1214 'null' => NullFileOp::class
1215 ];
1216
1217 $performOps = []; // array of FileOp objects
1218 // Build up ordered array of FileOps...
1219 foreach ( $ops as $operation ) {
1220 $opName = $operation['op'];
1221 if ( isset( $supportedOps[$opName] ) ) {
1222 $class = $supportedOps[$opName];
1223 // Get params for this operation
1224 $params = $operation;
1225 // Append the FileOp class
1226 $performOps[] = new $class( $this, $params, $this->logger );
1227 } else {
1228 throw new FileBackendError( "Operation '$opName' is not supported." );
1229 }
1230 }
1231
1232 return $performOps;
1233 }
1234
1245 final public function getPathsToLockForOpsInternal( array $performOps ) {
1246 // Build up a list of files to lock...
1247 $paths = [ 'sh' => [], 'ex' => [] ];
1248 foreach ( $performOps as $fileOp ) {
1249 $paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() );
1250 $paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() );
1251 }
1252 // Optimization: if doing an EX lock anyway, don't also set an SH one
1253 $paths['sh'] = array_diff( $paths['sh'], $paths['ex'] );
1254 // Get a shared lock on the parent directory of each path changed
1255 $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) );
1256
1257 return [
1258 LockManager::LOCK_UW => $paths['sh'],
1259 LockManager::LOCK_EX => $paths['ex']
1260 ];
1261 }
1262
1263 public function getScopedLocksForOps( array $ops, StatusValue $status ) {
1264 $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) );
1265
1266 return $this->getScopedFileLocks( $paths, 'mixed', $status );
1267 }
1268
1269 final protected function doOperationsInternal( array $ops, array $opts ) {
1271 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1272 $status = $this->newStatus();
1273
1274 // Fix up custom header name/value pairs
1275 $ops = array_map( [ $this, 'sanitizeOpHeaders' ], $ops );
1276 // Build up a list of FileOps and involved paths
1277 $fileOps = $this->getOperationsInternal( $ops );
1278 $pathsUsed = [];
1279 foreach ( $fileOps as $fileOp ) {
1280 $pathsUsed = array_merge( $pathsUsed, $fileOp->storagePathsReadOrChanged() );
1281 }
1282
1283 // Acquire any locks as needed for the scope of this function
1284 if ( empty( $opts['nonLocking'] ) ) {
1285 $pathsByLockType = $this->getPathsToLockForOpsInternal( $fileOps );
1287 $scopeLock = $this->getScopedFileLocks( $pathsByLockType, 'mixed', $status );
1288 if ( !$status->isOK() ) {
1289 return $status; // abort
1290 }
1291 }
1292
1293 // Clear any file cache entries (after locks acquired)
1294 if ( empty( $opts['preserveCache'] ) ) {
1295 $this->clearCache( $pathsUsed );
1296 }
1297
1298 // Enlarge the cache to fit the stat entries of these files
1299 $this->cheapCache->setMaxSize( max( 2 * count( $pathsUsed ), self::CACHE_CHEAP_SIZE ) );
1300
1301 // Load from the persistent container caches
1302 $this->primeContainerCache( $pathsUsed );
1303 // Get the latest stat info for all the files (having locked them)
1304 $ok = $this->preloadFileStat( [ 'srcs' => $pathsUsed, 'latest' => true ] );
1305
1306 if ( $ok ) {
1307 // Actually attempt the operation batch...
1308 $opts = $this->setConcurrencyFlags( $opts );
1309 $subStatus = FileOpBatch::attempt( $fileOps, $opts );
1310 } else {
1311 // If we could not even stat some files, then bail out
1312 $subStatus = $this->newStatus( 'backend-fail-internal', $this->name );
1313 foreach ( $ops as $i => $op ) { // mark each op as failed
1314 $subStatus->success[$i] = false;
1315 ++$subStatus->failCount;
1316 }
1317 $this->logger->error( static::class . "-{$this->name} " .
1318 " stat failure; aborted operations: " . FormatJson::encode( $ops ) );
1319 }
1320
1321 // Merge errors into StatusValue fields
1322 $status->merge( $subStatus );
1323 $status->success = $subStatus->success; // not done in merge()
1324
1325 // Shrink the stat cache back to normal size
1326 $this->cheapCache->setMaxSize( self::CACHE_CHEAP_SIZE );
1327
1328 return $status;
1329 }
1330
1331 final protected function doQuickOperationsInternal( array $ops, array $opts ) {
1333 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1334 $status = $this->newStatus();
1335
1336 // Fix up custom header name/value pairs
1337 $ops = array_map( [ $this, 'sanitizeOpHeaders' ], $ops );
1338 // Build up a list of FileOps and involved paths
1339 $fileOps = $this->getOperationsInternal( $ops );
1340 $pathsUsed = [];
1341 foreach ( $fileOps as $fileOp ) {
1342 $pathsUsed = array_merge( $pathsUsed, $fileOp->storagePathsReadOrChanged() );
1343 }
1344
1345 // Clear any file cache entries for involved paths
1346 $this->clearCache( $pathsUsed );
1347
1348 // Parallel ops may be disabled in config due to dependencies (e.g. needing popen())
1349 $async = ( $this->parallelize === 'implicit' && count( $ops ) > 1 );
1350 $maxConcurrency = $this->concurrency; // throttle
1352 $statuses = []; // array of (index => StatusValue)
1353 $fileOpHandles = []; // list of (index => handle) arrays
1354 $curFileOpHandles = []; // current handle batch
1355 // Perform the sync-only ops and build up op handles for the async ops...
1356 foreach ( $fileOps as $index => $fileOp ) {
1357 $subStatus = $async
1358 ? $fileOp->attemptAsyncQuick()
1359 : $fileOp->attemptQuick();
1360 if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { // async
1361 if ( count( $curFileOpHandles ) >= $maxConcurrency ) {
1362 $fileOpHandles[] = $curFileOpHandles; // push this batch
1363 $curFileOpHandles = [];
1364 }
1365 $curFileOpHandles[$index] = $subStatus->value; // keep index
1366 } else { // error or completed
1367 $statuses[$index] = $subStatus; // keep index
1368 }
1369 }
1370 if ( count( $curFileOpHandles ) ) {
1371 $fileOpHandles[] = $curFileOpHandles; // last batch
1372 }
1373 // Do all the async ops that can be done concurrently...
1374 foreach ( $fileOpHandles as $fileHandleBatch ) {
1375 $statuses += $this->executeOpHandlesInternal( $fileHandleBatch );
1376 }
1377 // Marshall and merge all the responses...
1378 foreach ( $statuses as $index => $subStatus ) {
1379 $status->merge( $subStatus );
1380 if ( $subStatus->isOK() ) {
1381 $status->success[$index] = true;
1382 ++$status->successCount;
1383 } else {
1384 $status->success[$index] = false;
1385 ++$status->failCount;
1386 }
1387 }
1388
1389 $this->clearCache( $pathsUsed );
1390
1391 return $status;
1392 }
1393
1403 final public function executeOpHandlesInternal( array $fileOpHandles ) {
1405 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1406
1407 foreach ( $fileOpHandles as $fileOpHandle ) {
1408 if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) {
1409 throw new InvalidArgumentException( "Expected FileBackendStoreOpHandle object." );
1410 } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) {
1411 throw new InvalidArgumentException( "Expected handle for this file backend." );
1412 }
1413 }
1414
1415 $statuses = $this->doExecuteOpHandlesInternal( $fileOpHandles );
1416 foreach ( $fileOpHandles as $fileOpHandle ) {
1417 $fileOpHandle->closeResources();
1418 }
1419
1420 return $statuses;
1421 }
1422
1432 protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1433 if ( count( $fileOpHandles ) ) {
1434 throw new FileBackendError( "Backend does not support asynchronous operations." );
1435 }
1436
1437 return [];
1438 }
1439
1451 protected function sanitizeOpHeaders( array $op ) {
1452 static $longs = [ 'content-disposition' ];
1453
1454 if ( isset( $op['headers'] ) ) { // op sets HTTP headers
1455 $newHeaders = [];
1456 foreach ( $op['headers'] as $name => $value ) {
1457 $name = strtolower( $name );
1458 $maxHVLen = in_array( $name, $longs ) ? INF : 255;
1459 if ( strlen( $name ) > 255 || strlen( $value ) > $maxHVLen ) {
1460 $this->logger->error( "Header '{header}' is too long.", [
1461 'filebackend' => $this->name,
1462 'header' => "$name: $value",
1463 ] );
1464 } else {
1465 $newHeaders[$name] = strlen( $value ) ? $value : ''; // null/false => ""
1466 }
1467 }
1468 $op['headers'] = $newHeaders;
1469 }
1470
1471 return $op;
1472 }
1473
1474 final public function preloadCache( array $paths ) {
1475 $fullConts = []; // full container names
1476 foreach ( $paths as $path ) {
1477 [ $fullCont, , ] = $this->resolveStoragePath( $path );
1478 $fullConts[] = $fullCont;
1479 }
1480 // Load from the persistent file and container caches
1481 $this->primeContainerCache( $fullConts );
1482 $this->primeFileCache( $paths );
1483 }
1484
1485 final public function clearCache( array $paths = null ) {
1486 if ( is_array( $paths ) ) {
1487 $paths = array_map( [ FileBackend::class, 'normalizeStoragePath' ], $paths );
1488 $paths = array_filter( $paths, 'strlen' ); // remove nulls
1489 }
1490 if ( $paths === null ) {
1491 $this->cheapCache->clear();
1492 $this->expensiveCache->clear();
1493 } else {
1494 foreach ( $paths as $path ) {
1495 $this->cheapCache->clear( $path );
1496 $this->expensiveCache->clear( $path );
1497 }
1498 }
1499 $this->doClearCache( $paths );
1500 }
1501
1510 protected function doClearCache( array $paths = null ) {
1511 }
1512
1513 final public function preloadFileStat( array $params ) {
1515 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1516
1517 $params['concurrency'] = ( $this->parallelize !== 'off' ) ? $this->concurrency : 1;
1518 $stats = $this->doGetFileStatMulti( $params );
1519 if ( $stats === null ) {
1520 return true; // not supported
1521 }
1522
1523 // Whether this queried the backend in high consistency mode
1524 $latest = !empty( $params['latest'] );
1525
1526 return $this->ingestFreshFileStats( $stats, $latest );
1527 }
1528
1542 protected function doGetFileStatMulti( array $params ) {
1543 return null; // not supported
1544 }
1545
1553 abstract protected function directoriesAreVirtual();
1554
1565 final protected static function isValidShortContainerName( $container ) {
1566 // Suffixes like '.xxx' (hex shard chars) or '.seg' (file segments)
1567 // might be used by subclasses. Reserve the dot character.
1568 // The only way dots end up in containers (e.g. resolveStoragePath)
1569 // is due to the wikiId container prefix or the above suffixes.
1570 return self::isValidContainerName( $container ) && !preg_match( '/[.]/', $container );
1571 }
1572
1582 final protected static function isValidContainerName( $container ) {
1583 // This accounts for NTFS, Swift, and Ceph restrictions
1584 // and disallows directory separators or traversal characters.
1585 // Note that matching strings URL encode to the same string;
1586 // in Swift/Ceph, the length restriction is *after* URL encoding.
1587 return (bool)preg_match( '/^[a-z0-9][a-z0-9-_.]{0,199}$/i', $container );
1588 }
1589
1603 final protected function resolveStoragePath( $storagePath ) {
1604 [ $backend, $shortCont, $relPath ] = self::splitStoragePath( $storagePath );
1605 if ( $backend === $this->name ) { // must be for this backend
1606 $relPath = self::normalizeContainerPath( $relPath );
1607 if ( $relPath !== null && self::isValidShortContainerName( $shortCont ) ) {
1608 // Get shard for the normalized path if this container is sharded
1609 $cShard = $this->getContainerShard( $shortCont, $relPath );
1610 // Validate and sanitize the relative path (backend-specific)
1611 $relPath = $this->resolveContainerPath( $shortCont, $relPath );
1612 if ( $relPath !== null ) {
1613 // Prepend any domain ID prefix to the container name
1614 $container = $this->fullContainerName( $shortCont );
1615 if ( self::isValidContainerName( $container ) ) {
1616 // Validate and sanitize the container name (backend-specific)
1617 $container = $this->resolveContainerName( "{$container}{$cShard}" );
1618 if ( $container !== null ) {
1619 return [ $container, $relPath, $cShard ];
1620 }
1621 }
1622 }
1623 }
1624 }
1625
1626 return [ null, null, null ];
1627 }
1628
1644 final protected function resolveStoragePathReal( $storagePath ) {
1645 [ $container, $relPath, $cShard ] = $this->resolveStoragePath( $storagePath );
1646 if ( $cShard !== null && substr( $relPath, -1 ) !== '/' ) {
1647 return [ $container, $relPath ];
1648 }
1649
1650 return [ null, null ];
1651 }
1652
1661 final protected function getContainerShard( $container, $relPath ) {
1662 [ $levels, $base, $repeat ] = $this->getContainerHashLevels( $container );
1663 if ( $levels == 1 || $levels == 2 ) {
1664 // Hash characters are either base 16 or 36
1665 $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]';
1666 // Get a regex that represents the shard portion of paths.
1667 // The concatenation of the captures gives us the shard.
1668 if ( $levels === 1 ) { // 16 or 36 shards per container
1669 $hashDirRegex = '(' . $char . ')';
1670 } else { // 256 or 1296 shards per container
1671 if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc")
1672 $hashDirRegex = $char . '/(' . $char . '{2})';
1673 } else { // short hash dir format (e.g. "a/b/c")
1674 $hashDirRegex = '(' . $char . ')/(' . $char . ')';
1675 }
1676 }
1677 // Allow certain directories to be above the hash dirs so as
1678 // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
1679 // They must be 2+ chars to avoid any hash directory ambiguity.
1680 $m = [];
1681 if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1682 return '.' . implode( '', array_slice( $m, 1 ) );
1683 }
1684
1685 return null; // failed to match
1686 }
1687
1688 return ''; // no sharding
1689 }
1690
1699 final public function isSingleShardPathInternal( $storagePath ) {
1700 [ , , $shard ] = $this->resolveStoragePath( $storagePath );
1701
1702 return ( $shard !== null );
1703 }
1704
1713 final protected function getContainerHashLevels( $container ) {
1714 if ( isset( $this->shardViaHashLevels[$container] ) ) {
1715 $config = $this->shardViaHashLevels[$container];
1716 $hashLevels = (int)$config['levels'];
1717 if ( $hashLevels == 1 || $hashLevels == 2 ) {
1718 $hashBase = (int)$config['base'];
1719 if ( $hashBase == 16 || $hashBase == 36 ) {
1720 return [ $hashLevels, $hashBase, $config['repeat'] ];
1721 }
1722 }
1723 }
1724
1725 return [ 0, 0, false ]; // no sharding
1726 }
1727
1734 final protected function getContainerSuffixes( $container ) {
1735 $shards = [];
1736 [ $digits, $base ] = $this->getContainerHashLevels( $container );
1737 if ( $digits > 0 ) {
1738 $numShards = $base ** $digits;
1739 for ( $index = 0; $index < $numShards; $index++ ) {
1740 $shards[] = '.' . Wikimedia\base_convert( (string)$index, 10, $base, $digits );
1741 }
1742 }
1743
1744 return $shards;
1745 }
1746
1753 final protected function fullContainerName( $container ) {
1754 if ( $this->domainId != '' ) {
1755 return "{$this->domainId}-$container";
1756 } else {
1757 return $container;
1758 }
1759 }
1760
1770 protected function resolveContainerName( $container ) {
1771 return $container;
1772 }
1773
1785 protected function resolveContainerPath( $container, $relStoragePath ) {
1786 return $relStoragePath;
1787 }
1788
1795 private function containerCacheKey( $container ) {
1796 return "filebackend:{$this->name}:{$this->domainId}:container:{$container}";
1797 }
1798
1805 final protected function setContainerCache( $container, array $val ) {
1806 if ( !$this->memCache->set( $this->containerCacheKey( $container ), $val, 14 * 86400 ) ) {
1807 $this->logger->warning( "Unable to set stat cache for container {container}.",
1808 [ 'filebackend' => $this->name, 'container' => $container ]
1809 );
1810 }
1811 }
1812
1819 final protected function deleteContainerCache( $container ) {
1820 if ( !$this->memCache->delete( $this->containerCacheKey( $container ), 300 ) ) {
1821 $this->logger->warning( "Unable to delete stat cache for container {container}.",
1822 [ 'filebackend' => $this->name, 'container' => $container ]
1823 );
1824 }
1825 }
1826
1834 final protected function primeContainerCache( array $items ) {
1836 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1837
1838 $paths = []; // list of storage paths
1839 $contNames = []; // (cache key => resolved container name)
1840 // Get all the paths/containers from the items...
1841 foreach ( $items as $item ) {
1842 if ( self::isStoragePath( $item ) ) {
1843 $paths[] = $item;
1844 } elseif ( is_string( $item ) ) { // full container name
1845 $contNames[$this->containerCacheKey( $item )] = $item;
1846 }
1847 }
1848 // Get all the corresponding cache keys for paths...
1849 foreach ( $paths as $path ) {
1850 [ $fullCont, , ] = $this->resolveStoragePath( $path );
1851 if ( $fullCont !== null ) { // valid path for this backend
1852 $contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
1853 }
1854 }
1855
1856 $contInfo = []; // (resolved container name => cache value)
1857 // Get all cache entries for these container cache keys...
1858 $values = $this->memCache->getMulti( array_keys( $contNames ) );
1859 foreach ( $values as $cacheKey => $val ) {
1860 $contInfo[$contNames[$cacheKey]] = $val;
1861 }
1862
1863 // Populate the container process cache for the backend...
1864 $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) );
1865 }
1866
1875 protected function doPrimeContainerCache( array $containerInfo ) {
1876 }
1877
1884 private function fileCacheKey( $path ) {
1885 return "filebackend:{$this->name}:{$this->domainId}:file:" . sha1( $path );
1886 }
1887
1896 final protected function setFileCache( $path, array $val ) {
1898 if ( $path === null ) {
1899 return; // invalid storage path
1900 }
1901 $mtime = (int)ConvertibleTimestamp::convert( TS_UNIX, $val['mtime'] );
1902 $ttl = $this->memCache->adaptiveTTL( $mtime, 7 * 86400, 300, 0.1 );
1903 $key = $this->fileCacheKey( $path );
1904 // Set the cache unless it is currently salted.
1905 if ( !$this->memCache->set( $key, $val, $ttl ) ) {
1906 $this->logger->warning( "Unable to set stat cache for file {path}.",
1907 [ 'filebackend' => $this->name, 'path' => $path ]
1908 );
1909 }
1910 }
1911
1920 final protected function deleteFileCache( $path ) {
1922 if ( $path === null ) {
1923 return; // invalid storage path
1924 }
1925 if ( !$this->memCache->delete( $this->fileCacheKey( $path ), 300 ) ) {
1926 $this->logger->warning( "Unable to delete stat cache for file {path}.",
1927 [ 'filebackend' => $this->name, 'path' => $path ]
1928 );
1929 }
1930 }
1931
1939 final protected function primeFileCache( array $items ) {
1941 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1942
1943 $paths = []; // list of storage paths
1944 $pathNames = []; // (cache key => storage path)
1945 // Get all the paths/containers from the items...
1946 foreach ( $items as $item ) {
1947 if ( self::isStoragePath( $item ) ) {
1949 if ( $path !== null ) {
1950 $paths[] = $path;
1951 }
1952 }
1953 }
1954 // Get all the corresponding cache keys for paths...
1955 foreach ( $paths as $path ) {
1956 [ , $rel, ] = $this->resolveStoragePath( $path );
1957 if ( $rel !== null ) { // valid path for this backend
1958 $pathNames[$this->fileCacheKey( $path )] = $path;
1959 }
1960 }
1961 // Get all cache entries for these file cache keys.
1962 // Note that negatives are not cached by getFileStat()/preloadFileStat().
1963 $values = $this->memCache->getMulti( array_keys( $pathNames ) );
1964 // Load all of the results into process cache...
1965 foreach ( array_filter( $values, 'is_array' ) as $cacheKey => $stat ) {
1966 $path = $pathNames[$cacheKey];
1967 // This flag only applies to stat info loaded directly
1968 // from a high consistency backend query to the process cache
1969 unset( $stat['latest'] );
1970
1971 $this->cheapCache->setField( $path, 'stat', $stat );
1972 if ( isset( $stat['sha1'] ) && strlen( $stat['sha1'] ) == 31 ) {
1973 // Some backends store SHA-1 as metadata
1974 $this->cheapCache->setField(
1975 $path,
1976 'sha1',
1977 [ 'hash' => $stat['sha1'], 'latest' => false ]
1978 );
1979 }
1980 if ( isset( $stat['xattr'] ) && is_array( $stat['xattr'] ) ) {
1981 // Some backends store custom headers/metadata
1982 $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] );
1983 $this->cheapCache->setField(
1984 $path,
1985 'xattr',
1986 [ 'map' => $stat['xattr'], 'latest' => false ]
1987 );
1988 }
1989 }
1990 }
1991
1999 final protected static function normalizeXAttributes( array $xattr ) {
2000 $newXAttr = [ 'headers' => [], 'metadata' => [] ];
2001
2002 foreach ( $xattr['headers'] as $name => $value ) {
2003 $newXAttr['headers'][strtolower( $name )] = $value;
2004 }
2005
2006 foreach ( $xattr['metadata'] as $name => $value ) {
2007 $newXAttr['metadata'][strtolower( $name )] = $value;
2008 }
2009
2010 return $newXAttr;
2011 }
2012
2019 final protected function setConcurrencyFlags( array $opts ) {
2020 $opts['concurrency'] = 1; // off
2021 if ( $this->parallelize === 'implicit' ) {
2022 if ( $opts['parallelize'] ?? true ) {
2023 $opts['concurrency'] = $this->concurrency;
2024 }
2025 } elseif ( $this->parallelize === 'explicit' ) {
2026 if ( !empty( $opts['parallelize'] ) ) {
2027 $opts['concurrency'] = $this->concurrency;
2028 }
2029 }
2030
2031 return $opts;
2032 }
2033
2043 protected function getContentType( $storagePath, $content, $fsPath ) {
2044 if ( $this->mimeCallback ) {
2045 return call_user_func_array( $this->mimeCallback, func_get_args() );
2046 }
2047
2048 $mime = ( $fsPath !== null ) ? mime_content_type( $fsPath ) : false;
2049 return $mime ?: 'unknown/unknown';
2050 }
2051}
Class representing a cache/ephemeral data store.
Definition BagOStuff.php:85
A BagOStuff object with no objects in it.
Class representing a non-directory file on the file system.
Definition FSFile.php:32
static placeholderProps()
Placeholder file properties to use for files that don't exist.
Definition FSFile.php:150
File backend exception for checked exceptions (e.g.
FileBackendStore helper class for performing asynchronous file operations.
Base class for all backends using particular storage medium.
preloadFileStat(array $params)
Preload file stat information (concurrently if possible) into in-process cache.
doGetLocalReferenceMulti(array $params)
resolveContainerName( $container)
Resolve a container name, checking if it's allowed by the backend.
static normalizeXAttributes(array $xattr)
Normalize file headers/metadata to the FileBackend::getFileXAttributes() format.
getLocalReferenceMulti(array $params)
Like getLocalReference() except it takes an array of storage paths and yields an order-preserved map ...
setFileCache( $path, array $val)
Set the cached stat info for a file path.
doPublish(array $params)
moveInternal(array $params)
Move a file from one storage path to another in the backend.
setContainerCache( $container, array $val)
Set the cached info for a container.
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path in the backend.
doGetFileContentsMulti(array $params)
doStreamFile(array $params)
doSecure(array $params)
MapCacheLRU $expensiveCache
Map of paths to large (RAM/disk) cache items.
doClean(array $params)
getFileTimestamp(array $params)
Get the last-modified timestamp of the file at a storage path.
doGetLocalCopyMulti(array $params)
getScopedLocksForOps(array $ops, StatusValue $status)
Get an array of scoped locks needed for a batch of file operations.
doCleanInternal( $container, $dir, array $params)
getPathsToLockForOpsInternal(array $performOps)
Get a list of storage paths to lock for a list of operations Returns an array with LockManager::LOCK_...
executeOpHandlesInternal(array $fileOpHandles)
Execute a list of FileBackendStoreOpHandle handles in parallel.
doPrepareInternal( $container, $dir, array $params)
concatenate(array $params)
Concatenate a list of storage files into a single file system file.
storeInternal(array $params)
Store a file into the backend from a file on disk.
getFileProps(array $params)
Get the properties of the content of the file at a storage path in the backend.
getDirectoryList(array $params)
Get an iterator to list all directories under a storage directory.
doQuickOperationsInternal(array $ops, array $opts)
resolveStoragePath( $storagePath)
Splits a storage path into an internal container name, an internal relative file name,...
copyInternal(array $params)
Copy a file from one storage path to another in the backend.
doStoreInternal(array $params)
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
doGetFileStat(array $params)
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
MapCacheLRU $cheapCache
Map of paths to small (RAM/disk) cache items.
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
doPublishInternal( $container, $dir, array $params)
doGetFileStatMulti(array $params)
Get file stat information (concurrently if possible) for several files.
getFileSha1Base36(array $params)
Get a SHA-1 hash of the content of the file at a storage path in the backend.
static isValidContainerName( $container)
Check if a full container name is valid.
ingestFreshFileStats(array $stats, $latest)
Ingest file stat entries that just came from querying the backend (not cache)
getFileListInternal( $container, $dir, array $params)
Do not call this function from places outside FileBackend.
doCopyInternal(array $params)
getLocalCopyMulti(array $params)
Like getLocalCopy() except it takes an array of storage paths and yields an order preserved-map of st...
getFileContentsMulti(array $params)
Like getFileContents() except it takes an array of storage paths and returns an order preserved map o...
doClearCache(array $paths=null)
Clears any additional stat caches for storage paths.
fullContainerName( $container)
Get the full container name, including the domain ID prefix.
primeFileCache(array $items)
Do a batch lookup from cache for file stats for all paths used in a list of storage paths or FileOp o...
doOperationsInternal(array $ops, array $opts)
isSingleShardPathInternal( $storagePath)
Check if a storage path maps to a single shard.
directoryExists(array $params)
Check if a directory exists at a given storage path.
doDeleteInternal(array $params)
getContainerSuffixes( $container)
Get a list of full container shard suffixes for a container.
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
sanitizeOpHeaders(array $op)
Normalize and filter HTTP headers from a file operation.
doDescribeInternal(array $params)
doGetFileSha1Base36(array $params)
deleteInternal(array $params)
Delete a file at the storage path.
doPrimeContainerCache(array $containerInfo)
Fill the backend-specific process cache given an array of resolved container names and their correspo...
doMoveInternal(array $params)
maxFileSizeInternal()
Get the maximum allowable file size given backend medium restrictions and basic performance constrain...
describeInternal(array $params)
Alter metadata for a file at the storage path.
streamFile(array $params)
Stream the content of the file at a storage path in the backend.
getOperationsInternal(array $ops)
Return a list of FileOp objects from a list of operations.
deleteFileCache( $path)
Delete the cached stat info for a file path.
setConcurrencyFlags(array $opts)
Set the 'concurrency' option from a list of operation options.
getFileXAttributes(array $params)
Get metadata about a file at a storage path in the backend.
getContentType( $storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
preloadCache(array $paths)
Preload persistent file stat cache and property cache into in-process cache.
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
doDirectoryExists( $container, $dir, array $params)
WANObjectCache $memCache
doPrepare(array $params)
FileBackend::prepare() StatusValue Good status without value for success, fatal otherwise.
getFileHttpUrl(array $params)
__construct(array $config)
doGetFileXAttributes(array $params)
doExecuteOpHandlesInternal(array $fileOpHandles)
nullInternal(array $params)
No-op file operation that does nothing.
array< string, array > $shardViaHashLevels
Map of container names to sharding config.
getFileSize(array $params)
Get the size (bytes) of a file at a storage path in the backend.
callable null $mimeCallback
Method to get the MIME type of files.
doConcatenate(array $params)
doSecureInternal( $container, $dir, array $params)
doCreateInternal(array $params)
fileExists(array $params)
Check if a file exists at a storage path in the backend.
getDirectoryListInternal( $container, $dir, array $params)
Do not call this function from places outside FileBackend.
static isValidShortContainerName( $container)
Check if a short container name is valid.
getContainerHashLevels( $container)
Get the sharding config for a container.
getContainerShard( $container, $relPath)
Get the container name shard suffix for a given path.
deleteContainerCache( $container)
Delete the cached info for a container.
createInternal(array $params)
Create a file in the backend with the given contents.
getFileList(array $params)
Get an iterator to list all stored files under a storage directory.
Base class for all file backend classes (including multi-write backends).
string $name
Unique backend name.
static splitStoragePath( $storagePath)
Split a storage path into a backend name, a container name, and a relative file path.
static normalizeContainerPath( $path)
Validate and normalize a relative storage path.
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
getTopDirectoryList(array $params)
Same as FileBackend::getDirectoryList() except only lists directories that are immediately under the ...
scopedProfileSection( $section)
newStatus(... $args)
Yields the result of the status wrapper callback on either:
static normalizeStoragePath( $storagePath)
Normalize a storage path by cleaning up directory separators.
int $concurrency
How many operations can be done in parallel.
getLocalReference(array $params)
Returns a file system file, identical in content to the file at a storage path.
static attempt(array $performOps, array $opts)
Attempt to perform a series of file operations.
Functions related to the output of file content.
Store key-value entries in a size-limited in-memory LRU cache.
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Multi-datacenter aware caching interface.