MediaWiki master
FileBackendStore.php
Go to the documentation of this file.
1<?php
24namespace Wikimedia\FileBackend;
25
26use InvalidArgumentException;
27use LockManager;
28use Shellbox\Command\BoxedCommand;
29use StatusValue;
30use Traversable;
31use Wikimedia\AtEase\AtEase;
48use Wikimedia\Timestamp\ConvertibleTimestamp;
49
64abstract class FileBackendStore extends FileBackend {
66 protected $memCache;
68 protected $srvCache;
70 protected $cheapCache;
72 protected $expensiveCache;
73
75 protected $shardViaHashLevels = [];
76
78 protected $mimeCallback;
79
81 protected $maxFileSize = 32 * 1024 * 1024 * 1024;
82
83 protected const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries
84 protected const CACHE_CHEAP_SIZE = 500; // integer; max entries in "cheap cache"
85 protected const CACHE_EXPENSIVE_SIZE = 5; // integer; max entries in "expensive cache"
86
88 protected const RES_ABSENT = false;
90 protected const RES_ERROR = null;
91
93 protected const ABSENT_NORMAL = 'FNE-N';
95 protected const ABSENT_LATEST = 'FNE-L';
96
110 public function __construct( array $config ) {
111 parent::__construct( $config );
112 $this->mimeCallback = $config['mimeCallback'] ?? null;
113 $this->srvCache = new EmptyBagOStuff(); // disabled by default
114 $this->memCache = WANObjectCache::newEmpty(); // disabled by default
115 $this->cheapCache = new MapCacheLRU( self::CACHE_CHEAP_SIZE );
116 $this->expensiveCache = new MapCacheLRU( self::CACHE_EXPENSIVE_SIZE );
117 }
118
126 final public function maxFileSizeInternal() {
127 return min( $this->maxFileSize, PHP_INT_MAX );
128 }
129
140 abstract public function isPathUsableInternal( $storagePath );
141
160 final public function createInternal( array $params ) {
162 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
163
164 if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
165 $status = $this->newStatus( 'backend-fail-maxsize',
166 $params['dst'], $this->maxFileSizeInternal() );
167 } else {
168 $status = $this->doCreateInternal( $params );
169 $this->clearCache( [ $params['dst'] ] );
170 if ( $params['dstExists'] ?? true ) {
171 $this->deleteFileCache( $params['dst'] ); // persistent cache
172 }
173 }
174
175 return $status;
176 }
177
183 abstract protected function doCreateInternal( array $params );
184
203 final public function storeInternal( array $params ) {
205 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
206
207 if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
208 $status = $this->newStatus( 'backend-fail-maxsize',
209 $params['dst'], $this->maxFileSizeInternal() );
210 } else {
211 $status = $this->doStoreInternal( $params );
212 $this->clearCache( [ $params['dst'] ] );
213 if ( $params['dstExists'] ?? true ) {
214 $this->deleteFileCache( $params['dst'] ); // persistent cache
215 }
216 }
217
218 return $status;
219 }
220
226 abstract protected function doStoreInternal( array $params );
227
247 final public function copyInternal( array $params ) {
249 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
250
251 $status = $this->doCopyInternal( $params );
252 $this->clearCache( [ $params['dst'] ] );
253 if ( $params['dstExists'] ?? true ) {
254 $this->deleteFileCache( $params['dst'] ); // persistent cache
255 }
256
257 return $status;
258 }
259
265 abstract protected function doCopyInternal( array $params );
266
281 final public function deleteInternal( array $params ) {
283 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
284
285 $status = $this->doDeleteInternal( $params );
286 $this->clearCache( [ $params['src'] ] );
287 $this->deleteFileCache( $params['src'] ); // persistent cache
288 return $status;
289 }
290
296 abstract protected function doDeleteInternal( array $params );
297
317 final public function moveInternal( array $params ) {
319 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
320
321 $status = $this->doMoveInternal( $params );
322 $this->clearCache( [ $params['src'], $params['dst'] ] );
323 $this->deleteFileCache( $params['src'] ); // persistent cache
324 if ( $params['dstExists'] ?? true ) {
325 $this->deleteFileCache( $params['dst'] ); // persistent cache
326 }
327
328 return $status;
329 }
330
336 abstract protected function doMoveInternal( array $params );
337
352 final public function describeInternal( array $params ) {
354 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
355
356 if ( count( $params['headers'] ) ) {
357 $status = $this->doDescribeInternal( $params );
358 $this->clearCache( [ $params['src'] ] );
359 $this->deleteFileCache( $params['src'] ); // persistent cache
360 } else {
361 $status = $this->newStatus(); // nothing to do
362 }
363
364 return $status;
365 }
366
373 protected function doDescribeInternal( array $params ) {
374 return $this->newStatus();
375 }
376
384 final public function nullInternal( array $params ) {
385 return $this->newStatus();
386 }
387
388 final public function concatenate( array $params ) {
390 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
391 $status = $this->newStatus();
392
393 // Try to lock the source files for the scope of this function
395 $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
396 if ( $status->isOK() ) {
397 // Actually do the file concatenation...
398 $hrStart = hrtime( true );
399 $status->merge( $this->doConcatenate( $params ) );
400 $sec = ( hrtime( true ) - $hrStart ) / 1e9;
401 if ( !$status->isOK() ) {
402 $this->logger->error( static::class . "-{$this->name}" .
403 " failed to concatenate " . count( $params['srcs'] ) . " file(s) [$sec sec]" );
404 }
405 }
406
407 return $status;
408 }
409
416 protected function doConcatenate( array $params ) {
417 $status = $this->newStatus();
418 $tmpPath = $params['dst'];
419 unset( $params['latest'] );
420
421 // Check that the specified temp file is valid...
422 AtEase::suppressWarnings();
423 $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 );
424 AtEase::restoreWarnings();
425 if ( !$ok ) { // not present or not empty
426 $status->fatal( 'backend-fail-opentemp', $tmpPath );
427
428 return $status;
429 }
430
431 // Get local FS versions of the chunks needed for the concatenation...
432 $fsFiles = $this->getLocalReferenceMulti( $params );
433 foreach ( $fsFiles as $path => &$fsFile ) {
434 if ( !$fsFile ) { // chunk failed to download?
435 $fsFile = $this->getLocalReference( [ 'src' => $path ] );
436 if ( !$fsFile ) { // retry failed?
437 $status->fatal(
438 $fsFile === self::RES_ERROR ? 'backend-fail-read' : 'backend-fail-notexists',
439 $path
440 );
441
442 return $status;
443 }
444 }
445 }
446 unset( $fsFile ); // unset reference so we can reuse $fsFile
447
448 // Get a handle for the destination temp file
449 $tmpHandle = fopen( $tmpPath, 'ab' );
450 if ( $tmpHandle === false ) {
451 $status->fatal( 'backend-fail-opentemp', $tmpPath );
452
453 return $status;
454 }
455
456 // Build up the temp file using the source chunks (in order)...
457 foreach ( $fsFiles as $virtualSource => $fsFile ) {
458 // Get a handle to the local FS version
459 $sourceHandle = fopen( $fsFile->getPath(), 'rb' );
460 if ( $sourceHandle === false ) {
461 fclose( $tmpHandle );
462 $status->fatal( 'backend-fail-read', $virtualSource );
463
464 return $status;
465 }
466 // Append chunk to file (pass chunk size to avoid magic quotes)
467 if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
468 fclose( $sourceHandle );
469 fclose( $tmpHandle );
470 $status->fatal( 'backend-fail-writetemp', $tmpPath );
471
472 return $status;
473 }
474 fclose( $sourceHandle );
475 }
476 if ( !fclose( $tmpHandle ) ) {
477 $status->fatal( 'backend-fail-closetemp', $tmpPath );
478
479 return $status;
480 }
481
482 clearstatcache(); // temp file changed
483
484 return $status;
485 }
486
490 final protected function doPrepare( array $params ) {
492 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
493 $status = $this->newStatus();
494
495 [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
496 if ( $dir === null ) {
497 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
498
499 return $status; // invalid storage path
500 }
501
502 if ( $shard !== null ) { // confined to a single container/shard
503 $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
504 } else { // directory is on several shards
505 $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
506 [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
507 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
508 $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
509 }
510 }
511
512 return $status;
513 }
514
523 protected function doPrepareInternal( $container, $dir, array $params ) {
524 return $this->newStatus();
525 }
526
527 final protected function doSecure( array $params ) {
529 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
530 $status = $this->newStatus();
531
532 [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
533 if ( $dir === null ) {
534 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
535
536 return $status; // invalid storage path
537 }
538
539 if ( $shard !== null ) { // confined to a single container/shard
540 $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
541 } else { // directory is on several shards
542 $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
543 [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
544 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
545 $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
546 }
547 }
548
549 return $status;
550 }
551
560 protected function doSecureInternal( $container, $dir, array $params ) {
561 return $this->newStatus();
562 }
563
564 final protected function doPublish( array $params ) {
566 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
567 $status = $this->newStatus();
568
569 [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
570 if ( $dir === null ) {
571 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
572
573 return $status; // invalid storage path
574 }
575
576 if ( $shard !== null ) { // confined to a single container/shard
577 $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
578 } else { // directory is on several shards
579 $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
580 [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
581 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
582 $status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) );
583 }
584 }
585
586 return $status;
587 }
588
597 protected function doPublishInternal( $container, $dir, array $params ) {
598 return $this->newStatus();
599 }
600
601 final protected function doClean( array $params ) {
603 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
604 $status = $this->newStatus();
605
606 // Recursive: first delete all empty subdirs recursively
607 if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) {
608 $subDirsRel = $this->getTopDirectoryList( [ 'dir' => $params['dir'] ] );
609 if ( $subDirsRel !== null ) { // no errors
610 foreach ( $subDirsRel as $subDirRel ) {
611 $subDir = $params['dir'] . "/{$subDirRel}"; // full path
612 $status->merge( $this->doClean( [ 'dir' => $subDir ] + $params ) );
613 }
614 unset( $subDirsRel ); // free directory for rmdir() on Windows (for FS backends)
615 }
616 }
617
618 [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
619 if ( $dir === null ) {
620 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
621
622 return $status; // invalid storage path
623 }
624
625 // Attempt to lock this directory...
626 $filesLockEx = [ $params['dir'] ];
628 $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
629 if ( !$status->isOK() ) {
630 return $status; // abort
631 }
632
633 if ( $shard !== null ) { // confined to a single container/shard
634 $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
635 $this->deleteContainerCache( $fullCont ); // purge cache
636 } else { // directory is on several shards
637 $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
638 [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
639 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
640 $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
641 $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache
642 }
643 }
644
645 return $status;
646 }
647
656 protected function doCleanInternal( $container, $dir, array $params ) {
657 return $this->newStatus();
658 }
659
660 final public function fileExists( array $params ) {
662 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
663
664 $stat = $this->getFileStat( $params );
665 if ( is_array( $stat ) ) {
666 return true;
667 }
668
669 return $stat === self::RES_ABSENT ? false : self::EXISTENCE_ERROR;
670 }
671
672 final public function getFileTimestamp( array $params ) {
674 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
675
676 $stat = $this->getFileStat( $params );
677 if ( is_array( $stat ) ) {
678 return $stat['mtime'];
679 }
680
681 return self::TIMESTAMP_FAIL; // all failure cases
682 }
683
684 final public function getFileSize( array $params ) {
686 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
687
688 $stat = $this->getFileStat( $params );
689 if ( is_array( $stat ) ) {
690 return $stat['size'];
691 }
692
693 return self::SIZE_FAIL; // all failure cases
694 }
695
696 final public function getFileStat( array $params ) {
698 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
699
700 $path = self::normalizeStoragePath( $params['src'] );
701 if ( $path === null ) {
702 return self::STAT_ERROR; // invalid storage path
703 }
704
705 // Whether to bypass cache except for process cache entries loaded directly from
706 // high consistency backend queries (caller handles any cache flushing and locking)
707 $latest = !empty( $params['latest'] );
708 // Whether to ignore cache entries missing the SHA-1 field for existing files
709 $requireSHA1 = !empty( $params['requireSHA1'] );
710
711 $stat = $this->cheapCache->getField( $path, 'stat', self::CACHE_TTL );
712 // Load the persistent stat cache into process cache if needed
713 if ( !$latest ) {
714 if (
715 // File stat is not in process cache
716 $stat === null ||
717 // Key/value store backends might opportunistically set file stat process
718 // cache entries from object listings that do not include the SHA-1. In that
719 // case, loading the persistent stat cache will likely yield the SHA-1.
720 ( $requireSHA1 && is_array( $stat ) && !isset( $stat['sha1'] ) )
721 ) {
722 $this->primeFileCache( [ $path ] );
723 // Get any newly process-cached entry
724 $stat = $this->cheapCache->getField( $path, 'stat', self::CACHE_TTL );
725 }
726 }
727
728 if ( is_array( $stat ) ) {
729 if (
730 ( !$latest || !empty( $stat['latest'] ) ) &&
731 ( !$requireSHA1 || isset( $stat['sha1'] ) )
732 ) {
733 return $stat;
734 }
735 } elseif ( $stat === self::ABSENT_LATEST ) {
736 return self::STAT_ABSENT;
737 } elseif ( $stat === self::ABSENT_NORMAL ) {
738 if ( !$latest ) {
739 return self::STAT_ABSENT;
740 }
741 }
742
743 // Load the file stat from the backend and update caches
744 $stat = $this->doGetFileStat( $params );
745 $this->ingestFreshFileStats( [ $path => $stat ], $latest );
746
747 if ( is_array( $stat ) ) {
748 return $stat;
749 }
750
751 return $stat === self::RES_ERROR ? self::STAT_ERROR : self::STAT_ABSENT;
752 }
753
761 final protected function ingestFreshFileStats( array $stats, $latest ) {
762 $success = true;
763
764 foreach ( $stats as $path => $stat ) {
765 if ( is_array( $stat ) ) {
766 // Strongly consistent backends might automatically set this flag
767 $stat['latest'] ??= $latest;
768
769 $this->cheapCache->setField( $path, 'stat', $stat );
770 if ( isset( $stat['sha1'] ) ) {
771 // Some backends store the SHA-1 hash as metadata
772 $this->cheapCache->setField(
773 $path,
774 'sha1',
775 [ 'hash' => $stat['sha1'], 'latest' => $latest ]
776 );
777 }
778 if ( isset( $stat['xattr'] ) ) {
779 // Some backends store custom headers/metadata
780 $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] );
781 $this->cheapCache->setField(
782 $path,
783 'xattr',
784 [ 'map' => $stat['xattr'], 'latest' => $latest ]
785 );
786 }
787 // Update persistent cache (@TODO: set all entries in one batch)
788 $this->setFileCache( $path, $stat );
789 } elseif ( $stat === self::RES_ABSENT ) {
790 $this->cheapCache->setField(
791 $path,
792 'stat',
793 $latest ? self::ABSENT_LATEST : self::ABSENT_NORMAL
794 );
795 $this->cheapCache->setField(
796 $path,
797 'xattr',
798 [ 'map' => self::XATTRS_FAIL, 'latest' => $latest ]
799 );
800 $this->cheapCache->setField(
801 $path,
802 'sha1',
803 [ 'hash' => self::SHA1_FAIL, 'latest' => $latest ]
804 );
805 $this->logger->debug(
806 __METHOD__ . ': File {path} does not exist',
807 [ 'path' => $path ]
808 );
809 } else {
810 $success = false;
811 $this->logger->error(
812 __METHOD__ . ': Could not stat file {path}',
813 [ 'path' => $path ]
814 );
815 }
816 }
817
818 return $success;
819 }
820
826 abstract protected function doGetFileStat( array $params );
827
828 public function getFileContentsMulti( array $params ) {
830 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
831
832 $params = $this->setConcurrencyFlags( $params );
833 $contents = $this->doGetFileContentsMulti( $params );
834 foreach ( $contents as $path => $content ) {
835 if ( !is_string( $content ) ) {
836 $contents[$path] = self::CONTENT_FAIL; // used for all failure cases
837 }
838 }
839
840 return $contents;
841 }
842
849 protected function doGetFileContentsMulti( array $params ) {
850 $contents = [];
851 foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
852 if ( $fsFile instanceof FSFile ) {
853 AtEase::suppressWarnings();
854 $content = file_get_contents( $fsFile->getPath() );
855 AtEase::restoreWarnings();
856 $contents[$path] = is_string( $content ) ? $content : self::RES_ERROR;
857 } else {
858 // self::RES_ERROR or self::RES_ABSENT
859 $contents[$path] = $fsFile;
860 }
861 }
862
863 return $contents;
864 }
865
866 final public function getFileXAttributes( array $params ) {
868 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
869
870 $path = self::normalizeStoragePath( $params['src'] );
871 if ( $path === null ) {
872 return self::XATTRS_FAIL; // invalid storage path
873 }
874 $latest = !empty( $params['latest'] ); // use latest data?
875 if ( $this->cheapCache->hasField( $path, 'xattr', self::CACHE_TTL ) ) {
876 $stat = $this->cheapCache->getField( $path, 'xattr' );
877 // If we want the latest data, check that this cached
878 // value was in fact fetched with the latest available data.
879 if ( !$latest || $stat['latest'] ) {
880 return $stat['map'];
881 }
882 }
883 $fields = $this->doGetFileXAttributes( $params );
884 if ( is_array( $fields ) ) {
885 $fields = self::normalizeXAttributes( $fields );
886 $this->cheapCache->setField(
887 $path,
888 'xattr',
889 [ 'map' => $fields, 'latest' => $latest ]
890 );
891 } elseif ( $fields === self::RES_ABSENT ) {
892 $this->cheapCache->setField(
893 $path,
894 'xattr',
895 [ 'map' => self::XATTRS_FAIL, 'latest' => $latest ]
896 );
897 } else {
898 $fields = self::XATTRS_FAIL; // used for all failure cases
899 }
900
901 return $fields;
902 }
903
910 protected function doGetFileXAttributes( array $params ) {
911 return [ 'headers' => [], 'metadata' => [] ]; // not supported
912 }
913
914 final public function getFileSha1Base36( array $params ) {
916 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
917
918 $path = self::normalizeStoragePath( $params['src'] );
919 if ( $path === null ) {
920 return self::SHA1_FAIL; // invalid storage path
921 }
922 $latest = !empty( $params['latest'] ); // use latest data?
923 if ( $this->cheapCache->hasField( $path, 'sha1', self::CACHE_TTL ) ) {
924 $stat = $this->cheapCache->getField( $path, 'sha1' );
925 // If we want the latest data, check that this cached
926 // value was in fact fetched with the latest available data.
927 if ( !$latest || $stat['latest'] ) {
928 return $stat['hash'];
929 }
930 }
931 $sha1 = $this->doGetFileSha1Base36( $params );
932 if ( is_string( $sha1 ) ) {
933 $this->cheapCache->setField(
934 $path,
935 'sha1',
936 [ 'hash' => $sha1, 'latest' => $latest ]
937 );
938 } elseif ( $sha1 === self::RES_ABSENT ) {
939 $this->cheapCache->setField(
940 $path,
941 'sha1',
942 [ 'hash' => self::SHA1_FAIL, 'latest' => $latest ]
943 );
944 } else {
945 $sha1 = self::SHA1_FAIL; // used for all failure cases
946 }
947
948 return $sha1;
949 }
950
957 protected function doGetFileSha1Base36( array $params ) {
958 $fsFile = $this->getLocalReference( $params );
959 if ( $fsFile instanceof FSFile ) {
960 $sha1 = $fsFile->getSha1Base36();
961
962 return is_string( $sha1 ) ? $sha1 : self::RES_ERROR;
963 }
964
965 return $fsFile === self::RES_ERROR ? self::RES_ERROR : self::RES_ABSENT;
966 }
967
968 final public function getFileProps( array $params ) {
970 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
971
972 $fsFile = $this->getLocalReference( $params );
973
974 return $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
975 }
976
977 final public function getLocalReferenceMulti( array $params ) {
979 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
980
981 $params = $this->setConcurrencyFlags( $params );
982
983 $fsFiles = []; // (path => FSFile)
984 $latest = !empty( $params['latest'] ); // use latest data?
985 // Reuse any files already in process cache...
986 foreach ( $params['srcs'] as $src ) {
988 if ( $path === null ) {
989 $fsFiles[$src] = self::RES_ERROR; // invalid storage path
990 } elseif ( $this->expensiveCache->hasField( $path, 'localRef' ) ) {
991 $val = $this->expensiveCache->getField( $path, 'localRef' );
992 // If we want the latest data, check that this cached
993 // value was in fact fetched with the latest available data.
994 if ( !$latest || $val['latest'] ) {
995 $fsFiles[$src] = $val['object'];
996 }
997 }
998 }
999 // Fetch local references of any remaining files...
1000 $params['srcs'] = array_diff( $params['srcs'], array_keys( $fsFiles ) );
1001 foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
1002 $fsFiles[$path] = $fsFile;
1003 if ( $fsFile instanceof FSFile ) {
1004 $this->expensiveCache->setField(
1005 $path,
1006 'localRef',
1007 [ 'object' => $fsFile, 'latest' => $latest ]
1008 );
1009 }
1010 }
1011
1012 return $fsFiles;
1013 }
1014
1021 protected function doGetLocalReferenceMulti( array $params ) {
1022 return $this->doGetLocalCopyMulti( $params );
1023 }
1024
1025 final public function getLocalCopyMulti( array $params ) {
1027 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1028
1029 $params = $this->setConcurrencyFlags( $params );
1030
1031 return $this->doGetLocalCopyMulti( $params );
1032 }
1033
1039 abstract protected function doGetLocalCopyMulti( array $params );
1040
1047 public function getFileHttpUrl( array $params ) {
1048 return self::TEMPURL_ERROR; // not supported
1049 }
1050
1051 public function addShellboxInputFile( BoxedCommand $command, string $boxedName,
1052 array $params
1053 ) {
1054 $ref = $this->getLocalReference( [ 'src' => $params['src'] ] );
1055 if ( $ref === false ) {
1056 return $this->newStatus( 'backend-fail-notexists', $params['src'] );
1057 } elseif ( $ref === null ) {
1058 return $this->newStatus( 'backend-fail-read', $params['src'] );
1059 } else {
1060 $file = $command->newInputFileFromFile( $ref->getPath() )
1061 ->userData( __CLASS__, $ref );
1062 $command->inputFile( $boxedName, $file );
1063 return $this->newStatus();
1064 }
1065 }
1066
1067 final public function streamFile( array $params ) {
1069 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1070 $status = $this->newStatus();
1071
1072 // Always set some fields for subclass convenience
1073 $params['options'] ??= [];
1074 $params['headers'] ??= [];
1075
1076 // Don't stream it out as text/html if there was a PHP error
1077 if ( ( empty( $params['headless'] ) || $params['headers'] ) && headers_sent() ) {
1078 print "Headers already sent, terminating.\n";
1079 $status->fatal( 'backend-fail-stream', $params['src'] );
1080 return $status;
1081 }
1082
1083 $status->merge( $this->doStreamFile( $params ) );
1084
1085 return $status;
1086 }
1087
1094 protected function doStreamFile( array $params ) {
1095 $status = $this->newStatus();
1096
1097 $flags = 0;
1098 $flags |= !empty( $params['headless'] ) ? HTTPFileStreamer::STREAM_HEADLESS : 0;
1099 $flags |= !empty( $params['allowOB'] ) ? HTTPFileStreamer::STREAM_ALLOW_OB : 0;
1100
1101 $fsFile = $this->getLocalReference( $params );
1102 if ( $fsFile ) {
1103 $streamer = new HTTPFileStreamer(
1104 $fsFile->getPath(),
1105 $this->getStreamerOptions()
1106 );
1107 $res = $streamer->stream( $params['headers'], true, $params['options'], $flags );
1108 } else {
1109 $res = false;
1110 HTTPFileStreamer::send404Message( $params['src'], $flags );
1111 }
1112
1113 if ( !$res ) {
1114 $status->fatal( 'backend-fail-stream', $params['src'] );
1115 }
1116
1117 return $status;
1118 }
1119
1120 final public function directoryExists( array $params ) {
1121 [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
1122 if ( $dir === null ) {
1123 return self::EXISTENCE_ERROR; // invalid storage path
1124 }
1125 if ( $shard !== null ) { // confined to a single container/shard
1126 return $this->doDirectoryExists( $fullCont, $dir, $params );
1127 } else { // directory is on several shards
1128 $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
1129 [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
1130 $res = false; // response
1131 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
1132 $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params );
1133 if ( $exists === true ) {
1134 $res = true;
1135 break; // found one!
1136 } elseif ( $exists === self::RES_ERROR ) {
1137 $res = self::EXISTENCE_ERROR;
1138 }
1139 }
1140
1141 return $res;
1142 }
1143 }
1144
1153 abstract protected function doDirectoryExists( $container, $dir, array $params );
1154
1155 final public function getDirectoryList( array $params ) {
1156 [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
1157 if ( $dir === null ) {
1158 return self::EXISTENCE_ERROR; // invalid storage path
1159 }
1160 if ( $shard !== null ) {
1161 // File listing is confined to a single container/shard
1162 return $this->getDirectoryListInternal( $fullCont, $dir, $params );
1163 } else {
1164 $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
1165 // File listing spans multiple containers/shards
1166 [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
1167
1168 return new FileBackendStoreShardDirIterator( $this,
1169 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
1170 }
1171 }
1172
1183 abstract public function getDirectoryListInternal( $container, $dir, array $params );
1184
1185 final public function getFileList( array $params ) {
1186 [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
1187 if ( $dir === null ) {
1188 return self::LIST_ERROR; // invalid storage path
1189 }
1190 if ( $shard !== null ) {
1191 // File listing is confined to a single container/shard
1192 return $this->getFileListInternal( $fullCont, $dir, $params );
1193 } else {
1194 $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
1195 // File listing spans multiple containers/shards
1196 [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
1197
1198 return new FileBackendStoreShardFileIterator( $this,
1199 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
1200 }
1201 }
1202
1213 abstract public function getFileListInternal( $container, $dir, array $params );
1214
1226 final public function getOperationsInternal( array $ops ) {
1227 $supportedOps = [
1228 'store' => StoreFileOp::class,
1229 'copy' => CopyFileOp::class,
1230 'move' => MoveFileOp::class,
1231 'delete' => DeleteFileOp::class,
1232 'create' => CreateFileOp::class,
1233 'describe' => DescribeFileOp::class,
1234 'null' => NullFileOp::class
1235 ];
1236
1237 $performOps = []; // array of FileOp objects
1238 // Build up ordered array of FileOps...
1239 foreach ( $ops as $operation ) {
1240 $opName = $operation['op'];
1241 if ( isset( $supportedOps[$opName] ) ) {
1242 $class = $supportedOps[$opName];
1243 // Get params for this operation
1244 $params = $operation;
1245 // Append the FileOp class
1246 $performOps[] = new $class( $this, $params, $this->logger );
1247 } else {
1248 throw new FileBackendError( "Operation '$opName' is not supported." );
1249 }
1250 }
1251
1252 return $performOps;
1253 }
1254
1265 final public function getPathsToLockForOpsInternal( array $performOps ) {
1266 // Build up a list of files to lock...
1267 $paths = [ 'sh' => [], 'ex' => [] ];
1268 foreach ( $performOps as $fileOp ) {
1269 $paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() );
1270 $paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() );
1271 }
1272 // Optimization: if doing an EX lock anyway, don't also set an SH one
1273 $paths['sh'] = array_diff( $paths['sh'], $paths['ex'] );
1274 // Get a shared lock on the parent directory of each path changed
1275 $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) );
1276
1277 return [
1278 LockManager::LOCK_UW => $paths['sh'],
1279 LockManager::LOCK_EX => $paths['ex']
1280 ];
1281 }
1282
1283 public function getScopedLocksForOps( array $ops, StatusValue $status ) {
1284 $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) );
1285
1286 return $this->getScopedFileLocks( $paths, 'mixed', $status );
1287 }
1288
1289 final protected function doOperationsInternal( array $ops, array $opts ) {
1291 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1292 $status = $this->newStatus();
1293
1294 // Fix up custom header name/value pairs
1295 $ops = array_map( $this->sanitizeOpHeaders( ... ), $ops );
1296 // Build up a list of FileOps and involved paths
1297 $fileOps = $this->getOperationsInternal( $ops );
1298 $pathsUsed = [];
1299 foreach ( $fileOps as $fileOp ) {
1300 $pathsUsed = array_merge( $pathsUsed, $fileOp->storagePathsReadOrChanged() );
1301 }
1302
1303 // Acquire any locks as needed for the scope of this function
1304 if ( empty( $opts['nonLocking'] ) ) {
1305 $pathsByLockType = $this->getPathsToLockForOpsInternal( $fileOps );
1307 $scopeLock = $this->getScopedFileLocks( $pathsByLockType, 'mixed', $status );
1308 if ( !$status->isOK() ) {
1309 return $status; // abort
1310 }
1311 }
1312
1313 // Clear any file cache entries (after locks acquired)
1314 if ( empty( $opts['preserveCache'] ) ) {
1315 $this->clearCache( $pathsUsed );
1316 }
1317
1318 // Enlarge the cache to fit the stat entries of these files
1319 $this->cheapCache->setMaxSize( max( 2 * count( $pathsUsed ), self::CACHE_CHEAP_SIZE ) );
1320
1321 // Load from the persistent container caches
1322 $this->primeContainerCache( $pathsUsed );
1323 // Get the latest stat info for all the files (having locked them)
1324 $ok = $this->preloadFileStat( [ 'srcs' => $pathsUsed, 'latest' => true ] );
1325
1326 if ( $ok ) {
1327 // Actually attempt the operation batch...
1328 $opts = $this->setConcurrencyFlags( $opts );
1329 $subStatus = FileOpBatch::attempt( $fileOps, $opts );
1330 } else {
1331 // If we could not even stat some files, then bail out
1332 $subStatus = $this->newStatus( 'backend-fail-internal', $this->name );
1333 foreach ( $ops as $i => $op ) { // mark each op as failed
1334 $subStatus->success[$i] = false;
1335 ++$subStatus->failCount;
1336 }
1337 $this->logger->error( static::class . "-{$this->name} stat failure",
1338 [ 'aborted_operations' => $ops ]
1339 );
1340 }
1341
1342 // Merge errors into StatusValue fields
1343 $status->merge( $subStatus );
1344 $status->success = $subStatus->success; // not done in merge()
1345
1346 // Shrink the stat cache back to normal size
1347 $this->cheapCache->setMaxSize( self::CACHE_CHEAP_SIZE );
1348
1349 return $status;
1350 }
1351
1352 final protected function doQuickOperationsInternal( array $ops, array $opts ) {
1354 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1355 $status = $this->newStatus();
1356
1357 // Fix up custom header name/value pairs
1358 $ops = array_map( $this->sanitizeOpHeaders( ... ), $ops );
1359 // Build up a list of FileOps and involved paths
1360 $fileOps = $this->getOperationsInternal( $ops );
1361 $pathsUsed = [];
1362 foreach ( $fileOps as $fileOp ) {
1363 $pathsUsed = array_merge( $pathsUsed, $fileOp->storagePathsReadOrChanged() );
1364 }
1365
1366 // Clear any file cache entries for involved paths
1367 $this->clearCache( $pathsUsed );
1368
1369 // Parallel ops may be disabled in config due to dependencies (e.g. needing popen())
1370 $async = ( $this->parallelize === 'implicit' && count( $ops ) > 1 );
1371 $maxConcurrency = $this->concurrency; // throttle
1373 $statuses = []; // array of (index => StatusValue)
1375 $batch = [];
1376 foreach ( $fileOps as $index => $fileOp ) {
1377 $subStatus = $async
1378 ? $fileOp->attemptAsyncQuick()
1379 : $fileOp->attemptQuick();
1380 if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { // async
1381 if ( count( $batch ) >= $maxConcurrency ) {
1382 // Execute this batch. Don't queue any more ops since they contain
1383 // open filehandles which are a limited resource (T230245).
1384 $statuses += $this->executeOpHandlesInternal( $batch );
1385 $batch = [];
1386 }
1387 $batch[$index] = $subStatus->value; // keep index
1388 } else { // error or completed
1389 $statuses[$index] = $subStatus; // keep index
1390 }
1391 }
1392 if ( count( $batch ) ) {
1393 $statuses += $this->executeOpHandlesInternal( $batch );
1394 }
1395 // Marshall and merge all the responses...
1396 foreach ( $statuses as $index => $subStatus ) {
1397 $status->merge( $subStatus );
1398 if ( $subStatus->isOK() ) {
1399 $status->success[$index] = true;
1400 ++$status->successCount;
1401 } else {
1402 $status->success[$index] = false;
1403 ++$status->failCount;
1404 }
1405 }
1406
1407 $this->clearCache( $pathsUsed );
1408
1409 return $status;
1410 }
1411
1421 final public function executeOpHandlesInternal( array $fileOpHandles ) {
1423 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1424
1425 foreach ( $fileOpHandles as $fileOpHandle ) {
1426 if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) {
1427 throw new InvalidArgumentException( "Expected FileBackendStoreOpHandle object." );
1428 } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) {
1429 throw new InvalidArgumentException( "Expected handle for this file backend." );
1430 }
1431 }
1432
1433 $statuses = $this->doExecuteOpHandlesInternal( $fileOpHandles );
1434 foreach ( $fileOpHandles as $fileOpHandle ) {
1435 $fileOpHandle->closeResources();
1436 }
1437
1438 return $statuses;
1439 }
1440
1450 protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1451 if ( count( $fileOpHandles ) ) {
1452 throw new FileBackendError( "Backend does not support asynchronous operations." );
1453 }
1454
1455 return [];
1456 }
1457
1469 protected function sanitizeOpHeaders( array $op ) {
1470 static $longs = [ 'content-disposition' ];
1471
1472 if ( isset( $op['headers'] ) ) { // op sets HTTP headers
1473 $newHeaders = [];
1474 foreach ( $op['headers'] as $name => $value ) {
1475 $name = strtolower( $name );
1476 $maxHVLen = in_array( $name, $longs ) ? INF : 255;
1477 if ( strlen( $name ) > 255 || strlen( $value ) > $maxHVLen ) {
1478 $this->logger->error( "Header '{header}' is too long.", [
1479 'filebackend' => $this->name,
1480 'header' => "$name: $value",
1481 ] );
1482 } else {
1483 $newHeaders[$name] = strlen( $value ) ? $value : ''; // null/false => ""
1484 }
1485 }
1486 $op['headers'] = $newHeaders;
1487 }
1488
1489 return $op;
1490 }
1491
1492 final public function preloadCache( array $paths ) {
1493 $fullConts = []; // full container names
1494 foreach ( $paths as $path ) {
1495 [ $fullCont, , ] = $this->resolveStoragePath( $path );
1496 $fullConts[] = $fullCont;
1497 }
1498 // Load from the persistent file and container caches
1499 $this->primeContainerCache( $fullConts );
1500 $this->primeFileCache( $paths );
1501 }
1502
1503 final public function clearCache( ?array $paths = null ) {
1504 if ( is_array( $paths ) ) {
1505 $paths = array_map( [ FileBackend::class, 'normalizeStoragePath' ], $paths );
1506 $paths = array_filter( $paths, 'strlen' ); // remove nulls
1507 }
1508 if ( $paths === null ) {
1509 $this->cheapCache->clear();
1510 $this->expensiveCache->clear();
1511 } else {
1512 foreach ( $paths as $path ) {
1513 $this->cheapCache->clear( $path );
1514 $this->expensiveCache->clear( $path );
1515 }
1516 }
1517 $this->doClearCache( $paths );
1518 }
1519
1528 protected function doClearCache( ?array $paths = null ) {
1529 }
1530
1531 final public function preloadFileStat( array $params ) {
1533 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1534
1535 $params['concurrency'] = ( $this->parallelize !== 'off' ) ? $this->concurrency : 1;
1536 $stats = $this->doGetFileStatMulti( $params );
1537 if ( $stats === null ) {
1538 return true; // not supported
1539 }
1540
1541 // Whether this queried the backend in high consistency mode
1542 $latest = !empty( $params['latest'] );
1543
1544 return $this->ingestFreshFileStats( $stats, $latest );
1545 }
1546
1560 protected function doGetFileStatMulti( array $params ) {
1561 return null; // not supported
1562 }
1563
1571 abstract protected function directoriesAreVirtual();
1572
1583 final protected static function isValidShortContainerName( $container ) {
1584 // Suffixes like '.xxx' (hex shard chars) or '.seg' (file segments)
1585 // might be used by subclasses. Reserve the dot character.
1586 // The only way dots end up in containers (e.g. resolveStoragePath)
1587 // is due to the wikiId container prefix or the above suffixes.
1588 return self::isValidContainerName( $container ) && !preg_match( '/[.]/', $container );
1589 }
1590
1600 final protected static function isValidContainerName( $container ) {
1601 // This accounts for NTFS, Swift, and Ceph restrictions
1602 // and disallows directory separators or traversal characters.
1603 // Note that matching strings URL encode to the same string;
1604 // in Swift/Ceph, the length restriction is *after* URL encoding.
1605 return (bool)preg_match( '/^[a-z0-9][a-z0-9-_.]{0,199}$/i', $container );
1606 }
1607
1621 final protected function resolveStoragePath( $storagePath ) {
1622 [ $backend, $shortCont, $relPath ] = self::splitStoragePath( $storagePath );
1623 if ( $backend === $this->name && $relPath !== null ) { // must be for this backend
1624 $relPath = self::normalizeContainerPath( $relPath );
1625 if ( $relPath !== null && self::isValidShortContainerName( $shortCont ) ) {
1626 // Get shard for the normalized path if this container is sharded
1627 $cShard = $this->getContainerShard( $shortCont, $relPath );
1628 // Validate and sanitize the relative path (backend-specific)
1629 $relPath = $this->resolveContainerPath( $shortCont, $relPath );
1630 if ( $relPath !== null ) {
1631 // Prepend any domain ID prefix to the container name
1632 $container = $this->fullContainerName( $shortCont );
1633 if ( self::isValidContainerName( $container ) ) {
1634 // Validate and sanitize the container name (backend-specific)
1635 $container = $this->resolveContainerName( "{$container}{$cShard}" );
1636 if ( $container !== null ) {
1637 return [ $container, $relPath, $cShard ];
1638 }
1639 }
1640 }
1641 }
1642 }
1643
1644 return [ null, null, null ];
1645 }
1646
1662 final protected function resolveStoragePathReal( $storagePath ) {
1663 [ $container, $relPath, $cShard ] = $this->resolveStoragePath( $storagePath );
1664 if ( $cShard !== null && substr( $relPath, -1 ) !== '/' ) {
1665 return [ $container, $relPath ];
1666 }
1667
1668 return [ null, null ];
1669 }
1670
1679 final protected function getContainerShard( $container, $relPath ) {
1680 [ $levels, $base, $repeat ] = $this->getContainerHashLevels( $container );
1681 if ( $levels == 1 || $levels == 2 ) {
1682 // Hash characters are either base 16 or 36
1683 $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]';
1684 // Get a regex that represents the shard portion of paths.
1685 // The concatenation of the captures gives us the shard.
1686 if ( $levels === 1 ) { // 16 or 36 shards per container
1687 $hashDirRegex = '(' . $char . ')';
1688 } else { // 256 or 1296 shards per container
1689 if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc")
1690 $hashDirRegex = $char . '/(' . $char . '{2})';
1691 } else { // short hash dir format (e.g. "a/b/c")
1692 $hashDirRegex = '(' . $char . ')/(' . $char . ')';
1693 }
1694 }
1695 // Allow certain directories to be above the hash dirs so as
1696 // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
1697 // They must be 2+ chars to avoid any hash directory ambiguity.
1698 $m = [];
1699 if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1700 return '.' . implode( '', array_slice( $m, 1 ) );
1701 }
1702
1703 return null; // failed to match
1704 }
1705
1706 return ''; // no sharding
1707 }
1708
1717 final public function isSingleShardPathInternal( $storagePath ) {
1718 [ , , $shard ] = $this->resolveStoragePath( $storagePath );
1719
1720 return ( $shard !== null );
1721 }
1722
1731 final protected function getContainerHashLevels( $container ) {
1732 if ( isset( $this->shardViaHashLevels[$container] ) ) {
1733 $config = $this->shardViaHashLevels[$container];
1734 $hashLevels = (int)$config['levels'];
1735 if ( $hashLevels == 1 || $hashLevels == 2 ) {
1736 $hashBase = (int)$config['base'];
1737 if ( $hashBase == 16 || $hashBase == 36 ) {
1738 return [ $hashLevels, $hashBase, $config['repeat'] ];
1739 }
1740 }
1741 }
1742
1743 return [ 0, 0, false ]; // no sharding
1744 }
1745
1752 final protected function getContainerSuffixes( $container ) {
1753 $shards = [];
1754 [ $digits, $base ] = $this->getContainerHashLevels( $container );
1755 if ( $digits > 0 ) {
1756 $numShards = $base ** $digits;
1757 for ( $index = 0; $index < $numShards; $index++ ) {
1758 $shards[] = '.' . \Wikimedia\base_convert( (string)$index, 10, $base, $digits );
1759 }
1760 }
1761
1762 return $shards;
1763 }
1764
1771 final protected function fullContainerName( $container ) {
1772 if ( $this->domainId != '' ) {
1773 return "{$this->domainId}-$container";
1774 } else {
1775 return $container;
1776 }
1777 }
1778
1788 protected function resolveContainerName( $container ) {
1789 return $container;
1790 }
1791
1803 protected function resolveContainerPath( $container, $relStoragePath ) {
1804 return $relStoragePath;
1805 }
1806
1813 private function containerCacheKey( $container ) {
1814 return "filebackend:{$this->name}:{$this->domainId}:container:{$container}";
1815 }
1816
1823 final protected function setContainerCache( $container, array $val ) {
1824 if ( !$this->memCache->set( $this->containerCacheKey( $container ), $val, 14 * 86400 ) ) {
1825 $this->logger->warning( "Unable to set stat cache for container {container}.",
1826 [ 'filebackend' => $this->name, 'container' => $container ]
1827 );
1828 }
1829 }
1830
1837 final protected function deleteContainerCache( $container ) {
1838 if ( !$this->memCache->delete( $this->containerCacheKey( $container ), 300 ) ) {
1839 $this->logger->warning( "Unable to delete stat cache for container {container}.",
1840 [ 'filebackend' => $this->name, 'container' => $container ]
1841 );
1842 }
1843 }
1844
1850 final protected function primeContainerCache( array $items ) {
1852 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1853
1854 $paths = []; // list of storage paths
1855 $contNames = []; // (cache key => resolved container name)
1856 // Get all the paths/containers from the items...
1857 foreach ( $items as $item ) {
1858 if ( self::isStoragePath( $item ) ) {
1859 $paths[] = $item;
1860 } elseif ( is_string( $item ) ) { // full container name
1861 $contNames[$this->containerCacheKey( $item )] = $item;
1862 }
1863 }
1864 // Get all the corresponding cache keys for paths...
1865 foreach ( $paths as $path ) {
1866 [ $fullCont, , ] = $this->resolveStoragePath( $path );
1867 if ( $fullCont !== null ) { // valid path for this backend
1868 $contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
1869 }
1870 }
1871
1872 $contInfo = []; // (resolved container name => cache value)
1873 // Get all cache entries for these container cache keys...
1874 $values = $this->memCache->getMulti( array_keys( $contNames ) );
1875 foreach ( $values as $cacheKey => $val ) {
1876 $contInfo[$contNames[$cacheKey]] = $val;
1877 }
1878
1879 // Populate the container process cache for the backend...
1880 $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) );
1881 }
1882
1891 protected function doPrimeContainerCache( array $containerInfo ) {
1892 }
1893
1900 private function fileCacheKey( $path ) {
1901 return "filebackend:{$this->name}:{$this->domainId}:file:" . sha1( $path );
1902 }
1903
1912 final protected function setFileCache( $path, array $val ) {
1914 if ( $path === null ) {
1915 return; // invalid storage path
1916 }
1917 $mtime = (int)ConvertibleTimestamp::convert( TS_UNIX, $val['mtime'] );
1918 $ttl = $this->memCache->adaptiveTTL( $mtime, 7 * 86400, 300, 0.1 );
1919 $key = $this->fileCacheKey( $path );
1920 // Set the cache unless it is currently salted.
1921 if ( !$this->memCache->set( $key, $val, $ttl ) ) {
1922 $this->logger->warning( "Unable to set stat cache for file {path}.",
1923 [ 'filebackend' => $this->name, 'path' => $path ]
1924 );
1925 }
1926 }
1927
1936 final protected function deleteFileCache( $path ) {
1938 if ( $path === null ) {
1939 return; // invalid storage path
1940 }
1941 if ( !$this->memCache->delete( $this->fileCacheKey( $path ), 300 ) ) {
1942 $this->logger->warning( "Unable to delete stat cache for file {path}.",
1943 [ 'filebackend' => $this->name, 'path' => $path ]
1944 );
1945 }
1946 }
1947
1955 final protected function primeFileCache( array $items ) {
1957 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1958
1959 $paths = []; // list of storage paths
1960 $pathNames = []; // (cache key => storage path)
1961 // Get all the paths/containers from the items...
1962 foreach ( $items as $item ) {
1963 if ( self::isStoragePath( $item ) ) {
1965 if ( $path !== null ) {
1966 $paths[] = $path;
1967 }
1968 }
1969 }
1970 // Get all the corresponding cache keys for paths...
1971 foreach ( $paths as $path ) {
1972 [ , $rel, ] = $this->resolveStoragePath( $path );
1973 if ( $rel !== null ) { // valid path for this backend
1974 $pathNames[$this->fileCacheKey( $path )] = $path;
1975 }
1976 }
1977 // Get all cache entries for these file cache keys.
1978 // Note that negatives are not cached by getFileStat()/preloadFileStat().
1979 $values = $this->memCache->getMulti( array_keys( $pathNames ) );
1980 // Load all of the results into process cache...
1981 foreach ( array_filter( $values, 'is_array' ) as $cacheKey => $stat ) {
1982 $path = $pathNames[$cacheKey];
1983 // This flag only applies to stat info loaded directly
1984 // from a high consistency backend query to the process cache
1985 unset( $stat['latest'] );
1986
1987 $this->cheapCache->setField( $path, 'stat', $stat );
1988 if ( isset( $stat['sha1'] ) && strlen( $stat['sha1'] ) == 31 ) {
1989 // Some backends store SHA-1 as metadata
1990 $this->cheapCache->setField(
1991 $path,
1992 'sha1',
1993 [ 'hash' => $stat['sha1'], 'latest' => false ]
1994 );
1995 }
1996 if ( isset( $stat['xattr'] ) && is_array( $stat['xattr'] ) ) {
1997 // Some backends store custom headers/metadata
1998 $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] );
1999 $this->cheapCache->setField(
2000 $path,
2001 'xattr',
2002 [ 'map' => $stat['xattr'], 'latest' => false ]
2003 );
2004 }
2005 }
2006 }
2007
2015 final protected static function normalizeXAttributes( array $xattr ) {
2016 $newXAttr = [ 'headers' => [], 'metadata' => [] ];
2017
2018 foreach ( $xattr['headers'] as $name => $value ) {
2019 $newXAttr['headers'][strtolower( $name )] = $value;
2020 }
2021
2022 foreach ( $xattr['metadata'] as $name => $value ) {
2023 $newXAttr['metadata'][strtolower( $name )] = $value;
2024 }
2025
2026 return $newXAttr;
2027 }
2028
2035 final protected function setConcurrencyFlags( array $opts ) {
2036 $opts['concurrency'] = 1; // off
2037 if ( $this->parallelize === 'implicit' ) {
2038 if ( $opts['parallelize'] ?? true ) {
2039 $opts['concurrency'] = $this->concurrency;
2040 }
2041 } elseif ( $this->parallelize === 'explicit' ) {
2042 if ( !empty( $opts['parallelize'] ) ) {
2043 $opts['concurrency'] = $this->concurrency;
2044 }
2045 }
2046
2047 return $opts;
2048 }
2049
2059 protected function getContentType( $storagePath, $content, $fsPath ) {
2060 if ( $this->mimeCallback ) {
2061 return ( $this->mimeCallback )( $storagePath, $content, $fsPath );
2062 }
2063
2064 $mime = ( $fsPath !== null ) ? mime_content_type( $fsPath ) : false;
2065 return $mime ?: 'unknown/unknown';
2066 }
2067}
2068
2070class_alias( FileBackendStore::class, 'FileBackendStore' );
Resource locking handling.
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Class representing a non-directory file on the file system.
Definition FSFile.php:34
File backend exception for checked exceptions (e.g.
Base class for all backends using particular storage medium.
getContainerHashLevels( $container)
Get the sharding config for a container.
createInternal(array $params)
Create a file in the backend with the given contents.
static isValidContainerName( $container)
Check if a full container name is valid.
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
preloadCache(array $paths)
Preload persistent file stat cache and property cache into in-process cache.
getLocalCopyMulti(array $params)
Like getLocalCopy() except it takes an array of storage paths and yields an order preserved-map of st...
getFileXAttributes(array $params)
Get metadata about a file at a storage path in the backend.
getFileList(array $params)
Get an iterator to list all stored files under a storage directory.
getContentType( $storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
doDirectoryExists( $container, $dir, array $params)
doOperationsInternal(array $ops, array $opts)
ingestFreshFileStats(array $stats, $latest)
Ingest file stat entries that just came from querying the backend (not cache)
moveInternal(array $params)
Move a file from one storage path to another in the backend.
getContainerSuffixes( $container)
Get a list of full container shard suffixes for a container.
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
getPathsToLockForOpsInternal(array $performOps)
Get a list of storage paths to lock for a list of operations Returns an array with LockManager::LOCK_...
getContainerShard( $container, $relPath)
Get the container name shard suffix for a given path.
executeOpHandlesInternal(array $fileOpHandles)
Execute a list of FileBackendStoreOpHandle handles in parallel.
primeFileCache(array $items)
Do a batch lookup from cache for file stats for all paths used in a list of storage paths or FileOp o...
setConcurrencyFlags(array $opts)
Set the 'concurrency' option from a list of operation options.
getScopedLocksForOps(array $ops, StatusValue $status)
Get an array of scoped locks needed for a batch of file operations.
concatenate(array $params)
Concatenate a list of storage files into a single file system file.
describeInternal(array $params)
Alter metadata for a file at the storage path.
MapCacheLRU $cheapCache
Map of paths to small (RAM/disk) cache items.
static normalizeXAttributes(array $xattr)
Normalize file headers/metadata to the FileBackend::getFileXAttributes() format.
MapCacheLRU $expensiveCache
Map of paths to large (RAM/disk) cache items.
directoryExists(array $params)
Check if a directory exists at a given storage path.
getFileContentsMulti(array $params)
Like getFileContents() except it takes an array of storage paths and returns an order preserved map o...
doQuickOperationsInternal(array $ops, array $opts)
setFileCache( $path, array $val)
Set the cached stat info for a file path.
doPrimeContainerCache(array $containerInfo)
Fill the backend-specific process cache given an array of resolved container names and their correspo...
static isValidShortContainerName( $container)
Check if a short container name is valid.
isSingleShardPathInternal( $storagePath)
Check if a storage path maps to a single shard.
doSecureInternal( $container, $dir, array $params)
storeInternal(array $params)
Store a file into the backend from a file on disk.
deleteInternal(array $params)
Delete a file at the storage path.
doGetFileStatMulti(array $params)
Get file stat information (concurrently if possible) for several files.
getFileProps(array $params)
Get the properties of the content of the file at a storage path in the backend.
setContainerCache( $container, array $val)
Set the cached info for a container.
maxFileSizeInternal()
Get the maximum allowable file size given backend medium restrictions and basic performance constrain...
doClearCache(?array $paths=null)
Clears any additional stat caches for storage paths.
int $maxFileSize
Size in bytes, defaults to 32 GiB.
doPrepare(array $params)
FileBackend::prepare() StatusValue Good status without value for success, fatal otherwise.
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
fullContainerName( $container)
Get the full container name, including the domain ID prefix.
getDirectoryListInternal( $container, $dir, array $params)
Do not call this function from places outside FileBackend.
fileExists(array $params)
Check if a file exists at a storage path in the backend.
callable null $mimeCallback
Method to get the MIME type of files.
deleteContainerCache( $container)
Delete the cached info for a container.
array< string, array > $shardViaHashLevels
Map of container names to sharding config.
getFileListInternal( $container, $dir, array $params)
Do not call this function from places outside FileBackend.
streamFile(array $params)
Stream the content of the file at a storage path in the backend.
clearCache(?array $paths=null)
Invalidate any in-process file stat and property cache.
getFileTimestamp(array $params)
Get the last-modified timestamp of the file at a storage path.
deleteFileCache( $path)
Delete the cached stat info for a file path.
resolveContainerName( $container)
Resolve a container name, checking if it's allowed by the backend.
doPrepareInternal( $container, $dir, array $params)
copyInternal(array $params)
Copy a file from one storage path to another in the backend.
resolveStoragePath( $storagePath)
Splits a storage path into an internal container name, an internal relative file name,...
getLocalReferenceMulti(array $params)
Like getLocalReference() except it takes an array of storage paths and yields an order-preserved map ...
getDirectoryList(array $params)
Get an iterator to list all directories under a storage directory.
doPublishInternal( $container, $dir, array $params)
preloadFileStat(array $params)
Preload file stat information (concurrently if possible) into in-process cache.
sanitizeOpHeaders(array $op)
Normalize and filter HTTP headers from a file operation.
getFileSize(array $params)
Get the size (bytes) of a file at a storage path in the backend.
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path in the backend.
nullInternal(array $params)
No-op file operation that does nothing.
doCleanInternal( $container, $dir, array $params)
getOperationsInternal(array $ops)
Return a list of FileOp objects from a list of operations.
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
getFileSha1Base36(array $params)
Get a SHA-1 hash of the content of the file at a storage path in the backend.
addShellboxInputFile(BoxedCommand $command, string $boxedName, array $params)
Add a file to a Shellbox command as an input file.
Base class for all file backend classes (including multi-write backends).
string $name
Unique backend name.
static normalizeContainerPath( $path)
Validate and normalize a relative storage path.
static splitStoragePath( $storagePath)
Split a storage path into a backend name, a container name, and a relative file path.
getLocalReference(array $params)
Returns a file system file, identical in content to the file at a storage path.
getTopDirectoryList(array $params)
Same as FileBackend::getDirectoryList() except only lists directories that are immediately under the ...
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
static normalizeStoragePath( $storagePath)
Normalize a storage path by cleaning up directory separators.
newStatus( $message=null,... $params)
Yields the result of the status wrapper callback on either:
int $concurrency
How many operations can be done in parallel.
static attempt(array $performOps, array $opts)
Attempt to perform a series of file operations.
FileBackendStore helper class for performing asynchronous file operations.
Copy a file from one storage path to another in the backend.
Create a file in the backend with the given content.
Delete a file at the given storage path from the backend.
Change metadata for a file at the given storage path in the backend.
FileBackend helper class for representing operations.
Definition FileOp.php:46
Move a file from one storage path to another in the backend.
Placeholder operation that has no params and does nothing.
Store a file into the backend from a file on the file system.
Functions related to the output of file content.
static send404Message( $fname, $flags=0)
Send out a standard 404 message for a file.
Store key-value entries in a size-limited in-memory LRU cache.
Abstract class for any ephemeral data store.
Definition BagOStuff.php:87
No-op implementation that stores nothing.
Multi-datacenter aware caching interface.