MediaWiki REL1_39
FileBackendStore.php
Go to the documentation of this file.
1<?php
24use Wikimedia\AtEase\AtEase;
25use Wikimedia\Timestamp\ConvertibleTimestamp;
26
41abstract class FileBackendStore extends FileBackend {
43 protected $memCache;
45 protected $srvCache;
47 protected $cheapCache;
49 protected $expensiveCache;
50
52 protected $shardViaHashLevels = [];
53
55 protected $mimeCallback;
56
57 protected $maxFileSize = 4294967296; // integer bytes (4GiB)
58
59 protected const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries
60 protected const CACHE_CHEAP_SIZE = 500; // integer; max entries in "cheap cache"
61 protected const CACHE_EXPENSIVE_SIZE = 5; // integer; max entries in "expensive cache"
62
64 protected static $RES_ABSENT = false;
66 protected static $RES_ERROR = null;
67
69 protected static $ABSENT_NORMAL = 'FNE-N';
71 protected static $ABSENT_LATEST = 'FNE-L';
72
86 public function __construct( array $config ) {
87 parent::__construct( $config );
88 $this->mimeCallback = $config['mimeCallback'] ?? null;
89 $this->srvCache = new EmptyBagOStuff(); // disabled by default
90 $this->memCache = WANObjectCache::newEmpty(); // disabled by default
91 $this->cheapCache = new MapCacheLRU( self::CACHE_CHEAP_SIZE );
92 $this->expensiveCache = new MapCacheLRU( self::CACHE_EXPENSIVE_SIZE );
93 }
94
102 final public function maxFileSizeInternal() {
103 return $this->maxFileSize;
104 }
105
116 abstract public function isPathUsableInternal( $storagePath );
117
136 final public function createInternal( array $params ) {
138 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
139
140 if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
141 $status = $this->newStatus( 'backend-fail-maxsize',
142 $params['dst'], $this->maxFileSizeInternal() );
143 } else {
144 $status = $this->doCreateInternal( $params );
145 $this->clearCache( [ $params['dst'] ] );
146 if ( $params['dstExists'] ?? true ) {
147 $this->deleteFileCache( $params['dst'] ); // persistent cache
148 }
149 }
150
151 return $status;
152 }
153
159 abstract protected function doCreateInternal( array $params );
160
179 final public function storeInternal( array $params ) {
181 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
182
183 if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
184 $status = $this->newStatus( 'backend-fail-maxsize',
185 $params['dst'], $this->maxFileSizeInternal() );
186 } else {
187 $status = $this->doStoreInternal( $params );
188 $this->clearCache( [ $params['dst'] ] );
189 if ( $params['dstExists'] ?? true ) {
190 $this->deleteFileCache( $params['dst'] ); // persistent cache
191 }
192 }
193
194 return $status;
195 }
196
202 abstract protected function doStoreInternal( array $params );
203
223 final public function copyInternal( array $params ) {
225 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
226
227 $status = $this->doCopyInternal( $params );
228 $this->clearCache( [ $params['dst'] ] );
229 if ( $params['dstExists'] ?? true ) {
230 $this->deleteFileCache( $params['dst'] ); // persistent cache
231 }
232
233 return $status;
234 }
235
241 abstract protected function doCopyInternal( array $params );
242
257 final public function deleteInternal( array $params ) {
259 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
260
261 $status = $this->doDeleteInternal( $params );
262 $this->clearCache( [ $params['src'] ] );
263 $this->deleteFileCache( $params['src'] ); // persistent cache
264 return $status;
265 }
266
272 abstract protected function doDeleteInternal( array $params );
273
293 final public function moveInternal( array $params ) {
295 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
296
297 $status = $this->doMoveInternal( $params );
298 $this->clearCache( [ $params['src'], $params['dst'] ] );
299 $this->deleteFileCache( $params['src'] ); // persistent cache
300 if ( $params['dstExists'] ?? true ) {
301 $this->deleteFileCache( $params['dst'] ); // persistent cache
302 }
303
304 return $status;
305 }
306
313 protected function doMoveInternal( array $params ) {
314 unset( $params['async'] ); // two steps, won't work here :)
315 $nsrc = FileBackend::normalizeStoragePath( $params['src'] );
316 $ndst = FileBackend::normalizeStoragePath( $params['dst'] );
317 // Copy source to dest
318 $status = $this->copyInternal( $params );
319 if ( $nsrc !== $ndst && $status->isOK() ) {
320 // Delete source (only fails due to races or network problems)
321 $status->merge( $this->deleteInternal( [ 'src' => $params['src'] ] ) );
322 $status->setResult( true, $status->value ); // ignore delete() errors
323 }
324
325 return $status;
326 }
327
342 final public function describeInternal( array $params ) {
344 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
345
346 if ( count( $params['headers'] ) ) {
347 $status = $this->doDescribeInternal( $params );
348 $this->clearCache( [ $params['src'] ] );
349 $this->deleteFileCache( $params['src'] ); // persistent cache
350 } else {
351 $status = $this->newStatus(); // nothing to do
352 }
353
354 return $status;
355 }
356
363 protected function doDescribeInternal( array $params ) {
364 return $this->newStatus();
365 }
366
374 final public function nullInternal( array $params ) {
375 return $this->newStatus();
376 }
377
378 final public function concatenate( array $params ) {
380 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
381 $status = $this->newStatus();
382
383 // Try to lock the source files for the scope of this function
385 $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
386 if ( $status->isOK() ) {
387 // Actually do the file concatenation...
388 $start_time = microtime( true );
389 $status->merge( $this->doConcatenate( $params ) );
390 $sec = microtime( true ) - $start_time;
391 if ( !$status->isOK() ) {
392 $this->logger->error( static::class . "-{$this->name}" .
393 " failed to concatenate " . count( $params['srcs'] ) . " file(s) [$sec sec]" );
394 }
395 }
396
397 return $status;
398 }
399
406 protected function doConcatenate( array $params ) {
407 $status = $this->newStatus();
408 $tmpPath = $params['dst'];
409 unset( $params['latest'] );
410
411 // Check that the specified temp file is valid...
412 AtEase::suppressWarnings();
413 $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 );
414 AtEase::restoreWarnings();
415 if ( !$ok ) { // not present or not empty
416 $status->fatal( 'backend-fail-opentemp', $tmpPath );
417
418 return $status;
419 }
420
421 // Get local FS versions of the chunks needed for the concatenation...
422 $fsFiles = $this->getLocalReferenceMulti( $params );
423 foreach ( $fsFiles as $path => &$fsFile ) {
424 if ( !$fsFile ) { // chunk failed to download?
425 $fsFile = $this->getLocalReference( [ 'src' => $path ] );
426 if ( !$fsFile ) { // retry failed?
427 $status->fatal( 'backend-fail-read', $path );
428
429 return $status;
430 }
431 }
432 }
433 unset( $fsFile ); // unset reference so we can reuse $fsFile
434
435 // Get a handle for the destination temp file
436 $tmpHandle = fopen( $tmpPath, 'ab' );
437 if ( $tmpHandle === false ) {
438 $status->fatal( 'backend-fail-opentemp', $tmpPath );
439
440 return $status;
441 }
442
443 // Build up the temp file using the source chunks (in order)...
444 foreach ( $fsFiles as $virtualSource => $fsFile ) {
445 // Get a handle to the local FS version
446 $sourceHandle = fopen( $fsFile->getPath(), 'rb' );
447 if ( $sourceHandle === false ) {
448 fclose( $tmpHandle );
449 $status->fatal( 'backend-fail-read', $virtualSource );
450
451 return $status;
452 }
453 // Append chunk to file (pass chunk size to avoid magic quotes)
454 if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
455 fclose( $sourceHandle );
456 fclose( $tmpHandle );
457 $status->fatal( 'backend-fail-writetemp', $tmpPath );
458
459 return $status;
460 }
461 fclose( $sourceHandle );
462 }
463 if ( !fclose( $tmpHandle ) ) {
464 $status->fatal( 'backend-fail-closetemp', $tmpPath );
465
466 return $status;
467 }
468
469 clearstatcache(); // temp file changed
470
471 return $status;
472 }
473
477 final protected function doPrepare( array $params ) {
479 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
480 $status = $this->newStatus();
481
482 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
483 if ( $dir === null ) {
484 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
485
486 return $status; // invalid storage path
487 }
488
489 if ( $shard !== null ) { // confined to a single container/shard
490 $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
491 } else { // directory is on several shards
492 $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
493 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
494 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
495 $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
496 }
497 }
498
499 return $status;
500 }
501
510 protected function doPrepareInternal( $container, $dir, array $params ) {
511 return $this->newStatus();
512 }
513
514 final protected function doSecure( array $params ) {
516 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
517 $status = $this->newStatus();
518
519 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
520 if ( $dir === null ) {
521 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
522
523 return $status; // invalid storage path
524 }
525
526 if ( $shard !== null ) { // confined to a single container/shard
527 $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
528 } else { // directory is on several shards
529 $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
530 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
531 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
532 $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
533 }
534 }
535
536 return $status;
537 }
538
547 protected function doSecureInternal( $container, $dir, array $params ) {
548 return $this->newStatus();
549 }
550
551 final protected function doPublish( array $params ) {
553 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
554 $status = $this->newStatus();
555
556 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
557 if ( $dir === null ) {
558 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
559
560 return $status; // invalid storage path
561 }
562
563 if ( $shard !== null ) { // confined to a single container/shard
564 $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
565 } else { // directory is on several shards
566 $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
567 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
568 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
569 $status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) );
570 }
571 }
572
573 return $status;
574 }
575
584 protected function doPublishInternal( $container, $dir, array $params ) {
585 return $this->newStatus();
586 }
587
588 final protected function doClean( array $params ) {
590 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
591 $status = $this->newStatus();
592
593 // Recursive: first delete all empty subdirs recursively
594 if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) {
595 $subDirsRel = $this->getTopDirectoryList( [ 'dir' => $params['dir'] ] );
596 if ( $subDirsRel !== null ) { // no errors
597 foreach ( $subDirsRel as $subDirRel ) {
598 $subDir = $params['dir'] . "/{$subDirRel}"; // full path
599 $status->merge( $this->doClean( [ 'dir' => $subDir ] + $params ) );
600 }
601 unset( $subDirsRel ); // free directory for rmdir() on Windows (for FS backends)
602 }
603 }
604
605 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
606 if ( $dir === null ) {
607 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
608
609 return $status; // invalid storage path
610 }
611
612 // Attempt to lock this directory...
613 $filesLockEx = [ $params['dir'] ];
615 $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
616 if ( !$status->isOK() ) {
617 return $status; // abort
618 }
619
620 if ( $shard !== null ) { // confined to a single container/shard
621 $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
622 $this->deleteContainerCache( $fullCont ); // purge cache
623 } else { // directory is on several shards
624 $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
625 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
626 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
627 $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
628 $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache
629 }
630 }
631
632 return $status;
633 }
634
643 protected function doCleanInternal( $container, $dir, array $params ) {
644 return $this->newStatus();
645 }
646
647 final public function fileExists( array $params ) {
649 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
650
651 $stat = $this->getFileStat( $params );
652 if ( is_array( $stat ) ) {
653 return true;
654 }
655
656 return ( $stat === self::$RES_ABSENT ) ? false : self::EXISTENCE_ERROR;
657 }
658
659 final public function getFileTimestamp( array $params ) {
661 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
662
663 $stat = $this->getFileStat( $params );
664 if ( is_array( $stat ) ) {
665 return $stat['mtime'];
666 }
667
668 return self::TIMESTAMP_FAIL; // all failure cases
669 }
670
671 final public function getFileSize( array $params ) {
673 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
674
675 $stat = $this->getFileStat( $params );
676 if ( is_array( $stat ) ) {
677 return $stat['size'];
678 }
679
680 return self::SIZE_FAIL; // all failure cases
681 }
682
683 final public function getFileStat( array $params ) {
685 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
686
687 $path = self::normalizeStoragePath( $params['src'] );
688 if ( $path === null ) {
689 return self::STAT_ERROR; // invalid storage path
690 }
691
692 // Whether to bypass cache except for process cache entries loaded directly from
693 // high consistency backend queries (caller handles any cache flushing and locking)
694 $latest = !empty( $params['latest'] );
695 // Whether to ignore cache entries missing the SHA-1 field for existing files
696 $requireSHA1 = !empty( $params['requireSHA1'] );
697
698 $stat = $this->cheapCache->getField( $path, 'stat', self::CACHE_TTL );
699 // Load the persistent stat cache into process cache if needed
700 if ( !$latest ) {
701 if (
702 // File stat is not in process cache
703 $stat === null ||
704 // Key/value store backends might opportunistically set file stat process
705 // cache entries from object listings that do not include the SHA-1. In that
706 // case, loading the persistent stat cache will likely yield the SHA-1.
707 ( $requireSHA1 && is_array( $stat ) && !isset( $stat['sha1'] ) )
708 ) {
709 $this->primeFileCache( [ $path ] );
710 // Get any newly process-cached entry
711 $stat = $this->cheapCache->getField( $path, 'stat', self::CACHE_TTL );
712 }
713 }
714
715 if ( is_array( $stat ) ) {
716 if (
717 ( !$latest || $stat['latest'] ) &&
718 ( !$requireSHA1 || isset( $stat['sha1'] ) )
719 ) {
720 return $stat;
721 }
722 } elseif ( $stat === self::$ABSENT_LATEST ) {
723 return self::STAT_ABSENT;
724 } elseif ( $stat === self::$ABSENT_NORMAL ) {
725 if ( !$latest ) {
726 return self::STAT_ABSENT;
727 }
728 }
729
730 // Load the file stat from the backend and update caches
731 $stat = $this->doGetFileStat( $params );
732 $this->ingestFreshFileStats( [ $path => $stat ], $latest );
733
734 if ( is_array( $stat ) ) {
735 return $stat;
736 }
737
738 return ( $stat === self::$RES_ERROR ) ? self::STAT_ERROR : self::STAT_ABSENT;
739 }
740
748 final protected function ingestFreshFileStats( array $stats, $latest ) {
749 $success = true;
750
751 foreach ( $stats as $path => $stat ) {
752 if ( is_array( $stat ) ) {
753 // Strongly consistent backends might automatically set this flag
754 $stat['latest'] = $stat['latest'] ?? $latest;
755
756 $this->cheapCache->setField( $path, 'stat', $stat );
757 if ( isset( $stat['sha1'] ) ) {
758 // Some backends store the SHA-1 hash as metadata
759 $this->cheapCache->setField(
760 $path,
761 'sha1',
762 [ 'hash' => $stat['sha1'], 'latest' => $latest ]
763 );
764 }
765 if ( isset( $stat['xattr'] ) ) {
766 // Some backends store custom headers/metadata
767 $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] );
768 $this->cheapCache->setField(
769 $path,
770 'xattr',
771 [ 'map' => $stat['xattr'], 'latest' => $latest ]
772 );
773 }
774 // Update persistent cache (@TODO: set all entries in one batch)
775 $this->setFileCache( $path, $stat );
776 } elseif ( $stat === self::$RES_ABSENT ) {
777 $this->cheapCache->setField(
778 $path,
779 'stat',
780 $latest ? self::$ABSENT_LATEST : self::$ABSENT_NORMAL
781 );
782 $this->cheapCache->setField(
783 $path,
784 'xattr',
785 [ 'map' => self::XATTRS_FAIL, 'latest' => $latest ]
786 );
787 $this->cheapCache->setField(
788 $path,
789 'sha1',
790 [ 'hash' => self::SHA1_FAIL, 'latest' => $latest ]
791 );
792 $this->logger->debug(
793 __METHOD__ . ': File {path} does not exist',
794 [ 'path' => $path ]
795 );
796 } else {
797 $success = false;
798 $this->logger->error(
799 __METHOD__ . ': Could not stat file {path}',
800 [ 'path' => $path ]
801 );
802 }
803 }
804
805 return $success;
806 }
807
812 abstract protected function doGetFileStat( array $params );
813
814 public function getFileContentsMulti( array $params ) {
816 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
817
818 $params = $this->setConcurrencyFlags( $params );
819 $contents = $this->doGetFileContentsMulti( $params );
820 foreach ( $contents as $path => $content ) {
821 if ( !is_string( $content ) ) {
822 $contents[$path] = self::CONTENT_FAIL; // used for all failure cases
823 }
824 }
825
826 return $contents;
827 }
828
835 protected function doGetFileContentsMulti( array $params ) {
836 $contents = [];
837 foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
838 if ( $fsFile instanceof FSFile ) {
839 AtEase::suppressWarnings();
840 $content = file_get_contents( $fsFile->getPath() );
841 AtEase::restoreWarnings();
842 $contents[$path] = is_string( $content ) ? $content : self::$RES_ERROR;
843 } elseif ( $fsFile === self::$RES_ABSENT ) {
844 $contents[$path] = self::$RES_ABSENT;
845 } else {
846 $contents[$path] = self::$RES_ERROR;
847 }
848 }
849
850 return $contents;
851 }
852
853 final public function getFileXAttributes( array $params ) {
855 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
856
857 $path = self::normalizeStoragePath( $params['src'] );
858 if ( $path === null ) {
859 return self::XATTRS_FAIL; // invalid storage path
860 }
861 $latest = !empty( $params['latest'] ); // use latest data?
862 if ( $this->cheapCache->hasField( $path, 'xattr', self::CACHE_TTL ) ) {
863 $stat = $this->cheapCache->getField( $path, 'xattr' );
864 // If we want the latest data, check that this cached
865 // value was in fact fetched with the latest available data.
866 if ( !$latest || $stat['latest'] ) {
867 return $stat['map'];
868 }
869 }
870 $fields = $this->doGetFileXAttributes( $params );
871 if ( is_array( $fields ) ) {
872 $fields = self::normalizeXAttributes( $fields );
873 $this->cheapCache->setField(
874 $path,
875 'xattr',
876 [ 'map' => $fields, 'latest' => $latest ]
877 );
878 } elseif ( $fields === self::$RES_ABSENT ) {
879 $this->cheapCache->setField(
880 $path,
881 'xattr',
882 [ 'map' => self::XATTRS_FAIL, 'latest' => $latest ]
883 );
884 } else {
885 $fields = self::XATTRS_FAIL; // used for all failure cases
886 }
887
888 return $fields;
889 }
890
897 protected function doGetFileXAttributes( array $params ) {
898 return [ 'headers' => [], 'metadata' => [] ]; // not supported
899 }
900
901 final public function getFileSha1Base36( array $params ) {
903 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
904
905 $path = self::normalizeStoragePath( $params['src'] );
906 if ( $path === null ) {
907 return self::SHA1_FAIL; // invalid storage path
908 }
909 $latest = !empty( $params['latest'] ); // use latest data?
910 if ( $this->cheapCache->hasField( $path, 'sha1', self::CACHE_TTL ) ) {
911 $stat = $this->cheapCache->getField( $path, 'sha1' );
912 // If we want the latest data, check that this cached
913 // value was in fact fetched with the latest available data.
914 if ( !$latest || $stat['latest'] ) {
915 return $stat['hash'];
916 }
917 }
918 $sha1 = $this->doGetFileSha1Base36( $params );
919 if ( is_string( $sha1 ) ) {
920 $this->cheapCache->setField(
921 $path,
922 'sha1',
923 [ 'hash' => $sha1, 'latest' => $latest ]
924 );
925 } elseif ( $sha1 === self::$RES_ABSENT ) {
926 $this->cheapCache->setField(
927 $path,
928 'sha1',
929 [ 'hash' => self::SHA1_FAIL, 'latest' => $latest ]
930 );
931 } else {
932 $sha1 = self::SHA1_FAIL; // used for all failure cases
933 }
934
935 return $sha1;
936 }
937
944 protected function doGetFileSha1Base36( array $params ) {
945 $fsFile = $this->getLocalReference( $params );
946 if ( $fsFile instanceof FSFile ) {
947 $sha1 = $fsFile->getSha1Base36();
948
949 return is_string( $sha1 ) ? $sha1 : self::$RES_ERROR;
950 }
951
952 return ( $fsFile === self::$RES_ERROR ) ? self::$RES_ERROR : self::$RES_ABSENT;
953 }
954
955 final public function getFileProps( array $params ) {
957 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
958
959 $fsFile = $this->getLocalReference( $params );
960
961 return $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
962 }
963
964 final public function getLocalReferenceMulti( array $params ) {
966 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
967
968 $params = $this->setConcurrencyFlags( $params );
969
970 $fsFiles = []; // (path => FSFile)
971 $latest = !empty( $params['latest'] ); // use latest data?
972 // Reuse any files already in process cache...
973 foreach ( $params['srcs'] as $src ) {
975 if ( $path === null ) {
976 $fsFiles[$src] = null; // invalid storage path
977 } elseif ( $this->expensiveCache->hasField( $path, 'localRef' ) ) {
978 $val = $this->expensiveCache->getField( $path, 'localRef' );
979 // If we want the latest data, check that this cached
980 // value was in fact fetched with the latest available data.
981 if ( !$latest || $val['latest'] ) {
982 $fsFiles[$src] = $val['object'];
983 }
984 }
985 }
986 // Fetch local references of any remaining files...
987 $params['srcs'] = array_diff( $params['srcs'], array_keys( $fsFiles ) );
988 foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
989 if ( $fsFile instanceof FSFile ) {
990 $fsFiles[$path] = $fsFile;
991 $this->expensiveCache->setField(
992 $path,
993 'localRef',
994 [ 'object' => $fsFile, 'latest' => $latest ]
995 );
996 } else {
997 $fsFiles[$path] = null; // used for all failure cases
998 }
999 }
1000
1001 return $fsFiles;
1002 }
1003
1010 protected function doGetLocalReferenceMulti( array $params ) {
1011 return $this->doGetLocalCopyMulti( $params );
1012 }
1013
1014 final public function getLocalCopyMulti( array $params ) {
1016 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1017
1018 $params = $this->setConcurrencyFlags( $params );
1019 $tmpFiles = $this->doGetLocalCopyMulti( $params );
1020 foreach ( $tmpFiles as $path => $tmpFile ) {
1021 if ( !$tmpFile ) {
1022 $tmpFiles[$path] = null; // used for all failure cases
1023 }
1024 }
1025
1026 return $tmpFiles;
1027 }
1028
1034 abstract protected function doGetLocalCopyMulti( array $params );
1035
1042 public function getFileHttpUrl( array $params ) {
1043 return self::TEMPURL_ERROR; // not supported
1044 }
1045
1046 final public function streamFile( array $params ) {
1048 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1049 $status = $this->newStatus();
1050
1051 // Always set some fields for subclass convenience
1052 $params['options'] = $params['options'] ?? [];
1053 $params['headers'] = $params['headers'] ?? [];
1054
1055 // Don't stream it out as text/html if there was a PHP error
1056 if ( ( empty( $params['headless'] ) || $params['headers'] ) && headers_sent() ) {
1057 print "Headers already sent, terminating.\n";
1058 $status->fatal( 'backend-fail-stream', $params['src'] );
1059 return $status;
1060 }
1061
1062 $status->merge( $this->doStreamFile( $params ) );
1063
1064 return $status;
1065 }
1066
1073 protected function doStreamFile( array $params ) {
1074 $status = $this->newStatus();
1075
1076 $flags = 0;
1077 $flags |= !empty( $params['headless'] ) ? HTTPFileStreamer::STREAM_HEADLESS : 0;
1078 $flags |= !empty( $params['allowOB'] ) ? HTTPFileStreamer::STREAM_ALLOW_OB : 0;
1079
1080 $fsFile = $this->getLocalReference( $params );
1081 if ( $fsFile ) {
1082 $streamer = new HTTPFileStreamer(
1083 $fsFile->getPath(),
1084 [
1085 'obResetFunc' => $this->obResetFunc,
1086 'streamMimeFunc' => $this->streamMimeFunc
1087 ]
1088 );
1089 $res = $streamer->stream( $params['headers'], true, $params['options'], $flags );
1090 } else {
1091 $res = false;
1092 HTTPFileStreamer::send404Message( $params['src'], $flags );
1093 }
1094
1095 if ( !$res ) {
1096 $status->fatal( 'backend-fail-stream', $params['src'] );
1097 }
1098
1099 return $status;
1100 }
1101
1102 final public function directoryExists( array $params ) {
1103 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
1104 if ( $dir === null ) {
1105 return self::EXISTENCE_ERROR; // invalid storage path
1106 }
1107 if ( $shard !== null ) { // confined to a single container/shard
1108 return $this->doDirectoryExists( $fullCont, $dir, $params );
1109 } else { // directory is on several shards
1110 $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
1111 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
1112 $res = false; // response
1113 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
1114 $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params );
1115 if ( $exists === true ) {
1116 $res = true;
1117 break; // found one!
1118 } elseif ( $exists === self::$RES_ERROR ) {
1119 $res = self::EXISTENCE_ERROR;
1120 }
1121 }
1122
1123 return $res;
1124 }
1125 }
1126
1135 abstract protected function doDirectoryExists( $container, $dir, array $params );
1136
1137 final public function getDirectoryList( array $params ) {
1138 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
1139 if ( $dir === null ) {
1140 return self::EXISTENCE_ERROR; // invalid storage path
1141 }
1142 if ( $shard !== null ) {
1143 // File listing is confined to a single container/shard
1144 return $this->getDirectoryListInternal( $fullCont, $dir, $params );
1145 } else {
1146 $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
1147 // File listing spans multiple containers/shards
1148 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
1149
1150 return new FileBackendStoreShardDirIterator( $this,
1151 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
1152 }
1153 }
1154
1165 abstract public function getDirectoryListInternal( $container, $dir, array $params );
1166
1167 final public function getFileList( array $params ) {
1168 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
1169 if ( $dir === null ) {
1170 return self::LIST_ERROR; // invalid storage path
1171 }
1172 if ( $shard !== null ) {
1173 // File listing is confined to a single container/shard
1174 return $this->getFileListInternal( $fullCont, $dir, $params );
1175 } else {
1176 $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
1177 // File listing spans multiple containers/shards
1178 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
1179
1180 return new FileBackendStoreShardFileIterator( $this,
1181 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
1182 }
1183 }
1184
1195 abstract public function getFileListInternal( $container, $dir, array $params );
1196
1208 final public function getOperationsInternal( array $ops ) {
1209 $supportedOps = [
1210 'store' => StoreFileOp::class,
1211 'copy' => CopyFileOp::class,
1212 'move' => MoveFileOp::class,
1213 'delete' => DeleteFileOp::class,
1214 'create' => CreateFileOp::class,
1215 'describe' => DescribeFileOp::class,
1216 'null' => NullFileOp::class
1217 ];
1218
1219 $performOps = []; // array of FileOp objects
1220 // Build up ordered array of FileOps...
1221 foreach ( $ops as $operation ) {
1222 $opName = $operation['op'];
1223 if ( isset( $supportedOps[$opName] ) ) {
1224 $class = $supportedOps[$opName];
1225 // Get params for this operation
1226 $params = $operation;
1227 // Append the FileOp class
1228 $performOps[] = new $class( $this, $params, $this->logger );
1229 } else {
1230 throw new FileBackendError( "Operation '$opName' is not supported." );
1231 }
1232 }
1233
1234 return $performOps;
1235 }
1236
1247 final public function getPathsToLockForOpsInternal( array $performOps ) {
1248 // Build up a list of files to lock...
1249 $paths = [ 'sh' => [], 'ex' => [] ];
1250 foreach ( $performOps as $fileOp ) {
1251 $paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() );
1252 $paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() );
1253 }
1254 // Optimization: if doing an EX lock anyway, don't also set an SH one
1255 $paths['sh'] = array_diff( $paths['sh'], $paths['ex'] );
1256 // Get a shared lock on the parent directory of each path changed
1257 $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) );
1258
1259 return [
1260 LockManager::LOCK_UW => $paths['sh'],
1261 LockManager::LOCK_EX => $paths['ex']
1262 ];
1263 }
1264
1265 public function getScopedLocksForOps( array $ops, StatusValue $status ) {
1266 $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) );
1267
1268 return $this->getScopedFileLocks( $paths, 'mixed', $status );
1269 }
1270
1271 final protected function doOperationsInternal( array $ops, array $opts ) {
1273 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1274 $status = $this->newStatus();
1275
1276 // Fix up custom header name/value pairs
1277 $ops = array_map( [ $this, 'sanitizeOpHeaders' ], $ops );
1278 // Build up a list of FileOps and involved paths
1279 $fileOps = $this->getOperationsInternal( $ops );
1280 $pathsUsed = [];
1281 foreach ( $fileOps as $fileOp ) {
1282 $pathsUsed = array_merge( $pathsUsed, $fileOp->storagePathsReadOrChanged() );
1283 }
1284
1285 // Acquire any locks as needed for the scope of this function
1286 if ( empty( $opts['nonLocking'] ) ) {
1287 $pathsByLockType = $this->getPathsToLockForOpsInternal( $fileOps );
1289 $scopeLock = $this->getScopedFileLocks( $pathsByLockType, 'mixed', $status );
1290 if ( !$status->isOK() ) {
1291 return $status; // abort
1292 }
1293 }
1294
1295 // Clear any file cache entries (after locks acquired)
1296 if ( empty( $opts['preserveCache'] ) ) {
1297 $this->clearCache( $pathsUsed );
1298 }
1299
1300 // Enlarge the cache to fit the stat entries of these files
1301 $this->cheapCache->setMaxSize( max( 2 * count( $pathsUsed ), self::CACHE_CHEAP_SIZE ) );
1302
1303 // Load from the persistent container caches
1304 $this->primeContainerCache( $pathsUsed );
1305 // Get the latest stat info for all the files (having locked them)
1306 $ok = $this->preloadFileStat( [ 'srcs' => $pathsUsed, 'latest' => true ] );
1307
1308 if ( $ok ) {
1309 // Actually attempt the operation batch...
1310 $opts = $this->setConcurrencyFlags( $opts );
1311 $subStatus = FileOpBatch::attempt( $fileOps, $opts );
1312 } else {
1313 // If we could not even stat some files, then bail out
1314 $subStatus = $this->newStatus( 'backend-fail-internal', $this->name );
1315 foreach ( $ops as $i => $op ) { // mark each op as failed
1316 $subStatus->success[$i] = false;
1317 ++$subStatus->failCount;
1318 }
1319 $this->logger->error( static::class . "-{$this->name} " .
1320 " stat failure; aborted operations: " . FormatJson::encode( $ops ) );
1321 }
1322
1323 // Merge errors into StatusValue fields
1324 $status->merge( $subStatus );
1325 $status->success = $subStatus->success; // not done in merge()
1326
1327 // Shrink the stat cache back to normal size
1328 $this->cheapCache->setMaxSize( self::CACHE_CHEAP_SIZE );
1329
1330 return $status;
1331 }
1332
1333 final protected function doQuickOperationsInternal( array $ops, array $opts ) {
1335 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1336 $status = $this->newStatus();
1337
1338 // Fix up custom header name/value pairs
1339 $ops = array_map( [ $this, 'sanitizeOpHeaders' ], $ops );
1340 // Build up a list of FileOps and involved paths
1341 $fileOps = $this->getOperationsInternal( $ops );
1342 $pathsUsed = [];
1343 foreach ( $fileOps as $fileOp ) {
1344 $pathsUsed = array_merge( $pathsUsed, $fileOp->storagePathsReadOrChanged() );
1345 }
1346
1347 // Clear any file cache entries for involved paths
1348 $this->clearCache( $pathsUsed );
1349
1350 // Parallel ops may be disabled in config due to dependencies (e.g. needing popen())
1351 $async = ( $this->parallelize === 'implicit' && count( $ops ) > 1 );
1352 $maxConcurrency = $this->concurrency; // throttle
1354 $statuses = []; // array of (index => StatusValue)
1355 $fileOpHandles = []; // list of (index => handle) arrays
1356 $curFileOpHandles = []; // current handle batch
1357 // Perform the sync-only ops and build up op handles for the async ops...
1358 foreach ( $fileOps as $index => $fileOp ) {
1359 $subStatus = $async
1360 ? $fileOp->attemptAsyncQuick()
1361 : $fileOp->attemptQuick();
1362 if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { // async
1363 if ( count( $curFileOpHandles ) >= $maxConcurrency ) {
1364 $fileOpHandles[] = $curFileOpHandles; // push this batch
1365 $curFileOpHandles = [];
1366 }
1367 $curFileOpHandles[$index] = $subStatus->value; // keep index
1368 } else { // error or completed
1369 $statuses[$index] = $subStatus; // keep index
1370 }
1371 }
1372 if ( count( $curFileOpHandles ) ) {
1373 $fileOpHandles[] = $curFileOpHandles; // last batch
1374 }
1375 // Do all the async ops that can be done concurrently...
1376 foreach ( $fileOpHandles as $fileHandleBatch ) {
1377 $statuses += $this->executeOpHandlesInternal( $fileHandleBatch );
1378 }
1379 // Marshall and merge all the responses...
1380 foreach ( $statuses as $index => $subStatus ) {
1381 $status->merge( $subStatus );
1382 if ( $subStatus->isOK() ) {
1383 $status->success[$index] = true;
1384 ++$status->successCount;
1385 } else {
1386 $status->success[$index] = false;
1387 ++$status->failCount;
1388 }
1389 }
1390
1391 $this->clearCache( $pathsUsed );
1392
1393 return $status;
1394 }
1395
1405 final public function executeOpHandlesInternal( array $fileOpHandles ) {
1407 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1408
1409 foreach ( $fileOpHandles as $fileOpHandle ) {
1410 if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) {
1411 throw new InvalidArgumentException( "Expected FileBackendStoreOpHandle object." );
1412 } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) {
1413 throw new InvalidArgumentException( "Expected handle for this file backend." );
1414 }
1415 }
1416
1417 $statuses = $this->doExecuteOpHandlesInternal( $fileOpHandles );
1418 foreach ( $fileOpHandles as $fileOpHandle ) {
1419 $fileOpHandle->closeResources();
1420 }
1421
1422 return $statuses;
1423 }
1424
1434 protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1435 if ( count( $fileOpHandles ) ) {
1436 throw new FileBackendError( "Backend does not support asynchronous operations." );
1437 }
1438
1439 return [];
1440 }
1441
1453 protected function sanitizeOpHeaders( array $op ) {
1454 static $longs = [ 'content-disposition' ];
1455
1456 if ( isset( $op['headers'] ) ) { // op sets HTTP headers
1457 $newHeaders = [];
1458 foreach ( $op['headers'] as $name => $value ) {
1459 $name = strtolower( $name );
1460 $maxHVLen = in_array( $name, $longs ) ? INF : 255;
1461 if ( strlen( $name ) > 255 || strlen( $value ) > $maxHVLen ) {
1462 $this->logger->error( "Header '{header}' is too long.", [
1463 'filebackend' => $this->name,
1464 'header' => "$name: $value",
1465 ] );
1466 } else {
1467 $newHeaders[$name] = strlen( $value ) ? $value : ''; // null/false => ""
1468 }
1469 }
1470 $op['headers'] = $newHeaders;
1471 }
1472
1473 return $op;
1474 }
1475
1476 final public function preloadCache( array $paths ) {
1477 $fullConts = []; // full container names
1478 foreach ( $paths as $path ) {
1479 list( $fullCont, , ) = $this->resolveStoragePath( $path );
1480 $fullConts[] = $fullCont;
1481 }
1482 // Load from the persistent file and container caches
1483 $this->primeContainerCache( $fullConts );
1484 $this->primeFileCache( $paths );
1485 }
1486
1487 final public function clearCache( array $paths = null ) {
1488 if ( is_array( $paths ) ) {
1489 $paths = array_map( [ FileBackend::class, 'normalizeStoragePath' ], $paths );
1490 $paths = array_filter( $paths, 'strlen' ); // remove nulls
1491 }
1492 if ( $paths === null ) {
1493 $this->cheapCache->clear();
1494 $this->expensiveCache->clear();
1495 } else {
1496 foreach ( $paths as $path ) {
1497 $this->cheapCache->clear( $path );
1498 $this->expensiveCache->clear( $path );
1499 }
1500 }
1501 $this->doClearCache( $paths );
1502 }
1503
1512 protected function doClearCache( array $paths = null ) {
1513 }
1514
1515 final public function preloadFileStat( array $params ) {
1517 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1518
1519 $params['concurrency'] = ( $this->parallelize !== 'off' ) ? $this->concurrency : 1;
1520 $stats = $this->doGetFileStatMulti( $params );
1521 if ( $stats === null ) {
1522 return true; // not supported
1523 }
1524
1525 // Whether this queried the backend in high consistency mode
1526 $latest = !empty( $params['latest'] );
1527
1528 return $this->ingestFreshFileStats( $stats, $latest );
1529 }
1530
1543 protected function doGetFileStatMulti( array $params ) {
1544 return null; // not supported
1545 }
1546
1554 abstract protected function directoriesAreVirtual();
1555
1566 final protected static function isValidShortContainerName( $container ) {
1567 // Suffixes like '.xxx' (hex shard chars) or '.seg' (file segments)
1568 // might be used by subclasses. Reserve the dot character.
1569 // The only way dots end up in containers (e.g. resolveStoragePath)
1570 // is due to the wikiId container prefix or the above suffixes.
1571 return self::isValidContainerName( $container ) && !preg_match( '/[.]/', $container );
1572 }
1573
1583 final protected static function isValidContainerName( $container ) {
1584 // This accounts for NTFS, Swift, and Ceph restrictions
1585 // and disallows directory separators or traversal characters.
1586 // Note that matching strings URL encode to the same string;
1587 // in Swift/Ceph, the length restriction is *after* URL encoding.
1588 return (bool)preg_match( '/^[a-z0-9][a-z0-9-_.]{0,199}$/i', $container );
1589 }
1590
1604 final protected function resolveStoragePath( $storagePath ) {
1605 list( $backend, $shortCont, $relPath ) = self::splitStoragePath( $storagePath );
1606 if ( $backend === $this->name ) { // must be for this backend
1607 $relPath = self::normalizeContainerPath( $relPath );
1608 if ( $relPath !== null && self::isValidShortContainerName( $shortCont ) ) {
1609 // Get shard for the normalized path if this container is sharded
1610 $cShard = $this->getContainerShard( $shortCont, $relPath );
1611 // Validate and sanitize the relative path (backend-specific)
1612 $relPath = $this->resolveContainerPath( $shortCont, $relPath );
1613 if ( $relPath !== null ) {
1614 // Prepend any domain ID prefix to the container name
1615 $container = $this->fullContainerName( $shortCont );
1616 if ( self::isValidContainerName( $container ) ) {
1617 // Validate and sanitize the container name (backend-specific)
1618 $container = $this->resolveContainerName( "{$container}{$cShard}" );
1619 if ( $container !== null ) {
1620 return [ $container, $relPath, $cShard ];
1621 }
1622 }
1623 }
1624 }
1625 }
1626
1627 return [ null, null, null ];
1628 }
1629
1645 final protected function resolveStoragePathReal( $storagePath ) {
1646 list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
1647 if ( $cShard !== null && substr( $relPath, -1 ) !== '/' ) {
1648 return [ $container, $relPath ];
1649 }
1650
1651 return [ null, null ];
1652 }
1653
1662 final protected function getContainerShard( $container, $relPath ) {
1663 list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container );
1664 if ( $levels == 1 || $levels == 2 ) {
1665 // Hash characters are either base 16 or 36
1666 $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]';
1667 // Get a regex that represents the shard portion of paths.
1668 // The concatenation of the captures gives us the shard.
1669 if ( $levels === 1 ) { // 16 or 36 shards per container
1670 $hashDirRegex = '(' . $char . ')';
1671 } else { // 256 or 1296 shards per container
1672 if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc")
1673 $hashDirRegex = $char . '/(' . $char . '{2})';
1674 } else { // short hash dir format (e.g. "a/b/c")
1675 $hashDirRegex = '(' . $char . ')/(' . $char . ')';
1676 }
1677 }
1678 // Allow certain directories to be above the hash dirs so as
1679 // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
1680 // They must be 2+ chars to avoid any hash directory ambiguity.
1681 $m = [];
1682 if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1683 return '.' . implode( '', array_slice( $m, 1 ) );
1684 }
1685
1686 return null; // failed to match
1687 }
1688
1689 return ''; // no sharding
1690 }
1691
1700 final public function isSingleShardPathInternal( $storagePath ) {
1701 list( , , $shard ) = $this->resolveStoragePath( $storagePath );
1702
1703 return ( $shard !== null );
1704 }
1705
1714 final protected function getContainerHashLevels( $container ) {
1715 if ( isset( $this->shardViaHashLevels[$container] ) ) {
1716 $config = $this->shardViaHashLevels[$container];
1717 $hashLevels = (int)$config['levels'];
1718 if ( $hashLevels == 1 || $hashLevels == 2 ) {
1719 $hashBase = (int)$config['base'];
1720 if ( $hashBase == 16 || $hashBase == 36 ) {
1721 return [ $hashLevels, $hashBase, $config['repeat'] ];
1722 }
1723 }
1724 }
1725
1726 return [ 0, 0, false ]; // no sharding
1727 }
1728
1735 final protected function getContainerSuffixes( $container ) {
1736 $shards = [];
1737 list( $digits, $base ) = $this->getContainerHashLevels( $container );
1738 if ( $digits > 0 ) {
1739 $numShards = $base ** $digits;
1740 for ( $index = 0; $index < $numShards; $index++ ) {
1741 $shards[] = '.' . Wikimedia\base_convert( (string)$index, 10, $base, $digits );
1742 }
1743 }
1744
1745 return $shards;
1746 }
1747
1754 final protected function fullContainerName( $container ) {
1755 if ( $this->domainId != '' ) {
1756 return "{$this->domainId}-$container";
1757 } else {
1758 return $container;
1759 }
1760 }
1761
1771 protected function resolveContainerName( $container ) {
1772 return $container;
1773 }
1774
1786 protected function resolveContainerPath( $container, $relStoragePath ) {
1787 return $relStoragePath;
1788 }
1789
1796 private function containerCacheKey( $container ) {
1797 return "filebackend:{$this->name}:{$this->domainId}:container:{$container}";
1798 }
1799
1806 final protected function setContainerCache( $container, array $val ) {
1807 $this->memCache->set( $this->containerCacheKey( $container ), $val, 14 * 86400 );
1808 }
1809
1816 final protected function deleteContainerCache( $container ) {
1817 if ( !$this->memCache->delete( $this->containerCacheKey( $container ), 300 ) ) {
1818 $this->logger->warning( "Unable to delete stat cache for container {container}.",
1819 [ 'filebackend' => $this->name, 'container' => $container ]
1820 );
1821 }
1822 }
1823
1831 final protected function primeContainerCache( array $items ) {
1833 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1834
1835 $paths = []; // list of storage paths
1836 $contNames = []; // (cache key => resolved container name)
1837 // Get all the paths/containers from the items...
1838 foreach ( $items as $item ) {
1839 if ( self::isStoragePath( $item ) ) {
1840 $paths[] = $item;
1841 } elseif ( is_string( $item ) ) { // full container name
1842 $contNames[$this->containerCacheKey( $item )] = $item;
1843 }
1844 }
1845 // Get all the corresponding cache keys for paths...
1846 foreach ( $paths as $path ) {
1847 list( $fullCont, , ) = $this->resolveStoragePath( $path );
1848 if ( $fullCont !== null ) { // valid path for this backend
1849 $contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
1850 }
1851 }
1852
1853 $contInfo = []; // (resolved container name => cache value)
1854 // Get all cache entries for these container cache keys...
1855 $values = $this->memCache->getMulti( array_keys( $contNames ) );
1856 foreach ( $values as $cacheKey => $val ) {
1857 $contInfo[$contNames[$cacheKey]] = $val;
1858 }
1859
1860 // Populate the container process cache for the backend...
1861 $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) );
1862 }
1863
1872 protected function doPrimeContainerCache( array $containerInfo ) {
1873 }
1874
1881 private function fileCacheKey( $path ) {
1882 return "filebackend:{$this->name}:{$this->domainId}:file:" . sha1( $path );
1883 }
1884
1893 final protected function setFileCache( $path, array $val ) {
1895 if ( $path === null ) {
1896 return; // invalid storage path
1897 }
1898 $mtime = (int)ConvertibleTimestamp::convert( TS_UNIX, $val['mtime'] );
1899 $ttl = $this->memCache->adaptiveTTL( $mtime, 7 * 86400, 300, 0.1 );
1900 $key = $this->fileCacheKey( $path );
1901 // Set the cache unless it is currently salted.
1902 $this->memCache->set( $key, $val, $ttl );
1903 }
1904
1913 final protected function deleteFileCache( $path ) {
1915 if ( $path === null ) {
1916 return; // invalid storage path
1917 }
1918 if ( !$this->memCache->delete( $this->fileCacheKey( $path ), 300 ) ) {
1919 $this->logger->warning( "Unable to delete stat cache for file {path}.",
1920 [ 'filebackend' => $this->name, 'path' => $path ]
1921 );
1922 }
1923 }
1924
1932 final protected function primeFileCache( array $items ) {
1934 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1935
1936 $paths = []; // list of storage paths
1937 $pathNames = []; // (cache key => storage path)
1938 // Get all the paths/containers from the items...
1939 foreach ( $items as $item ) {
1940 if ( self::isStoragePath( $item ) ) {
1942 if ( $path !== null ) {
1943 $paths[] = $path;
1944 }
1945 }
1946 }
1947 // Get rid of any paths that failed normalization
1948 $paths = array_filter( $paths, 'strlen' ); // remove nulls
1949 // Get all the corresponding cache keys for paths...
1950 foreach ( $paths as $path ) {
1951 list( , $rel, ) = $this->resolveStoragePath( $path );
1952 if ( $rel !== null ) { // valid path for this backend
1953 $pathNames[$this->fileCacheKey( $path )] = $path;
1954 }
1955 }
1956 // Get all cache entries for these file cache keys.
1957 // Note that negatives are not cached by getFileStat()/preloadFileStat().
1958 $values = $this->memCache->getMulti( array_keys( $pathNames ) );
1959 // Load all of the results into process cache...
1960 foreach ( array_filter( $values, 'is_array' ) as $cacheKey => $stat ) {
1961 $path = $pathNames[$cacheKey];
1962 // This flag only applies to stat info loaded directly
1963 // from a high consistency backend query to the process cache
1964 unset( $stat['latest'] );
1965
1966 $this->cheapCache->setField( $path, 'stat', $stat );
1967 if ( isset( $stat['sha1'] ) && strlen( $stat['sha1'] ) == 31 ) {
1968 // Some backends store SHA-1 as metadata
1969 $this->cheapCache->setField(
1970 $path,
1971 'sha1',
1972 [ 'hash' => $stat['sha1'], 'latest' => false ]
1973 );
1974 }
1975 if ( isset( $stat['xattr'] ) && is_array( $stat['xattr'] ) ) {
1976 // Some backends store custom headers/metadata
1977 $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] );
1978 $this->cheapCache->setField(
1979 $path,
1980 'xattr',
1981 [ 'map' => $stat['xattr'], 'latest' => false ]
1982 );
1983 }
1984 }
1985 }
1986
1994 final protected static function normalizeXAttributes( array $xattr ) {
1995 $newXAttr = [ 'headers' => [], 'metadata' => [] ];
1996
1997 foreach ( $xattr['headers'] as $name => $value ) {
1998 $newXAttr['headers'][strtolower( $name )] = $value;
1999 }
2000
2001 foreach ( $xattr['metadata'] as $name => $value ) {
2002 $newXAttr['metadata'][strtolower( $name )] = $value;
2003 }
2004
2005 return $newXAttr;
2006 }
2007
2014 final protected function setConcurrencyFlags( array $opts ) {
2015 $opts['concurrency'] = 1; // off
2016 if ( $this->parallelize === 'implicit' ) {
2017 if ( $opts['parallelize'] ?? true ) {
2018 $opts['concurrency'] = $this->concurrency;
2019 }
2020 } elseif ( $this->parallelize === 'explicit' ) {
2021 if ( !empty( $opts['parallelize'] ) ) {
2022 $opts['concurrency'] = $this->concurrency;
2023 }
2024 }
2025
2026 return $opts;
2027 }
2028
2038 protected function getContentType( $storagePath, $content, $fsPath ) {
2039 if ( $this->mimeCallback ) {
2040 return call_user_func_array( $this->mimeCallback, func_get_args() );
2041 }
2042
2043 $mime = ( $fsPath !== null ) ? mime_content_type( $fsPath ) : false;
2044 return $mime ?: 'unknown/unknown';
2045 }
2046}
Class representing a cache/ephemeral data store.
Definition BagOStuff.php:85
A BagOStuff object with no objects in it.
Class representing a non-directory file on the file system.
Definition FSFile.php:32
static placeholderProps()
Placeholder file properties to use for files that don't exist.
Definition FSFile.php:150
File backend exception for checked exceptions (e.g.
FileBackendStore helper class for performing asynchronous file operations.
Base class for all backends using particular storage medium.
preloadFileStat(array $params)
Preload file stat information (concurrently if possible) into in-process cache.
doGetLocalReferenceMulti(array $params)
resolveContainerName( $container)
Resolve a container name, checking if it's allowed by the backend.
static normalizeXAttributes(array $xattr)
Normalize file headers/metadata to the FileBackend::getFileXAttributes() format.
getLocalReferenceMulti(array $params)
Like getLocalReference() except it takes an array of storage paths and yields an order-preserved map ...
setFileCache( $path, array $val)
Set the cached stat info for a file path.
doPublish(array $params)
moveInternal(array $params)
Move a file from one storage path to another in the backend.
setContainerCache( $container, array $val)
Set the cached info for a container.
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path in the backend.
doGetFileContentsMulti(array $params)
static string $ABSENT_NORMAL
File does not exist according to a normal stat query.
doStreamFile(array $params)
doSecure(array $params)
MapCacheLRU $expensiveCache
Map of paths to large (RAM/disk) cache items.
doClean(array $params)
getFileTimestamp(array $params)
Get the last-modified timestamp of the file at a storage path.
doGetLocalCopyMulti(array $params)
getScopedLocksForOps(array $ops, StatusValue $status)
Get an array of scoped locks needed for a batch of file operations.
doCleanInternal( $container, $dir, array $params)
callable $mimeCallback
Method to get the MIME type of files.
getPathsToLockForOpsInternal(array $performOps)
Get a list of storage paths to lock for a list of operations Returns an array with LockManager::LOCK_...
executeOpHandlesInternal(array $fileOpHandles)
Execute a list of FileBackendStoreOpHandle handles in parallel.
doPrepareInternal( $container, $dir, array $params)
concatenate(array $params)
Concatenate a list of storage files into a single file system file.
storeInternal(array $params)
Store a file into the backend from a file on disk.
getFileProps(array $params)
Get the properties of the content of the file at a storage path in the backend.
getDirectoryList(array $params)
Get an iterator to list all directories under a storage directory.
doQuickOperationsInternal(array $ops, array $opts)
resolveStoragePath( $storagePath)
Splits a storage path into an internal container name, an internal relative file name,...
copyInternal(array $params)
Copy a file from one storage path to another in the backend.
doStoreInternal(array $params)
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
doGetFileStat(array $params)
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
MapCacheLRU $cheapCache
Map of paths to small (RAM/disk) cache items.
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
doPublishInternal( $container, $dir, array $params)
doGetFileStatMulti(array $params)
Get file stat information (concurrently if possible) for several files.
getFileSha1Base36(array $params)
Get a SHA-1 hash of the content of the file at a storage path in the backend.
static isValidContainerName( $container)
Check if a full container name is valid.
ingestFreshFileStats(array $stats, $latest)
Ingest file stat entries that just came from querying the backend (not cache)
getFileListInternal( $container, $dir, array $params)
Do not call this function from places outside FileBackend.
doCopyInternal(array $params)
getLocalCopyMulti(array $params)
Like getLocalCopy() except it takes an array of storage paths and yields an order preserved-map of st...
getFileContentsMulti(array $params)
Like getFileContents() except it takes an array of storage paths and returns an order preserved map o...
doClearCache(array $paths=null)
Clears any additional stat caches for storage paths.
fullContainerName( $container)
Get the full container name, including the domain ID prefix.
primeFileCache(array $items)
Do a batch lookup from cache for file stats for all paths used in a list of storage paths or FileOp o...
doOperationsInternal(array $ops, array $opts)
isSingleShardPathInternal( $storagePath)
Check if a storage path maps to a single shard.
directoryExists(array $params)
Check if a directory exists at a given storage path.
doDeleteInternal(array $params)
static string $ABSENT_LATEST
File does not exist according to a "latest"-mode stat query.
getContainerSuffixes( $container)
Get a list of full container shard suffixes for a container.
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
sanitizeOpHeaders(array $op)
Normalize and filter HTTP headers from a file operation.
doDescribeInternal(array $params)
doGetFileSha1Base36(array $params)
deleteInternal(array $params)
Delete a file at the storage path.
doPrimeContainerCache(array $containerInfo)
Fill the backend-specific process cache given an array of resolved container names and their correspo...
doMoveInternal(array $params)
maxFileSizeInternal()
Get the maximum allowable file size given backend medium restrictions and basic performance constrain...
describeInternal(array $params)
Alter metadata for a file at the storage path.
streamFile(array $params)
Stream the content of the file at a storage path in the backend.
getOperationsInternal(array $ops)
Return a list of FileOp objects from a list of operations.
deleteFileCache( $path)
Delete the cached stat info for a file path.
setConcurrencyFlags(array $opts)
Set the 'concurrency' option from a list of operation options.
getFileXAttributes(array $params)
Get metadata about a file at a storage path in the backend.
getContentType( $storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
preloadCache(array $paths)
Preload persistent file stat cache and property cache into in-process cache.
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
doDirectoryExists( $container, $dir, array $params)
WANObjectCache $memCache
static false $RES_ABSENT
Idiom for "no result due to missing file" (since 1.34)
doPrepare(array $params)
FileBackend::prepare() StatusValue Good status without value for success, fatal otherwise.
getFileHttpUrl(array $params)
__construct(array $config)
doGetFileXAttributes(array $params)
static null $RES_ERROR
Idiom for "no result due to I/O errors" (since 1.34)
doExecuteOpHandlesInternal(array $fileOpHandles)
nullInternal(array $params)
No-op file operation that does nothing.
getFileSize(array $params)
Get the size (bytes) of a file at a storage path in the backend.
array $shardViaHashLevels
Map of container names to sharding config.
doConcatenate(array $params)
doSecureInternal( $container, $dir, array $params)
doCreateInternal(array $params)
fileExists(array $params)
Check if a file exists at a storage path in the backend.
getDirectoryListInternal( $container, $dir, array $params)
Do not call this function from places outside FileBackend.
static isValidShortContainerName( $container)
Check if a short container name is valid.
getContainerHashLevels( $container)
Get the sharding config for a container.
getContainerShard( $container, $relPath)
Get the container name shard suffix for a given path.
deleteContainerCache( $container)
Delete the cached info for a container.
createInternal(array $params)
Create a file in the backend with the given contents.
getFileList(array $params)
Get an iterator to list all stored files under a storage directory.
Base class for all file backend classes (including multi-write backends).
string $name
Unique backend name.
static splitStoragePath( $storagePath)
Split a storage path into a backend name, a container name, and a relative file path.
static normalizeContainerPath( $path)
Validate and normalize a relative storage path.
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
getTopDirectoryList(array $params)
Same as FileBackend::getDirectoryList() except only lists directories that are immediately under the ...
scopedProfileSection( $section)
newStatus(... $args)
Yields the result of the status wrapper callback on either:
static normalizeStoragePath( $storagePath)
Normalize a storage path by cleaning up directory separators.
int $concurrency
How many operations can be done in parallel.
getLocalReference(array $params)
Returns a file system file, identical in content to the file at a storage path.
static attempt(array $performOps, array $opts)
Attempt to perform a series of file operations.
Functions related to the output of file content.
static send404Message( $fname, $flags=0)
Send out a standard 404 message for a file.
Handles a simple LRU key/value map with a maximum number of entries.
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Multi-datacenter aware caching interface.
while(( $__line=Maintenance::readconsole()) !==false) print
Definition eval.php:69
$content
Definition router.php:76
$mime
Definition router.php:60