MediaWiki  master
FileBackendStore.php
Go to the documentation of this file.
1 <?php
24 use Wikimedia\AtEase\AtEase;
25 use Wikimedia\Timestamp\ConvertibleTimestamp;
26 
41 abstract class FileBackendStore extends FileBackend {
43  protected $memCache;
45  protected $srvCache;
47  protected $cheapCache;
49  protected $expensiveCache;
50 
52  protected $shardViaHashLevels = [];
53 
55  protected $mimeCallback;
56 
57  protected $maxFileSize = 4294967296; // integer bytes (4GiB)
58 
59  protected const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries
60  protected const CACHE_CHEAP_SIZE = 500; // integer; max entries in "cheap cache"
61  protected const CACHE_EXPENSIVE_SIZE = 5; // integer; max entries in "expensive cache"
62 
64  protected static $RES_ABSENT = false;
66  protected static $RES_ERROR = null;
67 
69  protected static $ABSENT_NORMAL = 'FNE-N';
71  protected static $ABSENT_LATEST = 'FNE-L';
72 
86  public function __construct( array $config ) {
87  parent::__construct( $config );
88  $this->mimeCallback = $config['mimeCallback'] ?? null;
89  $this->srvCache = new EmptyBagOStuff(); // disabled by default
90  $this->memCache = WANObjectCache::newEmpty(); // disabled by default
91  $this->cheapCache = new MapCacheLRU( self::CACHE_CHEAP_SIZE );
92  $this->expensiveCache = new MapCacheLRU( self::CACHE_EXPENSIVE_SIZE );
93  }
94 
102  final public function maxFileSizeInternal() {
103  return $this->maxFileSize;
104  }
105 
116  abstract public function isPathUsableInternal( $storagePath );
117 
136  final public function createInternal( array $params ) {
138  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
139 
140  if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
141  $status = $this->newStatus( 'backend-fail-maxsize',
142  $params['dst'], $this->maxFileSizeInternal() );
143  } else {
144  $status = $this->doCreateInternal( $params );
145  $this->clearCache( [ $params['dst'] ] );
146  if ( $params['dstExists'] ?? true ) {
147  $this->deleteFileCache( $params['dst'] ); // persistent cache
148  }
149  }
150 
151  return $status;
152  }
153 
159  abstract protected function doCreateInternal( array $params );
160 
179  final public function storeInternal( array $params ) {
181  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
182 
183  if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
184  $status = $this->newStatus( 'backend-fail-maxsize',
185  $params['dst'], $this->maxFileSizeInternal() );
186  } else {
187  $status = $this->doStoreInternal( $params );
188  $this->clearCache( [ $params['dst'] ] );
189  if ( $params['dstExists'] ?? true ) {
190  $this->deleteFileCache( $params['dst'] ); // persistent cache
191  }
192  }
193 
194  return $status;
195  }
196 
202  abstract protected function doStoreInternal( array $params );
203 
223  final public function copyInternal( array $params ) {
225  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
226 
227  $status = $this->doCopyInternal( $params );
228  $this->clearCache( [ $params['dst'] ] );
229  if ( $params['dstExists'] ?? true ) {
230  $this->deleteFileCache( $params['dst'] ); // persistent cache
231  }
232 
233  return $status;
234  }
235 
241  abstract protected function doCopyInternal( array $params );
242 
257  final public function deleteInternal( array $params ) {
259  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
260 
261  $status = $this->doDeleteInternal( $params );
262  $this->clearCache( [ $params['src'] ] );
263  $this->deleteFileCache( $params['src'] ); // persistent cache
264  return $status;
265  }
266 
272  abstract protected function doDeleteInternal( array $params );
273 
293  final public function moveInternal( array $params ) {
295  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
296 
297  $status = $this->doMoveInternal( $params );
298  $this->clearCache( [ $params['src'], $params['dst'] ] );
299  $this->deleteFileCache( $params['src'] ); // persistent cache
300  if ( $params['dstExists'] ?? true ) {
301  $this->deleteFileCache( $params['dst'] ); // persistent cache
302  }
303 
304  return $status;
305  }
306 
313  protected function doMoveInternal( array $params ) {
314  unset( $params['async'] ); // two steps, won't work here :)
315  $nsrc = FileBackend::normalizeStoragePath( $params['src'] );
316  $ndst = FileBackend::normalizeStoragePath( $params['dst'] );
317  // Copy source to dest
318  $status = $this->copyInternal( $params );
319  if ( $nsrc !== $ndst && $status->isOK() ) {
320  // Delete source (only fails due to races or network problems)
321  $status->merge( $this->deleteInternal( [ 'src' => $params['src'] ] ) );
322  $status->setResult( true, $status->value ); // ignore delete() errors
323  }
324 
325  return $status;
326  }
327 
342  final public function describeInternal( array $params ) {
344  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
345 
346  if ( count( $params['headers'] ) ) {
347  $status = $this->doDescribeInternal( $params );
348  $this->clearCache( [ $params['src'] ] );
349  $this->deleteFileCache( $params['src'] ); // persistent cache
350  } else {
351  $status = $this->newStatus(); // nothing to do
352  }
353 
354  return $status;
355  }
356 
363  protected function doDescribeInternal( array $params ) {
364  return $this->newStatus();
365  }
366 
374  final public function nullInternal( array $params ) {
375  return $this->newStatus();
376  }
377 
378  final public function concatenate( array $params ) {
380  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
381  $status = $this->newStatus();
382 
383  // Try to lock the source files for the scope of this function
385  $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
386  if ( $status->isOK() ) {
387  // Actually do the file concatenation...
388  $start_time = microtime( true );
389  $status->merge( $this->doConcatenate( $params ) );
390  $sec = microtime( true ) - $start_time;
391  if ( !$status->isOK() ) {
392  $this->logger->error( static::class . "-{$this->name}" .
393  " failed to concatenate " . count( $params['srcs'] ) . " file(s) [$sec sec]" );
394  }
395  }
396 
397  return $status;
398  }
399 
406  protected function doConcatenate( array $params ) {
407  $status = $this->newStatus();
408  $tmpPath = $params['dst'];
409  unset( $params['latest'] );
410 
411  // Check that the specified temp file is valid...
412  AtEase::suppressWarnings();
413  $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 );
414  AtEase::restoreWarnings();
415  if ( !$ok ) { // not present or not empty
416  $status->fatal( 'backend-fail-opentemp', $tmpPath );
417 
418  return $status;
419  }
420 
421  // Get local FS versions of the chunks needed for the concatenation...
422  $fsFiles = $this->getLocalReferenceMulti( $params );
423  foreach ( $fsFiles as $path => &$fsFile ) {
424  if ( !$fsFile ) { // chunk failed to download?
425  $fsFile = $this->getLocalReference( [ 'src' => $path ] );
426  if ( !$fsFile ) { // retry failed?
427  $status->fatal(
428  $fsFile === self::$RES_ERROR ? 'backend-fail-read' : 'backend-fail-notexists',
429  $path
430  );
431 
432  return $status;
433  }
434  }
435  }
436  unset( $fsFile ); // unset reference so we can reuse $fsFile
437 
438  // Get a handle for the destination temp file
439  $tmpHandle = fopen( $tmpPath, 'ab' );
440  if ( $tmpHandle === false ) {
441  $status->fatal( 'backend-fail-opentemp', $tmpPath );
442 
443  return $status;
444  }
445 
446  // Build up the temp file using the source chunks (in order)...
447  foreach ( $fsFiles as $virtualSource => $fsFile ) {
448  // Get a handle to the local FS version
449  $sourceHandle = fopen( $fsFile->getPath(), 'rb' );
450  if ( $sourceHandle === false ) {
451  fclose( $tmpHandle );
452  $status->fatal( 'backend-fail-read', $virtualSource );
453 
454  return $status;
455  }
456  // Append chunk to file (pass chunk size to avoid magic quotes)
457  if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
458  fclose( $sourceHandle );
459  fclose( $tmpHandle );
460  $status->fatal( 'backend-fail-writetemp', $tmpPath );
461 
462  return $status;
463  }
464  fclose( $sourceHandle );
465  }
466  if ( !fclose( $tmpHandle ) ) {
467  $status->fatal( 'backend-fail-closetemp', $tmpPath );
468 
469  return $status;
470  }
471 
472  clearstatcache(); // temp file changed
473 
474  return $status;
475  }
476 
480  final protected function doPrepare( array $params ) {
482  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
483  $status = $this->newStatus();
484 
485  [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
486  if ( $dir === null ) {
487  $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
488 
489  return $status; // invalid storage path
490  }
491 
492  if ( $shard !== null ) { // confined to a single container/shard
493  $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
494  } else { // directory is on several shards
495  $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
496  [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
497  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
498  $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
499  }
500  }
501 
502  return $status;
503  }
504 
513  protected function doPrepareInternal( $container, $dir, array $params ) {
514  return $this->newStatus();
515  }
516 
517  final protected function doSecure( array $params ) {
519  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
520  $status = $this->newStatus();
521 
522  [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
523  if ( $dir === null ) {
524  $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
525 
526  return $status; // invalid storage path
527  }
528 
529  if ( $shard !== null ) { // confined to a single container/shard
530  $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
531  } else { // directory is on several shards
532  $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
533  [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
534  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
535  $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
536  }
537  }
538 
539  return $status;
540  }
541 
550  protected function doSecureInternal( $container, $dir, array $params ) {
551  return $this->newStatus();
552  }
553 
554  final protected function doPublish( array $params ) {
556  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
557  $status = $this->newStatus();
558 
559  [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
560  if ( $dir === null ) {
561  $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
562 
563  return $status; // invalid storage path
564  }
565 
566  if ( $shard !== null ) { // confined to a single container/shard
567  $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
568  } else { // directory is on several shards
569  $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
570  [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
571  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
572  $status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) );
573  }
574  }
575 
576  return $status;
577  }
578 
587  protected function doPublishInternal( $container, $dir, array $params ) {
588  return $this->newStatus();
589  }
590 
591  final protected function doClean( array $params ) {
593  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
594  $status = $this->newStatus();
595 
596  // Recursive: first delete all empty subdirs recursively
597  if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) {
598  $subDirsRel = $this->getTopDirectoryList( [ 'dir' => $params['dir'] ] );
599  if ( $subDirsRel !== null ) { // no errors
600  foreach ( $subDirsRel as $subDirRel ) {
601  $subDir = $params['dir'] . "/{$subDirRel}"; // full path
602  $status->merge( $this->doClean( [ 'dir' => $subDir ] + $params ) );
603  }
604  unset( $subDirsRel ); // free directory for rmdir() on Windows (for FS backends)
605  }
606  }
607 
608  [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
609  if ( $dir === null ) {
610  $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
611 
612  return $status; // invalid storage path
613  }
614 
615  // Attempt to lock this directory...
616  $filesLockEx = [ $params['dir'] ];
618  $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
619  if ( !$status->isOK() ) {
620  return $status; // abort
621  }
622 
623  if ( $shard !== null ) { // confined to a single container/shard
624  $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
625  $this->deleteContainerCache( $fullCont ); // purge cache
626  } else { // directory is on several shards
627  $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
628  [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
629  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
630  $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
631  $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache
632  }
633  }
634 
635  return $status;
636  }
637 
646  protected function doCleanInternal( $container, $dir, array $params ) {
647  return $this->newStatus();
648  }
649 
650  final public function fileExists( array $params ) {
652  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
653 
654  $stat = $this->getFileStat( $params );
655  if ( is_array( $stat ) ) {
656  return true;
657  }
658 
659  return ( $stat === self::$RES_ABSENT ) ? false : self::EXISTENCE_ERROR;
660  }
661 
662  final public function getFileTimestamp( array $params ) {
664  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
665 
666  $stat = $this->getFileStat( $params );
667  if ( is_array( $stat ) ) {
668  return $stat['mtime'];
669  }
670 
671  return self::TIMESTAMP_FAIL; // all failure cases
672  }
673 
674  final public function getFileSize( array $params ) {
676  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
677 
678  $stat = $this->getFileStat( $params );
679  if ( is_array( $stat ) ) {
680  return $stat['size'];
681  }
682 
683  return self::SIZE_FAIL; // all failure cases
684  }
685 
686  final public function getFileStat( array $params ) {
688  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
689 
690  $path = self::normalizeStoragePath( $params['src'] );
691  if ( $path === null ) {
692  return self::STAT_ERROR; // invalid storage path
693  }
694 
695  // Whether to bypass cache except for process cache entries loaded directly from
696  // high consistency backend queries (caller handles any cache flushing and locking)
697  $latest = !empty( $params['latest'] );
698  // Whether to ignore cache entries missing the SHA-1 field for existing files
699  $requireSHA1 = !empty( $params['requireSHA1'] );
700 
701  $stat = $this->cheapCache->getField( $path, 'stat', self::CACHE_TTL );
702  // Load the persistent stat cache into process cache if needed
703  if ( !$latest ) {
704  if (
705  // File stat is not in process cache
706  $stat === null ||
707  // Key/value store backends might opportunistically set file stat process
708  // cache entries from object listings that do not include the SHA-1. In that
709  // case, loading the persistent stat cache will likely yield the SHA-1.
710  ( $requireSHA1 && is_array( $stat ) && !isset( $stat['sha1'] ) )
711  ) {
712  $this->primeFileCache( [ $path ] );
713  // Get any newly process-cached entry
714  $stat = $this->cheapCache->getField( $path, 'stat', self::CACHE_TTL );
715  }
716  }
717 
718  if ( is_array( $stat ) ) {
719  if (
720  ( !$latest || !empty( $stat['latest'] ) ) &&
721  ( !$requireSHA1 || isset( $stat['sha1'] ) )
722  ) {
723  return $stat;
724  }
725  } elseif ( $stat === self::$ABSENT_LATEST ) {
726  return self::STAT_ABSENT;
727  } elseif ( $stat === self::$ABSENT_NORMAL ) {
728  if ( !$latest ) {
729  return self::STAT_ABSENT;
730  }
731  }
732 
733  // Load the file stat from the backend and update caches
734  $stat = $this->doGetFileStat( $params );
735  $this->ingestFreshFileStats( [ $path => $stat ], $latest );
736 
737  if ( is_array( $stat ) ) {
738  return $stat;
739  }
740 
741  return ( $stat === self::$RES_ERROR ) ? self::STAT_ERROR : self::STAT_ABSENT;
742  }
743 
751  final protected function ingestFreshFileStats( array $stats, $latest ) {
752  $success = true;
753 
754  foreach ( $stats as $path => $stat ) {
755  if ( is_array( $stat ) ) {
756  // Strongly consistent backends might automatically set this flag
757  $stat['latest'] ??= $latest;
758 
759  $this->cheapCache->setField( $path, 'stat', $stat );
760  if ( isset( $stat['sha1'] ) ) {
761  // Some backends store the SHA-1 hash as metadata
762  $this->cheapCache->setField(
763  $path,
764  'sha1',
765  [ 'hash' => $stat['sha1'], 'latest' => $latest ]
766  );
767  }
768  if ( isset( $stat['xattr'] ) ) {
769  // Some backends store custom headers/metadata
770  $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] );
771  $this->cheapCache->setField(
772  $path,
773  'xattr',
774  [ 'map' => $stat['xattr'], 'latest' => $latest ]
775  );
776  }
777  // Update persistent cache (@TODO: set all entries in one batch)
778  $this->setFileCache( $path, $stat );
779  } elseif ( $stat === self::$RES_ABSENT ) {
780  $this->cheapCache->setField(
781  $path,
782  'stat',
783  $latest ? self::$ABSENT_LATEST : self::$ABSENT_NORMAL
784  );
785  $this->cheapCache->setField(
786  $path,
787  'xattr',
788  [ 'map' => self::XATTRS_FAIL, 'latest' => $latest ]
789  );
790  $this->cheapCache->setField(
791  $path,
792  'sha1',
793  [ 'hash' => self::SHA1_FAIL, 'latest' => $latest ]
794  );
795  $this->logger->debug(
796  __METHOD__ . ': File {path} does not exist',
797  [ 'path' => $path ]
798  );
799  } else {
800  $success = false;
801  $this->logger->error(
802  __METHOD__ . ': Could not stat file {path}',
803  [ 'path' => $path ]
804  );
805  }
806  }
807 
808  return $success;
809  }
810 
815  abstract protected function doGetFileStat( array $params );
816 
817  public function getFileContentsMulti( array $params ) {
819  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
820 
821  $params = $this->setConcurrencyFlags( $params );
822  $contents = $this->doGetFileContentsMulti( $params );
823  foreach ( $contents as $path => $content ) {
824  if ( !is_string( $content ) ) {
825  $contents[$path] = self::CONTENT_FAIL; // used for all failure cases
826  }
827  }
828 
829  return $contents;
830  }
831 
838  protected function doGetFileContentsMulti( array $params ) {
839  $contents = [];
840  foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
841  if ( $fsFile instanceof FSFile ) {
842  AtEase::suppressWarnings();
843  $content = file_get_contents( $fsFile->getPath() );
844  AtEase::restoreWarnings();
845  $contents[$path] = is_string( $content ) ? $content : self::$RES_ERROR;
846  } else {
847  // self::$RES_ERROR or self::$RES_ABSENT
848  $contents[$path] = $fsFile;
849  }
850  }
851 
852  return $contents;
853  }
854 
855  final public function getFileXAttributes( array $params ) {
857  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
858 
859  $path = self::normalizeStoragePath( $params['src'] );
860  if ( $path === null ) {
861  return self::XATTRS_FAIL; // invalid storage path
862  }
863  $latest = !empty( $params['latest'] ); // use latest data?
864  if ( $this->cheapCache->hasField( $path, 'xattr', self::CACHE_TTL ) ) {
865  $stat = $this->cheapCache->getField( $path, 'xattr' );
866  // If we want the latest data, check that this cached
867  // value was in fact fetched with the latest available data.
868  if ( !$latest || $stat['latest'] ) {
869  return $stat['map'];
870  }
871  }
872  $fields = $this->doGetFileXAttributes( $params );
873  if ( is_array( $fields ) ) {
874  $fields = self::normalizeXAttributes( $fields );
875  $this->cheapCache->setField(
876  $path,
877  'xattr',
878  [ 'map' => $fields, 'latest' => $latest ]
879  );
880  } elseif ( $fields === self::$RES_ABSENT ) {
881  $this->cheapCache->setField(
882  $path,
883  'xattr',
884  [ 'map' => self::XATTRS_FAIL, 'latest' => $latest ]
885  );
886  } else {
887  $fields = self::XATTRS_FAIL; // used for all failure cases
888  }
889 
890  return $fields;
891  }
892 
899  protected function doGetFileXAttributes( array $params ) {
900  return [ 'headers' => [], 'metadata' => [] ]; // not supported
901  }
902 
903  final public function getFileSha1Base36( array $params ) {
905  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
906 
907  $path = self::normalizeStoragePath( $params['src'] );
908  if ( $path === null ) {
909  return self::SHA1_FAIL; // invalid storage path
910  }
911  $latest = !empty( $params['latest'] ); // use latest data?
912  if ( $this->cheapCache->hasField( $path, 'sha1', self::CACHE_TTL ) ) {
913  $stat = $this->cheapCache->getField( $path, 'sha1' );
914  // If we want the latest data, check that this cached
915  // value was in fact fetched with the latest available data.
916  if ( !$latest || $stat['latest'] ) {
917  return $stat['hash'];
918  }
919  }
920  $sha1 = $this->doGetFileSha1Base36( $params );
921  if ( is_string( $sha1 ) ) {
922  $this->cheapCache->setField(
923  $path,
924  'sha1',
925  [ 'hash' => $sha1, 'latest' => $latest ]
926  );
927  } elseif ( $sha1 === self::$RES_ABSENT ) {
928  $this->cheapCache->setField(
929  $path,
930  'sha1',
931  [ 'hash' => self::SHA1_FAIL, 'latest' => $latest ]
932  );
933  } else {
934  $sha1 = self::SHA1_FAIL; // used for all failure cases
935  }
936 
937  return $sha1;
938  }
939 
946  protected function doGetFileSha1Base36( array $params ) {
947  $fsFile = $this->getLocalReference( $params );
948  if ( $fsFile instanceof FSFile ) {
949  $sha1 = $fsFile->getSha1Base36();
950 
951  return is_string( $sha1 ) ? $sha1 : self::$RES_ERROR;
952  }
953 
954  return ( $fsFile === self::$RES_ERROR ) ? self::$RES_ERROR : self::$RES_ABSENT;
955  }
956 
957  final public function getFileProps( array $params ) {
959  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
960 
961  $fsFile = $this->getLocalReference( $params );
962 
963  return $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
964  }
965 
966  final public function getLocalReferenceMulti( array $params ) {
968  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
969 
970  $params = $this->setConcurrencyFlags( $params );
971 
972  $fsFiles = []; // (path => FSFile)
973  $latest = !empty( $params['latest'] ); // use latest data?
974  // Reuse any files already in process cache...
975  foreach ( $params['srcs'] as $src ) {
977  if ( $path === null ) {
978  $fsFiles[$src] = self::$RES_ERROR; // invalid storage path
979  } elseif ( $this->expensiveCache->hasField( $path, 'localRef' ) ) {
980  $val = $this->expensiveCache->getField( $path, 'localRef' );
981  // If we want the latest data, check that this cached
982  // value was in fact fetched with the latest available data.
983  if ( !$latest || $val['latest'] ) {
984  $fsFiles[$src] = $val['object'];
985  }
986  }
987  }
988  // Fetch local references of any remaining files...
989  $params['srcs'] = array_diff( $params['srcs'], array_keys( $fsFiles ) );
990  foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
991  if ( $fsFile instanceof FSFile ) {
992  $fsFiles[$path] = $fsFile;
993  $this->expensiveCache->setField(
994  $path,
995  'localRef',
996  [ 'object' => $fsFile, 'latest' => $latest ]
997  );
998  } else {
999  // self::$RES_ERROR or self::$RES_ABSENT
1000  $fsFiles[$path] = $fsFile;
1001  }
1002  }
1003 
1004  return $fsFiles;
1005  }
1006 
1013  protected function doGetLocalReferenceMulti( array $params ) {
1014  return $this->doGetLocalCopyMulti( $params );
1015  }
1016 
1017  final public function getLocalCopyMulti( array $params ) {
1019  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1020 
1021  $params = $this->setConcurrencyFlags( $params );
1022 
1023  return $this->doGetLocalCopyMulti( $params );
1024  }
1025 
1031  abstract protected function doGetLocalCopyMulti( array $params );
1032 
1039  public function getFileHttpUrl( array $params ) {
1040  return self::TEMPURL_ERROR; // not supported
1041  }
1042 
1043  final public function streamFile( array $params ) {
1045  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1046  $status = $this->newStatus();
1047 
1048  // Always set some fields for subclass convenience
1049  $params['options'] ??= [];
1050  $params['headers'] ??= [];
1051 
1052  // Don't stream it out as text/html if there was a PHP error
1053  if ( ( empty( $params['headless'] ) || $params['headers'] ) && headers_sent() ) {
1054  print "Headers already sent, terminating.\n";
1055  $status->fatal( 'backend-fail-stream', $params['src'] );
1056  return $status;
1057  }
1058 
1059  $status->merge( $this->doStreamFile( $params ) );
1060 
1061  return $status;
1062  }
1063 
1070  protected function doStreamFile( array $params ) {
1071  $status = $this->newStatus();
1072 
1073  $flags = 0;
1074  $flags |= !empty( $params['headless'] ) ? HTTPFileStreamer::STREAM_HEADLESS : 0;
1075  $flags |= !empty( $params['allowOB'] ) ? HTTPFileStreamer::STREAM_ALLOW_OB : 0;
1076 
1077  $fsFile = $this->getLocalReference( $params );
1078  if ( $fsFile ) {
1079  $streamer = new HTTPFileStreamer(
1080  $fsFile->getPath(),
1081  [
1082  'obResetFunc' => $this->obResetFunc,
1083  'streamMimeFunc' => $this->streamMimeFunc
1084  ]
1085  );
1086  $res = $streamer->stream( $params['headers'], true, $params['options'], $flags );
1087  } else {
1088  $res = false;
1089  HTTPFileStreamer::send404Message( $params['src'], $flags );
1090  }
1091 
1092  if ( !$res ) {
1093  $status->fatal( 'backend-fail-stream', $params['src'] );
1094  }
1095 
1096  return $status;
1097  }
1098 
1099  final public function directoryExists( array $params ) {
1100  [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
1101  if ( $dir === null ) {
1102  return self::EXISTENCE_ERROR; // invalid storage path
1103  }
1104  if ( $shard !== null ) { // confined to a single container/shard
1105  return $this->doDirectoryExists( $fullCont, $dir, $params );
1106  } else { // directory is on several shards
1107  $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
1108  [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
1109  $res = false; // response
1110  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
1111  $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params );
1112  if ( $exists === true ) {
1113  $res = true;
1114  break; // found one!
1115  } elseif ( $exists === self::$RES_ERROR ) {
1116  $res = self::EXISTENCE_ERROR;
1117  }
1118  }
1119 
1120  return $res;
1121  }
1122  }
1123 
1132  abstract protected function doDirectoryExists( $container, $dir, array $params );
1133 
1134  final public function getDirectoryList( array $params ) {
1135  [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
1136  if ( $dir === null ) {
1137  return self::EXISTENCE_ERROR; // invalid storage path
1138  }
1139  if ( $shard !== null ) {
1140  // File listing is confined to a single container/shard
1141  return $this->getDirectoryListInternal( $fullCont, $dir, $params );
1142  } else {
1143  $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
1144  // File listing spans multiple containers/shards
1145  [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
1146 
1147  return new FileBackendStoreShardDirIterator( $this,
1148  $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
1149  }
1150  }
1151 
1162  abstract public function getDirectoryListInternal( $container, $dir, array $params );
1163 
1164  final public function getFileList( array $params ) {
1165  [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
1166  if ( $dir === null ) {
1167  return self::LIST_ERROR; // invalid storage path
1168  }
1169  if ( $shard !== null ) {
1170  // File listing is confined to a single container/shard
1171  return $this->getFileListInternal( $fullCont, $dir, $params );
1172  } else {
1173  $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
1174  // File listing spans multiple containers/shards
1175  [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
1176 
1177  return new FileBackendStoreShardFileIterator( $this,
1178  $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
1179  }
1180  }
1181 
1192  abstract public function getFileListInternal( $container, $dir, array $params );
1193 
1205  final public function getOperationsInternal( array $ops ) {
1206  $supportedOps = [
1207  'store' => StoreFileOp::class,
1208  'copy' => CopyFileOp::class,
1209  'move' => MoveFileOp::class,
1210  'delete' => DeleteFileOp::class,
1211  'create' => CreateFileOp::class,
1212  'describe' => DescribeFileOp::class,
1213  'null' => NullFileOp::class
1214  ];
1215 
1216  $performOps = []; // array of FileOp objects
1217  // Build up ordered array of FileOps...
1218  foreach ( $ops as $operation ) {
1219  $opName = $operation['op'];
1220  if ( isset( $supportedOps[$opName] ) ) {
1221  $class = $supportedOps[$opName];
1222  // Get params for this operation
1223  $params = $operation;
1224  // Append the FileOp class
1225  $performOps[] = new $class( $this, $params, $this->logger );
1226  } else {
1227  throw new FileBackendError( "Operation '$opName' is not supported." );
1228  }
1229  }
1230 
1231  return $performOps;
1232  }
1233 
1244  final public function getPathsToLockForOpsInternal( array $performOps ) {
1245  // Build up a list of files to lock...
1246  $paths = [ 'sh' => [], 'ex' => [] ];
1247  foreach ( $performOps as $fileOp ) {
1248  $paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() );
1249  $paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() );
1250  }
1251  // Optimization: if doing an EX lock anyway, don't also set an SH one
1252  $paths['sh'] = array_diff( $paths['sh'], $paths['ex'] );
1253  // Get a shared lock on the parent directory of each path changed
1254  $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) );
1255 
1256  return [
1257  LockManager::LOCK_UW => $paths['sh'],
1258  LockManager::LOCK_EX => $paths['ex']
1259  ];
1260  }
1261 
1262  public function getScopedLocksForOps( array $ops, StatusValue $status ) {
1263  $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) );
1264 
1265  return $this->getScopedFileLocks( $paths, 'mixed', $status );
1266  }
1267 
1268  final protected function doOperationsInternal( array $ops, array $opts ) {
1270  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1271  $status = $this->newStatus();
1272 
1273  // Fix up custom header name/value pairs
1274  $ops = array_map( [ $this, 'sanitizeOpHeaders' ], $ops );
1275  // Build up a list of FileOps and involved paths
1276  $fileOps = $this->getOperationsInternal( $ops );
1277  $pathsUsed = [];
1278  foreach ( $fileOps as $fileOp ) {
1279  $pathsUsed = array_merge( $pathsUsed, $fileOp->storagePathsReadOrChanged() );
1280  }
1281 
1282  // Acquire any locks as needed for the scope of this function
1283  if ( empty( $opts['nonLocking'] ) ) {
1284  $pathsByLockType = $this->getPathsToLockForOpsInternal( $fileOps );
1286  $scopeLock = $this->getScopedFileLocks( $pathsByLockType, 'mixed', $status );
1287  if ( !$status->isOK() ) {
1288  return $status; // abort
1289  }
1290  }
1291 
1292  // Clear any file cache entries (after locks acquired)
1293  if ( empty( $opts['preserveCache'] ) ) {
1294  $this->clearCache( $pathsUsed );
1295  }
1296 
1297  // Enlarge the cache to fit the stat entries of these files
1298  $this->cheapCache->setMaxSize( max( 2 * count( $pathsUsed ), self::CACHE_CHEAP_SIZE ) );
1299 
1300  // Load from the persistent container caches
1301  $this->primeContainerCache( $pathsUsed );
1302  // Get the latest stat info for all the files (having locked them)
1303  $ok = $this->preloadFileStat( [ 'srcs' => $pathsUsed, 'latest' => true ] );
1304 
1305  if ( $ok ) {
1306  // Actually attempt the operation batch...
1307  $opts = $this->setConcurrencyFlags( $opts );
1308  $subStatus = FileOpBatch::attempt( $fileOps, $opts );
1309  } else {
1310  // If we could not even stat some files, then bail out
1311  $subStatus = $this->newStatus( 'backend-fail-internal', $this->name );
1312  foreach ( $ops as $i => $op ) { // mark each op as failed
1313  $subStatus->success[$i] = false;
1314  ++$subStatus->failCount;
1315  }
1316  $this->logger->error( static::class . "-{$this->name} " .
1317  " stat failure; aborted operations: " . FormatJson::encode( $ops ) );
1318  }
1319 
1320  // Merge errors into StatusValue fields
1321  $status->merge( $subStatus );
1322  $status->success = $subStatus->success; // not done in merge()
1323 
1324  // Shrink the stat cache back to normal size
1325  $this->cheapCache->setMaxSize( self::CACHE_CHEAP_SIZE );
1326 
1327  return $status;
1328  }
1329 
1330  final protected function doQuickOperationsInternal( array $ops, array $opts ) {
1332  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1333  $status = $this->newStatus();
1334 
1335  // Fix up custom header name/value pairs
1336  $ops = array_map( [ $this, 'sanitizeOpHeaders' ], $ops );
1337  // Build up a list of FileOps and involved paths
1338  $fileOps = $this->getOperationsInternal( $ops );
1339  $pathsUsed = [];
1340  foreach ( $fileOps as $fileOp ) {
1341  $pathsUsed = array_merge( $pathsUsed, $fileOp->storagePathsReadOrChanged() );
1342  }
1343 
1344  // Clear any file cache entries for involved paths
1345  $this->clearCache( $pathsUsed );
1346 
1347  // Parallel ops may be disabled in config due to dependencies (e.g. needing popen())
1348  $async = ( $this->parallelize === 'implicit' && count( $ops ) > 1 );
1349  $maxConcurrency = $this->concurrency; // throttle
1351  $statuses = []; // array of (index => StatusValue)
1352  $fileOpHandles = []; // list of (index => handle) arrays
1353  $curFileOpHandles = []; // current handle batch
1354  // Perform the sync-only ops and build up op handles for the async ops...
1355  foreach ( $fileOps as $index => $fileOp ) {
1356  $subStatus = $async
1357  ? $fileOp->attemptAsyncQuick()
1358  : $fileOp->attemptQuick();
1359  if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { // async
1360  if ( count( $curFileOpHandles ) >= $maxConcurrency ) {
1361  $fileOpHandles[] = $curFileOpHandles; // push this batch
1362  $curFileOpHandles = [];
1363  }
1364  $curFileOpHandles[$index] = $subStatus->value; // keep index
1365  } else { // error or completed
1366  $statuses[$index] = $subStatus; // keep index
1367  }
1368  }
1369  if ( count( $curFileOpHandles ) ) {
1370  $fileOpHandles[] = $curFileOpHandles; // last batch
1371  }
1372  // Do all the async ops that can be done concurrently...
1373  foreach ( $fileOpHandles as $fileHandleBatch ) {
1374  $statuses += $this->executeOpHandlesInternal( $fileHandleBatch );
1375  }
1376  // Marshall and merge all the responses...
1377  foreach ( $statuses as $index => $subStatus ) {
1378  $status->merge( $subStatus );
1379  if ( $subStatus->isOK() ) {
1380  $status->success[$index] = true;
1381  ++$status->successCount;
1382  } else {
1383  $status->success[$index] = false;
1384  ++$status->failCount;
1385  }
1386  }
1387 
1388  $this->clearCache( $pathsUsed );
1389 
1390  return $status;
1391  }
1392 
1402  final public function executeOpHandlesInternal( array $fileOpHandles ) {
1404  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1405 
1406  foreach ( $fileOpHandles as $fileOpHandle ) {
1407  if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) {
1408  throw new InvalidArgumentException( "Expected FileBackendStoreOpHandle object." );
1409  } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) {
1410  throw new InvalidArgumentException( "Expected handle for this file backend." );
1411  }
1412  }
1413 
1414  $statuses = $this->doExecuteOpHandlesInternal( $fileOpHandles );
1415  foreach ( $fileOpHandles as $fileOpHandle ) {
1416  $fileOpHandle->closeResources();
1417  }
1418 
1419  return $statuses;
1420  }
1421 
1431  protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1432  if ( count( $fileOpHandles ) ) {
1433  throw new FileBackendError( "Backend does not support asynchronous operations." );
1434  }
1435 
1436  return [];
1437  }
1438 
1450  protected function sanitizeOpHeaders( array $op ) {
1451  static $longs = [ 'content-disposition' ];
1452 
1453  if ( isset( $op['headers'] ) ) { // op sets HTTP headers
1454  $newHeaders = [];
1455  foreach ( $op['headers'] as $name => $value ) {
1456  $name = strtolower( $name );
1457  $maxHVLen = in_array( $name, $longs ) ? INF : 255;
1458  if ( strlen( $name ) > 255 || strlen( $value ) > $maxHVLen ) {
1459  $this->logger->error( "Header '{header}' is too long.", [
1460  'filebackend' => $this->name,
1461  'header' => "$name: $value",
1462  ] );
1463  } else {
1464  $newHeaders[$name] = strlen( $value ) ? $value : ''; // null/false => ""
1465  }
1466  }
1467  $op['headers'] = $newHeaders;
1468  }
1469 
1470  return $op;
1471  }
1472 
1473  final public function preloadCache( array $paths ) {
1474  $fullConts = []; // full container names
1475  foreach ( $paths as $path ) {
1476  [ $fullCont, , ] = $this->resolveStoragePath( $path );
1477  $fullConts[] = $fullCont;
1478  }
1479  // Load from the persistent file and container caches
1480  $this->primeContainerCache( $fullConts );
1481  $this->primeFileCache( $paths );
1482  }
1483 
1484  final public function clearCache( array $paths = null ) {
1485  if ( is_array( $paths ) ) {
1486  $paths = array_map( [ FileBackend::class, 'normalizeStoragePath' ], $paths );
1487  $paths = array_filter( $paths, 'strlen' ); // remove nulls
1488  }
1489  if ( $paths === null ) {
1490  $this->cheapCache->clear();
1491  $this->expensiveCache->clear();
1492  } else {
1493  foreach ( $paths as $path ) {
1494  $this->cheapCache->clear( $path );
1495  $this->expensiveCache->clear( $path );
1496  }
1497  }
1498  $this->doClearCache( $paths );
1499  }
1500 
1509  protected function doClearCache( array $paths = null ) {
1510  }
1511 
1512  final public function preloadFileStat( array $params ) {
1514  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1515 
1516  $params['concurrency'] = ( $this->parallelize !== 'off' ) ? $this->concurrency : 1;
1517  $stats = $this->doGetFileStatMulti( $params );
1518  if ( $stats === null ) {
1519  return true; // not supported
1520  }
1521 
1522  // Whether this queried the backend in high consistency mode
1523  $latest = !empty( $params['latest'] );
1524 
1525  return $this->ingestFreshFileStats( $stats, $latest );
1526  }
1527 
1540  protected function doGetFileStatMulti( array $params ) {
1541  return null; // not supported
1542  }
1543 
1551  abstract protected function directoriesAreVirtual();
1552 
1563  final protected static function isValidShortContainerName( $container ) {
1564  // Suffixes like '.xxx' (hex shard chars) or '.seg' (file segments)
1565  // might be used by subclasses. Reserve the dot character.
1566  // The only way dots end up in containers (e.g. resolveStoragePath)
1567  // is due to the wikiId container prefix or the above suffixes.
1568  return self::isValidContainerName( $container ) && !preg_match( '/[.]/', $container );
1569  }
1570 
1580  final protected static function isValidContainerName( $container ) {
1581  // This accounts for NTFS, Swift, and Ceph restrictions
1582  // and disallows directory separators or traversal characters.
1583  // Note that matching strings URL encode to the same string;
1584  // in Swift/Ceph, the length restriction is *after* URL encoding.
1585  return (bool)preg_match( '/^[a-z0-9][a-z0-9-_.]{0,199}$/i', $container );
1586  }
1587 
1601  final protected function resolveStoragePath( $storagePath ) {
1602  [ $backend, $shortCont, $relPath ] = self::splitStoragePath( $storagePath );
1603  if ( $backend === $this->name ) { // must be for this backend
1604  $relPath = self::normalizeContainerPath( $relPath );
1605  if ( $relPath !== null && self::isValidShortContainerName( $shortCont ) ) {
1606  // Get shard for the normalized path if this container is sharded
1607  $cShard = $this->getContainerShard( $shortCont, $relPath );
1608  // Validate and sanitize the relative path (backend-specific)
1609  $relPath = $this->resolveContainerPath( $shortCont, $relPath );
1610  if ( $relPath !== null ) {
1611  // Prepend any domain ID prefix to the container name
1612  $container = $this->fullContainerName( $shortCont );
1613  if ( self::isValidContainerName( $container ) ) {
1614  // Validate and sanitize the container name (backend-specific)
1615  $container = $this->resolveContainerName( "{$container}{$cShard}" );
1616  if ( $container !== null ) {
1617  return [ $container, $relPath, $cShard ];
1618  }
1619  }
1620  }
1621  }
1622  }
1623 
1624  return [ null, null, null ];
1625  }
1626 
1642  final protected function resolveStoragePathReal( $storagePath ) {
1643  [ $container, $relPath, $cShard ] = $this->resolveStoragePath( $storagePath );
1644  if ( $cShard !== null && substr( $relPath, -1 ) !== '/' ) {
1645  return [ $container, $relPath ];
1646  }
1647 
1648  return [ null, null ];
1649  }
1650 
1659  final protected function getContainerShard( $container, $relPath ) {
1660  [ $levels, $base, $repeat ] = $this->getContainerHashLevels( $container );
1661  if ( $levels == 1 || $levels == 2 ) {
1662  // Hash characters are either base 16 or 36
1663  $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]';
1664  // Get a regex that represents the shard portion of paths.
1665  // The concatenation of the captures gives us the shard.
1666  if ( $levels === 1 ) { // 16 or 36 shards per container
1667  $hashDirRegex = '(' . $char . ')';
1668  } else { // 256 or 1296 shards per container
1669  if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc")
1670  $hashDirRegex = $char . '/(' . $char . '{2})';
1671  } else { // short hash dir format (e.g. "a/b/c")
1672  $hashDirRegex = '(' . $char . ')/(' . $char . ')';
1673  }
1674  }
1675  // Allow certain directories to be above the hash dirs so as
1676  // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
1677  // They must be 2+ chars to avoid any hash directory ambiguity.
1678  $m = [];
1679  if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1680  return '.' . implode( '', array_slice( $m, 1 ) );
1681  }
1682 
1683  return null; // failed to match
1684  }
1685 
1686  return ''; // no sharding
1687  }
1688 
1697  final public function isSingleShardPathInternal( $storagePath ) {
1698  [ , , $shard ] = $this->resolveStoragePath( $storagePath );
1699 
1700  return ( $shard !== null );
1701  }
1702 
1711  final protected function getContainerHashLevels( $container ) {
1712  if ( isset( $this->shardViaHashLevels[$container] ) ) {
1713  $config = $this->shardViaHashLevels[$container];
1714  $hashLevels = (int)$config['levels'];
1715  if ( $hashLevels == 1 || $hashLevels == 2 ) {
1716  $hashBase = (int)$config['base'];
1717  if ( $hashBase == 16 || $hashBase == 36 ) {
1718  return [ $hashLevels, $hashBase, $config['repeat'] ];
1719  }
1720  }
1721  }
1722 
1723  return [ 0, 0, false ]; // no sharding
1724  }
1725 
1732  final protected function getContainerSuffixes( $container ) {
1733  $shards = [];
1734  [ $digits, $base ] = $this->getContainerHashLevels( $container );
1735  if ( $digits > 0 ) {
1736  $numShards = $base ** $digits;
1737  for ( $index = 0; $index < $numShards; $index++ ) {
1738  $shards[] = '.' . Wikimedia\base_convert( (string)$index, 10, $base, $digits );
1739  }
1740  }
1741 
1742  return $shards;
1743  }
1744 
1751  final protected function fullContainerName( $container ) {
1752  if ( $this->domainId != '' ) {
1753  return "{$this->domainId}-$container";
1754  } else {
1755  return $container;
1756  }
1757  }
1758 
1768  protected function resolveContainerName( $container ) {
1769  return $container;
1770  }
1771 
1783  protected function resolveContainerPath( $container, $relStoragePath ) {
1784  return $relStoragePath;
1785  }
1786 
1793  private function containerCacheKey( $container ) {
1794  return "filebackend:{$this->name}:{$this->domainId}:container:{$container}";
1795  }
1796 
1803  final protected function setContainerCache( $container, array $val ) {
1804  if ( !$this->memCache->set( $this->containerCacheKey( $container ), $val, 14 * 86400 ) ) {
1805  $this->logger->warning( "Unable to set stat cache for container {container}.",
1806  [ 'filebackend' => $this->name, 'container' => $container ]
1807  );
1808  }
1809  }
1810 
1817  final protected function deleteContainerCache( $container ) {
1818  if ( !$this->memCache->delete( $this->containerCacheKey( $container ), 300 ) ) {
1819  $this->logger->warning( "Unable to delete stat cache for container {container}.",
1820  [ 'filebackend' => $this->name, 'container' => $container ]
1821  );
1822  }
1823  }
1824 
1832  final protected function primeContainerCache( array $items ) {
1834  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1835 
1836  $paths = []; // list of storage paths
1837  $contNames = []; // (cache key => resolved container name)
1838  // Get all the paths/containers from the items...
1839  foreach ( $items as $item ) {
1840  if ( self::isStoragePath( $item ) ) {
1841  $paths[] = $item;
1842  } elseif ( is_string( $item ) ) { // full container name
1843  $contNames[$this->containerCacheKey( $item )] = $item;
1844  }
1845  }
1846  // Get all the corresponding cache keys for paths...
1847  foreach ( $paths as $path ) {
1848  [ $fullCont, , ] = $this->resolveStoragePath( $path );
1849  if ( $fullCont !== null ) { // valid path for this backend
1850  $contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
1851  }
1852  }
1853 
1854  $contInfo = []; // (resolved container name => cache value)
1855  // Get all cache entries for these container cache keys...
1856  $values = $this->memCache->getMulti( array_keys( $contNames ) );
1857  foreach ( $values as $cacheKey => $val ) {
1858  $contInfo[$contNames[$cacheKey]] = $val;
1859  }
1860 
1861  // Populate the container process cache for the backend...
1862  $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) );
1863  }
1864 
1873  protected function doPrimeContainerCache( array $containerInfo ) {
1874  }
1875 
1882  private function fileCacheKey( $path ) {
1883  return "filebackend:{$this->name}:{$this->domainId}:file:" . sha1( $path );
1884  }
1885 
1894  final protected function setFileCache( $path, array $val ) {
1896  if ( $path === null ) {
1897  return; // invalid storage path
1898  }
1899  $mtime = (int)ConvertibleTimestamp::convert( TS_UNIX, $val['mtime'] );
1900  $ttl = $this->memCache->adaptiveTTL( $mtime, 7 * 86400, 300, 0.1 );
1901  $key = $this->fileCacheKey( $path );
1902  // Set the cache unless it is currently salted.
1903  if ( !$this->memCache->set( $key, $val, $ttl ) ) {
1904  $this->logger->warning( "Unable to set stat cache for file {path}.",
1905  [ 'filebackend' => $this->name, 'path' => $path ]
1906  );
1907  }
1908  }
1909 
1918  final protected function deleteFileCache( $path ) {
1920  if ( $path === null ) {
1921  return; // invalid storage path
1922  }
1923  if ( !$this->memCache->delete( $this->fileCacheKey( $path ), 300 ) ) {
1924  $this->logger->warning( "Unable to delete stat cache for file {path}.",
1925  [ 'filebackend' => $this->name, 'path' => $path ]
1926  );
1927  }
1928  }
1929 
1937  final protected function primeFileCache( array $items ) {
1939  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1940 
1941  $paths = []; // list of storage paths
1942  $pathNames = []; // (cache key => storage path)
1943  // Get all the paths/containers from the items...
1944  foreach ( $items as $item ) {
1945  if ( self::isStoragePath( $item ) ) {
1947  if ( $path !== null ) {
1948  $paths[] = $path;
1949  }
1950  }
1951  }
1952  // Get all the corresponding cache keys for paths...
1953  foreach ( $paths as $path ) {
1954  [ , $rel, ] = $this->resolveStoragePath( $path );
1955  if ( $rel !== null ) { // valid path for this backend
1956  $pathNames[$this->fileCacheKey( $path )] = $path;
1957  }
1958  }
1959  // Get all cache entries for these file cache keys.
1960  // Note that negatives are not cached by getFileStat()/preloadFileStat().
1961  $values = $this->memCache->getMulti( array_keys( $pathNames ) );
1962  // Load all of the results into process cache...
1963  foreach ( array_filter( $values, 'is_array' ) as $cacheKey => $stat ) {
1964  $path = $pathNames[$cacheKey];
1965  // This flag only applies to stat info loaded directly
1966  // from a high consistency backend query to the process cache
1967  unset( $stat['latest'] );
1968 
1969  $this->cheapCache->setField( $path, 'stat', $stat );
1970  if ( isset( $stat['sha1'] ) && strlen( $stat['sha1'] ) == 31 ) {
1971  // Some backends store SHA-1 as metadata
1972  $this->cheapCache->setField(
1973  $path,
1974  'sha1',
1975  [ 'hash' => $stat['sha1'], 'latest' => false ]
1976  );
1977  }
1978  if ( isset( $stat['xattr'] ) && is_array( $stat['xattr'] ) ) {
1979  // Some backends store custom headers/metadata
1980  $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] );
1981  $this->cheapCache->setField(
1982  $path,
1983  'xattr',
1984  [ 'map' => $stat['xattr'], 'latest' => false ]
1985  );
1986  }
1987  }
1988  }
1989 
1997  final protected static function normalizeXAttributes( array $xattr ) {
1998  $newXAttr = [ 'headers' => [], 'metadata' => [] ];
1999 
2000  foreach ( $xattr['headers'] as $name => $value ) {
2001  $newXAttr['headers'][strtolower( $name )] = $value;
2002  }
2003 
2004  foreach ( $xattr['metadata'] as $name => $value ) {
2005  $newXAttr['metadata'][strtolower( $name )] = $value;
2006  }
2007 
2008  return $newXAttr;
2009  }
2010 
2017  final protected function setConcurrencyFlags( array $opts ) {
2018  $opts['concurrency'] = 1; // off
2019  if ( $this->parallelize === 'implicit' ) {
2020  if ( $opts['parallelize'] ?? true ) {
2021  $opts['concurrency'] = $this->concurrency;
2022  }
2023  } elseif ( $this->parallelize === 'explicit' ) {
2024  if ( !empty( $opts['parallelize'] ) ) {
2025  $opts['concurrency'] = $this->concurrency;
2026  }
2027  }
2028 
2029  return $opts;
2030  }
2031 
2041  protected function getContentType( $storagePath, $content, $fsPath ) {
2042  if ( $this->mimeCallback ) {
2043  return call_user_func_array( $this->mimeCallback, func_get_args() );
2044  }
2045 
2046  $mime = ( $fsPath !== null ) ? mime_content_type( $fsPath ) : false;
2047  return $mime ?: 'unknown/unknown';
2048  }
2049 }
$success
A BagOStuff object with no objects in it.
Class representing a non-directory file on the file system.
Definition: FSFile.php:32
static placeholderProps()
Placeholder file properties to use for files that don't exist.
Definition: FSFile.php:150
File backend exception for checked exceptions (e.g.
FileBackendStore helper class for performing asynchronous file operations.
Base class for all backends using particular storage medium.
preloadFileStat(array $params)
Preload file stat information (concurrently if possible) into in-process cache.
doGetLocalReferenceMulti(array $params)
resolveContainerName( $container)
Resolve a container name, checking if it's allowed by the backend.
static normalizeXAttributes(array $xattr)
Normalize file headers/metadata to the FileBackend::getFileXAttributes() format.
getLocalReferenceMulti(array $params)
Like getLocalReference() except it takes an array of storage paths and yields an order-preserved map ...
setFileCache( $path, array $val)
Set the cached stat info for a file path.
doPublish(array $params)
moveInternal(array $params)
Move a file from one storage path to another in the backend.
setContainerCache( $container, array $val)
Set the cached info for a container.
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path in the backend.
doGetFileContentsMulti(array $params)
static string $ABSENT_NORMAL
File does not exist according to a normal stat query.
doStreamFile(array $params)
doSecure(array $params)
MapCacheLRU $expensiveCache
Map of paths to large (RAM/disk) cache items.
doClean(array $params)
getFileTimestamp(array $params)
Get the last-modified timestamp of the file at a storage path.
doGetLocalCopyMulti(array $params)
getScopedLocksForOps(array $ops, StatusValue $status)
Get an array of scoped locks needed for a batch of file operations.
doCleanInternal( $container, $dir, array $params)
callable $mimeCallback
Method to get the MIME type of files.
getPathsToLockForOpsInternal(array $performOps)
Get a list of storage paths to lock for a list of operations Returns an array with LockManager::LOCK_...
executeOpHandlesInternal(array $fileOpHandles)
Execute a list of FileBackendStoreOpHandle handles in parallel.
doPrepareInternal( $container, $dir, array $params)
concatenate(array $params)
Concatenate a list of storage files into a single file system file.
storeInternal(array $params)
Store a file into the backend from a file on disk.
getFileProps(array $params)
Get the properties of the content of the file at a storage path in the backend.
getDirectoryList(array $params)
Get an iterator to list all directories under a storage directory.
doQuickOperationsInternal(array $ops, array $opts)
resolveStoragePath( $storagePath)
Splits a storage path into an internal container name, an internal relative file name,...
copyInternal(array $params)
Copy a file from one storage path to another in the backend.
doStoreInternal(array $params)
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
doGetFileStat(array $params)
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
MapCacheLRU $cheapCache
Map of paths to small (RAM/disk) cache items.
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
doPublishInternal( $container, $dir, array $params)
doGetFileStatMulti(array $params)
Get file stat information (concurrently if possible) for several files.
getFileSha1Base36(array $params)
Get a SHA-1 hash of the content of the file at a storage path in the backend.
static isValidContainerName( $container)
Check if a full container name is valid.
ingestFreshFileStats(array $stats, $latest)
Ingest file stat entries that just came from querying the backend (not cache)
getFileListInternal( $container, $dir, array $params)
Do not call this function from places outside FileBackend.
doCopyInternal(array $params)
getLocalCopyMulti(array $params)
Like getLocalCopy() except it takes an array of storage paths and yields an order preserved-map of st...
getFileContentsMulti(array $params)
Like getFileContents() except it takes an array of storage paths and returns an order preserved map o...
doClearCache(array $paths=null)
Clears any additional stat caches for storage paths.
fullContainerName( $container)
Get the full container name, including the domain ID prefix.
primeFileCache(array $items)
Do a batch lookup from cache for file stats for all paths used in a list of storage paths or FileOp o...
doOperationsInternal(array $ops, array $opts)
isSingleShardPathInternal( $storagePath)
Check if a storage path maps to a single shard.
directoryExists(array $params)
Check if a directory exists at a given storage path.
doDeleteInternal(array $params)
static string $ABSENT_LATEST
File does not exist according to a "latest"-mode stat query.
getContainerSuffixes( $container)
Get a list of full container shard suffixes for a container.
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
sanitizeOpHeaders(array $op)
Normalize and filter HTTP headers from a file operation.
doDescribeInternal(array $params)
doGetFileSha1Base36(array $params)
deleteInternal(array $params)
Delete a file at the storage path.
doPrimeContainerCache(array $containerInfo)
Fill the backend-specific process cache given an array of resolved container names and their correspo...
doMoveInternal(array $params)
maxFileSizeInternal()
Get the maximum allowable file size given backend medium restrictions and basic performance constrain...
describeInternal(array $params)
Alter metadata for a file at the storage path.
streamFile(array $params)
Stream the content of the file at a storage path in the backend.
getOperationsInternal(array $ops)
Return a list of FileOp objects from a list of operations.
deleteFileCache( $path)
Delete the cached stat info for a file path.
setConcurrencyFlags(array $opts)
Set the 'concurrency' option from a list of operation options.
getFileXAttributes(array $params)
Get metadata about a file at a storage path in the backend.
getContentType( $storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
preloadCache(array $paths)
Preload persistent file stat cache and property cache into in-process cache.
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
doDirectoryExists( $container, $dir, array $params)
WANObjectCache $memCache
static false $RES_ABSENT
Idiom for "no result due to missing file" (since 1.34)
doPrepare(array $params)
FileBackend::prepare() StatusValue Good status without value for success, fatal otherwise.
getFileHttpUrl(array $params)
__construct(array $config)
doGetFileXAttributes(array $params)
static null $RES_ERROR
Idiom for "no result due to I/O errors" (since 1.34)
doExecuteOpHandlesInternal(array $fileOpHandles)
nullInternal(array $params)
No-op file operation that does nothing.
getFileSize(array $params)
Get the size (bytes) of a file at a storage path in the backend.
array $shardViaHashLevels
Map of container names to sharding config.
doConcatenate(array $params)
doSecureInternal( $container, $dir, array $params)
doCreateInternal(array $params)
fileExists(array $params)
Check if a file exists at a storage path in the backend.
getDirectoryListInternal( $container, $dir, array $params)
Do not call this function from places outside FileBackend.
static isValidShortContainerName( $container)
Check if a short container name is valid.
getContainerHashLevels( $container)
Get the sharding config for a container.
getContainerShard( $container, $relPath)
Get the container name shard suffix for a given path.
deleteContainerCache( $container)
Delete the cached info for a container.
createInternal(array $params)
Create a file in the backend with the given contents.
getFileList(array $params)
Get an iterator to list all stored files under a storage directory.
Base class for all file backend classes (including multi-write backends).
Definition: FileBackend.php:99
string $name
Unique backend name.
static splitStoragePath( $storagePath)
Split a storage path into a backend name, a container name, and a relative file path.
static normalizeContainerPath( $path)
Validate and normalize a relative storage path.
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
getTopDirectoryList(array $params)
Same as FileBackend::getDirectoryList() except only lists directories that are immediately under the ...
scopedProfileSection( $section)
newStatus(... $args)
Yields the result of the status wrapper callback on either:
static normalizeStoragePath( $storagePath)
Normalize a storage path by cleaning up directory separators.
int $concurrency
How many operations can be done in parallel.
getLocalReference(array $params)
Returns a file system file, identical in content to the file at a storage path.
static attempt(array $performOps, array $opts)
Attempt to perform a series of file operations.
Definition: FileOpBatch.php:54
static encode( $value, $pretty=false, $escaping=0)
Returns the JSON representation of a value.
Definition: FormatJson.php:98
Functions related to the output of file content.
static send404Message( $fname, $flags=0)
Send out a standard 404 message for a file.
const LOCK_EX
Definition: LockManager.php:70
const LOCK_UW
Definition: LockManager.php:69
Store key-value entries in a size-limited in-memory LRU cache.
Definition: MapCacheLRU.php:34
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Definition: StatusValue.php:46
static newEmpty()
Get an instance that wraps EmptyBagOStuff.
$content
Definition: router.php:76
$mime
Definition: router.php:60