MediaWiki  master
FileBackendStore.php
Go to the documentation of this file.
1 <?php
24 use Wikimedia\AtEase\AtEase;
25 use Wikimedia\Timestamp\ConvertibleTimestamp;
26 
41 abstract class FileBackendStore extends FileBackend {
43  protected $memCache;
45  protected $srvCache;
47  protected $cheapCache;
49  protected $expensiveCache;
50 
52  protected $shardViaHashLevels = [];
53 
55  protected $mimeCallback;
56 
57  protected $maxFileSize = 4294967296; // integer bytes (4GiB)
58 
59  protected const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries
60  protected const CACHE_CHEAP_SIZE = 500; // integer; max entries in "cheap cache"
61  protected const CACHE_EXPENSIVE_SIZE = 5; // integer; max entries in "expensive cache"
62 
64  protected static $RES_ABSENT = false;
66  protected static $RES_ERROR = null;
67 
69  protected static $ABSENT_NORMAL = 'FNE-N';
71  protected static $ABSENT_LATEST = 'FNE-L';
72 
86  public function __construct( array $config ) {
87  parent::__construct( $config );
88  $this->mimeCallback = $config['mimeCallback'] ?? null;
89  $this->srvCache = new EmptyBagOStuff(); // disabled by default
90  $this->memCache = WANObjectCache::newEmpty(); // disabled by default
91  $this->cheapCache = new MapCacheLRU( self::CACHE_CHEAP_SIZE );
92  $this->expensiveCache = new MapCacheLRU( self::CACHE_EXPENSIVE_SIZE );
93  }
94 
102  final public function maxFileSizeInternal() {
103  return $this->maxFileSize;
104  }
105 
116  abstract public function isPathUsableInternal( $storagePath );
117 
136  final public function createInternal( array $params ) {
138  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
139 
140  if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
141  $status = $this->newStatus( 'backend-fail-maxsize',
142  $params['dst'], $this->maxFileSizeInternal() );
143  } else {
144  $status = $this->doCreateInternal( $params );
145  $this->clearCache( [ $params['dst'] ] );
146  if ( $params['dstExists'] ?? true ) {
147  $this->deleteFileCache( $params['dst'] ); // persistent cache
148  }
149  }
150 
151  return $status;
152  }
153 
159  abstract protected function doCreateInternal( array $params );
160 
179  final public function storeInternal( array $params ) {
181  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
182 
183  if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
184  $status = $this->newStatus( 'backend-fail-maxsize',
185  $params['dst'], $this->maxFileSizeInternal() );
186  } else {
187  $status = $this->doStoreInternal( $params );
188  $this->clearCache( [ $params['dst'] ] );
189  if ( $params['dstExists'] ?? true ) {
190  $this->deleteFileCache( $params['dst'] ); // persistent cache
191  }
192  }
193 
194  return $status;
195  }
196 
202  abstract protected function doStoreInternal( array $params );
203 
223  final public function copyInternal( array $params ) {
225  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
226 
227  $status = $this->doCopyInternal( $params );
228  $this->clearCache( [ $params['dst'] ] );
229  if ( $params['dstExists'] ?? true ) {
230  $this->deleteFileCache( $params['dst'] ); // persistent cache
231  }
232 
233  return $status;
234  }
235 
241  abstract protected function doCopyInternal( array $params );
242 
257  final public function deleteInternal( array $params ) {
259  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
260 
261  $status = $this->doDeleteInternal( $params );
262  $this->clearCache( [ $params['src'] ] );
263  $this->deleteFileCache( $params['src'] ); // persistent cache
264  return $status;
265  }
266 
272  abstract protected function doDeleteInternal( array $params );
273 
293  final public function moveInternal( array $params ) {
295  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
296 
297  $status = $this->doMoveInternal( $params );
298  $this->clearCache( [ $params['src'], $params['dst'] ] );
299  $this->deleteFileCache( $params['src'] ); // persistent cache
300  if ( $params['dstExists'] ?? true ) {
301  $this->deleteFileCache( $params['dst'] ); // persistent cache
302  }
303 
304  return $status;
305  }
306 
313  protected function doMoveInternal( array $params ) {
314  unset( $params['async'] ); // two steps, won't work here :)
315  $nsrc = FileBackend::normalizeStoragePath( $params['src'] );
316  $ndst = FileBackend::normalizeStoragePath( $params['dst'] );
317  // Copy source to dest
318  $status = $this->copyInternal( $params );
319  if ( $nsrc !== $ndst && $status->isOK() ) {
320  // Delete source (only fails due to races or network problems)
321  $status->merge( $this->deleteInternal( [ 'src' => $params['src'] ] ) );
322  $status->setResult( true, $status->value ); // ignore delete() errors
323  }
324 
325  return $status;
326  }
327 
342  final public function describeInternal( array $params ) {
344  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
345 
346  if ( count( $params['headers'] ) ) {
347  $status = $this->doDescribeInternal( $params );
348  $this->clearCache( [ $params['src'] ] );
349  $this->deleteFileCache( $params['src'] ); // persistent cache
350  } else {
351  $status = $this->newStatus(); // nothing to do
352  }
353 
354  return $status;
355  }
356 
363  protected function doDescribeInternal( array $params ) {
364  return $this->newStatus();
365  }
366 
374  final public function nullInternal( array $params ) {
375  return $this->newStatus();
376  }
377 
378  final public function concatenate( array $params ) {
380  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
381  $status = $this->newStatus();
382 
383  // Try to lock the source files for the scope of this function
385  $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
386  if ( $status->isOK() ) {
387  // Actually do the file concatenation...
388  $start_time = microtime( true );
389  $status->merge( $this->doConcatenate( $params ) );
390  $sec = microtime( true ) - $start_time;
391  if ( !$status->isOK() ) {
392  $this->logger->error( static::class . "-{$this->name}" .
393  " failed to concatenate " . count( $params['srcs'] ) . " file(s) [$sec sec]" );
394  }
395  }
396 
397  return $status;
398  }
399 
406  protected function doConcatenate( array $params ) {
407  $status = $this->newStatus();
408  $tmpPath = $params['dst'];
409  unset( $params['latest'] );
410 
411  // Check that the specified temp file is valid...
412  AtEase::suppressWarnings();
413  $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 );
414  AtEase::restoreWarnings();
415  if ( !$ok ) { // not present or not empty
416  $status->fatal( 'backend-fail-opentemp', $tmpPath );
417 
418  return $status;
419  }
420 
421  // Get local FS versions of the chunks needed for the concatenation...
422  $fsFiles = $this->getLocalReferenceMulti( $params );
423  foreach ( $fsFiles as $path => &$fsFile ) {
424  if ( !$fsFile ) { // chunk failed to download?
425  $fsFile = $this->getLocalReference( [ 'src' => $path ] );
426  if ( !$fsFile ) { // retry failed?
427  $status->fatal( 'backend-fail-read', $path );
428 
429  return $status;
430  }
431  }
432  }
433  unset( $fsFile ); // unset reference so we can reuse $fsFile
434 
435  // Get a handle for the destination temp file
436  $tmpHandle = fopen( $tmpPath, 'ab' );
437  if ( $tmpHandle === false ) {
438  $status->fatal( 'backend-fail-opentemp', $tmpPath );
439 
440  return $status;
441  }
442 
443  // Build up the temp file using the source chunks (in order)...
444  foreach ( $fsFiles as $virtualSource => $fsFile ) {
445  // Get a handle to the local FS version
446  $sourceHandle = fopen( $fsFile->getPath(), 'rb' );
447  if ( $sourceHandle === false ) {
448  fclose( $tmpHandle );
449  $status->fatal( 'backend-fail-read', $virtualSource );
450 
451  return $status;
452  }
453  // Append chunk to file (pass chunk size to avoid magic quotes)
454  if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
455  fclose( $sourceHandle );
456  fclose( $tmpHandle );
457  $status->fatal( 'backend-fail-writetemp', $tmpPath );
458 
459  return $status;
460  }
461  fclose( $sourceHandle );
462  }
463  if ( !fclose( $tmpHandle ) ) {
464  $status->fatal( 'backend-fail-closetemp', $tmpPath );
465 
466  return $status;
467  }
468 
469  clearstatcache(); // temp file changed
470 
471  return $status;
472  }
473 
477  final protected function doPrepare( array $params ) {
479  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
480  $status = $this->newStatus();
481 
482  [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
483  if ( $dir === null ) {
484  $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
485 
486  return $status; // invalid storage path
487  }
488 
489  if ( $shard !== null ) { // confined to a single container/shard
490  $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
491  } else { // directory is on several shards
492  $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
493  [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
494  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
495  $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
496  }
497  }
498 
499  return $status;
500  }
501 
510  protected function doPrepareInternal( $container, $dir, array $params ) {
511  return $this->newStatus();
512  }
513 
514  final protected function doSecure( array $params ) {
516  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
517  $status = $this->newStatus();
518 
519  [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
520  if ( $dir === null ) {
521  $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
522 
523  return $status; // invalid storage path
524  }
525 
526  if ( $shard !== null ) { // confined to a single container/shard
527  $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
528  } else { // directory is on several shards
529  $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
530  [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
531  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
532  $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
533  }
534  }
535 
536  return $status;
537  }
538 
547  protected function doSecureInternal( $container, $dir, array $params ) {
548  return $this->newStatus();
549  }
550 
551  final protected function doPublish( array $params ) {
553  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
554  $status = $this->newStatus();
555 
556  [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
557  if ( $dir === null ) {
558  $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
559 
560  return $status; // invalid storage path
561  }
562 
563  if ( $shard !== null ) { // confined to a single container/shard
564  $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
565  } else { // directory is on several shards
566  $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
567  [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
568  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
569  $status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) );
570  }
571  }
572 
573  return $status;
574  }
575 
584  protected function doPublishInternal( $container, $dir, array $params ) {
585  return $this->newStatus();
586  }
587 
588  final protected function doClean( array $params ) {
590  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
591  $status = $this->newStatus();
592 
593  // Recursive: first delete all empty subdirs recursively
594  if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) {
595  $subDirsRel = $this->getTopDirectoryList( [ 'dir' => $params['dir'] ] );
596  if ( $subDirsRel !== null ) { // no errors
597  foreach ( $subDirsRel as $subDirRel ) {
598  $subDir = $params['dir'] . "/{$subDirRel}"; // full path
599  $status->merge( $this->doClean( [ 'dir' => $subDir ] + $params ) );
600  }
601  unset( $subDirsRel ); // free directory for rmdir() on Windows (for FS backends)
602  }
603  }
604 
605  [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
606  if ( $dir === null ) {
607  $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
608 
609  return $status; // invalid storage path
610  }
611 
612  // Attempt to lock this directory...
613  $filesLockEx = [ $params['dir'] ];
615  $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
616  if ( !$status->isOK() ) {
617  return $status; // abort
618  }
619 
620  if ( $shard !== null ) { // confined to a single container/shard
621  $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
622  $this->deleteContainerCache( $fullCont ); // purge cache
623  } else { // directory is on several shards
624  $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
625  [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
626  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
627  $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
628  $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache
629  }
630  }
631 
632  return $status;
633  }
634 
643  protected function doCleanInternal( $container, $dir, array $params ) {
644  return $this->newStatus();
645  }
646 
647  final public function fileExists( array $params ) {
649  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
650 
651  $stat = $this->getFileStat( $params );
652  if ( is_array( $stat ) ) {
653  return true;
654  }
655 
656  return ( $stat === self::$RES_ABSENT ) ? false : self::EXISTENCE_ERROR;
657  }
658 
659  final public function getFileTimestamp( array $params ) {
661  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
662 
663  $stat = $this->getFileStat( $params );
664  if ( is_array( $stat ) ) {
665  return $stat['mtime'];
666  }
667 
668  return self::TIMESTAMP_FAIL; // all failure cases
669  }
670 
671  final public function getFileSize( array $params ) {
673  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
674 
675  $stat = $this->getFileStat( $params );
676  if ( is_array( $stat ) ) {
677  return $stat['size'];
678  }
679 
680  return self::SIZE_FAIL; // all failure cases
681  }
682 
683  final public function getFileStat( array $params ) {
685  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
686 
687  $path = self::normalizeStoragePath( $params['src'] );
688  if ( $path === null ) {
689  return self::STAT_ERROR; // invalid storage path
690  }
691 
692  // Whether to bypass cache except for process cache entries loaded directly from
693  // high consistency backend queries (caller handles any cache flushing and locking)
694  $latest = !empty( $params['latest'] );
695  // Whether to ignore cache entries missing the SHA-1 field for existing files
696  $requireSHA1 = !empty( $params['requireSHA1'] );
697 
698  $stat = $this->cheapCache->getField( $path, 'stat', self::CACHE_TTL );
699  // Load the persistent stat cache into process cache if needed
700  if ( !$latest ) {
701  if (
702  // File stat is not in process cache
703  $stat === null ||
704  // Key/value store backends might opportunistically set file stat process
705  // cache entries from object listings that do not include the SHA-1. In that
706  // case, loading the persistent stat cache will likely yield the SHA-1.
707  ( $requireSHA1 && is_array( $stat ) && !isset( $stat['sha1'] ) )
708  ) {
709  $this->primeFileCache( [ $path ] );
710  // Get any newly process-cached entry
711  $stat = $this->cheapCache->getField( $path, 'stat', self::CACHE_TTL );
712  }
713  }
714 
715  if ( is_array( $stat ) ) {
716  if (
717  ( !$latest || $stat['latest'] ) &&
718  ( !$requireSHA1 || isset( $stat['sha1'] ) )
719  ) {
720  return $stat;
721  }
722  } elseif ( $stat === self::$ABSENT_LATEST ) {
723  return self::STAT_ABSENT;
724  } elseif ( $stat === self::$ABSENT_NORMAL ) {
725  if ( !$latest ) {
726  return self::STAT_ABSENT;
727  }
728  }
729 
730  // Load the file stat from the backend and update caches
731  $stat = $this->doGetFileStat( $params );
732  $this->ingestFreshFileStats( [ $path => $stat ], $latest );
733 
734  if ( is_array( $stat ) ) {
735  return $stat;
736  }
737 
738  return ( $stat === self::$RES_ERROR ) ? self::STAT_ERROR : self::STAT_ABSENT;
739  }
740 
748  final protected function ingestFreshFileStats( array $stats, $latest ) {
749  $success = true;
750 
751  foreach ( $stats as $path => $stat ) {
752  if ( is_array( $stat ) ) {
753  // Strongly consistent backends might automatically set this flag
754  $stat['latest'] ??= $latest;
755 
756  $this->cheapCache->setField( $path, 'stat', $stat );
757  if ( isset( $stat['sha1'] ) ) {
758  // Some backends store the SHA-1 hash as metadata
759  $this->cheapCache->setField(
760  $path,
761  'sha1',
762  [ 'hash' => $stat['sha1'], 'latest' => $latest ]
763  );
764  }
765  if ( isset( $stat['xattr'] ) ) {
766  // Some backends store custom headers/metadata
767  $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] );
768  $this->cheapCache->setField(
769  $path,
770  'xattr',
771  [ 'map' => $stat['xattr'], 'latest' => $latest ]
772  );
773  }
774  // Update persistent cache (@TODO: set all entries in one batch)
775  $this->setFileCache( $path, $stat );
776  } elseif ( $stat === self::$RES_ABSENT ) {
777  $this->cheapCache->setField(
778  $path,
779  'stat',
780  $latest ? self::$ABSENT_LATEST : self::$ABSENT_NORMAL
781  );
782  $this->cheapCache->setField(
783  $path,
784  'xattr',
785  [ 'map' => self::XATTRS_FAIL, 'latest' => $latest ]
786  );
787  $this->cheapCache->setField(
788  $path,
789  'sha1',
790  [ 'hash' => self::SHA1_FAIL, 'latest' => $latest ]
791  );
792  $this->logger->debug(
793  __METHOD__ . ': File {path} does not exist',
794  [ 'path' => $path ]
795  );
796  } else {
797  $success = false;
798  $this->logger->error(
799  __METHOD__ . ': Could not stat file {path}',
800  [ 'path' => $path ]
801  );
802  }
803  }
804 
805  return $success;
806  }
807 
812  abstract protected function doGetFileStat( array $params );
813 
814  public function getFileContentsMulti( array $params ) {
816  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
817 
818  $params = $this->setConcurrencyFlags( $params );
819  $contents = $this->doGetFileContentsMulti( $params );
820  foreach ( $contents as $path => $content ) {
821  if ( !is_string( $content ) ) {
822  $contents[$path] = self::CONTENT_FAIL; // used for all failure cases
823  }
824  }
825 
826  return $contents;
827  }
828 
835  protected function doGetFileContentsMulti( array $params ) {
836  $contents = [];
837  foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
838  if ( $fsFile instanceof FSFile ) {
839  AtEase::suppressWarnings();
840  $content = file_get_contents( $fsFile->getPath() );
841  AtEase::restoreWarnings();
842  $contents[$path] = is_string( $content ) ? $content : self::$RES_ERROR;
843  } elseif ( $fsFile === self::$RES_ABSENT ) {
844  $contents[$path] = self::$RES_ABSENT;
845  } else {
846  $contents[$path] = self::$RES_ERROR;
847  }
848  }
849 
850  return $contents;
851  }
852 
853  final public function getFileXAttributes( array $params ) {
855  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
856 
857  $path = self::normalizeStoragePath( $params['src'] );
858  if ( $path === null ) {
859  return self::XATTRS_FAIL; // invalid storage path
860  }
861  $latest = !empty( $params['latest'] ); // use latest data?
862  if ( $this->cheapCache->hasField( $path, 'xattr', self::CACHE_TTL ) ) {
863  $stat = $this->cheapCache->getField( $path, 'xattr' );
864  // If we want the latest data, check that this cached
865  // value was in fact fetched with the latest available data.
866  if ( !$latest || $stat['latest'] ) {
867  return $stat['map'];
868  }
869  }
870  $fields = $this->doGetFileXAttributes( $params );
871  if ( is_array( $fields ) ) {
872  $fields = self::normalizeXAttributes( $fields );
873  $this->cheapCache->setField(
874  $path,
875  'xattr',
876  [ 'map' => $fields, 'latest' => $latest ]
877  );
878  } elseif ( $fields === self::$RES_ABSENT ) {
879  $this->cheapCache->setField(
880  $path,
881  'xattr',
882  [ 'map' => self::XATTRS_FAIL, 'latest' => $latest ]
883  );
884  } else {
885  $fields = self::XATTRS_FAIL; // used for all failure cases
886  }
887 
888  return $fields;
889  }
890 
897  protected function doGetFileXAttributes( array $params ) {
898  return [ 'headers' => [], 'metadata' => [] ]; // not supported
899  }
900 
901  final public function getFileSha1Base36( array $params ) {
903  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
904 
905  $path = self::normalizeStoragePath( $params['src'] );
906  if ( $path === null ) {
907  return self::SHA1_FAIL; // invalid storage path
908  }
909  $latest = !empty( $params['latest'] ); // use latest data?
910  if ( $this->cheapCache->hasField( $path, 'sha1', self::CACHE_TTL ) ) {
911  $stat = $this->cheapCache->getField( $path, 'sha1' );
912  // If we want the latest data, check that this cached
913  // value was in fact fetched with the latest available data.
914  if ( !$latest || $stat['latest'] ) {
915  return $stat['hash'];
916  }
917  }
918  $sha1 = $this->doGetFileSha1Base36( $params );
919  if ( is_string( $sha1 ) ) {
920  $this->cheapCache->setField(
921  $path,
922  'sha1',
923  [ 'hash' => $sha1, 'latest' => $latest ]
924  );
925  } elseif ( $sha1 === self::$RES_ABSENT ) {
926  $this->cheapCache->setField(
927  $path,
928  'sha1',
929  [ 'hash' => self::SHA1_FAIL, 'latest' => $latest ]
930  );
931  } else {
932  $sha1 = self::SHA1_FAIL; // used for all failure cases
933  }
934 
935  return $sha1;
936  }
937 
944  protected function doGetFileSha1Base36( array $params ) {
945  $fsFile = $this->getLocalReference( $params );
946  if ( $fsFile instanceof FSFile ) {
947  $sha1 = $fsFile->getSha1Base36();
948 
949  return is_string( $sha1 ) ? $sha1 : self::$RES_ERROR;
950  }
951 
952  return ( $fsFile === self::$RES_ERROR ) ? self::$RES_ERROR : self::$RES_ABSENT;
953  }
954 
955  final public function getFileProps( array $params ) {
957  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
958 
959  $fsFile = $this->getLocalReference( $params );
960 
961  return $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
962  }
963 
964  final public function getLocalReferenceMulti( array $params ) {
966  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
967 
968  $params = $this->setConcurrencyFlags( $params );
969 
970  $fsFiles = []; // (path => FSFile)
971  $latest = !empty( $params['latest'] ); // use latest data?
972  // Reuse any files already in process cache...
973  foreach ( $params['srcs'] as $src ) {
975  if ( $path === null ) {
976  $fsFiles[$src] = null; // invalid storage path
977  } elseif ( $this->expensiveCache->hasField( $path, 'localRef' ) ) {
978  $val = $this->expensiveCache->getField( $path, 'localRef' );
979  // If we want the latest data, check that this cached
980  // value was in fact fetched with the latest available data.
981  if ( !$latest || $val['latest'] ) {
982  $fsFiles[$src] = $val['object'];
983  }
984  }
985  }
986  // Fetch local references of any remaining files...
987  $params['srcs'] = array_diff( $params['srcs'], array_keys( $fsFiles ) );
988  foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
989  if ( $fsFile instanceof FSFile ) {
990  $fsFiles[$path] = $fsFile;
991  $this->expensiveCache->setField(
992  $path,
993  'localRef',
994  [ 'object' => $fsFile, 'latest' => $latest ]
995  );
996  } else {
997  $fsFiles[$path] = null; // used for all failure cases
998  }
999  }
1000 
1001  return $fsFiles;
1002  }
1003 
1010  protected function doGetLocalReferenceMulti( array $params ) {
1011  return $this->doGetLocalCopyMulti( $params );
1012  }
1013 
1014  final public function getLocalCopyMulti( array $params ) {
1016  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1017 
1018  $params = $this->setConcurrencyFlags( $params );
1019  $tmpFiles = $this->doGetLocalCopyMulti( $params );
1020  foreach ( $tmpFiles as $path => $tmpFile ) {
1021  if ( !$tmpFile ) {
1022  $tmpFiles[$path] = null; // used for all failure cases
1023  }
1024  }
1025 
1026  return $tmpFiles;
1027  }
1028 
1034  abstract protected function doGetLocalCopyMulti( array $params );
1035 
1042  public function getFileHttpUrl( array $params ) {
1043  return self::TEMPURL_ERROR; // not supported
1044  }
1045 
1046  final public function streamFile( array $params ) {
1048  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1049  $status = $this->newStatus();
1050 
1051  // Always set some fields for subclass convenience
1052  $params['options'] ??= [];
1053  $params['headers'] ??= [];
1054 
1055  // Don't stream it out as text/html if there was a PHP error
1056  if ( ( empty( $params['headless'] ) || $params['headers'] ) && headers_sent() ) {
1057  print "Headers already sent, terminating.\n";
1058  $status->fatal( 'backend-fail-stream', $params['src'] );
1059  return $status;
1060  }
1061 
1062  $status->merge( $this->doStreamFile( $params ) );
1063 
1064  return $status;
1065  }
1066 
1073  protected function doStreamFile( array $params ) {
1074  $status = $this->newStatus();
1075 
1076  $flags = 0;
1077  $flags |= !empty( $params['headless'] ) ? HTTPFileStreamer::STREAM_HEADLESS : 0;
1078  $flags |= !empty( $params['allowOB'] ) ? HTTPFileStreamer::STREAM_ALLOW_OB : 0;
1079 
1080  $fsFile = $this->getLocalReference( $params );
1081  if ( $fsFile ) {
1082  $streamer = new HTTPFileStreamer(
1083  $fsFile->getPath(),
1084  [
1085  'obResetFunc' => $this->obResetFunc,
1086  'streamMimeFunc' => $this->streamMimeFunc
1087  ]
1088  );
1089  $res = $streamer->stream( $params['headers'], true, $params['options'], $flags );
1090  } else {
1091  $res = false;
1092  HTTPFileStreamer::send404Message( $params['src'], $flags );
1093  }
1094 
1095  if ( !$res ) {
1096  $status->fatal( 'backend-fail-stream', $params['src'] );
1097  }
1098 
1099  return $status;
1100  }
1101 
1102  final public function directoryExists( array $params ) {
1103  [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
1104  if ( $dir === null ) {
1105  return self::EXISTENCE_ERROR; // invalid storage path
1106  }
1107  if ( $shard !== null ) { // confined to a single container/shard
1108  return $this->doDirectoryExists( $fullCont, $dir, $params );
1109  } else { // directory is on several shards
1110  $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
1111  [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
1112  $res = false; // response
1113  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
1114  $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params );
1115  if ( $exists === true ) {
1116  $res = true;
1117  break; // found one!
1118  } elseif ( $exists === self::$RES_ERROR ) {
1119  $res = self::EXISTENCE_ERROR;
1120  }
1121  }
1122 
1123  return $res;
1124  }
1125  }
1126 
1135  abstract protected function doDirectoryExists( $container, $dir, array $params );
1136 
1137  final public function getDirectoryList( array $params ) {
1138  [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
1139  if ( $dir === null ) {
1140  return self::EXISTENCE_ERROR; // invalid storage path
1141  }
1142  if ( $shard !== null ) {
1143  // File listing is confined to a single container/shard
1144  return $this->getDirectoryListInternal( $fullCont, $dir, $params );
1145  } else {
1146  $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
1147  // File listing spans multiple containers/shards
1148  [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
1149 
1150  return new FileBackendStoreShardDirIterator( $this,
1151  $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
1152  }
1153  }
1154 
1165  abstract public function getDirectoryListInternal( $container, $dir, array $params );
1166 
1167  final public function getFileList( array $params ) {
1168  [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
1169  if ( $dir === null ) {
1170  return self::LIST_ERROR; // invalid storage path
1171  }
1172  if ( $shard !== null ) {
1173  // File listing is confined to a single container/shard
1174  return $this->getFileListInternal( $fullCont, $dir, $params );
1175  } else {
1176  $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
1177  // File listing spans multiple containers/shards
1178  [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
1179 
1180  return new FileBackendStoreShardFileIterator( $this,
1181  $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
1182  }
1183  }
1184 
1195  abstract public function getFileListInternal( $container, $dir, array $params );
1196 
1208  final public function getOperationsInternal( array $ops ) {
1209  $supportedOps = [
1210  'store' => StoreFileOp::class,
1211  'copy' => CopyFileOp::class,
1212  'move' => MoveFileOp::class,
1213  'delete' => DeleteFileOp::class,
1214  'create' => CreateFileOp::class,
1215  'describe' => DescribeFileOp::class,
1216  'null' => NullFileOp::class
1217  ];
1218 
1219  $performOps = []; // array of FileOp objects
1220  // Build up ordered array of FileOps...
1221  foreach ( $ops as $operation ) {
1222  $opName = $operation['op'];
1223  if ( isset( $supportedOps[$opName] ) ) {
1224  $class = $supportedOps[$opName];
1225  // Get params for this operation
1226  $params = $operation;
1227  // Append the FileOp class
1228  $performOps[] = new $class( $this, $params, $this->logger );
1229  } else {
1230  throw new FileBackendError( "Operation '$opName' is not supported." );
1231  }
1232  }
1233 
1234  return $performOps;
1235  }
1236 
1247  final public function getPathsToLockForOpsInternal( array $performOps ) {
1248  // Build up a list of files to lock...
1249  $paths = [ 'sh' => [], 'ex' => [] ];
1250  foreach ( $performOps as $fileOp ) {
1251  $paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() );
1252  $paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() );
1253  }
1254  // Optimization: if doing an EX lock anyway, don't also set an SH one
1255  $paths['sh'] = array_diff( $paths['sh'], $paths['ex'] );
1256  // Get a shared lock on the parent directory of each path changed
1257  $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) );
1258 
1259  return [
1260  LockManager::LOCK_UW => $paths['sh'],
1261  LockManager::LOCK_EX => $paths['ex']
1262  ];
1263  }
1264 
1265  public function getScopedLocksForOps( array $ops, StatusValue $status ) {
1266  $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) );
1267 
1268  return $this->getScopedFileLocks( $paths, 'mixed', $status );
1269  }
1270 
1271  final protected function doOperationsInternal( array $ops, array $opts ) {
1273  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1274  $status = $this->newStatus();
1275 
1276  // Fix up custom header name/value pairs
1277  $ops = array_map( [ $this, 'sanitizeOpHeaders' ], $ops );
1278  // Build up a list of FileOps and involved paths
1279  $fileOps = $this->getOperationsInternal( $ops );
1280  $pathsUsed = [];
1281  foreach ( $fileOps as $fileOp ) {
1282  $pathsUsed = array_merge( $pathsUsed, $fileOp->storagePathsReadOrChanged() );
1283  }
1284 
1285  // Acquire any locks as needed for the scope of this function
1286  if ( empty( $opts['nonLocking'] ) ) {
1287  $pathsByLockType = $this->getPathsToLockForOpsInternal( $fileOps );
1289  $scopeLock = $this->getScopedFileLocks( $pathsByLockType, 'mixed', $status );
1290  if ( !$status->isOK() ) {
1291  return $status; // abort
1292  }
1293  }
1294 
1295  // Clear any file cache entries (after locks acquired)
1296  if ( empty( $opts['preserveCache'] ) ) {
1297  $this->clearCache( $pathsUsed );
1298  }
1299 
1300  // Enlarge the cache to fit the stat entries of these files
1301  $this->cheapCache->setMaxSize( max( 2 * count( $pathsUsed ), self::CACHE_CHEAP_SIZE ) );
1302 
1303  // Load from the persistent container caches
1304  $this->primeContainerCache( $pathsUsed );
1305  // Get the latest stat info for all the files (having locked them)
1306  $ok = $this->preloadFileStat( [ 'srcs' => $pathsUsed, 'latest' => true ] );
1307 
1308  if ( $ok ) {
1309  // Actually attempt the operation batch...
1310  $opts = $this->setConcurrencyFlags( $opts );
1311  $subStatus = FileOpBatch::attempt( $fileOps, $opts );
1312  } else {
1313  // If we could not even stat some files, then bail out
1314  $subStatus = $this->newStatus( 'backend-fail-internal', $this->name );
1315  foreach ( $ops as $i => $op ) { // mark each op as failed
1316  $subStatus->success[$i] = false;
1317  ++$subStatus->failCount;
1318  }
1319  $this->logger->error( static::class . "-{$this->name} " .
1320  " stat failure; aborted operations: " . FormatJson::encode( $ops ) );
1321  }
1322 
1323  // Merge errors into StatusValue fields
1324  $status->merge( $subStatus );
1325  $status->success = $subStatus->success; // not done in merge()
1326 
1327  // Shrink the stat cache back to normal size
1328  $this->cheapCache->setMaxSize( self::CACHE_CHEAP_SIZE );
1329 
1330  return $status;
1331  }
1332 
1333  final protected function doQuickOperationsInternal( array $ops, array $opts ) {
1335  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1336  $status = $this->newStatus();
1337 
1338  // Fix up custom header name/value pairs
1339  $ops = array_map( [ $this, 'sanitizeOpHeaders' ], $ops );
1340  // Build up a list of FileOps and involved paths
1341  $fileOps = $this->getOperationsInternal( $ops );
1342  $pathsUsed = [];
1343  foreach ( $fileOps as $fileOp ) {
1344  $pathsUsed = array_merge( $pathsUsed, $fileOp->storagePathsReadOrChanged() );
1345  }
1346 
1347  // Clear any file cache entries for involved paths
1348  $this->clearCache( $pathsUsed );
1349 
1350  // Parallel ops may be disabled in config due to dependencies (e.g. needing popen())
1351  $async = ( $this->parallelize === 'implicit' && count( $ops ) > 1 );
1352  $maxConcurrency = $this->concurrency; // throttle
1354  $statuses = []; // array of (index => StatusValue)
1355  $fileOpHandles = []; // list of (index => handle) arrays
1356  $curFileOpHandles = []; // current handle batch
1357  // Perform the sync-only ops and build up op handles for the async ops...
1358  foreach ( $fileOps as $index => $fileOp ) {
1359  $subStatus = $async
1360  ? $fileOp->attemptAsyncQuick()
1361  : $fileOp->attemptQuick();
1362  if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { // async
1363  if ( count( $curFileOpHandles ) >= $maxConcurrency ) {
1364  $fileOpHandles[] = $curFileOpHandles; // push this batch
1365  $curFileOpHandles = [];
1366  }
1367  $curFileOpHandles[$index] = $subStatus->value; // keep index
1368  } else { // error or completed
1369  $statuses[$index] = $subStatus; // keep index
1370  }
1371  }
1372  if ( count( $curFileOpHandles ) ) {
1373  $fileOpHandles[] = $curFileOpHandles; // last batch
1374  }
1375  // Do all the async ops that can be done concurrently...
1376  foreach ( $fileOpHandles as $fileHandleBatch ) {
1377  $statuses += $this->executeOpHandlesInternal( $fileHandleBatch );
1378  }
1379  // Marshall and merge all the responses...
1380  foreach ( $statuses as $index => $subStatus ) {
1381  $status->merge( $subStatus );
1382  if ( $subStatus->isOK() ) {
1383  $status->success[$index] = true;
1384  ++$status->successCount;
1385  } else {
1386  $status->success[$index] = false;
1387  ++$status->failCount;
1388  }
1389  }
1390 
1391  $this->clearCache( $pathsUsed );
1392 
1393  return $status;
1394  }
1395 
1405  final public function executeOpHandlesInternal( array $fileOpHandles ) {
1407  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1408 
1409  foreach ( $fileOpHandles as $fileOpHandle ) {
1410  if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) {
1411  throw new InvalidArgumentException( "Expected FileBackendStoreOpHandle object." );
1412  } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) {
1413  throw new InvalidArgumentException( "Expected handle for this file backend." );
1414  }
1415  }
1416 
1417  $statuses = $this->doExecuteOpHandlesInternal( $fileOpHandles );
1418  foreach ( $fileOpHandles as $fileOpHandle ) {
1419  $fileOpHandle->closeResources();
1420  }
1421 
1422  return $statuses;
1423  }
1424 
1434  protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1435  if ( count( $fileOpHandles ) ) {
1436  throw new FileBackendError( "Backend does not support asynchronous operations." );
1437  }
1438 
1439  return [];
1440  }
1441 
1453  protected function sanitizeOpHeaders( array $op ) {
1454  static $longs = [ 'content-disposition' ];
1455 
1456  if ( isset( $op['headers'] ) ) { // op sets HTTP headers
1457  $newHeaders = [];
1458  foreach ( $op['headers'] as $name => $value ) {
1459  $name = strtolower( $name );
1460  $maxHVLen = in_array( $name, $longs ) ? INF : 255;
1461  if ( strlen( $name ) > 255 || strlen( $value ) > $maxHVLen ) {
1462  $this->logger->error( "Header '{header}' is too long.", [
1463  'filebackend' => $this->name,
1464  'header' => "$name: $value",
1465  ] );
1466  } else {
1467  $newHeaders[$name] = strlen( $value ) ? $value : ''; // null/false => ""
1468  }
1469  }
1470  $op['headers'] = $newHeaders;
1471  }
1472 
1473  return $op;
1474  }
1475 
1476  final public function preloadCache( array $paths ) {
1477  $fullConts = []; // full container names
1478  foreach ( $paths as $path ) {
1479  [ $fullCont, , ] = $this->resolveStoragePath( $path );
1480  $fullConts[] = $fullCont;
1481  }
1482  // Load from the persistent file and container caches
1483  $this->primeContainerCache( $fullConts );
1484  $this->primeFileCache( $paths );
1485  }
1486 
1487  final public function clearCache( array $paths = null ) {
1488  if ( is_array( $paths ) ) {
1489  $paths = array_map( [ FileBackend::class, 'normalizeStoragePath' ], $paths );
1490  $paths = array_filter( $paths, 'strlen' ); // remove nulls
1491  }
1492  if ( $paths === null ) {
1493  $this->cheapCache->clear();
1494  $this->expensiveCache->clear();
1495  } else {
1496  foreach ( $paths as $path ) {
1497  $this->cheapCache->clear( $path );
1498  $this->expensiveCache->clear( $path );
1499  }
1500  }
1501  $this->doClearCache( $paths );
1502  }
1503 
1512  protected function doClearCache( array $paths = null ) {
1513  }
1514 
1515  final public function preloadFileStat( array $params ) {
1517  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1518 
1519  $params['concurrency'] = ( $this->parallelize !== 'off' ) ? $this->concurrency : 1;
1520  $stats = $this->doGetFileStatMulti( $params );
1521  if ( $stats === null ) {
1522  return true; // not supported
1523  }
1524 
1525  // Whether this queried the backend in high consistency mode
1526  $latest = !empty( $params['latest'] );
1527 
1528  return $this->ingestFreshFileStats( $stats, $latest );
1529  }
1530 
1543  protected function doGetFileStatMulti( array $params ) {
1544  return null; // not supported
1545  }
1546 
1554  abstract protected function directoriesAreVirtual();
1555 
1566  final protected static function isValidShortContainerName( $container ) {
1567  // Suffixes like '.xxx' (hex shard chars) or '.seg' (file segments)
1568  // might be used by subclasses. Reserve the dot character.
1569  // The only way dots end up in containers (e.g. resolveStoragePath)
1570  // is due to the wikiId container prefix or the above suffixes.
1571  return self::isValidContainerName( $container ) && !preg_match( '/[.]/', $container );
1572  }
1573 
1583  final protected static function isValidContainerName( $container ) {
1584  // This accounts for NTFS, Swift, and Ceph restrictions
1585  // and disallows directory separators or traversal characters.
1586  // Note that matching strings URL encode to the same string;
1587  // in Swift/Ceph, the length restriction is *after* URL encoding.
1588  return (bool)preg_match( '/^[a-z0-9][a-z0-9-_.]{0,199}$/i', $container );
1589  }
1590 
1604  final protected function resolveStoragePath( $storagePath ) {
1605  [ $backend, $shortCont, $relPath ] = self::splitStoragePath( $storagePath );
1606  if ( $backend === $this->name ) { // must be for this backend
1607  $relPath = self::normalizeContainerPath( $relPath );
1608  if ( $relPath !== null && self::isValidShortContainerName( $shortCont ) ) {
1609  // Get shard for the normalized path if this container is sharded
1610  $cShard = $this->getContainerShard( $shortCont, $relPath );
1611  // Validate and sanitize the relative path (backend-specific)
1612  $relPath = $this->resolveContainerPath( $shortCont, $relPath );
1613  if ( $relPath !== null ) {
1614  // Prepend any domain ID prefix to the container name
1615  $container = $this->fullContainerName( $shortCont );
1616  if ( self::isValidContainerName( $container ) ) {
1617  // Validate and sanitize the container name (backend-specific)
1618  $container = $this->resolveContainerName( "{$container}{$cShard}" );
1619  if ( $container !== null ) {
1620  return [ $container, $relPath, $cShard ];
1621  }
1622  }
1623  }
1624  }
1625  }
1626 
1627  return [ null, null, null ];
1628  }
1629 
1645  final protected function resolveStoragePathReal( $storagePath ) {
1646  [ $container, $relPath, $cShard ] = $this->resolveStoragePath( $storagePath );
1647  if ( $cShard !== null && substr( $relPath, -1 ) !== '/' ) {
1648  return [ $container, $relPath ];
1649  }
1650 
1651  return [ null, null ];
1652  }
1653 
1662  final protected function getContainerShard( $container, $relPath ) {
1663  [ $levels, $base, $repeat ] = $this->getContainerHashLevels( $container );
1664  if ( $levels == 1 || $levels == 2 ) {
1665  // Hash characters are either base 16 or 36
1666  $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]';
1667  // Get a regex that represents the shard portion of paths.
1668  // The concatenation of the captures gives us the shard.
1669  if ( $levels === 1 ) { // 16 or 36 shards per container
1670  $hashDirRegex = '(' . $char . ')';
1671  } else { // 256 or 1296 shards per container
1672  if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc")
1673  $hashDirRegex = $char . '/(' . $char . '{2})';
1674  } else { // short hash dir format (e.g. "a/b/c")
1675  $hashDirRegex = '(' . $char . ')/(' . $char . ')';
1676  }
1677  }
1678  // Allow certain directories to be above the hash dirs so as
1679  // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
1680  // They must be 2+ chars to avoid any hash directory ambiguity.
1681  $m = [];
1682  if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1683  return '.' . implode( '', array_slice( $m, 1 ) );
1684  }
1685 
1686  return null; // failed to match
1687  }
1688 
1689  return ''; // no sharding
1690  }
1691 
1700  final public function isSingleShardPathInternal( $storagePath ) {
1701  [ , , $shard ] = $this->resolveStoragePath( $storagePath );
1702 
1703  return ( $shard !== null );
1704  }
1705 
1714  final protected function getContainerHashLevels( $container ) {
1715  if ( isset( $this->shardViaHashLevels[$container] ) ) {
1716  $config = $this->shardViaHashLevels[$container];
1717  $hashLevels = (int)$config['levels'];
1718  if ( $hashLevels == 1 || $hashLevels == 2 ) {
1719  $hashBase = (int)$config['base'];
1720  if ( $hashBase == 16 || $hashBase == 36 ) {
1721  return [ $hashLevels, $hashBase, $config['repeat'] ];
1722  }
1723  }
1724  }
1725 
1726  return [ 0, 0, false ]; // no sharding
1727  }
1728 
1735  final protected function getContainerSuffixes( $container ) {
1736  $shards = [];
1737  [ $digits, $base ] = $this->getContainerHashLevels( $container );
1738  if ( $digits > 0 ) {
1739  $numShards = $base ** $digits;
1740  for ( $index = 0; $index < $numShards; $index++ ) {
1741  $shards[] = '.' . Wikimedia\base_convert( (string)$index, 10, $base, $digits );
1742  }
1743  }
1744 
1745  return $shards;
1746  }
1747 
1754  final protected function fullContainerName( $container ) {
1755  if ( $this->domainId != '' ) {
1756  return "{$this->domainId}-$container";
1757  } else {
1758  return $container;
1759  }
1760  }
1761 
1771  protected function resolveContainerName( $container ) {
1772  return $container;
1773  }
1774 
1786  protected function resolveContainerPath( $container, $relStoragePath ) {
1787  return $relStoragePath;
1788  }
1789 
1796  private function containerCacheKey( $container ) {
1797  return "filebackend:{$this->name}:{$this->domainId}:container:{$container}";
1798  }
1799 
1806  final protected function setContainerCache( $container, array $val ) {
1807  $this->memCache->set( $this->containerCacheKey( $container ), $val, 14 * 86400 );
1808  }
1809 
1816  final protected function deleteContainerCache( $container ) {
1817  if ( !$this->memCache->delete( $this->containerCacheKey( $container ), 300 ) ) {
1818  $this->logger->warning( "Unable to delete stat cache for container {container}.",
1819  [ 'filebackend' => $this->name, 'container' => $container ]
1820  );
1821  }
1822  }
1823 
1831  final protected function primeContainerCache( array $items ) {
1833  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1834 
1835  $paths = []; // list of storage paths
1836  $contNames = []; // (cache key => resolved container name)
1837  // Get all the paths/containers from the items...
1838  foreach ( $items as $item ) {
1839  if ( self::isStoragePath( $item ) ) {
1840  $paths[] = $item;
1841  } elseif ( is_string( $item ) ) { // full container name
1842  $contNames[$this->containerCacheKey( $item )] = $item;
1843  }
1844  }
1845  // Get all the corresponding cache keys for paths...
1846  foreach ( $paths as $path ) {
1847  [ $fullCont, , ] = $this->resolveStoragePath( $path );
1848  if ( $fullCont !== null ) { // valid path for this backend
1849  $contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
1850  }
1851  }
1852 
1853  $contInfo = []; // (resolved container name => cache value)
1854  // Get all cache entries for these container cache keys...
1855  $values = $this->memCache->getMulti( array_keys( $contNames ) );
1856  foreach ( $values as $cacheKey => $val ) {
1857  $contInfo[$contNames[$cacheKey]] = $val;
1858  }
1859 
1860  // Populate the container process cache for the backend...
1861  $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) );
1862  }
1863 
1872  protected function doPrimeContainerCache( array $containerInfo ) {
1873  }
1874 
1881  private function fileCacheKey( $path ) {
1882  return "filebackend:{$this->name}:{$this->domainId}:file:" . sha1( $path );
1883  }
1884 
1893  final protected function setFileCache( $path, array $val ) {
1895  if ( $path === null ) {
1896  return; // invalid storage path
1897  }
1898  $mtime = (int)ConvertibleTimestamp::convert( TS_UNIX, $val['mtime'] );
1899  $ttl = $this->memCache->adaptiveTTL( $mtime, 7 * 86400, 300, 0.1 );
1900  $key = $this->fileCacheKey( $path );
1901  // Set the cache unless it is currently salted.
1902  $this->memCache->set( $key, $val, $ttl );
1903  }
1904 
1913  final protected function deleteFileCache( $path ) {
1915  if ( $path === null ) {
1916  return; // invalid storage path
1917  }
1918  if ( !$this->memCache->delete( $this->fileCacheKey( $path ), 300 ) ) {
1919  $this->logger->warning( "Unable to delete stat cache for file {path}.",
1920  [ 'filebackend' => $this->name, 'path' => $path ]
1921  );
1922  }
1923  }
1924 
1932  final protected function primeFileCache( array $items ) {
1934  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1935 
1936  $paths = []; // list of storage paths
1937  $pathNames = []; // (cache key => storage path)
1938  // Get all the paths/containers from the items...
1939  foreach ( $items as $item ) {
1940  if ( self::isStoragePath( $item ) ) {
1942  if ( $path !== null ) {
1943  $paths[] = $path;
1944  }
1945  }
1946  }
1947  // Get rid of any paths that failed normalization
1948  $paths = array_filter( $paths, 'strlen' ); // remove nulls
1949  // Get all the corresponding cache keys for paths...
1950  foreach ( $paths as $path ) {
1951  [ , $rel, ] = $this->resolveStoragePath( $path );
1952  if ( $rel !== null ) { // valid path for this backend
1953  $pathNames[$this->fileCacheKey( $path )] = $path;
1954  }
1955  }
1956  // Get all cache entries for these file cache keys.
1957  // Note that negatives are not cached by getFileStat()/preloadFileStat().
1958  $values = $this->memCache->getMulti( array_keys( $pathNames ) );
1959  // Load all of the results into process cache...
1960  foreach ( array_filter( $values, 'is_array' ) as $cacheKey => $stat ) {
1961  $path = $pathNames[$cacheKey];
1962  // This flag only applies to stat info loaded directly
1963  // from a high consistency backend query to the process cache
1964  unset( $stat['latest'] );
1965 
1966  $this->cheapCache->setField( $path, 'stat', $stat );
1967  if ( isset( $stat['sha1'] ) && strlen( $stat['sha1'] ) == 31 ) {
1968  // Some backends store SHA-1 as metadata
1969  $this->cheapCache->setField(
1970  $path,
1971  'sha1',
1972  [ 'hash' => $stat['sha1'], 'latest' => false ]
1973  );
1974  }
1975  if ( isset( $stat['xattr'] ) && is_array( $stat['xattr'] ) ) {
1976  // Some backends store custom headers/metadata
1977  $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] );
1978  $this->cheapCache->setField(
1979  $path,
1980  'xattr',
1981  [ 'map' => $stat['xattr'], 'latest' => false ]
1982  );
1983  }
1984  }
1985  }
1986 
1994  final protected static function normalizeXAttributes( array $xattr ) {
1995  $newXAttr = [ 'headers' => [], 'metadata' => [] ];
1996 
1997  foreach ( $xattr['headers'] as $name => $value ) {
1998  $newXAttr['headers'][strtolower( $name )] = $value;
1999  }
2000 
2001  foreach ( $xattr['metadata'] as $name => $value ) {
2002  $newXAttr['metadata'][strtolower( $name )] = $value;
2003  }
2004 
2005  return $newXAttr;
2006  }
2007 
2014  final protected function setConcurrencyFlags( array $opts ) {
2015  $opts['concurrency'] = 1; // off
2016  if ( $this->parallelize === 'implicit' ) {
2017  if ( $opts['parallelize'] ?? true ) {
2018  $opts['concurrency'] = $this->concurrency;
2019  }
2020  } elseif ( $this->parallelize === 'explicit' ) {
2021  if ( !empty( $opts['parallelize'] ) ) {
2022  $opts['concurrency'] = $this->concurrency;
2023  }
2024  }
2025 
2026  return $opts;
2027  }
2028 
2038  protected function getContentType( $storagePath, $content, $fsPath ) {
2039  if ( $this->mimeCallback ) {
2040  return call_user_func_array( $this->mimeCallback, func_get_args() );
2041  }
2042 
2043  $mime = ( $fsPath !== null ) ? mime_content_type( $fsPath ) : false;
2044  return $mime ?: 'unknown/unknown';
2045  }
2046 }
$success
A BagOStuff object with no objects in it.
Class representing a non-directory file on the file system.
Definition: FSFile.php:32
static placeholderProps()
Placeholder file properties to use for files that don't exist.
Definition: FSFile.php:150
File backend exception for checked exceptions (e.g.
FileBackendStore helper class for performing asynchronous file operations.
Base class for all backends using particular storage medium.
preloadFileStat(array $params)
Preload file stat information (concurrently if possible) into in-process cache.
doGetLocalReferenceMulti(array $params)
resolveContainerName( $container)
Resolve a container name, checking if it's allowed by the backend.
static normalizeXAttributes(array $xattr)
Normalize file headers/metadata to the FileBackend::getFileXAttributes() format.
getLocalReferenceMulti(array $params)
Like getLocalReference() except it takes an array of storage paths and yields an order-preserved map ...
setFileCache( $path, array $val)
Set the cached stat info for a file path.
doPublish(array $params)
moveInternal(array $params)
Move a file from one storage path to another in the backend.
setContainerCache( $container, array $val)
Set the cached info for a container.
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path in the backend.
doGetFileContentsMulti(array $params)
static string $ABSENT_NORMAL
File does not exist according to a normal stat query.
doStreamFile(array $params)
doSecure(array $params)
MapCacheLRU $expensiveCache
Map of paths to large (RAM/disk) cache items.
doClean(array $params)
getFileTimestamp(array $params)
Get the last-modified timestamp of the file at a storage path.
doGetLocalCopyMulti(array $params)
getScopedLocksForOps(array $ops, StatusValue $status)
Get an array of scoped locks needed for a batch of file operations.
doCleanInternal( $container, $dir, array $params)
callable $mimeCallback
Method to get the MIME type of files.
getPathsToLockForOpsInternal(array $performOps)
Get a list of storage paths to lock for a list of operations Returns an array with LockManager::LOCK_...
executeOpHandlesInternal(array $fileOpHandles)
Execute a list of FileBackendStoreOpHandle handles in parallel.
doPrepareInternal( $container, $dir, array $params)
concatenate(array $params)
Concatenate a list of storage files into a single file system file.
storeInternal(array $params)
Store a file into the backend from a file on disk.
getFileProps(array $params)
Get the properties of the content of the file at a storage path in the backend.
getDirectoryList(array $params)
Get an iterator to list all directories under a storage directory.
doQuickOperationsInternal(array $ops, array $opts)
resolveStoragePath( $storagePath)
Splits a storage path into an internal container name, an internal relative file name,...
copyInternal(array $params)
Copy a file from one storage path to another in the backend.
doStoreInternal(array $params)
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
doGetFileStat(array $params)
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
MapCacheLRU $cheapCache
Map of paths to small (RAM/disk) cache items.
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
doPublishInternal( $container, $dir, array $params)
doGetFileStatMulti(array $params)
Get file stat information (concurrently if possible) for several files.
getFileSha1Base36(array $params)
Get a SHA-1 hash of the content of the file at a storage path in the backend.
static isValidContainerName( $container)
Check if a full container name is valid.
ingestFreshFileStats(array $stats, $latest)
Ingest file stat entries that just came from querying the backend (not cache)
getFileListInternal( $container, $dir, array $params)
Do not call this function from places outside FileBackend.
doCopyInternal(array $params)
getLocalCopyMulti(array $params)
Like getLocalCopy() except it takes an array of storage paths and yields an order preserved-map of st...
getFileContentsMulti(array $params)
Like getFileContents() except it takes an array of storage paths and returns an order preserved map o...
doClearCache(array $paths=null)
Clears any additional stat caches for storage paths.
fullContainerName( $container)
Get the full container name, including the domain ID prefix.
primeFileCache(array $items)
Do a batch lookup from cache for file stats for all paths used in a list of storage paths or FileOp o...
doOperationsInternal(array $ops, array $opts)
isSingleShardPathInternal( $storagePath)
Check if a storage path maps to a single shard.
directoryExists(array $params)
Check if a directory exists at a given storage path.
doDeleteInternal(array $params)
static string $ABSENT_LATEST
File does not exist according to a "latest"-mode stat query.
getContainerSuffixes( $container)
Get a list of full container shard suffixes for a container.
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
sanitizeOpHeaders(array $op)
Normalize and filter HTTP headers from a file operation.
doDescribeInternal(array $params)
doGetFileSha1Base36(array $params)
deleteInternal(array $params)
Delete a file at the storage path.
doPrimeContainerCache(array $containerInfo)
Fill the backend-specific process cache given an array of resolved container names and their correspo...
doMoveInternal(array $params)
maxFileSizeInternal()
Get the maximum allowable file size given backend medium restrictions and basic performance constrain...
describeInternal(array $params)
Alter metadata for a file at the storage path.
streamFile(array $params)
Stream the content of the file at a storage path in the backend.
getOperationsInternal(array $ops)
Return a list of FileOp objects from a list of operations.
deleteFileCache( $path)
Delete the cached stat info for a file path.
setConcurrencyFlags(array $opts)
Set the 'concurrency' option from a list of operation options.
getFileXAttributes(array $params)
Get metadata about a file at a storage path in the backend.
getContentType( $storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
preloadCache(array $paths)
Preload persistent file stat cache and property cache into in-process cache.
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
doDirectoryExists( $container, $dir, array $params)
WANObjectCache $memCache
static false $RES_ABSENT
Idiom for "no result due to missing file" (since 1.34)
doPrepare(array $params)
FileBackend::prepare() StatusValue Good status without value for success, fatal otherwise.
getFileHttpUrl(array $params)
__construct(array $config)
doGetFileXAttributes(array $params)
static null $RES_ERROR
Idiom for "no result due to I/O errors" (since 1.34)
doExecuteOpHandlesInternal(array $fileOpHandles)
nullInternal(array $params)
No-op file operation that does nothing.
getFileSize(array $params)
Get the size (bytes) of a file at a storage path in the backend.
array $shardViaHashLevels
Map of container names to sharding config.
doConcatenate(array $params)
doSecureInternal( $container, $dir, array $params)
doCreateInternal(array $params)
fileExists(array $params)
Check if a file exists at a storage path in the backend.
getDirectoryListInternal( $container, $dir, array $params)
Do not call this function from places outside FileBackend.
static isValidShortContainerName( $container)
Check if a short container name is valid.
getContainerHashLevels( $container)
Get the sharding config for a container.
getContainerShard( $container, $relPath)
Get the container name shard suffix for a given path.
deleteContainerCache( $container)
Delete the cached info for a container.
createInternal(array $params)
Create a file in the backend with the given contents.
getFileList(array $params)
Get an iterator to list all stored files under a storage directory.
Base class for all file backend classes (including multi-write backends).
Definition: FileBackend.php:99
string $name
Unique backend name.
static splitStoragePath( $storagePath)
Split a storage path into a backend name, a container name, and a relative file path.
static normalizeContainerPath( $path)
Validate and normalize a relative storage path.
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
getTopDirectoryList(array $params)
Same as FileBackend::getDirectoryList() except only lists directories that are immediately under the ...
scopedProfileSection( $section)
newStatus(... $args)
Yields the result of the status wrapper callback on either:
static normalizeStoragePath( $storagePath)
Normalize a storage path by cleaning up directory separators.
int $concurrency
How many operations can be done in parallel.
getLocalReference(array $params)
Returns a file system file, identical in content to the file at a storage path.
static attempt(array $performOps, array $opts)
Attempt to perform a series of file operations.
Definition: FileOpBatch.php:54
static encode( $value, $pretty=false, $escaping=0)
Returns the JSON representation of a value.
Definition: FormatJson.php:96
Functions related to the output of file content.
static send404Message( $fname, $flags=0)
Send out a standard 404 message for a file.
const LOCK_EX
Definition: LockManager.php:70
const LOCK_UW
Definition: LockManager.php:69
Handles a simple LRU key/value map with a maximum number of entries.
Definition: MapCacheLRU.php:36
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Definition: StatusValue.php:46
static newEmpty()
Get an instance that wraps EmptyBagOStuff.
$content
Definition: router.php:76
$mime
Definition: router.php:60