MediaWiki  master
FileBackendStore.php
Go to the documentation of this file.
1 <?php
26 
40 abstract class FileBackendStore extends FileBackend {
42  protected $memCache;
44  protected $srvCache;
46  protected $cheapCache;
48  protected $expensiveCache;
49 
51  protected $shardViaHashLevels = [];
52 
54  protected $mimeCallback;
55 
56  protected $maxFileSize = 4294967296; // integer bytes (4GiB)
57 
58  const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries
59  const CACHE_CHEAP_SIZE = 500; // integer; max entries in "cheap cache"
60  const CACHE_EXPENSIVE_SIZE = 5; // integer; max entries in "expensive cache"
61 
63  protected static $RES_ABSENT = false;
65  protected static $RES_ERROR = null;
66 
68  protected static $ABSENT_NORMAL = 'FNE-N';
70  protected static $ABSENT_LATEST = 'FNE-L';
71 
83  public function __construct( array $config ) {
84  parent::__construct( $config );
85  $this->mimeCallback = $config['mimeCallback'] ?? null;
86  $this->srvCache = new EmptyBagOStuff(); // disabled by default
87  $this->memCache = WANObjectCache::newEmpty(); // disabled by default
88  $this->cheapCache = new MapCacheLRU( self::CACHE_CHEAP_SIZE );
89  $this->expensiveCache = new MapCacheLRU( self::CACHE_EXPENSIVE_SIZE );
90  }
91 
99  final public function maxFileSizeInternal() {
100  return $this->maxFileSize;
101  }
102 
113  abstract public function isPathUsableInternal( $storagePath );
114 
133  final public function createInternal( array $params ) {
135  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
136 
137  if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
138  $status = $this->newStatus( 'backend-fail-maxsize',
139  $params['dst'], $this->maxFileSizeInternal() );
140  } else {
141  $status = $this->doCreateInternal( $params );
142  $this->clearCache( [ $params['dst'] ] );
143  if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
144  $this->deleteFileCache( $params['dst'] ); // persistent cache
145  }
146  }
147 
148  return $status;
149  }
150 
156  abstract protected function doCreateInternal( array $params );
157 
176  final public function storeInternal( array $params ) {
178  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
179 
180  if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
181  $status = $this->newStatus( 'backend-fail-maxsize',
182  $params['dst'], $this->maxFileSizeInternal() );
183  } else {
184  $status = $this->doStoreInternal( $params );
185  $this->clearCache( [ $params['dst'] ] );
186  if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
187  $this->deleteFileCache( $params['dst'] ); // persistent cache
188  }
189  }
190 
191  return $status;
192  }
193 
199  abstract protected function doStoreInternal( array $params );
200 
220  final public function copyInternal( array $params ) {
222  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
223 
224  $status = $this->doCopyInternal( $params );
225  $this->clearCache( [ $params['dst'] ] );
226  if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
227  $this->deleteFileCache( $params['dst'] ); // persistent cache
228  }
229 
230  return $status;
231  }
232 
238  abstract protected function doCopyInternal( array $params );
239 
254  final public function deleteInternal( array $params ) {
256  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
257 
258  $status = $this->doDeleteInternal( $params );
259  $this->clearCache( [ $params['src'] ] );
260  $this->deleteFileCache( $params['src'] ); // persistent cache
261  return $status;
262  }
263 
269  abstract protected function doDeleteInternal( array $params );
270 
290  final public function moveInternal( array $params ) {
292  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
293 
294  $status = $this->doMoveInternal( $params );
295  $this->clearCache( [ $params['src'], $params['dst'] ] );
296  $this->deleteFileCache( $params['src'] ); // persistent cache
297  if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
298  $this->deleteFileCache( $params['dst'] ); // persistent cache
299  }
300 
301  return $status;
302  }
303 
309  protected function doMoveInternal( array $params ) {
310  unset( $params['async'] ); // two steps, won't work here :)
311  $nsrc = FileBackend::normalizeStoragePath( $params['src'] );
312  $ndst = FileBackend::normalizeStoragePath( $params['dst'] );
313  // Copy source to dest
314  $status = $this->copyInternal( $params );
315  if ( $nsrc !== $ndst && $status->isOK() ) {
316  // Delete source (only fails due to races or network problems)
317  $status->merge( $this->deleteInternal( [ 'src' => $params['src'] ] ) );
318  $status->setResult( true, $status->value ); // ignore delete() errors
319  }
320 
321  return $status;
322  }
323 
338  final public function describeInternal( array $params ) {
340  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
341 
342  if ( count( $params['headers'] ) ) {
343  $status = $this->doDescribeInternal( $params );
344  $this->clearCache( [ $params['src'] ] );
345  $this->deleteFileCache( $params['src'] ); // persistent cache
346  } else {
347  $status = $this->newStatus(); // nothing to do
348  }
349 
350  return $status;
351  }
352 
358  protected function doDescribeInternal( array $params ) {
359  return $this->newStatus();
360  }
361 
369  final public function nullInternal( array $params ) {
370  return $this->newStatus();
371  }
372 
373  final public function concatenate( array $params ) {
375  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
376  $status = $this->newStatus();
377 
378  // Try to lock the source files for the scope of this function
380  $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
381  if ( $status->isOK() ) {
382  // Actually do the file concatenation...
383  $start_time = microtime( true );
384  $status->merge( $this->doConcatenate( $params ) );
385  $sec = microtime( true ) - $start_time;
386  if ( !$status->isOK() ) {
387  $this->logger->error( static::class . "-{$this->name}" .
388  " failed to concatenate " . count( $params['srcs'] ) . " file(s) [$sec sec]" );
389  }
390  }
391 
392  return $status;
393  }
394 
400  protected function doConcatenate( array $params ) {
401  $status = $this->newStatus();
402  $tmpPath = $params['dst']; // convenience
403  unset( $params['latest'] ); // sanity
404 
405  // Check that the specified temp file is valid...
406  AtEase::suppressWarnings();
407  $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 );
408  AtEase::restoreWarnings();
409  if ( !$ok ) { // not present or not empty
410  $status->fatal( 'backend-fail-opentemp', $tmpPath );
411 
412  return $status;
413  }
414 
415  // Get local FS versions of the chunks needed for the concatenation...
416  $fsFiles = $this->getLocalReferenceMulti( $params );
417  foreach ( $fsFiles as $path => &$fsFile ) {
418  if ( !$fsFile ) { // chunk failed to download?
419  $fsFile = $this->getLocalReference( [ 'src' => $path ] );
420  if ( !$fsFile ) { // retry failed?
421  $status->fatal( 'backend-fail-read', $path );
422 
423  return $status;
424  }
425  }
426  }
427  unset( $fsFile ); // unset reference so we can reuse $fsFile
428 
429  // Get a handle for the destination temp file
430  $tmpHandle = fopen( $tmpPath, 'ab' );
431  if ( $tmpHandle === false ) {
432  $status->fatal( 'backend-fail-opentemp', $tmpPath );
433 
434  return $status;
435  }
436 
437  // Build up the temp file using the source chunks (in order)...
438  foreach ( $fsFiles as $virtualSource => $fsFile ) {
439  // Get a handle to the local FS version
440  $sourceHandle = fopen( $fsFile->getPath(), 'rb' );
441  if ( $sourceHandle === false ) {
442  fclose( $tmpHandle );
443  $status->fatal( 'backend-fail-read', $virtualSource );
444 
445  return $status;
446  }
447  // Append chunk to file (pass chunk size to avoid magic quotes)
448  if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
449  fclose( $sourceHandle );
450  fclose( $tmpHandle );
451  $status->fatal( 'backend-fail-writetemp', $tmpPath );
452 
453  return $status;
454  }
455  fclose( $sourceHandle );
456  }
457  if ( !fclose( $tmpHandle ) ) {
458  $status->fatal( 'backend-fail-closetemp', $tmpPath );
459 
460  return $status;
461  }
462 
463  clearstatcache(); // temp file changed
464 
465  return $status;
466  }
467 
468  final protected function doPrepare( array $params ) {
470  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
471  $status = $this->newStatus();
472 
473  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
474  if ( $dir === null ) {
475  $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
476 
477  return $status; // invalid storage path
478  }
479 
480  if ( $shard !== null ) { // confined to a single container/shard
481  $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
482  } else { // directory is on several shards
483  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
484  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
485  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
486  $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
487  }
488  }
489 
490  return $status;
491  }
492 
500  protected function doPrepareInternal( $container, $dir, array $params ) {
501  return $this->newStatus();
502  }
503 
504  final protected function doSecure( array $params ) {
506  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
507  $status = $this->newStatus();
508 
509  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
510  if ( $dir === null ) {
511  $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
512 
513  return $status; // invalid storage path
514  }
515 
516  if ( $shard !== null ) { // confined to a single container/shard
517  $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
518  } else { // directory is on several shards
519  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
520  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
521  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
522  $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
523  }
524  }
525 
526  return $status;
527  }
528 
536  protected function doSecureInternal( $container, $dir, array $params ) {
537  return $this->newStatus();
538  }
539 
540  final protected function doPublish( array $params ) {
542  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
543  $status = $this->newStatus();
544 
545  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
546  if ( $dir === null ) {
547  $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
548 
549  return $status; // invalid storage path
550  }
551 
552  if ( $shard !== null ) { // confined to a single container/shard
553  $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
554  } else { // directory is on several shards
555  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
556  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
557  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
558  $status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) );
559  }
560  }
561 
562  return $status;
563  }
564 
572  protected function doPublishInternal( $container, $dir, array $params ) {
573  return $this->newStatus();
574  }
575 
576  final protected function doClean( array $params ) {
578  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
579  $status = $this->newStatus();
580 
581  // Recursive: first delete all empty subdirs recursively
582  if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) {
583  $subDirsRel = $this->getTopDirectoryList( [ 'dir' => $params['dir'] ] );
584  if ( $subDirsRel !== null ) { // no errors
585  foreach ( $subDirsRel as $subDirRel ) {
586  $subDir = $params['dir'] . "/{$subDirRel}"; // full path
587  $status->merge( $this->doClean( [ 'dir' => $subDir ] + $params ) );
588  }
589  unset( $subDirsRel ); // free directory for rmdir() on Windows (for FS backends)
590  }
591  }
592 
593  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
594  if ( $dir === null ) {
595  $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
596 
597  return $status; // invalid storage path
598  }
599 
600  // Attempt to lock this directory...
601  $filesLockEx = [ $params['dir'] ];
603  $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
604  if ( !$status->isOK() ) {
605  return $status; // abort
606  }
607 
608  if ( $shard !== null ) { // confined to a single container/shard
609  $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
610  $this->deleteContainerCache( $fullCont ); // purge cache
611  } else { // directory is on several shards
612  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
613  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
614  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
615  $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
616  $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache
617  }
618  }
619 
620  return $status;
621  }
622 
630  protected function doCleanInternal( $container, $dir, array $params ) {
631  return $this->newStatus();
632  }
633 
634  final public function fileExists( array $params ) {
636  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
637 
638  $stat = $this->getFileStat( $params );
639  if ( is_array( $stat ) ) {
640  return true;
641  }
642 
643  return ( $stat === self::$RES_ABSENT ) ? false : self::EXISTENCE_ERROR;
644  }
645 
646  final public function getFileTimestamp( array $params ) {
648  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
649 
650  $stat = $this->getFileStat( $params );
651  if ( is_array( $stat ) ) {
652  return $stat['mtime'];
653  }
654 
655  return self::TIMESTAMP_FAIL; // all failure cases
656  }
657 
658  final public function getFileSize( array $params ) {
660  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
661 
662  $stat = $this->getFileStat( $params );
663  if ( is_array( $stat ) ) {
664  return $stat['size'];
665  }
666 
667  return self::SIZE_FAIL; // all failure cases
668  }
669 
670  final public function getFileStat( array $params ) {
672  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
673 
674  $path = self::normalizeStoragePath( $params['src'] );
675  if ( $path === null ) {
676  return self::STAT_ERROR; // invalid storage path
677  }
678 
679  // Whether to bypass cache except for process cache entries loaded directly from
680  // high consistency backend queries (caller handles any cache flushing and locking)
681  $latest = !empty( $params['latest'] );
682  // Whether to ignore cache entries missing the SHA-1 field for existing files
683  $requireSHA1 = !empty( $params['requireSHA1'] );
684 
685  $stat = $this->cheapCache->getField( $path, 'stat', self::CACHE_TTL );
686  // Load the persistent stat cache into process cache if needed
687  if ( !$latest ) {
688  if (
689  // File stat is not in process cache
690  $stat === null ||
691  // Key/value store backends might opportunistically set file stat process
692  // cache entries from object listings that do not include the SHA-1. In that
693  // case, loading the persistent stat cache will likely yield the SHA-1.
694  ( $requireSHA1 && is_array( $stat ) && !isset( $stat['sha1'] ) )
695  ) {
696  $this->primeFileCache( [ $path ] );
697  // Get any newly process-cached entry
698  $stat = $this->cheapCache->getField( $path, 'stat', self::CACHE_TTL );
699  }
700  }
701 
702  if ( is_array( $stat ) ) {
703  if (
704  ( !$latest || $stat['latest'] ) &&
705  ( !$requireSHA1 || isset( $stat['sha1'] ) )
706  ) {
707  return $stat;
708  }
709  } elseif ( $stat === self::$ABSENT_LATEST ) {
710  return self::STAT_ABSENT;
711  } elseif ( $stat === self::$ABSENT_NORMAL ) {
712  if ( !$latest ) {
713  return self::STAT_ABSENT;
714  }
715  }
716 
717  // Load the file stat from the backend and update caches
718  $stat = $this->doGetFileStat( $params );
719  $this->ingestFreshFileStats( [ $path => $stat ], $latest );
720 
721  if ( is_array( $stat ) ) {
722  return $stat;
723  }
724 
725  return ( $stat === self::$RES_ERROR ) ? self::STAT_ERROR : self::STAT_ABSENT;
726  }
727 
735  final protected function ingestFreshFileStats( array $stats, $latest ) {
736  $success = true;
737 
738  foreach ( $stats as $path => $stat ) {
739  if ( is_array( $stat ) ) {
740  // Strongly consistent backends might automatically set this flag
741  $stat['latest'] = $stat['latest'] ?? $latest;
742 
743  $this->cheapCache->setField( $path, 'stat', $stat );
744  if ( isset( $stat['sha1'] ) ) {
745  // Some backends store the SHA-1 hash as metadata
746  $this->cheapCache->setField(
747  $path,
748  'sha1',
749  [ 'hash' => $stat['sha1'], 'latest' => $latest ]
750  );
751  }
752  if ( isset( $stat['xattr'] ) ) {
753  // Some backends store custom headers/metadata
754  $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] );
755  $this->cheapCache->setField(
756  $path,
757  'xattr',
758  [ 'map' => $stat['xattr'], 'latest' => $latest ]
759  );
760  }
761  // Update persistent cache (@TODO: set all entries in one batch)
762  $this->setFileCache( $path, $stat );
763  } elseif ( $stat === self::$RES_ABSENT ) {
764  $this->cheapCache->setField(
765  $path,
766  'stat',
767  $latest ? self::$ABSENT_LATEST : self::$ABSENT_NORMAL
768  );
769  $this->cheapCache->setField(
770  $path,
771  'xattr',
772  [ 'map' => self::XATTRS_FAIL, 'latest' => $latest ]
773  );
774  $this->cheapCache->setField(
775  $path,
776  'sha1',
777  [ 'hash' => self::SHA1_FAIL, 'latest' => $latest ]
778  );
779  $this->logger->debug(
780  __METHOD__ . ': File {path} does not exist',
781  [ 'path' => $path ]
782  );
783  } else {
784  $success = false;
785  $this->logger->error(
786  __METHOD__ . ': Could not stat file {path}',
787  [ 'path' => $path ]
788  );
789  }
790  }
791 
792  return $success;
793  }
794 
799  abstract protected function doGetFileStat( array $params );
800 
801  public function getFileContentsMulti( array $params ) {
803  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
804 
805  $params = $this->setConcurrencyFlags( $params );
806  $contents = $this->doGetFileContentsMulti( $params );
807  foreach ( $contents as $path => $content ) {
808  if ( !is_string( $content ) ) {
809  $contents[$path] = self::CONTENT_FAIL; // used for all failure cases
810  }
811  }
812 
813  return $contents;
814  }
815 
821  protected function doGetFileContentsMulti( array $params ) {
822  $contents = [];
823  foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
824  if ( $fsFile instanceof FSFile ) {
825  AtEase::suppressWarnings();
826  $content = file_get_contents( $fsFile->getPath() );
827  AtEase::restoreWarnings();
828  $contents[$path] = is_string( $content ) ? $content : self::$RES_ERROR;
829  } elseif ( $fsFile === self::$RES_ABSENT ) {
830  $contents[$path] = self::$RES_ABSENT;
831  } else {
832  $contents[$path] = self::$RES_ERROR;
833  }
834  }
835 
836  return $contents;
837  }
838 
839  final public function getFileXAttributes( array $params ) {
841  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
842 
843  $path = self::normalizeStoragePath( $params['src'] );
844  if ( $path === null ) {
845  return self::XATTRS_FAIL; // invalid storage path
846  }
847  $latest = !empty( $params['latest'] ); // use latest data?
848  if ( $this->cheapCache->hasField( $path, 'xattr', self::CACHE_TTL ) ) {
849  $stat = $this->cheapCache->getField( $path, 'xattr' );
850  // If we want the latest data, check that this cached
851  // value was in fact fetched with the latest available data.
852  if ( !$latest || $stat['latest'] ) {
853  return $stat['map'];
854  }
855  }
856  $fields = $this->doGetFileXAttributes( $params );
857  if ( is_array( $fields ) ) {
858  $fields = self::normalizeXAttributes( $fields );
859  $this->cheapCache->setField(
860  $path,
861  'xattr',
862  [ 'map' => $fields, 'latest' => $latest ]
863  );
864  } elseif ( $fields === self::$RES_ABSENT ) {
865  $this->cheapCache->setField(
866  $path,
867  'xattr',
868  [ 'map' => self::XATTRS_FAIL, 'latest' => $latest ]
869  );
870  } else {
871  $fields = self::XATTRS_FAIL; // used for all failure cases
872  }
873 
874  return $fields;
875  }
876 
882  protected function doGetFileXAttributes( array $params ) {
883  return [ 'headers' => [], 'metadata' => [] ]; // not supported
884  }
885 
886  final public function getFileSha1Base36( array $params ) {
888  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
889 
890  $path = self::normalizeStoragePath( $params['src'] );
891  if ( $path === null ) {
892  return self::SHA1_FAIL; // invalid storage path
893  }
894  $latest = !empty( $params['latest'] ); // use latest data?
895  if ( $this->cheapCache->hasField( $path, 'sha1', self::CACHE_TTL ) ) {
896  $stat = $this->cheapCache->getField( $path, 'sha1' );
897  // If we want the latest data, check that this cached
898  // value was in fact fetched with the latest available data.
899  if ( !$latest || $stat['latest'] ) {
900  return $stat['hash'];
901  }
902  }
903  $sha1 = $this->doGetFileSha1Base36( $params );
904  if ( is_string( $sha1 ) ) {
905  $this->cheapCache->setField(
906  $path,
907  'sha1',
908  [ 'hash' => $sha1, 'latest' => $latest ]
909  );
910  } elseif ( $sha1 === self::$RES_ABSENT ) {
911  $this->cheapCache->setField(
912  $path,
913  'sha1',
914  [ 'hash' => self::SHA1_FAIL, 'latest' => $latest ]
915  );
916  } else {
917  $sha1 = self::SHA1_FAIL; // used for all failure cases
918  }
919 
920  return $sha1;
921  }
922 
928  protected function doGetFileSha1Base36( array $params ) {
929  $fsFile = $this->getLocalReference( $params );
930  if ( $fsFile instanceof FSFile ) {
931  $sha1 = $fsFile->getSha1Base36();
932 
933  return is_string( $sha1 ) ? $sha1 : self::$RES_ERROR;
934  }
935 
936  return ( $fsFile === self::$RES_ERROR ) ? self::$RES_ERROR : self::$RES_ABSENT;
937  }
938 
939  final public function getFileProps( array $params ) {
941  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
942 
943  $fsFile = $this->getLocalReference( $params );
944 
945  return $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
946  }
947 
948  final public function getLocalReferenceMulti( array $params ) {
950  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
951 
952  $params = $this->setConcurrencyFlags( $params );
953 
954  $fsFiles = []; // (path => FSFile)
955  $latest = !empty( $params['latest'] ); // use latest data?
956  // Reuse any files already in process cache...
957  foreach ( $params['srcs'] as $src ) {
958  $path = self::normalizeStoragePath( $src );
959  if ( $path === null ) {
960  $fsFiles[$src] = null; // invalid storage path
961  } elseif ( $this->expensiveCache->hasField( $path, 'localRef' ) ) {
962  $val = $this->expensiveCache->getField( $path, 'localRef' );
963  // If we want the latest data, check that this cached
964  // value was in fact fetched with the latest available data.
965  if ( !$latest || $val['latest'] ) {
966  $fsFiles[$src] = $val['object'];
967  }
968  }
969  }
970  // Fetch local references of any remaning files...
971  $params['srcs'] = array_diff( $params['srcs'], array_keys( $fsFiles ) );
972  foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
973  if ( $fsFile instanceof FSFile ) {
974  $fsFiles[$path] = $fsFile;
975  $this->expensiveCache->setField(
976  $path,
977  'localRef',
978  [ 'object' => $fsFile, 'latest' => $latest ]
979  );
980  } else {
981  $fsFiles[$path] = null; // used for all failure cases
982  }
983  }
984 
985  return $fsFiles;
986  }
987 
993  protected function doGetLocalReferenceMulti( array $params ) {
994  return $this->doGetLocalCopyMulti( $params );
995  }
996 
997  final public function getLocalCopyMulti( array $params ) {
999  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1000 
1001  $params = $this->setConcurrencyFlags( $params );
1002  $tmpFiles = $this->doGetLocalCopyMulti( $params );
1003  foreach ( $tmpFiles as $path => $tmpFile ) {
1004  if ( !$tmpFile ) {
1005  $tmpFiles[$path] = null; // used for all failure cases
1006  }
1007  }
1008 
1009  return $tmpFiles;
1010  }
1011 
1017  abstract protected function doGetLocalCopyMulti( array $params );
1018 
1024  public function getFileHttpUrl( array $params ) {
1025  return self::TEMPURL_ERROR; // not supported
1026  }
1027 
1028  final public function streamFile( array $params ) {
1030  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1031  $status = $this->newStatus();
1032 
1033  // Always set some fields for subclass convenience
1034  $params['options'] = $params['options'] ?? [];
1035  $params['headers'] = $params['headers'] ?? [];
1036 
1037  // Don't stream it out as text/html if there was a PHP error
1038  if ( ( empty( $params['headless'] ) || $params['headers'] ) && headers_sent() ) {
1039  print "Headers already sent, terminating.\n";
1040  $status->fatal( 'backend-fail-stream', $params['src'] );
1041  return $status;
1042  }
1043 
1044  $status->merge( $this->doStreamFile( $params ) );
1045 
1046  return $status;
1047  }
1048 
1054  protected function doStreamFile( array $params ) {
1055  $status = $this->newStatus();
1056 
1057  $flags = 0;
1058  $flags |= !empty( $params['headless'] ) ? HTTPFileStreamer::STREAM_HEADLESS : 0;
1059  $flags |= !empty( $params['allowOB'] ) ? HTTPFileStreamer::STREAM_ALLOW_OB : 0;
1060 
1061  $fsFile = $this->getLocalReference( $params );
1062  if ( $fsFile ) {
1063  $streamer = new HTTPFileStreamer(
1064  $fsFile->getPath(),
1065  [
1066  'obResetFunc' => $this->obResetFunc,
1067  'streamMimeFunc' => $this->streamMimeFunc
1068  ]
1069  );
1070  $res = $streamer->stream( $params['headers'], true, $params['options'], $flags );
1071  } else {
1072  $res = false;
1073  HTTPFileStreamer::send404Message( $params['src'], $flags );
1074  }
1075 
1076  if ( !$res ) {
1077  $status->fatal( 'backend-fail-stream', $params['src'] );
1078  }
1079 
1080  return $status;
1081  }
1082 
1083  final public function directoryExists( array $params ) {
1084  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
1085  if ( $dir === null ) {
1086  return self::EXISTENCE_ERROR; // invalid storage path
1087  }
1088  if ( $shard !== null ) { // confined to a single container/shard
1089  return $this->doDirectoryExists( $fullCont, $dir, $params );
1090  } else { // directory is on several shards
1091  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
1092  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
1093  $res = false; // response
1094  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
1095  $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params );
1096  if ( $exists === true ) {
1097  $res = true;
1098  break; // found one!
1099  } elseif ( $exists === self::$RES_ERROR ) {
1100  $res = self::EXISTENCE_ERROR;
1101  }
1102  }
1103 
1104  return $res;
1105  }
1106  }
1107 
1116  abstract protected function doDirectoryExists( $container, $dir, array $params );
1117 
1118  final public function getDirectoryList( array $params ) {
1119  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
1120  if ( $dir === null ) {
1121  return self::EXISTENCE_ERROR; // invalid storage path
1122  }
1123  if ( $shard !== null ) {
1124  // File listing is confined to a single container/shard
1125  return $this->getDirectoryListInternal( $fullCont, $dir, $params );
1126  } else {
1127  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
1128  // File listing spans multiple containers/shards
1129  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
1130 
1131  return new FileBackendStoreShardDirIterator( $this,
1132  $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
1133  }
1134  }
1135 
1146  abstract public function getDirectoryListInternal( $container, $dir, array $params );
1147 
1148  final public function getFileList( array $params ) {
1149  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
1150  if ( $dir === null ) {
1151  return self::LIST_ERROR; // invalid storage path
1152  }
1153  if ( $shard !== null ) {
1154  // File listing is confined to a single container/shard
1155  return $this->getFileListInternal( $fullCont, $dir, $params );
1156  } else {
1157  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
1158  // File listing spans multiple containers/shards
1159  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
1160 
1161  return new FileBackendStoreShardFileIterator( $this,
1162  $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
1163  }
1164  }
1165 
1176  abstract public function getFileListInternal( $container, $dir, array $params );
1177 
1189  final public function getOperationsInternal( array $ops ) {
1190  $supportedOps = [
1191  'store' => StoreFileOp::class,
1192  'copy' => CopyFileOp::class,
1193  'move' => MoveFileOp::class,
1194  'delete' => DeleteFileOp::class,
1195  'create' => CreateFileOp::class,
1196  'describe' => DescribeFileOp::class,
1197  'null' => NullFileOp::class
1198  ];
1199 
1200  $performOps = []; // array of FileOp objects
1201  // Build up ordered array of FileOps...
1202  foreach ( $ops as $operation ) {
1203  $opName = $operation['op'];
1204  if ( isset( $supportedOps[$opName] ) ) {
1205  $class = $supportedOps[$opName];
1206  // Get params for this operation
1207  $params = $operation;
1208  // Append the FileOp class
1209  $performOps[] = new $class( $this, $params, $this->logger );
1210  } else {
1211  throw new FileBackendError( "Operation '$opName' is not supported." );
1212  }
1213  }
1214 
1215  return $performOps;
1216  }
1217 
1228  final public function getPathsToLockForOpsInternal( array $performOps ) {
1229  // Build up a list of files to lock...
1230  $paths = [ 'sh' => [], 'ex' => [] ];
1231  foreach ( $performOps as $fileOp ) {
1232  $paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() );
1233  $paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() );
1234  }
1235  // Optimization: if doing an EX lock anyway, don't also set an SH one
1236  $paths['sh'] = array_diff( $paths['sh'], $paths['ex'] );
1237  // Get a shared lock on the parent directory of each path changed
1238  $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) );
1239 
1240  return [
1241  LockManager::LOCK_UW => $paths['sh'],
1242  LockManager::LOCK_EX => $paths['ex']
1243  ];
1244  }
1245 
1246  public function getScopedLocksForOps( array $ops, StatusValue $status ) {
1247  $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) );
1248 
1249  return $this->getScopedFileLocks( $paths, 'mixed', $status );
1250  }
1251 
1252  final protected function doOperationsInternal( array $ops, array $opts ) {
1254  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1255  $status = $this->newStatus();
1256 
1257  // Fix up custom header name/value pairs...
1258  $ops = array_map( [ $this, 'sanitizeOpHeaders' ], $ops );
1259 
1260  // Build up a list of FileOps...
1261  $performOps = $this->getOperationsInternal( $ops );
1262 
1263  // Acquire any locks as needed...
1264  if ( empty( $opts['nonLocking'] ) ) {
1265  // Build up a list of files to lock...
1266  $paths = $this->getPathsToLockForOpsInternal( $performOps );
1267  // Try to lock those files for the scope of this function...
1269  $scopeLock = $this->getScopedFileLocks( $paths, 'mixed', $status );
1270  if ( !$status->isOK() ) {
1271  return $status; // abort
1272  }
1273  }
1274 
1275  // Clear any file cache entries (after locks acquired)
1276  if ( empty( $opts['preserveCache'] ) ) {
1277  $this->clearCache();
1278  }
1279 
1280  // Build the list of paths involved
1281  $paths = [];
1282  foreach ( $performOps as $performOp ) {
1283  $paths = array_merge( $paths, $performOp->storagePathsRead() );
1284  $paths = array_merge( $paths, $performOp->storagePathsChanged() );
1285  }
1286 
1287  // Enlarge the cache to fit the stat entries of these files
1288  $this->cheapCache->setMaxSize( max( 2 * count( $paths ), self::CACHE_CHEAP_SIZE ) );
1289 
1290  // Load from the persistent container caches
1291  $this->primeContainerCache( $paths );
1292  // Get the latest stat info for all the files (having locked them)
1293  $ok = $this->preloadFileStat( [ 'srcs' => $paths, 'latest' => true ] );
1294 
1295  if ( $ok ) {
1296  // Actually attempt the operation batch...
1297  $opts = $this->setConcurrencyFlags( $opts );
1298  $subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal );
1299  } else {
1300  // If we could not even stat some files, then bail out...
1301  $subStatus = $this->newStatus( 'backend-fail-internal', $this->name );
1302  foreach ( $ops as $i => $op ) { // mark each op as failed
1303  $subStatus->success[$i] = false;
1304  ++$subStatus->failCount;
1305  }
1306  $this->logger->error( static::class . "-{$this->name} " .
1307  " stat failure; aborted operations: " . FormatJson::encode( $ops ) );
1308  }
1309 
1310  // Merge errors into StatusValue fields
1311  $status->merge( $subStatus );
1312  $status->success = $subStatus->success; // not done in merge()
1313 
1314  // Shrink the stat cache back to normal size
1315  $this->cheapCache->setMaxSize( self::CACHE_CHEAP_SIZE );
1316 
1317  return $status;
1318  }
1319 
1320  final protected function doQuickOperationsInternal( array $ops ) {
1322  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1323  $status = $this->newStatus();
1324 
1325  // Fix up custom header name/value pairs...
1326  $ops = array_map( [ $this, 'sanitizeOpHeaders' ], $ops );
1327 
1328  // Clear any file cache entries
1329  $this->clearCache();
1330 
1331  $supportedOps = [ 'create', 'store', 'copy', 'move', 'delete', 'describe', 'null' ];
1332  // Parallel ops may be disabled in config due to dependencies (e.g. needing popen())
1333  $async = ( $this->parallelize === 'implicit' && count( $ops ) > 1 );
1334  $maxConcurrency = $this->concurrency; // throttle
1336  $statuses = []; // array of (index => StatusValue)
1337  $fileOpHandles = []; // list of (index => handle) arrays
1338  $curFileOpHandles = []; // current handle batch
1339  // Perform the sync-only ops and build up op handles for the async ops...
1340  foreach ( $ops as $index => $params ) {
1341  if ( !in_array( $params['op'], $supportedOps ) ) {
1342  throw new FileBackendError( "Operation '{$params['op']}' is not supported." );
1343  }
1344  $method = $params['op'] . 'Internal'; // e.g. "storeInternal"
1345  $subStatus = $this->$method( [ 'async' => $async ] + $params );
1346  if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { // async
1347  if ( count( $curFileOpHandles ) >= $maxConcurrency ) {
1348  $fileOpHandles[] = $curFileOpHandles; // push this batch
1349  $curFileOpHandles = [];
1350  }
1351  $curFileOpHandles[$index] = $subStatus->value; // keep index
1352  } else { // error or completed
1353  $statuses[$index] = $subStatus; // keep index
1354  }
1355  }
1356  if ( count( $curFileOpHandles ) ) {
1357  $fileOpHandles[] = $curFileOpHandles; // last batch
1358  }
1359  // Do all the async ops that can be done concurrently...
1360  foreach ( $fileOpHandles as $fileHandleBatch ) {
1361  $statuses = $statuses + $this->executeOpHandlesInternal( $fileHandleBatch );
1362  }
1363  // Marshall and merge all the responses...
1364  foreach ( $statuses as $index => $subStatus ) {
1365  $status->merge( $subStatus );
1366  if ( $subStatus->isOK() ) {
1367  $status->success[$index] = true;
1368  ++$status->successCount;
1369  } else {
1370  $status->success[$index] = false;
1371  ++$status->failCount;
1372  }
1373  }
1374 
1375  return $status;
1376  }
1377 
1387  final public function executeOpHandlesInternal( array $fileOpHandles ) {
1389  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1390 
1391  foreach ( $fileOpHandles as $fileOpHandle ) {
1392  if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) {
1393  throw new InvalidArgumentException( "Expected FileBackendStoreOpHandle object." );
1394  } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) {
1395  throw new InvalidArgumentException( "Expected handle for this file backend." );
1396  }
1397  }
1398 
1399  $statuses = $this->doExecuteOpHandlesInternal( $fileOpHandles );
1400  foreach ( $fileOpHandles as $fileOpHandle ) {
1401  $fileOpHandle->closeResources();
1402  }
1403 
1404  return $statuses;
1405  }
1406 
1415  protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1416  if ( count( $fileOpHandles ) ) {
1417  throw new FileBackendError( "Backend does not support asynchronous operations." );
1418  }
1419 
1420  return [];
1421  }
1422 
1434  protected function sanitizeOpHeaders( array $op ) {
1435  static $longs = [ 'content-disposition' ];
1436 
1437  if ( isset( $op['headers'] ) ) { // op sets HTTP headers
1438  $newHeaders = [];
1439  foreach ( $op['headers'] as $name => $value ) {
1440  $name = strtolower( $name );
1441  $maxHVLen = in_array( $name, $longs ) ? INF : 255;
1442  if ( strlen( $name ) > 255 || strlen( $value ) > $maxHVLen ) {
1443  $this->logger->error( "Header '{header}' is too long.", [
1444  'filebackend' => $this->name,
1445  'header' => "$name: $value",
1446  ] );
1447  } else {
1448  $newHeaders[$name] = strlen( $value ) ? $value : ''; // null/false => ""
1449  }
1450  }
1451  $op['headers'] = $newHeaders;
1452  }
1453 
1454  return $op;
1455  }
1456 
1457  final public function preloadCache( array $paths ) {
1458  $fullConts = []; // full container names
1459  foreach ( $paths as $path ) {
1460  list( $fullCont, , ) = $this->resolveStoragePath( $path );
1461  $fullConts[] = $fullCont;
1462  }
1463  // Load from the persistent file and container caches
1464  $this->primeContainerCache( $fullConts );
1465  $this->primeFileCache( $paths );
1466  }
1467 
1468  final public function clearCache( array $paths = null ) {
1469  if ( is_array( $paths ) ) {
1470  $paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
1471  $paths = array_filter( $paths, 'strlen' ); // remove nulls
1472  }
1473  if ( $paths === null ) {
1474  $this->cheapCache->clear();
1475  $this->expensiveCache->clear();
1476  } else {
1477  foreach ( $paths as $path ) {
1478  $this->cheapCache->clear( $path );
1479  $this->expensiveCache->clear( $path );
1480  }
1481  }
1482  $this->doClearCache( $paths );
1483  }
1484 
1492  protected function doClearCache( array $paths = null ) {
1493  }
1494 
1495  final public function preloadFileStat( array $params ) {
1497  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1498 
1499  $params['concurrency'] = ( $this->parallelize !== 'off' ) ? $this->concurrency : 1;
1500  $stats = $this->doGetFileStatMulti( $params );
1501  if ( $stats === null ) {
1502  return true; // not supported
1503  }
1504 
1505  // Whether this queried the backend in high consistency mode
1506  $latest = !empty( $params['latest'] );
1507 
1508  return $this->ingestFreshFileStats( $stats, $latest );
1509  }
1510 
1522  protected function doGetFileStatMulti( array $params ) {
1523  return null; // not supported
1524  }
1525 
1533  abstract protected function directoriesAreVirtual();
1534 
1545  final protected static function isValidShortContainerName( $container ) {
1546  // Suffixes like '.xxx' (hex shard chars) or '.seg' (file segments)
1547  // might be used by subclasses. Reserve the dot character for sanity.
1548  // The only way dots end up in containers (e.g. resolveStoragePath)
1549  // is due to the wikiId container prefix or the above suffixes.
1550  return self::isValidContainerName( $container ) && !preg_match( '/[.]/', $container );
1551  }
1552 
1562  final protected static function isValidContainerName( $container ) {
1563  // This accounts for NTFS, Swift, and Ceph restrictions
1564  // and disallows directory separators or traversal characters.
1565  // Note that matching strings URL encode to the same string;
1566  // in Swift/Ceph, the length restriction is *after* URL encoding.
1567  return (bool)preg_match( '/^[a-z0-9][a-z0-9-_.]{0,199}$/i', $container );
1568  }
1569 
1583  final protected function resolveStoragePath( $storagePath ) {
1584  list( $backend, $shortCont, $relPath ) = self::splitStoragePath( $storagePath );
1585  if ( $backend === $this->name ) { // must be for this backend
1586  $relPath = self::normalizeContainerPath( $relPath );
1587  if ( $relPath !== null && self::isValidShortContainerName( $shortCont ) ) {
1588  // Get shard for the normalized path if this container is sharded
1589  $cShard = $this->getContainerShard( $shortCont, $relPath );
1590  // Validate and sanitize the relative path (backend-specific)
1591  $relPath = $this->resolveContainerPath( $shortCont, $relPath );
1592  if ( $relPath !== null ) {
1593  // Prepend any domain ID prefix to the container name
1594  $container = $this->fullContainerName( $shortCont );
1595  if ( self::isValidContainerName( $container ) ) {
1596  // Validate and sanitize the container name (backend-specific)
1597  $container = $this->resolveContainerName( "{$container}{$cShard}" );
1598  if ( $container !== null ) {
1599  return [ $container, $relPath, $cShard ];
1600  }
1601  }
1602  }
1603  }
1604  }
1605 
1606  return [ null, null, null ];
1607  }
1608 
1624  final protected function resolveStoragePathReal( $storagePath ) {
1625  list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
1626  if ( $cShard !== null && substr( $relPath, -1 ) !== '/' ) {
1627  return [ $container, $relPath ];
1628  }
1629 
1630  return [ null, null ];
1631  }
1632 
1641  final protected function getContainerShard( $container, $relPath ) {
1642  list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container );
1643  if ( $levels == 1 || $levels == 2 ) {
1644  // Hash characters are either base 16 or 36
1645  $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]';
1646  // Get a regex that represents the shard portion of paths.
1647  // The concatenation of the captures gives us the shard.
1648  if ( $levels === 1 ) { // 16 or 36 shards per container
1649  $hashDirRegex = '(' . $char . ')';
1650  } else { // 256 or 1296 shards per container
1651  if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc")
1652  $hashDirRegex = $char . '/(' . $char . '{2})';
1653  } else { // short hash dir format (e.g. "a/b/c")
1654  $hashDirRegex = '(' . $char . ')/(' . $char . ')';
1655  }
1656  }
1657  // Allow certain directories to be above the hash dirs so as
1658  // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
1659  // They must be 2+ chars to avoid any hash directory ambiguity.
1660  $m = [];
1661  if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1662  return '.' . implode( '', array_slice( $m, 1 ) );
1663  }
1664 
1665  return null; // failed to match
1666  }
1667 
1668  return ''; // no sharding
1669  }
1670 
1679  final public function isSingleShardPathInternal( $storagePath ) {
1680  list( , , $shard ) = $this->resolveStoragePath( $storagePath );
1681 
1682  return ( $shard !== null );
1683  }
1684 
1693  final protected function getContainerHashLevels( $container ) {
1694  if ( isset( $this->shardViaHashLevels[$container] ) ) {
1695  $config = $this->shardViaHashLevels[$container];
1696  $hashLevels = (int)$config['levels'];
1697  if ( $hashLevels == 1 || $hashLevels == 2 ) {
1698  $hashBase = (int)$config['base'];
1699  if ( $hashBase == 16 || $hashBase == 36 ) {
1700  return [ $hashLevels, $hashBase, $config['repeat'] ];
1701  }
1702  }
1703  }
1704 
1705  return [ 0, 0, false ]; // no sharding
1706  }
1707 
1714  final protected function getContainerSuffixes( $container ) {
1715  $shards = [];
1716  list( $digits, $base ) = $this->getContainerHashLevels( $container );
1717  if ( $digits > 0 ) {
1718  $numShards = $base ** $digits;
1719  for ( $index = 0; $index < $numShards; $index++ ) {
1720  $shards[] = '.' . Wikimedia\base_convert( $index, 10, $base, $digits );
1721  }
1722  }
1723 
1724  return $shards;
1725  }
1726 
1733  final protected function fullContainerName( $container ) {
1734  if ( $this->domainId != '' ) {
1735  return "{$this->domainId}-$container";
1736  } else {
1737  return $container;
1738  }
1739  }
1740 
1749  protected function resolveContainerName( $container ) {
1750  return $container;
1751  }
1752 
1763  protected function resolveContainerPath( $container, $relStoragePath ) {
1764  return $relStoragePath;
1765  }
1766 
1773  private function containerCacheKey( $container ) {
1774  return "filebackend:{$this->name}:{$this->domainId}:container:{$container}";
1775  }
1776 
1783  final protected function setContainerCache( $container, array $val ) {
1784  $this->memCache->set( $this->containerCacheKey( $container ), $val, 14 * 86400 );
1785  }
1786 
1793  final protected function deleteContainerCache( $container ) {
1794  if ( !$this->memCache->delete( $this->containerCacheKey( $container ), 300 ) ) {
1795  $this->logger->warning( "Unable to delete stat cache for container {container}.",
1796  [ 'filebackend' => $this->name, 'container' => $container ]
1797  );
1798  }
1799  }
1800 
1808  final protected function primeContainerCache( array $items ) {
1810  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1811 
1812  $paths = []; // list of storage paths
1813  $contNames = []; // (cache key => resolved container name)
1814  // Get all the paths/containers from the items...
1815  foreach ( $items as $item ) {
1816  if ( self::isStoragePath( $item ) ) {
1817  $paths[] = $item;
1818  } elseif ( is_string( $item ) ) { // full container name
1819  $contNames[$this->containerCacheKey( $item )] = $item;
1820  }
1821  }
1822  // Get all the corresponding cache keys for paths...
1823  foreach ( $paths as $path ) {
1824  list( $fullCont, , ) = $this->resolveStoragePath( $path );
1825  if ( $fullCont !== null ) { // valid path for this backend
1826  $contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
1827  }
1828  }
1829 
1830  $contInfo = []; // (resolved container name => cache value)
1831  // Get all cache entries for these container cache keys...
1832  $values = $this->memCache->getMulti( array_keys( $contNames ) );
1833  foreach ( $values as $cacheKey => $val ) {
1834  $contInfo[$contNames[$cacheKey]] = $val;
1835  }
1836 
1837  // Populate the container process cache for the backend...
1838  $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) );
1839  }
1840 
1848  protected function doPrimeContainerCache( array $containerInfo ) {
1849  }
1850 
1857  private function fileCacheKey( $path ) {
1858  return "filebackend:{$this->name}:{$this->domainId}:file:" . sha1( $path );
1859  }
1860 
1869  final protected function setFileCache( $path, array $val ) {
1871  if ( $path === null ) {
1872  return; // invalid storage path
1873  }
1874  $mtime = ConvertibleTimestamp::convert( TS_UNIX, $val['mtime'] );
1875  $ttl = $this->memCache->adaptiveTTL( $mtime, 7 * 86400, 300, 0.1 );
1876  $key = $this->fileCacheKey( $path );
1877  // Set the cache unless it is currently salted.
1878  $this->memCache->set( $key, $val, $ttl );
1879  }
1880 
1889  final protected function deleteFileCache( $path ) {
1891  if ( $path === null ) {
1892  return; // invalid storage path
1893  }
1894  if ( !$this->memCache->delete( $this->fileCacheKey( $path ), 300 ) ) {
1895  $this->logger->warning( "Unable to delete stat cache for file {path}.",
1896  [ 'filebackend' => $this->name, 'path' => $path ]
1897  );
1898  }
1899  }
1900 
1908  final protected function primeFileCache( array $items ) {
1910  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1911 
1912  $paths = []; // list of storage paths
1913  $pathNames = []; // (cache key => storage path)
1914  // Get all the paths/containers from the items...
1915  foreach ( $items as $item ) {
1916  if ( self::isStoragePath( $item ) ) {
1917  $paths[] = FileBackend::normalizeStoragePath( $item );
1918  }
1919  }
1920  // Get rid of any paths that failed normalization
1921  $paths = array_filter( $paths, 'strlen' ); // remove nulls
1922  // Get all the corresponding cache keys for paths...
1923  foreach ( $paths as $path ) {
1924  list( , $rel, ) = $this->resolveStoragePath( $path );
1925  if ( $rel !== null ) { // valid path for this backend
1926  $pathNames[$this->fileCacheKey( $path )] = $path;
1927  }
1928  }
1929  // Get all cache entries for these file cache keys.
1930  // Note that negatives are not cached by getFileStat()/preloadFileStat().
1931  $values = $this->memCache->getMulti( array_keys( $pathNames ) );
1932  // Load all of the results into process cache...
1933  foreach ( array_filter( $values, 'is_array' ) as $cacheKey => $stat ) {
1934  $path = $pathNames[$cacheKey];
1935  // Sanity; this flag only applies to stat info loaded directly
1936  // from a high consistency backend query to the process cache
1937  unset( $stat['latest'] );
1938 
1939  $this->cheapCache->setField( $path, 'stat', $stat );
1940  if ( isset( $stat['sha1'] ) && strlen( $stat['sha1'] ) == 31 ) {
1941  // Some backends store SHA-1 as metadata
1942  $this->cheapCache->setField(
1943  $path,
1944  'sha1',
1945  [ 'hash' => $stat['sha1'], 'latest' => false ]
1946  );
1947  }
1948  if ( isset( $stat['xattr'] ) && is_array( $stat['xattr'] ) ) {
1949  // Some backends store custom headers/metadata
1950  $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] );
1951  $this->cheapCache->setField(
1952  $path,
1953  'xattr',
1954  [ 'map' => $stat['xattr'], 'latest' => false ]
1955  );
1956  }
1957  }
1958  }
1959 
1967  final protected static function normalizeXAttributes( array $xattr ) {
1968  $newXAttr = [ 'headers' => [], 'metadata' => [] ];
1969 
1970  foreach ( $xattr['headers'] as $name => $value ) {
1971  $newXAttr['headers'][strtolower( $name )] = $value;
1972  }
1973 
1974  foreach ( $xattr['metadata'] as $name => $value ) {
1975  $newXAttr['metadata'][strtolower( $name )] = $value;
1976  }
1977 
1978  return $newXAttr;
1979  }
1980 
1987  final protected function setConcurrencyFlags( array $opts ) {
1988  $opts['concurrency'] = 1; // off
1989  if ( $this->parallelize === 'implicit' ) {
1990  if ( !isset( $opts['parallelize'] ) || $opts['parallelize'] ) {
1991  $opts['concurrency'] = $this->concurrency;
1992  }
1993  } elseif ( $this->parallelize === 'explicit' ) {
1994  if ( !empty( $opts['parallelize'] ) ) {
1995  $opts['concurrency'] = $this->concurrency;
1996  }
1997  }
1998 
1999  return $opts;
2000  }
2001 
2010  protected function getContentType( $storagePath, $content, $fsPath ) {
2011  if ( $this->mimeCallback ) {
2012  return call_user_func_array( $this->mimeCallback, func_get_args() );
2013  }
2014 
2015  $mime = ( $fsPath !== null ) ? mime_content_type( $fsPath ) : false;
2016  return $mime ?: 'unknown/unknown';
2017  }
2018 }
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
doDirectoryExists( $container, $dir, array $params)
getContainerHashLevels( $container)
Get the sharding config for a container.
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
getFileHttpUrl(array $params)
setFileCache( $path, array $val)
Set the cached stat info for a file path.
doOperationsInternal(array $ops, array $opts)
MapCacheLRU $cheapCache
Map of paths to small (RAM/disk) cache items.
setConcurrencyFlags(array $opts)
Set the &#39;concurrency&#39; option from a list of operation options.
getContainerShard( $container, $relPath)
Get the container name shard suffix for a given path.
$success
moveInternal(array $params)
Move a file from one storage path to another in the backend.
concatenate(array $params)
doGetLocalReferenceMulti(array $params)
doSecureInternal( $container, $dir, array $params)
scopedProfileSection( $section)
getPathsToLockForOpsInternal(array $performOps)
Get a list of storage paths to lock for a list of operations Returns an array with LockManager::LOCK_...
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it&#39;s allowed by the backend.
doPublishInternal( $container, $dir, array $params)
static normalizeXAttributes(array $xattr)
Normalize file headers/metadata to the FileBackend::getFileXAttributes() format.
doGetFileXAttributes(array $params)
getFileStat(array $params)
setContainerCache( $container, array $val)
Set the cached info for a container.
__construct(array $config)
getName()
Get the unique backend name.
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
fileCacheKey( $path)
Get the cache key for a file path.
static encode( $value, $pretty=false, $escaping=0)
Returns the JSON representation of a value.
Definition: FormatJson.php:115
getDirectoryListInternal( $container, $dir, array $params)
Do not call this function from places outside FileBackend.
static normalizeStoragePath( $storagePath)
Normalize a storage path by cleaning up directory separators.
int $concurrency
How many operations can be done in parallel.
storeInternal(array $params)
Store a file into the backend from a file on disk.
array $shardViaHashLevels
Map of container names to sharding config.
doCreateInternal(array $params)
static newEmpty()
Get an instance that wraps EmptyBagOStuff.
WANObjectCache $memCache
streamFile(array $params)
getFileSha1Base36(array $params)
const LOCK_UW
Definition: LockManager.php:69
doClean(array $params)
static string $ABSENT_LATEST
File does not exist according to a "latest"-mode stat query.
MapCacheLRU $expensiveCache
Map of paths to large (RAM/disk) cache items.
executeOpHandlesInternal(array $fileOpHandles)
Execute a list of FileBackendStoreOpHandle handles in parallel.
getContentType( $storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
ingestFreshFileStats(array $stats, $latest)
Ingest file stat entries that just came from querying the backend (not cache)
getFileSize(array $params)
doQuickOperationsInternal(array $ops)
doPrepareInternal( $container, $dir, array $params)
static false $RES_ABSENT
Idiom for "no result due to missing file" (since 1.34)
File backend exception for checked exceptions (e.g.
Functions related to the output of file content.
createInternal(array $params)
Create a file in the backend with the given contents.
fileExists(array $params)
directoryExists(array $params)
const LOCK_EX
Definition: LockManager.php:70
doStreamFile(array $params)
getFileXAttributes(array $params)
nullInternal(array $params)
No-op file operation that does nothing.
deleteInternal(array $params)
Delete a file at the storage path.
preloadCache(array $paths)
callable $obResetFunc
doGetLocalCopyMulti(array $params)
getFileContentsMulti(array $params)
getFileTimestamp(array $params)
maxFileSizeInternal()
Get the maximum allowable file size given backend medium restrictions and basic performance constrain...
static placeholderProps()
Placeholder file properties to use for files that don&#39;t exist.
Definition: FSFile.php:150
getOperationsInternal(array $ops)
Return a list of FileOp objects from a list of operations.
callable $streamMimeFunc
clearCache(array $paths=null)
newStatus(... $args)
Yields the result of the status wrapper callback on either:
deleteFileCache( $path)
Delete the cached stat info for a file path.
callable $mimeCallback
Method to get the MIME type of files.
getFileListInternal( $container, $dir, array $params)
Do not call this function from places outside FileBackend.
static send404Message( $fname, $flags=0)
Send out a standard 404 message for a file.
static isValidShortContainerName( $container)
Check if a short container name is valid.
doPrepare(array $params)
isSingleShardPathInternal( $storagePath)
Check if a storage path maps to a single shard.
static isValidContainerName( $container)
Check if a full container name is valid.
doGetFileStatMulti(array $params)
Get file stat information (concurrently if possible) for several files.
copyInternal(array $params)
Copy a file from one storage path to another in the backend.
doGetFileStat(array $params)
string $name
Unique backend name.
Definition: FileBackend.php:96
doGetFileContentsMulti(array $params)
Base class for all backends using particular storage medium.
resolveContainerName( $container)
Resolve a container name, checking if it&#39;s allowed by the backend.
preloadFileStat(array $params)
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path in the backend. ...
getFileList(array $params)
doSecure(array $params)
static attempt(array $performOps, array $opts, FileJournal $journal)
Attempt to perform a series of file operations.
Definition: FileOpBatch.php:56
resolveStoragePath( $storagePath)
Splits a storage path into an internal container name, an internal relative file name, and a container shard suffix.
doConcatenate(array $params)
Class representing a non-directory file on the file system.
Definition: FSFile.php:32
doPublish(array $params)
deleteContainerCache( $container)
Delete the cached info for a container.
doPrimeContainerCache(array $containerInfo)
Fill the backend-specific process cache given an array of resolved container names and their correspo...
doClearCache(array $paths=null)
Clears any additional stat caches for storage paths.
describeInternal(array $params)
Alter metadata for a file at the storage path.
doStoreInternal(array $params)
Base class for all file backend classes (including multi-write backends).
Definition: FileBackend.php:94
doDeleteInternal(array $params)
getTopDirectoryList(array $params)
Same as FileBackend::getDirectoryList() except only lists directories that are immediately under the ...
doCleanInternal( $container, $dir, array $params)
$content
Definition: router.php:78
getScopedLocksForOps(array $ops, StatusValue $status)
static null $RES_ERROR
Idiom for "no result due to I/O errors" (since 1.34)
fullContainerName( $container)
Get the full container name, including the domain ID prefix.
doMoveInternal(array $params)
sanitizeOpHeaders(array $op)
Normalize and filter HTTP headers from a file operation.
doDescribeInternal(array $params)
getLocalReferenceMulti(array $params)
FileBackendStore helper class for performing asynchronous file operations.
doGetFileSha1Base36(array $params)
primeFileCache(array $items)
Do a batch lookup from cache for file stats for all paths used in a list of storage paths or FileOp o...
doCopyInternal(array $params)
getLocalReference(array $params)
Returns a file system file, identical in content to the file at a storage path.
getContainerSuffixes( $container)
Get a list of full container shard suffixes for a container.
getDirectoryList(array $params)
getLocalCopyMulti(array $params)
getFileProps(array $params)
containerCacheKey( $container)
Get the cache key for a container.
static string $ABSENT_NORMAL
File does not exist according to a normal stat query.
doExecuteOpHandlesInternal(array $fileOpHandles)