MediaWiki  1.28.0
FileBackendStore.php
Go to the documentation of this file.
1 <?php
38 abstract class FileBackendStore extends FileBackend {
40  protected $memCache;
42  protected $srvCache;
44  protected $cheapCache;
46  protected $expensiveCache;
47 
49  protected $shardViaHashLevels = [];
50 
52  protected $mimeCallback;
53 
54  protected $maxFileSize = 4294967296; // integer bytes (4GiB)
55 
56  const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries
57  const CACHE_CHEAP_SIZE = 500; // integer; max entries in "cheap cache"
58  const CACHE_EXPENSIVE_SIZE = 5; // integer; max entries in "expensive cache"
59 
71  public function __construct( array $config ) {
72  parent::__construct( $config );
73  $this->mimeCallback = isset( $config['mimeCallback'] )
74  ? $config['mimeCallback']
75  : null;
76  $this->srvCache = new EmptyBagOStuff(); // disabled by default
77  $this->memCache = WANObjectCache::newEmpty(); // disabled by default
78  $this->cheapCache = new ProcessCacheLRU( self::CACHE_CHEAP_SIZE );
79  $this->expensiveCache = new ProcessCacheLRU( self::CACHE_EXPENSIVE_SIZE );
80  }
81 
89  final public function maxFileSizeInternal() {
90  return $this->maxFileSize;
91  }
92 
102  abstract public function isPathUsableInternal( $storagePath );
103 
122  final public function createInternal( array $params ) {
123  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
124  if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
125  $status = $this->newStatus( 'backend-fail-maxsize',
126  $params['dst'], $this->maxFileSizeInternal() );
127  } else {
128  $status = $this->doCreateInternal( $params );
129  $this->clearCache( [ $params['dst'] ] );
130  if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
131  $this->deleteFileCache( $params['dst'] ); // persistent cache
132  }
133  }
134 
135  return $status;
136  }
137 
143  abstract protected function doCreateInternal( array $params );
144 
163  final public function storeInternal( array $params ) {
164  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
165  if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
166  $status = $this->newStatus( 'backend-fail-maxsize',
167  $params['dst'], $this->maxFileSizeInternal() );
168  } else {
169  $status = $this->doStoreInternal( $params );
170  $this->clearCache( [ $params['dst'] ] );
171  if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
172  $this->deleteFileCache( $params['dst'] ); // persistent cache
173  }
174  }
175 
176  return $status;
177  }
178 
184  abstract protected function doStoreInternal( array $params );
185 
205  final public function copyInternal( array $params ) {
206  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
207  $status = $this->doCopyInternal( $params );
208  $this->clearCache( [ $params['dst'] ] );
209  if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
210  $this->deleteFileCache( $params['dst'] ); // persistent cache
211  }
212 
213  return $status;
214  }
215 
221  abstract protected function doCopyInternal( array $params );
222 
237  final public function deleteInternal( array $params ) {
238  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
239  $status = $this->doDeleteInternal( $params );
240  $this->clearCache( [ $params['src'] ] );
241  $this->deleteFileCache( $params['src'] ); // persistent cache
242  return $status;
243  }
244 
250  abstract protected function doDeleteInternal( array $params );
251 
271  final public function moveInternal( array $params ) {
272  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
273  $status = $this->doMoveInternal( $params );
274  $this->clearCache( [ $params['src'], $params['dst'] ] );
275  $this->deleteFileCache( $params['src'] ); // persistent cache
276  if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
277  $this->deleteFileCache( $params['dst'] ); // persistent cache
278  }
279 
280  return $status;
281  }
282 
288  protected function doMoveInternal( array $params ) {
289  unset( $params['async'] ); // two steps, won't work here :)
290  $nsrc = FileBackend::normalizeStoragePath( $params['src'] );
291  $ndst = FileBackend::normalizeStoragePath( $params['dst'] );
292  // Copy source to dest
293  $status = $this->copyInternal( $params );
294  if ( $nsrc !== $ndst && $status->isOK() ) {
295  // Delete source (only fails due to races or network problems)
296  $status->merge( $this->deleteInternal( [ 'src' => $params['src'] ] ) );
297  $status->setResult( true, $status->value ); // ignore delete() errors
298  }
299 
300  return $status;
301  }
302 
317  final public function describeInternal( array $params ) {
318  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
319  if ( count( $params['headers'] ) ) {
320  $status = $this->doDescribeInternal( $params );
321  $this->clearCache( [ $params['src'] ] );
322  $this->deleteFileCache( $params['src'] ); // persistent cache
323  } else {
324  $status = $this->newStatus(); // nothing to do
325  }
326 
327  return $status;
328  }
329 
335  protected function doDescribeInternal( array $params ) {
336  return $this->newStatus();
337  }
338 
346  final public function nullInternal( array $params ) {
347  return $this->newStatus();
348  }
349 
350  final public function concatenate( array $params ) {
351  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
352  $status = $this->newStatus();
353 
354  // Try to lock the source files for the scope of this function
355  $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
356  if ( $status->isOK() ) {
357  // Actually do the file concatenation...
358  $start_time = microtime( true );
359  $status->merge( $this->doConcatenate( $params ) );
360  $sec = microtime( true ) - $start_time;
361  if ( !$status->isOK() ) {
362  $this->logger->error( get_class( $this ) . "-{$this->name}" .
363  " failed to concatenate " . count( $params['srcs'] ) . " file(s) [$sec sec]" );
364  }
365  }
366 
367  return $status;
368  }
369 
375  protected function doConcatenate( array $params ) {
376  $status = $this->newStatus();
377  $tmpPath = $params['dst']; // convenience
378  unset( $params['latest'] ); // sanity
379 
380  // Check that the specified temp file is valid...
381  MediaWiki\suppressWarnings();
382  $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 );
383  MediaWiki\restoreWarnings();
384  if ( !$ok ) { // not present or not empty
385  $status->fatal( 'backend-fail-opentemp', $tmpPath );
386 
387  return $status;
388  }
389 
390  // Get local FS versions of the chunks needed for the concatenation...
391  $fsFiles = $this->getLocalReferenceMulti( $params );
392  foreach ( $fsFiles as $path => &$fsFile ) {
393  if ( !$fsFile ) { // chunk failed to download?
394  $fsFile = $this->getLocalReference( [ 'src' => $path ] );
395  if ( !$fsFile ) { // retry failed?
396  $status->fatal( 'backend-fail-read', $path );
397 
398  return $status;
399  }
400  }
401  }
402  unset( $fsFile ); // unset reference so we can reuse $fsFile
403 
404  // Get a handle for the destination temp file
405  $tmpHandle = fopen( $tmpPath, 'ab' );
406  if ( $tmpHandle === false ) {
407  $status->fatal( 'backend-fail-opentemp', $tmpPath );
408 
409  return $status;
410  }
411 
412  // Build up the temp file using the source chunks (in order)...
413  foreach ( $fsFiles as $virtualSource => $fsFile ) {
414  // Get a handle to the local FS version
415  $sourceHandle = fopen( $fsFile->getPath(), 'rb' );
416  if ( $sourceHandle === false ) {
417  fclose( $tmpHandle );
418  $status->fatal( 'backend-fail-read', $virtualSource );
419 
420  return $status;
421  }
422  // Append chunk to file (pass chunk size to avoid magic quotes)
423  if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
424  fclose( $sourceHandle );
425  fclose( $tmpHandle );
426  $status->fatal( 'backend-fail-writetemp', $tmpPath );
427 
428  return $status;
429  }
430  fclose( $sourceHandle );
431  }
432  if ( !fclose( $tmpHandle ) ) {
433  $status->fatal( 'backend-fail-closetemp', $tmpPath );
434 
435  return $status;
436  }
437 
438  clearstatcache(); // temp file changed
439 
440  return $status;
441  }
442 
443  final protected function doPrepare( array $params ) {
444  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
445  $status = $this->newStatus();
446 
447  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
448  if ( $dir === null ) {
449  $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
450 
451  return $status; // invalid storage path
452  }
453 
454  if ( $shard !== null ) { // confined to a single container/shard
455  $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
456  } else { // directory is on several shards
457  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
458  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
459  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
460  $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
461  }
462  }
463 
464  return $status;
465  }
466 
474  protected function doPrepareInternal( $container, $dir, array $params ) {
475  return $this->newStatus();
476  }
477 
478  final protected function doSecure( array $params ) {
479  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
480  $status = $this->newStatus();
481 
482  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
483  if ( $dir === null ) {
484  $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
485 
486  return $status; // invalid storage path
487  }
488 
489  if ( $shard !== null ) { // confined to a single container/shard
490  $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
491  } else { // directory is on several shards
492  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
493  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
494  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
495  $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
496  }
497  }
498 
499  return $status;
500  }
501 
509  protected function doSecureInternal( $container, $dir, array $params ) {
510  return $this->newStatus();
511  }
512 
513  final protected function doPublish( array $params ) {
514  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
515  $status = $this->newStatus();
516 
517  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
518  if ( $dir === null ) {
519  $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
520 
521  return $status; // invalid storage path
522  }
523 
524  if ( $shard !== null ) { // confined to a single container/shard
525  $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
526  } else { // directory is on several shards
527  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
528  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
529  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
530  $status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) );
531  }
532  }
533 
534  return $status;
535  }
536 
544  protected function doPublishInternal( $container, $dir, array $params ) {
545  return $this->newStatus();
546  }
547 
548  final protected function doClean( array $params ) {
549  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
550  $status = $this->newStatus();
551 
552  // Recursive: first delete all empty subdirs recursively
553  if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) {
554  $subDirsRel = $this->getTopDirectoryList( [ 'dir' => $params['dir'] ] );
555  if ( $subDirsRel !== null ) { // no errors
556  foreach ( $subDirsRel as $subDirRel ) {
557  $subDir = $params['dir'] . "/{$subDirRel}"; // full path
558  $status->merge( $this->doClean( [ 'dir' => $subDir ] + $params ) );
559  }
560  unset( $subDirsRel ); // free directory for rmdir() on Windows (for FS backends)
561  }
562  }
563 
564  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
565  if ( $dir === null ) {
566  $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
567 
568  return $status; // invalid storage path
569  }
570 
571  // Attempt to lock this directory...
572  $filesLockEx = [ $params['dir'] ];
573  $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
574  if ( !$status->isOK() ) {
575  return $status; // abort
576  }
577 
578  if ( $shard !== null ) { // confined to a single container/shard
579  $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
580  $this->deleteContainerCache( $fullCont ); // purge cache
581  } else { // directory is on several shards
582  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
583  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
584  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
585  $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
586  $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache
587  }
588  }
589 
590  return $status;
591  }
592 
600  protected function doCleanInternal( $container, $dir, array $params ) {
601  return $this->newStatus();
602  }
603 
604  final public function fileExists( array $params ) {
605  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
606  $stat = $this->getFileStat( $params );
607 
608  return ( $stat === null ) ? null : (bool)$stat; // null => failure
609  }
610 
611  final public function getFileTimestamp( array $params ) {
612  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
613  $stat = $this->getFileStat( $params );
614 
615  return $stat ? $stat['mtime'] : false;
616  }
617 
618  final public function getFileSize( array $params ) {
619  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
620  $stat = $this->getFileStat( $params );
621 
622  return $stat ? $stat['size'] : false;
623  }
624 
625  final public function getFileStat( array $params ) {
626  $path = self::normalizeStoragePath( $params['src'] );
627  if ( $path === null ) {
628  return false; // invalid storage path
629  }
630  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
631  $latest = !empty( $params['latest'] ); // use latest data?
632  if ( !$latest && !$this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) {
633  $this->primeFileCache( [ $path ] ); // check persistent cache
634  }
635  if ( $this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) {
636  $stat = $this->cheapCache->get( $path, 'stat' );
637  // If we want the latest data, check that this cached
638  // value was in fact fetched with the latest available data.
639  if ( is_array( $stat ) ) {
640  if ( !$latest || $stat['latest'] ) {
641  return $stat;
642  }
643  } elseif ( in_array( $stat, [ 'NOT_EXIST', 'NOT_EXIST_LATEST' ] ) ) {
644  if ( !$latest || $stat === 'NOT_EXIST_LATEST' ) {
645  return false;
646  }
647  }
648  }
649  $stat = $this->doGetFileStat( $params );
650  if ( is_array( $stat ) ) { // file exists
651  // Strongly consistent backends can automatically set "latest"
652  $stat['latest'] = isset( $stat['latest'] ) ? $stat['latest'] : $latest;
653  $this->cheapCache->set( $path, 'stat', $stat );
654  $this->setFileCache( $path, $stat ); // update persistent cache
655  if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata
656  $this->cheapCache->set( $path, 'sha1',
657  [ 'hash' => $stat['sha1'], 'latest' => $latest ] );
658  }
659  if ( isset( $stat['xattr'] ) ) { // some backends store headers/metadata
660  $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] );
661  $this->cheapCache->set( $path, 'xattr',
662  [ 'map' => $stat['xattr'], 'latest' => $latest ] );
663  }
664  } elseif ( $stat === false ) { // file does not exist
665  $this->cheapCache->set( $path, 'stat', $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' );
666  $this->cheapCache->set( $path, 'xattr', [ 'map' => false, 'latest' => $latest ] );
667  $this->cheapCache->set( $path, 'sha1', [ 'hash' => false, 'latest' => $latest ] );
668  $this->logger->debug( __METHOD__ . ": File $path does not exist.\n" );
669  } else { // an error occurred
670  $this->logger->warning( __METHOD__ . ": Could not stat file $path.\n" );
671  }
672 
673  return $stat;
674  }
675 
680  abstract protected function doGetFileStat( array $params );
681 
682  public function getFileContentsMulti( array $params ) {
683  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
684 
685  $params = $this->setConcurrencyFlags( $params );
686  $contents = $this->doGetFileContentsMulti( $params );
687 
688  return $contents;
689  }
690 
696  protected function doGetFileContentsMulti( array $params ) {
697  $contents = [];
698  foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
699  MediaWiki\suppressWarnings();
700  $contents[$path] = $fsFile ? file_get_contents( $fsFile->getPath() ) : false;
701  MediaWiki\restoreWarnings();
702  }
703 
704  return $contents;
705  }
706 
707  final public function getFileXAttributes( array $params ) {
708  $path = self::normalizeStoragePath( $params['src'] );
709  if ( $path === null ) {
710  return false; // invalid storage path
711  }
712  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
713  $latest = !empty( $params['latest'] ); // use latest data?
714  if ( $this->cheapCache->has( $path, 'xattr', self::CACHE_TTL ) ) {
715  $stat = $this->cheapCache->get( $path, 'xattr' );
716  // If we want the latest data, check that this cached
717  // value was in fact fetched with the latest available data.
718  if ( !$latest || $stat['latest'] ) {
719  return $stat['map'];
720  }
721  }
722  $fields = $this->doGetFileXAttributes( $params );
723  $fields = is_array( $fields ) ? self::normalizeXAttributes( $fields ) : false;
724  $this->cheapCache->set( $path, 'xattr', [ 'map' => $fields, 'latest' => $latest ] );
725 
726  return $fields;
727  }
728 
734  protected function doGetFileXAttributes( array $params ) {
735  return [ 'headers' => [], 'metadata' => [] ]; // not supported
736  }
737 
738  final public function getFileSha1Base36( array $params ) {
739  $path = self::normalizeStoragePath( $params['src'] );
740  if ( $path === null ) {
741  return false; // invalid storage path
742  }
743  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
744  $latest = !empty( $params['latest'] ); // use latest data?
745  if ( $this->cheapCache->has( $path, 'sha1', self::CACHE_TTL ) ) {
746  $stat = $this->cheapCache->get( $path, 'sha1' );
747  // If we want the latest data, check that this cached
748  // value was in fact fetched with the latest available data.
749  if ( !$latest || $stat['latest'] ) {
750  return $stat['hash'];
751  }
752  }
753  $hash = $this->doGetFileSha1Base36( $params );
754  $this->cheapCache->set( $path, 'sha1', [ 'hash' => $hash, 'latest' => $latest ] );
755 
756  return $hash;
757  }
758 
764  protected function doGetFileSha1Base36( array $params ) {
765  $fsFile = $this->getLocalReference( $params );
766  if ( !$fsFile ) {
767  return false;
768  } else {
769  return $fsFile->getSha1Base36();
770  }
771  }
772 
773  final public function getFileProps( array $params ) {
774  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
775  $fsFile = $this->getLocalReference( $params );
776  $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
777 
778  return $props;
779  }
780 
781  final public function getLocalReferenceMulti( array $params ) {
782  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
783 
784  $params = $this->setConcurrencyFlags( $params );
785 
786  $fsFiles = []; // (path => FSFile)
787  $latest = !empty( $params['latest'] ); // use latest data?
788  // Reuse any files already in process cache...
789  foreach ( $params['srcs'] as $src ) {
790  $path = self::normalizeStoragePath( $src );
791  if ( $path === null ) {
792  $fsFiles[$src] = null; // invalid storage path
793  } elseif ( $this->expensiveCache->has( $path, 'localRef' ) ) {
794  $val = $this->expensiveCache->get( $path, 'localRef' );
795  // If we want the latest data, check that this cached
796  // value was in fact fetched with the latest available data.
797  if ( !$latest || $val['latest'] ) {
798  $fsFiles[$src] = $val['object'];
799  }
800  }
801  }
802  // Fetch local references of any remaning files...
803  $params['srcs'] = array_diff( $params['srcs'], array_keys( $fsFiles ) );
804  foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
805  $fsFiles[$path] = $fsFile;
806  if ( $fsFile ) { // update the process cache...
807  $this->expensiveCache->set( $path, 'localRef',
808  [ 'object' => $fsFile, 'latest' => $latest ] );
809  }
810  }
811 
812  return $fsFiles;
813  }
814 
820  protected function doGetLocalReferenceMulti( array $params ) {
821  return $this->doGetLocalCopyMulti( $params );
822  }
823 
824  final public function getLocalCopyMulti( array $params ) {
825  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
826 
827  $params = $this->setConcurrencyFlags( $params );
828  $tmpFiles = $this->doGetLocalCopyMulti( $params );
829 
830  return $tmpFiles;
831  }
832 
838  abstract protected function doGetLocalCopyMulti( array $params );
839 
845  public function getFileHttpUrl( array $params ) {
846  return null; // not supported
847  }
848 
849  final public function streamFile( array $params ) {
850  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
851  $status = $this->newStatus();
852 
853  // Always set some fields for subclass convenience
854  $params['options'] = isset( $params['options'] ) ? $params['options'] : [];
855  $params['headers'] = isset( $params['headers'] ) ? $params['headers'] : [];
856 
857  // Don't stream it out as text/html if there was a PHP error
858  if ( ( empty( $params['headless'] ) || $params['headers'] ) && headers_sent() ) {
859  print "Headers already sent, terminating.\n";
860  $status->fatal( 'backend-fail-stream', $params['src'] );
861  return $status;
862  }
863 
864  $status->merge( $this->doStreamFile( $params ) );
865 
866  return $status;
867  }
868 
874  protected function doStreamFile( array $params ) {
875  $status = $this->newStatus();
876 
877  $flags = 0;
878  $flags |= !empty( $params['headless'] ) ? HTTPFileStreamer::STREAM_HEADLESS : 0;
879  $flags |= !empty( $params['allowOB'] ) ? HTTPFileStreamer::STREAM_ALLOW_OB : 0;
880 
881  $fsFile = $this->getLocalReference( $params );
882  if ( $fsFile ) {
883  $streamer = new HTTPFileStreamer(
884  $fsFile->getPath(),
885  [
886  'obResetFunc' => $this->obResetFunc,
887  'streamMimeFunc' => $this->streamMimeFunc
888  ]
889  );
890  $res = $streamer->stream( $params['headers'], true, $params['options'], $flags );
891  } else {
892  $res = false;
893  HTTPFileStreamer::send404Message( $params['src'], $flags );
894  }
895 
896  if ( !$res ) {
897  $status->fatal( 'backend-fail-stream', $params['src'] );
898  }
899 
900  return $status;
901  }
902 
903  final public function directoryExists( array $params ) {
904  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
905  if ( $dir === null ) {
906  return false; // invalid storage path
907  }
908  if ( $shard !== null ) { // confined to a single container/shard
909  return $this->doDirectoryExists( $fullCont, $dir, $params );
910  } else { // directory is on several shards
911  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
912  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
913  $res = false; // response
914  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
915  $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params );
916  if ( $exists ) {
917  $res = true;
918  break; // found one!
919  } elseif ( $exists === null ) { // error?
920  $res = null; // if we don't find anything, it is indeterminate
921  }
922  }
923 
924  return $res;
925  }
926  }
927 
936  abstract protected function doDirectoryExists( $container, $dir, array $params );
937 
938  final public function getDirectoryList( array $params ) {
939  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
940  if ( $dir === null ) { // invalid storage path
941  return null;
942  }
943  if ( $shard !== null ) {
944  // File listing is confined to a single container/shard
945  return $this->getDirectoryListInternal( $fullCont, $dir, $params );
946  } else {
947  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
948  // File listing spans multiple containers/shards
949  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
950 
951  return new FileBackendStoreShardDirIterator( $this,
952  $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
953  }
954  }
955 
966  abstract public function getDirectoryListInternal( $container, $dir, array $params );
967 
968  final public function getFileList( array $params ) {
969  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
970  if ( $dir === null ) { // invalid storage path
971  return null;
972  }
973  if ( $shard !== null ) {
974  // File listing is confined to a single container/shard
975  return $this->getFileListInternal( $fullCont, $dir, $params );
976  } else {
977  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
978  // File listing spans multiple containers/shards
979  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
980 
981  return new FileBackendStoreShardFileIterator( $this,
982  $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
983  }
984  }
985 
996  abstract public function getFileListInternal( $container, $dir, array $params );
997 
1009  final public function getOperationsInternal( array $ops ) {
1010  $supportedOps = [
1011  'store' => 'StoreFileOp',
1012  'copy' => 'CopyFileOp',
1013  'move' => 'MoveFileOp',
1014  'delete' => 'DeleteFileOp',
1015  'create' => 'CreateFileOp',
1016  'describe' => 'DescribeFileOp',
1017  'null' => 'NullFileOp'
1018  ];
1019 
1020  $performOps = []; // array of FileOp objects
1021  // Build up ordered array of FileOps...
1022  foreach ( $ops as $operation ) {
1023  $opName = $operation['op'];
1024  if ( isset( $supportedOps[$opName] ) ) {
1025  $class = $supportedOps[$opName];
1026  // Get params for this operation
1027  $params = $operation;
1028  // Append the FileOp class
1029  $performOps[] = new $class( $this, $params, $this->logger );
1030  } else {
1031  throw new FileBackendError( "Operation '$opName' is not supported." );
1032  }
1033  }
1034 
1035  return $performOps;
1036  }
1037 
1048  final public function getPathsToLockForOpsInternal( array $performOps ) {
1049  // Build up a list of files to lock...
1050  $paths = [ 'sh' => [], 'ex' => [] ];
1051  foreach ( $performOps as $fileOp ) {
1052  $paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() );
1053  $paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() );
1054  }
1055  // Optimization: if doing an EX lock anyway, don't also set an SH one
1056  $paths['sh'] = array_diff( $paths['sh'], $paths['ex'] );
1057  // Get a shared lock on the parent directory of each path changed
1058  $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) );
1059 
1060  return [
1061  LockManager::LOCK_UW => $paths['sh'],
1062  LockManager::LOCK_EX => $paths['ex']
1063  ];
1064  }
1065 
1066  public function getScopedLocksForOps( array $ops, StatusValue $status ) {
1067  $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) );
1068 
1069  return $this->getScopedFileLocks( $paths, 'mixed', $status );
1070  }
1071 
1072  final protected function doOperationsInternal( array $ops, array $opts ) {
1073  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1074  $status = $this->newStatus();
1075 
1076  // Fix up custom header name/value pairs...
1077  $ops = array_map( [ $this, 'sanitizeOpHeaders' ], $ops );
1078 
1079  // Build up a list of FileOps...
1080  $performOps = $this->getOperationsInternal( $ops );
1081 
1082  // Acquire any locks as needed...
1083  if ( empty( $opts['nonLocking'] ) ) {
1084  // Build up a list of files to lock...
1085  $paths = $this->getPathsToLockForOpsInternal( $performOps );
1086  // Try to lock those files for the scope of this function...
1087 
1088  $scopeLock = $this->getScopedFileLocks( $paths, 'mixed', $status );
1089  if ( !$status->isOK() ) {
1090  return $status; // abort
1091  }
1092  }
1093 
1094  // Clear any file cache entries (after locks acquired)
1095  if ( empty( $opts['preserveCache'] ) ) {
1096  $this->clearCache();
1097  }
1098 
1099  // Build the list of paths involved
1100  $paths = [];
1101  foreach ( $performOps as $performOp ) {
1102  $paths = array_merge( $paths, $performOp->storagePathsRead() );
1103  $paths = array_merge( $paths, $performOp->storagePathsChanged() );
1104  }
1105 
1106  // Enlarge the cache to fit the stat entries of these files
1107  $this->cheapCache->resize( max( 2 * count( $paths ), self::CACHE_CHEAP_SIZE ) );
1108 
1109  // Load from the persistent container caches
1110  $this->primeContainerCache( $paths );
1111  // Get the latest stat info for all the files (having locked them)
1112  $ok = $this->preloadFileStat( [ 'srcs' => $paths, 'latest' => true ] );
1113 
1114  if ( $ok ) {
1115  // Actually attempt the operation batch...
1116  $opts = $this->setConcurrencyFlags( $opts );
1117  $subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal );
1118  } else {
1119  // If we could not even stat some files, then bail out...
1120  $subStatus = $this->newStatus( 'backend-fail-internal', $this->name );
1121  foreach ( $ops as $i => $op ) { // mark each op as failed
1122  $subStatus->success[$i] = false;
1123  ++$subStatus->failCount;
1124  }
1125  $this->logger->error( get_class( $this ) . "-{$this->name} " .
1126  " stat failure; aborted operations: " . FormatJson::encode( $ops ) );
1127  }
1128 
1129  // Merge errors into StatusValue fields
1130  $status->merge( $subStatus );
1131  $status->success = $subStatus->success; // not done in merge()
1132 
1133  // Shrink the stat cache back to normal size
1134  $this->cheapCache->resize( self::CACHE_CHEAP_SIZE );
1135 
1136  return $status;
1137  }
1138 
1139  final protected function doQuickOperationsInternal( array $ops ) {
1140  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1141  $status = $this->newStatus();
1142 
1143  // Fix up custom header name/value pairs...
1144  $ops = array_map( [ $this, 'sanitizeOpHeaders' ], $ops );
1145 
1146  // Clear any file cache entries
1147  $this->clearCache();
1148 
1149  $supportedOps = [ 'create', 'store', 'copy', 'move', 'delete', 'describe', 'null' ];
1150  // Parallel ops may be disabled in config due to dependencies (e.g. needing popen())
1151  $async = ( $this->parallelize === 'implicit' && count( $ops ) > 1 );
1152  $maxConcurrency = $this->concurrency; // throttle
1154  $statuses = []; // array of (index => StatusValue)
1155  $fileOpHandles = []; // list of (index => handle) arrays
1156  $curFileOpHandles = []; // current handle batch
1157  // Perform the sync-only ops and build up op handles for the async ops...
1158  foreach ( $ops as $index => $params ) {
1159  if ( !in_array( $params['op'], $supportedOps ) ) {
1160  throw new FileBackendError( "Operation '{$params['op']}' is not supported." );
1161  }
1162  $method = $params['op'] . 'Internal'; // e.g. "storeInternal"
1163  $subStatus = $this->$method( [ 'async' => $async ] + $params );
1164  if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { // async
1165  if ( count( $curFileOpHandles ) >= $maxConcurrency ) {
1166  $fileOpHandles[] = $curFileOpHandles; // push this batch
1167  $curFileOpHandles = [];
1168  }
1169  $curFileOpHandles[$index] = $subStatus->value; // keep index
1170  } else { // error or completed
1171  $statuses[$index] = $subStatus; // keep index
1172  }
1173  }
1174  if ( count( $curFileOpHandles ) ) {
1175  $fileOpHandles[] = $curFileOpHandles; // last batch
1176  }
1177  // Do all the async ops that can be done concurrently...
1178  foreach ( $fileOpHandles as $fileHandleBatch ) {
1179  $statuses = $statuses + $this->executeOpHandlesInternal( $fileHandleBatch );
1180  }
1181  // Marshall and merge all the responses...
1182  foreach ( $statuses as $index => $subStatus ) {
1183  $status->merge( $subStatus );
1184  if ( $subStatus->isOK() ) {
1185  $status->success[$index] = true;
1186  ++$status->successCount;
1187  } else {
1188  $status->success[$index] = false;
1189  ++$status->failCount;
1190  }
1191  }
1192 
1193  return $status;
1194  }
1195 
1206  final public function executeOpHandlesInternal( array $fileOpHandles ) {
1207  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1208 
1209  foreach ( $fileOpHandles as $fileOpHandle ) {
1210  if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) {
1211  throw new InvalidArgumentException( "Got a non-FileBackendStoreOpHandle object." );
1212  } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) {
1213  throw new InvalidArgumentException(
1214  "Got a FileBackendStoreOpHandle for the wrong backend." );
1215  }
1216  }
1217  $res = $this->doExecuteOpHandlesInternal( $fileOpHandles );
1218  foreach ( $fileOpHandles as $fileOpHandle ) {
1219  $fileOpHandle->closeResources();
1220  }
1221 
1222  return $res;
1223  }
1224 
1233  protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1234  if ( count( $fileOpHandles ) ) {
1235  throw new LogicException( "Backend does not support asynchronous operations." );
1236  }
1237 
1238  return [];
1239  }
1240 
1252  protected function sanitizeOpHeaders( array $op ) {
1253  static $longs = [ 'content-disposition' ];
1254 
1255  if ( isset( $op['headers'] ) ) { // op sets HTTP headers
1256  $newHeaders = [];
1257  foreach ( $op['headers'] as $name => $value ) {
1258  $name = strtolower( $name );
1259  $maxHVLen = in_array( $name, $longs ) ? INF : 255;
1260  if ( strlen( $name ) > 255 || strlen( $value ) > $maxHVLen ) {
1261  trigger_error( "Header '$name: $value' is too long." );
1262  } else {
1263  $newHeaders[$name] = strlen( $value ) ? $value : ''; // null/false => ""
1264  }
1265  }
1266  $op['headers'] = $newHeaders;
1267  }
1268 
1269  return $op;
1270  }
1271 
1272  final public function preloadCache( array $paths ) {
1273  $fullConts = []; // full container names
1274  foreach ( $paths as $path ) {
1275  list( $fullCont, , ) = $this->resolveStoragePath( $path );
1276  $fullConts[] = $fullCont;
1277  }
1278  // Load from the persistent file and container caches
1279  $this->primeContainerCache( $fullConts );
1280  $this->primeFileCache( $paths );
1281  }
1282 
1283  final public function clearCache( array $paths = null ) {
1284  if ( is_array( $paths ) ) {
1285  $paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
1286  $paths = array_filter( $paths, 'strlen' ); // remove nulls
1287  }
1288  if ( $paths === null ) {
1289  $this->cheapCache->clear();
1290  $this->expensiveCache->clear();
1291  } else {
1292  foreach ( $paths as $path ) {
1293  $this->cheapCache->clear( $path );
1294  $this->expensiveCache->clear( $path );
1295  }
1296  }
1297  $this->doClearCache( $paths );
1298  }
1299 
1307  protected function doClearCache( array $paths = null ) {
1308  }
1309 
1310  final public function preloadFileStat( array $params ) {
1311  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1312  $success = true; // no network errors
1313 
1314  $params['concurrency'] = ( $this->parallelize !== 'off' ) ? $this->concurrency : 1;
1315  $stats = $this->doGetFileStatMulti( $params );
1316  if ( $stats === null ) {
1317  return true; // not supported
1318  }
1319 
1320  $latest = !empty( $params['latest'] ); // use latest data?
1321  foreach ( $stats as $path => $stat ) {
1323  if ( $path === null ) {
1324  continue; // this shouldn't happen
1325  }
1326  if ( is_array( $stat ) ) { // file exists
1327  // Strongly consistent backends can automatically set "latest"
1328  $stat['latest'] = isset( $stat['latest'] ) ? $stat['latest'] : $latest;
1329  $this->cheapCache->set( $path, 'stat', $stat );
1330  $this->setFileCache( $path, $stat ); // update persistent cache
1331  if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata
1332  $this->cheapCache->set( $path, 'sha1',
1333  [ 'hash' => $stat['sha1'], 'latest' => $latest ] );
1334  }
1335  if ( isset( $stat['xattr'] ) ) { // some backends store headers/metadata
1336  $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] );
1337  $this->cheapCache->set( $path, 'xattr',
1338  [ 'map' => $stat['xattr'], 'latest' => $latest ] );
1339  }
1340  } elseif ( $stat === false ) { // file does not exist
1341  $this->cheapCache->set( $path, 'stat',
1342  $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' );
1343  $this->cheapCache->set( $path, 'xattr',
1344  [ 'map' => false, 'latest' => $latest ] );
1345  $this->cheapCache->set( $path, 'sha1',
1346  [ 'hash' => false, 'latest' => $latest ] );
1347  $this->logger->debug( __METHOD__ . ": File $path does not exist.\n" );
1348  } else { // an error occurred
1349  $success = false;
1350  $this->logger->warning( __METHOD__ . ": Could not stat file $path.\n" );
1351  }
1352  }
1353 
1354  return $success;
1355  }
1356 
1368  protected function doGetFileStatMulti( array $params ) {
1369  return null; // not supported
1370  }
1371 
1379  abstract protected function directoriesAreVirtual();
1380 
1391  final protected static function isValidShortContainerName( $container ) {
1392  // Suffixes like '.xxx' (hex shard chars) or '.seg' (file segments)
1393  // might be used by subclasses. Reserve the dot character for sanity.
1394  // The only way dots end up in containers (e.g. resolveStoragePath)
1395  // is due to the wikiId container prefix or the above suffixes.
1396  return self::isValidContainerName( $container ) && !preg_match( '/[.]/', $container );
1397  }
1398 
1408  final protected static function isValidContainerName( $container ) {
1409  // This accounts for NTFS, Swift, and Ceph restrictions
1410  // and disallows directory separators or traversal characters.
1411  // Note that matching strings URL encode to the same string;
1412  // in Swift/Ceph, the length restriction is *after* URL encoding.
1413  return (bool)preg_match( '/^[a-z0-9][a-z0-9-_.]{0,199}$/i', $container );
1414  }
1415 
1429  final protected function resolveStoragePath( $storagePath ) {
1430  list( $backend, $shortCont, $relPath ) = self::splitStoragePath( $storagePath );
1431  if ( $backend === $this->name ) { // must be for this backend
1432  $relPath = self::normalizeContainerPath( $relPath );
1433  if ( $relPath !== null && self::isValidShortContainerName( $shortCont ) ) {
1434  // Get shard for the normalized path if this container is sharded
1435  $cShard = $this->getContainerShard( $shortCont, $relPath );
1436  // Validate and sanitize the relative path (backend-specific)
1437  $relPath = $this->resolveContainerPath( $shortCont, $relPath );
1438  if ( $relPath !== null ) {
1439  // Prepend any wiki ID prefix to the container name
1440  $container = $this->fullContainerName( $shortCont );
1441  if ( self::isValidContainerName( $container ) ) {
1442  // Validate and sanitize the container name (backend-specific)
1443  $container = $this->resolveContainerName( "{$container}{$cShard}" );
1444  if ( $container !== null ) {
1445  return [ $container, $relPath, $cShard ];
1446  }
1447  }
1448  }
1449  }
1450  }
1451 
1452  return [ null, null, null ];
1453  }
1454 
1470  final protected function resolveStoragePathReal( $storagePath ) {
1471  list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
1472  if ( $cShard !== null && substr( $relPath, -1 ) !== '/' ) {
1473  return [ $container, $relPath ];
1474  }
1475 
1476  return [ null, null ];
1477  }
1478 
1487  final protected function getContainerShard( $container, $relPath ) {
1488  list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container );
1489  if ( $levels == 1 || $levels == 2 ) {
1490  // Hash characters are either base 16 or 36
1491  $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]';
1492  // Get a regex that represents the shard portion of paths.
1493  // The concatenation of the captures gives us the shard.
1494  if ( $levels === 1 ) { // 16 or 36 shards per container
1495  $hashDirRegex = '(' . $char . ')';
1496  } else { // 256 or 1296 shards per container
1497  if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc")
1498  $hashDirRegex = $char . '/(' . $char . '{2})';
1499  } else { // short hash dir format (e.g. "a/b/c")
1500  $hashDirRegex = '(' . $char . ')/(' . $char . ')';
1501  }
1502  }
1503  // Allow certain directories to be above the hash dirs so as
1504  // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
1505  // They must be 2+ chars to avoid any hash directory ambiguity.
1506  $m = [];
1507  if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1508  return '.' . implode( '', array_slice( $m, 1 ) );
1509  }
1510 
1511  return null; // failed to match
1512  }
1513 
1514  return ''; // no sharding
1515  }
1516 
1525  final public function isSingleShardPathInternal( $storagePath ) {
1526  list( , , $shard ) = $this->resolveStoragePath( $storagePath );
1527 
1528  return ( $shard !== null );
1529  }
1530 
1539  final protected function getContainerHashLevels( $container ) {
1540  if ( isset( $this->shardViaHashLevels[$container] ) ) {
1541  $config = $this->shardViaHashLevels[$container];
1542  $hashLevels = (int)$config['levels'];
1543  if ( $hashLevels == 1 || $hashLevels == 2 ) {
1544  $hashBase = (int)$config['base'];
1545  if ( $hashBase == 16 || $hashBase == 36 ) {
1546  return [ $hashLevels, $hashBase, $config['repeat'] ];
1547  }
1548  }
1549  }
1550 
1551  return [ 0, 0, false ]; // no sharding
1552  }
1553 
1560  final protected function getContainerSuffixes( $container ) {
1561  $shards = [];
1562  list( $digits, $base ) = $this->getContainerHashLevels( $container );
1563  if ( $digits > 0 ) {
1564  $numShards = pow( $base, $digits );
1565  for ( $index = 0; $index < $numShards; $index++ ) {
1566  $shards[] = '.' . Wikimedia\base_convert( $index, 10, $base, $digits );
1567  }
1568  }
1569 
1570  return $shards;
1571  }
1572 
1579  final protected function fullContainerName( $container ) {
1580  if ( $this->domainId != '' ) {
1581  return "{$this->domainId}-$container";
1582  } else {
1583  return $container;
1584  }
1585  }
1586 
1595  protected function resolveContainerName( $container ) {
1596  return $container;
1597  }
1598 
1609  protected function resolveContainerPath( $container, $relStoragePath ) {
1610  return $relStoragePath;
1611  }
1612 
1619  private function containerCacheKey( $container ) {
1620  return "filebackend:{$this->name}:{$this->domainId}:container:{$container}";
1621  }
1622 
1629  final protected function setContainerCache( $container, array $val ) {
1630  $this->memCache->set( $this->containerCacheKey( $container ), $val, 14 * 86400 );
1631  }
1632 
1639  final protected function deleteContainerCache( $container ) {
1640  if ( !$this->memCache->delete( $this->containerCacheKey( $container ), 300 ) ) {
1641  trigger_error( "Unable to delete stat cache for container $container." );
1642  }
1643  }
1644 
1652  final protected function primeContainerCache( array $items ) {
1653  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1654 
1655  $paths = []; // list of storage paths
1656  $contNames = []; // (cache key => resolved container name)
1657  // Get all the paths/containers from the items...
1658  foreach ( $items as $item ) {
1659  if ( self::isStoragePath( $item ) ) {
1660  $paths[] = $item;
1661  } elseif ( is_string( $item ) ) { // full container name
1662  $contNames[$this->containerCacheKey( $item )] = $item;
1663  }
1664  }
1665  // Get all the corresponding cache keys for paths...
1666  foreach ( $paths as $path ) {
1667  list( $fullCont, , ) = $this->resolveStoragePath( $path );
1668  if ( $fullCont !== null ) { // valid path for this backend
1669  $contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
1670  }
1671  }
1672 
1673  $contInfo = []; // (resolved container name => cache value)
1674  // Get all cache entries for these container cache keys...
1675  $values = $this->memCache->getMulti( array_keys( $contNames ) );
1676  foreach ( $values as $cacheKey => $val ) {
1677  $contInfo[$contNames[$cacheKey]] = $val;
1678  }
1679 
1680  // Populate the container process cache for the backend...
1681  $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) );
1682  }
1683 
1691  protected function doPrimeContainerCache( array $containerInfo ) {
1692  }
1693 
1700  private function fileCacheKey( $path ) {
1701  return "filebackend:{$this->name}:{$this->domainId}:file:" . sha1( $path );
1702  }
1703 
1712  final protected function setFileCache( $path, array $val ) {
1714  if ( $path === null ) {
1715  return; // invalid storage path
1716  }
1717  $mtime = ConvertibleTimestamp::convert( TS_UNIX, $val['mtime'] );
1718  $ttl = $this->memCache->adaptiveTTL( $mtime, 7 * 86400, 300, .1 );
1719  $key = $this->fileCacheKey( $path );
1720  // Set the cache unless it is currently salted.
1721  $this->memCache->set( $key, $val, $ttl );
1722  }
1723 
1732  final protected function deleteFileCache( $path ) {
1734  if ( $path === null ) {
1735  return; // invalid storage path
1736  }
1737  if ( !$this->memCache->delete( $this->fileCacheKey( $path ), 300 ) ) {
1738  trigger_error( "Unable to delete stat cache for file $path." );
1739  }
1740  }
1741 
1749  final protected function primeFileCache( array $items ) {
1750  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1751 
1752  $paths = []; // list of storage paths
1753  $pathNames = []; // (cache key => storage path)
1754  // Get all the paths/containers from the items...
1755  foreach ( $items as $item ) {
1756  if ( self::isStoragePath( $item ) ) {
1757  $paths[] = FileBackend::normalizeStoragePath( $item );
1758  }
1759  }
1760  // Get rid of any paths that failed normalization...
1761  $paths = array_filter( $paths, 'strlen' ); // remove nulls
1762  // Get all the corresponding cache keys for paths...
1763  foreach ( $paths as $path ) {
1764  list( , $rel, ) = $this->resolveStoragePath( $path );
1765  if ( $rel !== null ) { // valid path for this backend
1766  $pathNames[$this->fileCacheKey( $path )] = $path;
1767  }
1768  }
1769  // Get all cache entries for these file cache keys...
1770  $values = $this->memCache->getMulti( array_keys( $pathNames ) );
1771  foreach ( $values as $cacheKey => $val ) {
1772  $path = $pathNames[$cacheKey];
1773  if ( is_array( $val ) ) {
1774  $val['latest'] = false; // never completely trust cache
1775  $this->cheapCache->set( $path, 'stat', $val );
1776  if ( isset( $val['sha1'] ) ) { // some backends store SHA-1 as metadata
1777  $this->cheapCache->set( $path, 'sha1',
1778  [ 'hash' => $val['sha1'], 'latest' => false ] );
1779  }
1780  if ( isset( $val['xattr'] ) ) { // some backends store headers/metadata
1781  $val['xattr'] = self::normalizeXAttributes( $val['xattr'] );
1782  $this->cheapCache->set( $path, 'xattr',
1783  [ 'map' => $val['xattr'], 'latest' => false ] );
1784  }
1785  }
1786  }
1787  }
1788 
1796  final protected static function normalizeXAttributes( array $xattr ) {
1797  $newXAttr = [ 'headers' => [], 'metadata' => [] ];
1798 
1799  foreach ( $xattr['headers'] as $name => $value ) {
1800  $newXAttr['headers'][strtolower( $name )] = $value;
1801  }
1802 
1803  foreach ( $xattr['metadata'] as $name => $value ) {
1804  $newXAttr['metadata'][strtolower( $name )] = $value;
1805  }
1806 
1807  return $newXAttr;
1808  }
1809 
1816  final protected function setConcurrencyFlags( array $opts ) {
1817  $opts['concurrency'] = 1; // off
1818  if ( $this->parallelize === 'implicit' ) {
1819  if ( !isset( $opts['parallelize'] ) || $opts['parallelize'] ) {
1820  $opts['concurrency'] = $this->concurrency;
1821  }
1822  } elseif ( $this->parallelize === 'explicit' ) {
1823  if ( !empty( $opts['parallelize'] ) ) {
1824  $opts['concurrency'] = $this->concurrency;
1825  }
1826  }
1827 
1828  return $opts;
1829  }
1830 
1839  protected function getContentType( $storagePath, $content, $fsPath ) {
1840  if ( $this->mimeCallback ) {
1841  return call_user_func_array( $this->mimeCallback, func_get_args() );
1842  }
1843 
1844  $mime = null;
1845  if ( $fsPath !== null && function_exists( 'finfo_file' ) ) {
1846  $finfo = finfo_open( FILEINFO_MIME_TYPE );
1847  $mime = finfo_file( $finfo, $fsPath );
1848  finfo_close( $finfo );
1849  }
1850 
1851  return is_string( $mime ) ? $mime : 'unknown/unknown';
1852  }
1853 }
1854 
1865  public $params = []; // params to caller functions
1867  public $backend;
1869  public $resourcesToClose = [];
1870 
1871  public $call; // string; name that identifies the function called
1872 
1876  public function closeResources() {
1877  array_map( 'fclose', $this->resourcesToClose );
1878  }
1879 }
1880 
1887 abstract class FileBackendStoreShardListIterator extends FilterIterator {
1889  protected $backend;
1890 
1892  protected $params;
1893 
1895  protected $container;
1896 
1898  protected $directory;
1899 
1901  protected $multiShardPaths = []; // (rel path => 1)
1902 
1910  public function __construct(
1912  ) {
1913  $this->backend = $backend;
1914  $this->container = $container;
1915  $this->directory = $dir;
1916  $this->params = $params;
1917 
1918  $iter = new AppendIterator();
1919  foreach ( $suffixes as $suffix ) {
1920  $iter->append( $this->listFromShard( $this->container . $suffix ) );
1921  }
1922 
1923  parent::__construct( $iter );
1924  }
1925 
1926  public function accept() {
1927  $rel = $this->getInnerIterator()->current(); // path relative to given directory
1928  $path = $this->params['dir'] . "/{$rel}"; // full storage path
1929  if ( $this->backend->isSingleShardPathInternal( $path ) ) {
1930  return true; // path is only on one shard; no issue with duplicates
1931  } elseif ( isset( $this->multiShardPaths[$rel] ) ) {
1932  // Don't keep listing paths that are on multiple shards
1933  return false;
1934  } else {
1935  $this->multiShardPaths[$rel] = 1;
1936 
1937  return true;
1938  }
1939  }
1940 
1941  public function rewind() {
1942  parent::rewind();
1943  $this->multiShardPaths = [];
1944  }
1945 
1952  abstract protected function listFromShard( $container );
1953 }
1954 
1959  protected function listFromShard( $container ) {
1960  $list = $this->backend->getDirectoryListInternal(
1961  $container, $this->directory, $this->params );
1962  if ( $list === null ) {
1963  return new ArrayIterator( [] );
1964  } else {
1965  return is_array( $list ) ? new ArrayIterator( $list ) : $list;
1966  }
1967  }
1968 }
1969 
1974  protected function listFromShard( $container ) {
1975  $list = $this->backend->getFileListInternal(
1976  $container, $this->directory, $this->params );
1977  if ( $list === null ) {
1978  return new ArrayIterator( [] );
1979  } else {
1980  return is_array( $list ) ? new ArrayIterator( $list ) : $list;
1981  }
1982  }
1983 }
newStatus()
Yields the result of the status wrapper callback on either:
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
doPublishInternal($container, $dir, array $params)
getFileHttpUrl(array $params)
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition: deferred.txt:11
doOperationsInternal(array $ops, array $opts)
setConcurrencyFlags(array $opts)
Set the 'concurrency' option from a list of operation options.
the array() calling protocol came about after MediaWiki 1.4rc1.
if(count($args)==0) $dir
$success
setFileCache($path, array $val)
Set the cached stat info for a file path.
Iterator for listing regular files.
moveInternal(array $params)
Move a file from one storage path to another in the backend.
processing should stop and the error should be shown to the user * false
Definition: hooks.txt:189
concatenate(array $params)
doPrepareInternal($container, $dir, array $params)
doGetLocalReferenceMulti(array $params)
resolveContainerName($container)
Resolve a container name, checking if it's allowed by the backend.
getPathsToLockForOpsInternal(array $performOps)
Get a list of storage paths to lock for a list of operations Returns an array with LockManager::LOCK_...
scopedProfileSection($section)
static normalizeXAttributes(array $xattr)
Normalize file headers/metadata to the FileBackend::getFileXAttributes() format.
listFromShard($container)
Get the list for a given container shard.
doGetFileXAttributes(array $params)
getFileStat(array $params)
getContainerSuffixes($container)
Get a list of full container shard suffixes for a container.
$value
The most up to date schema for the tables in the database will always be tables sql in the maintenance directory
Definition: schema.txt:2
__construct(array $config)
if($ext== 'php'||$ext== 'php5') $mime
Definition: router.php:65
getName()
Get the unique backend name.
string $directory
Resolved relative path.
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
it s the revision text itself In either if gzip is the revision text is gzipped $flags
Definition: hooks.txt:2703
string $container
Full container name.
ProcessCacheLRU $expensiveCache
Map of paths to large (RAM/disk) cache items.
fileCacheKey($path)
Get the cache key for a file path.
FileBackendStore helper function to handle listings that span container shards.
static isValidContainerName($container)
Check if a full container name is valid.
int $concurrency
How many operations can be done in parallel.
storeInternal(array $params)
Store a file into the backend from a file on disk.
array $shardViaHashLevels
Map of container names to sharding config.
doSecureInternal($container, $dir, array $params)
const TS_UNIX
Unix time - the number of seconds since 1970-01-01 00:00:00 UTC.
Definition: defines.php:6
doCreateInternal(array $params)
static newEmpty()
Get an instance that wraps EmptyBagOStuff.
WANObjectCache $memCache
streamFile(array $params)
getFileSha1Base36(array $params)
const LOCK_UW
Definition: LockManager.php:69
doClean(array $params)
executeOpHandlesInternal(array $fileOpHandles)
Execute a list of FileBackendStoreOpHandle handles in parallel.
getFileSize(array $params)
doQuickOperationsInternal(array $ops)
File backend exception for checked exceptions (e.g.
isPathUsableInternal($storagePath)
Check if a file can be created or changed at a given storage path.
Functions related to the output of file content.
ProcessCacheLRU $cheapCache
Map of paths to small (RAM/disk) cache items.
createInternal(array $params)
Create a file in the backend with the given contents.
fileExists(array $params)
directoryExists(array $params)
const LOCK_EX
Definition: LockManager.php:70
$res
Definition: database.txt:21
doStreamFile(array $params)
getFileXAttributes(array $params)
nullInternal(array $params)
No-op file operation that does nothing.
static encode($value, $pretty=false, $escaping=0)
Returns the JSON representation of a value.
Definition: FormatJson.php:127
deleteInternal(array $params)
Delete a file at the storage path.
preloadCache(array $paths)
$params
callable $obResetFunc
doGetLocalCopyMulti(array $params)
getFileContentsMulti(array $params)
getFileTimestamp(array $params)
maxFileSizeInternal()
Get the maximum allowable file size given backend medium restrictions and basic performance constrain...
static placeholderProps()
Placeholder file properties to use for files that don't exist.
Definition: FSFile.php:143
getOperationsInternal(array $ops)
Return a list of FileOp objects from a list of operations.
callable $streamMimeFunc
clearCache(array $paths=null)
A BagOStuff object with no objects in it.
callable $mimeCallback
Method to get the MIME type of files.
resolveStoragePathReal($storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
doPrepare(array $params)
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
doGetFileStatMulti(array $params)
Get file stat information (concurrently if possible) for several files.
getContentType($storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
copyInternal(array $params)
Copy a file from one storage path to another in the backend.
doGetFileStat(array $params)
string $name
Unique backend name.
Definition: FileBackend.php:95
doGetFileContentsMulti(array $params)
Base class for all backends using particular storage medium.
preloadFileStat(array $params)
fullContainerName($container)
Get the full container name, including the wiki ID prefix.
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
getFileList(array $params)
closeResources()
Close all open file handles.
__construct(FileBackendStore $backend, $container, $dir, array $suffixes, array $params)
containerCacheKey($container)
Get the cache key for a container.
Iterator for listing directories.
getContainerHashLevels($container)
Get the sharding config for a container.
static normalizeStoragePath($storagePath)
Normalize a storage path by cleaning up directory separators.
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
doSecure(array $params)
static attempt(array $performOps, array $opts, FileJournal $journal)
Attempt to perform a series of file operations.
Definition: FileOpBatch.php:57
resolveContainerPath($container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
setContainerCache($container, array $val)
Set the cached info for a container.
doConcatenate(array $params)
getContainerShard($container, $relPath)
Get the container name shard suffix for a given path.
getDirectoryListInternal($container, $dir, array $params)
Do not call this function from places outside FileBackend.
doPublish(array $params)
doPrimeContainerCache(array $containerInfo)
Fill the backend-specific process cache given an array of resolved container names and their correspo...
static send404Message($fname, $flags=0)
Send out a standard 404 message for a file.
this hook is for auditing only RecentChangesLinked and Watchlist RecentChangesLinked and Watchlist e g Watchlist removed from all revisions and log entries to which it was applied This gives extensions a chance to take it off their books as the deletion has already been partly carried out by this point or something similar the user will be unable to create the tag set and then return false from the hook function Ensure you consume the ChangeTagAfterDelete hook to carry out custom deletion actions as context called by AbstractContent::getParserOutput May be used to override the normal model specific rendering of page content $content
Definition: hooks.txt:1046
doClearCache(array $paths=null)
Clears any additional stat caches for storage paths.
describeInternal(array $params)
Alter metadata for a file at the storage path.
design txt This is a brief overview of the new design More thorough and up to date information is available on the documentation wiki at name
Definition: design.txt:12
doStoreInternal(array $params)
isSingleShardPathInternal($storagePath)
Check if a storage path maps to a single shard.
Base class for all file backend classes (including multi-write backends).
Definition: FileBackend.php:93
doDeleteInternal(array $params)
getTopDirectoryList(array $params)
Same as FileBackend::getDirectoryList() except only lists directories that are immediately under the ...
this hook is for auditing only RecentChangesLinked and Watchlist RecentChangesLinked and Watchlist e g Watchlist removed from all revisions and log entries to which it was applied This gives extensions a chance to take it off their books as the deletion has already been partly carried out by this point or something similar the user will be unable to create the tag set $status
Definition: hooks.txt:1046
static convert($style=TS_UNIX, $ts)
Convert a timestamp string to a given format.
deleteFileCache($path)
Delete the cached stat info for a file path.
getScopedLocksForOps(array $ops, StatusValue $status)
doMoveInternal(array $params)
sanitizeOpHeaders(array $op)
Normalize and filter HTTP headers from a file operation.
doDescribeInternal(array $params)
static isValidShortContainerName($container)
Check if a short container name is valid.
doCleanInternal($container, $dir, array $params)
getLocalReferenceMulti(array $params)
FileBackendStore helper class for performing asynchronous file operations.
doGetFileSha1Base36(array $params)
deleteContainerCache($container)
Delete the cached info for a container.
primeFileCache(array $items)
Do a batch lookup from cache for file stats for all paths used in a list of storage paths or FileOp o...
getFileListInternal($container, $dir, array $params)
Do not call this function from places outside FileBackend.
doCopyInternal(array $params)
getLocalReference(array $params)
Returns a file system file, identical to the file at a storage path.
getDirectoryList(array $params)
Handles per process caching of items.
getLocalCopyMulti(array $params)
getFileProps(array $params)
doDirectoryExists($container, $dir, array $params)
resolveStoragePath($storagePath)
Splits a storage path into an internal container name, an internal relative file name, and a container shard suffix.
doExecuteOpHandlesInternal(array $fileOpHandles)