MediaWiki  master
FileBackendStore.php
Go to the documentation of this file.
1 <?php
24 
38 abstract class FileBackendStore extends FileBackend {
40  protected $memCache;
42  protected $srvCache;
44  protected $cheapCache;
46  protected $expensiveCache;
47 
49  protected $shardViaHashLevels = [];
50 
52  protected $mimeCallback;
53 
54  protected $maxFileSize = 4294967296; // integer bytes (4GiB)
55 
56  const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries
57  const CACHE_CHEAP_SIZE = 500; // integer; max entries in "cheap cache"
58  const CACHE_EXPENSIVE_SIZE = 5; // integer; max entries in "expensive cache"
59 
71  public function __construct( array $config ) {
72  parent::__construct( $config );
73  $this->mimeCallback = isset( $config['mimeCallback'] )
74  ? $config['mimeCallback']
75  : null;
76  $this->srvCache = new EmptyBagOStuff(); // disabled by default
77  $this->memCache = WANObjectCache::newEmpty(); // disabled by default
78  $this->cheapCache = new ProcessCacheLRU( self::CACHE_CHEAP_SIZE );
79  $this->expensiveCache = new ProcessCacheLRU( self::CACHE_EXPENSIVE_SIZE );
80  }
81 
89  final public function maxFileSizeInternal() {
90  return $this->maxFileSize;
91  }
92 
102  abstract public function isPathUsableInternal( $storagePath );
103 
122  final public function createInternal( array $params ) {
123  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
124  if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
125  $status = $this->newStatus( 'backend-fail-maxsize',
126  $params['dst'], $this->maxFileSizeInternal() );
127  } else {
128  $status = $this->doCreateInternal( $params );
129  $this->clearCache( [ $params['dst'] ] );
130  if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
131  $this->deleteFileCache( $params['dst'] ); // persistent cache
132  }
133  }
134 
135  return $status;
136  }
137 
143  abstract protected function doCreateInternal( array $params );
144 
163  final public function storeInternal( array $params ) {
164  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
165  if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
166  $status = $this->newStatus( 'backend-fail-maxsize',
167  $params['dst'], $this->maxFileSizeInternal() );
168  } else {
169  $status = $this->doStoreInternal( $params );
170  $this->clearCache( [ $params['dst'] ] );
171  if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
172  $this->deleteFileCache( $params['dst'] ); // persistent cache
173  }
174  }
175 
176  return $status;
177  }
178 
184  abstract protected function doStoreInternal( array $params );
185 
205  final public function copyInternal( array $params ) {
206  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
207  $status = $this->doCopyInternal( $params );
208  $this->clearCache( [ $params['dst'] ] );
209  if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
210  $this->deleteFileCache( $params['dst'] ); // persistent cache
211  }
212 
213  return $status;
214  }
215 
221  abstract protected function doCopyInternal( array $params );
222 
237  final public function deleteInternal( array $params ) {
238  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
239  $status = $this->doDeleteInternal( $params );
240  $this->clearCache( [ $params['src'] ] );
241  $this->deleteFileCache( $params['src'] ); // persistent cache
242  return $status;
243  }
244 
250  abstract protected function doDeleteInternal( array $params );
251 
271  final public function moveInternal( array $params ) {
272  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
273  $status = $this->doMoveInternal( $params );
274  $this->clearCache( [ $params['src'], $params['dst'] ] );
275  $this->deleteFileCache( $params['src'] ); // persistent cache
276  if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
277  $this->deleteFileCache( $params['dst'] ); // persistent cache
278  }
279 
280  return $status;
281  }
282 
288  protected function doMoveInternal( array $params ) {
289  unset( $params['async'] ); // two steps, won't work here :)
290  $nsrc = FileBackend::normalizeStoragePath( $params['src'] );
291  $ndst = FileBackend::normalizeStoragePath( $params['dst'] );
292  // Copy source to dest
293  $status = $this->copyInternal( $params );
294  if ( $nsrc !== $ndst && $status->isOK() ) {
295  // Delete source (only fails due to races or network problems)
296  $status->merge( $this->deleteInternal( [ 'src' => $params['src'] ] ) );
297  $status->setResult( true, $status->value ); // ignore delete() errors
298  }
299 
300  return $status;
301  }
302 
317  final public function describeInternal( array $params ) {
318  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
319  if ( count( $params['headers'] ) ) {
320  $status = $this->doDescribeInternal( $params );
321  $this->clearCache( [ $params['src'] ] );
322  $this->deleteFileCache( $params['src'] ); // persistent cache
323  } else {
324  $status = $this->newStatus(); // nothing to do
325  }
326 
327  return $status;
328  }
329 
335  protected function doDescribeInternal( array $params ) {
336  return $this->newStatus();
337  }
338 
346  final public function nullInternal( array $params ) {
347  return $this->newStatus();
348  }
349 
350  final public function concatenate( array $params ) {
351  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
352  $status = $this->newStatus();
353 
354  // Try to lock the source files for the scope of this function
355  $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
356  if ( $status->isOK() ) {
357  // Actually do the file concatenation...
358  $start_time = microtime( true );
359  $status->merge( $this->doConcatenate( $params ) );
360  $sec = microtime( true ) - $start_time;
361  if ( !$status->isOK() ) {
362  $this->logger->error( static::class . "-{$this->name}" .
363  " failed to concatenate " . count( $params['srcs'] ) . " file(s) [$sec sec]" );
364  }
365  }
366 
367  return $status;
368  }
369 
375  protected function doConcatenate( array $params ) {
376  $status = $this->newStatus();
377  $tmpPath = $params['dst']; // convenience
378  unset( $params['latest'] ); // sanity
379 
380  // Check that the specified temp file is valid...
381  MediaWiki\suppressWarnings();
382  $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 );
383  MediaWiki\restoreWarnings();
384  if ( !$ok ) { // not present or not empty
385  $status->fatal( 'backend-fail-opentemp', $tmpPath );
386 
387  return $status;
388  }
389 
390  // Get local FS versions of the chunks needed for the concatenation...
391  $fsFiles = $this->getLocalReferenceMulti( $params );
392  foreach ( $fsFiles as $path => &$fsFile ) {
393  if ( !$fsFile ) { // chunk failed to download?
394  $fsFile = $this->getLocalReference( [ 'src' => $path ] );
395  if ( !$fsFile ) { // retry failed?
396  $status->fatal( 'backend-fail-read', $path );
397 
398  return $status;
399  }
400  }
401  }
402  unset( $fsFile ); // unset reference so we can reuse $fsFile
403 
404  // Get a handle for the destination temp file
405  $tmpHandle = fopen( $tmpPath, 'ab' );
406  if ( $tmpHandle === false ) {
407  $status->fatal( 'backend-fail-opentemp', $tmpPath );
408 
409  return $status;
410  }
411 
412  // Build up the temp file using the source chunks (in order)...
413  foreach ( $fsFiles as $virtualSource => $fsFile ) {
414  // Get a handle to the local FS version
415  $sourceHandle = fopen( $fsFile->getPath(), 'rb' );
416  if ( $sourceHandle === false ) {
417  fclose( $tmpHandle );
418  $status->fatal( 'backend-fail-read', $virtualSource );
419 
420  return $status;
421  }
422  // Append chunk to file (pass chunk size to avoid magic quotes)
423  if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
424  fclose( $sourceHandle );
425  fclose( $tmpHandle );
426  $status->fatal( 'backend-fail-writetemp', $tmpPath );
427 
428  return $status;
429  }
430  fclose( $sourceHandle );
431  }
432  if ( !fclose( $tmpHandle ) ) {
433  $status->fatal( 'backend-fail-closetemp', $tmpPath );
434 
435  return $status;
436  }
437 
438  clearstatcache(); // temp file changed
439 
440  return $status;
441  }
442 
443  final protected function doPrepare( array $params ) {
444  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
445  $status = $this->newStatus();
446 
447  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
448  if ( $dir === null ) {
449  $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
450 
451  return $status; // invalid storage path
452  }
453 
454  if ( $shard !== null ) { // confined to a single container/shard
455  $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
456  } else { // directory is on several shards
457  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
458  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
459  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
460  $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
461  }
462  }
463 
464  return $status;
465  }
466 
474  protected function doPrepareInternal( $container, $dir, array $params ) {
475  return $this->newStatus();
476  }
477 
478  final protected function doSecure( array $params ) {
479  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
480  $status = $this->newStatus();
481 
482  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
483  if ( $dir === null ) {
484  $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
485 
486  return $status; // invalid storage path
487  }
488 
489  if ( $shard !== null ) { // confined to a single container/shard
490  $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
491  } else { // directory is on several shards
492  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
493  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
494  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
495  $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
496  }
497  }
498 
499  return $status;
500  }
501 
509  protected function doSecureInternal( $container, $dir, array $params ) {
510  return $this->newStatus();
511  }
512 
513  final protected function doPublish( array $params ) {
514  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
515  $status = $this->newStatus();
516 
517  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
518  if ( $dir === null ) {
519  $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
520 
521  return $status; // invalid storage path
522  }
523 
524  if ( $shard !== null ) { // confined to a single container/shard
525  $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
526  } else { // directory is on several shards
527  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
528  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
529  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
530  $status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) );
531  }
532  }
533 
534  return $status;
535  }
536 
544  protected function doPublishInternal( $container, $dir, array $params ) {
545  return $this->newStatus();
546  }
547 
548  final protected function doClean( array $params ) {
549  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
550  $status = $this->newStatus();
551 
552  // Recursive: first delete all empty subdirs recursively
553  if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) {
554  $subDirsRel = $this->getTopDirectoryList( [ 'dir' => $params['dir'] ] );
555  if ( $subDirsRel !== null ) { // no errors
556  foreach ( $subDirsRel as $subDirRel ) {
557  $subDir = $params['dir'] . "/{$subDirRel}"; // full path
558  $status->merge( $this->doClean( [ 'dir' => $subDir ] + $params ) );
559  }
560  unset( $subDirsRel ); // free directory for rmdir() on Windows (for FS backends)
561  }
562  }
563 
564  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
565  if ( $dir === null ) {
566  $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
567 
568  return $status; // invalid storage path
569  }
570 
571  // Attempt to lock this directory...
572  $filesLockEx = [ $params['dir'] ];
573  $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
574  if ( !$status->isOK() ) {
575  return $status; // abort
576  }
577 
578  if ( $shard !== null ) { // confined to a single container/shard
579  $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
580  $this->deleteContainerCache( $fullCont ); // purge cache
581  } else { // directory is on several shards
582  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
583  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
584  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
585  $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
586  $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache
587  }
588  }
589 
590  return $status;
591  }
592 
600  protected function doCleanInternal( $container, $dir, array $params ) {
601  return $this->newStatus();
602  }
603 
604  final public function fileExists( array $params ) {
605  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
606  $stat = $this->getFileStat( $params );
607 
608  return ( $stat === null ) ? null : (bool)$stat; // null => failure
609  }
610 
611  final public function getFileTimestamp( array $params ) {
612  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
613  $stat = $this->getFileStat( $params );
614 
615  return $stat ? $stat['mtime'] : false;
616  }
617 
618  final public function getFileSize( array $params ) {
619  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
620  $stat = $this->getFileStat( $params );
621 
622  return $stat ? $stat['size'] : false;
623  }
624 
625  final public function getFileStat( array $params ) {
626  $path = self::normalizeStoragePath( $params['src'] );
627  if ( $path === null ) {
628  return false; // invalid storage path
629  }
630  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
631  $latest = !empty( $params['latest'] ); // use latest data?
632  if ( !$latest && !$this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) {
633  $this->primeFileCache( [ $path ] ); // check persistent cache
634  }
635  if ( $this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) {
636  $stat = $this->cheapCache->get( $path, 'stat' );
637  // If we want the latest data, check that this cached
638  // value was in fact fetched with the latest available data.
639  if ( is_array( $stat ) ) {
640  if ( !$latest || $stat['latest'] ) {
641  return $stat;
642  }
643  } elseif ( in_array( $stat, [ 'NOT_EXIST', 'NOT_EXIST_LATEST' ] ) ) {
644  if ( !$latest || $stat === 'NOT_EXIST_LATEST' ) {
645  return false;
646  }
647  }
648  }
649  $stat = $this->doGetFileStat( $params );
650  if ( is_array( $stat ) ) { // file exists
651  // Strongly consistent backends can automatically set "latest"
652  $stat['latest'] = isset( $stat['latest'] ) ? $stat['latest'] : $latest;
653  $this->cheapCache->set( $path, 'stat', $stat );
654  $this->setFileCache( $path, $stat ); // update persistent cache
655  if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata
656  $this->cheapCache->set( $path, 'sha1',
657  [ 'hash' => $stat['sha1'], 'latest' => $latest ] );
658  }
659  if ( isset( $stat['xattr'] ) ) { // some backends store headers/metadata
660  $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] );
661  $this->cheapCache->set( $path, 'xattr',
662  [ 'map' => $stat['xattr'], 'latest' => $latest ] );
663  }
664  } elseif ( $stat === false ) { // file does not exist
665  $this->cheapCache->set( $path, 'stat', $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' );
666  $this->cheapCache->set( $path, 'xattr', [ 'map' => false, 'latest' => $latest ] );
667  $this->cheapCache->set( $path, 'sha1', [ 'hash' => false, 'latest' => $latest ] );
668  $this->logger->debug( __METHOD__ . ": File $path does not exist.\n" );
669  } else { // an error occurred
670  $this->logger->warning( __METHOD__ . ": Could not stat file $path.\n" );
671  }
672 
673  return $stat;
674  }
675 
680  abstract protected function doGetFileStat( array $params );
681 
682  public function getFileContentsMulti( array $params ) {
683  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
684 
685  $params = $this->setConcurrencyFlags( $params );
686  $contents = $this->doGetFileContentsMulti( $params );
687 
688  return $contents;
689  }
690 
696  protected function doGetFileContentsMulti( array $params ) {
697  $contents = [];
698  foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
699  MediaWiki\suppressWarnings();
700  $contents[$path] = $fsFile ? file_get_contents( $fsFile->getPath() ) : false;
701  MediaWiki\restoreWarnings();
702  }
703 
704  return $contents;
705  }
706 
707  final public function getFileXAttributes( array $params ) {
708  $path = self::normalizeStoragePath( $params['src'] );
709  if ( $path === null ) {
710  return false; // invalid storage path
711  }
712  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
713  $latest = !empty( $params['latest'] ); // use latest data?
714  if ( $this->cheapCache->has( $path, 'xattr', self::CACHE_TTL ) ) {
715  $stat = $this->cheapCache->get( $path, 'xattr' );
716  // If we want the latest data, check that this cached
717  // value was in fact fetched with the latest available data.
718  if ( !$latest || $stat['latest'] ) {
719  return $stat['map'];
720  }
721  }
722  $fields = $this->doGetFileXAttributes( $params );
723  $fields = is_array( $fields ) ? self::normalizeXAttributes( $fields ) : false;
724  $this->cheapCache->set( $path, 'xattr', [ 'map' => $fields, 'latest' => $latest ] );
725 
726  return $fields;
727  }
728 
734  protected function doGetFileXAttributes( array $params ) {
735  return [ 'headers' => [], 'metadata' => [] ]; // not supported
736  }
737 
738  final public function getFileSha1Base36( array $params ) {
739  $path = self::normalizeStoragePath( $params['src'] );
740  if ( $path === null ) {
741  return false; // invalid storage path
742  }
743  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
744  $latest = !empty( $params['latest'] ); // use latest data?
745  if ( $this->cheapCache->has( $path, 'sha1', self::CACHE_TTL ) ) {
746  $stat = $this->cheapCache->get( $path, 'sha1' );
747  // If we want the latest data, check that this cached
748  // value was in fact fetched with the latest available data.
749  if ( !$latest || $stat['latest'] ) {
750  return $stat['hash'];
751  }
752  }
753  $hash = $this->doGetFileSha1Base36( $params );
754  $this->cheapCache->set( $path, 'sha1', [ 'hash' => $hash, 'latest' => $latest ] );
755 
756  return $hash;
757  }
758 
764  protected function doGetFileSha1Base36( array $params ) {
765  $fsFile = $this->getLocalReference( $params );
766  if ( !$fsFile ) {
767  return false;
768  } else {
769  return $fsFile->getSha1Base36();
770  }
771  }
772 
773  final public function getFileProps( array $params ) {
774  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
775  $fsFile = $this->getLocalReference( $params );
776  $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
777 
778  return $props;
779  }
780 
781  final public function getLocalReferenceMulti( array $params ) {
782  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
783 
784  $params = $this->setConcurrencyFlags( $params );
785 
786  $fsFiles = []; // (path => FSFile)
787  $latest = !empty( $params['latest'] ); // use latest data?
788  // Reuse any files already in process cache...
789  foreach ( $params['srcs'] as $src ) {
790  $path = self::normalizeStoragePath( $src );
791  if ( $path === null ) {
792  $fsFiles[$src] = null; // invalid storage path
793  } elseif ( $this->expensiveCache->has( $path, 'localRef' ) ) {
794  $val = $this->expensiveCache->get( $path, 'localRef' );
795  // If we want the latest data, check that this cached
796  // value was in fact fetched with the latest available data.
797  if ( !$latest || $val['latest'] ) {
798  $fsFiles[$src] = $val['object'];
799  }
800  }
801  }
802  // Fetch local references of any remaning files...
803  $params['srcs'] = array_diff( $params['srcs'], array_keys( $fsFiles ) );
804  foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
805  $fsFiles[$path] = $fsFile;
806  if ( $fsFile ) { // update the process cache...
807  $this->expensiveCache->set( $path, 'localRef',
808  [ 'object' => $fsFile, 'latest' => $latest ] );
809  }
810  }
811 
812  return $fsFiles;
813  }
814 
820  protected function doGetLocalReferenceMulti( array $params ) {
821  return $this->doGetLocalCopyMulti( $params );
822  }
823 
824  final public function getLocalCopyMulti( array $params ) {
825  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
826 
827  $params = $this->setConcurrencyFlags( $params );
828  $tmpFiles = $this->doGetLocalCopyMulti( $params );
829 
830  return $tmpFiles;
831  }
832 
838  abstract protected function doGetLocalCopyMulti( array $params );
839 
845  public function getFileHttpUrl( array $params ) {
846  return null; // not supported
847  }
848 
849  final public function streamFile( array $params ) {
850  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
851  $status = $this->newStatus();
852 
853  // Always set some fields for subclass convenience
854  $params['options'] = isset( $params['options'] ) ? $params['options'] : [];
855  $params['headers'] = isset( $params['headers'] ) ? $params['headers'] : [];
856 
857  // Don't stream it out as text/html if there was a PHP error
858  if ( ( empty( $params['headless'] ) || $params['headers'] ) && headers_sent() ) {
859  print "Headers already sent, terminating.\n";
860  $status->fatal( 'backend-fail-stream', $params['src'] );
861  return $status;
862  }
863 
864  $status->merge( $this->doStreamFile( $params ) );
865 
866  return $status;
867  }
868 
874  protected function doStreamFile( array $params ) {
875  $status = $this->newStatus();
876 
877  $flags = 0;
878  $flags |= !empty( $params['headless'] ) ? HTTPFileStreamer::STREAM_HEADLESS : 0;
879  $flags |= !empty( $params['allowOB'] ) ? HTTPFileStreamer::STREAM_ALLOW_OB : 0;
880 
881  $fsFile = $this->getLocalReference( $params );
882  if ( $fsFile ) {
883  $streamer = new HTTPFileStreamer(
884  $fsFile->getPath(),
885  [
886  'obResetFunc' => $this->obResetFunc,
887  'streamMimeFunc' => $this->streamMimeFunc
888  ]
889  );
890  $res = $streamer->stream( $params['headers'], true, $params['options'], $flags );
891  } else {
892  $res = false;
893  HTTPFileStreamer::send404Message( $params['src'], $flags );
894  }
895 
896  if ( !$res ) {
897  $status->fatal( 'backend-fail-stream', $params['src'] );
898  }
899 
900  return $status;
901  }
902 
903  final public function directoryExists( array $params ) {
904  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
905  if ( $dir === null ) {
906  return false; // invalid storage path
907  }
908  if ( $shard !== null ) { // confined to a single container/shard
909  return $this->doDirectoryExists( $fullCont, $dir, $params );
910  } else { // directory is on several shards
911  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
912  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
913  $res = false; // response
914  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
915  $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params );
916  if ( $exists ) {
917  $res = true;
918  break; // found one!
919  } elseif ( $exists === null ) { // error?
920  $res = null; // if we don't find anything, it is indeterminate
921  }
922  }
923 
924  return $res;
925  }
926  }
927 
936  abstract protected function doDirectoryExists( $container, $dir, array $params );
937 
938  final public function getDirectoryList( array $params ) {
939  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
940  if ( $dir === null ) { // invalid storage path
941  return null;
942  }
943  if ( $shard !== null ) {
944  // File listing is confined to a single container/shard
945  return $this->getDirectoryListInternal( $fullCont, $dir, $params );
946  } else {
947  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
948  // File listing spans multiple containers/shards
949  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
950 
951  return new FileBackendStoreShardDirIterator( $this,
952  $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
953  }
954  }
955 
966  abstract public function getDirectoryListInternal( $container, $dir, array $params );
967 
968  final public function getFileList( array $params ) {
969  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
970  if ( $dir === null ) { // invalid storage path
971  return null;
972  }
973  if ( $shard !== null ) {
974  // File listing is confined to a single container/shard
975  return $this->getFileListInternal( $fullCont, $dir, $params );
976  } else {
977  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
978  // File listing spans multiple containers/shards
979  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
980 
981  return new FileBackendStoreShardFileIterator( $this,
982  $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
983  }
984  }
985 
996  abstract public function getFileListInternal( $container, $dir, array $params );
997 
1009  final public function getOperationsInternal( array $ops ) {
1010  $supportedOps = [
1011  'store' => 'StoreFileOp',
1012  'copy' => 'CopyFileOp',
1013  'move' => 'MoveFileOp',
1014  'delete' => 'DeleteFileOp',
1015  'create' => 'CreateFileOp',
1016  'describe' => 'DescribeFileOp',
1017  'null' => 'NullFileOp'
1018  ];
1019 
1020  $performOps = []; // array of FileOp objects
1021  // Build up ordered array of FileOps...
1022  foreach ( $ops as $operation ) {
1023  $opName = $operation['op'];
1024  if ( isset( $supportedOps[$opName] ) ) {
1025  $class = $supportedOps[$opName];
1026  // Get params for this operation
1027  $params = $operation;
1028  // Append the FileOp class
1029  $performOps[] = new $class( $this, $params, $this->logger );
1030  } else {
1031  throw new FileBackendError( "Operation '$opName' is not supported." );
1032  }
1033  }
1034 
1035  return $performOps;
1036  }
1037 
1048  final public function getPathsToLockForOpsInternal( array $performOps ) {
1049  // Build up a list of files to lock...
1050  $paths = [ 'sh' => [], 'ex' => [] ];
1051  foreach ( $performOps as $fileOp ) {
1052  $paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() );
1053  $paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() );
1054  }
1055  // Optimization: if doing an EX lock anyway, don't also set an SH one
1056  $paths['sh'] = array_diff( $paths['sh'], $paths['ex'] );
1057  // Get a shared lock on the parent directory of each path changed
1058  $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) );
1059 
1060  return [
1061  LockManager::LOCK_UW => $paths['sh'],
1062  LockManager::LOCK_EX => $paths['ex']
1063  ];
1064  }
1065 
1066  public function getScopedLocksForOps( array $ops, StatusValue $status ) {
1067  $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) );
1068 
1069  return $this->getScopedFileLocks( $paths, 'mixed', $status );
1070  }
1071 
1072  final protected function doOperationsInternal( array $ops, array $opts ) {
1073  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1074  $status = $this->newStatus();
1075 
1076  // Fix up custom header name/value pairs...
1077  $ops = array_map( [ $this, 'sanitizeOpHeaders' ], $ops );
1078 
1079  // Build up a list of FileOps...
1080  $performOps = $this->getOperationsInternal( $ops );
1081 
1082  // Acquire any locks as needed...
1083  if ( empty( $opts['nonLocking'] ) ) {
1084  // Build up a list of files to lock...
1085  $paths = $this->getPathsToLockForOpsInternal( $performOps );
1086  // Try to lock those files for the scope of this function...
1087 
1088  $scopeLock = $this->getScopedFileLocks( $paths, 'mixed', $status );
1089  if ( !$status->isOK() ) {
1090  return $status; // abort
1091  }
1092  }
1093 
1094  // Clear any file cache entries (after locks acquired)
1095  if ( empty( $opts['preserveCache'] ) ) {
1096  $this->clearCache();
1097  }
1098 
1099  // Build the list of paths involved
1100  $paths = [];
1101  foreach ( $performOps as $performOp ) {
1102  $paths = array_merge( $paths, $performOp->storagePathsRead() );
1103  $paths = array_merge( $paths, $performOp->storagePathsChanged() );
1104  }
1105 
1106  // Enlarge the cache to fit the stat entries of these files
1107  $this->cheapCache->resize( max( 2 * count( $paths ), self::CACHE_CHEAP_SIZE ) );
1108 
1109  // Load from the persistent container caches
1110  $this->primeContainerCache( $paths );
1111  // Get the latest stat info for all the files (having locked them)
1112  $ok = $this->preloadFileStat( [ 'srcs' => $paths, 'latest' => true ] );
1113 
1114  if ( $ok ) {
1115  // Actually attempt the operation batch...
1116  $opts = $this->setConcurrencyFlags( $opts );
1117  $subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal );
1118  } else {
1119  // If we could not even stat some files, then bail out...
1120  $subStatus = $this->newStatus( 'backend-fail-internal', $this->name );
1121  foreach ( $ops as $i => $op ) { // mark each op as failed
1122  $subStatus->success[$i] = false;
1123  ++$subStatus->failCount;
1124  }
1125  $this->logger->error( static::class . "-{$this->name} " .
1126  " stat failure; aborted operations: " . FormatJson::encode( $ops ) );
1127  }
1128 
1129  // Merge errors into StatusValue fields
1130  $status->merge( $subStatus );
1131  $status->success = $subStatus->success; // not done in merge()
1132 
1133  // Shrink the stat cache back to normal size
1134  $this->cheapCache->resize( self::CACHE_CHEAP_SIZE );
1135 
1136  return $status;
1137  }
1138 
1139  final protected function doQuickOperationsInternal( array $ops ) {
1140  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1141  $status = $this->newStatus();
1142 
1143  // Fix up custom header name/value pairs...
1144  $ops = array_map( [ $this, 'sanitizeOpHeaders' ], $ops );
1145 
1146  // Clear any file cache entries
1147  $this->clearCache();
1148 
1149  $supportedOps = [ 'create', 'store', 'copy', 'move', 'delete', 'describe', 'null' ];
1150  // Parallel ops may be disabled in config due to dependencies (e.g. needing popen())
1151  $async = ( $this->parallelize === 'implicit' && count( $ops ) > 1 );
1152  $maxConcurrency = $this->concurrency; // throttle
1154  $statuses = []; // array of (index => StatusValue)
1155  $fileOpHandles = []; // list of (index => handle) arrays
1156  $curFileOpHandles = []; // current handle batch
1157  // Perform the sync-only ops and build up op handles for the async ops...
1158  foreach ( $ops as $index => $params ) {
1159  if ( !in_array( $params['op'], $supportedOps ) ) {
1160  throw new FileBackendError( "Operation '{$params['op']}' is not supported." );
1161  }
1162  $method = $params['op'] . 'Internal'; // e.g. "storeInternal"
1163  $subStatus = $this->$method( [ 'async' => $async ] + $params );
1164  if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { // async
1165  if ( count( $curFileOpHandles ) >= $maxConcurrency ) {
1166  $fileOpHandles[] = $curFileOpHandles; // push this batch
1167  $curFileOpHandles = [];
1168  }
1169  $curFileOpHandles[$index] = $subStatus->value; // keep index
1170  } else { // error or completed
1171  $statuses[$index] = $subStatus; // keep index
1172  }
1173  }
1174  if ( count( $curFileOpHandles ) ) {
1175  $fileOpHandles[] = $curFileOpHandles; // last batch
1176  }
1177  // Do all the async ops that can be done concurrently...
1178  foreach ( $fileOpHandles as $fileHandleBatch ) {
1179  $statuses = $statuses + $this->executeOpHandlesInternal( $fileHandleBatch );
1180  }
1181  // Marshall and merge all the responses...
1182  foreach ( $statuses as $index => $subStatus ) {
1183  $status->merge( $subStatus );
1184  if ( $subStatus->isOK() ) {
1185  $status->success[$index] = true;
1186  ++$status->successCount;
1187  } else {
1188  $status->success[$index] = false;
1189  ++$status->failCount;
1190  }
1191  }
1192 
1193  return $status;
1194  }
1195 
1205  final public function executeOpHandlesInternal( array $fileOpHandles ) {
1206  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1207 
1208  foreach ( $fileOpHandles as $fileOpHandle ) {
1209  if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) {
1210  throw new InvalidArgumentException( "Expected FileBackendStoreOpHandle object." );
1211  } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) {
1212  throw new InvalidArgumentException( "Expected handle for this file backend." );
1213  }
1214  }
1215 
1216  $res = $this->doExecuteOpHandlesInternal( $fileOpHandles );
1217  foreach ( $fileOpHandles as $fileOpHandle ) {
1218  $fileOpHandle->closeResources();
1219  }
1220 
1221  return $res;
1222  }
1223 
1232  protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1233  if ( count( $fileOpHandles ) ) {
1234  throw new LogicException( "Backend does not support asynchronous operations." );
1235  }
1236 
1237  return [];
1238  }
1239 
1251  protected function sanitizeOpHeaders( array $op ) {
1252  static $longs = [ 'content-disposition' ];
1253 
1254  if ( isset( $op['headers'] ) ) { // op sets HTTP headers
1255  $newHeaders = [];
1256  foreach ( $op['headers'] as $name => $value ) {
1257  $name = strtolower( $name );
1258  $maxHVLen = in_array( $name, $longs ) ? INF : 255;
1259  if ( strlen( $name ) > 255 || strlen( $value ) > $maxHVLen ) {
1260  trigger_error( "Header '$name: $value' is too long." );
1261  } else {
1262  $newHeaders[$name] = strlen( $value ) ? $value : ''; // null/false => ""
1263  }
1264  }
1265  $op['headers'] = $newHeaders;
1266  }
1267 
1268  return $op;
1269  }
1270 
1271  final public function preloadCache( array $paths ) {
1272  $fullConts = []; // full container names
1273  foreach ( $paths as $path ) {
1274  list( $fullCont, , ) = $this->resolveStoragePath( $path );
1275  $fullConts[] = $fullCont;
1276  }
1277  // Load from the persistent file and container caches
1278  $this->primeContainerCache( $fullConts );
1279  $this->primeFileCache( $paths );
1280  }
1281 
1282  final public function clearCache( array $paths = null ) {
1283  if ( is_array( $paths ) ) {
1284  $paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
1285  $paths = array_filter( $paths, 'strlen' ); // remove nulls
1286  }
1287  if ( $paths === null ) {
1288  $this->cheapCache->clear();
1289  $this->expensiveCache->clear();
1290  } else {
1291  foreach ( $paths as $path ) {
1292  $this->cheapCache->clear( $path );
1293  $this->expensiveCache->clear( $path );
1294  }
1295  }
1296  $this->doClearCache( $paths );
1297  }
1298 
1306  protected function doClearCache( array $paths = null ) {
1307  }
1308 
1309  final public function preloadFileStat( array $params ) {
1310  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1311  $success = true; // no network errors
1312 
1313  $params['concurrency'] = ( $this->parallelize !== 'off' ) ? $this->concurrency : 1;
1314  $stats = $this->doGetFileStatMulti( $params );
1315  if ( $stats === null ) {
1316  return true; // not supported
1317  }
1318 
1319  $latest = !empty( $params['latest'] ); // use latest data?
1320  foreach ( $stats as $path => $stat ) {
1322  if ( $path === null ) {
1323  continue; // this shouldn't happen
1324  }
1325  if ( is_array( $stat ) ) { // file exists
1326  // Strongly consistent backends can automatically set "latest"
1327  $stat['latest'] = isset( $stat['latest'] ) ? $stat['latest'] : $latest;
1328  $this->cheapCache->set( $path, 'stat', $stat );
1329  $this->setFileCache( $path, $stat ); // update persistent cache
1330  if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata
1331  $this->cheapCache->set( $path, 'sha1',
1332  [ 'hash' => $stat['sha1'], 'latest' => $latest ] );
1333  }
1334  if ( isset( $stat['xattr'] ) ) { // some backends store headers/metadata
1335  $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] );
1336  $this->cheapCache->set( $path, 'xattr',
1337  [ 'map' => $stat['xattr'], 'latest' => $latest ] );
1338  }
1339  } elseif ( $stat === false ) { // file does not exist
1340  $this->cheapCache->set( $path, 'stat',
1341  $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' );
1342  $this->cheapCache->set( $path, 'xattr',
1343  [ 'map' => false, 'latest' => $latest ] );
1344  $this->cheapCache->set( $path, 'sha1',
1345  [ 'hash' => false, 'latest' => $latest ] );
1346  $this->logger->debug( __METHOD__ . ": File $path does not exist.\n" );
1347  } else { // an error occurred
1348  $success = false;
1349  $this->logger->warning( __METHOD__ . ": Could not stat file $path.\n" );
1350  }
1351  }
1352 
1353  return $success;
1354  }
1355 
1367  protected function doGetFileStatMulti( array $params ) {
1368  return null; // not supported
1369  }
1370 
1378  abstract protected function directoriesAreVirtual();
1379 
1390  final protected static function isValidShortContainerName( $container ) {
1391  // Suffixes like '.xxx' (hex shard chars) or '.seg' (file segments)
1392  // might be used by subclasses. Reserve the dot character for sanity.
1393  // The only way dots end up in containers (e.g. resolveStoragePath)
1394  // is due to the wikiId container prefix or the above suffixes.
1395  return self::isValidContainerName( $container ) && !preg_match( '/[.]/', $container );
1396  }
1397 
1407  final protected static function isValidContainerName( $container ) {
1408  // This accounts for NTFS, Swift, and Ceph restrictions
1409  // and disallows directory separators or traversal characters.
1410  // Note that matching strings URL encode to the same string;
1411  // in Swift/Ceph, the length restriction is *after* URL encoding.
1412  return (bool)preg_match( '/^[a-z0-9][a-z0-9-_.]{0,199}$/i', $container );
1413  }
1414 
1428  final protected function resolveStoragePath( $storagePath ) {
1429  list( $backend, $shortCont, $relPath ) = self::splitStoragePath( $storagePath );
1430  if ( $backend === $this->name ) { // must be for this backend
1431  $relPath = self::normalizeContainerPath( $relPath );
1432  if ( $relPath !== null && self::isValidShortContainerName( $shortCont ) ) {
1433  // Get shard for the normalized path if this container is sharded
1434  $cShard = $this->getContainerShard( $shortCont, $relPath );
1435  // Validate and sanitize the relative path (backend-specific)
1436  $relPath = $this->resolveContainerPath( $shortCont, $relPath );
1437  if ( $relPath !== null ) {
1438  // Prepend any wiki ID prefix to the container name
1439  $container = $this->fullContainerName( $shortCont );
1440  if ( self::isValidContainerName( $container ) ) {
1441  // Validate and sanitize the container name (backend-specific)
1442  $container = $this->resolveContainerName( "{$container}{$cShard}" );
1443  if ( $container !== null ) {
1444  return [ $container, $relPath, $cShard ];
1445  }
1446  }
1447  }
1448  }
1449  }
1450 
1451  return [ null, null, null ];
1452  }
1453 
1469  final protected function resolveStoragePathReal( $storagePath ) {
1470  list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
1471  if ( $cShard !== null && substr( $relPath, -1 ) !== '/' ) {
1472  return [ $container, $relPath ];
1473  }
1474 
1475  return [ null, null ];
1476  }
1477 
1486  final protected function getContainerShard( $container, $relPath ) {
1487  list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container );
1488  if ( $levels == 1 || $levels == 2 ) {
1489  // Hash characters are either base 16 or 36
1490  $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]';
1491  // Get a regex that represents the shard portion of paths.
1492  // The concatenation of the captures gives us the shard.
1493  if ( $levels === 1 ) { // 16 or 36 shards per container
1494  $hashDirRegex = '(' . $char . ')';
1495  } else { // 256 or 1296 shards per container
1496  if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc")
1497  $hashDirRegex = $char . '/(' . $char . '{2})';
1498  } else { // short hash dir format (e.g. "a/b/c")
1499  $hashDirRegex = '(' . $char . ')/(' . $char . ')';
1500  }
1501  }
1502  // Allow certain directories to be above the hash dirs so as
1503  // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
1504  // They must be 2+ chars to avoid any hash directory ambiguity.
1505  $m = [];
1506  if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1507  return '.' . implode( '', array_slice( $m, 1 ) );
1508  }
1509 
1510  return null; // failed to match
1511  }
1512 
1513  return ''; // no sharding
1514  }
1515 
1524  final public function isSingleShardPathInternal( $storagePath ) {
1525  list( , , $shard ) = $this->resolveStoragePath( $storagePath );
1526 
1527  return ( $shard !== null );
1528  }
1529 
1538  final protected function getContainerHashLevels( $container ) {
1539  if ( isset( $this->shardViaHashLevels[$container] ) ) {
1540  $config = $this->shardViaHashLevels[$container];
1541  $hashLevels = (int)$config['levels'];
1542  if ( $hashLevels == 1 || $hashLevels == 2 ) {
1543  $hashBase = (int)$config['base'];
1544  if ( $hashBase == 16 || $hashBase == 36 ) {
1545  return [ $hashLevels, $hashBase, $config['repeat'] ];
1546  }
1547  }
1548  }
1549 
1550  return [ 0, 0, false ]; // no sharding
1551  }
1552 
1559  final protected function getContainerSuffixes( $container ) {
1560  $shards = [];
1561  list( $digits, $base ) = $this->getContainerHashLevels( $container );
1562  if ( $digits > 0 ) {
1563  $numShards = pow( $base, $digits );
1564  for ( $index = 0; $index < $numShards; $index++ ) {
1565  $shards[] = '.' . Wikimedia\base_convert( $index, 10, $base, $digits );
1566  }
1567  }
1568 
1569  return $shards;
1570  }
1571 
1578  final protected function fullContainerName( $container ) {
1579  if ( $this->domainId != '' ) {
1580  return "{$this->domainId}-$container";
1581  } else {
1582  return $container;
1583  }
1584  }
1585 
1594  protected function resolveContainerName( $container ) {
1595  return $container;
1596  }
1597 
1608  protected function resolveContainerPath( $container, $relStoragePath ) {
1609  return $relStoragePath;
1610  }
1611 
1618  private function containerCacheKey( $container ) {
1619  return "filebackend:{$this->name}:{$this->domainId}:container:{$container}";
1620  }
1621 
1628  final protected function setContainerCache( $container, array $val ) {
1629  $this->memCache->set( $this->containerCacheKey( $container ), $val, 14 * 86400 );
1630  }
1631 
1638  final protected function deleteContainerCache( $container ) {
1639  if ( !$this->memCache->delete( $this->containerCacheKey( $container ), 300 ) ) {
1640  trigger_error( "Unable to delete stat cache for container $container." );
1641  }
1642  }
1643 
1651  final protected function primeContainerCache( array $items ) {
1652  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1653 
1654  $paths = []; // list of storage paths
1655  $contNames = []; // (cache key => resolved container name)
1656  // Get all the paths/containers from the items...
1657  foreach ( $items as $item ) {
1658  if ( self::isStoragePath( $item ) ) {
1659  $paths[] = $item;
1660  } elseif ( is_string( $item ) ) { // full container name
1661  $contNames[$this->containerCacheKey( $item )] = $item;
1662  }
1663  }
1664  // Get all the corresponding cache keys for paths...
1665  foreach ( $paths as $path ) {
1666  list( $fullCont, , ) = $this->resolveStoragePath( $path );
1667  if ( $fullCont !== null ) { // valid path for this backend
1668  $contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
1669  }
1670  }
1671 
1672  $contInfo = []; // (resolved container name => cache value)
1673  // Get all cache entries for these container cache keys...
1674  $values = $this->memCache->getMulti( array_keys( $contNames ) );
1675  foreach ( $values as $cacheKey => $val ) {
1676  $contInfo[$contNames[$cacheKey]] = $val;
1677  }
1678 
1679  // Populate the container process cache for the backend...
1680  $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) );
1681  }
1682 
1690  protected function doPrimeContainerCache( array $containerInfo ) {
1691  }
1692 
1699  private function fileCacheKey( $path ) {
1700  return "filebackend:{$this->name}:{$this->domainId}:file:" . sha1( $path );
1701  }
1702 
1711  final protected function setFileCache( $path, array $val ) {
1713  if ( $path === null ) {
1714  return; // invalid storage path
1715  }
1716  $mtime = ConvertibleTimestamp::convert( TS_UNIX, $val['mtime'] );
1717  $ttl = $this->memCache->adaptiveTTL( $mtime, 7 * 86400, 300, 0.1 );
1718  $key = $this->fileCacheKey( $path );
1719  // Set the cache unless it is currently salted.
1720  $this->memCache->set( $key, $val, $ttl );
1721  }
1722 
1731  final protected function deleteFileCache( $path ) {
1733  if ( $path === null ) {
1734  return; // invalid storage path
1735  }
1736  if ( !$this->memCache->delete( $this->fileCacheKey( $path ), 300 ) ) {
1737  trigger_error( "Unable to delete stat cache for file $path." );
1738  }
1739  }
1740 
1748  final protected function primeFileCache( array $items ) {
1749  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1750 
1751  $paths = []; // list of storage paths
1752  $pathNames = []; // (cache key => storage path)
1753  // Get all the paths/containers from the items...
1754  foreach ( $items as $item ) {
1755  if ( self::isStoragePath( $item ) ) {
1756  $paths[] = FileBackend::normalizeStoragePath( $item );
1757  }
1758  }
1759  // Get rid of any paths that failed normalization...
1760  $paths = array_filter( $paths, 'strlen' ); // remove nulls
1761  // Get all the corresponding cache keys for paths...
1762  foreach ( $paths as $path ) {
1763  list( , $rel, ) = $this->resolveStoragePath( $path );
1764  if ( $rel !== null ) { // valid path for this backend
1765  $pathNames[$this->fileCacheKey( $path )] = $path;
1766  }
1767  }
1768  // Get all cache entries for these file cache keys...
1769  $values = $this->memCache->getMulti( array_keys( $pathNames ) );
1770  foreach ( $values as $cacheKey => $val ) {
1771  $path = $pathNames[$cacheKey];
1772  if ( is_array( $val ) ) {
1773  $val['latest'] = false; // never completely trust cache
1774  $this->cheapCache->set( $path, 'stat', $val );
1775  if ( isset( $val['sha1'] ) ) { // some backends store SHA-1 as metadata
1776  $this->cheapCache->set( $path, 'sha1',
1777  [ 'hash' => $val['sha1'], 'latest' => false ] );
1778  }
1779  if ( isset( $val['xattr'] ) ) { // some backends store headers/metadata
1780  $val['xattr'] = self::normalizeXAttributes( $val['xattr'] );
1781  $this->cheapCache->set( $path, 'xattr',
1782  [ 'map' => $val['xattr'], 'latest' => false ] );
1783  }
1784  }
1785  }
1786  }
1787 
1795  final protected static function normalizeXAttributes( array $xattr ) {
1796  $newXAttr = [ 'headers' => [], 'metadata' => [] ];
1797 
1798  foreach ( $xattr['headers'] as $name => $value ) {
1799  $newXAttr['headers'][strtolower( $name )] = $value;
1800  }
1801 
1802  foreach ( $xattr['metadata'] as $name => $value ) {
1803  $newXAttr['metadata'][strtolower( $name )] = $value;
1804  }
1805 
1806  return $newXAttr;
1807  }
1808 
1815  final protected function setConcurrencyFlags( array $opts ) {
1816  $opts['concurrency'] = 1; // off
1817  if ( $this->parallelize === 'implicit' ) {
1818  if ( !isset( $opts['parallelize'] ) || $opts['parallelize'] ) {
1819  $opts['concurrency'] = $this->concurrency;
1820  }
1821  } elseif ( $this->parallelize === 'explicit' ) {
1822  if ( !empty( $opts['parallelize'] ) ) {
1823  $opts['concurrency'] = $this->concurrency;
1824  }
1825  }
1826 
1827  return $opts;
1828  }
1829 
1838  protected function getContentType( $storagePath, $content, $fsPath ) {
1839  if ( $this->mimeCallback ) {
1840  return call_user_func_array( $this->mimeCallback, func_get_args() );
1841  }
1842 
1843  $mime = ( $fsPath !== null ) ? mime_content_type( $fsPath ) : false;
1844  return $mime ?: 'unknown/unknown';
1845  }
1846 }
1847 
1858  public $params = []; // params to caller functions
1860  public $backend;
1862  public $resourcesToClose = [];
1863 
1864  public $call; // string; name that identifies the function called
1865 
1869  public function closeResources() {
1870  array_map( 'fclose', $this->resourcesToClose );
1871  }
1872 }
1873 
1880 abstract class FileBackendStoreShardListIterator extends FilterIterator {
1882  protected $backend;
1883 
1885  protected $params;
1886 
1888  protected $container;
1889 
1891  protected $directory;
1892 
1894  protected $multiShardPaths = []; // (rel path => 1)
1895 
1903  public function __construct(
1905  ) {
1906  $this->backend = $backend;
1907  $this->container = $container;
1908  $this->directory = $dir;
1909  $this->params = $params;
1910 
1911  $iter = new AppendIterator();
1912  foreach ( $suffixes as $suffix ) {
1913  $iter->append( $this->listFromShard( $this->container . $suffix ) );
1914  }
1915 
1916  parent::__construct( $iter );
1917  }
1918 
1919  public function accept() {
1920  $rel = $this->getInnerIterator()->current(); // path relative to given directory
1921  $path = $this->params['dir'] . "/{$rel}"; // full storage path
1922  if ( $this->backend->isSingleShardPathInternal( $path ) ) {
1923  return true; // path is only on one shard; no issue with duplicates
1924  } elseif ( isset( $this->multiShardPaths[$rel] ) ) {
1925  // Don't keep listing paths that are on multiple shards
1926  return false;
1927  } else {
1928  $this->multiShardPaths[$rel] = 1;
1929 
1930  return true;
1931  }
1932  }
1933 
1934  public function rewind() {
1935  parent::rewind();
1936  $this->multiShardPaths = [];
1937  }
1938 
1945  abstract protected function listFromShard( $container );
1946 }
1947 
1952  protected function listFromShard( $container ) {
1953  $list = $this->backend->getDirectoryListInternal(
1954  $container, $this->directory, $this->params );
1955  if ( $list === null ) {
1956  return new ArrayIterator( [] );
1957  } else {
1958  return is_array( $list ) ? new ArrayIterator( $list ) : $list;
1959  }
1960  }
1961 }
1962 
1967  protected function listFromShard( $container ) {
1968  $list = $this->backend->getFileListInternal(
1969  $container, $this->directory, $this->params );
1970  if ( $list === null ) {
1971  return new ArrayIterator( [] );
1972  } else {
1973  return is_array( $list ) ? new ArrayIterator( $list ) : $list;
1974  }
1975  }
1976 }
newStatus()
Yields the result of the status wrapper callback on either:
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
doPublishInternal($container, $dir, array $params)
getFileHttpUrl(array $params)
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition: deferred.txt:11
doOperationsInternal(array $ops, array $opts)
setConcurrencyFlags(array $opts)
Set the 'concurrency' option from a list of operation options.
the array() calling protocol came about after MediaWiki 1.4rc1.
magic word the default is to use $key to get the and $key value or $key value text $key value html to format the value $key
Definition: hooks.txt:2558
$success
setFileCache($path, array $val)
Set the cached stat info for a file path.
Iterator for listing regular files.
moveInternal(array $params)
Move a file from one storage path to another in the backend.
concatenate(array $params)
doPrepareInternal($container, $dir, array $params)
doGetLocalReferenceMulti(array $params)
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
resolveContainerName($container)
Resolve a container name, checking if it's allowed by the backend.
getPathsToLockForOpsInternal(array $performOps)
Get a list of storage paths to lock for a list of operations Returns an array with LockManager::LOCK_...
scopedProfileSection($section)
static normalizeXAttributes(array $xattr)
Normalize file headers/metadata to the FileBackend::getFileXAttributes() format.
processing should stop and the error should be shown to the user * false
Definition: hooks.txt:187
listFromShard($container)
Get the list for a given container shard.
doGetFileXAttributes(array $params)
getFileStat(array $params)
getContainerSuffixes($container)
Get a list of full container shard suffixes for a container.
$value
The most up to date schema for the tables in the database will always be tables sql in the maintenance directory
Definition: schema.txt:2
__construct(array $config)
if($ext== 'php'||$ext== 'php5') $mime
Definition: router.php:65
getName()
Get the unique backend name.
string $directory
Resolved relative path.
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
it s the revision text itself In either if gzip is the revision text is gzipped $flags
Definition: hooks.txt:2805
string $container
Full container name.
ProcessCacheLRU $expensiveCache
Map of paths to large (RAM/disk) cache items.
fileCacheKey($path)
Get the cache key for a file path.
FileBackendStore helper function to handle listings that span container shards.
static isValidContainerName($container)
Check if a full container name is valid.
int $concurrency
How many operations can be done in parallel.
storeInternal(array $params)
Store a file into the backend from a file on disk.
array $shardViaHashLevels
Map of container names to sharding config.
doSecureInternal($container, $dir, array $params)
doCreateInternal(array $params)
static newEmpty()
Get an instance that wraps EmptyBagOStuff.
WANObjectCache $memCache
streamFile(array $params)
getFileSha1Base36(array $params)
const LOCK_UW
Definition: LockManager.php:68
doClean(array $params)
executeOpHandlesInternal(array $fileOpHandles)
Execute a list of FileBackendStoreOpHandle handles in parallel.
getFileSize(array $params)
doQuickOperationsInternal(array $ops)
File backend exception for checked exceptions (e.g.
isPathUsableInternal($storagePath)
Check if a file can be created or changed at a given storage path.
Functions related to the output of file content.
ProcessCacheLRU $cheapCache
Map of paths to small (RAM/disk) cache items.
createInternal(array $params)
Create a file in the backend with the given contents.
fileExists(array $params)
directoryExists(array $params)
const LOCK_EX
Definition: LockManager.php:69
$res
Definition: database.txt:21
doStreamFile(array $params)
getFileXAttributes(array $params)
nullInternal(array $params)
No-op file operation that does nothing.
static encode($value, $pretty=false, $escaping=0)
Returns the JSON representation of a value.
Definition: FormatJson.php:127
deleteInternal(array $params)
Delete a file at the storage path.
preloadCache(array $paths)
$params
callable $obResetFunc
doGetLocalCopyMulti(array $params)
getFileContentsMulti(array $params)
getFileTimestamp(array $params)
maxFileSizeInternal()
Get the maximum allowable file size given backend medium restrictions and basic performance constrain...
static placeholderProps()
Placeholder file properties to use for files that don't exist.
Definition: FSFile.php:143
getOperationsInternal(array $ops)
Return a list of FileOp objects from a list of operations.
callable $streamMimeFunc
clearCache(array $paths=null)
callable $mimeCallback
Method to get the MIME type of files.
resolveStoragePathReal($storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
doPrepare(array $params)
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
doGetFileStatMulti(array $params)
Get file stat information (concurrently if possible) for several files.
getContentType($storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
copyInternal(array $params)
Copy a file from one storage path to another in the backend.
doGetFileStat(array $params)
string $name
Unique backend name.
Definition: FileBackend.php:94
doGetFileContentsMulti(array $params)
Base class for all backends using particular storage medium.
preloadFileStat(array $params)
fullContainerName($container)
Get the full container name, including the wiki ID prefix.
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
getFileList(array $params)
closeResources()
Close all open file handles.
__construct(FileBackendStore $backend, $container, $dir, array $suffixes, array $params)
containerCacheKey($container)
Get the cache key for a container.
Iterator for listing directories.
getContainerHashLevels($container)
Get the sharding config for a container.
static normalizeStoragePath($storagePath)
Normalize a storage path by cleaning up directory separators.
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
doSecure(array $params)
static attempt(array $performOps, array $opts, FileJournal $journal)
Attempt to perform a series of file operations.
Definition: FileOpBatch.php:56
resolveContainerPath($container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
setContainerCache($container, array $val)
Set the cached info for a container.
doConcatenate(array $params)
getContainerShard($container, $relPath)
Get the container name shard suffix for a given path.
getDirectoryListInternal($container, $dir, array $params)
Do not call this function from places outside FileBackend.
doPublish(array $params)
you have access to all of the normal MediaWiki so you can get a DB use the etc For full docs on the Maintenance class
Definition: maintenance.txt:52
doPrimeContainerCache(array $containerInfo)
Fill the backend-specific process cache given an array of resolved container names and their correspo...
static send404Message($fname, $flags=0)
Send out a standard 404 message for a file.
doClearCache(array $paths=null)
Clears any additional stat caches for storage paths.
describeInternal(array $params)
Alter metadata for a file at the storage path.
design txt This is a brief overview of the new design More thorough and up to date information is available on the documentation wiki at name
Definition: design.txt:12
doStoreInternal(array $params)
isSingleShardPathInternal($storagePath)
Check if a storage path maps to a single shard.
Base class for all file backend classes (including multi-write backends).
Definition: FileBackend.php:92
doDeleteInternal(array $params)
getTopDirectoryList(array $params)
Same as FileBackend::getDirectoryList() except only lists directories that are immediately under the ...
Status::newGood()`to allow deletion, and then`return false`from the hook function.Ensure you consume the 'ChangeTagAfterDelete'hook to carry out custom deletion actions.$tag:name of the tag $user:user initiating the action &$status:Status object.See above. 'ChangeTagsListActive':Allows you to nominate which of the tags your extension uses are in active use.&$tags:list of all active tags.Append to this array. 'ChangeTagsAfterUpdateTags':Called after tags have been updated with the ChangeTags::updateTags function.Params:$addedTags:tags effectively added in the update $removedTags:tags effectively removed in the update $prevTags:tags that were present prior to the update $rc_id:recentchanges table id $rev_id:revision table id $log_id:logging table id $params:tag params $rc:RecentChange being tagged when the tagging accompanies the action or null $user:User who performed the tagging when the tagging is subsequent to the action or null 'ChangeTagsAllowedAdd':Called when checking if a user can add tags to a change.&$allowedTags:List of all the tags the user is allowed to add.Any tags the user wants to add($addTags) that are not in this array will cause it to fail.You may add or remove tags to this array as required.$addTags:List of tags user intends to add.$user:User who is adding the tags. 'ChangeUserGroups':Called before user groups are changed.$performer:The User who will perform the change $user:The User whose groups will be changed &$add:The groups that will be added &$remove:The groups that will be removed 'Collation::factory':Called if $wgCategoryCollation is an unknown collation.$collationName:Name of the collation in question &$collationObject:Null.Replace with a subclass of the Collation class that implements the collation given in $collationName. 'ConfirmEmailComplete':Called after a user's email has been confirmed successfully.$user:user(object) whose email is being confirmed 'ContentAlterParserOutput':Modify parser output for a given content object.Called by Content::getParserOutput after parsing has finished.Can be used for changes that depend on the result of the parsing but have to be done before LinksUpdate is called(such as adding tracking categories based on the rendered HTML).$content:The Content to render $title:Title of the page, as context $parserOutput:ParserOutput to manipulate 'ContentGetParserOutput':Customize parser output for a given content object, called by AbstractContent::getParserOutput.May be used to override the normal model-specific rendering of page content.$content:The Content to render $title:Title of the page, as context $revId:The revision ID, as context $options:ParserOptions for rendering.To avoid confusing the parser cache, the output can only depend on parameters provided to this hook function, not on global state.$generateHtml:boolean, indicating whether full HTML should be generated.If false, generation of HTML may be skipped, but other information should still be present in the ParserOutput object.&$output:ParserOutput, to manipulate or replace 'ContentHandlerDefaultModelFor':Called when the default content model is determined for a given title.May be used to assign a different model for that title.$title:the Title in question &$model:the model name.Use with CONTENT_MODEL_XXX constants. 'ContentHandlerForModelID':Called when a ContentHandler is requested for a given content model name, but no entry for that model exists in $wgContentHandlers.Note:if your extension implements additional models via this hook, please use GetContentModels hook to make them known to core.$modeName:the requested content model name &$handler:set this to a ContentHandler object, if desired. 'ContentModelCanBeUsedOn':Called to determine whether that content model can be used on a given page.This is especially useful to prevent some content models to be used in some special location.$contentModel:ID of the content model in question $title:the Title in question.&$ok:Output parameter, whether it is OK to use $contentModel on $title.Handler functions that modify $ok should generally return false to prevent further hooks from further modifying $ok. 'ContribsPager::getQueryInfo':Before the contributions query is about to run &$pager:Pager object for contributions &$queryInfo:The query for the contribs Pager 'ContribsPager::reallyDoQuery':Called before really executing the query for My Contributions &$data:an array of results of all contribs queries $pager:The ContribsPager object hooked into $offset:Index offset, inclusive $limit:Exact query limit $descending:Query direction, false for ascending, true for descending 'ContributionsLineEnding':Called before a contributions HTML line is finished $page:SpecialPage object for contributions &$ret:the HTML line $row:the DB row for this line &$classes:the classes to add to the surrounding< li > &$attribs:associative array of other HTML attributes for the< li > element.Currently only data attributes reserved to MediaWiki are allowed(see Sanitizer::isReservedDataAttribute). 'ContributionsToolLinks':Change tool links above Special:Contributions $id:User identifier $title:User page title &$tools:Array of tool links $specialPage:SpecialPage instance for context and services.Can be either SpecialContributions or DeletedContributionsPage.Extensions should type hint against a generic SpecialPage though. 'ConvertContent':Called by AbstractContent::convert when a conversion to another content model is requested.Handler functions that modify $result should generally return false to disable further attempts at conversion.$content:The Content object to be converted.$toModel:The ID of the content model to convert to.$lossy:boolean indicating whether lossy conversion is allowed.&$result:Output parameter, in case the handler function wants to provide a converted Content object.Note that $result->getContentModel() must return $toModel. 'CustomEditor':When invoking the page editor Return true to allow the normal editor to be used, or false if implementing a custom editor, e.g.for a special namespace, etc.$article:Article being edited $user:User performing the edit 'DatabaseOraclePostInit':Called after initialising an Oracle database $db:the DatabaseOracle object 'DeletedContribsPager::reallyDoQuery':Called before really executing the query for Special:DeletedContributions Similar to ContribsPager::reallyDoQuery &$data:an array of results of all contribs queries $pager:The DeletedContribsPager object hooked into $offset:Index offset, inclusive $limit:Exact query limit $descending:Query direction, false for ascending, true for descending 'DeletedContributionsLineEnding':Called before a DeletedContributions HTML line is finished.Similar to ContributionsLineEnding $page:SpecialPage object for DeletedContributions &$ret:the HTML line $row:the DB row for this line &$classes:the classes to add to the surrounding< li > &$attribs:associative array of other HTML attributes for the< li > element.Currently only data attributes reserved to MediaWiki are allowed(see Sanitizer::isReservedDataAttribute). 'DifferenceEngineAfterLoadNewText':called in DifferenceEngine::loadNewText() after the new revision's content has been loaded into the class member variable $differenceEngine->mNewContent but before returning true from this function.$differenceEngine:DifferenceEngine object 'DifferenceEngineLoadTextAfterNewContentIsLoaded':called in DifferenceEngine::loadText() after the new revision's content has been loaded into the class member variable $differenceEngine->mNewContent but before checking if the variable's value is null.This hook can be used to inject content into said class member variable.$differenceEngine:DifferenceEngine object 'DifferenceEngineMarkPatrolledLink':Allows extensions to change the"mark as patrolled"link which is shown both on the diff header as well as on the bottom of a page, usually wrapped in a span element which has class="patrollink".$differenceEngine:DifferenceEngine object &$markAsPatrolledLink:The"mark as patrolled"link HTML(string) $rcid:Recent change ID(rc_id) for this change(int) 'DifferenceEngineMarkPatrolledRCID':Allows extensions to possibly change the rcid parameter.For example the rcid might be set to zero due to the user being the same as the performer of the change but an extension might still want to show it under certain conditions.&$rcid:rc_id(int) of the change or 0 $differenceEngine:DifferenceEngine object $change:RecentChange object $user:User object representing the current user 'DifferenceEngineNewHeader':Allows extensions to change the $newHeader variable, which contains information about the new revision, such as the revision's author, whether the revision was marked as a minor edit or not, etc.$differenceEngine:DifferenceEngine object &$newHeader:The string containing the various#mw-diff-otitle[1-5] divs, which include things like revision author info, revision comment, RevisionDelete link and more $formattedRevisionTools:Array containing revision tools, some of which may have been injected with the DiffRevisionTools hook $nextlink:String containing the link to the next revision(if any) $status
Definition: hooks.txt:1248
deleteFileCache($path)
Delete the cached stat info for a file path.
getScopedLocksForOps(array $ops, StatusValue $status)
doMoveInternal(array $params)
sanitizeOpHeaders(array $op)
Normalize and filter HTTP headers from a file operation.
doDescribeInternal(array $params)
static isValidShortContainerName($container)
Check if a short container name is valid.
doCleanInternal($container, $dir, array $params)
getLocalReferenceMulti(array $params)
FileBackendStore helper class for performing asynchronous file operations.
doGetFileSha1Base36(array $params)
deleteContainerCache($container)
Delete the cached info for a container.
primeFileCache(array $items)
Do a batch lookup from cache for file stats for all paths used in a list of storage paths or FileOp o...
getFileListInternal($container, $dir, array $params)
Do not call this function from places outside FileBackend.
doCopyInternal(array $params)
getLocalReference(array $params)
Returns a file system file, identical to the file at a storage path.
getDirectoryList(array $params)
Class for process caching individual properties of expiring items.
getLocalCopyMulti(array $params)
getFileProps(array $params)
doDirectoryExists($container, $dir, array $params)
resolveStoragePath($storagePath)
Splits a storage path into an internal container name, an internal relative file name, and a container shard suffix.
doExecuteOpHandlesInternal(array $fileOpHandles)