MediaWiki  master
FileBackendStore.php
Go to the documentation of this file.
1 <?php
24 
38 abstract class FileBackendStore extends FileBackend {
40  protected $memCache;
42  protected $srvCache;
44  protected $cheapCache;
46  protected $expensiveCache;
47 
49  protected $shardViaHashLevels = [];
50 
52  protected $mimeCallback;
53 
54  protected $maxFileSize = 4294967296; // integer bytes (4GiB)
55 
56  const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries
57  const CACHE_CHEAP_SIZE = 500; // integer; max entries in "cheap cache"
58  const CACHE_EXPENSIVE_SIZE = 5; // integer; max entries in "expensive cache"
59 
71  public function __construct( array $config ) {
72  parent::__construct( $config );
73  $this->mimeCallback = $config['mimeCallback'] ?? null;
74  $this->srvCache = new EmptyBagOStuff(); // disabled by default
75  $this->memCache = WANObjectCache::newEmpty(); // disabled by default
76  $this->cheapCache = new MapCacheLRU( self::CACHE_CHEAP_SIZE );
77  $this->expensiveCache = new MapCacheLRU( self::CACHE_EXPENSIVE_SIZE );
78  }
79 
87  final public function maxFileSizeInternal() {
88  return $this->maxFileSize;
89  }
90 
100  abstract public function isPathUsableInternal( $storagePath );
101 
120  final public function createInternal( array $params ) {
121  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
122  if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
123  $status = $this->newStatus( 'backend-fail-maxsize',
124  $params['dst'], $this->maxFileSizeInternal() );
125  } else {
126  $status = $this->doCreateInternal( $params );
127  $this->clearCache( [ $params['dst'] ] );
128  if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
129  $this->deleteFileCache( $params['dst'] ); // persistent cache
130  }
131  }
132 
133  return $status;
134  }
135 
141  abstract protected function doCreateInternal( array $params );
142 
161  final public function storeInternal( array $params ) {
162  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
163  if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
164  $status = $this->newStatus( 'backend-fail-maxsize',
165  $params['dst'], $this->maxFileSizeInternal() );
166  } else {
167  $status = $this->doStoreInternal( $params );
168  $this->clearCache( [ $params['dst'] ] );
169  if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
170  $this->deleteFileCache( $params['dst'] ); // persistent cache
171  }
172  }
173 
174  return $status;
175  }
176 
182  abstract protected function doStoreInternal( array $params );
183 
203  final public function copyInternal( array $params ) {
204  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
205  $status = $this->doCopyInternal( $params );
206  $this->clearCache( [ $params['dst'] ] );
207  if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
208  $this->deleteFileCache( $params['dst'] ); // persistent cache
209  }
210 
211  return $status;
212  }
213 
219  abstract protected function doCopyInternal( array $params );
220 
235  final public function deleteInternal( array $params ) {
236  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
237  $status = $this->doDeleteInternal( $params );
238  $this->clearCache( [ $params['src'] ] );
239  $this->deleteFileCache( $params['src'] ); // persistent cache
240  return $status;
241  }
242 
248  abstract protected function doDeleteInternal( array $params );
249 
269  final public function moveInternal( array $params ) {
270  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
271  $status = $this->doMoveInternal( $params );
272  $this->clearCache( [ $params['src'], $params['dst'] ] );
273  $this->deleteFileCache( $params['src'] ); // persistent cache
274  if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
275  $this->deleteFileCache( $params['dst'] ); // persistent cache
276  }
277 
278  return $status;
279  }
280 
286  protected function doMoveInternal( array $params ) {
287  unset( $params['async'] ); // two steps, won't work here :)
288  $nsrc = FileBackend::normalizeStoragePath( $params['src'] );
289  $ndst = FileBackend::normalizeStoragePath( $params['dst'] );
290  // Copy source to dest
291  $status = $this->copyInternal( $params );
292  if ( $nsrc !== $ndst && $status->isOK() ) {
293  // Delete source (only fails due to races or network problems)
294  $status->merge( $this->deleteInternal( [ 'src' => $params['src'] ] ) );
295  $status->setResult( true, $status->value ); // ignore delete() errors
296  }
297 
298  return $status;
299  }
300 
315  final public function describeInternal( array $params ) {
316  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
317  if ( count( $params['headers'] ) ) {
318  $status = $this->doDescribeInternal( $params );
319  $this->clearCache( [ $params['src'] ] );
320  $this->deleteFileCache( $params['src'] ); // persistent cache
321  } else {
322  $status = $this->newStatus(); // nothing to do
323  }
324 
325  return $status;
326  }
327 
333  protected function doDescribeInternal( array $params ) {
334  return $this->newStatus();
335  }
336 
344  final public function nullInternal( array $params ) {
345  return $this->newStatus();
346  }
347 
348  final public function concatenate( array $params ) {
349  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
350  $status = $this->newStatus();
351 
352  // Try to lock the source files for the scope of this function
353  $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
354  if ( $status->isOK() ) {
355  // Actually do the file concatenation...
356  $start_time = microtime( true );
357  $status->merge( $this->doConcatenate( $params ) );
358  $sec = microtime( true ) - $start_time;
359  if ( !$status->isOK() ) {
360  $this->logger->error( static::class . "-{$this->name}" .
361  " failed to concatenate " . count( $params['srcs'] ) . " file(s) [$sec sec]" );
362  }
363  }
364 
365  return $status;
366  }
367 
373  protected function doConcatenate( array $params ) {
374  $status = $this->newStatus();
375  $tmpPath = $params['dst']; // convenience
376  unset( $params['latest'] ); // sanity
377 
378  // Check that the specified temp file is valid...
379  Wikimedia\suppressWarnings();
380  $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 );
381  Wikimedia\restoreWarnings();
382  if ( !$ok ) { // not present or not empty
383  $status->fatal( 'backend-fail-opentemp', $tmpPath );
384 
385  return $status;
386  }
387 
388  // Get local FS versions of the chunks needed for the concatenation...
389  $fsFiles = $this->getLocalReferenceMulti( $params );
390  foreach ( $fsFiles as $path => &$fsFile ) {
391  if ( !$fsFile ) { // chunk failed to download?
392  $fsFile = $this->getLocalReference( [ 'src' => $path ] );
393  if ( !$fsFile ) { // retry failed?
394  $status->fatal( 'backend-fail-read', $path );
395 
396  return $status;
397  }
398  }
399  }
400  unset( $fsFile ); // unset reference so we can reuse $fsFile
401 
402  // Get a handle for the destination temp file
403  $tmpHandle = fopen( $tmpPath, 'ab' );
404  if ( $tmpHandle === false ) {
405  $status->fatal( 'backend-fail-opentemp', $tmpPath );
406 
407  return $status;
408  }
409 
410  // Build up the temp file using the source chunks (in order)...
411  foreach ( $fsFiles as $virtualSource => $fsFile ) {
412  // Get a handle to the local FS version
413  $sourceHandle = fopen( $fsFile->getPath(), 'rb' );
414  if ( $sourceHandle === false ) {
415  fclose( $tmpHandle );
416  $status->fatal( 'backend-fail-read', $virtualSource );
417 
418  return $status;
419  }
420  // Append chunk to file (pass chunk size to avoid magic quotes)
421  if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
422  fclose( $sourceHandle );
423  fclose( $tmpHandle );
424  $status->fatal( 'backend-fail-writetemp', $tmpPath );
425 
426  return $status;
427  }
428  fclose( $sourceHandle );
429  }
430  if ( !fclose( $tmpHandle ) ) {
431  $status->fatal( 'backend-fail-closetemp', $tmpPath );
432 
433  return $status;
434  }
435 
436  clearstatcache(); // temp file changed
437 
438  return $status;
439  }
440 
441  final protected function doPrepare( array $params ) {
442  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
443  $status = $this->newStatus();
444 
445  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
446  if ( $dir === null ) {
447  $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
448 
449  return $status; // invalid storage path
450  }
451 
452  if ( $shard !== null ) { // confined to a single container/shard
453  $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
454  } else { // directory is on several shards
455  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
456  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
457  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
458  $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
459  }
460  }
461 
462  return $status;
463  }
464 
472  protected function doPrepareInternal( $container, $dir, array $params ) {
473  return $this->newStatus();
474  }
475 
476  final protected function doSecure( array $params ) {
477  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
478  $status = $this->newStatus();
479 
480  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
481  if ( $dir === null ) {
482  $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
483 
484  return $status; // invalid storage path
485  }
486 
487  if ( $shard !== null ) { // confined to a single container/shard
488  $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
489  } else { // directory is on several shards
490  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
491  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
492  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
493  $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
494  }
495  }
496 
497  return $status;
498  }
499 
507  protected function doSecureInternal( $container, $dir, array $params ) {
508  return $this->newStatus();
509  }
510 
511  final protected function doPublish( array $params ) {
512  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
513  $status = $this->newStatus();
514 
515  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
516  if ( $dir === null ) {
517  $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
518 
519  return $status; // invalid storage path
520  }
521 
522  if ( $shard !== null ) { // confined to a single container/shard
523  $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
524  } else { // directory is on several shards
525  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
526  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
527  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
528  $status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) );
529  }
530  }
531 
532  return $status;
533  }
534 
542  protected function doPublishInternal( $container, $dir, array $params ) {
543  return $this->newStatus();
544  }
545 
546  final protected function doClean( array $params ) {
547  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
548  $status = $this->newStatus();
549 
550  // Recursive: first delete all empty subdirs recursively
551  if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) {
552  $subDirsRel = $this->getTopDirectoryList( [ 'dir' => $params['dir'] ] );
553  if ( $subDirsRel !== null ) { // no errors
554  foreach ( $subDirsRel as $subDirRel ) {
555  $subDir = $params['dir'] . "/{$subDirRel}"; // full path
556  $status->merge( $this->doClean( [ 'dir' => $subDir ] + $params ) );
557  }
558  unset( $subDirsRel ); // free directory for rmdir() on Windows (for FS backends)
559  }
560  }
561 
562  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
563  if ( $dir === null ) {
564  $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
565 
566  return $status; // invalid storage path
567  }
568 
569  // Attempt to lock this directory...
570  $filesLockEx = [ $params['dir'] ];
571  $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
572  if ( !$status->isOK() ) {
573  return $status; // abort
574  }
575 
576  if ( $shard !== null ) { // confined to a single container/shard
577  $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
578  $this->deleteContainerCache( $fullCont ); // purge cache
579  } else { // directory is on several shards
580  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
581  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
582  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
583  $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
584  $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache
585  }
586  }
587 
588  return $status;
589  }
590 
598  protected function doCleanInternal( $container, $dir, array $params ) {
599  return $this->newStatus();
600  }
601 
602  final public function fileExists( array $params ) {
603  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
604  $stat = $this->getFileStat( $params );
605 
606  return ( $stat === null ) ? null : (bool)$stat; // null => failure
607  }
608 
609  final public function getFileTimestamp( array $params ) {
610  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
611  $stat = $this->getFileStat( $params );
612 
613  return $stat ? $stat['mtime'] : false;
614  }
615 
616  final public function getFileSize( array $params ) {
617  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
618  $stat = $this->getFileStat( $params );
619 
620  return $stat ? $stat['size'] : false;
621  }
622 
623  final public function getFileStat( array $params ) {
624  $path = self::normalizeStoragePath( $params['src'] );
625  if ( $path === null ) {
626  return false; // invalid storage path
627  }
628  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
629 
630  $latest = !empty( $params['latest'] ); // use latest data?
631  $requireSHA1 = !empty( $params['requireSHA1'] ); // require SHA-1 if file exists?
632 
633  if ( !$latest ) {
634  $stat = $this->cheapCache->getField( $path, 'stat', self::CACHE_TTL );
635  // Note that some backends, like SwiftFileBackend, sometimes set file stat process
636  // cache entries from mass object listings that do not include the SHA-1. In that
637  // case, loading the persistent stat cache will likely yield the SHA-1.
638  if (
639  $stat === null ||
640  ( $requireSHA1 && is_array( $stat ) && !isset( $stat['sha1'] ) )
641  ) {
642  $this->primeFileCache( [ $path ] ); // check persistent cache
643  }
644  }
645 
646  $stat = $this->cheapCache->getField( $path, 'stat', self::CACHE_TTL );
647  // If we want the latest data, check that this cached
648  // value was in fact fetched with the latest available data.
649  if ( is_array( $stat ) ) {
650  if (
651  ( !$latest || $stat['latest'] ) &&
652  ( !$requireSHA1 || isset( $stat['sha1'] ) )
653  ) {
654  return $stat;
655  }
656  } elseif ( in_array( $stat, [ 'NOT_EXIST', 'NOT_EXIST_LATEST' ], true ) ) {
657  if ( !$latest || $stat === 'NOT_EXIST_LATEST' ) {
658  return false;
659  }
660  }
661 
662  $stat = $this->doGetFileStat( $params );
663 
664  if ( is_array( $stat ) ) { // file exists
665  // Strongly consistent backends can automatically set "latest"
666  $stat['latest'] = $stat['latest'] ?? $latest;
667  $this->cheapCache->setField( $path, 'stat', $stat );
668  $this->setFileCache( $path, $stat ); // update persistent cache
669  if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata
670  $this->cheapCache->setField( $path, 'sha1',
671  [ 'hash' => $stat['sha1'], 'latest' => $latest ] );
672  }
673  if ( isset( $stat['xattr'] ) ) { // some backends store headers/metadata
674  $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] );
675  $this->cheapCache->setField( $path, 'xattr',
676  [ 'map' => $stat['xattr'], 'latest' => $latest ] );
677  }
678  } elseif ( $stat === false ) { // file does not exist
679  $this->cheapCache->setField( $path, 'stat', $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' );
680  $this->cheapCache->setField( $path, 'xattr', [ 'map' => false, 'latest' => $latest ] );
681  $this->cheapCache->setField( $path, 'sha1', [ 'hash' => false, 'latest' => $latest ] );
682  $this->logger->debug( __METHOD__ . ': File {path} does not exist', [
683  'path' => $path,
684  ] );
685  } else { // an error occurred
686  $this->logger->warning( __METHOD__ . ': Could not stat file {path}', [
687  'path' => $path,
688  ] );
689  }
690 
691  return $stat;
692  }
693 
698  abstract protected function doGetFileStat( array $params );
699 
700  public function getFileContentsMulti( array $params ) {
701  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
702 
703  $params = $this->setConcurrencyFlags( $params );
704  $contents = $this->doGetFileContentsMulti( $params );
705 
706  return $contents;
707  }
708 
714  protected function doGetFileContentsMulti( array $params ) {
715  $contents = [];
716  foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
717  Wikimedia\suppressWarnings();
718  $contents[$path] = $fsFile ? file_get_contents( $fsFile->getPath() ) : false;
719  Wikimedia\restoreWarnings();
720  }
721 
722  return $contents;
723  }
724 
725  final public function getFileXAttributes( array $params ) {
726  $path = self::normalizeStoragePath( $params['src'] );
727  if ( $path === null ) {
728  return false; // invalid storage path
729  }
730  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
731  $latest = !empty( $params['latest'] ); // use latest data?
732  if ( $this->cheapCache->hasField( $path, 'xattr', self::CACHE_TTL ) ) {
733  $stat = $this->cheapCache->getField( $path, 'xattr' );
734  // If we want the latest data, check that this cached
735  // value was in fact fetched with the latest available data.
736  if ( !$latest || $stat['latest'] ) {
737  return $stat['map'];
738  }
739  }
740  $fields = $this->doGetFileXAttributes( $params );
741  $fields = is_array( $fields ) ? self::normalizeXAttributes( $fields ) : false;
742  $this->cheapCache->setField( $path, 'xattr', [ 'map' => $fields, 'latest' => $latest ] );
743 
744  return $fields;
745  }
746 
752  protected function doGetFileXAttributes( array $params ) {
753  return [ 'headers' => [], 'metadata' => [] ]; // not supported
754  }
755 
756  final public function getFileSha1Base36( array $params ) {
757  $path = self::normalizeStoragePath( $params['src'] );
758  if ( $path === null ) {
759  return false; // invalid storage path
760  }
761  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
762  $latest = !empty( $params['latest'] ); // use latest data?
763  if ( $this->cheapCache->hasField( $path, 'sha1', self::CACHE_TTL ) ) {
764  $stat = $this->cheapCache->getField( $path, 'sha1' );
765  // If we want the latest data, check that this cached
766  // value was in fact fetched with the latest available data.
767  if ( !$latest || $stat['latest'] ) {
768  return $stat['hash'];
769  }
770  }
771  $hash = $this->doGetFileSha1Base36( $params );
772  $this->cheapCache->setField( $path, 'sha1', [ 'hash' => $hash, 'latest' => $latest ] );
773 
774  return $hash;
775  }
776 
782  protected function doGetFileSha1Base36( array $params ) {
783  $fsFile = $this->getLocalReference( $params );
784  if ( !$fsFile ) {
785  return false;
786  } else {
787  return $fsFile->getSha1Base36();
788  }
789  }
790 
791  final public function getFileProps( array $params ) {
792  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
793  $fsFile = $this->getLocalReference( $params );
794  $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
795 
796  return $props;
797  }
798 
799  final public function getLocalReferenceMulti( array $params ) {
800  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
801 
802  $params = $this->setConcurrencyFlags( $params );
803 
804  $fsFiles = []; // (path => FSFile)
805  $latest = !empty( $params['latest'] ); // use latest data?
806  // Reuse any files already in process cache...
807  foreach ( $params['srcs'] as $src ) {
808  $path = self::normalizeStoragePath( $src );
809  if ( $path === null ) {
810  $fsFiles[$src] = null; // invalid storage path
811  } elseif ( $this->expensiveCache->hasField( $path, 'localRef' ) ) {
812  $val = $this->expensiveCache->getField( $path, 'localRef' );
813  // If we want the latest data, check that this cached
814  // value was in fact fetched with the latest available data.
815  if ( !$latest || $val['latest'] ) {
816  $fsFiles[$src] = $val['object'];
817  }
818  }
819  }
820  // Fetch local references of any remaning files...
821  $params['srcs'] = array_diff( $params['srcs'], array_keys( $fsFiles ) );
822  foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
823  $fsFiles[$path] = $fsFile;
824  if ( $fsFile ) { // update the process cache...
825  $this->expensiveCache->setField( $path, 'localRef',
826  [ 'object' => $fsFile, 'latest' => $latest ] );
827  }
828  }
829 
830  return $fsFiles;
831  }
832 
838  protected function doGetLocalReferenceMulti( array $params ) {
839  return $this->doGetLocalCopyMulti( $params );
840  }
841 
842  final public function getLocalCopyMulti( array $params ) {
843  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
844 
845  $params = $this->setConcurrencyFlags( $params );
846  $tmpFiles = $this->doGetLocalCopyMulti( $params );
847 
848  return $tmpFiles;
849  }
850 
856  abstract protected function doGetLocalCopyMulti( array $params );
857 
863  public function getFileHttpUrl( array $params ) {
864  return null; // not supported
865  }
866 
867  final public function streamFile( array $params ) {
868  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
869  $status = $this->newStatus();
870 
871  // Always set some fields for subclass convenience
872  $params['options'] = $params['options'] ?? [];
873  $params['headers'] = $params['headers'] ?? [];
874 
875  // Don't stream it out as text/html if there was a PHP error
876  if ( ( empty( $params['headless'] ) || $params['headers'] ) && headers_sent() ) {
877  print "Headers already sent, terminating.\n";
878  $status->fatal( 'backend-fail-stream', $params['src'] );
879  return $status;
880  }
881 
882  $status->merge( $this->doStreamFile( $params ) );
883 
884  return $status;
885  }
886 
892  protected function doStreamFile( array $params ) {
893  $status = $this->newStatus();
894 
895  $flags = 0;
896  $flags |= !empty( $params['headless'] ) ? HTTPFileStreamer::STREAM_HEADLESS : 0;
897  $flags |= !empty( $params['allowOB'] ) ? HTTPFileStreamer::STREAM_ALLOW_OB : 0;
898 
899  $fsFile = $this->getLocalReference( $params );
900  if ( $fsFile ) {
901  $streamer = new HTTPFileStreamer(
902  $fsFile->getPath(),
903  [
904  'obResetFunc' => $this->obResetFunc,
905  'streamMimeFunc' => $this->streamMimeFunc
906  ]
907  );
908  $res = $streamer->stream( $params['headers'], true, $params['options'], $flags );
909  } else {
910  $res = false;
911  HTTPFileStreamer::send404Message( $params['src'], $flags );
912  }
913 
914  if ( !$res ) {
915  $status->fatal( 'backend-fail-stream', $params['src'] );
916  }
917 
918  return $status;
919  }
920 
921  final public function directoryExists( array $params ) {
922  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
923  if ( $dir === null ) {
924  return false; // invalid storage path
925  }
926  if ( $shard !== null ) { // confined to a single container/shard
927  return $this->doDirectoryExists( $fullCont, $dir, $params );
928  } else { // directory is on several shards
929  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
930  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
931  $res = false; // response
932  foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
933  $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params );
934  if ( $exists ) {
935  $res = true;
936  break; // found one!
937  } elseif ( $exists === null ) { // error?
938  $res = null; // if we don't find anything, it is indeterminate
939  }
940  }
941 
942  return $res;
943  }
944  }
945 
954  abstract protected function doDirectoryExists( $container, $dir, array $params );
955 
956  final public function getDirectoryList( array $params ) {
957  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
958  if ( $dir === null ) { // invalid storage path
959  return null;
960  }
961  if ( $shard !== null ) {
962  // File listing is confined to a single container/shard
963  return $this->getDirectoryListInternal( $fullCont, $dir, $params );
964  } else {
965  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
966  // File listing spans multiple containers/shards
967  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
968 
969  return new FileBackendStoreShardDirIterator( $this,
970  $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
971  }
972  }
973 
984  abstract public function getDirectoryListInternal( $container, $dir, array $params );
985 
986  final public function getFileList( array $params ) {
987  list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
988  if ( $dir === null ) { // invalid storage path
989  return null;
990  }
991  if ( $shard !== null ) {
992  // File listing is confined to a single container/shard
993  return $this->getFileListInternal( $fullCont, $dir, $params );
994  } else {
995  $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
996  // File listing spans multiple containers/shards
997  list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
998 
999  return new FileBackendStoreShardFileIterator( $this,
1000  $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
1001  }
1002  }
1003 
1014  abstract public function getFileListInternal( $container, $dir, array $params );
1015 
1027  final public function getOperationsInternal( array $ops ) {
1028  $supportedOps = [
1029  'store' => StoreFileOp::class,
1030  'copy' => CopyFileOp::class,
1031  'move' => MoveFileOp::class,
1032  'delete' => DeleteFileOp::class,
1033  'create' => CreateFileOp::class,
1034  'describe' => DescribeFileOp::class,
1035  'null' => NullFileOp::class
1036  ];
1037 
1038  $performOps = []; // array of FileOp objects
1039  // Build up ordered array of FileOps...
1040  foreach ( $ops as $operation ) {
1041  $opName = $operation['op'];
1042  if ( isset( $supportedOps[$opName] ) ) {
1043  $class = $supportedOps[$opName];
1044  // Get params for this operation
1045  $params = $operation;
1046  // Append the FileOp class
1047  $performOps[] = new $class( $this, $params, $this->logger );
1048  } else {
1049  throw new FileBackendError( "Operation '$opName' is not supported." );
1050  }
1051  }
1052 
1053  return $performOps;
1054  }
1055 
1066  final public function getPathsToLockForOpsInternal( array $performOps ) {
1067  // Build up a list of files to lock...
1068  $paths = [ 'sh' => [], 'ex' => [] ];
1069  foreach ( $performOps as $fileOp ) {
1070  $paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() );
1071  $paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() );
1072  }
1073  // Optimization: if doing an EX lock anyway, don't also set an SH one
1074  $paths['sh'] = array_diff( $paths['sh'], $paths['ex'] );
1075  // Get a shared lock on the parent directory of each path changed
1076  $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) );
1077 
1078  return [
1079  LockManager::LOCK_UW => $paths['sh'],
1080  LockManager::LOCK_EX => $paths['ex']
1081  ];
1082  }
1083 
1084  public function getScopedLocksForOps( array $ops, StatusValue $status ) {
1085  $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) );
1086 
1087  return $this->getScopedFileLocks( $paths, 'mixed', $status );
1088  }
1089 
1090  final protected function doOperationsInternal( array $ops, array $opts ) {
1091  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1092  $status = $this->newStatus();
1093 
1094  // Fix up custom header name/value pairs...
1095  $ops = array_map( [ $this, 'sanitizeOpHeaders' ], $ops );
1096 
1097  // Build up a list of FileOps...
1098  $performOps = $this->getOperationsInternal( $ops );
1099 
1100  // Acquire any locks as needed...
1101  if ( empty( $opts['nonLocking'] ) ) {
1102  // Build up a list of files to lock...
1103  $paths = $this->getPathsToLockForOpsInternal( $performOps );
1104  // Try to lock those files for the scope of this function...
1105 
1106  $scopeLock = $this->getScopedFileLocks( $paths, 'mixed', $status );
1107  if ( !$status->isOK() ) {
1108  return $status; // abort
1109  }
1110  }
1111 
1112  // Clear any file cache entries (after locks acquired)
1113  if ( empty( $opts['preserveCache'] ) ) {
1114  $this->clearCache();
1115  }
1116 
1117  // Build the list of paths involved
1118  $paths = [];
1119  foreach ( $performOps as $performOp ) {
1120  $paths = array_merge( $paths, $performOp->storagePathsRead() );
1121  $paths = array_merge( $paths, $performOp->storagePathsChanged() );
1122  }
1123 
1124  // Enlarge the cache to fit the stat entries of these files
1125  $this->cheapCache->setMaxSize( max( 2 * count( $paths ), self::CACHE_CHEAP_SIZE ) );
1126 
1127  // Load from the persistent container caches
1128  $this->primeContainerCache( $paths );
1129  // Get the latest stat info for all the files (having locked them)
1130  $ok = $this->preloadFileStat( [ 'srcs' => $paths, 'latest' => true ] );
1131 
1132  if ( $ok ) {
1133  // Actually attempt the operation batch...
1134  $opts = $this->setConcurrencyFlags( $opts );
1135  $subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal );
1136  } else {
1137  // If we could not even stat some files, then bail out...
1138  $subStatus = $this->newStatus( 'backend-fail-internal', $this->name );
1139  foreach ( $ops as $i => $op ) { // mark each op as failed
1140  $subStatus->success[$i] = false;
1141  ++$subStatus->failCount;
1142  }
1143  $this->logger->error( static::class . "-{$this->name} " .
1144  " stat failure; aborted operations: " . FormatJson::encode( $ops ) );
1145  }
1146 
1147  // Merge errors into StatusValue fields
1148  $status->merge( $subStatus );
1149  $status->success = $subStatus->success; // not done in merge()
1150 
1151  // Shrink the stat cache back to normal size
1152  $this->cheapCache->setMaxSize( self::CACHE_CHEAP_SIZE );
1153 
1154  return $status;
1155  }
1156 
1157  final protected function doQuickOperationsInternal( array $ops ) {
1158  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1159  $status = $this->newStatus();
1160 
1161  // Fix up custom header name/value pairs...
1162  $ops = array_map( [ $this, 'sanitizeOpHeaders' ], $ops );
1163 
1164  // Clear any file cache entries
1165  $this->clearCache();
1166 
1167  $supportedOps = [ 'create', 'store', 'copy', 'move', 'delete', 'describe', 'null' ];
1168  // Parallel ops may be disabled in config due to dependencies (e.g. needing popen())
1169  $async = ( $this->parallelize === 'implicit' && count( $ops ) > 1 );
1170  $maxConcurrency = $this->concurrency; // throttle
1172  $statuses = []; // array of (index => StatusValue)
1173  $fileOpHandles = []; // list of (index => handle) arrays
1174  $curFileOpHandles = []; // current handle batch
1175  // Perform the sync-only ops and build up op handles for the async ops...
1176  foreach ( $ops as $index => $params ) {
1177  if ( !in_array( $params['op'], $supportedOps ) ) {
1178  throw new FileBackendError( "Operation '{$params['op']}' is not supported." );
1179  }
1180  $method = $params['op'] . 'Internal'; // e.g. "storeInternal"
1181  $subStatus = $this->$method( [ 'async' => $async ] + $params );
1182  if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { // async
1183  if ( count( $curFileOpHandles ) >= $maxConcurrency ) {
1184  $fileOpHandles[] = $curFileOpHandles; // push this batch
1185  $curFileOpHandles = [];
1186  }
1187  $curFileOpHandles[$index] = $subStatus->value; // keep index
1188  } else { // error or completed
1189  $statuses[$index] = $subStatus; // keep index
1190  }
1191  }
1192  if ( count( $curFileOpHandles ) ) {
1193  $fileOpHandles[] = $curFileOpHandles; // last batch
1194  }
1195  // Do all the async ops that can be done concurrently...
1196  foreach ( $fileOpHandles as $fileHandleBatch ) {
1197  $statuses = $statuses + $this->executeOpHandlesInternal( $fileHandleBatch );
1198  }
1199  // Marshall and merge all the responses...
1200  foreach ( $statuses as $index => $subStatus ) {
1201  $status->merge( $subStatus );
1202  if ( $subStatus->isOK() ) {
1203  $status->success[$index] = true;
1204  ++$status->successCount;
1205  } else {
1206  $status->success[$index] = false;
1207  ++$status->failCount;
1208  }
1209  }
1210 
1211  return $status;
1212  }
1213 
1223  final public function executeOpHandlesInternal( array $fileOpHandles ) {
1224  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1225 
1226  foreach ( $fileOpHandles as $fileOpHandle ) {
1227  if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) {
1228  throw new InvalidArgumentException( "Expected FileBackendStoreOpHandle object." );
1229  } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) {
1230  throw new InvalidArgumentException( "Expected handle for this file backend." );
1231  }
1232  }
1233 
1234  $res = $this->doExecuteOpHandlesInternal( $fileOpHandles );
1235  foreach ( $fileOpHandles as $fileOpHandle ) {
1236  $fileOpHandle->closeResources();
1237  }
1238 
1239  return $res;
1240  }
1241 
1250  protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1251  if ( count( $fileOpHandles ) ) {
1252  throw new LogicException( "Backend does not support asynchronous operations." );
1253  }
1254 
1255  return [];
1256  }
1257 
1269  protected function sanitizeOpHeaders( array $op ) {
1270  static $longs = [ 'content-disposition' ];
1271 
1272  if ( isset( $op['headers'] ) ) { // op sets HTTP headers
1273  $newHeaders = [];
1274  foreach ( $op['headers'] as $name => $value ) {
1275  $name = strtolower( $name );
1276  $maxHVLen = in_array( $name, $longs ) ? INF : 255;
1277  if ( strlen( $name ) > 255 || strlen( $value ) > $maxHVLen ) {
1278  trigger_error( "Header '$name: $value' is too long." );
1279  } else {
1280  $newHeaders[$name] = strlen( $value ) ? $value : ''; // null/false => ""
1281  }
1282  }
1283  $op['headers'] = $newHeaders;
1284  }
1285 
1286  return $op;
1287  }
1288 
1289  final public function preloadCache( array $paths ) {
1290  $fullConts = []; // full container names
1291  foreach ( $paths as $path ) {
1292  list( $fullCont, , ) = $this->resolveStoragePath( $path );
1293  $fullConts[] = $fullCont;
1294  }
1295  // Load from the persistent file and container caches
1296  $this->primeContainerCache( $fullConts );
1297  $this->primeFileCache( $paths );
1298  }
1299 
1300  final public function clearCache( array $paths = null ) {
1301  if ( is_array( $paths ) ) {
1302  $paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
1303  $paths = array_filter( $paths, 'strlen' ); // remove nulls
1304  }
1305  if ( $paths === null ) {
1306  $this->cheapCache->clear();
1307  $this->expensiveCache->clear();
1308  } else {
1309  foreach ( $paths as $path ) {
1310  $this->cheapCache->clear( $path );
1311  $this->expensiveCache->clear( $path );
1312  }
1313  }
1314  $this->doClearCache( $paths );
1315  }
1316 
1324  protected function doClearCache( array $paths = null ) {
1325  }
1326 
1327  final public function preloadFileStat( array $params ) {
1328  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1329  $success = true; // no network errors
1330 
1331  $params['concurrency'] = ( $this->parallelize !== 'off' ) ? $this->concurrency : 1;
1332  $stats = $this->doGetFileStatMulti( $params );
1333  if ( $stats === null ) {
1334  return true; // not supported
1335  }
1336 
1337  $latest = !empty( $params['latest'] ); // use latest data?
1338  foreach ( $stats as $path => $stat ) {
1340  if ( $path === null ) {
1341  continue; // this shouldn't happen
1342  }
1343  if ( is_array( $stat ) ) { // file exists
1344  // Strongly consistent backends can automatically set "latest"
1345  $stat['latest'] = $stat['latest'] ?? $latest;
1346  $this->cheapCache->setField( $path, 'stat', $stat );
1347  $this->setFileCache( $path, $stat ); // update persistent cache
1348  if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata
1349  $this->cheapCache->setField( $path, 'sha1',
1350  [ 'hash' => $stat['sha1'], 'latest' => $latest ] );
1351  }
1352  if ( isset( $stat['xattr'] ) ) { // some backends store headers/metadata
1353  $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] );
1354  $this->cheapCache->setField( $path, 'xattr',
1355  [ 'map' => $stat['xattr'], 'latest' => $latest ] );
1356  }
1357  } elseif ( $stat === false ) { // file does not exist
1358  $this->cheapCache->setField( $path, 'stat',
1359  $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' );
1360  $this->cheapCache->setField( $path, 'xattr',
1361  [ 'map' => false, 'latest' => $latest ] );
1362  $this->cheapCache->setField( $path, 'sha1',
1363  [ 'hash' => false, 'latest' => $latest ] );
1364  $this->logger->debug( __METHOD__ . ': File {path} does not exist', [
1365  'path' => $path,
1366  ] );
1367  } else { // an error occurred
1368  $success = false;
1369  $this->logger->warning( __METHOD__ . ': Could not stat file {path}', [
1370  'path' => $path,
1371  ] );
1372  }
1373  }
1374 
1375  return $success;
1376  }
1377 
1389  protected function doGetFileStatMulti( array $params ) {
1390  return null; // not supported
1391  }
1392 
1400  abstract protected function directoriesAreVirtual();
1401 
1412  final protected static function isValidShortContainerName( $container ) {
1413  // Suffixes like '.xxx' (hex shard chars) or '.seg' (file segments)
1414  // might be used by subclasses. Reserve the dot character for sanity.
1415  // The only way dots end up in containers (e.g. resolveStoragePath)
1416  // is due to the wikiId container prefix or the above suffixes.
1417  return self::isValidContainerName( $container ) && !preg_match( '/[.]/', $container );
1418  }
1419 
1429  final protected static function isValidContainerName( $container ) {
1430  // This accounts for NTFS, Swift, and Ceph restrictions
1431  // and disallows directory separators or traversal characters.
1432  // Note that matching strings URL encode to the same string;
1433  // in Swift/Ceph, the length restriction is *after* URL encoding.
1434  return (bool)preg_match( '/^[a-z0-9][a-z0-9-_.]{0,199}$/i', $container );
1435  }
1436 
1450  final protected function resolveStoragePath( $storagePath ) {
1451  list( $backend, $shortCont, $relPath ) = self::splitStoragePath( $storagePath );
1452  if ( $backend === $this->name ) { // must be for this backend
1453  $relPath = self::normalizeContainerPath( $relPath );
1454  if ( $relPath !== null && self::isValidShortContainerName( $shortCont ) ) {
1455  // Get shard for the normalized path if this container is sharded
1456  $cShard = $this->getContainerShard( $shortCont, $relPath );
1457  // Validate and sanitize the relative path (backend-specific)
1458  $relPath = $this->resolveContainerPath( $shortCont, $relPath );
1459  if ( $relPath !== null ) {
1460  // Prepend any wiki ID prefix to the container name
1461  $container = $this->fullContainerName( $shortCont );
1462  if ( self::isValidContainerName( $container ) ) {
1463  // Validate and sanitize the container name (backend-specific)
1464  $container = $this->resolveContainerName( "{$container}{$cShard}" );
1465  if ( $container !== null ) {
1466  return [ $container, $relPath, $cShard ];
1467  }
1468  }
1469  }
1470  }
1471  }
1472 
1473  return [ null, null, null ];
1474  }
1475 
1491  final protected function resolveStoragePathReal( $storagePath ) {
1492  list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
1493  if ( $cShard !== null && substr( $relPath, -1 ) !== '/' ) {
1494  return [ $container, $relPath ];
1495  }
1496 
1497  return [ null, null ];
1498  }
1499 
1508  final protected function getContainerShard( $container, $relPath ) {
1509  list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container );
1510  if ( $levels == 1 || $levels == 2 ) {
1511  // Hash characters are either base 16 or 36
1512  $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]';
1513  // Get a regex that represents the shard portion of paths.
1514  // The concatenation of the captures gives us the shard.
1515  if ( $levels === 1 ) { // 16 or 36 shards per container
1516  $hashDirRegex = '(' . $char . ')';
1517  } else { // 256 or 1296 shards per container
1518  if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc")
1519  $hashDirRegex = $char . '/(' . $char . '{2})';
1520  } else { // short hash dir format (e.g. "a/b/c")
1521  $hashDirRegex = '(' . $char . ')/(' . $char . ')';
1522  }
1523  }
1524  // Allow certain directories to be above the hash dirs so as
1525  // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
1526  // They must be 2+ chars to avoid any hash directory ambiguity.
1527  $m = [];
1528  if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1529  return '.' . implode( '', array_slice( $m, 1 ) );
1530  }
1531 
1532  return null; // failed to match
1533  }
1534 
1535  return ''; // no sharding
1536  }
1537 
1546  final public function isSingleShardPathInternal( $storagePath ) {
1547  list( , , $shard ) = $this->resolveStoragePath( $storagePath );
1548 
1549  return ( $shard !== null );
1550  }
1551 
1560  final protected function getContainerHashLevels( $container ) {
1561  if ( isset( $this->shardViaHashLevels[$container] ) ) {
1562  $config = $this->shardViaHashLevels[$container];
1563  $hashLevels = (int)$config['levels'];
1564  if ( $hashLevels == 1 || $hashLevels == 2 ) {
1565  $hashBase = (int)$config['base'];
1566  if ( $hashBase == 16 || $hashBase == 36 ) {
1567  return [ $hashLevels, $hashBase, $config['repeat'] ];
1568  }
1569  }
1570  }
1571 
1572  return [ 0, 0, false ]; // no sharding
1573  }
1574 
1581  final protected function getContainerSuffixes( $container ) {
1582  $shards = [];
1583  list( $digits, $base ) = $this->getContainerHashLevels( $container );
1584  if ( $digits > 0 ) {
1585  $numShards = $base ** $digits;
1586  for ( $index = 0; $index < $numShards; $index++ ) {
1587  $shards[] = '.' . Wikimedia\base_convert( $index, 10, $base, $digits );
1588  }
1589  }
1590 
1591  return $shards;
1592  }
1593 
1600  final protected function fullContainerName( $container ) {
1601  if ( $this->domainId != '' ) {
1602  return "{$this->domainId}-$container";
1603  } else {
1604  return $container;
1605  }
1606  }
1607 
1616  protected function resolveContainerName( $container ) {
1617  return $container;
1618  }
1619 
1630  protected function resolveContainerPath( $container, $relStoragePath ) {
1631  return $relStoragePath;
1632  }
1633 
1640  private function containerCacheKey( $container ) {
1641  return "filebackend:{$this->name}:{$this->domainId}:container:{$container}";
1642  }
1643 
1650  final protected function setContainerCache( $container, array $val ) {
1651  $this->memCache->set( $this->containerCacheKey( $container ), $val, 14 * 86400 );
1652  }
1653 
1660  final protected function deleteContainerCache( $container ) {
1661  if ( !$this->memCache->delete( $this->containerCacheKey( $container ), 300 ) ) {
1662  trigger_error( "Unable to delete stat cache for container $container." );
1663  }
1664  }
1665 
1673  final protected function primeContainerCache( array $items ) {
1674  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1675 
1676  $paths = []; // list of storage paths
1677  $contNames = []; // (cache key => resolved container name)
1678  // Get all the paths/containers from the items...
1679  foreach ( $items as $item ) {
1680  if ( self::isStoragePath( $item ) ) {
1681  $paths[] = $item;
1682  } elseif ( is_string( $item ) ) { // full container name
1683  $contNames[$this->containerCacheKey( $item )] = $item;
1684  }
1685  }
1686  // Get all the corresponding cache keys for paths...
1687  foreach ( $paths as $path ) {
1688  list( $fullCont, , ) = $this->resolveStoragePath( $path );
1689  if ( $fullCont !== null ) { // valid path for this backend
1690  $contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
1691  }
1692  }
1693 
1694  $contInfo = []; // (resolved container name => cache value)
1695  // Get all cache entries for these container cache keys...
1696  $values = $this->memCache->getMulti( array_keys( $contNames ) );
1697  foreach ( $values as $cacheKey => $val ) {
1698  $contInfo[$contNames[$cacheKey]] = $val;
1699  }
1700 
1701  // Populate the container process cache for the backend...
1702  $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) );
1703  }
1704 
1712  protected function doPrimeContainerCache( array $containerInfo ) {
1713  }
1714 
1721  private function fileCacheKey( $path ) {
1722  return "filebackend:{$this->name}:{$this->domainId}:file:" . sha1( $path );
1723  }
1724 
1733  final protected function setFileCache( $path, array $val ) {
1735  if ( $path === null ) {
1736  return; // invalid storage path
1737  }
1738  $mtime = ConvertibleTimestamp::convert( TS_UNIX, $val['mtime'] );
1739  $ttl = $this->memCache->adaptiveTTL( $mtime, 7 * 86400, 300, 0.1 );
1740  $key = $this->fileCacheKey( $path );
1741  // Set the cache unless it is currently salted.
1742  $this->memCache->set( $key, $val, $ttl );
1743  }
1744 
1753  final protected function deleteFileCache( $path ) {
1755  if ( $path === null ) {
1756  return; // invalid storage path
1757  }
1758  if ( !$this->memCache->delete( $this->fileCacheKey( $path ), 300 ) ) {
1759  trigger_error( "Unable to delete stat cache for file $path." );
1760  }
1761  }
1762 
1770  final protected function primeFileCache( array $items ) {
1771  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1772 
1773  $paths = []; // list of storage paths
1774  $pathNames = []; // (cache key => storage path)
1775  // Get all the paths/containers from the items...
1776  foreach ( $items as $item ) {
1777  if ( self::isStoragePath( $item ) ) {
1778  $paths[] = FileBackend::normalizeStoragePath( $item );
1779  }
1780  }
1781  // Get rid of any paths that failed normalization...
1782  $paths = array_filter( $paths, 'strlen' ); // remove nulls
1783  // Get all the corresponding cache keys for paths...
1784  foreach ( $paths as $path ) {
1785  list( , $rel, ) = $this->resolveStoragePath( $path );
1786  if ( $rel !== null ) { // valid path for this backend
1787  $pathNames[$this->fileCacheKey( $path )] = $path;
1788  }
1789  }
1790  // Get all cache entries for these file cache keys...
1791  $values = $this->memCache->getMulti( array_keys( $pathNames ) );
1792  foreach ( $values as $cacheKey => $val ) {
1793  $path = $pathNames[$cacheKey];
1794  if ( is_array( $val ) ) {
1795  $val['latest'] = false; // never completely trust cache
1796  $this->cheapCache->setField( $path, 'stat', $val );
1797  if ( isset( $val['sha1'] ) ) { // some backends store SHA-1 as metadata
1798  $this->cheapCache->setField( $path, 'sha1',
1799  [ 'hash' => $val['sha1'], 'latest' => false ] );
1800  }
1801  if ( isset( $val['xattr'] ) ) { // some backends store headers/metadata
1802  $val['xattr'] = self::normalizeXAttributes( $val['xattr'] );
1803  $this->cheapCache->setField( $path, 'xattr',
1804  [ 'map' => $val['xattr'], 'latest' => false ] );
1805  }
1806  }
1807  }
1808  }
1809 
1817  final protected static function normalizeXAttributes( array $xattr ) {
1818  $newXAttr = [ 'headers' => [], 'metadata' => [] ];
1819 
1820  foreach ( $xattr['headers'] as $name => $value ) {
1821  $newXAttr['headers'][strtolower( $name )] = $value;
1822  }
1823 
1824  foreach ( $xattr['metadata'] as $name => $value ) {
1825  $newXAttr['metadata'][strtolower( $name )] = $value;
1826  }
1827 
1828  return $newXAttr;
1829  }
1830 
1837  final protected function setConcurrencyFlags( array $opts ) {
1838  $opts['concurrency'] = 1; // off
1839  if ( $this->parallelize === 'implicit' ) {
1840  if ( !isset( $opts['parallelize'] ) || $opts['parallelize'] ) {
1841  $opts['concurrency'] = $this->concurrency;
1842  }
1843  } elseif ( $this->parallelize === 'explicit' ) {
1844  if ( !empty( $opts['parallelize'] ) ) {
1845  $opts['concurrency'] = $this->concurrency;
1846  }
1847  }
1848 
1849  return $opts;
1850  }
1851 
1860  protected function getContentType( $storagePath, $content, $fsPath ) {
1861  if ( $this->mimeCallback ) {
1862  return call_user_func_array( $this->mimeCallback, func_get_args() );
1863  }
1864 
1865  $mime = ( $fsPath !== null ) ? mime_content_type( $fsPath ) : false;
1866  return $mime ?: 'unknown/unknown';
1867  }
1868 }
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
The wiki should then use memcached to cache various data To use multiple just add more items to the array To increase the weight of a make its entry a array("192.168.0.1:11211", 2))
doDirectoryExists( $container, $dir, array $params)
getContainerHashLevels( $container)
Get the sharding config for a container.
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
getFileHttpUrl(array $params)
setFileCache( $path, array $val)
Set the cached stat info for a file path.
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition: deferred.txt:11
doOperationsInternal(array $ops, array $opts)
MapCacheLRU $cheapCache
Map of paths to small (RAM/disk) cache items.
setConcurrencyFlags(array $opts)
Set the &#39;concurrency&#39; option from a list of operation options.
getContainerShard( $container, $relPath)
Get the container name shard suffix for a given path.
$success
moveInternal(array $params)
Move a file from one storage path to another in the backend.
processing should stop and the error should be shown to the user * false
Definition: hooks.txt:187
concatenate(array $params)
doGetLocalReferenceMulti(array $params)
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
doSecureInternal( $container, $dir, array $params)
scopedProfileSection( $section)
getPathsToLockForOpsInternal(array $performOps)
Get a list of storage paths to lock for a list of operations Returns an array with LockManager::LOCK_...
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it&#39;s allowed by the backend.
doPublishInternal( $container, $dir, array $params)
static normalizeXAttributes(array $xattr)
Normalize file headers/metadata to the FileBackend::getFileXAttributes() format.
doGetFileXAttributes(array $params)
getFileStat(array $params)
$value
setContainerCache( $container, array $val)
Set the cached info for a container.
__construct(array $config)
getName()
Get the unique backend name.
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
fileCacheKey( $path)
Get the cache key for a file path.
static encode( $value, $pretty=false, $escaping=0)
Returns the JSON representation of a value.
Definition: FormatJson.php:115
getDirectoryListInternal( $container, $dir, array $params)
Do not call this function from places outside FileBackend.
static normalizeStoragePath( $storagePath)
Normalize a storage path by cleaning up directory separators.
int $concurrency
How many operations can be done in parallel.
storeInternal(array $params)
Store a file into the backend from a file on disk.
array $shardViaHashLevels
Map of container names to sharding config.
and how to run hooks for an and one after Each event has a name
Definition: hooks.txt:6
doCreateInternal(array $params)
Status::newGood()` to allow deletion, and then `return false` from the hook function. Ensure you consume the 'ChangeTagAfterDelete' hook to carry out custom deletion actions. $tag:name of the tag $user:user initiating the action & $status:Status object. See above. 'ChangeTagsListActive':Allows you to nominate which of the tags your extension uses are in active use. & $tags:list of all active tags. Append to this array. 'ChangeTagsAfterUpdateTags':Called after tags have been updated with the ChangeTags::updateTags function. Params:$addedTags:tags effectively added in the update $removedTags:tags effectively removed in the update $prevTags:tags that were present prior to the update $rc_id:recentchanges table id $rev_id:revision table id $log_id:logging table id $params:tag params $rc:RecentChange being tagged when the tagging accompanies the action, or null $user:User who performed the tagging when the tagging is subsequent to the action, or null 'ChangeTagsAllowedAdd':Called when checking if a user can add tags to a change. & $allowedTags:List of all the tags the user is allowed to add. Any tags the user wants to add( $addTags) that are not in this array will cause it to fail. You may add or remove tags to this array as required. $addTags:List of tags user intends to add. $user:User who is adding the tags. 'ChangeUserGroups':Called before user groups are changed. $performer:The User who will perform the change $user:The User whose groups will be changed & $add:The groups that will be added & $remove:The groups that will be removed 'Collation::factory':Called if $wgCategoryCollation is an unknown collation. $collationName:Name of the collation in question & $collationObject:Null. Replace with a subclass of the Collation class that implements the collation given in $collationName. 'ConfirmEmailComplete':Called after a user 's email has been confirmed successfully. $user:user(object) whose email is being confirmed 'ContentAlterParserOutput':Modify parser output for a given content object. Called by Content::getParserOutput after parsing has finished. Can be used for changes that depend on the result of the parsing but have to be done before LinksUpdate is called(such as adding tracking categories based on the rendered HTML). $content:The Content to render $title:Title of the page, as context $parserOutput:ParserOutput to manipulate 'ContentGetParserOutput':Customize parser output for a given content object, called by AbstractContent::getParserOutput. May be used to override the normal model-specific rendering of page content. $content:The Content to render $title:Title of the page, as context $revId:The revision ID, as context $options:ParserOptions for rendering. To avoid confusing the parser cache, the output can only depend on parameters provided to this hook function, not on global state. $generateHtml:boolean, indicating whether full HTML should be generated. If false, generation of HTML may be skipped, but other information should still be present in the ParserOutput object. & $output:ParserOutput, to manipulate or replace 'ContentHandlerDefaultModelFor':Called when the default content model is determined for a given title. May be used to assign a different model for that title. $title:the Title in question & $model:the model name. Use with CONTENT_MODEL_XXX constants. 'ContentHandlerForModelID':Called when a ContentHandler is requested for a given content model name, but no entry for that model exists in $wgContentHandlers. Note:if your extension implements additional models via this hook, please use GetContentModels hook to make them known to core. $modeName:the requested content model name & $handler:set this to a ContentHandler object, if desired. 'ContentModelCanBeUsedOn':Called to determine whether that content model can be used on a given page. This is especially useful to prevent some content models to be used in some special location. $contentModel:ID of the content model in question $title:the Title in question. & $ok:Output parameter, whether it is OK to use $contentModel on $title. Handler functions that modify $ok should generally return false to prevent further hooks from further modifying $ok. 'ContribsPager::getQueryInfo':Before the contributions query is about to run & $pager:Pager object for contributions & $queryInfo:The query for the contribs Pager 'ContribsPager::reallyDoQuery':Called before really executing the query for My Contributions & $data:an array of results of all contribs queries $pager:The ContribsPager object hooked into $offset:Index offset, inclusive $limit:Exact query limit $descending:Query direction, false for ascending, true for descending 'ContributionsLineEnding':Called before a contributions HTML line is finished $page:SpecialPage object for contributions & $ret:the HTML line $row:the DB row for this line & $classes:the classes to add to the surrounding< li > & $attribs:associative array of other HTML attributes for the< li > element. Currently only data attributes reserved to MediaWiki are allowed(see Sanitizer::isReservedDataAttribute). 'ContributionsToolLinks':Change tool links above Special:Contributions $id:User identifier $title:User page title & $tools:Array of tool links $specialPage:SpecialPage instance for context and services. Can be either SpecialContributions or DeletedContributionsPage. Extensions should type hint against a generic SpecialPage though. 'ConvertContent':Called by AbstractContent::convert when a conversion to another content model is requested. Handler functions that modify $result should generally return false to disable further attempts at conversion. $content:The Content object to be converted. $toModel:The ID of the content model to convert to. $lossy:boolean indicating whether lossy conversion is allowed. & $result:Output parameter, in case the handler function wants to provide a converted Content object. Note that $result->getContentModel() must return $toModel. 'ContentSecurityPolicyDefaultSource':Modify the allowed CSP load sources. This affects all directives except for the script directive. If you want to add a script source, see ContentSecurityPolicyScriptSource hook. & $defaultSrc:Array of Content-Security-Policy allowed sources $policyConfig:Current configuration for the Content-Security-Policy header $mode:ContentSecurityPolicy::REPORT_ONLY_MODE or ContentSecurityPolicy::FULL_MODE depending on type of header 'ContentSecurityPolicyDirectives':Modify the content security policy directives. Use this only if ContentSecurityPolicyDefaultSource and ContentSecurityPolicyScriptSource do not meet your needs. & $directives:Array of CSP directives $policyConfig:Current configuration for the CSP header $mode:ContentSecurityPolicy::REPORT_ONLY_MODE or ContentSecurityPolicy::FULL_MODE depending on type of header 'ContentSecurityPolicyScriptSource':Modify the allowed CSP script sources. Note that you also have to use ContentSecurityPolicyDefaultSource if you want non-script sources to be loaded from whatever you add. & $scriptSrc:Array of CSP directives $policyConfig:Current configuration for the CSP header $mode:ContentSecurityPolicy::REPORT_ONLY_MODE or ContentSecurityPolicy::FULL_MODE depending on type of header 'CustomEditor':When invoking the page editor Return true to allow the normal editor to be used, or false if implementing a custom editor, e.g. for a special namespace, etc. $article:Article being edited $user:User performing the edit 'DeletedContribsPager::reallyDoQuery':Called before really executing the query for Special:DeletedContributions Similar to ContribsPager::reallyDoQuery & $data:an array of results of all contribs queries $pager:The DeletedContribsPager object hooked into $offset:Index offset, inclusive $limit:Exact query limit $descending:Query direction, false for ascending, true for descending 'DeletedContributionsLineEnding':Called before a DeletedContributions HTML line is finished. Similar to ContributionsLineEnding $page:SpecialPage object for DeletedContributions & $ret:the HTML line $row:the DB row for this line & $classes:the classes to add to the surrounding< li > & $attribs:associative array of other HTML attributes for the< li > element. Currently only data attributes reserved to MediaWiki are allowed(see Sanitizer::isReservedDataAttribute). 'DeleteUnknownPreferences':Called by the cleanupPreferences.php maintenance script to build a WHERE clause with which to delete preferences that are not known about. This hook is used by extensions that have dynamically-named preferences that should not be deleted in the usual cleanup process. For example, the Gadgets extension creates preferences prefixed with 'gadget-', and so anything with that prefix is excluded from the deletion. &where:An array that will be passed as the $cond parameter to IDatabase::select() to determine what will be deleted from the user_properties table. $db:The IDatabase object, useful for accessing $db->buildLike() etc. 'DifferenceEngineAfterLoadNewText':called in DifferenceEngine::loadNewText() after the new revision 's content has been loaded into the class member variable $differenceEngine->mNewContent but before returning true from this function. $differenceEngine:DifferenceEngine object 'DifferenceEngineLoadTextAfterNewContentIsLoaded':called in DifferenceEngine::loadText() after the new revision 's content has been loaded into the class member variable $differenceEngine->mNewContent but before checking if the variable 's value is null. This hook can be used to inject content into said class member variable. $differenceEngine:DifferenceEngine object 'DifferenceEngineMarkPatrolledLink':Allows extensions to change the "mark as patrolled" link which is shown both on the diff header as well as on the bottom of a page, usually wrapped in a span element which has class="patrollink". $differenceEngine:DifferenceEngine object & $markAsPatrolledLink:The "mark as patrolled" link HTML(string) $rcid:Recent change ID(rc_id) for this change(int) 'DifferenceEngineMarkPatrolledRCID':Allows extensions to possibly change the rcid parameter. For example the rcid might be set to zero due to the user being the same as the performer of the change but an extension might still want to show it under certain conditions. & $rcid:rc_id(int) of the change or 0 $differenceEngine:DifferenceEngine object $change:RecentChange object $user:User object representing the current user 'DifferenceEngineNewHeader':Allows extensions to change the $newHeader variable, which contains information about the new revision, such as the revision 's author, whether the revision was marked as a minor edit or not, etc. $differenceEngine:DifferenceEngine object & $newHeader:The string containing the various #mw-diff-otitle[1-5] divs, which include things like revision author info, revision comment, RevisionDelete link and more $formattedRevisionTools:Array containing revision tools, some of which may have been injected with the DiffRevisionTools hook $nextlink:String containing the link to the next revision(if any) $status
Definition: hooks.txt:1263
static newEmpty()
Get an instance that wraps EmptyBagOStuff.
WANObjectCache $memCache
streamFile(array $params)
getFileSha1Base36(array $params)
const LOCK_UW
Definition: LockManager.php:68
doClean(array $params)
MapCacheLRU $expensiveCache
Map of paths to large (RAM/disk) cache items.
executeOpHandlesInternal(array $fileOpHandles)
Execute a list of FileBackendStoreOpHandle handles in parallel.
getContentType( $storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
getFileSize(array $params)
doQuickOperationsInternal(array $ops)
doPrepareInternal( $container, $dir, array $params)
File backend exception for checked exceptions (e.g.
Functions related to the output of file content.
createInternal(array $params)
Create a file in the backend with the given contents.
fileExists(array $params)
directoryExists(array $params)
const LOCK_EX
Definition: LockManager.php:69
$res
Definition: database.txt:21
doStreamFile(array $params)
getFileXAttributes(array $params)
nullInternal(array $params)
No-op file operation that does nothing.
deleteInternal(array $params)
Delete a file at the storage path.
preloadCache(array $paths)
$params
callable $obResetFunc
doGetLocalCopyMulti(array $params)
getFileContentsMulti(array $params)
getFileTimestamp(array $params)
maxFileSizeInternal()
Get the maximum allowable file size given backend medium restrictions and basic performance constrain...
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that When $user is not null
Definition: hooks.txt:780
static placeholderProps()
Placeholder file properties to use for files that don&#39;t exist.
Definition: FSFile.php:143
getOperationsInternal(array $ops)
Return a list of FileOp objects from a list of operations.
callable $streamMimeFunc
clearCache(array $paths=null)
newStatus(... $args)
Yields the result of the status wrapper callback on either:
deleteFileCache( $path)
Delete the cached stat info for a file path.
callable $mimeCallback
Method to get the MIME type of files.
getFileListInternal( $container, $dir, array $params)
Do not call this function from places outside FileBackend.
static send404Message( $fname, $flags=0)
Send out a standard 404 message for a file.
static isValidShortContainerName( $container)
Check if a short container name is valid.
doPrepare(array $params)
isSingleShardPathInternal( $storagePath)
Check if a storage path maps to a single shard.
static isValidContainerName( $container)
Check if a full container name is valid.
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
doGetFileStatMulti(array $params)
Get file stat information (concurrently if possible) for several files.
copyInternal(array $params)
Copy a file from one storage path to another in the backend.
doGetFileStat(array $params)
string $name
Unique backend name.
Definition: FileBackend.php:94
doGetFileContentsMulti(array $params)
Base class for all backends using particular storage medium.
resolveContainerName( $container)
Resolve a container name, checking if it&#39;s allowed by the backend.
preloadFileStat(array $params)
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path.
getFileList(array $params)
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
doSecure(array $params)
static attempt(array $performOps, array $opts, FileJournal $journal)
Attempt to perform a series of file operations.
Definition: FileOpBatch.php:56
resolveStoragePath( $storagePath)
Splits a storage path into an internal container name, an internal relative file name, and a container shard suffix.
doConcatenate(array $params)
doPublish(array $params)
deleteContainerCache( $container)
Delete the cached info for a container.
you have access to all of the normal MediaWiki so you can get a DB use the etc For full docs on the Maintenance class
Definition: maintenance.txt:52
doPrimeContainerCache(array $containerInfo)
Fill the backend-specific process cache given an array of resolved container names and their correspo...
doClearCache(array $paths=null)
Clears any additional stat caches for storage paths.
describeInternal(array $params)
Alter metadata for a file at the storage path.
doStoreInternal(array $params)
Base class for all file backend classes (including multi-write backends).
Definition: FileBackend.php:92
doDeleteInternal(array $params)
getTopDirectoryList(array $params)
Same as FileBackend::getDirectoryList() except only lists directories that are immediately under the ...
doCleanInternal( $container, $dir, array $params)
getScopedLocksForOps(array $ops, StatusValue $status)
fullContainerName( $container)
Get the full container name, including the wiki ID prefix.
doMoveInternal(array $params)
sanitizeOpHeaders(array $op)
Normalize and filter HTTP headers from a file operation.
doDescribeInternal(array $params)
getLocalReferenceMulti(array $params)
$content
Definition: pageupdater.txt:72
FileBackendStore helper class for performing asynchronous file operations.
doGetFileSha1Base36(array $params)
primeFileCache(array $items)
Do a batch lookup from cache for file stats for all paths used in a list of storage paths or FileOp o...
doCopyInternal(array $params)
getLocalReference(array $params)
Returns a file system file, identical to the file at a storage path.
getContainerSuffixes( $container)
Get a list of full container shard suffixes for a container.
getDirectoryList(array $params)
getLocalCopyMulti(array $params)
getFileProps(array $params)
containerCacheKey( $container)
Get the cache key for a container.
doExecuteOpHandlesInternal(array $fileOpHandles)