MediaWiki REL1_30
FileBackendStore.php
Go to the documentation of this file.
1<?php
23use Wikimedia\Timestamp\ConvertibleTimestamp;
24
38abstract class FileBackendStore extends FileBackend {
40 protected $memCache;
42 protected $srvCache;
44 protected $cheapCache;
46 protected $expensiveCache;
47
49 protected $shardViaHashLevels = [];
50
52 protected $mimeCallback;
53
54 protected $maxFileSize = 4294967296; // integer bytes (4GiB)
55
56 const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries
57 const CACHE_CHEAP_SIZE = 500; // integer; max entries in "cheap cache"
58 const CACHE_EXPENSIVE_SIZE = 5; // integer; max entries in "expensive cache"
59
71 public function __construct( array $config ) {
72 parent::__construct( $config );
73 $this->mimeCallback = isset( $config['mimeCallback'] )
74 ? $config['mimeCallback']
75 : null;
76 $this->srvCache = new EmptyBagOStuff(); // disabled by default
77 $this->memCache = WANObjectCache::newEmpty(); // disabled by default
78 $this->cheapCache = new ProcessCacheLRU( self::CACHE_CHEAP_SIZE );
79 $this->expensiveCache = new ProcessCacheLRU( self::CACHE_EXPENSIVE_SIZE );
80 }
81
89 final public function maxFileSizeInternal() {
90 return $this->maxFileSize;
91 }
92
102 abstract public function isPathUsableInternal( $storagePath );
103
122 final public function createInternal( array $params ) {
123 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
124 if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
125 $status = $this->newStatus( 'backend-fail-maxsize',
126 $params['dst'], $this->maxFileSizeInternal() );
127 } else {
128 $status = $this->doCreateInternal( $params );
129 $this->clearCache( [ $params['dst'] ] );
130 if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
131 $this->deleteFileCache( $params['dst'] ); // persistent cache
132 }
133 }
134
135 return $status;
136 }
137
143 abstract protected function doCreateInternal( array $params );
144
163 final public function storeInternal( array $params ) {
164 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
165 if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
166 $status = $this->newStatus( 'backend-fail-maxsize',
167 $params['dst'], $this->maxFileSizeInternal() );
168 } else {
169 $status = $this->doStoreInternal( $params );
170 $this->clearCache( [ $params['dst'] ] );
171 if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
172 $this->deleteFileCache( $params['dst'] ); // persistent cache
173 }
174 }
175
176 return $status;
177 }
178
184 abstract protected function doStoreInternal( array $params );
185
205 final public function copyInternal( array $params ) {
206 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
207 $status = $this->doCopyInternal( $params );
208 $this->clearCache( [ $params['dst'] ] );
209 if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
210 $this->deleteFileCache( $params['dst'] ); // persistent cache
211 }
212
213 return $status;
214 }
215
221 abstract protected function doCopyInternal( array $params );
222
237 final public function deleteInternal( array $params ) {
238 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
239 $status = $this->doDeleteInternal( $params );
240 $this->clearCache( [ $params['src'] ] );
241 $this->deleteFileCache( $params['src'] ); // persistent cache
242 return $status;
243 }
244
250 abstract protected function doDeleteInternal( array $params );
251
271 final public function moveInternal( array $params ) {
272 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
273 $status = $this->doMoveInternal( $params );
274 $this->clearCache( [ $params['src'], $params['dst'] ] );
275 $this->deleteFileCache( $params['src'] ); // persistent cache
276 if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
277 $this->deleteFileCache( $params['dst'] ); // persistent cache
278 }
279
280 return $status;
281 }
282
288 protected function doMoveInternal( array $params ) {
289 unset( $params['async'] ); // two steps, won't work here :)
292 // Copy source to dest
293 $status = $this->copyInternal( $params );
294 if ( $nsrc !== $ndst && $status->isOK() ) {
295 // Delete source (only fails due to races or network problems)
296 $status->merge( $this->deleteInternal( [ 'src' => $params['src'] ] ) );
297 $status->setResult( true, $status->value ); // ignore delete() errors
298 }
299
300 return $status;
301 }
302
317 final public function describeInternal( array $params ) {
318 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
319 if ( count( $params['headers'] ) ) {
320 $status = $this->doDescribeInternal( $params );
321 $this->clearCache( [ $params['src'] ] );
322 $this->deleteFileCache( $params['src'] ); // persistent cache
323 } else {
324 $status = $this->newStatus(); // nothing to do
325 }
326
327 return $status;
328 }
329
335 protected function doDescribeInternal( array $params ) {
336 return $this->newStatus();
337 }
338
346 final public function nullInternal( array $params ) {
347 return $this->newStatus();
348 }
349
350 final public function concatenate( array $params ) {
351 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
352 $status = $this->newStatus();
353
354 // Try to lock the source files for the scope of this function
355 $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
356 if ( $status->isOK() ) {
357 // Actually do the file concatenation...
358 $start_time = microtime( true );
359 $status->merge( $this->doConcatenate( $params ) );
360 $sec = microtime( true ) - $start_time;
361 if ( !$status->isOK() ) {
362 $this->logger->error( static::class . "-{$this->name}" .
363 " failed to concatenate " . count( $params['srcs'] ) . " file(s) [$sec sec]" );
364 }
365 }
366
367 return $status;
368 }
369
375 protected function doConcatenate( array $params ) {
376 $status = $this->newStatus();
377 $tmpPath = $params['dst']; // convenience
378 unset( $params['latest'] ); // sanity
379
380 // Check that the specified temp file is valid...
381 MediaWiki\suppressWarnings();
382 $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 );
383 MediaWiki\restoreWarnings();
384 if ( !$ok ) { // not present or not empty
385 $status->fatal( 'backend-fail-opentemp', $tmpPath );
386
387 return $status;
388 }
389
390 // Get local FS versions of the chunks needed for the concatenation...
391 $fsFiles = $this->getLocalReferenceMulti( $params );
392 foreach ( $fsFiles as $path => &$fsFile ) {
393 if ( !$fsFile ) { // chunk failed to download?
394 $fsFile = $this->getLocalReference( [ 'src' => $path ] );
395 if ( !$fsFile ) { // retry failed?
396 $status->fatal( 'backend-fail-read', $path );
397
398 return $status;
399 }
400 }
401 }
402 unset( $fsFile ); // unset reference so we can reuse $fsFile
403
404 // Get a handle for the destination temp file
405 $tmpHandle = fopen( $tmpPath, 'ab' );
406 if ( $tmpHandle === false ) {
407 $status->fatal( 'backend-fail-opentemp', $tmpPath );
408
409 return $status;
410 }
411
412 // Build up the temp file using the source chunks (in order)...
413 foreach ( $fsFiles as $virtualSource => $fsFile ) {
414 // Get a handle to the local FS version
415 $sourceHandle = fopen( $fsFile->getPath(), 'rb' );
416 if ( $sourceHandle === false ) {
417 fclose( $tmpHandle );
418 $status->fatal( 'backend-fail-read', $virtualSource );
419
420 return $status;
421 }
422 // Append chunk to file (pass chunk size to avoid magic quotes)
423 if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
424 fclose( $sourceHandle );
425 fclose( $tmpHandle );
426 $status->fatal( 'backend-fail-writetemp', $tmpPath );
427
428 return $status;
429 }
430 fclose( $sourceHandle );
431 }
432 if ( !fclose( $tmpHandle ) ) {
433 $status->fatal( 'backend-fail-closetemp', $tmpPath );
434
435 return $status;
436 }
437
438 clearstatcache(); // temp file changed
439
440 return $status;
441 }
442
443 final protected function doPrepare( array $params ) {
444 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
445 $status = $this->newStatus();
446
447 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
448 if ( $dir === null ) {
449 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
450
451 return $status; // invalid storage path
452 }
453
454 if ( $shard !== null ) { // confined to a single container/shard
455 $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
456 } else { // directory is on several shards
457 $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
458 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
459 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
460 $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
461 }
462 }
463
464 return $status;
465 }
466
474 protected function doPrepareInternal( $container, $dir, array $params ) {
475 return $this->newStatus();
476 }
477
478 final protected function doSecure( array $params ) {
479 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
480 $status = $this->newStatus();
481
482 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
483 if ( $dir === null ) {
484 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
485
486 return $status; // invalid storage path
487 }
488
489 if ( $shard !== null ) { // confined to a single container/shard
490 $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
491 } else { // directory is on several shards
492 $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
493 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
494 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
495 $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
496 }
497 }
498
499 return $status;
500 }
501
509 protected function doSecureInternal( $container, $dir, array $params ) {
510 return $this->newStatus();
511 }
512
513 final protected function doPublish( array $params ) {
514 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
515 $status = $this->newStatus();
516
517 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
518 if ( $dir === null ) {
519 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
520
521 return $status; // invalid storage path
522 }
523
524 if ( $shard !== null ) { // confined to a single container/shard
525 $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
526 } else { // directory is on several shards
527 $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
528 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
529 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
530 $status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) );
531 }
532 }
533
534 return $status;
535 }
536
544 protected function doPublishInternal( $container, $dir, array $params ) {
545 return $this->newStatus();
546 }
547
548 final protected function doClean( array $params ) {
549 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
550 $status = $this->newStatus();
551
552 // Recursive: first delete all empty subdirs recursively
553 if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) {
554 $subDirsRel = $this->getTopDirectoryList( [ 'dir' => $params['dir'] ] );
555 if ( $subDirsRel !== null ) { // no errors
556 foreach ( $subDirsRel as $subDirRel ) {
557 $subDir = $params['dir'] . "/{$subDirRel}"; // full path
558 $status->merge( $this->doClean( [ 'dir' => $subDir ] + $params ) );
559 }
560 unset( $subDirsRel ); // free directory for rmdir() on Windows (for FS backends)
561 }
562 }
563
564 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
565 if ( $dir === null ) {
566 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
567
568 return $status; // invalid storage path
569 }
570
571 // Attempt to lock this directory...
572 $filesLockEx = [ $params['dir'] ];
573 $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
574 if ( !$status->isOK() ) {
575 return $status; // abort
576 }
577
578 if ( $shard !== null ) { // confined to a single container/shard
579 $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
580 $this->deleteContainerCache( $fullCont ); // purge cache
581 } else { // directory is on several shards
582 $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
583 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
584 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
585 $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
586 $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache
587 }
588 }
589
590 return $status;
591 }
592
600 protected function doCleanInternal( $container, $dir, array $params ) {
601 return $this->newStatus();
602 }
603
604 final public function fileExists( array $params ) {
605 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
606 $stat = $this->getFileStat( $params );
607
608 return ( $stat === null ) ? null : (bool)$stat; // null => failure
609 }
610
611 final public function getFileTimestamp( array $params ) {
612 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
613 $stat = $this->getFileStat( $params );
614
615 return $stat ? $stat['mtime'] : false;
616 }
617
618 final public function getFileSize( array $params ) {
619 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
620 $stat = $this->getFileStat( $params );
621
622 return $stat ? $stat['size'] : false;
623 }
624
625 final public function getFileStat( array $params ) {
626 $path = self::normalizeStoragePath( $params['src'] );
627 if ( $path === null ) {
628 return false; // invalid storage path
629 }
630 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
631 $latest = !empty( $params['latest'] ); // use latest data?
632 if ( !$latest && !$this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) {
633 $this->primeFileCache( [ $path ] ); // check persistent cache
634 }
635 if ( $this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) {
636 $stat = $this->cheapCache->get( $path, 'stat' );
637 // If we want the latest data, check that this cached
638 // value was in fact fetched with the latest available data.
639 if ( is_array( $stat ) ) {
640 if ( !$latest || $stat['latest'] ) {
641 return $stat;
642 }
643 } elseif ( in_array( $stat, [ 'NOT_EXIST', 'NOT_EXIST_LATEST' ] ) ) {
644 if ( !$latest || $stat === 'NOT_EXIST_LATEST' ) {
645 return false;
646 }
647 }
648 }
649 $stat = $this->doGetFileStat( $params );
650 if ( is_array( $stat ) ) { // file exists
651 // Strongly consistent backends can automatically set "latest"
652 $stat['latest'] = isset( $stat['latest'] ) ? $stat['latest'] : $latest;
653 $this->cheapCache->set( $path, 'stat', $stat );
654 $this->setFileCache( $path, $stat ); // update persistent cache
655 if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata
656 $this->cheapCache->set( $path, 'sha1',
657 [ 'hash' => $stat['sha1'], 'latest' => $latest ] );
658 }
659 if ( isset( $stat['xattr'] ) ) { // some backends store headers/metadata
660 $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] );
661 $this->cheapCache->set( $path, 'xattr',
662 [ 'map' => $stat['xattr'], 'latest' => $latest ] );
663 }
664 } elseif ( $stat === false ) { // file does not exist
665 $this->cheapCache->set( $path, 'stat', $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' );
666 $this->cheapCache->set( $path, 'xattr', [ 'map' => false, 'latest' => $latest ] );
667 $this->cheapCache->set( $path, 'sha1', [ 'hash' => false, 'latest' => $latest ] );
668 $this->logger->debug( __METHOD__ . ": File $path does not exist.\n" );
669 } else { // an error occurred
670 $this->logger->warning( __METHOD__ . ": Could not stat file $path.\n" );
671 }
672
673 return $stat;
674 }
675
680 abstract protected function doGetFileStat( array $params );
681
682 public function getFileContentsMulti( array $params ) {
683 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
684
685 $params = $this->setConcurrencyFlags( $params );
686 $contents = $this->doGetFileContentsMulti( $params );
687
688 return $contents;
689 }
690
696 protected function doGetFileContentsMulti( array $params ) {
697 $contents = [];
698 foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
699 MediaWiki\suppressWarnings();
700 $contents[$path] = $fsFile ? file_get_contents( $fsFile->getPath() ) : false;
701 MediaWiki\restoreWarnings();
702 }
703
704 return $contents;
705 }
706
707 final public function getFileXAttributes( array $params ) {
708 $path = self::normalizeStoragePath( $params['src'] );
709 if ( $path === null ) {
710 return false; // invalid storage path
711 }
712 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
713 $latest = !empty( $params['latest'] ); // use latest data?
714 if ( $this->cheapCache->has( $path, 'xattr', self::CACHE_TTL ) ) {
715 $stat = $this->cheapCache->get( $path, 'xattr' );
716 // If we want the latest data, check that this cached
717 // value was in fact fetched with the latest available data.
718 if ( !$latest || $stat['latest'] ) {
719 return $stat['map'];
720 }
721 }
722 $fields = $this->doGetFileXAttributes( $params );
723 $fields = is_array( $fields ) ? self::normalizeXAttributes( $fields ) : false;
724 $this->cheapCache->set( $path, 'xattr', [ 'map' => $fields, 'latest' => $latest ] );
725
726 return $fields;
727 }
728
734 protected function doGetFileXAttributes( array $params ) {
735 return [ 'headers' => [], 'metadata' => [] ]; // not supported
736 }
737
738 final public function getFileSha1Base36( array $params ) {
739 $path = self::normalizeStoragePath( $params['src'] );
740 if ( $path === null ) {
741 return false; // invalid storage path
742 }
743 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
744 $latest = !empty( $params['latest'] ); // use latest data?
745 if ( $this->cheapCache->has( $path, 'sha1', self::CACHE_TTL ) ) {
746 $stat = $this->cheapCache->get( $path, 'sha1' );
747 // If we want the latest data, check that this cached
748 // value was in fact fetched with the latest available data.
749 if ( !$latest || $stat['latest'] ) {
750 return $stat['hash'];
751 }
752 }
753 $hash = $this->doGetFileSha1Base36( $params );
754 $this->cheapCache->set( $path, 'sha1', [ 'hash' => $hash, 'latest' => $latest ] );
755
756 return $hash;
757 }
758
764 protected function doGetFileSha1Base36( array $params ) {
765 $fsFile = $this->getLocalReference( $params );
766 if ( !$fsFile ) {
767 return false;
768 } else {
769 return $fsFile->getSha1Base36();
770 }
771 }
772
773 final public function getFileProps( array $params ) {
774 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
775 $fsFile = $this->getLocalReference( $params );
776 $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
777
778 return $props;
779 }
780
781 final public function getLocalReferenceMulti( array $params ) {
782 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
783
784 $params = $this->setConcurrencyFlags( $params );
785
786 $fsFiles = []; // (path => FSFile)
787 $latest = !empty( $params['latest'] ); // use latest data?
788 // Reuse any files already in process cache...
789 foreach ( $params['srcs'] as $src ) {
790 $path = self::normalizeStoragePath( $src );
791 if ( $path === null ) {
792 $fsFiles[$src] = null; // invalid storage path
793 } elseif ( $this->expensiveCache->has( $path, 'localRef' ) ) {
794 $val = $this->expensiveCache->get( $path, 'localRef' );
795 // If we want the latest data, check that this cached
796 // value was in fact fetched with the latest available data.
797 if ( !$latest || $val['latest'] ) {
798 $fsFiles[$src] = $val['object'];
799 }
800 }
801 }
802 // Fetch local references of any remaning files...
803 $params['srcs'] = array_diff( $params['srcs'], array_keys( $fsFiles ) );
804 foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
805 $fsFiles[$path] = $fsFile;
806 if ( $fsFile ) { // update the process cache...
807 $this->expensiveCache->set( $path, 'localRef',
808 [ 'object' => $fsFile, 'latest' => $latest ] );
809 }
810 }
811
812 return $fsFiles;
813 }
814
820 protected function doGetLocalReferenceMulti( array $params ) {
821 return $this->doGetLocalCopyMulti( $params );
822 }
823
824 final public function getLocalCopyMulti( array $params ) {
825 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
826
827 $params = $this->setConcurrencyFlags( $params );
828 $tmpFiles = $this->doGetLocalCopyMulti( $params );
829
830 return $tmpFiles;
831 }
832
838 abstract protected function doGetLocalCopyMulti( array $params );
839
845 public function getFileHttpUrl( array $params ) {
846 return null; // not supported
847 }
848
849 final public function streamFile( array $params ) {
850 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
851 $status = $this->newStatus();
852
853 // Always set some fields for subclass convenience
854 $params['options'] = isset( $params['options'] ) ? $params['options'] : [];
855 $params['headers'] = isset( $params['headers'] ) ? $params['headers'] : [];
856
857 // Don't stream it out as text/html if there was a PHP error
858 if ( ( empty( $params['headless'] ) || $params['headers'] ) && headers_sent() ) {
859 print "Headers already sent, terminating.\n";
860 $status->fatal( 'backend-fail-stream', $params['src'] );
861 return $status;
862 }
863
864 $status->merge( $this->doStreamFile( $params ) );
865
866 return $status;
867 }
868
874 protected function doStreamFile( array $params ) {
875 $status = $this->newStatus();
876
877 $flags = 0;
878 $flags |= !empty( $params['headless'] ) ? HTTPFileStreamer::STREAM_HEADLESS : 0;
879 $flags |= !empty( $params['allowOB'] ) ? HTTPFileStreamer::STREAM_ALLOW_OB : 0;
880
881 $fsFile = $this->getLocalReference( $params );
882 if ( $fsFile ) {
883 $streamer = new HTTPFileStreamer(
884 $fsFile->getPath(),
885 [
886 'obResetFunc' => $this->obResetFunc,
887 'streamMimeFunc' => $this->streamMimeFunc
888 ]
889 );
890 $res = $streamer->stream( $params['headers'], true, $params['options'], $flags );
891 } else {
892 $res = false;
894 }
895
896 if ( !$res ) {
897 $status->fatal( 'backend-fail-stream', $params['src'] );
898 }
899
900 return $status;
901 }
902
903 final public function directoryExists( array $params ) {
904 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
905 if ( $dir === null ) {
906 return false; // invalid storage path
907 }
908 if ( $shard !== null ) { // confined to a single container/shard
909 return $this->doDirectoryExists( $fullCont, $dir, $params );
910 } else { // directory is on several shards
911 $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
912 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
913 $res = false; // response
914 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
915 $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params );
916 if ( $exists ) {
917 $res = true;
918 break; // found one!
919 } elseif ( $exists === null ) { // error?
920 $res = null; // if we don't find anything, it is indeterminate
921 }
922 }
923
924 return $res;
925 }
926 }
927
936 abstract protected function doDirectoryExists( $container, $dir, array $params );
937
938 final public function getDirectoryList( array $params ) {
939 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
940 if ( $dir === null ) { // invalid storage path
941 return null;
942 }
943 if ( $shard !== null ) {
944 // File listing is confined to a single container/shard
945 return $this->getDirectoryListInternal( $fullCont, $dir, $params );
946 } else {
947 $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
948 // File listing spans multiple containers/shards
949 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
950
951 return new FileBackendStoreShardDirIterator( $this,
952 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
953 }
954 }
955
966 abstract public function getDirectoryListInternal( $container, $dir, array $params );
967
968 final public function getFileList( array $params ) {
969 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
970 if ( $dir === null ) { // invalid storage path
971 return null;
972 }
973 if ( $shard !== null ) {
974 // File listing is confined to a single container/shard
975 return $this->getFileListInternal( $fullCont, $dir, $params );
976 } else {
977 $this->logger->debug( __METHOD__ . ": iterating over all container shards.\n" );
978 // File listing spans multiple containers/shards
979 list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
980
981 return new FileBackendStoreShardFileIterator( $this,
982 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
983 }
984 }
985
996 abstract public function getFileListInternal( $container, $dir, array $params );
997
1009 final public function getOperationsInternal( array $ops ) {
1010 $supportedOps = [
1011 'store' => 'StoreFileOp',
1012 'copy' => 'CopyFileOp',
1013 'move' => 'MoveFileOp',
1014 'delete' => 'DeleteFileOp',
1015 'create' => 'CreateFileOp',
1016 'describe' => 'DescribeFileOp',
1017 'null' => 'NullFileOp'
1018 ];
1019
1020 $performOps = []; // array of FileOp objects
1021 // Build up ordered array of FileOps...
1022 foreach ( $ops as $operation ) {
1023 $opName = $operation['op'];
1024 if ( isset( $supportedOps[$opName] ) ) {
1025 $class = $supportedOps[$opName];
1026 // Get params for this operation
1027 $params = $operation;
1028 // Append the FileOp class
1029 $performOps[] = new $class( $this, $params, $this->logger );
1030 } else {
1031 throw new FileBackendError( "Operation '$opName' is not supported." );
1032 }
1033 }
1034
1035 return $performOps;
1036 }
1037
1048 final public function getPathsToLockForOpsInternal( array $performOps ) {
1049 // Build up a list of files to lock...
1050 $paths = [ 'sh' => [], 'ex' => [] ];
1051 foreach ( $performOps as $fileOp ) {
1052 $paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() );
1053 $paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() );
1054 }
1055 // Optimization: if doing an EX lock anyway, don't also set an SH one
1056 $paths['sh'] = array_diff( $paths['sh'], $paths['ex'] );
1057 // Get a shared lock on the parent directory of each path changed
1058 $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) );
1059
1060 return [
1061 LockManager::LOCK_UW => $paths['sh'],
1062 LockManager::LOCK_EX => $paths['ex']
1063 ];
1064 }
1065
1066 public function getScopedLocksForOps( array $ops, StatusValue $status ) {
1067 $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) );
1068
1069 return $this->getScopedFileLocks( $paths, 'mixed', $status );
1070 }
1071
1072 final protected function doOperationsInternal( array $ops, array $opts ) {
1073 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1074 $status = $this->newStatus();
1075
1076 // Fix up custom header name/value pairs...
1077 $ops = array_map( [ $this, 'sanitizeOpHeaders' ], $ops );
1078
1079 // Build up a list of FileOps...
1080 $performOps = $this->getOperationsInternal( $ops );
1081
1082 // Acquire any locks as needed...
1083 if ( empty( $opts['nonLocking'] ) ) {
1084 // Build up a list of files to lock...
1085 $paths = $this->getPathsToLockForOpsInternal( $performOps );
1086 // Try to lock those files for the scope of this function...
1087
1088 $scopeLock = $this->getScopedFileLocks( $paths, 'mixed', $status );
1089 if ( !$status->isOK() ) {
1090 return $status; // abort
1091 }
1092 }
1093
1094 // Clear any file cache entries (after locks acquired)
1095 if ( empty( $opts['preserveCache'] ) ) {
1096 $this->clearCache();
1097 }
1098
1099 // Build the list of paths involved
1100 $paths = [];
1101 foreach ( $performOps as $performOp ) {
1102 $paths = array_merge( $paths, $performOp->storagePathsRead() );
1103 $paths = array_merge( $paths, $performOp->storagePathsChanged() );
1104 }
1105
1106 // Enlarge the cache to fit the stat entries of these files
1107 $this->cheapCache->resize( max( 2 * count( $paths ), self::CACHE_CHEAP_SIZE ) );
1108
1109 // Load from the persistent container caches
1110 $this->primeContainerCache( $paths );
1111 // Get the latest stat info for all the files (having locked them)
1112 $ok = $this->preloadFileStat( [ 'srcs' => $paths, 'latest' => true ] );
1113
1114 if ( $ok ) {
1115 // Actually attempt the operation batch...
1116 $opts = $this->setConcurrencyFlags( $opts );
1117 $subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal );
1118 } else {
1119 // If we could not even stat some files, then bail out...
1120 $subStatus = $this->newStatus( 'backend-fail-internal', $this->name );
1121 foreach ( $ops as $i => $op ) { // mark each op as failed
1122 $subStatus->success[$i] = false;
1123 ++$subStatus->failCount;
1124 }
1125 $this->logger->error( static::class . "-{$this->name} " .
1126 " stat failure; aborted operations: " . FormatJson::encode( $ops ) );
1127 }
1128
1129 // Merge errors into StatusValue fields
1130 $status->merge( $subStatus );
1131 $status->success = $subStatus->success; // not done in merge()
1132
1133 // Shrink the stat cache back to normal size
1134 $this->cheapCache->resize( self::CACHE_CHEAP_SIZE );
1135
1136 return $status;
1137 }
1138
1139 final protected function doQuickOperationsInternal( array $ops ) {
1140 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1141 $status = $this->newStatus();
1142
1143 // Fix up custom header name/value pairs...
1144 $ops = array_map( [ $this, 'sanitizeOpHeaders' ], $ops );
1145
1146 // Clear any file cache entries
1147 $this->clearCache();
1148
1149 $supportedOps = [ 'create', 'store', 'copy', 'move', 'delete', 'describe', 'null' ];
1150 // Parallel ops may be disabled in config due to dependencies (e.g. needing popen())
1151 $async = ( $this->parallelize === 'implicit' && count( $ops ) > 1 );
1152 $maxConcurrency = $this->concurrency; // throttle
1154 $statuses = []; // array of (index => StatusValue)
1155 $fileOpHandles = []; // list of (index => handle) arrays
1156 $curFileOpHandles = []; // current handle batch
1157 // Perform the sync-only ops and build up op handles for the async ops...
1158 foreach ( $ops as $index => $params ) {
1159 if ( !in_array( $params['op'], $supportedOps ) ) {
1160 throw new FileBackendError( "Operation '{$params['op']}' is not supported." );
1161 }
1162 $method = $params['op'] . 'Internal'; // e.g. "storeInternal"
1163 $subStatus = $this->$method( [ 'async' => $async ] + $params );
1164 if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { // async
1165 if ( count( $curFileOpHandles ) >= $maxConcurrency ) {
1166 $fileOpHandles[] = $curFileOpHandles; // push this batch
1167 $curFileOpHandles = [];
1168 }
1169 $curFileOpHandles[$index] = $subStatus->value; // keep index
1170 } else { // error or completed
1171 $statuses[$index] = $subStatus; // keep index
1172 }
1173 }
1174 if ( count( $curFileOpHandles ) ) {
1175 $fileOpHandles[] = $curFileOpHandles; // last batch
1176 }
1177 // Do all the async ops that can be done concurrently...
1178 foreach ( $fileOpHandles as $fileHandleBatch ) {
1179 $statuses = $statuses + $this->executeOpHandlesInternal( $fileHandleBatch );
1180 }
1181 // Marshall and merge all the responses...
1182 foreach ( $statuses as $index => $subStatus ) {
1183 $status->merge( $subStatus );
1184 if ( $subStatus->isOK() ) {
1185 $status->success[$index] = true;
1186 ++$status->successCount;
1187 } else {
1188 $status->success[$index] = false;
1189 ++$status->failCount;
1190 }
1191 }
1192
1193 return $status;
1194 }
1195
1205 final public function executeOpHandlesInternal( array $fileOpHandles ) {
1206 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1207
1208 foreach ( $fileOpHandles as $fileOpHandle ) {
1209 if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) {
1210 throw new InvalidArgumentException( "Expected FileBackendStoreOpHandle object." );
1211 } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) {
1212 throw new InvalidArgumentException( "Expected handle for this file backend." );
1213 }
1214 }
1215
1216 $res = $this->doExecuteOpHandlesInternal( $fileOpHandles );
1217 foreach ( $fileOpHandles as $fileOpHandle ) {
1218 $fileOpHandle->closeResources();
1219 }
1220
1221 return $res;
1222 }
1223
1232 protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1233 if ( count( $fileOpHandles ) ) {
1234 throw new LogicException( "Backend does not support asynchronous operations." );
1235 }
1236
1237 return [];
1238 }
1239
1251 protected function sanitizeOpHeaders( array $op ) {
1252 static $longs = [ 'content-disposition' ];
1253
1254 if ( isset( $op['headers'] ) ) { // op sets HTTP headers
1255 $newHeaders = [];
1256 foreach ( $op['headers'] as $name => $value ) {
1257 $name = strtolower( $name );
1258 $maxHVLen = in_array( $name, $longs ) ? INF : 255;
1259 if ( strlen( $name ) > 255 || strlen( $value ) > $maxHVLen ) {
1260 trigger_error( "Header '$name: $value' is too long." );
1261 } else {
1262 $newHeaders[$name] = strlen( $value ) ? $value : ''; // null/false => ""
1263 }
1264 }
1265 $op['headers'] = $newHeaders;
1266 }
1267
1268 return $op;
1269 }
1270
1271 final public function preloadCache( array $paths ) {
1272 $fullConts = []; // full container names
1273 foreach ( $paths as $path ) {
1274 list( $fullCont, , ) = $this->resolveStoragePath( $path );
1275 $fullConts[] = $fullCont;
1276 }
1277 // Load from the persistent file and container caches
1278 $this->primeContainerCache( $fullConts );
1279 $this->primeFileCache( $paths );
1280 }
1281
1282 final public function clearCache( array $paths = null ) {
1283 if ( is_array( $paths ) ) {
1284 $paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
1285 $paths = array_filter( $paths, 'strlen' ); // remove nulls
1286 }
1287 if ( $paths === null ) {
1288 $this->cheapCache->clear();
1289 $this->expensiveCache->clear();
1290 } else {
1291 foreach ( $paths as $path ) {
1292 $this->cheapCache->clear( $path );
1293 $this->expensiveCache->clear( $path );
1294 }
1295 }
1296 $this->doClearCache( $paths );
1297 }
1298
1306 protected function doClearCache( array $paths = null ) {
1307 }
1308
1309 final public function preloadFileStat( array $params ) {
1310 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1311 $success = true; // no network errors
1312
1313 $params['concurrency'] = ( $this->parallelize !== 'off' ) ? $this->concurrency : 1;
1314 $stats = $this->doGetFileStatMulti( $params );
1315 if ( $stats === null ) {
1316 return true; // not supported
1317 }
1318
1319 $latest = !empty( $params['latest'] ); // use latest data?
1320 foreach ( $stats as $path => $stat ) {
1321 $path = FileBackend::normalizeStoragePath( $path );
1322 if ( $path === null ) {
1323 continue; // this shouldn't happen
1324 }
1325 if ( is_array( $stat ) ) { // file exists
1326 // Strongly consistent backends can automatically set "latest"
1327 $stat['latest'] = isset( $stat['latest'] ) ? $stat['latest'] : $latest;
1328 $this->cheapCache->set( $path, 'stat', $stat );
1329 $this->setFileCache( $path, $stat ); // update persistent cache
1330 if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata
1331 $this->cheapCache->set( $path, 'sha1',
1332 [ 'hash' => $stat['sha1'], 'latest' => $latest ] );
1333 }
1334 if ( isset( $stat['xattr'] ) ) { // some backends store headers/metadata
1335 $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] );
1336 $this->cheapCache->set( $path, 'xattr',
1337 [ 'map' => $stat['xattr'], 'latest' => $latest ] );
1338 }
1339 } elseif ( $stat === false ) { // file does not exist
1340 $this->cheapCache->set( $path, 'stat',
1341 $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' );
1342 $this->cheapCache->set( $path, 'xattr',
1343 [ 'map' => false, 'latest' => $latest ] );
1344 $this->cheapCache->set( $path, 'sha1',
1345 [ 'hash' => false, 'latest' => $latest ] );
1346 $this->logger->debug( __METHOD__ . ": File $path does not exist.\n" );
1347 } else { // an error occurred
1348 $success = false;
1349 $this->logger->warning( __METHOD__ . ": Could not stat file $path.\n" );
1350 }
1351 }
1352
1353 return $success;
1354 }
1355
1367 protected function doGetFileStatMulti( array $params ) {
1368 return null; // not supported
1369 }
1370
1378 abstract protected function directoriesAreVirtual();
1379
1390 final protected static function isValidShortContainerName( $container ) {
1391 // Suffixes like '.xxx' (hex shard chars) or '.seg' (file segments)
1392 // might be used by subclasses. Reserve the dot character for sanity.
1393 // The only way dots end up in containers (e.g. resolveStoragePath)
1394 // is due to the wikiId container prefix or the above suffixes.
1395 return self::isValidContainerName( $container ) && !preg_match( '/[.]/', $container );
1396 }
1397
1407 final protected static function isValidContainerName( $container ) {
1408 // This accounts for NTFS, Swift, and Ceph restrictions
1409 // and disallows directory separators or traversal characters.
1410 // Note that matching strings URL encode to the same string;
1411 // in Swift/Ceph, the length restriction is *after* URL encoding.
1412 return (bool)preg_match( '/^[a-z0-9][a-z0-9-_.]{0,199}$/i', $container );
1413 }
1414
1428 final protected function resolveStoragePath( $storagePath ) {
1429 list( $backend, $shortCont, $relPath ) = self::splitStoragePath( $storagePath );
1430 if ( $backend === $this->name ) { // must be for this backend
1431 $relPath = self::normalizeContainerPath( $relPath );
1432 if ( $relPath !== null && self::isValidShortContainerName( $shortCont ) ) {
1433 // Get shard for the normalized path if this container is sharded
1434 $cShard = $this->getContainerShard( $shortCont, $relPath );
1435 // Validate and sanitize the relative path (backend-specific)
1436 $relPath = $this->resolveContainerPath( $shortCont, $relPath );
1437 if ( $relPath !== null ) {
1438 // Prepend any wiki ID prefix to the container name
1439 $container = $this->fullContainerName( $shortCont );
1440 if ( self::isValidContainerName( $container ) ) {
1441 // Validate and sanitize the container name (backend-specific)
1442 $container = $this->resolveContainerName( "{$container}{$cShard}" );
1443 if ( $container !== null ) {
1444 return [ $container, $relPath, $cShard ];
1445 }
1446 }
1447 }
1448 }
1449 }
1450
1451 return [ null, null, null ];
1452 }
1453
1469 final protected function resolveStoragePathReal( $storagePath ) {
1470 list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
1471 if ( $cShard !== null && substr( $relPath, -1 ) !== '/' ) {
1472 return [ $container, $relPath ];
1473 }
1474
1475 return [ null, null ];
1476 }
1477
1486 final protected function getContainerShard( $container, $relPath ) {
1487 list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container );
1488 if ( $levels == 1 || $levels == 2 ) {
1489 // Hash characters are either base 16 or 36
1490 $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]';
1491 // Get a regex that represents the shard portion of paths.
1492 // The concatenation of the captures gives us the shard.
1493 if ( $levels === 1 ) { // 16 or 36 shards per container
1494 $hashDirRegex = '(' . $char . ')';
1495 } else { // 256 or 1296 shards per container
1496 if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc")
1497 $hashDirRegex = $char . '/(' . $char . '{2})';
1498 } else { // short hash dir format (e.g. "a/b/c")
1499 $hashDirRegex = '(' . $char . ')/(' . $char . ')';
1500 }
1501 }
1502 // Allow certain directories to be above the hash dirs so as
1503 // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
1504 // They must be 2+ chars to avoid any hash directory ambiguity.
1505 $m = [];
1506 if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1507 return '.' . implode( '', array_slice( $m, 1 ) );
1508 }
1509
1510 return null; // failed to match
1511 }
1512
1513 return ''; // no sharding
1514 }
1515
1524 final public function isSingleShardPathInternal( $storagePath ) {
1525 list( , , $shard ) = $this->resolveStoragePath( $storagePath );
1526
1527 return ( $shard !== null );
1528 }
1529
1538 final protected function getContainerHashLevels( $container ) {
1539 if ( isset( $this->shardViaHashLevels[$container] ) ) {
1540 $config = $this->shardViaHashLevels[$container];
1541 $hashLevels = (int)$config['levels'];
1542 if ( $hashLevels == 1 || $hashLevels == 2 ) {
1543 $hashBase = (int)$config['base'];
1544 if ( $hashBase == 16 || $hashBase == 36 ) {
1545 return [ $hashLevels, $hashBase, $config['repeat'] ];
1546 }
1547 }
1548 }
1549
1550 return [ 0, 0, false ]; // no sharding
1551 }
1552
1559 final protected function getContainerSuffixes( $container ) {
1560 $shards = [];
1561 list( $digits, $base ) = $this->getContainerHashLevels( $container );
1562 if ( $digits > 0 ) {
1563 $numShards = pow( $base, $digits );
1564 for ( $index = 0; $index < $numShards; $index++ ) {
1565 $shards[] = '.' . Wikimedia\base_convert( $index, 10, $base, $digits );
1566 }
1567 }
1568
1569 return $shards;
1570 }
1571
1578 final protected function fullContainerName( $container ) {
1579 if ( $this->domainId != '' ) {
1580 return "{$this->domainId}-$container";
1581 } else {
1582 return $container;
1583 }
1584 }
1585
1594 protected function resolveContainerName( $container ) {
1595 return $container;
1596 }
1597
1608 protected function resolveContainerPath( $container, $relStoragePath ) {
1609 return $relStoragePath;
1610 }
1611
1618 private function containerCacheKey( $container ) {
1619 return "filebackend:{$this->name}:{$this->domainId}:container:{$container}";
1620 }
1621
1628 final protected function setContainerCache( $container, array $val ) {
1629 $this->memCache->set( $this->containerCacheKey( $container ), $val, 14 * 86400 );
1630 }
1631
1638 final protected function deleteContainerCache( $container ) {
1639 if ( !$this->memCache->delete( $this->containerCacheKey( $container ), 300 ) ) {
1640 trigger_error( "Unable to delete stat cache for container $container." );
1641 }
1642 }
1643
1651 final protected function primeContainerCache( array $items ) {
1652 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1653
1654 $paths = []; // list of storage paths
1655 $contNames = []; // (cache key => resolved container name)
1656 // Get all the paths/containers from the items...
1657 foreach ( $items as $item ) {
1658 if ( self::isStoragePath( $item ) ) {
1659 $paths[] = $item;
1660 } elseif ( is_string( $item ) ) { // full container name
1661 $contNames[$this->containerCacheKey( $item )] = $item;
1662 }
1663 }
1664 // Get all the corresponding cache keys for paths...
1665 foreach ( $paths as $path ) {
1666 list( $fullCont, , ) = $this->resolveStoragePath( $path );
1667 if ( $fullCont !== null ) { // valid path for this backend
1668 $contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
1669 }
1670 }
1671
1672 $contInfo = []; // (resolved container name => cache value)
1673 // Get all cache entries for these container cache keys...
1674 $values = $this->memCache->getMulti( array_keys( $contNames ) );
1675 foreach ( $values as $cacheKey => $val ) {
1676 $contInfo[$contNames[$cacheKey]] = $val;
1677 }
1678
1679 // Populate the container process cache for the backend...
1680 $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) );
1681 }
1682
1690 protected function doPrimeContainerCache( array $containerInfo ) {
1691 }
1692
1699 private function fileCacheKey( $path ) {
1700 return "filebackend:{$this->name}:{$this->domainId}:file:" . sha1( $path );
1701 }
1702
1711 final protected function setFileCache( $path, array $val ) {
1712 $path = FileBackend::normalizeStoragePath( $path );
1713 if ( $path === null ) {
1714 return; // invalid storage path
1715 }
1716 $mtime = ConvertibleTimestamp::convert( TS_UNIX, $val['mtime'] );
1717 $ttl = $this->memCache->adaptiveTTL( $mtime, 7 * 86400, 300, 0.1 );
1718 $key = $this->fileCacheKey( $path );
1719 // Set the cache unless it is currently salted.
1720 $this->memCache->set( $key, $val, $ttl );
1721 }
1722
1731 final protected function deleteFileCache( $path ) {
1732 $path = FileBackend::normalizeStoragePath( $path );
1733 if ( $path === null ) {
1734 return; // invalid storage path
1735 }
1736 if ( !$this->memCache->delete( $this->fileCacheKey( $path ), 300 ) ) {
1737 trigger_error( "Unable to delete stat cache for file $path." );
1738 }
1739 }
1740
1748 final protected function primeFileCache( array $items ) {
1749 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1750
1751 $paths = []; // list of storage paths
1752 $pathNames = []; // (cache key => storage path)
1753 // Get all the paths/containers from the items...
1754 foreach ( $items as $item ) {
1755 if ( self::isStoragePath( $item ) ) {
1756 $paths[] = FileBackend::normalizeStoragePath( $item );
1757 }
1758 }
1759 // Get rid of any paths that failed normalization...
1760 $paths = array_filter( $paths, 'strlen' ); // remove nulls
1761 // Get all the corresponding cache keys for paths...
1762 foreach ( $paths as $path ) {
1763 list( , $rel, ) = $this->resolveStoragePath( $path );
1764 if ( $rel !== null ) { // valid path for this backend
1765 $pathNames[$this->fileCacheKey( $path )] = $path;
1766 }
1767 }
1768 // Get all cache entries for these file cache keys...
1769 $values = $this->memCache->getMulti( array_keys( $pathNames ) );
1770 foreach ( $values as $cacheKey => $val ) {
1771 $path = $pathNames[$cacheKey];
1772 if ( is_array( $val ) ) {
1773 $val['latest'] = false; // never completely trust cache
1774 $this->cheapCache->set( $path, 'stat', $val );
1775 if ( isset( $val['sha1'] ) ) { // some backends store SHA-1 as metadata
1776 $this->cheapCache->set( $path, 'sha1',
1777 [ 'hash' => $val['sha1'], 'latest' => false ] );
1778 }
1779 if ( isset( $val['xattr'] ) ) { // some backends store headers/metadata
1780 $val['xattr'] = self::normalizeXAttributes( $val['xattr'] );
1781 $this->cheapCache->set( $path, 'xattr',
1782 [ 'map' => $val['xattr'], 'latest' => false ] );
1783 }
1784 }
1785 }
1786 }
1787
1795 final protected static function normalizeXAttributes( array $xattr ) {
1796 $newXAttr = [ 'headers' => [], 'metadata' => [] ];
1797
1798 foreach ( $xattr['headers'] as $name => $value ) {
1799 $newXAttr['headers'][strtolower( $name )] = $value;
1800 }
1801
1802 foreach ( $xattr['metadata'] as $name => $value ) {
1803 $newXAttr['metadata'][strtolower( $name )] = $value;
1804 }
1805
1806 return $newXAttr;
1807 }
1808
1815 final protected function setConcurrencyFlags( array $opts ) {
1816 $opts['concurrency'] = 1; // off
1817 if ( $this->parallelize === 'implicit' ) {
1818 if ( !isset( $opts['parallelize'] ) || $opts['parallelize'] ) {
1819 $opts['concurrency'] = $this->concurrency;
1820 }
1821 } elseif ( $this->parallelize === 'explicit' ) {
1822 if ( !empty( $opts['parallelize'] ) ) {
1823 $opts['concurrency'] = $this->concurrency;
1824 }
1825 }
1826
1827 return $opts;
1828 }
1829
1838 protected function getContentType( $storagePath, $content, $fsPath ) {
1839 if ( $this->mimeCallback ) {
1840 return call_user_func_array( $this->mimeCallback, func_get_args() );
1841 }
1842
1843 $mime = ( $fsPath !== null ) ? mime_content_type( $fsPath ) : false;
1844 return $mime ?: 'unknown/unknown';
1845 }
1846}
1847
1858 public $params = []; // params to caller functions
1860 public $backend;
1863
1864 public $call; // string; name that identifies the function called
1865
1869 public function closeResources() {
1870 array_map( 'fclose', $this->resourcesToClose );
1871 }
1872}
1873
1880abstract class FileBackendStoreShardListIterator extends FilterIterator {
1882 protected $backend;
1883
1885 protected $params;
1886
1888 protected $container;
1889
1891 protected $directory;
1892
1894 protected $multiShardPaths = []; // (rel path => 1)
1895
1903 public function __construct(
1904 FileBackendStore $backend, $container, $dir, array $suffixes, array $params
1905 ) {
1906 $this->backend = $backend;
1907 $this->container = $container;
1908 $this->directory = $dir;
1909 $this->params = $params;
1910
1911 $iter = new AppendIterator();
1912 foreach ( $suffixes as $suffix ) {
1913 $iter->append( $this->listFromShard( $this->container . $suffix ) );
1914 }
1915
1916 parent::__construct( $iter );
1917 }
1918
1919 public function accept() {
1920 $rel = $this->getInnerIterator()->current(); // path relative to given directory
1921 $path = $this->params['dir'] . "/{$rel}"; // full storage path
1922 if ( $this->backend->isSingleShardPathInternal( $path ) ) {
1923 return true; // path is only on one shard; no issue with duplicates
1924 } elseif ( isset( $this->multiShardPaths[$rel] ) ) {
1925 // Don't keep listing paths that are on multiple shards
1926 return false;
1927 } else {
1928 $this->multiShardPaths[$rel] = 1;
1929
1930 return true;
1931 }
1932 }
1933
1934 public function rewind() {
1935 parent::rewind();
1936 $this->multiShardPaths = [];
1937 }
1938
1945 abstract protected function listFromShard( $container );
1946}
1947
1952 protected function listFromShard( $container ) {
1953 $list = $this->backend->getDirectoryListInternal(
1954 $container, $this->directory, $this->params );
1955 if ( $list === null ) {
1956 return new ArrayIterator( [] );
1957 } else {
1958 return is_array( $list ) ? new ArrayIterator( $list ) : $list;
1959 }
1960 }
1961}
1962
1967 protected function listFromShard( $container ) {
1968 $list = $this->backend->getFileListInternal(
1969 $container, $this->directory, $this->params );
1970 if ( $list === null ) {
1971 return new ArrayIterator( [] );
1972 } else {
1973 return is_array( $list ) ? new ArrayIterator( $list ) : $list;
1974 }
1975 }
1976}
$dir
Definition Autoload.php:8
interface is intended to be more or less compatible with the PHP memcached client.
Definition BagOStuff.php:47
A BagOStuff object with no objects in it.
static placeholderProps()
Placeholder file properties to use for files that don't exist.
Definition FSFile.php:143
File backend exception for checked exceptions (e.g.
FileBackendStore helper class for performing asynchronous file operations.
closeResources()
Close all open file handles.
Iterator for listing directories.
listFromShard( $container)
Get the list for a given container shard.
Iterator for listing regular files.
listFromShard( $container)
Get the list for a given container shard.
FileBackendStore helper function to handle listings that span container shards.
__construct(FileBackendStore $backend, $container, $dir, array $suffixes, array $params)
string $container
Full container name.
string $directory
Resolved relative path.
listFromShard( $container)
Get the list for a given container shard.
Base class for all backends using particular storage medium.
preloadFileStat(array $params)
Preload file stat information (concurrently if possible) into in-process cache.
containerCacheKey( $container)
Get the cache key for a container.
doGetLocalReferenceMulti(array $params)
resolveContainerName( $container)
Resolve a container name, checking if it's allowed by the backend.
static normalizeXAttributes(array $xattr)
Normalize file headers/metadata to the FileBackend::getFileXAttributes() format.
getLocalReferenceMulti(array $params)
Like getLocalReference() except it takes an array of storage paths and returns a map of storage paths...
setFileCache( $path, array $val)
Set the cached stat info for a file path.
doPublish(array $params)
moveInternal(array $params)
Move a file from one storage path to another in the backend.
setContainerCache( $container, array $val)
Set the cached info for a container.
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path.
doGetFileContentsMulti(array $params)
doStreamFile(array $params)
doSecure(array $params)
doClean(array $params)
getFileTimestamp(array $params)
Get the last-modified timestamp of the file at a storage path.
doGetLocalCopyMulti(array $params)
getScopedLocksForOps(array $ops, StatusValue $status)
Get an array of scoped locks needed for a batch of file operations.
doQuickOperationsInternal(array $ops)
doCleanInternal( $container, $dir, array $params)
callable $mimeCallback
Method to get the MIME type of files.
getPathsToLockForOpsInternal(array $performOps)
Get a list of storage paths to lock for a list of operations Returns an array with LockManager::LOCK_...
executeOpHandlesInternal(array $fileOpHandles)
Execute a list of FileBackendStoreOpHandle handles in parallel.
doPrepareInternal( $container, $dir, array $params)
concatenate(array $params)
Concatenate a list of storage files into a single file system file.
storeInternal(array $params)
Store a file into the backend from a file on disk.
getFileProps(array $params)
Get the properties of the file at a storage path in the backend.
getDirectoryList(array $params)
Get an iterator to list all directories under a storage directory.
resolveStoragePath( $storagePath)
Splits a storage path into an internal container name, an internal relative file name,...
copyInternal(array $params)
Copy a file from one storage path to another in the backend.
doStoreInternal(array $params)
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
doGetFileStat(array $params)
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
doPublishInternal( $container, $dir, array $params)
doGetFileStatMulti(array $params)
Get file stat information (concurrently if possible) for several files.
ProcessCacheLRU $expensiveCache
Map of paths to large (RAM/disk) cache items.
getFileSha1Base36(array $params)
Get a SHA-1 hash of the file at a storage path in the backend.
static isValidContainerName( $container)
Check if a full container name is valid.
getFileListInternal( $container, $dir, array $params)
Do not call this function from places outside FileBackend.
doCopyInternal(array $params)
getLocalCopyMulti(array $params)
Like getLocalCopy() except it takes an array of storage paths and returns a map of storage paths to T...
getFileContentsMulti(array $params)
Like getFileContents() except it takes an array of storage paths and returns a map of storage paths t...
doClearCache(array $paths=null)
Clears any additional stat caches for storage paths.
fullContainerName( $container)
Get the full container name, including the wiki ID prefix.
primeFileCache(array $items)
Do a batch lookup from cache for file stats for all paths used in a list of storage paths or FileOp o...
doOperationsInternal(array $ops, array $opts)
isSingleShardPathInternal( $storagePath)
Check if a storage path maps to a single shard.
directoryExists(array $params)
Check if a directory exists at a given storage path.
doDeleteInternal(array $params)
getContainerSuffixes( $container)
Get a list of full container shard suffixes for a container.
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
sanitizeOpHeaders(array $op)
Normalize and filter HTTP headers from a file operation.
doDescribeInternal(array $params)
doGetFileSha1Base36(array $params)
deleteInternal(array $params)
Delete a file at the storage path.
doPrimeContainerCache(array $containerInfo)
Fill the backend-specific process cache given an array of resolved container names and their correspo...
doMoveInternal(array $params)
maxFileSizeInternal()
Get the maximum allowable file size given backend medium restrictions and basic performance constrain...
describeInternal(array $params)
Alter metadata for a file at the storage path.
streamFile(array $params)
Stream the file at a storage path in the backend.
getOperationsInternal(array $ops)
Return a list of FileOp objects from a list of operations.
deleteFileCache( $path)
Delete the cached stat info for a file path.
setConcurrencyFlags(array $opts)
Set the 'concurrency' option from a list of operation options.
getFileXAttributes(array $params)
Get metadata about a file at a storage path in the backend.
getContentType( $storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
ProcessCacheLRU $cheapCache
Map of paths to small (RAM/disk) cache items.
preloadCache(array $paths)
Preload persistent file stat cache and property cache into in-process cache.
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
doDirectoryExists( $container, $dir, array $params)
WANObjectCache $memCache
doPrepare(array $params)
getFileHttpUrl(array $params)
__construct(array $config)
doGetFileXAttributes(array $params)
doExecuteOpHandlesInternal(array $fileOpHandles)
nullInternal(array $params)
No-op file operation that does nothing.
getFileSize(array $params)
Get the size (bytes) of a file at a storage path in the backend.
array $shardViaHashLevels
Map of container names to sharding config.
fileCacheKey( $path)
Get the cache key for a file path.
doConcatenate(array $params)
doSecureInternal( $container, $dir, array $params)
doCreateInternal(array $params)
fileExists(array $params)
Check if a file exists at a storage path in the backend.
getDirectoryListInternal( $container, $dir, array $params)
Do not call this function from places outside FileBackend.
static isValidShortContainerName( $container)
Check if a short container name is valid.
getContainerHashLevels( $container)
Get the sharding config for a container.
getContainerShard( $container, $relPath)
Get the container name shard suffix for a given path.
deleteContainerCache( $container)
Delete the cached info for a container.
createInternal(array $params)
Create a file in the backend with the given contents.
getFileList(array $params)
Get an iterator to list all stored files under a storage directory.
Base class for all file backend classes (including multi-write backends).
string $name
Unique backend name.
static splitStoragePath( $storagePath)
Split a storage path into a backend name, a container name, and a relative file path.
static normalizeContainerPath( $path)
Validate and normalize a relative storage path.
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
getTopDirectoryList(array $params)
Same as FileBackend::getDirectoryList() except only lists directories that are immediately under the ...
newStatus()
Yields the result of the status wrapper callback on either:
scopedProfileSection( $section)
static normalizeStoragePath( $storagePath)
Normalize a storage path by cleaning up directory separators.
int $concurrency
How many operations can be done in parallel.
getLocalReference(array $params)
Returns a file system file, identical to the file at a storage path.
static attempt(array $performOps, array $opts, FileJournal $journal)
Attempt to perform a series of file operations.
Functions related to the output of file content.
static send404Message( $fname, $flags=0)
Send out a standard 404 message for a file.
Handles per process caching of items.
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Multi-datacenter aware caching interface.
print
Definition cleanup.php:99
$res
Definition database.txt:21
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition deferred.txt:11
it s the revision text itself In either if gzip is the revision text is gzipped $flags
Definition hooks.txt:2805
Status::newGood()` to allow deletion, and then `return false` from the hook function. Ensure you consume the 'ChangeTagAfterDelete' hook to carry out custom deletion actions. $tag:name of the tag $user:user initiating the action & $status:Status object. See above. 'ChangeTagsListActive':Allows you to nominate which of the tags your extension uses are in active use. & $tags:list of all active tags. Append to this array. 'ChangeTagsAfterUpdateTags':Called after tags have been updated with the ChangeTags::updateTags function. Params:$addedTags:tags effectively added in the update $removedTags:tags effectively removed in the update $prevTags:tags that were present prior to the update $rc_id:recentchanges table id $rev_id:revision table id $log_id:logging table id $params:tag params $rc:RecentChange being tagged when the tagging accompanies the action or null $user:User who performed the tagging when the tagging is subsequent to the action or null 'ChangeTagsAllowedAdd':Called when checking if a user can add tags to a change. & $allowedTags:List of all the tags the user is allowed to add. Any tags the user wants to add( $addTags) that are not in this array will cause it to fail. You may add or remove tags to this array as required. $addTags:List of tags user intends to add. $user:User who is adding the tags. 'ChangeUserGroups':Called before user groups are changed. $performer:The User who will perform the change $user:The User whose groups will be changed & $add:The groups that will be added & $remove:The groups that will be removed 'Collation::factory':Called if $wgCategoryCollation is an unknown collation. $collationName:Name of the collation in question & $collationObject:Null. Replace with a subclass of the Collation class that implements the collation given in $collationName. 'ConfirmEmailComplete':Called after a user 's email has been confirmed successfully. $user:user(object) whose email is being confirmed 'ContentAlterParserOutput':Modify parser output for a given content object. Called by Content::getParserOutput after parsing has finished. Can be used for changes that depend on the result of the parsing but have to be done before LinksUpdate is called(such as adding tracking categories based on the rendered HTML). $content:The Content to render $title:Title of the page, as context $parserOutput:ParserOutput to manipulate 'ContentGetParserOutput':Customize parser output for a given content object, called by AbstractContent::getParserOutput. May be used to override the normal model-specific rendering of page content. $content:The Content to render $title:Title of the page, as context $revId:The revision ID, as context $options:ParserOptions for rendering. To avoid confusing the parser cache, the output can only depend on parameters provided to this hook function, not on global state. $generateHtml:boolean, indicating whether full HTML should be generated. If false, generation of HTML may be skipped, but other information should still be present in the ParserOutput object. & $output:ParserOutput, to manipulate or replace 'ContentHandlerDefaultModelFor':Called when the default content model is determined for a given title. May be used to assign a different model for that title. $title:the Title in question & $model:the model name. Use with CONTENT_MODEL_XXX constants. 'ContentHandlerForModelID':Called when a ContentHandler is requested for a given content model name, but no entry for that model exists in $wgContentHandlers. Note:if your extension implements additional models via this hook, please use GetContentModels hook to make them known to core. $modeName:the requested content model name & $handler:set this to a ContentHandler object, if desired. 'ContentModelCanBeUsedOn':Called to determine whether that content model can be used on a given page. This is especially useful to prevent some content models to be used in some special location. $contentModel:ID of the content model in question $title:the Title in question. & $ok:Output parameter, whether it is OK to use $contentModel on $title. Handler functions that modify $ok should generally return false to prevent further hooks from further modifying $ok. 'ContribsPager::getQueryInfo':Before the contributions query is about to run & $pager:Pager object for contributions & $queryInfo:The query for the contribs Pager 'ContribsPager::reallyDoQuery':Called before really executing the query for My Contributions & $data:an array of results of all contribs queries $pager:The ContribsPager object hooked into $offset:Index offset, inclusive $limit:Exact query limit $descending:Query direction, false for ascending, true for descending 'ContributionsLineEnding':Called before a contributions HTML line is finished $page:SpecialPage object for contributions & $ret:the HTML line $row:the DB row for this line & $classes:the classes to add to the surrounding< li > & $attribs:associative array of other HTML attributes for the< li > element. Currently only data attributes reserved to MediaWiki are allowed(see Sanitizer::isReservedDataAttribute). 'ContributionsToolLinks':Change tool links above Special:Contributions $id:User identifier $title:User page title & $tools:Array of tool links $specialPage:SpecialPage instance for context and services. Can be either SpecialContributions or DeletedContributionsPage. Extensions should type hint against a generic SpecialPage though. 'ConvertContent':Called by AbstractContent::convert when a conversion to another content model is requested. Handler functions that modify $result should generally return false to disable further attempts at conversion. $content:The Content object to be converted. $toModel:The ID of the content model to convert to. $lossy: boolean indicating whether lossy conversion is allowed. & $result:Output parameter, in case the handler function wants to provide a converted Content object. Note that $result->getContentModel() must return $toModel. 'CustomEditor':When invoking the page editor Return true to allow the normal editor to be used, or false if implementing a custom editor, e.g. for a special namespace, etc. $article:Article being edited $user:User performing the edit 'DatabaseOraclePostInit':Called after initialising an Oracle database $db:the DatabaseOracle object 'DeletedContribsPager::reallyDoQuery':Called before really executing the query for Special:DeletedContributions Similar to ContribsPager::reallyDoQuery & $data:an array of results of all contribs queries $pager:The DeletedContribsPager object hooked into $offset:Index offset, inclusive $limit:Exact query limit $descending:Query direction, false for ascending, true for descending 'DeletedContributionsLineEnding':Called before a DeletedContributions HTML line is finished. Similar to ContributionsLineEnding $page:SpecialPage object for DeletedContributions & $ret:the HTML line $row:the DB row for this line & $classes:the classes to add to the surrounding< li > & $attribs:associative array of other HTML attributes for the< li > element. Currently only data attributes reserved to MediaWiki are allowed(see Sanitizer::isReservedDataAttribute). 'DifferenceEngineAfterLoadNewText':called in DifferenceEngine::loadNewText() after the new revision 's content has been loaded into the class member variable $differenceEngine->mNewContent but before returning true from this function. $differenceEngine:DifferenceEngine object 'DifferenceEngineLoadTextAfterNewContentIsLoaded':called in DifferenceEngine::loadText() after the new revision 's content has been loaded into the class member variable $differenceEngine->mNewContent but before checking if the variable 's value is null. This hook can be used to inject content into said class member variable. $differenceEngine:DifferenceEngine object 'DifferenceEngineMarkPatrolledLink':Allows extensions to change the "mark as patrolled" link which is shown both on the diff header as well as on the bottom of a page, usually wrapped in a span element which has class="patrollink". $differenceEngine:DifferenceEngine object & $markAsPatrolledLink:The "mark as patrolled" link HTML(string) $rcid:Recent change ID(rc_id) for this change(int) 'DifferenceEngineMarkPatrolledRCID':Allows extensions to possibly change the rcid parameter. For example the rcid might be set to zero due to the user being the same as the performer of the change but an extension might still want to show it under certain conditions. & $rcid:rc_id(int) of the change or 0 $differenceEngine:DifferenceEngine object $change:RecentChange object $user:User object representing the current user 'DifferenceEngineNewHeader':Allows extensions to change the $newHeader variable, which contains information about the new revision, such as the revision 's author, whether the revision was marked as a minor edit or not, etc. $differenceEngine:DifferenceEngine object & $newHeader:The string containing the various #mw-diff-otitle[1-5] divs, which include things like revision author info, revision comment, RevisionDelete link and more $formattedRevisionTools:Array containing revision tools, some of which may have been injected with the DiffRevisionTools hook $nextlink:String containing the link to the next revision(if any) $status
Definition hooks.txt:1049
processing should stop and the error should be shown to the user * false
Definition hooks.txt:187
if( $ext=='php'|| $ext=='php5') $mime
Definition router.php:59
The most up to date schema for the tables in the database will always be tables sql in the maintenance directory
Definition schema.txt:2
$params