45use Wikimedia\AtEase\AtEase;
47use Wikimedia\Timestamp\ConvertibleTimestamp;
88 private $warningTrapStack = [];
101 parent::__construct( $config );
103 if ( PHP_OS_FAMILY ===
'Windows' ) {
104 $this->os =
'Windows';
105 } elseif ( PHP_OS_FAMILY ===
'BSD' || PHP_OS_FAMILY ===
'Darwin' ) {
111 if ( isset( $config[
'basePath'] ) ) {
112 $this->basePath = rtrim( $config[
'basePath'],
'/' );
114 $this->basePath =
null;
117 $this->containerPaths = [];
118 foreach ( ( $config[
'containerPaths'] ?? [] ) as $container => $fsPath ) {
119 $this->containerPaths[$container] = rtrim( $fsPath,
'/' );
122 $this->fileMode = $config[
'fileMode'] ?? 0644;
123 $this->dirMode = $config[
'directoryMode'] ?? 0777;
124 if ( isset( $config[
'fileOwner'] ) && function_exists(
'posix_getuid' ) ) {
125 $this->fileOwner = $config[
'fileOwner'];
127 $this->currentUser = posix_getpwuid( posix_getuid() )[
'name'];
130 $this->usableDirCache =
new MapCacheLRU( self::CACHE_CHEAP_SIZE );
134 return self::ATTR_UNICODE_PATHS;
139 if ( isset( $this->containerPaths[$container] ) || isset( $this->basePath ) ) {
142 return $relStoragePath;
157 if ( preg_match(
'![^/]{256}!', $fsPath ) ) {
160 if ( $this->os ===
'Windows' ) {
161 return !preg_match(
'![:*?"<>|]!', $fsPath );
176 if ( isset( $this->containerPaths[$shortCont] ) ) {
177 return $this->containerPaths[$shortCont];
178 } elseif ( isset( $this->basePath ) ) {
179 return "{$this->basePath}/{$fullCont}";
193 if ( $relPath ===
null ) {
196 [ , $shortCont, ] = FileBackend::splitStoragePath( $storagePath );
198 if ( $relPath !=
'' ) {
199 $fsPath .=
"/{$relPath}";
207 if ( $fsPath ===
null ) {
211 if ( $this->fileOwner !==
null && $this->currentUser !== $this->fileOwner ) {
212 trigger_error( __METHOD__ .
": PHP process owner is not '{$this->fileOwner}'." );
216 $fsDirectory = dirname( $fsPath );
217 $usable = $this->usableDirCache->get( $fsDirectory, MapCacheLRU::TTL_PROC_SHORT );
218 if ( $usable ===
null ) {
219 AtEase::suppressWarnings();
220 $usable = is_dir( $fsDirectory ) && is_writable( $fsDirectory );
221 AtEase::restoreWarnings();
222 $this->usableDirCache->set( $fsDirectory, $usable ? 1 : 0 );
232 if ( $fsDstPath ===
null ) {
233 $status->fatal(
'backend-fail-invalidpath',
$params[
'dst'] );
238 if ( !empty(
$params[
'async'] ) ) {
241 $status->fatal(
'backend-fail-create',
$params[
'dst'] );
245 $cmd = $this->makeCopyCommand( $tempFile->getPath(), $fsDstPath,
false );
247 if ( $errors !==
'' && !( $this->os ===
'Windows' && $errors[0] ===
" " ) ) {
248 $status->fatal(
'backend-fail-create',
$params[
'dst'] );
249 trigger_error(
"$cmd\n$errors", E_USER_WARNING );
253 $tempFile->bind( $status->value );
259 $fsStagePath = $this->makeStagingPath( $fsDstPath );
261 $stageHandle = fopen( $fsStagePath,
'xb' );
262 if ( $stageHandle ) {
263 $bytes = fwrite( $stageHandle,
$params[
'content'] );
264 $created = ( $bytes === strlen(
$params[
'content'] ) );
265 fclose( $stageHandle );
266 $created = $created ? rename( $fsStagePath, $fsDstPath ) :
false;
269 if ( $hadError || !$created ) {
270 $status->fatal(
'backend-fail-create',
$params[
'dst'] );
274 $this->
chmod( $fsDstPath );
285 if ( $fsDstPath ===
null ) {
286 $status->fatal(
'backend-fail-invalidpath',
$params[
'dst'] );
291 if ( $fsSrcPath === $fsDstPath ) {
292 $status->fatal(
'backend-fail-internal', $this->name );
297 if ( !empty(
$params[
'async'] ) ) {
298 $cmd = $this->makeCopyCommand( $fsSrcPath, $fsDstPath,
false );
300 if ( $errors !==
'' && !( $this->os ===
'Windows' && $errors[0] ===
" " ) ) {
301 $status->fatal(
'backend-fail-store',
$params[
'src'],
$params[
'dst'] );
302 trigger_error(
"$cmd\n$errors", E_USER_WARNING );
311 $fsStagePath = $this->makeStagingPath( $fsDstPath );
313 $srcHandle = fopen( $fsSrcPath,
'rb' );
315 $stageHandle = fopen( $fsStagePath,
'xb' );
316 if ( $stageHandle ) {
317 $bytes = stream_copy_to_stream( $srcHandle, $stageHandle );
318 $stored = ( $bytes !==
false && $bytes === fstat( $srcHandle )[
'size'] );
319 fclose( $stageHandle );
320 $stored = $stored ? rename( $fsStagePath, $fsDstPath ) :
false;
322 fclose( $srcHandle );
325 if ( $hadError || !$stored ) {
326 $status->fatal(
'backend-fail-store',
$params[
'src'],
$params[
'dst'] );
330 $this->
chmod( $fsDstPath );
340 if ( $fsSrcPath ===
null ) {
341 $status->fatal(
'backend-fail-invalidpath',
$params[
'src'] );
347 if ( $fsDstPath ===
null ) {
348 $status->fatal(
'backend-fail-invalidpath',
$params[
'dst'] );
353 if ( $fsSrcPath === $fsDstPath ) {
357 $ignoreMissing = !empty(
$params[
'ignoreMissingSource'] );
359 if ( !empty(
$params[
'async'] ) ) {
360 $cmd = $this->makeCopyCommand( $fsSrcPath, $fsDstPath, $ignoreMissing );
362 if ( $errors !==
'' && !( $this->os ===
'Windows' && $errors[0] ===
" " ) ) {
363 $status->fatal(
'backend-fail-copy',
$params[
'src'],
$params[
'dst'] );
364 trigger_error(
"$cmd\n$errors", E_USER_WARNING );
373 $fsStagePath = $this->makeStagingPath( $fsDstPath );
375 $srcHandle = fopen( $fsSrcPath,
'rb' );
377 $stageHandle = fopen( $fsStagePath,
'xb' );
378 if ( $stageHandle ) {
379 $bytes = stream_copy_to_stream( $srcHandle, $stageHandle );
380 $copied = ( $bytes !==
false && $bytes === fstat( $srcHandle )[
'size'] );
381 fclose( $stageHandle );
382 $copied = $copied ? rename( $fsStagePath, $fsDstPath ) :
false;
384 fclose( $srcHandle );
387 if ( $hadError || ( !$copied && !$ignoreMissing ) ) {
388 $status->fatal(
'backend-fail-copy',
$params[
'src'],
$params[
'dst'] );
393 $this->
chmod( $fsDstPath );
404 if ( $fsSrcPath ===
null ) {
405 $status->fatal(
'backend-fail-invalidpath',
$params[
'src'] );
411 if ( $fsDstPath ===
null ) {
412 $status->fatal(
'backend-fail-invalidpath',
$params[
'dst'] );
417 if ( $fsSrcPath === $fsDstPath ) {
421 $ignoreMissing = !empty(
$params[
'ignoreMissingSource'] );
423 if ( !empty(
$params[
'async'] ) ) {
424 $cmd = $this->makeMoveCommand( $fsSrcPath, $fsDstPath, $ignoreMissing );
426 if ( $errors !==
'' && !( $this->os ===
'Windows' && $errors[0] ===
" " ) ) {
427 $status->fatal(
'backend-fail-move',
$params[
'src'],
$params[
'dst'] );
428 trigger_error(
"$cmd\n$errors", E_USER_WARNING );
437 $moved = rename( $fsSrcPath, $fsDstPath );
439 if ( $hadError || ( !$moved && !$ignoreMissing ) ) {
440 $status->fatal(
'backend-fail-move',
$params[
'src'],
$params[
'dst'] );
453 if ( $fsSrcPath ===
null ) {
454 $status->fatal(
'backend-fail-invalidpath',
$params[
'src'] );
459 $ignoreMissing = !empty(
$params[
'ignoreMissingSource'] );
461 if ( !empty(
$params[
'async'] ) ) {
462 $cmd = $this->makeUnlinkCommand( $fsSrcPath, $ignoreMissing );
464 if ( $errors !==
'' && !( $this->os ===
'Windows' && $errors[0] ===
" " ) ) {
465 $status->fatal(
'backend-fail-delete',
$params[
'src'] );
466 trigger_error(
"$cmd\n$errors", E_USER_WARNING );
472 $deleted =
unlink( $fsSrcPath );
474 if ( $hadError || ( !$deleted && !$ignoreMissing ) ) {
475 $status->fatal(
'backend-fail-delete',
$params[
'src'] );
489 [ , $shortCont, ] = FileBackend::splitStoragePath(
$params[
'dir'] );
491 $fsDirectory = ( $dirRel !=
'' ) ?
"{$contRoot}/{$dirRel}" : $contRoot;
494 AtEase::suppressWarnings();
495 $alreadyExisted = is_dir( $fsDirectory );
496 if ( !$alreadyExisted ) {
497 $created = mkdir( $fsDirectory, $this->dirMode,
true );
499 $alreadyExisted = is_dir( $fsDirectory );
502 $isWritable = $created ?: is_writable( $fsDirectory );
503 AtEase::restoreWarnings();
504 if ( !$alreadyExisted && !$created ) {
505 $this->logger->error( __METHOD__ .
": cannot create directory $fsDirectory" );
506 $status->fatal(
'directorycreateerror',
$params[
'dir'] );
507 } elseif ( !$isWritable ) {
508 $this->logger->error( __METHOD__ .
": directory $fsDirectory is read-only" );
509 $status->fatal(
'directoryreadonlyerror',
$params[
'dir'] );
516 if ( $status->isGood() ) {
517 $this->usableDirCache->set( $fsDirectory, 1 );
525 [ , $shortCont, ] = FileBackend::splitStoragePath(
$params[
'dir'] );
527 $fsDirectory = ( $dirRel !=
'' ) ?
"{$contRoot}/{$dirRel}" : $contRoot;
529 if ( !empty(
$params[
'noListing'] ) && !is_file(
"{$fsDirectory}/index.html" ) ) {
531 $bytes = file_put_contents(
"{$fsDirectory}/index.html", $this->
indexHtmlPrivate() );
533 if ( $bytes ===
false ) {
534 $status->fatal(
'backend-fail-create',
$params[
'dir'] .
'/index.html' );
538 if ( !empty(
$params[
'noAccess'] ) && !is_file(
"{$contRoot}/.htaccess" ) ) {
539 AtEase::suppressWarnings();
540 $bytes = file_put_contents(
"{$contRoot}/.htaccess", $this->
htaccessPrivate() );
541 AtEase::restoreWarnings();
542 if ( $bytes ===
false ) {
543 $storeDir =
"mwstore://{$this->name}/{$shortCont}";
544 $status->fatal(
'backend-fail-create',
"{$storeDir}/.htaccess" );
553 [ , $shortCont, ] = FileBackend::splitStoragePath(
$params[
'dir'] );
555 $fsDirectory = ( $dirRel !=
'' ) ?
"{$contRoot}/{$dirRel}" : $contRoot;
557 if ( !empty(
$params[
'listing'] ) && is_file(
"{$fsDirectory}/index.html" ) ) {
558 $exists = ( file_get_contents(
"{$fsDirectory}/index.html" ) === $this->
indexHtmlPrivate() );
559 if ( $exists && !$this->
unlink(
"{$fsDirectory}/index.html" ) ) {
560 $status->fatal(
'backend-fail-delete',
$params[
'dir'] .
'/index.html' );
564 if ( !empty(
$params[
'access'] ) && is_file(
"{$contRoot}/.htaccess" ) ) {
565 $exists = ( file_get_contents(
"{$contRoot}/.htaccess" ) === $this->
htaccessPrivate() );
566 if ( $exists && !$this->
unlink(
"{$contRoot}/.htaccess" ) ) {
567 $storeDir =
"mwstore://{$this->name}/{$shortCont}";
568 $status->fatal(
'backend-fail-delete',
"{$storeDir}/.htaccess" );
577 [ , $shortCont, ] = FileBackend::splitStoragePath(
$params[
'dir'] );
579 $fsDirectory = ( $dirRel !=
'' ) ?
"{$contRoot}/{$dirRel}" : $contRoot;
581 $this->
rmdir( $fsDirectory );
588 if ( $fsSrcPath ===
null ) {
589 return self::RES_ERROR;
593 $stat = is_file( $fsSrcPath ) ? stat( $fsSrcPath ) :
false;
596 if ( is_array( $stat ) ) {
597 $ct =
new ConvertibleTimestamp( $stat[
'mtime'] );
600 'mtime' => $ct->getTimestamp( TS_MW ),
601 'size' => $stat[
'size']
605 return $hadError ? self::RES_ERROR : self::RES_ABSENT;
609 if ( is_array( $paths ) ) {
610 foreach ( $paths as
$path ) {
612 if ( $fsPath !==
null ) {
613 clearstatcache(
true, $fsPath );
614 $this->usableDirCache->clear( $fsPath );
618 clearstatcache(
true );
619 $this->usableDirCache->clear();
624 [ , $shortCont, ] = FileBackend::splitStoragePath(
$params[
'dir'] );
626 $fsDirectory = ( $dirRel !=
'' ) ?
"{$contRoot}/{$dirRel}" : $contRoot;
629 $exists = is_dir( $fsDirectory );
632 return $hadError ? self::RES_ERROR : $exists;
643 [ , $shortCont, ] = FileBackend::splitStoragePath(
$params[
'dir'] );
645 $fsDirectory = ( $dirRel !=
'' ) ?
"{$contRoot}/{$dirRel}" : $contRoot;
648 $error = $list->getLastError();
649 if ( $error !==
null ) {
651 $this->logger->info( __METHOD__ .
": non-existant directory: '$fsDirectory'" );
654 } elseif ( is_dir( $fsDirectory ) ) {
655 $this->logger->warning( __METHOD__ .
": unreadable directory: '$fsDirectory'" );
657 return self::RES_ERROR;
659 $this->logger->warning( __METHOD__ .
": unreachable directory: '$fsDirectory'" );
661 return self::RES_ERROR;
676 [ , $shortCont, ] = FileBackend::splitStoragePath(
$params[
'dir'] );
678 $fsDirectory = ( $dirRel !=
'' ) ?
"{$contRoot}/{$dirRel}" : $contRoot;
681 $error = $list->getLastError();
682 if ( $error !==
null ) {
684 $this->logger->info( __METHOD__ .
": non-existent directory: '$fsDirectory'" );
687 } elseif ( is_dir( $fsDirectory ) ) {
688 $this->logger->warning( __METHOD__ .
689 ": unreadable directory: '$fsDirectory': $error" );
691 return self::RES_ERROR;
693 $this->logger->warning( __METHOD__ .
694 ": unreachable directory: '$fsDirectory': $error" );
696 return self::RES_ERROR;
706 foreach (
$params[
'srcs'] as $src ) {
709 $fsFiles[$src] = self::RES_ERROR;
719 } elseif ( $hadError ) {
720 $fsFiles[$src] = self::RES_ERROR;
722 $fsFiles[$src] = self::RES_ABSENT;
732 foreach (
$params[
'srcs'] as $src ) {
735 $tmpFiles[$src] = self::RES_ERROR;
739 $ext = FileBackend::extensionFromPath( $src );
740 $tmpFile = $this->tmpFileFactory->newTempFSFile(
'localcopy_', $ext );
742 $tmpFiles[$src] = self::RES_ERROR;
746 $tmpPath = $tmpFile->getPath();
750 $copySuccess = $isFile ?
copy(
$source, $tmpPath ) :
false;
753 if ( $copySuccess ) {
754 $this->
chmod( $tmpPath );
755 $tmpFiles[$src] = $tmpFile;
756 } elseif ( $hadError ) {
757 $tmpFiles[$src] = self::RES_ERROR;
759 $tmpFiles[$src] = self::RES_ABSENT;
779 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
780 $pipes[$index] = popen( $fileOpHandle->cmd,
'r' );
784 foreach ( $pipes as $index => $pipe ) {
787 $errs[$index] = stream_get_contents( $pipe );
791 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
793 $function = $fileOpHandle->callback;
794 $function( $errs[$index], $status, $fileOpHandle->params, $fileOpHandle->cmd );
795 $statuses[$index] = $status;
805 private function makeStagingPath( $fsPath ) {
806 $time = dechex( time() );
807 $hash = \Wikimedia\base_convert( md5( basename( $fsPath ) ), 16, 36, 25 );
808 $unique = \Wikimedia\base_convert( bin2hex( random_bytes( 16 ) ), 16, 36, 25 );
810 return dirname( $fsPath ) .
"/.{$time}_{$hash}_{$unique}.tmpfsfile";
819 private function makeCopyCommand( $fsSrcPath, $fsDstPath, $ignoreMissing ) {
823 $fsStagePath = $this->makeStagingPath( $fsDstPath );
827 if ( $this->os ===
'Windows' ) {
830 $cmdWrite =
"COPY /B /Y $encSrc $encStage 2>&1 && MOVE /Y $encStage $encDst 2>&1";
831 $cmd = $ignoreMissing ?
"IF EXIST $encSrc $cmdWrite" : $cmdWrite;
835 $cmdWrite =
"cp $encSrc $encStage 2>&1 && mv $encStage $encDst 2>&1";
836 $cmd = $ignoreMissing ?
"test -f $encSrc && $cmdWrite" : $cmdWrite;
838 $octalPermissions =
'0' . decoct( $this->fileMode );
839 if ( strlen( $octalPermissions ) == 4 ) {
840 $cmd .=
" && chmod $octalPermissions $encDst 2>/dev/null";
853 private function makeMoveCommand( $fsSrcPath, $fsDstPath, $ignoreMissing =
false ) {
858 if ( $this->os ===
'Windows' ) {
859 $writeCmd =
"MOVE /Y $encSrc $encDst 2>&1";
860 $cmd = $ignoreMissing ?
"IF EXIST $encSrc $writeCmd" : $writeCmd;
862 $writeCmd =
"mv -f $encSrc $encDst 2>&1";
863 $cmd = $ignoreMissing ?
"test -f $encSrc && $writeCmd" : $writeCmd;
874 private function makeUnlinkCommand( $fsPath, $ignoreMissing =
false ) {
878 if ( $this->os ===
'Windows' ) {
879 $writeCmd =
"DEL /Q $encSrc 2>&1";
880 $cmd = $ignoreMissing ?
"IF EXIST $encSrc $writeCmd" : $writeCmd;
882 $cmd = $ignoreMissing ?
"rm -f $encSrc 2>&1" :
"rm $encSrc 2>&1";
894 protected function chmod( $fsPath ) {
895 if ( $this->os ===
'Windows' ) {
899 AtEase::suppressWarnings();
900 $ok =
chmod( $fsPath, $this->fileMode );
901 AtEase::restoreWarnings();
913 AtEase::suppressWarnings();
915 AtEase::restoreWarnings();
916 clearstatcache(
true, $fsPath );
927 protected function rmdir( $fsDirectory ) {
928 AtEase::suppressWarnings();
929 $ok =
rmdir( $fsDirectory );
930 AtEase::restoreWarnings();
931 clearstatcache(
true, $fsDirectory );
941 $tempFile = $this->tmpFileFactory->newTempFSFile(
'create_',
'tmp' );
946 AtEase::suppressWarnings();
947 if ( file_put_contents( $tempFile->getPath(),
$params[
'content'] ) ===
false ) {
950 AtEase::restoreWarnings();
970 return "Require all denied\n";
980 return ( $this->os ===
'Windows' ) ? strtr( $fsPath,
'/',
'\\' ) : $fsPath;
989 $this->warningTrapStack[] =
false;
990 set_error_handler(
function ( $errno, $errstr ) use ( $regexIgnore ) {
991 if ( $regexIgnore ===
null || !preg_match( $regexIgnore, $errstr ) ) {
992 $this->logger->error( $errstr );
993 $this->warningTrapStack[count( $this->warningTrapStack ) - 1] =
true;
1012 restore_error_handler();
1014 return array_pop( $this->warningTrapStack );
1024 if ( $regex ===
null ) {
1026 $alternatives = [
': No such file or directory' ];
1027 if ( $this->os ===
'Windows' ) {
1030 $alternatives[] =
' \(code: [23]\)';
1032 if ( function_exists(
'pcntl_strerror' ) ) {
1033 $alternatives[] = preg_quote(
': ' . pcntl_strerror( 2 ),
'/' );
1034 } elseif ( function_exists(
'socket_strerror' ) && defined(
'SOCKET_ENOENT' ) ) {
1035 $alternatives[] = preg_quote(
': ' . socket_strerror( SOCKET_ENOENT ),
'/' );
1037 $regex =
'/(' . implode(
'|', $alternatives ) .
')$/';
array $params
The job parameters.
Class for a file system (FS) based file backend.
doCopyInternal(array $params)
doDirectoryExists( $fullCont, $dirRel, array $params)
newTempFileWithContent(array $params)
__construct(array $config)
getDirectoryListInternal( $fullCont, $dirRel, array $params)
doDeleteInternal(array $params)
string $os
Simpler version of PHP_OS_FAMILY.
indexHtmlPrivate()
Return the text of an index.html file to hide directory listings.
doExecuteOpHandlesInternal(array $fileOpHandles)
getFileNotFoundRegex()
Get a regex matching file not found errors.
resolveToFSPath( $storagePath)
Get the absolute file system path for a storage path.
isFileNotFoundError( $error)
Determine whether a given error message is a file not found error.
doGetFileStat(array $params)
rmdir( $fsDirectory)
Remove an empty directory, suppressing the warnings.
htaccessPrivate()
Return the text of a .htaccess file to make a directory private.
doGetLocalCopyMulti(array $params)
MapCacheLRU $usableDirCache
Cache for known prepared/usable directories.
untrapWarnings()
Stop listening for E_WARNING errors and get whether any happened.
string $fileOwner
Required OS username to own files.
doPublishInternal( $fullCont, $dirRel, array $params)
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
isLegalRelPath( $fsPath)
Check a relative file system path for validity.
doCleanInternal( $fullCont, $dirRel, array $params)
chmod( $fsPath)
Chmod a file, suppressing the warnings.
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
doSecureInternal( $fullCont, $dirRel, array $params)
doMoveInternal(array $params)
doClearCache(array $paths=null)
Clears any additional stat caches for storage paths.
trapWarnings( $regexIgnore=null)
Listen for E_WARNING errors and track whether any that happen.
doStoreInternal(array $params)
doPrepareInternal( $fullCont, $dirRel, array $params)
FileBackendStore::doPrepare() to override StatusValue Good status without value for success,...
getFeatures()
Get the a bitfield of extra features supported by the backend medium.
containerFSRoot( $shortCont, $fullCont)
Given the short (unresolved) and full (resolved) name of a container, return the file system path of ...
unlink( $fsPath)
Unlink a file, suppressing the warnings.
int $fileMode
File permission mode.
string null $basePath
Directory holding the container directories.
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path in the backend.
trapWarningsIgnoringNotFound()
Track E_WARNING errors but ignore any that correspond to ENOENT "No such file or directory".
cleanPathSlashes( $fsPath)
Clean up directory separators for the given OS.
doCreateInternal(array $params)
doGetLocalReferenceMulti(array $params)
string $currentUser
OS username running this script.
array< string, string > $containerPaths
Map of container names to root paths for custom container paths.
int $dirMode
Directory permission mode.
getFileListInternal( $fullCont, $dirRel, array $params)
Class representing a non-directory file on the file system.
Base class for all backends using particular storage medium.
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
Store key-value entries in a size-limited in-memory LRU cache.
Generic operation result class Has warning/error list, boolean status and arbitrary value.