47use Shellbox\Command\BoxedCommand;
50use Wikimedia\AtEase\AtEase;
56use Wikimedia\Timestamp\ConvertibleTimestamp;
97 private $warningTrapStack = [];
110 parent::__construct( $config );
112 if ( PHP_OS_FAMILY ===
'Windows' ) {
113 $this->os =
'Windows';
114 } elseif ( PHP_OS_FAMILY ===
'BSD' || PHP_OS_FAMILY ===
'Darwin' ) {
120 if ( isset( $config[
'basePath'] ) ) {
121 $this->basePath = rtrim( $config[
'basePath'],
'/' );
123 $this->basePath =
null;
126 $this->containerPaths = [];
127 foreach ( ( $config[
'containerPaths'] ?? [] ) as $container => $fsPath ) {
128 $this->containerPaths[$container] = rtrim( $fsPath,
'/' );
131 $this->fileMode = $config[
'fileMode'] ?? 0644;
132 $this->dirMode = $config[
'directoryMode'] ?? 0777;
133 if ( isset( $config[
'fileOwner'] ) && function_exists(
'posix_getuid' ) ) {
134 $this->fileOwner = $config[
'fileOwner'];
136 $this->currentUser = posix_getpwuid( posix_getuid() )[
'name'];
139 $this->usableDirCache =
new MapCacheLRU( self::CACHE_CHEAP_SIZE );
148 if ( isset( $this->containerPaths[$container] ) || $this->basePath !==
null ) {
151 return $relStoragePath;
166 if ( preg_match(
'![^/]{256}!', $fsPath ) ) {
169 if ( $this->os ===
'Windows' ) {
170 return !preg_match(
'![:*?"<>|]!', $fsPath );
185 if ( isset( $this->containerPaths[$shortCont] ) ) {
186 return $this->containerPaths[$shortCont];
187 } elseif ( $this->basePath !==
null ) {
188 return "{$this->basePath}/{$fullCont}";
202 if ( $relPath ===
null ) {
207 if ( $relPath !=
'' ) {
208 $fsPath .=
"/{$relPath}";
216 if ( $fsPath ===
null ) {
220 if ( $this->fileOwner !==
null && $this->currentUser !== $this->fileOwner ) {
221 trigger_error( __METHOD__ .
": PHP process owner is not '{$this->fileOwner}'." );
225 $fsDirectory = dirname( $fsPath );
226 $usable = $this->usableDirCache->get( $fsDirectory, MapCacheLRU::TTL_PROC_SHORT );
227 if ( $usable ===
null ) {
228 AtEase::suppressWarnings();
229 $usable = is_dir( $fsDirectory ) && is_writable( $fsDirectory );
230 AtEase::restoreWarnings();
231 $this->usableDirCache->set( $fsDirectory, $usable ? 1 : 0 );
241 if ( $fsDstPath ===
null ) {
242 $status->fatal(
'backend-fail-invalidpath',
$params[
'dst'] );
247 if ( !empty(
$params[
'async'] ) ) {
250 $status->fatal(
'backend-fail-create',
$params[
'dst'] );
254 $cmd = $this->makeCopyCommand( $tempFile->getPath(), $fsDstPath,
false );
256 if ( $errors !==
'' && !( $this->os ===
'Windows' && $errors[0] ===
" " ) ) {
257 $status->fatal(
'backend-fail-create',
$params[
'dst'] );
258 trigger_error(
"$cmd\n$errors", E_USER_WARNING );
262 $tempFile->bind( $status->value );
268 $fsStagePath = $this->makeStagingPath( $fsDstPath );
270 $stageHandle = fopen( $fsStagePath,
'xb' );
271 if ( $stageHandle ) {
272 $bytes = fwrite( $stageHandle,
$params[
'content'] );
273 $created = ( $bytes === strlen(
$params[
'content'] ) );
274 fclose( $stageHandle );
275 $created = $created ? rename( $fsStagePath, $fsDstPath ) :
false;
278 if ( $hadError || !$created ) {
279 $status->fatal(
'backend-fail-create',
$params[
'dst'] );
283 $this->
chmod( $fsDstPath );
294 if ( $fsDstPath ===
null ) {
295 $status->fatal(
'backend-fail-invalidpath',
$params[
'dst'] );
300 if ( $fsSrcPath === $fsDstPath ) {
301 $status->fatal(
'backend-fail-internal', $this->name );
306 if ( !empty(
$params[
'async'] ) ) {
307 $cmd = $this->makeCopyCommand( $fsSrcPath, $fsDstPath,
false );
309 if ( $errors !==
'' && !( $this->os ===
'Windows' && $errors[0] ===
" " ) ) {
310 $status->fatal(
'backend-fail-store',
$params[
'src'],
$params[
'dst'] );
311 trigger_error(
"$cmd\n$errors", E_USER_WARNING );
320 $fsStagePath = $this->makeStagingPath( $fsDstPath );
322 $srcHandle = fopen( $fsSrcPath,
'rb' );
324 $stageHandle = fopen( $fsStagePath,
'xb' );
325 if ( $stageHandle ) {
326 $bytes = stream_copy_to_stream( $srcHandle, $stageHandle );
327 $stored = ( $bytes !==
false && $bytes === fstat( $srcHandle )[
'size'] );
328 fclose( $stageHandle );
329 $stored = $stored ? rename( $fsStagePath, $fsDstPath ) :
false;
331 fclose( $srcHandle );
334 if ( $hadError || !$stored ) {
335 $status->fatal(
'backend-fail-store',
$params[
'src'],
$params[
'dst'] );
339 $this->
chmod( $fsDstPath );
349 if ( $fsSrcPath ===
null ) {
350 $status->fatal(
'backend-fail-invalidpath',
$params[
'src'] );
356 if ( $fsDstPath ===
null ) {
357 $status->fatal(
'backend-fail-invalidpath',
$params[
'dst'] );
362 if ( $fsSrcPath === $fsDstPath ) {
366 $ignoreMissing = !empty(
$params[
'ignoreMissingSource'] );
368 if ( !empty(
$params[
'async'] ) ) {
369 $cmd = $this->makeCopyCommand( $fsSrcPath, $fsDstPath, $ignoreMissing );
371 if ( $errors !==
'' && !( $this->os ===
'Windows' && $errors[0] ===
" " ) ) {
372 $status->fatal(
'backend-fail-copy',
$params[
'src'],
$params[
'dst'] );
373 trigger_error(
"$cmd\n$errors", E_USER_WARNING );
382 $fsStagePath = $this->makeStagingPath( $fsDstPath );
384 $srcHandle = fopen( $fsSrcPath,
'rb' );
386 $stageHandle = fopen( $fsStagePath,
'xb' );
387 if ( $stageHandle ) {
388 $bytes = stream_copy_to_stream( $srcHandle, $stageHandle );
389 $copied = ( $bytes !==
false && $bytes === fstat( $srcHandle )[
'size'] );
390 fclose( $stageHandle );
391 $copied = $copied ? rename( $fsStagePath, $fsDstPath ) :
false;
393 fclose( $srcHandle );
396 if ( $hadError || ( !$copied && !$ignoreMissing ) ) {
397 $status->fatal(
'backend-fail-copy',
$params[
'src'],
$params[
'dst'] );
402 $this->
chmod( $fsDstPath );
413 if ( $fsSrcPath ===
null ) {
414 $status->fatal(
'backend-fail-invalidpath',
$params[
'src'] );
420 if ( $fsDstPath ===
null ) {
421 $status->fatal(
'backend-fail-invalidpath',
$params[
'dst'] );
426 if ( $fsSrcPath === $fsDstPath ) {
430 $ignoreMissing = !empty(
$params[
'ignoreMissingSource'] );
432 if ( !empty(
$params[
'async'] ) ) {
433 $cmd = $this->makeMoveCommand( $fsSrcPath, $fsDstPath, $ignoreMissing );
435 if ( $errors !==
'' && !( $this->os ===
'Windows' && $errors[0] ===
" " ) ) {
436 $status->fatal(
'backend-fail-move',
$params[
'src'],
$params[
'dst'] );
437 trigger_error(
"$cmd\n$errors", E_USER_WARNING );
446 $moved = rename( $fsSrcPath, $fsDstPath );
448 if ( $hadError || ( !$moved && !$ignoreMissing ) ) {
449 $status->fatal(
'backend-fail-move',
$params[
'src'],
$params[
'dst'] );
462 if ( $fsSrcPath ===
null ) {
463 $status->fatal(
'backend-fail-invalidpath',
$params[
'src'] );
468 $ignoreMissing = !empty(
$params[
'ignoreMissingSource'] );
470 if ( !empty(
$params[
'async'] ) ) {
471 $cmd = $this->makeUnlinkCommand( $fsSrcPath, $ignoreMissing );
473 if ( $errors !==
'' && !( $this->os ===
'Windows' && $errors[0] ===
" " ) ) {
474 $status->fatal(
'backend-fail-delete',
$params[
'src'] );
475 trigger_error(
"$cmd\n$errors", E_USER_WARNING );
481 $deleted =
unlink( $fsSrcPath );
483 if ( $hadError || ( !$deleted && !$ignoreMissing ) ) {
484 $status->fatal(
'backend-fail-delete',
$params[
'src'] );
500 $fsDirectory = ( $dirRel !=
'' ) ?
"{$contRoot}/{$dirRel}" : $contRoot;
503 AtEase::suppressWarnings();
504 $alreadyExisted = is_dir( $fsDirectory );
505 if ( !$alreadyExisted ) {
506 $created = mkdir( $fsDirectory, $this->dirMode,
true );
508 $alreadyExisted = is_dir( $fsDirectory );
511 $isWritable = $created ?: is_writable( $fsDirectory );
512 AtEase::restoreWarnings();
513 if ( !$alreadyExisted && !$created ) {
514 $this->logger->error( __METHOD__ .
": cannot create directory $fsDirectory" );
515 $status->fatal(
'directorycreateerror',
$params[
'dir'] );
516 } elseif ( !$isWritable ) {
517 $this->logger->error( __METHOD__ .
": directory $fsDirectory is read-only" );
518 $status->fatal(
'directoryreadonlyerror',
$params[
'dir'] );
525 if ( $status->isGood() ) {
526 $this->usableDirCache->set( $fsDirectory, 1 );
536 $fsDirectory = ( $dirRel !=
'' ) ?
"{$contRoot}/{$dirRel}" : $contRoot;
538 if ( !empty(
$params[
'noListing'] ) && !is_file(
"{$fsDirectory}/index.html" ) ) {
540 $bytes = file_put_contents(
"{$fsDirectory}/index.html", $this->
indexHtmlPrivate() );
542 if ( $bytes ===
false ) {
543 $status->fatal(
'backend-fail-create',
$params[
'dir'] .
'/index.html' );
547 if ( !empty(
$params[
'noAccess'] ) && !is_file(
"{$contRoot}/.htaccess" ) ) {
548 AtEase::suppressWarnings();
549 $bytes = file_put_contents(
"{$contRoot}/.htaccess", $this->
htaccessPrivate() );
550 AtEase::restoreWarnings();
551 if ( $bytes ===
false ) {
552 $storeDir =
"mwstore://{$this->name}/{$shortCont}";
553 $status->fatal(
'backend-fail-create',
"{$storeDir}/.htaccess" );
564 $fsDirectory = ( $dirRel !=
'' ) ?
"{$contRoot}/{$dirRel}" : $contRoot;
566 if ( !empty(
$params[
'listing'] ) && is_file(
"{$fsDirectory}/index.html" ) ) {
567 $exists = ( file_get_contents(
"{$fsDirectory}/index.html" ) === $this->
indexHtmlPrivate() );
568 if ( $exists && !$this->
unlink(
"{$fsDirectory}/index.html" ) ) {
569 $status->fatal(
'backend-fail-delete',
$params[
'dir'] .
'/index.html' );
573 if ( !empty(
$params[
'access'] ) && is_file(
"{$contRoot}/.htaccess" ) ) {
574 $exists = ( file_get_contents(
"{$contRoot}/.htaccess" ) === $this->
htaccessPrivate() );
575 if ( $exists && !$this->
unlink(
"{$contRoot}/.htaccess" ) ) {
576 $storeDir =
"mwstore://{$this->name}/{$shortCont}";
577 $status->fatal(
'backend-fail-delete',
"{$storeDir}/.htaccess" );
588 $fsDirectory = ( $dirRel !=
'' ) ?
"{$contRoot}/{$dirRel}" : $contRoot;
590 $this->
rmdir( $fsDirectory );
597 if ( $fsSrcPath ===
null ) {
598 return self::RES_ERROR;
602 $stat = is_file( $fsSrcPath ) ? stat( $fsSrcPath ) :
false;
605 if ( is_array( $stat ) ) {
606 $ct =
new ConvertibleTimestamp( $stat[
'mtime'] );
609 'mtime' => $ct->getTimestamp( TS_MW ),
610 'size' => $stat[
'size']
614 return $hadError ? self::RES_ERROR : self::RES_ABSENT;
618 if ( is_array( $paths ) ) {
619 foreach ( $paths as
$path ) {
621 if ( $fsPath !==
null ) {
622 clearstatcache(
true, $fsPath );
623 $this->usableDirCache->clear( $fsPath );
627 clearstatcache(
true );
628 $this->usableDirCache->clear();
635 $fsDirectory = ( $dirRel !=
'' ) ?
"{$contRoot}/{$dirRel}" : $contRoot;
638 $exists = is_dir( $fsDirectory );
641 return $hadError ? self::RES_ERROR : $exists;
654 $fsDirectory = ( $dirRel !=
'' ) ?
"{$contRoot}/{$dirRel}" : $contRoot;
657 $error = $list->getLastError();
658 if ( $error !==
null ) {
660 $this->logger->info( __METHOD__ .
": non-existant directory: '$fsDirectory'" );
663 } elseif ( is_dir( $fsDirectory ) ) {
664 $this->logger->warning( __METHOD__ .
": unreadable directory: '$fsDirectory'" );
666 return self::RES_ERROR;
668 $this->logger->warning( __METHOD__ .
": unreachable directory: '$fsDirectory'" );
670 return self::RES_ERROR;
687 $fsDirectory = ( $dirRel !=
'' ) ?
"{$contRoot}/{$dirRel}" : $contRoot;
690 $error = $list->getLastError();
691 if ( $error !==
null ) {
693 $this->logger->info( __METHOD__ .
": non-existent directory: '$fsDirectory'" );
696 } elseif ( is_dir( $fsDirectory ) ) {
697 $this->logger->warning( __METHOD__ .
698 ": unreadable directory: '$fsDirectory': $error" );
700 return self::RES_ERROR;
702 $this->logger->warning( __METHOD__ .
703 ": unreachable directory: '$fsDirectory': $error" );
705 return self::RES_ERROR;
715 foreach (
$params[
'srcs'] as $src ) {
718 $fsFiles[$src] = self::RES_ERROR;
728 } elseif ( $hadError ) {
729 $fsFiles[$src] = self::RES_ERROR;
731 $fsFiles[$src] = self::RES_ABSENT;
741 foreach (
$params[
'srcs'] as $src ) {
744 $tmpFiles[$src] = self::RES_ERROR;
749 $tmpFile = $this->tmpFileFactory->newTempFSFile(
'localcopy_', $ext );
751 $tmpFiles[$src] = self::RES_ERROR;
755 $tmpPath = $tmpFile->getPath();
759 $copySuccess = $isFile ?
copy(
$source, $tmpPath ) :
false;
762 if ( $copySuccess ) {
763 $this->
chmod( $tmpPath );
764 $tmpFiles[$src] = $tmpFile;
765 } elseif ( $hadError ) {
766 $tmpFiles[$src] = self::RES_ERROR;
768 $tmpFiles[$src] = self::RES_ABSENT;
779 if (
$path ===
null ) {
780 return $this->
newStatus(
'backend-fail-invalidpath', $params[
'src'] );
782 $command->inputFileFromFile( $boxedName,
$path );
799 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
800 $pipes[$index] = popen( $fileOpHandle->cmd,
'r' );
804 foreach ( $pipes as $index => $pipe ) {
807 $errs[$index] = stream_get_contents( $pipe );
811 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
813 $function = $fileOpHandle->callback;
814 $function( $errs[$index], $status, $fileOpHandle->params, $fileOpHandle->cmd );
815 $statuses[$index] = $status;
825 private function makeStagingPath( $fsPath ) {
826 $time = dechex( time() );
827 $hash = \Wikimedia\base_convert( md5( basename( $fsPath ) ), 16, 36, 25 );
828 $unique = \Wikimedia\base_convert( bin2hex( random_bytes( 16 ) ), 16, 36, 25 );
830 return dirname( $fsPath ) .
"/.{$time}_{$hash}_{$unique}.tmpfsfile";
839 private function makeCopyCommand( $fsSrcPath, $fsDstPath, $ignoreMissing ) {
843 $fsStagePath = $this->makeStagingPath( $fsDstPath );
847 if ( $this->os ===
'Windows' ) {
850 $cmdWrite =
"COPY /B /Y $encSrc $encStage 2>&1 && MOVE /Y $encStage $encDst 2>&1";
851 $cmd = $ignoreMissing ?
"IF EXIST $encSrc $cmdWrite" : $cmdWrite;
855 $cmdWrite =
"cp $encSrc $encStage 2>&1 && mv $encStage $encDst 2>&1";
856 $cmd = $ignoreMissing ?
"test -f $encSrc && $cmdWrite" : $cmdWrite;
858 $octalPermissions =
'0' . decoct( $this->fileMode );
859 if ( strlen( $octalPermissions ) == 4 ) {
860 $cmd .=
" && chmod $octalPermissions $encDst 2>/dev/null";
873 private function makeMoveCommand( $fsSrcPath, $fsDstPath, $ignoreMissing =
false ) {
878 if ( $this->os ===
'Windows' ) {
879 $writeCmd =
"MOVE /Y $encSrc $encDst 2>&1";
880 $cmd = $ignoreMissing ?
"IF EXIST $encSrc $writeCmd" : $writeCmd;
882 $writeCmd =
"mv -f $encSrc $encDst 2>&1";
883 $cmd = $ignoreMissing ?
"test -f $encSrc && $writeCmd" : $writeCmd;
894 private function makeUnlinkCommand( $fsPath, $ignoreMissing =
false ) {
898 if ( $this->os ===
'Windows' ) {
899 $writeCmd =
"DEL /Q $encSrc 2>&1";
900 $cmd = $ignoreMissing ?
"IF EXIST $encSrc $writeCmd" : $writeCmd;
902 $cmd = $ignoreMissing ?
"rm -f $encSrc 2>&1" :
"rm $encSrc 2>&1";
914 protected function chmod( $fsPath ) {
915 if ( $this->os ===
'Windows' ) {
919 AtEase::suppressWarnings();
920 $ok =
chmod( $fsPath, $this->fileMode );
921 AtEase::restoreWarnings();
933 AtEase::suppressWarnings();
935 AtEase::restoreWarnings();
936 clearstatcache(
true, $fsPath );
947 protected function rmdir( $fsDirectory ) {
948 AtEase::suppressWarnings();
949 $ok =
rmdir( $fsDirectory );
950 AtEase::restoreWarnings();
951 clearstatcache(
true, $fsDirectory );
961 $tempFile = $this->tmpFileFactory->newTempFSFile(
'create_',
'tmp' );
966 AtEase::suppressWarnings();
967 if ( file_put_contents( $tempFile->getPath(),
$params[
'content'] ) ===
false ) {
970 AtEase::restoreWarnings();
990 return "Require all denied\n";
1000 return ( $this->os ===
'Windows' ) ? strtr( $fsPath,
'/',
'\\' ) : $fsPath;
1009 $this->warningTrapStack[] =
false;
1010 set_error_handler(
function ( $errno, $errstr ) use ( $regexIgnore ) {
1011 if ( $regexIgnore ===
null || !preg_match( $regexIgnore, $errstr ) ) {
1012 $this->logger->error( $errstr );
1013 $this->warningTrapStack[count( $this->warningTrapStack ) - 1] =
true;
1032 restore_error_handler();
1034 return array_pop( $this->warningTrapStack );
1044 if ( $regex ===
null ) {
1046 $alternatives = [
': No such file or directory' ];
1047 if ( $this->os ===
'Windows' ) {
1050 $alternatives[] =
' \(code: [23]\)';
1052 if ( function_exists(
'pcntl_strerror' ) ) {
1053 $alternatives[] = preg_quote(
': ' . pcntl_strerror( 2 ),
'/' );
1054 } elseif ( function_exists(
'socket_strerror' ) && defined(
'SOCKET_ENOENT' ) ) {
1055 $alternatives[] = preg_quote(
': ' . socket_strerror( SOCKET_ENOENT ),
'/' );
1057 $regex =
'/(' . implode(
'|', $alternatives ) .
')$/';
1074class_alias( FSFileBackend::class,
'FSFileBackend' );
array $params
The job parameters.
Store key-value entries in a size-limited in-memory LRU cache.
Generic operation result class Has warning/error list, boolean status and arbitrary value.