MediaWiki master
FSFileBackend.php
Go to the documentation of this file.
1<?php
44namespace Wikimedia\FileBackend;
45
46use MapCacheLRU;
47use Shellbox\Command\BoxedCommand;
48use Shellbox\Shellbox;
49use StatusValue;
50use Wikimedia\AtEase\AtEase;
56use Wikimedia\Timestamp\ConvertibleTimestamp;
57
76 protected $usableDirCache;
77
79 protected $basePath;
80
82 protected $containerPaths;
83
85 protected $dirMode;
87 protected $fileMode;
89 protected $fileOwner;
90
92 protected $os;
94 protected $currentUser;
95
97 private $warningTrapStack = [];
98
109 public function __construct( array $config ) {
110 parent::__construct( $config );
111
112 if ( PHP_OS_FAMILY === 'Windows' ) {
113 $this->os = 'Windows';
114 } elseif ( PHP_OS_FAMILY === 'BSD' || PHP_OS_FAMILY === 'Darwin' ) {
115 $this->os = 'BSD';
116 } else {
117 $this->os = 'Linux';
118 }
119 // Remove any possible trailing slash from directories
120 if ( isset( $config['basePath'] ) ) {
121 $this->basePath = rtrim( $config['basePath'], '/' ); // remove trailing slash
122 } else {
123 $this->basePath = null; // none; containers must have explicit paths
124 }
125
126 $this->containerPaths = [];
127 foreach ( ( $config['containerPaths'] ?? [] ) as $container => $fsPath ) {
128 $this->containerPaths[$container] = rtrim( $fsPath, '/' ); // remove trailing slash
129 }
130
131 $this->fileMode = $config['fileMode'] ?? 0644;
132 $this->dirMode = $config['directoryMode'] ?? 0777;
133 if ( isset( $config['fileOwner'] ) && function_exists( 'posix_getuid' ) ) {
134 $this->fileOwner = $config['fileOwner'];
135 // Cache this, assuming it doesn't change
136 $this->currentUser = posix_getpwuid( posix_getuid() )['name'];
137 }
138
139 $this->usableDirCache = new MapCacheLRU( self::CACHE_CHEAP_SIZE );
140 }
141
142 public function getFeatures() {
144 }
145
146 protected function resolveContainerPath( $container, $relStoragePath ) {
147 // Check that container has a root directory
148 if ( isset( $this->containerPaths[$container] ) || $this->basePath !== null ) {
149 // Check for sensible relative paths (assume the base paths are OK)
150 if ( $this->isLegalRelPath( $relStoragePath ) ) {
151 return $relStoragePath;
152 }
153 }
154
155 return null; // invalid
156 }
157
164 protected function isLegalRelPath( $fsPath ) {
165 // Check for file names longer than 255 chars
166 if ( preg_match( '![^/]{256}!', $fsPath ) ) { // ext3/NTFS
167 return false;
168 }
169 if ( $this->os === 'Windows' ) { // NTFS
170 return !preg_match( '![:*?"<>|]!', $fsPath );
171 } else {
172 return true;
173 }
174 }
175
184 protected function containerFSRoot( $shortCont, $fullCont ) {
185 if ( isset( $this->containerPaths[$shortCont] ) ) {
186 return $this->containerPaths[$shortCont];
187 } elseif ( $this->basePath !== null ) {
188 return "{$this->basePath}/{$fullCont}";
189 }
190
191 return null; // no container base path defined
192 }
193
200 protected function resolveToFSPath( $storagePath ) {
201 [ $fullCont, $relPath ] = $this->resolveStoragePathReal( $storagePath );
202 if ( $relPath === null ) {
203 return null; // invalid
204 }
205 [ , $shortCont, ] = FileBackend::splitStoragePath( $storagePath );
206 $fsPath = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
207 if ( $relPath != '' ) {
208 $fsPath .= "/{$relPath}";
209 }
210
211 return $fsPath;
212 }
213
214 public function isPathUsableInternal( $storagePath ) {
215 $fsPath = $this->resolveToFSPath( $storagePath );
216 if ( $fsPath === null ) {
217 return false; // invalid
218 }
219
220 if ( $this->fileOwner !== null && $this->currentUser !== $this->fileOwner ) {
221 trigger_error( __METHOD__ . ": PHP process owner is not '{$this->fileOwner}'." );
222 return false;
223 }
224
225 $fsDirectory = dirname( $fsPath );
226 $usable = $this->usableDirCache->get( $fsDirectory, MapCacheLRU::TTL_PROC_SHORT );
227 if ( $usable === null ) {
228 AtEase::suppressWarnings();
229 $usable = is_dir( $fsDirectory ) && is_writable( $fsDirectory );
230 AtEase::restoreWarnings();
231 $this->usableDirCache->set( $fsDirectory, $usable ? 1 : 0 );
232 }
233
234 return $usable;
235 }
236
237 protected function doCreateInternal( array $params ) {
238 $status = $this->newStatus();
239
240 $fsDstPath = $this->resolveToFSPath( $params['dst'] );
241 if ( $fsDstPath === null ) {
242 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
243
244 return $status;
245 }
246
247 if ( !empty( $params['async'] ) ) { // deferred
248 $tempFile = $this->newTempFileWithContent( $params );
249 if ( !$tempFile ) {
250 $status->fatal( 'backend-fail-create', $params['dst'] );
251
252 return $status;
253 }
254 $cmd = $this->makeCopyCommand( $tempFile->getPath(), $fsDstPath, false );
255 $handler = function ( $errors, StatusValue $status, array $params, $cmd ) {
256 if ( $errors !== '' && !( $this->os === 'Windows' && $errors[0] === " " ) ) {
257 $status->fatal( 'backend-fail-create', $params['dst'] );
258 trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
259 }
260 };
261 $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd );
262 $tempFile->bind( $status->value );
263 } else { // immediate write
264 $created = false;
265 // Use fwrite+rename since (a) this clears xattrs, (b) threads still reading the old
266 // inode are unaffected since it writes to a new inode, and (c) new threads reading
267 // the file will either totally see the old version or totally see the new version
268 $fsStagePath = $this->makeStagingPath( $fsDstPath );
270 $stageHandle = fopen( $fsStagePath, 'xb' );
271 if ( $stageHandle ) {
272 $bytes = fwrite( $stageHandle, $params['content'] );
273 $created = ( $bytes === strlen( $params['content'] ) );
274 fclose( $stageHandle );
275 $created = $created ? rename( $fsStagePath, $fsDstPath ) : false;
276 }
277 $hadError = $this->untrapWarnings();
278 if ( $hadError || !$created ) {
279 $status->fatal( 'backend-fail-create', $params['dst'] );
280
281 return $status;
282 }
283 $this->chmod( $fsDstPath );
284 }
285
286 return $status;
287 }
288
289 protected function doStoreInternal( array $params ) {
290 $status = $this->newStatus();
291
292 $fsSrcPath = $params['src']; // file system path
293 $fsDstPath = $this->resolveToFSPath( $params['dst'] );
294 if ( $fsDstPath === null ) {
295 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
296
297 return $status;
298 }
299
300 if ( $fsSrcPath === $fsDstPath ) {
301 $status->fatal( 'backend-fail-internal', $this->name );
302
303 return $status;
304 }
305
306 if ( !empty( $params['async'] ) ) { // deferred
307 $cmd = $this->makeCopyCommand( $fsSrcPath, $fsDstPath, false );
308 $handler = function ( $errors, StatusValue $status, array $params, $cmd ) {
309 if ( $errors !== '' && !( $this->os === 'Windows' && $errors[0] === " " ) ) {
310 $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
311 trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
312 }
313 };
314 $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd );
315 } else { // immediate write
316 $stored = false;
317 // Use fwrite+rename since (a) this clears xattrs, (b) threads still reading the old
318 // inode are unaffected since it writes to a new inode, and (c) new threads reading
319 // the file will either totally see the old version or totally see the new version
320 $fsStagePath = $this->makeStagingPath( $fsDstPath );
322 $srcHandle = fopen( $fsSrcPath, 'rb' );
323 if ( $srcHandle ) {
324 $stageHandle = fopen( $fsStagePath, 'xb' );
325 if ( $stageHandle ) {
326 $bytes = stream_copy_to_stream( $srcHandle, $stageHandle );
327 $stored = ( $bytes !== false && $bytes === fstat( $srcHandle )['size'] );
328 fclose( $stageHandle );
329 $stored = $stored ? rename( $fsStagePath, $fsDstPath ) : false;
330 }
331 fclose( $srcHandle );
332 }
333 $hadError = $this->untrapWarnings();
334 if ( $hadError || !$stored ) {
335 $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
336
337 return $status;
338 }
339 $this->chmod( $fsDstPath );
340 }
341
342 return $status;
343 }
344
345 protected function doCopyInternal( array $params ) {
346 $status = $this->newStatus();
347
348 $fsSrcPath = $this->resolveToFSPath( $params['src'] );
349 if ( $fsSrcPath === null ) {
350 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
351
352 return $status;
353 }
354
355 $fsDstPath = $this->resolveToFSPath( $params['dst'] );
356 if ( $fsDstPath === null ) {
357 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
358
359 return $status;
360 }
361
362 if ( $fsSrcPath === $fsDstPath ) {
363 return $status; // no-op
364 }
365
366 $ignoreMissing = !empty( $params['ignoreMissingSource'] );
367
368 if ( !empty( $params['async'] ) ) { // deferred
369 $cmd = $this->makeCopyCommand( $fsSrcPath, $fsDstPath, $ignoreMissing );
370 $handler = function ( $errors, StatusValue $status, array $params, $cmd ) {
371 if ( $errors !== '' && !( $this->os === 'Windows' && $errors[0] === " " ) ) {
372 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
373 trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
374 }
375 };
376 $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd );
377 } else { // immediate write
378 $copied = false;
379 // Use fwrite+rename since (a) this clears xattrs, (b) threads still reading the old
380 // inode are unaffected since it writes to a new inode, and (c) new threads reading
381 // the file will either totally see the old version or totally see the new version
382 $fsStagePath = $this->makeStagingPath( $fsDstPath );
384 $srcHandle = fopen( $fsSrcPath, 'rb' );
385 if ( $srcHandle ) {
386 $stageHandle = fopen( $fsStagePath, 'xb' );
387 if ( $stageHandle ) {
388 $bytes = stream_copy_to_stream( $srcHandle, $stageHandle );
389 $copied = ( $bytes !== false && $bytes === fstat( $srcHandle )['size'] );
390 fclose( $stageHandle );
391 $copied = $copied ? rename( $fsStagePath, $fsDstPath ) : false;
392 }
393 fclose( $srcHandle );
394 }
395 $hadError = $this->untrapWarnings();
396 if ( $hadError || ( !$copied && !$ignoreMissing ) ) {
397 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
398
399 return $status;
400 }
401 if ( $copied ) {
402 $this->chmod( $fsDstPath );
403 }
404 }
405
406 return $status;
407 }
408
409 protected function doMoveInternal( array $params ) {
410 $status = $this->newStatus();
411
412 $fsSrcPath = $this->resolveToFSPath( $params['src'] );
413 if ( $fsSrcPath === null ) {
414 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
415
416 return $status;
417 }
418
419 $fsDstPath = $this->resolveToFSPath( $params['dst'] );
420 if ( $fsDstPath === null ) {
421 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
422
423 return $status;
424 }
425
426 if ( $fsSrcPath === $fsDstPath ) {
427 return $status; // no-op
428 }
429
430 $ignoreMissing = !empty( $params['ignoreMissingSource'] );
431
432 if ( !empty( $params['async'] ) ) { // deferred
433 $cmd = $this->makeMoveCommand( $fsSrcPath, $fsDstPath, $ignoreMissing );
434 $handler = function ( $errors, StatusValue $status, array $params, $cmd ) {
435 if ( $errors !== '' && !( $this->os === 'Windows' && $errors[0] === " " ) ) {
436 $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
437 trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
438 }
439 };
440 $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd );
441 } else { // immediate write
442 // Use rename() here since (a) this clears xattrs, (b) any threads still reading the
443 // old inode are unaffected since it writes to a new inode, and (c) this is fast and
444 // atomic within a file system volume (as is normally the case)
446 $moved = rename( $fsSrcPath, $fsDstPath );
447 $hadError = $this->untrapWarnings();
448 if ( $hadError || ( !$moved && !$ignoreMissing ) ) {
449 $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
450
451 return $status;
452 }
453 }
454
455 return $status;
456 }
457
458 protected function doDeleteInternal( array $params ) {
459 $status = $this->newStatus();
460
461 $fsSrcPath = $this->resolveToFSPath( $params['src'] );
462 if ( $fsSrcPath === null ) {
463 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
464
465 return $status;
466 }
467
468 $ignoreMissing = !empty( $params['ignoreMissingSource'] );
469
470 if ( !empty( $params['async'] ) ) { // deferred
471 $cmd = $this->makeUnlinkCommand( $fsSrcPath, $ignoreMissing );
472 $handler = function ( $errors, StatusValue $status, array $params, $cmd ) {
473 if ( $errors !== '' && !( $this->os === 'Windows' && $errors[0] === " " ) ) {
474 $status->fatal( 'backend-fail-delete', $params['src'] );
475 trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
476 }
477 };
478 $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd );
479 } else { // immediate write
481 $deleted = unlink( $fsSrcPath );
482 $hadError = $this->untrapWarnings();
483 if ( $hadError || ( !$deleted && !$ignoreMissing ) ) {
484 $status->fatal( 'backend-fail-delete', $params['src'] );
485
486 return $status;
487 }
488 }
489
490 return $status;
491 }
492
496 protected function doPrepareInternal( $fullCont, $dirRel, array $params ) {
497 $status = $this->newStatus();
498 [ , $shortCont, ] = FileBackend::splitStoragePath( $params['dir'] );
499 $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
500 $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
501 // Create the directory and its parents as needed...
502 $created = false;
503 AtEase::suppressWarnings();
504 $alreadyExisted = is_dir( $fsDirectory ); // already there?
505 if ( !$alreadyExisted ) {
506 $created = mkdir( $fsDirectory, $this->dirMode, true );
507 if ( !$created ) {
508 $alreadyExisted = is_dir( $fsDirectory ); // another thread made it?
509 }
510 }
511 $isWritable = $created ?: is_writable( $fsDirectory ); // assume writable if created here
512 AtEase::restoreWarnings();
513 if ( !$alreadyExisted && !$created ) {
514 $this->logger->error( __METHOD__ . ": cannot create directory $fsDirectory" );
515 $status->fatal( 'directorycreateerror', $params['dir'] ); // fails on races
516 } elseif ( !$isWritable ) {
517 $this->logger->error( __METHOD__ . ": directory $fsDirectory is read-only" );
518 $status->fatal( 'directoryreadonlyerror', $params['dir'] );
519 }
520 // Respect any 'noAccess' or 'noListing' flags...
521 if ( $created ) {
522 $status->merge( $this->doSecureInternal( $fullCont, $dirRel, $params ) );
523 }
524
525 if ( $status->isGood() ) {
526 $this->usableDirCache->set( $fsDirectory, 1 );
527 }
528
529 return $status;
530 }
531
532 protected function doSecureInternal( $fullCont, $dirRel, array $params ) {
533 $status = $this->newStatus();
534 [ , $shortCont, ] = FileBackend::splitStoragePath( $params['dir'] );
535 $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
536 $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
537 // Seed new directories with a blank index.html, to prevent crawling...
538 if ( !empty( $params['noListing'] ) && !is_file( "{$fsDirectory}/index.html" ) ) {
539 $this->trapWarnings();
540 $bytes = file_put_contents( "{$fsDirectory}/index.html", $this->indexHtmlPrivate() );
541 $this->untrapWarnings();
542 if ( $bytes === false ) {
543 $status->fatal( 'backend-fail-create', $params['dir'] . '/index.html' );
544 }
545 }
546 // Add a .htaccess file to the root of the container...
547 if ( !empty( $params['noAccess'] ) && !is_file( "{$contRoot}/.htaccess" ) ) {
548 AtEase::suppressWarnings();
549 $bytes = file_put_contents( "{$contRoot}/.htaccess", $this->htaccessPrivate() );
550 AtEase::restoreWarnings();
551 if ( $bytes === false ) {
552 $storeDir = "mwstore://{$this->name}/{$shortCont}";
553 $status->fatal( 'backend-fail-create', "{$storeDir}/.htaccess" );
554 }
555 }
556
557 return $status;
558 }
559
560 protected function doPublishInternal( $fullCont, $dirRel, array $params ) {
561 $status = $this->newStatus();
562 [ , $shortCont, ] = FileBackend::splitStoragePath( $params['dir'] );
563 $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
564 $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
565 // Unseed new directories with a blank index.html, to allow crawling...
566 if ( !empty( $params['listing'] ) && is_file( "{$fsDirectory}/index.html" ) ) {
567 $exists = ( file_get_contents( "{$fsDirectory}/index.html" ) === $this->indexHtmlPrivate() );
568 if ( $exists && !$this->unlink( "{$fsDirectory}/index.html" ) ) { // reverse secure()
569 $status->fatal( 'backend-fail-delete', $params['dir'] . '/index.html' );
570 }
571 }
572 // Remove the .htaccess file from the root of the container...
573 if ( !empty( $params['access'] ) && is_file( "{$contRoot}/.htaccess" ) ) {
574 $exists = ( file_get_contents( "{$contRoot}/.htaccess" ) === $this->htaccessPrivate() );
575 if ( $exists && !$this->unlink( "{$contRoot}/.htaccess" ) ) { // reverse secure()
576 $storeDir = "mwstore://{$this->name}/{$shortCont}";
577 $status->fatal( 'backend-fail-delete', "{$storeDir}/.htaccess" );
578 }
579 }
580
581 return $status;
582 }
583
584 protected function doCleanInternal( $fullCont, $dirRel, array $params ) {
585 $status = $this->newStatus();
586 [ , $shortCont, ] = FileBackend::splitStoragePath( $params['dir'] );
587 $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
588 $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
589
590 $this->rmdir( $fsDirectory );
591
592 return $status;
593 }
594
595 protected function doGetFileStat( array $params ) {
596 $fsSrcPath = $this->resolveToFSPath( $params['src'] );
597 if ( $fsSrcPath === null ) {
598 return self::RES_ERROR; // invalid storage path
599 }
600
601 $this->trapWarnings(); // don't trust 'false' if there were errors
602 $stat = is_file( $fsSrcPath ) ? stat( $fsSrcPath ) : false; // regular files only
603 $hadError = $this->untrapWarnings();
604
605 if ( is_array( $stat ) ) {
606 $ct = new ConvertibleTimestamp( $stat['mtime'] );
607
608 return [
609 'mtime' => $ct->getTimestamp( TS_MW ),
610 'size' => $stat['size']
611 ];
612 }
613
614 return $hadError ? self::RES_ERROR : self::RES_ABSENT;
615 }
616
617 protected function doClearCache( ?array $paths = null ) {
618 if ( is_array( $paths ) ) {
619 foreach ( $paths as $path ) {
620 $fsPath = $this->resolveToFSPath( $path );
621 if ( $fsPath !== null ) {
622 clearstatcache( true, $fsPath );
623 $this->usableDirCache->clear( $fsPath );
624 }
625 }
626 } else {
627 clearstatcache( true ); // clear the PHP file stat cache
628 $this->usableDirCache->clear();
629 }
630 }
631
632 protected function doDirectoryExists( $fullCont, $dirRel, array $params ) {
633 [ , $shortCont, ] = FileBackend::splitStoragePath( $params['dir'] );
634 $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
635 $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
636
637 $this->trapWarnings(); // don't trust 'false' if there were errors
638 $exists = is_dir( $fsDirectory );
639 $hadError = $this->untrapWarnings();
640
641 return $hadError ? self::RES_ERROR : $exists;
642 }
643
651 public function getDirectoryListInternal( $fullCont, $dirRel, array $params ) {
652 [ , $shortCont, ] = FileBackend::splitStoragePath( $params['dir'] );
653 $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
654 $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
655
656 $list = new FSFileBackendDirList( $fsDirectory, $params );
657 $error = $list->getLastError();
658 if ( $error !== null ) {
659 if ( $this->isFileNotFoundError( $error ) ) {
660 $this->logger->info( __METHOD__ . ": non-existant directory: '$fsDirectory'" );
661
662 return []; // nothing under this dir
663 } elseif ( is_dir( $fsDirectory ) ) {
664 $this->logger->warning( __METHOD__ . ": unreadable directory: '$fsDirectory'" );
665
666 return self::RES_ERROR; // bad permissions?
667 } else {
668 $this->logger->warning( __METHOD__ . ": unreachable directory: '$fsDirectory'" );
669
670 return self::RES_ERROR;
671 }
672 }
673
674 return $list;
675 }
676
684 public function getFileListInternal( $fullCont, $dirRel, array $params ) {
685 [ , $shortCont, ] = FileBackend::splitStoragePath( $params['dir'] );
686 $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
687 $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
688
689 $list = new FSFileBackendFileList( $fsDirectory, $params );
690 $error = $list->getLastError();
691 if ( $error !== null ) {
692 if ( $this->isFileNotFoundError( $error ) ) {
693 $this->logger->info( __METHOD__ . ": non-existent directory: '$fsDirectory'" );
694
695 return []; // nothing under this dir
696 } elseif ( is_dir( $fsDirectory ) ) {
697 $this->logger->warning( __METHOD__ .
698 ": unreadable directory: '$fsDirectory': $error" );
699
700 return self::RES_ERROR; // bad permissions?
701 } else {
702 $this->logger->warning( __METHOD__ .
703 ": unreachable directory: '$fsDirectory': $error" );
704
705 return self::RES_ERROR;
706 }
707 }
708
709 return $list;
710 }
711
712 protected function doGetLocalReferenceMulti( array $params ) {
713 $fsFiles = []; // (path => FSFile)
714
715 foreach ( $params['srcs'] as $src ) {
716 $source = $this->resolveToFSPath( $src );
717 if ( $source === null ) {
718 $fsFiles[$src] = self::RES_ERROR; // invalid path
719 continue;
720 }
721
722 $this->trapWarnings(); // don't trust 'false' if there were errors
723 $isFile = is_file( $source ); // regular files only
724 $hadError = $this->untrapWarnings();
725
726 if ( $isFile ) {
727 $fsFiles[$src] = new FSFile( $source );
728 } elseif ( $hadError ) {
729 $fsFiles[$src] = self::RES_ERROR;
730 } else {
731 $fsFiles[$src] = self::RES_ABSENT;
732 }
733 }
734
735 return $fsFiles;
736 }
737
738 protected function doGetLocalCopyMulti( array $params ) {
739 $tmpFiles = []; // (path => TempFSFile)
740
741 foreach ( $params['srcs'] as $src ) {
742 $source = $this->resolveToFSPath( $src );
743 if ( $source === null ) {
744 $tmpFiles[$src] = self::RES_ERROR; // invalid path
745 continue;
746 }
747 // Create a new temporary file with the same extension...
748 $ext = FileBackend::extensionFromPath( $src );
749 $tmpFile = $this->tmpFileFactory->newTempFSFile( 'localcopy_', $ext );
750 if ( !$tmpFile ) {
751 $tmpFiles[$src] = self::RES_ERROR;
752 continue;
753 }
754
755 $tmpPath = $tmpFile->getPath();
756 // Copy the source file over the temp file
757 $this->trapWarnings(); // don't trust 'false' if there were errors
758 $isFile = is_file( $source ); // regular files only
759 $copySuccess = $isFile ? copy( $source, $tmpPath ) : false;
760 $hadError = $this->untrapWarnings();
761
762 if ( $copySuccess ) {
763 $this->chmod( $tmpPath );
764 $tmpFiles[$src] = $tmpFile;
765 } elseif ( $hadError ) {
766 $tmpFiles[$src] = self::RES_ERROR; // copy failed
767 } else {
768 $tmpFiles[$src] = self::RES_ABSENT;
769 }
770 }
771
772 return $tmpFiles;
773 }
774
775 public function addShellboxInputFile( BoxedCommand $command, string $boxedName,
776 array $params
777 ) {
778 $path = $this->resolveToFSPath( $params['src'] );
779 if ( $path === null ) {
780 return $this->newStatus( 'backend-fail-invalidpath', $params['src'] );
781 }
782 $command->inputFileFromFile( $boxedName, $path );
783 return $this->newStatus();
784 }
785
786 protected function directoriesAreVirtual() {
787 return false;
788 }
789
795 protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
796 $statuses = [];
797
798 $pipes = [];
799 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
800 $pipes[$index] = popen( $fileOpHandle->cmd, 'r' );
801 }
802
803 $errs = [];
804 foreach ( $pipes as $index => $pipe ) {
805 // Result will be empty on success in *NIX. On Windows,
806 // it may be something like " 1 file(s) [copied|moved].".
807 $errs[$index] = stream_get_contents( $pipe );
808 fclose( $pipe );
809 }
810
811 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
812 $status = $this->newStatus();
813 $function = $fileOpHandle->callback;
814 $function( $errs[$index], $status, $fileOpHandle->params, $fileOpHandle->cmd );
815 $statuses[$index] = $status;
816 }
817
818 return $statuses;
819 }
820
825 private function makeStagingPath( $fsPath ) {
826 $time = dechex( time() ); // make it easy to find old orphans
827 $hash = \Wikimedia\base_convert( md5( basename( $fsPath ) ), 16, 36, 25 );
828 $unique = \Wikimedia\base_convert( bin2hex( random_bytes( 16 ) ), 16, 36, 25 );
829
830 return dirname( $fsPath ) . "/.{$time}_{$hash}_{$unique}.tmpfsfile";
831 }
832
839 private function makeCopyCommand( $fsSrcPath, $fsDstPath, $ignoreMissing ) {
840 // Use copy+rename since (a) this clears xattrs, (b) threads still reading the old
841 // inode are unaffected since it writes to a new inode, and (c) new threads reading
842 // the file will either totally see the old version or totally see the new version
843 $fsStagePath = $this->makeStagingPath( $fsDstPath );
844 $encSrc = Shellbox::escape( $this->cleanPathSlashes( $fsSrcPath ) );
845 $encStage = Shellbox::escape( $this->cleanPathSlashes( $fsStagePath ) );
846 $encDst = Shellbox::escape( $this->cleanPathSlashes( $fsDstPath ) );
847 if ( $this->os === 'Windows' ) {
848 // https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/copy
849 // https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/move
850 $cmdWrite = "COPY /B /Y $encSrc $encStage 2>&1 && MOVE /Y $encStage $encDst 2>&1";
851 $cmd = $ignoreMissing ? "IF EXIST $encSrc $cmdWrite" : $cmdWrite;
852 } else {
853 // https://manpages.debian.org/buster/coreutils/cp.1.en.html
854 // https://manpages.debian.org/buster/coreutils/mv.1.en.html
855 $cmdWrite = "cp $encSrc $encStage 2>&1 && mv $encStage $encDst 2>&1";
856 $cmd = $ignoreMissing ? "test -f $encSrc && $cmdWrite" : $cmdWrite;
857 // Clean up permissions on any newly created destination file
858 $octalPermissions = '0' . decoct( $this->fileMode );
859 if ( strlen( $octalPermissions ) == 4 ) {
860 $cmd .= " && chmod $octalPermissions $encDst 2>/dev/null";
861 }
862 }
863
864 return $cmd;
865 }
866
873 private function makeMoveCommand( $fsSrcPath, $fsDstPath, $ignoreMissing = false ) {
874 // https://manpages.debian.org/buster/coreutils/mv.1.en.html
875 // https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/move
876 $encSrc = Shellbox::escape( $this->cleanPathSlashes( $fsSrcPath ) );
877 $encDst = Shellbox::escape( $this->cleanPathSlashes( $fsDstPath ) );
878 if ( $this->os === 'Windows' ) {
879 $writeCmd = "MOVE /Y $encSrc $encDst 2>&1";
880 $cmd = $ignoreMissing ? "IF EXIST $encSrc $writeCmd" : $writeCmd;
881 } else {
882 $writeCmd = "mv -f $encSrc $encDst 2>&1";
883 $cmd = $ignoreMissing ? "test -f $encSrc && $writeCmd" : $writeCmd;
884 }
885
886 return $cmd;
887 }
888
894 private function makeUnlinkCommand( $fsPath, $ignoreMissing = false ) {
895 // https://manpages.debian.org/buster/coreutils/rm.1.en.html
896 // https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/del
897 $encSrc = Shellbox::escape( $this->cleanPathSlashes( $fsPath ) );
898 if ( $this->os === 'Windows' ) {
899 $writeCmd = "DEL /Q $encSrc 2>&1";
900 $cmd = $ignoreMissing ? "IF EXIST $encSrc $writeCmd" : $writeCmd;
901 } else {
902 $cmd = $ignoreMissing ? "rm -f $encSrc 2>&1" : "rm $encSrc 2>&1";
903 }
904
905 return $cmd;
906 }
907
914 protected function chmod( $fsPath ) {
915 if ( $this->os === 'Windows' ) {
916 return true;
917 }
918
919 AtEase::suppressWarnings();
920 $ok = chmod( $fsPath, $this->fileMode );
921 AtEase::restoreWarnings();
922
923 return $ok;
924 }
925
932 protected function unlink( $fsPath ) {
933 AtEase::suppressWarnings();
934 $ok = unlink( $fsPath );
935 AtEase::restoreWarnings();
936 clearstatcache( true, $fsPath );
937
938 return $ok;
939 }
940
947 protected function rmdir( $fsDirectory ) {
948 AtEase::suppressWarnings();
949 $ok = rmdir( $fsDirectory ); // remove directory if empty
950 AtEase::restoreWarnings();
951 clearstatcache( true, $fsDirectory );
952
953 return $ok;
954 }
955
960 protected function newTempFileWithContent( array $params ) {
961 $tempFile = $this->tmpFileFactory->newTempFSFile( 'create_', 'tmp' );
962 if ( !$tempFile ) {
963 return null;
964 }
965
966 AtEase::suppressWarnings();
967 if ( file_put_contents( $tempFile->getPath(), $params['content'] ) === false ) {
968 $tempFile = null;
969 }
970 AtEase::restoreWarnings();
971
972 return $tempFile;
973 }
974
980 protected function indexHtmlPrivate() {
981 return '';
982 }
983
989 protected function htaccessPrivate() {
990 return "Require all denied\n";
991 }
992
999 protected function cleanPathSlashes( $fsPath ) {
1000 return ( $this->os === 'Windows' ) ? strtr( $fsPath, '/', '\\' ) : $fsPath;
1001 }
1002
1008 protected function trapWarnings( $regexIgnore = null ) {
1009 $this->warningTrapStack[] = false;
1010 set_error_handler( function ( $errno, $errstr ) use ( $regexIgnore ) {
1011 if ( $regexIgnore === null || !preg_match( $regexIgnore, $errstr ) ) {
1012 $this->logger->error( $errstr );
1013 $this->warningTrapStack[count( $this->warningTrapStack ) - 1] = true;
1014 }
1015 return true; // suppress from PHP handler
1016 }, E_WARNING );
1017 }
1018
1022 protected function trapWarningsIgnoringNotFound() {
1023 $this->trapWarnings( $this->getFileNotFoundRegex() );
1024 }
1025
1031 protected function untrapWarnings() {
1032 restore_error_handler();
1033
1034 return array_pop( $this->warningTrapStack );
1035 }
1036
1042 protected function getFileNotFoundRegex() {
1043 static $regex;
1044 if ( $regex === null ) {
1045 // "No such file or directory": string literal in spl_directory.c etc.
1046 $alternatives = [ ': No such file or directory' ];
1047 if ( $this->os === 'Windows' ) {
1048 // 2 = The system cannot find the file specified.
1049 // 3 = The system cannot find the path specified.
1050 $alternatives[] = ' \‍(code: [23]\‍)';
1051 }
1052 if ( function_exists( 'pcntl_strerror' ) ) {
1053 $alternatives[] = preg_quote( ': ' . pcntl_strerror( 2 ), '/' );
1054 } elseif ( function_exists( 'socket_strerror' ) && defined( 'SOCKET_ENOENT' ) ) {
1055 $alternatives[] = preg_quote( ': ' . socket_strerror( SOCKET_ENOENT ), '/' );
1056 }
1057 $regex = '/(' . implode( '|', $alternatives ) . ')$/';
1058 }
1059 return $regex;
1060 }
1061
1068 protected function isFileNotFoundError( $error ) {
1069 return (bool)preg_match( $this->getFileNotFoundRegex(), $error );
1070 }
1071}
1072
1074class_alias( FSFileBackend::class, 'FSFileBackend' );
array $params
The job parameters.
Store key-value entries in a size-limited in-memory LRU cache.
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Class for a file system (FS) based file backend.
doDirectoryExists( $fullCont, $dirRel, array $params)
addShellboxInputFile(BoxedCommand $command, string $boxedName, array $params)
Add a file to a Shellbox command as an input file.
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path in the backend.
doPrepareInternal( $fullCont, $dirRel, array $params)
FileBackendStore::doPrepare() to override StatusValue Good status without value for success,...
indexHtmlPrivate()
Return the text of an index.html file to hide directory listings.
doSecureInternal( $fullCont, $dirRel, array $params)
MapCacheLRU $usableDirCache
Cache for known prepared/usable directories.
string $fileOwner
Required OS username to own files.
htaccessPrivate()
Return the text of a .htaccess file to make a directory private.
getFeatures()
Get the a bitfield of extra features supported by the backend medium.
isLegalRelPath( $fsPath)
Check a relative file system path for validity.
resolveToFSPath( $storagePath)
Get the absolute file system path for a storage path.
string $currentUser
OS username running this script.
trapWarningsIgnoringNotFound()
Track E_WARNING errors but ignore any that correspond to ENOENT "No such file or directory".
unlink( $fsPath)
Unlink a file, suppressing the warnings.
getDirectoryListInternal( $fullCont, $dirRel, array $params)
int $dirMode
Directory permission mode.
doExecuteOpHandlesInternal(array $fileOpHandles)
containerFSRoot( $shortCont, $fullCont)
Given the short (unresolved) and full (resolved) name of a container, return the file system path of ...
untrapWarnings()
Stop listening for E_WARNING errors and get whether any happened.
array< string, string > $containerPaths
Map of container names to root paths for custom container paths.
isFileNotFoundError( $error)
Determine whether a given error message is a file not found error.
trapWarnings( $regexIgnore=null)
Listen for E_WARNING errors and track whether any that happen.
doPublishInternal( $fullCont, $dirRel, array $params)
string null $basePath
Directory holding the container directories.
cleanPathSlashes( $fsPath)
Clean up directory separators for the given OS.
doCleanInternal( $fullCont, $dirRel, array $params)
string $os
Simpler version of PHP_OS_FAMILY.
doClearCache(?array $paths=null)
Clears any additional stat caches for storage paths.
int $fileMode
File permission mode.
rmdir( $fsDirectory)
Remove an empty directory, suppressing the warnings.
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
chmod( $fsPath)
Chmod a file, suppressing the warnings.
getFileNotFoundRegex()
Get a regex matching file not found errors.
getFileListInternal( $fullCont, $dirRel, array $params)
Class representing a non-directory file on the file system.
Definition FSFile.php:34
This class is used to hold the location and do limited manipulation of files stored temporarily (this...
Base class for all backends using particular storage medium.
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
static extensionFromPath( $path, $case='lowercase')
Get the final extension from a storage or FS path.
static splitStoragePath( $storagePath)
Split a storage path into a backend name, a container name, and a relative file path.
newStatus( $message=null,... $params)
Yields the result of the status wrapper callback on either:
copy(array $params, array $opts=[])
Performs a single copy operation.
$source