MediaWiki master
FSFileBackend.php
Go to the documentation of this file.
1<?php
44use Shellbox\Shellbox;
45use Wikimedia\AtEase\AtEase;
47use Wikimedia\Timestamp\ConvertibleTimestamp;
48
67 protected $usableDirCache;
68
70 protected $basePath;
71
73 protected $containerPaths;
74
76 protected $dirMode;
78 protected $fileMode;
80 protected $fileOwner;
81
83 protected $os;
85 protected $currentUser;
86
88 private $warningTrapStack = [];
89
100 public function __construct( array $config ) {
101 parent::__construct( $config );
102
103 if ( PHP_OS_FAMILY === 'Windows' ) {
104 $this->os = 'Windows';
105 } elseif ( PHP_OS_FAMILY === 'BSD' || PHP_OS_FAMILY === 'Darwin' ) {
106 $this->os = 'BSD';
107 } else {
108 $this->os = 'Linux';
109 }
110 // Remove any possible trailing slash from directories
111 if ( isset( $config['basePath'] ) ) {
112 $this->basePath = rtrim( $config['basePath'], '/' ); // remove trailing slash
113 } else {
114 $this->basePath = null; // none; containers must have explicit paths
115 }
116
117 $this->containerPaths = [];
118 foreach ( ( $config['containerPaths'] ?? [] ) as $container => $fsPath ) {
119 $this->containerPaths[$container] = rtrim( $fsPath, '/' ); // remove trailing slash
120 }
121
122 $this->fileMode = $config['fileMode'] ?? 0644;
123 $this->dirMode = $config['directoryMode'] ?? 0777;
124 if ( isset( $config['fileOwner'] ) && function_exists( 'posix_getuid' ) ) {
125 $this->fileOwner = $config['fileOwner'];
126 // Cache this, assuming it doesn't change
127 $this->currentUser = posix_getpwuid( posix_getuid() )['name'];
128 }
129
130 $this->usableDirCache = new MapCacheLRU( self::CACHE_CHEAP_SIZE );
131 }
132
133 public function getFeatures() {
134 return self::ATTR_UNICODE_PATHS;
135 }
136
137 protected function resolveContainerPath( $container, $relStoragePath ) {
138 // Check that container has a root directory
139 if ( isset( $this->containerPaths[$container] ) || isset( $this->basePath ) ) {
140 // Check for sensible relative paths (assume the base paths are OK)
141 if ( $this->isLegalRelPath( $relStoragePath ) ) {
142 return $relStoragePath;
143 }
144 }
145
146 return null; // invalid
147 }
148
155 protected function isLegalRelPath( $fsPath ) {
156 // Check for file names longer than 255 chars
157 if ( preg_match( '![^/]{256}!', $fsPath ) ) { // ext3/NTFS
158 return false;
159 }
160 if ( $this->os === 'Windows' ) { // NTFS
161 return !preg_match( '![:*?"<>|]!', $fsPath );
162 } else {
163 return true;
164 }
165 }
166
175 protected function containerFSRoot( $shortCont, $fullCont ) {
176 if ( isset( $this->containerPaths[$shortCont] ) ) {
177 return $this->containerPaths[$shortCont];
178 } elseif ( isset( $this->basePath ) ) {
179 return "{$this->basePath}/{$fullCont}";
180 }
181
182 return null; // no container base path defined
183 }
184
191 protected function resolveToFSPath( $storagePath ) {
192 [ $fullCont, $relPath ] = $this->resolveStoragePathReal( $storagePath );
193 if ( $relPath === null ) {
194 return null; // invalid
195 }
196 [ , $shortCont, ] = FileBackend::splitStoragePath( $storagePath );
197 $fsPath = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
198 if ( $relPath != '' ) {
199 $fsPath .= "/{$relPath}";
200 }
201
202 return $fsPath;
203 }
204
205 public function isPathUsableInternal( $storagePath ) {
206 $fsPath = $this->resolveToFSPath( $storagePath );
207 if ( $fsPath === null ) {
208 return false; // invalid
209 }
210
211 if ( $this->fileOwner !== null && $this->currentUser !== $this->fileOwner ) {
212 trigger_error( __METHOD__ . ": PHP process owner is not '{$this->fileOwner}'." );
213 return false;
214 }
215
216 $fsDirectory = dirname( $fsPath );
217 $usable = $this->usableDirCache->get( $fsDirectory, MapCacheLRU::TTL_PROC_SHORT );
218 if ( $usable === null ) {
219 AtEase::suppressWarnings();
220 $usable = is_dir( $fsDirectory ) && is_writable( $fsDirectory );
221 AtEase::restoreWarnings();
222 $this->usableDirCache->set( $fsDirectory, $usable ? 1 : 0 );
223 }
224
225 return $usable;
226 }
227
228 protected function doCreateInternal( array $params ) {
229 $status = $this->newStatus();
230
231 $fsDstPath = $this->resolveToFSPath( $params['dst'] );
232 if ( $fsDstPath === null ) {
233 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
234
235 return $status;
236 }
237
238 if ( !empty( $params['async'] ) ) { // deferred
239 $tempFile = $this->newTempFileWithContent( $params );
240 if ( !$tempFile ) {
241 $status->fatal( 'backend-fail-create', $params['dst'] );
242
243 return $status;
244 }
245 $cmd = $this->makeCopyCommand( $tempFile->getPath(), $fsDstPath, false );
246 $handler = function ( $errors, StatusValue $status, array $params, $cmd ) {
247 if ( $errors !== '' && !( $this->os === 'Windows' && $errors[0] === " " ) ) {
248 $status->fatal( 'backend-fail-create', $params['dst'] );
249 trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
250 }
251 };
252 $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd );
253 $tempFile->bind( $status->value );
254 } else { // immediate write
255 $created = false;
256 // Use fwrite+rename since (a) this clears xattrs, (b) threads still reading the old
257 // inode are unaffected since it writes to a new inode, and (c) new threads reading
258 // the file will either totally see the old version or totally see the new version
259 $fsStagePath = $this->makeStagingPath( $fsDstPath );
261 $stageHandle = fopen( $fsStagePath, 'xb' );
262 if ( $stageHandle ) {
263 $bytes = fwrite( $stageHandle, $params['content'] );
264 $created = ( $bytes === strlen( $params['content'] ) );
265 fclose( $stageHandle );
266 $created = $created ? rename( $fsStagePath, $fsDstPath ) : false;
267 }
268 $hadError = $this->untrapWarnings();
269 if ( $hadError || !$created ) {
270 $status->fatal( 'backend-fail-create', $params['dst'] );
271
272 return $status;
273 }
274 $this->chmod( $fsDstPath );
275 }
276
277 return $status;
278 }
279
280 protected function doStoreInternal( array $params ) {
281 $status = $this->newStatus();
282
283 $fsSrcPath = $params['src']; // file system path
284 $fsDstPath = $this->resolveToFSPath( $params['dst'] );
285 if ( $fsDstPath === null ) {
286 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
287
288 return $status;
289 }
290
291 if ( $fsSrcPath === $fsDstPath ) {
292 $status->fatal( 'backend-fail-internal', $this->name );
293
294 return $status;
295 }
296
297 if ( !empty( $params['async'] ) ) { // deferred
298 $cmd = $this->makeCopyCommand( $fsSrcPath, $fsDstPath, false );
299 $handler = function ( $errors, StatusValue $status, array $params, $cmd ) {
300 if ( $errors !== '' && !( $this->os === 'Windows' && $errors[0] === " " ) ) {
301 $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
302 trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
303 }
304 };
305 $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd );
306 } else { // immediate write
307 $stored = false;
308 // Use fwrite+rename since (a) this clears xattrs, (b) threads still reading the old
309 // inode are unaffected since it writes to a new inode, and (c) new threads reading
310 // the file will either totally see the old version or totally see the new version
311 $fsStagePath = $this->makeStagingPath( $fsDstPath );
313 $srcHandle = fopen( $fsSrcPath, 'rb' );
314 if ( $srcHandle ) {
315 $stageHandle = fopen( $fsStagePath, 'xb' );
316 if ( $stageHandle ) {
317 $bytes = stream_copy_to_stream( $srcHandle, $stageHandle );
318 $stored = ( $bytes !== false && $bytes === fstat( $srcHandle )['size'] );
319 fclose( $stageHandle );
320 $stored = $stored ? rename( $fsStagePath, $fsDstPath ) : false;
321 }
322 fclose( $srcHandle );
323 }
324 $hadError = $this->untrapWarnings();
325 if ( $hadError || !$stored ) {
326 $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
327
328 return $status;
329 }
330 $this->chmod( $fsDstPath );
331 }
332
333 return $status;
334 }
335
336 protected function doCopyInternal( array $params ) {
337 $status = $this->newStatus();
338
339 $fsSrcPath = $this->resolveToFSPath( $params['src'] );
340 if ( $fsSrcPath === null ) {
341 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
342
343 return $status;
344 }
345
346 $fsDstPath = $this->resolveToFSPath( $params['dst'] );
347 if ( $fsDstPath === null ) {
348 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
349
350 return $status;
351 }
352
353 if ( $fsSrcPath === $fsDstPath ) {
354 return $status; // no-op
355 }
356
357 $ignoreMissing = !empty( $params['ignoreMissingSource'] );
358
359 if ( !empty( $params['async'] ) ) { // deferred
360 $cmd = $this->makeCopyCommand( $fsSrcPath, $fsDstPath, $ignoreMissing );
361 $handler = function ( $errors, StatusValue $status, array $params, $cmd ) {
362 if ( $errors !== '' && !( $this->os === 'Windows' && $errors[0] === " " ) ) {
363 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
364 trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
365 }
366 };
367 $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd );
368 } else { // immediate write
369 $copied = false;
370 // Use fwrite+rename since (a) this clears xattrs, (b) threads still reading the old
371 // inode are unaffected since it writes to a new inode, and (c) new threads reading
372 // the file will either totally see the old version or totally see the new version
373 $fsStagePath = $this->makeStagingPath( $fsDstPath );
375 $srcHandle = fopen( $fsSrcPath, 'rb' );
376 if ( $srcHandle ) {
377 $stageHandle = fopen( $fsStagePath, 'xb' );
378 if ( $stageHandle ) {
379 $bytes = stream_copy_to_stream( $srcHandle, $stageHandle );
380 $copied = ( $bytes !== false && $bytes === fstat( $srcHandle )['size'] );
381 fclose( $stageHandle );
382 $copied = $copied ? rename( $fsStagePath, $fsDstPath ) : false;
383 }
384 fclose( $srcHandle );
385 }
386 $hadError = $this->untrapWarnings();
387 if ( $hadError || ( !$copied && !$ignoreMissing ) ) {
388 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
389
390 return $status;
391 }
392 if ( $copied ) {
393 $this->chmod( $fsDstPath );
394 }
395 }
396
397 return $status;
398 }
399
400 protected function doMoveInternal( array $params ) {
401 $status = $this->newStatus();
402
403 $fsSrcPath = $this->resolveToFSPath( $params['src'] );
404 if ( $fsSrcPath === null ) {
405 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
406
407 return $status;
408 }
409
410 $fsDstPath = $this->resolveToFSPath( $params['dst'] );
411 if ( $fsDstPath === null ) {
412 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
413
414 return $status;
415 }
416
417 if ( $fsSrcPath === $fsDstPath ) {
418 return $status; // no-op
419 }
420
421 $ignoreMissing = !empty( $params['ignoreMissingSource'] );
422
423 if ( !empty( $params['async'] ) ) { // deferred
424 $cmd = $this->makeMoveCommand( $fsSrcPath, $fsDstPath, $ignoreMissing );
425 $handler = function ( $errors, StatusValue $status, array $params, $cmd ) {
426 if ( $errors !== '' && !( $this->os === 'Windows' && $errors[0] === " " ) ) {
427 $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
428 trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
429 }
430 };
431 $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd );
432 } else { // immediate write
433 // Use rename() here since (a) this clears xattrs, (b) any threads still reading the
434 // old inode are unaffected since it writes to a new inode, and (c) this is fast and
435 // atomic within a file system volume (as is normally the case)
437 $moved = rename( $fsSrcPath, $fsDstPath );
438 $hadError = $this->untrapWarnings();
439 if ( $hadError || ( !$moved && !$ignoreMissing ) ) {
440 $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
441
442 return $status;
443 }
444 }
445
446 return $status;
447 }
448
449 protected function doDeleteInternal( array $params ) {
450 $status = $this->newStatus();
451
452 $fsSrcPath = $this->resolveToFSPath( $params['src'] );
453 if ( $fsSrcPath === null ) {
454 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
455
456 return $status;
457 }
458
459 $ignoreMissing = !empty( $params['ignoreMissingSource'] );
460
461 if ( !empty( $params['async'] ) ) { // deferred
462 $cmd = $this->makeUnlinkCommand( $fsSrcPath, $ignoreMissing );
463 $handler = function ( $errors, StatusValue $status, array $params, $cmd ) {
464 if ( $errors !== '' && !( $this->os === 'Windows' && $errors[0] === " " ) ) {
465 $status->fatal( 'backend-fail-delete', $params['src'] );
466 trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
467 }
468 };
469 $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd );
470 } else { // immediate write
472 $deleted = unlink( $fsSrcPath );
473 $hadError = $this->untrapWarnings();
474 if ( $hadError || ( !$deleted && !$ignoreMissing ) ) {
475 $status->fatal( 'backend-fail-delete', $params['src'] );
476
477 return $status;
478 }
479 }
480
481 return $status;
482 }
483
487 protected function doPrepareInternal( $fullCont, $dirRel, array $params ) {
488 $status = $this->newStatus();
489 [ , $shortCont, ] = FileBackend::splitStoragePath( $params['dir'] );
490 $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
491 $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
492 // Create the directory and its parents as needed...
493 $created = false;
494 AtEase::suppressWarnings();
495 $alreadyExisted = is_dir( $fsDirectory ); // already there?
496 if ( !$alreadyExisted ) {
497 $created = mkdir( $fsDirectory, $this->dirMode, true );
498 if ( !$created ) {
499 $alreadyExisted = is_dir( $fsDirectory ); // another thread made it?
500 }
501 }
502 $isWritable = $created ?: is_writable( $fsDirectory ); // assume writable if created here
503 AtEase::restoreWarnings();
504 if ( !$alreadyExisted && !$created ) {
505 $this->logger->error( __METHOD__ . ": cannot create directory $fsDirectory" );
506 $status->fatal( 'directorycreateerror', $params['dir'] ); // fails on races
507 } elseif ( !$isWritable ) {
508 $this->logger->error( __METHOD__ . ": directory $fsDirectory is read-only" );
509 $status->fatal( 'directoryreadonlyerror', $params['dir'] );
510 }
511 // Respect any 'noAccess' or 'noListing' flags...
512 if ( $created ) {
513 $status->merge( $this->doSecureInternal( $fullCont, $dirRel, $params ) );
514 }
515
516 if ( $status->isGood() ) {
517 $this->usableDirCache->set( $fsDirectory, 1 );
518 }
519
520 return $status;
521 }
522
523 protected function doSecureInternal( $fullCont, $dirRel, array $params ) {
524 $status = $this->newStatus();
525 [ , $shortCont, ] = FileBackend::splitStoragePath( $params['dir'] );
526 $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
527 $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
528 // Seed new directories with a blank index.html, to prevent crawling...
529 if ( !empty( $params['noListing'] ) && !is_file( "{$fsDirectory}/index.html" ) ) {
530 $this->trapWarnings();
531 $bytes = file_put_contents( "{$fsDirectory}/index.html", $this->indexHtmlPrivate() );
532 $this->untrapWarnings();
533 if ( $bytes === false ) {
534 $status->fatal( 'backend-fail-create', $params['dir'] . '/index.html' );
535 }
536 }
537 // Add a .htaccess file to the root of the container...
538 if ( !empty( $params['noAccess'] ) && !is_file( "{$contRoot}/.htaccess" ) ) {
539 AtEase::suppressWarnings();
540 $bytes = file_put_contents( "{$contRoot}/.htaccess", $this->htaccessPrivate() );
541 AtEase::restoreWarnings();
542 if ( $bytes === false ) {
543 $storeDir = "mwstore://{$this->name}/{$shortCont}";
544 $status->fatal( 'backend-fail-create', "{$storeDir}/.htaccess" );
545 }
546 }
547
548 return $status;
549 }
550
551 protected function doPublishInternal( $fullCont, $dirRel, array $params ) {
552 $status = $this->newStatus();
553 [ , $shortCont, ] = FileBackend::splitStoragePath( $params['dir'] );
554 $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
555 $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
556 // Unseed new directories with a blank index.html, to allow crawling...
557 if ( !empty( $params['listing'] ) && is_file( "{$fsDirectory}/index.html" ) ) {
558 $exists = ( file_get_contents( "{$fsDirectory}/index.html" ) === $this->indexHtmlPrivate() );
559 if ( $exists && !$this->unlink( "{$fsDirectory}/index.html" ) ) { // reverse secure()
560 $status->fatal( 'backend-fail-delete', $params['dir'] . '/index.html' );
561 }
562 }
563 // Remove the .htaccess file from the root of the container...
564 if ( !empty( $params['access'] ) && is_file( "{$contRoot}/.htaccess" ) ) {
565 $exists = ( file_get_contents( "{$contRoot}/.htaccess" ) === $this->htaccessPrivate() );
566 if ( $exists && !$this->unlink( "{$contRoot}/.htaccess" ) ) { // reverse secure()
567 $storeDir = "mwstore://{$this->name}/{$shortCont}";
568 $status->fatal( 'backend-fail-delete', "{$storeDir}/.htaccess" );
569 }
570 }
571
572 return $status;
573 }
574
575 protected function doCleanInternal( $fullCont, $dirRel, array $params ) {
576 $status = $this->newStatus();
577 [ , $shortCont, ] = FileBackend::splitStoragePath( $params['dir'] );
578 $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
579 $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
580
581 $this->rmdir( $fsDirectory );
582
583 return $status;
584 }
585
586 protected function doGetFileStat( array $params ) {
587 $fsSrcPath = $this->resolveToFSPath( $params['src'] );
588 if ( $fsSrcPath === null ) {
589 return self::RES_ERROR; // invalid storage path
590 }
591
592 $this->trapWarnings(); // don't trust 'false' if there were errors
593 $stat = is_file( $fsSrcPath ) ? stat( $fsSrcPath ) : false; // regular files only
594 $hadError = $this->untrapWarnings();
595
596 if ( is_array( $stat ) ) {
597 $ct = new ConvertibleTimestamp( $stat['mtime'] );
598
599 return [
600 'mtime' => $ct->getTimestamp( TS_MW ),
601 'size' => $stat['size']
602 ];
603 }
604
605 return $hadError ? self::RES_ERROR : self::RES_ABSENT;
606 }
607
608 protected function doClearCache( array $paths = null ) {
609 if ( is_array( $paths ) ) {
610 foreach ( $paths as $path ) {
611 $fsPath = $this->resolveToFSPath( $path );
612 if ( $fsPath !== null ) {
613 clearstatcache( true, $fsPath );
614 $this->usableDirCache->clear( $fsPath );
615 }
616 }
617 } else {
618 clearstatcache( true ); // clear the PHP file stat cache
619 $this->usableDirCache->clear();
620 }
621 }
622
623 protected function doDirectoryExists( $fullCont, $dirRel, array $params ) {
624 [ , $shortCont, ] = FileBackend::splitStoragePath( $params['dir'] );
625 $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
626 $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
627
628 $this->trapWarnings(); // don't trust 'false' if there were errors
629 $exists = is_dir( $fsDirectory );
630 $hadError = $this->untrapWarnings();
631
632 return $hadError ? self::RES_ERROR : $exists;
633 }
634
642 public function getDirectoryListInternal( $fullCont, $dirRel, array $params ) {
643 [ , $shortCont, ] = FileBackend::splitStoragePath( $params['dir'] );
644 $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
645 $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
646
647 $list = new FSFileBackendDirList( $fsDirectory, $params );
648 $error = $list->getLastError();
649 if ( $error !== null ) {
650 if ( $this->isFileNotFoundError( $error ) ) {
651 $this->logger->info( __METHOD__ . ": non-existant directory: '$fsDirectory'" );
652
653 return []; // nothing under this dir
654 } elseif ( is_dir( $fsDirectory ) ) {
655 $this->logger->warning( __METHOD__ . ": unreadable directory: '$fsDirectory'" );
656
657 return self::RES_ERROR; // bad permissions?
658 } else {
659 $this->logger->warning( __METHOD__ . ": unreachable directory: '$fsDirectory'" );
660
661 return self::RES_ERROR;
662 }
663 }
664
665 return $list;
666 }
667
675 public function getFileListInternal( $fullCont, $dirRel, array $params ) {
676 [ , $shortCont, ] = FileBackend::splitStoragePath( $params['dir'] );
677 $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
678 $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
679
680 $list = new FSFileBackendFileList( $fsDirectory, $params );
681 $error = $list->getLastError();
682 if ( $error !== null ) {
683 if ( $this->isFileNotFoundError( $error ) ) {
684 $this->logger->info( __METHOD__ . ": non-existent directory: '$fsDirectory'" );
685
686 return []; // nothing under this dir
687 } elseif ( is_dir( $fsDirectory ) ) {
688 $this->logger->warning( __METHOD__ .
689 ": unreadable directory: '$fsDirectory': $error" );
690
691 return self::RES_ERROR; // bad permissions?
692 } else {
693 $this->logger->warning( __METHOD__ .
694 ": unreachable directory: '$fsDirectory': $error" );
695
696 return self::RES_ERROR;
697 }
698 }
699
700 return $list;
701 }
702
703 protected function doGetLocalReferenceMulti( array $params ) {
704 $fsFiles = []; // (path => FSFile)
705
706 foreach ( $params['srcs'] as $src ) {
707 $source = $this->resolveToFSPath( $src );
708 if ( $source === null ) {
709 $fsFiles[$src] = self::RES_ERROR; // invalid path
710 continue;
711 }
712
713 $this->trapWarnings(); // don't trust 'false' if there were errors
714 $isFile = is_file( $source ); // regular files only
715 $hadError = $this->untrapWarnings();
716
717 if ( $isFile ) {
718 $fsFiles[$src] = new FSFile( $source );
719 } elseif ( $hadError ) {
720 $fsFiles[$src] = self::RES_ERROR;
721 } else {
722 $fsFiles[$src] = self::RES_ABSENT;
723 }
724 }
725
726 return $fsFiles;
727 }
728
729 protected function doGetLocalCopyMulti( array $params ) {
730 $tmpFiles = []; // (path => TempFSFile)
731
732 foreach ( $params['srcs'] as $src ) {
733 $source = $this->resolveToFSPath( $src );
734 if ( $source === null ) {
735 $tmpFiles[$src] = self::RES_ERROR; // invalid path
736 continue;
737 }
738 // Create a new temporary file with the same extension...
739 $ext = FileBackend::extensionFromPath( $src );
740 $tmpFile = $this->tmpFileFactory->newTempFSFile( 'localcopy_', $ext );
741 if ( !$tmpFile ) {
742 $tmpFiles[$src] = self::RES_ERROR;
743 continue;
744 }
745
746 $tmpPath = $tmpFile->getPath();
747 // Copy the source file over the temp file
748 $this->trapWarnings(); // don't trust 'false' if there were errors
749 $isFile = is_file( $source ); // regular files only
750 $copySuccess = $isFile ? copy( $source, $tmpPath ) : false;
751 $hadError = $this->untrapWarnings();
752
753 if ( $copySuccess ) {
754 $this->chmod( $tmpPath );
755 $tmpFiles[$src] = $tmpFile;
756 } elseif ( $hadError ) {
757 $tmpFiles[$src] = self::RES_ERROR; // copy failed
758 } else {
759 $tmpFiles[$src] = self::RES_ABSENT;
760 }
761 }
762
763 return $tmpFiles;
764 }
765
766 protected function directoriesAreVirtual() {
767 return false;
768 }
769
775 protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
776 $statuses = [];
777
778 $pipes = [];
779 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
780 $pipes[$index] = popen( $fileOpHandle->cmd, 'r' );
781 }
782
783 $errs = [];
784 foreach ( $pipes as $index => $pipe ) {
785 // Result will be empty on success in *NIX. On Windows,
786 // it may be something like " 1 file(s) [copied|moved].".
787 $errs[$index] = stream_get_contents( $pipe );
788 fclose( $pipe );
789 }
790
791 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
792 $status = $this->newStatus();
793 $function = $fileOpHandle->callback;
794 $function( $errs[$index], $status, $fileOpHandle->params, $fileOpHandle->cmd );
795 $statuses[$index] = $status;
796 }
797
798 return $statuses;
799 }
800
805 private function makeStagingPath( $fsPath ) {
806 $time = dechex( time() ); // make it easy to find old orphans
807 $hash = \Wikimedia\base_convert( md5( basename( $fsPath ) ), 16, 36, 25 );
808 $unique = \Wikimedia\base_convert( bin2hex( random_bytes( 16 ) ), 16, 36, 25 );
809
810 return dirname( $fsPath ) . "/.{$time}_{$hash}_{$unique}.tmpfsfile";
811 }
812
819 private function makeCopyCommand( $fsSrcPath, $fsDstPath, $ignoreMissing ) {
820 // Use copy+rename since (a) this clears xattrs, (b) threads still reading the old
821 // inode are unaffected since it writes to a new inode, and (c) new threads reading
822 // the file will either totally see the old version or totally see the new version
823 $fsStagePath = $this->makeStagingPath( $fsDstPath );
824 $encSrc = Shellbox::escape( $this->cleanPathSlashes( $fsSrcPath ) );
825 $encStage = Shellbox::escape( $this->cleanPathSlashes( $fsStagePath ) );
826 $encDst = Shellbox::escape( $this->cleanPathSlashes( $fsDstPath ) );
827 if ( $this->os === 'Windows' ) {
828 // https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/copy
829 // https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/move
830 $cmdWrite = "COPY /B /Y $encSrc $encStage 2>&1 && MOVE /Y $encStage $encDst 2>&1";
831 $cmd = $ignoreMissing ? "IF EXIST $encSrc $cmdWrite" : $cmdWrite;
832 } else {
833 // https://manpages.debian.org/buster/coreutils/cp.1.en.html
834 // https://manpages.debian.org/buster/coreutils/mv.1.en.html
835 $cmdWrite = "cp $encSrc $encStage 2>&1 && mv $encStage $encDst 2>&1";
836 $cmd = $ignoreMissing ? "test -f $encSrc && $cmdWrite" : $cmdWrite;
837 // Clean up permissions on any newly created destination file
838 $octalPermissions = '0' . decoct( $this->fileMode );
839 if ( strlen( $octalPermissions ) == 4 ) {
840 $cmd .= " && chmod $octalPermissions $encDst 2>/dev/null";
841 }
842 }
843
844 return $cmd;
845 }
846
853 private function makeMoveCommand( $fsSrcPath, $fsDstPath, $ignoreMissing = false ) {
854 // https://manpages.debian.org/buster/coreutils/mv.1.en.html
855 // https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/move
856 $encSrc = Shellbox::escape( $this->cleanPathSlashes( $fsSrcPath ) );
857 $encDst = Shellbox::escape( $this->cleanPathSlashes( $fsDstPath ) );
858 if ( $this->os === 'Windows' ) {
859 $writeCmd = "MOVE /Y $encSrc $encDst 2>&1";
860 $cmd = $ignoreMissing ? "IF EXIST $encSrc $writeCmd" : $writeCmd;
861 } else {
862 $writeCmd = "mv -f $encSrc $encDst 2>&1";
863 $cmd = $ignoreMissing ? "test -f $encSrc && $writeCmd" : $writeCmd;
864 }
865
866 return $cmd;
867 }
868
874 private function makeUnlinkCommand( $fsPath, $ignoreMissing = false ) {
875 // https://manpages.debian.org/buster/coreutils/rm.1.en.html
876 // https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/del
877 $encSrc = Shellbox::escape( $this->cleanPathSlashes( $fsPath ) );
878 if ( $this->os === 'Windows' ) {
879 $writeCmd = "DEL /Q $encSrc 2>&1";
880 $cmd = $ignoreMissing ? "IF EXIST $encSrc $writeCmd" : $writeCmd;
881 } else {
882 $cmd = $ignoreMissing ? "rm -f $encSrc 2>&1" : "rm $encSrc 2>&1";
883 }
884
885 return $cmd;
886 }
887
894 protected function chmod( $fsPath ) {
895 if ( $this->os === 'Windows' ) {
896 return true;
897 }
898
899 AtEase::suppressWarnings();
900 $ok = chmod( $fsPath, $this->fileMode );
901 AtEase::restoreWarnings();
902
903 return $ok;
904 }
905
912 protected function unlink( $fsPath ) {
913 AtEase::suppressWarnings();
914 $ok = unlink( $fsPath );
915 AtEase::restoreWarnings();
916 clearstatcache( true, $fsPath );
917
918 return $ok;
919 }
920
927 protected function rmdir( $fsDirectory ) {
928 AtEase::suppressWarnings();
929 $ok = rmdir( $fsDirectory ); // remove directory if empty
930 AtEase::restoreWarnings();
931 clearstatcache( true, $fsDirectory );
932
933 return $ok;
934 }
935
940 protected function newTempFileWithContent( array $params ) {
941 $tempFile = $this->tmpFileFactory->newTempFSFile( 'create_', 'tmp' );
942 if ( !$tempFile ) {
943 return null;
944 }
945
946 AtEase::suppressWarnings();
947 if ( file_put_contents( $tempFile->getPath(), $params['content'] ) === false ) {
948 $tempFile = null;
949 }
950 AtEase::restoreWarnings();
951
952 return $tempFile;
953 }
954
960 protected function indexHtmlPrivate() {
961 return '';
962 }
963
969 protected function htaccessPrivate() {
970 return "Require all denied\n";
971 }
972
979 protected function cleanPathSlashes( $fsPath ) {
980 return ( $this->os === 'Windows' ) ? strtr( $fsPath, '/', '\\' ) : $fsPath;
981 }
982
988 protected function trapWarnings( $regexIgnore = null ) {
989 $this->warningTrapStack[] = false;
990 set_error_handler( function ( $errno, $errstr ) use ( $regexIgnore ) {
991 if ( $regexIgnore === null || !preg_match( $regexIgnore, $errstr ) ) {
992 $this->logger->error( $errstr );
993 $this->warningTrapStack[count( $this->warningTrapStack ) - 1] = true;
994 }
995 return true; // suppress from PHP handler
996 }, E_WARNING );
997 }
998
1002 protected function trapWarningsIgnoringNotFound() {
1003 $this->trapWarnings( $this->getFileNotFoundRegex() );
1004 }
1005
1011 protected function untrapWarnings() {
1012 restore_error_handler();
1013
1014 return array_pop( $this->warningTrapStack );
1015 }
1016
1022 protected function getFileNotFoundRegex() {
1023 static $regex;
1024 if ( $regex === null ) {
1025 // "No such file or directory": string literal in spl_directory.c etc.
1026 $alternatives = [ ': No such file or directory' ];
1027 if ( $this->os === 'Windows' ) {
1028 // 2 = The system cannot find the file specified.
1029 // 3 = The system cannot find the path specified.
1030 $alternatives[] = ' \‍(code: [23]\‍)';
1031 }
1032 if ( function_exists( 'pcntl_strerror' ) ) {
1033 $alternatives[] = preg_quote( ': ' . pcntl_strerror( 2 ), '/' );
1034 } elseif ( function_exists( 'socket_strerror' ) && defined( 'SOCKET_ENOENT' ) ) {
1035 $alternatives[] = preg_quote( ': ' . socket_strerror( SOCKET_ENOENT ), '/' );
1036 }
1037 $regex = '/(' . implode( '|', $alternatives ) . ')$/';
1038 }
1039 return $regex;
1040 }
1041
1048 protected function isFileNotFoundError( $error ) {
1049 return (bool)preg_match( $this->getFileNotFoundRegex(), $error );
1050 }
1051}
array $params
The job parameters.
Class for a file system (FS) based file backend.
doCopyInternal(array $params)
doDirectoryExists( $fullCont, $dirRel, array $params)
newTempFileWithContent(array $params)
__construct(array $config)
getDirectoryListInternal( $fullCont, $dirRel, array $params)
doDeleteInternal(array $params)
string $os
Simpler version of PHP_OS_FAMILY.
indexHtmlPrivate()
Return the text of an index.html file to hide directory listings.
doExecuteOpHandlesInternal(array $fileOpHandles)
getFileNotFoundRegex()
Get a regex matching file not found errors.
resolveToFSPath( $storagePath)
Get the absolute file system path for a storage path.
isFileNotFoundError( $error)
Determine whether a given error message is a file not found error.
doGetFileStat(array $params)
rmdir( $fsDirectory)
Remove an empty directory, suppressing the warnings.
htaccessPrivate()
Return the text of a .htaccess file to make a directory private.
doGetLocalCopyMulti(array $params)
MapCacheLRU $usableDirCache
Cache for known prepared/usable directories.
untrapWarnings()
Stop listening for E_WARNING errors and get whether any happened.
string $fileOwner
Required OS username to own files.
doPublishInternal( $fullCont, $dirRel, array $params)
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
isLegalRelPath( $fsPath)
Check a relative file system path for validity.
doCleanInternal( $fullCont, $dirRel, array $params)
chmod( $fsPath)
Chmod a file, suppressing the warnings.
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
doSecureInternal( $fullCont, $dirRel, array $params)
doMoveInternal(array $params)
doClearCache(array $paths=null)
Clears any additional stat caches for storage paths.
trapWarnings( $regexIgnore=null)
Listen for E_WARNING errors and track whether any that happen.
doStoreInternal(array $params)
doPrepareInternal( $fullCont, $dirRel, array $params)
FileBackendStore::doPrepare() to override StatusValue Good status without value for success,...
getFeatures()
Get the a bitfield of extra features supported by the backend medium.
containerFSRoot( $shortCont, $fullCont)
Given the short (unresolved) and full (resolved) name of a container, return the file system path of ...
unlink( $fsPath)
Unlink a file, suppressing the warnings.
int $fileMode
File permission mode.
string null $basePath
Directory holding the container directories.
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path in the backend.
trapWarningsIgnoringNotFound()
Track E_WARNING errors but ignore any that correspond to ENOENT "No such file or directory".
cleanPathSlashes( $fsPath)
Clean up directory separators for the given OS.
doCreateInternal(array $params)
doGetLocalReferenceMulti(array $params)
string $currentUser
OS username running this script.
array< string, string > $containerPaths
Map of container names to root paths for custom container paths.
int $dirMode
Directory permission mode.
getFileListInternal( $fullCont, $dirRel, array $params)
Class representing a non-directory file on the file system.
Definition FSFile.php:32
Base class for all backends using particular storage medium.
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
Store key-value entries in a size-limited in-memory LRU cache.
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Base class for all file backend classes (including multi-write backends).
newStatus(... $args)
Yields the result of the status wrapper callback on either:
copy(array $params, array $opts=[])
Performs a single copy operation.
$source