MediaWiki  master
FSFileBackend.php
Go to the documentation of this file.
1 <?php
44 use Wikimedia\AtEase\AtEase;
45 use Wikimedia\Timestamp\ConvertibleTimestamp;
46 
65  protected $usableDirCache;
66 
68  protected $basePath;
69 
71  protected $containerPaths;
72 
74  protected $dirMode;
76  protected $fileMode;
78  protected $fileOwner;
79 
81  protected $os;
83  protected $currentUser;
84 
86  private $warningTrapStack = [];
87 
98  public function __construct( array $config ) {
99  parent::__construct( $config );
100 
101  if ( substr( PHP_OS, 0, 3 ) === 'WIN' || PHP_OS === 'Windows' ) {
102  $this->os = 'Windows';
103  } elseif ( substr( PHP_OS, 0, -3 ) === 'BSD' || PHP_OS === 'Darwin' ) {
104  $this->os = 'BSD';
105  } else {
106  $this->os = 'Linux';
107  }
108  // Remove any possible trailing slash from directories
109  if ( isset( $config['basePath'] ) ) {
110  $this->basePath = rtrim( $config['basePath'], '/' ); // remove trailing slash
111  } else {
112  $this->basePath = null; // none; containers must have explicit paths
113  }
114 
115  $this->containerPaths = [];
116  foreach ( ( $config['containerPaths'] ?? [] ) as $container => $fsPath ) {
117  $this->containerPaths[$container] = rtrim( $fsPath, '/' ); // remove trailing slash
118  }
119 
120  $this->fileMode = $config['fileMode'] ?? 0644;
121  $this->dirMode = $config['directoryMode'] ?? 0777;
122  if ( isset( $config['fileOwner'] ) && function_exists( 'posix_getuid' ) ) {
123  $this->fileOwner = $config['fileOwner'];
124  // Cache this, assuming it doesn't change
125  $this->currentUser = posix_getpwuid( posix_getuid() )['name'];
126  }
127 
128  $this->usableDirCache = new MapCacheLRU( self::CACHE_CHEAP_SIZE );
129  }
130 
131  public function getFeatures() {
133  }
134 
135  protected function resolveContainerPath( $container, $relStoragePath ) {
136  // Check that container has a root directory
137  if ( isset( $this->containerPaths[$container] ) || isset( $this->basePath ) ) {
138  // Check for sane relative paths (assume the base paths are OK)
139  if ( $this->isLegalRelPath( $relStoragePath ) ) {
140  return $relStoragePath;
141  }
142  }
143 
144  return null; // invalid
145  }
146 
153  protected function isLegalRelPath( $fsPath ) {
154  // Check for file names longer than 255 chars
155  if ( preg_match( '![^/]{256}!', $fsPath ) ) { // ext3/NTFS
156  return false;
157  }
158  if ( $this->os === 'Windows' ) { // NTFS
159  return !preg_match( '![:*?"<>|]!', $fsPath );
160  } else {
161  return true;
162  }
163  }
164 
173  protected function containerFSRoot( $shortCont, $fullCont ) {
174  if ( isset( $this->containerPaths[$shortCont] ) ) {
175  return $this->containerPaths[$shortCont];
176  } elseif ( isset( $this->basePath ) ) {
177  return "{$this->basePath}/{$fullCont}";
178  }
179 
180  return null; // no container base path defined
181  }
182 
189  protected function resolveToFSPath( $storagePath ) {
190  list( $fullCont, $relPath ) = $this->resolveStoragePathReal( $storagePath );
191  if ( $relPath === null ) {
192  return null; // invalid
193  }
194  list( , $shortCont, ) = FileBackend::splitStoragePath( $storagePath );
195  $fsPath = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
196  if ( $relPath != '' ) {
197  $fsPath .= "/{$relPath}";
198  }
199 
200  return $fsPath;
201  }
202 
203  public function isPathUsableInternal( $storagePath ) {
204  $fsPath = $this->resolveToFSPath( $storagePath );
205  if ( $fsPath === null ) {
206  return false; // invalid
207  }
208 
209  if ( $this->fileOwner !== null && $this->currentUser !== $this->fileOwner ) {
210  trigger_error( __METHOD__ . ": PHP process owner is not '{$this->fileOwner}'." );
211  return false;
212  }
213 
214  $fsDirectory = dirname( $fsPath );
215  $usable = $this->usableDirCache->get( $fsDirectory, MapCacheLRU::TTL_PROC_SHORT );
216  if ( $usable === null ) {
217  AtEase::suppressWarnings();
218  $usable = is_dir( $fsDirectory ) && is_writable( $fsDirectory );
219  AtEase::restoreWarnings();
220  $this->usableDirCache->set( $fsDirectory, $usable ? 1 : 0 );
221  }
222 
223  return $usable;
224  }
225 
226  protected function doCreateInternal( array $params ) {
227  $status = $this->newStatus();
228 
229  $fsDstPath = $this->resolveToFSPath( $params['dst'] );
230  if ( $fsDstPath === null ) {
231  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
232 
233  return $status;
234  }
235 
236  if ( !empty( $params['async'] ) ) { // deferred
237  $tempFile = $this->newTempFileWithContent( $params );
238  if ( !$tempFile ) {
239  $status->fatal( 'backend-fail-create', $params['dst'] );
240 
241  return $status;
242  }
243  $cmd = $this->makeCopyCommand( $tempFile->getPath(), $fsDstPath, false );
244  $handler = function ( $errors, StatusValue $status, array $params, $cmd ) {
245  if ( $errors !== '' && !( $this->os === 'Windows' && $errors[0] === " " ) ) {
246  $status->fatal( 'backend-fail-create', $params['dst'] );
247  trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
248  }
249  };
250  $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd );
251  $tempFile->bind( $status->value );
252  } else { // immediate write
253  $created = false;
254  // Use fwrite+rename since (a) this clears xattrs, (b) threads still reading the old
255  // inode are unaffected since it writes to a new inode, and (c) new threads reading
256  // the file will either totally see the old version or totally see the new version
257  $fsStagePath = $this->makeStagingPath( $fsDstPath );
258  $this->trapWarnings( '/: No such file or directory$/' );
259  $stageHandle = fopen( $fsStagePath, 'xb' );
260  if ( $stageHandle ) {
261  $bytes = fwrite( $stageHandle, $params['content'] );
262  $created = ( $bytes === strlen( $params['content'] ) );
263  fclose( $stageHandle );
264  $created = $created ? rename( $fsStagePath, $fsDstPath ) : false;
265  }
266  $hadError = $this->untrapWarnings();
267  if ( $hadError || !$created ) {
268  $status->fatal( 'backend-fail-create', $params['dst'] );
269 
270  return $status;
271  }
272  $this->chmod( $fsDstPath );
273  }
274 
275  return $status;
276  }
277 
278  protected function doStoreInternal( array $params ) {
279  $status = $this->newStatus();
280 
281  $fsSrcPath = $params['src']; // file system path
282  $fsDstPath = $this->resolveToFSPath( $params['dst'] );
283  if ( $fsDstPath === null ) {
284  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
285 
286  return $status;
287  }
288 
289  if ( $fsSrcPath === $fsDstPath ) {
290  $status->fatal( 'backend-fail-internal', $this->name );
291 
292  return $status; // sanity
293  }
294 
295  if ( !empty( $params['async'] ) ) { // deferred
296  $cmd = $this->makeCopyCommand( $fsSrcPath, $fsDstPath, false );
297  $handler = function ( $errors, StatusValue $status, array $params, $cmd ) {
298  if ( $errors !== '' && !( $this->os === 'Windows' && $errors[0] === " " ) ) {
299  $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
300  trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
301  }
302  };
303  $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd );
304  } else { // immediate write
305  $stored = false;
306  // Use fwrite+rename since (a) this clears xattrs, (b) threads still reading the old
307  // inode are unaffected since it writes to a new inode, and (c) new threads reading
308  // the file will either totally see the old version or totally see the new version
309  $fsStagePath = $this->makeStagingPath( $fsDstPath );
310  $this->trapWarnings( '/: No such file or directory$/' );
311  $srcHandle = fopen( $fsSrcPath, 'rb' );
312  if ( $srcHandle ) {
313  $stageHandle = fopen( $fsStagePath, 'xb' );
314  if ( $stageHandle ) {
315  $bytes = stream_copy_to_stream( $srcHandle, $stageHandle );
316  $stored = ( $bytes !== false && $bytes === fstat( $srcHandle )['size'] );
317  fclose( $stageHandle );
318  $stored = $stored ? rename( $fsStagePath, $fsDstPath ) : false;
319  }
320  fclose( $srcHandle );
321  }
322  $hadError = $this->untrapWarnings();
323  if ( $hadError || !$stored ) {
324  $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
325 
326  return $status;
327  }
328  $this->chmod( $fsDstPath );
329  }
330 
331  return $status;
332  }
333 
334  protected function doCopyInternal( array $params ) {
335  $status = $this->newStatus();
336 
337  $fsSrcPath = $this->resolveToFSPath( $params['src'] );
338  if ( $fsSrcPath === null ) {
339  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
340 
341  return $status;
342  }
343 
344  $fsDstPath = $this->resolveToFSPath( $params['dst'] );
345  if ( $fsDstPath === null ) {
346  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
347 
348  return $status;
349  }
350 
351  if ( $fsSrcPath === $fsDstPath ) {
352  return $status; // no-op
353  }
354 
355  $ignoreMissing = !empty( $params['ignoreMissingSource'] );
356 
357  if ( !empty( $params['async'] ) ) { // deferred
358  $cmd = $this->makeCopyCommand( $fsSrcPath, $fsDstPath, $ignoreMissing );
359  $handler = function ( $errors, StatusValue $status, array $params, $cmd ) {
360  if ( $errors !== '' && !( $this->os === 'Windows' && $errors[0] === " " ) ) {
361  $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
362  trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
363  }
364  };
365  $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd );
366  } else { // immediate write
367  $copied = false;
368  // Use fwrite+rename since (a) this clears xattrs, (b) threads still reading the old
369  // inode are unaffected since it writes to a new inode, and (c) new threads reading
370  // the file will either totally see the old version or totally see the new version
371  $fsStagePath = $this->makeStagingPath( $fsDstPath );
372  $this->trapWarnings( '/: No such file or directory$/' );
373  $srcHandle = fopen( $fsSrcPath, 'rb' );
374  if ( $srcHandle ) {
375  $stageHandle = fopen( $fsStagePath, 'xb' );
376  if ( $stageHandle ) {
377  $bytes = stream_copy_to_stream( $srcHandle, $stageHandle );
378  $copied = ( $bytes !== false && $bytes === fstat( $srcHandle )['size'] );
379  fclose( $stageHandle );
380  $copied = $copied ? rename( $fsStagePath, $fsDstPath ) : false;
381  }
382  fclose( $srcHandle );
383  }
384  $hadError = $this->untrapWarnings();
385  if ( $hadError || ( !$copied && !$ignoreMissing ) ) {
386  $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
387 
388  return $status;
389  }
390  if ( $copied ) {
391  $this->chmod( $fsDstPath );
392  }
393  }
394 
395  return $status;
396  }
397 
398  protected function doMoveInternal( array $params ) {
399  $status = $this->newStatus();
400 
401  $fsSrcPath = $this->resolveToFSPath( $params['src'] );
402  if ( $fsSrcPath === null ) {
403  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
404 
405  return $status;
406  }
407 
408  $fsDstPath = $this->resolveToFSPath( $params['dst'] );
409  if ( $fsDstPath === null ) {
410  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
411 
412  return $status;
413  }
414 
415  if ( $fsSrcPath === $fsDstPath ) {
416  return $status; // no-op
417  }
418 
419  $ignoreMissing = !empty( $params['ignoreMissingSource'] );
420 
421  if ( !empty( $params['async'] ) ) { // deferred
422  $cmd = $this->makeMoveCommand( $fsSrcPath, $fsDstPath, $ignoreMissing );
423  $handler = function ( $errors, StatusValue $status, array $params, $cmd ) {
424  if ( $errors !== '' && !( $this->os === 'Windows' && $errors[0] === " " ) ) {
425  $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
426  trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
427  }
428  };
429  $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd );
430  } else { // immediate write
431  // Use rename() here since (a) this clears xattrs, (b) any threads still reading the
432  // old inode are unaffected since it writes to a new inode, and (c) this is fast and
433  // atomic within a file system volume (as is normally the case)
434  $this->trapWarnings( '/: No such file or directory$/' );
435  $moved = rename( $fsSrcPath, $fsDstPath );
436  $hadError = $this->untrapWarnings();
437  if ( $hadError || ( !$moved && !$ignoreMissing ) ) {
438  $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
439 
440  return $status;
441  }
442  }
443 
444  return $status;
445  }
446 
447  protected function doDeleteInternal( array $params ) {
448  $status = $this->newStatus();
449 
450  $fsSrcPath = $this->resolveToFSPath( $params['src'] );
451  if ( $fsSrcPath === null ) {
452  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
453 
454  return $status;
455  }
456 
457  $ignoreMissing = !empty( $params['ignoreMissingSource'] );
458 
459  if ( !empty( $params['async'] ) ) { // deferred
460  $cmd = $this->makeUnlinkCommand( $fsSrcPath, $ignoreMissing );
461  $handler = function ( $errors, StatusValue $status, array $params, $cmd ) {
462  if ( $errors !== '' && !( $this->os === 'Windows' && $errors[0] === " " ) ) {
463  $status->fatal( 'backend-fail-delete', $params['src'] );
464  trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
465  }
466  };
467  $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd );
468  } else { // immediate write
469  $this->trapWarnings( '/: No such file or directory$/' );
470  $deleted = unlink( $fsSrcPath );
471  $hadError = $this->untrapWarnings();
472  if ( $hadError || ( !$deleted && !$ignoreMissing ) ) {
473  $status->fatal( 'backend-fail-delete', $params['src'] );
474 
475  return $status;
476  }
477  }
478 
479  return $status;
480  }
481 
488  protected function doPrepareInternal( $fullCont, $dirRel, array $params ) {
489  $status = $this->newStatus();
490  list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
491  $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
492  $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
493  // Create the directory and its parents as needed...
494  $created = false;
495  AtEase::suppressWarnings();
496  $alreadyExisted = is_dir( $fsDirectory ); // already there?
497  if ( !$alreadyExisted ) {
498  $created = mkdir( $fsDirectory, $this->dirMode, true );
499  if ( !$created ) {
500  $alreadyExisted = is_dir( $fsDirectory ); // another thread made it?
501  }
502  }
503  $isWritable = $created ?: is_writable( $fsDirectory ); // assume writable if created here
504  AtEase::restoreWarnings();
505  if ( !$alreadyExisted && !$created ) {
506  $this->logger->error( __METHOD__ . ": cannot create directory $fsDirectory" );
507  $status->fatal( 'directorycreateerror', $params['dir'] ); // fails on races
508  } elseif ( !$isWritable ) {
509  $this->logger->error( __METHOD__ . ": directory $fsDirectory is read-only" );
510  $status->fatal( 'directoryreadonlyerror', $params['dir'] );
511  }
512  // Respect any 'noAccess' or 'noListing' flags...
513  if ( $created ) {
514  $status->merge( $this->doSecureInternal( $fullCont, $dirRel, $params ) );
515  }
516 
517  if ( $status->isOK() ) {
518  $this->usableDirCache->set( $fsDirectory, 1 );
519  }
520 
521  return $status;
522  }
523 
524  protected function doSecureInternal( $fullCont, $dirRel, array $params ) {
525  $status = $this->newStatus();
526  list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
527  $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
528  $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
529  // Seed new directories with a blank index.html, to prevent crawling...
530  if ( !empty( $params['noListing'] ) && !is_file( "{$fsDirectory}/index.html" ) ) {
531  $this->trapWarnings();
532  $bytes = file_put_contents( "{$fsDirectory}/index.html", $this->indexHtmlPrivate() );
533  $this->untrapWarnings();
534  if ( $bytes === false ) {
535  $status->fatal( 'backend-fail-create', $params['dir'] . '/index.html' );
536  }
537  }
538  // Add a .htaccess file to the root of the container...
539  if ( !empty( $params['noAccess'] ) && !is_file( "{$contRoot}/.htaccess" ) ) {
540  AtEase::suppressWarnings();
541  $bytes = file_put_contents( "{$contRoot}/.htaccess", $this->htaccessPrivate() );
542  AtEase::restoreWarnings();
543  if ( $bytes === false ) {
544  $storeDir = "mwstore://{$this->name}/{$shortCont}";
545  $status->fatal( 'backend-fail-create', "{$storeDir}/.htaccess" );
546  }
547  }
548 
549  return $status;
550  }
551 
552  protected function doPublishInternal( $fullCont, $dirRel, array $params ) {
553  $status = $this->newStatus();
554  list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
555  $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
556  $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
557  // Unseed new directories with a blank index.html, to allow crawling...
558  if ( !empty( $params['listing'] ) && is_file( "{$fsDirectory}/index.html" ) ) {
559  $exists = ( file_get_contents( "{$fsDirectory}/index.html" ) === $this->indexHtmlPrivate() );
560  if ( $exists && !$this->unlink( "{$fsDirectory}/index.html" ) ) { // reverse secure()
561  $status->fatal( 'backend-fail-delete', $params['dir'] . '/index.html' );
562  }
563  }
564  // Remove the .htaccess file from the root of the container...
565  if ( !empty( $params['access'] ) && is_file( "{$contRoot}/.htaccess" ) ) {
566  $exists = ( file_get_contents( "{$contRoot}/.htaccess" ) === $this->htaccessPrivate() );
567  if ( $exists && !$this->unlink( "{$contRoot}/.htaccess" ) ) { // reverse secure()
568  $storeDir = "mwstore://{$this->name}/{$shortCont}";
569  $status->fatal( 'backend-fail-delete', "{$storeDir}/.htaccess" );
570  }
571  }
572 
573  return $status;
574  }
575 
576  protected function doCleanInternal( $fullCont, $dirRel, array $params ) {
577  $status = $this->newStatus();
578  list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
579  $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
580  $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
581 
582  $this->rmdir( $fsDirectory );
583 
584  return $status;
585  }
586 
587  protected function doGetFileStat( array $params ) {
588  $fsSrcPath = $this->resolveToFSPath( $params['src'] );
589  if ( $fsSrcPath === null ) {
590  return self::$RES_ERROR; // invalid storage path
591  }
592 
593  $this->trapWarnings(); // don't trust 'false' if there were errors
594  $stat = is_file( $fsSrcPath ) ? stat( $fsSrcPath ) : false; // regular files only
595  $hadError = $this->untrapWarnings();
596 
597  if ( is_array( $stat ) ) {
598  $ct = new ConvertibleTimestamp( $stat['mtime'] );
599 
600  return [
601  'mtime' => $ct->getTimestamp( TS_MW ),
602  'size' => $stat['size']
603  ];
604  }
605 
606  return $hadError ? self::$RES_ERROR : self::$RES_ABSENT;
607  }
608 
609  protected function doClearCache( array $paths = null ) {
610  if ( is_array( $paths ) ) {
611  foreach ( $paths as $path ) {
612  $fsPath = $this->resolveToFSPath( $path );
613  if ( $fsPath !== null ) {
614  clearstatcache( true, $fsPath );
615  $this->usableDirCache->clear( $fsPath );
616  }
617  }
618  } else {
619  clearstatcache( true ); // clear the PHP file stat cache
620  $this->usableDirCache->clear();
621  }
622  }
623 
624  protected function doDirectoryExists( $fullCont, $dirRel, array $params ) {
625  list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
626  $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
627  $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
628 
629  $this->trapWarnings(); // don't trust 'false' if there were errors
630  $exists = is_dir( $fsDirectory );
631  $hadError = $this->untrapWarnings();
632 
633  return $hadError ? self::$RES_ERROR : $exists;
634  }
635 
643  public function getDirectoryListInternal( $fullCont, $dirRel, array $params ) {
644  list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
645  $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
646  $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
647 
648  $list = new FSFileBackendDirList( $fsDirectory, $params );
649  $error = $list->getLastError();
650  if ( $error !== null ) {
651  if ( preg_match( '/: No such file or directory$/', $error ) ) {
652  $this->logger->info( __METHOD__ . ": non-existant directory: '$fsDirectory'" );
653 
654  return []; // nothing under this dir
655  } elseif ( is_dir( $fsDirectory ) ) {
656  $this->logger->warning( __METHOD__ . ": unreadable directory: '$fsDirectory'" );
657 
658  return self::$RES_ERROR; // bad permissions?
659  } else {
660  $this->logger->warning( __METHOD__ . ": unreachable directory: '$fsDirectory'" );
661 
662  return self::$RES_ERROR;
663  }
664  }
665 
666  return $list;
667  }
668 
676  public function getFileListInternal( $fullCont, $dirRel, array $params ) {
677  list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
678  $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
679  $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
680 
681  $list = new FSFileBackendFileList( $fsDirectory, $params );
682  $error = $list->getLastError();
683  if ( $error !== null ) {
684  if ( preg_match( '/: No such file or directory$/', $error ) ) {
685  $this->logger->info( __METHOD__ . ": non-existant directory: '$fsDirectory'" );
686 
687  return []; // nothing under this dir
688  } elseif ( is_dir( $fsDirectory ) ) {
689  $this->logger->warning( __METHOD__ . ": unreadable directory: '$fsDirectory'" );
690 
691  return self::$RES_ERROR; // bad permissions?
692  } else {
693  $this->logger->warning( __METHOD__ . ": unreachable directory: '$fsDirectory'" );
694 
695  return self::$RES_ERROR;
696  }
697  }
698 
699  return $list;
700  }
701 
702  protected function doGetLocalReferenceMulti( array $params ) {
703  $fsFiles = []; // (path => FSFile)
704 
705  foreach ( $params['srcs'] as $src ) {
706  $source = $this->resolveToFSPath( $src );
707  if ( $source === null ) {
708  $fsFiles[$src] = self::$RES_ERROR; // invalid path
709  continue;
710  }
711 
712  $this->trapWarnings(); // don't trust 'false' if there were errors
713  $isFile = is_file( $source ); // regular files only
714  $hadError = $this->untrapWarnings();
715 
716  if ( $isFile ) {
717  $fsFiles[$src] = new FSFile( $source );
718  } elseif ( $hadError ) {
719  $fsFiles[$src] = self::$RES_ERROR;
720  } else {
721  $fsFiles[$src] = self::$RES_ABSENT;
722  }
723  }
724 
725  return $fsFiles;
726  }
727 
728  protected function doGetLocalCopyMulti( array $params ) {
729  $tmpFiles = []; // (path => TempFSFile)
730 
731  foreach ( $params['srcs'] as $src ) {
732  $source = $this->resolveToFSPath( $src );
733  if ( $source === null ) {
734  $tmpFiles[$src] = self::$RES_ERROR; // invalid path
735  continue;
736  }
737  // Create a new temporary file with the same extension...
739  $tmpFile = $this->tmpFileFactory->newTempFSFile( 'localcopy_', $ext );
740  if ( !$tmpFile ) {
741  $tmpFiles[$src] = self::$RES_ERROR;
742  continue;
743  }
744 
745  $tmpPath = $tmpFile->getPath();
746  // Copy the source file over the temp file
747  $this->trapWarnings(); // don't trust 'false' if there were errors
748  $isFile = is_file( $source ); // regular files only
749  $copySuccess = $isFile ? copy( $source, $tmpPath ) : false;
750  $hadError = $this->untrapWarnings();
751 
752  if ( $copySuccess ) {
753  $this->chmod( $tmpPath );
754  $tmpFiles[$src] = $tmpFile;
755  } elseif ( $hadError ) {
756  $tmpFiles[$src] = self::$RES_ERROR; // copy failed
757  } else {
758  $tmpFiles[$src] = self::$RES_ABSENT;
759  }
760  }
761 
762  return $tmpFiles;
763  }
764 
765  protected function directoriesAreVirtual() {
766  return false;
767  }
768 
774  protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
775  $statuses = [];
776 
777  $pipes = [];
778  foreach ( $fileOpHandles as $index => $fileOpHandle ) {
779  $pipes[$index] = popen( $fileOpHandle->cmd, 'r' );
780  }
781 
782  $errs = [];
783  foreach ( $pipes as $index => $pipe ) {
784  // Result will be empty on success in *NIX. On Windows,
785  // it may be something like " 1 file(s) [copied|moved].".
786  $errs[$index] = stream_get_contents( $pipe );
787  fclose( $pipe );
788  }
789 
790  foreach ( $fileOpHandles as $index => $fileOpHandle ) {
791  $status = $this->newStatus();
792  $function = $fileOpHandle->callback;
793  $function( $errs[$index], $status, $fileOpHandle->params, $fileOpHandle->cmd );
794  $statuses[$index] = $status;
795  }
796 
797  return $statuses;
798  }
799 
804  private function makeStagingPath( $fsPath ) {
805  $time = dechex( time() ); // make it easy to find old orphans
806  $hash = \Wikimedia\base_convert( md5( basename( $fsPath ) ), 16, 36, 25 );
807  $unique = \Wikimedia\base_convert( bin2hex( random_bytes( 16 ) ), 16, 36, 25 );
808 
809  return dirname( $fsPath ) . "/.{$time}_{$hash}_{$unique}.tmpfsfile";
810  }
811 
818  private function makeCopyCommand( $fsSrcPath, $fsDstPath, $ignoreMissing ) {
819  // Use copy+rename since (a) this clears xattrs, (b) threads still reading the old
820  // inode are unaffected since it writes to a new inode, and (c) new threads reading
821  // the file will either totally see the old version or totally see the new version
822  $fsStagePath = $this->makeStagingPath( $fsDstPath );
823  $encSrc = escapeshellarg( $this->cleanPathSlashes( $fsSrcPath ) );
824  $encStage = escapeshellarg( $this->cleanPathSlashes( $fsStagePath ) );
825  $encDst = escapeshellarg( $this->cleanPathSlashes( $fsDstPath ) );
826  if ( $this->os === 'Windows' ) {
827  // https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/copy
828  // https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/move
829  $cmdWrite = "COPY /B /Y $encSrc $encStage 2>&1 && MOVE /Y $encStage $encDst 2>&1";
830  $cmd = $ignoreMissing ? "IF EXIST $encSrc $cmdWrite" : $cmdWrite;
831  } else {
832  // https://manpages.debian.org/buster/coreutils/cp.1.en.html
833  // https://manpages.debian.org/buster/coreutils/mv.1.en.html
834  $cmdWrite = "cp $encSrc $encStage 2>&1 && mv $encStage $encDst 2>&1";
835  $cmd = $ignoreMissing ? "test -f $encSrc && $cmdWrite" : $cmdWrite;
836  // Clean up permissions on any newly created destination file
837  $octalPermissions = '0' . decoct( $this->fileMode );
838  if ( strlen( $octalPermissions ) == 4 ) {
839  $cmd .= " && chmod $octalPermissions $encDst 2>/dev/null";
840  }
841  }
842 
843  return $cmd;
844  }
845 
852  private function makeMoveCommand( $fsSrcPath, $fsDstPath, $ignoreMissing = false ) {
853  // https://manpages.debian.org/buster/coreutils/mv.1.en.html
854  // https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/move
855  $encSrc = escapeshellarg( $this->cleanPathSlashes( $fsSrcPath ) );
856  $encDst = escapeshellarg( $this->cleanPathSlashes( $fsDstPath ) );
857  if ( $this->os === 'Windows' ) {
858  $writeCmd = "MOVE /Y $encSrc $encDst 2>&1";
859  $cmd = $ignoreMissing ? "IF EXIST $encSrc $writeCmd" : $writeCmd;
860  } else {
861  $writeCmd = "mv -f $encSrc $encDst 2>&1";
862  $cmd = $ignoreMissing ? "test -f $encSrc && $writeCmd" : $writeCmd;
863  }
864 
865  return $cmd;
866  }
867 
873  private function makeUnlinkCommand( $fsPath, $ignoreMissing = false ) {
874  // https://manpages.debian.org/buster/coreutils/rm.1.en.html
875  // https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/del
876  $encSrc = escapeshellarg( $this->cleanPathSlashes( $fsPath ) );
877  if ( $this->os === 'Windows' ) {
878  $writeCmd = "DEL /Q $encSrc 2>&1";
879  $cmd = $ignoreMissing ? "IF EXIST $encSrc $writeCmd" : $writeCmd;
880  } else {
881  $cmd = $ignoreMissing ? "rm -f $encSrc 2>&1" : "rm $encSrc 2>&1";
882  }
883 
884  return $cmd;
885  }
886 
893  protected function chmod( $fsPath ) {
894  if ( $this->os === 'Windows' ) {
895  return true;
896  }
897 
898  AtEase::suppressWarnings();
899  $ok = chmod( $fsPath, $this->fileMode );
900  AtEase::restoreWarnings();
901 
902  return $ok;
903  }
904 
911  protected function unlink( $fsPath ) {
912  AtEase::suppressWarnings();
913  $ok = unlink( $fsPath );
914  AtEase::restoreWarnings();
915  clearstatcache( true, $fsPath );
916 
917  return $ok;
918  }
919 
926  protected function rmdir( $fsDirectory ) {
927  AtEase::suppressWarnings();
928  $ok = rmdir( $fsDirectory ); // remove directory if empty
929  AtEase::restoreWarnings();
930  clearstatcache( true, $fsDirectory );
931 
932  return $ok;
933  }
934 
939  protected function newTempFileWithContent( array $params ) {
940  $tempFile = $this->tmpFileFactory->newTempFSFile( 'create_', 'tmp' );
941  if ( !$tempFile ) {
942  return null;
943  }
944 
945  AtEase::suppressWarnings();
946  if ( file_put_contents( $tempFile->getPath(), $params['content'] ) === false ) {
947  $tempFile = null;
948  }
949  AtEase::restoreWarnings();
950 
951  return $tempFile;
952  }
953 
959  protected function indexHtmlPrivate() {
960  return '';
961  }
962 
968  protected function htaccessPrivate() {
969  return "Deny from all\n";
970  }
971 
978  protected function cleanPathSlashes( $fsPath ) {
979  return ( $this->os === 'Windows' ) ? strtr( $fsPath, '/', '\\' ) : $fsPath;
980  }
981 
987  protected function trapWarnings( $regexIgnore = null ) {
988  $this->warningTrapStack[] = false;
989  set_error_handler( function ( $errno, $errstr ) use ( $regexIgnore ) {
990  if ( $regexIgnore === null || !preg_match( $regexIgnore, $errstr ) ) {
991  $this->logger->error( $errstr );
992  $this->warningTrapStack[count( $this->warningTrapStack ) - 1] = true;
993  }
994  return true; // suppress from PHP handler
995  }, E_WARNING );
996  }
997 
1003  protected function untrapWarnings() {
1004  restore_error_handler();
1005 
1006  return array_pop( $this->warningTrapStack );
1007  }
1008 }
FileBackend\splitStoragePath
static splitStoragePath( $storagePath)
Split a storage path into a backend name, a container name, and a relative file path.
Definition: FileBackend.php:1536
FSFileBackend\__construct
__construct(array $config)
Definition: FSFileBackend.php:98
FSFileBackend\doCleanInternal
doCleanInternal( $fullCont, $dirRel, array $params)
Definition: FSFileBackend.php:576
FSFileOpHandle
Definition: FSFileOpHandle.php:22
StatusValue
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Definition: StatusValue.php:42
FSFileBackend\doStoreInternal
doStoreInternal(array $params)
Definition: FSFileBackend.php:278
FSFileBackend\doClearCache
doClearCache(array $paths=null)
Clears any additional stat caches for storage paths.
Definition: FSFileBackend.php:609
FSFileBackend\doPublishInternal
doPublishInternal( $fullCont, $dirRel, array $params)
Definition: FSFileBackend.php:552
FSFileBackend\$currentUser
string $currentUser
OS username running this script.
Definition: FSFileBackend.php:83
FSFileBackend\doPrepareInternal
doPrepareInternal( $fullCont, $dirRel, array $params)
Definition: FSFileBackend.php:488
FSFileBackend\makeUnlinkCommand
makeUnlinkCommand( $fsPath, $ignoreMissing=false)
Definition: FSFileBackend.php:873
FileBackend\extensionFromPath
static extensionFromPath( $path, $case='lowercase')
Get the final extension from a storage or FS path.
Definition: FileBackend.php:1598
FSFileBackend\$dirMode
int $dirMode
Directory permission mode.
Definition: FSFileBackend.php:74
FSFileBackend\htaccessPrivate
htaccessPrivate()
Return the text of a .htaccess file to make a directory private.
Definition: FSFileBackend.php:968
FSFileBackend\doGetFileStat
doGetFileStat(array $params)
Definition: FSFileBackend.php:587
FSFileBackend\directoriesAreVirtual
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
Definition: FSFileBackend.php:765
FSFileBackend\$fileOwner
string $fileOwner
Required OS username to own files.
Definition: FSFileBackend.php:78
FSFileBackend\makeStagingPath
makeStagingPath( $fsPath)
Definition: FSFileBackend.php:804
FSFileBackend\cleanPathSlashes
cleanPathSlashes( $fsPath)
Clean up directory separators for the given OS.
Definition: FSFileBackend.php:978
FileBackendStore\$RES_ERROR
static null $RES_ERROR
Idiom for "no result due to I/O errors" (since 1.34)
Definition: FileBackendStore.php:65
FSFileBackend\doGetLocalReferenceMulti
doGetLocalReferenceMulti(array $params)
Definition: FSFileBackend.php:702
FSFileBackend\$basePath
string $basePath
Directory holding the container directories.
Definition: FSFileBackend.php:68
FSFileBackend\makeMoveCommand
makeMoveCommand( $fsSrcPath, $fsDstPath, $ignoreMissing=false)
Definition: FSFileBackend.php:852
FSFileBackend\doDirectoryExists
doDirectoryExists( $fullCont, $dirRel, array $params)
Definition: FSFileBackend.php:624
FSFileBackend\getDirectoryListInternal
getDirectoryListInternal( $fullCont, $dirRel, array $params)
Definition: FSFileBackend.php:643
FSFileBackend\getFeatures
getFeatures()
Get the a bitfield of extra features supported by the backend medium.
Definition: FSFileBackend.php:131
FSFileBackendFileList
Definition: FSFileBackendFileList.php:22
FSFileBackend\containerFSRoot
containerFSRoot( $shortCont, $fullCont)
Given the short (unresolved) and full (resolved) name of a container, return the file system path of ...
Definition: FSFileBackend.php:173
FSFileBackendDirList
Definition: FSFileBackendDirList.php:22
FSFileBackend\rmdir
rmdir( $fsDirectory)
Remove an empty directory, suppressing the warnings.
Definition: FSFileBackend.php:926
FileBackend\ATTR_UNICODE_PATHS
const ATTR_UNICODE_PATHS
Definition: FileBackend.php:134
MapCacheLRU
Handles a simple LRU key/value map with a maximum number of entries.
Definition: MapCacheLRU.php:38
FSFileBackend\$warningTrapStack
bool[] $warningTrapStack
Map of (stack index => whether a warning happened)
Definition: FSFileBackend.php:86
FSFileBackend\doSecureInternal
doSecureInternal( $fullCont, $dirRel, array $params)
Definition: FSFileBackend.php:524
FSFileBackend\makeCopyCommand
makeCopyCommand( $fsSrcPath, $fsDstPath, $ignoreMissing)
Definition: FSFileBackend.php:818
FSFileBackend\doExecuteOpHandlesInternal
doExecuteOpHandlesInternal(array $fileOpHandles)
Definition: FSFileBackend.php:774
FSFileBackend\$containerPaths
array $containerPaths
Map of container names to root paths for custom container paths.
Definition: FSFileBackend.php:71
FSFileBackend\resolveToFSPath
resolveToFSPath( $storagePath)
Get the absolute file system path for a storage path.
Definition: FSFileBackend.php:189
FSFileBackend\doDeleteInternal
doDeleteInternal(array $params)
Definition: FSFileBackend.php:447
FSFileBackend\$fileMode
int $fileMode
File permission mode.
Definition: FSFileBackend.php:76
FSFileBackend\resolveContainerPath
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
Definition: FSFileBackend.php:135
FileBackendStore\resolveStoragePathReal
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
Definition: FileBackendStore.php:1623
FileBackendStore
Base class for all backends using particular storage medium.
Definition: FileBackendStore.php:40
FileBackendStore\$RES_ABSENT
static false $RES_ABSENT
Idiom for "no result due to missing file" (since 1.34)
Definition: FileBackendStore.php:63
FSFileBackend\unlink
unlink( $fsPath)
Unlink a file, suppressing the warnings.
Definition: FSFileBackend.php:911
FSFile
Class representing a non-directory file on the file system.
Definition: FSFile.php:32
FileBackend\newStatus
newStatus(... $args)
Yields the result of the status wrapper callback on either:
Definition: FileBackend.php:1691
FSFileBackend
Class for a file system (FS) based file backend.
Definition: FSFileBackend.php:63
FSFileBackend\indexHtmlPrivate
indexHtmlPrivate()
Return the text of an index.html file to hide directory listings.
Definition: FSFileBackend.php:959
$path
$path
Definition: NoLocalSettings.php:25
FSFileBackend\doCreateInternal
doCreateInternal(array $params)
Definition: FSFileBackend.php:226
FSFileBackend\$usableDirCache
MapCacheLRU $usableDirCache
Cache for known prepared/usable directorries.
Definition: FSFileBackend.php:65
$source
$source
Definition: mwdoc-filter.php:34
FSFileBackend\isPathUsableInternal
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path in the backend.
Definition: FSFileBackend.php:203
FSFileBackend\$os
bool $os
Simpler version of PHP_OS_FAMILY (PHP 7.2)
Definition: FSFileBackend.php:81
$ext
if(!is_readable( $file)) $ext
Definition: router.php:48
FileBackend\copy
copy(array $params, array $opts=[])
Performs a single copy operation.
Definition: FileBackend.php:544
FSFileBackend\chmod
chmod( $fsPath)
Chmod a file, suppressing the warnings.
Definition: FSFileBackend.php:893
FSFileBackend\untrapWarnings
untrapWarnings()
Stop listening for E_WARNING errors and get whether any happened.
Definition: FSFileBackend.php:1003
FSFileBackend\doGetLocalCopyMulti
doGetLocalCopyMulti(array $params)
Definition: FSFileBackend.php:728
FSFileBackend\newTempFileWithContent
newTempFileWithContent(array $params)
Definition: FSFileBackend.php:939
FSFileBackend\doCopyInternal
doCopyInternal(array $params)
Definition: FSFileBackend.php:334
FSFileBackend\trapWarnings
trapWarnings( $regexIgnore=null)
Listen for E_WARNING errors and track whether any that happen.
Definition: FSFileBackend.php:987
FSFileBackend\isLegalRelPath
isLegalRelPath( $fsPath)
Sanity check a relative file system path for validity.
Definition: FSFileBackend.php:153
FSFileBackend\getFileListInternal
getFileListInternal( $fullCont, $dirRel, array $params)
Definition: FSFileBackend.php:676
FSFileBackend\doMoveInternal
doMoveInternal(array $params)
Definition: FSFileBackend.php:398