MediaWiki  master
FSFileBackend.php
Go to the documentation of this file.
1 <?php
44 use Shellbox\Shellbox;
45 use Wikimedia\AtEase\AtEase;
46 use Wikimedia\Timestamp\ConvertibleTimestamp;
47 
66  protected $usableDirCache;
67 
69  protected $basePath;
70 
72  protected $containerPaths;
73 
75  protected $dirMode;
77  protected $fileMode;
79  protected $fileOwner;
80 
82  protected $os;
84  protected $currentUser;
85 
87  private $warningTrapStack = [];
88 
99  public function __construct( array $config ) {
100  parent::__construct( $config );
101 
102  if ( PHP_OS_FAMILY === 'Windows' ) {
103  $this->os = 'Windows';
104  } elseif ( PHP_OS_FAMILY === 'BSD' || PHP_OS_FAMILY === 'Darwin' ) {
105  $this->os = 'BSD';
106  } else {
107  $this->os = 'Linux';
108  }
109  // Remove any possible trailing slash from directories
110  if ( isset( $config['basePath'] ) ) {
111  $this->basePath = rtrim( $config['basePath'], '/' ); // remove trailing slash
112  } else {
113  $this->basePath = null; // none; containers must have explicit paths
114  }
115 
116  $this->containerPaths = [];
117  foreach ( ( $config['containerPaths'] ?? [] ) as $container => $fsPath ) {
118  $this->containerPaths[$container] = rtrim( $fsPath, '/' ); // remove trailing slash
119  }
120 
121  $this->fileMode = $config['fileMode'] ?? 0644;
122  $this->dirMode = $config['directoryMode'] ?? 0777;
123  if ( isset( $config['fileOwner'] ) && function_exists( 'posix_getuid' ) ) {
124  $this->fileOwner = $config['fileOwner'];
125  // Cache this, assuming it doesn't change
126  $this->currentUser = posix_getpwuid( posix_getuid() )['name'];
127  }
128 
129  $this->usableDirCache = new MapCacheLRU( self::CACHE_CHEAP_SIZE );
130  }
131 
132  public function getFeatures() {
134  }
135 
136  protected function resolveContainerPath( $container, $relStoragePath ) {
137  // Check that container has a root directory
138  if ( isset( $this->containerPaths[$container] ) || isset( $this->basePath ) ) {
139  // Check for sane relative paths (assume the base paths are OK)
140  if ( $this->isLegalRelPath( $relStoragePath ) ) {
141  return $relStoragePath;
142  }
143  }
144 
145  return null; // invalid
146  }
147 
154  protected function isLegalRelPath( $fsPath ) {
155  // Check for file names longer than 255 chars
156  if ( preg_match( '![^/]{256}!', $fsPath ) ) { // ext3/NTFS
157  return false;
158  }
159  if ( $this->os === 'Windows' ) { // NTFS
160  return !preg_match( '![:*?"<>|]!', $fsPath );
161  } else {
162  return true;
163  }
164  }
165 
174  protected function containerFSRoot( $shortCont, $fullCont ) {
175  if ( isset( $this->containerPaths[$shortCont] ) ) {
176  return $this->containerPaths[$shortCont];
177  } elseif ( isset( $this->basePath ) ) {
178  return "{$this->basePath}/{$fullCont}";
179  }
180 
181  return null; // no container base path defined
182  }
183 
190  protected function resolveToFSPath( $storagePath ) {
191  list( $fullCont, $relPath ) = $this->resolveStoragePathReal( $storagePath );
192  if ( $relPath === null ) {
193  return null; // invalid
194  }
195  list( , $shortCont, ) = FileBackend::splitStoragePath( $storagePath );
196  $fsPath = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
197  if ( $relPath != '' ) {
198  $fsPath .= "/{$relPath}";
199  }
200 
201  return $fsPath;
202  }
203 
204  public function isPathUsableInternal( $storagePath ) {
205  $fsPath = $this->resolveToFSPath( $storagePath );
206  if ( $fsPath === null ) {
207  return false; // invalid
208  }
209 
210  if ( $this->fileOwner !== null && $this->currentUser !== $this->fileOwner ) {
211  trigger_error( __METHOD__ . ": PHP process owner is not '{$this->fileOwner}'." );
212  return false;
213  }
214 
215  $fsDirectory = dirname( $fsPath );
216  $usable = $this->usableDirCache->get( $fsDirectory, MapCacheLRU::TTL_PROC_SHORT );
217  if ( $usable === null ) {
218  AtEase::suppressWarnings();
219  $usable = is_dir( $fsDirectory ) && is_writable( $fsDirectory );
220  AtEase::restoreWarnings();
221  $this->usableDirCache->set( $fsDirectory, $usable ? 1 : 0 );
222  }
223 
224  return $usable;
225  }
226 
227  protected function doCreateInternal( array $params ) {
228  $status = $this->newStatus();
229 
230  $fsDstPath = $this->resolveToFSPath( $params['dst'] );
231  if ( $fsDstPath === null ) {
232  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
233 
234  return $status;
235  }
236 
237  if ( !empty( $params['async'] ) ) { // deferred
238  $tempFile = $this->newTempFileWithContent( $params );
239  if ( !$tempFile ) {
240  $status->fatal( 'backend-fail-create', $params['dst'] );
241 
242  return $status;
243  }
244  $cmd = $this->makeCopyCommand( $tempFile->getPath(), $fsDstPath, false );
245  $handler = function ( $errors, StatusValue $status, array $params, $cmd ) {
246  if ( $errors !== '' && !( $this->os === 'Windows' && $errors[0] === " " ) ) {
247  $status->fatal( 'backend-fail-create', $params['dst'] );
248  trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
249  }
250  };
251  $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd );
252  $tempFile->bind( $status->value );
253  } else { // immediate write
254  $created = false;
255  // Use fwrite+rename since (a) this clears xattrs, (b) threads still reading the old
256  // inode are unaffected since it writes to a new inode, and (c) new threads reading
257  // the file will either totally see the old version or totally see the new version
258  $fsStagePath = $this->makeStagingPath( $fsDstPath );
260  $stageHandle = fopen( $fsStagePath, 'xb' );
261  if ( $stageHandle ) {
262  $bytes = fwrite( $stageHandle, $params['content'] );
263  $created = ( $bytes === strlen( $params['content'] ) );
264  fclose( $stageHandle );
265  $created = $created ? rename( $fsStagePath, $fsDstPath ) : false;
266  }
267  $hadError = $this->untrapWarnings();
268  if ( $hadError || !$created ) {
269  $status->fatal( 'backend-fail-create', $params['dst'] );
270 
271  return $status;
272  }
273  $this->chmod( $fsDstPath );
274  }
275 
276  return $status;
277  }
278 
279  protected function doStoreInternal( array $params ) {
280  $status = $this->newStatus();
281 
282  $fsSrcPath = $params['src']; // file system path
283  $fsDstPath = $this->resolveToFSPath( $params['dst'] );
284  if ( $fsDstPath === null ) {
285  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
286 
287  return $status;
288  }
289 
290  if ( $fsSrcPath === $fsDstPath ) {
291  $status->fatal( 'backend-fail-internal', $this->name );
292 
293  return $status; // sanity
294  }
295 
296  if ( !empty( $params['async'] ) ) { // deferred
297  $cmd = $this->makeCopyCommand( $fsSrcPath, $fsDstPath, false );
298  $handler = function ( $errors, StatusValue $status, array $params, $cmd ) {
299  if ( $errors !== '' && !( $this->os === 'Windows' && $errors[0] === " " ) ) {
300  $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
301  trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
302  }
303  };
304  $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd );
305  } else { // immediate write
306  $stored = false;
307  // Use fwrite+rename since (a) this clears xattrs, (b) threads still reading the old
308  // inode are unaffected since it writes to a new inode, and (c) new threads reading
309  // the file will either totally see the old version or totally see the new version
310  $fsStagePath = $this->makeStagingPath( $fsDstPath );
312  $srcHandle = fopen( $fsSrcPath, 'rb' );
313  if ( $srcHandle ) {
314  $stageHandle = fopen( $fsStagePath, 'xb' );
315  if ( $stageHandle ) {
316  $bytes = stream_copy_to_stream( $srcHandle, $stageHandle );
317  $stored = ( $bytes !== false && $bytes === fstat( $srcHandle )['size'] );
318  fclose( $stageHandle );
319  $stored = $stored ? rename( $fsStagePath, $fsDstPath ) : false;
320  }
321  fclose( $srcHandle );
322  }
323  $hadError = $this->untrapWarnings();
324  if ( $hadError || !$stored ) {
325  $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
326 
327  return $status;
328  }
329  $this->chmod( $fsDstPath );
330  }
331 
332  return $status;
333  }
334 
335  protected function doCopyInternal( array $params ) {
336  $status = $this->newStatus();
337 
338  $fsSrcPath = $this->resolveToFSPath( $params['src'] );
339  if ( $fsSrcPath === null ) {
340  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
341 
342  return $status;
343  }
344 
345  $fsDstPath = $this->resolveToFSPath( $params['dst'] );
346  if ( $fsDstPath === null ) {
347  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
348 
349  return $status;
350  }
351 
352  if ( $fsSrcPath === $fsDstPath ) {
353  return $status; // no-op
354  }
355 
356  $ignoreMissing = !empty( $params['ignoreMissingSource'] );
357 
358  if ( !empty( $params['async'] ) ) { // deferred
359  $cmd = $this->makeCopyCommand( $fsSrcPath, $fsDstPath, $ignoreMissing );
360  $handler = function ( $errors, StatusValue $status, array $params, $cmd ) {
361  if ( $errors !== '' && !( $this->os === 'Windows' && $errors[0] === " " ) ) {
362  $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
363  trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
364  }
365  };
366  $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd );
367  } else { // immediate write
368  $copied = false;
369  // Use fwrite+rename since (a) this clears xattrs, (b) threads still reading the old
370  // inode are unaffected since it writes to a new inode, and (c) new threads reading
371  // the file will either totally see the old version or totally see the new version
372  $fsStagePath = $this->makeStagingPath( $fsDstPath );
374  $srcHandle = fopen( $fsSrcPath, 'rb' );
375  if ( $srcHandle ) {
376  $stageHandle = fopen( $fsStagePath, 'xb' );
377  if ( $stageHandle ) {
378  $bytes = stream_copy_to_stream( $srcHandle, $stageHandle );
379  $copied = ( $bytes !== false && $bytes === fstat( $srcHandle )['size'] );
380  fclose( $stageHandle );
381  $copied = $copied ? rename( $fsStagePath, $fsDstPath ) : false;
382  }
383  fclose( $srcHandle );
384  }
385  $hadError = $this->untrapWarnings();
386  if ( $hadError || ( !$copied && !$ignoreMissing ) ) {
387  $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
388 
389  return $status;
390  }
391  if ( $copied ) {
392  $this->chmod( $fsDstPath );
393  }
394  }
395 
396  return $status;
397  }
398 
399  protected function doMoveInternal( array $params ) {
400  $status = $this->newStatus();
401 
402  $fsSrcPath = $this->resolveToFSPath( $params['src'] );
403  if ( $fsSrcPath === null ) {
404  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
405 
406  return $status;
407  }
408 
409  $fsDstPath = $this->resolveToFSPath( $params['dst'] );
410  if ( $fsDstPath === null ) {
411  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
412 
413  return $status;
414  }
415 
416  if ( $fsSrcPath === $fsDstPath ) {
417  return $status; // no-op
418  }
419 
420  $ignoreMissing = !empty( $params['ignoreMissingSource'] );
421 
422  if ( !empty( $params['async'] ) ) { // deferred
423  $cmd = $this->makeMoveCommand( $fsSrcPath, $fsDstPath, $ignoreMissing );
424  $handler = function ( $errors, StatusValue $status, array $params, $cmd ) {
425  if ( $errors !== '' && !( $this->os === 'Windows' && $errors[0] === " " ) ) {
426  $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
427  trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
428  }
429  };
430  $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd );
431  } else { // immediate write
432  // Use rename() here since (a) this clears xattrs, (b) any threads still reading the
433  // old inode are unaffected since it writes to a new inode, and (c) this is fast and
434  // atomic within a file system volume (as is normally the case)
436  $moved = rename( $fsSrcPath, $fsDstPath );
437  $hadError = $this->untrapWarnings();
438  if ( $hadError || ( !$moved && !$ignoreMissing ) ) {
439  $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
440 
441  return $status;
442  }
443  }
444 
445  return $status;
446  }
447 
448  protected function doDeleteInternal( array $params ) {
449  $status = $this->newStatus();
450 
451  $fsSrcPath = $this->resolveToFSPath( $params['src'] );
452  if ( $fsSrcPath === null ) {
453  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
454 
455  return $status;
456  }
457 
458  $ignoreMissing = !empty( $params['ignoreMissingSource'] );
459 
460  if ( !empty( $params['async'] ) ) { // deferred
461  $cmd = $this->makeUnlinkCommand( $fsSrcPath, $ignoreMissing );
462  $handler = function ( $errors, StatusValue $status, array $params, $cmd ) {
463  if ( $errors !== '' && !( $this->os === 'Windows' && $errors[0] === " " ) ) {
464  $status->fatal( 'backend-fail-delete', $params['src'] );
465  trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
466  }
467  };
468  $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd );
469  } else { // immediate write
471  $deleted = unlink( $fsSrcPath );
472  $hadError = $this->untrapWarnings();
473  if ( $hadError || ( !$deleted && !$ignoreMissing ) ) {
474  $status->fatal( 'backend-fail-delete', $params['src'] );
475 
476  return $status;
477  }
478  }
479 
480  return $status;
481  }
482 
489  protected function doPrepareInternal( $fullCont, $dirRel, array $params ) {
490  $status = $this->newStatus();
491  list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
492  $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
493  $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
494  // Create the directory and its parents as needed...
495  $created = false;
496  AtEase::suppressWarnings();
497  $alreadyExisted = is_dir( $fsDirectory ); // already there?
498  if ( !$alreadyExisted ) {
499  $created = mkdir( $fsDirectory, $this->dirMode, true );
500  if ( !$created ) {
501  $alreadyExisted = is_dir( $fsDirectory ); // another thread made it?
502  }
503  }
504  $isWritable = $created ?: is_writable( $fsDirectory ); // assume writable if created here
505  AtEase::restoreWarnings();
506  if ( !$alreadyExisted && !$created ) {
507  $this->logger->error( __METHOD__ . ": cannot create directory $fsDirectory" );
508  $status->fatal( 'directorycreateerror', $params['dir'] ); // fails on races
509  } elseif ( !$isWritable ) {
510  $this->logger->error( __METHOD__ . ": directory $fsDirectory is read-only" );
511  $status->fatal( 'directoryreadonlyerror', $params['dir'] );
512  }
513  // Respect any 'noAccess' or 'noListing' flags...
514  if ( $created ) {
515  $status->merge( $this->doSecureInternal( $fullCont, $dirRel, $params ) );
516  }
517 
518  if ( $status->isOK() ) {
519  $this->usableDirCache->set( $fsDirectory, 1 );
520  }
521 
522  return $status;
523  }
524 
525  protected function doSecureInternal( $fullCont, $dirRel, array $params ) {
526  $status = $this->newStatus();
527  list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
528  $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
529  $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
530  // Seed new directories with a blank index.html, to prevent crawling...
531  if ( !empty( $params['noListing'] ) && !is_file( "{$fsDirectory}/index.html" ) ) {
532  $this->trapWarnings();
533  $bytes = file_put_contents( "{$fsDirectory}/index.html", $this->indexHtmlPrivate() );
534  $this->untrapWarnings();
535  if ( $bytes === false ) {
536  $status->fatal( 'backend-fail-create', $params['dir'] . '/index.html' );
537  }
538  }
539  // Add a .htaccess file to the root of the container...
540  if ( !empty( $params['noAccess'] ) && !is_file( "{$contRoot}/.htaccess" ) ) {
541  AtEase::suppressWarnings();
542  $bytes = file_put_contents( "{$contRoot}/.htaccess", $this->htaccessPrivate() );
543  AtEase::restoreWarnings();
544  if ( $bytes === false ) {
545  $storeDir = "mwstore://{$this->name}/{$shortCont}";
546  $status->fatal( 'backend-fail-create', "{$storeDir}/.htaccess" );
547  }
548  }
549 
550  return $status;
551  }
552 
553  protected function doPublishInternal( $fullCont, $dirRel, array $params ) {
554  $status = $this->newStatus();
555  list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
556  $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
557  $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
558  // Unseed new directories with a blank index.html, to allow crawling...
559  if ( !empty( $params['listing'] ) && is_file( "{$fsDirectory}/index.html" ) ) {
560  $exists = ( file_get_contents( "{$fsDirectory}/index.html" ) === $this->indexHtmlPrivate() );
561  if ( $exists && !$this->unlink( "{$fsDirectory}/index.html" ) ) { // reverse secure()
562  $status->fatal( 'backend-fail-delete', $params['dir'] . '/index.html' );
563  }
564  }
565  // Remove the .htaccess file from the root of the container...
566  if ( !empty( $params['access'] ) && is_file( "{$contRoot}/.htaccess" ) ) {
567  $exists = ( file_get_contents( "{$contRoot}/.htaccess" ) === $this->htaccessPrivate() );
568  if ( $exists && !$this->unlink( "{$contRoot}/.htaccess" ) ) { // reverse secure()
569  $storeDir = "mwstore://{$this->name}/{$shortCont}";
570  $status->fatal( 'backend-fail-delete', "{$storeDir}/.htaccess" );
571  }
572  }
573 
574  return $status;
575  }
576 
577  protected function doCleanInternal( $fullCont, $dirRel, array $params ) {
578  $status = $this->newStatus();
579  list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
580  $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
581  $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
582 
583  $this->rmdir( $fsDirectory );
584 
585  return $status;
586  }
587 
588  protected function doGetFileStat( array $params ) {
589  $fsSrcPath = $this->resolveToFSPath( $params['src'] );
590  if ( $fsSrcPath === null ) {
591  return self::$RES_ERROR; // invalid storage path
592  }
593 
594  $this->trapWarnings(); // don't trust 'false' if there were errors
595  $stat = is_file( $fsSrcPath ) ? stat( $fsSrcPath ) : false; // regular files only
596  $hadError = $this->untrapWarnings();
597 
598  if ( is_array( $stat ) ) {
599  $ct = new ConvertibleTimestamp( $stat['mtime'] );
600 
601  return [
602  'mtime' => $ct->getTimestamp( TS_MW ),
603  'size' => $stat['size']
604  ];
605  }
606 
607  return $hadError ? self::$RES_ERROR : self::$RES_ABSENT;
608  }
609 
610  protected function doClearCache( array $paths = null ) {
611  if ( is_array( $paths ) ) {
612  foreach ( $paths as $path ) {
613  $fsPath = $this->resolveToFSPath( $path );
614  if ( $fsPath !== null ) {
615  clearstatcache( true, $fsPath );
616  $this->usableDirCache->clear( $fsPath );
617  }
618  }
619  } else {
620  clearstatcache( true ); // clear the PHP file stat cache
621  $this->usableDirCache->clear();
622  }
623  }
624 
625  protected function doDirectoryExists( $fullCont, $dirRel, array $params ) {
626  list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
627  $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
628  $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
629 
630  $this->trapWarnings(); // don't trust 'false' if there were errors
631  $exists = is_dir( $fsDirectory );
632  $hadError = $this->untrapWarnings();
633 
634  return $hadError ? self::$RES_ERROR : $exists;
635  }
636 
644  public function getDirectoryListInternal( $fullCont, $dirRel, array $params ) {
645  list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
646  $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
647  $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
648 
649  $list = new FSFileBackendDirList( $fsDirectory, $params );
650  $error = $list->getLastError();
651  if ( $error !== null ) {
652  if ( $this->isFileNotFoundError( $error ) ) {
653  $this->logger->info( __METHOD__ . ": non-existant directory: '$fsDirectory'" );
654 
655  return []; // nothing under this dir
656  } elseif ( is_dir( $fsDirectory ) ) {
657  $this->logger->warning( __METHOD__ . ": unreadable directory: '$fsDirectory'" );
658 
659  return self::$RES_ERROR; // bad permissions?
660  } else {
661  $this->logger->warning( __METHOD__ . ": unreachable directory: '$fsDirectory'" );
662 
663  return self::$RES_ERROR;
664  }
665  }
666 
667  return $list;
668  }
669 
677  public function getFileListInternal( $fullCont, $dirRel, array $params ) {
678  list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
679  $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
680  $fsDirectory = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
681 
682  $list = new FSFileBackendFileList( $fsDirectory, $params );
683  $error = $list->getLastError();
684  if ( $error !== null ) {
685  if ( $this->isFileNotFoundError( $error ) ) {
686  $this->logger->info( __METHOD__ . ": non-existent directory: '$fsDirectory'" );
687 
688  return []; // nothing under this dir
689  } elseif ( is_dir( $fsDirectory ) ) {
690  $this->logger->warning( __METHOD__ .
691  ": unreadable directory: '$fsDirectory': $error" );
692 
693  return self::$RES_ERROR; // bad permissions?
694  } else {
695  $this->logger->warning( __METHOD__ .
696  ": unreachable directory: '$fsDirectory': $error" );
697 
698  return self::$RES_ERROR;
699  }
700  }
701 
702  return $list;
703  }
704 
705  protected function doGetLocalReferenceMulti( array $params ) {
706  $fsFiles = []; // (path => FSFile)
707 
708  foreach ( $params['srcs'] as $src ) {
709  $source = $this->resolveToFSPath( $src );
710  if ( $source === null ) {
711  $fsFiles[$src] = self::$RES_ERROR; // invalid path
712  continue;
713  }
714 
715  $this->trapWarnings(); // don't trust 'false' if there were errors
716  $isFile = is_file( $source ); // regular files only
717  $hadError = $this->untrapWarnings();
718 
719  if ( $isFile ) {
720  $fsFiles[$src] = new FSFile( $source );
721  } elseif ( $hadError ) {
722  $fsFiles[$src] = self::$RES_ERROR;
723  } else {
724  $fsFiles[$src] = self::$RES_ABSENT;
725  }
726  }
727 
728  return $fsFiles;
729  }
730 
731  protected function doGetLocalCopyMulti( array $params ) {
732  $tmpFiles = []; // (path => TempFSFile)
733 
734  foreach ( $params['srcs'] as $src ) {
735  $source = $this->resolveToFSPath( $src );
736  if ( $source === null ) {
737  $tmpFiles[$src] = self::$RES_ERROR; // invalid path
738  continue;
739  }
740  // Create a new temporary file with the same extension...
742  $tmpFile = $this->tmpFileFactory->newTempFSFile( 'localcopy_', $ext );
743  if ( !$tmpFile ) {
744  $tmpFiles[$src] = self::$RES_ERROR;
745  continue;
746  }
747 
748  $tmpPath = $tmpFile->getPath();
749  // Copy the source file over the temp file
750  $this->trapWarnings(); // don't trust 'false' if there were errors
751  $isFile = is_file( $source ); // regular files only
752  $copySuccess = $isFile ? copy( $source, $tmpPath ) : false;
753  $hadError = $this->untrapWarnings();
754 
755  if ( $copySuccess ) {
756  $this->chmod( $tmpPath );
757  $tmpFiles[$src] = $tmpFile;
758  } elseif ( $hadError ) {
759  $tmpFiles[$src] = self::$RES_ERROR; // copy failed
760  } else {
761  $tmpFiles[$src] = self::$RES_ABSENT;
762  }
763  }
764 
765  return $tmpFiles;
766  }
767 
768  protected function directoriesAreVirtual() {
769  return false;
770  }
771 
777  protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
778  $statuses = [];
779 
780  $pipes = [];
781  foreach ( $fileOpHandles as $index => $fileOpHandle ) {
782  $pipes[$index] = popen( $fileOpHandle->cmd, 'r' );
783  }
784 
785  $errs = [];
786  foreach ( $pipes as $index => $pipe ) {
787  // Result will be empty on success in *NIX. On Windows,
788  // it may be something like " 1 file(s) [copied|moved].".
789  $errs[$index] = stream_get_contents( $pipe );
790  fclose( $pipe );
791  }
792 
793  foreach ( $fileOpHandles as $index => $fileOpHandle ) {
794  $status = $this->newStatus();
795  $function = $fileOpHandle->callback;
796  $function( $errs[$index], $status, $fileOpHandle->params, $fileOpHandle->cmd );
797  $statuses[$index] = $status;
798  }
799 
800  return $statuses;
801  }
802 
807  private function makeStagingPath( $fsPath ) {
808  $time = dechex( time() ); // make it easy to find old orphans
809  $hash = \Wikimedia\base_convert( md5( basename( $fsPath ) ), 16, 36, 25 );
810  $unique = \Wikimedia\base_convert( bin2hex( random_bytes( 16 ) ), 16, 36, 25 );
811 
812  return dirname( $fsPath ) . "/.{$time}_{$hash}_{$unique}.tmpfsfile";
813  }
814 
821  private function makeCopyCommand( $fsSrcPath, $fsDstPath, $ignoreMissing ) {
822  // Use copy+rename since (a) this clears xattrs, (b) threads still reading the old
823  // inode are unaffected since it writes to a new inode, and (c) new threads reading
824  // the file will either totally see the old version or totally see the new version
825  $fsStagePath = $this->makeStagingPath( $fsDstPath );
826  $encSrc = Shellbox::escape( $this->cleanPathSlashes( $fsSrcPath ) );
827  $encStage = Shellbox::escape( $this->cleanPathSlashes( $fsStagePath ) );
828  $encDst = Shellbox::escape( $this->cleanPathSlashes( $fsDstPath ) );
829  if ( $this->os === 'Windows' ) {
830  // https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/copy
831  // https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/move
832  $cmdWrite = "COPY /B /Y $encSrc $encStage 2>&1 && MOVE /Y $encStage $encDst 2>&1";
833  $cmd = $ignoreMissing ? "IF EXIST $encSrc $cmdWrite" : $cmdWrite;
834  } else {
835  // https://manpages.debian.org/buster/coreutils/cp.1.en.html
836  // https://manpages.debian.org/buster/coreutils/mv.1.en.html
837  $cmdWrite = "cp $encSrc $encStage 2>&1 && mv $encStage $encDst 2>&1";
838  $cmd = $ignoreMissing ? "test -f $encSrc && $cmdWrite" : $cmdWrite;
839  // Clean up permissions on any newly created destination file
840  $octalPermissions = '0' . decoct( $this->fileMode );
841  if ( strlen( $octalPermissions ) == 4 ) {
842  $cmd .= " && chmod $octalPermissions $encDst 2>/dev/null";
843  }
844  }
845 
846  return $cmd;
847  }
848 
855  private function makeMoveCommand( $fsSrcPath, $fsDstPath, $ignoreMissing = false ) {
856  // https://manpages.debian.org/buster/coreutils/mv.1.en.html
857  // https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/move
858  $encSrc = Shellbox::escape( $this->cleanPathSlashes( $fsSrcPath ) );
859  $encDst = Shellbox::escape( $this->cleanPathSlashes( $fsDstPath ) );
860  if ( $this->os === 'Windows' ) {
861  $writeCmd = "MOVE /Y $encSrc $encDst 2>&1";
862  $cmd = $ignoreMissing ? "IF EXIST $encSrc $writeCmd" : $writeCmd;
863  } else {
864  $writeCmd = "mv -f $encSrc $encDst 2>&1";
865  $cmd = $ignoreMissing ? "test -f $encSrc && $writeCmd" : $writeCmd;
866  }
867 
868  return $cmd;
869  }
870 
876  private function makeUnlinkCommand( $fsPath, $ignoreMissing = false ) {
877  // https://manpages.debian.org/buster/coreutils/rm.1.en.html
878  // https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/del
879  $encSrc = Shellbox::escape( $this->cleanPathSlashes( $fsPath ) );
880  if ( $this->os === 'Windows' ) {
881  $writeCmd = "DEL /Q $encSrc 2>&1";
882  $cmd = $ignoreMissing ? "IF EXIST $encSrc $writeCmd" : $writeCmd;
883  } else {
884  $cmd = $ignoreMissing ? "rm -f $encSrc 2>&1" : "rm $encSrc 2>&1";
885  }
886 
887  return $cmd;
888  }
889 
896  protected function chmod( $fsPath ) {
897  if ( $this->os === 'Windows' ) {
898  return true;
899  }
900 
901  AtEase::suppressWarnings();
902  $ok = chmod( $fsPath, $this->fileMode );
903  AtEase::restoreWarnings();
904 
905  return $ok;
906  }
907 
914  protected function unlink( $fsPath ) {
915  AtEase::suppressWarnings();
916  $ok = unlink( $fsPath );
917  AtEase::restoreWarnings();
918  clearstatcache( true, $fsPath );
919 
920  return $ok;
921  }
922 
929  protected function rmdir( $fsDirectory ) {
930  AtEase::suppressWarnings();
931  $ok = rmdir( $fsDirectory ); // remove directory if empty
932  AtEase::restoreWarnings();
933  clearstatcache( true, $fsDirectory );
934 
935  return $ok;
936  }
937 
942  protected function newTempFileWithContent( array $params ) {
943  $tempFile = $this->tmpFileFactory->newTempFSFile( 'create_', 'tmp' );
944  if ( !$tempFile ) {
945  return null;
946  }
947 
948  AtEase::suppressWarnings();
949  if ( file_put_contents( $tempFile->getPath(), $params['content'] ) === false ) {
950  $tempFile = null;
951  }
952  AtEase::restoreWarnings();
953 
954  return $tempFile;
955  }
956 
962  protected function indexHtmlPrivate() {
963  return '';
964  }
965 
971  protected function htaccessPrivate() {
972  return "Deny from all\n";
973  }
974 
981  protected function cleanPathSlashes( $fsPath ) {
982  return ( $this->os === 'Windows' ) ? strtr( $fsPath, '/', '\\' ) : $fsPath;
983  }
984 
990  protected function trapWarnings( $regexIgnore = null ) {
991  $this->warningTrapStack[] = false;
992  set_error_handler( function ( $errno, $errstr ) use ( $regexIgnore ) {
993  if ( $regexIgnore === null || !preg_match( $regexIgnore, $errstr ) ) {
994  $this->logger->error( $errstr );
995  $this->warningTrapStack[count( $this->warningTrapStack ) - 1] = true;
996  }
997  return true; // suppress from PHP handler
998  }, E_WARNING );
999  }
1000 
1004  protected function trapWarningsIgnoringNotFound() {
1005  $this->trapWarnings( $this->getFileNotFoundRegex() );
1006  }
1007 
1013  protected function untrapWarnings() {
1014  restore_error_handler();
1015 
1016  return array_pop( $this->warningTrapStack );
1017  }
1018 
1024  protected function getFileNotFoundRegex() {
1025  static $regex;
1026  if ( $regex === null ) {
1027  // "No such file or directory": string literal in spl_directory.c etc.
1028  $alternatives = [ ': No such file or directory' ];
1029  if ( $this->os === 'Windows' ) {
1030  // 2 = The system cannot find the file specified.
1031  // 3 = The system cannot find the path specified.
1032  $alternatives[] = ' \‍(code: [23]\‍)';
1033  }
1034  if ( function_exists( 'pcntl_strerror' ) ) {
1035  $alternatives[] = preg_quote( ': ' . pcntl_strerror( 2 ), '/' );
1036  } elseif ( function_exists( 'socket_strerror' ) && defined( 'SOCKET_ENOENT' ) ) {
1037  $alternatives[] = preg_quote( ': ' . socket_strerror( SOCKET_ENOENT ), '/' );
1038  }
1039  $regex = '/(' . implode( '|', $alternatives ) . ')$/';
1040  }
1041  return $regex;
1042  }
1043 
1050  protected function isFileNotFoundError( $error ) {
1051  return (bool)preg_match( $this->getFileNotFoundRegex(), $error );
1052  }
1053 }
FileBackend\splitStoragePath
static splitStoragePath( $storagePath)
Split a storage path into a backend name, a container name, and a relative file path.
Definition: FileBackend.php:1537
FSFileBackend\__construct
__construct(array $config)
Definition: FSFileBackend.php:99
FSFileBackend\doCleanInternal
doCleanInternal( $fullCont, $dirRel, array $params)
Definition: FSFileBackend.php:577
FSFileOpHandle
Definition: FSFileOpHandle.php:22
StatusValue
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Definition: StatusValue.php:43
FSFileBackend\doStoreInternal
doStoreInternal(array $params)
Definition: FSFileBackend.php:279
FSFileBackend\trapWarningsIgnoringNotFound
trapWarningsIgnoringNotFound()
Track E_WARNING errors but ignore any that correspond to ENOENT "No such file or directory".
Definition: FSFileBackend.php:1004
FSFileBackend\doClearCache
doClearCache(array $paths=null)
Clears any additional stat caches for storage paths.
Definition: FSFileBackend.php:610
FSFileBackend\getFileNotFoundRegex
getFileNotFoundRegex()
Get a regex matching file not found errors.
Definition: FSFileBackend.php:1024
FSFileBackend\doPublishInternal
doPublishInternal( $fullCont, $dirRel, array $params)
Definition: FSFileBackend.php:553
FSFileBackend\$currentUser
string $currentUser
OS username running this script.
Definition: FSFileBackend.php:84
FSFileBackend\doPrepareInternal
doPrepareInternal( $fullCont, $dirRel, array $params)
Definition: FSFileBackend.php:489
FSFileBackend\makeUnlinkCommand
makeUnlinkCommand( $fsPath, $ignoreMissing=false)
Definition: FSFileBackend.php:876
FileBackend\extensionFromPath
static extensionFromPath( $path, $case='lowercase')
Get the final extension from a storage or FS path.
Definition: FileBackend.php:1599
FSFileBackend\$dirMode
int $dirMode
Directory permission mode.
Definition: FSFileBackend.php:75
FSFileBackend\htaccessPrivate
htaccessPrivate()
Return the text of a .htaccess file to make a directory private.
Definition: FSFileBackend.php:971
FSFileBackend\doGetFileStat
doGetFileStat(array $params)
Definition: FSFileBackend.php:588
FSFileBackend\directoriesAreVirtual
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
Definition: FSFileBackend.php:768
FSFileBackend\$fileOwner
string $fileOwner
Required OS username to own files.
Definition: FSFileBackend.php:79
FSFileBackend\makeStagingPath
makeStagingPath( $fsPath)
Definition: FSFileBackend.php:807
FSFileBackend\cleanPathSlashes
cleanPathSlashes( $fsPath)
Clean up directory separators for the given OS.
Definition: FSFileBackend.php:981
FileBackendStore\$RES_ERROR
static null $RES_ERROR
Idiom for "no result due to I/O errors" (since 1.34)
Definition: FileBackendStore.php:66
FSFileBackend\doGetLocalReferenceMulti
doGetLocalReferenceMulti(array $params)
Definition: FSFileBackend.php:705
FSFileBackend\$basePath
string $basePath
Directory holding the container directories.
Definition: FSFileBackend.php:69
FSFileBackend\makeMoveCommand
makeMoveCommand( $fsSrcPath, $fsDstPath, $ignoreMissing=false)
Definition: FSFileBackend.php:855
FSFileBackend\doDirectoryExists
doDirectoryExists( $fullCont, $dirRel, array $params)
Definition: FSFileBackend.php:625
FSFileBackend\isFileNotFoundError
isFileNotFoundError( $error)
Determine whether a given error message is a file not found error.
Definition: FSFileBackend.php:1050
FSFileBackend\getDirectoryListInternal
getDirectoryListInternal( $fullCont, $dirRel, array $params)
Definition: FSFileBackend.php:644
FSFileBackend\getFeatures
getFeatures()
Get the a bitfield of extra features supported by the backend medium.
Definition: FSFileBackend.php:132
FSFileBackendFileList
Definition: FSFileBackendFileList.php:22
FSFileBackend\containerFSRoot
containerFSRoot( $shortCont, $fullCont)
Given the short (unresolved) and full (resolved) name of a container, return the file system path of ...
Definition: FSFileBackend.php:174
FSFileBackendDirList
Definition: FSFileBackendDirList.php:22
FSFileBackend\rmdir
rmdir( $fsDirectory)
Remove an empty directory, suppressing the warnings.
Definition: FSFileBackend.php:929
FileBackend\ATTR_UNICODE_PATHS
const ATTR_UNICODE_PATHS
Definition: FileBackend.php:136
MapCacheLRU
Handles a simple LRU key/value map with a maximum number of entries.
Definition: MapCacheLRU.php:36
FSFileBackend\$warningTrapStack
bool[] $warningTrapStack
Map of (stack index => whether a warning happened)
Definition: FSFileBackend.php:87
FSFileBackend\doSecureInternal
doSecureInternal( $fullCont, $dirRel, array $params)
Definition: FSFileBackend.php:525
FSFileBackend\makeCopyCommand
makeCopyCommand( $fsSrcPath, $fsDstPath, $ignoreMissing)
Definition: FSFileBackend.php:821
FSFileBackend\doExecuteOpHandlesInternal
doExecuteOpHandlesInternal(array $fileOpHandles)
Definition: FSFileBackend.php:777
FSFileBackend\$containerPaths
array $containerPaths
Map of container names to root paths for custom container paths.
Definition: FSFileBackend.php:72
FSFileBackend\resolveToFSPath
resolveToFSPath( $storagePath)
Get the absolute file system path for a storage path.
Definition: FSFileBackend.php:190
FSFileBackend\doDeleteInternal
doDeleteInternal(array $params)
Definition: FSFileBackend.php:448
FSFileBackend\$fileMode
int $fileMode
File permission mode.
Definition: FSFileBackend.php:77
FSFileBackend\resolveContainerPath
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
Definition: FSFileBackend.php:136
FileBackendStore\resolveStoragePathReal
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
Definition: FileBackendStore.php:1642
FileBackendStore
Base class for all backends using particular storage medium.
Definition: FileBackendStore.php:41
FileBackendStore\$RES_ABSENT
static false $RES_ABSENT
Idiom for "no result due to missing file" (since 1.34)
Definition: FileBackendStore.php:64
FSFileBackend\unlink
unlink( $fsPath)
Unlink a file, suppressing the warnings.
Definition: FSFileBackend.php:914
FSFile
Class representing a non-directory file on the file system.
Definition: FSFile.php:32
FileBackend\newStatus
newStatus(... $args)
Yields the result of the status wrapper callback on either:
Definition: FileBackend.php:1692
FSFileBackend
Class for a file system (FS) based file backend.
Definition: FSFileBackend.php:64
FSFileBackend\indexHtmlPrivate
indexHtmlPrivate()
Return the text of an index.html file to hide directory listings.
Definition: FSFileBackend.php:962
$path
$path
Definition: NoLocalSettings.php:25
FSFileBackend\doCreateInternal
doCreateInternal(array $params)
Definition: FSFileBackend.php:227
FSFileBackend\$usableDirCache
MapCacheLRU $usableDirCache
Cache for known prepared/usable directorries.
Definition: FSFileBackend.php:66
$source
$source
Definition: mwdoc-filter.php:34
FSFileBackend\isPathUsableInternal
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path in the backend.
Definition: FSFileBackend.php:204
FSFileBackend\$os
bool $os
Simpler version of PHP_OS_FAMILY.
Definition: FSFileBackend.php:82
$ext
if(!is_readable( $file)) $ext
Definition: router.php:48
FileBackend\copy
copy(array $params, array $opts=[])
Performs a single copy operation.
Definition: FileBackend.php:547
FSFileBackend\chmod
chmod( $fsPath)
Chmod a file, suppressing the warnings.
Definition: FSFileBackend.php:896
FSFileBackend\untrapWarnings
untrapWarnings()
Stop listening for E_WARNING errors and get whether any happened.
Definition: FSFileBackend.php:1013
FSFileBackend\doGetLocalCopyMulti
doGetLocalCopyMulti(array $params)
Definition: FSFileBackend.php:731
FSFileBackend\newTempFileWithContent
newTempFileWithContent(array $params)
Definition: FSFileBackend.php:942
FSFileBackend\doCopyInternal
doCopyInternal(array $params)
Definition: FSFileBackend.php:335
FSFileBackend\trapWarnings
trapWarnings( $regexIgnore=null)
Listen for E_WARNING errors and track whether any that happen.
Definition: FSFileBackend.php:990
FSFileBackend\isLegalRelPath
isLegalRelPath( $fsPath)
Sanity check a relative file system path for validity.
Definition: FSFileBackend.php:154
FSFileBackend\getFileListInternal
getFileListInternal( $fullCont, $dirRel, array $params)
Definition: FSFileBackend.php:677
FSFileBackend\doMoveInternal
doMoveInternal(array $params)
Definition: FSFileBackend.php:399