MediaWiki  master
FileOp.php
Go to the documentation of this file.
1 <?php
23 use Psr\Log\LoggerInterface;
24 
36 abstract class FileOp {
38  protected $backend;
40  protected $logger;
41 
43  protected $params = [];
44 
46  protected $state = self::STATE_NEW;
48  protected $failed = false;
50  protected $async = false;
52  protected $batchId;
54  protected $cancelled = false;
55 
57  protected $sourceSize;
59  protected $sourceSha1;
60 
62  protected $overwriteSameCase;
63 
65  protected $destExists;
66 
67  /* Object life-cycle */
68  private const STATE_NEW = 1;
69  private const STATE_CHECKED = 2;
70  private const STATE_ATTEMPTED = 3;
71 
72  protected const ASSUMED_SHA1 = 'sha1';
73  protected const ASSUMED_EXISTS = 'exists';
74  protected const ASSUMED_SIZE = 'size';
75 
84  final public function __construct(
85  FileBackendStore $backend, array $params, LoggerInterface $logger
86  ) {
87  $this->backend = $backend;
88  $this->logger = $logger;
89  list( $required, $optional, $paths ) = $this->allowedParams();
90  foreach ( $required as $name ) {
91  if ( isset( $params[$name] ) ) {
92  $this->params[$name] = $params[$name];
93  } else {
94  throw new InvalidArgumentException( "File operation missing parameter '$name'." );
95  }
96  }
97  foreach ( $optional as $name ) {
98  if ( isset( $params[$name] ) ) {
99  $this->params[$name] = $params[$name];
100  }
101  }
102  foreach ( $paths as $name ) {
103  if ( isset( $this->params[$name] ) ) {
104  // Normalize paths so the paths to the same file have the same string
105  $this->params[$name] = self::normalizeIfValidStoragePath( $this->params[$name] );
106  }
107  }
108  }
109 
116  protected static function normalizeIfValidStoragePath( $path ) {
119 
120  return $res ?? $path;
121  }
122 
123  return $path;
124  }
125 
131  final public function setBatchId( $batchId ) {
132  $this->batchId = $batchId;
133  }
134 
141  final public function getParam( $name ) {
142  return $this->params[$name] ?? null;
143  }
144 
150  final public function failed() {
151  return $this->failed;
152  }
153 
159  final public static function newPredicates() {
160  return [ self::ASSUMED_EXISTS => [], self::ASSUMED_SHA1 => [], self::ASSUMED_SIZE => [] ];
161  }
162 
168  final public static function newDependencies() {
169  return [ 'read' => [], 'write' => [] ];
170  }
171 
178  final public function applyDependencies( array $deps ) {
179  $deps['read'] += array_fill_keys( $this->storagePathsRead(), 1 );
180  $deps['write'] += array_fill_keys( $this->storagePathsChanged(), 1 );
181 
182  return $deps;
183  }
184 
191  final public function dependsOn( array $deps ) {
192  foreach ( $this->storagePathsChanged() as $path ) {
193  if ( isset( $deps['read'][$path] ) || isset( $deps['write'][$path] ) ) {
194  return true; // "output" or "anti" dependency
195  }
196  }
197  foreach ( $this->storagePathsRead() as $path ) {
198  if ( isset( $deps['write'][$path] ) ) {
199  return true; // "flow" dependency
200  }
201  }
202 
203  return false;
204  }
205 
213  final public function getJournalEntries( array $oPredicates, array $nPredicates ) {
214  if ( $this->cancelled ) {
215  return []; // this is a no-op
216  }
217  $nullEntries = [];
218  $updateEntries = [];
219  $deleteEntries = [];
220  foreach ( $this->storagePathsReadOrChanged() as $path ) {
221  $nullEntries[] = [ // assertion for recovery
222  'op' => 'null',
223  'path' => $path,
224  'newSha1' => $this->fileSha1( $path, $oPredicates )
225  ];
226  }
227  foreach ( $this->storagePathsChanged() as $path ) {
228  if ( $nPredicates[self::ASSUMED_SHA1][$path] === false ) { // deleted
229  $deleteEntries[] = [
230  'op' => 'delete',
231  'path' => $path,
232  'newSha1' => ''
233  ];
234  } else { // created/updated
235  $updateEntries[] = [
236  'op' => $this->fileExists( $path, $oPredicates ) ? 'update' : 'create',
237  'path' => $path,
238  'newSha1' => $nPredicates[self::ASSUMED_SHA1][$path]
239  ];
240  }
241  }
242 
243  return array_merge( $nullEntries, $updateEntries, $deleteEntries );
244  }
245 
254  final public function precheck( array &$predicates ) {
255  if ( $this->state !== self::STATE_NEW ) {
256  return StatusValue::newFatal( 'fileop-fail-state', self::STATE_NEW, $this->state );
257  }
258  $this->state = self::STATE_CHECKED;
259 
260  $status = StatusValue::newGood();
261  foreach ( $this->storagePathsReadOrChanged() as $path ) {
262  if ( !$this->backend->isPathUsableInternal( $path ) ) {
263  $status->fatal( 'backend-fail-usable', $path );
264  }
265  }
266  if ( !$status->isOK() ) {
267  return $status;
268  }
269 
270  $status = $this->doPrecheck( $predicates );
271  if ( !$status->isOK() ) {
272  $this->failed = true;
273  }
274 
275  return $status;
276  }
277 
282  protected function doPrecheck( array &$predicates ) {
283  return StatusValue::newGood();
284  }
285 
291  final public function attempt() {
292  if ( $this->state !== self::STATE_CHECKED ) {
293  return StatusValue::newFatal( 'fileop-fail-state', self::STATE_CHECKED, $this->state );
294  } elseif ( $this->failed ) { // failed precheck
295  return StatusValue::newFatal( 'fileop-fail-attempt-precheck' );
296  }
297  $this->state = self::STATE_ATTEMPTED;
298  if ( $this->cancelled ) {
299  $status = StatusValue::newGood(); // no-op
300  } else {
301  $status = $this->doAttempt();
302  if ( !$status->isOK() ) {
303  $this->failed = true;
304  $this->logFailure( 'attempt' );
305  }
306  }
307 
308  return $status;
309  }
310 
314  protected function doAttempt() {
315  return StatusValue::newGood();
316  }
317 
323  final public function attemptAsync() {
324  $this->async = true;
325  $result = $this->attempt();
326  $this->async = false;
327 
328  return $result;
329  }
330 
336  final public function attemptQuick() {
337  $this->state = self::STATE_CHECKED; // bypassed
338 
339  return $this->attempt();
340  }
341 
347  final public function attemptAsyncQuick() {
348  $this->state = self::STATE_CHECKED; // bypassed
349 
350  return $this->attemptAsync();
351  }
352 
358  protected function allowedParams() {
359  return [ [], [], [] ];
360  }
361 
368  protected function setFlags( array $params ) {
369  return [ 'async' => $this->async ] + $params;
370  }
371 
377  public function storagePathsRead() {
378  return [];
379  }
380 
386  public function storagePathsChanged() {
387  return [];
388  }
389 
395  final public function storagePathsReadOrChanged() {
396  return array_values( array_unique(
397  array_merge( $this->storagePathsRead(), $this->storagePathsChanged() )
398  ) );
399  }
400 
409  protected function precheckDestExistence( array $predicates ) {
410  $status = StatusValue::newGood();
411  // Record the size of source file/string
412  $this->sourceSize = $this->getSourceSize(); // FS file or data string
413  if ( $this->sourceSize === null ) { // file in storage?
414  $this->sourceSize = $this->fileSize( $this->params['src'], $predicates );
415  }
416  // Record the hash of source file/string
417  $this->sourceSha1 = $this->getSourceSha1Base36(); // FS file or data string
418  if ( $this->sourceSha1 === null ) { // file in storage?
419  $this->sourceSha1 = $this->fileSha1( $this->params['src'], $predicates );
420  }
421  // Record the existence of destination file
422  $this->destExists = $this->fileExists( $this->params['dst'], $predicates );
423  // Check if an incompatible file exists at the destination
424  $this->overwriteSameCase = false;
425  if ( $this->destExists ) {
426  if ( $this->getParam( 'overwrite' ) ) {
427  return $status; // OK, no conflict
428  } elseif ( $this->getParam( 'overwriteSame' ) ) {
429  // Operation does nothing other than return an OK or bad status
430  $dhash = $this->fileSha1( $this->params['dst'], $predicates );
431  $dsize = $this->fileSize( $this->params['dst'], $predicates );
432  // Check if hashes are valid and match each other...
433  if ( !strlen( $this->sourceSha1 ) || !strlen( $dhash ) ) {
434  $status->fatal( 'backend-fail-hashes' );
435  } elseif ( !is_int( $this->sourceSize ) || !is_int( $dsize ) ) {
436  $status->fatal( 'backend-fail-sizes' );
437  } elseif ( $this->sourceSha1 !== $dhash || $this->sourceSize !== $dsize ) {
438  // Give an error if the files are not identical
439  $status->fatal( 'backend-fail-notsame', $this->params['dst'] );
440  } else {
441  $this->overwriteSameCase = true; // OK
442  }
443  } else {
444  $status->fatal( 'backend-fail-alreadyexists', $this->params['dst'] );
445  }
446  } elseif ( $this->destExists === FileBackend::EXISTENCE_ERROR ) {
447  $status->fatal( 'backend-fail-stat', $this->params['dst'] );
448  }
449 
450  return $status;
451  }
452 
459  protected function getSourceSize() {
460  return null; // N/A
461  }
462 
469  protected function getSourceSha1Base36() {
470  return null; // N/A
471  }
472 
483  final protected function fileExists( $source, array $predicates ) {
484  if ( isset( $predicates[self::ASSUMED_EXISTS][$source] ) ) {
485  return $predicates[self::ASSUMED_EXISTS][$source]; // previous op assures this
486  } else {
487  $params = [ 'src' => $source, 'latest' => true ];
488 
489  return $this->backend->fileExists( $params );
490  }
491  }
492 
504  final protected function fileSize( $source, array $predicates ) {
505  if ( isset( $predicates[self::ASSUMED_SIZE][$source] ) ) {
506  return $predicates[self::ASSUMED_SIZE][$source]; // previous op assures this
507  } elseif (
508  isset( $predicates[self::ASSUMED_EXISTS][$source] ) &&
509  !$predicates[self::ASSUMED_EXISTS][$source]
510  ) {
511  return false; // previous op assures this
512  } else {
513  $params = [ 'src' => $source, 'latest' => true ];
514 
515  return $this->backend->getFileSize( $params );
516  }
517  }
518 
526  final protected function fileSha1( $source, array $predicates ) {
527  if ( isset( $predicates[self::ASSUMED_SHA1][$source] ) ) {
528  return $predicates[self::ASSUMED_SHA1][$source]; // previous op assures this
529  } elseif (
530  isset( $predicates[self::ASSUMED_EXISTS][$source] ) &&
531  !$predicates[self::ASSUMED_EXISTS][$source]
532  ) {
533  return false; // previous op assures this
534  } else {
535  $params = [ 'src' => $source, 'latest' => true ];
536 
537  return $this->backend->getFileSha1Base36( $params );
538  }
539  }
540 
546  public function getBackend() {
547  return $this->backend;
548  }
549 
555  final public function logFailure( $action ) {
557  $params['failedAction'] = $action;
558  try {
559  $this->logger->error( static::class .
560  " failed (batch #{$this->batchId}): " . FormatJson::encode( $params ) );
561  } catch ( Exception $e ) {
562  // bad config? debug log error?
563  }
564  }
565 }
FileOp\logFailure
logFailure( $action)
Log a file operation failure and preserve any temp files.
Definition: FileOp.php:555
FileOp\setBatchId
setBatchId( $batchId)
Set the batch UUID this operation belongs to.
Definition: FileOp.php:131
FileOp\fileSha1
fileSha1( $source, array $predicates)
Get the SHA-1 of a file in storage when this operation is attempted.
Definition: FileOp.php:526
StatusValue\newFatal
static newFatal( $message,... $parameters)
Factory function for fatal errors.
Definition: StatusValue.php:70
FileOp\allowedParams
allowedParams()
Get the file operation parameters.
Definition: FileOp.php:358
FileOp\attemptAsyncQuick
attemptAsyncQuick()
Attempt the operation in the background without regards to prechecks.
Definition: FileOp.php:347
FileOp
FileBackend helper class for representing operations.
Definition: FileOp.php:36
true
return true
Definition: router.php:90
FileOp\$destExists
bool $destExists
Definition: FileOp.php:65
FileOp\doAttempt
doAttempt()
Definition: FileOp.php:314
FileOp\attemptQuick
attemptQuick()
Attempt the operation without regards to prechecks.
Definition: FileOp.php:336
FileOp\precheck
precheck(array &$predicates)
Check preconditions of the operation without writing anything.
Definition: FileOp.php:254
FileBackend\normalizeStoragePath
static normalizeStoragePath( $storagePath)
Normalize a storage path by cleaning up directory separators.
Definition: FileBackend.php:1563
FileOp\getJournalEntries
getJournalEntries(array $oPredicates, array $nPredicates)
Get the file journal entries for this file operation.
Definition: FileOp.php:213
FileOp\doPrecheck
doPrecheck(array &$predicates)
Definition: FileOp.php:282
FileOp\$logger
LoggerInterface $logger
Definition: FileOp.php:40
FileOp\storagePathsChanged
storagePathsChanged()
Get a list of storage paths written to for this operation.
Definition: FileOp.php:386
FileOp\ASSUMED_EXISTS
const ASSUMED_EXISTS
Definition: FileOp.php:73
$res
$res
Definition: testCompression.php:57
FileOp\$failed
bool $failed
Definition: FileOp.php:48
FileOp\applyDependencies
applyDependencies(array $deps)
Update a dependency tracking array to account for this operation.
Definition: FileOp.php:178
FileOp\attemptAsync
attemptAsync()
Attempt the operation in the background.
Definition: FileOp.php:323
FileOp\__construct
__construct(FileBackendStore $backend, array $params, LoggerInterface $logger)
Build a new batch file operation transaction.
Definition: FileOp.php:84
FileOp\$async
bool $async
Definition: FileOp.php:50
FormatJson\encode
static encode( $value, $pretty=false, $escaping=0)
Returns the JSON representation of a value.
Definition: FormatJson.php:115
FileOp\getBackend
getBackend()
Get the backend this operation is for.
Definition: FileOp.php:546
FileOp\getSourceSize
getSourceSize()
precheckDestExistence() helper function to get the source file size.
Definition: FileOp.php:459
FileOp\$batchId
string $batchId
Definition: FileOp.php:52
FileOp\failed
failed()
Check if this operation failed precheck() or attempt()
Definition: FileOp.php:150
FileOp\ASSUMED_SHA1
const ASSUMED_SHA1
Definition: FileOp.php:72
FileBackend\isStoragePath
static isStoragePath( $path)
Check if a given path is a "mwstore://" path.
Definition: FileBackend.php:1528
FileOp\setFlags
setFlags(array $params)
Adjust params to FileBackendStore internal file calls.
Definition: FileOp.php:368
FileOp\fileSize
fileSize( $source, array $predicates)
Get the size a file in storage will have when this operation is attempted.
Definition: FileOp.php:504
FileOp\$backend
FileBackendStore $backend
Definition: FileOp.php:38
FileOp\attempt
attempt()
Attempt the operation.
Definition: FileOp.php:291
FileOp\getParam
getParam( $name)
Get the value of the parameter with the given name.
Definition: FileOp.php:141
FileOp\STATE_NEW
const STATE_NEW
Definition: FileOp.php:68
FileOp\STATE_ATTEMPTED
const STATE_ATTEMPTED
Definition: FileOp.php:70
FileOp\dependsOn
dependsOn(array $deps)
Check if this operation changes files listed in $paths.
Definition: FileOp.php:191
FileOp\$sourceSha1
string bool $sourceSha1
Definition: FileOp.php:59
FileOp\$params
array $params
Definition: FileOp.php:43
FileOp\$state
int $state
Definition: FileOp.php:46
StatusValue\newGood
static newGood( $value=null)
Factory function for good results.
Definition: StatusValue.php:82
FileOp\$cancelled
bool $cancelled
Definition: FileOp.php:54
FileBackendStore
Base class for all backends using particular storage medium.
Definition: FileBackendStore.php:41
FileOp\getSourceSha1Base36
getSourceSha1Base36()
precheckDestExistence() helper function to get the source file SHA-1.
Definition: FileOp.php:469
FileOp\STATE_CHECKED
const STATE_CHECKED
Definition: FileOp.php:69
FileOp\$overwriteSameCase
bool $overwriteSameCase
Definition: FileOp.php:62
$path
$path
Definition: NoLocalSettings.php:25
FileOp\ASSUMED_SIZE
const ASSUMED_SIZE
Definition: FileOp.php:74
FileOp\$sourceSize
int bool $sourceSize
Definition: FileOp.php:57
$source
$source
Definition: mwdoc-filter.php:34
FileOp\fileExists
fileExists( $source, array $predicates)
Check if a file will exist in storage when this operation is attempted.
Definition: FileOp.php:483
FileOp\precheckDestExistence
precheckDestExistence(array $predicates)
Check for errors with regards to the destination file already existing.
Definition: FileOp.php:409
FileOp\newDependencies
static newDependencies()
Get a new empty dependency tracking array for paths read/written to.
Definition: FileOp.php:168
FileOp\normalizeIfValidStoragePath
static normalizeIfValidStoragePath( $path)
Normalize a string if it is a valid storage path.
Definition: FileOp.php:116
FileOp\storagePathsReadOrChanged
storagePathsReadOrChanged()
Get a list of storage paths read from or written to for this operation.
Definition: FileOp.php:395
FileOp\newPredicates
static newPredicates()
Get a new empty predicates array for precheck()
Definition: FileOp.php:159
FileOp\storagePathsRead
storagePathsRead()
Get a list of storage paths read from for this operation.
Definition: FileOp.php:377