Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
80.00% |
92 / 115 |
|
68.00% |
17 / 25 |
CRAP | |
0.00% |
0 / 1 |
| FileOp | |
80.70% |
92 / 114 |
|
68.00% |
17 / 25 |
80.35 | |
0.00% |
0 / 1 |
| __construct | |
92.31% |
12 / 13 |
|
0.00% |
0 / 1 |
7.02 | |||
| normalizeIfValidStoragePath | |
100.00% |
4 / 4 |
|
100.00% |
1 / 1 |
2 | |||
| getParam | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| failed | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| newDependencies | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| applyDependencies | |
100.00% |
3 / 3 |
|
100.00% |
1 / 1 |
1 | |||
| dependsOn | |
100.00% |
7 / 7 |
|
100.00% |
1 / 1 |
6 | |||
| precheck | |
87.50% |
7 / 8 |
|
0.00% |
0 / 1 |
3.02 | |||
| doPrecheck | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| attempt | |
66.67% |
8 / 12 |
|
0.00% |
0 / 1 |
5.93 | |||
| doAttempt | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| attemptAsync | |
0.00% |
0 / 4 |
|
0.00% |
0 / 1 |
2 | |||
| attemptQuick | |
100.00% |
2 / 2 |
|
100.00% |
1 / 1 |
1 | |||
| attemptAsyncQuick | |
0.00% |
0 / 2 |
|
0.00% |
0 / 1 |
2 | |||
| allowedParams | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| setFlags | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| storagePathsRead | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| storagePathsChanged | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| storagePathsReadOrChanged | |
100.00% |
3 / 3 |
|
100.00% |
1 / 1 |
1 | |||
| precheckDestExistence | |
86.36% |
19 / 22 |
|
0.00% |
0 / 1 |
13.43 | |||
| resolveFileExistence | |
100.00% |
6 / 6 |
|
100.00% |
1 / 1 |
1 | |||
| resolveFileSize | |
100.00% |
6 / 6 |
|
100.00% |
1 / 1 |
1 | |||
| resolveFileSha1Base36 | |
100.00% |
6 / 6 |
|
100.00% |
1 / 1 |
1 | |||
| getBackend | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| logFailure | |
0.00% |
0 / 6 |
|
0.00% |
0 / 1 |
12 | |||
| 1 | <?php |
| 2 | /** |
| 3 | * Helper class for representing operations with transaction support. |
| 4 | * |
| 5 | * @license GPL-2.0-or-later |
| 6 | * @file |
| 7 | * @ingroup FileBackend |
| 8 | */ |
| 9 | |
| 10 | namespace Wikimedia\FileBackend\FileOps; |
| 11 | |
| 12 | use Closure; |
| 13 | use Exception; |
| 14 | use InvalidArgumentException; |
| 15 | use Psr\Log\LoggerInterface; |
| 16 | use StatusValue; |
| 17 | use Wikimedia\FileBackend\FileBackend; |
| 18 | use Wikimedia\FileBackend\FileBackendStore; |
| 19 | use Wikimedia\RequestTimeout\TimeoutException; |
| 20 | |
| 21 | /** |
| 22 | * FileBackend helper class for representing operations. |
| 23 | * Do not use this class from places outside FileBackend. |
| 24 | * |
| 25 | * Methods called from FileOpBatch::attempt() should avoid throwing |
| 26 | * exceptions at all costs. FileOp objects should be lightweight in order |
| 27 | * to support large arrays in memory and serialization. |
| 28 | * |
| 29 | * @ingroup FileBackend |
| 30 | * @since 1.19 |
| 31 | */ |
| 32 | abstract class FileOp { |
| 33 | /** @var FileBackendStore */ |
| 34 | protected $backend; |
| 35 | /** @var LoggerInterface */ |
| 36 | protected $logger; |
| 37 | |
| 38 | /** @var array */ |
| 39 | protected $params = []; |
| 40 | |
| 41 | /** @var int Stage in the operation life-cycle */ |
| 42 | protected $state = self::STATE_NEW; |
| 43 | /** @var bool Whether the operation pre-check or attempt stage failed */ |
| 44 | protected $failed = false; |
| 45 | /** @var bool Whether the operation is part of a concurrent sub-batch of operation */ |
| 46 | protected $async = false; |
| 47 | /** @var bool Whether the operation pre-check stage marked the attempt stage as a no-op */ |
| 48 | protected $noOp = false; |
| 49 | |
| 50 | /** @var bool|null */ |
| 51 | protected $overwriteSameCase; |
| 52 | /** @var bool|null */ |
| 53 | protected $destExists; |
| 54 | |
| 55 | /** Operation has not yet been pre-checked nor run */ |
| 56 | private const STATE_NEW = 1; |
| 57 | /** Operation has been pre-checked but not yet attempted */ |
| 58 | private const STATE_CHECKED = 2; |
| 59 | /** Operation has been attempted */ |
| 60 | private const STATE_ATTEMPTED = 3; |
| 61 | |
| 62 | /** |
| 63 | * Build a new batch file operation transaction |
| 64 | * |
| 65 | * @param FileBackendStore $backend |
| 66 | * @param array $params |
| 67 | * @param LoggerInterface $logger PSR logger instance |
| 68 | */ |
| 69 | final public function __construct( |
| 70 | FileBackendStore $backend, array $params, LoggerInterface $logger |
| 71 | ) { |
| 72 | $this->backend = $backend; |
| 73 | $this->logger = $logger; |
| 74 | [ $required, $optional, $paths ] = $this->allowedParams(); |
| 75 | foreach ( $required as $name ) { |
| 76 | if ( isset( $params[$name] ) ) { |
| 77 | $this->params[$name] = $params[$name]; |
| 78 | } else { |
| 79 | throw new InvalidArgumentException( "File operation missing parameter '$name'." ); |
| 80 | } |
| 81 | } |
| 82 | foreach ( $optional as $name ) { |
| 83 | if ( isset( $params[$name] ) ) { |
| 84 | $this->params[$name] = $params[$name]; |
| 85 | } |
| 86 | } |
| 87 | foreach ( $paths as $name ) { |
| 88 | if ( isset( $this->params[$name] ) ) { |
| 89 | // Normalize paths so the paths to the same file have the same string |
| 90 | $this->params[$name] = self::normalizeIfValidStoragePath( $this->params[$name] ); |
| 91 | } |
| 92 | } |
| 93 | } |
| 94 | |
| 95 | /** |
| 96 | * Normalize a string if it is a valid storage path |
| 97 | * |
| 98 | * @param string $path |
| 99 | * @return string |
| 100 | */ |
| 101 | protected static function normalizeIfValidStoragePath( $path ) { |
| 102 | if ( FileBackend::isStoragePath( $path ) ) { |
| 103 | $res = FileBackend::normalizeStoragePath( $path ); |
| 104 | |
| 105 | return $res ?? $path; |
| 106 | } |
| 107 | |
| 108 | return $path; |
| 109 | } |
| 110 | |
| 111 | /** |
| 112 | * Get the value of the parameter with the given name |
| 113 | * |
| 114 | * @param string $name |
| 115 | * @return mixed Returns null if the parameter is not set |
| 116 | */ |
| 117 | final public function getParam( $name ) { |
| 118 | return $this->params[$name] ?? null; |
| 119 | } |
| 120 | |
| 121 | /** |
| 122 | * Check if this operation failed precheck() or attempt() |
| 123 | * |
| 124 | * @return bool |
| 125 | */ |
| 126 | final public function failed() { |
| 127 | return $this->failed; |
| 128 | } |
| 129 | |
| 130 | /** |
| 131 | * Get a new empty dependency tracking array for paths read/written to |
| 132 | * |
| 133 | * @return array |
| 134 | */ |
| 135 | final public static function newDependencies() { |
| 136 | return [ 'read' => [], 'write' => [] ]; |
| 137 | } |
| 138 | |
| 139 | /** |
| 140 | * Update a dependency tracking array to account for this operation |
| 141 | * |
| 142 | * @param array $deps Prior path reads/writes; format of FileOp::newDependencies() |
| 143 | * @return array |
| 144 | */ |
| 145 | final public function applyDependencies( array $deps ) { |
| 146 | $deps['read'] += array_fill_keys( $this->storagePathsRead(), 1 ); |
| 147 | $deps['write'] += array_fill_keys( $this->storagePathsChanged(), 1 ); |
| 148 | |
| 149 | return $deps; |
| 150 | } |
| 151 | |
| 152 | /** |
| 153 | * Check if this operation changes files listed in $paths |
| 154 | * |
| 155 | * @param array $deps Prior path reads/writes; format of FileOp::newDependencies() |
| 156 | * @return bool |
| 157 | */ |
| 158 | final public function dependsOn( array $deps ) { |
| 159 | foreach ( $this->storagePathsChanged() as $path ) { |
| 160 | if ( isset( $deps['read'][$path] ) || isset( $deps['write'][$path] ) ) { |
| 161 | return true; // "output" or "anti" dependency |
| 162 | } |
| 163 | } |
| 164 | foreach ( $this->storagePathsRead() as $path ) { |
| 165 | if ( isset( $deps['write'][$path] ) ) { |
| 166 | return true; // "flow" dependency |
| 167 | } |
| 168 | } |
| 169 | |
| 170 | return false; |
| 171 | } |
| 172 | |
| 173 | /** |
| 174 | * Do a dry-run precondition check of the operation in the context of op batch |
| 175 | * |
| 176 | * Updates the batch predicates for all paths this op can change if an OK status is returned |
| 177 | * |
| 178 | * @param FileStatePredicates $predicates Counterfactual file states for the op batch |
| 179 | * @return StatusValue |
| 180 | */ |
| 181 | final public function precheck( FileStatePredicates $predicates ) { |
| 182 | if ( $this->state !== self::STATE_NEW ) { |
| 183 | return StatusValue::newFatal( 'fileop-fail-state', self::STATE_NEW, $this->state ); |
| 184 | } |
| 185 | $this->state = self::STATE_CHECKED; |
| 186 | |
| 187 | $opPredicates = $predicates->snapshot( $this->storagePathsReadOrChanged() ); |
| 188 | $status = $this->doPrecheck( $opPredicates, $predicates ); |
| 189 | if ( !$status->isOK() ) { |
| 190 | $this->failed = true; |
| 191 | } |
| 192 | |
| 193 | return $status; |
| 194 | } |
| 195 | |
| 196 | /** |
| 197 | * Do a dry-run precondition check of the operation in the context of op batch |
| 198 | * |
| 199 | * Updates the batch predicates for all paths this op can change if an OK status is returned |
| 200 | * |
| 201 | * @param FileStatePredicates $opPredicates Counterfactual file states for op paths at op start |
| 202 | * @param FileStatePredicates $batchPredicates Counterfactual file states for the op batch |
| 203 | * @return StatusValue |
| 204 | */ |
| 205 | protected function doPrecheck( |
| 206 | FileStatePredicates $opPredicates, |
| 207 | FileStatePredicates $batchPredicates |
| 208 | ) { |
| 209 | return StatusValue::newGood(); |
| 210 | } |
| 211 | |
| 212 | /** |
| 213 | * Attempt the operation |
| 214 | * |
| 215 | * @return StatusValue |
| 216 | */ |
| 217 | final public function attempt() { |
| 218 | if ( $this->state !== self::STATE_CHECKED ) { |
| 219 | return StatusValue::newFatal( 'fileop-fail-state', self::STATE_CHECKED, $this->state ); |
| 220 | } elseif ( $this->failed ) { // failed precheck |
| 221 | return StatusValue::newFatal( 'fileop-fail-attempt-precheck' ); |
| 222 | } |
| 223 | $this->state = self::STATE_ATTEMPTED; |
| 224 | if ( $this->noOp ) { |
| 225 | $status = StatusValue::newGood(); // no-op |
| 226 | } else { |
| 227 | $status = $this->doAttempt(); |
| 228 | if ( !$status->isOK() ) { |
| 229 | $this->failed = true; |
| 230 | $this->logFailure( 'attempt' ); |
| 231 | } |
| 232 | } |
| 233 | |
| 234 | return $status; |
| 235 | } |
| 236 | |
| 237 | /** |
| 238 | * @return StatusValue |
| 239 | */ |
| 240 | protected function doAttempt() { |
| 241 | return StatusValue::newGood(); |
| 242 | } |
| 243 | |
| 244 | /** |
| 245 | * Attempt the operation in the background |
| 246 | * |
| 247 | * @return StatusValue |
| 248 | */ |
| 249 | final public function attemptAsync() { |
| 250 | $this->async = true; |
| 251 | $result = $this->attempt(); |
| 252 | $this->async = false; |
| 253 | |
| 254 | return $result; |
| 255 | } |
| 256 | |
| 257 | /** |
| 258 | * Attempt the operation without regards to prechecks |
| 259 | * |
| 260 | * @return StatusValue |
| 261 | */ |
| 262 | final public function attemptQuick() { |
| 263 | $this->state = self::STATE_CHECKED; // bypassed |
| 264 | |
| 265 | return $this->attempt(); |
| 266 | } |
| 267 | |
| 268 | /** |
| 269 | * Attempt the operation in the background without regards to prechecks |
| 270 | * |
| 271 | * @return StatusValue |
| 272 | */ |
| 273 | final public function attemptAsyncQuick() { |
| 274 | $this->state = self::STATE_CHECKED; // bypassed |
| 275 | |
| 276 | return $this->attemptAsync(); |
| 277 | } |
| 278 | |
| 279 | /** |
| 280 | * Get the file operation parameters |
| 281 | * |
| 282 | * @return array (required params list, optional params list, list of params that are paths) |
| 283 | */ |
| 284 | protected function allowedParams() { |
| 285 | return [ [], [], [] ]; |
| 286 | } |
| 287 | |
| 288 | /** |
| 289 | * Adjust params to FileBackendStore internal file calls |
| 290 | * |
| 291 | * @param array $params |
| 292 | * @return array (required params list, optional params list) |
| 293 | */ |
| 294 | protected function setFlags( array $params ) { |
| 295 | return [ 'async' => $this->async ] + $params; |
| 296 | } |
| 297 | |
| 298 | /** |
| 299 | * Get a list of storage paths read from for this operation |
| 300 | * |
| 301 | * @return array |
| 302 | */ |
| 303 | public function storagePathsRead() { |
| 304 | return []; |
| 305 | } |
| 306 | |
| 307 | /** |
| 308 | * Get a list of storage paths written to for this operation |
| 309 | * |
| 310 | * @return array |
| 311 | */ |
| 312 | public function storagePathsChanged() { |
| 313 | return []; |
| 314 | } |
| 315 | |
| 316 | /** |
| 317 | * Get a list of storage paths read from or written to for this operation |
| 318 | * |
| 319 | * @return array |
| 320 | */ |
| 321 | final public function storagePathsReadOrChanged() { |
| 322 | return array_values( array_unique( |
| 323 | array_merge( $this->storagePathsRead(), $this->storagePathsChanged() ) |
| 324 | ) ); |
| 325 | } |
| 326 | |
| 327 | /** |
| 328 | * Check for errors with regards to the destination file already existing |
| 329 | * |
| 330 | * Also set the destExists and overwriteSameCase member variables. |
| 331 | * A bad StatusValue will be returned if there is no chance it can be overwritten. |
| 332 | * |
| 333 | * @param FileStatePredicates $opPredicates Counterfactual storage path states for this op |
| 334 | * @param int|false|Closure $sourceSize Source size or idempotent function yielding the size |
| 335 | * @param string|Closure $sourceSha1 Source hash, or, idempotent function yielding the hash |
| 336 | * @return StatusValue |
| 337 | */ |
| 338 | protected function precheckDestExistence( |
| 339 | FileStatePredicates $opPredicates, |
| 340 | $sourceSize, |
| 341 | $sourceSha1 |
| 342 | ) { |
| 343 | $status = StatusValue::newGood(); |
| 344 | // Record the existence of destination file |
| 345 | $this->destExists = $this->resolveFileExistence( $this->params['dst'], $opPredicates ); |
| 346 | // Check if an incompatible file exists at the destination |
| 347 | $this->overwriteSameCase = false; |
| 348 | if ( $this->destExists ) { |
| 349 | if ( $this->getParam( 'overwrite' ) ) { |
| 350 | return $status; // OK, no conflict |
| 351 | } elseif ( $this->getParam( 'overwriteSame' ) ) { |
| 352 | // Operation does nothing other than return an OK or bad status |
| 353 | $sourceSize = ( $sourceSize instanceof Closure ) ? $sourceSize() : $sourceSize; |
| 354 | $sourceSha1 = ( $sourceSha1 instanceof Closure ) ? $sourceSha1() : $sourceSha1; |
| 355 | $dstSha1 = $this->resolveFileSha1Base36( $this->params['dst'], $opPredicates ); |
| 356 | $dstSize = $this->resolveFileSize( $this->params['dst'], $opPredicates ); |
| 357 | // Check if hashes are valid and match each other... |
| 358 | if ( !strlen( $sourceSha1 ) || !strlen( $dstSha1 ) ) { |
| 359 | $status->fatal( 'backend-fail-hashes' ); |
| 360 | } elseif ( !is_int( $sourceSize ) || !is_int( $dstSize ) ) { |
| 361 | $status->fatal( 'backend-fail-sizes' ); |
| 362 | } elseif ( $sourceSha1 !== $dstSha1 || $sourceSize !== $dstSize ) { |
| 363 | // Give an error if the files are not identical |
| 364 | $status->fatal( 'backend-fail-notsame', $this->params['dst'] ); |
| 365 | } else { |
| 366 | $this->overwriteSameCase = true; // OK |
| 367 | } |
| 368 | } else { |
| 369 | $status->fatal( 'backend-fail-alreadyexists', $this->params['dst'] ); |
| 370 | } |
| 371 | } elseif ( $this->destExists === FileBackend::EXISTENCE_ERROR ) { |
| 372 | $status->fatal( 'backend-fail-stat', $this->params['dst'] ); |
| 373 | } |
| 374 | |
| 375 | return $status; |
| 376 | } |
| 377 | |
| 378 | /** |
| 379 | * Check if a file will exist in storage when this operation is attempted |
| 380 | * |
| 381 | * Ideally, the file stat entry should already be preloaded via preloadFileStat(). |
| 382 | * Otherwise, this will query the backend. |
| 383 | * |
| 384 | * @param string $source Storage path |
| 385 | * @param FileStatePredicates $opPredicates Counterfactual storage path states for this op |
| 386 | * @return bool|null Whether the file will exist or null on error |
| 387 | */ |
| 388 | final protected function resolveFileExistence( $source, FileStatePredicates $opPredicates ) { |
| 389 | return $opPredicates->resolveFileExistence( |
| 390 | $source, |
| 391 | function ( $path ) { |
| 392 | return $this->backend->fileExists( [ 'src' => $path, 'latest' => true ] ); |
| 393 | } |
| 394 | ); |
| 395 | } |
| 396 | |
| 397 | /** |
| 398 | * Get the size a file in storage will have when this operation is attempted |
| 399 | * |
| 400 | * Ideally, file the stat entry should already be preloaded via preloadFileStat() and |
| 401 | * the backend tracks hashes as extended attributes. Otherwise, this will query the backend. |
| 402 | * Get the size of a file in storage when this operation is attempted |
| 403 | * |
| 404 | * @param string $source Storage path |
| 405 | * @param FileStatePredicates $opPredicates Counterfactual storage path states for this op |
| 406 | * @return int|false False on failure |
| 407 | */ |
| 408 | final protected function resolveFileSize( $source, FileStatePredicates $opPredicates ) { |
| 409 | return $opPredicates->resolveFileSize( |
| 410 | $source, |
| 411 | function ( $path ) { |
| 412 | return $this->backend->getFileSize( [ 'src' => $path, 'latest' => true ] ); |
| 413 | } |
| 414 | ); |
| 415 | } |
| 416 | |
| 417 | /** |
| 418 | * Get the SHA-1 of a file in storage when this operation is attempted |
| 419 | * |
| 420 | * @param string $source Storage path |
| 421 | * @param FileStatePredicates $opPredicates Counterfactual storage path states for this op |
| 422 | * @return string|false The SHA-1 hash the file will have or false if non-existent or on error |
| 423 | */ |
| 424 | final protected function resolveFileSha1Base36( $source, FileStatePredicates $opPredicates ) { |
| 425 | return $opPredicates->resolveFileSha1Base36( |
| 426 | $source, |
| 427 | function ( $path ) { |
| 428 | return $this->backend->getFileSha1Base36( [ 'src' => $path, 'latest' => true ] ); |
| 429 | } |
| 430 | ); |
| 431 | } |
| 432 | |
| 433 | /** |
| 434 | * Get the backend this operation is for |
| 435 | * |
| 436 | * @return FileBackendStore |
| 437 | */ |
| 438 | public function getBackend() { |
| 439 | return $this->backend; |
| 440 | } |
| 441 | |
| 442 | /** |
| 443 | * Log a file operation failure and preserve any temp files |
| 444 | * |
| 445 | * @param string $action |
| 446 | */ |
| 447 | final public function logFailure( $action ) { |
| 448 | try { |
| 449 | $this->logger->error( static::class . ' failed: ' . $action, |
| 450 | [ 'params' => $this->params ] |
| 451 | ); |
| 452 | } catch ( TimeoutException $e ) { |
| 453 | throw $e; |
| 454 | } catch ( Exception ) { |
| 455 | // bad config? debug log error? |
| 456 | } |
| 457 | } |
| 458 | } |
| 459 | |
| 460 | /** @deprecated class alias since 1.43 */ |
| 461 | class_alias( FileOp::class, 'FileOp' ); |