Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
Total | |
80.99% |
98 / 121 |
|
68.00% |
17 / 25 |
CRAP | |
0.00% |
0 / 1 |
FileOp | |
80.99% |
98 / 121 |
|
68.00% |
17 / 25 |
84.72 | |
0.00% |
0 / 1 |
__construct | |
92.31% |
12 / 13 |
|
0.00% |
0 / 1 |
7.02 | |||
normalizeIfValidStoragePath | |
100.00% |
4 / 4 |
|
100.00% |
1 / 1 |
2 | |||
getParam | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
failed | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
newDependencies | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
applyDependencies | |
100.00% |
3 / 3 |
|
100.00% |
1 / 1 |
1 | |||
dependsOn | |
100.00% |
7 / 7 |
|
100.00% |
1 / 1 |
6 | |||
precheck | |
92.86% |
13 / 14 |
|
0.00% |
0 / 1 |
6.01 | |||
doPrecheck | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
attempt | |
66.67% |
8 / 12 |
|
0.00% |
0 / 1 |
5.93 | |||
doAttempt | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
attemptAsync | |
0.00% |
0 / 4 |
|
0.00% |
0 / 1 |
2 | |||
attemptQuick | |
100.00% |
2 / 2 |
|
100.00% |
1 / 1 |
1 | |||
attemptAsyncQuick | |
0.00% |
0 / 2 |
|
0.00% |
0 / 1 |
2 | |||
allowedParams | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
setFlags | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
storagePathsRead | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
storagePathsChanged | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
storagePathsReadOrChanged | |
100.00% |
3 / 3 |
|
100.00% |
1 / 1 |
1 | |||
precheckDestExistence | |
86.36% |
19 / 22 |
|
0.00% |
0 / 1 |
13.43 | |||
resolveFileExistence | |
100.00% |
6 / 6 |
|
100.00% |
1 / 1 |
1 | |||
resolveFileSize | |
100.00% |
6 / 6 |
|
100.00% |
1 / 1 |
1 | |||
resolveFileSha1Base36 | |
100.00% |
6 / 6 |
|
100.00% |
1 / 1 |
1 | |||
getBackend | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
logFailure | |
0.00% |
0 / 7 |
|
0.00% |
0 / 1 |
12 |
1 | <?php |
2 | /** |
3 | * Helper class for representing operations with transaction support. |
4 | * |
5 | * This program is free software; you can redistribute it and/or modify |
6 | * it under the terms of the GNU General Public License as published by |
7 | * the Free Software Foundation; either version 2 of the License, or |
8 | * (at your option) any later version. |
9 | * |
10 | * This program is distributed in the hope that it will be useful, |
11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
13 | * GNU General Public License for more details. |
14 | * |
15 | * You should have received a copy of the GNU General Public License along |
16 | * with this program; if not, write to the Free Software Foundation, Inc., |
17 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
18 | * http://www.gnu.org/copyleft/gpl.html |
19 | * |
20 | * @file |
21 | * @ingroup FileBackend |
22 | */ |
23 | use Psr\Log\LoggerInterface; |
24 | use Wikimedia\RequestTimeout\TimeoutException; |
25 | |
26 | /** |
27 | * FileBackend helper class for representing operations. |
28 | * Do not use this class from places outside FileBackend. |
29 | * |
30 | * Methods called from FileOpBatch::attempt() should avoid throwing |
31 | * exceptions at all costs. FileOp objects should be lightweight in order |
32 | * to support large arrays in memory and serialization. |
33 | * |
34 | * @ingroup FileBackend |
35 | * @since 1.19 |
36 | */ |
37 | abstract class FileOp { |
38 | /** @var FileBackendStore */ |
39 | protected $backend; |
40 | /** @var LoggerInterface */ |
41 | protected $logger; |
42 | |
43 | /** @var array */ |
44 | protected $params = []; |
45 | |
46 | /** @var int Stage in the operation life-cycle */ |
47 | protected $state = self::STATE_NEW; |
48 | /** @var bool Whether the operation pre-check or attempt stage failed */ |
49 | protected $failed = false; |
50 | /** @var bool Whether the operation is part of a concurrent sub-batch of operation */ |
51 | protected $async = false; |
52 | /** @var bool Whether the operation pre-check stage marked the attempt stage as a no-op */ |
53 | protected $noOp = false; |
54 | |
55 | /** @var bool|null */ |
56 | protected $overwriteSameCase; |
57 | /** @var bool|null */ |
58 | protected $destExists; |
59 | |
60 | /** Operation has not yet been pre-checked nor run */ |
61 | private const STATE_NEW = 1; |
62 | /** Operation has been pre-checked but not yet attempted */ |
63 | private const STATE_CHECKED = 2; |
64 | /** Operation has been attempted */ |
65 | private const STATE_ATTEMPTED = 3; |
66 | |
67 | /** |
68 | * Build a new batch file operation transaction |
69 | * |
70 | * @param FileBackendStore $backend |
71 | * @param array $params |
72 | * @param LoggerInterface $logger PSR logger instance |
73 | * @throws InvalidArgumentException |
74 | */ |
75 | final public function __construct( |
76 | FileBackendStore $backend, array $params, LoggerInterface $logger |
77 | ) { |
78 | $this->backend = $backend; |
79 | $this->logger = $logger; |
80 | [ $required, $optional, $paths ] = $this->allowedParams(); |
81 | foreach ( $required as $name ) { |
82 | if ( isset( $params[$name] ) ) { |
83 | $this->params[$name] = $params[$name]; |
84 | } else { |
85 | throw new InvalidArgumentException( "File operation missing parameter '$name'." ); |
86 | } |
87 | } |
88 | foreach ( $optional as $name ) { |
89 | if ( isset( $params[$name] ) ) { |
90 | $this->params[$name] = $params[$name]; |
91 | } |
92 | } |
93 | foreach ( $paths as $name ) { |
94 | if ( isset( $this->params[$name] ) ) { |
95 | // Normalize paths so the paths to the same file have the same string |
96 | $this->params[$name] = self::normalizeIfValidStoragePath( $this->params[$name] ); |
97 | } |
98 | } |
99 | } |
100 | |
101 | /** |
102 | * Normalize a string if it is a valid storage path |
103 | * |
104 | * @param string $path |
105 | * @return string |
106 | */ |
107 | protected static function normalizeIfValidStoragePath( $path ) { |
108 | if ( FileBackend::isStoragePath( $path ) ) { |
109 | $res = FileBackend::normalizeStoragePath( $path ); |
110 | |
111 | return $res ?? $path; |
112 | } |
113 | |
114 | return $path; |
115 | } |
116 | |
117 | /** |
118 | * Get the value of the parameter with the given name |
119 | * |
120 | * @param string $name |
121 | * @return mixed Returns null if the parameter is not set |
122 | */ |
123 | final public function getParam( $name ) { |
124 | return $this->params[$name] ?? null; |
125 | } |
126 | |
127 | /** |
128 | * Check if this operation failed precheck() or attempt() |
129 | * |
130 | * @return bool |
131 | */ |
132 | final public function failed() { |
133 | return $this->failed; |
134 | } |
135 | |
136 | /** |
137 | * Get a new empty dependency tracking array for paths read/written to |
138 | * |
139 | * @return array |
140 | */ |
141 | final public static function newDependencies() { |
142 | return [ 'read' => [], 'write' => [] ]; |
143 | } |
144 | |
145 | /** |
146 | * Update a dependency tracking array to account for this operation |
147 | * |
148 | * @param array $deps Prior path reads/writes; format of FileOp::newDependencies() |
149 | * @return array |
150 | */ |
151 | final public function applyDependencies( array $deps ) { |
152 | $deps['read'] += array_fill_keys( $this->storagePathsRead(), 1 ); |
153 | $deps['write'] += array_fill_keys( $this->storagePathsChanged(), 1 ); |
154 | |
155 | return $deps; |
156 | } |
157 | |
158 | /** |
159 | * Check if this operation changes files listed in $paths |
160 | * |
161 | * @param array $deps Prior path reads/writes; format of FileOp::newDependencies() |
162 | * @return bool |
163 | */ |
164 | final public function dependsOn( array $deps ) { |
165 | foreach ( $this->storagePathsChanged() as $path ) { |
166 | if ( isset( $deps['read'][$path] ) || isset( $deps['write'][$path] ) ) { |
167 | return true; // "output" or "anti" dependency |
168 | } |
169 | } |
170 | foreach ( $this->storagePathsRead() as $path ) { |
171 | if ( isset( $deps['write'][$path] ) ) { |
172 | return true; // "flow" dependency |
173 | } |
174 | } |
175 | |
176 | return false; |
177 | } |
178 | |
179 | /** |
180 | * Do a dry-run precondition check of the operation in the context of op batch |
181 | * |
182 | * Updates the batch predicates for all paths this op can change if an OK status is returned |
183 | * |
184 | * @param FileStatePredicates $predicates Counterfactual file states for the op batch |
185 | * @return StatusValue |
186 | */ |
187 | final public function precheck( FileStatePredicates $predicates ) { |
188 | if ( $this->state !== self::STATE_NEW ) { |
189 | return StatusValue::newFatal( 'fileop-fail-state', self::STATE_NEW, $this->state ); |
190 | } |
191 | $this->state = self::STATE_CHECKED; |
192 | |
193 | $status = StatusValue::newGood(); |
194 | foreach ( $this->storagePathsReadOrChanged() as $path ) { |
195 | if ( !$this->backend->isPathUsableInternal( $path ) ) { |
196 | $status->fatal( 'backend-fail-usable', $path ); |
197 | } |
198 | } |
199 | if ( !$status->isOK() ) { |
200 | return $status; |
201 | } |
202 | |
203 | $opPredicates = $predicates->snapshot( $this->storagePathsReadOrChanged() ); |
204 | $status = $this->doPrecheck( $opPredicates, $predicates ); |
205 | if ( !$status->isOK() ) { |
206 | $this->failed = true; |
207 | } |
208 | |
209 | return $status; |
210 | } |
211 | |
212 | /** |
213 | * Do a dry-run precondition check of the operation in the context of op batch |
214 | * |
215 | * Updates the batch predicates for all paths this op can change if an OK status is returned |
216 | * |
217 | * @param FileStatePredicates $opPredicates Counterfactual file states for op paths at op start |
218 | * @param FileStatePredicates $batchPredicates Counterfactual file states for the op batch |
219 | * @return StatusValue |
220 | */ |
221 | protected function doPrecheck( |
222 | FileStatePredicates $opPredicates, |
223 | FileStatePredicates $batchPredicates |
224 | ) { |
225 | return StatusValue::newGood(); |
226 | } |
227 | |
228 | /** |
229 | * Attempt the operation |
230 | * |
231 | * @return StatusValue |
232 | */ |
233 | final public function attempt() { |
234 | if ( $this->state !== self::STATE_CHECKED ) { |
235 | return StatusValue::newFatal( 'fileop-fail-state', self::STATE_CHECKED, $this->state ); |
236 | } elseif ( $this->failed ) { // failed precheck |
237 | return StatusValue::newFatal( 'fileop-fail-attempt-precheck' ); |
238 | } |
239 | $this->state = self::STATE_ATTEMPTED; |
240 | if ( $this->noOp ) { |
241 | $status = StatusValue::newGood(); // no-op |
242 | } else { |
243 | $status = $this->doAttempt(); |
244 | if ( !$status->isOK() ) { |
245 | $this->failed = true; |
246 | $this->logFailure( 'attempt' ); |
247 | } |
248 | } |
249 | |
250 | return $status; |
251 | } |
252 | |
253 | /** |
254 | * @return StatusValue |
255 | */ |
256 | protected function doAttempt() { |
257 | return StatusValue::newGood(); |
258 | } |
259 | |
260 | /** |
261 | * Attempt the operation in the background |
262 | * |
263 | * @return StatusValue |
264 | */ |
265 | final public function attemptAsync() { |
266 | $this->async = true; |
267 | $result = $this->attempt(); |
268 | $this->async = false; |
269 | |
270 | return $result; |
271 | } |
272 | |
273 | /** |
274 | * Attempt the operation without regards to prechecks |
275 | * |
276 | * @return StatusValue |
277 | */ |
278 | final public function attemptQuick() { |
279 | $this->state = self::STATE_CHECKED; // bypassed |
280 | |
281 | return $this->attempt(); |
282 | } |
283 | |
284 | /** |
285 | * Attempt the operation in the background without regards to prechecks |
286 | * |
287 | * @return StatusValue |
288 | */ |
289 | final public function attemptAsyncQuick() { |
290 | $this->state = self::STATE_CHECKED; // bypassed |
291 | |
292 | return $this->attemptAsync(); |
293 | } |
294 | |
295 | /** |
296 | * Get the file operation parameters |
297 | * |
298 | * @return array (required params list, optional params list, list of params that are paths) |
299 | */ |
300 | protected function allowedParams() { |
301 | return [ [], [], [] ]; |
302 | } |
303 | |
304 | /** |
305 | * Adjust params to FileBackendStore internal file calls |
306 | * |
307 | * @param array $params |
308 | * @return array (required params list, optional params list) |
309 | */ |
310 | protected function setFlags( array $params ) { |
311 | return [ 'async' => $this->async ] + $params; |
312 | } |
313 | |
314 | /** |
315 | * Get a list of storage paths read from for this operation |
316 | * |
317 | * @return array |
318 | */ |
319 | public function storagePathsRead() { |
320 | return []; |
321 | } |
322 | |
323 | /** |
324 | * Get a list of storage paths written to for this operation |
325 | * |
326 | * @return array |
327 | */ |
328 | public function storagePathsChanged() { |
329 | return []; |
330 | } |
331 | |
332 | /** |
333 | * Get a list of storage paths read from or written to for this operation |
334 | * |
335 | * @return array |
336 | */ |
337 | final public function storagePathsReadOrChanged() { |
338 | return array_values( array_unique( |
339 | array_merge( $this->storagePathsRead(), $this->storagePathsChanged() ) |
340 | ) ); |
341 | } |
342 | |
343 | /** |
344 | * Check for errors with regards to the destination file already existing |
345 | * |
346 | * Also set the destExists and overwriteSameCase member variables. |
347 | * A bad StatusValue will be returned if there is no chance it can be overwritten. |
348 | * |
349 | * @param FileStatePredicates $opPredicates Counterfactual storage path states for this op |
350 | * @param int|false|Closure $sourceSize Source size or idempotent function yielding the size |
351 | * @param string|Closure $sourceSha1 Source hash, or, idempotent function yielding the hash |
352 | * @return StatusValue |
353 | */ |
354 | protected function precheckDestExistence( |
355 | FileStatePredicates $opPredicates, |
356 | $sourceSize, |
357 | $sourceSha1 |
358 | ) { |
359 | $status = StatusValue::newGood(); |
360 | // Record the existence of destination file |
361 | $this->destExists = $this->resolveFileExistence( $this->params['dst'], $opPredicates ); |
362 | // Check if an incompatible file exists at the destination |
363 | $this->overwriteSameCase = false; |
364 | if ( $this->destExists ) { |
365 | if ( $this->getParam( 'overwrite' ) ) { |
366 | return $status; // OK, no conflict |
367 | } elseif ( $this->getParam( 'overwriteSame' ) ) { |
368 | // Operation does nothing other than return an OK or bad status |
369 | $sourceSize = ( $sourceSize instanceof Closure ) ? $sourceSize() : $sourceSize; |
370 | $sourceSha1 = ( $sourceSha1 instanceof Closure ) ? $sourceSha1() : $sourceSha1; |
371 | $dstSha1 = $this->resolveFileSha1Base36( $this->params['dst'], $opPredicates ); |
372 | $dstSize = $this->resolveFileSize( $this->params['dst'], $opPredicates ); |
373 | // Check if hashes are valid and match each other... |
374 | if ( !strlen( $sourceSha1 ) || !strlen( $dstSha1 ) ) { |
375 | $status->fatal( 'backend-fail-hashes' ); |
376 | } elseif ( !is_int( $sourceSize ) || !is_int( $dstSize ) ) { |
377 | $status->fatal( 'backend-fail-sizes' ); |
378 | } elseif ( $sourceSha1 !== $dstSha1 || $sourceSize !== $dstSize ) { |
379 | // Give an error if the files are not identical |
380 | $status->fatal( 'backend-fail-notsame', $this->params['dst'] ); |
381 | } else { |
382 | $this->overwriteSameCase = true; // OK |
383 | } |
384 | } else { |
385 | $status->fatal( 'backend-fail-alreadyexists', $this->params['dst'] ); |
386 | } |
387 | } elseif ( $this->destExists === FileBackend::EXISTENCE_ERROR ) { |
388 | $status->fatal( 'backend-fail-stat', $this->params['dst'] ); |
389 | } |
390 | |
391 | return $status; |
392 | } |
393 | |
394 | /** |
395 | * Check if a file will exist in storage when this operation is attempted |
396 | * |
397 | * Ideally, the file stat entry should already be preloaded via preloadFileStat(). |
398 | * Otherwise, this will query the backend. |
399 | * |
400 | * @param string $source Storage path |
401 | * @param FileStatePredicates $opPredicates Counterfactual storage path states for this op |
402 | * @return bool|null Whether the file will exist or null on error |
403 | */ |
404 | final protected function resolveFileExistence( $source, FileStatePredicates $opPredicates ) { |
405 | return $opPredicates->resolveFileExistence( |
406 | $source, |
407 | function ( $path ) { |
408 | return $this->backend->fileExists( [ 'src' => $path, 'latest' => true ] ); |
409 | } |
410 | ); |
411 | } |
412 | |
413 | /** |
414 | * Get the size a file in storage will have when this operation is attempted |
415 | * |
416 | * Ideally, file the stat entry should already be preloaded via preloadFileStat() and |
417 | * the backend tracks hashes as extended attributes. Otherwise, this will query the backend. |
418 | * Get the size of a file in storage when this operation is attempted |
419 | * |
420 | * @param string $source Storage path |
421 | * @param FileStatePredicates $opPredicates Counterfactual storage path states for this op |
422 | * @return int|false False on failure |
423 | */ |
424 | final protected function resolveFileSize( $source, FileStatePredicates $opPredicates ) { |
425 | return $opPredicates->resolveFileSize( |
426 | $source, |
427 | function ( $path ) { |
428 | return $this->backend->getFileSize( [ 'src' => $path, 'latest' => true ] ); |
429 | } |
430 | ); |
431 | } |
432 | |
433 | /** |
434 | * Get the SHA-1 of a file in storage when this operation is attempted |
435 | * |
436 | * @param string $source Storage path |
437 | * @param FileStatePredicates $opPredicates Counterfactual storage path states for this op |
438 | * @return string|false The SHA-1 hash the file will have or false if non-existent or on error |
439 | */ |
440 | final protected function resolveFileSha1Base36( $source, FileStatePredicates $opPredicates ) { |
441 | return $opPredicates->resolveFileSha1Base36( |
442 | $source, |
443 | function ( $path ) { |
444 | return $this->backend->getFileSha1Base36( [ 'src' => $path, 'latest' => true ] ); |
445 | } |
446 | ); |
447 | } |
448 | |
449 | /** |
450 | * Get the backend this operation is for |
451 | * |
452 | * @return FileBackendStore |
453 | */ |
454 | public function getBackend() { |
455 | return $this->backend; |
456 | } |
457 | |
458 | /** |
459 | * Log a file operation failure and preserve any temp files |
460 | * |
461 | * @param string $action |
462 | */ |
463 | final public function logFailure( $action ) { |
464 | $params = $this->params; |
465 | $params['failedAction'] = $action; |
466 | try { |
467 | $this->logger->error( static::class . |
468 | " failed: " . FormatJson::encode( $params ) ); |
469 | } catch ( TimeoutException $e ) { |
470 | throw $e; |
471 | } catch ( Exception $e ) { |
472 | // bad config? debug log error? |
473 | } |
474 | } |
475 | } |