Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
Total | |
81.82% |
54 / 66 |
|
0.00% |
0 / 2 |
CRAP | |
0.00% |
0 / 1 |
FileOpBatch | |
81.82% |
54 / 66 |
|
0.00% |
0 / 2 |
24.91 | |
0.00% |
0 / 1 |
attempt | |
94.44% |
34 / 36 |
|
0.00% |
0 / 1 |
11.02 | |||
runParallelBatches | |
66.67% |
20 / 30 |
|
0.00% |
0 / 1 |
15.48 |
1 | <?php |
2 | /** |
3 | * Helper class for representing batch file operations. |
4 | * |
5 | * This program is free software; you can redistribute it and/or modify |
6 | * it under the terms of the GNU General Public License as published by |
7 | * the Free Software Foundation; either version 2 of the License, or |
8 | * (at your option) any later version. |
9 | * |
10 | * This program is distributed in the hope that it will be useful, |
11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
13 | * GNU General Public License for more details. |
14 | * |
15 | * You should have received a copy of the GNU General Public License along |
16 | * with this program; if not, write to the Free Software Foundation, Inc., |
17 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
18 | * http://www.gnu.org/copyleft/gpl.html |
19 | * |
20 | * @file |
21 | * @ingroup FileBackend |
22 | */ |
23 | |
24 | /** |
25 | * Helper class for representing batch file operations. |
26 | * Do not use this class from places outside FileBackend. |
27 | * |
28 | * Methods should avoid throwing exceptions at all costs. |
29 | * |
30 | * @ingroup FileBackend |
31 | * @since 1.20 |
32 | */ |
33 | class FileOpBatch { |
34 | /* Timeout related parameters */ |
35 | private const MAX_BATCH_SIZE = 1000; // integer |
36 | |
37 | /** |
38 | * Attempt to perform a series of file operations. |
39 | * Callers are responsible for handling file locking. |
40 | * |
41 | * $opts is an array of options, including: |
42 | * - force : Errors that would normally cause a rollback do not. |
43 | * The remaining operations are still attempted if any fail. |
44 | * - concurrency : Try to do this many operations in parallel when possible. |
45 | * |
46 | * The resulting StatusValue will be "OK" unless: |
47 | * - a) unexpected operation errors occurred (network partitions, disk full...) |
48 | * - b) predicted operation errors occurred and 'force' was not set |
49 | * |
50 | * @param FileOp[] $performOps List of FileOp operations |
51 | * @param array $opts Batch operation options |
52 | * @return StatusValue |
53 | */ |
54 | public static function attempt( array $performOps, array $opts ) { |
55 | $status = StatusValue::newGood(); |
56 | |
57 | $n = count( $performOps ); |
58 | if ( $n > self::MAX_BATCH_SIZE ) { |
59 | $status->fatal( 'backend-fail-batchsize', $n, self::MAX_BATCH_SIZE ); |
60 | |
61 | return $status; |
62 | } |
63 | |
64 | $ignoreErrors = !empty( $opts['force'] ); |
65 | $maxConcurrency = $opts['concurrency'] ?? 1; |
66 | |
67 | $predicates = new FileStatePredicates(); // account for previous ops in prechecks |
68 | $curBatch = []; // concurrent FileOp sub-batch accumulation |
69 | $curBatchDeps = FileOp::newDependencies(); // paths used in FileOp sub-batch |
70 | $pPerformOps = []; // ordered list of concurrent FileOp sub-batches |
71 | $lastBackend = null; // last op backend name |
72 | // Do pre-checks for each operation; abort on failure... |
73 | foreach ( $performOps as $index => $fileOp ) { |
74 | $backendName = $fileOp->getBackend()->getName(); |
75 | // Decide if this op can be done concurrently within this sub-batch |
76 | // or if a new concurrent sub-batch must be started after this one... |
77 | if ( $fileOp->dependsOn( $curBatchDeps ) |
78 | || count( $curBatch ) >= $maxConcurrency |
79 | || ( $backendName !== $lastBackend && count( $curBatch ) ) |
80 | ) { |
81 | $pPerformOps[] = $curBatch; // push this batch |
82 | $curBatch = []; // start a new sub-batch |
83 | $curBatchDeps = FileOp::newDependencies(); |
84 | } |
85 | $lastBackend = $backendName; |
86 | $curBatch[$index] = $fileOp; // keep index |
87 | // Update list of affected paths in this batch |
88 | $curBatchDeps = $fileOp->applyDependencies( $curBatchDeps ); |
89 | // Simulate performing the operation... |
90 | $subStatus = $fileOp->precheck( $predicates ); // updates $predicates |
91 | $status->merge( $subStatus ); |
92 | if ( !$subStatus->isOK() ) { |
93 | // operation failed? |
94 | $status->success[$index] = false; |
95 | ++$status->failCount; |
96 | if ( !$ignoreErrors ) { |
97 | return $status; // abort |
98 | } |
99 | } |
100 | } |
101 | // Push the last sub-batch |
102 | if ( count( $curBatch ) ) { |
103 | $pPerformOps[] = $curBatch; |
104 | } |
105 | |
106 | if ( $ignoreErrors ) { // treat precheck() fatals as mere warnings |
107 | $status->setResult( true, $status->value ); |
108 | } |
109 | |
110 | // Attempt each operation (in parallel if allowed and possible)... |
111 | self::runParallelBatches( $pPerformOps, $status ); |
112 | |
113 | return $status; |
114 | } |
115 | |
116 | /** |
117 | * Attempt a list of file operations sub-batches in series. |
118 | * |
119 | * The operations *in* each sub-batch will be done in parallel. |
120 | * The caller is responsible for making sure the operations |
121 | * within any given sub-batch do not depend on each other. |
122 | * This will abort remaining ops on failure. |
123 | * |
124 | * @param FileOp[][] $pPerformOps Batches of file ops (batches use original indexes) |
125 | * @param StatusValue $status |
126 | */ |
127 | protected static function runParallelBatches( array $pPerformOps, StatusValue $status ) { |
128 | $aborted = false; // set to true on unexpected errors |
129 | foreach ( $pPerformOps as $performOpsBatch ) { |
130 | if ( $aborted ) { // check batch op abort flag... |
131 | // We can't continue (even with $ignoreErrors) as $predicates is wrong. |
132 | // Log the remaining ops as failed for recovery... |
133 | foreach ( $performOpsBatch as $i => $fileOp ) { |
134 | $status->success[$i] = false; |
135 | ++$status->failCount; |
136 | $fileOp->logFailure( 'attempt_aborted' ); |
137 | } |
138 | continue; |
139 | } |
140 | /** @var StatusValue[] $statuses */ |
141 | $statuses = []; |
142 | $opHandles = []; |
143 | // Get the backend; all sub-batch ops belong to a single backend |
144 | /** @var FileBackendStore $backend */ |
145 | $backend = reset( $performOpsBatch )->getBackend(); |
146 | // Get the operation handles or actually do it if there is just one. |
147 | // If attemptAsync() returns a StatusValue, it was either due to an error |
148 | // or the backend does not support async ops and did it synchronously. |
149 | foreach ( $performOpsBatch as $i => $fileOp ) { |
150 | if ( !isset( $status->success[$i] ) ) { // didn't already fail in precheck() |
151 | // Parallel ops may be disabled in config due to missing dependencies, |
152 | // (e.g. needing popen()). When they are, $performOpsBatch has size 1. |
153 | $subStatus = ( count( $performOpsBatch ) > 1 ) |
154 | ? $fileOp->attemptAsync() |
155 | : $fileOp->attempt(); |
156 | if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { |
157 | $opHandles[$i] = $subStatus->value; // deferred |
158 | } else { |
159 | $statuses[$i] = $subStatus; // done already |
160 | } |
161 | } |
162 | } |
163 | // Try to do all the operations concurrently... |
164 | $statuses += $backend->executeOpHandlesInternal( $opHandles ); |
165 | // Marshall and merge all the responses (blocking)... |
166 | foreach ( $performOpsBatch as $i => $fileOp ) { |
167 | if ( !isset( $status->success[$i] ) ) { // didn't already fail in precheck() |
168 | $subStatus = $statuses[$i]; |
169 | $status->merge( $subStatus ); |
170 | if ( $subStatus->isOK() ) { |
171 | $status->success[$i] = true; |
172 | ++$status->successCount; |
173 | } else { |
174 | $status->success[$i] = false; |
175 | ++$status->failCount; |
176 | $aborted = true; // set abort flag; we can't continue |
177 | } |
178 | } |
179 | } |
180 | } |
181 | } |
182 | } |