MediaWiki  master
FileOpBatch.php
Go to the documentation of this file.
1 <?php
33 class FileOpBatch {
34  /* Timeout related parameters */
35  private const MAX_BATCH_SIZE = 1000; // integer
36 
54  public static function attempt( array $performOps, array $opts ) {
55  $status = StatusValue::newGood();
56 
57  $n = count( $performOps );
58  if ( $n > self::MAX_BATCH_SIZE ) {
59  $status->fatal( 'backend-fail-batchsize', $n, self::MAX_BATCH_SIZE );
60 
61  return $status;
62  }
63 
64  $ignoreErrors = !empty( $opts['force'] );
65  $maxConcurrency = $opts['concurrency'] ?? 1;
66 
67  $predicates = FileOp::newPredicates(); // account for previous ops in prechecks
68  $curBatch = []; // concurrent FileOp sub-batch accumulation
69  $curBatchDeps = FileOp::newDependencies(); // paths used in FileOp sub-batch
70  $pPerformOps = []; // ordered list of concurrent FileOp sub-batches
71  $lastBackend = null; // last op backend name
72  // Do pre-checks for each operation; abort on failure...
73  foreach ( $performOps as $index => $fileOp ) {
74  $backendName = $fileOp->getBackend()->getName();
75  // Decide if this op can be done concurrently within this sub-batch
76  // or if a new concurrent sub-batch must be started after this one...
77  if ( $fileOp->dependsOn( $curBatchDeps )
78  || count( $curBatch ) >= $maxConcurrency
79  || ( $backendName !== $lastBackend && count( $curBatch ) )
80  ) {
81  $pPerformOps[] = $curBatch; // push this batch
82  $curBatch = []; // start a new sub-batch
83  $curBatchDeps = FileOp::newDependencies();
84  }
85  $lastBackend = $backendName;
86  $curBatch[$index] = $fileOp; // keep index
87  // Update list of affected paths in this batch
88  $curBatchDeps = $fileOp->applyDependencies( $curBatchDeps );
89  // Simulate performing the operation...
90  $subStatus = $fileOp->precheck( $predicates ); // updates $predicates
91  $status->merge( $subStatus );
92  if ( !$subStatus->isOK() ) {
93  // operation failed?
94  $status->success[$index] = false;
95  ++$status->failCount;
96  if ( !$ignoreErrors ) {
97  return $status; // abort
98  }
99  }
100  }
101  // Push the last sub-batch
102  if ( count( $curBatch ) ) {
103  $pPerformOps[] = $curBatch;
104  }
105 
106  if ( $ignoreErrors ) { // treat precheck() fatals as mere warnings
107  $status->setResult( true, $status->value );
108  }
109 
110  // Attempt each operation (in parallel if allowed and possible)...
111  self::runParallelBatches( $pPerformOps, $status );
112 
113  return $status;
114  }
115 
127  protected static function runParallelBatches( array $pPerformOps, StatusValue $status ) {
128  $aborted = false; // set to true on unexpected errors
129  foreach ( $pPerformOps as $performOpsBatch ) {
130  if ( $aborted ) { // check batch op abort flag...
131  // We can't continue (even with $ignoreErrors) as $predicates is wrong.
132  // Log the remaining ops as failed for recovery...
133  foreach ( $performOpsBatch as $i => $fileOp ) {
134  $status->success[$i] = false;
135  ++$status->failCount;
136  $fileOp->logFailure( 'attempt_aborted' );
137  }
138  continue;
139  }
141  $statuses = [];
142  $opHandles = [];
143  // Get the backend; all sub-batch ops belong to a single backend
145  $backend = reset( $performOpsBatch )->getBackend();
146  // Get the operation handles or actually do it if there is just one.
147  // If attemptAsync() returns a StatusValue, it was either due to an error
148  // or the backend does not support async ops and did it synchronously.
149  foreach ( $performOpsBatch as $i => $fileOp ) {
150  if ( !isset( $status->success[$i] ) ) { // didn't already fail in precheck()
151  // Parallel ops may be disabled in config due to missing dependencies,
152  // (e.g. needing popen()). When they are, $performOpsBatch has size 1.
153  $subStatus = ( count( $performOpsBatch ) > 1 )
154  ? $fileOp->attemptAsync()
155  : $fileOp->attempt();
156  if ( $subStatus->value instanceof FileBackendStoreOpHandle ) {
157  $opHandles[$i] = $subStatus->value; // deferred
158  } else {
159  $statuses[$i] = $subStatus; // done already
160  }
161  }
162  }
163  // Try to do all the operations concurrently...
164  $statuses += $backend->executeOpHandlesInternal( $opHandles );
165  // Marshall and merge all the responses (blocking)...
166  foreach ( $performOpsBatch as $i => $fileOp ) {
167  if ( !isset( $status->success[$i] ) ) { // didn't already fail in precheck()
168  $subStatus = $statuses[$i];
169  $status->merge( $subStatus );
170  if ( $subStatus->isOK() ) {
171  $status->success[$i] = true;
172  ++$status->successCount;
173  } else {
174  $status->success[$i] = false;
175  ++$status->failCount;
176  $aborted = true; // set abort flag; we can't continue
177  }
178  }
179  }
180  }
181  }
182 }
FileBackendStore helper class for performing asynchronous file operations.
Helper class for representing batch file operations.
Definition: FileOpBatch.php:33
static runParallelBatches(array $pPerformOps, StatusValue $status)
Attempt a list of file operations sub-batches in series.
static attempt(array $performOps, array $opts)
Attempt to perform a series of file operations.
Definition: FileOpBatch.php:54
static newDependencies()
Get a new empty dependency tracking array for paths read/written to.
Definition: FileOp.php:158
static newPredicates()
Get a new empty predicates array for precheck()
Definition: FileOp.php:149
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Definition: StatusValue.php:46
merge( $other, $overwriteValue=false)
Merge another status object into this one.
static newGood( $value=null)
Factory function for good results.
Definition: StatusValue.php:85