MediaWiki  master
FileOpBatch.php
Go to the documentation of this file.
1 <?php
33 class FileOpBatch {
34  /* Timeout related parameters */
35  private const MAX_BATCH_SIZE = 1000; // integer
36 
54  public static function attempt( array $performOps, array $opts ) {
55  $status = StatusValue::newGood();
56 
57  $n = count( $performOps );
58  if ( $n > self::MAX_BATCH_SIZE ) {
59  $status->fatal( 'backend-fail-batchsize', $n, self::MAX_BATCH_SIZE );
60 
61  return $status;
62  }
63 
64  $ignoreErrors = !empty( $opts['force'] );
65  $maxConcurrency = $opts['concurrency'] ?? 1;
66 
67  $predicates = FileOp::newPredicates(); // account for previous ops in prechecks
68  $curBatch = []; // concurrent FileOp sub-batch accumulation
69  $curBatchDeps = FileOp::newDependencies(); // paths used in FileOp sub-batch
70  $pPerformOps = []; // ordered list of concurrent FileOp sub-batches
71  $lastBackend = null; // last op backend name
72  // Do pre-checks for each operation; abort on failure...
73  foreach ( $performOps as $index => $fileOp ) {
74  $backendName = $fileOp->getBackend()->getName();
75  // Decide if this op can be done concurrently within this sub-batch
76  // or if a new concurrent sub-batch must be started after this one...
77  if ( $fileOp->dependsOn( $curBatchDeps )
78  || count( $curBatch ) >= $maxConcurrency
79  || ( $backendName !== $lastBackend && count( $curBatch ) )
80  ) {
81  $pPerformOps[] = $curBatch; // push this batch
82  $curBatch = []; // start a new sub-batch
83  $curBatchDeps = FileOp::newDependencies();
84  }
85  $lastBackend = $backendName;
86  $curBatch[$index] = $fileOp; // keep index
87  // Update list of affected paths in this batch
88  $curBatchDeps = $fileOp->applyDependencies( $curBatchDeps );
89  // Simulate performing the operation...
90  $oldPredicates = $predicates;
91  $subStatus = $fileOp->precheck( $predicates ); // updates $predicates
92  $status->merge( $subStatus );
93  if ( !$subStatus->isOK() ) {
94  // operation failed?
95  $status->success[$index] = false;
96  ++$status->failCount;
97  if ( !$ignoreErrors ) {
98  return $status; // abort
99  }
100  }
101  }
102  // Push the last sub-batch
103  if ( count( $curBatch ) ) {
104  $pPerformOps[] = $curBatch;
105  }
106 
107  if ( $ignoreErrors ) { // treat precheck() fatals as mere warnings
108  $status->setResult( true, $status->value );
109  }
110 
111  // Attempt each operation (in parallel if allowed and possible)...
112  self::runParallelBatches( $pPerformOps, $status );
113 
114  return $status;
115  }
116 
128  protected static function runParallelBatches( array $pPerformOps, StatusValue $status ) {
129  $aborted = false; // set to true on unexpected errors
130  foreach ( $pPerformOps as $performOpsBatch ) {
131  if ( $aborted ) { // check batch op abort flag...
132  // We can't continue (even with $ignoreErrors) as $predicates is wrong.
133  // Log the remaining ops as failed for recovery...
134  foreach ( $performOpsBatch as $i => $fileOp ) {
135  $status->success[$i] = false;
136  ++$status->failCount;
137  $fileOp->logFailure( 'attempt_aborted' );
138  }
139  continue;
140  }
142  $statuses = [];
143  $opHandles = [];
144  // Get the backend; all sub-batch ops belong to a single backend
146  $backend = reset( $performOpsBatch )->getBackend();
147  // Get the operation handles or actually do it if there is just one.
148  // If attemptAsync() returns a StatusValue, it was either due to an error
149  // or the backend does not support async ops and did it synchronously.
150  foreach ( $performOpsBatch as $i => $fileOp ) {
151  if ( !isset( $status->success[$i] ) ) { // didn't already fail in precheck()
152  // Parallel ops may be disabled in config due to missing dependencies,
153  // (e.g. needing popen()). When they are, $performOpsBatch has size 1.
154  $subStatus = ( count( $performOpsBatch ) > 1 )
155  ? $fileOp->attemptAsync()
156  : $fileOp->attempt();
157  if ( $subStatus->value instanceof FileBackendStoreOpHandle ) {
158  $opHandles[$i] = $subStatus->value; // deferred
159  } else {
160  $statuses[$i] = $subStatus; // done already
161  }
162  }
163  }
164  // Try to do all the operations concurrently...
165  $statuses += $backend->executeOpHandlesInternal( $opHandles );
166  // Marshall and merge all the responses (blocking)...
167  foreach ( $performOpsBatch as $i => $fileOp ) {
168  if ( !isset( $status->success[$i] ) ) { // didn't already fail in precheck()
169  $subStatus = $statuses[$i];
170  $status->merge( $subStatus );
171  if ( $subStatus->isOK() ) {
172  $status->success[$i] = true;
173  ++$status->successCount;
174  } else {
175  $status->success[$i] = false;
176  ++$status->failCount;
177  $aborted = true; // set abort flag; we can't continue
178  }
179  }
180  }
181  }
182  }
183 }
StatusValue
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Definition: StatusValue.php:43
StatusValue\merge
merge( $other, $overwriteValue=false)
Merge another status object into this one.
Definition: StatusValue.php:278
StatusValue\newGood
static newGood( $value=null)
Factory function for good results.
Definition: StatusValue.php:82
FileOpBatch
Helper class for representing batch file operations.
Definition: FileOpBatch.php:33
FileBackendStoreOpHandle
FileBackendStore helper class for performing asynchronous file operations.
Definition: FileBackendStoreOpHandle.php:32
FileOpBatch\attempt
static attempt(array $performOps, array $opts)
Attempt to perform a series of file operations.
Definition: FileOpBatch.php:54
FileOpBatch\runParallelBatches
static runParallelBatches(array $pPerformOps, StatusValue $status)
Attempt a list of file operations sub-batches in series.
Definition: FileOpBatch.php:128
FileOpBatch\MAX_BATCH_SIZE
const MAX_BATCH_SIZE
Definition: FileOpBatch.php:35
FileOp\newDependencies
static newDependencies()
Get a new empty dependency tracking array for paths read/written to.
Definition: FileOp.php:157
FileOp\newPredicates
static newPredicates()
Get a new empty predicates array for precheck()
Definition: FileOp.php:148