MediaWiki  master
FileOpBatch.php
Go to the documentation of this file.
1 <?php
33 class FileOpBatch {
34  /* Timeout related parameters */
35  const MAX_BATCH_SIZE = 1000; // integer
36 
56  public static function attempt( array $performOps, array $opts, FileJournal $journal ) {
57  $status = StatusValue::newGood();
58 
59  $n = count( $performOps );
60  if ( $n > self::MAX_BATCH_SIZE ) {
61  $status->fatal( 'backend-fail-batchsize', $n, self::MAX_BATCH_SIZE );
62 
63  return $status;
64  }
65 
66  $batchId = $journal->getTimestampedUUID();
67  $ignoreErrors = !empty( $opts['force'] );
68  $journaled = empty( $opts['nonJournaled'] );
69  $maxConcurrency = $opts['concurrency'] ?? 1;
70 
71  $entries = []; // file journal entry list
72  $predicates = FileOp::newPredicates(); // account for previous ops in prechecks
73  $curBatch = []; // concurrent FileOp sub-batch accumulation
74  $curBatchDeps = FileOp::newDependencies(); // paths used in FileOp sub-batch
75  $pPerformOps = []; // ordered list of concurrent FileOp sub-batches
76  $lastBackend = null; // last op backend name
77  // Do pre-checks for each operation; abort on failure...
78  foreach ( $performOps as $index => $fileOp ) {
79  $backendName = $fileOp->getBackend()->getName();
80  $fileOp->setBatchId( $batchId ); // transaction ID
81  // Decide if this op can be done concurrently within this sub-batch
82  // or if a new concurrent sub-batch must be started after this one...
83  if ( $fileOp->dependsOn( $curBatchDeps )
84  || count( $curBatch ) >= $maxConcurrency
85  || ( $backendName !== $lastBackend && count( $curBatch ) )
86  ) {
87  $pPerformOps[] = $curBatch; // push this batch
88  $curBatch = []; // start a new sub-batch
89  $curBatchDeps = FileOp::newDependencies();
90  }
91  $lastBackend = $backendName;
92  $curBatch[$index] = $fileOp; // keep index
93  // Update list of affected paths in this batch
94  $curBatchDeps = $fileOp->applyDependencies( $curBatchDeps );
95  // Simulate performing the operation...
96  $oldPredicates = $predicates;
97  $subStatus = $fileOp->precheck( $predicates ); // updates $predicates
98  $status->merge( $subStatus );
99  if ( $subStatus->isOK() ) {
100  if ( $journaled ) { // journal log entries
101  $entries = array_merge( $entries,
102  $fileOp->getJournalEntries( $oldPredicates, $predicates ) );
103  }
104  } else { // operation failed?
105  $status->success[$index] = false;
106  ++$status->failCount;
107  if ( !$ignoreErrors ) {
108  return $status; // abort
109  }
110  }
111  }
112  // Push the last sub-batch
113  if ( count( $curBatch ) ) {
114  $pPerformOps[] = $curBatch;
115  }
116 
117  // Log the operations in the file journal...
118  if ( count( $entries ) ) {
119  $subStatus = $journal->logChangeBatch( $entries, $batchId );
120  if ( !$subStatus->isOK() ) {
121  $status->merge( $subStatus );
122 
123  return $status; // abort
124  }
125  }
126 
127  if ( $ignoreErrors ) { // treat precheck() fatals as mere warnings
128  $status->setResult( true, $status->value );
129  }
130 
131  // Attempt each operation (in parallel if allowed and possible)...
132  self::runParallelBatches( $pPerformOps, $status );
133 
134  return $status;
135  }
136 
148  protected static function runParallelBatches( array $pPerformOps, StatusValue $status ) {
149  $aborted = false; // set to true on unexpected errors
150  foreach ( $pPerformOps as $performOpsBatch ) {
152  if ( $aborted ) { // check batch op abort flag...
153  // We can't continue (even with $ignoreErrors) as $predicates is wrong.
154  // Log the remaining ops as failed for recovery...
155  foreach ( $performOpsBatch as $i => $fileOp ) {
156  $status->success[$i] = false;
157  ++$status->failCount;
158  $performOpsBatch[$i]->logFailure( 'attempt_aborted' );
159  }
160  continue;
161  }
163  $statuses = [];
164  $opHandles = [];
165  // Get the backend; all sub-batch ops belong to a single backend
167  $backend = reset( $performOpsBatch )->getBackend();
168  // Get the operation handles or actually do it if there is just one.
169  // If attemptAsync() returns a StatusValue, it was either due to an error
170  // or the backend does not support async ops and did it synchronously.
171  foreach ( $performOpsBatch as $i => $fileOp ) {
172  if ( !isset( $status->success[$i] ) ) { // didn't already fail in precheck()
173  // Parallel ops may be disabled in config due to missing dependencies,
174  // (e.g. needing popen()). When they are, $performOpsBatch has size 1.
175  $subStatus = ( count( $performOpsBatch ) > 1 )
176  ? $fileOp->attemptAsync()
177  : $fileOp->attempt();
178  if ( $subStatus->value instanceof FileBackendStoreOpHandle ) {
179  $opHandles[$i] = $subStatus->value; // deferred
180  } else {
181  $statuses[$i] = $subStatus; // done already
182  }
183  }
184  }
185  // Try to do all the operations concurrently...
186  $statuses = $statuses + $backend->executeOpHandlesInternal( $opHandles );
187  // Marshall and merge all the responses (blocking)...
188  foreach ( $performOpsBatch as $i => $fileOp ) {
189  if ( !isset( $status->success[$i] ) ) { // didn't already fail in precheck()
190  $subStatus = $statuses[$i];
191  $status->merge( $subStatus );
192  if ( $subStatus->isOK() ) {
193  $status->success[$i] = true;
194  ++$status->successCount;
195  } else {
196  $status->success[$i] = false;
197  ++$status->failCount;
198  $aborted = true; // set abort flag; we can't continue
199  }
200  }
201  }
202  }
203  }
204 }
Helper class for representing batch file operations.
Definition: FileOpBatch.php:33
static newPredicates()
Get a new empty predicates array for precheck()
Definition: FileOp.php:157
static newGood( $value=null)
Factory function for good results.
Definition: StatusValue.php:81
static runParallelBatches(array $pPerformOps, StatusValue $status)
Attempt a list of file operations sub-batches in series.
merge( $other, $overwriteValue=false)
Merge another status object into this one.
logChangeBatch(array $entries, $batchId)
Log changes made by a batch file operation.
static attempt(array $performOps, array $opts, FileJournal $journal)
Attempt to perform a series of file operations.
Definition: FileOpBatch.php:56
static newDependencies()
Get a new empty dependency tracking array for paths read/written to.
Definition: FileOp.php:166
getTimestampedUUID()
Get a statistically unique ID string.
Definition: FileJournal.php:79
FileBackendStore helper class for performing asynchronous file operations.
const MAX_BATCH_SIZE
Definition: FileOpBatch.php:35
Class for handling file operation journaling.
Definition: FileJournal.php:40