Code Coverage
 
Lines
Functions and Methods
Classes and Traits
Total
69.14% covered (warning)
69.14%
242 / 350
70.00% covered (warning)
70.00%
28 / 40
CRAP
0.00% covered (danger)
0.00%
0 / 1
FileBackendMultiWrite
69.14% covered (warning)
69.14%
242 / 350
70.00% covered (warning)
70.00%
28 / 40
609.38
0.00% covered (danger)
0.00%
0 / 1
 __construct
85.71% covered (warning)
85.71%
24 / 28
0.00% covered (danger)
0.00%
0 / 1
10.29
 doOperationsInternal
61.54% covered (warning)
61.54%
32 / 52
0.00% covered (danger)
0.00%
0 / 1
22.62
 consistencyCheck
79.07% covered (warning)
79.07%
34 / 43
0.00% covered (danger)
0.00%
0 / 1
22.31
 accessibilityCheck
77.78% covered (warning)
77.78%
7 / 9
0.00% covered (danger)
0.00%
0 / 1
5.27
 resyncFiles
0.00% covered (danger)
0.00%
0 / 51
0.00% covered (danger)
0.00%
0 / 1
272
 fileStoragePathsForOps
90.91% covered (success)
90.91%
10 / 11
0.00% covered (danger)
0.00%
0 / 1
7.04
 substOpBatchPaths
100.00% covered (success)
100.00%
8 / 8
100.00% covered (success)
100.00%
1 / 1
4
 substOpPaths
100.00% covered (success)
100.00%
2 / 2
100.00% covered (success)
100.00%
1 / 1
1
 substPaths
100.00% covered (success)
100.00%
5 / 5
100.00% covered (success)
100.00%
1 / 1
1
 unsubstPaths
100.00% covered (success)
100.00%
5 / 5
100.00% covered (success)
100.00%
1 / 1
1
 hasVolatileSources
75.00% covered (warning)
75.00%
3 / 4
0.00% covered (danger)
0.00%
0 / 1
4.25
 doQuickOperationsInternal
100.00% covered (success)
100.00%
19 / 19
100.00% covered (success)
100.00%
1 / 1
5
 doPrepare
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 doSecure
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 doPublish
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 doClean
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 doDirectoryOp
68.75% covered (warning)
68.75%
11 / 16
0.00% covered (danger)
0.00%
0 / 1
4.49
 concatenate
100.00% covered (success)
100.00%
5 / 5
100.00% covered (success)
100.00%
1 / 1
1
 fileExists
100.00% covered (success)
100.00%
3 / 3
100.00% covered (success)
100.00%
1 / 1
1
 getFileTimestamp
100.00% covered (success)
100.00%
3 / 3
100.00% covered (success)
100.00%
1 / 1
1
 getFileSize
100.00% covered (success)
100.00%
3 / 3
100.00% covered (success)
100.00%
1 / 1
1
 getFileStat
100.00% covered (success)
100.00%
3 / 3
100.00% covered (success)
100.00%
1 / 1
1
 getFileXAttributes
0.00% covered (danger)
0.00%
0 / 3
0.00% covered (danger)
0.00%
0 / 1
2
 getFileContentsMulti
100.00% covered (success)
100.00%
7 / 7
100.00% covered (success)
100.00%
1 / 1
2
 getFileSha1Base36
100.00% covered (success)
100.00%
3 / 3
100.00% covered (success)
100.00%
1 / 1
1
 getFileProps
100.00% covered (success)
100.00%
3 / 3
100.00% covered (success)
100.00%
1 / 1
1
 streamFile
100.00% covered (success)
100.00%
3 / 3
100.00% covered (success)
100.00%
1 / 1
1
 getLocalReferenceMulti
100.00% covered (success)
100.00%
7 / 7
100.00% covered (success)
100.00%
1 / 1
2
 getLocalCopyMulti
100.00% covered (success)
100.00%
7 / 7
100.00% covered (success)
100.00%
1 / 1
2
 getFileHttpUrl
100.00% covered (success)
100.00%
3 / 3
100.00% covered (success)
100.00%
1 / 1
1
 directoryExists
100.00% covered (success)
100.00%
2 / 2
100.00% covered (success)
100.00%
1 / 1
1
 getDirectoryList
100.00% covered (success)
100.00%
2 / 2
100.00% covered (success)
100.00%
1 / 1
1
 getFileList
75.00% covered (warning)
75.00%
3 / 4
0.00% covered (danger)
0.00%
0 / 1
3.14
 getFileListForWrite
0.00% covered (danger)
0.00%
0 / 8
0.00% covered (danger)
0.00%
0 / 1
20
 getFeatures
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 clearCache
100.00% covered (success)
100.00%
3 / 3
100.00% covered (success)
100.00%
1 / 1
3
 preloadCache
100.00% covered (success)
100.00%
2 / 2
100.00% covered (success)
100.00%
1 / 1
1
 preloadFileStat
0.00% covered (danger)
0.00%
0 / 3
0.00% covered (danger)
0.00%
0 / 1
2
 getScopedLocksForOps
100.00% covered (success)
100.00%
14 / 14
100.00% covered (success)
100.00%
1 / 1
1
 getReadIndexFromParams
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
2
1<?php
2/**
3 * Proxy backend that mirrors writes to several internal backends.
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
19 *
20 * @file
21 * @ingroup FileBackend
22 */
23
24use MediaWiki\Deferred\DeferredUpdates;
25use MediaWiki\Json\FormatJson;
26use Wikimedia\FileBackend\FileBackend;
27use Wikimedia\Timestamp\ConvertibleTimestamp;
28
29/**
30 * @brief Proxy backend that mirrors writes to several internal backends.
31 *
32 * This class defines a multi-write backend. Multiple backends can be
33 * registered to this proxy backend and it will act as a single backend.
34 * Use this when all access to those backends is through this proxy backend.
35 * At least one of the backends must be declared the "master" backend.
36 *
37 * Only use this class when transitioning from one storage system to another.
38 *
39 * Read operations are only done on the 'master' backend for consistency.
40 * Except on getting list of thumbnails for write operations.
41 * Write operations are performed on all backends, starting with the master.
42 * This makes a best-effort to have transactional semantics, but since requests
43 * may sometimes fail, the use of "autoResync" or background scripts to fix
44 * inconsistencies is important.
45 *
46 * @ingroup FileBackend
47 * @since 1.19
48 */
49class FileBackendMultiWrite extends FileBackend {
50    /** @var FileBackendStore[] Prioritized list of FileBackendStore objects */
51    protected $backends = [];
52
53    /** @var int Index of master backend */
54    protected $masterIndex = -1;
55    /** @var int Index of read affinity backend */
56    protected $readIndex = -1;
57
58    /** @var int Bitfield */
59    protected $syncChecks = 0;
60    /** @var string|bool */
61    protected $autoResync = false;
62
63    /** @var bool */
64    protected $asyncWrites = false;
65
66    /** @var int Compare file sizes among backends */
67    private const CHECK_SIZE = 1;
68    /** @var int Compare file mtimes among backends */
69    private const CHECK_TIME = 2;
70    /** @var int Compare file hashes among backends */
71    private const CHECK_SHA1 = 4;
72
73    /**
74     * Construct a proxy backend that consists of several internal backends.
75     * Locking and read-only checks are handled by the proxy backend.
76     *
77     * Additional $config params include:
78     *   - backends       : Array of backend config and multi-backend settings.
79     *                      Each value is the config used in the constructor of a
80     *                      FileBackendStore class, but with these additional settings:
81     *                        - class         : The name of the backend class
82     *                        - isMultiMaster : This must be set for one backend.
83     *                        - readAffinity  : Use this for reads without 'latest' set.
84     *   - syncChecks     : Integer bitfield of internal backend sync checks to perform.
85     *                      Possible bits include the FileBackendMultiWrite::CHECK_* constants.
86     *                      There are constants for SIZE, TIME, and SHA1.
87     *                      The checks are done before allowing any file operations.
88     *   - autoResync     : Automatically resync the clone backends to the master backend
89     *                      when pre-operation sync checks fail. This should only be used
90     *                      if the master backend is stable and not missing any files.
91     *                      Use "conservative" to limit resyncing to copying newer master
92     *                      backend files over older (or non-existing) clone backend files.
93     *                      Cases that cannot be handled will result in operation abortion.
94     *   - replication    : Set to 'async' to defer file operations on the non-master backends.
95     *                      This will apply such updates post-send for web requests. Note that
96     *                      any checks from "syncChecks" are still synchronous.
97     *
98     * @param array $config
99     * @throws LogicException
100     */
101    public function __construct( array $config ) {
102        parent::__construct( $config );
103        $this->syncChecks = $config['syncChecks'] ?? self::CHECK_SIZE;
104        $this->autoResync = $config['autoResync'] ?? false;
105        $this->asyncWrites = isset( $config['replication'] ) && $config['replication'] === 'async';
106        // Construct backends here rather than via registration
107        // to keep these backends hidden from outside the proxy.
108        $namesUsed = [];
109        foreach ( $config['backends'] as $index => $beConfig ) {
110            $name = $beConfig['name'];
111            if ( isset( $namesUsed[$name] ) ) { // don't break FileOp predicates
112                throw new LogicException( "Two or more backends defined with the name $name." );
113            }
114            $namesUsed[$name] = 1;
115            // Alter certain sub-backend settings
116            unset( $beConfig['readOnly'] ); // use proxy backend setting
117            unset( $beConfig['lockManager'] ); // lock under proxy backend
118            $beConfig['domainId'] = $this->domainId; // use the proxy backend wiki ID
119            $beConfig['logger'] = $this->logger; // use the proxy backend logger
120            if ( !empty( $beConfig['isMultiMaster'] ) ) {
121                if ( $this->masterIndex >= 0 ) {
122                    throw new LogicException( 'More than one master backend defined.' );
123                }
124                $this->masterIndex = $index; // this is the "master"
125            }
126            if ( !empty( $beConfig['readAffinity'] ) ) {
127                $this->readIndex = $index; // prefer this for reads
128            }
129            // Create sub-backend object
130            if ( !isset( $beConfig['class'] ) ) {
131                throw new InvalidArgumentException( 'No class given for a backend config.' );
132            }
133            $class = $beConfig['class'];
134            $this->backends[$index] = new $class( $beConfig );
135        }
136        if ( $this->masterIndex < 0 ) { // need backends and must have a master
137            throw new LogicException( 'No master backend defined.' );
138        }
139        if ( $this->readIndex < 0 ) {
140            $this->readIndex = $this->masterIndex; // default
141        }
142    }
143
144    final protected function doOperationsInternal( array $ops, array $opts ) {
145        $status = $this->newStatus();
146
147        $fname = __METHOD__;
148        $mbe = $this->backends[$this->masterIndex]; // convenience
149
150        // Acquire any locks as needed
151        $scopeLock = null;
152        if ( empty( $opts['nonLocking'] ) ) {
153            $scopeLock = $this->getScopedLocksForOps( $ops, $status );
154            if ( !$status->isOK() ) {
155                return $status; // abort
156            }
157        }
158        // Get the list of paths to read/write
159        $relevantPaths = $this->fileStoragePathsForOps( $ops );
160        // Clear any cache entries (after locks acquired)
161        $this->clearCache( $relevantPaths );
162        $opts['preserveCache'] = true; // only locked files are cached
163        // Check if the paths are valid and accessible on all backends
164        $status->merge( $this->accessibilityCheck( $relevantPaths ) );
165        if ( !$status->isOK() ) {
166            return $status; // abort
167        }
168        // Do a consistency check to see if the backends are consistent
169        $syncStatus = $this->consistencyCheck( $relevantPaths );
170        if ( !$syncStatus->isOK() ) {
171            $this->logger->error(
172                "$fname: failed sync check: " . FormatJson::encode( $relevantPaths )
173            );
174            // Try to resync the clone backends to the master on the spot
175            if (
176                $this->autoResync === false ||
177                !$this->resyncFiles( $relevantPaths, $this->autoResync )->isOK()
178            ) {
179                $status->merge( $syncStatus );
180
181                return $status; // abort
182            }
183        }
184        // Actually attempt the operation batch on the master backend
185        $realOps = $this->substOpBatchPaths( $ops, $mbe );
186        $masterStatus = $mbe->doOperations( $realOps, $opts );
187        $status->merge( $masterStatus );
188        // Propagate the operations to the clone backends if there were no unexpected errors
189        // and everything didn't fail due to predicted errors. If $ops only had one operation,
190        // this might avoid backend sync inconsistencies.
191        if ( $masterStatus->isOK() && $masterStatus->successCount > 0 ) {
192            foreach ( $this->backends as $index => $backend ) {
193                if ( $index === $this->masterIndex ) {
194                    continue; // done already
195                }
196
197                $realOps = $this->substOpBatchPaths( $ops, $backend );
198                if ( $this->asyncWrites && !$this->hasVolatileSources( $ops ) ) {
199                    // Bind $scopeLock to the callback to preserve locks
200                    DeferredUpdates::addCallableUpdate(
201                        function () use (
202                            $backend, $realOps, $opts, $scopeLock, $relevantPaths, $fname
203                        ) {
204                            $this->logger->debug(
205                                "$fname: '{$backend->getName()}' async replication; paths: " .
206                                FormatJson::encode( $relevantPaths )
207                            );
208                            $backend->doOperations( $realOps, $opts );
209                        }
210                    );
211                } else {
212                    $this->logger->debug(
213                        "$fname: '{$backend->getName()}' sync replication; paths: " .
214                        FormatJson::encode( $relevantPaths )
215                    );
216                    $status->merge( $backend->doOperations( $realOps, $opts ) );
217                }
218            }
219        }
220        // Make 'success', 'successCount', and 'failCount' fields reflect
221        // the overall operation, rather than all the batches for each backend.
222        // Do this by only using success values from the master backend's batch.
223        $status->success = $masterStatus->success;
224        $status->successCount = $masterStatus->successCount;
225        $status->failCount = $masterStatus->failCount;
226
227        return $status;
228    }
229
230    /**
231     * Check that a set of files are consistent across all internal backends
232     *
233     * This method should only be called if the files are locked or the backend
234     * is in read-only mode
235     *
236     * @param array $paths List of storage paths
237     * @return StatusValue
238     */
239    public function consistencyCheck( array $paths ) {
240        $status = $this->newStatus();
241        if ( $this->syncChecks == 0 || count( $this->backends ) <= 1 ) {
242            return $status; // skip checks
243        }
244
245        // Preload all of the stat info in as few round trips as possible
246        foreach ( $this->backends as $backend ) {
247            $realPaths = $this->substPaths( $paths, $backend );
248            $backend->preloadFileStat( [ 'srcs' => $realPaths, 'latest' => true ] );
249        }
250
251        foreach ( $paths as $path ) {
252            $params = [ 'src' => $path, 'latest' => true ];
253            // Get the state of the file on the master backend
254            $masterBackend = $this->backends[$this->masterIndex];
255            $masterParams = $this->substOpPaths( $params, $masterBackend );
256            $masterStat = $masterBackend->getFileStat( $masterParams );
257            if ( $masterStat === self::STAT_ERROR ) {
258                $status->fatal( 'backend-fail-stat', $path );
259                continue;
260            }
261            if ( $this->syncChecks & self::CHECK_SHA1 ) {
262                $masterSha1 = $masterBackend->getFileSha1Base36( $masterParams );
263                if ( ( $masterSha1 !== false ) !== (bool)$masterStat ) {
264                    $status->fatal( 'backend-fail-hash', $path );
265                    continue;
266                }
267            } else {
268                $masterSha1 = null; // unused
269            }
270
271            // Check if all clone backends agree with the master...
272            foreach ( $this->backends as $index => $cloneBackend ) {
273                if ( $index === $this->masterIndex ) {
274                    continue; // master
275                }
276
277                // Get the state of the file on the clone backend
278                $cloneParams = $this->substOpPaths( $params, $cloneBackend );
279                $cloneStat = $cloneBackend->getFileStat( $cloneParams );
280
281                if ( $masterStat ) {
282                    // File exists in the master backend
283                    if ( !$cloneStat ) {
284                        // File is missing from the clone backend
285                        $status->fatal( 'backend-fail-synced', $path );
286                    } elseif (
287                        ( $this->syncChecks & self::CHECK_SIZE ) &&
288                        $cloneStat['size'] !== $masterStat['size']
289                    ) {
290                        // File in the clone backend is different
291                        $status->fatal( 'backend-fail-synced', $path );
292                    } elseif (
293                        ( $this->syncChecks & self::CHECK_TIME ) &&
294                        abs(
295                            (int)ConvertibleTimestamp::convert( TS_UNIX, $masterStat['mtime'] ) -
296                            (int)ConvertibleTimestamp::convert( TS_UNIX, $cloneStat['mtime'] )
297                        ) > 30
298                    ) {
299                        // File in the clone backend is significantly newer or older
300                        $status->fatal( 'backend-fail-synced', $path );
301                    } elseif (
302                        ( $this->syncChecks & self::CHECK_SHA1 ) &&
303                        $cloneBackend->getFileSha1Base36( $cloneParams ) !== $masterSha1
304                    ) {
305                        // File in the clone backend is different
306                        $status->fatal( 'backend-fail-synced', $path );
307                    }
308                } else {
309                    // File does not exist in the master backend
310                    if ( $cloneStat ) {
311                        // Stray file exists in the clone backend
312                        $status->fatal( 'backend-fail-synced', $path );
313                    }
314                }
315            }
316        }
317
318        return $status;
319    }
320
321    /**
322     * Check that a set of file paths are usable across all internal backends
323     *
324     * @param array $paths List of storage paths
325     * @return StatusValue
326     */
327    public function accessibilityCheck( array $paths ) {
328        $status = $this->newStatus();
329        if ( count( $this->backends ) <= 1 ) {
330            return $status; // skip checks
331        }
332
333        foreach ( $paths as $path ) {
334            foreach ( $this->backends as $backend ) {
335                $realPath = $this->substPaths( $path, $backend );
336                if ( !$backend->isPathUsableInternal( $realPath ) ) {
337                    $status->fatal( 'backend-fail-usable', $path );
338                }
339            }
340        }
341
342        return $status;
343    }
344
345    /**
346     * Check that a set of files are consistent across all internal backends
347     * and re-synchronize those files against the "multi master" if needed.
348     *
349     * This method should only be called if the files are locked
350     *
351     * @param array $paths List of storage paths
352     * @param string|bool $resyncMode False, True, or "conservative"; see __construct()
353     * @return StatusValue
354     */
355    public function resyncFiles( array $paths, $resyncMode = true ) {
356        $status = $this->newStatus();
357
358        $fname = __METHOD__;
359        foreach ( $paths as $path ) {
360            $params = [ 'src' => $path, 'latest' => true ];
361            // Get the state of the file on the master backend
362            $masterBackend = $this->backends[$this->masterIndex];
363            $masterParams = $this->substOpPaths( $params, $masterBackend );
364            $masterPath = $masterParams['src'];
365            $masterStat = $masterBackend->getFileStat( $masterParams );
366            if ( $masterStat === self::STAT_ERROR ) {
367                $status->fatal( 'backend-fail-stat', $path );
368                $this->logger->error( "$fname: file '$masterPath' is not available" );
369                continue;
370            }
371            $masterSha1 = $masterBackend->getFileSha1Base36( $masterParams );
372            if ( ( $masterSha1 !== false ) !== (bool)$masterStat ) {
373                $status->fatal( 'backend-fail-hash', $path );
374                $this->logger->error( "$fname: file '$masterPath' hash does not match stat" );
375                continue;
376            }
377
378            // Check of all clone backends agree with the master...
379            foreach ( $this->backends as $index => $cloneBackend ) {
380                if ( $index === $this->masterIndex ) {
381                    continue; // master
382                }
383
384                // Get the state of the file on the clone backend
385                $cloneParams = $this->substOpPaths( $params, $cloneBackend );
386                $clonePath = $cloneParams['src'];
387                $cloneStat = $cloneBackend->getFileStat( $cloneParams );
388                if ( $cloneStat === self::STAT_ERROR ) {
389                    $status->fatal( 'backend-fail-stat', $path );
390                    $this->logger->error( "$fname: file '$clonePath' is not available" );
391                    continue;
392                }
393                $cloneSha1 = $cloneBackend->getFileSha1Base36( $cloneParams );
394                if ( ( $cloneSha1 !== false ) !== (bool)$cloneStat ) {
395                    $status->fatal( 'backend-fail-hash', $path );
396                    $this->logger->error( "$fname: file '$clonePath' hash does not match stat" );
397                    continue;
398                }
399
400                if ( $masterSha1 === $cloneSha1 ) {
401                    // File is either the same in both backends or absent from both backends
402                    $this->logger->debug( "$fname: file '$clonePath' matches '$masterPath'" );
403                } elseif ( $masterSha1 !== false ) {
404                    // File is either missing from or different in the clone backend
405                    if (
406                        $resyncMode === 'conservative' &&
407                        $cloneStat &&
408                        // @phan-suppress-next-line PhanTypeArraySuspiciousNullable
409                        $cloneStat['mtime'] > $masterStat['mtime']
410                    ) {
411                        // Do not replace files with older ones; reduces the risk of data loss
412                        $status->fatal( 'backend-fail-synced', $path );
413                    } else {
414                        // Copy the master backend file to the clone backend in overwrite mode
415                        $fsFile = $masterBackend->getLocalReference( $masterParams );
416                        $status->merge( $cloneBackend->quickStore( [
417                            'src' => $fsFile,
418                            'dst' => $clonePath
419                        ] ) );
420                    }
421                } elseif ( $masterStat === false ) {
422                    // Stray file exists in the clone backend
423                    if ( $resyncMode === 'conservative' ) {
424                        // Do not delete stray files; reduces the risk of data loss
425                        $status->fatal( 'backend-fail-synced', $path );
426                        $this->logger->error( "$fname: not allowed to delete file '$clonePath'" );
427                    } else {
428                        // Delete the stay file from the clone backend
429                        $status->merge( $cloneBackend->quickDelete( [ 'src' => $clonePath ] ) );
430                    }
431                }
432            }
433        }
434
435        if ( !$status->isOK() ) {
436            $this->logger->error( "$fname: failed to resync: " . FormatJson::encode( $paths ) );
437        }
438
439        return $status;
440    }
441
442    /**
443     * Get a list of file storage paths to read or write for a list of operations
444     *
445     * @param array $ops Same format as doOperations()
446     * @return array List of storage paths to files (does not include directories)
447     */
448    protected function fileStoragePathsForOps( array $ops ) {
449        $paths = [];
450        foreach ( $ops as $op ) {
451            if ( isset( $op['src'] ) ) {
452                // For things like copy/move/delete with "ignoreMissingSource" and there
453                // is no source file, nothing should happen and there should be no errors.
454                if ( empty( $op['ignoreMissingSource'] )
455                    || $this->fileExists( [ 'src' => $op['src'] ] )
456                ) {
457                    $paths[] = $op['src'];
458                }
459            }
460            if ( isset( $op['srcs'] ) ) {
461                $paths = array_merge( $paths, $op['srcs'] );
462            }
463            if ( isset( $op['dst'] ) ) {
464                $paths[] = $op['dst'];
465            }
466        }
467
468        return array_values( array_unique( array_filter( $paths, [ FileBackend::class, 'isStoragePath' ] ) ) );
469    }
470
471    /**
472     * Substitute the backend name in storage path parameters
473     * for a set of operations with that of a given internal backend.
474     *
475     * @param array $ops List of file operation arrays
476     * @param FileBackendStore $backend
477     * @return array
478     */
479    protected function substOpBatchPaths( array $ops, FileBackendStore $backend ) {
480        $newOps = []; // operations