Code Coverage
 
Lines
Functions and Methods
Classes and Traits
Total
69.85% covered (warning)
69.85%
498 / 713
48.05% covered (danger)
48.05%
37 / 77
CRAP
0.00% covered (danger)
0.00%
0 / 1
FileBackendStore
69.85% covered (warning)
69.85%
498 / 713
48.05% covered (danger)
48.05%
37 / 77
2316.49
0.00% covered (danger)
0.00%
0 / 1
 __construct
100.00% covered (success)
100.00%
6 / 6
100.00% covered (success)
100.00%
1 / 1
1
 maxFileSizeInternal
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 isPathUsableInternal
n/a
0 / 0
n/a
0 / 0
0
 createInternal
77.78% covered (warning)
77.78%
7 / 9
0.00% covered (danger)
0.00%
0 / 1
3.10
 doCreateInternal
n/a
0 / 0
n/a
0 / 0
0
 storeInternal
77.78% covered (warning)
77.78%
7 / 9
0.00% covered (danger)
0.00%
0 / 1
3.10
 doStoreInternal
n/a
0 / 0
n/a
0 / 0
0
 copyInternal
100.00% covered (success)
100.00%
6 / 6
100.00% covered (success)
100.00%
1 / 1
2
 doCopyInternal
n/a
0 / 0
n/a
0 / 0
0
 deleteInternal
100.00% covered (success)
100.00%
5 / 5
100.00% covered (success)
100.00%
1 / 1
1
 doDeleteInternal
n/a
0 / 0
n/a
0 / 0
0
 moveInternal
100.00% covered (success)
100.00%
7 / 7
100.00% covered (success)
100.00%
1 / 1
2
 doMoveInternal
n/a
0 / 0
n/a
0 / 0
0
 describeInternal
100.00% covered (success)
100.00%
7 / 7
100.00% covered (success)
100.00%
1 / 1
2
 doDescribeInternal
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 nullInternal
0.00% covered (danger)
0.00%
0 / 1
0.00% covered (danger)
0.00%
0 / 1
2
 concatenate
100.00% covered (success)
100.00%
11 / 11
100.00% covered (success)
100.00%
1 / 1
3
 doConcatenate
56.10% covered (warning)
56.10%
23 / 41
0.00% covered (danger)
0.00%
0 / 1
24.19
 doPrepare
69.23% covered (warning)
69.23%
9 / 13
0.00% covered (danger)
0.00%
0 / 1
4.47
 doPrepareInternal
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 doSecure
53.85% covered (warning)
53.85%
7 / 13
0.00% covered (danger)
0.00%
0 / 1
5.57
 doSecureInternal
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 doPublish
53.85% covered (warning)
53.85%
7 / 13
0.00% covered (danger)
0.00%
0 / 1
5.57
 doPublishInternal
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 doClean
69.23% covered (warning)
69.23%
18 / 26
0.00% covered (danger)
0.00%
0 / 1
11.36
 doCleanInternal
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 fileExists
100.00% covered (success)
100.00%
5 / 5
100.00% covered (success)
100.00%
1 / 1
3
 getFileTimestamp
100.00% covered (success)
100.00%
5 / 5
100.00% covered (success)
100.00%
1 / 1
2
 getFileSize
100.00% covered (success)
100.00%
5 / 5
100.00% covered (success)
100.00%
1 / 1
2
 getFileStat
96.15% covered (success)
96.15%
25 / 26
0.00% covered (danger)
0.00%
0 / 1
17
 ingestFreshFileStats
75.56% covered (warning)
75.56%
34 / 45
0.00% covered (danger)
0.00%
0 / 1
7.72
 doGetFileStat
n/a
0 / 0
n/a
0 / 0
0
 getFileContentsMulti
100.00% covered (success)
100.00%
7 / 7
100.00% covered (success)
100.00%
1 / 1
3
 doGetFileContentsMulti
100.00% covered (success)
100.00%
9 / 9
100.00% covered (success)
100.00%
1 / 1
4
 getFileXAttributes
0.00% covered (danger)
0.00%
0 / 25
0.00% covered (danger)
0.00%
0 / 1
56
 doGetFileXAttributes
0.00% covered (danger)
0.00%
0 / 1
0.00% covered (danger)
0.00%
0 / 1
2
 getFileSha1Base36
66.67% covered (warning)
66.67%
16 / 24
0.00% covered (danger)
0.00%
0 / 1
8.81
 doGetFileSha1Base36
80.00% covered (warning)
80.00%
4 / 5
0.00% covered (danger)
0.00%
0 / 1
4.13
 getFileProps
100.00% covered (success)
100.00%
3 / 3
100.00% covered (success)
100.00%
1 / 1
2
 getLocalReferenceMulti
95.65% covered (success)
95.65%
22 / 23
0.00% covered (danger)
0.00%
0 / 1
8
 doGetLocalReferenceMulti
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 getLocalCopyMulti
100.00% covered (success)
100.00%
3 / 3
100.00% covered (success)
100.00%
1 / 1
1
 doGetLocalCopyMulti
n/a
0 / 0
n/a
0 / 0
0
 getFileHttpUrl
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 streamFile
70.00% covered (warning)
70.00%
7 / 10
0.00% covered (danger)
0.00%
0 / 1
4.43
 doStreamFile
100.00% covered (success)
100.00%
16 / 16
100.00% covered (success)
100.00%
1 / 1
5
 directoryExists
25.00% covered (danger)
25.00%
4 / 16
0.00% covered (danger)
0.00%
0 / 1
21.19
 doDirectoryExists
n/a
0 / 0
n/a
0 / 0
0
 getDirectoryList
44.44% covered (danger)
44.44%
4 / 9
0.00% covered (danger)
0.00%
0 / 1
4.54
 getDirectoryListInternal
n/a
0 / 0
n/a
0 / 0
0
 getFileList
55.56% covered (warning)
55.56%
5 / 9
0.00% covered (danger)
0.00%
0 / 1
3.79
 getFileListInternal
n/a
0 / 0
n/a
0 / 0
0
 getOperationsInternal
94.44% covered (success)
94.44%
17 / 18
0.00% covered (danger)
0.00%
0 / 1
3.00
 getPathsToLockForOpsInternal
100.00% covered (success)
100.00%
10 / 10
100.00% covered (success)
100.00%
1 / 1
2
 getScopedLocksForOps
0.00% covered (danger)
0.00%
0 / 2
0.00% covered (danger)
0.00%
0 / 1
2
 doOperationsInternal
76.67% covered (warning)
76.67%
23 / 30
0.00% covered (danger)
0.00%
0 / 1
7.62
 doQuickOperationsInternal
75.76% covered (warning)
75.76%
25 / 33
0.00% covered (danger)
0.00%
0 / 1
11.42
 executeOpHandlesInternal
50.00% covered (danger)
50.00%
5 / 10
0.00% covered (danger)
0.00%
0 / 1
8.12
 doExecuteOpHandlesInternal
66.67% covered (warning)
66.67%
2 / 3
0.00% covered (danger)
0.00%
0 / 1
2.15
 sanitizeOpHeaders
100.00% covered (success)
100.00%
14 / 14
100.00% covered (success)
100.00%
1 / 1
7
 preloadCache
100.00% covered (success)
100.00%
6 / 6
100.00% covered (success)
100.00%
1 / 1
2
 clearCache
80.00% covered (warning)
80.00%
8 / 10
0.00% covered (danger)
0.00%
0 / 1
4.13
 doClearCache
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 preloadFileStat
71.43% covered (warning)
71.43%
5 / 7
0.00% covered (danger)
0.00%
0 / 1
3.21
 doGetFileStatMulti
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 directoriesAreVirtual
n/a
0 / 0
n/a
0 / 0
0
 isValidShortContainerName
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
2
 isValidContainerName
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 resolveStoragePath
100.00% covered (success)
100.00%
13 / 13
100.00% covered (success)
100.00%
1 / 1
7
 resolveStoragePathReal
100.00% covered (success)
100.00%
4 / 4
100.00% covered (success)
100.00%
1 / 1
3
 getContainerShard
23.08% covered (danger)
23.08%
3 / 13
0.00% covered (danger)
0.00%
0 / 1
29.30
 isSingleShardPathInternal
0.00% covered (danger)
0.00%
0 / 2
0.00% covered (danger)
0.00%
0 / 1
2
 getContainerHashLevels
25.00% covered (danger)
25.00%
2 / 8
0.00% covered (danger)
0.00%
0 / 1
21.19
 getContainerSuffixes
0.00% covered (danger)
0.00%
0 / 7
0.00% covered (danger)
0.00%
0 / 1
12
 fullContainerName
66.67% covered (warning)
66.67%
2 / 3
0.00% covered (danger)
0.00%
0 / 1
2.15
 resolveContainerName
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 resolveContainerPath
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 containerCacheKey
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 setContainerCache
0.00% covered (danger)
0.00%
0 / 4
0.00% covered (danger)
0.00%
0 / 1
6
 deleteContainerCache
25.00% covered (danger)
25.00%
1 / 4
0.00% covered (danger)
0.00%
0 / 1
3.69
 primeContainerCache
94.12% covered (success)
94.12%
16 / 17
0.00% covered (danger)
0.00%
0 / 1
7.01
 doPrimeContainerCache
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 fileCacheKey
100.00% covered (success)
100.00%
1 / 1
100.00% covered (success)
100.00%
1 / 1
1
 setFileCache
60.00% covered (warning)
60.00%
6 / 10
0.00% covered (danger)
0.00%
0 / 1
3.58
 deleteFileCache
42.86% covered (danger)
42.86%
3 / 7
0.00% covered (danger)
0.00%
0 / 1
4.68
 primeFileCache
46.67% covered (danger)
46.67%
14 / 30
0.00% covered (danger)
0.00%
0 / 1
29.36
 normalizeXAttributes
0.00% covered (danger)
0.00%
0 / 6
0.00% covered (danger)
0.00%
0 / 1
12
 setConcurrencyFlags
50.00% covered (danger)
50.00%
4 / 8
0.00% covered (danger)
0.00%
0 / 1
8.12
 getContentType
100.00% covered (success)
100.00%
4 / 4
100.00% covered (success)
100.00%
1 / 1
4
1<?php
2/**
3 * Base class for all backends using particular storage medium.
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
19 *
20 * @file
21 * @ingroup FileBackend
22 */
23
24use MediaWiki\Json\FormatJson;
25use Wikimedia\AtEase\AtEase;
26use Wikimedia\FileBackend\FileBackend;
27use Wikimedia\ObjectCache\BagOStuff;
28use Wikimedia\ObjectCache\EmptyBagOStuff;
29use Wikimedia\Timestamp\ConvertibleTimestamp;
30
31/**
32 * @brief Base class for all backends using particular storage medium.
33 *
34 * This class defines the methods as abstract that subclasses must implement.
35 * Outside callers should *not* use functions with "Internal" in the name.
36 *
37 * The FileBackend operations are implemented using basic functions
38 * such as storeInternal(), copyInternal(), deleteInternal() and the like.
39 * This class is also responsible for path resolution and sanitization.
40 *
41 * @stable to extend
42 * @ingroup FileBackend
43 * @since 1.19
44 */
45abstract class FileBackendStore extends FileBackend {
46    /** @var WANObjectCache */
47    protected $memCache;
48    /** @var BagOStuff */
49    protected $srvCache;
50    /** @var MapCacheLRU Map of paths to small (RAM/disk) cache items */
51    protected $cheapCache;
52    /** @var MapCacheLRU Map of paths to large (RAM/disk) cache items */
53    protected $expensiveCache;
54
55    /** @var array<string,array> Map of container names to sharding config */
56    protected $shardViaHashLevels = [];
57
58    /** @var callable|null Method to get the MIME type of files */
59    protected $mimeCallback;
60
61    protected $maxFileSize = 32 * 1024 * 1024 * 1024; // integer bytes (32GiB)
62
63    protected const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries
64    protected const CACHE_CHEAP_SIZE = 500; // integer; max entries in "cheap cache"
65    protected const CACHE_EXPENSIVE_SIZE = 5; // integer; max entries in "expensive cache"
66
67    /** @var false Idiom for "no result due to missing file" (since 1.34) */
68    protected const RES_ABSENT = false;
69    /** @var null Idiom for "no result due to I/O errors" (since 1.34) */
70    protected const RES_ERROR = null;
71
72    /** @var string File does not exist according to a normal stat query */
73    protected const ABSENT_NORMAL = 'FNE-N';
74    /** @var string File does not exist according to a "latest"-mode stat query */
75    protected const ABSENT_LATEST = 'FNE-L';
76
77    /**
78     * @see FileBackend::__construct()
79     * Additional $config params include:
80     *   - srvCache     : BagOStuff cache to APC or the like.
81     *   - wanCache     : WANObjectCache object to use for persistent caching.
82     *   - mimeCallback : Callback that takes (storage path, content, file system path) and
83     *                    returns the MIME type of the file or 'unknown/unknown'. The file
84     *                    system path parameter should be used if the content one is null.
85     *
86     * @stable to call
87     *
88     * @param array $config
89     */
90    public function __construct( array $config ) {
91        parent::__construct( $config );
92        $this->mimeCallback = $config['mimeCallback'] ?? null;
93        $this->srvCache = new EmptyBagOStuff(); // disabled by default
94        $this->memCache = WANObjectCache::newEmpty(); // disabled by default
95        $this->cheapCache = new MapCacheLRU( self::CACHE_CHEAP_SIZE );
96        $this->expensiveCache = new MapCacheLRU( self::CACHE_EXPENSIVE_SIZE );
97    }
98
99    /**
100     * Get the maximum allowable file size given backend
101     * medium restrictions and basic performance constraints.
102     * Do not call this function from places outside FileBackend and FileOp.
103     *
104     * @return int Bytes
105     */
106    final public function maxFileSizeInternal() {
107        return min( $this->maxFileSize, PHP_INT_MAX );
108    }
109
110    /**
111     * Check if a file can be created or changed at a given storage path in the backend
112     *
113     * FS backends should check that the parent directory exists, files can be written
114     * under it, and that any file already there is both readable and writable.
115     * Backends using key/value stores should check if the container exists.
116     *
117     * @param string $storagePath
118     * @return bool
119     */
120    abstract public function isPathUsableInternal( $storagePath );
121
122    /**
123     * Create a file in the backend with the given contents.
124     * This will overwrite any file that exists at the destination.
125     * Do not call this function from places outside FileBackend and FileOp.
126     *
127     * $params include:
128     *   - content     : the raw file contents
129     *   - dst         : destination storage path
130     *   - headers     : HTTP header name/value map
131     *   - async       : StatusValue will be returned immediately if supported.
132     *                   If the StatusValue is OK, then its value field will be
133     *                   set to a FileBackendStoreOpHandle object.
134     *   - dstExists   : Whether a file exists at the destination (optimization).
135     *                   Callers can use "false" if no existing file is being changed.
136     *
137     * @param array $params
138     * @return StatusValue
139     */
140    final public function createInternal( array $params ) {
141        /** @noinspection PhpUnusedLocalVariableInspection */
142        $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
143
144        if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
145            $status = $this->newStatus( 'backend-fail-maxsize',
146                $params['dst'], $this->maxFileSizeInternal() );
147        } else {
148            $status = $this->doCreateInternal( $params );
149            $this->clearCache( [ $params['dst'] ] );
150            if ( $params['dstExists'] ?? true ) {
151                $this->deleteFileCache( $params['dst'] ); // persistent cache
152            }
153        }
154
155        return $status;
156    }
157
158    /**
159     * @see FileBackendStore::createInternal()
160     * @param array $params
161     * @return StatusValue
162     */
163    abstract protected function doCreateInternal( array $params );
164
165    /**
166     * Store a file into the backend from a file on disk.
167     * This will overwrite any file that exists at the destination.
168     * Do not call this function from places outside FileBackend and FileOp.
169     *
170     * $params include:
171     *   - src         : source path on disk
172     *   - dst         : destination storage path
173     *   - headers     : HTTP header name/value map
174     *   - async       : StatusValue will be returned immediately if supported.
175     *                   If the StatusValue is OK, then its value field will be
176     *                   set to a FileBackendStoreOpHandle object.
177     *   - dstExists   : Whether a file exists at the destination (optimization).
178     *                   Callers can use "false" if no existing file is being changed.
179     *
180     * @param array $params
181     * @return StatusValue
182     */
183    final public function storeInternal( array $params ) {
184        /** @noinspection PhpUnusedLocalVariableInspection */
185        $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
186
187        if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
188            $status = $this->newStatus( 'backend-fail-maxsize',
189                $params['dst'], $this->maxFileSizeInternal() );
190        } else {
191            $status = $this->doStoreInternal( $params );
192            $this->clearCache( [ $params['dst'] ] );
193            if ( $params['dstExists'] ?? true ) {
194                $this->deleteFileCache( $params['dst'] ); // persistent cache
195            }
196        }
197
198        return $status;
199    }
200
201    /**
202     * @see FileBackendStore::storeInternal()
203     * @param array $params
204     * @return StatusValue
205     */
206    abstract protected function doStoreInternal( array $params );
207
208    /**
209     * Copy a file from one storage path to another in the backend.
210     * This will overwrite any file that exists at the destination.
211     * Do not call this function from places outside FileBackend and FileOp.
212     *
213     * $params include:
214     *   - src                 : source storage path
215     *   - dst                 : destination storage path
216     *   - ignoreMissingSource : do nothing if the source file does not exist
217     *   - headers             : HTTP header name/value map
218     *   - async               : StatusValue will be returned immediately if supported.
219     *                           If the StatusValue is OK, then its value field will be
220     *                           set to a FileBackendStoreOpHandle object.
221     *   - dstExists           : Whether a file exists at the destination (optimization).
222     *                           Callers can use "false" if no existing file is being changed.
223     *
224     * @param array $params
225     * @return StatusValue
226     */
227    final public function copyInternal( array $params ) {
228        /** @noinspection PhpUnusedLocalVariableInspection */
229        $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
230
231        $status = $this->doCopyInternal( $params );
232        $this->clearCache( [ $params['dst'] ] );
233        if ( $params['dstExists'] ?? true ) {
234            $this->deleteFileCache( $params['dst'] ); // persistent cache
235        }
236
237        return $status;
238    }
239
240    /**
241     * @see FileBackendStore::copyInternal()
242     * @param array $params
243     * @return StatusValue
244     */
245    abstract protected function doCopyInternal( array $params );
246
247    /**
248     * Delete a file at the storage path.
249     * Do not call this function from places outside FileBackend and FileOp.
250     *
251     * $params include:
252     *   - src                 : source storage path
253     *   - ignoreMissingSource : do nothing if the source file does not exist
254     *   - async               : StatusValue will be returned immediately if supported.
255     *                           If the StatusValue is OK, then its value field will be
256     *                           set to a FileBackendStoreOpHandle object.
257     *
258     * @param array $params
259     * @return StatusValue
260     */
261    final public function deleteInternal( array $params ) {
262        /** @noinspection PhpUnusedLocalVariableInspection */
263        $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
264
265        $status = $this->doDeleteInternal( $params );
266        $this->clearCache( [ $params['src'] ] );
267        $this->deleteFileCache( $params['src'] ); // persistent cache
268        return $status;
269    }
270
271    /**
272     * @see FileBackendStore::deleteInternal()
273     * @param array $params
274     * @return StatusValue
275     */
276    abstract protected function doDeleteInternal( array $params );
277
278    /**
279     * Move a file from one storage path to another in the backend.
280     * This will overwrite any file that exists at the destination.
281     * Do not call this function from places outside FileBackend and FileOp.
282     *
283     * $params include:
284     *   - src                 : source storage path
285     *   - dst                 : destination storage path
286     *   - ignoreMissingSource : do nothing if the source file does not exist
287     *   - headers             : HTTP header name/value map
288     *   - async               : StatusValue will be returned immediately if supported.
289     *                           If the StatusValue is OK, then its value field will be
290     *                           set to a FileBackendStoreOpHandle object.
291     *   - dstExists           : Whether a file exists at the destination (optimization).
292     *                           Callers can use "false" if no existing file is being changed.
293     *
294     * @param array $params
295     * @return StatusValue
296     */
297    final public function moveInternal( array $params ) {
298        /** @noinspection PhpUnusedLocalVariableInspection */
299        $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
300
301        $status = $this->doMoveInternal( $params );
302        $this->clearCache( [ $params['src'], $params['dst'] ] );
303        $this->deleteFileCache( $params['src'] ); // persistent cache
304        if ( $params['dstExists'] ?? true ) {
305            $this->deleteFileCache( $params['dst'] ); // persistent cache
306        }
307
308        return $status;
309    }
310
311    /**
312     * @see FileBackendStore::moveInternal()
313     * @param array $params
314     * @return StatusValue
315     */
316    abstract protected function doMoveInternal( array $params );
317
318    /**
319     * Alter metadata for a file at the storage path.
320     * Do not call this function from places outside FileBackend and FileOp.
321     *
322     * $params include:
323     *   - src           : source storage path
324     *   - headers       : HTTP header name/value map
325     *   - async         : StatusValue will be returned immediately if supported.
326     *                     If the StatusValue is OK, then its value field will be
327     *                     set to a FileBackendStoreOpHandle object.
328     *
329     * @param array $params
330     * @return StatusValue
331     */
332    final public function describeInternal( array $params ) {
333        /** @noinspection PhpUnusedLocalVariableInspection */
334        $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
335
336        if ( count( $params['headers'] ) ) {
337            $status = $this->doDescribeInternal( $params );
338            $this->clearCache( [ $params['src'] ] );
339            $this->deleteFileCache( $params['src'] ); // persistent cache
340        } else {
341            $status = $this->newStatus(); // nothing to do
342        }
343
344        return $status;
345    }
346
347    /**
348     * @see FileBackendStore::describeInternal()
349     * @stable to override
350     * @param array $params
351     * @return StatusValue
352     */
353    protected function doDescribeInternal( array $params ) {
354        return $this->newStatus();
355    }
356
357    /**
358     * No-op file operation that does nothing.
359     * Do not call this function from places outside FileBackend and FileOp.
360     *
361     * @param array $params
362     * @return StatusValue
363     */
364    final public function nullInternal( array $params ) {
365        return $this->newStatus();
366    }
367
368    final public function concatenate( array $params ) {
369        /** @noinspection PhpUnusedLocalVariableInspection */
370        $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
371        $status = $this->newStatus();
372
373        // Try to lock the source files for the scope of this function
374        /** @noinspection PhpUnusedLocalVariableInspection */
375        $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
376        if ( $status->isOK() ) {
377            // Actually do the file concatenation...
378            $start_time = microtime( true );
379            $status->merge( $this->doConcatenate( $params ) );
380            $sec = microtime( true ) - $start_time;
381            if ( !$status->isOK() ) {
382                $this->logger->error( static::class . "-{$this->name}" .
383                    " failed to concatenate " . count( $params['srcs'] ) . " file(s) [$sec sec]" );
384            }
385        }
386
387        return $status;
388    }
389
390    /**
391     * @see FileBackendStore::concatenate()
392     * @stable to override
393     * @param array $params
394     * @return StatusValue
395     */
396    protected function doConcatenate( array $params ) {
397        $status = $this->newStatus();
398        $tmpPath = $params['dst'];
399        unset( $params['latest'] );
400
401        // Check that the specified temp file is valid...
402        AtEase::suppressWarnings();
403        $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 );
404        AtEase::restoreWarnings();
405        if ( !$ok ) { // not present or not empty
406            $status->fatal( 'backend-fail-opentemp', $tmpPath );
407
408            return $status;
409        }
410
411        // Get local FS versions of the chunks needed for the concatenation...
412        $fsFiles = $this->getLocalReferenceMulti( $params );
413        foreach ( $fsFiles as $path => &$fsFile ) {
414            if ( !$fsFile ) { // chunk failed to download?
415                $fsFile = $this->getLocalReference( [ 'src' => $path ] );
416                if ( !$fsFile ) { // retry failed?
417                    $status->fatal(
418                        $fsFile === self::RES_ERROR ? 'backend-fail-read' : 'backend-fail-notexists',
419                        $path
420                    );
421
422                    return $status;
423                }
424            }
425        }
426        unset( $fsFile ); // unset reference so we can reuse $fsFile
427
428        // Get a handle for the destination temp file
429        $tmpHandle = fopen( $tmpPath, 'ab' );
430        if ( $tmpHandle === false ) {
431            $status->fatal( 'backend-fail-opentemp', $tmpPath );
432
433            return $status;
434        }
435
436        // Build up the temp file using the source chunks (in order)...
437        foreach ( $fsFiles as $virtualSource => $fsFile ) {
438            // Get a handle to the local FS version
439            $sourceHandle = fopen( $fsFile->getPath(), 'rb' );
440            if ( $sourceHandle === false ) {
441                fclose( $tmpHandle );
442                $status->fatal( 'backend-fail-read', $virtualSource );
443
444                return $status;
445            }
446            // Append chunk to file (pass chunk size to avoid magic quotes)
447            if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
448                fclose( $sourceHandle );
449                fclose( $tmpHandle );
450                $status->fatal( 'backend-fail-writetemp', $tmpPath );
451
452                return $status;
453            }
454            fclose( $sourceHandle );
455        }
456        if ( !fclose( $tmpHandle ) ) {
457            $status->fatal( 'backend-fail-closetemp', $tmpPath );
458
459            return $status;
460        }
461
462        clearstatcache(); // temp file changed
463
464        return $status;
465    }
466
467    /**
468     * @inheritDoc
469     */
470    final protected function doPrepare( array $params ) {
471        /** @noinspection PhpUnusedLocalVariableInspection */
472        $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
473        $status = $this->newStatus();
474
475        [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
476        if ( $dir === null ) {
477            $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
478
479            return $status; // invalid storage path
480        }
481
482        if ( $shard !== null ) { // confined to a single container/shard
483            $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
484        } else { // directory is on several shards
485            $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
486            [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
487            foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
488                $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
489            }
490        }
491
492        return $status;
493    }
494
495    /**
496     * @see FileBackendStore::doPrepare()
497     * @stable to override
498     * @param string $container
499     * @param string $dir
500     * @param array $params
501     * @return StatusValue Good status without value for success, fatal otherwise.
502     */
503    protected function doPrepareInternal( $container, $dir, array $params ) {
504        return $this->newStatus();
505    }
506
507    final protected function doSecure( array $params ) {
508        /** @noinspection PhpUnusedLocalVariableInspection */
509        $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
510        $status = $this->newStatus();
511
512        [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
513        if ( $dir === null ) {
514            $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
515
516            return $status; // invalid storage path
517        }
518
519        if ( $shard !== null ) { // confined to a single container/shard
520            $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
521        } else { // directory is on several shards
522            $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
523            [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
524            foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
525                $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
526            }
527        }
528
529        return $status;
530    }
531
532    /**
533     * @see FileBackendStore::doSecure()
534     * @stable to override
535     * @param string $container
536     * @param string $dir
537     * @param array $params
538     * @return StatusValue Good status without value for success, fatal otherwise.
539     */
540    protected function doSecureInternal( $container, $dir, array $params ) {
541        return $this->newStatus();
542    }
543
544    final protected function doPublish( array $params ) {
545        /** @noinspection PhpUnusedLocalVariableInspection */
546        $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
547        $status = $this->newStatus();
548
549        [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
550        if ( $dir === null ) {
551            $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
552
553            return $status; // invalid storage path
554        }
555
556        if ( $shard !== null ) { // confined to a single container/shard
557            $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
558        } else { // directory is on several shards
559            $this->logger->debug( __METHOD__ . ": iterating over all container shards." );
560            [ , $shortCont, ] = self::splitStoragePath( $params['dir'] );
561            foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
562                $status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) );
563            }
564        }
565
566        return $status;
567    }
568
569    /**
570     * @see FileBackendStore::doPublish()
571     * @stable to override
572     * @param string $container
573     * @param string $dir
574     * @param array $params
575     * @return StatusValue
576     */
577    protected function doPublishInternal( $container, $dir, array $params ) {
578        return $this->newStatus();
579    }
580
581    final protected function doClean( array $params ) {
582        /** @noinspection PhpUnusedLocalVariableInspection */
583        $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
584        $status = $this->newStatus();
585
586        // Recursive: first delete all empty subdirs recursively
587        if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) {
588            $subDirsRel = $this->getTopDirectoryList( [ 'dir' => $params['dir'] ] );
589            if ( $subDirsRel !== null ) { // no errors
590                foreach ( $subDirsRel as $subDirRel ) {
591                    $subDir = $params['dir'] . "/{$subDirRel}"; // full path
592                    $status->merge( $this->doClean( [ 'dir' => $subDir ] + $params ) );
593                }
594                unset( $subDirsRel ); // free directory for rmdir() on Windows (for FS backends)
595            }
596        }
597
598        [ $fullCont, $dir, $shard ] = $this->resolveStoragePath( $params['dir'] );
599        if ( $dir === null ) {
600            $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
601
602            return $status; // invalid storage path
603        }
604
605        // Attempt to lock this directory...
606        $filesLockEx = [ $params['dir'] ];
607        /** @noinspection PhpUnusedLocalVariableInspection */
608        $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
609        if ( !$status->isOK() ) {
610            return $status; // abort
611        }
612
613        if ( $shard !== null ) { // confined to a single container/shard