Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
Total | |
25.48% |
241 / 946 |
|
12.50% |
7 / 56 |
CRAP | |
0.00% |
0 / 1 |
SwiftFileBackend | |
25.48% |
241 / 946 |
|
12.50% |
7 / 56 |
35340.35 | |
0.00% |
0 / 1 |
__construct | |
88.89% |
24 / 27 |
|
0.00% |
0 / 1 |
7.07 | |||
setLogger | |
0.00% |
0 / 2 |
|
0.00% |
0 / 1 |
2 | |||
getFeatures | |
0.00% |
0 / 5 |
|
0.00% |
0 / 1 |
2 | |||
resolveContainerPath | |
60.00% |
3 / 5 |
|
0.00% |
0 / 1 |
3.58 | |||
isPathUsableInternal | |
0.00% |
0 / 4 |
|
0.00% |
0 / 1 |
6 | |||
extractMutableContentHeaders | |
95.00% |
19 / 20 |
|
0.00% |
0 / 1 |
12 | |||
extractMetadataHeaders | |
100.00% |
6 / 6 |
|
100.00% |
1 / 1 |
3 | |||
getMetadataFromHeaders | |
100.00% |
5 / 5 |
|
100.00% |
1 / 1 |
2 | |||
doCreateInternal | |
0.00% |
0 / 37 |
|
0.00% |
0 / 1 |
42 | |||
doStoreInternal | |
0.00% |
0 / 57 |
|
0.00% |
0 / 1 |
90 | |||
doCopyInternal | |
0.00% |
0 / 36 |
|
0.00% |
0 / 1 |
56 | |||
doMoveInternal | |
0.00% |
0 / 45 |
|
0.00% |
0 / 1 |
132 | |||
doDeleteInternal | |
0.00% |
0 / 26 |
|
0.00% |
0 / 1 |
42 | |||
doDescribeInternal | |
0.00% |
0 / 35 |
|
0.00% |
0 / 1 |
90 | |||
doPrepareInternal | |
70.00% |
7 / 10 |
|
0.00% |
0 / 1 |
3.24 | |||
doSecureInternal | |
47.06% |
8 / 17 |
|
0.00% |
0 / 1 |
6.37 | |||
doPublishInternal | |
0.00% |
0 / 15 |
|
0.00% |
0 / 1 |
12 | |||
doCleanInternal | |
61.54% |
8 / 13 |
|
0.00% |
0 / 1 |
8.05 | |||
doGetFileStat | |
100.00% |
4 / 4 |
|
100.00% |
1 / 1 |
1 | |||
convertSwiftDate | |
0.00% |
0 / 6 |
|
0.00% |
0 / 1 |
12 | |||
addMissingHashMetadata | |
0.00% |
0 / 30 |
|
0.00% |
0 / 1 |
56 | |||
doGetFileContentsMulti | |
71.79% |
28 / 39 |
|
0.00% |
0 / 1 |
10.82 | |||
doDirectoryExists | |
0.00% |
0 / 5 |
|
0.00% |
0 / 1 |
12 | |||
getDirectoryListInternal | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
getFileListInternal | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
getDirListPageInternal | |
0.00% |
0 / 36 |
|
0.00% |
0 / 1 |
306 | |||
getFileListPageInternal | |
42.86% |
9 / 21 |
|
0.00% |
0 / 1 |
24.11 | |||
buildFileObjectListing | |
0.00% |
0 / 16 |
|
0.00% |
0 / 1 |
56 | |||
loadListingStatInternal | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
doGetFileXAttributes | |
0.00% |
0 / 7 |
|
0.00% |
0 / 1 |
30 | |||
doGetFileSha1base36 | |
0.00% |
0 / 5 |
|
0.00% |
0 / 1 |
12 | |||
doStreamFile | |
0.00% |
0 / 36 |
|
0.00% |
0 / 1 |
156 | |||
doGetLocalCopyMulti | |
65.31% |
32 / 49 |
|
0.00% |
0 / 1 |
18.01 | |||
getFileHttpUrl | |
0.00% |
0 / 34 |
|
0.00% |
0 / 1 |
56 | |||
directoriesAreVirtual | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
headersFromParams | |
75.00% |
3 / 4 |
|
0.00% |
0 / 1 |
2.06 | |||
doExecuteOpHandlesInternal | |
0.00% |
0 / 20 |
|
0.00% |
0 / 1 |
72 | |||
setContainerAccess | |
0.00% |
0 / 14 |
|
0.00% |
0 / 1 |
12 | |||
getContainerStat | |
60.00% |
15 / 25 |
|
0.00% |
0 / 1 |
10.14 | |||
createContainer | |
0.00% |
0 / 18 |
|
0.00% |
0 / 1 |
20 | |||
deleteContainer | |
0.00% |
0 / 12 |
|
0.00% |
0 / 1 |
30 | |||
objectListing | |
65.38% |
17 / 26 |
|
0.00% |
0 / 1 |
12.36 | |||
doPrimeContainerCache | |
50.00% |
1 / 2 |
|
0.00% |
0 / 1 |
2.50 | |||
doGetFileStatMulti | |
42.11% |
16 / 38 |
|
0.00% |
0 / 1 |
34.48 | |||
getStatFromHeaders | |
0.00% |
0 / 9 |
|
0.00% |
0 / 1 |
12 | |||
getAuthentication | |
42.11% |
8 / 19 |
|
0.00% |
0 / 1 |
20.42 | |||
setAuthCreds | |
0.00% |
0 / 9 |
|
0.00% |
0 / 1 |
20 | |||
refreshAuthentication | |
0.00% |
0 / 28 |
|
0.00% |
0 / 1 |
30 | |||
storageUrl | |
0.00% |
0 / 6 |
|
0.00% |
0 / 1 |
12 | |||
authTokenHeaders | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
getCredsCacheKey | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
requestWithAuth | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
requestMultiWithAuth | |
39.13% |
9 / 23 |
|
0.00% |
0 / 1 |
38.29 | |||
getAuthFailureResponse | |
100.00% |
12 / 12 |
|
100.00% |
1 / 1 |
1 | |||
isAuthFailureResponse | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
2 | |||
onError | |
20.00% |
4 / 20 |
|
0.00% |
0 / 1 |
24.43 |
1 | <?php |
2 | /** |
3 | * OpenStack Swift based file backend. |
4 | * |
5 | * This program is free software; you can redistribute it and/or modify |
6 | * it under the terms of the GNU General Public License as published by |
7 | * the Free Software Foundation; either version 2 of the License, or |
8 | * (at your option) any later version. |
9 | * |
10 | * This program is distributed in the hope that it will be useful, |
11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
13 | * GNU General Public License for more details. |
14 | * |
15 | * You should have received a copy of the GNU General Public License along |
16 | * with this program; if not, write to the Free Software Foundation, Inc., |
17 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
18 | * http://www.gnu.org/copyleft/gpl.html |
19 | * |
20 | * @file |
21 | * @ingroup FileBackend |
22 | * @author Russ Nelson |
23 | */ |
24 | |
25 | use MediaWiki\Utils\MWTimestamp; |
26 | use Psr\Log\LoggerInterface; |
27 | use Wikimedia\AtEase\AtEase; |
28 | use Wikimedia\RequestTimeout\TimeoutException; |
29 | |
30 | /** |
31 | * @brief Class for an OpenStack Swift (or Ceph RGW) based file backend. |
32 | * |
33 | * StatusValue messages should avoid mentioning the Swift account name. |
34 | * Likewise, error suppression should be used to avoid path disclosure. |
35 | * |
36 | * @ingroup FileBackend |
37 | * @since 1.19 |
38 | */ |
39 | class SwiftFileBackend extends FileBackendStore { |
40 | private const DEFAULT_HTTP_OPTIONS = [ 'httpVersion' => 'v1.1' ]; |
41 | private const AUTH_FAILURE_ERROR = 'Could not connect due to prior authentication failure'; |
42 | |
43 | /** @var MultiHttpClient */ |
44 | protected $http; |
45 | /** @var int TTL in seconds */ |
46 | protected $authTTL; |
47 | /** @var string Authentication base URL (without version) */ |
48 | protected $swiftAuthUrl; |
49 | /** @var string Override of storage base URL */ |
50 | protected $swiftStorageUrl; |
51 | /** @var string Swift user (account:user) to authenticate as */ |
52 | protected $swiftUser; |
53 | /** @var string Secret key for user */ |
54 | protected $swiftKey; |
55 | /** @var string Shared secret value for making temp URLs */ |
56 | protected $swiftTempUrlKey; |
57 | /** @var string S3 access key (RADOS Gateway) */ |
58 | protected $rgwS3AccessKey; |
59 | /** @var string S3 authentication key (RADOS Gateway) */ |
60 | protected $rgwS3SecretKey; |
61 | /** @var array Additional users (account:user) with read permissions on public containers */ |
62 | protected $readUsers; |
63 | /** @var array Additional users (account:user) with write permissions on public containers */ |
64 | protected $writeUsers; |
65 | /** @var array Additional users (account:user) with read permissions on private containers */ |
66 | protected $secureReadUsers; |
67 | /** @var array Additional users (account:user) with write permissions on private containers */ |
68 | protected $secureWriteUsers; |
69 | |
70 | /** @var BagOStuff */ |
71 | protected $srvCache; |
72 | |
73 | /** @var MapCacheLRU Container stat cache */ |
74 | protected $containerStatCache; |
75 | |
76 | /** @var array|null */ |
77 | protected $authCreds; |
78 | /** @var int|null UNIX timestamp */ |
79 | protected $authErrorTimestamp = null; |
80 | |
81 | /** @var bool Whether the server is an Ceph RGW */ |
82 | protected $isRGW = false; |
83 | |
84 | /** |
85 | * @see FileBackendStore::__construct() |
86 | * @param array $config Params include: |
87 | * - swiftAuthUrl : Swift authentication server URL |
88 | * - swiftUser : Swift user used by MediaWiki (account:username) |
89 | * - swiftKey : Swift authentication key for the above user |
90 | * - swiftAuthTTL : Swift authentication TTL (seconds) |
91 | * - swiftTempUrlKey : Swift "X-Account-Meta-Temp-URL-Key" value on the account. |
92 | * Do not set this until it has been set in the backend. |
93 | * - swiftStorageUrl : Swift storage URL (overrides that of the authentication response). |
94 | * This is useful to set if a TLS proxy is in use. |
95 | * - shardViaHashLevels : Map of container names to sharding config with: |
96 | * - base : base of hash characters, 16 or 36 |
97 | * - levels : the number of hash levels (and digits) |
98 | * - repeat : hash subdirectories are prefixed with all the |
99 | * parent hash directory names (e.g. "a/ab/abc") |
100 | * - cacheAuthInfo : Whether to cache authentication tokens in APC, etc. |
101 | * If those are not available, then the main cache will be used. |
102 | * This is probably insecure in shared hosting environments. |
103 | * - rgwS3AccessKey : Rados Gateway S3 "access key" value on the account. |
104 | * Do not set this until it has been set in the backend. |
105 | * This is used for generating expiring pre-authenticated URLs. |
106 | * Only use this when using rgw and to work around |
107 | * http://tracker.newdream.net/issues/3454. |
108 | * - rgwS3SecretKey : Rados Gateway S3 "secret key" value on the account. |
109 | * Do not set this until it has been set in the backend. |
110 | * This is used for generating expiring pre-authenticated URLs. |
111 | * Only use this when using rgw and to work around |
112 | * http://tracker.newdream.net/issues/3454. |
113 | * - readUsers : Swift users with read access to public containers (account:username) |
114 | * - writeUsers : Swift users with write access to public containers (account:username) |
115 | * - secureReadUsers : Swift users with read access to private containers (account:username) |
116 | * - secureWriteUsers : Swift users with write access to private containers (account:username) |
117 | * - connTimeout : The HTTP connect timeout to use when connecting to Swift, in |
118 | * seconds. |
119 | * - reqTimeout : The HTTP request timeout to use when communicating with Swift, in |
120 | * seconds. |
121 | */ |
122 | public function __construct( array $config ) { |
123 | parent::__construct( $config ); |
124 | // Required settings |
125 | $this->swiftAuthUrl = $config['swiftAuthUrl']; |
126 | $this->swiftUser = $config['swiftUser']; |
127 | $this->swiftKey = $config['swiftKey']; |
128 | // Optional settings |
129 | $this->authTTL = $config['swiftAuthTTL'] ?? 15 * 60; // some sensible number |
130 | $this->swiftTempUrlKey = $config['swiftTempUrlKey'] ?? ''; |
131 | $this->swiftStorageUrl = $config['swiftStorageUrl'] ?? null; |
132 | $this->shardViaHashLevels = $config['shardViaHashLevels'] ?? ''; |
133 | $this->rgwS3AccessKey = $config['rgwS3AccessKey'] ?? ''; |
134 | $this->rgwS3SecretKey = $config['rgwS3SecretKey'] ?? ''; |
135 | |
136 | // HTTP helper client |
137 | $httpOptions = []; |
138 | foreach ( [ 'connTimeout', 'reqTimeout' ] as $optionName ) { |
139 | if ( isset( $config[$optionName] ) ) { |
140 | $httpOptions[$optionName] = $config[$optionName]; |
141 | } |
142 | } |
143 | $this->http = new MultiHttpClient( $httpOptions ); |
144 | $this->http->setLogger( $this->logger ); |
145 | |
146 | // Cache container information to mask latency |
147 | if ( isset( $config['wanCache'] ) && $config['wanCache'] instanceof WANObjectCache ) { |
148 | $this->memCache = $config['wanCache']; |
149 | } |
150 | // Process cache for container info |
151 | $this->containerStatCache = new MapCacheLRU( 300 ); |
152 | // Cache auth token information to avoid RTTs |
153 | if ( !empty( $config['cacheAuthInfo'] ) && isset( $config['srvCache'] ) ) { |
154 | $this->srvCache = $config['srvCache']; |
155 | } else { |
156 | $this->srvCache = new EmptyBagOStuff(); |
157 | } |
158 | $this->readUsers = $config['readUsers'] ?? []; |
159 | $this->writeUsers = $config['writeUsers'] ?? []; |
160 | $this->secureReadUsers = $config['secureReadUsers'] ?? []; |
161 | $this->secureWriteUsers = $config['secureWriteUsers'] ?? []; |
162 | // Per https://docs.openstack.org/swift/latest/overview_large_objects.html |
163 | // we need to split objects if they are larger than 5 GB. Support for |
164 | // splitting objects has not yet been implemented by this class |
165 | // so limit max file size to 5GiB. |
166 | $this->maxFileSize = 5 * 1024 * 1024 * 1024; |
167 | } |
168 | |
169 | public function setLogger( LoggerInterface $logger ) { |
170 | parent::setLogger( $logger ); |
171 | $this->http->setLogger( $logger ); |
172 | } |
173 | |
174 | public function getFeatures() { |
175 | return ( |
176 | self::ATTR_UNICODE_PATHS | |
177 | self::ATTR_HEADERS | |
178 | self::ATTR_METADATA |
179 | ); |
180 | } |
181 | |
182 | protected function resolveContainerPath( $container, $relStoragePath ) { |
183 | if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) { |
184 | return null; // not UTF-8, makes it hard to use CF and the swift HTTP API |
185 | } elseif ( strlen( rawurlencode( $relStoragePath ) ) > 1024 ) { |
186 | return null; // too long for Swift |
187 | } |
188 | |
189 | return $relStoragePath; |
190 | } |
191 | |
192 | public function isPathUsableInternal( $storagePath ) { |
193 | [ $container, $rel ] = $this->resolveStoragePathReal( $storagePath ); |
194 | if ( $rel === null ) { |
195 | return false; // invalid |
196 | } |
197 | |
198 | return is_array( $this->getContainerStat( $container ) ); |
199 | } |
200 | |
201 | /** |
202 | * Filter/normalize a header map to only include mutable "content-"/"x-content-" headers |
203 | * |
204 | * Mutable headers can be changed via HTTP POST even if the file content is the same |
205 | * |
206 | * @see https://docs.openstack.org/api-ref/object-store |
207 | * @param string[] $headers Map of (header => value) for a swift object |
208 | * @return string[] Map of (header => value) for Content-* headers mutable via POST |
209 | */ |
210 | protected function extractMutableContentHeaders( array $headers ) { |
211 | $contentHeaders = []; |
212 | // Normalize casing, and strip out illegal headers |
213 | foreach ( $headers as $name => $value ) { |
214 | $name = strtolower( $name ); |
215 | if ( $name === 'x-delete-at' && is_numeric( $value ) ) { |
216 | // Expects a Unix Epoch date |
217 | $contentHeaders[$name] = $value; |
218 | } elseif ( $name === 'x-delete-after' && is_numeric( $value ) ) { |
219 | // Expects number of minutes time to live. |
220 | $contentHeaders[$name] = $value; |
221 | } elseif ( preg_match( '/^(x-)?content-(?!length$)/', $name ) ) { |
222 | // Only allow content-* and x-content-* headers (but not content-length) |
223 | $contentHeaders[$name] = $value; |
224 | } elseif ( $name === 'content-type' && strlen( $value ) ) { |
225 | // This header can be set to a value but not unset |
226 | $contentHeaders[$name] = $value; |
227 | } |
228 | } |
229 | // By default, Swift has annoyingly low maximum header value limits |
230 | if ( isset( $contentHeaders['content-disposition'] ) ) { |
231 | $maxLength = 255; |
232 | // @note: assume FileBackend::makeContentDisposition() already used |
233 | $offset = $maxLength - strlen( $contentHeaders['content-disposition'] ); |
234 | if ( $offset < 0 ) { |
235 | $pos = strrpos( $contentHeaders['content-disposition'], ';', $offset ); |
236 | $contentHeaders['content-disposition'] = $pos === false |
237 | ? '' |
238 | : trim( substr( $contentHeaders['content-disposition'], 0, $pos ) ); |
239 | } |
240 | } |
241 | |
242 | return $contentHeaders; |
243 | } |
244 | |
245 | /** |
246 | * @see https://docs.openstack.org/api-ref/object-store |
247 | * @param string[] $headers Map of (header => value) for a swift object |
248 | * @return string[] Map of (metadata header name => metadata value) |
249 | */ |
250 | protected function extractMetadataHeaders( array $headers ) { |
251 | $metadataHeaders = []; |
252 | foreach ( $headers as $name => $value ) { |
253 | $name = strtolower( $name ); |
254 | if ( strpos( $name, 'x-object-meta-' ) === 0 ) { |
255 | $metadataHeaders[$name] = $value; |
256 | } |
257 | } |
258 | |
259 | return $metadataHeaders; |
260 | } |
261 | |
262 | /** |
263 | * @see https://docs.openstack.org/api-ref/object-store |
264 | * @param string[] $headers Map of (header => value) for a swift object |
265 | * @return string[] Map of (metadata key name => metadata value) |
266 | */ |
267 | protected function getMetadataFromHeaders( array $headers ) { |
268 | $prefixLen = strlen( 'x-object-meta-' ); |
269 | |
270 | $metadata = []; |
271 | foreach ( $this->extractMetadataHeaders( $headers ) as $name => $value ) { |
272 | $metadata[substr( $name, $prefixLen )] = $value; |
273 | } |
274 | |
275 | return $metadata; |
276 | } |
277 | |
278 | protected function doCreateInternal( array $params ) { |
279 | $status = $this->newStatus(); |
280 | |
281 | [ $dstCont, $dstRel ] = $this->resolveStoragePathReal( $params['dst'] ); |
282 | if ( $dstRel === null ) { |
283 | $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); |
284 | |
285 | return $status; |
286 | } |
287 | |
288 | // Headers that are not strictly a function of the file content |
289 | $mutableHeaders = $this->extractMutableContentHeaders( $params['headers'] ?? [] ); |
290 | // Make sure that the "content-type" header is set to something sensible |
291 | $mutableHeaders['content-type'] |
292 | ??= $this->getContentType( $params['dst'], $params['content'], null ); |
293 | |
294 | $reqs = [ [ |
295 | 'method' => 'PUT', |
296 | 'container' => $dstCont, |
297 | 'relPath' => $dstRel, |
298 | 'headers' => array_merge( |
299 | $mutableHeaders, |
300 | [ |
301 | 'etag' => md5( $params['content'] ), |
302 | 'content-length' => strlen( $params['content'] ), |
303 | 'x-object-meta-sha1base36' => |
304 | Wikimedia\base_convert( sha1( $params['content'] ), 16, 36, 31 ) |
305 | ] |
306 | ), |
307 | 'body' => $params['content'] |
308 | ] ]; |
309 | |
310 | $method = __METHOD__; |
311 | $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) { |
312 | [ $rcode, $rdesc, , $rbody, $rerr ] = $request['response']; |
313 | if ( $rcode === 201 || $rcode === 202 ) { |
314 | // good |
315 | } elseif ( $rcode === 412 ) { |
316 | $status->fatal( 'backend-fail-contenttype', $params['dst'] ); |
317 | } else { |
318 | $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc, $rbody ); |
319 | } |
320 | |
321 | return SwiftFileOpHandle::CONTINUE_IF_OK; |
322 | }; |
323 | |
324 | $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs ); |
325 | if ( !empty( $params['async'] ) ) { // deferred |
326 | $status->value = $opHandle; |
327 | } else { // actually write the object in Swift |
328 | $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) ); |
329 | } |
330 | |
331 | return $status; |
332 | } |
333 | |
334 | protected function doStoreInternal( array $params ) { |
335 | $status = $this->newStatus(); |
336 | |
337 | [ $dstCont, $dstRel ] = $this->resolveStoragePathReal( $params['dst'] ); |
338 | if ( $dstRel === null ) { |
339 | $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); |
340 | |
341 | return $status; |
342 | } |
343 | |
344 | // Open a handle to the source file so that it can be streamed. The size and hash |
345 | // will be computed using the handle. In the off chance that the source file changes |
346 | // during this operation, the PUT will fail due to an ETag mismatch and be aborted. |
347 | AtEase::suppressWarnings(); |
348 | $srcHandle = fopen( $params['src'], 'rb' ); |
349 | AtEase::restoreWarnings(); |
350 | if ( $srcHandle === false ) { // source doesn't exist? |
351 | $status->fatal( 'backend-fail-notexists', $params['src'] ); |
352 | |
353 | return $status; |
354 | } |
355 | |
356 | // Compute the MD5 and SHA-1 hashes in one pass |
357 | $srcSize = fstat( $srcHandle )['size']; |
358 | $md5Context = hash_init( 'md5' ); |
359 | $sha1Context = hash_init( 'sha1' ); |
360 | $hashDigestSize = 0; |
361 | while ( !feof( $srcHandle ) ) { |
362 | $buffer = (string)fread( $srcHandle, 131_072 ); // 128 KiB |
363 | hash_update( $md5Context, $buffer ); |
364 | hash_update( $sha1Context, $buffer ); |
365 | $hashDigestSize += strlen( $buffer ); |
366 | } |
367 | // Reset the handle back to the beginning so that it can be streamed |
368 | rewind( $srcHandle ); |
369 | |
370 | if ( $hashDigestSize !== $srcSize ) { |
371 | $status->fatal( 'backend-fail-hash', $params['src'] ); |
372 | |
373 | return $status; |
374 | } |
375 | |
376 | // Headers that are not strictly a function of the file content |
377 | $mutableHeaders = $this->extractMutableContentHeaders( $params['headers'] ?? [] ); |
378 | // Make sure that the "content-type" header is set to something sensible |
379 | $mutableHeaders['content-type'] |
380 | ??= $this->getContentType( $params['dst'], null, $params['src'] ); |
381 | |
382 | $reqs = [ [ |
383 | 'method' => 'PUT', |
384 | 'container' => $dstCont, |
385 | 'relPath' => $dstRel, |
386 | 'headers' => array_merge( |
387 | $mutableHeaders, |
388 | [ |
389 | 'content-length' => $srcSize, |
390 | 'etag' => hash_final( $md5Context ), |
391 | 'x-object-meta-sha1base36' => |
392 | Wikimedia\base_convert( hash_final( $sha1Context ), 16, 36, 31 ) |
393 | ] |
394 | ), |
395 | 'body' => $srcHandle // resource |
396 | ] ]; |
397 | |
398 | $method = __METHOD__; |
399 | $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) { |
400 | [ $rcode, $rdesc, , $rbody, $rerr ] = $request['response']; |
401 | if ( $rcode === 201 || $rcode === 202 ) { |
402 | // good |
403 | } elseif ( $rcode === 412 ) { |
404 | $status->fatal( 'backend-fail-contenttype', $params['dst'] ); |
405 | } else { |
406 | $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc, $rbody ); |
407 | } |
408 | |
409 | return SwiftFileOpHandle::CONTINUE_IF_OK; |
410 | }; |
411 | |
412 | $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs ); |
413 | $opHandle->resourcesToClose[] = $srcHandle; |
414 | |
415 | if ( !empty( $params['async'] ) ) { // deferred |
416 | $status->value = $opHandle; |
417 | } else { // actually write the object in Swift |
418 | $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) ); |
419 | } |
420 | |
421 | return $status; |
422 | } |
423 | |
424 | protected function doCopyInternal( array $params ) { |
425 | $status = $this->newStatus(); |
426 | |
427 | [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $params['src'] ); |
428 | if ( $srcRel === null ) { |
429 | $status->fatal( 'backend-fail-invalidpath', $params['src'] ); |
430 | |
431 | return $status; |
432 | } |
433 | |
434 | [ $dstCont, $dstRel ] = $this->resolveStoragePathReal( $params['dst'] ); |
435 | if ( $dstRel === null ) { |
436 | $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); |
437 | |
438 | return $status; |
439 | } |
440 | |
441 | $reqs = [ [ |
442 | 'method' => 'PUT', |
443 | 'container' => $dstCont, |
444 | 'relPath' => $dstRel, |
445 | 'headers' => array_merge( |
446 | $this->extractMutableContentHeaders( $params['headers'] ?? [] ), |
447 | [ |
448 | 'x-copy-from' => '/' . rawurlencode( $srcCont ) . '/' . |
449 | str_replace( "%2F", "/", rawurlencode( $srcRel ) ) |
450 | ] |
451 | ) |
452 | ] ]; |
453 | |
454 | $method = __METHOD__; |
455 | $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) { |
456 | [ $rcode, $rdesc, , $rbody, $rerr ] = $request['response']; |
457 | if ( $rcode === 201 ) { |
458 | // good |
459 | } elseif ( $rcode === 404 ) { |
460 | if ( empty( $params['ignoreMissingSource'] ) ) { |
461 | $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); |
462 | } |
463 | } else { |
464 | $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc, $rbody ); |
465 | } |
466 | |
467 | return SwiftFileOpHandle::CONTINUE_IF_OK; |
468 | }; |
469 | |
470 | $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs ); |
471 | if ( !empty( $params['async'] ) ) { // deferred |
472 | $status->value = $opHandle; |
473 | } else { // actually write the object in Swift |
474 | $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) ); |
475 | } |
476 | |
477 | return $status; |
478 | } |
479 | |
480 | protected function doMoveInternal( array $params ) { |
481 | $status = $this->newStatus(); |
482 | |
483 | [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $params['src'] ); |
484 | if ( $srcRel === null ) { |
485 | $status->fatal( 'backend-fail-invalidpath', $params['src'] ); |
486 | |
487 | return $status; |
488 | } |
489 | |
490 | [ $dstCont, $dstRel ] = $this->resolveStoragePathReal( $params['dst'] ); |
491 | if ( $dstRel === null ) { |
492 | $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); |
493 | |
494 | return $status; |
495 | } |
496 | |
497 | $reqs = [ [ |
498 | 'method' => 'PUT', |
499 | 'container' => $dstCont, |
500 | 'relPath' => $dstRel, |
501 | 'headers' => array_merge( |
502 | $this->extractMutableContentHeaders( $params['headers'] ?? [] ), |
503 | [ |
504 | 'x-copy-from' => '/' . rawurlencode( $srcCont ) . '/' . |
505 | str_replace( "%2F", "/", rawurlencode( $srcRel ) ) |
506 | ] |
507 | ) |
508 | ] ]; |
509 | if ( "{$srcCont}/{$srcRel}" !== "{$dstCont}/{$dstRel}" ) { |
510 | $reqs[] = [ |
511 | 'method' => 'DELETE', |
512 | 'container' => $srcCont, |
513 | 'relPath' => $srcRel, |
514 | 'headers' => [] |
515 | ]; |
516 | } |
517 | |
518 | $method = __METHOD__; |
519 | $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) { |
520 | [ $rcode, $rdesc, , $rbody, $rerr ] = $request['response']; |
521 | if ( $request['method'] === 'PUT' && $rcode === 201 ) { |
522 | // good |
523 | } elseif ( $request['method'] === 'DELETE' && $rcode === 204 ) { |
524 | // good |
525 | } elseif ( $rcode === 404 ) { |
526 | if ( empty( $params['ignoreMissingSource'] ) ) { |
527 | $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); |
528 | } else { |
529 | // Leave Status as OK but skip the DELETE request |
530 | return SwiftFileOpHandle::CONTINUE_NO; |
531 | } |
532 | } else { |
533 | $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc, $rbody ); |
534 | } |
535 | |
536 | return SwiftFileOpHandle::CONTINUE_IF_OK; |
537 | }; |
538 | |
539 | $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs ); |
540 | if ( !empty( $params['async'] ) ) { // deferred |
541 | $status->value = $opHandle; |
542 | } else { // actually move the object in Swift |
543 | $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) ); |
544 | } |
545 | |
546 | return $status; |
547 | } |
548 | |
549 | protected function doDeleteInternal( array $params ) { |
550 | $status = $this->newStatus(); |
551 | |
552 | [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $params['src'] ); |
553 | if ( $srcRel === null ) { |
554 | $status->fatal( 'backend-fail-invalidpath', $params['src'] ); |
555 | |
556 | return $status; |
557 | } |
558 | |
559 | $reqs = [ [ |
560 | 'method' => 'DELETE', |
561 | 'container' => $srcCont, |
562 | 'relPath' => $srcRel, |
563 | 'headers' => [] |
564 | ] ]; |
565 | |
566 | $method = __METHOD__; |
567 | $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) { |
568 | [ $rcode, $rdesc, , $rbody, $rerr ] = $request['response']; |
569 | if ( $rcode === 204 ) { |
570 | // good |
571 | } elseif ( $rcode === 404 ) { |
572 | if ( empty( $params['ignoreMissingSource'] ) ) { |
573 | $status->fatal( 'backend-fail-delete', $params['src'] ); |
574 | } |
575 | } else { |
576 | $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc, $rbody ); |
577 | } |
578 | |
579 | return SwiftFileOpHandle::CONTINUE_IF_OK; |
580 | }; |
581 | |
582 | $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs ); |
583 | if ( !empty( $params['async'] ) ) { // deferred |
584 | $status->value = $opHandle; |
585 | } else { // actually delete the object in Swift |
586 | $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) ); |
587 | } |
588 | |
589 | return $status; |
590 | } |
591 | |
592 | protected function doDescribeInternal( array $params ) { |
593 | $status = $this->newStatus(); |
594 | |
595 | [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $params['src'] ); |
596 | if ( $srcRel === null ) { |
597 | $status->fatal( 'backend-fail-invalidpath', $params['src'] ); |
598 | |
599 | return $status; |
600 | } |
601 | |
602 | // Fetch the old object headers/metadata...this should be in stat cache by now |
603 | $stat = $this->getFileStat( [ 'src' => $params['src'], 'latest' => 1 ] ); |
604 | if ( $stat && !isset( $stat['xattr'] ) ) { // older cache entry |
605 | $stat = $this->doGetFileStat( [ 'src' => $params['src'], 'latest' => 1 ] ); |
606 | } |
607 | if ( !$stat ) { |
608 | $status->fatal( 'backend-fail-describe', $params['src'] ); |
609 | |
610 | return $status; |
611 | } |
612 | |
613 | // Swift object POST clears any prior headers, so merge the new and old headers here. |
614 | // Also, during, POST, libcurl adds "Content-Type: application/x-www-form-urlencoded" |
615 | // if "Content-Type" is not set, which would clobber the header value for the object. |
616 | $oldMetadataHeaders = []; |
617 | foreach ( $stat['xattr']['metadata'] as $name => $value ) { |
618 | $oldMetadataHeaders["x-object-meta-$name"] = $value; |
619 | } |
620 | $newContentHeaders = $this->extractMutableContentHeaders( $params['headers'] ?? [] ); |
621 | $oldContentHeaders = $stat['xattr']['headers']; |
622 | |
623 | $reqs = [ [ |
624 | 'method' => 'POST', |
625 | 'container' => $srcCont, |
626 | 'relPath' => $srcRel, |
627 | 'headers' => $oldMetadataHeaders + $newContentHeaders + $oldContentHeaders |
628 | ] ]; |
629 | |
630 | $method = __METHOD__; |
631 | $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) { |
632 | [ $rcode, $rdesc, , $rbody, $rerr ] = $request['response']; |
633 | if ( $rcode === 202 ) { |
634 | // good |
635 | } elseif ( $rcode === 404 ) { |
636 | $status->fatal( 'backend-fail-describe', $params['src'] ); |
637 | } else { |
638 | $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc, $rbody ); |
639 | } |
640 | }; |
641 | |
642 | $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs ); |
643 | if ( !empty( $params['async'] ) ) { // deferred |
644 | $status->value = $opHandle; |
645 | } else { // actually change the object in Swift |
646 | $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) ); |
647 | } |
648 | |
649 | return $status; |
650 | } |
651 | |
652 | /** |
653 | * @inheritDoc |
654 | */ |
655 | protected function doPrepareInternal( $fullCont, $dir, array $params ) { |
656 | $status = $this->newStatus(); |
657 | |
658 | // (a) Check if container already exists |
659 | $stat = $this->getContainerStat( $fullCont ); |
660 | if ( is_array( $stat ) ) { |
661 | return $status; // already there |
662 | } elseif ( $stat === self::RES_ERROR ) { |
663 | $status->fatal( 'backend-fail-internal', $this->name ); |
664 | $this->logger->error( __METHOD__ . ': cannot get container stat' ); |
665 | } else { |
666 | // (b) Create container as needed with proper ACLs |
667 | $params['op'] = 'prepare'; |
668 | $status->merge( $this->createContainer( $fullCont, $params ) ); |
669 | } |
670 | |
671 | return $status; |
672 | } |
673 | |
674 | protected function doSecureInternal( $fullCont, $dir, array $params ) { |
675 | $status = $this->newStatus(); |
676 | if ( empty( $params['noAccess'] ) ) { |
677 | return $status; // nothing to do |
678 | } |
679 | |
680 | $stat = $this->getContainerStat( $fullCont ); |
681 | if ( is_array( $stat ) ) { |
682 | $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] ); |
683 | $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] ); |
684 | // Make container private to end-users... |
685 | $status->merge( $this->setContainerAccess( |
686 | $fullCont, |
687 | $readUsers, |
688 | $writeUsers |
689 | ) ); |
690 | } elseif ( $stat === self::RES_ABSENT ) { |
691 | $status->fatal( 'backend-fail-usable', $params['dir'] ); |
692 | } else { |
693 | $status->fatal( 'backend-fail-internal', $this->name ); |
694 | $this->logger->error( __METHOD__ . ': cannot get container stat' ); |
695 | } |
696 | |
697 | return $status; |
698 | } |
699 | |
700 | protected function doPublishInternal( $fullCont, $dir, array $params ) { |
701 | $status = $this->newStatus(); |
702 | |
703 | $stat = $this->getContainerStat( $fullCont ); |
704 | if ( is_array( $stat ) ) { |
705 | $readUsers = array_merge( $this->readUsers, [ $this->swiftUser, '.r:*' ] ); |
706 | $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] ); |
707 | |
708 | // Make container public to end-users... |
709 | $status->merge( $this->setContainerAccess( |
710 | $fullCont, |
711 | $readUsers, |
712 | $writeUsers |
713 | ) ); |
714 | } elseif ( $stat === self::RES_ABSENT ) { |
715 | $status->fatal( 'backend-fail-usable', $params['dir'] ); |
716 | } else { |
717 | $status->fatal( 'backend-fail-internal', $this->name ); |
718 | $this->logger->error( __METHOD__ . ': cannot get container stat' ); |
719 | } |
720 | |
721 | return $status; |
722 | } |
723 | |
724 | protected function doCleanInternal( $fullCont, $dir, array $params ) { |
725 | $status = $this->newStatus(); |
726 | |
727 | // Only containers themselves can be removed, all else is virtual |
728 | if ( $dir != '' ) { |
729 | return $status; // nothing to do |
730 | } |
731 | |
732 | // (a) Check the container |
733 | $stat = $this->getContainerStat( $fullCont, true ); |
734 | if ( $stat === self::RES_ABSENT ) { |
735 | return $status; // ok, nothing to do |
736 | } elseif ( $stat === self::RES_ERROR ) { |
737 | $status->fatal( 'backend-fail-internal', $this->name ); |
738 | $this->logger->error( __METHOD__ . ': cannot get container stat' ); |
739 | } elseif ( is_array( $stat ) && $stat['count'] == 0 ) { |
740 | // (b) Delete the container if empty |
741 | $params['op'] = 'clean'; |
742 | $status->merge( $this->deleteContainer( $fullCont, $params ) ); |
743 | } |
744 | |
745 | return $status; |
746 | } |
747 | |
748 | protected function doGetFileStat( array $params ) { |
749 | $params = [ 'srcs' => [ $params['src'] ], 'concurrency' => 1 ] + $params; |
750 | unset( $params['src'] ); |
751 | $stats = $this->doGetFileStatMulti( $params ); |
752 | |
753 | return reset( $stats ); |
754 | } |
755 | |
756 | /** |
757 | * Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT"/"2013-05-11T07:37:27.678360Z". |
758 | * Dates might also come in like "2013-05-11T07:37:27.678360" from Swift listings, |
759 | * missing the timezone suffix (though Ceph RGW does not appear to have this bug). |
760 | * |
761 | * @param string $ts |
762 | * @param int $format Output format (TS_* constant) |
763 | * @return string |
764 | * @throws FileBackendError |
765 | */ |
766 | protected function convertSwiftDate( $ts, $format = TS_MW ) { |
767 | try { |
768 | $timestamp = new MWTimestamp( $ts ); |
769 | |
770 | return $timestamp->getTimestamp( $format ); |
771 | } catch ( TimeoutException $e ) { |
772 | throw $e; |
773 | } catch ( Exception $e ) { |
774 | throw new FileBackendError( $e->getMessage() ); |
775 | } |
776 | } |
777 | |
778 | /** |
779 | * Fill in any missing object metadata and save it to Swift |
780 | * |
781 | * @param array $objHdrs Object response headers |
782 | * @param string $path Storage path to object |
783 | * @return array New headers |
784 | */ |
785 | protected function addMissingHashMetadata( array $objHdrs, $path ) { |
786 | if ( isset( $objHdrs['x-object-meta-sha1base36'] ) ) { |
787 | return $objHdrs; // nothing to do |
788 | } |
789 | |
790 | /** @noinspection PhpUnusedLocalVariableInspection */ |
791 | $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); |
792 | $this->logger->error( __METHOD__ . ": {path} was not stored with SHA-1 metadata.", |
793 | [ 'path' => $path ] ); |
794 | |
795 | $objHdrs['x-object-meta-sha1base36'] = false; |
796 | |
797 | // Find prior custom HTTP headers |
798 | $postHeaders = $this->extractMutableContentHeaders( $objHdrs ); |
799 | // Find prior metadata headers |
800 | $postHeaders += $this->extractMetadataHeaders( $objHdrs ); |
801 | |
802 | $status = $this->newStatus(); |
803 | /** @noinspection PhpUnusedLocalVariableInspection */ |
804 | $scopeLockS = $this->getScopedFileLocks( [ $path ], LockManager::LOCK_UW, $status ); |
805 | if ( $status->isOK() ) { |
806 | $tmpFile = $this->getLocalCopy( [ 'src' => $path, 'latest' => 1 ] ); |
807 | if ( $tmpFile ) { |
808 | $hash = $tmpFile->getSha1Base36(); |
809 | if ( $hash !== false ) { |
810 | $objHdrs['x-object-meta-sha1base36'] = $hash; |
811 | // Merge new SHA1 header into the old ones |
812 | $postHeaders['x-object-meta-sha1base36'] = $hash; |
813 | [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $path ); |
814 | [ $rcode ] = $this->requestWithAuth( [ |
815 | 'method' => 'POST', |
816 | 'container' => $srcCont, |
817 | 'relPath' => $srcRel, |
818 | 'headers' => $postHeaders |
819 | ] ); |
820 | if ( $rcode >= 200 && $rcode <= 299 ) { |
821 | $this->deleteFileCache( $path ); |
822 | |
823 | return $objHdrs; // success |
824 | } |
825 | } |
826 | } |
827 | } |
828 | |
829 | $this->logger->error( __METHOD__ . ': unable to set SHA-1 metadata for {path}', |
830 | [ 'path' => $path ] ); |
831 | |
832 | return $objHdrs; // failed |
833 | } |
834 | |
835 | protected function doGetFileContentsMulti( array $params ) { |
836 | $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging |
837 | // Blindly create tmp files and stream to them, catching any exception |
838 | // if the file does not exist. Do not waste time doing file stats here. |
839 | $reqs = []; // (path => op) |
840 | |
841 | // Initial dummy values to preserve path order |
842 | $contents = array_fill_keys( $params['srcs'], self::RES_ERROR ); |
843 | foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch |
844 | [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $path ); |
845 | if ( $srcRel === null ) { |
846 | continue; // invalid storage path |
847 | } |
848 | // Create a new temporary memory file... |
849 | $handle = fopen( 'php://temp', 'wb' ); |
850 | if ( $handle ) { |
851 | $reqs[$path] = [ |
852 | 'method' => 'GET', |
853 | 'container' => $srcCont, |
854 | 'relPath' => $srcRel, |
855 | 'headers' => $this->headersFromParams( $params ), |
856 | 'stream' => $handle, |
857 | ]; |
858 | } |
859 | } |
860 | |
861 | $reqs = $this->requestMultiWithAuth( |
862 | $reqs, |
863 | [ 'maxConnsPerHost' => $params['concurrency'] ] |
864 | ); |
865 | foreach ( $reqs as $path => $op ) { |
866 | [ $rcode, $rdesc, $rhdrs, $rbody, $rerr ] = $op['response']; |
867 | if ( $rcode >= 200 && $rcode <= 299 ) { |
868 | rewind( $op['stream'] ); // start from the beginning |
869 | $content = (string)stream_get_contents( $op['stream'] ); |
870 | $size = strlen( $content ); |
871 | // Make sure that stream finished |
872 | if ( $size === (int)$rhdrs['content-length'] ) { |
873 | $contents[$path] = $content; |
874 | } else { |
875 | $contents[$path] = self::RES_ERROR; |
876 | $rerr = "Got {$size}/{$rhdrs['content-length']} bytes"; |
877 | $this->onError( null, __METHOD__, |
878 | [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc ); |
879 | } |
880 | } elseif ( $rcode === 404 ) { |
881 | $contents[$path] = self::RES_ABSENT; |
882 | } else { |
883 | $contents[$path] = self::RES_ERROR; |
884 | $this->onError( null, __METHOD__, |
885 | [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc, $rbody ); |
886 | } |
887 | fclose( $op['stream'] ); // close open handle |
888 | } |
889 | |
890 | return $contents; |
891 | } |
892 | |
893 | protected function doDirectoryExists( $fullCont, $dir, array $params ) { |
894 | $prefix = ( $dir == '' ) ? null : "{$dir}/"; |
895 | $status = $this->objectListing( $fullCont, 'names', 1, null, $prefix ); |
896 | if ( $status->isOK() ) { |
897 | return ( count( $status->value ) ) > 0; |
898 | } |
899 | |
900 | return self::RES_ERROR; |
901 | } |
902 | |
903 | /** |
904 | * @see FileBackendStore::getDirectoryListInternal() |
905 | * @param string $fullCont |
906 | * @param string $dir |
907 | * @param array $params |
908 | * @return SwiftFileBackendDirList |
909 | */ |
910 | public function getDirectoryListInternal( $fullCont, $dir, array $params ) { |
911 | return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params ); |
912 | } |
913 | |
914 | /** |
915 | * @see FileBackendStore::getFileListInternal() |
916 | * @param string $fullCont |
917 | * @param string $dir |
918 | * @param array $params |
919 | * @return SwiftFileBackendFileList |
920 | */ |
921 | public function getFileListInternal( $fullCont, $dir, array $params ) { |
922 | return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params ); |
923 | } |
924 | |
925 | /** |
926 | * Do not call this function outside of SwiftFileBackendFileList |
927 | * |
928 | * @param string $fullCont Resolved container name |
929 | * @param string $dir Resolved storage directory with no trailing slash |
930 | * @param string|null &$after Resolved container relative path used for continuation paging |
931 | * @param int $limit Max number of items to list |
932 | * @param array $params Parameters for {@link getDirectoryList()} |
933 | * @return string[] List of resolved container relative directories directly under $dir |
934 | * @throws FileBackendError |
935 | */ |
936 | public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) { |
937 | $dirs = []; |
938 | if ( $after === INF ) { |
939 | return $dirs; // nothing more |
940 | } |
941 | |
942 | /** @noinspection PhpUnusedLocalVariableInspection */ |
943 | $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); |
944 | |
945 | $prefix = ( $dir == '' ) ? null : "{$dir}/"; |
946 | // Non-recursive: only list dirs right under $dir |
947 | if ( !empty( $params['topOnly'] ) ) { |
948 | $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' ); |
949 | if ( !$status->isOK() ) { |
950 | throw new FileBackendError( "Iterator page I/O error." ); |
951 | } |
952 | $objects = $status->value; |
953 | // @phan-suppress-next-line PhanTypeSuspiciousNonTraversableForeach |
954 | foreach ( $objects as $object ) { // files and directories |
955 | if ( substr( $object, -1 ) === '/' ) { |
956 | $dirs[] = $object; // directories end in '/' |
957 | } |
958 | } |
959 | } else { |
960 | // Recursive: list all dirs under $dir and its subdirs |
961 | $getParentDir = static function ( $path ) { |
962 | return ( $path !== null && strpos( $path, '/' ) !== false ) ? dirname( $path ) : false; |
963 | }; |
964 | |
965 | // Get directory from last item of prior page |
966 | $lastDir = $getParentDir( $after ); // must be first page |
967 | $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix ); |
968 | |
969 | if ( !$status->isOK() ) { |
970 | throw new FileBackendError( "Iterator page I/O error." ); |
971 | } |
972 | |
973 | $objects = $status->value; |
974 | |
975 | // @phan-suppress-next-line PhanTypeSuspiciousNonTraversableForeach |
976 | foreach ( $objects as $object ) { // files |
977 | $objectDir = $getParentDir( $object ); // directory of object |
978 | |
979 | if ( $objectDir !== false && $objectDir !== $dir ) { |
980 | // Swift stores paths in UTF-8, using binary sorting. |
981 | // See function "create_container_table" in common/db.py. |
982 | // If a directory is not "greater" than the last one, |
983 | // then it was already listed by the calling iterator. |
984 | if ( strcmp( $objectDir, $lastDir ) > 0 ) { |
985 | $pDir = $objectDir; |
986 | do { // add dir and all its parent dirs |
987 | $dirs[] = "{$pDir}/"; |
988 | $pDir = $getParentDir( $pDir ); |
989 | } while ( $pDir !== false |
990 | && strcmp( $pDir, $lastDir ) > 0 // not done already |
991 | && strlen( $pDir ) > strlen( $dir ) // within $dir |
992 | ); |
993 | } |
994 | $lastDir = $objectDir; |
995 | } |
996 | } |
997 | } |
998 | // Page on the unfiltered directory listing (what is returned may be filtered) |
999 | if ( count( $objects ) < $limit ) { |
1000 | $after = INF; // avoid a second RTT |
1001 | } else { |
1002 | $after = end( $objects ); // update last item |
1003 | } |
1004 | |
1005 | return $dirs; |
1006 | } |
1007 | |
1008 | /** |
1009 | * Do not call this function outside of SwiftFileBackendFileList |
1010 | * |
1011 | * @param string $fullCont Resolved container name |
1012 | * @param string $dir Resolved storage directory with no trailing slash |
1013 | * @param string|null &$after Resolved container relative path of file to list items after |
1014 | * @param int $limit Max number of items to list |
1015 | * @param array $params Parameters for {@link getFileList()} |
1016 | * @return array[] List of (name, stat map or null) tuples under $dir |
1017 | * @throws FileBackendError |
1018 | */ |
1019 | public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) { |
1020 | $files = []; // list of (path, stat map or null) entries |
1021 | if ( $after === INF ) { |
1022 | return $files; // nothing more |
1023 | } |
1024 | |
1025 | /** @noinspection PhpUnusedLocalVariableInspection */ |
1026 | $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); |
1027 | |
1028 | $prefix = ( $dir == '' ) ? null : "{$dir}/"; |
1029 | // $objects will contain a list of unfiltered names or stdClass items |
1030 | // Non-recursive: only list files right under $dir |
1031 | if ( !empty( $params['topOnly'] ) ) { |
1032 | if ( !empty( $params['adviseStat'] ) ) { |
1033 | $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix, '/' ); |
1034 | } else { |
1035 | $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' ); |
1036 | } |
1037 | } else { |
1038 | // Recursive: list all files under $dir and its subdirs |
1039 | if ( !empty( $params['adviseStat'] ) ) { |
1040 | $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix ); |
1041 | } else { |
1042 | $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix ); |
1043 | } |
1044 | } |
1045 | |
1046 | // Reformat this list into a list of (name, stat map or null) entries |
1047 | if ( !$status->isOK() ) { |
1048 | throw new FileBackendError( "Iterator page I/O error." ); |
1049 | } |
1050 | |
1051 | $objects = $status->value; |
1052 | $files = $this->buildFileObjectListing( $objects ); |
1053 | |
1054 | // Page on the unfiltered object listing (what is returned may be filtered) |
1055 | if ( count( $objects ) < $limit ) { |
1056 | $after = INF; // avoid a second RTT |
1057 | } else { |
1058 | $after = end( $objects ); // update last item |
1059 | $after = is_object( $after ) ? $after->name : $after; |
1060 | } |
1061 | |
1062 | return $files; |
1063 | } |
1064 | |
1065 | /** |
1066 | * Build a list of file objects, filtering out any directories |
1067 | * and extracting any stat info if provided in $objects |
1068 | * |
1069 | * @param stdClass[]|string[] $objects List of stdClass items or object names |
1070 | * @return array[] List of (name, stat map or null) entries |
1071 | */ |
1072 | private function buildFileObjectListing( array $objects ) { |
1073 | $names = []; |
1074 | foreach ( $objects as $object ) { |
1075 | if ( is_object( $object ) ) { |
1076 | if ( isset( $object->subdir ) || !isset( $object->name ) ) { |
1077 | continue; // virtual directory entry; ignore |
1078 | } |
1079 | $stat = [ |
1080 | // Convert various random Swift dates to TS_MW |
1081 | 'mtime' => $this->convertSwiftDate( $object->last_modified, TS_MW ), |
1082 | 'size' => (int)$object->bytes, |
1083 | 'sha1' => null, |
1084 | // Note: manifest ETags are not an MD5 of the file |
1085 | 'md5' => ctype_xdigit( $object->hash ) ? $object->hash : null, |
1086 | 'latest' => false // eventually consistent |
1087 | ]; |
1088 | $names[] = [ $object->name, $stat ]; |
1089 | } elseif ( substr( $object, -1 ) !== '/' ) { |
1090 | // Omit directories, which end in '/' in listings |
1091 | $names[] = [ $object, null ]; |
1092 | } |
1093 | } |
1094 | |
1095 | return $names; |
1096 | } |
1097 | |
1098 | /** |
1099 | * Do not call this function outside of SwiftFileBackendFileList |
1100 | * |
1101 | * @param string $path Storage path |
1102 | * @param array $val Stat value |
1103 | */ |
1104 | public function loadListingStatInternal( $path, array $val ) { |
1105 | $this->cheapCache->setField( $path, 'stat', $val ); |
1106 | } |
1107 | |
1108 | protected function doGetFileXAttributes( array $params ) { |
1109 | $stat = $this->getFileStat( $params ); |
1110 | // Stat entries filled by file listings don't include metadata/headers |
1111 | if ( is_array( $stat ) && !isset( $stat['xattr'] ) ) { |
1112 | $this->clearCache( [ $params['src'] ] ); |
1113 | $stat = $this->getFileStat( $params ); |
1114 | } |
1115 | |
1116 | if ( is_array( $stat ) ) { |
1117 | return $stat['xattr']; |
1118 | } |
1119 | |
1120 | return $stat === self::RES_ERROR ? self::RES_ERROR : self::RES_ABSENT; |
1121 | } |
1122 | |
1123 | protected function doGetFileSha1base36( array $params ) { |
1124 | // Avoid using stat entries from file listings, which never include the SHA-1 hash. |
1125 | // Also, recompute the hash if it's not part of the metadata headers for some reason. |
1126 | $params['requireSHA1'] = true; |
1127 | |
1128 | $stat = $this->getFileStat( $params ); |
1129 | if ( is_array( $stat ) ) { |
1130 | return $stat['sha1']; |
1131 | } |
1132 | |
1133 | return $stat === self::RES_ERROR ? self::RES_ERROR : self::RES_ABSENT; |
1134 | } |
1135 | |
1136 | protected function doStreamFile( array $params ) { |
1137 | $status = $this->newStatus(); |
1138 | |
1139 | $flags = !empty( $params['headless'] ) ? HTTPFileStreamer::STREAM_HEADLESS : 0; |
1140 | |
1141 | [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $params['src'] ); |
1142 | if ( $srcRel === null ) { |
1143 | HTTPFileStreamer::send404Message( $params['src'], $flags ); |
1144 | $status->fatal( 'backend-fail-invalidpath', $params['src'] ); |
1145 | |
1146 | return $status; |
1147 | } |
1148 | |
1149 | if ( !is_array( $this->getContainerStat( $srcCont ) ) ) { |
1150 | HTTPFileStreamer::send404Message( $params['src'], $flags ); |
1151 | $status->fatal( 'backend-fail-stream', $params['src'] ); |
1152 | |
1153 | return $status; |
1154 | } |
1155 | |
1156 | // If "headers" is set, we only want to send them if the file is there. |
1157 | // Do not bother checking if the file exists if headers are not set though. |
1158 | if ( $params['headers'] && !$this->fileExists( $params ) ) { |
1159 | HTTPFileStreamer::send404Message( $params['src'], $flags ); |
1160 | $status->fatal( 'backend-fail-stream', $params['src'] ); |
1161 | |
1162 | return $status; |
1163 | } |
1164 | |
1165 | // Send the requested additional headers |
1166 | if ( empty( $params['headless'] ) ) { |
1167 | foreach ( $params['headers'] as $header ) { |
1168 | header( $header ); |
1169 | } |
1170 | } |
1171 | |
1172 | if ( empty( $params['allowOB'] ) ) { |
1173 | // Cancel output buffering and gzipping if set |
1174 | ( $this->obResetFunc )(); |
1175 | } |
1176 | |
1177 | $handle = fopen( 'php://output', 'wb' ); |
1178 | [ $rcode, $rdesc, , $rbody, $rerr ] = $this->requestWithAuth( [ |
1179 | 'method' => 'GET', |
1180 | 'container' => $srcCont, |
1181 | 'relPath' => $srcRel, |
1182 | 'headers' => $this->headersFromParams( $params ) + $params['options'], |
1183 | 'stream' => $handle, |
1184 | 'flags' => [ 'relayResponseHeaders' => empty( $params['headless'] ) ] |
1185 | ] ); |
1186 | |
1187 | if ( $rcode >= 200 && $rcode <= 299 ) { |
1188 | // good |
1189 | } elseif ( $rcode === 404 ) { |
1190 | $status->fatal( 'backend-fail-stream', $params['src'] ); |
1191 | // Per T43113, nasty things can happen if bad cache entries get |
1192 | // stuck in cache. It's also possible that this error can come up |
1193 | // with simple race conditions. Clear out the stat cache to be safe. |
1194 | $this->clearCache( [ $params['src'] ] ); |
1195 | $this->deleteFileCache( $params['src'] ); |
1196 | } else { |
1197 | $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc, $rbody ); |
1198 | } |
1199 | |
1200 | return $status; |
1201 | } |
1202 | |
1203 | protected function doGetLocalCopyMulti( array $params ) { |
1204 | $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging |
1205 | // Blindly create tmp files and stream to them, catching any exception |
1206 | // if the file does not exist. Do not waste time doing file stats here. |
1207 | $reqs = []; // (path => op) |
1208 | |
1209 | // Initial dummy values to preserve path order |
1210 | $tmpFiles = array_fill_keys( $params['srcs'], self::RES_ERROR ); |
1211 | foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch |
1212 | [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $path ); |
1213 | if ( $srcRel === null ) { |
1214 | continue; // invalid storage path |
1215 | } |
1216 | // Get source file extension |
1217 | $ext = FileBackend::extensionFromPath( $path ); |
1218 | // Create a new temporary file... |
1219 | $tmpFile = $this->tmpFileFactory->newTempFSFile( 'localcopy_', $ext ); |
1220 | $handle = $tmpFile ? fopen( $tmpFile->getPath(), 'wb' ) : false; |
1221 | if ( $handle ) { |
1222 | $reqs[$path] = [ |
1223 | 'method' => 'GET', |
1224 | 'container' => $srcCont, |
1225 | 'relPath' => $srcRel, |
1226 | 'headers' => $this->headersFromParams( $params ), |
1227 | 'stream' => $handle, |
1228 | ]; |
1229 | $tmpFiles[$path] = $tmpFile; |
1230 | } |
1231 | } |
1232 | |
1233 | // Ceph RADOS Gateway is in use (strong consistency) or X-Newest will be used |
1234 | $latest = ( $this->isRGW || !empty( $params['latest'] ) ); |
1235 | |
1236 | $reqs = $this->requestMultiWithAuth( |
1237 | $reqs, |
1238 | [ 'maxConnsPerHost' => $params['concurrency'] ] |
1239 | ); |
1240 | foreach ( $reqs as $path => $op ) { |
1241 | [ $rcode, $rdesc, $rhdrs, $rbody, $rerr ] = $op['response']; |
1242 | fclose( $op['stream'] ); // close open handle |
1243 | if ( $rcode >= 200 && $rcode <= 299 ) { |
1244 | /** @var TempFSFile $tmpFile */ |
1245 | $tmpFile = $tmpFiles[$path]; |
1246 | // Make sure that the stream finished and fully wrote to disk |
1247 | $size = $tmpFile->getSize(); |
1248 | if ( $size !== (int)$rhdrs['content-length'] ) { |
1249 | $tmpFiles[$path] = self::RES_ERROR; |
1250 | $rerr = "Got {$size}/{$rhdrs['content-length']} bytes"; |
1251 | $this->onError( null, __METHOD__, |
1252 | [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc ); |
1253 | } |
1254 | // Set the file stat process cache in passing |
1255 | $stat = $this->getStatFromHeaders( $rhdrs ); |
1256 | $stat['latest'] = $latest; |
1257 | $this->cheapCache->setField( $path, 'stat', $stat ); |
1258 | } elseif ( $rcode === 404 ) { |
1259 | $tmpFiles[$path] = self::RES_ABSENT; |
1260 | $this->cheapCache->setField( |
1261 | $path, |
1262 | 'stat', |
1263 | $latest ? self::ABSENT_LATEST : self::ABSENT_NORMAL |
1264 | ); |
1265 | } else { |
1266 | $tmpFiles[$path] = self::RES_ERROR; |
1267 | $this->onError( null, __METHOD__, |
1268 | [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc, $rbody ); |
1269 | } |
1270 | } |
1271 | |
1272 | return $tmpFiles; |
1273 | } |
1274 | |
1275 | public function getFileHttpUrl( array $params ) { |
1276 | if ( $this->swiftTempUrlKey != '' || |
1277 | ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' ) |
1278 | ) { |
1279 | [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $params['src'] ); |
1280 | if ( $srcRel === null ) { |
1281 | return self::TEMPURL_ERROR; // invalid path |
1282 | } |
1283 | |
1284 | $auth = $this->getAuthentication(); |
1285 | if ( !$auth ) { |
1286 | return self::TEMPURL_ERROR; |
1287 | } |
1288 | |
1289 | $ttl = $params['ttl'] ?? 86400; |
1290 | $expires = time() + $ttl; |
1291 | |
1292 | if ( $this->swiftTempUrlKey != '' ) { |
1293 | $url = $this->storageUrl( $auth, $srcCont, $srcRel ); |
1294 | // Swift wants the signature based on the unencoded object name |
1295 | $contPath = parse_url( $this->storageUrl( $auth, $srcCont ), PHP_URL_PATH ); |
1296 | $signature = hash_hmac( 'sha1', |
1297 | "GET\n{$expires}\n{$contPath}/{$srcRel}", |
1298 | $this->swiftTempUrlKey |
1299 | ); |
1300 | |
1301 | return "{$url}?temp_url_sig={$signature}&temp_url_expires={$expires}"; |
1302 | } else { // give S3 API URL for rgw |
1303 | // Path for signature starts with the bucket |
1304 | $spath = '/' . rawurlencode( $srcCont ) . '/' . |
1305 | str_replace( '%2F', '/', rawurlencode( $srcRel ) ); |
1306 | // Calculate the hash |
1307 | $signature = base64_encode( hash_hmac( |
1308 | 'sha1', |
1309 | "GET\n\n\n{$expires}\n{$spath}", |
1310 | $this->rgwS3SecretKey, |
1311 | true // raw |
1312 | ) ); |
1313 | // See https://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html. |
1314 | // Note: adding a newline for empty CanonicalizedAmzHeaders does not work. |
1315 | // Note: S3 API is the rgw default; remove the /swift/ URL bit. |
1316 | return str_replace( '/swift/v1', '', $this->storageUrl( $auth ) . $spath ) . |
1317 | '?' . |
1318 | http_build_query( [ |
1319 | 'Signature' => $signature, |
1320 | 'Expires' => $expires, |
1321 | 'AWSAccessKeyId' => $this->rgwS3AccessKey |
1322 | ] ); |
1323 | } |
1324 | } |
1325 | |
1326 | return self::TEMPURL_ERROR; |
1327 | } |
1328 | |
1329 | protected function directoriesAreVirtual() { |
1330 | return true; |
1331 | } |
1332 | |
1333 | /** |
1334 | * Get headers to send to Swift when reading a file based |
1335 | * on a FileBackend params array, e.g. that of getLocalCopy(). |
1336 | * $params is currently only checked for a 'latest' flag. |
1337 | * |
1338 | * @param array $params |
1339 | * @return array |
1340 | */ |
1341 | protected function headersFromParams( array $params ) { |
1342 | $hdrs = []; |
1343 | if ( !empty( $params['latest'] ) ) { |
1344 | $hdrs['x-newest'] = 'true'; |
1345 | } |
1346 | |
1347 | return $hdrs; |
1348 | } |
1349 | |
1350 | protected function doExecuteOpHandlesInternal( array $fileOpHandles ) { |
1351 | /** @var SwiftFileOpHandle[] $fileOpHandles */ |
1352 | '@phan-var SwiftFileOpHandle[] $fileOpHandles'; |
1353 | |
1354 | /** @var StatusValue[] $statuses */ |
1355 | $statuses = []; |
1356 | |
1357 | // Split the HTTP requests into stages that can be done concurrently |
1358 | $httpReqsByStage = []; // map of (stage => index => HTTP request) |
1359 | foreach ( $fileOpHandles as $index => $fileOpHandle ) { |
1360 | $reqs = $fileOpHandle->httpOp; |
1361 | foreach ( $reqs as $stage => $req ) { |
1362 | $httpReqsByStage[$stage][$index] = $req; |
1363 | } |
1364 | $statuses[$index] = $this->newStatus(); |
1365 | } |
1366 | |
1367 | // Run all requests for the first stage, then the next, and so on |
1368 | $reqCount = count( $httpReqsByStage ); |
1369 | for ( $stage = 0; $stage < $reqCount; ++$stage ) { |
1370 | $httpReqs = $this->requestMultiWithAuth( $httpReqsByStage[$stage] ); |
1371 | foreach ( $httpReqs as $index => $httpReq ) { |
1372 | /** @var SwiftFileOpHandle $fileOpHandle */ |
1373 | $fileOpHandle = $fileOpHandles[$index]; |
1374 | // Run the callback for each request of this operation |
1375 | $status = $statuses[$index]; |
1376 | ( $fileOpHandle->callback )( $httpReq, $status ); |
1377 | // On failure, abort all remaining requests for this operation. This is used |
1378 | // in "move" operations to abort the DELETE request if the PUT request fails. |
1379 | if ( |
1380 | !$status->isOK() || |
1381 | $fileOpHandle->state === $fileOpHandle::CONTINUE_NO |
1382 | ) { |
1383 | $stages = count( $fileOpHandle->httpOp ); |
1384 | for ( $s = ( $stage + 1 ); $s < $stages; ++$s ) { |
1385 | unset( $httpReqsByStage[$s][$index] ); |
1386 | } |
1387 | } |
1388 | } |
1389 | } |
1390 | |
1391 | return $statuses; |
1392 | } |
1393 | |
1394 | /** |
1395 | * Set read/write permissions for a Swift container. |
1396 | * |
1397 | * @see http://docs.openstack.org/developer/swift/misc.html#acls |
1398 | * |
1399 | * In general, we don't allow listings to end-users. It's not useful, isn't well-defined |
1400 | * (lists are truncated to 10000 item with no way to page), and is just a performance risk. |
1401 | * |
1402 | * @param string $container Resolved Swift container |
1403 | * @param array $readUsers List of the possible criteria for a request to have |
1404 | * access to read a container. Each item is one of the following formats: |
1405 | * - account:user : Grants access if the request is by the given user |
1406 | * - ".r:<regex>" : Grants access if the request is from a referrer host that |
1407 | * matches the expression and the request is not for a listing. |
1408 | * Setting this to '*' effectively makes a container public. |
1409 | * -".rlistings:<regex>" : Grants access if the request is from a referrer host that |
1410 | * matches the expression and the request is for a listing. |
1411 | * @param array $writeUsers A list of the possible criteria for a request to have |
1412 | * access to write to a container. Each item is of the following format: |
1413 | * - account:user : Grants access if the request is by the given user |
1414 | * @return StatusValue Good status without value for success, fatal otherwise. |
1415 | */ |
1416 | protected function setContainerAccess( $container, array $readUsers, array $writeUsers ) { |
1417 | $status = $this->newStatus(); |
1418 | |
1419 | [ $rcode, , , , ] = $this->requestWithAuth( [ |
1420 | 'method' => 'POST', |
1421 | 'container' => $container, |
1422 | 'headers' => [ |
1423 | 'x-container-read' => implode( ',', $readUsers ), |
1424 | 'x-container-write' => implode( ',', $writeUsers ) |
1425 | ] |
1426 | ] ); |
1427 | |
1428 | if ( $rcode != 204 && $rcode !== 202 ) { |
1429 | $status->fatal( 'backend-fail-internal', $this->name ); |
1430 | $this->logger->error( __METHOD__ . ': unexpected rcode value ({rcode})', |
1431 | [ 'rcode' => $rcode ] ); |
1432 | } |
1433 | |
1434 | return $status; |
1435 | } |
1436 | |
1437 | /** |
1438 | * Get a Swift container stat map, possibly from process cache. |
1439 | * Use $reCache if the file count or byte count is needed. |
1440 | * |
1441 | * @param string $container Container name |
1442 | * @param bool $bypassCache Bypass all caches and load from Swift |
1443 | * @return array|false|null False on 404, null on failure |
1444 | */ |
1445 | protected function getContainerStat( $container, $bypassCache = false ) { |
1446 | /** @noinspection PhpUnusedLocalVariableInspection */ |
1447 | $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" ); |
1448 | |
1449 | if ( $bypassCache ) { // purge cache |
1450 | $this->containerStatCache->clear( $container ); |
1451 | } elseif ( !$this->containerStatCache->hasField( $container, 'stat' ) ) { |
1452 | $this->primeContainerCache( [ $container ] ); // check persistent cache |
1453 | } |
1454 | if ( !$this->containerStatCache->hasField( $container, 'stat' ) ) { |
1455 | [ $rcode, $rdesc, $rhdrs, $rbody, $rerr ] = $this->requestWithAuth( [ |
1456 | 'method' => 'HEAD', |
1457 | 'container' => $container |
1458 | ] ); |
1459 | |
1460 | if ( $rcode === 204 ) { |
1461 | $stat = [ |
1462 | 'count' => $rhdrs['x-container-object-count'], |
1463 | 'bytes' => $rhdrs['x-container-bytes-used'] |
1464 | ]; |
1465 | if ( $bypassCache ) { |
1466 | return $stat; |
1467 | } else { |
1468 | $this->containerStatCache->setField( $container, 'stat', $stat ); // cache it |
1469 | $this->setContainerCache( $container, $stat ); // update persistent cache |
1470 | } |
1471 | } elseif ( $rcode === 404 ) { |
1472 | return self::RES_ABSENT; |
1473 | } else { |
1474 | $this->onError( null, __METHOD__, |
1475 | [ 'cont' => $container ], $rerr, $rcode, $rdesc, $rbody ); |
1476 | |
1477 | return self::RES_ERROR; |
1478 | } |
1479 | } |
1480 | |
1481 | return $this->containerStatCache->getField( $container, 'stat' ); |
1482 | } |
1483 | |
1484 | /** |
1485 | * Create a Swift container |
1486 | * |
1487 | * @param string $container Container name |
1488 | * @param array $params |
1489 | * @return StatusValue Good status without value for success, fatal otherwise. |
1490 | */ |
1491 | protected function createContainer( $container, array $params ) { |
1492 | $status = $this->newStatus(); |
1493 | |
1494 | // @see SwiftFileBackend::setContainerAccess() |
1495 | if ( empty( $params['noAccess'] ) ) { |
1496 | // public |
1497 | $readUsers = array_merge( $this->readUsers, [ '.r:*', $this->swiftUser ] ); |
1498 | $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] ); |
1499 | } else { |
1500 | // private |
1501 | $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] ); |
1502 | $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] ); |
1503 | } |
1504 | |
1505 | [ $rcode, $rdesc, , $rbody, $rerr ] = $this->requestWithAuth( [ |
1506 | 'method' => 'PUT', |
1507 | 'container' => $container, |
1508 | 'headers' => [ |
1509 | 'x-container-read' => implode( ',', $readUsers ), |
1510 | 'x-container-write' => implode( ',', $writeUsers ) |
1511 | ] |
1512 | ] ); |
1513 | |
1514 | if ( $rcode === 201 ) { // new |
1515 | // good |
1516 | } elseif ( $rcode === 202 ) { // already there |
1517 | // this shouldn't really happen, but is OK |
1518 | } else { |
1519 | $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc, $rbody ); |
1520 | } |
1521 | |
1522 | return $status; |
1523 | } |
1524 | |
1525 | /** |
1526 | * Delete a Swift container |
1527 | * |
1528 | * @param string $container Container name |
1529 | * @param array $params |
1530 | * @return StatusValue |
1531 | */ |
1532 | protected function deleteContainer( $container, array $params ) { |
1533 | $status = $this->newStatus(); |
1534 | |
1535 | [ $rcode, $rdesc, , $rbody, $rerr ] = $this->requestWithAuth( [ |
1536 | 'method' => 'DELETE', |
1537 | 'container' => $container |
1538 | ] ); |
1539 | |
1540 | if ( $rcode >= 200 && $rcode <= 299 ) { // deleted |
1541 | $this->containerStatCache->clear( $container ); // purge |
1542 | } elseif ( $rcode === 404 ) { // not there |
1543 | // this shouldn't really happen, but is OK |
1544 | } elseif ( $rcode === 409 ) { // not empty |
1545 | $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); // race? |
1546 | } else { |
1547 | $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc, $rbody ); |
1548 | } |
1549 | |
1550 | return $status; |
1551 | } |
1552 | |
1553 | /** |
1554 | * Get a list of objects under a container. |
1555 | * Either just the names or a list of stdClass objects with details can be returned. |
1556 | * |
1557 | * @param string $fullCont |
1558 | * @param string $type ('info' for a list of object detail maps, 'names' for names only) |
1559 | * @param int $limit |
1560 | * @param string|null $after |
1561 | * @param string|null $prefix |
1562 | * @param string|null $delim |
1563 | * @return StatusValue With the list as value |
1564 | */ |
1565 | private function objectListing( |
1566 | $fullCont, $type, $limit, $after = null, $prefix = null, $delim = null |
1567 | ) { |
1568 | $status = $this->newStatus(); |
1569 | |
1570 | $query = [ 'limit' => $limit ]; |
1571 | if ( $type === 'info' ) { |
1572 | $query['format'] = 'json'; |
1573 | } |
1574 | if ( $after !== null ) { |
1575 | $query['marker'] = $after; |
1576 | } |
1577 | if ( $prefix !== null ) { |
1578 | $query['prefix'] = $prefix; |
1579 | } |
1580 | if ( $delim !== null ) { |
1581 | $query['delimiter'] = $delim; |
1582 | } |
1583 | |
1584 | [ $rcode, $rdesc, , $rbody, $rerr ] = $this->requestWithAuth( [ |
1585 | 'method' => 'GET', |
1586 | 'container' => $fullCont, |
1587 | 'query' => $query, |
1588 | ] ); |
1589 | |
1590 | $params = [ 'cont' => $fullCont, 'prefix' => $prefix, 'delim' => $delim ]; |
1591 | if ( $rcode === 200 ) { // good |
1592 | if ( $type === 'info' ) { |
1593 | $status->value = FormatJson::decode( trim( $rbody ) ); |
1594 | } else { |
1595 | $status->value = explode( "\n", trim( $rbody ) ); |
1596 | } |
1597 | } elseif ( $rcode === 204 ) { |
1598 | $status->value = []; // empty container |
1599 | } elseif ( $rcode === 404 ) { |
1600 | $status->value = []; // no container |
1601 | } else { |
1602 | $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc, $rbody ); |
1603 | } |
1604 | |
1605 | return $status; |
1606 | } |
1607 | |
1608 | protected function doPrimeContainerCache( array $containerInfo ) { |
1609 | foreach ( $containerInfo as $container => $info ) { |
1610 | $this->containerStatCache->setField( $container, 'stat', $info ); |
1611 | } |
1612 | } |
1613 | |
1614 | protected function doGetFileStatMulti( array $params ) { |
1615 | $stats = []; |
1616 | |
1617 | $reqs = []; // (path => op) |
1618 | // (a) Check the containers of the paths... |
1619 | foreach ( $params['srcs'] as $path ) { |
1620 | [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $path ); |
1621 | if ( $srcRel === null ) { |
1622 | // invalid storage path |
1623 | $stats[$path] = self::RES_ERROR; |
1624 | continue; |
1625 | } |
1626 | |
1627 | $cstat = $this->getContainerStat( $srcCont ); |
1628 | if ( $cstat === self::RES_ABSENT ) { |
1629 | $stats[$path] = self::RES_ABSENT; |
1630 | continue; // ok, nothing to do |
1631 | } elseif ( $cstat === self::RES_ERROR ) { |
1632 | $stats[$path] = self::RES_ERROR; |
1633 | continue; |
1634 | } |
1635 | |
1636 | $reqs[$path] = [ |
1637 | 'method' => 'HEAD', |
1638 | 'container' => $srcCont, |
1639 | 'relPath' => $srcRel, |
1640 | 'headers' => $this->headersFromParams( $params ) |
1641 | ]; |
1642 | } |
1643 | |
1644 | // (b) Check the files themselves... |
1645 | $reqs = $this->requestMultiWithAuth( |
1646 | $reqs, |
1647 | [ 'maxConnsPerHost' => $params['concurrency'] ] |
1648 | ); |
1649 | foreach ( $reqs as $path => $op ) { |
1650 | [ $rcode, $rdesc, $rhdrs, $rbody, $rerr ] = $op['response']; |
1651 | if ( $rcode === 200 || $rcode === 204 ) { |
1652 | // Update the object if it is missing some headers |
1653 | if ( !empty( $params['requireSHA1'] ) ) { |
1654 | $rhdrs = $this->addMissingHashMetadata( $rhdrs, $path ); |
1655 | } |
1656 | // Load the stat map from the headers |
1657 | $stat = $this->getStatFromHeaders( $rhdrs ); |
1658 | if ( $this->isRGW ) { |
1659 | $stat['latest'] = true; // strong consistency |
1660 | } |
1661 | } elseif ( $rcode === 404 ) { |
1662 | $stat = self::RES_ABSENT; |
1663 | } else { |
1664 | $stat = self::RES_ERROR; |
1665 | $this->onError( null, __METHOD__, $params, $rerr, $rcode, $rdesc, $rbody ); |
1666 | } |
1667 | $stats[$path] = $stat; |
1668 | } |
1669 | |
1670 | return $stats; |
1671 | } |
1672 | |
1673 | /** |
1674 | * @param array $rhdrs |
1675 | * @return array |
1676 | */ |
1677 | protected function getStatFromHeaders( array $rhdrs ) { |
1678 | // Fetch all of the custom metadata headers |
1679 | $metadata = $this->getMetadataFromHeaders( $rhdrs ); |
1680 | // Fetch all of the custom raw HTTP headers |
1681 | $headers = $this->extractMutableContentHeaders( $rhdrs ); |
1682 | |
1683 | return [ |
1684 | // Convert various random Swift dates to TS_MW |
1685 | 'mtime' => $this->convertSwiftDate( $rhdrs['last-modified'], TS_MW ), |
1686 | // Empty objects actually return no content-length header in Ceph |
1687 | 'size' => isset( $rhdrs['content-length'] ) ? (int)$rhdrs['content-length'] : 0, |
1688 | 'sha1' => $metadata['sha1base36'] ?? null, |
1689 | // Note: manifest ETags are not an MD5 of the file |
1690 | 'md5' => ctype_xdigit( $rhdrs['etag'] ) ? $rhdrs['etag'] : null, |
1691 | 'xattr' => [ 'metadata' => $metadata, 'headers' => $headers ] |
1692 | ]; |
1693 | } |
1694 | |
1695 | /** |
1696 | * Get the cached auth token. |
1697 | * |
1698 | * @return array|null Credential map |
1699 | */ |
1700 | protected function getAuthentication() { |
1701 | if ( $this->authErrorTimestamp !== null ) { |
1702 | $interval = time() - $this->authErrorTimestamp; |
1703 | if ( $interval < 60 ) { |
1704 | $this->logger->debug( |
1705 | 'rejecting request since auth failure occurred {interval} seconds ago', |
1706 | [ 'interval' => $interval ] |
1707 | ); |
1708 | return null; |
1709 | } else { // actually retry this time |
1710 | $this->authErrorTimestamp = null; |
1711 | } |
1712 | } |
1713 | // Authenticate with proxy and get a session key... |
1714 | if ( !$this->authCreds ) { |
1715 | $cacheKey = $this->getCredsCacheKey( $this->swiftUser ); |
1716 | $creds = $this->srvCache->get( $cacheKey ); // credentials |
1717 | // Try to use the credential cache |
1718 | if ( isset( $creds['auth_token'] ) |
1719 | && isset( $creds['storage_url'] ) |
1720 | && isset( $creds['expiry_time'] ) |
1721 | && $creds['expiry_time'] > time() |
1722 | ) { |
1723 | $this->setAuthCreds( $creds ); |
1724 | } else { // cache miss |
1725 | $this->refreshAuthentication(); |
1726 | } |
1727 | } |
1728 | |
1729 | return $this->authCreds; |
1730 | } |
1731 | |
1732 | /** |
1733 | * Update the auth credentials |
1734 | * |
1735 | * @param array|null $creds |
1736 | */ |
1737 | private function setAuthCreds( ?array $creds ) { |
1738 | $this->logger->debug( 'Using auth token with expiry_time={expiry_time}', |
1739 | [ |
1740 | 'expiry_time' => isset( $creds['expiry_time'] ) |
1741 | ? gmdate( 'c', $creds['expiry_time'] ) : 'null' |
1742 | ] |
1743 | ); |
1744 | $this->authCreds = $creds; |
1745 | // Ceph RGW does not use <account> in URLs (OpenStack Swift uses "/v1/<account>") |
1746 | if ( $creds && str_ends_with( $creds['storage_url'], '/v1' ) ) { |
1747 | $this->isRGW = true; // take advantage of strong consistency in Ceph |
1748 | } |
1749 | } |
1750 | |
1751 | /** |
1752 | * Fetch the auth token from the server, without caching. |
1753 | * |
1754 | * @return array|null Credential map |
1755 | */ |
1756 | private function refreshAuthentication() { |
1757 | [ $rcode, , $rhdrs, $rbody, ] = $this->http->run( [ |
1758 | 'method' => 'GET', |
1759 | 'url' => "{$this->swiftAuthUrl}/v1.0", |
1760 | 'headers' => [ |
1761 | 'x-auth-user' => $this->swiftUser, |
1762 | 'x-auth-key' => $this->swiftKey |
1763 | ] |
1764 | ], self::DEFAULT_HTTP_OPTIONS ); |
1765 | |
1766 | if ( $rcode >= 200 && $rcode <= 299 ) { // OK |
1767 | if ( isset( $rhdrs['x-auth-token-expires'] ) ) { |
1768 | $ttl = intval( $rhdrs['x-auth-token-expires'] ); |
1769 | } else { |
1770 | $ttl = $this->authTTL; |
1771 | } |
1772 | $expiryTime = time() + $ttl; |
1773 | $creds = [ |
1774 | 'auth_token' => $rhdrs['x-auth-token'], |
1775 | 'storage_url' => $this->swiftStorageUrl ?? $rhdrs['x-storage-url'], |
1776 | 'expiry_time' => $expiryTime, |
1777 | ]; |
1778 | $this->srvCache->set( $this->getCredsCacheKey( $this->swiftUser ), $creds, $expiryTime ); |
1779 | } elseif ( $rcode === 401 ) { |
1780 | $this->onError( null, __METHOD__, [], "Authentication failed.", $rcode ); |
1781 | $this->authErrorTimestamp = time(); |
1782 | $creds = null; |
1783 | } else { |
1784 | $this->onError( null, __METHOD__, [], "HTTP return code: $rcode", $rcode, $rbody ); |
1785 | $this->authErrorTimestamp = time(); |
1786 | $creds = null; |
1787 | } |
1788 | $this->setAuthCreds( $creds ); |
1789 | return $creds; |
1790 | } |
1791 | |
1792 | /** |
1793 | * @param array $creds From getAuthentication() |
1794 | * @param string|null $container |
1795 | * @param string|null $object |
1796 | * @return string |
1797 | */ |
1798 | protected function storageUrl( array $creds, $container = null, $object = null ) { |
1799 | $parts = [ $creds['storage_url'] ]; |
1800 | if ( strlen( $container ?? '' ) ) { |
1801 | $parts[] = rawurlencode( $container ); |
1802 | } |
1803 | if ( strlen( $object ?? '' ) ) { |
1804 | $parts[] = str_replace( "%2F", "/", rawurlencode( $object ) ); |
1805 | } |
1806 | |
1807 | return implode( '/', $parts ); |
1808 | } |
1809 | |
1810 | /** |
1811 | * @param array $creds From getAuthentication() |
1812 | * @return array |
1813 | */ |
1814 | protected function authTokenHeaders( array $creds ) { |
1815 | return [ 'x-auth-token' => $creds['auth_token'] ]; |
1816 | } |
1817 | |
1818 | /** |
1819 | * Get the cache key for a container |
1820 | * |
1821 | * @param string $username |
1822 | * @return string |
1823 | */ |
1824 | private function getCredsCacheKey( $username ) { |
1825 | return 'swiftcredentials:' . md5( $username . ':' . $this->swiftAuthUrl ); |
1826 | } |
1827 | |
1828 | /** |
1829 | * Perform an authenticated HTTP request |
1830 | * |
1831 | * @param array $req The request data, including: |
1832 | * - container: The name of the container (required) |
1833 | * - relPath: The relative path under the container. If this is omitted, |
1834 | * the request will refer to the container itself. |
1835 | * - headers: An array of request headers to send, in addition to the |
1836 | * auth headers. |
1837 | * - Other keys to be passed through to MultiHttpClient::run() |
1838 | * @param array $options Options to pass through to MultiHttpClient, in |
1839 | * addition to the default options DEFAULT_HTTP_OPTIONS |
1840 | * @return array The response array from MultiHttpClient::run() |
1841 | */ |
1842 | private function requestWithAuth( array $req, array $options = [] ) { |
1843 | return $this->requestMultiWithAuth( [ $req ], $options )[0]['response']; |
1844 | } |
1845 | |
1846 | /** |
1847 | * Perform a batch of authenticated HTTP requests |
1848 | * |
1849 | * @param array $reqs An array of request data arrays. See self::requestWithAuth() |
1850 | * @param array $options Options to pass through to MultiHttpClient, in |
1851 | * addition to the default options DEFAULT_HTTP_OPTIONS |
1852 | * @return array The request array with responses populated, as returned by |
1853 | * MultiHttpClient::runMulti() |
1854 | */ |
1855 | private function requestMultiWithAuth( array $reqs, $options = [] ) { |
1856 | $remainingTries = 2; |
1857 | $auth = $this->getAuthentication(); |
1858 | while ( true ) { |
1859 | if ( !$auth ) { |
1860 | foreach ( $reqs as &$req ) { |
1861 | if ( !isset( $req['response'] ) ) { |
1862 | $req['response'] = $this->getAuthFailureResponse(); |
1863 | } |
1864 | } |
1865 | break; |
1866 | } |
1867 | foreach ( $reqs as &$req ) { |
1868 | '@phan-var array $req'; // Not array[] |
1869 | if ( isset( $req['response'] ) ) { |
1870 | // Request was attempted before |
1871 | // Retry only if it gave a 401 response code |
1872 | if ( $req['response']['code'] !== 401 ) { |
1873 | continue; |
1874 | } |
1875 | } |
1876 | $req['headers'] = $this->authTokenHeaders( $auth ) + ( $req['headers'] ?? [] ); |
1877 | $req['url'] = $this->storageUrl( $auth, $req['container'], $req['relPath'] ?? null ); |
1878 | } |
1879 | unset( $req ); |
1880 | $reqs = $this->http->runMulti( $reqs, $options + self::DEFAULT_HTTP_OPTIONS ); |
1881 | if ( --$remainingTries > 0 ) { |
1882 | // Retry if any request failed with 401 "not authorized" |
1883 | foreach ( $reqs as $req ) { |
1884 | if ( $req['response']['code'] === 401 ) { |
1885 | $auth = $this->refreshAuthentication(); |
1886 | continue 2; |
1887 | } |
1888 | } |
1889 | } |
1890 | break; |
1891 | } |
1892 | return $reqs; |
1893 | } |
1894 | |
1895 | /** |
1896 | * Get a synthetic response to return from requestWithAuth() or requestMultiWithAuth() |
1897 | * if the request could not be issued due to failure of a prior authentication request. |
1898 | * This failure should not be logged as an HTTP error since the original failure would |
1899 | * have been logged. |
1900 | * |
1901 | * @return array |
1902 | */ |
1903 | private function getAuthFailureResponse() { |
1904 | return [ |
1905 | 'code' => 0, |
1906 | 0 => 0, |
1907 | 'reason' => '', |
1908 | 1 => '', |
1909 | 'headers' => [], |
1910 | 2 => [], |
1911 | 'body' => '', |
1912 | 3 => '', |
1913 | 'error' => self::AUTH_FAILURE_ERROR, |
1914 | 4 => self::AUTH_FAILURE_ERROR |
1915 | ]; |
1916 | } |
1917 | |
1918 | /** |
1919 | * Determine whether an HTTP response was generated by getAuthFailureResponse() |
1920 | * |
1921 | * @param int $code |
1922 | * @param string $error |
1923 | * @return bool |
1924 | */ |
1925 | private function isAuthFailureResponse( $code, $error ) { |
1926 | return $code === 0 && $error === self::AUTH_FAILURE_ERROR; |
1927 | } |
1928 | |
1929 | /** |
1930 | * Log an unexpected exception for this backend. |
1931 | * This also sets the StatusValue object to have a fatal error. |
1932 | * |
1933 | * @param StatusValue|null $status To add fatal errors to |
1934 | * @param string $func |
1935 | * @param array $params |
1936 | * @param string $err Error string |
1937 | * @param int $code HTTP status |
1938 | * @param string $desc HTTP StatusValue description |
1939 | * @param string $body HTTP body |
1940 | */ |
1941 | public function onError( $status, $func, array $params, $err = '', $code = 0, $desc = '', $body = '' ) { |
1942 | if ( $this->isAuthFailureResponse( $code, $err ) ) { |
1943 | if ( $status instanceof StatusValue ) { |
1944 | $status->fatal( 'backend-fail-connect', $this->name ); |
1945 | } |
1946 | // Already logged |
1947 | return; |
1948 | } |
1949 | if ( $status instanceof StatusValue ) { |
1950 | $status->fatal( 'backend-fail-internal', $this->name ); |
1951 | } |
1952 | $msg = "HTTP {code} ({desc}) in '{func}' (given '{req_params}')"; |
1953 | $msgParams = [ |
1954 | 'code' => $code, |
1955 | 'desc' => $desc, |
1956 | 'func' => $func, |
1957 | 'req_params' => FormatJson::encode( $params ), |
1958 | ]; |
1959 | if ( $err ) { |
1960 | $msg .= ': {err}'; |
1961 | $msgParams['err'] = $err; |
1962 | } |
1963 | if ( $code == 502 ) { |
1964 | $msg .= ' ({truncatedBody})'; |
1965 | $msgParams['truncatedBody'] = substr( strip_tags( $body ), 0, 100 ); |
1966 | } |
1967 | $this->logger->error( $msg, $msgParams ); |
1968 | } |
1969 | } |