MediaWiki master
SwiftFileBackend.php
Go to the documentation of this file.
1<?php
26use Psr\Log\LoggerInterface;
27use Wikimedia\AtEase\AtEase;
28use Wikimedia\RequestTimeout\TimeoutException;
29
40 private const DEFAULT_HTTP_OPTIONS = [ 'httpVersion' => 'v1.1' ];
41 private const AUTH_FAILURE_ERROR = 'Could not connect due to prior authentication failure';
42
44 protected $http;
46 protected $authTTL;
48 protected $swiftAuthUrl;
52 protected $swiftUser;
54 protected $swiftKey;
58 protected $rgwS3AccessKey;
60 protected $rgwS3SecretKey;
62 protected $readUsers;
64 protected $writeUsers;
69
71 protected $srvCache;
72
75
77 protected $authCreds;
79 protected $authErrorTimestamp = null;
80
82 protected $isRGW = false;
83
122 public function __construct( array $config ) {
123 parent::__construct( $config );
124 // Required settings
125 $this->swiftAuthUrl = $config['swiftAuthUrl'];
126 $this->swiftUser = $config['swiftUser'];
127 $this->swiftKey = $config['swiftKey'];
128 // Optional settings
129 $this->authTTL = $config['swiftAuthTTL'] ?? 15 * 60; // some sensible number
130 $this->swiftTempUrlKey = $config['swiftTempUrlKey'] ?? '';
131 $this->swiftStorageUrl = $config['swiftStorageUrl'] ?? null;
132 $this->shardViaHashLevels = $config['shardViaHashLevels'] ?? '';
133 $this->rgwS3AccessKey = $config['rgwS3AccessKey'] ?? '';
134 $this->rgwS3SecretKey = $config['rgwS3SecretKey'] ?? '';
135
136 // HTTP helper client
137 $httpOptions = [];
138 foreach ( [ 'connTimeout', 'reqTimeout' ] as $optionName ) {
139 if ( isset( $config[$optionName] ) ) {
140 $httpOptions[$optionName] = $config[$optionName];
141 }
142 }
143 $this->http = new MultiHttpClient( $httpOptions );
144 $this->http->setLogger( $this->logger );
145
146 // Cache container information to mask latency
147 if ( isset( $config['wanCache'] ) && $config['wanCache'] instanceof WANObjectCache ) {
148 $this->memCache = $config['wanCache'];
149 }
150 // Process cache for container info
151 $this->containerStatCache = new MapCacheLRU( 300 );
152 // Cache auth token information to avoid RTTs
153 if ( !empty( $config['cacheAuthInfo'] ) && isset( $config['srvCache'] ) ) {
154 $this->srvCache = $config['srvCache'];
155 } else {
156 $this->srvCache = new EmptyBagOStuff();
157 }
158 $this->readUsers = $config['readUsers'] ?? [];
159 $this->writeUsers = $config['writeUsers'] ?? [];
160 $this->secureReadUsers = $config['secureReadUsers'] ?? [];
161 $this->secureWriteUsers = $config['secureWriteUsers'] ?? [];
162 // Per https://docs.openstack.org/swift/latest/overview_large_objects.html
163 // we need to split objects if they are larger than 5 GB. Support for
164 // splitting objects has not yet been implemented by this class
165 // so limit max file size to 5GiB.
166 $this->maxFileSize = 5 * 1024 * 1024 * 1024;
167 }
168
169 public function setLogger( LoggerInterface $logger ) {
170 parent::setLogger( $logger );
171 $this->http->setLogger( $logger );
172 }
173
174 public function getFeatures() {
175 return (
176 self::ATTR_UNICODE_PATHS |
177 self::ATTR_HEADERS |
178 self::ATTR_METADATA
179 );
180 }
181
182 protected function resolveContainerPath( $container, $relStoragePath ) {
183 if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) {
184 return null; // not UTF-8, makes it hard to use CF and the swift HTTP API
185 } elseif ( strlen( rawurlencode( $relStoragePath ) ) > 1024 ) {
186 return null; // too long for Swift
187 }
188
189 return $relStoragePath;
190 }
191
192 public function isPathUsableInternal( $storagePath ) {
193 [ $container, $rel ] = $this->resolveStoragePathReal( $storagePath );
194 if ( $rel === null ) {
195 return false; // invalid
196 }
197
198 return is_array( $this->getContainerStat( $container ) );
199 }
200
210 protected function extractMutableContentHeaders( array $headers ) {
211 $contentHeaders = [];
212 // Normalize casing, and strip out illegal headers
213 foreach ( $headers as $name => $value ) {
214 $name = strtolower( $name );
215 if ( $name === 'x-delete-at' && is_numeric( $value ) ) {
216 // Expects a Unix Epoch date
217 $contentHeaders[$name] = $value;
218 } elseif ( $name === 'x-delete-after' && is_numeric( $value ) ) {
219 // Expects number of minutes time to live.
220 $contentHeaders[$name] = $value;
221 } elseif ( preg_match( '/^(x-)?content-(?!length$)/', $name ) ) {
222 // Only allow content-* and x-content-* headers (but not content-length)
223 $contentHeaders[$name] = $value;
224 } elseif ( $name === 'content-type' && strlen( $value ) ) {
225 // This header can be set to a value but not unset
226 $contentHeaders[$name] = $value;
227 }
228 }
229 // By default, Swift has annoyingly low maximum header value limits
230 if ( isset( $contentHeaders['content-disposition'] ) ) {
231 $maxLength = 255;
232 // @note: assume FileBackend::makeContentDisposition() already used
233 $offset = $maxLength - strlen( $contentHeaders['content-disposition'] );
234 if ( $offset < 0 ) {
235 $pos = strrpos( $contentHeaders['content-disposition'], ';', $offset );
236 $contentHeaders['content-disposition'] = $pos === false
237 ? ''
238 : trim( substr( $contentHeaders['content-disposition'], 0, $pos ) );
239 }
240 }
241
242 return $contentHeaders;
243 }
244
250 protected function extractMetadataHeaders( array $headers ) {
251 $metadataHeaders = [];
252 foreach ( $headers as $name => $value ) {
253 $name = strtolower( $name );
254 if ( strpos( $name, 'x-object-meta-' ) === 0 ) {
255 $metadataHeaders[$name] = $value;
256 }
257 }
258
259 return $metadataHeaders;
260 }
261
267 protected function getMetadataFromHeaders( array $headers ) {
268 $prefixLen = strlen( 'x-object-meta-' );
269
270 $metadata = [];
271 foreach ( $this->extractMetadataHeaders( $headers ) as $name => $value ) {
272 $metadata[substr( $name, $prefixLen )] = $value;
273 }
274
275 return $metadata;
276 }
277
278 protected function doCreateInternal( array $params ) {
279 $status = $this->newStatus();
280
281 [ $dstCont, $dstRel ] = $this->resolveStoragePathReal( $params['dst'] );
282 if ( $dstRel === null ) {
283 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
284
285 return $status;
286 }
287
288 // Headers that are not strictly a function of the file content
289 $mutableHeaders = $this->extractMutableContentHeaders( $params['headers'] ?? [] );
290 // Make sure that the "content-type" header is set to something sensible
291 $mutableHeaders['content-type']
292 ??= $this->getContentType( $params['dst'], $params['content'], null );
293
294 $reqs = [ [
295 'method' => 'PUT',
296 'container' => $dstCont,
297 'relPath' => $dstRel,
298 'headers' => array_merge(
299 $mutableHeaders,
300 [
301 'etag' => md5( $params['content'] ),
302 'content-length' => strlen( $params['content'] ),
303 'x-object-meta-sha1base36' =>
304 Wikimedia\base_convert( sha1( $params['content'] ), 16, 36, 31 )
305 ]
306 ),
307 'body' => $params['content']
308 ] ];
309
310 $method = __METHOD__;
311 $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
312 [ $rcode, $rdesc, , $rbody, $rerr ] = $request['response'];
313 if ( $rcode === 201 || $rcode === 202 ) {
314 // good
315 } elseif ( $rcode === 412 ) {
316 $status->fatal( 'backend-fail-contenttype', $params['dst'] );
317 } else {
318 $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc, $rbody );
319 }
320
321 return SwiftFileOpHandle::CONTINUE_IF_OK;
322 };
323
324 $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
325 if ( !empty( $params['async'] ) ) { // deferred
326 $status->value = $opHandle;
327 } else { // actually write the object in Swift
328 $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
329 }
330
331 return $status;
332 }
333
334 protected function doStoreInternal( array $params ) {
335 $status = $this->newStatus();
336
337 [ $dstCont, $dstRel ] = $this->resolveStoragePathReal( $params['dst'] );
338 if ( $dstRel === null ) {
339 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
340
341 return $status;
342 }
343
344 // Open a handle to the source file so that it can be streamed. The size and hash
345 // will be computed using the handle. In the off chance that the source file changes
346 // during this operation, the PUT will fail due to an ETag mismatch and be aborted.
347 AtEase::suppressWarnings();
348 $srcHandle = fopen( $params['src'], 'rb' );
349 AtEase::restoreWarnings();
350 if ( $srcHandle === false ) { // source doesn't exist?
351 $status->fatal( 'backend-fail-notexists', $params['src'] );
352
353 return $status;
354 }
355
356 // Compute the MD5 and SHA-1 hashes in one pass
357 $srcSize = fstat( $srcHandle )['size'];
358 $md5Context = hash_init( 'md5' );
359 $sha1Context = hash_init( 'sha1' );
360 $hashDigestSize = 0;
361 while ( !feof( $srcHandle ) ) {
362 $buffer = (string)fread( $srcHandle, 131_072 ); // 128 KiB
363 hash_update( $md5Context, $buffer );
364 hash_update( $sha1Context, $buffer );
365 $hashDigestSize += strlen( $buffer );
366 }
367 // Reset the handle back to the beginning so that it can be streamed
368 rewind( $srcHandle );
369
370 if ( $hashDigestSize !== $srcSize ) {
371 $status->fatal( 'backend-fail-hash', $params['src'] );
372
373 return $status;
374 }
375
376 // Headers that are not strictly a function of the file content
377 $mutableHeaders = $this->extractMutableContentHeaders( $params['headers'] ?? [] );
378 // Make sure that the "content-type" header is set to something sensible
379 $mutableHeaders['content-type']
380 ??= $this->getContentType( $params['dst'], null, $params['src'] );
381
382 $reqs = [ [
383 'method' => 'PUT',
384 'container' => $dstCont,
385 'relPath' => $dstRel,
386 'headers' => array_merge(
387 $mutableHeaders,
388 [
389 'content-length' => $srcSize,
390 'etag' => hash_final( $md5Context ),
391 'x-object-meta-sha1base36' =>
392 Wikimedia\base_convert( hash_final( $sha1Context ), 16, 36, 31 )
393 ]
394 ),
395 'body' => $srcHandle // resource
396 ] ];
397
398 $method = __METHOD__;
399 $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
400 [ $rcode, $rdesc, , $rbody, $rerr ] = $request['response'];
401 if ( $rcode === 201 || $rcode === 202 ) {
402 // good
403 } elseif ( $rcode === 412 ) {
404 $status->fatal( 'backend-fail-contenttype', $params['dst'] );
405 } else {
406 $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc, $rbody );
407 }
408
409 return SwiftFileOpHandle::CONTINUE_IF_OK;
410 };
411
412 $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
413 $opHandle->resourcesToClose[] = $srcHandle;
414
415 if ( !empty( $params['async'] ) ) { // deferred
416 $status->value = $opHandle;
417 } else { // actually write the object in Swift
418 $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
419 }
420
421 return $status;
422 }
423
424 protected function doCopyInternal( array $params ) {
425 $status = $this->newStatus();
426
427 [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $params['src'] );
428 if ( $srcRel === null ) {
429 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
430
431 return $status;
432 }
433
434 [ $dstCont, $dstRel ] = $this->resolveStoragePathReal( $params['dst'] );
435 if ( $dstRel === null ) {
436 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
437
438 return $status;
439 }
440
441 $reqs = [ [
442 'method' => 'PUT',
443 'container' => $dstCont,
444 'relPath' => $dstRel,
445 'headers' => array_merge(
446 $this->extractMutableContentHeaders( $params['headers'] ?? [] ),
447 [
448 'x-copy-from' => '/' . rawurlencode( $srcCont ) . '/' .
449 str_replace( "%2F", "/", rawurlencode( $srcRel ) )
450 ]
451 )
452 ] ];
453
454 $method = __METHOD__;
455 $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
456 [ $rcode, $rdesc, , $rbody, $rerr ] = $request['response'];
457 if ( $rcode === 201 ) {
458 // good
459 } elseif ( $rcode === 404 ) {
460 if ( empty( $params['ignoreMissingSource'] ) ) {
461 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
462 }
463 } else {
464 $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc, $rbody );
465 }
466
467 return SwiftFileOpHandle::CONTINUE_IF_OK;
468 };
469
470 $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
471 if ( !empty( $params['async'] ) ) { // deferred
472 $status->value = $opHandle;
473 } else { // actually write the object in Swift
474 $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
475 }
476
477 return $status;
478 }
479
480 protected function doMoveInternal( array $params ) {
481 $status = $this->newStatus();
482
483 [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $params['src'] );
484 if ( $srcRel === null ) {
485 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
486
487 return $status;
488 }
489
490 [ $dstCont, $dstRel ] = $this->resolveStoragePathReal( $params['dst'] );
491 if ( $dstRel === null ) {
492 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
493
494 return $status;
495 }
496
497 $reqs = [ [
498 'method' => 'PUT',
499 'container' => $dstCont,
500 'relPath' => $dstRel,
501 'headers' => array_merge(
502 $this->extractMutableContentHeaders( $params['headers'] ?? [] ),
503 [
504 'x-copy-from' => '/' . rawurlencode( $srcCont ) . '/' .
505 str_replace( "%2F", "/", rawurlencode( $srcRel ) )
506 ]
507 )
508 ] ];
509 if ( "{$srcCont}/{$srcRel}" !== "{$dstCont}/{$dstRel}" ) {
510 $reqs[] = [
511 'method' => 'DELETE',
512 'container' => $srcCont,
513 'relPath' => $srcRel,
514 'headers' => []
515 ];
516 }
517
518 $method = __METHOD__;
519 $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
520 [ $rcode, $rdesc, , $rbody, $rerr ] = $request['response'];
521 if ( $request['method'] === 'PUT' && $rcode === 201 ) {
522 // good
523 } elseif ( $request['method'] === 'DELETE' && $rcode === 204 ) {
524 // good
525 } elseif ( $rcode === 404 ) {
526 if ( empty( $params['ignoreMissingSource'] ) ) {
527 $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
528 } else {
529 // Leave Status as OK but skip the DELETE request
530 return SwiftFileOpHandle::CONTINUE_NO;
531 }
532 } else {
533 $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc, $rbody );
534 }
535
536 return SwiftFileOpHandle::CONTINUE_IF_OK;
537 };
538
539 $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
540 if ( !empty( $params['async'] ) ) { // deferred
541 $status->value = $opHandle;
542 } else { // actually move the object in Swift
543 $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
544 }
545
546 return $status;
547 }
548
549 protected function doDeleteInternal( array $params ) {
550 $status = $this->newStatus();
551
552 [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $params['src'] );
553 if ( $srcRel === null ) {
554 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
555
556 return $status;
557 }
558
559 $reqs = [ [
560 'method' => 'DELETE',
561 'container' => $srcCont,
562 'relPath' => $srcRel,
563 'headers' => []
564 ] ];
565
566 $method = __METHOD__;
567 $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
568 [ $rcode, $rdesc, , $rbody, $rerr ] = $request['response'];
569 if ( $rcode === 204 ) {
570 // good
571 } elseif ( $rcode === 404 ) {
572 if ( empty( $params['ignoreMissingSource'] ) ) {
573 $status->fatal( 'backend-fail-delete', $params['src'] );
574 }
575 } else {
576 $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc, $rbody );
577 }
578
579 return SwiftFileOpHandle::CONTINUE_IF_OK;
580 };
581
582 $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
583 if ( !empty( $params['async'] ) ) { // deferred
584 $status->value = $opHandle;
585 } else { // actually delete the object in Swift
586 $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
587 }
588
589 return $status;
590 }
591
592 protected function doDescribeInternal( array $params ) {
593 $status = $this->newStatus();
594
595 [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $params['src'] );
596 if ( $srcRel === null ) {
597 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
598
599 return $status;
600 }
601
602 // Fetch the old object headers/metadata...this should be in stat cache by now
603 $stat = $this->getFileStat( [ 'src' => $params['src'], 'latest' => 1 ] );
604 if ( $stat && !isset( $stat['xattr'] ) ) { // older cache entry
605 $stat = $this->doGetFileStat( [ 'src' => $params['src'], 'latest' => 1 ] );
606 }
607 if ( !$stat ) {
608 $status->fatal( 'backend-fail-describe', $params['src'] );
609
610 return $status;
611 }
612
613 // Swift object POST clears any prior headers, so merge the new and old headers here.
614 // Also, during, POST, libcurl adds "Content-Type: application/x-www-form-urlencoded"
615 // if "Content-Type" is not set, which would clobber the header value for the object.
616 $oldMetadataHeaders = [];
617 foreach ( $stat['xattr']['metadata'] as $name => $value ) {
618 $oldMetadataHeaders["x-object-meta-$name"] = $value;
619 }
620 $newContentHeaders = $this->extractMutableContentHeaders( $params['headers'] ?? [] );
621 $oldContentHeaders = $stat['xattr']['headers'];
622
623 $reqs = [ [
624 'method' => 'POST',
625 'container' => $srcCont,
626 'relPath' => $srcRel,
627 'headers' => $oldMetadataHeaders + $newContentHeaders + $oldContentHeaders
628 ] ];
629
630 $method = __METHOD__;
631 $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
632 [ $rcode, $rdesc, , $rbody, $rerr ] = $request['response'];
633 if ( $rcode === 202 ) {
634 // good
635 } elseif ( $rcode === 404 ) {
636 $status->fatal( 'backend-fail-describe', $params['src'] );
637 } else {
638 $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc, $rbody );
639 }
640 };
641
642 $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
643 if ( !empty( $params['async'] ) ) { // deferred
644 $status->value = $opHandle;
645 } else { // actually change the object in Swift
646 $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
647 }
648
649 return $status;
650 }
651
655 protected function doPrepareInternal( $fullCont, $dir, array $params ) {
656 $status = $this->newStatus();
657
658 // (a) Check if container already exists
659 $stat = $this->getContainerStat( $fullCont );
660 if ( is_array( $stat ) ) {
661 return $status; // already there
662 } elseif ( $stat === self::RES_ERROR ) {
663 $status->fatal( 'backend-fail-internal', $this->name );
664 $this->logger->error( __METHOD__ . ': cannot get container stat' );
665 } else {
666 // (b) Create container as needed with proper ACLs
667 $params['op'] = 'prepare';
668 $status->merge( $this->createContainer( $fullCont, $params ) );
669 }
670
671 return $status;
672 }
673
674 protected function doSecureInternal( $fullCont, $dir, array $params ) {
675 $status = $this->newStatus();
676 if ( empty( $params['noAccess'] ) ) {
677 return $status; // nothing to do
678 }
679
680 $stat = $this->getContainerStat( $fullCont );
681 if ( is_array( $stat ) ) {
682 $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
683 $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
684 // Make container private to end-users...
685 $status->merge( $this->setContainerAccess(
686 $fullCont,
689 ) );
690 } elseif ( $stat === self::RES_ABSENT ) {
691 $status->fatal( 'backend-fail-usable', $params['dir'] );
692 } else {
693 $status->fatal( 'backend-fail-internal', $this->name );
694 $this->logger->error( __METHOD__ . ': cannot get container stat' );
695 }
696
697 return $status;
698 }
699
700 protected function doPublishInternal( $fullCont, $dir, array $params ) {
701 $status = $this->newStatus();
702
703 $stat = $this->getContainerStat( $fullCont );
704 if ( is_array( $stat ) ) {
705 $readUsers = array_merge( $this->readUsers, [ $this->swiftUser, '.r:*' ] );
706 $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
707
708 // Make container public to end-users...
709 $status->merge( $this->setContainerAccess(
710 $fullCont,
713 ) );
714 } elseif ( $stat === self::RES_ABSENT ) {
715 $status->fatal( 'backend-fail-usable', $params['dir'] );
716 } else {
717 $status->fatal( 'backend-fail-internal', $this->name );
718 $this->logger->error( __METHOD__ . ': cannot get container stat' );
719 }
720
721 return $status;
722 }
723
724 protected function doCleanInternal( $fullCont, $dir, array $params ) {
725 $status = $this->newStatus();
726
727 // Only containers themselves can be removed, all else is virtual
728 if ( $dir != '' ) {
729 return $status; // nothing to do
730 }
731
732 // (a) Check the container
733 $stat = $this->getContainerStat( $fullCont, true );
734 if ( $stat === self::RES_ABSENT ) {
735 return $status; // ok, nothing to do
736 } elseif ( $stat === self::RES_ERROR ) {
737 $status->fatal( 'backend-fail-internal', $this->name );
738 $this->logger->error( __METHOD__ . ': cannot get container stat' );
739 } elseif ( is_array( $stat ) && $stat['count'] == 0 ) {
740 // (b) Delete the container if empty
741 $params['op'] = 'clean';
742 $status->merge( $this->deleteContainer( $fullCont, $params ) );
743 }
744
745 return $status;
746 }
747
748 protected function doGetFileStat( array $params ) {
749 $params = [ 'srcs' => [ $params['src'] ], 'concurrency' => 1 ] + $params;
750 unset( $params['src'] );
751 $stats = $this->doGetFileStatMulti( $params );
752
753 return reset( $stats );
754 }
755
766 protected function convertSwiftDate( $ts, $format = TS_MW ) {
767 try {
768 $timestamp = new MWTimestamp( $ts );
769
770 return $timestamp->getTimestamp( $format );
771 } catch ( TimeoutException $e ) {
772 throw $e;
773 } catch ( Exception $e ) {
774 throw new FileBackendError( $e->getMessage() );
775 }
776 }
777
785 protected function addMissingHashMetadata( array $objHdrs, $path ) {
786 if ( isset( $objHdrs['x-object-meta-sha1base36'] ) ) {
787 return $objHdrs; // nothing to do
788 }
789
791 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
792 $this->logger->error( __METHOD__ . ": {path} was not stored with SHA-1 metadata.",
793 [ 'path' => $path ] );
794
795 $objHdrs['x-object-meta-sha1base36'] = false;
796
797 // Find prior custom HTTP headers
798 $postHeaders = $this->extractMutableContentHeaders( $objHdrs );
799 // Find prior metadata headers
800 $postHeaders += $this->extractMetadataHeaders( $objHdrs );
801
802 $status = $this->newStatus();
804 $scopeLockS = $this->getScopedFileLocks( [ $path ], LockManager::LOCK_UW, $status );
805 if ( $status->isOK() ) {
806 $tmpFile = $this->getLocalCopy( [ 'src' => $path, 'latest' => 1 ] );
807 if ( $tmpFile ) {
808 $hash = $tmpFile->getSha1Base36();
809 if ( $hash !== false ) {
810 $objHdrs['x-object-meta-sha1base36'] = $hash;
811 // Merge new SHA1 header into the old ones
812 $postHeaders['x-object-meta-sha1base36'] = $hash;
813 [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $path );
814 [ $rcode ] = $this->requestWithAuth( [
815 'method' => 'POST',
816 'container' => $srcCont,
817 'relPath' => $srcRel,
818 'headers' => $postHeaders
819 ] );
820 if ( $rcode >= 200 && $rcode <= 299 ) {
821 $this->deleteFileCache( $path );
822
823 return $objHdrs; // success
824 }
825 }
826 }
827 }
828
829 $this->logger->error( __METHOD__ . ': unable to set SHA-1 metadata for {path}',
830 [ 'path' => $path ] );
831
832 return $objHdrs; // failed
833 }
834
835 protected function doGetFileContentsMulti( array $params ) {
836 $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging
837 // Blindly create tmp files and stream to them, catching any exception
838 // if the file does not exist. Do not waste time doing file stats here.
839 $reqs = []; // (path => op)
840
841 // Initial dummy values to preserve path order
842 $contents = array_fill_keys( $params['srcs'], self::RES_ERROR );
843 foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch
844 [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $path );
845 if ( $srcRel === null ) {
846 continue; // invalid storage path
847 }
848 // Create a new temporary memory file...
849 $handle = fopen( 'php://temp', 'wb' );
850 if ( $handle ) {
851 $reqs[$path] = [
852 'method' => 'GET',
853 'container' => $srcCont,
854 'relPath' => $srcRel,
855 'headers' => $this->headersFromParams( $params ),
856 'stream' => $handle,
857 ];
858 }
859 }
860
861 $reqs = $this->requestMultiWithAuth(
862 $reqs,
863 [ 'maxConnsPerHost' => $params['concurrency'] ]
864 );
865 foreach ( $reqs as $path => $op ) {
866 [ $rcode, $rdesc, $rhdrs, $rbody, $rerr ] = $op['response'];
867 if ( $rcode >= 200 && $rcode <= 299 ) {
868 rewind( $op['stream'] ); // start from the beginning
869 $content = (string)stream_get_contents( $op['stream'] );
870 $size = strlen( $content );
871 // Make sure that stream finished
872 if ( $size === (int)$rhdrs['content-length'] ) {
873 $contents[$path] = $content;
874 } else {
875 $contents[$path] = self::RES_ERROR;
876 $rerr = "Got {$size}/{$rhdrs['content-length']} bytes";
877 $this->onError( null, __METHOD__,
878 [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
879 }
880 } elseif ( $rcode === 404 ) {
881 $contents[$path] = self::RES_ABSENT;
882 } else {
883 $contents[$path] = self::RES_ERROR;
884 $this->onError( null, __METHOD__,
885 [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc, $rbody );
886 }
887 fclose( $op['stream'] ); // close open handle
888 }
889
890 return $contents;
891 }
892
893 protected function doDirectoryExists( $fullCont, $dir, array $params ) {
894 $prefix = ( $dir == '' ) ? null : "{$dir}/";
895 $status = $this->objectListing( $fullCont, 'names', 1, null, $prefix );
896 if ( $status->isOK() ) {
897 return ( count( $status->value ) ) > 0;
898 }
899
900 return self::RES_ERROR;
901 }
902
910 public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
911 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
912 }
913
921 public function getFileListInternal( $fullCont, $dir, array $params ) {
922 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
923 }
924
936 public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
937 $dirs = [];
938 if ( $after === INF ) {
939 return $dirs; // nothing more
940 }
941
943 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
944
945 $prefix = ( $dir == '' ) ? null : "{$dir}/";
946 // Non-recursive: only list dirs right under $dir
947 if ( !empty( $params['topOnly'] ) ) {
948 $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
949 if ( !$status->isOK() ) {
950 throw new FileBackendError( "Iterator page I/O error." );
951 }
952 $objects = $status->value;
953 // @phan-suppress-next-line PhanTypeSuspiciousNonTraversableForeach
954 foreach ( $objects as $object ) { // files and directories
955 if ( substr( $object, -1 ) === '/' ) {
956 $dirs[] = $object; // directories end in '/'
957 }
958 }
959 } else {
960 // Recursive: list all dirs under $dir and its subdirs
961 $getParentDir = static function ( $path ) {
962 return ( $path !== null && strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
963 };
964
965 // Get directory from last item of prior page
966 $lastDir = $getParentDir( $after ); // must be first page
967 $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
968
969 if ( !$status->isOK() ) {
970 throw new FileBackendError( "Iterator page I/O error." );
971 }
972
973 $objects = $status->value;
974
975 // @phan-suppress-next-line PhanTypeSuspiciousNonTraversableForeach
976 foreach ( $objects as $object ) { // files
977 $objectDir = $getParentDir( $object ); // directory of object
978
979 if ( $objectDir !== false && $objectDir !== $dir ) {
980 // Swift stores paths in UTF-8, using binary sorting.
981 // See function "create_container_table" in common/db.py.
982 // If a directory is not "greater" than the last one,
983 // then it was already listed by the calling iterator.
984 if ( strcmp( $objectDir, $lastDir ) > 0 ) {
985 $pDir = $objectDir;
986 do { // add dir and all its parent dirs
987 $dirs[] = "{$pDir}/";
988 $pDir = $getParentDir( $pDir );
989 } while ( $pDir !== false
990 && strcmp( $pDir, $lastDir ) > 0 // not done already
991 && strlen( $pDir ) > strlen( $dir ) // within $dir
992 );
993 }
994 $lastDir = $objectDir;
995 }
996 }
997 }
998 // Page on the unfiltered directory listing (what is returned may be filtered)
999 if ( count( $objects ) < $limit ) {
1000 $after = INF; // avoid a second RTT
1001 } else {
1002 $after = end( $objects ); // update last item
1003 }
1004
1005 return $dirs;
1006 }
1007
1019 public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
1020 $files = []; // list of (path, stat map or null) entries
1021 if ( $after === INF ) {
1022 return $files; // nothing more
1023 }
1024
1026 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1027
1028 $prefix = ( $dir == '' ) ? null : "{$dir}/";
1029 // $objects will contain a list of unfiltered names or stdClass items
1030 // Non-recursive: only list files right under $dir
1031 if ( !empty( $params['topOnly'] ) ) {
1032 if ( !empty( $params['adviseStat'] ) ) {
1033 $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix, '/' );
1034 } else {
1035 $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
1036 }
1037 } else {
1038 // Recursive: list all files under $dir and its subdirs
1039 if ( !empty( $params['adviseStat'] ) ) {
1040 $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix );
1041 } else {
1042 $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
1043 }
1044 }
1045
1046 // Reformat this list into a list of (name, stat map or null) entries
1047 if ( !$status->isOK() ) {
1048 throw new FileBackendError( "Iterator page I/O error." );
1049 }
1050
1051 $objects = $status->value;
1052 $files = $this->buildFileObjectListing( $objects );
1053
1054 // Page on the unfiltered object listing (what is returned may be filtered)
1055 if ( count( $objects ) < $limit ) {
1056 $after = INF; // avoid a second RTT
1057 } else {
1058 $after = end( $objects ); // update last item
1059 $after = is_object( $after ) ? $after->name : $after;
1060 }
1061
1062 return $files;
1063 }
1064
1072 private function buildFileObjectListing( array $objects ) {
1073 $names = [];
1074 foreach ( $objects as $object ) {
1075 if ( is_object( $object ) ) {
1076 if ( isset( $object->subdir ) || !isset( $object->name ) ) {
1077 continue; // virtual directory entry; ignore
1078 }
1079 $stat = [
1080 // Convert various random Swift dates to TS_MW
1081 'mtime' => $this->convertSwiftDate( $object->last_modified, TS_MW ),
1082 'size' => (int)$object->bytes,
1083 'sha1' => null,
1084 // Note: manifest ETags are not an MD5 of the file
1085 'md5' => ctype_xdigit( $object->hash ) ? $object->hash : null,
1086 'latest' => false // eventually consistent
1087 ];
1088 $names[] = [ $object->name, $stat ];
1089 } elseif ( substr( $object, -1 ) !== '/' ) {
1090 // Omit directories, which end in '/' in listings
1091 $names[] = [ $object, null ];
1092 }
1093 }
1094
1095 return $names;
1096 }
1097
1104 public function loadListingStatInternal( $path, array $val ) {
1105 $this->cheapCache->setField( $path, 'stat', $val );
1106 }
1107
1108 protected function doGetFileXAttributes( array $params ) {
1109 $stat = $this->getFileStat( $params );
1110 // Stat entries filled by file listings don't include metadata/headers
1111 if ( is_array( $stat ) && !isset( $stat['xattr'] ) ) {
1112 $this->clearCache( [ $params['src'] ] );
1113 $stat = $this->getFileStat( $params );
1114 }
1115
1116 if ( is_array( $stat ) ) {
1117 return $stat['xattr'];
1118 }
1119
1120 return $stat === self::RES_ERROR ? self::RES_ERROR : self::RES_ABSENT;
1121 }
1122
1123 protected function doGetFileSha1base36( array $params ) {
1124 // Avoid using stat entries from file listings, which never include the SHA-1 hash.
1125 // Also, recompute the hash if it's not part of the metadata headers for some reason.
1126 $params['requireSHA1'] = true;
1127
1128 $stat = $this->getFileStat( $params );
1129 if ( is_array( $stat ) ) {
1130 return $stat['sha1'];
1131 }
1132
1133 return $stat === self::RES_ERROR ? self::RES_ERROR : self::RES_ABSENT;
1134 }
1135
1136 protected function doStreamFile( array $params ) {
1137 $status = $this->newStatus();
1138
1139 $flags = !empty( $params['headless'] ) ? HTTPFileStreamer::STREAM_HEADLESS : 0;
1140
1141 [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $params['src'] );
1142 if ( $srcRel === null ) {
1143 HTTPFileStreamer::send404Message( $params['src'], $flags );
1144 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
1145
1146 return $status;
1147 }
1148
1149 if ( !is_array( $this->getContainerStat( $srcCont ) ) ) {
1150 HTTPFileStreamer::send404Message( $params['src'], $flags );
1151 $status->fatal( 'backend-fail-stream', $params['src'] );
1152
1153 return $status;
1154 }
1155
1156 // If "headers" is set, we only want to send them if the file is there.
1157 // Do not bother checking if the file exists if headers are not set though.
1158 if ( $params['headers'] && !$this->fileExists( $params ) ) {
1159 HTTPFileStreamer::send404Message( $params['src'], $flags );
1160 $status->fatal( 'backend-fail-stream', $params['src'] );
1161
1162 return $status;
1163 }
1164
1165 // Send the requested additional headers
1166 if ( empty( $params['headless'] ) ) {
1167 foreach ( $params['headers'] as $header ) {
1168 header( $header );
1169 }
1170 }
1171
1172 if ( empty( $params['allowOB'] ) ) {
1173 // Cancel output buffering and gzipping if set
1174 ( $this->obResetFunc )();
1175 }
1176
1177 $handle = fopen( 'php://output', 'wb' );
1178 [ $rcode, $rdesc, , $rbody, $rerr ] = $this->requestWithAuth( [
1179 'method' => 'GET',
1180 'container' => $srcCont,
1181 'relPath' => $srcRel,
1182 'headers' => $this->headersFromParams( $params ) + $params['options'],
1183 'stream' => $handle,
1184 'flags' => [ 'relayResponseHeaders' => empty( $params['headless'] ) ]
1185 ] );
1186
1187 if ( $rcode >= 200 && $rcode <= 299 ) {
1188 // good
1189 } elseif ( $rcode === 404 ) {
1190 $status->fatal( 'backend-fail-stream', $params['src'] );
1191 // Per T43113, nasty things can happen if bad cache entries get
1192 // stuck in cache. It's also possible that this error can come up
1193 // with simple race conditions. Clear out the stat cache to be safe.
1194 $this->clearCache( [ $params['src'] ] );
1195 $this->deleteFileCache( $params['src'] );
1196 } else {
1197 $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc, $rbody );
1198 }
1199
1200 return $status;
1201 }
1202
1203 protected function doGetLocalCopyMulti( array $params ) {
1204 $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging
1205 // Blindly create tmp files and stream to them, catching any exception
1206 // if the file does not exist. Do not waste time doing file stats here.
1207 $reqs = []; // (path => op)
1208
1209 // Initial dummy values to preserve path order
1210 $tmpFiles = array_fill_keys( $params['srcs'], self::RES_ERROR );
1211 foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch
1212 [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $path );
1213 if ( $srcRel === null ) {
1214 continue; // invalid storage path
1215 }
1216 // Get source file extension
1218 // Create a new temporary file...
1219 $tmpFile = $this->tmpFileFactory->newTempFSFile( 'localcopy_', $ext );
1220 $handle = $tmpFile ? fopen( $tmpFile->getPath(), 'wb' ) : false;
1221 if ( $handle ) {
1222 $reqs[$path] = [
1223 'method' => 'GET',
1224 'container' => $srcCont,
1225 'relPath' => $srcRel,
1226 'headers' => $this->headersFromParams( $params ),
1227 'stream' => $handle,
1228 ];
1229 $tmpFiles[$path] = $tmpFile;
1230 }
1231 }
1232
1233 // Ceph RADOS Gateway is in use (strong consistency) or X-Newest will be used
1234 $latest = ( $this->isRGW || !empty( $params['latest'] ) );
1235
1236 $reqs = $this->requestMultiWithAuth(
1237 $reqs,
1238 [ 'maxConnsPerHost' => $params['concurrency'] ]
1239 );
1240 foreach ( $reqs as $path => $op ) {
1241 [ $rcode, $rdesc, $rhdrs, $rbody, $rerr ] = $op['response'];
1242 fclose( $op['stream'] ); // close open handle
1243 if ( $rcode >= 200 && $rcode <= 299 ) {
1245 $tmpFile = $tmpFiles[$path];
1246 // Make sure that the stream finished and fully wrote to disk
1247 $size = $tmpFile->getSize();
1248 if ( $size !== (int)$rhdrs['content-length'] ) {
1249 $tmpFiles[$path] = self::RES_ERROR;
1250 $rerr = "Got {$size}/{$rhdrs['content-length']} bytes";
1251 $this->onError( null, __METHOD__,
1252 [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
1253 }
1254 // Set the file stat process cache in passing
1255 $stat = $this->getStatFromHeaders( $rhdrs );
1256 $stat['latest'] = $latest;
1257 $this->cheapCache->setField( $path, 'stat', $stat );
1258 } elseif ( $rcode === 404 ) {
1259 $tmpFiles[$path] = self::RES_ABSENT;
1260 $this->cheapCache->setField(
1261 $path,
1262 'stat',
1263 $latest ? self::ABSENT_LATEST : self::ABSENT_NORMAL
1264 );
1265 } else {
1266 $tmpFiles[$path] = self::RES_ERROR;
1267 $this->onError( null, __METHOD__,
1268 [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc, $rbody );
1269 }
1270 }
1271
1272 return $tmpFiles;
1273 }
1274
1275 public function getFileHttpUrl( array $params ) {
1276 if ( $this->swiftTempUrlKey != '' ||
1277 ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' )
1278 ) {
1279 [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $params['src'] );
1280 if ( $srcRel === null ) {
1281 return self::TEMPURL_ERROR; // invalid path
1282 }
1283
1284 $auth = $this->getAuthentication();
1285 if ( !$auth ) {
1286 return self::TEMPURL_ERROR;
1287 }
1288
1289 $ttl = $params['ttl'] ?? 86400;
1290 $expires = time() + $ttl;
1291
1292 if ( $this->swiftTempUrlKey != '' ) {
1293 $url = $this->storageUrl( $auth, $srcCont, $srcRel );
1294 // Swift wants the signature based on the unencoded object name
1295 $contPath = parse_url( $this->storageUrl( $auth, $srcCont ), PHP_URL_PATH );
1296 $signature = hash_hmac( 'sha1',
1297 "GET\n{$expires}\n{$contPath}/{$srcRel}",
1298 $this->swiftTempUrlKey
1299 );
1300
1301 return "{$url}?temp_url_sig={$signature}&temp_url_expires={$expires}";
1302 } else { // give S3 API URL for rgw
1303 // Path for signature starts with the bucket
1304 $spath = '/' . rawurlencode( $srcCont ) . '/' .
1305 str_replace( '%2F', '/', rawurlencode( $srcRel ) );
1306 // Calculate the hash
1307 $signature = base64_encode( hash_hmac(
1308 'sha1',
1309 "GET\n\n\n{$expires}\n{$spath}",
1310 $this->rgwS3SecretKey,
1311 true // raw
1312 ) );
1313 // See https://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html.
1314 // Note: adding a newline for empty CanonicalizedAmzHeaders does not work.
1315 // Note: S3 API is the rgw default; remove the /swift/ URL bit.
1316 return str_replace( '/swift/v1', '', $this->storageUrl( $auth ) . $spath ) .
1317 '?' .
1318 http_build_query( [
1319 'Signature' => $signature,
1320 'Expires' => $expires,
1321 'AWSAccessKeyId' => $this->rgwS3AccessKey
1322 ] );
1323 }
1324 }
1325
1326 return self::TEMPURL_ERROR;
1327 }
1328
1329 protected function directoriesAreVirtual() {
1330 return true;
1331 }
1332
1341 protected function headersFromParams( array $params ) {
1342 $hdrs = [];
1343 if ( !empty( $params['latest'] ) ) {
1344 $hdrs['x-newest'] = 'true';
1345 }
1346
1347 return $hdrs;
1348 }
1349
1350 protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1352 '@phan-var SwiftFileOpHandle[] $fileOpHandles';
1353
1355 $statuses = [];
1356
1357 // Split the HTTP requests into stages that can be done concurrently
1358 $httpReqsByStage = []; // map of (stage => index => HTTP request)
1359 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1360 $reqs = $fileOpHandle->httpOp;
1361 foreach ( $reqs as $stage => $req ) {
1362 $httpReqsByStage[$stage][$index] = $req;
1363 }
1364 $statuses[$index] = $this->newStatus();
1365 }
1366
1367 // Run all requests for the first stage, then the next, and so on
1368 $reqCount = count( $httpReqsByStage );
1369 for ( $stage = 0; $stage < $reqCount; ++$stage ) {
1370 $httpReqs = $this->requestMultiWithAuth( $httpReqsByStage[$stage] );
1371 foreach ( $httpReqs as $index => $httpReq ) {
1373 $fileOpHandle = $fileOpHandles[$index];
1374 // Run the callback for each request of this operation
1375 $status = $statuses[$index];
1376 ( $fileOpHandle->callback )( $httpReq, $status );
1377 // On failure, abort all remaining requests for this operation. This is used
1378 // in "move" operations to abort the DELETE request if the PUT request fails.
1379 if (
1380 !$status->isOK() ||
1381 $fileOpHandle->state === $fileOpHandle::CONTINUE_NO
1382 ) {
1383 $stages = count( $fileOpHandle->httpOp );
1384 for ( $s = ( $stage + 1 ); $s < $stages; ++$s ) {
1385 unset( $httpReqsByStage[$s][$index] );
1386 }
1387 }
1388 }
1389 }
1390
1391 return $statuses;
1392 }
1393
1416 protected function setContainerAccess( $container, array $readUsers, array $writeUsers ) {
1417 $status = $this->newStatus();
1418
1419 [ $rcode, , , , ] = $this->requestWithAuth( [
1420 'method' => 'POST',
1421 'container' => $container,
1422 'headers' => [
1423 'x-container-read' => implode( ',', $readUsers ),
1424 'x-container-write' => implode( ',', $writeUsers )
1425 ]
1426 ] );
1427
1428 if ( $rcode != 204 && $rcode !== 202 ) {
1429 $status->fatal( 'backend-fail-internal', $this->name );
1430 $this->logger->error( __METHOD__ . ': unexpected rcode value ({rcode})',
1431 [ 'rcode' => $rcode ] );
1432 }
1433
1434 return $status;
1435 }
1436
1445 protected function getContainerStat( $container, $bypassCache = false ) {
1447 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1448
1449 if ( $bypassCache ) { // purge cache
1450 $this->containerStatCache->clear( $container );
1451 } elseif ( !$this->containerStatCache->hasField( $container, 'stat' ) ) {
1452 $this->primeContainerCache( [ $container ] ); // check persistent cache
1453 }
1454 if ( !$this->containerStatCache->hasField( $container, 'stat' ) ) {
1455 [ $rcode, $rdesc, $rhdrs, $rbody, $rerr ] = $this->requestWithAuth( [
1456 'method' => 'HEAD',
1457 'container' => $container
1458 ] );
1459
1460 if ( $rcode === 204 ) {
1461 $stat = [
1462 'count' => $rhdrs['x-container-object-count'],
1463 'bytes' => $rhdrs['x-container-bytes-used']
1464 ];
1465 if ( $bypassCache ) {
1466 return $stat;
1467 } else {
1468 $this->containerStatCache->setField( $container, 'stat', $stat ); // cache it
1469 $this->setContainerCache( $container, $stat ); // update persistent cache
1470 }
1471 } elseif ( $rcode === 404 ) {
1472 return self::RES_ABSENT;
1473 } else {
1474 $this->onError( null, __METHOD__,
1475 [ 'cont' => $container ], $rerr, $rcode, $rdesc, $rbody );
1476
1477 return self::RES_ERROR;
1478 }
1479 }
1480
1481 return $this->containerStatCache->getField( $container, 'stat' );
1482 }
1483
1491 protected function createContainer( $container, array $params ) {
1492 $status = $this->newStatus();
1493
1494 // @see SwiftFileBackend::setContainerAccess()
1495 if ( empty( $params['noAccess'] ) ) {
1496 // public
1497 $readUsers = array_merge( $this->readUsers, [ '.r:*', $this->swiftUser ] );
1498 $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
1499 } else {
1500 // private
1501 $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
1502 $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
1503 }
1504
1505 [ $rcode, $rdesc, , $rbody, $rerr ] = $this->requestWithAuth( [
1506 'method' => 'PUT',
1507 'container' => $container,
1508 'headers' => [
1509 'x-container-read' => implode( ',', $readUsers ),
1510 'x-container-write' => implode( ',', $writeUsers )
1511 ]
1512 ] );
1513
1514 if ( $rcode === 201 ) { // new
1515 // good
1516 } elseif ( $rcode === 202 ) { // already there
1517 // this shouldn't really happen, but is OK
1518 } else {
1519 $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc, $rbody );
1520 }
1521
1522 return $status;
1523 }
1524
1532 protected function deleteContainer( $container, array $params ) {
1533 $status = $this->newStatus();
1534
1535 [ $rcode, $rdesc, , $rbody, $rerr ] = $this->requestWithAuth( [
1536 'method' => 'DELETE',
1537 'container' => $container
1538 ] );
1539
1540 if ( $rcode >= 200 && $rcode <= 299 ) { // deleted
1541 $this->containerStatCache->clear( $container ); // purge
1542 } elseif ( $rcode === 404 ) { // not there
1543 // this shouldn't really happen, but is OK
1544 } elseif ( $rcode === 409 ) { // not empty
1545 $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); // race?
1546 } else {
1547 $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc, $rbody );
1548 }
1549
1550 return $status;
1551 }
1552
1565 private function objectListing(
1566 $fullCont, $type, $limit, $after = null, $prefix = null, $delim = null
1567 ) {
1568 $status = $this->newStatus();
1569
1570 $query = [ 'limit' => $limit ];
1571 if ( $type === 'info' ) {
1572 $query['format'] = 'json';
1573 }
1574 if ( $after !== null ) {
1575 $query['marker'] = $after;
1576 }
1577 if ( $prefix !== null ) {
1578 $query['prefix'] = $prefix;
1579 }
1580 if ( $delim !== null ) {
1581 $query['delimiter'] = $delim;
1582 }
1583
1584 [ $rcode, $rdesc, , $rbody, $rerr ] = $this->requestWithAuth( [
1585 'method' => 'GET',
1586 'container' => $fullCont,
1587 'query' => $query,
1588 ] );
1589
1590 $params = [ 'cont' => $fullCont, 'prefix' => $prefix, 'delim' => $delim ];
1591 if ( $rcode === 200 ) { // good
1592 if ( $type === 'info' ) {
1593 $status->value = FormatJson::decode( trim( $rbody ) );
1594 } else {
1595 $status->value = explode( "\n", trim( $rbody ) );
1596 }
1597 } elseif ( $rcode === 204 ) {
1598 $status->value = []; // empty container
1599 } elseif ( $rcode === 404 ) {
1600 $status->value = []; // no container
1601 } else {
1602 $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc, $rbody );
1603 }
1604
1605 return $status;
1606 }
1607
1608 protected function doPrimeContainerCache( array $containerInfo ) {
1609 foreach ( $containerInfo as $container => $info ) {
1610 $this->containerStatCache->setField( $container, 'stat', $info );
1611 }
1612 }
1613
1614 protected function doGetFileStatMulti( array $params ) {
1615 $stats = [];
1616
1617 $reqs = []; // (path => op)
1618 // (a) Check the containers of the paths...
1619 foreach ( $params['srcs'] as $path ) {
1620 [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $path );
1621 if ( $srcRel === null ) {
1622 // invalid storage path
1623 $stats[$path] = self::RES_ERROR;
1624 continue;
1625 }
1626
1627 $cstat = $this->getContainerStat( $srcCont );
1628 if ( $cstat === self::RES_ABSENT ) {
1629 $stats[$path] = self::RES_ABSENT;
1630 continue; // ok, nothing to do
1631 } elseif ( $cstat === self::RES_ERROR ) {
1632 $stats[$path] = self::RES_ERROR;
1633 continue;
1634 }
1635
1636 $reqs[$path] = [
1637 'method' => 'HEAD',
1638 'container' => $srcCont,
1639 'relPath' => $srcRel,
1640 'headers' => $this->headersFromParams( $params )
1641 ];
1642 }
1643
1644 // (b) Check the files themselves...
1645 $reqs = $this->requestMultiWithAuth(
1646 $reqs,
1647 [ 'maxConnsPerHost' => $params['concurrency'] ]
1648 );
1649 foreach ( $reqs as $path => $op ) {
1650 [ $rcode, $rdesc, $rhdrs, $rbody, $rerr ] = $op['response'];
1651 if ( $rcode === 200 || $rcode === 204 ) {
1652 // Update the object if it is missing some headers
1653 if ( !empty( $params['requireSHA1'] ) ) {
1654 $rhdrs = $this->addMissingHashMetadata( $rhdrs, $path );
1655 }
1656 // Load the stat map from the headers
1657 $stat = $this->getStatFromHeaders( $rhdrs );
1658 if ( $this->isRGW ) {
1659 $stat['latest'] = true; // strong consistency
1660 }
1661 } elseif ( $rcode === 404 ) {
1662 $stat = self::RES_ABSENT;
1663 } else {
1664 $stat = self::RES_ERROR;
1665 $this->onError( null, __METHOD__, $params, $rerr, $rcode, $rdesc, $rbody );
1666 }
1667 $stats[$path] = $stat;
1668 }
1669
1670 return $stats;
1671 }
1672
1677 protected function getStatFromHeaders( array $rhdrs ) {
1678 // Fetch all of the custom metadata headers
1679 $metadata = $this->getMetadataFromHeaders( $rhdrs );
1680 // Fetch all of the custom raw HTTP headers
1681 $headers = $this->extractMutableContentHeaders( $rhdrs );
1682
1683 return [
1684 // Convert various random Swift dates to TS_MW
1685 'mtime' => $this->convertSwiftDate( $rhdrs['last-modified'], TS_MW ),
1686 // Empty objects actually return no content-length header in Ceph
1687 'size' => isset( $rhdrs['content-length'] ) ? (int)$rhdrs['content-length'] : 0,
1688 'sha1' => $metadata['sha1base36'] ?? null,
1689 // Note: manifest ETags are not an MD5 of the file
1690 'md5' => ctype_xdigit( $rhdrs['etag'] ) ? $rhdrs['etag'] : null,
1691 'xattr' => [ 'metadata' => $metadata, 'headers' => $headers ]
1692 ];
1693 }
1694
1700 protected function getAuthentication() {
1701 if ( $this->authErrorTimestamp !== null ) {
1702 $interval = time() - $this->authErrorTimestamp;
1703 if ( $interval < 60 ) {
1704 $this->logger->debug(
1705 'rejecting request since auth failure occurred {interval} seconds ago',
1706 [ 'interval' => $interval ]
1707 );
1708 return null;
1709 } else { // actually retry this time
1710 $this->authErrorTimestamp = null;
1711 }
1712 }
1713 // Authenticate with proxy and get a session key...
1714 if ( !$this->authCreds ) {
1715 $cacheKey = $this->getCredsCacheKey( $this->swiftUser );
1716 $creds = $this->srvCache->get( $cacheKey ); // credentials
1717 // Try to use the credential cache
1718 if ( isset( $creds['auth_token'] )
1719 && isset( $creds['storage_url'] )
1720 && isset( $creds['expiry_time'] )
1721 && $creds['expiry_time'] > time()
1722 ) {
1723 $this->setAuthCreds( $creds );
1724 } else { // cache miss
1725 $this->refreshAuthentication();
1726 }
1727 }
1728
1729 return $this->authCreds;
1730 }
1731
1737 private function setAuthCreds( ?array $creds ) {
1738 $this->logger->debug( 'Using auth token with expiry_time={expiry_time}',
1739 [
1740 'expiry_time' => isset( $creds['expiry_time'] )
1741 ? gmdate( 'c', $creds['expiry_time'] ) : 'null'
1742 ]
1743 );
1744 $this->authCreds = $creds;
1745 // Ceph RGW does not use <account> in URLs (OpenStack Swift uses "/v1/<account>")
1746 if ( $creds && str_ends_with( $creds['storage_url'], '/v1' ) ) {
1747 $this->isRGW = true; // take advantage of strong consistency in Ceph
1748 }
1749 }
1750
1756 private function refreshAuthentication() {
1757 [ $rcode, , $rhdrs, $rbody, ] = $this->http->run( [
1758 'method' => 'GET',
1759 'url' => "{$this->swiftAuthUrl}/v1.0",
1760 'headers' => [
1761 'x-auth-user' => $this->swiftUser,
1762 'x-auth-key' => $this->swiftKey
1763 ]
1764 ], self::DEFAULT_HTTP_OPTIONS );
1765
1766 if ( $rcode >= 200 && $rcode <= 299 ) { // OK
1767 if ( isset( $rhdrs['x-auth-token-expires'] ) ) {
1768 $ttl = intval( $rhdrs['x-auth-token-expires'] );
1769 } else {
1770 $ttl = $this->authTTL;
1771 }
1772 $expiryTime = time() + $ttl;
1773 $creds = [
1774 'auth_token' => $rhdrs['x-auth-token'],
1775 'storage_url' => $this->swiftStorageUrl ?? $rhdrs['x-storage-url'],
1776 'expiry_time' => $expiryTime,
1777 ];
1778 $this->srvCache->set( $this->getCredsCacheKey( $this->swiftUser ), $creds, $expiryTime );
1779 } elseif ( $rcode === 401 ) {
1780 $this->onError( null, __METHOD__, [], "Authentication failed.", $rcode );
1781 $this->authErrorTimestamp = time();
1782 $creds = null;
1783 } else {
1784 $this->onError( null, __METHOD__, [], "HTTP return code: $rcode", $rcode, $rbody );
1785 $this->authErrorTimestamp = time();
1786 $creds = null;
1787 }
1788 $this->setAuthCreds( $creds );
1789 return $creds;
1790 }
1791
1798 protected function storageUrl( array $creds, $container = null, $object = null ) {
1799 $parts = [ $creds['storage_url'] ];
1800 if ( strlen( $container ?? '' ) ) {
1801 $parts[] = rawurlencode( $container );
1802 }
1803 if ( strlen( $object ?? '' ) ) {
1804 $parts[] = str_replace( "%2F", "/", rawurlencode( $object ) );
1805 }
1806
1807 return implode( '/', $parts );
1808 }
1809
1814 protected function authTokenHeaders( array $creds ) {
1815 return [ 'x-auth-token' => $creds['auth_token'] ];
1816 }
1817
1824 private function getCredsCacheKey( $username ) {
1825 return 'swiftcredentials:' . md5( $username . ':' . $this->swiftAuthUrl );
1826 }
1827
1842 private function requestWithAuth( array $req, array $options = [] ) {
1843 return $this->requestMultiWithAuth( [ $req ], $options )[0]['response'];
1844 }
1845
1855 private function requestMultiWithAuth( array $reqs, $options = [] ) {
1856 $remainingTries = 2;
1857 $auth = $this->getAuthentication();
1858 while ( true ) {
1859 if ( !$auth ) {
1860 foreach ( $reqs as &$req ) {
1861 if ( !isset( $req['response'] ) ) {
1862 $req['response'] = $this->getAuthFailureResponse();
1863 }
1864 }
1865 break;
1866 }
1867 foreach ( $reqs as &$req ) {
1868 '@phan-var array $req'; // Not array[]
1869 if ( isset( $req['response'] ) ) {
1870 // Request was attempted before
1871 // Retry only if it gave a 401 response code
1872 if ( $req['response']['code'] !== 401 ) {
1873 continue;
1874 }
1875 }
1876 $req['headers'] = $this->authTokenHeaders( $auth ) + ( $req['headers'] ?? [] );
1877 $req['url'] = $this->storageUrl( $auth, $req['container'], $req['relPath'] ?? null );
1878 }
1879 unset( $req );
1880 $reqs = $this->http->runMulti( $reqs, $options + self::DEFAULT_HTTP_OPTIONS );
1881 if ( --$remainingTries > 0 ) {
1882 // Retry if any request failed with 401 "not authorized"
1883 foreach ( $reqs as $req ) {
1884 if ( $req['response']['code'] === 401 ) {
1885 $auth = $this->refreshAuthentication();
1886 continue 2;
1887 }
1888 }
1889 }
1890 break;
1891 }
1892 return $reqs;
1893 }
1894
1903 private function getAuthFailureResponse() {
1904 return [
1905 'code' => 0,
1906 0 => 0,
1907 'reason' => '',
1908 1 => '',
1909 'headers' => [],
1910 2 => [],
1911 'body' => '',
1912 3 => '',
1913 'error' => self::AUTH_FAILURE_ERROR,
1914 4 => self::AUTH_FAILURE_ERROR
1915 ];
1916 }
1917
1925 private function isAuthFailureResponse( $code, $error ) {
1926 return $code === 0 && $error === self::AUTH_FAILURE_ERROR;
1927 }
1928
1941 public function onError( $status, $func, array $params, $err = '', $code = 0, $desc = '', $body = '' ) {
1942 if ( $this->isAuthFailureResponse( $code, $err ) ) {
1943 if ( $status instanceof StatusValue ) {
1944 $status->fatal( 'backend-fail-connect', $this->name );
1945 }
1946 // Already logged
1947 return;
1948 }
1949 if ( $status instanceof StatusValue ) {
1950 $status->fatal( 'backend-fail-internal', $this->name );
1951 }
1952 $msg = "HTTP {code} ({desc}) in '{func}' (given '{req_params}')";
1953 $msgParams = [
1954 'code' => $code,
1955 'desc' => $desc,
1956 'func' => $func,
1957 'req_params' => FormatJson::encode( $params ),
1958 ];
1959 if ( $err ) {
1960 $msg .= ': {err}';
1961 $msgParams['err'] = $err;
1962 }
1963 if ( $code == 502 ) {
1964 $msg .= ' ({truncatedBody})';
1965 $msgParams['truncatedBody'] = substr( strip_tags( $body ), 0, 100 );
1966 }
1967 $this->logger->error( $msg, $msgParams );
1968 }
1969}
array $params
The job parameters.
Class representing a cache/ephemeral data store.
Definition BagOStuff.php:85
A BagOStuff object with no objects in it.
File backend exception for checked exceptions (e.g.
Base class for all backends using particular storage medium.
setContainerCache( $container, array $val)
Set the cached info for a container.
executeOpHandlesInternal(array $fileOpHandles)
Execute a list of FileBackendStoreOpHandle handles in parallel.
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
deleteFileCache( $path)
Delete the cached stat info for a file path.
getContentType( $storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
fileExists(array $params)
Check if a file exists at a storage path in the backend.
getLocalCopy(array $params)
Get a local copy on disk of the file at a storage path in the backend.
string $name
Unique backend name.
callable $obResetFunc
LoggerInterface $logger
static extensionFromPath( $path, $case='lowercase')
Get the final extension from a storage or FS path.
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
scopedProfileSection( $section)
newStatus(... $args)
Yields the result of the status wrapper callback on either:
static decode( $value, $assoc=false)
Decodes a JSON string.
Store key-value entries in a size-limited in-memory LRU cache.
Library for creating and parsing MW-style timestamps.
Class to handle multiple HTTP requests.
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Iterator for listing directories.
Iterator for listing regular files.
Class for an OpenStack Swift (or Ceph RGW) based file backend.
string $swiftUser
Swift user (account:user) to authenticate as.
string $swiftAuthUrl
Authentication base URL (without version)
string $swiftTempUrlKey
Shared secret value for making temp URLs.
MapCacheLRU $containerStatCache
Container stat cache.
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path in the backend.
getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
doPublishInternal( $fullCont, $dir, array $params)
doCreateInternal(array $params)
doGetFileStatMulti(array $params)
Get file stat information (concurrently if possible) for several files.
doGetFileSha1base36(array $params)
int null $authErrorTimestamp
UNIX timestamp.
array $writeUsers
Additional users (account:user) with write permissions on public containers.
__construct(array $config)
MultiHttpClient $http
doGetFileXAttributes(array $params)
authTokenHeaders(array $creds)
getStatFromHeaders(array $rhdrs)
string $swiftStorageUrl
Override of storage base URL.
createContainer( $container, array $params)
Create a Swift container.
doCopyInternal(array $params)
getDirectoryListInternal( $fullCont, $dir, array $params)
string $rgwS3AccessKey
S3 access key (RADOS Gateway)
setContainerAccess( $container, array $readUsers, array $writeUsers)
Set read/write permissions for a Swift container.
getFileHttpUrl(array $params)
array $secureWriteUsers
Additional users (account:user) with write permissions on private containers.
extractMetadataHeaders(array $headers)
int $authTTL
TTL in seconds.
headersFromParams(array $params)
Get headers to send to Swift when reading a file based on a FileBackend params array,...
bool $isRGW
Whether the server is an Ceph RGW.
doStoreInternal(array $params)
onError( $status, $func, array $params, $err='', $code=0, $desc='', $body='')
Log an unexpected exception for this backend.
loadListingStatInternal( $path, array $val)
Do not call this function outside of SwiftFileBackendFileList.
doPrepareInternal( $fullCont, $dir, array $params)
FileBackendStore::doPrepare() to override StatusValue Good status without value for success,...
setLogger(LoggerInterface $logger)
doSecureInternal( $fullCont, $dir, array $params)
getFileListInternal( $fullCont, $dir, array $params)
getMetadataFromHeaders(array $headers)
doMoveInternal(array $params)
addMissingHashMetadata(array $objHdrs, $path)
Fill in any missing object metadata and save it to Swift.
getFeatures()
Get the a bitfield of extra features supported by the backend medium.
deleteContainer( $container, array $params)
Delete a Swift container.
doGetFileStat(array $params)
getAuthentication()
Get the cached auth token.
doGetLocalCopyMulti(array $params)
string $rgwS3SecretKey
S3 authentication key (RADOS Gateway)
doGetFileContentsMulti(array $params)
storageUrl(array $creds, $container=null, $object=null)
convertSwiftDate( $ts, $format=TS_MW)
Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT"/"2013-05-11T07:37:27.678360Z".
doStreamFile(array $params)
doPrimeContainerCache(array $containerInfo)
Fill the backend-specific process cache given an array of resolved container names and their correspo...
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
array $readUsers
Additional users (account:user) with read permissions on public containers.
array $secureReadUsers
Additional users (account:user) with read permissions on private containers.
doCleanInternal( $fullCont, $dir, array $params)
getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
string $swiftKey
Secret key for user.
doDirectoryExists( $fullCont, $dir, array $params)
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
doExecuteOpHandlesInternal(array $fileOpHandles)
doDeleteInternal(array $params)
doDescribeInternal(array $params)
extractMutableContentHeaders(array $headers)
Filter/normalize a header map to only include mutable "content-"/"x-content-" headers.
getContainerStat( $container, $bypassCache=false)
Get a Swift container stat map, possibly from process cache.
Multi-datacenter aware caching interface.
This program is free software; you can redistribute it and/or modify it under the terms of the GNU Ge...
$header