MediaWiki  master
SwiftFileBackend.php
Go to the documentation of this file.
1 <?php
25 use Psr\Log\LoggerInterface;
26 use Wikimedia\AtEase\AtEase;
27 use Wikimedia\RequestTimeout\TimeoutException;
28 
39  private const DEFAULT_HTTP_OPTIONS = [ 'httpVersion' => 'v1.1' ];
40 
42  protected $http;
44  protected $authTTL;
46  protected $swiftAuthUrl;
48  protected $swiftStorageUrl;
50  protected $swiftUser;
52  protected $swiftKey;
54  protected $swiftTempUrlKey;
56  protected $rgwS3AccessKey;
58  protected $rgwS3SecretKey;
60  protected $readUsers;
62  protected $writeUsers;
64  protected $secureReadUsers;
66  protected $secureWriteUsers;
67 
69  protected $srvCache;
70 
73 
75  protected $authCreds;
77  protected $authSessionTimestamp = 0;
79  protected $authErrorTimestamp = null;
80 
82  protected $isRGW = false;
83 
122  public function __construct( array $config ) {
123  parent::__construct( $config );
124  // Required settings
125  $this->swiftAuthUrl = $config['swiftAuthUrl'];
126  $this->swiftUser = $config['swiftUser'];
127  $this->swiftKey = $config['swiftKey'];
128  // Optional settings
129  $this->authTTL = $config['swiftAuthTTL'] ?? 15 * 60; // some sensible number
130  $this->swiftTempUrlKey = $config['swiftTempUrlKey'] ?? '';
131  $this->swiftStorageUrl = $config['swiftStorageUrl'] ?? null;
132  $this->shardViaHashLevels = $config['shardViaHashLevels'] ?? '';
133  $this->rgwS3AccessKey = $config['rgwS3AccessKey'] ?? '';
134  $this->rgwS3SecretKey = $config['rgwS3SecretKey'] ?? '';
135 
136  // HTTP helper client
137  $httpOptions = [];
138  foreach ( [ 'connTimeout', 'reqTimeout' ] as $optionName ) {
139  if ( isset( $config[$optionName] ) ) {
140  $httpOptions[$optionName] = $config[$optionName];
141  }
142  }
143  $this->http = new MultiHttpClient( $httpOptions );
144  $this->http->setLogger( $this->logger );
145 
146  // Cache container information to mask latency
147  if ( isset( $config['wanCache'] ) && $config['wanCache'] instanceof WANObjectCache ) {
148  $this->memCache = $config['wanCache'];
149  }
150  // Process cache for container info
151  $this->containerStatCache = new MapCacheLRU( 300 );
152  // Cache auth token information to avoid RTTs
153  if ( !empty( $config['cacheAuthInfo'] ) && isset( $config['srvCache'] ) ) {
154  $this->srvCache = $config['srvCache'];
155  } else {
156  $this->srvCache = new EmptyBagOStuff();
157  }
158  $this->readUsers = $config['readUsers'] ?? [];
159  $this->writeUsers = $config['writeUsers'] ?? [];
160  $this->secureReadUsers = $config['secureReadUsers'] ?? [];
161  $this->secureWriteUsers = $config['secureWriteUsers'] ?? [];
162  }
163 
164  public function setLogger( LoggerInterface $logger ) {
165  parent::setLogger( $logger );
166  $this->http->setLogger( $logger );
167  }
168 
169  public function getFeatures() {
170  return (
171  self::ATTR_UNICODE_PATHS |
172  self::ATTR_HEADERS |
173  self::ATTR_METADATA
174  );
175  }
176 
177  protected function resolveContainerPath( $container, $relStoragePath ) {
178  if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) {
179  return null; // not UTF-8, makes it hard to use CF and the swift HTTP API
180  } elseif ( strlen( rawurlencode( $relStoragePath ) ) > 1024 ) {
181  return null; // too long for Swift
182  }
183 
184  return $relStoragePath;
185  }
186 
187  public function isPathUsableInternal( $storagePath ) {
188  [ $container, $rel ] = $this->resolveStoragePathReal( $storagePath );
189  if ( $rel === null ) {
190  return false; // invalid
191  }
192 
193  return is_array( $this->getContainerStat( $container ) );
194  }
195 
205  protected function extractMutableContentHeaders( array $headers ) {
206  $contentHeaders = [];
207  // Normalize casing, and strip out illegal headers
208  foreach ( $headers as $name => $value ) {
209  $name = strtolower( $name );
210  if ( $name === 'x-delete-at' && is_numeric( $value ) ) {
211  // Expects a Unix Epoch date
212  $contentHeaders[$name] = $value;
213  } elseif ( $name === 'x-delete-after' && is_numeric( $value ) ) {
214  // Expects number of minutes time to live.
215  $contentHeaders[$name] = $value;
216  } elseif ( preg_match( '/^(x-)?content-(?!length$)/', $name ) ) {
217  // Only allow content-* and x-content-* headers (but not content-length)
218  $contentHeaders[$name] = $value;
219  } elseif ( $name === 'content-type' && strlen( $value ) ) {
220  // This header can be set to a value but not unset
221  $contentHeaders[$name] = $value;
222  }
223  }
224  // By default, Swift has annoyingly low maximum header value limits
225  if ( isset( $contentHeaders['content-disposition'] ) ) {
226  $maxLength = 255;
227  // @note: assume FileBackend::makeContentDisposition() already used
228  $offset = $maxLength - strlen( $contentHeaders['content-disposition'] );
229  if ( $offset < 0 ) {
230  $pos = strrpos( $contentHeaders['content-disposition'], ';', $offset );
231  $contentHeaders['content-disposition'] = $pos === false
232  ? ''
233  : trim( substr( $contentHeaders['content-disposition'], 0, $pos ) );
234  }
235  }
236 
237  return $contentHeaders;
238  }
239 
245  protected function extractMetadataHeaders( array $headers ) {
246  $metadataHeaders = [];
247  foreach ( $headers as $name => $value ) {
248  $name = strtolower( $name );
249  if ( strpos( $name, 'x-object-meta-' ) === 0 ) {
250  $metadataHeaders[$name] = $value;
251  }
252  }
253 
254  return $metadataHeaders;
255  }
256 
262  protected function getMetadataFromHeaders( array $headers ) {
263  $prefixLen = strlen( 'x-object-meta-' );
264 
265  $metadata = [];
266  foreach ( $this->extractMetadataHeaders( $headers ) as $name => $value ) {
267  $metadata[substr( $name, $prefixLen )] = $value;
268  }
269 
270  return $metadata;
271  }
272 
273  protected function doCreateInternal( array $params ) {
274  $status = $this->newStatus();
275 
276  [ $dstCont, $dstRel ] = $this->resolveStoragePathReal( $params['dst'] );
277  if ( $dstRel === null ) {
278  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
279 
280  return $status;
281  }
282 
283  // Headers that are not strictly a function of the file content
284  $mutableHeaders = $this->extractMutableContentHeaders( $params['headers'] ?? [] );
285  // Make sure that the "content-type" header is set to something sensible
286  $mutableHeaders['content-type']
287  ??= $this->getContentType( $params['dst'], $params['content'], null );
288 
289  $reqs = [ [
290  'method' => 'PUT',
291  'url' => [ $dstCont, $dstRel ],
292  'headers' => array_merge(
293  $mutableHeaders,
294  [
295  'etag' => md5( $params['content'] ),
296  'content-length' => strlen( $params['content'] ),
297  'x-object-meta-sha1base36' =>
298  Wikimedia\base_convert( sha1( $params['content'] ), 16, 36, 31 )
299  ]
300  ),
301  'body' => $params['content']
302  ] ];
303 
304  $method = __METHOD__;
305  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
306  [ $rcode, $rdesc, , , $rerr ] = $request['response'];
307  if ( $rcode === 201 || $rcode === 202 ) {
308  // good
309  } elseif ( $rcode === 412 ) {
310  $status->fatal( 'backend-fail-contenttype', $params['dst'] );
311  } else {
312  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
313  }
314 
315  return SwiftFileOpHandle::CONTINUE_IF_OK;
316  };
317 
318  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
319  if ( !empty( $params['async'] ) ) { // deferred
320  $status->value = $opHandle;
321  } else { // actually write the object in Swift
322  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
323  }
324 
325  return $status;
326  }
327 
328  protected function doStoreInternal( array $params ) {
329  $status = $this->newStatus();
330 
331  [ $dstCont, $dstRel ] = $this->resolveStoragePathReal( $params['dst'] );
332  if ( $dstRel === null ) {
333  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
334 
335  return $status;
336  }
337 
338  // Open a handle to the source file so that it can be streamed. The size and hash
339  // will be computed using the handle. In the off chance that the source file changes
340  // during this operation, the PUT will fail due to an ETag mismatch and be aborted.
341  AtEase::suppressWarnings();
342  $srcHandle = fopen( $params['src'], 'rb' );
343  AtEase::restoreWarnings();
344  if ( $srcHandle === false ) { // source doesn't exist?
345  $status->fatal( 'backend-fail-notexists', $params['src'] );
346 
347  return $status;
348  }
349 
350  // Compute the MD5 and SHA-1 hashes in one pass
351  $srcSize = fstat( $srcHandle )['size'];
352  $md5Context = hash_init( 'md5' );
353  $sha1Context = hash_init( 'sha1' );
354  $hashDigestSize = 0;
355  while ( !feof( $srcHandle ) ) {
356  $buffer = (string)fread( $srcHandle, 131072 ); // 128 KiB
357  hash_update( $md5Context, $buffer );
358  hash_update( $sha1Context, $buffer );
359  $hashDigestSize += strlen( $buffer );
360  }
361  // Reset the handle back to the beginning so that it can be streamed
362  rewind( $srcHandle );
363 
364  if ( $hashDigestSize !== $srcSize ) {
365  $status->fatal( 'backend-fail-hash', $params['src'] );
366 
367  return $status;
368  }
369 
370  // Headers that are not strictly a function of the file content
371  $mutableHeaders = $this->extractMutableContentHeaders( $params['headers'] ?? [] );
372  // Make sure that the "content-type" header is set to something sensible
373  $mutableHeaders['content-type']
374  ??= $this->getContentType( $params['dst'], null, $params['src'] );
375 
376  $reqs = [ [
377  'method' => 'PUT',
378  'url' => [ $dstCont, $dstRel ],
379  'headers' => array_merge(
380  $mutableHeaders,
381  [
382  'content-length' => $srcSize,
383  'etag' => hash_final( $md5Context ),
384  'x-object-meta-sha1base36' =>
385  Wikimedia\base_convert( hash_final( $sha1Context ), 16, 36, 31 )
386  ]
387  ),
388  'body' => $srcHandle // resource
389  ] ];
390 
391  $method = __METHOD__;
392  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
393  [ $rcode, $rdesc, , , $rerr ] = $request['response'];
394  if ( $rcode === 201 || $rcode === 202 ) {
395  // good
396  } elseif ( $rcode === 412 ) {
397  $status->fatal( 'backend-fail-contenttype', $params['dst'] );
398  } else {
399  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
400  }
401 
402  return SwiftFileOpHandle::CONTINUE_IF_OK;
403  };
404 
405  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
406  $opHandle->resourcesToClose[] = $srcHandle;
407 
408  if ( !empty( $params['async'] ) ) { // deferred
409  $status->value = $opHandle;
410  } else { // actually write the object in Swift
411  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
412  }
413 
414  return $status;
415  }
416 
417  protected function doCopyInternal( array $params ) {
418  $status = $this->newStatus();
419 
420  [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $params['src'] );
421  if ( $srcRel === null ) {
422  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
423 
424  return $status;
425  }
426 
427  [ $dstCont, $dstRel ] = $this->resolveStoragePathReal( $params['dst'] );
428  if ( $dstRel === null ) {
429  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
430 
431  return $status;
432  }
433 
434  $reqs = [ [
435  'method' => 'PUT',
436  'url' => [ $dstCont, $dstRel ],
437  'headers' => array_merge(
438  $this->extractMutableContentHeaders( $params['headers'] ?? [] ),
439  [
440  'x-copy-from' => '/' . rawurlencode( $srcCont ) . '/' .
441  str_replace( "%2F", "/", rawurlencode( $srcRel ) )
442  ]
443  )
444  ] ];
445 
446  $method = __METHOD__;
447  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
448  [ $rcode, $rdesc, , , $rerr ] = $request['response'];
449  if ( $rcode === 201 ) {
450  // good
451  } elseif ( $rcode === 404 ) {
452  if ( empty( $params['ignoreMissingSource'] ) ) {
453  $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
454  }
455  } else {
456  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
457  }
458 
459  return SwiftFileOpHandle::CONTINUE_IF_OK;
460  };
461 
462  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
463  if ( !empty( $params['async'] ) ) { // deferred
464  $status->value = $opHandle;
465  } else { // actually write the object in Swift
466  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
467  }
468 
469  return $status;
470  }
471 
472  protected function doMoveInternal( array $params ) {
473  $status = $this->newStatus();
474 
475  [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $params['src'] );
476  if ( $srcRel === null ) {
477  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
478 
479  return $status;
480  }
481 
482  [ $dstCont, $dstRel ] = $this->resolveStoragePathReal( $params['dst'] );
483  if ( $dstRel === null ) {
484  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
485 
486  return $status;
487  }
488 
489  $reqs = [ [
490  'method' => 'PUT',
491  'url' => [ $dstCont, $dstRel ],
492  'headers' => array_merge(
493  $this->extractMutableContentHeaders( $params['headers'] ?? [] ),
494  [
495  'x-copy-from' => '/' . rawurlencode( $srcCont ) . '/' .
496  str_replace( "%2F", "/", rawurlencode( $srcRel ) )
497  ]
498  )
499  ] ];
500  if ( "{$srcCont}/{$srcRel}" !== "{$dstCont}/{$dstRel}" ) {
501  $reqs[] = [
502  'method' => 'DELETE',
503  'url' => [ $srcCont, $srcRel ],
504  'headers' => []
505  ];
506  }
507 
508  $method = __METHOD__;
509  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
510  [ $rcode, $rdesc, , , $rerr ] = $request['response'];
511  if ( $request['method'] === 'PUT' && $rcode === 201 ) {
512  // good
513  } elseif ( $request['method'] === 'DELETE' && $rcode === 204 ) {
514  // good
515  } elseif ( $rcode === 404 ) {
516  if ( empty( $params['ignoreMissingSource'] ) ) {
517  $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
518  } else {
519  // Leave Status as OK but skip the DELETE request
520  return SwiftFileOpHandle::CONTINUE_NO;
521  }
522  } else {
523  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
524  }
525 
526  return SwiftFileOpHandle::CONTINUE_IF_OK;
527  };
528 
529  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
530  if ( !empty( $params['async'] ) ) { // deferred
531  $status->value = $opHandle;
532  } else { // actually move the object in Swift
533  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
534  }
535 
536  return $status;
537  }
538 
539  protected function doDeleteInternal( array $params ) {
540  $status = $this->newStatus();
541 
542  [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $params['src'] );
543  if ( $srcRel === null ) {
544  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
545 
546  return $status;
547  }
548 
549  $reqs = [ [
550  'method' => 'DELETE',
551  'url' => [ $srcCont, $srcRel ],
552  'headers' => []
553  ] ];
554 
555  $method = __METHOD__;
556  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
557  [ $rcode, $rdesc, , , $rerr ] = $request['response'];
558  if ( $rcode === 204 ) {
559  // good
560  } elseif ( $rcode === 404 ) {
561  if ( empty( $params['ignoreMissingSource'] ) ) {
562  $status->fatal( 'backend-fail-delete', $params['src'] );
563  }
564  } else {
565  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
566  }
567 
568  return SwiftFileOpHandle::CONTINUE_IF_OK;
569  };
570 
571  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
572  if ( !empty( $params['async'] ) ) { // deferred
573  $status->value = $opHandle;
574  } else { // actually delete the object in Swift
575  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
576  }
577 
578  return $status;
579  }
580 
581  protected function doDescribeInternal( array $params ) {
582  $status = $this->newStatus();
583 
584  [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $params['src'] );
585  if ( $srcRel === null ) {
586  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
587 
588  return $status;
589  }
590 
591  // Fetch the old object headers/metadata...this should be in stat cache by now
592  $stat = $this->getFileStat( [ 'src' => $params['src'], 'latest' => 1 ] );
593  if ( $stat && !isset( $stat['xattr'] ) ) { // older cache entry
594  $stat = $this->doGetFileStat( [ 'src' => $params['src'], 'latest' => 1 ] );
595  }
596  if ( !$stat ) {
597  $status->fatal( 'backend-fail-describe', $params['src'] );
598 
599  return $status;
600  }
601 
602  // Swift object POST clears any prior headers, so merge the new and old headers here.
603  // Also, during, POST, libcurl adds "Content-Type: application/x-www-form-urlencoded"
604  // if "Content-Type" is not set, which would clobber the header value for the object.
605  $oldMetadataHeaders = [];
606  foreach ( $stat['xattr']['metadata'] as $name => $value ) {
607  $oldMetadataHeaders["x-object-meta-$name"] = $value;
608  }
609  $newContentHeaders = $this->extractMutableContentHeaders( $params['headers'] ?? [] );
610  $oldContentHeaders = $stat['xattr']['headers'];
611 
612  $reqs = [ [
613  'method' => 'POST',
614  'url' => [ $srcCont, $srcRel ],
615  'headers' => $oldMetadataHeaders + $newContentHeaders + $oldContentHeaders
616  ] ];
617 
618  $method = __METHOD__;
619  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
620  [ $rcode, $rdesc, , , $rerr ] = $request['response'];
621  if ( $rcode === 202 ) {
622  // good
623  } elseif ( $rcode === 404 ) {
624  $status->fatal( 'backend-fail-describe', $params['src'] );
625  } else {
626  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
627  }
628  };
629 
630  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
631  if ( !empty( $params['async'] ) ) { // deferred
632  $status->value = $opHandle;
633  } else { // actually change the object in Swift
634  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
635  }
636 
637  return $status;
638  }
639 
643  protected function doPrepareInternal( $fullCont, $dir, array $params ) {
644  $status = $this->newStatus();
645 
646  // (a) Check if container already exists
647  $stat = $this->getContainerStat( $fullCont );
648  if ( is_array( $stat ) ) {
649  return $status; // already there
650  } elseif ( $stat === self::$RES_ERROR ) {
651  $status->fatal( 'backend-fail-internal', $this->name );
652  $this->logger->error( __METHOD__ . ': cannot get container stat' );
653 
654  return $status;
655  }
656 
657  // (b) Create container as needed with proper ACLs
658  if ( $stat === false ) {
659  $params['op'] = 'prepare';
660  $status->merge( $this->createContainer( $fullCont, $params ) );
661  }
662 
663  return $status;
664  }
665 
666  protected function doSecureInternal( $fullCont, $dir, array $params ) {
667  $status = $this->newStatus();
668  if ( empty( $params['noAccess'] ) ) {
669  return $status; // nothing to do
670  }
671 
672  $stat = $this->getContainerStat( $fullCont );
673  if ( is_array( $stat ) ) {
674  $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
675  $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
676  // Make container private to end-users...
677  $status->merge( $this->setContainerAccess(
678  $fullCont,
679  $readUsers,
681  ) );
682  } elseif ( $stat === false ) {
683  $status->fatal( 'backend-fail-usable', $params['dir'] );
684  } else {
685  $status->fatal( 'backend-fail-internal', $this->name );
686  $this->logger->error( __METHOD__ . ': cannot get container stat' );
687  }
688 
689  return $status;
690  }
691 
692  protected function doPublishInternal( $fullCont, $dir, array $params ) {
693  $status = $this->newStatus();
694 
695  $stat = $this->getContainerStat( $fullCont );
696  if ( is_array( $stat ) ) {
697  $readUsers = array_merge( $this->readUsers, [ $this->swiftUser, '.r:*' ] );
698  $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
699 
700  // Make container public to end-users...
701  $status->merge( $this->setContainerAccess(
702  $fullCont,
703  $readUsers,
705  ) );
706  } elseif ( $stat === false ) {
707  $status->fatal( 'backend-fail-usable', $params['dir'] );
708  } else {
709  $status->fatal( 'backend-fail-internal', $this->name );
710  $this->logger->error( __METHOD__ . ': cannot get container stat' );
711  }
712 
713  return $status;
714  }
715 
716  protected function doCleanInternal( $fullCont, $dir, array $params ) {
717  $status = $this->newStatus();
718 
719  // Only containers themselves can be removed, all else is virtual
720  if ( $dir != '' ) {
721  return $status; // nothing to do
722  }
723 
724  // (a) Check the container
725  $stat = $this->getContainerStat( $fullCont, true );
726  if ( $stat === false ) {
727  return $status; // ok, nothing to do
728  } elseif ( !is_array( $stat ) ) {
729  $status->fatal( 'backend-fail-internal', $this->name );
730  $this->logger->error( __METHOD__ . ': cannot get container stat' );
731 
732  return $status;
733  }
734 
735  // (b) Delete the container if empty
736  if ( $stat['count'] == 0 ) {
737  $params['op'] = 'clean';
738  $status->merge( $this->deleteContainer( $fullCont, $params ) );
739  }
740 
741  return $status;
742  }
743 
744  protected function doGetFileStat( array $params ) {
745  $params = [ 'srcs' => [ $params['src'] ], 'concurrency' => 1 ] + $params;
746  unset( $params['src'] );
747  $stats = $this->doGetFileStatMulti( $params );
748 
749  return reset( $stats );
750  }
751 
762  protected function convertSwiftDate( $ts, $format = TS_MW ) {
763  try {
764  $timestamp = new MWTimestamp( $ts );
765 
766  return $timestamp->getTimestamp( $format );
767  } catch ( TimeoutException $e ) {
768  throw $e;
769  } catch ( Exception $e ) {
770  throw new FileBackendError( $e->getMessage() );
771  }
772  }
773 
781  protected function addMissingHashMetadata( array $objHdrs, $path ) {
782  if ( isset( $objHdrs['x-object-meta-sha1base36'] ) ) {
783  return $objHdrs; // nothing to do
784  }
785 
787  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
788  $this->logger->error( __METHOD__ . ": {path} was not stored with SHA-1 metadata.",
789  [ 'path' => $path ] );
790 
791  $objHdrs['x-object-meta-sha1base36'] = false;
792 
793  $auth = $this->getAuthentication();
794  if ( !$auth ) {
795  return $objHdrs; // failed
796  }
797 
798  // Find prior custom HTTP headers
799  $postHeaders = $this->extractMutableContentHeaders( $objHdrs );
800  // Find prior metadata headers
801  $postHeaders += $this->extractMetadataHeaders( $objHdrs );
802 
803  $status = $this->newStatus();
805  $scopeLockS = $this->getScopedFileLocks( [ $path ], LockManager::LOCK_UW, $status );
806  if ( $status->isOK() ) {
807  $tmpFile = $this->getLocalCopy( [ 'src' => $path, 'latest' => 1 ] );
808  if ( $tmpFile ) {
809  $hash = $tmpFile->getSha1Base36();
810  if ( $hash !== false ) {
811  $objHdrs['x-object-meta-sha1base36'] = $hash;
812  // Merge new SHA1 header into the old ones
813  $postHeaders['x-object-meta-sha1base36'] = $hash;
814  [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $path );
815  [ $rcode ] = $this->http->run( [
816  'method' => 'POST',
817  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
818  'headers' => $this->authTokenHeaders( $auth ) + $postHeaders
819  ], self::DEFAULT_HTTP_OPTIONS );
820  if ( $rcode >= 200 && $rcode <= 299 ) {
821  $this->deleteFileCache( $path );
822 
823  return $objHdrs; // success
824  }
825  }
826  }
827  }
828 
829  $this->logger->error( __METHOD__ . ': unable to set SHA-1 metadata for {path}',
830  [ 'path' => $path ] );
831 
832  return $objHdrs; // failed
833  }
834 
835  protected function doGetFileContentsMulti( array $params ) {
836  $auth = $this->getAuthentication();
837 
838  $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging
839  // Blindly create tmp files and stream to them, catching any exception
840  // if the file does not exist. Do not waste time doing file stats here.
841  $reqs = []; // (path => op)
842 
843  // Initial dummy values to preserve path order
844  $contents = array_fill_keys( $params['srcs'], self::$RES_ERROR );
845  foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch
846  [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $path );
847  if ( $srcRel === null || !$auth ) {
848  continue; // invalid storage path or auth error
849  }
850  // Create a new temporary memory file...
851  $handle = fopen( 'php://temp', 'wb' );
852  if ( $handle ) {
853  $reqs[$path] = [
854  'method' => 'GET',
855  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
856  'headers' => $this->authTokenHeaders( $auth )
857  + $this->headersFromParams( $params ),
858  'stream' => $handle,
859  ];
860  }
861  }
862 
863  $opts = [
864  'maxConnsPerHost' => $params['concurrency'],
865  ] + self::DEFAULT_HTTP_OPTIONS;
866  $reqs = $this->http->runMulti( $reqs, $opts );
867  foreach ( $reqs as $path => $op ) {
868  [ $rcode, $rdesc, $rhdrs, , $rerr ] = $op['response'];
869  if ( $rcode >= 200 && $rcode <= 299 ) {
870  rewind( $op['stream'] ); // start from the beginning
871  $content = (string)stream_get_contents( $op['stream'] );
872  $size = strlen( $content );
873  // Make sure that stream finished
874  if ( $size === (int)$rhdrs['content-length'] ) {
875  $contents[$path] = $content;
876  } else {
877  $contents[$path] = self::$RES_ERROR;
878  $rerr = "Got {$size}/{$rhdrs['content-length']} bytes";
879  $this->onError( null, __METHOD__,
880  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
881  }
882  } elseif ( $rcode === 404 ) {
883  $contents[$path] = self::$RES_ABSENT;
884  } else {
885  $contents[$path] = self::$RES_ERROR;
886  $this->onError( null, __METHOD__,
887  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
888  }
889  fclose( $op['stream'] ); // close open handle
890  }
891 
892  return $contents;
893  }
894 
895  protected function doDirectoryExists( $fullCont, $dir, array $params ) {
896  $prefix = ( $dir == '' ) ? null : "{$dir}/";
897  $status = $this->objectListing( $fullCont, 'names', 1, null, $prefix );
898  if ( $status->isOK() ) {
899  return ( count( $status->value ) ) > 0;
900  }
901 
902  return self::$RES_ERROR;
903  }
904 
912  public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
913  return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
914  }
915 
923  public function getFileListInternal( $fullCont, $dir, array $params ) {
924  return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
925  }
926 
938  public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
939  $dirs = [];
940  if ( $after === INF ) {
941  return $dirs; // nothing more
942  }
943 
945  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
946 
947  $prefix = ( $dir == '' ) ? null : "{$dir}/";
948  // Non-recursive: only list dirs right under $dir
949  if ( !empty( $params['topOnly'] ) ) {
950  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
951  if ( !$status->isOK() ) {
952  throw new FileBackendError( "Iterator page I/O error." );
953  }
954  $objects = $status->value;
955  // @phan-suppress-next-line PhanTypeSuspiciousNonTraversableForeach
956  foreach ( $objects as $object ) { // files and directories
957  if ( substr( $object, -1 ) === '/' ) {
958  $dirs[] = $object; // directories end in '/'
959  }
960  }
961  } else {
962  // Recursive: list all dirs under $dir and its subdirs
963  $getParentDir = static function ( $path ) {
964  return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
965  };
966 
967  // Get directory from last item of prior page
968  $lastDir = $getParentDir( $after ); // must be first page
969  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
970 
971  if ( !$status->isOK() ) {
972  throw new FileBackendError( "Iterator page I/O error." );
973  }
974 
975  $objects = $status->value;
976 
977  // @phan-suppress-next-line PhanTypeSuspiciousNonTraversableForeach
978  foreach ( $objects as $object ) { // files
979  $objectDir = $getParentDir( $object ); // directory of object
980 
981  if ( $objectDir !== false && $objectDir !== $dir ) {
982  // Swift stores paths in UTF-8, using binary sorting.
983  // See function "create_container_table" in common/db.py.
984  // If a directory is not "greater" than the last one,
985  // then it was already listed by the calling iterator.
986  if ( strcmp( $objectDir, $lastDir ) > 0 ) {
987  $pDir = $objectDir;
988  do { // add dir and all its parent dirs
989  $dirs[] = "{$pDir}/";
990  $pDir = $getParentDir( $pDir );
991  } while ( $pDir !== false
992  && strcmp( $pDir, $lastDir ) > 0 // not done already
993  && strlen( $pDir ) > strlen( $dir ) // within $dir
994  );
995  }
996  $lastDir = $objectDir;
997  }
998  }
999  }
1000  // Page on the unfiltered directory listing (what is returned may be filtered)
1001  if ( count( $objects ) < $limit ) {
1002  $after = INF; // avoid a second RTT
1003  } else {
1004  $after = end( $objects ); // update last item
1005  }
1006 
1007  return $dirs;
1008  }
1009 
1021  public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
1022  $files = []; // list of (path, stat array or null) entries
1023  if ( $after === INF ) {
1024  return $files; // nothing more
1025  }
1026 
1028  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1029 
1030  $prefix = ( $dir == '' ) ? null : "{$dir}/";
1031  // $objects will contain a list of unfiltered names or stdClass items
1032  // Non-recursive: only list files right under $dir
1033  if ( !empty( $params['topOnly'] ) ) {
1034  if ( !empty( $params['adviseStat'] ) ) {
1035  $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix, '/' );
1036  } else {
1037  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
1038  }
1039  } else {
1040  // Recursive: list all files under $dir and its subdirs
1041  if ( !empty( $params['adviseStat'] ) ) {
1042  $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix );
1043  } else {
1044  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
1045  }
1046  }
1047 
1048  // Reformat this list into a list of (name, stat array or null) entries
1049  if ( !$status->isOK() ) {
1050  throw new FileBackendError( "Iterator page I/O error." );
1051  }
1052 
1053  $objects = $status->value;
1054  $files = $this->buildFileObjectListing( $objects );
1055 
1056  // Page on the unfiltered object listing (what is returned may be filtered)
1057  if ( count( $objects ) < $limit ) {
1058  $after = INF; // avoid a second RTT
1059  } else {
1060  $after = end( $objects ); // update last item
1061  $after = is_object( $after ) ? $after->name : $after;
1062  }
1063 
1064  return $files;
1065  }
1066 
1074  private function buildFileObjectListing( array $objects ) {
1075  $names = [];
1076  foreach ( $objects as $object ) {
1077  if ( is_object( $object ) ) {
1078  if ( isset( $object->subdir ) || !isset( $object->name ) ) {
1079  continue; // virtual directory entry; ignore
1080  }
1081  $stat = [
1082  // Convert various random Swift dates to TS_MW
1083  'mtime' => $this->convertSwiftDate( $object->last_modified, TS_MW ),
1084  'size' => (int)$object->bytes,
1085  'sha1' => null,
1086  // Note: manifest ETags are not an MD5 of the file
1087  'md5' => ctype_xdigit( $object->hash ) ? $object->hash : null,
1088  'latest' => false // eventually consistent
1089  ];
1090  $names[] = [ $object->name, $stat ];
1091  } elseif ( substr( $object, -1 ) !== '/' ) {
1092  // Omit directories, which end in '/' in listings
1093  $names[] = [ $object, null ];
1094  }
1095  }
1096 
1097  return $names;
1098  }
1099 
1106  public function loadListingStatInternal( $path, array $val ) {
1107  $this->cheapCache->setField( $path, 'stat', $val );
1108  }
1109 
1110  protected function doGetFileXAttributes( array $params ) {
1111  $stat = $this->getFileStat( $params );
1112  // Stat entries filled by file listings don't include metadata/headers
1113  if ( is_array( $stat ) && !isset( $stat['xattr'] ) ) {
1114  $this->clearCache( [ $params['src'] ] );
1115  $stat = $this->getFileStat( $params );
1116  }
1117 
1118  if ( is_array( $stat ) ) {
1119  return $stat['xattr'];
1120  }
1121 
1122  return ( $stat === self::$RES_ERROR ) ? self::$RES_ERROR : self::$RES_ABSENT;
1123  }
1124 
1125  protected function doGetFileSha1base36( array $params ) {
1126  // Avoid using stat entries from file listings, which never include the SHA-1 hash.
1127  // Also, recompute the hash if it's not part of the metadata headers for some reason.
1128  $params['requireSHA1'] = true;
1129 
1130  $stat = $this->getFileStat( $params );
1131  if ( is_array( $stat ) ) {
1132  return $stat['sha1'];
1133  }
1134 
1135  return ( $stat === self::$RES_ERROR ) ? self::$RES_ERROR : self::$RES_ABSENT;
1136  }
1137 
1138  protected function doStreamFile( array $params ) {
1139  $status = $this->newStatus();
1140 
1141  $flags = !empty( $params['headless'] ) ? HTTPFileStreamer::STREAM_HEADLESS : 0;
1142 
1143  [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $params['src'] );
1144  if ( $srcRel === null ) {
1145  HTTPFileStreamer::send404Message( $params['src'], $flags );
1146  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
1147 
1148  return $status;
1149  }
1150 
1151  $auth = $this->getAuthentication();
1152  if ( !$auth || !is_array( $this->getContainerStat( $srcCont ) ) ) {
1153  HTTPFileStreamer::send404Message( $params['src'], $flags );
1154  $status->fatal( 'backend-fail-stream', $params['src'] );
1155 
1156  return $status;
1157  }
1158 
1159  // If "headers" is set, we only want to send them if the file is there.
1160  // Do not bother checking if the file exists if headers are not set though.
1161  if ( $params['headers'] && !$this->fileExists( $params ) ) {
1162  HTTPFileStreamer::send404Message( $params['src'], $flags );
1163  $status->fatal( 'backend-fail-stream', $params['src'] );
1164 
1165  return $status;
1166  }
1167 
1168  // Send the requested additional headers
1169  foreach ( $params['headers'] as $header ) {
1170  header( $header ); // always send
1171  }
1172 
1173  if ( empty( $params['allowOB'] ) ) {
1174  // Cancel output buffering and gzipping if set
1175  ( $this->obResetFunc )();
1176  }
1177 
1178  $handle = fopen( 'php://output', 'wb' );
1179  [ $rcode, $rdesc, , , $rerr ] = $this->http->run( [
1180  'method' => 'GET',
1181  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1182  'headers' => $this->authTokenHeaders( $auth )
1183  + $this->headersFromParams( $params ) + $params['options'],
1184  'stream' => $handle,
1185  'flags' => [ 'relayResponseHeaders' => empty( $params['headless'] ) ]
1186  ], self::DEFAULT_HTTP_OPTIONS );
1187 
1188  if ( $rcode >= 200 && $rcode <= 299 ) {
1189  // good
1190  } elseif ( $rcode === 404 ) {
1191  $status->fatal( 'backend-fail-stream', $params['src'] );
1192  // Per T43113, nasty things can happen if bad cache entries get
1193  // stuck in cache. It's also possible that this error can come up
1194  // with simple race conditions. Clear out the stat cache to be safe.
1195  $this->clearCache( [ $params['src'] ] );
1196  $this->deleteFileCache( $params['src'] );
1197  } else {
1198  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1199  }
1200 
1201  return $status;
1202  }
1203 
1204  protected function doGetLocalCopyMulti( array $params ) {
1205  $auth = $this->getAuthentication();
1206 
1207  $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging
1208  // Blindly create tmp files and stream to them, catching any exception
1209  // if the file does not exist. Do not waste time doing file stats here.
1210  $reqs = []; // (path => op)
1211 
1212  // Initial dummy values to preserve path order
1213  $tmpFiles = array_fill_keys( $params['srcs'], self::$RES_ERROR );
1214  foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch
1215  [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $path );
1216  if ( $srcRel === null || !$auth ) {
1217  continue; // invalid storage path or auth error
1218  }
1219  // Get source file extension
1221  // Create a new temporary file...
1222  $tmpFile = $this->tmpFileFactory->newTempFSFile( 'localcopy_', $ext );
1223  $handle = $tmpFile ? fopen( $tmpFile->getPath(), 'wb' ) : false;
1224  if ( $handle ) {
1225  $reqs[$path] = [
1226  'method' => 'GET',
1227  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1228  'headers' => $this->authTokenHeaders( $auth )
1229  + $this->headersFromParams( $params ),
1230  'stream' => $handle,
1231  ];
1232  $tmpFiles[$path] = $tmpFile;
1233  }
1234  }
1235 
1236  // Ceph RADOS Gateway is in use (strong consistency) or X-Newest will be used
1237  $latest = ( $this->isRGW || !empty( $params['latest'] ) );
1238 
1239  $opts = [
1240  'maxConnsPerHost' => $params['concurrency'],
1241  ] + self::DEFAULT_HTTP_OPTIONS;
1242  $reqs = $this->http->runMulti( $reqs, $opts );
1243  foreach ( $reqs as $path => $op ) {
1244  [ $rcode, $rdesc, $rhdrs, , $rerr ] = $op['response'];
1245  fclose( $op['stream'] ); // close open handle
1246  if ( $rcode >= 200 && $rcode <= 299 ) {
1248  $tmpFile = $tmpFiles[$path];
1249  // Make sure that the stream finished and fully wrote to disk
1250  $size = $tmpFile->getSize();
1251  if ( $size !== (int)$rhdrs['content-length'] ) {
1252  $tmpFiles[$path] = self::$RES_ERROR;
1253  $rerr = "Got {$size}/{$rhdrs['content-length']} bytes";
1254  $this->onError( null, __METHOD__,
1255  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
1256  }
1257  // Set the file stat process cache in passing
1258  $stat = $this->getStatFromHeaders( $rhdrs );
1259  $stat['latest'] = $latest;
1260  $this->cheapCache->setField( $path, 'stat', $stat );
1261  } elseif ( $rcode === 404 ) {
1262  $tmpFiles[$path] = self::$RES_ABSENT;
1263  $this->cheapCache->setField(
1264  $path,
1265  'stat',
1266  $latest ? self::$ABSENT_LATEST : self::$ABSENT_NORMAL
1267  );
1268  } else {
1269  $tmpFiles[$path] = self::$RES_ERROR;
1270  $this->onError( null, __METHOD__,
1271  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
1272  }
1273  }
1274 
1275  return $tmpFiles;
1276  }
1277 
1278  public function getFileHttpUrl( array $params ) {
1279  if ( $this->swiftTempUrlKey != '' ||
1280  ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' )
1281  ) {
1282  [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $params['src'] );
1283  if ( $srcRel === null ) {
1284  return self::TEMPURL_ERROR; // invalid path
1285  }
1286 
1287  $auth = $this->getAuthentication();
1288  if ( !$auth ) {
1289  return self::TEMPURL_ERROR;
1290  }
1291 
1292  $ttl = $params['ttl'] ?? 86400;
1293  $expires = time() + $ttl;
1294 
1295  if ( $this->swiftTempUrlKey != '' ) {
1296  $url = $this->storageUrl( $auth, $srcCont, $srcRel );
1297  // Swift wants the signature based on the unencoded object name
1298  $contPath = parse_url( $this->storageUrl( $auth, $srcCont ), PHP_URL_PATH );
1299  $signature = hash_hmac( 'sha1',
1300  "GET\n{$expires}\n{$contPath}/{$srcRel}",
1301  $this->swiftTempUrlKey
1302  );
1303 
1304  return "{$url}?temp_url_sig={$signature}&temp_url_expires={$expires}";
1305  } else { // give S3 API URL for rgw
1306  // Path for signature starts with the bucket
1307  $spath = '/' . rawurlencode( $srcCont ) . '/' .
1308  str_replace( '%2F', '/', rawurlencode( $srcRel ) );
1309  // Calculate the hash
1310  $signature = base64_encode( hash_hmac(
1311  'sha1',
1312  "GET\n\n\n{$expires}\n{$spath}",
1313  $this->rgwS3SecretKey,
1314  true // raw
1315  ) );
1316  // See https://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html.
1317  // Note: adding a newline for empty CanonicalizedAmzHeaders does not work.
1318  // Note: S3 API is the rgw default; remove the /swift/ URL bit.
1319  return str_replace( '/swift/v1', '', $this->storageUrl( $auth ) . $spath ) .
1320  '?' .
1321  http_build_query( [
1322  'Signature' => $signature,
1323  'Expires' => $expires,
1324  'AWSAccessKeyId' => $this->rgwS3AccessKey
1325  ] );
1326  }
1327  }
1328 
1329  return self::TEMPURL_ERROR;
1330  }
1331 
1332  protected function directoriesAreVirtual() {
1333  return true;
1334  }
1335 
1344  protected function headersFromParams( array $params ) {
1345  $hdrs = [];
1346  if ( !empty( $params['latest'] ) ) {
1347  $hdrs['x-newest'] = 'true';
1348  }
1349 
1350  return $hdrs;
1351  }
1352 
1353  protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1355  '@phan-var SwiftFileOpHandle[] $fileOpHandles';
1356 
1358  $statuses = [];
1359 
1360  $auth = $this->getAuthentication();
1361  if ( !$auth ) {
1362  foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1363  $statuses[$index] = $this->newStatus( 'backend-fail-connect', $this->name );
1364  }
1365 
1366  return $statuses;
1367  }
1368 
1369  // Split the HTTP requests into stages that can be done concurrently
1370  $httpReqsByStage = []; // map of (stage => index => HTTP request)
1371  foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1372  $reqs = $fileOpHandle->httpOp;
1373  // Convert the 'url' parameter to an actual URL using $auth
1374  foreach ( $reqs as $stage => &$req ) {
1375  [ $container, $relPath ] = $req['url'];
1376  $req['url'] = $this->storageUrl( $auth, $container, $relPath );
1377  $req['headers'] ??= [];
1378  $req['headers'] = $this->authTokenHeaders( $auth ) + $req['headers'];
1379  $httpReqsByStage[$stage][$index] = $req;
1380  }
1381  $statuses[$index] = $this->newStatus();
1382  }
1383 
1384  // Run all requests for the first stage, then the next, and so on
1385  $reqCount = count( $httpReqsByStage );
1386  for ( $stage = 0; $stage < $reqCount; ++$stage ) {
1387  $httpReqs = $this->http->runMulti( $httpReqsByStage[$stage], self::DEFAULT_HTTP_OPTIONS );
1388  foreach ( $httpReqs as $index => $httpReq ) {
1390  $fileOpHandle = $fileOpHandles[$index];
1391  // Run the callback for each request of this operation
1392  $status = $statuses[$index];
1393  ( $fileOpHandle->callback )( $httpReq, $status );
1394  // On failure, abort all remaining requests for this operation. This is used
1395  // in "move" operations to abort the DELETE request if the PUT request fails.
1396  if (
1397  !$status->isOK() ||
1398  $fileOpHandle->state === $fileOpHandle::CONTINUE_NO
1399  ) {
1400  $stages = count( $fileOpHandle->httpOp );
1401  for ( $s = ( $stage + 1 ); $s < $stages; ++$s ) {
1402  unset( $httpReqsByStage[$s][$index] );
1403  }
1404  }
1405  }
1406  }
1407 
1408  return $statuses;
1409  }
1410 
1433  protected function setContainerAccess( $container, array $readUsers, array $writeUsers ) {
1434  $status = $this->newStatus();
1435  $auth = $this->getAuthentication();
1436 
1437  if ( !$auth ) {
1438  $status->fatal( 'backend-fail-connect', $this->name );
1439 
1440  return $status;
1441  }
1442 
1443  [ $rcode, $rdesc, $rhdrs, $rbody, $rerr ] = $this->http->run( [
1444  'method' => 'POST',
1445  'url' => $this->storageUrl( $auth, $container ),
1446  'headers' => $this->authTokenHeaders( $auth ) + [
1447  'x-container-read' => implode( ',', $readUsers ),
1448  'x-container-write' => implode( ',', $writeUsers )
1449  ]
1450  ], self::DEFAULT_HTTP_OPTIONS );
1451 
1452  if ( $rcode != 204 && $rcode !== 202 ) {
1453  $status->fatal( 'backend-fail-internal', $this->name );
1454  $this->logger->error( __METHOD__ . ': unexpected rcode value ({rcode})',
1455  [ 'rcode' => $rcode ] );
1456  }
1457 
1458  return $status;
1459  }
1460 
1469  protected function getContainerStat( $container, $bypassCache = false ) {
1471  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1472 
1473  if ( $bypassCache ) { // purge cache
1474  $this->containerStatCache->clear( $container );
1475  } elseif ( !$this->containerStatCache->hasField( $container, 'stat' ) ) {
1476  $this->primeContainerCache( [ $container ] ); // check persistent cache
1477  }
1478  if ( !$this->containerStatCache->hasField( $container, 'stat' ) ) {
1479  $auth = $this->getAuthentication();
1480  if ( !$auth ) {
1481  return self::$RES_ERROR;
1482  }
1483 
1484  [ $rcode, $rdesc, $rhdrs, , $rerr ] = $this->http->run( [
1485  'method' => 'HEAD',
1486  'url' => $this->storageUrl( $auth, $container ),
1487  'headers' => $this->authTokenHeaders( $auth )
1488  ], self::DEFAULT_HTTP_OPTIONS );
1489 
1490  if ( $rcode === 204 ) {
1491  $stat = [
1492  'count' => $rhdrs['x-container-object-count'],
1493  'bytes' => $rhdrs['x-container-bytes-used']
1494  ];
1495  if ( $bypassCache ) {
1496  return $stat;
1497  } else {
1498  $this->containerStatCache->setField( $container, 'stat', $stat ); // cache it
1499  $this->setContainerCache( $container, $stat ); // update persistent cache
1500  }
1501  } elseif ( $rcode === 404 ) {
1502  return self::$RES_ABSENT;
1503  } else {
1504  $this->onError( null, __METHOD__,
1505  [ 'cont' => $container ], $rerr, $rcode, $rdesc );
1506 
1507  return self::$RES_ERROR;
1508  }
1509  }
1510 
1511  return $this->containerStatCache->getField( $container, 'stat' );
1512  }
1513 
1521  protected function createContainer( $container, array $params ) {
1522  $status = $this->newStatus();
1523 
1524  $auth = $this->getAuthentication();
1525  if ( !$auth ) {
1526  $status->fatal( 'backend-fail-connect', $this->name );
1527 
1528  return $status;
1529  }
1530 
1531  // @see SwiftFileBackend::setContainerAccess()
1532  if ( empty( $params['noAccess'] ) ) {
1533  // public
1534  $readUsers = array_merge( $this->readUsers, [ '.r:*', $this->swiftUser ] );
1535  $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
1536  } else {
1537  // private
1538  $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
1539  $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
1540  }
1541 
1542  [ $rcode, $rdesc, , , $rerr ] = $this->http->run( [
1543  'method' => 'PUT',
1544  'url' => $this->storageUrl( $auth, $container ),
1545  'headers' => $this->authTokenHeaders( $auth ) + [
1546  'x-container-read' => implode( ',', $readUsers ),
1547  'x-container-write' => implode( ',', $writeUsers )
1548  ]
1549  ], self::DEFAULT_HTTP_OPTIONS );
1550 
1551  if ( $rcode === 201 ) { // new
1552  // good
1553  } elseif ( $rcode === 202 ) { // already there
1554  // this shouldn't really happen, but is OK
1555  } else {
1556  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1557  }
1558 
1559  return $status;
1560  }
1561 
1569  protected function deleteContainer( $container, array $params ) {
1570  $status = $this->newStatus();
1571 
1572  $auth = $this->getAuthentication();
1573  if ( !$auth ) {
1574  $status->fatal( 'backend-fail-connect', $this->name );
1575 
1576  return $status;
1577  }
1578 
1579  [ $rcode, $rdesc, , , $rerr ] = $this->http->run( [
1580  'method' => 'DELETE',
1581  'url' => $this->storageUrl( $auth, $container ),
1582  'headers' => $this->authTokenHeaders( $auth )
1583  ], self::DEFAULT_HTTP_OPTIONS );
1584 
1585  if ( $rcode >= 200 && $rcode <= 299 ) { // deleted
1586  $this->containerStatCache->clear( $container ); // purge
1587  } elseif ( $rcode === 404 ) { // not there
1588  // this shouldn't really happen, but is OK
1589  } elseif ( $rcode === 409 ) { // not empty
1590  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); // race?
1591  } else {
1592  // @phan-suppress-previous-line PhanPluginDuplicateIfStatements
1593  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1594  }
1595 
1596  return $status;
1597  }
1598 
1611  private function objectListing(
1612  $fullCont, $type, $limit, $after = null, $prefix = null, $delim = null
1613  ) {
1614  $status = $this->newStatus();
1615 
1616  $auth = $this->getAuthentication();
1617  if ( !$auth ) {
1618  $status->fatal( 'backend-fail-connect', $this->name );
1619 
1620  return $status;
1621  }
1622 
1623  $query = [ 'limit' => $limit ];
1624  if ( $type === 'info' ) {
1625  $query['format'] = 'json';
1626  }
1627  if ( $after !== null ) {
1628  $query['marker'] = $after;
1629  }
1630  if ( $prefix !== null ) {
1631  $query['prefix'] = $prefix;
1632  }
1633  if ( $delim !== null ) {
1634  $query['delimiter'] = $delim;
1635  }
1636 
1637  [ $rcode, $rdesc, , $rbody, $rerr ] = $this->http->run( [
1638  'method' => 'GET',
1639  'url' => $this->storageUrl( $auth, $fullCont ),
1640  'query' => $query,
1641  'headers' => $this->authTokenHeaders( $auth )
1642  ], self::DEFAULT_HTTP_OPTIONS );
1643 
1644  $params = [ 'cont' => $fullCont, 'prefix' => $prefix, 'delim' => $delim ];
1645  if ( $rcode === 200 ) { // good
1646  if ( $type === 'info' ) {
1647  $status->value = FormatJson::decode( trim( $rbody ) );
1648  } else {
1649  $status->value = explode( "\n", trim( $rbody ) );
1650  }
1651  } elseif ( $rcode === 204 ) {
1652  $status->value = []; // empty container
1653  } elseif ( $rcode === 404 ) {
1654  $status->value = []; // no container
1655  } else {
1656  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1657  }
1658 
1659  return $status;
1660  }
1661 
1662  protected function doPrimeContainerCache( array $containerInfo ) {
1663  foreach ( $containerInfo as $container => $info ) {
1664  $this->containerStatCache->setField( $container, 'stat', $info );
1665  }
1666  }
1667 
1668  protected function doGetFileStatMulti( array $params ) {
1669  $stats = [];
1670 
1671  $auth = $this->getAuthentication();
1672 
1673  $reqs = []; // (path => op)
1674  // (a) Check the containers of the paths...
1675  foreach ( $params['srcs'] as $path ) {
1676  [ $srcCont, $srcRel ] = $this->resolveStoragePathReal( $path );
1677  if ( $srcRel === null || !$auth ) {
1678  $stats[$path] = self::$RES_ERROR;
1679  continue; // invalid storage path or auth error
1680  }
1681 
1682  $cstat = $this->getContainerStat( $srcCont );
1683  if ( $cstat === self::$RES_ABSENT ) {
1684  $stats[$path] = self::$RES_ABSENT;
1685  continue; // ok, nothing to do
1686  } elseif ( !is_array( $cstat ) ) {
1687  $stats[$path] = self::$RES_ERROR;
1688  continue;
1689  }
1690 
1691  $reqs[$path] = [
1692  'method' => 'HEAD',
1693  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1694  'headers' => $this->authTokenHeaders( $auth ) + $this->headersFromParams( $params )
1695  ];
1696  }
1697 
1698  // (b) Check the files themselves...
1699  $opts = [
1700  'maxConnsPerHost' => $params['concurrency'],
1701  ] + self::DEFAULT_HTTP_OPTIONS;
1702  $reqs = $this->http->runMulti( $reqs, $opts );
1703  foreach ( $reqs as $path => $op ) {
1704  [ $rcode, $rdesc, $rhdrs, , $rerr ] = $op['response'];
1705  if ( $rcode === 200 || $rcode === 204 ) {
1706  // Update the object if it is missing some headers
1707  if ( !empty( $params['requireSHA1'] ) ) {
1708  $rhdrs = $this->addMissingHashMetadata( $rhdrs, $path );
1709  }
1710  // Load the stat array from the headers
1711  $stat = $this->getStatFromHeaders( $rhdrs );
1712  if ( $this->isRGW ) {
1713  $stat['latest'] = true; // strong consistency
1714  }
1715  } elseif ( $rcode === 404 ) {
1716  $stat = self::$RES_ABSENT;
1717  } else {
1718  $stat = self::$RES_ERROR;
1719  $this->onError( null, __METHOD__, $params, $rerr, $rcode, $rdesc );
1720  }
1721  $stats[$path] = $stat;
1722  }
1723 
1724  return $stats;
1725  }
1726 
1731  protected function getStatFromHeaders( array $rhdrs ) {
1732  // Fetch all of the custom metadata headers
1733  $metadata = $this->getMetadataFromHeaders( $rhdrs );
1734  // Fetch all of the custom raw HTTP headers
1735  $headers = $this->extractMutableContentHeaders( $rhdrs );
1736 
1737  return [
1738  // Convert various random Swift dates to TS_MW
1739  'mtime' => $this->convertSwiftDate( $rhdrs['last-modified'], TS_MW ),
1740  // Empty objects actually return no content-length header in Ceph
1741  'size' => isset( $rhdrs['content-length'] ) ? (int)$rhdrs['content-length'] : 0,
1742  'sha1' => $metadata['sha1base36'] ?? null,
1743  // Note: manifest ETags are not an MD5 of the file
1744  'md5' => ctype_xdigit( $rhdrs['etag'] ) ? $rhdrs['etag'] : null,
1745  'xattr' => [ 'metadata' => $metadata, 'headers' => $headers ]
1746  ];
1747  }
1748 
1752  protected function getAuthentication() {
1753  if ( $this->authErrorTimestamp !== null ) {
1754  if ( ( time() - $this->authErrorTimestamp ) < 60 ) {
1755  return null; // failed last attempt; don't bother
1756  } else { // actually retry this time
1757  $this->authErrorTimestamp = null;
1758  }
1759  }
1760  // Session keys expire after a while, so we renew them periodically
1761  $reAuth = ( ( time() - $this->authSessionTimestamp ) > $this->authTTL );
1762  // Authenticate with proxy and get a session key...
1763  if ( !$this->authCreds || $reAuth ) {
1764  $this->authSessionTimestamp = 0;
1765  $cacheKey = $this->getCredsCacheKey( $this->swiftUser );
1766  $creds = $this->srvCache->get( $cacheKey ); // credentials
1767  // Try to use the credential cache
1768  if ( isset( $creds['auth_token'] ) && isset( $creds['storage_url'] ) ) {
1769  $this->authCreds = $creds;
1770  // Skew the timestamp for worst case to avoid using stale credentials
1771  $this->authSessionTimestamp = time() - (int)ceil( $this->authTTL / 2 );
1772  } else { // cache miss
1773  [ $rcode, , $rhdrs, , ] = $this->http->run( [
1774  'method' => 'GET',
1775  'url' => "{$this->swiftAuthUrl}/v1.0",
1776  'headers' => [
1777  'x-auth-user' => $this->swiftUser,
1778  'x-auth-key' => $this->swiftKey
1779  ]
1780  ], self::DEFAULT_HTTP_OPTIONS );
1781 
1782  if ( $rcode >= 200 && $rcode <= 299 ) { // OK
1783  $this->authCreds = [
1784  'auth_token' => $rhdrs['x-auth-token'],
1785  'storage_url' => $this->swiftStorageUrl ?? $rhdrs['x-storage-url']
1786  ];
1787 
1788  $this->srvCache->set( $cacheKey, $this->authCreds, ceil( $this->authTTL / 2 ) );
1789  $this->authSessionTimestamp = time();
1790  } elseif ( $rcode === 401 ) {
1791  $this->onError( null, __METHOD__, [], "Authentication failed.", $rcode );
1792  $this->authErrorTimestamp = time();
1793 
1794  return null;
1795  } else {
1796  $this->onError( null, __METHOD__, [], "HTTP return code: $rcode", $rcode );
1797  $this->authErrorTimestamp = time();
1798 
1799  return null;
1800  }
1801  }
1802  // Ceph RGW does not use <account> in URLs (OpenStack Swift uses "/v1/<account>")
1803  if ( substr( $this->authCreds['storage_url'], -3 ) === '/v1' ) {
1804  $this->isRGW = true; // take advantage of strong consistency in Ceph
1805  }
1806  }
1807 
1808  return $this->authCreds;
1809  }
1810 
1817  protected function storageUrl( array $creds, $container = null, $object = null ) {
1818  $parts = [ $creds['storage_url'] ];
1819  if ( strlen( $container ) ) {
1820  $parts[] = rawurlencode( $container );
1821  }
1822  if ( strlen( $object ) ) {
1823  $parts[] = str_replace( "%2F", "/", rawurlencode( $object ) );
1824  }
1825 
1826  return implode( '/', $parts );
1827  }
1828 
1833  protected function authTokenHeaders( array $creds ) {
1834  return [ 'x-auth-token' => $creds['auth_token'] ];
1835  }
1836 
1843  private function getCredsCacheKey( $username ) {
1844  return 'swiftcredentials:' . md5( $username . ':' . $this->swiftAuthUrl );
1845  }
1846 
1858  public function onError( $status, $func, array $params, $err = '', $code = 0, $desc = '' ) {
1859  if ( $status instanceof StatusValue ) {
1860  $status->fatal( 'backend-fail-internal', $this->name );
1861  }
1862  if ( $code == 401 ) { // possibly a stale token
1863  $this->srvCache->delete( $this->getCredsCacheKey( $this->swiftUser ) );
1864  }
1865  $msg = "HTTP {code} ({desc}) in '{func}' (given '{req_params}')";
1866  $msgParams = [
1867  'code' => $code,
1868  'desc' => $desc,
1869  'func' => $func,
1870  'req_params' => FormatJson::encode( $params ),
1871  ];
1872  if ( $err ) {
1873  $msg .= ': {err}';
1874  $msgParams['err'] = $err;
1875  }
1876  $this->logger->error( $msg, $msgParams );
1877  }
1878 }
A BagOStuff object with no objects in it.
File backend exception for checked exceptions (e.g.
Base class for all backends using particular storage medium.
setContainerCache( $container, array $val)
Set the cached info for a container.
executeOpHandlesInternal(array $fileOpHandles)
Execute a list of FileBackendStoreOpHandle handles in parallel.
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
deleteFileCache( $path)
Delete the cached stat info for a file path.
getContentType( $storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
static false $RES_ABSENT
Idiom for "no result due to missing file" (since 1.34)
static null $RES_ERROR
Idiom for "no result due to I/O errors" (since 1.34)
fileExists(array $params)
Check if a file exists at a storage path in the backend.
getLocalCopy(array $params)
Get a local copy on disk of the file at a storage path in the backend.
string $name
Unique backend name.
callable $obResetFunc
LoggerInterface $logger
static extensionFromPath( $path, $case='lowercase')
Get the final extension from a storage or FS path.
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
scopedProfileSection( $section)
newStatus(... $args)
Yields the result of the status wrapper callback on either:
static encode( $value, $pretty=false, $escaping=0)
Returns the JSON representation of a value.
Definition: FormatJson.php:96
static decode( $value, $assoc=false)
Decodes a JSON string.
Definition: FormatJson.php:146
static send404Message( $fname, $flags=0)
Send out a standard 404 message for a file.
const LOCK_UW
Definition: LockManager.php:69
Library for creating and parsing MW-style timestamps.
Definition: MWTimestamp.php:39
Handles a simple LRU key/value map with a maximum number of entries.
Definition: MapCacheLRU.php:36
Class to handle multiple HTTP requests.
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Definition: StatusValue.php:46
Iterator for listing directories.
Iterator for listing regular files.
Class for an OpenStack Swift (or Ceph RGW) based file backend.
string $swiftUser
Swift user (account:user) to authenticate as.
string $swiftAuthUrl
Authentication base URL (without version)
string $swiftTempUrlKey
Shared secret value for making temp URLs.
MapCacheLRU $containerStatCache
Container stat cache.
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path in the backend.
getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
doPublishInternal( $fullCont, $dir, array $params)
doCreateInternal(array $params)
doGetFileStatMulti(array $params)
Get file stat information (concurrently if possible) for several files.
doGetFileSha1base36(array $params)
int null $authErrorTimestamp
UNIX timestamp.
array $writeUsers
Additional users (account:user) with write permissions on public containers.
__construct(array $config)
MultiHttpClient $http
doGetFileXAttributes(array $params)
onError( $status, $func, array $params, $err='', $code=0, $desc='')
Log an unexpected exception for this backend.
authTokenHeaders(array $creds)
getStatFromHeaders(array $rhdrs)
string $swiftStorageUrl
Override of storage base URL.
createContainer( $container, array $params)
Create a Swift container.
doCopyInternal(array $params)
getDirectoryListInternal( $fullCont, $dir, array $params)
string $rgwS3AccessKey
S3 access key (RADOS Gateway)
setContainerAccess( $container, array $readUsers, array $writeUsers)
Set read/write permissions for a Swift container.
getFileHttpUrl(array $params)
array $secureWriteUsers
Additional users (account:user) with write permissions on private containers.
extractMetadataHeaders(array $headers)
int $authTTL
TTL in seconds.
headersFromParams(array $params)
Get headers to send to Swift when reading a file based on a FileBackend params array,...
bool $isRGW
Whether the server is an Ceph RGW.
doStoreInternal(array $params)
int $authSessionTimestamp
UNIX timestamp.
loadListingStatInternal( $path, array $val)
Do not call this function outside of SwiftFileBackendFileList.
doPrepareInternal( $fullCont, $dir, array $params)
FileBackendStore::doPrepare() Stability: stableto override StatusValue Good status without value for ...
setLogger(LoggerInterface $logger)
doSecureInternal( $fullCont, $dir, array $params)
getFileListInternal( $fullCont, $dir, array $params)
getMetadataFromHeaders(array $headers)
doMoveInternal(array $params)
addMissingHashMetadata(array $objHdrs, $path)
Fill in any missing object metadata and save it to Swift.
getFeatures()
Get the a bitfield of extra features supported by the backend medium.
deleteContainer( $container, array $params)
Delete a Swift container.
doGetFileStat(array $params)
doGetLocalCopyMulti(array $params)
string $rgwS3SecretKey
S3 authentication key (RADOS Gateway)
doGetFileContentsMulti(array $params)
storageUrl(array $creds, $container=null, $object=null)
convertSwiftDate( $ts, $format=TS_MW)
Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT"/"2013-05-11T07:37:27.678360Z".
doStreamFile(array $params)
doPrimeContainerCache(array $containerInfo)
Fill the backend-specific process cache given an array of resolved container names and their correspo...
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
array $readUsers
Additional users (account:user) with read permissions on public containers.
array $secureReadUsers
Additional users (account:user) with read permissions on private containers.
doCleanInternal( $fullCont, $dir, array $params)
getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
string $swiftKey
Secret key for user.
doDirectoryExists( $fullCont, $dir, array $params)
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
doExecuteOpHandlesInternal(array $fileOpHandles)
doDeleteInternal(array $params)
doDescribeInternal(array $params)
extractMutableContentHeaders(array $headers)
Filter/normalize a header map to only include mutable "content-"/"x-content-" headers.
getContainerStat( $container, $bypassCache=false)
Get a Swift container stat array, possibly from process cache.
Multi-datacenter aware caching interface.
foreach( $mmfl['setupFiles'] as $fileName) if( $queue) if(empty( $mmfl['quiet'])) $s
$content
Definition: router.php:76
if(!is_readable( $file)) $ext
Definition: router.php:48
$header