MediaWiki  master
SwiftFileBackend.php
Go to the documentation of this file.
1 <?php
25 use Psr\Log\LoggerInterface;
26 use Wikimedia\AtEase\AtEase;
27 
38  private const DEFAULT_HTTP_OPTIONS = [ 'httpVersion' => 'v1.1' ];
39 
41  protected $http;
43  protected $authTTL;
45  protected $swiftAuthUrl;
47  protected $swiftStorageUrl;
49  protected $swiftUser;
51  protected $swiftKey;
53  protected $swiftTempUrlKey;
55  protected $rgwS3AccessKey;
57  protected $rgwS3SecretKey;
59  protected $readUsers;
61  protected $writeUsers;
63  protected $secureReadUsers;
65  protected $secureWriteUsers;
66 
68  protected $srvCache;
69 
72 
74  protected $authCreds;
76  protected $authSessionTimestamp = 0;
78  protected $authErrorTimestamp = null;
79 
81  protected $isRGW = false;
82 
121  public function __construct( array $config ) {
122  parent::__construct( $config );
123  // Required settings
124  $this->swiftAuthUrl = $config['swiftAuthUrl'];
125  $this->swiftUser = $config['swiftUser'];
126  $this->swiftKey = $config['swiftKey'];
127  // Optional settings
128  $this->authTTL = $config['swiftAuthTTL'] ?? 15 * 60; // some sensible number
129  $this->swiftTempUrlKey = $config['swiftTempUrlKey'] ?? '';
130  $this->swiftStorageUrl = $config['swiftStorageUrl'] ?? null;
131  $this->shardViaHashLevels = $config['shardViaHashLevels'] ?? '';
132  $this->rgwS3AccessKey = $config['rgwS3AccessKey'] ?? '';
133  $this->rgwS3SecretKey = $config['rgwS3SecretKey'] ?? '';
134 
135  // HTTP helper client
136  $httpOptions = [];
137  foreach ( [ 'connTimeout', 'reqTimeout' ] as $optionName ) {
138  if ( isset( $config[$optionName] ) ) {
139  $httpOptions[$optionName] = $config[$optionName];
140  }
141  }
142  $this->http = new MultiHttpClient( $httpOptions );
143  $this->http->setLogger( $this->logger );
144 
145  // Cache container information to mask latency
146  if ( isset( $config['wanCache'] ) && $config['wanCache'] instanceof WANObjectCache ) {
147  $this->memCache = $config['wanCache'];
148  }
149  // Process cache for container info
150  $this->containerStatCache = new MapCacheLRU( 300 );
151  // Cache auth token information to avoid RTTs
152  if ( !empty( $config['cacheAuthInfo'] ) && isset( $config['srvCache'] ) ) {
153  $this->srvCache = $config['srvCache'];
154  } else {
155  $this->srvCache = new EmptyBagOStuff();
156  }
157  $this->readUsers = $config['readUsers'] ?? [];
158  $this->writeUsers = $config['writeUsers'] ?? [];
159  $this->secureReadUsers = $config['secureReadUsers'] ?? [];
160  $this->secureWriteUsers = $config['secureWriteUsers'] ?? [];
161  }
162 
163  public function setLogger( LoggerInterface $logger ) {
164  parent::setLogger( $logger );
165  $this->http->setLogger( $logger );
166  }
167 
168  public function getFeatures() {
169  return (
170  self::ATTR_UNICODE_PATHS |
171  self::ATTR_HEADERS |
172  self::ATTR_METADATA
173  );
174  }
175 
176  protected function resolveContainerPath( $container, $relStoragePath ) {
177  if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) {
178  return null; // not UTF-8, makes it hard to use CF and the swift HTTP API
179  } elseif ( strlen( rawurlencode( $relStoragePath ) ) > 1024 ) {
180  return null; // too long for Swift
181  }
182 
183  return $relStoragePath;
184  }
185 
186  public function isPathUsableInternal( $storagePath ) {
187  list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
188  if ( $rel === null ) {
189  return false; // invalid
190  }
191 
192  return is_array( $this->getContainerStat( $container ) );
193  }
194 
204  protected function extractMutableContentHeaders( array $headers ) {
205  $contentHeaders = [];
206  // Normalize casing, and strip out illegal headers
207  foreach ( $headers as $name => $value ) {
208  $name = strtolower( $name );
209  if ( $name === 'x-delete-at' && is_numeric( $value ) ) {
210  // Expects a Unix Epoch date
211  $contentHeaders[$name] = $value;
212  } elseif ( $name === 'x-delete-after' && is_numeric( $value ) ) {
213  // Expects number of minutes time to live.
214  $contentHeaders[$name] = $value;
215  } elseif ( preg_match( '/^(x-)?content-(?!length$)/', $name ) ) {
216  // Only allow content-* and x-content-* headers (but not content-length)
217  $contentHeaders[$name] = $value;
218  } elseif ( $name === 'content-type' && strlen( $value ) ) {
219  // This header can be set to a value but not unset
220  $contentHeaders[$name] = $value;
221  }
222  }
223  // By default, Swift has annoyingly low maximum header value limits
224  if ( isset( $contentHeaders['content-disposition'] ) ) {
225  $maxLength = 255;
226  // @note: assume FileBackend::makeContentDisposition() already used
227  $offset = $maxLength - strlen( $contentHeaders['content-disposition'] );
228  if ( $offset < 0 ) {
229  $pos = strrpos( $contentHeaders['content-disposition'], ';', $offset );
230  $contentHeaders['content-disposition'] = $pos === false
231  ? ''
232  : trim( substr( $contentHeaders['content-disposition'], 0, $pos ) );
233  }
234  }
235 
236  return $contentHeaders;
237  }
238 
244  protected function extractMetadataHeaders( array $headers ) {
245  $metadataHeaders = [];
246  foreach ( $headers as $name => $value ) {
247  $name = strtolower( $name );
248  if ( strpos( $name, 'x-object-meta-' ) === 0 ) {
249  $metadataHeaders[$name] = $value;
250  }
251  }
252 
253  return $metadataHeaders;
254  }
255 
261  protected function getMetadataFromHeaders( array $headers ) {
262  $prefixLen = strlen( 'x-object-meta-' );
263 
264  $metadata = [];
265  foreach ( $this->extractMetadataHeaders( $headers ) as $name => $value ) {
266  $metadata[substr( $name, $prefixLen )] = $value;
267  }
268 
269  return $metadata;
270  }
271 
272  protected function doCreateInternal( array $params ) {
273  $status = $this->newStatus();
274 
275  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
276  if ( $dstRel === null ) {
277  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
278 
279  return $status;
280  }
281 
282  // Headers that are not strictly a function of the file content
283  $mutableHeaders = $this->extractMutableContentHeaders( $params['headers'] ?? [] );
284  // Make sure that the "content-type" header is set to something sensible
285  $mutableHeaders['content-type'] = $mutableHeaders['content-type']
286  ?? $this->getContentType( $params['dst'], $params['content'], null );
287 
288  $reqs = [ [
289  'method' => 'PUT',
290  'url' => [ $dstCont, $dstRel ],
291  'headers' => array_merge(
292  $mutableHeaders,
293  [
294  'etag' => md5( $params['content'] ),
295  'content-length' => strlen( $params['content'] ),
296  'x-object-meta-sha1base36' =>
297  Wikimedia\base_convert( sha1( $params['content'] ), 16, 36, 31 )
298  ]
299  ),
300  'body' => $params['content']
301  ] ];
302 
303  $method = __METHOD__;
304  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
305  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
306  if ( $rcode === 201 || $rcode === 202 ) {
307  // good
308  } elseif ( $rcode === 412 ) {
309  $status->fatal( 'backend-fail-contenttype', $params['dst'] );
310  } else {
311  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
312  }
313 
314  return SwiftFileOpHandle::CONTINUE_IF_OK;
315  };
316 
317  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
318  if ( !empty( $params['async'] ) ) { // deferred
319  $status->value = $opHandle;
320  } else { // actually write the object in Swift
321  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
322  }
323 
324  return $status;
325  }
326 
327  protected function doStoreInternal( array $params ) {
328  $status = $this->newStatus();
329 
330  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
331  if ( $dstRel === null ) {
332  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
333 
334  return $status;
335  }
336 
337  // Open a handle to the source file so that it can be streamed. The size and hash
338  // will be computed using the handle. In the off chance that the source file changes
339  // during this operation, the PUT will fail due to an ETag mismatch and be aborted.
340  AtEase::suppressWarnings();
341  $srcHandle = fopen( $params['src'], 'rb' );
342  AtEase::restoreWarnings();
343  if ( $srcHandle === false ) { // source doesn't exist?
344  $status->fatal( 'backend-fail-notexists', $params['src'] );
345 
346  return $status;
347  }
348 
349  // Compute the MD5 and SHA-1 hashes in one pass
350  $srcSize = fstat( $srcHandle )['size'];
351  $md5Context = hash_init( 'md5' );
352  $sha1Context = hash_init( 'sha1' );
353  $hashDigestSize = 0;
354  while ( !feof( $srcHandle ) ) {
355  $buffer = (string)fread( $srcHandle, 131072 ); // 128 KiB
356  hash_update( $md5Context, $buffer );
357  hash_update( $sha1Context, $buffer );
358  $hashDigestSize += strlen( $buffer );
359  }
360  // Reset the handle back to the beginning so that it can be streamed
361  rewind( $srcHandle );
362 
363  if ( $hashDigestSize !== $srcSize ) {
364  $status->fatal( 'backend-fail-hash', $params['src'] );
365 
366  return $status;
367  }
368 
369  // Headers that are not strictly a function of the file content
370  $mutableHeaders = $this->extractMutableContentHeaders( $params['headers'] ?? [] );
371  // Make sure that the "content-type" header is set to something sensible
372  $mutableHeaders['content-type'] = $mutableHeaders['content-type']
373  ?? $this->getContentType( $params['dst'], null, $params['src'] );
374 
375  $reqs = [ [
376  'method' => 'PUT',
377  'url' => [ $dstCont, $dstRel ],
378  'headers' => array_merge(
379  $mutableHeaders,
380  [
381  'content-length' => $srcSize,
382  'etag' => hash_final( $md5Context ),
383  'x-object-meta-sha1base36' =>
384  Wikimedia\base_convert( hash_final( $sha1Context ), 16, 36, 31 )
385  ]
386  ),
387  'body' => $srcHandle // resource
388  ] ];
389 
390  $method = __METHOD__;
391  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
392  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
393  if ( $rcode === 201 || $rcode === 202 ) {
394  // good
395  } elseif ( $rcode === 412 ) {
396  $status->fatal( 'backend-fail-contenttype', $params['dst'] );
397  } else {
398  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
399  }
400 
401  return SwiftFileOpHandle::CONTINUE_IF_OK;
402  };
403 
404  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
405  $opHandle->resourcesToClose[] = $srcHandle;
406 
407  if ( !empty( $params['async'] ) ) { // deferred
408  $status->value = $opHandle;
409  } else { // actually write the object in Swift
410  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
411  }
412 
413  return $status;
414  }
415 
416  protected function doCopyInternal( array $params ) {
417  $status = $this->newStatus();
418 
419  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
420  if ( $srcRel === null ) {
421  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
422 
423  return $status;
424  }
425 
426  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
427  if ( $dstRel === null ) {
428  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
429 
430  return $status;
431  }
432 
433  $reqs = [ [
434  'method' => 'PUT',
435  'url' => [ $dstCont, $dstRel ],
436  'headers' => array_merge(
437  $this->extractMutableContentHeaders( $params['headers'] ?? [] ),
438  [
439  'x-copy-from' => '/' . rawurlencode( $srcCont ) . '/' .
440  str_replace( "%2F", "/", rawurlencode( $srcRel ) )
441  ]
442  )
443  ] ];
444 
445  $method = __METHOD__;
446  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
447  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
448  if ( $rcode === 201 ) {
449  // good
450  } elseif ( $rcode === 404 ) {
451  if ( empty( $params['ignoreMissingSource'] ) ) {
452  $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
453  }
454  } else {
455  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
456  }
457 
458  return SwiftFileOpHandle::CONTINUE_IF_OK;
459  };
460 
461  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
462  if ( !empty( $params['async'] ) ) { // deferred
463  $status->value = $opHandle;
464  } else { // actually write the object in Swift
465  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
466  }
467 
468  return $status;
469  }
470 
471  protected function doMoveInternal( array $params ) {
472  $status = $this->newStatus();
473 
474  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
475  if ( $srcRel === null ) {
476  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
477 
478  return $status;
479  }
480 
481  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
482  if ( $dstRel === null ) {
483  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
484 
485  return $status;
486  }
487 
488  $reqs = [ [
489  'method' => 'PUT',
490  'url' => [ $dstCont, $dstRel ],
491  'headers' => array_merge(
492  $this->extractMutableContentHeaders( $params['headers'] ?? [] ),
493  [
494  'x-copy-from' => '/' . rawurlencode( $srcCont ) . '/' .
495  str_replace( "%2F", "/", rawurlencode( $srcRel ) )
496  ]
497  )
498  ] ];
499  if ( "{$srcCont}/{$srcRel}" !== "{$dstCont}/{$dstRel}" ) {
500  $reqs[] = [
501  'method' => 'DELETE',
502  'url' => [ $srcCont, $srcRel ],
503  'headers' => []
504  ];
505  }
506 
507  $method = __METHOD__;
508  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
509  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
510  if ( $request['method'] === 'PUT' && $rcode === 201 ) {
511  // good
512  } elseif ( $request['method'] === 'DELETE' && $rcode === 204 ) {
513  // good
514  } elseif ( $rcode === 404 ) {
515  if ( empty( $params['ignoreMissingSource'] ) ) {
516  $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
517  } else {
518  // Leave Status as OK but skip the DELETE request
519  return SwiftFileOpHandle::CONTINUE_NO;
520  }
521  } else {
522  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
523  }
524 
525  return SwiftFileOpHandle::CONTINUE_IF_OK;
526  };
527 
528  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
529  if ( !empty( $params['async'] ) ) { // deferred
530  $status->value = $opHandle;
531  } else { // actually move the object in Swift
532  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
533  }
534 
535  return $status;
536  }
537 
538  protected function doDeleteInternal( array $params ) {
539  $status = $this->newStatus();
540 
541  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
542  if ( $srcRel === null ) {
543  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
544 
545  return $status;
546  }
547 
548  $reqs = [ [
549  'method' => 'DELETE',
550  'url' => [ $srcCont, $srcRel ],
551  'headers' => []
552  ] ];
553 
554  $method = __METHOD__;
555  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
556  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
557  if ( $rcode === 204 ) {
558  // good
559  } elseif ( $rcode === 404 ) {
560  if ( empty( $params['ignoreMissingSource'] ) ) {
561  $status->fatal( 'backend-fail-delete', $params['src'] );
562  }
563  } else {
564  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
565  }
566 
567  return SwiftFileOpHandle::CONTINUE_IF_OK;
568  };
569 
570  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
571  if ( !empty( $params['async'] ) ) { // deferred
572  $status->value = $opHandle;
573  } else { // actually delete the object in Swift
574  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
575  }
576 
577  return $status;
578  }
579 
580  protected function doDescribeInternal( array $params ) {
581  $status = $this->newStatus();
582 
583  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
584  if ( $srcRel === null ) {
585  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
586 
587  return $status;
588  }
589 
590  // Fetch the old object headers/metadata...this should be in stat cache by now
591  $stat = $this->getFileStat( [ 'src' => $params['src'], 'latest' => 1 ] );
592  if ( $stat && !isset( $stat['xattr'] ) ) { // older cache entry
593  $stat = $this->doGetFileStat( [ 'src' => $params['src'], 'latest' => 1 ] );
594  }
595  if ( !$stat ) {
596  $status->fatal( 'backend-fail-describe', $params['src'] );
597 
598  return $status;
599  }
600 
601  // Swift object POST clears any prior headers, so merge the new and old headers here.
602  // Also, during, POST, libcurl adds "Content-Type: application/x-www-form-urlencoded"
603  // if "Content-Type" is not set, which would clobber the header value for the object.
604  $oldMetadataHeaders = [];
605  foreach ( $stat['xattr']['metadata'] as $name => $value ) {
606  $oldMetadataHeaders["x-object-meta-$name"] = $value;
607  }
608  $newContentHeaders = $this->extractMutableContentHeaders( $params['headers'] ?? [] );
609  $oldContentHeaders = $stat['xattr']['headers'];
610 
611  $reqs = [ [
612  'method' => 'POST',
613  'url' => [ $srcCont, $srcRel ],
614  'headers' => $oldMetadataHeaders + $newContentHeaders + $oldContentHeaders
615  ] ];
616 
617  $method = __METHOD__;
618  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
619  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
620  if ( $rcode === 202 ) {
621  // good
622  } elseif ( $rcode === 404 ) {
623  $status->fatal( 'backend-fail-describe', $params['src'] );
624  } else {
625  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
626  }
627  };
628 
629  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
630  if ( !empty( $params['async'] ) ) { // deferred
631  $status->value = $opHandle;
632  } else { // actually change the object in Swift
633  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
634  }
635 
636  return $status;
637  }
638 
642  protected function doPrepareInternal( $fullCont, $dir, array $params ) {
643  $status = $this->newStatus();
644 
645  // (a) Check if container already exists
646  $stat = $this->getContainerStat( $fullCont );
647  if ( is_array( $stat ) ) {
648  return $status; // already there
649  } elseif ( $stat === self::$RES_ERROR ) {
650  $status->fatal( 'backend-fail-internal', $this->name );
651  $this->logger->error( __METHOD__ . ': cannot get container stat' );
652 
653  return $status;
654  }
655 
656  // (b) Create container as needed with proper ACLs
657  if ( $stat === false ) {
658  $params['op'] = 'prepare';
659  $status->merge( $this->createContainer( $fullCont, $params ) );
660  }
661 
662  return $status;
663  }
664 
665  protected function doSecureInternal( $fullCont, $dir, array $params ) {
666  $status = $this->newStatus();
667  if ( empty( $params['noAccess'] ) ) {
668  return $status; // nothing to do
669  }
670 
671  $stat = $this->getContainerStat( $fullCont );
672  if ( is_array( $stat ) ) {
673  $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
674  $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
675  // Make container private to end-users...
676  $status->merge( $this->setContainerAccess(
677  $fullCont,
678  $readUsers,
680  ) );
681  } elseif ( $stat === false ) {
682  $status->fatal( 'backend-fail-usable', $params['dir'] );
683  } else {
684  $status->fatal( 'backend-fail-internal', $this->name );
685  $this->logger->error( __METHOD__ . ': cannot get container stat' );
686  }
687 
688  return $status;
689  }
690 
691  protected function doPublishInternal( $fullCont, $dir, array $params ) {
692  $status = $this->newStatus();
693 
694  $stat = $this->getContainerStat( $fullCont );
695  if ( is_array( $stat ) ) {
696  $readUsers = array_merge( $this->readUsers, [ $this->swiftUser, '.r:*' ] );
697  $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
698 
699  // Make container public to end-users...
700  $status->merge( $this->setContainerAccess(
701  $fullCont,
702  $readUsers,
704  ) );
705  } elseif ( $stat === false ) {
706  $status->fatal( 'backend-fail-usable', $params['dir'] );
707  } else {
708  $status->fatal( 'backend-fail-internal', $this->name );
709  $this->logger->error( __METHOD__ . ': cannot get container stat' );
710  }
711 
712  return $status;
713  }
714 
715  protected function doCleanInternal( $fullCont, $dir, array $params ) {
716  $status = $this->newStatus();
717 
718  // Only containers themselves can be removed, all else is virtual
719  if ( $dir != '' ) {
720  return $status; // nothing to do
721  }
722 
723  // (a) Check the container
724  $stat = $this->getContainerStat( $fullCont, true );
725  if ( $stat === false ) {
726  return $status; // ok, nothing to do
727  } elseif ( !is_array( $stat ) ) {
728  $status->fatal( 'backend-fail-internal', $this->name );
729  $this->logger->error( __METHOD__ . ': cannot get container stat' );
730 
731  return $status;
732  }
733 
734  // (b) Delete the container if empty
735  if ( $stat['count'] == 0 ) {
736  $params['op'] = 'clean';
737  $status->merge( $this->deleteContainer( $fullCont, $params ) );
738  }
739 
740  return $status;
741  }
742 
743  protected function doGetFileStat( array $params ) {
744  $params = [ 'srcs' => [ $params['src'] ], 'concurrency' => 1 ] + $params;
745  unset( $params['src'] );
746  $stats = $this->doGetFileStatMulti( $params );
747 
748  return reset( $stats );
749  }
750 
761  protected function convertSwiftDate( $ts, $format = TS_MW ) {
762  try {
763  $timestamp = new MWTimestamp( $ts );
764 
765  return $timestamp->getTimestamp( $format );
766  } catch ( Exception $e ) {
767  throw new FileBackendError( $e->getMessage() );
768  }
769  }
770 
778  protected function addMissingHashMetadata( array $objHdrs, $path ) {
779  if ( isset( $objHdrs['x-object-meta-sha1base36'] ) ) {
780  return $objHdrs; // nothing to do
781  }
782 
784  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
785  $this->logger->error( __METHOD__ . ": {path} was not stored with SHA-1 metadata.",
786  [ 'path' => $path ] );
787 
788  $objHdrs['x-object-meta-sha1base36'] = false;
789 
790  $auth = $this->getAuthentication();
791  if ( !$auth ) {
792  return $objHdrs; // failed
793  }
794 
795  // Find prior custom HTTP headers
796  $postHeaders = $this->extractMutableContentHeaders( $objHdrs );
797  // Find prior metadata headers
798  $postHeaders += $this->extractMetadataHeaders( $objHdrs );
799 
800  $status = $this->newStatus();
802  $scopeLockS = $this->getScopedFileLocks( [ $path ], LockManager::LOCK_UW, $status );
803  if ( $status->isOK() ) {
804  $tmpFile = $this->getLocalCopy( [ 'src' => $path, 'latest' => 1 ] );
805  if ( $tmpFile ) {
806  $hash = $tmpFile->getSha1Base36();
807  if ( $hash !== false ) {
808  $objHdrs['x-object-meta-sha1base36'] = $hash;
809  // Merge new SHA1 header into the old ones
810  $postHeaders['x-object-meta-sha1base36'] = $hash;
811  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
812  list( $rcode ) = $this->http->run( [
813  'method' => 'POST',
814  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
815  'headers' => $this->authTokenHeaders( $auth ) + $postHeaders
816  ], self::DEFAULT_HTTP_OPTIONS );
817  if ( $rcode >= 200 && $rcode <= 299 ) {
818  $this->deleteFileCache( $path );
819 
820  return $objHdrs; // success
821  }
822  }
823  }
824  }
825 
826  $this->logger->error( __METHOD__ . ': unable to set SHA-1 metadata for {path}',
827  [ 'path' => $path ] );
828 
829  return $objHdrs; // failed
830  }
831 
832  protected function doGetFileContentsMulti( array $params ) {
833  $auth = $this->getAuthentication();
834 
835  $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging
836  // Blindly create tmp files and stream to them, catching any exception
837  // if the file does not exist. Do not waste time doing file stats here.
838  $reqs = []; // (path => op)
839 
840  // Initial dummy values to preserve path order
841  $contents = array_fill_keys( $params['srcs'], self::$RES_ERROR );
842  foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch
843  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
844  if ( $srcRel === null || !$auth ) {
845  continue; // invalid storage path or auth error
846  }
847  // Create a new temporary memory file...
848  $handle = fopen( 'php://temp', 'wb' );
849  if ( $handle ) {
850  $reqs[$path] = [
851  'method' => 'GET',
852  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
853  'headers' => $this->authTokenHeaders( $auth )
854  + $this->headersFromParams( $params ),
855  'stream' => $handle,
856  ];
857  }
858  }
859 
860  $opts = [
861  'maxConnsPerHost' => $params['concurrency'],
863  $reqs = $this->http->runMulti( $reqs, $opts );
864  foreach ( $reqs as $path => $op ) {
865  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response'];
866  if ( $rcode >= 200 && $rcode <= 299 ) {
867  rewind( $op['stream'] ); // start from the beginning
868  $content = (string)stream_get_contents( $op['stream'] );
869  $size = strlen( $content );
870  // Make sure that stream finished
871  if ( $size === (int)$rhdrs['content-length'] ) {
872  $contents[$path] = $content;
873  } else {
874  $contents[$path] = self::$RES_ERROR;
875  $rerr = "Got {$size}/{$rhdrs['content-length']} bytes";
876  $this->onError( null, __METHOD__,
877  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
878  }
879  } elseif ( $rcode === 404 ) {
880  $contents[$path] = self::$RES_ABSENT;
881  } else {
882  $contents[$path] = self::$RES_ERROR;
883  $this->onError( null, __METHOD__,
884  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
885  }
886  fclose( $op['stream'] ); // close open handle
887  }
888 
889  return $contents;
890  }
891 
892  protected function doDirectoryExists( $fullCont, $dir, array $params ) {
893  $prefix = ( $dir == '' ) ? null : "{$dir}/";
894  $status = $this->objectListing( $fullCont, 'names', 1, null, $prefix );
895  if ( $status->isOK() ) {
896  return ( count( $status->value ) ) > 0;
897  }
898 
899  return self::$RES_ERROR;
900  }
901 
909  public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
910  return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
911  }
912 
920  public function getFileListInternal( $fullCont, $dir, array $params ) {
921  return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
922  }
923 
935  public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
936  $dirs = [];
937  if ( $after === INF ) {
938  return $dirs; // nothing more
939  }
940 
942  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
943 
944  $prefix = ( $dir == '' ) ? null : "{$dir}/";
945  // Non-recursive: only list dirs right under $dir
946  if ( !empty( $params['topOnly'] ) ) {
947  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
948  if ( !$status->isOK() ) {
949  throw new FileBackendError( "Iterator page I/O error." );
950  }
951  $objects = $status->value;
952  // @phan-suppress-next-line PhanTypeSuspiciousNonTraversableForeach
953  foreach ( $objects as $object ) { // files and directories
954  if ( substr( $object, -1 ) === '/' ) {
955  $dirs[] = $object; // directories end in '/'
956  }
957  }
958  } else {
959  // Recursive: list all dirs under $dir and its subdirs
960  $getParentDir = static function ( $path ) {
961  return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
962  };
963 
964  // Get directory from last item of prior page
965  $lastDir = $getParentDir( $after ); // must be first page
966  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
967 
968  if ( !$status->isOK() ) {
969  throw new FileBackendError( "Iterator page I/O error." );
970  }
971 
972  $objects = $status->value;
973 
974  // @phan-suppress-next-line PhanTypeSuspiciousNonTraversableForeach
975  foreach ( $objects as $object ) { // files
976  $objectDir = $getParentDir( $object ); // directory of object
977 
978  if ( $objectDir !== false && $objectDir !== $dir ) {
979  // Swift stores paths in UTF-8, using binary sorting.
980  // See function "create_container_table" in common/db.py.
981  // If a directory is not "greater" than the last one,
982  // then it was already listed by the calling iterator.
983  if ( strcmp( $objectDir, $lastDir ) > 0 ) {
984  $pDir = $objectDir;
985  do { // add dir and all its parent dirs
986  $dirs[] = "{$pDir}/";
987  $pDir = $getParentDir( $pDir );
988  } while ( $pDir !== false
989  && strcmp( $pDir, $lastDir ) > 0 // not done already
990  && strlen( $pDir ) > strlen( $dir ) // within $dir
991  );
992  }
993  $lastDir = $objectDir;
994  }
995  }
996  }
997  // Page on the unfiltered directory listing (what is returned may be filtered)
998  if ( count( $objects ) < $limit ) {
999  $after = INF; // avoid a second RTT
1000  } else {
1001  $after = end( $objects ); // update last item
1002  }
1003 
1004  return $dirs;
1005  }
1006 
1018  public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
1019  $files = []; // list of (path, stat array or null) entries
1020  if ( $after === INF ) {
1021  return $files; // nothing more
1022  }
1023 
1025  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1026 
1027  $prefix = ( $dir == '' ) ? null : "{$dir}/";
1028  // $objects will contain a list of unfiltered names or stdClass items
1029  // Non-recursive: only list files right under $dir
1030  if ( !empty( $params['topOnly'] ) ) {
1031  if ( !empty( $params['adviseStat'] ) ) {
1032  $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix, '/' );
1033  } else {
1034  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
1035  }
1036  } else {
1037  // Recursive: list all files under $dir and its subdirs
1038  if ( !empty( $params['adviseStat'] ) ) {
1039  $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix );
1040  } else {
1041  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
1042  }
1043  }
1044 
1045  // Reformat this list into a list of (name, stat array or null) entries
1046  if ( !$status->isOK() ) {
1047  throw new FileBackendError( "Iterator page I/O error." );
1048  }
1049 
1050  $objects = $status->value;
1051  $files = $this->buildFileObjectListing( $objects );
1052 
1053  // Page on the unfiltered object listing (what is returned may be filtered)
1054  if ( count( $objects ) < $limit ) {
1055  $after = INF; // avoid a second RTT
1056  } else {
1057  $after = end( $objects ); // update last item
1058  $after = is_object( $after ) ? $after->name : $after;
1059  }
1060 
1061  return $files;
1062  }
1063 
1071  private function buildFileObjectListing( array $objects ) {
1072  $names = [];
1073  foreach ( $objects as $object ) {
1074  if ( is_object( $object ) ) {
1075  if ( isset( $object->subdir ) || !isset( $object->name ) ) {
1076  continue; // virtual directory entry; ignore
1077  }
1078  $stat = [
1079  // Convert various random Swift dates to TS_MW
1080  'mtime' => $this->convertSwiftDate( $object->last_modified, TS_MW ),
1081  'size' => (int)$object->bytes,
1082  'sha1' => null,
1083  // Note: manifiest ETags are not an MD5 of the file
1084  'md5' => ctype_xdigit( $object->hash ) ? $object->hash : null,
1085  'latest' => false // eventually consistent
1086  ];
1087  $names[] = [ $object->name, $stat ];
1088  } elseif ( substr( $object, -1 ) !== '/' ) {
1089  // Omit directories, which end in '/' in listings
1090  $names[] = [ $object, null ];
1091  }
1092  }
1093 
1094  return $names;
1095  }
1096 
1103  public function loadListingStatInternal( $path, array $val ) {
1104  $this->cheapCache->setField( $path, 'stat', $val );
1105  }
1106 
1107  protected function doGetFileXAttributes( array $params ) {
1108  $stat = $this->getFileStat( $params );
1109  // Stat entries filled by file listings don't include metadata/headers
1110  if ( is_array( $stat ) && !isset( $stat['xattr'] ) ) {
1111  $this->clearCache( [ $params['src'] ] );
1112  $stat = $this->getFileStat( $params );
1113  }
1114 
1115  if ( is_array( $stat ) ) {
1116  return $stat['xattr'];
1117  }
1118 
1119  return ( $stat === self::$RES_ERROR ) ? self::$RES_ERROR : self::$RES_ABSENT;
1120  }
1121 
1122  protected function doGetFileSha1base36( array $params ) {
1123  // Avoid using stat entries from file listings, which never include the SHA-1 hash.
1124  // Also, recompute the hash if it's not part of the metadata headers for some reason.
1125  $params['requireSHA1'] = true;
1126 
1127  $stat = $this->getFileStat( $params );
1128  if ( is_array( $stat ) ) {
1129  return $stat['sha1'];
1130  }
1131 
1132  return ( $stat === self::$RES_ERROR ) ? self::$RES_ERROR : self::$RES_ABSENT;
1133  }
1134 
1135  protected function doStreamFile( array $params ) {
1136  $status = $this->newStatus();
1137 
1138  $flags = !empty( $params['headless'] ) ? HTTPFileStreamer::STREAM_HEADLESS : 0;
1139 
1140  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1141  if ( $srcRel === null ) {
1142  HTTPFileStreamer::send404Message( $params['src'], $flags );
1143  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
1144 
1145  return $status;
1146  }
1147 
1148  $auth = $this->getAuthentication();
1149  if ( !$auth || !is_array( $this->getContainerStat( $srcCont ) ) ) {
1150  HTTPFileStreamer::send404Message( $params['src'], $flags );
1151  $status->fatal( 'backend-fail-stream', $params['src'] );
1152 
1153  return $status;
1154  }
1155 
1156  // If "headers" is set, we only want to send them if the file is there.
1157  // Do not bother checking if the file exists if headers are not set though.
1158  if ( $params['headers'] && !$this->fileExists( $params ) ) {
1159  HTTPFileStreamer::send404Message( $params['src'], $flags );
1160  $status->fatal( 'backend-fail-stream', $params['src'] );
1161 
1162  return $status;
1163  }
1164 
1165  // Send the requested additional headers
1166  foreach ( $params['headers'] as $header ) {
1167  header( $header ); // always send
1168  }
1169 
1170  if ( empty( $params['allowOB'] ) ) {
1171  // Cancel output buffering and gzipping if set
1172  ( $this->obResetFunc )();
1173  }
1174 
1175  $handle = fopen( 'php://output', 'wb' );
1176  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1177  'method' => 'GET',
1178  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1179  'headers' => $this->authTokenHeaders( $auth )
1180  + $this->headersFromParams( $params ) + $params['options'],
1181  'stream' => $handle,
1182  'flags' => [ 'relayResponseHeaders' => empty( $params['headless'] ) ]
1183  ], self::DEFAULT_HTTP_OPTIONS );
1184 
1185  if ( $rcode >= 200 && $rcode <= 299 ) {
1186  // good
1187  } elseif ( $rcode === 404 ) {
1188  $status->fatal( 'backend-fail-stream', $params['src'] );
1189  // Per T43113, nasty things can happen if bad cache entries get
1190  // stuck in cache. It's also possible that this error can come up
1191  // with simple race conditions. Clear out the stat cache to be safe.
1192  $this->clearCache( [ $params['src'] ] );
1193  $this->deleteFileCache( $params['src'] );
1194  } else {
1195  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1196  }
1197 
1198  return $status;
1199  }
1200 
1201  protected function doGetLocalCopyMulti( array $params ) {
1202  $auth = $this->getAuthentication();
1203 
1204  $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging
1205  // Blindly create tmp files and stream to them, catching any exception
1206  // if the file does not exist. Do not waste time doing file stats here.
1207  $reqs = []; // (path => op)
1208 
1209  // Initial dummy values to preserve path order
1210  $tmpFiles = array_fill_keys( $params['srcs'], self::$RES_ERROR );
1211  foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch
1212  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
1213  if ( $srcRel === null || !$auth ) {
1214  continue; // invalid storage path or auth error
1215  }
1216  // Get source file extension
1218  // Create a new temporary file...
1219  $tmpFile = $this->tmpFileFactory->newTempFSFile( 'localcopy_', $ext );
1220  $handle = $tmpFile ? fopen( $tmpFile->getPath(), 'wb' ) : false;
1221  if ( $handle ) {
1222  $reqs[$path] = [
1223  'method' => 'GET',
1224  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1225  'headers' => $this->authTokenHeaders( $auth )
1226  + $this->headersFromParams( $params ),
1227  'stream' => $handle,
1228  ];
1229  $tmpFiles[$path] = $tmpFile;
1230  }
1231  }
1232 
1233  // Ceph RADOS Gateway is in use (strong consistency) or X-Newest will be used
1234  $latest = ( $this->isRGW || !empty( $params['latest'] ) );
1235 
1236  $opts = [
1237  'maxConnsPerHost' => $params['concurrency'],
1239  $reqs = $this->http->runMulti( $reqs, $opts );
1240  foreach ( $reqs as $path => $op ) {
1241  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response'];
1242  fclose( $op['stream'] ); // close open handle
1243  if ( $rcode >= 200 && $rcode <= 299 ) {
1245  $tmpFile = $tmpFiles[$path];
1246  // Make sure that the stream finished and fully wrote to disk
1247  $size = $tmpFile->getSize();
1248  if ( $size !== (int)$rhdrs['content-length'] ) {
1249  $tmpFiles[$path] = self::$RES_ERROR;
1250  $rerr = "Got {$size}/{$rhdrs['content-length']} bytes";
1251  $this->onError( null, __METHOD__,
1252  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
1253  }
1254  // Set the file stat process cache in passing
1255  $stat = $this->getStatFromHeaders( $rhdrs );
1256  $stat['latest'] = $latest;
1257  $this->cheapCache->setField( $path, 'stat', $stat );
1258  } elseif ( $rcode === 404 ) {
1259  $tmpFiles[$path] = self::$RES_ABSENT;
1260  $this->cheapCache->setField(
1261  $path,
1262  'stat',
1263  $latest ? self::$ABSENT_LATEST : self::$ABSENT_NORMAL
1264  );
1265  } else {
1266  $tmpFiles[$path] = self::$RES_ERROR;
1267  $this->onError( null, __METHOD__,
1268  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
1269  }
1270  }
1271 
1272  return $tmpFiles;
1273  }
1274 
1275  public function getFileHttpUrl( array $params ) {
1276  if ( $this->swiftTempUrlKey != '' ||
1277  ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' )
1278  ) {
1279  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1280  if ( $srcRel === null ) {
1281  return self::TEMPURL_ERROR; // invalid path
1282  }
1283 
1284  $auth = $this->getAuthentication();
1285  if ( !$auth ) {
1286  return self::TEMPURL_ERROR;
1287  }
1288 
1289  $ttl = $params['ttl'] ?? 86400;
1290  $expires = time() + $ttl;
1291 
1292  if ( $this->swiftTempUrlKey != '' ) {
1293  $url = $this->storageUrl( $auth, $srcCont, $srcRel );
1294  // Swift wants the signature based on the unencoded object name
1295  $contPath = parse_url( $this->storageUrl( $auth, $srcCont ), PHP_URL_PATH );
1296  $signature = hash_hmac( 'sha1',
1297  "GET\n{$expires}\n{$contPath}/{$srcRel}",
1298  $this->swiftTempUrlKey
1299  );
1300 
1301  return "{$url}?temp_url_sig={$signature}&temp_url_expires={$expires}";
1302  } else { // give S3 API URL for rgw
1303  // Path for signature starts with the bucket
1304  $spath = '/' . rawurlencode( $srcCont ) . '/' .
1305  str_replace( '%2F', '/', rawurlencode( $srcRel ) );
1306  // Calculate the hash
1307  $signature = base64_encode( hash_hmac(
1308  'sha1',
1309  "GET\n\n\n{$expires}\n{$spath}",
1310  $this->rgwS3SecretKey,
1311  true // raw
1312  ) );
1313  // See https://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html.
1314  // Note: adding a newline for empty CanonicalizedAmzHeaders does not work.
1315  // Note: S3 API is the rgw default; remove the /swift/ URL bit.
1316  return str_replace( '/swift/v1', '', $this->storageUrl( $auth ) . $spath ) .
1317  '?' .
1318  http_build_query( [
1319  'Signature' => $signature,
1320  'Expires' => $expires,
1321  'AWSAccessKeyId' => $this->rgwS3AccessKey
1322  ] );
1323  }
1324  }
1325 
1326  return self::TEMPURL_ERROR;
1327  }
1328 
1329  protected function directoriesAreVirtual() {
1330  return true;
1331  }
1332 
1341  protected function headersFromParams( array $params ) {
1342  $hdrs = [];
1343  if ( !empty( $params['latest'] ) ) {
1344  $hdrs['x-newest'] = 'true';
1345  }
1346 
1347  return $hdrs;
1348  }
1349 
1350  protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1352  '@phan-var SwiftFileOpHandle[] $fileOpHandles';
1353 
1355  $statuses = [];
1356 
1357  $auth = $this->getAuthentication();
1358  if ( !$auth ) {
1359  foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1360  $statuses[$index] = $this->newStatus( 'backend-fail-connect', $this->name );
1361  }
1362 
1363  return $statuses;
1364  }
1365 
1366  // Split the HTTP requests into stages that can be done concurrently
1367  $httpReqsByStage = []; // map of (stage => index => HTTP request)
1368  foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1369  $reqs = $fileOpHandle->httpOp;
1370  // Convert the 'url' parameter to an actual URL using $auth
1371  foreach ( $reqs as $stage => &$req ) {
1372  list( $container, $relPath ) = $req['url'];
1373  $req['url'] = $this->storageUrl( $auth, $container, $relPath );
1374  $req['headers'] = $req['headers'] ?? [];
1375  $req['headers'] = $this->authTokenHeaders( $auth ) + $req['headers'];
1376  $httpReqsByStage[$stage][$index] = $req;
1377  }
1378  $statuses[$index] = $this->newStatus();
1379  }
1380 
1381  // Run all requests for the first stage, then the next, and so on
1382  $reqCount = count( $httpReqsByStage );
1383  for ( $stage = 0; $stage < $reqCount; ++$stage ) {
1384  $httpReqs = $this->http->runMulti( $httpReqsByStage[$stage], self::DEFAULT_HTTP_OPTIONS );
1385  foreach ( $httpReqs as $index => $httpReq ) {
1387  $fileOpHandle = $fileOpHandles[$index];
1388  // Run the callback for each request of this operation
1389  $status = $statuses[$index];
1390  ( $fileOpHandle->callback )( $httpReq, $status );
1391  // On failure, abort all remaining requests for this operation. This is used
1392  // in "move" operations to abort the DELETE request if the PUT request fails.
1393  if (
1394  !$status->isOK() ||
1395  $fileOpHandle->state === $fileOpHandle::CONTINUE_NO
1396  ) {
1397  $stages = count( $fileOpHandle->httpOp );
1398  for ( $s = ( $stage + 1 ); $s < $stages; ++$s ) {
1399  unset( $httpReqsByStage[$s][$index] );
1400  }
1401  }
1402  }
1403  }
1404 
1405  return $statuses;
1406  }
1407 
1430  protected function setContainerAccess( $container, array $readUsers, array $writeUsers ) {
1431  $status = $this->newStatus();
1432  $auth = $this->getAuthentication();
1433 
1434  if ( !$auth ) {
1435  $status->fatal( 'backend-fail-connect', $this->name );
1436 
1437  return $status;
1438  }
1439 
1440  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1441  'method' => 'POST',
1442  'url' => $this->storageUrl( $auth, $container ),
1443  'headers' => $this->authTokenHeaders( $auth ) + [
1444  'x-container-read' => implode( ',', $readUsers ),
1445  'x-container-write' => implode( ',', $writeUsers )
1446  ]
1447  ], self::DEFAULT_HTTP_OPTIONS );
1448 
1449  if ( $rcode != 204 && $rcode !== 202 ) {
1450  $status->fatal( 'backend-fail-internal', $this->name );
1451  $this->logger->error( __METHOD__ . ': unexpected rcode value ({rcode})',
1452  [ 'rcode' => $rcode ] );
1453  }
1454 
1455  return $status;
1456  }
1457 
1466  protected function getContainerStat( $container, $bypassCache = false ) {
1468  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1469 
1470  if ( $bypassCache ) { // purge cache
1471  $this->containerStatCache->clear( $container );
1472  } elseif ( !$this->containerStatCache->hasField( $container, 'stat' ) ) {
1473  $this->primeContainerCache( [ $container ] ); // check persistent cache
1474  }
1475  if ( !$this->containerStatCache->hasField( $container, 'stat' ) ) {
1476  $auth = $this->getAuthentication();
1477  if ( !$auth ) {
1478  return self::$RES_ERROR;
1479  }
1480 
1481  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1482  'method' => 'HEAD',
1483  'url' => $this->storageUrl( $auth, $container ),
1484  'headers' => $this->authTokenHeaders( $auth )
1485  ], self::DEFAULT_HTTP_OPTIONS );
1486 
1487  if ( $rcode === 204 ) {
1488  $stat = [
1489  'count' => $rhdrs['x-container-object-count'],
1490  'bytes' => $rhdrs['x-container-bytes-used']
1491  ];
1492  if ( $bypassCache ) {
1493  return $stat;
1494  } else {
1495  $this->containerStatCache->setField( $container, 'stat', $stat ); // cache it
1496  $this->setContainerCache( $container, $stat ); // update persistent cache
1497  }
1498  } elseif ( $rcode === 404 ) {
1499  return self::$RES_ABSENT;
1500  } else {
1501  $this->onError( null, __METHOD__,
1502  [ 'cont' => $container ], $rerr, $rcode, $rdesc );
1503 
1504  return self::$RES_ERROR;
1505  }
1506  }
1507 
1508  return $this->containerStatCache->getField( $container, 'stat' );
1509  }
1510 
1518  protected function createContainer( $container, array $params ) {
1519  $status = $this->newStatus();
1520 
1521  $auth = $this->getAuthentication();
1522  if ( !$auth ) {
1523  $status->fatal( 'backend-fail-connect', $this->name );
1524 
1525  return $status;
1526  }
1527 
1528  // @see SwiftFileBackend::setContainerAccess()
1529  if ( empty( $params['noAccess'] ) ) {
1530  // public
1531  $readUsers = array_merge( $this->readUsers, [ '.r:*', $this->swiftUser ] );
1532  $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
1533  } else {
1534  // private
1535  $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
1536  $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
1537  }
1538 
1539  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1540  'method' => 'PUT',
1541  'url' => $this->storageUrl( $auth, $container ),
1542  'headers' => $this->authTokenHeaders( $auth ) + [
1543  'x-container-read' => implode( ',', $readUsers ),
1544  'x-container-write' => implode( ',', $writeUsers )
1545  ]
1546  ], self::DEFAULT_HTTP_OPTIONS );
1547 
1548  if ( $rcode === 201 ) { // new
1549  // good
1550  } elseif ( $rcode === 202 ) { // already there
1551  // this shouldn't really happen, but is OK
1552  } else {
1553  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1554  }
1555 
1556  return $status;
1557  }
1558 
1566  protected function deleteContainer( $container, array $params ) {
1567  $status = $this->newStatus();
1568 
1569  $auth = $this->getAuthentication();
1570  if ( !$auth ) {
1571  $status->fatal( 'backend-fail-connect', $this->name );
1572 
1573  return $status;
1574  }
1575 
1576  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1577  'method' => 'DELETE',
1578  'url' => $this->storageUrl( $auth, $container ),
1579  'headers' => $this->authTokenHeaders( $auth )
1580  ], self::DEFAULT_HTTP_OPTIONS );
1581 
1582  if ( $rcode >= 200 && $rcode <= 299 ) { // deleted
1583  $this->containerStatCache->clear( $container ); // purge
1584  } elseif ( $rcode === 404 ) { // not there
1585  // this shouldn't really happen, but is OK
1586  } elseif ( $rcode === 409 ) { // not empty
1587  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); // race?
1588  } else {
1589  // @phan-suppress-previous-line PhanPluginDuplicateIfStatements
1590  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1591  }
1592 
1593  return $status;
1594  }
1595 
1608  private function objectListing(
1609  $fullCont, $type, $limit, $after = null, $prefix = null, $delim = null
1610  ) {
1611  $status = $this->newStatus();
1612 
1613  $auth = $this->getAuthentication();
1614  if ( !$auth ) {
1615  $status->fatal( 'backend-fail-connect', $this->name );
1616 
1617  return $status;
1618  }
1619 
1620  $query = [ 'limit' => $limit ];
1621  if ( $type === 'info' ) {
1622  $query['format'] = 'json';
1623  }
1624  if ( $after !== null ) {
1625  $query['marker'] = $after;
1626  }
1627  if ( $prefix !== null ) {
1628  $query['prefix'] = $prefix;
1629  }
1630  if ( $delim !== null ) {
1631  $query['delimiter'] = $delim;
1632  }
1633 
1634  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1635  'method' => 'GET',
1636  'url' => $this->storageUrl( $auth, $fullCont ),
1637  'query' => $query,
1638  'headers' => $this->authTokenHeaders( $auth )
1639  ], self::DEFAULT_HTTP_OPTIONS );
1640 
1641  $params = [ 'cont' => $fullCont, 'prefix' => $prefix, 'delim' => $delim ];
1642  if ( $rcode === 200 ) { // good
1643  if ( $type === 'info' ) {
1644  $status->value = FormatJson::decode( trim( $rbody ) );
1645  } else {
1646  $status->value = explode( "\n", trim( $rbody ) );
1647  }
1648  } elseif ( $rcode === 204 ) {
1649  $status->value = []; // empty container
1650  } elseif ( $rcode === 404 ) {
1651  $status->value = []; // no container
1652  } else {
1653  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1654  }
1655 
1656  return $status;
1657  }
1658 
1659  protected function doPrimeContainerCache( array $containerInfo ) {
1660  foreach ( $containerInfo as $container => $info ) {
1661  $this->containerStatCache->setField( $container, 'stat', $info );
1662  }
1663  }
1664 
1665  protected function doGetFileStatMulti( array $params ) {
1666  $stats = [];
1667 
1668  $auth = $this->getAuthentication();
1669 
1670  $reqs = []; // (path => op)
1671  // (a) Check the containers of the paths...
1672  foreach ( $params['srcs'] as $path ) {
1673  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
1674  if ( $srcRel === null || !$auth ) {
1675  $stats[$path] = self::$RES_ERROR;
1676  continue; // invalid storage path or auth error
1677  }
1678 
1679  $cstat = $this->getContainerStat( $srcCont );
1680  if ( $cstat === self::$RES_ABSENT ) {
1681  $stats[$path] = self::$RES_ABSENT;
1682  continue; // ok, nothing to do
1683  } elseif ( !is_array( $cstat ) ) {
1684  $stats[$path] = self::$RES_ERROR;
1685  continue;
1686  }
1687 
1688  $reqs[$path] = [
1689  'method' => 'HEAD',
1690  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1691  'headers' => $this->authTokenHeaders( $auth ) + $this->headersFromParams( $params )
1692  ];
1693  }
1694 
1695  // (b) Check the files themselves...
1696  $opts = [
1697  'maxConnsPerHost' => $params['concurrency'],
1699  $reqs = $this->http->runMulti( $reqs, $opts );
1700  foreach ( $reqs as $path => $op ) {
1701  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response'];
1702  if ( $rcode === 200 || $rcode === 204 ) {
1703  // Update the object if it is missing some headers
1704  if ( !empty( $params['requireSHA1'] ) ) {
1705  $rhdrs = $this->addMissingHashMetadata( $rhdrs, $path );
1706  }
1707  // Load the stat array from the headers
1708  $stat = $this->getStatFromHeaders( $rhdrs );
1709  if ( $this->isRGW ) {
1710  $stat['latest'] = true; // strong consistency
1711  }
1712  } elseif ( $rcode === 404 ) {
1713  $stat = self::$RES_ABSENT;
1714  } else {
1715  $stat = self::$RES_ERROR;
1716  $this->onError( null, __METHOD__, $params, $rerr, $rcode, $rdesc );
1717  }
1718  $stats[$path] = $stat;
1719  }
1720 
1721  return $stats;
1722  }
1723 
1728  protected function getStatFromHeaders( array $rhdrs ) {
1729  // Fetch all of the custom metadata headers
1730  $metadata = $this->getMetadataFromHeaders( $rhdrs );
1731  // Fetch all of the custom raw HTTP headers
1732  $headers = $this->extractMutableContentHeaders( $rhdrs );
1733 
1734  return [
1735  // Convert various random Swift dates to TS_MW
1736  'mtime' => $this->convertSwiftDate( $rhdrs['last-modified'], TS_MW ),
1737  // Empty objects actually return no content-length header in Ceph
1738  'size' => isset( $rhdrs['content-length'] ) ? (int)$rhdrs['content-length'] : 0,
1739  'sha1' => $metadata['sha1base36'] ?? null,
1740  // Note: manifiest ETags are not an MD5 of the file
1741  'md5' => ctype_xdigit( $rhdrs['etag'] ) ? $rhdrs['etag'] : null,
1742  'xattr' => [ 'metadata' => $metadata, 'headers' => $headers ]
1743  ];
1744  }
1745 
1749  protected function getAuthentication() {
1750  if ( $this->authErrorTimestamp !== null ) {
1751  if ( ( time() - $this->authErrorTimestamp ) < 60 ) {
1752  return null; // failed last attempt; don't bother
1753  } else { // actually retry this time
1754  $this->authErrorTimestamp = null;
1755  }
1756  }
1757  // Session keys expire after a while, so we renew them periodically
1758  $reAuth = ( ( time() - $this->authSessionTimestamp ) > $this->authTTL );
1759  // Authenticate with proxy and get a session key...
1760  if ( !$this->authCreds || $reAuth ) {
1761  $this->authSessionTimestamp = 0;
1762  $cacheKey = $this->getCredsCacheKey( $this->swiftUser );
1763  $creds = $this->srvCache->get( $cacheKey ); // credentials
1764  // Try to use the credential cache
1765  if ( isset( $creds['auth_token'] ) && isset( $creds['storage_url'] ) ) {
1766  $this->authCreds = $creds;
1767  // Skew the timestamp for worst case to avoid using stale credentials
1768  $this->authSessionTimestamp = time() - ceil( $this->authTTL / 2 );
1769  } else { // cache miss
1770  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1771  'method' => 'GET',
1772  'url' => "{$this->swiftAuthUrl}/v1.0",
1773  'headers' => [
1774  'x-auth-user' => $this->swiftUser,
1775  'x-auth-key' => $this->swiftKey
1776  ]
1777  ], self::DEFAULT_HTTP_OPTIONS );
1778 
1779  if ( $rcode >= 200 && $rcode <= 299 ) { // OK
1780  $this->authCreds = [
1781  'auth_token' => $rhdrs['x-auth-token'],
1782  'storage_url' => $this->swiftStorageUrl ?? $rhdrs['x-storage-url']
1783  ];
1784 
1785  $this->srvCache->set( $cacheKey, $this->authCreds, ceil( $this->authTTL / 2 ) );
1786  $this->authSessionTimestamp = time();
1787  } elseif ( $rcode === 401 ) {
1788  $this->onError( null, __METHOD__, [], "Authentication failed.", $rcode );
1789  $this->authErrorTimestamp = time();
1790 
1791  return null;
1792  } else {
1793  $this->onError( null, __METHOD__, [], "HTTP return code: $rcode", $rcode );
1794  $this->authErrorTimestamp = time();
1795 
1796  return null;
1797  }
1798  }
1799  // Ceph RGW does not use <account> in URLs (OpenStack Swift uses "/v1/<account>")
1800  if ( substr( $this->authCreds['storage_url'], -3 ) === '/v1' ) {
1801  $this->isRGW = true; // take advantage of strong consistency in Ceph
1802  }
1803  }
1804 
1805  return $this->authCreds;
1806  }
1807 
1814  protected function storageUrl( array $creds, $container = null, $object = null ) {
1815  $parts = [ $creds['storage_url'] ];
1816  if ( strlen( $container ) ) {
1817  $parts[] = rawurlencode( $container );
1818  }
1819  if ( strlen( $object ) ) {
1820  $parts[] = str_replace( "%2F", "/", rawurlencode( $object ) );
1821  }
1822 
1823  return implode( '/', $parts );
1824  }
1825 
1830  protected function authTokenHeaders( array $creds ) {
1831  return [ 'x-auth-token' => $creds['auth_token'] ];
1832  }
1833 
1840  private function getCredsCacheKey( $username ) {
1841  return 'swiftcredentials:' . md5( $username . ':' . $this->swiftAuthUrl );
1842  }
1843 
1855  public function onError( $status, $func, array $params, $err = '', $code = 0, $desc = '' ) {
1856  if ( $status instanceof StatusValue ) {
1857  $status->fatal( 'backend-fail-internal', $this->name );
1858  }
1859  if ( $code == 401 ) { // possibly a stale token
1860  $this->srvCache->delete( $this->getCredsCacheKey( $this->swiftUser ) );
1861  }
1862  $msg = "HTTP {code} ({desc}) in '{func}' (given '{req_params}')";
1863  $msgParams = [
1864  'code' => $code,
1865  'desc' => $desc,
1866  'func' => $func,
1867  'req_params' => FormatJson::encode( $params ),
1868  ];
1869  if ( $err ) {
1870  $msg .= ': {err}';
1871  $msgParams['err'] = $err;
1872  }
1873  $this->logger->error( $msg, $msgParams );
1874  }
1875 }
SwiftFileOpHandle
Definition: SwiftFileOpHandle.php:25
MWTimestamp
Library for creating and parsing MW-style timestamps.
Definition: MWTimestamp.php:38
SwiftFileBackend\doPrepareInternal
doPrepareInternal( $fullCont, $dir, array $params)
FileBackendStore::doPrepare() Stability: stableto override StatusValue Good status without value for ...
Definition: SwiftFileBackend.php:642
SwiftFileBackend\$isRGW
bool $isRGW
Whether the server is an Ceph RGW.
Definition: SwiftFileBackend.php:81
MultiHttpClient
Class to handle multiple HTTP requests.
Definition: MultiHttpClient.php:55
StatusValue
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Definition: StatusValue.php:43
SwiftFileBackend\isPathUsableInternal
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path in the backend.
Definition: SwiftFileBackend.php:186
SwiftFileBackend\$authErrorTimestamp
int $authErrorTimestamp
UNIX timestamp.
Definition: SwiftFileBackend.php:78
SwiftFileBackend\getAuthentication
getAuthentication()
Definition: SwiftFileBackend.php:1749
SwiftFileBackend\getContainerStat
getContainerStat( $container, $bypassCache=false)
Get a Swift container stat array, possibly from process cache.
Definition: SwiftFileBackend.php:1466
SwiftFileBackend\buildFileObjectListing
buildFileObjectListing(array $objects)
Build a list of file objects, filtering out any directories and extracting any stat info if provided ...
Definition: SwiftFileBackend.php:1071
EmptyBagOStuff
A BagOStuff object with no objects in it.
Definition: EmptyBagOStuff.php:29
SwiftFileBackend\$containerStatCache
MapCacheLRU $containerStatCache
Container stat cache.
Definition: SwiftFileBackend.php:71
SwiftFileBackend\$swiftKey
string $swiftKey
Secret key for user.
Definition: SwiftFileBackend.php:51
SwiftFileBackend\getStatFromHeaders
getStatFromHeaders(array $rhdrs)
Definition: SwiftFileBackend.php:1728
SwiftFileBackend\addMissingHashMetadata
addMissingHashMetadata(array $objHdrs, $path)
Fill in any missing object metadata and save it to Swift.
Definition: SwiftFileBackend.php:778
LockManager\LOCK_UW
const LOCK_UW
Definition: LockManager.php:71
SwiftFileBackend\doGetFileXAttributes
doGetFileXAttributes(array $params)
Definition: SwiftFileBackend.php:1107
FileBackendError
File backend exception for checked exceptions (e.g.
Definition: FileBackendError.php:10
FileBackend\getScopedFileLocks
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
Definition: FileBackend.php:1427
FileBackendStore\fileExists
fileExists(array $params)
Check if a file exists at a storage path in the backend.
Definition: FileBackendStore.php:647
SwiftFileBackend\$swiftUser
string $swiftUser
Swift user (account:user) to authenticate as.
Definition: SwiftFileBackend.php:49
SwiftFileBackend\loadListingStatInternal
loadListingStatInternal( $path, array $val)
Do not call this function outside of SwiftFileBackendFileList.
Definition: SwiftFileBackend.php:1103
FileBackend\extensionFromPath
static extensionFromPath( $path, $case='lowercase')
Get the final extension from a storage or FS path.
Definition: FileBackend.php:1584
FileBackendStore\deleteFileCache
deleteFileCache( $path)
Delete the cached stat info for a file path.
Definition: FileBackendStore.php:1913
SwiftFileBackend\resolveContainerPath
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
Definition: SwiftFileBackend.php:176
FileBackendStore\executeOpHandlesInternal
executeOpHandlesInternal(array $fileOpHandles)
Execute a list of FileBackendStoreOpHandle handles in parallel.
Definition: FileBackendStore.php:1405
BagOStuff
Class representing a cache/ephemeral data store.
Definition: BagOStuff.php:86
SwiftFileBackend\DEFAULT_HTTP_OPTIONS
const DEFAULT_HTTP_OPTIONS
Definition: SwiftFileBackend.php:38
SwiftFileBackend\doGetFileContentsMulti
doGetFileContentsMulti(array $params)
Definition: SwiftFileBackend.php:832
SwiftFileBackend\createContainer
createContainer( $container, array $params)
Create a Swift container.
Definition: SwiftFileBackend.php:1518
SwiftFileBackend\extractMetadataHeaders
extractMetadataHeaders(array $headers)
Definition: SwiftFileBackend.php:244
SwiftFileBackend\getFileListPageInternal
getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
Definition: SwiftFileBackend.php:1018
SwiftFileBackend\getMetadataFromHeaders
getMetadataFromHeaders(array $headers)
Definition: SwiftFileBackend.php:261
SwiftFileBackend\doGetFileSha1base36
doGetFileSha1base36(array $params)
Definition: SwiftFileBackend.php:1122
FileBackendStore\clearCache
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
Definition: FileBackendStore.php:1487
SwiftFileBackend\getFeatures
getFeatures()
Get the a bitfield of extra features supported by the backend medium.
Definition: SwiftFileBackend.php:168
FileBackendStore\$RES_ERROR
static null $RES_ERROR
Idiom for "no result due to I/O errors" (since 1.34)
Definition: FileBackendStore.php:66
SwiftFileBackend\doDeleteInternal
doDeleteInternal(array $params)
Definition: SwiftFileBackend.php:538
SwiftFileBackend\setLogger
setLogger(LoggerInterface $logger)
Definition: SwiftFileBackend.php:163
SwiftFileBackend\headersFromParams
headersFromParams(array $params)
Get headers to send to Swift when reading a file based on a FileBackend params array,...
Definition: SwiftFileBackend.php:1341
SwiftFileBackend\deleteContainer
deleteContainer( $container, array $params)
Delete a Swift container.
Definition: SwiftFileBackend.php:1566
SwiftFileBackend\getCredsCacheKey
getCredsCacheKey( $username)
Get the cache key for a container.
Definition: SwiftFileBackend.php:1840
FormatJson\decode
static decode( $value, $assoc=false)
Decodes a JSON string.
Definition: FormatJson.php:146
FormatJson\encode
static encode( $value, $pretty=false, $escaping=0)
Returns the JSON representation of a value.
Definition: FormatJson.php:96
SwiftFileBackend\doDirectoryExists
doDirectoryExists( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:892
SwiftFileBackend
Class for an OpenStack Swift (or Ceph RGW) based file backend.
Definition: SwiftFileBackend.php:37
SwiftFileBackend\getDirectoryListInternal
getDirectoryListInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:909
SwiftFileBackend\setContainerAccess
setContainerAccess( $container, array $readUsers, array $writeUsers)
Set read/write permissions for a Swift container.
Definition: SwiftFileBackend.php:1430
SwiftFileBackend\getFileHttpUrl
getFileHttpUrl(array $params)
Definition: SwiftFileBackend.php:1275
SwiftFileBackend\doMoveInternal
doMoveInternal(array $params)
Definition: SwiftFileBackend.php:471
MapCacheLRU
Handles a simple LRU key/value map with a maximum number of entries.
Definition: MapCacheLRU.php:36
SwiftFileBackend\$authCreds
array $authCreds
Definition: SwiftFileBackend.php:74
$dirs
$dirs
Definition: mergeMessageFileList.php:213
SwiftFileBackend\extractMutableContentHeaders
extractMutableContentHeaders(array $headers)
Filter/normalize a header map to only include mutable "content-"/"x-content-" headers.
Definition: SwiftFileBackend.php:204
SwiftFileBackend\doGetLocalCopyMulti
doGetLocalCopyMulti(array $params)
Definition: SwiftFileBackend.php:1201
SwiftFileBackend\onError
onError( $status, $func, array $params, $err='', $code=0, $desc='')
Log an unexpected exception for this backend.
Definition: SwiftFileBackend.php:1855
SwiftFileBackend\$secureWriteUsers
array $secureWriteUsers
Additional users (account:user) with write permissions on private containers.
Definition: SwiftFileBackend.php:65
SwiftFileBackend\doDescribeInternal
doDescribeInternal(array $params)
Definition: SwiftFileBackend.php:580
SwiftFileBackend\storageUrl
storageUrl(array $creds, $container=null, $object=null)
Definition: SwiftFileBackend.php:1814
FileBackendStore\getContentType
getContentType( $storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
Definition: FileBackendStore.php:2035
FileBackend\$obResetFunc
callable $obResetFunc
Definition: FileBackend.php:126
SwiftFileBackendDirList
Iterator for listing directories.
Definition: SwiftFileBackendDirList.php:28
$content
$content
Definition: router.php:76
SwiftFileBackend\doSecureInternal
doSecureInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:665
$s
foreach( $mmfl['setupFiles'] as $fileName) if( $queue) if(empty( $mmfl['quiet'])) $s
Definition: mergeMessageFileList.php:206
FileBackendStore\getFileStat
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
Definition: FileBackendStore.php:683
$header
$header
Definition: updateCredits.php:37
SwiftFileBackend\__construct
__construct(array $config)
Definition: SwiftFileBackend.php:121
SwiftFileBackend\directoriesAreVirtual
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
Definition: SwiftFileBackend.php:1329
FileBackendStore\resolveStoragePathReal
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
Definition: FileBackendStore.php:1645
SwiftFileBackend\authTokenHeaders
authTokenHeaders(array $creds)
Definition: SwiftFileBackend.php:1830
WANObjectCache
Multi-datacenter aware caching interface.
Definition: WANObjectCache.php:131
SwiftFileBackend\doCreateInternal
doCreateInternal(array $params)
Definition: SwiftFileBackend.php:272
SwiftFileBackend\$authSessionTimestamp
int $authSessionTimestamp
UNIX timestamp.
Definition: SwiftFileBackend.php:76
FileBackendStore
Base class for all backends using particular storage medium.
Definition: FileBackendStore.php:41
HTTPFileStreamer\STREAM_HEADLESS
const STREAM_HEADLESS
Definition: HTTPFileStreamer.php:40
FileBackendStore\$RES_ABSENT
static false $RES_ABSENT
Idiom for "no result due to missing file" (since 1.34)
Definition: FileBackendStore.php:64
SwiftFileBackend\doExecuteOpHandlesInternal
doExecuteOpHandlesInternal(array $fileOpHandles)
Definition: SwiftFileBackend.php:1350
FileBackend\$logger
LoggerInterface $logger
Definition: FileBackend.php:121
SwiftFileBackend\$swiftAuthUrl
string $swiftAuthUrl
Authentication base URL (without version)
Definition: SwiftFileBackend.php:45
SwiftFileBackend\doStreamFile
doStreamFile(array $params)
Definition: SwiftFileBackend.php:1135
HTTPFileStreamer\send404Message
static send404Message( $fname, $flags=0)
Send out a standard 404 message for a file.
Definition: HTTPFileStreamer.php:203
FileBackend\newStatus
newStatus(... $args)
Yields the result of the status wrapper callback on either:
Definition: FileBackend.php:1677
SwiftFileBackend\$writeUsers
array $writeUsers
Additional users (account:user) with write permissions on public containers.
Definition: SwiftFileBackend.php:61
SwiftFileBackend\$rgwS3SecretKey
string $rgwS3SecretKey
S3 authentication key (RADOS Gateway)
Definition: SwiftFileBackend.php:57
FileBackendStore\primeContainerCache
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
Definition: FileBackendStore.php:1831
SwiftFileBackend\$authTTL
int $authTTL
TTL in seconds.
Definition: SwiftFileBackend.php:43
SwiftFileBackend\$readUsers
array $readUsers
Additional users (account:user) with read permissions on public containers.
Definition: SwiftFileBackend.php:59
SwiftFileBackend\doCopyInternal
doCopyInternal(array $params)
Definition: SwiftFileBackend.php:416
FileBackend\$name
string $name
Unique backend name.
Definition: FileBackend.php:101
SwiftFileBackend\$rgwS3AccessKey
string $rgwS3AccessKey
S3 access key (RADOS Gateway)
Definition: SwiftFileBackend.php:55
FileBackend\getLocalCopy
getLocalCopy(array $params)
Get a local copy on disk of the file at a storage path in the backend.
Definition: FileBackend.php:1189
SwiftFileBackend\objectListing
objectListing( $fullCont, $type, $limit, $after=null, $prefix=null, $delim=null)
Get a list of objects under a container.
Definition: SwiftFileBackend.php:1608
SwiftFileBackend\$swiftStorageUrl
string $swiftStorageUrl
Override of storage base URL.
Definition: SwiftFileBackend.php:47
$path
$path
Definition: NoLocalSettings.php:25
SwiftFileBackend\convertSwiftDate
convertSwiftDate( $ts, $format=TS_MW)
Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT"/"2013-05-11T07:37:27.678360Z".
Definition: SwiftFileBackend.php:761
SwiftFileBackend\doPublishInternal
doPublishInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:691
SwiftFileBackend\doGetFileStat
doGetFileStat(array $params)
Definition: SwiftFileBackend.php:743
Wikimedia
This program is free software; you can redistribute it and/or modify it under the terms of the GNU Ge...
SwiftFileBackend\$swiftTempUrlKey
string $swiftTempUrlKey
Shared secret value for making temp URLs.
Definition: SwiftFileBackend.php:53
SwiftFileBackend\doStoreInternal
doStoreInternal(array $params)
Definition: SwiftFileBackend.php:327
SwiftFileBackend\doGetFileStatMulti
doGetFileStatMulti(array $params)
Get file stat information (concurrently if possible) for several files.
Definition: SwiftFileBackend.php:1665
$ext
if(!is_readable( $file)) $ext
Definition: router.php:48
SwiftFileBackend\doCleanInternal
doCleanInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:715
SwiftFileBackend\$srvCache
BagOStuff $srvCache
Definition: SwiftFileBackend.php:68
SwiftFileBackendFileList
Iterator for listing regular files.
Definition: SwiftFileBackendFileList.php:28
SwiftFileBackend\getFileListInternal
getFileListInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:920
SwiftFileBackend\doPrimeContainerCache
doPrimeContainerCache(array $containerInfo)
Fill the backend-specific process cache given an array of resolved container names and their correspo...
Definition: SwiftFileBackend.php:1659
SwiftFileBackend\$secureReadUsers
array $secureReadUsers
Additional users (account:user) with read permissions on private containers.
Definition: SwiftFileBackend.php:63
FileBackendStore\setContainerCache
setContainerCache( $container, array $val)
Set the cached info for a container.
Definition: FileBackendStore.php:1806
FileBackend\scopedProfileSection
scopedProfileSection( $section)
Definition: FileBackend.php:1699
SwiftFileBackend\getDirListPageInternal
getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
Definition: SwiftFileBackend.php:935
SwiftFileBackend\$http
MultiHttpClient $http
Definition: SwiftFileBackend.php:41
$type
$type
Definition: testCompression.php:52