MediaWiki  master
SwiftFileBackend.php
Go to the documentation of this file.
1 <?php
25 use Psr\Log\LoggerInterface;
26 use Wikimedia\AtEase\AtEase;
27 
39  protected $http;
41  protected $authTTL;
43  protected $swiftAuthUrl;
45  protected $swiftStorageUrl;
47  protected $swiftUser;
49  protected $swiftKey;
51  protected $swiftTempUrlKey;
53  protected $rgwS3AccessKey;
55  protected $rgwS3SecretKey;
57  protected $readUsers;
59  protected $writeUsers;
61  protected $secureReadUsers;
63  protected $secureWriteUsers;
64 
66  protected $srvCache;
67 
70 
72  protected $authCreds;
74  protected $authSessionTimestamp = 0;
76  protected $authErrorTimestamp = null;
77 
79  protected $isRGW = false;
80 
119  public function __construct( array $config ) {
120  parent::__construct( $config );
121  // Required settings
122  $this->swiftAuthUrl = $config['swiftAuthUrl'];
123  $this->swiftUser = $config['swiftUser'];
124  $this->swiftKey = $config['swiftKey'];
125  // Optional settings
126  $this->authTTL = $config['swiftAuthTTL'] ?? 15 * 60; // some sane number
127  $this->swiftTempUrlKey = $config['swiftTempUrlKey'] ?? '';
128  $this->swiftStorageUrl = $config['swiftStorageUrl'] ?? null;
129  $this->shardViaHashLevels = $config['shardViaHashLevels'] ?? '';
130  $this->rgwS3AccessKey = $config['rgwS3AccessKey'] ?? '';
131  $this->rgwS3SecretKey = $config['rgwS3SecretKey'] ?? '';
132 
133  // HTTP helper client
134  $httpOptions = [];
135  foreach ( [ 'connTimeout', 'reqTimeout' ] as $optionName ) {
136  if ( isset( $config[$optionName] ) ) {
137  $httpOptions[$optionName] = $config[$optionName];
138  }
139  }
140  $this->http = new MultiHttpClient( $httpOptions );
141  $this->http->setLogger( $this->logger );
142 
143  // Cache container information to mask latency
144  if ( isset( $config['wanCache'] ) && $config['wanCache'] instanceof WANObjectCache ) {
145  $this->memCache = $config['wanCache'];
146  }
147  // Process cache for container info
148  $this->containerStatCache = new MapCacheLRU( 300 );
149  // Cache auth token information to avoid RTTs
150  if ( !empty( $config['cacheAuthInfo'] ) && isset( $config['srvCache'] ) ) {
151  $this->srvCache = $config['srvCache'];
152  } else {
153  $this->srvCache = new EmptyBagOStuff();
154  }
155  $this->readUsers = $config['readUsers'] ?? [];
156  $this->writeUsers = $config['writeUsers'] ?? [];
157  $this->secureReadUsers = $config['secureReadUsers'] ?? [];
158  $this->secureWriteUsers = $config['secureWriteUsers'] ?? [];
159  }
160 
161  public function setLogger( LoggerInterface $logger ) {
162  parent::setLogger( $logger );
163  $this->http->setLogger( $logger );
164  }
165 
166  public function getFeatures() {
167  return (
168  self::ATTR_UNICODE_PATHS |
169  self::ATTR_HEADERS |
170  self::ATTR_METADATA
171  );
172  }
173 
174  protected function resolveContainerPath( $container, $relStoragePath ) {
175  if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) {
176  return null; // not UTF-8, makes it hard to use CF and the swift HTTP API
177  } elseif ( strlen( rawurlencode( $relStoragePath ) ) > 1024 ) {
178  return null; // too long for Swift
179  }
180 
181  return $relStoragePath;
182  }
183 
184  public function isPathUsableInternal( $storagePath ) {
185  list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
186  if ( $rel === null ) {
187  return false; // invalid
188  }
189 
190  return is_array( $this->getContainerStat( $container ) );
191  }
192 
202  protected function extractMutableContentHeaders( array $headers ) {
203  $contentHeaders = [];
204  // Normalize casing, and strip out illegal headers
205  foreach ( $headers as $name => $value ) {
206  $name = strtolower( $name );
207  if ( $name === 'x-delete-at' && is_numeric( $value ) ) {
208  // Expects a Unix Epoch date
209  $contentHeaders[$name] = $value;
210  } elseif ( $name === 'x-delete-after' && is_numeric( $value ) ) {
211  // Expects number of minutes time to live.
212  $contentHeaders[$name] = $value;
213  } elseif ( preg_match( '/^(x-)?content-(?!length$)/', $name ) ) {
214  // Only allow content-* and x-content-* headers (but not content-length)
215  $contentHeaders[$name] = $value;
216  } elseif ( $name === 'content-type' && strlen( $value ) ) {
217  // This header can be set to a value but not unset for sanity
218  $contentHeaders[$name] = $value;
219  }
220  }
221  // By default, Swift has annoyingly low maximum header value limits
222  if ( isset( $contentHeaders['content-disposition'] ) ) {
223  $maxLength = 255;
224  // @note: assume FileBackend::makeContentDisposition() already used
225  $offset = $maxLength - strlen( $contentHeaders['content-disposition'] );
226  if ( $offset < 0 ) {
227  $pos = strrpos( $contentHeaders['content-disposition'], ';', $offset );
228  $contentHeaders['content-disposition'] = $pos === false
229  ? ''
230  : trim( substr( $contentHeaders['content-disposition'], 0, $pos ) );
231  }
232  }
233 
234  return $contentHeaders;
235  }
236 
242  protected function extractMetadataHeaders( array $headers ) {
243  $metadataHeaders = [];
244  foreach ( $headers as $name => $value ) {
245  $name = strtolower( $name );
246  if ( strpos( $name, 'x-object-meta-' ) === 0 ) {
247  $metadataHeaders[$name] = $value;
248  }
249  }
250 
251  return $metadataHeaders;
252  }
253 
259  protected function getMetadataFromHeaders( array $headers ) {
260  $prefixLen = strlen( 'x-object-meta-' );
261 
262  $metadata = [];
263  foreach ( $this->extractMetadataHeaders( $headers ) as $name => $value ) {
264  $metadata[substr( $name, $prefixLen )] = $value;
265  }
266 
267  return $metadata;
268  }
269 
270  protected function doCreateInternal( array $params ) {
271  $status = $this->newStatus();
272 
273  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
274  if ( $dstRel === null ) {
275  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
276 
277  return $status;
278  }
279 
280  // Headers that are not strictly a function of the file content
281  $mutableHeaders = $this->extractMutableContentHeaders( $params['headers'] ?? [] );
282  // Make sure that the "content-type" header is set to something sensible
283  $mutableHeaders['content-type'] = $mutableHeaders['content-type']
284  ?? $this->getContentType( $params['dst'], $params['content'], null );
285 
286  $reqs = [ [
287  'method' => 'PUT',
288  'url' => [ $dstCont, $dstRel ],
289  'headers' => array_merge(
290  $mutableHeaders,
291  [
292  'etag' => md5( $params['content'] ),
293  'content-length' => strlen( $params['content'] ),
294  'x-object-meta-sha1base36' =>
295  Wikimedia\base_convert( sha1( $params['content'] ), 16, 36, 31 )
296  ]
297  ),
298  'body' => $params['content']
299  ] ];
300 
301  $method = __METHOD__;
302  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
303  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
304  if ( $rcode === 201 || $rcode === 202 ) {
305  // good
306  } elseif ( $rcode === 412 ) {
307  $status->fatal( 'backend-fail-contenttype', $params['dst'] );
308  } else {
309  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
310  }
311 
312  return SwiftFileOpHandle::CONTINUE_IF_OK;
313  };
314 
315  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
316  if ( !empty( $params['async'] ) ) { // deferred
317  $status->value = $opHandle;
318  } else { // actually write the object in Swift
319  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
320  }
321 
322  return $status;
323  }
324 
325  protected function doStoreInternal( array $params ) {
326  $status = $this->newStatus();
327 
328  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
329  if ( $dstRel === null ) {
330  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
331 
332  return $status;
333  }
334 
335  // Open a handle to the source file so that it can be streamed. The size and hash
336  // will be computed using the handle. In the off chance that the source file changes
337  // during this operation, the PUT will fail due to an ETag mismatch and be aborted.
338  AtEase::suppressWarnings();
339  $srcHandle = fopen( $params['src'], 'rb' );
340  AtEase::restoreWarnings();
341  if ( $srcHandle === false ) { // source doesn't exist?
342  $status->fatal( 'backend-fail-notexists', $params['src'] );
343 
344  return $status;
345  }
346 
347  // Compute the MD5 and SHA-1 hashes in one pass
348  $srcSize = fstat( $srcHandle )['size'];
349  $md5Context = hash_init( 'md5' );
350  $sha1Context = hash_init( 'sha1' );
351  $hashDigestSize = 0;
352  while ( !feof( $srcHandle ) ) {
353  $buffer = (string)fread( $srcHandle, 131072 ); // 128 KiB
354  hash_update( $md5Context, $buffer );
355  hash_update( $sha1Context, $buffer );
356  $hashDigestSize += strlen( $buffer );
357  }
358  // Reset the handle back to the beginning so that it can be streamed
359  rewind( $srcHandle );
360 
361  if ( $hashDigestSize !== $srcSize ) {
362  $status->fatal( 'backend-fail-hash', $params['src'] );
363 
364  return $status;
365  }
366 
367  // Headers that are not strictly a function of the file content
368  $mutableHeaders = $this->extractMutableContentHeaders( $params['headers'] ?? [] );
369  // Make sure that the "content-type" header is set to something sensible
370  $mutableHeaders['content-type'] = $mutableHeaders['content-type']
371  ?? $this->getContentType( $params['dst'], null, $params['src'] );
372 
373  $reqs = [ [
374  'method' => 'PUT',
375  'url' => [ $dstCont, $dstRel ],
376  'headers' => array_merge(
377  $mutableHeaders,
378  [
379  'content-length' => $srcSize,
380  'etag' => hash_final( $md5Context ),
381  'x-object-meta-sha1base36' =>
382  Wikimedia\base_convert( hash_final( $sha1Context ), 16, 36, 31 )
383  ]
384  ),
385  'body' => $srcHandle // resource
386  ] ];
387 
388  $method = __METHOD__;
389  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
390  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
391  if ( $rcode === 201 || $rcode === 202 ) {
392  // good
393  } elseif ( $rcode === 412 ) {
394  $status->fatal( 'backend-fail-contenttype', $params['dst'] );
395  } else {
396  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
397  }
398 
399  return SwiftFileOpHandle::CONTINUE_IF_OK;
400  };
401 
402  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
403  $opHandle->resourcesToClose[] = $srcHandle;
404 
405  if ( !empty( $params['async'] ) ) { // deferred
406  $status->value = $opHandle;
407  } else { // actually write the object in Swift
408  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
409  }
410 
411  return $status;
412  }
413 
414  protected function doCopyInternal( array $params ) {
415  $status = $this->newStatus();
416 
417  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
418  if ( $srcRel === null ) {
419  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
420 
421  return $status;
422  }
423 
424  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
425  if ( $dstRel === null ) {
426  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
427 
428  return $status;
429  }
430 
431  $reqs = [ [
432  'method' => 'PUT',
433  'url' => [ $dstCont, $dstRel ],
434  'headers' => array_merge(
435  $this->extractMutableContentHeaders( $params['headers'] ?? [] ),
436  [
437  'x-copy-from' => '/' . rawurlencode( $srcCont ) . '/' .
438  str_replace( "%2F", "/", rawurlencode( $srcRel ) )
439  ]
440  )
441  ] ];
442 
443  $method = __METHOD__;
444  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
445  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
446  if ( $rcode === 201 ) {
447  // good
448  } elseif ( $rcode === 404 ) {
449  if ( empty( $params['ignoreMissingSource'] ) ) {
450  $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
451  }
452  } else {
453  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
454  }
455 
456  return SwiftFileOpHandle::CONTINUE_IF_OK;
457  };
458 
459  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
460  if ( !empty( $params['async'] ) ) { // deferred
461  $status->value = $opHandle;
462  } else { // actually write the object in Swift
463  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
464  }
465 
466  return $status;
467  }
468 
469  protected function doMoveInternal( array $params ) {
470  $status = $this->newStatus();
471 
472  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
473  if ( $srcRel === null ) {
474  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
475 
476  return $status;
477  }
478 
479  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
480  if ( $dstRel === null ) {
481  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
482 
483  return $status;
484  }
485 
486  $reqs = [ [
487  'method' => 'PUT',
488  'url' => [ $dstCont, $dstRel ],
489  'headers' => array_merge(
490  $this->extractMutableContentHeaders( $params['headers'] ?? [] ),
491  [
492  'x-copy-from' => '/' . rawurlencode( $srcCont ) . '/' .
493  str_replace( "%2F", "/", rawurlencode( $srcRel ) )
494  ]
495  )
496  ] ];
497  if ( "{$srcCont}/{$srcRel}" !== "{$dstCont}/{$dstRel}" ) {
498  $reqs[] = [
499  'method' => 'DELETE',
500  'url' => [ $srcCont, $srcRel ],
501  'headers' => []
502  ];
503  }
504 
505  $method = __METHOD__;
506  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
507  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
508  if ( $request['method'] === 'PUT' && $rcode === 201 ) {
509  // good
510  } elseif ( $request['method'] === 'DELETE' && $rcode === 204 ) {
511  // good
512  } elseif ( $rcode === 404 ) {
513  if ( empty( $params['ignoreMissingSource'] ) ) {
514  $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
515  } else {
516  // Leave Status as OK but skip the DELETE request
517  return SwiftFileOpHandle::CONTINUE_NO;
518  }
519  } else {
520  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
521  }
522 
523  return SwiftFileOpHandle::CONTINUE_IF_OK;
524  };
525 
526  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
527  if ( !empty( $params['async'] ) ) { // deferred
528  $status->value = $opHandle;
529  } else { // actually move the object in Swift
530  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
531  }
532 
533  return $status;
534  }
535 
536  protected function doDeleteInternal( array $params ) {
537  $status = $this->newStatus();
538 
539  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
540  if ( $srcRel === null ) {
541  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
542 
543  return $status;
544  }
545 
546  $reqs = [ [
547  'method' => 'DELETE',
548  'url' => [ $srcCont, $srcRel ],
549  'headers' => []
550  ] ];
551 
552  $method = __METHOD__;
553  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
554  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
555  if ( $rcode === 204 ) {
556  // good
557  } elseif ( $rcode === 404 ) {
558  if ( empty( $params['ignoreMissingSource'] ) ) {
559  $status->fatal( 'backend-fail-delete', $params['src'] );
560  }
561  } else {
562  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
563  }
564 
565  return SwiftFileOpHandle::CONTINUE_IF_OK;
566  };
567 
568  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
569  if ( !empty( $params['async'] ) ) { // deferred
570  $status->value = $opHandle;
571  } else { // actually delete the object in Swift
572  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
573  }
574 
575  return $status;
576  }
577 
578  protected function doDescribeInternal( array $params ) {
579  $status = $this->newStatus();
580 
581  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
582  if ( $srcRel === null ) {
583  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
584 
585  return $status;
586  }
587 
588  // Fetch the old object headers/metadata...this should be in stat cache by now
589  $stat = $this->getFileStat( [ 'src' => $params['src'], 'latest' => 1 ] );
590  if ( $stat && !isset( $stat['xattr'] ) ) { // older cache entry
591  $stat = $this->doGetFileStat( [ 'src' => $params['src'], 'latest' => 1 ] );
592  }
593  if ( !$stat ) {
594  $status->fatal( 'backend-fail-describe', $params['src'] );
595 
596  return $status;
597  }
598 
599  // Swift object POST clears any prior headers, so merge the new and old headers here.
600  // Also, during, POST, libcurl adds "Content-Type: application/x-www-form-urlencoded"
601  // if "Content-Type" is not set, which would clobber the header value for the object.
602  $oldMetadataHeaders = [];
603  foreach ( $stat['xattr']['metadata'] as $name => $value ) {
604  $oldMetadataHeaders["x-object-meta-$name"] = $value;
605  }
606  $newContentHeaders = $this->extractMutableContentHeaders( $params['headers'] ?? [] );
607  $oldContentHeaders = $stat['xattr']['headers'];
608 
609  $reqs = [ [
610  'method' => 'POST',
611  'url' => [ $srcCont, $srcRel ],
612  'headers' => $oldMetadataHeaders + $newContentHeaders + $oldContentHeaders
613  ] ];
614 
615  $method = __METHOD__;
616  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
617  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
618  if ( $rcode === 202 ) {
619  // good
620  } elseif ( $rcode === 404 ) {
621  $status->fatal( 'backend-fail-describe', $params['src'] );
622  } else {
623  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
624  }
625  };
626 
627  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
628  if ( !empty( $params['async'] ) ) { // deferred
629  $status->value = $opHandle;
630  } else { // actually change the object in Swift
631  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
632  }
633 
634  return $status;
635  }
636 
637  protected function doPrepareInternal( $fullCont, $dir, array $params ) {
638  $status = $this->newStatus();
639 
640  // (a) Check if container already exists
641  $stat = $this->getContainerStat( $fullCont );
642  if ( is_array( $stat ) ) {
643  return $status; // already there
644  } elseif ( $stat === self::$RES_ERROR ) {
645  $status->fatal( 'backend-fail-internal', $this->name );
646  $this->logger->error( __METHOD__ . ': cannot get container stat' );
647 
648  return $status;
649  }
650 
651  // (b) Create container as needed with proper ACLs
652  if ( $stat === false ) {
653  $params['op'] = 'prepare';
654  $status->merge( $this->createContainer( $fullCont, $params ) );
655  }
656 
657  return $status;
658  }
659 
660  protected function doSecureInternal( $fullCont, $dir, array $params ) {
661  $status = $this->newStatus();
662  if ( empty( $params['noAccess'] ) ) {
663  return $status; // nothing to do
664  }
665 
666  $stat = $this->getContainerStat( $fullCont );
667  if ( is_array( $stat ) ) {
668  $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
669  $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
670  // Make container private to end-users...
671  $status->merge( $this->setContainerAccess(
672  $fullCont,
673  $readUsers,
675  ) );
676  } elseif ( $stat === false ) {
677  $status->fatal( 'backend-fail-usable', $params['dir'] );
678  } else {
679  $status->fatal( 'backend-fail-internal', $this->name );
680  $this->logger->error( __METHOD__ . ': cannot get container stat' );
681  }
682 
683  return $status;
684  }
685 
686  protected function doPublishInternal( $fullCont, $dir, array $params ) {
687  $status = $this->newStatus();
688 
689  $stat = $this->getContainerStat( $fullCont );
690  if ( is_array( $stat ) ) {
691  $readUsers = array_merge( $this->readUsers, [ $this->swiftUser, '.r:*' ] );
692  $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
693 
694  // Make container public to end-users...
695  $status->merge( $this->setContainerAccess(
696  $fullCont,
697  $readUsers,
699  ) );
700  } elseif ( $stat === false ) {
701  $status->fatal( 'backend-fail-usable', $params['dir'] );
702  } else {
703  $status->fatal( 'backend-fail-internal', $this->name );
704  $this->logger->error( __METHOD__ . ': cannot get container stat' );
705  }
706 
707  return $status;
708  }
709 
710  protected function doCleanInternal( $fullCont, $dir, array $params ) {
711  $status = $this->newStatus();
712 
713  // Only containers themselves can be removed, all else is virtual
714  if ( $dir != '' ) {
715  return $status; // nothing to do
716  }
717 
718  // (a) Check the container
719  $stat = $this->getContainerStat( $fullCont, true );
720  if ( $stat === false ) {
721  return $status; // ok, nothing to do
722  } elseif ( !is_array( $stat ) ) {
723  $status->fatal( 'backend-fail-internal', $this->name );
724  $this->logger->error( __METHOD__ . ': cannot get container stat' );
725 
726  return $status;
727  }
728 
729  // (b) Delete the container if empty
730  if ( $stat['count'] == 0 ) {
731  $params['op'] = 'clean';
732  $status->merge( $this->deleteContainer( $fullCont, $params ) );
733  }
734 
735  return $status;
736  }
737 
738  protected function doGetFileStat( array $params ) {
739  $params = [ 'srcs' => [ $params['src'] ], 'concurrency' => 1 ] + $params;
740  unset( $params['src'] );
741  $stats = $this->doGetFileStatMulti( $params );
742 
743  return reset( $stats );
744  }
745 
756  protected function convertSwiftDate( $ts, $format = TS_MW ) {
757  try {
758  $timestamp = new MWTimestamp( $ts );
759 
760  return $timestamp->getTimestamp( $format );
761  } catch ( Exception $e ) {
762  throw new FileBackendError( $e->getMessage() );
763  }
764  }
765 
773  protected function addMissingHashMetadata( array $objHdrs, $path ) {
774  if ( isset( $objHdrs['x-object-meta-sha1base36'] ) ) {
775  return $objHdrs; // nothing to do
776  }
777 
779  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
780  $this->logger->error( __METHOD__ . ": {path} was not stored with SHA-1 metadata.",
781  [ 'path' => $path ] );
782 
783  $objHdrs['x-object-meta-sha1base36'] = false;
784 
785  $auth = $this->getAuthentication();
786  if ( !$auth ) {
787  return $objHdrs; // failed
788  }
789 
790  // Find prior custom HTTP headers
791  $postHeaders = $this->extractMutableContentHeaders( $objHdrs );
792  // Find prior metadata headers
793  $postHeaders += $this->extractMetadataHeaders( $objHdrs );
794 
795  $status = $this->newStatus();
797  $scopeLockS = $this->getScopedFileLocks( [ $path ], LockManager::LOCK_UW, $status );
798  if ( $status->isOK() ) {
799  $tmpFile = $this->getLocalCopy( [ 'src' => $path, 'latest' => 1 ] );
800  if ( $tmpFile ) {
801  $hash = $tmpFile->getSha1Base36();
802  if ( $hash !== false ) {
803  $objHdrs['x-object-meta-sha1base36'] = $hash;
804  // Merge new SHA1 header into the old ones
805  $postHeaders['x-object-meta-sha1base36'] = $hash;
806  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
807  list( $rcode ) = $this->http->run( [
808  'method' => 'POST',
809  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
810  'headers' => $this->authTokenHeaders( $auth ) + $postHeaders
811  ] );
812  if ( $rcode >= 200 && $rcode <= 299 ) {
813  $this->deleteFileCache( $path );
814 
815  return $objHdrs; // success
816  }
817  }
818  }
819  }
820 
821  $this->logger->error( __METHOD__ . ': unable to set SHA-1 metadata for {path}',
822  [ 'path' => $path ] );
823 
824  return $objHdrs; // failed
825  }
826 
827  protected function doGetFileContentsMulti( array $params ) {
828  $auth = $this->getAuthentication();
829 
830  $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging
831  // Blindly create tmp files and stream to them, catching any exception
832  // if the file does not exist. Do not waste time doing file stats here.
833  $reqs = []; // (path => op)
834 
835  // Initial dummy values to preserve path order
836  $contents = array_fill_keys( $params['srcs'], self::$RES_ERROR );
837  foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch
838  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
839  if ( $srcRel === null || !$auth ) {
840  continue; // invalid storage path or auth error
841  }
842  // Create a new temporary memory file...
843  $handle = fopen( 'php://temp', 'wb' );
844  if ( $handle ) {
845  $reqs[$path] = [
846  'method' => 'GET',
847  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
848  'headers' => $this->authTokenHeaders( $auth )
849  + $this->headersFromParams( $params ),
850  'stream' => $handle,
851  ];
852  }
853  }
854 
855  $opts = [ 'maxConnsPerHost' => $params['concurrency'] ];
856  $reqs = $this->http->runMulti( $reqs, $opts );
857  foreach ( $reqs as $path => $op ) {
858  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response'];
859  if ( $rcode >= 200 && $rcode <= 299 ) {
860  rewind( $op['stream'] ); // start from the beginning
861  $content = (string)stream_get_contents( $op['stream'] );
862  $size = strlen( $content );
863  // Make sure that stream finished
864  if ( $size === (int)$rhdrs['content-length'] ) {
865  $contents[$path] = $content;
866  } else {
867  $contents[$path] = self::$RES_ERROR;
868  $rerr = "Got {$size}/{$rhdrs['content-length']} bytes";
869  $this->onError( null, __METHOD__,
870  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
871  }
872  } elseif ( $rcode === 404 ) {
873  $contents[$path] = self::$RES_ABSENT;
874  } else {
875  $contents[$path] = self::$RES_ERROR;
876  $this->onError( null, __METHOD__,
877  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
878  }
879  fclose( $op['stream'] ); // close open handle
880  }
881 
882  return $contents;
883  }
884 
885  protected function doDirectoryExists( $fullCont, $dir, array $params ) {
886  $prefix = ( $dir == '' ) ? null : "{$dir}/";
887  $status = $this->objectListing( $fullCont, 'names', 1, null, $prefix );
888  if ( $status->isOK() ) {
889  return ( count( $status->value ) ) > 0;
890  }
891 
892  return self::$RES_ERROR;
893  }
894 
902  public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
903  return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
904  }
905 
913  public function getFileListInternal( $fullCont, $dir, array $params ) {
914  return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
915  }
916 
928  public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
929  $dirs = [];
930  if ( $after === INF ) {
931  return $dirs; // nothing more
932  }
933 
935  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
936 
937  $prefix = ( $dir == '' ) ? null : "{$dir}/";
938  // Non-recursive: only list dirs right under $dir
939  if ( !empty( $params['topOnly'] ) ) {
940  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
941  if ( !$status->isOK() ) {
942  throw new FileBackendError( "Iterator page I/O error." );
943  }
944  $objects = $status->value;
945  // @phan-suppress-next-line PhanTypeSuspiciousNonTraversableForeach
946  foreach ( $objects as $object ) { // files and directories
947  if ( substr( $object, -1 ) === '/' ) {
948  $dirs[] = $object; // directories end in '/'
949  }
950  }
951  } else {
952  // Recursive: list all dirs under $dir and its subdirs
953  $getParentDir = static function ( $path ) {
954  return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
955  };
956 
957  // Get directory from last item of prior page
958  $lastDir = $getParentDir( $after ); // must be first page
959  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
960 
961  if ( !$status->isOK() ) {
962  throw new FileBackendError( "Iterator page I/O error." );
963  }
964 
965  $objects = $status->value;
966 
967  // @phan-suppress-next-line PhanTypeSuspiciousNonTraversableForeach
968  foreach ( $objects as $object ) { // files
969  $objectDir = $getParentDir( $object ); // directory of object
970 
971  if ( $objectDir !== false && $objectDir !== $dir ) {
972  // Swift stores paths in UTF-8, using binary sorting.
973  // See function "create_container_table" in common/db.py.
974  // If a directory is not "greater" than the last one,
975  // then it was already listed by the calling iterator.
976  if ( strcmp( $objectDir, $lastDir ) > 0 ) {
977  $pDir = $objectDir;
978  do { // add dir and all its parent dirs
979  $dirs[] = "{$pDir}/";
980  $pDir = $getParentDir( $pDir );
981  } while ( $pDir !== false // sanity
982  && strcmp( $pDir, $lastDir ) > 0 // not done already
983  && strlen( $pDir ) > strlen( $dir ) // within $dir
984  );
985  }
986  $lastDir = $objectDir;
987  }
988  }
989  }
990  // Page on the unfiltered directory listing (what is returned may be filtered)
991  if ( count( $objects ) < $limit ) {
992  $after = INF; // avoid a second RTT
993  } else {
994  $after = end( $objects ); // update last item
995  }
996 
997  return $dirs;
998  }
999 
1011  public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
1012  $files = []; // list of (path, stat array or null) entries
1013  if ( $after === INF ) {
1014  return $files; // nothing more
1015  }
1016 
1018  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1019 
1020  $prefix = ( $dir == '' ) ? null : "{$dir}/";
1021  // $objects will contain a list of unfiltered names or stdClass items
1022  // Non-recursive: only list files right under $dir
1023  if ( !empty( $params['topOnly'] ) ) {
1024  if ( !empty( $params['adviseStat'] ) ) {
1025  $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix, '/' );
1026  } else {
1027  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
1028  }
1029  } else {
1030  // Recursive: list all files under $dir and its subdirs
1031  if ( !empty( $params['adviseStat'] ) ) {
1032  $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix );
1033  } else {
1034  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
1035  }
1036  }
1037 
1038  // Reformat this list into a list of (name, stat array or null) entries
1039  if ( !$status->isOK() ) {
1040  throw new FileBackendError( "Iterator page I/O error." );
1041  }
1042 
1043  $objects = $status->value;
1044  $files = $this->buildFileObjectListing( $objects );
1045 
1046  // Page on the unfiltered object listing (what is returned may be filtered)
1047  if ( count( $objects ) < $limit ) {
1048  $after = INF; // avoid a second RTT
1049  } else {
1050  $after = end( $objects ); // update last item
1051  $after = is_object( $after ) ? $after->name : $after;
1052  }
1053 
1054  return $files;
1055  }
1056 
1064  private function buildFileObjectListing( array $objects ) {
1065  $names = [];
1066  foreach ( $objects as $object ) {
1067  if ( is_object( $object ) ) {
1068  if ( isset( $object->subdir ) || !isset( $object->name ) ) {
1069  continue; // virtual directory entry; ignore
1070  }
1071  $stat = [
1072  // Convert various random Swift dates to TS_MW
1073  'mtime' => $this->convertSwiftDate( $object->last_modified, TS_MW ),
1074  'size' => (int)$object->bytes,
1075  'sha1' => null,
1076  // Note: manifiest ETags are not an MD5 of the file
1077  'md5' => ctype_xdigit( $object->hash ) ? $object->hash : null,
1078  'latest' => false // eventually consistent
1079  ];
1080  $names[] = [ $object->name, $stat ];
1081  } elseif ( substr( $object, -1 ) !== '/' ) {
1082  // Omit directories, which end in '/' in listings
1083  $names[] = [ $object, null ];
1084  }
1085  }
1086 
1087  return $names;
1088  }
1089 
1096  public function loadListingStatInternal( $path, array $val ) {
1097  $this->cheapCache->setField( $path, 'stat', $val );
1098  }
1099 
1100  protected function doGetFileXAttributes( array $params ) {
1101  $stat = $this->getFileStat( $params );
1102  // Stat entries filled by file listings don't include metadata/headers
1103  if ( is_array( $stat ) && !isset( $stat['xattr'] ) ) {
1104  $this->clearCache( [ $params['src'] ] );
1105  $stat = $this->getFileStat( $params );
1106  }
1107 
1108  if ( is_array( $stat ) ) {
1109  return $stat['xattr'];
1110  }
1111 
1112  return ( $stat === self::$RES_ERROR ) ? self::$RES_ERROR : self::$RES_ABSENT;
1113  }
1114 
1115  protected function doGetFileSha1base36( array $params ) {
1116  // Avoid using stat entries from file listings, which never include the SHA-1 hash.
1117  // Also, recompute the hash if it's not part of the metadata headers for some reason.
1118  $params['requireSHA1'] = true;
1119 
1120  $stat = $this->getFileStat( $params );
1121  if ( is_array( $stat ) ) {
1122  return $stat['sha1'];
1123  }
1124 
1125  return ( $stat === self::$RES_ERROR ) ? self::$RES_ERROR : self::$RES_ABSENT;
1126  }
1127 
1128  protected function doStreamFile( array $params ) {
1129  $status = $this->newStatus();
1130 
1131  $flags = !empty( $params['headless'] ) ? HTTPFileStreamer::STREAM_HEADLESS : 0;
1132 
1133  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1134  if ( $srcRel === null ) {
1135  HTTPFileStreamer::send404Message( $params['src'], $flags );
1136  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
1137 
1138  return $status;
1139  }
1140 
1141  $auth = $this->getAuthentication();
1142  if ( !$auth || !is_array( $this->getContainerStat( $srcCont ) ) ) {
1143  HTTPFileStreamer::send404Message( $params['src'], $flags );
1144  $status->fatal( 'backend-fail-stream', $params['src'] );
1145 
1146  return $status;
1147  }
1148 
1149  // If "headers" is set, we only want to send them if the file is there.
1150  // Do not bother checking if the file exists if headers are not set though.
1151  if ( $params['headers'] && !$this->fileExists( $params ) ) {
1152  HTTPFileStreamer::send404Message( $params['src'], $flags );
1153  $status->fatal( 'backend-fail-stream', $params['src'] );
1154 
1155  return $status;
1156  }
1157 
1158  // Send the requested additional headers
1159  foreach ( $params['headers'] as $header ) {
1160  header( $header ); // aways send
1161  }
1162 
1163  if ( empty( $params['allowOB'] ) ) {
1164  // Cancel output buffering and gzipping if set
1165  ( $this->obResetFunc )();
1166  }
1167 
1168  $handle = fopen( 'php://output', 'wb' );
1169  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1170  'method' => 'GET',
1171  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1172  'headers' => $this->authTokenHeaders( $auth )
1173  + $this->headersFromParams( $params ) + $params['options'],
1174  'stream' => $handle,
1175  'flags' => [ 'relayResponseHeaders' => empty( $params['headless'] ) ]
1176  ] );
1177 
1178  if ( $rcode >= 200 && $rcode <= 299 ) {
1179  // good
1180  } elseif ( $rcode === 404 ) {
1181  $status->fatal( 'backend-fail-stream', $params['src'] );
1182  // Per T43113, nasty things can happen if bad cache entries get
1183  // stuck in cache. It's also possible that this error can come up
1184  // with simple race conditions. Clear out the stat cache to be safe.
1185  $this->clearCache( [ $params['src'] ] );
1186  $this->deleteFileCache( $params['src'] );
1187  } else {
1188  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1189  }
1190 
1191  return $status;
1192  }
1193 
1194  protected function doGetLocalCopyMulti( array $params ) {
1195  $auth = $this->getAuthentication();
1196 
1197  $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging
1198  // Blindly create tmp files and stream to them, catching any exception
1199  // if the file does not exist. Do not waste time doing file stats here.
1200  $reqs = []; // (path => op)
1201 
1202  // Initial dummy values to preserve path order
1203  $tmpFiles = array_fill_keys( $params['srcs'], self::$RES_ERROR );
1204  foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch
1205  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
1206  if ( $srcRel === null || !$auth ) {
1207  continue; // invalid storage path or auth error
1208  }
1209  // Get source file extension
1211  // Create a new temporary file...
1212  $tmpFile = $this->tmpFileFactory->newTempFSFile( 'localcopy_', $ext );
1213  $handle = $tmpFile ? fopen( $tmpFile->getPath(), 'wb' ) : false;
1214  if ( $handle ) {
1215  $reqs[$path] = [
1216  'method' => 'GET',
1217  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1218  'headers' => $this->authTokenHeaders( $auth )
1219  + $this->headersFromParams( $params ),
1220  'stream' => $handle,
1221  ];
1222  $tmpFiles[$path] = $tmpFile;
1223  }
1224  }
1225 
1226  // Ceph RADOS Gateway is in use (strong consistency) or X-Newest will be used
1227  $latest = ( $this->isRGW || !empty( $params['latest'] ) );
1228 
1229  $opts = [ 'maxConnsPerHost' => $params['concurrency'] ];
1230  $reqs = $this->http->runMulti( $reqs, $opts );
1231  foreach ( $reqs as $path => $op ) {
1232  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response'];
1233  fclose( $op['stream'] ); // close open handle
1234  if ( $rcode >= 200 && $rcode <= 299 ) {
1236  $tmpFile = $tmpFiles[$path];
1237  // Make sure that the stream finished and fully wrote to disk
1238  $size = $tmpFile->getSize();
1239  if ( $size !== (int)$rhdrs['content-length'] ) {
1240  $tmpFiles[$path] = self::$RES_ERROR;
1241  $rerr = "Got {$size}/{$rhdrs['content-length']} bytes";
1242  $this->onError( null, __METHOD__,
1243  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
1244  }
1245  // Set the file stat process cache in passing
1246  $stat = $this->getStatFromHeaders( $rhdrs );
1247  $stat['latest'] = $latest;
1248  $this->cheapCache->setField( $path, 'stat', $stat );
1249  } elseif ( $rcode === 404 ) {
1250  $tmpFiles[$path] = self::$RES_ABSENT;
1251  $this->cheapCache->setField(
1252  $path,
1253  'stat',
1254  $latest ? self::$ABSENT_LATEST : self::$ABSENT_NORMAL
1255  );
1256  } else {
1257  $tmpFiles[$path] = self::$RES_ERROR;
1258  $this->onError( null, __METHOD__,
1259  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
1260  }
1261  }
1262 
1263  return $tmpFiles;
1264  }
1265 
1266  public function getFileHttpUrl( array $params ) {
1267  if ( $this->swiftTempUrlKey != '' ||
1268  ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' )
1269  ) {
1270  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1271  if ( $srcRel === null ) {
1272  return self::TEMPURL_ERROR; // invalid path
1273  }
1274 
1275  $auth = $this->getAuthentication();
1276  if ( !$auth ) {
1277  return self::TEMPURL_ERROR;
1278  }
1279 
1280  $ttl = $params['ttl'] ?? 86400;
1281  $expires = time() + $ttl;
1282 
1283  if ( $this->swiftTempUrlKey != '' ) {
1284  $url = $this->storageUrl( $auth, $srcCont, $srcRel );
1285  // Swift wants the signature based on the unencoded object name
1286  $contPath = parse_url( $this->storageUrl( $auth, $srcCont ), PHP_URL_PATH );
1287  $signature = hash_hmac( 'sha1',
1288  "GET\n{$expires}\n{$contPath}/{$srcRel}",
1289  $this->swiftTempUrlKey
1290  );
1291 
1292  return "{$url}?temp_url_sig={$signature}&temp_url_expires={$expires}";
1293  } else { // give S3 API URL for rgw
1294  // Path for signature starts with the bucket
1295  $spath = '/' . rawurlencode( $srcCont ) . '/' .
1296  str_replace( '%2F', '/', rawurlencode( $srcRel ) );
1297  // Calculate the hash
1298  $signature = base64_encode( hash_hmac(
1299  'sha1',
1300  "GET\n\n\n{$expires}\n{$spath}",
1301  $this->rgwS3SecretKey,
1302  true // raw
1303  ) );
1304  // See https://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html.
1305  // Note: adding a newline for empty CanonicalizedAmzHeaders does not work.
1306  // Note: S3 API is the rgw default; remove the /swift/ URL bit.
1307  return str_replace( '/swift/v1', '', $this->storageUrl( $auth ) . $spath ) .
1308  '?' .
1309  http_build_query( [
1310  'Signature' => $signature,
1311  'Expires' => $expires,
1312  'AWSAccessKeyId' => $this->rgwS3AccessKey
1313  ] );
1314  }
1315  }
1316 
1317  return self::TEMPURL_ERROR;
1318  }
1319 
1320  protected function directoriesAreVirtual() {
1321  return true;
1322  }
1323 
1332  protected function headersFromParams( array $params ) {
1333  $hdrs = [];
1334  if ( !empty( $params['latest'] ) ) {
1335  $hdrs['x-newest'] = 'true';
1336  }
1337 
1338  return $hdrs;
1339  }
1340 
1341  protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1343  '@phan-var SwiftFileOpHandle[] $fileOpHandles';
1344 
1346  $statuses = [];
1347 
1348  $auth = $this->getAuthentication();
1349  if ( !$auth ) {
1350  foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1351  $statuses[$index] = $this->newStatus( 'backend-fail-connect', $this->name );
1352  }
1353 
1354  return $statuses;
1355  }
1356 
1357  // Split the HTTP requests into stages that can be done concurrently
1358  $httpReqsByStage = []; // map of (stage => index => HTTP request)
1359  foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1360  $reqs = $fileOpHandle->httpOp;
1361  // Convert the 'url' parameter to an actual URL using $auth
1362  foreach ( $reqs as $stage => &$req ) {
1363  list( $container, $relPath ) = $req['url'];
1364  $req['url'] = $this->storageUrl( $auth, $container, $relPath );
1365  $req['headers'] = $req['headers'] ?? [];
1366  $req['headers'] = $this->authTokenHeaders( $auth ) + $req['headers'];
1367  $httpReqsByStage[$stage][$index] = $req;
1368  }
1369  $statuses[$index] = $this->newStatus();
1370  }
1371 
1372  // Run all requests for the first stage, then the next, and so on
1373  $reqCount = count( $httpReqsByStage );
1374  for ( $stage = 0; $stage < $reqCount; ++$stage ) {
1375  $httpReqs = $this->http->runMulti( $httpReqsByStage[$stage] );
1376  foreach ( $httpReqs as $index => $httpReq ) {
1378  $fileOpHandle = $fileOpHandles[$index];
1379  // Run the callback for each request of this operation
1380  $status = $statuses[$index];
1381  ( $fileOpHandle->callback )( $httpReq, $status );
1382  // On failure, abort all remaining requests for this operation. This is used
1383  // in "move" operations to abort the DELETE request if the PUT request fails.
1384  if (
1385  !$status->isOK() ||
1386  $fileOpHandle->state === $fileOpHandle::CONTINUE_NO
1387  ) {
1388  $stages = count( $fileOpHandle->httpOp );
1389  for ( $s = ( $stage + 1 ); $s < $stages; ++$s ) {
1390  unset( $httpReqsByStage[$s][$index] );
1391  }
1392  }
1393  }
1394  }
1395 
1396  return $statuses;
1397  }
1398 
1421  protected function setContainerAccess( $container, array $readUsers, array $writeUsers ) {
1422  $status = $this->newStatus();
1423  $auth = $this->getAuthentication();
1424 
1425  if ( !$auth ) {
1426  $status->fatal( 'backend-fail-connect', $this->name );
1427 
1428  return $status;
1429  }
1430 
1431  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1432  'method' => 'POST',
1433  'url' => $this->storageUrl( $auth, $container ),
1434  'headers' => $this->authTokenHeaders( $auth ) + [
1435  'x-container-read' => implode( ',', $readUsers ),
1436  'x-container-write' => implode( ',', $writeUsers )
1437  ]
1438  ] );
1439 
1440  if ( $rcode != 204 && $rcode !== 202 ) {
1441  $status->fatal( 'backend-fail-internal', $this->name );
1442  $this->logger->error( __METHOD__ . ': unexpected rcode value ({rcode})',
1443  [ 'rcode' => $rcode ] );
1444  }
1445 
1446  return $status;
1447  }
1448 
1457  protected function getContainerStat( $container, $bypassCache = false ) {
1459  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1460 
1461  if ( $bypassCache ) { // purge cache
1462  $this->containerStatCache->clear( $container );
1463  } elseif ( !$this->containerStatCache->hasField( $container, 'stat' ) ) {
1464  $this->primeContainerCache( [ $container ] ); // check persistent cache
1465  }
1466  if ( !$this->containerStatCache->hasField( $container, 'stat' ) ) {
1467  $auth = $this->getAuthentication();
1468  if ( !$auth ) {
1469  return self::$RES_ERROR;
1470  }
1471 
1472  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1473  'method' => 'HEAD',
1474  'url' => $this->storageUrl( $auth, $container ),
1475  'headers' => $this->authTokenHeaders( $auth )
1476  ] );
1477 
1478  if ( $rcode === 204 ) {
1479  $stat = [
1480  'count' => $rhdrs['x-container-object-count'],
1481  'bytes' => $rhdrs['x-container-bytes-used']
1482  ];
1483  if ( $bypassCache ) {
1484  return $stat;
1485  } else {
1486  $this->containerStatCache->setField( $container, 'stat', $stat ); // cache it
1487  $this->setContainerCache( $container, $stat ); // update persistent cache
1488  }
1489  } elseif ( $rcode === 404 ) {
1490  return self::$RES_ABSENT;
1491  } else {
1492  $this->onError( null, __METHOD__,
1493  [ 'cont' => $container ], $rerr, $rcode, $rdesc );
1494 
1495  return self::$RES_ERROR;
1496  }
1497  }
1498 
1499  return $this->containerStatCache->getField( $container, 'stat' );
1500  }
1501 
1509  protected function createContainer( $container, array $params ) {
1510  $status = $this->newStatus();
1511 
1512  $auth = $this->getAuthentication();
1513  if ( !$auth ) {
1514  $status->fatal( 'backend-fail-connect', $this->name );
1515 
1516  return $status;
1517  }
1518 
1519  // @see SwiftFileBackend::setContainerAccess()
1520  if ( empty( $params['noAccess'] ) ) {
1521  // public
1522  $readUsers = array_merge( $this->readUsers, [ '.r:*', $this->swiftUser ] );
1523  $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
1524  } else {
1525  // private
1526  $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
1527  $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
1528  }
1529 
1530  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1531  'method' => 'PUT',
1532  'url' => $this->storageUrl( $auth, $container ),
1533  'headers' => $this->authTokenHeaders( $auth ) + [
1534  'x-container-read' => implode( ',', $readUsers ),
1535  'x-container-write' => implode( ',', $writeUsers )
1536  ]
1537  ] );
1538 
1539  if ( $rcode === 201 ) { // new
1540  // good
1541  } elseif ( $rcode === 202 ) { // already there
1542  // this shouldn't really happen, but is OK
1543  } else {
1544  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1545  }
1546 
1547  return $status;
1548  }
1549 
1557  protected function deleteContainer( $container, array $params ) {
1558  $status = $this->newStatus();
1559 
1560  $auth = $this->getAuthentication();
1561  if ( !$auth ) {
1562  $status->fatal( 'backend-fail-connect', $this->name );
1563 
1564  return $status;
1565  }
1566 
1567  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1568  'method' => 'DELETE',
1569  'url' => $this->storageUrl( $auth, $container ),
1570  'headers' => $this->authTokenHeaders( $auth )
1571  ] );
1572 
1573  if ( $rcode >= 200 && $rcode <= 299 ) { // deleted
1574  $this->containerStatCache->clear( $container ); // purge
1575  } elseif ( $rcode === 404 ) { // not there
1576  // this shouldn't really happen, but is OK
1577  } elseif ( $rcode === 409 ) { // not empty
1578  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); // race?
1579  } else {
1580  // @phan-suppress-previous-line PhanPluginDuplicateIfStatements
1581  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1582  }
1583 
1584  return $status;
1585  }
1586 
1599  private function objectListing(
1600  $fullCont, $type, $limit, $after = null, $prefix = null, $delim = null
1601  ) {
1602  $status = $this->newStatus();
1603 
1604  $auth = $this->getAuthentication();
1605  if ( !$auth ) {
1606  $status->fatal( 'backend-fail-connect', $this->name );
1607 
1608  return $status;
1609  }
1610 
1611  $query = [ 'limit' => $limit ];
1612  if ( $type === 'info' ) {
1613  $query['format'] = 'json';
1614  }
1615  if ( $after !== null ) {
1616  $query['marker'] = $after;
1617  }
1618  if ( $prefix !== null ) {
1619  $query['prefix'] = $prefix;
1620  }
1621  if ( $delim !== null ) {
1622  $query['delimiter'] = $delim;
1623  }
1624 
1625  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1626  'method' => 'GET',
1627  'url' => $this->storageUrl( $auth, $fullCont ),
1628  'query' => $query,
1629  'headers' => $this->authTokenHeaders( $auth )
1630  ] );
1631 
1632  $params = [ 'cont' => $fullCont, 'prefix' => $prefix, 'delim' => $delim ];
1633  if ( $rcode === 200 ) { // good
1634  if ( $type === 'info' ) {
1635  $status->value = FormatJson::decode( trim( $rbody ) );
1636  } else {
1637  $status->value = explode( "\n", trim( $rbody ) );
1638  }
1639  } elseif ( $rcode === 204 ) {
1640  $status->value = []; // empty container
1641  } elseif ( $rcode === 404 ) {
1642  $status->value = []; // no container
1643  } else {
1644  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1645  }
1646 
1647  return $status;
1648  }
1649 
1650  protected function doPrimeContainerCache( array $containerInfo ) {
1651  foreach ( $containerInfo as $container => $info ) {
1652  $this->containerStatCache->setField( $container, 'stat', $info );
1653  }
1654  }
1655 
1656  protected function doGetFileStatMulti( array $params ) {
1657  $stats = [];
1658 
1659  $auth = $this->getAuthentication();
1660 
1661  $reqs = []; // (path => op)
1662  // (a) Check the containers of the paths...
1663  foreach ( $params['srcs'] as $path ) {
1664  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
1665  if ( $srcRel === null || !$auth ) {
1666  $stats[$path] = self::$RES_ERROR;
1667  continue; // invalid storage path or auth error
1668  }
1669 
1670  $cstat = $this->getContainerStat( $srcCont );
1671  if ( $cstat === self::$RES_ABSENT ) {
1672  $stats[$path] = self::$RES_ABSENT;
1673  continue; // ok, nothing to do
1674  } elseif ( !is_array( $cstat ) ) {
1675  $stats[$path] = self::$RES_ERROR;
1676  continue;
1677  }
1678 
1679  $reqs[$path] = [
1680  'method' => 'HEAD',
1681  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1682  'headers' => $this->authTokenHeaders( $auth ) + $this->headersFromParams( $params )
1683  ];
1684  }
1685 
1686  // (b) Check the files themselves...
1687  $opts = [ 'maxConnsPerHost' => $params['concurrency'] ];
1688  $reqs = $this->http->runMulti( $reqs, $opts );
1689  foreach ( $reqs as $path => $op ) {
1690  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response'];
1691  if ( $rcode === 200 || $rcode === 204 ) {
1692  // Update the object if it is missing some headers
1693  if ( !empty( $params['requireSHA1'] ) ) {
1694  $rhdrs = $this->addMissingHashMetadata( $rhdrs, $path );
1695  }
1696  // Load the stat array from the headers
1697  $stat = $this->getStatFromHeaders( $rhdrs );
1698  if ( $this->isRGW ) {
1699  $stat['latest'] = true; // strong consistency
1700  }
1701  } elseif ( $rcode === 404 ) {
1702  $stat = self::$RES_ABSENT;
1703  } else {
1704  $stat = self::$RES_ERROR;
1705  $this->onError( null, __METHOD__, $params, $rerr, $rcode, $rdesc );
1706  }
1707  $stats[$path] = $stat;
1708  }
1709 
1710  return $stats;
1711  }
1712 
1717  protected function getStatFromHeaders( array $rhdrs ) {
1718  // Fetch all of the custom metadata headers
1719  $metadata = $this->getMetadataFromHeaders( $rhdrs );
1720  // Fetch all of the custom raw HTTP headers
1721  $headers = $this->extractMutableContentHeaders( $rhdrs );
1722 
1723  return [
1724  // Convert various random Swift dates to TS_MW
1725  'mtime' => $this->convertSwiftDate( $rhdrs['last-modified'], TS_MW ),
1726  // Empty objects actually return no content-length header in Ceph
1727  'size' => isset( $rhdrs['content-length'] ) ? (int)$rhdrs['content-length'] : 0,
1728  'sha1' => $metadata['sha1base36'] ?? null,
1729  // Note: manifiest ETags are not an MD5 of the file
1730  'md5' => ctype_xdigit( $rhdrs['etag'] ) ? $rhdrs['etag'] : null,
1731  'xattr' => [ 'metadata' => $metadata, 'headers' => $headers ]
1732  ];
1733  }
1734 
1738  protected function getAuthentication() {
1739  if ( $this->authErrorTimestamp !== null ) {
1740  if ( ( time() - $this->authErrorTimestamp ) < 60 ) {
1741  return null; // failed last attempt; don't bother
1742  } else { // actually retry this time
1743  $this->authErrorTimestamp = null;
1744  }
1745  }
1746  // Session keys expire after a while, so we renew them periodically
1747  $reAuth = ( ( time() - $this->authSessionTimestamp ) > $this->authTTL );
1748  // Authenticate with proxy and get a session key...
1749  if ( !$this->authCreds || $reAuth ) {
1750  $this->authSessionTimestamp = 0;
1751  $cacheKey = $this->getCredsCacheKey( $this->swiftUser );
1752  $creds = $this->srvCache->get( $cacheKey ); // credentials
1753  // Try to use the credential cache
1754  if ( isset( $creds['auth_token'] ) && isset( $creds['storage_url'] ) ) {
1755  $this->authCreds = $creds;
1756  // Skew the timestamp for worst case to avoid using stale credentials
1757  $this->authSessionTimestamp = time() - ceil( $this->authTTL / 2 );
1758  } else { // cache miss
1759  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1760  'method' => 'GET',
1761  'url' => "{$this->swiftAuthUrl}/v1.0",
1762  'headers' => [
1763  'x-auth-user' => $this->swiftUser,
1764  'x-auth-key' => $this->swiftKey
1765  ]
1766  ] );
1767 
1768  if ( $rcode >= 200 && $rcode <= 299 ) { // OK
1769  $this->authCreds = [
1770  'auth_token' => $rhdrs['x-auth-token'],
1771  'storage_url' => $this->swiftStorageUrl ?? $rhdrs['x-storage-url']
1772  ];
1773 
1774  $this->srvCache->set( $cacheKey, $this->authCreds, ceil( $this->authTTL / 2 ) );
1775  $this->authSessionTimestamp = time();
1776  } elseif ( $rcode === 401 ) {
1777  $this->onError( null, __METHOD__, [], "Authentication failed.", $rcode );
1778  $this->authErrorTimestamp = time();
1779 
1780  return null;
1781  } else {
1782  $this->onError( null, __METHOD__, [], "HTTP return code: $rcode", $rcode );
1783  $this->authErrorTimestamp = time();
1784 
1785  return null;
1786  }
1787  }
1788  // Ceph RGW does not use <account> in URLs (OpenStack Swift uses "/v1/<account>")
1789  if ( substr( $this->authCreds['storage_url'], -3 ) === '/v1' ) {
1790  $this->isRGW = true; // take advantage of strong consistency in Ceph
1791  }
1792  }
1793 
1794  return $this->authCreds;
1795  }
1796 
1803  protected function storageUrl( array $creds, $container = null, $object = null ) {
1804  $parts = [ $creds['storage_url'] ];
1805  if ( strlen( $container ) ) {
1806  $parts[] = rawurlencode( $container );
1807  }
1808  if ( strlen( $object ) ) {
1809  $parts[] = str_replace( "%2F", "/", rawurlencode( $object ) );
1810  }
1811 
1812  return implode( '/', $parts );
1813  }
1814 
1819  protected function authTokenHeaders( array $creds ) {
1820  return [ 'x-auth-token' => $creds['auth_token'] ];
1821  }
1822 
1829  private function getCredsCacheKey( $username ) {
1830  return 'swiftcredentials:' . md5( $username . ':' . $this->swiftAuthUrl );
1831  }
1832 
1844  public function onError( $status, $func, array $params, $err = '', $code = 0, $desc = '' ) {
1845  if ( $status instanceof StatusValue ) {
1846  $status->fatal( 'backend-fail-internal', $this->name );
1847  }
1848  if ( $code == 401 ) { // possibly a stale token
1849  $this->srvCache->delete( $this->getCredsCacheKey( $this->swiftUser ) );
1850  }
1851  $msg = "HTTP {code} ({desc}) in '{func}' (given '{req_params}')";
1852  $msgParams = [
1853  'code' => $code,
1854  'desc' => $desc,
1855  'func' => $func,
1856  'req_params' => FormatJson::encode( $params ),
1857  ];
1858  if ( $err ) {
1859  $msg .= ': {err}';
1860  $msgParams['err'] = $err;
1861  }
1862  $this->logger->error( $msg, $msgParams );
1863  }
1864 }
SwiftFileOpHandle
Definition: SwiftFileOpHandle.php:25
MWTimestamp
Library for creating and parsing MW-style timestamps.
Definition: MWTimestamp.php:38
SwiftFileBackend\doPrepareInternal
doPrepareInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:637
SwiftFileBackend\$isRGW
bool $isRGW
Whether the server is an Ceph RGW.
Definition: SwiftFileBackend.php:79
MultiHttpClient
Class to handle multiple HTTP requests.
Definition: MultiHttpClient.php:55
StatusValue
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Definition: StatusValue.php:43
SwiftFileBackend\isPathUsableInternal
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path in the backend.
Definition: SwiftFileBackend.php:184
SwiftFileBackend\$authErrorTimestamp
int $authErrorTimestamp
UNIX timestamp.
Definition: SwiftFileBackend.php:76
SwiftFileBackend\getAuthentication
getAuthentication()
Definition: SwiftFileBackend.php:1738
SwiftFileBackend\getContainerStat
getContainerStat( $container, $bypassCache=false)
Get a Swift container stat array, possibly from process cache.
Definition: SwiftFileBackend.php:1457
SwiftFileBackend\buildFileObjectListing
buildFileObjectListing(array $objects)
Build a list of file objects, filtering out any directories and extracting any stat info if provided ...
Definition: SwiftFileBackend.php:1064
EmptyBagOStuff
A BagOStuff object with no objects in it.
Definition: EmptyBagOStuff.php:29
SwiftFileBackend\$containerStatCache
MapCacheLRU $containerStatCache
Container stat cache.
Definition: SwiftFileBackend.php:69
SwiftFileBackend\$swiftKey
string $swiftKey
Secret key for user.
Definition: SwiftFileBackend.php:49
SwiftFileBackend\getStatFromHeaders
getStatFromHeaders(array $rhdrs)
Definition: SwiftFileBackend.php:1717
SwiftFileBackend\addMissingHashMetadata
addMissingHashMetadata(array $objHdrs, $path)
Fill in any missing object metadata and save it to Swift.
Definition: SwiftFileBackend.php:773
LockManager\LOCK_UW
const LOCK_UW
Definition: LockManager.php:70
SwiftFileBackend\doGetFileXAttributes
doGetFileXAttributes(array $params)
Definition: SwiftFileBackend.php:1100
FileBackendError
File backend exception for checked exceptions (e.g.
Definition: FileBackendError.php:10
FileBackend\getScopedFileLocks
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
Definition: FileBackend.php:1433
FileBackendStore\fileExists
fileExists(array $params)
Check if a file exists at a storage path in the backend.
Definition: FileBackendStore.php:644
SwiftFileBackend\$swiftUser
string $swiftUser
Swift user (account:user) to authenticate as.
Definition: SwiftFileBackend.php:47
SwiftFileBackend\loadListingStatInternal
loadListingStatInternal( $path, array $val)
Do not call this function outside of SwiftFileBackendFileList.
Definition: SwiftFileBackend.php:1096
FileBackend\extensionFromPath
static extensionFromPath( $path, $case='lowercase')
Get the final extension from a storage or FS path.
Definition: FileBackend.php:1599
FileBackendStore\deleteFileCache
deleteFileCache( $path)
Delete the cached stat info for a file path.
Definition: FileBackendStore.php:1910
SwiftFileBackend\resolveContainerPath
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
Definition: SwiftFileBackend.php:174
FileBackendStore\executeOpHandlesInternal
executeOpHandlesInternal(array $fileOpHandles)
Execute a list of FileBackendStoreOpHandle handles in parallel.
Definition: FileBackendStore.php:1402
BagOStuff
Class representing a cache/ephemeral data store.
Definition: BagOStuff.php:86
SwiftFileBackend\doGetFileContentsMulti
doGetFileContentsMulti(array $params)
Definition: SwiftFileBackend.php:827
SwiftFileBackend\createContainer
createContainer( $container, array $params)
Create a Swift container.
Definition: SwiftFileBackend.php:1509
SwiftFileBackend\extractMetadataHeaders
extractMetadataHeaders(array $headers)
Definition: SwiftFileBackend.php:242
SwiftFileBackend\getFileListPageInternal
getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
Definition: SwiftFileBackend.php:1011
SwiftFileBackend\getMetadataFromHeaders
getMetadataFromHeaders(array $headers)
Definition: SwiftFileBackend.php:259
SwiftFileBackend\doGetFileSha1base36
doGetFileSha1base36(array $params)
Definition: SwiftFileBackend.php:1115
FileBackendStore\clearCache
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
Definition: FileBackendStore.php:1484
SwiftFileBackend\getFeatures
getFeatures()
Get the a bitfield of extra features supported by the backend medium.
Definition: SwiftFileBackend.php:166
FileBackendStore\$RES_ERROR
static null $RES_ERROR
Idiom for "no result due to I/O errors" (since 1.34)
Definition: FileBackendStore.php:66
SwiftFileBackend\doDeleteInternal
doDeleteInternal(array $params)
Definition: SwiftFileBackend.php:536
SwiftFileBackend\setLogger
setLogger(LoggerInterface $logger)
Definition: SwiftFileBackend.php:161
SwiftFileBackend\headersFromParams
headersFromParams(array $params)
Get headers to send to Swift when reading a file based on a FileBackend params array,...
Definition: SwiftFileBackend.php:1332
SwiftFileBackend\deleteContainer
deleteContainer( $container, array $params)
Delete a Swift container.
Definition: SwiftFileBackend.php:1557
SwiftFileBackend\getCredsCacheKey
getCredsCacheKey( $username)
Get the cache key for a container.
Definition: SwiftFileBackend.php:1829
FormatJson\decode
static decode( $value, $assoc=false)
Decodes a JSON string.
Definition: FormatJson.php:146
FormatJson\encode
static encode( $value, $pretty=false, $escaping=0)
Returns the JSON representation of a value.
Definition: FormatJson.php:96
SwiftFileBackend\doDirectoryExists
doDirectoryExists( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:885
SwiftFileBackend
Class for an OpenStack Swift (or Ceph RGW) based file backend.
Definition: SwiftFileBackend.php:37
SwiftFileBackend\getDirectoryListInternal
getDirectoryListInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:902
SwiftFileBackend\setContainerAccess
setContainerAccess( $container, array $readUsers, array $writeUsers)
Set read/write permissions for a Swift container.
Definition: SwiftFileBackend.php:1421
SwiftFileBackend\getFileHttpUrl
getFileHttpUrl(array $params)
Definition: SwiftFileBackend.php:1266
SwiftFileBackend\doMoveInternal
doMoveInternal(array $params)
Definition: SwiftFileBackend.php:469
MapCacheLRU
Handles a simple LRU key/value map with a maximum number of entries.
Definition: MapCacheLRU.php:36
SwiftFileBackend\$authCreds
array $authCreds
Definition: SwiftFileBackend.php:72
$dirs
$dirs
Definition: mergeMessageFileList.php:213
SwiftFileBackend\extractMutableContentHeaders
extractMutableContentHeaders(array $headers)
Filter/normalize a header map to only include mutable "content-"/"x-content-" headers.
Definition: SwiftFileBackend.php:202
SwiftFileBackend\doGetLocalCopyMulti
doGetLocalCopyMulti(array $params)
Definition: SwiftFileBackend.php:1194
SwiftFileBackend\onError
onError( $status, $func, array $params, $err='', $code=0, $desc='')
Log an unexpected exception for this backend.
Definition: SwiftFileBackend.php:1844
SwiftFileBackend\$secureWriteUsers
array $secureWriteUsers
Additional users (account:user) with write permissions on private containers.
Definition: SwiftFileBackend.php:63
SwiftFileBackend\doDescribeInternal
doDescribeInternal(array $params)
Definition: SwiftFileBackend.php:578
SwiftFileBackend\storageUrl
storageUrl(array $creds, $container=null, $object=null)
Definition: SwiftFileBackend.php:1803
FileBackendStore\getContentType
getContentType( $storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
Definition: FileBackendStore.php:2032
FileBackend\$obResetFunc
callable $obResetFunc
Definition: FileBackend.php:127
SwiftFileBackendDirList
Iterator for listing directories.
Definition: SwiftFileBackendDirList.php:28
$content
$content
Definition: router.php:76
SwiftFileBackend\doSecureInternal
doSecureInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:660
$s
foreach( $mmfl['setupFiles'] as $fileName) if( $queue) if(empty( $mmfl['quiet'])) $s
Definition: mergeMessageFileList.php:206
FileBackendStore\getFileStat
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
Definition: FileBackendStore.php:680
$header
$header
Definition: updateCredits.php:37
SwiftFileBackend\__construct
__construct(array $config)
Definition: SwiftFileBackend.php:119
SwiftFileBackend\directoriesAreVirtual
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
Definition: SwiftFileBackend.php:1320
FileBackendStore\resolveStoragePathReal
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
Definition: FileBackendStore.php:1642
SwiftFileBackend\authTokenHeaders
authTokenHeaders(array $creds)
Definition: SwiftFileBackend.php:1819
WANObjectCache
Multi-datacenter aware caching interface.
Definition: WANObjectCache.php:137
SwiftFileBackend\doCreateInternal
doCreateInternal(array $params)
Definition: SwiftFileBackend.php:270
SwiftFileBackend\$authSessionTimestamp
int $authSessionTimestamp
UNIX timestamp.
Definition: SwiftFileBackend.php:74
FileBackendStore
Base class for all backends using particular storage medium.
Definition: FileBackendStore.php:41
HTTPFileStreamer\STREAM_HEADLESS
const STREAM_HEADLESS
Definition: HTTPFileStreamer.php:40
FileBackendStore\$RES_ABSENT
static false $RES_ABSENT
Idiom for "no result due to missing file" (since 1.34)
Definition: FileBackendStore.php:64
SwiftFileBackend\doExecuteOpHandlesInternal
doExecuteOpHandlesInternal(array $fileOpHandles)
Definition: SwiftFileBackend.php:1341
FileBackend\$logger
LoggerInterface $logger
Definition: FileBackend.php:122
SwiftFileBackend\$swiftAuthUrl
string $swiftAuthUrl
Authentication base URL (without version)
Definition: SwiftFileBackend.php:43
SwiftFileBackend\doStreamFile
doStreamFile(array $params)
Definition: SwiftFileBackend.php:1128
HTTPFileStreamer\send404Message
static send404Message( $fname, $flags=0)
Send out a standard 404 message for a file.
Definition: HTTPFileStreamer.php:203
FileBackend\newStatus
newStatus(... $args)
Yields the result of the status wrapper callback on either:
Definition: FileBackend.php:1692
SwiftFileBackend\$writeUsers
array $writeUsers
Additional users (account:user) with write permissions on public containers.
Definition: SwiftFileBackend.php:59
SwiftFileBackend\$rgwS3SecretKey
string $rgwS3SecretKey
S3 authentication key (RADOS Gateway)
Definition: SwiftFileBackend.php:55
FileBackendStore\primeContainerCache
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
Definition: FileBackendStore.php:1828
SwiftFileBackend\$authTTL
int $authTTL
TTL in seconds.
Definition: SwiftFileBackend.php:41
SwiftFileBackend\$readUsers
array $readUsers
Additional users (account:user) with read permissions on public containers.
Definition: SwiftFileBackend.php:57
SwiftFileBackend\doCopyInternal
doCopyInternal(array $params)
Definition: SwiftFileBackend.php:414
FileBackend\$name
string $name
Unique backend name.
Definition: FileBackend.php:100
SwiftFileBackend\$rgwS3AccessKey
string $rgwS3AccessKey
S3 access key (RADOS Gateway)
Definition: SwiftFileBackend.php:53
FileBackend\getLocalCopy
getLocalCopy(array $params)
Get a local copy on disk of the file at a storage path in the backend.
Definition: FileBackend.php:1195
SwiftFileBackend\objectListing
objectListing( $fullCont, $type, $limit, $after=null, $prefix=null, $delim=null)
Get a list of objects under a container.
Definition: SwiftFileBackend.php:1599
SwiftFileBackend\$swiftStorageUrl
string $swiftStorageUrl
Override of storage base URL.
Definition: SwiftFileBackend.php:45
$path
$path
Definition: NoLocalSettings.php:25
SwiftFileBackend\convertSwiftDate
convertSwiftDate( $ts, $format=TS_MW)
Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT"/"2013-05-11T07:37:27.678360Z".
Definition: SwiftFileBackend.php:756
SwiftFileBackend\doPublishInternal
doPublishInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:686
SwiftFileBackend\doGetFileStat
doGetFileStat(array $params)
Definition: SwiftFileBackend.php:738
Wikimedia
This program is free software; you can redistribute it and/or modify it under the terms of the GNU Ge...
SwiftFileBackend\$swiftTempUrlKey
string $swiftTempUrlKey
Shared secret value for making temp URLs.
Definition: SwiftFileBackend.php:51
SwiftFileBackend\doStoreInternal
doStoreInternal(array $params)
Definition: SwiftFileBackend.php:325
SwiftFileBackend\doGetFileStatMulti
doGetFileStatMulti(array $params)
Get file stat information (concurrently if possible) for several files.
Definition: SwiftFileBackend.php:1656
$ext
if(!is_readable( $file)) $ext
Definition: router.php:48
SwiftFileBackend\doCleanInternal
doCleanInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:710
SwiftFileBackend\$srvCache
BagOStuff $srvCache
Definition: SwiftFileBackend.php:66
SwiftFileBackendFileList
Iterator for listing regular files.
Definition: SwiftFileBackendFileList.php:28
SwiftFileBackend\getFileListInternal
getFileListInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:913
SwiftFileBackend\doPrimeContainerCache
doPrimeContainerCache(array $containerInfo)
Fill the backend-specific process cache given an array of resolved container names and their correspo...
Definition: SwiftFileBackend.php:1650
SwiftFileBackend\$secureReadUsers
array $secureReadUsers
Additional users (account:user) with read permissions on private containers.
Definition: SwiftFileBackend.php:61
FileBackendStore\setContainerCache
setContainerCache( $container, array $val)
Set the cached info for a container.
Definition: FileBackendStore.php:1803
FileBackend\scopedProfileSection
scopedProfileSection( $section)
Definition: FileBackend.php:1714
SwiftFileBackend\getDirListPageInternal
getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
Definition: SwiftFileBackend.php:928
SwiftFileBackend\$http
MultiHttpClient $http
Definition: SwiftFileBackend.php:39
$type
$type
Definition: testCompression.php:52