MediaWiki  master
SwiftFileBackend.php
Go to the documentation of this file.
1 <?php
25 use Wikimedia\AtEase\AtEase;
26 
38  protected $http;
40  protected $authTTL;
42  protected $swiftAuthUrl;
44  protected $swiftStorageUrl;
46  protected $swiftUser;
48  protected $swiftKey;
50  protected $swiftTempUrlKey;
52  protected $rgwS3AccessKey;
54  protected $rgwS3SecretKey;
56  protected $readUsers;
58  protected $writeUsers;
60  protected $secureReadUsers;
62  protected $secureWriteUsers;
63 
65  protected $srvCache;
66 
69 
71  protected $authCreds;
73  protected $authSessionTimestamp = 0;
75  protected $authErrorTimestamp = null;
76 
78  protected $isRGW = false;
79 
118  public function __construct( array $config ) {
119  parent::__construct( $config );
120  // Required settings
121  $this->swiftAuthUrl = $config['swiftAuthUrl'];
122  $this->swiftUser = $config['swiftUser'];
123  $this->swiftKey = $config['swiftKey'];
124  // Optional settings
125  $this->authTTL = $config['swiftAuthTTL'] ?? 15 * 60; // some sane number
126  $this->swiftTempUrlKey = $config['swiftTempUrlKey'] ?? '';
127  $this->swiftStorageUrl = $config['swiftStorageUrl'] ?? null;
128  $this->shardViaHashLevels = $config['shardViaHashLevels'] ?? '';
129  $this->rgwS3AccessKey = $config['rgwS3AccessKey'] ?? '';
130  $this->rgwS3SecretKey = $config['rgwS3SecretKey'] ?? '';
131 
132  // HTTP helper client
133  $httpOptions = [];
134  foreach ( [ 'connTimeout', 'reqTimeout' ] as $optionName ) {
135  if ( isset( $config[$optionName] ) ) {
136  $httpOptions[$optionName] = $config[$optionName];
137  }
138  }
139  $this->http = new MultiHttpClient( $httpOptions );
140 
141  // Cache container information to mask latency
142  if ( isset( $config['wanCache'] ) && $config['wanCache'] instanceof WANObjectCache ) {
143  $this->memCache = $config['wanCache'];
144  }
145  // Process cache for container info
146  $this->containerStatCache = new MapCacheLRU( 300 );
147  // Cache auth token information to avoid RTTs
148  if ( !empty( $config['cacheAuthInfo'] ) && isset( $config['srvCache'] ) ) {
149  $this->srvCache = $config['srvCache'];
150  } else {
151  $this->srvCache = new EmptyBagOStuff();
152  }
153  $this->readUsers = $config['readUsers'] ?? [];
154  $this->writeUsers = $config['writeUsers'] ?? [];
155  $this->secureReadUsers = $config['secureReadUsers'] ?? [];
156  $this->secureWriteUsers = $config['secureWriteUsers'] ?? [];
157  }
158 
159  public function getFeatures() {
160  return (
161  self::ATTR_UNICODE_PATHS |
162  self::ATTR_HEADERS |
163  self::ATTR_METADATA
164  );
165  }
166 
167  protected function resolveContainerPath( $container, $relStoragePath ) {
168  if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) {
169  return null; // not UTF-8, makes it hard to use CF and the swift HTTP API
170  } elseif ( strlen( rawurlencode( $relStoragePath ) ) > 1024 ) {
171  return null; // too long for Swift
172  }
173 
174  return $relStoragePath;
175  }
176 
177  public function isPathUsableInternal( $storagePath ) {
178  list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
179  if ( $rel === null ) {
180  return false; // invalid
181  }
182 
183  return is_array( $this->getContainerStat( $container ) );
184  }
185 
195  protected function extractMutableContentHeaders( array $headers ) {
196  $contentHeaders = [];
197  // Normalize casing, and strip out illegal headers
198  foreach ( $headers as $name => $value ) {
199  $name = strtolower( $name );
200  if ( !preg_match( '/^(x-)?content-(?!length$)/', $name ) ) {
201  // Only allow content-* and x-content-* headers (but not content-length)
202  continue;
203  } elseif ( $name === 'content-type' && !strlen( $value ) ) {
204  // This header can be set to a value but not unset for sanity
205  continue;
206  }
207  $contentHeaders[$name] = $value;
208  }
209  // By default, Swift has annoyingly low maximum header value limits
210  if ( isset( $contentHeaders['content-disposition'] ) ) {
211  $maxLength = 255;
212  // @note: assume FileBackend::makeContentDisposition() already used
213  $offset = $maxLength - strlen( $contentHeaders['content-disposition'] );
214  if ( $offset < 0 ) {
215  $pos = strrpos( $contentHeaders['content-disposition'], ';', $offset );
216  $contentHeaders['content-disposition'] = $pos === false
217  ? ''
218  : trim( substr( $contentHeaders['content-disposition'], 0, $pos ) );
219  }
220  }
221 
222  return $contentHeaders;
223  }
224 
230  protected function extractMetadataHeaders( array $headers ) {
231  $metadataHeaders = [];
232  foreach ( $headers as $name => $value ) {
233  $name = strtolower( $name );
234  if ( strpos( $name, 'x-object-meta-' ) === 0 ) {
235  $metadataHeaders[$name] = $value;
236  }
237  }
238 
239  return $metadataHeaders;
240  }
241 
247  protected function getMetadataFromHeaders( array $headers ) {
248  $prefixLen = strlen( 'x-object-meta-' );
249 
250  $metadata = [];
251  foreach ( $this->extractMetadataHeaders( $headers ) as $name => $value ) {
252  $metadata[substr( $name, $prefixLen )] = $value;
253  }
254 
255  return $metadata;
256  }
257 
258  protected function doCreateInternal( array $params ) {
259  $status = $this->newStatus();
260 
261  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
262  if ( $dstRel === null ) {
263  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
264 
265  return $status;
266  }
267 
268  // Headers that are not strictly a function of the file content
269  $mutableHeaders = $this->extractMutableContentHeaders( $params['headers'] ?? [] );
270  // Make sure that the "content-type" header is set to something sensible
271  $mutableHeaders['content-type'] = $mutableHeaders['content-type']
272  ?? $this->getContentType( $params['dst'], $params['content'], null );
273 
274  $reqs = [ [
275  'method' => 'PUT',
276  'url' => [ $dstCont, $dstRel ],
277  'headers' => array_merge(
278  $mutableHeaders,
279  [
280  'etag' => md5( $params['content'] ),
281  'content-length' => strlen( $params['content'] ),
282  'x-object-meta-sha1base36' =>
283  Wikimedia\base_convert( sha1( $params['content'] ), 16, 36, 31 )
284  ]
285  ),
286  'body' => $params['content']
287  ] ];
288 
289  $method = __METHOD__;
290  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
291  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
292  if ( $rcode === 201 || $rcode === 202 ) {
293  // good
294  } elseif ( $rcode === 412 ) {
295  $status->fatal( 'backend-fail-contenttype', $params['dst'] );
296  } else {
297  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
298  }
299 
300  return SwiftFileOpHandle::CONTINUE_IF_OK;
301  };
302 
303  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
304  if ( !empty( $params['async'] ) ) { // deferred
305  $status->value = $opHandle;
306  } else { // actually write the object in Swift
307  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
308  }
309 
310  return $status;
311  }
312 
313  protected function doStoreInternal( array $params ) {
314  $status = $this->newStatus();
315 
316  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
317  if ( $dstRel === null ) {
318  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
319 
320  return $status;
321  }
322 
323  // Open a handle to the source file so that it can be streamed. The size and hash
324  // will be computed using the handle. In the off chance that the source file changes
325  // during this operation, the PUT will fail due to an ETag mismatch and be aborted.
326  AtEase::suppressWarnings();
327  $srcHandle = fopen( $params['src'], 'rb' );
328  AtEase::restoreWarnings();
329  if ( $srcHandle === false ) { // source doesn't exist?
330  $status->fatal( 'backend-fail-notexists', $params['src'] );
331 
332  return $status;
333  }
334 
335  // Compute the MD5 and SHA-1 hashes in one pass
336  $srcSize = fstat( $srcHandle )['size'];
337  $md5Context = hash_init( 'md5' );
338  $sha1Context = hash_init( 'sha1' );
339  $hashDigestSize = 0;
340  while ( !feof( $srcHandle ) ) {
341  $buffer = (string)fread( $srcHandle, 131072 ); // 128 KiB
342  hash_update( $md5Context, $buffer );
343  hash_update( $sha1Context, $buffer );
344  $hashDigestSize += strlen( $buffer );
345  }
346  // Reset the handle back to the beginning so that it can be streamed
347  rewind( $srcHandle );
348 
349  if ( $hashDigestSize !== $srcSize ) {
350  $status->fatal( 'backend-fail-hash', $params['src'] );
351 
352  return $status;
353  }
354 
355  // Headers that are not strictly a function of the file content
356  $mutableHeaders = $this->extractMutableContentHeaders( $params['headers'] ?? [] );
357  // Make sure that the "content-type" header is set to something sensible
358  $mutableHeaders['content-type'] = $mutableHeaders['content-type']
359  ?? $this->getContentType( $params['dst'], null, $params['src'] );
360 
361  $reqs = [ [
362  'method' => 'PUT',
363  'url' => [ $dstCont, $dstRel ],
364  'headers' => array_merge(
365  $mutableHeaders,
366  [
367  'content-length' => $srcSize,
368  'etag' => hash_final( $md5Context ),
369  'x-object-meta-sha1base36' =>
370  Wikimedia\base_convert( hash_final( $sha1Context ), 16, 36, 31 )
371  ]
372  ),
373  'body' => $srcHandle // resource
374  ] ];
375 
376  $method = __METHOD__;
377  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
378  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
379  if ( $rcode === 201 || $rcode === 202 ) {
380  // good
381  } elseif ( $rcode === 412 ) {
382  $status->fatal( 'backend-fail-contenttype', $params['dst'] );
383  } else {
384  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
385  }
386 
387  return SwiftFileOpHandle::CONTINUE_IF_OK;
388  };
389 
390  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
391  $opHandle->resourcesToClose[] = $srcHandle;
392 
393  if ( !empty( $params['async'] ) ) { // deferred
394  $status->value = $opHandle;
395  } else { // actually write the object in Swift
396  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
397  }
398 
399  return $status;
400  }
401 
402  protected function doCopyInternal( array $params ) {
403  $status = $this->newStatus();
404 
405  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
406  if ( $srcRel === null ) {
407  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
408 
409  return $status;
410  }
411 
412  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
413  if ( $dstRel === null ) {
414  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
415 
416  return $status;
417  }
418 
419  $reqs = [ [
420  'method' => 'PUT',
421  'url' => [ $dstCont, $dstRel ],
422  'headers' => array_merge(
423  $this->extractMutableContentHeaders( $params['headers'] ?? [] ),
424  [
425  'x-copy-from' => '/' . rawurlencode( $srcCont ) . '/' .
426  str_replace( "%2F", "/", rawurlencode( $srcRel ) )
427  ]
428  )
429  ] ];
430 
431  $method = __METHOD__;
432  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
433  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
434  if ( $rcode === 201 ) {
435  // good
436  } elseif ( $rcode === 404 ) {
437  if ( empty( $params['ignoreMissingSource'] ) ) {
438  $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
439  }
440  } else {
441  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
442  }
443 
444  return SwiftFileOpHandle::CONTINUE_IF_OK;
445  };
446 
447  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
448  if ( !empty( $params['async'] ) ) { // deferred
449  $status->value = $opHandle;
450  } else { // actually write the object in Swift
451  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
452  }
453 
454  return $status;
455  }
456 
457  protected function doMoveInternal( array $params ) {
458  $status = $this->newStatus();
459 
460  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
461  if ( $srcRel === null ) {
462  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
463 
464  return $status;
465  }
466 
467  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
468  if ( $dstRel === null ) {
469  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
470 
471  return $status;
472  }
473 
474  $reqs = [ [
475  'method' => 'PUT',
476  'url' => [ $dstCont, $dstRel ],
477  'headers' => array_merge(
478  $this->extractMutableContentHeaders( $params['headers'] ?? [] ),
479  [
480  'x-copy-from' => '/' . rawurlencode( $srcCont ) . '/' .
481  str_replace( "%2F", "/", rawurlencode( $srcRel ) )
482  ]
483  )
484  ] ];
485  if ( "{$srcCont}/{$srcRel}" !== "{$dstCont}/{$dstRel}" ) {
486  $reqs[] = [
487  'method' => 'DELETE',
488  'url' => [ $srcCont, $srcRel ],
489  'headers' => []
490  ];
491  }
492 
493  $method = __METHOD__;
494  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
495  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
496  if ( $request['method'] === 'PUT' && $rcode === 201 ) {
497  // good
498  } elseif ( $request['method'] === 'DELETE' && $rcode === 204 ) {
499  // good
500  } elseif ( $rcode === 404 ) {
501  if ( empty( $params['ignoreMissingSource'] ) ) {
502  $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
503  } else {
504  // Leave Status as OK but skip the DELETE request
505  return SwiftFileOpHandle::CONTINUE_NO;
506  }
507  } else {
508  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
509  }
510 
511  return SwiftFileOpHandle::CONTINUE_IF_OK;
512  };
513 
514  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
515  if ( !empty( $params['async'] ) ) { // deferred
516  $status->value = $opHandle;
517  } else { // actually move the object in Swift
518  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
519  }
520 
521  return $status;
522  }
523 
524  protected function doDeleteInternal( array $params ) {
525  $status = $this->newStatus();
526 
527  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
528  if ( $srcRel === null ) {
529  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
530 
531  return $status;
532  }
533 
534  $reqs = [ [
535  'method' => 'DELETE',
536  'url' => [ $srcCont, $srcRel ],
537  'headers' => []
538  ] ];
539 
540  $method = __METHOD__;
541  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
542  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
543  if ( $rcode === 204 ) {
544  // good
545  } elseif ( $rcode === 404 ) {
546  if ( empty( $params['ignoreMissingSource'] ) ) {
547  $status->fatal( 'backend-fail-delete', $params['src'] );
548  }
549  } else {
550  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
551  }
552 
553  return SwiftFileOpHandle::CONTINUE_IF_OK;
554  };
555 
556  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
557  if ( !empty( $params['async'] ) ) { // deferred
558  $status->value = $opHandle;
559  } else { // actually delete the object in Swift
560  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
561  }
562 
563  return $status;
564  }
565 
566  protected function doDescribeInternal( array $params ) {
567  $status = $this->newStatus();
568 
569  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
570  if ( $srcRel === null ) {
571  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
572 
573  return $status;
574  }
575 
576  // Fetch the old object headers/metadata...this should be in stat cache by now
577  $stat = $this->getFileStat( [ 'src' => $params['src'], 'latest' => 1 ] );
578  if ( $stat && !isset( $stat['xattr'] ) ) { // older cache entry
579  $stat = $this->doGetFileStat( [ 'src' => $params['src'], 'latest' => 1 ] );
580  }
581  if ( !$stat ) {
582  $status->fatal( 'backend-fail-describe', $params['src'] );
583 
584  return $status;
585  }
586 
587  // Swift object POST clears any prior headers, so merge the new and old headers here.
588  // Also, during, POST, libcurl adds "Content-Type: application/x-www-form-urlencoded"
589  // if "Content-Type" is not set, which would clobber the header value for the object.
590  $oldMetadataHeaders = [];
591  foreach ( $stat['xattr']['metadata'] as $name => $value ) {
592  $oldMetadataHeaders["x-object-meta-$name"] = $value;
593  }
594  $newContentHeaders = $this->extractMutableContentHeaders( $params['headers'] ?? [] );
595  $oldContentHeaders = $stat['xattr']['headers'];
596 
597  $reqs = [ [
598  'method' => 'POST',
599  'url' => [ $srcCont, $srcRel ],
600  'headers' => $oldMetadataHeaders + $newContentHeaders + $oldContentHeaders
601  ] ];
602 
603  $method = __METHOD__;
604  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
605  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
606  if ( $rcode === 202 ) {
607  // good
608  } elseif ( $rcode === 404 ) {
609  $status->fatal( 'backend-fail-describe', $params['src'] );
610  } else {
611  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
612  }
613  };
614 
615  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
616  if ( !empty( $params['async'] ) ) { // deferred
617  $status->value = $opHandle;
618  } else { // actually change the object in Swift
619  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
620  }
621 
622  return $status;
623  }
624 
625  protected function doPrepareInternal( $fullCont, $dir, array $params ) {
626  $status = $this->newStatus();
627 
628  // (a) Check if container already exists
629  $stat = $this->getContainerStat( $fullCont );
630  if ( is_array( $stat ) ) {
631  return $status; // already there
632  } elseif ( $stat === self::$RES_ERROR ) {
633  $status->fatal( 'backend-fail-internal', $this->name );
634  $this->logger->error( __METHOD__ . ': cannot get container stat' );
635 
636  return $status;
637  }
638 
639  // (b) Create container as needed with proper ACLs
640  if ( $stat === false ) {
641  $params['op'] = 'prepare';
642  $status->merge( $this->createContainer( $fullCont, $params ) );
643  }
644 
645  return $status;
646  }
647 
648  protected function doSecureInternal( $fullCont, $dir, array $params ) {
649  $status = $this->newStatus();
650  if ( empty( $params['noAccess'] ) ) {
651  return $status; // nothing to do
652  }
653 
654  $stat = $this->getContainerStat( $fullCont );
655  if ( is_array( $stat ) ) {
656  $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
657  $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
658  // Make container private to end-users...
659  $status->merge( $this->setContainerAccess(
660  $fullCont,
661  $readUsers,
663  ) );
664  } elseif ( $stat === false ) {
665  $status->fatal( 'backend-fail-usable', $params['dir'] );
666  } else {
667  $status->fatal( 'backend-fail-internal', $this->name );
668  $this->logger->error( __METHOD__ . ': cannot get container stat' );
669  }
670 
671  return $status;
672  }
673 
674  protected function doPublishInternal( $fullCont, $dir, array $params ) {
675  $status = $this->newStatus();
676 
677  $stat = $this->getContainerStat( $fullCont );
678  if ( is_array( $stat ) ) {
679  $readUsers = array_merge( $this->readUsers, [ $this->swiftUser, '.r:*' ] );
680  $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
681 
682  // Make container public to end-users...
683  $status->merge( $this->setContainerAccess(
684  $fullCont,
685  $readUsers,
687  ) );
688  } elseif ( $stat === false ) {
689  $status->fatal( 'backend-fail-usable', $params['dir'] );
690  } else {
691  $status->fatal( 'backend-fail-internal', $this->name );
692  $this->logger->error( __METHOD__ . ': cannot get container stat' );
693  }
694 
695  return $status;
696  }
697 
698  protected function doCleanInternal( $fullCont, $dir, array $params ) {
699  $status = $this->newStatus();
700 
701  // Only containers themselves can be removed, all else is virtual
702  if ( $dir != '' ) {
703  return $status; // nothing to do
704  }
705 
706  // (a) Check the container
707  $stat = $this->getContainerStat( $fullCont, true );
708  if ( $stat === false ) {
709  return $status; // ok, nothing to do
710  } elseif ( !is_array( $stat ) ) {
711  $status->fatal( 'backend-fail-internal', $this->name );
712  $this->logger->error( __METHOD__ . ': cannot get container stat' );
713 
714  return $status;
715  }
716 
717  // (b) Delete the container if empty
718  if ( $stat['count'] == 0 ) {
719  $params['op'] = 'clean';
720  $status->merge( $this->deleteContainer( $fullCont, $params ) );
721  }
722 
723  return $status;
724  }
725 
726  protected function doGetFileStat( array $params ) {
727  $params = [ 'srcs' => [ $params['src'] ], 'concurrency' => 1 ] + $params;
728  unset( $params['src'] );
729  $stats = $this->doGetFileStatMulti( $params );
730 
731  return reset( $stats );
732  }
733 
744  protected function convertSwiftDate( $ts, $format = TS_MW ) {
745  try {
746  $timestamp = new MWTimestamp( $ts );
747 
748  return $timestamp->getTimestamp( $format );
749  } catch ( Exception $e ) {
750  throw new FileBackendError( $e->getMessage() );
751  }
752  }
753 
761  protected function addMissingHashMetadata( array $objHdrs, $path ) {
762  if ( isset( $objHdrs['x-object-meta-sha1base36'] ) ) {
763  return $objHdrs; // nothing to do
764  }
765 
767  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
768  $this->logger->error( __METHOD__ . ": {path} was not stored with SHA-1 metadata.",
769  [ 'path' => $path ] );
770 
771  $objHdrs['x-object-meta-sha1base36'] = false;
772 
773  $auth = $this->getAuthentication();
774  if ( !$auth ) {
775  return $objHdrs; // failed
776  }
777 
778  // Find prior custom HTTP headers
779  $postHeaders = $this->extractMutableContentHeaders( $objHdrs );
780  // Find prior metadata headers
781  $postHeaders += $this->extractMetadataHeaders( $objHdrs );
782 
783  $status = $this->newStatus();
785  $scopeLockS = $this->getScopedFileLocks( [ $path ], LockManager::LOCK_UW, $status );
786  if ( $status->isOK() ) {
787  $tmpFile = $this->getLocalCopy( [ 'src' => $path, 'latest' => 1 ] );
788  if ( $tmpFile ) {
789  $hash = $tmpFile->getSha1Base36();
790  if ( $hash !== false ) {
791  $objHdrs['x-object-meta-sha1base36'] = $hash;
792  // Merge new SHA1 header into the old ones
793  $postHeaders['x-object-meta-sha1base36'] = $hash;
794  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
795  list( $rcode ) = $this->http->run( [
796  'method' => 'POST',
797  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
798  'headers' => $this->authTokenHeaders( $auth ) + $postHeaders
799  ] );
800  if ( $rcode >= 200 && $rcode <= 299 ) {
801  $this->deleteFileCache( $path );
802 
803  return $objHdrs; // success
804  }
805  }
806  }
807  }
808 
809  $this->logger->error( __METHOD__ . ': unable to set SHA-1 metadata for {path}',
810  [ 'path' => $path ] );
811 
812  return $objHdrs; // failed
813  }
814 
815  protected function doGetFileContentsMulti( array $params ) {
816  $auth = $this->getAuthentication();
817 
818  $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging
819  // Blindly create tmp files and stream to them, catching any exception
820  // if the file does not exist. Do not waste time doing file stats here.
821  $reqs = []; // (path => op)
822 
823  // Initial dummy values to preserve path order
824  $contents = array_fill_keys( $params['srcs'], self::$RES_ERROR );
825  foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch
826  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
827  if ( $srcRel === null || !$auth ) {
828  continue; // invalid storage path or auth error
829  }
830  // Create a new temporary memory file...
831  $handle = fopen( 'php://temp', 'wb' );
832  if ( $handle ) {
833  $reqs[$path] = [
834  'method' => 'GET',
835  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
836  'headers' => $this->authTokenHeaders( $auth )
837  + $this->headersFromParams( $params ),
838  'stream' => $handle,
839  ];
840  }
841  }
842 
843  $opts = [ 'maxConnsPerHost' => $params['concurrency'] ];
844  $reqs = $this->http->runMulti( $reqs, $opts );
845  foreach ( $reqs as $path => $op ) {
846  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response'];
847  if ( $rcode >= 200 && $rcode <= 299 ) {
848  rewind( $op['stream'] ); // start from the beginning
849  $content = (string)stream_get_contents( $op['stream'] );
850  $size = strlen( $content );
851  // Make sure that stream finished
852  if ( $size === (int)$rhdrs['content-length'] ) {
853  $contents[$path] = $content;
854  } else {
855  $contents[$path] = self::$RES_ERROR;
856  $rerr = "Got {$size}/{$rhdrs['content-length']} bytes";
857  $this->onError( null, __METHOD__,
858  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
859  }
860  } elseif ( $rcode === 404 ) {
861  $contents[$path] = self::$RES_ABSENT;
862  } else {
863  $contents[$path] = self::$RES_ERROR;
864  $this->onError( null, __METHOD__,
865  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
866  }
867  fclose( $op['stream'] ); // close open handle
868  }
869 
870  return $contents;
871  }
872 
873  protected function doDirectoryExists( $fullCont, $dir, array $params ) {
874  $prefix = ( $dir == '' ) ? null : "{$dir}/";
875  $status = $this->objectListing( $fullCont, 'names', 1, null, $prefix );
876  if ( $status->isOK() ) {
877  return ( count( $status->value ) ) > 0;
878  }
879 
880  return self::$RES_ERROR;
881  }
882 
890  public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
891  return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
892  }
893 
901  public function getFileListInternal( $fullCont, $dir, array $params ) {
902  return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
903  }
904 
916  public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
917  $dirs = [];
918  if ( $after === INF ) {
919  return $dirs; // nothing more
920  }
921 
923  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
924 
925  $prefix = ( $dir == '' ) ? null : "{$dir}/";
926  // Non-recursive: only list dirs right under $dir
927  if ( !empty( $params['topOnly'] ) ) {
928  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
929  if ( !$status->isOK() ) {
930  throw new FileBackendError( "Iterator page I/O error." );
931  }
932  $objects = $status->value;
933  // @phan-suppress-next-line PhanTypeSuspiciousNonTraversableForeach
934  foreach ( $objects as $object ) { // files and directories
935  if ( substr( $object, -1 ) === '/' ) {
936  $dirs[] = $object; // directories end in '/'
937  }
938  }
939  } else {
940  // Recursive: list all dirs under $dir and its subdirs
941  $getParentDir = function ( $path ) {
942  return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
943  };
944 
945  // Get directory from last item of prior page
946  $lastDir = $getParentDir( $after ); // must be first page
947  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
948 
949  if ( !$status->isOK() ) {
950  throw new FileBackendError( "Iterator page I/O error." );
951  }
952 
953  $objects = $status->value;
954 
955  // @phan-suppress-next-line PhanTypeSuspiciousNonTraversableForeach
956  foreach ( $objects as $object ) { // files
957  $objectDir = $getParentDir( $object ); // directory of object
958 
959  if ( $objectDir !== false && $objectDir !== $dir ) {
960  // Swift stores paths in UTF-8, using binary sorting.
961  // See function "create_container_table" in common/db.py.
962  // If a directory is not "greater" than the last one,
963  // then it was already listed by the calling iterator.
964  if ( strcmp( $objectDir, $lastDir ) > 0 ) {
965  $pDir = $objectDir;
966  do { // add dir and all its parent dirs
967  $dirs[] = "{$pDir}/";
968  $pDir = $getParentDir( $pDir );
969  } while ( $pDir !== false // sanity
970  && strcmp( $pDir, $lastDir ) > 0 // not done already
971  && strlen( $pDir ) > strlen( $dir ) // within $dir
972  );
973  }
974  $lastDir = $objectDir;
975  }
976  }
977  }
978  // Page on the unfiltered directory listing (what is returned may be filtered)
979  if ( count( $objects ) < $limit ) {
980  $after = INF; // avoid a second RTT
981  } else {
982  $after = end( $objects ); // update last item
983  }
984 
985  return $dirs;
986  }
987 
999  public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
1000  $files = []; // list of (path, stat array or null) entries
1001  if ( $after === INF ) {
1002  return $files; // nothing more
1003  }
1004 
1006  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1007 
1008  $prefix = ( $dir == '' ) ? null : "{$dir}/";
1009  // $objects will contain a list of unfiltered names or stdClass items
1010  // Non-recursive: only list files right under $dir
1011  if ( !empty( $params['topOnly'] ) ) {
1012  if ( !empty( $params['adviseStat'] ) ) {
1013  $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix, '/' );
1014  } else {
1015  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
1016  }
1017  } else {
1018  // Recursive: list all files under $dir and its subdirs
1019  if ( !empty( $params['adviseStat'] ) ) {
1020  $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix );
1021  } else {
1022  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
1023  }
1024  }
1025 
1026  // Reformat this list into a list of (name, stat array or null) entries
1027  if ( !$status->isOK() ) {
1028  throw new FileBackendError( "Iterator page I/O error." );
1029  }
1030 
1031  $objects = $status->value;
1032  $files = $this->buildFileObjectListing( $objects );
1033 
1034  // Page on the unfiltered object listing (what is returned may be filtered)
1035  if ( count( $objects ) < $limit ) {
1036  $after = INF; // avoid a second RTT
1037  } else {
1038  $after = end( $objects ); // update last item
1039  $after = is_object( $after ) ? $after->name : $after;
1040  }
1041 
1042  return $files;
1043  }
1044 
1052  private function buildFileObjectListing( array $objects ) {
1053  $names = [];
1054  foreach ( $objects as $object ) {
1055  if ( is_object( $object ) ) {
1056  if ( isset( $object->subdir ) || !isset( $object->name ) ) {
1057  continue; // virtual directory entry; ignore
1058  }
1059  $stat = [
1060  // Convert various random Swift dates to TS_MW
1061  'mtime' => $this->convertSwiftDate( $object->last_modified, TS_MW ),
1062  'size' => (int)$object->bytes,
1063  'sha1' => null,
1064  // Note: manifiest ETags are not an MD5 of the file
1065  'md5' => ctype_xdigit( $object->hash ) ? $object->hash : null,
1066  'latest' => false // eventually consistent
1067  ];
1068  $names[] = [ $object->name, $stat ];
1069  } elseif ( substr( $object, -1 ) !== '/' ) {
1070  // Omit directories, which end in '/' in listings
1071  $names[] = [ $object, null ];
1072  }
1073  }
1074 
1075  return $names;
1076  }
1077 
1084  public function loadListingStatInternal( $path, array $val ) {
1085  $this->cheapCache->setField( $path, 'stat', $val );
1086  }
1087 
1088  protected function doGetFileXAttributes( array $params ) {
1089  $stat = $this->getFileStat( $params );
1090  // Stat entries filled by file listings don't include metadata/headers
1091  if ( is_array( $stat ) && !isset( $stat['xattr'] ) ) {
1092  $this->clearCache( [ $params['src'] ] );
1093  $stat = $this->getFileStat( $params );
1094  }
1095 
1096  if ( is_array( $stat ) ) {
1097  return $stat['xattr'];
1098  }
1099 
1100  return ( $stat === self::$RES_ERROR ) ? self::$RES_ERROR : self::$RES_ABSENT;
1101  }
1102 
1103  protected function doGetFileSha1base36( array $params ) {
1104  // Avoid using stat entries from file listings, which never include the SHA-1 hash.
1105  // Also, recompute the hash if it's not part of the metadata headers for some reason.
1106  $params['requireSHA1'] = true;
1107 
1108  $stat = $this->getFileStat( $params );
1109  if ( is_array( $stat ) ) {
1110  return $stat['sha1'];
1111  }
1112 
1113  return ( $stat === self::$RES_ERROR ) ? self::$RES_ERROR : self::$RES_ABSENT;
1114  }
1115 
1116  protected function doStreamFile( array $params ) {
1117  $status = $this->newStatus();
1118 
1119  $flags = !empty( $params['headless'] ) ? HTTPFileStreamer::STREAM_HEADLESS : 0;
1120 
1121  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1122  if ( $srcRel === null ) {
1123  HTTPFileStreamer::send404Message( $params['src'], $flags );
1124  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
1125 
1126  return $status;
1127  }
1128 
1129  $auth = $this->getAuthentication();
1130  if ( !$auth || !is_array( $this->getContainerStat( $srcCont ) ) ) {
1131  HTTPFileStreamer::send404Message( $params['src'], $flags );
1132  $status->fatal( 'backend-fail-stream', $params['src'] );
1133 
1134  return $status;
1135  }
1136 
1137  // If "headers" is set, we only want to send them if the file is there.
1138  // Do not bother checking if the file exists if headers are not set though.
1139  if ( $params['headers'] && !$this->fileExists( $params ) ) {
1140  HTTPFileStreamer::send404Message( $params['src'], $flags );
1141  $status->fatal( 'backend-fail-stream', $params['src'] );
1142 
1143  return $status;
1144  }
1145 
1146  // Send the requested additional headers
1147  foreach ( $params['headers'] as $header ) {
1148  header( $header ); // aways send
1149  }
1150 
1151  if ( empty( $params['allowOB'] ) ) {
1152  // Cancel output buffering and gzipping if set
1153  ( $this->obResetFunc )();
1154  }
1155 
1156  $handle = fopen( 'php://output', 'wb' );
1157  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1158  'method' => 'GET',
1159  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1160  'headers' => $this->authTokenHeaders( $auth )
1161  + $this->headersFromParams( $params ) + $params['options'],
1162  'stream' => $handle,
1163  'flags' => [ 'relayResponseHeaders' => empty( $params['headless'] ) ]
1164  ] );
1165 
1166  if ( $rcode >= 200 && $rcode <= 299 ) {
1167  // good
1168  } elseif ( $rcode === 404 ) {
1169  $status->fatal( 'backend-fail-stream', $params['src'] );
1170  // Per T43113, nasty things can happen if bad cache entries get
1171  // stuck in cache. It's also possible that this error can come up
1172  // with simple race conditions. Clear out the stat cache to be safe.
1173  $this->clearCache( [ $params['src'] ] );
1174  $this->deleteFileCache( $params['src'] );
1175  } else {
1176  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1177  }
1178 
1179  return $status;
1180  }
1181 
1182  protected function doGetLocalCopyMulti( array $params ) {
1183  $auth = $this->getAuthentication();
1184 
1185  $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging
1186  // Blindly create tmp files and stream to them, catching any exception
1187  // if the file does not exist. Do not waste time doing file stats here.
1188  $reqs = []; // (path => op)
1189 
1190  // Initial dummy values to preserve path order
1191  $tmpFiles = array_fill_keys( $params['srcs'], self::$RES_ERROR );
1192  foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch
1193  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
1194  if ( $srcRel === null || !$auth ) {
1195  continue; // invalid storage path or auth error
1196  }
1197  // Get source file extension
1199  // Create a new temporary file...
1200  $tmpFile = $this->tmpFileFactory->newTempFSFile( 'localcopy_', $ext );
1201  $handle = $tmpFile ? fopen( $tmpFile->getPath(), 'wb' ) : false;
1202  if ( $handle ) {
1203  $reqs[$path] = [
1204  'method' => 'GET',
1205  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1206  'headers' => $this->authTokenHeaders( $auth )
1207  + $this->headersFromParams( $params ),
1208  'stream' => $handle,
1209  ];
1210  $tmpFiles[$path] = $tmpFile;
1211  }
1212  }
1213 
1214  // Ceph RADOS Gateway is in use (strong consistency) or X-Newest will be used
1215  $latest = ( $this->isRGW || !empty( $params['latest'] ) );
1216 
1217  $opts = [ 'maxConnsPerHost' => $params['concurrency'] ];
1218  $reqs = $this->http->runMulti( $reqs, $opts );
1219  foreach ( $reqs as $path => $op ) {
1220  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response'];
1221  fclose( $op['stream'] ); // close open handle
1222  if ( $rcode >= 200 && $rcode <= 299 ) {
1224  $tmpFile = $tmpFiles[$path];
1225  // Make sure that the stream finished and fully wrote to disk
1226  $size = $tmpFile->getSize();
1227  if ( $size !== (int)$rhdrs['content-length'] ) {
1228  $tmpFiles[$path] = self::$RES_ERROR;
1229  $rerr = "Got {$size}/{$rhdrs['content-length']} bytes";
1230  $this->onError( null, __METHOD__,
1231  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
1232  }
1233  // Set the file stat process cache in passing
1234  $stat = $this->getStatFromHeaders( $rhdrs );
1235  $stat['latest'] = $latest;
1236  $this->cheapCache->setField( $path, 'stat', $stat );
1237  } elseif ( $rcode === 404 ) {
1238  $tmpFiles[$path] = self::$RES_ABSENT;
1239  $this->cheapCache->setField(
1240  $path,
1241  'stat',
1242  $latest ? self::$ABSENT_LATEST : self::$ABSENT_NORMAL
1243  );
1244  } else {
1245  $tmpFiles[$path] = self::$RES_ERROR;
1246  $this->onError( null, __METHOD__,
1247  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
1248  }
1249  }
1250 
1251  return $tmpFiles;
1252  }
1253 
1254  public function getFileHttpUrl( array $params ) {
1255  if ( $this->swiftTempUrlKey != '' ||
1256  ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' )
1257  ) {
1258  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1259  if ( $srcRel === null ) {
1260  return self::TEMPURL_ERROR; // invalid path
1261  }
1262 
1263  $auth = $this->getAuthentication();
1264  if ( !$auth ) {
1265  return self::TEMPURL_ERROR;
1266  }
1267 
1268  $ttl = $params['ttl'] ?? 86400;
1269  $expires = time() + $ttl;
1270 
1271  if ( $this->swiftTempUrlKey != '' ) {
1272  $url = $this->storageUrl( $auth, $srcCont, $srcRel );
1273  // Swift wants the signature based on the unencoded object name
1274  $contPath = parse_url( $this->storageUrl( $auth, $srcCont ), PHP_URL_PATH );
1275  $signature = hash_hmac( 'sha1',
1276  "GET\n{$expires}\n{$contPath}/{$srcRel}",
1277  $this->swiftTempUrlKey
1278  );
1279 
1280  return "{$url}?temp_url_sig={$signature}&temp_url_expires={$expires}";
1281  } else { // give S3 API URL for rgw
1282  // Path for signature starts with the bucket
1283  $spath = '/' . rawurlencode( $srcCont ) . '/' .
1284  str_replace( '%2F', '/', rawurlencode( $srcRel ) );
1285  // Calculate the hash
1286  $signature = base64_encode( hash_hmac(
1287  'sha1',
1288  "GET\n\n\n{$expires}\n{$spath}",
1289  $this->rgwS3SecretKey,
1290  true // raw
1291  ) );
1292  // See https://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html.
1293  // Note: adding a newline for empty CanonicalizedAmzHeaders does not work.
1294  // Note: S3 API is the rgw default; remove the /swift/ URL bit.
1295  return str_replace( '/swift/v1', '', $this->storageUrl( $auth ) . $spath ) .
1296  '?' .
1297  http_build_query( [
1298  'Signature' => $signature,
1299  'Expires' => $expires,
1300  'AWSAccessKeyId' => $this->rgwS3AccessKey
1301  ] );
1302  }
1303  }
1304 
1305  return self::TEMPURL_ERROR;
1306  }
1307 
1308  protected function directoriesAreVirtual() {
1309  return true;
1310  }
1311 
1320  protected function headersFromParams( array $params ) {
1321  $hdrs = [];
1322  if ( !empty( $params['latest'] ) ) {
1323  $hdrs['x-newest'] = 'true';
1324  }
1325 
1326  return $hdrs;
1327  }
1328 
1329  protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1331  '@phan-var SwiftFileOpHandle[] $fileOpHandles';
1332 
1334  $statuses = [];
1335 
1336  $auth = $this->getAuthentication();
1337  if ( !$auth ) {
1338  foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1339  $statuses[$index] = $this->newStatus( 'backend-fail-connect', $this->name );
1340  }
1341 
1342  return $statuses;
1343  }
1344 
1345  // Split the HTTP requests into stages that can be done concurrently
1346  $httpReqsByStage = []; // map of (stage => index => HTTP request)
1347  foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1348  $reqs = $fileOpHandle->httpOp;
1349  // Convert the 'url' parameter to an actual URL using $auth
1350  foreach ( $reqs as $stage => &$req ) {
1351  list( $container, $relPath ) = $req['url'];
1352  $req['url'] = $this->storageUrl( $auth, $container, $relPath );
1353  $req['headers'] = $req['headers'] ?? [];
1354  $req['headers'] = $this->authTokenHeaders( $auth ) + $req['headers'];
1355  $httpReqsByStage[$stage][$index] = $req;
1356  }
1357  $statuses[$index] = $this->newStatus();
1358  }
1359 
1360  // Run all requests for the first stage, then the next, and so on
1361  $reqCount = count( $httpReqsByStage );
1362  for ( $stage = 0; $stage < $reqCount; ++$stage ) {
1363  $httpReqs = $this->http->runMulti( $httpReqsByStage[$stage] );
1364  foreach ( $httpReqs as $index => $httpReq ) {
1366  $fileOpHandle = $fileOpHandles[$index];
1367  // Run the callback for each request of this operation
1368  $status = $statuses[$index];
1369  ( $fileOpHandle->callback )( $httpReq, $status );
1370  // On failure, abort all remaining requests for this operation. This is used
1371  // in "move" operations to abort the DELETE request if the PUT request fails.
1372  if (
1373  !$status->isOK() ||
1374  $fileOpHandle->state === $fileOpHandle::CONTINUE_NO
1375  ) {
1376  $stages = count( $fileOpHandle->httpOp );
1377  for ( $s = ( $stage + 1 ); $s < $stages; ++$s ) {
1378  unset( $httpReqsByStage[$s][$index] );
1379  }
1380  }
1381  }
1382  }
1383 
1384  return $statuses;
1385  }
1386 
1409  protected function setContainerAccess( $container, array $readUsers, array $writeUsers ) {
1410  $status = $this->newStatus();
1411  $auth = $this->getAuthentication();
1412 
1413  if ( !$auth ) {
1414  $status->fatal( 'backend-fail-connect', $this->name );
1415 
1416  return $status;
1417  }
1418 
1419  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1420  'method' => 'POST',
1421  'url' => $this->storageUrl( $auth, $container ),
1422  'headers' => $this->authTokenHeaders( $auth ) + [
1423  'x-container-read' => implode( ',', $readUsers ),
1424  'x-container-write' => implode( ',', $writeUsers )
1425  ]
1426  ] );
1427 
1428  if ( $rcode != 204 && $rcode !== 202 ) {
1429  $status->fatal( 'backend-fail-internal', $this->name );
1430  $this->logger->error( __METHOD__ . ': unexpected rcode value ({rcode})',
1431  [ 'rcode' => $rcode ] );
1432  }
1433 
1434  return $status;
1435  }
1436 
1445  protected function getContainerStat( $container, $bypassCache = false ) {
1447  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1448 
1449  if ( $bypassCache ) { // purge cache
1450  $this->containerStatCache->clear( $container );
1451  } elseif ( !$this->containerStatCache->hasField( $container, 'stat' ) ) {
1452  $this->primeContainerCache( [ $container ] ); // check persistent cache
1453  }
1454  if ( !$this->containerStatCache->hasField( $container, 'stat' ) ) {
1455  $auth = $this->getAuthentication();
1456  if ( !$auth ) {
1457  return self::$RES_ERROR;
1458  }
1459 
1460  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1461  'method' => 'HEAD',
1462  'url' => $this->storageUrl( $auth, $container ),
1463  'headers' => $this->authTokenHeaders( $auth )
1464  ] );
1465 
1466  if ( $rcode === 204 ) {
1467  $stat = [
1468  'count' => $rhdrs['x-container-object-count'],
1469  'bytes' => $rhdrs['x-container-bytes-used']
1470  ];
1471  if ( $bypassCache ) {
1472  return $stat;
1473  } else {
1474  $this->containerStatCache->setField( $container, 'stat', $stat ); // cache it
1475  $this->setContainerCache( $container, $stat ); // update persistent cache
1476  }
1477  } elseif ( $rcode === 404 ) {
1478  return self::$RES_ABSENT;
1479  } else {
1480  $this->onError( null, __METHOD__,
1481  [ 'cont' => $container ], $rerr, $rcode, $rdesc );
1482 
1483  return self::$RES_ERROR;
1484  }
1485  }
1486 
1487  return $this->containerStatCache->getField( $container, 'stat' );
1488  }
1489 
1497  protected function createContainer( $container, array $params ) {
1498  $status = $this->newStatus();
1499 
1500  $auth = $this->getAuthentication();
1501  if ( !$auth ) {
1502  $status->fatal( 'backend-fail-connect', $this->name );
1503 
1504  return $status;
1505  }
1506 
1507  // @see SwiftFileBackend::setContainerAccess()
1508  if ( empty( $params['noAccess'] ) ) {
1509  // public
1510  $readUsers = array_merge( $this->readUsers, [ '.r:*', $this->swiftUser ] );
1511  $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
1512  } else {
1513  // private
1514  $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
1515  $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
1516  }
1517 
1518  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1519  'method' => 'PUT',
1520  'url' => $this->storageUrl( $auth, $container ),
1521  'headers' => $this->authTokenHeaders( $auth ) + [
1522  'x-container-read' => implode( ',', $readUsers ),
1523  'x-container-write' => implode( ',', $writeUsers )
1524  ]
1525  ] );
1526 
1527  if ( $rcode === 201 ) { // new
1528  // good
1529  } elseif ( $rcode === 202 ) { // already there
1530  // this shouldn't really happen, but is OK
1531  } else {
1532  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1533  }
1534 
1535  return $status;
1536  }
1537 
1545  protected function deleteContainer( $container, array $params ) {
1546  $status = $this->newStatus();
1547 
1548  $auth = $this->getAuthentication();
1549  if ( !$auth ) {
1550  $status->fatal( 'backend-fail-connect', $this->name );
1551 
1552  return $status;
1553  }
1554 
1555  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1556  'method' => 'DELETE',
1557  'url' => $this->storageUrl( $auth, $container ),
1558  'headers' => $this->authTokenHeaders( $auth )
1559  ] );
1560 
1561  if ( $rcode >= 200 && $rcode <= 299 ) { // deleted
1562  $this->containerStatCache->clear( $container ); // purge
1563  } elseif ( $rcode === 404 ) { // not there
1564  // this shouldn't really happen, but is OK
1565  } elseif ( $rcode === 409 ) { // not empty
1566  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); // race?
1567  } else {
1568  // @phan-suppress-previous-line PhanPluginDuplicateIfStatements
1569  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1570  }
1571 
1572  return $status;
1573  }
1574 
1587  private function objectListing(
1588  $fullCont, $type, $limit, $after = null, $prefix = null, $delim = null
1589  ) {
1590  $status = $this->newStatus();
1591 
1592  $auth = $this->getAuthentication();
1593  if ( !$auth ) {
1594  $status->fatal( 'backend-fail-connect', $this->name );
1595 
1596  return $status;
1597  }
1598 
1599  $query = [ 'limit' => $limit ];
1600  if ( $type === 'info' ) {
1601  $query['format'] = 'json';
1602  }
1603  if ( $after !== null ) {
1604  $query['marker'] = $after;
1605  }
1606  if ( $prefix !== null ) {
1607  $query['prefix'] = $prefix;
1608  }
1609  if ( $delim !== null ) {
1610  $query['delimiter'] = $delim;
1611  }
1612 
1613  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1614  'method' => 'GET',
1615  'url' => $this->storageUrl( $auth, $fullCont ),
1616  'query' => $query,
1617  'headers' => $this->authTokenHeaders( $auth )
1618  ] );
1619 
1620  $params = [ 'cont' => $fullCont, 'prefix' => $prefix, 'delim' => $delim ];
1621  if ( $rcode === 200 ) { // good
1622  if ( $type === 'info' ) {
1623  $status->value = FormatJson::decode( trim( $rbody ) );
1624  } else {
1625  $status->value = explode( "\n", trim( $rbody ) );
1626  }
1627  } elseif ( $rcode === 204 ) {
1628  $status->value = []; // empty container
1629  } elseif ( $rcode === 404 ) {
1630  $status->value = []; // no container
1631  } else {
1632  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1633  }
1634 
1635  return $status;
1636  }
1637 
1638  protected function doPrimeContainerCache( array $containerInfo ) {
1639  foreach ( $containerInfo as $container => $info ) {
1640  $this->containerStatCache->setField( $container, 'stat', $info );
1641  }
1642  }
1643 
1644  protected function doGetFileStatMulti( array $params ) {
1645  $stats = [];
1646 
1647  $auth = $this->getAuthentication();
1648 
1649  $reqs = []; // (path => op)
1650  // (a) Check the containers of the paths...
1651  foreach ( $params['srcs'] as $path ) {
1652  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
1653  if ( $srcRel === null || !$auth ) {
1654  $stats[$path] = self::$RES_ERROR;
1655  continue; // invalid storage path or auth error
1656  }
1657 
1658  $cstat = $this->getContainerStat( $srcCont );
1659  if ( $cstat === self::$RES_ABSENT ) {
1660  $stats[$path] = self::$RES_ABSENT;
1661  continue; // ok, nothing to do
1662  } elseif ( !is_array( $cstat ) ) {
1663  $stats[$path] = self::$RES_ERROR;
1664  continue;
1665  }
1666 
1667  $reqs[$path] = [
1668  'method' => 'HEAD',
1669  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1670  'headers' => $this->authTokenHeaders( $auth ) + $this->headersFromParams( $params )
1671  ];
1672  }
1673 
1674  // (b) Check the files themselves...
1675  $opts = [ 'maxConnsPerHost' => $params['concurrency'] ];
1676  $reqs = $this->http->runMulti( $reqs, $opts );
1677  foreach ( $reqs as $path => $op ) {
1678  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response'];
1679  if ( $rcode === 200 || $rcode === 204 ) {
1680  // Update the object if it is missing some headers
1681  if ( !empty( $params['requireSHA1'] ) ) {
1682  $rhdrs = $this->addMissingHashMetadata( $rhdrs, $path );
1683  }
1684  // Load the stat array from the headers
1685  $stat = $this->getStatFromHeaders( $rhdrs );
1686  if ( $this->isRGW ) {
1687  $stat['latest'] = true; // strong consistency
1688  }
1689  } elseif ( $rcode === 404 ) {
1690  $stat = self::$RES_ABSENT;
1691  } else {
1692  $stat = self::$RES_ERROR;
1693  $this->onError( null, __METHOD__, $params, $rerr, $rcode, $rdesc );
1694  }
1695  $stats[$path] = $stat;
1696  }
1697 
1698  return $stats;
1699  }
1700 
1705  protected function getStatFromHeaders( array $rhdrs ) {
1706  // Fetch all of the custom metadata headers
1707  $metadata = $this->getMetadataFromHeaders( $rhdrs );
1708  // Fetch all of the custom raw HTTP headers
1709  $headers = $this->extractMutableContentHeaders( $rhdrs );
1710 
1711  return [
1712  // Convert various random Swift dates to TS_MW
1713  'mtime' => $this->convertSwiftDate( $rhdrs['last-modified'], TS_MW ),
1714  // Empty objects actually return no content-length header in Ceph
1715  'size' => isset( $rhdrs['content-length'] ) ? (int)$rhdrs['content-length'] : 0,
1716  'sha1' => $metadata['sha1base36'] ?? null,
1717  // Note: manifiest ETags are not an MD5 of the file
1718  'md5' => ctype_xdigit( $rhdrs['etag'] ) ? $rhdrs['etag'] : null,
1719  'xattr' => [ 'metadata' => $metadata, 'headers' => $headers ]
1720  ];
1721  }
1722 
1726  protected function getAuthentication() {
1727  if ( $this->authErrorTimestamp !== null ) {
1728  if ( ( time() - $this->authErrorTimestamp ) < 60 ) {
1729  return null; // failed last attempt; don't bother
1730  } else { // actually retry this time
1731  $this->authErrorTimestamp = null;
1732  }
1733  }
1734  // Session keys expire after a while, so we renew them periodically
1735  $reAuth = ( ( time() - $this->authSessionTimestamp ) > $this->authTTL );
1736  // Authenticate with proxy and get a session key...
1737  if ( !$this->authCreds || $reAuth ) {
1738  $this->authSessionTimestamp = 0;
1739  $cacheKey = $this->getCredsCacheKey( $this->swiftUser );
1740  $creds = $this->srvCache->get( $cacheKey ); // credentials
1741  // Try to use the credential cache
1742  if ( isset( $creds['auth_token'] ) && isset( $creds['storage_url'] ) ) {
1743  $this->authCreds = $creds;
1744  // Skew the timestamp for worst case to avoid using stale credentials
1745  $this->authSessionTimestamp = time() - ceil( $this->authTTL / 2 );
1746  } else { // cache miss
1747  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1748  'method' => 'GET',
1749  'url' => "{$this->swiftAuthUrl}/v1.0",
1750  'headers' => [
1751  'x-auth-user' => $this->swiftUser,
1752  'x-auth-key' => $this->swiftKey
1753  ]
1754  ] );
1755 
1756  if ( $rcode >= 200 && $rcode <= 299 ) { // OK
1757  $this->authCreds = [
1758  'auth_token' => $rhdrs['x-auth-token'],
1759  'storage_url' => $this->swiftStorageUrl ?? $rhdrs['x-storage-url']
1760  ];
1761 
1762  $this->srvCache->set( $cacheKey, $this->authCreds, ceil( $this->authTTL / 2 ) );
1763  $this->authSessionTimestamp = time();
1764  } elseif ( $rcode === 401 ) {
1765  $this->onError( null, __METHOD__, [], "Authentication failed.", $rcode );
1766  $this->authErrorTimestamp = time();
1767 
1768  return null;
1769  } else {
1770  $this->onError( null, __METHOD__, [], "HTTP return code: $rcode", $rcode );
1771  $this->authErrorTimestamp = time();
1772 
1773  return null;
1774  }
1775  }
1776  // Ceph RGW does not use <account> in URLs (OpenStack Swift uses "/v1/<account>")
1777  if ( substr( $this->authCreds['storage_url'], -3 ) === '/v1' ) {
1778  $this->isRGW = true; // take advantage of strong consistency in Ceph
1779  }
1780  }
1781 
1782  return $this->authCreds;
1783  }
1784 
1791  protected function storageUrl( array $creds, $container = null, $object = null ) {
1792  $parts = [ $creds['storage_url'] ];
1793  if ( strlen( $container ) ) {
1794  $parts[] = rawurlencode( $container );
1795  }
1796  if ( strlen( $object ) ) {
1797  $parts[] = str_replace( "%2F", "/", rawurlencode( $object ) );
1798  }
1799 
1800  return implode( '/', $parts );
1801  }
1802 
1807  protected function authTokenHeaders( array $creds ) {
1808  return [ 'x-auth-token' => $creds['auth_token'] ];
1809  }
1810 
1817  private function getCredsCacheKey( $username ) {
1818  return 'swiftcredentials:' . md5( $username . ':' . $this->swiftAuthUrl );
1819  }
1820 
1832  public function onError( $status, $func, array $params, $err = '', $code = 0, $desc = '' ) {
1833  if ( $status instanceof StatusValue ) {
1834  $status->fatal( 'backend-fail-internal', $this->name );
1835  }
1836  if ( $code == 401 ) { // possibly a stale token
1837  $this->srvCache->delete( $this->getCredsCacheKey( $this->swiftUser ) );
1838  }
1839  $msg = "HTTP {code} ({desc}) in '{func}' (given '{req_params}')";
1840  $msgParams = [
1841  'code' => $code,
1842  'desc' => $desc,
1843  'func' => $func,
1844  'req_params' => FormatJson::encode( $params ),
1845  ];
1846  if ( $err ) {
1847  $msg .= ': {err}';
1848  $msgParams['err'] = $err;
1849  }
1850  $this->logger->error( $msg, $msgParams );
1851  }
1852 }
SwiftFileOpHandle
Definition: SwiftFileOpHandle.php:25
MWTimestamp
Library for creating and parsing MW-style timestamps.
Definition: MWTimestamp.php:34
SwiftFileBackend\doPrepareInternal
doPrepareInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:625
SwiftFileBackend\$isRGW
bool $isRGW
Whether the server is an Ceph RGW.
Definition: SwiftFileBackend.php:78
MultiHttpClient
Class to handle multiple HTTP requests.
Definition: MultiHttpClient.php:55
StatusValue
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Definition: StatusValue.php:43
SwiftFileBackend\isPathUsableInternal
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path in the backend.
Definition: SwiftFileBackend.php:177
SwiftFileBackend\$authErrorTimestamp
int $authErrorTimestamp
UNIX timestamp.
Definition: SwiftFileBackend.php:75
SwiftFileBackend\getAuthentication
getAuthentication()
Definition: SwiftFileBackend.php:1726
SwiftFileBackend\getContainerStat
getContainerStat( $container, $bypassCache=false)
Get a Swift container stat array, possibly from process cache.
Definition: SwiftFileBackend.php:1445
SwiftFileBackend\buildFileObjectListing
buildFileObjectListing(array $objects)
Build a list of file objects, filtering out any directories and extracting any stat info if provided ...
Definition: SwiftFileBackend.php:1052
EmptyBagOStuff
A BagOStuff object with no objects in it.
Definition: EmptyBagOStuff.php:29
SwiftFileBackend\$containerStatCache
MapCacheLRU $containerStatCache
Container stat cache.
Definition: SwiftFileBackend.php:68
SwiftFileBackend\$swiftKey
string $swiftKey
Secret key for user.
Definition: SwiftFileBackend.php:48
SwiftFileBackend\getStatFromHeaders
getStatFromHeaders(array $rhdrs)
Definition: SwiftFileBackend.php:1705
SwiftFileBackend\addMissingHashMetadata
addMissingHashMetadata(array $objHdrs, $path)
Fill in any missing object metadata and save it to Swift.
Definition: SwiftFileBackend.php:761
LockManager\LOCK_UW
const LOCK_UW
Definition: LockManager.php:70
SwiftFileBackend\doGetFileXAttributes
doGetFileXAttributes(array $params)
Definition: SwiftFileBackend.php:1088
FileBackendError
File backend exception for checked exceptions (e.g.
Definition: FileBackendError.php:10
FileBackend\getScopedFileLocks
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
Definition: FileBackend.php:1436
FileBackendStore\fileExists
fileExists(array $params)
Check if a file exists at a storage path in the backend.
Definition: FileBackendStore.php:644
SwiftFileBackend\$swiftUser
string $swiftUser
Swift user (account:user) to authenticate as.
Definition: SwiftFileBackend.php:46
SwiftFileBackend\loadListingStatInternal
loadListingStatInternal( $path, array $val)
Do not call this function outside of SwiftFileBackendFileList.
Definition: SwiftFileBackend.php:1084
FileBackend\extensionFromPath
static extensionFromPath( $path, $case='lowercase')
Get the final extension from a storage or FS path.
Definition: FileBackend.php:1602
FileBackendStore\deleteFileCache
deleteFileCache( $path)
Delete the cached stat info for a file path.
Definition: FileBackendStore.php:1910
SwiftFileBackend\resolveContainerPath
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
Definition: SwiftFileBackend.php:167
FileBackendStore\executeOpHandlesInternal
executeOpHandlesInternal(array $fileOpHandles)
Execute a list of FileBackendStoreOpHandle handles in parallel.
Definition: FileBackendStore.php:1402
BagOStuff
Class representing a cache/ephemeral data store.
Definition: BagOStuff.php:71
$s
$s
Definition: mergeMessageFileList.php:185
SwiftFileBackend\doGetFileContentsMulti
doGetFileContentsMulti(array $params)
Definition: SwiftFileBackend.php:815
SwiftFileBackend\createContainer
createContainer( $container, array $params)
Create a Swift container.
Definition: SwiftFileBackend.php:1497
SwiftFileBackend\extractMetadataHeaders
extractMetadataHeaders(array $headers)
Definition: SwiftFileBackend.php:230
SwiftFileBackend\getFileListPageInternal
getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
Definition: SwiftFileBackend.php:999
SwiftFileBackend\getMetadataFromHeaders
getMetadataFromHeaders(array $headers)
Definition: SwiftFileBackend.php:247
SwiftFileBackend\doGetFileSha1base36
doGetFileSha1base36(array $params)
Definition: SwiftFileBackend.php:1103
FileBackendStore\clearCache
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
Definition: FileBackendStore.php:1484
SwiftFileBackend\getFeatures
getFeatures()
Get the a bitfield of extra features supported by the backend medium Stable to override.
Definition: SwiftFileBackend.php:159
FileBackendStore\$RES_ERROR
static null $RES_ERROR
Idiom for "no result due to I/O errors" (since 1.34)
Definition: FileBackendStore.php:66
SwiftFileBackend\doDeleteInternal
doDeleteInternal(array $params)
Definition: SwiftFileBackend.php:524
SwiftFileBackend\headersFromParams
headersFromParams(array $params)
Get headers to send to Swift when reading a file based on a FileBackend params array,...
Definition: SwiftFileBackend.php:1320
SwiftFileBackend\deleteContainer
deleteContainer( $container, array $params)
Delete a Swift container.
Definition: SwiftFileBackend.php:1545
SwiftFileBackend\getCredsCacheKey
getCredsCacheKey( $username)
Get the cache key for a container.
Definition: SwiftFileBackend.php:1817
FormatJson\decode
static decode( $value, $assoc=false)
Decodes a JSON string.
Definition: FormatJson.php:174
FormatJson\encode
static encode( $value, $pretty=false, $escaping=0)
Returns the JSON representation of a value.
Definition: FormatJson.php:115
SwiftFileBackend\doDirectoryExists
doDirectoryExists( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:873
SwiftFileBackend
Class for an OpenStack Swift (or Ceph RGW) based file backend.
Definition: SwiftFileBackend.php:36
SwiftFileBackend\getDirectoryListInternal
getDirectoryListInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:890
SwiftFileBackend\setContainerAccess
setContainerAccess( $container, array $readUsers, array $writeUsers)
Set read/write permissions for a Swift container.
Definition: SwiftFileBackend.php:1409
SwiftFileBackend\getFileHttpUrl
getFileHttpUrl(array $params)
Definition: SwiftFileBackend.php:1254
SwiftFileBackend\doMoveInternal
doMoveInternal(array $params)
Definition: SwiftFileBackend.php:457
MapCacheLRU
Handles a simple LRU key/value map with a maximum number of entries.
Definition: MapCacheLRU.php:38
SwiftFileBackend\$authCreds
array $authCreds
Definition: SwiftFileBackend.php:71
$dirs
$dirs
Definition: mergeMessageFileList.php:192
SwiftFileBackend\extractMutableContentHeaders
extractMutableContentHeaders(array $headers)
Filter/normalize a header map to only include mutable "content-"/"x-content-" headers.
Definition: SwiftFileBackend.php:195
SwiftFileBackend\doGetLocalCopyMulti
doGetLocalCopyMulti(array $params)
Definition: SwiftFileBackend.php:1182
SwiftFileBackend\onError
onError( $status, $func, array $params, $err='', $code=0, $desc='')
Log an unexpected exception for this backend.
Definition: SwiftFileBackend.php:1832
SwiftFileBackend\$secureWriteUsers
array $secureWriteUsers
Additional users (account:user) with write permissions on private containers.
Definition: SwiftFileBackend.php:62
SwiftFileBackend\doDescribeInternal
doDescribeInternal(array $params)
Definition: SwiftFileBackend.php:566
SwiftFileBackend\storageUrl
storageUrl(array $creds, $container=null, $object=null)
Definition: SwiftFileBackend.php:1791
FileBackendStore\getContentType
getContentType( $storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file Stable to override.
Definition: FileBackendStore.php:2032
FileBackend\$obResetFunc
callable $obResetFunc
Definition: FileBackend.php:127
SwiftFileBackendDirList
Iterator for listing directories.
Definition: SwiftFileBackendDirList.php:28
$content
$content
Definition: router.php:76
SwiftFileBackend\doSecureInternal
doSecureInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:648
FileBackendStore\getFileStat
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
Definition: FileBackendStore.php:680
$header
$header
Definition: updateCredits.php:41
SwiftFileBackend\__construct
__construct(array $config)
Definition: SwiftFileBackend.php:118
SwiftFileBackend\directoriesAreVirtual
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
Definition: SwiftFileBackend.php:1308
FileBackendStore\resolveStoragePathReal
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
Definition: FileBackendStore.php:1642
SwiftFileBackend\authTokenHeaders
authTokenHeaders(array $creds)
Definition: SwiftFileBackend.php:1807
WANObjectCache
Multi-datacenter aware caching interface.
Definition: WANObjectCache.php:125
SwiftFileBackend\doCreateInternal
doCreateInternal(array $params)
Definition: SwiftFileBackend.php:258
SwiftFileBackend\$authSessionTimestamp
int $authSessionTimestamp
UNIX timestamp.
Definition: SwiftFileBackend.php:73
FileBackendStore
Base class for all backends using particular storage medium.
Definition: FileBackendStore.php:41
HTTPFileStreamer\STREAM_HEADLESS
const STREAM_HEADLESS
Definition: HTTPFileStreamer.php:40
FileBackendStore\$RES_ABSENT
static false $RES_ABSENT
Idiom for "no result due to missing file" (since 1.34)
Definition: FileBackendStore.php:64
SwiftFileBackend\doExecuteOpHandlesInternal
doExecuteOpHandlesInternal(array $fileOpHandles)
Definition: SwiftFileBackend.php:1329
SwiftFileBackend\$swiftAuthUrl
string $swiftAuthUrl
Authentication base URL (without version)
Definition: SwiftFileBackend.php:42
SwiftFileBackend\doStreamFile
doStreamFile(array $params)
Definition: SwiftFileBackend.php:1116
HTTPFileStreamer\send404Message
static send404Message( $fname, $flags=0)
Send out a standard 404 message for a file.
Definition: HTTPFileStreamer.php:203
FileBackend\newStatus
newStatus(... $args)
Yields the result of the status wrapper callback on either:
Definition: FileBackend.php:1695
SwiftFileBackend\$writeUsers
array $writeUsers
Additional users (account:user) with write permissions on public containers.
Definition: SwiftFileBackend.php:58
SwiftFileBackend\$rgwS3SecretKey
string $rgwS3SecretKey
S3 authentication key (RADOS Gateway)
Definition: SwiftFileBackend.php:54
FileBackendStore\primeContainerCache
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
Definition: FileBackendStore.php:1828
SwiftFileBackend\$authTTL
int $authTTL
TTL in seconds.
Definition: SwiftFileBackend.php:40
SwiftFileBackend\$readUsers
array $readUsers
Additional users (account:user) with read permissions on public containers.
Definition: SwiftFileBackend.php:56
SwiftFileBackend\doCopyInternal
doCopyInternal(array $params)
Definition: SwiftFileBackend.php:402
FileBackend\$name
string $name
Unique backend name.
Definition: FileBackend.php:100
SwiftFileBackend\$rgwS3AccessKey
string $rgwS3AccessKey
S3 access key (RADOS Gateway)
Definition: SwiftFileBackend.php:52
FileBackend\getLocalCopy
getLocalCopy(array $params)
Get a local copy on disk of the file at a storage path in the backend.
Definition: FileBackend.php:1198
SwiftFileBackend\objectListing
objectListing( $fullCont, $type, $limit, $after=null, $prefix=null, $delim=null)
Get a list of objects under a container.
Definition: SwiftFileBackend.php:1587
SwiftFileBackend\$swiftStorageUrl
string $swiftStorageUrl
Override of storage base URL.
Definition: SwiftFileBackend.php:44
$path
$path
Definition: NoLocalSettings.php:25
SwiftFileBackend\convertSwiftDate
convertSwiftDate( $ts, $format=TS_MW)
Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT"/"2013-05-11T07:37:27.678360Z".
Definition: SwiftFileBackend.php:744
SwiftFileBackend\doPublishInternal
doPublishInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:674
SwiftFileBackend\doGetFileStat
doGetFileStat(array $params)
Definition: SwiftFileBackend.php:726
Wikimedia
This program is free software; you can redistribute it and/or modify it under the terms of the GNU Ge...
SwiftFileBackend\$swiftTempUrlKey
string $swiftTempUrlKey
Shared secret value for making temp URLs.
Definition: SwiftFileBackend.php:50
SwiftFileBackend\doStoreInternal
doStoreInternal(array $params)
Definition: SwiftFileBackend.php:313
SwiftFileBackend\doGetFileStatMulti
doGetFileStatMulti(array $params)
Get file stat information (concurrently if possible) for several files Stable to override.
Definition: SwiftFileBackend.php:1644
$ext
if(!is_readable( $file)) $ext
Definition: router.php:48
SwiftFileBackend\doCleanInternal
doCleanInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:698
SwiftFileBackend\$srvCache
BagOStuff $srvCache
Definition: SwiftFileBackend.php:65
SwiftFileBackendFileList
Iterator for listing regular files.
Definition: SwiftFileBackendFileList.php:28
SwiftFileBackend\getFileListInternal
getFileListInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:901
SwiftFileBackend\doPrimeContainerCache
doPrimeContainerCache(array $containerInfo)
Fill the backend-specific process cache given an array of resolved container names and their correspo...
Definition: SwiftFileBackend.php:1638
SwiftFileBackend\$secureReadUsers
array $secureReadUsers
Additional users (account:user) with read permissions on private containers.
Definition: SwiftFileBackend.php:60
FileBackendStore\setContainerCache
setContainerCache( $container, array $val)
Set the cached info for a container.
Definition: FileBackendStore.php:1803
FileBackend\scopedProfileSection
scopedProfileSection( $section)
Definition: FileBackend.php:1717
SwiftFileBackend\getDirListPageInternal
getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
Definition: SwiftFileBackend.php:916
SwiftFileBackend\$http
MultiHttpClient $http
Definition: SwiftFileBackend.php:38
$type
$type
Definition: testCompression.php:52