MediaWiki  master
SwiftFileBackend.php
Go to the documentation of this file.
1 <?php
25 use Wikimedia\AtEase\AtEase;
26 
38  protected $http;
40  protected $authTTL;
42  protected $swiftAuthUrl;
44  protected $swiftStorageUrl;
46  protected $swiftUser;
48  protected $swiftKey;
50  protected $swiftTempUrlKey;
52  protected $rgwS3AccessKey;
54  protected $rgwS3SecretKey;
56  protected $readUsers;
58  protected $writeUsers;
60  protected $secureReadUsers;
62  protected $secureWriteUsers;
63 
65  protected $srvCache;
66 
69 
71  protected $authCreds;
73  protected $authSessionTimestamp = 0;
75  protected $authErrorTimestamp = null;
76 
78  protected $isRGW = false;
79 
118  public function __construct( array $config ) {
119  parent::__construct( $config );
120  // Required settings
121  $this->swiftAuthUrl = $config['swiftAuthUrl'];
122  $this->swiftUser = $config['swiftUser'];
123  $this->swiftKey = $config['swiftKey'];
124  // Optional settings
125  $this->authTTL = $config['swiftAuthTTL'] ?? 15 * 60; // some sane number
126  $this->swiftTempUrlKey = $config['swiftTempUrlKey'] ?? '';
127  $this->swiftStorageUrl = $config['swiftStorageUrl'] ?? null;
128  $this->shardViaHashLevels = $config['shardViaHashLevels'] ?? '';
129  $this->rgwS3AccessKey = $config['rgwS3AccessKey'] ?? '';
130  $this->rgwS3SecretKey = $config['rgwS3SecretKey'] ?? '';
131 
132  // HTTP helper client
133  $httpOptions = [];
134  foreach ( [ 'connTimeout', 'reqTimeout' ] as $optionName ) {
135  if ( isset( $config[$optionName] ) ) {
136  $httpOptions[$optionName] = $config[$optionName];
137  }
138  }
139  $this->http = new MultiHttpClient( $httpOptions );
140 
141  // Cache container information to mask latency
142  if ( isset( $config['wanCache'] ) && $config['wanCache'] instanceof WANObjectCache ) {
143  $this->memCache = $config['wanCache'];
144  }
145  // Process cache for container info
146  $this->containerStatCache = new MapCacheLRU( 300 );
147  // Cache auth token information to avoid RTTs
148  if ( !empty( $config['cacheAuthInfo'] ) && isset( $config['srvCache'] ) ) {
149  $this->srvCache = $config['srvCache'];
150  } else {
151  $this->srvCache = new EmptyBagOStuff();
152  }
153  $this->readUsers = $config['readUsers'] ?? [];
154  $this->writeUsers = $config['writeUsers'] ?? [];
155  $this->secureReadUsers = $config['secureReadUsers'] ?? [];
156  $this->secureWriteUsers = $config['secureWriteUsers'] ?? [];
157  }
158 
159  public function getFeatures() {
160  return (
161  self::ATTR_UNICODE_PATHS |
162  self::ATTR_HEADERS |
163  self::ATTR_METADATA
164  );
165  }
166 
167  protected function resolveContainerPath( $container, $relStoragePath ) {
168  if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) {
169  return null; // not UTF-8, makes it hard to use CF and the swift HTTP API
170  } elseif ( strlen( rawurlencode( $relStoragePath ) ) > 1024 ) {
171  return null; // too long for Swift
172  }
173 
174  return $relStoragePath;
175  }
176 
177  public function isPathUsableInternal( $storagePath ) {
178  list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
179  if ( $rel === null ) {
180  return false; // invalid
181  }
182 
183  return is_array( $this->getContainerStat( $container ) );
184  }
185 
195  protected function extractMutableContentHeaders( array $headers ) {
196  $contentHeaders = [];
197  // Normalize casing, and strip out illegal headers
198  foreach ( $headers as $name => $value ) {
199  $name = strtolower( $name );
200  if ( !preg_match( '/^(x-)?content-(?!length$)/', $name ) ) {
201  // Only allow content-* and x-content-* headers (but not content-length)
202  continue;
203  } elseif ( $name === 'content-type' && !strlen( $value ) ) {
204  // This header can be set to a value but not unset for sanity
205  continue;
206  }
207  $contentHeaders[$name] = $value;
208  }
209  // By default, Swift has annoyingly low maximum header value limits
210  if ( isset( $contentHeaders['content-disposition'] ) ) {
211  $disposition = '';
212  // @note: assume FileBackend::makeContentDisposition() already used
213  foreach ( explode( ';', $contentHeaders['content-disposition'] ) as $part ) {
214  $part = trim( $part );
215  $new = ( $disposition === '' ) ? $part : "{$disposition};{$part}";
216  if ( strlen( $new ) <= 255 ) {
217  $disposition = $new;
218  } else {
219  break; // too long; sigh
220  }
221  }
222  $contentHeaders['content-disposition'] = $disposition;
223  }
224 
225  return $contentHeaders;
226  }
227 
233  protected function extractMetadataHeaders( array $headers ) {
234  $metadataHeaders = [];
235  foreach ( $headers as $name => $value ) {
236  $name = strtolower( $name );
237  if ( strpos( $name, 'x-object-meta-' ) === 0 ) {
238  $metadataHeaders[$name] = $value;
239  }
240  }
241 
242  return $metadataHeaders;
243  }
244 
250  protected function getMetadataFromHeaders( array $headers ) {
251  $prefixLen = strlen( 'x-object-meta-' );
252 
253  $metadata = [];
254  foreach ( $this->extractMetadataHeaders( $headers ) as $name => $value ) {
255  $metadata[substr( $name, $prefixLen )] = $value;
256  }
257 
258  return $metadata;
259  }
260 
261  protected function doCreateInternal( array $params ) {
262  $status = $this->newStatus();
263 
264  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
265  if ( $dstRel === null ) {
266  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
267 
268  return $status;
269  }
270 
271  // Headers that are not strictly a function of the file content
272  $mutableHeaders = $this->extractMutableContentHeaders( $params['headers'] ?? [] );
273  // Make sure that the "content-type" header is set to something sensible
274  $mutableHeaders['content-type'] = $mutableHeaders['content-type']
275  ?? $this->getContentType( $params['dst'], $params['content'], null );
276 
277  $reqs = [ [
278  'method' => 'PUT',
279  'url' => [ $dstCont, $dstRel ],
280  'headers' => array_merge(
281  $mutableHeaders,
282  [
283  'etag' => md5( $params['content'] ),
284  'content-length' => strlen( $params['content'] ),
285  'x-object-meta-sha1base36' =>
286  Wikimedia\base_convert( sha1( $params['content'] ), 16, 36, 31 )
287  ]
288  ),
289  'body' => $params['content']
290  ] ];
291 
292  $method = __METHOD__;
293  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
294  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
295  if ( $rcode === 201 || $rcode === 202 ) {
296  // good
297  } elseif ( $rcode === 412 ) {
298  $status->fatal( 'backend-fail-contenttype', $params['dst'] );
299  } else {
300  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
301  }
302 
303  return SwiftFileOpHandle::CONTINUE_IF_OK;
304  };
305 
306  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
307  if ( !empty( $params['async'] ) ) { // deferred
308  $status->value = $opHandle;
309  } else { // actually write the object in Swift
310  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
311  }
312 
313  return $status;
314  }
315 
316  protected function doStoreInternal( array $params ) {
317  $status = $this->newStatus();
318 
319  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
320  if ( $dstRel === null ) {
321  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
322 
323  return $status;
324  }
325 
326  // Open a handle to the source file so that it can be streamed. The size and hash
327  // will be computed using the handle. In the off chance that the source file changes
328  // during this operation, the PUT will fail due to an ETag mismatch and be aborted.
329  AtEase::suppressWarnings();
330  $srcHandle = fopen( $params['src'], 'rb' );
331  AtEase::restoreWarnings();
332  if ( $srcHandle === false ) { // source doesn't exist?
333  $status->fatal( 'backend-fail-notexists', $params['src'] );
334 
335  return $status;
336  }
337 
338  // Compute the MD5 and SHA-1 hashes in one pass
339  $srcSize = fstat( $srcHandle )['size'];
340  $md5Context = hash_init( 'md5' );
341  $sha1Context = hash_init( 'sha1' );
342  $hashDigestSize = 0;
343  while ( !feof( $srcHandle ) ) {
344  $buffer = (string)fread( $srcHandle, 131072 ); // 128 KiB
345  hash_update( $md5Context, $buffer );
346  hash_update( $sha1Context, $buffer );
347  $hashDigestSize += strlen( $buffer );
348  }
349  // Reset the handle back to the beginning so that it can be streamed
350  rewind( $srcHandle );
351 
352  if ( $hashDigestSize !== $srcSize ) {
353  $status->fatal( 'backend-fail-hash', $params['src'] );
354 
355  return $status;
356  }
357 
358  // Headers that are not strictly a function of the file content
359  $mutableHeaders = $this->extractMutableContentHeaders( $params['headers'] ?? [] );
360  // Make sure that the "content-type" header is set to something sensible
361  $mutableHeaders['content-type'] = $mutableHeaders['content-type']
362  ?? $this->getContentType( $params['dst'], null, $params['src'] );
363 
364  $reqs = [ [
365  'method' => 'PUT',
366  'url' => [ $dstCont, $dstRel ],
367  'headers' => array_merge(
368  $mutableHeaders,
369  [
370  'content-length' => $srcSize,
371  'etag' => hash_final( $md5Context ),
372  'x-object-meta-sha1base36' =>
373  Wikimedia\base_convert( hash_final( $sha1Context ), 16, 36, 31 )
374  ]
375  ),
376  'body' => $srcHandle // resource
377  ] ];
378 
379  $method = __METHOD__;
380  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
381  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
382  if ( $rcode === 201 || $rcode === 202 ) {
383  // good
384  } elseif ( $rcode === 412 ) {
385  $status->fatal( 'backend-fail-contenttype', $params['dst'] );
386  } else {
387  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
388  }
389 
390  return SwiftFileOpHandle::CONTINUE_IF_OK;
391  };
392 
393  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
394  $opHandle->resourcesToClose[] = $srcHandle;
395 
396  if ( !empty( $params['async'] ) ) { // deferred
397  $status->value = $opHandle;
398  } else { // actually write the object in Swift
399  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
400  }
401 
402  return $status;
403  }
404 
405  protected function doCopyInternal( array $params ) {
406  $status = $this->newStatus();
407 
408  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
409  if ( $srcRel === null ) {
410  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
411 
412  return $status;
413  }
414 
415  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
416  if ( $dstRel === null ) {
417  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
418 
419  return $status;
420  }
421 
422  $reqs = [ [
423  'method' => 'PUT',
424  'url' => [ $dstCont, $dstRel ],
425  'headers' => array_merge(
426  $this->extractMutableContentHeaders( $params['headers'] ?? [] ),
427  [
428  'x-copy-from' => '/' . rawurlencode( $srcCont ) . '/' .
429  str_replace( "%2F", "/", rawurlencode( $srcRel ) )
430  ]
431  )
432  ] ];
433 
434  $method = __METHOD__;
435  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
436  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
437  if ( $rcode === 201 ) {
438  // good
439  } elseif ( $rcode === 404 ) {
440  if ( empty( $params['ignoreMissingSource'] ) ) {
441  $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
442  }
443  } else {
444  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
445  }
446 
447  return SwiftFileOpHandle::CONTINUE_IF_OK;
448  };
449 
450  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
451  if ( !empty( $params['async'] ) ) { // deferred
452  $status->value = $opHandle;
453  } else { // actually write the object in Swift
454  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
455  }
456 
457  return $status;
458  }
459 
460  protected function doMoveInternal( array $params ) {
461  $status = $this->newStatus();
462 
463  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
464  if ( $srcRel === null ) {
465  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
466 
467  return $status;
468  }
469 
470  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
471  if ( $dstRel === null ) {
472  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
473 
474  return $status;
475  }
476 
477  $reqs = [ [
478  'method' => 'PUT',
479  'url' => [ $dstCont, $dstRel ],
480  'headers' => array_merge(
481  $this->extractMutableContentHeaders( $params['headers'] ?? [] ),
482  [
483  'x-copy-from' => '/' . rawurlencode( $srcCont ) . '/' .
484  str_replace( "%2F", "/", rawurlencode( $srcRel ) )
485  ]
486  )
487  ] ];
488  if ( "{$srcCont}/{$srcRel}" !== "{$dstCont}/{$dstRel}" ) {
489  $reqs[] = [
490  'method' => 'DELETE',
491  'url' => [ $srcCont, $srcRel ],
492  'headers' => []
493  ];
494  }
495 
496  $method = __METHOD__;
497  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
498  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
499  if ( $request['method'] === 'PUT' && $rcode === 201 ) {
500  // good
501  } elseif ( $request['method'] === 'DELETE' && $rcode === 204 ) {
502  // good
503  } elseif ( $rcode === 404 ) {
504  if ( empty( $params['ignoreMissingSource'] ) ) {
505  $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
506  } else {
507  // Leave Status as OK but skip the DELETE request
508  return SwiftFileOpHandle::CONTINUE_NO;
509  }
510  } else {
511  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
512  }
513 
514  return SwiftFileOpHandle::CONTINUE_IF_OK;
515  };
516 
517  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
518  if ( !empty( $params['async'] ) ) { // deferred
519  $status->value = $opHandle;
520  } else { // actually move the object in Swift
521  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
522  }
523 
524  return $status;
525  }
526 
527  protected function doDeleteInternal( array $params ) {
528  $status = $this->newStatus();
529 
530  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
531  if ( $srcRel === null ) {
532  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
533 
534  return $status;
535  }
536 
537  $reqs = [ [
538  'method' => 'DELETE',
539  'url' => [ $srcCont, $srcRel ],
540  'headers' => []
541  ] ];
542 
543  $method = __METHOD__;
544  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
545  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
546  if ( $rcode === 204 ) {
547  // good
548  } elseif ( $rcode === 404 ) {
549  if ( empty( $params['ignoreMissingSource'] ) ) {
550  $status->fatal( 'backend-fail-delete', $params['src'] );
551  }
552  } else {
553  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
554  }
555 
556  return SwiftFileOpHandle::CONTINUE_IF_OK;
557  };
558 
559  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
560  if ( !empty( $params['async'] ) ) { // deferred
561  $status->value = $opHandle;
562  } else { // actually delete the object in Swift
563  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
564  }
565 
566  return $status;
567  }
568 
569  protected function doDescribeInternal( array $params ) {
570  $status = $this->newStatus();
571 
572  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
573  if ( $srcRel === null ) {
574  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
575 
576  return $status;
577  }
578 
579  // Fetch the old object headers/metadata...this should be in stat cache by now
580  $stat = $this->getFileStat( [ 'src' => $params['src'], 'latest' => 1 ] );
581  if ( $stat && !isset( $stat['xattr'] ) ) { // older cache entry
582  $stat = $this->doGetFileStat( [ 'src' => $params['src'], 'latest' => 1 ] );
583  }
584  if ( !$stat ) {
585  $status->fatal( 'backend-fail-describe', $params['src'] );
586 
587  return $status;
588  }
589 
590  // Swift object POST clears any prior headers, so merge the new and old headers here.
591  // Also, during, POST, libcurl adds "Content-Type: application/x-www-form-urlencoded"
592  // if "Content-Type" is not set, which would clobber the header value for the object.
593  $oldMetadataHeaders = [];
594  foreach ( $stat['xattr']['metadata'] as $name => $value ) {
595  $oldMetadataHeaders["x-object-meta-$name"] = $value;
596  }
597  $newContentHeaders = $this->extractMutableContentHeaders( $params['headers'] ?? [] );
598  $oldContentHeaders = $stat['xattr']['headers'];
599 
600  $reqs = [ [
601  'method' => 'POST',
602  'url' => [ $srcCont, $srcRel ],
603  'headers' => $oldMetadataHeaders + $newContentHeaders + $oldContentHeaders
604  ] ];
605 
606  $method = __METHOD__;
607  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
608  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
609  if ( $rcode === 202 ) {
610  // good
611  } elseif ( $rcode === 404 ) {
612  $status->fatal( 'backend-fail-describe', $params['src'] );
613  } else {
614  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
615  }
616  };
617 
618  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
619  if ( !empty( $params['async'] ) ) { // deferred
620  $status->value = $opHandle;
621  } else { // actually change the object in Swift
622  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
623  }
624 
625  return $status;
626  }
627 
628  protected function doPrepareInternal( $fullCont, $dir, array $params ) {
629  $status = $this->newStatus();
630 
631  // (a) Check if container already exists
632  $stat = $this->getContainerStat( $fullCont );
633  if ( is_array( $stat ) ) {
634  return $status; // already there
635  } elseif ( $stat === self::$RES_ERROR ) {
636  $status->fatal( 'backend-fail-internal', $this->name );
637  $this->logger->error( __METHOD__ . ': cannot get container stat' );
638 
639  return $status;
640  }
641 
642  // (b) Create container as needed with proper ACLs
643  if ( $stat === false ) {
644  $params['op'] = 'prepare';
645  $status->merge( $this->createContainer( $fullCont, $params ) );
646  }
647 
648  return $status;
649  }
650 
651  protected function doSecureInternal( $fullCont, $dir, array $params ) {
652  $status = $this->newStatus();
653  if ( empty( $params['noAccess'] ) ) {
654  return $status; // nothing to do
655  }
656 
657  $stat = $this->getContainerStat( $fullCont );
658  if ( is_array( $stat ) ) {
659  $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
660  $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
661  // Make container private to end-users...
662  $status->merge( $this->setContainerAccess(
663  $fullCont,
664  $readUsers,
666  ) );
667  } elseif ( $stat === false ) {
668  $status->fatal( 'backend-fail-usable', $params['dir'] );
669  } else {
670  $status->fatal( 'backend-fail-internal', $this->name );
671  $this->logger->error( __METHOD__ . ': cannot get container stat' );
672  }
673 
674  return $status;
675  }
676 
677  protected function doPublishInternal( $fullCont, $dir, array $params ) {
678  $status = $this->newStatus();
679 
680  $stat = $this->getContainerStat( $fullCont );
681  if ( is_array( $stat ) ) {
682  $readUsers = array_merge( $this->readUsers, [ $this->swiftUser, '.r:*' ] );
683  $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
684 
685  // Make container public to end-users...
686  $status->merge( $this->setContainerAccess(
687  $fullCont,
688  $readUsers,
690  ) );
691  } elseif ( $stat === false ) {
692  $status->fatal( 'backend-fail-usable', $params['dir'] );
693  } else {
694  $status->fatal( 'backend-fail-internal', $this->name );
695  $this->logger->error( __METHOD__ . ': cannot get container stat' );
696  }
697 
698  return $status;
699  }
700 
701  protected function doCleanInternal( $fullCont, $dir, array $params ) {
702  $status = $this->newStatus();
703 
704  // Only containers themselves can be removed, all else is virtual
705  if ( $dir != '' ) {
706  return $status; // nothing to do
707  }
708 
709  // (a) Check the container
710  $stat = $this->getContainerStat( $fullCont, true );
711  if ( $stat === false ) {
712  return $status; // ok, nothing to do
713  } elseif ( !is_array( $stat ) ) {
714  $status->fatal( 'backend-fail-internal', $this->name );
715  $this->logger->error( __METHOD__ . ': cannot get container stat' );
716 
717  return $status;
718  }
719 
720  // (b) Delete the container if empty
721  if ( $stat['count'] == 0 ) {
722  $params['op'] = 'clean';
723  $status->merge( $this->deleteContainer( $fullCont, $params ) );
724  }
725 
726  return $status;
727  }
728 
729  protected function doGetFileStat( array $params ) {
730  $params = [ 'srcs' => [ $params['src'] ], 'concurrency' => 1 ] + $params;
731  unset( $params['src'] );
732  $stats = $this->doGetFileStatMulti( $params );
733 
734  return reset( $stats );
735  }
736 
747  protected function convertSwiftDate( $ts, $format = TS_MW ) {
748  try {
749  $timestamp = new MWTimestamp( $ts );
750 
751  return $timestamp->getTimestamp( $format );
752  } catch ( Exception $e ) {
753  throw new FileBackendError( $e->getMessage() );
754  }
755  }
756 
764  protected function addMissingHashMetadata( array $objHdrs, $path ) {
765  if ( isset( $objHdrs['x-object-meta-sha1base36'] ) ) {
766  return $objHdrs; // nothing to do
767  }
768 
770  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
771  $this->logger->error( __METHOD__ . ": {path} was not stored with SHA-1 metadata.",
772  [ 'path' => $path ] );
773 
774  $objHdrs['x-object-meta-sha1base36'] = false;
775 
776  $auth = $this->getAuthentication();
777  if ( !$auth ) {
778  return $objHdrs; // failed
779  }
780 
781  // Find prior custom HTTP headers
782  $postHeaders = $this->extractMutableContentHeaders( $objHdrs );
783  // Find prior metadata headers
784  $postHeaders += $this->extractMetadataHeaders( $objHdrs );
785 
786  $status = $this->newStatus();
788  $scopeLockS = $this->getScopedFileLocks( [ $path ], LockManager::LOCK_UW, $status );
789  if ( $status->isOK() ) {
790  $tmpFile = $this->getLocalCopy( [ 'src' => $path, 'latest' => 1 ] );
791  if ( $tmpFile ) {
792  $hash = $tmpFile->getSha1Base36();
793  if ( $hash !== false ) {
794  $objHdrs['x-object-meta-sha1base36'] = $hash;
795  // Merge new SHA1 header into the old ones
796  $postHeaders['x-object-meta-sha1base36'] = $hash;
797  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
798  list( $rcode ) = $this->http->run( [
799  'method' => 'POST',
800  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
801  'headers' => $this->authTokenHeaders( $auth ) + $postHeaders
802  ] );
803  if ( $rcode >= 200 && $rcode <= 299 ) {
804  $this->deleteFileCache( $path );
805 
806  return $objHdrs; // success
807  }
808  }
809  }
810  }
811 
812  $this->logger->error( __METHOD__ . ': unable to set SHA-1 metadata for {path}',
813  [ 'path' => $path ] );
814 
815  return $objHdrs; // failed
816  }
817 
818  protected function doGetFileContentsMulti( array $params ) {
819  $auth = $this->getAuthentication();
820 
821  $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging
822  // Blindly create tmp files and stream to them, catching any exception
823  // if the file does not exist. Do not waste time doing file stats here.
824  $reqs = []; // (path => op)
825 
826  // Initial dummy values to preserve path order
827  $contents = array_fill_keys( $params['srcs'], self::$RES_ERROR );
828  foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch
829  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
830  if ( $srcRel === null || !$auth ) {
831  continue; // invalid storage path or auth error
832  }
833  // Create a new temporary memory file...
834  $handle = fopen( 'php://temp', 'wb' );
835  if ( $handle ) {
836  $reqs[$path] = [
837  'method' => 'GET',
838  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
839  'headers' => $this->authTokenHeaders( $auth )
840  + $this->headersFromParams( $params ),
841  'stream' => $handle,
842  ];
843  }
844  }
845 
846  $opts = [ 'maxConnsPerHost' => $params['concurrency'] ];
847  $reqs = $this->http->runMulti( $reqs, $opts );
848  foreach ( $reqs as $path => $op ) {
849  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response'];
850  if ( $rcode >= 200 && $rcode <= 299 ) {
851  rewind( $op['stream'] ); // start from the beginning
852  $content = (string)stream_get_contents( $op['stream'] );
853  $size = strlen( $content );
854  // Make sure that stream finished
855  if ( $size === (int)$rhdrs['content-length'] ) {
856  $contents[$path] = $content;
857  } else {
858  $contents[$path] = self::$RES_ERROR;
859  $rerr = "Got {$size}/{$rhdrs['content-length']} bytes";
860  $this->onError( null, __METHOD__,
861  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
862  }
863  } elseif ( $rcode === 404 ) {
864  $contents[$path] = self::$RES_ABSENT;
865  } else {
866  $contents[$path] = self::$RES_ERROR;
867  $this->onError( null, __METHOD__,
868  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
869  }
870  fclose( $op['stream'] ); // close open handle
871  }
872 
873  return $contents;
874  }
875 
876  protected function doDirectoryExists( $fullCont, $dir, array $params ) {
877  $prefix = ( $dir == '' ) ? null : "{$dir}/";
878  $status = $this->objectListing( $fullCont, 'names', 1, null, $prefix );
879  if ( $status->isOK() ) {
880  return ( count( $status->value ) ) > 0;
881  }
882 
883  return self::$RES_ERROR;
884  }
885 
893  public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
894  return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
895  }
896 
904  public function getFileListInternal( $fullCont, $dir, array $params ) {
905  return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
906  }
907 
919  public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
920  $dirs = [];
921  if ( $after === INF ) {
922  return $dirs; // nothing more
923  }
924 
926  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
927 
928  $prefix = ( $dir == '' ) ? null : "{$dir}/";
929  // Non-recursive: only list dirs right under $dir
930  if ( !empty( $params['topOnly'] ) ) {
931  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
932  if ( !$status->isOK() ) {
933  throw new FileBackendError( "Iterator page I/O error." );
934  }
935  $objects = $status->value;
936  // @phan-suppress-next-line PhanTypeSuspiciousNonTraversableForeach
937  foreach ( $objects as $object ) { // files and directories
938  if ( substr( $object, -1 ) === '/' ) {
939  $dirs[] = $object; // directories end in '/'
940  }
941  }
942  } else {
943  // Recursive: list all dirs under $dir and its subdirs
944  $getParentDir = function ( $path ) {
945  return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
946  };
947 
948  // Get directory from last item of prior page
949  $lastDir = $getParentDir( $after ); // must be first page
950  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
951 
952  if ( !$status->isOK() ) {
953  throw new FileBackendError( "Iterator page I/O error." );
954  }
955 
956  $objects = $status->value;
957 
958  // @phan-suppress-next-line PhanTypeSuspiciousNonTraversableForeach
959  foreach ( $objects as $object ) { // files
960  $objectDir = $getParentDir( $object ); // directory of object
961 
962  if ( $objectDir !== false && $objectDir !== $dir ) {
963  // Swift stores paths in UTF-8, using binary sorting.
964  // See function "create_container_table" in common/db.py.
965  // If a directory is not "greater" than the last one,
966  // then it was already listed by the calling iterator.
967  if ( strcmp( $objectDir, $lastDir ) > 0 ) {
968  $pDir = $objectDir;
969  do { // add dir and all its parent dirs
970  $dirs[] = "{$pDir}/";
971  $pDir = $getParentDir( $pDir );
972  } while ( $pDir !== false // sanity
973  && strcmp( $pDir, $lastDir ) > 0 // not done already
974  && strlen( $pDir ) > strlen( $dir ) // within $dir
975  );
976  }
977  $lastDir = $objectDir;
978  }
979  }
980  }
981  // Page on the unfiltered directory listing (what is returned may be filtered)
982  if ( count( $objects ) < $limit ) {
983  $after = INF; // avoid a second RTT
984  } else {
985  $after = end( $objects ); // update last item
986  }
987 
988  return $dirs;
989  }
990 
1002  public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
1003  $files = []; // list of (path, stat array or null) entries
1004  if ( $after === INF ) {
1005  return $files; // nothing more
1006  }
1007 
1009  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1010 
1011  $prefix = ( $dir == '' ) ? null : "{$dir}/";
1012  // $objects will contain a list of unfiltered names or stdClass items
1013  // Non-recursive: only list files right under $dir
1014  if ( !empty( $params['topOnly'] ) ) {
1015  if ( !empty( $params['adviseStat'] ) ) {
1016  $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix, '/' );
1017  } else {
1018  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
1019  }
1020  } else {
1021  // Recursive: list all files under $dir and its subdirs
1022  if ( !empty( $params['adviseStat'] ) ) {
1023  $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix );
1024  } else {
1025  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
1026  }
1027  }
1028 
1029  // Reformat this list into a list of (name, stat array or null) entries
1030  if ( !$status->isOK() ) {
1031  throw new FileBackendError( "Iterator page I/O error." );
1032  }
1033 
1034  $objects = $status->value;
1035  $files = $this->buildFileObjectListing( $objects );
1036 
1037  // Page on the unfiltered object listing (what is returned may be filtered)
1038  if ( count( $objects ) < $limit ) {
1039  $after = INF; // avoid a second RTT
1040  } else {
1041  $after = end( $objects ); // update last item
1042  $after = is_object( $after ) ? $after->name : $after;
1043  }
1044 
1045  return $files;
1046  }
1047 
1055  private function buildFileObjectListing( array $objects ) {
1056  $names = [];
1057  foreach ( $objects as $object ) {
1058  if ( is_object( $object ) ) {
1059  if ( isset( $object->subdir ) || !isset( $object->name ) ) {
1060  continue; // virtual directory entry; ignore
1061  }
1062  $stat = [
1063  // Convert various random Swift dates to TS_MW
1064  'mtime' => $this->convertSwiftDate( $object->last_modified, TS_MW ),
1065  'size' => (int)$object->bytes,
1066  'sha1' => null,
1067  // Note: manifiest ETags are not an MD5 of the file
1068  'md5' => ctype_xdigit( $object->hash ) ? $object->hash : null,
1069  'latest' => false // eventually consistent
1070  ];
1071  $names[] = [ $object->name, $stat ];
1072  } elseif ( substr( $object, -1 ) !== '/' ) {
1073  // Omit directories, which end in '/' in listings
1074  $names[] = [ $object, null ];
1075  }
1076  }
1077 
1078  return $names;
1079  }
1080 
1087  public function loadListingStatInternal( $path, array $val ) {
1088  $this->cheapCache->setField( $path, 'stat', $val );
1089  }
1090 
1091  protected function doGetFileXAttributes( array $params ) {
1092  $stat = $this->getFileStat( $params );
1093  // Stat entries filled by file listings don't include metadata/headers
1094  if ( is_array( $stat ) && !isset( $stat['xattr'] ) ) {
1095  $this->clearCache( [ $params['src'] ] );
1096  $stat = $this->getFileStat( $params );
1097  }
1098 
1099  if ( is_array( $stat ) ) {
1100  return $stat['xattr'];
1101  }
1102 
1103  return ( $stat === self::$RES_ERROR ) ? self::$RES_ERROR : self::$RES_ABSENT;
1104  }
1105 
1106  protected function doGetFileSha1base36( array $params ) {
1107  // Avoid using stat entries from file listings, which never include the SHA-1 hash.
1108  // Also, recompute the hash if it's not part of the metadata headers for some reason.
1109  $params['requireSHA1'] = true;
1110 
1111  $stat = $this->getFileStat( $params );
1112  if ( is_array( $stat ) ) {
1113  return $stat['sha1'];
1114  }
1115 
1116  return ( $stat === self::$RES_ERROR ) ? self::$RES_ERROR : self::$RES_ABSENT;
1117  }
1118 
1119  protected function doStreamFile( array $params ) {
1120  $status = $this->newStatus();
1121 
1122  $flags = !empty( $params['headless'] ) ? HTTPFileStreamer::STREAM_HEADLESS : 0;
1123 
1124  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1125  if ( $srcRel === null ) {
1126  HTTPFileStreamer::send404Message( $params['src'], $flags );
1127  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
1128 
1129  return $status;
1130  }
1131 
1132  $auth = $this->getAuthentication();
1133  if ( !$auth || !is_array( $this->getContainerStat( $srcCont ) ) ) {
1134  HTTPFileStreamer::send404Message( $params['src'], $flags );
1135  $status->fatal( 'backend-fail-stream', $params['src'] );
1136 
1137  return $status;
1138  }
1139 
1140  // If "headers" is set, we only want to send them if the file is there.
1141  // Do not bother checking if the file exists if headers are not set though.
1142  if ( $params['headers'] && !$this->fileExists( $params ) ) {
1143  HTTPFileStreamer::send404Message( $params['src'], $flags );
1144  $status->fatal( 'backend-fail-stream', $params['src'] );
1145 
1146  return $status;
1147  }
1148 
1149  // Send the requested additional headers
1150  foreach ( $params['headers'] as $header ) {
1151  header( $header ); // aways send
1152  }
1153 
1154  if ( empty( $params['allowOB'] ) ) {
1155  // Cancel output buffering and gzipping if set
1156  ( $this->obResetFunc )();
1157  }
1158 
1159  $handle = fopen( 'php://output', 'wb' );
1160  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1161  'method' => 'GET',
1162  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1163  'headers' => $this->authTokenHeaders( $auth )
1164  + $this->headersFromParams( $params ) + $params['options'],
1165  'stream' => $handle,
1166  'flags' => [ 'relayResponseHeaders' => empty( $params['headless'] ) ]
1167  ] );
1168 
1169  if ( $rcode >= 200 && $rcode <= 299 ) {
1170  // good
1171  } elseif ( $rcode === 404 ) {
1172  $status->fatal( 'backend-fail-stream', $params['src'] );
1173  // Per T43113, nasty things can happen if bad cache entries get
1174  // stuck in cache. It's also possible that this error can come up
1175  // with simple race conditions. Clear out the stat cache to be safe.
1176  $this->clearCache( [ $params['src'] ] );
1177  $this->deleteFileCache( $params['src'] );
1178  } else {
1179  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1180  }
1181 
1182  return $status;
1183  }
1184 
1185  protected function doGetLocalCopyMulti( array $params ) {
1186  $auth = $this->getAuthentication();
1187 
1188  $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging
1189  // Blindly create tmp files and stream to them, catching any exception
1190  // if the file does not exist. Do not waste time doing file stats here.
1191  $reqs = []; // (path => op)
1192 
1193  // Initial dummy values to preserve path order
1194  $tmpFiles = array_fill_keys( $params['srcs'], self::$RES_ERROR );
1195  foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch
1196  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
1197  if ( $srcRel === null || !$auth ) {
1198  continue; // invalid storage path or auth error
1199  }
1200  // Get source file extension
1202  // Create a new temporary file...
1203  $tmpFile = $this->tmpFileFactory->newTempFSFile( 'localcopy_', $ext );
1204  $handle = $tmpFile ? fopen( $tmpFile->getPath(), 'wb' ) : false;
1205  if ( $handle ) {
1206  $reqs[$path] = [
1207  'method' => 'GET',
1208  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1209  'headers' => $this->authTokenHeaders( $auth )
1210  + $this->headersFromParams( $params ),
1211  'stream' => $handle,
1212  ];
1213  $tmpFiles[$path] = $tmpFile;
1214  }
1215  }
1216 
1217  // Ceph RADOS Gateway is in use (strong consistency) or X-Newest will be used
1218  $latest = ( $this->isRGW || !empty( $params['latest'] ) );
1219 
1220  $opts = [ 'maxConnsPerHost' => $params['concurrency'] ];
1221  $reqs = $this->http->runMulti( $reqs, $opts );
1222  foreach ( $reqs as $path => $op ) {
1223  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response'];
1224  fclose( $op['stream'] ); // close open handle
1225  if ( $rcode >= 200 && $rcode <= 299 ) {
1227  $tmpFile = $tmpFiles[$path];
1228  // Make sure that the stream finished and fully wrote to disk
1229  $size = $tmpFile->getSize();
1230  if ( $size !== (int)$rhdrs['content-length'] ) {
1231  $tmpFiles[$path] = self::$RES_ERROR;
1232  $rerr = "Got {$size}/{$rhdrs['content-length']} bytes";
1233  $this->onError( null, __METHOD__,
1234  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
1235  }
1236  // Set the file stat process cache in passing
1237  $stat = $this->getStatFromHeaders( $rhdrs );
1238  $stat['latest'] = $latest;
1239  $this->cheapCache->setField( $path, 'stat', $stat );
1240  } elseif ( $rcode === 404 ) {
1241  $tmpFiles[$path] = self::$RES_ABSENT;
1242  $this->cheapCache->setField(
1243  $path,
1244  'stat',
1245  $latest ? self::$ABSENT_LATEST : self::$ABSENT_NORMAL
1246  );
1247  } else {
1248  $tmpFiles[$path] = self::$RES_ERROR;
1249  $this->onError( null, __METHOD__,
1250  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
1251  }
1252  }
1253 
1254  return $tmpFiles;
1255  }
1256 
1257  public function getFileHttpUrl( array $params ) {
1258  if ( $this->swiftTempUrlKey != '' ||
1259  ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' )
1260  ) {
1261  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1262  if ( $srcRel === null ) {
1263  return self::TEMPURL_ERROR; // invalid path
1264  }
1265 
1266  $auth = $this->getAuthentication();
1267  if ( !$auth ) {
1268  return self::TEMPURL_ERROR;
1269  }
1270 
1271  $ttl = $params['ttl'] ?? 86400;
1272  $expires = time() + $ttl;
1273 
1274  if ( $this->swiftTempUrlKey != '' ) {
1275  $url = $this->storageUrl( $auth, $srcCont, $srcRel );
1276  // Swift wants the signature based on the unencoded object name
1277  $contPath = parse_url( $this->storageUrl( $auth, $srcCont ), PHP_URL_PATH );
1278  $signature = hash_hmac( 'sha1',
1279  "GET\n{$expires}\n{$contPath}/{$srcRel}",
1280  $this->swiftTempUrlKey
1281  );
1282 
1283  return "{$url}?temp_url_sig={$signature}&temp_url_expires={$expires}";
1284  } else { // give S3 API URL for rgw
1285  // Path for signature starts with the bucket
1286  $spath = '/' . rawurlencode( $srcCont ) . '/' .
1287  str_replace( '%2F', '/', rawurlencode( $srcRel ) );
1288  // Calculate the hash
1289  $signature = base64_encode( hash_hmac(
1290  'sha1',
1291  "GET\n\n\n{$expires}\n{$spath}",
1292  $this->rgwS3SecretKey,
1293  true // raw
1294  ) );
1295  // See https://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html.
1296  // Note: adding a newline for empty CanonicalizedAmzHeaders does not work.
1297  // Note: S3 API is the rgw default; remove the /swift/ URL bit.
1298  return str_replace( '/swift/v1', '', $this->storageUrl( $auth ) . $spath ) .
1299  '?' .
1300  http_build_query( [
1301  'Signature' => $signature,
1302  'Expires' => $expires,
1303  'AWSAccessKeyId' => $this->rgwS3AccessKey
1304  ] );
1305  }
1306  }
1307 
1308  return self::TEMPURL_ERROR;
1309  }
1310 
1311  protected function directoriesAreVirtual() {
1312  return true;
1313  }
1314 
1323  protected function headersFromParams( array $params ) {
1324  $hdrs = [];
1325  if ( !empty( $params['latest'] ) ) {
1326  $hdrs['x-newest'] = 'true';
1327  }
1328 
1329  return $hdrs;
1330  }
1331 
1332  protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1334  '@phan-var SwiftFileOpHandle[] $fileOpHandles';
1335 
1337  $statuses = [];
1338 
1339  $auth = $this->getAuthentication();
1340  if ( !$auth ) {
1341  foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1342  $statuses[$index] = $this->newStatus( 'backend-fail-connect', $this->name );
1343  }
1344 
1345  return $statuses;
1346  }
1347 
1348  // Split the HTTP requests into stages that can be done concurrently
1349  $httpReqsByStage = []; // map of (stage => index => HTTP request)
1350  foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1351  $reqs = $fileOpHandle->httpOp;
1352  // Convert the 'url' parameter to an actual URL using $auth
1353  foreach ( $reqs as $stage => &$req ) {
1354  list( $container, $relPath ) = $req['url'];
1355  $req['url'] = $this->storageUrl( $auth, $container, $relPath );
1356  $req['headers'] = $req['headers'] ?? [];
1357  $req['headers'] = $this->authTokenHeaders( $auth ) + $req['headers'];
1358  $httpReqsByStage[$stage][$index] = $req;
1359  }
1360  $statuses[$index] = $this->newStatus();
1361  }
1362 
1363  // Run all requests for the first stage, then the next, and so on
1364  $reqCount = count( $httpReqsByStage );
1365  for ( $stage = 0; $stage < $reqCount; ++$stage ) {
1366  $httpReqs = $this->http->runMulti( $httpReqsByStage[$stage] );
1367  foreach ( $httpReqs as $index => $httpReq ) {
1369  $fileOpHandle = $fileOpHandles[$index];
1370  // Run the callback for each request of this operation
1371  $status = $statuses[$index];
1372  ( $fileOpHandle->callback )( $httpReq, $status );
1373  // On failure, abort all remaining requests for this operation. This is used
1374  // in "move" operations to abort the DELETE request if the PUT request fails.
1375  if (
1376  !$status->isOK() ||
1377  $fileOpHandle->state === $fileOpHandle::CONTINUE_NO
1378  ) {
1379  $stages = count( $fileOpHandle->httpOp );
1380  for ( $s = ( $stage + 1 ); $s < $stages; ++$s ) {
1381  unset( $httpReqsByStage[$s][$index] );
1382  }
1383  }
1384  }
1385  }
1386 
1387  return $statuses;
1388  }
1389 
1412  protected function setContainerAccess( $container, array $readUsers, array $writeUsers ) {
1413  $status = $this->newStatus();
1414  $auth = $this->getAuthentication();
1415 
1416  if ( !$auth ) {
1417  $status->fatal( 'backend-fail-connect', $this->name );
1418 
1419  return $status;
1420  }
1421 
1422  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1423  'method' => 'POST',
1424  'url' => $this->storageUrl( $auth, $container ),
1425  'headers' => $this->authTokenHeaders( $auth ) + [
1426  'x-container-read' => implode( ',', $readUsers ),
1427  'x-container-write' => implode( ',', $writeUsers )
1428  ]
1429  ] );
1430 
1431  if ( $rcode != 204 && $rcode !== 202 ) {
1432  $status->fatal( 'backend-fail-internal', $this->name );
1433  $this->logger->error( __METHOD__ . ': unexpected rcode value ({rcode})',
1434  [ 'rcode' => $rcode ] );
1435  }
1436 
1437  return $status;
1438  }
1439 
1448  protected function getContainerStat( $container, $bypassCache = false ) {
1450  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1451 
1452  if ( $bypassCache ) { // purge cache
1453  $this->containerStatCache->clear( $container );
1454  } elseif ( !$this->containerStatCache->hasField( $container, 'stat' ) ) {
1455  $this->primeContainerCache( [ $container ] ); // check persistent cache
1456  }
1457  if ( !$this->containerStatCache->hasField( $container, 'stat' ) ) {
1458  $auth = $this->getAuthentication();
1459  if ( !$auth ) {
1460  return self::$RES_ERROR;
1461  }
1462 
1463  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1464  'method' => 'HEAD',
1465  'url' => $this->storageUrl( $auth, $container ),
1466  'headers' => $this->authTokenHeaders( $auth )
1467  ] );
1468 
1469  if ( $rcode === 204 ) {
1470  $stat = [
1471  'count' => $rhdrs['x-container-object-count'],
1472  'bytes' => $rhdrs['x-container-bytes-used']
1473  ];
1474  if ( $bypassCache ) {
1475  return $stat;
1476  } else {
1477  $this->containerStatCache->setField( $container, 'stat', $stat ); // cache it
1478  $this->setContainerCache( $container, $stat ); // update persistent cache
1479  }
1480  } elseif ( $rcode === 404 ) {
1481  return self::$RES_ABSENT;
1482  } else {
1483  $this->onError( null, __METHOD__,
1484  [ 'cont' => $container ], $rerr, $rcode, $rdesc );
1485 
1486  return self::$RES_ERROR;
1487  }
1488  }
1489 
1490  return $this->containerStatCache->getField( $container, 'stat' );
1491  }
1492 
1500  protected function createContainer( $container, array $params ) {
1501  $status = $this->newStatus();
1502 
1503  $auth = $this->getAuthentication();
1504  if ( !$auth ) {
1505  $status->fatal( 'backend-fail-connect', $this->name );
1506 
1507  return $status;
1508  }
1509 
1510  // @see SwiftFileBackend::setContainerAccess()
1511  if ( empty( $params['noAccess'] ) ) {
1512  // public
1513  $readUsers = array_merge( $this->readUsers, [ '.r:*', $this->swiftUser ] );
1514  $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
1515  } else {
1516  // private
1517  $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
1518  $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
1519  }
1520 
1521  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1522  'method' => 'PUT',
1523  'url' => $this->storageUrl( $auth, $container ),
1524  'headers' => $this->authTokenHeaders( $auth ) + [
1525  'x-container-read' => implode( ',', $readUsers ),
1526  'x-container-write' => implode( ',', $writeUsers )
1527  ]
1528  ] );
1529 
1530  if ( $rcode === 201 ) { // new
1531  // good
1532  } elseif ( $rcode === 202 ) { // already there
1533  // this shouldn't really happen, but is OK
1534  } else {
1535  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1536  }
1537 
1538  return $status;
1539  }
1540 
1548  protected function deleteContainer( $container, array $params ) {
1549  $status = $this->newStatus();
1550 
1551  $auth = $this->getAuthentication();
1552  if ( !$auth ) {
1553  $status->fatal( 'backend-fail-connect', $this->name );
1554 
1555  return $status;
1556  }
1557 
1558  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1559  'method' => 'DELETE',
1560  'url' => $this->storageUrl( $auth, $container ),
1561  'headers' => $this->authTokenHeaders( $auth )
1562  ] );
1563 
1564  if ( $rcode >= 200 && $rcode <= 299 ) { // deleted
1565  $this->containerStatCache->clear( $container ); // purge
1566  } elseif ( $rcode === 404 ) { // not there
1567  // this shouldn't really happen, but is OK
1568  } elseif ( $rcode === 409 ) { // not empty
1569  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); // race?
1570  } else {
1571  // @phan-suppress-previous-line PhanPluginDuplicateIfStatements
1572  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1573  }
1574 
1575  return $status;
1576  }
1577 
1590  private function objectListing(
1591  $fullCont, $type, $limit, $after = null, $prefix = null, $delim = null
1592  ) {
1593  $status = $this->newStatus();
1594 
1595  $auth = $this->getAuthentication();
1596  if ( !$auth ) {
1597  $status->fatal( 'backend-fail-connect', $this->name );
1598 
1599  return $status;
1600  }
1601 
1602  $query = [ 'limit' => $limit ];
1603  if ( $type === 'info' ) {
1604  $query['format'] = 'json';
1605  }
1606  if ( $after !== null ) {
1607  $query['marker'] = $after;
1608  }
1609  if ( $prefix !== null ) {
1610  $query['prefix'] = $prefix;
1611  }
1612  if ( $delim !== null ) {
1613  $query['delimiter'] = $delim;
1614  }
1615 
1616  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1617  'method' => 'GET',
1618  'url' => $this->storageUrl( $auth, $fullCont ),
1619  'query' => $query,
1620  'headers' => $this->authTokenHeaders( $auth )
1621  ] );
1622 
1623  $params = [ 'cont' => $fullCont, 'prefix' => $prefix, 'delim' => $delim ];
1624  if ( $rcode === 200 ) { // good
1625  if ( $type === 'info' ) {
1626  $status->value = FormatJson::decode( trim( $rbody ) );
1627  } else {
1628  $status->value = explode( "\n", trim( $rbody ) );
1629  }
1630  } elseif ( $rcode === 204 ) {
1631  $status->value = []; // empty container
1632  } elseif ( $rcode === 404 ) {
1633  $status->value = []; // no container
1634  } else {
1635  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1636  }
1637 
1638  return $status;
1639  }
1640 
1641  protected function doPrimeContainerCache( array $containerInfo ) {
1642  foreach ( $containerInfo as $container => $info ) {
1643  $this->containerStatCache->setField( $container, 'stat', $info );
1644  }
1645  }
1646 
1647  protected function doGetFileStatMulti( array $params ) {
1648  $stats = [];
1649 
1650  $auth = $this->getAuthentication();
1651 
1652  $reqs = []; // (path => op)
1653  // (a) Check the containers of the paths...
1654  foreach ( $params['srcs'] as $path ) {
1655  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
1656  if ( $srcRel === null || !$auth ) {
1657  $stats[$path] = self::$RES_ERROR;
1658  continue; // invalid storage path or auth error
1659  }
1660 
1661  $cstat = $this->getContainerStat( $srcCont );
1662  if ( $cstat === self::$RES_ABSENT ) {
1663  $stats[$path] = self::$RES_ABSENT;
1664  continue; // ok, nothing to do
1665  } elseif ( !is_array( $cstat ) ) {
1666  $stats[$path] = self::$RES_ERROR;
1667  continue;
1668  }
1669 
1670  $reqs[$path] = [
1671  'method' => 'HEAD',
1672  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1673  'headers' => $this->authTokenHeaders( $auth ) + $this->headersFromParams( $params )
1674  ];
1675  }
1676 
1677  // (b) Check the files themselves...
1678  $opts = [ 'maxConnsPerHost' => $params['concurrency'] ];
1679  $reqs = $this->http->runMulti( $reqs, $opts );
1680  foreach ( $reqs as $path => $op ) {
1681  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response'];
1682  if ( $rcode === 200 || $rcode === 204 ) {
1683  // Update the object if it is missing some headers
1684  if ( !empty( $params['requireSHA1'] ) ) {
1685  $rhdrs = $this->addMissingHashMetadata( $rhdrs, $path );
1686  }
1687  // Load the stat array from the headers
1688  $stat = $this->getStatFromHeaders( $rhdrs );
1689  if ( $this->isRGW ) {
1690  $stat['latest'] = true; // strong consistency
1691  }
1692  } elseif ( $rcode === 404 ) {
1693  $stat = self::$RES_ABSENT;
1694  } else {
1695  $stat = self::$RES_ERROR;
1696  $this->onError( null, __METHOD__, $params, $rerr, $rcode, $rdesc );
1697  }
1698  $stats[$path] = $stat;
1699  }
1700 
1701  return $stats;
1702  }
1703 
1708  protected function getStatFromHeaders( array $rhdrs ) {
1709  // Fetch all of the custom metadata headers
1710  $metadata = $this->getMetadataFromHeaders( $rhdrs );
1711  // Fetch all of the custom raw HTTP headers
1712  $headers = $this->extractMutableContentHeaders( $rhdrs );
1713 
1714  return [
1715  // Convert various random Swift dates to TS_MW
1716  'mtime' => $this->convertSwiftDate( $rhdrs['last-modified'], TS_MW ),
1717  // Empty objects actually return no content-length header in Ceph
1718  'size' => isset( $rhdrs['content-length'] ) ? (int)$rhdrs['content-length'] : 0,
1719  'sha1' => $metadata['sha1base36'] ?? null,
1720  // Note: manifiest ETags are not an MD5 of the file
1721  'md5' => ctype_xdigit( $rhdrs['etag'] ) ? $rhdrs['etag'] : null,
1722  'xattr' => [ 'metadata' => $metadata, 'headers' => $headers ]
1723  ];
1724  }
1725 
1729  protected function getAuthentication() {
1730  if ( $this->authErrorTimestamp !== null ) {
1731  if ( ( time() - $this->authErrorTimestamp ) < 60 ) {
1732  return null; // failed last attempt; don't bother
1733  } else { // actually retry this time
1734  $this->authErrorTimestamp = null;
1735  }
1736  }
1737  // Session keys expire after a while, so we renew them periodically
1738  $reAuth = ( ( time() - $this->authSessionTimestamp ) > $this->authTTL );
1739  // Authenticate with proxy and get a session key...
1740  if ( !$this->authCreds || $reAuth ) {
1741  $this->authSessionTimestamp = 0;
1742  $cacheKey = $this->getCredsCacheKey( $this->swiftUser );
1743  $creds = $this->srvCache->get( $cacheKey ); // credentials
1744  // Try to use the credential cache
1745  if ( isset( $creds['auth_token'] ) && isset( $creds['storage_url'] ) ) {
1746  $this->authCreds = $creds;
1747  // Skew the timestamp for worst case to avoid using stale credentials
1748  $this->authSessionTimestamp = time() - ceil( $this->authTTL / 2 );
1749  } else { // cache miss
1750  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1751  'method' => 'GET',
1752  'url' => "{$this->swiftAuthUrl}/v1.0",
1753  'headers' => [
1754  'x-auth-user' => $this->swiftUser,
1755  'x-auth-key' => $this->swiftKey
1756  ]
1757  ] );
1758 
1759  if ( $rcode >= 200 && $rcode <= 299 ) { // OK
1760  $this->authCreds = [
1761  'auth_token' => $rhdrs['x-auth-token'],
1762  'storage_url' => $this->swiftStorageUrl ?? $rhdrs['x-storage-url']
1763  ];
1764 
1765  $this->srvCache->set( $cacheKey, $this->authCreds, ceil( $this->authTTL / 2 ) );
1766  $this->authSessionTimestamp = time();
1767  } elseif ( $rcode === 401 ) {
1768  $this->onError( null, __METHOD__, [], "Authentication failed.", $rcode );
1769  $this->authErrorTimestamp = time();
1770 
1771  return null;
1772  } else {
1773  $this->onError( null, __METHOD__, [], "HTTP return code: $rcode", $rcode );
1774  $this->authErrorTimestamp = time();
1775 
1776  return null;
1777  }
1778  }
1779  // Ceph RGW does not use <account> in URLs (OpenStack Swift uses "/v1/<account>")
1780  if ( substr( $this->authCreds['storage_url'], -3 ) === '/v1' ) {
1781  $this->isRGW = true; // take advantage of strong consistency in Ceph
1782  }
1783  }
1784 
1785  return $this->authCreds;
1786  }
1787 
1794  protected function storageUrl( array $creds, $container = null, $object = null ) {
1795  $parts = [ $creds['storage_url'] ];
1796  if ( strlen( $container ) ) {
1797  $parts[] = rawurlencode( $container );
1798  }
1799  if ( strlen( $object ) ) {
1800  $parts[] = str_replace( "%2F", "/", rawurlencode( $object ) );
1801  }
1802 
1803  return implode( '/', $parts );
1804  }
1805 
1810  protected function authTokenHeaders( array $creds ) {
1811  return [ 'x-auth-token' => $creds['auth_token'] ];
1812  }
1813 
1820  private function getCredsCacheKey( $username ) {
1821  return 'swiftcredentials:' . md5( $username . ':' . $this->swiftAuthUrl );
1822  }
1823 
1835  public function onError( $status, $func, array $params, $err = '', $code = 0, $desc = '' ) {
1836  if ( $status instanceof StatusValue ) {
1837  $status->fatal( 'backend-fail-internal', $this->name );
1838  }
1839  if ( $code == 401 ) { // possibly a stale token
1840  $this->srvCache->delete( $this->getCredsCacheKey( $this->swiftUser ) );
1841  }
1842  $msg = "HTTP {code} ({desc}) in '{func}' (given '{req_params}')";
1843  $msgParams = [
1844  'code' => $code,
1845  'desc' => $desc,
1846  'func' => $func,
1847  'req_params' => FormatJson::encode( $params ),
1848  ];
1849  if ( $err ) {
1850  $msg .= ': {err}';
1851  $msgParams['err'] = $err;
1852  }
1853  $this->logger->error( $msg, $msgParams );
1854  }
1855 }
SwiftFileOpHandle
Definition: SwiftFileOpHandle.php:25
MWTimestamp
Library for creating and parsing MW-style timestamps.
Definition: MWTimestamp.php:32
SwiftFileBackend\doPrepareInternal
doPrepareInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:628
SwiftFileBackend\$isRGW
bool $isRGW
Whether the server is an Ceph RGW.
Definition: SwiftFileBackend.php:78
MultiHttpClient
Class to handle multiple HTTP requests.
Definition: MultiHttpClient.php:55
StatusValue
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Definition: StatusValue.php:42
SwiftFileBackend\isPathUsableInternal
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path in the backend.
Definition: SwiftFileBackend.php:177
SwiftFileBackend\$authErrorTimestamp
int $authErrorTimestamp
UNIX timestamp.
Definition: SwiftFileBackend.php:75
SwiftFileBackend\getAuthentication
getAuthentication()
Definition: SwiftFileBackend.php:1729
SwiftFileBackend\getContainerStat
getContainerStat( $container, $bypassCache=false)
Get a Swift container stat array, possibly from process cache.
Definition: SwiftFileBackend.php:1448
SwiftFileBackend\buildFileObjectListing
buildFileObjectListing(array $objects)
Build a list of file objects, filtering out any directories and extracting any stat info if provided ...
Definition: SwiftFileBackend.php:1055
EmptyBagOStuff
A BagOStuff object with no objects in it.
Definition: EmptyBagOStuff.php:29
SwiftFileBackend\$containerStatCache
MapCacheLRU $containerStatCache
Container stat cache.
Definition: SwiftFileBackend.php:68
SwiftFileBackend\$swiftKey
string $swiftKey
Secret key for user.
Definition: SwiftFileBackend.php:48
SwiftFileBackend\getStatFromHeaders
getStatFromHeaders(array $rhdrs)
Definition: SwiftFileBackend.php:1708
SwiftFileBackend\addMissingHashMetadata
addMissingHashMetadata(array $objHdrs, $path)
Fill in any missing object metadata and save it to Swift.
Definition: SwiftFileBackend.php:764
LockManager\LOCK_UW
const LOCK_UW
Definition: LockManager.php:69
SwiftFileBackend\doGetFileXAttributes
doGetFileXAttributes(array $params)
Definition: SwiftFileBackend.php:1091
FileBackendError
File backend exception for checked exceptions (e.g.
Definition: FileBackendError.php:8
FileBackend\getScopedFileLocks
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
Definition: FileBackend.php:1432
FileBackendStore\fileExists
fileExists(array $params)
Check if a file exists at a storage path in the backend.
Definition: FileBackendStore.php:634
SwiftFileBackend\$swiftUser
string $swiftUser
Swift user (account:user) to authenticate as.
Definition: SwiftFileBackend.php:46
SwiftFileBackend\loadListingStatInternal
loadListingStatInternal( $path, array $val)
Do not call this function outside of SwiftFileBackendFileList.
Definition: SwiftFileBackend.php:1087
FileBackend\extensionFromPath
static extensionFromPath( $path, $case='lowercase')
Get the final extension from a storage or FS path.
Definition: FileBackend.php:1598
FileBackendStore\deleteFileCache
deleteFileCache( $path)
Delete the cached stat info for a file path.
Definition: FileBackendStore.php:1888
SwiftFileBackend\resolveContainerPath
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
Definition: SwiftFileBackend.php:167
FileBackendStore\executeOpHandlesInternal
executeOpHandlesInternal(array $fileOpHandles)
Execute a list of FileBackendStoreOpHandle handles in parallel.
Definition: FileBackendStore.php:1386
BagOStuff
Class representing a cache/ephemeral data store.
Definition: BagOStuff.php:65
$s
$s
Definition: mergeMessageFileList.php:185
SwiftFileBackend\doGetFileContentsMulti
doGetFileContentsMulti(array $params)
Definition: SwiftFileBackend.php:818
SwiftFileBackend\createContainer
createContainer( $container, array $params)
Create a Swift container.
Definition: SwiftFileBackend.php:1500
SwiftFileBackend\extractMetadataHeaders
extractMetadataHeaders(array $headers)
Definition: SwiftFileBackend.php:233
SwiftFileBackend\getFileListPageInternal
getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
Definition: SwiftFileBackend.php:1002
SwiftFileBackend\getMetadataFromHeaders
getMetadataFromHeaders(array $headers)
Definition: SwiftFileBackend.php:250
SwiftFileBackend\doGetFileSha1base36
doGetFileSha1base36(array $params)
Definition: SwiftFileBackend.php:1106
FileBackendStore\clearCache
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
Definition: FileBackendStore.php:1467
SwiftFileBackend\getFeatures
getFeatures()
Get the a bitfield of extra features supported by the backend medium.
Definition: SwiftFileBackend.php:159
FileBackendStore\$RES_ERROR
static null $RES_ERROR
Idiom for "no result due to I/O errors" (since 1.34)
Definition: FileBackendStore.php:65
SwiftFileBackend\doDeleteInternal
doDeleteInternal(array $params)
Definition: SwiftFileBackend.php:527
SwiftFileBackend\headersFromParams
headersFromParams(array $params)
Get headers to send to Swift when reading a file based on a FileBackend params array,...
Definition: SwiftFileBackend.php:1323
SwiftFileBackend\deleteContainer
deleteContainer( $container, array $params)
Delete a Swift container.
Definition: SwiftFileBackend.php:1548
SwiftFileBackend\getCredsCacheKey
getCredsCacheKey( $username)
Get the cache key for a container.
Definition: SwiftFileBackend.php:1820
FormatJson\decode
static decode( $value, $assoc=false)
Decodes a JSON string.
Definition: FormatJson.php:174
FormatJson\encode
static encode( $value, $pretty=false, $escaping=0)
Returns the JSON representation of a value.
Definition: FormatJson.php:115
SwiftFileBackend\doDirectoryExists
doDirectoryExists( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:876
SwiftFileBackend
Class for an OpenStack Swift (or Ceph RGW) based file backend.
Definition: SwiftFileBackend.php:36
SwiftFileBackend\getDirectoryListInternal
getDirectoryListInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:893
SwiftFileBackend\setContainerAccess
setContainerAccess( $container, array $readUsers, array $writeUsers)
Set read/write permissions for a Swift container.
Definition: SwiftFileBackend.php:1412
SwiftFileBackend\getFileHttpUrl
getFileHttpUrl(array $params)
Definition: SwiftFileBackend.php:1257
SwiftFileBackend\doMoveInternal
doMoveInternal(array $params)
Definition: SwiftFileBackend.php:460
MapCacheLRU
Handles a simple LRU key/value map with a maximum number of entries.
Definition: MapCacheLRU.php:38
SwiftFileBackend\$authCreds
array $authCreds
Definition: SwiftFileBackend.php:71
$dirs
$dirs
Definition: mergeMessageFileList.php:192
SwiftFileBackend\extractMutableContentHeaders
extractMutableContentHeaders(array $headers)
Filter/normalize a header map to only include mutable "content-"/"x-content-" headers.
Definition: SwiftFileBackend.php:195
SwiftFileBackend\doGetLocalCopyMulti
doGetLocalCopyMulti(array $params)
Definition: SwiftFileBackend.php:1185
SwiftFileBackend\onError
onError( $status, $func, array $params, $err='', $code=0, $desc='')
Log an unexpected exception for this backend.
Definition: SwiftFileBackend.php:1835
SwiftFileBackend\$secureWriteUsers
array $secureWriteUsers
Additional users (account:user) with write permissions on private containers.
Definition: SwiftFileBackend.php:62
SwiftFileBackend\doDescribeInternal
doDescribeInternal(array $params)
Definition: SwiftFileBackend.php:569
SwiftFileBackend\storageUrl
storageUrl(array $creds, $container=null, $object=null)
Definition: SwiftFileBackend.php:1794
FileBackendStore\getContentType
getContentType( $storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
Definition: FileBackendStore.php:2009
FileBackend\$obResetFunc
callable $obResetFunc
Definition: FileBackend.php:125
SwiftFileBackendDirList
Iterator for listing directories.
Definition: SwiftFileBackendDirList.php:28
$content
$content
Definition: router.php:76
SwiftFileBackend\doSecureInternal
doSecureInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:651
FileBackendStore\getFileStat
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
Definition: FileBackendStore.php:670
$header
$header
Definition: updateCredits.php:41
SwiftFileBackend\__construct
__construct(array $config)
Definition: SwiftFileBackend.php:118
SwiftFileBackend\directoriesAreVirtual
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
Definition: SwiftFileBackend.php:1311
FileBackendStore\resolveStoragePathReal
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
Definition: FileBackendStore.php:1623
SwiftFileBackend\authTokenHeaders
authTokenHeaders(array $creds)
Definition: SwiftFileBackend.php:1810
WANObjectCache
Multi-datacenter aware caching interface.
Definition: WANObjectCache.php:120
SwiftFileBackend\doCreateInternal
doCreateInternal(array $params)
Definition: SwiftFileBackend.php:261
SwiftFileBackend\$authSessionTimestamp
int $authSessionTimestamp
UNIX timestamp.
Definition: SwiftFileBackend.php:73
FileBackendStore
Base class for all backends using particular storage medium.
Definition: FileBackendStore.php:40
HTTPFileStreamer\STREAM_HEADLESS
const STREAM_HEADLESS
Definition: HTTPFileStreamer.php:40
FileBackendStore\$RES_ABSENT
static false $RES_ABSENT
Idiom for "no result due to missing file" (since 1.34)
Definition: FileBackendStore.php:63
SwiftFileBackend\doExecuteOpHandlesInternal
doExecuteOpHandlesInternal(array $fileOpHandles)
Definition: SwiftFileBackend.php:1332
SwiftFileBackend\$swiftAuthUrl
string $swiftAuthUrl
Authentication base URL (without version)
Definition: SwiftFileBackend.php:42
SwiftFileBackend\doStreamFile
doStreamFile(array $params)
Definition: SwiftFileBackend.php:1119
HTTPFileStreamer\send404Message
static send404Message( $fname, $flags=0)
Send out a standard 404 message for a file.
Definition: HTTPFileStreamer.php:203
FileBackend\newStatus
newStatus(... $args)
Yields the result of the status wrapper callback on either:
Definition: FileBackend.php:1691
SwiftFileBackend\$writeUsers
array $writeUsers
Additional users (account:user) with write permissions on public containers.
Definition: SwiftFileBackend.php:58
SwiftFileBackend\$rgwS3SecretKey
string $rgwS3SecretKey
S3 authentication key (RADOS Gateway)
Definition: SwiftFileBackend.php:54
FileBackendStore\primeContainerCache
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
Definition: FileBackendStore.php:1807
SwiftFileBackend\$authTTL
int $authTTL
TTL in seconds.
Definition: SwiftFileBackend.php:40
SwiftFileBackend\$readUsers
array $readUsers
Additional users (account:user) with read permissions on public containers.
Definition: SwiftFileBackend.php:56
SwiftFileBackend\doCopyInternal
doCopyInternal(array $params)
Definition: SwiftFileBackend.php:405
FileBackend\$name
string $name
Unique backend name.
Definition: FileBackend.php:98
SwiftFileBackend\$rgwS3AccessKey
string $rgwS3AccessKey
S3 access key (RADOS Gateway)
Definition: SwiftFileBackend.php:52
FileBackend\getLocalCopy
getLocalCopy(array $params)
Get a local copy on disk of the file at a storage path in the backend.
Definition: FileBackend.php:1194
SwiftFileBackend\objectListing
objectListing( $fullCont, $type, $limit, $after=null, $prefix=null, $delim=null)
Get a list of objects under a container.
Definition: SwiftFileBackend.php:1590
SwiftFileBackend\$swiftStorageUrl
string $swiftStorageUrl
Override of storage base URL.
Definition: SwiftFileBackend.php:44
$path
$path
Definition: NoLocalSettings.php:25
SwiftFileBackend\convertSwiftDate
convertSwiftDate( $ts, $format=TS_MW)
Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT"/"2013-05-11T07:37:27.678360Z".
Definition: SwiftFileBackend.php:747
SwiftFileBackend\doPublishInternal
doPublishInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:677
SwiftFileBackend\doGetFileStat
doGetFileStat(array $params)
Definition: SwiftFileBackend.php:729
Wikimedia
This program is free software; you can redistribute it and/or modify it under the terms of the GNU Ge...
SwiftFileBackend\$swiftTempUrlKey
string $swiftTempUrlKey
Shared secret value for making temp URLs.
Definition: SwiftFileBackend.php:50
SwiftFileBackend\doStoreInternal
doStoreInternal(array $params)
Definition: SwiftFileBackend.php:316
SwiftFileBackend\doGetFileStatMulti
doGetFileStatMulti(array $params)
Get file stat information (concurrently if possible) for several files.
Definition: SwiftFileBackend.php:1647
$ext
if(!is_readable( $file)) $ext
Definition: router.php:48
SwiftFileBackend\doCleanInternal
doCleanInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:701
SwiftFileBackend\$srvCache
BagOStuff $srvCache
Definition: SwiftFileBackend.php:65
SwiftFileBackendFileList
Iterator for listing regular files.
Definition: SwiftFileBackendFileList.php:28
SwiftFileBackend\getFileListInternal
getFileListInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:904
SwiftFileBackend\doPrimeContainerCache
doPrimeContainerCache(array $containerInfo)
Fill the backend-specific process cache given an array of resolved container names and their correspo...
Definition: SwiftFileBackend.php:1641
SwiftFileBackend\$secureReadUsers
array $secureReadUsers
Additional users (account:user) with read permissions on private containers.
Definition: SwiftFileBackend.php:60
FileBackendStore\setContainerCache
setContainerCache( $container, array $val)
Set the cached info for a container.
Definition: FileBackendStore.php:1782
FileBackend\scopedProfileSection
scopedProfileSection( $section)
Definition: FileBackend.php:1713
SwiftFileBackend\getDirListPageInternal
getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
Definition: SwiftFileBackend.php:919
SwiftFileBackend\$http
MultiHttpClient $http
Definition: SwiftFileBackend.php:38
$type
$type
Definition: testCompression.php:52