MediaWiki  1.28.0
SwiftFileBackend.php
Go to the documentation of this file.
1 <?php
37  protected $http;
38 
40  protected $authTTL;
41 
43  protected $swiftAuthUrl;
44 
46  protected $swiftUser;
47 
49  protected $swiftKey;
50 
52  protected $swiftTempUrlKey;
53 
55  protected $rgwS3AccessKey;
56 
58  protected $rgwS3SecretKey;
59 
61  protected $srvCache;
62 
65 
67  protected $authCreds;
68 
70  protected $authSessionTimestamp = 0;
71 
73  protected $authErrorTimestamp = null;
74 
76  protected $isRGW = false;
77 
106  public function __construct( array $config ) {
107  parent::__construct( $config );
108  // Required settings
109  $this->swiftAuthUrl = $config['swiftAuthUrl'];
110  $this->swiftUser = $config['swiftUser'];
111  $this->swiftKey = $config['swiftKey'];
112  // Optional settings
113  $this->authTTL = isset( $config['swiftAuthTTL'] )
114  ? $config['swiftAuthTTL']
115  : 15 * 60; // some sane number
116  $this->swiftTempUrlKey = isset( $config['swiftTempUrlKey'] )
117  ? $config['swiftTempUrlKey']
118  : '';
119  $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
120  ? $config['shardViaHashLevels']
121  : '';
122  $this->rgwS3AccessKey = isset( $config['rgwS3AccessKey'] )
123  ? $config['rgwS3AccessKey']
124  : '';
125  $this->rgwS3SecretKey = isset( $config['rgwS3SecretKey'] )
126  ? $config['rgwS3SecretKey']
127  : '';
128  // HTTP helper client
129  $this->http = new MultiHttpClient( [] );
130  // Cache container information to mask latency
131  if ( isset( $config['wanCache'] ) && $config['wanCache'] instanceof WANObjectCache ) {
132  $this->memCache = $config['wanCache'];
133  }
134  // Process cache for container info
135  $this->containerStatCache = new ProcessCacheLRU( 300 );
136  // Cache auth token information to avoid RTTs
137  if ( !empty( $config['cacheAuthInfo'] ) && isset( $config['srvCache'] ) ) {
138  $this->srvCache = $config['srvCache'];
139  } else {
140  $this->srvCache = new EmptyBagOStuff();
141  }
142  }
143 
144  public function getFeatures() {
147  }
148 
149  protected function resolveContainerPath( $container, $relStoragePath ) {
150  if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) {
151  return null; // not UTF-8, makes it hard to use CF and the swift HTTP API
152  } elseif ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
153  return null; // too long for Swift
154  }
155 
156  return $relStoragePath;
157  }
158 
159  public function isPathUsableInternal( $storagePath ) {
160  list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
161  if ( $rel === null ) {
162  return false; // invalid
163  }
164 
165  return is_array( $this->getContainerStat( $container ) );
166  }
167 
175  protected function sanitizeHdrs( array $params ) {
176  return isset( $params['headers'] )
177  ? $this->getCustomHeaders( $params['headers'] )
178  : [];
179 
180  }
181 
186  protected function getCustomHeaders( array $rawHeaders ) {
187  $headers = [];
188 
189  // Normalize casing, and strip out illegal headers
190  foreach ( $rawHeaders as $name => $value ) {
191  $name = strtolower( $name );
192  if ( preg_match( '/^content-(type|length)$/', $name ) ) {
193  continue; // blacklisted
194  } elseif ( preg_match( '/^(x-)?content-/', $name ) ) {
195  $headers[$name] = $value; // allowed
196  } elseif ( preg_match( '/^content-(disposition)/', $name ) ) {
197  $headers[$name] = $value; // allowed
198  }
199  }
200  // By default, Swift has annoyingly low maximum header value limits
201  if ( isset( $headers['content-disposition'] ) ) {
202  $disposition = '';
203  // @note: assume FileBackend::makeContentDisposition() already used
204  foreach ( explode( ';', $headers['content-disposition'] ) as $part ) {
205  $part = trim( $part );
206  $new = ( $disposition === '' ) ? $part : "{$disposition};{$part}";
207  if ( strlen( $new ) <= 255 ) {
208  $disposition = $new;
209  } else {
210  break; // too long; sigh
211  }
212  }
213  $headers['content-disposition'] = $disposition;
214  }
215 
216  return $headers;
217  }
218 
223  protected function getMetadataHeaders( array $rawHeaders ) {
224  $headers = [];
225  foreach ( $rawHeaders as $name => $value ) {
226  $name = strtolower( $name );
227  if ( strpos( $name, 'x-object-meta-' ) === 0 ) {
228  $headers[$name] = $value;
229  }
230  }
231 
232  return $headers;
233  }
234 
239  protected function getMetadata( array $rawHeaders ) {
240  $metadata = [];
241  foreach ( $this->getMetadataHeaders( $rawHeaders ) as $name => $value ) {
242  $metadata[substr( $name, strlen( 'x-object-meta-' ) )] = $value;
243  }
244 
245  return $metadata;
246  }
247 
248  protected function doCreateInternal( array $params ) {
249  $status = $this->newStatus();
250 
251  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
252  if ( $dstRel === null ) {
253  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
254 
255  return $status;
256  }
257 
258  $sha1Hash = Wikimedia\base_convert( sha1( $params['content'] ), 16, 36, 31 );
259  $contentType = isset( $params['headers']['content-type'] )
260  ? $params['headers']['content-type']
261  : $this->getContentType( $params['dst'], $params['content'], null );
262 
263  $reqs = [ [
264  'method' => 'PUT',
265  'url' => [ $dstCont, $dstRel ],
266  'headers' => [
267  'content-length' => strlen( $params['content'] ),
268  'etag' => md5( $params['content'] ),
269  'content-type' => $contentType,
270  'x-object-meta-sha1base36' => $sha1Hash
271  ] + $this->sanitizeHdrs( $params ),
272  'body' => $params['content']
273  ] ];
274 
275  $method = __METHOD__;
276  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
277  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
278  if ( $rcode === 201 ) {
279  // good
280  } elseif ( $rcode === 412 ) {
281  $status->fatal( 'backend-fail-contenttype', $params['dst'] );
282  } else {
283  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
284  }
285  };
286 
287  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
288  if ( !empty( $params['async'] ) ) { // deferred
289  $status->value = $opHandle;
290  } else { // actually write the object in Swift
291  $status->merge( current( $this->doExecuteOpHandlesInternal( [ $opHandle ] ) ) );
292  }
293 
294  return $status;
295  }
296 
297  protected function doStoreInternal( array $params ) {
298  $status = $this->newStatus();
299 
300  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
301  if ( $dstRel === null ) {
302  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
303 
304  return $status;
305  }
306 
307  MediaWiki\suppressWarnings();
308  $sha1Hash = sha1_file( $params['src'] );
309  MediaWiki\restoreWarnings();
310  if ( $sha1Hash === false ) { // source doesn't exist?
311  $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
312 
313  return $status;
314  }
315  $sha1Hash = Wikimedia\base_convert( $sha1Hash, 16, 36, 31 );
316  $contentType = isset( $params['headers']['content-type'] )
317  ? $params['headers']['content-type']
318  : $this->getContentType( $params['dst'], null, $params['src'] );
319 
320  $handle = fopen( $params['src'], 'rb' );
321  if ( $handle === false ) { // source doesn't exist?
322  $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
323 
324  return $status;
325  }
326 
327  $reqs = [ [
328  'method' => 'PUT',
329  'url' => [ $dstCont, $dstRel ],
330  'headers' => [
331  'content-length' => filesize( $params['src'] ),
332  'etag' => md5_file( $params['src'] ),
333  'content-type' => $contentType,
334  'x-object-meta-sha1base36' => $sha1Hash
335  ] + $this->sanitizeHdrs( $params ),
336  'body' => $handle // resource
337  ] ];
338 
339  $method = __METHOD__;
340  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
341  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
342  if ( $rcode === 201 ) {
343  // good
344  } elseif ( $rcode === 412 ) {
345  $status->fatal( 'backend-fail-contenttype', $params['dst'] );
346  } else {
347  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
348  }
349  };
350 
351  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
352  if ( !empty( $params['async'] ) ) { // deferred
353  $status->value = $opHandle;
354  } else { // actually write the object in Swift
355  $status->merge( current( $this->doExecuteOpHandlesInternal( [ $opHandle ] ) ) );
356  }
357 
358  return $status;
359  }
360 
361  protected function doCopyInternal( array $params ) {
362  $status = $this->newStatus();
363 
364  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
365  if ( $srcRel === null ) {
366  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
367 
368  return $status;
369  }
370 
371  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
372  if ( $dstRel === null ) {
373  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
374 
375  return $status;
376  }
377 
378  $reqs = [ [
379  'method' => 'PUT',
380  'url' => [ $dstCont, $dstRel ],
381  'headers' => [
382  'x-copy-from' => '/' . rawurlencode( $srcCont ) .
383  '/' . str_replace( "%2F", "/", rawurlencode( $srcRel ) )
384  ] + $this->sanitizeHdrs( $params ), // extra headers merged into object
385  ] ];
386 
387  $method = __METHOD__;
388  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
389  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
390  if ( $rcode === 201 ) {
391  // good
392  } elseif ( $rcode === 404 ) {
393  $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
394  } else {
395  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
396  }
397  };
398 
399  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
400  if ( !empty( $params['async'] ) ) { // deferred
401  $status->value = $opHandle;
402  } else { // actually write the object in Swift
403  $status->merge( current( $this->doExecuteOpHandlesInternal( [ $opHandle ] ) ) );
404  }
405 
406  return $status;
407  }
408 
409  protected function doMoveInternal( array $params ) {
410  $status = $this->newStatus();
411 
412  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
413  if ( $srcRel === null ) {
414  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
415 
416  return $status;
417  }
418 
419  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
420  if ( $dstRel === null ) {
421  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
422 
423  return $status;
424  }
425 
426  $reqs = [
427  [
428  'method' => 'PUT',
429  'url' => [ $dstCont, $dstRel ],
430  'headers' => [
431  'x-copy-from' => '/' . rawurlencode( $srcCont ) .
432  '/' . str_replace( "%2F", "/", rawurlencode( $srcRel ) )
433  ] + $this->sanitizeHdrs( $params ) // extra headers merged into object
434  ]
435  ];
436  if ( "{$srcCont}/{$srcRel}" !== "{$dstCont}/{$dstRel}" ) {
437  $reqs[] = [
438  'method' => 'DELETE',
439  'url' => [ $srcCont, $srcRel ],
440  'headers' => []
441  ];
442  }
443 
444  $method = __METHOD__;
445  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
446  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
447  if ( $request['method'] === 'PUT' && $rcode === 201 ) {
448  // good
449  } elseif ( $request['method'] === 'DELETE' && $rcode === 204 ) {
450  // good
451  } elseif ( $rcode === 404 ) {
452  $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
453  } else {
454  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
455  }
456  };
457 
458  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
459  if ( !empty( $params['async'] ) ) { // deferred
460  $status->value = $opHandle;
461  } else { // actually move the object in Swift
462  $status->merge( current( $this->doExecuteOpHandlesInternal( [ $opHandle ] ) ) );
463  }
464 
465  return $status;
466  }
467 
468  protected function doDeleteInternal( array $params ) {
469  $status = $this->newStatus();
470 
471  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
472  if ( $srcRel === null ) {
473  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
474 
475  return $status;
476  }
477 
478  $reqs = [ [
479  'method' => 'DELETE',
480  'url' => [ $srcCont, $srcRel ],
481  'headers' => []
482  ] ];
483 
484  $method = __METHOD__;
485  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
486  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
487  if ( $rcode === 204 ) {
488  // good
489  } elseif ( $rcode === 404 ) {
490  if ( empty( $params['ignoreMissingSource'] ) ) {
491  $status->fatal( 'backend-fail-delete', $params['src'] );
492  }
493  } else {
494  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
495  }
496  };
497 
498  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
499  if ( !empty( $params['async'] ) ) { // deferred
500  $status->value = $opHandle;
501  } else { // actually delete the object in Swift
502  $status->merge( current( $this->doExecuteOpHandlesInternal( [ $opHandle ] ) ) );
503  }
504 
505  return $status;
506  }
507 
508  protected function doDescribeInternal( array $params ) {
509  $status = $this->newStatus();
510 
511  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
512  if ( $srcRel === null ) {
513  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
514 
515  return $status;
516  }
517 
518  // Fetch the old object headers/metadata...this should be in stat cache by now
519  $stat = $this->getFileStat( [ 'src' => $params['src'], 'latest' => 1 ] );
520  if ( $stat && !isset( $stat['xattr'] ) ) { // older cache entry
521  $stat = $this->doGetFileStat( [ 'src' => $params['src'], 'latest' => 1 ] );
522  }
523  if ( !$stat ) {
524  $status->fatal( 'backend-fail-describe', $params['src'] );
525 
526  return $status;
527  }
528 
529  // POST clears prior headers, so we need to merge the changes in to the old ones
530  $metaHdrs = [];
531  foreach ( $stat['xattr']['metadata'] as $name => $value ) {
532  $metaHdrs["x-object-meta-$name"] = $value;
533  }
534  $customHdrs = $this->sanitizeHdrs( $params ) + $stat['xattr']['headers'];
535 
536  $reqs = [ [
537  'method' => 'POST',
538  'url' => [ $srcCont, $srcRel ],
539  'headers' => $metaHdrs + $customHdrs
540  ] ];
541 
542  $method = __METHOD__;
543  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
544  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
545  if ( $rcode === 202 ) {
546  // good
547  } elseif ( $rcode === 404 ) {
548  $status->fatal( 'backend-fail-describe', $params['src'] );
549  } else {
550  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
551  }
552  };
553 
554  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
555  if ( !empty( $params['async'] ) ) { // deferred
556  $status->value = $opHandle;
557  } else { // actually change the object in Swift
558  $status->merge( current( $this->doExecuteOpHandlesInternal( [ $opHandle ] ) ) );
559  }
560 
561  return $status;
562  }
563 
564  protected function doPrepareInternal( $fullCont, $dir, array $params ) {
565  $status = $this->newStatus();
566 
567  // (a) Check if container already exists
568  $stat = $this->getContainerStat( $fullCont );
569  if ( is_array( $stat ) ) {
570  return $status; // already there
571  } elseif ( $stat === null ) {
572  $status->fatal( 'backend-fail-internal', $this->name );
573  $this->logger->error( __METHOD__ . ': cannot get container stat' );
574 
575  return $status;
576  }
577 
578  // (b) Create container as needed with proper ACLs
579  if ( $stat === false ) {
580  $params['op'] = 'prepare';
581  $status->merge( $this->createContainer( $fullCont, $params ) );
582  }
583 
584  return $status;
585  }
586 
587  protected function doSecureInternal( $fullCont, $dir, array $params ) {
588  $status = $this->newStatus();
589  if ( empty( $params['noAccess'] ) ) {
590  return $status; // nothing to do
591  }
592 
593  $stat = $this->getContainerStat( $fullCont );
594  if ( is_array( $stat ) ) {
595  // Make container private to end-users...
596  $status->merge( $this->setContainerAccess(
597  $fullCont,
598  [ $this->swiftUser ], // read
599  [ $this->swiftUser ] // write
600  ) );
601  } elseif ( $stat === false ) {
602  $status->fatal( 'backend-fail-usable', $params['dir'] );
603  } else {
604  $status->fatal( 'backend-fail-internal', $this->name );
605  $this->logger->error( __METHOD__ . ': cannot get container stat' );
606  }
607 
608  return $status;
609  }
610 
611  protected function doPublishInternal( $fullCont, $dir, array $params ) {
612  $status = $this->newStatus();
613 
614  $stat = $this->getContainerStat( $fullCont );
615  if ( is_array( $stat ) ) {
616  // Make container public to end-users...
617  $status->merge( $this->setContainerAccess(
618  $fullCont,
619  [ $this->swiftUser, '.r:*' ], // read
620  [ $this->swiftUser ] // write
621  ) );
622  } elseif ( $stat === false ) {
623  $status->fatal( 'backend-fail-usable', $params['dir'] );
624  } else {
625  $status->fatal( 'backend-fail-internal', $this->name );
626  $this->logger->error( __METHOD__ . ': cannot get container stat' );
627  }
628 
629  return $status;
630  }
631 
632  protected function doCleanInternal( $fullCont, $dir, array $params ) {
633  $status = $this->newStatus();
634 
635  // Only containers themselves can be removed, all else is virtual
636  if ( $dir != '' ) {
637  return $status; // nothing to do
638  }
639 
640  // (a) Check the container
641  $stat = $this->getContainerStat( $fullCont, true );
642  if ( $stat === false ) {
643  return $status; // ok, nothing to do
644  } elseif ( !is_array( $stat ) ) {
645  $status->fatal( 'backend-fail-internal', $this->name );
646  $this->logger->error( __METHOD__ . ': cannot get container stat' );
647 
648  return $status;
649  }
650 
651  // (b) Delete the container if empty
652  if ( $stat['count'] == 0 ) {
653  $params['op'] = 'clean';
654  $status->merge( $this->deleteContainer( $fullCont, $params ) );
655  }
656 
657  return $status;
658  }
659 
660  protected function doGetFileStat( array $params ) {
661  $params = [ 'srcs' => [ $params['src'] ], 'concurrency' => 1 ] + $params;
662  unset( $params['src'] );
663  $stats = $this->doGetFileStatMulti( $params );
664 
665  return reset( $stats );
666  }
667 
678  protected function convertSwiftDate( $ts, $format = TS_MW ) {
679  try {
680  $timestamp = new MWTimestamp( $ts );
681 
682  return $timestamp->getTimestamp( $format );
683  } catch ( Exception $e ) {
684  throw new FileBackendError( $e->getMessage() );
685  }
686  }
687 
695  protected function addMissingMetadata( array $objHdrs, $path ) {
696  if ( isset( $objHdrs['x-object-meta-sha1base36'] ) ) {
697  return $objHdrs; // nothing to do
698  }
699 
701  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
702  $this->logger->error( __METHOD__ . ": $path was not stored with SHA-1 metadata." );
703 
704  $objHdrs['x-object-meta-sha1base36'] = false;
705 
706  $auth = $this->getAuthentication();
707  if ( !$auth ) {
708  return $objHdrs; // failed
709  }
710 
711  // Find prior custom HTTP headers
712  $postHeaders = $this->getCustomHeaders( $objHdrs );
713  // Find prior metadata headers
714  $postHeaders += $this->getMetadataHeaders( $objHdrs );
715 
716  $status = $this->newStatus();
718  $scopeLockS = $this->getScopedFileLocks( [ $path ], LockManager::LOCK_UW, $status );
719  if ( $status->isOK() ) {
720  $tmpFile = $this->getLocalCopy( [ 'src' => $path, 'latest' => 1 ] );
721  if ( $tmpFile ) {
722  $hash = $tmpFile->getSha1Base36();
723  if ( $hash !== false ) {
724  $objHdrs['x-object-meta-sha1base36'] = $hash;
725  // Merge new SHA1 header into the old ones
726  $postHeaders['x-object-meta-sha1base36'] = $hash;
727  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
728  list( $rcode ) = $this->http->run( [
729  'method' => 'POST',
730  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
731  'headers' => $this->authTokenHeaders( $auth ) + $postHeaders
732  ] );
733  if ( $rcode >= 200 && $rcode <= 299 ) {
734  $this->deleteFileCache( $path );
735 
736  return $objHdrs; // success
737  }
738  }
739  }
740  }
741 
742  $this->logger->error( __METHOD__ . ": unable to set SHA-1 metadata for $path" );
743 
744  return $objHdrs; // failed
745  }
746 
747  protected function doGetFileContentsMulti( array $params ) {
748  $contents = [];
749 
750  $auth = $this->getAuthentication();
751 
752  $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging
753  // Blindly create tmp files and stream to them, catching any exception if the file does
754  // not exist. Doing stats here is useless and will loop infinitely in addMissingMetadata().
755  $reqs = []; // (path => op)
756 
757  foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch
758  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
759  if ( $srcRel === null || !$auth ) {
760  $contents[$path] = false;
761  continue;
762  }
763  // Create a new temporary memory file...
764  $handle = fopen( 'php://temp', 'wb' );
765  if ( $handle ) {
766  $reqs[$path] = [
767  'method' => 'GET',
768  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
769  'headers' => $this->authTokenHeaders( $auth )
770  + $this->headersFromParams( $params ),
771  'stream' => $handle,
772  ];
773  }
774  $contents[$path] = false;
775  }
776 
777  $opts = [ 'maxConnsPerHost' => $params['concurrency'] ];
778  $reqs = $this->http->runMulti( $reqs, $opts );
779  foreach ( $reqs as $path => $op ) {
780  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response'];
781  if ( $rcode >= 200 && $rcode <= 299 ) {
782  rewind( $op['stream'] ); // start from the beginning
783  $contents[$path] = stream_get_contents( $op['stream'] );
784  } elseif ( $rcode === 404 ) {
785  $contents[$path] = false;
786  } else {
787  $this->onError( null, __METHOD__,
788  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
789  }
790  fclose( $op['stream'] ); // close open handle
791  }
792 
793  return $contents;
794  }
795 
796  protected function doDirectoryExists( $fullCont, $dir, array $params ) {
797  $prefix = ( $dir == '' ) ? null : "{$dir}/";
798  $status = $this->objectListing( $fullCont, 'names', 1, null, $prefix );
799  if ( $status->isOK() ) {
800  return ( count( $status->value ) ) > 0;
801  }
802 
803  return null; // error
804  }
805 
813  public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
814  return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
815  }
816 
824  public function getFileListInternal( $fullCont, $dir, array $params ) {
825  return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
826  }
827 
839  public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
840  $dirs = [];
841  if ( $after === INF ) {
842  return $dirs; // nothing more
843  }
844 
845  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
846 
847  $prefix = ( $dir == '' ) ? null : "{$dir}/";
848  // Non-recursive: only list dirs right under $dir
849  if ( !empty( $params['topOnly'] ) ) {
850  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
851  if ( !$status->isOK() ) {
852  throw new FileBackendError( "Iterator page I/O error." );
853  }
854  $objects = $status->value;
855  foreach ( $objects as $object ) { // files and directories
856  if ( substr( $object, -1 ) === '/' ) {
857  $dirs[] = $object; // directories end in '/'
858  }
859  }
860  } else {
861  // Recursive: list all dirs under $dir and its subdirs
862  $getParentDir = function ( $path ) {
863  return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
864  };
865 
866  // Get directory from last item of prior page
867  $lastDir = $getParentDir( $after ); // must be first page
868  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
869 
870  if ( !$status->isOK() ) {
871  throw new FileBackendError( "Iterator page I/O error." );
872  }
873 
874  $objects = $status->value;
875 
876  foreach ( $objects as $object ) { // files
877  $objectDir = $getParentDir( $object ); // directory of object
878 
879  if ( $objectDir !== false && $objectDir !== $dir ) {
880  // Swift stores paths in UTF-8, using binary sorting.
881  // See function "create_container_table" in common/db.py.
882  // If a directory is not "greater" than the last one,
883  // then it was already listed by the calling iterator.
884  if ( strcmp( $objectDir, $lastDir ) > 0 ) {
885  $pDir = $objectDir;
886  do { // add dir and all its parent dirs
887  $dirs[] = "{$pDir}/";
888  $pDir = $getParentDir( $pDir );
889  } while ( $pDir !== false // sanity
890  && strcmp( $pDir, $lastDir ) > 0 // not done already
891  && strlen( $pDir ) > strlen( $dir ) // within $dir
892  );
893  }
894  $lastDir = $objectDir;
895  }
896  }
897  }
898  // Page on the unfiltered directory listing (what is returned may be filtered)
899  if ( count( $objects ) < $limit ) {
900  $after = INF; // avoid a second RTT
901  } else {
902  $after = end( $objects ); // update last item
903  }
904 
905  return $dirs;
906  }
907 
919  public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
920  $files = []; // list of (path, stat array or null) entries
921  if ( $after === INF ) {
922  return $files; // nothing more
923  }
924 
925  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
926 
927  $prefix = ( $dir == '' ) ? null : "{$dir}/";
928  // $objects will contain a list of unfiltered names or CF_Object items
929  // Non-recursive: only list files right under $dir
930  if ( !empty( $params['topOnly'] ) ) {
931  if ( !empty( $params['adviseStat'] ) ) {
932  $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix, '/' );
933  } else {
934  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
935  }
936  } else {
937  // Recursive: list all files under $dir and its subdirs
938  if ( !empty( $params['adviseStat'] ) ) {
939  $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix );
940  } else {
941  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
942  }
943  }
944 
945  // Reformat this list into a list of (name, stat array or null) entries
946  if ( !$status->isOK() ) {
947  throw new FileBackendError( "Iterator page I/O error." );
948  }
949 
950  $objects = $status->value;
951  $files = $this->buildFileObjectListing( $params, $dir, $objects );
952 
953  // Page on the unfiltered object listing (what is returned may be filtered)
954  if ( count( $objects ) < $limit ) {
955  $after = INF; // avoid a second RTT
956  } else {
957  $after = end( $objects ); // update last item
958  $after = is_object( $after ) ? $after->name : $after;
959  }
960 
961  return $files;
962  }
963 
973  private function buildFileObjectListing( array $params, $dir, array $objects ) {
974  $names = [];
975  foreach ( $objects as $object ) {
976  if ( is_object( $object ) ) {
977  if ( isset( $object->subdir ) || !isset( $object->name ) ) {
978  continue; // virtual directory entry; ignore
979  }
980  $stat = [
981  // Convert various random Swift dates to TS_MW
982  'mtime' => $this->convertSwiftDate( $object->last_modified, TS_MW ),
983  'size' => (int)$object->bytes,
984  'sha1' => null,
985  // Note: manifiest ETags are not an MD5 of the file
986  'md5' => ctype_xdigit( $object->hash ) ? $object->hash : null,
987  'latest' => false // eventually consistent
988  ];
989  $names[] = [ $object->name, $stat ];
990  } elseif ( substr( $object, -1 ) !== '/' ) {
991  // Omit directories, which end in '/' in listings
992  $names[] = [ $object, null ];
993  }
994  }
995 
996  return $names;
997  }
998 
1005  public function loadListingStatInternal( $path, array $val ) {
1006  $this->cheapCache->set( $path, 'stat', $val );
1007  }
1008 
1009  protected function doGetFileXAttributes( array $params ) {
1010  $stat = $this->getFileStat( $params );
1011  if ( $stat ) {
1012  if ( !isset( $stat['xattr'] ) ) {
1013  // Stat entries filled by file listings don't include metadata/headers
1014  $this->clearCache( [ $params['src'] ] );
1015  $stat = $this->getFileStat( $params );
1016  }
1017 
1018  return $stat['xattr'];
1019  } else {
1020  return false;
1021  }
1022  }
1023 
1024  protected function doGetFileSha1base36( array $params ) {
1025  $stat = $this->getFileStat( $params );
1026  if ( $stat ) {
1027  if ( !isset( $stat['sha1'] ) ) {
1028  // Stat entries filled by file listings don't include SHA1
1029  $this->clearCache( [ $params['src'] ] );
1030  $stat = $this->getFileStat( $params );
1031  }
1032 
1033  return $stat['sha1'];
1034  } else {
1035  return false;
1036  }
1037  }
1038 
1039  protected function doStreamFile( array $params ) {
1040  $status = $this->newStatus();
1041 
1042  $flags = !empty( $params['headless'] ) ? StreamFile::STREAM_HEADLESS : 0;
1043 
1044  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1045  if ( $srcRel === null ) {
1046  StreamFile::send404Message( $params['src'], $flags );
1047  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
1048 
1049  return $status;
1050  }
1051 
1052  $auth = $this->getAuthentication();
1053  if ( !$auth || !is_array( $this->getContainerStat( $srcCont ) ) ) {
1054  StreamFile::send404Message( $params['src'], $flags );
1055  $status->fatal( 'backend-fail-stream', $params['src'] );
1056 
1057  return $status;
1058  }
1059 
1060  // If "headers" is set, we only want to send them if the file is there.
1061  // Do not bother checking if the file exists if headers are not set though.
1062  if ( $params['headers'] && !$this->fileExists( $params ) ) {
1063  StreamFile::send404Message( $params['src'], $flags );
1064  $status->fatal( 'backend-fail-stream', $params['src'] );
1065 
1066  return $status;
1067  }
1068 
1069  // Send the requested additional headers
1070  foreach ( $params['headers'] as $header ) {
1071  header( $header ); // aways send
1072  }
1073 
1074  if ( empty( $params['allowOB'] ) ) {
1075  // Cancel output buffering and gzipping if set
1076  call_user_func( $this->obResetFunc );
1077  }
1078 
1079  $handle = fopen( 'php://output', 'wb' );
1080  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1081  'method' => 'GET',
1082  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1083  'headers' => $this->authTokenHeaders( $auth )
1084  + $this->headersFromParams( $params ) + $params['options'],
1085  'stream' => $handle,
1086  'flags' => [ 'relayResponseHeaders' => empty( $params['headless'] ) ]
1087  ] );
1088 
1089  if ( $rcode >= 200 && $rcode <= 299 ) {
1090  // good
1091  } elseif ( $rcode === 404 ) {
1092  $status->fatal( 'backend-fail-stream', $params['src'] );
1093  // Per bug 41113, nasty things can happen if bad cache entries get
1094  // stuck in cache. It's also possible that this error can come up
1095  // with simple race conditions. Clear out the stat cache to be safe.
1096  $this->clearCache( [ $params['src'] ] );
1097  $this->deleteFileCache( $params['src'] );
1098  } else {
1099  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1100  }
1101 
1102  return $status;
1103  }
1104 
1105  protected function doGetLocalCopyMulti( array $params ) {
1107  $tmpFiles = [];
1108 
1109  $auth = $this->getAuthentication();
1110 
1111  $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging
1112  // Blindly create tmp files and stream to them, catching any exception if the file does
1113  // not exist. Doing a stat here is useless causes infinite loops in addMissingMetadata().
1114  $reqs = []; // (path => op)
1115 
1116  foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch
1117  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
1118  if ( $srcRel === null || !$auth ) {
1119  $tmpFiles[$path] = null;
1120  continue;
1121  }
1122  // Get source file extension
1124  // Create a new temporary file...
1125  $tmpFile = TempFSFile::factory( 'localcopy_', $ext, $this->tmpDirectory );
1126  if ( $tmpFile ) {
1127  $handle = fopen( $tmpFile->getPath(), 'wb' );
1128  if ( $handle ) {
1129  $reqs[$path] = [
1130  'method' => 'GET',
1131  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1132  'headers' => $this->authTokenHeaders( $auth )
1133  + $this->headersFromParams( $params ),
1134  'stream' => $handle,
1135  ];
1136  } else {
1137  $tmpFile = null;
1138  }
1139  }
1140  $tmpFiles[$path] = $tmpFile;
1141  }
1142 
1143  $isLatest = ( $this->isRGW || !empty( $params['latest'] ) );
1144  $opts = [ 'maxConnsPerHost' => $params['concurrency'] ];
1145  $reqs = $this->http->runMulti( $reqs, $opts );
1146  foreach ( $reqs as $path => $op ) {
1147  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response'];
1148  fclose( $op['stream'] ); // close open handle
1149  if ( $rcode >= 200 && $rcode <= 299 ) {
1150  $size = $tmpFiles[$path] ? $tmpFiles[$path]->getSize() : 0;
1151  // Double check that the disk is not full/broken
1152  if ( $size != $rhdrs['content-length'] ) {
1153  $tmpFiles[$path] = null;
1154  $rerr = "Got {$size}/{$rhdrs['content-length']} bytes";
1155  $this->onError( null, __METHOD__,
1156  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
1157  }
1158  // Set the file stat process cache in passing
1159  $stat = $this->getStatFromHeaders( $rhdrs );
1160  $stat['latest'] = $isLatest;
1161  $this->cheapCache->set( $path, 'stat', $stat );
1162  } elseif ( $rcode === 404 ) {
1163  $tmpFiles[$path] = false;
1164  } else {
1165  $tmpFiles[$path] = null;
1166  $this->onError( null, __METHOD__,
1167  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
1168  }
1169  }
1170 
1171  return $tmpFiles;
1172  }
1173 
1174  public function getFileHttpUrl( array $params ) {
1175  if ( $this->swiftTempUrlKey != '' ||
1176  ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' )
1177  ) {
1178  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1179  if ( $srcRel === null ) {
1180  return null; // invalid path
1181  }
1182 
1183  $auth = $this->getAuthentication();
1184  if ( !$auth ) {
1185  return null;
1186  }
1187 
1188  $ttl = isset( $params['ttl'] ) ? $params['ttl'] : 86400;
1189  $expires = time() + $ttl;
1190 
1191  if ( $this->swiftTempUrlKey != '' ) {
1192  $url = $this->storageUrl( $auth, $srcCont, $srcRel );
1193  // Swift wants the signature based on the unencoded object name
1194  $contPath = parse_url( $this->storageUrl( $auth, $srcCont ), PHP_URL_PATH );
1195  $signature = hash_hmac( 'sha1',
1196  "GET\n{$expires}\n{$contPath}/{$srcRel}",
1197  $this->swiftTempUrlKey
1198  );
1199 
1200  return "{$url}?temp_url_sig={$signature}&temp_url_expires={$expires}";
1201  } else { // give S3 API URL for rgw
1202  // Path for signature starts with the bucket
1203  $spath = '/' . rawurlencode( $srcCont ) . '/' .
1204  str_replace( '%2F', '/', rawurlencode( $srcRel ) );
1205  // Calculate the hash
1206  $signature = base64_encode( hash_hmac(
1207  'sha1',
1208  "GET\n\n\n{$expires}\n{$spath}",
1209  $this->rgwS3SecretKey,
1210  true // raw
1211  ) );
1212  // See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html.
1213  // Note: adding a newline for empty CanonicalizedAmzHeaders does not work.
1214  // Note: S3 API is the rgw default; remove the /swift/ URL bit.
1215  return str_replace( '/swift/v1', '', $this->storageUrl( $auth ) . $spath ) .
1216  '?' .
1217  http_build_query( [
1218  'Signature' => $signature,
1219  'Expires' => $expires,
1220  'AWSAccessKeyId' => $this->rgwS3AccessKey
1221  ] );
1222  }
1223  }
1224 
1225  return null;
1226  }
1227 
1228  protected function directoriesAreVirtual() {
1229  return true;
1230  }
1231 
1240  protected function headersFromParams( array $params ) {
1241  $hdrs = [];
1242  if ( !empty( $params['latest'] ) ) {
1243  $hdrs['x-newest'] = 'true';
1244  }
1245 
1246  return $hdrs;
1247  }
1248 
1254  protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1256  $statuses = [];
1257 
1258  $auth = $this->getAuthentication();
1259  if ( !$auth ) {
1260  foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1261  $statuses[$index] = $this->newStatus( 'backend-fail-connect', $this->name );
1262  }
1263 
1264  return $statuses;
1265  }
1266 
1267  // Split the HTTP requests into stages that can be done concurrently
1268  $httpReqsByStage = []; // map of (stage => index => HTTP request)
1269  foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1271  $reqs = $fileOpHandle->httpOp;
1272  // Convert the 'url' parameter to an actual URL using $auth
1273  foreach ( $reqs as $stage => &$req ) {
1274  list( $container, $relPath ) = $req['url'];
1275  $req['url'] = $this->storageUrl( $auth, $container, $relPath );
1276  $req['headers'] = isset( $req['headers'] ) ? $req['headers'] : [];
1277  $req['headers'] = $this->authTokenHeaders( $auth ) + $req['headers'];
1278  $httpReqsByStage[$stage][$index] = $req;
1279  }
1280  $statuses[$index] = $this->newStatus();
1281  }
1282 
1283  // Run all requests for the first stage, then the next, and so on
1284  $reqCount = count( $httpReqsByStage );
1285  for ( $stage = 0; $stage < $reqCount; ++$stage ) {
1286  $httpReqs = $this->http->runMulti( $httpReqsByStage[$stage] );
1287  foreach ( $httpReqs as $index => $httpReq ) {
1288  // Run the callback for each request of this operation
1289  $callback = $fileOpHandles[$index]->callback;
1290  call_user_func_array( $callback, [ $httpReq, $statuses[$index] ] );
1291  // On failure, abort all remaining requests for this operation
1292  // (e.g. abort the DELETE request if the COPY request fails for a move)
1293  if ( !$statuses[$index]->isOK() ) {
1294  $stages = count( $fileOpHandles[$index]->httpOp );
1295  for ( $s = ( $stage + 1 ); $s < $stages; ++$s ) {
1296  unset( $httpReqsByStage[$s][$index] );
1297  }
1298  }
1299  }
1300  }
1301 
1302  return $statuses;
1303  }
1304 
1327  protected function setContainerAccess( $container, array $readGrps, array $writeGrps ) {
1328  $status = $this->newStatus();
1329  $auth = $this->getAuthentication();
1330 
1331  if ( !$auth ) {
1332  $status->fatal( 'backend-fail-connect', $this->name );
1333 
1334  return $status;
1335  }
1336 
1337  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1338  'method' => 'POST',
1339  'url' => $this->storageUrl( $auth, $container ),
1340  'headers' => $this->authTokenHeaders( $auth ) + [
1341  'x-container-read' => implode( ',', $readGrps ),
1342  'x-container-write' => implode( ',', $writeGrps )
1343  ]
1344  ] );
1345 
1346  if ( $rcode != 204 && $rcode !== 202 ) {
1347  $status->fatal( 'backend-fail-internal', $this->name );
1348  $this->logger->error( __METHOD__ . ': unexpected rcode value (' . $rcode . ')' );
1349  }
1350 
1351  return $status;
1352  }
1353 
1362  protected function getContainerStat( $container, $bypassCache = false ) {
1363  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1364 
1365  if ( $bypassCache ) { // purge cache
1366  $this->containerStatCache->clear( $container );
1367  } elseif ( !$this->containerStatCache->has( $container, 'stat' ) ) {
1368  $this->primeContainerCache( [ $container ] ); // check persistent cache
1369  }
1370  if ( !$this->containerStatCache->has( $container, 'stat' ) ) {
1371  $auth = $this->getAuthentication();
1372  if ( !$auth ) {
1373  return null;
1374  }
1375 
1376  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1377  'method' => 'HEAD',
1378  'url' => $this->storageUrl( $auth, $container ),
1379  'headers' => $this->authTokenHeaders( $auth )
1380  ] );
1381 
1382  if ( $rcode === 204 ) {
1383  $stat = [
1384  'count' => $rhdrs['x-container-object-count'],
1385  'bytes' => $rhdrs['x-container-bytes-used']
1386  ];
1387  if ( $bypassCache ) {
1388  return $stat;
1389  } else {
1390  $this->containerStatCache->set( $container, 'stat', $stat ); // cache it
1391  $this->setContainerCache( $container, $stat ); // update persistent cache
1392  }
1393  } elseif ( $rcode === 404 ) {
1394  return false;
1395  } else {
1396  $this->onError( null, __METHOD__,
1397  [ 'cont' => $container ], $rerr, $rcode, $rdesc );
1398 
1399  return null;
1400  }
1401  }
1402 
1403  return $this->containerStatCache->get( $container, 'stat' );
1404  }
1405 
1413  protected function createContainer( $container, array $params ) {
1414  $status = $this->newStatus();
1415 
1416  $auth = $this->getAuthentication();
1417  if ( !$auth ) {
1418  $status->fatal( 'backend-fail-connect', $this->name );
1419 
1420  return $status;
1421  }
1422 
1423  // @see SwiftFileBackend::setContainerAccess()
1424  if ( empty( $params['noAccess'] ) ) {
1425  $readGrps = [ '.r:*', $this->swiftUser ]; // public
1426  } else {
1427  $readGrps = [ $this->swiftUser ]; // private
1428  }
1429  $writeGrps = [ $this->swiftUser ]; // sanity
1430 
1431  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1432  'method' => 'PUT',
1433  'url' => $this->storageUrl( $auth, $container ),
1434  'headers' => $this->authTokenHeaders( $auth ) + [
1435  'x-container-read' => implode( ',', $readGrps ),
1436  'x-container-write' => implode( ',', $writeGrps )
1437  ]
1438  ] );
1439 
1440  if ( $rcode === 201 ) { // new
1441  // good
1442  } elseif ( $rcode === 202 ) { // already there
1443  // this shouldn't really happen, but is OK
1444  } else {
1445  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1446  }
1447 
1448  return $status;
1449  }
1450 
1458  protected function deleteContainer( $container, array $params ) {
1459  $status = $this->newStatus();
1460 
1461  $auth = $this->getAuthentication();
1462  if ( !$auth ) {
1463  $status->fatal( 'backend-fail-connect', $this->name );
1464 
1465  return $status;
1466  }
1467 
1468  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1469  'method' => 'DELETE',
1470  'url' => $this->storageUrl( $auth, $container ),
1471  'headers' => $this->authTokenHeaders( $auth )
1472  ] );
1473 
1474  if ( $rcode >= 200 && $rcode <= 299 ) { // deleted
1475  $this->containerStatCache->clear( $container ); // purge
1476  } elseif ( $rcode === 404 ) { // not there
1477  // this shouldn't really happen, but is OK
1478  } elseif ( $rcode === 409 ) { // not empty
1479  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); // race?
1480  } else {
1481  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1482  }
1483 
1484  return $status;
1485  }
1486 
1499  private function objectListing(
1500  $fullCont, $type, $limit, $after = null, $prefix = null, $delim = null
1501  ) {
1502  $status = $this->newStatus();
1503 
1504  $auth = $this->getAuthentication();
1505  if ( !$auth ) {
1506  $status->fatal( 'backend-fail-connect', $this->name );
1507 
1508  return $status;
1509  }
1510 
1511  $query = [ 'limit' => $limit ];
1512  if ( $type === 'info' ) {
1513  $query['format'] = 'json';
1514  }
1515  if ( $after !== null ) {
1516  $query['marker'] = $after;
1517  }
1518  if ( $prefix !== null ) {
1519  $query['prefix'] = $prefix;
1520  }
1521  if ( $delim !== null ) {
1522  $query['delimiter'] = $delim;
1523  }
1524 
1525  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1526  'method' => 'GET',
1527  'url' => $this->storageUrl( $auth, $fullCont ),
1528  'query' => $query,
1529  'headers' => $this->authTokenHeaders( $auth )
1530  ] );
1531 
1532  $params = [ 'cont' => $fullCont, 'prefix' => $prefix, 'delim' => $delim ];
1533  if ( $rcode === 200 ) { // good
1534  if ( $type === 'info' ) {
1535  $status->value = FormatJson::decode( trim( $rbody ) );
1536  } else {
1537  $status->value = explode( "\n", trim( $rbody ) );
1538  }
1539  } elseif ( $rcode === 204 ) {
1540  $status->value = []; // empty container
1541  } elseif ( $rcode === 404 ) {
1542  $status->value = []; // no container
1543  } else {
1544  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1545  }
1546 
1547  return $status;
1548  }
1549 
1550  protected function doPrimeContainerCache( array $containerInfo ) {
1551  foreach ( $containerInfo as $container => $info ) {
1552  $this->containerStatCache->set( $container, 'stat', $info );
1553  }
1554  }
1555 
1556  protected function doGetFileStatMulti( array $params ) {
1557  $stats = [];
1558 
1559  $auth = $this->getAuthentication();
1560 
1561  $reqs = [];
1562  foreach ( $params['srcs'] as $path ) {
1563  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
1564  if ( $srcRel === null ) {
1565  $stats[$path] = false;
1566  continue; // invalid storage path
1567  } elseif ( !$auth ) {
1568  $stats[$path] = null;
1569  continue;
1570  }
1571 
1572  // (a) Check the container
1573  $cstat = $this->getContainerStat( $srcCont );
1574  if ( $cstat === false ) {
1575  $stats[$path] = false;
1576  continue; // ok, nothing to do
1577  } elseif ( !is_array( $cstat ) ) {
1578  $stats[$path] = null;
1579  continue;
1580  }
1581 
1582  $reqs[$path] = [
1583  'method' => 'HEAD',
1584  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1585  'headers' => $this->authTokenHeaders( $auth ) + $this->headersFromParams( $params )
1586  ];
1587  }
1588 
1589  $opts = [ 'maxConnsPerHost' => $params['concurrency'] ];
1590  $reqs = $this->http->runMulti( $reqs, $opts );
1591 
1592  foreach ( $params['srcs'] as $path ) {
1593  if ( array_key_exists( $path, $stats ) ) {
1594  continue; // some sort of failure above
1595  }
1596  // (b) Check the file
1597  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $reqs[$path]['response'];
1598  if ( $rcode === 200 || $rcode === 204 ) {
1599  // Update the object if it is missing some headers
1600  $rhdrs = $this->addMissingMetadata( $rhdrs, $path );
1601  // Load the stat array from the headers
1602  $stat = $this->getStatFromHeaders( $rhdrs );
1603  if ( $this->isRGW ) {
1604  $stat['latest'] = true; // strong consistency
1605  }
1606  } elseif ( $rcode === 404 ) {
1607  $stat = false;
1608  } else {
1609  $stat = null;
1610  $this->onError( null, __METHOD__, $params, $rerr, $rcode, $rdesc );
1611  }
1612  $stats[$path] = $stat;
1613  }
1614 
1615  return $stats;
1616  }
1617 
1622  protected function getStatFromHeaders( array $rhdrs ) {
1623  // Fetch all of the custom metadata headers
1624  $metadata = $this->getMetadata( $rhdrs );
1625  // Fetch all of the custom raw HTTP headers
1626  $headers = $this->sanitizeHdrs( [ 'headers' => $rhdrs ] );
1627 
1628  return [
1629  // Convert various random Swift dates to TS_MW
1630  'mtime' => $this->convertSwiftDate( $rhdrs['last-modified'], TS_MW ),
1631  // Empty objects actually return no content-length header in Ceph
1632  'size' => isset( $rhdrs['content-length'] ) ? (int)$rhdrs['content-length'] : 0,
1633  'sha1' => isset( $metadata['sha1base36'] ) ? $metadata['sha1base36'] : null,
1634  // Note: manifiest ETags are not an MD5 of the file
1635  'md5' => ctype_xdigit( $rhdrs['etag'] ) ? $rhdrs['etag'] : null,
1636  'xattr' => [ 'metadata' => $metadata, 'headers' => $headers ]
1637  ];
1638  }
1639 
1643  protected function getAuthentication() {
1644  if ( $this->authErrorTimestamp !== null ) {
1645  if ( ( time() - $this->authErrorTimestamp ) < 60 ) {
1646  return null; // failed last attempt; don't bother
1647  } else { // actually retry this time
1648  $this->authErrorTimestamp = null;
1649  }
1650  }
1651  // Session keys expire after a while, so we renew them periodically
1652  $reAuth = ( ( time() - $this->authSessionTimestamp ) > $this->authTTL );
1653  // Authenticate with proxy and get a session key...
1654  if ( !$this->authCreds || $reAuth ) {
1655  $this->authSessionTimestamp = 0;
1656  $cacheKey = $this->getCredsCacheKey( $this->swiftUser );
1657  $creds = $this->srvCache->get( $cacheKey ); // credentials
1658  // Try to use the credential cache
1659  if ( isset( $creds['auth_token'] ) && isset( $creds['storage_url'] ) ) {
1660  $this->authCreds = $creds;
1661  // Skew the timestamp for worst case to avoid using stale credentials
1662  $this->authSessionTimestamp = time() - ceil( $this->authTTL / 2 );
1663  } else { // cache miss
1664  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1665  'method' => 'GET',
1666  'url' => "{$this->swiftAuthUrl}/v1.0",
1667  'headers' => [
1668  'x-auth-user' => $this->swiftUser,
1669  'x-auth-key' => $this->swiftKey
1670  ]
1671  ] );
1672 
1673  if ( $rcode >= 200 && $rcode <= 299 ) { // OK
1674  $this->authCreds = [
1675  'auth_token' => $rhdrs['x-auth-token'],
1676  'storage_url' => $rhdrs['x-storage-url']
1677  ];
1678  $this->srvCache->set( $cacheKey, $this->authCreds, ceil( $this->authTTL / 2 ) );
1679  $this->authSessionTimestamp = time();
1680  } elseif ( $rcode === 401 ) {
1681  $this->onError( null, __METHOD__, [], "Authentication failed.", $rcode );
1682  $this->authErrorTimestamp = time();
1683 
1684  return null;
1685  } else {
1686  $this->onError( null, __METHOD__, [], "HTTP return code: $rcode", $rcode );
1687  $this->authErrorTimestamp = time();
1688 
1689  return null;
1690  }
1691  }
1692  // Ceph RGW does not use <account> in URLs (OpenStack Swift uses "/v1/<account>")
1693  if ( substr( $this->authCreds['storage_url'], -3 ) === '/v1' ) {
1694  $this->isRGW = true; // take advantage of strong consistency in Ceph
1695  }
1696  }
1697 
1698  return $this->authCreds;
1699  }
1700 
1707  protected function storageUrl( array $creds, $container = null, $object = null ) {
1708  $parts = [ $creds['storage_url'] ];
1709  if ( strlen( $container ) ) {
1710  $parts[] = rawurlencode( $container );
1711  }
1712  if ( strlen( $object ) ) {
1713  $parts[] = str_replace( "%2F", "/", rawurlencode( $object ) );
1714  }
1715 
1716  return implode( '/', $parts );
1717  }
1718 
1723  protected function authTokenHeaders( array $creds ) {
1724  return [ 'x-auth-token' => $creds['auth_token'] ];
1725  }
1726 
1733  private function getCredsCacheKey( $username ) {
1734  return 'swiftcredentials:' . md5( $username . ':' . $this->swiftAuthUrl );
1735  }
1736 
1748  public function onError( $status, $func, array $params, $err = '', $code = 0, $desc = '' ) {
1749  if ( $status instanceof StatusValue ) {
1750  $status->fatal( 'backend-fail-internal', $this->name );
1751  }
1752  if ( $code == 401 ) { // possibly a stale token
1753  $this->srvCache->delete( $this->getCredsCacheKey( $this->swiftUser ) );
1754  }
1755  $this->logger->error(
1756  "HTTP $code ($desc) in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
1757  ( $err ? ": $err" : "" )
1758  );
1759  }
1760 }
1761 
1767  public $httpOp;
1769  public $callback;
1770 
1777  $this->backend = $backend;
1778  $this->callback = $callback;
1779  $this->httpOp = $httpOp;
1780  }
1781 }
1782 
1790 abstract class SwiftFileBackendList implements Iterator {
1792  protected $bufferIter = [];
1793 
1795  protected $bufferAfter = null;
1796 
1798  protected $pos = 0;
1799 
1801  protected $params = [];
1802 
1804  protected $backend;
1805 
1807  protected $container;
1808 
1810  protected $dir;
1811 
1813  protected $suffixStart;
1814 
1815  const PAGE_SIZE = 9000; // file listing buffer size
1816 
1823  public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
1824  $this->backend = $backend;
1825  $this->container = $fullCont;
1826  $this->dir = $dir;
1827  if ( substr( $this->dir, -1 ) === '/' ) {
1828  $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
1829  }
1830  if ( $this->dir == '' ) { // whole container
1831  $this->suffixStart = 0;
1832  } else { // dir within container
1833  $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
1834  }
1835  $this->params = $params;
1836  }
1837 
1842  public function key() {
1843  return $this->pos;
1844  }
1845 
1849  public function next() {
1850  // Advance to the next file in the page
1851  next( $this->bufferIter );
1852  ++$this->pos;
1853  // Check if there are no files left in this page and
1854  // advance to the next page if this page was not empty.
1855  if ( !$this->valid() && count( $this->bufferIter ) ) {
1856  $this->bufferIter = $this->pageFromList(
1857  $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1858  ); // updates $this->bufferAfter
1859  }
1860  }
1861 
1865  public function rewind() {
1866  $this->pos = 0;
1867  $this->bufferAfter = null;
1868  $this->bufferIter = $this->pageFromList(
1869  $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1870  ); // updates $this->bufferAfter
1871  }
1872 
1877  public function valid() {
1878  if ( $this->bufferIter === null ) {
1879  return false; // some failure?
1880  } else {
1881  return ( current( $this->bufferIter ) !== false ); // no paths can have this value
1882  }
1883  }
1884 
1895  abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1896 }
1897 
1906  public function current() {
1907  return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
1908  }
1909 
1910  protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1911  return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1912  }
1913 }
1914 
1923  public function current() {
1924  list( $path, $stat ) = current( $this->bufferIter );
1925  $relPath = substr( $path, $this->suffixStart );
1926  if ( is_array( $stat ) ) {
1927  $storageDir = rtrim( $this->params['dir'], '/' );
1928  $this->backend->loadListingStatInternal( "$storageDir/$relPath", $stat );
1929  }
1930 
1931  return $relPath;
1932  }
1933 
1934  protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1935  return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
1936  }
1937 }
doStoreInternal(array $params)
doPrepareInternal($fullCont, $dir, array $params)
doGetFileStatMulti(array $params)
newStatus()
Yields the result of the status wrapper callback on either:
getFileListPageInternal($fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
buildFileObjectListing(array $params, $dir, array $objects)
Build a list of file objects, filtering out any directories and extracting any stat info if provided ...
doCleanInternal($fullCont, $dir, array $params)
getFileListInternal($fullCont, $dir, array $params)
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition: deferred.txt:11
the array() calling protocol came about after MediaWiki 1.4rc1.
resolveContainerPath($container, $relStoragePath)
null for the local wiki Added should default to null in handler for backwards compatibility add a value to it if you want to add a cookie that have to vary cache options can modify $query
Definition: hooks.txt:1555
getCustomHeaders(array $rawHeaders)
if(count($args)==0) $dir
Iterator for listing directories.
processing should stop and the error should be shown to the user * false
Definition: hooks.txt:189
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
div flags Integer display flags(NO_ACTION_LINK, NO_EXTRA_USER_LINKS) 'LogException'returning false will NOT prevent logging $e
Definition: hooks.txt:2102
string $swiftUser
Swift user (account:user) to authenticate as.
int $authErrorTimestamp
UNIX timestamp.
scopedProfileSection($section)
deleteContainer($container, array $params)
Delete a Swift container.
getFileStat(array $params)
const ATTR_HEADERS
Bitfield flags for supported features.
doDirectoryExists($fullCont, $dir, array $params)
doPublishInternal($fullCont, $dir, array $params)
string $bufferAfter
List items after this path.
$value
$files
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
it s the revision text itself In either if gzip is the revision text is gzipped $flags
Definition: hooks.txt:2703
SwiftFileBackend helper class to page through listings.
doDeleteInternal(array $params)
static send404Message($fname, $flags=0)
Send out a standard 404 message for a file.
Definition: StreamFile.php:70
doMoveInternal(array $params)
getStatFromHeaders(array $rhdrs)
static extensionFromPath($path, $case= 'lowercase')
Get the final extension from a storage or FS path.
setContainerAccess($container, array $readGrps, array $writeGrps)
Set read/write permissions for a Swift container.
doGetFileXAttributes(array $params)
static factory($prefix, $extension= '', $tmpDirectory=null)
Make a new temporary file on the file system.
Definition: TempFSFile.php:55
loadListingStatInternal($path, array $val)
Do not call this function outside of SwiftFileBackendFileList.
Apache License January http
doGetFileContentsMulti(array $params)
addMissingMetadata(array $objHdrs, $path)
Fill in any missing object metadata and save it to Swift.
doGetLocalCopyMulti(array $params)
doGetFileSha1base36(array $params)
getContainerStat($container, $bypassCache=false)
Get a Swift container stat array, possibly from process cache.
doDescribeInternal(array $params)
getMetadata(array $rawHeaders)
getFileHttpUrl(array $params)
const LOCK_UW
Definition: LockManager.php:69
File backend exception for checked exceptions (e.g.
fileExists(array $params)
if($limit) $timestamp
authTokenHeaders(array $creds)
headersFromParams(array $params)
Get headers to send to Swift when reading a file based on a FileBackend params array, e.g.
array $bufferIter
List of path or (path,stat array) entries.
storageUrl(array $creds, $container=null, $object=null)
isPathUsableInternal($storagePath)
__construct(SwiftFileBackend $backend, $fullCont, $dir, array $params)
static encode($value, $pretty=false, $escaping=0)
Returns the JSON representation of a value.
Definition: FormatJson.php:127
$params
const STREAM_HEADLESS
Definition: StreamFile.php:28
string $swiftAuthUrl
Authentication base URL (without version)
string $dir
Storage directory.
const TS_MW
MediaWiki concatenated string timestamp (YYYYMMDDHHMMSS)
Definition: defines.php:11
const ATTR_UNICODE_PATHS
getCredsCacheKey($username)
Get the cache key for a container.
const ATTR_METADATA
clearCache(array $paths=null)
A BagOStuff object with no objects in it.
resolveStoragePathReal($storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
objectListing($fullCont, $type, $limit, $after=null, $prefix=null, $delim=null)
Get a list of objects under a container.
$header
string $container
Container name.
int $authTTL
TTL in seconds.
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that probably a stub it is not rendered in wiki pages or galleries in category pages allow injecting custom HTML after the section Any uses of the hook need to handle escaping see BaseTemplate::getToolbox and BaseTemplate::makeListItem for details on the format of individual items inside of this array or by returning and letting standard HTTP rendering take place modifiable or by returning false and taking over the output modifiable & $code
Definition: hooks.txt:802
getContentType($storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
onError($status, $func, array $params, $err= '', $code=0, $desc= '')
Log an unexpected exception for this backend.
string $name
Unique backend name.
Definition: FileBackend.php:95
Base class for all backends using particular storage medium.
getLocalCopy(array $params)
Get a local copy on disk of the file at a storage path in the backend.
doStreamFile(array $params)
pageFromList($container, $dir, &$after, $limit, array $params)
pageFromList($container, $dir, &$after, $limit, array $params)
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
this hook is for auditing only $req
Definition: hooks.txt:1007
this hook is for auditing only or null if authentication failed before getting that far $username
Definition: hooks.txt:802
Iterator for listing regular files.
setContainerCache($container, array $val)
Set the cached info for a container.
getDirectoryListInternal($fullCont, $dir, array $params)
__construct(array $config)
error also a ContextSource you ll probably need to make sure the header is varied on $request
Definition: hooks.txt:2573
SwiftFileBackend $backend
sanitizeHdrs(array $params)
Sanitize and filter the custom headers from a $params array.
doCopyInternal(array $params)
convertSwiftDate($ts, $format=TS_MW)
Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT"/"2013-05-11T07:37:27.678360Z". ...
string $rgwS3AccessKey
S3 access key (RADOS Gateway)
doCreateInternal(array $params)
array $httpOp
List of Requests for MultiHttpClient.
doPrimeContainerCache(array $containerInfo)
Class for an OpenStack Swift (or Ceph RGW) based file backend.
design txt This is a brief overview of the new design More thorough and up to date information is available on the documentation wiki at name
Definition: design.txt:12
this hook is for auditing only RecentChangesLinked and Watchlist RecentChangesLinked and Watchlist e g Watchlist removed from all revisions and log entries to which it was applied This gives extensions a chance to take it off their books as the deletion has already been partly carried out by this point or something similar the user will be unable to create the tag set and then return false from the hook function Ensure you consume the ChangeTagAfterDelete hook to carry out custom deletion actions as context called by AbstractContent::getParserOutput May be used to override the normal model specific rendering of page content as context as context the output can only depend on parameters provided to this hook not on global state indicating whether full HTML should be generated If generation of HTML may be but other information should still be present in the ParserOutput object to manipulate or replace but no entry for that model exists in $wgContentHandlers if desired whether it is OK to use $contentModel on $title Handler functions that modify $ok should generally return false to prevent further hooks from further modifying $ok inclusive $limit
Definition: hooks.txt:1046
pageFromList($container, $dir, &$after, $limit, array $params)
Get the given list portion (page)
this hook is for auditing only RecentChangesLinked and Watchlist RecentChangesLinked and Watchlist e g Watchlist removed from all revisions and log entries to which it was applied This gives extensions a chance to take it off their books as the deletion has already been partly carried out by this point or something similar the user will be unable to create the tag set $status
Definition: hooks.txt:1046
getMetadataHeaders(array $rawHeaders)
int $authSessionTimestamp
UNIX timestamp.
MultiHttpClient $http
deleteFileCache($path)
Delete the cached stat info for a file path.
ProcessCacheLRU $containerStatCache
Container stat cache.
getDirListPageInternal($fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
doExecuteOpHandlesInternal(array $fileOpHandles)
doSecureInternal($fullCont, $dir, array $params)
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that probably a stub it is not rendered in wiki pages or galleries in category pages allow injecting custom HTML after the section Any uses of the hook need to handle escaping see BaseTemplate::getToolbox and BaseTemplate::makeListItem for details on the format of individual items inside of this array or by returning and letting standard HTTP rendering take place modifiable or by returning false and taking over the output modifiable modifiable after all normalizations have been except for the $wgMaxImageArea check set to true or false to override the $wgMaxImageArea check result gives extension the possibility to transform it themselves $handler
Definition: hooks.txt:802
Class to handle concurrent HTTP requests.
static decode($value, $assoc=false)
Decodes a JSON string.
Definition: FormatJson.php:187
doGetFileStat(array $params)
createContainer($container, array $params)
Create a Swift container.
FileBackendStore helper class for performing asynchronous file operations.
string $rgwS3SecretKey
S3 authentication key (RADOS Gateway)
Handles per process caching of items.
Library for creating and parsing MW-style timestamps.
Definition: MWTimestamp.php:31
do that in ParserLimitReportFormat instead use this to modify the parameters of the image and a DIV can begin in one section and end in another Make sure your code can handle that case gracefully See the EditSectionClearerLink extension for an example zero but section is usually empty its values are the globals values before the output is cached one of or reset my talk my contributions etc etc otherwise the built in rate limiting checks are if enabled allows for interception of redirect as a string mapping parameter names to values & $type
Definition: hooks.txt:2491
__construct(SwiftFileBackend $backend, Closure $callback, array $httpOp)
string $swiftKey
Secret key for user.
bool $isRGW
Whether the server is an Ceph RGW.
string $swiftTempUrlKey
Shared secret value for making temp URLs.