MediaWiki  1.27.2
SwiftFileBackend.php
Go to the documentation of this file.
1 <?php
37  protected $http;
38 
40  protected $authTTL;
41 
43  protected $swiftAuthUrl;
44 
46  protected $swiftUser;
47 
49  protected $swiftKey;
50 
52  protected $swiftTempUrlKey;
53 
55  protected $rgwS3AccessKey;
56 
58  protected $rgwS3SecretKey;
59 
61  protected $srvCache;
62 
65 
67  protected $authCreds;
68 
70  protected $authSessionTimestamp = 0;
71 
73  protected $authErrorTimestamp = null;
74 
76  protected $isRGW = false;
77 
106  public function __construct( array $config ) {
107  parent::__construct( $config );
108  // Required settings
109  $this->swiftAuthUrl = $config['swiftAuthUrl'];
110  $this->swiftUser = $config['swiftUser'];
111  $this->swiftKey = $config['swiftKey'];
112  // Optional settings
113  $this->authTTL = isset( $config['swiftAuthTTL'] )
114  ? $config['swiftAuthTTL']
115  : 15 * 60; // some sane number
116  $this->swiftTempUrlKey = isset( $config['swiftTempUrlKey'] )
117  ? $config['swiftTempUrlKey']
118  : '';
119  $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
120  ? $config['shardViaHashLevels']
121  : '';
122  $this->rgwS3AccessKey = isset( $config['rgwS3AccessKey'] )
123  ? $config['rgwS3AccessKey']
124  : '';
125  $this->rgwS3SecretKey = isset( $config['rgwS3SecretKey'] )
126  ? $config['rgwS3SecretKey']
127  : '';
128  // HTTP helper client
129  $this->http = new MultiHttpClient( [] );
130  // Cache container information to mask latency
131  if ( isset( $config['wanCache'] ) && $config['wanCache'] instanceof WANObjectCache ) {
132  $this->memCache = $config['wanCache'];
133  }
134  // Process cache for container info
135  $this->containerStatCache = new ProcessCacheLRU( 300 );
136  // Cache auth token information to avoid RTTs
137  if ( !empty( $config['cacheAuthInfo'] ) ) {
138  if ( PHP_SAPI === 'cli' ) {
139  // Preferrably memcached
140  $this->srvCache = ObjectCache::getLocalClusterInstance();
141  } else {
142  // Look for APC, XCache, WinCache, ect...
143  $this->srvCache = ObjectCache::getLocalServerInstance( CACHE_NONE );
144  }
145  } else {
146  $this->srvCache = new EmptyBagOStuff();
147  }
148  }
149 
150  public function getFeatures() {
153  }
154 
155  protected function resolveContainerPath( $container, $relStoragePath ) {
156  if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) {
157  return null; // not UTF-8, makes it hard to use CF and the swift HTTP API
158  } elseif ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
159  return null; // too long for Swift
160  }
161 
162  return $relStoragePath;
163  }
164 
165  public function isPathUsableInternal( $storagePath ) {
166  list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
167  if ( $rel === null ) {
168  return false; // invalid
169  }
170 
171  return is_array( $this->getContainerStat( $container ) );
172  }
173 
181  protected function sanitizeHdrs( array $params ) {
182  return isset( $params['headers'] )
183  ? $this->getCustomHeaders( $params['headers'] )
184  : [];
185 
186  }
187 
192  protected function getCustomHeaders( array $rawHeaders ) {
193  $headers = [];
194 
195  // Normalize casing, and strip out illegal headers
196  foreach ( $rawHeaders as $name => $value ) {
197  $name = strtolower( $name );
198  if ( preg_match( '/^content-(type|length)$/', $name ) ) {
199  continue; // blacklisted
200  } elseif ( preg_match( '/^(x-)?content-/', $name ) ) {
201  $headers[$name] = $value; // allowed
202  } elseif ( preg_match( '/^content-(disposition)/', $name ) ) {
203  $headers[$name] = $value; // allowed
204  }
205  }
206  // By default, Swift has annoyingly low maximum header value limits
207  if ( isset( $headers['content-disposition'] ) ) {
208  $disposition = '';
209  // @note: assume FileBackend::makeContentDisposition() already used
210  foreach ( explode( ';', $headers['content-disposition'] ) as $part ) {
211  $part = trim( $part );
212  $new = ( $disposition === '' ) ? $part : "{$disposition};{$part}";
213  if ( strlen( $new ) <= 255 ) {
214  $disposition = $new;
215  } else {
216  break; // too long; sigh
217  }
218  }
219  $headers['content-disposition'] = $disposition;
220  }
221 
222  return $headers;
223  }
224 
229  protected function getMetadataHeaders( array $rawHeaders ) {
230  $headers = [];
231  foreach ( $rawHeaders as $name => $value ) {
232  $name = strtolower( $name );
233  if ( strpos( $name, 'x-object-meta-' ) === 0 ) {
234  $headers[$name] = $value;
235  }
236  }
237 
238  return $headers;
239  }
240 
245  protected function getMetadata( array $rawHeaders ) {
246  $metadata = [];
247  foreach ( $this->getMetadataHeaders( $rawHeaders ) as $name => $value ) {
248  $metadata[substr( $name, strlen( 'x-object-meta-' ) )] = $value;
249  }
250 
251  return $metadata;
252  }
253 
254  protected function doCreateInternal( array $params ) {
256 
257  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
258  if ( $dstRel === null ) {
259  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
260 
261  return $status;
262  }
263 
264  $sha1Hash = Wikimedia\base_convert( sha1( $params['content'] ), 16, 36, 31 );
265  $contentType = isset( $params['headers']['content-type'] )
266  ? $params['headers']['content-type']
267  : $this->getContentType( $params['dst'], $params['content'], null );
268 
269  $reqs = [ [
270  'method' => 'PUT',
271  'url' => [ $dstCont, $dstRel ],
272  'headers' => [
273  'content-length' => strlen( $params['content'] ),
274  'etag' => md5( $params['content'] ),
275  'content-type' => $contentType,
276  'x-object-meta-sha1base36' => $sha1Hash
277  ] + $this->sanitizeHdrs( $params ),
278  'body' => $params['content']
279  ] ];
280 
281  $method = __METHOD__;
282  $handler = function ( array $request, Status $status ) use ( $method, $params ) {
283  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
284  if ( $rcode === 201 ) {
285  // good
286  } elseif ( $rcode === 412 ) {
287  $status->fatal( 'backend-fail-contenttype', $params['dst'] );
288  } else {
289  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
290  }
291  };
292 
293  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
294  if ( !empty( $params['async'] ) ) { // deferred
295  $status->value = $opHandle;
296  } else { // actually write the object in Swift
297  $status->merge( current( $this->doExecuteOpHandlesInternal( [ $opHandle ] ) ) );
298  }
299 
300  return $status;
301  }
302 
303  protected function doStoreInternal( array $params ) {
305 
306  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
307  if ( $dstRel === null ) {
308  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
309 
310  return $status;
311  }
312 
313  MediaWiki\suppressWarnings();
314  $sha1Hash = sha1_file( $params['src'] );
315  MediaWiki\restoreWarnings();
316  if ( $sha1Hash === false ) { // source doesn't exist?
317  $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
318 
319  return $status;
320  }
321  $sha1Hash = Wikimedia\base_convert( $sha1Hash, 16, 36, 31 );
322  $contentType = isset( $params['headers']['content-type'] )
323  ? $params['headers']['content-type']
324  : $this->getContentType( $params['dst'], null, $params['src'] );
325 
326  $handle = fopen( $params['src'], 'rb' );
327  if ( $handle === false ) { // source doesn't exist?
328  $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
329 
330  return $status;
331  }
332 
333  $reqs = [ [
334  'method' => 'PUT',
335  'url' => [ $dstCont, $dstRel ],
336  'headers' => [
337  'content-length' => filesize( $params['src'] ),
338  'etag' => md5_file( $params['src'] ),
339  'content-type' => $contentType,
340  'x-object-meta-sha1base36' => $sha1Hash
341  ] + $this->sanitizeHdrs( $params ),
342  'body' => $handle // resource
343  ] ];
344 
345  $method = __METHOD__;
346  $handler = function ( array $request, Status $status ) use ( $method, $params ) {
347  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
348  if ( $rcode === 201 ) {
349  // good
350  } elseif ( $rcode === 412 ) {
351  $status->fatal( 'backend-fail-contenttype', $params['dst'] );
352  } else {
353  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
354  }
355  };
356 
357  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
358  if ( !empty( $params['async'] ) ) { // deferred
359  $status->value = $opHandle;
360  } else { // actually write the object in Swift
361  $status->merge( current( $this->doExecuteOpHandlesInternal( [ $opHandle ] ) ) );
362  }
363 
364  return $status;
365  }
366 
367  protected function doCopyInternal( array $params ) {
369 
370  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
371  if ( $srcRel === null ) {
372  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
373 
374  return $status;
375  }
376 
377  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
378  if ( $dstRel === null ) {
379  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
380 
381  return $status;
382  }
383 
384  $reqs = [ [
385  'method' => 'PUT',
386  'url' => [ $dstCont, $dstRel ],
387  'headers' => [
388  'x-copy-from' => '/' . rawurlencode( $srcCont ) .
389  '/' . str_replace( "%2F", "/", rawurlencode( $srcRel ) )
390  ] + $this->sanitizeHdrs( $params ), // extra headers merged into object
391  ] ];
392 
393  $method = __METHOD__;
394  $handler = function ( array $request, Status $status ) use ( $method, $params ) {
395  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
396  if ( $rcode === 201 ) {
397  // good
398  } elseif ( $rcode === 404 ) {
399  $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
400  } else {
401  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
402  }
403  };
404 
405  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
406  if ( !empty( $params['async'] ) ) { // deferred
407  $status->value = $opHandle;
408  } else { // actually write the object in Swift
409  $status->merge( current( $this->doExecuteOpHandlesInternal( [ $opHandle ] ) ) );
410  }
411 
412  return $status;
413  }
414 
415  protected function doMoveInternal( array $params ) {
417 
418  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
419  if ( $srcRel === null ) {
420  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
421 
422  return $status;
423  }
424 
425  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
426  if ( $dstRel === null ) {
427  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
428 
429  return $status;
430  }
431 
432  $reqs = [
433  [
434  'method' => 'PUT',
435  'url' => [ $dstCont, $dstRel ],
436  'headers' => [
437  'x-copy-from' => '/' . rawurlencode( $srcCont ) .
438  '/' . str_replace( "%2F", "/", rawurlencode( $srcRel ) )
439  ] + $this->sanitizeHdrs( $params ) // extra headers merged into object
440  ]
441  ];
442  if ( "{$srcCont}/{$srcRel}" !== "{$dstCont}/{$dstRel}" ) {
443  $reqs[] = [
444  'method' => 'DELETE',
445  'url' => [ $srcCont, $srcRel ],
446  'headers' => []
447  ];
448  }
449 
450  $method = __METHOD__;
451  $handler = function ( array $request, Status $status ) use ( $method, $params ) {
452  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
453  if ( $request['method'] === 'PUT' && $rcode === 201 ) {
454  // good
455  } elseif ( $request['method'] === 'DELETE' && $rcode === 204 ) {
456  // good
457  } elseif ( $rcode === 404 ) {
458  $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
459  } else {
460  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
461  }
462  };
463 
464  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
465  if ( !empty( $params['async'] ) ) { // deferred
466  $status->value = $opHandle;
467  } else { // actually move the object in Swift
468  $status->merge( current( $this->doExecuteOpHandlesInternal( [ $opHandle ] ) ) );
469  }
470 
471  return $status;
472  }
473 
474  protected function doDeleteInternal( array $params ) {
476 
477  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
478  if ( $srcRel === null ) {
479  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
480 
481  return $status;
482  }
483 
484  $reqs = [ [
485  'method' => 'DELETE',
486  'url' => [ $srcCont, $srcRel ],
487  'headers' => []
488  ] ];
489 
490  $method = __METHOD__;
491  $handler = function ( array $request, Status $status ) use ( $method, $params ) {
492  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
493  if ( $rcode === 204 ) {
494  // good
495  } elseif ( $rcode === 404 ) {
496  if ( empty( $params['ignoreMissingSource'] ) ) {
497  $status->fatal( 'backend-fail-delete', $params['src'] );
498  }
499  } else {
500  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
501  }
502  };
503 
504  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
505  if ( !empty( $params['async'] ) ) { // deferred
506  $status->value = $opHandle;
507  } else { // actually delete the object in Swift
508  $status->merge( current( $this->doExecuteOpHandlesInternal( [ $opHandle ] ) ) );
509  }
510 
511  return $status;
512  }
513 
514  protected function doDescribeInternal( array $params ) {
516 
517  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
518  if ( $srcRel === null ) {
519  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
520 
521  return $status;
522  }
523 
524  // Fetch the old object headers/metadata...this should be in stat cache by now
525  $stat = $this->getFileStat( [ 'src' => $params['src'], 'latest' => 1 ] );
526  if ( $stat && !isset( $stat['xattr'] ) ) { // older cache entry
527  $stat = $this->doGetFileStat( [ 'src' => $params['src'], 'latest' => 1 ] );
528  }
529  if ( !$stat ) {
530  $status->fatal( 'backend-fail-describe', $params['src'] );
531 
532  return $status;
533  }
534 
535  // POST clears prior headers, so we need to merge the changes in to the old ones
536  $metaHdrs = [];
537  foreach ( $stat['xattr']['metadata'] as $name => $value ) {
538  $metaHdrs["x-object-meta-$name"] = $value;
539  }
540  $customHdrs = $this->sanitizeHdrs( $params ) + $stat['xattr']['headers'];
541 
542  $reqs = [ [
543  'method' => 'POST',
544  'url' => [ $srcCont, $srcRel ],
545  'headers' => $metaHdrs + $customHdrs
546  ] ];
547 
548  $method = __METHOD__;
549  $handler = function ( array $request, Status $status ) use ( $method, $params ) {
550  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
551  if ( $rcode === 202 ) {
552  // good
553  } elseif ( $rcode === 404 ) {
554  $status->fatal( 'backend-fail-describe', $params['src'] );
555  } else {
556  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
557  }
558  };
559 
560  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
561  if ( !empty( $params['async'] ) ) { // deferred
562  $status->value = $opHandle;
563  } else { // actually change the object in Swift
564  $status->merge( current( $this->doExecuteOpHandlesInternal( [ $opHandle ] ) ) );
565  }
566 
567  return $status;
568  }
569 
570  protected function doPrepareInternal( $fullCont, $dir, array $params ) {
572 
573  // (a) Check if container already exists
574  $stat = $this->getContainerStat( $fullCont );
575  if ( is_array( $stat ) ) {
576  return $status; // already there
577  } elseif ( $stat === null ) {
578  $status->fatal( 'backend-fail-internal', $this->name );
579  wfDebugLog( 'SwiftBackend', __METHOD__ . ': cannot get container stat' );
580 
581  return $status;
582  }
583 
584  // (b) Create container as needed with proper ACLs
585  if ( $stat === false ) {
586  $params['op'] = 'prepare';
587  $status->merge( $this->createContainer( $fullCont, $params ) );
588  }
589 
590  return $status;
591  }
592 
593  protected function doSecureInternal( $fullCont, $dir, array $params ) {
595  if ( empty( $params['noAccess'] ) ) {
596  return $status; // nothing to do
597  }
598 
599  $stat = $this->getContainerStat( $fullCont );
600  if ( is_array( $stat ) ) {
601  // Make container private to end-users...
602  $status->merge( $this->setContainerAccess(
603  $fullCont,
604  [ $this->swiftUser ], // read
605  [ $this->swiftUser ] // write
606  ) );
607  } elseif ( $stat === false ) {
608  $status->fatal( 'backend-fail-usable', $params['dir'] );
609  } else {
610  $status->fatal( 'backend-fail-internal', $this->name );
611  wfDebugLog( 'SwiftBackend', __METHOD__ . ': cannot get container stat' );
612  }
613 
614  return $status;
615  }
616 
617  protected function doPublishInternal( $fullCont, $dir, array $params ) {
619 
620  $stat = $this->getContainerStat( $fullCont );
621  if ( is_array( $stat ) ) {
622  // Make container public to end-users...
623  $status->merge( $this->setContainerAccess(
624  $fullCont,
625  [ $this->swiftUser, '.r:*' ], // read
626  [ $this->swiftUser ] // write
627  ) );
628  } elseif ( $stat === false ) {
629  $status->fatal( 'backend-fail-usable', $params['dir'] );
630  } else {
631  $status->fatal( 'backend-fail-internal', $this->name );
632  wfDebugLog( 'SwiftBackend', __METHOD__ . ': cannot get container stat' );
633  }
634 
635  return $status;
636  }
637 
638  protected function doCleanInternal( $fullCont, $dir, array $params ) {
640 
641  // Only containers themselves can be removed, all else is virtual
642  if ( $dir != '' ) {
643  return $status; // nothing to do
644  }
645 
646  // (a) Check the container
647  $stat = $this->getContainerStat( $fullCont, true );
648  if ( $stat === false ) {
649  return $status; // ok, nothing to do
650  } elseif ( !is_array( $stat ) ) {
651  $status->fatal( 'backend-fail-internal', $this->name );
652  wfDebugLog( 'SwiftBackend', __METHOD__ . ': cannot get container stat' );
653 
654  return $status;
655  }
656 
657  // (b) Delete the container if empty
658  if ( $stat['count'] == 0 ) {
659  $params['op'] = 'clean';
660  $status->merge( $this->deleteContainer( $fullCont, $params ) );
661  }
662 
663  return $status;
664  }
665 
666  protected function doGetFileStat( array $params ) {
667  $params = [ 'srcs' => [ $params['src'] ], 'concurrency' => 1 ] + $params;
668  unset( $params['src'] );
669  $stats = $this->doGetFileStatMulti( $params );
670 
671  return reset( $stats );
672  }
673 
684  protected function convertSwiftDate( $ts, $format = TS_MW ) {
685  try {
686  $timestamp = new MWTimestamp( $ts );
687 
688  return $timestamp->getTimestamp( $format );
689  } catch ( Exception $e ) {
690  throw new FileBackendError( $e->getMessage() );
691  }
692  }
693 
701  protected function addMissingMetadata( array $objHdrs, $path ) {
702  if ( isset( $objHdrs['x-object-meta-sha1base36'] ) ) {
703  return $objHdrs; // nothing to do
704  }
705 
707  $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" );
708  wfDebugLog( 'SwiftBackend', __METHOD__ . ": $path was not stored with SHA-1 metadata." );
709 
710  $objHdrs['x-object-meta-sha1base36'] = false;
711 
712  $auth = $this->getAuthentication();
713  if ( !$auth ) {
714  return $objHdrs; // failed
715  }
716 
717  // Find prior custom HTTP headers
718  $postHeaders = $this->getCustomHeaders( $objHdrs );
719  // Find prior metadata headers
720  $postHeaders += $this->getMetadataHeaders( $objHdrs );
721 
724  $scopeLockS = $this->getScopedFileLocks( [ $path ], LockManager::LOCK_UW, $status );
725  if ( $status->isOK() ) {
726  $tmpFile = $this->getLocalCopy( [ 'src' => $path, 'latest' => 1 ] );
727  if ( $tmpFile ) {
728  $hash = $tmpFile->getSha1Base36();
729  if ( $hash !== false ) {
730  $objHdrs['x-object-meta-sha1base36'] = $hash;
731  // Merge new SHA1 header into the old ones
732  $postHeaders['x-object-meta-sha1base36'] = $hash;
733  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
734  list( $rcode ) = $this->http->run( [
735  'method' => 'POST',
736  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
737  'headers' => $this->authTokenHeaders( $auth ) + $postHeaders
738  ] );
739  if ( $rcode >= 200 && $rcode <= 299 ) {
740  $this->deleteFileCache( $path );
741 
742  return $objHdrs; // success
743  }
744  }
745  }
746  }
747 
748  wfDebugLog( 'SwiftBackend', __METHOD__ . ": unable to set SHA-1 metadata for $path" );
749 
750  return $objHdrs; // failed
751  }
752 
753  protected function doGetFileContentsMulti( array $params ) {
754  $contents = [];
755 
756  $auth = $this->getAuthentication();
757 
758  $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging
759  // Blindly create tmp files and stream to them, catching any exception if the file does
760  // not exist. Doing stats here is useless and will loop infinitely in addMissingMetadata().
761  $reqs = []; // (path => op)
762 
763  foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch
764  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
765  if ( $srcRel === null || !$auth ) {
766  $contents[$path] = false;
767  continue;
768  }
769  // Create a new temporary memory file...
770  $handle = fopen( 'php://temp', 'wb' );
771  if ( $handle ) {
772  $reqs[$path] = [
773  'method' => 'GET',
774  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
775  'headers' => $this->authTokenHeaders( $auth )
776  + $this->headersFromParams( $params ),
777  'stream' => $handle,
778  ];
779  }
780  $contents[$path] = false;
781  }
782 
783  $opts = [ 'maxConnsPerHost' => $params['concurrency'] ];
784  $reqs = $this->http->runMulti( $reqs, $opts );
785  foreach ( $reqs as $path => $op ) {
786  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response'];
787  if ( $rcode >= 200 && $rcode <= 299 ) {
788  rewind( $op['stream'] ); // start from the beginning
789  $contents[$path] = stream_get_contents( $op['stream'] );
790  } elseif ( $rcode === 404 ) {
791  $contents[$path] = false;
792  } else {
793  $this->onError( null, __METHOD__,
794  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
795  }
796  fclose( $op['stream'] ); // close open handle
797  }
798 
799  return $contents;
800  }
801 
802  protected function doDirectoryExists( $fullCont, $dir, array $params ) {
803  $prefix = ( $dir == '' ) ? null : "{$dir}/";
804  $status = $this->objectListing( $fullCont, 'names', 1, null, $prefix );
805  if ( $status->isOK() ) {
806  return ( count( $status->value ) ) > 0;
807  }
808 
809  return null; // error
810  }
811 
819  public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
820  return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
821  }
822 
830  public function getFileListInternal( $fullCont, $dir, array $params ) {
831  return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
832  }
833 
845  public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
846  $dirs = [];
847  if ( $after === INF ) {
848  return $dirs; // nothing more
849  }
850 
851  $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" );
852 
853  $prefix = ( $dir == '' ) ? null : "{$dir}/";
854  // Non-recursive: only list dirs right under $dir
855  if ( !empty( $params['topOnly'] ) ) {
856  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
857  if ( !$status->isOK() ) {
858  throw new FileBackendError( "Iterator page I/O error: {$status->getMessage()}" );
859  }
860  $objects = $status->value;
861  foreach ( $objects as $object ) { // files and directories
862  if ( substr( $object, -1 ) === '/' ) {
863  $dirs[] = $object; // directories end in '/'
864  }
865  }
866  } else {
867  // Recursive: list all dirs under $dir and its subdirs
868  $getParentDir = function ( $path ) {
869  return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
870  };
871 
872  // Get directory from last item of prior page
873  $lastDir = $getParentDir( $after ); // must be first page
874  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
875 
876  if ( !$status->isOK() ) {
877  throw new FileBackendError( "Iterator page I/O error: {$status->getMessage()}" );
878  }
879 
880  $objects = $status->value;
881 
882  foreach ( $objects as $object ) { // files
883  $objectDir = $getParentDir( $object ); // directory of object
884 
885  if ( $objectDir !== false && $objectDir !== $dir ) {
886  // Swift stores paths in UTF-8, using binary sorting.
887  // See function "create_container_table" in common/db.py.
888  // If a directory is not "greater" than the last one,
889  // then it was already listed by the calling iterator.
890  if ( strcmp( $objectDir, $lastDir ) > 0 ) {
891  $pDir = $objectDir;
892  do { // add dir and all its parent dirs
893  $dirs[] = "{$pDir}/";
894  $pDir = $getParentDir( $pDir );
895  } while ( $pDir !== false // sanity
896  && strcmp( $pDir, $lastDir ) > 0 // not done already
897  && strlen( $pDir ) > strlen( $dir ) // within $dir
898  );
899  }
900  $lastDir = $objectDir;
901  }
902  }
903  }
904  // Page on the unfiltered directory listing (what is returned may be filtered)
905  if ( count( $objects ) < $limit ) {
906  $after = INF; // avoid a second RTT
907  } else {
908  $after = end( $objects ); // update last item
909  }
910 
911  return $dirs;
912  }
913 
925  public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
926  $files = []; // list of (path, stat array or null) entries
927  if ( $after === INF ) {
928  return $files; // nothing more
929  }
930 
931  $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" );
932 
933  $prefix = ( $dir == '' ) ? null : "{$dir}/";
934  // $objects will contain a list of unfiltered names or CF_Object items
935  // Non-recursive: only list files right under $dir
936  if ( !empty( $params['topOnly'] ) ) {
937  if ( !empty( $params['adviseStat'] ) ) {
938  $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix, '/' );
939  } else {
940  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
941  }
942  } else {
943  // Recursive: list all files under $dir and its subdirs
944  if ( !empty( $params['adviseStat'] ) ) {
945  $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix );
946  } else {
947  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
948  }
949  }
950 
951  // Reformat this list into a list of (name, stat array or null) entries
952  if ( !$status->isOK() ) {
953  throw new FileBackendError( "Iterator page I/O error: {$status->getMessage()}" );
954  }
955 
956  $objects = $status->value;
957  $files = $this->buildFileObjectListing( $params, $dir, $objects );
958 
959  // Page on the unfiltered object listing (what is returned may be filtered)
960  if ( count( $objects ) < $limit ) {
961  $after = INF; // avoid a second RTT
962  } else {
963  $after = end( $objects ); // update last item
964  $after = is_object( $after ) ? $after->name : $after;
965  }
966 
967  return $files;
968  }
969 
979  private function buildFileObjectListing( array $params, $dir, array $objects ) {
980  $names = [];
981  foreach ( $objects as $object ) {
982  if ( is_object( $object ) ) {
983  if ( isset( $object->subdir ) || !isset( $object->name ) ) {
984  continue; // virtual directory entry; ignore
985  }
986  $stat = [
987  // Convert various random Swift dates to TS_MW
988  'mtime' => $this->convertSwiftDate( $object->last_modified, TS_MW ),
989  'size' => (int)$object->bytes,
990  'sha1' => null,
991  // Note: manifiest ETags are not an MD5 of the file
992  'md5' => ctype_xdigit( $object->hash ) ? $object->hash : null,
993  'latest' => false // eventually consistent
994  ];
995  $names[] = [ $object->name, $stat ];
996  } elseif ( substr( $object, -1 ) !== '/' ) {
997  // Omit directories, which end in '/' in listings
998  $names[] = [ $object, null ];
999  }
1000  }
1001 
1002  return $names;
1003  }
1004 
1011  public function loadListingStatInternal( $path, array $val ) {
1012  $this->cheapCache->set( $path, 'stat', $val );
1013  }
1014 
1015  protected function doGetFileXAttributes( array $params ) {
1016  $stat = $this->getFileStat( $params );
1017  if ( $stat ) {
1018  if ( !isset( $stat['xattr'] ) ) {
1019  // Stat entries filled by file listings don't include metadata/headers
1020  $this->clearCache( [ $params['src'] ] );
1021  $stat = $this->getFileStat( $params );
1022  }
1023 
1024  return $stat['xattr'];
1025  } else {
1026  return false;
1027  }
1028  }
1029 
1030  protected function doGetFileSha1base36( array $params ) {
1031  $stat = $this->getFileStat( $params );
1032  if ( $stat ) {
1033  if ( !isset( $stat['sha1'] ) ) {
1034  // Stat entries filled by file listings don't include SHA1
1035  $this->clearCache( [ $params['src'] ] );
1036  $stat = $this->getFileStat( $params );
1037  }
1038 
1039  return $stat['sha1'];
1040  } else {
1041  return false;
1042  }
1043  }
1044 
1045  protected function doStreamFile( array $params ) {
1047 
1048  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1049  if ( $srcRel === null ) {
1050  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
1051  }
1052 
1053  $auth = $this->getAuthentication();
1054  if ( !$auth || !is_array( $this->getContainerStat( $srcCont ) ) ) {
1055  $status->fatal( 'backend-fail-stream', $params['src'] );
1056 
1057  return $status;
1058  }
1059 
1060  $handle = fopen( 'php://output', 'wb' );
1061 
1062  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1063  'method' => 'GET',
1064  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1065  'headers' => $this->authTokenHeaders( $auth )
1066  + $this->headersFromParams( $params ),
1067  'stream' => $handle,
1068  ] );
1069 
1070  if ( $rcode >= 200 && $rcode <= 299 ) {
1071  // good
1072  } elseif ( $rcode === 404 ) {
1073  $status->fatal( 'backend-fail-stream', $params['src'] );
1074  } else {
1075  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1076  }
1077 
1078  return $status;
1079  }
1080 
1081  protected function doGetLocalCopyMulti( array $params ) {
1082  $tmpFiles = [];
1083 
1084  $auth = $this->getAuthentication();
1085 
1086  $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging
1087  // Blindly create tmp files and stream to them, catching any exception if the file does
1088  // not exist. Doing a stat here is useless causes infinite loops in addMissingMetadata().
1089  $reqs = []; // (path => op)
1090 
1091  foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch
1092  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
1093  if ( $srcRel === null || !$auth ) {
1094  $tmpFiles[$path] = null;
1095  continue;
1096  }
1097  // Get source file extension
1099  // Create a new temporary file...
1100  $tmpFile = TempFSFile::factory( 'localcopy_', $ext );
1101  if ( $tmpFile ) {
1102  $handle = fopen( $tmpFile->getPath(), 'wb' );
1103  if ( $handle ) {
1104  $reqs[$path] = [
1105  'method' => 'GET',
1106  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1107  'headers' => $this->authTokenHeaders( $auth )
1108  + $this->headersFromParams( $params ),
1109  'stream' => $handle,
1110  ];
1111  } else {
1112  $tmpFile = null;
1113  }
1114  }
1115  $tmpFiles[$path] = $tmpFile;
1116  }
1117 
1118  $isLatest = ( $this->isRGW || !empty( $params['latest'] ) );
1119  $opts = [ 'maxConnsPerHost' => $params['concurrency'] ];
1120  $reqs = $this->http->runMulti( $reqs, $opts );
1121  foreach ( $reqs as $path => $op ) {
1122  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response'];
1123  fclose( $op['stream'] ); // close open handle
1124  if ( $rcode >= 200 && $rcode <= 299 ) {
1125  $size = $tmpFiles[$path] ? $tmpFiles[$path]->getSize() : 0;
1126  // Double check that the disk is not full/broken
1127  if ( $size != $rhdrs['content-length'] ) {
1128  $tmpFiles[$path] = null;
1129  $rerr = "Got {$size}/{$rhdrs['content-length']} bytes";
1130  $this->onError( null, __METHOD__,
1131  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
1132  }
1133  // Set the file stat process cache in passing
1134  $stat = $this->getStatFromHeaders( $rhdrs );
1135  $stat['latest'] = $isLatest;
1136  $this->cheapCache->set( $path, 'stat', $stat );
1137  } elseif ( $rcode === 404 ) {
1138  $tmpFiles[$path] = false;
1139  } else {
1140  $tmpFiles[$path] = null;
1141  $this->onError( null, __METHOD__,
1142  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
1143  }
1144  }
1145 
1146  return $tmpFiles;
1147  }
1148 
1149  public function getFileHttpUrl( array $params ) {
1150  if ( $this->swiftTempUrlKey != '' ||
1151  ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' )
1152  ) {
1153  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1154  if ( $srcRel === null ) {
1155  return null; // invalid path
1156  }
1157 
1158  $auth = $this->getAuthentication();
1159  if ( !$auth ) {
1160  return null;
1161  }
1162 
1163  $ttl = isset( $params['ttl'] ) ? $params['ttl'] : 86400;
1164  $expires = time() + $ttl;
1165 
1166  if ( $this->swiftTempUrlKey != '' ) {
1167  $url = $this->storageUrl( $auth, $srcCont, $srcRel );
1168  // Swift wants the signature based on the unencoded object name
1169  $contPath = parse_url( $this->storageUrl( $auth, $srcCont ), PHP_URL_PATH );
1170  $signature = hash_hmac( 'sha1',
1171  "GET\n{$expires}\n{$contPath}/{$srcRel}",
1172  $this->swiftTempUrlKey
1173  );
1174 
1175  return "{$url}?temp_url_sig={$signature}&temp_url_expires={$expires}";
1176  } else { // give S3 API URL for rgw
1177  // Path for signature starts with the bucket
1178  $spath = '/' . rawurlencode( $srcCont ) . '/' .
1179  str_replace( '%2F', '/', rawurlencode( $srcRel ) );
1180  // Calculate the hash
1181  $signature = base64_encode( hash_hmac(
1182  'sha1',
1183  "GET\n\n\n{$expires}\n{$spath}",
1184  $this->rgwS3SecretKey,
1185  true // raw
1186  ) );
1187  // See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html.
1188  // Note: adding a newline for empty CanonicalizedAmzHeaders does not work.
1189  return wfAppendQuery(
1190  str_replace( '/swift/v1', '', // S3 API is the rgw default
1191  $this->storageUrl( $auth ) . $spath ),
1192  [
1193  'Signature' => $signature,
1194  'Expires' => $expires,
1195  'AWSAccessKeyId' => $this->rgwS3AccessKey ]
1196  );
1197  }
1198  }
1199 
1200  return null;
1201  }
1202 
1203  protected function directoriesAreVirtual() {
1204  return true;
1205  }
1206 
1215  protected function headersFromParams( array $params ) {
1216  $hdrs = [];
1217  if ( !empty( $params['latest'] ) ) {
1218  $hdrs['x-newest'] = 'true';
1219  }
1220 
1221  return $hdrs;
1222  }
1223 
1229  protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1230  $statuses = [];
1231 
1232  $auth = $this->getAuthentication();
1233  if ( !$auth ) {
1234  foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1235  $statuses[$index] = Status::newFatal( 'backend-fail-connect', $this->name );
1236  }
1237 
1238  return $statuses;
1239  }
1240 
1241  // Split the HTTP requests into stages that can be done concurrently
1242  $httpReqsByStage = []; // map of (stage => index => HTTP request)
1243  foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1244  $reqs = $fileOpHandle->httpOp;
1245  // Convert the 'url' parameter to an actual URL using $auth
1246  foreach ( $reqs as $stage => &$req ) {
1247  list( $container, $relPath ) = $req['url'];
1248  $req['url'] = $this->storageUrl( $auth, $container, $relPath );
1249  $req['headers'] = isset( $req['headers'] ) ? $req['headers'] : [];
1250  $req['headers'] = $this->authTokenHeaders( $auth ) + $req['headers'];
1251  $httpReqsByStage[$stage][$index] = $req;
1252  }
1253  $statuses[$index] = Status::newGood();
1254  }
1255 
1256  // Run all requests for the first stage, then the next, and so on
1257  $reqCount = count( $httpReqsByStage );
1258  for ( $stage = 0; $stage < $reqCount; ++$stage ) {
1259  $httpReqs = $this->http->runMulti( $httpReqsByStage[$stage] );
1260  foreach ( $httpReqs as $index => $httpReq ) {
1261  // Run the callback for each request of this operation
1262  $callback = $fileOpHandles[$index]->callback;
1263  call_user_func_array( $callback, [ $httpReq, $statuses[$index] ] );
1264  // On failure, abort all remaining requests for this operation
1265  // (e.g. abort the DELETE request if the COPY request fails for a move)
1266  if ( !$statuses[$index]->isOK() ) {
1267  $stages = count( $fileOpHandles[$index]->httpOp );
1268  for ( $s = ( $stage + 1 ); $s < $stages; ++$s ) {
1269  unset( $httpReqsByStage[$s][$index] );
1270  }
1271  }
1272  }
1273  }
1274 
1275  return $statuses;
1276  }
1277 
1300  protected function setContainerAccess( $container, array $readGrps, array $writeGrps ) {
1302  $auth = $this->getAuthentication();
1303 
1304  if ( !$auth ) {
1305  $status->fatal( 'backend-fail-connect', $this->name );
1306 
1307  return $status;
1308  }
1309 
1310  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1311  'method' => 'POST',
1312  'url' => $this->storageUrl( $auth, $container ),
1313  'headers' => $this->authTokenHeaders( $auth ) + [
1314  'x-container-read' => implode( ',', $readGrps ),
1315  'x-container-write' => implode( ',', $writeGrps )
1316  ]
1317  ] );
1318 
1319  if ( $rcode != 204 && $rcode !== 202 ) {
1320  $status->fatal( 'backend-fail-internal', $this->name );
1321  wfDebugLog( 'SwiftBackend', __METHOD__ . ': unexpected rcode value (' . $rcode . ')' );
1322  }
1323 
1324  return $status;
1325  }
1326 
1335  protected function getContainerStat( $container, $bypassCache = false ) {
1336  $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" );
1337 
1338  if ( $bypassCache ) { // purge cache
1339  $this->containerStatCache->clear( $container );
1340  } elseif ( !$this->containerStatCache->has( $container, 'stat' ) ) {
1341  $this->primeContainerCache( [ $container ] ); // check persistent cache
1342  }
1343  if ( !$this->containerStatCache->has( $container, 'stat' ) ) {
1344  $auth = $this->getAuthentication();
1345  if ( !$auth ) {
1346  return null;
1347  }
1348 
1349  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1350  'method' => 'HEAD',
1351  'url' => $this->storageUrl( $auth, $container ),
1352  'headers' => $this->authTokenHeaders( $auth )
1353  ] );
1354 
1355  if ( $rcode === 204 ) {
1356  $stat = [
1357  'count' => $rhdrs['x-container-object-count'],
1358  'bytes' => $rhdrs['x-container-bytes-used']
1359  ];
1360  if ( $bypassCache ) {
1361  return $stat;
1362  } else {
1363  $this->containerStatCache->set( $container, 'stat', $stat ); // cache it
1364  $this->setContainerCache( $container, $stat ); // update persistent cache
1365  }
1366  } elseif ( $rcode === 404 ) {
1367  return false;
1368  } else {
1369  $this->onError( null, __METHOD__,
1370  [ 'cont' => $container ], $rerr, $rcode, $rdesc );
1371 
1372  return null;
1373  }
1374  }
1375 
1376  return $this->containerStatCache->get( $container, 'stat' );
1377  }
1378 
1386  protected function createContainer( $container, array $params ) {
1388 
1389  $auth = $this->getAuthentication();
1390  if ( !$auth ) {
1391  $status->fatal( 'backend-fail-connect', $this->name );
1392 
1393  return $status;
1394  }
1395 
1396  // @see SwiftFileBackend::setContainerAccess()
1397  if ( empty( $params['noAccess'] ) ) {
1398  $readGrps = [ '.r:*', $this->swiftUser ]; // public
1399  } else {
1400  $readGrps = [ $this->swiftUser ]; // private
1401  }
1402  $writeGrps = [ $this->swiftUser ]; // sanity
1403 
1404  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1405  'method' => 'PUT',
1406  'url' => $this->storageUrl( $auth, $container ),
1407  'headers' => $this->authTokenHeaders( $auth ) + [
1408  'x-container-read' => implode( ',', $readGrps ),
1409  'x-container-write' => implode( ',', $writeGrps )
1410  ]
1411  ] );
1412 
1413  if ( $rcode === 201 ) { // new
1414  // good
1415  } elseif ( $rcode === 202 ) { // already there
1416  // this shouldn't really happen, but is OK
1417  } else {
1418  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1419  }
1420 
1421  return $status;
1422  }
1423 
1431  protected function deleteContainer( $container, array $params ) {
1433 
1434  $auth = $this->getAuthentication();
1435  if ( !$auth ) {
1436  $status->fatal( 'backend-fail-connect', $this->name );
1437 
1438  return $status;
1439  }
1440 
1441  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1442  'method' => 'DELETE',
1443  'url' => $this->storageUrl( $auth, $container ),
1444  'headers' => $this->authTokenHeaders( $auth )
1445  ] );
1446 
1447  if ( $rcode >= 200 && $rcode <= 299 ) { // deleted
1448  $this->containerStatCache->clear( $container ); // purge
1449  } elseif ( $rcode === 404 ) { // not there
1450  // this shouldn't really happen, but is OK
1451  } elseif ( $rcode === 409 ) { // not empty
1452  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); // race?
1453  } else {
1454  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1455  }
1456 
1457  return $status;
1458  }
1459 
1472  private function objectListing(
1473  $fullCont, $type, $limit, $after = null, $prefix = null, $delim = null
1474  ) {
1476 
1477  $auth = $this->getAuthentication();
1478  if ( !$auth ) {
1479  $status->fatal( 'backend-fail-connect', $this->name );
1480 
1481  return $status;
1482  }
1483 
1484  $query = [ 'limit' => $limit ];
1485  if ( $type === 'info' ) {
1486  $query['format'] = 'json';
1487  }
1488  if ( $after !== null ) {
1489  $query['marker'] = $after;
1490  }
1491  if ( $prefix !== null ) {
1492  $query['prefix'] = $prefix;
1493  }
1494  if ( $delim !== null ) {
1495  $query['delimiter'] = $delim;
1496  }
1497 
1498  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1499  'method' => 'GET',
1500  'url' => $this->storageUrl( $auth, $fullCont ),
1501  'query' => $query,
1502  'headers' => $this->authTokenHeaders( $auth )
1503  ] );
1504 
1505  $params = [ 'cont' => $fullCont, 'prefix' => $prefix, 'delim' => $delim ];
1506  if ( $rcode === 200 ) { // good
1507  if ( $type === 'info' ) {
1508  $status->value = FormatJson::decode( trim( $rbody ) );
1509  } else {
1510  $status->value = explode( "\n", trim( $rbody ) );
1511  }
1512  } elseif ( $rcode === 204 ) {
1513  $status->value = []; // empty container
1514  } elseif ( $rcode === 404 ) {
1515  $status->value = []; // no container
1516  } else {
1517  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1518  }
1519 
1520  return $status;
1521  }
1522 
1523  protected function doPrimeContainerCache( array $containerInfo ) {
1524  foreach ( $containerInfo as $container => $info ) {
1525  $this->containerStatCache->set( $container, 'stat', $info );
1526  }
1527  }
1528 
1529  protected function doGetFileStatMulti( array $params ) {
1530  $stats = [];
1531 
1532  $auth = $this->getAuthentication();
1533 
1534  $reqs = [];
1535  foreach ( $params['srcs'] as $path ) {
1536  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
1537  if ( $srcRel === null ) {
1538  $stats[$path] = false;
1539  continue; // invalid storage path
1540  } elseif ( !$auth ) {
1541  $stats[$path] = null;
1542  continue;
1543  }
1544 
1545  // (a) Check the container
1546  $cstat = $this->getContainerStat( $srcCont );
1547  if ( $cstat === false ) {
1548  $stats[$path] = false;
1549  continue; // ok, nothing to do
1550  } elseif ( !is_array( $cstat ) ) {
1551  $stats[$path] = null;
1552  continue;
1553  }
1554 
1555  $reqs[$path] = [
1556  'method' => 'HEAD',
1557  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1558  'headers' => $this->authTokenHeaders( $auth ) + $this->headersFromParams( $params )
1559  ];
1560  }
1561 
1562  $opts = [ 'maxConnsPerHost' => $params['concurrency'] ];
1563  $reqs = $this->http->runMulti( $reqs, $opts );
1564 
1565  foreach ( $params['srcs'] as $path ) {
1566  if ( array_key_exists( $path, $stats ) ) {
1567  continue; // some sort of failure above
1568  }
1569  // (b) Check the file
1570  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $reqs[$path]['response'];
1571  if ( $rcode === 200 || $rcode === 204 ) {
1572  // Update the object if it is missing some headers
1573  $rhdrs = $this->addMissingMetadata( $rhdrs, $path );
1574  // Load the stat array from the headers
1575  $stat = $this->getStatFromHeaders( $rhdrs );
1576  if ( $this->isRGW ) {
1577  $stat['latest'] = true; // strong consistency
1578  }
1579  } elseif ( $rcode === 404 ) {
1580  $stat = false;
1581  } else {
1582  $stat = null;
1583  $this->onError( null, __METHOD__, $params, $rerr, $rcode, $rdesc );
1584  }
1585  $stats[$path] = $stat;
1586  }
1587 
1588  return $stats;
1589  }
1590 
1595  protected function getStatFromHeaders( array $rhdrs ) {
1596  // Fetch all of the custom metadata headers
1597  $metadata = $this->getMetadata( $rhdrs );
1598  // Fetch all of the custom raw HTTP headers
1599  $headers = $this->sanitizeHdrs( [ 'headers' => $rhdrs ] );
1600 
1601  return [
1602  // Convert various random Swift dates to TS_MW
1603  'mtime' => $this->convertSwiftDate( $rhdrs['last-modified'], TS_MW ),
1604  // Empty objects actually return no content-length header in Ceph
1605  'size' => isset( $rhdrs['content-length'] ) ? (int)$rhdrs['content-length'] : 0,
1606  'sha1' => isset( $metadata['sha1base36'] ) ? $metadata['sha1base36'] : null,
1607  // Note: manifiest ETags are not an MD5 of the file
1608  'md5' => ctype_xdigit( $rhdrs['etag'] ) ? $rhdrs['etag'] : null,
1609  'xattr' => [ 'metadata' => $metadata, 'headers' => $headers ]
1610  ];
1611  }
1612 
1616  protected function getAuthentication() {
1617  if ( $this->authErrorTimestamp !== null ) {
1618  if ( ( time() - $this->authErrorTimestamp ) < 60 ) {
1619  return null; // failed last attempt; don't bother
1620  } else { // actually retry this time
1621  $this->authErrorTimestamp = null;
1622  }
1623  }
1624  // Session keys expire after a while, so we renew them periodically
1625  $reAuth = ( ( time() - $this->authSessionTimestamp ) > $this->authTTL );
1626  // Authenticate with proxy and get a session key...
1627  if ( !$this->authCreds || $reAuth ) {
1628  $this->authSessionTimestamp = 0;
1629  $cacheKey = $this->getCredsCacheKey( $this->swiftUser );
1630  $creds = $this->srvCache->get( $cacheKey ); // credentials
1631  // Try to use the credential cache
1632  if ( isset( $creds['auth_token'] ) && isset( $creds['storage_url'] ) ) {
1633  $this->authCreds = $creds;
1634  // Skew the timestamp for worst case to avoid using stale credentials
1635  $this->authSessionTimestamp = time() - ceil( $this->authTTL / 2 );
1636  } else { // cache miss
1637  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1638  'method' => 'GET',
1639  'url' => "{$this->swiftAuthUrl}/v1.0",
1640  'headers' => [
1641  'x-auth-user' => $this->swiftUser,
1642  'x-auth-key' => $this->swiftKey
1643  ]
1644  ] );
1645 
1646  if ( $rcode >= 200 && $rcode <= 299 ) { // OK
1647  $this->authCreds = [
1648  'auth_token' => $rhdrs['x-auth-token'],
1649  'storage_url' => $rhdrs['x-storage-url']
1650  ];
1651  $this->srvCache->set( $cacheKey, $this->authCreds, ceil( $this->authTTL / 2 ) );
1652  $this->authSessionTimestamp = time();
1653  } elseif ( $rcode === 401 ) {
1654  $this->onError( null, __METHOD__, [], "Authentication failed.", $rcode );
1655  $this->authErrorTimestamp = time();
1656 
1657  return null;
1658  } else {
1659  $this->onError( null, __METHOD__, [], "HTTP return code: $rcode", $rcode );
1660  $this->authErrorTimestamp = time();
1661 
1662  return null;
1663  }
1664  }
1665  // Ceph RGW does not use <account> in URLs (OpenStack Swift uses "/v1/<account>")
1666  if ( substr( $this->authCreds['storage_url'], -3 ) === '/v1' ) {
1667  $this->isRGW = true; // take advantage of strong consistency in Ceph
1668  }
1669  }
1670 
1671  return $this->authCreds;
1672  }
1673 
1680  protected function storageUrl( array $creds, $container = null, $object = null ) {
1681  $parts = [ $creds['storage_url'] ];
1682  if ( strlen( $container ) ) {
1683  $parts[] = rawurlencode( $container );
1684  }
1685  if ( strlen( $object ) ) {
1686  $parts[] = str_replace( "%2F", "/", rawurlencode( $object ) );
1687  }
1688 
1689  return implode( '/', $parts );
1690  }
1691 
1696  protected function authTokenHeaders( array $creds ) {
1697  return [ 'x-auth-token' => $creds['auth_token'] ];
1698  }
1699 
1706  private function getCredsCacheKey( $username ) {
1707  return 'swiftcredentials:' . md5( $username . ':' . $this->swiftAuthUrl );
1708  }
1709 
1721  public function onError( $status, $func, array $params, $err = '', $code = 0, $desc = '' ) {
1722  if ( $status instanceof Status ) {
1723  $status->fatal( 'backend-fail-internal', $this->name );
1724  }
1725  if ( $code == 401 ) { // possibly a stale token
1726  $this->srvCache->delete( $this->getCredsCacheKey( $this->swiftUser ) );
1727  }
1728  wfDebugLog( 'SwiftBackend',
1729  "HTTP $code ($desc) in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
1730  ( $err ? ": $err" : "" )
1731  );
1732  }
1733 }
1734 
1740  public $httpOp;
1742  public $callback;
1743 
1750  $this->backend = $backend;
1751  $this->callback = $callback;
1752  $this->httpOp = $httpOp;
1753  }
1754 }
1755 
1763 abstract class SwiftFileBackendList implements Iterator {
1765  protected $bufferIter = [];
1766 
1768  protected $bufferAfter = null;
1769 
1771  protected $pos = 0;
1772 
1774  protected $params = [];
1775 
1777  protected $backend;
1778 
1780  protected $container;
1781 
1783  protected $dir;
1784 
1786  protected $suffixStart;
1787 
1788  const PAGE_SIZE = 9000; // file listing buffer size
1789 
1796  public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
1797  $this->backend = $backend;
1798  $this->container = $fullCont;
1799  $this->dir = $dir;
1800  if ( substr( $this->dir, -1 ) === '/' ) {
1801  $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
1802  }
1803  if ( $this->dir == '' ) { // whole container
1804  $this->suffixStart = 0;
1805  } else { // dir within container
1806  $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
1807  }
1808  $this->params = $params;
1809  }
1810 
1815  public function key() {
1816  return $this->pos;
1817  }
1818 
1822  public function next() {
1823  // Advance to the next file in the page
1824  next( $this->bufferIter );
1825  ++$this->pos;
1826  // Check if there are no files left in this page and
1827  // advance to the next page if this page was not empty.
1828  if ( !$this->valid() && count( $this->bufferIter ) ) {
1829  $this->bufferIter = $this->pageFromList(
1830  $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1831  ); // updates $this->bufferAfter
1832  }
1833  }
1834 
1838  public function rewind() {
1839  $this->pos = 0;
1840  $this->bufferAfter = null;
1841  $this->bufferIter = $this->pageFromList(
1842  $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1843  ); // updates $this->bufferAfter
1844  }
1845 
1850  public function valid() {
1851  if ( $this->bufferIter === null ) {
1852  return false; // some failure?
1853  } else {
1854  return ( current( $this->bufferIter ) !== false ); // no paths can have this value
1855  }
1856  }
1857 
1868  abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1869 }
1870 
1879  public function current() {
1880  return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
1881  }
1882 
1883  protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1884  return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1885  }
1886 }
1887 
1896  public function current() {
1897  list( $path, $stat ) = current( $this->bufferIter );
1898  $relPath = substr( $path, $this->suffixStart );
1899  if ( is_array( $stat ) ) {
1900  $storageDir = rtrim( $this->params['dir'], '/' );
1901  $this->backend->loadListingStatInternal( "$storageDir/$relPath", $stat );
1902  }
1903 
1904  return $relPath;
1905  }
1906 
1907  protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1908  return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
1909  }
1910 }
static factory($prefix, $extension= '')
Make a new temporary file on the file system.
Definition: TempFSFile.php:54
doStoreInternal(array $params)
doPrepareInternal($fullCont, $dir, array $params)
doGetFileStatMulti(array $params)
getFileListPageInternal($fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
buildFileObjectListing(array $params, $dir, array $objects)
Build a list of file objects, filtering out any directories and extracting any stat info if provided ...
doCleanInternal($fullCont, $dir, array $params)
getFileListInternal($fullCont, $dir, array $params)
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition: deferred.txt:11
the array() calling protocol came about after MediaWiki 1.4rc1.
resolveContainerPath($container, $relStoragePath)
null for the local wiki Added should default to null in handler for backwards compatibility add a value to it if you want to add a cookie that have to vary cache options can modify $query
Definition: hooks.txt:1418
getCustomHeaders(array $rawHeaders)
if(count($args)==0) $dir
Iterator for listing directories.
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
div flags Integer display flags(NO_ACTION_LINK, NO_EXTRA_USER_LINKS) 'LogException'returning false will NOT prevent logging $e
Definition: hooks.txt:1932
string $swiftUser
Swift user (account:user) to authenticate as.
int $authErrorTimestamp
UNIX timestamp.
static instance()
Singleton.
Definition: Profiler.php:60
processing should stop and the error should be shown to the user * false
Definition: hooks.txt:189
deleteContainer($container, array $params)
Delete a Swift container.
getFileStat(array $params)
const ATTR_HEADERS
Bitfield flags for supported features.
doDirectoryExists($fullCont, $dir, array $params)
doPublishInternal($fullCont, $dir, array $params)
string $bufferAfter
List items after this path.
$value
$files
static getLocalClusterInstance()
Get the main cluster-local cache object.
SwiftFileBackend helper class to page through listings.
doDeleteInternal(array $params)
static newFatal($message)
Factory function for fatal errors.
Definition: Status.php:89
doMoveInternal(array $params)
getStatFromHeaders(array $rhdrs)
Multi-datacenter aware caching interface.
static extensionFromPath($path, $case= 'lowercase')
Get the final extension from a storage or FS path.
setContainerAccess($container, array $readGrps, array $writeGrps)
Set read/write permissions for a Swift container.
doGetFileXAttributes(array $params)
loadListingStatInternal($path, array $val)
Do not call this function outside of SwiftFileBackendFileList.
Apache License January http
doGetFileContentsMulti(array $params)
addMissingMetadata(array $objHdrs, $path)
Fill in any missing object metadata and save it to Swift.
doGetLocalCopyMulti(array $params)
doGetFileSha1base36(array $params)
getContainerStat($container, $bypassCache=false)
Get a Swift container stat array, possibly from process cache.
wfDebugLog($logGroup, $text, $dest= 'all', array $context=[])
Send a line to a supplementary debug log file, if configured, or main debug log if not...
doDescribeInternal(array $params)
getMetadata(array $rawHeaders)
getFileHttpUrl(array $params)
const LOCK_UW
Definition: LockManager.php:61
File backend exception for checked exceptions (e.g.
wfAppendQuery($url, $query)
Append a query string to an existing URL, which may or may not already have query string parameters a...
if($limit) $timestamp
authTokenHeaders(array $creds)
headersFromParams(array $params)
Get headers to send to Swift when reading a file based on a FileBackend params array, e.g.
array $bufferIter
List of path or (path,stat array) entries.
storageUrl(array $creds, $container=null, $object=null)
isPathUsableInternal($storagePath)
__construct(SwiftFileBackend $backend, $fullCont, $dir, array $params)
static encode($value, $pretty=false, $escaping=0)
Returns the JSON representation of a value.
Definition: FormatJson.php:127
$params
string $swiftAuthUrl
Authentication base URL (without version)
string $dir
Storage directory.
const ATTR_UNICODE_PATHS
getCredsCacheKey($username)
Get the cache key for a container.
const ATTR_METADATA
clearCache(array $paths=null)
A BagOStuff object with no objects in it.
resolveStoragePathReal($storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
objectListing($fullCont, $type, $limit, $after=null, $prefix=null, $delim=null)
Get a list of objects under a container.
string $container
Container name.
int $authTTL
TTL in seconds.
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that probably a stub it is not rendered in wiki pages or galleries in category pages allow injecting custom HTML after the section Any uses of the hook need to handle escaping see BaseTemplate::getToolbox and BaseTemplate::makeListItem for details on the format of individual items inside of this array or by returning and letting standard HTTP rendering take place modifiable or by returning false and taking over the output modifiable & $code
Definition: hooks.txt:762
getContentType($storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
onError($status, $func, array $params, $err= '', $code=0, $desc= '')
Log an unexpected exception for this backend.
string $name
Unique backend name.
Definition: FileBackend.php:87
Base class for all backends using particular storage medium.
getLocalCopy(array $params)
Get a local copy on disk of the file at a storage path in the backend.
doStreamFile(array $params)
pageFromList($container, $dir, &$after, $limit, array $params)
const TS_MW
MediaWiki concatenated string timestamp (YYYYMMDDHHMMSS)
pageFromList($container, $dir, &$after, $limit, array $params)
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
this hook is for auditing only $req
Definition: hooks.txt:965
this hook is for auditing only or null if authentication failed before getting that far $username
Definition: hooks.txt:762
Iterator for listing regular files.
setContainerCache($container, array $val)
Set the cached info for a container.
getDirectoryListInternal($fullCont, $dir, array $params)
__construct(array $config)
error also a ContextSource you ll probably need to make sure the header is varied on $request
Definition: hooks.txt:2418
SwiftFileBackend $backend
sanitizeHdrs(array $params)
Sanitize and filter the custom headers from a $params array.
doCopyInternal(array $params)
convertSwiftDate($ts, $format=TS_MW)
Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT"/"2013-05-11T07:37:27.678360Z". ...
string $rgwS3AccessKey
S3 access key (RADOS Gateway)
doCreateInternal(array $params)
static getLocalServerInstance($fallback=CACHE_NONE)
Factory function for CACHE_ACCEL (referenced from DefaultSettings.php)
array $httpOp
List of Requests for MultiHttpClient.
doPrimeContainerCache(array $containerInfo)
Class for an OpenStack Swift (or Ceph RGW) based file backend.
design txt This is a brief overview of the new design More thorough and up to date information is available on the documentation wiki at name
Definition: design.txt:12
this hook is for auditing only RecentChangesLinked and Watchlist RecentChangesLinked and Watchlist e g Watchlist removed from all revisions and log entries to which it was applied This gives extensions a chance to take it off their books as the deletion has already been partly carried out by this point or something similar the user will be unable to create the tag set and then return false from the hook function Ensure you consume the ChangeTagAfterDelete hook to carry out custom deletion actions as context called by AbstractContent::getParserOutput May be used to override the normal model specific rendering of page content as context as context the output can only depend on parameters provided to this hook not on global state indicating whether full HTML should be generated If generation of HTML may be but other information should still be present in the ParserOutput object to manipulate or replace but no entry for that model exists in $wgContentHandlers if desired whether it is OK to use $contentModel on $title Handler functions that modify $ok should generally return false to prevent further hooks from further modifying $ok inclusive $limit
Definition: hooks.txt:1004
pageFromList($container, $dir, &$after, $limit, array $params)
Get the given list portion (page)
this hook is for auditing only RecentChangesLinked and Watchlist RecentChangesLinked and Watchlist e g Watchlist removed from all revisions and log entries to which it was applied This gives extensions a chance to take it off their books as the deletion has already been partly carried out by this point or something similar the user will be unable to create the tag set $status
Definition: hooks.txt:1004
getMetadataHeaders(array $rawHeaders)
int $authSessionTimestamp
UNIX timestamp.
MultiHttpClient $http
deleteFileCache($path)
Delete the cached stat info for a file path.
ProcessCacheLRU $containerStatCache
Container stat cache.
getDirListPageInternal($fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
doExecuteOpHandlesInternal(array $fileOpHandles)
doSecureInternal($fullCont, $dir, array $params)
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that probably a stub it is not rendered in wiki pages or galleries in category pages allow injecting custom HTML after the section Any uses of the hook need to handle escaping see BaseTemplate::getToolbox and BaseTemplate::makeListItem for details on the format of individual items inside of this array or by returning and letting standard HTTP rendering take place modifiable or by returning false and taking over the output modifiable modifiable after all normalizations have been except for the $wgMaxImageArea check set to true or false to override the $wgMaxImageArea check result gives extension the possibility to transform it themselves $handler
Definition: hooks.txt:762
Class to handle concurrent HTTP requests.
const CACHE_NONE
Definition: Defines.php:102
static decode($value, $assoc=false)
Decodes a JSON string.
Definition: FormatJson.php:187
doGetFileStat(array $params)
createContainer($container, array $params)
Create a Swift container.
FileBackendStore helper class for performing asynchronous file operations.
string $rgwS3SecretKey
S3 authentication key (RADOS Gateway)
Handles per process caching of items.
Library for creating and parsing MW-style timestamps.
Definition: MWTimestamp.php:31
do that in ParserLimitReportFormat instead use this to modify the parameters of the image and a DIV can begin in one section and end in another Make sure your code can handle that case gracefully See the EditSectionClearerLink extension for an example zero but section is usually empty its values are the globals values before the output is cached one of or reset my talk my contributions etc etc otherwise the built in rate limiting checks are if enabled allows for interception of redirect as a string mapping parameter names to values & $type
Definition: hooks.txt:2338
__construct(SwiftFileBackend $backend, Closure $callback, array $httpOp)
string $swiftKey
Secret key for user.
static newGood($value=null)
Factory function for good results.
Definition: Status.php:101
bool $isRGW
Whether the server is an Ceph RGW.
getScopedFileLocks(array $paths, $type, Status $status, $timeout=0)
Lock the files at the given storage paths in the backend.
string $swiftTempUrlKey
Shared secret value for making temp URLs.