MediaWiki  REL1_31
SwiftFileBackend.php
Go to the documentation of this file.
1 <?php
36  protected $http;
38  protected $authTTL;
40  protected $swiftAuthUrl;
42  protected $swiftStorageUrl;
44  protected $swiftUser;
46  protected $swiftKey;
48  protected $swiftTempUrlKey;
50  protected $rgwS3AccessKey;
52  protected $rgwS3SecretKey;
54  protected $readUsers;
56  protected $writeUsers;
58  protected $secureReadUsers;
60  protected $secureWriteUsers;
61 
63  protected $srvCache;
64 
67 
69  protected $authCreds;
71  protected $authSessionTimestamp = 0;
73  protected $authErrorTimestamp = null;
74 
76  protected $isRGW = false;
77 
112  public function __construct( array $config ) {
113  parent::__construct( $config );
114  // Required settings
115  $this->swiftAuthUrl = $config['swiftAuthUrl'];
116  $this->swiftUser = $config['swiftUser'];
117  $this->swiftKey = $config['swiftKey'];
118  // Optional settings
119  $this->authTTL = isset( $config['swiftAuthTTL'] )
120  ? $config['swiftAuthTTL']
121  : 15 * 60; // some sane number
122  $this->swiftTempUrlKey = isset( $config['swiftTempUrlKey'] )
123  ? $config['swiftTempUrlKey']
124  : '';
125  $this->swiftStorageUrl = isset( $config['swiftStorageUrl'] )
126  ? $config['swiftStorageUrl']
127  : null;
128  $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
129  ? $config['shardViaHashLevels']
130  : '';
131  $this->rgwS3AccessKey = isset( $config['rgwS3AccessKey'] )
132  ? $config['rgwS3AccessKey']
133  : '';
134  $this->rgwS3SecretKey = isset( $config['rgwS3SecretKey'] )
135  ? $config['rgwS3SecretKey']
136  : '';
137  // HTTP helper client
138  $this->http = new MultiHttpClient( [] );
139  // Cache container information to mask latency
140  if ( isset( $config['wanCache'] ) && $config['wanCache'] instanceof WANObjectCache ) {
141  $this->memCache = $config['wanCache'];
142  }
143  // Process cache for container info
144  $this->containerStatCache = new ProcessCacheLRU( 300 );
145  // Cache auth token information to avoid RTTs
146  if ( !empty( $config['cacheAuthInfo'] ) && isset( $config['srvCache'] ) ) {
147  $this->srvCache = $config['srvCache'];
148  } else {
149  $this->srvCache = new EmptyBagOStuff();
150  }
151  $this->readUsers = isset( $config['readUsers'] )
152  ? $config['readUsers']
153  : [];
154  $this->writeUsers = isset( $config['writeUsers'] )
155  ? $config['writeUsers']
156  : [];
157  $this->secureReadUsers = isset( $config['secureReadUsers'] )
158  ? $config['secureReadUsers']
159  : [];
160  $this->secureWriteUsers = isset( $config['secureWriteUsers'] )
161  ? $config['secureWriteUsers']
162  : [];
163  }
164 
165  public function getFeatures() {
168  }
169 
170  protected function resolveContainerPath( $container, $relStoragePath ) {
171  if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) {
172  return null; // not UTF-8, makes it hard to use CF and the swift HTTP API
173  } elseif ( strlen( rawurlencode( $relStoragePath ) ) > 1024 ) {
174  return null; // too long for Swift
175  }
176 
177  return $relStoragePath;
178  }
179 
180  public function isPathUsableInternal( $storagePath ) {
181  list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
182  if ( $rel === null ) {
183  return false; // invalid
184  }
185 
186  return is_array( $this->getContainerStat( $container ) );
187  }
188 
196  protected function sanitizeHdrsStrict( array $params ) {
197  if ( !isset( $params['headers'] ) ) {
198  return [];
199  }
200 
201  $headers = $this->getCustomHeaders( $params['headers'] );
202  unset( $headers[ 'content-type' ] );
203 
204  return $headers;
205  }
206 
219  protected function sanitizeHdrs( array $params ) {
220  return isset( $params['headers'] )
221  ? $this->getCustomHeaders( $params['headers'] )
222  : [];
223  }
224 
229  protected function getCustomHeaders( array $rawHeaders ) {
230  $headers = [];
231 
232  // Normalize casing, and strip out illegal headers
233  foreach ( $rawHeaders as $name => $value ) {
234  $name = strtolower( $name );
235  if ( preg_match( '/^content-length$/', $name ) ) {
236  continue; // blacklisted
237  } elseif ( preg_match( '/^(x-)?content-/', $name ) ) {
238  $headers[$name] = $value; // allowed
239  } elseif ( preg_match( '/^content-(disposition)/', $name ) ) {
240  $headers[$name] = $value; // allowed
241  }
242  }
243  // By default, Swift has annoyingly low maximum header value limits
244  if ( isset( $headers['content-disposition'] ) ) {
245  $disposition = '';
246  // @note: assume FileBackend::makeContentDisposition() already used
247  foreach ( explode( ';', $headers['content-disposition'] ) as $part ) {
248  $part = trim( $part );
249  $new = ( $disposition === '' ) ? $part : "{$disposition};{$part}";
250  if ( strlen( $new ) <= 255 ) {
251  $disposition = $new;
252  } else {
253  break; // too long; sigh
254  }
255  }
256  $headers['content-disposition'] = $disposition;
257  }
258 
259  return $headers;
260  }
261 
266  protected function getMetadataHeaders( array $rawHeaders ) {
267  $headers = [];
268  foreach ( $rawHeaders as $name => $value ) {
269  $name = strtolower( $name );
270  if ( strpos( $name, 'x-object-meta-' ) === 0 ) {
271  $headers[$name] = $value;
272  }
273  }
274 
275  return $headers;
276  }
277 
282  protected function getMetadata( array $rawHeaders ) {
283  $metadata = [];
284  foreach ( $this->getMetadataHeaders( $rawHeaders ) as $name => $value ) {
285  $metadata[substr( $name, strlen( 'x-object-meta-' ) )] = $value;
286  }
287 
288  return $metadata;
289  }
290 
291  protected function doCreateInternal( array $params ) {
292  $status = $this->newStatus();
293 
294  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
295  if ( $dstRel === null ) {
296  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
297 
298  return $status;
299  }
300 
301  $sha1Hash = Wikimedia\base_convert( sha1( $params['content'] ), 16, 36, 31 );
302  $contentType = isset( $params['headers']['content-type'] )
303  ? $params['headers']['content-type']
304  : $this->getContentType( $params['dst'], $params['content'], null );
305 
306  $reqs = [ [
307  'method' => 'PUT',
308  'url' => [ $dstCont, $dstRel ],
309  'headers' => [
310  'content-length' => strlen( $params['content'] ),
311  'etag' => md5( $params['content'] ),
312  'content-type' => $contentType,
313  'x-object-meta-sha1base36' => $sha1Hash
314  ] + $this->sanitizeHdrsStrict( $params ),
315  'body' => $params['content']
316  ] ];
317 
318  $method = __METHOD__;
319  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
320  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
321  if ( $rcode === 201 ) {
322  // good
323  } elseif ( $rcode === 412 ) {
324  $status->fatal( 'backend-fail-contenttype', $params['dst'] );
325  } else {
326  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
327  }
328  };
329 
330  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
331  if ( !empty( $params['async'] ) ) { // deferred
332  $status->value = $opHandle;
333  } else { // actually write the object in Swift
334  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
335  }
336 
337  return $status;
338  }
339 
340  protected function doStoreInternal( array $params ) {
341  $status = $this->newStatus();
342 
343  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
344  if ( $dstRel === null ) {
345  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
346 
347  return $status;
348  }
349 
350  Wikimedia\suppressWarnings();
351  $sha1Hash = sha1_file( $params['src'] );
352  Wikimedia\restoreWarnings();
353  if ( $sha1Hash === false ) { // source doesn't exist?
354  $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
355 
356  return $status;
357  }
358  $sha1Hash = Wikimedia\base_convert( $sha1Hash, 16, 36, 31 );
359  $contentType = isset( $params['headers']['content-type'] )
360  ? $params['headers']['content-type']
361  : $this->getContentType( $params['dst'], null, $params['src'] );
362 
363  $handle = fopen( $params['src'], 'rb' );
364  if ( $handle === false ) { // source doesn't exist?
365  $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
366 
367  return $status;
368  }
369 
370  $reqs = [ [
371  'method' => 'PUT',
372  'url' => [ $dstCont, $dstRel ],
373  'headers' => [
374  'content-length' => filesize( $params['src'] ),
375  'etag' => md5_file( $params['src'] ),
376  'content-type' => $contentType,
377  'x-object-meta-sha1base36' => $sha1Hash
378  ] + $this->sanitizeHdrsStrict( $params ),
379  'body' => $handle // resource
380  ] ];
381 
382  $method = __METHOD__;
383  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
384  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
385  if ( $rcode === 201 ) {
386  // good
387  } elseif ( $rcode === 412 ) {
388  $status->fatal( 'backend-fail-contenttype', $params['dst'] );
389  } else {
390  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
391  }
392  };
393 
394  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
395  $opHandle->resourcesToClose[] = $handle;
396 
397  if ( !empty( $params['async'] ) ) { // deferred
398  $status->value = $opHandle;
399  } else { // actually write the object in Swift
400  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
401  }
402 
403  return $status;
404  }
405 
406  protected function doCopyInternal( array $params ) {
407  $status = $this->newStatus();
408 
409  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
410  if ( $srcRel === null ) {
411  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
412 
413  return $status;
414  }
415 
416  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
417  if ( $dstRel === null ) {
418  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
419 
420  return $status;
421  }
422 
423  $reqs = [ [
424  'method' => 'PUT',
425  'url' => [ $dstCont, $dstRel ],
426  'headers' => [
427  'x-copy-from' => '/' . rawurlencode( $srcCont ) .
428  '/' . str_replace( "%2F", "/", rawurlencode( $srcRel ) )
429  ] + $this->sanitizeHdrsStrict( $params ), // extra headers merged into object
430  ] ];
431 
432  $method = __METHOD__;
433  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
434  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
435  if ( $rcode === 201 ) {
436  // good
437  } elseif ( $rcode === 404 ) {
438  $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
439  } else {
440  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
441  }
442  };
443 
444  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
445  if ( !empty( $params['async'] ) ) { // deferred
446  $status->value = $opHandle;
447  } else { // actually write the object in Swift
448  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
449  }
450 
451  return $status;
452  }
453 
454  protected function doMoveInternal( array $params ) {
455  $status = $this->newStatus();
456 
457  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
458  if ( $srcRel === null ) {
459  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
460 
461  return $status;
462  }
463 
464  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
465  if ( $dstRel === null ) {
466  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
467 
468  return $status;
469  }
470 
471  $reqs = [
472  [
473  'method' => 'PUT',
474  'url' => [ $dstCont, $dstRel ],
475  'headers' => [
476  'x-copy-from' => '/' . rawurlencode( $srcCont ) .
477  '/' . str_replace( "%2F", "/", rawurlencode( $srcRel ) )
478  ] + $this->sanitizeHdrsStrict( $params ) // extra headers merged into object
479  ]
480  ];
481  if ( "{$srcCont}/{$srcRel}" !== "{$dstCont}/{$dstRel}" ) {
482  $reqs[] = [
483  'method' => 'DELETE',
484  'url' => [ $srcCont, $srcRel ],
485  'headers' => []
486  ];
487  }
488 
489  $method = __METHOD__;
490  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
491  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
492  if ( $request['method'] === 'PUT' && $rcode === 201 ) {
493  // good
494  } elseif ( $request['method'] === 'DELETE' && $rcode === 204 ) {
495  // good
496  } elseif ( $rcode === 404 ) {
497  $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
498  } else {
499  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
500  }
501  };
502 
503  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
504  if ( !empty( $params['async'] ) ) { // deferred
505  $status->value = $opHandle;
506  } else { // actually move the object in Swift
507  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
508  }
509 
510  return $status;
511  }
512 
513  protected function doDeleteInternal( array $params ) {
514  $status = $this->newStatus();
515 
516  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
517  if ( $srcRel === null ) {
518  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
519 
520  return $status;
521  }
522 
523  $reqs = [ [
524  'method' => 'DELETE',
525  'url' => [ $srcCont, $srcRel ],
526  'headers' => []
527  ] ];
528 
529  $method = __METHOD__;
530  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
531  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
532  if ( $rcode === 204 ) {
533  // good
534  } elseif ( $rcode === 404 ) {
535  if ( empty( $params['ignoreMissingSource'] ) ) {
536  $status->fatal( 'backend-fail-delete', $params['src'] );
537  }
538  } else {
539  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
540  }
541  };
542 
543  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
544  if ( !empty( $params['async'] ) ) { // deferred
545  $status->value = $opHandle;
546  } else { // actually delete the object in Swift
547  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
548  }
549 
550  return $status;
551  }
552 
553  protected function doDescribeInternal( array $params ) {
554  $status = $this->newStatus();
555 
556  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
557  if ( $srcRel === null ) {
558  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
559 
560  return $status;
561  }
562 
563  // Fetch the old object headers/metadata...this should be in stat cache by now
564  $stat = $this->getFileStat( [ 'src' => $params['src'], 'latest' => 1 ] );
565  if ( $stat && !isset( $stat['xattr'] ) ) { // older cache entry
566  $stat = $this->doGetFileStat( [ 'src' => $params['src'], 'latest' => 1 ] );
567  }
568  if ( !$stat ) {
569  $status->fatal( 'backend-fail-describe', $params['src'] );
570 
571  return $status;
572  }
573 
574  // POST clears prior headers, so we need to merge the changes in to the old ones
575  $metaHdrs = [];
576  foreach ( $stat['xattr']['metadata'] as $name => $value ) {
577  $metaHdrs["x-object-meta-$name"] = $value;
578  }
579  $customHdrs = $this->sanitizeHdrs( $params ) + $stat['xattr']['headers'];
580 
581  $reqs = [ [
582  'method' => 'POST',
583  'url' => [ $srcCont, $srcRel ],
584  'headers' => $metaHdrs + $customHdrs
585  ] ];
586 
587  $method = __METHOD__;
588  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
589  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
590  if ( $rcode === 202 ) {
591  // good
592  } elseif ( $rcode === 404 ) {
593  $status->fatal( 'backend-fail-describe', $params['src'] );
594  } else {
595  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
596  }
597  };
598 
599  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
600  if ( !empty( $params['async'] ) ) { // deferred
601  $status->value = $opHandle;
602  } else { // actually change the object in Swift
603  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
604  }
605 
606  return $status;
607  }
608 
609  protected function doPrepareInternal( $fullCont, $dir, array $params ) {
610  $status = $this->newStatus();
611 
612  // (a) Check if container already exists
613  $stat = $this->getContainerStat( $fullCont );
614  if ( is_array( $stat ) ) {
615  return $status; // already there
616  } elseif ( $stat === null ) {
617  $status->fatal( 'backend-fail-internal', $this->name );
618  $this->logger->error( __METHOD__ . ': cannot get container stat' );
619 
620  return $status;
621  }
622 
623  // (b) Create container as needed with proper ACLs
624  if ( $stat === false ) {
625  $params['op'] = 'prepare';
626  $status->merge( $this->createContainer( $fullCont, $params ) );
627  }
628 
629  return $status;
630  }
631 
632  protected function doSecureInternal( $fullCont, $dir, array $params ) {
633  $status = $this->newStatus();
634  if ( empty( $params['noAccess'] ) ) {
635  return $status; // nothing to do
636  }
637 
638  $stat = $this->getContainerStat( $fullCont );
639  if ( is_array( $stat ) ) {
640  $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
641  $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
642  // Make container private to end-users...
643  $status->merge( $this->setContainerAccess(
644  $fullCont,
645  $readUsers,
647  ) );
648  } elseif ( $stat === false ) {
649  $status->fatal( 'backend-fail-usable', $params['dir'] );
650  } else {
651  $status->fatal( 'backend-fail-internal', $this->name );
652  $this->logger->error( __METHOD__ . ': cannot get container stat' );
653  }
654 
655  return $status;
656  }
657 
658  protected function doPublishInternal( $fullCont, $dir, array $params ) {
659  $status = $this->newStatus();
660 
661  $stat = $this->getContainerStat( $fullCont );
662  if ( is_array( $stat ) ) {
663  $readUsers = array_merge( $this->readUsers, [ $this->swiftUser, '.r:*' ] );
664  $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
665 
666  // Make container public to end-users...
667  $status->merge( $this->setContainerAccess(
668  $fullCont,
669  $readUsers,
671  ) );
672  } elseif ( $stat === false ) {
673  $status->fatal( 'backend-fail-usable', $params['dir'] );
674  } else {
675  $status->fatal( 'backend-fail-internal', $this->name );
676  $this->logger->error( __METHOD__ . ': cannot get container stat' );
677  }
678 
679  return $status;
680  }
681 
682  protected function doCleanInternal( $fullCont, $dir, array $params ) {
683  $status = $this->newStatus();
684 
685  // Only containers themselves can be removed, all else is virtual
686  if ( $dir != '' ) {
687  return $status; // nothing to do
688  }
689 
690  // (a) Check the container
691  $stat = $this->getContainerStat( $fullCont, true );
692  if ( $stat === false ) {
693  return $status; // ok, nothing to do
694  } elseif ( !is_array( $stat ) ) {
695  $status->fatal( 'backend-fail-internal', $this->name );
696  $this->logger->error( __METHOD__ . ': cannot get container stat' );
697 
698  return $status;
699  }
700 
701  // (b) Delete the container if empty
702  if ( $stat['count'] == 0 ) {
703  $params['op'] = 'clean';
704  $status->merge( $this->deleteContainer( $fullCont, $params ) );
705  }
706 
707  return $status;
708  }
709 
710  protected function doGetFileStat( array $params ) {
711  $params = [ 'srcs' => [ $params['src'] ], 'concurrency' => 1 ] + $params;
712  unset( $params['src'] );
713  $stats = $this->doGetFileStatMulti( $params );
714 
715  return reset( $stats );
716  }
717 
728  protected function convertSwiftDate( $ts, $format = TS_MW ) {
729  try {
730  $timestamp = new MWTimestamp( $ts );
731 
732  return $timestamp->getTimestamp( $format );
733  } catch ( Exception $e ) {
734  throw new FileBackendError( $e->getMessage() );
735  }
736  }
737 
745  protected function addMissingMetadata( array $objHdrs, $path ) {
746  if ( isset( $objHdrs['x-object-meta-sha1base36'] ) ) {
747  return $objHdrs; // nothing to do
748  }
749 
751  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
752  $this->logger->error( __METHOD__ . ": {path} was not stored with SHA-1 metadata.",
753  [ 'path' => $path ] );
754 
755  $objHdrs['x-object-meta-sha1base36'] = false;
756 
757  $auth = $this->getAuthentication();
758  if ( !$auth ) {
759  return $objHdrs; // failed
760  }
761 
762  // Find prior custom HTTP headers
763  $postHeaders = $this->getCustomHeaders( $objHdrs );
764  // Find prior metadata headers
765  $postHeaders += $this->getMetadataHeaders( $objHdrs );
766 
767  $status = $this->newStatus();
769  $scopeLockS = $this->getScopedFileLocks( [ $path ], LockManager::LOCK_UW, $status );
770  if ( $status->isOK() ) {
771  $tmpFile = $this->getLocalCopy( [ 'src' => $path, 'latest' => 1 ] );
772  if ( $tmpFile ) {
773  $hash = $tmpFile->getSha1Base36();
774  if ( $hash !== false ) {
775  $objHdrs['x-object-meta-sha1base36'] = $hash;
776  // Merge new SHA1 header into the old ones
777  $postHeaders['x-object-meta-sha1base36'] = $hash;
778  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
779  list( $rcode ) = $this->http->run( [
780  'method' => 'POST',
781  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
782  'headers' => $this->authTokenHeaders( $auth ) + $postHeaders
783  ] );
784  if ( $rcode >= 200 && $rcode <= 299 ) {
785  $this->deleteFileCache( $path );
786 
787  return $objHdrs; // success
788  }
789  }
790  }
791  }
792 
793  $this->logger->error( __METHOD__ . ': unable to set SHA-1 metadata for {path}',
794  [ 'path' => $path ] );
795 
796  return $objHdrs; // failed
797  }
798 
799  protected function doGetFileContentsMulti( array $params ) {
800  $contents = [];
801 
802  $auth = $this->getAuthentication();
803 
804  $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging
805  // Blindly create tmp files and stream to them, catching any exception if the file does
806  // not exist. Doing stats here is useless and will loop infinitely in addMissingMetadata().
807  $reqs = []; // (path => op)
808 
809  foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch
810  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
811  if ( $srcRel === null || !$auth ) {
812  $contents[$path] = false;
813  continue;
814  }
815  // Create a new temporary memory file...
816  $handle = fopen( 'php://temp', 'wb' );
817  if ( $handle ) {
818  $reqs[$path] = [
819  'method' => 'GET',
820  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
821  'headers' => $this->authTokenHeaders( $auth )
822  + $this->headersFromParams( $params ),
823  'stream' => $handle,
824  ];
825  }
826  $contents[$path] = false;
827  }
828 
829  $opts = [ 'maxConnsPerHost' => $params['concurrency'] ];
830  $reqs = $this->http->runMulti( $reqs, $opts );
831  foreach ( $reqs as $path => $op ) {
832  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response'];
833  if ( $rcode >= 200 && $rcode <= 299 ) {
834  rewind( $op['stream'] ); // start from the beginning
835  $contents[$path] = stream_get_contents( $op['stream'] );
836  } elseif ( $rcode === 404 ) {
837  $contents[$path] = false;
838  } else {
839  $this->onError( null, __METHOD__,
840  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
841  }
842  fclose( $op['stream'] ); // close open handle
843  }
844 
845  return $contents;
846  }
847 
848  protected function doDirectoryExists( $fullCont, $dir, array $params ) {
849  $prefix = ( $dir == '' ) ? null : "{$dir}/";
850  $status = $this->objectListing( $fullCont, 'names', 1, null, $prefix );
851  if ( $status->isOK() ) {
852  return ( count( $status->value ) ) > 0;
853  }
854 
855  return null; // error
856  }
857 
865  public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
866  return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
867  }
868 
876  public function getFileListInternal( $fullCont, $dir, array $params ) {
877  return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
878  }
879 
891  public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
892  $dirs = [];
893  if ( $after === INF ) {
894  return $dirs; // nothing more
895  }
896 
897  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
898 
899  $prefix = ( $dir == '' ) ? null : "{$dir}/";
900  // Non-recursive: only list dirs right under $dir
901  if ( !empty( $params['topOnly'] ) ) {
902  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
903  if ( !$status->isOK() ) {
904  throw new FileBackendError( "Iterator page I/O error." );
905  }
906  $objects = $status->value;
907  foreach ( $objects as $object ) { // files and directories
908  if ( substr( $object, -1 ) === '/' ) {
909  $dirs[] = $object; // directories end in '/'
910  }
911  }
912  } else {
913  // Recursive: list all dirs under $dir and its subdirs
914  $getParentDir = function ( $path ) {
915  return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
916  };
917 
918  // Get directory from last item of prior page
919  $lastDir = $getParentDir( $after ); // must be first page
920  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
921 
922  if ( !$status->isOK() ) {
923  throw new FileBackendError( "Iterator page I/O error." );
924  }
925 
926  $objects = $status->value;
927 
928  foreach ( $objects as $object ) { // files
929  $objectDir = $getParentDir( $object ); // directory of object
930 
931  if ( $objectDir !== false && $objectDir !== $dir ) {
932  // Swift stores paths in UTF-8, using binary sorting.
933  // See function "create_container_table" in common/db.py.
934  // If a directory is not "greater" than the last one,
935  // then it was already listed by the calling iterator.
936  if ( strcmp( $objectDir, $lastDir ) > 0 ) {
937  $pDir = $objectDir;
938  do { // add dir and all its parent dirs
939  $dirs[] = "{$pDir}/";
940  $pDir = $getParentDir( $pDir );
941  } while ( $pDir !== false // sanity
942  && strcmp( $pDir, $lastDir ) > 0 // not done already
943  && strlen( $pDir ) > strlen( $dir ) // within $dir
944  );
945  }
946  $lastDir = $objectDir;
947  }
948  }
949  }
950  // Page on the unfiltered directory listing (what is returned may be filtered)
951  if ( count( $objects ) < $limit ) {
952  $after = INF; // avoid a second RTT
953  } else {
954  $after = end( $objects ); // update last item
955  }
956 
957  return $dirs;
958  }
959 
971  public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
972  $files = []; // list of (path, stat array or null) entries
973  if ( $after === INF ) {
974  return $files; // nothing more
975  }
976 
977  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
978 
979  $prefix = ( $dir == '' ) ? null : "{$dir}/";
980  // $objects will contain a list of unfiltered names or CF_Object items
981  // Non-recursive: only list files right under $dir
982  if ( !empty( $params['topOnly'] ) ) {
983  if ( !empty( $params['adviseStat'] ) ) {
984  $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix, '/' );
985  } else {
986  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
987  }
988  } else {
989  // Recursive: list all files under $dir and its subdirs
990  if ( !empty( $params['adviseStat'] ) ) {
991  $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix );
992  } else {
993  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
994  }
995  }
996 
997  // Reformat this list into a list of (name, stat array or null) entries
998  if ( !$status->isOK() ) {
999  throw new FileBackendError( "Iterator page I/O error." );
1000  }
1001 
1002  $objects = $status->value;
1003  $files = $this->buildFileObjectListing( $params, $dir, $objects );
1004 
1005  // Page on the unfiltered object listing (what is returned may be filtered)
1006  if ( count( $objects ) < $limit ) {
1007  $after = INF; // avoid a second RTT
1008  } else {
1009  $after = end( $objects ); // update last item
1010  $after = is_object( $after ) ? $after->name : $after;
1011  }
1012 
1013  return $files;
1014  }
1015 
1025  private function buildFileObjectListing( array $params, $dir, array $objects ) {
1026  $names = [];
1027  foreach ( $objects as $object ) {
1028  if ( is_object( $object ) ) {
1029  if ( isset( $object->subdir ) || !isset( $object->name ) ) {
1030  continue; // virtual directory entry; ignore
1031  }
1032  $stat = [
1033  // Convert various random Swift dates to TS_MW
1034  'mtime' => $this->convertSwiftDate( $object->last_modified, TS_MW ),
1035  'size' => (int)$object->bytes,
1036  'sha1' => null,
1037  // Note: manifiest ETags are not an MD5 of the file
1038  'md5' => ctype_xdigit( $object->hash ) ? $object->hash : null,
1039  'latest' => false // eventually consistent
1040  ];
1041  $names[] = [ $object->name, $stat ];
1042  } elseif ( substr( $object, -1 ) !== '/' ) {
1043  // Omit directories, which end in '/' in listings
1044  $names[] = [ $object, null ];
1045  }
1046  }
1047 
1048  return $names;
1049  }
1050 
1057  public function loadListingStatInternal( $path, array $val ) {
1058  $this->cheapCache->set( $path, 'stat', $val );
1059  }
1060 
1061  protected function doGetFileXAttributes( array $params ) {
1062  $stat = $this->getFileStat( $params );
1063  if ( $stat ) {
1064  if ( !isset( $stat['xattr'] ) ) {
1065  // Stat entries filled by file listings don't include metadata/headers
1066  $this->clearCache( [ $params['src'] ] );
1067  $stat = $this->getFileStat( $params );
1068  }
1069 
1070  return $stat['xattr'];
1071  } else {
1072  return false;
1073  }
1074  }
1075 
1076  protected function doGetFileSha1base36( array $params ) {
1077  $stat = $this->getFileStat( $params );
1078  if ( $stat ) {
1079  if ( !isset( $stat['sha1'] ) ) {
1080  // Stat entries filled by file listings don't include SHA1
1081  $this->clearCache( [ $params['src'] ] );
1082  $stat = $this->getFileStat( $params );
1083  }
1084 
1085  return $stat['sha1'];
1086  } else {
1087  return false;
1088  }
1089  }
1090 
1091  protected function doStreamFile( array $params ) {
1092  $status = $this->newStatus();
1093 
1094  $flags = !empty( $params['headless'] ) ? StreamFile::STREAM_HEADLESS : 0;
1095 
1096  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1097  if ( $srcRel === null ) {
1098  StreamFile::send404Message( $params['src'], $flags );
1099  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
1100 
1101  return $status;
1102  }
1103 
1104  $auth = $this->getAuthentication();
1105  if ( !$auth || !is_array( $this->getContainerStat( $srcCont ) ) ) {
1106  StreamFile::send404Message( $params['src'], $flags );
1107  $status->fatal( 'backend-fail-stream', $params['src'] );
1108 
1109  return $status;
1110  }
1111 
1112  // If "headers" is set, we only want to send them if the file is there.
1113  // Do not bother checking if the file exists if headers are not set though.
1114  if ( $params['headers'] && !$this->fileExists( $params ) ) {
1115  StreamFile::send404Message( $params['src'], $flags );
1116  $status->fatal( 'backend-fail-stream', $params['src'] );
1117 
1118  return $status;
1119  }
1120 
1121  // Send the requested additional headers
1122  foreach ( $params['headers'] as $header ) {
1123  header( $header ); // aways send
1124  }
1125 
1126  if ( empty( $params['allowOB'] ) ) {
1127  // Cancel output buffering and gzipping if set
1128  call_user_func( $this->obResetFunc );
1129  }
1130 
1131  $handle = fopen( 'php://output', 'wb' );
1132  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1133  'method' => 'GET',
1134  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1135  'headers' => $this->authTokenHeaders( $auth )
1136  + $this->headersFromParams( $params ) + $params['options'],
1137  'stream' => $handle,
1138  'flags' => [ 'relayResponseHeaders' => empty( $params['headless'] ) ]
1139  ] );
1140 
1141  if ( $rcode >= 200 && $rcode <= 299 ) {
1142  // good
1143  } elseif ( $rcode === 404 ) {
1144  $status->fatal( 'backend-fail-stream', $params['src'] );
1145  // Per T43113, nasty things can happen if bad cache entries get
1146  // stuck in cache. It's also possible that this error can come up
1147  // with simple race conditions. Clear out the stat cache to be safe.
1148  $this->clearCache( [ $params['src'] ] );
1149  $this->deleteFileCache( $params['src'] );
1150  } else {
1151  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1152  }
1153 
1154  return $status;
1155  }
1156 
1157  protected function doGetLocalCopyMulti( array $params ) {
1159  $tmpFiles = [];
1160 
1161  $auth = $this->getAuthentication();
1162 
1163  $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging
1164  // Blindly create tmp files and stream to them, catching any exception if the file does
1165  // not exist. Doing a stat here is useless causes infinite loops in addMissingMetadata().
1166  $reqs = []; // (path => op)
1167 
1168  foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch
1169  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
1170  if ( $srcRel === null || !$auth ) {
1171  $tmpFiles[$path] = null;
1172  continue;
1173  }
1174  // Get source file extension
1176  // Create a new temporary file...
1177  $tmpFile = TempFSFile::factory( 'localcopy_', $ext, $this->tmpDirectory );
1178  if ( $tmpFile ) {
1179  $handle = fopen( $tmpFile->getPath(), 'wb' );
1180  if ( $handle ) {
1181  $reqs[$path] = [
1182  'method' => 'GET',
1183  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1184  'headers' => $this->authTokenHeaders( $auth )
1185  + $this->headersFromParams( $params ),
1186  'stream' => $handle,
1187  ];
1188  } else {
1189  $tmpFile = null;
1190  }
1191  }
1192  $tmpFiles[$path] = $tmpFile;
1193  }
1194 
1195  $isLatest = ( $this->isRGW || !empty( $params['latest'] ) );
1196  $opts = [ 'maxConnsPerHost' => $params['concurrency'] ];
1197  $reqs = $this->http->runMulti( $reqs, $opts );
1198  foreach ( $reqs as $path => $op ) {
1199  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response'];
1200  fclose( $op['stream'] ); // close open handle
1201  if ( $rcode >= 200 && $rcode <= 299 ) {
1202  $size = $tmpFiles[$path] ? $tmpFiles[$path]->getSize() : 0;
1203  // Double check that the disk is not full/broken
1204  if ( $size != $rhdrs['content-length'] ) {
1205  $tmpFiles[$path] = null;
1206  $rerr = "Got {$size}/{$rhdrs['content-length']} bytes";
1207  $this->onError( null, __METHOD__,
1208  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
1209  }
1210  // Set the file stat process cache in passing
1211  $stat = $this->getStatFromHeaders( $rhdrs );
1212  $stat['latest'] = $isLatest;
1213  $this->cheapCache->set( $path, 'stat', $stat );
1214  } elseif ( $rcode === 404 ) {
1215  $tmpFiles[$path] = false;
1216  } else {
1217  $tmpFiles[$path] = null;
1218  $this->onError( null, __METHOD__,
1219  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
1220  }
1221  }
1222 
1223  return $tmpFiles;
1224  }
1225 
1226  public function getFileHttpUrl( array $params ) {
1227  if ( $this->swiftTempUrlKey != '' ||
1228  ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' )
1229  ) {
1230  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1231  if ( $srcRel === null ) {
1232  return null; // invalid path
1233  }
1234 
1235  $auth = $this->getAuthentication();
1236  if ( !$auth ) {
1237  return null;
1238  }
1239 
1240  $ttl = isset( $params['ttl'] ) ? $params['ttl'] : 86400;
1241  $expires = time() + $ttl;
1242 
1243  if ( $this->swiftTempUrlKey != '' ) {
1244  $url = $this->storageUrl( $auth, $srcCont, $srcRel );
1245  // Swift wants the signature based on the unencoded object name
1246  $contPath = parse_url( $this->storageUrl( $auth, $srcCont ), PHP_URL_PATH );
1247  $signature = hash_hmac( 'sha1',
1248  "GET\n{$expires}\n{$contPath}/{$srcRel}",
1249  $this->swiftTempUrlKey
1250  );
1251 
1252  return "{$url}?temp_url_sig={$signature}&temp_url_expires={$expires}";
1253  } else { // give S3 API URL for rgw
1254  // Path for signature starts with the bucket
1255  $spath = '/' . rawurlencode( $srcCont ) . '/' .
1256  str_replace( '%2F', '/', rawurlencode( $srcRel ) );
1257  // Calculate the hash
1258  $signature = base64_encode( hash_hmac(
1259  'sha1',
1260  "GET\n\n\n{$expires}\n{$spath}",
1261  $this->rgwS3SecretKey,
1262  true // raw
1263  ) );
1264  // See https://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html.
1265  // Note: adding a newline for empty CanonicalizedAmzHeaders does not work.
1266  // Note: S3 API is the rgw default; remove the /swift/ URL bit.
1267  return str_replace( '/swift/v1', '', $this->storageUrl( $auth ) . $spath ) .
1268  '?' .
1269  http_build_query( [
1270  'Signature' => $signature,
1271  'Expires' => $expires,
1272  'AWSAccessKeyId' => $this->rgwS3AccessKey
1273  ] );
1274  }
1275  }
1276 
1277  return null;
1278  }
1279 
1280  protected function directoriesAreVirtual() {
1281  return true;
1282  }
1283 
1292  protected function headersFromParams( array $params ) {
1293  $hdrs = [];
1294  if ( !empty( $params['latest'] ) ) {
1295  $hdrs['x-newest'] = 'true';
1296  }
1297 
1298  return $hdrs;
1299  }
1300 
1306  protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1308  $statuses = [];
1309 
1310  $auth = $this->getAuthentication();
1311  if ( !$auth ) {
1312  foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1313  $statuses[$index] = $this->newStatus( 'backend-fail-connect', $this->name );
1314  }
1315 
1316  return $statuses;
1317  }
1318 
1319  // Split the HTTP requests into stages that can be done concurrently
1320  $httpReqsByStage = []; // map of (stage => index => HTTP request)
1321  foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1323  $reqs = $fileOpHandle->httpOp;
1324  // Convert the 'url' parameter to an actual URL using $auth
1325  foreach ( $reqs as $stage => &$req ) {
1326  list( $container, $relPath ) = $req['url'];
1327  $req['url'] = $this->storageUrl( $auth, $container, $relPath );
1328  $req['headers'] = isset( $req['headers'] ) ? $req['headers'] : [];
1329  $req['headers'] = $this->authTokenHeaders( $auth ) + $req['headers'];
1330  $httpReqsByStage[$stage][$index] = $req;
1331  }
1332  $statuses[$index] = $this->newStatus();
1333  }
1334 
1335  // Run all requests for the first stage, then the next, and so on
1336  $reqCount = count( $httpReqsByStage );
1337  for ( $stage = 0; $stage < $reqCount; ++$stage ) {
1338  $httpReqs = $this->http->runMulti( $httpReqsByStage[$stage] );
1339  foreach ( $httpReqs as $index => $httpReq ) {
1340  // Run the callback for each request of this operation
1341  $callback = $fileOpHandles[$index]->callback;
1342  call_user_func_array( $callback, [ $httpReq, $statuses[$index] ] );
1343  // On failure, abort all remaining requests for this operation
1344  // (e.g. abort the DELETE request if the COPY request fails for a move)
1345  if ( !$statuses[$index]->isOK() ) {
1346  $stages = count( $fileOpHandles[$index]->httpOp );
1347  for ( $s = ( $stage + 1 ); $s < $stages; ++$s ) {
1348  unset( $httpReqsByStage[$s][$index] );
1349  }
1350  }
1351  }
1352  }
1353 
1354  return $statuses;
1355  }
1356 
1379  protected function setContainerAccess( $container, array $readUsers, array $writeUsers ) {
1380  $status = $this->newStatus();
1381  $auth = $this->getAuthentication();
1382 
1383  if ( !$auth ) {
1384  $status->fatal( 'backend-fail-connect', $this->name );
1385 
1386  return $status;
1387  }
1388 
1389  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1390  'method' => 'POST',
1391  'url' => $this->storageUrl( $auth, $container ),
1392  'headers' => $this->authTokenHeaders( $auth ) + [
1393  'x-container-read' => implode( ',', $readUsers ),
1394  'x-container-write' => implode( ',', $writeUsers )
1395  ]
1396  ] );
1397 
1398  if ( $rcode != 204 && $rcode !== 202 ) {
1399  $status->fatal( 'backend-fail-internal', $this->name );
1400  $this->logger->error( __METHOD__ . ': unexpected rcode value ({rcode})',
1401  [ 'rcode' => $rcode ] );
1402  }
1403 
1404  return $status;
1405  }
1406 
1415  protected function getContainerStat( $container, $bypassCache = false ) {
1416  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1417 
1418  if ( $bypassCache ) { // purge cache
1419  $this->containerStatCache->clear( $container );
1420  } elseif ( !$this->containerStatCache->has( $container, 'stat' ) ) {
1421  $this->primeContainerCache( [ $container ] ); // check persistent cache
1422  }
1423  if ( !$this->containerStatCache->has( $container, 'stat' ) ) {
1424  $auth = $this->getAuthentication();
1425  if ( !$auth ) {
1426  return null;
1427  }
1428 
1429  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1430  'method' => 'HEAD',
1431  'url' => $this->storageUrl( $auth, $container ),
1432  'headers' => $this->authTokenHeaders( $auth )
1433  ] );
1434 
1435  if ( $rcode === 204 ) {
1436  $stat = [
1437  'count' => $rhdrs['x-container-object-count'],
1438  'bytes' => $rhdrs['x-container-bytes-used']
1439  ];
1440  if ( $bypassCache ) {
1441  return $stat;
1442  } else {
1443  $this->containerStatCache->set( $container, 'stat', $stat ); // cache it
1444  $this->setContainerCache( $container, $stat ); // update persistent cache
1445  }
1446  } elseif ( $rcode === 404 ) {
1447  return false;
1448  } else {
1449  $this->onError( null, __METHOD__,
1450  [ 'cont' => $container ], $rerr, $rcode, $rdesc );
1451 
1452  return null;
1453  }
1454  }
1455 
1456  return $this->containerStatCache->get( $container, 'stat' );
1457  }
1458 
1466  protected function createContainer( $container, array $params ) {
1467  $status = $this->newStatus();
1468 
1469  $auth = $this->getAuthentication();
1470  if ( !$auth ) {
1471  $status->fatal( 'backend-fail-connect', $this->name );
1472 
1473  return $status;
1474  }
1475 
1476  // @see SwiftFileBackend::setContainerAccess()
1477  if ( empty( $params['noAccess'] ) ) {
1478  // public
1479  $readUsers = array_merge( $this->readUsers, [ '.r:*', $this->swiftUser ] );
1480  $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
1481  } else {
1482  // private
1483  $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
1484  $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
1485  }
1486 
1487  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1488  'method' => 'PUT',
1489  'url' => $this->storageUrl( $auth, $container ),
1490  'headers' => $this->authTokenHeaders( $auth ) + [
1491  'x-container-read' => implode( ',', $readUsers ),
1492  'x-container-write' => implode( ',', $writeUsers )
1493  ]
1494  ] );
1495 
1496  if ( $rcode === 201 ) { // new
1497  // good
1498  } elseif ( $rcode === 202 ) { // already there
1499  // this shouldn't really happen, but is OK
1500  } else {
1501  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1502  }
1503 
1504  return $status;
1505  }
1506 
1514  protected function deleteContainer( $container, array $params ) {
1515  $status = $this->newStatus();
1516 
1517  $auth = $this->getAuthentication();
1518  if ( !$auth ) {
1519  $status->fatal( 'backend-fail-connect', $this->name );
1520 
1521  return $status;
1522  }
1523 
1524  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1525  'method' => 'DELETE',
1526  'url' => $this->storageUrl( $auth, $container ),
1527  'headers' => $this->authTokenHeaders( $auth )
1528  ] );
1529 
1530  if ( $rcode >= 200 && $rcode <= 299 ) { // deleted
1531  $this->containerStatCache->clear( $container ); // purge
1532  } elseif ( $rcode === 404 ) { // not there
1533  // this shouldn't really happen, but is OK
1534  } elseif ( $rcode === 409 ) { // not empty
1535  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); // race?
1536  } else {
1537  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1538  }
1539 
1540  return $status;
1541  }
1542 
1555  private function objectListing(
1556  $fullCont, $type, $limit, $after = null, $prefix = null, $delim = null
1557  ) {
1558  $status = $this->newStatus();
1559 
1560  $auth = $this->getAuthentication();
1561  if ( !$auth ) {
1562  $status->fatal( 'backend-fail-connect', $this->name );
1563 
1564  return $status;
1565  }
1566 
1567  $query = [ 'limit' => $limit ];
1568  if ( $type === 'info' ) {
1569  $query['format'] = 'json';
1570  }
1571  if ( $after !== null ) {
1572  $query['marker'] = $after;
1573  }
1574  if ( $prefix !== null ) {
1575  $query['prefix'] = $prefix;
1576  }
1577  if ( $delim !== null ) {
1578  $query['delimiter'] = $delim;
1579  }
1580 
1581  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1582  'method' => 'GET',
1583  'url' => $this->storageUrl( $auth, $fullCont ),
1584  'query' => $query,
1585  'headers' => $this->authTokenHeaders( $auth )
1586  ] );
1587 
1588  $params = [ 'cont' => $fullCont, 'prefix' => $prefix, 'delim' => $delim ];
1589  if ( $rcode === 200 ) { // good
1590  if ( $type === 'info' ) {
1591  $status->value = FormatJson::decode( trim( $rbody ) );
1592  } else {
1593  $status->value = explode( "\n", trim( $rbody ) );
1594  }
1595  } elseif ( $rcode === 204 ) {
1596  $status->value = []; // empty container
1597  } elseif ( $rcode === 404 ) {
1598  $status->value = []; // no container
1599  } else {
1600  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1601  }
1602 
1603  return $status;
1604  }
1605 
1606  protected function doPrimeContainerCache( array $containerInfo ) {
1607  foreach ( $containerInfo as $container => $info ) {
1608  $this->containerStatCache->set( $container, 'stat', $info );
1609  }
1610  }
1611 
1612  protected function doGetFileStatMulti( array $params ) {
1613  $stats = [];
1614 
1615  $auth = $this->getAuthentication();
1616 
1617  $reqs = [];
1618  foreach ( $params['srcs'] as $path ) {
1619  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
1620  if ( $srcRel === null ) {
1621  $stats[$path] = false;
1622  continue; // invalid storage path
1623  } elseif ( !$auth ) {
1624  $stats[$path] = null;
1625  continue;
1626  }
1627 
1628  // (a) Check the container
1629  $cstat = $this->getContainerStat( $srcCont );
1630  if ( $cstat === false ) {
1631  $stats[$path] = false;
1632  continue; // ok, nothing to do
1633  } elseif ( !is_array( $cstat ) ) {
1634  $stats[$path] = null;
1635  continue;
1636  }
1637 
1638  $reqs[$path] = [
1639  'method' => 'HEAD',
1640  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1641  'headers' => $this->authTokenHeaders( $auth ) + $this->headersFromParams( $params )
1642  ];
1643  }
1644 
1645  $opts = [ 'maxConnsPerHost' => $params['concurrency'] ];
1646  $reqs = $this->http->runMulti( $reqs, $opts );
1647 
1648  foreach ( $params['srcs'] as $path ) {
1649  if ( array_key_exists( $path, $stats ) ) {
1650  continue; // some sort of failure above
1651  }
1652  // (b) Check the file
1653  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $reqs[$path]['response'];
1654  if ( $rcode === 200 || $rcode === 204 ) {
1655  // Update the object if it is missing some headers
1656  $rhdrs = $this->addMissingMetadata( $rhdrs, $path );
1657  // Load the stat array from the headers
1658  $stat = $this->getStatFromHeaders( $rhdrs );
1659  if ( $this->isRGW ) {
1660  $stat['latest'] = true; // strong consistency
1661  }
1662  } elseif ( $rcode === 404 ) {
1663  $stat = false;
1664  } else {
1665  $stat = null;
1666  $this->onError( null, __METHOD__, $params, $rerr, $rcode, $rdesc );
1667  }
1668  $stats[$path] = $stat;
1669  }
1670 
1671  return $stats;
1672  }
1673 
1678  protected function getStatFromHeaders( array $rhdrs ) {
1679  // Fetch all of the custom metadata headers
1680  $metadata = $this->getMetadata( $rhdrs );
1681  // Fetch all of the custom raw HTTP headers
1682  $headers = $this->sanitizeHdrs( [ 'headers' => $rhdrs ] );
1683 
1684  return [
1685  // Convert various random Swift dates to TS_MW
1686  'mtime' => $this->convertSwiftDate( $rhdrs['last-modified'], TS_MW ),
1687  // Empty objects actually return no content-length header in Ceph
1688  'size' => isset( $rhdrs['content-length'] ) ? (int)$rhdrs['content-length'] : 0,
1689  'sha1' => isset( $metadata['sha1base36'] ) ? $metadata['sha1base36'] : null,
1690  // Note: manifiest ETags are not an MD5 of the file
1691  'md5' => ctype_xdigit( $rhdrs['etag'] ) ? $rhdrs['etag'] : null,
1692  'xattr' => [ 'metadata' => $metadata, 'headers' => $headers ]
1693  ];
1694  }
1695 
1699  protected function getAuthentication() {
1700  if ( $this->authErrorTimestamp !== null ) {
1701  if ( ( time() - $this->authErrorTimestamp ) < 60 ) {
1702  return null; // failed last attempt; don't bother
1703  } else { // actually retry this time
1704  $this->authErrorTimestamp = null;
1705  }
1706  }
1707  // Session keys expire after a while, so we renew them periodically
1708  $reAuth = ( ( time() - $this->authSessionTimestamp ) > $this->authTTL );
1709  // Authenticate with proxy and get a session key...
1710  if ( !$this->authCreds || $reAuth ) {
1711  $this->authSessionTimestamp = 0;
1712  $cacheKey = $this->getCredsCacheKey( $this->swiftUser );
1713  $creds = $this->srvCache->get( $cacheKey ); // credentials
1714  // Try to use the credential cache
1715  if ( isset( $creds['auth_token'] ) && isset( $creds['storage_url'] ) ) {
1716  $this->authCreds = $creds;
1717  // Skew the timestamp for worst case to avoid using stale credentials
1718  $this->authSessionTimestamp = time() - ceil( $this->authTTL / 2 );
1719  } else { // cache miss
1720  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1721  'method' => 'GET',
1722  'url' => "{$this->swiftAuthUrl}/v1.0",
1723  'headers' => [
1724  'x-auth-user' => $this->swiftUser,
1725  'x-auth-key' => $this->swiftKey
1726  ]
1727  ] );
1728 
1729  if ( $rcode >= 200 && $rcode <= 299 ) { // OK
1730  $this->authCreds = [
1731  'auth_token' => $rhdrs['x-auth-token'],
1732  'storage_url' => ( $this->swiftStorageUrl !== null )
1733  ? $this->swiftStorageUrl
1734  : $rhdrs['x-storage-url']
1735  ];
1736 
1737  $this->srvCache->set( $cacheKey, $this->authCreds, ceil( $this->authTTL / 2 ) );
1738  $this->authSessionTimestamp = time();
1739  } elseif ( $rcode === 401 ) {
1740  $this->onError( null, __METHOD__, [], "Authentication failed.", $rcode );
1741  $this->authErrorTimestamp = time();
1742 
1743  return null;
1744  } else {
1745  $this->onError( null, __METHOD__, [], "HTTP return code: $rcode", $rcode );
1746  $this->authErrorTimestamp = time();
1747 
1748  return null;
1749  }
1750  }
1751  // Ceph RGW does not use <account> in URLs (OpenStack Swift uses "/v1/<account>")
1752  if ( substr( $this->authCreds['storage_url'], -3 ) === '/v1' ) {
1753  $this->isRGW = true; // take advantage of strong consistency in Ceph
1754  }
1755  }
1756 
1757  return $this->authCreds;
1758  }
1759 
1766  protected function storageUrl( array $creds, $container = null, $object = null ) {
1767  $parts = [ $creds['storage_url'] ];
1768  if ( strlen( $container ) ) {
1769  $parts[] = rawurlencode( $container );
1770  }
1771  if ( strlen( $object ) ) {
1772  $parts[] = str_replace( "%2F", "/", rawurlencode( $object ) );
1773  }
1774 
1775  return implode( '/', $parts );
1776  }
1777 
1782  protected function authTokenHeaders( array $creds ) {
1783  return [ 'x-auth-token' => $creds['auth_token'] ];
1784  }
1785 
1792  private function getCredsCacheKey( $username ) {
1793  return 'swiftcredentials:' . md5( $username . ':' . $this->swiftAuthUrl );
1794  }
1795 
1807  public function onError( $status, $func, array $params, $err = '', $code = 0, $desc = '' ) {
1808  if ( $status instanceof StatusValue ) {
1809  $status->fatal( 'backend-fail-internal', $this->name );
1810  }
1811  if ( $code == 401 ) { // possibly a stale token
1812  $this->srvCache->delete( $this->getCredsCacheKey( $this->swiftUser ) );
1813  }
1814  $msg = "HTTP {code} ({desc}) in '{func}' (given '{params}')";
1815  $msgParams = [
1816  'code' => $code,
1817  'desc' => $desc,
1818  'func' => $func,
1819  'req_params' => FormatJson::encode( $params ),
1820  ];
1821  if ( $err ) {
1822  $msg .= ': {err}';
1823  $msgParams['err'] = $err;
1824  }
1825  $this->logger->error( $msg, $msgParams );
1826  }
1827 }
1828 
1834  public $httpOp;
1836  public $callback;
1837 
1844  $this->backend = $backend;
1845  $this->callback = $callback;
1846  $this->httpOp = $httpOp;
1847  }
1848 }
1849 
1857 abstract class SwiftFileBackendList implements Iterator {
1859  protected $bufferIter = [];
1860 
1862  protected $bufferAfter = null;
1863 
1865  protected $pos = 0;
1866 
1868  protected $params = [];
1869 
1871  protected $backend;
1872 
1874  protected $container;
1875 
1877  protected $dir;
1878 
1880  protected $suffixStart;
1881 
1882  const PAGE_SIZE = 9000; // file listing buffer size
1883 
1890  public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
1891  $this->backend = $backend;
1892  $this->container = $fullCont;
1893  $this->dir = $dir;
1894  if ( substr( $this->dir, -1 ) === '/' ) {
1895  $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
1896  }
1897  if ( $this->dir == '' ) { // whole container
1898  $this->suffixStart = 0;
1899  } else { // dir within container
1900  $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
1901  }
1902  $this->params = $params;
1903  }
1904 
1909  public function key() {
1910  return $this->pos;
1911  }
1912 
1916  public function next() {
1917  // Advance to the next file in the page
1918  next( $this->bufferIter );
1919  ++$this->pos;
1920  // Check if there are no files left in this page and
1921  // advance to the next page if this page was not empty.
1922  if ( !$this->valid() && count( $this->bufferIter ) ) {
1923  $this->bufferIter = $this->pageFromList(
1924  $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1925  ); // updates $this->bufferAfter
1926  }
1927  }
1928 
1932  public function rewind() {
1933  $this->pos = 0;
1934  $this->bufferAfter = null;
1935  $this->bufferIter = $this->pageFromList(
1936  $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1937  ); // updates $this->bufferAfter
1938  }
1939 
1944  public function valid() {
1945  if ( $this->bufferIter === null ) {
1946  return false; // some failure?
1947  } else {
1948  return ( current( $this->bufferIter ) !== false ); // no paths can have this value
1949  }
1950  }
1951 
1962  abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1963 }
1964 
1973  public function current() {
1974  return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
1975  }
1976 
1977  protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1978  return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1979  }
1980 }
1981 
1990  public function current() {
1991  list( $path, $stat ) = current( $this->bufferIter );
1992  $relPath = substr( $path, $this->suffixStart );
1993  if ( is_array( $stat ) ) {
1994  $storageDir = rtrim( $this->params['dir'], '/' );
1995  $this->backend->loadListingStatInternal( "$storageDir/$relPath", $stat );
1996  }
1997 
1998  return $relPath;
1999  }
2000 
2001  protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
2002  return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
2003  }
2004 }
SwiftFileBackend\$containerStatCache
ProcessCacheLRU $containerStatCache
Container stat cache.
Definition: SwiftFileBackend.php:66
SwiftFileBackendList\rewind
rewind()
Definition: SwiftFileBackend.php:1932
SwiftFileOpHandle
Definition: SwiftFileBackend.php:1832
MWTimestamp
Library for creating and parsing MW-style timestamps.
Definition: MWTimestamp.php:32
SwiftFileBackend\doPrepareInternal
doPrepareInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:609
SwiftFileBackend\$isRGW
bool $isRGW
Whether the server is an Ceph RGW.
Definition: SwiftFileBackend.php:76
$handler
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that probably a stub it is not rendered in wiki pages or galleries in category pages allow injecting custom HTML after the section Any uses of the hook need to handle escaping see BaseTemplate::getToolbox and BaseTemplate::makeListItem for details on the format of individual items inside of this array or by returning and letting standard HTTP rendering take place modifiable or by returning false and taking over the output modifiable modifiable after all normalizations have been except for the $wgMaxImageArea check set to true or false to override the $wgMaxImageArea check result gives extension the possibility to transform it themselves $handler
Definition: hooks.txt:903
MultiHttpClient
Class to handle concurrent HTTP requests.
Definition: MultiHttpClient.php:48
StatusValue
Generic operation result class Has warning/error list, boolean status and arbitrary value.
Definition: StatusValue.php:42
SwiftFileBackend\isPathUsableInternal
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path.
Definition: SwiftFileBackend.php:180
SwiftFileBackend\$authErrorTimestamp
int $authErrorTimestamp
UNIX timestamp.
Definition: SwiftFileBackend.php:73
use
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
Definition: APACHE-LICENSE-2.0.txt:10
SwiftFileBackend\getAuthentication
getAuthentication()
Definition: SwiftFileBackend.php:1699
SwiftFileBackend\getContainerStat
getContainerStat( $container, $bypassCache=false)
Get a Swift container stat array, possibly from process cache.
Definition: SwiftFileBackend.php:1415
SwiftFileBackendDirList\pageFromList
pageFromList( $container, $dir, &$after, $limit, array $params)
Get the given list portion (page)
Definition: SwiftFileBackend.php:1977
EmptyBagOStuff
A BagOStuff object with no objects in it.
Definition: EmptyBagOStuff.php:29
array
the array() calling protocol came about after MediaWiki 1.4rc1.
SwiftFileBackend\$swiftKey
string $swiftKey
Secret key for user.
Definition: SwiftFileBackend.php:46
SwiftFileBackend\getStatFromHeaders
getStatFromHeaders(array $rhdrs)
Definition: SwiftFileBackend.php:1678
FileBackend\ATTR_HEADERS
const ATTR_HEADERS
Bitfield flags for supported features.
Definition: FileBackend.php:128
SwiftFileBackend\getCustomHeaders
getCustomHeaders(array $rawHeaders)
Definition: SwiftFileBackend.php:229
LockManager\LOCK_UW
const LOCK_UW
Definition: LockManager.php:68
SwiftFileBackend\doGetFileXAttributes
doGetFileXAttributes(array $params)
Definition: SwiftFileBackend.php:1061
FileBackendError
File backend exception for checked exceptions (e.g.
Definition: FileBackendError.php:8
FileBackend\getScopedFileLocks
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
Definition: FileBackend.php:1343
SwiftFileBackend\buildFileObjectListing
buildFileObjectListing(array $params, $dir, array $objects)
Build a list of file objects, filtering out any directories and extracting any stat info if provided ...
Definition: SwiftFileBackend.php:1025
FileBackendStore\fileExists
fileExists(array $params)
Check if a file exists at a storage path in the backend.
Definition: FileBackendStore.php:604
SwiftFileBackend\$swiftUser
string $swiftUser
Swift user (account:user) to authenticate as.
Definition: SwiftFileBackend.php:44
SwiftFileBackend\loadListingStatInternal
loadListingStatInternal( $path, array $val)
Do not call this function outside of SwiftFileBackendFileList.
Definition: SwiftFileBackend.php:1057
FileBackend\extensionFromPath
static extensionFromPath( $path, $case='lowercase')
Get the final extension from a storage or FS path.
Definition: FileBackend.php:1506
FileBackendStore\deleteFileCache
deleteFileCache( $path)
Delete the cached stat info for a file path.
Definition: FileBackendStore.php:1731
SwiftFileBackend\resolveContainerPath
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
Definition: SwiftFileBackend.php:170
$params
$params
Definition: styleTest.css.php:40
FileBackendStore\executeOpHandlesInternal
executeOpHandlesInternal(array $fileOpHandles)
Execute a list of FileBackendStoreOpHandle handles in parallel.
Definition: FileBackendStore.php:1205
BagOStuff
interface is intended to be more or less compatible with the PHP memcached client.
Definition: BagOStuff.php:47
$s
$s
Definition: mergeMessageFileList.php:187
SwiftFileBackend\doGetFileContentsMulti
doGetFileContentsMulti(array $params)
Definition: SwiftFileBackend.php:799
SwiftFileBackend\createContainer
createContainer( $container, array $params)
Create a Swift container.
Definition: SwiftFileBackend.php:1466
http
Apache License January http
Definition: APACHE-LICENSE-2.0.txt:6
SwiftFileBackend\getFileListPageInternal
getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
Definition: SwiftFileBackend.php:971
SwiftFileBackendList\PAGE_SIZE
const PAGE_SIZE
Definition: SwiftFileBackend.php:1882
SwiftFileBackend\doGetFileSha1base36
doGetFileSha1base36(array $params)
Definition: SwiftFileBackend.php:1076
FileBackendStore\clearCache
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
Definition: FileBackendStore.php:1282
SwiftFileBackend\getFeatures
getFeatures()
Get the a bitfield of extra features supported by the backend medium.
Definition: SwiftFileBackend.php:165
php
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:37
SwiftFileBackend\doDeleteInternal
doDeleteInternal(array $params)
Definition: SwiftFileBackend.php:513
SwiftFileBackend\headersFromParams
headersFromParams(array $params)
Get headers to send to Swift when reading a file based on a FileBackend params array,...
Definition: SwiftFileBackend.php:1292
SwiftFileBackend\deleteContainer
deleteContainer( $container, array $params)
Delete a Swift container.
Definition: SwiftFileBackend.php:1514
SwiftFileBackendList\$suffixStart
int $suffixStart
Definition: SwiftFileBackend.php:1880
SwiftFileBackend\getCredsCacheKey
getCredsCacheKey( $username)
Get the cache key for a container.
Definition: SwiftFileBackend.php:1792
FormatJson\decode
static decode( $value, $assoc=false)
Decodes a JSON string.
Definition: FormatJson.php:187
FormatJson\encode
static encode( $value, $pretty=false, $escaping=0)
Returns the JSON representation of a value.
Definition: FormatJson.php:127
SwiftFileBackend\doDirectoryExists
doDirectoryExists( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:848
SwiftFileBackend\sanitizeHdrsStrict
sanitizeHdrsStrict(array $params)
Sanitize and filter the custom headers from a $params array.
Definition: SwiftFileBackend.php:196
SwiftFileBackendList\$params
array $params
Definition: SwiftFileBackend.php:1868
SwiftFileBackend
Class for an OpenStack Swift (or Ceph RGW) based file backend.
Definition: SwiftFileBackend.php:34
SwiftFileBackendList
SwiftFileBackend helper class to page through listings.
Definition: SwiftFileBackend.php:1857
SwiftFileBackend\addMissingMetadata
addMissingMetadata(array $objHdrs, $path)
Fill in any missing object metadata and save it to Swift.
Definition: SwiftFileBackend.php:745
SwiftFileBackendList\valid
valid()
Definition: SwiftFileBackend.php:1944
SwiftFileBackend\getDirectoryListInternal
getDirectoryListInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:865
SwiftFileBackend\setContainerAccess
setContainerAccess( $container, array $readUsers, array $writeUsers)
Set read/write permissions for a Swift container.
Definition: SwiftFileBackend.php:1379
FileBackendStoreOpHandle\$backend
FileBackendStore $backend
Definition: FileBackendStore.php:1860
SwiftFileBackend\getMetadata
getMetadata(array $rawHeaders)
Definition: SwiftFileBackend.php:282
SwiftFileBackend\getFileHttpUrl
getFileHttpUrl(array $params)
Definition: SwiftFileBackend.php:1226
SwiftFileBackend\doMoveInternal
doMoveInternal(array $params)
Definition: SwiftFileBackend.php:454
SwiftFileBackendList\$dir
string $dir
Storage directory.
Definition: SwiftFileBackend.php:1877
FileBackend\ATTR_UNICODE_PATHS
const ATTR_UNICODE_PATHS
Definition: FileBackend.php:130
SwiftFileBackend\$authCreds
array $authCreds
Definition: SwiftFileBackend.php:69
SwiftFileBackendList\pageFromList
pageFromList( $container, $dir, &$after, $limit, array $params)
Get the given list portion (page)
$dirs
$dirs
Definition: mergeMessageFileList.php:194
SwiftFileBackend\doGetLocalCopyMulti
doGetLocalCopyMulti(array $params)
Definition: SwiftFileBackend.php:1157
SwiftFileBackend\onError
onError( $status, $func, array $params, $err='', $code=0, $desc='')
Log an unexpected exception for this backend.
Definition: SwiftFileBackend.php:1807
SwiftFileBackendDirList\current
current()
Definition: SwiftFileBackend.php:1973
StreamFile\send404Message
static send404Message( $fname, $flags=0)
Send out a standard 404 message for a file.
Definition: StreamFile.php:70
SwiftFileBackend\$secureWriteUsers
array $secureWriteUsers
Additional users (account:user) with write permissions on private containers.
Definition: SwiftFileBackend.php:60
SwiftFileBackend\doDescribeInternal
doDescribeInternal(array $params)
Definition: SwiftFileBackend.php:553
list
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition: deferred.txt:11
SwiftFileBackend\sanitizeHdrs
sanitizeHdrs(array $params)
Sanitize and filter the custom headers from a $params array.
Definition: SwiftFileBackend.php:219
SwiftFileBackend\storageUrl
storageUrl(array $creds, $container=null, $object=null)
Definition: SwiftFileBackend.php:1766
FileBackendStore\getContentType
getContentType( $storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
Definition: FileBackendStore.php:1838
TempFSFile\factory
static factory( $prefix, $extension='', $tmpDirectory=null)
Make a new temporary file on the file system.
Definition: TempFSFile.php:55
SwiftFileBackendFileList\current
current()
Definition: SwiftFileBackend.php:1990
SwiftFileBackendDirList
Iterator for listing directories.
Definition: SwiftFileBackend.php:1968
SwiftFileBackendList\$container
string $container
Container name.
Definition: SwiftFileBackend.php:1874
SwiftFileBackend\doSecureInternal
doSecureInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:632
FileBackend\ATTR_METADATA
const ATTR_METADATA
Definition: FileBackend.php:129
$value
$value
Definition: styleTest.css.php:45
FileBackendStore\getFileStat
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
Definition: FileBackendStore.php:625
$header
$header
Definition: updateCredits.php:35
SwiftFileBackend\__construct
__construct(array $config)
Definition: SwiftFileBackend.php:112
SwiftFileBackend\directoriesAreVirtual
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
Definition: SwiftFileBackend.php:1280
FileBackendStore\resolveStoragePathReal
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
Definition: FileBackendStore.php:1469
SwiftFileBackend\authTokenHeaders
authTokenHeaders(array $creds)
Definition: SwiftFileBackend.php:1782
SwiftFileBackendList\$bufferIter
array $bufferIter
List of path or (path,stat array) entries.
Definition: SwiftFileBackend.php:1859
WANObjectCache
Multi-datacenter aware caching interface.
Definition: WANObjectCache.php:87
SwiftFileBackendList\$backend
SwiftFileBackend $backend
Definition: SwiftFileBackend.php:1871
SwiftFileBackend\doCreateInternal
doCreateInternal(array $params)
Definition: SwiftFileBackend.php:291
SwiftFileBackend\$authSessionTimestamp
int $authSessionTimestamp
UNIX timestamp.
Definition: SwiftFileBackend.php:71
FileBackendStore
Base class for all backends using particular storage medium.
Definition: FileBackendStore.php:38
SwiftFileBackend\doExecuteOpHandlesInternal
doExecuteOpHandlesInternal(array $fileOpHandles)
Definition: SwiftFileBackend.php:1306
$status
Status::newGood()` to allow deletion, and then `return false` from the hook function. Ensure you consume the 'ChangeTagAfterDelete' hook to carry out custom deletion actions. $tag:name of the tag $user:user initiating the action & $status:Status object. See above. 'ChangeTagsListActive':Allows you to nominate which of the tags your extension uses are in active use. & $tags:list of all active tags. Append to this array. 'ChangeTagsAfterUpdateTags':Called after tags have been updated with the ChangeTags::updateTags function. Params:$addedTags:tags effectively added in the update $removedTags:tags effectively removed in the update $prevTags:tags that were present prior to the update $rc_id:recentchanges table id $rev_id:revision table id $log_id:logging table id $params:tag params $rc:RecentChange being tagged when the tagging accompanies the action or null $user:User who performed the tagging when the tagging is subsequent to the action or null 'ChangeTagsAllowedAdd':Called when checking if a user can add tags to a change. & $allowedTags:List of all the tags the user is allowed to add. Any tags the user wants to add( $addTags) that are not in this array will cause it to fail. You may add or remove tags to this array as required. $addTags:List of tags user intends to add. $user:User who is adding the tags. 'ChangeUserGroups':Called before user groups are changed. $performer:The User who will perform the change $user:The User whose groups will be changed & $add:The groups that will be added & $remove:The groups that will be removed 'Collation::factory':Called if $wgCategoryCollation is an unknown collation. $collationName:Name of the collation in question & $collationObject:Null. Replace with a subclass of the Collation class that implements the collation given in $collationName. 'ConfirmEmailComplete':Called after a user 's email has been confirmed successfully. $user:user(object) whose email is being confirmed 'ContentAlterParserOutput':Modify parser output for a given content object. Called by Content::getParserOutput after parsing has finished. Can be used for changes that depend on the result of the parsing but have to be done before LinksUpdate is called(such as adding tracking categories based on the rendered HTML). $content:The Content to render $title:Title of the page, as context $parserOutput:ParserOutput to manipulate 'ContentGetParserOutput':Customize parser output for a given content object, called by AbstractContent::getParserOutput. May be used to override the normal model-specific rendering of page content. $content:The Content to render $title:Title of the page, as context $revId:The revision ID, as context $options:ParserOptions for rendering. To avoid confusing the parser cache, the output can only depend on parameters provided to this hook function, not on global state. $generateHtml:boolean, indicating whether full HTML should be generated. If false, generation of HTML may be skipped, but other information should still be present in the ParserOutput object. & $output:ParserOutput, to manipulate or replace 'ContentHandlerDefaultModelFor':Called when the default content model is determined for a given title. May be used to assign a different model for that title. $title:the Title in question & $model:the model name. Use with CONTENT_MODEL_XXX constants. 'ContentHandlerForModelID':Called when a ContentHandler is requested for a given content model name, but no entry for that model exists in $wgContentHandlers. Note:if your extension implements additional models via this hook, please use GetContentModels hook to make them known to core. $modeName:the requested content model name & $handler:set this to a ContentHandler object, if desired. 'ContentModelCanBeUsedOn':Called to determine whether that content model can be used on a given page. This is especially useful to prevent some content models to be used in some special location. $contentModel:ID of the content model in question $title:the Title in question. & $ok:Output parameter, whether it is OK to use $contentModel on $title. Handler functions that modify $ok should generally return false to prevent further hooks from further modifying $ok. 'ContribsPager::getQueryInfo':Before the contributions query is about to run & $pager:Pager object for contributions & $queryInfo:The query for the contribs Pager 'ContribsPager::reallyDoQuery':Called before really executing the query for My Contributions & $data:an array of results of all contribs queries $pager:The ContribsPager object hooked into $offset:Index offset, inclusive $limit:Exact query limit $descending:Query direction, false for ascending, true for descending 'ContributionsLineEnding':Called before a contributions HTML line is finished $page:SpecialPage object for contributions & $ret:the HTML line $row:the DB row for this line & $classes:the classes to add to the surrounding< li > & $attribs:associative array of other HTML attributes for the< li > element. Currently only data attributes reserved to MediaWiki are allowed(see Sanitizer::isReservedDataAttribute). 'ContributionsToolLinks':Change tool links above Special:Contributions $id:User identifier $title:User page title & $tools:Array of tool links $specialPage:SpecialPage instance for context and services. Can be either SpecialContributions or DeletedContributionsPage. Extensions should type hint against a generic SpecialPage though. 'ConvertContent':Called by AbstractContent::convert when a conversion to another content model is requested. Handler functions that modify $result should generally return false to disable further attempts at conversion. $content:The Content object to be converted. $toModel:The ID of the content model to convert to. $lossy:boolean indicating whether lossy conversion is allowed. & $result:Output parameter, in case the handler function wants to provide a converted Content object. Note that $result->getContentModel() must return $toModel. 'CustomEditor':When invoking the page editor Return true to allow the normal editor to be used, or false if implementing a custom editor, e.g. for a special namespace, etc. $article:Article being edited $user:User performing the edit 'DatabaseOraclePostInit':Called after initialising an Oracle database $db:the DatabaseOracle object 'DeletedContribsPager::reallyDoQuery':Called before really executing the query for Special:DeletedContributions Similar to ContribsPager::reallyDoQuery & $data:an array of results of all contribs queries $pager:The DeletedContribsPager object hooked into $offset:Index offset, inclusive $limit:Exact query limit $descending:Query direction, false for ascending, true for descending 'DeletedContributionsLineEnding':Called before a DeletedContributions HTML line is finished. Similar to ContributionsLineEnding $page:SpecialPage object for DeletedContributions & $ret:the HTML line $row:the DB row for this line & $classes:the classes to add to the surrounding< li > & $attribs:associative array of other HTML attributes for the< li > element. Currently only data attributes reserved to MediaWiki are allowed(see Sanitizer::isReservedDataAttribute). 'DeleteUnknownPreferences':Called by the cleanupPreferences.php maintenance script to build a WHERE clause with which to delete preferences that are not known about. This hook is used by extensions that have dynamically-named preferences that should not be deleted in the usual cleanup process. For example, the Gadgets extension creates preferences prefixed with 'gadget-', and so anything with that prefix is excluded from the deletion. &where:An array that will be passed as the $cond parameter to IDatabase::select() to determine what will be deleted from the user_properties table. $db:The IDatabase object, useful for accessing $db->buildLike() etc. 'DifferenceEngineAfterLoadNewText':called in DifferenceEngine::loadNewText() after the new revision 's content has been loaded into the class member variable $differenceEngine->mNewContent but before returning true from this function. $differenceEngine:DifferenceEngine object 'DifferenceEngineLoadTextAfterNewContentIsLoaded':called in DifferenceEngine::loadText() after the new revision 's content has been loaded into the class member variable $differenceEngine->mNewContent but before checking if the variable 's value is null. This hook can be used to inject content into said class member variable. $differenceEngine:DifferenceEngine object 'DifferenceEngineMarkPatrolledLink':Allows extensions to change the "mark as patrolled" link which is shown both on the diff header as well as on the bottom of a page, usually wrapped in a span element which has class="patrollink". $differenceEngine:DifferenceEngine object & $markAsPatrolledLink:The "mark as patrolled" link HTML(string) $rcid:Recent change ID(rc_id) for this change(int) 'DifferenceEngineMarkPatrolledRCID':Allows extensions to possibly change the rcid parameter. For example the rcid might be set to zero due to the user being the same as the performer of the change but an extension might still want to show it under certain conditions. & $rcid:rc_id(int) of the change or 0 $differenceEngine:DifferenceEngine object $change:RecentChange object $user:User object representing the current user 'DifferenceEngineNewHeader':Allows extensions to change the $newHeader variable, which contains information about the new revision, such as the revision 's author, whether the revision was marked as a minor edit or not, etc. $differenceEngine:DifferenceEngine object & $newHeader:The string containing the various #mw-diff-otitle[1-5] divs, which include things like revision author info, revision comment, RevisionDelete link and more $formattedRevisionTools:Array containing revision tools, some of which may have been injected with the DiffRevisionTools hook $nextlink:String containing the link to the next revision(if any) $status
Definition: hooks.txt:1255
SwiftFileBackend\$swiftAuthUrl
string $swiftAuthUrl
Authentication base URL (without version)
Definition: SwiftFileBackend.php:40
FileBackendStoreOpHandle
FileBackendStore helper class for performing asynchronous file operations.
Definition: FileBackendStore.php:1856
SwiftFileBackend\doStreamFile
doStreamFile(array $params)
Definition: SwiftFileBackend.php:1091
SwiftFileBackendList\$pos
int $pos
Definition: SwiftFileBackend.php:1865
SwiftFileBackend\$writeUsers
array $writeUsers
Additional users (account:user) with write permissions on public containers.
Definition: SwiftFileBackend.php:56
$req
this hook is for auditing only $req
Definition: hooks.txt:990
SwiftFileBackendList\next
next()
Definition: SwiftFileBackend.php:1916
SwiftFileBackendList\$bufferAfter
string $bufferAfter
List items after this path.
Definition: SwiftFileBackend.php:1862
SwiftFileBackend\$rgwS3SecretKey
string $rgwS3SecretKey
S3 authentication key (RADOS Gateway)
Definition: SwiftFileBackend.php:52
FileBackendStore\primeContainerCache
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
Definition: FileBackendStore.php:1651
SwiftFileBackend\$authTTL
int $authTTL
TTL in seconds.
Definition: SwiftFileBackend.php:38
$code
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that probably a stub it is not rendered in wiki pages or galleries in category pages allow injecting custom HTML after the section Any uses of the hook need to handle escaping see BaseTemplate::getToolbox and BaseTemplate::makeListItem for details on the format of individual items inside of this array or by returning and letting standard HTTP rendering take place modifiable or by returning false and taking over the output modifiable & $code
Definition: hooks.txt:865
SwiftFileBackend\$readUsers
array $readUsers
Additional users (account:user) with read permissions on public containers.
Definition: SwiftFileBackend.php:54
SwiftFileBackend\doCopyInternal
doCopyInternal(array $params)
Definition: SwiftFileBackend.php:406
FileBackend\$name
string $name
Unique backend name.
Definition: FileBackend.php:94
SwiftFileBackend\$rgwS3AccessKey
string $rgwS3AccessKey
S3 access key (RADOS Gateway)
Definition: SwiftFileBackend.php:50
FileBackend\getLocalCopy
getLocalCopy(array $params)
Get a local copy on disk of the file at a storage path in the backend.
Definition: FileBackend.php:1131
SwiftFileBackend\objectListing
objectListing( $fullCont, $type, $limit, $after=null, $prefix=null, $delim=null)
Get a list of objects under a container.
Definition: SwiftFileBackend.php:1555
SwiftFileOpHandle\$httpOp
array $httpOp
List of Requests for MultiHttpClient.
Definition: SwiftFileBackend.php:1834
SwiftFileBackend\$swiftStorageUrl
string $swiftStorageUrl
Override of storage base URL.
Definition: SwiftFileBackend.php:42
$path
$path
Definition: NoLocalSettings.php:25
SwiftFileBackend\convertSwiftDate
convertSwiftDate( $ts, $format=TS_MW)
Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT"/"2013-05-11T07:37:27.678360Z".
Definition: SwiftFileBackend.php:728
as
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:22
SwiftFileBackend\doPublishInternal
doPublishInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:658
SwiftFileOpHandle\$callback
Closure $callback
Definition: SwiftFileBackend.php:1836
SwiftFileBackendFileList\pageFromList
pageFromList( $container, $dir, &$after, $limit, array $params)
Get the given list portion (page)
Definition: SwiftFileBackend.php:2001
SwiftFileBackend\doGetFileStat
doGetFileStat(array $params)
Definition: SwiftFileBackend.php:710
SwiftFileBackend\$swiftTempUrlKey
string $swiftTempUrlKey
Shared secret value for making temp URLs.
Definition: SwiftFileBackend.php:48
SwiftFileBackend\doStoreInternal
doStoreInternal(array $params)
Definition: SwiftFileBackend.php:340
$username
this hook is for auditing only or null if authentication failed before getting that far $username
Definition: hooks.txt:785
SwiftFileBackend\doGetFileStatMulti
doGetFileStatMulti(array $params)
Get file stat information (concurrently if possible) for several files.
Definition: SwiftFileBackend.php:1612
name
design txt This is a brief overview of the new design More thorough and up to date information is available on the documentation wiki at name
Definition: design.txt:12
$request
do that in ParserLimitReportFormat instead use this to modify the parameters of the image all existing parser cache entries will be invalid To avoid you ll need to handle that somehow(e.g. with the RejectParserCacheValue hook) because MediaWiki won 't do it for you. & $defaults also a ContextSource after deleting those rows but within the same transaction you ll probably need to make sure the header is varied on $request
Definition: hooks.txt:2806
SwiftFileBackend\doCleanInternal
doCleanInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:682
ProcessCacheLRU
Class for process caching individual properties of expiring items.
Definition: ProcessCacheLRU.php:32
SwiftFileBackend\$srvCache
BagOStuff $srvCache
Definition: SwiftFileBackend.php:63
SwiftFileOpHandle\__construct
__construct(SwiftFileBackend $backend, Closure $callback, array $httpOp)
Definition: SwiftFileBackend.php:1843
$query
null for the wiki Added should default to null in handler for backwards compatibility add a value to it if you want to add a cookie that have to vary cache options can modify $query
Definition: hooks.txt:1620
SwiftFileBackendFileList
Iterator for listing regular files.
Definition: SwiftFileBackend.php:1985
SwiftFileBackend\getFileListInternal
getFileListInternal( $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:876
SwiftFileBackend\doPrimeContainerCache
doPrimeContainerCache(array $containerInfo)
Fill the backend-specific process cache given an array of resolved container names and their correspo...
Definition: SwiftFileBackend.php:1606
$ext
$ext
Definition: router.php:55
SwiftFileBackend\$secureReadUsers
array $secureReadUsers
Additional users (account:user) with read permissions on private containers.
Definition: SwiftFileBackend.php:58
false
processing should stop and the error should be shown to the user * false
Definition: hooks.txt:187
FileBackendStore\setContainerCache
setContainerCache( $container, array $val)
Set the cached info for a container.
Definition: FileBackendStore.php:1628
FileBackend\scopedProfileSection
scopedProfileSection( $section)
Definition: FileBackend.php:1620
SwiftFileBackend\getDirListPageInternal
getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
Definition: SwiftFileBackend.php:891
FileBackend\newStatus
newStatus()
Yields the result of the status wrapper callback on either:
Definition: FileBackend.php:1597
StreamFile\STREAM_HEADLESS
const STREAM_HEADLESS
Definition: StreamFile.php:28
$e
div flags Integer display flags(NO_ACTION_LINK, NO_EXTRA_USER_LINKS) 'LogException' returning false will NOT prevent logging $e
Definition: hooks.txt:2171
SwiftFileBackend\getMetadataHeaders
getMetadataHeaders(array $rawHeaders)
Definition: SwiftFileBackend.php:266
SwiftFileBackendList\__construct
__construct(SwiftFileBackend $backend, $fullCont, $dir, array $params)
Definition: SwiftFileBackend.php:1890
SwiftFileBackend\$http
MultiHttpClient $http
Definition: SwiftFileBackend.php:36
$type
$type
Definition: testCompression.php:48
SwiftFileBackendList\key
key()
Definition: SwiftFileBackend.php:1909