MediaWiki  master
SwiftFileBackend.php
Go to the documentation of this file.
1 <?php
36  protected $http;
38  protected $authTTL;
40  protected $swiftAuthUrl;
42  protected $swiftStorageUrl;
44  protected $swiftUser;
46  protected $swiftKey;
48  protected $swiftTempUrlKey;
50  protected $rgwS3AccessKey;
52  protected $rgwS3SecretKey;
54  protected $readUsers;
56  protected $writeUsers;
58  protected $secureReadUsers;
60  protected $secureWriteUsers;
61 
63  protected $srvCache;
64 
67 
69  protected $authCreds;
71  protected $authSessionTimestamp = 0;
74 
76  protected $isRGW = false;
77 
112  public function __construct( array $config ) {
113  parent::__construct( $config );
114  // Required settings
115  $this->swiftAuthUrl = $config['swiftAuthUrl'];
116  $this->swiftUser = $config['swiftUser'];
117  $this->swiftKey = $config['swiftKey'];
118  // Optional settings
119  $this->authTTL = $config['swiftAuthTTL'] ?? 15 * 60; // some sane number
120  $this->swiftTempUrlKey = $config['swiftTempUrlKey'] ?? '';
121  $this->swiftStorageUrl = $config['swiftStorageUrl'] ?? null;
122  $this->shardViaHashLevels = $config['shardViaHashLevels'] ?? '';
123  $this->rgwS3AccessKey = $config['rgwS3AccessKey'] ?? '';
124  $this->rgwS3SecretKey = $config['rgwS3SecretKey'] ?? '';
125  // HTTP helper client
126  $this->http = new MultiHttpClient( [] );
127  // Cache container information to mask latency
128  if ( isset( $config['wanCache'] ) && $config['wanCache'] instanceof WANObjectCache ) {
129  $this->memCache = $config['wanCache'];
130  }
131  // Process cache for container info
132  $this->containerStatCache = new MapCacheLRU( 300 );
133  // Cache auth token information to avoid RTTs
134  if ( !empty( $config['cacheAuthInfo'] ) && isset( $config['srvCache'] ) ) {
135  $this->srvCache = $config['srvCache'];
136  } else {
137  $this->srvCache = new EmptyBagOStuff();
138  }
139  $this->readUsers = $config['readUsers'] ?? [];
140  $this->writeUsers = $config['writeUsers'] ?? [];
141  $this->secureReadUsers = $config['secureReadUsers'] ?? [];
142  $this->secureWriteUsers = $config['secureWriteUsers'] ?? [];
143  }
144 
145  public function getFeatures() {
148  }
149 
150  protected function resolveContainerPath( $container, $relStoragePath ) {
151  if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) {
152  return null; // not UTF-8, makes it hard to use CF and the swift HTTP API
153  } elseif ( strlen( rawurlencode( $relStoragePath ) ) > 1024 ) {
154  return null; // too long for Swift
155  }
156 
157  return $relStoragePath;
158  }
159 
160  public function isPathUsableInternal( $storagePath ) {
161  list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
162  if ( $rel === null ) {
163  return false; // invalid
164  }
165 
166  return is_array( $this->getContainerStat( $container ) );
167  }
168 
176  protected function sanitizeHdrsStrict( array $params ) {
177  if ( !isset( $params['headers'] ) ) {
178  return [];
179  }
180 
181  $headers = $this->getCustomHeaders( $params['headers'] );
182  unset( $headers[ 'content-type' ] );
183 
184  return $headers;
185  }
186 
199  protected function sanitizeHdrs( array $params ) {
200  return isset( $params['headers'] )
201  ? $this->getCustomHeaders( $params['headers'] )
202  : [];
203  }
204 
209  protected function getCustomHeaders( array $rawHeaders ) {
210  $headers = [];
211 
212  // Normalize casing, and strip out illegal headers
213  foreach ( $rawHeaders as $name => $value ) {
214  $name = strtolower( $name );
215  if ( preg_match( '/^content-length$/', $name ) ) {
216  continue; // blacklisted
217  } elseif ( preg_match( '/^(x-)?content-/', $name ) ) {
218  $headers[$name] = $value; // allowed
219  } elseif ( preg_match( '/^content-(disposition)/', $name ) ) {
220  $headers[$name] = $value; // allowed
221  }
222  }
223  // By default, Swift has annoyingly low maximum header value limits
224  if ( isset( $headers['content-disposition'] ) ) {
225  $disposition = '';
226  // @note: assume FileBackend::makeContentDisposition() already used
227  foreach ( explode( ';', $headers['content-disposition'] ) as $part ) {
228  $part = trim( $part );
229  $new = ( $disposition === '' ) ? $part : "{$disposition};{$part}";
230  if ( strlen( $new ) <= 255 ) {
231  $disposition = $new;
232  } else {
233  break; // too long; sigh
234  }
235  }
236  $headers['content-disposition'] = $disposition;
237  }
238 
239  return $headers;
240  }
241 
246  protected function getMetadataHeaders( array $rawHeaders ) {
247  $headers = [];
248  foreach ( $rawHeaders as $name => $value ) {
249  $name = strtolower( $name );
250  if ( strpos( $name, 'x-object-meta-' ) === 0 ) {
251  $headers[$name] = $value;
252  }
253  }
254 
255  return $headers;
256  }
257 
262  protected function getMetadata( array $rawHeaders ) {
263  $metadata = [];
264  foreach ( $this->getMetadataHeaders( $rawHeaders ) as $name => $value ) {
265  $metadata[substr( $name, strlen( 'x-object-meta-' ) )] = $value;
266  }
267 
268  return $metadata;
269  }
270 
271  protected function doCreateInternal( array $params ) {
272  $status = $this->newStatus();
273 
274  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
275  if ( $dstRel === null ) {
276  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
277 
278  return $status;
279  }
280 
281  $sha1Hash = Wikimedia\base_convert( sha1( $params['content'] ), 16, 36, 31 );
282  $contentType = $params['headers']['content-type']
283  ?? $this->getContentType( $params['dst'], $params['content'], null );
284 
285  $reqs = [ [
286  'method' => 'PUT',
287  'url' => [ $dstCont, $dstRel ],
288  'headers' => [
289  'content-length' => strlen( $params['content'] ),
290  'etag' => md5( $params['content'] ),
291  'content-type' => $contentType,
292  'x-object-meta-sha1base36' => $sha1Hash
293  ] + $this->sanitizeHdrsStrict( $params ),
294  'body' => $params['content']
295  ] ];
296 
297  $method = __METHOD__;
298  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
299  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
300  if ( $rcode === 201 ) {
301  // good
302  } elseif ( $rcode === 412 ) {
303  $status->fatal( 'backend-fail-contenttype', $params['dst'] );
304  } else {
305  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
306  }
307  };
308 
309  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
310  if ( !empty( $params['async'] ) ) { // deferred
311  $status->value = $opHandle;
312  } else { // actually write the object in Swift
313  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
314  }
315 
316  return $status;
317  }
318 
319  protected function doStoreInternal( array $params ) {
320  $status = $this->newStatus();
321 
322  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
323  if ( $dstRel === null ) {
324  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
325 
326  return $status;
327  }
328 
329  Wikimedia\suppressWarnings();
330  $sha1Hash = sha1_file( $params['src'] );
331  Wikimedia\restoreWarnings();
332  if ( $sha1Hash === false ) { // source doesn't exist?
333  $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
334 
335  return $status;
336  }
337  $sha1Hash = Wikimedia\base_convert( $sha1Hash, 16, 36, 31 );
338  $contentType = $params['headers']['content-type']
339  ?? $this->getContentType( $params['dst'], null, $params['src'] );
340 
341  $handle = fopen( $params['src'], 'rb' );
342  if ( $handle === false ) { // source doesn't exist?
343  $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
344 
345  return $status;
346  }
347 
348  $reqs = [ [
349  'method' => 'PUT',
350  'url' => [ $dstCont, $dstRel ],
351  'headers' => [
352  'content-length' => filesize( $params['src'] ),
353  'etag' => md5_file( $params['src'] ),
354  'content-type' => $contentType,
355  'x-object-meta-sha1base36' => $sha1Hash
356  ] + $this->sanitizeHdrsStrict( $params ),
357  'body' => $handle // resource
358  ] ];
359 
360  $method = __METHOD__;
361  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
362  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
363  if ( $rcode === 201 ) {
364  // good
365  } elseif ( $rcode === 412 ) {
366  $status->fatal( 'backend-fail-contenttype', $params['dst'] );
367  } else {
368  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
369  }
370  };
371 
372  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
373  $opHandle->resourcesToClose[] = $handle;
374 
375  if ( !empty( $params['async'] ) ) { // deferred
376  $status->value = $opHandle;
377  } else { // actually write the object in Swift
378  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
379  }
380 
381  return $status;
382  }
383 
384  protected function doCopyInternal( array $params ) {
385  $status = $this->newStatus();
386 
387  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
388  if ( $srcRel === null ) {
389  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
390 
391  return $status;
392  }
393 
394  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
395  if ( $dstRel === null ) {
396  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
397 
398  return $status;
399  }
400 
401  $reqs = [ [
402  'method' => 'PUT',
403  'url' => [ $dstCont, $dstRel ],
404  'headers' => [
405  'x-copy-from' => '/' . rawurlencode( $srcCont ) .
406  '/' . str_replace( "%2F", "/", rawurlencode( $srcRel ) )
407  ] + $this->sanitizeHdrsStrict( $params ), // extra headers merged into object
408  ] ];
409 
410  $method = __METHOD__;
411  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
412  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
413  if ( $rcode === 201 ) {
414  // good
415  } elseif ( $rcode === 404 ) {
416  $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
417  } else {
418  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
419  }
420  };
421 
422  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
423  if ( !empty( $params['async'] ) ) { // deferred
424  $status->value = $opHandle;
425  } else { // actually write the object in Swift
426  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
427  }
428 
429  return $status;
430  }
431 
432  protected function doMoveInternal( array $params ) {
433  $status = $this->newStatus();
434 
435  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
436  if ( $srcRel === null ) {
437  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
438 
439  return $status;
440  }
441 
442  list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
443  if ( $dstRel === null ) {
444  $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
445 
446  return $status;
447  }
448 
449  $reqs = [
450  [
451  'method' => 'PUT',
452  'url' => [ $dstCont, $dstRel ],
453  'headers' => [
454  'x-copy-from' => '/' . rawurlencode( $srcCont ) .
455  '/' . str_replace( "%2F", "/", rawurlencode( $srcRel ) )
456  ] + $this->sanitizeHdrsStrict( $params ) // extra headers merged into object
457  ]
458  ];
459  if ( "{$srcCont}/{$srcRel}" !== "{$dstCont}/{$dstRel}" ) {
460  $reqs[] = [
461  'method' => 'DELETE',
462  'url' => [ $srcCont, $srcRel ],
463  'headers' => []
464  ];
465  }
466 
467  $method = __METHOD__;
468  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
469  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
470  if ( $request['method'] === 'PUT' && $rcode === 201 ) {
471  // good
472  } elseif ( $request['method'] === 'DELETE' && $rcode === 204 ) {
473  // good
474  } elseif ( $rcode === 404 ) {
475  $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
476  } else {
477  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
478  }
479  };
480 
481  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
482  if ( !empty( $params['async'] ) ) { // deferred
483  $status->value = $opHandle;
484  } else { // actually move the object in Swift
485  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
486  }
487 
488  return $status;
489  }
490 
491  protected function doDeleteInternal( array $params ) {
492  $status = $this->newStatus();
493 
494  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
495  if ( $srcRel === null ) {
496  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
497 
498  return $status;
499  }
500 
501  $reqs = [ [
502  'method' => 'DELETE',
503  'url' => [ $srcCont, $srcRel ],
504  'headers' => []
505  ] ];
506 
507  $method = __METHOD__;
508  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
509  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
510  if ( $rcode === 204 ) {
511  // good
512  } elseif ( $rcode === 404 ) {
513  if ( empty( $params['ignoreMissingSource'] ) ) {
514  $status->fatal( 'backend-fail-delete', $params['src'] );
515  }
516  } else {
517  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
518  }
519  };
520 
521  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
522  if ( !empty( $params['async'] ) ) { // deferred
523  $status->value = $opHandle;
524  } else { // actually delete the object in Swift
525  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
526  }
527 
528  return $status;
529  }
530 
531  protected function doDescribeInternal( array $params ) {
532  $status = $this->newStatus();
533 
534  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
535  if ( $srcRel === null ) {
536  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
537 
538  return $status;
539  }
540 
541  // Fetch the old object headers/metadata...this should be in stat cache by now
542  $stat = $this->getFileStat( [ 'src' => $params['src'], 'latest' => 1 ] );
543  if ( $stat && !isset( $stat['xattr'] ) ) { // older cache entry
544  $stat = $this->doGetFileStat( [ 'src' => $params['src'], 'latest' => 1 ] );
545  }
546  if ( !$stat ) {
547  $status->fatal( 'backend-fail-describe', $params['src'] );
548 
549  return $status;
550  }
551 
552  // POST clears prior headers, so we need to merge the changes in to the old ones
553  $metaHdrs = [];
554  foreach ( $stat['xattr']['metadata'] as $name => $value ) {
555  $metaHdrs["x-object-meta-$name"] = $value;
556  }
557  $customHdrs = $this->sanitizeHdrs( $params ) + $stat['xattr']['headers'];
558 
559  $reqs = [ [
560  'method' => 'POST',
561  'url' => [ $srcCont, $srcRel ],
562  'headers' => $metaHdrs + $customHdrs
563  ] ];
564 
565  $method = __METHOD__;
566  $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
567  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
568  if ( $rcode === 202 ) {
569  // good
570  } elseif ( $rcode === 404 ) {
571  $status->fatal( 'backend-fail-describe', $params['src'] );
572  } else {
573  $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
574  }
575  };
576 
577  $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
578  if ( !empty( $params['async'] ) ) { // deferred
579  $status->value = $opHandle;
580  } else { // actually change the object in Swift
581  $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
582  }
583 
584  return $status;
585  }
586 
587  protected function doPrepareInternal( $fullCont, $dir, array $params ) {
588  $status = $this->newStatus();
589 
590  // (a) Check if container already exists
591  $stat = $this->getContainerStat( $fullCont );
592  if ( is_array( $stat ) ) {
593  return $status; // already there
594  } elseif ( $stat === null ) {
595  $status->fatal( 'backend-fail-internal', $this->name );
596  $this->logger->error( __METHOD__ . ': cannot get container stat' );
597 
598  return $status;
599  }
600 
601  // (b) Create container as needed with proper ACLs
602  if ( $stat === false ) {
603  $params['op'] = 'prepare';
604  $status->merge( $this->createContainer( $fullCont, $params ) );
605  }
606 
607  return $status;
608  }
609 
610  protected function doSecureInternal( $fullCont, $dir, array $params ) {
611  $status = $this->newStatus();
612  if ( empty( $params['noAccess'] ) ) {
613  return $status; // nothing to do
614  }
615 
616  $stat = $this->getContainerStat( $fullCont );
617  if ( is_array( $stat ) ) {
618  $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
619  $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
620  // Make container private to end-users...
621  $status->merge( $this->setContainerAccess(
622  $fullCont,
623  $readUsers,
625  ) );
626  } elseif ( $stat === false ) {
627  $status->fatal( 'backend-fail-usable', $params['dir'] );
628  } else {
629  $status->fatal( 'backend-fail-internal', $this->name );
630  $this->logger->error( __METHOD__ . ': cannot get container stat' );
631  }
632 
633  return $status;
634  }
635 
636  protected function doPublishInternal( $fullCont, $dir, array $params ) {
637  $status = $this->newStatus();
638 
639  $stat = $this->getContainerStat( $fullCont );
640  if ( is_array( $stat ) ) {
641  $readUsers = array_merge( $this->readUsers, [ $this->swiftUser, '.r:*' ] );
642  $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
643 
644  // Make container public to end-users...
645  $status->merge( $this->setContainerAccess(
646  $fullCont,
647  $readUsers,
649  ) );
650  } elseif ( $stat === false ) {
651  $status->fatal( 'backend-fail-usable', $params['dir'] );
652  } else {
653  $status->fatal( 'backend-fail-internal', $this->name );
654  $this->logger->error( __METHOD__ . ': cannot get container stat' );
655  }
656 
657  return $status;
658  }
659 
660  protected function doCleanInternal( $fullCont, $dir, array $params ) {
661  $status = $this->newStatus();
662 
663  // Only containers themselves can be removed, all else is virtual
664  if ( $dir != '' ) {
665  return $status; // nothing to do
666  }
667 
668  // (a) Check the container
669  $stat = $this->getContainerStat( $fullCont, true );
670  if ( $stat === false ) {
671  return $status; // ok, nothing to do
672  } elseif ( !is_array( $stat ) ) {
673  $status->fatal( 'backend-fail-internal', $this->name );
674  $this->logger->error( __METHOD__ . ': cannot get container stat' );
675 
676  return $status;
677  }
678 
679  // (b) Delete the container if empty
680  if ( $stat['count'] == 0 ) {
681  $params['op'] = 'clean';
682  $status->merge( $this->deleteContainer( $fullCont, $params ) );
683  }
684 
685  return $status;
686  }
687 
688  protected function doGetFileStat( array $params ) {
689  $params = [ 'srcs' => [ $params['src'] ], 'concurrency' => 1 ] + $params;
690  unset( $params['src'] );
691  $stats = $this->doGetFileStatMulti( $params );
692 
693  return reset( $stats );
694  }
695 
706  protected function convertSwiftDate( $ts, $format = TS_MW ) {
707  try {
708  $timestamp = new MWTimestamp( $ts );
709 
710  return $timestamp->getTimestamp( $format );
711  } catch ( Exception $e ) {
712  throw new FileBackendError( $e->getMessage() );
713  }
714  }
715 
723  protected function addMissingHashMetadata( array $objHdrs, $path ) {
724  if ( isset( $objHdrs['x-object-meta-sha1base36'] ) ) {
725  return $objHdrs; // nothing to do
726  }
727 
729  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
730  $this->logger->error( __METHOD__ . ": {path} was not stored with SHA-1 metadata.",
731  [ 'path' => $path ] );
732 
733  $objHdrs['x-object-meta-sha1base36'] = false;
734 
735  $auth = $this->getAuthentication();
736  if ( !$auth ) {
737  return $objHdrs; // failed
738  }
739 
740  // Find prior custom HTTP headers
741  $postHeaders = $this->getCustomHeaders( $objHdrs );
742  // Find prior metadata headers
743  $postHeaders += $this->getMetadataHeaders( $objHdrs );
744 
745  $status = $this->newStatus();
747  $scopeLockS = $this->getScopedFileLocks( [ $path ], LockManager::LOCK_UW, $status );
748  if ( $status->isOK() ) {
749  $tmpFile = $this->getLocalCopy( [ 'src' => $path, 'latest' => 1 ] );
750  if ( $tmpFile ) {
751  $hash = $tmpFile->getSha1Base36();
752  if ( $hash !== false ) {
753  $objHdrs['x-object-meta-sha1base36'] = $hash;
754  // Merge new SHA1 header into the old ones
755  $postHeaders['x-object-meta-sha1base36'] = $hash;
756  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
757  list( $rcode ) = $this->http->run( [
758  'method' => 'POST',
759  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
760  'headers' => $this->authTokenHeaders( $auth ) + $postHeaders
761  ] );
762  if ( $rcode >= 200 && $rcode <= 299 ) {
763  $this->deleteFileCache( $path );
764 
765  return $objHdrs; // success
766  }
767  }
768  }
769  }
770 
771  $this->logger->error( __METHOD__ . ': unable to set SHA-1 metadata for {path}',
772  [ 'path' => $path ] );
773 
774  return $objHdrs; // failed
775  }
776 
777  protected function doGetFileContentsMulti( array $params ) {
778  $contents = [];
779 
780  $auth = $this->getAuthentication();
781 
782  $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging
783  // Blindly create tmp files and stream to them, catching any exception
784  // if the file does not exist. Do not waste time doing file stats here.
785  $reqs = []; // (path => op)
786 
787  foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch
788  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
789  if ( $srcRel === null || !$auth ) {
790  $contents[$path] = false;
791  continue;
792  }
793  // Create a new temporary memory file...
794  $handle = fopen( 'php://temp', 'wb' );
795  if ( $handle ) {
796  $reqs[$path] = [
797  'method' => 'GET',
798  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
799  'headers' => $this->authTokenHeaders( $auth )
800  + $this->headersFromParams( $params ),
801  'stream' => $handle,
802  ];
803  }
804  $contents[$path] = false;
805  }
806 
807  $opts = [ 'maxConnsPerHost' => $params['concurrency'] ];
808  $reqs = $this->http->runMulti( $reqs, $opts );
809  foreach ( $reqs as $path => $op ) {
810  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response'];
811  if ( $rcode >= 200 && $rcode <= 299 ) {
812  rewind( $op['stream'] ); // start from the beginning
813  $contents[$path] = stream_get_contents( $op['stream'] );
814  } elseif ( $rcode === 404 ) {
815  $contents[$path] = false;
816  } else {
817  $this->onError( null, __METHOD__,
818  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
819  }
820  fclose( $op['stream'] ); // close open handle
821  }
822 
823  return $contents;
824  }
825 
826  protected function doDirectoryExists( $fullCont, $dir, array $params ) {
827  $prefix = ( $dir == '' ) ? null : "{$dir}/";
828  $status = $this->objectListing( $fullCont, 'names', 1, null, $prefix );
829  if ( $status->isOK() ) {
830  return ( count( $status->value ) ) > 0;
831  }
832 
833  return null; // error
834  }
835 
843  public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
844  return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
845  }
846 
854  public function getFileListInternal( $fullCont, $dir, array $params ) {
855  return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
856  }
857 
869  public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
870  $dirs = [];
871  if ( $after === INF ) {
872  return $dirs; // nothing more
873  }
874 
875  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
876 
877  $prefix = ( $dir == '' ) ? null : "{$dir}/";
878  // Non-recursive: only list dirs right under $dir
879  if ( !empty( $params['topOnly'] ) ) {
880  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
881  if ( !$status->isOK() ) {
882  throw new FileBackendError( "Iterator page I/O error." );
883  }
884  $objects = $status->value;
885  foreach ( $objects as $object ) { // files and directories
886  if ( substr( $object, -1 ) === '/' ) {
887  $dirs[] = $object; // directories end in '/'
888  }
889  }
890  } else {
891  // Recursive: list all dirs under $dir and its subdirs
892  $getParentDir = function ( $path ) {
893  return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
894  };
895 
896  // Get directory from last item of prior page
897  $lastDir = $getParentDir( $after ); // must be first page
898  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
899 
900  if ( !$status->isOK() ) {
901  throw new FileBackendError( "Iterator page I/O error." );
902  }
903 
904  $objects = $status->value;
905 
906  foreach ( $objects as $object ) { // files
907  $objectDir = $getParentDir( $object ); // directory of object
908 
909  if ( $objectDir !== false && $objectDir !== $dir ) {
910  // Swift stores paths in UTF-8, using binary sorting.
911  // See function "create_container_table" in common/db.py.
912  // If a directory is not "greater" than the last one,
913  // then it was already listed by the calling iterator.
914  if ( strcmp( $objectDir, $lastDir ) > 0 ) {
915  $pDir = $objectDir;
916  do { // add dir and all its parent dirs
917  $dirs[] = "{$pDir}/";
918  $pDir = $getParentDir( $pDir );
919  } while ( $pDir !== false // sanity
920  && strcmp( $pDir, $lastDir ) > 0 // not done already
921  && strlen( $pDir ) > strlen( $dir ) // within $dir
922  );
923  }
924  $lastDir = $objectDir;
925  }
926  }
927  }
928  // Page on the unfiltered directory listing (what is returned may be filtered)
929  if ( count( $objects ) < $limit ) {
930  $after = INF; // avoid a second RTT
931  } else {
932  $after = end( $objects ); // update last item
933  }
934 
935  return $dirs;
936  }
937 
949  public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
950  $files = []; // list of (path, stat array or null) entries
951  if ( $after === INF ) {
952  return $files; // nothing more
953  }
954 
955  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
956 
957  $prefix = ( $dir == '' ) ? null : "{$dir}/";
958  // $objects will contain a list of unfiltered names or CF_Object items
959  // Non-recursive: only list files right under $dir
960  if ( !empty( $params['topOnly'] ) ) {
961  if ( !empty( $params['adviseStat'] ) ) {
962  $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix, '/' );
963  } else {
964  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
965  }
966  } else {
967  // Recursive: list all files under $dir and its subdirs
968  if ( !empty( $params['adviseStat'] ) ) {
969  $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix );
970  } else {
971  $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
972  }
973  }
974 
975  // Reformat this list into a list of (name, stat array or null) entries
976  if ( !$status->isOK() ) {
977  throw new FileBackendError( "Iterator page I/O error." );
978  }
979 
980  $objects = $status->value;
981  $files = $this->buildFileObjectListing( $params, $dir, $objects );
982 
983  // Page on the unfiltered object listing (what is returned may be filtered)
984  if ( count( $objects ) < $limit ) {
985  $after = INF; // avoid a second RTT
986  } else {
987  $after = end( $objects ); // update last item
988  $after = is_object( $after ) ? $after->name : $after;
989  }
990 
991  return $files;
992  }
993 
1003  private function buildFileObjectListing( array $params, $dir, array $objects ) {
1004  $names = [];
1005  foreach ( $objects as $object ) {
1006  if ( is_object( $object ) ) {
1007  if ( isset( $object->subdir ) || !isset( $object->name ) ) {
1008  continue; // virtual directory entry; ignore
1009  }
1010  $stat = [
1011  // Convert various random Swift dates to TS_MW
1012  'mtime' => $this->convertSwiftDate( $object->last_modified, TS_MW ),
1013  'size' => (int)$object->bytes,
1014  'sha1' => null,
1015  // Note: manifiest ETags are not an MD5 of the file
1016  'md5' => ctype_xdigit( $object->hash ) ? $object->hash : null,
1017  'latest' => false // eventually consistent
1018  ];
1019  $names[] = [ $object->name, $stat ];
1020  } elseif ( substr( $object, -1 ) !== '/' ) {
1021  // Omit directories, which end in '/' in listings
1022  $names[] = [ $object, null ];
1023  }
1024  }
1025 
1026  return $names;
1027  }
1028 
1035  public function loadListingStatInternal( $path, array $val ) {
1036  $this->cheapCache->setField( $path, 'stat', $val );
1037  }
1038 
1039  protected function doGetFileXAttributes( array $params ) {
1040  $stat = $this->getFileStat( $params );
1041  if ( $stat ) {
1042  if ( !isset( $stat['xattr'] ) ) {
1043  // Stat entries filled by file listings don't include metadata/headers
1044  $this->clearCache( [ $params['src'] ] );
1045  $stat = $this->getFileStat( $params );
1046  }
1047 
1048  return $stat['xattr'];
1049  } else {
1050  return false;
1051  }
1052  }
1053 
1054  protected function doGetFileSha1base36( array $params ) {
1055  // Avoid using stat entries from file listings, which never include the SHA-1 hash.
1056  // Also, recompute the hash if it's not part of the metadata headers for some reason.
1057  $params['requireSHA1'] = true;
1058 
1059  $stat = $this->getFileStat( $params );
1060  if ( $stat ) {
1061  return $stat['sha1'];
1062  } else {
1063  return false;
1064  }
1065  }
1066 
1067  protected function doStreamFile( array $params ) {
1068  $status = $this->newStatus();
1069 
1070  $flags = !empty( $params['headless'] ) ? StreamFile::STREAM_HEADLESS : 0;
1071 
1072  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1073  if ( $srcRel === null ) {
1074  StreamFile::send404Message( $params['src'], $flags );
1075  $status->fatal( 'backend-fail-invalidpath', $params['src'] );
1076 
1077  return $status;
1078  }
1079 
1080  $auth = $this->getAuthentication();
1081  if ( !$auth || !is_array( $this->getContainerStat( $srcCont ) ) ) {
1082  StreamFile::send404Message( $params['src'], $flags );
1083  $status->fatal( 'backend-fail-stream', $params['src'] );
1084 
1085  return $status;
1086  }
1087 
1088  // If "headers" is set, we only want to send them if the file is there.
1089  // Do not bother checking if the file exists if headers are not set though.
1090  if ( $params['headers'] && !$this->fileExists( $params ) ) {
1091  StreamFile::send404Message( $params['src'], $flags );
1092  $status->fatal( 'backend-fail-stream', $params['src'] );
1093 
1094  return $status;
1095  }
1096 
1097  // Send the requested additional headers
1098  foreach ( $params['headers'] as $header ) {
1099  header( $header ); // aways send
1100  }
1101 
1102  if ( empty( $params['allowOB'] ) ) {
1103  // Cancel output buffering and gzipping if set
1104  ( $this->obResetFunc )();
1105  }
1106 
1107  $handle = fopen( 'php://output', 'wb' );
1108  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1109  'method' => 'GET',
1110  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1111  'headers' => $this->authTokenHeaders( $auth )
1112  + $this->headersFromParams( $params ) + $params['options'],
1113  'stream' => $handle,
1114  'flags' => [ 'relayResponseHeaders' => empty( $params['headless'] ) ]
1115  ] );
1116 
1117  if ( $rcode >= 200 && $rcode <= 299 ) {
1118  // good
1119  } elseif ( $rcode === 404 ) {
1120  $status->fatal( 'backend-fail-stream', $params['src'] );
1121  // Per T43113, nasty things can happen if bad cache entries get
1122  // stuck in cache. It's also possible that this error can come up
1123  // with simple race conditions. Clear out the stat cache to be safe.
1124  $this->clearCache( [ $params['src'] ] );
1125  $this->deleteFileCache( $params['src'] );
1126  } else {
1127  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1128  }
1129 
1130  return $status;
1131  }
1132 
1133  protected function doGetLocalCopyMulti( array $params ) {
1135  $tmpFiles = [];
1136 
1137  $auth = $this->getAuthentication();
1138 
1139  $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging
1140  // Blindly create tmp files and stream to them, catching any exception
1141  // if the file does not exist. Do not waste time doing file stats here.
1142  $reqs = []; // (path => op)
1143 
1144  foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch
1145  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
1146  if ( $srcRel === null || !$auth ) {
1147  $tmpFiles[$path] = null;
1148  continue;
1149  }
1150  // Get source file extension
1152  // Create a new temporary file...
1153  $tmpFile = TempFSFile::factory( 'localcopy_', $ext, $this->tmpDirectory );
1154  if ( $tmpFile ) {
1155  $handle = fopen( $tmpFile->getPath(), 'wb' );
1156  if ( $handle ) {
1157  $reqs[$path] = [
1158  'method' => 'GET',
1159  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1160  'headers' => $this->authTokenHeaders( $auth )
1161  + $this->headersFromParams( $params ),
1162  'stream' => $handle,
1163  ];
1164  } else {
1165  $tmpFile = null;
1166  }
1167  }
1168  $tmpFiles[$path] = $tmpFile;
1169  }
1170 
1171  $isLatest = ( $this->isRGW || !empty( $params['latest'] ) );
1172  $opts = [ 'maxConnsPerHost' => $params['concurrency'] ];
1173  $reqs = $this->http->runMulti( $reqs, $opts );
1174  foreach ( $reqs as $path => $op ) {
1175  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response'];
1176  fclose( $op['stream'] ); // close open handle
1177  if ( $rcode >= 200 && $rcode <= 299 ) {
1178  $size = $tmpFiles[$path] ? $tmpFiles[$path]->getSize() : 0;
1179  // Double check that the disk is not full/broken
1180  if ( $size != $rhdrs['content-length'] ) {
1181  $tmpFiles[$path] = null;
1182  $rerr = "Got {$size}/{$rhdrs['content-length']} bytes";
1183  $this->onError( null, __METHOD__,
1184  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
1185  }
1186  // Set the file stat process cache in passing
1187  $stat = $this->getStatFromHeaders( $rhdrs );
1188  $stat['latest'] = $isLatest;
1189  $this->cheapCache->setField( $path, 'stat', $stat );
1190  } elseif ( $rcode === 404 ) {
1191  $tmpFiles[$path] = false;
1192  } else {
1193  $tmpFiles[$path] = null;
1194  $this->onError( null, __METHOD__,
1195  [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
1196  }
1197  }
1198 
1199  return $tmpFiles;
1200  }
1201 
1202  public function getFileHttpUrl( array $params ) {
1203  if ( $this->swiftTempUrlKey != '' ||
1204  ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' )
1205  ) {
1206  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1207  if ( $srcRel === null ) {
1208  return null; // invalid path
1209  }
1210 
1211  $auth = $this->getAuthentication();
1212  if ( !$auth ) {
1213  return null;
1214  }
1215 
1216  $ttl = $params['ttl'] ?? 86400;
1217  $expires = time() + $ttl;
1218 
1219  if ( $this->swiftTempUrlKey != '' ) {
1220  $url = $this->storageUrl( $auth, $srcCont, $srcRel );
1221  // Swift wants the signature based on the unencoded object name
1222  $contPath = parse_url( $this->storageUrl( $auth, $srcCont ), PHP_URL_PATH );
1223  $signature = hash_hmac( 'sha1',
1224  "GET\n{$expires}\n{$contPath}/{$srcRel}",
1225  $this->swiftTempUrlKey
1226  );
1227 
1228  return "{$url}?temp_url_sig={$signature}&temp_url_expires={$expires}";
1229  } else { // give S3 API URL for rgw
1230  // Path for signature starts with the bucket
1231  $spath = '/' . rawurlencode( $srcCont ) . '/' .
1232  str_replace( '%2F', '/', rawurlencode( $srcRel ) );
1233  // Calculate the hash
1234  $signature = base64_encode( hash_hmac(
1235  'sha1',
1236  "GET\n\n\n{$expires}\n{$spath}",
1237  $this->rgwS3SecretKey,
1238  true // raw
1239  ) );
1240  // See https://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html.
1241  // Note: adding a newline for empty CanonicalizedAmzHeaders does not work.
1242  // Note: S3 API is the rgw default; remove the /swift/ URL bit.
1243  return str_replace( '/swift/v1', '', $this->storageUrl( $auth ) . $spath ) .
1244  '?' .
1245  http_build_query( [
1246  'Signature' => $signature,
1247  'Expires' => $expires,
1248  'AWSAccessKeyId' => $this->rgwS3AccessKey
1249  ] );
1250  }
1251  }
1252 
1253  return null;
1254  }
1255 
1256  protected function directoriesAreVirtual() {
1257  return true;
1258  }
1259 
1268  protected function headersFromParams( array $params ) {
1269  $hdrs = [];
1270  if ( !empty( $params['latest'] ) ) {
1271  $hdrs['x-newest'] = 'true';
1272  }
1273 
1274  return $hdrs;
1275  }
1276 
1282  protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1284  $statuses = [];
1285 
1286  $auth = $this->getAuthentication();
1287  if ( !$auth ) {
1288  foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1289  $statuses[$index] = $this->newStatus( 'backend-fail-connect', $this->name );
1290  }
1291 
1292  return $statuses;
1293  }
1294 
1295  // Split the HTTP requests into stages that can be done concurrently
1296  $httpReqsByStage = []; // map of (stage => index => HTTP request)
1297  foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1299  $reqs = $fileOpHandle->httpOp;
1300  // Convert the 'url' parameter to an actual URL using $auth
1301  foreach ( $reqs as $stage => &$req ) {
1302  list( $container, $relPath ) = $req['url'];
1303  $req['url'] = $this->storageUrl( $auth, $container, $relPath );
1304  $req['headers'] = $req['headers'] ?? [];
1305  $req['headers'] = $this->authTokenHeaders( $auth ) + $req['headers'];
1306  $httpReqsByStage[$stage][$index] = $req;
1307  }
1308  $statuses[$index] = $this->newStatus();
1309  }
1310 
1311  // Run all requests for the first stage, then the next, and so on
1312  $reqCount = count( $httpReqsByStage );
1313  for ( $stage = 0; $stage < $reqCount; ++$stage ) {
1314  $httpReqs = $this->http->runMulti( $httpReqsByStage[$stage] );
1315  foreach ( $httpReqs as $index => $httpReq ) {
1316  // Run the callback for each request of this operation
1317  $callback = $fileOpHandles[$index]->callback;
1318  $callback( $httpReq, $statuses[$index] );
1319  // On failure, abort all remaining requests for this operation
1320  // (e.g. abort the DELETE request if the COPY request fails for a move)
1321  if ( !$statuses[$index]->isOK() ) {
1322  $stages = count( $fileOpHandles[$index]->httpOp );
1323  for ( $s = ( $stage + 1 ); $s < $stages; ++$s ) {
1324  unset( $httpReqsByStage[$s][$index] );
1325  }
1326  }
1327  }
1328  }
1329 
1330  return $statuses;
1331  }
1332 
1355  protected function setContainerAccess( $container, array $readUsers, array $writeUsers ) {
1356  $status = $this->newStatus();
1357  $auth = $this->getAuthentication();
1358 
1359  if ( !$auth ) {
1360  $status->fatal( 'backend-fail-connect', $this->name );
1361 
1362  return $status;
1363  }
1364 
1365  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1366  'method' => 'POST',
1367  'url' => $this->storageUrl( $auth, $container ),
1368  'headers' => $this->authTokenHeaders( $auth ) + [
1369  'x-container-read' => implode( ',', $readUsers ),
1370  'x-container-write' => implode( ',', $writeUsers )
1371  ]
1372  ] );
1373 
1374  if ( $rcode != 204 && $rcode !== 202 ) {
1375  $status->fatal( 'backend-fail-internal', $this->name );
1376  $this->logger->error( __METHOD__ . ': unexpected rcode value ({rcode})',
1377  [ 'rcode' => $rcode ] );
1378  }
1379 
1380  return $status;
1381  }
1382 
1391  protected function getContainerStat( $container, $bypassCache = false ) {
1392  $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1393 
1394  if ( $bypassCache ) { // purge cache
1395  $this->containerStatCache->clear( $container );
1396  } elseif ( !$this->containerStatCache->hasField( $container, 'stat' ) ) {
1397  $this->primeContainerCache( [ $container ] ); // check persistent cache
1398  }
1399  if ( !$this->containerStatCache->hasField( $container, 'stat' ) ) {
1400  $auth = $this->getAuthentication();
1401  if ( !$auth ) {
1402  return null;
1403  }
1404 
1405  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1406  'method' => 'HEAD',
1407  'url' => $this->storageUrl( $auth, $container ),
1408  'headers' => $this->authTokenHeaders( $auth )
1409  ] );
1410 
1411  if ( $rcode === 204 ) {
1412  $stat = [
1413  'count' => $rhdrs['x-container-object-count'],
1414  'bytes' => $rhdrs['x-container-bytes-used']
1415  ];
1416  if ( $bypassCache ) {
1417  return $stat;
1418  } else {
1419  $this->containerStatCache->setField( $container, 'stat', $stat ); // cache it
1420  $this->setContainerCache( $container, $stat ); // update persistent cache
1421  }
1422  } elseif ( $rcode === 404 ) {
1423  return false;
1424  } else {
1425  $this->onError( null, __METHOD__,
1426  [ 'cont' => $container ], $rerr, $rcode, $rdesc );
1427 
1428  return null;
1429  }
1430  }
1431 
1432  return $this->containerStatCache->getField( $container, 'stat' );
1433  }
1434 
1442  protected function createContainer( $container, array $params ) {
1443  $status = $this->newStatus();
1444 
1445  $auth = $this->getAuthentication();
1446  if ( !$auth ) {
1447  $status->fatal( 'backend-fail-connect', $this->name );
1448 
1449  return $status;
1450  }
1451 
1452  // @see SwiftFileBackend::setContainerAccess()
1453  if ( empty( $params['noAccess'] ) ) {
1454  // public
1455  $readUsers = array_merge( $this->readUsers, [ '.r:*', $this->swiftUser ] );
1456  $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
1457  } else {
1458  // private
1459  $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
1460  $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
1461  }
1462 
1463  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1464  'method' => 'PUT',
1465  'url' => $this->storageUrl( $auth, $container ),
1466  'headers' => $this->authTokenHeaders( $auth ) + [
1467  'x-container-read' => implode( ',', $readUsers ),
1468  'x-container-write' => implode( ',', $writeUsers )
1469  ]
1470  ] );
1471 
1472  if ( $rcode === 201 ) { // new
1473  // good
1474  } elseif ( $rcode === 202 ) { // already there
1475  // this shouldn't really happen, but is OK
1476  } else {
1477  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1478  }
1479 
1480  return $status;
1481  }
1482 
1490  protected function deleteContainer( $container, array $params ) {
1491  $status = $this->newStatus();
1492 
1493  $auth = $this->getAuthentication();
1494  if ( !$auth ) {
1495  $status->fatal( 'backend-fail-connect', $this->name );
1496 
1497  return $status;
1498  }
1499 
1500  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1501  'method' => 'DELETE',
1502  'url' => $this->storageUrl( $auth, $container ),
1503  'headers' => $this->authTokenHeaders( $auth )
1504  ] );
1505 
1506  if ( $rcode >= 200 && $rcode <= 299 ) { // deleted
1507  $this->containerStatCache->clear( $container ); // purge
1508  } elseif ( $rcode === 404 ) { // not there
1509  // this shouldn't really happen, but is OK
1510  } elseif ( $rcode === 409 ) { // not empty
1511  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); // race?
1512  } else {
1513  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1514  }
1515 
1516  return $status;
1517  }
1518 
1531  private function objectListing(
1532  $fullCont, $type, $limit, $after = null, $prefix = null, $delim = null
1533  ) {
1534  $status = $this->newStatus();
1535 
1536  $auth = $this->getAuthentication();
1537  if ( !$auth ) {
1538  $status->fatal( 'backend-fail-connect', $this->name );
1539 
1540  return $status;
1541  }
1542 
1543  $query = [ 'limit' => $limit ];
1544  if ( $type === 'info' ) {
1545  $query['format'] = 'json';
1546  }
1547  if ( $after !== null ) {
1548  $query['marker'] = $after;
1549  }
1550  if ( $prefix !== null ) {
1551  $query['prefix'] = $prefix;
1552  }
1553  if ( $delim !== null ) {
1554  $query['delimiter'] = $delim;
1555  }
1556 
1557  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1558  'method' => 'GET',
1559  'url' => $this->storageUrl( $auth, $fullCont ),
1560  'query' => $query,
1561  'headers' => $this->authTokenHeaders( $auth )
1562  ] );
1563 
1564  $params = [ 'cont' => $fullCont, 'prefix' => $prefix, 'delim' => $delim ];
1565  if ( $rcode === 200 ) { // good
1566  if ( $type === 'info' ) {
1567  $status->value = FormatJson::decode( trim( $rbody ) );
1568  } else {
1569  $status->value = explode( "\n", trim( $rbody ) );
1570  }
1571  } elseif ( $rcode === 204 ) {
1572  $status->value = []; // empty container
1573  } elseif ( $rcode === 404 ) {
1574  $status->value = []; // no container
1575  } else {
1576  $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1577  }
1578 
1579  return $status;
1580  }
1581 
1582  protected function doPrimeContainerCache( array $containerInfo ) {
1583  foreach ( $containerInfo as $container => $info ) {
1584  $this->containerStatCache->setField( $container, 'stat', $info );
1585  }
1586  }
1587 
1588  protected function doGetFileStatMulti( array $params ) {
1589  $stats = [];
1590 
1591  $auth = $this->getAuthentication();
1592 
1593  $reqs = [];
1594  foreach ( $params['srcs'] as $path ) {
1595  list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
1596  if ( $srcRel === null ) {
1597  $stats[$path] = false;
1598  continue; // invalid storage path
1599  } elseif ( !$auth ) {
1600  $stats[$path] = null;
1601  continue;
1602  }
1603 
1604  // (a) Check the container
1605  $cstat = $this->getContainerStat( $srcCont );
1606  if ( $cstat === false ) {
1607  $stats[$path] = false;
1608  continue; // ok, nothing to do
1609  } elseif ( !is_array( $cstat ) ) {
1610  $stats[$path] = null;
1611  continue;
1612  }
1613 
1614  $reqs[$path] = [
1615  'method' => 'HEAD',
1616  'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1617  'headers' => $this->authTokenHeaders( $auth ) + $this->headersFromParams( $params )
1618  ];
1619  }
1620 
1621  $opts = [ 'maxConnsPerHost' => $params['concurrency'] ];
1622  $reqs = $this->http->runMulti( $reqs, $opts );
1623 
1624  foreach ( $params['srcs'] as $path ) {
1625  if ( array_key_exists( $path, $stats ) ) {
1626  continue; // some sort of failure above
1627  }
1628  // (b) Check the file
1629  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $reqs[$path]['response'];
1630  if ( $rcode === 200 || $rcode === 204 ) {
1631  // Update the object if it is missing some headers
1632  if ( !empty( $params['requireSHA1'] ) ) {
1633  $rhdrs = $this->addMissingHashMetadata( $rhdrs, $path );
1634  }
1635  // Load the stat array from the headers
1636  $stat = $this->getStatFromHeaders( $rhdrs );
1637  if ( $this->isRGW ) {
1638  $stat['latest'] = true; // strong consistency
1639  }
1640  } elseif ( $rcode === 404 ) {
1641  $stat = false;
1642  } else {
1643  $stat = null;
1644  $this->onError( null, __METHOD__, $params, $rerr, $rcode, $rdesc );
1645  }
1646  $stats[$path] = $stat;
1647  }
1648 
1649  return $stats;
1650  }
1651 
1656  protected function getStatFromHeaders( array $rhdrs ) {
1657  // Fetch all of the custom metadata headers
1658  $metadata = $this->getMetadata( $rhdrs );
1659  // Fetch all of the custom raw HTTP headers
1660  $headers = $this->sanitizeHdrs( [ 'headers' => $rhdrs ] );
1661 
1662  return [
1663  // Convert various random Swift dates to TS_MW
1664  'mtime' => $this->convertSwiftDate( $rhdrs['last-modified'], TS_MW ),
1665  // Empty objects actually return no content-length header in Ceph
1666  'size' => isset( $rhdrs['content-length'] ) ? (int)$rhdrs['content-length'] : 0,
1667  'sha1' => $metadata['sha1base36'] ?? null,
1668  // Note: manifiest ETags are not an MD5 of the file
1669  'md5' => ctype_xdigit( $rhdrs['etag'] ) ? $rhdrs['etag'] : null,
1670  'xattr' => [ 'metadata' => $metadata, 'headers' => $headers ]
1671  ];
1672  }
1673 
1677  protected function getAuthentication() {
1678  if ( $this->authErrorTimestamp !== null ) {
1679  if ( ( time() - $this->authErrorTimestamp ) < 60 ) {
1680  return null; // failed last attempt; don't bother
1681  } else { // actually retry this time
1682  $this->authErrorTimestamp = null;
1683  }
1684  }
1685  // Session keys expire after a while, so we renew them periodically
1686  $reAuth = ( ( time() - $this->authSessionTimestamp ) > $this->authTTL );
1687  // Authenticate with proxy and get a session key...
1688  if ( !$this->authCreds || $reAuth ) {
1689  $this->authSessionTimestamp = 0;
1690  $cacheKey = $this->getCredsCacheKey( $this->swiftUser );
1691  $creds = $this->srvCache->get( $cacheKey ); // credentials
1692  // Try to use the credential cache
1693  if ( isset( $creds['auth_token'] ) && isset( $creds['storage_url'] ) ) {
1694  $this->authCreds = $creds;
1695  // Skew the timestamp for worst case to avoid using stale credentials
1696  $this->authSessionTimestamp = time() - ceil( $this->authTTL / 2 );
1697  } else { // cache miss
1698  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1699  'method' => 'GET',
1700  'url' => "{$this->swiftAuthUrl}/v1.0",
1701  'headers' => [
1702  'x-auth-user' => $this->swiftUser,
1703  'x-auth-key' => $this->swiftKey
1704  ]
1705  ] );
1706 
1707  if ( $rcode >= 200 && $rcode <= 299 ) { // OK
1708  $this->authCreds = [
1709  'auth_token' => $rhdrs['x-auth-token'],
1710  'storage_url' => ( $this->swiftStorageUrl !== null )
1711  ? $this->swiftStorageUrl
1712  : $rhdrs['x-storage-url']
1713  ];
1714 
1715  $this->srvCache->set( $cacheKey, $this->authCreds, ceil( $this->authTTL / 2 ) );
1716  $this->authSessionTimestamp = time();
1717  } elseif ( $rcode === 401 ) {
1718  $this->onError( null, __METHOD__, [], "Authentication failed.", $rcode );
1719  $this->authErrorTimestamp = time();
1720 
1721  return null;
1722  } else {
1723  $this->onError( null, __METHOD__, [], "HTTP return code: $rcode", $rcode );
1724  $this->authErrorTimestamp = time();
1725 
1726  return null;
1727  }
1728  }
1729  // Ceph RGW does not use <account> in URLs (OpenStack Swift uses "/v1/<account>")
1730  if ( substr( $this->authCreds['storage_url'], -3 ) === '/v1' ) {
1731  $this->isRGW = true; // take advantage of strong consistency in Ceph
1732  }
1733  }
1734 
1735  return $this->authCreds;
1736  }
1737 
1744  protected function storageUrl( array $creds, $container = null, $object = null ) {
1745  $parts = [ $creds['storage_url'] ];
1746  if ( strlen( $container ) ) {
1747  $parts[] = rawurlencode( $container );
1748  }
1749  if ( strlen( $object ) ) {
1750  $parts[] = str_replace( "%2F", "/", rawurlencode( $object ) );
1751  }
1752 
1753  return implode( '/', $parts );
1754  }
1755 
1760  protected function authTokenHeaders( array $creds ) {
1761  return [ 'x-auth-token' => $creds['auth_token'] ];
1762  }
1763 
1770  private function getCredsCacheKey( $username ) {
1771  return 'swiftcredentials:' . md5( $username . ':' . $this->swiftAuthUrl );
1772  }
1773 
1785  public function onError( $status, $func, array $params, $err = '', $code = 0, $desc = '' ) {
1786  if ( $status instanceof StatusValue ) {
1787  $status->fatal( 'backend-fail-internal', $this->name );
1788  }
1789  if ( $code == 401 ) { // possibly a stale token
1790  $this->srvCache->delete( $this->getCredsCacheKey( $this->swiftUser ) );
1791  }
1792  $msg = "HTTP {code} ({desc}) in '{func}' (given '{req_params}')";
1793  $msgParams = [
1794  'code' => $code,
1795  'desc' => $desc,
1796  'func' => $func,
1797  'req_params' => FormatJson::encode( $params ),
1798  ];
1799  if ( $err ) {
1800  $msg .= ': {err}';
1801  $msgParams['err'] = $err;
1802  }
1803  $this->logger->error( $msg, $msgParams );
1804  }
1805 }
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
doStoreInternal(array $params)
The wiki should then use memcached to cache various data To use multiple just add more items to the array To increase the weight of a make its entry a array("192.168.0.1:11211", 2))
doGetFileStatMulti(array $params)
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
buildFileObjectListing(array $params, $dir, array $objects)
Build a list of file objects, filtering out any directories and extracting any stat info if provided ...
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition: deferred.txt:11
null for the local wiki Added should default to null in handler for backwards compatibility add a value to it if you want to add a cookie that have to vary cache options can modify $query
Definition: hooks.txt:1585
getCustomHeaders(array $rawHeaders)
static send404Message( $fname, $flags=0)
Send out a standard 404 message for a file.
Definition: StreamFile.php:70
Iterator for listing directories.
processing should stop and the error should be shown to the user * false
Definition: hooks.txt:187
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
div flags Integer display flags(NO_ACTION_LINK, NO_EXTRA_USER_LINKS) 'LogException' returning false will NOT prevent logging $e
Definition: hooks.txt:2159
string $swiftUser
Swift user (account:user) to authenticate as.
int $authErrorTimestamp
UNIX timestamp.
scopedProfileSection( $section)
getFileStat(array $params)
const ATTR_HEADERS
Bitfield flags for supported features.
createContainer( $container, array $params)
Create a Swift container.
$value
setContainerCache( $container, array $val)
Set the cached info for a container.
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
static encode( $value, $pretty=false, $escaping=0)
Returns the JSON representation of a value.
Definition: FormatJson.php:115
getContainerStat( $container, $bypassCache=false)
Get a Swift container stat array, possibly from process cache.
doDeleteInternal(array $params)
static extensionFromPath( $path, $case='lowercase')
Get the final extension from a storage or FS path.
doMoveInternal(array $params)
getStatFromHeaders(array $rhdrs)
doGetFileXAttributes(array $params)
doDirectoryExists( $fullCont, $dir, array $params)
Apache License January http
sanitizeHdrsStrict(array $params)
Sanitize and filter the custom headers from a $params array.
doGetFileContentsMulti(array $params)
and how to run hooks for an and one after Each event has a name
Definition: hooks.txt:6
getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that When $user is not it can be in the form of< username >< more info > e g for bot passwords intended to be added to log contexts Fields it might only if the login was with a bot password it is not rendered in wiki pages or galleries in category pages allow injecting custom HTML after the section Any uses of the hook need to handle escaping see BaseTemplate::getToolbox and BaseTemplate::makeListItem for details on the format of individual items inside of this array or by returning and letting standard HTTP rendering take place modifiable or by returning false and taking over the output modifiable modifiable after all normalizations have been except for the $wgMaxImageArea check set to true or false to override the $wgMaxImageArea check result gives extension the possibility to transform it themselves $handler
Definition: hooks.txt:780
doGetLocalCopyMulti(array $params)
setContainerAccess( $container, array $readUsers, array $writeUsers)
Set read/write permissions for a Swift container.
doGetFileSha1base36(array $params)
loadListingStatInternal( $path, array $val)
Do not call this function outside of SwiftFileBackendFileList.
Status::newGood()` to allow deletion, and then `return false` from the hook function. Ensure you consume the 'ChangeTagAfterDelete' hook to carry out custom deletion actions. $tag:name of the tag $user:user initiating the action & $status:Status object. See above. 'ChangeTagsListActive':Allows you to nominate which of the tags your extension uses are in active use. & $tags:list of all active tags. Append to this array. 'ChangeTagsAfterUpdateTags':Called after tags have been updated with the ChangeTags::updateTags function. Params:$addedTags:tags effectively added in the update $removedTags:tags effectively removed in the update $prevTags:tags that were present prior to the update $rc_id:recentchanges table id $rev_id:revision table id $log_id:logging table id $params:tag params $rc:RecentChange being tagged when the tagging accompanies the action, or null $user:User who performed the tagging when the tagging is subsequent to the action, or null 'ChangeTagsAllowedAdd':Called when checking if a user can add tags to a change. & $allowedTags:List of all the tags the user is allowed to add. Any tags the user wants to add( $addTags) that are not in this array will cause it to fail. You may add or remove tags to this array as required. $addTags:List of tags user intends to add. $user:User who is adding the tags. 'ChangeUserGroups':Called before user groups are changed. $performer:The User who will perform the change $user:The User whose groups will be changed & $add:The groups that will be added & $remove:The groups that will be removed 'Collation::factory':Called if $wgCategoryCollation is an unknown collation. $collationName:Name of the collation in question & $collationObject:Null. Replace with a subclass of the Collation class that implements the collation given in $collationName. 'ConfirmEmailComplete':Called after a user 's email has been confirmed successfully. $user:user(object) whose email is being confirmed 'ContentAlterParserOutput':Modify parser output for a given content object. Called by Content::getParserOutput after parsing has finished. Can be used for changes that depend on the result of the parsing but have to be done before LinksUpdate is called(such as adding tracking categories based on the rendered HTML). $content:The Content to render $title:Title of the page, as context $parserOutput:ParserOutput to manipulate 'ContentGetParserOutput':Customize parser output for a given content object, called by AbstractContent::getParserOutput. May be used to override the normal model-specific rendering of page content. $content:The Content to render $title:Title of the page, as context $revId:The revision ID, as context $options:ParserOptions for rendering. To avoid confusing the parser cache, the output can only depend on parameters provided to this hook function, not on global state. $generateHtml:boolean, indicating whether full HTML should be generated. If false, generation of HTML may be skipped, but other information should still be present in the ParserOutput object. & $output:ParserOutput, to manipulate or replace 'ContentHandlerDefaultModelFor':Called when the default content model is determined for a given title. May be used to assign a different model for that title. $title:the Title in question & $model:the model name. Use with CONTENT_MODEL_XXX constants. 'ContentHandlerForModelID':Called when a ContentHandler is requested for a given content model name, but no entry for that model exists in $wgContentHandlers. Note:if your extension implements additional models via this hook, please use GetContentModels hook to make them known to core. $modeName:the requested content model name & $handler:set this to a ContentHandler object, if desired. 'ContentModelCanBeUsedOn':Called to determine whether that content model can be used on a given page. This is especially useful to prevent some content models to be used in some special location. $contentModel:ID of the content model in question $title:the Title in question. & $ok:Output parameter, whether it is OK to use $contentModel on $title. Handler functions that modify $ok should generally return false to prevent further hooks from further modifying $ok. 'ContribsPager::getQueryInfo':Before the contributions query is about to run & $pager:Pager object for contributions & $queryInfo:The query for the contribs Pager 'ContribsPager::reallyDoQuery':Called before really executing the query for My Contributions & $data:an array of results of all contribs queries $pager:The ContribsPager object hooked into $offset:Index offset, inclusive $limit:Exact query limit $descending:Query direction, false for ascending, true for descending 'ContributionsLineEnding':Called before a contributions HTML line is finished $page:SpecialPage object for contributions & $ret:the HTML line $row:the DB row for this line & $classes:the classes to add to the surrounding< li > & $attribs:associative array of other HTML attributes for the< li > element. Currently only data attributes reserved to MediaWiki are allowed(see Sanitizer::isReservedDataAttribute). 'ContributionsToolLinks':Change tool links above Special:Contributions $id:User identifier $title:User page title & $tools:Array of tool links $specialPage:SpecialPage instance for context and services. Can be either SpecialContributions or DeletedContributionsPage. Extensions should type hint against a generic SpecialPage though. 'ConvertContent':Called by AbstractContent::convert when a conversion to another content model is requested. Handler functions that modify $result should generally return false to disable further attempts at conversion. $content:The Content object to be converted. $toModel:The ID of the content model to convert to. $lossy:boolean indicating whether lossy conversion is allowed. & $result:Output parameter, in case the handler function wants to provide a converted Content object. Note that $result->getContentModel() must return $toModel. 'ContentSecurityPolicyDefaultSource':Modify the allowed CSP load sources. This affects all directives except for the script directive. If you want to add a script source, see ContentSecurityPolicyScriptSource hook. & $defaultSrc:Array of Content-Security-Policy allowed sources $policyConfig:Current configuration for the Content-Security-Policy header $mode:ContentSecurityPolicy::REPORT_ONLY_MODE or ContentSecurityPolicy::FULL_MODE depending on type of header 'ContentSecurityPolicyDirectives':Modify the content security policy directives. Use this only if ContentSecurityPolicyDefaultSource and ContentSecurityPolicyScriptSource do not meet your needs. & $directives:Array of CSP directives $policyConfig:Current configuration for the CSP header $mode:ContentSecurityPolicy::REPORT_ONLY_MODE or ContentSecurityPolicy::FULL_MODE depending on type of header 'ContentSecurityPolicyScriptSource':Modify the allowed CSP script sources. Note that you also have to use ContentSecurityPolicyDefaultSource if you want non-script sources to be loaded from whatever you add. & $scriptSrc:Array of CSP directives $policyConfig:Current configuration for the CSP header $mode:ContentSecurityPolicy::REPORT_ONLY_MODE or ContentSecurityPolicy::FULL_MODE depending on type of header 'CustomEditor':When invoking the page editor Return true to allow the normal editor to be used, or false if implementing a custom editor, e.g. for a special namespace, etc. $article:Article being edited $user:User performing the edit 'DeletedContribsPager::reallyDoQuery':Called before really executing the query for Special:DeletedContributions Similar to ContribsPager::reallyDoQuery & $data:an array of results of all contribs queries $pager:The DeletedContribsPager object hooked into $offset:Index offset, inclusive $limit:Exact query limit $descending:Query direction, false for ascending, true for descending 'DeletedContributionsLineEnding':Called before a DeletedContributions HTML line is finished. Similar to ContributionsLineEnding $page:SpecialPage object for DeletedContributions & $ret:the HTML line $row:the DB row for this line & $classes:the classes to add to the surrounding< li > & $attribs:associative array of other HTML attributes for the< li > element. Currently only data attributes reserved to MediaWiki are allowed(see Sanitizer::isReservedDataAttribute). 'DeleteUnknownPreferences':Called by the cleanupPreferences.php maintenance script to build a WHERE clause with which to delete preferences that are not known about. This hook is used by extensions that have dynamically-named preferences that should not be deleted in the usual cleanup process. For example, the Gadgets extension creates preferences prefixed with 'gadget-', and so anything with that prefix is excluded from the deletion. &where:An array that will be passed as the $cond parameter to IDatabase::select() to determine what will be deleted from the user_properties table. $db:The IDatabase object, useful for accessing $db->buildLike() etc. 'DifferenceEngineAfterLoadNewText':called in DifferenceEngine::loadNewText() after the new revision 's content has been loaded into the class member variable $differenceEngine->mNewContent but before returning true from this function. $differenceEngine:DifferenceEngine object 'DifferenceEngineLoadTextAfterNewContentIsLoaded':called in DifferenceEngine::loadText() after the new revision 's content has been loaded into the class member variable $differenceEngine->mNewContent but before checking if the variable 's value is null. This hook can be used to inject content into said class member variable. $differenceEngine:DifferenceEngine object 'DifferenceEngineMarkPatrolledLink':Allows extensions to change the "mark as patrolled" link which is shown both on the diff header as well as on the bottom of a page, usually wrapped in a span element which has class="patrollink". $differenceEngine:DifferenceEngine object & $markAsPatrolledLink:The "mark as patrolled" link HTML(string) $rcid:Recent change ID(rc_id) for this change(int) 'DifferenceEngineMarkPatrolledRCID':Allows extensions to possibly change the rcid parameter. For example the rcid might be set to zero due to the user being the same as the performer of the change but an extension might still want to show it under certain conditions. & $rcid:rc_id(int) of the change or 0 $differenceEngine:DifferenceEngine object $change:RecentChange object $user:User object representing the current user 'DifferenceEngineNewHeader':Allows extensions to change the $newHeader variable, which contains information about the new revision, such as the revision 's author, whether the revision was marked as a minor edit or not, etc. $differenceEngine:DifferenceEngine object & $newHeader:The string containing the various #mw-diff-otitle[1-5] divs, which include things like revision author info, revision comment, RevisionDelete link and more $formattedRevisionTools:Array containing revision tools, some of which may have been injected with the DiffRevisionTools hook $nextlink:String containing the link to the next revision(if any) $status
Definition: hooks.txt:1263
doDescribeInternal(array $params)
getMetadata(array $rawHeaders)
getFileHttpUrl(array $params)
resolveContainerPath( $container, $relStoragePath)
const LOCK_UW
Definition: LockManager.php:68
executeOpHandlesInternal(array $fileOpHandles)
Execute a list of FileBackendStoreOpHandle handles in parallel.
getContentType( $storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
static factory( $prefix, $extension='', $tmpDirectory=null)
Make a new temporary file on the file system.
Definition: TempFSFile.php:55
File backend exception for checked exceptions (e.g.
fileExists(array $params)
onError( $status, $func, array $params, $err='', $code=0, $desc='')
Log an unexpected exception for this backend.
array $secureWriteUsers
Additional users (account:user) with write permissions on private containers.
static decode( $value, $assoc=false)
Decodes a JSON string.
Definition: FormatJson.php:174
authTokenHeaders(array $creds)
headersFromParams(array $params)
Get headers to send to Swift when reading a file based on a FileBackend params array, e.g.
deleteContainer( $container, array $params)
Delete a Swift container.
storageUrl(array $creds, $container=null, $object=null)
$params
getCredsCacheKey( $username)
Get the cache key for a container.
callable $obResetFunc
const STREAM_HEADLESS
Definition: StreamFile.php:28
string $swiftAuthUrl
Authentication base URL (without version)
const ATTR_UNICODE_PATHS
doSecureInternal( $fullCont, $dir, array $params)
const ATTR_METADATA
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that When $user is not null
Definition: hooks.txt:780
clearCache(array $paths=null)
newStatus(... $args)
Yields the result of the status wrapper callback on either:
deleteFileCache( $path)
Delete the cached stat info for a file path.
getDirectoryListInternal( $fullCont, $dir, array $params)
$header
int $authTTL
TTL in seconds.
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that When $user is not it can be in the form of< username >< more info > e g for bot passwords intended to be added to log contexts Fields it might only if the login was with a bot password it is not rendered in wiki pages or galleries in category pages allow injecting custom HTML after the section Any uses of the hook need to handle escaping see BaseTemplate::getToolbox and BaseTemplate::makeListItem for details on the format of individual items inside of this array or by returning and letting standard HTTP rendering take place modifiable or by returning false and taking over the output modifiable & $code
Definition: hooks.txt:780
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
string $name
Unique backend name.
Definition: FileBackend.php:94
Base class for all backends using particular storage medium.
getLocalCopy(array $params)
Get a local copy on disk of the file at a storage path in the backend.
doStreamFile(array $params)
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
this hook is for auditing only $req
Definition: hooks.txt:979
this hook is for auditing only or null if authentication failed before getting that far $username
Definition: hooks.txt:780
Iterator for listing regular files.
__construct(array $config)
sanitizeHdrs(array $params)
Sanitize and filter the custom headers from a $params array.
array $readUsers
Additional users (account:user) with read permissions on public containers.
doCopyInternal(array $params)
string $rgwS3AccessKey
S3 access key (RADOS Gateway)
doCreateInternal(array $params)
string $swiftStorageUrl
Override of storage base URL.
doPublishInternal( $fullCont, $dir, array $params)
doPrimeContainerCache(array $containerInfo)
Class for an OpenStack Swift (or Ceph RGW) based file backend.
if(!is_readable( $file)) $ext
Definition: router.php:48
array $writeUsers
Additional users (account:user) with write permissions on public containers.
getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
getMetadataHeaders(array $rawHeaders)
int $authSessionTimestamp
UNIX timestamp.
MultiHttpClient $http
ProcessCacheLRU $containerStatCache
Container stat cache.
doExecuteOpHandlesInternal(array $fileOpHandles)
getFileListInternal( $fullCont, $dir, array $params)
objectListing( $fullCont, $type, $limit, $after=null, $prefix=null, $delim=null)
Get a list of objects under a container.
convertSwiftDate( $ts, $format=TS_MW)
Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT"/"2013-05-11T07:37:27.678360Z". ...
Class to handle multiple HTTP requests.
doGetFileStat(array $params)
string $rgwS3SecretKey
S3 authentication key (RADOS Gateway)
do that in ParserLimitReportFormat instead use this to modify the parameters of the image all existing parser cache entries will be invalid To avoid you ll need to handle that somehow(e.g. with the RejectParserCacheValue hook) because MediaWiki won 't do it for you. & $defaults also a ContextSource after deleting those rows but within the same transaction you ll probably need to make sure the header is varied on $request
Definition: hooks.txt:2633
doCleanInternal( $fullCont, $dir, array $params)
doPrepareInternal( $fullCont, $dir, array $params)
string $swiftKey
Secret key for user.
bool $isRGW
Whether the server is an Ceph RGW.
addMissingHashMetadata(array $objHdrs, $path)
Fill in any missing object metadata and save it to Swift.
isPathUsableInternal( $storagePath)
array $secureReadUsers
Additional users (account:user) with read permissions on private containers.
string $swiftTempUrlKey
Shared secret value for making temp URLs.