MediaWiki REL1_32
SwiftFileBackend.php
Go to the documentation of this file.
1<?php
36 protected $http;
38 protected $authTTL;
40 protected $swiftAuthUrl;
44 protected $swiftUser;
46 protected $swiftKey;
50 protected $rgwS3AccessKey;
52 protected $rgwS3SecretKey;
54 protected $readUsers;
56 protected $writeUsers;
61
63 protected $srvCache;
64
67
69 protected $authCreds;
71 protected $authSessionTimestamp = 0;
73 protected $authErrorTimestamp = null;
74
76 protected $isRGW = false;
77
112 public function __construct( array $config ) {
113 parent::__construct( $config );
114 // Required settings
115 $this->swiftAuthUrl = $config['swiftAuthUrl'];
116 $this->swiftUser = $config['swiftUser'];
117 $this->swiftKey = $config['swiftKey'];
118 // Optional settings
119 $this->authTTL = $config['swiftAuthTTL'] ?? 15 * 60; // some sane number
120 $this->swiftTempUrlKey = $config['swiftTempUrlKey'] ?? '';
121 $this->swiftStorageUrl = $config['swiftStorageUrl'] ?? null;
122 $this->shardViaHashLevels = $config['shardViaHashLevels'] ?? '';
123 $this->rgwS3AccessKey = $config['rgwS3AccessKey'] ?? '';
124 $this->rgwS3SecretKey = $config['rgwS3SecretKey'] ?? '';
125 // HTTP helper client
126 $this->http = new MultiHttpClient( [] );
127 // Cache container information to mask latency
128 if ( isset( $config['wanCache'] ) && $config['wanCache'] instanceof WANObjectCache ) {
129 $this->memCache = $config['wanCache'];
130 }
131 // Process cache for container info
132 $this->containerStatCache = new MapCacheLRU( 300 );
133 // Cache auth token information to avoid RTTs
134 if ( !empty( $config['cacheAuthInfo'] ) && isset( $config['srvCache'] ) ) {
135 $this->srvCache = $config['srvCache'];
136 } else {
137 $this->srvCache = new EmptyBagOStuff();
138 }
139 $this->readUsers = $config['readUsers'] ?? [];
140 $this->writeUsers = $config['writeUsers'] ?? [];
141 $this->secureReadUsers = $config['secureReadUsers'] ?? [];
142 $this->secureWriteUsers = $config['secureWriteUsers'] ?? [];
143 }
144
149
150 protected function resolveContainerPath( $container, $relStoragePath ) {
151 if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) {
152 return null; // not UTF-8, makes it hard to use CF and the swift HTTP API
153 } elseif ( strlen( rawurlencode( $relStoragePath ) ) > 1024 ) {
154 return null; // too long for Swift
155 }
156
157 return $relStoragePath;
158 }
159
160 public function isPathUsableInternal( $storagePath ) {
161 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
162 if ( $rel === null ) {
163 return false; // invalid
164 }
165
166 return is_array( $this->getContainerStat( $container ) );
167 }
168
176 protected function sanitizeHdrsStrict( array $params ) {
177 if ( !isset( $params['headers'] ) ) {
178 return [];
179 }
180
181 $headers = $this->getCustomHeaders( $params['headers'] );
182 unset( $headers[ 'content-type' ] );
183
184 return $headers;
185 }
186
199 protected function sanitizeHdrs( array $params ) {
200 return isset( $params['headers'] )
201 ? $this->getCustomHeaders( $params['headers'] )
202 : [];
203 }
204
209 protected function getCustomHeaders( array $rawHeaders ) {
210 $headers = [];
211
212 // Normalize casing, and strip out illegal headers
213 foreach ( $rawHeaders as $name => $value ) {
214 $name = strtolower( $name );
215 if ( preg_match( '/^content-length$/', $name ) ) {
216 continue; // blacklisted
217 } elseif ( preg_match( '/^(x-)?content-/', $name ) ) {
218 $headers[$name] = $value; // allowed
219 } elseif ( preg_match( '/^content-(disposition)/', $name ) ) {
220 $headers[$name] = $value; // allowed
221 }
222 }
223 // By default, Swift has annoyingly low maximum header value limits
224 if ( isset( $headers['content-disposition'] ) ) {
225 $disposition = '';
226 // @note: assume FileBackend::makeContentDisposition() already used
227 foreach ( explode( ';', $headers['content-disposition'] ) as $part ) {
228 $part = trim( $part );
229 $new = ( $disposition === '' ) ? $part : "{$disposition};{$part}";
230 if ( strlen( $new ) <= 255 ) {
231 $disposition = $new;
232 } else {
233 break; // too long; sigh
234 }
235 }
236 $headers['content-disposition'] = $disposition;
237 }
238
239 return $headers;
240 }
241
246 protected function getMetadataHeaders( array $rawHeaders ) {
247 $headers = [];
248 foreach ( $rawHeaders as $name => $value ) {
249 $name = strtolower( $name );
250 if ( strpos( $name, 'x-object-meta-' ) === 0 ) {
251 $headers[$name] = $value;
252 }
253 }
254
255 return $headers;
256 }
257
262 protected function getMetadata( array $rawHeaders ) {
263 $metadata = [];
264 foreach ( $this->getMetadataHeaders( $rawHeaders ) as $name => $value ) {
265 $metadata[substr( $name, strlen( 'x-object-meta-' ) )] = $value;
266 }
267
268 return $metadata;
269 }
270
271 protected function doCreateInternal( array $params ) {
272 $status = $this->newStatus();
273
274 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
275 if ( $dstRel === null ) {
276 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
277
278 return $status;
279 }
280
281 $sha1Hash = Wikimedia\base_convert( sha1( $params['content'] ), 16, 36, 31 );
282 $contentType = $params['headers']['content-type']
283 ?? $this->getContentType( $params['dst'], $params['content'], null );
284
285 $reqs = [ [
286 'method' => 'PUT',
287 'url' => [ $dstCont, $dstRel ],
288 'headers' => [
289 'content-length' => strlen( $params['content'] ),
290 'etag' => md5( $params['content'] ),
291 'content-type' => $contentType,
292 'x-object-meta-sha1base36' => $sha1Hash
293 ] + $this->sanitizeHdrsStrict( $params ),
294 'body' => $params['content']
295 ] ];
296
297 $method = __METHOD__;
298 $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
299 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
300 if ( $rcode === 201 ) {
301 // good
302 } elseif ( $rcode === 412 ) {
303 $status->fatal( 'backend-fail-contenttype', $params['dst'] );
304 } else {
305 $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
306 }
307 };
308
309 $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
310 if ( !empty( $params['async'] ) ) { // deferred
311 $status->value = $opHandle;
312 } else { // actually write the object in Swift
313 $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
314 }
315
316 return $status;
317 }
318
319 protected function doStoreInternal( array $params ) {
320 $status = $this->newStatus();
321
322 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
323 if ( $dstRel === null ) {
324 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
325
326 return $status;
327 }
328
329 Wikimedia\suppressWarnings();
330 $sha1Hash = sha1_file( $params['src'] );
331 Wikimedia\restoreWarnings();
332 if ( $sha1Hash === false ) { // source doesn't exist?
333 $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
334
335 return $status;
336 }
337 $sha1Hash = Wikimedia\base_convert( $sha1Hash, 16, 36, 31 );
338 $contentType = $params['headers']['content-type']
339 ?? $this->getContentType( $params['dst'], null, $params['src'] );
340
341 $handle = fopen( $params['src'], 'rb' );
342 if ( $handle === false ) { // source doesn't exist?
343 $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
344
345 return $status;
346 }
347
348 $reqs = [ [
349 'method' => 'PUT',
350 'url' => [ $dstCont, $dstRel ],
351 'headers' => [
352 'content-length' => filesize( $params['src'] ),
353 'etag' => md5_file( $params['src'] ),
354 'content-type' => $contentType,
355 'x-object-meta-sha1base36' => $sha1Hash
356 ] + $this->sanitizeHdrsStrict( $params ),
357 'body' => $handle // resource
358 ] ];
359
360 $method = __METHOD__;
361 $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
362 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
363 if ( $rcode === 201 ) {
364 // good
365 } elseif ( $rcode === 412 ) {
366 $status->fatal( 'backend-fail-contenttype', $params['dst'] );
367 } else {
368 $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
369 }
370 };
371
372 $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
373 $opHandle->resourcesToClose[] = $handle;
374
375 if ( !empty( $params['async'] ) ) { // deferred
376 $status->value = $opHandle;
377 } else { // actually write the object in Swift
378 $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
379 }
380
381 return $status;
382 }
383
384 protected function doCopyInternal( array $params ) {
385 $status = $this->newStatus();
386
387 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
388 if ( $srcRel === null ) {
389 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
390
391 return $status;
392 }
393
394 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
395 if ( $dstRel === null ) {
396 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
397
398 return $status;
399 }
400
401 $reqs = [ [
402 'method' => 'PUT',
403 'url' => [ $dstCont, $dstRel ],
404 'headers' => [
405 'x-copy-from' => '/' . rawurlencode( $srcCont ) .
406 '/' . str_replace( "%2F", "/", rawurlencode( $srcRel ) )
407 ] + $this->sanitizeHdrsStrict( $params ), // extra headers merged into object
408 ] ];
409
410 $method = __METHOD__;
411 $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
412 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
413 if ( $rcode === 201 ) {
414 // good
415 } elseif ( $rcode === 404 ) {
416 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
417 } else {
418 $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
419 }
420 };
421
422 $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
423 if ( !empty( $params['async'] ) ) { // deferred
424 $status->value = $opHandle;
425 } else { // actually write the object in Swift
426 $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
427 }
428
429 return $status;
430 }
431
432 protected function doMoveInternal( array $params ) {
433 $status = $this->newStatus();
434
435 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
436 if ( $srcRel === null ) {
437 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
438
439 return $status;
440 }
441
442 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
443 if ( $dstRel === null ) {
444 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
445
446 return $status;
447 }
448
449 $reqs = [
450 [
451 'method' => 'PUT',
452 'url' => [ $dstCont, $dstRel ],
453 'headers' => [
454 'x-copy-from' => '/' . rawurlencode( $srcCont ) .
455 '/' . str_replace( "%2F", "/", rawurlencode( $srcRel ) )
456 ] + $this->sanitizeHdrsStrict( $params ) // extra headers merged into object
457 ]
458 ];
459 if ( "{$srcCont}/{$srcRel}" !== "{$dstCont}/{$dstRel}" ) {
460 $reqs[] = [
461 'method' => 'DELETE',
462 'url' => [ $srcCont, $srcRel ],
463 'headers' => []
464 ];
465 }
466
467 $method = __METHOD__;
468 $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
469 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
470 if ( $request['method'] === 'PUT' && $rcode === 201 ) {
471 // good
472 } elseif ( $request['method'] === 'DELETE' && $rcode === 204 ) {
473 // good
474 } elseif ( $rcode === 404 ) {
475 $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
476 } else {
477 $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
478 }
479 };
480
481 $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
482 if ( !empty( $params['async'] ) ) { // deferred
483 $status->value = $opHandle;
484 } else { // actually move the object in Swift
485 $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
486 }
487
488 return $status;
489 }
490
491 protected function doDeleteInternal( array $params ) {
492 $status = $this->newStatus();
493
494 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
495 if ( $srcRel === null ) {
496 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
497
498 return $status;
499 }
500
501 $reqs = [ [
502 'method' => 'DELETE',
503 'url' => [ $srcCont, $srcRel ],
504 'headers' => []
505 ] ];
506
507 $method = __METHOD__;
508 $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
509 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
510 if ( $rcode === 204 ) {
511 // good
512 } elseif ( $rcode === 404 ) {
513 if ( empty( $params['ignoreMissingSource'] ) ) {
514 $status->fatal( 'backend-fail-delete', $params['src'] );
515 }
516 } else {
517 $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
518 }
519 };
520
521 $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
522 if ( !empty( $params['async'] ) ) { // deferred
523 $status->value = $opHandle;
524 } else { // actually delete the object in Swift
525 $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
526 }
527
528 return $status;
529 }
530
531 protected function doDescribeInternal( array $params ) {
532 $status = $this->newStatus();
533
534 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
535 if ( $srcRel === null ) {
536 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
537
538 return $status;
539 }
540
541 // Fetch the old object headers/metadata...this should be in stat cache by now
542 $stat = $this->getFileStat( [ 'src' => $params['src'], 'latest' => 1 ] );
543 if ( $stat && !isset( $stat['xattr'] ) ) { // older cache entry
544 $stat = $this->doGetFileStat( [ 'src' => $params['src'], 'latest' => 1 ] );
545 }
546 if ( !$stat ) {
547 $status->fatal( 'backend-fail-describe', $params['src'] );
548
549 return $status;
550 }
551
552 // POST clears prior headers, so we need to merge the changes in to the old ones
553 $metaHdrs = [];
554 foreach ( $stat['xattr']['metadata'] as $name => $value ) {
555 $metaHdrs["x-object-meta-$name"] = $value;
556 }
557 $customHdrs = $this->sanitizeHdrs( $params ) + $stat['xattr']['headers'];
558
559 $reqs = [ [
560 'method' => 'POST',
561 'url' => [ $srcCont, $srcRel ],
562 'headers' => $metaHdrs + $customHdrs
563 ] ];
564
565 $method = __METHOD__;
566 $handler = function ( array $request, StatusValue $status ) use ( $method, $params ) {
567 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
568 if ( $rcode === 202 ) {
569 // good
570 } elseif ( $rcode === 404 ) {
571 $status->fatal( 'backend-fail-describe', $params['src'] );
572 } else {
573 $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
574 }
575 };
576
577 $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
578 if ( !empty( $params['async'] ) ) { // deferred
579 $status->value = $opHandle;
580 } else { // actually change the object in Swift
581 $status->merge( current( $this->executeOpHandlesInternal( [ $opHandle ] ) ) );
582 }
583
584 return $status;
585 }
586
587 protected function doPrepareInternal( $fullCont, $dir, array $params ) {
588 $status = $this->newStatus();
589
590 // (a) Check if container already exists
591 $stat = $this->getContainerStat( $fullCont );
592 if ( is_array( $stat ) ) {
593 return $status; // already there
594 } elseif ( $stat === null ) {
595 $status->fatal( 'backend-fail-internal', $this->name );
596 $this->logger->error( __METHOD__ . ': cannot get container stat' );
597
598 return $status;
599 }
600
601 // (b) Create container as needed with proper ACLs
602 if ( $stat === false ) {
603 $params['op'] = 'prepare';
604 $status->merge( $this->createContainer( $fullCont, $params ) );
605 }
606
607 return $status;
608 }
609
610 protected function doSecureInternal( $fullCont, $dir, array $params ) {
611 $status = $this->newStatus();
612 if ( empty( $params['noAccess'] ) ) {
613 return $status; // nothing to do
614 }
615
616 $stat = $this->getContainerStat( $fullCont );
617 if ( is_array( $stat ) ) {
618 $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
619 $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
620 // Make container private to end-users...
621 $status->merge( $this->setContainerAccess(
622 $fullCont,
625 ) );
626 } elseif ( $stat === false ) {
627 $status->fatal( 'backend-fail-usable', $params['dir'] );
628 } else {
629 $status->fatal( 'backend-fail-internal', $this->name );
630 $this->logger->error( __METHOD__ . ': cannot get container stat' );
631 }
632
633 return $status;
634 }
635
636 protected function doPublishInternal( $fullCont, $dir, array $params ) {
637 $status = $this->newStatus();
638
639 $stat = $this->getContainerStat( $fullCont );
640 if ( is_array( $stat ) ) {
641 $readUsers = array_merge( $this->readUsers, [ $this->swiftUser, '.r:*' ] );
642 $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
643
644 // Make container public to end-users...
645 $status->merge( $this->setContainerAccess(
646 $fullCont,
649 ) );
650 } elseif ( $stat === false ) {
651 $status->fatal( 'backend-fail-usable', $params['dir'] );
652 } else {
653 $status->fatal( 'backend-fail-internal', $this->name );
654 $this->logger->error( __METHOD__ . ': cannot get container stat' );
655 }
656
657 return $status;
658 }
659
660 protected function doCleanInternal( $fullCont, $dir, array $params ) {
661 $status = $this->newStatus();
662
663 // Only containers themselves can be removed, all else is virtual
664 if ( $dir != '' ) {
665 return $status; // nothing to do
666 }
667
668 // (a) Check the container
669 $stat = $this->getContainerStat( $fullCont, true );
670 if ( $stat === false ) {
671 return $status; // ok, nothing to do
672 } elseif ( !is_array( $stat ) ) {
673 $status->fatal( 'backend-fail-internal', $this->name );
674 $this->logger->error( __METHOD__ . ': cannot get container stat' );
675
676 return $status;
677 }
678
679 // (b) Delete the container if empty
680 if ( $stat['count'] == 0 ) {
681 $params['op'] = 'clean';
682 $status->merge( $this->deleteContainer( $fullCont, $params ) );
683 }
684
685 return $status;
686 }
687
688 protected function doGetFileStat( array $params ) {
689 $params = [ 'srcs' => [ $params['src'] ], 'concurrency' => 1 ] + $params;
690 unset( $params['src'] );
691 $stats = $this->doGetFileStatMulti( $params );
692
693 return reset( $stats );
694 }
695
706 protected function convertSwiftDate( $ts, $format = TS_MW ) {
707 try {
708 $timestamp = new MWTimestamp( $ts );
709
710 return $timestamp->getTimestamp( $format );
711 } catch ( Exception $e ) {
712 throw new FileBackendError( $e->getMessage() );
713 }
714 }
715
723 protected function addMissingMetadata( array $objHdrs, $path ) {
724 if ( isset( $objHdrs['x-object-meta-sha1base36'] ) ) {
725 return $objHdrs; // nothing to do
726 }
727
729 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
730 $this->logger->error( __METHOD__ . ": {path} was not stored with SHA-1 metadata.",
731 [ 'path' => $path ] );
732
733 $objHdrs['x-object-meta-sha1base36'] = false;
734
735 $auth = $this->getAuthentication();
736 if ( !$auth ) {
737 return $objHdrs; // failed
738 }
739
740 // Find prior custom HTTP headers
741 $postHeaders = $this->getCustomHeaders( $objHdrs );
742 // Find prior metadata headers
743 $postHeaders += $this->getMetadataHeaders( $objHdrs );
744
745 $status = $this->newStatus();
747 $scopeLockS = $this->getScopedFileLocks( [ $path ], LockManager::LOCK_UW, $status );
748 if ( $status->isOK() ) {
749 $tmpFile = $this->getLocalCopy( [ 'src' => $path, 'latest' => 1 ] );
750 if ( $tmpFile ) {
751 $hash = $tmpFile->getSha1Base36();
752 if ( $hash !== false ) {
753 $objHdrs['x-object-meta-sha1base36'] = $hash;
754 // Merge new SHA1 header into the old ones
755 $postHeaders['x-object-meta-sha1base36'] = $hash;
756 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
757 list( $rcode ) = $this->http->run( [
758 'method' => 'POST',
759 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
760 'headers' => $this->authTokenHeaders( $auth ) + $postHeaders
761 ] );
762 if ( $rcode >= 200 && $rcode <= 299 ) {
763 $this->deleteFileCache( $path );
764
765 return $objHdrs; // success
766 }
767 }
768 }
769 }
770
771 $this->logger->error( __METHOD__ . ': unable to set SHA-1 metadata for {path}',
772 [ 'path' => $path ] );
773
774 return $objHdrs; // failed
775 }
776
777 protected function doGetFileContentsMulti( array $params ) {
778 $contents = [];
779
780 $auth = $this->getAuthentication();
781
782 $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging
783 // Blindly create tmp files and stream to them, catching any exception if the file does
784 // not exist. Doing stats here is useless and will loop infinitely in addMissingMetadata().
785 $reqs = []; // (path => op)
786
787 foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch
788 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
789 if ( $srcRel === null || !$auth ) {
790 $contents[$path] = false;
791 continue;
792 }
793 // Create a new temporary memory file...
794 $handle = fopen( 'php://temp', 'wb' );
795 if ( $handle ) {
796 $reqs[$path] = [
797 'method' => 'GET',
798 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
799 'headers' => $this->authTokenHeaders( $auth )
800 + $this->headersFromParams( $params ),
801 'stream' => $handle,
802 ];
803 }
804 $contents[$path] = false;
805 }
806
807 $opts = [ 'maxConnsPerHost' => $params['concurrency'] ];
808 $reqs = $this->http->runMulti( $reqs, $opts );
809 foreach ( $reqs as $path => $op ) {
810 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response'];
811 if ( $rcode >= 200 && $rcode <= 299 ) {
812 rewind( $op['stream'] ); // start from the beginning
813 $contents[$path] = stream_get_contents( $op['stream'] );
814 } elseif ( $rcode === 404 ) {
815 $contents[$path] = false;
816 } else {
817 $this->onError( null, __METHOD__,
818 [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
819 }
820 fclose( $op['stream'] ); // close open handle
821 }
822
823 return $contents;
824 }
825
826 protected function doDirectoryExists( $fullCont, $dir, array $params ) {
827 $prefix = ( $dir == '' ) ? null : "{$dir}/";
828 $status = $this->objectListing( $fullCont, 'names', 1, null, $prefix );
829 if ( $status->isOK() ) {
830 return ( count( $status->value ) ) > 0;
831 }
832
833 return null; // error
834 }
835
843 public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
844 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
845 }
846
854 public function getFileListInternal( $fullCont, $dir, array $params ) {
855 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
856 }
857
869 public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
870 $dirs = [];
871 if ( $after === INF ) {
872 return $dirs; // nothing more
873 }
874
875 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
876
877 $prefix = ( $dir == '' ) ? null : "{$dir}/";
878 // Non-recursive: only list dirs right under $dir
879 if ( !empty( $params['topOnly'] ) ) {
880 $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
881 if ( !$status->isOK() ) {
882 throw new FileBackendError( "Iterator page I/O error." );
883 }
884 $objects = $status->value;
885 foreach ( $objects as $object ) { // files and directories
886 if ( substr( $object, -1 ) === '/' ) {
887 $dirs[] = $object; // directories end in '/'
888 }
889 }
890 } else {
891 // Recursive: list all dirs under $dir and its subdirs
892 $getParentDir = function ( $path ) {
893 return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
894 };
895
896 // Get directory from last item of prior page
897 $lastDir = $getParentDir( $after ); // must be first page
898 $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
899
900 if ( !$status->isOK() ) {
901 throw new FileBackendError( "Iterator page I/O error." );
902 }
903
904 $objects = $status->value;
905
906 foreach ( $objects as $object ) { // files
907 $objectDir = $getParentDir( $object ); // directory of object
908
909 if ( $objectDir !== false && $objectDir !== $dir ) {
910 // Swift stores paths in UTF-8, using binary sorting.
911 // See function "create_container_table" in common/db.py.
912 // If a directory is not "greater" than the last one,
913 // then it was already listed by the calling iterator.
914 if ( strcmp( $objectDir, $lastDir ) > 0 ) {
915 $pDir = $objectDir;
916 do { // add dir and all its parent dirs
917 $dirs[] = "{$pDir}/";
918 $pDir = $getParentDir( $pDir );
919 } while ( $pDir !== false // sanity
920 && strcmp( $pDir, $lastDir ) > 0 // not done already
921 && strlen( $pDir ) > strlen( $dir ) // within $dir
922 );
923 }
924 $lastDir = $objectDir;
925 }
926 }
927 }
928 // Page on the unfiltered directory listing (what is returned may be filtered)
929 if ( count( $objects ) < $limit ) {
930 $after = INF; // avoid a second RTT
931 } else {
932 $after = end( $objects ); // update last item
933 }
934
935 return $dirs;
936 }
937
949 public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
950 $files = []; // list of (path, stat array or null) entries
951 if ( $after === INF ) {
952 return $files; // nothing more
953 }
954
955 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
956
957 $prefix = ( $dir == '' ) ? null : "{$dir}/";
958 // $objects will contain a list of unfiltered names or CF_Object items
959 // Non-recursive: only list files right under $dir
960 if ( !empty( $params['topOnly'] ) ) {
961 if ( !empty( $params['adviseStat'] ) ) {
962 $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix, '/' );
963 } else {
964 $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
965 }
966 } else {
967 // Recursive: list all files under $dir and its subdirs
968 if ( !empty( $params['adviseStat'] ) ) {
969 $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix );
970 } else {
971 $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
972 }
973 }
974
975 // Reformat this list into a list of (name, stat array or null) entries
976 if ( !$status->isOK() ) {
977 throw new FileBackendError( "Iterator page I/O error." );
978 }
979
980 $objects = $status->value;
981 $files = $this->buildFileObjectListing( $params, $dir, $objects );
982
983 // Page on the unfiltered object listing (what is returned may be filtered)
984 if ( count( $objects ) < $limit ) {
985 $after = INF; // avoid a second RTT
986 } else {
987 $after = end( $objects ); // update last item
988 $after = is_object( $after ) ? $after->name : $after;
989 }
990
991 return $files;
992 }
993
1003 private function buildFileObjectListing( array $params, $dir, array $objects ) {
1004 $names = [];
1005 foreach ( $objects as $object ) {
1006 if ( is_object( $object ) ) {
1007 if ( isset( $object->subdir ) || !isset( $object->name ) ) {
1008 continue; // virtual directory entry; ignore
1009 }
1010 $stat = [
1011 // Convert various random Swift dates to TS_MW
1012 'mtime' => $this->convertSwiftDate( $object->last_modified, TS_MW ),
1013 'size' => (int)$object->bytes,
1014 'sha1' => null,
1015 // Note: manifiest ETags are not an MD5 of the file
1016 'md5' => ctype_xdigit( $object->hash ) ? $object->hash : null,
1017 'latest' => false // eventually consistent
1018 ];
1019 $names[] = [ $object->name, $stat ];
1020 } elseif ( substr( $object, -1 ) !== '/' ) {
1021 // Omit directories, which end in '/' in listings
1022 $names[] = [ $object, null ];
1023 }
1024 }
1025
1026 return $names;
1027 }
1028
1035 public function loadListingStatInternal( $path, array $val ) {
1036 $this->cheapCache->setField( $path, 'stat', $val );
1037 }
1038
1039 protected function doGetFileXAttributes( array $params ) {
1040 $stat = $this->getFileStat( $params );
1041 if ( $stat ) {
1042 if ( !isset( $stat['xattr'] ) ) {
1043 // Stat entries filled by file listings don't include metadata/headers
1044 $this->clearCache( [ $params['src'] ] );
1045 $stat = $this->getFileStat( $params );
1046 }
1047
1048 return $stat['xattr'];
1049 } else {
1050 return false;
1051 }
1052 }
1053
1054 protected function doGetFileSha1base36( array $params ) {
1055 $stat = $this->getFileStat( $params );
1056 if ( $stat ) {
1057 if ( !isset( $stat['sha1'] ) ) {
1058 // Stat entries filled by file listings don't include SHA1
1059 $this->clearCache( [ $params['src'] ] );
1060 $stat = $this->getFileStat( $params );
1061 }
1062
1063 return $stat['sha1'];
1064 } else {
1065 return false;
1066 }
1067 }
1068
1069 protected function doStreamFile( array $params ) {
1070 $status = $this->newStatus();
1071
1072 $flags = !empty( $params['headless'] ) ? StreamFile::STREAM_HEADLESS : 0;
1073
1074 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1075 if ( $srcRel === null ) {
1076 StreamFile::send404Message( $params['src'], $flags );
1077 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
1078
1079 return $status;
1080 }
1081
1082 $auth = $this->getAuthentication();
1083 if ( !$auth || !is_array( $this->getContainerStat( $srcCont ) ) ) {
1084 StreamFile::send404Message( $params['src'], $flags );
1085 $status->fatal( 'backend-fail-stream', $params['src'] );
1086
1087 return $status;
1088 }
1089
1090 // If "headers" is set, we only want to send them if the file is there.
1091 // Do not bother checking if the file exists if headers are not set though.
1092 if ( $params['headers'] && !$this->fileExists( $params ) ) {
1093 StreamFile::send404Message( $params['src'], $flags );
1094 $status->fatal( 'backend-fail-stream', $params['src'] );
1095
1096 return $status;
1097 }
1098
1099 // Send the requested additional headers
1100 foreach ( $params['headers'] as $header ) {
1101 header( $header ); // aways send
1102 }
1103
1104 if ( empty( $params['allowOB'] ) ) {
1105 // Cancel output buffering and gzipping if set
1106 ( $this->obResetFunc )();
1107 }
1108
1109 $handle = fopen( 'php://output', 'wb' );
1110 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1111 'method' => 'GET',
1112 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1113 'headers' => $this->authTokenHeaders( $auth )
1114 + $this->headersFromParams( $params ) + $params['options'],
1115 'stream' => $handle,
1116 'flags' => [ 'relayResponseHeaders' => empty( $params['headless'] ) ]
1117 ] );
1118
1119 if ( $rcode >= 200 && $rcode <= 299 ) {
1120 // good
1121 } elseif ( $rcode === 404 ) {
1122 $status->fatal( 'backend-fail-stream', $params['src'] );
1123 // Per T43113, nasty things can happen if bad cache entries get
1124 // stuck in cache. It's also possible that this error can come up
1125 // with simple race conditions. Clear out the stat cache to be safe.
1126 $this->clearCache( [ $params['src'] ] );
1127 $this->deleteFileCache( $params['src'] );
1128 } else {
1129 $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1130 }
1131
1132 return $status;
1133 }
1134
1135 protected function doGetLocalCopyMulti( array $params ) {
1137 $tmpFiles = [];
1138
1139 $auth = $this->getAuthentication();
1140
1141 $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging
1142 // Blindly create tmp files and stream to them, catching any exception if the file does
1143 // not exist. Doing a stat here is useless causes infinite loops in addMissingMetadata().
1144 $reqs = []; // (path => op)
1145
1146 foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch
1147 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
1148 if ( $srcRel === null || !$auth ) {
1149 $tmpFiles[$path] = null;
1150 continue;
1151 }
1152 // Get source file extension
1154 // Create a new temporary file...
1155 $tmpFile = TempFSFile::factory( 'localcopy_', $ext, $this->tmpDirectory );
1156 if ( $tmpFile ) {
1157 $handle = fopen( $tmpFile->getPath(), 'wb' );
1158 if ( $handle ) {
1159 $reqs[$path] = [
1160 'method' => 'GET',
1161 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1162 'headers' => $this->authTokenHeaders( $auth )
1163 + $this->headersFromParams( $params ),
1164 'stream' => $handle,
1165 ];
1166 } else {
1167 $tmpFile = null;
1168 }
1169 }
1170 $tmpFiles[$path] = $tmpFile;
1171 }
1172
1173 $isLatest = ( $this->isRGW || !empty( $params['latest'] ) );
1174 $opts = [ 'maxConnsPerHost' => $params['concurrency'] ];
1175 $reqs = $this->http->runMulti( $reqs, $opts );
1176 foreach ( $reqs as $path => $op ) {
1177 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response'];
1178 fclose( $op['stream'] ); // close open handle
1179 if ( $rcode >= 200 && $rcode <= 299 ) {
1180 $size = $tmpFiles[$path] ? $tmpFiles[$path]->getSize() : 0;
1181 // Double check that the disk is not full/broken
1182 if ( $size != $rhdrs['content-length'] ) {
1183 $tmpFiles[$path] = null;
1184 $rerr = "Got {$size}/{$rhdrs['content-length']} bytes";
1185 $this->onError( null, __METHOD__,
1186 [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
1187 }
1188 // Set the file stat process cache in passing
1189 $stat = $this->getStatFromHeaders( $rhdrs );
1190 $stat['latest'] = $isLatest;
1191 $this->cheapCache->setField( $path, 'stat', $stat );
1192 } elseif ( $rcode === 404 ) {
1193 $tmpFiles[$path] = false;
1194 } else {
1195 $tmpFiles[$path] = null;
1196 $this->onError( null, __METHOD__,
1197 [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
1198 }
1199 }
1200
1201 return $tmpFiles;
1202 }
1203
1204 public function getFileHttpUrl( array $params ) {
1205 if ( $this->swiftTempUrlKey != '' ||
1206 ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' )
1207 ) {
1208 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1209 if ( $srcRel === null ) {
1210 return null; // invalid path
1211 }
1212
1213 $auth = $this->getAuthentication();
1214 if ( !$auth ) {
1215 return null;
1216 }
1217
1218 $ttl = $params['ttl'] ?? 86400;
1219 $expires = time() + $ttl;
1220
1221 if ( $this->swiftTempUrlKey != '' ) {
1222 $url = $this->storageUrl( $auth, $srcCont, $srcRel );
1223 // Swift wants the signature based on the unencoded object name
1224 $contPath = parse_url( $this->storageUrl( $auth, $srcCont ), PHP_URL_PATH );
1225 $signature = hash_hmac( 'sha1',
1226 "GET\n{$expires}\n{$contPath}/{$srcRel}",
1227 $this->swiftTempUrlKey
1228 );
1229
1230 return "{$url}?temp_url_sig={$signature}&temp_url_expires={$expires}";
1231 } else { // give S3 API URL for rgw
1232 // Path for signature starts with the bucket
1233 $spath = '/' . rawurlencode( $srcCont ) . '/' .
1234 str_replace( '%2F', '/', rawurlencode( $srcRel ) );
1235 // Calculate the hash
1236 $signature = base64_encode( hash_hmac(
1237 'sha1',
1238 "GET\n\n\n{$expires}\n{$spath}",
1239 $this->rgwS3SecretKey,
1240 true // raw
1241 ) );
1242 // See https://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html.
1243 // Note: adding a newline for empty CanonicalizedAmzHeaders does not work.
1244 // Note: S3 API is the rgw default; remove the /swift/ URL bit.
1245 return str_replace( '/swift/v1', '', $this->storageUrl( $auth ) . $spath ) .
1246 '?' .
1247 http_build_query( [
1248 'Signature' => $signature,
1249 'Expires' => $expires,
1250 'AWSAccessKeyId' => $this->rgwS3AccessKey
1251 ] );
1252 }
1253 }
1254
1255 return null;
1256 }
1257
1258 protected function directoriesAreVirtual() {
1259 return true;
1260 }
1261
1270 protected function headersFromParams( array $params ) {
1271 $hdrs = [];
1272 if ( !empty( $params['latest'] ) ) {
1273 $hdrs['x-newest'] = 'true';
1274 }
1275
1276 return $hdrs;
1277 }
1278
1284 protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1286 $statuses = [];
1287
1288 $auth = $this->getAuthentication();
1289 if ( !$auth ) {
1290 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1291 $statuses[$index] = $this->newStatus( 'backend-fail-connect', $this->name );
1292 }
1293
1294 return $statuses;
1295 }
1296
1297 // Split the HTTP requests into stages that can be done concurrently
1298 $httpReqsByStage = []; // map of (stage => index => HTTP request)
1299 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1301 $reqs = $fileOpHandle->httpOp;
1302 // Convert the 'url' parameter to an actual URL using $auth
1303 foreach ( $reqs as $stage => &$req ) {
1304 list( $container, $relPath ) = $req['url'];
1305 $req['url'] = $this->storageUrl( $auth, $container, $relPath );
1306 $req['headers'] = $req['headers'] ?? [];
1307 $req['headers'] = $this->authTokenHeaders( $auth ) + $req['headers'];
1308 $httpReqsByStage[$stage][$index] = $req;
1309 }
1310 $statuses[$index] = $this->newStatus();
1311 }
1312
1313 // Run all requests for the first stage, then the next, and so on
1314 $reqCount = count( $httpReqsByStage );
1315 for ( $stage = 0; $stage < $reqCount; ++$stage ) {
1316 $httpReqs = $this->http->runMulti( $httpReqsByStage[$stage] );
1317 foreach ( $httpReqs as $index => $httpReq ) {
1318 // Run the callback for each request of this operation
1319 $callback = $fileOpHandles[$index]->callback;
1320 $callback( $httpReq, $statuses[$index] );
1321 // On failure, abort all remaining requests for this operation
1322 // (e.g. abort the DELETE request if the COPY request fails for a move)
1323 if ( !$statuses[$index]->isOK() ) {
1324 $stages = count( $fileOpHandles[$index]->httpOp );
1325 for ( $s = ( $stage + 1 ); $s < $stages; ++$s ) {
1326 unset( $httpReqsByStage[$s][$index] );
1327 }
1328 }
1329 }
1330 }
1331
1332 return $statuses;
1333 }
1334
1357 protected function setContainerAccess( $container, array $readUsers, array $writeUsers ) {
1358 $status = $this->newStatus();
1359 $auth = $this->getAuthentication();
1360
1361 if ( !$auth ) {
1362 $status->fatal( 'backend-fail-connect', $this->name );
1363
1364 return $status;
1365 }
1366
1367 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1368 'method' => 'POST',
1369 'url' => $this->storageUrl( $auth, $container ),
1370 'headers' => $this->authTokenHeaders( $auth ) + [
1371 'x-container-read' => implode( ',', $readUsers ),
1372 'x-container-write' => implode( ',', $writeUsers )
1373 ]
1374 ] );
1375
1376 if ( $rcode != 204 && $rcode !== 202 ) {
1377 $status->fatal( 'backend-fail-internal', $this->name );
1378 $this->logger->error( __METHOD__ . ': unexpected rcode value ({rcode})',
1379 [ 'rcode' => $rcode ] );
1380 }
1381
1382 return $status;
1383 }
1384
1393 protected function getContainerStat( $container, $bypassCache = false ) {
1394 $ps = $this->scopedProfileSection( __METHOD__ . "-{$this->name}" );
1395
1396 if ( $bypassCache ) { // purge cache
1397 $this->containerStatCache->clear( $container );
1398 } elseif ( !$this->containerStatCache->hasField( $container, 'stat' ) ) {
1399 $this->primeContainerCache( [ $container ] ); // check persistent cache
1400 }
1401 if ( !$this->containerStatCache->hasField( $container, 'stat' ) ) {
1402 $auth = $this->getAuthentication();
1403 if ( !$auth ) {
1404 return null;
1405 }
1406
1407 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1408 'method' => 'HEAD',
1409 'url' => $this->storageUrl( $auth, $container ),
1410 'headers' => $this->authTokenHeaders( $auth )
1411 ] );
1412
1413 if ( $rcode === 204 ) {
1414 $stat = [
1415 'count' => $rhdrs['x-container-object-count'],
1416 'bytes' => $rhdrs['x-container-bytes-used']
1417 ];
1418 if ( $bypassCache ) {
1419 return $stat;
1420 } else {
1421 $this->containerStatCache->setField( $container, 'stat', $stat ); // cache it
1422 $this->setContainerCache( $container, $stat ); // update persistent cache
1423 }
1424 } elseif ( $rcode === 404 ) {
1425 return false;
1426 } else {
1427 $this->onError( null, __METHOD__,
1428 [ 'cont' => $container ], $rerr, $rcode, $rdesc );
1429
1430 return null;
1431 }
1432 }
1433
1434 return $this->containerStatCache->getField( $container, 'stat' );
1435 }
1436
1444 protected function createContainer( $container, array $params ) {
1445 $status = $this->newStatus();
1446
1447 $auth = $this->getAuthentication();
1448 if ( !$auth ) {
1449 $status->fatal( 'backend-fail-connect', $this->name );
1450
1451 return $status;
1452 }
1453
1454 // @see SwiftFileBackend::setContainerAccess()
1455 if ( empty( $params['noAccess'] ) ) {
1456 // public
1457 $readUsers = array_merge( $this->readUsers, [ '.r:*', $this->swiftUser ] );
1458 $writeUsers = array_merge( $this->writeUsers, [ $this->swiftUser ] );
1459 } else {
1460 // private
1461 $readUsers = array_merge( $this->secureReadUsers, [ $this->swiftUser ] );
1462 $writeUsers = array_merge( $this->secureWriteUsers, [ $this->swiftUser ] );
1463 }
1464
1465 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1466 'method' => 'PUT',
1467 'url' => $this->storageUrl( $auth, $container ),
1468 'headers' => $this->authTokenHeaders( $auth ) + [
1469 'x-container-read' => implode( ',', $readUsers ),
1470 'x-container-write' => implode( ',', $writeUsers )
1471 ]
1472 ] );
1473
1474 if ( $rcode === 201 ) { // new
1475 // good
1476 } elseif ( $rcode === 202 ) { // already there
1477 // this shouldn't really happen, but is OK
1478 } else {
1479 $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1480 }
1481
1482 return $status;
1483 }
1484
1492 protected function deleteContainer( $container, array $params ) {
1493 $status = $this->newStatus();
1494
1495 $auth = $this->getAuthentication();
1496 if ( !$auth ) {
1497 $status->fatal( 'backend-fail-connect', $this->name );
1498
1499 return $status;
1500 }
1501
1502 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1503 'method' => 'DELETE',
1504 'url' => $this->storageUrl( $auth, $container ),
1505 'headers' => $this->authTokenHeaders( $auth )
1506 ] );
1507
1508 if ( $rcode >= 200 && $rcode <= 299 ) { // deleted
1509 $this->containerStatCache->clear( $container ); // purge
1510 } elseif ( $rcode === 404 ) { // not there
1511 // this shouldn't really happen, but is OK
1512 } elseif ( $rcode === 409 ) { // not empty
1513 $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); // race?
1514 } else {
1515 $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1516 }
1517
1518 return $status;
1519 }
1520
1533 private function objectListing(
1534 $fullCont, $type, $limit, $after = null, $prefix = null, $delim = null
1535 ) {
1536 $status = $this->newStatus();
1537
1538 $auth = $this->getAuthentication();
1539 if ( !$auth ) {
1540 $status->fatal( 'backend-fail-connect', $this->name );
1541
1542 return $status;
1543 }
1544
1545 $query = [ 'limit' => $limit ];
1546 if ( $type === 'info' ) {
1547 $query['format'] = 'json';
1548 }
1549 if ( $after !== null ) {
1550 $query['marker'] = $after;
1551 }
1552 if ( $prefix !== null ) {
1553 $query['prefix'] = $prefix;
1554 }
1555 if ( $delim !== null ) {
1556 $query['delimiter'] = $delim;
1557 }
1558
1559 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1560 'method' => 'GET',
1561 'url' => $this->storageUrl( $auth, $fullCont ),
1562 'query' => $query,
1563 'headers' => $this->authTokenHeaders( $auth )
1564 ] );
1565
1566 $params = [ 'cont' => $fullCont, 'prefix' => $prefix, 'delim' => $delim ];
1567 if ( $rcode === 200 ) { // good
1568 if ( $type === 'info' ) {
1569 $status->value = FormatJson::decode( trim( $rbody ) );
1570 } else {
1571 $status->value = explode( "\n", trim( $rbody ) );
1572 }
1573 } elseif ( $rcode === 204 ) {
1574 $status->value = []; // empty container
1575 } elseif ( $rcode === 404 ) {
1576 $status->value = []; // no container
1577 } else {
1578 $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1579 }
1580
1581 return $status;
1582 }
1583
1584 protected function doPrimeContainerCache( array $containerInfo ) {
1585 foreach ( $containerInfo as $container => $info ) {
1586 $this->containerStatCache->setField( $container, 'stat', $info );
1587 }
1588 }
1589
1590 protected function doGetFileStatMulti( array $params ) {
1591 $stats = [];
1592
1593 $auth = $this->getAuthentication();
1594
1595 $reqs = [];
1596 foreach ( $params['srcs'] as $path ) {
1597 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
1598 if ( $srcRel === null ) {
1599 $stats[$path] = false;
1600 continue; // invalid storage path
1601 } elseif ( !$auth ) {
1602 $stats[$path] = null;
1603 continue;
1604 }
1605
1606 // (a) Check the container
1607 $cstat = $this->getContainerStat( $srcCont );
1608 if ( $cstat === false ) {
1609 $stats[$path] = false;
1610 continue; // ok, nothing to do
1611 } elseif ( !is_array( $cstat ) ) {
1612 $stats[$path] = null;
1613 continue;
1614 }
1615
1616 $reqs[$path] = [
1617 'method' => 'HEAD',
1618 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
1619 'headers' => $this->authTokenHeaders( $auth ) + $this->headersFromParams( $params )
1620 ];
1621 }
1622
1623 $opts = [ 'maxConnsPerHost' => $params['concurrency'] ];
1624 $reqs = $this->http->runMulti( $reqs, $opts );
1625
1626 foreach ( $params['srcs'] as $path ) {
1627 if ( array_key_exists( $path, $stats ) ) {
1628 continue; // some sort of failure above
1629 }
1630 // (b) Check the file
1631 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $reqs[$path]['response'];
1632 if ( $rcode === 200 || $rcode === 204 ) {
1633 // Update the object if it is missing some headers
1634 $rhdrs = $this->addMissingMetadata( $rhdrs, $path );
1635 // Load the stat array from the headers
1636 $stat = $this->getStatFromHeaders( $rhdrs );
1637 if ( $this->isRGW ) {
1638 $stat['latest'] = true; // strong consistency
1639 }
1640 } elseif ( $rcode === 404 ) {
1641 $stat = false;
1642 } else {
1643 $stat = null;
1644 $this->onError( null, __METHOD__, $params, $rerr, $rcode, $rdesc );
1645 }
1646 $stats[$path] = $stat;
1647 }
1648
1649 return $stats;
1650 }
1651
1656 protected function getStatFromHeaders( array $rhdrs ) {
1657 // Fetch all of the custom metadata headers
1658 $metadata = $this->getMetadata( $rhdrs );
1659 // Fetch all of the custom raw HTTP headers
1660 $headers = $this->sanitizeHdrs( [ 'headers' => $rhdrs ] );
1661
1662 return [
1663 // Convert various random Swift dates to TS_MW
1664 'mtime' => $this->convertSwiftDate( $rhdrs['last-modified'], TS_MW ),
1665 // Empty objects actually return no content-length header in Ceph
1666 'size' => isset( $rhdrs['content-length'] ) ? (int)$rhdrs['content-length'] : 0,
1667 'sha1' => $metadata['sha1base36'] ?? null,
1668 // Note: manifiest ETags are not an MD5 of the file
1669 'md5' => ctype_xdigit( $rhdrs['etag'] ) ? $rhdrs['etag'] : null,
1670 'xattr' => [ 'metadata' => $metadata, 'headers' => $headers ]
1671 ];
1672 }
1673
1677 protected function getAuthentication() {
1678 if ( $this->authErrorTimestamp !== null ) {
1679 if ( ( time() - $this->authErrorTimestamp ) < 60 ) {
1680 return null; // failed last attempt; don't bother
1681 } else { // actually retry this time
1682 $this->authErrorTimestamp = null;
1683 }
1684 }
1685 // Session keys expire after a while, so we renew them periodically
1686 $reAuth = ( ( time() - $this->authSessionTimestamp ) > $this->authTTL );
1687 // Authenticate with proxy and get a session key...
1688 if ( !$this->authCreds || $reAuth ) {
1689 $this->authSessionTimestamp = 0;
1690 $cacheKey = $this->getCredsCacheKey( $this->swiftUser );
1691 $creds = $this->srvCache->get( $cacheKey ); // credentials
1692 // Try to use the credential cache
1693 if ( isset( $creds['auth_token'] ) && isset( $creds['storage_url'] ) ) {
1694 $this->authCreds = $creds;
1695 // Skew the timestamp for worst case to avoid using stale credentials
1696 $this->authSessionTimestamp = time() - ceil( $this->authTTL / 2 );
1697 } else { // cache miss
1698 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [
1699 'method' => 'GET',
1700 'url' => "{$this->swiftAuthUrl}/v1.0",
1701 'headers' => [
1702 'x-auth-user' => $this->swiftUser,
1703 'x-auth-key' => $this->swiftKey
1704 ]
1705 ] );
1706
1707 if ( $rcode >= 200 && $rcode <= 299 ) { // OK
1708 $this->authCreds = [
1709 'auth_token' => $rhdrs['x-auth-token'],
1710 'storage_url' => ( $this->swiftStorageUrl !== null )
1711 ? $this->swiftStorageUrl
1712 : $rhdrs['x-storage-url']
1713 ];
1714
1715 $this->srvCache->set( $cacheKey, $this->authCreds, ceil( $this->authTTL / 2 ) );
1716 $this->authSessionTimestamp = time();
1717 } elseif ( $rcode === 401 ) {
1718 $this->onError( null, __METHOD__, [], "Authentication failed.", $rcode );
1719 $this->authErrorTimestamp = time();
1720
1721 return null;
1722 } else {
1723 $this->onError( null, __METHOD__, [], "HTTP return code: $rcode", $rcode );
1724 $this->authErrorTimestamp = time();
1725
1726 return null;
1727 }
1728 }
1729 // Ceph RGW does not use <account> in URLs (OpenStack Swift uses "/v1/<account>")
1730 if ( substr( $this->authCreds['storage_url'], -3 ) === '/v1' ) {
1731 $this->isRGW = true; // take advantage of strong consistency in Ceph
1732 }
1733 }
1734
1735 return $this->authCreds;
1736 }
1737
1744 protected function storageUrl( array $creds, $container = null, $object = null ) {
1745 $parts = [ $creds['storage_url'] ];
1746 if ( strlen( $container ) ) {
1747 $parts[] = rawurlencode( $container );
1748 }
1749 if ( strlen( $object ) ) {
1750 $parts[] = str_replace( "%2F", "/", rawurlencode( $object ) );
1751 }
1752
1753 return implode( '/', $parts );
1754 }
1755
1760 protected function authTokenHeaders( array $creds ) {
1761 return [ 'x-auth-token' => $creds['auth_token'] ];
1762 }
1763
1770 private function getCredsCacheKey( $username ) {
1771 return 'swiftcredentials:' . md5( $username . ':' . $this->swiftAuthUrl );
1772 }
1773
1785 public function onError( $status, $func, array $params, $err = '', $code = 0, $desc = '' ) {
1786 if ( $status instanceof StatusValue ) {
1787 $status->fatal( 'backend-fail-internal', $this->name );
1788 }
1789 if ( $code == 401 ) { // possibly a stale token
1790 $this->srvCache->delete( $this->getCredsCacheKey( $this->swiftUser ) );
1791 }
1792 $msg = "HTTP {code} ({desc}) in '{func}' (given '{req_params}')";
1793 $msgParams = [
1794 'code' => $code,
1795 'desc' => $desc,
1796 'func' => $func,
1797 'req_params' => FormatJson::encode( $params ),
1798 ];
1799 if ( $err ) {
1800 $msg .= ': {err}';
1801 $msgParams['err'] = $err;
1802 }
1803 $this->logger->error( $msg, $msgParams );
1804 }
1805}
1806
1812 public $httpOp;
1815
1822 $this->backend = $backend;
1823 $this->callback = $callback;
1824 $this->httpOp = $httpOp;
1825 }
1826}
1827
1835abstract class SwiftFileBackendList implements Iterator {
1837 protected $bufferIter = [];
1838
1840 protected $bufferAfter = null;
1841
1843 protected $pos = 0;
1844
1846 protected $params = [];
1847
1849 protected $backend;
1850
1852 protected $container;
1853
1855 protected $dir;
1856
1858 protected $suffixStart;
1859
1860 const PAGE_SIZE = 9000; // file listing buffer size
1861
1868 public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
1869 $this->backend = $backend;
1870 $this->container = $fullCont;
1871 $this->dir = $dir;
1872 if ( substr( $this->dir, -1 ) === '/' ) {
1873 $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
1874 }
1875 if ( $this->dir == '' ) { // whole container
1876 $this->suffixStart = 0;
1877 } else { // dir within container
1878 $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
1879 }
1880 $this->params = $params;
1881 }
1882
1887 public function key() {
1888 return $this->pos;
1889 }
1890
1894 public function next() {
1895 // Advance to the next file in the page
1896 next( $this->bufferIter );
1897 ++$this->pos;
1898 // Check if there are no files left in this page and
1899 // advance to the next page if this page was not empty.
1900 if ( !$this->valid() && count( $this->bufferIter ) ) {
1901 $this->bufferIter = $this->pageFromList(
1902 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1903 ); // updates $this->bufferAfter
1904 }
1905 }
1906
1910 public function rewind() {
1911 $this->pos = 0;
1912 $this->bufferAfter = null;
1913 $this->bufferIter = $this->pageFromList(
1914 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1915 ); // updates $this->bufferAfter
1916 }
1917
1922 public function valid() {
1923 if ( $this->bufferIter === null ) {
1924 return false; // some failure?
1925 } else {
1926 return ( current( $this->bufferIter ) !== false ); // no paths can have this value
1927 }
1928 }
1929
1940 abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1941}
1942
1951 public function current() {
1952 return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
1953 }
1954
1955 protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1956 return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1957 }
1958}
1959
1968 public function current() {
1969 list( $path, $stat ) = current( $this->bufferIter );
1970 $relPath = substr( $path, $this->suffixStart );
1971 if ( is_array( $stat ) ) {
1972 $storageDir = rtrim( $this->params['dir'], '/' );
1973 $this->backend->loadListingStatInternal( "$storageDir/$relPath", $stat );
1974 }
1975
1976 return $relPath;
1977 }
1978
1979 protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1980 return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
1981 }
1982}
Apache License January http
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
Class representing a cache/ephemeral data store.
Definition BagOStuff.php:58
A BagOStuff object with no objects in it.
File backend exception for checked exceptions (e.g.
FileBackendStore helper class for performing asynchronous file operations.
Base class for all backends using particular storage medium.
setContainerCache( $container, array $val)
Set the cached info for a container.
executeOpHandlesInternal(array $fileOpHandles)
Execute a list of FileBackendStoreOpHandle handles in parallel.
getFileStat(array $params)
Get quick information about a file at a storage path in the backend.
resolveStoragePathReal( $storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
clearCache(array $paths=null)
Invalidate any in-process file stat and property cache.
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
deleteFileCache( $path)
Delete the cached stat info for a file path.
getContentType( $storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
fileExists(array $params)
Check if a file exists at a storage path in the backend.
getLocalCopy(array $params)
Get a local copy on disk of the file at a storage path in the backend.
string $name
Unique backend name.
const ATTR_UNICODE_PATHS
callable $obResetFunc
static extensionFromPath( $path, $case='lowercase')
Get the final extension from a storage or FS path.
getScopedFileLocks(array $paths, $type, StatusValue $status, $timeout=0)
Lock the files at the given storage paths in the backend.
newStatus()
Yields the result of the status wrapper callback on either:
scopedProfileSection( $section)
const ATTR_METADATA
const ATTR_HEADERS
Bitfield flags for supported features.
Library for creating and parsing MW-style timestamps.
Handles a simple LRU key/value map with a maximum number of entries.
Class to handle multiple HTTP requests.
Class for process caching individual properties of expiring items.
Generic operation result class Has warning/error list, boolean status and arbitrary value.
const STREAM_HEADLESS
static send404Message( $fname, $flags=0)
Send out a standard 404 message for a file.
Iterator for listing directories.
pageFromList( $container, $dir, &$after, $limit, array $params)
Get the given list portion (page)
Iterator for listing regular files.
pageFromList( $container, $dir, &$after, $limit, array $params)
Get the given list portion (page)
SwiftFileBackend helper class to page through listings.
string $dir
Storage directory.
string $container
Container name.
__construct(SwiftFileBackend $backend, $fullCont, $dir, array $params)
string $bufferAfter
List items after this path.
pageFromList( $container, $dir, &$after, $limit, array $params)
Get the given list portion (page)
array $bufferIter
List of path or (path,stat array) entries.
SwiftFileBackend $backend
Class for an OpenStack Swift (or Ceph RGW) based file backend.
string $swiftUser
Swift user (account:user) to authenticate as.
sanitizeHdrs(array $params)
Sanitize and filter the custom headers from a $params array.
string $swiftAuthUrl
Authentication base URL (without version)
string $swiftTempUrlKey
Shared secret value for making temp URLs.
isPathUsableInternal( $storagePath)
Check if a file can be created or changed at a given storage path.
getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
doPublishInternal( $fullCont, $dir, array $params)
doCreateInternal(array $params)
doGetFileStatMulti(array $params)
Get file stat information (concurrently if possible) for several files.
doGetFileSha1base36(array $params)
array $writeUsers
Additional users (account:user) with write permissions on public containers.
__construct(array $config)
MultiHttpClient $http
doGetFileXAttributes(array $params)
onError( $status, $func, array $params, $err='', $code=0, $desc='')
Log an unexpected exception for this backend.
buildFileObjectListing(array $params, $dir, array $objects)
Build a list of file objects, filtering out any directories and extracting any stat info if provided ...
authTokenHeaders(array $creds)
getStatFromHeaders(array $rhdrs)
string $swiftStorageUrl
Override of storage base URL.
createContainer( $container, array $params)
Create a Swift container.
doCopyInternal(array $params)
getCredsCacheKey( $username)
Get the cache key for a container.
getDirectoryListInternal( $fullCont, $dir, array $params)
string $rgwS3AccessKey
S3 access key (RADOS Gateway)
setContainerAccess( $container, array $readUsers, array $writeUsers)
Set read/write permissions for a Swift container.
getFileHttpUrl(array $params)
array $secureWriteUsers
Additional users (account:user) with write permissions on private containers.
getCustomHeaders(array $rawHeaders)
int $authTTL
TTL in seconds.
headersFromParams(array $params)
Get headers to send to Swift when reading a file based on a FileBackend params array,...
getMetadata(array $rawHeaders)
bool $isRGW
Whether the server is an Ceph RGW.
addMissingMetadata(array $objHdrs, $path)
Fill in any missing object metadata and save it to Swift.
doStoreInternal(array $params)
int $authErrorTimestamp
UNIX timestamp.
getMetadataHeaders(array $rawHeaders)
int $authSessionTimestamp
UNIX timestamp.
loadListingStatInternal( $path, array $val)
Do not call this function outside of SwiftFileBackendFileList.
doPrepareInternal( $fullCont, $dir, array $params)
doSecureInternal( $fullCont, $dir, array $params)
getFileListInternal( $fullCont, $dir, array $params)
doMoveInternal(array $params)
getFeatures()
Get the a bitfield of extra features supported by the backend medium.
deleteContainer( $container, array $params)
Delete a Swift container.
doGetFileStat(array $params)
doGetLocalCopyMulti(array $params)
string $rgwS3SecretKey
S3 authentication key (RADOS Gateway)
doGetFileContentsMulti(array $params)
storageUrl(array $creds, $container=null, $object=null)
convertSwiftDate( $ts, $format=TS_MW)
Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT"/"2013-05-11T07:37:27.678360Z".
doStreamFile(array $params)
doPrimeContainerCache(array $containerInfo)
Fill the backend-specific process cache given an array of resolved container names and their correspo...
sanitizeHdrsStrict(array $params)
Sanitize and filter the custom headers from a $params array.
resolveContainerPath( $container, $relStoragePath)
Resolve a relative storage path, checking if it's allowed by the backend.
array $readUsers
Additional users (account:user) with read permissions on public containers.
array $secureReadUsers
Additional users (account:user) with read permissions on private containers.
doCleanInternal( $fullCont, $dir, array $params)
ProcessCacheLRU $containerStatCache
Container stat cache.
getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
string $swiftKey
Secret key for user.
doDirectoryExists( $fullCont, $dir, array $params)
objectListing( $fullCont, $type, $limit, $after=null, $prefix=null, $delim=null)
Get a list of objects under a container.
directoriesAreVirtual()
Is this a key/value store where directories are just virtual? Virtual directories exists in so much a...
doExecuteOpHandlesInternal(array $fileOpHandles)
doDeleteInternal(array $params)
doDescribeInternal(array $params)
getContainerStat( $container, $bypassCache=false)
Get a Swift container stat array, possibly from process cache.
array $httpOp
List of Requests for MultiHttpClient.
__construct(SwiftFileBackend $backend, Closure $callback, array $httpOp)
static factory( $prefix, $extension='', $tmpDirectory=null)
Make a new temporary file on the file system.
Multi-datacenter aware caching interface.
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition deferred.txt:11
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
this hook is for auditing only $req
Definition hooks.txt:1018
do that in ParserLimitReportFormat instead use this to modify the parameters of the image all existing parser cache entries will be invalid To avoid you ll need to handle that somehow(e.g. with the RejectParserCacheValue hook) because MediaWiki won 't do it for you. & $defaults also a ContextSource after deleting those rows but within the same transaction you ll probably need to make sure the header is varied on $request
Definition hooks.txt:2880
Status::newGood()` to allow deletion, and then `return false` from the hook function. Ensure you consume the 'ChangeTagAfterDelete' hook to carry out custom deletion actions. $tag:name of the tag $user:user initiating the action & $status:Status object. See above. 'ChangeTagsListActive':Allows you to nominate which of the tags your extension uses are in active use. & $tags:list of all active tags. Append to this array. 'ChangeTagsAfterUpdateTags':Called after tags have been updated with the ChangeTags::updateTags function. Params:$addedTags:tags effectively added in the update $removedTags:tags effectively removed in the update $prevTags:tags that were present prior to the update $rc_id:recentchanges table id $rev_id:revision table id $log_id:logging table id $params:tag params $rc:RecentChange being tagged when the tagging accompanies the action, or null $user:User who performed the tagging when the tagging is subsequent to the action, or null 'ChangeTagsAllowedAdd':Called when checking if a user can add tags to a change. & $allowedTags:List of all the tags the user is allowed to add. Any tags the user wants to add( $addTags) that are not in this array will cause it to fail. You may add or remove tags to this array as required. $addTags:List of tags user intends to add. $user:User who is adding the tags. 'ChangeUserGroups':Called before user groups are changed. $performer:The User who will perform the change $user:The User whose groups will be changed & $add:The groups that will be added & $remove:The groups that will be removed 'Collation::factory':Called if $wgCategoryCollation is an unknown collation. $collationName:Name of the collation in question & $collationObject:Null. Replace with a subclass of the Collation class that implements the collation given in $collationName. 'ConfirmEmailComplete':Called after a user 's email has been confirmed successfully. $user:user(object) whose email is being confirmed 'ContentAlterParserOutput':Modify parser output for a given content object. Called by Content::getParserOutput after parsing has finished. Can be used for changes that depend on the result of the parsing but have to be done before LinksUpdate is called(such as adding tracking categories based on the rendered HTML). $content:The Content to render $title:Title of the page, as context $parserOutput:ParserOutput to manipulate 'ContentGetParserOutput':Customize parser output for a given content object, called by AbstractContent::getParserOutput. May be used to override the normal model-specific rendering of page content. $content:The Content to render $title:Title of the page, as context $revId:The revision ID, as context $options:ParserOptions for rendering. To avoid confusing the parser cache, the output can only depend on parameters provided to this hook function, not on global state. $generateHtml:boolean, indicating whether full HTML should be generated. If false, generation of HTML may be skipped, but other information should still be present in the ParserOutput object. & $output:ParserOutput, to manipulate or replace 'ContentHandlerDefaultModelFor':Called when the default content model is determined for a given title. May be used to assign a different model for that title. $title:the Title in question & $model:the model name. Use with CONTENT_MODEL_XXX constants. 'ContentHandlerForModelID':Called when a ContentHandler is requested for a given content model name, but no entry for that model exists in $wgContentHandlers. Note:if your extension implements additional models via this hook, please use GetContentModels hook to make them known to core. $modeName:the requested content model name & $handler:set this to a ContentHandler object, if desired. 'ContentModelCanBeUsedOn':Called to determine whether that content model can be used on a given page. This is especially useful to prevent some content models to be used in some special location. $contentModel:ID of the content model in question $title:the Title in question. & $ok:Output parameter, whether it is OK to use $contentModel on $title. Handler functions that modify $ok should generally return false to prevent further hooks from further modifying $ok. 'ContribsPager::getQueryInfo':Before the contributions query is about to run & $pager:Pager object for contributions & $queryInfo:The query for the contribs Pager 'ContribsPager::reallyDoQuery':Called before really executing the query for My Contributions & $data:an array of results of all contribs queries $pager:The ContribsPager object hooked into $offset:Index offset, inclusive $limit:Exact query limit $descending:Query direction, false for ascending, true for descending 'ContributionsLineEnding':Called before a contributions HTML line is finished $page:SpecialPage object for contributions & $ret:the HTML line $row:the DB row for this line & $classes:the classes to add to the surrounding< li > & $attribs:associative array of other HTML attributes for the< li > element. Currently only data attributes reserved to MediaWiki are allowed(see Sanitizer::isReservedDataAttribute). 'ContributionsToolLinks':Change tool links above Special:Contributions $id:User identifier $title:User page title & $tools:Array of tool links $specialPage:SpecialPage instance for context and services. Can be either SpecialContributions or DeletedContributionsPage. Extensions should type hint against a generic SpecialPage though. 'ConvertContent':Called by AbstractContent::convert when a conversion to another content model is requested. Handler functions that modify $result should generally return false to disable further attempts at conversion. $content:The Content object to be converted. $toModel:The ID of the content model to convert to. $lossy:boolean indicating whether lossy conversion is allowed. & $result:Output parameter, in case the handler function wants to provide a converted Content object. Note that $result->getContentModel() must return $toModel. 'ContentSecurityPolicyDefaultSource':Modify the allowed CSP load sources. This affects all directives except for the script directive. If you want to add a script source, see ContentSecurityPolicyScriptSource hook. & $defaultSrc:Array of Content-Security-Policy allowed sources $policyConfig:Current configuration for the Content-Security-Policy header $mode:ContentSecurityPolicy::REPORT_ONLY_MODE or ContentSecurityPolicy::FULL_MODE depending on type of header 'ContentSecurityPolicyDirectives':Modify the content security policy directives. Use this only if ContentSecurityPolicyDefaultSource and ContentSecurityPolicyScriptSource do not meet your needs. & $directives:Array of CSP directives $policyConfig:Current configuration for the CSP header $mode:ContentSecurityPolicy::REPORT_ONLY_MODE or ContentSecurityPolicy::FULL_MODE depending on type of header 'ContentSecurityPolicyScriptSource':Modify the allowed CSP script sources. Note that you also have to use ContentSecurityPolicyDefaultSource if you want non-script sources to be loaded from whatever you add. & $scriptSrc:Array of CSP directives $policyConfig:Current configuration for the CSP header $mode:ContentSecurityPolicy::REPORT_ONLY_MODE or ContentSecurityPolicy::FULL_MODE depending on type of header 'CustomEditor':When invoking the page editor Return true to allow the normal editor to be used, or false if implementing a custom editor, e.g. for a special namespace, etc. $article:Article being edited $user:User performing the edit 'DatabaseOraclePostInit':Called after initialising an Oracle database $db:the DatabaseOracle object 'DeletedContribsPager::reallyDoQuery':Called before really executing the query for Special:DeletedContributions Similar to ContribsPager::reallyDoQuery & $data:an array of results of all contribs queries $pager:The DeletedContribsPager object hooked into $offset:Index offset, inclusive $limit:Exact query limit $descending:Query direction, false for ascending, true for descending 'DeletedContributionsLineEnding':Called before a DeletedContributions HTML line is finished. Similar to ContributionsLineEnding $page:SpecialPage object for DeletedContributions & $ret:the HTML line $row:the DB row for this line & $classes:the classes to add to the surrounding< li > & $attribs:associative array of other HTML attributes for the< li > element. Currently only data attributes reserved to MediaWiki are allowed(see Sanitizer::isReservedDataAttribute). 'DeleteUnknownPreferences':Called by the cleanupPreferences.php maintenance script to build a WHERE clause with which to delete preferences that are not known about. This hook is used by extensions that have dynamically-named preferences that should not be deleted in the usual cleanup process. For example, the Gadgets extension creates preferences prefixed with 'gadget-', and so anything with that prefix is excluded from the deletion. &where:An array that will be passed as the $cond parameter to IDatabase::select() to determine what will be deleted from the user_properties table. $db:The IDatabase object, useful for accessing $db->buildLike() etc. 'DifferenceEngineAfterLoadNewText':called in DifferenceEngine::loadNewText() after the new revision 's content has been loaded into the class member variable $differenceEngine->mNewContent but before returning true from this function. $differenceEngine:DifferenceEngine object 'DifferenceEngineLoadTextAfterNewContentIsLoaded':called in DifferenceEngine::loadText() after the new revision 's content has been loaded into the class member variable $differenceEngine->mNewContent but before checking if the variable 's value is null. This hook can be used to inject content into said class member variable. $differenceEngine:DifferenceEngine object 'DifferenceEngineMarkPatrolledLink':Allows extensions to change the "mark as patrolled" link which is shown both on the diff header as well as on the bottom of a page, usually wrapped in a span element which has class="patrollink". $differenceEngine:DifferenceEngine object & $markAsPatrolledLink:The "mark as patrolled" link HTML(string) $rcid:Recent change ID(rc_id) for this change(int) 'DifferenceEngineMarkPatrolledRCID':Allows extensions to possibly change the rcid parameter. For example the rcid might be set to zero due to the user being the same as the performer of the change but an extension might still want to show it under certain conditions. & $rcid:rc_id(int) of the change or 0 $differenceEngine:DifferenceEngine object $change:RecentChange object $user:User object representing the current user 'DifferenceEngineNewHeader':Allows extensions to change the $newHeader variable, which contains information about the new revision, such as the revision 's author, whether the revision was marked as a minor edit or not, etc. $differenceEngine:DifferenceEngine object & $newHeader:The string containing the various #mw-diff-otitle[1-5] divs, which include things like revision author info, revision comment, RevisionDelete link and more $formattedRevisionTools:Array containing revision tools, some of which may have been injected with the DiffRevisionTools hook $nextlink:String containing the link to the next revision(if any) $status
Definition hooks.txt:1305
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that probably a stub it is not rendered in wiki pages or galleries in category pages allow injecting custom HTML after the section Any uses of the hook need to handle escaping see BaseTemplate::getToolbox and BaseTemplate::makeListItem for details on the format of individual items inside of this array or by returning and letting standard HTTP rendering take place modifiable or by returning false and taking over the output modifiable & $code
Definition hooks.txt:895
this hook is for auditing only or null if authentication failed before getting that far $username
Definition hooks.txt:815
and how to run hooks for an and one after Each event has a name
Definition hooks.txt:12
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that probably a stub it is not rendered in wiki pages or galleries in category pages allow injecting custom HTML after the section Any uses of the hook need to handle escaping see BaseTemplate::getToolbox and BaseTemplate::makeListItem for details on the format of individual items inside of this array or by returning and letting standard HTTP rendering take place modifiable or by returning false and taking over the output modifiable modifiable after all normalizations have been except for the $wgMaxImageArea check set to true or false to override the $wgMaxImageArea check result gives extension the possibility to transform it themselves $handler
Definition hooks.txt:933
null for the local wiki Added should default to null in handler for backwards compatibility add a value to it if you want to add a cookie that have to vary cache options can modify $query
Definition hooks.txt:1656
processing should stop and the error should be shown to the user * false
Definition hooks.txt:187
returning false will NOT prevent logging $e
Definition hooks.txt:2226
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition injection.txt:37
The wiki should then use memcached to cache various data To use multiple just add more items to the array To increase the weight of a make its entry a array("192.168.0.1:11211", 2))
if(!is_readable( $file)) $ext
Definition router.php:55
$params
$header