107 parent::__construct( $config );
109 $this->swiftAuthUrl = $config[
'swiftAuthUrl'];
110 $this->swiftUser = $config[
'swiftUser'];
111 $this->swiftKey = $config[
'swiftKey'];
113 $this->authTTL = isset( $config[
'swiftAuthTTL'] )
114 ? $config[
'swiftAuthTTL']
116 $this->swiftTempUrlKey = isset( $config[
'swiftTempUrlKey'] )
117 ? $config[
'swiftTempUrlKey']
119 $this->shardViaHashLevels = isset( $config[
'shardViaHashLevels'] )
120 ? $config[
'shardViaHashLevels']
122 $this->rgwS3AccessKey = isset( $config[
'rgwS3AccessKey'] )
123 ? $config[
'rgwS3AccessKey']
125 $this->rgwS3SecretKey = isset( $config[
'rgwS3SecretKey'] )
126 ? $config[
'rgwS3SecretKey']
131 if ( isset( $config[
'wanCache'] ) && $config[
'wanCache'] instanceof
WANObjectCache ) {
132 $this->memCache = $config[
'wanCache'];
137 if ( !empty( $config[
'cacheAuthInfo'] ) ) {
138 if ( PHP_SAPI ===
'cli' ) {
156 if ( !mb_check_encoding( $relStoragePath,
'UTF-8' ) ) {
158 } elseif ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
162 return $relStoragePath;
167 if ( $rel === null ) {
182 return isset( $params[
'headers'] )
198 if ( preg_match(
'/^content-(type|length)$/',
$name ) ) {
200 } elseif ( preg_match(
'/^(x-)?content-/',
$name ) ) {
202 } elseif ( preg_match(
'/^content-(disposition)/',
$name ) ) {
207 if ( isset( $headers[
'content-disposition'] ) ) {
210 foreach ( explode(
';', $headers[
'content-disposition'] )
as $part ) {
211 $part = trim( $part );
212 $new = ( $disposition ===
'' ) ? $part :
"{$disposition};{$part}";
213 if ( strlen( $new ) <= 255 ) {
219 $headers[
'content-disposition'] = $disposition;
233 if ( strpos(
$name,
'x-object-meta-' ) === 0 ) {
248 $metadata[substr(
$name, strlen(
'x-object-meta-' ) )] =
$value;
258 if ( $dstRel === null ) {
259 $status->fatal(
'backend-fail-invalidpath', $params[
'dst'] );
264 $sha1Hash = Wikimedia\base_convert( sha1( $params[
'content'] ), 16, 36, 31 );
265 $contentType = isset( $params[
'headers'][
'content-type'] )
266 ? $params[
'headers'][
'content-type']
267 : $this->
getContentType( $params[
'dst'], $params[
'content'], null );
271 'url' => [ $dstCont, $dstRel ],
273 'content-length' => strlen( $params[
'content'] ),
274 'etag' => md5( $params[
'content'] ),
275 'content-type' => $contentType,
276 'x-object-meta-sha1base36' => $sha1Hash
278 'body' => $params[
'content']
281 $method = __METHOD__;
283 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request[
'response'];
284 if ( $rcode === 201 ) {
286 } elseif ( $rcode === 412 ) {
287 $status->fatal(
'backend-fail-contenttype', $params[
'dst'] );
289 $this->
onError(
$status, $method, $params, $rerr, $rcode, $rdesc );
294 if ( !empty( $params[
'async'] ) ) {
307 if ( $dstRel === null ) {
308 $status->fatal(
'backend-fail-invalidpath', $params[
'dst'] );
313 MediaWiki\suppressWarnings();
314 $sha1Hash = sha1_file( $params[
'src'] );
315 MediaWiki\restoreWarnings();
316 if ( $sha1Hash ===
false ) {
317 $status->fatal(
'backend-fail-store', $params[
'src'], $params[
'dst'] );
321 $sha1Hash = Wikimedia\base_convert( $sha1Hash, 16, 36, 31 );
322 $contentType = isset( $params[
'headers'][
'content-type'] )
323 ? $params[
'headers'][
'content-type']
326 $handle = fopen( $params[
'src'],
'rb' );
327 if ( $handle ===
false ) {
328 $status->fatal(
'backend-fail-store', $params[
'src'], $params[
'dst'] );
335 'url' => [ $dstCont, $dstRel ],
337 'content-length' => filesize( $params[
'src'] ),
338 'etag' => md5_file( $params[
'src'] ),
339 'content-type' => $contentType,
340 'x-object-meta-sha1base36' => $sha1Hash
345 $method = __METHOD__;
347 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request[
'response'];
348 if ( $rcode === 201 ) {
350 } elseif ( $rcode === 412 ) {
351 $status->fatal(
'backend-fail-contenttype', $params[
'dst'] );
353 $this->
onError(
$status, $method, $params, $rerr, $rcode, $rdesc );
358 if ( !empty( $params[
'async'] ) ) {
371 if ( $srcRel === null ) {
372 $status->fatal(
'backend-fail-invalidpath', $params[
'src'] );
378 if ( $dstRel === null ) {
379 $status->fatal(
'backend-fail-invalidpath', $params[
'dst'] );
386 'url' => [ $dstCont, $dstRel ],
388 'x-copy-from' =>
'/' . rawurlencode( $srcCont ) .
389 '/' . str_replace(
"%2F",
"/", rawurlencode( $srcRel ) )
393 $method = __METHOD__;
395 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request[
'response'];
396 if ( $rcode === 201 ) {
398 } elseif ( $rcode === 404 ) {
399 $status->fatal(
'backend-fail-copy', $params[
'src'], $params[
'dst'] );
401 $this->
onError(
$status, $method, $params, $rerr, $rcode, $rdesc );
406 if ( !empty( $params[
'async'] ) ) {
419 if ( $srcRel === null ) {
420 $status->fatal(
'backend-fail-invalidpath', $params[
'src'] );
426 if ( $dstRel === null ) {
427 $status->fatal(
'backend-fail-invalidpath', $params[
'dst'] );
435 'url' => [ $dstCont, $dstRel ],
437 'x-copy-from' =>
'/' . rawurlencode( $srcCont ) .
438 '/' . str_replace(
"%2F",
"/", rawurlencode( $srcRel ) )
442 if (
"{$srcCont}/{$srcRel}" !==
"{$dstCont}/{$dstRel}" ) {
444 'method' =>
'DELETE',
445 'url' => [ $srcCont, $srcRel ],
450 $method = __METHOD__;
452 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request[
'response'];
453 if ( $request[
'method'] ===
'PUT' && $rcode === 201 ) {
455 } elseif ( $request[
'method'] ===
'DELETE' && $rcode === 204 ) {
457 } elseif ( $rcode === 404 ) {
458 $status->fatal(
'backend-fail-move', $params[
'src'], $params[
'dst'] );
460 $this->
onError(
$status, $method, $params, $rerr, $rcode, $rdesc );
465 if ( !empty( $params[
'async'] ) ) {
478 if ( $srcRel === null ) {
479 $status->fatal(
'backend-fail-invalidpath', $params[
'src'] );
485 'method' =>
'DELETE',
486 'url' => [ $srcCont, $srcRel ],
490 $method = __METHOD__;
492 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request[
'response'];
493 if ( $rcode === 204 ) {
495 } elseif ( $rcode === 404 ) {
496 if ( empty( $params[
'ignoreMissingSource'] ) ) {
497 $status->fatal(
'backend-fail-delete', $params[
'src'] );
500 $this->
onError(
$status, $method, $params, $rerr, $rcode, $rdesc );
505 if ( !empty( $params[
'async'] ) ) {
518 if ( $srcRel === null ) {
519 $status->fatal(
'backend-fail-invalidpath', $params[
'src'] );
525 $stat = $this->
getFileStat( [
'src' => $params[
'src'],
'latest' => 1 ] );
526 if ( $stat && !isset( $stat[
'xattr'] ) ) {
527 $stat = $this->
doGetFileStat( [
'src' => $params[
'src'],
'latest' => 1 ] );
530 $status->fatal(
'backend-fail-describe', $params[
'src'] );
538 $metaHdrs[
"x-object-meta-$name"] =
$value;
540 $customHdrs = $this->
sanitizeHdrs( $params ) + $stat[
'xattr'][
'headers'];
544 'url' => [ $srcCont, $srcRel ],
545 'headers' => $metaHdrs + $customHdrs
548 $method = __METHOD__;
550 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request[
'response'];
551 if ( $rcode === 202 ) {
553 } elseif ( $rcode === 404 ) {
554 $status->fatal(
'backend-fail-describe', $params[
'src'] );
556 $this->
onError(
$status, $method, $params, $rerr, $rcode, $rdesc );
561 if ( !empty( $params[
'async'] ) ) {
575 if ( is_array( $stat ) ) {
577 } elseif ( $stat === null ) {
578 $status->fatal(
'backend-fail-internal', $this->
name );
579 wfDebugLog(
'SwiftBackend', __METHOD__ .
': cannot get container stat' );
585 if ( $stat ===
false ) {
586 $params[
'op'] =
'prepare';
595 if ( empty( $params[
'noAccess'] ) ) {
600 if ( is_array( $stat ) ) {
604 [ $this->swiftUser ],
607 } elseif ( $stat ===
false ) {
608 $status->fatal(
'backend-fail-usable', $params[
'dir'] );
610 $status->fatal(
'backend-fail-internal', $this->
name );
611 wfDebugLog(
'SwiftBackend', __METHOD__ .
': cannot get container stat' );
621 if ( is_array( $stat ) ) {
625 [ $this->swiftUser,
'.r:*' ],
628 } elseif ( $stat ===
false ) {
629 $status->fatal(
'backend-fail-usable', $params[
'dir'] );
631 $status->fatal(
'backend-fail-internal', $this->
name );
632 wfDebugLog(
'SwiftBackend', __METHOD__ .
': cannot get container stat' );
648 if ( $stat ===
false ) {
650 } elseif ( !is_array( $stat ) ) {
651 $status->fatal(
'backend-fail-internal', $this->
name );
652 wfDebugLog(
'SwiftBackend', __METHOD__ .
': cannot get container stat' );
658 if ( $stat[
'count'] == 0 ) {
659 $params[
'op'] =
'clean';
667 $params = [
'srcs' => [ $params[
'src'] ],
'concurrency' => 1 ] +
$params;
668 unset( $params[
'src'] );
671 return reset( $stats );
702 if ( isset( $objHdrs[
'x-object-meta-sha1base36'] ) ) {
708 wfDebugLog(
'SwiftBackend', __METHOD__ .
": $path was not stored with SHA-1 metadata." );
710 $objHdrs[
'x-object-meta-sha1base36'] =
false;
728 $hash = $tmpFile->getSha1Base36();
729 if ( $hash !==
false ) {
730 $objHdrs[
'x-object-meta-sha1base36'] = $hash;
732 $postHeaders[
'x-object-meta-sha1base36'] = $hash;
734 list( $rcode ) = $this->
http->run( [
736 'url' => $this->
storageUrl( $auth, $srcCont, $srcRel ),
739 if ( $rcode >= 200 && $rcode <= 299 ) {
748 wfDebugLog(
'SwiftBackend', __METHOD__ .
": unable to set SHA-1 metadata for $path" );
758 $ep = array_diff_key( $params, [
'srcs' => 1 ] );
763 foreach ( $params[
'srcs']
as $path ) {
765 if ( $srcRel === null || !$auth ) {
766 $contents[
$path] =
false;
770 $handle = fopen(
'php://temp',
'wb' );
774 'url' => $this->
storageUrl( $auth, $srcCont, $srcRel ),
780 $contents[
$path] =
false;
783 $opts = [
'maxConnsPerHost' => $params[
'concurrency'] ];
784 $reqs = $this->
http->runMulti( $reqs, $opts );
785 foreach ( $reqs
as $path => $op ) {
786 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op[
'response'];
787 if ( $rcode >= 200 && $rcode <= 299 ) {
788 rewind( $op[
'stream'] );
789 $contents[
$path] = stream_get_contents( $op[
'stream'] );
790 } elseif ( $rcode === 404 ) {
791 $contents[
$path] =
false;
793 $this->
onError( null, __METHOD__,
794 [
'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
796 fclose( $op[
'stream'] );
803 $prefix = (
$dir ==
'' ) ? null :
"{$dir}/";
806 return ( count(
$status->value ) ) > 0;
847 if ( $after === INF ) {
853 $prefix = (
$dir ==
'' ) ? null :
"{$dir}/";
855 if ( !empty( $params[
'topOnly'] ) ) {
858 throw new FileBackendError(
"Iterator page I/O error: {$status->getMessage()}" );
861 foreach ( $objects
as $object ) {
862 if ( substr( $object, -1 ) ===
'/' ) {
868 $getParentDir =
function (
$path ) {
873 $lastDir = $getParentDir( $after );
877 throw new FileBackendError(
"Iterator page I/O error: {$status->getMessage()}" );
882 foreach ( $objects
as $object ) {
883 $objectDir = $getParentDir( $object );
885 if ( $objectDir !==
false && $objectDir !==
$dir ) {
890 if ( strcmp( $objectDir, $lastDir ) > 0 ) {
893 $dirs[] =
"{$pDir}/";
894 $pDir = $getParentDir( $pDir );
895 }
while ( $pDir !==
false
896 && strcmp( $pDir, $lastDir ) > 0
897 && strlen( $pDir ) > strlen(
$dir )
900 $lastDir = $objectDir;
905 if ( count( $objects ) <
$limit ) {
908 $after = end( $objects );
927 if ( $after === INF ) {
933 $prefix = (
$dir ==
'' ) ? null :
"{$dir}/";
936 if ( !empty( $params[
'topOnly'] ) ) {
937 if ( !empty( $params[
'adviseStat'] ) ) {
944 if ( !empty( $params[
'adviseStat'] ) ) {
953 throw new FileBackendError(
"Iterator page I/O error: {$status->getMessage()}" );
960 if ( count( $objects ) <
$limit ) {
963 $after = end( $objects );
964 $after = is_object( $after ) ? $after->name : $after;
981 foreach ( $objects
as $object ) {
982 if ( is_object( $object ) ) {
983 if ( isset( $object->subdir ) || !isset( $object->name ) ) {
989 'size' => (int)$object->bytes,
992 'md5' => ctype_xdigit( $object->hash ) ? $object->hash : null,
995 $names[] = [ $object->name, $stat ];
996 } elseif ( substr( $object, -1 ) !==
'/' ) {
998 $names[] = [ $object, null ];
1012 $this->cheapCache->set(
$path,
'stat', $val );
1018 if ( !isset( $stat[
'xattr'] ) ) {
1024 return $stat[
'xattr'];
1033 if ( !isset( $stat[
'sha1'] ) ) {
1039 return $stat[
'sha1'];
1049 if ( $srcRel === null ) {
1050 $status->fatal(
'backend-fail-invalidpath', $params[
'src'] );
1055 $status->fatal(
'backend-fail-stream', $params[
'src'] );
1060 $handle = fopen(
'php://output',
'wb' );
1062 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->
http->run( [
1064 'url' => $this->
storageUrl( $auth, $srcCont, $srcRel ),
1067 'stream' => $handle,
1070 if ( $rcode >= 200 && $rcode <= 299 ) {
1072 } elseif ( $rcode === 404 ) {
1073 $status->fatal(
'backend-fail-stream', $params[
'src'] );
1075 $this->
onError(
$status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1086 $ep = array_diff_key( $params, [
'srcs' => 1 ] );
1091 foreach ( $params[
'srcs']
as $path ) {
1093 if ( $srcRel === null || !$auth ) {
1094 $tmpFiles[
$path] = null;
1102 $handle = fopen( $tmpFile->getPath(),
'wb' );
1106 'url' => $this->
storageUrl( $auth, $srcCont, $srcRel ),
1109 'stream' => $handle,
1115 $tmpFiles[
$path] = $tmpFile;
1118 $isLatest = ( $this->isRGW || !empty( $params[
'latest'] ) );
1119 $opts = [
'maxConnsPerHost' => $params[
'concurrency'] ];
1120 $reqs = $this->
http->runMulti( $reqs, $opts );
1121 foreach ( $reqs
as $path => $op ) {
1122 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op[
'response'];
1123 fclose( $op[
'stream'] );
1124 if ( $rcode >= 200 && $rcode <= 299 ) {
1125 $size = $tmpFiles[
$path] ? $tmpFiles[
$path]->getSize() : 0;
1127 if ( $size != $rhdrs[
'content-length'] ) {
1128 $tmpFiles[
$path] = null;
1129 $rerr =
"Got {$size}/{$rhdrs['content-length']} bytes";
1130 $this->
onError( null, __METHOD__,
1131 [
'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
1135 $stat[
'latest'] = $isLatest;
1136 $this->cheapCache->set( $path,
'stat', $stat );
1137 } elseif ( $rcode === 404 ) {
1138 $tmpFiles[
$path] =
false;
1140 $tmpFiles[
$path] = null;
1141 $this->
onError( null, __METHOD__,
1142 [
'src' => $path ] + $ep, $rerr, $rcode, $rdesc );
1150 if ( $this->swiftTempUrlKey !=
'' ||
1151 ( $this->rgwS3AccessKey !=
'' && $this->rgwS3SecretKey !=
'' )
1154 if ( $srcRel === null ) {
1163 $ttl = isset( $params[
'ttl'] ) ? $params[
'ttl'] : 86400;
1164 $expires = time() + $ttl;
1166 if ( $this->swiftTempUrlKey !=
'' ) {
1167 $url = $this->
storageUrl( $auth, $srcCont, $srcRel );
1169 $contPath = parse_url( $this->
storageUrl( $auth, $srcCont ), PHP_URL_PATH );
1170 $signature = hash_hmac(
'sha1',
1171 "GET\n{$expires}\n{$contPath}/{$srcRel}",
1172 $this->swiftTempUrlKey
1175 return "{$url}?temp_url_sig={$signature}&temp_url_expires={$expires}";
1178 $spath =
'/' . rawurlencode( $srcCont ) .
'/' .
1179 str_replace(
'%2F',
'/', rawurlencode( $srcRel ) );
1181 $signature = base64_encode( hash_hmac(
1183 "GET\n\n\n{$expires}\n{$spath}",
1184 $this->rgwS3SecretKey,
1190 str_replace(
'/swift/v1',
'',
1193 'Signature' => $signature,
1194 'Expires' => $expires,
1195 'AWSAccessKeyId' => $this->rgwS3AccessKey ]
1217 if ( !empty( $params[
'latest'] ) ) {
1218 $hdrs[
'x-newest'] =
'true';
1234 foreach ( $fileOpHandles
as $index => $fileOpHandle ) {
1242 $httpReqsByStage = [];
1243 foreach ( $fileOpHandles
as $index => $fileOpHandle ) {
1244 $reqs = $fileOpHandle->httpOp;
1246 foreach ( $reqs
as $stage => &
$req ) {
1247 list( $container, $relPath ) =
$req[
'url'];
1249 $req[
'headers'] = isset(
$req[
'headers'] ) ?
$req[
'headers'] : [];
1251 $httpReqsByStage[$stage][$index] =
$req;
1257 $reqCount = count( $httpReqsByStage );
1258 for ( $stage = 0; $stage < $reqCount; ++$stage ) {
1259 $httpReqs = $this->
http->runMulti( $httpReqsByStage[$stage] );
1260 foreach ( $httpReqs
as $index => $httpReq ) {
1262 $callback = $fileOpHandles[$index]->callback;
1263 call_user_func_array( $callback, [ $httpReq, $statuses[$index] ] );
1266 if ( !$statuses[$index]->isOK() ) {
1267 $stages = count( $fileOpHandles[$index]->httpOp );
1268 for (
$s = ( $stage + 1 );
$s < $stages; ++
$s ) {
1269 unset( $httpReqsByStage[
$s][$index] );
1305 $status->fatal(
'backend-fail-connect', $this->
name );
1310 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->
http->run( [
1312 'url' => $this->
storageUrl( $auth, $container ),
1314 'x-container-read' => implode(
',', $readGrps ),
1315 'x-container-write' => implode(
',', $writeGrps )
1319 if ( $rcode != 204 && $rcode !== 202 ) {
1320 $status->fatal(
'backend-fail-internal', $this->
name );
1321 wfDebugLog(
'SwiftBackend', __METHOD__ .
': unexpected rcode value (' . $rcode .
')' );
1338 if ( $bypassCache ) {
1339 $this->containerStatCache->clear( $container );
1340 } elseif ( !$this->containerStatCache->has( $container,
'stat' ) ) {
1343 if ( !$this->containerStatCache->has( $container,
'stat' ) ) {
1349 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->
http->run( [
1351 'url' => $this->
storageUrl( $auth, $container ),
1355 if ( $rcode === 204 ) {
1357 'count' => $rhdrs[
'x-container-object-count'],
1358 'bytes' => $rhdrs[
'x-container-bytes-used']
1360 if ( $bypassCache ) {
1363 $this->containerStatCache->set( $container,
'stat', $stat );
1366 } elseif ( $rcode === 404 ) {
1369 $this->
onError( null, __METHOD__,
1370 [
'cont' => $container ], $rerr, $rcode, $rdesc );
1376 return $this->containerStatCache->get( $container,
'stat' );
1391 $status->fatal(
'backend-fail-connect', $this->
name );
1397 if ( empty( $params[
'noAccess'] ) ) {
1404 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->
http->run( [
1406 'url' => $this->
storageUrl( $auth, $container ),
1408 'x-container-read' => implode(
',', $readGrps ),
1409 'x-container-write' => implode(
',', $writeGrps )
1413 if ( $rcode === 201 ) {
1415 } elseif ( $rcode === 202 ) {
1418 $this->
onError(
$status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1436 $status->fatal(
'backend-fail-connect', $this->
name );
1441 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->
http->run( [
1442 'method' =>
'DELETE',
1443 'url' => $this->
storageUrl( $auth, $container ),
1447 if ( $rcode >= 200 && $rcode <= 299 ) {
1448 $this->containerStatCache->clear( $container );
1449 } elseif ( $rcode === 404 ) {
1451 } elseif ( $rcode === 409 ) {
1452 $this->
onError(
$status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1454 $this->
onError(
$status, __METHOD__, $params, $rerr, $rcode, $rdesc );
1473 $fullCont,
$type,
$limit, $after = null, $prefix = null, $delim = null
1479 $status->fatal(
'backend-fail-connect', $this->
name );
1485 if (
$type ===
'info' ) {
1486 $query[
'format'] =
'json';
1488 if ( $after !== null ) {
1489 $query[
'marker'] = $after;
1491 if ( $prefix !== null ) {
1492 $query[
'prefix'] = $prefix;
1494 if ( $delim !== null ) {
1495 $query[
'delimiter'] = $delim;
1498 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->
http->run( [
1500 'url' => $this->
storageUrl( $auth, $fullCont ),
1505 $params = [
'cont' => $fullCont,
'prefix' => $prefix,
'delim' => $delim ];
1506 if ( $rcode === 200 ) {
1507 if (
$type ===
'info' ) {
1510 $status->value = explode(
"\n", trim( $rbody ) );
1512 } elseif ( $rcode === 204 ) {
1514 } elseif ( $rcode === 404 ) {
1524 foreach ( $containerInfo
as $container => $info ) {
1525 $this->containerStatCache->set( $container,
'stat', $info );
1535 foreach ( $params[
'srcs']
as $path ) {
1537 if ( $srcRel === null ) {
1538 $stats[
$path] =
false;
1540 } elseif ( !$auth ) {
1541 $stats[
$path] = null;
1547 if ( $cstat ===
false ) {
1548 $stats[
$path] =
false;
1550 } elseif ( !is_array( $cstat ) ) {
1551 $stats[
$path] = null;
1557 'url' => $this->
storageUrl( $auth, $srcCont, $srcRel ),
1562 $opts = [
'maxConnsPerHost' => $params[
'concurrency'] ];
1563 $reqs = $this->
http->runMulti( $reqs, $opts );
1565 foreach ( $params[
'srcs']
as $path ) {
1566 if ( array_key_exists( $path, $stats ) ) {
1570 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $reqs[
$path][
'response'];
1571 if ( $rcode === 200 || $rcode === 204 ) {
1576 if ( $this->isRGW ) {
1577 $stat[
'latest'] =
true;
1579 } elseif ( $rcode === 404 ) {
1583 $this->
onError( null, __METHOD__, $params, $rerr, $rcode, $rdesc );
1585 $stats[
$path] = $stat;
1599 $headers = $this->
sanitizeHdrs( [
'headers' => $rhdrs ] );
1605 'size' => isset( $rhdrs[
'content-length'] ) ? (int)$rhdrs[
'content-length'] : 0,
1606 'sha1' => isset( $metadata[
'sha1base36'] ) ? $metadata[
'sha1base36'] : null,
1608 'md5' => ctype_xdigit( $rhdrs[
'etag'] ) ? $rhdrs[
'etag'] : null,
1609 'xattr' => [
'metadata' => $metadata,
'headers' => $headers ]
1617 if ( $this->authErrorTimestamp !== null ) {
1618 if ( ( time() - $this->authErrorTimestamp ) < 60 ) {
1621 $this->authErrorTimestamp = null;
1627 if ( !$this->authCreds || $reAuth ) {
1628 $this->authSessionTimestamp = 0;
1630 $creds = $this->srvCache->get( $cacheKey );
1632 if ( isset( $creds[
'auth_token'] ) && isset( $creds[
'storage_url'] ) ) {
1633 $this->authCreds = $creds;
1635 $this->authSessionTimestamp = time() - ceil( $this->authTTL / 2 );
1637 list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->
http->run( [
1639 'url' =>
"{$this->swiftAuthUrl}/v1.0",
1641 'x-auth-user' => $this->swiftUser,
1642 'x-auth-key' => $this->swiftKey
1646 if ( $rcode >= 200 && $rcode <= 299 ) {
1647 $this->authCreds = [
1648 'auth_token' => $rhdrs[
'x-auth-token'],
1649 'storage_url' => $rhdrs[
'x-storage-url']
1651 $this->srvCache->set( $cacheKey, $this->authCreds, ceil( $this->authTTL / 2 ) );
1652 $this->authSessionTimestamp = time();
1653 } elseif ( $rcode === 401 ) {
1654 $this->
onError( null, __METHOD__, [],
"Authentication failed.", $rcode );
1655 $this->authErrorTimestamp = time();
1659 $this->
onError( null, __METHOD__, [],
"HTTP return code: $rcode", $rcode );
1660 $this->authErrorTimestamp = time();
1666 if ( substr( $this->authCreds[
'storage_url'], -3 ) ===
'/v1' ) {
1667 $this->isRGW =
true;
1681 $parts = [ $creds[
'storage_url'] ];
1682 if ( strlen( $container ) ) {
1683 $parts[] = rawurlencode( $container );
1685 if ( strlen( $object ) ) {
1686 $parts[] = str_replace(
"%2F",
"/", rawurlencode( $object ) );
1689 return implode(
'/', $parts );
1697 return [
'x-auth-token' => $creds[
'auth_token'] ];
1707 return 'swiftcredentials:' . md5(
$username .
':' . $this->swiftAuthUrl );
1723 $status->fatal(
'backend-fail-internal', $this->
name );
1725 if (
$code == 401 ) {
1730 ( $err ?
": $err" :
"" )
1798 $this->container = $fullCont;
1800 if ( substr( $this->dir, -1 ) ===
'/' ) {
1801 $this->dir = substr( $this->dir, 0, -1 );
1803 if ( $this->dir ==
'' ) {
1804 $this->suffixStart = 0;
1806 $this->suffixStart = strlen( $this->dir ) + 1;
1824 next( $this->bufferIter );
1828 if ( !$this->
valid() && count( $this->bufferIter ) ) {
1830 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1840 $this->bufferAfter = null;
1842 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1851 if ( $this->bufferIter === null ) {
1854 return ( current( $this->bufferIter ) !==
false );
1880 return substr(
current( $this->bufferIter ), $this->suffixStart, -1 );
1898 $relPath = substr(
$path, $this->suffixStart );
1899 if ( is_array( $stat ) ) {
1900 $storageDir = rtrim( $this->params[
'dir'],
'/' );
1901 $this->backend->loadListingStatInternal(
"$storageDir/$relPath", $stat );
static factory($prefix, $extension= '')
Make a new temporary file on the file system.
doStoreInternal(array $params)
doPrepareInternal($fullCont, $dir, array $params)
doGetFileStatMulti(array $params)
getFileListPageInternal($fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
primeContainerCache(array $items)
Do a batch lookup from cache for container stats for all containers used in a list of container names...
buildFileObjectListing(array $params, $dir, array $objects)
Build a list of file objects, filtering out any directories and extracting any stat info if provided ...
doCleanInternal($fullCont, $dir, array $params)
getFileListInternal($fullCont, $dir, array $params)
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
the array() calling protocol came about after MediaWiki 1.4rc1.
resolveContainerPath($container, $relStoragePath)
null for the local wiki Added should default to null in handler for backwards compatibility add a value to it if you want to add a cookie that have to vary cache options can modify $query
getCustomHeaders(array $rawHeaders)
Iterator for listing directories.
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
div flags Integer display flags(NO_ACTION_LINK, NO_EXTRA_USER_LINKS) 'LogException'returning false will NOT prevent logging $e
string $swiftUser
Swift user (account:user) to authenticate as.
int $authErrorTimestamp
UNIX timestamp.
static instance()
Singleton.
processing should stop and the error should be shown to the user * false
deleteContainer($container, array $params)
Delete a Swift container.
getFileStat(array $params)
const ATTR_HEADERS
Bitfield flags for supported features.
doDirectoryExists($fullCont, $dir, array $params)
doPublishInternal($fullCont, $dir, array $params)
string $bufferAfter
List items after this path.
static getLocalClusterInstance()
Get the main cluster-local cache object.
SwiftFileBackend helper class to page through listings.
doDeleteInternal(array $params)
static newFatal($message)
Factory function for fatal errors.
doMoveInternal(array $params)
getStatFromHeaders(array $rhdrs)
Multi-datacenter aware caching interface.
static extensionFromPath($path, $case= 'lowercase')
Get the final extension from a storage or FS path.
setContainerAccess($container, array $readGrps, array $writeGrps)
Set read/write permissions for a Swift container.
doGetFileXAttributes(array $params)
loadListingStatInternal($path, array $val)
Do not call this function outside of SwiftFileBackendFileList.
Apache License January http
doGetFileContentsMulti(array $params)
addMissingMetadata(array $objHdrs, $path)
Fill in any missing object metadata and save it to Swift.
doGetLocalCopyMulti(array $params)
doGetFileSha1base36(array $params)
getContainerStat($container, $bypassCache=false)
Get a Swift container stat array, possibly from process cache.
wfDebugLog($logGroup, $text, $dest= 'all', array $context=[])
Send a line to a supplementary debug log file, if configured, or main debug log if not...
doDescribeInternal(array $params)
getMetadata(array $rawHeaders)
getFileHttpUrl(array $params)
File backend exception for checked exceptions (e.g.
wfAppendQuery($url, $query)
Append a query string to an existing URL, which may or may not already have query string parameters a...
authTokenHeaders(array $creds)
headersFromParams(array $params)
Get headers to send to Swift when reading a file based on a FileBackend params array, e.g.
array $bufferIter
List of path or (path,stat array) entries.
storageUrl(array $creds, $container=null, $object=null)
isPathUsableInternal($storagePath)
__construct(SwiftFileBackend $backend, $fullCont, $dir, array $params)
string $swiftAuthUrl
Authentication base URL (without version)
string $dir
Storage directory.
getCredsCacheKey($username)
Get the cache key for a container.
FileBackendStore $backend
clearCache(array $paths=null)
A BagOStuff object with no objects in it.
resolveStoragePathReal($storagePath)
Like resolveStoragePath() except null values are returned if the container is sharded and the shard c...
objectListing($fullCont, $type, $limit, $after=null, $prefix=null, $delim=null)
Get a list of objects under a container.
string $container
Container name.
int $authTTL
TTL in seconds.
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that probably a stub it is not rendered in wiki pages or galleries in category pages allow injecting custom HTML after the section Any uses of the hook need to handle escaping see BaseTemplate::getToolbox and BaseTemplate::makeListItem for details on the format of individual items inside of this array or by returning and letting standard HTTP rendering take place modifiable or by returning false and taking over the output modifiable & $code
getContentType($storagePath, $content, $fsPath)
Get the content type to use in HEAD/GET requests for a file.
onError($status, $func, array $params, $err= '', $code=0, $desc= '')
Log an unexpected exception for this backend.
string $name
Unique backend name.
Base class for all backends using particular storage medium.
getLocalCopy(array $params)
Get a local copy on disk of the file at a storage path in the backend.
doStreamFile(array $params)
pageFromList($container, $dir, &$after, $limit, array $params)
const TS_MW
MediaWiki concatenated string timestamp (YYYYMMDDHHMMSS)
pageFromList($container, $dir, &$after, $limit, array $params)
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
this hook is for auditing only $req
this hook is for auditing only or null if authentication failed before getting that far $username
Iterator for listing regular files.
setContainerCache($container, array $val)
Set the cached info for a container.
getDirectoryListInternal($fullCont, $dir, array $params)
__construct(array $config)
error also a ContextSource you ll probably need to make sure the header is varied on $request
SwiftFileBackend $backend
sanitizeHdrs(array $params)
Sanitize and filter the custom headers from a $params array.
doCopyInternal(array $params)
convertSwiftDate($ts, $format=TS_MW)
Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT"/"2013-05-11T07:37:27.678360Z". ...
string $rgwS3AccessKey
S3 access key (RADOS Gateway)
doCreateInternal(array $params)
static getLocalServerInstance($fallback=CACHE_NONE)
Factory function for CACHE_ACCEL (referenced from DefaultSettings.php)
array $httpOp
List of Requests for MultiHttpClient.
doPrimeContainerCache(array $containerInfo)
Class for an OpenStack Swift (or Ceph RGW) based file backend.
design txt This is a brief overview of the new design More thorough and up to date information is available on the documentation wiki at name
this hook is for auditing only RecentChangesLinked and Watchlist RecentChangesLinked and Watchlist e g Watchlist removed from all revisions and log entries to which it was applied This gives extensions a chance to take it off their books as the deletion has already been partly carried out by this point or something similar the user will be unable to create the tag set and then return false from the hook function Ensure you consume the ChangeTagAfterDelete hook to carry out custom deletion actions as context called by AbstractContent::getParserOutput May be used to override the normal model specific rendering of page content as context as context the output can only depend on parameters provided to this hook not on global state indicating whether full HTML should be generated If generation of HTML may be but other information should still be present in the ParserOutput object to manipulate or replace but no entry for that model exists in $wgContentHandlers if desired whether it is OK to use $contentModel on $title Handler functions that modify $ok should generally return false to prevent further hooks from further modifying $ok inclusive $limit
pageFromList($container, $dir, &$after, $limit, array $params)
Get the given list portion (page)
this hook is for auditing only RecentChangesLinked and Watchlist RecentChangesLinked and Watchlist e g Watchlist removed from all revisions and log entries to which it was applied This gives extensions a chance to take it off their books as the deletion has already been partly carried out by this point or something similar the user will be unable to create the tag set $status
getMetadataHeaders(array $rawHeaders)
int $authSessionTimestamp
UNIX timestamp.
deleteFileCache($path)
Delete the cached stat info for a file path.
ProcessCacheLRU $containerStatCache
Container stat cache.
getDirListPageInternal($fullCont, $dir, &$after, $limit, array $params)
Do not call this function outside of SwiftFileBackendFileList.
doExecuteOpHandlesInternal(array $fileOpHandles)
doSecureInternal($fullCont, $dir, array $params)
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that probably a stub it is not rendered in wiki pages or galleries in category pages allow injecting custom HTML after the section Any uses of the hook need to handle escaping see BaseTemplate::getToolbox and BaseTemplate::makeListItem for details on the format of individual items inside of this array or by returning and letting standard HTTP rendering take place modifiable or by returning false and taking over the output modifiable modifiable after all normalizations have been except for the $wgMaxImageArea check set to true or false to override the $wgMaxImageArea check result gives extension the possibility to transform it themselves $handler
Class to handle concurrent HTTP requests.
doGetFileStat(array $params)
createContainer($container, array $params)
Create a Swift container.
FileBackendStore helper class for performing asynchronous file operations.
string $rgwS3SecretKey
S3 authentication key (RADOS Gateway)
Handles per process caching of items.
Library for creating and parsing MW-style timestamps.
do that in ParserLimitReportFormat instead use this to modify the parameters of the image and a DIV can begin in one section and end in another Make sure your code can handle that case gracefully See the EditSectionClearerLink extension for an example zero but section is usually empty its values are the globals values before the output is cached one of or reset my talk my contributions etc etc otherwise the built in rate limiting checks are if enabled allows for interception of redirect as a string mapping parameter names to values & $type
__construct(SwiftFileBackend $backend, Closure $callback, array $httpOp)
string $swiftKey
Secret key for user.
static newGood($value=null)
Factory function for good results.
bool $isRGW
Whether the server is an Ceph RGW.
getScopedFileLocks(array $paths, $type, Status $status, $timeout=0)
Lock the files at the given storage paths in the backend.
string $swiftTempUrlKey
Shared secret value for making temp URLs.