Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
68.42% |
91 / 133 |
|
20.00% |
2 / 10 |
CRAP | |
0.00% |
0 / 1 |
| RESTBagOStuff | |
68.42% |
91 / 133 |
|
20.00% |
2 / 10 |
124.61 | |
0.00% |
0 / 1 |
| __construct | |
60.87% |
14 / 23 |
|
0.00% |
0 / 1 |
6.50 | |||
| setLogger | |
100.00% |
2 / 2 |
|
100.00% |
1 / 1 |
1 | |||
| doGet | |
100.00% |
20 / 20 |
|
100.00% |
1 / 1 |
8 | |||
| doSet | |
84.62% |
11 / 13 |
|
0.00% |
0 / 1 |
4.06 | |||
| doAdd | |
0.00% |
0 / 3 |
|
0.00% |
0 / 1 |
6 | |||
| doDelete | |
83.33% |
10 / 12 |
|
0.00% |
0 / 1 |
2.02 | |||
| doIncrWithInit | |
0.00% |
0 / 8 |
|
0.00% |
0 / 1 |
30 | |||
| decodeBody | |
70.59% |
12 / 17 |
|
0.00% |
0 / 1 |
11.06 | |||
| encodeBody | |
77.78% |
14 / 18 |
|
0.00% |
0 / 1 |
6.40 | |||
| handleError | |
47.06% |
8 / 17 |
|
0.00% |
0 / 1 |
14.27 | |||
| 1 | <?php |
| 2 | |
| 3 | use Psr\Log\LoggerInterface; |
| 4 | |
| 5 | /** |
| 6 | * Interface to key-value storage behind an HTTP server. |
| 7 | * |
| 8 | * ### Important caveats |
| 9 | * |
| 10 | * This interface is currently an incomplete BagOStuff implementation, |
| 11 | * supported only for use with MediaWiki features that accept a dedicated |
| 12 | * cache type to use for a narrow set of cache keys that share the same |
| 13 | * key expiry and replication requirements, and where the key-value server |
| 14 | * in question is statically configured with domain knowledge of said |
| 15 | * key expiry and replication requirements. |
| 16 | * |
| 17 | * Specifically, RESTBagOStuff has the following limitations: |
| 18 | * |
| 19 | * - The expiry parameter is ignored in methods like `set()`. |
| 20 | * |
| 21 | * There is not currently an agreed protocol for sending this to a |
| 22 | * server. This class is written for use with MediaWiki\Session\SessionManager |
| 23 | * and Kask/Cassanda at WMF, which does not expose a customizable expiry. |
| 24 | * |
| 25 | * As such, it is not recommended to use RESTBagOStuff to back a general |
| 26 | * purpose cache type (such as MediaWiki's main cache, or main stash). |
| 27 | * Instead, it is only supported toMediaWiki features where a cache type can |
| 28 | * be pointed for a narrow set of keys that naturally share the same TTL |
| 29 | * anyway, or where the feature behaves correctly even if the logical expiry |
| 30 | * is longer than specified (e.g. immutable keys, or value verification) |
| 31 | * |
| 32 | * - Most methods are non-atomic. |
| 33 | * |
| 34 | * The class should only be used for get, set, and delete operations. |
| 35 | * Advanced methods like `incr()`, `add()` and `lock()` do exist but |
| 36 | * inherit a native and best-effort implementation based on get+set. |
| 37 | * These should not be relied upon. |
| 38 | * |
| 39 | * ### Backend requirements |
| 40 | * |
| 41 | * The HTTP server will receive requests for URLs like `{baseURL}/{KEY}`. It |
| 42 | * must implement the GET, PUT and DELETE methods. |
| 43 | * |
| 44 | * E.g., when the base URL is `/sessions/v1`, then `set()` will: |
| 45 | * |
| 46 | * `PUT /sessions/v1/mykeyhere` |
| 47 | * |
| 48 | * and `get()` would do: |
| 49 | * |
| 50 | * `GET /sessions/v1/mykeyhere` |
| 51 | * |
| 52 | * and `delete()` would do: |
| 53 | * |
| 54 | * `DELETE /sessions/v1/mykeyhere` |
| 55 | * |
| 56 | * ### Example configuration |
| 57 | * |
| 58 | * Minimal generic configuration: |
| 59 | * |
| 60 | * @code |
| 61 | * $wgObjectCaches['sessions'] = array( |
| 62 | * 'class' => 'RESTBagOStuff', |
| 63 | * 'url' => 'http://localhost:7231/example/' |
| 64 | * ); |
| 65 | * @endcode |
| 66 | * |
| 67 | * |
| 68 | * Configuration for [Kask](https://www.mediawiki.org/wiki/Kask) session store: |
| 69 | * @code |
| 70 | * $wgObjectCaches['sessions'] = array( |
| 71 | * 'class' => 'RESTBagOStuff', |
| 72 | * 'url' => 'https://kaskhost:1234/sessions/v1/', |
| 73 | * 'httpParams' => [ |
| 74 | * 'readHeaders' => [], |
| 75 | * 'writeHeaders' => [ 'content-type' => 'application/octet-stream' ], |
| 76 | * 'deleteHeaders' => [], |
| 77 | * 'writeMethod' => 'POST', |
| 78 | * ], |
| 79 | * 'serialization_type' => 'JSON', |
| 80 | * 'extendedErrorBodyFields' => [ 'type', 'title', 'detail', 'instance' ] |
| 81 | * ); |
| 82 | * $wgSessionCacheType = 'sessions'; |
| 83 | * @endcode |
| 84 | */ |
| 85 | class RESTBagOStuff extends MediumSpecificBagOStuff { |
| 86 | /** |
| 87 | * Default connection timeout in seconds. The kernel retransmits the SYN |
| 88 | * packet after 1 second, so 1.2 seconds allows for 1 retransmit without |
| 89 | * permanent failure. |
| 90 | */ |
| 91 | private const DEFAULT_CONN_TIMEOUT = 1.2; |
| 92 | |
| 93 | /** |
| 94 | * Default request timeout |
| 95 | */ |
| 96 | private const DEFAULT_REQ_TIMEOUT = 3.0; |
| 97 | |
| 98 | /** |
| 99 | * @var MultiHttpClient |
| 100 | */ |
| 101 | private $client; |
| 102 | |
| 103 | /** |
| 104 | * REST URL to use for storage. |
| 105 | * @var string |
| 106 | */ |
| 107 | private $url; |
| 108 | |
| 109 | /** |
| 110 | * HTTP parameters: readHeaders, writeHeaders, deleteHeaders, writeMethod. |
| 111 | * @var array |
| 112 | */ |
| 113 | private $httpParams; |
| 114 | |
| 115 | /** |
| 116 | * Optional serialization type to use. Allowed values: "PHP", "JSON". |
| 117 | * @var string |
| 118 | */ |
| 119 | private $serializationType; |
| 120 | |
| 121 | /** |
| 122 | * Optional HMAC Key for protecting the serialized blob. If omitted no protection is done |
| 123 | * @var string |
| 124 | */ |
| 125 | private $hmacKey; |
| 126 | |
| 127 | /** |
| 128 | * @var array additional body fields to log on error, if possible |
| 129 | */ |
| 130 | private $extendedErrorBodyFields; |
| 131 | |
| 132 | public function __construct( $params ) { |
| 133 | $params['segmentationSize'] ??= INF; |
| 134 | if ( empty( $params['url'] ) ) { |
| 135 | throw new InvalidArgumentException( 'URL parameter is required' ); |
| 136 | } |
| 137 | |
| 138 | if ( empty( $params['client'] ) ) { |
| 139 | // Pass through some params to the HTTP client. |
| 140 | $clientParams = [ |
| 141 | 'connTimeout' => $params['connTimeout'] ?? self::DEFAULT_CONN_TIMEOUT, |
| 142 | 'reqTimeout' => $params['reqTimeout'] ?? self::DEFAULT_REQ_TIMEOUT, |
| 143 | ]; |
| 144 | foreach ( [ 'caBundlePath', 'proxy', 'telemetry' ] as $key ) { |
| 145 | if ( isset( $params[$key] ) ) { |
| 146 | $clientParams[$key] = $params[$key]; |
| 147 | } |
| 148 | } |
| 149 | $this->client = new MultiHttpClient( $clientParams ); |
| 150 | } else { |
| 151 | $this->client = $params['client']; |
| 152 | } |
| 153 | |
| 154 | $this->httpParams['writeMethod'] = $params['httpParams']['writeMethod'] ?? 'PUT'; |
| 155 | $this->httpParams['readHeaders'] = $params['httpParams']['readHeaders'] ?? []; |
| 156 | $this->httpParams['writeHeaders'] = $params['httpParams']['writeHeaders'] ?? []; |
| 157 | $this->httpParams['deleteHeaders'] = $params['httpParams']['deleteHeaders'] ?? []; |
| 158 | $this->extendedErrorBodyFields = $params['extendedErrorBodyFields'] ?? []; |
| 159 | $this->serializationType = $params['serialization_type'] ?? 'PHP'; |
| 160 | $this->hmacKey = $params['hmac_key'] ?? ''; |
| 161 | |
| 162 | // The parent constructor calls setLogger() which sets the logger in $this->client |
| 163 | parent::__construct( $params ); |
| 164 | |
| 165 | // Make sure URL ends with / |
| 166 | $this->url = rtrim( $params['url'], '/' ) . '/'; |
| 167 | |
| 168 | $this->attrMap[self::ATTR_DURABILITY] = self::QOS_DURABILITY_DISK; |
| 169 | } |
| 170 | |
| 171 | public function setLogger( LoggerInterface $logger ) { |
| 172 | parent::setLogger( $logger ); |
| 173 | $this->client->setLogger( $logger ); |
| 174 | } |
| 175 | |
| 176 | protected function doGet( $key, $flags = 0, &$casToken = null ) { |
| 177 | $getToken = ( $casToken === self::PASS_BY_REF ); |
| 178 | $casToken = null; |
| 179 | |
| 180 | $req = [ |
| 181 | 'method' => 'GET', |
| 182 | 'url' => $this->url . rawurlencode( $key ), |
| 183 | 'headers' => $this->httpParams['readHeaders'], |
| 184 | ]; |
| 185 | |
| 186 | $value = false; |
| 187 | $valueSize = false; |
| 188 | [ $rcode, , $rhdrs, $rbody, $rerr ] = $this->client->run( $req ); |
| 189 | if ( $rcode === 200 && is_string( $rbody ) ) { |
| 190 | $value = $this->decodeBody( $rbody ); |
| 191 | $valueSize = strlen( $rbody ); |
| 192 | // @FIXME: use some kind of hash or UUID header as CAS token |
| 193 | if ( $getToken && $value !== false ) { |
| 194 | $casToken = $rbody; |
| 195 | } |
| 196 | } elseif ( $rcode === 0 || ( $rcode >= 400 && $rcode != 404 ) ) { |
| 197 | $this->handleError( 'Failed to fetch {cacheKey}', $rcode, $rerr, $rhdrs, $rbody, |
| 198 | [ 'cacheKey' => $key ] ); |
| 199 | } |
| 200 | |
| 201 | $this->updateOpStats( self::METRIC_OP_GET, [ $key => [ 0, $valueSize ] ] ); |
| 202 | |
| 203 | return $value; |
| 204 | } |
| 205 | |
| 206 | protected function doSet( $key, $value, $exptime = 0, $flags = 0 ) { |
| 207 | $req = [ |
| 208 | 'method' => $this->httpParams['writeMethod'], |
| 209 | 'url' => $this->url . rawurlencode( $key ), |
| 210 | 'body' => $this->encodeBody( $value ), |
| 211 | 'headers' => $this->httpParams['writeHeaders'], |
| 212 | ]; |
| 213 | |
| 214 | [ $rcode, , $rhdrs, $rbody, $rerr ] = $this->client->run( $req ); |
| 215 | $res = ( $rcode === 200 || $rcode === 201 || $rcode === 204 ); |
| 216 | if ( !$res ) { |
| 217 | $this->handleError( 'Failed to store {cacheKey}', $rcode, $rerr, $rhdrs, $rbody, |
| 218 | [ 'cacheKey' => $key ] ); |
| 219 | } |
| 220 | |
| 221 | $this->updateOpStats( self::METRIC_OP_SET, [ $key => [ strlen( $rbody ), 0 ] ] ); |
| 222 | |
| 223 | return $res; |
| 224 | } |
| 225 | |
| 226 | protected function doAdd( $key, $value, $exptime = 0, $flags = 0 ) { |
| 227 | // NOTE: This is non-atomic |
| 228 | if ( $this->get( $key ) === false ) { |
| 229 | return $this->set( $key, $value, $exptime, $flags ); |
| 230 | } |
| 231 | |
| 232 | // key already set |
| 233 | return false; |
| 234 | } |
| 235 | |
| 236 | protected function doDelete( $key, $flags = 0 ) { |
| 237 | $req = [ |
| 238 | 'method' => 'DELETE', |
| 239 | 'url' => $this->url . rawurlencode( $key ), |
| 240 | 'headers' => $this->httpParams['deleteHeaders'], |
| 241 | ]; |
| 242 | |
| 243 | [ $rcode, , $rhdrs, $rbody, $rerr ] = $this->client->run( $req ); |
| 244 | $res = in_array( $rcode, [ 200, 204, 205, 404, 410 ] ); |
| 245 | if ( !$res ) { |
| 246 | $this->handleError( 'Failed to delete {cacheKey}', $rcode, $rerr, $rhdrs, $rbody, |
| 247 | [ 'cacheKey' => $key ] ); |
| 248 | } |
| 249 | |
| 250 | $this->updateOpStats( self::METRIC_OP_DELETE, [ $key ] ); |
| 251 | |
| 252 | return $res; |
| 253 | } |
| 254 | |
| 255 | protected function doIncrWithInit( $key, $exptime, $step, $init, $flags ) { |
| 256 | // NOTE: This is non-atomic |
| 257 | $curValue = $this->doGet( $key ); |
| 258 | if ( $curValue === false ) { |
| 259 | $newValue = $this->doSet( $key, $init, $exptime ) ? $init : false; |
| 260 | } elseif ( $this->isInteger( $curValue ) ) { |
| 261 | $sum = max( $curValue + $step, 0 ); |
| 262 | $newValue = $this->doSet( $key, $sum, $exptime ) ? $sum : false; |
| 263 | } else { |
| 264 | $newValue = false; |
| 265 | } |
| 266 | |
| 267 | return $newValue; |
| 268 | } |
| 269 | |
| 270 | /** |
| 271 | * Processes the response body. |
| 272 | * |
| 273 | * @param string $body request body to process |
| 274 | * @return mixed|bool the processed body, or false on error |
| 275 | */ |
| 276 | private function decodeBody( $body ) { |
| 277 | $pieces = explode( '.', $body, 3 ); |
| 278 | if ( count( $pieces ) !== 3 || $pieces[0] !== $this->serializationType ) { |
| 279 | return false; |
| 280 | } |
| 281 | [ , $hmac, $serialized ] = $pieces; |
| 282 | if ( $this->hmacKey !== '' ) { |
| 283 | $checkHmac = hash_hmac( 'sha256', $serialized, $this->hmacKey, true ); |
| 284 | if ( !hash_equals( $checkHmac, base64_decode( $hmac ) ) ) { |
| 285 | return false; |
| 286 | } |
| 287 | } |
| 288 | |
| 289 | switch ( $this->serializationType ) { |
| 290 | case 'JSON': |
| 291 | $value = json_decode( $serialized, true ); |
| 292 | return ( json_last_error() === JSON_ERROR_NONE ) ? $value : false; |
| 293 | |
| 294 | case 'PHP': |
| 295 | return unserialize( $serialized ); |
| 296 | |
| 297 | default: |
| 298 | throw new \DomainException( |
| 299 | "Unknown serialization type: $this->serializationType" |
| 300 | ); |
| 301 | } |
| 302 | } |
| 303 | |
| 304 | /** |
| 305 | * Prepares the request body (the "value" portion of our key/value store) for transmission. |
| 306 | * |
| 307 | * @param string $body request body to prepare |
| 308 | * @return string the prepared body |
| 309 | * @throws LogicException |
| 310 | */ |
| 311 | private function encodeBody( $body ) { |
| 312 | switch ( $this->serializationType ) { |
| 313 | case 'JSON': |
| 314 | $value = json_encode( $body ); |
| 315 | if ( $value === false ) { |
| 316 | throw new InvalidArgumentException( __METHOD__ . ": body could not be encoded." ); |
| 317 | } |
| 318 | break; |
| 319 | |
| 320 | case 'PHP': |
| 321 | $value = serialize( $body ); |
| 322 | break; |
| 323 | |
| 324 | default: |
| 325 | throw new \DomainException( |
| 326 | "Unknown serialization type: $this->serializationType" |
| 327 | ); |
| 328 | } |
| 329 | |
| 330 | if ( $this->hmacKey !== '' ) { |
| 331 | $hmac = base64_encode( |
| 332 | hash_hmac( 'sha256', $value, $this->hmacKey, true ) |
| 333 | ); |
| 334 | } else { |
| 335 | $hmac = ''; |
| 336 | } |
| 337 | return $this->serializationType . '.' . $hmac . '.' . $value; |
| 338 | } |
| 339 | |
| 340 | /** |
| 341 | * Handle storage error |
| 342 | * |
| 343 | * @param string $msg Error message |
| 344 | * @param int $rcode Error code from client |
| 345 | * @param string $rerr Error message from client |
| 346 | * @param array $rhdrs Response headers |
| 347 | * @param string $rbody Error body from client (if any) |
| 348 | * @param array $context Error context for PSR-3 logging |
| 349 | */ |
| 350 | protected function handleError( $msg, $rcode, $rerr, $rhdrs, $rbody, $context = [] ) { |
| 351 | $message = "$msg : ({code}) {error}"; |
| 352 | $context = [ |
| 353 | 'code' => $rcode, |
| 354 | 'error' => $rerr |
| 355 | ] + $context; |
| 356 | |
| 357 | if ( $this->extendedErrorBodyFields !== [] ) { |
| 358 | $body = $this->decodeBody( $rbody ); |
| 359 | if ( $body ) { |
| 360 | $extraFields = ''; |
| 361 | foreach ( $this->extendedErrorBodyFields as $field ) { |
| 362 | if ( isset( $body[$field] ) ) { |
| 363 | $extraFields .= " : ({$field}) {$body[$field]}"; |
| 364 | } |
| 365 | } |
| 366 | if ( $extraFields !== '' ) { |
| 367 | $message .= " {extra_fields}"; |
| 368 | $context['extra_fields'] = $extraFields; |
| 369 | } |
| 370 | } |
| 371 | } |
| 372 | |
| 373 | $this->logger->error( $message, $context ); |
| 374 | $this->setLastError( $rcode === 0 ? self::ERR_UNREACHABLE : self::ERR_UNEXPECTED ); |
| 375 | } |
| 376 | } |