Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
Total | |
67.91% |
91 / 134 |
|
20.00% |
2 / 10 |
CRAP | |
0.00% |
0 / 1 |
RESTBagOStuff | |
68.42% |
91 / 133 |
|
20.00% |
2 / 10 |
124.61 | |
0.00% |
0 / 1 |
__construct | |
60.87% |
14 / 23 |
|
0.00% |
0 / 1 |
6.50 | |||
setLogger | |
100.00% |
2 / 2 |
|
100.00% |
1 / 1 |
1 | |||
doGet | |
100.00% |
20 / 20 |
|
100.00% |
1 / 1 |
8 | |||
doSet | |
84.62% |
11 / 13 |
|
0.00% |
0 / 1 |
4.06 | |||
doAdd | |
0.00% |
0 / 3 |
|
0.00% |
0 / 1 |
6 | |||
doDelete | |
83.33% |
10 / 12 |
|
0.00% |
0 / 1 |
2.02 | |||
doIncrWithInit | |
0.00% |
0 / 8 |
|
0.00% |
0 / 1 |
30 | |||
decodeBody | |
70.59% |
12 / 17 |
|
0.00% |
0 / 1 |
11.06 | |||
encodeBody | |
77.78% |
14 / 18 |
|
0.00% |
0 / 1 |
6.40 | |||
handleError | |
47.06% |
8 / 17 |
|
0.00% |
0 / 1 |
14.27 |
1 | <?php |
2 | |
3 | namespace Wikimedia\ObjectCache; |
4 | |
5 | use InvalidArgumentException; |
6 | use LogicException; |
7 | use Psr\Log\LoggerInterface; |
8 | use Wikimedia\Http\MultiHttpClient; |
9 | |
10 | /** |
11 | * Interface to key-value storage behind an HTTP server. |
12 | * |
13 | * ### Important caveats |
14 | * |
15 | * This interface is currently an incomplete BagOStuff implementation, |
16 | * supported only for use with MediaWiki features that accept a dedicated |
17 | * cache type to use for a narrow set of cache keys that share the same |
18 | * key expiry and replication requirements, and where the key-value server |
19 | * in question is statically configured with domain knowledge of said |
20 | * key expiry and replication requirements. |
21 | * |
22 | * Specifically, RESTBagOStuff has the following limitations: |
23 | * |
24 | * - The expiry parameter is ignored in methods like `set()`. |
25 | * |
26 | * There is not currently an agreed protocol for sending this to a |
27 | * server. This class is written for use with MediaWiki\Session\SessionManager |
28 | * and Kask/Cassanda at WMF, which does not expose a customizable expiry. |
29 | * |
30 | * As such, it is not recommended to use RESTBagOStuff to back a general |
31 | * purpose cache type (such as MediaWiki's main cache, or main stash). |
32 | * Instead, it is only supported toMediaWiki features where a cache type can |
33 | * be pointed for a narrow set of keys that naturally share the same TTL |
34 | * anyway, or where the feature behaves correctly even if the logical expiry |
35 | * is longer than specified (e.g. immutable keys, or value verification) |
36 | * |
37 | * - Most methods are non-atomic. |
38 | * |
39 | * The class should only be used for get, set, and delete operations. |
40 | * Advanced methods like `incr()`, `add()` and `lock()` do exist but |
41 | * inherit a native and best-effort implementation based on get+set. |
42 | * These should not be relied upon. |
43 | * |
44 | * ### Backend requirements |
45 | * |
46 | * The HTTP server will receive requests for URLs like `{baseURL}/{KEY}`. It |
47 | * must implement the GET, PUT and DELETE methods. |
48 | * |
49 | * E.g., when the base URL is `/sessions/v1`, then `set()` will: |
50 | * |
51 | * `PUT /sessions/v1/mykeyhere` |
52 | * |
53 | * and `get()` would do: |
54 | * |
55 | * `GET /sessions/v1/mykeyhere` |
56 | * |
57 | * and `delete()` would do: |
58 | * |
59 | * `DELETE /sessions/v1/mykeyhere` |
60 | * |
61 | * ### Example configuration |
62 | * |
63 | * Minimal generic configuration: |
64 | * |
65 | * @code |
66 | * $wgObjectCaches['sessions'] = array( |
67 | * 'class' => 'RESTBagOStuff', |
68 | * 'url' => 'http://localhost:7231/example/' |
69 | * ); |
70 | * @endcode |
71 | * |
72 | * |
73 | * Configuration for [Kask](https://www.mediawiki.org/wiki/Kask) session store: |
74 | * @code |
75 | * $wgObjectCaches['sessions'] = array( |
76 | * 'class' => 'RESTBagOStuff', |
77 | * 'url' => 'https://kaskhost:1234/sessions/v1/', |
78 | * 'httpParams' => [ |
79 | * 'readHeaders' => [], |
80 | * 'writeHeaders' => [ 'content-type' => 'application/octet-stream' ], |
81 | * 'deleteHeaders' => [], |
82 | * 'writeMethod' => 'POST', |
83 | * ], |
84 | * 'serialization_type' => 'JSON', |
85 | * 'extendedErrorBodyFields' => [ 'type', 'title', 'detail', 'instance' ] |
86 | * ); |
87 | * $wgSessionCacheType = 'sessions'; |
88 | * @endcode |
89 | */ |
90 | class RESTBagOStuff extends MediumSpecificBagOStuff { |
91 | /** |
92 | * Default connection timeout in seconds. The kernel retransmits the SYN |
93 | * packet after 1 second, so 1.2 seconds allows for 1 retransmit without |
94 | * permanent failure. |
95 | */ |
96 | private const DEFAULT_CONN_TIMEOUT = 1.2; |
97 | |
98 | /** |
99 | * Default request timeout |
100 | */ |
101 | private const DEFAULT_REQ_TIMEOUT = 3.0; |
102 | |
103 | /** |
104 | * @var MultiHttpClient |
105 | */ |
106 | private $client; |
107 | |
108 | /** |
109 | * REST URL to use for storage. |
110 | * |
111 | * @var string |
112 | */ |
113 | private $url; |
114 | |
115 | /** |
116 | * HTTP parameters: readHeaders, writeHeaders, deleteHeaders, writeMethod. |
117 | * |
118 | * @var array |
119 | */ |
120 | private $httpParams; |
121 | |
122 | /** |
123 | * Optional serialization type to use. Allowed values: "PHP", "JSON". |
124 | * |
125 | * @var string |
126 | */ |
127 | private $serializationType; |
128 | |
129 | /** |
130 | * Optional HMAC Key for protecting the serialized blob. If omitted no protection is done |
131 | * |
132 | * @var string |
133 | */ |
134 | private $hmacKey; |
135 | |
136 | /** |
137 | * @var array additional body fields to log on error, if possible |
138 | */ |
139 | private $extendedErrorBodyFields; |
140 | |
141 | public function __construct( $params ) { |
142 | $params['segmentationSize'] ??= INF; |
143 | if ( empty( $params['url'] ) ) { |
144 | throw new InvalidArgumentException( 'URL parameter is required' ); |
145 | } |
146 | |
147 | if ( empty( $params['client'] ) ) { |
148 | // Pass through some params to the HTTP client. |
149 | $clientParams = [ |
150 | 'connTimeout' => $params['connTimeout'] ?? self::DEFAULT_CONN_TIMEOUT, |
151 | 'reqTimeout' => $params['reqTimeout'] ?? self::DEFAULT_REQ_TIMEOUT, |
152 | ]; |
153 | foreach ( [ 'caBundlePath', 'proxy', 'telemetry' ] as $key ) { |
154 | if ( isset( $params[$key] ) ) { |
155 | $clientParams[$key] = $params[$key]; |
156 | } |
157 | } |
158 | $this->client = new MultiHttpClient( $clientParams ); |
159 | } else { |
160 | $this->client = $params['client']; |
161 | } |
162 | |
163 | $this->httpParams['writeMethod'] = $params['httpParams']['writeMethod'] ?? 'PUT'; |
164 | $this->httpParams['readHeaders'] = $params['httpParams']['readHeaders'] ?? []; |
165 | $this->httpParams['writeHeaders'] = $params['httpParams']['writeHeaders'] ?? []; |
166 | $this->httpParams['deleteHeaders'] = $params['httpParams']['deleteHeaders'] ?? []; |
167 | $this->extendedErrorBodyFields = $params['extendedErrorBodyFields'] ?? []; |
168 | $this->serializationType = $params['serialization_type'] ?? 'PHP'; |
169 | $this->hmacKey = $params['hmac_key'] ?? ''; |
170 | |
171 | // The parent constructor calls setLogger() which sets the logger in $this->client |
172 | parent::__construct( $params ); |
173 | |
174 | // Make sure URL ends with / |
175 | $this->url = rtrim( $params['url'], '/' ) . '/'; |
176 | |
177 | $this->attrMap[self::ATTR_DURABILITY] = self::QOS_DURABILITY_DISK; |
178 | } |
179 | |
180 | public function setLogger( LoggerInterface $logger ) { |
181 | parent::setLogger( $logger ); |
182 | $this->client->setLogger( $logger ); |
183 | } |
184 | |
185 | protected function doGet( $key, $flags = 0, &$casToken = null ) { |
186 | $getToken = ( $casToken === self::PASS_BY_REF ); |
187 | $casToken = null; |
188 | |
189 | $req = [ |
190 | 'method' => 'GET', |
191 | 'url' => $this->url . rawurlencode( $key ), |
192 | 'headers' => $this->httpParams['readHeaders'], |
193 | ]; |
194 | |
195 | $value = false; |
196 | $valueSize = false; |
197 | [ $rcode, , $rhdrs, $rbody, $rerr ] = $this->client->run( $req ); |
198 | if ( $rcode === 200 && is_string( $rbody ) ) { |
199 | $value = $this->decodeBody( $rbody ); |
200 | $valueSize = strlen( $rbody ); |
201 | // @FIXME: use some kind of hash or UUID header as CAS token |
202 | if ( $getToken && $value !== false ) { |
203 | $casToken = $rbody; |
204 | } |
205 | } elseif ( $rcode === 0 || ( $rcode >= 400 && $rcode != 404 ) ) { |
206 | $this->handleError( 'Failed to fetch {cacheKey}', $rcode, $rerr, $rhdrs, $rbody, |
207 | [ 'cacheKey' => $key ] ); |
208 | } |
209 | |
210 | $this->updateOpStats( self::METRIC_OP_GET, [ $key => [ 0, $valueSize ] ] ); |
211 | |
212 | return $value; |
213 | } |
214 | |
215 | protected function doSet( $key, $value, $exptime = 0, $flags = 0 ) { |
216 | $req = [ |
217 | 'method' => $this->httpParams['writeMethod'], |
218 | 'url' => $this->url . rawurlencode( $key ), |
219 | 'body' => $this->encodeBody( $value ), |
220 | 'headers' => $this->httpParams['writeHeaders'], |
221 | ]; |
222 | |
223 | [ $rcode, , $rhdrs, $rbody, $rerr ] = $this->client->run( $req ); |
224 | $res = ( $rcode === 200 || $rcode === 201 || $rcode === 204 ); |
225 | if ( !$res ) { |
226 | $this->handleError( 'Failed to store {cacheKey}', $rcode, $rerr, $rhdrs, $rbody, |
227 | [ 'cacheKey' => $key ] ); |
228 | } |
229 | |
230 | $this->updateOpStats( self::METRIC_OP_SET, [ $key => [ strlen( $rbody ), 0 ] ] ); |
231 | |
232 | return $res; |
233 | } |
234 | |
235 | protected function doAdd( $key, $value, $exptime = 0, $flags = 0 ) { |
236 | // NOTE: This is non-atomic |
237 | if ( $this->get( $key ) === false ) { |
238 | return $this->set( $key, $value, $exptime, $flags ); |
239 | } |
240 | |
241 | // key already set |
242 | return false; |
243 | } |
244 | |
245 | protected function doDelete( $key, $flags = 0 ) { |
246 | $req = [ |
247 | 'method' => 'DELETE', |
248 | 'url' => $this->url . rawurlencode( $key ), |
249 | 'headers' => $this->httpParams['deleteHeaders'], |
250 | ]; |
251 | |
252 | [ $rcode, , $rhdrs, $rbody, $rerr ] = $this->client->run( $req ); |
253 | $res = in_array( $rcode, [ 200, 204, 205, 404, 410 ] ); |
254 | if ( !$res ) { |
255 | $this->handleError( 'Failed to delete {cacheKey}', $rcode, $rerr, $rhdrs, $rbody, |
256 | [ 'cacheKey' => $key ] ); |
257 | } |
258 | |
259 | $this->updateOpStats( self::METRIC_OP_DELETE, [ $key ] ); |
260 | |
261 | return $res; |
262 | } |
263 | |
264 | protected function doIncrWithInit( $key, $exptime, $step, $init, $flags ) { |
265 | // NOTE: This is non-atomic |
266 | $curValue = $this->doGet( $key ); |
267 | if ( $curValue === false ) { |
268 | $newValue = $this->doSet( $key, $init, $exptime ) ? $init : false; |
269 | } elseif ( $this->isInteger( $curValue ) ) { |
270 | $sum = max( $curValue + $step, 0 ); |
271 | $newValue = $this->doSet( $key, $sum, $exptime ) ? $sum : false; |
272 | } else { |
273 | $newValue = false; |
274 | } |
275 | |
276 | return $newValue; |
277 | } |
278 | |
279 | /** |
280 | * Processes the response body. |
281 | * |
282 | * @param string $body request body to process |
283 | * |
284 | * @return mixed|bool the processed body, or false on error |
285 | */ |
286 | private function decodeBody( $body ) { |
287 | $pieces = explode( '.', $body, 3 ); |
288 | if ( count( $pieces ) !== 3 || $pieces[0] !== $this->serializationType ) { |
289 | return false; |
290 | } |
291 | [ , $hmac, $serialized ] = $pieces; |
292 | if ( $this->hmacKey !== '' ) { |
293 | $checkHmac = hash_hmac( 'sha256', $serialized, $this->hmacKey, true ); |
294 | if ( !hash_equals( $checkHmac, base64_decode( $hmac ) ) ) { |
295 | return false; |
296 | } |
297 | } |
298 | |
299 | switch ( $this->serializationType ) { |
300 | case 'JSON': |
301 | $value = json_decode( $serialized, true ); |
302 | return ( json_last_error() === JSON_ERROR_NONE ) ? $value : false; |
303 | |
304 | case 'PHP': |
305 | return unserialize( $serialized ); |
306 | |
307 | default: |
308 | throw new \DomainException( |
309 | "Unknown serialization type: $this->serializationType" |
310 | ); |
311 | } |
312 | } |
313 | |
314 | /** |
315 | * Prepares the request body (the "value" portion of our key/value store) for transmission. |
316 | * |
317 | * @param string $body request body to prepare |
318 | * |
319 | * @return string the prepared body |
320 | * @throws LogicException |
321 | */ |
322 | private function encodeBody( $body ) { |
323 | switch ( $this->serializationType ) { |
324 | case 'JSON': |
325 | $value = json_encode( $body ); |
326 | if ( $value === false ) { |
327 | throw new InvalidArgumentException( __METHOD__ . ": body could not be encoded." ); |
328 | } |
329 | break; |
330 | |
331 | case 'PHP': |
332 | $value = serialize( $body ); |
333 | break; |
334 | |
335 | default: |
336 | throw new \DomainException( |
337 | "Unknown serialization type: $this->serializationType" |
338 | ); |
339 | } |
340 | |
341 | if ( $this->hmacKey !== '' ) { |
342 | $hmac = base64_encode( |
343 | hash_hmac( 'sha256', $value, $this->hmacKey, true ) |
344 | ); |
345 | } else { |
346 | $hmac = ''; |
347 | } |
348 | return $this->serializationType . '.' . $hmac . '.' . $value; |
349 | } |
350 | |
351 | /** |
352 | * Handle storage error |
353 | * |
354 | * @param string $msg Error message |
355 | * @param int $rcode Error code from client |
356 | * @param string $rerr Error message from client |
357 | * @param array $rhdrs Response headers |
358 | * @param string $rbody Error body from client (if any) |
359 | * @param array $context Error context for PSR-3 logging |
360 | */ |
361 | protected function handleError( $msg, $rcode, $rerr, $rhdrs, $rbody, $context = [] ) { |
362 | $message = "$msg : ({code}) {error}"; |
363 | $context = [ |
364 | 'code' => $rcode, |
365 | 'error' => $rerr |
366 | ] + $context; |
367 | |
368 | if ( $this->extendedErrorBodyFields !== [] ) { |
369 | $body = $this->decodeBody( $rbody ); |
370 | if ( $body ) { |
371 | $extraFields = ''; |
372 | foreach ( $this->extendedErrorBodyFields as $field ) { |
373 | if ( isset( $body[$field] ) ) { |
374 | $extraFields .= " : ({$field}) {$body[$field]}"; |
375 | } |
376 | } |
377 | if ( $extraFields !== '' ) { |
378 | $message .= " {extra_fields}"; |
379 | $context['extra_fields'] = $extraFields; |
380 | } |
381 | } |
382 | } |
383 | |
384 | $this->logger->error( $message, $context ); |
385 | $this->setLastError( $rcode === 0 ? self::ERR_UNREACHABLE : self::ERR_UNEXPECTED ); |
386 | } |
387 | } |
388 | |
389 | /** @deprecated class alias since 1.43 */ |
390 | class_alias( RESTBagOStuff::class, 'RESTBagOStuff' ); |