MediaWiki  master
RESTBagOStuff.php
Go to the documentation of this file.
1 <?php
2 
3 use Psr\Log\LoggerInterface;
4 
54  private const DEFAULT_CONN_TIMEOUT = 1.2;
55 
59  private const DEFAULT_REQ_TIMEOUT = 3.0;
60 
64  private $client;
65 
70  private $url;
71 
75  private $httpParams;
76 
82 
87  private $hmacKey;
88 
93 
94  public function __construct( $params ) {
95  $params['segmentationSize'] = $params['segmentationSize'] ?? INF;
96  if ( empty( $params['url'] ) ) {
97  throw new InvalidArgumentException( 'URL parameter is required' );
98  }
99 
100  if ( empty( $params['client'] ) ) {
101  // Pass through some params to the HTTP client.
102  $clientParams = [
103  'connTimeout' => $params['connTimeout'] ?? self::DEFAULT_CONN_TIMEOUT,
104  'reqTimeout' => $params['reqTimeout'] ?? self::DEFAULT_REQ_TIMEOUT,
105  ];
106  foreach ( [ 'caBundlePath', 'proxy' ] as $key ) {
107  if ( isset( $params[$key] ) ) {
108  $clientParams[$key] = $params[$key];
109  }
110  }
111  $this->client = new MultiHttpClient( $clientParams );
112  } else {
113  $this->client = $params['client'];
114  }
115 
116  $this->httpParams['writeMethod'] = $params['httpParams']['writeMethod'] ?? 'PUT';
117  $this->httpParams['readHeaders'] = $params['httpParams']['readHeaders'] ?? [];
118  $this->httpParams['writeHeaders'] = $params['httpParams']['writeHeaders'] ?? [];
119  $this->httpParams['deleteHeaders'] = $params['httpParams']['deleteHeaders'] ?? [];
120  $this->extendedErrorBodyFields = $params['extendedErrorBodyFields'] ?? [];
121  $this->serializationType = $params['serialization_type'] ?? 'PHP';
122  $this->hmacKey = $params['hmac_key'] ?? '';
123 
124  // The parent constructor calls setLogger() which sets the logger in $this->client
125  parent::__construct( $params );
126 
127  // Make sure URL ends with /
128  $this->url = rtrim( $params['url'], '/' ) . '/';
129 
131  }
132 
133  public function setLogger( LoggerInterface $logger ) {
134  parent::setLogger( $logger );
135  $this->client->setLogger( $logger );
136  }
137 
138  protected function doGet( $key, $flags = 0, &$casToken = null ) {
139  $getToken = ( $casToken === self::PASS_BY_REF );
140  $casToken = null;
141 
142  $req = [
143  'method' => 'GET',
144  'url' => $this->url . rawurlencode( $key ),
145  'headers' => $this->httpParams['readHeaders'],
146  ];
147 
148  $value = false;
149  $valueSize = false;
150  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->client->run( $req );
151  if ( $rcode === 200 && is_string( $rbody ) ) {
152  $value = $this->decodeBody( $rbody );
153  $valueSize = strlen( $rbody );
154  // @FIXME: use some kind of hash or UUID header as CAS token
155  if ( $getToken && $value !== false ) {
156  $casToken = $rbody;
157  }
158  } elseif ( $rcode === 0 || ( $rcode >= 400 && $rcode != 404 ) ) {
159  $this->handleError( "Failed to fetch $key", $rcode, $rerr, $rhdrs, $rbody );
160  }
161 
162  $this->updateOpStats( self::METRIC_OP_GET, [ $key => [ null, $valueSize ] ] );
163 
164  return $value;
165  }
166 
167  protected function doSet( $key, $value, $exptime = 0, $flags = 0 ) {
168  // @TODO: respect WRITE_SYNC (e.g. EACH_QUORUM)
169  // @TODO: respect $exptime
170  $req = [
171  'method' => $this->httpParams['writeMethod'],
172  'url' => $this->url . rawurlencode( $key ),
173  'body' => $this->encodeBody( $value ),
174  'headers' => $this->httpParams['writeHeaders'],
175  ];
176 
177  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->client->run( $req );
178  $res = ( $rcode === 200 || $rcode === 201 || $rcode === 204 );
179  if ( !$res ) {
180  $this->handleError( "Failed to store $key", $rcode, $rerr, $rhdrs, $rbody );
181  }
182 
183  $this->updateOpStats( self::METRIC_OP_SET, [ $key => [ strlen( $rbody ), null ] ] );
184 
185  return $res;
186  }
187 
188  protected function doAdd( $key, $value, $exptime = 0, $flags = 0 ) {
189  // @TODO: make this atomic
190  if ( $this->get( $key ) === false ) {
191  return $this->set( $key, $value, $exptime, $flags );
192  }
193 
194  return false; // key already set
195  }
196 
197  protected function doDelete( $key, $flags = 0 ) {
198  // @TODO: respect WRITE_SYNC (e.g. EACH_QUORUM)
199  $req = [
200  'method' => 'DELETE',
201  'url' => $this->url . rawurlencode( $key ),
202  'headers' => $this->httpParams['deleteHeaders'],
203  ];
204 
205  list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->client->run( $req );
206  $res = in_array( $rcode, [ 200, 204, 205, 404, 410 ] );
207  if ( !$res ) {
208  $this->handleError( "Failed to delete $key", $rcode, $rerr, $rhdrs, $rbody );
209  }
210 
211  $this->updateOpStats( self::METRIC_OP_DELETE, [ $key ] );
212 
213  return $res;
214  }
215 
216  public function incr( $key, $value = 1, $flags = 0 ) {
217  // @TODO: make this atomic
218  $n = $this->get( $key, self::READ_LATEST );
219  if ( $this->isInteger( $n ) ) { // key exists?
220  $n = max( $n + (int)$value, 0 );
221  // @TODO: respect $exptime
222  return $this->set( $key, $n ) ? $n : false;
223  }
224 
225  return false;
226  }
227 
228  public function decr( $key, $value = 1, $flags = 0 ) {
229  return $this->incr( $key, -$value, $flags );
230  }
231 
232  public function makeKeyInternal( $keyspace, $components ) {
233  return $this->genericKeyFromComponents( $keyspace, ...$components );
234  }
235 
236  protected function convertGenericKey( $key ) {
237  return $key; // short-circuit; already uses "generic" keys
238  }
239 
246  private function decodeBody( $body ) {
247  $pieces = explode( '.', $body, 3 );
248  if ( count( $pieces ) !== 3 || $pieces[0] !== $this->serializationType ) {
249  return false;
250  }
251  list( , $hmac, $serialized ) = $pieces;
252  if ( $this->hmacKey !== '' ) {
253  $checkHmac = hash_hmac( 'sha256', $serialized, $this->hmacKey, true );
254  if ( !hash_equals( $checkHmac, base64_decode( $hmac ) ) ) {
255  return false;
256  }
257  }
258 
259  switch ( $this->serializationType ) {
260  case 'JSON':
261  $value = json_decode( $serialized, true );
262  return ( json_last_error() === JSON_ERROR_NONE ) ? $value : false;
263 
264  case 'PHP':
265  return unserialize( $serialized );
266 
267  default:
268  throw new \DomainException(
269  "Unknown serialization type: $this->serializationType"
270  );
271  }
272  }
273 
281  private function encodeBody( $body ) {
282  switch ( $this->serializationType ) {
283  case 'JSON':
284  $value = json_encode( $body );
285  if ( $value === false ) {
286  throw new InvalidArgumentException( __METHOD__ . ": body could not be encoded." );
287  }
288  break;
289 
290  case 'PHP':
291  $value = serialize( $body );
292  break;
293 
294  default:
295  throw new \DomainException(
296  "Unknown serialization type: $this->serializationType"
297  );
298  }
299 
300  if ( $this->hmacKey !== '' ) {
301  $hmac = base64_encode(
302  hash_hmac( 'sha256', $value, $this->hmacKey, true )
303  );
304  } else {
305  $hmac = '';
306  }
307  return $this->serializationType . '.' . $hmac . '.' . $value;
308  }
309 
318  protected function handleError( $msg, $rcode, $rerr, $rhdrs, $rbody ) {
319  $message = "$msg : ({code}) {error}";
320  $context = [
321  'code' => $rcode,
322  'error' => $rerr
323  ];
324 
325  if ( $this->extendedErrorBodyFields !== [] ) {
326  $body = $this->decodeBody( $rbody );
327  if ( $body ) {
328  $extraFields = '';
329  foreach ( $this->extendedErrorBodyFields as $field ) {
330  if ( isset( $body[$field] ) ) {
331  $extraFields .= " : ({$field}) {$body[$field]}";
332  }
333  }
334  if ( $extraFields !== '' ) {
335  $message .= " {extra_fields}";
336  $context['extra_fields'] = $extraFields;
337  }
338  }
339  }
340 
341  $this->logger->error( $message, $context );
342  $this->setLastError( $rcode === 0 ? self::ERR_UNREACHABLE : self::ERR_UNEXPECTED );
343  }
344 }
MediumSpecificBagOStuff\setLastError
setLastError( $err)
Set the "last error" registry.
Definition: MediumSpecificBagOStuff.php:830
MediumSpecificBagOStuff\isInteger
isInteger( $value)
Check if a value is an integer.
Definition: MediumSpecificBagOStuff.php:953
MultiHttpClient
Class to handle multiple HTTP requests.
Definition: MultiHttpClient.php:55
$serialized
foreach( $res as $row) $serialized
Definition: testCompression.php:88
RESTBagOStuff\doDelete
doDelete( $key, $flags=0)
Delete an item.
Definition: RESTBagOStuff.php:197
BagOStuff\genericKeyFromComponents
genericKeyFromComponents(... $components)
At a minimum, there must be a keyspace and collection name component.
Definition: BagOStuff.php:676
RESTBagOStuff\$hmacKey
string $hmacKey
Optional HMAC Key for protecting the serialized blob.
Definition: RESTBagOStuff.php:87
RESTBagOStuff\decr
decr( $key, $value=1, $flags=0)
Decrease stored value of $key by $value while preserving its TTL.
Definition: RESTBagOStuff.php:228
MediumSpecificBagOStuff\serialize
serialize( $value)
Definition: MediumSpecificBagOStuff.php:1164
RESTBagOStuff\decodeBody
decodeBody( $body)
Processes the response body.
Definition: RESTBagOStuff.php:246
$res
$res
Definition: testCompression.php:57
RESTBagOStuff
Interface to key-value storage behind an HTTP server.
Definition: RESTBagOStuff.php:48
BagOStuff\$logger
LoggerInterface $logger
Definition: BagOStuff.php:90
RESTBagOStuff\incr
incr( $key, $value=1, $flags=0)
Increase stored value of $key by $value while preserving its TTL.
Definition: RESTBagOStuff.php:216
RESTBagOStuff\__construct
__construct( $params)
Definition: RESTBagOStuff.php:94
BagOStuff\READ_LATEST
const READ_LATEST
Bitfield constants for get()/getMulti(); these are only advisory.
Definition: BagOStuff.php:115
RESTBagOStuff\$serializationType
string $serializationType
Optional serialization type to use.
Definition: RESTBagOStuff.php:81
MediumSpecificBagOStuff
Storage medium specific cache for storing items (e.g.
Definition: MediumSpecificBagOStuff.php:34
RESTBagOStuff\handleError
handleError( $msg, $rcode, $rerr, $rhdrs, $rbody)
Handle storage error.
Definition: RESTBagOStuff.php:318
MediumSpecificBagOStuff\updateOpStats
updateOpStats(string $op, array $keyInfo)
Definition: MediumSpecificBagOStuff.php:1191
RESTBagOStuff\DEFAULT_CONN_TIMEOUT
const DEFAULT_CONN_TIMEOUT
Default connection timeout in seconds.
Definition: RESTBagOStuff.php:54
Wikimedia\LightweightObjectStore\StorageAwareness\QOS_DURABILITY_DISK
const QOS_DURABILITY_DISK
Data is saved to disk and writes do not usually block on fsync()
Definition: StorageAwareness.php:62
RESTBagOStuff\doSet
doSet( $key, $value, $exptime=0, $flags=0)
Set an item.
Definition: RESTBagOStuff.php:167
RESTBagOStuff\$httpParams
array $httpParams
http parameters: readHeaders, writeHeaders, deleteHeaders, writeMethod
Definition: RESTBagOStuff.php:75
RESTBagOStuff\$extendedErrorBodyFields
array $extendedErrorBodyFields
additional body fields to log on error, if possible
Definition: RESTBagOStuff.php:92
RESTBagOStuff\$client
MultiHttpClient $client
Definition: RESTBagOStuff.php:64
RESTBagOStuff\$url
string $url
REST URL to use for storage.
Definition: RESTBagOStuff.php:70
RESTBagOStuff\doAdd
doAdd( $key, $value, $exptime=0, $flags=0)
Insert an item if it does not already exist.
Definition: RESTBagOStuff.php:188
Wikimedia\LightweightObjectStore\StorageAwareness\ATTR_DURABILITY
const ATTR_DURABILITY
Durability of writes; see QOS_DURABILITY_* (higher means stronger)
Definition: StorageAwareness.php:45
RESTBagOStuff\convertGenericKey
convertGenericKey( $key)
Convert a "generic" reversible cache key into one for this cache.
Definition: RESTBagOStuff.php:236
RESTBagOStuff\setLogger
setLogger(LoggerInterface $logger)
Definition: RESTBagOStuff.php:133
RESTBagOStuff\DEFAULT_REQ_TIMEOUT
const DEFAULT_REQ_TIMEOUT
Default request timeout.
Definition: RESTBagOStuff.php:59
MediumSpecificBagOStuff\unserialize
unserialize( $value)
Definition: MediumSpecificBagOStuff.php:1173
RESTBagOStuff\doGet
doGet( $key, $flags=0, &$casToken=null)
Definition: RESTBagOStuff.php:138
BagOStuff\$keyspace
string $keyspace
Default keyspace; used by makeKey()
Definition: BagOStuff.php:103
RESTBagOStuff\encodeBody
encodeBody( $body)
Prepares the request body (the "value" portion of our key/value store) for transmission.
Definition: RESTBagOStuff.php:281
RESTBagOStuff\makeKeyInternal
makeKeyInternal( $keyspace, $components)
Make a cache key for the given keyspace and components.
Definition: RESTBagOStuff.php:232