MediaWiki  master
EtcdConfig.php
Go to the documentation of this file.
1 <?php
25 
31 class EtcdConfig implements Config, LoggerAwareInterface {
33  private $http;
35  private $srvCache;
37  private $procCache;
39  private $logger;
40 
42  private $host;
44  private $protocol;
46  private $directory;
48  private $encoding;
50  private $baseCacheTTL;
52  private $skewCacheTTL;
54  private $timeout;
55 
68  public function __construct( array $params ) {
69  $params += [
70  'protocol' => 'http',
71  'encoding' => 'JSON',
72  'cacheTTL' => 10,
73  'skewTTL' => 1,
74  'timeout' => 2
75  ];
76 
77  $this->host = $params['host'];
78  $this->protocol = $params['protocol'];
79  $this->directory = trim( $params['directory'], '/' );
80  $this->encoding = $params['encoding'];
81  $this->skewCacheTTL = $params['skewTTL'];
82  $this->baseCacheTTL = max( $params['cacheTTL'] - $this->skewCacheTTL, 0 );
83  $this->timeout = $params['timeout'];
84 
85  if ( !isset( $params['cache'] ) ) {
86  $this->srvCache = new HashBagOStuff();
87  } elseif ( $params['cache'] instanceof BagOStuff ) {
88  $this->srvCache = $params['cache'];
89  } else {
90  $this->srvCache = ObjectFactory::getObjectFromSpec( $params['cache'] );
91  }
92 
93  $this->logger = new Psr\Log\NullLogger();
94  $this->http = new MultiHttpClient( [
95  'connTimeout' => $this->timeout,
96  'reqTimeout' => $this->timeout,
97  'logger' => $this->logger
98  ] );
99  }
100 
101  public function setLogger( LoggerInterface $logger ) {
102  $this->logger = $logger;
103  $this->http->setLogger( $logger );
104  }
105 
106  public function has( $name ) {
107  $this->load();
108 
109  return array_key_exists( $name, $this->procCache['config'] );
110  }
111 
112  public function get( $name ) {
113  $this->load();
114 
115  if ( !array_key_exists( $name, $this->procCache['config'] ) ) {
116  throw new ConfigException( "No entry found for '$name'." );
117  }
118 
119  return $this->procCache['config'][$name];
120  }
121 
122  public function getModifiedIndex() {
123  $this->load();
124  return $this->procCache['modifiedIndex'];
125  }
126 
130  private function load() {
131  if ( $this->procCache !== null ) {
132  return; // already loaded
133  }
134 
135  $now = microtime( true );
136  $key = $this->srvCache->makeGlobalKey(
137  __CLASS__,
138  $this->host,
139  $this->directory
140  );
141 
142  // Get the cached value or block until it is regenerated (by this or another thread)...
143  $data = null; // latest config info
144  $error = null; // last error message
145  $loop = new WaitConditionLoop(
146  function () use ( $key, $now, &$data, &$error ) {
147  // Check if the values are in cache yet...
148  $data = $this->srvCache->get( $key );
149  if ( is_array( $data ) && $data['expires'] > $now ) {
150  $this->logger->debug( "Found up-to-date etcd configuration cache." );
151 
152  return WaitConditionLoop::CONDITION_REACHED;
153  }
154 
155  // Cache is either empty or stale;
156  // refresh the cache from etcd, using a mutex to reduce stampedes...
157  if ( $this->srvCache->lock( $key, 0, $this->baseCacheTTL ) ) {
158  try {
159  $etcdResponse = $this->fetchAllFromEtcd();
160  $error = $etcdResponse['error'];
161  if ( is_array( $etcdResponse['config'] ) ) {
162  // Avoid having all servers expire cache keys at the same time
163  $expiry = microtime( true ) + $this->baseCacheTTL;
164  // @phan-suppress-next-line PhanTypeMismatchArgumentInternal
165  $expiry += mt_rand( 0, 1e6 ) / 1e6 * $this->skewCacheTTL;
166  $data = [
167  'config' => $etcdResponse['config'],
168  'expires' => $expiry,
169  'modifiedIndex' => $etcdResponse['modifiedIndex']
170  ];
171  $this->srvCache->set( $key, $data, BagOStuff::TTL_INDEFINITE );
172 
173  $this->logger->info( "Refreshed stale etcd configuration cache." );
174 
175  return WaitConditionLoop::CONDITION_REACHED;
176  } else {
177  $this->logger->error( "Failed to fetch configuration: $error" );
178  if ( !$etcdResponse['retry'] ) {
179  // Fail fast since the error is likely to keep happening
180  return WaitConditionLoop::CONDITION_FAILED;
181  }
182  }
183  } finally {
184  $this->srvCache->unlock( $key ); // release mutex
185  }
186  }
187 
188  if ( is_array( $data ) ) {
189  $this->logger->info( "Using stale etcd configuration cache." );
190 
191  return WaitConditionLoop::CONDITION_REACHED;
192  }
193 
194  return WaitConditionLoop::CONDITION_CONTINUE;
195  },
197  );
198 
199  if ( $loop->invoke() !== WaitConditionLoop::CONDITION_REACHED ) {
200  // No cached value exists and etcd query failed; throw an error
201  throw new ConfigException( "Failed to load configuration from etcd: $error" );
202  }
203 
204  $this->procCache = $data;
205  }
206 
210  public function fetchAllFromEtcd() {
211  // TODO: inject DnsSrvDiscoverer in order to be able to test this method
212  $dsd = new DnsSrvDiscoverer( $this->host );
213  $servers = $dsd->getServers();
214  if ( !$servers ) {
215  return $this->fetchAllFromEtcdServer( $this->host );
216  }
217 
218  do {
219  // Pick a random etcd server from dns
220  $server = $dsd->pickServer( $servers );
221  $host = IP::combineHostAndPort( $server['target'], $server['port'] );
222  // Try to load the config from this particular server
224  if ( is_array( $response['config'] ) || $response['retry'] ) {
225  break;
226  }
227 
228  // Avoid the server next time if that failed
229  $servers = $dsd->removeServer( $server, $servers );
230  } while ( $servers );
231 
232  return $response;
233  }
234 
239  protected function fetchAllFromEtcdServer( $address ) {
240  // Retrieve all the values under the MediaWiki config directory
241  list( $rcode, $rdesc, /* $rhdrs */, $rbody, $rerr ) = $this->http->run( [
242  'method' => 'GET',
243  'url' => "{$this->protocol}://{$address}/v2/keys/{$this->directory}/?recursive=true",
244  'headers' => [ 'content-type' => 'application/json' ]
245  ] );
246 
247  $response = [ 'config' => null, 'error' => null, 'retry' => false, 'modifiedIndex' => 0 ];
248 
249  static $terminalCodes = [ 404 => true ];
250  if ( $rcode < 200 || $rcode > 399 ) {
251  $response['error'] = strlen( $rerr ) ? $rerr : "HTTP $rcode ($rdesc)";
252  $response['retry'] = empty( $terminalCodes[$rcode] );
253  return $response;
254  }
255 
256  try {
257  $parsedResponse = $this->parseResponse( $rbody );
258  } catch ( EtcdConfigParseError $e ) {
259  $parsedResponse = [ 'error' => $e->getMessage() ];
260  }
261  return array_merge( $response, $parsedResponse );
262  }
263 
270  protected function parseResponse( $rbody ) {
271  $info = json_decode( $rbody, true );
272  if ( $info === null ) {
273  throw new EtcdConfigParseError( "Error unserializing JSON response." );
274  }
275  if ( !isset( $info['node'] ) || !is_array( $info['node'] ) ) {
276  throw new EtcdConfigParseError(
277  "Unexpected JSON response: Missing or invalid node at top level." );
278  }
279  $config = [];
280  $lastModifiedIndex = $this->parseDirectory( '', $info['node'], $config );
281  return [ 'modifiedIndex' => $lastModifiedIndex, 'config' => $config ];
282  }
283 
293  protected function parseDirectory( $dirName, $dirNode, &$config ) {
294  $lastModifiedIndex = 0;
295  if ( !isset( $dirNode['nodes'] ) ) {
296  throw new EtcdConfigParseError(
297  "Unexpected JSON response in dir '$dirName'; missing 'nodes' list." );
298  }
299  if ( !is_array( $dirNode['nodes'] ) ) {
300  throw new EtcdConfigParseError(
301  "Unexpected JSON response in dir '$dirName'; 'nodes' is not an array." );
302  }
303 
304  foreach ( $dirNode['nodes'] as $node ) {
305  '@phan-var array $node';
306  $baseName = basename( $node['key'] );
307  $fullName = $dirName === '' ? $baseName : "$dirName/$baseName";
308  if ( !empty( $node['dir'] ) ) {
309  $lastModifiedIndex = max(
310  $this->parseDirectory( $fullName, $node, $config ),
311  $lastModifiedIndex );
312  } else {
313  $value = $this->unserialize( $node['value'] );
314  if ( !is_array( $value ) || !array_key_exists( 'val', $value ) ) {
315  throw new EtcdConfigParseError( "Failed to parse value for '$fullName'." );
316  }
317  $lastModifiedIndex = max( $node['modifiedIndex'], $lastModifiedIndex );
318  $config[$fullName] = $value['val'];
319  }
320  }
321  return $lastModifiedIndex;
322  }
323 
328  private function unserialize( $string ) {
329  if ( $this->encoding === 'YAML' ) {
330  return yaml_parse( $string );
331  } else {
332  return json_decode( $string, true );
333  }
334  }
335 }
Interface for configuration instances.
Definition: EtcdConfig.php:31
array $procCache
Definition: EtcdConfig.php:37
$response
int $skewCacheTTL
Definition: EtcdConfig.php:52
LoggerInterface $logger
Definition: EtcdConfig.php:39
int $baseCacheTTL
Definition: EtcdConfig.php:50
parseDirectory( $dirName, $dirNode, &$config)
Recursively parse a directory node and populate the array passed by reference, throwing EtcdConfigPar...
Definition: EtcdConfig.php:293
static combineHostAndPort( $host, $port, $defaultPort=false)
Given a host name and a port, combine them into host/port string like you might find in a URL...
Definition: IP.php:302
string $protocol
Definition: EtcdConfig.php:44
BagOStuff $srvCache
Definition: EtcdConfig.php:35
string $host
Definition: EtcdConfig.php:42
Exceptions for config failures.
$fullName
getModifiedIndex()
Definition: EtcdConfig.php:122
has( $name)
Check whether a configuration option is set for the given name.
Definition: EtcdConfig.php:106
int $timeout
Definition: EtcdConfig.php:54
Interface for configuration instances.
Definition: Config.php:28
string $encoding
Definition: EtcdConfig.php:48
parseResponse( $rbody)
Parse a response body, throwing EtcdConfigParseError if there is a validation error.
Definition: EtcdConfig.php:270
MultiHttpClient $http
Definition: EtcdConfig.php:33
__construct(array $params)
Definition: EtcdConfig.php:68
fetchAllFromEtcd()
Definition: EtcdConfig.php:210
fetchAllFromEtcdServer( $address)
Definition: EtcdConfig.php:239
string $directory
Definition: EtcdConfig.php:46
setLogger(LoggerInterface $logger)
Definition: EtcdConfig.php:101
Class to handle multiple HTTP requests.
return true
Definition: router.php:92
unserialize( $string)
Definition: EtcdConfig.php:328