Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
52.94% |
108 / 204 |
|
0.00% |
0 / 7 |
CRAP | |
0.00% |
0 / 1 |
| ElasticaErrorHandler | |
52.94% |
108 / 204 |
|
0.00% |
0 / 7 |
288.11 | |
0.00% |
0 / 1 |
| logRequestResponse | |
0.00% |
0 / 6 |
|
0.00% |
0 / 1 |
6 | |||
| extractMessage | |
0.00% |
0 / 2 |
|
0.00% |
0 / 1 |
2 | |||
| extractFullError | |
51.85% |
28 / 54 |
|
0.00% |
0 / 1 |
31.86 | |||
| classifyError | |
98.51% |
66 / 67 |
|
0.00% |
0 / 1 |
10 | |||
| isParseError | |
0.00% |
0 / 4 |
|
0.00% |
0 / 1 |
12 | |||
| extractMessageAndStatus | |
12.50% |
7 / 56 |
|
0.00% |
0 / 1 |
126.22 | |||
| formatMessage | |
46.67% |
7 / 15 |
|
0.00% |
0 / 1 |
11.46 | |||
| 1 | <?php |
| 2 | |
| 3 | namespace CirrusSearch; |
| 4 | |
| 5 | use Elastica\Exception\Bulk\ResponseException as BulkResponseException; |
| 6 | use Elastica\Exception\Connection\HttpException; |
| 7 | use Elastica\Exception\PartialShardFailureException; |
| 8 | use Elastica\Exception\ResponseException; |
| 9 | use MediaWiki\Logger\LoggerFactory; |
| 10 | use MediaWiki\Status\Status; |
| 11 | |
| 12 | /** |
| 13 | * Generic functions for extracting and reporting on errors/exceptions |
| 14 | * from Elastica. |
| 15 | */ |
| 16 | class ElasticaErrorHandler { |
| 17 | |
| 18 | public static function logRequestResponse( Connection $conn, string $message, array $context = [] ) { |
| 19 | $client = $conn->getClient(); |
| 20 | LoggerFactory::getInstance( LogChannel::DEFAULT )->info( $message, $context + [ |
| 21 | 'cluster' => $conn->getClusterName(), |
| 22 | 'elasticsearch_request' => (string)$client->getLastRequest(), |
| 23 | 'elasticsearch_response' => $client->getLastResponse() !== null ? json_encode( $client->getLastResponse()->getData() ) : "NULL", |
| 24 | ] ); |
| 25 | } |
| 26 | |
| 27 | /** |
| 28 | * @param \Elastica\Exception\ExceptionInterface $exception |
| 29 | * @return string |
| 30 | */ |
| 31 | public static function extractMessage( \Elastica\Exception\ExceptionInterface $exception ) { |
| 32 | $error = self::extractFullError( $exception ); |
| 33 | return self::formatMessage( $error ); |
| 34 | } |
| 35 | |
| 36 | /** |
| 37 | * Extract an error message from an exception thrown by Elastica. |
| 38 | * @param \Elastica\Exception\ExceptionInterface $exception exception from which to extract a message |
| 39 | * @return array structuerd error from the exception |
| 40 | */ |
| 41 | public static function extractFullError( \Elastica\Exception\ExceptionInterface $exception ): array { |
| 42 | if ( $exception instanceof BulkResponseException ) { |
| 43 | $actionReasons = []; |
| 44 | foreach ( $exception->getActionExceptions() as $actionException ) { |
| 45 | $actionReasons[] = $actionException->getMessage() . ': ' |
| 46 | . self::formatMessage( $actionException->getResponse()->getFullError() ); |
| 47 | } |
| 48 | return [ |
| 49 | 'type' => 'bulk', |
| 50 | 'reason' => $exception->getMessage(), |
| 51 | 'actionReasons' => $actionReasons, |
| 52 | ]; |
| 53 | } elseif ( $exception instanceof HttpException ) { |
| 54 | return [ |
| 55 | 'type' => 'http_exception', |
| 56 | 'reason' => $exception->getMessage() |
| 57 | ]; |
| 58 | } elseif ( !( $exception instanceof ResponseException ) ) { |
| 59 | // simulate the basic full error structure |
| 60 | return [ |
| 61 | 'type' => 'unknown', |
| 62 | 'reason' => $exception->getMessage() |
| 63 | ]; |
| 64 | } |
| 65 | if ( $exception instanceof PartialShardFailureException ) { |
| 66 | // @todo still needs to be fixed, need a way to trigger this |
| 67 | // failure |
| 68 | $shardStats = $exception->getResponse()->getShardsStatistics(); |
| 69 | $message = []; |
| 70 | $type = null; |
| 71 | foreach ( $shardStats[ 'failures' ] as $failure ) { |
| 72 | $message[] = $failure['reason']['reason']; |
| 73 | $type ??= $failure['reason']['type']; |
| 74 | } |
| 75 | |
| 76 | return [ |
| 77 | 'type' => $type, |
| 78 | 'reason' => 'Partial failure: ' . implode( ',', $message ), |
| 79 | 'partial' => true |
| 80 | ]; |
| 81 | } |
| 82 | |
| 83 | $response = $exception->getResponse(); |
| 84 | $error = $response->getFullError(); |
| 85 | if ( is_string( $error ) ) { |
| 86 | $error = [ |
| 87 | 'type' => 'unknown', |
| 88 | 'reason' => $error, |
| 89 | ]; |
| 90 | } elseif ( $error === null ) { |
| 91 | // response wasnt json or didn't contain 'error' key |
| 92 | // in this case elastica reports nothing. |
| 93 | $data = $response->getData(); |
| 94 | $parts = []; |
| 95 | if ( $response->getStatus() !== null ) { |
| 96 | $parts[] = 'Status code ' . $response->getStatus(); |
| 97 | } |
| 98 | if ( isset( $data['message'] ) ) { |
| 99 | // Client puts non-json responses here |
| 100 | $parts[] = substr( $data['message'], 0, 200 ); |
| 101 | } elseif ( is_string( $data ) && $data !== "" ) { |
| 102 | // pre-6.0.3 versions of Elastica |
| 103 | $parts[] = substr( $data, 0, 200 ); |
| 104 | } |
| 105 | $reason = implode( "; ", $parts ); |
| 106 | |
| 107 | $error = [ |
| 108 | 'type' => 'unknown', |
| 109 | 'reason' => $reason, |
| 110 | ]; |
| 111 | } |
| 112 | |
| 113 | return $error; |
| 114 | } |
| 115 | |
| 116 | /** |
| 117 | * Broadly classify the error message into failures where |
| 118 | * we decided to not serve the query, and failures where |
| 119 | * we just failed to answer |
| 120 | * |
| 121 | * @param \Elastica\Exception\ExceptionInterface|null $exception |
| 122 | * @return string Either 'rejected', 'failed' or 'unknown' |
| 123 | */ |
| 124 | public static function classifyError( ?\Elastica\Exception\ExceptionInterface $exception = null ) { |
| 125 | if ( $exception === null ) { |
| 126 | return 'unknown'; |
| 127 | } |
| 128 | $error = self::extractFullError( $exception ); |
| 129 | if ( isset( $error['root_cause'][0]['type'] ) ) { |
| 130 | $error = reset( $error['root_cause'] ); |
| 131 | } elseif ( !( isset( $error['type'] ) && isset( $error['reason'] ) ) ) { |
| 132 | return 'unknown'; |
| 133 | } |
| 134 | |
| 135 | $heuristics = [ |
| 136 | 'rejected' => [ |
| 137 | 'type_regexes' => [ |
| 138 | '(^|_)regex_', |
| 139 | '^too_complex_to_determinize_exception$', |
| 140 | '^elasticsearch_parse_exception$', |
| 141 | '^search_parse_exception$', |
| 142 | '^query_shard_exception$', |
| 143 | '^illegal_argument_exception$', |
| 144 | '^too_many_clauses$', |
| 145 | '^too_many_nested_clauses$', |
| 146 | '^parsing_exception$', |
| 147 | '^parse_exception$', |
| 148 | '^script_exception$', |
| 149 | ], |
| 150 | 'msg_regexes' => [ |
| 151 | ], |
| 152 | ], |
| 153 | 'failed' => [ |
| 154 | 'type_regexes' => [ |
| 155 | '^es_rejected_execution_exception$', |
| 156 | '^search_phase_execution_exception', |
| 157 | '^remote_transport_exception$', |
| 158 | '^search_context_missing_exception$', |
| 159 | '^null_pointer_exception$', |
| 160 | '^elasticsearch_timeout_exception$', |
| 161 | '^retry_on_primary_exception$', |
| 162 | // These are exceptions thrown by elastica itself |
| 163 | // (generally connectivity issues in cURL) |
| 164 | '^http_exception$', |
| 165 | ], |
| 166 | 'msg_regexes' => [ |
| 167 | // ClientException thrown by Elastica |
| 168 | '^No enabled connection', |
| 169 | // These are problems raised by the http intermediary layers (nginx/envoy) |
| 170 | '^Status code 503', |
| 171 | '^\Qupstream connect error or disconnect/reset\E', |
| 172 | '^upstream request timeout', |
| 173 | // see \CirrusSearch\Query\CompSuggestQueryBuilder::postProcess, not ideal to rely |
| 174 | // on our own exception message for error classification... |
| 175 | '^\QInvalid response returned from the backend (probable shard failure during the fetch phase)\E', |
| 176 | ], |
| 177 | ], |
| 178 | 'config_issue' => [ |
| 179 | 'type_regexes' => [ |
| 180 | '^index_not_found_exception$', |
| 181 | ], |
| 182 | 'msg_regexes' => [ |
| 183 | // for 'bulk' errors index_not_found_exception is set |
| 184 | // in message and not type |
| 185 | 'index_not_found_exception', |
| 186 | ], |
| 187 | ], |
| 188 | 'memory_issue' => [ |
| 189 | 'type_regexes' => [ |
| 190 | '^circuit_breaking_exception$', |
| 191 | ], |
| 192 | 'msg_regexes' => [], |
| 193 | ], |
| 194 | ]; |
| 195 | |
| 196 | foreach ( $heuristics as $type => $heuristic ) { |
| 197 | $regex = implode( '|', $heuristic['type_regexes'] ); |
| 198 | if ( $regex && preg_match( "#$regex#", $error['type'] ) ) { |
| 199 | return $type; |
| 200 | } |
| 201 | $regex = implode( '|', $heuristic['msg_regexes'] ); |
| 202 | if ( $regex && preg_match( "#$regex#", $error['reason'] ) ) { |
| 203 | return $type; |
| 204 | } |
| 205 | } |
| 206 | return "unknown"; |
| 207 | } |
| 208 | |
| 209 | /** |
| 210 | * Does this status represent an Elasticsearch parse error? |
| 211 | * @param Status $status Status to check |
| 212 | * @return bool is this a parse error? |
| 213 | */ |
| 214 | public static function isParseError( Status $status ) { |
| 215 | foreach ( $status->getMessages() as $msg ) { |
| 216 | if ( $msg->getKey() === 'cirrussearch-parse-error' ) { |
| 217 | return true; |
| 218 | } |
| 219 | } |
| 220 | return false; |
| 221 | } |
| 222 | |
| 223 | /** |
| 224 | * @param \Elastica\Exception\ExceptionInterface|null $exception |
| 225 | * @return array Two elements, first is Status object, second is string. |
| 226 | */ |
| 227 | public static function extractMessageAndStatus( ?\Elastica\Exception\ExceptionInterface $exception = null ) { |
| 228 | if ( !$exception ) { |
| 229 | return [ Status::newFatal( 'cirrussearch-backend-error' ), '' ]; |
| 230 | } |
| 231 | |
| 232 | // Lots of times these are the same as getFullError(), but sometimes |
| 233 | // they're not. I'm looking at you PartialShardFailureException. |
| 234 | $error = self::extractFullError( $exception ); |
| 235 | |
| 236 | // These can be top level errors, or exceptions that don't extend from |
| 237 | // ResponseException like PartialShardFailureException or errors |
| 238 | // contacting the cluster. |
| 239 | if ( !isset( $error['root_cause'][0]['type'] ) ) { |
| 240 | return [ |
| 241 | Status::newFatal( 'cirrussearch-backend-error' ), |
| 242 | self::formatMessage( $error ) |
| 243 | ]; |
| 244 | } |
| 245 | |
| 246 | // We can have multiple root causes if the error is not the |
| 247 | // same on different shards. Errors will be deduplicated based |
| 248 | // on their type. Currently we display only the first one if |
| 249 | // it happens. |
| 250 | $cause = reset( $error['root_cause'] ); |
| 251 | |
| 252 | if ( $cause['type'] === 'query_shard_exception' ) { |
| 253 | // The important part of the parse error message is embedded a few levels down |
| 254 | // and comes before the next new line so lets slurp it up and log it rather than |
| 255 | // the huge clump of error. |
| 256 | $shardFailure = reset( $error['failed_shards'] ); |
| 257 | if ( !empty( $shardFailure['reason'] ) ) { |
| 258 | if ( !empty( $shardFailure['reason']['caused_by'] ) ) { |
| 259 | $message = $shardFailure['reason']['caused_by']['reason']; |
| 260 | } else { |
| 261 | $message = $shardFailure['reason']['reason']; |
| 262 | } |
| 263 | } else { |
| 264 | $message = "???"; |
| 265 | } |
| 266 | $end = strpos( $message, "\n", 0 ); |
| 267 | if ( $end === false ) { |
| 268 | $end = strlen( $message ); |
| 269 | } |
| 270 | $parseError = substr( $message, 0, $end ); |
| 271 | |
| 272 | return [ |
| 273 | Status::newFatal( 'cirrussearch-parse-error' ), |
| 274 | 'Parse error on ' . $parseError |
| 275 | ]; |
| 276 | } |
| 277 | |
| 278 | if ( $cause['type'] === 'too_complex_to_determinize_exception' ) { |
| 279 | return [ Status::newFatal( |
| 280 | 'cirrussearch-regex-too-complex-error' ), |
| 281 | $cause['reason'] |
| 282 | ]; |
| 283 | } |
| 284 | |
| 285 | if ( in_array( $cause['type'], [ 'too_many_nested_clauses', 'too_many_clauses' ] ) ) { |
| 286 | return [ Status::newFatal( |
| 287 | 'cirrussearch-query-too-complex-error' ), |
| 288 | $cause['reason'] |
| 289 | ]; |
| 290 | } |
| 291 | |
| 292 | if ( $cause['type'] === 'script_exception' ) { |
| 293 | // do not use $cause which won't contain the caused_by chain |
| 294 | $formattedMessage = self::formatMessage( $error['caused_by'] ); |
| 295 | $formattedMessage .= "\n\t" . implode( "\n\t", $cause['script_stack'] ) . "\n"; |
| 296 | return [ |
| 297 | Status::newFatal( 'cirrussearch-backend-error' ), |
| 298 | $formattedMessage |
| 299 | ]; |
| 300 | } |
| 301 | |
| 302 | if ( preg_match( '/(^|_)regex_/', $cause['type'] ) ) { |
| 303 | $syntaxError = $cause['reason']; |
| 304 | $errorMessage = 'unknown'; |
| 305 | $position = 'unknown'; |
| 306 | // Note: we support only errors coming from the extra plugin. |
| 307 | // Without the plugin regex is unavailable, so there is no other |
| 308 | // source of regex errors to handle here. |
| 309 | |
| 310 | $matches = []; |
| 311 | // In some cases elastic will serialize the exception by adding |
| 312 | // an extra message prefix with the exception type. |
| 313 | // If the exception is serialized through Transport: |
| 314 | // invalid_regex_exception: expected ']' at position 2 |
| 315 | // Or if the exception is thrown locally by the node receiving the query: |
| 316 | // expected ']' at position 2 |
| 317 | if ( preg_match( '/(?:[a-z_]+: )?(.+) at position (\d+)/', $syntaxError, $matches ) ) { |
| 318 | [ , $errorMessage, $position ] = $matches; |
| 319 | } elseif ( $syntaxError === 'unexpected end-of-string' ) { |
| 320 | $errorMessage = 'regex too short to be correct'; |
| 321 | } |
| 322 | $status = Status::newFatal( 'cirrussearch-regex-syntax-error', $errorMessage, $position ); |
| 323 | |
| 324 | return [ $status, 'Regex syntax error: ' . $syntaxError ]; |
| 325 | } |
| 326 | |
| 327 | return [ |
| 328 | Status::newFatal( 'cirrussearch-backend-error' ), |
| 329 | self::formatMessage( $cause ) |
| 330 | ]; |
| 331 | } |
| 332 | |
| 333 | /** |
| 334 | * Takes an error and converts it into a useful message. Mostly this is to deal with |
| 335 | * errors where the useful part is hidden inside a caused_by chain. |
| 336 | * WARNING: In some circumstances, like bulk update failures, this could be multiple |
| 337 | * megabytes. |
| 338 | * |
| 339 | * @param array $error An error array, such as the one returned by extractFullError(). |
| 340 | * @return string |
| 341 | */ |
| 342 | protected static function formatMessage( array $error ) { |
| 343 | if ( isset( $error['actionReasons'] ) ) { |
| 344 | $message = $error['type'] . ': ' . $error['reason']; |
| 345 | foreach ( $error['actionReasons'] as $actionReason ) { |
| 346 | $message .= " - $actionReason\n"; |
| 347 | } |
| 348 | return $message; |
| 349 | } |
| 350 | |
| 351 | $causeChain = []; |
| 352 | $errorCursor = $error; |
| 353 | while ( isset( $errorCursor['caused_by'] ) ) { |
| 354 | $errorCursor = $errorCursor['caused_by']; |
| 355 | if ( $errorCursor['reason'] ) { |
| 356 | $causeChain[] = $errorCursor['reason']; |
| 357 | } |
| 358 | } |
| 359 | $message = $error['type'] . ': ' . $error['reason']; |
| 360 | if ( $causeChain ) { |
| 361 | $message .= ' (' . implode( ' -> ', array_reverse( $causeChain ) ) . ')'; |
| 362 | } |
| 363 | return $message; |
| 364 | } |
| 365 | |
| 366 | } |