Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
84.29% |
279 / 331 |
|
80.00% |
12 / 15 |
CRAP | |
0.00% |
0 / 1 |
| OrchestratorRequest | |
84.29% |
279 / 331 |
|
80.00% |
12 / 15 |
75.43 | |
0.00% |
0 / 1 |
| __construct | |
100.00% |
7 / 7 |
|
100.00% |
1 / 1 |
2 | |||
| getHost | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| orchestrate | |
98.36% |
60 / 61 |
|
0.00% |
0 / 1 |
14 | |||
| orchestrateTestExecution | |
60.75% |
65 / 107 |
|
0.00% |
0 / 1 |
18.32 | |||
| validateCachedResponse | |
18.18% |
2 / 11 |
|
0.00% |
0 / 1 |
4.19 | |||
| handleGuzzleRequestForEvaluate | |
100.00% |
35 / 35 |
|
100.00% |
1 / 1 |
7 | |||
| substituteLoneSurrogates | |
100.00% |
9 / 9 |
|
100.00% |
1 / 1 |
3 | |||
| getSupportedProgrammingLanguages | |
100.00% |
3 / 3 |
|
100.00% |
1 / 1 |
2 | |||
| persistToCache | |
100.00% |
11 / 11 |
|
100.00% |
1 / 1 |
1 | |||
| getResponseEnvelope | |
100.00% |
69 / 69 |
|
100.00% |
1 / 1 |
5 | |||
| getFailedResponseEnvelope | |
100.00% |
2 / 2 |
|
100.00% |
1 / 1 |
1 | |||
| getOrchestratorResponseTTL | |
100.00% |
9 / 9 |
|
100.00% |
1 / 1 |
7 | |||
| isAWFragmentRequest | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| isWikifunctionsClientRequest | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| getFunctionZidForCacheKey | |
100.00% |
4 / 4 |
|
100.00% |
1 / 1 |
3 | |||
| 1 | <?php |
| 2 | /** |
| 3 | * WikiLambda Orchestrator Interface base class |
| 4 | * |
| 5 | * @file |
| 6 | * @ingroup Extensions |
| 7 | * @copyright 2020– Abstract Wikipedia team; see AUTHORS.txt |
| 8 | * @license MIT |
| 9 | */ |
| 10 | |
| 11 | namespace MediaWiki\Extension\WikiLambda; |
| 12 | |
| 13 | use GuzzleHttp\Client; |
| 14 | use GuzzleHttp\Exception\ConnectException; |
| 15 | use GuzzleHttp\Exception\GuzzleException; |
| 16 | use GuzzleHttp\Exception\TooManyRedirectsException; |
| 17 | use JsonException; |
| 18 | use MediaWiki\Extension\WikiLambda\Cache\MemcachedWrapper; |
| 19 | use MediaWiki\Extension\WikiLambda\Registry\ZErrorTypeRegistry; |
| 20 | use MediaWiki\Extension\WikiLambda\Registry\ZTypeRegistry; |
| 21 | use MediaWiki\Extension\WikiLambda\ZObjects\ZBoolean; |
| 22 | use MediaWiki\Extension\WikiLambda\ZObjects\ZError; |
| 23 | use MediaWiki\Extension\WikiLambda\ZObjects\ZObject; |
| 24 | use MediaWiki\Extension\WikiLambda\ZObjects\ZReference; |
| 25 | use MediaWiki\Extension\WikiLambda\ZObjects\ZResponseEnvelope; |
| 26 | use MediaWiki\Extension\WikiLambda\ZObjects\ZString; |
| 27 | use MediaWiki\Json\FormatJson; |
| 28 | use MediaWiki\Logger\LoggerFactory; |
| 29 | use MediaWiki\Utils\GitInfo; |
| 30 | use Psr\Http\Message\ResponseInterface; |
| 31 | use Psr\Log\LoggerInterface; |
| 32 | use stdClass; |
| 33 | use Wikimedia\RequestTimeout\TimeoutException; |
| 34 | use Wikimedia\Telemetry\TracerInterface; |
| 35 | use Wikimedia\Timestamp\ConvertibleTimestamp; |
| 36 | use Wikimedia\Timestamp\TimestampFormat as TS; |
| 37 | |
| 38 | class OrchestratorRequest { |
| 39 | |
| 40 | protected string $userAgentString; |
| 41 | protected MemcachedWrapper $objectCache; |
| 42 | protected LoggerInterface $logger; |
| 43 | |
| 44 | public const AW_FRAGMENT_ORIGIN_HEADER = 'aw-fragment'; |
| 45 | public const WF_CLIENT_ORIGIN_HEADER = 'wf-client'; |
| 46 | public const FUNCTIONCALL_CACHE_KEY_PREFIX = 'WikiLambdaFunctionCall'; |
| 47 | /** Change this to stop reading all earlier entries. */ |
| 48 | public const FUNCTIONCALL_CACHE_KEY_VERSION = 'v2'; |
| 49 | |
| 50 | /** |
| 51 | * The specialised request interface to control all network access to the function-orchestrator. |
| 52 | * |
| 53 | * @param TracerInterface $tracer Telemetry tracer used to propagate the trace context |
| 54 | * @param Client $guzzleClient GuzzleHttp Client used for requests |
| 55 | */ |
| 56 | public function __construct( |
| 57 | protected readonly TracerInterface $tracer, |
| 58 | protected readonly Client $guzzleClient |
| 59 | ) { |
| 60 | // We generate a user agent string for better traceability of requests |
| 61 | $this->userAgentString = 'wikifunctions-request/' . MW_VERSION; |
| 62 | $gitInfo = new GitInfo( MW_INSTALL_PATH . '/extensions/WikiLambda' ); |
| 63 | $gitHash = $gitInfo->getHeadSHA1(); |
| 64 | if ( $gitHash !== false ) { |
| 65 | $this->userAgentString .= '-WL' . substr( $gitHash, 0, 8 ); |
| 66 | } |
| 67 | |
| 68 | // Non-injected items |
| 69 | $this->objectCache = WikiLambdaServices::getMemcachedWrapper(); |
| 70 | |
| 71 | // Use a separate channel for the orchestrator. This class writes a message for each |
| 72 | // call, and those messages can hide the other messages on the WikiLambda channel. |
| 73 | $this->logger = LoggerFactory::getInstance( 'WikiLambdaOrchestrator' ); |
| 74 | } |
| 75 | |
| 76 | /** |
| 77 | * Returns the orchestrator host |
| 78 | * |
| 79 | * @return string |
| 80 | */ |
| 81 | public function getHost(): string { |
| 82 | return $this->guzzleClient->getConfig( 'base_uri' ); |
| 83 | } |
| 84 | |
| 85 | /** |
| 86 | * Returns the evaluated result for a given function call. |
| 87 | * |
| 88 | * First checks whether the value is cached. If miss, it requests the |
| 89 | * evaluation to the backend services. When the new value returns, it |
| 90 | * sets it in the cache. |
| 91 | * |
| 92 | * The following flags alter the cache-related behavior: |
| 93 | * * If bypassCache is true, sends the request directly to the orchestrator and |
| 94 | * does not update the cached value. |
| 95 | * * If evaluateOnMis is false, after a cache miss it returns false, rather than |
| 96 | * triggering a call |
| 97 | * |
| 98 | * @param array $query |
| 99 | * @param bool $bypassCache - Whether to bypass the MediaWiki-side function call cache; this is |
| 100 | * only to be used for special circumstances, as it's potentially expensive. |
| 101 | * @param bool $evaluateOnMiss - Whether to make an http request to the evaluate endpoint on cache miss. |
| 102 | * Default behavior is true, but we can disable this flag if we want to orchestrate asynchronous |
| 103 | * calls when there's no cached result (e.g. with bulk requests). |
| 104 | * @param string $origin - Content of the X-WikiLambda-Request-Origin header from the caller request |
| 105 | * @return array|false containing Response object (Z22) returned by orchestrator, down-cast to a |
| 106 | * string, the actual http status code from the Orchestrator, and a 'cached' flag that states |
| 107 | * whether the object cache served the value |
| 108 | * @throws OrchestratorException on any transport failure |
| 109 | */ |
| 110 | public function orchestrate( |
| 111 | array $query, |
| 112 | $bypassCache = false, |
| 113 | $evaluateOnMiss = true, |
| 114 | $origin = '' |
| 115 | ): array|false { |
| 116 | // (T365053) Propagate request tracing headers |
| 117 | $requestHeaders = $this->tracer->getRequestHeaders(); |
| 118 | $requestHeaders['User-Agent'] = $this->userAgentString; |
| 119 | |
| 120 | $getFreshResult = (bool)( $query[ 'getFreshResult' ] ?? false ); |
| 121 | |
| 122 | // Remove getFreshResult from cache key |
| 123 | $cacheableQuery = $query; |
| 124 | unset( $cacheableQuery[ 'getFreshResult' ] ); |
| 125 | |
| 126 | $requestKey = $this->objectCache->makeKey( |
| 127 | self::FUNCTIONCALL_CACHE_KEY_PREFIX, |
| 128 | self::FUNCTIONCALL_CACHE_KEY_VERSION, |
| 129 | self::getFunctionZidForCacheKey( $cacheableQuery ), |
| 130 | ZObjectUtils::makeCacheKeyFromZObject( $cacheableQuery ) |
| 131 | ); |
| 132 | |
| 133 | // 1. Get from the Object Cache (except when bypassCache=true or getFreshResult=true) |
| 134 | if ( !$bypassCache && !$getFreshResult ) { |
| 135 | $response = $this->objectCache->get( $requestKey ); |
| 136 | |
| 137 | // 1.a. Cache hit, exit early with validated cached value |
| 138 | if ( $response !== false ) { |
| 139 | $this->logger->debug( __METHOD__ . ' cache hit for {key}', [ 'key' => $requestKey ] ); |
| 140 | // The flag is set here, not stored, so that an entry written before this code |
| 141 | // shipped still reports correctly. |
| 142 | return array_merge( |
| 143 | $this->validateCachedResponse( $response, $requestKey ), |
| 144 | [ 'cached' => true ] |
| 145 | ); |
| 146 | } |
| 147 | |
| 148 | // 1.b. Cache miss, re-evaluate or return false (if evaluateOnMiss is unset) |
| 149 | $this->logger->info( __METHOD__ . ' cache miss for {key}', [ 'key' => $requestKey ] ); |
| 150 | if ( !$evaluateOnMiss ) { |
| 151 | return false; |
| 152 | } |
| 153 | } |
| 154 | |
| 155 | // 2. Call the orchestrator |
| 156 | try { |
| 157 | $response = $this->handleGuzzleRequestForEvaluate( $query, $requestHeaders ); |
| 158 | } catch ( GuzzleException $e ) { |
| 159 | throw new OrchestratorException( $e->getMessage(), $query, 0, $e ); |
| 160 | } |
| 161 | |
| 162 | // 3. Standardized log for Orchestrator Request result for every case |
| 163 | $httpStatus = $response['httpStatusCode'] ?? HttpStatus::INTERNAL_SERVER_ERROR; |
| 164 | $loggerArgs = [ __METHOD__ . ' evaluated response for {key} returned HTTP {status}', [ |
| 165 | 'key' => $requestKey, |
| 166 | 'status' => $httpStatus, |
| 167 | 'source' => $origin ?: 'wf-repo' |
| 168 | ] ]; |
| 169 | |
| 170 | if ( |
| 171 | ( $httpStatus >= HttpStatus::INTERNAL_SERVER_ERROR ) || |
| 172 | ( $httpStatus === HttpStatus::TOO_MANY_REQUESTS ) || |
| 173 | ( $httpStatus === HttpStatus::REQUEST_TIMEOUT ) |
| 174 | ) { |
| 175 | // System error (http 408, 429 or >=500): log as WARNING |
| 176 | $this->logger->warning( ...$loggerArgs ); |
| 177 | } else { |
| 178 | // Successful request or content error: log as INFO |
| 179 | $this->logger->info( ...$loggerArgs ); |
| 180 | } |
| 181 | |
| 182 | // 4. Write to the Object Cache (except when bypassCache=true) |
| 183 | if ( !$bypassCache ) { |
| 184 | // Get TTL conditioned by status and origin |
| 185 | $exptime = self::getOrchestratorResponseTTL( $httpStatus, $origin ); |
| 186 | |
| 187 | // Let's avoid deserializing-serializing just to add a few metadata keys before caching |
| 188 | $stampedResponse = json_decode( $response[ 'result' ] ); |
| 189 | $requestTimestamp = ConvertibleTimestamp::now( TS::ISO_8601 ); |
| 190 | $stampedResponse = ZObjectUtils::setMetaDataValue( $stampedResponse, |
| 191 | 'functionCallCachedOn', $requestTimestamp ); |
| 192 | $stampedResponse = ZObjectUtils::setMetaDataValue( $stampedResponse, |
| 193 | 'functionCallCacheKey', $requestKey ); |
| 194 | |
| 195 | // Cache stamped response metadata if it was stamped on successfully, else don't worry, |
| 196 | // this is not critical data. However, make sure to return the unstamped response, so |
| 197 | // we don't give the impression the call was retrieved from the cache when it was not. |
| 198 | $cachedResponse = $stampedResponse !== null |
| 199 | ? array_merge( $response, [ 'result' => json_encode( $stampedResponse ) ] ) |
| 200 | : $response; |
| 201 | |
| 202 | $this->objectCache->set( $requestKey, $cachedResponse, $exptime ); |
| 203 | $this->logger->debug( __METHOD__ . ' cache store for {key}, TTL {ttl}', [ |
| 204 | 'key' => $requestKey, |
| 205 | 'ttl' => $exptime |
| 206 | ] ); |
| 207 | } |
| 208 | |
| 209 | // 5. Add metadata key about the call being newly generated with latest data; |
| 210 | // get cachedWikidataEntities key: if missing, the call is considered fresh |
| 211 | $stampedResponse = json_decode( $response[ 'result' ] ); |
| 212 | $cachedWikidataEntities = ZObjectUtils::getMetaDataValue( $stampedResponse, 'cachedWikidataEntities' ); |
| 213 | if ( $cachedWikidataEntities === null ) { |
| 214 | $ts = ConvertibleTimestamp::now( TS::ISO_8601 ); |
| 215 | $stampedResponse = ZObjectUtils::setMetaDataValue( $stampedResponse, 'functionCallFreshResult', $ts ); |
| 216 | if ( $stampedResponse !== null ) { |
| 217 | // Assign new stamped response only if valid (e.g. no change if Z22 contained a Z24 |
| 218 | $response['result'] = json_encode( $stampedResponse ); |
| 219 | } |
| 220 | } else { |
| 221 | // When the call was evaluated but used cached entities, return in the response array |
| 222 | // so that the caller logs the information without having to parse the response again. |
| 223 | // Note than when returning a cached response, we don't inspect the cached wikidata |
| 224 | // entities, as this information is most likely irrelevant and outdated, and extracting |
| 225 | // this information at this point is costless while parsing cached response to get |
| 226 | // the metadata is not. |
| 227 | $response['cachedEntities'] = $cachedWikidataEntities; |
| 228 | } |
| 229 | |
| 230 | return array_merge( $response, [ 'cached' => false ] ); |
| 231 | } |
| 232 | |
| 233 | /** |
| 234 | * Orchestrates a full test execution: runs the test call (Z20K2) and validation |
| 235 | * call (Z20K3) against the orchestrator, interprets the result, and returns it. |
| 236 | * |
| 237 | * The returned array contains the keys: |
| 238 | * * passed: boolean flag |
| 239 | * * value: ZObject with ZBoolean containing passed or not passed |
| 240 | * * metadata: ZObject with ZTypedMap containing overall test metadata |
| 241 | * * hasErrors: boolean flag |
| 242 | * |
| 243 | * Returns false if either call is a cache miss and $evaluateOnMiss is false, |
| 244 | * meaning the caller should queue a job to execute the test asynchronously. |
| 245 | * |
| 246 | * @param stdClass $testCall Dereferenced ZFunctionCall for the test (Z20K2) |
| 247 | * @param stdClass $validationCall ZFunctionCall for the validation (Z20K3), without K1 set |
| 248 | * @param bool $evaluateOnMiss Whether to call the orchestrator on a cache miss |
| 249 | * @return array|false |
| 250 | * @throws OrchestratorException |
| 251 | * @throws TimeoutException |
| 252 | */ |
| 253 | public function orchestrateTestExecution( |
| 254 | stdClass $testCall, |
| 255 | stdClass $validationCall, |
| 256 | bool $evaluateOnMiss |
| 257 | ) { |
| 258 | // 1. Execute the test call (Z20K2) |
| 259 | // ================================ |
| 260 | |
| 261 | // 1.a. validate test call; if not a function call then this |
| 262 | // test is not valid and we need to return an error (object type mismatch/Z518) |
| 263 | if ( !ZObjectUtils::isFunctionCall( $testCall ) ) { |
| 264 | $this->logger->warning( __METHOD__ . ' input test call is not a valid function call', [ |
| 265 | 'testCall' => $testCall |
| 266 | ] ); |
| 267 | $zError = ZErrorFactory::createZErrorInstance( ZErrorTypeRegistry::Z_ERROR_OBJECT_TYPE_MISMATCH, [ |
| 268 | 'expected' => ZTypeRegistry::Z_FUNCTIONCALL, |
| 269 | 'actual' => $testCall |
| 270 | ] ); |
| 271 | return [ |
| 272 | 'passed' => false, |
| 273 | 'value' => new ZBoolean( false ), |
| 274 | 'metadata' => ZResponseEnvelope::wrapInResponseMap( 'errors', $zError ), |
| 275 | 'hasErrors' => true |
| 276 | ]; |
| 277 | } |
| 278 | |
| 279 | // 1.b. call orchestrator service with evaluateOnMiss flag |
| 280 | // can throw OrchestratorException or TimoutException |
| 281 | $response = $this->orchestrate( |
| 282 | [ 'zobject' => $testCall, 'doValidate' => true ], |
| 283 | /* bypassCache= */ false, |
| 284 | $evaluateOnMiss |
| 285 | ); |
| 286 | |
| 287 | // 1.c. if response is false, there was a cache miss with evaluateOnMiss=false |
| 288 | if ( $response === false ) { |
| 289 | return false; |
| 290 | } |
| 291 | |
| 292 | // 1.d. parse the response, into a ZResponseEnvelope object |
| 293 | $callResponse = $this->getResponseEnvelope( |
| 294 | $response['result'], |
| 295 | json_encode( $testCall ) |
| 296 | ); |
| 297 | |
| 298 | // 1.e. (T394107) check that Z22K1 contains a valid object, else set validateErrors |
| 299 | $callResponseValue = $callResponse->getZValue(); |
| 300 | if ( !( $callResponseValue instanceof ZObject ) ) { |
| 301 | $this->logger->warning( __METHOD__ . ' test call does not return a valid response envelope', [ |
| 302 | 'testCallResponse' => var_export( $callResponseValue, true ) |
| 303 | ] ); |
| 304 | $zError = ZErrorFactory::createZErrorInstance( ZErrorTypeRegistry::Z_ERROR_UNKNOWN, [ |
| 305 | 'message' => 'Test call returned invalid Z22K1' |
| 306 | ] ); |
| 307 | return [ |
| 308 | 'passed' => false, |
| 309 | 'value' => new ZBoolean( false ), |
| 310 | 'metadata' => ZResponseEnvelope::wrapInResponseMap( 'validateErrors', $zError ), |
| 311 | 'hasErrors' => true |
| 312 | ]; |
| 313 | } |
| 314 | |
| 315 | // 1.f. If the call was retrieved from the cache, get stored timestamp from the response |
| 316 | $callCachedTimestamp = ZObjectUtils::getMetaDataValue( |
| 317 | $callResponse->getSerialized(), 'functionCallCachedOn' ); |
| 318 | |
| 319 | // 2. Execute the validation call (Z20K3) |
| 320 | // ====================================== |
| 321 | |
| 322 | // 2.a. validate test validation call; if not a function call then this |
| 323 | // test is not valid and we need to return an error (object type mismatch/Z518) |
| 324 | if ( !ZObjectUtils::isFunctionCall( $validationCall ) ) { |
| 325 | $this->logger->warning( __METHOD__ . ' input validation call is not a valid function call', [ |
| 326 | 'validationCall' => $validationCall |
| 327 | ] ); |
| 328 | $zError = ZErrorFactory::createZErrorInstance( ZErrorTypeRegistry::Z_ERROR_OBJECT_TYPE_MISMATCH, [ |
| 329 | 'expected' => ZTypeRegistry::Z_FUNCTIONCALL, |
| 330 | 'actual' => $validationCall |
| 331 | ] ); |
| 332 | return [ |
| 333 | 'passed' => false, |
| 334 | 'value' => new ZBoolean( false ), |
| 335 | 'metadata' => ZResponseEnvelope::wrapInResponseMap( 'errors', $zError ), |
| 336 | 'hasErrors' => true |
| 337 | ]; |
| 338 | } |
| 339 | |
| 340 | $testMetadata = $callResponse->getValueByKey( ZTypeRegistry::Z_RESPONSEENVELOPE_METADATA ); |
| 341 | '@phan-var \MediaWiki\Extension\WikiLambda\ZObjects\ZTypedMap $testMetadata'; |
| 342 | |
| 343 | // 2.b. build the validation call by adding the first key |
| 344 | $validationCallFunctionZid = $validationCall->{ ZTypeRegistry::Z_FUNCTIONCALL_FUNCTION }; |
| 345 | $validationCall->{ $validationCallFunctionZid . 'K1' } = $callResponseValue->getSerialized(); |
| 346 | |
| 347 | // 2.c. call orchestrator service with evaluateOnMiss flag |
| 348 | // can throw OrchestratorException or TimeoutException |
| 349 | $response = $this->orchestrate( |
| 350 | [ 'zobject' => $validationCall, 'doValidate' => false ], |
| 351 | /* bypassCache= */ false, |
| 352 | $evaluateOnMiss |
| 353 | ); |
| 354 | |
| 355 | // 2.d. if response is false, there was a cache miss with evaluateOnMiss=false |
| 356 | if ( $response === false ) { |
| 357 | return false; |
| 358 | } |
| 359 | |
| 360 | // 2.e. parse the response into a Response Envelope Z22 |
| 361 | $validationResponse = $this->getResponseEnvelope( |
| 362 | $response['result'], |
| 363 | json_encode( $validationCall ) |
| 364 | ); |
| 365 | |
| 366 | // 2.f. If the call was retrieved from the cache, get stored timestamp from the response |
| 367 | $validationCachedTimestamp = ZObjectUtils::getMetaDataValue( |
| 368 | $validationResponse->getSerialized(), 'functionCallCachedOn' ); |
| 369 | |
| 370 | // 3. Interpret the result and compile additional metadata |
| 371 | // ======================================================= |
| 372 | |
| 373 | // must be ZBoolean( true or false ): |
| 374 | $validationResponseValue = $validationResponse->getZValue(); |
| 375 | $passed = true; |
| 376 | |
| 377 | // 3.a. If the validation call fails, set it as false in the metadata map |
| 378 | if ( $validationResponse->hasErrors() ) { |
| 379 | $validationResponseValue = new ZReference( ZTypeRegistry::Z_BOOLEAN_FALSE ); |
| 380 | $testMetadata->setValueForKey( new ZString( 'validateErrors' ), $validationResponse->getErrors() ); |
| 381 | } |
| 382 | |
| 383 | // 3.b. If the validation call is false, set metadata keys with actual vs expected result |
| 384 | if ( ZObjectUtils::isFalse( $validationResponseValue ) ) { |
| 385 | $passed = false; |
| 386 | // The expected value is author-supplied (the validator's second input) and might not |
| 387 | // round-trip through ZObjectFactory, e.g. an incomplete literal missing a required key. |
| 388 | // That's only metadata enrichment for an already-failed test, so degrade to Z24/Void |
| 389 | // rather than letting the ZErrorException bubble up and fail the whole request. (T430082) |
| 390 | try { |
| 391 | // validator might not be a two-input call, set expected as Z24/Void if that's the case |
| 392 | $expectedValue = ZObjectFactory::create( |
| 393 | $validationCall->{ $validationCallFunctionZid . 'K2' } ?? ZTypeRegistry::Z_VOID |
| 394 | ); |
| 395 | } catch ( ZErrorException $e ) { |
| 396 | $this->logger->info( |
| 397 | __METHOD__ . ' could not build expected test result: {message}', |
| 398 | [ 'message' => $e->getMessage(), 'exception' => $e ] |
| 399 | ); |
| 400 | $expectedValue = new ZReference( ZTypeRegistry::Z_VOID ); |
| 401 | } |
| 402 | $testMetadata->setValueForKey( new ZString( 'actualTestResult' ), $callResponseValue ); |
| 403 | $testMetadata->setValueForKey( new ZString( 'expectedTestResult' ), $expectedValue ); |
| 404 | } |
| 405 | |
| 406 | // 3.c. If any of the calls was retrieved from the cache, add timestamps to metadata |
| 407 | if ( $callCachedTimestamp ) { |
| 408 | $testMetadata->setValueForKey( |
| 409 | new ZString( 'testCallCachedOn' ), |
| 410 | new ZString( $callCachedTimestamp ) |
| 411 | ); |
| 412 | } |
| 413 | if ( $validationCachedTimestamp ) { |
| 414 | $testMetadata->setValueForKey( |
| 415 | new ZString( 'validationCallCachedOn' ), |
| 416 | new ZString( $validationCachedTimestamp ) |
| 417 | ); |
| 418 | } |
| 419 | |
| 420 | // 3.d. Remove generic function call cache metadata keys |
| 421 | $testMetadata->setValueForKey( new ZString( 'functionCallCachedOn' ), null ); |
| 422 | $testMetadata->setValueForKey( new ZString( 'functionCallFreshResult' ), null ); |
| 423 | |
| 424 | return [ |
| 425 | 'passed' => $passed, |
| 426 | 'value' => $validationResponseValue, |
| 427 | 'metadata' => $testMetadata, |
| 428 | 'hasErrors' => $validationResponse->hasErrors(), |
| 429 | ]; |
| 430 | } |
| 431 | |
| 432 | /** |
| 433 | * Validates and returns an orchestrator cache response. |
| 434 | * If the stored value is not an array, it deletes. |
| 435 | * |
| 436 | * @param mixed $response |
| 437 | * @param string $requestKey |
| 438 | * @return array |
| 439 | */ |
| 440 | private function validateCachedResponse( $response, string $requestKey ): array { |
| 441 | if ( is_array( $response ) ) { |
| 442 | return $response; |
| 443 | } |
| 444 | |
| 445 | $this->logger->error( |
| 446 | 'Cached orchestrator response was somehow not an array', |
| 447 | [ |
| 448 | 'requestKey' => $requestKey, |
| 449 | // Shortened to avoid over-loading the logging system |
| 450 | 'response' => substr( var_export( $response, true ), 0, 1000 ) |
| 451 | ] |
| 452 | ); |
| 453 | $this->objectCache->delete( $requestKey ); |
| 454 | return [ 'result' => null, 'httpStatusCode' => 500 ]; |
| 455 | } |
| 456 | |
| 457 | /** |
| 458 | * Helper function to handle client-side HTTP error codes. |
| 459 | * |
| 460 | * Guzzle by default throws an exception on any non-2xx status, in this case returned from the Orchestrator. |
| 461 | * By calling the post method with `http_errors => false`, 400 and 500 errors are returned in the |
| 462 | * response payload instead of being thrown as ClientException or ServerException. |
| 463 | * |
| 464 | * Guzzle throws four types of exception, all extend TransferException (implements GuzzleException): |
| 465 | * * ConnectException: in case of networking error |
| 466 | * * ClientException: for http 400 errors; with `http_errors => false` these are not thrown. |
| 467 | * * ServerException: for http 500 errors; with `http_errors => false` these are not thrown. |
| 468 | * * TooManyRedirectsException: in case of too many redirects followed |
| 469 | * |
| 470 | * See: https://docs.guzzlephp.org/en/stable/quickstart.html#exceptions |
| 471 | * |
| 472 | * @param stdClass|array $query |
| 473 | * @param array $requestHeaders |
| 474 | * @return array containing Response object (Z22) returned by orchestrator, down-cast to a string |
| 475 | * and the actual http status code from the Orchestrator |
| 476 | * @throws ConnectException If the request fails to connect |
| 477 | * @throws TooManyRedirectsException If the request exceeds the allowed number of redirects |
| 478 | */ |
| 479 | private function handleGuzzleRequestForEvaluate( $query, $requestHeaders ): array { |
| 480 | // TODO (T338242): Use postAsync here. |
| 481 | $response = $this->guzzleClient->post( '/1/v2/evaluate/', [ |
| 482 | 'json' => $query, |
| 483 | 'headers' => $requestHeaders, |
| 484 | // http 400/500 errors from Orchestrator will be suppressed so that they will not throw exceptions |
| 485 | 'http_errors' => false |
| 486 | ] ); |
| 487 | $httpStatusCode = $response->getStatusCode(); |
| 488 | $responseBody = $response->getBody()->getContents(); |
| 489 | |
| 490 | try { |
| 491 | // (T414062) Check if the response body is a valid JSON string, and a Z22 as expected. |
| 492 | try { |
| 493 | $responseBodyObject = json_decode( $responseBody, true, 512, JSON_THROW_ON_ERROR ); |
| 494 | } catch ( JsonException $e ) { |
| 495 | // The orchestrator runs on Node.js, whose UTF-16 strings (and thus JSON.stringify) |
| 496 | // tolerate lone surrogates; when it echoes back malformed user input it can emit |
| 497 | // escapes such as "\udff3" or "\ud83c" that PHP's json_decode rejects outright with |
| 498 | // JSON_ERROR_UTF16. Substitute any lone surrogate with U+FFFD and retry, so that we |
| 499 | // recover from this one tolerable case but still let genuinely malformed JSON throw. |
| 500 | if ( $e->getCode() !== JSON_ERROR_UTF16 ) { |
| 501 | throw $e; |
| 502 | } |
| 503 | $responseBody = self::substituteLoneSurrogates( $responseBody ); |
| 504 | $responseBodyObject = json_decode( $responseBody, true, 512, JSON_THROW_ON_ERROR ); |
| 505 | } |
| 506 | // Z1K1 may be canonical ("Z22") or in normal form ({ Z1K1: "Z9", Z9K1: "Z22" }); accept either. |
| 507 | $responseType = is_array( $responseBodyObject ) ? ( $responseBodyObject['Z1K1'] ?? null ) : null; |
| 508 | if ( is_array( $responseType ) ) { |
| 509 | $responseType = $responseType['Z9K1'] ?? null; |
| 510 | } |
| 511 | if ( $responseType !== ZTypeRegistry::Z_RESPONSEENVELOPE ) { |
| 512 | throw new JsonException( 'Response is not a Z22: ' . var_export( $responseBody, true ) ); |
| 513 | } |
| 514 | } catch ( JsonException $e ) { |
| 515 | $this->logger->warning( |
| 516 | 'Orchestrator response was either not JSON, or somehow not a Z22', |
| 517 | [ |
| 518 | // Shortened to avoid over-loading the logging system |
| 519 | 'responseBody' => substr( $responseBody, 0, 1000 ), |
| 520 | 'httpStatusCode' => $httpStatusCode, |
| 521 | 'exceptionMessage' => $e->getMessage(), |
| 522 | 'exception' => $e, |
| 523 | ] |
| 524 | ); |
| 525 | |
| 526 | // Make an actual Z22 response for the user of a Z24 with with a Z577 error inside, quoting the bad response |
| 527 | $responseError = ZErrorFactory::createZErrorInstance( |
| 528 | ZErrorTypeRegistry::Z_ERROR_INVALID_ORCHESTRATOR_RESULT, |
| 529 | [ 'request' => $query, 'response' => $responseBody ] |
| 530 | ); |
| 531 | $badResponse = new ZResponseEnvelope( null, ZResponseEnvelope::wrapErrorInResponseMap( $responseError ) ); |
| 532 | return [ 'result' => FormatJson::encode( $badResponse->getSerialized() ), 'httpStatusCode' => 500 ]; |
| 533 | } |
| 534 | |
| 535 | return [ 'result' => $responseBody, 'httpStatusCode' => $httpStatusCode ]; |
| 536 | } |
| 537 | |
| 538 | /** |
| 539 | * Replace any lone UTF-16 surrogate escape in a JSON string with the U+FFFD replacement |
| 540 | * character, leaving valid high+low surrogate pairs (and all other content) untouched. |
| 541 | * |
| 542 | * PHP's json_decode rejects lone surrogate escapes with JSON_ERROR_UTF16, whereas the |
| 543 | * Node.js orchestrator can emit them when serialising mangled user input. This lets us |
| 544 | * recover from that specific case without masking otherwise-malformed JSON. |
| 545 | * |
| 546 | * @param string $json A JSON string that failed to decode with JSON_ERROR_UTF16 |
| 547 | * @return string The same JSON with lone surrogate escapes replaced by "�" |
| 548 | */ |
| 549 | private static function substituteLoneSurrogates( string $json ): string { |
| 550 | return preg_replace_callback( |
| 551 | // Branch 1 consumes a complete high+low pair (kept as-is); anything left matching |
| 552 | // branch 2 is therefore a lone surrogate and is replaced. |
| 553 | '/\\\\u(d[89ab][0-9a-f]{2})\\\\u(d[c-f][0-9a-f]{2})|\\\\u(d[89a-f][0-9a-f]{2})/i', |
| 554 | static function ( array $matches ): string { |
| 555 | return isset( $matches[3] ) && $matches[3] !== '' ? '\\ufffd' : $matches[0]; |
| 556 | }, |
| 557 | $json |
| 558 | // On a PCRE engine failure preg_replace_callback returns null; fall back to the original |
| 559 | // string, which then re-throws JSON_ERROR_UTF16 and is wrapped as a Z577 error as before. |
| 560 | ) ?? $json; |
| 561 | } |
| 562 | |
| 563 | /** |
| 564 | * Ask the function-orchestrator for the list of programming languages with evaluators currently configured. |
| 565 | * |
| 566 | * @return ResponseInterface Response interface returned by orchestrator network call. |
| 567 | * @throws OrchestratorException If the orchestrator cannot be reached |
| 568 | */ |
| 569 | public function getSupportedProgrammingLanguages(): ResponseInterface { |
| 570 | // TODO (T338242): Use getAsync here. |
| 571 | try { |
| 572 | return $this->guzzleClient->get( '/1/v1/supported-programming-languages/', [ 'http_errors' => false ] ); |
| 573 | } catch ( GuzzleException $e ) { |
| 574 | throw new OrchestratorException( $e->getMessage(), [], 0, $e ); |
| 575 | } |
| 576 | } |
| 577 | |
| 578 | /** |
| 579 | * Ask the function-orchestrator to store a Persistent ZObject (Z2) in its cache. |
| 580 | * |
| 581 | * @param stdClass $Z2 The ZObject to persist to cache. |
| 582 | * @return ResponseInterface Response interface returned by orchestrator network call. |
| 583 | */ |
| 584 | public function persistToCache( $Z2 ): ResponseInterface { |
| 585 | // (T365053) Propagate request tracing headers |
| 586 | $requestHeaders = $this->tracer->getRequestHeaders(); |
| 587 | $requestHeaders['User-Agent'] = $this->userAgentString; |
| 588 | |
| 589 | $query = [ |
| 590 | 'ZObject' => $Z2, |
| 591 | 'ZID' => $Z2->{ 'Z2K1' }->{ 'Z6K1' } |
| 592 | ]; |
| 593 | |
| 594 | // TODO (T338242): Use postAsync here. |
| 595 | return $this->guzzleClient->post( '/1/v1/persist-to-cache', [ |
| 596 | 'json' => $query, |
| 597 | 'headers' => $requestHeaders, |
| 598 | 'http_errors' => false |
| 599 | ] ); |
| 600 | } |
| 601 | |
| 602 | /** |
| 603 | * Converts a response string into a valid ZResponseEnvelope and |
| 604 | * handles all possible errors from the conversion. |
| 605 | * |
| 606 | * If the response cannot be validated or the response type is |
| 607 | * not correct, it builds a new failure Response Envelope/Z22 |
| 608 | * with the error details in its "errors" key. |
| 609 | * |
| 610 | * All errors captured and returned in this method will be wrapped |
| 611 | * in a Z507/Evaluator error, as they are errors carried by the |
| 612 | * response from the orchestrator service. |
| 613 | * |
| 614 | * This method will NOT throw any exception nor cause the API |
| 615 | * to die with error. |
| 616 | * |
| 617 | * @param string $response The JSON string being decoded |
| 618 | * @param string $call The JSON string of the function call |
| 619 | * @return ZResponseEnvelope |
| 620 | */ |
| 621 | private function getResponseEnvelope( string $response, string $call ): ZResponseEnvelope { |
| 622 | try { |
| 623 | // Decode string JSON into stdClass |
| 624 | $responseContents = json_decode( $response, false, 512, JSON_THROW_ON_ERROR ); |
| 625 | } catch ( JsonException $e ) { |
| 626 | $this->logger->error( |
| 627 | __METHOD__ . ' failed to execute, server response not valid Json: {exception}', |
| 628 | [ |
| 629 | 'request' => $call, |
| 630 | 'response' => $response, |
| 631 | 'exception' => $e, |
| 632 | ] |
| 633 | ); |
| 634 | // ZError: Invalid JSON returned by the evaluator |
| 635 | $zErrorJson = ZErrorFactory::createZErrorInstance( ZErrorTypeRegistry::Z_ERROR_INVALID_JSON, [ |
| 636 | 'message' => $e->getMessage(), |
| 637 | 'data' => $response |
| 638 | ] ); |
| 639 | $zError = ZErrorFactory::createZErrorInstance( ZErrorTypeRegistry::Z_ERROR_EVALUATION, [ |
| 640 | 'functionCall' => $call, |
| 641 | 'error' => $zErrorJson |
| 642 | ] ); |
| 643 | return $this->getFailedResponseEnvelope( $zError ); |
| 644 | } |
| 645 | |
| 646 | // (T414752) Trivial check if server hasn't responded with a ZObject, rather than loading all of ZObjectFactory |
| 647 | // e.g. "{"error":"Payload too large"}" |
| 648 | if ( !property_exists( $responseContents, ZTypeRegistry::Z_OBJECT_TYPE ) ) { |
| 649 | $this->logger->warning( |
| 650 | __METHOD__ . ' failed to execute, server response was not a ZObject: {response}', |
| 651 | [ |
| 652 | 'request' => $call, |
| 653 | 'response' => $response |
| 654 | ] |
| 655 | ); |
| 656 | // ZError: Invalid ZObject returned by the evaluator |
| 657 | $zErrorType = ZErrorFactory::createZErrorInstance( ZErrorTypeRegistry::Z_ERROR_OBJECT_TYPE_MISMATCH, [ |
| 658 | 'expected' => ZTypeRegistry::Z_RESPONSEENVELOPE, |
| 659 | // If it's not a ZObject at all, just say it's a string |
| 660 | 'actual' => ZTypeRegistry::Z_STRING |
| 661 | ] ); |
| 662 | $zError = ZErrorFactory::createZErrorInstance( ZErrorTypeRegistry::Z_ERROR_EVALUATION, [ |
| 663 | 'functionCall' => $call, |
| 664 | 'error' => $zErrorType |
| 665 | ] ); |
| 666 | return $this->getFailedResponseEnvelope( $zError ); |
| 667 | } |
| 668 | |
| 669 | // Convert stdClass into ZObject by passing it through ZObjectFactory::create |
| 670 | try { |
| 671 | $responseObject = ZObjectFactory::create( $responseContents ); |
| 672 | } catch ( ZErrorException $e ) { |
| 673 | $this->logger->error( |
| 674 | __METHOD__ . ' failed to execute, server response not wellformed: {exception}', |
| 675 | [ |
| 676 | 'request' => $call, |
| 677 | 'response' => $response, |
| 678 | 'exception' => $e, |
| 679 | ] |
| 680 | ); |
| 681 | // ZError: Invalid ZObject returned by the evaluator |
| 682 | $zError = ZErrorFactory::createZErrorInstance( ZErrorTypeRegistry::Z_ERROR_EVALUATION, [ |
| 683 | 'functionCall' => $call, |
| 684 | 'error' => $e->getZError() |
| 685 | ] ); |
| 686 | return $this->getFailedResponseEnvelope( $zError ); |
| 687 | } |
| 688 | |
| 689 | // Check that returned object is of the right type |
| 690 | if ( !( $responseObject instanceof ZResponseEnvelope ) ) { |
| 691 | $this->logger->error( |
| 692 | __METHOD__ . ' failed to execute, server response was not a Response Envelope', |
| 693 | [ |
| 694 | 'request' => $call, |
| 695 | 'response' => $response |
| 696 | ] |
| 697 | ); |
| 698 | // ZError: Invalid ZObject returned by the evaluator |
| 699 | $zErrorType = ZErrorFactory::createZErrorInstance( ZErrorTypeRegistry::Z_ERROR_OBJECT_TYPE_MISMATCH, [ |
| 700 | 'expected' => ZTypeRegistry::Z_RESPONSEENVELOPE, |
| 701 | 'actual' => $responseObject->getZType() |
| 702 | ] ); |
| 703 | $zError = ZErrorFactory::createZErrorInstance( ZErrorTypeRegistry::Z_ERROR_EVALUATION, [ |
| 704 | 'functionCall' => $call, |
| 705 | 'error' => $zErrorType |
| 706 | ] ); |
| 707 | return $this->getFailedResponseEnvelope( $zError ); |
| 708 | } |
| 709 | |
| 710 | // Response Envelope returned by the orchestrator is valid! |
| 711 | return $responseObject; |
| 712 | } |
| 713 | |
| 714 | /** |
| 715 | * Builds and returns a Z22/ResponseEnvelope object wrapping a failure, so: |
| 716 | * * Z22K1 is Void/Z24, and |
| 717 | * * Z22K2 contains a Map with an "error" key with the given zError object |
| 718 | * |
| 719 | * @param ZError $zErrorObject |
| 720 | * @return ZResponseEnvelope |
| 721 | */ |
| 722 | private function getFailedResponseEnvelope( $zErrorObject ): ZResponseEnvelope { |
| 723 | $zResponseMap = ZResponseEnvelope::wrapErrorInResponseMap( $zErrorObject ); |
| 724 | return new ZResponseEnvelope( null, $zResponseMap ); |
| 725 | } |
| 726 | |
| 727 | /** |
| 728 | * Returns the TTL for the caching of the response received from the Orchestrator. |
| 729 | * |
| 730 | * (T338243) Set TTL conditionally depending on the returned status: |
| 731 | * - success (http 200): TTL_MONTH |
| 732 | * - failure due to a content error (http 400-422): TTL_WEEK |
| 733 | * - failure due to a recoverable error (http 408, 429 or >=500): TTL_MINUTE |
| 734 | * |
| 735 | * (T432869) Set all values to a minute TTL when the request |
| 736 | * is for rendering an Abstract Wiki or an Embedded fragment. |
| 737 | * |
| 738 | * @param int $httpStatus |
| 739 | * @param string $origin |
| 740 | * @return int |
| 741 | */ |
| 742 | private static function getOrchestratorResponseTTL( $httpStatus, $origin ): int { |
| 743 | // Request originated from an AW or Embedded fragment: set to TTL_MINUTE |
| 744 | if ( self::isAWFragmentRequest( $origin ) || self::isWikifunctionsClientRequest( $origin ) ) { |
| 745 | return MemcachedWrapper::TTL_MINUTE; |
| 746 | } |
| 747 | |
| 748 | // Recoverable system errors: set to TTL_MINUTE |
| 749 | if ( |
| 750 | ( $httpStatus >= HttpStatus::INTERNAL_SERVER_ERROR ) || |
| 751 | ( $httpStatus === HttpStatus::TOO_MANY_REQUESTS ) || |
| 752 | ( $httpStatus === HttpStatus::REQUEST_TIMEOUT ) |
| 753 | ) { |
| 754 | return MemcachedWrapper::TTL_MINUTE; |
| 755 | } |
| 756 | |
| 757 | // Successful value: set to TTL_MONTH |
| 758 | if ( $httpStatus === HttpStatus::OK ) { |
| 759 | return MemcachedWrapper::TTL_MONTH; |
| 760 | } |
| 761 | |
| 762 | // Default: TTL_WEEK |
| 763 | return MemcachedWrapper::TTL_WEEK; |
| 764 | } |
| 765 | |
| 766 | /** |
| 767 | * Returns whether the value of the X-WikiLambda-Request-Origin header (passed as |
| 768 | * parameter) identifies this request as coming from an Abstract Wiki fragment call. |
| 769 | * |
| 770 | * @param string $origin |
| 771 | * @return bool |
| 772 | */ |
| 773 | private static function isAWFragmentRequest( string $origin ): bool { |
| 774 | return $origin === self::AW_FRAGMENT_ORIGIN_HEADER; |
| 775 | } |
| 776 | |
| 777 | /** |
| 778 | * Returns whether the value of the X-WikiLambda-Request-Origin header (passed as parameter) |
| 779 | * parameter) identifies this request as coming from a Wikifunctions Client embedded fragment. |
| 780 | * |
| 781 | * @param string $origin |
| 782 | * @return bool |
| 783 | */ |
| 784 | private static function isWikifunctionsClientRequest( string $origin ): bool { |
| 785 | return $origin === self::WF_CLIENT_ORIGIN_HEADER; |
| 786 | } |
| 787 | |
| 788 | /** |
| 789 | * Returns the ZID of the called function, so that a cache key is easy to find in the logs. |
| 790 | * |
| 791 | * @param array $query |
| 792 | * @return string The function ZID, or Z0 if the call has no function reference |
| 793 | */ |
| 794 | private static function getFunctionZidForCacheKey( array $query ): string { |
| 795 | $functionZid = ( (array)( $query['zobject'] ?? [] ) )[ ZTypeRegistry::Z_FUNCTIONCALL_FUNCTION ] ?? null; |
| 796 | return is_string( $functionZid ) && ZObjectUtils::isValidZObjectReference( $functionZid ) ? |
| 797 | $functionZid : |
| 798 | ZTypeRegistry::Z_NULL_REFERENCE; |
| 799 | } |
| 800 | } |