Code Coverage
 
Lines
Functions and Methods
Classes and Traits
Total
95.77% covered (success)
95.77%
204 / 213
60.00% covered (warning)
60.00%
3 / 5
CRAP
0.00% covered (danger)
0.00%
0 / 1
AbstractWikiRequest
95.77% covered (success)
95.77%
204 / 213
60.00% covered (warning)
60.00%
3 / 5
28
0.00% covered (danger)
0.00%
0 / 1
 __construct
100.00% covered (success)
100.00%
2 / 2
100.00% covered (success)
100.00%
1 / 1
1
 fetchRenderedAWFragment
100.00% covered (success)
100.00%
70 / 70
100.00% covered (success)
100.00%
1 / 1
6
 buildRenderAWFragmentCall
100.00% covered (success)
100.00%
28 / 28
100.00% covered (success)
100.00%
1 / 1
1
 getRenderAWFragmentFunction
85.71% covered (warning)
85.71%
6 / 7
0.00% covered (danger)
0.00%
0 / 1
2.01
 callRenderFunctionCall
92.45% covered (success)
92.45%
98 / 106
0.00% covered (danger)
0.00%
0 / 1
18.14
1<?php
2/**
3 * AbstractWiki Request Service
4 *
5 * @file
6 * @ingroup Extensions
7 * @copyright 2020– Abstract Wikipedia team; see AUTHORS.txt
8 * @license MIT
9 */
10
11namespace MediaWiki\Extension\WikiLambda\AbstractContent;
12
13use JsonException;
14use MediaWiki\Config\Config;
15use MediaWiki\Extension\WikiLambda\AWStorage\AWFragmentStore;
16use MediaWiki\Extension\WikiLambda\HttpStatus;
17use MediaWiki\Extension\WikiLambda\OrchestratorRequest;
18use MediaWiki\Extension\WikiLambda\Registry\ZTypeRegistry;
19use MediaWiki\Extension\WikiLambda\Renderer\WikifunctionsFragmentRenderer;
20use MediaWiki\Extension\WikiLambda\WikifunctionCallException;
21use MediaWiki\Extension\WikiLambda\ZObjectUtils;
22use MediaWiki\Http\HttpRequestFactory;
23use MediaWiki\Logger\LoggerFactory;
24use Psr\Log\LoggerInterface;
25use RuntimeException;
26use Wikimedia\Stats\StatsFactory;
27use Wikimedia\Timestamp\ConvertibleTimestamp;
28
29class AbstractWikiRequest {
30
31    /**
32     * HTTP status codes that we log at warning level when a fragment fails to render.
33     *
34     * These do not have one cause. Some are our fault, e.g. NOT_IMPLEMENTED, when the wiki has
35     * no configuration for a target API. Some are permission problems, e.g. UNAUTHORIZED and
36     * FORBIDDEN. Some can be load problems, e.g. REQUEST_TIMEOUT. We log most of them noisily
37     * for now, until we know how frequently each one occurs in production.
38     *
39     * Failures caused by the content are in HttpStatus::CONTENT_ERROR_CODES, and we log those
40     * quietly. TOO_MANY_REQUESTS is an expected result of the rate limits, and we log it quietly
41     * too. A code in none of these is one that we did not expect, and we log it as an error.
42     */
43    private const WARNING_STATUS_CODES = [
44        // e.g. Z559/User not permitted to evaluate function
45        HttpStatus::UNAUTHORIZED,
46        // e.g. Wikifunctions refused our request
47        HttpStatus::FORBIDDEN,
48        // e.g. Z574/Orchestrator time limit
49        HttpStatus::REQUEST_TIMEOUT,
50        // e.g. Z530/API failure
51        HttpStatus::INTERNAL_SERVER_ERROR,
52        // e.g. this wiki has no Wikifunctions target API
53        HttpStatus::NOT_IMPLEMENTED,
54        // e.g. Z577/Invalid orchestrator result
55        HttpStatus::BAD_GATEWAY,
56        // e.g. we cannot reach Wikifunctions
57        HttpStatus::SERVICE_UNAVAILABLE,
58        // e.g. Z576/Evaluator WASM limit
59        HttpStatus::GATEWAY_TIMEOUT,
60    ];
61
62    private LoggerInterface $logger;
63    private StatsFactory $statsFactory;
64
65    public function __construct(
66        private readonly Config $config,
67        private readonly HttpRequestFactory $httpRequestFactory,
68        private readonly AWFragmentStore $fragmentStore,
69        private readonly WikifunctionsFragmentRenderer $fragmentRenderer,
70        StatsFactory $statsFactory
71    ) {
72        // Non-injected items
73        $this->logger = LoggerFactory::getInstance( 'WikiLambdaAbstract' );
74        $this->statsFactory = $statsFactory->withComponent( 'WikiLambda' );
75    }
76
77    /**
78     * Re-generates an Abstract Wikipedia fragment by requesting it remotely from
79     * Wikifunctions (repo), sanitises it, and stores it in the AWFragmentStore.
80     *
81     * Returns an associative array containing the cached value, either a
82     * successfully rendered and sanitised fragment, or a failed one.
83     *
84     * Callers:
85     * * Jobs/CacheAbstractContentFragmentJob
86     * * ActionAPIs/ApiAbstractWikiRunFragment
87     *
88     * @param array $fragment
89     * @param string $topicQid
90     * @param string $languageZid
91     * @param string $datetime
92     * @param string $fragmentKey
93     * @return array
94     */
95    public function fetchRenderedAWFragment(
96        array $fragment,
97        string $topicQid,
98        string $languageZid,
99        string $datetime,
100        string $fragmentKey
101    ): array {
102        $renderedValue = [];
103
104        // 1. Build the Z825 function call with the fragment as composition implementation
105        $functionCall = $this->buildRenderAWFragmentCall(
106            $fragment,
107            $topicQid,
108            $languageZid,
109            $datetime
110        );
111
112        try {
113            // 2. Run fragment function call, should return a Z89/Html fragment object
114            $htmlFragment = $this->callRenderFunctionCall( $functionCall );
115
116            // 2.1. If successful, render the Z89K1/Html fragment value
117            $sanitizedHtml = $this->fragmentRenderer->render(
118                $htmlFragment[ ZTypeRegistry::Z_HTML_FRAGMENT_VALUE ],
119                [
120                    'qid' => $topicQid,
121                    'language' => $languageZid,
122                    'fragmentKey' => $fragmentKey,
123                ]
124            );
125
126            // ... and prepare rendered and sanitized response to be stored
127            $renderedValue[ 'success' ] = true;
128            $renderedValue[ 'value' ] = $sanitizedHtml;
129
130        } catch ( WikifunctionCallException $e ) {
131            // 2.2. If failure, log and prepare the error payload to be stored
132            $renderedValue[ 'success' ] = false;
133            $renderedValue[ 'value' ] = $e->toArray();
134
135            $httpStatusCode = $e->getHttpStatusCode();
136
137            if ( in_array( $httpStatusCode, HttpStatus::CONTENT_ERROR_CODES, true ) ) {
138                // The content caused the error, and an editor must correct it. Log it with the
139                // extra data, but do not make noise.
140                $logContext = [];
141                if ( $e->hasZError() ) {
142                    $logContext[ 'zerror' ] = $e->getZError();
143                }
144                $this->logger->info(
145                    __METHOD__ . ': AbstractWikiRequest::callRenderFunctionCall failed with a user error: {error}',
146                    [
147                        'error' => $e->getMessage(),
148                        'fragmentKey' => $fragmentKey,
149                        'httpStatusCode' => (string)$httpStatusCode,
150                    ] + $logContext
151                );
152            } elseif ( $httpStatusCode === HttpStatus::TOO_MANY_REQUESTS ) {
153                // e.g. Z570/Orchestrator rate limit, or the Wikifunctions API limits. The job
154                // tries again later, so log it without the trace.
155                $this->logger->info(
156                    __METHOD__ . ': AbstractWikiRequest::callRenderFunctionCall was rate limited: {error}',
157                    [
158                        'error' => $e->getMessage(),
159                        'fragmentKey' => $fragmentKey,
160                        'httpStatusCode' => (string)$httpStatusCode,
161                    ]
162                );
163            } elseif ( in_array( $httpStatusCode, self::WARNING_STATUS_CODES, true ) ) {
164                $this->logger->warning(
165                    __METHOD__ . ': AbstractWikiRequest::callRenderFunctionCall got a server issue: {error}',
166                    [
167                        'error' => $e->getMessage(),
168                        'exception' => $e,
169                        'fragmentKey' => $fragmentKey,
170                        'httpStatusCode' => (string)$httpStatusCode,
171                    ]
172                );
173            } else {
174                // We do not know this status code, so we cannot tell who caused the failure.
175                // Log it as an error, so that we can find it and add it to a list above.
176                $this->logger->error(
177                    __METHOD__ . ': AbstractWikiRequest::callRenderFunctionCall got unhandled error: {error}',
178                    [
179                        'error' => $e->getMessage(),
180                        'exception' => $e,
181                        'fragmentKey' => $fragmentKey,
182                        'httpStatusCode' => (string)$httpStatusCode,
183                    ]
184                );
185            }
186        }
187
188        // 4. Set the fragment in the AWFragmentStore
189        $this->fragmentStore->setRenderedAWFragment(
190            $topicQid,
191            $languageZid,
192            $datetime,
193            $fragmentKey,
194            $renderedValue
195        );
196
197        return $renderedValue;
198    }
199
200    /**
201     * Utility function to build a function call to the virtual function
202     * Z825/Run Abstract Fragment, using the fragment as an implementation and
203     * the args Qid, Language and Date as values for Z825K1, Z825K2 and Z825K3.
204     *
205     * Gets the data definition of the virtual function from function-schemata
206     * data/definitions/Z825.json file, as this will be present in the Abstract
207     * Repo environment (and any other)
208     *
209     * Returns the function call, or false if it could not be created
210     *
211     * @param array $fragment
212     * @param string $qid
213     * @param string $language
214     * @param string $datetime
215     * @return array
216     */
217    private function buildRenderAWFragmentCall(
218        array $fragment,
219        string $qid,
220        string $language,
221        string $datetime
222    ): array {
223        // Transform datetime ('YmdHis') into the date format needed by the cache key
224        $date = ( new ConvertibleTimestamp( $datetime ) )->format( 'Y-m-d' );
225
226        // We get the function definition from schemata because:
227        // * it will be available in the Abstract repo
228        // * we don't need the user-contributed labels
229        // * we can avoid making a remote fetch from wikifunctions
230        $function = $this->getRenderAWFragmentFunction();
231
232        // Set function's only implementation as fragment to execute:
233        $function[ ZTypeRegistry::Z_FUNCTION_IMPLEMENTATIONS ] = [
234            ZTypeRegistry::Z_IMPLEMENTATION,
235            [
236                ZTypeRegistry::Z_OBJECT_TYPE => ZTypeRegistry::Z_IMPLEMENTATION,
237                ZTypeRegistry::Z_IMPLEMENTATION_FUNCTION => ZTypeRegistry::Z_RUN_ABSTRACT_FRAGMENT,
238                ZTypeRegistry::Z_IMPLEMENTATION_COMPOSITION => $fragment
239            ]
240        ];
241
242        // Build argument: Wikidata reference object from qid
243        $wikidataReference = [
244            ZTypeRegistry::Z_OBJECT_TYPE => ZTypeRegistry::Z_WIKIDATA_REFERENCE_ITEM,
245            ZTypeRegistry::Z_WIKIDATA_REFERENCE_ITEM_ID => $qid
246        ];
247
248        // Build argument: Date parser function call from date string and language
249        $dateParser = [
250            ZTypeRegistry::Z_OBJECT_TYPE => ZTypeRegistry::Z_FUNCTIONCALL,
251            ZTypeRegistry::Z_FUNCTIONCALL_FUNCTION => ZTypeRegistry::Z_DATE_PARSER,
252            ZTypeRegistry::Z_DATE_PARSER_STRING => $date,
253            ZTypeRegistry::Z_DATE_PARSER_LANGUAGE => $language
254        ];
255
256        // Build function call to literal function:
257        $functionCall = [
258            ZTypeRegistry::Z_OBJECT_TYPE => ZTypeRegistry::Z_FUNCTIONCALL,
259            ZTypeRegistry::Z_FUNCTIONCALL_FUNCTION => $function,
260            ZTypeRegistry::Z_RUN_ABSTRACT_FRAGMENT_QID => $wikidataReference,
261            ZTypeRegistry::Z_RUN_ABSTRACT_FRAGMENT_LANGUAGE => $language,
262            ZTypeRegistry::Z_RUN_ABSTRACT_FRAGMENT_DATE => $dateParser
263        ];
264
265        return $functionCall;
266    }
267
268    /**
269     * Utility function to get the function definition for Z825/Render Abstract
270     * Fragment from the function schemata data definitions directory.
271     *
272     * @return array
273     * @throws RuntimeException
274     */
275    private function getRenderAWFragmentFunction(): array {
276        $functionPath = dirname( __DIR__ ) . '/../function-schemata/data/definitions/';
277        $functionFile = ZTypeRegistry::Z_RUN_ABSTRACT_FRAGMENT . '.json';
278
279        $functionDefinitionStr = file_get_contents( $functionPath . $functionFile );
280        $functionDefinition = json_decode( $functionDefinitionStr, true );
281        if ( !is_array( $functionDefinition ) ) {
282            // If the file was not found, or decoding it was not possible:
283            throw new RuntimeException( "Failed to load function-schemata definition for Z825" );
284        }
285
286        return $functionDefinition[ ZTypeRegistry::Z_PERSISTENTOBJECT_VALUE ];
287    }
288
289    /**
290     * Performs remote call to Wikifunctions' wikilambda_function_call Action API
291     *
292     * @param array $functionCall
293     * @return array HTML fragment (Z89) response object
294     * @throws WikifunctionCallException
295     */
296    public function callRenderFunctionCall( array $functionCall ): array {
297        // Base API URL from config
298        $targetUrl = $this->config->get( 'WikiLambdaClientTargetAPI' );
299        if ( !$targetUrl ) {
300            // Missing configuration
301            throw new WikifunctionCallException(
302                'apierror-abstractwiki_run_fragment-not-enabled',
303                HttpStatus::NOT_IMPLEMENTED
304            );
305        }
306        // The action must be present in the URL query string (not just the POST body)
307        // so it shows up in HTTP-layer logs on the remote wiki.
308        $apiUrl = $targetUrl . '/w/api.php?action=wikilambda_function_call';
309
310        try {
311            // Stringify the function call
312            $functionCallEncoded = json_encode( $functionCall, JSON_THROW_ON_ERROR );
313        } catch ( JsonException ) {
314            // Missing or corrupt schemata files means bad configuration
315            throw new WikifunctionCallException(
316                'apierror-abstractwiki_run_fragment-not-enabled',
317                HttpStatus::NOT_IMPLEMENTED
318            );
319        }
320
321        // Build POST params
322        $params = [
323            'format' => 'json',
324            'action' => 'wikilambda_function_call',
325            'wikilambda_function_call_zobject' => $functionCallEncoded,
326        ];
327
328        // Create and execute request
329        $request = $this->httpRequestFactory->create(
330            $apiUrl,
331            [
332                'method' => 'POST',
333                'postData' => $params
334            ],
335            __METHOD__
336        );
337
338        // Set request origin header
339        $request->setHeader( 'X-WikiLambda-Request-Origin', OrchestratorRequest::AW_FRAGMENT_ORIGIN_HEADER );
340
341        // How long does the orchestrator take to respond per fragment?
342        // Grafana: mediawiki.WikiLambda.aw_orchestrator_call_seconds{}
343        $orchestratorCallStart = microtime( true );
344        $status = $request->execute();
345        $this->statsFactory->getTiming( 'aw_orchestrator_call_seconds' )
346            ->observeSeconds( microtime( true ) - $orchestratorCallStart );
347        $apiHttpStatusCode = $request->getStatus();
348
349        // HTTP 503
350        // Transport level error; Wikifunctions could not be reached.
351        if ( !$status->isOK() && $apiHttpStatusCode === 0 ) {
352            throw new WikifunctionCallException(
353                'apierror-abstractwiki_run_fragment-service-unavailable',
354                HttpStatus::SERVICE_UNAVAILABLE
355            );
356        }
357
358        // HTTP 500
359        // Decode the response and check that content is a valid JSON object.
360        // If that's not the case, there's an unknown internal server error
361        $responseData = json_decode( $request->getContent(), true );
362        if ( !is_array( $responseData ) ) {
363            throw new WikifunctionCallException(
364                'apierror-abstractwiki_run_fragment-unknown-error',
365                HttpStatus::INTERNAL_SERVER_ERROR
366            );
367        }
368
369        // If there's an error key in the API response, it could be:
370        // * error.code !== 'wikilambda-zerror'
371        //   * 200/internal_api_error_* - some unknown error
372        //   * 429/apierror-wikilambda_function_call-concurrency-limit - too many requests
373        //   * 503/timeouterror-text - timeout
374        // * error.code === 'wikilambda-zerror'
375        //   * 503/Z529 - unable to connect to the orchestrator
376        //   * 403/Z559 - no permissions
377        //   * 400/Z501 - JSON styntax error
378        //   * 400/Z518 - ZObject type mismatch
379        if ( array_key_exists( 'error', $responseData ) ) {
380            $errorCode = $responseData[ 'error' ][ 'code' ];
381            // For 503 and 429, currently unavailable, try again later. Keep the code, so that
382            // we can tell a rate limit from an outage.
383            if (
384                $apiHttpStatusCode === HttpStatus::SERVICE_UNAVAILABLE ||
385                $apiHttpStatusCode === HttpStatus::TOO_MANY_REQUESTS
386            ) {
387                throw new WikifunctionCallException(
388                    'apierror-abstractwiki_run_fragment-service-unavailable',
389                    $apiHttpStatusCode
390                );
391            }
392
393            // For 403, permissions error, return forbidden:
394            if ( $apiHttpStatusCode === HttpStatus::FORBIDDEN ) {
395                throw new WikifunctionCallException(
396                    'apierror-abstractwiki_run_fragment-forbidden',
397                    HttpStatus::FORBIDDEN
398                );
399            }
400
401            // For 400, either wrong JSON or wrong ZObject type, return bad request:
402            if ( $apiHttpStatusCode === HttpStatus::BAD_REQUEST ) {
403                throw new WikifunctionCallException(
404                    'apierror-abstractwiki_run_fragment-bad-fragment',
405                    HttpStatus::BAD_REQUEST
406                );
407            }
408
409            // Else, return unknown issue:
410            throw new WikifunctionCallException(
411                'apierror-abstractwiki_run_fragment-unknown-error',
412                HttpStatus::INTERNAL_SERVER_ERROR
413            );
414        }
415
416        // The response does not have an error, but also doesn't have the
417        // expected content, so throw an unknown error.
418        if ( !array_key_exists( 'wikilambda_function_call', $responseData ) ) {
419            throw new WikifunctionCallException(
420                'apierror-abstractwiki_run_fragment-unknown-error',
421                HttpStatus::INTERNAL_SERVER_ERROR
422            );
423        }
424
425        // Give phan some assistance on what we expect the response to look like
426        '@phan-var array{wikilambda_function_call?: array{data?: string}} $responseData';
427
428        $responseEnvelopeStr = $responseData[ 'wikilambda_function_call' ][ 'data' ] ?? '';
429        $responseEnvelope = json_decode( $responseEnvelopeStr );
430
431        // The response is not a valid JSON, which probably means
432        // that there is some unknown bug somewhere, return unknown:
433        if ( !is_object( $responseEnvelope ) ) {
434            throw new WikifunctionCallException(
435                'apierror-abstractwiki_run_fragment-unknown-error',
436                HttpStatus::INTERNAL_SERVER_ERROR
437            );
438        }
439
440        // Give phan some assistance on what we expect the envelope to look like
441        '@phan-var object{Z22K1:array<string,string|array>,Z22K2:array<string,string|array>} $responseEnvelope';
442
443        // How long did the orchestrator itself spend evaluating, excluding network time?
444        // Grafana: mediawiki.WikiLambda.aw_orchestration_duration_seconds{}
445        $orchestrationDuration = ZObjectUtils::getMetaDataValue( $responseEnvelope, 'orchestrationDuration' );
446        if ( $orchestrationDuration !== null ) {
447            $this->statsFactory->getTiming( 'aw_orchestration_duration_seconds' )
448                ->observe( floatval( $orchestrationDuration ) );
449        }
450
451        $htmlFragment = $responseEnvelope->{ ZTypeRegistry::Z_RESPONSEENVELOPE_VALUE };
452
453        // If the response value is Void, there is an error in the response metadata.
454        // We can capture it and show some stuff.
455        if ( $htmlFragment === ZTypeRegistry::Z_VOID ) {
456            $zerror = ZObjectUtils::getMetaDataValue( $responseEnvelope, 'errors' );
457            // (T432333) Surface http status code assigned and returned by the orchestrator
458            // This http status code maps with the inner zerror zid according to the mappings
459            // in `function-schemata/test_data/errors/http_status_mappings.yaml`
460            $orchestratorHttpStatusCode = $responseData['wikilambda_function_call']['orchestratorHttpStatusCode'] ??
461                HttpStatus::BAD_REQUEST;
462            throw new WikifunctionCallException(
463                'apierror-abstractwiki_run_fragment-returned-zerror',
464                $orchestratorHttpStatusCode,
465                $zerror,
466                [ $zerror->{'Z5K1'} ?? 'No error code provided' ]
467            );
468        }
469
470        // We make sure that the result is a Z89/HTML fragment.
471        // If not the case, the fragment is probably wrong, so return 400:
472        if ( !is_object( $htmlFragment ) || !property_exists( $htmlFragment, ZTypeRegistry::Z_HTML_FRAGMENT_VALUE ) ) {
473            throw new WikifunctionCallException(
474                'apierror-abstractwiki_run_fragment-bad-response',
475                HttpStatus::BAD_REQUEST
476            );
477        }
478
479        // The orchestrator may emit Z89K1 in canonical form ("foo") or normal form
480        // ({ Z1K1: "Z6", Z6K1: "foo" }); canonicalise so downstream always sees a string (T426297).
481        $htmlFragment->{ ZTypeRegistry::Z_HTML_FRAGMENT_VALUE } = ZObjectUtils::canonicalize(
482            $htmlFragment->{ ZTypeRegistry::Z_HTML_FRAGMENT_VALUE }
483        );
484
485        if ( !is_string( $htmlFragment->{ ZTypeRegistry::Z_HTML_FRAGMENT_VALUE } ) ) {
486            throw new WikifunctionCallException(
487                'apierror-abstractwiki_run_fragment-bad-response',
488                HttpStatus::BAD_REQUEST
489            );
490        }
491
492        return (array)$htmlFragment;
493    }
494}