Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
86.59% |
310 / 358 |
|
54.55% |
6 / 11 |
CRAP | |
0.00% |
0 / 1 |
| ApiPerformTest | |
86.59% |
310 / 358 |
|
54.55% |
6 / 11 |
91.29 | |
0.00% |
0 / 1 |
| __construct | |
100.00% |
5 / 5 |
|
100.00% |
1 / 1 |
1 | |||
| execute | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| executeGenerator | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| run | |
80.93% |
174 / 215 |
|
0.00% |
0 / 1 |
46.49 | |||
| getImplementationListEntry | |
100.00% |
7 / 7 |
|
100.00% |
1 / 1 |
4 | |||
| getTesterObject | |
100.00% |
11 / 11 |
|
100.00% |
1 / 1 |
6 | |||
| isFalse | |
63.64% |
7 / 11 |
|
0.00% |
0 / 1 |
9.36 | |||
| getAllowedParams | n/a |
0 / 0 |
n/a |
0 / 0 |
1 | |||||
| getExamplesMessages | n/a |
0 / 0 |
n/a |
0 / 0 |
1 | |||||
| isInternal | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| getNumericMetadataValue | |
100.00% |
6 / 6 |
|
100.00% |
1 / 1 |
2 | |||
| compareImplementationStats | |
88.89% |
8 / 9 |
|
0.00% |
0 / 1 |
5.03 | |||
| maybeUpdateImplementationRanking | |
100.00% |
91 / 91 |
|
100.00% |
1 / 1 |
10 | |||
| 1 | <?php |
| 2 | /** |
| 3 | * WikiLambda function call API |
| 4 | * |
| 5 | * @file |
| 6 | * @ingroup Extensions |
| 7 | * @copyright 2020– Abstract Wikipedia team; see AUTHORS.txt |
| 8 | * @license MIT |
| 9 | */ |
| 10 | |
| 11 | namespace MediaWiki\Extension\WikiLambda\ActionAPI; |
| 12 | |
| 13 | use ApiPageSet; |
| 14 | use FormatJson; |
| 15 | use JobQueueGroup; |
| 16 | use MediaWiki\Extension\WikiLambda\Jobs\CacheTesterResultsJob; |
| 17 | use MediaWiki\Extension\WikiLambda\Jobs\UpdateImplementationsJob; |
| 18 | use MediaWiki\Extension\WikiLambda\Registry\ZErrorTypeRegistry; |
| 19 | use MediaWiki\Extension\WikiLambda\Registry\ZTypeRegistry; |
| 20 | use MediaWiki\Extension\WikiLambda\ZErrorException; |
| 21 | use MediaWiki\Extension\WikiLambda\ZErrorFactory; |
| 22 | use MediaWiki\Extension\WikiLambda\ZObjectFactory; |
| 23 | use MediaWiki\Extension\WikiLambda\ZObjects\ZObject; |
| 24 | use MediaWiki\Extension\WikiLambda\ZObjects\ZReference; |
| 25 | use MediaWiki\Extension\WikiLambda\ZObjects\ZResponseEnvelope; |
| 26 | use MediaWiki\Extension\WikiLambda\ZObjects\ZString; |
| 27 | use MediaWiki\Extension\WikiLambda\ZObjects\ZTypedList; |
| 28 | use MediaWiki\Extension\WikiLambda\ZObjects\ZTypedMap; |
| 29 | use MediaWiki\Extension\WikiLambda\ZObjectStore; |
| 30 | use MediaWiki\Extension\WikiLambda\ZObjectUtils; |
| 31 | use MediaWiki\Logger\LoggerFactory; |
| 32 | use MediaWiki\MediaWikiServices; |
| 33 | use MediaWiki\Title\Title; |
| 34 | use Wikimedia\ParamValidator\ParamValidator; |
| 35 | |
| 36 | class ApiPerformTest extends WikiLambdaApiBase { |
| 37 | |
| 38 | private ZObjectStore $zObjectStore; |
| 39 | private JobQueueGroup $jobQueueGroup; |
| 40 | |
| 41 | public function __construct( $query, $moduleName, ZObjectStore $zObjectStore ) { |
| 42 | parent::__construct( $query, $moduleName, 'wikilambda_perform_test_' ); |
| 43 | |
| 44 | $this->zObjectStore = $zObjectStore; |
| 45 | |
| 46 | $this->setUp(); |
| 47 | |
| 48 | // TODO (T330033): Consider injecting this service rather than just fetching from main |
| 49 | $services = MediaWikiServices::getInstance(); |
| 50 | $this->jobQueueGroup = $services->getJobQueueGroup(); |
| 51 | } |
| 52 | |
| 53 | public function execute() { |
| 54 | $this->run(); |
| 55 | } |
| 56 | |
| 57 | public function executeGenerator( $resultPageSet ) { |
| 58 | $this->run( $resultPageSet ); |
| 59 | } |
| 60 | |
| 61 | /** |
| 62 | * @param ApiPageSet|null $resultPageSet |
| 63 | */ |
| 64 | private function run( $resultPageSet = null ) { |
| 65 | $params = $this->extractRequestParams(); |
| 66 | $pageResult = $this->getResult(); |
| 67 | $functionZid = $params[ 'zfunction' ]; |
| 68 | $requestedImplementations = $params[ 'zimplementations' ] ?: []; |
| 69 | $requestedTesters = $params[ 'ztesters' ] ?: []; |
| 70 | |
| 71 | // 1. Work out matrix of what we want for what |
| 72 | // TODO (T362190): Consider handling an inline ZFunction (for when it's not been created yet)? |
| 73 | $targetTitle = Title::newFromText( $functionZid, NS_MAIN ); |
| 74 | if ( !$targetTitle || !( $targetTitle->exists() ) ) { |
| 75 | $this->dieWithError( [ "wikilambda-performtest-error-unknown-zid", $functionZid ], null, null, 404 ); |
| 76 | } |
| 77 | |
| 78 | // Needed for caching. |
| 79 | $functionRevision = $targetTitle->getLatestRevID(); |
| 80 | |
| 81 | // @phan-suppress-next-line PhanTypeMismatchArgumentNullable |
| 82 | $targetObject = $this->zObjectStore->fetchZObjectByTitle( $targetTitle ); |
| 83 | if ( $targetObject->getZType() !== ZTypeRegistry::Z_FUNCTION ) { |
| 84 | $this->dieWithError( [ "wikilambda-performtest-error-nonfunction", $functionZid ], null, null, 400 ); |
| 85 | } |
| 86 | $targetFunction = $targetObject->getInnerZObject(); |
| 87 | '@phan-var \MediaWiki\Extension\WikiLambda\ZObjects\ZFunction $targetFunction'; |
| 88 | |
| 89 | if ( !count( $requestedImplementations ) ) { |
| 90 | $targetFunctionImplementions = $targetFunction->getValueByKey( ZTypeRegistry::Z_FUNCTION_IMPLEMENTATIONS ); |
| 91 | '@phan-var \MediaWiki\Extension\WikiLambda\ZObjects\ZTypedList $targetFunctionImplementions'; |
| 92 | $requestedImplementations = $targetFunctionImplementions->getAsArray(); |
| 93 | } |
| 94 | |
| 95 | if ( !count( $requestedTesters ) ) { |
| 96 | $targetFunctionTesters = $targetFunction->getValueByKey( ZTypeRegistry::Z_FUNCTION_TESTERS ); |
| 97 | '@phan-var \MediaWiki\Extension\WikiLambda\ZObjects\ZTypedList $targetFunctionTesters'; |
| 98 | $requestedTesters = $targetFunctionTesters->getAsArray(); |
| 99 | } |
| 100 | |
| 101 | // We only update the implementation ranking for attached implementations and testers, |
| 102 | // and only if all attached implementations and testers are included in the results, |
| 103 | // and at least one result is live (not from cache). These vars are used to track |
| 104 | // those conditions. |
| 105 | $attachedImplementationZids = $targetFunction->getImplementationZids(); |
| 106 | $attachedTesterZids = $targetFunction->getTesterZids(); |
| 107 | $canUpdateImplementationRanking = false; |
| 108 | |
| 109 | // 2. For each implementation, run each tester |
| 110 | $responseArray = []; |
| 111 | // Map of $implementationZid:$testerMap; used for implementation ranking |
| 112 | $implementationMap = []; |
| 113 | foreach ( $requestedImplementations as $implementation ) { |
| 114 | $inlineImplementation = false; |
| 115 | if ( is_string( $implementation ) ) { |
| 116 | // (T358089) Decode any '|' characters of ZObjects that were escaped for the API transit |
| 117 | $implementation = str_replace( '🪈', '|', $implementation ); |
| 118 | $decodedJson = FormatJson::decode( $implementation ); |
| 119 | // If not JSON, assume we have received a ZID. |
| 120 | if ( $decodedJson ) { |
| 121 | $inlineImplementation = true; |
| 122 | |
| 123 | if ( !$this->getContext()->getAuthority()->isAllowed( 'wikilambda-create-implementation' ) ) { |
| 124 | $zError = ZErrorFactory::createZErrorInstance( |
| 125 | ZErrorTypeRegistry::Z_ERROR_USER_CANNOT_RUN, |
| 126 | [] |
| 127 | ); |
| 128 | $this->dieWithZError( $zError, 403 ); |
| 129 | } |
| 130 | |
| 131 | try { |
| 132 | $implementation = ZObjectFactory::create( $decodedJson ); |
| 133 | } catch ( ZErrorException $e ) { |
| 134 | $this->dieWithError( |
| 135 | [ |
| 136 | 'wikilambda-performtest-error-invalidimplementation', |
| 137 | $e->getZErrorMessage() |
| 138 | ], |
| 139 | null, null, 400 |
| 140 | ); |
| 141 | } |
| 142 | } else { |
| 143 | $implementation = new ZReference( $implementation ); |
| 144 | } |
| 145 | } |
| 146 | $implementationZid = ZObjectUtils::getZid( $implementation ); |
| 147 | $implementationListEntry = $this->getImplementationListEntry( $implementation ); |
| 148 | |
| 149 | // Note that the Implementation ZID can be non-Z0 if it's being run on an unsaved edit. |
| 150 | $implementationRevision = null; |
| 151 | if ( !$inlineImplementation ) { |
| 152 | $title = Title::newFromText( $implementationZid, NS_MAIN ); |
| 153 | if ( $title ) { |
| 154 | $implementationRevision = $title->getLatestRevID(); |
| 155 | } else { |
| 156 | $this->dieWithError( |
| 157 | [ |
| 158 | 'wikilambda-performtest-error-invalidimplementation', |
| 159 | $implementationZid |
| 160 | ], |
| 161 | null, null, 400 |
| 162 | ); |
| 163 | } |
| 164 | } |
| 165 | |
| 166 | // Re-use our copy of the target function, setting the implementations to just the one |
| 167 | // we're testing now |
| 168 | $targetFunction->setValueByKey( |
| 169 | ZTypeRegistry::Z_FUNCTION_IMPLEMENTATIONS, |
| 170 | new ZTypedList( |
| 171 | ZTypedList::buildType( new ZReference( ZTypeRegistry::Z_IMPLEMENTATION ) ), |
| 172 | $implementationListEntry |
| 173 | ) |
| 174 | ); |
| 175 | // Map of $testerZid:$testResult for a particular implementation |
| 176 | $testerMap = []; |
| 177 | foreach ( $requestedTesters as $requestedTester ) { |
| 178 | $passed = true; |
| 179 | $testResult = [ |
| 180 | 'zFunctionId' => $functionZid, |
| 181 | 'zImplementationId' => $implementationZid, |
| 182 | ]; |
| 183 | |
| 184 | $inlineTester = false; |
| 185 | if ( is_string( $requestedTester ) ) { |
| 186 | // (T358089) Decode any '|' characters of ZObjects that were escaped for the API transit |
| 187 | $requestedTester = str_replace( '🪈', '|', $requestedTester ); |
| 188 | $decodedJson = FormatJson::decode( $requestedTester ); |
| 189 | // If not JSON, assume we have received a ZID. |
| 190 | if ( $decodedJson ) { |
| 191 | $inlineTester = true; |
| 192 | |
| 193 | if ( !$this->getContext()->getAuthority()->isAllowed( 'wikilambda-create-tester' ) ) { |
| 194 | $zError = ZErrorFactory::createZErrorInstance( |
| 195 | ZErrorTypeRegistry::Z_ERROR_USER_CANNOT_RUN, |
| 196 | [] |
| 197 | ); |
| 198 | $this->dieWithZError( $zError, 403 ); |
| 199 | } |
| 200 | |
| 201 | try { |
| 202 | $requestedTester = ZObjectFactory::create( $decodedJson ); |
| 203 | } catch ( ZErrorException $e ) { |
| 204 | $this->dieWithError( |
| 205 | [ |
| 206 | 'wikilambda-performtest-error-invalidtester', |
| 207 | $e->getZErrorMessage() |
| 208 | ], |
| 209 | null, null, 400 |
| 210 | ); |
| 211 | } |
| 212 | } else { |
| 213 | $requestedTester = new ZReference( $requestedTester ); |
| 214 | } |
| 215 | } |
| 216 | |
| 217 | $testerZid = ZObjectUtils::getZid( $requestedTester ); |
| 218 | $testResult[ 'zTesterId' ] = $testerZid; |
| 219 | $testerObject = $this->getTesterObject( $requestedTester ); |
| 220 | |
| 221 | // Note that the Tester ZID can be non-Z0 if it's being run on an unsaved edit. |
| 222 | $testerRevision = null; |
| 223 | if ( !$inlineTester ) { |
| 224 | $title = Title::newFromText( $testerZid, NS_MAIN ); |
| 225 | if ( $title ) { |
| 226 | $testerRevision = $title->getLatestRevID(); |
| 227 | } else { |
| 228 | $this->dieWithError( |
| 229 | [ |
| 230 | 'wikilambda-performtest-error-invalidtester', |
| 231 | $testerZid |
| 232 | ], |
| 233 | null, null, 400 |
| 234 | ); |
| 235 | } |
| 236 | } |
| 237 | |
| 238 | // (T297707): Work out if this has been cached before (checking revisions of objects), |
| 239 | // and if so reply with that instead of executing. |
| 240 | if ( !$inlineImplementation && !$inlineTester ) { |
| 241 | $possiblyCachedResult = $this->zObjectStore->findZTesterResult( |
| 242 | $functionZid, |
| 243 | $functionRevision, |
| 244 | $implementationZid, |
| 245 | $implementationRevision, |
| 246 | $testerZid, |
| 247 | $testerRevision, |
| 248 | ); |
| 249 | |
| 250 | if ( $possiblyCachedResult ) { |
| 251 | $possiblyCachedResult->setMetaDataValue( |
| 252 | "loadedFromMediaWikiCache", |
| 253 | new ZString( date( 'Y-m-d\TH:i:s\Z' ) ) |
| 254 | ); |
| 255 | |
| 256 | $this->getLogger()->debug( |
| 257 | 'Cache result hit: ' . $possiblyCachedResult->getZValue(), |
| 258 | [] |
| 259 | ); |
| 260 | $testResult[ 'validateStatus' ] = $possiblyCachedResult->getZValue(); |
| 261 | $testResult[ 'testMetadata'] = $possiblyCachedResult->getZMetadata(); |
| 262 | |
| 263 | $responseArray[] = $testResult; |
| 264 | // Update bookkeeping for the call to maybeUpdateImplementationRanking, if needed. |
| 265 | // Implementation ranking only involves attached implementations and testers. |
| 266 | if ( in_array( $testerZid, $attachedTesterZids ) && |
| 267 | in_array( $implementationZid, $attachedImplementationZids ) ) { |
| 268 | $testerMap[$testerZid] = $testResult; |
| 269 | } |
| 270 | continue; |
| 271 | } |
| 272 | } |
| 273 | |
| 274 | // Use tester to create a function call of the test case inputs |
| 275 | $testFunctionCall = $testerObject->getValueByKey( ZTypeRegistry::Z_TESTER_CALL ); |
| 276 | '@phan-var \MediaWiki\Extension\WikiLambda\ZObjects\ZFunctionCall $testFunctionCall'; |
| 277 | |
| 278 | // Set the target function of the call to our modified copy of the target function with only the |
| 279 | // current implementation |
| 280 | $testFunctionCall->setValueByKey( ZTypeRegistry::Z_FUNCTIONCALL_FUNCTION, $targetFunction ); |
| 281 | |
| 282 | // Execute the test case function call |
| 283 | $testResultObject = $this->executeFunctionCall( $testFunctionCall, true ); |
| 284 | $testMetadata = $testResultObject->getValueByKey( ZTypeRegistry::Z_RESPONSEENVELOPE_METADATA ); |
| 285 | '@phan-var \MediaWiki\Extension\WikiLambda\ZObjects\ZTypedMap $testMetadata'; |
| 286 | |
| 287 | // Use tester to create a function call validating the output |
| 288 | $validateTestValue = $testResultObject->hasErrors() ? |
| 289 | $testResultObject->getErrors() : |
| 290 | $testResultObject->getZValue(); |
| 291 | |
| 292 | $validateFunctionCall = $testerObject->getValueByKey( ZTypeRegistry::Z_TESTER_VALIDATION ); |
| 293 | '@phan-var \MediaWiki\Extension\WikiLambda\ZObjects\ZFunctionCall $validateFunctionCall'; |
| 294 | |
| 295 | $targetValidationFunctionZID = $validateFunctionCall->getZValue(); |
| 296 | $validateFunctionCall->setValueByKey( $targetValidationFunctionZID . 'K1', $validateTestValue ); |
| 297 | |
| 298 | // Execute the validation function call and stash it |
| 299 | $validateResult = $this->executeFunctionCall( $validateFunctionCall, false ); |
| 300 | |
| 301 | // If the running failed, we explicitly set the tester passing as false |
| 302 | if ( $validateResult->hasErrors() ) { |
| 303 | $validateResult->setValueByKey( |
| 304 | ZTypeRegistry::Z_RESPONSEENVELOPE_VALUE, |
| 305 | new ZReference( ZTypeRegistry::Z_BOOLEAN_FALSE ) |
| 306 | ); |
| 307 | // Add the validator errors to the metadata map |
| 308 | $testMetadata->setValueForKey( |
| 309 | new ZString( "validateErrors" ), |
| 310 | $validateResult->getErrors() ); |
| 311 | } |
| 312 | |
| 313 | $validateResultItem = $validateResult->getZValue(); |
| 314 | |
| 315 | if ( self::isFalse( $validateResultItem ) ) { |
| 316 | $passed = false; |
| 317 | // Add the expected and actual values to the metadata map |
| 318 | $testMetadata->setValueForKey( |
| 319 | new ZString( "actualTestResult" ), |
| 320 | $validateTestValue |
| 321 | ); |
| 322 | $testMetadata->setValueForKey( |
| 323 | new ZString( "expectedTestResult" ), |
| 324 | $validateFunctionCall->getValueByKey( $targetValidationFunctionZID . 'K2' ) |
| 325 | ); |
| 326 | } |
| 327 | |
| 328 | // (T297707): Store this response in a DB table for faster future responses. |
| 329 | // We can only do this for persisted revisions, not inline items, as we can't |
| 330 | // version them otherwise, so use truthiness (neither null nor 0, non-extant). |
| 331 | // We also only do this if the validation step didn't have an error itself. |
| 332 | if ( |
| 333 | !$inlineImplementation && !$inlineTester && |
| 334 | !$validateResult->hasErrors() |
| 335 | ) { |
| 336 | // Store a fake ZResponseEnvelope of the validation result and the real meta-data run |
| 337 | // via an asynchronous job so that we don't trigger a "DB write on API GET" performance |
| 338 | // error. |
| 339 | $this->getLogger()->debug( |
| 340 | 'Tester result cache job triggered', |
| 341 | [ |
| 342 | 'functionZid' => $functionZid, |
| 343 | 'functionRevision' => $functionRevision |
| 344 | ] |
| 345 | ); |
| 346 | |
| 347 | $stashedResult = new ZResponseEnvelope( $validateResultItem, $testMetadata ); |
| 348 | |
| 349 | $cacheTesterResultsJob = new CacheTesterResultsJob( |
| 350 | [ |
| 351 | 'functionZid' => $functionZid, |
| 352 | 'functionRevision' => $functionRevision, |
| 353 | 'implementationZid' => $implementationZid, |
| 354 | 'implementationRevision' => $implementationRevision, |
| 355 | 'testerZid' => $testerZid, |
| 356 | 'testerRevision' => $testerRevision, |
| 357 | 'passed' => $passed, |
| 358 | 'stashedResult' => $stashedResult->__toString() |
| 359 | ] |
| 360 | ); |
| 361 | |
| 362 | $this->jobQueueGroup->push( $cacheTesterResultsJob ); |
| 363 | } |
| 364 | |
| 365 | // Stash the response |
| 366 | $testResult[ 'validateStatus' ] = $validateResultItem; |
| 367 | $testResult[ 'testMetadata' ] = $testMetadata; |
| 368 | $responseArray[] = $testResult; |
| 369 | |
| 370 | // Update bookkeeping for the call to maybeUpdateImplementationRanking, if needed. |
| 371 | // Implementation ranking only involves attached implementations and testers. |
| 372 | if ( in_array( $testerZid, $attachedTesterZids ) && |
| 373 | in_array( $implementationZid, $attachedImplementationZids ) ) { |
| 374 | $testerMap[$testerZid] = $testResult; |
| 375 | // Since this $testResult is "live" (not from cache), indicating that the |
| 376 | // function, implementation, or tester has changed, we should check |
| 377 | // if there is an improved implementation ranking |
| 378 | // TODO (T330370): Revisit this strategy when we have more experience with it |
| 379 | $canUpdateImplementationRanking = true; |
| 380 | } |
| 381 | } |
| 382 | // Update bookkeeping for the call to maybeUpdateImplementationRanking, if needed. |
| 383 | if ( in_array( $implementationZid, $attachedImplementationZids ) ) { |
| 384 | $implementationMap[ $implementationZid ] = $testerMap; |
| 385 | } |
| 386 | } |
| 387 | |
| 388 | // 3. Maybe update implementation ranking (in persistent storage) |
| 389 | if ( $canUpdateImplementationRanking ) { |
| 390 | $this->maybeUpdateImplementationRanking( |
| 391 | $functionZid, |
| 392 | $functionRevision, |
| 393 | $implementationMap, |
| 394 | $attachedImplementationZids, |
| 395 | $attachedTesterZids |
| 396 | ); |
| 397 | } else { |
| 398 | $this->getLogger()->info( |
| 399 | __METHOD__ . ' Not updating {functionZid} implementation ranking; no live results', |
| 400 | [ |
| 401 | 'functionZid' => $functionZid, |
| 402 | 'canUpdateImplementationRanking' => $canUpdateImplementationRanking |
| 403 | ] |
| 404 | ); |
| 405 | } |
| 406 | |
| 407 | // 4. Return the response. |
| 408 | $pageResult->addValue( [ 'query' ], $this->getModuleName(), $responseArray ); |
| 409 | } |
| 410 | |
| 411 | private function getImplementationListEntry( $zobject ) { |
| 412 | if ( $zobject->getZType() === ZTypeRegistry::Z_REFERENCE || |
| 413 | $zobject->getZType() === ZTypeRegistry::Z_IMPLEMENTATION ) { |
| 414 | return $zobject; |
| 415 | } elseif ( $zobject->getZType() === ZTypeRegistry::Z_PERSISTENTOBJECT ) { |
| 416 | return $this->getImplementationListEntry( |
| 417 | $zobject->getValueByKey( ZTypeRegistry::Z_PERSISTENTOBJECT_VALUE ) ); |
| 418 | } |
| 419 | $this->dieWithError( [ "wikilambda-performtest-error-nonimplementation", $zobject ], null, null, 400 ); |
| 420 | } |
| 421 | |
| 422 | private function getTesterObject( $zobject ) { |
| 423 | if ( $zobject->getZType() === ZTypeRegistry::Z_REFERENCE ) { |
| 424 | $zid = ZObjectUtils::getZid( $zobject ); |
| 425 | $title = Title::newFromText( $zid, NS_MAIN ); |
| 426 | if ( !( $title instanceof Title ) || !$title->exists() ) { |
| 427 | $this->dieWithError( [ "wikilambda-performtest-error-unknown-zid", $zid ], null, null, 404 ); |
| 428 | } |
| 429 | // @phan-suppress-next-line PhanTypeMismatchArgumentNullable |
| 430 | return $this->getTesterObject( $this->zObjectStore->fetchZObjectByTitle( $title )->getInnerZObject() ); |
| 431 | } elseif ( $zobject->getZType() === ZTypeRegistry::Z_PERSISTENTOBJECT ) { |
| 432 | return $this->getTesterObject( $zobject->getValueByKey( ZTypeRegistry::Z_PERSISTENTOBJECT_VALUE ) ); |
| 433 | } elseif ( $zobject->getZType() === ZTypeRegistry::Z_TESTER ) { |
| 434 | return $zobject; |
| 435 | } |
| 436 | $this->dieWithError( [ "wikilambda-performtest-error-nontester", $zobject ], null, null, 400 ); |
| 437 | } |
| 438 | |
| 439 | private static function isFalse( $object ) { |
| 440 | if ( $object instanceof ZObject ) { |
| 441 | if ( $object instanceof ZReference ) { |
| 442 | return self::isFalse( $object->getZValue() ); |
| 443 | } elseif ( $object->getZType() === ZTypeRegistry::Z_BOOLEAN ) { |
| 444 | return self::isFalse( $object->getValueByKey( ZTypeRegistry::Z_BOOLEAN_VALUE ) ); |
| 445 | } |
| 446 | } elseif ( $object instanceof \stdClass ) { |
| 447 | if ( $object->{ ZTypeRegistry::Z_OBJECT_TYPE } === ZTypeRegistry::Z_REFERENCE ) { |
| 448 | return self::isFalse( $object->{ ZTypeRegistry::Z_REFERENCE_VALUE } ); |
| 449 | } elseif ( $object->{ ZTypeRegistry::Z_OBJECT_TYPE } === ZTypeRegistry::Z_BOOLEAN ) { |
| 450 | return self::isFalse( $object->{ ZTypeRegistry::Z_BOOLEAN_VALUE } ); |
| 451 | } |
| 452 | } |
| 453 | return $object === ZTypeRegistry::Z_BOOLEAN_FALSE; |
| 454 | } |
| 455 | |
| 456 | /** |
| 457 | * @inheritDoc |
| 458 | * @codeCoverageIgnore |
| 459 | */ |
| 460 | protected function getAllowedParams(): array { |
| 461 | return [ |
| 462 | 'zfunction' => [ |
| 463 | ParamValidator::PARAM_TYPE => 'text', |
| 464 | ParamValidator::PARAM_REQUIRED => true, |
| 465 | ], |
| 466 | 'zimplementations' => [ |
| 467 | ParamValidator::PARAM_ISMULTI => true, |
| 468 | ParamValidator::PARAM_REQUIRED => false, |
| 469 | ], |
| 470 | 'ztesters' => [ |
| 471 | ParamValidator::PARAM_ISMULTI => true, |
| 472 | ParamValidator::PARAM_REQUIRED => false, |
| 473 | ], |
| 474 | ]; |
| 475 | } |
| 476 | |
| 477 | /** |
| 478 | * @see ApiBase::getExamplesMessages() |
| 479 | * @return array |
| 480 | * @codeCoverageIgnore |
| 481 | */ |
| 482 | protected function getExamplesMessages() { |
| 483 | $exampleZid = $this->zObjectStore->findFirstZImplementationFunction(); |
| 484 | |
| 485 | $queryPrefix = 'action=wikilambda_perform_test&format=json&wikilambda_perform_test_zfunction='; |
| 486 | |
| 487 | return [ |
| 488 | $queryPrefix . $exampleZid |
| 489 | => 'apihelp-wikilambda_perform_test-example', |
| 490 | $queryPrefix . 'Z801' |
| 491 | => 'apihelp-wikilambda_perform_test-z801', |
| 492 | $queryPrefix . 'Z801&wikilambda_perform_test_zimplementations=Z901' |
| 493 | => 'apihelp-wikilambda_perform_test-z801-implementation', |
| 494 | $queryPrefix . 'Z801&wikilambda_perform_test_ztesters=Z8010|Z8011' |
| 495 | => 'apihelp-wikilambda_perform_test-z801-tester', |
| 496 | $queryPrefix . 'Z801&wikilambda_perform_test_zimplementations=Z901&wikilambda_perform_test_ztesters=Z8010' |
| 497 | => 'apihelp-wikilambda_perform_test-z801-implementation-and-testers', |
| 498 | ]; |
| 499 | } |
| 500 | |
| 501 | /** |
| 502 | * Mark as internal. This isn't meant to be user-facing, and can change at any time. |
| 503 | * @return bool |
| 504 | */ |
| 505 | public function isInternal() { |
| 506 | return true; |
| 507 | } |
| 508 | |
| 509 | /** |
| 510 | * Retrieves the $metadataMap value for ZString($keyString) and converts it to a float. It must |
| 511 | * be a value of type ZString, whose underlying string begins with a float, e.g. '320.815 ms'. |
| 512 | * If ZString($keyString) isn't used in $metadataMap, returns zero. |
| 513 | * |
| 514 | * N.B. We do not check the units here; we assume that they are always the same (i.e., |
| 515 | * milliseconds) for the values retrieved by this function. This consistency is primarily |
| 516 | * the responsibility of the backend services that generate the metadata elements. |
| 517 | * |
| 518 | * @param ZTypedMap $metadataMap |
| 519 | * @param string $keyString |
| 520 | * @return float |
| 521 | */ |
| 522 | private static function getNumericMetadataValue( $metadataMap, $keyString ) { |
| 523 | $key = new ZString( $keyString ); |
| 524 | $value = $metadataMap->getValueGivenKey( $key ); |
| 525 | if ( !$value ) { |
| 526 | return 0; |
| 527 | } |
| 528 | $value = $value->getZValue(); |
| 529 | return floatval( $value ); |
| 530 | } |
| 531 | |
| 532 | /** |
| 533 | * Callback for uasort() to order the implementations for a function that's been tested |
| 534 | * @param array $a Implementation stats |
| 535 | * @param array $b Implementation stats |
| 536 | * @return int Result of comparison |
| 537 | */ |
| 538 | private static function compareImplementationStats( $a, $b ) { |
| 539 | if ( $a[ 'numFailed' ] < $b[ 'numFailed' ] ) { |
| 540 | return -1; |
| 541 | } |
| 542 | if ( $b[ 'numFailed' ] < $a[ 'numFailed' ] ) { |
| 543 | return 1; |
| 544 | } |
| 545 | if ( $a[ 'averageTime' ] < $b[ 'averageTime' ] ) { |
| 546 | return -1; |
| 547 | } |
| 548 | if ( $b[ 'averageTime' ] < $a[ 'averageTime' ] ) { |
| 549 | return 1; |
| 550 | } |
| 551 | return 0; |
| 552 | } |
| 553 | |
| 554 | /** |
| 555 | * Based on tester results contained in $implementationMap, order the implementations of the |
| 556 | * given function from best-performing to worst-performing (in terms of speed). If the |
| 557 | * ordering is significantly different than the previous ordering for this function, instantiate |
| 558 | * an asynchronous job to update Z8K4/implementations in the function's persistent storage. |
| 559 | * |
| 560 | * TODO (T329138): Consider whether average Cpu usage is good enough to determine the ranking. |
| 561 | * Should we eliminate implementations that are outliers relative to others on the same test? |
| 562 | * Should we consider non-CPU time needed to, e.g., retrieve info from Wikidata? |
| 563 | * |
| 564 | * @param string $functionZid |
| 565 | * @param int $functionRevision |
| 566 | * @param array $implementationMap contains $implementationZid => $testerMap, for each tested |
| 567 | * implementation. $testerMap contains $testerZid => $testResult for each tester. See |
| 568 | * ApiPerformTest::run for the structure of $testResult. |
| 569 | * @param array $attachedImplementationZids |
| 570 | * @param array $attachedTesterZids |
| 571 | */ |
| 572 | public static function maybeUpdateImplementationRanking( |
| 573 | $functionZid, $functionRevision, $implementationMap, $attachedImplementationZids, $attachedTesterZids |
| 574 | ) { |
| 575 | // NOTE: As this code is static for testing purposes, we can't use $this->getLogger() here |
| 576 | $logger = LoggerFactory::getInstance( 'WikiLambda' ); |
| 577 | |
| 578 | // We don't currently support updates involving a Z0, and we don't expect to get any here. |
| 579 | // (However, it maybe could happen if the value of Z8K4 has been manually edited.) |
| 580 | unset( $implementationMap[ ZTypeRegistry::Z_NULL_REFERENCE ] ); |
| 581 | |
| 582 | if ( count( $attachedImplementationZids ) <= 1 ) { |
| 583 | // No point in updating. |
| 584 | $logger->debug( |
| 585 | __METHOD__ . ' Not updating {functionZid}: Implementation count <= 1', |
| 586 | [ |
| 587 | 'functionZid' => $functionZid, |
| 588 | 'functionRevision' => $functionRevision, |
| 589 | 'implementationMap' => $implementationMap |
| 590 | ] |
| 591 | ); |
| 592 | return; |
| 593 | } |
| 594 | |
| 595 | // We only update if we have results for all currently attached implementations, |
| 596 | // and all currently attached testers. We already know that the implementations and |
| 597 | // testers in the maps are attached; now we check whether all attached ones are present. |
| 598 | $implementationZids = array_keys( $implementationMap ); |
| 599 | $testerZids = array_keys( reset( $implementationMap ) ); |
| 600 | if ( array_diff( $attachedImplementationZids, $implementationZids ) || |
| 601 | array_diff( $attachedTesterZids, $testerZids ) ) { |
| 602 | $logger->debug( |
| 603 | __METHOD__ . ' Not updating {functionZid}: Missing results for attached implementations or testers', |
| 604 | [ |
| 605 | 'functionZid' => $functionZid, |
| 606 | 'functionRevision' => $functionRevision, |
| 607 | 'attachedImplementationZids' => $attachedImplementationZids, |
| 608 | 'implementationZids' => $implementationZids, |
| 609 | 'attachedTesterZids' => $attachedTesterZids, |
| 610 | 'testerZids' => $testerZids, |
| 611 | 'implementationMap' => $implementationMap |
| 612 | ] |
| 613 | ); |
| 614 | return; |
| 615 | } |
| 616 | |
| 617 | // Record which implementation is first in Z8K4 before this update happens |
| 618 | $previousFirst = $attachedImplementationZids[ 0 ]; |
| 619 | |
| 620 | // For each implementation, get (count of tests-failed) and (average runtime of tests) |
| 621 | // and add them into $implementationMap. |
| 622 | // TODO (T314539): Revisit Use of (count of tests-failed) after failing implementations are |
| 623 | // routinely deactivated |
| 624 | foreach ( $implementationMap as $implementationZid => $testerMap ) { |
| 625 | $numFailed = 0; |
| 626 | $averageTime = 0.0; |
| 627 | foreach ( $testerMap as $testerId => $testResult ) { |
| 628 | if ( self::isFalse( $testResult[ 'validateStatus' ] ) ) { |
| 629 | $numFailed++; |
| 630 | } |
| 631 | $metadataMap = $testResult[ 'testMetadata' ]; |
| 632 | '@phan-var \MediaWiki\Extension\WikiLambda\ZObjects\ZTypedMap $metadataMap'; |
| 633 | $averageTime += |
| 634 | ( self::getNumericMetadataValue( $metadataMap, 'orchestrationCpuUsage' ) |
| 635 | + self::getNumericMetadataValue( $metadataMap, 'evaluationCpuUsage' ) |
| 636 | + self::getNumericMetadataValue( $metadataMap, 'executionCpuUsage' ) ); |
| 637 | } |
| 638 | $averageTime /= count( $testerMap ); |
| 639 | $implementationMap[ $implementationZid ][ 'numFailed' ] = $numFailed; |
| 640 | $implementationMap[ $implementationZid ][ 'averageTime' ] = $averageTime; |
| 641 | } |
| 642 | |
| 643 | uasort( $implementationMap, [ self::class, 'compareImplementationStats' ] ); |
| 644 | // Get the ranked Zids |
| 645 | |
| 646 | // Bail out if the new first element is the same as the previous |
| 647 | $newFirst = array_key_first( $implementationMap ); |
| 648 | if ( $newFirst === $previousFirst ) { |
| 649 | $logger->debug( |
| 650 | __METHOD__ . ' Not updating {functionZid}: Same first element', |
| 651 | [ |
| 652 | 'functionZid' => $functionZid, |
| 653 | 'functionRevision' => $functionRevision, |
| 654 | 'previousFirst' => $previousFirst, |
| 655 | 'implementationMap' => $implementationMap |
| 656 | ] |
| 657 | ); |
| 658 | return; |
| 659 | } |
| 660 | |
| 661 | // Bail out if the performance of $newFirst is only marginally better than the |
| 662 | // performance of $previousFirst. Note: if numFailed of $newFirst is less than |
| 663 | // numFailed of $previousFirst, then we should *not* bail out. |
| 664 | // TODO (T329138): Also consider: |
| 665 | // Check if all of the average times are roughly indistinguishable. |
| 666 | $previousFirstStats = $implementationMap[ $previousFirst ]; |
| 667 | $newFirstStats = $implementationMap[ $newFirst ]; |
| 668 | $relativeThreshold = 0.8; |
| 669 | if ( $newFirstStats[ 'averageTime' ] >= $relativeThreshold * $previousFirstStats[ 'averageTime' ] && |
| 670 | $newFirstStats[ 'numFailed' ] >= $previousFirstStats[ 'numFailed' ] ) { |
| 671 | $logger->debug( |
| 672 | __METHOD__ . ' Not updating {functionZid}: New first element only marginally better than previous', |
| 673 | [ |
| 674 | 'functionZid' => $functionZid, |
| 675 | 'functionRevision' => $functionRevision, |
| 676 | 'previousFirst' => $previousFirst, |
| 677 | 'newFirst' => $newFirst, |
| 678 | 'implementationMap' => $implementationMap |
| 679 | ] |
| 680 | ); |
| 681 | return; |
| 682 | } |
| 683 | |
| 684 | $implementationRankingZids = array_keys( $implementationMap ); |
| 685 | $logger->info( |
| 686 | __METHOD__ . ' Creating UpdateImplementationsJob for {functionZid}', |
| 687 | [ |
| 688 | 'functionZid' => $functionZid, |
| 689 | 'functionRevision' => $functionRevision, |
| 690 | 'implementationRankingZids' => $implementationRankingZids, |
| 691 | 'implementationMap' => $implementationMap |
| 692 | ] |
| 693 | ); |
| 694 | |
| 695 | $updateImplementationsJob = new UpdateImplementationsJob( |
| 696 | [ 'functionZid' => $functionZid, |
| 697 | 'functionRevision' => $functionRevision, |
| 698 | 'implementationRankingZids' => $implementationRankingZids |
| 699 | ] ); |
| 700 | // NOTE: As this code is static for testing purposes, we can't use $this->jobQueueGroup here |
| 701 | // TODO (T330033): Consider using an injected service for the following |
| 702 | $services = MediaWikiServices::getInstance(); |
| 703 | $jobQueueGroup = $services->getJobQueueGroup(); |
| 704 | $jobQueueGroup->push( $updateImplementationsJob ); |
| 705 | } |
| 706 | } |