Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
82.44% |
108 / 131 |
|
42.86% |
3 / 7 |
CRAP | |
0.00% |
0 / 1 |
| CheckSanity | |
82.44% |
108 / 131 |
|
42.86% |
3 / 7 |
22.16 | |
0.00% |
0 / 1 |
| execute | |
100.00% |
12 / 12 |
|
100.00% |
1 / 1 |
1 | |||
| makeChecker | |
0.00% |
0 / 14 |
|
0.00% |
0 / 1 |
2 | |||
| makeIsOldClosure | |
0.00% |
0 / 7 |
|
0.00% |
0 / 1 |
6 | |||
| check | |
100.00% |
5 / 5 |
|
100.00% |
1 / 1 |
3 | |||
| reformat | |
98.46% |
64 / 65 |
|
0.00% |
0 / 1 |
11 | |||
| getAllowedParams | |
100.00% |
27 / 27 |
|
100.00% |
1 / 1 |
1 | |||
| isInternal | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| 1 | <?php |
| 2 | |
| 3 | namespace CirrusSearch\Api; |
| 4 | |
| 5 | use CirrusSearch\Connection; |
| 6 | use CirrusSearch\Sanity\BufferedRemediator; |
| 7 | use CirrusSearch\Sanity\Checker; |
| 8 | use CirrusSearch\Sanity\CheckerException; |
| 9 | use CirrusSearch\Sanity\Remediator; |
| 10 | use CirrusSearch\SearchConfig; |
| 11 | use CirrusSearch\Searcher; |
| 12 | use CirrusSearch\Util; |
| 13 | use MediaWiki\Api\ApiBase; |
| 14 | use MediaWiki\WikiMap\WikiMap; |
| 15 | use Wikimedia\ParamValidator\ParamValidator; |
| 16 | use WikiMedia\ParamValidator\TypeDef\IntegerDef; |
| 17 | |
| 18 | /** |
| 19 | * Validates the sanity of the search indexes for a range of page id's |
| 20 | * |
| 21 | * Invokes the cirrus sanity checker which compares a range of page ids |
| 22 | * current state in the sql database against the elasticsearch indexes. |
| 23 | * Reports on issues found such as missing pages, pages that should have |
| 24 | * been deleted, and old versions in the search index. |
| 25 | * |
| 26 | * Also offers a constant rerender-over-time through the sequenceid and |
| 27 | * rerenderfrequency options. The sequenceid should be incremented each |
| 28 | * time the same set of page ids is sent to the checker. A subset of |
| 29 | * the page ids will be emit as `oldDocument` in each batch, such that |
| 30 | * after `rerenderfrequency` increments of `sequenceid` all pages will |
| 31 | * have been rerendered. The purpose of the over-time rerender is to |
| 32 | * ensure changes to how pages are rendered make it into the search indexes |
| 33 | * within an expected timeframe. |
| 34 | * |
| 35 | * @license GPL-2.0-or-later |
| 36 | */ |
| 37 | class CheckSanity extends ApiBase { |
| 38 | use ApiTrait; |
| 39 | |
| 40 | public function execute() { |
| 41 | $cluster = $this->getParameter( 'cluster' ); |
| 42 | // Start and end values are inclusive |
| 43 | $start = $this->getParameter( 'from' ); |
| 44 | $end = $start + $this->getParameter( 'limit' ) - 1; |
| 45 | |
| 46 | $remediator = new BufferedRemediator(); |
| 47 | $this->check( $this->makeChecker( $cluster, $remediator ), $start, $end ); |
| 48 | $problems = $remediator->getActions(); |
| 49 | |
| 50 | $result = $this->getResult(); |
| 51 | $result->addValue( null, 'wikiId', WikiMap::getCurrentWikiId() ); |
| 52 | $result->addValue( |
| 53 | null, 'clusterGroup', |
| 54 | $this->getSearchConfig()->getClusterAssignment()->getCrossClusterName() ); |
| 55 | $result->addValue( null, 'problems', $this->reformat( $problems ) ); |
| 56 | } |
| 57 | |
| 58 | protected function makeChecker( string $cluster, Remediator $remediator ): Checker { |
| 59 | $searchConfig = $this->getSearchConfig(); |
| 60 | $connection = Connection::getPool( $searchConfig, $cluster ); |
| 61 | $searcher = new Searcher( $connection, 0, 0, $searchConfig, [], null ); |
| 62 | |
| 63 | return new Checker( |
| 64 | $searchConfig, |
| 65 | $connection, |
| 66 | $remediator, |
| 67 | $searcher, |
| 68 | Util::getStatsFactory(), |
| 69 | false, // logSane |
| 70 | false, // fastRedirectCheck |
| 71 | null, // pageCache |
| 72 | $this->makeIsOldClosure() |
| 73 | ); |
| 74 | } |
| 75 | |
| 76 | private function makeIsOldClosure(): ?\Closure { |
| 77 | $sequenceId = $this->getParameter( 'sequenceid' ); |
| 78 | if ( $sequenceId === null ) { |
| 79 | return null; |
| 80 | } |
| 81 | return Checker::makeIsOldClosure( |
| 82 | $sequenceId, |
| 83 | $this->getParameter( 'rerenderfrequency' ) |
| 84 | ); |
| 85 | } |
| 86 | |
| 87 | private function check( Checker $checker, int $start, int $end, int $batchSize = 10 ) { |
| 88 | $ranges = array_chunk( range( $start, $end ), $batchSize ); |
| 89 | foreach ( $ranges as $pageIds ) { |
| 90 | try { |
| 91 | $checker->check( $pageIds ); |
| 92 | } catch ( CheckerException $e ) { |
| 93 | // This mostly happens when there is a transient data loading problem. |
| 94 | // The request should be retried. |
| 95 | $this->dieWithException( $e ); |
| 96 | } |
| 97 | } |
| 98 | } |
| 99 | |
| 100 | /** |
| 101 | * Reformat Saneitizer problems for output |
| 102 | * |
| 103 | * Intentionally only emits numeric ids to avoid responding with |
| 104 | * any user generated data. As a list of page ids and index states |
| 105 | * this shouldn't be capable of leaking information thats not already |
| 106 | * known. |
| 107 | */ |
| 108 | private function reformat( array $problems ): array { |
| 109 | $clean = []; |
| 110 | $indexBaseName = $this->getSearchConfig()->get( SearchConfig::INDEX_BASE_NAME ); |
| 111 | // Generic connection for resolving index names, its always the same everywhere |
| 112 | $connection = Connection::getPool( $this->getSearchConfig() ); |
| 113 | foreach ( $problems as [ $problem, $args ] ) { |
| 114 | switch ( $problem ) { |
| 115 | case 'redirectInIndex': |
| 116 | [ $docId, $page, $indexSuffix ] = $args; |
| 117 | $target = $page->getRedirectTarget(); |
| 118 | $problem = [ |
| 119 | 'indexName' => $connection->getIndexName( $indexBaseName, $indexSuffix ), |
| 120 | 'errorType' => $problem, |
| 121 | 'pageId' => $page->getId(), |
| 122 | 'namespaceId' => $page->getNamespace(), |
| 123 | ]; |
| 124 | // Page could redirect to a Special page or even another wiki, |
| 125 | // target information is only useful on pages that exist locally. |
| 126 | if ( $target != null && $target->canExist() ) { |
| 127 | $targetIndexSuffix = $connection->getIndexSuffixForNamespace( $target->getNamespace() ); |
| 128 | $problem['target'] = [ |
| 129 | 'pageId' => $target->getId(), |
| 130 | 'namespaceId' => $target->getNamespace(), |
| 131 | 'indexName' => $connection->getIndexName( $indexBaseName, $targetIndexSuffix ), |
| 132 | ]; |
| 133 | } |
| 134 | $clean[] = $problem; |
| 135 | break; |
| 136 | |
| 137 | case 'pageNotInIndex': |
| 138 | case 'oldDocument': |
| 139 | [ $page ] = $args; |
| 140 | $indexSuffix = $connection->getIndexSuffixForNamespace( $page->getNamespace() ); |
| 141 | $clean[] = [ |
| 142 | 'indexName' => $connection->getIndexName( $indexBaseName, $indexSuffix ), |
| 143 | 'errorType' => $problem, |
| 144 | 'pageId' => $page->getId(), |
| 145 | 'namespaceId' => $page->getNamespace(), |
| 146 | ]; |
| 147 | break; |
| 148 | |
| 149 | case 'ghostPageInIndex': |
| 150 | [ $docId, $title ] = $args; |
| 151 | $indexSuffix = $connection->getIndexSuffixForNamespace( $title->getNamespace() ); |
| 152 | $clean[] = [ |
| 153 | 'indexName' => $connection->getIndexName( $indexBaseName, $indexSuffix ), |
| 154 | 'errorType' => $problem, |
| 155 | 'pageId' => (int)$docId, |
| 156 | 'namespaceId' => $title->getNamespace(), |
| 157 | ]; |
| 158 | break; |
| 159 | |
| 160 | case 'pageInWrongIndex': |
| 161 | [ $docId, $page, $wrongIndexSuffix ] = $args; |
| 162 | $indexSuffix = $connection->getIndexSuffixForNamespace( $page->getNamespace() ); |
| 163 | $clean[] = [ |
| 164 | 'wrongIndexName' => $connection->getIndexName( $indexBaseName, $wrongIndexSuffix ), |
| 165 | 'indexName' => $connection->getIndexName( $indexBaseName, $indexSuffix ), |
| 166 | 'errorType' => $problem, |
| 167 | 'pageId' => $page->getId(), |
| 168 | 'namespaceId' => $page->getNamespace(), |
| 169 | ]; |
| 170 | break; |
| 171 | |
| 172 | case 'oldVersionInIndex': |
| 173 | // kinda random this one provides the suffix directly |
| 174 | [ $docId, $page, $indexSuffix ] = $args; |
| 175 | $clean[] = [ |
| 176 | 'indexName' => $connection->getIndexName( $indexBaseName, $indexSuffix ), |
| 177 | 'errorType' => $problem, |
| 178 | 'pageId' => $page->getId(), |
| 179 | 'namespaceId' => $page->getNamespace(), |
| 180 | ]; |
| 181 | break; |
| 182 | |
| 183 | default: |
| 184 | $this->dieDebug( __METHOD__, "Unknown remediation: $problem" ); |
| 185 | } |
| 186 | } |
| 187 | |
| 188 | return $clean; |
| 189 | } |
| 190 | |
| 191 | /** @inheritDoc */ |
| 192 | public function getAllowedParams() { |
| 193 | $assignment = $this->getSearchConfig()->getClusterAssignment(); |
| 194 | return [ |
| 195 | 'cluster' => [ |
| 196 | ParamValidator::PARAM_REQUIRED => true, |
| 197 | ParamValidator::PARAM_TYPE => $assignment->getManagedClusters(), |
| 198 | ], |
| 199 | 'from' => [ |
| 200 | ParamValidator::PARAM_TYPE => 'integer', |
| 201 | ParamValidator::PARAM_REQUIRED => true, |
| 202 | IntegerDef::PARAM_MIN => 0, |
| 203 | ], |
| 204 | 'limit' => [ |
| 205 | ParamValidator::PARAM_DEFAULT => 100, |
| 206 | ParamValidator::PARAM_TYPE => 'limit', |
| 207 | IntegerDef::PARAM_MIN => 1, |
| 208 | IntegerDef::PARAM_MAX => ApiBase::LIMIT_BIG1, |
| 209 | IntegerDef::PARAM_MAX2 => ApiBase::LIMIT_BIG2 |
| 210 | ], |
| 211 | // The caller must increment the sequenceid each successive |
| 212 | // time it invokes the sanity check for the same set of pages. |
| 213 | // Pages within the batch will emit an `oldDocument` problem |
| 214 | // spread over `rerenderfrequency` invocations of the api. |
| 215 | // This supports a slow and constant rerender of all content, |
| 216 | // ensuring the search indices stay aligned with changes to |
| 217 | // indexing and rendering code. |
| 218 | 'sequenceid' => [ |
| 219 | // Providing this enables the "old document" checks |
| 220 | // which provide constant re-rendering over time. |
| 221 | ParamValidator::PARAM_TYPE => 'integer', |
| 222 | ], |
| 223 | // Controls how often a page is flagged with the `oldDocument` |
| 224 | // problem. If the caller scans all page ids every week, then |
| 225 | // the default value of 16 would emit an `oldDocument` problem |
| 226 | // for all existing pages spread over 16 weeks. |
| 227 | 'rerenderfrequency' => [ |
| 228 | ParamValidator::PARAM_DEFAULT => 16, |
| 229 | ParamValidator::PARAM_TYPE => 'integer', |
| 230 | IntegerDef::PARAM_MIN => 2, |
| 231 | ] |
| 232 | ]; |
| 233 | } |
| 234 | |
| 235 | /** |
| 236 | * Mark as internal. This isn't meant to be used by normal api users |
| 237 | * @return bool |
| 238 | */ |
| 239 | public function isInternal() { |
| 240 | return true; |
| 241 | } |
| 242 | |
| 243 | } |