Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
94.55% |
104 / 110 |
|
76.92% |
10 / 13 |
CRAP | |
0.00% |
0 / 1 |
| CompSuggestQueryBuilder | |
94.55% |
104 / 110 |
|
76.92% |
10 / 13 |
45.33 | |
0.00% |
0 / 1 |
| __construct | |
100.00% |
9 / 9 |
|
100.00% |
1 / 1 |
3 | |||
| areResultsPossible | |
75.00% |
3 / 4 |
|
0.00% |
0 / 1 |
3.14 | |||
| build | |
100.00% |
10 / 10 |
|
100.00% |
1 / 1 |
3 | |||
| buildSuggestQueries | |
100.00% |
7 / 7 |
|
100.00% |
1 / 1 |
3 | |||
| resolveFuzzy | |
88.89% |
8 / 9 |
|
0.00% |
0 / 1 |
4.02 | |||
| buildSuggestQuery | |
100.00% |
12 / 12 |
|
100.00% |
1 / 1 |
5 | |||
| handleVariants | |
100.00% |
19 / 19 |
|
100.00% |
1 / 1 |
6 | |||
| buildVariantProfile | |
100.00% |
3 / 3 |
|
100.00% |
1 / 1 |
1 | |||
| postProcess | |
86.21% |
25 / 29 |
|
0.00% |
0 / 1 |
12.38 | |||
| decodeId | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| getMergedProfiles | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| computeHardLimit | |
100.00% |
5 / 5 |
|
100.00% |
1 / 1 |
2 | |||
| getLimit | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| 1 | <?php |
| 2 | |
| 3 | namespace CirrusSearch\Query; |
| 4 | |
| 5 | use CirrusSearch\BuildDocument\Completion\SuggestBuilder; |
| 6 | use CirrusSearch\CirrusConfigNames; |
| 7 | use CirrusSearch\Search\CompletionResultsCollector; |
| 8 | use CirrusSearch\Search\SearchContext; |
| 9 | use CirrusSearch\SearchConfig; |
| 10 | use CirrusSearch\SecondTry\SecondTryRunner; |
| 11 | use Elastica\ResultSet; |
| 12 | use Elastica\Suggest; |
| 13 | use Elastica\Suggest\Completion; |
| 14 | use MediaWiki\Search\SearchSuggestion; |
| 15 | use Wikimedia\Assert\Assert; |
| 16 | |
| 17 | /** |
| 18 | * Suggest (Completion) query builder. |
| 19 | * Unlike classic query builders it will : |
| 20 | * - handle limit differently as offsets are not accepted during suggest queries |
| 21 | * - store a mutable state in mergedProfiles |
| 22 | */ |
| 23 | class CompSuggestQueryBuilder { |
| 24 | use QueryBuilderTraits; |
| 25 | |
| 26 | public const VARIANT_EXTRA_DISCOUNT = 0.0001; |
| 27 | |
| 28 | /** @var SearchContext (final) */ |
| 29 | private $searchContext; |
| 30 | |
| 31 | /** @var array (final) */ |
| 32 | private $profile; |
| 33 | |
| 34 | /** @var int (final) */ |
| 35 | private $limit; |
| 36 | |
| 37 | /** @var int (final) */ |
| 38 | private $hardLimit; |
| 39 | |
| 40 | /** @var int (final) */ |
| 41 | private $offset; |
| 42 | |
| 43 | /** @var array (mutable) state built after calling self::build */ |
| 44 | private $mergedProfiles; |
| 45 | private SecondTryRunner $secondTryRunner; |
| 46 | |
| 47 | /** |
| 48 | * @param SearchContext $context |
| 49 | * @param array $profile settings as definied in profiles/SuggestProfiles.config.php |
| 50 | * @param SecondTryRunner $secondTryRunner |
| 51 | * @param int $limit the number of results to display |
| 52 | * @param int $offset |
| 53 | */ |
| 54 | public function __construct( SearchContext $context, array $profile, SecondTryRunner $secondTryRunner, $limit, $offset = 0 ) { |
| 55 | $this->searchContext = $context; |
| 56 | $this->profile = $profile['fst']; |
| 57 | Assert::parameter( count( $this->profile ) > 0, '$profile', 'Profile must not be empty' ); |
| 58 | $this->secondTryRunner = $secondTryRunner; |
| 59 | $this->hardLimit = self::computeHardLimit( $limit, $offset, $context->getConfig() ); |
| 60 | if ( $limit > $this->hardLimit - $offset ) { |
| 61 | $limit = $this->hardLimit - $offset; |
| 62 | } |
| 63 | $this->limit = $limit > 0 ? $limit : 0; |
| 64 | $this->offset = $offset; |
| 65 | } |
| 66 | |
| 67 | /** |
| 68 | * Check the builder settings to determine if results are possible. |
| 69 | * If this method returns false the query must not have to be sent to elastic |
| 70 | * |
| 71 | * @return bool true if results are possible false otherwise |
| 72 | */ |
| 73 | public function areResultsPossible() { |
| 74 | $namespaces = $this->searchContext->getNamespaces(); |
| 75 | if ( $namespaces !== null && !in_array( NS_MAIN, $namespaces ) ) { |
| 76 | return false; |
| 77 | } |
| 78 | // If the offset requested is greater than the hard limit |
| 79 | // allowed we will always return an empty set so let's do it |
| 80 | // asap. |
| 81 | return $this->limit > 0; |
| 82 | } |
| 83 | |
| 84 | /** |
| 85 | * Build the suggest query |
| 86 | * @param string $term |
| 87 | * @param array<string, string[]> $secondTryCandidates |
| 88 | * @return Suggest |
| 89 | */ |
| 90 | public function build( string $term, array $secondTryCandidates = [] ): Suggest { |
| 91 | $this->checkTitleSearchRequestLength( $term, $this->searchContext ); |
| 92 | $origTerm = $term; |
| 93 | if ( mb_strlen( $term ) > SuggestBuilder::MAX_INPUT_LENGTH ) { |
| 94 | // Trim the query otherwise we won't find results |
| 95 | $term = mb_substr( $term, 0, SuggestBuilder::MAX_INPUT_LENGTH ); |
| 96 | } |
| 97 | |
| 98 | $queryLen = mb_strlen( trim( $term ) ); // Avoid cheating with spaces |
| 99 | |
| 100 | $this->mergedProfiles = $this->profile; |
| 101 | $suggest = $this->buildSuggestQueries( $this->profile, $term, $queryLen ); |
| 102 | |
| 103 | // Handle variants, update the set of profiles and suggest queries |
| 104 | if ( $secondTryCandidates ) { |
| 105 | $this->handleVariants( $suggest, $secondTryCandidates, $queryLen, $origTerm ); |
| 106 | } |
| 107 | return $suggest; |
| 108 | } |
| 109 | |
| 110 | /** |
| 111 | * Builds a set of suggest query by reading the list of profiles |
| 112 | * @param array $profiles |
| 113 | * @param string $query |
| 114 | * @param int $queryLen the length to use when checking min/max_query_len |
| 115 | * @return Suggest a set of suggest queries ready to for elastic |
| 116 | */ |
| 117 | private function buildSuggestQueries( array $profiles, $query, $queryLen ) { |
| 118 | $suggest = new Suggest(); |
| 119 | foreach ( $profiles as $name => $config ) { |
| 120 | $sugg = $this->buildSuggestQuery( $name, $config, $query, $queryLen ); |
| 121 | if ( $sugg === null ) { |
| 122 | continue; |
| 123 | } |
| 124 | $suggest->addSuggestion( $sugg ); |
| 125 | } |
| 126 | return $suggest; |
| 127 | } |
| 128 | |
| 129 | /** |
| 130 | * Resolves AUTO fuzziness into a constant value |
| 131 | * @param array $fuzzy FST Fuzziness configuration |
| 132 | * @param int $queryLen The number of codepoints in the query |
| 133 | * @return array Resolve FST Fuzziness configuration |
| 134 | */ |
| 135 | private function resolveFuzzy( array $fuzzy, $queryLen ): array { |
| 136 | // TODO: We could support `AUTO:2,8` syntax as well, but didnt seem necessary |
| 137 | if ( ( $fuzzy['fuzziness'] ?? null ) === 'AUTO' ) { |
| 138 | $low = 3; |
| 139 | $high = 6; |
| 140 | if ( $queryLen < $low ) { |
| 141 | $fuzzy['fuzziness'] = 0; |
| 142 | } elseif ( $queryLen < $high ) { |
| 143 | $fuzzy['fuzziness'] = 1; |
| 144 | } else { |
| 145 | $fuzzy['fuzziness'] = 2; |
| 146 | } |
| 147 | } |
| 148 | return $fuzzy; |
| 149 | } |
| 150 | |
| 151 | /** |
| 152 | * Builds a suggest query from a profile |
| 153 | * @param string $name name of the suggestion |
| 154 | * @param array $config Profile |
| 155 | * @param string $query |
| 156 | * @param int $queryLen the length to use when checking min/max_query_len |
| 157 | * @return Completion|null suggest query ready to for elastic or null |
| 158 | */ |
| 159 | private function buildSuggestQuery( $name, array $config, $query, $queryLen ) { |
| 160 | // Do not remove spaces at the end, the user might tell us he finished writing a word |
| 161 | $query = ltrim( $query ); |
| 162 | if ( $config['min_query_len'] > $queryLen ) { |
| 163 | return null; |
| 164 | } |
| 165 | if ( isset( $config['max_query_len'] ) && $queryLen > $config['max_query_len'] ) { |
| 166 | return null; |
| 167 | } |
| 168 | $field = $config['field']; |
| 169 | $sug = new Completion( $name, $field ); |
| 170 | $sug->setPrefix( $query ); |
| 171 | $sug->setSize( $this->hardLimit * $config['fetch_limit_factor'] ); |
| 172 | if ( isset( $config['fuzzy'] ) ) { |
| 173 | $sug->setFuzzy( $this->resolveFuzzy( $config['fuzzy'], $queryLen ) ); |
| 174 | } |
| 175 | return $sug; |
| 176 | } |
| 177 | |
| 178 | /** |
| 179 | * Update the suggest queries and return additional profiles flagged the 'fallback' key |
| 180 | * with a discount factor = originalDiscount * 0.0001/(variantIndex+1). |
| 181 | * @param Suggest $suggests |
| 182 | * @param array<string, string[]> $secondTryCandidates candidates as returned by {@link SecondTryRunner::candidate} |
| 183 | * @param int $queryLen the original query length |
| 184 | * @param string $term original term (used to dedup) |
| 185 | * @internal param array $profiles the default profiles |
| 186 | */ |
| 187 | private function handleVariants( Suggest $suggests, array $secondTryCandidates, int $queryLen, string $term ): void { |
| 188 | $done = [ $term ]; |
| 189 | $variantIndex = 0; |
| 190 | foreach ( $secondTryCandidates as $strategy => $candidates ) { |
| 191 | foreach ( $candidates as $candidate ) { |
| 192 | if ( in_array( $candidate, $done, true ) ) { |
| 193 | continue; |
| 194 | } |
| 195 | $done[] = $candidate; |
| 196 | $variantIndex++; |
| 197 | foreach ( $this->profile as $name => $profile ) { |
| 198 | $variantProfName = $name . '-second-try-' . $strategy . '-' . $variantIndex; |
| 199 | $profile = $this->buildVariantProfile( |
| 200 | $profile, ( self::VARIANT_EXTRA_DISCOUNT * $this->secondTryRunner->weight( $strategy ) ) / $variantIndex |
| 201 | ); |
| 202 | $suggest = $this->buildSuggestQuery( |
| 203 | $variantProfName, $profile, $candidate, $queryLen |
| 204 | ); |
| 205 | if ( $suggest !== null ) { |
| 206 | $suggests->addSuggestion( $suggest ); |
| 207 | $this->mergedProfiles[$variantProfName] = $profile; |
| 208 | } |
| 209 | } |
| 210 | |
| 211 | } |
| 212 | } |
| 213 | } |
| 214 | |
| 215 | /** |
| 216 | * Creates a copy of $profile[$name] with a custom '-variant-SEQ' suffix. |
| 217 | * And applies an extra discount factor of 0.0001. |
| 218 | * The copy is added to the profiles container. |
| 219 | * @param array $profile profile to copy |
| 220 | * @param float $extraDiscount extra discount factor to rank variant suggestion lower. |
| 221 | * @return array |
| 222 | */ |
| 223 | protected function buildVariantProfile( array $profile, $extraDiscount = 0.0001 ) { |
| 224 | // mark the profile as a fallback query |
| 225 | $profile['fallback'] = true; |
| 226 | $profile['discount'] *= $extraDiscount; |
| 227 | return $profile; |
| 228 | } |
| 229 | |
| 230 | /** |
| 231 | * Post process the response from elastic to build the SearchSuggestionSet. |
| 232 | * |
| 233 | * Merge top level multi-queries and resolve returned pageIds into Title objects. |
| 234 | * |
| 235 | * @param CompletionResultsCollector $collector |
| 236 | * @param ResultSet $results |
| 237 | * @param string $indexName |
| 238 | * @return int total hits |
| 239 | */ |
| 240 | public function postProcess( CompletionResultsCollector $collector, ResultSet $results, $indexName ) { |
| 241 | $suggestResp = $results->getSuggests(); |
| 242 | if ( $suggestResp === [] ) { |
| 243 | // Edge case where the index contains 0 documents and does not even return the 'suggest' field |
| 244 | return 0; |
| 245 | } |
| 246 | $hitsTotal = 0; |
| 247 | foreach ( $suggestResp as $name => $sug ) { |
| 248 | $discount = $this->mergedProfiles[$name]['discount']; |
| 249 | foreach ( $sug as $suggested ) { |
| 250 | $hitsTotal += count( $suggested['options'] ); |
| 251 | foreach ( $suggested['options'] as $suggest ) { |
| 252 | $page = $suggest['text']; |
| 253 | if ( !isset( $suggest['_id'] ) ) { |
| 254 | // likely a shard failure during the fetch phase |
| 255 | // https://github.com/elastic/elasticsearch/issues/32467 |
| 256 | throw new \Elastica\Exception\RuntimeException( "Invalid response returned from " . |
| 257 | "the backend (probable shard failure during the fetch phase)" ); |
| 258 | } |
| 259 | $targetTitle = $page; |
| 260 | $targetTitleNS = NS_MAIN; |
| 261 | if ( isset( $suggest['_source']['target_title'] ) ) { |
| 262 | $targetTitle = $suggest['_source']['target_title']['title']; |
| 263 | $targetTitleNS = $suggest['_source']['target_title']['namespace']; |
| 264 | } |
| 265 | [ $docId, $type ] = $this->decodeId( $suggest['_id'] ); |
| 266 | $score = $discount * $suggest['_score']; |
| 267 | $pageId = $this->searchContext->getConfig()->makePageId( $docId ); |
| 268 | $suggestion = new SearchSuggestion( $score, null, null, $pageId ); |
| 269 | if ( $collector->collect( $suggestion, $name, $indexName ) ) { |
| 270 | if ( $type === SuggestBuilder::TITLE_SUGGESTION && $targetTitleNS === NS_MAIN ) { |
| 271 | // For title suggestions we always use the target_title |
| 272 | // This is because we may encounter default_sort or subphrases that are not |
| 273 | // valid titles... And we prefer to display the title over close redirects |
| 274 | // for CrossNS redirect we prefer the returned suggestion |
| 275 | $suggestion->setText( $targetTitle ); |
| 276 | |
| 277 | } else { |
| 278 | $suggestion->setText( $page ); |
| 279 | } |
| 280 | } else { |
| 281 | // Results are returned in order by elastic skip the rest if no more |
| 282 | // results from this suggest can be collected |
| 283 | if ( $collector->isFull() && $collector->getMinScore() > $score ) { |
| 284 | break; |
| 285 | } |
| 286 | } |
| 287 | } |
| 288 | } |
| 289 | } |
| 290 | return $hitsTotal; |
| 291 | } |
| 292 | |
| 293 | /** |
| 294 | * @param string $id compacted id (id + $type) |
| 295 | * @return array 2 elt array [ $id, $type ] |
| 296 | */ |
| 297 | private function decodeId( $id ) { |
| 298 | return [ intval( substr( $id, 0, -1 ) ), substr( $id, -1 ) ]; |
| 299 | } |
| 300 | |
| 301 | /** |
| 302 | * (public for tests) |
| 303 | * @return array |
| 304 | */ |
| 305 | public function getMergedProfiles() { |
| 306 | return $this->mergedProfiles; |
| 307 | } |
| 308 | |
| 309 | /** |
| 310 | * Get the hard limit |
| 311 | * The completion api does not supports offset we have to add a hack |
| 312 | * here to work around this limitation. |
| 313 | * To avoid ridiculously large queries we set also a hard limit. |
| 314 | * Note that this limit will be changed by fetch_limit_factor set to 2 or 1.5 |
| 315 | * depending on the profile. |
| 316 | * @param int $limit limit requested |
| 317 | * @param int $offset offset requested |
| 318 | * @param SearchConfig $config |
| 319 | * @return int the number of results to fetch from elastic |
| 320 | */ |
| 321 | public static function computeHardLimit( $limit, $offset, SearchConfig $config ) { |
| 322 | $limit += $offset; |
| 323 | $hardLimit = $config->get( CirrusConfigNames::CompletionSuggesterHardLimit ) ?? 50; |
| 324 | if ( $limit > $hardLimit ) { |
| 325 | return $hardLimit; |
| 326 | } |
| 327 | return $limit; |
| 328 | } |
| 329 | |
| 330 | /** |
| 331 | * Number of results we could display |
| 332 | * @return int |
| 333 | */ |
| 334 | public function getLimit() { |
| 335 | return $this->limit; |
| 336 | } |
| 337 | } |