Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
93.94% |
124 / 132 |
|
62.50% |
5 / 8 |
CRAP | |
0.00% |
0 / 1 |
| EntityFullTextQueryBuilder | |
93.94% |
124 / 132 |
|
62.50% |
5 / 8 |
24.13 | |
0.00% |
0 / 1 |
| __construct | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| newFromGlobals | |
100.00% |
9 / 9 |
|
100.00% |
1 / 1 |
1 | |||
| build | |
90.00% |
9 / 10 |
|
0.00% |
0 / 1 |
3.01 | |||
| buildDegraded | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| buildEntitySearchQuery | |
100.00% |
64 / 64 |
|
100.00% |
1 / 1 |
9 | |||
| buildSimpleAllFilter | |
100.00% |
10 / 10 |
|
100.00% |
1 / 1 |
3 | |||
| buildFieldMatch | |
100.00% |
5 / 5 |
|
100.00% |
1 / 1 |
2 | |||
| buildPhraseRescore | |
81.25% |
26 / 32 |
|
0.00% |
0 / 1 |
4.11 | |||
| 1 | <?php |
| 2 | |
| 3 | namespace Wikibase\Search\Elastic; |
| 4 | |
| 5 | use CirrusSearch\Extra\Query\TokenCountRouter; |
| 6 | use CirrusSearch\Parser\FullTextKeywordRegistry; |
| 7 | use CirrusSearch\Query\FullTextQueryBuilder; |
| 8 | use CirrusSearch\Search\SearchContext; |
| 9 | use Elastica\Query\AbstractQuery; |
| 10 | use Elastica\Query\BoolQuery; |
| 11 | use Elastica\Query\DisMax; |
| 12 | use Elastica\Query\MatchNone; |
| 13 | use Elastica\Query\MatchQuery; |
| 14 | use Elastica\Query\MultiMatch; |
| 15 | use Elastica\Query\Term; |
| 16 | use MediaWiki\Context\RequestContext; |
| 17 | use MediaWiki\MediaWikiServices; |
| 18 | use Wikibase\DataModel\Entity\EntityIdParser; |
| 19 | use Wikibase\Lib\LanguageFallbackChainFactory; |
| 20 | use Wikibase\Repo\WikibaseRepo; |
| 21 | |
| 22 | /** |
| 23 | * Builder for entity fulltext queries |
| 24 | */ |
| 25 | class EntityFullTextQueryBuilder implements FullTextQueryBuilder { |
| 26 | public const ENTITY_FULL_TEXT_MARKER = 'entity_full_text'; |
| 27 | |
| 28 | /** |
| 29 | * @param array $stemmingSettings Stemming settings from UseStemming config entry |
| 30 | * @param array $settings Settings from EntitySearchProfiles.php |
| 31 | * @param LanguageFallbackChainFactory $languageFallbackChainFactory |
| 32 | * @param EntityIdParser $entityIdParser |
| 33 | * @param string $userLanguage User's language code |
| 34 | */ |
| 35 | public function __construct( |
| 36 | private readonly array $stemmingSettings, |
| 37 | private readonly array $settings, |
| 38 | private readonly LanguageFallbackChainFactory $languageFallbackChainFactory, |
| 39 | private readonly EntityIdParser $entityIdParser, |
| 40 | private readonly string $userLanguage, |
| 41 | ) { |
| 42 | } |
| 43 | |
| 44 | /** |
| 45 | * Create fulltext builder from global environment. |
| 46 | * @param array $settings Configuration from config file |
| 47 | * @return EntityFullTextQueryBuilder |
| 48 | */ |
| 49 | public static function newFromGlobals( array $settings ) { |
| 50 | $services = MediaWikiServices::getInstance(); |
| 51 | $config = $services->getConfigFactory()->makeConfig( 'WikibaseCirrusSearch' ); |
| 52 | return new static( |
| 53 | $config->get( 'UseStemming' ), |
| 54 | $settings, |
| 55 | WikibaseRepo::getLanguageFallbackChainFactory( $services ), |
| 56 | WikibaseRepo::getEntityIdParser( $services ), |
| 57 | RequestContext::getMain()->getLanguage()->getCode() |
| 58 | ); |
| 59 | } |
| 60 | |
| 61 | /** |
| 62 | * Search articles with provided term. |
| 63 | * |
| 64 | * @param SearchContext $searchContext |
| 65 | * @param string $term term to search |
| 66 | */ |
| 67 | public function build( SearchContext $searchContext, $term ) { |
| 68 | // Strip keyword features from the search term (T425253). |
| 69 | // Keyword features (e.g. haswbstatement:P31=Q5) add their filters |
| 70 | // directly to SearchContext and must be removed from $term so the |
| 71 | // entity search query only receives the plain text portion. |
| 72 | $features = ( new FullTextKeywordRegistry( $searchContext->getConfig() ) )->getKeywords(); |
| 73 | foreach ( $features as $feature ) { |
| 74 | $term = $feature->apply( $searchContext, $term ); |
| 75 | } |
| 76 | $term = trim( $term ); |
| 77 | |
| 78 | $searchContext->addSyntaxUsed( self::ENTITY_FULL_TEXT_MARKER, 10 ); |
| 79 | |
| 80 | if ( $term === '' ) { |
| 81 | return; |
| 82 | } |
| 83 | |
| 84 | $this->buildEntitySearchQuery( $searchContext, $term ); |
| 85 | |
| 86 | // FIXME: make it dispatch by content model |
| 87 | $searchContext->setResultsType( new EntityResultType( $this->userLanguage, |
| 88 | $this->languageFallbackChainFactory->newFromLanguageCode( $this->userLanguage ) ) ); |
| 89 | } |
| 90 | |
| 91 | /** |
| 92 | * @param SearchContext $searchContext |
| 93 | * @return bool |
| 94 | */ |
| 95 | public function buildDegraded( SearchContext $searchContext ) { |
| 96 | // Not doing anything for now |
| 97 | return false; |
| 98 | } |
| 99 | |
| 100 | /** |
| 101 | * Build a fulltext query for Wikibase entity. |
| 102 | * @param SearchContext $searchContext |
| 103 | * @param string $term Search term |
| 104 | */ |
| 105 | protected function buildEntitySearchQuery( SearchContext $searchContext, $term ) { |
| 106 | /* |
| 107 | * Overall query structure is as follows: |
| 108 | * - Bool with: |
| 109 | * Filter of namespace = N |
| 110 | * OR (Should with 1 mininmum) of: |
| 111 | * title.keyword = QUERY |
| 112 | * fulltext match query |
| 113 | * |
| 114 | * Fulltext match query is: |
| 115 | * Filter of: |
| 116 | * at least one of: all, all.plain matching |
| 117 | * description (for stemmed) or description.en (for non-stemmed) matching, with fallback |
| 118 | * OR (should with 0 minimum) of: |
| 119 | * DISMAX query of: all labels.near_match in fallback chain |
| 120 | * OR (should with 0 minimum) of: |
| 121 | * all |
| 122 | * all.plain |
| 123 | * DISMAX of: all fulltext matches for tokenized fields |
| 124 | */ |
| 125 | |
| 126 | $profile = $this->settings; |
| 127 | // $fields is collecting all the fields for dismax query to be used in |
| 128 | // scoring match |
| 129 | $fields = [ |
| 130 | [ "labels.{$this->userLanguage}.near_match", $profile['lang-exact'] ], |
| 131 | [ "labels.{$this->userLanguage}.near_match_folded", $profile['lang-folded'] ], |
| 132 | ]; |
| 133 | |
| 134 | $fieldsTokenized = [ |
| 135 | [ "labels.{$this->userLanguage}.plain", $profile['lang-partial'] ], |
| 136 | [ "descriptions.{$this->userLanguage}.plain", $profile['lang-partial'] ], |
| 137 | ]; |
| 138 | if ( !empty( $this->stemmingSettings[$this->userLanguage]['query'] ) ) { |
| 139 | $fieldsTokenized[] = [ "labels.{$this->userLanguage}", $profile['lang-partial'] ]; |
| 140 | $fieldsTokenized[] = [ "descriptions.{$this->userLanguage}", $profile['lang-partial'] ]; |
| 141 | } |
| 142 | |
| 143 | $searchLanguageCodes = $this->languageFallbackChainFactory->newFromLanguageCode( $this->userLanguage ) |
| 144 | ->getFetchLanguageCodes(); |
| 145 | |
| 146 | $discount = $profile['fallback-discount']; |
| 147 | $stemFilterFields = []; |
| 148 | |
| 149 | foreach ( $searchLanguageCodes as $fallbackCode ) { |
| 150 | if ( empty( $this->stemmingSettings[$fallbackCode]['query'] ) ) { |
| 151 | $stemFilterFields[] = "descriptions.{$fallbackCode}.plain"; |
| 152 | } else { |
| 153 | $stemFilterFields[] = "descriptions.{$fallbackCode}"; |
| 154 | // only add the stemmed version in the filter |
| 155 | // labels should be copied to the text field and thus be captured by the filter on the all field |
| 156 | $stemFilterFields[] = "labels.{$fallbackCode}"; |
| 157 | } |
| 158 | |
| 159 | if ( $fallbackCode === $this->userLanguage ) { |
| 160 | continue; |
| 161 | } |
| 162 | |
| 163 | $weight = $profile['fallback-exact'] * $discount; |
| 164 | $fields[] = [ "labels.{$fallbackCode}.near_match", $weight ]; |
| 165 | |
| 166 | $weight = $profile['fallback-folded'] * $discount; |
| 167 | $fields[] = [ "labels.{$fallbackCode}.near_match_folded", $weight ]; |
| 168 | |
| 169 | $weight = $profile['fallback-partial'] * $discount; |
| 170 | $fieldsTokenized[] = [ "labels.{$fallbackCode}.plain", $weight ]; |
| 171 | $fieldsTokenized[] = [ "descriptions.{$fallbackCode}.plain", $weight ]; |
| 172 | if ( !empty( $this->stemmingSettings[$fallbackCode]['query'] ) ) { |
| 173 | $fieldsTokenized[] = [ "descriptions.{$fallbackCode}", $weight ]; |
| 174 | $fieldsTokenized[] = [ "labels.{$fallbackCode}", $weight ]; |
| 175 | } |
| 176 | |
| 177 | $discount *= $profile['fallback-discount']; |
| 178 | } |
| 179 | |
| 180 | $titleMatch = new Term( [ |
| 181 | 'title.keyword' => EntitySearchUtils::normalizeId( $term, $this->entityIdParser ), |
| 182 | ] ); |
| 183 | |
| 184 | // Main query filter |
| 185 | $filterQuery = $this->buildSimpleAllFilter( $term ); |
| 186 | foreach ( $stemFilterFields as $filterField ) { |
| 187 | $filterQuery->addShould( $this->buildFieldMatch( $filterField, $term, 'AND' ) ); |
| 188 | } |
| 189 | |
| 190 | // Near match ones, they use constant score |
| 191 | $nearMatchQuery = new DisMax(); |
| 192 | $nearMatchQuery->setTieBreaker( 0 ); |
| 193 | foreach ( $fields as $field ) { |
| 194 | $nearMatchQuery->addQuery( EntitySearchUtils::makeConstScoreQuery( $field[0], $field[1], |
| 195 | $term ) ); |
| 196 | } |
| 197 | |
| 198 | // Tokenized ones |
| 199 | $tokenizedQuery = $this->buildSimpleAllFilter( $term, 'OR', $profile['any'] ); |
| 200 | $tokenizedQueryFields = new DisMax(); |
| 201 | $tokenizedQueryFields->setTieBreaker( 0.2 ); |
| 202 | foreach ( $fieldsTokenized as $field ) { |
| 203 | $m = $this->buildFieldMatch( $field[0], $term ); |
| 204 | $m->setFieldBoost( $field[0], $field[1] ); |
| 205 | $tokenizedQueryFields->addQuery( $m ); |
| 206 | } |
| 207 | $tokenizedQuery->addShould( $tokenizedQueryFields ); |
| 208 | |
| 209 | // Main labels/desc query |
| 210 | $labelsDescQuery = new BoolQuery(); |
| 211 | $labelsDescQuery->setMinimumShouldMatch( 0 ); |
| 212 | $labelsDescQuery->addFilter( $filterQuery ); |
| 213 | $labelsDescQuery->addShould( $nearMatchQuery ); |
| 214 | $labelsDescQuery->addShould( $tokenizedQuery ); |
| 215 | |
| 216 | // Main query |
| 217 | $query = new BoolQuery(); |
| 218 | |
| 219 | // Match either labels or exact match to title |
| 220 | $query->addShould( $titleMatch ); |
| 221 | $query->addShould( $labelsDescQuery ); |
| 222 | $query->setMinimumShouldMatch( 1 ); |
| 223 | |
| 224 | $searchContext->setMainQuery( $query ); |
| 225 | $searchContext->setPhraseRescoreQuery( $this->buildPhraseRescore( $term, $searchContext, $profile ) ); |
| 226 | } |
| 227 | |
| 228 | /** |
| 229 | * Builds a simple filter on all and all.plain when all terms must match |
| 230 | * |
| 231 | * @param string $query |
| 232 | * @param string $operator |
| 233 | * @param null $boost |
| 234 | * @return BoolQuery |
| 235 | */ |
| 236 | private function buildSimpleAllFilter( $query, $operator = 'AND', $boost = null ) { |
| 237 | $filter = new BoolQuery(); |
| 238 | $filter->setMinimumShouldMatch( 1 ); |
| 239 | // FIXME: We can't use solely the stem field here |
| 240 | // - Depending on languages it may lack stopwords, |
| 241 | // A dedicated field used for filtering would be nice |
| 242 | foreach ( [ 'all', 'all.plain' ] as $field ) { |
| 243 | $m = new MatchQuery(); |
| 244 | $m->setFieldQuery( $field, $query ); |
| 245 | $m->setFieldOperator( $field, $operator ); |
| 246 | if ( $boost ) { |
| 247 | $m->setFieldBoost( $field, $boost ); |
| 248 | } |
| 249 | $filter->addShould( $m ); |
| 250 | } |
| 251 | return $filter; |
| 252 | } |
| 253 | |
| 254 | /** |
| 255 | * Build simple match clause, matching field against term |
| 256 | * @param string $field |
| 257 | * @param string $term |
| 258 | * @param string|null $operator |
| 259 | * @return MatchQuery |
| 260 | */ |
| 261 | private function buildFieldMatch( $field, $term, $operator = null ) { |
| 262 | $m = new MatchQuery(); |
| 263 | $m->setFieldQuery( $field, $term ); |
| 264 | if ( $operator ) { |
| 265 | $m->setFieldOperator( $field, $operator ); |
| 266 | } |
| 267 | return $m; |
| 268 | } |
| 269 | |
| 270 | /** |
| 271 | * Create phrase rescore query for "all" fields |
| 272 | * @param string $queryText |
| 273 | * @param SearchContext $context |
| 274 | * @param float[][] $profile Must contain $profile['phrase'] with keys 'all', 'slop', 'all.plain' |
| 275 | * @return AbstractQuery|null |
| 276 | */ |
| 277 | private function buildPhraseRescore( $queryText, SearchContext $context, array $profile ) { |
| 278 | if ( empty( $profile['phrase'] ) ) { |
| 279 | return null; |
| 280 | } else { |
| 281 | $phraseProfile = $profile['phrase']; |
| 282 | } |
| 283 | $useRouter = $context->getConfig()->getElement( 'CirrusSearchWikimediaExtraPlugin', 'token_count_router' ) === true; |
| 284 | $phrase = new MultiMatch(); |
| 285 | $phrase->setParam( 'type', 'phrase' ); |
| 286 | $phrase->setParam( 'slop', $phraseProfile['slop'] ); |
| 287 | $fields = [ |
| 288 | "all^{$phraseProfile['all']}", "all.plain^{$phraseProfile['all.plain']}" |
| 289 | ]; |
| 290 | $phrase->setFields( $fields ); |
| 291 | $phrase->setQuery( $queryText ); |
| 292 | if ( !$useRouter ) { |
| 293 | return $phrase; |
| 294 | } |
| 295 | $tokCount = new TokenCountRouter( |
| 296 | // text |
| 297 | $queryText, |
| 298 | // fallback |
| 299 | new MatchNone(), |
| 300 | // field |
| 301 | "text" |
| 302 | ); |
| 303 | $tokCount->addCondition( |
| 304 | TokenCountRouter::GT, |
| 305 | 1, |
| 306 | $phrase |
| 307 | ); |
| 308 | $maxTokens = $context->getConfig()->get( 'CirrusSearchMaxPhraseTokens' ); |
| 309 | if ( $maxTokens ) { |
| 310 | $tokCount->addCondition( |
| 311 | TokenCountRouter::GT, |
| 312 | $maxTokens, |
| 313 | new \Elastica\Query\MatchNone() |
| 314 | ); |
| 315 | } |
| 316 | return $tokCount; |
| 317 | } |
| 318 | |
| 319 | } |