Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
80.95% |
170 / 210 |
|
40.00% |
2 / 5 |
CRAP | |
0.00% |
0 / 1 |
| SuggesterAnalysisConfigBuilder | |
80.95% |
170 / 210 |
|
40.00% |
2 / 5 |
20.24 | |
0.00% |
0 / 1 |
| defaults | |
82.32% |
135 / 164 |
|
0.00% |
0 / 1 |
6.20 | |||
| customize | |
76.19% |
32 / 42 |
|
0.00% |
0 / 1 |
10.09 | |||
| buildConfig | |
100.00% |
2 / 2 |
|
100.00% |
1 / 1 |
1 | |||
| getDefaultStopSet | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| hasStopWords | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| 1 | <?php |
| 2 | /** |
| 3 | * @license GPL-2.0-or-later |
| 4 | */ |
| 5 | |
| 6 | namespace CirrusSearch\Maintenance; |
| 7 | |
| 8 | use CirrusSearch\CirrusConfigNames; |
| 9 | |
| 10 | /** |
| 11 | * Builds search analysis config arrays for the completion suggester |
| 12 | * index. |
| 13 | */ |
| 14 | class SuggesterAnalysisConfigBuilder extends AnalysisConfigBuilder { |
| 15 | /** |
| 16 | * Build an analysis config with sane defaults |
| 17 | * |
| 18 | * @param string $language Config language |
| 19 | * @return array |
| 20 | */ |
| 21 | protected function defaults( $language ) { |
| 22 | // Use default lowercase filter |
| 23 | $lowercase_type = [ 'type' => 'lowercase' ]; |
| 24 | if ( $this->isIcuAvailable() ) { |
| 25 | $lowercase_type = [ |
| 26 | "type" => "icu_normalizer", |
| 27 | "name" => "nfkc_cf", |
| 28 | ]; |
| 29 | } |
| 30 | // Use the default Lucene ASCII filter |
| 31 | $folding_type = [ 'type' => 'asciifolding' ]; |
| 32 | if ( $this->shouldActivateIcuFolding( $language ) ) { |
| 33 | // Use ICU Folding if the plugin is available and activated in the config |
| 34 | $folding_type = [ 'type' => 'icu_folding' ]; |
| 35 | $unicodeSetFilter = $this->getICUSetFilter( $language ); |
| 36 | if ( $unicodeSetFilter !== null ) { |
| 37 | $folding_type['unicode_set_filter'] = $unicodeSetFilter; |
| 38 | } |
| 39 | } |
| 40 | $textTokenizer = 'standard'; |
| 41 | $plainTokenizer = $this->config->get( CirrusConfigNames::CompletionPlainTokenizer ); |
| 42 | if ( $this->shouldActivateIcuTokenization( $language ) ) { |
| 43 | $textTokenizer = 'icu_tokenizer'; |
| 44 | // We cannot use the icu_tokenizer for plain here |
| 45 | // even if icu tokenization is mostly needed for languages |
| 46 | // where space is not used to break words. We don't want |
| 47 | // to break some punctuation chars like ':' |
| 48 | } |
| 49 | $defaults = [ |
| 50 | 'char_filter' => [ |
| 51 | 'apostrophe_norm' => [ |
| 52 | 'type' => 'mapping', |
| 53 | 'mappings' => [ |
| 54 | "`=>'", // grave accent |
| 55 | "´=>'", // acute accent |
| 56 | "ʹ=>'", // modifier letter prime |
| 57 | "ʻ=>'", // modifier letter turned comma |
| 58 | "ʼ=>'", // modifier letter apostrophe |
| 59 | "ʽ=>'", // modifier letter reversed comma |
| 60 | "ʾ=>'", // modifier letter right half ring |
| 61 | "ʿ=>'", // modifier letter left half ring |
| 62 | "ˋ=>'", // modifier letter grave accent |
| 63 | "՚=>'", // Armenian apostrophe |
| 64 | "\u05F3=>'", // Hebrew punctuation geresh |
| 65 | "‘=>'", // left single quotation mark |
| 66 | "’=>'", // right single quotation mark |
| 67 | "‛=>'", // single high-reversed-9 quotation mark |
| 68 | "′=>'", // prime |
| 69 | "‵=>'", // reversed prime |
| 70 | "ꞌ=>'", // Latin small letter saltillo |
| 71 | "＇=>'", // fullwidth apostrophe |
| 72 | "｀=>'", // fullwidth grave accent |
| 73 | ], |
| 74 | ], |
| 75 | 'word_break_helper' => [ |
| 76 | 'type' => 'mapping', |
| 77 | 'mappings' => [ |
| 78 | '_=>\u0020', // a space for mw |
| 79 | ',=>\u0020', // useful for "Lastname, Firstname" |
| 80 | '"=>\u0020', // " certainly phrase search? |
| 81 | '-=>\u0020', // useful for hyphenated names |
| 82 | // Not sure about ( and )... |
| 83 | // very useful to search for : |
| 84 | // "john smith explo" instead of "john smith (expl" |
| 85 | // but annoying to search for "(C)" |
| 86 | // ')=>\u0020', |
| 87 | // '(=>\u0020', |
| 88 | // Ignoring : can be misleading for expert users |
| 89 | // Because we will return unrelated pages when the user |
| 90 | // search for "magic keywords" like WP:WP which are sometimes |
| 91 | // pages in the main namespace that redirect to other namespace |
| 92 | // ':=>\u0020', |
| 93 | // Others are the ones ignored by common search engines |
| 94 | ';=>\u0020', |
| 95 | '\\[=>\u0020', |
| 96 | '\\]=>\u0020', |
| 97 | '{=>\u0020', |
| 98 | '}=>\u0020', |
| 99 | '\\\\=>\u0020', |
| 100 | // Unicode white spaces |
| 101 | // cause issues with completion |
| 102 | // only few of them where actually |
| 103 | // identified as problematic but |
| 104 | // more are added for extra safety |
| 105 | // see: T156234 |
| 106 | // TODO: reevaluate with es5 |
| 107 | '\u00a0=>\u0020', |
| 108 | '\u1680=>\u0020', |
| 109 | '\u180e=>\u0020', |
| 110 | '\u2000=>\u0020', |
| 111 | '\u2001=>\u0020', |
| 112 | '\u2002=>\u0020', |
| 113 | '\u2003=>\u0020', |
| 114 | '\u2004=>\u0020', |
| 115 | '\u2005=>\u0020', |
| 116 | '\u2006=>\u0020', |
| 117 | '\u2007=>\u0020', |
| 118 | '\u2008=>\u0020', |
| 119 | '\u2009=>\u0020', |
| 120 | '\u200a=>\u0020', |
| 121 | '\u200b=>\u0020', // causes issue |
| 122 | '\u200c=>\u0020', // causes issue |
| 123 | '\u200d=>\u0020', // causes issue |
| 124 | '\u202f=>\u0020', |
| 125 | '\u205f=>\u0020', |
| 126 | '\u3000=>\u0020', |
| 127 | '\ufeff=>\u0020', // causes issue |
| 128 | ], |
| 129 | ], |
| 130 | ], |
| 131 | 'filter' => [ |
| 132 | "stop_filter" => [ |
| 133 | "type" => "stop", |
| 134 | "stopwords" => "_none_", |
| 135 | "remove_trailing" => "true" |
| 136 | ], |
| 137 | "lowercase" => $lowercase_type, |
| 138 | "accentfolding" => $folding_type, |
| 139 | "token_limit" => [ |
| 140 | "type" => "limit", |
| 141 | "max_token_count" => "20" |
| 142 | ], |
| 143 | // Workaround what seems to be a bug in the |
| 144 | // completion suggester, empty tokens cause an |
| 145 | // issue similar to |
| 146 | // https://github.com/elastic/elasticsearch/pull/11158 |
| 147 | // can be removed with es5 if we want |
| 148 | // note that icu_folding can introduce empty tokens, so |
| 149 | // maybe it is best to leave this in place |
| 150 | "remove_empty" => [ |
| 151 | "type" => "length", |
| 152 | "min" => 1, |
| 153 | ], |
| 154 | ], |
| 155 | 'analyzer' => [ |
| 156 | "stop_analyzer" => [ |
| 157 | "type" => "custom", |
| 158 | "char_filter" => [ 'apostrophe_norm' ], |
| 159 | "filter" => [ |
| 160 | "lowercase", |
| 161 | "stop_filter", |
| 162 | "accentfolding", |
| 163 | "remove_empty", |
| 164 | "token_limit" |
| 165 | ], |
| 166 | "tokenizer" => $textTokenizer, |
| 167 | ], |
| 168 | // We do not remove stop words when searching, |
| 169 | // this leads to extremely weird behaviors while |
| 170 | // writing "to be or no to be" |
| 171 | "stop_analyzer_search" => [ |
| 172 | "type" => "custom", |
| 173 | "char_filter" => [ 'apostrophe_norm' ], |
| 174 | "filter" => [ |
| 175 | "lowercase", |
| 176 | "accentfolding", |
| 177 | "remove_empty", |
| 178 | "token_limit" |
| 179 | ], |
| 180 | "tokenizer" => $textTokenizer, |
| 181 | ], |
| 182 | "plain" => [ |
| 183 | "type" => "custom", |
| 184 | "char_filter" => [ 'apostrophe_norm', 'word_break_helper' ], |
| 185 | "filter" => [ |
| 186 | "remove_empty", |
| 187 | "token_limit", |
| 188 | "lowercase" |
| 189 | ], |
| 190 | "tokenizer" => $plainTokenizer, |
| 191 | ], |
| 192 | "plain_search" => [ |
| 193 | "type" => "custom", |
| 194 | "char_filter" => [ 'apostrophe_norm', 'word_break_helper' ], |
| 195 | "filter" => [ |
| 196 | "remove_empty", |
| 197 | "token_limit", |
| 198 | "lowercase" |
| 199 | ], |
| 200 | "tokenizer" => $plainTokenizer, |
| 201 | ], |
| 202 | ], |
| 203 | ]; |
| 204 | if ( $this->config->getElement( CirrusConfigNames::CompletionSuggesterSubphrases, 'build' ) ) { |
| 205 | $defaults['analyzer']['subphrases'] = [ |
| 206 | "type" => "custom", |
| 207 | "filter" => [ |
| 208 | "lowercase", |
| 209 | "accentfolding", |
| 210 | "remove_empty", |
| 211 | "token_limit" |
| 212 | ], |
| 213 | "tokenizer" => $textTokenizer, |
| 214 | ]; |
| 215 | $defaults['analyzer']['subphrases_search'] = [ |
| 216 | "type" => "custom", |
| 217 | "filter" => [ |
| 218 | "lowercase", |
| 219 | "accentfolding", |
| 220 | "remove_empty", |
| 221 | "token_limit" |
| 222 | ], |
| 223 | "tokenizer" => $textTokenizer, |
| 224 | ]; |
| 225 | } |
| 226 | return $defaults; |
| 227 | } |
| 228 | |
| 229 | /** |
| 230 | * @param array $config |
| 231 | * @param string $language |
| 232 | * @return array |
| 233 | */ |
| 234 | private function customize( array $config, $language ) { |
| 235 | $defaultStopSet = $this->getDefaultStopSet( $language ); |
| 236 | $config['filter']['stop_filter']['stopwords'] = $defaultStopSet; |
| 237 | |
| 238 | switch ( $this->getDefaultTextAnalyzerType( $language ) ) { |
| 239 | // Please add languages in alphabetical order. |
| 240 | case 'arabic': |
| 241 | $config[ 'char_filter' ][ 'arabic_numeral_map' ] = [ |
| 242 | // T117217 fold Eastern Arabic Numerals (٠۱۲۳...) into Western (0123...) |
| 243 | 'type' => 'mapping', |
| 244 | 'mappings' => [ |
| 245 | '\u0660=>0', '\u0661=>1', '\u0662=>2', |
| 246 | '\u0663=>3', '\u0664=>4', '\u0665=>5', |
| 247 | '\u0666=>6', '\u0667=>7', '\u0668=>8', |
| 248 | '\u0669=>9', |
| 249 | ], |
| 250 | ]; |
| 251 | |
| 252 | // add arabic_numeral_map to plain and copy plain to plain_search |
| 253 | $config[ 'analyzer' ][ 'plain' ][ 'char_filter' ][] = 'arabic_numeral_map'; |
| 254 | $config[ 'analyzer' ][ 'plain_search' ] = $config[ 'analyzer' ][ 'plain' ]; |
| 255 | break; |
| 256 | case 'russian': |
| 257 | $config[ 'char_filter' ][ 'russian_diacritic_map' ] = [ |
| 258 | // T117217 fold Eastern Arabic Numerals (٠۱۲۳...) into Western (0123...) |
| 259 | 'type' => 'mapping', |
| 260 | 'mappings' => [ |
| 261 | // T102298 ignore combining acute / stress accents |
| 262 | '\u0301=>', |
| 263 | // T124592 fold ё=>е and Ё=>Е, precomposed or with combining diacritic |
| 264 | '\u0451=>\u0435', |
| 265 | '\u0401=>\u0415', |
| 266 | '\u0435\u0308=>\u0435', |
| 267 | '\u0415\u0308=>\u0415', |
| 268 | |
| 269 | ], |
| 270 | ]; |
| 271 | |
| 272 | // add arabic_numeral_map to plain and copy plain to plain_search |
| 273 | $config[ 'analyzer' ][ 'plain' ][ 'char_filter' ][] = 'russian_diacritic_map'; |
| 274 | $config[ 'analyzer' ][ 'plain_search' ] = $config[ 'analyzer' ][ 'plain' ]; |
| 275 | break; |
| 276 | } |
| 277 | |
| 278 | if ( $this->isIcuAvailable() ) { |
| 279 | foreach ( $config[ 'analyzer' ] as $k => &$analyzer ) { |
| 280 | if ( $k != "stop_analyzer" && $k != "stop_analyzer_search" ) { |
| 281 | continue; |
| 282 | } |
| 283 | if ( !isset( $analyzer[ 'filter' ] ) ) { |
| 284 | continue; |
| 285 | } |
| 286 | $analyzer[ 'filter' ] = array_map( static function ( $filter ) { |
| 287 | if ( $filter === 'lowercase' ) { |
| 288 | return 'icu_normalizer'; |
| 289 | } |
| 290 | return $filter; |
| 291 | }, $analyzer[ 'filter' ] ); |
| 292 | } |
| 293 | } |
| 294 | return $config; |
| 295 | } |
| 296 | |
| 297 | /** |
| 298 | * Build the analysis config. |
| 299 | * |
| 300 | * @param string|null $language Config language |
| 301 | * @return array the analysis config |
| 302 | */ |
| 303 | public function buildConfig( $language = null ) { |
| 304 | $language ??= $this->defaultLanguage; |
| 305 | return $this->customize( $this->defaults( $language ), $language ); |
| 306 | } |
| 307 | |
| 308 | /** @var string[] */ |
| 309 | private static $stopwords = [ |
| 310 | 'ar' => '_arabic_', |
| 311 | 'hy' => '_armenian_', |
| 312 | 'eu' => '_basque_', |
| 313 | 'pt-br' => '_brazilian_', |
| 314 | 'bg' => '_bulgarian_', |
| 315 | 'ca' => '_catalan_', |
| 316 | 'cs' => '_czech_', |
| 317 | 'da' => '_danish_', |
| 318 | 'nl' => '_dutch_', |
| 319 | 'en' => '_english_', |
| 320 | 'en-ca' => '_english_', |
| 321 | 'en-gb' => '_english_', |
| 322 | 'simple' => '_english_', |
| 323 | 'fi' => '_finnish_', |
| 324 | 'fr' => '_french_', |
| 325 | 'gl' => '_galician_', |
| 326 | 'de' => '_german_', |
| 327 | 'el' => '_greek_', |
| 328 | 'hi' => '_hindi_', |
| 329 | 'hu' => '_hungarian_', |
| 330 | 'id' => '_indonesian_', |
| 331 | 'lt' => '_lithuanian_', |
| 332 | 'lv' => '_latvian_', |
| 333 | 'ga' => '_irish_', |
| 334 | 'it' => '_italian_', |
| 335 | 'nb' => '_norwegian_', |
| 336 | 'nn' => '_norwegian_', |
| 337 | 'fa' => '_persian_', |
| 338 | 'pt' => '_portuguese_', |
| 339 | 'ro' => '_romanian_', |
| 340 | 'ru' => '_russian_', |
| 341 | 'ckb' => '_sorani_', |
| 342 | 'es' => '_spanish_', |
| 343 | 'sv' => '_swedish_', |
| 344 | 'th' => '_thai_', |
| 345 | 'tr' => '_turkish_' |
| 346 | ]; |
| 347 | |
| 348 | /** |
| 349 | * @param string $lang |
| 350 | * @return string |
| 351 | */ |
| 352 | private function getDefaultStopSet( $lang ) { |
| 353 | return self::$stopwords[$lang] ?? '_none_'; |
| 354 | } |
| 355 | |
| 356 | /** |
| 357 | * @param string $lang |
| 358 | * @return bool |
| 359 | */ |
| 360 | public static function hasStopWords( $lang ) { |
| 361 | return isset( self::$stopwords[$lang] ); |
| 362 | } |
| 363 | } |