MediaWiki REL1_39
SearchMySQL.php
Go to the documentation of this file.
1<?php
28use Wikimedia\AtEase\AtEase;
29
35 protected $strictMatching = true;
36
37 private static $mMinSearchLength;
38
48 private function parseQuery( $filteredText, $fulltext ) {
49 $lc = $this->legalSearchChars( self::CHARS_NO_SYNTAX ); // Minus syntax chars (" and *)
50 $searchon = '';
51 $this->searchTerms = [];
52
53 # @todo FIXME: This doesn't handle parenthetical expressions.
54 $m = [];
55 if ( preg_match_all( '/([-+<>~]?)(([' . $lc . ']+)(\*?)|"[^"]*")/',
56 $filteredText, $m, PREG_SET_ORDER )
57 ) {
58 $services = MediaWikiServices::getInstance();
59 $contLang = $services->getContentLanguage();
60 $langConverter = $services->getLanguageConverterFactory()->getLanguageConverter( $contLang );
61 foreach ( $m as $bits ) {
62 AtEase::suppressWarnings();
63 list( /* all */, $modifier, $term, $nonQuoted, $wildcard ) = $bits;
64 AtEase::restoreWarnings();
65
66 if ( $nonQuoted != '' ) {
67 $term = $nonQuoted;
68 $quote = '';
69 } else {
70 $term = str_replace( '"', '', $term );
71 $quote = '"';
72 }
73
74 if ( $searchon !== '' ) {
75 $searchon .= ' ';
76 }
77 if ( $this->strictMatching && ( $modifier == '' ) ) {
78 // If we leave this out, boolean op defaults to OR which is rarely helpful.
79 $modifier = '+';
80 }
81
82 // Some languages such as Serbian store the input form in the search index,
83 // so we may need to search for matches in multiple writing system variants.
84 $convertedVariants = $langConverter->autoConvertToAllVariants( $term );
85 if ( is_array( $convertedVariants ) ) {
86 $variants = array_unique( array_values( $convertedVariants ) );
87 } else {
88 $variants = [ $term ];
89 }
90
91 // The low-level search index does some processing on input to work
92 // around problems with minimum lengths and encoding in MySQL's
93 // fulltext engine.
94 // For Chinese this also inserts spaces between adjacent Han characters.
95 $strippedVariants = array_map( [ $contLang, 'normalizeForSearch' ], $variants );
96
97 // Some languages such as Chinese force all variants to a canonical
98 // form when stripping to the low-level search index, so to be sure
99 // let's check our variants list for unique items after stripping.
100 $strippedVariants = array_unique( $strippedVariants );
101
102 $searchon .= $modifier;
103 if ( count( $strippedVariants ) > 1 ) {
104 $searchon .= '(';
105 }
106 foreach ( $strippedVariants as $stripped ) {
107 $stripped = $this->normalizeText( $stripped );
108 if ( $nonQuoted && strpos( $stripped, ' ' ) !== false ) {
109 // Hack for Chinese: we need to toss in quotes for
110 // multiple-character phrases since normalizeForSearch()
111 // added spaces between them to make word breaks.
112 $stripped = '"' . trim( $stripped ) . '"';
113 }
114 $searchon .= "$quote$stripped$quote$wildcard ";
115 }
116 if ( count( $strippedVariants ) > 1 ) {
117 $searchon .= ')';
118 }
119
120 // Match individual terms or quoted phrase in result highlighting...
121 // Note that variants will be introduced in a later stage for highlighting!
122 $regexp = $this->regexTerm( $term, $wildcard );
123 $this->searchTerms[] = $regexp;
124 }
125 wfDebug( __METHOD__ . ": Would search with '$searchon'" );
126 wfDebug( __METHOD__ . ': Match with /' . implode( '|', $this->searchTerms ) . "/" );
127 } else {
128 wfDebug( __METHOD__ . ": Can't understand search query '{$filteredText}'" );
129 }
130
131 $dbr = $this->lb->getConnectionRef( DB_REPLICA );
132 $searchon = $dbr->addQuotes( $searchon );
133 $field = $this->getIndexField( $fulltext );
134 return [
135 " MATCH($field) AGAINST($searchon IN BOOLEAN MODE) ",
136 " MATCH($field) AGAINST($searchon IN NATURAL LANGUAGE MODE) DESC "
137 ];
138 }
139
140 private function regexTerm( $string, $wildcard ) {
141 $regex = preg_quote( $string, '/' );
142 if ( MediaWikiServices::getInstance()->getContentLanguage()->hasWordBreaks() ) {
143 if ( $wildcard ) {
144 // Don't cut off the final bit!
145 $regex = "\b$regex";
146 } else {
147 $regex = "\b$regex\b";
148 }
149 } else {
150 // For Chinese, words may legitimately abut other words in the text literal.
151 // Don't add \b boundary checks... note this could cause false positives
152 // for Latin chars.
153 }
154 return $regex;
155 }
156
157 public function legalSearchChars( $type = self::CHARS_ALL ) {
158 $searchChars = parent::legalSearchChars( $type );
159 if ( $type === self::CHARS_ALL ) {
160 // " for phrase, * for wildcard
161 $searchChars = "\"*" . $searchChars;
162 }
163 return $searchChars;
164 }
165
172 protected function doSearchTextInDB( $term ) {
173 return $this->searchInternal( $term, true );
174 }
175
182 protected function doSearchTitleInDB( $term ) {
183 return $this->searchInternal( $term, false );
184 }
185
186 protected function searchInternal( $term, $fulltext ) {
187 // This seems out of place, why is this called with empty term?
188 if ( trim( $term ) === '' ) {
189 return null;
190 }
191
192 $filteredTerm = $this->filter( $term );
193 $query = $this->getQuery( $filteredTerm, $fulltext );
194 $dbr = $this->lb->getConnectionRef( DB_REPLICA );
195 $resultSet = $dbr->select(
196 $query['tables'], $query['fields'], $query['conds'],
197 __METHOD__, $query['options'], $query['joins']
198 );
199
200 $total = null;
201 $query = $this->getCountQuery( $filteredTerm, $fulltext );
202 $totalResult = $dbr->select(
203 $query['tables'], $query['fields'], $query['conds'],
204 __METHOD__, $query['options'], $query['joins']
205 );
206
207 $row = $totalResult->fetchObject();
208 if ( $row ) {
209 $total = intval( $row->c );
210 }
211 $totalResult->free();
212
213 return new SqlSearchResultSet( $resultSet, $this->searchTerms, $total );
214 }
215
216 public function supports( $feature ) {
217 switch ( $feature ) {
218 case 'title-suffix-filter':
219 return true;
220 default:
221 return parent::supports( $feature );
222 }
223 }
224
230 protected function queryFeatures( &$query ) {
231 foreach ( $this->features as $feature => $value ) {
232 if ( $feature === 'title-suffix-filter' && $value ) {
233 $dbr = $this->lb->getConnectionRef( DB_REPLICA );
234 $query['conds'][] = 'page_title' . $dbr->buildLike( $dbr->anyString(), $value );
235 }
236 }
237 }
238
244 private function queryNamespaces( &$query ) {
245 if ( is_array( $this->namespaces ) ) {
246 if ( count( $this->namespaces ) === 0 ) {
247 $this->namespaces[] = NS_MAIN;
248 }
249 $query['conds']['page_namespace'] = $this->namespaces;
250 }
251 }
252
258 protected function limitResult( &$query ) {
259 $query['options']['LIMIT'] = $this->limit;
260 $query['options']['OFFSET'] = $this->offset;
261 }
262
271 private function getQuery( $filteredTerm, $fulltext ) {
272 $query = [
273 'tables' => [],
274 'fields' => [],
275 'conds' => [],
276 'options' => [],
277 'joins' => [],
278 ];
279
280 $this->queryMain( $query, $filteredTerm, $fulltext );
281 $this->queryFeatures( $query );
282 $this->queryNamespaces( $query );
283 $this->limitResult( $query );
284
285 return $query;
286 }
287
293 private function getIndexField( $fulltext ) {
294 return $fulltext ? 'si_text' : 'si_title';
295 }
296
305 private function queryMain( &$query, $filteredTerm, $fulltext ) {
306 $match = $this->parseQuery( $filteredTerm, $fulltext );
307 $query['tables'][] = 'page';
308 $query['tables'][] = 'searchindex';
309 $query['fields'][] = 'page_id';
310 $query['fields'][] = 'page_namespace';
311 $query['fields'][] = 'page_title';
312 $query['conds'][] = 'page_id=si_page';
313 $query['conds'][] = $match[0];
314 $query['options']['ORDER BY'] = $match[1];
315 }
316
323 private function getCountQuery( $filteredTerm, $fulltext ) {
324 $match = $this->parseQuery( $filteredTerm, $fulltext );
325
326 $query = [
327 'tables' => [ 'page', 'searchindex' ],
328 'fields' => [ 'COUNT(*) as c' ],
329 'conds' => [ 'page_id=si_page', $match[0] ],
330 'options' => [],
331 'joins' => [],
332 ];
333
334 $this->queryFeatures( $query );
335 $this->queryNamespaces( $query );
336
337 return $query;
338 }
339
348 public function update( $id, $title, $text ) {
349 $dbw = $this->lb->getConnectionRef( DB_PRIMARY );
350 $dbw->replace(
351 'searchindex',
352 'si_page',
353 [
354 'si_page' => $id,
355 'si_title' => $this->normalizeText( $title ),
356 'si_text' => $this->normalizeText( $text )
357 ],
358 __METHOD__
359 );
360 }
361
369 public function updateTitle( $id, $title ) {
370 $dbw = $this->lb->getConnectionRef( DB_PRIMARY );
371 $dbw->update( 'searchindex',
372 [ 'si_title' => $this->normalizeText( $title ) ],
373 [ 'si_page' => $id ],
374 __METHOD__
375 );
376 }
377
385 public function delete( $id, $title ) {
386 $dbw = $this->lb->getConnectionRef( DB_PRIMARY );
387 $dbw->delete( 'searchindex', [ 'si_page' => $id ], __METHOD__ );
388 }
389
396 public function normalizeText( $string ) {
397 $out = parent::normalizeText( $string );
398
399 // MySQL fulltext index doesn't grok utf-8, so we
400 // need to fold cases and convert to hex
401 $out = preg_replace_callback(
402 "/([\\xc0-\\xff][\\x80-\\xbf]*)/",
403 [ $this, 'stripForSearchCallback' ],
404 MediaWikiServices::getInstance()->getContentLanguage()->lc( $out ) );
405
406 // And to add insult to injury, the default indexing
407 // ignores short words... Pad them so we can pass them
408 // through without reconfiguring the server...
409 $minLength = $this->minSearchLength();
410 if ( $minLength > 1 ) {
411 $n = $minLength - 1;
412 $out = preg_replace(
413 "/\b(\w{1,$n})\b/",
414 "$1u800",
415 $out );
416 }
417
418 // Periods within things like hostnames and IP addresses
419 // are also important -- we want a search for "example.com"
420 // or "192.168.1.1" to work sensibly.
421 // MySQL's search seems to ignore them, so you'd match on
422 // "example.wikipedia.com" and "192.168.83.1" as well.
423 return preg_replace(
424 "/(\w)\.(\w|\*)/u",
425 "$1u82e$2",
426 $out
427 );
428 }
429
437 protected function stripForSearchCallback( $matches ) {
438 return 'u8' . bin2hex( $matches[1] );
439 }
440
447 protected function minSearchLength() {
448 if ( self::$mMinSearchLength === null ) {
449 $sql = "SHOW GLOBAL VARIABLES LIKE 'ft\\_min\\_word\\_len'";
450
451 $dbr = $this->lb->getConnectionRef( DB_REPLICA );
452 $result = $dbr->query( $sql, __METHOD__ );
453 $row = $result->fetchObject();
454 $result->free();
455
456 if ( $row && $row->Variable_name == 'ft_min_word_len' ) {
457 self::$mMinSearchLength = intval( $row->Value );
458 } else {
459 self::$mMinSearchLength = 0;
460 }
461 }
462 return self::$mMinSearchLength;
463 }
464}
const NS_MAIN
Definition Defines.php:64
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
Service locator for MediaWiki core services.
Base search engine base class for database-backed searches.
filter( $text)
Return a 'cleaned up' search string.
int[] null $namespaces
Search engine hook for MySQL.
updateTitle( $id, $title)
Update a search index record's title only.
doSearchTitleInDB( $term)
Perform a title-only search query and return a result set.
stripForSearchCallback( $matches)
Armor a case-folded UTF-8 string to get through MySQL's fulltext search without being mucked up by fu...
queryFeatures(&$query)
Add special conditions.
update( $id, $title, $text)
Create or update the search index record for the given page.
supports( $feature)
searchInternal( $term, $fulltext)
limitResult(&$query)
Add limit options.
legalSearchChars( $type=self::CHARS_ALL)
Get chars legal for search.
normalizeText( $string)
Converts some characters for MySQL's indexing to grok it correctly, and pads short words to overcome ...
minSearchLength()
Check MySQL server's ft_min_word_len setting so we know if we need to pad short words....
doSearchTextInDB( $term)
Perform a full text search query and return a result set.
This class is used for different SQL-based search engines shipped with MediaWiki.
const DB_REPLICA
Definition defines.php:26
const DB_PRIMARY
Definition defines.php:28