MediaWiki  master
SearchMySQL.php
Go to the documentation of this file.
1 <?php
28 use Wikimedia\AtEase\AtEase;
29 
34 class SearchMySQL extends SearchDatabase {
35  protected $strictMatching = true;
36 
37  private static $mMinSearchLength;
38 
48  private function parseQuery( $filteredText, $fulltext ) {
49  $lc = $this->legalSearchChars( self::CHARS_NO_SYNTAX ); // Minus syntax chars (" and *)
50  $searchon = '';
51  $this->searchTerms = [];
52 
53  # @todo FIXME: This doesn't handle parenthetical expressions.
54  $m = [];
55  if ( preg_match_all( '/([-+<>~]?)(([' . $lc . ']+)(\*?)|"[^"]*")/',
56  $filteredText, $m, PREG_SET_ORDER )
57  ) {
58  $services = MediaWikiServices::getInstance();
59  $contLang = $services->getContentLanguage();
60  $langConverter = $services->getLanguageConverterFactory()->getLanguageConverter( $contLang );
61  foreach ( $m as $bits ) {
62  AtEase::suppressWarnings();
63  [ /* all */, $modifier, $term, $nonQuoted, $wildcard ] = $bits;
64  AtEase::restoreWarnings();
65 
66  if ( $nonQuoted != '' ) {
67  $term = $nonQuoted;
68  $quote = '';
69  } else {
70  $term = str_replace( '"', '', $term );
71  $quote = '"';
72  }
73 
74  if ( $searchon !== '' ) {
75  $searchon .= ' ';
76  }
77  if ( $this->strictMatching && ( $modifier == '' ) ) {
78  // If we leave this out, boolean op defaults to OR which is rarely helpful.
79  $modifier = '+';
80  }
81 
82  // Some languages such as Serbian store the input form in the search index,
83  // so we may need to search for matches in multiple writing system variants.
84  $convertedVariants = $langConverter->autoConvertToAllVariants( $term );
85  if ( is_array( $convertedVariants ) ) {
86  $variants = array_unique( array_values( $convertedVariants ) );
87  } else {
88  $variants = [ $term ];
89  }
90 
91  // The low-level search index does some processing on input to work
92  // around problems with minimum lengths and encoding in MySQL's
93  // fulltext engine.
94  // For Chinese this also inserts spaces between adjacent Han characters.
95  $strippedVariants = array_map( [ $contLang, 'normalizeForSearch' ], $variants );
96 
97  // Some languages such as Chinese force all variants to a canonical
98  // form when stripping to the low-level search index, so to be sure
99  // let's check our variants list for unique items after stripping.
100  $strippedVariants = array_unique( $strippedVariants );
101 
102  $searchon .= $modifier;
103  if ( count( $strippedVariants ) > 1 ) {
104  $searchon .= '(';
105  }
106  foreach ( $strippedVariants as $stripped ) {
107  $stripped = $this->normalizeText( $stripped );
108  if ( $nonQuoted && strpos( $stripped, ' ' ) !== false ) {
109  // Hack for Chinese: we need to toss in quotes for
110  // multiple-character phrases since normalizeForSearch()
111  // added spaces between them to make word breaks.
112  $stripped = '"' . trim( $stripped ) . '"';
113  }
114  $searchon .= "$quote$stripped$quote$wildcard ";
115  }
116  if ( count( $strippedVariants ) > 1 ) {
117  $searchon .= ')';
118  }
119 
120  // Match individual terms or quoted phrase in result highlighting...
121  // Note that variants will be introduced in a later stage for highlighting!
122  $regexp = $this->regexTerm( $term, $wildcard );
123  $this->searchTerms[] = $regexp;
124  }
125  wfDebug( __METHOD__ . ": Would search with '$searchon'" );
126  wfDebug( __METHOD__ . ': Match with /' . implode( '|', $this->searchTerms ) . "/" );
127  } else {
128  wfDebug( __METHOD__ . ": Can't understand search query '{$filteredText}'" );
129  }
130 
131  $dbr = $this->lb->getConnectionRef( DB_REPLICA );
132  $searchon = $dbr->addQuotes( $searchon );
133  $field = $this->getIndexField( $fulltext );
134  return [
135  " MATCH($field) AGAINST($searchon IN BOOLEAN MODE) ",
136  " MATCH($field) AGAINST($searchon IN NATURAL LANGUAGE MODE) DESC "
137  ];
138  }
139 
140  private function regexTerm( $string, $wildcard ) {
141  $regex = preg_quote( $string, '/' );
142  if ( MediaWikiServices::getInstance()->getContentLanguage()->hasWordBreaks() ) {
143  if ( $wildcard ) {
144  // Don't cut off the final bit!
145  $regex = "\b$regex";
146  } else {
147  $regex = "\b$regex\b";
148  }
149  } else {
150  // For Chinese, words may legitimately abut other words in the text literal.
151  // Don't add \b boundary checks... note this could cause false positives
152  // for Latin chars.
153  }
154  return $regex;
155  }
156 
157  public function legalSearchChars( $type = self::CHARS_ALL ) {
158  $searchChars = parent::legalSearchChars( $type );
159  if ( $type === self::CHARS_ALL ) {
160  // " for phrase, * for wildcard
161  $searchChars = "\"*" . $searchChars;
162  }
163  return $searchChars;
164  }
165 
172  protected function doSearchTextInDB( $term ) {
173  return $this->searchInternal( $term, true );
174  }
175 
182  protected function doSearchTitleInDB( $term ) {
183  return $this->searchInternal( $term, false );
184  }
185 
186  protected function searchInternal( $term, $fulltext ) {
187  // This seems out of place, why is this called with empty term?
188  if ( trim( $term ) === '' ) {
189  return null;
190  }
191 
192  $filteredTerm = $this->filter( $term );
193  $query = $this->getQuery( $filteredTerm, $fulltext );
194  $dbr = $this->lb->getConnectionRef( DB_REPLICA );
195  $resultSet = $dbr->select(
196  $query['tables'], $query['fields'], $query['conds'],
197  __METHOD__, $query['options'], $query['joins']
198  );
199 
200  $total = null;
201  $query = $this->getCountQuery( $filteredTerm, $fulltext );
202  $totalResult = $dbr->select(
203  $query['tables'], $query['fields'], $query['conds'],
204  __METHOD__, $query['options'], $query['joins']
205  );
206 
207  $row = $totalResult->fetchObject();
208  if ( $row ) {
209  $total = intval( $row->c );
210  }
211  $totalResult->free();
212 
213  return new SqlSearchResultSet( $resultSet, $this->searchTerms, $total );
214  }
215 
216  public function supports( $feature ) {
217  switch ( $feature ) {
218  case 'title-suffix-filter':
219  return true;
220  default:
221  return parent::supports( $feature );
222  }
223  }
224 
230  protected function queryFeatures( &$query ) {
231  foreach ( $this->features as $feature => $value ) {
232  if ( $feature === 'title-suffix-filter' && $value ) {
233  $dbr = $this->lb->getConnectionRef( DB_REPLICA );
234  $query['conds'][] = 'page_title' . $dbr->buildLike( $dbr->anyString(), $value );
235  }
236  }
237  }
238 
244  private function queryNamespaces( &$query ) {
245  if ( is_array( $this->namespaces ) ) {
246  if ( count( $this->namespaces ) === 0 ) {
247  $this->namespaces[] = NS_MAIN;
248  }
249  $query['conds']['page_namespace'] = $this->namespaces;
250  }
251  }
252 
258  protected function limitResult( &$query ) {
259  $query['options']['LIMIT'] = $this->limit;
260  $query['options']['OFFSET'] = $this->offset;
261  }
262 
271  private function getQuery( $filteredTerm, $fulltext ) {
272  $query = [
273  'tables' => [],
274  'fields' => [],
275  'conds' => [],
276  'options' => [],
277  'joins' => [],
278  ];
279 
280  $this->queryMain( $query, $filteredTerm, $fulltext );
281  $this->queryFeatures( $query );
282  $this->queryNamespaces( $query );
283  $this->limitResult( $query );
284 
285  return $query;
286  }
287 
293  private function getIndexField( $fulltext ) {
294  return $fulltext ? 'si_text' : 'si_title';
295  }
296 
305  private function queryMain( &$query, $filteredTerm, $fulltext ) {
306  $match = $this->parseQuery( $filteredTerm, $fulltext );
307  $query['tables'][] = 'page';
308  $query['tables'][] = 'searchindex';
309  $query['fields'][] = 'page_id';
310  $query['fields'][] = 'page_namespace';
311  $query['fields'][] = 'page_title';
312  $query['conds'][] = 'page_id=si_page';
313  $query['conds'][] = $match[0];
314  $query['options']['ORDER BY'] = $match[1];
315  }
316 
323  private function getCountQuery( $filteredTerm, $fulltext ) {
324  $match = $this->parseQuery( $filteredTerm, $fulltext );
325 
326  $query = [
327  'tables' => [ 'page', 'searchindex' ],
328  'fields' => [ 'COUNT(*) as c' ],
329  'conds' => [ 'page_id=si_page', $match[0] ],
330  'options' => [],
331  'joins' => [],
332  ];
333 
334  $this->queryFeatures( $query );
335  $this->queryNamespaces( $query );
336 
337  return $query;
338  }
339 
348  public function update( $id, $title, $text ) {
349  $dbw = $this->lb->getConnectionRef( DB_PRIMARY );
350  $dbw->replace(
351  'searchindex',
352  'si_page',
353  [
354  'si_page' => $id,
355  'si_title' => $this->normalizeText( $title ),
356  'si_text' => $this->normalizeText( $text )
357  ],
358  __METHOD__
359  );
360  }
361 
369  public function updateTitle( $id, $title ) {
370  $dbw = $this->lb->getConnectionRef( DB_PRIMARY );
371  $dbw->update( 'searchindex',
372  [ 'si_title' => $this->normalizeText( $title ) ],
373  [ 'si_page' => $id ],
374  __METHOD__
375  );
376  }
377 
385  public function delete( $id, $title ) {
386  $dbw = $this->lb->getConnectionRef( DB_PRIMARY );
387  $dbw->delete( 'searchindex', [ 'si_page' => $id ], __METHOD__ );
388  }
389 
396  public function normalizeText( $string ) {
397  $out = parent::normalizeText( $string );
398 
399  // MySQL fulltext index doesn't grok utf-8, so we
400  // need to fold cases and convert to hex
401  $out = preg_replace_callback(
402  "/([\\xc0-\\xff][\\x80-\\xbf]*)/",
403  [ $this, 'stripForSearchCallback' ],
404  MediaWikiServices::getInstance()->getContentLanguage()->lc( $out ) );
405 
406  // And to add insult to injury, the default indexing
407  // ignores short words... Pad them so we can pass them
408  // through without reconfiguring the server...
409  $minLength = $this->minSearchLength();
410  if ( $minLength > 1 ) {
411  $n = $minLength - 1;
412  $out = preg_replace(
413  "/\b(\w{1,$n})\b/",
414  "$1u800",
415  $out );
416  }
417 
418  // Periods within things like hostnames and IP addresses
419  // are also important -- we want a search for "example.com"
420  // or "192.168.1.1" to work sensibly.
421  // MySQL's search seems to ignore them, so you'd match on
422  // "example.wikipedia.com" and "192.168.83.1" as well.
423  return preg_replace(
424  "/(\w)\.(\w|\*)/u",
425  "$1u82e$2",
426  $out
427  );
428  }
429 
437  protected function stripForSearchCallback( $matches ) {
438  return 'u8' . bin2hex( $matches[1] );
439  }
440 
447  protected function minSearchLength() {
448  if ( self::$mMinSearchLength === null ) {
449  $sql = "SHOW GLOBAL VARIABLES LIKE 'ft\\_min\\_word\\_len'";
450 
451  $dbr = $this->lb->getConnectionRef( DB_REPLICA );
452  $result = $dbr->query( $sql, __METHOD__ );
453  $row = $result->fetchObject();
454  $result->free();
455 
456  if ( $row && $row->Variable_name == 'ft_min_word_len' ) {
457  self::$mMinSearchLength = intval( $row->Value );
458  } else {
459  self::$mMinSearchLength = 0;
460  }
461  }
462  return self::$mMinSearchLength;
463  }
464 }
const NS_MAIN
Definition: Defines.php:64
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
$matches
Service locator for MediaWiki core services.
Base search engine base class for database-backed searches.
filter( $text)
Return a 'cleaned up' search string.
int[] null $namespaces
Search engine hook for MySQL.
Definition: SearchMySQL.php:34
updateTitle( $id, $title)
Update a search index record's title only.
doSearchTitleInDB( $term)
Perform a title-only search query and return a result set.
stripForSearchCallback( $matches)
Armor a case-folded UTF-8 string to get through MySQL's fulltext search without being mucked up by fu...
queryFeatures(&$query)
Add special conditions.
update( $id, $title, $text)
Create or update the search index record for the given page.
supports( $feature)
searchInternal( $term, $fulltext)
limitResult(&$query)
Add limit options.
legalSearchChars( $type=self::CHARS_ALL)
Get chars legal for search.
normalizeText( $string)
Converts some characters for MySQL's indexing to grok it correctly, and pads short words to overcome ...
minSearchLength()
Check MySQL server's ft_min_word_len setting so we know if we need to pad short words....
doSearchTextInDB( $term)
Perform a full text search query and return a result set.
This class is used for different SQL-based search engines shipped with MediaWiki.
const DB_REPLICA
Definition: defines.php:26
const DB_PRIMARY
Definition: defines.php:28