MediaWiki  master
SearchMySQL.php
Go to the documentation of this file.
1 <?php
28 
33 class SearchMySQL extends SearchDatabase {
34  protected $strictMatching = true;
35 
36  private static $mMinSearchLength;
37 
47  private function parseQuery( $filteredText, $fulltext ) {
48  $lc = $this->legalSearchChars( self::CHARS_NO_SYNTAX ); // Minus syntax chars (" and *)
49  $searchon = '';
50  $this->searchTerms = [];
51 
52  # @todo FIXME: This doesn't handle parenthetical expressions.
53  $m = [];
54  if ( preg_match_all( '/([-+<>~]?)(([' . $lc . ']+)(\*?)|"[^"]*")/',
55  $filteredText, $m, PREG_SET_ORDER ) ) {
56  foreach ( $m as $bits ) {
57  Wikimedia\suppressWarnings();
58  list( /* all */, $modifier, $term, $nonQuoted, $wildcard ) = $bits;
59  Wikimedia\restoreWarnings();
60 
61  if ( $nonQuoted != '' ) {
62  $term = $nonQuoted;
63  $quote = '';
64  } else {
65  $term = str_replace( '"', '', $term );
66  $quote = '"';
67  }
68 
69  if ( $searchon !== '' ) {
70  $searchon .= ' ';
71  }
72  if ( $this->strictMatching && ( $modifier == '' ) ) {
73  // If we leave this out, boolean op defaults to OR which is rarely helpful.
74  $modifier = '+';
75  }
76 
77  // Some languages such as Serbian store the input form in the search index,
78  // so we may need to search for matches in multiple writing system variants.
79  $contLang = MediaWikiServices::getInstance()->getContentLanguage();
80  $convertedVariants = $contLang->autoConvertToAllVariants( $term );
81  if ( is_array( $convertedVariants ) ) {
82  $variants = array_unique( array_values( $convertedVariants ) );
83  } else {
84  $variants = [ $term ];
85  }
86 
87  // The low-level search index does some processing on input to work
88  // around problems with minimum lengths and encoding in MySQL's
89  // fulltext engine.
90  // For Chinese this also inserts spaces between adjacent Han characters.
91  $strippedVariants = array_map( [ $contLang, 'normalizeForSearch' ], $variants );
92 
93  // Some languages such as Chinese force all variants to a canonical
94  // form when stripping to the low-level search index, so to be sure
95  // let's check our variants list for unique items after stripping.
96  $strippedVariants = array_unique( $strippedVariants );
97 
98  $searchon .= $modifier;
99  if ( count( $strippedVariants ) > 1 ) {
100  $searchon .= '(';
101  }
102  foreach ( $strippedVariants as $stripped ) {
103  $stripped = $this->normalizeText( $stripped );
104  if ( $nonQuoted && strpos( $stripped, ' ' ) !== false ) {
105  // Hack for Chinese: we need to toss in quotes for
106  // multiple-character phrases since normalizeForSearch()
107  // added spaces between them to make word breaks.
108  $stripped = '"' . trim( $stripped ) . '"';
109  }
110  $searchon .= "$quote$stripped$quote$wildcard ";
111  }
112  if ( count( $strippedVariants ) > 1 ) {
113  $searchon .= ')';
114  }
115 
116  // Match individual terms or quoted phrase in result highlighting...
117  // Note that variants will be introduced in a later stage for highlighting!
118  $regexp = $this->regexTerm( $term, $wildcard );
119  $this->searchTerms[] = $regexp;
120  }
121  wfDebug( __METHOD__ . ": Would search with '$searchon'\n" );
122  wfDebug( __METHOD__ . ': Match with /' . implode( '|', $this->searchTerms ) . "/\n" );
123  } else {
124  wfDebug( __METHOD__ . ": Can't understand search query '{$filteredText}'\n" );
125  }
126 
127  $dbr = $this->lb->getConnectionRef( DB_REPLICA );
128  $searchon = $dbr->addQuotes( $searchon );
129  $field = $this->getIndexField( $fulltext );
130  return [
131  " MATCH($field) AGAINST($searchon IN BOOLEAN MODE) ",
132  " MATCH($field) AGAINST($searchon IN NATURAL LANGUAGE MODE) DESC "
133  ];
134  }
135 
136  private function regexTerm( $string, $wildcard ) {
137  $regex = preg_quote( $string, '/' );
138  if ( MediaWikiServices::getInstance()->getContentLanguage()->hasWordBreaks() ) {
139  if ( $wildcard ) {
140  // Don't cut off the final bit!
141  $regex = "\b$regex";
142  } else {
143  $regex = "\b$regex\b";
144  }
145  } else {
146  // For Chinese, words may legitimately abut other words in the text literal.
147  // Don't add \b boundary checks... note this could cause false positives
148  // for Latin chars.
149  }
150  return $regex;
151  }
152 
153  public function legalSearchChars( $type = self::CHARS_ALL ) {
154  $searchChars = parent::legalSearchChars( $type );
155  if ( $type === self::CHARS_ALL ) {
156  // " for phrase, * for wildcard
157  $searchChars = "\"*" . $searchChars;
158  }
159  return $searchChars;
160  }
161 
168  protected function doSearchTextInDB( $term ) {
169  return $this->searchInternal( $term, true );
170  }
171 
178  protected function doSearchTitleInDB( $term ) {
179  return $this->searchInternal( $term, false );
180  }
181 
182  protected function searchInternal( $term, $fulltext ) {
183  // This seems out of place, why is this called with empty term?
184  if ( trim( $term ) === '' ) {
185  return null;
186  }
187 
188  $filteredTerm = $this->filter( $term );
189  $query = $this->getQuery( $filteredTerm, $fulltext );
190  $dbr = $this->lb->getConnectionRef( DB_REPLICA );
191  $resultSet = $dbr->select(
192  $query['tables'], $query['fields'], $query['conds'],
193  __METHOD__, $query['options'], $query['joins']
194  );
195 
196  $total = null;
197  $query = $this->getCountQuery( $filteredTerm, $fulltext );
198  $totalResult = $dbr->select(
199  $query['tables'], $query['fields'], $query['conds'],
200  __METHOD__, $query['options'], $query['joins']
201  );
202 
203  $row = $totalResult->fetchObject();
204  if ( $row ) {
205  $total = intval( $row->c );
206  }
207  $totalResult->free();
208 
209  return new SqlSearchResultSet( $resultSet, $this->searchTerms, $total );
210  }
211 
212  public function supports( $feature ) {
213  switch ( $feature ) {
214  case 'title-suffix-filter':
215  return true;
216  default:
217  return parent::supports( $feature );
218  }
219  }
220 
226  protected function queryFeatures( &$query ) {
227  foreach ( $this->features as $feature => $value ) {
228  if ( $feature === 'title-suffix-filter' && $value ) {
229  $dbr = $this->lb->getConnectionRef( DB_REPLICA );
230  $query['conds'][] = 'page_title' . $dbr->buildLike( $dbr->anyString(), $value );
231  }
232  }
233  }
234 
240  function queryNamespaces( &$query ) {
241  if ( is_array( $this->namespaces ) ) {
242  if ( count( $this->namespaces ) === 0 ) {
243  $this->namespaces[] = '0';
244  }
245  $query['conds']['page_namespace'] = $this->namespaces;
246  }
247  }
248 
254  protected function limitResult( &$query ) {
255  $query['options']['LIMIT'] = $this->limit;
256  $query['options']['OFFSET'] = $this->offset;
257  }
258 
267  private function getQuery( $filteredTerm, $fulltext ) {
268  $query = [
269  'tables' => [],
270  'fields' => [],
271  'conds' => [],
272  'options' => [],
273  'joins' => [],
274  ];
275 
276  $this->queryMain( $query, $filteredTerm, $fulltext );
277  $this->queryFeatures( $query );
278  $this->queryNamespaces( $query );
279  $this->limitResult( $query );
280 
281  return $query;
282  }
283 
289  private function getIndexField( $fulltext ) {
290  return $fulltext ? 'si_text' : 'si_title';
291  }
292 
301  private function queryMain( &$query, $filteredTerm, $fulltext ) {
302  $match = $this->parseQuery( $filteredTerm, $fulltext );
303  $query['tables'][] = 'page';
304  $query['tables'][] = 'searchindex';
305  $query['fields'][] = 'page_id';
306  $query['fields'][] = 'page_namespace';
307  $query['fields'][] = 'page_title';
308  $query['conds'][] = 'page_id=si_page';
309  $query['conds'][] = $match[0];
310  $query['options']['ORDER BY'] = $match[1];
311  }
312 
319  private function getCountQuery( $filteredTerm, $fulltext ) {
320  $match = $this->parseQuery( $filteredTerm, $fulltext );
321 
322  $query = [
323  'tables' => [ 'page', 'searchindex' ],
324  'fields' => [ 'COUNT(*) as c' ],
325  'conds' => [ 'page_id=si_page', $match[0] ],
326  'options' => [],
327  'joins' => [],
328  ];
329 
330  $this->queryFeatures( $query );
331  $this->queryNamespaces( $query );
332 
333  return $query;
334  }
335 
344  function update( $id, $title, $text ) {
345  $dbw = $this->lb->getConnectionRef( DB_MASTER );
346  $dbw->replace( 'searchindex',
347  [ 'si_page' ],
348  [
349  'si_page' => $id,
350  'si_title' => $this->normalizeText( $title ),
351  'si_text' => $this->normalizeText( $text )
352  ], __METHOD__ );
353  }
354 
362  function updateTitle( $id, $title ) {
363  $dbw = $this->lb->getConnectionRef( DB_MASTER );
364  $dbw->update( 'searchindex',
365  [ 'si_title' => $this->normalizeText( $title ) ],
366  [ 'si_page' => $id ],
367  __METHOD__
368  );
369  }
370 
378  function delete( $id, $title ) {
379  $dbw = $this->lb->getConnectionRef( DB_MASTER );
380  $dbw->delete( 'searchindex', [ 'si_page' => $id ], __METHOD__ );
381  }
382 
389  function normalizeText( $string ) {
390  $out = parent::normalizeText( $string );
391 
392  // MySQL fulltext index doesn't grok utf-8, so we
393  // need to fold cases and convert to hex
394  $out = preg_replace_callback(
395  "/([\\xc0-\\xff][\\x80-\\xbf]*)/",
396  [ $this, 'stripForSearchCallback' ],
397  MediaWikiServices::getInstance()->getContentLanguage()->lc( $out ) );
398 
399  // And to add insult to injury, the default indexing
400  // ignores short words... Pad them so we can pass them
401  // through without reconfiguring the server...
402  $minLength = $this->minSearchLength();
403  if ( $minLength > 1 ) {
404  $n = $minLength - 1;
405  $out = preg_replace(
406  "/\b(\w{1,$n})\b/",
407  "$1u800",
408  $out );
409  }
410 
411  // Periods within things like hostnames and IP addresses
412  // are also important -- we want a search for "example.com"
413  // or "192.168.1.1" to work sanely.
414  // MySQL's search seems to ignore them, so you'd match on
415  // "example.wikipedia.com" and "192.168.83.1" as well.
416  $out = preg_replace(
417  "/(\w)\.(\w|\*)/u",
418  "$1u82e$2",
419  $out );
420 
421  return $out;
422  }
423 
431  protected function stripForSearchCallback( $matches ) {
432  return 'u8' . bin2hex( $matches[1] );
433  }
434 
441  protected function minSearchLength() {
442  if ( is_null( self::$mMinSearchLength ) ) {
443  $sql = "SHOW GLOBAL VARIABLES LIKE 'ft\\_min\\_word\\_len'";
444 
445  $dbr = $this->lb->getConnectionRef( DB_REPLICA );
446  $result = $dbr->query( $sql, __METHOD__ );
447  $row = $result->fetchObject();
448  $result->free();
449 
450  if ( $row && $row->Variable_name == 'ft_min_word_len' ) {
451  self::$mMinSearchLength = intval( $row->Value );
452  } else {
453  self::$mMinSearchLength = 0;
454  }
455  }
456  return self::$mMinSearchLength;
457  }
458 }
minSearchLength()
Check MySQL server&#39;s ft_min_word_len setting so we know if we need to pad short words...
limitResult(&$query)
Add limit options.
queryMain(&$query, $filteredTerm, $fulltext)
Get the base part of the search query.
getIndexField( $fulltext)
Picks which field to index on, depending on what type of query.
normalizeText( $string)
Converts some characters for MySQL&#39;s indexing to grok it correctly, and pads short words to overcome ...
const DB_MASTER
Definition: defines.php:26
updateTitle( $id, $title)
Update a search index record&#39;s title only.
doSearchTitleInDB( $term)
Perform a title-only search query and return a result set.
regexTerm( $string, $wildcard)
Search engine hook for MySQL 4+.
Definition: SearchMySQL.php:33
queryFeatures(&$query)
Add special conditions.
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
getQuery( $filteredTerm, $fulltext)
Construct the SQL query to do the search.
supports( $feature)
This class is used for different SQL-based search engines shipped with MediaWiki. ...
stripForSearchCallback( $matches)
Armor a case-folded UTF-8 string to get through MySQL&#39;s fulltext search without being mucked up by fu...
getCountQuery( $filteredTerm, $fulltext)
static $mMinSearchLength
Definition: SearchMySQL.php:36
parseQuery( $filteredText, $fulltext)
Parse the user&#39;s query and transform it into two SQL fragments: a WHERE condition and an ORDER BY exp...
Definition: SearchMySQL.php:47
int [] null $namespaces
filter( $text)
Return a &#39;cleaned up&#39; search string.
legalSearchChars( $type=self::CHARS_ALL)
update( $id, $title, $text)
Create or update the search index record for the given page.
Base search engine base class for database-backed searches.
searchInternal( $term, $fulltext)
doSearchTextInDB( $term)
Perform a full text search query and return a result set.
const DB_REPLICA
Definition: defines.php:25
queryNamespaces(&$query)
Add namespace conditions.
$matches