MediaWiki  master
SearchMySQL.php
Go to the documentation of this file.
1 <?php
28 
33 class SearchMySQL extends SearchDatabase {
34  protected $strictMatching = true;
35 
36  private static $mMinSearchLength;
37 
47  private function parseQuery( $filteredText, $fulltext ) {
48  $lc = $this->legalSearchChars( self::CHARS_NO_SYNTAX ); // Minus syntax chars (" and *)
49  $searchon = '';
50  $this->searchTerms = [];
51 
52  # @todo FIXME: This doesn't handle parenthetical expressions.
53  $m = [];
54  if ( preg_match_all( '/([-+<>~]?)(([' . $lc . ']+)(\*?)|"[^"]*")/',
55  $filteredText, $m, PREG_SET_ORDER )
56  ) {
57  $services = MediaWikiServices::getInstance();
58  $contLang = $services->getContentLanguage();
59  $langConverter = $services->getLanguageConverterFactory()->getLanguageConverter( $contLang );
60  foreach ( $m as $bits ) {
61  Wikimedia\suppressWarnings();
62  list( /* all */, $modifier, $term, $nonQuoted, $wildcard ) = $bits;
63  Wikimedia\restoreWarnings();
64 
65  if ( $nonQuoted != '' ) {
66  $term = $nonQuoted;
67  $quote = '';
68  } else {
69  $term = str_replace( '"', '', $term );
70  $quote = '"';
71  }
72 
73  if ( $searchon !== '' ) {
74  $searchon .= ' ';
75  }
76  if ( $this->strictMatching && ( $modifier == '' ) ) {
77  // If we leave this out, boolean op defaults to OR which is rarely helpful.
78  $modifier = '+';
79  }
80 
81  // Some languages such as Serbian store the input form in the search index,
82  // so we may need to search for matches in multiple writing system variants.
83  $convertedVariants = $langConverter->autoConvertToAllVariants( $term );
84  if ( is_array( $convertedVariants ) ) {
85  $variants = array_unique( array_values( $convertedVariants ) );
86  } else {
87  $variants = [ $term ];
88  }
89 
90  // The low-level search index does some processing on input to work
91  // around problems with minimum lengths and encoding in MySQL's
92  // fulltext engine.
93  // For Chinese this also inserts spaces between adjacent Han characters.
94  $strippedVariants = array_map( [ $contLang, 'normalizeForSearch' ], $variants );
95 
96  // Some languages such as Chinese force all variants to a canonical
97  // form when stripping to the low-level search index, so to be sure
98  // let's check our variants list for unique items after stripping.
99  $strippedVariants = array_unique( $strippedVariants );
100 
101  $searchon .= $modifier;
102  if ( count( $strippedVariants ) > 1 ) {
103  $searchon .= '(';
104  }
105  foreach ( $strippedVariants as $stripped ) {
106  $stripped = $this->normalizeText( $stripped );
107  if ( $nonQuoted && strpos( $stripped, ' ' ) !== false ) {
108  // Hack for Chinese: we need to toss in quotes for
109  // multiple-character phrases since normalizeForSearch()
110  // added spaces between them to make word breaks.
111  $stripped = '"' . trim( $stripped ) . '"';
112  }
113  $searchon .= "$quote$stripped$quote$wildcard ";
114  }
115  if ( count( $strippedVariants ) > 1 ) {
116  $searchon .= ')';
117  }
118 
119  // Match individual terms or quoted phrase in result highlighting...
120  // Note that variants will be introduced in a later stage for highlighting!
121  $regexp = $this->regexTerm( $term, $wildcard );
122  $this->searchTerms[] = $regexp;
123  }
124  wfDebug( __METHOD__ . ": Would search with '$searchon'" );
125  wfDebug( __METHOD__ . ': Match with /' . implode( '|', $this->searchTerms ) . "/" );
126  } else {
127  wfDebug( __METHOD__ . ": Can't understand search query '{$filteredText}'" );
128  }
129 
130  $dbr = $this->lb->getConnectionRef( DB_REPLICA );
131  $searchon = $dbr->addQuotes( $searchon );
132  $field = $this->getIndexField( $fulltext );
133  return [
134  " MATCH($field) AGAINST($searchon IN BOOLEAN MODE) ",
135  " MATCH($field) AGAINST($searchon IN NATURAL LANGUAGE MODE) DESC "
136  ];
137  }
138 
139  private function regexTerm( $string, $wildcard ) {
140  $regex = preg_quote( $string, '/' );
141  if ( MediaWikiServices::getInstance()->getContentLanguage()->hasWordBreaks() ) {
142  if ( $wildcard ) {
143  // Don't cut off the final bit!
144  $regex = "\b$regex";
145  } else {
146  $regex = "\b$regex\b";
147  }
148  } else {
149  // For Chinese, words may legitimately abut other words in the text literal.
150  // Don't add \b boundary checks... note this could cause false positives
151  // for Latin chars.
152  }
153  return $regex;
154  }
155 
156  public function legalSearchChars( $type = self::CHARS_ALL ) {
157  $searchChars = parent::legalSearchChars( $type );
158  if ( $type === self::CHARS_ALL ) {
159  // " for phrase, * for wildcard
160  $searchChars = "\"*" . $searchChars;
161  }
162  return $searchChars;
163  }
164 
171  protected function doSearchTextInDB( $term ) {
172  return $this->searchInternal( $term, true );
173  }
174 
181  protected function doSearchTitleInDB( $term ) {
182  return $this->searchInternal( $term, false );
183  }
184 
185  protected function searchInternal( $term, $fulltext ) {
186  // This seems out of place, why is this called with empty term?
187  if ( trim( $term ) === '' ) {
188  return null;
189  }
190 
191  $filteredTerm = $this->filter( $term );
192  $query = $this->getQuery( $filteredTerm, $fulltext );
193  $dbr = $this->lb->getConnectionRef( DB_REPLICA );
194  $resultSet = $dbr->select(
195  $query['tables'], $query['fields'], $query['conds'],
196  __METHOD__, $query['options'], $query['joins']
197  );
198 
199  $total = null;
200  $query = $this->getCountQuery( $filteredTerm, $fulltext );
201  $totalResult = $dbr->select(
202  $query['tables'], $query['fields'], $query['conds'],
203  __METHOD__, $query['options'], $query['joins']
204  );
205 
206  $row = $totalResult->fetchObject();
207  if ( $row ) {
208  $total = intval( $row->c );
209  }
210  $totalResult->free();
211 
212  return new SqlSearchResultSet( $resultSet, $this->searchTerms, $total );
213  }
214 
215  public function supports( $feature ) {
216  switch ( $feature ) {
217  case 'title-suffix-filter':
218  return true;
219  default:
220  return parent::supports( $feature );
221  }
222  }
223 
229  protected function queryFeatures( &$query ) {
230  foreach ( $this->features as $feature => $value ) {
231  if ( $feature === 'title-suffix-filter' && $value ) {
232  $dbr = $this->lb->getConnectionRef( DB_REPLICA );
233  $query['conds'][] = 'page_title' . $dbr->buildLike( $dbr->anyString(), $value );
234  }
235  }
236  }
237 
243  private function queryNamespaces( &$query ) {
244  if ( is_array( $this->namespaces ) ) {
245  if ( count( $this->namespaces ) === 0 ) {
246  $this->namespaces[] = '0';
247  }
248  $query['conds']['page_namespace'] = $this->namespaces;
249  }
250  }
251 
257  protected function limitResult( &$query ) {
258  $query['options']['LIMIT'] = $this->limit;
259  $query['options']['OFFSET'] = $this->offset;
260  }
261 
270  private function getQuery( $filteredTerm, $fulltext ) {
271  $query = [
272  'tables' => [],
273  'fields' => [],
274  'conds' => [],
275  'options' => [],
276  'joins' => [],
277  ];
278 
279  $this->queryMain( $query, $filteredTerm, $fulltext );
280  $this->queryFeatures( $query );
281  $this->queryNamespaces( $query );
282  $this->limitResult( $query );
283 
284  return $query;
285  }
286 
292  private function getIndexField( $fulltext ) {
293  return $fulltext ? 'si_text' : 'si_title';
294  }
295 
304  private function queryMain( &$query, $filteredTerm, $fulltext ) {
305  $match = $this->parseQuery( $filteredTerm, $fulltext );
306  $query['tables'][] = 'page';
307  $query['tables'][] = 'searchindex';
308  $query['fields'][] = 'page_id';
309  $query['fields'][] = 'page_namespace';
310  $query['fields'][] = 'page_title';
311  $query['conds'][] = 'page_id=si_page';
312  $query['conds'][] = $match[0];
313  $query['options']['ORDER BY'] = $match[1];
314  }
315 
322  private function getCountQuery( $filteredTerm, $fulltext ) {
323  $match = $this->parseQuery( $filteredTerm, $fulltext );
324 
325  $query = [
326  'tables' => [ 'page', 'searchindex' ],
327  'fields' => [ 'COUNT(*) as c' ],
328  'conds' => [ 'page_id=si_page', $match[0] ],
329  'options' => [],
330  'joins' => [],
331  ];
332 
333  $this->queryFeatures( $query );
334  $this->queryNamespaces( $query );
335 
336  return $query;
337  }
338 
347  public function update( $id, $title, $text ) {
348  $dbw = $this->lb->getConnectionRef( DB_PRIMARY );
349  $dbw->replace(
350  'searchindex',
351  'si_page',
352  [
353  'si_page' => $id,
354  'si_title' => $this->normalizeText( $title ),
355  'si_text' => $this->normalizeText( $text )
356  ],
357  __METHOD__
358  );
359  }
360 
368  public function updateTitle( $id, $title ) {
369  $dbw = $this->lb->getConnectionRef( DB_PRIMARY );
370  $dbw->update( 'searchindex',
371  [ 'si_title' => $this->normalizeText( $title ) ],
372  [ 'si_page' => $id ],
373  __METHOD__
374  );
375  }
376 
384  public function delete( $id, $title ) {
385  $dbw = $this->lb->getConnectionRef( DB_PRIMARY );
386  $dbw->delete( 'searchindex', [ 'si_page' => $id ], __METHOD__ );
387  }
388 
395  public function normalizeText( $string ) {
396  $out = parent::normalizeText( $string );
397 
398  // MySQL fulltext index doesn't grok utf-8, so we
399  // need to fold cases and convert to hex
400  $out = preg_replace_callback(
401  "/([\\xc0-\\xff][\\x80-\\xbf]*)/",
402  [ $this, 'stripForSearchCallback' ],
403  MediaWikiServices::getInstance()->getContentLanguage()->lc( $out ) );
404 
405  // And to add insult to injury, the default indexing
406  // ignores short words... Pad them so we can pass them
407  // through without reconfiguring the server...
408  $minLength = $this->minSearchLength();
409  if ( $minLength > 1 ) {
410  $n = $minLength - 1;
411  $out = preg_replace(
412  "/\b(\w{1,$n})\b/",
413  "$1u800",
414  $out );
415  }
416 
417  // Periods within things like hostnames and IP addresses
418  // are also important -- we want a search for "example.com"
419  // or "192.168.1.1" to work sanely.
420  // MySQL's search seems to ignore them, so you'd match on
421  // "example.wikipedia.com" and "192.168.83.1" as well.
422  $out = preg_replace(
423  "/(\w)\.(\w|\*)/u",
424  "$1u82e$2",
425  $out );
426 
427  return $out;
428  }
429 
437  protected function stripForSearchCallback( $matches ) {
438  return 'u8' . bin2hex( $matches[1] );
439  }
440 
447  protected function minSearchLength() {
448  if ( self::$mMinSearchLength === null ) {
449  $sql = "SHOW GLOBAL VARIABLES LIKE 'ft\\_min\\_word\\_len'";
450 
451  $dbr = $this->lb->getConnectionRef( DB_REPLICA );
452  $result = $dbr->query( $sql, __METHOD__ );
453  $row = $result->fetchObject();
454  $result->free();
455 
456  if ( $row && $row->Variable_name == 'ft_min_word_len' ) {
457  self::$mMinSearchLength = intval( $row->Value );
458  } else {
459  self::$mMinSearchLength = 0;
460  }
461  }
463  }
464 }
SearchEngine\$limit
int $limit
Definition: SearchEngine.php:47
SearchMySQL\limitResult
limitResult(&$query)
Add limit options.
Definition: SearchMySQL.php:257
SearchMySQL\queryNamespaces
queryNamespaces(&$query)
Add namespace conditions.
Definition: SearchMySQL.php:243
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:191
SearchMySQL
Search engine hook for MySQL.
Definition: SearchMySQL.php:33
SearchMySQL\minSearchLength
minSearchLength()
Check MySQL server's ft_min_word_len setting so we know if we need to pad short words....
Definition: SearchMySQL.php:447
SearchMySQL\getIndexField
getIndexField( $fulltext)
Picks which field to index on, depending on what type of query.
Definition: SearchMySQL.php:292
SearchEngine\$namespaces
int[] null $namespaces
Definition: SearchEngine.php:44
SearchMySQL\normalizeText
normalizeText( $string)
Converts some characters for MySQL's indexing to grok it correctly, and pads short words to overcome ...
Definition: SearchMySQL.php:395
SearchMySQL\queryMain
queryMain(&$query, $filteredTerm, $fulltext)
Get the base part of the search query.
Definition: SearchMySQL.php:304
SearchMySQL\queryFeatures
queryFeatures(&$query)
Add special conditions.
Definition: SearchMySQL.php:229
$dbr
$dbr
Definition: testCompression.php:54
SearchDatabase
Base search engine base class for database-backed searches.
Definition: SearchDatabase.php:33
SearchMySQL\stripForSearchCallback
stripForSearchCallback( $matches)
Armor a case-folded UTF-8 string to get through MySQL's fulltext search without being mucked up by fu...
Definition: SearchMySQL.php:437
SearchMySQL\updateTitle
updateTitle( $id, $title)
Update a search index record's title only.
Definition: SearchMySQL.php:368
SearchEngine\$offset
int $offset
Definition: SearchEngine.php:50
SearchMySQL\regexTerm
regexTerm( $string, $wildcard)
Definition: SearchMySQL.php:139
SearchMySQL\getQuery
getQuery( $filteredTerm, $fulltext)
Construct the SQL query to do the search.
Definition: SearchMySQL.php:270
$matches
$matches
Definition: NoLocalSettings.php:24
$title
$title
Definition: testCompression.php:38
SearchMySQL\getCountQuery
getCountQuery( $filteredTerm, $fulltext)
Definition: SearchMySQL.php:322
DB_REPLICA
const DB_REPLICA
Definition: defines.php:25
SearchMySQL\doSearchTitleInDB
doSearchTitleInDB( $term)
Perform a title-only search query and return a result set.
Definition: SearchMySQL.php:181
wfDebug
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
Definition: GlobalFunctions.php:894
SqlSearchResultSet
This class is used for different SQL-based search engines shipped with MediaWiki.
Definition: SqlSearchResultSet.php:9
SearchMySQL\legalSearchChars
legalSearchChars( $type=self::CHARS_ALL)
Get chars legal for search.
Definition: SearchMySQL.php:156
DB_PRIMARY
const DB_PRIMARY
Definition: defines.php:27
SearchMySQL\supports
supports( $feature)
Definition: SearchMySQL.php:215
SearchMySQL\doSearchTextInDB
doSearchTextInDB( $term)
Perform a full text search query and return a result set.
Definition: SearchMySQL.php:171
SearchMySQL\parseQuery
parseQuery( $filteredText, $fulltext)
Parse the user's query and transform it into two SQL fragments: a WHERE condition and an ORDER BY exp...
Definition: SearchMySQL.php:47
SearchMySQL\update
update( $id, $title, $text)
Create or update the search index record for the given page.
Definition: SearchMySQL.php:347
SearchMySQL\$strictMatching
$strictMatching
Definition: SearchMySQL.php:34
SearchMySQL\$mMinSearchLength
static $mMinSearchLength
Definition: SearchMySQL.php:36
SearchDatabase\filter
filter( $text)
Return a 'cleaned up' search string.
Definition: SearchDatabase.php:91
SearchMySQL\searchInternal
searchInternal( $term, $fulltext)
Definition: SearchMySQL.php:185
$type
$type
Definition: testCompression.php:52