MediaWiki  master
SearchMySQL.php
Go to the documentation of this file.
1 <?php
28 use Wikimedia\AtEase\AtEase;
30 
35 class SearchMySQL extends SearchDatabase {
36  protected $strictMatching = true;
37 
38  private static $mMinSearchLength;
39 
49  private function parseQuery( $filteredText, $fulltext ) {
50  $lc = $this->legalSearchChars( self::CHARS_NO_SYNTAX ); // Minus syntax chars (" and *)
51  $searchon = '';
52  $this->searchTerms = [];
53 
54  # @todo FIXME: This doesn't handle parenthetical expressions.
55  $m = [];
56  if ( preg_match_all( '/([-+<>~]?)(([' . $lc . ']+)(\*?)|"[^"]*")/',
57  $filteredText, $m, PREG_SET_ORDER )
58  ) {
59  $services = MediaWikiServices::getInstance();
60  $contLang = $services->getContentLanguage();
61  $langConverter = $services->getLanguageConverterFactory()->getLanguageConverter( $contLang );
62  foreach ( $m as $bits ) {
63  AtEase::suppressWarnings();
64  [ /* all */, $modifier, $term, $nonQuoted, $wildcard ] = $bits;
65  AtEase::restoreWarnings();
66 
67  if ( $nonQuoted != '' ) {
68  $term = $nonQuoted;
69  $quote = '';
70  } else {
71  $term = str_replace( '"', '', $term );
72  $quote = '"';
73  }
74 
75  if ( $searchon !== '' ) {
76  $searchon .= ' ';
77  }
78  if ( $this->strictMatching && ( $modifier == '' ) ) {
79  // If we leave this out, boolean op defaults to OR which is rarely helpful.
80  $modifier = '+';
81  }
82 
83  // Some languages such as Serbian store the input form in the search index,
84  // so we may need to search for matches in multiple writing system variants.
85  $convertedVariants = $langConverter->autoConvertToAllVariants( $term );
86  if ( is_array( $convertedVariants ) ) {
87  $variants = array_unique( array_values( $convertedVariants ) );
88  } else {
89  $variants = [ $term ];
90  }
91 
92  // The low-level search index does some processing on input to work
93  // around problems with minimum lengths and encoding in MySQL's
94  // fulltext engine.
95  // For Chinese this also inserts spaces between adjacent Han characters.
96  $strippedVariants = array_map( [ $contLang, 'normalizeForSearch' ], $variants );
97 
98  // Some languages such as Chinese force all variants to a canonical
99  // form when stripping to the low-level search index, so to be sure
100  // let's check our variants list for unique items after stripping.
101  $strippedVariants = array_unique( $strippedVariants );
102 
103  $searchon .= $modifier;
104  if ( count( $strippedVariants ) > 1 ) {
105  $searchon .= '(';
106  }
107  foreach ( $strippedVariants as $stripped ) {
108  $stripped = $this->normalizeText( $stripped );
109  if ( $nonQuoted && strpos( $stripped, ' ' ) !== false ) {
110  // Hack for Chinese: we need to toss in quotes for
111  // multiple-character phrases since normalizeForSearch()
112  // added spaces between them to make word breaks.
113  $stripped = '"' . trim( $stripped ) . '"';
114  }
115  $searchon .= "$quote$stripped$quote$wildcard ";
116  }
117  if ( count( $strippedVariants ) > 1 ) {
118  $searchon .= ')';
119  }
120 
121  // Match individual terms or quoted phrase in result highlighting...
122  // Note that variants will be introduced in a later stage for highlighting!
123  $regexp = $this->regexTerm( $term, $wildcard );
124  $this->searchTerms[] = $regexp;
125  }
126  wfDebug( __METHOD__ . ": Would search with '$searchon'" );
127  wfDebug( __METHOD__ . ': Match with /' . implode( '|', $this->searchTerms ) . "/" );
128  } else {
129  wfDebug( __METHOD__ . ": Can't understand search query '{$filteredText}'" );
130  }
131 
132  $dbr = $this->dbProvider->getReplicaDatabase();
133  $searchon = $dbr->addQuotes( $searchon );
134  $field = $this->getIndexField( $fulltext );
135  return [
136  " MATCH($field) AGAINST($searchon IN BOOLEAN MODE) ",
137  " MATCH($field) AGAINST($searchon IN NATURAL LANGUAGE MODE) DESC "
138  ];
139  }
140 
141  private function regexTerm( $string, $wildcard ) {
142  $regex = preg_quote( $string, '/' );
143  if ( MediaWikiServices::getInstance()->getContentLanguage()->hasWordBreaks() ) {
144  if ( $wildcard ) {
145  // Don't cut off the final bit!
146  $regex = "\b$regex";
147  } else {
148  $regex = "\b$regex\b";
149  }
150  } else {
151  // For Chinese, words may legitimately abut other words in the text literal.
152  // Don't add \b boundary checks... note this could cause false positives
153  // for Latin chars.
154  }
155  return $regex;
156  }
157 
158  public function legalSearchChars( $type = self::CHARS_ALL ) {
159  $searchChars = parent::legalSearchChars( $type );
160  if ( $type === self::CHARS_ALL ) {
161  // " for phrase, * for wildcard
162  $searchChars = "\"*" . $searchChars;
163  }
164  return $searchChars;
165  }
166 
173  protected function doSearchTextInDB( $term ) {
174  return $this->searchInternal( $term, true );
175  }
176 
183  protected function doSearchTitleInDB( $term ) {
184  return $this->searchInternal( $term, false );
185  }
186 
187  protected function searchInternal( $term, $fulltext ) {
188  // This seems out of place, why is this called with empty term?
189  if ( trim( $term ) === '' ) {
190  return null;
191  }
192 
193  $filteredTerm = $this->filter( $term );
194  $queryBuilder = $this->getQueryBuilder( $filteredTerm, $fulltext );
195  $resultSet = $queryBuilder->caller( __METHOD__ )->fetchResultSet();
196 
197  $total = null;
198  $queryBuilder = $this->getCountQueryBuilder( $filteredTerm, $fulltext );
199  $totalResult = $queryBuilder->caller( __METHOD__ )->fetchResultSet();
200 
201  $row = $totalResult->fetchObject();
202  if ( $row ) {
203  $total = intval( $row->c );
204  }
205  $totalResult->free();
206 
207  return new SqlSearchResultSet( $resultSet, $this->searchTerms, $total );
208  }
209 
210  public function supports( $feature ) {
211  switch ( $feature ) {
212  case 'title-suffix-filter':
213  return true;
214  default:
215  return parent::supports( $feature );
216  }
217  }
218 
224  protected function queryFeatures( SelectQueryBuilder $queryBuilder ) {
225  foreach ( $this->features as $feature => $value ) {
226  if ( $feature === 'title-suffix-filter' && $value ) {
227  $dbr = $this->dbProvider->getReplicaDatabase();
228  $queryBuilder->andWhere( 'page_title' . $dbr->buildLike( $dbr->anyString(), $value ) );
229  }
230  }
231  }
232 
238  private function queryNamespaces( $queryBuilder ) {
239  if ( is_array( $this->namespaces ) ) {
240  if ( count( $this->namespaces ) === 0 ) {
241  $this->namespaces[] = NS_MAIN;
242  }
243  $queryBuilder->andWhere( [ 'page_namespace' => $this->namespaces ] );
244  }
245  }
246 
254  private function getQueryBuilder( $filteredTerm, $fulltext ): SelectQueryBuilder {
255  $queryBuilder = $this->dbProvider->getReplicaDatabase()->newSelectQueryBuilder();
256 
257  $this->queryMain( $queryBuilder, $filteredTerm, $fulltext );
258  $this->queryFeatures( $queryBuilder );
259  $this->queryNamespaces( $queryBuilder );
260  $queryBuilder->limit( $this->limit )
261  ->offset( $this->offset );
262 
263  return $queryBuilder;
264  }
265 
271  private function getIndexField( $fulltext ) {
272  return $fulltext ? 'si_text' : 'si_title';
273  }
274 
283  private function queryMain( SelectQueryBuilder $queryBuilder, $filteredTerm, $fulltext ) {
284  $match = $this->parseQuery( $filteredTerm, $fulltext );
285  $queryBuilder->select( [ 'page_id', 'page_namespace', 'page_title' ] )
286  ->from( 'page' )
287  ->join( 'searchindex', null, 'page_id=si_page' )
288  ->where( $match[0] )
289  ->orderBy( $match[1] );
290  }
291 
298  private function getCountQueryBuilder( $filteredTerm, $fulltext ): SelectQueryBuilder {
299  $match = $this->parseQuery( $filteredTerm, $fulltext );
300  $queryBuilder = $this->dbProvider->getReplicaDatabase()->newSelectQueryBuilder()
301  ->select( [ 'c' => 'COUNT(*)' ] )
302  ->from( 'page' )
303  ->join( 'searchindex', null, 'page_id=si_page' )
304  ->where( $match[0] );
305 
306  $this->queryFeatures( $queryBuilder );
307  $this->queryNamespaces( $queryBuilder );
308 
309  return $queryBuilder;
310  }
311 
320  public function update( $id, $title, $text ) {
321  $this->dbProvider->getPrimaryDatabase()->newReplaceQueryBuilder()
322  ->replaceInto( 'searchindex' )
323  ->uniqueIndexFields( [ 'si_page' ] )
324  ->rows( [
325  'si_page' => $id,
326  'si_title' => $this->normalizeText( $title ),
327  'si_text' => $this->normalizeText( $text )
328  ] )
329  ->caller( __METHOD__ )->execute();
330  }
331 
339  public function updateTitle( $id, $title ) {
340  $this->dbProvider->getPrimaryDatabase()->newUpdateQueryBuilder()
341  ->update( 'searchindex' )
342  ->set( [ 'si_title' => $this->normalizeText( $title ) ] )
343  ->where( [ 'si_page' => $id ] )
344  ->caller( __METHOD__ )->execute();
345  }
346 
354  public function delete( $id, $title ) {
355  $this->dbProvider->getPrimaryDatabase()->newDeleteQueryBuilder()
356  ->deleteFrom( 'searchindex' )
357  ->where( [ 'si_page' => $id ] )
358  ->caller( __METHOD__ )->execute();
359  }
360 
367  public function normalizeText( $string ) {
368  $out = parent::normalizeText( $string );
369 
370  // MySQL fulltext index doesn't grok utf-8, so we
371  // need to fold cases and convert to hex
372  $out = preg_replace_callback(
373  "/([\\xc0-\\xff][\\x80-\\xbf]*)/",
374  [ $this, 'stripForSearchCallback' ],
375  MediaWikiServices::getInstance()->getContentLanguage()->lc( $out ) );
376 
377  // And to add insult to injury, the default indexing
378  // ignores short words... Pad them so we can pass them
379  // through without reconfiguring the server...
380  $minLength = $this->minSearchLength();
381  if ( $minLength > 1 ) {
382  $n = $minLength - 1;
383  $out = preg_replace(
384  "/\b(\w{1,$n})\b/",
385  "$1u800",
386  $out );
387  }
388 
389  // Periods within things like hostnames and IP addresses
390  // are also important -- we want a search for "example.com"
391  // or "192.168.1.1" to work sensibly.
392  // MySQL's search seems to ignore them, so you'd match on
393  // "example.wikipedia.com" and "192.168.83.1" as well.
394  return preg_replace(
395  "/(\w)\.(\w|\*)/u",
396  "$1u82e$2",
397  $out
398  );
399  }
400 
408  protected function stripForSearchCallback( $matches ) {
409  return 'u8' . bin2hex( $matches[1] );
410  }
411 
418  protected function minSearchLength() {
419  if ( self::$mMinSearchLength === null ) {
420  $sql = "SHOW GLOBAL VARIABLES LIKE 'ft\\_min\\_word\\_len'";
421 
422  $dbr = $this->dbProvider->getReplicaDatabase();
423  // phpcs:ignore MediaWiki.Usage.DbrQueryUsage.DbrQueryFound
424  $result = $dbr->query( $sql, __METHOD__ );
425  $row = $result->fetchObject();
426  $result->free();
427 
428  if ( $row && $row->Variable_name == 'ft_min_word_len' ) {
429  self::$mMinSearchLength = intval( $row->Value );
430  } else {
431  self::$mMinSearchLength = 0;
432  }
433  }
434  return self::$mMinSearchLength;
435  }
436 }
const NS_MAIN
Definition: Defines.php:64
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
$matches
Service locator for MediaWiki core services.
Base search engine base class for database-backed searches.
filter( $text)
Return a 'cleaned up' search string.
Search engine hook for MySQL.
Definition: SearchMySQL.php:35
queryFeatures(SelectQueryBuilder $queryBuilder)
Add special conditions.
updateTitle( $id, $title)
Update a search index record's title only.
doSearchTitleInDB( $term)
Perform a title-only search query and return a result set.
stripForSearchCallback( $matches)
Armor a case-folded UTF-8 string to get through MySQL's fulltext search without being mucked up by fu...
update( $id, $title, $text)
Create or update the search index record for the given page.
supports( $feature)
searchInternal( $term, $fulltext)
legalSearchChars( $type=self::CHARS_ALL)
Get chars legal for search.
normalizeText( $string)
Converts some characters for MySQL's indexing to grok it correctly, and pads short words to overcome ...
minSearchLength()
Check MySQL server's ft_min_word_len setting so we know if we need to pad short words....
doSearchTextInDB( $term)
Perform a full text search query and return a result set.
This class is used for different SQL-based search engines shipped with MediaWiki.
Build SELECT queries with a fluent interface.
limit( $limit)
Set the query limit.
andWhere( $conds)
Add conditions to the query.
select( $fields)
Add a field or an array of fields to the query.
caller( $fname)
Set the method name to be included in an SQL comment.