MediaWiki master
SearchMySQL.php
Go to the documentation of this file.
1<?php
28use Wikimedia\AtEase\AtEase;
32
38 protected $strictMatching = true;
39
40 private static $mMinSearchLength;
41
51 private function parseQuery( $filteredText, $fulltext ) {
52 $lc = $this->legalSearchChars( self::CHARS_NO_SYNTAX ); // Minus syntax chars (" and *)
53 $searchon = '';
54 $this->searchTerms = [];
55
56 # @todo FIXME: This doesn't handle parenthetical expressions.
57 $m = [];
58 if ( preg_match_all( '/([-+<>~]?)(([' . $lc . ']+)(\*?)|"[^"]*")/',
59 $filteredText, $m, PREG_SET_ORDER )
60 ) {
61 $services = MediaWikiServices::getInstance();
62 $contLang = $services->getContentLanguage();
63 $langConverter = $services->getLanguageConverterFactory()->getLanguageConverter( $contLang );
64 foreach ( $m as $bits ) {
65 AtEase::suppressWarnings();
66 [ /* all */, $modifier, $term, $nonQuoted, $wildcard ] = $bits;
67 AtEase::restoreWarnings();
68
69 if ( $nonQuoted != '' ) {
70 $term = $nonQuoted;
71 $quote = '';
72 } else {
73 $term = str_replace( '"', '', $term );
74 $quote = '"';
75 }
76
77 if ( $searchon !== '' ) {
78 $searchon .= ' ';
79 }
80 if ( $this->strictMatching && ( $modifier == '' ) ) {
81 // If we leave this out, boolean op defaults to OR which is rarely helpful.
82 $modifier = '+';
83 }
84
85 // Some languages such as Serbian store the input form in the search index,
86 // so we may need to search for matches in multiple writing system variants.
87 $convertedVariants = $langConverter->autoConvertToAllVariants( $term );
88 if ( is_array( $convertedVariants ) ) {
89 $variants = array_unique( array_values( $convertedVariants ) );
90 } else {
91 $variants = [ $term ];
92 }
93
94 // The low-level search index does some processing on input to work
95 // around problems with minimum lengths and encoding in MySQL's
96 // fulltext engine.
97 // For Chinese this also inserts spaces between adjacent Han characters.
98 $strippedVariants = array_map( [ $contLang, 'normalizeForSearch' ], $variants );
99
100 // Some languages such as Chinese force all variants to a canonical
101 // form when stripping to the low-level search index, so to be sure
102 // let's check our variants list for unique items after stripping.
103 $strippedVariants = array_unique( $strippedVariants );
104
105 $searchon .= $modifier;
106 if ( count( $strippedVariants ) > 1 ) {
107 $searchon .= '(';
108 }
109 foreach ( $strippedVariants as $stripped ) {
110 $stripped = $this->normalizeText( $stripped );
111 if ( $nonQuoted && strpos( $stripped, ' ' ) !== false ) {
112 // Hack for Chinese: we need to toss in quotes for
113 // multiple-character phrases since normalizeForSearch()
114 // added spaces between them to make word breaks.
115 $stripped = '"' . trim( $stripped ) . '"';
116 }
117 $searchon .= "$quote$stripped$quote$wildcard ";
118 }
119 if ( count( $strippedVariants ) > 1 ) {
120 $searchon .= ')';
121 }
122
123 // Match individual terms or quoted phrase in result highlighting...
124 // Note that variants will be introduced in a later stage for highlighting!
125 $regexp = $this->regexTerm( $term, $wildcard );
126 $this->searchTerms[] = $regexp;
127 }
128 wfDebug( __METHOD__ . ": Would search with '$searchon'" );
129 wfDebug( __METHOD__ . ': Match with /' . implode( '|', $this->searchTerms ) . "/" );
130 } else {
131 wfDebug( __METHOD__ . ": Can't understand search query '{$filteredText}'" );
132 }
133
134 $dbr = $this->dbProvider->getReplicaDatabase();
135 $searchon = $dbr->addQuotes( $searchon );
136 $field = $this->getIndexField( $fulltext );
137 return [
138 " MATCH($field) AGAINST($searchon IN BOOLEAN MODE) ",
139 " MATCH($field) AGAINST($searchon IN NATURAL LANGUAGE MODE) DESC "
140 ];
141 }
142
143 private function regexTerm( $string, $wildcard ) {
144 $regex = preg_quote( $string, '/' );
145 if ( MediaWikiServices::getInstance()->getContentLanguage()->hasWordBreaks() ) {
146 if ( $wildcard ) {
147 // Don't cut off the final bit!
148 $regex = "\b$regex";
149 } else {
150 $regex = "\b$regex\b";
151 }
152 } else {
153 // For Chinese, words may legitimately abut other words in the text literal.
154 // Don't add \b boundary checks... note this could cause false positives
155 // for Latin chars.
156 }
157 return $regex;
158 }
159
160 public function legalSearchChars( $type = self::CHARS_ALL ) {
161 $searchChars = parent::legalSearchChars( $type );
162 if ( $type === self::CHARS_ALL ) {
163 // " for phrase, * for wildcard
164 $searchChars = "\"*" . $searchChars;
165 }
166 return $searchChars;
167 }
168
175 protected function doSearchTextInDB( $term ) {
176 return $this->searchInternal( $term, true );
177 }
178
185 protected function doSearchTitleInDB( $term ) {
186 return $this->searchInternal( $term, false );
187 }
188
189 protected function searchInternal( $term, $fulltext ) {
190 // This seems out of place, why is this called with empty term?
191 if ( trim( $term ) === '' ) {
192 return null;
193 }
194
195 $filteredTerm = $this->filter( $term );
196 $queryBuilder = $this->getQueryBuilder( $filteredTerm, $fulltext );
197 $resultSet = $queryBuilder->caller( __METHOD__ )->fetchResultSet();
198
199 $total = null;
200 $queryBuilder = $this->getCountQueryBuilder( $filteredTerm, $fulltext );
201 $totalResult = $queryBuilder->caller( __METHOD__ )->fetchResultSet();
202
203 $row = $totalResult->fetchObject();
204 if ( $row ) {
205 $total = intval( $row->c );
206 }
207 $totalResult->free();
208
209 return new SqlSearchResultSet( $resultSet, $this->searchTerms, $total );
210 }
211
212 public function supports( $feature ) {
213 switch ( $feature ) {
214 case 'title-suffix-filter':
215 return true;
216 default:
217 return parent::supports( $feature );
218 }
219 }
220
226 protected function queryFeatures( SelectQueryBuilder $queryBuilder ) {
227 foreach ( $this->features as $feature => $value ) {
228 if ( $feature === 'title-suffix-filter' && $value ) {
229 $dbr = $this->dbProvider->getReplicaDatabase();
230 $queryBuilder->andWhere(
231 $dbr->expr( 'page_title', IExpression::LIKE, new LikeValue( $dbr->anyString(), $value ) )
232 );
233 }
234 }
235 }
236
242 private function queryNamespaces( $queryBuilder ) {
243 if ( is_array( $this->namespaces ) ) {
244 if ( count( $this->namespaces ) === 0 ) {
245 $this->namespaces[] = NS_MAIN;
246 }
247 $queryBuilder->andWhere( [ 'page_namespace' => $this->namespaces ] );
248 }
249 }
250
258 private function getQueryBuilder( $filteredTerm, $fulltext ): SelectQueryBuilder {
259 $queryBuilder = $this->dbProvider->getReplicaDatabase()->newSelectQueryBuilder();
260
261 $this->queryMain( $queryBuilder, $filteredTerm, $fulltext );
262 $this->queryFeatures( $queryBuilder );
263 $this->queryNamespaces( $queryBuilder );
264 $queryBuilder->limit( $this->limit )
265 ->offset( $this->offset );
266
267 return $queryBuilder;
268 }
269
275 private function getIndexField( $fulltext ) {
276 return $fulltext ? 'si_text' : 'si_title';
277 }
278
287 private function queryMain( SelectQueryBuilder $queryBuilder, $filteredTerm, $fulltext ) {
288 $match = $this->parseQuery( $filteredTerm, $fulltext );
289 $queryBuilder->select( [ 'page_id', 'page_namespace', 'page_title' ] )
290 ->from( 'page' )
291 ->join( 'searchindex', null, 'page_id=si_page' )
292 ->where( $match[0] )
293 ->orderBy( $match[1] );
294 }
295
302 private function getCountQueryBuilder( $filteredTerm, $fulltext ): SelectQueryBuilder {
303 $match = $this->parseQuery( $filteredTerm, $fulltext );
304 $queryBuilder = $this->dbProvider->getReplicaDatabase()->newSelectQueryBuilder()
305 ->select( [ 'c' => 'COUNT(*)' ] )
306 ->from( 'page' )
307 ->join( 'searchindex', null, 'page_id=si_page' )
308 ->where( $match[0] );
309
310 $this->queryFeatures( $queryBuilder );
311 $this->queryNamespaces( $queryBuilder );
312
313 return $queryBuilder;
314 }
315
324 public function update( $id, $title, $text ) {
325 $this->dbProvider->getPrimaryDatabase()->newReplaceQueryBuilder()
326 ->replaceInto( 'searchindex' )
327 ->uniqueIndexFields( [ 'si_page' ] )
328 ->rows( [
329 'si_page' => $id,
330 'si_title' => $this->normalizeText( $title ),
331 'si_text' => $this->normalizeText( $text )
332 ] )
333 ->caller( __METHOD__ )->execute();
334 }
335
343 public function updateTitle( $id, $title ) {
344 $this->dbProvider->getPrimaryDatabase()->newUpdateQueryBuilder()
345 ->update( 'searchindex' )
346 ->set( [ 'si_title' => $this->normalizeText( $title ) ] )
347 ->where( [ 'si_page' => $id ] )
348 ->caller( __METHOD__ )->execute();
349 }
350
358 public function delete( $id, $title ) {
359 $this->dbProvider->getPrimaryDatabase()->newDeleteQueryBuilder()
360 ->deleteFrom( 'searchindex' )
361 ->where( [ 'si_page' => $id ] )
362 ->caller( __METHOD__ )->execute();
363 }
364
371 public function normalizeText( $string ) {
372 $out = parent::normalizeText( $string );
373
374 // MySQL fulltext index doesn't grok utf-8, so we
375 // need to fold cases and convert to hex
376 $out = preg_replace_callback(
377 "/([\\xc0-\\xff][\\x80-\\xbf]*)/",
378 [ $this, 'stripForSearchCallback' ],
379 MediaWikiServices::getInstance()->getContentLanguage()->lc( $out ) );
380
381 // And to add insult to injury, the default indexing
382 // ignores short words... Pad them so we can pass them
383 // through without reconfiguring the server...
384 $minLength = $this->minSearchLength();
385 if ( $minLength > 1 ) {
386 $n = $minLength - 1;
387 $out = preg_replace(
388 "/\b(\w{1,$n})\b/",
389 "$1u800",
390 $out );
391 }
392
393 // Periods within things like hostnames and IP addresses
394 // are also important -- we want a search for "example.com"
395 // or "192.168.1.1" to work sensibly.
396 // MySQL's search seems to ignore them, so you'd match on
397 // "example.wikipedia.com" and "192.168.83.1" as well.
398 return preg_replace(
399 "/(\w)\.(\w|\*)/u",
400 "$1u82e$2",
401 $out
402 );
403 }
404
412 protected function stripForSearchCallback( $matches ) {
413 return 'u8' . bin2hex( $matches[1] );
414 }
415
422 protected function minSearchLength() {
423 if ( self::$mMinSearchLength === null ) {
424 $sql = "SHOW GLOBAL VARIABLES LIKE 'ft\\_min\\_word\\_len'";
425
426 $dbr = $this->dbProvider->getReplicaDatabase();
427 // phpcs:ignore MediaWiki.Usage.DbrQueryUsage.DbrQueryFound
428 $result = $dbr->query( $sql, __METHOD__ );
429 $row = $result->fetchObject();
430 $result->free();
431
432 if ( $row && $row->Variable_name == 'ft_min_word_len' ) {
433 self::$mMinSearchLength = intval( $row->Value );
434 } else {
435 self::$mMinSearchLength = 0;
436 }
437 }
438 return self::$mMinSearchLength;
439 }
440}
getQueryBuilder()
const NS_MAIN
Definition Defines.php:64
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
Service locator for MediaWiki core services.
Base search engine base class for database-backed searches.
filter( $text)
Return a 'cleaned up' search string.
Search engine hook for MySQL.
queryFeatures(SelectQueryBuilder $queryBuilder)
Add special conditions.
updateTitle( $id, $title)
Update a search index record's title only.
doSearchTitleInDB( $term)
Perform a title-only search query and return a result set.
stripForSearchCallback( $matches)
Armor a case-folded UTF-8 string to get through MySQL's fulltext search without being mucked up by fu...
update( $id, $title, $text)
Create or update the search index record for the given page.
supports( $feature)
searchInternal( $term, $fulltext)
legalSearchChars( $type=self::CHARS_ALL)
Get chars legal for search.
normalizeText( $string)
Converts some characters for MySQL's indexing to grok it correctly, and pads short words to overcome ...
minSearchLength()
Check MySQL server's ft_min_word_len setting so we know if we need to pad short words....
doSearchTextInDB( $term)
Perform a full text search query and return a result set.
This class is used for different SQL-based search engines shipped with MediaWiki.
Content of like value.
Definition LikeValue.php:14
Build SELECT queries with a fluent interface.
limit( $limit)
Set the query limit.
andWhere( $conds)
Add conditions to the query.
select( $fields)
Add a field or an array of fields to the query.
caller( $fname)
Set the method name to be included in an SQL comment.