MediaWiki REL1_34
SearchMySQL.php
Go to the documentation of this file.
1<?php
28
34 protected $strictMatching = true;
35
36 private static $mMinSearchLength;
37
47 private function parseQuery( $filteredText, $fulltext ) {
48 $lc = $this->legalSearchChars( self::CHARS_NO_SYNTAX ); // Minus syntax chars (" and *)
49 $searchon = '';
50 $this->searchTerms = [];
51
52 # @todo FIXME: This doesn't handle parenthetical expressions.
53 $m = [];
54 if ( preg_match_all( '/([-+<>~]?)(([' . $lc . ']+)(\*?)|"[^"]*")/',
55 $filteredText, $m, PREG_SET_ORDER ) ) {
56 foreach ( $m as $bits ) {
57 Wikimedia\suppressWarnings();
58 list( /* all */, $modifier, $term, $nonQuoted, $wildcard ) = $bits;
59 Wikimedia\restoreWarnings();
60
61 if ( $nonQuoted != '' ) {
62 $term = $nonQuoted;
63 $quote = '';
64 } else {
65 $term = str_replace( '"', '', $term );
66 $quote = '"';
67 }
68
69 if ( $searchon !== '' ) {
70 $searchon .= ' ';
71 }
72 if ( $this->strictMatching && ( $modifier == '' ) ) {
73 // If we leave this out, boolean op defaults to OR which is rarely helpful.
74 $modifier = '+';
75 }
76
77 // Some languages such as Serbian store the input form in the search index,
78 // so we may need to search for matches in multiple writing system variants.
79 $contLang = MediaWikiServices::getInstance()->getContentLanguage();
80 $convertedVariants = $contLang->autoConvertToAllVariants( $term );
81 if ( is_array( $convertedVariants ) ) {
82 $variants = array_unique( array_values( $convertedVariants ) );
83 } else {
84 $variants = [ $term ];
85 }
86
87 // The low-level search index does some processing on input to work
88 // around problems with minimum lengths and encoding in MySQL's
89 // fulltext engine.
90 // For Chinese this also inserts spaces between adjacent Han characters.
91 $strippedVariants = array_map( [ $contLang, 'normalizeForSearch' ], $variants );
92
93 // Some languages such as Chinese force all variants to a canonical
94 // form when stripping to the low-level search index, so to be sure
95 // let's check our variants list for unique items after stripping.
96 $strippedVariants = array_unique( $strippedVariants );
97
98 $searchon .= $modifier;
99 if ( count( $strippedVariants ) > 1 ) {
100 $searchon .= '(';
101 }
102 foreach ( $strippedVariants as $stripped ) {
103 $stripped = $this->normalizeText( $stripped );
104 if ( $nonQuoted && strpos( $stripped, ' ' ) !== false ) {
105 // Hack for Chinese: we need to toss in quotes for
106 // multiple-character phrases since normalizeForSearch()
107 // added spaces between them to make word breaks.
108 $stripped = '"' . trim( $stripped ) . '"';
109 }
110 $searchon .= "$quote$stripped$quote$wildcard ";
111 }
112 if ( count( $strippedVariants ) > 1 ) {
113 $searchon .= ')';
114 }
115
116 // Match individual terms or quoted phrase in result highlighting...
117 // Note that variants will be introduced in a later stage for highlighting!
118 $regexp = $this->regexTerm( $term, $wildcard );
119 $this->searchTerms[] = $regexp;
120 }
121 wfDebug( __METHOD__ . ": Would search with '$searchon'\n" );
122 wfDebug( __METHOD__ . ': Match with /' . implode( '|', $this->searchTerms ) . "/\n" );
123 } else {
124 wfDebug( __METHOD__ . ": Can't understand search query '{$filteredText}'\n" );
125 }
126
127 $dbr = $this->lb->getConnectionRef( DB_REPLICA );
128 $searchon = $dbr->addQuotes( $searchon );
129 $field = $this->getIndexField( $fulltext );
130 return [
131 " MATCH($field) AGAINST($searchon IN BOOLEAN MODE) ",
132 " MATCH($field) AGAINST($searchon IN NATURAL LANGUAGE MODE) DESC "
133 ];
134 }
135
136 private function regexTerm( $string, $wildcard ) {
137 $regex = preg_quote( $string, '/' );
138 if ( MediaWikiServices::getInstance()->getContentLanguage()->hasWordBreaks() ) {
139 if ( $wildcard ) {
140 // Don't cut off the final bit!
141 $regex = "\b$regex";
142 } else {
143 $regex = "\b$regex\b";
144 }
145 } else {
146 // For Chinese, words may legitimately abut other words in the text literal.
147 // Don't add \b boundary checks... note this could cause false positives
148 // for Latin chars.
149 }
150 return $regex;
151 }
152
153 public function legalSearchChars( $type = self::CHARS_ALL ) {
154 $searchChars = parent::legalSearchChars( $type );
155 if ( $type === self::CHARS_ALL ) {
156 // " for phrase, * for wildcard
157 $searchChars = "\"*" . $searchChars;
158 }
159 return $searchChars;
160 }
161
168 protected function doSearchTextInDB( $term ) {
169 return $this->searchInternal( $term, true );
170 }
171
178 protected function doSearchTitleInDB( $term ) {
179 return $this->searchInternal( $term, false );
180 }
181
182 protected function searchInternal( $term, $fulltext ) {
183 // This seems out of place, why is this called with empty term?
184 if ( trim( $term ) === '' ) {
185 return null;
186 }
187
188 $filteredTerm = $this->filter( $term );
189 $query = $this->getQuery( $filteredTerm, $fulltext );
190 $dbr = $this->lb->getConnectionRef( DB_REPLICA );
191 $resultSet = $dbr->select(
192 $query['tables'], $query['fields'], $query['conds'],
193 __METHOD__, $query['options'], $query['joins']
194 );
195
196 $total = null;
197 $query = $this->getCountQuery( $filteredTerm, $fulltext );
198 $totalResult = $dbr->select(
199 $query['tables'], $query['fields'], $query['conds'],
200 __METHOD__, $query['options'], $query['joins']
201 );
202
203 $row = $totalResult->fetchObject();
204 if ( $row ) {
205 $total = intval( $row->c );
206 }
207 $totalResult->free();
208
209 return new SqlSearchResultSet( $resultSet, $this->searchTerms, $total );
210 }
211
212 public function supports( $feature ) {
213 switch ( $feature ) {
214 case 'title-suffix-filter':
215 return true;
216 default:
217 return parent::supports( $feature );
218 }
219 }
220
226 protected function queryFeatures( &$query ) {
227 foreach ( $this->features as $feature => $value ) {
228 if ( $feature === 'title-suffix-filter' && $value ) {
229 $dbr = $this->lb->getConnectionRef( DB_REPLICA );
230 $query['conds'][] = 'page_title' . $dbr->buildLike( $dbr->anyString(), $value );
231 }
232 }
233 }
234
240 function queryNamespaces( &$query ) {
241 if ( is_array( $this->namespaces ) ) {
242 if ( count( $this->namespaces ) === 0 ) {
243 $this->namespaces[] = '0';
244 }
245 $query['conds']['page_namespace'] = $this->namespaces;
246 }
247 }
248
254 protected function limitResult( &$query ) {
255 $query['options']['LIMIT'] = $this->limit;
256 $query['options']['OFFSET'] = $this->offset;
257 }
258
267 private function getQuery( $filteredTerm, $fulltext ) {
268 $query = [
269 'tables' => [],
270 'fields' => [],
271 'conds' => [],
272 'options' => [],
273 'joins' => [],
274 ];
275
276 $this->queryMain( $query, $filteredTerm, $fulltext );
277 $this->queryFeatures( $query );
278 $this->queryNamespaces( $query );
279 $this->limitResult( $query );
280
281 return $query;
282 }
283
289 private function getIndexField( $fulltext ) {
290 return $fulltext ? 'si_text' : 'si_title';
291 }
292
301 private function queryMain( &$query, $filteredTerm, $fulltext ) {
302 $match = $this->parseQuery( $filteredTerm, $fulltext );
303 $query['tables'][] = 'page';
304 $query['tables'][] = 'searchindex';
305 $query['fields'][] = 'page_id';
306 $query['fields'][] = 'page_namespace';
307 $query['fields'][] = 'page_title';
308 $query['conds'][] = 'page_id=si_page';
309 $query['conds'][] = $match[0];
310 $query['options']['ORDER BY'] = $match[1];
311 }
312
319 private function getCountQuery( $filteredTerm, $fulltext ) {
320 $match = $this->parseQuery( $filteredTerm, $fulltext );
321
322 $query = [
323 'tables' => [ 'page', 'searchindex' ],
324 'fields' => [ 'COUNT(*) as c' ],
325 'conds' => [ 'page_id=si_page', $match[0] ],
326 'options' => [],
327 'joins' => [],
328 ];
329
330 $this->queryFeatures( $query );
331 $this->queryNamespaces( $query );
332
333 return $query;
334 }
335
344 function update( $id, $title, $text ) {
345 $dbw = $this->lb->getConnectionRef( DB_MASTER );
346 $dbw->replace( 'searchindex',
347 [ 'si_page' ],
348 [
349 'si_page' => $id,
350 'si_title' => $this->normalizeText( $title ),
351 'si_text' => $this->normalizeText( $text )
352 ], __METHOD__ );
353 }
354
362 function updateTitle( $id, $title ) {
363 $dbw = $this->lb->getConnectionRef( DB_MASTER );
364 $dbw->update( 'searchindex',
365 [ 'si_title' => $this->normalizeText( $title ) ],
366 [ 'si_page' => $id ],
367 __METHOD__
368 );
369 }
370
378 function delete( $id, $title ) {
379 $dbw = $this->lb->getConnectionRef( DB_MASTER );
380 $dbw->delete( 'searchindex', [ 'si_page' => $id ], __METHOD__ );
381 }
382
389 function normalizeText( $string ) {
390 $out = parent::normalizeText( $string );
391
392 // MySQL fulltext index doesn't grok utf-8, so we
393 // need to fold cases and convert to hex
394 $out = preg_replace_callback(
395 "/([\\xc0-\\xff][\\x80-\\xbf]*)/",
396 [ $this, 'stripForSearchCallback' ],
397 MediaWikiServices::getInstance()->getContentLanguage()->lc( $out ) );
398
399 // And to add insult to injury, the default indexing
400 // ignores short words... Pad them so we can pass them
401 // through without reconfiguring the server...
402 $minLength = $this->minSearchLength();
403 if ( $minLength > 1 ) {
404 $n = $minLength - 1;
405 $out = preg_replace(
406 "/\b(\w{1,$n})\b/",
407 "$1u800",
408 $out );
409 }
410
411 // Periods within things like hostnames and IP addresses
412 // are also important -- we want a search for "example.com"
413 // or "192.168.1.1" to work sanely.
414 // MySQL's search seems to ignore them, so you'd match on
415 // "example.wikipedia.com" and "192.168.83.1" as well.
416 $out = preg_replace(
417 "/(\w)\.(\w|\*)/u",
418 "$1u82e$2",
419 $out );
420
421 return $out;
422 }
423
431 protected function stripForSearchCallback( $matches ) {
432 return 'u8' . bin2hex( $matches[1] );
433 }
434
441 protected function minSearchLength() {
442 if ( is_null( self::$mMinSearchLength ) ) {
443 $sql = "SHOW GLOBAL VARIABLES LIKE 'ft\\_min\\_word\\_len'";
444
445 $dbr = $this->lb->getConnectionRef( DB_REPLICA );
446 $result = $dbr->query( $sql, __METHOD__ );
447 $row = $result->fetchObject();
448 $result->free();
449
450 if ( $row && $row->Variable_name == 'ft_min_word_len' ) {
451 self::$mMinSearchLength = intval( $row->Value );
452 } else {
453 self::$mMinSearchLength = 0;
454 }
455 }
457 }
458}
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
MediaWikiServices is the service locator for the application scope of MediaWiki.
Base search engine base class for database-backed searches.
filter( $text)
Return a 'cleaned up' search string.
int[] null $namespaces
Search engine hook for MySQL 4+.
parseQuery( $filteredText, $fulltext)
Parse the user's query and transform it into two SQL fragments: a WHERE condition and an ORDER BY exp...
queryMain(&$query, $filteredTerm, $fulltext)
Get the base part of the search query.
updateTitle( $id, $title)
Update a search index record's title only.
doSearchTitleInDB( $term)
Perform a title-only search query and return a result set.
getQuery( $filteredTerm, $fulltext)
Construct the SQL query to do the search.
stripForSearchCallback( $matches)
Armor a case-folded UTF-8 string to get through MySQL's fulltext search without being mucked up by fu...
getIndexField( $fulltext)
Picks which field to index on, depending on what type of query.
queryFeatures(&$query)
Add special conditions.
update( $id, $title, $text)
Create or update the search index record for the given page.
supports( $feature)
searchInternal( $term, $fulltext)
static $mMinSearchLength
limitResult(&$query)
Add limit options.
regexTerm( $string, $wildcard)
queryNamespaces(&$query)
Add namespace conditions.
legalSearchChars( $type=self::CHARS_ALL)
Get chars legal for search.
getCountQuery( $filteredTerm, $fulltext)
normalizeText( $string)
Converts some characters for MySQL's indexing to grok it correctly, and pads short words to overcome ...
minSearchLength()
Check MySQL server's ft_min_word_len setting so we know if we need to pad short words....
doSearchTextInDB( $term)
Perform a full text search query and return a result set.
This class is used for different SQL-based search engines shipped with MediaWiki.
const DB_REPLICA
Definition defines.php:25
const DB_MASTER
Definition defines.php:26