MediaWiki  master
SearchSqlite.php
Go to the documentation of this file.
1 <?php
25 use Wikimedia\AtEase\AtEase;
26 
36  private function fulltextSearchSupported() {
37  $dbr = $this->lb->getMaintenanceConnectionRef( DB_REPLICA );
38  $sql = (string)$dbr->selectField(
39  $dbr->addIdentifierQuotes( 'sqlite_master' ),
40  'sql',
41  [ 'tbl_name' => $dbr->tableName( 'searchindex', 'raw' ) ],
42  __METHOD__
43  );
44 
45  return ( stristr( $sql, 'fts' ) !== false );
46  }
47 
56  private function parseQuery( $filteredText, $fulltext ) {
57  $lc = $this->legalSearchChars( self::CHARS_NO_SYNTAX ); // Minus syntax chars (" and *)
58  $searchon = '';
59  $this->searchTerms = [];
60 
61  $m = [];
62  if ( preg_match_all( '/([-+<>~]?)(([' . $lc . ']+)(\*?)|"[^"]*")/',
63  $filteredText, $m, PREG_SET_ORDER ) ) {
64  foreach ( $m as $bits ) {
65  AtEase::suppressWarnings();
66  [ /* all */, $modifier, $term, $nonQuoted, $wildcard ] = $bits;
67  AtEase::restoreWarnings();
68 
69  if ( $nonQuoted != '' ) {
70  $term = $nonQuoted;
71  $quote = '';
72  } else {
73  $term = str_replace( '"', '', $term );
74  $quote = '"';
75  }
76 
77  if ( $searchon !== '' ) {
78  $searchon .= ' ';
79  }
80 
81  // Some languages such as Serbian store the input form in the search index,
82  // so we may need to search for matches in multiple writing system variants.
83 
84  $converter = MediaWikiServices::getInstance()->getLanguageConverterFactory()
85  ->getLanguageConverter();
86  $convertedVariants = $converter->autoConvertToAllVariants( $term );
87  if ( is_array( $convertedVariants ) ) {
88  $variants = array_unique( array_values( $convertedVariants ) );
89  } else {
90  $variants = [ $term ];
91  }
92 
93  // The low-level search index does some processing on input to work
94  // around problems with minimum lengths and encoding in MySQL's
95  // fulltext engine.
96  // For Chinese this also inserts spaces between adjacent Han characters.
97  $strippedVariants = array_map(
98  [ MediaWikiServices::getInstance()->getContentLanguage(),
99  'normalizeForSearch' ],
100  $variants );
101 
102  // Some languages such as Chinese force all variants to a canonical
103  // form when stripping to the low-level search index, so to be sure
104  // let's check our variants list for unique items after stripping.
105  $strippedVariants = array_unique( $strippedVariants );
106 
107  $searchon .= $modifier;
108  if ( count( $strippedVariants ) > 1 ) {
109  $searchon .= '(';
110  }
111  foreach ( $strippedVariants as $stripped ) {
112  if ( $nonQuoted && strpos( $stripped, ' ' ) !== false ) {
113  // Hack for Chinese: we need to toss in quotes for
114  // multiple-character phrases since normalizeForSearch()
115  // added spaces between them to make word breaks.
116  $stripped = '"' . trim( $stripped ) . '"';
117  }
118  $searchon .= "$quote$stripped$quote$wildcard ";
119  }
120  if ( count( $strippedVariants ) > 1 ) {
121  $searchon .= ')';
122  }
123 
124  // Match individual terms or quoted phrase in result highlighting...
125  // Note that variants will be introduced in a later stage for highlighting!
126  $regexp = $this->regexTerm( $term, $wildcard );
127  $this->searchTerms[] = $regexp;
128  }
129 
130  } else {
131  wfDebug( __METHOD__ . ": Can't understand search query '{$filteredText}'" );
132  }
133 
134  $dbr = $this->lb->getConnectionRef( DB_REPLICA );
135  $searchon = $dbr->addQuotes( $searchon );
136  $field = $this->getIndexField( $fulltext );
137 
138  return " $field MATCH $searchon ";
139  }
140 
141  private function regexTerm( $string, $wildcard ) {
142  $regex = preg_quote( $string, '/' );
143  if ( MediaWikiServices::getInstance()->getContentLanguage()->hasWordBreaks() ) {
144  if ( $wildcard ) {
145  // Don't cut off the final bit!
146  $regex = "\b$regex";
147  } else {
148  $regex = "\b$regex\b";
149  }
150  } else {
151  // For Chinese, words may legitimately abut other words in the text literal.
152  // Don't add \b boundary checks... note this could cause false positives
153  // for Latin chars.
154  }
155  return $regex;
156  }
157 
158  public function legalSearchChars( $type = self::CHARS_ALL ) {
159  $searchChars = parent::legalSearchChars( $type );
160  if ( $type === self::CHARS_ALL ) {
161  // " for phrase, * for wildcard
162  $searchChars = "\"*" . $searchChars;
163  }
164  return $searchChars;
165  }
166 
173  protected function doSearchTextInDB( $term ) {
174  return $this->searchInternal( $term, true );
175  }
176 
183  protected function doSearchTitleInDB( $term ) {
184  return $this->searchInternal( $term, false );
185  }
186 
187  protected function searchInternal( $term, $fulltext ) {
188  if ( !$this->fulltextSearchSupported() ) {
189  return null;
190  }
191 
192  $filteredTerm =
193  $this->filter( MediaWikiServices::getInstance()->getContentLanguage()->lc( $term ) );
194  $dbr = $this->lb->getConnectionRef( DB_REPLICA );
195  $resultSet = $dbr->query( $this->getQuery( $filteredTerm, $fulltext ), __METHOD__ );
196 
197  $total = null;
198  $totalResult = $dbr->query( $this->getCountQuery( $filteredTerm, $fulltext ), __METHOD__ );
199  $row = $totalResult->fetchObject();
200  if ( $row ) {
201  $total = intval( $row->c );
202  }
203  $totalResult->free();
204 
205  return new SqlSearchResultSet( $resultSet, $this->searchTerms, $total );
206  }
207 
212  private function queryNamespaces() {
213  if ( $this->namespaces === null ) {
214  return ''; # search all
215  }
216  if ( $this->namespaces === [] ) {
218  } else {
219  $dbr = $this->lb->getConnectionRef( DB_REPLICA );
220  $namespaces = $dbr->makeList( $this->namespaces );
221  }
222  return 'AND page_namespace IN (' . $namespaces . ')';
223  }
224 
230  private function limitResult( $sql ) {
231  $dbr = $this->lb->getConnectionRef( DB_REPLICA );
232 
233  return $dbr->limitResult( $sql, $this->limit, $this->offset );
234  }
235 
243  private function getQuery( $filteredTerm, $fulltext ) {
244  return $this->limitResult(
245  $this->queryMain( $filteredTerm, $fulltext ) . ' ' .
246  $this->queryNamespaces()
247  );
248  }
249 
255  private function getIndexField( $fulltext ) {
256  return $fulltext ? 'si_text' : 'si_title';
257  }
258 
266  private function queryMain( $filteredTerm, $fulltext ) {
267  $match = $this->parseQuery( $filteredTerm, $fulltext );
268  $dbr = $this->lb->getMaintenanceConnectionRef( DB_REPLICA );
269  $page = $dbr->tableName( 'page' );
270  $searchindex = $dbr->tableName( 'searchindex' );
271  return "SELECT $searchindex.rowid, page_namespace, page_title " .
272  "FROM $page,$searchindex " .
273  "WHERE page_id=$searchindex.rowid AND $match";
274  }
275 
276  private function getCountQuery( $filteredTerm, $fulltext ) {
277  $match = $this->parseQuery( $filteredTerm, $fulltext );
278  $dbr = $this->lb->getMaintenanceConnectionRef( DB_REPLICA );
279  $page = $dbr->tableName( 'page' );
280  $searchindex = $dbr->tableName( 'searchindex' );
281  return "SELECT COUNT(*) AS c " .
282  "FROM $page,$searchindex " .
283  "WHERE page_id=$searchindex.rowid AND $match " .
284  $this->queryNamespaces();
285  }
286 
295  public function update( $id, $title, $text ) {
296  if ( !$this->fulltextSearchSupported() ) {
297  return;
298  }
299  // @todo find a method to do it in a single request,
300  // couldn't do it so far due to typelessness of FTS3 tables.
301  $dbw = $this->lb->getConnectionRef( DB_PRIMARY );
302  $dbw->delete( 'searchindex', [ 'rowid' => $id ], __METHOD__ );
303  $dbw->insert( 'searchindex',
304  [
305  'rowid' => $id,
306  'si_title' => $title,
307  'si_text' => $text
308  ], __METHOD__ );
309  }
310 
318  public function updateTitle( $id, $title ) {
319  if ( !$this->fulltextSearchSupported() ) {
320  return;
321  }
322 
323  $dbw = $this->lb->getConnectionRef( DB_PRIMARY );
324  $dbw->update( 'searchindex',
325  [ 'si_title' => $title ],
326  [ 'rowid' => $id ],
327  __METHOD__ );
328  }
329 }
const NS_MAIN
Definition: Defines.php:64
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
Service locator for MediaWiki core services.
Base search engine base class for database-backed searches.
filter( $text)
Return a 'cleaned up' search string.
int[] null $namespaces
Search engine hook for SQLite.
searchInternal( $term, $fulltext)
update( $id, $title, $text)
Create or update the search index record for the given page.
updateTitle( $id, $title)
Update a search index record's title only.
doSearchTitleInDB( $term)
Perform a title-only search query and return a result set.
doSearchTextInDB( $term)
Perform a full text search query and return a result set.
legalSearchChars( $type=self::CHARS_ALL)
Get chars legal for search.
This class is used for different SQL-based search engines shipped with MediaWiki.
const DB_REPLICA
Definition: defines.php:26
const DB_PRIMARY
Definition: defines.php:28