MediaWiki  master
SearchSqlite.php
Go to the documentation of this file.
1 <?php
25 
35  private function fulltextSearchSupported() {
36  $dbr = $this->lb->getMaintenanceConnectionRef( DB_REPLICA );
37  $sql = (string)$dbr->selectField(
38  $dbr->addIdentifierQuotes( 'sqlite_master' ),
39  'sql',
40  [ 'tbl_name' => $dbr->tableName( 'searchindex', 'raw' ) ],
41  __METHOD__
42  );
43 
44  return ( stristr( $sql, 'fts' ) !== false );
45  }
46 
55  private function parseQuery( $filteredText, $fulltext ) {
56  $lc = $this->legalSearchChars( self::CHARS_NO_SYNTAX ); // Minus syntax chars (" and *)
57  $searchon = '';
58  $this->searchTerms = [];
59 
60  $m = [];
61  if ( preg_match_all( '/([-+<>~]?)(([' . $lc . ']+)(\*?)|"[^"]*")/',
62  $filteredText, $m, PREG_SET_ORDER ) ) {
63  foreach ( $m as $bits ) {
64  Wikimedia\suppressWarnings();
65  list( /* all */, $modifier, $term, $nonQuoted, $wildcard ) = $bits;
66  Wikimedia\restoreWarnings();
67 
68  if ( $nonQuoted != '' ) {
69  $term = $nonQuoted;
70  $quote = '';
71  } else {
72  $term = str_replace( '"', '', $term );
73  $quote = '"';
74  }
75 
76  if ( $searchon !== '' ) {
77  $searchon .= ' ';
78  }
79 
80  // Some languages such as Serbian store the input form in the search index,
81  // so we may need to search for matches in multiple writing system variants.
82  $convertedVariants = MediaWikiServices::getInstance()->getContentLanguage()->
83  autoConvertToAllVariants( $term );
84  if ( is_array( $convertedVariants ) ) {
85  $variants = array_unique( array_values( $convertedVariants ) );
86  } else {
87  $variants = [ $term ];
88  }
89 
90  // The low-level search index does some processing on input to work
91  // around problems with minimum lengths and encoding in MySQL's
92  // fulltext engine.
93  // For Chinese this also inserts spaces between adjacent Han characters.
94  $strippedVariants = array_map(
95  [ MediaWikiServices::getInstance()->getContentLanguage(),
96  'normalizeForSearch' ],
97  $variants );
98 
99  // Some languages such as Chinese force all variants to a canonical
100  // form when stripping to the low-level search index, so to be sure
101  // let's check our variants list for unique items after stripping.
102  $strippedVariants = array_unique( $strippedVariants );
103 
104  $searchon .= $modifier;
105  if ( count( $strippedVariants ) > 1 ) {
106  $searchon .= '(';
107  }
108  foreach ( $strippedVariants as $stripped ) {
109  if ( $nonQuoted && strpos( $stripped, ' ' ) !== false ) {
110  // Hack for Chinese: we need to toss in quotes for
111  // multiple-character phrases since normalizeForSearch()
112  // added spaces between them to make word breaks.
113  $stripped = '"' . trim( $stripped ) . '"';
114  }
115  $searchon .= "$quote$stripped$quote$wildcard ";
116  }
117  if ( count( $strippedVariants ) > 1 ) {
118  $searchon .= ')';
119  }
120 
121  // Match individual terms or quoted phrase in result highlighting...
122  // Note that variants will be introduced in a later stage for highlighting!
123  $regexp = $this->regexTerm( $term, $wildcard );
124  $this->searchTerms[] = $regexp;
125  }
126 
127  } else {
128  wfDebug( __METHOD__ . ": Can't understand search query '{$filteredText}'\n" );
129  }
130 
131  $dbr = $this->lb->getConnectionRef( DB_REPLICA );
132  $searchon = $dbr->addQuotes( $searchon );
133  $field = $this->getIndexField( $fulltext );
134 
135  return " $field MATCH $searchon ";
136  }
137 
138  private function regexTerm( $string, $wildcard ) {
139  $regex = preg_quote( $string, '/' );
140  if ( MediaWikiServices::getInstance()->getContentLanguage()->hasWordBreaks() ) {
141  if ( $wildcard ) {
142  // Don't cut off the final bit!
143  $regex = "\b$regex";
144  } else {
145  $regex = "\b$regex\b";
146  }
147  } else {
148  // For Chinese, words may legitimately abut other words in the text literal.
149  // Don't add \b boundary checks... note this could cause false positives
150  // for Latin chars.
151  }
152  return $regex;
153  }
154 
155  public function legalSearchChars( $type = self::CHARS_ALL ) {
156  $searchChars = parent::legalSearchChars( $type );
157  if ( $type === self::CHARS_ALL ) {
158  // " for phrase, * for wildcard
159  $searchChars = "\"*" . $searchChars;
160  }
161  return $searchChars;
162  }
163 
170  protected function doSearchTextInDB( $term ) {
171  return $this->searchInternal( $term, true );
172  }
173 
180  protected function doSearchTitleInDB( $term ) {
181  return $this->searchInternal( $term, false );
182  }
183 
184  protected function searchInternal( $term, $fulltext ) {
185  if ( !$this->fulltextSearchSupported() ) {
186  return null;
187  }
188 
189  $filteredTerm =
190  $this->filter( MediaWikiServices::getInstance()->getContentLanguage()->lc( $term ) );
191  $dbr = $this->lb->getConnectionRef( DB_REPLICA );
192  $resultSet = $dbr->query( $this->getQuery( $filteredTerm, $fulltext ) );
193 
194  $total = null;
195  $totalResult = $dbr->query( $this->getCountQuery( $filteredTerm, $fulltext ) );
196  $row = $totalResult->fetchObject();
197  if ( $row ) {
198  $total = intval( $row->c );
199  }
200  $totalResult->free();
201 
202  return new SqlSearchResultSet( $resultSet, $this->searchTerms, $total );
203  }
204 
209  private function queryNamespaces() {
210  if ( is_null( $this->namespaces ) ) {
211  return ''; # search all
212  }
213  if ( $this->namespaces === [] ) {
214  $namespaces = '0';
215  } else {
216  $dbr = $this->lb->getConnectionRef( DB_REPLICA );
217  $namespaces = $dbr->makeList( $this->namespaces );
218  }
219  return 'AND page_namespace IN (' . $namespaces . ')';
220  }
221 
227  private function limitResult( $sql ) {
228  $dbr = $this->lb->getConnectionRef( DB_REPLICA );
229 
230  return $dbr->limitResult( $sql, $this->limit, $this->offset );
231  }
232 
240  private function getQuery( $filteredTerm, $fulltext ) {
241  return $this->limitResult(
242  $this->queryMain( $filteredTerm, $fulltext ) . ' ' .
243  $this->queryNamespaces()
244  );
245  }
246 
252  private function getIndexField( $fulltext ) {
253  return $fulltext ? 'si_text' : 'si_title';
254  }
255 
263  private function queryMain( $filteredTerm, $fulltext ) {
264  $match = $this->parseQuery( $filteredTerm, $fulltext );
265  $dbr = $this->lb->getMaintenanceConnectionRef( DB_REPLICA );
266  $page = $dbr->tableName( 'page' );
267  $searchindex = $dbr->tableName( 'searchindex' );
268  return "SELECT $searchindex.rowid, page_namespace, page_title " .
269  "FROM $page,$searchindex " .
270  "WHERE page_id=$searchindex.rowid AND $match";
271  }
272 
273  private function getCountQuery( $filteredTerm, $fulltext ) {
274  $match = $this->parseQuery( $filteredTerm, $fulltext );
275  $dbr = $this->lb->getMaintenanceConnectionRef( DB_REPLICA );
276  $page = $dbr->tableName( 'page' );
277  $searchindex = $dbr->tableName( 'searchindex' );
278  return "SELECT COUNT(*) AS c " .
279  "FROM $page,$searchindex " .
280  "WHERE page_id=$searchindex.rowid AND $match " .
281  $this->queryNamespaces();
282  }
283 
292  public function update( $id, $title, $text ) {
293  if ( !$this->fulltextSearchSupported() ) {
294  return;
295  }
296  // @todo find a method to do it in a single request,
297  // couldn't do it so far due to typelessness of FTS3 tables.
298  $dbw = $this->lb->getConnectionRef( DB_MASTER );
299  $dbw->delete( 'searchindex', [ 'rowid' => $id ], __METHOD__ );
300  $dbw->insert( 'searchindex',
301  [
302  'rowid' => $id,
303  'si_title' => $title,
304  'si_text' => $text
305  ], __METHOD__ );
306  }
307 
315  public function updateTitle( $id, $title ) {
316  if ( !$this->fulltextSearchSupported() ) {
317  return;
318  }
319 
320  $dbw = $this->lb->getConnectionRef( DB_MASTER );
321  $dbw->update( 'searchindex',
322  [ 'si_title' => $title ],
323  [ 'rowid' => $id ],
324  __METHOD__ );
325  }
326 }
queryMain( $filteredTerm, $fulltext)
Get the base part of the search query.
doSearchTextInDB( $term)
Perform a full text search query and return a result set.
parseQuery( $filteredText, $fulltext)
Parse the user&#39;s query and transform it into an SQL fragment which will become part of a WHERE clause...
update( $id, $title, $text)
Create or update the search index record for the given page.
regexTerm( $string, $wildcard)
getCountQuery( $filteredTerm, $fulltext)
const DB_MASTER
Definition: defines.php:26
legalSearchChars( $type=self::CHARS_ALL)
limitResult( $sql)
Returns a query with limit for number of results set.
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
This class is used for different SQL-based search engines shipped with MediaWiki. ...
doSearchTitleInDB( $term)
Perform a title-only search query and return a result set.
updateTitle( $id, $title)
Update a search index record&#39;s title only.
int [] null $namespaces
fulltextSearchSupported()
Whether fulltext search is supported by current schema.
filter( $text)
Return a &#39;cleaned up&#39; search string.
queryNamespaces()
Return a partial WHERE clause to limit the search to the given namespaces.
Base search engine base class for database-backed searches.
const DB_REPLICA
Definition: defines.php:25
getIndexField( $fulltext)
Picks which field to index on, depending on what type of query.
Search engine hook for SQLite.
searchInternal( $term, $fulltext)
getQuery( $filteredTerm, $fulltext)
Construct the full SQL query to do the search.