MediaWiki REL1_39
SearchSqlite.php
Go to the documentation of this file.
1<?php
25use Wikimedia\AtEase\AtEase;
26
36 private function fulltextSearchSupported() {
37 $dbr = $this->lb->getMaintenanceConnectionRef( DB_REPLICA );
38 $sql = (string)$dbr->selectField(
39 $dbr->addIdentifierQuotes( 'sqlite_master' ),
40 'sql',
41 [ 'tbl_name' => $dbr->tableName( 'searchindex', 'raw' ) ],
42 __METHOD__
43 );
44
45 return ( stristr( $sql, 'fts' ) !== false );
46 }
47
56 private function parseQuery( $filteredText, $fulltext ) {
57 $lc = $this->legalSearchChars( self::CHARS_NO_SYNTAX ); // Minus syntax chars (" and *)
58 $searchon = '';
59 $this->searchTerms = [];
60
61 $m = [];
62 if ( preg_match_all( '/([-+<>~]?)(([' . $lc . ']+)(\*?)|"[^"]*")/',
63 $filteredText, $m, PREG_SET_ORDER ) ) {
64 foreach ( $m as $bits ) {
65 AtEase::suppressWarnings();
66 list( /* all */, $modifier, $term, $nonQuoted, $wildcard ) = $bits;
67 AtEase::restoreWarnings();
68
69 if ( $nonQuoted != '' ) {
70 $term = $nonQuoted;
71 $quote = '';
72 } else {
73 $term = str_replace( '"', '', $term );
74 $quote = '"';
75 }
76
77 if ( $searchon !== '' ) {
78 $searchon .= ' ';
79 }
80
81 // Some languages such as Serbian store the input form in the search index,
82 // so we may need to search for matches in multiple writing system variants.
83
84 $converter = MediaWikiServices::getInstance()->getLanguageConverterFactory()
85 ->getLanguageConverter();
86 $convertedVariants = $converter->autoConvertToAllVariants( $term );
87 if ( is_array( $convertedVariants ) ) {
88 $variants = array_unique( array_values( $convertedVariants ) );
89 } else {
90 $variants = [ $term ];
91 }
92
93 // The low-level search index does some processing on input to work
94 // around problems with minimum lengths and encoding in MySQL's
95 // fulltext engine.
96 // For Chinese this also inserts spaces between adjacent Han characters.
97 $strippedVariants = array_map(
98 [ MediaWikiServices::getInstance()->getContentLanguage(),
99 'normalizeForSearch' ],
100 $variants );
101
102 // Some languages such as Chinese force all variants to a canonical
103 // form when stripping to the low-level search index, so to be sure
104 // let's check our variants list for unique items after stripping.
105 $strippedVariants = array_unique( $strippedVariants );
106
107 $searchon .= $modifier;
108 if ( count( $strippedVariants ) > 1 ) {
109 $searchon .= '(';
110 }
111 foreach ( $strippedVariants as $stripped ) {
112 if ( $nonQuoted && strpos( $stripped, ' ' ) !== false ) {
113 // Hack for Chinese: we need to toss in quotes for
114 // multiple-character phrases since normalizeForSearch()
115 // added spaces between them to make word breaks.
116 $stripped = '"' . trim( $stripped ) . '"';
117 }
118 $searchon .= "$quote$stripped$quote$wildcard ";
119 }
120 if ( count( $strippedVariants ) > 1 ) {
121 $searchon .= ')';
122 }
123
124 // Match individual terms or quoted phrase in result highlighting...
125 // Note that variants will be introduced in a later stage for highlighting!
126 $regexp = $this->regexTerm( $term, $wildcard );
127 $this->searchTerms[] = $regexp;
128 }
129
130 } else {
131 wfDebug( __METHOD__ . ": Can't understand search query '{$filteredText}'" );
132 }
133
134 $dbr = $this->lb->getConnectionRef( DB_REPLICA );
135 $searchon = $dbr->addQuotes( $searchon );
136 $field = $this->getIndexField( $fulltext );
137
138 return " $field MATCH $searchon ";
139 }
140
141 private function regexTerm( $string, $wildcard ) {
142 $regex = preg_quote( $string, '/' );
143 if ( MediaWikiServices::getInstance()->getContentLanguage()->hasWordBreaks() ) {
144 if ( $wildcard ) {
145 // Don't cut off the final bit!
146 $regex = "\b$regex";
147 } else {
148 $regex = "\b$regex\b";
149 }
150 } else {
151 // For Chinese, words may legitimately abut other words in the text literal.
152 // Don't add \b boundary checks... note this could cause false positives
153 // for Latin chars.
154 }
155 return $regex;
156 }
157
158 public function legalSearchChars( $type = self::CHARS_ALL ) {
159 $searchChars = parent::legalSearchChars( $type );
160 if ( $type === self::CHARS_ALL ) {
161 // " for phrase, * for wildcard
162 $searchChars = "\"*" . $searchChars;
163 }
164 return $searchChars;
165 }
166
173 protected function doSearchTextInDB( $term ) {
174 return $this->searchInternal( $term, true );
175 }
176
183 protected function doSearchTitleInDB( $term ) {
184 return $this->searchInternal( $term, false );
185 }
186
187 protected function searchInternal( $term, $fulltext ) {
188 if ( !$this->fulltextSearchSupported() ) {
189 return null;
190 }
191
192 $filteredTerm =
193 $this->filter( MediaWikiServices::getInstance()->getContentLanguage()->lc( $term ) );
194 $dbr = $this->lb->getConnectionRef( DB_REPLICA );
195 $resultSet = $dbr->query( $this->getQuery( $filteredTerm, $fulltext ), __METHOD__ );
196
197 $total = null;
198 $totalResult = $dbr->query( $this->getCountQuery( $filteredTerm, $fulltext ), __METHOD__ );
199 $row = $totalResult->fetchObject();
200 if ( $row ) {
201 $total = intval( $row->c );
202 }
203 $totalResult->free();
204
205 return new SqlSearchResultSet( $resultSet, $this->searchTerms, $total );
206 }
207
212 private function queryNamespaces() {
213 if ( $this->namespaces === null ) {
214 return ''; # search all
215 }
216 if ( $this->namespaces === [] ) {
218 } else {
219 $dbr = $this->lb->getConnectionRef( DB_REPLICA );
220 $namespaces = $dbr->makeList( $this->namespaces );
221 }
222 return 'AND page_namespace IN (' . $namespaces . ')';
223 }
224
230 private function limitResult( $sql ) {
231 $dbr = $this->lb->getConnectionRef( DB_REPLICA );
232
233 return $dbr->limitResult( $sql, $this->limit, $this->offset );
234 }
235
243 private function getQuery( $filteredTerm, $fulltext ) {
244 return $this->limitResult(
245 $this->queryMain( $filteredTerm, $fulltext ) . ' ' .
246 $this->queryNamespaces()
247 );
248 }
249
255 private function getIndexField( $fulltext ) {
256 return $fulltext ? 'si_text' : 'si_title';
257 }
258
266 private function queryMain( $filteredTerm, $fulltext ) {
267 $match = $this->parseQuery( $filteredTerm, $fulltext );
268 $dbr = $this->lb->getMaintenanceConnectionRef( DB_REPLICA );
269 $page = $dbr->tableName( 'page' );
270 $searchindex = $dbr->tableName( 'searchindex' );
271 return "SELECT $searchindex.rowid, page_namespace, page_title " .
272 "FROM $page,$searchindex " .
273 "WHERE page_id=$searchindex.rowid AND $match";
274 }
275
276 private function getCountQuery( $filteredTerm, $fulltext ) {
277 $match = $this->parseQuery( $filteredTerm, $fulltext );
278 $dbr = $this->lb->getMaintenanceConnectionRef( DB_REPLICA );
279 $page = $dbr->tableName( 'page' );
280 $searchindex = $dbr->tableName( 'searchindex' );
281 return "SELECT COUNT(*) AS c " .
282 "FROM $page,$searchindex " .
283 "WHERE page_id=$searchindex.rowid AND $match " .
284 $this->queryNamespaces();
285 }
286
295 public function update( $id, $title, $text ) {
296 if ( !$this->fulltextSearchSupported() ) {
297 return;
298 }
299 // @todo find a method to do it in a single request,
300 // couldn't do it so far due to typelessness of FTS3 tables.
301 $dbw = $this->lb->getConnectionRef( DB_PRIMARY );
302 $dbw->delete( 'searchindex', [ 'rowid' => $id ], __METHOD__ );
303 $dbw->insert( 'searchindex',
304 [
305 'rowid' => $id,
306 'si_title' => $title,
307 'si_text' => $text
308 ], __METHOD__ );
309 }
310
318 public function updateTitle( $id, $title ) {
319 if ( !$this->fulltextSearchSupported() ) {
320 return;
321 }
322
323 $dbw = $this->lb->getConnectionRef( DB_PRIMARY );
324 $dbw->update( 'searchindex',
325 [ 'si_title' => $title ],
326 [ 'rowid' => $id ],
327 __METHOD__ );
328 }
329}
const NS_MAIN
Definition Defines.php:64
wfDebug( $text, $dest='all', array $context=[])
Sends a line to the debug log if enabled or, optionally, to a comment in output.
Service locator for MediaWiki core services.
Base search engine base class for database-backed searches.
filter( $text)
Return a 'cleaned up' search string.
int[] null $namespaces
Search engine hook for SQLite.
searchInternal( $term, $fulltext)
update( $id, $title, $text)
Create or update the search index record for the given page.
updateTitle( $id, $title)
Update a search index record's title only.
doSearchTitleInDB( $term)
Perform a title-only search query and return a result set.
doSearchTextInDB( $term)
Perform a full text search query and return a result set.
legalSearchChars( $type=self::CHARS_ALL)
Get chars legal for search.
This class is used for different SQL-based search engines shipped with MediaWiki.
const DB_REPLICA
Definition defines.php:26
const DB_PRIMARY
Definition defines.php:28