Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
87.06% |
74 / 85 |
|
63.64% |
7 / 11 |
CRAP | |
0.00% |
0 / 1 |
| ParserOutputPageProperties | |
87.06% |
74 / 85 |
|
63.64% |
7 / 11 |
33.08 | |
0.00% |
0 / 1 |
| __construct | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| initialize | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| finishInitializeBatch | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| finalize | |
0.00% |
0 / 2 |
|
0.00% |
0 / 1 |
2 | |||
| finalizeReal | |
81.58% |
31 / 38 |
|
0.00% |
0 / 1 |
6.23 | |||
| extractDisplayTitle | |
100.00% |
24 / 24 |
|
100.00% |
1 / 1 |
10 | |||
| isSameString | |
100.00% |
3 / 3 |
|
100.00% |
1 / 1 |
1 | |||
| fixAndFlagInvalidUTF8InSource | |
100.00% |
9 / 9 |
|
100.00% |
1 / 1 |
3 | |||
| flagProblem | |
100.00% |
2 / 2 |
|
100.00% |
1 / 1 |
1 | |||
| truncateFileTextContent | |
100.00% |
3 / 3 |
|
100.00% |
1 / 1 |
4 | |||
| truncateFileContent | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
2 | |||
| 1 | <?php |
| 2 | |
| 3 | namespace CirrusSearch\BuildDocument; |
| 4 | |
| 5 | use CirrusSearch\CirrusSearch; |
| 6 | use CirrusSearch\Search\CirrusIndexField; |
| 7 | use CirrusSearch\SearchConfig; |
| 8 | use Elastica\Document; |
| 9 | use MediaWiki\Logger\LoggerFactory; |
| 10 | use MediaWiki\MediaWikiServices; |
| 11 | use MediaWiki\Page\WikiPage; |
| 12 | use MediaWiki\Parser\ParserOutput; |
| 13 | use MediaWiki\Parser\Sanitizer; |
| 14 | use MediaWiki\Revision\BadRevisionException; |
| 15 | use MediaWiki\Revision\RevisionRecord; |
| 16 | use MediaWiki\Title\Title; |
| 17 | use Wikimedia\LightweightObjectStore\ExpirationAwareness; |
| 18 | |
| 19 | /** |
| 20 | * Extract searchable properties from the MediaWiki ParserOutput |
| 21 | */ |
| 22 | class ParserOutputPageProperties implements PagePropertyBuilder { |
| 23 | /** @var SearchConfig */ |
| 24 | private $config; |
| 25 | |
| 26 | public function __construct( SearchConfig $config ) { |
| 27 | $this->config = $config; |
| 28 | } |
| 29 | |
| 30 | /** |
| 31 | * {@inheritDoc} |
| 32 | */ |
| 33 | public function initialize( Document $doc, WikiPage $page, RevisionRecord $revision ): void { |
| 34 | // NOOP |
| 35 | } |
| 36 | |
| 37 | /** |
| 38 | * {@inheritDoc} |
| 39 | */ |
| 40 | public function finishInitializeBatch(): void { |
| 41 | // NOOP |
| 42 | } |
| 43 | |
| 44 | /** |
| 45 | * {@inheritDoc} |
| 46 | */ |
| 47 | public function finalize( Document $doc, Title $title, RevisionRecord $revision ): void { |
| 48 | $page = MediaWikiServices::getInstance()->getWikiPageFactory()->newFromTitle( $title ); |
| 49 | $this->finalizeReal( $doc, $page, new CirrusSearch, $revision ); |
| 50 | } |
| 51 | |
| 52 | /** |
| 53 | * Visible for testing. Much simpler to test with all objects resolved. |
| 54 | * |
| 55 | * @param Document $doc Document to finalize |
| 56 | * @param WikiPage $page WikiPage to scope operation to |
| 57 | * @param CirrusSearch $engine SearchEngine implementation |
| 58 | * @param RevisionRecord $revision The page revision to use |
| 59 | * @throws BuildDocumentException |
| 60 | */ |
| 61 | public function finalizeReal( |
| 62 | Document $doc, |
| 63 | WikiPage $page, |
| 64 | CirrusSearch $engine, |
| 65 | RevisionRecord $revision |
| 66 | ): void { |
| 67 | $wanCache = MediaWikiServices::getInstance()->getMainWANObjectCache(); |
| 68 | $cacheKey = $wanCache->makeKey( |
| 69 | 'CirrusSearchParserOutputPageProperties', |
| 70 | $page->getId(), |
| 71 | $revision->getId(), |
| 72 | $page->getTouched(), |
| 73 | 'v2' |
| 74 | ); |
| 75 | |
| 76 | // We are having problems with low hit rates, but haven't been able to |
| 77 | // track down why that is. Log a sample of keys so we can evaluate if |
| 78 | // the problem is that $page->getTouched() is changing between |
| 79 | // invocations. -- eb 2024 july 9 |
| 80 | if ( $page->getId() % 1000 === 0 ) { |
| 81 | LoggerFactory::getInstance( 'CirrusSearch' )->debug( |
| 82 | 'Sampling of CirrusSearchParserOutputPageProperties cache keys: {cache_key}', |
| 83 | [ |
| 84 | 'cache_key' => $cacheKey, |
| 85 | 'revision_id' => $revision->getId(), |
| 86 | 'page_id' => $page->getId(), |
| 87 | ] ); |
| 88 | } |
| 89 | |
| 90 | $fieldContent = $wanCache->getWithSetCallback( |
| 91 | $cacheKey, |
| 92 | ExpirationAwareness::TTL_HOUR * 6, |
| 93 | function () use ( $page, $revision, $engine ) { |
| 94 | $contentHandler = $page->getContentHandler(); |
| 95 | // TODO: Should see if we can change content handler api to avoid |
| 96 | // the WikiPage god object, but currently parser cache is still |
| 97 | // tied to WikiPage as well. |
| 98 | try { |
| 99 | $output = $contentHandler->getParserOutputForIndexing( $page, null, $revision ); |
| 100 | } catch ( BadRevisionException $e ) { |
| 101 | // The revision is corrupted in the db and has been marked as permanently missing, |
| 102 | //we can't do much about it so flag that page as broken in CirrusSearch. |
| 103 | return self::flagProblem( [], 'CirrusSearchBadRevision' ); |
| 104 | } |
| 105 | |
| 106 | if ( !$output ) { |
| 107 | throw new BuildDocumentException( "ParserOutput cannot be obtained." ); |
| 108 | } |
| 109 | |
| 110 | $fieldContent = $contentHandler->getDataForSearchIndex( $page, $output, $engine, $revision ); |
| 111 | $fieldContent['display_title'] = self::extractDisplayTitle( $page->getTitle(), $output ); |
| 112 | return self::fixAndFlagInvalidUTF8InSource( $fieldContent, $page->getId() ); |
| 113 | } |
| 114 | ); |
| 115 | |
| 116 | $fieldContent = $this->truncateFileContent( $fieldContent ); |
| 117 | $fieldDefinitions = $engine->getSearchIndexFields(); |
| 118 | foreach ( $fieldContent as $field => $fieldData ) { |
| 119 | $doc->set( $field, $fieldData ); |
| 120 | if ( isset( $fieldDefinitions[$field] ) ) { |
| 121 | $hints = $fieldDefinitions[$field]->getEngineHints( $engine ); |
| 122 | CirrusIndexField::addIndexingHints( $doc, $field, $hints ); |
| 123 | } |
| 124 | } |
| 125 | } |
| 126 | |
| 127 | /** |
| 128 | * @param Title $title |
| 129 | * @param ParserOutput $output |
| 130 | * @return string|null |
| 131 | */ |
| 132 | private static function extractDisplayTitle( Title $title, ParserOutput $output ): ?string { |
| 133 | $titleText = $title->getText(); |
| 134 | $titlePrefixedText = $title->getPrefixedText(); |
| 135 | |
| 136 | $raw = $output->getDisplayTitle(); |
| 137 | if ( $raw === false ) { |
| 138 | return null; |
| 139 | } |
| 140 | $clean = Sanitizer::stripAllTags( $raw ); |
| 141 | // Only index display titles that differ from the normal title |
| 142 | if ( self::isSameString( $clean, $titleText ) || |
| 143 | self::isSameString( $clean, $titlePrefixedText ) |
| 144 | ) { |
| 145 | return null; |
| 146 | } |
| 147 | if ( $title->getNamespace() === 0 || strpos( $clean, ':' ) === false ) { |
| 148 | return $clean; |
| 149 | } |
| 150 | // There is no official way that namespaces work in display title, it |
| 151 | // is an arbitrary string. Even so some use cases, such as the |
| 152 | // Translate extension, will translate the namespace as well. Here |
| 153 | // `Help:foo` will have a display title of `Aide:bar`. If we were to |
| 154 | // simply index as is the autocomplete and near matcher would see |
| 155 | // Help:Aide:bar, which doesn't seem particularly useful. |
| 156 | // The strategy here is to see if the portion before the : is a valid namespace |
| 157 | // in either the language of the wiki or the language of the page. If it is |
| 158 | // then we strip it from the display title. |
| 159 | [ $maybeNs, $maybeDisplayTitle ] = explode( ':', $clean, 2 ); |
| 160 | $cleanTitle = Title::newFromText( $clean ); |
| 161 | if ( $cleanTitle === null ) { |
| 162 | // The title is invalid, we cannot extract the ns prefix |
| 163 | return $clean; |
| 164 | } |
| 165 | if ( $cleanTitle->getNamespace() == $title->getNamespace() ) { |
| 166 | // While it doesn't really matter, $cleanTitle->getText() may |
| 167 | // have had ucfirst() applied depending on settings so we |
| 168 | // return the unmodified $maybeDisplayTitle. |
| 169 | return $maybeDisplayTitle; |
| 170 | } |
| 171 | |
| 172 | $docLang = $title->getPageLanguage(); |
| 173 | $nsIndex = $docLang->getNsIndex( $maybeNs ); |
| 174 | if ( $nsIndex !== $title->getNamespace() ) { |
| 175 | // Valid namespace but not the same as the actual page. |
| 176 | // Keep the namespace in the display title. |
| 177 | return $clean; |
| 178 | } |
| 179 | |
| 180 | return self::isSameString( $maybeDisplayTitle, $titleText ) |
| 181 | ? null |
| 182 | : $maybeDisplayTitle; |
| 183 | } |
| 184 | |
| 185 | private static function isSameString( string $a, string $b ): bool { |
| 186 | $a = mb_strtolower( strtr( $a, '_', ' ' ) ); |
| 187 | $b = mb_strtolower( strtr( $b, '_', ' ' ) ); |
| 188 | return $a === $b; |
| 189 | } |
| 190 | |
| 191 | /** |
| 192 | * Find invalid UTF-8 sequence in the source text. |
| 193 | * Fix them and flag the doc with the CirrusSearchInvalidUTF8 template. |
| 194 | * |
| 195 | * Temporary solution to help investigate/fix T225200 |
| 196 | * |
| 197 | * Visible for testing only |
| 198 | * @param array $fields |
| 199 | * @param int $pageId |
| 200 | * @return array |
| 201 | */ |
| 202 | public static function fixAndFlagInvalidUTF8InSource( array $fields, int $pageId ): array { |
| 203 | if ( isset( $fields['source_text'] ) ) { |
| 204 | $fixedVersion = mb_convert_encoding( $fields['source_text'], 'UTF-8', 'UTF-8' ); |
| 205 | if ( $fixedVersion !== $fields['source_text'] ) { |
| 206 | LoggerFactory::getInstance( 'CirrusSearch' ) |
| 207 | ->warning( 'Fixing invalid UTF-8 sequences in source text for page id {page_id}', |
| 208 | [ 'page_id' => $pageId ] ); |
| 209 | $fields['source_text'] = $fixedVersion; |
| 210 | $fields = self::flagProblem( $fields, 'CirrusSearchInvalidUTF8' ); |
| 211 | } |
| 212 | } |
| 213 | return $fields; |
| 214 | } |
| 215 | |
| 216 | private static function flagProblem( array $fields, string $template ): array { |
| 217 | $fields['template'][] = Title::makeTitle( NS_TEMPLATE, $template )->getPrefixedText(); |
| 218 | return $fields; |
| 219 | } |
| 220 | |
| 221 | /** |
| 222 | * Visible for testing only |
| 223 | * @param int $maxLen |
| 224 | * @param array $fieldContent |
| 225 | * @return array |
| 226 | */ |
| 227 | public static function truncateFileTextContent( int $maxLen, array $fieldContent ): array { |
| 228 | if ( $maxLen >= 0 && isset( $fieldContent['file_text'] ) && strlen( $fieldContent['file_text'] ) > $maxLen ) { |
| 229 | $fieldContent['file_text'] = mb_strcut( $fieldContent['file_text'], 0, $maxLen ); |
| 230 | } |
| 231 | |
| 232 | return $fieldContent; |
| 233 | } |
| 234 | |
| 235 | private function truncateFileContent( array $fieldContent ): array { |
| 236 | return self::truncateFileTextContent( $this->config->get( 'CirrusSearchMaxFileTextLength' ) ?: -1, $fieldContent ); |
| 237 | } |
| 238 | } |