Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
0.00% |
0 / 88 |
|
0.00% |
0 / 8 |
CRAP | |
0.00% |
0 / 1 |
| RedirectsAndIncomingLinks | |
0.00% |
0 / 88 |
|
0.00% |
0 / 8 |
506 | |
0.00% |
0 / 1 |
| __construct | |
0.00% |
0 / 5 |
|
0.00% |
0 / 1 |
2 | |||
| initialize | |
0.00% |
0 / 22 |
|
0.00% |
0 / 1 |
30 | |||
| finishInitializeBatch | |
0.00% |
0 / 22 |
|
0.00% |
0 / 1 |
56 | |||
| finalize | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| raiseLinkCountException | |
0.00% |
0 / 12 |
|
0.00% |
0 / 1 |
12 | |||
| buildCount | |
0.00% |
0 / 14 |
|
0.00% |
0 / 1 |
2 | |||
| newLog | |
0.00% |
0 / 7 |
|
0.00% |
0 / 1 |
2 | |||
| raiseResponseException | |
0.00% |
0 / 5 |
|
0.00% |
0 / 1 |
12 | |||
| 1 | <?php |
| 2 | |
| 3 | namespace CirrusSearch\BuildDocument; |
| 4 | |
| 5 | use CirrusSearch\CirrusConfigNames; |
| 6 | use CirrusSearch\Connection; |
| 7 | use CirrusSearch\ElasticaErrorHandler; |
| 8 | use CirrusSearch\ElasticsearchIntermediary; |
| 9 | use CirrusSearch\Search\CirrusIndexField; |
| 10 | use CirrusSearch\Search\Filters; |
| 11 | use CirrusSearch\SearchConfig; |
| 12 | use CirrusSearch\SearchRequestLog; |
| 13 | use Elastica\Document; |
| 14 | use Elastica\Exception\ResponseException; |
| 15 | use Elastica\Multi\ResultSet; |
| 16 | use Elastica\Multi\Search as MultiSearch; |
| 17 | use Elastica\Query\BoolQuery; |
| 18 | use Elastica\Query\Term; |
| 19 | use Elastica\Query\Terms; |
| 20 | use Elastica\Search; |
| 21 | use MediaWiki\Cache\BacklinkCacheFactory; |
| 22 | use MediaWiki\Logger\LoggerFactory; |
| 23 | use MediaWiki\Page\PageIdentity; |
| 24 | use MediaWiki\Page\WikiPage; |
| 25 | use MediaWiki\Revision\RevisionRecord; |
| 26 | use MediaWiki\Title\Title; |
| 27 | use MediaWiki\Title\TitleFormatter; |
| 28 | |
| 29 | /** |
| 30 | * Adds redirects and incoming links to the documents. These are done together |
| 31 | * because one needs the other. |
| 32 | * |
| 33 | * @license GPL-2.0-or-later |
| 34 | */ |
| 35 | class RedirectsAndIncomingLinks extends ElasticsearchIntermediary implements PagePropertyBuilder { |
| 36 | /** |
| 37 | * @var SearchConfig |
| 38 | */ |
| 39 | private $config; |
| 40 | |
| 41 | /** |
| 42 | * @var MultiSearch |
| 43 | */ |
| 44 | private $linkCountMultiSearch; |
| 45 | |
| 46 | /** |
| 47 | * @var callable[] Callables expecting to recieve a single argument, the total hits |
| 48 | * of the related query added to linkCountMultiSearch. Array is in same order |
| 49 | * as queries added to the multi-search. |
| 50 | */ |
| 51 | private $linkCountClosures = []; |
| 52 | |
| 53 | /** |
| 54 | * @var int[] List of page id's in current batch. Only for debug purposes. |
| 55 | */ |
| 56 | private $pageIds = []; |
| 57 | |
| 58 | /** |
| 59 | * @var BacklinkCacheFactory |
| 60 | */ |
| 61 | private $backlinkCacheFactory; |
| 62 | |
| 63 | /** |
| 64 | * @var TitleFormatter |
| 65 | */ |
| 66 | private $titleFormatter; |
| 67 | |
| 68 | /** |
| 69 | * @param Connection $conn |
| 70 | * @param BacklinkCacheFactory $backlinkCacheFactory |
| 71 | * @param TitleFormatter $titleFormatter |
| 72 | */ |
| 73 | public function __construct( |
| 74 | Connection $conn, |
| 75 | BacklinkCacheFactory $backlinkCacheFactory, |
| 76 | TitleFormatter $titleFormatter |
| 77 | ) { |
| 78 | parent::__construct( $conn, null, 0 ); |
| 79 | $this->config = $conn->getConfig(); |
| 80 | $this->linkCountMultiSearch = new MultiSearch( $this->connection->getClient() ); |
| 81 | $this->backlinkCacheFactory = $backlinkCacheFactory; |
| 82 | $this->titleFormatter = $titleFormatter; |
| 83 | } |
| 84 | |
| 85 | /** |
| 86 | * {@inheritDoc} |
| 87 | */ |
| 88 | public function initialize( Document $doc, WikiPage $page, RevisionRecord $revision, bool $isRedirect ): void { |
| 89 | $title = $page->getTitle(); |
| 90 | $this->pageIds[] = $page->getId(); |
| 91 | $outgoingLinksToCount = [ $title->getPrefixedDBkey() ]; |
| 92 | |
| 93 | // Gather redirects to this page |
| 94 | $redirectPageIdentities = $this->backlinkCacheFactory->getBacklinkCache( $title ) |
| 95 | ->getLinkPages( 'redirect', false, false, $this->config->get( CirrusConfigNames::IndexedRedirects ) ); |
| 96 | $redirects = []; |
| 97 | /** @var PageIdentity $redirect */ |
| 98 | foreach ( $redirectPageIdentities as $redirect ) { |
| 99 | // If the redirect is in main OR the same namespace as the article the index it |
| 100 | if ( $redirect->getNamespace() === NS_MAIN || $redirect->getNamespace() === $title->getNamespace() ) { |
| 101 | $redirects[] = [ |
| 102 | 'namespace' => $redirect->getNamespace(), |
| 103 | 'title' => $this->titleFormatter->getText( $redirect ) |
| 104 | ]; |
| 105 | $outgoingLinksToCount[] = $this->titleFormatter->getPrefixedDBkey( $redirect ); |
| 106 | } |
| 107 | } |
| 108 | $doc->set( 'redirect', $redirects ); |
| 109 | |
| 110 | if ( !$this->config->get( CirrusConfigNames::EnableIncomingLinkCounting ) ) { |
| 111 | return; |
| 112 | } |
| 113 | |
| 114 | // Count links |
| 115 | // Incoming links is the sum of: |
| 116 | // #1 Number of redirects to the page |
| 117 | // #2 Number of links to the title |
| 118 | // #3 Number of links to all the redirects |
| 119 | |
| 120 | // #1 we have a list of the "first" $wgCirrusSearchIndexedRedirects redirect so we just count it: |
| 121 | $redirectCount = count( $redirects ); |
| 122 | |
| 123 | // #2 and #3 we count the number of links to the page with Elasticsearch. |
| 124 | // Since we only have $wgCirrusSearchIndexedRedirects we only count that many terms. |
| 125 | $this->linkCountMultiSearch->addSearch( $this->buildCount( $outgoingLinksToCount ) ); |
| 126 | $this->linkCountClosures[] = static function ( $count ) use( $doc, $redirectCount ) { |
| 127 | $doc->set( 'incoming_links', $count + $redirectCount ); |
| 128 | CirrusIndexField::addNoopHandler( $doc, 'incoming_links', 'within 20%' ); |
| 129 | }; |
| 130 | } |
| 131 | |
| 132 | /** |
| 133 | * {@inheritDoc} |
| 134 | */ |
| 135 | public function finishInitializeBatch(): void { |
| 136 | if ( !$this->linkCountClosures ) { |
| 137 | return; |
| 138 | } |
| 139 | $linkCountClosureCount = count( $this->linkCountClosures ); |
| 140 | try { |
| 141 | $this->startNewLog( "counting links to {pageCount} pages", 'count_links', [ |
| 142 | 'pageCount' => $linkCountClosureCount, |
| 143 | 'query' => $linkCountClosureCount, |
| 144 | ] ); |
| 145 | $result = $this->linkCountMultiSearch->search(); |
| 146 | |
| 147 | if ( $result->count() <= 0 ) { |
| 148 | $this->raiseResponseException(); |
| 149 | } |
| 150 | |
| 151 | $foundNull = false; |
| 152 | for ( $index = 0; $index < $linkCountClosureCount; $index++ ) { |
| 153 | if ( $result[$index] === null ) { |
| 154 | // Finish updating other docs that have results before |
| 155 | // throwing the exception. |
| 156 | $foundNull = true; |
| 157 | } else { |
| 158 | $this->linkCountClosures[ $index ]( $result[ $index ]->getTotalHits() ); |
| 159 | } |
| 160 | } |
| 161 | if ( $foundNull ) { |
| 162 | $this->raiseLinkCountException( $result ); |
| 163 | } |
| 164 | $this->success(); |
| 165 | } catch ( \Elastica\Exception\ExceptionInterface $e ) { |
| 166 | // Note that we do not abort the update operation on failure, we simply |
| 167 | // complain about it and let the remainder of the update continue. The |
| 168 | // counts can simply be allowed to drift until resolved. |
| 169 | $this->failure( $e ); |
| 170 | LoggerFactory::getInstance( 'CirrusSearchChangeFailed' )->info( |
| 171 | 'Links for page ids: ' . implode( ',', $this->pageIds ) ); |
| 172 | } |
| 173 | } |
| 174 | |
| 175 | /** |
| 176 | * {@inheritDoc} |
| 177 | */ |
| 178 | public function finalize( Document $doc, Title $title, RevisionRecord $revision ): void { |
| 179 | // NOOP |
| 180 | } |
| 181 | |
| 182 | /** |
| 183 | * @param ResultSet $result |
| 184 | * @return never |
| 185 | */ |
| 186 | private function raiseLinkCountException( $result ): void { |
| 187 | $linkCountClosureCount = count( $this->linkCountClosures ); |
| 188 | // Seems to happen during connection issues? Treat it the |
| 189 | // same as an exception even though it wasn't thrown (why?) |
| 190 | $numNulls = 0; |
| 191 | for ( $i = 0; $i < $linkCountClosureCount; $i++ ) { |
| 192 | if ( $result[$i] === null ) { |
| 193 | $numNulls++; |
| 194 | } |
| 195 | } |
| 196 | |
| 197 | // Log the raw request/response until we understand how these happen |
| 198 | ElasticaErrorHandler::logRequestResponse( $this->connection, |
| 199 | "Received null for link count on {numNulls} out of {linkCountClosureCount} pages", [ |
| 200 | 'numNulls' => $numNulls, |
| 201 | 'linkCountClosureCount' => $linkCountClosureCount, |
| 202 | ] ); |
| 203 | |
| 204 | throw new \Elastica\Exception\RuntimeException( |
| 205 | "Received null for link count on $numNulls out of $linkCountClosureCount pages" ); |
| 206 | } |
| 207 | |
| 208 | /** |
| 209 | * Build a Search that will count all pages that link to $titles. |
| 210 | * |
| 211 | * @param string[] $titles title in prefixedDBKey form |
| 212 | * @return Search that counts all pages that link to $titles |
| 213 | */ |
| 214 | private function buildCount( array $titles ): Search { |
| 215 | $bool = new BoolQuery(); |
| 216 | $bool->addFilter( Filters::unify( |
| 217 | // must |
| 218 | [ new Terms( 'outgoing_link', $titles ) ], |
| 219 | // must_not |
| 220 | [ new Term( [ 'page_type' => 'redirect' ] ) ] |
| 221 | ) ); |
| 222 | |
| 223 | $indexPrefix = $this->config->get( SearchConfig::INDEX_BASE_NAME ); |
| 224 | $index = $this->connection->getIndex( $indexPrefix ); |
| 225 | $search = new Search( $index->getClient() ); |
| 226 | $search->addIndex( $index ); |
| 227 | $search->setQuery( $bool ); |
| 228 | $search->getQuery()->setTrackTotalHits( true ); |
| 229 | $search->getQuery()->addParam( 'stats', 'link_count' ); |
| 230 | $search->getQuery()->setSize( 0 ); |
| 231 | |
| 232 | return $search; |
| 233 | } |
| 234 | |
| 235 | /** |
| 236 | * @param string $description |
| 237 | * @param string $queryType |
| 238 | * @param array $extra |
| 239 | * @return SearchRequestLog |
| 240 | */ |
| 241 | protected function newLog( $description, $queryType, array $extra = [] ) { |
| 242 | return new SearchRequestLog( |
| 243 | $this->connection->getClient(), |
| 244 | $description, |
| 245 | $queryType, |
| 246 | $extra |
| 247 | ); |
| 248 | } |
| 249 | |
| 250 | /** |
| 251 | * @throws ResponseException |
| 252 | * @return void |
| 253 | */ |
| 254 | private function raiseResponseException(): void { |
| 255 | $client = $this->connection->getClient(); |
| 256 | $request = $client->getLastRequest(); |
| 257 | $response = $client->getLastResponse(); |
| 258 | |
| 259 | if ( $request && $response ) { |
| 260 | throw new ResponseException( $request, $response ); |
| 261 | } |
| 262 | } |
| 263 | } |