Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
95.78% |
318 / 332 |
|
82.35% |
14 / 17 |
CRAP | |
0.00% |
0 / 1 |
| ParserOutputAccess | |
95.78% |
318 / 332 |
|
82.35% |
14 / 17 |
124 | |
0.00% |
0 / 1 |
| __construct | |
100.00% |
11 / 11 |
|
100.00% |
1 / 1 |
1 | |||
| setLogger | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| normalizeOptions | |
100.00% |
17 / 17 |
|
100.00% |
1 / 1 |
8 | |||
| shouldUseCache | |
100.00% |
11 / 11 |
|
100.00% |
1 / 1 |
8 | |||
| getCachedParserOutput | |
100.00% |
34 / 34 |
|
100.00% |
1 / 1 |
18 | |||
| getFallbackOutputForLatest | |
88.89% |
16 / 18 |
|
0.00% |
0 / 1 |
6.05 | |||
| getParserOutput | |
100.00% |
59 / 59 |
|
100.00% |
1 / 1 |
22 | |||
| renderRevision | |
100.00% |
49 / 49 |
|
100.00% |
1 / 1 |
19 | |||
| checkPreconditions | |
100.00% |
19 / 19 |
|
100.00% |
1 / 1 |
10 | |||
| newPoolWork | |
96.36% |
53 / 55 |
|
0.00% |
0 / 1 |
9 | |||
| getPrimaryCache | |
100.00% |
4 / 4 |
|
100.00% |
1 / 1 |
3 | |||
| getSecondaryCache | |
100.00% |
4 / 4 |
|
100.00% |
1 / 1 |
3 | |||
| startOperationSpan | |
23.08% |
3 / 13 |
|
0.00% |
0 / 1 |
7.10 | |||
| clearLocalCache | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| saveToCache | |
100.00% |
8 / 8 |
|
100.00% |
1 / 1 |
5 | |||
| postprocess | |
100.00% |
23 / 23 |
|
100.00% |
1 / 1 |
4 | |||
| shouldCheckCache | |
100.00% |
5 / 5 |
|
100.00% |
1 / 1 |
3 | |||
| 1 | <?php |
| 2 | /** |
| 3 | * @license GPL-2.0-or-later |
| 4 | * @file |
| 5 | */ |
| 6 | namespace MediaWiki\Page; |
| 7 | |
| 8 | use InvalidArgumentException; |
| 9 | use MediaWiki\MainConfigNames; |
| 10 | use MediaWiki\MediaWikiServices; |
| 11 | use MediaWiki\Parser\ParserCache; |
| 12 | use MediaWiki\Parser\ParserCacheFactory; |
| 13 | use MediaWiki\Parser\ParserOptions; |
| 14 | use MediaWiki\Parser\ParserOutput; |
| 15 | use MediaWiki\Parser\RevisionOutputCache; |
| 16 | use MediaWiki\PoolCounter\PoolCounterFactory; |
| 17 | use MediaWiki\PoolCounter\PoolCounterWork; |
| 18 | use MediaWiki\PoolCounter\PoolCounterWorkViaCallback; |
| 19 | use MediaWiki\Revision\RevisionLookup; |
| 20 | use MediaWiki\Revision\RevisionRecord; |
| 21 | use MediaWiki\Revision\RevisionRenderer; |
| 22 | use MediaWiki\Status\Status; |
| 23 | use MediaWiki\Title\TitleFormatter; |
| 24 | use MediaWiki\Utils\MWTimestamp; |
| 25 | use MediaWiki\WikiMap\WikiMap; |
| 26 | use Psr\Log\LoggerAwareInterface; |
| 27 | use Psr\Log\LoggerInterface; |
| 28 | use Psr\Log\NullLogger; |
| 29 | use Wikimedia\Assert\Assert; |
| 30 | use Wikimedia\MapCacheLRU\MapCacheLRU; |
| 31 | use Wikimedia\Parsoid\Parsoid; |
| 32 | use Wikimedia\Rdbms\ChronologyProtector; |
| 33 | use Wikimedia\Stats\StatsFactory; |
| 34 | use Wikimedia\Telemetry\SpanInterface; |
| 35 | use Wikimedia\Telemetry\TracerInterface; |
| 36 | |
| 37 | /** |
| 38 | * Service for getting rendered output of a given page. |
| 39 | * |
| 40 | * This is a high level service, encapsulating concerns like caching |
| 41 | * and stampede protection via PoolCounter. |
| 42 | * |
| 43 | * @since 1.36 |
| 44 | * @ingroup Page |
| 45 | */ |
| 46 | class ParserOutputAccess implements LoggerAwareInterface { |
| 47 | |
| 48 | /** @internal */ |
| 49 | public const PARSOID_PCACHE_NAME = 'parsoid-' . ParserCacheFactory::DEFAULT_NAME; |
| 50 | |
| 51 | /** @internal */ |
| 52 | public const PARSOID_RCACHE_NAME = 'parsoid-' . ParserCacheFactory::DEFAULT_RCACHE_NAME; |
| 53 | |
| 54 | /** @internal */ |
| 55 | public const POSTPROC_CACHE_PREFIX = 'postproc-'; |
| 56 | |
| 57 | /** |
| 58 | * @var int Do not check the cache before parsing (force parse) |
| 59 | */ |
| 60 | public const OPT_NO_CHECK_CACHE = 1; |
| 61 | |
| 62 | /** @var int Alias for NO_CHECK_CACHE */ |
| 63 | public const OPT_FORCE_PARSE = self::OPT_NO_CHECK_CACHE; |
| 64 | |
| 65 | /** |
| 66 | * @var int Do not update the cache after parsing. |
| 67 | */ |
| 68 | public const OPT_NO_UPDATE_CACHE = 2; |
| 69 | |
| 70 | /** |
| 71 | * @var int Bypass audience check for deleted/suppressed revisions. |
| 72 | * The caller is responsible for ensuring that unauthorized access is prevented. |
| 73 | * If not set, output generation will fail if the revision is not public. |
| 74 | */ |
| 75 | public const OPT_NO_AUDIENCE_CHECK = 4; |
| 76 | |
| 77 | /** |
| 78 | * @var int Do not check the cache before parsing, |
| 79 | * and do not update the cache after parsing (not cacheable). |
| 80 | */ |
| 81 | public const OPT_NO_CACHE = self::OPT_NO_UPDATE_CACHE | self::OPT_NO_CHECK_CACHE; |
| 82 | |
| 83 | /** |
| 84 | * @var int Do perform an opportunistic LinksUpdate on cache miss |
| 85 | * @since 1.41 |
| 86 | */ |
| 87 | public const OPT_LINKS_UPDATE = 8; |
| 88 | |
| 89 | /** |
| 90 | * Apply page view semantics. This relaxes some guarantees, specifically: |
| 91 | * - Use PoolCounter for stampede protection, causing the request to |
| 92 | * block until another process has finished rendering the content. |
| 93 | * - Allow stale parser output to be returned to prevent long waits for |
| 94 | * slow renders. |
| 95 | * - Allow cacheable placeholder output to be returned when PoolCounter |
| 96 | * fails to obtain a lock. See the PoolCounterConf setting for details. |
| 97 | * |
| 98 | * @see Bug T352837 |
| 99 | * @since 1.42 |
| 100 | * @deprecated since 1.45, instead use OPT_POOL_COUNTER => POOL_COUNTER_ARTICLE_VIEW |
| 101 | * and OPT_POOL_COUNTER_FALLBACK => true. |
| 102 | */ |
| 103 | public const OPT_FOR_ARTICLE_VIEW = 16; |
| 104 | |
| 105 | /** |
| 106 | * @var int Ignore the profile version of the result from the cache. |
| 107 | * Otherwise, if it's not Parsoid's default, it will be invalidated. |
| 108 | */ |
| 109 | public const OPT_IGNORE_PROFILE_VERSION = 128; |
| 110 | |
| 111 | /** |
| 112 | * @var int ignore postprocessing cache |
| 113 | */ |
| 114 | public const OPT_NO_POSTPROC_CACHE = 256; |
| 115 | |
| 116 | /** |
| 117 | * Whether to fall back to using stale content when failing to |
| 118 | * get a poolcounter lock. |
| 119 | */ |
| 120 | public const OPT_POOL_COUNTER_FALLBACK = 'poolcounter-fallback'; |
| 121 | |
| 122 | /** |
| 123 | * @see MainConfigSchema::PoolCounterConf |
| 124 | */ |
| 125 | public const OPT_POOL_COUNTER = 'poolcounter-type'; |
| 126 | |
| 127 | /** |
| 128 | * @see MainConfigSchema::PoolCounterConf |
| 129 | */ |
| 130 | public const POOL_COUNTER_ARTICLE_VIEW = 'ArticleView'; |
| 131 | |
| 132 | /** |
| 133 | * @see MainConfigSchema::PoolCounterConf |
| 134 | */ |
| 135 | public const POOL_COUNTER_REST_API = 'HtmlRestApi'; |
| 136 | |
| 137 | /** |
| 138 | * Defaults for options that are not covered by initializing |
| 139 | * bit-based keys to zero. |
| 140 | */ |
| 141 | private const DEFAULT_OPTIONS = [ |
| 142 | self::OPT_POOL_COUNTER => null, |
| 143 | self::OPT_POOL_COUNTER_FALLBACK => false |
| 144 | ]; |
| 145 | |
| 146 | /** @var string Do not read or write any cache */ |
| 147 | private const CACHE_NONE = 'none'; |
| 148 | |
| 149 | /** @var string Use primary cache */ |
| 150 | private const CACHE_PRIMARY = 'primary'; |
| 151 | |
| 152 | /** @var string Use secondary cache */ |
| 153 | private const CACHE_SECONDARY = 'secondary'; |
| 154 | |
| 155 | /** |
| 156 | * In cases that an extension tries to get the same ParserOutput of |
| 157 | * the page right after it was parsed (T301310). |
| 158 | * @var MapCacheLRU<string,ParserOutput> |
| 159 | */ |
| 160 | private MapCacheLRU $localCache; |
| 161 | |
| 162 | private ParserCacheFactory $parserCacheFactory; |
| 163 | private RevisionLookup $revisionLookup; |
| 164 | private RevisionRenderer $revisionRenderer; |
| 165 | private StatsFactory $statsFactory; |
| 166 | private ChronologyProtector $chronologyProtector; |
| 167 | private WikiPageFactory $wikiPageFactory; |
| 168 | private TitleFormatter $titleFormatter; |
| 169 | private TracerInterface $tracer; |
| 170 | private PoolCounterFactory $poolCounterFactory; |
| 171 | private LoggerInterface $logger; |
| 172 | |
| 173 | public function __construct( |
| 174 | ParserCacheFactory $parserCacheFactory, |
| 175 | RevisionLookup $revisionLookup, |
| 176 | RevisionRenderer $revisionRenderer, |
| 177 | StatsFactory $statsFactory, |
| 178 | ChronologyProtector $chronologyProtector, |
| 179 | WikiPageFactory $wikiPageFactory, |
| 180 | TitleFormatter $titleFormatter, |
| 181 | TracerInterface $tracer, |
| 182 | PoolCounterFactory $poolCounterFactory |
| 183 | ) { |
| 184 | $this->parserCacheFactory = $parserCacheFactory; |
| 185 | $this->revisionLookup = $revisionLookup; |
| 186 | $this->revisionRenderer = $revisionRenderer; |
| 187 | $this->statsFactory = $statsFactory; |
| 188 | $this->chronologyProtector = $chronologyProtector; |
| 189 | $this->wikiPageFactory = $wikiPageFactory; |
| 190 | $this->titleFormatter = $titleFormatter; |
| 191 | $this->tracer = $tracer; |
| 192 | $this->poolCounterFactory = $poolCounterFactory; |
| 193 | |
| 194 | $this->localCache = new MapCacheLRU( 10 ); |
| 195 | $this->logger = new NullLogger(); |
| 196 | } |
| 197 | |
| 198 | public function setLogger( LoggerInterface $logger ): void { |
| 199 | $this->logger = $logger; |
| 200 | } |
| 201 | |
| 202 | /** |
| 203 | * Converts bitfield options to an associative array. |
| 204 | * |
| 205 | * If the input is an array, any integer key that has multiple bits set will |
| 206 | * be split into separate keys for each bit. String keys remain unchanged. |
| 207 | * |
| 208 | * @param int|array $options |
| 209 | * |
| 210 | * @return array An associative array with one key for each bit, |
| 211 | * plus any keys already present in the input. |
| 212 | */ |
| 213 | private static function normalizeOptions( $options ): array { |
| 214 | $bits = 0; |
| 215 | |
| 216 | // TODO: Starting in 1.46, emit deprecation warnings when getting an int. |
| 217 | |
| 218 | if ( is_array( $options ) ) { |
| 219 | if ( $options['_normalized_'] ?? false ) { |
| 220 | // already normalized. |
| 221 | return $options; |
| 222 | } |
| 223 | |
| 224 | // Collect all bits from array keys, in case one of the keys |
| 225 | // sets multiple bits. |
| 226 | foreach ( $options as $opt => $enabled ) { |
| 227 | if ( is_int( $opt ) && $enabled === true ) { |
| 228 | $bits |= $opt; |
| 229 | } |
| 230 | } |
| 231 | } else { |
| 232 | $bits = $options; |
| 233 | $options = []; |
| 234 | } |
| 235 | |
| 236 | // From the (numerically) smallest to the largest option that can possibly exist |
| 237 | for ( $b = self::OPT_NO_CHECK_CACHE; $b <= self::OPT_NO_POSTPROC_CACHE; $b <<= 1 ) { |
| 238 | $options[$b] = (bool)( $bits & $b ); |
| 239 | } |
| 240 | |
| 241 | if ( $options[ self::OPT_FOR_ARTICLE_VIEW ] ) { |
| 242 | $options[ self::OPT_POOL_COUNTER ] = self::POOL_COUNTER_ARTICLE_VIEW; |
| 243 | $options[ self::OPT_POOL_COUNTER_FALLBACK ] = true; |
| 244 | } |
| 245 | |
| 246 | $options += self::DEFAULT_OPTIONS; |
| 247 | |
| 248 | $options['_normalized_'] = true; |
| 249 | return $options; |
| 250 | } |
| 251 | |
| 252 | /** |
| 253 | * Use a cache? |
| 254 | * |
| 255 | * @param PageRecord $page |
| 256 | * @param RevisionRecord|null $rev |
| 257 | * |
| 258 | * @return string One of the CACHE_XXX constants. |
| 259 | */ |
| 260 | private function shouldUseCache( |
| 261 | PageRecord $page, |
| 262 | ?RevisionRecord $rev |
| 263 | ) { |
| 264 | if ( $rev && !$rev->getId() ) { |
| 265 | // The revision isn't from the database, so the output can't safely be cached. |
| 266 | return self::CACHE_NONE; |
| 267 | } |
| 268 | |
| 269 | // NOTE: Keep in sync with ParserWikiPage::shouldCheckParserCache(). |
| 270 | // NOTE: when we allow caching of old revisions in the future, |
| 271 | // we must not allow caching of deleted revisions. |
| 272 | |
| 273 | $wikiPage = $this->wikiPageFactory->newFromTitle( $page ); |
| 274 | if ( !$page->exists() || !$wikiPage->getContentHandler()->isParserCacheSupported() ) { |
| 275 | return self::CACHE_NONE; |
| 276 | } |
| 277 | |
| 278 | $isOld = $rev && $rev->getId() !== $page->getLatest(); |
| 279 | if ( !$isOld ) { |
| 280 | return self::CACHE_PRIMARY; |
| 281 | } |
| 282 | |
| 283 | if ( !$rev->audienceCan( RevisionRecord::DELETED_TEXT, RevisionRecord::FOR_PUBLIC ) ) { |
| 284 | // deleted/suppressed revision |
| 285 | return self::CACHE_NONE; |
| 286 | } |
| 287 | |
| 288 | return self::CACHE_SECONDARY; |
| 289 | } |
| 290 | |
| 291 | /** |
| 292 | * Get the rendered output for the given page if it is present in the cache. |
| 293 | * |
| 294 | * @param PageRecord $page |
| 295 | * @param ParserOptions $parserOptions |
| 296 | * @param RevisionRecord|null $revision |
| 297 | * @param int|array $options Bitfield or associative array using the OPT_XXX constants. |
| 298 | * Passing an int is deprecated and will trigger deprecation warnings |
| 299 | * in the future. |
| 300 | * @return ParserOutput|null |
| 301 | */ |
| 302 | public function getCachedParserOutput( |
| 303 | PageRecord $page, |
| 304 | ParserOptions $parserOptions, |
| 305 | ?RevisionRecord $revision = null, |
| 306 | $options = [] |
| 307 | ): ?ParserOutput { |
| 308 | $options = self::normalizeOptions( $options ); |
| 309 | |
| 310 | $span = $this->startOperationSpan( __FUNCTION__, $page, $revision ); |
| 311 | $isOld = $revision && $revision->getId() !== $page->getLatest(); |
| 312 | $useCache = $this->shouldUseCache( $page, $revision ); |
| 313 | $primaryCache = $this->getPrimaryCache( $parserOptions ); |
| 314 | $classCacheKey = $primaryCache->makeParserOutputKey( $page, $parserOptions ); |
| 315 | |
| 316 | if ( $useCache === self::CACHE_PRIMARY ) { |
| 317 | if ( !$isOld && $this->localCache->hasField( $classCacheKey, $page->getLatest() ) ) { |
| 318 | return $this->localCache->getField( $classCacheKey, $page->getLatest() ); |
| 319 | } |
| 320 | $output = $primaryCache->get( $page, $parserOptions ); |
| 321 | } elseif ( $useCache === self::CACHE_SECONDARY && $revision ) { |
| 322 | $secondaryCache = $this->getSecondaryCache( $parserOptions ); |
| 323 | $output = $secondaryCache->get( $revision, $parserOptions ); |
| 324 | } else { |
| 325 | $output = null; |
| 326 | } |
| 327 | |
| 328 | $statType = $statReason = $output ? 'hit' : 'miss'; |
| 329 | |
| 330 | if ( |
| 331 | $output && !$options[ self::OPT_IGNORE_PROFILE_VERSION ] && |
| 332 | $output->getContentHolder()->isParsoidContent() |
| 333 | ) { |
| 334 | $pageBundle = $output->getContentHolder()->getBasePageBundle(); |
| 335 | // T333606: Force a reparse if the version coming from cache is not the default |
| 336 | $cachedVersion = $pageBundle->version ?? null; |
| 337 | if ( |
| 338 | $cachedVersion !== null && // T325137: BadContentModel, no sense in reparsing |
| 339 | $cachedVersion !== Parsoid::defaultHTMLVersion() |
| 340 | ) { |
| 341 | $statType = 'miss'; |
| 342 | $statReason = 'obsolete'; |
| 343 | $output = null; |
| 344 | } |
| 345 | } |
| 346 | |
| 347 | if ( $output && !$isOld && !$parserOptions->getPostproc() ) { |
| 348 | $this->localCache->setField( $classCacheKey, $page->getLatest(), $output ); |
| 349 | } |
| 350 | |
| 351 | $this->statsFactory |
| 352 | ->getCounter( 'parseroutputaccess_cache_total' ) |
| 353 | ->setLabel( 'cache', $useCache ) |
| 354 | ->setLabel( 'reason', $statReason ) |
| 355 | ->setLabel( 'type', $statType ) |
| 356 | ->setLabel( 'postproc', $parserOptions->getPostproc() ? 'true' : 'false' ) |
| 357 | ->increment(); |
| 358 | |
| 359 | return $output ?: null; // convert false to null |
| 360 | } |
| 361 | |
| 362 | /** |
| 363 | * Fallback for use with PoolCounterWork. |
| 364 | * Returns stale cached output if appropriate. |
| 365 | * |
| 366 | * @return Status<ParserOutput>|false |
| 367 | */ |
| 368 | private function getFallbackOutputForLatest( |
| 369 | PageRecord $page, |
| 370 | ParserOptions $parserOptions, |
| 371 | string $workKey, |
| 372 | bool $fast |
| 373 | ) { |
| 374 | $parserOutput = $this->getPrimaryCache( $parserOptions ) |
| 375 | ->getDirty( $page, $parserOptions ); |
| 376 | |
| 377 | if ( !$parserOutput ) { |
| 378 | $this->logger->info( 'dirty missing' ); |
| 379 | return false; |
| 380 | } |
| 381 | |
| 382 | if ( $fast ) { |
| 383 | // If this user recently made DB changes, then don't eagerly serve stale output, |
| 384 | // so that users generally see their own edits after page save. |
| 385 | // |
| 386 | // If PoolCounter is overloaded, we may end up here a second time (with fast=false), |
| 387 | // in which case we will serve a stale fallback then. |
| 388 | // |
| 389 | // Note that CP reports anything in the last 10 seconds from the same client, |
| 390 | // including to other pages and other databases, so we bias towards avoiding |
| 391 | // fast-stale responses for several seconds after saving an edit. |
| 392 | if ( $this->chronologyProtector->getTouched() ) { |
| 393 | $this->logger->info( |
| 394 | 'declining fast-fallback to stale output since ChronologyProtector ' . |
| 395 | 'reports the client recently made changes', |
| 396 | [ 'workKey' => $workKey ] |
| 397 | ); |
| 398 | // Forget this ParserOutput -- we will request it again if |
| 399 | // necessary in slow mode. There might be a newer entry |
| 400 | // available by that time. |
| 401 | return false; |
| 402 | } |
| 403 | } |
| 404 | |
| 405 | $this->logger->info( $fast ? 'fast dirty output' : 'dirty output', [ 'workKey' => $workKey ] ); |
| 406 | |
| 407 | $status = Status::newGood( $parserOutput ); |
| 408 | $status->warning( 'view-pool-dirty-output' ); |
| 409 | $status->warning( $fast ? 'view-pool-contention' : 'view-pool-overload' ); |
| 410 | return $status; |
| 411 | } |
| 412 | |
| 413 | /** |
| 414 | * Returns the rendered output for the given page. |
| 415 | * Caching and concurrency control is applied. |
| 416 | * |
| 417 | * @param PageRecord $page |
| 418 | * @param ParserOptions $parserOptions |
| 419 | * @param RevisionRecord|null $revision |
| 420 | * @param int|array $options Bitfield or associative array using the OPT_XXX constants. |
| 421 | * Passing an int is deprecated and will trigger deprecation warnings |
| 422 | * in the future. |
| 423 | * |
| 424 | * @return Status<ParserOutput> containing a ParserOutput if no error occurred. |
| 425 | * Well-known errors and warnings include the following messages: |
| 426 | * - 'view-pool-dirty-output' (warning) The output is dirty (from a stale cache entry). |
| 427 | * - 'view-pool-contention' (warning) Dirty output was returned immediately instead of |
| 428 | * waiting to acquire a work lock (when "fast stale" mode is enabled in PoolCounter). |
| 429 | * - 'view-pool-timeout' (warning) Dirty output was returned after failing to acquire |
| 430 | * a work lock (got QUEUE_FULL or TIMEOUT from PoolCounter). |
| 431 | * - 'pool-queuefull' (error) unable to acquire work lock, and no cached content found. |
| 432 | * - 'pool-timeout' (error) unable to acquire work lock, and no cached content found. |
| 433 | * - 'pool-servererror' (error) PoolCounterWork failed due to a lock service error. |
| 434 | * - 'pool-unknownerror' (error) PoolCounterWork failed for an unknown reason. |
| 435 | * - 'nopagetext' (error) The page does not exist |
| 436 | */ |
| 437 | public function getParserOutput( |
| 438 | PageRecord $page, |
| 439 | ParserOptions $parserOptions, |
| 440 | ?RevisionRecord $revision = null, |
| 441 | $options = [] |
| 442 | ): Status { |
| 443 | $options = self::normalizeOptions( $options ); |
| 444 | |
| 445 | $span = $this->startOperationSpan( __FUNCTION__, $page, $revision ); |
| 446 | $error = $this->checkPreconditions( $page, $revision, $options ); |
| 447 | if ( $error ) { |
| 448 | $this->statsFactory |
| 449 | ->getCounter( 'parseroutputaccess_case' ) |
| 450 | ->setLabel( 'case', 'error' ) |
| 451 | ->setLabel( 'postproc', $parserOptions->getPostproc() ? 'true' : 'false' ) |
| 452 | ->increment(); |
| 453 | return $error; |
| 454 | } |
| 455 | |
| 456 | $isOld = $revision && $revision->getId() !== $page->getLatest(); |
| 457 | if ( $isOld ) { |
| 458 | $this->statsFactory |
| 459 | ->getCounter( 'parseroutputaccess_case' ) |
| 460 | ->setLabel( 'case', 'old' ) |
| 461 | ->setLabel( 'postproc', $parserOptions->getPostproc() ? 'true' : 'false' ) |
| 462 | ->increment(); |
| 463 | } else { |
| 464 | $this->statsFactory |
| 465 | ->getCounter( 'parseroutputaccess_case' ) |
| 466 | ->setLabel( 'case', 'current' ) |
| 467 | ->setLabel( 'postproc', $parserOptions->getPostproc() ? 'true' : 'false' ) |
| 468 | ->increment(); |
| 469 | } |
| 470 | |
| 471 | if ( $this->shouldCheckCache( $parserOptions, $options ) ) { |
| 472 | $output = $this->getCachedParserOutput( $page, $parserOptions, $revision ); |
| 473 | if ( $output ) { |
| 474 | return Status::newGood( $output ); |
| 475 | } |
| 476 | } |
| 477 | |
| 478 | if ( !$revision ) { |
| 479 | $revId = $page->getLatest(); |
| 480 | $revision = $revId ? $this->revisionLookup->getRevisionById( $revId ) : null; |
| 481 | |
| 482 | if ( !$revision ) { |
| 483 | $this->statsFactory |
| 484 | ->getCounter( 'parseroutputaccess_status' ) |
| 485 | ->setLabel( 'status', 'norev' ) |
| 486 | ->setLabel( 'postproc', $parserOptions->getPostproc() ? 'true' : 'false' ) |
| 487 | ->increment(); |
| 488 | return Status::newFatal( 'missing-revision', $revId ); |
| 489 | } |
| 490 | } |
| 491 | |
| 492 | if ( $options[ self::OPT_POOL_COUNTER ] ) { |
| 493 | $work = $this->newPoolWork( $page, $parserOptions, $revision, $options ); |
| 494 | /** @var Status $status */ |
| 495 | $status = $work->execute(); |
| 496 | } else { |
| 497 | // XXX: we could try harder to reuse a cache lookup above to |
| 498 | // provide the $previous argument here |
| 499 | $this->statsFactory->getCounter( 'parseroutputaccess_render_total' ) |
| 500 | ->setLabel( 'pool', 'none' ) |
| 501 | ->setLabel( 'cache', self::CACHE_NONE ) |
| 502 | ->setLabel( 'postproc', $parserOptions->getPostproc() ? 'true' : 'false' ) |
| 503 | ->increment(); |
| 504 | |
| 505 | $status = $this->renderRevision( $page, $parserOptions, $revision, $options, null ); |
| 506 | } |
| 507 | |
| 508 | $output = $status->getValue(); |
| 509 | Assert::postcondition( $output || !$status->isOK(), 'Inconsistent status' ); |
| 510 | |
| 511 | // T301310: cache even uncacheable content locally |
| 512 | // T348255: temporarily disable local cache of postprocessed |
| 513 | // content out of an abundance of caution |
| 514 | if ( $output && !$isOld && !$parserOptions->getPostproc() ) { |
| 515 | $primaryCache = $this->getPrimaryCache( $parserOptions ); |
| 516 | $classCacheKey = $primaryCache->makeParserOutputKey( $page, $parserOptions ); |
| 517 | $this->localCache->setField( $classCacheKey, $page->getLatest(), $output ); |
| 518 | } |
| 519 | |
| 520 | $labels = [ |
| 521 | 'postproc' => $parserOptions->getPostproc() ? 'true' : 'false', |
| 522 | 'status' => $status->isGood() ? 'good' : ( $status->isOK() ? 'ok' : 'error' ), |
| 523 | ]; |
| 524 | |
| 525 | $this->statsFactory->getCounter( 'parseroutputaccess_status' ) |
| 526 | ->setLabels( $labels ) |
| 527 | ->increment(); |
| 528 | |
| 529 | return $status; |
| 530 | } |
| 531 | |
| 532 | /** |
| 533 | * Render the given revision. |
| 534 | * |
| 535 | * This method will update the parser cache if appropriate, and will |
| 536 | * trigger a links update if OPT_LINKS_UPDATE is set. |
| 537 | * |
| 538 | * This method does not perform access checks, and will not load content |
| 539 | * from caches. The caller is assumed to have taken care of that. |
| 540 | * |
| 541 | * Where possible, pass in a $previousOutput, which will prevent an |
| 542 | * unnecessary double-lookup in the cache. |
| 543 | * |
| 544 | * @see PoolWorkArticleView::renderRevision |
| 545 | * @return Status<ParserOutput> |
| 546 | */ |
| 547 | private function renderRevision( |
| 548 | PageRecord $page, |
| 549 | ParserOptions $parserOptions, |
| 550 | RevisionRecord $revision, |
| 551 | array $options, |
| 552 | ?ParserOutput $previousOutput = null |
| 553 | ): Status { |
| 554 | $span = $this->startOperationSpan( __FUNCTION__, $page, $revision ); |
| 555 | |
| 556 | $isCurrent = $revision->getId() === $page->getLatest(); |
| 557 | |
| 558 | // T371713: Temporary statistics collection code to determine |
| 559 | // feasibility of Parsoid selective update |
| 560 | $sampleRate = MediaWikiServices::getInstance()->getMainConfig()->get( |
| 561 | MainConfigNames::ParsoidSelectiveUpdateSampleRate |
| 562 | ); |
| 563 | $doSample = ( $sampleRate && mt_rand( 1, $sampleRate ) === 1 ); |
| 564 | |
| 565 | if ( $previousOutput === null && ( $doSample || $parserOptions->getUseParsoid() ) ) { |
| 566 | // If $useCache === self::CACHE_SECONDARY we could potentially |
| 567 | // try to reuse the parse of $revision-1 from the secondary cache, |
| 568 | // but it is likely those template transclusions are out of date. |
| 569 | // Try to reuse the template transclusions from the most recent |
| 570 | // parse, which are more likely to reflect the current template. |
| 571 | if ( $this->shouldCheckCache( $parserOptions, $options ) ) { |
| 572 | $previousOutput = $this->getPrimaryCache( $parserOptions )->getDirty( $page, $parserOptions ) ?: null; |
| 573 | } |
| 574 | } |
| 575 | |
| 576 | $preStatus = null; |
| 577 | if ( $parserOptions->getPostproc() ) { |
| 578 | $preParserOptions = $parserOptions->clearPostproc(); |
| 579 | $preStatus = $this->getParserOutput( $page, $preParserOptions, $revision, $options ); |
| 580 | $output = $preStatus->getValue(); |
| 581 | if ( $output ) { |
| 582 | $output = $this->postprocess( $output, $parserOptions, $page ); |
| 583 | } |
| 584 | } else { |
| 585 | $renderedRev = $this->revisionRenderer->getRenderedRevision( $revision, $parserOptions, null, [ |
| 586 | 'audience' => RevisionRecord::RAW, |
| 587 | 'previous-output' => $previousOutput, |
| 588 | ] ); |
| 589 | |
| 590 | $output = $renderedRev->getRevisionParserOutput(); |
| 591 | } |
| 592 | |
| 593 | if ( $doSample ) { |
| 594 | # Keep these labels in sync with those in RefreshLinksJob |
| 595 | $labels = [ |
| 596 | 'source' => 'ParserOutputAccess', |
| 597 | 'type' => $previousOutput === null ? 'full' : 'selective', |
| 598 | 'reason' => $parserOptions->getRenderReason(), |
| 599 | 'parser' => $parserOptions->getUseParsoid() ? 'parsoid' : 'legacy', |
| 600 | 'opportunistic' => 'false', |
| 601 | 'wiki' => WikiMap::getCurrentWikiId(), |
| 602 | 'model' => $revision->getMainContentModel(), |
| 603 | 'postproc' => $parserOptions->getPostproc() ? 'true' : 'false', |
| 604 | ]; |
| 605 | $this->statsFactory |
| 606 | ->getCounter( 'ParserCache_selective_total' ) |
| 607 | ->setLabels( $labels ) |
| 608 | ->increment(); |
| 609 | $this->statsFactory |
| 610 | ->getCounter( 'ParserCache_selective_cpu_seconds' ) |
| 611 | ->setLabels( $labels ) |
| 612 | ->incrementBy( $output->getTimeProfile( 'cpu' ) ?? 0 ); |
| 613 | } |
| 614 | |
| 615 | $res = Status::newGood( $output ); |
| 616 | if ( $preStatus ) { |
| 617 | $res->merge( $preStatus ); |
| 618 | } |
| 619 | |
| 620 | if ( $output && $res->isGood() ) { |
| 621 | // do not cache the result if the parsercache result wasn't good (e.g. stale) |
| 622 | $this->saveToCache( $parserOptions, $output, $page, $revision, $options ); |
| 623 | } |
| 624 | |
| 625 | if ( $output && $options[ self::OPT_LINKS_UPDATE ] && !$parserOptions->getPostproc() ) { |
| 626 | $this->wikiPageFactory->newFromTitle( $page ) |
| 627 | ->triggerOpportunisticLinksUpdate( $output ); |
| 628 | } |
| 629 | |
| 630 | return $res; |
| 631 | } |
| 632 | |
| 633 | private function checkPreconditions( |
| 634 | PageRecord $page, |
| 635 | ?RevisionRecord $revision = null, |
| 636 | array $options = [] |
| 637 | ): ?Status { |
| 638 | if ( !$page->exists() ) { |
| 639 | return Status::newFatal( 'nopagetext' ); |
| 640 | } |
| 641 | |
| 642 | if ( !$options[ self::OPT_NO_UPDATE_CACHE ] && $revision && !$revision->getId() ) { |
| 643 | throw new InvalidArgumentException( |
| 644 | 'The revision does not have a known ID. Use OPT_NO_CACHE.' |
| 645 | ); |
| 646 | } |
| 647 | |
| 648 | if ( $revision && $revision->getPageId() !== $page->getId() ) { |
| 649 | throw new InvalidArgumentException( |
| 650 | 'The revision does not belong to the given page.' |
| 651 | ); |
| 652 | } |
| 653 | |
| 654 | if ( $revision && !$options[ self::OPT_NO_AUDIENCE_CHECK ] ) { |
| 655 | // NOTE: If per-user checks are desired, the caller should perform them and |
| 656 | // then set OPT_NO_AUDIENCE_CHECK if they passed. |
| 657 | if ( !$revision->audienceCan( RevisionRecord::DELETED_TEXT, RevisionRecord::FOR_PUBLIC ) ) { |
| 658 | return Status::newFatal( |
| 659 | 'missing-revision-permission', |
| 660 | $revision->getId(), |
| 661 | $revision->getTimestamp(), |
| 662 | $this->titleFormatter->getPrefixedURL( $page ) |
| 663 | ); |
| 664 | } |
| 665 | } |
| 666 | |
| 667 | return null; |
| 668 | } |
| 669 | |
| 670 | protected function newPoolWork( |
| 671 | PageRecord $page, |
| 672 | ParserOptions $parserOptions, |
| 673 | RevisionRecord $revision, |
| 674 | array $options |
| 675 | ): PoolCounterWork { |
| 676 | $profile = $options[ self::OPT_POOL_COUNTER ]; |
| 677 | // Once we're in a pool counter, don't spawn another poolcounter job |
| 678 | $options[self::OPT_POOL_COUNTER] = false; |
| 679 | // Default behavior (no caching) |
| 680 | $callbacks = [ |
| 681 | 'doWork' => fn () => $this->renderRevision( |
| 682 | $page, |
| 683 | $parserOptions, |
| 684 | $revision, |
| 685 | $options |
| 686 | ), |
| 687 | // uncached |
| 688 | 'doCachedWork' => static fn () => false, |
| 689 | // no fallback |
| 690 | 'fallback' => static fn ( $fast ) => false, |
| 691 | 'error' => static fn ( $status ) => $status, |
| 692 | ]; |
| 693 | |
| 694 | $useCache = $this->shouldUseCache( $page, $revision ); |
| 695 | |
| 696 | $this->statsFactory->getCounter( 'parseroutputaccess_render_total' ) |
| 697 | ->setLabel( 'pool', 'articleview' ) |
| 698 | ->setLabel( 'cache', $useCache ) |
| 699 | ->setLabel( 'postproc', $parserOptions->getPostproc() ? 'true' : 'false' ) |
| 700 | ->increment(); |
| 701 | |
| 702 | switch ( $useCache ) { |
| 703 | case self::CACHE_PRIMARY: |
| 704 | $primaryCache = $this->getPrimaryCache( $parserOptions ); |
| 705 | $parserCacheMetadata = $primaryCache->getMetadata( $page ); |
| 706 | $cacheKey = $primaryCache->makeParserOutputKey( $page, $parserOptions, |
| 707 | $parserCacheMetadata ? $parserCacheMetadata->getUsedOptions() : null |
| 708 | ); |
| 709 | |
| 710 | $workKey = $cacheKey . ':revid:' . $revision->getId(); |
| 711 | |
| 712 | $callbacks['doCachedWork'] = |
| 713 | static function () use ( $primaryCache, $page, $parserOptions ) { |
| 714 | $parserOutput = $primaryCache->get( $page, $parserOptions ); |
| 715 | return $parserOutput ? Status::newGood( $parserOutput ) : false; |
| 716 | }; |
| 717 | |
| 718 | $callbacks['fallback'] = |
| 719 | function ( $fast ) use ( $page, $parserOptions, $workKey, $options ) { |
| 720 | if ( $options[ self::OPT_POOL_COUNTER_FALLBACK ] ) { |
| 721 | return $this->getFallbackOutputForLatest( |
| 722 | $page, $parserOptions, $workKey, $fast |
| 723 | ); |
| 724 | } else { |
| 725 | return false; |
| 726 | } |
| 727 | }; |
| 728 | |
| 729 | break; |
| 730 | |
| 731 | case self::CACHE_SECONDARY: |
| 732 | $secondaryCache = $this->getSecondaryCache( $parserOptions ); |
| 733 | $workKey = $secondaryCache->makeParserOutputKey( $revision, $parserOptions ); |
| 734 | |
| 735 | $callbacks['doCachedWork'] = |
| 736 | static function () use ( $secondaryCache, $revision, $parserOptions ) { |
| 737 | $parserOutput = $secondaryCache->get( $revision, $parserOptions ); |
| 738 | |
| 739 | return $parserOutput ? Status::newGood( $parserOutput ) : false; |
| 740 | }; |
| 741 | |
| 742 | break; |
| 743 | |
| 744 | default: |
| 745 | $secondaryCache = $this->getSecondaryCache( $parserOptions ); |
| 746 | $workKey = $secondaryCache->makeParserOutputKeyOptionalRevId( $revision, $parserOptions ); |
| 747 | } |
| 748 | |
| 749 | $pool = $this->poolCounterFactory->create( $profile, $workKey ); |
| 750 | return new PoolCounterWorkViaCallback( |
| 751 | $pool, |
| 752 | $workKey, |
| 753 | $callbacks |
| 754 | ); |
| 755 | } |
| 756 | |
| 757 | private function getPrimaryCache( ParserOptions $pOpts ): ParserCache { |
| 758 | $name = $pOpts->getUseParsoid() ? self::PARSOID_PCACHE_NAME : ParserCacheFactory::DEFAULT_NAME; |
| 759 | if ( $pOpts->getPostproc() ) { |
| 760 | $name = self::POSTPROC_CACHE_PREFIX . $name; |
| 761 | } |
| 762 | return $this->parserCacheFactory->getParserCache( $name ); |
| 763 | } |
| 764 | |
| 765 | private function getSecondaryCache( ParserOptions $pOpts ): RevisionOutputCache { |
| 766 | $name = $pOpts->getUseParsoid() ? self::PARSOID_RCACHE_NAME : ParserCacheFactory::DEFAULT_RCACHE_NAME; |
| 767 | if ( $pOpts->getPostproc() ) { |
| 768 | $name = self::POSTPROC_CACHE_PREFIX . $name; |
| 769 | } |
| 770 | return $this->parserCacheFactory->getRevisionOutputCache( $name ); |
| 771 | } |
| 772 | |
| 773 | private function startOperationSpan( |
| 774 | string $opName, |
| 775 | PageRecord $page, |
| 776 | ?RevisionRecord $revision = null |
| 777 | ): SpanInterface { |
| 778 | $span = $this->tracer->createSpan( "ParserOutputAccess::$opName" ); |
| 779 | if ( $span->getContext()->isSampled() ) { |
| 780 | $span->setAttributes( [ |
| 781 | 'org.wikimedia.parser.page' => $page->__toString(), |
| 782 | 'org.wikimedia.parser.page.id' => $page->getId(), |
| 783 | 'org.wikimedia.parser.page.wiki' => $page->getWikiId(), |
| 784 | ] ); |
| 785 | if ( $revision ) { |
| 786 | $span->setAttributes( [ |
| 787 | 'org.wikimedia.parser.revision.id' => $revision->getId(), |
| 788 | 'org.wikimedia.parser.revision.parent_id' => $revision->getParentId(), |
| 789 | ] ); |
| 790 | } |
| 791 | } |
| 792 | return $span->start()->activate(); |
| 793 | } |
| 794 | |
| 795 | /** |
| 796 | * Clear the local cache |
| 797 | * @since 1.45 |
| 798 | */ |
| 799 | public function clearLocalCache() { |
| 800 | $this->localCache->clear(); |
| 801 | } |
| 802 | |
| 803 | private function saveToCache( |
| 804 | ParserOptions $parserOptions, ParserOutput $output, PageRecord $page, RevisionRecord $revision, array $options |
| 805 | ): void { |
| 806 | $useCache = $this->shouldUseCache( $page, $revision ); |
| 807 | if ( !$options[ self::OPT_NO_UPDATE_CACHE ] && $output->isCacheable() ) { |
| 808 | if ( $useCache === self::CACHE_PRIMARY ) { |
| 809 | $primaryCache = $this->getPrimaryCache( $parserOptions ); |
| 810 | $primaryCache->save( $output, $page, $parserOptions ); |
| 811 | } elseif ( $useCache === self::CACHE_SECONDARY ) { |
| 812 | $secondaryCache = $this->getSecondaryCache( $parserOptions ); |
| 813 | $secondaryCache->save( $output, $revision, $parserOptions ); |
| 814 | } |
| 815 | } |
| 816 | } |
| 817 | |
| 818 | /** |
| 819 | * Postprocess the given ParserOutput. |
| 820 | */ |
| 821 | public function postprocess( ParserOutput $output, ParserOptions $parserOptions, PageRecord $page ): ParserOutput { |
| 822 | // Kludgey workaround: extract $textOptions from the $parserOptions |
| 823 | $textOptions = []; |
| 824 | // Don't add these to the used options set of $output because we |
| 825 | // don't want to mutate that, and the actual return value ParserOutput |
| 826 | // doesn't yet exist. |
| 827 | $parserOptions->registerWatcher( null ); |
| 828 | foreach ( ParserOptions::$postprocOptions as $key ) { |
| 829 | $textOptions[$key] = $parserOptions->getOption( $key ); |
| 830 | } |
| 831 | $textOptions = [ |
| 832 | 'allowClone' => true, |
| 833 | ] + $textOptions; |
| 834 | |
| 835 | $pipeline = MediaWikiServices::getInstance()->getDefaultOutputPipeline(); |
| 836 | $output = $pipeline->run( $output, $parserOptions, $textOptions ); |
| 837 | // Ensure this ParserOptions is watching the resulting ParserOutput, |
| 838 | // now that it exists. |
| 839 | $parserOptions->registerWatcher( $output->recordOption( ... ) ); |
| 840 | // Ensure "postproc" is in the set of used options |
| 841 | // (Probably not necessary, but it doesn't hurt to be safe.) |
| 842 | $parserOptions->getPostproc(); |
| 843 | // Ensure all postprocOptions are in the set of used options |
| 844 | // (Since we can't detect accesses via $textOptions) |
| 845 | foreach ( ParserOptions::$postprocOptions as $key ) { |
| 846 | $parserOptions->getOption( $key ); |
| 847 | } |
| 848 | // Add a cache message if debug info is requested (this used to |
| 849 | // be part of $textOptions) |
| 850 | if ( $parserOptions->getOption( 'includeDebugInfo' ) ) { |
| 851 | # Note that we can't make the key before postprocessing because |
| 852 | # the set of used options may vary during postprocessing; similarly |
| 853 | # we can't use ParserOutput::addCacheMsg() because the |
| 854 | # RenderDebugInfo stage has already run by the time we get here. |
| 855 | # So add the debug info "the hard way", but consistent with how |
| 856 | # RenderDebugInfo does it. |
| 857 | $parserOutputKey = $this->getPrimaryCache( $parserOptions ) |
| 858 | ->makeParserOutputKey( $page, $parserOptions, $output->getUsedOptions() ); |
| 859 | $timestamp = MWTimestamp::now(); |
| 860 | $msg = "Post-processing cache key $parserOutputKey, generated at $timestamp"; |
| 861 | // Sanitize for comment. Note '‐' in the replacement is U+2010, |
| 862 | // which looks much like the problematic '-'. |
| 863 | $msg = str_replace( [ '-', '>' ], [ '‐', '>' ], $msg ); |
| 864 | $output->setContentHolderText( |
| 865 | $output->getContentHolderText() . "<!--\n$msg\n-->" |
| 866 | ); |
| 867 | } |
| 868 | return $output; |
| 869 | } |
| 870 | |
| 871 | private function shouldCheckCache( ParserOptions $parserOptions, array $options ): bool { |
| 872 | if ( $options[ self::OPT_NO_CHECK_CACHE ] ) { |
| 873 | return false; |
| 874 | } |
| 875 | if ( $parserOptions->getPostproc() ) { |
| 876 | return !$options[ self::OPT_NO_POSTPROC_CACHE ]; |
| 877 | } |
| 878 | return true; |
| 879 | } |
| 880 | } |