Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
95.94% |
331 / 345 |
|
83.33% |
15 / 18 |
CRAP | |
0.00% |
0 / 1 |
| ParserOutputAccess | |
95.94% |
331 / 345 |
|
83.33% |
15 / 18 |
126 | |
0.00% |
0 / 1 |
| __construct | |
100.00% |
11 / 11 |
|
100.00% |
1 / 1 |
1 | |||
| setLogger | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| normalizeOptions | |
100.00% |
17 / 17 |
|
100.00% |
1 / 1 |
8 | |||
| shouldUseCache | |
100.00% |
11 / 11 |
|
100.00% |
1 / 1 |
8 | |||
| getCachedParserOutput | |
100.00% |
34 / 34 |
|
100.00% |
1 / 1 |
18 | |||
| getFallbackOutputForLatest | |
88.89% |
16 / 18 |
|
0.00% |
0 / 1 |
6.05 | |||
| getParserOutput | |
100.00% |
59 / 59 |
|
100.00% |
1 / 1 |
22 | |||
| renderRevision | |
100.00% |
49 / 49 |
|
100.00% |
1 / 1 |
19 | |||
| checkPreconditions | |
100.00% |
19 / 19 |
|
100.00% |
1 / 1 |
10 | |||
| newPoolWork | |
96.36% |
53 / 55 |
|
0.00% |
0 / 1 |
9 | |||
| getPrimaryCache | |
100.00% |
4 / 4 |
|
100.00% |
1 / 1 |
3 | |||
| getSecondaryCache | |
100.00% |
4 / 4 |
|
100.00% |
1 / 1 |
3 | |||
| startOperationSpan | |
23.08% |
3 / 13 |
|
0.00% |
0 / 1 |
7.10 | |||
| clearLocalCache | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| saveToCache | |
100.00% |
8 / 8 |
|
100.00% |
1 / 1 |
5 | |||
| postprocess | |
100.00% |
14 / 14 |
|
100.00% |
1 / 1 |
1 | |||
| postprocessInPipeline | |
100.00% |
22 / 22 |
|
100.00% |
1 / 1 |
5 | |||
| shouldCheckCache | |
100.00% |
5 / 5 |
|
100.00% |
1 / 1 |
3 | |||
| 1 | <?php |
| 2 | /** |
| 3 | * @license GPL-2.0-or-later |
| 4 | * @file |
| 5 | */ |
| 6 | namespace MediaWiki\Page; |
| 7 | |
| 8 | use InvalidArgumentException; |
| 9 | use MediaWiki\Config\Config; |
| 10 | use MediaWiki\MainConfigNames; |
| 11 | use MediaWiki\OutputTransform\OutputTransformPipeline; |
| 12 | use MediaWiki\Parser\ParserCache; |
| 13 | use MediaWiki\Parser\ParserCacheFactory; |
| 14 | use MediaWiki\Parser\ParserOptions; |
| 15 | use MediaWiki\Parser\ParserOutput; |
| 16 | use MediaWiki\Parser\RevisionOutputCache; |
| 17 | use MediaWiki\PoolCounter\PoolCounterFactory; |
| 18 | use MediaWiki\PoolCounter\PoolCounterWork; |
| 19 | use MediaWiki\PoolCounter\PoolCounterWorkViaCallback; |
| 20 | use MediaWiki\Revision\RevisionLookup; |
| 21 | use MediaWiki\Revision\RevisionRecord; |
| 22 | use MediaWiki\Revision\RevisionRenderer; |
| 23 | use MediaWiki\Status\Status; |
| 24 | use MediaWiki\Title\TitleFormatter; |
| 25 | use MediaWiki\Utils\MWTimestamp; |
| 26 | use MediaWiki\WikiMap\WikiMap; |
| 27 | use Psr\Log\LoggerAwareInterface; |
| 28 | use Psr\Log\LoggerInterface; |
| 29 | use Psr\Log\NullLogger; |
| 30 | use Wikimedia\Assert\Assert; |
| 31 | use Wikimedia\ObjectCache\MapCacheLRU; |
| 32 | use Wikimedia\Parsoid\Parsoid; |
| 33 | use Wikimedia\Rdbms\ChronologyProtector; |
| 34 | use Wikimedia\Stats\StatsFactory; |
| 35 | use Wikimedia\Telemetry\SpanInterface; |
| 36 | use Wikimedia\Telemetry\TracerInterface; |
| 37 | |
| 38 | /** |
| 39 | * Service for getting rendered output of a given page. |
| 40 | * |
| 41 | * This is a high level service, encapsulating concerns like caching |
| 42 | * and stampede protection via PoolCounter. |
| 43 | * |
| 44 | * @since 1.36 |
| 45 | * @ingroup Page |
| 46 | */ |
| 47 | class ParserOutputAccess implements LoggerAwareInterface { |
| 48 | |
| 49 | /** @internal */ |
| 50 | public const PARSOID_PCACHE_NAME = 'parsoid-' . ParserCacheFactory::DEFAULT_NAME; |
| 51 | |
| 52 | /** @internal */ |
| 53 | public const PARSOID_RCACHE_NAME = 'parsoid-' . ParserCacheFactory::DEFAULT_RCACHE_NAME; |
| 54 | |
| 55 | /** @internal */ |
| 56 | public const POSTPROC_CACHE_PREFIX = 'postproc-'; |
| 57 | |
| 58 | /** |
| 59 | * @var int Do not check the cache before parsing (force parse) |
| 60 | */ |
| 61 | public const OPT_NO_CHECK_CACHE = 1; |
| 62 | |
| 63 | /** @var int Alias for NO_CHECK_CACHE */ |
| 64 | public const OPT_FORCE_PARSE = self::OPT_NO_CHECK_CACHE; |
| 65 | |
| 66 | /** |
| 67 | * @var int Do not update the cache after parsing. |
| 68 | */ |
| 69 | public const OPT_NO_UPDATE_CACHE = 2; |
| 70 | |
| 71 | /** |
| 72 | * @var int Bypass audience check for deleted/suppressed revisions. |
| 73 | * The caller is responsible for ensuring that unauthorized access is prevented. |
| 74 | * If not set, output generation will fail if the revision is not public. |
| 75 | */ |
| 76 | public const OPT_NO_AUDIENCE_CHECK = 4; |
| 77 | |
| 78 | /** |
| 79 | * @var int Do not check the cache before parsing, |
| 80 | * and do not update the cache after parsing (not cacheable). |
| 81 | */ |
| 82 | public const OPT_NO_CACHE = self::OPT_NO_UPDATE_CACHE | self::OPT_NO_CHECK_CACHE; |
| 83 | |
| 84 | /** |
| 85 | * @var int Do perform an opportunistic LinksUpdate on cache miss |
| 86 | * @since 1.41 |
| 87 | */ |
| 88 | public const OPT_LINKS_UPDATE = 8; |
| 89 | |
| 90 | /** |
| 91 | * Apply page view semantics. This relaxes some guarantees, specifically: |
| 92 | * - Use PoolCounter for stampede protection, causing the request to |
| 93 | * block until another process has finished rendering the content. |
| 94 | * - Allow stale parser output to be returned to prevent long waits for |
| 95 | * slow renders. |
| 96 | * - Allow cacheable placeholder output to be returned when PoolCounter |
| 97 | * fails to obtain a lock. See the PoolCounterConf setting for details. |
| 98 | * |
| 99 | * @see Bug T352837 |
| 100 | * @since 1.42 |
| 101 | * @deprecated since 1.45, instead use OPT_POOL_COUNTER => POOL_COUNTER_ARTICLE_VIEW |
| 102 | * and OPT_POOL_COUNTER_FALLBACK => true. |
| 103 | */ |
| 104 | public const OPT_FOR_ARTICLE_VIEW = 16; |
| 105 | |
| 106 | /** |
| 107 | * @var int Ignore the profile version of the result from the cache. |
| 108 | * Otherwise, if it's not Parsoid's default, it will be invalidated. |
| 109 | */ |
| 110 | public const OPT_IGNORE_PROFILE_VERSION = 128; |
| 111 | |
| 112 | /** |
| 113 | * @var int ignore postprocessing cache |
| 114 | */ |
| 115 | public const OPT_NO_POSTPROC_CACHE = 256; |
| 116 | |
| 117 | /** |
| 118 | * Whether to fall back to using stale content when failing to |
| 119 | * get a poolcounter lock. |
| 120 | */ |
| 121 | public const OPT_POOL_COUNTER_FALLBACK = 'poolcounter-fallback'; |
| 122 | |
| 123 | /** |
| 124 | * @see MainConfigSchema::PoolCounterConf |
| 125 | */ |
| 126 | public const OPT_POOL_COUNTER = 'poolcounter-type'; |
| 127 | |
| 128 | /** |
| 129 | * @see MainConfigSchema::PoolCounterConf |
| 130 | */ |
| 131 | public const POOL_COUNTER_ARTICLE_VIEW = 'ArticleView'; |
| 132 | |
| 133 | /** |
| 134 | * @see MainConfigSchema::PoolCounterConf |
| 135 | */ |
| 136 | public const POOL_COUNTER_REST_API = 'HtmlRestApi'; |
| 137 | |
| 138 | /** |
| 139 | * Defaults for options that are not covered by initializing |
| 140 | * bit-based keys to zero. |
| 141 | */ |
| 142 | private const DEFAULT_OPTIONS = [ |
| 143 | self::OPT_POOL_COUNTER => null, |
| 144 | self::OPT_POOL_COUNTER_FALLBACK => false |
| 145 | ]; |
| 146 | |
| 147 | /** @var string Do not read or write any cache */ |
| 148 | private const CACHE_NONE = 'none'; |
| 149 | |
| 150 | /** @var string Use primary cache */ |
| 151 | private const CACHE_PRIMARY = 'primary'; |
| 152 | |
| 153 | /** @var string Use secondary cache */ |
| 154 | private const CACHE_SECONDARY = 'secondary'; |
| 155 | |
| 156 | /** |
| 157 | * In cases that an extension tries to get the same ParserOutput of |
| 158 | * the page right after it was parsed (T301310). |
| 159 | * @var MapCacheLRU<string,ParserOutput> |
| 160 | */ |
| 161 | private MapCacheLRU $localCache; |
| 162 | |
| 163 | private ParserCacheFactory $parserCacheFactory; |
| 164 | private RevisionLookup $revisionLookup; |
| 165 | private RevisionRenderer $revisionRenderer; |
| 166 | private StatsFactory $statsFactory; |
| 167 | private ChronologyProtector $chronologyProtector; |
| 168 | private WikiPageFactory $wikiPageFactory; |
| 169 | private TitleFormatter $titleFormatter; |
| 170 | private TracerInterface $tracer; |
| 171 | private PoolCounterFactory $poolCounterFactory; |
| 172 | private LoggerInterface $logger; |
| 173 | |
| 174 | public function __construct( |
| 175 | private readonly Config $config, |
| 176 | private readonly OutputTransformPipeline $outputTransformPipeline, |
| 177 | ParserCacheFactory $parserCacheFactory, |
| 178 | RevisionLookup $revisionLookup, |
| 179 | RevisionRenderer $revisionRenderer, |
| 180 | StatsFactory $statsFactory, |
| 181 | ChronologyProtector $chronologyProtector, |
| 182 | WikiPageFactory $wikiPageFactory, |
| 183 | TitleFormatter $titleFormatter, |
| 184 | TracerInterface $tracer, |
| 185 | PoolCounterFactory $poolCounterFactory |
| 186 | ) { |
| 187 | $this->parserCacheFactory = $parserCacheFactory; |
| 188 | $this->revisionLookup = $revisionLookup; |
| 189 | $this->revisionRenderer = $revisionRenderer; |
| 190 | $this->statsFactory = $statsFactory; |
| 191 | $this->chronologyProtector = $chronologyProtector; |
| 192 | $this->wikiPageFactory = $wikiPageFactory; |
| 193 | $this->titleFormatter = $titleFormatter; |
| 194 | $this->tracer = $tracer; |
| 195 | $this->poolCounterFactory = $poolCounterFactory; |
| 196 | |
| 197 | $this->localCache = new MapCacheLRU( 10 ); |
| 198 | $this->logger = new NullLogger(); |
| 199 | } |
| 200 | |
| 201 | public function setLogger( LoggerInterface $logger ): void { |
| 202 | $this->logger = $logger; |
| 203 | } |
| 204 | |
| 205 | /** |
| 206 | * Converts bitfield options to an associative array. |
| 207 | * |
| 208 | * If the input is an array, any integer key that has multiple bits set will |
| 209 | * be split into separate keys for each bit. String keys remain unchanged. |
| 210 | * |
| 211 | * @param int|array $options |
| 212 | * |
| 213 | * @return array An associative array with one key for each bit, |
| 214 | * plus any keys already present in the input. |
| 215 | */ |
| 216 | private static function normalizeOptions( $options ): array { |
| 217 | $bits = 0; |
| 218 | |
| 219 | // TODO: Starting in 1.46, emit deprecation warnings when getting an int. |
| 220 | |
| 221 | if ( is_array( $options ) ) { |
| 222 | if ( $options['_normalized_'] ?? false ) { |
| 223 | // already normalized. |
| 224 | return $options; |
| 225 | } |
| 226 | |
| 227 | // Collect all bits from array keys, in case one of the keys |
| 228 | // sets multiple bits. |
| 229 | foreach ( $options as $opt => $enabled ) { |
| 230 | if ( is_int( $opt ) && $enabled === true ) { |
| 231 | $bits |= $opt; |
| 232 | } |
| 233 | } |
| 234 | } else { |
| 235 | $bits = $options; |
| 236 | $options = []; |
| 237 | } |
| 238 | |
| 239 | // From the (numerically) smallest to the largest option that can possibly exist |
| 240 | for ( $b = self::OPT_NO_CHECK_CACHE; $b <= self::OPT_NO_POSTPROC_CACHE; $b <<= 1 ) { |
| 241 | $options[$b] = (bool)( $bits & $b ); |
| 242 | } |
| 243 | |
| 244 | if ( $options[ self::OPT_FOR_ARTICLE_VIEW ] ) { |
| 245 | $options[ self::OPT_POOL_COUNTER ] = self::POOL_COUNTER_ARTICLE_VIEW; |
| 246 | $options[ self::OPT_POOL_COUNTER_FALLBACK ] = true; |
| 247 | } |
| 248 | |
| 249 | $options += self::DEFAULT_OPTIONS; |
| 250 | |
| 251 | $options['_normalized_'] = true; |
| 252 | return $options; |
| 253 | } |
| 254 | |
| 255 | /** |
| 256 | * Use a cache? |
| 257 | * |
| 258 | * @param PageRecord $page |
| 259 | * @param RevisionRecord|null $rev |
| 260 | * |
| 261 | * @return string One of the CACHE_XXX constants. |
| 262 | */ |
| 263 | private function shouldUseCache( |
| 264 | PageRecord $page, |
| 265 | ?RevisionRecord $rev |
| 266 | ) { |
| 267 | if ( $rev && !$rev->getId() ) { |
| 268 | // The revision isn't from the database, so the output can't safely be cached. |
| 269 | return self::CACHE_NONE; |
| 270 | } |
| 271 | |
| 272 | // NOTE: Keep in sync with ParserWikiPage::shouldCheckParserCache(). |
| 273 | // NOTE: when we allow caching of old revisions in the future, |
| 274 | // we must not allow caching of deleted revisions. |
| 275 | |
| 276 | $wikiPage = $this->wikiPageFactory->newFromTitle( $page ); |
| 277 | if ( !$page->exists() || !$wikiPage->getContentHandler()->isParserCacheSupported() ) { |
| 278 | return self::CACHE_NONE; |
| 279 | } |
| 280 | |
| 281 | $isOld = $rev && $rev->getId() !== $page->getLatest(); |
| 282 | if ( !$isOld ) { |
| 283 | return self::CACHE_PRIMARY; |
| 284 | } |
| 285 | |
| 286 | if ( !$rev->audienceCan( RevisionRecord::DELETED_TEXT, RevisionRecord::FOR_PUBLIC ) ) { |
| 287 | // deleted/suppressed revision |
| 288 | return self::CACHE_NONE; |
| 289 | } |
| 290 | |
| 291 | return self::CACHE_SECONDARY; |
| 292 | } |
| 293 | |
| 294 | /** |
| 295 | * Get the rendered output for the given page if it is present in the cache. |
| 296 | * |
| 297 | * @param PageRecord $page |
| 298 | * @param ParserOptions $parserOptions |
| 299 | * @param RevisionRecord|null $revision |
| 300 | * @param int|array $options Bitfield or associative array using the OPT_XXX constants. |
| 301 | * Passing an int is deprecated and will trigger deprecation warnings |
| 302 | * in the future. |
| 303 | * @return ParserOutput|null |
| 304 | */ |
| 305 | public function getCachedParserOutput( |
| 306 | PageRecord $page, |
| 307 | ParserOptions $parserOptions, |
| 308 | ?RevisionRecord $revision = null, |
| 309 | $options = [] |
| 310 | ): ?ParserOutput { |
| 311 | $options = self::normalizeOptions( $options ); |
| 312 | |
| 313 | $span = $this->startOperationSpan( __FUNCTION__, $page, $revision ); |
| 314 | $isOld = $revision && $revision->getId() !== $page->getLatest(); |
| 315 | $useCache = $this->shouldUseCache( $page, $revision ); |
| 316 | $primaryCache = $this->getPrimaryCache( $parserOptions ); |
| 317 | $classCacheKey = $primaryCache->makeParserOutputKey( $page, $parserOptions ); |
| 318 | |
| 319 | if ( $useCache === self::CACHE_PRIMARY ) { |
| 320 | if ( !$isOld && $this->localCache->hasField( $classCacheKey, $page->getLatest() ) ) { |
| 321 | return $this->localCache->getField( $classCacheKey, $page->getLatest() ); |
| 322 | } |
| 323 | $output = $primaryCache->get( $page, $parserOptions ); |
| 324 | } elseif ( $useCache === self::CACHE_SECONDARY && $revision ) { |
| 325 | $secondaryCache = $this->getSecondaryCache( $parserOptions ); |
| 326 | $output = $secondaryCache->get( $revision, $parserOptions ); |
| 327 | } else { |
| 328 | $output = null; |
| 329 | } |
| 330 | |
| 331 | $statType = $statReason = $output ? 'hit' : 'miss'; |
| 332 | |
| 333 | if ( |
| 334 | $output && !$options[ self::OPT_IGNORE_PROFILE_VERSION ] && |
| 335 | $output->getContentHolder()->isParsoidContent() |
| 336 | ) { |
| 337 | $pageBundle = $output->getContentHolder()->getBasePageBundle(); |
| 338 | // T333606: Force a reparse if the version coming from cache is not the default |
| 339 | $cachedVersion = $pageBundle->version ?? null; |
| 340 | if ( |
| 341 | $cachedVersion !== null && // T325137: BadContentModel, no sense in reparsing |
| 342 | $cachedVersion !== Parsoid::defaultHTMLVersion() |
| 343 | ) { |
| 344 | $statType = 'miss'; |
| 345 | $statReason = 'obsolete'; |
| 346 | $output = null; |
| 347 | } |
| 348 | } |
| 349 | |
| 350 | if ( $output && !$isOld && !$parserOptions->getPostproc() ) { |
| 351 | $this->localCache->setField( $classCacheKey, $page->getLatest(), $output ); |
| 352 | } |
| 353 | |
| 354 | $this->statsFactory |
| 355 | ->getCounter( 'parseroutputaccess_cache_total' ) |
| 356 | ->setLabel( 'cache', $useCache ) |
| 357 | ->setLabel( 'reason', $statReason ) |
| 358 | ->setLabel( 'type', $statType ) |
| 359 | ->setLabel( 'postproc', $parserOptions->getPostproc() ? 'true' : 'false' ) |
| 360 | ->increment(); |
| 361 | |
| 362 | return $output ?: null; // convert false to null |
| 363 | } |
| 364 | |
| 365 | /** |
| 366 | * Fallback for use with PoolCounterWork. |
| 367 | * Returns stale cached output if appropriate. |
| 368 | * |
| 369 | * @return Status<ParserOutput>|false |
| 370 | */ |
| 371 | private function getFallbackOutputForLatest( |
| 372 | PageRecord $page, |
| 373 | ParserOptions $parserOptions, |
| 374 | string $workKey, |
| 375 | bool $fast |
| 376 | ) { |
| 377 | $parserOutput = $this->getPrimaryCache( $parserOptions ) |
| 378 | ->getDirty( $page, $parserOptions ); |
| 379 | |
| 380 | if ( !$parserOutput ) { |
| 381 | $this->logger->info( 'dirty missing' ); |
| 382 | return false; |
| 383 | } |
| 384 | |
| 385 | if ( $fast ) { |
| 386 | // If this user recently made DB changes, then don't eagerly serve stale output, |
| 387 | // so that users generally see their own edits after page save. |
| 388 | // |
| 389 | // If PoolCounter is overloaded, we may end up here a second time (with fast=false), |
| 390 | // in which case we will serve a stale fallback then. |
| 391 | // |
| 392 | // Note that CP reports anything in the last 10 seconds from the same client, |
| 393 | // including to other pages and other databases, so we bias towards avoiding |
| 394 | // fast-stale responses for several seconds after saving an edit. |
| 395 | if ( $this->chronologyProtector->getTouched() ) { |
| 396 | $this->logger->info( |
| 397 | 'declining fast-fallback to stale output since ChronologyProtector ' . |
| 398 | 'reports the client recently made changes', |
| 399 | [ 'workKey' => $workKey ] |
| 400 | ); |
| 401 | // Forget this ParserOutput -- we will request it again if |
| 402 | // necessary in slow mode. There might be a newer entry |
| 403 | // available by that time. |
| 404 | return false; |
| 405 | } |
| 406 | } |
| 407 | |
| 408 | $this->logger->info( $fast ? 'fast dirty output' : 'dirty output', [ 'workKey' => $workKey ] ); |
| 409 | |
| 410 | $status = Status::newGood( $parserOutput ); |
| 411 | $status->warning( 'view-pool-dirty-output' ); |
| 412 | $status->warning( $fast ? 'view-pool-contention' : 'view-pool-overload' ); |
| 413 | return $status; |
| 414 | } |
| 415 | |
| 416 | /** |
| 417 | * Returns the rendered output for the given page. |
| 418 | * Caching and concurrency control is applied. |
| 419 | * |
| 420 | * @param PageRecord $page |
| 421 | * @param ParserOptions $parserOptions |
| 422 | * @param RevisionRecord|null $revision |
| 423 | * @param int|array $options Bitfield or associative array using the OPT_XXX constants. |
| 424 | * Passing an int is deprecated and will trigger deprecation warnings |
| 425 | * in the future. |
| 426 | * |
| 427 | * @return Status<ParserOutput> containing a ParserOutput if no error occurred. |
| 428 | * Well-known errors and warnings include the following messages: |
| 429 | * - 'view-pool-dirty-output' (warning) The output is dirty (from a stale cache entry). |
| 430 | * - 'view-pool-contention' (warning) Dirty output was returned immediately instead of |
| 431 | * waiting to acquire a work lock (when "fast stale" mode is enabled in PoolCounter). |
| 432 | * - 'view-pool-timeout' (warning) Dirty output was returned after failing to acquire |
| 433 | * a work lock (got QUEUE_FULL or TIMEOUT from PoolCounter). |
| 434 | * - 'pool-queuefull' (error) unable to acquire work lock, and no cached content found. |
| 435 | * - 'pool-timeout' (error) unable to acquire work lock, and no cached content found. |
| 436 | * - 'pool-servererror' (error) PoolCounterWork failed due to a lock service error. |
| 437 | * - 'pool-unknownerror' (error) PoolCounterWork failed for an unknown reason. |
| 438 | * - 'nopagetext' (error) The page does not exist |
| 439 | */ |
| 440 | public function getParserOutput( |
| 441 | PageRecord $page, |
| 442 | ParserOptions $parserOptions, |
| 443 | ?RevisionRecord $revision = null, |
| 444 | $options = [] |
| 445 | ): Status { |
| 446 | $options = self::normalizeOptions( $options ); |
| 447 | |
| 448 | $span = $this->startOperationSpan( __FUNCTION__, $page, $revision ); |
| 449 | $error = $this->checkPreconditions( $page, $revision, $options ); |
| 450 | if ( $error ) { |
| 451 | $this->statsFactory |
| 452 | ->getCounter( 'parseroutputaccess_case' ) |
| 453 | ->setLabel( 'case', 'error' ) |
| 454 | ->setLabel( 'postproc', $parserOptions->getPostproc() ? 'true' : 'false' ) |
| 455 | ->increment(); |
| 456 | return $error; |
| 457 | } |
| 458 | |
| 459 | $isOld = $revision && $revision->getId() !== $page->getLatest(); |
| 460 | if ( $isOld ) { |
| 461 | $this->statsFactory |
| 462 | ->getCounter( 'parseroutputaccess_case' ) |
| 463 | ->setLabel( 'case', 'old' ) |
| 464 | ->setLabel( 'postproc', $parserOptions->getPostproc() ? 'true' : 'false' ) |
| 465 | ->increment(); |
| 466 | } else { |
| 467 | $this->statsFactory |
| 468 | ->getCounter( 'parseroutputaccess_case' ) |
| 469 | ->setLabel( 'case', 'current' ) |
| 470 | ->setLabel( 'postproc', $parserOptions->getPostproc() ? 'true' : 'false' ) |
| 471 | ->increment(); |
| 472 | } |
| 473 | |
| 474 | if ( $this->shouldCheckCache( $parserOptions, $options ) ) { |
| 475 | $output = $this->getCachedParserOutput( $page, $parserOptions, $revision ); |
| 476 | if ( $output ) { |
| 477 | return Status::newGood( $output ); |
| 478 | } |
| 479 | } |
| 480 | |
| 481 | if ( !$revision ) { |
| 482 | $revId = $page->getLatest(); |
| 483 | $revision = $revId ? $this->revisionLookup->getRevisionById( $revId ) : null; |
| 484 | |
| 485 | if ( !$revision ) { |
| 486 | $this->statsFactory |
| 487 | ->getCounter( 'parseroutputaccess_status' ) |
| 488 | ->setLabel( 'status', 'norev' ) |
| 489 | ->setLabel( 'postproc', $parserOptions->getPostproc() ? 'true' : 'false' ) |
| 490 | ->increment(); |
| 491 | return Status::newFatal( 'missing-revision', $revId ); |
| 492 | } |
| 493 | } |
| 494 | |
| 495 | if ( $options[ self::OPT_POOL_COUNTER ] ) { |
| 496 | $work = $this->newPoolWork( $page, $parserOptions, $revision, $options ); |
| 497 | /** @var Status $status */ |
| 498 | $status = $work->execute(); |
| 499 | } else { |
| 500 | // XXX: we could try harder to reuse a cache lookup above to |
| 501 | // provide the $previous argument here |
| 502 | $this->statsFactory->getCounter( 'parseroutputaccess_render_total' ) |
| 503 | ->setLabel( 'pool', 'none' ) |
| 504 | ->setLabel( 'cache', self::CACHE_NONE ) |
| 505 | ->setLabel( 'postproc', $parserOptions->getPostproc() ? 'true' : 'false' ) |
| 506 | ->increment(); |
| 507 | |
| 508 | $status = $this->renderRevision( $page, $parserOptions, $revision, $options, null ); |
| 509 | } |
| 510 | |
| 511 | $output = $status->getValue(); |
| 512 | Assert::postcondition( $output || !$status->isOK(), 'Inconsistent status' ); |
| 513 | |
| 514 | // T301310: cache even uncacheable content locally |
| 515 | // T348255: temporarily disable local cache of postprocessed |
| 516 | // content out of an abundance of caution |
| 517 | if ( $output && !$isOld && !$parserOptions->getPostproc() ) { |
| 518 | $primaryCache = $this->getPrimaryCache( $parserOptions ); |
| 519 | $classCacheKey = $primaryCache->makeParserOutputKey( $page, $parserOptions ); |
| 520 | $this->localCache->setField( $classCacheKey, $page->getLatest(), $output ); |
| 521 | } |
| 522 | |
| 523 | $labels = [ |
| 524 | 'postproc' => $parserOptions->getPostproc() ? 'true' : 'false', |
| 525 | 'status' => $status->isGood() ? 'good' : ( $status->isOK() ? 'ok' : 'error' ), |
| 526 | ]; |
| 527 | |
| 528 | $this->statsFactory->getCounter( 'parseroutputaccess_status' ) |
| 529 | ->setLabels( $labels ) |
| 530 | ->increment(); |
| 531 | |
| 532 | return $status; |
| 533 | } |
| 534 | |
| 535 | /** |
| 536 | * Render the given revision. |
| 537 | * |
| 538 | * This method will update the parser cache if appropriate, and will |
| 539 | * trigger a links update if OPT_LINKS_UPDATE is set. |
| 540 | * |
| 541 | * This method does not perform access checks, and will not load content |
| 542 | * from caches. The caller is assumed to have taken care of that. |
| 543 | * |
| 544 | * Where possible, pass in a $previousOutput, which will prevent an |
| 545 | * unnecessary double-lookup in the cache. |
| 546 | * |
| 547 | * @see PoolWorkArticleView::renderRevision |
| 548 | * @return Status<ParserOutput> |
| 549 | */ |
| 550 | private function renderRevision( |
| 551 | PageRecord $page, |
| 552 | ParserOptions $parserOptions, |
| 553 | RevisionRecord $revision, |
| 554 | array $options, |
| 555 | ?ParserOutput $previousOutput = null |
| 556 | ): Status { |
| 557 | $span = $this->startOperationSpan( __FUNCTION__, $page, $revision ); |
| 558 | |
| 559 | $isCurrent = $revision->getId() === $page->getLatest(); |
| 560 | |
| 561 | // T371713: Temporary statistics collection code to determine |
| 562 | // feasibility of Parsoid selective update |
| 563 | $sampleRate = $this->config->get( |
| 564 | MainConfigNames::ParsoidSelectiveUpdateSampleRate |
| 565 | ); |
| 566 | $doSample = ( $sampleRate && mt_rand( 1, $sampleRate ) === 1 ); |
| 567 | |
| 568 | if ( $previousOutput === null && ( $doSample || $parserOptions->getUseParsoid() ) ) { |
| 569 | // If $useCache === self::CACHE_SECONDARY we could potentially |
| 570 | // try to reuse the parse of $revision-1 from the secondary cache, |
| 571 | // but it is likely those template transclusions are out of date. |
| 572 | // Try to reuse the template transclusions from the most recent |
| 573 | // parse, which are more likely to reflect the current template. |
| 574 | if ( $this->shouldCheckCache( $parserOptions, $options ) ) { |
| 575 | $previousOutput = $this->getPrimaryCache( $parserOptions )->getDirty( $page, $parserOptions ) ?: null; |
| 576 | } |
| 577 | } |
| 578 | |
| 579 | $preStatus = null; |
| 580 | if ( $parserOptions->getPostproc() ) { |
| 581 | $preParserOptions = $parserOptions->clearPostproc(); |
| 582 | $preStatus = $this->getParserOutput( $page, $preParserOptions, $revision, $options ); |
| 583 | $output = $preStatus->getValue(); |
| 584 | if ( $output ) { |
| 585 | $output = $this->postprocess( $output, $parserOptions, $page, $revision ); |
| 586 | } |
| 587 | } else { |
| 588 | $renderedRev = $this->revisionRenderer->getRenderedRevision( $revision, $parserOptions, null, [ |
| 589 | 'audience' => RevisionRecord::RAW, |
| 590 | 'previous-output' => $previousOutput, |
| 591 | ] ); |
| 592 | |
| 593 | $output = $renderedRev->getRevisionParserOutput(); |
| 594 | } |
| 595 | |
| 596 | if ( $doSample ) { |
| 597 | # Keep these labels in sync with those in RefreshLinksJob |
| 598 | $labels = [ |
| 599 | 'source' => 'ParserOutputAccess', |
| 600 | 'type' => $previousOutput === null ? 'full' : 'selective', |
| 601 | 'reason' => $parserOptions->getRenderReason(), |
| 602 | 'parser' => $parserOptions->getUseParsoid() ? 'parsoid' : 'legacy', |
| 603 | 'opportunistic' => 'false', |
| 604 | 'wiki' => WikiMap::getCurrentWikiId(), |
| 605 | 'model' => $revision->getMainContentModel(), |
| 606 | 'postproc' => $parserOptions->getPostproc() ? 'true' : 'false', |
| 607 | ]; |
| 608 | $this->statsFactory |
| 609 | ->getCounter( 'ParserCache_selective_total' ) |
| 610 | ->setLabels( $labels ) |
| 611 | ->increment(); |
| 612 | $this->statsFactory |
| 613 | ->getCounter( 'ParserCache_selective_cpu_seconds' ) |
| 614 | ->setLabels( $labels ) |
| 615 | ->incrementBy( $output->getTimeProfile( 'cpu' ) ?? 0 ); |
| 616 | } |
| 617 | |
| 618 | $res = Status::newGood( $output ); |
| 619 | if ( $preStatus ) { |
| 620 | $res->merge( $preStatus ); |
| 621 | } |
| 622 | |
| 623 | if ( $output && $res->isGood() ) { |
| 624 | // do not cache the result if the parsercache result wasn't good (e.g. stale) |
| 625 | $this->saveToCache( $parserOptions, $output, $page, $revision, $options ); |
| 626 | } |
| 627 | |
| 628 | if ( $output && $options[ self::OPT_LINKS_UPDATE ] && !$parserOptions->getPostproc() ) { |
| 629 | $this->wikiPageFactory->newFromTitle( $page ) |
| 630 | ->triggerOpportunisticLinksUpdate( $output ); |
| 631 | } |
| 632 | |
| 633 | return $res; |
| 634 | } |
| 635 | |
| 636 | private function checkPreconditions( |
| 637 | PageRecord $page, |
| 638 | ?RevisionRecord $revision = null, |
| 639 | array $options = [] |
| 640 | ): ?Status { |
| 641 | if ( !$page->exists() ) { |
| 642 | return Status::newFatal( 'nopagetext' ); |
| 643 | } |
| 644 | |
| 645 | if ( !$options[ self::OPT_NO_UPDATE_CACHE ] && $revision && !$revision->getId() ) { |
| 646 | throw new InvalidArgumentException( |
| 647 | 'The revision does not have a known ID. Use OPT_NO_CACHE.' |
| 648 | ); |
| 649 | } |
| 650 | |
| 651 | if ( $revision && $revision->getPageId() !== $page->getId() ) { |
| 652 | throw new InvalidArgumentException( |
| 653 | 'The revision does not belong to the given page.' |
| 654 | ); |
| 655 | } |
| 656 | |
| 657 | if ( $revision && !$options[ self::OPT_NO_AUDIENCE_CHECK ] ) { |
| 658 | // NOTE: If per-user checks are desired, the caller should perform them and |
| 659 | // then set OPT_NO_AUDIENCE_CHECK if they passed. |
| 660 | if ( !$revision->audienceCan( RevisionRecord::DELETED_TEXT, RevisionRecord::FOR_PUBLIC ) ) { |
| 661 | return Status::newFatal( |
| 662 | 'missing-revision-permission', |
| 663 | $revision->getId(), |
| 664 | $revision->getTimestamp(), |
| 665 | $this->titleFormatter->getPrefixedURL( $page ) |
| 666 | ); |
| 667 | } |
| 668 | } |
| 669 | |
| 670 | return null; |
| 671 | } |
| 672 | |
| 673 | protected function newPoolWork( |
| 674 | PageRecord $page, |
| 675 | ParserOptions $parserOptions, |
| 676 | RevisionRecord $revision, |
| 677 | array $options |
| 678 | ): PoolCounterWork { |
| 679 | $profile = $options[ self::OPT_POOL_COUNTER ]; |
| 680 | // Once we're in a pool counter, don't spawn another poolcounter job |
| 681 | $options[self::OPT_POOL_COUNTER] = false; |
| 682 | // Default behavior (no caching) |
| 683 | $callbacks = [ |
| 684 | 'doWork' => fn () => $this->renderRevision( |
| 685 | $page, |
| 686 | $parserOptions, |
| 687 | $revision, |
| 688 | $options |
| 689 | ), |
| 690 | // uncached |
| 691 | 'doCachedWork' => static fn () => false, |
| 692 | // no fallback |
| 693 | 'fallback' => static fn ( $fast ) => false, |
| 694 | 'error' => static fn ( $status ) => $status, |
| 695 | ]; |
| 696 | |
| 697 | $useCache = $this->shouldUseCache( $page, $revision ); |
| 698 | |
| 699 | $this->statsFactory->getCounter( 'parseroutputaccess_render_total' ) |
| 700 | ->setLabel( 'pool', 'articleview' ) |
| 701 | ->setLabel( 'cache', $useCache ) |
| 702 | ->setLabel( 'postproc', $parserOptions->getPostproc() ? 'true' : 'false' ) |
| 703 | ->increment(); |
| 704 | |
| 705 | switch ( $useCache ) { |
| 706 | case self::CACHE_PRIMARY: |
| 707 | $primaryCache = $this->getPrimaryCache( $parserOptions ); |
| 708 | $parserCacheMetadata = $primaryCache->getMetadata( $page ); |
| 709 | $cacheKey = $primaryCache->makeParserOutputKey( $page, $parserOptions, |
| 710 | $parserCacheMetadata ? $parserCacheMetadata->getUsedOptions() : null |
| 711 | ); |
| 712 | |
| 713 | $workKey = $cacheKey . ':revid:' . $revision->getId(); |
| 714 | |
| 715 | $callbacks['doCachedWork'] = |
| 716 | static function () use ( $primaryCache, $page, $parserOptions ) { |
| 717 | $parserOutput = $primaryCache->get( $page, $parserOptions ); |
| 718 | return $parserOutput ? Status::newGood( $parserOutput ) : false; |
| 719 | }; |
| 720 | |
| 721 | $callbacks['fallback'] = |
| 722 | function ( $fast ) use ( $page, $parserOptions, $workKey, $options ) { |
| 723 | if ( $options[ self::OPT_POOL_COUNTER_FALLBACK ] ) { |
| 724 | return $this->getFallbackOutputForLatest( |
| 725 | $page, $parserOptions, $workKey, $fast |
| 726 | ); |
| 727 | } else { |
| 728 | return false; |
| 729 | } |
| 730 | }; |
| 731 | |
| 732 | break; |
| 733 | |
| 734 | case self::CACHE_SECONDARY: |
| 735 | $secondaryCache = $this->getSecondaryCache( $parserOptions ); |
| 736 | $workKey = $secondaryCache->makeParserOutputKey( $revision, $parserOptions ); |
| 737 | |
| 738 | $callbacks['doCachedWork'] = |
| 739 | static function () use ( $secondaryCache, $revision, $parserOptions ) { |
| 740 | $parserOutput = $secondaryCache->get( $revision, $parserOptions ); |
| 741 | |
| 742 | return $parserOutput ? Status::newGood( $parserOutput ) : false; |
| 743 | }; |
| 744 | |
| 745 | break; |
| 746 | |
| 747 | default: |
| 748 | $secondaryCache = $this->getSecondaryCache( $parserOptions ); |
| 749 | $workKey = $secondaryCache->makeParserOutputKeyOptionalRevId( $revision, $parserOptions ); |
| 750 | } |
| 751 | |
| 752 | $pool = $this->poolCounterFactory->create( $profile, $workKey ); |
| 753 | return new PoolCounterWorkViaCallback( |
| 754 | $pool, |
| 755 | $workKey, |
| 756 | $callbacks |
| 757 | ); |
| 758 | } |
| 759 | |
| 760 | private function getPrimaryCache( ParserOptions $pOpts ): ParserCache { |
| 761 | $name = $pOpts->getUseParsoid() ? self::PARSOID_PCACHE_NAME : ParserCacheFactory::DEFAULT_NAME; |
| 762 | if ( $pOpts->getPostproc() ) { |
| 763 | $name = self::POSTPROC_CACHE_PREFIX . $name; |
| 764 | } |
| 765 | return $this->parserCacheFactory->getParserCache( $name ); |
| 766 | } |
| 767 | |
| 768 | private function getSecondaryCache( ParserOptions $pOpts ): RevisionOutputCache { |
| 769 | $name = $pOpts->getUseParsoid() ? self::PARSOID_RCACHE_NAME : ParserCacheFactory::DEFAULT_RCACHE_NAME; |
| 770 | if ( $pOpts->getPostproc() ) { |
| 771 | $name = self::POSTPROC_CACHE_PREFIX . $name; |
| 772 | } |
| 773 | return $this->parserCacheFactory->getRevisionOutputCache( $name ); |
| 774 | } |
| 775 | |
| 776 | private function startOperationSpan( |
| 777 | string $opName, |
| 778 | PageRecord $page, |
| 779 | ?RevisionRecord $revision = null |
| 780 | ): SpanInterface { |
| 781 | $span = $this->tracer->createSpan( "ParserOutputAccess::$opName" ); |
| 782 | if ( $span->getContext()->isSampled() ) { |
| 783 | $span->setAttributes( [ |
| 784 | 'org.wikimedia.parser.page' => $page->__toString(), |
| 785 | 'org.wikimedia.parser.page.id' => $page->getId(), |
| 786 | 'org.wikimedia.parser.page.wiki' => $page->getWikiId(), |
| 787 | ] ); |
| 788 | if ( $revision ) { |
| 789 | $span->setAttributes( [ |
| 790 | 'org.wikimedia.parser.revision.id' => $revision->getId(), |
| 791 | 'org.wikimedia.parser.revision.parent_id' => $revision->getParentId(), |
| 792 | ] ); |
| 793 | } |
| 794 | } |
| 795 | return $span->start()->activate(); |
| 796 | } |
| 797 | |
| 798 | /** |
| 799 | * Clear the local cache |
| 800 | * @since 1.45 |
| 801 | */ |
| 802 | public function clearLocalCache() { |
| 803 | $this->localCache->clear(); |
| 804 | } |
| 805 | |
| 806 | private function saveToCache( |
| 807 | ParserOptions $parserOptions, ParserOutput $output, PageRecord $page, RevisionRecord $revision, array $options |
| 808 | ): void { |
| 809 | $useCache = $this->shouldUseCache( $page, $revision ); |
| 810 | if ( !$options[ self::OPT_NO_UPDATE_CACHE ] && $output->isCacheable() ) { |
| 811 | if ( $useCache === self::CACHE_PRIMARY ) { |
| 812 | $primaryCache = $this->getPrimaryCache( $parserOptions ); |
| 813 | $primaryCache->save( $output, $page, $parserOptions ); |
| 814 | } elseif ( $useCache === self::CACHE_SECONDARY ) { |
| 815 | $secondaryCache = $this->getSecondaryCache( $parserOptions ); |
| 816 | $secondaryCache->save( $output, $revision, $parserOptions ); |
| 817 | } |
| 818 | } |
| 819 | } |
| 820 | |
| 821 | /** |
| 822 | * Postprocess the given ParserOutput. |
| 823 | * @internal |
| 824 | */ |
| 825 | public function postprocess( |
| 826 | ParserOutput $output, ParserOptions $parserOptions, |
| 827 | PageRecord $page, ?RevisionRecord $revision = null |
| 828 | ): ParserOutput { |
| 829 | $useCache = $this->shouldUseCache( $page, $revision ); |
| 830 | return self::postprocessInPipeline( |
| 831 | $this->outputTransformPipeline, $output, $parserOptions, $page, |
| 832 | fn ( $used ) => match ( $useCache ) { |
| 833 | self::CACHE_NONE => null, |
| 834 | self::CACHE_PRIMARY => |
| 835 | $this->getPrimaryCache( $parserOptions ) |
| 836 | ->makeParserOutputKey( $page, $parserOptions, $used ), |
| 837 | self::CACHE_SECONDARY => |
| 838 | $this->getSecondaryCache( $parserOptions ) |
| 839 | // @phan-suppress-next-line PhanTypeMismatchArgumentNullable $revision is non-null here |
| 840 | ->makeParserOutputKey( $revision, $parserOptions, $used ), |
| 841 | } |
| 842 | ); |
| 843 | } |
| 844 | |
| 845 | /** |
| 846 | * Postprocess the given ParserOutput in the given pipeline. |
| 847 | * |
| 848 | * If debugging information is requested in the parser options, |
| 849 | * $getCache will be used to obtain an appropriate ParserCache |
| 850 | * in order to include the cache key in the debugging output. |
| 851 | * |
| 852 | * @param OutputTransformPipeline $outputTransformPipeline |
| 853 | * @param ParserOutput $output |
| 854 | * @param ParserOptions $parserOptions |
| 855 | * @param PageRecord $page |
| 856 | * @param callable(array):(string|null) $getCacheKey a function to return |
| 857 | * the cache key used, given an array of used parser options; null |
| 858 | * indicates that the postprocessed output will not be cached |
| 859 | * @return ParserOutput a new ParserOutput |
| 860 | */ |
| 861 | public static function postprocessInPipeline( |
| 862 | OutputTransformPipeline $outputTransformPipeline, |
| 863 | ParserOutput $output, ParserOptions $parserOptions, PageRecord $page, |
| 864 | callable $getCacheKey |
| 865 | ): ParserOutput { |
| 866 | // Kludgey workaround: extract $textOptions from the $parserOptions |
| 867 | $textOptions = []; |
| 868 | // Don't add these to the used options set of $output because we |
| 869 | // don't want to mutate that, and the actual return value ParserOutput |
| 870 | // doesn't yet exist. |
| 871 | $parserOptions->registerWatcher( null ); |
| 872 | foreach ( ParserOptions::$postprocOptions as $key ) { |
| 873 | $textOptions[$key] = $parserOptions->getOption( $key ); |
| 874 | } |
| 875 | $textOptions = [ |
| 876 | 'allowClone' => true, |
| 877 | ] + $textOptions; |
| 878 | |
| 879 | $output = $outputTransformPipeline->run( $output, $parserOptions, $textOptions ); |
| 880 | // Ensure this ParserOptions is watching the resulting ParserOutput, |
| 881 | // now that it exists. |
| 882 | $parserOptions->registerWatcher( $output->recordOption( ... ) ); |
| 883 | // Ensure "postproc" is in the set of used options |
| 884 | // (Probably not necessary, but it doesn't hurt to be safe.) |
| 885 | $parserOptions->getPostproc(); |
| 886 | // Ensure all postprocOptions are in the set of used options |
| 887 | // (Since we can't detect accesses via $textOptions) |
| 888 | foreach ( ParserOptions::$postprocOptions as $key ) { |
| 889 | $parserOptions->getOption( $key ); |
| 890 | } |
| 891 | // Add a cache message if debug info is requested (this used to |
| 892 | // be part of $textOptions) |
| 893 | $keyForDebugInfo = $parserOptions->getOption( 'includeDebugInfo' ) ? |
| 894 | $getCacheKey( $output->getUsedOptions() ) : null; |
| 895 | if ( $keyForDebugInfo !== null ) { |
| 896 | # Note that we can't make the key before postprocessing because |
| 897 | # the set of used options may vary during postprocessing; similarly |
| 898 | # we can't use ParserOutput::addCacheMsg() because the |
| 899 | # RenderDebugInfo stage has already run by the time we get here. |
| 900 | # So add the debug info "the hard way", but consistent with how |
| 901 | # RenderDebugInfo does it. |
| 902 | $timestamp = MWTimestamp::now(); |
| 903 | $msg = "Post-processing cache key $keyForDebugInfo, generated at $timestamp"; |
| 904 | // Sanitize for comment. Note '‐' in the replacement is U+2010, |
| 905 | // which looks much like the problematic '-'. |
| 906 | $msg = str_replace( [ '-', '>' ], [ '‐', '>' ], $msg ); |
| 907 | $output->setContentHolderText( |
| 908 | $output->getContentHolderText() . "<!--\n$msg\n-->" |
| 909 | ); |
| 910 | } |
| 911 | return $output; |
| 912 | } |
| 913 | |
| 914 | private function shouldCheckCache( ParserOptions $parserOptions, array $options ): bool { |
| 915 | if ( $options[ self::OPT_NO_CHECK_CACHE ] ) { |
| 916 | return false; |
| 917 | } |
| 918 | if ( $parserOptions->getPostproc() ) { |
| 919 | return !$options[ self::OPT_NO_POSTPROC_CACHE ]; |
| 920 | } |
| 921 | return true; |
| 922 | } |
| 923 | } |