59 public const OPT_NO_CHECK_CACHE = 1;
62 public const OPT_FORCE_PARSE = self::OPT_NO_CHECK_CACHE;
67 public const OPT_NO_UPDATE_CACHE = 2;
74 public const OPT_NO_AUDIENCE_CHECK = 4;
80 public const OPT_NO_CACHE = self::OPT_NO_UPDATE_CACHE | self::OPT_NO_CHECK_CACHE;
86 public const OPT_LINKS_UPDATE = 8;
108 public const OPT_IGNORE_PROFILE_VERSION = 128;
113 public const OPT_NO_POSTPROC_CACHE = 256;
140 private const DEFAULT_OPTIONS = [
141 self::OPT_POOL_COUNTER =>
null,
142 self::OPT_POOL_COUNTER_FALLBACK => false
149 private const CACHE_PRIMARY =
'primary';
152 private const CACHE_SECONDARY =
'secondary';
170 private LoggerInterface $logger;
183 $this->parserCacheFactory = $parserCacheFactory;
184 $this->revisionLookup = $revisionLookup;
185 $this->revisionRenderer = $revisionRenderer;
186 $this->statsFactory = $statsFactory;
187 $this->chronologyProtector = $chronologyProtector;
188 $this->wikiPageFactory = $wikiPageFactory;
189 $this->titleFormatter = $titleFormatter;
190 $this->tracer = $tracer;
191 $this->poolCounterFactory = $poolCounterFactory;
194 $this->logger =
new NullLogger();
197 public function setLogger( LoggerInterface $logger ): void {
198 $this->logger = $logger;
212 private static function normalizeOptions( $options ): array {
217 if ( is_array( $options ) ) {
218 if ( $options[
'_normalized_'] ??
false ) {
225 foreach ( $options as $opt => $enabled ) {
226 if ( is_int( $opt ) && $enabled ===
true ) {
236 for ( $b = self::OPT_NO_CHECK_CACHE; $b <= self::OPT_NO_POSTPROC_CACHE; $b <<= 1 ) {
237 $options[$b] = (bool)( $bits & $b );
240 if ( $options[ self::OPT_FOR_ARTICLE_VIEW ] ) {
241 $options[ self::OPT_POOL_COUNTER ] = self::POOL_COUNTER_ARTICLE_VIEW;
242 $options[ self::OPT_POOL_COUNTER_FALLBACK ] =
true;
245 $options += self::DEFAULT_OPTIONS;
247 $options[
'_normalized_'] =
true;
259 private function shouldUseCache(
263 if ( $rev && !$rev->getId() ) {
272 $wikiPage = $this->wikiPageFactory->newFromTitle( $page );
273 if ( !$page->exists() || !$wikiPage->getContentHandler()->isParserCacheSupported() ) {
277 $isOld = $rev && $rev->getId() !== $page->getLatest();
279 return self::CACHE_PRIMARY;
282 if ( !$rev->audienceCan( RevisionRecord::DELETED_TEXT, RevisionRecord::FOR_PUBLIC ) ) {
287 return self::CACHE_SECONDARY;
307 $options = self::normalizeOptions( $options );
309 $span = $this->startOperationSpan( __FUNCTION__, $page, $revision );
311 $useCache = $this->shouldUseCache( $page, $revision );
312 $primaryCache = $this->getPrimaryCache( $parserOptions );
313 $classCacheKey = $primaryCache->makeParserOutputKey( $page, $parserOptions );
315 if ( $useCache === self::CACHE_PRIMARY ) {
316 if ( !$isOld && $this->localCache->hasField( $classCacheKey, $page->
getLatest() ) ) {
317 return $this->localCache->getField( $classCacheKey, $page->
getLatest() );
319 $output = $primaryCache->get( $page, $parserOptions );
320 } elseif ( $useCache === self::CACHE_SECONDARY && $revision ) {
321 $secondaryCache = $this->getSecondaryCache( $parserOptions );
322 $output = $secondaryCache->get( $revision, $parserOptions );
327 $statType = $statReason = $output ?
'hit' :
'miss';
330 $output && !$options[ self::OPT_IGNORE_PROFILE_VERSION ] &&
335 $cachedVersion = $pageBundle->version ??
null;
337 $cachedVersion !==
null &&
338 $cachedVersion !== Parsoid::defaultHTMLVersion()
341 $statReason =
'obsolete';
346 if ( $output && !$isOld && !$parserOptions->
getPostproc() ) {
347 $this->localCache->setField( $classCacheKey, $page->
getLatest(), $output );
351 ->getCounter(
'parseroutputaccess_cache_total' )
352 ->setLabel(
'cache', $useCache )
353 ->setLabel(
'reason', $statReason )
354 ->setLabel(
'type', $statType )
355 ->setLabel(
'postproc', $parserOptions->
getPostproc() ?
'true' :
'false' )
358 return $output ?:
null;
367 private function getFallbackOutputForLatest(
369 ParserOptions $parserOptions,
373 $parserOutput = $this->getPrimaryCache( $parserOptions )
374 ->getDirty( $page, $parserOptions );
376 if ( !$parserOutput ) {
377 $this->logger->info(
'dirty missing' );
391 if ( $this->chronologyProtector->getTouched() ) {
393 'declining fast-fallback to stale output since ChronologyProtector ' .
394 'reports the client recently made changes',
395 [
'workKey' => $workKey ]
404 $this->logger->info( $fast ?
'fast dirty output' :
'dirty output', [
'workKey' => $workKey ] );
406 $status = Status::newGood( $parserOutput );
407 $status->warning(
'view-pool-dirty-output' );
408 $status->warning( $fast ?
'view-pool-contention' :
'view-pool-overload' );
442 $options = self::normalizeOptions( $options );
444 $span = $this->startOperationSpan( __FUNCTION__, $page, $revision );
445 $error = $this->checkPreconditions( $page, $revision, $options );
448 ->getCounter(
'parseroutputaccess_case' )
449 ->setLabel(
'case',
'error' )
450 ->setLabel(
'postproc', $parserOptions->
getPostproc() ?
'true' :
'false' )
458 ->getCounter(
'parseroutputaccess_case' )
459 ->setLabel(
'case',
'old' )
460 ->setLabel(
'postproc', $parserOptions->
getPostproc() ?
'true' :
'false' )
464 ->getCounter(
'parseroutputaccess_case' )
465 ->setLabel(
'case',
'current' )
466 ->setLabel(
'postproc', $parserOptions->
getPostproc() ?
'true' :
'false' )
470 if ( $this->shouldCheckCache( $parserOptions, $options ) ) {
471 $output = $this->getCachedParserOutput( $page, $parserOptions, $revision );
473 return Status::newGood( $output );
479 $revision = $revId ? $this->revisionLookup->getRevisionById( $revId ) :
null;
483 ->getCounter(
'parseroutputaccess_status' )
484 ->setLabel(
'status',
'norev' )
485 ->setLabel(
'postproc', $parserOptions->
getPostproc() ?
'true' :
'false' )
487 return Status::newFatal(
'missing-revision', $revId );
491 if ( $options[ self::OPT_POOL_COUNTER ] ) {
492 $work = $this->newPoolWork( $page, $parserOptions, $revision, $options );
494 $status = $work->execute();
498 $this->statsFactory->getCounter(
'parseroutputaccess_render_total' )
499 ->setLabel(
'pool',
'none' )
500 ->setLabel(
'cache', self::CACHE_NONE )
501 ->setLabel(
'postproc', $parserOptions->
getPostproc() ?
'true' :
'false' )
504 $status = $this->renderRevision( $page, $parserOptions, $revision, $options,
null );
507 $output = $status->getValue();
508 Assert::postcondition( $output || !$status->isOK(),
'Inconsistent status' );
513 if ( $output && !$isOld && !$parserOptions->
getPostproc() ) {
514 $primaryCache = $this->getPrimaryCache( $parserOptions );
515 $classCacheKey = $primaryCache->makeParserOutputKey( $page, $parserOptions );
516 $this->localCache->setField( $classCacheKey, $page->
getLatest(), $output );
520 'postproc' => $parserOptions->
getPostproc() ?
'true' :
'false',
521 'status' => $status->isGood() ?
'good' : ( $status->isOK() ?
'ok' :
'error' ),
524 $this->statsFactory->getCounter(
'parseroutputaccess_status' )
525 ->setLabels( $labels )
546 private function renderRevision(
548 ParserOptions $parserOptions,
549 RevisionRecord $revision,
551 ?ParserOutput $previousOutput =
null
553 $span = $this->startOperationSpan( __FUNCTION__, $page, $revision );
555 $isCurrent = $revision->getId() === $page->getLatest();
559 $sampleRate = MediaWikiServices::getInstance()->getMainConfig()->get(
560 MainConfigNames::ParsoidSelectiveUpdateSampleRate
562 $doSample = ( $sampleRate && mt_rand( 1, $sampleRate ) === 1 );
564 if ( $previousOutput ===
null && ( $doSample || $parserOptions->getUseParsoid() ) ) {
570 if ( $this->shouldCheckCache( $parserOptions, $options ) ) {
571 $previousOutput = $this->getPrimaryCache( $parserOptions )->getDirty( $page, $parserOptions ) ?:
null;
576 if ( $parserOptions->getPostproc() ) {
577 $preParserOptions = $parserOptions->clearPostproc();
578 $preStatus = $this->getParserOutput( $page, $preParserOptions, $revision, $options );
579 $output = $preStatus->getValue();
581 $output = $this->postprocess( $output, $parserOptions );
584 $renderedRev = $this->revisionRenderer->getRenderedRevision( $revision, $parserOptions,
null, [
585 'audience' => RevisionRecord::RAW,
586 'previous-output' => $previousOutput,
589 $output = $renderedRev->getRevisionParserOutput();
594 'source' =>
'ParserOutputAccess',
595 'type' => $previousOutput ===
null ?
'full' :
'selective',
596 'reason' => $parserOptions->getRenderReason(),
597 'parser' => $parserOptions->getUseParsoid() ?
'parsoid' :
'legacy',
598 'opportunistic' =>
'false',
599 'wiki' => WikiMap::getCurrentWikiId(),
600 'model' => $revision->getMainContentModel(),
601 'postproc' => $parserOptions->getPostproc() ?
'true' :
'false',
604 ->getCounter(
'ParserCache_selective_total' )
605 ->setLabels( $labels )
608 ->getCounter(
'ParserCache_selective_cpu_seconds' )
609 ->setLabels( $labels )
613 $res = Status::newGood( $output );
615 $res->merge( $preStatus );
618 if ( $output && $res->isGood() ) {
620 $this->saveToCache( $parserOptions, $output, $page, $revision, $options );
623 if ( $output && $options[ self::OPT_LINKS_UPDATE ] && !$parserOptions->getPostproc() ) {
624 $this->wikiPageFactory->newFromTitle( $page )
625 ->triggerOpportunisticLinksUpdate( $output );
631 private function checkPreconditions(
633 ?RevisionRecord $revision =
null,
636 if ( !$page->exists() ) {
637 return Status::newFatal(
'nopagetext' );
640 if ( !$options[ self::OPT_NO_UPDATE_CACHE ] && $revision && !$revision->getId() ) {
641 throw new InvalidArgumentException(
642 'The revision does not have a known ID. Use OPT_NO_CACHE.'
646 if ( $revision && $revision->getPageId() !== $page->getId() ) {
647 throw new InvalidArgumentException(
648 'The revision does not belong to the given page.'
652 if ( $revision && !$options[ self::OPT_NO_AUDIENCE_CHECK ] ) {
655 if ( !$revision->audienceCan( RevisionRecord::DELETED_TEXT, RevisionRecord::FOR_PUBLIC ) ) {
656 return Status::newFatal(
657 'missing-revision-permission',
659 $revision->getTimestamp(),
660 $this->titleFormatter->getPrefixedURL( $page )
676 'doWork' => function () use ( $page, $parserOptions, $revision, $options ) {
677 return $this->renderRevision(
684 'doCachedWork' =>
static function () {
688 'fallback' =>
static function ( $fast ) {
692 'error' =>
static function ( $status ) {
697 $useCache = $this->shouldUseCache( $page, $revision );
699 $this->statsFactory->getCounter(
'parseroutputaccess_render_total' )
700 ->setLabel(
'pool',
'articleview' )
701 ->setLabel(
'cache', $useCache )
702 ->setLabel(
'postproc', $parserOptions->getPostproc() ?
'true' :
'false' )
705 switch ( $useCache ) {
706 case self::CACHE_PRIMARY:
707 $primaryCache = $this->getPrimaryCache( $parserOptions );
708 $parserCacheMetadata = $primaryCache->getMetadata( $page );
709 $cacheKey = $primaryCache->makeParserOutputKey( $page, $parserOptions,
710 $parserCacheMetadata ? $parserCacheMetadata->getUsedOptions() : null
713 $workKey = $cacheKey .
':revid:' . $revision->getId();
715 $callbacks[
'doCachedWork'] =
716 static function () use ( $primaryCache, $page, $parserOptions ) {
717 $parserOutput = $primaryCache->get( $page, $parserOptions );
718 return $parserOutput ? Status::newGood( $parserOutput ) : false;
721 $callbacks[
'fallback'] =
722 function ( $fast ) use ( $page, $parserOptions, $workKey, $options ) {
723 if ( $options[ self::OPT_POOL_COUNTER_FALLBACK ] ) {
724 return $this->getFallbackOutputForLatest(
725 $page, $parserOptions, $workKey, $fast
734 case self::CACHE_SECONDARY:
735 $secondaryCache = $this->getSecondaryCache( $parserOptions );
736 $workKey = $secondaryCache->makeParserOutputKey( $revision, $parserOptions );
738 $callbacks[
'doCachedWork'] =
739 static function () use ( $secondaryCache, $revision, $parserOptions ) {
740 $parserOutput = $secondaryCache->get( $revision, $parserOptions );
742 return $parserOutput ? Status::newGood( $parserOutput ) : false;
748 $secondaryCache = $this->getSecondaryCache( $parserOptions );
749 $workKey = $secondaryCache->makeParserOutputKeyOptionalRevId( $revision, $parserOptions );
752 $profile = $options[ self::OPT_POOL_COUNTER ];
753 $pool = $this->poolCounterFactory->create( $profile, $workKey );
754 return new PoolCounterWorkViaCallback(
761 private function getPrimaryCache( ParserOptions $pOpts ): ParserCache {
762 $name = $pOpts->getUseParsoid() ? self::PARSOID_PCACHE_NAME : ParserCacheFactory::DEFAULT_NAME;
763 if ( $pOpts->getPostproc() ) {
764 $name = self::POSTPROC_CACHE_PREFIX . $name;
766 return $this->parserCacheFactory->getParserCache( $name );
769 private function getSecondaryCache( ParserOptions $pOpts ): RevisionOutputCache {
770 $name = $pOpts->getUseParsoid() ? self::PARSOID_RCACHE_NAME : ParserCacheFactory::DEFAULT_RCACHE_NAME;
771 if ( $pOpts->getPostproc() ) {
772 $name = self::POSTPROC_CACHE_PREFIX . $name;
774 return $this->parserCacheFactory->getRevisionOutputCache( $name );
777 private function startOperationSpan(
780 ?RevisionRecord $revision =
null
782 $span = $this->tracer->createSpan(
"ParserOutputAccess::$opName" );
783 if ( $span->getContext()->isSampled() ) {
784 $span->setAttributes( [
785 'org.wikimedia.parser.page' => $page->__toString(),
786 'org.wikimedia.parser.page.id' => $page->getId(),
787 'org.wikimedia.parser.page.wiki' => $page->getWikiId(),
790 $span->setAttributes( [
791 'org.wikimedia.parser.revision.id' => $revision->getId(),
792 'org.wikimedia.parser.revision.parent_id' => $revision->getParentId(),
796 return $span->start()->activate();
804 $this->localCache->clear();
807 private function saveToCache(
810 $useCache = $this->shouldUseCache( $page, $revision );
811 if ( !$options[ self::OPT_NO_UPDATE_CACHE ] && $output->
isCacheable() ) {
812 if ( $useCache === self::CACHE_PRIMARY ) {
813 $primaryCache = $this->getPrimaryCache( $parserOptions );
814 $primaryCache->save( $output, $page, $parserOptions );
815 } elseif ( $useCache === self::CACHE_SECONDARY ) {
816 $secondaryCache = $this->getSecondaryCache( $parserOptions );
817 $secondaryCache->save( $output, $revision, $parserOptions );
832 foreach ( ParserOptions::$postprocOptions as $key ) {
833 $textOptions[$key] = $parserOptions->
getOption( $key );
836 'allowClone' =>
true,
839 $pipeline = MediaWikiServices::getInstance()->getDefaultOutputPipeline();
840 $output = $pipeline->run( $output, $parserOptions, $textOptions );
849 foreach ( ParserOptions::$postprocOptions as $key ) {
855 private function shouldCheckCache( ParserOptions $parserOptions, array $options ): bool {
856 if ( $options[ self::OPT_NO_CHECK_CACHE ] ) {
860 return !$options[ self::OPT_NO_POSTPROC_CACHE ];