61 public const OPT_NO_CHECK_CACHE = 1;
64 public const OPT_FORCE_PARSE = self::OPT_NO_CHECK_CACHE;
69 public const OPT_NO_UPDATE_CACHE = 2;
76 public const OPT_NO_AUDIENCE_CHECK = 4;
82 public const OPT_NO_CACHE = self::OPT_NO_UPDATE_CACHE | self::OPT_NO_CHECK_CACHE;
88 public const OPT_LINKS_UPDATE = 8;
110 public const OPT_IGNORE_PROFILE_VERSION = 128;
115 public const OPT_NO_POSTPROC_CACHE = 256;
142 private const DEFAULT_OPTIONS = [
143 self::OPT_POOL_COUNTER =>
null,
144 self::OPT_POOL_COUNTER_FALLBACK => false
151 private const CACHE_PRIMARY =
'primary';
154 private const CACHE_SECONDARY =
'secondary';
172 private LoggerInterface $logger;
175 private readonly
Config $config,
187 $this->parserCacheFactory = $parserCacheFactory;
188 $this->revisionLookup = $revisionLookup;
189 $this->revisionRenderer = $revisionRenderer;
190 $this->statsFactory = $statsFactory;
191 $this->chronologyProtector = $chronologyProtector;
192 $this->wikiPageFactory = $wikiPageFactory;
193 $this->titleFormatter = $titleFormatter;
194 $this->tracer = $tracer;
195 $this->poolCounterFactory = $poolCounterFactory;
198 $this->logger =
new NullLogger();
201 public function setLogger( LoggerInterface $logger ): void {
202 $this->logger = $logger;
216 private static function normalizeOptions( $options ): array {
221 if ( is_array( $options ) ) {
222 if ( $options[
'_normalized_'] ??
false ) {
229 foreach ( $options as $opt => $enabled ) {
230 if ( is_int( $opt ) && $enabled ===
true ) {
240 for ( $b = self::OPT_NO_CHECK_CACHE; $b <= self::OPT_NO_POSTPROC_CACHE; $b <<= 1 ) {
241 $options[$b] = (bool)( $bits & $b );
244 if ( $options[ self::OPT_FOR_ARTICLE_VIEW ] ) {
245 $options[ self::OPT_POOL_COUNTER ] = self::POOL_COUNTER_ARTICLE_VIEW;
246 $options[ self::OPT_POOL_COUNTER_FALLBACK ] =
true;
249 $options += self::DEFAULT_OPTIONS;
251 $options[
'_normalized_'] =
true;
263 private function shouldUseCache(
267 if ( $rev && !$rev->getId() ) {
276 $wikiPage = $this->wikiPageFactory->newFromTitle( $page );
277 if ( !$page->exists() || !$wikiPage->getContentHandler()->isParserCacheSupported() ) {
281 $isOld = $rev && $rev->getId() !== $page->getLatest();
283 return self::CACHE_PRIMARY;
286 if ( !$rev->audienceCan( RevisionRecord::DELETED_TEXT, RevisionRecord::FOR_PUBLIC ) ) {
291 return self::CACHE_SECONDARY;
311 $options = self::normalizeOptions( $options );
313 $span = $this->startOperationSpan( __FUNCTION__, $page, $revision );
315 $useCache = $this->shouldUseCache( $page, $revision );
316 $primaryCache = $this->getPrimaryCache( $parserOptions );
317 $classCacheKey = $primaryCache->makeParserOutputKey( $page, $parserOptions );
319 if ( $useCache === self::CACHE_PRIMARY ) {
320 if ( !$isOld && $this->localCache->hasField( $classCacheKey, $page->
getLatest() ) ) {
321 return $this->localCache->getField( $classCacheKey, $page->
getLatest() );
323 $output = $primaryCache->get( $page, $parserOptions );
324 } elseif ( $useCache === self::CACHE_SECONDARY && $revision ) {
325 $secondaryCache = $this->getSecondaryCache( $parserOptions );
326 $output = $secondaryCache->get( $revision, $parserOptions );
331 $statType = $statReason = $output ?
'hit' :
'miss';
334 $output && !$options[ self::OPT_IGNORE_PROFILE_VERSION ] &&
339 $cachedVersion = $pageBundle->version ??
null;
341 $cachedVersion !==
null &&
342 $cachedVersion !== Parsoid::defaultHTMLVersion()
345 $statReason =
'obsolete';
350 if ( $output && !$isOld && !$parserOptions->
getPostproc() ) {
351 $this->localCache->setField( $classCacheKey, $page->
getLatest(), $output );
355 ->getCounter(
'parseroutputaccess_cache_total' )
356 ->setLabel(
'cache', $useCache )
357 ->setLabel(
'reason', $statReason )
358 ->setLabel(
'type', $statType )
359 ->setLabel(
'postproc', $parserOptions->
getPostproc() ?
'true' :
'false' )
362 return $output ?:
null;
371 private function getFallbackOutputForLatest(
373 ParserOptions $parserOptions,
377 $parserOutput = $this->getPrimaryCache( $parserOptions )
378 ->getDirty( $page, $parserOptions );
380 if ( !$parserOutput ) {
381 $this->logger->info(
'dirty missing' );
395 if ( $this->chronologyProtector->getTouched() ) {
397 'declining fast-fallback to stale output since ChronologyProtector ' .
398 'reports the client recently made changes',
399 [
'workKey' => $workKey ]
408 $this->logger->info( $fast ?
'fast dirty output' :
'dirty output', [
'workKey' => $workKey ] );
410 $status = Status::newGood( $parserOutput );
411 $status->warning(
'view-pool-dirty-output' );
412 $status->warning( $fast ?
'view-pool-contention' :
'view-pool-overload' );
446 $options = self::normalizeOptions( $options );
448 $span = $this->startOperationSpan( __FUNCTION__, $page, $revision );
449 $error = $this->checkPreconditions( $page, $revision, $options );
452 ->getCounter(
'parseroutputaccess_case' )
453 ->setLabel(
'case',
'error' )
454 ->setLabel(
'postproc', $parserOptions->
getPostproc() ?
'true' :
'false' )
462 ->getCounter(
'parseroutputaccess_case' )
463 ->setLabel(
'case',
'old' )
464 ->setLabel(
'postproc', $parserOptions->
getPostproc() ?
'true' :
'false' )
468 ->getCounter(
'parseroutputaccess_case' )
469 ->setLabel(
'case',
'current' )
470 ->setLabel(
'postproc', $parserOptions->
getPostproc() ?
'true' :
'false' )
474 if ( $this->shouldCheckCache( $parserOptions, $options ) ) {
475 $output = $this->getCachedParserOutput( $page, $parserOptions, $revision );
477 return Status::newGood( $output );
483 $revision = $revId ? $this->revisionLookup->getRevisionById( $revId ) :
null;
487 ->getCounter(
'parseroutputaccess_status' )
488 ->setLabel(
'status',
'norev' )
489 ->setLabel(
'postproc', $parserOptions->
getPostproc() ?
'true' :
'false' )
491 return Status::newFatal(
'missing-revision', $revId );
495 if ( $options[ self::OPT_POOL_COUNTER ] ) {
496 $work = $this->newPoolWork( $page, $parserOptions, $revision, $options );
498 $status = $work->execute();
502 $this->statsFactory->getCounter(
'parseroutputaccess_render_total' )
503 ->setLabel(
'pool',
'none' )
504 ->setLabel(
'cache', self::CACHE_NONE )
505 ->setLabel(
'postproc', $parserOptions->
getPostproc() ?
'true' :
'false' )
508 $status = $this->renderRevision( $page, $parserOptions, $revision, $options,
null );
511 $output = $status->getValue();
512 Assert::postcondition( $output || !$status->isOK(),
'Inconsistent status' );
517 if ( $output && !$isOld && !$parserOptions->
getPostproc() ) {
518 $primaryCache = $this->getPrimaryCache( $parserOptions );
519 $classCacheKey = $primaryCache->makeParserOutputKey( $page, $parserOptions );
520 $this->localCache->setField( $classCacheKey, $page->
getLatest(), $output );
524 'postproc' => $parserOptions->
getPostproc() ?
'true' :
'false',
525 'status' => $status->isGood() ?
'good' : ( $status->isOK() ?
'ok' :
'error' ),
528 $this->statsFactory->getCounter(
'parseroutputaccess_status' )
529 ->setLabels( $labels )
550 private function renderRevision(
552 ParserOptions $parserOptions,
553 RevisionRecord $revision,
555 ?ParserOutput $previousOutput =
null
557 $span = $this->startOperationSpan( __FUNCTION__, $page, $revision );
559 $isCurrent = $revision->getId() === $page->getLatest();
563 $sampleRate = $this->config->get(
564 MainConfigNames::ParsoidSelectiveUpdateSampleRate
566 $doSample = ( $sampleRate && mt_rand( 1, $sampleRate ) === 1 );
568 if ( $previousOutput ===
null && ( $doSample || $parserOptions->getUseParsoid() ) ) {
574 if ( $this->shouldCheckCache( $parserOptions, $options ) ) {
575 $previousOutput = $this->getPrimaryCache( $parserOptions )->getDirty( $page, $parserOptions ) ?:
null;
580 if ( $parserOptions->getPostproc() ) {
581 $preParserOptions = $parserOptions->clearPostproc();
582 $preStatus = $this->getParserOutput( $page, $preParserOptions, $revision, $options );
583 $output = $preStatus->getValue();
585 $output = $this->postprocess( $output, $parserOptions, $page, $revision );
588 $renderedRev = $this->revisionRenderer->getRenderedRevision( $revision, $parserOptions,
null, [
589 'audience' => RevisionRecord::RAW,
590 'previous-output' => $previousOutput,
593 $output = $renderedRev->getRevisionParserOutput();
597 # Keep these labels in sync with those in RefreshLinksJob
599 'source' =>
'ParserOutputAccess',
600 'type' => $previousOutput ===
null ?
'full' :
'selective',
601 'reason' => $parserOptions->getRenderReason(),
602 'parser' => $parserOptions->getUseParsoid() ?
'parsoid' :
'legacy',
603 'opportunistic' =>
'false',
604 'wiki' => WikiMap::getCurrentWikiId(),
605 'model' => $revision->getMainContentModel(),
606 'postproc' => $parserOptions->getPostproc() ?
'true' :
'false',
609 ->getCounter(
'ParserCache_selective_total' )
610 ->setLabels( $labels )
613 ->getCounter(
'ParserCache_selective_cpu_seconds' )
614 ->setLabels( $labels )
618 $res = Status::newGood( $output );
620 $res->merge( $preStatus );
623 if ( $output && $res->isGood() ) {
625 $this->saveToCache( $parserOptions, $output, $page, $revision, $options );
628 if ( $output && $options[ self::OPT_LINKS_UPDATE ] && !$parserOptions->getPostproc() ) {
629 $this->wikiPageFactory->newFromTitle( $page )
630 ->triggerOpportunisticLinksUpdate( $output );
636 private function checkPreconditions(
638 ?RevisionRecord $revision =
null,
641 if ( !$page->exists() ) {
642 return Status::newFatal(
'nopagetext' );
645 if ( !$options[ self::OPT_NO_UPDATE_CACHE ] && $revision && !$revision->getId() ) {
646 throw new InvalidArgumentException(
647 'The revision does not have a known ID. Use OPT_NO_CACHE.'
651 if ( $revision && $revision->getPageId() !== $page->getId() ) {
652 throw new InvalidArgumentException(
653 'The revision does not belong to the given page.'
657 if ( $revision && !$options[ self::OPT_NO_AUDIENCE_CHECK ] ) {
660 if ( !$revision->audienceCan( RevisionRecord::DELETED_TEXT, RevisionRecord::FOR_PUBLIC ) ) {
661 return Status::newFatal(
662 'missing-revision-permission',
664 $revision->getTimestamp(),
665 $this->titleFormatter->getPrefixedURL( $page )
679 $profile = $options[ self::OPT_POOL_COUNTER ];
681 $options[self::OPT_POOL_COUNTER] =
false;
684 'doWork' => fn () => $this->renderRevision(
691 'doCachedWork' =>
static fn () =>
false,
693 'fallback' =>
static fn ( $fast ) =>
false,
694 'error' =>
static fn ( $status ) => $status,
697 $useCache = $this->shouldUseCache( $page, $revision );
699 $this->statsFactory->getCounter(
'parseroutputaccess_render_total' )
700 ->setLabel(
'pool',
'articleview' )
701 ->setLabel(
'cache', $useCache )
702 ->setLabel(
'postproc', $parserOptions->
getPostproc() ?
'true' :
'false' )
705 switch ( $useCache ) {
706 case self::CACHE_PRIMARY:
707 $primaryCache = $this->getPrimaryCache( $parserOptions );
708 $parserCacheMetadata = $primaryCache->getMetadata( $page );
709 $cacheKey = $primaryCache->makeParserOutputKey( $page, $parserOptions,
710 $parserCacheMetadata ? $parserCacheMetadata->getUsedOptions() : null
713 $workKey = $cacheKey .
':revid:' . $revision->
getId();
715 $callbacks[
'doCachedWork'] =
716 static function () use ( $primaryCache, $page, $parserOptions ) {
717 $parserOutput = $primaryCache->get( $page, $parserOptions );
718 return $parserOutput ? Status::newGood( $parserOutput ) : false;
721 $callbacks[
'fallback'] =
722 function ( $fast ) use ( $page, $parserOptions, $workKey, $options ) {
723 if ( $options[ self::OPT_POOL_COUNTER_FALLBACK ] ) {
724 return $this->getFallbackOutputForLatest(
725 $page, $parserOptions, $workKey, $fast
734 case self::CACHE_SECONDARY:
735 $secondaryCache = $this->getSecondaryCache( $parserOptions );
736 $workKey = $secondaryCache->makeParserOutputKey( $revision, $parserOptions );
738 $callbacks[
'doCachedWork'] =
739 static function () use ( $secondaryCache, $revision, $parserOptions ) {
740 $parserOutput = $secondaryCache->get( $revision, $parserOptions );
742 return $parserOutput ? Status::newGood( $parserOutput ) : false;
748 $secondaryCache = $this->getSecondaryCache( $parserOptions );
749 $workKey = $secondaryCache->makeParserOutputKeyOptionalRevId( $revision, $parserOptions );
752 $pool = $this->poolCounterFactory->create( $profile, $workKey );
753 return new PoolCounterWorkViaCallback(
760 private function getPrimaryCache( ParserOptions $pOpts ): ParserCache {
761 $name = $pOpts->getUseParsoid() ? self::PARSOID_PCACHE_NAME : ParserCacheFactory::DEFAULT_NAME;
762 if ( $pOpts->getPostproc() ) {
763 $name = self::POSTPROC_CACHE_PREFIX . $name;
765 return $this->parserCacheFactory->getParserCache( $name );
768 private function getSecondaryCache( ParserOptions $pOpts ): RevisionOutputCache {
769 $name = $pOpts->getUseParsoid() ? self::PARSOID_RCACHE_NAME : ParserCacheFactory::DEFAULT_RCACHE_NAME;
770 if ( $pOpts->getPostproc() ) {
771 $name = self::POSTPROC_CACHE_PREFIX . $name;
773 return $this->parserCacheFactory->getRevisionOutputCache( $name );
776 private function startOperationSpan(
779 ?RevisionRecord $revision =
null
781 $span = $this->tracer->createSpan(
"ParserOutputAccess::$opName" );
782 if ( $span->getContext()->isSampled() ) {
783 $span->setAttributes( [
784 'org.wikimedia.parser.page' => $page->__toString(),
785 'org.wikimedia.parser.page.id' => $page->getId(),
786 'org.wikimedia.parser.page.wiki' => $page->getWikiId(),
789 $span->setAttributes( [
790 'org.wikimedia.parser.revision.id' => $revision->
getId(),
791 'org.wikimedia.parser.revision.parent_id' => $revision->
getParentId(),
795 return $span->start()->activate();
803 $this->localCache->clear();
806 private function saveToCache(
809 $useCache = $this->shouldUseCache( $page, $revision );
810 if ( !$options[ self::OPT_NO_UPDATE_CACHE ] && $output->
isCacheable() ) {
811 if ( $useCache === self::CACHE_PRIMARY ) {
812 $primaryCache = $this->getPrimaryCache( $parserOptions );
813 $primaryCache->save( $output, $page, $parserOptions );
814 } elseif ( $useCache === self::CACHE_SECONDARY ) {
815 $secondaryCache = $this->getSecondaryCache( $parserOptions );
816 $secondaryCache->save( $output, $revision, $parserOptions );
829 $useCache = $this->shouldUseCache( $page, $revision );
830 return self::postprocessInPipeline(
831 $this->outputTransformPipeline, $output, $parserOptions, $page,
832 fn ( $used ) => match ( $useCache ) {
833 self::CACHE_NONE =>
null,
834 self::CACHE_PRIMARY =>
835 $this->getPrimaryCache( $parserOptions )
836 ->makeParserOutputKey( $page, $parserOptions, $used ),
837 self::CACHE_SECONDARY =>
838 $this->getSecondaryCache( $parserOptions )
840 ->makeParserOutputKey( $revision, $parserOptions, $used ),
864 callable $getCacheKey
872 foreach ( ParserOptions::$postprocOptions as $key ) {
873 $textOptions[$key] = $parserOptions->
getOption( $key );
876 'allowClone' =>
true,
879 $output = $outputTransformPipeline->
run( $output, $parserOptions, $textOptions );
888 foreach ( ParserOptions::$postprocOptions as $key ) {
893 $keyForDebugInfo = $parserOptions->
getOption(
'includeDebugInfo' ) ?
895 if ( $keyForDebugInfo !==
null ) {
896 # Note that we can't make the key before postprocessing because
897 # the set of used options may vary during postprocessing; similarly
898 # we can't use ParserOutput::addCacheMsg() because the
899 # RenderDebugInfo stage has already run by the time we get here.
900 # So add the debug info "the hard way", but consistent with how
901 # RenderDebugInfo does it.
902 $timestamp = MWTimestamp::now();
903 $msg =
"Post-processing cache key $keyForDebugInfo, generated at $timestamp";
906 $msg = str_replace( [
'-',
'>' ], [
'‐',
'>' ], $msg );
914 private function shouldCheckCache( ParserOptions $parserOptions, array $options ): bool {
915 if ( $options[ self::OPT_NO_CHECK_CACHE ] ) {
919 return !$options[ self::OPT_NO_POSTPROC_CACHE ];