ParsoidParser.php

Go to the documentation of this file.

<?php
declare( strict_types = 1 );
 
namespace MediaWiki\Parser\Parsoid;
 
use MediaWiki\Category\TrackingCategories;
use MediaWiki\Content\TextContent;
use MediaWiki\Content\WikitextContent;
use MediaWiki\Context\RequestContext;
use MediaWiki\Language\LanguageConverterFactory;
use MediaWiki\MainConfigNames;
use MediaWiki\MediaWikiServices;
use MediaWiki\Page\PageReference;
use MediaWiki\Parser\ParserOptions;
use MediaWiki\Parser\ParserOutput;
use MediaWiki\Parser\ParserOutputFlags;
use MediaWiki\Parser\Parsoid\Config\DataAccess;
use MediaWiki\Parser\Parsoid\Config\PageConfigFactory;
use MediaWiki\Revision\MutableRevisionRecord;
use MediaWiki\Revision\SlotRecord;
use MediaWiki\Title\NamespaceInfo;
use MediaWiki\Title\Title;
use MediaWiki\WikiMap\WikiMap;
use Wikimedia\Assert\Assert;
use Wikimedia\Parsoid\Config\PageConfig;
use Wikimedia\Parsoid\Config\SiteConfig;
use Wikimedia\Parsoid\Parsoid;
 
class ParsoidParser /* eventually this will extend \Parser */ {
    public function __construct(
        private Parsoid $parsoid,
        private readonly PageConfigFactory $pageConfigFactory,
        private readonly LanguageConverterFactory $languageConverterFactory,
        private readonly SiteConfig $siteConfig,
        private readonly DataAccess $dataAccess,
        private readonly NamespaceInfo $namespaceInfo,
        private readonly TrackingCategories $trackingCategories,
    ) {
    }
 
    private function genParserOutput(
        PageConfig $pageConfig, ParserOptions $options, ?ParserOutput $previousOutput
    ): ParserOutput {
        $parserOutput = new ParserOutput();
 
        // Parsoid itself does not vary output by parser options right now.
        // But, ensure that any option use by extensions, parser functions,
        // recursive parses, or (in the unlikely future scenario) Parsoid itself
        // are recorded as used.
        $options->registerWatcher( $parserOutput->recordOption( ... ) );
 
        // The enable/disable logic here matches that in Parser::internalParseHalfParsed(),
        // although __NOCONTENTCONVERT__ is handled internal to Parsoid.
        //
        // T349137: It might be preferable to handle __NOCONTENTCONVERT__ here rather than
        // by inspecting the DOM inside Parsoid. That will come in a separate patch.
        $htmlVariantLanguage = null;
        if ( !( $options->getDisableContentConversion() || $options->getInterfaceMessage() ) ) {
            // NOTES (some of these are TODOs for read views integration)
            // 1. This html variant conversion is a pre-cache transform. HtmlOutputRendererHelper
            //    has another variant conversion that is a post-cache transform based on the
            //    'Accept-Language' header. If that header is set, there is really no reason to
            //    do this conversion here. So, eventually, we are likely to either not pass in
            //    the htmlVariantLanguage option below OR disable language conversion from the
            //    wt2html path in Parsoid and this and the Accept-Language variant conversion
            //    both would have to be handled as post-cache transforms.
            //
            // 2. Parser.php calls convert() which computes a preferred variant from the
            //    target language. But, we cannot do that unconditionally here because REST API
            //    requests specify the exact variant via the 'Content-Language' header.
            //
            //    For Parsoid page views, either the callers will have to compute the
            //    preferred variant and set it in ParserOptions OR the REST API will have
            //    to set some other flag indicating that the preferred variant should not
            //    be computed. For now, I am adding a temporary hack, but this should be
            //    replaced with something more sensible (T267067).
            //
            // 3. Additionally, Parsoid's callers will have to set targetLanguage in ParserOptions
            //    to mimic the logic in Parser.php (missing right now).
            $langCode = $pageConfig->getPageLanguageBcp47();
            // TEMPORARY HACK
            if ( $options->getRenderReason() === 'page_view' || $options->getRenderReason() === 'page_view_old' ) {
                $langFactory = MediaWikiServices::getInstance()->getLanguageFactory();
                $lang = $langFactory->getLanguage( $langCode );
                $langConv = $this->languageConverterFactory->getLanguageConverter( $lang );
                $htmlVariantLanguage = $langFactory->getLanguage( $langConv->getPreferredVariant() );
            } else {
                $htmlVariantLanguage = $langCode;
            }
        }
        $oldPageConfig = null;
        $oldPageBundle = null;
 
        // T371713: Temporary statistics collection code to determine
        // feasibility of Parsoid selective update
        $sampleRate = MediaWikiServices::getInstance()->getMainConfig()->get(
            MainConfigNames::ParsoidSelectiveUpdateSampleRate
        );
        $doSample = ( $sampleRate && mt_rand( 1, $sampleRate ) === 1 );
        if ( $doSample && $previousOutput !== null && $previousOutput->getCacheRevisionId() ) {
            // Allow fetching the old wikitext corresponding to the
            // $previousOutput
            $oldPageConfig = $this->pageConfigFactory->createFromParserOptions(
                $options,
                Title::newFromLinkTarget( $pageConfig->getLinkTarget() ),
                $previousOutput->getCacheRevisionId(),
                $previousOutput->getLanguage()
            );
            $oldPageBundle =
                PageBundleParserOutputConverter::htmlPageBundleFromParserOutput(
                    $previousOutput, $this->siteConfig, bodyOnly: false,
                );
        }
        $defaultOptions = [
            'pageBundle' => true,
            'wrapSections' => true,
            'logLinterData' => true,
            // The canonical ParserOutput form is body-only; Parsoid emits the
            // body fragment and the full document is reconstructed on demand
            // from an HtmlPageBundle (getPageBundle()). Legacy full-document
            // cache entries are still handled by the lazy strip in ContentHolder.
            'body_only' => true,
            'htmlVariantLanguage' => $htmlVariantLanguage,
            // We're doing language conversion in postprocessing now.
            'skipLanguageConversionPass' => true,
            'offsetType' => 'byte',
            'outputContentVersion' => Parsoid::defaultHTMLVersion(),
            'previousOutput' => $oldPageBundle,
            'previousInput' => $oldPageConfig,
            // The following are passed for metrics & labelling
            'sampleStats' => $doSample,
            'renderReason' => $options->getRenderReason(),
            'userAgent' => RequestContext::getMain()->getRequest()->getHeader( 'User-Agent' ),
        ];
 
        $parserOutput->resetParseStartTime();
 
        // This can throw ClientError or ResourceLimitExceededException.
        // Callers are responsible for figuring out how to handle them.
        $pageBundle = $this->parsoid->wikitext2html(
            $pageConfig,
            $defaultOptions,
            $headers,
            $parserOutput );
 
        $parserOutput = PageBundleParserOutputConverter::parserOutputFromPageBundle(
            $pageBundle, $parserOutput,
            title: $pageConfig->getLinkTarget(),
            siteConfig: $this->siteConfig,
        );
 
        // Register a watcher again because the $parserOutput arg
        // and $parserOutput return value above are different objects!
        $options->registerWatcher( $parserOutput->recordOption( ... ) );
 
        $parserOutput->setFromParserOptions( $options );
 
        $parserOutput->recordTimeProfile();
        $this->dataAccess->makeLimitReport( $pageConfig, $options, $parserOutput );
 
        // T371713: Collect statistics on parsing time -vs- presence of
        // $previousOutput
        $stats = MediaWikiServices::getInstance()->getStatsFactory();
        $labels = [
            'type' => $previousOutput === null ? 'full' : 'selective',
            'wiki' => WikiMap::getCurrentWikiId(),
            'reason' => $options->getRenderReason() ?: 'unknown',
            'has_async_content' =>
                $parserOutput->getOutputFlag( ParserOutputFlags::HAS_ASYNC_CONTENT )
                ? 'true' : 'false',
            'async_not_ready' =>
                $parserOutput->getOutputFlag( ParserOutputFlags::ASYNC_NOT_READY )
                ? 'true' : 'false',
        ];
        $stats
            ->getCounter( 'Parsoid_parse_cpu_seconds' )
            ->setLabels( $labels )
            ->incrementBy( $parserOutput->getTimeProfile( 'cpu' ) );
        $stats
            ->getCounter( 'Parsoid_parse_total' )
            ->setLabels( $labels )
            ->increment();
 
        return $this->addMetadata( $parserOutput, $pageConfig );
    }
 
    public function addMetadata( ParserOutput $parserOutput, PageConfig $pageConfig ): ParserOutput {
        // Add Parsoid skinning module
        $parserOutput->addModuleStyles( [ 'mediawiki.skinning.content.parsoid' ] );
 
        // (T10068) Allow control over whether robots index a page.
        # __NOINDEX__ always overrides __INDEX__, see T16899
        foreach ( [ 'noindex', 'index' ] as $indexSwitch ) {
            if (
                $parserOutput->getPageProperty( $indexSwitch ) !== null &&
                $this->namespaceInfo->canUseNoindex(
                    $pageConfig->getLinkTarget()->getNamespace()
                )
            ) {
                $parserOutput->setIndexPolicy( $indexSwitch );
                // Tracking categories are 'index-category', 'noindex-category'
                $this->trackingCategories->addTrackingCategory(
                    $parserOutput,
                    $indexSwitch . '-category',
                    Title::newFromLinkTarget( $pageConfig->getLinkTarget() ),
                );
            }
        }
 
        // Record base uri
        $parserOutput->setExtensionData(
            'core:base-uri', $this->siteConfig->baseURI()
        );
 
        // Record Parsoid version in extension data; this allows
        // us to use the onRejectParserCacheValue hook to selectively
        // expire "bad" generated content in the event of a rollback.
        $parserOutput->setExtensionData(
            'core:parsoid-version', Parsoid::version()
        );
        $parserOutput->setExtensionData(
            'core:html-version', Parsoid::defaultHTMLVersion()
        );
        // Export Parsoid HTML version to client gadgets as well
        $parserOutput->setJsConfigVar(
            'wgParsoidHtmlVersion', Parsoid::defaultHTMLVersion()
        );
        // TEMPORARY during transition to new LanguageConverter
        // Ensure we can distinguish ParserOutputs created using the
        // older language converter implementation.
        $parserOutput->setExtensionData(
            'core:parsoid-languageconverter', 'postprocess'
        );
 
        return $parserOutput;
    }
 
    public function parse(
        $text, PageReference $page, ParserOptions $options,
        bool $linestart = true, bool $clearState = true, ?int $revId = null,
        ?ParserOutput $previousOutput = null
    ): ParserOutput {
        Assert::invariant( $linestart, '$linestart=false is not yet supported' );
        Assert::invariant( $clearState, '$clearState=false is not yet supported' );
        $title = Title::newFromPageReference( $page );
        $lang = $options->getTargetLanguage();
        if ( $lang === null && $options->getInterfaceMessage() ) {
            $lang = $options->getUserLangObj();
        }
        $pageConfig = $revId === null || $revId === 0 ? null : $this->pageConfigFactory->createFromParserOptions(
            $options, // T392113: transfers current revision record callback
            $title,
            $revId,
            $lang // defaults to title page language if null
        );
        $content = null;
        if ( $text instanceof TextContent ) {
            $content = $text;
            $text = $content->getText();
        }
        if ( !( $pageConfig && $pageConfig->getPageMainContent() === $text ) ) {
            // This is a bit awkward! But we really need to parse $text, which
            // may or may not correspond to the $revId provided!
            // T332928 suggests one solution: splitting the "have revid"
            // callers from the "bare text, no associated revision" callers.
            $revisionRecord = new MutableRevisionRecord( $title );
            if ( $revId !== null ) {
                $revisionRecord->setId( $revId );
            }
            $revisionRecord->setSlot(
                SlotRecord::newUnsaved(
                    SlotRecord::MAIN,
                    $content ?? new WikitextContent( $text )
                )
            );
            $pageConfig = $this->pageConfigFactory->createFromParserOptions(
                $options,
                $title,
                $revisionRecord,
                $lang // defaults to title page language if null
            );
        }
 
        return $this->genParserOutput( $pageConfig, $options, $previousOutput );
    }
}