LanguageVariantConverter.php

Go to the documentation of this file.

<?php
declare( strict_types = 1 );
 
namespace MediaWiki\Parser\Parsoid;
 
use MediaWiki\Language\LanguageCode;
use MediaWiki\Languages\LanguageConverterFactory;
use MediaWiki\Languages\LanguageFactory;
use MediaWiki\Page\PageIdentity;
use MediaWiki\Parser\ParserOptions;
use MediaWiki\Parser\ParserOutput;
use MediaWiki\Parser\Parsoid\Config\PageConfigFactory;
use MediaWiki\Rest\HttpException;
use MediaWiki\Rest\LocalizedHttpException;
use MediaWiki\Revision\RevisionAccessException;
use MediaWiki\Title\Title;
use MediaWiki\Title\TitleFactory;
use Wikimedia\Bcp47Code\Bcp47Code;
use Wikimedia\Bcp47Code\Bcp47CodeValue;
use Wikimedia\Message\MessageValue;
use Wikimedia\Parsoid\Config\PageConfig;
use Wikimedia\Parsoid\Config\SiteConfig;
use Wikimedia\Parsoid\Core\HtmlPageBundle;
use Wikimedia\Parsoid\DOM\Element;
use Wikimedia\Parsoid\Parsoid;
use Wikimedia\Parsoid\Utils\DOMCompat;
use Wikimedia\Parsoid\Utils\DOMUtils;
 
class LanguageVariantConverter {
    private ?PageConfig $pageConfig = null;
    private readonly Title $pageTitle;
    private ?Bcp47Code $pageLanguageOverride = null;
    private bool $isFallbackLanguageConverterEnabled = true;
 
    private ?ParserOptions $parserOptionsForTest = null;
 
    public function __construct(
        private readonly PageIdentity $pageIdentity,
        private readonly PageConfigFactory $pageConfigFactory,
        private readonly Parsoid $parsoid,
        private readonly SiteConfig $siteConfig,
        TitleFactory $titleFactory,
        private readonly LanguageConverterFactory $languageConverterFactory,
        private readonly LanguageFactory $languageFactory
    ) {
        $this->pageTitle = $titleFactory->newFromPageIdentity( $pageIdentity );
    }
 
    public function setPageConfig( PageConfig $pageConfig ) {
        $this->pageConfig = $pageConfig;
    }
 
    public function setPageLanguageOverride( Bcp47Code $language ) {
        $this->pageLanguageOverride = $language;
    }
 
    public function convertPageBundleVariant(
        HtmlPageBundle $pageBundle,
        Bcp47Code $targetVariant,
        ?Bcp47Code $sourceVariant = null
    ): HtmlPageBundle {
        [ $pageLanguage, $sourceVariant ] =
            $this->getBaseAndSourceLanguage( $pageBundle, $sourceVariant );
 
        if ( !$this->siteConfig->langConverterEnabledBcp47( $pageLanguage ) ) {
            // If the language doesn't support variants, just return the content unmodified.
            return $pageBundle;
        }
 
        $pageConfig = $this->getPageConfig( $pageLanguage, $sourceVariant );
 
        if ( $this->parsoid->implementsLanguageConversionBcp47( $pageConfig, $targetVariant ) ) {
            return $this->parsoid->pb2pb(
                $pageConfig, 'variant', $pageBundle,
                [
                    'variant' => [
                        'source' => $sourceVariant,
                        'target' => $targetVariant,
                    ]
                ]
            );
        } else {
            if ( !$this->isFallbackLanguageConverterEnabled ) {
                // Fallback variant conversion is not enabled, return the page bundle as is.
                return $pageBundle;
            }
 
            // LanguageConverter::hasVariant and LanguageConverter::convertTo
            // could take a string|Bcp47Code in the future, which would
            // allow us to avoid the $targetVariantCode conversion here.
            $baseLanguage = $this->languageFactory->getParentLanguage( $targetVariant );
            $languageConverter = $this->languageConverterFactory->getLanguageConverter( $baseLanguage );
            $targetVariantCode = $this->languageFactory->getLanguage( $targetVariant )->getCode();
            if ( $languageConverter->hasVariant( $targetVariantCode ) ) {
                // NOTE: This is not a convert() because we have the exact desired variant
                // and don't need to compute a preferred variant based on a base language.
                // Also see T267067 for why convert() should be avoided.
                $convertedHtml = $languageConverter->convertTo( $pageBundle->html, $targetVariantCode );
                $pageVariant = $targetVariant;
            } else {
                // No conversion possible - pass through original HTML in original language
                $convertedHtml = $pageBundle->html;
                $pageVariant = $pageConfig->getPageLanguageBcp47();
            }
 
            // Add a note so that we can identify what was used to perform the variant conversion
            $msg = "<!-- Variant conversion performed using the core LanguageConverter -->";
            $convertedHtml = $msg . $convertedHtml;
 
            // NOTE: Keep this in sync with code in Parsoid.php in Parsoid repo
            // Add meta information that Parsoid normally adds
            $headers = [
                'content-language' => $pageVariant->toBcp47Code(),
                'vary' => [ 'Accept', 'Accept-Language' ]
            ];
            $doc = DOMUtils::parseHTML( '<head></head><body></body>' );
            DOMUtils::addHttpEquivHeaders( $doc, $headers );
            $docElt = $doc->documentElement;
            '@phan-var Element $docElt';
            $docHtml = DOMCompat::getOuterHTML( $docElt );
            $convertedHtml = preg_replace( "#</body>#", $docHtml, "$convertedHtml</body>" );
            return new HtmlPageBundle(
                html: $convertedHtml, parsoid: [], mw: [],
                version: $pageBundle->version, headers: $headers
            );
        }
    }
 
    public function convertParserOutputVariant(
        ParserOutput $parserOutput,
        Bcp47Code $targetVariant,
        ?Bcp47Code $sourceVariant = null
    ): ParserOutput {
        $pageBundle = PageBundleParserOutputConverter::pageBundleFromParserOutput( $parserOutput );
        $modifiedPageBundle = $this->convertPageBundleVariant( $pageBundle, $targetVariant, $sourceVariant );
 
        return PageBundleParserOutputConverter::parserOutputFromPageBundle( $modifiedPageBundle, $parserOutput );
    }
 
    public function disableFallbackLanguageConverter(): void {
        $this->isFallbackLanguageConverterEnabled = false;
    }
 
    private function getPageConfig( Bcp47Code $pageLanguage, ?Bcp47Code $sourceVariant ): PageConfig {
        if ( $this->pageConfig ) {
            return $this->pageConfig;
        }
 
        try {
            $this->pageConfig = $this->pageConfigFactory->createFromParserOptions(
                // Hook for unit testing: can supply a mock for parser options
                $this->parserOptionsForTest ?? ParserOptions::newFromAnon(),
                $this->pageIdentity,
                null,
                $pageLanguage
            );
 
            if ( $sourceVariant ) {
                $this->pageConfig->setVariantBcp47( $sourceVariant );
            }
        } catch ( RevisionAccessException ) {
            // TODO: Throw a different exception, this class should not know
            //       about HTTP status codes.
            throw new LocalizedHttpException( new MessageValue( "rest-specified-revision-unavailable" ), 404 );
        }
 
        return $this->pageConfig;
    }
 
    private function getPageLanguage( HtmlPageBundle $pageBundle, ?Bcp47Code $default = null ): Bcp47Code {
        // If a language was set by calling setPageLanguageOverride(), always use it!
        if ( $this->pageLanguageOverride ) {
            return $this->pageLanguageOverride;
        }
 
        // If the page bundle contains a language code, use that.
        $pageBundleLanguage = $pageBundle->headers[ 'content-language' ] ?? null;
        if ( $pageBundleLanguage ) {
            // The HTTP header will contain a BCP-47 language code, not a
            // mediawiki-internal one.
            return new Bcp47CodeValue( $pageBundleLanguage );
        }
 
        // NOTE: Use explicit default *before* we try HtmlPageBundle, because PageConfig::getPageLanguage()
        //       falls back to Title::getPageLanguage(). If we did that first, $default would never be used.
        if ( $default ) {
            return $default;
        }
 
        // If we have a PageConfig, we can ask it for the page's language. Note that this will fall back to
        // Title::getPageLanguage(), so it has to be the last thing we try.
        if ( $this->pageConfig ) {
            return $this->pageConfig->getPageLanguageBcp47();
        }
 
        // Finally, just go by the code associated with the title. This may come from the database or
        // it may be determined based on the title itself.
        return $this->pageTitle->getPageLanguage();
    }
 
    private function getBaseAndSourceLanguage( HtmlPageBundle $pageBundle, ?Bcp47Code $sourceLanguage ): array {
        // Try to determine the language code associated with the content of the page.
        // The result may be a variant code.
        $baseLanguage = $this->getPageLanguage( $pageBundle, $sourceLanguage );
 
        // To find out if $baseLanguage is actually a variant, get the parent language and compare.
        $parentLang = $this->languageFactory->getParentLanguage( $baseLanguage );
 
        // If $parentLang is not the same language as $baseLanguage, this means that
        // $baseLanguage is a variant. In that case, set $sourceLanguage to that
        // variant (unless $sourceLanguage is already set), and set $baseLanguage
        // to the $parentLang
        if ( $parentLang && strcasecmp( $parentLang->toBcp47Code(), $baseLanguage->toBcp47Code() ) !== 0 ) {
            if ( !$sourceLanguage ) {
                $sourceLanguage = $baseLanguage;
            }
            $baseLanguage = $parentLang;
        }
 
        if ( $sourceLanguage !== null ) {
            $parentConverter = $this->languageConverterFactory->getLanguageConverter( $parentLang );
            // If the source variant isn't actually a variant, trigger auto-detection
            $sourceIsVariant = (
                strcasecmp( $parentLang->toBcp47Code(), $sourceLanguage->toBcp47Code() ) !== 0 &&
                $parentConverter->hasVariant(
                    LanguageCode::bcp47ToInternal( $sourceLanguage->toBcp47Code() )
                )
            );
            if ( !$sourceIsVariant ) {
                $sourceLanguage = null;
            }
        }
 
        return [ $baseLanguage, $sourceLanguage ];
    }
}