51 parent::__construct();
52 $this->
addDescription(
'Compares variant conversion output between Parser and HtmlOutputRendererHelper' );
55 'Name of the page to be parsed and compared',
60 'Target variant language code to transform the content to',
68 $pageName = $this->
getArg(
'page-title' );
69 $pageTitle = Title::newFromText( $pageName );
71 if ( !$pageTitle || !$pageTitle->
exists() ) {
72 $this->
fatalError(
"Title with name $pageName not found" );
75 $targetVariantCode = $this->
getArg(
'target-variant' );
76 $languageNameUtils = $mwInstance->getLanguageNameUtils();
77 if ( !$languageNameUtils->isValidBuiltInCode( $targetVariantCode ) ) {
78 $this->
fatalError(
"$targetVariantCode is not a supported variant" );
80 $targetVariant = $mwInstance->getLanguageFactory()->getLanguage(
84 $user = User::newSystemUser( User::MAINTENANCE_SCRIPT_USER, [
'steal' =>
true ] );
87 $parserOutput = $this->getParserOutput( $pageTitle, $baseLanguage, $targetVariant );
88 $parsoidOutput = $this->getParsoidOutput( $pageTitle, $targetVariant, $user );
89 $converterUsed = $this->getConverterUsed( $parsoidOutput );
91 $this->compareOutput( $parserOutput->getContentHolderText(), $parsoidOutput->
getContentHolderText(),
100 new ServiceOptions( PageRestHelperFactory::CONSTRUCTOR_OPTIONS, $services->getMainConfig() ),
101 $services->getRevisionLookup(),
102 $services->getRevisionRenderer(),
103 $services->getTitleFormatter(),
104 $services->getPageStore(),
105 $services->getParsoidOutputStash(),
107 $services->getParserOutputAccess(),
108 $services->getParsoidSiteConfig(),
109 $services->getHtmlTransformFactory(),
110 $services->getContentHandlerFactory(),
111 $services->getLanguageFactory(),
112 $services->getRedirectStore(),
113 $services->getLanguageConverterFactory(),
114 $services->getTitleFactory(),
115 $services->getConnectionProvider(),
116 $services->getChangeTagsStore(),
117 StatsFactory::newNull()
125 $parserOpts->disableContentConversion(
false );
126 $parserOpts->disableTitleConversion(
false );
131 private function getParserOutput(
141 $mwInstance = $this->getServiceContainer();
143 $languageFactory = $mwInstance->getLanguageFactory();
144 $parser = $mwInstance->getParser();
145 $parserOptions = $this->getParserOptions(
146 $languageFactory->getParentLanguage( $baseLanguage )
149 $content = $mwInstance->getRevisionLookup()
150 ->getRevisionByTitle( $pageTitle )
151 ->getContent( SlotRecord::MAIN );
152 $wikiContent = ( $content instanceof
TextContent ) ? $content->getText() :
'';
154 $po = $parser->parse( $wikiContent, $pageTitle, $parserOptions );
156 $pipeline = $mwInstance->getDefaultOutputPipeline();
157 $options = [
'deduplicateStyles' => false ];
158 return $pipeline->run( $po, $parserOptions, $options );
161 private function getParsoidOutput(
163 Bcp47Code $targetVariant,
167 $htmlOutputRendererHelper = $this->newPageRestHelperFactory()->newHtmlOutputRendererHelper( $pageTitle, [
170 ], $user,
null,
false, $parserOptions );
171 $htmlOutputRendererHelper->setVariantConversionLanguage( $targetVariant );
173 $po = $htmlOutputRendererHelper->getHtml();
174 $pipeline = $this->getServiceContainer()->getDefaultOutputPipeline();
175 $options = [
'deduplicateStyles' => false ];
176 return $pipeline->run( $po, $parserOptions, $options );
179 private function getWords(
string $output ): array {
180 $tagsRemoved = strip_tags( $output );
181 $words = preg_split(
'/\s+/', trim( $tagsRemoved ), -1, PREG_SPLIT_NO_EMPTY );
185 private function getBody(
string $output ): string {
186 $dom = new DOMDocument();
188 @$dom->loadHTML( $output );
189 $body = $dom->getElementsByTagName(
'body' )->item( 0 );
190 if ( $body ===
null ) {
195 return $body->textContent;
198 private function compareOutput(
201 string $converterUsed
203 $parsoidWords = $this->getWords( $this->getBody( $parsoidText ) );
204 $parserWords = $this->getWords( $parserText );
206 $parserWordCount = count( $parserWords );
207 $parsoidWordCount = count( $parsoidWords );
208 $this->output(
"Word count: Parsoid: $parsoidWordCount; Parser: $parserWordCount\n" );
210 $this->outputSimilarity( $parsoidWords, $parserWords );
211 $this->output(
"\n" );
212 $this->outputDiff( $parsoidWords, $parserWords, $converterUsed );
215 private function getConverterUsed(
ParserOutput $parsoidOutput ): string {
216 $isCoreConverterUsed = strpos(
217 $parsoidOutput->getRawText(),
218 'Variant conversion performed using the core LanguageConverter'
221 if ( $isCoreConverterUsed ) {
222 return 'Core LanguageConverter';
224 return 'Parsoid LanguageConverter';
229 private function mb_sprintf(
string $format, ...$args ): string {
233 preg_replace_callback(
234 '/(?<=%|%-)\d+(?=s)/',
236 $value = array_shift(
$params );
238 return (
string)( strlen( $value ) - mb_strlen( $value ) +
$matches[0] );
246 private function outputSimilarity( array $parsoidWords, array $parserWords ): void {
247 $parsoidOutput = implode(
' ', $parsoidWords );
248 $parserOutput = implode(
' ', $parserWords );
250 'Total characters: Parsoid: ' . strlen( $parsoidOutput ) .
251 '; Parser: ' . strlen( $parserOutput ) .
"\n"
254 $similarityPercent = 0;
255 $similarCharacters = similar_text( $parsoidOutput, $parserOutput, $similarityPercent );
256 $similarityPercent = round( $similarityPercent, 2 );
259 "Similarity via similar_text(): $similarityPercent%; Similar characters: $similarCharacters"
263 private function outputDiff( array $parsoidWords, array $parserWords,
string $converterUsed ): void {
264 $out = str_repeat(
'-', 96 ) .
"\n";
265 $out .= sprintf(
"| %5s | %-35s | %-35s | %-8s |\n",
'Line',
'Parsoid',
'Parser',
'Diff' );
266 $out .= sprintf(
"| %5s | %-35s | %-35s | %-8s |\n",
'',
"($converterUsed)",
'',
'' );
267 $out .= str_repeat(
'-', 96 ) .
"\n";
270 $diff =
new Diff( $parsoidWords, $parserWords );
272 $this->output( $e->getMessage() );
273 $this->error(
'Encountered ComplexityException while computing diff' );
278 foreach ( $wordDiffFormat as $index => $wordDiff ) {
279 $action = $wordDiff[
'action'];
280 $old = $wordDiff[
'old'] ??
null;
281 $new = $wordDiff[
'new'] ??
null;
283 $out .= $this->mb_sprintf(
284 "| %5s | %-35s | %-35s | %-8s |\n",
285 str_pad( (
string)( $index + 1 ), 5,
' ', STR_PAD_LEFT ),
286 mb_strimwidth( $old ??
'- N/A -', 0, 35,
'…' ),
287 mb_strimwidth( $new ??
'- N/A -', 0, 35,
'…' ),
293 $out .= str_repeat(
'-', 96 ) .
"\n";
294 $this->output(
"\n" . $out );