41 parent::__construct();
42 $this->
addDescription(
'Compares variant conversion output between Parser and HtmlOutputRendererHelper' );
45 'Name of the page to be parsed and compared',
50 'Target variant language code to transform the content to',
56 $mwInstance = MediaWikiServices::getInstance();
58 $pageName = $this->
getArg(
'page-title' );
59 $pageTitle = Title::newFromText( $pageName );
61 if ( !$pageTitle || !$pageTitle->
exists() ) {
62 $this->
fatalError(
"Title with name $pageName not found" );
65 $targetVariantCode = $this->
getArg(
'target-variant' );
66 $languageNameUtils = $mwInstance->getLanguageNameUtils();
67 if ( !$languageNameUtils->isValidBuiltInCode( $targetVariantCode ) ) {
68 $this->
fatalError(
"$targetVariantCode is not a supported variant" );
70 $targetVariant = $mwInstance->getLanguageFactory()->getLanguage(
77 $parserOutput = $this->getParserOutput( $pageTitle, $baseLanguage, $targetVariant );
78 $parsoidOutput = $this->getParsoidOutput( $pageTitle, $targetVariant, $user );
79 $converterUsed = $this->getConverterUsed( $parsoidOutput );
81 $this->compareOutput( $parserOutput, $parsoidOutput, $converterUsed );
90 $services->getParsoidOutputStash(),
92 $services->getParsoidOutputAccess(),
93 $services->getHtmlTransformFactory(),
94 $services->getContentHandlerFactory(),
95 $services->getLanguageFactory()
103 $parserOpts->disableContentConversion(
false );
104 $parserOpts->disableTitleConversion(
false );
109 private function getParserOutput(
119 $mwInstance = MediaWikiServices::getInstance();
121 $languageFactory = $mwInstance->getLanguageFactory();
122 $parser = $mwInstance->getParser();
123 $parserOptions = $this->getParserOptions(
124 $languageFactory->getParentLanguage( $baseLanguage )
127 $content = $mwInstance->getRevisionLookup()
128 ->getRevisionByTitle( $pageTitle )
129 ->getContent( SlotRecord::MAIN );
132 return $parser->parse( $wikiContent, $pageTitle, $parserOptions );
135 private function getParsoidOutput(
137 Bcp47Code $targetVariant,
140 $htmlOutputRendererHelper = $this->newHtmlOutputRendererHelper();
141 $htmlOutputRendererHelper->init( $pageTitle, [
145 $htmlOutputRendererHelper->setVariantConversionLanguage( $targetVariant );
147 return $htmlOutputRendererHelper->getHtml();
150 private function getWords(
string $output ): array {
151 $tagsRemoved = strip_tags( $output );
152 $words = preg_split(
'/\s+/', trim( $tagsRemoved ), -1, PREG_SPLIT_NO_EMPTY );
156 private function getBody(
string $output ): string {
157 $dom = new DOMDocument();
159 @$dom->loadHTML( $output );
160 $body = $dom->getElementsByTagName(
'body' )->item( 0 );
161 if ( $body ===
null ) {
166 return $body->textContent;
169 private function compareOutput(
172 string $converterUsed
174 $parsoidText = $parsoidOutput->getText( [
'deduplicateStyles' => false ] );
175 $parserText = $parserOutput->
getText( [
'deduplicateStyles' =>
false ] );
177 $parsoidWords = $this->getWords( $this->getBody( $parsoidText ) );
178 $parserWords = $this->getWords( $parserText );
180 $parserWordCount = count( $parserWords );
181 $parsoidWordCount = count( $parsoidWords );
182 $this->output(
"Word count: Parsoid: $parsoidWordCount; Parser: $parserWordCount\n" );
184 $this->outputSimilarity( $parsoidWords, $parserWords );
185 $this->output(
"\n" );
186 $this->outputDiff( $parsoidWords, $parserWords, $converterUsed );
189 private function getConverterUsed(
ParserOutput $parsoidOutput ): string {
190 $isCoreConverterUsed = strpos(
191 $parsoidOutput->getRawText(),
192 'Variant conversion performed using the core LanguageConverter'
195 if ( $isCoreConverterUsed ) {
196 return 'Core LanguageConverter';
198 return 'Parsoid LanguageConverter';
203 private function mb_sprintf(
string $format, ...$args ): string {
207 preg_replace_callback(
208 '/(?<=%|%-)\d+(?=s)/',
209 static function ( array
$matches ) use ( &$params ) {
210 $value = array_shift( $params );
212 return (
string)( strlen( $value ) - mb_strlen( $value ) +
$matches[0] );
220 private function outputSimilarity( array $parsoidWords, array $parserWords ): void {
221 $parsoidOutput = implode(
' ', $parsoidWords );
222 $parserOutput = implode(
' ', $parserWords );
224 'Total characters: Parsoid: ' . strlen( $parsoidOutput ) .
225 '; Parser: ' . strlen( $parserOutput ) .
"\n"
228 $similarityPercent = 0;
229 $similarCharacters = similar_text( $parsoidOutput, $parserOutput, $similarityPercent );
230 $similarityPercent = round( $similarityPercent, 2 );
233 "Similarity via similar_text(): $similarityPercent%; Similar characters: $similarCharacters"
237 private function outputDiff( array $parsoidWords, array $parserWords,
string $converterUsed ): void {
238 $out = str_repeat(
'-', 96 ) .
"\n";
239 $out .= sprintf(
"| %5s | %-35s | %-35s | %-8s |\n",
'Line',
'Parsoid',
'Parser',
'Diff' );
240 $out .= sprintf(
"| %5s | %-35s | %-35s | %-8s |\n",
'',
"($converterUsed)",
'',
'' );
241 $out .= str_repeat(
'-', 96 ) .
"\n";
244 $diff =
new Diff( $parsoidWords, $parserWords );
246 $this->output( $e->getMessage() );
247 $this->error(
'Encountered ComplexityException while computing diff' );
252 foreach ( $wordDiffFormat as $index => $wordDiff ) {
253 $action = $wordDiff[
'action'];
254 $old = $wordDiff[
'old'] ??
null;
255 $new = $wordDiff[
'new'] ??
null;
257 $out .= $this->mb_sprintf(
258 "| %5s | %-35s | %-35s | %-8s |\n",
259 str_pad( (
string)( $index + 1 ), 5,
' ', STR_PAD_LEFT ),
260 mb_strimwidth( $old ??
'- N/A -', 0, 35,
'…' ),
261 mb_strimwidth( $new ??
'- N/A -', 0, 35,
'…' ),
267 $out .= str_repeat(
'-', 96 ) .
"\n";
268 $this->output(
"\n" . $out );
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
addArg( $arg, $description, $required=true, $multi=false)
Add some args that are needed.
getArg( $argId=0, $default=null)
Get an argument.
addDescription( $text)
Set the description text.
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
Content object implementation for representing flat text.