MediaWiki  master
compareParsers.php
Go to the documentation of this file.
1 <?php
31 require_once __DIR__ . '/dumpIterator.php';
32 
40 
41  private $count = 0;
43  private $saveFailed;
49  private $showDiff;
51  private $options;
53  private $failed;
54 
55  public function __construct() {
56  parent::__construct();
57  $this->saveFailed = false;
58  $this->addDescription( 'Run a file or dump with several parsers' );
59  $this->addOption( 'parser1', 'The first parser to compare.', true, true );
60  $this->addOption( 'parser2', 'The second parser to compare.', true, true );
61  $this->addOption( 'tidy', 'Run tidy on the articles.', false, false );
62  $this->addOption(
63  'save-failed',
64  'Folder in which articles which differ will be stored.',
65  false,
66  true
67  );
68  $this->addOption( 'show-diff', 'Show a diff of the two renderings.', false, false );
69  $this->addOption(
70  'diff-bin',
71  'Binary to use for diffing (can also be provided by DIFF env var).',
72  false,
73  false
74  );
75  $this->addOption(
76  'strip-parameters',
77  'Remove parameters of html tags to increase readability.',
78  false,
79  false
80  );
81  $this->addOption(
82  'show-parsed-output',
83  'Show the parsed html if both Parsers give the same output.',
84  false,
85  false
86  );
87  }
88 
89  public function checkOptions() {
90  if ( $this->hasOption( 'save-failed' ) ) {
91  $this->saveFailed = $this->getOption( 'save-failed' );
92  }
93 
94  $this->stripParametersEnabled = $this->hasOption( 'strip-parameters' );
95  $this->showParsedOutput = $this->hasOption( 'show-parsed-output' );
96 
97  $this->showDiff = $this->hasOption( 'show-diff' );
98  if ( $this->showDiff ) {
99  $bin = $this->getOption( 'diff-bin', getenv( 'DIFF' ) );
100  if ( $bin != '' ) {
101  global $wgDiff;
102  $wgDiff = $bin;
103  }
104  }
105 
106  $user = new User();
107  $this->options = ParserOptions::newFromUser( $user );
108 
109  if ( $this->hasOption( 'tidy' ) ) {
110  if ( !MWTidy::isEnabled() ) {
111  $this->fatalError( 'Tidy was requested but $wgTidyConfig is not set in LocalSettings.php' );
112  }
113  $this->options->setTidy( true );
114  }
115 
116  $this->failed = 0;
117  }
118 
119  public function conclusions() {
120  $this->error( "{$this->failed} failed revisions out of {$this->count}" );
121  if ( $this->count > 0 ) {
122  $this->output( " (" . ( $this->failed / $this->count ) . "%)\n" );
123  }
124  }
125 
126  function stripParameters( $text ) {
127  if ( !$this->stripParametersEnabled ) {
128  return $text;
129  }
130 
131  return preg_replace( '/(<a) [^>]+>/', '$1>', $text );
132  }
133 
138  public function processRevision( $rev ) {
139  $title = $rev->getTitle();
140 
141  $parser1Name = $this->getOption( 'parser1' );
142  $parser2Name = $this->getOption( 'parser2' );
143 
144  self::checkParserLocally( $parser1Name );
145  self::checkParserLocally( $parser2Name );
146 
147  $parser1 = new $parser1Name();
148  $parser2 = new $parser2Name();
149 
150  $content = $rev->getContent();
151 
152  if ( $content->getModel() !== CONTENT_MODEL_WIKITEXT ) {
153  $this->error( "Page {$title->getPrefixedText()} does not contain wikitext "
154  . "but {$content->getModel()}\n" );
155 
156  return;
157  }
158 
160  '@phan-var WikitextContent $content';
161  $text = strval( $content->getText() );
162 
163  $output1 = $parser1->parse( $text, $title, $this->options );
164  $output2 = $parser2->parse( $text, $title, $this->options );
165 
166  if ( $output1->getText() != $output2->getText() ) {
167  $this->failed++;
168  $this->error( "Parsing for {$title->getPrefixedText()} differs\n" );
169 
170  if ( $this->saveFailed ) {
171  file_put_contents(
172  $this->saveFailed . '/' . rawurlencode( $title->getPrefixedText() ) . ".txt",
173  $text
174  );
175  }
176  if ( $this->showDiff ) {
177  $this->output( wfDiff(
178  $this->stripParameters( $output1->getText() ),
179  $this->stripParameters( $output2->getText() ),
180  ''
181  ) );
182  }
183  } else {
184  $this->output( $title->getPrefixedText() . "\tOK\n" );
185 
186  if ( $this->showParsedOutput ) {
187  $this->output( $this->stripParameters( $output1->getText() ) );
188  }
189  }
190  }
191 
192  private static function checkParserLocally( $parserName ) {
193  /* Look for the parser in a file appropiately named in the current folder */
194  if ( !class_exists( $parserName ) && file_exists( "$parserName.php" ) ) {
195  global $wgAutoloadClasses;
196  $wgAutoloadClasses[$parserName] = realpath( '.' ) . "/$parserName.php";
197  }
198  }
199 }
200 
201 $maintClass = CompareParsers::class;
202 require_once RUN_MAINTENANCE_IF_MAIN;
bool $stripParametersEnabled
const RUN_MAINTENANCE_IF_MAIN
Definition: Maintenance.php:39
const CONTENT_MODEL_WIKITEXT
Definition: Defines.php:215
error( $err, $die=0)
Throw an error to the user.
processRevision( $rev)
Callback function for each revision, parse with both parsers and compare.
getOption( $name, $default=null)
Get an option, or return the default.
static checkParserLocally( $parserName)
static isEnabled()
Definition: MWTidy.php:54
ParserOptions $options
Maintenance script to take page text out of an XML dump file and render basic HTML out to files...
hasOption( $name)
Checks to see if a particular option exists.
The User object encapsulates all of the user-specific settings (user_id, name, rights, email address, options, last login time).
Definition: User.php:51
$maintClass
addDescription( $text)
Set the description text.
wfDiff( $before, $after, $params='-u')
Returns unified plain-text diff of two texts.
output( $out, $channel=null)
Throw some output to the user.
Base class for interating over a dump.
stripParameters( $text)
static newFromUser( $user)
Get a ParserOptions object from a given user.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
$content
Definition: router.php:78
$wgAutoloadClasses
Array mapping class names to filenames, for autoloading.
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
$wgDiff
Path to the GNU diff utility.