MediaWiki  master
compareParsers.php
Go to the documentation of this file.
1 <?php
31 require_once __DIR__ . '/dumpIterator.php';
32 
40 
41  private $count = 0;
43  private $saveFailed;
49  private $showDiff;
51  private $options;
53  private $failed;
54 
55  public function __construct() {
56  parent::__construct();
57  $this->saveFailed = false;
58  $this->addDescription( 'Run a file or dump with several parsers' );
59  $this->addOption( 'parser1', 'The first parser to compare.', true, true );
60  $this->addOption( 'parser2', 'The second parser to compare.', true, true );
61  $this->addOption(
62  'save-failed',
63  'Folder in which articles which differ will be stored.',
64  false,
65  true
66  );
67  $this->addOption( 'show-diff', 'Show a diff of the two renderings.', false, false );
68  $this->addOption(
69  'diff-bin',
70  'Binary to use for diffing (can also be provided by DIFF env var).',
71  false,
72  false
73  );
74  $this->addOption(
75  'strip-parameters',
76  'Remove parameters of html tags to increase readability.',
77  false,
78  false
79  );
80  $this->addOption(
81  'show-parsed-output',
82  'Show the parsed html if both Parsers give the same output.',
83  false,
84  false
85  );
86  }
87 
88  public function checkOptions() {
89  if ( $this->hasOption( 'save-failed' ) ) {
90  $this->saveFailed = $this->getOption( 'save-failed' );
91  }
92 
93  $this->stripParametersEnabled = $this->hasOption( 'strip-parameters' );
94  $this->showParsedOutput = $this->hasOption( 'show-parsed-output' );
95 
96  $this->showDiff = $this->hasOption( 'show-diff' );
97  if ( $this->showDiff ) {
98  $bin = $this->getOption( 'diff-bin', getenv( 'DIFF' ) );
99  if ( $bin != '' ) {
100  global $wgDiff;
101  $wgDiff = $bin;
102  }
103  }
104 
105  $user = new User();
106  $this->options = ParserOptions::newFromUser( $user );
107 
108  $this->failed = 0;
109  }
110 
111  public function conclusions() {
112  $this->error( "{$this->failed} failed revisions out of {$this->count}" );
113  if ( $this->count > 0 ) {
114  $this->output( " (" . ( $this->failed / $this->count ) . "%)\n" );
115  }
116  }
117 
118  private function stripParameters( $text ) {
119  if ( !$this->stripParametersEnabled ) {
120  return $text;
121  }
122 
123  return preg_replace( '/(<a) [^>]+>/', '$1>', $text );
124  }
125 
130  public function processRevision( WikiRevision $rev ) {
131  $title = $rev->getTitle();
132 
133  $parser1Name = $this->getOption( 'parser1' );
134  $parser2Name = $this->getOption( 'parser2' );
135 
136  self::checkParserLocally( $parser1Name );
137  self::checkParserLocally( $parser2Name );
138 
139  $parser1 = new $parser1Name();
140  $parser2 = new $parser2Name();
141 
142  $content = $rev->getContent();
143 
144  if ( $content->getModel() !== CONTENT_MODEL_WIKITEXT ) {
145  $this->error( "Page {$title->getPrefixedText()} does not contain wikitext "
146  . "but {$content->getModel()}\n" );
147 
148  return;
149  }
150 
152  '@phan-var WikitextContent $content';
153  $text = strval( $content->getText() );
154 
155  $output1 = $parser1->parse( $text, $title, $this->options );
156  $output2 = $parser2->parse( $text, $title, $this->options );
157 
158  if ( $output1->getText() != $output2->getText() ) {
159  $this->failed++;
160  $this->error( "Parsing for {$title->getPrefixedText()} differs\n" );
161 
162  if ( $this->saveFailed ) {
163  file_put_contents(
164  $this->saveFailed . '/' . rawurlencode( $title->getPrefixedText() ) . ".txt",
165  $text
166  );
167  }
168  if ( $this->showDiff ) {
169  $diffs = new Diff(
170  explode( "\n", $this->stripParameters( $output1->getText() ) ),
171  explode( "\n", $this->stripParameters( $output2->getText() ) )
172  );
173  $formatter = new UnifiedDiffFormatter();
174  $unifiedDiff = $formatter->format( $diffs );
175 
176  $this->output( $unifiedDiff );
177  }
178  } else {
179  $this->output( $title->getPrefixedText() . "\tOK\n" );
180 
181  if ( $this->showParsedOutput ) {
182  $this->output( $this->stripParameters( $output1->getText() ) );
183  }
184  }
185  }
186 
187  private static function checkParserLocally( $parserName ) {
188  /* Look for the parser in a file appropiately named in the current folder */
189  if ( !class_exists( $parserName ) && file_exists( "$parserName.php" ) ) {
190  global $wgAutoloadClasses;
191  $wgAutoloadClasses[$parserName] = realpath( '.' ) . "/$parserName.php";
192  }
193  }
194 }
195 
196 $maintClass = CompareParsers::class;
197 require_once RUN_MAINTENANCE_IF_MAIN;
RUN_MAINTENANCE_IF_MAIN
const RUN_MAINTENANCE_IF_MAIN
Definition: Maintenance.php:38
ParserOptions
Set options of the Parser.
Definition: ParserOptions.php:44
CompareParsers\$count
$count
Definition: compareParsers.php:41
CompareParsers\checkOptions
checkOptions()
Stub function for processing additional options.
Definition: compareParsers.php:88
CompareParsers\stripParameters
stripParameters( $text)
Definition: compareParsers.php:118
CompareParsers\checkParserLocally
static checkParserLocally( $parserName)
Definition: compareParsers.php:187
Maintenance\addDescription
addDescription( $text)
Set the description text.
Definition: Maintenance.php:327
CompareParsers\$options
ParserOptions $options
Definition: compareParsers.php:51
WikiRevision\getTitle
getTitle()
Definition: WikiRevision.php:428
$wgDiff
$wgDiff
Path to the GNU diff utility.
Definition: DefaultSettings.php:7088
CONTENT_MODEL_WIKITEXT
const CONTENT_MODEL_WIKITEXT
Definition: Defines.php:224
CompareParsers\conclusions
conclusions()
Stub function for giving data about what was computed.
Definition: compareParsers.php:111
CompareParsers\$stripParametersEnabled
bool $stripParametersEnabled
Definition: compareParsers.php:45
$maintClass
$maintClass
Definition: compareParsers.php:196
CompareParsers\$failed
int $failed
Definition: compareParsers.php:53
CompareParsers\__construct
__construct()
Default constructor.
Definition: compareParsers.php:55
CompareParsers\$showDiff
bool $showDiff
Definition: compareParsers.php:49
Maintenance\addOption
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
Definition: Maintenance.php:245
$title
$title
Definition: testCompression.php:38
WikiRevision\getContent
getContent( $role=SlotRecord::MAIN)
Definition: WikiRevision.php:493
UnifiedDiffFormatter
A formatter that outputs unified diffs @newable.
Definition: UnifiedDiffFormatter.php:32
$content
$content
Definition: router.php:76
CompareParsers\processRevision
processRevision(WikiRevision $rev)
Callback function for each revision, parse with both parsers and compare.
Definition: compareParsers.php:130
CompareParsers\$saveFailed
bool $saveFailed
Definition: compareParsers.php:43
$wgAutoloadClasses
$wgAutoloadClasses
Array mapping class names to filenames, for autoloading.
Definition: DefaultSettings.php:7834
DumpIterator
Base class for interating over a dump.
Definition: dumpIterator.php:36
Maintenance\getOption
getOption( $name, $default=null)
Get an option, or return the default.
Definition: Maintenance.php:281
WikiRevision
Represents a revision, log entry or upload during the import process.
Definition: WikiRevision.php:39
CompareParsers
Maintenance script to take page text out of an XML dump file and render basic HTML out to files.
Definition: compareParsers.php:39
CompareParsers\$showParsedOutput
bool $showParsedOutput
Definition: compareParsers.php:47
Maintenance\error
error( $err, $die=0)
Throw an error to the user.
Definition: Maintenance.php:463
Maintenance\output
output( $out, $channel=null)
Throw some output to the user.
Definition: Maintenance.php:434
Maintenance\hasOption
hasOption( $name)
Checks to see if a particular option exists.
Definition: Maintenance.php:266
User
The User object encapsulates all of the user-specific settings (user_id, name, rights,...
Definition: User.php:56
ParserOptions\newFromUser
static newFromUser( $user)
Get a ParserOptions object from a given user.
Definition: ParserOptions.php:1077
Diff
Class representing a 'diff' between two sequences of strings.
Definition: Diff.php:32