MediaWiki  master
WikiExporter.php
Go to the documentation of this file.
1 <?php
37 
41 class WikiExporter {
43  public $list_authors = false;
44 
46  public $dumpUploads = false;
47 
49  public $dumpUploadFileContents = false;
50 
52  public $author_list = "";
53 
54  public const FULL = 1;
55  public const CURRENT = 2;
56  public const STABLE = 4; // extension defined
57  public const LOGS = 8;
58  public const RANGE = 16;
59 
62 
63  protected const BATCH_SIZE = 50000;
64 
66  public $text;
67 
69  public $sink;
70 
72  private $writer;
73 
75  protected $db;
76 
78  protected $history;
79 
81  protected $limitNamespaces;
82 
84  private $revisionStore;
85 
87  private $titleParser;
88 
90  private $hookRunner;
91 
96  public static function schemaVersion() {
99  }
100 
112  public function __construct(
113  $db,
114  $history = self::CURRENT,
115  $text = self::TEXT,
116  $limitNamespaces = null
117  ) {
118  $this->db = $db;
119  $this->history = $history;
120  $this->writer = new XmlDumpWriter( $text, self::schemaVersion() );
121  $this->sink = new DumpOutput();
122  $this->text = $text;
123  $this->limitNamespaces = $limitNamespaces;
124  $services = MediaWikiServices::getInstance();
125  $this->hookRunner = new HookRunner( $services->getHookContainer() );
126  $this->revisionStore = $services->getRevisionStore();
127  $this->titleParser = $services->getTitleParser();
128  }
129 
135  public function setSchemaVersion( $schemaVersion ) {
136  $this->writer = new XmlDumpWriter( $this->text, $schemaVersion );
137  }
138 
146  public function setOutputSink( &$sink ) {
147  $this->sink =& $sink;
148  }
149 
150  public function openStream() {
151  $output = $this->writer->openStream();
152  $this->sink->writeOpenStream( $output );
153  }
154 
155  public function closeStream() {
156  $output = $this->writer->closeStream();
157  $this->sink->writeCloseStream( $output );
158  }
159 
165  public function allPages() {
166  $this->dumpFrom( '' );
167  }
168 
177  public function pagesByRange( $start, $end, $orderRevs ) {
178  if ( $orderRevs ) {
179  $condition = 'rev_page >= ' . intval( $start );
180  if ( $end ) {
181  $condition .= ' AND rev_page < ' . intval( $end );
182  }
183  } else {
184  $condition = 'page_id >= ' . intval( $start );
185  if ( $end ) {
186  $condition .= ' AND page_id < ' . intval( $end );
187  }
188  }
189  $this->dumpFrom( $condition, $orderRevs );
190  }
191 
199  public function revsByRange( $start, $end ) {
200  $condition = 'rev_id >= ' . intval( $start );
201  if ( $end ) {
202  $condition .= ' AND rev_id < ' . intval( $end );
203  }
204  $this->dumpFrom( $condition );
205  }
206 
210  public function pageByTitle( PageIdentity $page ) {
211  $this->dumpFrom(
212  'page_namespace=' . $page->getNamespace() .
213  ' AND page_title=' . $this->db->addQuotes( $page->getDBkey() ) );
214  }
215 
220  public function pageByName( $name ) {
221  try {
222  $link = $this->titleParser->parseTitle( $name );
223  $this->dumpFrom(
224  'page_namespace=' . $link->getNamespace() .
225  ' AND page_title=' . $this->db->addQuotes( $link->getDBkey() ) );
226  } catch ( MalformedTitleException $ex ) {
227  throw new MWException( "Can't export invalid title" );
228  }
229  }
230 
234  public function pagesByName( $names ) {
235  foreach ( $names as $name ) {
236  $this->pageByName( $name );
237  }
238  }
239 
240  public function allLogs() {
241  $this->dumpFrom( '' );
242  }
243 
248  public function logsByRange( $start, $end ) {
249  $condition = 'log_id >= ' . intval( $start );
250  if ( $end ) {
251  $condition .= ' AND log_id < ' . intval( $end );
252  }
253  $this->dumpFrom( $condition );
254  }
255 
263  protected function do_list_authors( $cond ) {
264  $this->author_list = "<contributors>";
265  // rev_deleted
266 
267  $revQuery = $this->revisionStore->getQueryInfo( [ 'page' ] );
268  $res = $this->db->select(
269  $revQuery['tables'],
270  [
271  'rev_user_text' => $revQuery['fields']['rev_user_text'],
272  'rev_user' => $revQuery['fields']['rev_user'],
273  ],
274  [
275  $this->db->bitAnd( 'rev_deleted', RevisionRecord::DELETED_USER ) . ' = 0',
276  $cond,
277  ],
278  __METHOD__,
279  [ 'DISTINCT' ],
280  $revQuery['joins']
281  );
282 
283  foreach ( $res as $row ) {
284  $this->author_list .= "<contributor>" .
285  "<username>" .
286  htmlspecialchars( $row->rev_user_text ) .
287  "</username>" .
288  "<id>" .
289  ( (int)$row->rev_user ) .
290  "</id>" .
291  "</contributor>";
292  }
293  $this->author_list .= "</contributors>";
294  }
295 
302  protected function dumpFrom( $cond = '', $orderRevs = false ) {
303  if ( $this->history & self::LOGS ) {
304  $this->dumpLogs( $cond );
305  } else {
306  $this->dumpPages( $cond, $orderRevs );
307  }
308  }
309 
314  protected function dumpLogs( $cond ) {
315  $where = [];
316  # Hide private logs
317  $hideLogs = LogEventsList::getExcludeClause( $this->db );
318  if ( $hideLogs ) {
319  $where[] = $hideLogs;
320  }
321  # Add on any caller specified conditions
322  if ( $cond ) {
323  $where[] = $cond;
324  }
325  $result = null; // Assuring $result is not undefined, if exception occurs early
326 
327  $commentQuery = CommentStore::getStore()->getJoin( 'log_comment' );
328 
329  $tables = array_merge(
330  [ 'logging', 'actor' ], $commentQuery['tables']
331  );
332  $fields = [
333  'log_id', 'log_type', 'log_action', 'log_timestamp', 'log_namespace',
334  'log_title', 'log_params', 'log_deleted', 'actor_user', 'actor_name'
335  ] + $commentQuery['fields'];
336  $options = [
337  'ORDER BY' => 'log_id',
338  'USE INDEX' => [ 'logging' => 'PRIMARY' ],
339  'LIMIT' => self::BATCH_SIZE,
340  ];
341  $joins = [
342  'actor' => [ 'JOIN', 'actor_id=log_actor' ]
343  ] + $commentQuery['joins'];
344 
345  $lastLogId = 0;
346  while ( true ) {
347  $result = $this->db->select(
348  $tables,
349  $fields,
350  array_merge( $where, [ 'log_id > ' . intval( $lastLogId ) ] ),
351  __METHOD__,
352  $options,
353  $joins
354  );
355 
356  if ( !$result->numRows() ) {
357  break;
358  }
359 
360  $lastLogId = $this->outputLogStream( $result );
361  }
362  }
363 
370  protected function dumpPages( $cond, $orderRevs ) {
371  $revQuery = $this->revisionStore->getQueryInfo( [ 'page' ] );
372  $slotQuery = $this->revisionStore->getSlotsQueryInfo( [ 'content' ] );
373 
374  // We want page primary rather than revision.
375  // We also want to join in the slots and content tables.
376  // NOTE: This means we may get multiple rows per revision, and more rows
377  // than the batch size! Should be ok, since the max number of slots is
378  // fixed and low (dozens at worst).
379  $tables = array_merge( [ 'page' ], array_diff( $revQuery['tables'], [ 'page' ] ) );
380  $tables = array_merge( $tables, array_diff( $slotQuery['tables'], $tables ) );
381  $join = $revQuery['joins'] + [
382  'revision' => $revQuery['joins']['page'],
383  'slots' => [ 'JOIN', [ 'slot_revision_id = rev_id' ] ],
384  'content' => [ 'JOIN', [ 'content_id = slot_content_id' ] ],
385  ];
386  unset( $join['page'] );
387 
388  $fields = array_merge( $revQuery['fields'], $slotQuery['fields'] );
389  $fields[] = 'page_restrictions';
390 
391  if ( $this->text != self::STUB ) {
392  $fields['_load_content'] = '1';
393  }
394 
395  $conds = [];
396  if ( $cond !== '' ) {
397  $conds[] = $cond;
398  }
399  $opts = [ 'ORDER BY' => [ 'rev_page ASC', 'rev_id ASC' ] ];
400  $opts['USE INDEX'] = [];
401 
402  $op = '>';
403  if ( is_array( $this->history ) ) {
404  # Time offset/limit for all pages/history...
405  # Set time order
406  if ( $this->history['dir'] == 'asc' ) {
407  $opts['ORDER BY'] = 'rev_timestamp ASC';
408  } else {
409  $op = '<';
410  $opts['ORDER BY'] = 'rev_timestamp DESC';
411  }
412  # Set offset
413  if ( !empty( $this->history['offset'] ) ) {
414  $conds[] = "rev_timestamp $op " .
415  $this->db->addQuotes( $this->db->timestamp( $this->history['offset'] ) );
416  }
417  # Set query limit
418  if ( !empty( $this->history['limit'] ) ) {
419  $maxRowCount = intval( $this->history['limit'] );
420  }
421  } elseif ( $this->history & self::FULL ) {
422  # Full history dumps...
423  # query optimization for history stub dumps
424  if ( $this->text == self::STUB ) {
425  $opts[] = 'STRAIGHT_JOIN';
426  $opts['USE INDEX']['revision'] = 'rev_page_id';
427  unset( $join['revision'] );
428  $join['page'] = [ 'JOIN', 'rev_page=page_id' ];
429  }
430  } elseif ( $this->history & self::CURRENT ) {
431  # Latest revision dumps...
432  if ( $this->list_authors && $cond != '' ) { // List authors, if so desired
433  $this->do_list_authors( $cond );
434  }
435  $join['revision'] = [ 'JOIN', 'page_id=rev_page AND page_latest=rev_id' ];
436  $opts[ 'ORDER BY' ] = [ 'page_id ASC' ];
437  } elseif ( $this->history & self::STABLE ) {
438  # "Stable" revision dumps...
439  # Default JOIN, to be overridden...
440  $join['revision'] = [ 'JOIN', 'page_id=rev_page AND page_latest=rev_id' ];
441  # One, and only one hook should set this, and return false
442  if ( $this->hookRunner->onWikiExporter__dumpStableQuery( $tables, $opts, $join ) ) {
443  throw new MWException( __METHOD__ . " given invalid history dump type." );
444  }
445  } elseif ( $this->history & self::RANGE ) {
446  # Dump of revisions within a specified range. Condition already set in revsByRange().
447  } else {
448  # Unknown history specification parameter?
449  throw new MWException( __METHOD__ . " given invalid history dump type." );
450  }
451 
452  $result = null; // Assuring $result is not undefined, if exception occurs early
453  $done = false;
454  $lastRow = null;
455  $revPage = 0;
456  $revId = 0;
457  $rowCount = 0;
458 
459  $opts['LIMIT'] = self::BATCH_SIZE;
460 
461  $this->hookRunner->onModifyExportQuery(
462  $this->db, $tables, $cond, $opts, $join, $conds );
463 
464  while ( !$done ) {
465  // If necessary, impose the overall maximum and stop looping after this iteration.
466  if ( !empty( $maxRowCount ) && $rowCount + self::BATCH_SIZE > $maxRowCount ) {
467  $opts['LIMIT'] = $maxRowCount - $rowCount;
468  $done = true;
469  }
470 
471  $queryConds = $conds;
472  $queryConds[] = 'rev_page>' . intval( $revPage ) . ' OR (rev_page=' .
473  intval( $revPage ) . ' AND rev_id' . $op . intval( $revId ) . ')';
474 
475  # Do the query and process any results, remembering max ids for the next iteration.
476  $result = $this->db->select(
477  $tables,
478  $fields,
479  $queryConds,
480  __METHOD__,
481  $opts,
482  $join
483  );
484  if ( $result->numRows() > 0 ) {
485  $lastRow = $this->outputPageStreamBatch( $result, $lastRow );
486  $rowCount += $result->numRows();
487  $revPage = $lastRow->rev_page;
488  $revId = $lastRow->rev_id;
489  } else {
490  $done = true;
491  }
492 
493  // If we are finished, close off final page element (if any).
494  if ( $done && $lastRow ) {
495  $this->finishPageStreamOutput( $lastRow );
496  }
497  }
498  }
499 
509  protected function outputPageStreamBatch( $results, $lastRow ) {
510  $rowCarry = null;
511  while ( true ) {
512  $slotRows = $this->getSlotRowBatch( $results, $rowCarry );
513 
514  if ( !$slotRows ) {
515  break;
516  }
517 
518  // All revision info is present in all slot rows.
519  // Use the first slot row as the revision row.
520  $revRow = $slotRows[0];
521 
522  if ( $this->limitNamespaces &&
523  !in_array( $revRow->page_namespace, $this->limitNamespaces ) ) {
524  $lastRow = $revRow;
525  continue;
526  }
527 
528  if ( $lastRow === null ||
529  $lastRow->page_namespace !== $revRow->page_namespace ||
530  $lastRow->page_title !== $revRow->page_title ) {
531  if ( $lastRow !== null ) {
532  $output = '';
533  if ( $this->dumpUploads ) {
534  $output .= $this->writer->writeUploads( $lastRow, $this->dumpUploadFileContents );
535  }
536  $output .= $this->writer->closePage();
537  $this->sink->writeClosePage( $output );
538  }
539  $output = $this->writer->openPage( $revRow );
540  $this->sink->writeOpenPage( $revRow, $output );
541  }
542  $output = $this->writer->writeRevision( $revRow, $slotRows );
543  $this->sink->writeRevision( $revRow, $output );
544  $lastRow = $revRow;
545  }
546 
547  if ( $rowCarry ) {
548  throw new LogicException( 'Error while processing a stream of slot rows' );
549  }
550 
551  return $lastRow;
552  }
553 
563  protected function getSlotRowBatch( $results, &$carry = null ) {
564  $slotRows = [];
565  $prev = null;
566 
567  if ( $carry ) {
568  $slotRows[] = $carry;
569  $prev = $carry;
570  $carry = null;
571  }
572 
573  while ( $row = $results->fetchObject() ) {
574  if ( $prev && $prev->rev_id !== $row->rev_id ) {
575  $carry = $row;
576  break;
577  }
578  $slotRows[] = $row;
579  $prev = $row;
580  }
581 
582  return $slotRows;
583  }
584 
590  protected function finishPageStreamOutput( $lastRow ) {
591  $output = '';
592  if ( $this->dumpUploads ) {
593  $output .= $this->writer->writeUploads( $lastRow, $this->dumpUploadFileContents );
594  }
595  $output .= $this->author_list;
596  $output .= $this->writer->closePage();
597  $this->sink->writeClosePage( $output );
598  }
599 
604  protected function outputLogStream( $resultset ) {
605  foreach ( $resultset as $row ) {
606  $output = $this->writer->writeLogItem( $row );
607  $this->sink->writeLogItem( $row, $output );
608  }
609  return $row->log_id ?? null;
610  }
611 }
Page\PageIdentity
Interface for objects (potentially) representing an editable wiki page.
Definition: PageIdentity.php:64
WikiExporter\schemaVersion
static schemaVersion()
Returns the default export schema version, as defined by $wgXmlDumpSchemaVersion.
Definition: WikiExporter.php:96
MediaWiki\Revision\RevisionRecord
Page revision base class.
Definition: RevisionRecord.php:47
WikiExporter\revsByRange
revsByRange( $start, $end)
Dumps a series of page and revision records for those pages in the database with revisions falling wi...
Definition: WikiExporter.php:199
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:180
WikiExporter\$revisionStore
RevisionStore $revisionStore
Definition: WikiExporter.php:84
WikiExporter\CURRENT
const CURRENT
Definition: WikiExporter.php:55
MediaWiki\Revision\RevisionStore
Service for looking up page revisions.
Definition: RevisionStore.php:88
WikiExporter\finishPageStreamOutput
finishPageStreamOutput( $lastRow)
Final page stream output, after all batches are complete.
Definition: WikiExporter.php:590
WikiExporter\getSlotRowBatch
getSlotRowBatch( $results, &$carry=null)
Returns all slot rows for a revision.
Definition: WikiExporter.php:563
WikiExporter\$dumpUploadFileContents
bool $dumpUploadFileContents
Definition: WikiExporter.php:49
LogEventsList\getExcludeClause
static getExcludeClause( $db, $audience='public', Authority $performer=null)
SQL clause to skip forbidden log types for this user.
Definition: LogEventsList.php:795
$res
$res
Definition: testCompression.php:57
WikiExporter\allLogs
allLogs()
Definition: WikiExporter.php:240
$revQuery
$revQuery
Definition: testCompression.php:56
WikiExporter\dumpFrom
dumpFrom( $cond='', $orderRevs=false)
Definition: WikiExporter.php:302
WikiExporter\openStream
openStream()
Definition: WikiExporter.php:150
Wikimedia\Rdbms\IDatabase
Basic database interface for live and lazy-loaded relation database handles.
Definition: IDatabase.php:38
WikiExporter\STUB
const STUB
Definition: WikiExporter.php:61
WikiExporter\$text
int $text
Definition: WikiExporter.php:66
WikiExporter\$list_authors
bool $list_authors
Return distinct author list (when not returning full history)
Definition: WikiExporter.php:43
MWException
MediaWiki exception.
Definition: MWException.php:29
WikiExporter\$history
array int $history
Definition: WikiExporter.php:78
XmlDumpWriter\WRITE_CONTENT
const WRITE_CONTENT
Output serialized revision content.
Definition: XmlDumpWriter.php:42
WikiExporter\pagesByName
pagesByName( $names)
Definition: WikiExporter.php:234
$wgXmlDumpSchemaVersion
$wgXmlDumpSchemaVersion
The schema to use per default when generating XML dumps.
Definition: DefaultSettings.php:8401
Wikimedia\Rdbms\IResultWrapper
Result wrapper for grabbing data queried from an IDatabase object.
Definition: IResultWrapper.php:24
WikiExporter\TEXT
const TEXT
Definition: WikiExporter.php:60
Page\PageReference\getNamespace
getNamespace()
Returns the page's namespace number.
WikiExporter\outputLogStream
outputLogStream( $resultset)
Definition: WikiExporter.php:604
WikiExporter\closeStream
closeStream()
Definition: WikiExporter.php:155
WikiExporter\$limitNamespaces
array null $limitNamespaces
Definition: WikiExporter.php:81
WikiExporter\allPages
allPages()
Dumps a series of page and revision records for all pages in the database, either including complete ...
Definition: WikiExporter.php:165
WikiExporter\$dumpUploads
bool $dumpUploads
Definition: WikiExporter.php:46
WikiExporter\$hookRunner
HookRunner $hookRunner
Definition: WikiExporter.php:90
WikiExporter\STABLE
const STABLE
Definition: WikiExporter.php:56
WikiExporter\setSchemaVersion
setSchemaVersion( $schemaVersion)
Definition: WikiExporter.php:135
TitleParser
A title parser service for MediaWiki.
Definition: TitleParser.php:33
DumpOutput
Definition: DumpOutput.php:29
WikiExporter
Definition: WikiExporter.php:41
WikiExporter\$author_list
string $author_list
Definition: WikiExporter.php:52
WikiExporter\dumpPages
dumpPages( $cond, $orderRevs)
Definition: WikiExporter.php:370
WikiExporter\pagesByRange
pagesByRange( $start, $end, $orderRevs)
Dumps a series of page and revision records for those pages in the database falling within the page_i...
Definition: WikiExporter.php:177
Page\PageReference\getDBkey
getDBkey()
Get the page title in DB key form.
WikiExporter\do_list_authors
do_list_authors( $cond)
Generates the distinct list of authors of an article Not called by default (depends on $this->list_au...
Definition: WikiExporter.php:263
WikiExporter\FULL
const FULL
Definition: WikiExporter.php:54
WikiExporter\pageByTitle
pageByTitle(PageIdentity $page)
Definition: WikiExporter.php:210
WikiExporter\__construct
__construct( $db, $history=self::CURRENT, $text=self::TEXT, $limitNamespaces=null)
Definition: WikiExporter.php:112
WikiExporter\$writer
XmlDumpWriter $writer
Definition: WikiExporter.php:72
WikiExporter\$titleParser
TitleParser $titleParser
Definition: WikiExporter.php:87
WikiExporter\RANGE
const RANGE
Definition: WikiExporter.php:58
XmlDumpWriter\WRITE_STUB
const WRITE_STUB
Only output subs for revision content.
Definition: XmlDumpWriter.php:45
WikiExporter\setOutputSink
setOutputSink(&$sink)
Set the DumpOutput or DumpFilter object which will receive various row objects and XML output for fil...
Definition: WikiExporter.php:146
MalformedTitleException
MalformedTitleException is thrown when a TitleParser is unable to parse a title string.
Definition: MalformedTitleException.php:26
WikiExporter\LOGS
const LOGS
Definition: WikiExporter.php:57
WikiExporter\BATCH_SIZE
const BATCH_SIZE
Definition: WikiExporter.php:63
XmlDumpWriter
Definition: XmlDumpWriter.php:39
WikiExporter\logsByRange
logsByRange( $start, $end)
Definition: WikiExporter.php:248
WikiExporter\outputPageStreamBatch
outputPageStreamBatch( $results, $lastRow)
Runs through a query result set dumping page, revision, and slot records.
Definition: WikiExporter.php:509
MediaWiki\HookContainer\HookRunner
This class provides an implementation of the core hook interfaces, forwarding hook calls to HookConta...
Definition: HookRunner.php:558
WikiExporter\dumpLogs
dumpLogs( $cond)
Definition: WikiExporter.php:314
WikiExporter\pageByName
pageByName( $name)
Definition: WikiExporter.php:220
WikiExporter\$db
IDatabase $db
Definition: WikiExporter.php:75
WikiExporter\$sink
DumpOutput $sink
Definition: WikiExporter.php:69
CommentStore\getStore
static getStore()
Definition: CommentStore.php:120