MediaWiki  master
WikiExporter.php
Go to the documentation of this file.
1 <?php
34 
38 class WikiExporter {
40  public $list_authors = false;
41 
43  public $dumpUploads = false;
44 
46  public $dumpUploadFileContents = false;
47 
49  public $author_list = "";
50 
51  const FULL = 1;
52  const CURRENT = 2;
53  const STABLE = 4; // extension defined
54  const LOGS = 8;
55  const RANGE = 16;
56 
59 
60  const BATCH_SIZE = 50000;
61 
63  public $text;
64 
66  public $sink;
67 
69  private $writer;
70 
72  protected $db;
73 
75  protected $history;
76 
78  protected $limitNamespaces;
79 
84  public static function schemaVersion() {
87  }
88 
100  function __construct(
101  $db,
102  $history = self::CURRENT,
103  $text = self::TEXT,
104  $limitNamespaces = null
105  ) {
106  $this->db = $db;
107  $this->history = $history;
108  $this->writer = new XmlDumpWriter( $text, self::schemaVersion() );
109  $this->sink = new DumpOutput();
110  $this->text = $text;
111  $this->limitNamespaces = $limitNamespaces;
112  }
113 
119  public function setSchemaVersion( $schemaVersion ) {
120  $this->writer = new XmlDumpWriter( $this->text, $schemaVersion );
121  }
122 
130  public function setOutputSink( &$sink ) {
131  $this->sink =& $sink;
132  }
133 
134  public function openStream() {
135  $output = $this->writer->openStream();
136  $this->sink->writeOpenStream( $output );
137  }
138 
139  public function closeStream() {
140  $output = $this->writer->closeStream();
141  $this->sink->writeCloseStream( $output );
142  }
143 
149  public function allPages() {
150  $this->dumpFrom( '' );
151  }
152 
161  public function pagesByRange( $start, $end, $orderRevs ) {
162  if ( $orderRevs ) {
163  $condition = 'rev_page >= ' . intval( $start );
164  if ( $end ) {
165  $condition .= ' AND rev_page < ' . intval( $end );
166  }
167  } else {
168  $condition = 'page_id >= ' . intval( $start );
169  if ( $end ) {
170  $condition .= ' AND page_id < ' . intval( $end );
171  }
172  }
173  $this->dumpFrom( $condition, $orderRevs );
174  }
175 
183  public function revsByRange( $start, $end ) {
184  $condition = 'rev_id >= ' . intval( $start );
185  if ( $end ) {
186  $condition .= ' AND rev_id < ' . intval( $end );
187  }
188  $this->dumpFrom( $condition );
189  }
190 
194  public function pageByTitle( $title ) {
195  $this->dumpFrom(
196  'page_namespace=' . $title->getNamespace() .
197  ' AND page_title=' . $this->db->addQuotes( $title->getDBkey() ) );
198  }
199 
204  public function pageByName( $name ) {
205  $title = Title::newFromText( $name );
206  if ( is_null( $title ) ) {
207  throw new MWException( "Can't export invalid title" );
208  } else {
209  $this->pageByTitle( $title );
210  }
211  }
212 
216  public function pagesByName( $names ) {
217  foreach ( $names as $name ) {
218  $this->pageByName( $name );
219  }
220  }
221 
222  public function allLogs() {
223  $this->dumpFrom( '' );
224  }
225 
230  public function logsByRange( $start, $end ) {
231  $condition = 'log_id >= ' . intval( $start );
232  if ( $end ) {
233  $condition .= ' AND log_id < ' . intval( $end );
234  }
235  $this->dumpFrom( $condition );
236  }
237 
245  protected function do_list_authors( $cond ) {
246  $this->author_list = "<contributors>";
247  // rev_deleted
248 
249  $revQuery = Revision::getQueryInfo( [ 'page' ] );
250  $res = $this->db->select(
251  $revQuery['tables'],
252  [
253  'rev_user_text' => $revQuery['fields']['rev_user_text'],
254  'rev_user' => $revQuery['fields']['rev_user'],
255  ],
256  [
257  $this->db->bitAnd( 'rev_deleted', RevisionRecord::DELETED_USER ) . ' = 0',
258  $cond,
259  ],
260  __METHOD__,
261  [ 'DISTINCT' ],
262  $revQuery['joins']
263  );
264 
265  foreach ( $res as $row ) {
266  $this->author_list .= "<contributor>" .
267  "<username>" .
268  htmlspecialchars( $row->rev_user_text ) .
269  "</username>" .
270  "<id>" .
271  ( (int)$row->rev_user ) .
272  "</id>" .
273  "</contributor>";
274  }
275  $this->author_list .= "</contributors>";
276  }
277 
284  protected function dumpFrom( $cond = '', $orderRevs = false ) {
285  if ( $this->history & self::LOGS ) {
286  $this->dumpLogs( $cond );
287  } else {
288  $this->dumpPages( $cond, $orderRevs );
289  }
290  }
291 
296  protected function dumpLogs( $cond ) {
297  $where = [];
298  # Hide private logs
299  $hideLogs = LogEventsList::getExcludeClause( $this->db );
300  if ( $hideLogs ) {
301  $where[] = $hideLogs;
302  }
303  # Add on any caller specified conditions
304  if ( $cond ) {
305  $where[] = $cond;
306  }
307  $result = null; // Assuring $result is not undefined, if exception occurs early
308 
309  $commentQuery = CommentStore::getStore()->getJoin( 'log_comment' );
310  $actorQuery = ActorMigration::newMigration()->getJoin( 'log_user' );
311 
312  $tables = array_merge(
313  [ 'logging' ], $commentQuery['tables'], $actorQuery['tables'], [ 'user' ]
314  );
315  $fields = [
316  'log_id', 'log_type', 'log_action', 'log_timestamp', 'log_namespace',
317  'log_title', 'log_params', 'log_deleted', 'user_name'
318  ] + $commentQuery['fields'] + $actorQuery['fields'];
319  $options = [
320  'ORDER BY' => 'log_id',
321  'USE INDEX' => [ 'logging' => 'PRIMARY' ],
322  'LIMIT' => self::BATCH_SIZE,
323  ];
324  $joins = [
325  'user' => [ 'JOIN', 'user_id = ' . $actorQuery['fields']['log_user'] ]
326  ] + $commentQuery['joins'] + $actorQuery['joins'];
327 
328  $lastLogId = 0;
329  while ( true ) {
330  $result = $this->db->select(
331  $tables,
332  $fields,
333  array_merge( $where, [ 'log_id > ' . intval( $lastLogId ) ] ),
334  __METHOD__,
335  $options,
336  $joins
337  );
338 
339  if ( !$result->numRows() ) {
340  break;
341  }
342 
343  $lastLogId = $this->outputLogStream( $result );
344  }
345  }
346 
353  protected function dumpPages( $cond, $orderRevs ) {
355  if ( !( $wgMultiContentRevisionSchemaMigrationStage & SCHEMA_COMPAT_WRITE_OLD ) ) {
356  // TODO: Make XmlDumpWriter use a RevisionStore! (see T198706 and T174031)
357  throw new MWException(
358  'Cannot use WikiExporter with SCHEMA_COMPAT_WRITE_OLD mode disabled!'
359  . ' Support for dumping from the new schema is not implemented yet!'
360  );
361  }
362 
363  $revQuery = MediaWikiServicesAlias::getInstance()->getRevisionStore()->getQueryInfo(
364  [ 'page' ]
365  );
366  $slotQuery = MediaWikiServicesAlias::getInstance()->getRevisionStore()->getSlotsQueryInfo(
367  [ 'content' ]
368  );
369 
370  // We want page primary rather than revision.
371  // We also want to join in the slots and content tables.
372  // NOTE: This means we may get multiple rows per revision, and more rows
373  // than the batch size! Should be ok, since the max number of slots is
374  // fixed and low (dozens at worst).
375  $tables = array_merge( [ 'page' ], array_diff( $revQuery['tables'], [ 'page' ] ) );
376  $tables = array_merge( $tables, array_diff( $slotQuery['tables'], $tables ) );
377  $join = $revQuery['joins'] + [
378  'revision' => $revQuery['joins']['page'],
379  'slots' => [ 'JOIN', [ 'slot_revision_id = rev_id' ] ],
380  'content' => [ 'JOIN', [ 'content_id = slot_content_id' ] ],
381  ];
382  unset( $join['page'] );
383 
384  $fields = array_merge( $revQuery['fields'], $slotQuery['fields'] );
385  $fields[] = 'page_restrictions';
386 
387  if ( $this->text != self::STUB ) {
388  $fields['_load_content'] = '1';
389  }
390 
391  $conds = [];
392  if ( $cond !== '' ) {
393  $conds[] = $cond;
394  }
395  $opts = [ 'ORDER BY' => [ 'rev_page ASC', 'rev_id ASC' ] ];
396  $opts['USE INDEX'] = [];
397 
398  $op = '>';
399  if ( is_array( $this->history ) ) {
400  # Time offset/limit for all pages/history...
401  # Set time order
402  if ( $this->history['dir'] == 'asc' ) {
403  $opts['ORDER BY'] = 'rev_timestamp ASC';
404  } else {
405  $op = '<';
406  $opts['ORDER BY'] = 'rev_timestamp DESC';
407  }
408  # Set offset
409  if ( !empty( $this->history['offset'] ) ) {
410  $conds[] = "rev_timestamp $op " .
411  $this->db->addQuotes( $this->db->timestamp( $this->history['offset'] ) );
412  }
413  # Set query limit
414  if ( !empty( $this->history['limit'] ) ) {
415  $maxRowCount = intval( $this->history['limit'] );
416  }
417  } elseif ( $this->history & self::FULL ) {
418  # Full history dumps...
419  # query optimization for history stub dumps
420  if ( $this->text == self::STUB ) {
421  $opts[] = 'STRAIGHT_JOIN';
422  $opts['USE INDEX']['revision'] = 'rev_page_id';
423  unset( $join['revision'] );
424  $join['page'] = [ 'JOIN', 'rev_page=page_id' ];
425  }
426  } elseif ( $this->history & self::CURRENT ) {
427  # Latest revision dumps...
428  if ( $this->list_authors && $cond != '' ) { // List authors, if so desired
429  $this->do_list_authors( $cond );
430  }
431  $join['revision'] = [ 'JOIN', 'page_id=rev_page AND page_latest=rev_id' ];
432  $opts[ 'ORDER BY' ] = [ 'page_id ASC' ];
433  } elseif ( $this->history & self::STABLE ) {
434  # "Stable" revision dumps...
435  # Default JOIN, to be overridden...
436  $join['revision'] = [ 'JOIN', 'page_id=rev_page AND page_latest=rev_id' ];
437  # One, and only one hook should set this, and return false
438  if ( Hooks::run( 'WikiExporter::dumpStableQuery', [ &$tables, &$opts, &$join ] ) ) {
439  throw new MWException( __METHOD__ . " given invalid history dump type." );
440  }
441  } elseif ( $this->history & self::RANGE ) {
442  # Dump of revisions within a specified range. Condition already set in revsByRange().
443  } else {
444  # Unknown history specification parameter?
445  throw new MWException( __METHOD__ . " given invalid history dump type." );
446  }
447 
448  $result = null; // Assuring $result is not undefined, if exception occurs early
449  $done = false;
450  $lastRow = null;
451  $revPage = 0;
452  $revId = 0;
453  $rowCount = 0;
454 
455  $opts['LIMIT'] = self::BATCH_SIZE;
456 
457  Hooks::run( 'ModifyExportQuery',
458  [ $this->db, &$tables, &$cond, &$opts, &$join ] );
459 
460  while ( !$done ) {
461  // If necessary, impose the overall maximum and stop looping after this iteration.
462  if ( !empty( $maxRowCount ) && $rowCount + self::BATCH_SIZE > $maxRowCount ) {
463  $opts['LIMIT'] = $maxRowCount - $rowCount;
464  $done = true;
465  }
466 
467  $queryConds = $conds;
468  $queryConds[] = 'rev_page>' . intval( $revPage ) . ' OR (rev_page=' .
469  intval( $revPage ) . ' AND rev_id' . $op . intval( $revId ) . ')';
470 
471  # Do the query and process any results, remembering max ids for the next iteration.
472  $result = $this->db->select(
473  $tables,
474  $fields,
475  $queryConds,
476  __METHOD__,
477  $opts,
478  $join
479  );
480  if ( $result->numRows() > 0 ) {
481  $lastRow = $this->outputPageStreamBatch( $result, $lastRow );
482  $rowCount += $result->numRows();
483  $revPage = $lastRow->rev_page;
484  $revId = $lastRow->rev_id;
485  } else {
486  $done = true;
487  }
488 
489  // If we are finished, close off final page element (if any).
490  if ( $done && $lastRow ) {
491  $this->finishPageStreamOutput( $lastRow );
492  }
493  }
494  }
495 
505  protected function outputPageStreamBatch( $results, $lastRow ) {
506  $rowCarry = null;
507  while ( true ) {
508  $slotRows = $this->getSlotRowBatch( $results, $rowCarry );
509 
510  if ( !$slotRows ) {
511  break;
512  }
513 
514  // All revision info is present in all slot rows.
515  // Use the first slot row as the revision row.
516  $revRow = $slotRows[0];
517 
518  if ( $this->limitNamespaces &&
519  !in_array( $revRow->page_namespace, $this->limitNamespaces ) ) {
520  $lastRow = $revRow;
521  continue;
522  }
523 
524  if ( $lastRow === null ||
525  $lastRow->page_namespace !== $revRow->page_namespace ||
526  $lastRow->page_title !== $revRow->page_title ) {
527  if ( $lastRow !== null ) {
528  $output = '';
529  if ( $this->dumpUploads ) {
530  $output .= $this->writer->writeUploads( $lastRow, $this->dumpUploadFileContents );
531  }
532  $output .= $this->writer->closePage();
533  $this->sink->writeClosePage( $output );
534  }
535  $output = $this->writer->openPage( $revRow );
536  $this->sink->writeOpenPage( $revRow, $output );
537  }
538  $output = $this->writer->writeRevision( $revRow, $slotRows );
539  $this->sink->writeRevision( $revRow, $output );
540  $lastRow = $revRow;
541  }
542 
543  if ( $rowCarry ) {
544  throw new LogicException( 'Error while processing a stream of slot rows' );
545  }
546 
547  return $lastRow;
548  }
549 
559  protected function getSlotRowBatch( $results, &$carry = null ) {
560  $slotRows = [];
561  $prev = null;
562 
563  if ( $carry ) {
564  $slotRows[] = $carry;
565  $prev = $carry;
566  $carry = null;
567  }
568 
569  while ( $row = $results->fetchObject() ) {
570  if ( $prev && $prev->rev_id !== $row->rev_id ) {
571  $carry = $row;
572  break;
573  }
574  $slotRows[] = $row;
575  $prev = $row;
576  }
577 
578  return $slotRows;
579  }
580 
586  protected function finishPageStreamOutput( $lastRow ) {
587  $output = '';
588  if ( $this->dumpUploads ) {
589  $output .= $this->writer->writeUploads( $lastRow, $this->dumpUploadFileContents );
590  }
591  $output .= $this->author_list;
592  $output .= $this->writer->closePage();
593  $this->sink->writeClosePage( $output );
594  }
595 
600  protected function outputLogStream( $resultset ) {
601  foreach ( $resultset as $row ) {
602  $output = $this->writer->writeLogItem( $row );
603  $this->sink->writeLogItem( $row, $output );
604  }
605  return isset( $row ) ? $row->log_id : null;
606  }
607 }
const SCHEMA_COMPAT_WRITE_OLD
Definition: Defines.php:264
array null $limitNamespaces
DumpOutput $sink
bool $list_authors
Return distinct author list (when not returning full history)
int $wgMultiContentRevisionSchemaMigrationStage
RevisionStore table schema migration stage (content, slots, content_models & slot_roles tables)...
const WRITE_CONTENT
Output serialized revision content.
IDatabase $db
setOutputSink(&$sink)
Set the DumpOutput or DumpFilter object which will receive various row objects and XML output for fil...
const WRITE_STUB
Only output subs for revision content.
logsByRange( $start, $end)
XmlDumpWriter $writer
allPages()
Dumps a series of page and revision records for all pages in the database, either including complete ...
string $author_list
setSchemaVersion( $schemaVersion)
dumpPages( $cond, $orderRevs)
dumpLogs( $cond)
finishPageStreamOutput( $lastRow)
Final page stream output, after all batches are complete.
const BATCH_SIZE
static newMigration()
Static constructor.
outputPageStreamBatch( $results, $lastRow)
Runs through a query result set dumping page, revision, and slot records.
$wgXmlDumpSchemaVersion
The schema to use per default when generating XML dumps.
array int $history
do_list_authors( $cond)
Generates the distinct list of authors of an article Not called by default (depends on $this->list_au...
static getQueryInfo( $options=[])
Return the tables, fields, and join conditions to be selected to create a new revision object...
Definition: Revision.php:315
pageByTitle( $title)
pageByName( $name)
revsByRange( $start, $end)
Dumps a series of page and revision records for those pages in the database with revisions falling wi...
getSlotRowBatch( $results, &$carry=null)
Returns all slot rows for a revision.
static getStore()
bool $dumpUploadFileContents
static getExcludeClause( $db, $audience='public', User $user=null)
SQL clause to skip forbidden log types for this user.
static schemaVersion()
Returns the default export schema version, as defined by $wgXmlDumpSchemaVersion. ...
$revQuery
dumpFrom( $cond='', $orderRevs=false)
__construct( $db, $history=self::CURRENT, $text=self::TEXT, $limitNamespaces=null)
outputLogStream( $resultset)
pagesByName( $names)
pagesByRange( $start, $end, $orderRevs)
Dumps a series of page and revision records for those pages in the database falling within the page_i...
static run( $event, array $args=[], $deprecatedVersion=null)
Call hook functions defined in Hooks::register and $wgHooks.
Definition: Hooks.php:200
static newFromText( $text, $defaultNamespace=NS_MAIN)
Create a new Title from text, such as what one would find in a link.
Definition: Title.php:316