MediaWiki  master
importDump.php
Go to the documentation of this file.
1 <?php
28 
29 require_once __DIR__ . '/Maintenance.php';
30 
36 class BackupReader extends Maintenance {
37  public $reportingInterval = 100;
38  public $pageCount = 0;
39  public $revCount = 0;
40  public $dryRun = false;
41  public $uploads = false;
42  protected $uploadCount = 0;
43  public $imageBasePath = false;
45  public $nsFilter = false;
47  public $stderr;
49  protected $importCallback;
51  protected $logItemCallback;
53  protected $uploadCallback;
55  protected $startTime;
56 
57  public function __construct() {
58  parent::__construct();
59  $gz = in_array( 'compress.zlib', stream_get_wrappers() )
60  ? 'ok'
61  : '(disabled; requires PHP zlib module)';
62  $bz2 = in_array( 'compress.bzip2', stream_get_wrappers() )
63  ? 'ok'
64  : '(disabled; requires PHP bzip2 module)';
65 
66  $this->addDescription(
67  <<<TEXT
68 This script reads pages from an XML file as produced from Special:Export or
69 dumpBackup.php, and saves them into the current wiki.
70 
71 Compressed XML files may be read directly:
72  .gz $gz
73  .bz2 $bz2
74  .7z (if 7za executable is in PATH)
75 
76 Note that for very large data sets, importDump.php may be slow; there are
77 alternate methods which can be much faster for full site restoration:
78 <https://www.mediawiki.org/wiki/Manual:Importing_XML_dumps>
79 TEXT
80  );
81  $this->stderr = fopen( "php://stderr", "wt" );
82  $this->addOption( 'report',
83  'Report position and speed after every n pages processed', false, true );
84  $this->addOption( 'namespaces',
85  'Import only the pages from namespaces belonging to the list of ' .
86  'pipe-separated namespace names or namespace indexes', false, true );
87  $this->addOption( 'rootpage', 'Pages will be imported as subpages of the specified page',
88  false, true );
89  $this->addOption( 'dry-run', 'Parse dump without actually importing pages' );
90  $this->addOption( 'debug', 'Output extra verbose debug information' );
91  $this->addOption( 'uploads', 'Process file upload data if included (experimental)' );
92  $this->addOption(
93  'no-updates',
94  'Disable link table updates. Is faster but leaves the wiki in an inconsistent state'
95  );
96  $this->addOption( 'image-base-path', 'Import files from a specified path', false, true );
97  $this->addOption( 'skip-to', 'Start from nth page by skipping first n-1 pages', false, true );
98  $this->addOption( 'username-prefix', 'Prefix for interwiki usernames', false, true );
99  $this->addOption( 'no-local-users',
100  'Treat all usernames as interwiki. ' .
101  'The default is to assign edits to local users where they exist.',
102  false, false
103  );
104  $this->addArg( 'file', 'Dump file to import [else use stdin]', false );
105  }
106 
107  public function execute() {
108  if ( wfReadOnly() ) {
109  $this->fatalError( "Wiki is in read-only mode; you'll need to disable it for import to work." );
110  }
111 
112  $this->reportingInterval = intval( $this->getOption( 'report', 100 ) );
113  if ( !$this->reportingInterval ) {
114  $this->reportingInterval = 100; // avoid division by zero
115  }
116 
117  $this->dryRun = $this->hasOption( 'dry-run' );
118  $this->uploads = $this->hasOption( 'uploads' ); // experimental!
119  if ( $this->hasOption( 'image-base-path' ) ) {
120  $this->imageBasePath = $this->getOption( 'image-base-path' );
121  }
122  if ( $this->hasOption( 'namespaces' ) ) {
123  $this->setNsfilter( explode( '|', $this->getOption( 'namespaces' ) ) );
124  }
125 
126  if ( $this->hasArg( 0 ) ) {
127  $this->importFromFile( $this->getArg( 0 ) );
128  } else {
129  $this->importFromStdin();
130  }
131 
132  $this->output( "Done!\n" );
133  $this->output( "You might want to run rebuildrecentchanges.php to regenerate RecentChanges,\n" );
134  $this->output( "and initSiteStats.php to update page and revision counts\n" );
135  }
136 
137  function setNsfilter( array $namespaces ) {
138  if ( count( $namespaces ) == 0 ) {
139  $this->nsFilter = false;
140 
141  return;
142  }
143  $this->nsFilter = array_unique( array_map( [ $this, 'getNsIndex' ], $namespaces ) );
144  }
145 
146  private function getNsIndex( $namespace ) {
147  $contLang = MediaWikiServices::getInstance()->getContentLanguage();
148  $result = $contLang->getNsIndex( $namespace );
149  if ( $result !== false ) {
150  return $result;
151  }
152  $ns = intval( $namespace );
153  if ( strval( $ns ) === $namespace && $contLang->getNsText( $ns ) !== false ) {
154  return $ns;
155  }
156  $this->fatalError( "Unknown namespace text / index specified: $namespace" );
157  }
158 
164  private function skippedNamespace( $obj ) {
165  $title = null;
166  if ( $obj instanceof Title ) {
167  $title = $obj;
168  } elseif ( $obj instanceof Revision ) {
169  $title = $obj->getTitle();
170  } elseif ( $obj instanceof WikiRevision ) {
171  $title = $obj->title;
172  } else {
173  throw new MWException( "Cannot get namespace of object in " . __METHOD__ );
174  }
175 
176  if ( is_null( $title ) ) {
177  // Probably a log entry
178  return false;
179  }
180 
181  $ns = $title->getNamespace();
182 
183  return is_array( $this->nsFilter ) && !in_array( $ns, $this->nsFilter );
184  }
185 
186  function reportPage( $page ) {
187  $this->pageCount++;
188  }
189 
193  function handleRevision( $rev ) {
194  $title = $rev->getTitle();
195  if ( !$title ) {
196  $this->progress( "Got bogus revision with null title!" );
197 
198  return;
199  }
200 
201  if ( $this->skippedNamespace( $title ) ) {
202  return;
203  }
204 
205  $this->revCount++;
206  $this->report();
207 
208  if ( !$this->dryRun ) {
209  call_user_func( $this->importCallback, $rev );
210  }
211  }
212 
217  function handleUpload( $revision ) {
218  if ( $this->uploads ) {
219  if ( $this->skippedNamespace( $revision ) ) {
220  return false;
221  }
222  $this->uploadCount++;
223  // $this->report();
224  // @phan-suppress-next-line PhanUndeclaredMethod
225  $this->progress( "upload: " . $revision->getFilename() );
226 
227  if ( !$this->dryRun ) {
228  // bluuuh hack
229  // call_user_func( $this->uploadCallback, $revision );
230  $dbw = $this->getDB( DB_MASTER );
231 
232  return $dbw->deadlockLoop( [ $revision, 'importUpload' ] );
233  }
234  }
235 
236  return false;
237  }
238 
239  function handleLogItem( $rev ) {
240  if ( $this->skippedNamespace( $rev ) ) {
241  return;
242  }
243  $this->revCount++;
244  $this->report();
245 
246  if ( !$this->dryRun ) {
247  call_user_func( $this->logItemCallback, $rev );
248  }
249  }
250 
251  function report( $final = false ) {
252  if ( $final xor ( $this->pageCount % $this->reportingInterval == 0 ) ) {
253  $this->showReport();
254  }
255  }
256 
257  function showReport() {
258  if ( !$this->mQuiet ) {
259  $delta = microtime( true ) - $this->startTime;
260  if ( $delta ) {
261  $rate = sprintf( "%.2f", $this->pageCount / $delta );
262  $revrate = sprintf( "%.2f", $this->revCount / $delta );
263  } else {
264  $rate = '-';
265  $revrate = '-';
266  }
267  # Logs dumps don't have page tallies
268  if ( $this->pageCount ) {
269  $this->progress( "$this->pageCount ($rate pages/sec $revrate revs/sec)" );
270  } else {
271  $this->progress( "$this->revCount ($revrate revs/sec)" );
272  }
273  }
274  wfWaitForSlaves();
275  }
276 
277  function progress( $string ) {
278  fwrite( $this->stderr, $string . "\n" );
279  }
280 
281  function importFromFile( $filename ) {
282  if ( preg_match( '/\.gz$/', $filename ) ) {
283  $filename = 'compress.zlib://' . $filename;
284  } elseif ( preg_match( '/\.bz2$/', $filename ) ) {
285  $filename = 'compress.bzip2://' . $filename;
286  } elseif ( preg_match( '/\.7z$/', $filename ) ) {
287  $filename = 'mediawiki.compress.7z://' . $filename;
288  }
289 
290  $file = fopen( $filename, 'rt' );
291 
292  return $this->importFromHandle( $file );
293  }
294 
295  function importFromStdin() {
296  $file = fopen( 'php://stdin', 'rt' );
297  if ( self::posix_isatty( $file ) ) {
298  $this->maybeHelp( true );
299  }
300 
301  return $this->importFromHandle( $file );
302  }
303 
304  function importFromHandle( $handle ) {
305  $this->startTime = microtime( true );
306 
307  $source = new ImportStreamSource( $handle );
308  $importer = new WikiImporter( $source, $this->getConfig() );
309 
310  // Updating statistics require a lot of time so disable it
311  $importer->disableStatisticsUpdate();
312 
313  if ( $this->hasOption( 'debug' ) ) {
314  $importer->setDebug( true );
315  }
316  if ( $this->hasOption( 'no-updates' ) ) {
317  $importer->setNoUpdates( true );
318  }
319  if ( $this->hasOption( 'username-prefix' ) ) {
320  $importer->setUsernamePrefix(
321  $this->getOption( 'username-prefix' ),
322  !$this->hasOption( 'no-local-users' )
323  );
324  }
325  if ( $this->hasOption( 'rootpage' ) ) {
326  $statusRootPage = $importer->setTargetRootPage( $this->getOption( 'rootpage' ) );
327  if ( !$statusRootPage->isGood() ) {
328  // Die here so that it doesn't print "Done!"
329  $this->fatalError( $statusRootPage->getMessage( false, false, 'en' )->text() );
330  return false;
331  }
332  }
333  if ( $this->hasOption( 'skip-to' ) ) {
334  $nthPage = (int)$this->getOption( 'skip-to' );
335  $importer->setPageOffset( $nthPage );
336  $this->pageCount = $nthPage - 1;
337  }
338  $importer->setPageCallback( [ $this, 'reportPage' ] );
339  $importer->setNoticeCallback( function ( $msg, $params ) {
340  echo wfMessage( $msg, $params )->text() . "\n";
341  } );
342  $this->importCallback = $importer->setRevisionCallback(
343  [ $this, 'handleRevision' ] );
344  $this->uploadCallback = $importer->setUploadCallback(
345  [ $this, 'handleUpload' ] );
346  $this->logItemCallback = $importer->setLogItemCallback(
347  [ $this, 'handleLogItem' ] );
348  if ( $this->uploads ) {
349  $importer->setImportUploads( true );
350  }
351  if ( $this->imageBasePath ) {
352  $importer->setImageBasePath( $this->imageBasePath );
353  }
354 
355  if ( $this->dryRun ) {
356  $importer->setPageOutCallback( null );
357  }
358 
359  return $importer->doImport();
360  }
361 }
362 
363 $maintClass = BackupReader::class;
364 require_once RUN_MAINTENANCE_IF_MAIN;
callable null $uploadCallback
Definition: importDump.php:53
if(PHP_SAPI !='cli-server') if(!isset( $_SERVER['SCRIPT_FILENAME'])) $file
Item class for a filearchive table row.
Definition: router.php:42
handleLogItem( $rev)
Definition: importDump.php:239
importFromFile( $filename)
Definition: importDump.php:281
getArg( $argId=0, $default=null)
Get an argument.
const RUN_MAINTENANCE_IF_MAIN
Definition: Maintenance.php:39
maybeHelp( $force=false)
Maybe show the help.
setNsfilter(array $namespaces)
Definition: importDump.php:137
callable null $importCallback
Definition: importDump.php:49
XML file reader for the page data importer.
getOption( $name, $default=null)
Get an option, or return the default.
skippedNamespace( $obj)
Definition: importDump.php:164
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:86
handleUpload( $revision)
Definition: importDump.php:217
$source
callable null $logItemCallback
Definition: importDump.php:51
hasOption( $name)
Checks to see if a particular option exists.
const DB_MASTER
Definition: defines.php:26
array false $nsFilter
Definition: importDump.php:45
hasArg( $argId=0)
Does a given argument exist?
wfReadOnly()
Check whether the wiki is in read-only mode.
wfWaitForSlaves( $ifWritesSince=null, $wiki=false, $cluster=false, $timeout=null)
Waits for the replica DBs to catch up to the master position.
addDescription( $text)
Set the description text.
Represents a revision, log entry or upload during the import process.
Imports a XML dump from a file (either from file upload, files on disk, or HTTP)
addArg( $arg, $description, $required=true)
Add some args that are needed.
reportPage( $page)
Definition: importDump.php:186
Maintenance script that imports XML dump files into the current wiki.
Definition: importDump.php:36
output( $out, $channel=null)
Throw some output to the user.
bool resource $stderr
Definition: importDump.php:47
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
handleRevision( $rev)
Definition: importDump.php:193
importFromHandle( $handle)
Definition: importDump.php:304
getNsIndex( $namespace)
Definition: importDump.php:146
wfMessage( $key,... $params)
This is the function for getting translated interface messages.
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
$maintClass
Definition: importDump.php:351
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
report( $final=false)
Definition: importDump.php:251
progress( $string)
Definition: importDump.php:277