MediaWiki  1.28.0
backup.inc
Go to the documentation of this file.
1 <?php
27 require_once __DIR__ . '/Maintenance.php';
28 require_once __DIR__ . '/../includes/export/DumpFilter.php';
29 
33 class BackupDumper extends Maintenance {
34  public $reporting = true;
35  public $pages = null; // all pages
36  public $skipHeader = false; // don't output <mediawiki> and <siteinfo>
37  public $skipFooter = false; // don't output </mediawiki>
38  public $startId = 0;
39  public $endId = 0;
40  public $revStartId = 0;
41  public $revEndId = 0;
42  public $dumpUploads = false;
43  public $dumpUploadFileContents = false;
44  public $orderRevs = false;
45 
46  protected $reportingInterval = 100;
47  protected $pageCount = 0;
48  protected $revCount = 0;
49  protected $server = null; // use default
50  protected $sink = null; // Output filters
51  protected $lastTime = 0;
52  protected $pageCountLast = 0;
53  protected $revCountLast = 0;
54 
55  protected $outputTypes = [];
56  protected $filterTypes = [];
57 
58  protected $ID = 0;
59 
67  protected $forcedDb = null;
68 
70  protected $lb;
71 
72  // @todo Unused?
73  private $stubText = false; // include rev_text_id instead of text; for 2-pass dump
74 
78  function __construct( $args = null ) {
79  parent::__construct();
80  $this->stderr = fopen( "php://stderr", "wt" );
81 
82  // Built-in output and filter plugins
83  $this->registerOutput( 'file', 'DumpFileOutput' );
84  $this->registerOutput( 'gzip', 'DumpGZipOutput' );
85  $this->registerOutput( 'bzip2', 'DumpBZip2Output' );
86  $this->registerOutput( 'dbzip2', 'DumpDBZip2Output' );
87  $this->registerOutput( '7zip', 'Dump7ZipOutput' );
88 
89  $this->registerFilter( 'latest', 'DumpLatestFilter' );
90  $this->registerFilter( 'notalk', 'DumpNotalkFilter' );
91  $this->registerFilter( 'namespace', 'DumpNamespaceFilter' );
92 
93  // These three can be specified multiple times
94  $this->addOption( 'plugin', 'Load a dump plugin class. Specify as <class>[:<file>].',
95  false, true, false, true );
96  $this->addOption( 'output', 'Begin a filtered output stream; Specify as <type>:<file>. ' .
97  '<type>s: file, gzip, bzip2, 7zip, dbzip2', false, true, false, true );
98  $this->addOption( 'filter', 'Add a filter on an output branch. Specify as ' .
99  '<type>[:<options>]. <types>s: latest, notalk, namespace', false, true, false, true );
100  $this->addOption( 'report', 'Report position and speed after every n pages processed. ' .
101  'Default: 100.', false, true );
102  $this->addOption( 'server', 'Force reading from MySQL server', false, true );
103  $this->addOption( '7ziplevel', '7zip compression level for all 7zip outputs. Used for ' .
104  '-mx option to 7za command.', false, true );
105 
106  if ( $args ) {
107  // Args should be loaded and processed so that dump() can be called directly
108  // instead of execute()
109  $this->loadWithArgv( $args );
110  $this->processOptions();
111  }
112  }
113 
118  function registerOutput( $name, $class ) {
119  $this->outputTypes[$name] = $class;
120  }
121 
126  function registerFilter( $name, $class ) {
127  $this->filterTypes[$name] = $class;
128  }
129 
137  function loadPlugin( $class, $file ) {
138  if ( $file != '' ) {
139  require_once $file;
140  }
141  $register = [ $class, 'register' ];
142  call_user_func_array( $register, [ $this ] );
143  }
144 
145  function execute() {
146  throw new MWException( 'execute() must be overridden in subclasses' );
147  }
148 
152  function processOptions() {
153  $sink = null;
154  $sinks = [];
155 
157  foreach ( $options as $arg ) {
158  $opt = $arg[0];
159  $param = $arg[1];
160 
161  switch ( $opt ) {
162  case 'plugin':
163  $val = explode( ':', $param );
164 
165  if ( count( $val ) === 1 ) {
166  $this->loadPlugin( $val[0] );
167  } elseif ( count( $val ) === 2 ) {
168  $this->loadPlugin( $val[0], $val[1] );
169  } else {
170  $this->fatalError( 'Invalid plugin parameter' );
171  return;
172  }
173 
174  break;
175  case 'output':
176  $split = explode( ':', $param, 2 );
177  if ( count( $split ) !== 2 ) {
178  $this->fatalError( 'Invalid output parameter' );
179  }
180  list( $type, $file ) = $split;
181  if ( !is_null( $sink ) ) {
182  $sinks[] = $sink;
183  }
184  if ( !isset( $this->outputTypes[$type] ) ) {
185  $this->fatalError( "Unrecognized output sink type '$type'" );
186  }
187  $class = $this->outputTypes[$type];
188  if ( $type === "7zip" ) {
189  $sink = new $class( $file, intval( $this->getOption( '7ziplevel' ) ) );
190  } else {
191  $sink = new $class( $file );
192  }
193 
194  break;
195  case 'filter':
196  if ( is_null( $sink ) ) {
197  $sink = new DumpOutput();
198  }
199 
200  $split = explode( ':', $param );
201  $key = $split[0];
202 
203  if ( !isset( $this->filterTypes[$key] ) ) {
204  $this->fatalError( "Unrecognized filter type '$key'" );
205  }
206 
207  $type = $this->filterTypes[$key];
208 
209  if ( count( $split ) === 1 ) {
210  $filter = new $type( $sink );
211  } elseif ( count( $split ) === 2 ) {
212  $filter = new $type( $sink, $split[1] );
213  } else {
214  $this->fatalError( 'Invalid filter parameter' );
215  }
216 
217  // references are lame in php...
218  unset( $sink );
219  $sink = $filter;
220 
221  break;
222  }
223  }
224 
225  if ( $this->hasOption( 'report' ) ) {
226  $this->reportingInterval = intval( $this->getOption( 'report' ) );
227  }
228 
229  if ( $this->hasOption( 'server' ) ) {
230  $this->server = $this->getOption( 'server' );
231  }
232 
233  if ( is_null( $sink ) ) {
234  $sink = new DumpOutput();
235  }
236  $sinks[] = $sink;
237 
238  if ( count( $sinks ) > 1 ) {
239  $this->sink = new DumpMultiWriter( $sinks );
240  } else {
241  $this->sink = $sink;
242  }
243  }
244 
245  function dump( $history, $text = WikiExporter::TEXT ) {
246  # Notice messages will foul up your XML output even if they're
247  # relatively harmless.
248  if ( ini_get( 'display_errors' ) ) {
249  ini_set( 'display_errors', 'stderr' );
250  }
251 
252  $this->initProgress( $history );
253 
254  $db = $this->backupDb();
255  $exporter = new WikiExporter( $db, $history, WikiExporter::STREAM, $text );
256  $exporter->dumpUploads = $this->dumpUploads;
257  $exporter->dumpUploadFileContents = $this->dumpUploadFileContents;
258 
259  $wrapper = new ExportProgressFilter( $this->sink, $this );
260  $exporter->setOutputSink( $wrapper );
261 
262  if ( !$this->skipHeader ) {
263  $exporter->openStream();
264  }
265  # Log item dumps: all or by range
266  if ( $history & WikiExporter::LOGS ) {
267  if ( $this->startId || $this->endId ) {
268  $exporter->logsByRange( $this->startId, $this->endId );
269  } else {
270  $exporter->allLogs();
271  }
272  } elseif ( is_null( $this->pages ) ) {
273  # Page dumps: all or by page ID range
274  if ( $this->startId || $this->endId ) {
275  $exporter->pagesByRange( $this->startId, $this->endId, $this->orderRevs );
276  } elseif ( $this->revStartId || $this->revEndId ) {
277  $exporter->revsByRange( $this->revStartId, $this->revEndId );
278  } else {
279  $exporter->allPages();
280  }
281  } else {
282  # Dump of specific pages
283  $exporter->pagesByName( $this->pages );
284  }
285 
286  if ( !$this->skipFooter ) {
287  $exporter->closeStream();
288  }
289 
290  $this->report( true );
291  }
292 
299  function initProgress( $history = WikiExporter::FULL ) {
300  $table = ( $history == WikiExporter::CURRENT ) ? 'page' : 'revision';
301  $field = ( $history == WikiExporter::CURRENT ) ? 'page_id' : 'rev_id';
302 
304  if ( $this->forcedDb === null ) {
305  $dbr = wfGetDB( DB_REPLICA );
306  }
307  $this->maxCount = $dbr->selectField( $table, "MAX($field)", '', __METHOD__ );
308  $this->startTime = microtime( true );
309  $this->lastTime = $this->startTime;
310  $this->ID = getmypid();
311  }
312 
319  function backupDb() {
320  if ( $this->forcedDb !== null ) {
321  return $this->forcedDb;
322  }
323 
324  $this->lb = wfGetLBFactory()->newMainLB();
325  $db = $this->lb->getConnection( DB_REPLICA, 'dump' );
326 
327  // Discourage the server from disconnecting us if it takes a long time
328  // to read out the big ol' batch query.
329  $db->setSessionOptions( [ 'connTimeout' => 3600 * 24 ] );
330 
331  return $db;
332  }
333 
341  function setDB( IDatabase $db = null ) {
342  parent::setDB( $db );
343  $this->forcedDb = $db;
344  }
345 
346  function __destruct() {
347  if ( isset( $this->lb ) ) {
348  $this->lb->closeAll();
349  }
350  }
351 
352  function backupServer() {
354 
355  return $this->server
356  ? $this->server
357  : $wgDBserver;
358  }
359 
360  function reportPage() {
361  $this->pageCount++;
362  }
363 
364  function revCount() {
365  $this->revCount++;
366  $this->report();
367  }
368 
369  function report( $final = false ) {
370  if ( $final xor ( $this->revCount % $this->reportingInterval == 0 ) ) {
371  $this->showReport();
372  }
373  }
374 
375  function showReport() {
376  if ( $this->reporting ) {
377  $now = wfTimestamp( TS_DB );
378  $nowts = microtime( true );
379  $deltaAll = $nowts - $this->startTime;
380  $deltaPart = $nowts - $this->lastTime;
381  $this->pageCountPart = $this->pageCount - $this->pageCountLast;
382  $this->revCountPart = $this->revCount - $this->revCountLast;
383 
384  if ( $deltaAll ) {
385  $portion = $this->revCount / $this->maxCount;
386  $eta = $this->startTime + $deltaAll / $portion;
387  $etats = wfTimestamp( TS_DB, intval( $eta ) );
388  $pageRate = $this->pageCount / $deltaAll;
389  $revRate = $this->revCount / $deltaAll;
390  } else {
391  $pageRate = '-';
392  $revRate = '-';
393  $etats = '-';
394  }
395  if ( $deltaPart ) {
396  $pageRatePart = $this->pageCountPart / $deltaPart;
397  $revRatePart = $this->revCountPart / $deltaPart;
398  } else {
399  $pageRatePart = '-';
400  $revRatePart = '-';
401  }
402  $this->progress( sprintf(
403  "%s: %s (ID %d) %d pages (%0.1f|%0.1f/sec all|curr), "
404  . "%d revs (%0.1f|%0.1f/sec all|curr), ETA %s [max %d]",
405  $now, wfWikiID(), $this->ID, $this->pageCount, $pageRate,
406  $pageRatePart, $this->revCount, $revRate, $revRatePart, $etats,
407  $this->maxCount
408  ) );
409  $this->lastTime = $nowts;
410  $this->revCountLast = $this->revCount;
411  }
412  }
413 
414  function progress( $string ) {
415  if ( $this->reporting ) {
416  fwrite( $this->stderr, $string . "\n" );
417  }
418  }
419 
420  function fatalError( $msg ) {
421  $this->error( "$msg\n", 1 );
422  }
423 }
424 
426  function __construct( &$sink, &$progress ) {
427  parent::__construct( $sink );
428  $this->progress = $progress;
429  }
430 
431  function writeClosePage( $string ) {
432  parent::writeClosePage( $string );
433  $this->progress->reportPage();
434  }
435 
436  function writeRevision( $rev, $string ) {
437  parent::writeRevision( $rev, $string );
438  $this->progress->revCount();
439  }
440 }
setDB(IDatabase $db=null)
Force the dump to use the provided database connection for database operations, wherever possible...
Definition: backup.inc:341
processOptions()
Processes arguments and sets $this->$sink accordingly.
Definition: backup.inc:152
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition: deferred.txt:11
wfGetDB($db, $groups=[], $wiki=false)
Get a Database object.
LoadBalancer $lb
Definition: backup.inc:70
array $orderedOptions
Used to read the options in the order they were passed.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: maintenance.txt:39
$wgDBserver
Database host name or IP address.
hasOption($name)
Checks to see if a particular param exists.
when a variable name is used in a it is silently declared as a new local masking the global
Definition: design.txt:93
DumpOutput $sink
FIXME will need to be made protected whenever legacy code is updated.
Definition: DumpFilter.php:37
report($final=false)
Definition: backup.inc:369
$dumpUploadFileContents
Definition: backup.inc:43
if($line===false) $args
Definition: cdb.php:64
wfTimestamp($outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
addOption($name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
fatalError($msg)
Definition: backup.inc:420
this hook is for auditing only RecentChangesLinked and Watchlist RecentChangesLinked and Watchlist e g Watchlist removed from all revisions and log entries to which it was applied This gives extensions a chance to take it off their books as the deletion has already been partly carried out by this point or something similar the user will be unable to create the tag set and then return false from the hook function Ensure you consume the ChangeTagAfterDelete hook to carry out custom deletion actions as context called by AbstractContent::getParserOutput May be used to override the normal model specific rendering of page content as context as context $options
Definition: hooks.txt:1046
__construct($args=null)
Definition: backup.inc:78
wfWikiID()
Get an ASCII string identifying this wiki This is used as a prefix in memcached keys.
presenting them properly to the user as errors is done by the caller return true use this to change the list i e etc $rev
Definition: hooks.txt:1721
backupServer()
Definition: backup.inc:352
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
getOption($name, $default=null)
Get an option, or return the default.
loadWithArgv($argv)
Load params and arguments from a given array of command-line arguments.
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
wfGetLBFactory()
Get the load balancer factory object.
The ContentHandler facility adds support for arbitrary content types on wiki pages
writeRevision($rev, $string)
Definition: backup.inc:436
registerFilter($name, $class)
Definition: backup.inc:126
error($err, $die=0)
Throw an error to the user.
initProgress($history=WikiExporter::FULL)
Initialise starting time and maximum revision count.
Definition: backup.inc:299
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such and we might be restricted by PHP settings such as safe mode or open_basedir We cannot assume that the software even has read access anywhere useful Many shared hosts run all users web applications under the same so they can t rely on Unix and must forbid reads to even standard directories like tmp lest users read each others files We cannot assume that the user has the ability to install or run any programs not written as web accessible PHP scripts Since anything that works on cheap shared hosting will work if you have shell or root access MediaWiki s design is based around catering to the lowest common denominator Although we support higher end setups as the way many things work by default is tailored toward shared hosting These defaults are unconventional from the point of view of and they certainly aren t ideal for someone who s installing MediaWiki as MediaWiki does not conform to normal Unix filesystem layout Hopefully we ll offer direct support for standard layouts in the but for now *any change to the location of files is unsupported *Moving things and leaving symlinks will *probably *not break but it is *strongly *advised not to try any more intrusive changes to get MediaWiki to conform more closely to your filesystem hierarchy Any such attempt will almost certainly result in unnecessary bugs The standard recommended location to install relative to the web is it should be possible to enable the appropriate rewrite rules by if you can reconfigure the web server
$reportingInterval
Definition: backup.inc:46
const DB_REPLICA
Definition: defines.php:22
__construct(&$sink, &$progress)
Definition: backup.inc:426
registerOutput($name, $class)
Definition: backup.inc:118
writeClosePage($string)
Definition: backup.inc:431
progress($string)
Definition: backup.inc:414
loadPlugin($class, $file)
Load a plugin and register it.
Definition: backup.inc:137
do that in ParserLimitReportFormat instead use this to modify the parameters of the image and a DIV can begin in one section and end in another Make sure your code can handle that case gracefully See the EditSectionClearerLink extension for an example zero but section is usually empty its values are the globals values before the output is cached one of or reset my talk my contributions etc etc otherwise the built in rate limiting checks are if enabled allows for interception of redirect as a string mapping parameter names to values & $type
Definition: hooks.txt:2491
const TS_DB
MySQL DATETIME (YYYY-MM-DD HH:MM:SS)
Definition: defines.php:16
Basic database interface for live and lazy-loaded relation database handles.
Definition: IDatabase.php:34
DatabaseBase null $forcedDb
The dependency-injected database to use.
Definition: backup.inc:67
dump($history, $text=WikiExporter::TEXT)
Definition: backup.inc:245
Allows to change the fields on the form that will be generated $name
Definition: hooks.txt:300