MediaWiki  master
generateSitemap.php
Go to the documentation of this file.
1 <?php
30 
31 require_once __DIR__ . '/Maintenance.php';
32 
39  const GS_MAIN = -2;
40  const GS_TALK = -1;
41 
49  public $url_limit;
50 
58  public $size_limit;
59 
65  public $fspath;
66 
73  public $urlpath;
74 
80  public $compress;
81 
88 
94  public $limit = [];
95 
101  public $priorities = [];
102 
108  public $namespaces = [];
109 
115  public $timestamp;
116 
122  public $dbr;
123 
129  public $findex;
130 
136  public $file;
137 
143  private $identifier;
144 
145  public function __construct() {
146  parent::__construct();
147  $this->addDescription( 'Creates a sitemap for the site' );
148  $this->addOption(
149  'fspath',
150  'The file system path to save to, e.g. /tmp/sitemap; defaults to current directory',
151  false,
152  true
153  );
154  $this->addOption(
155  'urlpath',
156  'The URL path corresponding to --fspath, prepended to filenames in the index; '
157  . 'defaults to an empty string',
158  false,
159  true
160  );
161  $this->addOption(
162  'compress',
163  'Compress the sitemap files, can take value yes|no, default yes',
164  false,
165  true
166  );
167  $this->addOption( 'skip-redirects', 'Do not include redirecting articles in the sitemap' );
168  $this->addOption(
169  'identifier',
170  'What site identifier to use for the wiki, defaults to $wgDBname',
171  false,
172  true
173  );
174  }
175 
179  public function execute() {
180  $this->setNamespacePriorities();
181  $this->url_limit = 50000;
182  $this->size_limit = ( 2 ** 20 ) * 10;
183 
184  # Create directory if needed
185  $fspath = $this->getOption( 'fspath', getcwd() );
186  if ( !wfMkdirParents( $fspath, null, __METHOD__ ) ) {
187  $this->fatalError( "Can not create directory $fspath." );
188  }
189 
190  $this->fspath = realpath( $fspath ) . DIRECTORY_SEPARATOR;
191  $this->urlpath = $this->getOption( 'urlpath', "" );
192  if ( $this->urlpath !== "" && substr( $this->urlpath, -1 ) !== '/' ) {
193  $this->urlpath .= '/';
194  }
195  $this->identifier = $this->getOption( 'identifier', wfWikiID() );
196  $this->compress = $this->getOption( 'compress', 'yes' ) !== 'no';
197  $this->skipRedirects = $this->hasOption( 'skip-redirects' );
198  $this->dbr = $this->getDB( DB_REPLICA );
199  $this->generateNamespaces();
200  $this->timestamp = wfTimestamp( TS_ISO_8601, wfTimestampNow() );
201  $this->findex = fopen( "{$this->fspath}sitemap-index-{$this->identifier}.xml", 'wb' );
202  $this->main();
203  }
204 
205  private function setNamespacePriorities() {
207 
208  // Custom main namespaces
209  $this->priorities[self::GS_MAIN] = '0.5';
210  // Custom talk namesspaces
211  $this->priorities[self::GS_TALK] = '0.1';
212  // MediaWiki standard namespaces
213  $this->priorities[NS_MAIN] = '1.0';
214  $this->priorities[NS_TALK] = '0.1';
215  $this->priorities[NS_USER] = '0.5';
216  $this->priorities[NS_USER_TALK] = '0.1';
217  $this->priorities[NS_PROJECT] = '0.5';
218  $this->priorities[NS_PROJECT_TALK] = '0.1';
219  $this->priorities[NS_FILE] = '0.5';
220  $this->priorities[NS_FILE_TALK] = '0.1';
221  $this->priorities[NS_MEDIAWIKI] = '0.0';
222  $this->priorities[NS_MEDIAWIKI_TALK] = '0.1';
223  $this->priorities[NS_TEMPLATE] = '0.0';
224  $this->priorities[NS_TEMPLATE_TALK] = '0.1';
225  $this->priorities[NS_HELP] = '0.5';
226  $this->priorities[NS_HELP_TALK] = '0.1';
227  $this->priorities[NS_CATEGORY] = '0.5';
228  $this->priorities[NS_CATEGORY_TALK] = '0.1';
229 
230  // Custom priorities
231  if ( $wgSitemapNamespacesPriorities !== false ) {
235  foreach ( $wgSitemapNamespacesPriorities as $namespace => $priority ) {
236  $float = floatval( $priority );
237  if ( $float > 1.0 ) {
238  $priority = '1.0';
239  } elseif ( $float < 0.0 ) {
240  $priority = '0.0';
241  }
242  $this->priorities[$namespace] = $priority;
243  }
244  }
245  }
246 
250  function generateNamespaces() {
251  // Only generate for specific namespaces if $wgSitemapNamespaces is an array.
252  global $wgSitemapNamespaces;
253  if ( is_array( $wgSitemapNamespaces ) ) {
255 
256  return;
257  }
258 
259  $res = $this->dbr->select( 'page',
260  [ 'page_namespace' ],
261  [],
262  __METHOD__,
263  [
264  'GROUP BY' => 'page_namespace',
265  'ORDER BY' => 'page_namespace',
266  ]
267  );
268 
269  foreach ( $res as $row ) {
270  $this->namespaces[] = $row->page_namespace;
271  }
272  }
273 
280  function priority( $namespace ) {
281  return $this->priorities[$namespace] ?? $this->guessPriority( $namespace );
282  }
283 
292  function guessPriority( $namespace ) {
293  return MediaWikiServices::getInstance()->getNamespaceInfo()->isSubject( $namespace )
294  ? $this->priorities[self::GS_MAIN]
295  : $this->priorities[self::GS_TALK];
296  }
297 
304  function getPageRes( $namespace ) {
305  return $this->dbr->select( 'page',
306  [
307  'page_namespace',
308  'page_title',
309  'page_touched',
310  'page_is_redirect'
311  ],
312  [ 'page_namespace' => $namespace ],
313  __METHOD__
314  );
315  }
316 
320  public function main() {
321  $contLang = MediaWikiServices::getInstance()->getContentLanguage();
322 
323  fwrite( $this->findex, $this->openIndex() );
324 
325  foreach ( $this->namespaces as $namespace ) {
326  $res = $this->getPageRes( $namespace );
327  $this->file = false;
328  $this->generateLimit( $namespace );
329  $length = $this->limit[0];
330  $i = $smcount = 0;
331 
332  $fns = $contLang->getFormattedNsText( $namespace );
333  $this->output( "$namespace ($fns)\n" );
334  $skippedRedirects = 0; // Number of redirects skipped for that namespace
335  foreach ( $res as $row ) {
336  if ( $this->skipRedirects && $row->page_is_redirect ) {
337  $skippedRedirects++;
338  continue;
339  }
340 
341  if ( $i++ === 0
342  || $i === $this->url_limit + 1
343  || $length + $this->limit[1] + $this->limit[2] > $this->size_limit
344  ) {
345  if ( $this->file !== false ) {
346  $this->write( $this->file, $this->closeFile() );
347  $this->close( $this->file );
348  }
349  $filename = $this->sitemapFilename( $namespace, $smcount++ );
350  $this->file = $this->open( $this->fspath . $filename, 'wb' );
351  $this->write( $this->file, $this->openFile() );
352  fwrite( $this->findex, $this->indexEntry( $filename ) );
353  $this->output( "\t$this->fspath$filename\n" );
354  $length = $this->limit[0];
355  $i = 1;
356  }
357  $title = Title::makeTitle( $row->page_namespace, $row->page_title );
358  $date = wfTimestamp( TS_ISO_8601, $row->page_touched );
359  $entry = $this->fileEntry( $title->getCanonicalURL(), $date, $this->priority( $namespace ) );
360  $length += strlen( $entry );
361  $this->write( $this->file, $entry );
362  // generate pages for language variants
363  if ( $contLang->hasVariants() ) {
364  $variants = $contLang->getVariants();
365  foreach ( $variants as $vCode ) {
366  if ( $vCode == $contLang->getCode() ) {
367  continue; // we don't want default variant
368  }
369  $entry = $this->fileEntry(
370  $title->getCanonicalURL( '', $vCode ),
371  $date,
372  $this->priority( $namespace )
373  );
374  $length += strlen( $entry );
375  $this->write( $this->file, $entry );
376  }
377  }
378  }
379 
380  if ( $this->skipRedirects && $skippedRedirects > 0 ) {
381  $this->output( " skipped $skippedRedirects redirect(s)\n" );
382  }
383 
384  if ( $this->file ) {
385  $this->write( $this->file, $this->closeFile() );
386  $this->close( $this->file );
387  }
388  }
389  fwrite( $this->findex, $this->closeIndex() );
390  fclose( $this->findex );
391  }
392 
400  function open( $file, $flags ) {
401  $resource = $this->compress ? gzopen( $file, $flags ) : fopen( $file, $flags );
402  if ( $resource === false ) {
403  throw new MWException( __METHOD__
404  . " error opening file $file with flags $flags. Check permissions?" );
405  }
406 
407  return $resource;
408  }
409 
416  function write( &$handle, $str ) {
417  if ( $handle === true || $handle === false ) {
418  throw new MWException( __METHOD__ . " was passed a boolean as a file handle.\n" );
419  }
420  if ( $this->compress ) {
421  gzwrite( $handle, $str );
422  } else {
423  fwrite( $handle, $str );
424  }
425  }
426 
432  function close( &$handle ) {
433  if ( $this->compress ) {
434  gzclose( $handle );
435  } else {
436  fclose( $handle );
437  }
438  }
439 
447  function sitemapFilename( $namespace, $count ) {
448  $ext = $this->compress ? '.gz' : '';
449 
450  return "sitemap-{$this->identifier}-NS_$namespace-$count.xml$ext";
451  }
452 
458  function xmlHead() {
459  return '<?xml version="1.0" encoding="UTF-8"?>' . "\n";
460  }
461 
467  function xmlSchema() {
468  return 'http://www.sitemaps.org/schemas/sitemap/0.9';
469  }
470 
476  function openIndex() {
477  return $this->xmlHead() . '<sitemapindex xmlns="' . $this->xmlSchema() . '">' . "\n";
478  }
479 
486  function indexEntry( $filename ) {
487  return "\t<sitemap>\n" .
488  "\t\t<loc>" . wfGetServerUrl( PROTO_CANONICAL ) .
489  ( substr( $this->urlpath, 0, 1 ) === "/" ? "" : "/" ) .
490  "{$this->urlpath}$filename</loc>\n" .
491  "\t\t<lastmod>{$this->timestamp}</lastmod>\n" .
492  "\t</sitemap>\n";
493  }
494 
500  function closeIndex() {
501  return "</sitemapindex>\n";
502  }
503 
509  function openFile() {
510  return $this->xmlHead() . '<urlset xmlns="' . $this->xmlSchema() . '">' . "\n";
511  }
512 
521  function fileEntry( $url, $date, $priority ) {
522  return "\t<url>\n" .
523  // T36666: $url may contain bad characters such as ampersands.
524  "\t\t<loc>" . htmlspecialchars( $url ) . "</loc>\n" .
525  "\t\t<lastmod>$date</lastmod>\n" .
526  "\t\t<priority>$priority</priority>\n" .
527  "\t</url>\n";
528  }
529 
535  function closeFile() {
536  return "</urlset>\n";
537  }
538 
544  function generateLimit( $namespace ) {
545  // T19961: make a title with the longest possible URL in this namespace
546  $title = Title::makeTitle( $namespace, str_repeat( "\u{28B81}", 63 ) . "\u{5583}" );
547 
548  $this->limit = [
549  strlen( $this->openFile() ),
550  strlen( $this->fileEntry(
551  $title->getCanonicalURL(),
552  wfTimestamp( TS_ISO_8601, wfTimestamp() ),
553  $this->priority( $namespace )
554  ) ),
555  strlen( $this->closeFile() )
556  ];
557  }
558 }
559 
561 require_once RUN_MAINTENANCE_IF_MAIN;
resource $findex
A resource pointing to the sitemap index file.
Maintenance script that generates a sitemap for the site.
const NS_MAIN
Definition: Defines.php:64
to move a page</td >< td > &*You are moving the page across namespaces
open( $file, $flags)
gzopen() / fopen() wrapper
$wgSitemapNamespacesPriorities
Custom namespace priorities for sitemaps.
array $namespaces
A one-dimensional array of namespaces in the wiki.
array $limit
The number of entries to save in each sitemap file.
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
getOption( $name, $default=null)
Get an option, or return the default.
write(&$handle, $str)
gzwrite() / fwrite() wrapper
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: maintenance.txt:39
generateNamespaces()
Generate a one-dimensional array of existing namespaces.
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency MediaWikiServices
Definition: injection.txt:23
string $identifier
Identifier to use in filenames, default $wgDBname.
array $priorities
Key => value entries of namespaces and their priorities.
xmlHead()
Return the XML required to open an XML file.
hasOption( $name)
Checks to see if a particular option exists.
require_once RUN_MAINTENANCE_IF_MAIN
Definition: maintenance.txt:50
wfGetServerUrl( $proto)
Get the wiki&#39;s "server", i.e.
const NS_TEMPLATE
Definition: Defines.php:74
bool $compress
Whether or not to use compression.
closeFile()
Return the XML required to close sitemap file.
getPageRes( $namespace)
Return a database resolution of all the pages in a given namespace.
bool $skipRedirects
Whether or not to include redirection pages.
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
guessPriority( $namespace)
If the namespace isn&#39;t listed on the priority list return the default priority for the namespace...
const NS_PROJECT
Definition: Defines.php:68
Using a hook running we can avoid having all this option specific stuff in our mainline code Using the function We ve cleaned up the code here by removing clumps of infrequently used code and moving them off somewhere else It s much easier for someone working with this code to see what s _really_ going and make changes or fix bugs In we can take all the code that deals with the little used title reversing we can concentrate it all in an extension file
Definition: hooks.txt:91
int $url_limit
The maximum amount of urls in a sitemap file.
generateLimit( $namespace)
Populate $this->limit.
const NS_MEDIAWIKI_TALK
Definition: Defines.php:73
close(&$handle)
gzclose() / fclose() wrapper
closeIndex()
Return the XML required to close a sitemap index file.
const NS_PROJECT_TALK
Definition: Defines.php:69
$res
Definition: database.txt:21
addDescription( $text)
Set the description text.
openIndex()
Return the XML required to open a sitemap index file.
wfTimestampNow()
Convenience function; returns MediaWiki timestamp for the present time.
const NS_CATEGORY
Definition: Defines.php:78
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that When $user is not null
Definition: hooks.txt:780
namespace and then decline to actually register it file or subcat img or subcat $title
Definition: hooks.txt:925
getDB( $db, $groups=[], $wiki=false)
Returns a database to be used by current maintenance script.
wfWikiID()
Get an ASCII string identifying this wiki This is used as a prefix in memcached keys.
const NS_FILE
Definition: Defines.php:70
const NS_FILE_TALK
Definition: Defines.php:71
output( $out, $channel=null)
Throw some output to the user.
fileEntry( $url, $date, $priority)
Return the XML for a single sitemap entry.
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
const NS_MEDIAWIKI
Definition: Defines.php:72
sitemapFilename( $namespace, $count)
Get a sitemap filename.
static makeTitle( $ns, $title, $fragment='', $interwiki='')
Create a new Title from a namespace index and a DB key.
Definition: Title.php:589
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
const PROTO_CANONICAL
Definition: Defines.php:223
const NS_CATEGORY_TALK
Definition: Defines.php:79
const NS_HELP
Definition: Defines.php:76
int $size_limit
The maximum size of a sitemap file.
const NS_HELP_TALK
Definition: Defines.php:77
you have access to all of the normal MediaWiki so you can get a DB use the etc For full docs on the Maintenance class
Definition: maintenance.txt:52
wfMkdirParents( $dir, $mode=null, $caller=null)
Make directory, and make all parent directories if they don&#39;t exist.
object $dbr
A database replica DB object.
if(!is_readable( $file)) $ext
Definition: router.php:48
priority( $namespace)
Get the priority of a given namespace.
string $fspath
The path to prepend to the filename.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
const NS_TALK
Definition: Defines.php:65
const DB_REPLICA
Definition: defines.php:25
indexEntry( $filename)
Return the XML for a single sitemap indexfile entry.
$wgSitemapNamespaces
Array of namespaces to generate a Google sitemap for when the maintenance/generateSitemap.php script is run, or false if one is to be generated for all namespaces.
$maintClass
xmlSchema()
Return the XML schema being used.
string $urlpath
The URL path to prepend to filenames in the index; should resolve to the same directory as $fspath...
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
resource $file
A resource pointing to a sitemap file.
main()
Main loop.
const NS_USER_TALK
Definition: Defines.php:67
openFile()
Return the XML required to open a sitemap file.
const NS_TEMPLATE_TALK
Definition: Defines.php:75
string $timestamp
When this sitemap batch was generated.