MediaWiki  master
generateSitemap.php
Go to the documentation of this file.
1 <?php
31 
32 require_once __DIR__ . '/Maintenance.php';
33 
40  const GS_MAIN = -2;
41  const GS_TALK = -1;
42 
50  public $url_limit;
51 
59  public $size_limit;
60 
66  public $fspath;
67 
74  public $urlpath;
75 
81  public $compress;
82 
89 
95  public $limit = [];
96 
102  public $priorities = [];
103 
109  public $namespaces = [];
110 
116  public $timestamp;
117 
123  public $dbr;
124 
130  public $findex;
131 
137  public $file;
138 
144  private $identifier;
145 
146  public function __construct() {
147  parent::__construct();
148  $this->addDescription( 'Creates a sitemap for the site' );
149  $this->addOption(
150  'fspath',
151  'The file system path to save to, e.g. /tmp/sitemap; defaults to current directory',
152  false,
153  true
154  );
155  $this->addOption(
156  'urlpath',
157  'The URL path corresponding to --fspath, prepended to filenames in the index; '
158  . 'defaults to an empty string',
159  false,
160  true
161  );
162  $this->addOption(
163  'compress',
164  'Compress the sitemap files, can take value yes|no, default yes',
165  false,
166  true
167  );
168  $this->addOption( 'skip-redirects', 'Do not include redirecting articles in the sitemap' );
169  $this->addOption(
170  'identifier',
171  'What site identifier to use for the wiki, defaults to $wgDBname',
172  false,
173  true
174  );
175  }
176 
180  public function execute() {
181  $this->setNamespacePriorities();
182  $this->url_limit = 50000;
183  $this->size_limit = ( 2 ** 20 ) * 10;
184 
185  # Create directory if needed
186  $fspath = $this->getOption( 'fspath', getcwd() );
187  if ( !wfMkdirParents( $fspath, null, __METHOD__ ) ) {
188  $this->fatalError( "Can not create directory $fspath." );
189  }
190 
191  $dbDomain = WikiMap::getCurrentWikiDbDomain()->getId();
192  $this->fspath = realpath( $fspath ) . DIRECTORY_SEPARATOR;
193  $this->urlpath = $this->getOption( 'urlpath', "" );
194  if ( $this->urlpath !== "" && substr( $this->urlpath, -1 ) !== '/' ) {
195  $this->urlpath .= '/';
196  }
197  $this->identifier = $this->getOption( 'identifier', $dbDomain );
198  $this->compress = $this->getOption( 'compress', 'yes' ) !== 'no';
199  $this->skipRedirects = $this->hasOption( 'skip-redirects' );
200  $this->dbr = $this->getDB( DB_REPLICA );
201  $this->generateNamespaces();
202  $this->timestamp = wfTimestamp( TS_ISO_8601, wfTimestampNow() );
203  $encIdentifier = rawurlencode( $this->identifier );
204  $this->findex = fopen( "{$this->fspath}sitemap-index-{$encIdentifier}.xml", 'wb' );
205  $this->main();
206  }
207 
208  private function setNamespacePriorities() {
210 
211  // Custom main namespaces
212  $this->priorities[self::GS_MAIN] = '0.5';
213  // Custom talk namesspaces
214  $this->priorities[self::GS_TALK] = '0.1';
215  // MediaWiki standard namespaces
216  $this->priorities[NS_MAIN] = '1.0';
217  $this->priorities[NS_TALK] = '0.1';
218  $this->priorities[NS_USER] = '0.5';
219  $this->priorities[NS_USER_TALK] = '0.1';
220  $this->priorities[NS_PROJECT] = '0.5';
221  $this->priorities[NS_PROJECT_TALK] = '0.1';
222  $this->priorities[NS_FILE] = '0.5';
223  $this->priorities[NS_FILE_TALK] = '0.1';
224  $this->priorities[NS_MEDIAWIKI] = '0.0';
225  $this->priorities[NS_MEDIAWIKI_TALK] = '0.1';
226  $this->priorities[NS_TEMPLATE] = '0.0';
227  $this->priorities[NS_TEMPLATE_TALK] = '0.1';
228  $this->priorities[NS_HELP] = '0.5';
229  $this->priorities[NS_HELP_TALK] = '0.1';
230  $this->priorities[NS_CATEGORY] = '0.5';
231  $this->priorities[NS_CATEGORY_TALK] = '0.1';
232 
233  // Custom priorities
234  if ( $wgSitemapNamespacesPriorities !== false ) {
238  foreach ( $wgSitemapNamespacesPriorities as $namespace => $priority ) {
239  $float = floatval( $priority );
240  if ( $float > 1.0 ) {
241  $priority = '1.0';
242  } elseif ( $float < 0.0 ) {
243  $priority = '0.0';
244  }
245  $this->priorities[$namespace] = $priority;
246  }
247  }
248  }
249 
253  function generateNamespaces() {
254  // Only generate for specific namespaces if $wgSitemapNamespaces is an array.
255  global $wgSitemapNamespaces;
256  if ( is_array( $wgSitemapNamespaces ) ) {
258 
259  return;
260  }
261 
262  $res = $this->dbr->select( 'page',
263  [ 'page_namespace' ],
264  [],
265  __METHOD__,
266  [
267  'GROUP BY' => 'page_namespace',
268  'ORDER BY' => 'page_namespace',
269  ]
270  );
271 
272  foreach ( $res as $row ) {
273  $this->namespaces[] = $row->page_namespace;
274  }
275  }
276 
283  function priority( $namespace ) {
284  return $this->priorities[$namespace] ?? $this->guessPriority( $namespace );
285  }
286 
295  function guessPriority( $namespace ) {
296  return MediaWikiServices::getInstance()->getNamespaceInfo()->isSubject( $namespace )
297  ? $this->priorities[self::GS_MAIN]
298  : $this->priorities[self::GS_TALK];
299  }
300 
307  function getPageRes( $namespace ) {
308  return $this->dbr->select( 'page',
309  [
310  'page_namespace',
311  'page_title',
312  'page_touched',
313  'page_is_redirect'
314  ],
315  [ 'page_namespace' => $namespace ],
316  __METHOD__
317  );
318  }
319 
323  public function main() {
324  $contLang = MediaWikiServices::getInstance()->getContentLanguage();
325 
326  fwrite( $this->findex, $this->openIndex() );
327 
328  foreach ( $this->namespaces as $namespace ) {
329  $res = $this->getPageRes( $namespace );
330  $this->file = false;
331  $this->generateLimit( $namespace );
332  $length = $this->limit[0];
333  $i = $smcount = 0;
334 
335  $fns = $contLang->getFormattedNsText( $namespace );
336  $this->output( "$namespace ($fns)\n" );
337  $skippedRedirects = 0; // Number of redirects skipped for that namespace
338  foreach ( $res as $row ) {
339  if ( $this->skipRedirects && $row->page_is_redirect ) {
340  $skippedRedirects++;
341  continue;
342  }
343 
344  if ( $i++ === 0
345  || $i === $this->url_limit + 1
346  || $length + $this->limit[1] + $this->limit[2] > $this->size_limit
347  ) {
348  if ( $this->file !== false ) {
349  $this->write( $this->file, $this->closeFile() );
350  $this->close( $this->file );
351  }
352  $filename = $this->sitemapFilename( $namespace, $smcount++ );
353  $this->file = $this->open( $this->fspath . $filename, 'wb' );
354  $this->write( $this->file, $this->openFile() );
355  fwrite( $this->findex, $this->indexEntry( $filename ) );
356  $this->output( "\t$this->fspath$filename\n" );
357  $length = $this->limit[0];
358  $i = 1;
359  }
360  $title = Title::makeTitle( $row->page_namespace, $row->page_title );
361  $date = wfTimestamp( TS_ISO_8601, $row->page_touched );
362  $entry = $this->fileEntry( $title->getCanonicalURL(), $date, $this->priority( $namespace ) );
363  $length += strlen( $entry );
364  $this->write( $this->file, $entry );
365  // generate pages for language variants
366  if ( $contLang->hasVariants() ) {
367  $variants = $contLang->getVariants();
368  foreach ( $variants as $vCode ) {
369  if ( $vCode == $contLang->getCode() ) {
370  continue; // we don't want default variant
371  }
372  $entry = $this->fileEntry(
373  $title->getCanonicalURL( '', $vCode ),
374  $date,
375  $this->priority( $namespace )
376  );
377  $length += strlen( $entry );
378  $this->write( $this->file, $entry );
379  }
380  }
381  }
382 
383  if ( $this->skipRedirects && $skippedRedirects > 0 ) {
384  $this->output( " skipped $skippedRedirects redirect(s)\n" );
385  }
386 
387  if ( $this->file ) {
388  $this->write( $this->file, $this->closeFile() );
389  $this->close( $this->file );
390  }
391  }
392  fwrite( $this->findex, $this->closeIndex() );
393  fclose( $this->findex );
394  }
395 
403  function open( $file, $flags ) {
404  $resource = $this->compress ? gzopen( $file, $flags ) : fopen( $file, $flags );
405  if ( $resource === false ) {
406  throw new MWException( __METHOD__
407  . " error opening file $file with flags $flags. Check permissions?" );
408  }
409 
410  return $resource;
411  }
412 
419  function write( &$handle, $str ) {
420  if ( $handle === true || $handle === false ) {
421  throw new MWException( __METHOD__ . " was passed a boolean as a file handle.\n" );
422  }
423  if ( $this->compress ) {
424  gzwrite( $handle, $str );
425  } else {
426  fwrite( $handle, $str );
427  }
428  }
429 
435  function close( &$handle ) {
436  if ( $this->compress ) {
437  gzclose( $handle );
438  } else {
439  fclose( $handle );
440  }
441  }
442 
450  function sitemapFilename( $namespace, $count ) {
451  $ext = $this->compress ? '.gz' : '';
452 
453  return "sitemap-{$this->identifier}-NS_$namespace-$count.xml$ext";
454  }
455 
461  function xmlHead() {
462  return '<?xml version="1.0" encoding="UTF-8"?>' . "\n";
463  }
464 
470  function xmlSchema() {
471  return 'http://www.sitemaps.org/schemas/sitemap/0.9';
472  }
473 
479  function openIndex() {
480  return $this->xmlHead() . '<sitemapindex xmlns="' . $this->xmlSchema() . '">' . "\n";
481  }
482 
489  function indexEntry( $filename ) {
490  return "\t<sitemap>\n" .
491  "\t\t<loc>" . wfGetServerUrl( PROTO_CANONICAL ) .
492  ( substr( $this->urlpath, 0, 1 ) === "/" ? "" : "/" ) .
493  "{$this->urlpath}$filename</loc>\n" .
494  "\t\t<lastmod>{$this->timestamp}</lastmod>\n" .
495  "\t</sitemap>\n";
496  }
497 
503  function closeIndex() {
504  return "</sitemapindex>\n";
505  }
506 
512  function openFile() {
513  return $this->xmlHead() . '<urlset xmlns="' . $this->xmlSchema() . '">' . "\n";
514  }
515 
524  function fileEntry( $url, $date, $priority ) {
525  return "\t<url>\n" .
526  // T36666: $url may contain bad characters such as ampersands.
527  "\t\t<loc>" . htmlspecialchars( $url ) . "</loc>\n" .
528  "\t\t<lastmod>$date</lastmod>\n" .
529  "\t\t<priority>$priority</priority>\n" .
530  "\t</url>\n";
531  }
532 
538  function closeFile() {
539  return "</urlset>\n";
540  }
541 
547  function generateLimit( $namespace ) {
548  // T19961: make a title with the longest possible URL in this namespace
549  $title = Title::makeTitle( $namespace, str_repeat( "\u{28B81}", 63 ) . "\u{5583}" );
550 
551  $this->limit = [
552  strlen( $this->openFile() ),
553  strlen( $this->fileEntry(
554  $title->getCanonicalURL(),
555  wfTimestamp( TS_ISO_8601, wfTimestamp() ),
556  $this->priority( $namespace )
557  ) ),
558  strlen( $this->closeFile() )
559  ];
560  }
561 }
562 
564 require_once RUN_MAINTENANCE_IF_MAIN;
resource $findex
A resource pointing to the sitemap index file.
Maintenance script that generates a sitemap for the site.
const NS_MAIN
Definition: Defines.php:60
to move a page</td >< td > &*You are moving the page across namespaces
open( $file, $flags)
gzopen() / fopen() wrapper
$wgSitemapNamespacesPriorities
Custom namespace priorities for sitemaps.
array $namespaces
A one-dimensional array of namespaces in the wiki.
array $limit
The number of entries to save in each sitemap file.
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for use
getOption( $name, $default=null)
Get an option, or return the default.
write(&$handle, $str)
gzwrite() / fwrite() wrapper
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: maintenance.txt:39
generateNamespaces()
Generate a one-dimensional array of existing namespaces.
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency MediaWikiServices
Definition: injection.txt:23
string $identifier
Identifier to use in filenames, default $wgDBname.
array $priorities
Key => value entries of namespaces and their priorities.
xmlHead()
Return the XML required to open an XML file.
hasOption( $name)
Checks to see if a particular option exists.
require_once RUN_MAINTENANCE_IF_MAIN
Definition: maintenance.txt:50
wfGetServerUrl( $proto)
Get the wiki&#39;s "server", i.e.
const NS_TEMPLATE
Definition: Defines.php:70
bool $compress
Whether or not to use compression.
closeFile()
Return the XML required to close sitemap file.
getPageRes( $namespace)
Return a database resolution of all the pages in a given namespace.
bool $skipRedirects
Whether or not to include redirection pages.
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
guessPriority( $namespace)
If the namespace isn&#39;t listed on the priority list return the default priority for the namespace...
const NS_PROJECT
Definition: Defines.php:64
Using a hook running we can avoid having all this option specific stuff in our mainline code Using the function We ve cleaned up the code here by removing clumps of infrequently used code and moving them off somewhere else It s much easier for someone working with this code to see what s _really_ going and make changes or fix bugs In we can take all the code that deals with the little used title reversing we can concentrate it all in an extension file
Definition: hooks.txt:91
int $url_limit
The maximum amount of urls in a sitemap file.
generateLimit( $namespace)
Populate $this->limit.
const NS_MEDIAWIKI_TALK
Definition: Defines.php:69
close(&$handle)
gzclose() / fclose() wrapper
closeIndex()
Return the XML required to close a sitemap index file.
const NS_PROJECT_TALK
Definition: Defines.php:65
$res
Definition: database.txt:21
addDescription( $text)
Set the description text.
openIndex()
Return the XML required to open a sitemap index file.
wfTimestampNow()
Convenience function; returns MediaWiki timestamp for the present time.
const NS_CATEGORY
Definition: Defines.php:74
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that When $user is not null
Definition: hooks.txt:767
namespace and then decline to actually register it file or subcat img or subcat $title
Definition: hooks.txt:912
const NS_FILE
Definition: Defines.php:66
const NS_FILE_TALK
Definition: Defines.php:67
output( $out, $channel=null)
Throw some output to the user.
fileEntry( $url, $date, $priority)
Return the XML for a single sitemap entry.
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
const NS_MEDIAWIKI
Definition: Defines.php:68
sitemapFilename( $namespace, $count)
Get a sitemap filename.
static makeTitle( $ns, $title, $fragment='', $interwiki='')
Create a new Title from a namespace index and a DB key.
Definition: Title.php:592
static getCurrentWikiDbDomain()
Definition: WikiMap.php:293
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
const PROTO_CANONICAL
Definition: Defines.php:203
const NS_CATEGORY_TALK
Definition: Defines.php:75
const NS_HELP
Definition: Defines.php:72
int $size_limit
The maximum size of a sitemap file.
const NS_HELP_TALK
Definition: Defines.php:73
you have access to all of the normal MediaWiki so you can get a DB use the etc For full docs on the Maintenance class
Definition: maintenance.txt:52
wfMkdirParents( $dir, $mode=null, $caller=null)
Make directory, and make all parent directories if they don&#39;t exist.
object $dbr
A database replica DB object.
if(!is_readable( $file)) $ext
Definition: router.php:48
priority( $namespace)
Get the priority of a given namespace.
string $fspath
The path to prepend to the filename.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
const NS_TALK
Definition: Defines.php:61
const DB_REPLICA
Definition: defines.php:25
indexEntry( $filename)
Return the XML for a single sitemap indexfile entry.
$wgSitemapNamespaces
Array of namespaces to generate a Google sitemap for when the maintenance/generateSitemap.php script is run, or false if one is to be generated for all namespaces.
$maintClass
xmlSchema()
Return the XML schema being used.
string $urlpath
The URL path to prepend to filenames in the index; should resolve to the same directory as $fspath...
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
resource $file
A resource pointing to a sitemap file.
main()
Main loop.
const NS_USER_TALK
Definition: Defines.php:63
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
openFile()
Return the XML required to open a sitemap file.
const NS_TEMPLATE_TALK
Definition: Defines.php:71
string $timestamp
When this sitemap batch was generated.