MediaWiki  master
generateSitemap.php
Go to the documentation of this file.
1 <?php
31 
32 require_once __DIR__ . '/Maintenance.php';
33 
40  const GS_MAIN = -2;
41  const GS_TALK = -1;
42 
50  public $url_limit;
51 
59  public $size_limit;
60 
66  public $fspath;
67 
74  public $urlpath;
75 
81  public $compress;
82 
89 
95  public $limit = [];
96 
102  public $priorities = [];
103 
109  public $namespaces = [];
110 
116  public $timestamp;
117 
123  public $dbr;
124 
130  public $findex;
131 
137  public $file;
138 
144  private $identifier;
145 
146  public function __construct() {
147  parent::__construct();
148  $this->addDescription( 'Creates a sitemap for the site' );
149  $this->addOption(
150  'fspath',
151  'The file system path to save to, e.g. /tmp/sitemap; defaults to current directory',
152  false,
153  true
154  );
155  $this->addOption(
156  'urlpath',
157  'The URL path corresponding to --fspath, prepended to filenames in the index; '
158  . 'defaults to an empty string',
159  false,
160  true
161  );
162  $this->addOption(
163  'compress',
164  'Compress the sitemap files, can take value yes|no, default yes',
165  false,
166  true
167  );
168  $this->addOption( 'skip-redirects', 'Do not include redirecting articles in the sitemap' );
169  $this->addOption(
170  'identifier',
171  'What site identifier to use for the wiki, defaults to $wgDBname',
172  false,
173  true
174  );
175  }
176 
180  public function execute() {
181  $this->setNamespacePriorities();
182  $this->url_limit = 50000;
183  $this->size_limit = ( 2 ** 20 ) * 10;
184 
185  # Create directory if needed
186  $fspath = $this->getOption( 'fspath', getcwd() );
187  if ( !wfMkdirParents( $fspath, null, __METHOD__ ) ) {
188  $this->fatalError( "Can not create directory $fspath." );
189  }
190 
191  $dbDomain = WikiMap::getCurrentWikiDbDomain()->getId();
192  $this->fspath = realpath( $fspath ) . DIRECTORY_SEPARATOR;
193  $this->urlpath = $this->getOption( 'urlpath', "" );
194  if ( $this->urlpath !== "" && substr( $this->urlpath, -1 ) !== '/' ) {
195  $this->urlpath .= '/';
196  }
197  $this->identifier = $this->getOption( 'identifier', $dbDomain );
198  $this->compress = $this->getOption( 'compress', 'yes' ) !== 'no';
199  $this->skipRedirects = $this->hasOption( 'skip-redirects' );
200  $this->dbr = $this->getDB( DB_REPLICA );
201  $this->generateNamespaces();
202  $this->timestamp = wfTimestamp( TS_ISO_8601, wfTimestampNow() );
203  $encIdentifier = rawurlencode( $this->identifier );
204  $this->findex = fopen( "{$this->fspath}sitemap-index-{$encIdentifier}.xml", 'wb' );
205  $this->main();
206  }
207 
208  private function setNamespacePriorities() {
210 
211  // Custom main namespaces
212  $this->priorities[self::GS_MAIN] = '0.5';
213  // Custom talk namesspaces
214  $this->priorities[self::GS_TALK] = '0.1';
215  // MediaWiki standard namespaces
216  $this->priorities[NS_MAIN] = '1.0';
217  $this->priorities[NS_TALK] = '0.1';
218  $this->priorities[NS_USER] = '0.5';
219  $this->priorities[NS_USER_TALK] = '0.1';
220  $this->priorities[NS_PROJECT] = '0.5';
221  $this->priorities[NS_PROJECT_TALK] = '0.1';
222  $this->priorities[NS_FILE] = '0.5';
223  $this->priorities[NS_FILE_TALK] = '0.1';
224  $this->priorities[NS_MEDIAWIKI] = '0.0';
225  $this->priorities[NS_MEDIAWIKI_TALK] = '0.1';
226  $this->priorities[NS_TEMPLATE] = '0.0';
227  $this->priorities[NS_TEMPLATE_TALK] = '0.1';
228  $this->priorities[NS_HELP] = '0.5';
229  $this->priorities[NS_HELP_TALK] = '0.1';
230  $this->priorities[NS_CATEGORY] = '0.5';
231  $this->priorities[NS_CATEGORY_TALK] = '0.1';
232 
233  // Custom priorities
234  if ( $wgSitemapNamespacesPriorities !== false ) {
238  foreach ( $wgSitemapNamespacesPriorities as $namespace => $priority ) {
239  $float = floatval( $priority );
240  if ( $float > 1.0 ) {
241  $priority = '1.0';
242  } elseif ( $float < 0.0 ) {
243  $priority = '0.0';
244  }
245  $this->priorities[$namespace] = $priority;
246  }
247  }
248  }
249 
253  function generateNamespaces() {
254  // Only generate for specific namespaces if $wgSitemapNamespaces is an array.
255  global $wgSitemapNamespaces;
256  if ( is_array( $wgSitemapNamespaces ) ) {
257  $this->namespaces = $wgSitemapNamespaces;
258 
259  return;
260  }
261 
262  $res = $this->dbr->select( 'page',
263  [ 'page_namespace' ],
264  [],
265  __METHOD__,
266  [
267  'GROUP BY' => 'page_namespace',
268  'ORDER BY' => 'page_namespace',
269  ]
270  );
271 
272  foreach ( $res as $row ) {
273  $this->namespaces[] = $row->page_namespace;
274  }
275  }
276 
283  function priority( $namespace ) {
284  return $this->priorities[$namespace] ?? $this->guessPriority( $namespace );
285  }
286 
295  function guessPriority( $namespace ) {
296  return MediaWikiServices::getInstance()->getNamespaceInfo()->isSubject( $namespace )
297  ? $this->priorities[self::GS_MAIN]
298  : $this->priorities[self::GS_TALK];
299  }
300 
307  function getPageRes( $namespace ) {
308  return $this->dbr->select( 'page',
309  [
310  'page_namespace',
311  'page_title',
312  'page_touched',
313  'page_is_redirect'
314  ],
315  [ 'page_namespace' => $namespace ],
316  __METHOD__
317  );
318  }
319 
323  public function main() {
324  $contLang = MediaWikiServices::getInstance()->getContentLanguage();
325 
326  fwrite( $this->findex, $this->openIndex() );
327 
328  foreach ( $this->namespaces as $namespace ) {
329  $res = $this->getPageRes( $namespace );
330  $this->file = false;
331  $this->generateLimit( $namespace );
332  $length = $this->limit[0];
333  $i = $smcount = 0;
334 
335  $fns = $contLang->getFormattedNsText( $namespace );
336  $this->output( "$namespace ($fns)\n" );
337  $skippedRedirects = 0; // Number of redirects skipped for that namespace
338  foreach ( $res as $row ) {
339  if ( $this->skipRedirects && $row->page_is_redirect ) {
340  $skippedRedirects++;
341  continue;
342  }
343 
344  if ( $i++ === 0
345  || $i === $this->url_limit + 1
346  || $length + $this->limit[1] + $this->limit[2] > $this->size_limit
347  ) {
348  if ( $this->file !== false ) {
349  $this->write( $this->file, $this->closeFile() );
350  $this->close( $this->file );
351  }
352  $filename = $this->sitemapFilename( $namespace, $smcount++ );
353  $this->file = $this->open( $this->fspath . $filename, 'wb' );
354  $this->write( $this->file, $this->openFile() );
355  fwrite( $this->findex, $this->indexEntry( $filename ) );
356  $this->output( "\t$this->fspath$filename\n" );
357  $length = $this->limit[0];
358  $i = 1;
359  }
360  $title = Title::makeTitle( $row->page_namespace, $row->page_title );
361  $date = wfTimestamp( TS_ISO_8601, $row->page_touched );
362  $entry = $this->fileEntry( $title->getCanonicalURL(), $date, $this->priority( $namespace ) );
363  $length += strlen( $entry );
364  $this->write( $this->file, $entry );
365  // generate pages for language variants
366  if ( $contLang->hasVariants() ) {
367  $variants = $contLang->getVariants();
368  foreach ( $variants as $vCode ) {
369  if ( $vCode == $contLang->getCode() ) {
370  continue; // we don't want default variant
371  }
372  $entry = $this->fileEntry(
373  $title->getCanonicalURL( '', $vCode ),
374  $date,
375  $this->priority( $namespace )
376  );
377  $length += strlen( $entry );
378  $this->write( $this->file, $entry );
379  }
380  }
381  }
382 
383  if ( $this->skipRedirects && $skippedRedirects > 0 ) {
384  $this->output( " skipped $skippedRedirects redirect(s)\n" );
385  }
386 
387  if ( $this->file ) {
388  $this->write( $this->file, $this->closeFile() );
389  $this->close( $this->file );
390  }
391  }
392  fwrite( $this->findex, $this->closeIndex() );
393  fclose( $this->findex );
394  }
395 
403  function open( $file, $flags ) {
404  $resource = $this->compress ? gzopen( $file, $flags ) : fopen( $file, $flags );
405  if ( $resource === false ) {
406  throw new MWException( __METHOD__
407  . " error opening file $file with flags $flags. Check permissions?" );
408  }
409 
410  return $resource;
411  }
412 
419  function write( &$handle, $str ) {
420  if ( $handle === true || $handle === false ) {
421  throw new MWException( __METHOD__ . " was passed a boolean as a file handle.\n" );
422  }
423  if ( $this->compress ) {
424  gzwrite( $handle, $str );
425  } else {
426  fwrite( $handle, $str );
427  }
428  }
429 
435  function close( &$handle ) {
436  if ( $this->compress ) {
437  gzclose( $handle );
438  } else {
439  fclose( $handle );
440  }
441  }
442 
450  function sitemapFilename( $namespace, $count ) {
451  $ext = $this->compress ? '.gz' : '';
452 
453  return "sitemap-{$this->identifier}-NS_$namespace-$count.xml$ext";
454  }
455 
461  function xmlHead() {
462  return '<?xml version="1.0" encoding="UTF-8"?>' . "\n";
463  }
464 
470  function xmlSchema() {
471  return 'http://www.sitemaps.org/schemas/sitemap/0.9';
472  }
473 
479  function openIndex() {
480  return $this->xmlHead() . '<sitemapindex xmlns="' . $this->xmlSchema() . '">' . "\n";
481  }
482 
489  function indexEntry( $filename ) {
490  return "\t<sitemap>\n" .
491  "\t\t<loc>" . wfGetServerUrl( PROTO_CANONICAL ) .
492  ( substr( $this->urlpath, 0, 1 ) === "/" ? "" : "/" ) .
493  "{$this->urlpath}$filename</loc>\n" .
494  "\t\t<lastmod>{$this->timestamp}</lastmod>\n" .
495  "\t</sitemap>\n";
496  }
497 
503  function closeIndex() {
504  return "</sitemapindex>\n";
505  }
506 
512  function openFile() {
513  return $this->xmlHead() . '<urlset xmlns="' . $this->xmlSchema() . '">' . "\n";
514  }
515 
524  function fileEntry( $url, $date, $priority ) {
525  return "\t<url>\n" .
526  // T36666: $url may contain bad characters such as ampersands.
527  "\t\t<loc>" . htmlspecialchars( $url ) . "</loc>\n" .
528  "\t\t<lastmod>$date</lastmod>\n" .
529  "\t\t<priority>$priority</priority>\n" .
530  "\t</url>\n";
531  }
532 
538  function closeFile() {
539  return "</urlset>\n";
540  }
541 
547  function generateLimit( $namespace ) {
548  // T19961: make a title with the longest possible URL in this namespace
549  $title = Title::makeTitle( $namespace, str_repeat( "\u{28B81}", 63 ) . "\u{5583}" );
550 
551  $this->limit = [
552  strlen( $this->openFile() ),
553  strlen( $this->fileEntry(
554  $title->getCanonicalURL(),
555  wfTimestamp( TS_ISO_8601, wfTimestamp() ),
556  $this->priority( $namespace )
557  ) ),
558  strlen( $this->closeFile() )
559  ];
560  }
561 }
562 
563 $maintClass = GenerateSitemap::class;
564 require_once RUN_MAINTENANCE_IF_MAIN;
resource $findex
A resource pointing to the sitemap index file.
const RUN_MAINTENANCE_IF_MAIN
Definition: Maintenance.php:39
Maintenance script that generates a sitemap for the site.
const NS_MAIN
Definition: Defines.php:60
open( $file, $flags)
gzopen() / fopen() wrapper
$wgSitemapNamespacesPriorities
Custom namespace priorities for sitemaps.
array $namespaces
A one-dimensional array of namespaces in the wiki.
array $limit
The number of entries to save in each sitemap file.
getOption( $name, $default=null)
Get an option, or return the default.
write(&$handle, $str)
gzwrite() / fwrite() wrapper
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:86
generateNamespaces()
Generate a one-dimensional array of existing namespaces.
string $identifier
Identifier to use in filenames, default $wgDBname.
array $priorities
Key => value entries of namespaces and their priorities.
xmlHead()
Return the XML required to open an XML file.
hasOption( $name)
Checks to see if a particular option exists.
wfGetServerUrl( $proto)
Get the wiki&#39;s "server", i.e.
const NS_TEMPLATE
Definition: Defines.php:70
bool $compress
Whether or not to use compression.
closeFile()
Return the XML required to close sitemap file.
getPageRes( $namespace)
Return a database resolution of all the pages in a given namespace.
bool $skipRedirects
Whether or not to include redirection pages.
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
guessPriority( $namespace)
If the namespace isn&#39;t listed on the priority list return the default priority for the namespace...
const NS_PROJECT
Definition: Defines.php:64
int $url_limit
The maximum amount of urls in a sitemap file.
generateLimit( $namespace)
Populate $this->limit.
const NS_MEDIAWIKI_TALK
Definition: Defines.php:69
close(&$handle)
gzclose() / fclose() wrapper
closeIndex()
Return the XML required to close a sitemap index file.
const NS_PROJECT_TALK
Definition: Defines.php:65
resource false $file
A resource pointing to a sitemap file.
addDescription( $text)
Set the description text.
openIndex()
Return the XML required to open a sitemap index file.
wfTimestampNow()
Convenience function; returns MediaWiki timestamp for the present time.
const NS_CATEGORY
Definition: Defines.php:74
const NS_FILE
Definition: Defines.php:66
const NS_FILE_TALK
Definition: Defines.php:67
output( $out, $channel=null)
Throw some output to the user.
fileEntry( $url, $date, $priority)
Return the XML for a single sitemap entry.
const NS_MEDIAWIKI
Definition: Defines.php:68
sitemapFilename( $namespace, $count)
Get a sitemap filename.
static makeTitle( $ns, $title, $fragment='', $interwiki='')
Create a new Title from a namespace index and a DB key.
Definition: Title.php:586
static getCurrentWikiDbDomain()
Definition: WikiMap.php:293
const PROTO_CANONICAL
Definition: Defines.php:203
const NS_CATEGORY_TALK
Definition: Defines.php:75
const NS_HELP
Definition: Defines.php:72
int $size_limit
The maximum size of a sitemap file.
const NS_HELP_TALK
Definition: Defines.php:73
wfMkdirParents( $dir, $mode=null, $caller=null)
Make directory, and make all parent directories if they don&#39;t exist.
object $dbr
A database replica DB object.
if(!is_readable( $file)) $ext
Definition: router.php:48
priority( $namespace)
Get the priority of a given namespace.
string $fspath
The path to prepend to the filename.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
const NS_TALK
Definition: Defines.php:61
const DB_REPLICA
Definition: defines.php:25
indexEntry( $filename)
Return the XML for a single sitemap indexfile entry.
$wgSitemapNamespaces
Array of namespaces to generate a Google sitemap for when the maintenance/generateSitemap.php script is run, or false if one is to be generated for all namespaces.
$maintClass
xmlSchema()
Return the XML schema being used.
string $urlpath
The URL path to prepend to filenames in the index; should resolve to the same directory as $fspath...
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
main()
Main loop.
const NS_USER_TALK
Definition: Defines.php:63
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
openFile()
Return the XML required to open a sitemap file.
const NS_TEMPLATE_TALK
Definition: Defines.php:71
string $timestamp
When this sitemap batch was generated.