MediaWiki  master
generateSitemap.php
Go to the documentation of this file.
1 <?php
32 
33 require_once __DIR__ . '/Maintenance.php';
34 
41  private const GS_MAIN = -2;
42  private const GS_TALK = -1;
43 
51  public $url_limit;
52 
60  public $size_limit;
61 
67  public $fspath;
68 
75  public $urlpath;
76 
82  public $compress;
83 
90 
96  public $limit = [];
97 
103  public $priorities = [];
104 
110  public $namespaces = [];
111 
117  public $timestamp;
118 
124  public $dbr;
125 
131  public $findex;
132 
138  public $file;
139 
145  private $identifier;
146 
147  public function __construct() {
148  parent::__construct();
149  $this->addDescription( 'Creates a sitemap for the site' );
150  $this->addOption(
151  'fspath',
152  'The file system path to save to, e.g. /tmp/sitemap; defaults to current directory',
153  false,
154  true
155  );
156  $this->addOption(
157  'urlpath',
158  'The URL path corresponding to --fspath, prepended to filenames in the index; '
159  . 'defaults to an empty string',
160  false,
161  true
162  );
163  $this->addOption(
164  'compress',
165  'Compress the sitemap files, can take value yes|no, default yes',
166  false,
167  true
168  );
169  $this->addOption( 'skip-redirects', 'Do not include redirecting articles in the sitemap' );
170  $this->addOption(
171  'identifier',
172  'What site identifier to use for the wiki, defaults to $wgDBname',
173  false,
174  true
175  );
176  }
177 
181  public function execute() {
182  $this->setNamespacePriorities();
183  $this->url_limit = 50000;
184  $this->size_limit = ( 2 ** 20 ) * 10;
185 
186  # Create directory if needed
187  $fspath = $this->getOption( 'fspath', getcwd() );
188  if ( !wfMkdirParents( $fspath, null, __METHOD__ ) ) {
189  $this->fatalError( "Can not create directory $fspath." );
190  }
191 
192  $dbDomain = WikiMap::getCurrentWikiDbDomain()->getId();
193  $this->fspath = realpath( $fspath ) . DIRECTORY_SEPARATOR;
194  $this->urlpath = $this->getOption( 'urlpath', "" );
195  if ( $this->urlpath !== "" && substr( $this->urlpath, -1 ) !== '/' ) {
196  $this->urlpath .= '/';
197  }
198  $this->identifier = $this->getOption( 'identifier', $dbDomain );
199  $this->compress = $this->getOption( 'compress', 'yes' ) !== 'no';
200  $this->skipRedirects = $this->hasOption( 'skip-redirects' );
201  $this->dbr = $this->getDB( DB_REPLICA );
202  $this->generateNamespaces();
203  $this->timestamp = wfTimestamp( TS_ISO_8601, wfTimestampNow() );
204  $encIdentifier = rawurlencode( $this->identifier );
205  $this->findex = fopen( "{$this->fspath}sitemap-index-{$encIdentifier}.xml", 'wb' );
206  $this->main();
207  }
208 
209  private function setNamespacePriorities() {
211 
212  // Custom main namespaces
213  $this->priorities[self::GS_MAIN] = '0.5';
214  // Custom talk namesspaces
215  $this->priorities[self::GS_TALK] = '0.1';
216  // MediaWiki standard namespaces
217  $this->priorities[NS_MAIN] = '1.0';
218  $this->priorities[NS_TALK] = '0.1';
219  $this->priorities[NS_USER] = '0.5';
220  $this->priorities[NS_USER_TALK] = '0.1';
221  $this->priorities[NS_PROJECT] = '0.5';
222  $this->priorities[NS_PROJECT_TALK] = '0.1';
223  $this->priorities[NS_FILE] = '0.5';
224  $this->priorities[NS_FILE_TALK] = '0.1';
225  $this->priorities[NS_MEDIAWIKI] = '0.0';
226  $this->priorities[NS_MEDIAWIKI_TALK] = '0.1';
227  $this->priorities[NS_TEMPLATE] = '0.0';
228  $this->priorities[NS_TEMPLATE_TALK] = '0.1';
229  $this->priorities[NS_HELP] = '0.5';
230  $this->priorities[NS_HELP_TALK] = '0.1';
231  $this->priorities[NS_CATEGORY] = '0.5';
232  $this->priorities[NS_CATEGORY_TALK] = '0.1';
233 
234  // Custom priorities
235  if ( $wgSitemapNamespacesPriorities !== false ) {
239  foreach ( $wgSitemapNamespacesPriorities as $namespace => $priority ) {
240  $float = floatval( $priority );
241  if ( $float > 1.0 ) {
242  $priority = '1.0';
243  } elseif ( $float < 0.0 ) {
244  $priority = '0.0';
245  }
246  $this->priorities[$namespace] = $priority;
247  }
248  }
249  }
250 
254  private function generateNamespaces() {
255  // Only generate for specific namespaces if $wgSitemapNamespaces is an array.
256  global $wgSitemapNamespaces;
257  if ( is_array( $wgSitemapNamespaces ) ) {
258  $this->namespaces = $wgSitemapNamespaces;
259 
260  return;
261  }
262 
263  $res = $this->dbr->select( 'page',
264  [ 'page_namespace' ],
265  [],
266  __METHOD__,
267  [
268  'GROUP BY' => 'page_namespace',
269  'ORDER BY' => 'page_namespace',
270  ]
271  );
272 
273  foreach ( $res as $row ) {
274  $this->namespaces[] = $row->page_namespace;
275  }
276  }
277 
284  private function priority( $namespace ) {
285  return $this->priorities[$namespace] ?? $this->guessPriority( $namespace );
286  }
287 
296  private function guessPriority( $namespace ) {
297  return MediaWikiServices::getInstance()->getNamespaceInfo()->isSubject( $namespace )
298  ? $this->priorities[self::GS_MAIN]
299  : $this->priorities[self::GS_TALK];
300  }
301 
308  private function getPageRes( $namespace ) {
309  return $this->dbr->select(
310  [ 'page', 'page_props' ],
311  [
312  'page_namespace',
313  'page_title',
314  'page_touched',
315  'page_is_redirect',
316  'pp_propname',
317  ],
318  [ 'page_namespace' => $namespace ],
319  __METHOD__,
320  [],
321  [
322  'page_props' => [
323  'LEFT JOIN',
324  [
325  'page_id = pp_page',
326  'pp_propname' => 'noindex'
327  ]
328  ]
329  ]
330  );
331  }
332 
336  public function main() {
337  $contLang = MediaWikiServices::getInstance()->getContentLanguage();
338 
339  fwrite( $this->findex, $this->openIndex() );
340 
341  foreach ( $this->namespaces as $namespace ) {
342  $res = $this->getPageRes( $namespace );
343  $this->file = false;
344  $this->generateLimit( $namespace );
345  $length = $this->limit[0];
346  $i = $smcount = 0;
347 
348  $fns = $contLang->getFormattedNsText( $namespace );
349  $this->output( "$namespace ($fns)\n" );
350  $skippedRedirects = 0; // Number of redirects skipped for that namespace
351  $skippedNoindex = 0; // Number of pages with __NOINDEX__ switch for that NS
352  foreach ( $res as $row ) {
353  if ( $row->pp_propname === 'noindex' ) {
354  $skippedNoindex++;
355  continue;
356  }
357 
358  if ( $this->skipRedirects && $row->page_is_redirect ) {
359  $skippedRedirects++;
360  continue;
361  }
362 
363  if ( $i++ === 0
364  || $i === $this->url_limit + 1
365  || $length + $this->limit[1] + $this->limit[2] > $this->size_limit
366  ) {
367  if ( $this->file !== false ) {
368  $this->write( $this->file, $this->closeFile() );
369  $this->close( $this->file );
370  }
371  $filename = $this->sitemapFilename( $namespace, $smcount++ );
372  $this->file = $this->open( $this->fspath . $filename, 'wb' );
373  $this->write( $this->file, $this->openFile() );
374  fwrite( $this->findex, $this->indexEntry( $filename ) );
375  $this->output( "\t$this->fspath$filename\n" );
376  $length = $this->limit[0];
377  $i = 1;
378  }
379  $title = Title::makeTitle( $row->page_namespace, $row->page_title );
380  $date = wfTimestamp( TS_ISO_8601, $row->page_touched );
381  $entry = $this->fileEntry( $title->getCanonicalURL(), $date, $this->priority( $namespace ) );
382  $length += strlen( $entry );
383  $this->write( $this->file, $entry );
384  // generate pages for language variants
385  if ( $contLang->hasVariants() ) {
386  $variants = $contLang->getVariants();
387  foreach ( $variants as $vCode ) {
388  if ( $vCode == $contLang->getCode() ) {
389  continue; // we don't want default variant
390  }
391  $entry = $this->fileEntry(
392  $title->getCanonicalURL( '', $vCode ),
393  $date,
394  $this->priority( $namespace )
395  );
396  $length += strlen( $entry );
397  $this->write( $this->file, $entry );
398  }
399  }
400  }
401 
402  if ( $skippedNoindex > 0 ) {
403  $this->output( " skipped $skippedNoindex page(s) with __NOINDEX__ switch\n" );
404  }
405 
406  if ( $this->skipRedirects && $skippedRedirects > 0 ) {
407  $this->output( " skipped $skippedRedirects redirect(s)\n" );
408  }
409 
410  if ( $this->file ) {
411  $this->write( $this->file, $this->closeFile() );
412  $this->close( $this->file );
413  }
414  }
415  fwrite( $this->findex, $this->closeIndex() );
416  fclose( $this->findex );
417  }
418 
426  private function open( $file, $flags ) {
427  $resource = $this->compress ? gzopen( $file, $flags ) : fopen( $file, $flags );
428  if ( $resource === false ) {
429  throw new MWException( __METHOD__
430  . " error opening file $file with flags $flags. Check permissions?" );
431  }
432 
433  return $resource;
434  }
435 
442  private function write( &$handle, $str ) {
443  if ( $handle === true || $handle === false ) {
444  throw new MWException( __METHOD__ . " was passed a boolean as a file handle.\n" );
445  }
446  if ( $this->compress ) {
447  gzwrite( $handle, $str );
448  } else {
449  fwrite( $handle, $str );
450  }
451  }
452 
458  private function close( &$handle ) {
459  if ( $this->compress ) {
460  gzclose( $handle );
461  } else {
462  fclose( $handle );
463  }
464  }
465 
473  private function sitemapFilename( $namespace, $count ) {
474  $ext = $this->compress ? '.gz' : '';
475 
476  return "sitemap-{$this->identifier}-NS_$namespace-$count.xml$ext";
477  }
478 
484  private function xmlHead() {
485  return '<?xml version="1.0" encoding="UTF-8"?>' . "\n";
486  }
487 
493  private function xmlSchema() {
494  return 'http://www.sitemaps.org/schemas/sitemap/0.9';
495  }
496 
502  private function openIndex() {
503  return $this->xmlHead() . '<sitemapindex xmlns="' . $this->xmlSchema() . '">' . "\n";
504  }
505 
512  private function indexEntry( $filename ) {
513  return "\t<sitemap>\n" .
514  "\t\t<loc>" . wfGetServerUrl( PROTO_CANONICAL ) .
515  ( substr( $this->urlpath, 0, 1 ) === "/" ? "" : "/" ) .
516  "{$this->urlpath}$filename</loc>\n" .
517  "\t\t<lastmod>{$this->timestamp}</lastmod>\n" .
518  "\t</sitemap>\n";
519  }
520 
526  private function closeIndex() {
527  return "</sitemapindex>\n";
528  }
529 
535  private function openFile() {
536  return $this->xmlHead() . '<urlset xmlns="' . $this->xmlSchema() . '">' . "\n";
537  }
538 
547  private function fileEntry( $url, $date, $priority ) {
548  return "\t<url>\n" .
549  // T36666: $url may contain bad characters such as ampersands.
550  "\t\t<loc>" . htmlspecialchars( $url ) . "</loc>\n" .
551  "\t\t<lastmod>$date</lastmod>\n" .
552  "\t\t<priority>$priority</priority>\n" .
553  "\t</url>\n";
554  }
555 
561  private function closeFile() {
562  return "</urlset>\n";
563  }
564 
570  private function generateLimit( $namespace ) {
571  // T19961: make a title with the longest possible URL in this namespace
572  $title = Title::makeTitle( $namespace, str_repeat( "\u{28B81}", 63 ) . "\u{5583}" );
573 
574  $this->limit = [
575  strlen( $this->openFile() ),
576  strlen( $this->fileEntry(
577  $title->getCanonicalURL(),
578  wfTimestamp( TS_ISO_8601, wfTimestamp() ),
579  $this->priority( $namespace )
580  ) ),
581  strlen( $this->closeFile() )
582  ];
583  }
584 }
585 
586 $maintClass = GenerateSitemap::class;
587 require_once RUN_MAINTENANCE_IF_MAIN;
GenerateSitemap\main
main()
Main loop.
Definition: generateSitemap.php:336
RUN_MAINTENANCE_IF_MAIN
const RUN_MAINTENANCE_IF_MAIN
Definition: Maintenance.php:38
GenerateSitemap\$dbr
IDatabase $dbr
A database replica DB object.
Definition: generateSitemap.php:124
GenerateSitemap\generateNamespaces
generateNamespaces()
Generate a one-dimensional array of existing namespaces.
Definition: generateSitemap.php:254
$wgSitemapNamespacesPriorities
$wgSitemapNamespacesPriorities
Custom namespace priorities for sitemaps.
Definition: DefaultSettings.php:7076
PROTO_CANONICAL
const PROTO_CANONICAL
Definition: Defines.php:212
WikiMap\getCurrentWikiDbDomain
static getCurrentWikiDbDomain()
Definition: WikiMap.php:293
NS_HELP
const NS_HELP
Definition: Defines.php:81
wfMkdirParents
wfMkdirParents( $dir, $mode=null, $caller=null)
Make directory, and make all parent directories if they don't exist.
Definition: GlobalFunctions.php:1897
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:165
NS_TEMPLATE_TALK
const NS_TEMPLATE_TALK
Definition: Defines.php:80
Maintenance\fatalError
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
Definition: Maintenance.php:486
Maintenance\addDescription
addDescription( $text)
Set the description text.
Definition: Maintenance.php:327
GenerateSitemap\xmlHead
xmlHead()
Return the XML required to open an XML file.
Definition: generateSitemap.php:484
wfTimestamp
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Definition: GlobalFunctions.php:1815
GenerateSitemap\execute
execute()
Execute.
Definition: generateSitemap.php:181
NS_FILE
const NS_FILE
Definition: Defines.php:75
NS_TEMPLATE
const NS_TEMPLATE
Definition: Defines.php:79
Maintenance
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:55
GenerateSitemap\GS_MAIN
const GS_MAIN
Definition: generateSitemap.php:41
$res
$res
Definition: testCompression.php:57
GenerateSitemap\$url_limit
int $url_limit
The maximum amount of urls in a sitemap file.
Definition: generateSitemap.php:51
Wikimedia\Rdbms\IDatabase
Basic database interface for live and lazy-loaded relation database handles.
Definition: IDatabase.php:38
GenerateSitemap\generateLimit
generateLimit( $namespace)
Populate $this->limit.
Definition: generateSitemap.php:570
NS_MAIN
const NS_MAIN
Definition: Defines.php:69
GenerateSitemap\$timestamp
string $timestamp
When this sitemap batch was generated.
Definition: generateSitemap.php:117
GenerateSitemap\__construct
__construct()
Default constructor.
Definition: generateSitemap.php:147
MWException
MediaWiki exception.
Definition: MWException.php:29
GenerateSitemap\close
close(&$handle)
gzclose() / fclose() wrapper
Definition: generateSitemap.php:458
NS_PROJECT
const NS_PROJECT
Definition: Defines.php:73
Wikimedia\Rdbms\IResultWrapper
Result wrapper for grabbing data queried from an IDatabase object.
Definition: IResultWrapper.php:24
GenerateSitemap\$file
resource false $file
A resource pointing to a sitemap file.
Definition: generateSitemap.php:138
GenerateSitemap\$findex
resource $findex
A resource pointing to the sitemap index file.
Definition: generateSitemap.php:131
GenerateSitemap\setNamespacePriorities
setNamespacePriorities()
Definition: generateSitemap.php:209
NS_MEDIAWIKI_TALK
const NS_MEDIAWIKI_TALK
Definition: Defines.php:78
Maintenance\addOption
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
Definition: Maintenance.php:245
$title
$title
Definition: testCompression.php:38
GenerateSitemap\$priorities
array $priorities
Key => value entries of namespaces and their priorities.
Definition: generateSitemap.php:103
Title\makeTitle
static makeTitle( $ns, $title, $fragment='', $interwiki='')
Create a new Title from a namespace index and a DB key.
Definition: Title.php:591
DB_REPLICA
const DB_REPLICA
Definition: defines.php:25
wfTimestampNow
wfTimestampNow()
Convenience function; returns MediaWiki timestamp for the present time.
Definition: GlobalFunctions.php:1844
NS_CATEGORY
const NS_CATEGORY
Definition: Defines.php:83
GenerateSitemap\$skipRedirects
bool $skipRedirects
Whether or not to include redirection pages.
Definition: generateSitemap.php:89
GenerateSitemap\guessPriority
guessPriority( $namespace)
If the namespace isn't listed on the priority list return the default priority for the namespace,...
Definition: generateSitemap.php:296
GenerateSitemap\$compress
bool $compress
Whether or not to use compression.
Definition: generateSitemap.php:82
GenerateSitemap\closeIndex
closeIndex()
Return the XML required to close a sitemap index file.
Definition: generateSitemap.php:526
GenerateSitemap\fileEntry
fileEntry( $url, $date, $priority)
Return the XML for a single sitemap entry.
Definition: generateSitemap.php:547
GenerateSitemap\sitemapFilename
sitemapFilename( $namespace, $count)
Get a sitemap filename.
Definition: generateSitemap.php:473
NS_USER_TALK
const NS_USER_TALK
Definition: Defines.php:72
GenerateSitemap\closeFile
closeFile()
Return the XML required to close sitemap file.
Definition: generateSitemap.php:561
GenerateSitemap\GS_TALK
const GS_TALK
Definition: generateSitemap.php:42
GenerateSitemap\$size_limit
int $size_limit
The maximum size of a sitemap file.
Definition: generateSitemap.php:60
NS_PROJECT_TALK
const NS_PROJECT_TALK
Definition: Defines.php:74
GenerateSitemap\getPageRes
getPageRes( $namespace)
Return a database resolution of all the pages in a given namespace.
Definition: generateSitemap.php:308
Maintenance\getDB
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
Definition: Maintenance.php:1369
GenerateSitemap\$fspath
string $fspath
The path to prepend to the filename.
Definition: generateSitemap.php:67
$wgSitemapNamespaces
$wgSitemapNamespaces
Array of namespaces to generate a Google sitemap for when the maintenance/generateSitemap....
Definition: DefaultSettings.php:7060
GenerateSitemap\priority
priority( $namespace)
Get the priority of a given namespace.
Definition: generateSitemap.php:284
GenerateSitemap\xmlSchema
xmlSchema()
Return the XML schema being used.
Definition: generateSitemap.php:493
GenerateSitemap\$namespaces
array $namespaces
A one-dimensional array of namespaces in the wiki.
Definition: generateSitemap.php:110
GenerateSitemap\openIndex
openIndex()
Return the XML required to open a sitemap index file.
Definition: generateSitemap.php:502
Maintenance\getOption
getOption( $name, $default=null)
Get an option, or return the default.
Definition: Maintenance.php:281
NS_HELP_TALK
const NS_HELP_TALK
Definition: Defines.php:82
GenerateSitemap\openFile
openFile()
Return the XML required to open a sitemap file.
Definition: generateSitemap.php:535
NS_USER
const NS_USER
Definition: Defines.php:71
Maintenance\output
output( $out, $channel=null)
Throw some output to the user.
Definition: Maintenance.php:433
NS_TALK
const NS_TALK
Definition: Defines.php:70
$ext
if(!is_readable( $file)) $ext
Definition: router.php:48
NS_MEDIAWIKI
const NS_MEDIAWIKI
Definition: Defines.php:77
GenerateSitemap\$limit
array $limit
The number of entries to save in each sitemap file.
Definition: generateSitemap.php:96
NS_FILE_TALK
const NS_FILE_TALK
Definition: Defines.php:76
Maintenance\hasOption
hasOption( $name)
Checks to see if a particular option exists.
Definition: Maintenance.php:266
NS_CATEGORY_TALK
const NS_CATEGORY_TALK
Definition: Defines.php:84
wfGetServerUrl
wfGetServerUrl( $proto)
Get the wiki's "server", i.e.
Definition: GlobalFunctions.php:568
GenerateSitemap
Maintenance script that generates a sitemap for the site.
Definition: generateSitemap.php:40
GenerateSitemap\open
open( $file, $flags)
gzopen() / fopen() wrapper
Definition: generateSitemap.php:426
GenerateSitemap\$urlpath
string $urlpath
The URL path to prepend to filenames in the index; should resolve to the same directory as $fspath.
Definition: generateSitemap.php:75
GenerateSitemap\indexEntry
indexEntry( $filename)
Return the XML for a single sitemap indexfile entry.
Definition: generateSitemap.php:512
GenerateSitemap\$identifier
string $identifier
Identifier to use in filenames, default $wgDBname.
Definition: generateSitemap.php:145
GenerateSitemap\write
write(&$handle, $str)
gzwrite() / fwrite() wrapper
Definition: generateSitemap.php:442
$maintClass
$maintClass
Definition: generateSitemap.php:586