MediaWiki  master
generateSitemap.php
Go to the documentation of this file.
1 <?php
32 
33 require_once __DIR__ . '/Maintenance.php';
34 
41  private const GS_MAIN = -2;
42  private const GS_TALK = -1;
43 
51  public $url_limit;
52 
60  public $size_limit;
61 
67  public $fspath;
68 
75  public $urlpath;
76 
82  public $compress;
83 
90 
96  public $limit = [];
97 
103  public $priorities = [];
104 
110  public $namespaces = [];
111 
117  public $timestamp;
118 
124  public $dbr;
125 
131  public $findex;
132 
138  public $file;
139 
145  private $identifier;
146 
147  public function __construct() {
148  parent::__construct();
149  $this->addDescription( 'Creates a sitemap for the site' );
150  $this->addOption(
151  'fspath',
152  'The file system path to save to, e.g. /tmp/sitemap; defaults to current directory',
153  false,
154  true
155  );
156  $this->addOption(
157  'urlpath',
158  'The URL path corresponding to --fspath, prepended to filenames in the index; '
159  . 'defaults to an empty string',
160  false,
161  true
162  );
163  $this->addOption(
164  'compress',
165  'Compress the sitemap files, can take value yes|no, default yes',
166  false,
167  true
168  );
169  $this->addOption( 'skip-redirects', 'Do not include redirecting articles in the sitemap' );
170  $this->addOption(
171  'identifier',
172  'What site identifier to use for the wiki, defaults to $wgDBname',
173  false,
174  true
175  );
176  }
177 
181  public function execute() {
182  $this->setNamespacePriorities();
183  $this->url_limit = 50000;
184  $this->size_limit = ( 2 ** 20 ) * 10;
185 
186  # Create directory if needed
187  $fspath = $this->getOption( 'fspath', getcwd() );
188  if ( !wfMkdirParents( $fspath, null, __METHOD__ ) ) {
189  $this->fatalError( "Can not create directory $fspath." );
190  }
191 
192  $dbDomain = WikiMap::getCurrentWikiDbDomain()->getId();
193  $this->fspath = realpath( $fspath ) . DIRECTORY_SEPARATOR;
194  $this->urlpath = $this->getOption( 'urlpath', "" );
195  if ( $this->urlpath !== "" && substr( $this->urlpath, -1 ) !== '/' ) {
196  $this->urlpath .= '/';
197  }
198  $this->identifier = $this->getOption( 'identifier', $dbDomain );
199  $this->compress = $this->getOption( 'compress', 'yes' ) !== 'no';
200  $this->skipRedirects = $this->hasOption( 'skip-redirects' );
201  $this->dbr = $this->getDB( DB_REPLICA );
202  $this->generateNamespaces();
203  $this->timestamp = wfTimestamp( TS_ISO_8601, wfTimestampNow() );
204  $encIdentifier = rawurlencode( $this->identifier );
205  $this->findex = fopen( "{$this->fspath}sitemap-index-{$encIdentifier}.xml", 'wb' );
206  $this->main();
207  }
208 
209  private function setNamespacePriorities() {
211 
212  // Custom main namespaces
213  $this->priorities[self::GS_MAIN] = '0.5';
214  // Custom talk namesspaces
215  $this->priorities[self::GS_TALK] = '0.1';
216  // MediaWiki standard namespaces
217  $this->priorities[NS_MAIN] = '1.0';
218  $this->priorities[NS_TALK] = '0.1';
219  $this->priorities[NS_USER] = '0.5';
220  $this->priorities[NS_USER_TALK] = '0.1';
221  $this->priorities[NS_PROJECT] = '0.5';
222  $this->priorities[NS_PROJECT_TALK] = '0.1';
223  $this->priorities[NS_FILE] = '0.5';
224  $this->priorities[NS_FILE_TALK] = '0.1';
225  $this->priorities[NS_MEDIAWIKI] = '0.0';
226  $this->priorities[NS_MEDIAWIKI_TALK] = '0.1';
227  $this->priorities[NS_TEMPLATE] = '0.0';
228  $this->priorities[NS_TEMPLATE_TALK] = '0.1';
229  $this->priorities[NS_HELP] = '0.5';
230  $this->priorities[NS_HELP_TALK] = '0.1';
231  $this->priorities[NS_CATEGORY] = '0.5';
232  $this->priorities[NS_CATEGORY_TALK] = '0.1';
233 
234  // Custom priorities
235  if ( $wgSitemapNamespacesPriorities !== false ) {
239  foreach ( $wgSitemapNamespacesPriorities as $namespace => $priority ) {
240  $float = floatval( $priority );
241  if ( $float > 1.0 ) {
242  $priority = '1.0';
243  } elseif ( $float < 0.0 ) {
244  $priority = '0.0';
245  }
246  $this->priorities[$namespace] = $priority;
247  }
248  }
249  }
250 
254  private function generateNamespaces() {
255  // Only generate for specific namespaces if $wgSitemapNamespaces is an array.
256  global $wgSitemapNamespaces;
257  if ( is_array( $wgSitemapNamespaces ) ) {
258  $this->namespaces = $wgSitemapNamespaces;
259 
260  return;
261  }
262 
263  $res = $this->dbr->select( 'page',
264  [ 'page_namespace' ],
265  [],
266  __METHOD__,
267  [
268  'GROUP BY' => 'page_namespace',
269  'ORDER BY' => 'page_namespace',
270  ]
271  );
272 
273  foreach ( $res as $row ) {
274  $this->namespaces[] = $row->page_namespace;
275  }
276  }
277 
284  private function priority( $namespace ) {
285  return $this->priorities[$namespace] ?? $this->guessPriority( $namespace );
286  }
287 
296  private function guessPriority( $namespace ) {
297  return MediaWikiServices::getInstance()->getNamespaceInfo()->isSubject( $namespace )
298  ? $this->priorities[self::GS_MAIN]
299  : $this->priorities[self::GS_TALK];
300  }
301 
308  private function getPageRes( $namespace ) {
309  return $this->dbr->select(
310  [ 'page', 'page_props' ],
311  [
312  'page_namespace',
313  'page_title',
314  'page_touched',
315  'page_is_redirect',
316  'pp_propname',
317  ],
318  [ 'page_namespace' => $namespace ],
319  __METHOD__,
320  [],
321  [
322  'page_props' => [
323  'LEFT JOIN',
324  [
325  'page_id = pp_page',
326  'pp_propname' => 'noindex'
327  ]
328  ]
329  ]
330  );
331  }
332 
336  public function main() {
337  $services = MediaWikiServices::getInstance();
338  $contLang = $services->getContentLanguage();
339  $langConverter = $services->getLanguageConverterFactory()->getLanguageConverter( $contLang );
340 
341  fwrite( $this->findex, $this->openIndex() );
342 
343  foreach ( $this->namespaces as $namespace ) {
344  $res = $this->getPageRes( $namespace );
345  $this->file = false;
346  $this->generateLimit( $namespace );
347  $length = $this->limit[0];
348  $i = $smcount = 0;
349 
350  $fns = $contLang->getFormattedNsText( $namespace );
351  $this->output( "$namespace ($fns)\n" );
352  $skippedRedirects = 0; // Number of redirects skipped for that namespace
353  $skippedNoindex = 0; // Number of pages with __NOINDEX__ switch for that NS
354  foreach ( $res as $row ) {
355  if ( $row->pp_propname === 'noindex' ) {
356  $skippedNoindex++;
357  continue;
358  }
359 
360  if ( $this->skipRedirects && $row->page_is_redirect ) {
361  $skippedRedirects++;
362  continue;
363  }
364 
365  if ( $i++ === 0
366  || $i === $this->url_limit + 1
367  || $length + $this->limit[1] + $this->limit[2] > $this->size_limit
368  ) {
369  if ( $this->file !== false ) {
370  $this->write( $this->file, $this->closeFile() );
371  $this->close( $this->file );
372  }
373  $filename = $this->sitemapFilename( $namespace, $smcount++ );
374  $this->file = $this->open( $this->fspath . $filename, 'wb' );
375  $this->write( $this->file, $this->openFile() );
376  fwrite( $this->findex, $this->indexEntry( $filename ) );
377  $this->output( "\t$this->fspath$filename\n" );
378  $length = $this->limit[0];
379  $i = 1;
380  }
381  $title = Title::makeTitle( $row->page_namespace, $row->page_title );
382  $date = wfTimestamp( TS_ISO_8601, $row->page_touched );
383  $entry = $this->fileEntry( $title->getCanonicalURL(), $date, $this->priority( $namespace ) );
384  $length += strlen( $entry );
385  $this->write( $this->file, $entry );
386  // generate pages for language variants
387  if ( $langConverter->hasVariants() ) {
388  $variants = $langConverter->getVariants();
389  foreach ( $variants as $vCode ) {
390  if ( $vCode == $contLang->getCode() ) {
391  continue; // we don't want default variant
392  }
393  $entry = $this->fileEntry(
394  $title->getCanonicalURL( '', $vCode ),
395  $date,
396  $this->priority( $namespace )
397  );
398  $length += strlen( $entry );
399  $this->write( $this->file, $entry );
400  }
401  }
402  }
403 
404  if ( $skippedNoindex > 0 ) {
405  $this->output( " skipped $skippedNoindex page(s) with __NOINDEX__ switch\n" );
406  }
407 
408  if ( $this->skipRedirects && $skippedRedirects > 0 ) {
409  $this->output( " skipped $skippedRedirects redirect(s)\n" );
410  }
411 
412  if ( $this->file ) {
413  $this->write( $this->file, $this->closeFile() );
414  $this->close( $this->file );
415  }
416  }
417  fwrite( $this->findex, $this->closeIndex() );
418  fclose( $this->findex );
419  }
420 
428  private function open( $file, $flags ) {
429  $resource = $this->compress ? gzopen( $file, $flags ) : fopen( $file, $flags );
430  if ( $resource === false ) {
431  throw new MWException( __METHOD__
432  . " error opening file $file with flags $flags. Check permissions?" );
433  }
434 
435  return $resource;
436  }
437 
444  private function write( &$handle, $str ) {
445  if ( $handle === true || $handle === false ) {
446  throw new MWException( __METHOD__ . " was passed a boolean as a file handle.\n" );
447  }
448  if ( $this->compress ) {
449  gzwrite( $handle, $str );
450  } else {
451  fwrite( $handle, $str );
452  }
453  }
454 
460  private function close( &$handle ) {
461  if ( $this->compress ) {
462  gzclose( $handle );
463  } else {
464  fclose( $handle );
465  }
466  }
467 
475  private function sitemapFilename( $namespace, $count ) {
476  $ext = $this->compress ? '.gz' : '';
477 
478  return "sitemap-{$this->identifier}-NS_$namespace-$count.xml$ext";
479  }
480 
486  private function xmlHead() {
487  return '<?xml version="1.0" encoding="UTF-8"?>' . "\n";
488  }
489 
495  private function xmlSchema() {
496  return 'http://www.sitemaps.org/schemas/sitemap/0.9';
497  }
498 
504  private function openIndex() {
505  return $this->xmlHead() . '<sitemapindex xmlns="' . $this->xmlSchema() . '">' . "\n";
506  }
507 
514  private function indexEntry( $filename ) {
515  return "\t<sitemap>\n" .
516  "\t\t<loc>" . wfGetServerUrl( PROTO_CANONICAL ) .
517  ( substr( $this->urlpath, 0, 1 ) === "/" ? "" : "/" ) .
518  "{$this->urlpath}$filename</loc>\n" .
519  "\t\t<lastmod>{$this->timestamp}</lastmod>\n" .
520  "\t</sitemap>\n";
521  }
522 
528  private function closeIndex() {
529  return "</sitemapindex>\n";
530  }
531 
537  private function openFile() {
538  return $this->xmlHead() . '<urlset xmlns="' . $this->xmlSchema() . '">' . "\n";
539  }
540 
549  private function fileEntry( $url, $date, $priority ) {
550  return "\t<url>\n" .
551  // T36666: $url may contain bad characters such as ampersands.
552  "\t\t<loc>" . htmlspecialchars( $url ) . "</loc>\n" .
553  "\t\t<lastmod>$date</lastmod>\n" .
554  "\t\t<priority>$priority</priority>\n" .
555  "\t</url>\n";
556  }
557 
563  private function closeFile() {
564  return "</urlset>\n";
565  }
566 
572  private function generateLimit( $namespace ) {
573  // T19961: make a title with the longest possible URL in this namespace
574  $title = Title::makeTitle( $namespace, str_repeat( "\u{28B81}", 63 ) . "\u{5583}" );
575 
576  $this->limit = [
577  strlen( $this->openFile() ),
578  strlen( $this->fileEntry(
579  $title->getCanonicalURL(),
580  wfTimestamp( TS_ISO_8601, wfTimestamp() ),
581  $this->priority( $namespace )
582  ) ),
583  strlen( $this->closeFile() )
584  ];
585  }
586 }
587 
588 $maintClass = GenerateSitemap::class;
589 require_once RUN_MAINTENANCE_IF_MAIN;
GenerateSitemap\main
main()
Main loop.
Definition: generateSitemap.php:336
GenerateSitemap\$dbr
IDatabase $dbr
A database replica DB object.
Definition: generateSitemap.php:124
GenerateSitemap\generateNamespaces
generateNamespaces()
Generate a one-dimensional array of existing namespaces.
Definition: generateSitemap.php:254
$wgSitemapNamespacesPriorities
$wgSitemapNamespacesPriorities
Custom namespace priorities for sitemaps.
Definition: DefaultSettings.php:7727
NS_FILE_TALK
const NS_FILE_TALK
Definition: Defines.php:71
WikiMap\getCurrentWikiDbDomain
static getCurrentWikiDbDomain()
Definition: WikiMap.php:293
NS_MEDIAWIKI
const NS_MEDIAWIKI
Definition: Defines.php:72
wfMkdirParents
wfMkdirParents( $dir, $mode=null, $caller=null)
Make directory, and make all parent directories if they don't exist.
Definition: GlobalFunctions.php:1773
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:190
Maintenance\fatalError
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
Definition: Maintenance.php:489
Maintenance\addDescription
addDescription( $text)
Set the description text.
Definition: Maintenance.php:329
GenerateSitemap\xmlHead
xmlHead()
Return the XML required to open an XML file.
Definition: generateSitemap.php:486
NS_CATEGORY_TALK
const NS_CATEGORY_TALK
Definition: Defines.php:79
wfTimestamp
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Definition: GlobalFunctions.php:1691
NS_MEDIAWIKI_TALK
const NS_MEDIAWIKI_TALK
Definition: Defines.php:73
GenerateSitemap\execute
execute()
Execute.
Definition: generateSitemap.php:181
Maintenance
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:59
GenerateSitemap\GS_MAIN
const GS_MAIN
Definition: generateSitemap.php:41
$res
$res
Definition: testCompression.php:57
GenerateSitemap\$url_limit
int $url_limit
The maximum amount of urls in a sitemap file.
Definition: generateSitemap.php:51
Wikimedia\Rdbms\IDatabase
Basic database interface for live and lazy-loaded relation database handles.
Definition: IDatabase.php:38
NS_MAIN
const NS_MAIN
Definition: Defines.php:64
GenerateSitemap\generateLimit
generateLimit( $namespace)
Populate $this->limit.
Definition: generateSitemap.php:572
GenerateSitemap\$timestamp
string $timestamp
When this sitemap batch was generated.
Definition: generateSitemap.php:117
GenerateSitemap\__construct
__construct()
Default constructor.
Definition: generateSitemap.php:147
NS_HELP_TALK
const NS_HELP_TALK
Definition: Defines.php:77
MWException
MediaWiki exception.
Definition: MWException.php:29
GenerateSitemap\close
close(&$handle)
gzclose() / fclose() wrapper
Definition: generateSitemap.php:460
Wikimedia\Rdbms\IResultWrapper
Result wrapper for grabbing data queried from an IDatabase object.
Definition: IResultWrapper.php:26
GenerateSitemap\$file
resource false $file
A resource pointing to a sitemap file.
Definition: generateSitemap.php:138
GenerateSitemap\$findex
resource $findex
A resource pointing to the sitemap index file.
Definition: generateSitemap.php:131
GenerateSitemap\setNamespacePriorities
setNamespacePriorities()
Definition: generateSitemap.php:209
NS_TEMPLATE
const NS_TEMPLATE
Definition: Defines.php:74
NS_PROJECT
const NS_PROJECT
Definition: Defines.php:68
Maintenance\addOption
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
Definition: Maintenance.php:249
$title
$title
Definition: testCompression.php:38
GenerateSitemap\$priorities
array $priorities
Key => value entries of namespaces and their priorities.
Definition: generateSitemap.php:103
Title\makeTitle
static makeTitle( $ns, $title, $fragment='', $interwiki='')
Create a new Title from a namespace index and a DB key.
Definition: Title.php:651
DB_REPLICA
const DB_REPLICA
Definition: defines.php:25
wfTimestampNow
wfTimestampNow()
Convenience function; returns MediaWiki timestamp for the present time.
Definition: GlobalFunctions.php:1720
GenerateSitemap\$skipRedirects
bool $skipRedirects
Whether or not to include redirection pages.
Definition: generateSitemap.php:89
PROTO_CANONICAL
const PROTO_CANONICAL
Definition: Defines.php:196
GenerateSitemap\guessPriority
guessPriority( $namespace)
If the namespace isn't listed on the priority list return the default priority for the namespace,...
Definition: generateSitemap.php:296
GenerateSitemap\$compress
bool $compress
Whether or not to use compression.
Definition: generateSitemap.php:82
NS_TALK
const NS_TALK
Definition: Defines.php:65
GenerateSitemap\closeIndex
closeIndex()
Return the XML required to close a sitemap index file.
Definition: generateSitemap.php:528
GenerateSitemap\fileEntry
fileEntry( $url, $date, $priority)
Return the XML for a single sitemap entry.
Definition: generateSitemap.php:549
GenerateSitemap\sitemapFilename
sitemapFilename( $namespace, $count)
Get a sitemap filename.
Definition: generateSitemap.php:475
GenerateSitemap\closeFile
closeFile()
Return the XML required to close sitemap file.
Definition: generateSitemap.php:563
GenerateSitemap\GS_TALK
const GS_TALK
Definition: generateSitemap.php:42
GenerateSitemap\$size_limit
int $size_limit
The maximum size of a sitemap file.
Definition: generateSitemap.php:60
GenerateSitemap\getPageRes
getPageRes( $namespace)
Return a database resolution of all the pages in a given namespace.
Definition: generateSitemap.php:308
Maintenance\getDB
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
Definition: Maintenance.php:1362
GenerateSitemap\$fspath
string $fspath
The path to prepend to the filename.
Definition: generateSitemap.php:67
$wgSitemapNamespaces
$wgSitemapNamespaces
Array of namespaces to generate a Google sitemap for when the maintenance/generateSitemap....
Definition: DefaultSettings.php:7711
NS_USER
const NS_USER
Definition: Defines.php:66
GenerateSitemap\priority
priority( $namespace)
Get the priority of a given namespace.
Definition: generateSitemap.php:284
NS_HELP
const NS_HELP
Definition: Defines.php:76
GenerateSitemap\xmlSchema
xmlSchema()
Return the XML schema being used.
Definition: generateSitemap.php:495
GenerateSitemap\$namespaces
array $namespaces
A one-dimensional array of namespaces in the wiki.
Definition: generateSitemap.php:110
GenerateSitemap\openIndex
openIndex()
Return the XML required to open a sitemap index file.
Definition: generateSitemap.php:504
Maintenance\getOption
getOption( $name, $default=null)
Get an option, or return the default.
Definition: Maintenance.php:286
GenerateSitemap\openFile
openFile()
Return the XML required to open a sitemap file.
Definition: generateSitemap.php:537
NS_CATEGORY
const NS_CATEGORY
Definition: Defines.php:78
NS_USER_TALK
const NS_USER_TALK
Definition: Defines.php:67
Maintenance\output
output( $out, $channel=null)
Throw some output to the user.
Definition: Maintenance.php:435
$ext
if(!is_readable( $file)) $ext
Definition: router.php:48
GenerateSitemap\$limit
array $limit
The number of entries to save in each sitemap file.
Definition: generateSitemap.php:96
NS_FILE
const NS_FILE
Definition: Defines.php:70
Maintenance\hasOption
hasOption( $name)
Checks to see if a particular option was set.
Definition: Maintenance.php:271
NS_PROJECT_TALK
const NS_PROJECT_TALK
Definition: Defines.php:69
wfGetServerUrl
wfGetServerUrl( $proto)
Get the wiki's "server", i.e.
Definition: GlobalFunctions.php:552
GenerateSitemap
Maintenance script that generates a sitemap for the site.
Definition: generateSitemap.php:40
GenerateSitemap\open
open( $file, $flags)
gzopen() / fopen() wrapper
Definition: generateSitemap.php:428
GenerateSitemap\$urlpath
string $urlpath
The URL path to prepend to filenames in the index; should resolve to the same directory as $fspath.
Definition: generateSitemap.php:75
GenerateSitemap\indexEntry
indexEntry( $filename)
Return the XML for a single sitemap indexfile entry.
Definition: generateSitemap.php:514
NS_TEMPLATE_TALK
const NS_TEMPLATE_TALK
Definition: Defines.php:75
GenerateSitemap\$identifier
string $identifier
Identifier to use in filenames, default $wgDBname.
Definition: generateSitemap.php:145
GenerateSitemap\write
write(&$handle, $str)
gzwrite() / fwrite() wrapper
Definition: generateSitemap.php:444
$maintClass
$maintClass
Definition: generateSitemap.php:588