MediaWiki  master
generateSitemap.php
Go to the documentation of this file.
1 <?php
33 
34 require_once __DIR__ . '/Maintenance.php';
35 
42  private const GS_MAIN = -2;
43  private const GS_TALK = -1;
44 
52  public $url_limit;
53 
61  public $size_limit;
62 
68  public $fspath;
69 
76  public $urlpath;
77 
83  public $compress;
84 
91 
97  public $limit = [];
98 
104  public $priorities = [];
105 
111  public $namespaces = [];
112 
118  public $timestamp;
119 
125  public $dbr;
126 
132  public $findex;
133 
139  public $file;
140 
146  private $identifier;
147 
148  public function __construct() {
149  parent::__construct();
150  $this->addDescription( 'Creates a sitemap for the site' );
151  $this->addOption(
152  'fspath',
153  'The file system path to save to, e.g. /tmp/sitemap; defaults to current directory',
154  false,
155  true
156  );
157  $this->addOption(
158  'urlpath',
159  'The URL path corresponding to --fspath, prepended to filenames in the index; '
160  . 'defaults to an empty string',
161  false,
162  true
163  );
164  $this->addOption(
165  'compress',
166  'Compress the sitemap files, can take value yes|no, default yes',
167  false,
168  true
169  );
170  $this->addOption( 'skip-redirects', 'Do not include redirecting articles in the sitemap' );
171  $this->addOption(
172  'identifier',
173  'What site identifier to use for the wiki, defaults to $wgDBname',
174  false,
175  true
176  );
177  }
178 
182  public function execute() {
183  $this->setNamespacePriorities();
184  $this->url_limit = 50000;
185  $this->size_limit = ( 2 ** 20 ) * 10;
186 
187  # Create directory if needed
188  $fspath = $this->getOption( 'fspath', getcwd() );
189  if ( !wfMkdirParents( $fspath, null, __METHOD__ ) ) {
190  $this->fatalError( "Can not create directory $fspath." );
191  }
192 
193  $dbDomain = WikiMap::getCurrentWikiDbDomain()->getId();
194  $this->fspath = realpath( $fspath ) . DIRECTORY_SEPARATOR;
195  $this->urlpath = $this->getOption( 'urlpath', "" );
196  if ( $this->urlpath !== "" && substr( $this->urlpath, -1 ) !== '/' ) {
197  $this->urlpath .= '/';
198  }
199  $this->identifier = $this->getOption( 'identifier', $dbDomain );
200  $this->compress = $this->getOption( 'compress', 'yes' ) !== 'no';
201  $this->skipRedirects = $this->hasOption( 'skip-redirects' );
202  $this->dbr = $this->getDB( DB_REPLICA );
203  $this->generateNamespaces();
204  $this->timestamp = wfTimestamp( TS_ISO_8601, wfTimestampNow() );
205  $encIdentifier = rawurlencode( $this->identifier );
206  $this->findex = fopen( "{$this->fspath}sitemap-index-{$encIdentifier}.xml", 'wb' );
207  $this->main();
208  }
209 
210  private function setNamespacePriorities() {
211  $sitemapNamespacesPriorities = $this->getConfig()->get( MainConfigNames::SitemapNamespacesPriorities );
212 
213  // Custom main namespaces
214  $this->priorities[self::GS_MAIN] = '0.5';
215  // Custom talk namesspaces
216  $this->priorities[self::GS_TALK] = '0.1';
217  // MediaWiki standard namespaces
218  $this->priorities[NS_MAIN] = '1.0';
219  $this->priorities[NS_TALK] = '0.1';
220  $this->priorities[NS_USER] = '0.5';
221  $this->priorities[NS_USER_TALK] = '0.1';
222  $this->priorities[NS_PROJECT] = '0.5';
223  $this->priorities[NS_PROJECT_TALK] = '0.1';
224  $this->priorities[NS_FILE] = '0.5';
225  $this->priorities[NS_FILE_TALK] = '0.1';
226  $this->priorities[NS_MEDIAWIKI] = '0.0';
227  $this->priorities[NS_MEDIAWIKI_TALK] = '0.1';
228  $this->priorities[NS_TEMPLATE] = '0.0';
229  $this->priorities[NS_TEMPLATE_TALK] = '0.1';
230  $this->priorities[NS_HELP] = '0.5';
231  $this->priorities[NS_HELP_TALK] = '0.1';
232  $this->priorities[NS_CATEGORY] = '0.5';
233  $this->priorities[NS_CATEGORY_TALK] = '0.1';
234 
235  // Custom priorities
236  if ( $sitemapNamespacesPriorities !== false ) {
240  foreach ( $sitemapNamespacesPriorities as $namespace => $priority ) {
241  $float = floatval( $priority );
242  if ( $float > 1.0 ) {
243  $priority = '1.0';
244  } elseif ( $float < 0.0 ) {
245  $priority = '0.0';
246  }
247  $this->priorities[$namespace] = $priority;
248  }
249  }
250  }
251 
255  private function generateNamespaces() {
256  // Only generate for specific namespaces if $wgSitemapNamespaces is an array.
257  $sitemapNamespaces = $this->getConfig()->get( MainConfigNames::SitemapNamespaces );
258  if ( is_array( $sitemapNamespaces ) ) {
259  $this->namespaces = $sitemapNamespaces;
260 
261  return;
262  }
263 
264  $res = $this->dbr->select( 'page',
265  [ 'page_namespace' ],
266  [],
267  __METHOD__,
268  [
269  'GROUP BY' => 'page_namespace',
270  'ORDER BY' => 'page_namespace',
271  ]
272  );
273 
274  foreach ( $res as $row ) {
275  $this->namespaces[] = $row->page_namespace;
276  }
277  }
278 
285  private function priority( $namespace ) {
286  return $this->priorities[$namespace] ?? $this->guessPriority( $namespace );
287  }
288 
297  private function guessPriority( $namespace ) {
298  return MediaWikiServices::getInstance()->getNamespaceInfo()->isSubject( $namespace )
299  ? $this->priorities[self::GS_MAIN]
300  : $this->priorities[self::GS_TALK];
301  }
302 
309  private function getPageRes( $namespace ) {
310  return $this->dbr->select(
311  [ 'page', 'page_props' ],
312  [
313  'page_namespace',
314  'page_title',
315  'page_touched',
316  'page_is_redirect',
317  'pp_propname',
318  ],
319  [ 'page_namespace' => $namespace ],
320  __METHOD__,
321  [],
322  [
323  'page_props' => [
324  'LEFT JOIN',
325  [
326  'page_id = pp_page',
327  'pp_propname' => 'noindex'
328  ]
329  ]
330  ]
331  );
332  }
333 
337  public function main() {
338  $services = MediaWikiServices::getInstance();
339  $contLang = $services->getContentLanguage();
340  $langConverter = $services->getLanguageConverterFactory()->getLanguageConverter( $contLang );
341 
342  fwrite( $this->findex, $this->openIndex() );
343 
344  foreach ( $this->namespaces as $namespace ) {
345  $res = $this->getPageRes( $namespace );
346  $this->file = false;
347  $this->generateLimit( $namespace );
348  $length = $this->limit[0];
349  $i = $smcount = 0;
350 
351  $fns = $contLang->getFormattedNsText( $namespace );
352  $this->output( "$namespace ($fns)\n" );
353  $skippedRedirects = 0; // Number of redirects skipped for that namespace
354  $skippedNoindex = 0; // Number of pages with __NOINDEX__ switch for that NS
355  foreach ( $res as $row ) {
356  if ( $row->pp_propname === 'noindex' ) {
357  $skippedNoindex++;
358  continue;
359  }
360 
361  if ( $this->skipRedirects && $row->page_is_redirect ) {
362  $skippedRedirects++;
363  continue;
364  }
365 
366  if ( $i++ === 0
367  || $i === $this->url_limit + 1
368  || $length + $this->limit[1] + $this->limit[2] > $this->size_limit
369  ) {
370  if ( $this->file !== false ) {
371  $this->write( $this->file, $this->closeFile() );
372  $this->close( $this->file );
373  }
374  $filename = $this->sitemapFilename( $namespace, $smcount++ );
375  $this->file = $this->open( $this->fspath . $filename, 'wb' );
376  $this->write( $this->file, $this->openFile() );
377  fwrite( $this->findex, $this->indexEntry( $filename ) );
378  $this->output( "\t$this->fspath$filename\n" );
379  $length = $this->limit[0];
380  $i = 1;
381  }
382  $title = Title::makeTitle( $row->page_namespace, $row->page_title );
383  $date = wfTimestamp( TS_ISO_8601, $row->page_touched );
384  $entry = $this->fileEntry( $title->getCanonicalURL(), $date, $this->priority( $namespace ) );
385  $length += strlen( $entry );
386  $this->write( $this->file, $entry );
387  // generate pages for language variants
388  if ( $langConverter->hasVariants() ) {
389  $variants = $langConverter->getVariants();
390  foreach ( $variants as $vCode ) {
391  if ( $vCode == $contLang->getCode() ) {
392  continue; // we don't want default variant
393  }
394  $entry = $this->fileEntry(
395  $title->getCanonicalURL( '', $vCode ),
396  $date,
397  $this->priority( $namespace )
398  );
399  $length += strlen( $entry );
400  $this->write( $this->file, $entry );
401  }
402  }
403  }
404 
405  if ( $skippedNoindex > 0 ) {
406  $this->output( " skipped $skippedNoindex page(s) with __NOINDEX__ switch\n" );
407  }
408 
409  if ( $this->skipRedirects && $skippedRedirects > 0 ) {
410  $this->output( " skipped $skippedRedirects redirect(s)\n" );
411  }
412 
413  if ( $this->file ) {
414  $this->write( $this->file, $this->closeFile() );
415  $this->close( $this->file );
416  }
417  }
418  fwrite( $this->findex, $this->closeIndex() );
419  fclose( $this->findex );
420  }
421 
429  private function open( $file, $flags ) {
430  $resource = $this->compress ? gzopen( $file, $flags ) : fopen( $file, $flags );
431  if ( $resource === false ) {
432  throw new MWException( __METHOD__
433  . " error opening file $file with flags $flags. Check permissions?" );
434  }
435 
436  return $resource;
437  }
438 
445  private function write( &$handle, $str ) {
446  if ( $handle === true || $handle === false ) {
447  throw new MWException( __METHOD__ . " was passed a boolean as a file handle.\n" );
448  }
449  if ( $this->compress ) {
450  gzwrite( $handle, $str );
451  } else {
452  fwrite( $handle, $str );
453  }
454  }
455 
461  private function close( &$handle ) {
462  if ( $this->compress ) {
463  gzclose( $handle );
464  } else {
465  fclose( $handle );
466  }
467  }
468 
476  private function sitemapFilename( $namespace, $count ) {
477  $ext = $this->compress ? '.gz' : '';
478 
479  return "sitemap-{$this->identifier}-NS_$namespace-$count.xml$ext";
480  }
481 
487  private function xmlHead() {
488  return '<?xml version="1.0" encoding="UTF-8"?>' . "\n";
489  }
490 
496  private function xmlSchema() {
497  return 'http://www.sitemaps.org/schemas/sitemap/0.9';
498  }
499 
505  private function openIndex() {
506  return $this->xmlHead() . '<sitemapindex xmlns="' . $this->xmlSchema() . '">' . "\n";
507  }
508 
515  private function indexEntry( $filename ) {
516  return "\t<sitemap>\n" .
517  "\t\t<loc>" . wfGetServerUrl( PROTO_CANONICAL ) .
518  ( substr( $this->urlpath, 0, 1 ) === "/" ? "" : "/" ) .
519  "{$this->urlpath}$filename</loc>\n" .
520  "\t\t<lastmod>{$this->timestamp}</lastmod>\n" .
521  "\t</sitemap>\n";
522  }
523 
529  private function closeIndex() {
530  return "</sitemapindex>\n";
531  }
532 
538  private function openFile() {
539  return $this->xmlHead() . '<urlset xmlns="' . $this->xmlSchema() . '">' . "\n";
540  }
541 
550  private function fileEntry( $url, $date, $priority ) {
551  return "\t<url>\n" .
552  // T36666: $url may contain bad characters such as ampersands.
553  "\t\t<loc>" . htmlspecialchars( $url ) . "</loc>\n" .
554  "\t\t<lastmod>$date</lastmod>\n" .
555  "\t\t<priority>$priority</priority>\n" .
556  "\t</url>\n";
557  }
558 
564  private function closeFile() {
565  return "</urlset>\n";
566  }
567 
573  private function generateLimit( $namespace ) {
574  // T19961: make a title with the longest possible URL in this namespace
575  $title = Title::makeTitle( $namespace, str_repeat( "\u{28B81}", 63 ) . "\u{5583}" );
576 
577  $this->limit = [
578  strlen( $this->openFile() ),
579  strlen( $this->fileEntry(
580  $title->getCanonicalURL(),
581  wfTimestamp( TS_ISO_8601, wfTimestamp() ),
582  $this->priority( $namespace )
583  ) ),
584  strlen( $this->closeFile() )
585  ];
586  }
587 }
588 
589 $maintClass = GenerateSitemap::class;
590 require_once RUN_MAINTENANCE_IF_MAIN;
const PROTO_CANONICAL
Definition: Defines.php:199
const NS_HELP
Definition: Defines.php:76
const NS_USER
Definition: Defines.php:66
const NS_FILE
Definition: Defines.php:70
const NS_MEDIAWIKI_TALK
Definition: Defines.php:73
const NS_MAIN
Definition: Defines.php:64
const NS_PROJECT_TALK
Definition: Defines.php:69
const NS_MEDIAWIKI
Definition: Defines.php:72
const NS_TEMPLATE
Definition: Defines.php:74
const NS_FILE_TALK
Definition: Defines.php:71
const NS_HELP_TALK
Definition: Defines.php:77
const NS_CATEGORY_TALK
Definition: Defines.php:79
const NS_TALK
Definition: Defines.php:65
const NS_USER_TALK
Definition: Defines.php:67
const NS_PROJECT
Definition: Defines.php:68
const NS_CATEGORY
Definition: Defines.php:78
const NS_TEMPLATE_TALK
Definition: Defines.php:75
wfTimestampNow()
Convenience function; returns MediaWiki timestamp for the present time.
wfGetServerUrl( $proto)
Get the wiki's "server", i.e.
wfMkdirParents( $dir, $mode=null, $caller=null)
Make directory, and make all parent directories if they don't exist.
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Maintenance script that generates a sitemap for the site.
string $timestamp
When this sitemap batch was generated.
IDatabase $dbr
A database replica DB object.
string $fspath
The path to prepend to the filename.
array $priorities
Key => value entries of namespaces and their priorities.
bool $skipRedirects
Whether or not to include redirection pages.
__construct()
Default constructor.
array $limit
The number of entries to save in each sitemap file.
int $size_limit
The maximum size of a sitemap file.
bool $compress
Whether or not to use compression.
array $namespaces
A one-dimensional array of namespaces in the wiki.
string $urlpath
The URL path to prepend to filenames in the index; should resolve to the same directory as $fspath.
resource $findex
A resource pointing to the sitemap index file.
int $url_limit
The maximum amount of urls in a sitemap file.
resource false $file
A resource pointing to a sitemap file.
MediaWiki exception.
Definition: MWException.php:29
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:66
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
output( $out, $channel=null)
Throw some output to the user.
hasOption( $name)
Checks to see if a particular option was set.
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
A class containing constants representing the names of configuration variables.
Service locator for MediaWiki core services.
static makeTitle( $ns, $title, $fragment='', $interwiki='')
Create a new Title from a namespace index and a DB key.
Definition: Title.php:638
static getCurrentWikiDbDomain()
Definition: WikiMap.php:293
$maintClass
Basic database interface for live and lazy-loaded relation database handles.
Definition: IDatabase.php:39
Result wrapper for grabbing data queried from an IDatabase object.
const DB_REPLICA
Definition: defines.php:26
if(!is_readable( $file)) $ext
Definition: router.php:48