MediaWiki  master
generateSitemap.php
Go to the documentation of this file.
1 <?php
35 
36 require_once __DIR__ . '/Maintenance.php';
37 
44  private const GS_MAIN = -2;
45  private const GS_TALK = -1;
46 
54  public $url_limit;
55 
63  public $size_limit;
64 
70  public $fspath;
71 
78  public $urlpath;
79 
85  public $compress;
86 
93 
99  public $limit = [];
100 
106  public $priorities = [];
107 
113  public $namespaces = [];
114 
120  public $timestamp;
121 
127  public $dbr;
128 
134  public $findex;
135 
141  public $file;
142 
148  private $identifier;
149 
150  public function __construct() {
151  parent::__construct();
152  $this->addDescription( 'Creates a sitemap for the site' );
153  $this->addOption(
154  'fspath',
155  'The file system path to save to, e.g. /tmp/sitemap; defaults to current directory',
156  false,
157  true
158  );
159  $this->addOption(
160  'urlpath',
161  'The URL path corresponding to --fspath, prepended to filenames in the index; '
162  . 'defaults to an empty string',
163  false,
164  true
165  );
166  $this->addOption(
167  'compress',
168  'Compress the sitemap files, can take value yes|no, default yes',
169  false,
170  true
171  );
172  $this->addOption( 'skip-redirects', 'Do not include redirecting articles in the sitemap' );
173  $this->addOption(
174  'identifier',
175  'What site identifier to use for the wiki, defaults to $wgDBname',
176  false,
177  true
178  );
179  }
180 
184  public function execute() {
185  $this->setNamespacePriorities();
186  $this->url_limit = 50000;
187  $this->size_limit = ( 2 ** 20 ) * 10;
188 
189  # Create directory if needed
190  $fspath = $this->getOption( 'fspath', getcwd() );
191  if ( !wfMkdirParents( $fspath, null, __METHOD__ ) ) {
192  $this->fatalError( "Can not create directory $fspath." );
193  }
194 
195  $dbDomain = WikiMap::getCurrentWikiDbDomain()->getId();
196  $this->fspath = realpath( $fspath ) . DIRECTORY_SEPARATOR;
197  $this->urlpath = $this->getOption( 'urlpath', "" );
198  if ( $this->urlpath !== "" && substr( $this->urlpath, -1 ) !== '/' ) {
199  $this->urlpath .= '/';
200  }
201  $this->identifier = $this->getOption( 'identifier', $dbDomain );
202  $this->compress = $this->getOption( 'compress', 'yes' ) !== 'no';
203  $this->skipRedirects = $this->hasOption( 'skip-redirects' );
204  $this->dbr = $this->getDB( DB_REPLICA );
205  $this->generateNamespaces();
206  $this->timestamp = wfTimestamp( TS_ISO_8601, wfTimestampNow() );
207  $encIdentifier = rawurlencode( $this->identifier );
208  $this->findex = fopen( "{$this->fspath}sitemap-index-{$encIdentifier}.xml", 'wb' );
209  $this->main();
210  }
211 
212  private function setNamespacePriorities() {
213  $sitemapNamespacesPriorities = $this->getConfig()->get( MainConfigNames::SitemapNamespacesPriorities );
214 
215  // Custom main namespaces
216  $this->priorities[self::GS_MAIN] = '0.5';
217  // Custom talk namesspaces
218  $this->priorities[self::GS_TALK] = '0.1';
219  // MediaWiki standard namespaces
220  $this->priorities[NS_MAIN] = '1.0';
221  $this->priorities[NS_TALK] = '0.1';
222  $this->priorities[NS_USER] = '0.5';
223  $this->priorities[NS_USER_TALK] = '0.1';
224  $this->priorities[NS_PROJECT] = '0.5';
225  $this->priorities[NS_PROJECT_TALK] = '0.1';
226  $this->priorities[NS_FILE] = '0.5';
227  $this->priorities[NS_FILE_TALK] = '0.1';
228  $this->priorities[NS_MEDIAWIKI] = '0.0';
229  $this->priorities[NS_MEDIAWIKI_TALK] = '0.1';
230  $this->priorities[NS_TEMPLATE] = '0.0';
231  $this->priorities[NS_TEMPLATE_TALK] = '0.1';
232  $this->priorities[NS_HELP] = '0.5';
233  $this->priorities[NS_HELP_TALK] = '0.1';
234  $this->priorities[NS_CATEGORY] = '0.5';
235  $this->priorities[NS_CATEGORY_TALK] = '0.1';
236 
237  // Custom priorities
238  if ( $sitemapNamespacesPriorities !== false ) {
242  foreach ( $sitemapNamespacesPriorities as $namespace => $priority ) {
243  $float = floatval( $priority );
244  if ( $float > 1.0 ) {
245  $priority = '1.0';
246  } elseif ( $float < 0.0 ) {
247  $priority = '0.0';
248  }
249  $this->priorities[$namespace] = $priority;
250  }
251  }
252  }
253 
257  private function generateNamespaces() {
258  // Only generate for specific namespaces if $wgSitemapNamespaces is an array.
259  $sitemapNamespaces = $this->getConfig()->get( MainConfigNames::SitemapNamespaces );
260  if ( is_array( $sitemapNamespaces ) ) {
261  $this->namespaces = $sitemapNamespaces;
262 
263  return;
264  }
265 
266  $res = $this->dbr->select( 'page',
267  [ 'page_namespace' ],
268  [],
269  __METHOD__,
270  [
271  'GROUP BY' => 'page_namespace',
272  'ORDER BY' => 'page_namespace',
273  ]
274  );
275 
276  foreach ( $res as $row ) {
277  $this->namespaces[] = $row->page_namespace;
278  }
279  }
280 
287  private function priority( $namespace ) {
288  return $this->priorities[$namespace] ?? $this->guessPriority( $namespace );
289  }
290 
299  private function guessPriority( $namespace ) {
300  return MediaWikiServices::getInstance()->getNamespaceInfo()->isSubject( $namespace )
301  ? $this->priorities[self::GS_MAIN]
302  : $this->priorities[self::GS_TALK];
303  }
304 
311  private function getPageRes( $namespace ) {
312  return $this->dbr->select(
313  [ 'page', 'page_props' ],
314  [
315  'page_namespace',
316  'page_title',
317  'page_touched',
318  'page_is_redirect',
319  'pp_propname',
320  ],
321  [ 'page_namespace' => $namespace ],
322  __METHOD__,
323  [],
324  [
325  'page_props' => [
326  'LEFT JOIN',
327  [
328  'page_id = pp_page',
329  'pp_propname' => 'noindex'
330  ]
331  ]
332  ]
333  );
334  }
335 
339  public function main() {
340  $services = MediaWikiServices::getInstance();
341  $contLang = $services->getContentLanguage();
342  $langConverter = $services->getLanguageConverterFactory()->getLanguageConverter( $contLang );
343 
344  fwrite( $this->findex, $this->openIndex() );
345 
346  foreach ( $this->namespaces as $namespace ) {
347  $res = $this->getPageRes( $namespace );
348  $this->file = false;
349  $this->generateLimit( $namespace );
350  $length = $this->limit[0];
351  $i = $smcount = 0;
352 
353  $fns = $contLang->getFormattedNsText( $namespace );
354  $this->output( "$namespace ($fns)\n" );
355  $skippedRedirects = 0; // Number of redirects skipped for that namespace
356  $skippedNoindex = 0; // Number of pages with __NOINDEX__ switch for that NS
357  foreach ( $res as $row ) {
358  if ( $row->pp_propname === 'noindex' ) {
359  $skippedNoindex++;
360  continue;
361  }
362 
363  if ( $this->skipRedirects && $row->page_is_redirect ) {
364  $skippedRedirects++;
365  continue;
366  }
367 
368  if ( $i++ === 0
369  || $i === $this->url_limit + 1
370  || $length + $this->limit[1] + $this->limit[2] > $this->size_limit
371  ) {
372  if ( $this->file !== false ) {
373  $this->write( $this->file, $this->closeFile() );
374  $this->close( $this->file );
375  }
376  $filename = $this->sitemapFilename( $namespace, $smcount++ );
377  $this->file = $this->open( $this->fspath . $filename, 'wb' );
378  $this->write( $this->file, $this->openFile() );
379  fwrite( $this->findex, $this->indexEntry( $filename ) );
380  $this->output( "\t$this->fspath$filename\n" );
381  $length = $this->limit[0];
382  $i = 1;
383  }
384  $title = Title::makeTitle( $row->page_namespace, $row->page_title );
385  $date = wfTimestamp( TS_ISO_8601, $row->page_touched );
386  $entry = $this->fileEntry( $title->getCanonicalURL(), $date, $this->priority( $namespace ) );
387  $length += strlen( $entry );
388  $this->write( $this->file, $entry );
389  // generate pages for language variants
390  if ( $langConverter->hasVariants() ) {
391  $variants = $langConverter->getVariants();
392  foreach ( $variants as $vCode ) {
393  if ( $vCode == $contLang->getCode() ) {
394  continue; // we don't want default variant
395  }
396  $entry = $this->fileEntry(
397  $title->getCanonicalURL( '', $vCode ),
398  $date,
399  $this->priority( $namespace )
400  );
401  $length += strlen( $entry );
402  $this->write( $this->file, $entry );
403  }
404  }
405  }
406 
407  if ( $skippedNoindex > 0 ) {
408  $this->output( " skipped $skippedNoindex page(s) with __NOINDEX__ switch\n" );
409  }
410 
411  if ( $this->skipRedirects && $skippedRedirects > 0 ) {
412  $this->output( " skipped $skippedRedirects redirect(s)\n" );
413  }
414 
415  if ( $this->file ) {
416  $this->write( $this->file, $this->closeFile() );
417  $this->close( $this->file );
418  }
419  }
420  fwrite( $this->findex, $this->closeIndex() );
421  fclose( $this->findex );
422  }
423 
431  private function open( $file, $flags ) {
432  $resource = $this->compress ? gzopen( $file, $flags ) : fopen( $file, $flags );
433  if ( $resource === false ) {
434  throw new RuntimeException( __METHOD__
435  . " error opening file $file with flags $flags. Check permissions?" );
436  }
437 
438  return $resource;
439  }
440 
447  private function write( &$handle, $str ) {
448  if ( $handle === true || $handle === false ) {
449  throw new InvalidArgumentException( __METHOD__ . " was passed a boolean as a file handle.\n" );
450  }
451  if ( $this->compress ) {
452  gzwrite( $handle, $str );
453  } else {
454  fwrite( $handle, $str );
455  }
456  }
457 
463  private function close( &$handle ) {
464  if ( $this->compress ) {
465  gzclose( $handle );
466  } else {
467  fclose( $handle );
468  }
469  }
470 
478  private function sitemapFilename( $namespace, $count ) {
479  $ext = $this->compress ? '.gz' : '';
480 
481  return "sitemap-{$this->identifier}-NS_$namespace-$count.xml$ext";
482  }
483 
489  private function xmlHead() {
490  return '<?xml version="1.0" encoding="UTF-8"?>' . "\n";
491  }
492 
498  private function xmlSchema() {
499  return 'http://www.sitemaps.org/schemas/sitemap/0.9';
500  }
501 
507  private function openIndex() {
508  return $this->xmlHead() . '<sitemapindex xmlns="' . $this->xmlSchema() . '">' . "\n";
509  }
510 
517  private function indexEntry( $filename ) {
518  return "\t<sitemap>\n" .
519  "\t\t<loc>" . wfGetServerUrl( PROTO_CANONICAL ) .
520  ( substr( $this->urlpath, 0, 1 ) === "/" ? "" : "/" ) .
521  "{$this->urlpath}$filename</loc>\n" .
522  "\t\t<lastmod>{$this->timestamp}</lastmod>\n" .
523  "\t</sitemap>\n";
524  }
525 
531  private function closeIndex() {
532  return "</sitemapindex>\n";
533  }
534 
540  private function openFile() {
541  return $this->xmlHead() . '<urlset xmlns="' . $this->xmlSchema() . '">' . "\n";
542  }
543 
552  private function fileEntry( $url, $date, $priority ) {
553  return "\t<url>\n" .
554  // T36666: $url may contain bad characters such as ampersands.
555  "\t\t<loc>" . htmlspecialchars( $url ) . "</loc>\n" .
556  "\t\t<lastmod>$date</lastmod>\n" .
557  "\t\t<priority>$priority</priority>\n" .
558  "\t</url>\n";
559  }
560 
566  private function closeFile() {
567  return "</urlset>\n";
568  }
569 
575  private function generateLimit( $namespace ) {
576  // T19961: make a title with the longest possible URL in this namespace
577  $title = Title::makeTitle( $namespace, str_repeat( "\u{28B81}", 63 ) . "\u{5583}" );
578 
579  $this->limit = [
580  strlen( $this->openFile() ),
581  strlen( $this->fileEntry(
582  $title->getCanonicalURL(),
583  wfTimestamp( TS_ISO_8601, wfTimestamp() ),
584  $this->priority( $namespace )
585  ) ),
586  strlen( $this->closeFile() )
587  ];
588  }
589 }
590 
591 $maintClass = GenerateSitemap::class;
592 require_once RUN_MAINTENANCE_IF_MAIN;
const PROTO_CANONICAL
Definition: Defines.php:199
const NS_HELP
Definition: Defines.php:76
const NS_USER
Definition: Defines.php:66
const NS_FILE
Definition: Defines.php:70
const NS_MEDIAWIKI_TALK
Definition: Defines.php:73
const NS_MAIN
Definition: Defines.php:64
const NS_PROJECT_TALK
Definition: Defines.php:69
const NS_MEDIAWIKI
Definition: Defines.php:72
const NS_TEMPLATE
Definition: Defines.php:74
const NS_FILE_TALK
Definition: Defines.php:71
const NS_HELP_TALK
Definition: Defines.php:77
const NS_CATEGORY_TALK
Definition: Defines.php:79
const NS_TALK
Definition: Defines.php:65
const NS_USER_TALK
Definition: Defines.php:67
const NS_PROJECT
Definition: Defines.php:68
const NS_CATEGORY
Definition: Defines.php:78
const NS_TEMPLATE_TALK
Definition: Defines.php:75
wfTimestampNow()
Convenience function; returns MediaWiki timestamp for the present time.
wfGetServerUrl( $proto)
Get the wiki's "server", i.e.
wfMkdirParents( $dir, $mode=null, $caller=null)
Make directory, and make all parent directories if they don't exist.
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Maintenance script that generates a sitemap for the site.
string $timestamp
When this sitemap batch was generated.
IDatabase $dbr
A database replica DB object.
string $fspath
The path to prepend to the filename.
array $priorities
Key => value entries of namespaces and their priorities.
bool $skipRedirects
Whether or not to include redirection pages.
__construct()
Default constructor.
array $limit
The number of entries to save in each sitemap file.
int $size_limit
The maximum size of a sitemap file.
bool $compress
Whether or not to use compression.
array $namespaces
A one-dimensional array of namespaces in the wiki.
string $urlpath
The URL path to prepend to filenames in the index; should resolve to the same directory as $fspath.
resource $findex
A resource pointing to the sitemap index file.
int $url_limit
The maximum amount of urls in a sitemap file.
resource false $file
A resource pointing to a sitemap file.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:66
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
output( $out, $channel=null)
Throw some output to the user.
hasOption( $name)
Checks to see if a particular option was set.
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
A class containing constants representing the names of configuration variables.
Service locator for MediaWiki core services.
Represents a title within MediaWiki.
Definition: Title.php:82
Helper tools for dealing with other locally-hosted wikis.
Definition: WikiMap.php:33
$maintClass
Basic database interface for live and lazy-loaded relation database handles.
Definition: IDatabase.php:36
Result wrapper for grabbing data queried from an IDatabase object.
const DB_REPLICA
Definition: defines.php:26
if(!is_readable( $file)) $ext
Definition: router.php:48