MediaWiki  master
generateSitemap.php
Go to the documentation of this file.
1 <?php
34 
35 require_once __DIR__ . '/Maintenance.php';
36 
43  private const GS_MAIN = -2;
44  private const GS_TALK = -1;
45 
53  public $url_limit;
54 
62  public $size_limit;
63 
69  public $fspath;
70 
77  public $urlpath;
78 
84  public $compress;
85 
92 
98  public $limit = [];
99 
105  public $priorities = [];
106 
112  public $namespaces = [];
113 
119  public $timestamp;
120 
126  public $dbr;
127 
133  public $findex;
134 
140  public $file;
141 
147  private $identifier;
148 
149  public function __construct() {
150  parent::__construct();
151  $this->addDescription( 'Creates a sitemap for the site' );
152  $this->addOption(
153  'fspath',
154  'The file system path to save to, e.g. /tmp/sitemap; defaults to current directory',
155  false,
156  true
157  );
158  $this->addOption(
159  'urlpath',
160  'The URL path corresponding to --fspath, prepended to filenames in the index; '
161  . 'defaults to an empty string',
162  false,
163  true
164  );
165  $this->addOption(
166  'compress',
167  'Compress the sitemap files, can take value yes|no, default yes',
168  false,
169  true
170  );
171  $this->addOption( 'skip-redirects', 'Do not include redirecting articles in the sitemap' );
172  $this->addOption(
173  'identifier',
174  'What site identifier to use for the wiki, defaults to $wgDBname',
175  false,
176  true
177  );
178  }
179 
183  public function execute() {
184  $this->setNamespacePriorities();
185  $this->url_limit = 50000;
186  $this->size_limit = ( 2 ** 20 ) * 10;
187 
188  # Create directory if needed
189  $fspath = $this->getOption( 'fspath', getcwd() );
190  if ( !wfMkdirParents( $fspath, null, __METHOD__ ) ) {
191  $this->fatalError( "Can not create directory $fspath." );
192  }
193 
194  $dbDomain = WikiMap::getCurrentWikiDbDomain()->getId();
195  $this->fspath = realpath( $fspath ) . DIRECTORY_SEPARATOR;
196  $this->urlpath = $this->getOption( 'urlpath', "" );
197  if ( $this->urlpath !== "" && $this->urlpath[-1] !== '/' ) {
198  $this->urlpath .= '/';
199  }
200  $this->identifier = $this->getOption( 'identifier', $dbDomain );
201  $this->compress = $this->getOption( 'compress', 'yes' ) !== 'no';
202  $this->skipRedirects = $this->hasOption( 'skip-redirects' );
203  $this->dbr = $this->getDB( DB_REPLICA );
204  $this->generateNamespaces();
205  $this->timestamp = wfTimestamp( TS_ISO_8601, wfTimestampNow() );
206  $encIdentifier = rawurlencode( $this->identifier );
207  $this->findex = fopen( "{$this->fspath}sitemap-index-{$encIdentifier}.xml", 'wb' );
208  $this->main();
209  }
210 
211  private function setNamespacePriorities() {
212  $sitemapNamespacesPriorities = $this->getConfig()->get( MainConfigNames::SitemapNamespacesPriorities );
213 
214  // Custom main namespaces
215  $this->priorities[self::GS_MAIN] = '0.5';
216  // Custom talk namesspaces
217  $this->priorities[self::GS_TALK] = '0.1';
218  // MediaWiki standard namespaces
219  $this->priorities[NS_MAIN] = '1.0';
220  $this->priorities[NS_TALK] = '0.1';
221  $this->priorities[NS_USER] = '0.5';
222  $this->priorities[NS_USER_TALK] = '0.1';
223  $this->priorities[NS_PROJECT] = '0.5';
224  $this->priorities[NS_PROJECT_TALK] = '0.1';
225  $this->priorities[NS_FILE] = '0.5';
226  $this->priorities[NS_FILE_TALK] = '0.1';
227  $this->priorities[NS_MEDIAWIKI] = '0.0';
228  $this->priorities[NS_MEDIAWIKI_TALK] = '0.1';
229  $this->priorities[NS_TEMPLATE] = '0.0';
230  $this->priorities[NS_TEMPLATE_TALK] = '0.1';
231  $this->priorities[NS_HELP] = '0.5';
232  $this->priorities[NS_HELP_TALK] = '0.1';
233  $this->priorities[NS_CATEGORY] = '0.5';
234  $this->priorities[NS_CATEGORY_TALK] = '0.1';
235 
236  // Custom priorities
237  if ( $sitemapNamespacesPriorities !== false ) {
241  foreach ( $sitemapNamespacesPriorities as $namespace => $priority ) {
242  $float = floatval( $priority );
243  if ( $float > 1.0 ) {
244  $priority = '1.0';
245  } elseif ( $float < 0.0 ) {
246  $priority = '0.0';
247  }
248  $this->priorities[$namespace] = $priority;
249  }
250  }
251  }
252 
256  private function generateNamespaces() {
257  // Only generate for specific namespaces if $wgSitemapNamespaces is an array.
258  $sitemapNamespaces = $this->getConfig()->get( MainConfigNames::SitemapNamespaces );
259  if ( is_array( $sitemapNamespaces ) ) {
260  $this->namespaces = $sitemapNamespaces;
261 
262  return;
263  }
264 
265  $res = $this->dbr->newSelectQueryBuilder()
266  ->select( [ 'page_namespace' ] )
267  ->from( 'page' )
268  ->groupBy( 'page_namespace' )
269  ->orderBy( 'page_namespace' )
270  ->caller( __METHOD__ )->fetchResultSet();
271 
272  foreach ( $res as $row ) {
273  $this->namespaces[] = $row->page_namespace;
274  }
275  }
276 
283  private function priority( $namespace ) {
284  return $this->priorities[$namespace] ?? $this->guessPriority( $namespace );
285  }
286 
295  private function guessPriority( $namespace ) {
296  return $this->getServiceContainer()->getNamespaceInfo()->isSubject( $namespace )
297  ? $this->priorities[self::GS_MAIN]
298  : $this->priorities[self::GS_TALK];
299  }
300 
307  private function getPageRes( $namespace ) {
308  return $this->dbr->newSelectQueryBuilder()
309  ->select( [ 'page_namespace', 'page_title', 'page_touched', 'page_is_redirect', 'pp_propname' ] )
310  ->from( 'page' )
311  ->leftJoin( 'page_props', null, [ 'page_id = pp_page', 'pp_propname' => 'noindex' ] )
312  ->where( [ 'page_namespace' => $namespace ] )
313  ->caller( __METHOD__ )->fetchResultSet();
314  }
315 
319  public function main() {
320  $services = $this->getServiceContainer();
321  $contLang = $services->getContentLanguage();
322  $langConverter = $services->getLanguageConverterFactory()->getLanguageConverter( $contLang );
323 
324  fwrite( $this->findex, $this->openIndex() );
325 
326  foreach ( $this->namespaces as $namespace ) {
327  $res = $this->getPageRes( $namespace );
328  $this->file = false;
329  $this->generateLimit( $namespace );
330  $length = $this->limit[0];
331  $i = $smcount = 0;
332 
333  $fns = $contLang->getFormattedNsText( $namespace );
334  $this->output( "$namespace ($fns)\n" );
335  $skippedRedirects = 0; // Number of redirects skipped for that namespace
336  $skippedNoindex = 0; // Number of pages with __NOINDEX__ switch for that NS
337  foreach ( $res as $row ) {
338  if ( $row->pp_propname === 'noindex' ) {
339  $skippedNoindex++;
340  continue;
341  }
342 
343  if ( $this->skipRedirects && $row->page_is_redirect ) {
344  $skippedRedirects++;
345  continue;
346  }
347 
348  if ( $i++ === 0
349  || $i === $this->url_limit + 1
350  || $length + $this->limit[1] + $this->limit[2] > $this->size_limit
351  ) {
352  if ( $this->file !== false ) {
353  $this->write( $this->file, $this->closeFile() );
354  $this->close( $this->file );
355  }
356  $filename = $this->sitemapFilename( $namespace, $smcount++ );
357  $this->file = $this->open( $this->fspath . $filename, 'wb' );
358  $this->write( $this->file, $this->openFile() );
359  fwrite( $this->findex, $this->indexEntry( $filename ) );
360  $this->output( "\t$this->fspath$filename\n" );
361  $length = $this->limit[0];
362  $i = 1;
363  }
364  $title = Title::makeTitle( $row->page_namespace, $row->page_title );
365  $date = wfTimestamp( TS_ISO_8601, $row->page_touched );
366  $entry = $this->fileEntry( $title->getCanonicalURL(), $date, $this->priority( $namespace ) );
367  $length += strlen( $entry );
368  $this->write( $this->file, $entry );
369  // generate pages for language variants
370  if ( $langConverter->hasVariants() ) {
371  $variants = $langConverter->getVariants();
372  foreach ( $variants as $vCode ) {
373  if ( $vCode == $contLang->getCode() ) {
374  continue; // we don't want default variant
375  }
376  $entry = $this->fileEntry(
377  $title->getCanonicalURL( [ 'variant' => $vCode ] ),
378  $date,
379  $this->priority( $namespace )
380  );
381  $length += strlen( $entry );
382  $this->write( $this->file, $entry );
383  }
384  }
385  }
386 
387  if ( $skippedNoindex > 0 ) {
388  $this->output( " skipped $skippedNoindex page(s) with __NOINDEX__ switch\n" );
389  }
390 
391  if ( $this->skipRedirects && $skippedRedirects > 0 ) {
392  $this->output( " skipped $skippedRedirects redirect(s)\n" );
393  }
394 
395  if ( $this->file ) {
396  $this->write( $this->file, $this->closeFile() );
397  $this->close( $this->file );
398  }
399  }
400  fwrite( $this->findex, $this->closeIndex() );
401  fclose( $this->findex );
402  }
403 
411  private function open( $file, $flags ) {
412  $resource = $this->compress ? gzopen( $file, $flags ) : fopen( $file, $flags );
413  if ( $resource === false ) {
414  throw new RuntimeException( __METHOD__
415  . " error opening file $file with flags $flags. Check permissions?" );
416  }
417 
418  return $resource;
419  }
420 
427  private function write( &$handle, $str ) {
428  if ( $handle === true || $handle === false ) {
429  throw new InvalidArgumentException( __METHOD__ . " was passed a boolean as a file handle.\n" );
430  }
431  if ( $this->compress ) {
432  gzwrite( $handle, $str );
433  } else {
434  fwrite( $handle, $str );
435  }
436  }
437 
443  private function close( &$handle ) {
444  if ( $this->compress ) {
445  gzclose( $handle );
446  } else {
447  fclose( $handle );
448  }
449  }
450 
458  private function sitemapFilename( $namespace, $count ) {
459  $ext = $this->compress ? '.gz' : '';
460 
461  return "sitemap-{$this->identifier}-NS_$namespace-$count.xml$ext";
462  }
463 
469  private function xmlHead() {
470  return '<?xml version="1.0" encoding="UTF-8"?>' . "\n";
471  }
472 
478  private function xmlSchema() {
479  return 'http://www.sitemaps.org/schemas/sitemap/0.9';
480  }
481 
487  private function openIndex() {
488  return $this->xmlHead() . '<sitemapindex xmlns="' . $this->xmlSchema() . '">' . "\n";
489  }
490 
497  private function indexEntry( $filename ) {
498  return "\t<sitemap>\n" .
499  "\t\t<loc>" . wfGetServerUrl( PROTO_CANONICAL ) .
500  ( $this->urlpath[0] === "/" ? "" : "/" ) .
501  "{$this->urlpath}$filename</loc>\n" .
502  "\t\t<lastmod>{$this->timestamp}</lastmod>\n" .
503  "\t</sitemap>\n";
504  }
505 
511  private function closeIndex() {
512  return "</sitemapindex>\n";
513  }
514 
520  private function openFile() {
521  return $this->xmlHead() . '<urlset xmlns="' . $this->xmlSchema() . '">' . "\n";
522  }
523 
532  private function fileEntry( $url, $date, $priority ) {
533  return "\t<url>\n" .
534  // T36666: $url may contain bad characters such as ampersands.
535  "\t\t<loc>" . htmlspecialchars( $url ) . "</loc>\n" .
536  "\t\t<lastmod>$date</lastmod>\n" .
537  "\t\t<priority>$priority</priority>\n" .
538  "\t</url>\n";
539  }
540 
546  private function closeFile() {
547  return "</urlset>\n";
548  }
549 
555  private function generateLimit( $namespace ) {
556  // T19961: make a title with the longest possible URL in this namespace
557  $title = Title::makeTitle( $namespace, str_repeat( "\u{28B81}", 63 ) . "\u{5583}" );
558 
559  $this->limit = [
560  strlen( $this->openFile() ),
561  strlen( $this->fileEntry(
562  $title->getCanonicalURL(),
563  wfTimestamp( TS_ISO_8601, wfTimestamp() ),
564  $this->priority( $namespace )
565  ) ),
566  strlen( $this->closeFile() )
567  ];
568  }
569 }
570 
571 $maintClass = GenerateSitemap::class;
572 require_once RUN_MAINTENANCE_IF_MAIN;
const PROTO_CANONICAL
Definition: Defines.php:197
const NS_HELP
Definition: Defines.php:76
const NS_USER
Definition: Defines.php:66
const NS_FILE
Definition: Defines.php:70
const NS_MEDIAWIKI_TALK
Definition: Defines.php:73
const NS_MAIN
Definition: Defines.php:64
const NS_PROJECT_TALK
Definition: Defines.php:69
const NS_MEDIAWIKI
Definition: Defines.php:72
const NS_TEMPLATE
Definition: Defines.php:74
const NS_FILE_TALK
Definition: Defines.php:71
const NS_HELP_TALK
Definition: Defines.php:77
const NS_CATEGORY_TALK
Definition: Defines.php:79
const NS_TALK
Definition: Defines.php:65
const NS_USER_TALK
Definition: Defines.php:67
const NS_PROJECT
Definition: Defines.php:68
const NS_CATEGORY
Definition: Defines.php:78
const NS_TEMPLATE_TALK
Definition: Defines.php:75
wfTimestampNow()
Convenience function; returns MediaWiki timestamp for the present time.
wfGetServerUrl( $proto)
Get the wiki's "server", i.e.
wfMkdirParents( $dir, $mode=null, $caller=null)
Make directory, and make all parent directories if they don't exist.
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Maintenance script that generates a sitemap for the site.
string $timestamp
When this sitemap batch was generated.
IDatabase $dbr
A database replica DB object.
string $fspath
The path to prepend to the filename.
array $priorities
Key => value entries of namespaces and their priorities.
bool $skipRedirects
Whether or not to include redirection pages.
__construct()
Default constructor.
array $limit
The number of entries to save in each sitemap file.
int $size_limit
The maximum size of a sitemap file.
bool $compress
Whether or not to use compression.
array $namespaces
A one-dimensional array of namespaces in the wiki.
string $urlpath
The URL path to prepend to filenames in the index; should resolve to the same directory as $fspath.
resource $findex
A resource pointing to the sitemap index file.
int $url_limit
The maximum amount of urls in a sitemap file.
resource false $file
A resource pointing to a sitemap file.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:66
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
output( $out, $channel=null)
Throw some output to the user.
hasOption( $name)
Checks to see if a particular option was set.
getServiceContainer()
Returns the main service container.
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
A class containing constants representing the names of configuration variables.
Represents a title within MediaWiki.
Definition: Title.php:76
Tools for dealing with other locally-hosted wikis.
Definition: WikiMap.php:31
$maintClass
Basic database interface for live and lazy-loaded relation database handles.
Definition: IDatabase.php:36
Result wrapper for grabbing data queried from an IDatabase object.
const DB_REPLICA
Definition: defines.php:26
if(!is_readable( $file)) $ext
Definition: router.php:48