MediaWiki  1.27.2
generateSitemap.php
Go to the documentation of this file.
1 <?php
29 require_once __DIR__ . '/Maintenance.php';
30 
37  const GS_MAIN = -2;
38  const GS_TALK = -1;
39 
47  public $url_limit;
48 
56  public $size_limit;
57 
63  public $fspath;
64 
71  public $urlpath;
72 
78  public $compress;
79 
86 
92  public $limit = [];
93 
99  public $priorities = [];
100 
106  public $namespaces = [];
107 
113  public $timestamp;
114 
120  public $dbr;
121 
127  public $findex;
128 
134  public $file;
135 
141  private $identifier;
142 
146  public function __construct() {
147  parent::__construct();
148  $this->addDescription( 'Creates a sitemap for the site' );
149  $this->addOption(
150  'fspath',
151  'The file system path to save to, e.g. /tmp/sitemap; defaults to current directory',
152  false,
153  true
154  );
155  $this->addOption(
156  'urlpath',
157  'The URL path corresponding to --fspath, prepended to filenames in the index; '
158  . 'defaults to an empty string',
159  false,
160  true
161  );
162  $this->addOption(
163  'compress',
164  'Compress the sitemap files, can take value yes|no, default yes',
165  false,
166  true
167  );
168  $this->addOption( 'skip-redirects', 'Do not include redirecting articles in the sitemap' );
169  $this->addOption(
170  'identifier',
171  'What site identifier to use for the wiki, defaults to $wgDBname',
172  false,
173  true
174  );
175  }
176 
180  public function execute() {
181  $this->setNamespacePriorities();
182  $this->url_limit = 50000;
183  $this->size_limit = pow( 2, 20 ) * 10;
184 
185  # Create directory if needed
186  $fspath = $this->getOption( 'fspath', getcwd() );
187  if ( !wfMkdirParents( $fspath, null, __METHOD__ ) ) {
188  $this->error( "Can not create directory $fspath.", 1 );
189  }
190 
191  $this->fspath = realpath( $fspath ) . DIRECTORY_SEPARATOR;
192  $this->urlpath = $this->getOption( 'urlpath', "" );
193  if ( $this->urlpath !== "" && substr( $this->urlpath, -1 ) !== '/' ) {
194  $this->urlpath .= '/';
195  }
196  $this->identifier = $this->getOption( 'identifier', wfWikiID() );
197  $this->compress = $this->getOption( 'compress', 'yes' ) !== 'no';
198  $this->skipRedirects = $this->getOption( 'skip-redirects', false ) !== false;
199  $this->dbr = $this->getDB( DB_SLAVE );
200  $this->generateNamespaces();
201  $this->timestamp = wfTimestamp( TS_ISO_8601, wfTimestampNow() );
202  $this->findex = fopen( "{$this->fspath}sitemap-index-{$this->identifier}.xml", 'wb' );
203  $this->main();
204  }
205 
206  private function setNamespacePriorities() {
207  global $wgSitemapNamespacesPriorities;
208 
209  // Custom main namespaces
210  $this->priorities[self::GS_MAIN] = '0.5';
211  // Custom talk namesspaces
212  $this->priorities[self::GS_TALK] = '0.1';
213  // MediaWiki standard namespaces
214  $this->priorities[NS_MAIN] = '1.0';
215  $this->priorities[NS_TALK] = '0.1';
216  $this->priorities[NS_USER] = '0.5';
217  $this->priorities[NS_USER_TALK] = '0.1';
218  $this->priorities[NS_PROJECT] = '0.5';
219  $this->priorities[NS_PROJECT_TALK] = '0.1';
220  $this->priorities[NS_FILE] = '0.5';
221  $this->priorities[NS_FILE_TALK] = '0.1';
222  $this->priorities[NS_MEDIAWIKI] = '0.0';
223  $this->priorities[NS_MEDIAWIKI_TALK] = '0.1';
224  $this->priorities[NS_TEMPLATE] = '0.0';
225  $this->priorities[NS_TEMPLATE_TALK] = '0.1';
226  $this->priorities[NS_HELP] = '0.5';
227  $this->priorities[NS_HELP_TALK] = '0.1';
228  $this->priorities[NS_CATEGORY] = '0.5';
229  $this->priorities[NS_CATEGORY_TALK] = '0.1';
230 
231  // Custom priorities
232  if ( $wgSitemapNamespacesPriorities !== false ) {
236  foreach ( $wgSitemapNamespacesPriorities as $namespace => $priority ) {
237  $float = floatval( $priority );
238  if ( $float > 1.0 ) {
239  $priority = '1.0';
240  } elseif ( $float < 0.0 ) {
241  $priority = '0.0';
242  }
243  $this->priorities[$namespace] = $priority;
244  }
245  }
246  }
247 
251  function generateNamespaces() {
252  // Only generate for specific namespaces if $wgSitemapNamespaces is an array.
253  global $wgSitemapNamespaces;
254  if ( is_array( $wgSitemapNamespaces ) ) {
255  $this->namespaces = $wgSitemapNamespaces;
256 
257  return;
258  }
259 
260  $res = $this->dbr->select( 'page',
261  [ 'page_namespace' ],
262  [],
263  __METHOD__,
264  [
265  'GROUP BY' => 'page_namespace',
266  'ORDER BY' => 'page_namespace',
267  ]
268  );
269 
270  foreach ( $res as $row ) {
271  $this->namespaces[] = $row->page_namespace;
272  }
273  }
274 
281  function priority( $namespace ) {
282  return isset( $this->priorities[$namespace] )
283  ? $this->priorities[$namespace]
284  : $this->guessPriority( $namespace );
285  }
286 
295  function guessPriority( $namespace ) {
296  return MWNamespace::isSubject( $namespace )
297  ? $this->priorities[self::GS_MAIN]
298  : $this->priorities[self::GS_TALK];
299  }
300 
307  function getPageRes( $namespace ) {
308  return $this->dbr->select( 'page',
309  [
310  'page_namespace',
311  'page_title',
312  'page_touched',
313  'page_is_redirect'
314  ],
315  [ 'page_namespace' => $namespace ],
316  __METHOD__
317  );
318  }
319 
323  public function main() {
325 
326  fwrite( $this->findex, $this->openIndex() );
327 
328  foreach ( $this->namespaces as $namespace ) {
329  $res = $this->getPageRes( $namespace );
330  $this->file = false;
331  $this->generateLimit( $namespace );
332  $length = $this->limit[0];
333  $i = $smcount = 0;
334 
335  $fns = $wgContLang->getFormattedNsText( $namespace );
336  $this->output( "$namespace ($fns)\n" );
337  $skippedRedirects = 0; // Number of redirects skipped for that namespace
338  foreach ( $res as $row ) {
339  if ( $this->skipRedirects && $row->page_is_redirect ) {
340  $skippedRedirects++;
341  continue;
342  }
343 
344  if ( $i++ === 0
345  || $i === $this->url_limit + 1
346  || $length + $this->limit[1] + $this->limit[2] > $this->size_limit
347  ) {
348  if ( $this->file !== false ) {
349  $this->write( $this->file, $this->closeFile() );
350  $this->close( $this->file );
351  }
352  $filename = $this->sitemapFilename( $namespace, $smcount++ );
353  $this->file = $this->open( $this->fspath . $filename, 'wb' );
354  $this->write( $this->file, $this->openFile() );
355  fwrite( $this->findex, $this->indexEntry( $filename ) );
356  $this->output( "\t$this->fspath$filename\n" );
357  $length = $this->limit[0];
358  $i = 1;
359  }
360  $title = Title::makeTitle( $row->page_namespace, $row->page_title );
361  $date = wfTimestamp( TS_ISO_8601, $row->page_touched );
362  $entry = $this->fileEntry( $title->getCanonicalURL(), $date, $this->priority( $namespace ) );
363  $length += strlen( $entry );
364  $this->write( $this->file, $entry );
365  // generate pages for language variants
366  if ( $wgContLang->hasVariants() ) {
367  $variants = $wgContLang->getVariants();
368  foreach ( $variants as $vCode ) {
369  if ( $vCode == $wgContLang->getCode() ) {
370  continue; // we don't want default variant
371  }
372  $entry = $this->fileEntry(
373  $title->getCanonicalURL( '', $vCode ),
374  $date,
375  $this->priority( $namespace )
376  );
377  $length += strlen( $entry );
378  $this->write( $this->file, $entry );
379  }
380  }
381  }
382 
383  if ( $this->skipRedirects && $skippedRedirects > 0 ) {
384  $this->output( " skipped $skippedRedirects redirect(s)\n" );
385  }
386 
387  if ( $this->file ) {
388  $this->write( $this->file, $this->closeFile() );
389  $this->close( $this->file );
390  }
391  }
392  fwrite( $this->findex, $this->closeIndex() );
393  fclose( $this->findex );
394  }
395 
403  function open( $file, $flags ) {
404  $resource = $this->compress ? gzopen( $file, $flags ) : fopen( $file, $flags );
405  if ( $resource === false ) {
406  throw new MWException( __METHOD__
407  . " error opening file $file with flags $flags. Check permissions?" );
408  }
409 
410  return $resource;
411  }
412 
419  function write( &$handle, $str ) {
420  if ( $handle === true || $handle === false ) {
421  throw new MWException( __METHOD__ . " was passed a boolean as a file handle.\n" );
422  }
423  if ( $this->compress ) {
424  gzwrite( $handle, $str );
425  } else {
426  fwrite( $handle, $str );
427  }
428  }
429 
435  function close( &$handle ) {
436  if ( $this->compress ) {
437  gzclose( $handle );
438  } else {
439  fclose( $handle );
440  }
441  }
442 
450  function sitemapFilename( $namespace, $count ) {
451  $ext = $this->compress ? '.gz' : '';
452 
453  return "sitemap-{$this->identifier}-NS_$namespace-$count.xml$ext";
454  }
455 
461  function xmlHead() {
462  return '<?xml version="1.0" encoding="UTF-8"?>' . "\n";
463  }
464 
470  function xmlSchema() {
471  return 'http://www.sitemaps.org/schemas/sitemap/0.9';
472  }
473 
479  function openIndex() {
480  return $this->xmlHead() . '<sitemapindex xmlns="' . $this->xmlSchema() . '">' . "\n";
481  }
482 
489  function indexEntry( $filename ) {
490  return
491  "\t<sitemap>\n" .
492  "\t\t<loc>{$this->urlpath}$filename</loc>\n" .
493  "\t\t<lastmod>{$this->timestamp}</lastmod>\n" .
494  "\t</sitemap>\n";
495  }
496 
502  function closeIndex() {
503  return "</sitemapindex>\n";
504  }
505 
511  function openFile() {
512  return $this->xmlHead() . '<urlset xmlns="' . $this->xmlSchema() . '">' . "\n";
513  }
514 
523  function fileEntry( $url, $date, $priority ) {
524  return
525  "\t<url>\n" .
526  // bug 34666: $url may contain bad characters such as ampersands.
527  "\t\t<loc>" . htmlspecialchars( $url ) . "</loc>\n" .
528  "\t\t<lastmod>$date</lastmod>\n" .
529  "\t\t<priority>$priority</priority>\n" .
530  "\t</url>\n";
531  }
532 
538  function closeFile() {
539  return "</urlset>\n";
540  }
541 
547  function generateLimit( $namespace ) {
548  // bug 17961: make a title with the longest possible URL in this namespace
549  $title = Title::makeTitle( $namespace, str_repeat( "\xf0\xa8\xae\x81", 63 ) . "\xe5\x96\x83" );
550 
551  $this->limit = [
552  strlen( $this->openFile() ),
553  strlen( $this->fileEntry(
554  $title->getCanonicalURL(),
556  $this->priority( $namespace )
557  ) ),
558  strlen( $this->closeFile() )
559  ];
560  }
561 }
562 
563 $maintClass = "GenerateSitemap";
564 require_once RUN_MAINTENANCE_IF_MAIN;
resource $findex
A resource pointing to the sitemap index file.
indexEntry($filename)
Return the XML for a single sitemap indexfile entry.
Maintenance script that generates a sitemap for the site.
const NS_MAIN
Definition: Defines.php:69
to move a page</td >< td > &*You are moving the page across namespaces
array $namespaces
A one-dimensional array of namespaces in the wiki.
array $limit
The number of entries to save in each sitemap file.
wfMkdirParents($dir, $mode=null, $caller=null)
Make directory, and make all parent directories if they don't exist.
write(&$handle, $str)
gzwrite() / fwrite() wrapper
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: maintenance.txt:39
generateNamespaces()
Generate a one-dimensional array of existing namespaces.
getDB($db, $groups=[], $wiki=false)
Returns a database to be used by current maintenance script.
it s the revision text itself In either if gzip is the revision text is gzipped $flags
Definition: hooks.txt:2548
string $identifier
Identifier to use in filenames, default $wgDBname.
array $priorities
Key => value entries of namespaces and their priorities.
xmlHead()
Return the XML required to open an XML file.
require_once RUN_MAINTENANCE_IF_MAIN
Definition: maintenance.txt:50
guessPriority($namespace)
If the namespace isn't listed on the priority list return the default priority for the namespace...
when a variable name is used in a it is silently declared as a new local masking the global
Definition: design.txt:93
const NS_TEMPLATE
Definition: Defines.php:79
__construct()
Constructor.
bool $compress
Whether or not to use compression.
closeFile()
Return the XML required to close sitemap file.
bool $skipRedirects
Whether or not to include redirection pages.
wfTimestamp($outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
addOption($name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
const NS_PROJECT
Definition: Defines.php:73
We ve cleaned up the code here by removing clumps of infrequently used code and moving them off somewhere else It s much easier for someone working with this code to see what s _really_ going and make changes or fix bugs In we can take all the code that deals with the little used title reversing we can concentrate it all in an extension file
Definition: hooks.txt:93
int $url_limit
The maximum amount of urls in a sitemap file.
const NS_MEDIAWIKI_TALK
Definition: Defines.php:78
close(&$handle)
gzclose() / fclose() wrapper
closeIndex()
Return the XML required to close a sitemap index file.
const NS_PROJECT_TALK
Definition: Defines.php:74
const TS_ISO_8601
ISO 8601 format with no timezone: 1986-02-09T20:00:00Z.
$res
Definition: database.txt:21
openIndex()
Return the XML required to open a sitemap index file.
wfTimestampNow()
Convenience function; returns MediaWiki timestamp for the present time.
const NS_CATEGORY
Definition: Defines.php:83
const DB_SLAVE
Definition: Defines.php:46
namespace and then decline to actually register it file or subcat img or subcat $title
Definition: hooks.txt:912
wfWikiID()
Get an ASCII string identifying this wiki This is used as a prefix in memcached keys.
const NS_FILE
Definition: Defines.php:75
const NS_FILE_TALK
Definition: Defines.php:76
addDescription($text)
Set the description text.
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
const NS_MEDIAWIKI
Definition: Defines.php:77
getOption($name, $default=null)
Get an option, or return the default.
output($out, $channel=null)
Throw some output to the user.
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
static isSubject($index)
Is the given namespace is a subject (non-talk) namespace?
Definition: MWNamespace.php:87
const NS_CATEGORY_TALK
Definition: Defines.php:84
const NS_HELP
Definition: Defines.php:81
int $size_limit
The maximum size of a sitemap file.
const NS_HELP_TALK
Definition: Defines.php:82
object $dbr
A database slave object.
generateLimit($namespace)
Populate $this->limit.
error($err, $die=0)
Throw an error to the user.
this class mediates it Skin Encapsulates a look and feel for the wiki All of the functions that render HTML and make choices about how to render it are here and are called from various other places when and is meant to be subclassed with other skins that may override some of its functions The User object contains a reference to a and so rather than having a global skin object we just rely on the global User and get the skin with $wgUser and also has some character encoding functions and other locale stuff The current user interface language is instantiated as and the local content language as $wgContLang
Definition: design.txt:56
string $fspath
The path to prepend to the filename.
$count
const NS_TALK
Definition: Defines.php:70
getPageRes($namespace)
Return a database resolution of all the pages in a given namespace.
$maintClass
xmlSchema()
Return the XML schema being used.
string $urlpath
The URL path to prepend to filenames in the index; should resolve to the same directory as $fspath...
open($file, $flags)
gzopen() / fopen() wrapper
resource $file
A resource pointing to a sitemap file.
main()
Main loop.
const NS_USER_TALK
Definition: Defines.php:72
fileEntry($url, $date, $priority)
Return the XML for a single sitemap entry.
priority($namespace)
Get the priority of a given namespace.
openFile()
Return the XML required to open a sitemap file.
static & makeTitle($ns, $title, $fragment= '', $interwiki= '')
Create a new Title from a namespace index and a DB key.
Definition: Title.php:524
const NS_TEMPLATE_TALK
Definition: Defines.php:80
sitemapFilename($namespace, $count)
Get a sitemap filename.
string $timestamp
When this sitemap batch was generated.