MediaWiki  REL1_31
generateSitemap.php
Go to the documentation of this file.
1 <?php
29 require_once __DIR__ . '/Maintenance.php';
30 
37  const GS_MAIN = -2;
38  const GS_TALK = -1;
39 
47  public $url_limit;
48 
56  public $size_limit;
57 
63  public $fspath;
64 
71  public $urlpath;
72 
78  public $compress;
79 
86 
92  public $limit = [];
93 
99  public $priorities = [];
100 
106  public $namespaces = [];
107 
113  public $timestamp;
114 
120  public $dbr;
121 
127  public $findex;
128 
134  public $file;
135 
141  private $identifier;
142 
143  public function __construct() {
144  parent::__construct();
145  $this->addDescription( 'Creates a sitemap for the site' );
146  $this->addOption(
147  'fspath',
148  'The file system path to save to, e.g. /tmp/sitemap; defaults to current directory',
149  false,
150  true
151  );
152  $this->addOption(
153  'urlpath',
154  'The URL path corresponding to --fspath, prepended to filenames in the index; '
155  . 'defaults to an empty string',
156  false,
157  true
158  );
159  $this->addOption(
160  'compress',
161  'Compress the sitemap files, can take value yes|no, default yes',
162  false,
163  true
164  );
165  $this->addOption( 'skip-redirects', 'Do not include redirecting articles in the sitemap' );
166  $this->addOption(
167  'identifier',
168  'What site identifier to use for the wiki, defaults to $wgDBname',
169  false,
170  true
171  );
172  }
173 
177  public function execute() {
178  $this->setNamespacePriorities();
179  $this->url_limit = 50000;
180  $this->size_limit = pow( 2, 20 ) * 10;
181 
182  # Create directory if needed
183  $fspath = $this->getOption( 'fspath', getcwd() );
184  if ( !wfMkdirParents( $fspath, null, __METHOD__ ) ) {
185  $this->fatalError( "Can not create directory $fspath." );
186  }
187 
188  $this->fspath = realpath( $fspath ) . DIRECTORY_SEPARATOR;
189  $this->urlpath = $this->getOption( 'urlpath', "" );
190  if ( $this->urlpath !== "" && substr( $this->urlpath, -1 ) !== '/' ) {
191  $this->urlpath .= '/';
192  }
193  $this->identifier = $this->getOption( 'identifier', wfWikiID() );
194  $this->compress = $this->getOption( 'compress', 'yes' ) !== 'no';
195  $this->skipRedirects = $this->hasOption( 'skip-redirects' );
196  $this->dbr = $this->getDB( DB_REPLICA );
197  $this->generateNamespaces();
198  $this->timestamp = wfTimestamp( TS_ISO_8601, wfTimestampNow() );
199  $this->findex = fopen( "{$this->fspath}sitemap-index-{$this->identifier}.xml", 'wb' );
200  $this->main();
201  }
202 
203  private function setNamespacePriorities() {
205 
206  // Custom main namespaces
207  $this->priorities[self::GS_MAIN] = '0.5';
208  // Custom talk namesspaces
209  $this->priorities[self::GS_TALK] = '0.1';
210  // MediaWiki standard namespaces
211  $this->priorities[NS_MAIN] = '1.0';
212  $this->priorities[NS_TALK] = '0.1';
213  $this->priorities[NS_USER] = '0.5';
214  $this->priorities[NS_USER_TALK] = '0.1';
215  $this->priorities[NS_PROJECT] = '0.5';
216  $this->priorities[NS_PROJECT_TALK] = '0.1';
217  $this->priorities[NS_FILE] = '0.5';
218  $this->priorities[NS_FILE_TALK] = '0.1';
219  $this->priorities[NS_MEDIAWIKI] = '0.0';
220  $this->priorities[NS_MEDIAWIKI_TALK] = '0.1';
221  $this->priorities[NS_TEMPLATE] = '0.0';
222  $this->priorities[NS_TEMPLATE_TALK] = '0.1';
223  $this->priorities[NS_HELP] = '0.5';
224  $this->priorities[NS_HELP_TALK] = '0.1';
225  $this->priorities[NS_CATEGORY] = '0.5';
226  $this->priorities[NS_CATEGORY_TALK] = '0.1';
227 
228  // Custom priorities
229  if ( $wgSitemapNamespacesPriorities !== false ) {
233  foreach ( $wgSitemapNamespacesPriorities as $namespace => $priority ) {
234  $float = floatval( $priority );
235  if ( $float > 1.0 ) {
236  $priority = '1.0';
237  } elseif ( $float < 0.0 ) {
238  $priority = '0.0';
239  }
240  $this->priorities[$namespace] = $priority;
241  }
242  }
243  }
244 
248  function generateNamespaces() {
249  // Only generate for specific namespaces if $wgSitemapNamespaces is an array.
251  if ( is_array( $wgSitemapNamespaces ) ) {
253 
254  return;
255  }
256 
257  $res = $this->dbr->select( 'page',
258  [ 'page_namespace' ],
259  [],
260  __METHOD__,
261  [
262  'GROUP BY' => 'page_namespace',
263  'ORDER BY' => 'page_namespace',
264  ]
265  );
266 
267  foreach ( $res as $row ) {
268  $this->namespaces[] = $row->page_namespace;
269  }
270  }
271 
278  function priority( $namespace ) {
279  return isset( $this->priorities[$namespace] )
280  ? $this->priorities[$namespace]
281  : $this->guessPriority( $namespace );
282  }
283 
292  function guessPriority( $namespace ) {
293  return MWNamespace::isSubject( $namespace )
294  ? $this->priorities[self::GS_MAIN]
295  : $this->priorities[self::GS_TALK];
296  }
297 
304  function getPageRes( $namespace ) {
305  return $this->dbr->select( 'page',
306  [
307  'page_namespace',
308  'page_title',
309  'page_touched',
310  'page_is_redirect'
311  ],
312  [ 'page_namespace' => $namespace ],
313  __METHOD__
314  );
315  }
316 
320  public function main() {
322 
323  fwrite( $this->findex, $this->openIndex() );
324 
325  foreach ( $this->namespaces as $namespace ) {
326  $res = $this->getPageRes( $namespace );
327  $this->file = false;
328  $this->generateLimit( $namespace );
329  $length = $this->limit[0];
330  $i = $smcount = 0;
331 
332  $fns = $wgContLang->getFormattedNsText( $namespace );
333  $this->output( "$namespace ($fns)\n" );
334  $skippedRedirects = 0; // Number of redirects skipped for that namespace
335  foreach ( $res as $row ) {
336  if ( $this->skipRedirects && $row->page_is_redirect ) {
337  $skippedRedirects++;
338  continue;
339  }
340 
341  if ( $i++ === 0
342  || $i === $this->url_limit + 1
343  || $length + $this->limit[1] + $this->limit[2] > $this->size_limit
344  ) {
345  if ( $this->file !== false ) {
346  $this->write( $this->file, $this->closeFile() );
347  $this->close( $this->file );
348  }
349  $filename = $this->sitemapFilename( $namespace, $smcount++ );
350  $this->file = $this->open( $this->fspath . $filename, 'wb' );
351  $this->write( $this->file, $this->openFile() );
352  fwrite( $this->findex, $this->indexEntry( $filename ) );
353  $this->output( "\t$this->fspath$filename\n" );
354  $length = $this->limit[0];
355  $i = 1;
356  }
357  $title = Title::makeTitle( $row->page_namespace, $row->page_title );
358  $date = wfTimestamp( TS_ISO_8601, $row->page_touched );
359  $entry = $this->fileEntry( $title->getCanonicalURL(), $date, $this->priority( $namespace ) );
360  $length += strlen( $entry );
361  $this->write( $this->file, $entry );
362  // generate pages for language variants
363  if ( $wgContLang->hasVariants() ) {
364  $variants = $wgContLang->getVariants();
365  foreach ( $variants as $vCode ) {
366  if ( $vCode == $wgContLang->getCode() ) {
367  continue; // we don't want default variant
368  }
369  $entry = $this->fileEntry(
370  $title->getCanonicalURL( '', $vCode ),
371  $date,
372  $this->priority( $namespace )
373  );
374  $length += strlen( $entry );
375  $this->write( $this->file, $entry );
376  }
377  }
378  }
379 
380  if ( $this->skipRedirects && $skippedRedirects > 0 ) {
381  $this->output( " skipped $skippedRedirects redirect(s)\n" );
382  }
383 
384  if ( $this->file ) {
385  $this->write( $this->file, $this->closeFile() );
386  $this->close( $this->file );
387  }
388  }
389  fwrite( $this->findex, $this->closeIndex() );
390  fclose( $this->findex );
391  }
392 
400  function open( $file, $flags ) {
401  $resource = $this->compress ? gzopen( $file, $flags ) : fopen( $file, $flags );
402  if ( $resource === false ) {
403  throw new MWException( __METHOD__
404  . " error opening file $file with flags $flags. Check permissions?" );
405  }
406 
407  return $resource;
408  }
409 
416  function write( &$handle, $str ) {
417  if ( $handle === true || $handle === false ) {
418  throw new MWException( __METHOD__ . " was passed a boolean as a file handle.\n" );
419  }
420  if ( $this->compress ) {
421  gzwrite( $handle, $str );
422  } else {
423  fwrite( $handle, $str );
424  }
425  }
426 
432  function close( &$handle ) {
433  if ( $this->compress ) {
434  gzclose( $handle );
435  } else {
436  fclose( $handle );
437  }
438  }
439 
447  function sitemapFilename( $namespace, $count ) {
448  $ext = $this->compress ? '.gz' : '';
449 
450  return "sitemap-{$this->identifier}-NS_$namespace-$count.xml$ext";
451  }
452 
458  function xmlHead() {
459  return '<?xml version="1.0" encoding="UTF-8"?>' . "\n";
460  }
461 
467  function xmlSchema() {
468  return 'http://www.sitemaps.org/schemas/sitemap/0.9';
469  }
470 
476  function openIndex() {
477  return $this->xmlHead() . '<sitemapindex xmlns="' . $this->xmlSchema() . '">' . "\n";
478  }
479 
486  function indexEntry( $filename ) {
487  return "\t<sitemap>\n" .
488  "\t\t<loc>{$this->urlpath}$filename</loc>\n" .
489  "\t\t<lastmod>{$this->timestamp}</lastmod>\n" .
490  "\t</sitemap>\n";
491  }
492 
498  function closeIndex() {
499  return "</sitemapindex>\n";
500  }
501 
507  function openFile() {
508  return $this->xmlHead() . '<urlset xmlns="' . $this->xmlSchema() . '">' . "\n";
509  }
510 
519  function fileEntry( $url, $date, $priority ) {
520  return "\t<url>\n" .
521  // T36666: $url may contain bad characters such as ampersands.
522  "\t\t<loc>" . htmlspecialchars( $url ) . "</loc>\n" .
523  "\t\t<lastmod>$date</lastmod>\n" .
524  "\t\t<priority>$priority</priority>\n" .
525  "\t</url>\n";
526  }
527 
533  function closeFile() {
534  return "</urlset>\n";
535  }
536 
542  function generateLimit( $namespace ) {
543  // T19961: make a title with the longest possible URL in this namespace
544  $title = Title::makeTitle( $namespace, str_repeat( "\xf0\xa8\xae\x81", 63 ) . "\xe5\x96\x83" );
545 
546  $this->limit = [
547  strlen( $this->openFile() ),
548  strlen( $this->fileEntry(
549  $title->getCanonicalURL(),
550  wfTimestamp( TS_ISO_8601, wfTimestamp() ),
551  $this->priority( $namespace )
552  ) ),
553  strlen( $this->closeFile() )
554  ];
555  }
556 }
557 
559 require_once RUN_MAINTENANCE_IF_MAIN;
GenerateSitemap\main
main()
Main loop.
Definition: generateSitemap.php:320
GenerateSitemap\generateNamespaces
generateNamespaces()
Generate a one-dimensional array of existing namespaces.
Definition: generateSitemap.php:248
$wgSitemapNamespacesPriorities
$wgSitemapNamespacesPriorities
Custom namespace priorities for sitemaps.
Definition: DefaultSettings.php:6590
GenerateSitemap\$file
resource $file
A resource pointing to a sitemap file.
Definition: generateSitemap.php:134
NS_HELP
const NS_HELP
Definition: Defines.php:86
array
the array() calling protocol came about after MediaWiki 1.4rc1.
wfMkdirParents
wfMkdirParents( $dir, $mode=null, $caller=null)
Make directory, and make all parent directories if they don't exist.
Definition: GlobalFunctions.php:2076
NS_TEMPLATE_TALK
const NS_TEMPLATE_TALK
Definition: Defines.php:85
Maintenance\fatalError
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
Definition: Maintenance.php:439
Maintenance\addDescription
addDescription( $text)
Set the description text.
Definition: Maintenance.php:291
GenerateSitemap\xmlHead
xmlHead()
Return the XML required to open an XML file.
Definition: generateSitemap.php:458
wfTimestamp
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Definition: GlobalFunctions.php:1980
GenerateSitemap\execute
execute()
Execute.
Definition: generateSitemap.php:177
NS_FILE
const NS_FILE
Definition: Defines.php:80
RUN_MAINTENANCE_IF_MAIN
require_once RUN_MAINTENANCE_IF_MAIN
Definition: maintenance.txt:50
NS_TEMPLATE
const NS_TEMPLATE
Definition: Defines.php:84
$res
$res
Definition: database.txt:21
Maintenance
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: maintenance.txt:39
GenerateSitemap\GS_MAIN
const GS_MAIN
Definition: generateSitemap.php:37
GenerateSitemap\$url_limit
int $url_limit
The maximum amount of urls in a sitemap file.
Definition: generateSitemap.php:47
php
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:37
GenerateSitemap\generateLimit
generateLimit( $namespace)
Populate $this->limit.
Definition: generateSitemap.php:542
NS_MAIN
const NS_MAIN
Definition: Defines.php:74
file
We ve cleaned up the code here by removing clumps of infrequently used code and moving them off somewhere else It s much easier for someone working with this code to see what s _really_ going and make changes or fix bugs In we can take all the code that deals with the little used title reversing we can concentrate it all in an extension file
Definition: hooks.txt:106
GenerateSitemap\$timestamp
string $timestamp
When this sitemap batch was generated.
Definition: generateSitemap.php:113
GenerateSitemap\__construct
__construct()
Default constructor.
Definition: generateSitemap.php:143
namespaces
to move a page</td >< td > &*You are moving the page across namespaces
Definition: All_system_messages.txt:2670
MWException
MediaWiki exception.
Definition: MWException.php:26
GenerateSitemap\close
close(&$handle)
gzclose() / fclose() wrapper
Definition: generateSitemap.php:432
NS_PROJECT
const NS_PROJECT
Definition: Defines.php:78
GenerateSitemap\$findex
resource $findex
A resource pointing to the sitemap index file.
Definition: generateSitemap.php:127
GenerateSitemap\setNamespacePriorities
setNamespacePriorities()
Definition: generateSitemap.php:203
NS_MEDIAWIKI_TALK
const NS_MEDIAWIKI_TALK
Definition: Defines.php:83
Maintenance\addOption
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
Definition: Maintenance.php:219
$title
namespace and then decline to actually register it file or subcat img or subcat $title
Definition: hooks.txt:964
GenerateSitemap\$priorities
array $priorities
Key => value entries of namespaces and their priorities.
Definition: generateSitemap.php:99
Title\makeTitle
static makeTitle( $ns, $title, $fragment='', $interwiki='')
Create a new Title from a namespace index and a DB key.
Definition: Title.php:534
global
when a variable name is used in a it is silently declared as a new masking the global
Definition: design.txt:95
DB_REPLICA
const DB_REPLICA
Definition: defines.php:25
wfTimestampNow
wfTimestampNow()
Convenience function; returns MediaWiki timestamp for the present time.
Definition: GlobalFunctions.php:2009
NS_CATEGORY
const NS_CATEGORY
Definition: Defines.php:88
GenerateSitemap\$skipRedirects
bool $skipRedirects
Whether or not to include redirection pages.
Definition: generateSitemap.php:85
GenerateSitemap\guessPriority
guessPriority( $namespace)
If the namespace isn't listed on the priority list return the default priority for the namespace,...
Definition: generateSitemap.php:292
GenerateSitemap\$compress
bool $compress
Whether or not to use compression.
Definition: generateSitemap.php:78
GenerateSitemap\closeIndex
closeIndex()
Return the XML required to close a sitemap index file.
Definition: generateSitemap.php:498
MWNamespace\isSubject
static isSubject( $index)
Is the given namespace is a subject (non-talk) namespace?
Definition: MWNamespace.php:108
GenerateSitemap\fileEntry
fileEntry( $url, $date, $priority)
Return the XML for a single sitemap entry.
Definition: generateSitemap.php:519
GenerateSitemap\sitemapFilename
sitemapFilename( $namespace, $count)
Get a sitemap filename.
Definition: generateSitemap.php:447
wfWikiID
wfWikiID()
Get an ASCII string identifying this wiki This is used as a prefix in memcached keys.
Definition: GlobalFunctions.php:2763
NS_USER_TALK
const NS_USER_TALK
Definition: Defines.php:77
GenerateSitemap\closeFile
closeFile()
Return the XML required to close sitemap file.
Definition: generateSitemap.php:533
GenerateSitemap\GS_TALK
const GS_TALK
Definition: generateSitemap.php:38
GenerateSitemap\$size_limit
int $size_limit
The maximum size of a sitemap file.
Definition: generateSitemap.php:56
NS_PROJECT_TALK
const NS_PROJECT_TALK
Definition: Defines.php:79
GenerateSitemap\getPageRes
getPageRes( $namespace)
Return a database resolution of all the pages in a given namespace.
Definition: generateSitemap.php:304
GenerateSitemap\$fspath
string $fspath
The path to prepend to the filename.
Definition: generateSitemap.php:63
$wgSitemapNamespaces
$wgSitemapNamespaces
Array of namespaces to generate a Google sitemap for when the maintenance/generateSitemap....
Definition: DefaultSettings.php:6574
GenerateSitemap\priority
priority( $namespace)
Get the priority of a given namespace.
Definition: generateSitemap.php:278
GenerateSitemap\xmlSchema
xmlSchema()
Return the XML schema being used.
Definition: generateSitemap.php:467
GenerateSitemap\$namespaces
array $namespaces
A one-dimensional array of namespaces in the wiki.
Definition: generateSitemap.php:106
GenerateSitemap\openIndex
openIndex()
Return the XML required to open a sitemap index file.
Definition: generateSitemap.php:476
Maintenance\getOption
getOption( $name, $default=null)
Get an option, or return the default.
Definition: Maintenance.php:254
NS_HELP_TALK
const NS_HELP_TALK
Definition: Defines.php:87
as
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:22
GenerateSitemap\openFile
openFile()
Return the XML required to open a sitemap file.
Definition: generateSitemap.php:507
Maintenance\getDB
getDB( $db, $groups=[], $wiki=false)
Returns a database to be used by current maintenance script.
Definition: Maintenance.php:1309
NS_USER
const NS_USER
Definition: Defines.php:76
Maintenance\output
output( $out, $channel=null)
Throw some output to the user.
Definition: Maintenance.php:388
NS_TALK
const NS_TALK
Definition: Defines.php:75
GenerateSitemap\$dbr
object $dbr
A database replica DB object.
Definition: generateSitemap.php:120
NS_MEDIAWIKI
const NS_MEDIAWIKI
Definition: Defines.php:82
class
you have access to all of the normal MediaWiki so you can get a DB use the etc For full docs on the Maintenance class
Definition: maintenance.txt:56
GenerateSitemap\$limit
array $limit
The number of entries to save in each sitemap file.
Definition: generateSitemap.php:92
NS_FILE_TALK
const NS_FILE_TALK
Definition: Defines.php:81
Maintenance\hasOption
hasOption( $name)
Checks to see if a particular param exists.
Definition: Maintenance.php:240
NS_CATEGORY_TALK
const NS_CATEGORY_TALK
Definition: Defines.php:89
GenerateSitemap
Maintenance script that generates a sitemap for the site.
Definition: generateSitemap.php:36
$ext
$ext
Definition: router.php:55
GenerateSitemap\open
open( $file, $flags)
gzopen() / fopen() wrapper
Definition: generateSitemap.php:400
GenerateSitemap\$urlpath
string $urlpath
The URL path to prepend to filenames in the index; should resolve to the same directory as $fspath.
Definition: generateSitemap.php:71
GenerateSitemap\indexEntry
indexEntry( $filename)
Return the XML for a single sitemap indexfile entry.
Definition: generateSitemap.php:486
GenerateSitemap\$identifier
string $identifier
Identifier to use in filenames, default $wgDBname.
Definition: generateSitemap.php:141
GenerateSitemap\write
write(&$handle, $str)
gzwrite() / fwrite() wrapper
Definition: generateSitemap.php:416
$wgContLang
this class mediates it Skin Encapsulates a look and feel for the wiki All of the functions that render HTML and make choices about how to render it are here and are called from various other places when and is meant to be subclassed with other skins that may override some of its functions The User object contains a reference to a and so rather than having a global skin object we just rely on the global User and get the skin with $wgUser and also has some character encoding functions and other locale stuff The current user interface language is instantiated as and the content language as $wgContLang
Definition: design.txt:57
$maintClass
$maintClass
Definition: generateSitemap.php:558