MediaWiki REL1_30
generateSitemap.php
Go to the documentation of this file.
1<?php
29require_once __DIR__ . '/Maintenance.php';
30
37 const GS_MAIN = -2;
38 const GS_TALK = -1;
39
47 public $url_limit;
48
57
63 public $fspath;
64
71 public $urlpath;
72
78 public $compress;
79
86
92 public $limit = [];
93
99 public $priorities = [];
100
106 public $namespaces = [];
107
114
120 public $dbr;
121
127 public $findex;
128
134 public $file;
135
141 private $identifier;
142
143 public function __construct() {
144 parent::__construct();
145 $this->addDescription( 'Creates a sitemap for the site' );
146 $this->addOption(
147 'fspath',
148 'The file system path to save to, e.g. /tmp/sitemap; defaults to current directory',
149 false,
150 true
151 );
152 $this->addOption(
153 'urlpath',
154 'The URL path corresponding to --fspath, prepended to filenames in the index; '
155 . 'defaults to an empty string',
156 false,
157 true
158 );
159 $this->addOption(
160 'compress',
161 'Compress the sitemap files, can take value yes|no, default yes',
162 false,
163 true
164 );
165 $this->addOption( 'skip-redirects', 'Do not include redirecting articles in the sitemap' );
166 $this->addOption(
167 'identifier',
168 'What site identifier to use for the wiki, defaults to $wgDBname',
169 false,
170 true
171 );
172 }
173
177 public function execute() {
178 $this->setNamespacePriorities();
179 $this->url_limit = 50000;
180 $this->size_limit = pow( 2, 20 ) * 10;
181
182 # Create directory if needed
183 $fspath = $this->getOption( 'fspath', getcwd() );
184 if ( !wfMkdirParents( $fspath, null, __METHOD__ ) ) {
185 $this->error( "Can not create directory $fspath.", 1 );
186 }
187
188 $this->fspath = realpath( $fspath ) . DIRECTORY_SEPARATOR;
189 $this->urlpath = $this->getOption( 'urlpath', "" );
190 if ( $this->urlpath !== "" && substr( $this->urlpath, -1 ) !== '/' ) {
191 $this->urlpath .= '/';
192 }
193 $this->identifier = $this->getOption( 'identifier', wfWikiID() );
194 $this->compress = $this->getOption( 'compress', 'yes' ) !== 'no';
195 $this->skipRedirects = $this->hasOption( 'skip-redirects' );
196 $this->dbr = $this->getDB( DB_REPLICA );
197 $this->generateNamespaces();
198 $this->timestamp = wfTimestamp( TS_ISO_8601, wfTimestampNow() );
199 $this->findex = fopen( "{$this->fspath}sitemap-index-{$this->identifier}.xml", 'wb' );
200 $this->main();
201 }
202
203 private function setNamespacePriorities() {
205
206 // Custom main namespaces
207 $this->priorities[self::GS_MAIN] = '0.5';
208 // Custom talk namesspaces
209 $this->priorities[self::GS_TALK] = '0.1';
210 // MediaWiki standard namespaces
211 $this->priorities[NS_MAIN] = '1.0';
212 $this->priorities[NS_TALK] = '0.1';
213 $this->priorities[NS_USER] = '0.5';
214 $this->priorities[NS_USER_TALK] = '0.1';
215 $this->priorities[NS_PROJECT] = '0.5';
216 $this->priorities[NS_PROJECT_TALK] = '0.1';
217 $this->priorities[NS_FILE] = '0.5';
218 $this->priorities[NS_FILE_TALK] = '0.1';
219 $this->priorities[NS_MEDIAWIKI] = '0.0';
220 $this->priorities[NS_MEDIAWIKI_TALK] = '0.1';
221 $this->priorities[NS_TEMPLATE] = '0.0';
222 $this->priorities[NS_TEMPLATE_TALK] = '0.1';
223 $this->priorities[NS_HELP] = '0.5';
224 $this->priorities[NS_HELP_TALK] = '0.1';
225 $this->priorities[NS_CATEGORY] = '0.5';
226 $this->priorities[NS_CATEGORY_TALK] = '0.1';
227
228 // Custom priorities
229 if ( $wgSitemapNamespacesPriorities !== false ) {
233 foreach ( $wgSitemapNamespacesPriorities as $namespace => $priority ) {
234 $float = floatval( $priority );
235 if ( $float > 1.0 ) {
236 $priority = '1.0';
237 } elseif ( $float < 0.0 ) {
238 $priority = '0.0';
239 }
240 $this->priorities[$namespace] = $priority;
241 }
242 }
243 }
244
249 // Only generate for specific namespaces if $wgSitemapNamespaces is an array.
251 if ( is_array( $wgSitemapNamespaces ) ) {
253
254 return;
255 }
256
257 $res = $this->dbr->select( 'page',
258 [ 'page_namespace' ],
259 [],
260 __METHOD__,
261 [
262 'GROUP BY' => 'page_namespace',
263 'ORDER BY' => 'page_namespace',
264 ]
265 );
266
267 foreach ( $res as $row ) {
268 $this->namespaces[] = $row->page_namespace;
269 }
270 }
271
278 function priority( $namespace ) {
279 return isset( $this->priorities[$namespace] )
280 ? $this->priorities[$namespace]
281 : $this->guessPriority( $namespace );
282 }
283
292 function guessPriority( $namespace ) {
293 return MWNamespace::isSubject( $namespace )
294 ? $this->priorities[self::GS_MAIN]
295 : $this->priorities[self::GS_TALK];
296 }
297
304 function getPageRes( $namespace ) {
305 return $this->dbr->select( 'page',
306 [
307 'page_namespace',
308 'page_title',
309 'page_touched',
310 'page_is_redirect'
311 ],
312 [ 'page_namespace' => $namespace ],
313 __METHOD__
314 );
315 }
316
320 public function main() {
321 global $wgContLang;
322
323 fwrite( $this->findex, $this->openIndex() );
324
325 foreach ( $this->namespaces as $namespace ) {
326 $res = $this->getPageRes( $namespace );
327 $this->file = false;
328 $this->generateLimit( $namespace );
329 $length = $this->limit[0];
330 $i = $smcount = 0;
331
332 $fns = $wgContLang->getFormattedNsText( $namespace );
333 $this->output( "$namespace ($fns)\n" );
334 $skippedRedirects = 0; // Number of redirects skipped for that namespace
335 foreach ( $res as $row ) {
336 if ( $this->skipRedirects && $row->page_is_redirect ) {
337 $skippedRedirects++;
338 continue;
339 }
340
341 if ( $i++ === 0
342 || $i === $this->url_limit + 1
343 || $length + $this->limit[1] + $this->limit[2] > $this->size_limit
344 ) {
345 if ( $this->file !== false ) {
346 $this->write( $this->file, $this->closeFile() );
347 $this->close( $this->file );
348 }
349 $filename = $this->sitemapFilename( $namespace, $smcount++ );
350 $this->file = $this->open( $this->fspath . $filename, 'wb' );
351 $this->write( $this->file, $this->openFile() );
352 fwrite( $this->findex, $this->indexEntry( $filename ) );
353 $this->output( "\t$this->fspath$filename\n" );
354 $length = $this->limit[0];
355 $i = 1;
356 }
357 $title = Title::makeTitle( $row->page_namespace, $row->page_title );
358 $date = wfTimestamp( TS_ISO_8601, $row->page_touched );
359 $entry = $this->fileEntry( $title->getCanonicalURL(), $date, $this->priority( $namespace ) );
360 $length += strlen( $entry );
361 $this->write( $this->file, $entry );
362 // generate pages for language variants
363 if ( $wgContLang->hasVariants() ) {
364 $variants = $wgContLang->getVariants();
365 foreach ( $variants as $vCode ) {
366 if ( $vCode == $wgContLang->getCode() ) {
367 continue; // we don't want default variant
368 }
369 $entry = $this->fileEntry(
370 $title->getCanonicalURL( '', $vCode ),
371 $date,
372 $this->priority( $namespace )
373 );
374 $length += strlen( $entry );
375 $this->write( $this->file, $entry );
376 }
377 }
378 }
379
380 if ( $this->skipRedirects && $skippedRedirects > 0 ) {
381 $this->output( " skipped $skippedRedirects redirect(s)\n" );
382 }
383
384 if ( $this->file ) {
385 $this->write( $this->file, $this->closeFile() );
386 $this->close( $this->file );
387 }
388 }
389 fwrite( $this->findex, $this->closeIndex() );
390 fclose( $this->findex );
391 }
392
400 function open( $file, $flags ) {
401 $resource = $this->compress ? gzopen( $file, $flags ) : fopen( $file, $flags );
402 if ( $resource === false ) {
403 throw new MWException( __METHOD__
404 . " error opening file $file with flags $flags. Check permissions?" );
405 }
406
407 return $resource;
408 }
409
416 function write( &$handle, $str ) {
417 if ( $handle === true || $handle === false ) {
418 throw new MWException( __METHOD__ . " was passed a boolean as a file handle.\n" );
419 }
420 if ( $this->compress ) {
421 gzwrite( $handle, $str );
422 } else {
423 fwrite( $handle, $str );
424 }
425 }
426
432 function close( &$handle ) {
433 if ( $this->compress ) {
434 gzclose( $handle );
435 } else {
436 fclose( $handle );
437 }
438 }
439
447 function sitemapFilename( $namespace, $count ) {
448 $ext = $this->compress ? '.gz' : '';
449
450 return "sitemap-{$this->identifier}-NS_$namespace-$count.xml$ext";
451 }
452
458 function xmlHead() {
459 return '<?xml version="1.0" encoding="UTF-8"?>' . "\n";
460 }
461
467 function xmlSchema() {
468 return 'http://www.sitemaps.org/schemas/sitemap/0.9';
469 }
470
476 function openIndex() {
477 return $this->xmlHead() . '<sitemapindex xmlns="' . $this->xmlSchema() . '">' . "\n";
478 }
479
486 function indexEntry( $filename ) {
487 return
488 "\t<sitemap>\n" .
489 "\t\t<loc>{$this->urlpath}$filename</loc>\n" .
490 "\t\t<lastmod>{$this->timestamp}</lastmod>\n" .
491 "\t</sitemap>\n";
492 }
493
499 function closeIndex() {
500 return "</sitemapindex>\n";
501 }
502
508 function openFile() {
509 return $this->xmlHead() . '<urlset xmlns="' . $this->xmlSchema() . '">' . "\n";
510 }
511
520 function fileEntry( $url, $date, $priority ) {
521 return
522 "\t<url>\n" .
523 // T36666: $url may contain bad characters such as ampersands.
524 "\t\t<loc>" . htmlspecialchars( $url ) . "</loc>\n" .
525 "\t\t<lastmod>$date</lastmod>\n" .
526 "\t\t<priority>$priority</priority>\n" .
527 "\t</url>\n";
528 }
529
535 function closeFile() {
536 return "</urlset>\n";
537 }
538
544 function generateLimit( $namespace ) {
545 // T19961: make a title with the longest possible URL in this namespace
546 $title = Title::makeTitle( $namespace, str_repeat( "\xf0\xa8\xae\x81", 63 ) . "\xe5\x96\x83" );
547
548 $this->limit = [
549 strlen( $this->openFile() ),
550 strlen( $this->fileEntry(
551 $title->getCanonicalURL(),
552 wfTimestamp( TS_ISO_8601, wfTimestamp() ),
553 $this->priority( $namespace )
554 ) ),
555 strlen( $this->closeFile() )
556 ];
557 }
558}
559
560$maintClass = "GenerateSitemap";
561require_once RUN_MAINTENANCE_IF_MAIN;
to move a page</td >< td > &*You are moving the page across namespaces
$wgSitemapNamespaces
Array of namespaces to generate a Google sitemap for when the maintenance/generateSitemap....
$wgSitemapNamespacesPriorities
Custom namespace priorities for sitemaps.
wfTimestampNow()
Convenience function; returns MediaWiki timestamp for the present time.
wfMkdirParents( $dir, $mode=null, $caller=null)
Make directory, and make all parent directories if they don't exist.
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
wfWikiID()
Get an ASCII string identifying this wiki This is used as a prefix in memcached keys.
Maintenance script that generates a sitemap for the site.
generateLimit( $namespace)
Populate $this->limit.
string $timestamp
When this sitemap batch was generated.
xmlSchema()
Return the XML schema being used.
indexEntry( $filename)
Return the XML for a single sitemap indexfile entry.
close(&$handle)
gzclose() / fclose() wrapper
generateNamespaces()
Generate a one-dimensional array of existing namespaces.
closeFile()
Return the XML required to close sitemap file.
string $identifier
Identifier to use in filenames, default $wgDBname.
string $fspath
The path to prepend to the filename.
closeIndex()
Return the XML required to close a sitemap index file.
write(&$handle, $str)
gzwrite() / fwrite() wrapper
array $priorities
Key => value entries of namespaces and their priorities.
bool $skipRedirects
Whether or not to include redirection pages.
__construct()
Default constructor.
array $limit
The number of entries to save in each sitemap file.
open( $file, $flags)
gzopen() / fopen() wrapper
int $size_limit
The maximum size of a sitemap file.
xmlHead()
Return the XML required to open an XML file.
getPageRes( $namespace)
Return a database resolution of all the pages in a given namespace.
priority( $namespace)
Get the priority of a given namespace.
bool $compress
Whether or not to use compression.
array $namespaces
A one-dimensional array of namespaces in the wiki.
string $urlpath
The URL path to prepend to filenames in the index; should resolve to the same directory as $fspath.
guessPriority( $namespace)
If the namespace isn't listed on the priority list return the default priority for the namespace,...
resource $findex
A resource pointing to the sitemap index file.
int $url_limit
The maximum amount of urls in a sitemap file.
fileEntry( $url, $date, $priority)
Return the XML for a single sitemap entry.
openFile()
Return the XML required to open a sitemap file.
object $dbr
A database replica DB object.
resource $file
A resource pointing to a sitemap file.
sitemapFilename( $namespace, $count)
Get a sitemap filename.
openIndex()
Return the XML required to open a sitemap index file.
MediaWiki exception.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
getDB( $db, $groups=[], $wiki=false)
Returns a database to be used by current maintenance script.
hasOption( $name)
Checks to see if a particular param exists.
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
$res
Definition database.txt:21
this class mediates it Skin Encapsulates a look and feel for the wiki All of the functions that render HTML and make choices about how to render it are here and are called from various other places when and is meant to be subclassed with other skins that may override some of its functions The User object contains a reference to a and so rather than having a global skin object we just rely on the global User and get the skin with $wgUser and also has some character encoding functions and other locale stuff The current user interface language is instantiated as and the local content language as $wgContLang
Definition design.txt:57
design txt This is a brief overview of the new design More thorough and up to date information is available on the documentation wiki at etc Handles the details of getting and saving to the user table of the and dealing with sessions and cookies OutputPage Encapsulates the entire HTML page that will be sent in response to any server request It is used by calling its functions to add in any and then calling output() to send it all. It could be easily changed to send incrementally if that becomes useful
do that in ParserLimitReportFormat instead use this to modify the parameters of the image all existing parser cache entries will be invalid To avoid you ll need to handle that somehow(e.g. with the RejectParserCacheValue hook) because MediaWiki won 't do it for you. & $defaults error
Definition hooks.txt:2581
it s the revision text itself In either if gzip is the revision text is gzipped $flags
Definition hooks.txt:2805
const NS_HELP
Definition Defines.php:77
const NS_USER
Definition Defines.php:67
const NS_FILE
Definition Defines.php:71
const NS_MEDIAWIKI_TALK
Definition Defines.php:74
const NS_MAIN
Definition Defines.php:65
const NS_PROJECT_TALK
Definition Defines.php:70
const NS_MEDIAWIKI
Definition Defines.php:73
const NS_TEMPLATE
Definition Defines.php:75
const NS_FILE_TALK
Definition Defines.php:72
const NS_HELP_TALK
Definition Defines.php:78
const NS_CATEGORY_TALK
Definition Defines.php:80
const NS_TALK
Definition Defines.php:66
const NS_USER_TALK
Definition Defines.php:68
const NS_PROJECT
Definition Defines.php:69
const NS_CATEGORY
Definition Defines.php:79
const NS_TEMPLATE_TALK
Definition Defines.php:76
require_once RUN_MAINTENANCE_IF_MAIN
const DB_REPLICA
Definition defines.php:25