36 require_once __DIR__ .
'/Maintenance.php';
44 private const GS_MAIN = -2;
45 private const GS_TALK = -1;
151 parent::__construct();
155 'The file system path to save to, e.g. /tmp/sitemap; defaults to current directory',
161 'The URL path corresponding to --fspath, prepended to filenames in the index; '
162 .
'defaults to an empty string',
168 'Compress the sitemap files, can take value yes|no, default yes',
172 $this->
addOption(
'skip-redirects',
'Do not include redirecting articles in the sitemap' );
175 'What site identifier to use for the wiki, defaults to $wgDBname',
185 $this->setNamespacePriorities();
186 $this->url_limit = 50000;
187 $this->size_limit = ( 2 ** 20 ) * 10;
189 # Create directory if needed
192 $this->
fatalError(
"Can not create directory $fspath." );
195 $dbDomain = WikiMap::getCurrentWikiDbDomain()->getId();
196 $this->fspath = realpath(
$fspath ) . DIRECTORY_SEPARATOR;
197 $this->urlpath = $this->
getOption(
'urlpath',
"" );
198 if ( $this->urlpath !==
"" && substr( $this->urlpath, -1 ) !==
'/' ) {
199 $this->urlpath .=
'/';
201 $this->identifier = $this->
getOption(
'identifier', $dbDomain );
202 $this->compress = $this->
getOption(
'compress',
'yes' ) !==
'no';
203 $this->skipRedirects = $this->
hasOption(
'skip-redirects' );
205 $this->generateNamespaces();
207 $encIdentifier = rawurlencode( $this->identifier );
208 $this->findex = fopen(
"{$this->fspath}sitemap-index-{$encIdentifier}.xml",
'wb' );
212 private function setNamespacePriorities() {
213 $sitemapNamespacesPriorities = $this->
getConfig()->get( MainConfigNames::SitemapNamespacesPriorities );
216 $this->priorities[self::GS_MAIN] =
'0.5';
218 $this->priorities[self::GS_TALK] =
'0.1';
220 $this->priorities[
NS_MAIN] =
'1.0';
221 $this->priorities[
NS_TALK] =
'0.1';
222 $this->priorities[
NS_USER] =
'0.5';
226 $this->priorities[
NS_FILE] =
'0.5';
232 $this->priorities[
NS_HELP] =
'0.5';
238 if ( $sitemapNamespacesPriorities !==
false ) {
242 foreach ( $sitemapNamespacesPriorities as $namespace => $priority ) {
243 $float = floatval( $priority );
244 if ( $float > 1.0 ) {
246 } elseif ( $float < 0.0 ) {
249 $this->priorities[$namespace] = $priority;
257 private function generateNamespaces() {
259 $sitemapNamespaces = $this->
getConfig()->get( MainConfigNames::SitemapNamespaces );
260 if ( is_array( $sitemapNamespaces ) ) {
261 $this->namespaces = $sitemapNamespaces;
266 $res = $this->dbr->select(
'page',
267 [
'page_namespace' ],
271 'GROUP BY' =>
'page_namespace',
272 'ORDER BY' =>
'page_namespace',
276 foreach (
$res as $row ) {
277 $this->namespaces[] = $row->page_namespace;
287 private function priority( $namespace ) {
288 return $this->priorities[$namespace] ?? $this->guessPriority( $namespace );
299 private function guessPriority( $namespace ) {
300 return MediaWikiServices::getInstance()->getNamespaceInfo()->isSubject( $namespace )
301 ? $this->priorities[self::GS_MAIN]
302 : $this->priorities[self::GS_TALK];
311 private function getPageRes( $namespace ) {
312 return $this->dbr->select(
313 [
'page',
'page_props' ],
321 [
'page_namespace' => $namespace ],
329 'pp_propname' =>
'noindex'
340 $services = MediaWikiServices::getInstance();
341 $contLang = $services->getContentLanguage();
342 $langConverter = $services->getLanguageConverterFactory()->getLanguageConverter( $contLang );
344 fwrite( $this->findex, $this->openIndex() );
346 foreach ( $this->namespaces as $namespace ) {
347 $res = $this->getPageRes( $namespace );
349 $this->generateLimit( $namespace );
350 $length = $this->limit[0];
353 $fns = $contLang->getFormattedNsText( $namespace );
354 $this->
output(
"$namespace ($fns)\n" );
355 $skippedRedirects = 0;
357 foreach (
$res as $row ) {
358 if ( $row->pp_propname ===
'noindex' ) {
363 if ( $this->skipRedirects && $row->page_is_redirect ) {
369 || $i === $this->url_limit + 1
370 || $length + $this->limit[1] + $this->limit[2] > $this->size_limit
372 if ( $this->file !==
false ) {
373 $this->write( $this->file, $this->closeFile() );
374 $this->close( $this->file );
376 $filename = $this->sitemapFilename( $namespace, $smcount++ );
377 $this->file = $this->open( $this->fspath . $filename,
'wb' );
378 $this->write( $this->file, $this->openFile() );
379 fwrite( $this->findex, $this->indexEntry( $filename ) );
380 $this->
output(
"\t$this->fspath$filename\n" );
381 $length = $this->limit[0];
384 $title = Title::makeTitle( $row->page_namespace, $row->page_title );
385 $date =
wfTimestamp( TS_ISO_8601, $row->page_touched );
386 $entry = $this->fileEntry(
$title->getCanonicalURL(), $date, $this->priority( $namespace ) );
387 $length += strlen( $entry );
388 $this->write( $this->file, $entry );
390 if ( $langConverter->hasVariants() ) {
391 $variants = $langConverter->getVariants();
392 foreach ( $variants as $vCode ) {
393 if ( $vCode == $contLang->getCode() ) {
396 $entry = $this->fileEntry(
397 $title->getCanonicalURL(
'', $vCode ),
399 $this->priority( $namespace )
401 $length += strlen( $entry );
402 $this->write( $this->file, $entry );
407 if ( $skippedNoindex > 0 ) {
408 $this->
output(
" skipped $skippedNoindex page(s) with __NOINDEX__ switch\n" );
411 if ( $this->skipRedirects && $skippedRedirects > 0 ) {
412 $this->
output(
" skipped $skippedRedirects redirect(s)\n" );
416 $this->write( $this->file, $this->closeFile() );
417 $this->close( $this->file );
420 fwrite( $this->findex, $this->closeIndex() );
421 fclose( $this->findex );
431 private function open(
$file, $flags ) {
432 $resource = $this->compress ? gzopen(
$file, $flags ) : fopen(
$file, $flags );
433 if ( $resource ===
false ) {
434 throw new RuntimeException( __METHOD__
435 .
" error opening file $file with flags $flags. Check permissions?" );
447 private function write( &$handle, $str ) {
448 if ( $handle ===
true || $handle ===
false ) {
449 throw new InvalidArgumentException( __METHOD__ .
" was passed a boolean as a file handle.\n" );
451 if ( $this->compress ) {
452 gzwrite( $handle, $str );
454 fwrite( $handle, $str );
463 private function close( &$handle ) {
464 if ( $this->compress ) {
478 private function sitemapFilename( $namespace, $count ) {
479 $ext = $this->compress ?
'.gz' :
'';
481 return "sitemap-{$this->identifier}-NS_$namespace-$count.xml$ext";
489 private function xmlHead() {
490 return '<?xml version="1.0" encoding="UTF-8"?>' .
"\n";
498 private function xmlSchema() {
499 return 'http://www.sitemaps.org/schemas/sitemap/0.9';
507 private function openIndex() {
508 return $this->xmlHead() .
'<sitemapindex xmlns="' . $this->xmlSchema() .
'">' .
"\n";
517 private function indexEntry( $filename ) {
518 return "\t<sitemap>\n" .
520 ( substr( $this->urlpath, 0, 1 ) ===
"/" ?
"" :
"/" ) .
521 "{$this->urlpath}$filename</loc>\n" .
522 "\t\t<lastmod>{$this->timestamp}</lastmod>\n" .
531 private function closeIndex() {
532 return "</sitemapindex>\n";
540 private function openFile() {
541 return $this->xmlHead() .
'<urlset xmlns="' . $this->xmlSchema() .
'">' .
"\n";
552 private function fileEntry( $url, $date, $priority ) {
555 "\t\t<loc>" . htmlspecialchars( $url ) .
"</loc>\n" .
556 "\t\t<lastmod>$date</lastmod>\n" .
557 "\t\t<priority>$priority</priority>\n" .
566 private function closeFile() {
567 return "</urlset>\n";
575 private function generateLimit( $namespace ) {
577 $title = Title::makeTitle( $namespace, str_repeat(
"\u{28B81}", 63 ) .
"\u{5583}" );
580 strlen( $this->openFile() ),
581 strlen( $this->fileEntry(
582 $title->getCanonicalURL(),
584 $this->priority( $namespace )
586 strlen( $this->closeFile() )
592 require_once RUN_MAINTENANCE_IF_MAIN;
wfTimestampNow()
Convenience function; returns MediaWiki timestamp for the present time.
wfGetServerUrl( $proto)
Get the wiki's "server", i.e.
wfMkdirParents( $dir, $mode=null, $caller=null)
Make directory, and make all parent directories if they don't exist.
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Maintenance script that generates a sitemap for the site.
string $timestamp
When this sitemap batch was generated.
IDatabase $dbr
A database replica DB object.
string $fspath
The path to prepend to the filename.
array $priorities
Key => value entries of namespaces and their priorities.
bool $skipRedirects
Whether or not to include redirection pages.
__construct()
Default constructor.
array $limit
The number of entries to save in each sitemap file.
int $size_limit
The maximum size of a sitemap file.
bool $compress
Whether or not to use compression.
array $namespaces
A one-dimensional array of namespaces in the wiki.
string $urlpath
The URL path to prepend to filenames in the index; should resolve to the same directory as $fspath.
resource $findex
A resource pointing to the sitemap index file.
int $url_limit
The maximum amount of urls in a sitemap file.
resource false $file
A resource pointing to a sitemap file.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
output( $out, $channel=null)
Throw some output to the user.
hasOption( $name)
Checks to see if a particular option was set.
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
A class containing constants representing the names of configuration variables.
if(!is_readable( $file)) $ext