generateSitemap.php

Go to the documentation of this file.

<?php
use MediaWiki\MainConfigNames;
use MediaWiki\Maintenance\Maintenance;
use MediaWiki\Page\SitemapGenerator;
use MediaWiki\WikiMap\WikiMap;
use Wikimedia\Rdbms\IReadableDatabase;
use Wikimedia\Timestamp\TimestampFormat as TS;
 
// @codeCoverageIgnoreStart
require_once __DIR__ . '/Maintenance.php';
// @codeCoverageIgnoreEnd
 
class GenerateSitemap extends Maintenance {
    public $url_limit;
 
    public $fspath;
 
    public $urlpath;
 
    public $compress;
 
    public $skipRedirects;
 
    public $namespaces = [];
 
    public $timestamp;
 
    public $dbr;
 
    public $indexFile;
 
    public $file;
 
    private $identifier;
 
    public function __construct() {
        parent::__construct();
        $this->addDescription( 'Creates a sitemap for the site' );
        $this->addOption(
            'fspath',
            'The file system path to save to, e.g. /tmp/sitemap; defaults to current directory',
            false,
            true
        );
        $this->addOption(
            'urlpath',
            'The URL path corresponding to --fspath, prepended to filenames in the index; '
                . 'defaults to an empty string',
            false,
            true
        );
        $this->addOption(
            'compress',
            'Compress the sitemap files, can take value yes|no, default yes',
            false,
            true
        );
        $this->addOption( 'skip-redirects', 'Do not include redirecting articles in the sitemap' );
        $this->addOption(
            'identifier',
            'What site identifier to use for the wiki, defaults to $wgDBname',
            false,
            true
        );
        $this->addOption(
            'namespaces',
            'Only include pages in these namespaces in the sitemap, ' .
            'defaults to the value of wgSitemapNamespaces if not defined.',
            false, true, false, true
        );
        $this->addOption(
            'limit',
            'Maximum number of URLs per sitemap file. Default 50,000.',
            false,
            true
        );
    }
 
    public function execute() {
        $this->url_limit = $this->getOption( 'limit', 50_000 );
 
        # Create directory if needed
        $fspath = $this->getOption( 'fspath', getcwd() );
        if ( !wfMkdirParents( $fspath, null, __METHOD__ ) ) {
            $this->fatalError( "Can not create directory $fspath." );
        }
 
        $dbDomain = WikiMap::getCurrentWikiDbDomain()->getId();
        $this->fspath = realpath( $fspath ) . DIRECTORY_SEPARATOR;
        $this->urlpath = $this->getOption( 'urlpath', "" );
        if ( $this->urlpath !== "" && substr( $this->urlpath, -1 ) !== '/' ) {
            $this->urlpath .= '/';
        }
        $this->identifier = $this->getOption( 'identifier', $dbDomain );
        $this->compress = $this->getOption( 'compress', 'yes' ) !== 'no';
        $this->skipRedirects = $this->hasOption( 'skip-redirects' );
        $this->dbr = $this->getReplicaDB();
        $this->timestamp = wfTimestamp( TS::ISO_8601, wfTimestampNow() );
        $encIdentifier = rawurlencode( $this->identifier );
        $indexPath = "{$this->fspath}sitemap-index-{$encIdentifier}.xml";
        $this->indexFile = fopen( "{$this->fspath}sitemap-index-{$encIdentifier}.xml", 'wb' );
        $this->main();
        $this->output( "Wrote index: $indexPath\n" );
    }
 
    private function getNamespaces() {
        // Use the namespaces passed in via command line arguments if they are set.
        return $this->getOption( 'namespaces' )
            ?? $this->getConfig()->get( MainConfigNames::SitemapNamespaces )
            ?: null;
    }
 
    public function main() {
        $services = $this->getServiceContainer();
        $contLang = $services->getContentLanguage();
        $serverUrl = $services->getUrlUtils()->getServer( PROTO_CANONICAL ) ?? '';
 
        fwrite( $this->indexFile, $this->openIndex() );
 
        $generator = new SitemapGenerator(
            $contLang,
            $services->getLanguageConverterFactory(),
            $services->getGenderCache()
        );
        $generator->skipRedirects( $this->skipRedirects )
            ->namespaces( $this->getNamespaces() )
            ->limit( $this->url_limit );
 
        $sitemapId = 0;
        do {
            $filename = $this->sitemapFilename( $sitemapId++ );
            $filePath = $this->fspath . $filename;
            $file = $this->open( $filePath, 'wb' );
            $xml = $generator->getXml( $this->dbr );
            $this->write( $file, $xml );
            $this->close( $file );
            fwrite( $this->indexFile, $this->indexEntry( $filename, $serverUrl ) );
            $this->output( "Wrote sitemap: $filePath\n" );
        } while ( $generator->nextBatch() );
 
        fwrite( $this->indexFile, $this->closeIndex() );
        fclose( $this->indexFile );
    }
 
    private function open( $file, $flags ) {
        $resource = $this->compress ? gzopen( $file, $flags ) : fopen( $file, $flags );
        if ( $resource === false ) {
            throw new RuntimeException( __METHOD__
                . " error opening file $file with flags $flags. Check permissions?" );
        }
 
        return $resource;
    }
 
    private function write( &$handle, $str ) {
        if ( $handle === true || $handle === false ) {
            throw new InvalidArgumentException( __METHOD__ . " was passed a boolean as a file handle.\n" );
        }
        if ( $this->compress ) {
            gzwrite( $handle, $str );
        } else {
            fwrite( $handle, $str );
        }
    }
 
    private function close( &$handle ) {
        if ( $this->compress ) {
            gzclose( $handle );
        } else {
            fclose( $handle );
        }
    }
 
    private function sitemapFilename( $count ) {
        $ext = $this->compress ? '.gz' : '';
 
        return "sitemap-{$this->identifier}-$count.xml$ext";
    }
 
    private function xmlHead() {
        return '<?xml version="1.0" encoding="UTF-8"?>' . "\n";
    }
 
    private function xmlSchema() {
        return 'http://www.sitemaps.org/schemas/sitemap/0.9';
    }
 
    private function openIndex() {
        return $this->xmlHead() . '<sitemapindex xmlns="' . $this->xmlSchema() . '">' . "\n";
    }
 
    private function indexEntry( $filename, $serverUrl ) {
        return "\t<sitemap>\n" .
            "\t\t<loc>" . $serverUrl .
                ( substr( $this->urlpath, 0, 1 ) === "/" ? "" : "/" ) .
                "{$this->urlpath}$filename</loc>\n" .
            "\t\t<lastmod>{$this->timestamp}</lastmod>\n" .
            "\t</sitemap>\n";
    }
 
    private function closeIndex() {
        return "</sitemapindex>\n";
    }
}
 
// @codeCoverageIgnoreStart
$maintClass = GenerateSitemap::class;
require_once RUN_MAINTENANCE_IF_MAIN;
// @codeCoverageIgnoreEnd