SitemapGenerator.php

Go to the documentation of this file.

<?php
 
namespace MediaWiki\Page;
 
use MediaWiki\Cache\GenderCache;
use MediaWiki\Config\Config;
use MediaWiki\Language\Language;
use MediaWiki\Languages\LanguageConverterFactory;
use MediaWiki\MainConfigNames;
use MediaWiki\Title\Title;
use MediaWiki\Xml\Xml;
use Wikimedia\Rdbms\IReadableDatabase;
use Wikimedia\Timestamp\TimestampFormat as TS;
 
class SitemapGenerator {
    private ?array $selectedNamespaces = null;
    private ?array $allowedNamespaces = null;
    private ?array $excludedNamespaces = null;
    private ?int $startId = null;
    private ?int $endId = null;
    private array $variants = [];
    private bool $skipRedirects = false;
    private ?int $limit = null;
    private ?int $nextBatchStart = null;
 
    private ?GenderCache $genderCache;
 
    public static function getVariants(
        Language $contLang,
        LanguageConverterFactory $languageConverterFactory,
    ) {
        $converter = $languageConverterFactory->getLanguageConverter( $contLang );
        $variants = [];
        foreach ( $converter->getVariants() as $vCode ) {
            // We don't want the default variant
            if ( $vCode !== $contLang->getCode() ) {
                $variants[] = $vCode;
            }
        }
        return $variants;
    }
 
    public function __construct(
        Language $contLang,
        LanguageConverterFactory $languageConverterFactory,
        GenderCache $genderCache,
    ) {
        $this->variants = self::getVariants( $contLang, $languageConverterFactory );
        $this->genderCache = $contLang->needsGenderDistinction() ? $genderCache : null;
    }
 
    public function namespaces( ?array $namespaces ) {
        $this->selectedNamespaces = $namespaces;
        return $this;
    }
 
    public function additionalNamespaces( ?array $namespaces ) {
        if ( $namespaces && $this->selectedNamespaces !== null ) {
            $this->selectedNamespaces = array_unique( array_merge(
                $this->selectedNamespaces, $namespaces
            ) );
        }
        return $this;
    }
 
    public function namespacesFromConfig( Config $config ) {
        $this->allowedNamespaces = null;
        $this->excludedNamespaces = null;
 
        $sitemapNamespaces = $config->get( MainConfigNames::SitemapNamespaces );
        if ( $sitemapNamespaces ) {
            $this->selectedNamespaces = $sitemapNamespaces;
        }
 
        $defaultPolicy = $config->get( MainConfigNames::DefaultRobotPolicy );
        $namespacePolicies = $config->get( MainConfigNames::NamespaceRobotPolicies );
        if ( self::isNoIndex( $defaultPolicy ) ) {
            $namespaces = [];
            foreach ( $namespacePolicies as $ns => $policy ) {
                if ( !self::isNoIndex( $policy ) ) {
                    $namespaces[] = $ns;
                }
            }
            $this->allowedNamespaces = $namespaces;
        } else {
            $excluded = [];
            foreach ( $namespacePolicies as $ns => $policy ) {
                if ( self::isNoIndex( $policy ) ) {
                    $excluded[] = $ns;
                }
            }
            if ( $excluded ) {
                $this->excludedNamespaces = $excluded;
            }
        }
        return $this;
    }
 
    private static function isNoIndex( $policy ) {
        $policyArray = Article::formatRobotPolicy( $policy );
        return ( $policyArray['index'] ?? '' ) === 'noindex';
    }
 
    public function idRange( ?int $startId, ?int $endId ) {
        $this->startId = $startId;
        $this->endId = $endId;
        return $this;
    }
 
    public function skipRedirects( bool $skip = true ) {
        $this->skipRedirects = $skip;
        return $this;
    }
 
    public function limit( int $limit ) {
        $this->limit = $limit;
        return $this;
    }
 
    public function nextBatch() {
        if ( $this->nextBatchStart !== null ) {
            $this->startId = $this->nextBatchStart;
            return true;
        } else {
            return false;
        }
    }
 
    public function getXml( IReadableDatabase $dbr ) {
        $empty = false;
        $sqb = $dbr->newSelectQueryBuilder()
            ->select( [ 'page_id', 'page_namespace', 'page_title', 'page_touched' ] )
            ->from( 'page' )
            ->leftJoin( 'page_props', null, [ 'page_id = pp_page', 'pp_propname' => 'noindex' ] )
            ->where( [ 'pp_propname' => null ] )
            ->orderBy( 'page_id' )
            ->caller( __METHOD__ );
 
        if ( $this->startId !== null ) {
            $sqb->where( $dbr->expr( 'page_id', '>=', $this->startId ) );
        }
        if ( $this->endId !== null ) {
            $sqb->where( $dbr->expr( 'page_id', '<', $this->endId ) );
        }
        $namespaces = $this->getSelectedAndAllowedNamespaces();
        if ( $namespaces !== null ) {
            if ( $namespaces === [] ) {
                $empty = true;
            } else {
                $sqb->where( [ 'page_namespace' => $namespaces ] );
            }
        }
        if ( $this->excludedNamespaces !== null ) {
            $sqb->where( [ $dbr->expr( 'page_namespace', '!=', $this->excludedNamespaces ) ] );
        }
        if ( $this->skipRedirects ) {
            $sqb->where( [ 'page_is_redirect' => 0 ] );
        }
        $variants = [ null, ...$this->variants ];
        if ( $this->limit ) {
            $pageLimit = (int)( $this->limit / count( $variants ) );
            $sqb->limit( $pageLimit + 1 );
        } else {
            $pageLimit = null;
        }
 
        $res = $empty ? [] : $sqb->fetchResultSet();
        $this->genderCache?->doPageRows( $res );
 
        $xml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" .
            "<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n";
        $count = 0;
        $nextBatchStart = 0;
        foreach ( $res as $row ) {
            if ( $pageLimit !== null && ++$count > $pageLimit ) {
                $nextBatchStart = (int)$row->page_id;
                break;
            }
            $title = Title::makeTitle( $row->page_namespace, $row->page_title );
            foreach ( $variants as $variant ) {
                $query = $variant === null ? '' : 'variant=' . urlencode( $variant );
                $xml .= '<url>' .
                    Xml::element( 'loc', null, $title->getCanonicalURL( $query ) ) .
                    Xml::element( 'lastmod', null, wfTimestamp( TS::ISO_8601, $row->page_touched ) ) .
                    "</url>\n";
            }
        }
        $xml .= "</urlset>\n";
 
        if ( $nextBatchStart ) {
            $this->nextBatchStart = $nextBatchStart;
        } else {
            $this->nextBatchStart = null;
        }
 
        return $xml;
    }
 
    private function getSelectedAndAllowedNamespaces() {
        if ( $this->selectedNamespaces !== null ) {
            if ( $this->allowedNamespaces !== null ) {
                return array_intersect( $this->selectedNamespaces, $this->allowedNamespaces );
            } else {
                return $this->selectedNamespaces;
            }
        } else {
            return $this->allowedNamespaces;
        }
    }
}