35 require_once __DIR__ .
'/Maintenance.php';
43 private const GS_MAIN = -2;
44 private const GS_TALK = -1;
150 parent::__construct();
154 'The file system path to save to, e.g. /tmp/sitemap; defaults to current directory',
160 'The URL path corresponding to --fspath, prepended to filenames in the index; '
161 .
'defaults to an empty string',
167 'Compress the sitemap files, can take value yes|no, default yes',
171 $this->
addOption(
'skip-redirects',
'Do not include redirecting articles in the sitemap' );
174 'What site identifier to use for the wiki, defaults to $wgDBname',
184 $this->setNamespacePriorities();
185 $this->url_limit = 50000;
186 $this->size_limit = ( 2 ** 20 ) * 10;
188 # Create directory if needed
191 $this->
fatalError(
"Can not create directory $fspath." );
194 $dbDomain = WikiMap::getCurrentWikiDbDomain()->getId();
195 $this->fspath = realpath(
$fspath ) . DIRECTORY_SEPARATOR;
196 $this->urlpath = $this->
getOption(
'urlpath',
"" );
197 if ( $this->urlpath !==
"" && $this->urlpath[-1] !==
'/' ) {
198 $this->urlpath .=
'/';
200 $this->identifier = $this->
getOption(
'identifier', $dbDomain );
201 $this->compress = $this->
getOption(
'compress',
'yes' ) !==
'no';
202 $this->skipRedirects = $this->
hasOption(
'skip-redirects' );
204 $this->generateNamespaces();
206 $encIdentifier = rawurlencode( $this->identifier );
207 $this->findex = fopen(
"{$this->fspath}sitemap-index-{$encIdentifier}.xml",
'wb' );
211 private function setNamespacePriorities() {
212 $sitemapNamespacesPriorities = $this->
getConfig()->get( MainConfigNames::SitemapNamespacesPriorities );
215 $this->priorities[self::GS_MAIN] =
'0.5';
217 $this->priorities[self::GS_TALK] =
'0.1';
219 $this->priorities[
NS_MAIN] =
'1.0';
220 $this->priorities[
NS_TALK] =
'0.1';
221 $this->priorities[
NS_USER] =
'0.5';
225 $this->priorities[
NS_FILE] =
'0.5';
231 $this->priorities[
NS_HELP] =
'0.5';
237 if ( $sitemapNamespacesPriorities !==
false ) {
241 foreach ( $sitemapNamespacesPriorities as $namespace => $priority ) {
242 $float = floatval( $priority );
243 if ( $float > 1.0 ) {
245 } elseif ( $float < 0.0 ) {
248 $this->priorities[$namespace] = $priority;
256 private function generateNamespaces() {
258 $sitemapNamespaces = $this->
getConfig()->get( MainConfigNames::SitemapNamespaces );
259 if ( is_array( $sitemapNamespaces ) ) {
260 $this->namespaces = $sitemapNamespaces;
265 $res = $this->dbr->newSelectQueryBuilder()
266 ->select( [
'page_namespace' ] )
268 ->groupBy(
'page_namespace' )
269 ->orderBy(
'page_namespace' )
270 ->caller( __METHOD__ )->fetchResultSet();
272 foreach ( $res as $row ) {
273 $this->namespaces[] = $row->page_namespace;
283 private function priority( $namespace ) {
284 return $this->priorities[$namespace] ?? $this->guessPriority( $namespace );
295 private function guessPriority( $namespace ) {
297 ? $this->priorities[self::GS_MAIN]
298 : $this->priorities[self::GS_TALK];
307 private function getPageRes( $namespace ) {
308 return $this->dbr->newSelectQueryBuilder()
309 ->select( [
'page_namespace',
'page_title',
'page_touched',
'page_is_redirect',
'pp_propname' ] )
311 ->leftJoin(
'page_props',
null, [
'page_id = pp_page',
'pp_propname' =>
'noindex' ] )
312 ->where( [
'page_namespace' => $namespace ] )
313 ->caller( __METHOD__ )->fetchResultSet();
321 $contLang = $services->getContentLanguage();
322 $langConverter = $services->getLanguageConverterFactory()->getLanguageConverter( $contLang );
324 fwrite( $this->findex, $this->openIndex() );
326 foreach ( $this->namespaces as $namespace ) {
327 $res = $this->getPageRes( $namespace );
329 $this->generateLimit( $namespace );
330 $length = $this->limit[0];
333 $fns = $contLang->getFormattedNsText( $namespace );
334 $this->
output(
"$namespace ($fns)\n" );
335 $skippedRedirects = 0;
337 foreach ( $res as $row ) {
338 if ( $row->pp_propname ===
'noindex' ) {
343 if ( $this->skipRedirects && $row->page_is_redirect ) {
349 || $i === $this->url_limit + 1
350 || $length + $this->limit[1] + $this->limit[2] > $this->size_limit
352 if ( $this->file !==
false ) {
353 $this->write( $this->file, $this->closeFile() );
354 $this->close( $this->file );
356 $filename = $this->sitemapFilename( $namespace, $smcount++ );
357 $this->file = $this->open( $this->fspath . $filename,
'wb' );
358 $this->write( $this->file, $this->openFile() );
359 fwrite( $this->findex, $this->indexEntry( $filename ) );
360 $this->
output(
"\t$this->fspath$filename\n" );
361 $length = $this->limit[0];
364 $title = Title::makeTitle( $row->page_namespace, $row->page_title );
365 $date =
wfTimestamp( TS_ISO_8601, $row->page_touched );
366 $entry = $this->fileEntry( $title->getCanonicalURL(), $date, $this->priority( $namespace ) );
367 $length += strlen( $entry );
368 $this->write( $this->file, $entry );
370 if ( $langConverter->hasVariants() ) {
371 $variants = $langConverter->getVariants();
372 foreach ( $variants as $vCode ) {
373 if ( $vCode == $contLang->getCode() ) {
376 $entry = $this->fileEntry(
377 $title->getCanonicalURL( [
'variant' => $vCode ] ),
379 $this->priority( $namespace )
381 $length += strlen( $entry );
382 $this->write( $this->file, $entry );
387 if ( $skippedNoindex > 0 ) {
388 $this->
output(
" skipped $skippedNoindex page(s) with __NOINDEX__ switch\n" );
391 if ( $this->skipRedirects && $skippedRedirects > 0 ) {
392 $this->
output(
" skipped $skippedRedirects redirect(s)\n" );
396 $this->write( $this->file, $this->closeFile() );
397 $this->close( $this->file );
400 fwrite( $this->findex, $this->closeIndex() );
401 fclose( $this->findex );
411 private function open(
$file, $flags ) {
412 $resource = $this->compress ? gzopen(
$file, $flags ) : fopen(
$file, $flags );
413 if ( $resource ===
false ) {
414 throw new RuntimeException( __METHOD__
415 .
" error opening file $file with flags $flags. Check permissions?" );
427 private function write( &$handle, $str ) {
428 if ( $handle ===
true || $handle ===
false ) {
429 throw new InvalidArgumentException( __METHOD__ .
" was passed a boolean as a file handle.\n" );
431 if ( $this->compress ) {
432 gzwrite( $handle, $str );
434 fwrite( $handle, $str );
443 private function close( &$handle ) {
444 if ( $this->compress ) {
458 private function sitemapFilename( $namespace, $count ) {
459 $ext = $this->compress ?
'.gz' :
'';
461 return "sitemap-{$this->identifier}-NS_$namespace-$count.xml$ext";
469 private function xmlHead() {
470 return '<?xml version="1.0" encoding="UTF-8"?>' .
"\n";
478 private function xmlSchema() {
479 return 'http://www.sitemaps.org/schemas/sitemap/0.9';
487 private function openIndex() {
488 return $this->xmlHead() .
'<sitemapindex xmlns="' . $this->xmlSchema() .
'">' .
"\n";
497 private function indexEntry( $filename ) {
498 return "\t<sitemap>\n" .
500 ( $this->urlpath[0] ===
"/" ?
"" :
"/" ) .
501 "{$this->urlpath}$filename</loc>\n" .
502 "\t\t<lastmod>{$this->timestamp}</lastmod>\n" .
511 private function closeIndex() {
512 return "</sitemapindex>\n";
520 private function openFile() {
521 return $this->xmlHead() .
'<urlset xmlns="' . $this->xmlSchema() .
'">' .
"\n";
532 private function fileEntry( $url, $date, $priority ) {
535 "\t\t<loc>" . htmlspecialchars( $url ) .
"</loc>\n" .
536 "\t\t<lastmod>$date</lastmod>\n" .
537 "\t\t<priority>$priority</priority>\n" .
546 private function closeFile() {
547 return "</urlset>\n";
555 private function generateLimit( $namespace ) {
557 $title = Title::makeTitle( $namespace, str_repeat(
"\u{28B81}", 63 ) .
"\u{5583}" );
560 strlen( $this->openFile() ),
561 strlen( $this->fileEntry(
562 $title->getCanonicalURL(),
564 $this->priority( $namespace )
566 strlen( $this->closeFile() )
572 require_once RUN_MAINTENANCE_IF_MAIN;
wfTimestampNow()
Convenience function; returns MediaWiki timestamp for the present time.
wfGetServerUrl( $proto)
Get the wiki's "server", i.e.
wfMkdirParents( $dir, $mode=null, $caller=null)
Make directory, and make all parent directories if they don't exist.
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Maintenance script that generates a sitemap for the site.
string $timestamp
When this sitemap batch was generated.
IDatabase $dbr
A database replica DB object.
string $fspath
The path to prepend to the filename.
array $priorities
Key => value entries of namespaces and their priorities.
bool $skipRedirects
Whether or not to include redirection pages.
__construct()
Default constructor.
array $limit
The number of entries to save in each sitemap file.
int $size_limit
The maximum size of a sitemap file.
bool $compress
Whether or not to use compression.
array $namespaces
A one-dimensional array of namespaces in the wiki.
string $urlpath
The URL path to prepend to filenames in the index; should resolve to the same directory as $fspath.
resource $findex
A resource pointing to the sitemap index file.
int $url_limit
The maximum amount of urls in a sitemap file.
resource false $file
A resource pointing to a sitemap file.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
output( $out, $channel=null)
Throw some output to the user.
hasOption( $name)
Checks to see if a particular option was set.
getServiceContainer()
Returns the main service container.
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
A class containing constants representing the names of configuration variables.
if(!is_readable( $file)) $ext