43 private const GS_MAIN = -2;
44 private const GS_TALK = -1;
150 parent::__construct();
154 'The file system path to save to, e.g. /tmp/sitemap; defaults to current directory',
160 'The URL path corresponding to --fspath, prepended to filenames in the index; '
161 .
'defaults to an empty string',
167 'Compress the sitemap files, can take value yes|no, default yes',
171 $this->
addOption(
'skip-redirects',
'Do not include redirecting articles in the sitemap' );
174 'What site identifier to use for the wiki, defaults to $wgDBname',
180 'Only include pages in these namespaces in the sitemap, ' .
181 'defaults to the value of wgSitemapNamespaces if not defined.',
182 false,
true,
false,
true
190 $this->setNamespacePriorities();
191 $this->url_limit = 50000;
192 $this->size_limit = ( 2 ** 20 ) * 10;
194 # Create directory if needed
197 $this->
fatalError(
"Can not create directory $fspath." );
200 $dbDomain = WikiMap::getCurrentWikiDbDomain()->getId();
201 $this->fspath = realpath(
$fspath ) . DIRECTORY_SEPARATOR;
202 $this->urlpath = $this->
getOption(
'urlpath',
"" );
203 if ( $this->urlpath !==
"" && substr( $this->urlpath, -1 ) !==
'/' ) {
204 $this->urlpath .=
'/';
206 $this->identifier = $this->
getOption(
'identifier', $dbDomain );
207 $this->compress = $this->
getOption(
'compress',
'yes' ) !==
'no';
208 $this->skipRedirects = $this->
hasOption(
'skip-redirects' );
210 $this->generateNamespaces();
212 $encIdentifier = rawurlencode( $this->identifier );
213 $this->findex = fopen(
"{$this->fspath}sitemap-index-{$encIdentifier}.xml",
'wb' );
217 private function setNamespacePriorities() {
218 $sitemapNamespacesPriorities = $this->
getConfig()->get( MainConfigNames::SitemapNamespacesPriorities );
221 $this->priorities[self::GS_MAIN] =
'0.5';
223 $this->priorities[self::GS_TALK] =
'0.1';
225 $this->priorities[
NS_MAIN] =
'1.0';
226 $this->priorities[
NS_TALK] =
'0.1';
227 $this->priorities[
NS_USER] =
'0.5';
231 $this->priorities[
NS_FILE] =
'0.5';
237 $this->priorities[
NS_HELP] =
'0.5';
243 if ( $sitemapNamespacesPriorities !==
false ) {
247 foreach ( $sitemapNamespacesPriorities as $namespace => $priority ) {
248 $float = floatval( $priority );
249 if ( $float > 1.0 ) {
251 } elseif ( $float < 0.0 ) {
254 $this->priorities[$namespace] = $priority;
262 private function generateNamespaces() {
264 $sitemapNamespacesFromConfig = $this->
getOption(
'namespaces' );
265 if ( is_array( $sitemapNamespacesFromConfig ) && count( $sitemapNamespacesFromConfig ) > 0 ) {
266 $this->namespaces = $sitemapNamespacesFromConfig;
272 $sitemapNamespaces = $this->
getConfig()->get( MainConfigNames::SitemapNamespaces );
273 if ( is_array( $sitemapNamespaces ) ) {
274 $this->namespaces = $sitemapNamespaces;
279 $res = $this->dbr->newSelectQueryBuilder()
280 ->select( [
'page_namespace' ] )
282 ->groupBy(
'page_namespace' )
283 ->orderBy(
'page_namespace' )
284 ->caller( __METHOD__ )->fetchResultSet();
286 foreach ( $res as $row ) {
287 $this->namespaces[] = $row->page_namespace;
297 private function priority( $namespace ) {
298 return $this->priorities[$namespace] ?? $this->guessPriority( $namespace );
309 private function guessPriority( $namespace ) {
311 ? $this->priorities[self::GS_MAIN]
312 : $this->priorities[self::GS_TALK];
321 private function getPageRes( $namespace ) {
322 return $this->dbr->newSelectQueryBuilder()
323 ->select( [
'page_namespace',
'page_title',
'page_touched',
'page_is_redirect',
'pp_propname' ] )
325 ->leftJoin(
'page_props',
null, [
'page_id = pp_page',
'pp_propname' =>
'noindex' ] )
326 ->where( [
'page_namespace' => $namespace ] )
327 ->caller( __METHOD__ )->fetchResultSet();
335 $contLang = $services->getContentLanguage();
336 $langConverter = $services->getLanguageConverterFactory()->getLanguageConverter( $contLang );
338 fwrite( $this->findex, $this->openIndex() );
340 foreach ( $this->namespaces as $namespace ) {
341 $res = $this->getPageRes( $namespace );
343 $this->generateLimit( $namespace );
344 $length = $this->limit[0];
347 $fns = $contLang->getFormattedNsText( $namespace );
348 $this->
output(
"$namespace ($fns)\n" );
349 $skippedRedirects = 0;
351 foreach ( $res as $row ) {
352 if ( $row->pp_propname ===
'noindex' ) {
357 if ( $this->skipRedirects && $row->page_is_redirect ) {
363 || $i === $this->url_limit + 1
364 || $length + $this->limit[1] + $this->limit[2] > $this->size_limit
366 if ( $this->file !==
false ) {
367 $this->write( $this->file, $this->closeFile() );
368 $this->close( $this->file );
370 $filename = $this->sitemapFilename( $namespace, $smcount++ );
371 $this->file = $this->open( $this->fspath . $filename,
'wb' );
372 $this->write( $this->file, $this->openFile() );
373 fwrite( $this->findex, $this->indexEntry( $filename ) );
374 $this->
output(
"\t$this->fspath$filename\n" );
375 $length = $this->limit[0];
378 $title = Title::makeTitle( $row->page_namespace, $row->page_title );
379 $date =
wfTimestamp( TS_ISO_8601, $row->page_touched );
380 $entry = $this->fileEntry( $title->getCanonicalURL(), $date, $this->priority( $namespace ) );
381 $length += strlen( $entry );
382 $this->write( $this->file, $entry );
384 if ( $langConverter->hasVariants() ) {
385 $variants = $langConverter->getVariants();
386 foreach ( $variants as $vCode ) {
387 if ( $vCode == $contLang->getCode() ) {
390 $entry = $this->fileEntry(
391 $title->getCanonicalURL( [
'variant' => $vCode ] ),
393 $this->priority( $namespace )
395 $length += strlen( $entry );
396 $this->write( $this->file, $entry );
401 if ( $skippedNoindex > 0 ) {
402 $this->
output(
" skipped $skippedNoindex page(s) with __NOINDEX__ switch\n" );
405 if ( $this->skipRedirects && $skippedRedirects > 0 ) {
406 $this->
output(
" skipped $skippedRedirects redirect(s)\n" );
410 $this->write( $this->file, $this->closeFile() );
411 $this->close( $this->file );
414 fwrite( $this->findex, $this->closeIndex() );
415 fclose( $this->findex );
425 private function open(
$file, $flags ) {
426 $resource = $this->compress ? gzopen(
$file, $flags ) : fopen(
$file, $flags );
427 if ( $resource ===
false ) {
428 throw new RuntimeException( __METHOD__
429 .
" error opening file $file with flags $flags. Check permissions?" );
441 private function write( &$handle, $str ) {
442 if ( $handle ===
true || $handle ===
false ) {
443 throw new InvalidArgumentException( __METHOD__ .
" was passed a boolean as a file handle.\n" );
445 if ( $this->compress ) {
446 gzwrite( $handle, $str );
448 fwrite( $handle, $str );
457 private function close( &$handle ) {
458 if ( $this->compress ) {
472 private function sitemapFilename( $namespace, $count ) {
473 $ext = $this->compress ?
'.gz' :
'';
475 return "sitemap-{$this->identifier}-NS_$namespace-$count.xml$ext";
483 private function xmlHead() {
484 return '<?xml version="1.0" encoding="UTF-8"?>' .
"\n";
492 private function xmlSchema() {
493 return 'http://www.sitemaps.org/schemas/sitemap/0.9';
501 private function openIndex() {
502 return $this->xmlHead() .
'<sitemapindex xmlns="' . $this->xmlSchema() .
'">' .
"\n";
511 private function indexEntry( $filename ) {
512 return "\t<sitemap>\n" .
514 ( substr( $this->urlpath, 0, 1 ) ===
"/" ?
"" :
"/" ) .
515 "{$this->urlpath}$filename</loc>\n" .
516 "\t\t<lastmod>{$this->timestamp}</lastmod>\n" .
525 private function closeIndex() {
526 return "</sitemapindex>\n";
534 private function openFile() {
535 return $this->xmlHead() .
'<urlset xmlns="' . $this->xmlSchema() .
'">' .
"\n";
546 private function fileEntry( $url, $date, $priority ) {
549 "\t\t<loc>" . htmlspecialchars( $url ) .
"</loc>\n" .
550 "\t\t<lastmod>$date</lastmod>\n" .
551 "\t\t<priority>$priority</priority>\n" .
560 private function closeFile() {
561 return "</urlset>\n";
569 private function generateLimit( $namespace ) {
571 $title = Title::makeTitle( $namespace, str_repeat(
"\u{28B81}", 63 ) .
"\u{5583}" );
574 strlen( $this->openFile() ),
575 strlen( $this->fileEntry(
576 $title->getCanonicalURL(),
578 $this->priority( $namespace )
580 strlen( $this->closeFile() )
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
output( $out, $channel=null)
Throw some output to the user.
hasOption( $name)
Checks to see if a particular option was set.
getServiceContainer()
Returns the main service container.
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.