23use Wikimedia\Purtle\RdfWriter;
24use Wikimedia\Purtle\TurtleRdfWriter;
27require_once __DIR__ .
'/Maintenance.php';
39 private const SPARQL_INSERT = <<<SPARQL
49 private const SPARQL_DELETE = <<<SPARQLD
69 private $categoriesRdf;
82 parent::__construct();
84 $this->
addDescription(
"Generate RDF dump of category changes in a wiki." );
87 $this->
addOption(
'output',
"Output file (default is stdout). Will be overwritten.",
false,
89 $this->
addOption(
'start',
'Starting timestamp (inclusive), in ISO or Mediawiki format.',
91 $this->
addOption(
'end',
'Ending timestamp (exclusive), in ISO or Mediawiki format.',
true,
100 $this->rdfWriter =
new TurtleRdfWriter();
101 $this->categoriesRdf =
new CategoriesRdf( $this->rdfWriter );
110 $rcMaxAge = $this->
getConfig()->get( MainConfigNames::RCMaxAge );
112 if ( (
int)$now->getTimestamp( TS_UNIX ) - (
int)$startTS->getTimestamp( TS_UNIX ) > $rcMaxAge ) {
113 $this->
error(
"Start timestamp too old, maximum RC age is $rcMaxAge!" );
115 if ( (
int)$now->getTimestamp( TS_UNIX ) - (
int)$endTS->getTimestamp( TS_UNIX ) > $rcMaxAge ) {
116 $this->
error(
"End timestamp too old, maximum RC age is $rcMaxAge!" );
119 $this->startTS = $startTS->getTimestamp();
120 $this->endTS = $endTS->getTimestamp();
122 $outFile = $this->
getOption(
'output',
'php://stdout' );
123 if ( $outFile ===
'-' ) {
124 $outFile =
'php://stdout';
127 $output = fopen( $outFile,
'wb' );
129 $this->categoriesRdf->setupPrefixes();
130 $this->rdfWriter->start();
132 $prefixes = $this->
getRdf();
135 $prefixes = preg_replace( [
'/^@/m',
'/\s*[.]$/m' ],
'', $prefixes );
136 fwrite( $output, $prefixes );
156 fwrite( $output, $this->
updateTS( $this->endTS ) );
163 private function getInsertRdf() {
164 $rdfText = $this->
getRdf();
168 return sprintf( self::SPARQL_INSERT, $rdfText );
179 private function getCategoriesUpdate(
IReadableDatabase $dbr, $deleteUrls, $pages, $mark ) {
180 if ( !$deleteUrls ) {
185 $this->writeParentCategories( $dbr, $pages );
188 return "# $mark\n" . sprintf( self::SPARQL_DELETE, implode(
' ', $deleteUrls ) ) .
189 $this->getInsertRdf();
200 $this->categoriesRdf->writeCategoryLinkData( $pages[$row->cl_from], $row->cl_to );
210 $dumpUrl =
'<' . $this->categoriesRdf->getDumpURI() .
'>';
214 $dumpUrl schema:dateModified ?o .
217 $dumpUrl schema:dateModified ?o .
220 $dumpUrl schema:dateModified
"$ts"^^xsd:dateTime .
235 private function setupChangesIterator(
238 array $extra_tables = [],
241 $tables = [
'recentchanges',
'page_props',
'category' ];
242 if ( $extra_tables ) {
243 $tables = array_merge( $tables, $extra_tables );
250 $this->addTimestampConditions( $it, $dbr );
254 'LEFT JOIN', [
'pp_propname' =>
'hiddencat',
'pp_page = rc_cur_id' ]
257 'LEFT JOIN', [
'cat_title = rc_title' ]
280 $it = $this->setupChangesIterator( $dbr, [], [], $fname );
295 $it = $this->setupChangesIterator(
297 [
'page_title',
'page_namespace' ],
304 'rc_log_type' =>
'move',
308 'page' => [
'JOIN',
'rc_cur_id = page_id' ],
310 $this->addIndex( $it );
326 $this->addTimestampConditions( $it, $dbr );
330 'rc_log_type' =>
'delete',
331 'rc_log_action' =>
'delete',
335 'NOT EXISTS (SELECT * FROM page WHERE page_id = rc_cur_id)',
337 $this->addIndex( $it );
350 $it = $this->setupChangesIterator( $dbr, [], [], $fname );
354 'rc_log_type' =>
'delete',
355 'rc_log_action' =>
'restore',
358 'EXISTS (SELECT page_id FROM page WHERE page_id = rc_cur_id)',
360 $this->addIndex( $it );
372 $it = $this->setupChangesIterator( $dbr, [], [], $fname );
378 $this->addIndex( $it );
400 'USE INDEX' => [
'recentchanges' =>
'rc_new_name_timestamp' ]
415 [
'cl_from',
'cl_to' ],
419 'cl_type' =>
'subcat',
424 return new RecursiveIteratorIterator( $it );
432 return $this->rdfWriter->drain();
445 foreach ( $batch as $row ) {
447 $deleteUrls[] =
'<' . $this->categoriesRdf->labelToUrl( $row->rc_title ) .
'>';
448 $this->processed[$row->rc_cur_id] =
true;
450 fwrite( $output, $this->getCategoriesUpdate( $dbr, $deleteUrls, [],
"Deletes" ) );
458 private function writeCategoryData( $row ) {
459 $this->categoriesRdf->writeCategoryData(
461 $row->pp_propname ===
'hiddencat',
462 (
int)$row->cat_pages - (
int)$row->cat_subcats - (
int)$row->cat_files,
463 (
int)$row->cat_subcats
475 foreach ( $batch as $row ) {
476 $deleteUrls[] =
'<' . $this->categoriesRdf->labelToUrl( $row->rc_title ) .
'>';
478 if ( isset( $this->processed[$row->rc_cur_id] ) ) {
487 $row->rc_title = $row->page_title;
488 $this->writeCategoryData( $row );
489 $pages[$row->rc_cur_id] = $row->page_title;
490 $this->processed[$row->rc_cur_id] =
true;
493 fwrite( $output, $this->getCategoriesUpdate( $dbr, $deleteUrls, $pages,
"Moves" ) );
502 fwrite( $output,
"# Restores\n" );
507 foreach ( $batch as $row ) {
508 if ( isset( $this->processed[$row->rc_cur_id] ) ) {
512 $this->writeCategoryData( $row );
513 $pages[$row->rc_cur_id] = $row->rc_title;
514 $this->processed[$row->rc_cur_id] =
true;
521 $this->writeParentCategories( $dbr, $pages );
523 fwrite( $output, $this->getInsertRdf() );
532 fwrite( $output,
"# Additions\n" );
536 foreach ( $batch as $row ) {
537 if ( isset( $this->processed[$row->rc_cur_id] ) ) {
541 $this->writeCategoryData( $row );
542 $pages[$row->rc_cur_id] = $row->rc_title;
543 $this->processed[$row->rc_cur_id] =
true;
550 $this->writeParentCategories( $dbr, $pages );
551 fwrite( $output, $this->getInsertRdf() );
569 foreach ( $batch as $row ) {
572 if ( isset( $this->processed[$row->rc_cur_id] ) ) {
576 $this->writeCategoryData( $row );
577 $pages[$row->rc_cur_id] = $row->rc_title;
578 $this->processed[$row->rc_cur_id] =
true;
579 $deleteUrls[] =
'<' . $this->categoriesRdf->labelToUrl( $row->rc_title ) .
'>';
582 fwrite( $output, $this->getCategoriesUpdate( $dbr, $deleteUrls, $pages,
"Edits" ) );
592 $processedTitle = [];
608 foreach ( $batch as $row ) {
609 $childPages[$row->rc_cur_id] =
true;
610 $parentCats[$row->rc_title] =
true;
621 'rc_title' =>
'page_title',
628 ->leftJoin(
'page_props',
null, [
'pp_propname' =>
'hiddencat',
'pp_page = page_id' ] )
629 ->leftJoin(
'category',
null, [
'cat_title = page_title' ] )
630 ->where( [
'page_namespace' =>
NS_CATEGORY,
'page_id' => array_keys( $childPages ) ] )
631 ->caller( __METHOD__ )->fetchResultSet();
632 foreach ( $childRows as $row ) {
633 if ( isset( $this->processed[$row->page_id] ) ) {
637 $this->writeCategoryData( $row );
638 if ( $row->page_id ) {
639 $pages[$row->page_id] = $row->rc_title;
640 $deleteUrls[] =
'<' . $this->categoriesRdf->labelToUrl( $row->rc_title ) .
'>';
641 $this->processed[$row->page_id] =
true;
651 'rc_title' =>
'cat_title',
658 ->leftJoin(
'page',
null, [
'page_title = cat_title',
'page_namespace' =>
NS_CATEGORY ] )
659 ->leftJoin(
'page_props',
null, [
'pp_propname' =>
'hiddencat',
'pp_page = page_id' ] )
660 ->where( [
'cat_title' => array_map(
'strval', array_keys( $parentCats ) ) ] )
661 ->caller( __METHOD__ )->fetchResultSet();
662 foreach ( $parentRows as $row ) {
663 if ( $row->page_id && isset( $this->processed[$row->page_id] ) ) {
667 if ( isset( $processedTitle[$row->rc_title] ) ) {
671 $this->writeCategoryData( $row );
672 if ( $row->page_id ) {
673 $pages[$row->page_id] = $row->rc_title;
674 $deleteUrls[] =
'<' . $this->categoriesRdf->labelToUrl( $row->rc_title ) .
'>';
675 $this->processed[$row->page_id] =
true;
677 $processedTitle[$row->rc_title] =
true;
681 fwrite( $output, $this->getCategoriesUpdate( $dbr, $deleteUrls, $pages,
"Changes" ) );
687require_once RUN_MAINTENANCE_IF_MAIN;
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Allows iterating a large number of rows in batches transparently.
addConditions(array $conditions)
addJoinConditions(array $conditions)
setFetchColumns(array $columns)
setCaller( $caller)
Use ->setCaller( METHOD ) to indicate which code is using this class.
addOptions(array $options)
Maintenance script to provide RDF representation of the recent changes in category tree.
initialize()
Initialize external service classes.
handleRestores(IReadableDatabase $dbr, $output)
handleMoves(IReadableDatabase $dbr, $output)
getNewCatsIterator(IReadableDatabase $dbr, $fname)
Fetch newly created categories.
execute()
Do the actual work.
getDeletedCatsIterator(IReadableDatabase $dbr, $fname)
Fetch deleted categories.
handleCategorization(IReadableDatabase $dbr, $output)
Handles categorization changes.
true[] $processed
List of processed page IDs, so we don't try to process same thing twice.
getRestoredCatsIterator(IReadableDatabase $dbr, $fname)
Fetch restored categories.
getMovedCatsIterator(IReadableDatabase $dbr, $fname)
Fetch moved categories.
handleEdits(IReadableDatabase $dbr, $output)
Handle edits for category texts.
getRdf()
Get accumulated RDF.
getChangedCatsIterator(IReadableDatabase $dbr, $type, $fname)
Fetch categorization changes or edits.
updateTS( $timestamp)
Generate SPARQL Update code for updating dump timestamp.
handleAdds(IReadableDatabase $dbr, $output)
handleDeletes(IReadableDatabase $dbr, $output)
Handle category deletes.
__construct()
Default constructor.
getCategoryLinksIterator(IReadableDatabase $dbr, array $ids, $fname)
Get iterator for links for categories.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
error( $err, $die=0)
Throw an error to the user.
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
A class containing constants representing the names of configuration variables.