24use Wikimedia\Purtle\RdfWriter;
25use Wikimedia\Purtle\TurtleRdfWriter;
29require_once __DIR__ .
'/Maintenance.php';
42 private const SPARQL_INSERT = <<<SPARQL
52 private const SPARQL_DELETE = <<<SPARQLD
72 private $categoriesRdf;
87 parent::__construct();
89 $this->
addDescription(
"Generate RDF dump of category changes in a wiki." );
92 $this->
addOption(
'output',
"Output file (default is stdout). Will be overwritten.",
false,
94 $this->
addOption(
'start',
'Starting timestamp (inclusive), in ISO or MediaWiki format.',
96 $this->
addOption(
'end',
'Ending timestamp (exclusive), in ISO or MediaWiki format.',
true,
105 $this->rdfWriter =
new TurtleRdfWriter();
106 $this->categoriesRdf =
new CategoriesRdf( $this->rdfWriter );
115 $rcMaxAge = $this->
getConfig()->get( MainConfigNames::RCMaxAge );
117 if ( (
int)$now->getTimestamp( TS_UNIX ) - (
int)$startTS->getTimestamp( TS_UNIX ) > $rcMaxAge ) {
118 $this->
error(
"Start timestamp too old, maximum RC age is $rcMaxAge!" );
120 if ( (
int)$now->getTimestamp( TS_UNIX ) - (
int)$endTS->getTimestamp( TS_UNIX ) > $rcMaxAge ) {
121 $this->
error(
"End timestamp too old, maximum RC age is $rcMaxAge!" );
124 $this->startTS = $startTS->getTimestamp();
125 $this->endTS = $endTS->getTimestamp();
127 $outFile = $this->
getOption(
'output',
'php://stdout' );
128 if ( $outFile ===
'-' ) {
129 $outFile =
'php://stdout';
132 $output = fopen( $outFile,
'wb' );
134 $this->categoriesRdf->setupPrefixes();
135 $this->rdfWriter->start();
137 $prefixes = $this->
getRdf();
140 $prefixes = preg_replace( [
'/^@/m',
'/\s*[.]$/m' ],
'', $prefixes );
141 fwrite( $output, $prefixes );
161 fwrite( $output, $this->
updateTS( $this->endTS ) );
168 private function getInsertRdf() {
169 $rdfText = $this->
getRdf();
173 return sprintf( self::SPARQL_INSERT, $rdfText );
184 private function getCategoriesUpdate(
IReadableDatabase $dbr, $deleteUrls, $pages, $mark ) {
185 if ( !$deleteUrls ) {
190 $this->writeParentCategories( $dbr, $pages );
193 return "# $mark\n" . sprintf( self::SPARQL_DELETE, implode(
' ', $deleteUrls ) ) .
194 $this->getInsertRdf();
205 $this->categoriesRdf->writeCategoryLinkData( $pages[$row->cl_from], $row->cl_to );
215 $dumpUrl =
'<' . $this->categoriesRdf->getDumpURI() .
'>';
219 $dumpUrl schema:dateModified ?o .
222 $dumpUrl schema:dateModified ?o .
225 $dumpUrl schema:dateModified
"$ts"^^xsd:dateTime .
239 private function setupChangesIterator(
246 ->from(
'recentchanges' )
247 ->leftJoin(
'page_props',
null, [
'pp_propname' =>
'hiddencat',
'pp_page = rc_cur_id' ] )
248 ->leftJoin(
'category',
null, [
'cat_title = rc_title' ] )
249 ->select( array_merge( $columns, [
261 $this->addTimestampConditions( $it, $dbr );
272 $it = $this->setupChangesIterator( $dbr, [], $fname );
287 $it = $this->setupChangesIterator(
289 [
'page_title',
'page_namespace' ],
295 'rc_log_type' =>
'move',
298 $it->sqb->join(
'page',
null,
'rc_cur_id = page_id' );
299 $this->addIndex( $it );
312 ->from(
'recentchanges' )
313 ->select( [
'rc_cur_id',
'rc_title' ] )
317 'rc_log_type' =>
'delete',
318 'rc_log_action' =>
'delete',
322 'NOT EXISTS (SELECT * FROM page WHERE page_id = rc_cur_id)',
328 $this->addTimestampConditions( $it, $dbr );
329 $this->addIndex( $it );
340 $it = $this->setupChangesIterator( $dbr, [], $fname );
344 'rc_log_type' =>
'delete',
345 'rc_log_action' =>
'restore',
348 'EXISTS (SELECT page_id FROM page WHERE page_id = rc_cur_id)',
350 $this->addIndex( $it );
362 $it = $this->setupChangesIterator( $dbr, [], $fname );
368 $this->addIndex( $it );
379 $dbr->
expr(
'rc_timestamp',
'>=', $dbr->
timestamp( $this->startTS ) ),
380 $dbr->
expr(
'rc_timestamp',
'<', $dbr->
timestamp( $this->endTS ) ),
389 'USE INDEX' => [
'recentchanges' =>
'rc_new_name_timestamp' ]
404 ->from(
'categorylinks' )
405 ->select( [
'cl_from',
'cl_to' ] )
407 'cl_type' =>
'subcat',
411 [
'cl_from',
'cl_to' ],
414 return new RecursiveIteratorIterator( $it );
422 return $this->rdfWriter->drain();
435 foreach ( $batch as $row ) {
437 $deleteUrls[] =
'<' . $this->categoriesRdf->labelToUrl( $row->rc_title ) .
'>';
438 $this->processed[$row->rc_cur_id] =
true;
440 fwrite( $output, $this->getCategoriesUpdate( $dbr, $deleteUrls, [],
"Deletes" ) );
448 private function writeCategoryData( $row ) {
449 $this->categoriesRdf->writeCategoryData(
451 $row->pp_propname ===
'hiddencat',
452 (
int)$row->cat_pages - (
int)$row->cat_subcats - (
int)$row->cat_files,
453 (
int)$row->cat_subcats
465 foreach ( $batch as $row ) {
466 $deleteUrls[] =
'<' . $this->categoriesRdf->labelToUrl( $row->rc_title ) .
'>';
468 if ( isset( $this->processed[$row->rc_cur_id] ) ) {
477 $row->rc_title = $row->page_title;
478 $this->writeCategoryData( $row );
479 $pages[$row->rc_cur_id] = $row->page_title;
480 $this->processed[$row->rc_cur_id] =
true;
483 fwrite( $output, $this->getCategoriesUpdate( $dbr, $deleteUrls, $pages,
"Moves" ) );
492 fwrite( $output,
"# Restores\n" );
497 foreach ( $batch as $row ) {
498 if ( isset( $this->processed[$row->rc_cur_id] ) ) {
502 $this->writeCategoryData( $row );
503 $pages[$row->rc_cur_id] = $row->rc_title;
504 $this->processed[$row->rc_cur_id] =
true;
511 $this->writeParentCategories( $dbr, $pages );
513 fwrite( $output, $this->getInsertRdf() );
522 fwrite( $output,
"# Additions\n" );
526 foreach ( $batch as $row ) {
527 if ( isset( $this->processed[$row->rc_cur_id] ) ) {
531 $this->writeCategoryData( $row );
532 $pages[$row->rc_cur_id] = $row->rc_title;
533 $this->processed[$row->rc_cur_id] =
true;
540 $this->writeParentCategories( $dbr, $pages );
541 fwrite( $output, $this->getInsertRdf() );
559 foreach ( $batch as $row ) {
562 if ( isset( $this->processed[$row->rc_cur_id] ) ) {
566 $this->writeCategoryData( $row );
567 $pages[$row->rc_cur_id] = $row->rc_title;
568 $this->processed[$row->rc_cur_id] =
true;
569 $deleteUrls[] =
'<' . $this->categoriesRdf->labelToUrl( $row->rc_title ) .
'>';
572 fwrite( $output, $this->getCategoriesUpdate( $dbr, $deleteUrls, $pages,
"Edits" ) );
582 $processedTitle = [];
598 foreach ( $batch as $row ) {
599 $childPages[$row->rc_cur_id] =
true;
600 $parentCats[$row->rc_title] =
true;
611 'rc_title' =>
'page_title',
618 ->leftJoin(
'page_props',
null, [
'pp_propname' =>
'hiddencat',
'pp_page = page_id' ] )
619 ->leftJoin(
'category',
null, [
'cat_title = page_title' ] )
620 ->where( [
'page_namespace' =>
NS_CATEGORY,
'page_id' => array_keys( $childPages ) ] )
621 ->caller( __METHOD__ )->fetchResultSet();
622 foreach ( $childRows as $row ) {
623 if ( isset( $this->processed[$row->page_id] ) ) {
627 $this->writeCategoryData( $row );
628 if ( $row->page_id ) {
629 $pages[$row->page_id] = $row->rc_title;
630 $deleteUrls[] =
'<' . $this->categoriesRdf->labelToUrl( $row->rc_title ) .
'>';
631 $this->processed[$row->page_id] =
true;
641 'rc_title' =>
'cat_title',
648 ->leftJoin(
'page',
null, [
'page_title = cat_title',
'page_namespace' =>
NS_CATEGORY ] )
649 ->leftJoin(
'page_props',
null, [
'pp_propname' =>
'hiddencat',
'pp_page = page_id' ] )
650 ->where( [
'cat_title' => array_map(
'strval', array_keys( $parentCats ) ) ] )
651 ->caller( __METHOD__ )->fetchResultSet();
652 foreach ( $parentRows as $row ) {
653 if ( $row->page_id && isset( $this->processed[$row->page_id] ) ) {
657 if ( isset( $processedTitle[$row->rc_title] ) ) {
661 $this->writeCategoryData( $row );
662 if ( $row->page_id ) {
663 $pages[$row->page_id] = $row->rc_title;
664 $deleteUrls[] =
'<' . $this->categoriesRdf->labelToUrl( $row->rc_title ) .
'>';
665 $this->processed[$row->page_id] =
true;
667 $processedTitle[$row->rc_title] =
true;
671 fwrite( $output, $this->getCategoriesUpdate( $dbr, $deleteUrls, $pages,
"Changes" ) );
678require_once RUN_MAINTENANCE_IF_MAIN;
wfTimestamp( $outputtype=TS_UNIX, $ts=0)
Get a timestamp string in one of various formats.
Allows iterating a large number of rows in batches transparently.
Maintenance script to provide RDF representation of the recent changes in category tree.
initialize()
Initialize external service classes.
handleRestores(IReadableDatabase $dbr, $output)
handleMoves(IReadableDatabase $dbr, $output)
getNewCatsIterator(IReadableDatabase $dbr, $fname)
Fetch newly created categories.
execute()
Do the actual work.
getDeletedCatsIterator(IReadableDatabase $dbr, $fname)
Fetch deleted categories.
handleCategorization(IReadableDatabase $dbr, $output)
Handles categorization changes.
true[] $processed
List of processed page IDs, so we don't try to process same thing twice.
getRestoredCatsIterator(IReadableDatabase $dbr, $fname)
Fetch restored categories.
getMovedCatsIterator(IReadableDatabase $dbr, $fname)
Fetch moved categories.
handleEdits(IReadableDatabase $dbr, $output)
Handle edits for category texts.
getRdf()
Get accumulated RDF.
getChangedCatsIterator(IReadableDatabase $dbr, $type, $fname)
Fetch categorization changes or edits.
updateTS( $timestamp)
Generate SPARQL Update code for updating dump timestamp.
handleAdds(IReadableDatabase $dbr, $output)
handleDeletes(IReadableDatabase $dbr, $output)
Handle category deletes.
__construct()
Default constructor.
getCategoryLinksIterator(IReadableDatabase $dbr, array $ids, $fname)
Get iterator for links for categories.
A class containing constants representing the names of configuration variables.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
getOption( $name, $default=null)
Get an option, or return the default.
error( $err, $die=0)
Throw an error to the user.
addDescription( $text)
Set the description text.