ChangesListQuery.php

Go to the documentation of this file.

<?php
 
namespace MediaWiki\RecentChanges\ChangesListQuery;
 
use InvalidArgumentException;
use LogicException;
use MediaWiki\ChangeTags\ChangeTagsStore;
use MediaWiki\Config\ServiceOptions;
use MediaWiki\Deferred\LinksUpdate\LinksTable;
use MediaWiki\Linker\LinkTarget;
use MediaWiki\Linker\LinkTargetLookup;
use MediaWiki\Logging\LogPage;
use MediaWiki\MainConfigNames;
use MediaWiki\MediaWikiServices;
use MediaWiki\Page\PageIdentity;
use MediaWiki\Page\PageReference;
use MediaWiki\Permissions\Authority;
use MediaWiki\RecentChanges\RecentChange;
use MediaWiki\RecentChanges\RecentChangeLookup;
use MediaWiki\Revision\RevisionRecord;
use MediaWiki\Storage\NameTableAccessException;
use MediaWiki\Storage\NameTableStore;
use MediaWiki\Title\TitleValue;
use MediaWiki\User\TempUser\TempUserConfig;
use MediaWiki\User\UserFactory;
use MediaWiki\User\UserIdentity;
use MediaWiki\Watchlist\WatchedItemStoreInterface;
use Psr\Log\LoggerInterface;
use stdClass;
use Wikimedia\Rdbms\IExpression;
use Wikimedia\Rdbms\IReadableDatabase;
use Wikimedia\Rdbms\IResultWrapper;
use Wikimedia\Rdbms\RawSQLExpression;
use Wikimedia\Rdbms\SelectQueryBuilder;
use Wikimedia\Stats\StatsFactory;
use Wikimedia\Timestamp\ConvertibleTimestamp;
use Wikimedia\Timestamp\TimestampFormat as TS;
use function array_key_exists;
 
class ChangesListQuery implements QueryBackend, JoinDependencyProvider {
    public const CONSTRUCTOR_OPTIONS = [
        MainConfigNames::WatchlistExpiry,
        MainConfigNames::MiserMode,
        MainConfigNames::RCMaxAge,
        MainConfigNames::EnableChangesListQueryPartitioning,
        MainConfigNames::VirtualDomainsMapping,
        ...ExperienceCondition::CONSTRUCTOR_OPTIONS
    ];
 
    public const LINKS_FROM = 'from';
    public const LINKS_TO = 'to';
 
    private const LINK_TABLE_PREFIXES = [
        'pagelinks' => 'pl',
        'templatelinks' => 'tl',
        'categorylinks' => 'cl',
        'imagelinks' => 'il'
    ];
 
    public const PARTITION_THRESHOLD = 10000;
 
    public const SORT_TIMESTAMP_DESC = 'timestamp-desc';
    public const SORT_TIMESTAMP_ASC = 'timestamp-asc';
 
    private int|float $rcMaxAge;
    private bool $enablePartitioning;
    private bool $forcePartitioning = false;
    private array $virtualDomainsMapping;
 
    private array $densityTunables = [
        self::DENSITY_LINKS => 0.1,
        self::DENSITY_WATCHLIST => 0.1,
        self::DENSITY_USER => 0.1,
        self::DENSITY_CHANGE_TAG_THRESHOLD => 0.5,
    ];
 
    private $filterModules;
 
    private $joinModules;
 
    private $highlights = [];
 
    private $fields = [];
    private $conds = [];
 
    private $linkDirection = null;
    private $linkTables = [];
    private $linkTarget = null;
 
    private $preparedEmulatedUnion = false;
 
    private $prepareCallbacks = [];
 
    private $minTimestamp = null;
 
    private $startTimestamp = null;
    private $startId = null;
    private $endTimestamp = null;
    private $endId = null;
    private $sort = self::SORT_TIMESTAMP_DESC;
 
    private ?int $limit = null;
 
    private $density = 1;
 
    private $joinOrderHint = self::JOIN_ORDER_RECENTCHANGES;
 
    private ?Authority $audience = null;
 
    private $excludeDeletedAction = false;
    private $excludeDeletedUser = false;
 
    private $maxExecutionTime = null;
 
    private $forceEmptySet = false;
 
    private $distinct = false;
 
    private ?string $caller = null;
 
    private $legacyMutators = [];
    private $sqbMutators = [];
 
    public function __construct(
        private ServiceOptions $config,
        private RecentChangeLookup $recentChangeLookup,
        private WatchedItemStoreInterface $watchedItemStore,
        private TempUserConfig $tempUserConfig,
        private UserFactory $userFactory,
        private LinkTargetLookup $linkTargetLookup,
        private ChangeTagsStore $changeTagsStore,
        private StatsFactory $statsFactory,
        private NameTableStore $slotRoleStore,
        private LoggerInterface $logger,
        private IReadableDatabase $db,
        private TableStatsProvider $rcStats,
    ) {
        $this->filterModules = [
            'experience' => new ExperienceCondition(
                $config,
                $this->tempUserConfig,
                $this->userFactory,
            ),
            'user' => new UserCondition(),
            'named' => new NamedCondition( $this->tempUserConfig ),
            'bot' => new BooleanFieldCondition( 'rc_bot' ),
            'minor' => new BooleanFieldCondition( 'rc_minor' ),
            'redirect' => new BooleanJoinFieldCondition( 'page_is_redirect', 'page' ),
            'revisionType' => new RevisionTypeCondition(),
            'source' => new EnumFieldCondition(
                'rc_source',
                $this->recentChangeLookup->getAllSources()
            ),
            'logType' => new FieldEqualityCondition( 'rc_log_type', true ),
            'patrolled' => new EnumFieldCondition(
                'rc_patrolled',
                [
                    RecentChange::PRC_UNPATROLLED,
                    RecentChange::PRC_PATROLLED,
                    RecentChange::PRC_AUTOPATROLLED,
                ]
            ),
            'watched' => new WatchedCondition(
                (bool)$config->get( MainConfigNames::WatchlistExpiry )
            ),
            'seen' => new SeenCondition(
                $this->watchedItemStore
            ),
            'watchlistLabel' => new WatchlistLabelCondition(),
            'namespace' => new FieldEqualityCondition( 'rc_namespace' ),
            'title' => new TitleCondition(),
            'subpageof' => new SubpageOfCondition(),
        ];
 
        // ChangeTagsCondition consumes the density heuristic so it has to
        // be prepared after the other modules. Putting it late in the list
        // serves that purpose.
        $this->filterModules['changeTags'] = new ChangeTagsCondition(
            $this->changeTagsStore,
            $this->rcStats,
            $this->logger,
            (bool)$config->get( MainConfigNames::MiserMode ),
        );
 
        $this->joinModules = [
            'actor' => new BasicJoin( 'actor', 'recentchanges_actor', 'actor_id=rc_actor' ),
            'change_tag' => new BasicJoin(
                'change_tag',
                'changetagdisplay',
                'changetagdisplay.ct_rc_id=rc_id'
            ),
            'comment' => new BasicJoin( 'comment', 'recentchanges_comment', 'comment_id=rc_comment_id' ),
            'page' => new BasicJoin( 'page', '', 'page_id=rc_cur_id' ),
            'revision' => new BasicJoin( 'revision', '', 'rev_id=rc_this_oldid' ),
            'slots' => new SlotsJoin(),
            'user' => new BasicJoin( 'user', '', 'user_id=actor_user', 'actor' ),
            'watchlist' => new WatchlistJoin(),
            'watchlist_expiry' => new BasicJoin( 'watchlist_expiry', '', 'we_item=wl_id', 'watchlist' ),
            'watchlist_label_member' => new BasicJoin(
                'watchlist_label_member',
                '',
                'wlm_item=wl_id',
                [ 'watchlist' ]
            ),
        ];
 
        $this->rcMaxAge = (int)$config->get( MainConfigNames::RCMaxAge );
        $this->enablePartitioning = (bool)$config->get( MainConfigNames::EnableChangesListQueryPartitioning );
        $this->virtualDomainsMapping = $config->get( MainConfigNames::VirtualDomainsMapping );
    }
 
    public function applyAction( string $verb, string $moduleName, $value = null ) {
        $module = $this->getFilter( $moduleName );
        switch ( $verb ) {
            case 'require':
                $module->require( $value );
                break;
            case 'exclude':
                $module->exclude( $value );
                break;
            default:
                throw new InvalidArgumentException(
                    "Unknown filter action verb: \"$verb\"" );
        }
        return $this;
    }
 
    public function requireNamespaces( array $namespaces ) {
        return $this->applyArrayAction( 'require', 'namespace', $namespaces );
    }
 
    public function excludeNamespaces( array $namespaces ) {
        return $this->applyArrayAction( 'exclude', 'namespace', $namespaces );
    }
 
    private function applyArrayAction( string $verb, string $moduleName, array $values ) {
        foreach ( $values as $value ) {
            $this->applyAction( $verb, $moduleName, $value );
        }
        return $this;
    }
 
    public function requireSubpageOf( LinkTarget|PageReference $page ) {
        $this->getSubpageOfCondition()->require( $page );
        return $this;
    }
 
    public function requireTitle( LinkTarget|PageReference $title ) {
        $this->getTitleCondition()->require( $title );
        return $this;
    }
 
    public function requireWatched( $watchTypes = [ 'watchedold', 'watchednew' ] ) {
        return $this->applyArrayAction( 'require', 'watched', $watchTypes );
    }
 
    public function requireWatchlistLabelIds( array $labelIds ) {
        return $this->applyArrayAction( 'require', 'watchlistLabel', $labelIds );
    }
 
    public function excludeWatchlistLabelIds( array $labelIds ) {
        return $this->applyArrayAction( 'exclude', 'watchlistLabel', $labelIds );
    }
 
    public function requireLink( string $direction, array $tables, PageIdentity $page ) {
        if ( count( $tables ) == 0 ) {
            throw new InvalidArgumentException( 'Need at least one link table' );
        }
        $unknownTables = array_diff( $tables, array_keys( self::LINK_TABLE_PREFIXES ) );
        if ( $unknownTables ) {
            throw new InvalidArgumentException( 'Unknown link table(s): ' .
                implode( ', ', $unknownTables ) );
        }
 
        $this->linkDirection = $direction;
        $this->linkTables = $tables;
        $this->linkTarget = $page;
        return $this;
    }
 
    public function requireSources( array $sources ): self {
        return $this->applyArrayAction( 'require', 'source', $sources );
    }
 
    public function requireUser( UserIdentity $user ): self {
        $this->getUserFilter()->require( $user );
        return $this;
    }
 
    public function excludeUser( UserIdentity $user ): self {
        $this->getUserFilter()->exclude( $user );
        return $this;
    }
 
    public function requirePatrolled( $value ): self {
        $this->getPatrolledFilter()->require( $value );
        return $this;
    }
 
    public function requireChangeTags( $tagNames ): self {
        return $this->applyArrayAction( 'require', 'changeTags', $tagNames );
    }
 
    public function excludeChangeTags( $tagNames ): self {
        return $this->applyArrayAction( 'exclude', 'changeTags', $tagNames );
    }
 
    public function requireLatest(): self {
        $this->getRevisionTypeFilter()->require( 'latest' );
        return $this;
    }
 
    public function excludeOldRevisions(): self {
        $this->getRevisionTypeFilter()->exclude( 'old' );
        return $this;
    }
 
    public function requireSlotChanged( string $role ): self {
        try {
            $roleId = $this->slotRoleStore->getId( $role );
        } catch ( NameTableAccessException ) {
            // No revisions changed this role yet
            $this->forceEmptySet();
            return $this;
        }
 
        $this->prepareCallbacks['slotChanged'] = function () use ( $roleId ) {
            $slotsJoin = $this->getSlotsJoinModule();
            $slotsJoin->setRoleId( $roleId );
            $slotsJoin->forConds( $this )
                ->left();
 
            $slotsJoin->parentAlias()
                ->forConds()
                ->left();
 
            // Detecting whether the slot has been touched as follows:
            // 1. if slot_origin=slot_revision_id then the slot has been newly created or edited
            // with this revision
            // 2. otherwise if the content of a slot is different to the content of its parent slot,
            // then the content of the slot has been changed in this revision
            // (probably by a revert)
            $this->where( $this->db->orExpr( [
                new RawSQLExpression( 'slot.slot_origin = slot.slot_revision_id' ),
                new RawSQLExpression( 'slot.slot_content_id != parent_slot.slot_content_id' ),
                $this->db->expr( 'slot.slot_content_id', '=', null )->and( 'parent_slot.slot_content_id', '!=', null ),
                $this->db->expr( 'slot.slot_content_id', '!=', null )->and( 'parent_slot.slot_content_id', '=', null ),
            ] ) );
        };
        return $this;
    }
 
    public function excludeDeletedLogAction(): self {
        $this->excludeDeletedAction = true;
        return $this;
    }
 
    public function allowDeletedLogAction(): self {
        $this->excludeDeletedAction = false;
        return $this;
    }
 
    public function excludeDeletedUser(): self {
        $this->excludeDeletedUser = true;
        return $this;
    }
 
    public function denseRcSizeThreshold( $threshold ): self {
        $this->getChangeTagsFilter()->setDenseRcSizeThreshold( $threshold );
        return $this;
    }
 
    public function audience( ?Authority $authority ) {
        $this->audience = $authority;
        return $this;
    }
 
    public function highlight( string $name, string $verb, string $moduleName, $value = null ) {
        $module = $this->getFilter( $moduleName );
        // Validate now while the responsible caller is in the stack
        $value = $module->validateValue( $value );
        $module->capture();
        $sense = match ( $verb ) {
            'require' => true,
            'exclude' => false,
        };
        $this->highlights[$name][] = new ChangesListHighlight( $sense, $moduleName, $value );
        return $this;
    }
 
    public function minTimestamp( $timestamp ) {
        $this->minTimestamp = $timestamp;
        return $this;
    }
 
    public function startAt( string $timestamp, ?int $id = null ): self {
        $this->startTimestamp = $timestamp;
        $this->startId = $id;
        return $this;
    }
 
    public function endAt( string $timestamp, ?int $id = null ): self {
        $this->endTimestamp = $timestamp;
        $this->endId = $id;
        return $this;
    }
 
    public function orderBy( $sort ) {
        $this->sort = $sort;
        return $this;
    }
 
    public function limit( int $limit ) {
        $this->limit = $limit;
        $this->getChangeTagsFilter()->setLimit( $limit );
        return $this;
    }
 
    public function adjustDensity( $density ): self {
        if ( is_string( $density ) ) {
            if ( isset( $this->densityTunables[$density] ) ) {
                $density = $this->densityTunables[$density];
            } else {
                throw new \InvalidArgumentException( "Unknown density \"$density\"" );
            }
        }
        $this->density *= $density;
        $this->getChangeTagsFilter()->setDensityThresholdReached(
            $this->density >= $this->densityTunables[self::DENSITY_CHANGE_TAG_THRESHOLD]
        );
        return $this;
    }
 
    public function joinOrderHint( $order ): self {
        $this->joinOrderHint = $order;
        return $this;
    }
 
    public function forceEmptySet(): self {
        $this->forceEmptySet = true;
        return $this;
    }
 
    public function isEmptySet(): bool {
        return $this->forceEmptySet;
    }
 
    public function maxExecutionTime( float|int|null $time ) {
        $this->maxExecutionTime = $time;
        return $this;
    }
 
    public function enablePartitioning(): self {
        $this->enablePartitioning = true;
        return $this;
    }
 
    public function forcePartitioning(): self {
        $this->forcePartitioning = true;
        return $this;
    }
 
    private function getWatchlistJoinModule(): WatchlistJoin {
        return $this->joinModules['watchlist'];
    }
 
    private function getWatchlistExpiryJoinModule(): BasicJoin {
        return $this->joinModules['watchlist_expiry'];
    }
 
    private function getSlotsJoinModule(): SlotsJoin {
        return $this->joinModules['slots'];
    }
 
    private function getUserFilter(): UserCondition {
        return $this->filterModules['user'];
    }
 
    private function getWatchedFilter(): WatchedCondition {
        return $this->filterModules['watched'];
    }
 
    private function getSeenFilter(): SeenCondition {
        return $this->filterModules['seen'];
    }
 
    private function getChangeTagsFilter(): ChangeTagsCondition {
        return $this->filterModules['changeTags'];
    }
 
    private function getWatchlistLabelFilter(): WatchlistLabelCondition {
        return $this->filterModules['watchlistLabel'];
    }
 
    private function getRedirectFilter(): BooleanJoinFieldCondition {
        return $this->filterModules['redirect'];
    }
 
    private function getRevisionTypeFilter(): RevisionTypeCondition {
        return $this->filterModules['revisionType'];
    }
 
    private function getPatrolledFilter(): EnumFieldCondition {
        return $this->filterModules['patrolled'];
    }
 
    private function getTitleCondition(): TitleCondition {
        return $this->filterModules['title'];
    }
 
    private function getSubpageOfCondition(): SubpageOfCondition {
        return $this->filterModules['subpageof'];
    }
 
    public function watchlistUser( UserIdentity $user ) {
        $this->getWatchlistJoinModule()->setUser( $user );
        $this->getSeenFilter()->setUser( $user );
        $this->getWatchedFilter()->setUser( $user );
        return $this;
    }
 
    public function fields( $fields ): self {
        $fields = is_array( $fields ) ? $fields : [ $fields ];
        $this->fields = array_merge( $this->fields, $fields );
        return $this;
    }
 
    public function rcUserFields(): QueryBackend {
        $this->getJoin( 'actor' )->forFields( $this )->straight();
        $this->fields['rc_user'] = 'recentchanges_actor.actor_user';
        $this->fields['rc_user_text'] = 'recentchanges_actor.actor_name';
        return $this;
    }
 
    public function addChangeTagSummaryField(): self {
        $this->getChangeTagsFilter()->capture();
        return $this;
    }
 
    public function addWatchlistLabelSummaryField(): self {
        $this->getWatchlistLabelFilter()->capture();
        return $this;
    }
 
    public function recentChangeFields() {
        $this->prepareCallbacks['recentChangeFields'] = function () {
            $this->fields( RecentChange::getQueryInfo()['fields'] );
            $this->joinForFields( 'actor' )->straight();
            $this->joinForFields( 'comment' )->straight();
        };
        return $this;
    }
 
    public function watchlistFields(
        $fields = [ 'wl_user', 'wl_notificationtimestamp', 'we_expiry' ]
    ) {
        $this->prepareCallbacks['watchlistFields'] = function () use ( $fields ) {
            $wlFields = array_diff( $fields, [ 'we_expiry' ] );
            $weFields = array_intersect( $fields, [ 'we_expiry' ] );
            if ( $wlFields ) {
                $this->fields( $wlFields );
            }
            $this->joinForFields( 'watchlist' )->weakLeft();
            if ( $weFields && $this->config->get( MainConfigNames::WatchlistExpiry ) ) {
                $this->fields( $weFields );
                $this->joinForFields( 'watchlist_expiry' )->weakLeft();
            }
        };
        return $this;
    }
 
    public function sha1Fields() {
        $this->sqbMutators['sha1Fields'] = $this->applySha1Fields( ... );
        return $this;
    }
 
    private function applySha1Fields( SelectQueryBuilder $query ) {
        $pairExpr = $this->db->buildGroupConcat(
            $this->db->buildConcat( [ 'sr.role_name', $this->db->addQuotes( ':' ), 'c.content_sha1' ] ),
            ','
        );
        $revSha1Subquery = $this->db->newSelectQueryBuilder()
            ->select( [
                'rev_id',
                'rev_deleted',
                'rev_slot_pairs' => $pairExpr,
            ] )
            ->from( 'revision' )
            ->join( 'slots', 's', [ 'rev_id = s.slot_revision_id' ] )
            ->join( 'content', 'c', [ 's.slot_content_id = c.content_id' ] )
            ->join( 'slot_roles', 'sr', [ 's.slot_role_id = sr.role_id' ] )
            ->groupBy( [ 'rev_id', 'rev_deleted' ] )
            ->caller( __METHOD__ );
 
        $query->leftJoin(
            $revSha1Subquery,
            'revsha1',
            [ 'rc_this_oldid = revsha1.rev_id' ]
        );
        $query->fields( [
            'rev_deleted' => 'revsha1.rev_deleted',
            'rev_slot_pairs' => 'revsha1.rev_slot_pairs'
        ] );
    }
 
    public function addRedirectField(): self {
        $this->getRedirectFilter()->capture();
        return $this;
    }
 
    public function commentFields(): self {
        $this->prepareCallbacks['commentFields'] = function () {
            $this->joinForFields( 'comment' )->straight();
            $this->fields( [
                'rc_comment_text' => 'recentchanges_comment.comment_text',
                'rc_comment_data' => 'recentchanges_comment.comment_data',
                'rc_comment_cid' => 'recentchanges_comment.comment_id'
            ] );
        };
        return $this;
    }
 
    public function maybeAddWatchlistExpiryField(): self {
        if ( $this->config->get( MainConfigNames::WatchlistExpiry ) ) {
            $this->getWatchlistExpiryJoinModule()->forFields( $this )->weakLeft();
            $this->fields( 'we_expiry' );
        }
        return $this;
    }
 
    public function legacyMutator( callable $callback ) {
        $this->legacyMutators[] = $callback;
        return $this;
    }
 
    public function sqbMutator( callable $callback ) {
        $this->sqbMutators[] = $callback;
        return $this;
    }
 
    public function fetchResult(): ChangesListResult {
        $this->prepare();
        if ( $this->isEmptySet() ) {
            return $this->newResult();
        }
 
        $shouldPartition = $this->shouldDoPartitioning();
        if ( $shouldPartition ) {
            $this->prepareEmulatedUnion();
        }
 
        $sqb = $this->createQueryBuilder();
        if ( !$shouldPartition ) {
            $this->applyTimestampFilter( $sqb );
        }
        $sqb = $this->applyMutators( $sqb );
        if ( !$sqb || $this->isEmptySet() ) {
            return $this->newResult();
        }
 
        $queries = $this->applyLinkTarget( $sqb );
 
        $timer = $this->statsFactory->getTiming( 'ChangesListQuery_query_seconds' )
            ->setLabel( 'caller', $this->caller ?? 'unknown' )
            ->setLabel( 'union', (string)count( $queries ) )
            ->start();
 
        if ( $shouldPartition ) {
            $timer->setLabel( 'strategy', 'partition' );
            $res = $this->doPartitionUnion( $queries );
        } else {
            $timer->setLabel( 'strategy', 'simple' );
            $res = $this->maybeEmulateUnion( $queries );
        }
 
        $timer->stop();
 
        return $this->newResult( $res );
    }
 
    private function prepare() {
        if ( $this->linkTables ) {
            $this->adjustDensity( self::DENSITY_LINKS )
                ->joinOrderHint( self::JOIN_ORDER_OTHER );
        }
        if ( $this->audience !== null ) {
            $this->getChangeTagsFilter()->setAudience( $this->audience );
        }
        foreach ( $this->filterModules as $module ) {
            $module->prepareQuery( $this->db, $this );
        }
        foreach ( $this->prepareCallbacks as $callback ) {
            $callback();
        }
        $this->prepareAudienceCondition( $this->audience );
        if ( count( $this->linkTables ) > 1 ) {
            $this->prepareEmulatedUnion();
        }
    }
 
    private function prepareEmulatedUnion() {
        $this->preparedEmulatedUnion = true;
        $this->fields( [ 'rc_timestamp', 'rc_id' ] );
    }
 
    private function newResult( $rows = [] ): ChangesListResult {
        return new ChangesListResult( $rows, $this->getHighlightsFromRow( ... ) );
    }
 
    private function prepareAudienceCondition( ?Authority $authority ) {
        if ( $this->excludeDeletedUser ) {
            if ( !$authority || !$authority->isAllowed( 'deletedhistory' ) ) {
                $bitmask = RevisionRecord::DELETED_USER;
            } elseif ( !$authority->isAllowedAny( 'suppressrevision', 'viewsuppressed' ) ) {
                $bitmask = RevisionRecord::DELETED_USER | RevisionRecord::DELETED_RESTRICTED;
            } else {
                $bitmask = 0;
            }
            if ( $bitmask ) {
                $this->where( new RawSQLExpression(
                    $this->db->bitAnd( 'rc_deleted', $bitmask ) . " != $bitmask"
                ) );
            }
        }
        if ( $this->excludeDeletedAction ) {
            // Log entries with DELETED_ACTION must not show up unless the user has
            // the necessary rights.
            if ( !$authority || !$authority->isAllowed( 'deletedhistory' ) ) {
                $bitmask = LogPage::DELETED_ACTION;
            } elseif ( !$authority->isAllowedAny( 'suppressrevision', 'viewsuppressed' ) ) {
                $bitmask = LogPage::DELETED_ACTION | LogPage::DELETED_RESTRICTED;
            } else {
                $bitmask = 0;
            }
            if ( $bitmask ) {
                $this->where( $this->db->expr( 'rc_source', '!=', RecentChange::SRC_LOG )
                    ->orExpr( new RawSQLExpression(
                        $this->db->bitAnd( 'rc_deleted', $bitmask ) . " != $bitmask"
                    ) )
                );
            }
        }
    }
 
    private function createQueryBuilder(): SelectQueryBuilder {
        $sqb = $this->db->newSelectQueryBuilder()
            ->select( $this->getUniqueFields() )
            ->from( 'recentchanges' )
            ->where( $this->conds );
 
        $this->applyOptions( $sqb );
 
        foreach ( $this->joinModules as $join ) {
            $join->prepare( $sqb );
        }
        return $sqb;
    }
 
    private function applyOptions( SelectQueryBuilder $sqb ) {
        if ( $this->distinct ) {
            $sqb->distinct();
            // In order to prevent DISTINCT from causing query performance problems,
            // we have to GROUP BY the primary key.
            $sqb->groupBy( [ 'rc_timestamp', 'rc_id' ] );
        }
        $dir = $this->sort === self::SORT_TIMESTAMP_ASC ? 'ASC' : 'DESC';
        $sqb->orderBy( [ "rc_timestamp $dir", "rc_id $dir" ] );
 
        $sqb->caller( $this->caller ?? __CLASS__ );
        if ( $this->limit !== null ) {
            $sqb->limit( $this->limit );
        }
        if ( $this->maxExecutionTime !== null ) {
            $sqb->setMaxExecutionTime( $this->maxExecutionTime );
        }
    }
 
    private function applyTimestampFilter( SelectQueryBuilder $sqb ) {
        if ( $this->minTimestamp !== null ) {
            $sqb->andWhere( $this->db->expr( 'rc_timestamp', '>=',
                $this->db->timestamp( $this->minTimestamp ) ) );
        }
        $this->applyStartOrEnd( $sqb, true, $this->startTimestamp, $this->startId );
        $this->applyStartOrEnd( $sqb, false, $this->endTimestamp, $this->endId );
    }
 
    private function applyStartOrEnd( SelectQueryBuilder $sqb, $isStart, $ts, $id ) {
        if ( $ts === null ) {
            return;
        }
        $op = ( $isStart === ( $this->sort === self::SORT_TIMESTAMP_ASC ) ) ? '>=' : '<=';
        $conds = [ 'rc_timestamp' => $this->db->timestamp( $ts ) ];
        if ( $id !== null ) {
            $conds['rc_id'] = $id;
        }
        $sqb->andWhere( $this->db->buildComparison( $op, $conds ) );
    }
 
    private function getUniqueFields() {
        $seen = [];
        $fields = [];
        foreach ( $this->fields as $index => $value ) {
            if ( is_numeric( $index ) ) {
                if ( !array_key_exists( $index, $seen ) ) {
                    $fields[] = $value;
                    $seen[$index] = true;
                }
            } else {
                $fields[$index] = $value;
            }
        }
        return $fields;
    }
 
    private function applyMutators( SelectQueryBuilder $sqb ) {
        if ( $this->legacyMutators ) {
            $queryInfo = $sqb->getQueryInfo();
            foreach ( $this->legacyMutators as $mutator ) {
                $ret = $mutator(
                    $queryInfo['tables'],
                    $queryInfo['fields'],
                    $queryInfo['conds'],
                    $queryInfo['options'],
                    $queryInfo['join_conds']
                );
                if ( $ret === false ) {
                    $this->forceEmptySet();
                    return null;
                }
            }
            $sqb = $this->db->newSelectQueryBuilder()
                ->queryInfo( $queryInfo );
        }
        foreach ( $this->sqbMutators as $mutator ) {
            // Note $sqb may be passed by reference and reassigned
            $mutator( $sqb );
        }
        return $sqb;
    }
 
    private function applyLinkTarget( SelectQueryBuilder $mainQueryBuilder ): array {
        if ( !$this->linkTarget ) {
            return [ $mainQueryBuilder ];
        }
 
        $useVirtualDomains = $this->shouldUseVirtualDomains();
 
        $queries = [];
        foreach ( $this->linkTables as $linkTable ) {
            $queryBuilder = clone $mainQueryBuilder;
 
            if ( $this->linkDirection === self::LINKS_TO ) {
                $ok = $useVirtualDomains
                    ? $this->applyLinksToFilter( $queryBuilder, $this->linkTarget, $linkTable )
                    : $this->applyLinksToCondition( $queryBuilder, $this->linkTarget, $linkTable );
            } else {
                $ok = $useVirtualDomains
                    ? $this->applyLinksFromFilter( $queryBuilder, $this->linkTarget, $linkTable )
                    : $this->applyLinksFromCondition( $queryBuilder, $this->linkTarget, $linkTable );
            }
 
            if ( $ok ) {
                $queries[] = $queryBuilder;
            }
        }
        return $queries;
    }
 
    private function shouldUseVirtualDomains(): bool {
        return isset( $this->virtualDomainsMapping[LinksTable::VIRTUAL_DOMAIN] );
    }
 
    private function applyLinksToCondition(
        SelectQueryBuilder $queryBuilder,
        PageIdentity $page,
        string $linkTable
    ): bool {
        $prefix = self::LINK_TABLE_PREFIXES[$linkTable];
        $queryBuilder->join( $linkTable, null, "rc_cur_id = {$prefix}_from" );
        if ( $linkTable === 'imagelinks' ) {
            // The imagelinks table has no xx_namespace field and has xx_to instead of xx_target_id
            if ( $page->getNamespace() !== NS_FILE ) {
                // No imagelinks to a non-image page
                return false;
            }
            $title = TitleValue::newFromPage( $page );
            $cond = MediaWikiServices::getInstance()->getLinksMigration()->getLinksConditions( 'imagelinks', $title );
            $queryBuilder->where( $cond );
        } else {
            $linkTarget = $page instanceof LinkTarget ? $page : TitleValue::newFromPage( $page );
            $targetId = $this->linkTargetLookup->getLinkTargetId( $linkTarget );
            if ( !$targetId ) {
                return false;
            }
            $queryBuilder->where( $this->db->expr( "{$prefix}_target_id", '=', $targetId ) );
        }
        return true;
    }
 
    private function applyLinksToFilter(
        SelectQueryBuilder $queryBuilder,
        PageIdentity $page,
        string $linkTable
    ): bool {
        $linkTarget = $page instanceof LinkTarget ? $page : TitleValue::newFromPage( $page );
        $targetId = $this->linkTargetLookup->getLinkTargetId( $linkTarget );
        if ( !$targetId ) {
            return false;
        }
 
        $connProvider = MediaWikiServices::getInstance()->getConnectionProvider();
        $dbr = $connProvider->getReplicaDatabase( LinksTable::VIRTUAL_DOMAIN );
        $prefix = self::LINK_TABLE_PREFIXES[$linkTable];
 
        $res = $dbr->newSelectQuerybuilder()
            ->select( "{$prefix}_from" )
            ->from( $linkTable )
            ->where( [ "{$prefix}_target_id" => $targetId ] )
            ->limit( 5000 )
            ->caller( __METHOD__ )
            ->fetchFieldValues();
 
        if ( $res === [] ) {
            return false;
        }
 
        $queryBuilder->where( [ 'rc_cur_id' => $res ] );
 
        return true;
    }
 
    private function applyLinksFromCondition(
        SelectQueryBuilder $queryBuilder,
        PageIdentity $page,
        string $linkTable
    ): bool {
        if ( !$page->getId() ) {
            // No links from a non-existent page
            return false;
        }
        $prefix = self::LINK_TABLE_PREFIXES[$linkTable];
        $queryBuilder
            ->where( [ "{$prefix}_from" => $page->getId() ] )
            ->join( 'linktarget', null, [ 'rc_namespace = lt_namespace', 'rc_title = lt_title' ] )
            ->join( $linkTable, null, "{$prefix}_target_id = lt_id" );
        return true;
    }
 
    private function applyLinksFromFilter(
        SelectQueryBuilder $queryBuilder,
        PageIdentity $page,
        string $linkTable
    ): bool {
        if ( !$page->getId() ) {
            // No links from a non-existent page
            return false;
        }
 
        $connProvider = MediaWikiServices::getInstance()->getConnectionProvider();
        $dbr = $connProvider->getReplicaDatabase( LinksTable::VIRTUAL_DOMAIN );
        $prefix = self::LINK_TABLE_PREFIXES[$linkTable];
 
        $res = $dbr->newSelectQuerybuilder()
            ->select( 'page_id' )
            ->from( 'linktarget' )
            ->join( $linkTable, null, "{$prefix}_target_id = lt_id" )
            ->join( 'page', null, [ 'lt_namespace = page_namespace', 'lt_title = page_title' ] )
            ->where( [ "{$prefix}_from" => $page->getId() ] )
            ->limit( 5000 )
            ->caller( __METHOD__ )
            ->fetchFieldValues();
 
        if ( $res === [] ) {
            return false;
        }
 
        $queryBuilder->where( [ 'rc_cur_id' => $res ] );
 
        return true;
    }
 
    private function maybeEmulateUnion( $queries ) {
        if ( !$queries ) {
            return [];
        } elseif ( count( $queries ) === 1 ) {
            return $queries[0]->fetchResultSet();
        } else {
            $rows = [];
            $this->emulateUnion( $queries, $this->limit, $rows );
            return $rows;
        }
    }
 
    private function emulateUnion( array $queries, ?int $limit, &$rows ) {
        if ( !$this->preparedEmulatedUnion ) {
            throw new LogicException(
                'emulateUnion() was called but not prepareEmulatedUnion()' );
        }
        $unsortedRows = [];
        foreach ( $queries as $query ) {
            foreach ( $query->fetchResultSet() as $row ) {
                $unsortedRows[] = $row;
            }
        }
        $this->sortAndTruncate( $unsortedRows, $limit, $rows );
    }
 
    public function sortAndTruncate( array $inRows, ?int $limit, &$outRows ) {
        usort( $inRows, static fn ( $a, $b ) =>
            $b->rc_timestamp <=> $a->rc_timestamp ?:
            $b->rc_id <=> $a->rc_id
        );
        // Remove duplicates and slice
        $prevId = null;
        $numOut = 0;
        foreach ( $inRows as $row ) {
            if ( $prevId !== $row->rc_id ) {
                $outRows[] = $row;
                $numOut++;
                if ( $numOut === $limit ) {
                    break;
                }
            }
            $prevId = $row->rc_id;
        }
    }
 
    private function shouldDoPartitioning(): bool {
        return $this->forcePartitioning
            || ( $this->enablePartitioning
                && $this->limit !== null
                && $this->minTimestamp !== null
                && $this->joinOrderHint === self::JOIN_ORDER_OTHER
                && $this->sort === self::SORT_TIMESTAMP_DESC
                && $this->estimateSize() > self::PARTITION_THRESHOLD
            );
    }
 
    private function estimateSize() {
        $now = ConvertibleTimestamp::time();
        $min = (int)ConvertibleTimestamp::convert( TS::UNIX, $this->minTimestamp );
        $period = min( $now - $min, $this->rcMaxAge );
        return $this->rcStats->getIdDelta() * $this->density * $period;
    }
 
    private function doPartitionUnion( array $queries ) {
        if ( !$this->preparedEmulatedUnion ) {
            throw new LogicException(
                'doPartitionUnion() was called but not prepareEmulatedUnion()' );
        }
        $unsortedRows = [];
        foreach ( $queries as $query ) {
            $this->doPartitionQuery( $query, $unsortedRows );
        }
        if ( count( $queries ) > 1 ) {
            $rows = [];
            $this->sortAndTruncate( $unsortedRows, $this->limit, $rows );
            return $rows;
        } else {
            return $unsortedRows;
        }
    }
 
    private function doPartitionQuery( SelectQueryBuilder $sqb, &$rows ) {
        $now = ConvertibleTimestamp::time();
        $minTime = (int)ConvertibleTimestamp::convert( TS::UNIX,
            $this->minTimestamp ?? $now - $this->rcMaxAge );
        $limit = $this->limit ?? 10_000;
        $rcSize = $this->rcStats->getIdDelta();
 
        $this->logger->debug( 'Beginning partition request with density={density}, period={period}',
            [
                'period' => $now - $minTime,
                'limit' => $limit,
                'density' => $this->density,
                'rcSize' => $rcSize,
            ]
        );
 
        $partitioner = new TimestampRangePartitioner( $minTime, $now, $limit,
            null, $this->density, $rcSize, $this->rcMaxAge );
        do {
            [ $min, $max, $limit ] = $partitioner->getNextPartition();
 
            $partitionQuery = clone $sqb;
            if ( $min !== null ) {
                $partitionQuery->where( $this->db->expr(
                    'rc_timestamp', '>=', $this->db->timestamp( $min ) ) );
            }
            if ( $max !== null ) {
                $partitionQuery->where( $this->db->expr(
                    'rc_timestamp', '<=', $this->db->timestamp( $max ) ) );
            }
            $partitionQuery->limit( $limit );
 
            $row = null;
            $res = $partitionQuery->fetchResultSet();
            foreach ( $res as $row ) {
                $rows[] = $row;
            }
            $partitioner->notifyResult(
                $row ? (int)ConvertibleTimestamp::convert( TS::UNIX, $row->rc_timestamp ) : null,
                $res->numRows()
            );
        } while ( !$partitioner->isDone() );
 
        $m = $partitioner->getMetrics();
        $this->logger->debug( 'Finished partition request: ' .
            'got {actualRows} rows in {queryCount} queries, period={actualPeriod}',
            $m
        );
 
        $this->statsFactory->getCounter( 'ChangesListQuery_partition_queries_total' )
            ->incrementBy( $m['queryCount'] );
        $this->statsFactory->getCounter( 'ChangesListQuery_partition_requests_total' )
            ->increment();
        $this->statsFactory->getCounter( 'ChangesListQuery_partition_rows_total' )
            ->incrementBy( $m['actualRows' ] );
        $this->statsFactory->getCounter( 'ChangesListQuery_partition_overrun_total' )
            ->incrementBy(
                $m['actualRows'] * ( $m['queryPeriod'] / ( $m['actualPeriod'] ?: 1 ) - 1 )
            );
    }
 
    private function getHighlightsFromRow( stdClass $row ) {
        $activeHighlights = [];
        foreach ( $this->highlights as $name => $highlights ) {
            foreach ( $highlights as $hl ) {
                $module = $this->getFilter( $hl->moduleName );
                if ( $module->evaluate( $row, $hl->value ) === $hl->sense ) {
                    $activeHighlights[$name] = true;
                }
            }
        }
        return $activeHighlights;
    }
 
    private function getFilter( string $name ): ChangesListCondition {
        if ( !isset( $this->filterModules[$name] ) ) {
            throw new InvalidArgumentException( "Unknown filter module \"$name\"" );
        }
        return $this->filterModules[$name];
    }
 
    public function where( IExpression $expr ): self {
        $this->conds[] = $expr;
        return $this;
    }
 
    public function caller( string $caller ): self {
        $this->caller = $caller;
        return $this;
    }
 
    public function joinForFields( string $table ): ChangesListJoinBuilder {
        return $this->getJoin( $table )->forFields( $this );
    }
 
    public function joinForConds( string $table ): ChangesListJoinBuilder {
        return $this->getJoin( $table )->forConds( $this );
    }
 
    private function getJoin( string $name ): ChangesListJoinModule {
        if ( !isset( $this->joinModules[$name] ) ) {
            throw new InvalidArgumentException( "Unknown join module \"$name\"" );
        }
        return $this->joinModules[$name];
    }
 
    public function distinct(): QueryBackend {
        $this->distinct = true;
        return $this;
    }
 
    public function registerFilter( $name, ChangesListCondition $module ) {
        $this->filterModules[$name] = $module;
    }
 
}