LBFactory.php

Go to the documentation of this file.

<?php
namespace Wikimedia\Rdbms;
 
use Exception;
use Generator;
use Psr\Log\LoggerInterface;
use Psr\Log\NullLogger;
use RuntimeException;
use Throwable;
use Wikimedia\ObjectCache\BagOStuff;
use Wikimedia\ObjectCache\EmptyBagOStuff;
use Wikimedia\ObjectCache\WANObjectCache;
use Wikimedia\RequestTimeout\CriticalSectionProvider;
use Wikimedia\ScopedCallback;
use Wikimedia\Stats\StatsFactory;
use Wikimedia\Telemetry\NoopTracer;
use Wikimedia\Telemetry\TracerInterface;
 
abstract class LBFactory implements ILBFactory {
    private $csProvider;
    private $profiler;
    private $trxProfiler;
    private $tracer;
    private $statsFactory;
    private $logger;
    private $errorLogger;
    private $deprecationLogger;
 
    protected $chronologyProtector;
    protected $srvCache;
    protected $wanCache;
    protected $localDomain;
 
    private $cliMode;
    private $agent;
 
    private $tableAliases = [];
    protected $domainAliases = [];
    protected array $virtualDomainsMapping = [];
    protected array $virtualDomains = [];
    private $replicationWaitCallbacks = [];
 
    private $ticket;
    private $trxRoundFname = null;
    private $trxRoundStage = self::ROUND_CURSORY;
    private $replicationWaitTimeout;
 
    protected $readOnlyReason = false;
 
    private $defaultGroup = null;
    private bool $shuffleSharding = false;
    private ?string $uniqueIdentifier = null;
 
    private const ROUND_CURSORY = 'cursory';
    private const ROUND_BEGINNING = 'within-begin';
    private const ROUND_COMMITTING = 'within-commit';
    private const ROUND_ROLLING_BACK = 'within-rollback';
    private const ROUND_COMMIT_CALLBACKS = 'within-commit-callbacks';
    private const ROUND_ROLLBACK_CALLBACKS = 'within-rollback-callbacks';
    private const ROUND_ROLLBACK_SESSIONS = 'within-rollback-session';
 
    private $configCallback = null;
 
    public function __construct( array $conf ) {
        $this->configure( $conf );
 
        if ( isset( $conf['configCallback'] ) ) {
            $this->configCallback = $conf['configCallback'];
        }
    }
 
    protected function configure( array $conf ): void {
        $this->localDomain = isset( $conf['localDomain'] )
            ? DatabaseDomain::newFromId( $conf['localDomain'] )
            : DatabaseDomain::newUnspecified();
 
        if ( isset( $conf['readOnlyReason'] ) && is_string( $conf['readOnlyReason'] ) ) {
            $this->readOnlyReason = $conf['readOnlyReason'];
        }
 
        $this->chronologyProtector = $conf['chronologyProtector'] ?? new ChronologyProtector();
        $this->srvCache = $conf['srvCache'] ?? new EmptyBagOStuff();
        $this->wanCache = $conf['wanCache'] ?? WANObjectCache::newEmpty();
 
        $this->logger = $conf['logger'] ?? new NullLogger();
        $this->errorLogger = $conf['errorLogger'] ?? static function ( Throwable $e ) {
                trigger_error( get_class( $e ) . ': ' . $e->getMessage(), E_USER_WARNING );
        };
        $this->deprecationLogger = $conf['deprecationLogger'] ?? static function ( $msg ) {
                trigger_error( $msg, E_USER_DEPRECATED );
        };
 
        $this->profiler = $conf['profiler'] ?? null;
        $this->trxProfiler = $conf['trxProfiler'] ?? new TransactionProfiler();
        $this->statsFactory = $conf['statsFactory'] ?? StatsFactory::newNull();
        $this->tracer = $conf['tracer'] ?? new NoopTracer();
 
        $this->csProvider = $conf['criticalSectionProvider'] ?? null;
 
        $this->cliMode = $conf['cliMode'] ?? ( PHP_SAPI === 'cli' || PHP_SAPI === 'phpdbg' );
        $this->agent = $conf['agent'] ?? '';
        $this->replicationWaitTimeout = $this->cliMode ? 60 : 1;
        $this->virtualDomainsMapping = $conf['virtualDomainsMapping'] ?? [];
        $this->virtualDomains = $conf['virtualDomains'] ?? [];
 
        $this->shuffleSharding = $conf['shuffleSharding'] ?? false;
        $this->uniqueIdentifier = $conf['uniqueIdentifier'] ?? null;
 
        static $nextTicket;
        $this->ticket = $nextTicket = ( is_int( $nextTicket ) ? $nextTicket++ : mt_rand() );
    }
 
    public function destroy() {
        $scope = ScopedCallback::newScopedIgnoreUserAbort();
 
        foreach ( $this->getLBsForOwner() as $lb ) {
            $lb->disable( __METHOD__ );
        }
    }
 
    public function autoReconfigure(): void {
        if ( !$this->configCallback ) {
            return;
        }
 
        $conf = ( $this->configCallback )();
        if ( $conf ) {
            $this->reconfigure( $conf );
        }
    }
 
    public function reconfigure( array $conf ): void {
        if ( !$conf ) {
            return;
        }
 
        foreach ( $this->getLBsForOwner() as $lb ) {
            $lb->reconfigure( $conf );
        }
    }
 
    public function getLocalDomainID(): string {
        return $this->localDomain->getId();
    }
 
    public function shutdown(
        $flags = self::SHUTDOWN_NORMAL,
        ?callable $workCallback = null,
        &$cpIndex = null,
        &$cpClientId = null
    ) {
        $scope = ScopedCallback::newScopedIgnoreUserAbort();
 
        if ( ( $flags & self::SHUTDOWN_NO_CHRONPROT ) != self::SHUTDOWN_NO_CHRONPROT ) {
            // Remark all of the relevant DB primary positions
            foreach ( $this->getLBsForOwner() as $lb ) {
                if ( $lb->hasPrimaryConnection() ) {
                    $this->chronologyProtector->stageSessionPrimaryPos( $lb );
                }
            }
            // Write the positions to the persistent stash
            $this->chronologyProtector->persistSessionReplicationPositions( $cpIndex );
            $this->logger->debug( __METHOD__ . ': finished ChronologyProtector shutdown' );
        }
        $cpClientId = $this->chronologyProtector->getClientId();
 
        $this->commitPrimaryChanges( __METHOD__ );
 
        $this->logger->debug( 'LBFactory shutdown completed' );
    }
 
    public function getAllLBs() {
        foreach ( $this->getLBsForOwner() as $lb ) {
            yield $lb;
        }
    }
 
    abstract protected function getLBsForOwner();
 
    public function flushReplicaSnapshots( $fname = __METHOD__ ) {
        if ( $this->trxRoundFname !== null && $this->trxRoundFname !== $fname ) {
            $this->logger->warning(
                "$fname: transaction round '{$this->trxRoundFname}' still running",
                [ 'exception' => new RuntimeException() ]
            );
        }
        foreach ( $this->getLBsForOwner() as $lb ) {
            $lb->flushReplicaSnapshots( $fname );
        }
    }
 
    final public function beginPrimaryChanges( $fname = __METHOD__ ) {
        $this->assertTransactionRoundStage( self::ROUND_CURSORY );
        $scope = ScopedCallback::newScopedIgnoreUserAbort();
 
        foreach ( $this->getLBsForOwner() as $lb ) {
            $lb->flushReplicaSnapshots( $fname );
        }
 
        $this->trxRoundStage = self::ROUND_BEGINNING;
        if ( $this->trxRoundFname !== null ) {
            throw new DBTransactionError(
                null,
                "$fname: transaction round '{$this->trxRoundFname}' already started"
            );
        }
        $this->trxRoundFname = $fname;
        // Flush snapshots and appropriately set DBO_TRX on primary connections
        foreach ( $this->getLBsForOwner() as $lb ) {
            $lb->beginPrimaryChanges( $fname );
        }
        $this->trxRoundStage = self::ROUND_CURSORY;
    }
 
    final public function commitPrimaryChanges( $fname = __METHOD__, int $maxWriteDuration = 0 ) {
        $this->assertTransactionRoundStage( self::ROUND_CURSORY );
        $scope = ScopedCallback::newScopedIgnoreUserAbort();
 
        $this->trxRoundStage = self::ROUND_COMMITTING;
        if ( $this->trxRoundFname !== null && $this->trxRoundFname !== $fname ) {
            throw new DBTransactionError(
                null,
                "$fname: transaction round '{$this->trxRoundFname}' still running"
            );
        }
        // Run pre-commit callbacks and suppress post-commit callbacks, aborting on failure
        do {
            $count = 0; // number of callbacks executed this iteration
            foreach ( $this->getLBsForOwner() as $lb ) {
                $count += $lb->finalizePrimaryChanges( $fname );
            }
        } while ( $count > 0 );
        $this->trxRoundFname = null;
        // Perform pre-commit checks, aborting on failure
        foreach ( $this->getLBsForOwner() as $lb ) {
            $lb->approvePrimaryChanges( $maxWriteDuration, $fname );
        }
        // Log the DBs and methods involved in multi-DB transactions
        $this->logIfMultiDbTransaction();
        // Actually perform the commit on all primary DB connections and revert DBO_TRX
        foreach ( $this->getLBsForOwner() as $lb ) {
            $lb->commitPrimaryChanges( $fname );
        }
        // Run all post-commit callbacks in a separate step
        $this->trxRoundStage = self::ROUND_COMMIT_CALLBACKS;
        $e = $this->executePostTransactionCallbacks();
        $this->trxRoundStage = self::ROUND_CURSORY;
        // Throw any last post-commit callback error
        if ( $e instanceof Exception ) {
            // FIXME: The possible checked exception types should be better documented
            // @phan-suppress-next-line PhanThrowTypeMismatch
            throw $e;
        }
 
        foreach ( $this->getLBsForOwner() as $lb ) {
            $lb->flushReplicaSnapshots( $fname );
        }
    }
 
    final public function rollbackPrimaryChanges( $fname = __METHOD__ ) {
        $scope = ScopedCallback::newScopedIgnoreUserAbort();
 
        $this->trxRoundStage = self::ROUND_ROLLING_BACK;
        $this->trxRoundFname = null;
        // Actually perform the rollback on all primary DB connections and revert DBO_TRX
        foreach ( $this->getLBsForOwner() as $lb ) {
            $lb->rollbackPrimaryChanges( $fname );
        }
        // Run all post-commit callbacks in a separate step
        $this->trxRoundStage = self::ROUND_ROLLBACK_CALLBACKS;
        $this->executePostTransactionCallbacks();
        $this->trxRoundStage = self::ROUND_CURSORY;
 
        foreach ( $this->getLBsForOwner() as $lb ) {
            $lb->flushReplicaSnapshots( $fname );
        }
    }
 
    final public function flushPrimarySessions( $fname = __METHOD__ ) {
        $scope = ScopedCallback::newScopedIgnoreUserAbort();
 
        // Release named locks and table locks on all primary DB connections
        $this->trxRoundStage = self::ROUND_ROLLBACK_SESSIONS;
        foreach ( $this->getLBsForOwner() as $lb ) {
            $lb->flushPrimarySessions( $fname );
        }
        $this->trxRoundStage = self::ROUND_CURSORY;
    }
 
    private function executePostTransactionCallbacks() {
        $fname = __METHOD__;
        // Run all post-commit callbacks until new ones stop getting added
        $e = null; // first callback exception
        $iterations = 0;
        do {
            // Run any callbacks on tracked load balancers
            foreach ( $this->getLBsForOwner() as $lb ) {
                $ex = $lb->runPrimaryTransactionIdleCallbacks( $fname );
                $e = $e ?: $ex;
            }
            // T392913: log and break when this method seems to be in an obviously broken loop
            if ( ++$iterations >= 32 ) {
                throw new DBTransactionError(
                    null,
                    "Aborting likely infinite callback loop due to unresolvable pending writes"
                );
            }
        } while ( $this->hasPrimaryChanges() );
        // Run all listener callbacks once
        foreach ( $this->getLBsForOwner() as $lb ) {
            $ex = $lb->runPrimaryTransactionListenerCallbacks( $fname );
            $e = $e ?: $ex;
        }
 
        return $e;
    }
 
    public function hasTransactionRound() {
        // TODO: check for implicit rounds or rename and check for implicit rounds with writes?
        return ( $this->trxRoundFname !== null );
    }
 
    public function isReadyForRoundOperations() {
        return ( $this->trxRoundStage === self::ROUND_CURSORY );
    }
 
    private function logIfMultiDbTransaction() {
        $callersByDB = [];
        foreach ( $this->getLBsForOwner() as $lb ) {
            $primaryName = $lb->getServerName( ServerInfo::WRITER_INDEX );
            $callers = $lb->pendingPrimaryChangeCallers();
            if ( $callers ) {
                $callersByDB[$primaryName] = $callers;
            }
        }
 
        if ( count( $callersByDB ) >= 2 ) {
            $dbs = implode( ', ', array_keys( $callersByDB ) );
            $msg = "Multi-DB transaction [{$dbs}]:\n";
            foreach ( $callersByDB as $db => $callers ) {
                $msg .= "$db: " . implode( '; ', $callers ) . "\n";
            }
            $this->logger->info( $msg );
        }
    }
 
    public function hasPrimaryChanges() {
        foreach ( $this->getLBsForOwner() as $lb ) {
            if ( $lb->hasPrimaryChanges() ) {
                return true;
            }
        }
        return false;
    }
 
    public function laggedReplicaUsed() {
        foreach ( $this->getLBsForOwner() as $lb ) {
            if ( $lb->laggedReplicaUsed() ) {
                return true;
            }
        }
        return false;
    }
 
    public function hasOrMadeRecentPrimaryChanges( $age = null ) {
        foreach ( $this->getLBsForOwner() as $lb ) {
            if ( $lb->hasOrMadeRecentPrimaryChanges( $age ) ) {
                return true;
            }
        }
        return false;
    }
 
    public function waitForReplication( array $opts = [] ) {
        $opts += [
            'timeout' => $this->replicationWaitTimeout,
            'ifWritesSince' => null
        ];
 
        $lbs = [];
        foreach ( $this->getLBsForOwner() as $lb ) {
            $lbs[] = $lb;
        }
 
        // Get all the primary DB positions of applicable DBs right now.
        // This can be faster since waiting on one cluster reduces the
        // time needed to wait on the next clusters.
        $primaryPositions = array_fill( 0, count( $lbs ), false );
        foreach ( $lbs as $i => $lb ) {
            if (
                // No writes to wait on getting replicated
                !$lb->hasPrimaryConnection() ||
                // No replication; avoid getPrimaryPos() permissions errors (T29975)
                !$lb->hasStreamingReplicaServers() ||
                // No writes since the last replication wait
                (
                    $opts['ifWritesSince'] &&
                    $lb->lastPrimaryChangeTimestamp() < $opts['ifWritesSince']
                )
            ) {
                continue; // no need to wait
            }
 
            $primaryPositions[$i] = $lb->getPrimaryPos();
        }
 
        // Run any listener callbacks *after* getting the DB positions. The more
        // time spent in the callbacks, the less time is spent in waitForAll().
        foreach ( $this->replicationWaitCallbacks as $callback ) {
            $callback();
        }
 
        $failed = [];
        foreach ( $lbs as $i => $lb ) {
            if ( $primaryPositions[$i] ) {
                // The RDBMS may not support getPrimaryPos()
                if ( !$lb->waitForAll( $primaryPositions[$i], $opts['timeout'] ) ) {
                    $failed[] = $lb->getServerName( ServerInfo::WRITER_INDEX );
                }
            }
        }
 
        return !$failed;
    }
 
    public function setWaitForReplicationListener( $name, ?callable $callback = null ) {
        if ( $callback ) {
            $this->replicationWaitCallbacks[$name] = $callback;
        } else {
            unset( $this->replicationWaitCallbacks[$name] );
        }
    }
 
    public function getEmptyTransactionTicket( $fname ) {
        if ( $this->hasPrimaryChanges() ) {
            $this->logger->error(
                __METHOD__ . ": $fname does not have outer scope",
                [ 'exception' => new RuntimeException() ]
            );
 
            return null;
        }
 
        return $this->ticket;
    }
 
    public function getPrimaryDatabase( $domain = false ): IDatabase {
        return $this->getMappedDatabase( DB_PRIMARY, [], $domain );
    }
 
    public function getAutoCommitPrimaryConnection( $domain = false ): IDatabase {
        return $this->getLoadBalancer( $domain )
            ->getConnection( DB_PRIMARY, [], $this->getMappedDomain( $domain ), ILoadBalancer::CONN_TRX_AUTOCOMMIT );
    }
 
    public function getReplicaDatabase( string|false $domain = false, $group = null ): IReadableDatabase {
        if ( $group === null ) {
            $groups = [];
        } else {
            $groups = [ $group ];
        }
        return $this->getMappedDatabase( DB_REPLICA, $groups, $domain );
    }
 
    public function getLoadBalancer( $domain = false ): ILoadBalancer {
        if ( $domain !== false && in_array( $domain, $this->virtualDomains ) ) {
            if ( isset( $this->virtualDomainsMapping[$domain] ) ) {
                $config = $this->virtualDomainsMapping[$domain];
                if ( isset( $config['cluster'] ) ) {
                    return $this->getExternalLB( $config['cluster'] );
                }
                $domain = $config['db'];
            } else {
                // It's not configured, assume local db.
                $domain = false;
            }
        }
        return $this->getMainLB( $domain );
    }
 
    private function getMappedDatabase( $index, $groups, string|false $domain ) {
        return $this->getLoadBalancer( $domain )
            ->getConnection( $index, $groups, $this->getMappedDomain( $domain ) );
    }
 
    public function getMappedDomain( string|false $domain ): string|false {
        if ( $domain !== false && in_array( $domain, $this->virtualDomains ) ) {
            return $this->virtualDomainsMapping[$domain]['db'] ?? false;
        } else {
            return $domain;
        }
    }
 
    public function isLocalDomain( $domain ) {
        if ( $domain !== false && in_array( $domain, $this->virtualDomains ) ) {
            if ( isset( $this->virtualDomainsMapping[$domain] ) ) {
                $config = $this->virtualDomainsMapping[$domain];
                if ( isset( $config['cluster'] ) ) {
                    // In an external cluster
                    return false;
                }
                $domain = $config['db'];
            } else {
                // Unconfigured virtual domain is always local
                return true;
            }
        }
        return $domain === false || $domain === $this->getLocalDomainID();
    }
 
    public function isSharedVirtualDomain( $domain ) {
        if ( $domain !== false
            && in_array( $domain, $this->virtualDomains )
            && isset( $this->virtualDomainsMapping[$domain] )
        ) {
            return $this->virtualDomainsMapping[$domain]['db'] !== false;
        }
        return false;
    }
 
    final public function commitAndWaitForReplication( $fname, $ticket, array $opts = [] ) {
        if ( $ticket !== $this->ticket ) {
            $this->logger->error(
                __METHOD__ . ": $fname does not have outer scope ($ticket vs {$this->ticket})",
                [ 'exception' => new RuntimeException() ]
            );
 
            return false;
        }
 
        // The transaction owner and any caller with the empty transaction ticket can commit
        // so that getEmptyTransactionTicket() callers don't risk seeing DBTransactionError.
        if ( $this->trxRoundFname !== null && $fname !== $this->trxRoundFname ) {
            $this->logger->info( "$fname: committing on behalf of {$this->trxRoundFname}" );
            $fnameEffective = $this->trxRoundFname;
        } else {
            $fnameEffective = $fname;
        }
 
        $this->commitPrimaryChanges( $fnameEffective );
        $waitSucceeded = $this->waitForReplication( $opts );
        // If a nested caller committed on behalf of $fname, start another empty $fname
        // transaction, leaving the caller with the same empty transaction state as before.
        if ( $fnameEffective !== $fname ) {
            $this->beginPrimaryChanges( $fnameEffective );
        }
 
        return $waitSucceeded;
    }
 
    public function disableChronologyProtection() {
        $this->chronologyProtector->setEnabled( false );
    }
 
    public function setDefaultGroupName( string $defaultGroup ): void {
        // for future LBs
        $this->defaultGroup = $defaultGroup;
 
        // For existing LBs
        foreach ( $this->getLBsForOwner() as $lb ) {
            $lb->setDefaultGroupName( $defaultGroup );
        }
    }
 
    final protected function baseLoadBalancerParams() {
        if ( $this->trxRoundStage === self::ROUND_COMMIT_CALLBACKS ) {
            $initStage = ILoadBalancerForOwner::STAGE_POSTCOMMIT_CALLBACKS;
        } elseif ( $this->trxRoundStage === self::ROUND_ROLLBACK_CALLBACKS ) {
            $initStage = ILoadBalancerForOwner::STAGE_POSTROLLBACK_CALLBACKS;
        } else {
            $initStage = null;
        }
 
        return [
            'localDomain' => $this->localDomain,
            'readOnlyReason' => $this->readOnlyReason,
            'srvCache' => $this->srvCache,
            'wanCache' => $this->wanCache,
            'profiler' => $this->profiler,
            'trxProfiler' => $this->trxProfiler,
            'tracer' => $this->tracer,
            'logger' => $this->logger,
            'errorLogger' => $this->errorLogger,
            'deprecationLogger' => $this->deprecationLogger,
            'statsFactory' => $this->statsFactory,
            'cliMode' => $this->cliMode,
            'agent' => $this->agent,
            'defaultGroup' => $this->defaultGroup,
            'chronologyProtector' => $this->chronologyProtector,
            'roundStage' => $initStage,
            'criticalSectionProvider' => $this->csProvider,
            'shuffleSharding' => $this->shuffleSharding,
            'uniqueIdentifier' => $this->uniqueIdentifier,
        ];
    }
 
    protected function initLoadBalancer( ILoadBalancerForOwner $lb ) {
        if ( $this->trxRoundFname !== null ) {
            $lb->beginPrimaryChanges( $this->trxRoundFname ); // set DBO_TRX
        }
 
        $lb->setTableAliases( $this->tableAliases );
        $lb->setDomainAliases( $this->domainAliases );
    }
 
    public function setTableAliases( array $aliases ) {
        $this->tableAliases = $aliases;
    }
 
    public function setDomainAliases( array $aliases ) {
        $this->domainAliases = $aliases;
    }
 
    public function getTransactionProfiler(): TransactionProfiler {
        return $this->trxProfiler;
    }
 
    public function setLocalDomainPrefix( $prefix ) {
        $this->localDomain = new DatabaseDomain(
            $this->localDomain->getDatabase(),
            $this->localDomain->getSchema(),
            $prefix
        );
 
        foreach ( $this->getLBsForOwner() as $lb ) {
            $lb->setLocalDomainPrefix( $prefix );
        }
    }
 
    public function redefineLocalDomain( $domain ) {
        $this->closeAll( __METHOD__ );
 
        $this->localDomain = DatabaseDomain::newFromId( $domain );
 
        foreach ( $this->getLBsForOwner() as $lb ) {
            $lb->redefineLocalDomain( $this->localDomain );
        }
    }
 
    public function closeAll( $fname = __METHOD__ ) {
        $scope = ScopedCallback::newScopedIgnoreUserAbort();
 
        foreach ( $this->getLBsForOwner() as $lb ) {
            $lb->closeAll( $fname );
        }
    }
 
    public function setAgentName( $agent ) {
        $this->agent = $agent;
    }
 
    public function hasStreamingReplicaServers() {
        foreach ( $this->getLBsForOwner() as $lb ) {
            if ( $lb->hasStreamingReplicaServers() ) {
                return true;
            }
        }
        return false;
    }
 
    public function setDefaultReplicationWaitTimeout( $seconds ) {
        $old = $this->replicationWaitTimeout;
        $this->replicationWaitTimeout = max( 1, (int)$seconds );
 
        return $old;
    }
 
    private function assertTransactionRoundStage( $stage ) {
        if ( $this->trxRoundStage !== $stage ) {
            $transactionName = $this->trxRoundFname ?? '<unknown>';
            throw new DBTransactionError(
                null,
                "Transaction '$transactionName' round stage must be '$stage' (not '{$this->trxRoundStage}')"
            );
        }
    }
}