MediaWiki  master
LoadBalancer.php
Go to the documentation of this file.
1 <?php
22 namespace Wikimedia\Rdbms;
23 
27 use BagOStuff;
28 use EmptyBagOStuff;
29 use WANObjectCache;
30 use ArrayUtils;
31 use LogicException;
35 use Exception;
36 
42 class LoadBalancer implements ILoadBalancer {
44  private $loadMonitor;
48  private $srvCache;
50  private $wanCache;
52  private $profiler;
54  private $trxProfiler;
56  private $replLogger;
58  private $connLogger;
60  private $queryLogger;
62  private $perfLogger;
64  private $errorLogger;
67 
69  private $localDomain;
70 
74  private $conns;
75 
77  private $servers;
79  private $groupLoads;
81  private $maxLagByIndex;
83  private $waitTimeout;
89  private $maxLag;
91  private $defaultGroup;
92 
94  private $hostname;
96  private $cliMode;
98  private $agent;
99 
101  private $tableAliases = [];
103  private $indexAliases = [];
107  private $tempTablesOnlyMode = [];
108 
110  private $trxRoundId = false;
112  private $trxRoundStage = self::ROUND_CURSORY;
116  private $readIndexByGroup = [];
118  private $waitForPos;
120  private $allowLagged = false;
122  private $laggedReplicaMode = false;
124  private $lastError = 'Unknown error';
126  private $readOnlyReason = false;
128  private $connectionCounter = 0;
130  private $disabled = false;
132  private $connectionAttempted = false;
133 
135  private $id;
137  private $ownerId;
138 
139  private static $INFO_SERVER_INDEX = 'serverIndex';
140  private static $INFO_AUTOCOMMIT_ONLY = 'autoCommitOnly';
141  private static $INFO_FORIEGN = 'foreign';
142  private static $INFO_FOREIGN_REF_COUNT = 'foreignPoolRefCount';
143 
145  const CONN_HELD_WARN_THRESHOLD = 10;
146 
148  const MAX_LAG_DEFAULT = 6;
150  const MAX_WAIT_DEFAULT = 10;
152  const TTL_CACHE_READONLY = 5;
153 
154  const KEY_LOCAL = 'local';
155  const KEY_FOREIGN_FREE = 'foreignFree';
156  const KEY_FOREIGN_INUSE = 'foreignInUse';
157 
158  const KEY_LOCAL_NOROUND = 'localAutoCommit';
159  const KEY_FOREIGN_FREE_NOROUND = 'foreignFreeAutoCommit';
160  const KEY_FOREIGN_INUSE_NOROUND = 'foreignInUseAutoCommit';
161 
163  const ROUND_CURSORY = 'cursory';
165  const ROUND_FINALIZED = 'finalized';
167  const ROUND_APPROVED = 'approved';
169  const ROUND_COMMIT_CALLBACKS = 'commit-callbacks';
171  const ROUND_ROLLBACK_CALLBACKS = 'rollback-callbacks';
173  const ROUND_ERROR = 'error';
174 
175  public function __construct( array $params ) {
176  if ( !isset( $params['servers'] ) || !count( $params['servers'] ) ) {
177  throw new InvalidArgumentException( 'Missing or empty "servers" parameter' );
178  }
179 
180  $localDomain = isset( $params['localDomain'] )
181  ? DatabaseDomain::newFromId( $params['localDomain'] )
183  $this->setLocalDomain( $localDomain );
184 
185  $this->maxLag = $params['maxLag'] ?? self::MAX_LAG_DEFAULT;
186 
187  $listKey = -1;
188  $this->servers = [];
189  $this->groupLoads = [ self::GROUP_GENERIC => [] ];
190  foreach ( $params['servers'] as $i => $server ) {
191  if ( ++$listKey !== $i ) {
192  throw new UnexpectedValueException( 'List expected for "servers" parameter' );
193  }
194  $this->servers[$i] = $server;
195  foreach ( ( $server['groupLoads'] ?? [] ) as $group => $ratio ) {
196  $this->groupLoads[$group][$i] = $ratio;
197  }
198  $this->groupLoads[self::GROUP_GENERIC][$i] = $server['load'];
199  $this->maxLagByIndex[$i] = $server['max lag'] ?? $this->maxLag;
200  }
201 
202  $this->waitTimeout = $params['waitTimeout'] ?? self::MAX_WAIT_DEFAULT;
203 
204  $this->conns = self::newTrackedConnectionsArray();
205 
206  if ( isset( $params['readOnlyReason'] ) && is_string( $params['readOnlyReason'] ) ) {
207  $this->readOnlyReason = $params['readOnlyReason'];
208  }
209 
210  $this->loadMonitorConfig = $params['loadMonitor'] ?? [ 'class' => 'LoadMonitorNull' ];
211  $this->loadMonitorConfig += [ 'lagWarnThreshold' => $this->maxLag ];
212 
213  $this->srvCache = $params['srvCache'] ?? new EmptyBagOStuff();
214  $this->wanCache = $params['wanCache'] ?? WANObjectCache::newEmpty();
215  $this->profiler = $params['profiler'] ?? null;
216  $this->trxProfiler = $params['trxProfiler'] ?? new TransactionProfiler();
217 
218  $this->errorLogger = $params['errorLogger'] ?? function ( Exception $e ) {
219  trigger_error( get_class( $e ) . ': ' . $e->getMessage(), E_USER_WARNING );
220  };
221  $this->deprecationLogger = $params['deprecationLogger'] ?? function ( $msg ) {
222  trigger_error( $msg, E_USER_DEPRECATED );
223  };
224  foreach ( [ 'replLogger', 'connLogger', 'queryLogger', 'perfLogger' ] as $key ) {
225  $this->$key = $params[$key] ?? new NullLogger();
226  }
227 
228  $this->hostname = $params['hostname'] ?? ( gethostname() ?: 'unknown' );
229  $this->cliMode = $params['cliMode'] ?? ( PHP_SAPI === 'cli' || PHP_SAPI === 'phpdbg' );
230  $this->agent = $params['agent'] ?? '';
231 
232  if ( isset( $params['chronologyCallback'] ) ) {
233  $this->chronologyCallback = $params['chronologyCallback'];
234  }
235 
236  if ( isset( $params['roundStage'] ) ) {
237  if ( $params['roundStage'] === self::STAGE_POSTCOMMIT_CALLBACKS ) {
238  $this->trxRoundStage = self::ROUND_COMMIT_CALLBACKS;
239  } elseif ( $params['roundStage'] === self::STAGE_POSTROLLBACK_CALLBACKS ) {
240  $this->trxRoundStage = self::ROUND_ROLLBACK_CALLBACKS;
241  }
242  }
243 
244  $group = $params['defaultGroup'] ?? self::GROUP_GENERIC;
245  $this->defaultGroup = isset( $this->groupLoads[$group] ) ? $group : self::GROUP_GENERIC;
246 
247  static $nextId;
248  $this->id = $nextId = ( is_int( $nextId ) ? $nextId++ : mt_rand() );
249  $this->ownerId = $params['ownerId'] ?? null;
250  }
251 
252  private static function newTrackedConnectionsArray() {
253  return [
254  // Connection were transaction rounds may be applied
255  self::KEY_LOCAL => [],
256  self::KEY_FOREIGN_INUSE => [],
257  self::KEY_FOREIGN_FREE => [],
258  // Auto-committing counterpart connections that ignore transaction rounds
259  self::KEY_LOCAL_NOROUND => [],
260  self::KEY_FOREIGN_INUSE_NOROUND => [],
261  self::KEY_FOREIGN_FREE_NOROUND => []
262  ];
263  }
264 
265  public function getLocalDomainID() {
266  return $this->localDomain->getId();
267  }
268 
269  public function resolveDomainID( $domain ) {
270  if ( $domain === $this->localDomainIdAlias || $domain === false ) {
271  // Local connection requested via some backwards-compatibility domain alias
272  return $this->getLocalDomainID();
273  }
274 
275  return (string)$domain;
276  }
277 
285  private function resolveGroups( $groups, $i ) {
286  // If a specific replica server was specified, then $groups makes no sense
287  if ( $i > 0 && $groups !== [] && $groups !== false ) {
288  $list = implode( ', ', (array)$groups );
289  throw new LogicException( "Query group(s) ($list) given with server index (#$i)" );
290  }
291 
292  if ( $groups === [] || $groups === false || $groups === $this->defaultGroup ) {
293  $resolvedGroups = [ $this->defaultGroup ]; // common case
294  } elseif ( is_string( $groups ) && isset( $this->groupLoads[$groups] ) ) {
295  $resolvedGroups = [ $groups, $this->defaultGroup ];
296  } elseif ( is_array( $groups ) ) {
297  $resolvedGroups = array_keys( array_flip( $groups ) + [ self::GROUP_GENERIC => 1 ] );
298  } else {
299  $resolvedGroups = [ $this->defaultGroup ];
300  }
301 
302  return $resolvedGroups;
303  }
304 
311  private function sanitizeConnectionFlags( $flags, $i, $domain ) {
312  // Whether an outside caller is explicitly requesting the master database server
313  if ( $i === self::DB_MASTER || $i === $this->getWriterIndex() ) {
314  $flags |= self::CONN_INTENT_WRITABLE;
315  }
316 
317  if ( ( $flags & self::CONN_TRX_AUTOCOMMIT ) == self::CONN_TRX_AUTOCOMMIT ) {
318  // Callers use CONN_TRX_AUTOCOMMIT to bypass REPEATABLE-READ staleness without
319  // resorting to row locks (e.g. FOR UPDATE) or to make small out-of-band commits
320  // during larger transactions. This is useful for avoiding lock contention.
321 
322  // Master DB server attributes (should match those of the replica DB servers)
323  $attributes = $this->getServerAttributes( $this->getWriterIndex() );
324  if ( $attributes[Database::ATTR_DB_LEVEL_LOCKING] ) {
325  // The RDBMS does not support concurrent writes (e.g. SQLite), so attempts
326  // to use separate connections would just cause self-deadlocks. Note that
327  // REPEATABLE-READ staleness is not an issue since DB-level locking means
328  // that transactions are Strict Serializable anyway.
329  $flags &= ~self::CONN_TRX_AUTOCOMMIT;
330  $type = $this->getServerType( $this->getWriterIndex() );
331  $this->connLogger->info( __METHOD__ . ": CONN_TRX_AUTOCOMMIT disallowed ($type)" );
332  } elseif ( isset( $this->tempTablesOnlyMode[$domain] ) ) {
333  // T202116: integration tests are active and queries should be all be using
334  // temporary clone tables (via prefix). Such tables are not visible accross
335  // different connections nor can there be REPEATABLE-READ snapshot staleness,
336  // so use the same connection for everything.
337  $flags &= ~self::CONN_TRX_AUTOCOMMIT;
338  }
339  }
340 
341  return $flags;
342  }
343 
349  private function enforceConnectionFlags( IDatabase $conn, $flags ) {
350  if ( ( $flags & self::CONN_TRX_AUTOCOMMIT ) == self::CONN_TRX_AUTOCOMMIT ) {
351  if ( $conn->trxLevel() ) { // sanity
352  throw new DBUnexpectedError(
353  $conn,
354  'Handle requested with CONN_TRX_AUTOCOMMIT yet it has a transaction'
355  );
356  }
357 
358  $conn->clearFlag( $conn::DBO_TRX ); // auto-commit mode
359  }
360  }
361 
367  private function getLoadMonitor() {
368  if ( !isset( $this->loadMonitor ) ) {
369  $compat = [
370  'LoadMonitor' => LoadMonitor::class,
371  'LoadMonitorNull' => LoadMonitorNull::class,
372  'LoadMonitorMySQL' => LoadMonitorMySQL::class,
373  ];
374 
375  $class = $this->loadMonitorConfig['class'];
376  if ( isset( $compat[$class] ) ) {
377  $class = $compat[$class];
378  }
379 
380  $this->loadMonitor = new $class(
381  $this, $this->srvCache, $this->wanCache, $this->loadMonitorConfig );
382  $this->loadMonitor->setLogger( $this->replLogger );
383  }
384 
385  return $this->loadMonitor;
386  }
387 
394  private function getRandomNonLagged( array $loads, $domain = false, $maxLag = INF ) {
395  $lags = $this->getLagTimes( $domain );
396 
397  # Unset excessively lagged servers
398  foreach ( $lags as $i => $lag ) {
399  if ( $i !== $this->getWriterIndex() ) {
400  # How much lag this server nominally is allowed to have
401  $maxServerLag = $this->servers[$i]['max lag'] ?? $this->maxLag; // default
402  # Constrain that futher by $maxLag argument
403  $maxServerLag = min( $maxServerLag, $maxLag );
404 
405  $host = $this->getServerName( $i );
406  if ( $lag === false && !is_infinite( $maxServerLag ) ) {
407  $this->replLogger->debug(
408  __METHOD__ .
409  ": server {host} is not replicating?", [ 'host' => $host ] );
410  unset( $loads[$i] );
411  } elseif ( $lag > $maxServerLag ) {
412  $this->replLogger->debug(
413  __METHOD__ .
414  ": server {host} has {lag} seconds of lag (>= {maxlag})",
415  [ 'host' => $host, 'lag' => $lag, 'maxlag' => $maxServerLag ]
416  );
417  unset( $loads[$i] );
418  }
419  }
420  }
421 
422  # Find out if all the replica DBs with non-zero load are lagged
423  $sum = 0;
424  foreach ( $loads as $load ) {
425  $sum += $load;
426  }
427  if ( $sum == 0 ) {
428  # No appropriate DB servers except maybe the master and some replica DBs with zero load
429  # Do NOT use the master
430  # Instead, this function will return false, triggering read-only mode,
431  # and a lagged replica DB will be used instead.
432  return false;
433  }
434 
435  if ( count( $loads ) == 0 ) {
436  return false;
437  }
438 
439  # Return a random representative of the remainder
440  return ArrayUtils::pickRandom( $loads );
441  }
442 
451  private function getConnectionIndex( $i, array $groups, $domain ) {
452  if ( $i === self::DB_MASTER ) {
453  $i = $this->getWriterIndex();
454  } elseif ( $i === self::DB_REPLICA ) {
455  foreach ( $groups as $group ) {
456  $groupIndex = $this->getReaderIndex( $group, $domain );
457  if ( $groupIndex !== false ) {
458  $i = $groupIndex; // group connection succeeded
459  break;
460  }
461  }
462  } elseif ( !isset( $this->servers[$i] ) ) {
463  throw new UnexpectedValueException( "Invalid server index index #$i" );
464  }
465 
466  if ( $i === self::DB_REPLICA ) {
467  $this->lastError = 'Unknown error'; // set here in case of worse failure
468  $this->lastError = 'No working replica DB server: ' . $this->lastError;
469  $this->reportConnectionError();
470  return null; // unreachable due to exception
471  }
472 
473  return $i;
474  }
475 
476  public function getReaderIndex( $group = false, $domain = false ) {
477  if ( $this->getServerCount() == 1 ) {
478  // Skip the load balancing if there's only one server
479  return $this->getWriterIndex();
480  }
481 
482  $group = is_string( $group ) ? $group : self::GROUP_GENERIC;
483 
484  $index = $this->getExistingReaderIndex( $group );
485  if ( $index >= 0 ) {
486  // A reader index was already selected and "waitForPos" was handled
487  return $index;
488  }
489 
490  // Use the server weight array for this load group
491  if ( isset( $this->groupLoads[$group] ) ) {
492  $loads = $this->groupLoads[$group];
493  } else {
494  $this->connLogger->info( __METHOD__ . ": no loads for group $group" );
495 
496  return false;
497  }
498 
499  // Scale the configured load ratios according to each server's load and state
500  $this->getLoadMonitor()->scaleLoads( $loads, $domain );
501 
502  // Pick a server to use, accounting for weights, load, lag, and "waitForPos"
503  $this->lazyLoadReplicationPositions(); // optimizes server candidate selection
504  list( $i, $laggedReplicaMode ) = $this->pickReaderIndex( $loads, $domain );
505  if ( $i === false ) {
506  // DB connection unsuccessful
507  return false;
508  }
509 
510  // If data seen by queries is expected to reflect the transactions committed as of
511  // or after a given replication position then wait for the DB to apply those changes
512  if ( $this->waitForPos && $i !== $this->getWriterIndex() && !$this->doWait( $i ) ) {
513  // Data will be outdated compared to what was expected
514  $laggedReplicaMode = true;
515  }
516 
517  // Cache the reader index for future DB_REPLICA handles
518  $this->setExistingReaderIndex( $group, $i );
519  // Record whether the generic reader index is in "lagged replica DB" mode
520  if ( $group === self::GROUP_GENERIC && $laggedReplicaMode ) {
521  $this->laggedReplicaMode = true;
522  }
523 
524  $serverName = $this->getServerName( $i );
525  $this->connLogger->debug( __METHOD__ . ": using server $serverName for group '$group'" );
526 
527  return $i;
528  }
529 
536  protected function getExistingReaderIndex( $group ) {
537  return $this->readIndexByGroup[$group] ?? -1;
538  }
539 
546  private function setExistingReaderIndex( $group, $index ) {
547  if ( $index < 0 ) {
548  throw new UnexpectedValueException( "Cannot set a negative read server index" );
549  }
550  $this->readIndexByGroup[$group] = $index;
551  }
552 
558  private function pickReaderIndex( array $loads, $domain = false ) {
559  if ( $loads === [] ) {
560  throw new InvalidArgumentException( "Server configuration array is empty" );
561  }
562 
564  $i = false;
566  $laggedReplicaMode = false;
567 
568  // Quickly look through the available servers for a server that meets criteria...
569  $currentLoads = $loads;
570  while ( count( $currentLoads ) ) {
571  if ( $this->allowLagged || $laggedReplicaMode ) {
572  $i = ArrayUtils::pickRandom( $currentLoads );
573  } else {
574  $i = false;
575  if ( $this->waitForPos && $this->waitForPos->asOfTime() ) {
576  $this->replLogger->debug( __METHOD__ . ": replication positions detected" );
577  // "chronologyCallback" sets "waitForPos" for session consistency.
578  // This triggers doWait() after connect, so it's especially good to
579  // avoid lagged servers so as to avoid excessive delay in that method.
580  $ago = microtime( true ) - $this->waitForPos->asOfTime();
581  // Aim for <= 1 second of waiting (being too picky can backfire)
582  $i = $this->getRandomNonLagged( $currentLoads, $domain, $ago + 1 );
583  }
584  if ( $i === false ) {
585  // Any server with less lag than it's 'max lag' param is preferable
586  $i = $this->getRandomNonLagged( $currentLoads, $domain );
587  }
588  if ( $i === false && count( $currentLoads ) ) {
589  // All replica DBs lagged. Switch to read-only mode
590  $this->replLogger->error(
591  __METHOD__ . ": all replica DBs lagged. Switch to read-only mode" );
592  $i = ArrayUtils::pickRandom( $currentLoads );
593  $laggedReplicaMode = true;
594  }
595  }
596 
597  if ( $i === false ) {
598  // pickRandom() returned false.
599  // This is permanent and means the configuration or the load monitor
600  // wants us to return false.
601  $this->connLogger->debug( __METHOD__ . ": pickRandom() returned false" );
602 
603  return [ false, false ];
604  }
605 
606  $serverName = $this->getServerName( $i );
607  $this->connLogger->debug( __METHOD__ . ": Using reader #$i: $serverName..." );
608 
609  // Get a connection to this server without triggering other server connections
610  $conn = $this->getServerConnection( $i, $domain, self::CONN_SILENCE_ERRORS );
611  if ( !$conn ) {
612  $this->connLogger->warning( __METHOD__ . ": Failed connecting to $i/$domain" );
613  unset( $currentLoads[$i] ); // avoid this server next iteration
614  $i = false;
615  continue;
616  }
617 
618  // Decrement reference counter, we are finished with this connection.
619  // It will be incremented for the caller later.
620  if ( $domain !== false ) {
621  $this->reuseConnection( $conn );
622  }
623 
624  // Return this server
625  break;
626  }
627 
628  // If all servers were down, quit now
629  if ( $currentLoads === [] ) {
630  $this->connLogger->error( __METHOD__ . ": all servers down" );
631  }
632 
633  return [ $i, $laggedReplicaMode ];
634  }
635 
636  public function waitFor( $pos ) {
637  $oldPos = $this->waitForPos;
638  try {
639  $this->waitForPos = $pos;
640  // If a generic reader connection was already established, then wait now
641  $i = $this->getExistingReaderIndex( self::GROUP_GENERIC );
642  if ( $i > 0 && !$this->doWait( $i ) ) {
643  $this->laggedReplicaMode = true;
644  }
645  // Otherwise, wait until a connection is established in getReaderIndex()
646  } finally {
647  // Restore the older position if it was higher since this is used for lag-protection
648  $this->setWaitForPositionIfHigher( $oldPos );
649  }
650  }
651 
652  public function waitForOne( $pos, $timeout = null ) {
653  $oldPos = $this->waitForPos;
654  try {
655  $this->waitForPos = $pos;
656 
657  $i = $this->getExistingReaderIndex( self::GROUP_GENERIC );
658  if ( $i <= 0 ) {
659  // Pick a generic replica DB if there isn't one yet
660  $readLoads = $this->groupLoads[self::GROUP_GENERIC];
661  unset( $readLoads[$this->getWriterIndex()] ); // replica DBs only
662  $readLoads = array_filter( $readLoads ); // with non-zero load
663  $i = ArrayUtils::pickRandom( $readLoads );
664  }
665 
666  if ( $i > 0 ) {
667  $ok = $this->doWait( $i, true, $timeout );
668  } else {
669  $ok = true; // no applicable loads
670  }
671  } finally {
672  // Restore the old position; this is used for throttling, not lag-protection
673  $this->waitForPos = $oldPos;
674  }
675 
676  return $ok;
677  }
678 
679  public function waitForAll( $pos, $timeout = null ) {
680  $timeout = $timeout ?: $this->waitTimeout;
681 
682  $oldPos = $this->waitForPos;
683  try {
684  $this->waitForPos = $pos;
685  $serverCount = $this->getServerCount();
686 
687  $ok = true;
688  for ( $i = 1; $i < $serverCount; $i++ ) {
689  if ( $this->serverHasLoadInAnyGroup( $i ) ) {
690  $start = microtime( true );
691  $ok = $this->doWait( $i, true, $timeout ) && $ok;
692  $timeout -= intval( microtime( true ) - $start );
693  if ( $timeout <= 0 ) {
694  break; // timeout reached
695  }
696  }
697  }
698  } finally {
699  // Restore the old position; this is used for throttling, not lag-protection
700  $this->waitForPos = $oldPos;
701  }
702 
703  return $ok;
704  }
705 
710  private function serverHasLoadInAnyGroup( $i ) {
711  foreach ( $this->groupLoads as $loadsByIndex ) {
712  if ( ( $loadsByIndex[$i] ?? 0 ) > 0 ) {
713  return true;
714  }
715  }
716 
717  return false;
718  }
719 
723  private function setWaitForPositionIfHigher( $pos ) {
724  if ( !$pos ) {
725  return;
726  }
727 
728  if ( !$this->waitForPos || $pos->hasReached( $this->waitForPos ) ) {
729  $this->waitForPos = $pos;
730  }
731  }
732 
733  public function getAnyOpenConnection( $i, $flags = 0 ) {
734  $i = ( $i === self::DB_MASTER ) ? $this->getWriterIndex() : $i;
735  // Connection handles required to be in auto-commit mode use a separate connection
736  // pool since the main pool is effected by implicit and explicit transaction rounds
737  $autocommit = ( ( $flags & self::CONN_TRX_AUTOCOMMIT ) == self::CONN_TRX_AUTOCOMMIT );
738 
739  $conn = false;
740  foreach ( $this->conns as $connsByServer ) {
741  // Get the connection array server indexes to inspect
742  if ( $i === self::DB_REPLICA ) {
743  $indexes = array_keys( $connsByServer );
744  } else {
745  $indexes = isset( $connsByServer[$i] ) ? [ $i ] : [];
746  }
747 
748  foreach ( $indexes as $index ) {
749  $conn = $this->pickAnyOpenConnection( $connsByServer[$index], $autocommit );
750  if ( $conn ) {
751  break;
752  }
753  }
754  }
755 
756  if ( $conn ) {
757  $this->enforceConnectionFlags( $conn, $flags );
758  }
759 
760  return $conn;
761  }
762 
768  private function pickAnyOpenConnection( $candidateConns, $autocommit ) {
769  $conn = false;
770 
771  foreach ( $candidateConns as $candidateConn ) {
772  if ( !$candidateConn->isOpen() ) {
773  continue; // some sort of error occurred?
774  } elseif (
775  $autocommit &&
776  (
777  // Connection is transaction round aware
778  !$candidateConn->getLBInfo( self::$INFO_AUTOCOMMIT_ONLY ) ||
779  // Some sort of error left a transaction open?
780  $candidateConn->trxLevel()
781  )
782  ) {
783  continue; // some sort of error left a transaction open?
784  }
785 
786  $conn = $candidateConn;
787  }
788 
789  return $conn;
790  }
791 
799  protected function doWait( $index, $open = false, $timeout = null ) {
800  $timeout = max( 1, intval( $timeout ?: $this->waitTimeout ) );
801 
802  // Check if we already know that the DB has reached this point
803  $server = $this->getServerName( $index );
804  $key = $this->srvCache->makeGlobalKey( __CLASS__, 'last-known-pos', $server, 'v1' );
806  $knownReachedPos = $this->srvCache->get( $key );
807  if (
808  $knownReachedPos instanceof DBMasterPos &&
809  $knownReachedPos->hasReached( $this->waitForPos )
810  ) {
811  $this->replLogger->debug(
812  __METHOD__ .
813  ': replica DB {dbserver} known to be caught up (pos >= $knownReachedPos).',
814  [ 'dbserver' => $server ]
815  );
816  return true;
817  }
818 
819  // Find a connection to wait on, creating one if needed and allowed
820  $close = false; // close the connection afterwards
821  $flags = self::CONN_SILENCE_ERRORS;
822  $conn = $this->getAnyOpenConnection( $index, $flags );
823  if ( !$conn ) {
824  if ( !$open ) {
825  $this->replLogger->debug(
826  __METHOD__ . ': no connection open for {dbserver}',
827  [ 'dbserver' => $server ]
828  );
829 
830  return false;
831  }
832  // Get a connection to this server without triggering other server connections
833  $conn = $this->getServerConnection( $index, self::DOMAIN_ANY, $flags );
834  if ( !$conn ) {
835  $this->replLogger->warning(
836  __METHOD__ . ': failed to connect to {dbserver}',
837  [ 'dbserver' => $server ]
838  );
839 
840  return false;
841  }
842  // Avoid connection spam in waitForAll() when connections
843  // are made just for the sake of doing this lag check.
844  $close = true;
845  }
846 
847  $this->replLogger->info(
848  __METHOD__ .
849  ': waiting for replica DB {dbserver} to catch up...',
850  [ 'dbserver' => $server ]
851  );
852 
853  $result = $conn->masterPosWait( $this->waitForPos, $timeout );
854 
855  if ( $result === null ) {
856  $this->replLogger->warning(
857  __METHOD__ . ': Errored out waiting on {host} pos {pos}',
858  [
859  'host' => $server,
860  'pos' => $this->waitForPos,
861  'trace' => ( new RuntimeException() )->getTraceAsString()
862  ]
863  );
864  $ok = false;
865  } elseif ( $result == -1 ) {
866  $this->replLogger->warning(
867  __METHOD__ . ': Timed out waiting on {host} pos {pos}',
868  [
869  'host' => $server,
870  'pos' => $this->waitForPos,
871  'trace' => ( new RuntimeException() )->getTraceAsString()
872  ]
873  );
874  $ok = false;
875  } else {
876  $this->replLogger->debug( __METHOD__ . ": done waiting" );
877  $ok = true;
878  // Remember that the DB reached this point
879  $this->srvCache->set( $key, $this->waitForPos, BagOStuff::TTL_DAY );
880  }
881 
882  if ( $close ) {
883  $this->closeConnection( $conn );
884  }
885 
886  return $ok;
887  }
888 
889  public function getConnection( $i, $groups = [], $domain = false, $flags = 0 ) {
890  $domain = $this->resolveDomainID( $domain );
891  $groups = $this->resolveGroups( $groups, $i );
892  $flags = $this->sanitizeConnectionFlags( $flags, $i, $domain );
893  // If given DB_MASTER/DB_REPLICA, resolve it to a specific server index. Resolving
894  // DB_REPLICA might trigger getServerConnection() calls due to the getReaderIndex()
895  // connectivity checks or LoadMonitor::scaleLoads() server state cache regeneration.
896  // The use of getServerConnection() instead of getConnection() avoids infinite loops.
897  $serverIndex = $this->getConnectionIndex( $i, $groups, $domain );
898  // Get an open connection to that server (might trigger a new connection)
899  $conn = $this->getServerConnection( $serverIndex, $domain, $flags );
900  // Set master DB handles as read-only if there is high replication lag
901  if (
902  $serverIndex === $this->getWriterIndex() &&
903  $this->getLaggedReplicaMode( $domain ) &&
904  !is_string( $conn->getLBInfo( $conn::LB_READ_ONLY_REASON ) )
905  ) {
906  $reason = ( $this->getExistingReaderIndex( self::GROUP_GENERIC ) >= 0 )
907  ? 'The database is read-only until replication lag decreases.'
908  : 'The database is read-only until replica database servers becomes reachable.';
909  $conn->setLBInfo( $conn::LB_READ_ONLY_REASON, $reason );
910  }
911 
912  return $conn;
913  }
914 
922  public function getServerConnection( $i, $domain, $flags = 0 ) {
923  // Number of connections made before getting the server index and handle
924  $priorConnectionsMade = $this->connectionCounter;
925  // Get an open connection to this server (might trigger a new connection)
926  $conn = $this->localDomain->equals( $domain )
927  ? $this->getLocalConnection( $i, $flags )
928  : $this->getForeignConnection( $i, $domain, $flags );
929  // Throw an error or otherwise bail out if the connection attempt failed
930  if ( !( $conn instanceof IDatabase ) ) {
931  if ( ( $flags & self::CONN_SILENCE_ERRORS ) != self::CONN_SILENCE_ERRORS ) {
932  $this->reportConnectionError();
933  }
934 
935  return false;
936  }
937 
938  // Profile any new connections caused by this method
939  if ( $this->connectionCounter > $priorConnectionsMade ) {
940  $this->trxProfiler->recordConnection(
941  $conn->getServer(),
942  $conn->getDBname(),
943  ( ( $flags & self::CONN_INTENT_WRITABLE ) == self::CONN_INTENT_WRITABLE )
944  );
945  }
946 
947  if ( !$conn->isOpen() ) {
948  $this->errorConnection = $conn;
949  // Connection was made but later unrecoverably lost for some reason.
950  // Do not return a handle that will just throw exceptions on use, but
951  // let the calling code, e.g. getReaderIndex(), try another server.
952  return false;
953  }
954 
955  // Make sure that flags like CONN_TRX_AUTOCOMMIT are respected by this handle
956  $this->enforceConnectionFlags( $conn, $flags );
957  // Set master DB handles as read-only if the load balancer is configured as read-only
958  // or the master database server is running in server-side read-only mode. Note that
959  // replica DB handles are always read-only via Database::assertIsWritableMaster().
960  // Read-only mode due to replication lag is *avoided* here to avoid recursion.
961  if ( $i === $this->getWriterIndex() ) {
962  if ( $this->readOnlyReason !== false ) {
964  } elseif ( $this->isMasterConnectionReadOnly( $conn, $flags ) ) {
965  $readOnlyReason = 'The master database server is running in read-only mode.';
966  } else {
967  $readOnlyReason = false;
968  }
969  $conn->setLBInfo( $conn::LB_READ_ONLY_REASON, $readOnlyReason );
970  }
971 
972  return $conn;
973  }
974 
975  public function reuseConnection( IDatabase $conn ) {
976  $serverIndex = $conn->getLBInfo( self::$INFO_SERVER_INDEX );
977  $refCount = $conn->getLBInfo( self::$INFO_FOREIGN_REF_COUNT );
978  if ( $serverIndex === null || $refCount === null ) {
979  return; // non-foreign connection; no domain-use tracking to update
980  } elseif ( $conn instanceof DBConnRef ) {
981  // DBConnRef already handles calling reuseConnection() and only passes the live
982  // Database instance to this method. Any caller passing in a DBConnRef is broken.
983  $this->connLogger->error(
984  __METHOD__ . ": got DBConnRef instance.\n" .
985  ( new LogicException() )->getTraceAsString() );
986 
987  return;
988  }
989 
990  if ( $this->disabled ) {
991  return; // DBConnRef handle probably survived longer than the LoadBalancer
992  }
993 
994  if ( $conn->getLBInfo( self::$INFO_AUTOCOMMIT_ONLY ) ) {
995  $connFreeKey = self::KEY_FOREIGN_FREE_NOROUND;
996  $connInUseKey = self::KEY_FOREIGN_INUSE_NOROUND;
997  } else {
998  $connFreeKey = self::KEY_FOREIGN_FREE;
999  $connInUseKey = self::KEY_FOREIGN_INUSE;
1000  }
1001 
1002  $domain = $conn->getDomainID();
1003  if ( !isset( $this->conns[$connInUseKey][$serverIndex][$domain] ) ) {
1004  throw new InvalidArgumentException(
1005  "Connection $serverIndex/$domain not found; it may have already been freed" );
1006  } elseif ( $this->conns[$connInUseKey][$serverIndex][$domain] !== $conn ) {
1007  throw new InvalidArgumentException(
1008  "Connection $serverIndex/$domain mismatched; it may have already been freed" );
1009  }
1010 
1011  $conn->setLBInfo( self::$INFO_FOREIGN_REF_COUNT, --$refCount );
1012  if ( $refCount <= 0 ) {
1013  $this->conns[$connFreeKey][$serverIndex][$domain] = $conn;
1014  unset( $this->conns[$connInUseKey][$serverIndex][$domain] );
1015  if ( !$this->conns[$connInUseKey][$serverIndex] ) {
1016  unset( $this->conns[$connInUseKey][$serverIndex] ); // clean up
1017  }
1018  $this->connLogger->debug( __METHOD__ . ": freed connection $serverIndex/$domain" );
1019  } else {
1020  $this->connLogger->debug( __METHOD__ .
1021  ": reference count for $serverIndex/$domain reduced to $refCount" );
1022  }
1023  }
1024 
1025  public function getConnectionRef( $i, $groups = [], $domain = false, $flags = 0 ) {
1026  $domain = $this->resolveDomainID( $domain );
1027  $role = $this->getRoleFromIndex( $i );
1028 
1029  return new DBConnRef( $this, $this->getConnection( $i, $groups, $domain, $flags ), $role );
1030  }
1031 
1032  public function getLazyConnectionRef( $i, $groups = [], $domain = false, $flags = 0 ) {
1033  $domain = $this->resolveDomainID( $domain );
1034  $role = $this->getRoleFromIndex( $i );
1035 
1036  return new DBConnRef( $this, [ $i, $groups, $domain, $flags ], $role );
1037  }
1038 
1039  public function getMaintenanceConnectionRef( $i, $groups = [], $domain = false, $flags = 0 ) {
1040  $domain = $this->resolveDomainID( $domain );
1041  $role = $this->getRoleFromIndex( $i );
1042 
1043  return new MaintainableDBConnRef(
1044  $this, $this->getConnection( $i, $groups, $domain, $flags ), $role );
1045  }
1046 
1051  private function getRoleFromIndex( $i ) {
1052  return ( $i === self::DB_MASTER || $i === $this->getWriterIndex() )
1053  ? self::DB_MASTER
1054  : self::DB_REPLICA;
1055  }
1056 
1064  public function openConnection( $i, $domain = false, $flags = 0 ) {
1065  return $this->getConnection( $i, [], $domain, $flags | self::CONN_SILENCE_ERRORS );
1066  }
1067 
1082  private function getLocalConnection( $i, $flags = 0 ) {
1083  $autoCommit = ( ( $flags & self::CONN_TRX_AUTOCOMMIT ) == self::CONN_TRX_AUTOCOMMIT );
1084  // Connection handles required to be in auto-commit mode use a separate connection
1085  // pool since the main pool is effected by implicit and explicit transaction rounds
1086  $connKey = $autoCommit ? self::KEY_LOCAL_NOROUND : self::KEY_LOCAL;
1087 
1088  if ( isset( $this->conns[$connKey][$i][0] ) ) {
1089  $conn = $this->conns[$connKey][$i][0];
1090  } else {
1091  $conn = $this->reallyOpenConnection(
1092  $i,
1093  $this->localDomain,
1094  [ self::$INFO_AUTOCOMMIT_ONLY => $autoCommit ]
1095  );
1096  if ( $conn->isOpen() ) {
1097  $this->connLogger->debug( __METHOD__ . ": opened new connection for $i" );
1098  $this->conns[$connKey][$i][0] = $conn;
1099  } else {
1100  $this->connLogger->warning( __METHOD__ . ": connection error for $i" );
1101  $this->errorConnection = $conn;
1102  $conn = false;
1103  }
1104  }
1105 
1106  // Sanity check to make sure that the right domain is selected
1107  if (
1108  $conn instanceof IDatabase &&
1109  !$this->localDomain->isCompatible( $conn->getDomainID() )
1110  ) {
1111  throw new UnexpectedValueException(
1112  "Got connection to '{$conn->getDomainID()}', " .
1113  "but expected local domain ('{$this->localDomain}')"
1114  );
1115  }
1116 
1117  return $conn;
1118  }
1119 
1144  private function getForeignConnection( $i, $domain, $flags = 0 ) {
1145  $domainInstance = DatabaseDomain::newFromId( $domain );
1146  $autoCommit = ( ( $flags & self::CONN_TRX_AUTOCOMMIT ) == self::CONN_TRX_AUTOCOMMIT );
1147  // Connection handles required to be in auto-commit mode use a separate connection
1148  // pool since the main pool is effected by implicit and explicit transaction rounds
1149  if ( $autoCommit ) {
1150  $connFreeKey = self::KEY_FOREIGN_FREE_NOROUND;
1151  $connInUseKey = self::KEY_FOREIGN_INUSE_NOROUND;
1152  } else {
1153  $connFreeKey = self::KEY_FOREIGN_FREE;
1154  $connInUseKey = self::KEY_FOREIGN_INUSE;
1155  }
1156 
1158  $conn = null;
1159 
1160  if ( isset( $this->conns[$connInUseKey][$i][$domain] ) ) {
1161  // Reuse an in-use connection for the same domain
1162  $conn = $this->conns[$connInUseKey][$i][$domain];
1163  $this->connLogger->debug( __METHOD__ . ": reusing connection $i/$domain" );
1164  } elseif ( isset( $this->conns[$connFreeKey][$i][$domain] ) ) {
1165  // Reuse a free connection for the same domain
1166  $conn = $this->conns[$connFreeKey][$i][$domain];
1167  unset( $this->conns[$connFreeKey][$i][$domain] );
1168  $this->conns[$connInUseKey][$i][$domain] = $conn;
1169  $this->connLogger->debug( __METHOD__ . ": reusing free connection $i/$domain" );
1170  } elseif ( !empty( $this->conns[$connFreeKey][$i] ) ) {
1171  // Reuse a free connection from another domain if possible
1172  foreach ( $this->conns[$connFreeKey][$i] as $oldDomain => $conn ) {
1173  if ( $domainInstance->getDatabase() !== null ) {
1174  // Check if changing the database will require a new connection.
1175  // In that case, leave the connection handle alone and keep looking.
1176  // This prevents connections from being closed mid-transaction and can
1177  // also avoid overhead if the same database will later be requested.
1178  if (
1179  $conn->databasesAreIndependent() &&
1180  $conn->getDBname() !== $domainInstance->getDatabase()
1181  ) {
1182  continue;
1183  }
1184  // Select the new database, schema, and prefix
1185  $conn->selectDomain( $domainInstance );
1186  } else {
1187  // Stay on the current database, but update the schema/prefix
1188  $conn->dbSchema( $domainInstance->getSchema() );
1189  $conn->tablePrefix( $domainInstance->getTablePrefix() );
1190  }
1191  unset( $this->conns[$connFreeKey][$i][$oldDomain] );
1192  // Note that if $domain is an empty string, getDomainID() might not match it
1193  $this->conns[$connInUseKey][$i][$conn->getDomainID()] = $conn;
1194  $this->connLogger->debug( __METHOD__ .
1195  ": reusing free connection from $oldDomain for $domain" );
1196  break;
1197  }
1198  }
1199 
1200  if ( !$conn ) {
1201  $conn = $this->reallyOpenConnection(
1202  $i,
1203  $domainInstance,
1204  [
1205  self::$INFO_AUTOCOMMIT_ONLY => $autoCommit,
1206  self::$INFO_FORIEGN => true,
1207  self::$INFO_FOREIGN_REF_COUNT => 0
1208  ]
1209  );
1210  if ( $conn->isOpen() ) {
1211  // Note that if $domain is an empty string, getDomainID() might not match it
1212  $this->conns[$connInUseKey][$i][$conn->getDomainID()] = $conn;
1213  $this->connLogger->debug( __METHOD__ . ": opened new connection for $i/$domain" );
1214  } else {
1215  $this->connLogger->warning( __METHOD__ . ": connection error for $i/$domain" );
1216  $this->errorConnection = $conn;
1217  $conn = false;
1218  }
1219  }
1220 
1221  if ( $conn instanceof IDatabase ) {
1222  // Sanity check to make sure that the right domain is selected
1223  if ( !$domainInstance->isCompatible( $conn->getDomainID() ) ) {
1224  throw new UnexpectedValueException(
1225  "Got connection to '{$conn->getDomainID()}', but expected '$domain'" );
1226  }
1227  // Increment reference count
1228  $refCount = $conn->getLBInfo( self::$INFO_FOREIGN_REF_COUNT );
1229  $conn->setLBInfo( self::$INFO_FOREIGN_REF_COUNT, $refCount + 1 );
1230  }
1231 
1232  return $conn;
1233  }
1234 
1235  public function getServerAttributes( $i ) {
1237  $this->getServerType( $i ),
1238  $this->servers[$i]['driver'] ?? null
1239  );
1240  }
1241 
1248  private function isOpen( $index ) {
1249  return (bool)$this->getAnyOpenConnection( $index );
1250  }
1251 
1264  protected function reallyOpenConnection( $i, DatabaseDomain $domain, array $lbInfo ) {
1265  if ( $this->disabled ) {
1266  throw new DBAccessError();
1267  }
1268 
1269  $server = $this->getServerInfoStrict( $i );
1270 
1271  $conn = Database::factory(
1272  $server['type'],
1273  array_merge( $server, [
1274  // Basic replication role information
1275  'topologyRole' => $this->getTopologyRole( $i, $server ),
1276  'topologicalMaster' => $this->getMasterServerName(),
1277  // Use the database specified in $domain (null means "none or entrypoint DB");
1278  // fallback to the $server default if the RDBMs is an embedded library using a
1279  // file on disk since there would be nothing to access to without a DB/file name.
1280  'dbname' => $this->getServerAttributes( $i )[Database::ATTR_DB_IS_FILE]
1281  ? ( $domain->getDatabase() ?? $server['dbname'] ?? null )
1282  : $domain->getDatabase(),
1283  // Override the $server default schema with that of $domain if specified
1284  'schema' => $domain->getSchema() ?? $server['schema'] ?? null,
1285  // Use the table prefix specified in $domain
1286  'tablePrefix' => $domain->getTablePrefix(),
1287  // Participate in transaction rounds if $server does not specify otherwise
1288  'flags' => $this->initConnFlags( $server['flags'] ?? IDatabase::DBO_DEFAULT ),
1289  // Inject the PHP execution mode and the agent string
1290  'cliMode' => $this->cliMode,
1291  'agent' => $this->agent,
1292  'ownerId' => $this->id,
1293  // Inject object and callback dependencies
1294  'lazyMasterHandle' => $this->getLazyConnectionRef(
1296  [],
1297  $domain->getId()
1298  ),
1299  'srvCache' => $this->srvCache,
1300  'connLogger' => $this->connLogger,
1301  'queryLogger' => $this->queryLogger,
1302  'errorLogger' => $this->errorLogger,
1303  'deprecationLogger' => $this->deprecationLogger,
1304  'profiler' => $this->profiler,
1305  'trxProfiler' => $this->trxProfiler
1306  ] ),
1307  Database::NEW_UNCONNECTED
1308  );
1309  // Attach load balancer information to the handle
1310  $conn->setLBInfo( [ self::$INFO_SERVER_INDEX => $i ] + $lbInfo );
1311  // Set alternative table/index names before any queries can be issued
1312  $conn->setTableAliases( $this->tableAliases );
1313  $conn->setIndexAliases( $this->indexAliases );
1314  // Account for any active transaction round and listeners
1315  if ( $i === $this->getWriterIndex() ) {
1316  if ( $this->trxRoundId !== false ) {
1317  $this->applyTransactionRoundFlags( $conn );
1318  }
1319  foreach ( $this->trxRecurringCallbacks as $name => $callback ) {
1320  $conn->setTransactionListener( $name, $callback );
1321  }
1322  }
1323 
1324  // Make the connection handle live
1325  try {
1326  $conn->initConnection();
1328  } catch ( DBConnectionError $e ) {
1329  // ignore; let the DB handle the logging
1330  }
1331 
1332  // Try to maintain session consistency for clients that trigger write transactions
1333  // in a request or script and then return soon after in another request or script.
1334  // This requires cooperation with ChronologyProtector and the application wiring.
1335  if ( $conn->isOpen() ) {
1337  }
1338 
1339  // Log when many connection are made during a single request/script
1340  $count = $this->getCurrentConnectionCount();
1341  if ( $count >= self::CONN_HELD_WARN_THRESHOLD ) {
1342  $this->perfLogger->warning(
1343  __METHOD__ . ": {connections}+ connections made (master={masterdb})",
1344  [
1345  'connections' => $count,
1346  'dbserver' => $conn->getServer(),
1347  'masterdb' => $this->getMasterServerName()
1348  ]
1349  );
1350  }
1351 
1352  return $conn;
1353  }
1354 
1360  private function getTopologyRole( $i, array $server ) {
1361  if ( !empty( $server['is static'] ) ) {
1362  return IDatabase::ROLE_STATIC_CLONE;
1363  }
1364 
1365  return ( $i === $this->getWriterIndex() )
1366  ? IDatabase::ROLE_STREAMING_MASTER
1367  : IDatabase::ROLE_STREAMING_REPLICA;
1368  }
1369 
1375  private function initConnFlags( $flags ) {
1376  if ( ( $flags & IDatabase::DBO_DEFAULT ) === IDatabase::DBO_DEFAULT ) {
1377  if ( $this->cliMode ) {
1378  $flags &= ~IDatabase::DBO_TRX;
1379  } else {
1380  $flags |= IDatabase::DBO_TRX;
1381  }
1382  }
1383 
1384  return $flags;
1385  }
1386 
1390  private function lazyLoadReplicationPositions() {
1391  if ( !$this->connectionAttempted && $this->chronologyCallback ) {
1392  $this->connectionAttempted = true;
1393  ( $this->chronologyCallback )( $this ); // generally calls waitFor()
1394  $this->connLogger->debug( __METHOD__ . ': executed chronology callback.' );
1395  }
1396  }
1397 
1401  private function reportConnectionError() {
1402  $conn = $this->errorConnection; // the connection which caused the error
1403  $context = [
1404  'method' => __METHOD__,
1405  'last_error' => $this->lastError,
1406  ];
1407 
1408  if ( $conn instanceof IDatabase ) {
1409  $context['db_server'] = $conn->getServer();
1410  $this->connLogger->warning(
1411  __METHOD__ . ": connection error: {last_error} ({db_server})",
1412  $context
1413  );
1414 
1415  throw new DBConnectionError( $conn, "{$this->lastError} ({$context['db_server']})" );
1416  } else {
1417  // No last connection, probably due to all servers being too busy
1418  $this->connLogger->error(
1419  __METHOD__ .
1420  ": LB failure with no last connection. Connection error: {last_error}",
1421  $context
1422  );
1423 
1424  // If all servers were busy, "lastError" will contain something sensible
1425  throw new DBConnectionError( null, $this->lastError );
1426  }
1427  }
1428 
1429  public function getWriterIndex() {
1430  return 0;
1431  }
1432 
1440  public function haveIndex( $i ) {
1441  return array_key_exists( $i, $this->servers );
1442  }
1443 
1451  public function isNonZeroLoad( $i ) {
1452  return ( isset( $this->servers[$i] ) && $this->groupLoads[self::GROUP_GENERIC][$i] > 0 );
1453  }
1454 
1455  public function getServerCount() {
1456  return count( $this->servers );
1457  }
1458 
1459  public function hasReplicaServers() {
1460  return ( $this->getServerCount() > 1 );
1461  }
1462 
1463  public function hasStreamingReplicaServers() {
1464  foreach ( $this->servers as $i => $server ) {
1465  if ( $i !== $this->getWriterIndex() && empty( $server['is static'] ) ) {
1466  return true;
1467  }
1468  }
1469 
1470  return false;
1471  }
1472 
1473  public function getServerName( $i ) {
1474  $name = $this->servers[$i]['hostName'] ?? ( $this->servers[$i]['host'] ?? '' );
1475 
1476  return ( $name != '' ) ? $name : 'localhost';
1477  }
1478 
1479  public function getServerInfo( $i ) {
1480  return $this->servers[$i] ?? false;
1481  }
1482 
1483  public function getServerType( $i ) {
1484  return $this->servers[$i]['type'] ?? 'unknown';
1485  }
1486 
1487  public function getMasterPos() {
1488  $index = $this->getWriterIndex();
1489 
1490  $conn = $this->getAnyOpenConnection( $index );
1491  if ( $conn ) {
1492  return $conn->getMasterPos();
1493  }
1494 
1495  $conn = $this->getConnection( $index, self::CONN_SILENCE_ERRORS );
1496  if ( !$conn ) {
1497  $this->reportConnectionError();
1498  return null; // unreachable due to exception
1499  }
1500 
1501  try {
1502  $pos = $conn->getMasterPos();
1503  } finally {
1504  $this->closeConnection( $conn );
1505  }
1506 
1507  return $pos;
1508  }
1509 
1510  public function getReplicaResumePos() {
1511  // Get the position of any existing master server connection
1512  $masterConn = $this->getAnyOpenConnection( $this->getWriterIndex() );
1513  if ( $masterConn ) {
1514  return $masterConn->getMasterPos();
1515  }
1516 
1517  // Get the highest position of any existing replica server connection
1518  $highestPos = false;
1519  $serverCount = $this->getServerCount();
1520  for ( $i = 1; $i < $serverCount; $i++ ) {
1521  if ( !empty( $this->servers[$i]['is static'] ) ) {
1522  continue; // server does not use replication
1523  }
1524 
1525  $conn = $this->getAnyOpenConnection( $i );
1526  $pos = $conn ? $conn->getReplicaPos() : false;
1527  if ( !$pos ) {
1528  continue; // no open connection or could not get position
1529  }
1530 
1531  $highestPos = $highestPos ?: $pos;
1532  if ( $pos->hasReached( $highestPos ) ) {
1533  $highestPos = $pos;
1534  }
1535  }
1536 
1537  return $highestPos;
1538  }
1539 
1540  public function disable( $fname = __METHOD__, $owner = null ) {
1541  $this->assertOwnership( $fname, $owner );
1542  $this->closeAll( $fname, $owner );
1543  $this->disabled = true;
1544  }
1545 
1546  public function closeAll( $fname = __METHOD__, $owner = null ) {
1547  $this->assertOwnership( $fname, $owner );
1548  if ( $this->ownerId === null ) {
1550  $scope = ScopedCallback::newScopedIgnoreUserAbort();
1551  }
1552  $this->forEachOpenConnection( function ( IDatabase $conn ) use ( $fname ) {
1553  $host = $conn->getServer();
1554  $this->connLogger->debug( "$fname: closing connection to database '$host'." );
1555  $conn->close( $fname, $this->id );
1556  } );
1557 
1558  $this->conns = self::newTrackedConnectionsArray();
1559  }
1560 
1561  public function closeConnection( IDatabase $conn ) {
1562  if ( $conn instanceof DBConnRef ) {
1563  // Avoid calling close() but still leaving the handle in the pool
1564  throw new RuntimeException( 'Cannot close DBConnRef instance; it must be shareable' );
1565  }
1566 
1567  $serverIndex = $conn->getLBInfo( self::$INFO_SERVER_INDEX );
1568  foreach ( $this->conns as $type => $connsByServer ) {
1569  if ( !isset( $connsByServer[$serverIndex] ) ) {
1570  continue;
1571  }
1572 
1573  foreach ( $connsByServer[$serverIndex] as $i => $trackedConn ) {
1574  if ( $conn === $trackedConn ) {
1575  $host = $this->getServerName( $i );
1576  $this->connLogger->debug(
1577  __METHOD__ . ": closing connection to database $i at '$host'." );
1578  unset( $this->conns[$type][$serverIndex][$i] );
1579  break 2;
1580  }
1581  }
1582  }
1583 
1584  $conn->close( __METHOD__ );
1585  }
1586 
1587  public function commitAll( $fname = __METHOD__, $owner = null ) {
1588  $this->commitMasterChanges( $fname, $owner );
1589  $this->flushMasterSnapshots( $fname, $owner );
1590  $this->flushReplicaSnapshots( $fname, $owner );
1591  }
1592 
1593  public function finalizeMasterChanges( $fname = __METHOD__, $owner = null ) {
1594  $this->assertOwnership( $fname, $owner );
1595  $this->assertTransactionRoundStage( [ self::ROUND_CURSORY, self::ROUND_FINALIZED ] );
1596  if ( $this->ownerId === null ) {
1598  $scope = ScopedCallback::newScopedIgnoreUserAbort();
1599  }
1600 
1601  $this->trxRoundStage = self::ROUND_ERROR; // "failed" until proven otherwise
1602  // Loop until callbacks stop adding callbacks on other connections
1603  $total = 0;
1604  do {
1605  $count = 0; // callbacks execution attempts
1606  $this->forEachOpenMasterConnection( function ( Database $conn ) use ( &$count ) {
1607  // Run any pre-commit callbacks while leaving the post-commit ones suppressed.
1608  // Any error should cause all (peer) transactions to be rolled back together.
1609  $count += $conn->runOnTransactionPreCommitCallbacks();
1610  } );
1611  $total += $count;
1612  } while ( $count > 0 );
1613  // Defer post-commit callbacks until after COMMIT/ROLLBACK happens on all handles
1614  $this->forEachOpenMasterConnection( function ( Database $conn ) {
1615  $conn->setTrxEndCallbackSuppression( true );
1616  } );
1617  $this->trxRoundStage = self::ROUND_FINALIZED;
1618 
1619  return $total;
1620  }
1621 
1622  public function approveMasterChanges( array $options, $fname = __METHOD__, $owner = null ) {
1623  $this->assertOwnership( $fname, $owner );
1624  $this->assertTransactionRoundStage( self::ROUND_FINALIZED );
1625  if ( $this->ownerId === null ) {
1627  $scope = ScopedCallback::newScopedIgnoreUserAbort();
1628  }
1629 
1630  $limit = $options['maxWriteDuration'] ?? 0;
1631 
1632  $this->trxRoundStage = self::ROUND_ERROR; // "failed" until proven otherwise
1633  $this->forEachOpenMasterConnection( function ( IDatabase $conn ) use ( $limit ) {
1634  // If atomic sections or explicit transactions are still open, some caller must have
1635  // caught an exception but failed to properly rollback any changes. Detect that and
1636  // throw and error (causing rollback).
1637  $conn->assertNoOpenTransactions();
1638  // Assert that the time to replicate the transaction will be sane.
1639  // If this fails, then all DB transactions will be rollback back together.
1640  $time = $conn->pendingWriteQueryDuration( $conn::ESTIMATE_DB_APPLY );
1641  if ( $limit > 0 && $time > $limit ) {
1642  throw new DBTransactionSizeError(
1643  $conn,
1644  "Transaction spent $time second(s) in writes, exceeding the limit of $limit",
1645  [ $time, $limit ]
1646  );
1647  }
1648  // If a connection sits idle while slow queries execute on another, that connection
1649  // may end up dropped before the commit round is reached. Ping servers to detect this.
1650  if ( $conn->writesOrCallbacksPending() && !$conn->ping() ) {
1651  throw new DBTransactionError(
1652  $conn,
1653  "A connection to the {$conn->getDBname()} database was lost before commit"
1654  );
1655  }
1656  } );
1657  $this->trxRoundStage = self::ROUND_APPROVED;
1658  }
1659 
1660  public function beginMasterChanges( $fname = __METHOD__, $owner = null ) {
1661  $this->assertOwnership( $fname, $owner );
1662  if ( $this->trxRoundId !== false ) {
1663  throw new DBTransactionError(
1664  null,
1665  "$fname: Transaction round '{$this->trxRoundId}' already started"
1666  );
1667  }
1668  $this->assertTransactionRoundStage( self::ROUND_CURSORY );
1669  if ( $this->ownerId === null ) {
1671  $scope = ScopedCallback::newScopedIgnoreUserAbort();
1672  }
1673 
1674  // Clear any empty transactions (no writes/callbacks) from the implicit round
1675  $this->flushMasterSnapshots( $fname, $owner );
1676 
1677  $this->trxRoundId = $fname;
1678  $this->trxRoundStage = self::ROUND_ERROR; // "failed" until proven otherwise
1679  // Mark applicable handles as participating in this explicit transaction round.
1680  // For each of these handles, any writes and callbacks will be tied to a single
1681  // transaction. The (peer) handles will reject begin()/commit() calls unless they
1682  // are part of an en masse commit or an en masse rollback.
1683  $this->forEachOpenMasterConnection( function ( Database $conn ) {
1684  $this->applyTransactionRoundFlags( $conn );
1685  } );
1686  $this->trxRoundStage = self::ROUND_CURSORY;
1687  }
1688 
1689  public function commitMasterChanges( $fname = __METHOD__, $owner = null ) {
1690  $this->assertOwnership( $fname, $owner );
1691  $this->assertTransactionRoundStage( self::ROUND_APPROVED );
1692  if ( $this->ownerId === null ) {
1694  $scope = ScopedCallback::newScopedIgnoreUserAbort();
1695  }
1696 
1697  $failures = [];
1698 
1699  $restore = ( $this->trxRoundId !== false );
1700  $this->trxRoundId = false;
1701  $this->trxRoundStage = self::ROUND_ERROR; // "failed" until proven otherwise
1702  // Commit any writes and clear any snapshots as well (callbacks require AUTOCOMMIT).
1703  // Note that callbacks should already be suppressed due to finalizeMasterChanges().
1705  function ( IDatabase $conn ) use ( $fname, &$failures ) {
1706  try {
1707  $conn->commit( $fname, $conn::FLUSHING_ALL_PEERS );
1708  } catch ( DBError $e ) {
1709  ( $this->errorLogger )( $e );
1710  $failures[] = "{$conn->getServer()}: {$e->getMessage()}";
1711  }
1712  }
1713  );
1714  if ( $failures ) {
1715  throw new DBTransactionError(
1716  null,
1717  "$fname: Commit failed on server(s) " . implode( "\n", array_unique( $failures ) )
1718  );
1719  }
1720  if ( $restore ) {
1721  // Unmark handles as participating in this explicit transaction round
1722  $this->forEachOpenMasterConnection( function ( Database $conn ) {
1723  $this->undoTransactionRoundFlags( $conn );
1724  } );
1725  }
1726  $this->trxRoundStage = self::ROUND_COMMIT_CALLBACKS;
1727  }
1728 
1729  public function runMasterTransactionIdleCallbacks( $fname = __METHOD__, $owner = null ) {
1730  $this->assertOwnership( $fname, $owner );
1731  if ( $this->trxRoundStage === self::ROUND_COMMIT_CALLBACKS ) {
1732  $type = IDatabase::TRIGGER_COMMIT;
1733  } elseif ( $this->trxRoundStage === self::ROUND_ROLLBACK_CALLBACKS ) {
1734  $type = IDatabase::TRIGGER_ROLLBACK;
1735  } else {
1736  throw new DBTransactionError(
1737  null,
1738  "Transaction should be in the callback stage (not '{$this->trxRoundStage}')"
1739  );
1740  }
1741  if ( $this->ownerId === null ) {
1743  $scope = ScopedCallback::newScopedIgnoreUserAbort();
1744  }
1745 
1746  $oldStage = $this->trxRoundStage;
1747  $this->trxRoundStage = self::ROUND_ERROR; // "failed" until proven otherwise
1748 
1749  // Now that the COMMIT/ROLLBACK step is over, enable post-commit callback runs
1750  $this->forEachOpenMasterConnection( function ( Database $conn ) {
1751  $conn->setTrxEndCallbackSuppression( false );
1752  } );
1753 
1754  $e = null; // first exception
1755  $fname = __METHOD__;
1756  // Loop until callbacks stop adding callbacks on other connections
1757  do {
1758  // Run any pending callbacks for each connection...
1759  $count = 0; // callback execution attempts
1761  function ( Database $conn ) use ( $type, &$e, &$count ) {
1762  if ( $conn->trxLevel() ) {
1763  return; // retry in the next iteration, after commit() is called
1764  }
1765  try {
1766  $count += $conn->runOnTransactionIdleCallbacks( $type );
1767  } catch ( Exception $ex ) {
1768  $e = $e ?: $ex;
1769  }
1770  }
1771  );
1772  // Clear out any active transactions left over from callbacks...
1773  $this->forEachOpenMasterConnection( function ( Database $conn ) use ( &$e, $fname ) {
1774  if ( $conn->writesPending() ) {
1775  // A callback from another handle wrote to this one and DBO_TRX is set
1776  $this->queryLogger->warning( $fname . ": found writes pending." );
1777  $fnames = implode( ', ', $conn->pendingWriteAndCallbackCallers() );
1778  $this->queryLogger->warning(
1779  "$fname: found writes pending ($fnames).",
1780  [
1781  'db_server' => $conn->getServer(),
1782  'db_name' => $conn->getDBname()
1783  ]
1784  );
1785  } elseif ( $conn->trxLevel() ) {
1786  // A callback from another handle read from this one and DBO_TRX is set,
1787  // which can easily happen if there is only one DB (no replicas)
1788  $this->queryLogger->debug( "$fname: found empty transaction." );
1789  }
1790  try {
1791  $conn->commit( $fname, $conn::FLUSHING_ALL_PEERS );
1792  } catch ( Exception $ex ) {
1793  $e = $e ?: $ex;
1794  }
1795  } );
1796  } while ( $count > 0 );
1797 
1798  $this->trxRoundStage = $oldStage;
1799 
1800  return $e;
1801  }
1802 
1803  public function runMasterTransactionListenerCallbacks( $fname = __METHOD__, $owner = null ) {
1804  $this->assertOwnership( $fname, $owner );
1805  if ( $this->trxRoundStage === self::ROUND_COMMIT_CALLBACKS ) {
1806  $type = IDatabase::TRIGGER_COMMIT;
1807  } elseif ( $this->trxRoundStage === self::ROUND_ROLLBACK_CALLBACKS ) {
1808  $type = IDatabase::TRIGGER_ROLLBACK;
1809  } else {
1810  throw new DBTransactionError(
1811  null,
1812  "Transaction should be in the callback stage (not '{$this->trxRoundStage}')"
1813  );
1814  }
1815  if ( $this->ownerId === null ) {
1817  $scope = ScopedCallback::newScopedIgnoreUserAbort();
1818  }
1819 
1820  $e = null;
1821 
1822  $this->trxRoundStage = self::ROUND_ERROR; // "failed" until proven otherwise
1823  $this->forEachOpenMasterConnection( function ( Database $conn ) use ( $type, &$e ) {
1824  try {
1826  } catch ( Exception $ex ) {
1827  $e = $e ?: $ex;
1828  }
1829  } );
1830  $this->trxRoundStage = self::ROUND_CURSORY;
1831 
1832  return $e;
1833  }
1834 
1835  public function rollbackMasterChanges( $fname = __METHOD__, $owner = null ) {
1836  $this->assertOwnership( $fname, $owner );
1837  if ( $this->ownerId === null ) {
1839  $scope = ScopedCallback::newScopedIgnoreUserAbort();
1840  }
1841 
1842  $restore = ( $this->trxRoundId !== false );
1843  $this->trxRoundId = false;
1844  $this->trxRoundStage = self::ROUND_ERROR; // "failed" until proven otherwise
1845  $this->forEachOpenMasterConnection( function ( IDatabase $conn ) use ( $fname ) {
1846  $conn->rollback( $fname, $conn::FLUSHING_ALL_PEERS );
1847  } );
1848  if ( $restore ) {
1849  // Unmark handles as participating in this explicit transaction round
1850  $this->forEachOpenMasterConnection( function ( Database $conn ) {
1851  $this->undoTransactionRoundFlags( $conn );
1852  } );
1853  }
1854  $this->trxRoundStage = self::ROUND_ROLLBACK_CALLBACKS;
1855  }
1856 
1861  private function assertTransactionRoundStage( $stage ) {
1862  $stages = (array)$stage;
1863 
1864  if ( !in_array( $this->trxRoundStage, $stages, true ) ) {
1865  $stageList = implode(
1866  '/',
1867  array_map( function ( $v ) {
1868  return "'$v'";
1869  }, $stages )
1870  );
1871  throw new DBTransactionError(
1872  null,
1873  "Transaction round stage must be $stageList (not '{$this->trxRoundStage}')"
1874  );
1875  }
1876  }
1877 
1890  private function assertOwnership( $fname, $owner ) {
1891  if ( $this->ownerId !== null && $owner !== $this->ownerId && $owner !== $this->id ) {
1892  throw new DBTransactionError(
1893  null,
1894  "$fname: LoadBalancer is owned by ID '{$this->ownerId}' (got '$owner')."
1895  );
1896  }
1897  }
1898 
1908  private function applyTransactionRoundFlags( Database $conn ) {
1909  if ( $conn->getLBInfo( self::$INFO_AUTOCOMMIT_ONLY ) ) {
1910  return; // transaction rounds do not apply to these connections
1911  }
1912 
1913  if ( $conn->getFlag( $conn::DBO_DEFAULT ) ) {
1914  // DBO_TRX is controlled entirely by CLI mode presence with DBO_DEFAULT.
1915  // Force DBO_TRX even in CLI mode since a commit round is expected soon.
1916  $conn->setFlag( $conn::DBO_TRX, $conn::REMEMBER_PRIOR );
1917  }
1918 
1919  if ( $conn->getFlag( $conn::DBO_TRX ) ) {
1920  $conn->setLBInfo( $conn::LB_TRX_ROUND_ID, $this->trxRoundId );
1921  }
1922  }
1923 
1927  private function undoTransactionRoundFlags( Database $conn ) {
1928  if ( $conn->getLBInfo( self::$INFO_AUTOCOMMIT_ONLY ) ) {
1929  return; // transaction rounds do not apply to these connections
1930  }
1931 
1932  if ( $conn->getFlag( $conn::DBO_TRX ) ) {
1933  $conn->setLBInfo( $conn::LB_TRX_ROUND_ID, null ); // remove the round ID
1934  }
1935 
1936  if ( $conn->getFlag( $conn::DBO_DEFAULT ) ) {
1937  $conn->restoreFlags( $conn::RESTORE_PRIOR );
1938  }
1939  }
1940 
1941  public function flushReplicaSnapshots( $fname = __METHOD__, $owner = null ) {
1942  $this->assertOwnership( $fname, $owner );
1943  $this->forEachOpenReplicaConnection( function ( IDatabase $conn ) use ( $fname ) {
1944  $conn->flushSnapshot( $fname );
1945  } );
1946  }
1947 
1948  public function flushMasterSnapshots( $fname = __METHOD__, $owner = null ) {
1949  $this->assertOwnership( $fname, $owner );
1950  $this->forEachOpenMasterConnection( function ( IDatabase $conn ) use ( $fname ) {
1951  $conn->flushSnapshot( $fname );
1952  } );
1953  }
1954 
1959  public function getTransactionRoundStage() {
1960  return $this->trxRoundStage;
1961  }
1962 
1963  public function hasMasterConnection() {
1964  return $this->isOpen( $this->getWriterIndex() );
1965  }
1966 
1967  public function hasMasterChanges() {
1968  $pending = 0;
1969  $this->forEachOpenMasterConnection( function ( IDatabase $conn ) use ( &$pending ) {
1970  $pending |= $conn->writesOrCallbacksPending();
1971  } );
1972 
1973  return (bool)$pending;
1974  }
1975 
1976  public function lastMasterChangeTimestamp() {
1977  $lastTime = false;
1978  $this->forEachOpenMasterConnection( function ( IDatabase $conn ) use ( &$lastTime ) {
1979  $lastTime = max( $lastTime, $conn->lastDoneWrites() );
1980  } );
1981 
1982  return $lastTime;
1983  }
1984 
1985  public function hasOrMadeRecentMasterChanges( $age = null ) {
1986  $age = ( $age === null ) ? $this->waitTimeout : $age;
1987 
1988  return ( $this->hasMasterChanges()
1989  || $this->lastMasterChangeTimestamp() > microtime( true ) - $age );
1990  }
1991 
1992  public function pendingMasterChangeCallers() {
1993  $fnames = [];
1994  $this->forEachOpenMasterConnection( function ( IDatabase $conn ) use ( &$fnames ) {
1995  $fnames = array_merge( $fnames, $conn->pendingWriteCallers() );
1996  } );
1997 
1998  return $fnames;
1999  }
2000 
2001  public function getLaggedReplicaMode( $domain = false ) {
2002  if ( $this->laggedReplicaMode ) {
2003  return true; // stay in lagged replica mode
2004  }
2005 
2006  if ( $this->hasStreamingReplicaServers() ) {
2007  // This will set "laggedReplicaMode" as needed
2008  $this->getReaderIndex( self::GROUP_GENERIC, $domain );
2009  }
2010 
2011  return $this->laggedReplicaMode;
2012  }
2013 
2014  public function laggedReplicaUsed() {
2015  return $this->laggedReplicaMode;
2016  }
2017 
2018  public function getReadOnlyReason( $domain = false ) {
2019  $domainInstance = DatabaseDomain::newFromId( $this->resolveDomainID( $domain ) );
2020 
2021  if ( $this->readOnlyReason !== false ) {
2022  return $this->readOnlyReason;
2023  } elseif ( $this->isMasterRunningReadOnly( $domainInstance ) ) {
2024  return 'The master database server is running in read-only mode.';
2025  } elseif ( $this->getLaggedReplicaMode( $domain ) ) {
2026  return ( $this->getExistingReaderIndex( self::GROUP_GENERIC ) >= 0 )
2027  ? 'The database is read-only until replication lag decreases.'
2028  : 'The database is read-only until a replica database server becomes reachable.';
2029  }
2030 
2031  return false;
2032  }
2033 
2039  private function isMasterConnectionReadOnly( IDatabase $conn, $flags = 0 ) {
2040  // Note that table prefixes are not related to server-side read-only mode
2041  $key = $this->srvCache->makeGlobalKey(
2042  'rdbms-server-readonly',
2043  $conn->getServer(),
2044  $conn->getDBname(),
2045  $conn->dbSchema()
2046  );
2047 
2048  if ( ( $flags & self::CONN_REFRESH_READ_ONLY ) == self::CONN_REFRESH_READ_ONLY ) {
2049  try {
2050  $readOnly = (int)$conn->serverIsReadOnly();
2051  } catch ( DBError $e ) {
2052  $readOnly = 0;
2053  }
2054  $this->srvCache->set( $key, $readOnly, BagOStuff::TTL_PROC_SHORT );
2055  } else {
2056  $readOnly = $this->srvCache->getWithSetCallback(
2057  $key,
2059  function () use ( $conn ) {
2060  try {
2061  return (int)$conn->serverIsReadOnly();
2062  } catch ( DBError $e ) {
2063  return 0;
2064  }
2065  }
2066  );
2067  }
2068 
2069  return (bool)$readOnly;
2070  }
2071 
2076  private function isMasterRunningReadOnly( DatabaseDomain $domain ) {
2077  // Context will often be HTTP GET/HEAD; heavily cache the results
2078  return (bool)$this->wanCache->getWithSetCallback(
2079  // Note that table prefixes are not related to server-side read-only mode
2080  $this->wanCache->makeGlobalKey(
2081  'rdbms-server-readonly',
2082  $this->getMasterServerName(),
2083  $domain->getDatabase(),
2084  $domain->getSchema()
2085  ),
2086  self::TTL_CACHE_READONLY,
2087  function () use ( $domain ) {
2088  $old = $this->trxProfiler->setSilenced( true );
2089  try {
2090  $index = $this->getWriterIndex();
2091  // Reset the cache for isMasterConnectionReadOnly()
2092  $flags = self::CONN_REFRESH_READ_ONLY;
2093  $conn = $this->getServerConnection( $index, $domain->getId(), $flags );
2094  // Reuse the process cache set above
2095  $readOnly = (int)$this->isMasterConnectionReadOnly( $conn );
2096  $this->reuseConnection( $conn );
2097  } catch ( DBError $e ) {
2098  $readOnly = 0;
2099  }
2100  $this->trxProfiler->setSilenced( $old );
2101 
2102  return $readOnly;
2103  },
2104  [ 'pcTTL' => WANObjectCache::TTL_PROC_LONG, 'lockTSE' => 10, 'busyValue' => 0 ]
2105  );
2106  }
2107 
2108  public function allowLagged( $mode = null ) {
2109  if ( $mode === null ) {
2110  return $this->allowLagged;
2111  }
2112  $this->allowLagged = $mode;
2113 
2114  return $this->allowLagged;
2115  }
2116 
2117  public function pingAll() {
2118  $success = true;
2119  $this->forEachOpenConnection( function ( IDatabase $conn ) use ( &$success ) {
2120  if ( !$conn->ping() ) {
2121  $success = false;
2122  }
2123  } );
2124 
2125  return $success;
2126  }
2127 
2128  public function forEachOpenConnection( $callback, array $params = [] ) {
2129  foreach ( $this->conns as $connsByServer ) {
2130  foreach ( $connsByServer as $serverConns ) {
2131  foreach ( $serverConns as $conn ) {
2132  $callback( $conn, ...$params );
2133  }
2134  }
2135  }
2136  }
2137 
2138  public function forEachOpenMasterConnection( $callback, array $params = [] ) {
2139  $masterIndex = $this->getWriterIndex();
2140  foreach ( $this->conns as $connsByServer ) {
2141  if ( isset( $connsByServer[$masterIndex] ) ) {
2143  foreach ( $connsByServer[$masterIndex] as $conn ) {
2144  $callback( $conn, ...$params );
2145  }
2146  }
2147  }
2148  }
2149 
2150  public function forEachOpenReplicaConnection( $callback, array $params = [] ) {
2151  foreach ( $this->conns as $connsByServer ) {
2152  foreach ( $connsByServer as $i => $serverConns ) {
2153  if ( $i === $this->getWriterIndex() ) {
2154  continue; // skip master
2155  }
2156  foreach ( $serverConns as $conn ) {
2157  $callback( $conn, ...$params );
2158  }
2159  }
2160  }
2161  }
2162 
2166  private function getCurrentConnectionCount() {
2167  $count = 0;
2168  foreach ( $this->conns as $connsByServer ) {
2169  foreach ( $connsByServer as $serverConns ) {
2170  $count += count( $serverConns );
2171  }
2172  }
2173 
2174  return $count;
2175  }
2176 
2177  public function getMaxLag( $domain = false ) {
2178  $host = '';
2179  $maxLag = -1;
2180  $maxIndex = 0;
2181 
2182  if ( $this->hasReplicaServers() ) {
2183  $lagTimes = $this->getLagTimes( $domain );
2184  foreach ( $lagTimes as $i => $lag ) {
2185  if ( $this->groupLoads[self::GROUP_GENERIC][$i] > 0 && $lag > $maxLag ) {
2186  $maxLag = $lag;
2187  $host = $this->getServerInfoStrict( $i, 'host' );
2188  $maxIndex = $i;
2189  }
2190  }
2191  }
2192 
2193  return [ $host, $maxLag, $maxIndex ];
2194  }
2195 
2196  public function getLagTimes( $domain = false ) {
2197  if ( !$this->hasReplicaServers() ) {
2198  return [ $this->getWriterIndex() => 0 ]; // no replication = no lag
2199  }
2200 
2201  $knownLagTimes = []; // map of (server index => 0 seconds)
2202  $indexesWithLag = [];
2203  foreach ( $this->servers as $i => $server ) {
2204  if ( empty( $server['is static'] ) ) {
2205  $indexesWithLag[] = $i; // DB server might have replication lag
2206  } else {
2207  $knownLagTimes[$i] = 0; // DB server is a non-replicating and read-only archive
2208  }
2209  }
2210 
2211  return $this->getLoadMonitor()->getLagTimes( $indexesWithLag, $domain ) + $knownLagTimes;
2212  }
2213 
2229  public function safeGetLag( IDatabase $conn ) {
2230  return $conn->getLag();
2231  }
2232 
2233  public function waitForMasterPos( IDatabase $conn, $pos = false, $timeout = null ) {
2234  $timeout = max( 1, $timeout ?: $this->waitTimeout );
2235 
2236  if ( $conn->getLBInfo( self::$INFO_SERVER_INDEX ) === $this->getWriterIndex() ) {
2237  return true; // not a replica DB server
2238  }
2239 
2240  if ( !$pos ) {
2241  // Get the current master position, opening a connection if needed
2242  $index = $this->getWriterIndex();
2243  $flags = self::CONN_SILENCE_ERRORS;
2244  $masterConn = $this->getAnyOpenConnection( $index, $flags );
2245  if ( $masterConn ) {
2246  $pos = $masterConn->getMasterPos();
2247  } else {
2248  $masterConn = $this->getServerConnection( $index, self::DOMAIN_ANY, $flags );
2249  if ( !$masterConn ) {
2250  throw new DBReplicationWaitError(
2251  null,
2252  "Could not obtain a master database connection to get the position"
2253  );
2254  }
2255  $pos = $masterConn->getMasterPos();
2256  $this->closeConnection( $masterConn );
2257  }
2258  }
2259 
2260  if ( $pos instanceof DBMasterPos ) {
2261  $start = microtime( true );
2262  $result = $conn->masterPosWait( $pos, $timeout );
2263  $seconds = max( microtime( true ) - $start, 0 );
2264  if ( $result == -1 || is_null( $result ) ) {
2265  $msg = __METHOD__ . ': timed out waiting on {host} pos {pos} [{seconds}s]';
2266  $this->replLogger->warning( $msg, [
2267  'host' => $conn->getServer(),
2268  'pos' => $pos,
2269  'seconds' => round( $seconds, 6 ),
2270  'trace' => ( new RuntimeException() )->getTraceAsString()
2271  ] );
2272  $ok = false;
2273  } else {
2274  $this->replLogger->debug( __METHOD__ . ': done waiting' );
2275  $ok = true;
2276  }
2277  } else {
2278  $ok = false; // something is misconfigured
2279  $this->replLogger->error(
2280  __METHOD__ . ': could not get master pos for {host}',
2281  [
2282  'host' => $conn->getServer(),
2283  'trace' => ( new RuntimeException() )->getTraceAsString()
2284  ]
2285  );
2286  }
2287 
2288  return $ok;
2289  }
2290 
2303  public function safeWaitForMasterPos( IDatabase $conn, $pos = false, $timeout = null ) {
2304  return $this->waitForMasterPos( $conn, $pos, $timeout );
2305  }
2306 
2307  public function setTransactionListener( $name, callable $callback = null ) {
2308  if ( $callback ) {
2309  $this->trxRecurringCallbacks[$name] = $callback;
2310  } else {
2311  unset( $this->trxRecurringCallbacks[$name] );
2312  }
2314  function ( IDatabase $conn ) use ( $name, $callback ) {
2315  $conn->setTransactionListener( $name, $callback );
2316  }
2317  );
2318  }
2319 
2320  public function setTableAliases( array $aliases ) {
2321  $this->tableAliases = $aliases;
2322  }
2323 
2324  public function setIndexAliases( array $aliases ) {
2325  $this->indexAliases = $aliases;
2326  }
2327 
2328  public function setLocalDomainPrefix( $prefix ) {
2329  // Find connections to explicit foreign domains still marked as in-use...
2330  $domainsInUse = [];
2331  $this->forEachOpenConnection( function ( IDatabase $conn ) use ( &$domainsInUse ) {
2332  // Once reuseConnection() is called on a handle, its reference count goes from 1 to 0.
2333  // Until then, it is still in use by the caller (explicitly or via DBConnRef scope).
2334  if ( $conn->getLBInfo( self::$INFO_FOREIGN_REF_COUNT ) > 0 ) {
2335  $domainsInUse[] = $conn->getDomainID();
2336  }
2337  } );
2338 
2339  // Do not switch connections to explicit foreign domains unless marked as safe
2340  if ( $domainsInUse ) {
2341  $domains = implode( ', ', $domainsInUse );
2342  throw new DBUnexpectedError( null,
2343  "Foreign domain connections are still in use ($domains)" );
2344  }
2345 
2346  $this->setLocalDomain( new DatabaseDomain(
2347  $this->localDomain->getDatabase(),
2348  $this->localDomain->getSchema(),
2349  $prefix
2350  ) );
2351 
2352  // Update the prefix for all local connections...
2353  $this->forEachOpenConnection( function ( IDatabase $conn ) use ( $prefix ) {
2354  if ( !$conn->getLBInfo( self::$INFO_FORIEGN ) ) {
2355  $conn->tablePrefix( $prefix );
2356  }
2357  } );
2358  }
2359 
2360  public function redefineLocalDomain( $domain ) {
2361  $this->closeAll( __METHOD__, $this->id );
2362 
2363  $this->setLocalDomain( DatabaseDomain::newFromId( $domain ) );
2364  }
2365 
2366  public function setTempTablesOnlyMode( $value, $domain ) {
2367  $old = $this->tempTablesOnlyMode[$domain] ?? false;
2368  if ( $value ) {
2369  $this->tempTablesOnlyMode[$domain] = true;
2370  } else {
2371  unset( $this->tempTablesOnlyMode[$domain] );
2372  }
2373 
2374  return $old;
2375  }
2376 
2380  private function setLocalDomain( DatabaseDomain $domain ) {
2381  $this->localDomain = $domain;
2382  // In case a caller assumes that the domain ID is simply <db>-<prefix>, which is almost
2383  // always true, gracefully handle the case when they fail to account for escaping.
2384  if ( $this->localDomain->getTablePrefix() != '' ) {
2385  $this->localDomainIdAlias =
2386  $this->localDomain->getDatabase() . '-' . $this->localDomain->getTablePrefix();
2387  } else {
2388  $this->localDomainIdAlias = $this->localDomain->getDatabase();
2389  }
2390  }
2391 
2398  private function getServerInfoStrict( $i, $field = null ) {
2399  if ( !isset( $this->servers[$i] ) || !is_array( $this->servers[$i] ) ) {
2400  throw new InvalidArgumentException( "No server with index '$i'" );
2401  }
2402 
2403  if ( $field !== null ) {
2404  if ( !array_key_exists( $field, $this->servers[$i] ) ) {
2405  throw new InvalidArgumentException( "No field '$field' in server index '$i'" );
2406  }
2407 
2408  return $this->servers[$i][$field];
2409  }
2410 
2411  return $this->servers[$i];
2412  }
2413 
2417  private function getMasterServerName() {
2418  return $this->getServerName( $this->getWriterIndex() );
2419  }
2420 
2421  function __destruct() {
2422  // Avoid connection leaks for sanity
2423  $this->disable( __METHOD__, $this->ownerId );
2424  }
2425 }
2426 
2430 class_alias( LoadBalancer::class, 'LoadBalancer' );
Helper class that detects high-contention DB queries via profiling calls.
getLocalConnection( $i, $flags=0)
Open a connection to a local DB, or return one if it is already open.
array [] $tableAliases
$aliases Map of (table => (dbname, schema, prefix) map)
setExistingReaderIndex( $group, $index)
Set the server index chosen by the load balancer for use with the given query group.
Database $errorConnection
Connection handle that caused a problem.
runMasterTransactionListenerCallbacks( $fname=__METHOD__, $owner=null)
Run all recurring post-COMMIT/ROLLBACK listener callbacks.
callable $deprecationLogger
Deprecation logger.
getWriterIndex()
Get the server index of the master server.
pendingWriteCallers()
Get the list of method names that did write queries for this transaction.
enforceConnectionFlags(IDatabase $conn, $flags)
dbSchema( $schema=null)
Get/set the db schema.
array [] $groupLoads
Map of (group => server index => weight)
getConnectionRef( $i, $groups=[], $domain=false, $flags=0)
Get a live database handle reference for a real or virtual (DB_MASTER/DB_REPLICA) server index...
getLocalDomainID()
Get the local (and default) database domain ID of connection handles.
closeConnection(IDatabase $conn)
Close a connection.
setTransactionListener( $name, callable $callback=null)
Set a callback via IDatabase::setTransactionListener() on all current and future master connections o...
string bool $readOnlyReason
Reason this instance is read-only or false if not.
getFlag( $flag)
Returns a boolean whether the flag $flag is set for this connection.
Definition: Database.php:802
pickAnyOpenConnection( $candidateConns, $autocommit)
getConnectionIndex( $i, array $groups, $domain)
Get the server index to use for a specified server index and query group list.
isMasterRunningReadOnly(DatabaseDomain $domain)
$context
Definition: load.php:45
$success
forEachOpenReplicaConnection( $callback, array $params=[])
Call a function with each open replica DB connection object.
allowLagged( $mode=null)
Disables/enables lag checks.
getLBInfo( $name=null)
Get properties passed down from the server info array of the load balancer.
setLBInfo( $nameOrArray, $value=null)
Set the entire array or a particular key of the managing load balancer info array.
Definition: Database.php:616
setTempTablesOnlyMode( $value, $domain)
Indicate whether the tables on this domain are only temporary tables for testing. ...
getReaderIndex( $group=false, $domain=false)
Get the server index of the reader connection for a given group.
bool [] $tempTablesOnlyMode
Map of (domain => whether to use "temp tables only" mode)
getAnyOpenConnection( $i, $flags=0)
Get any open connection to a given server index, local or foreign.
applyTransactionRoundFlags(Database $conn)
Make all DB servers with DBO_DEFAULT/DBO_TRX set join the transaction round.
runMasterTransactionIdleCallbacks( $fname=__METHOD__, $owner=null)
Consume and run all pending post-COMMIT/ROLLBACK callbacks and commit dangling transactions.
string null $defaultGroup
Default query group to use with getConnection()
setLBInfo( $nameOrArray, $value=null)
Set the entire array or a particular key of the managing load balancer info array.
TransactionProfiler $trxProfiler
getServer()
Get the server hostname or IP address.
Definition: Database.php:2433
safeGetLag(IDatabase $conn)
Get the lag in seconds for a given connection, or zero if this load balancer does not have replicatio...
resolveGroups( $groups, $i)
Resolve $groups into a list of query groups defining as having database servers.
lazyLoadReplicationPositions()
Make sure that any "waitForPos" positions are loaded and available to doWait()
setTrxEndCallbackSuppression( $suppress)
Whether to disable running of post-COMMIT/ROLLBACK callbacks.
Definition: Database.php:3595
rollbackMasterChanges( $fname=__METHOD__, $owner=null)
Issue ROLLBACK only on master, only if queries were done on connection.
commit( $fname=__METHOD__, $flush=self::FLUSHING_ONE)
Commits a transaction previously started using begin()
hasReached(DBMasterPos $pos)
__construct(array $params)
Construct a manager of IDatabase connection objects.
getServerType( $i)
Get DB type of the server with the specified index.
sanitizeConnectionFlags( $flags, $i, $domain)
bool $connectionAttempted
Whether any connection has been attempted yet.
getReadOnlyReason( $domain=false)
trxLevel()
Gets the current transaction level.
Definition: Database.php:550
lastMasterChangeTimestamp()
Get the timestamp of the latest write query done by this thread.
getLag()
Get the amount of replication lag for this database server.
setLocalDomain(DatabaseDomain $domain)
hasOrMadeRecentMasterChanges( $age=null)
Check if this load balancer object had any recent or still pending writes issued against it by this P...
flushSnapshot( $fname=__METHOD__, $flush=self::FLUSHING_ONE)
Commit any transaction but error out if writes or callbacks are pending.
Helper class used for automatically marking an IDatabase connection as reusable (once it no longer ma...
Definition: DBConnRef.php:29
bool $allowLagged
Whether to disregard replica DB lag as a factor in replica DB selection.
trxLevel()
Gets the current transaction level.
getLBInfo( $name=null)
Get properties passed down from the server info array of the load balancer.
Definition: Database.php:604
hasStreamingReplicaServers()
Whether any replica servers use streaming replication from the master server.
getDomainID()
Return the currently selected domain ID.
const DB_MASTER
Definition: defines.php:26
waitForMasterPos(IDatabase $conn, $pos=false, $timeout=null)
Wait for a replica DB to reach a specified master position.
array $loadMonitorConfig
The LoadMonitor configuration.
int $maxLag
Amount of replication lag, in seconds, that is considered "high".
isMasterConnectionReadOnly(IDatabase $conn, $flags=0)
pendingWriteQueryDuration( $type=self::ESTIMATE_TOTAL)
Get the time spend running write queries for this transaction.
string [] $indexAliases
Map of (index alias => index)
writesOrCallbacksPending()
Whether there is a transaction open with either possible write queries or unresolved pre-commit/commi...
commitMasterChanges( $fname=__METHOD__, $owner=null)
Issue COMMIT on all open master connections to flush changes and view snapshots.
getDBname()
Get the current DB name.
Definition: Database.php:2429
getRandomNonLagged(array $loads, $domain=false, $maxLag=INF)
rollback( $fname=__METHOD__, $flush=self::FLUSHING_ONE)
Rollback a transaction previously started using begin()
static newEmpty()
Get an instance that wraps EmptyBagOStuff.
setLocalDomainPrefix( $prefix)
Set a new table prefix for the existing local domain ID for testing.
closeAll( $fname=__METHOD__, $owner=null)
Close all open connections.
assertNoOpenTransactions()
Assert that all explicit transactions or atomic sections have been closed.
lastDoneWrites()
Get the last time the connection may have been used for a write query.
getServerCount()
Get the number of servers defined in configuration.
getExistingReaderIndex( $group)
Get the server index chosen by the load balancer for use with the given query group.
flushMasterSnapshots( $fname=__METHOD__, $owner=null)
Commit all master DB transactions so as to flush any REPEATABLE-READ or SSI snapshots.
string $agent
Agent name for query profiling.
getLazyConnectionRef( $i, $groups=[], $domain=false, $flags=0)
Get a database handle reference for a real or virtual (DB_MASTER/DB_REPLICA) server index...
An object representing a master or replica DB position in a replicated setup.
Definition: DBMasterPos.php:12
forEachOpenConnection( $callback, array $params=[])
Call a function with each open connection object.
forEachOpenMasterConnection( $callback, array $params=[])
Call a function with each open connection object to a master.
haveIndex( $i)
Returns true if the specified index is a valid server index.
string bool $trxRoundId
Explicit DBO_TRX transaction round active or false if none.
waitFor( $pos)
Set the master position to reach before the next generic group DB handle query.
setTableAliases(array $aliases)
Make certain table names use their own database, schema, and table prefix when passed into SQL querie...
bool DBMasterPos $waitForPos
Replication sync position or false if not set.
laggedReplicaUsed()
Checks whether the database for generic connections this request was both:
mixed $profiler
Class name or object With profileIn/profileOut methods.
getDBname()
Get the current DB name.
getServerName( $i)
Get the host name or IP address of the server with the specified index.
string $trxRoundStage
Stage of the current transaction round in the transaction round life-cycle.
runOnTransactionIdleCallbacks( $trigger)
Actually consume and run any "on transaction idle/resolution" callbacks.
Definition: Database.php:3609
commitAll( $fname=__METHOD__, $owner=null)
Commit transactions on all open connections.
reallyOpenConnection( $i, DatabaseDomain $domain, array $lbInfo)
Open a new network connection to a server (uncached)
undoTransactionRoundFlags(Database $conn)
runTransactionListenerCallbacks( $trigger)
Actually run any "transaction listener" callbacks.
Definition: Database.php:3755
getMaxLag( $domain=false)
Get the hostname and lag time of the most-lagged replica server.
setFlag( $flag, $remember=self::REMEMBER_NOTHING)
Set a flag for this connection.
Definition: Database.php:759
hasMasterChanges()
Whether there are pending changes or callbacks in a transaction by this thread.
restoreFlags( $state=self::RESTORE_PRIOR)
Restore the flags to their prior state before the last setFlag/clearFlag call.
Definition: Database.php:789
waitForOne( $pos, $timeout=null)
Set the master wait position and wait for a generic replica DB to catch up to it. ...
hasReplicaServers()
Whether there are any replica servers configured.
setTransactionListener( $name, callable $callback=null)
Run a callback after each time any transaction commits or rolls back.
runOnTransactionPreCommitCallbacks()
Actually consume and run any "on transaction pre-commit" callbacks.
Definition: Database.php:3678
getLaggedReplicaMode( $domain=false)
const DBO_TRX
Definition: defines.php:12
assertOwnership( $fname, $owner)
Assure that if this instance is owned, the caller is either the owner or is internal.
disable( $fname=__METHOD__, $owner=null)
Close all connections and disable this load balancer.
getForeignConnection( $i, $domain, $flags=0)
Open a connection to a foreign DB, or return one if it is already open.
int $waitTimeout
Seconds to spend waiting on replica DB lag to resolve.
beginMasterChanges( $fname=__METHOD__, $owner=null)
Flush any master transaction snapshots and set DBO_TRX (if DBO_DEFAULT is set)
getLoadMonitor()
Get a LoadMonitor instance.
masterPosWait(DBMasterPos $pos, $timeout)
Wait for the replica DB to catch up to a given master position.
approveMasterChanges(array $options, $fname=__METHOD__, $owner=null)
Perform all pre-commit checks for things like replication safety.
const DBO_DEFAULT
Definition: defines.php:13
getMasterPos()
Get the current master replication position.
Database cluster connection, tracking, load balancing, and transaction manager interface.
Class to handle database/schema/prefix specifications for IDatabase.
Basic database interface for live and lazy-loaded relation database handles.
Definition: IDatabase.php:38
pickReaderIndex(array $loads, $domain=false)
LoggerInterface $queryLogger
setIndexAliases(array $aliases)
Convert certain index names to alternative names before querying the DB.
waitForAll( $pos, $timeout=null)
Set the master wait position and wait for ALL replica DBs to catch up to it.
int [] $readIndexByGroup
The group replica server indexes keyed by group.
Relational database abstraction object.
Definition: Database.php:49
static attributesFromType( $dbType, $driver=null)
Definition: Database.php:443
getConnection( $i, $groups=[], $domain=false, $flags=0)
Get a live handle for a real or virtual (DB_MASTER/DB_REPLICA) server index.
callable null $chronologyCallback
Callback to run before the first connection attempt.
pendingWriteAndCallbackCallers()
List the methods that have write queries or callbacks for the current transaction.
Definition: Database.php:730
commit( $fname=__METHOD__, $flush=self::FLUSHING_ONE)
Commits a transaction previously started using begin()
Definition: Database.php:4066
getLagTimes( $domain=false)
Get an estimate of replication lag (in seconds) for each server.
bool $laggedReplicaMode
Whether the generic reader fell back to a lagged replica DB.
IDatabase [][][] Database [][][] $conns
Map of (connection category => server index => IDatabase[])
$id
var int An identifier for this class instance
array [] $servers
Map of (server index => server config array)
getTopologyRole( $i, array $server)
getServer()
Get the server hostname or IP address.
Exception class for attempted DB access.
safeWaitForMasterPos(IDatabase $conn, $pos=false, $timeout=null)
Wait for a replica DB to reach a specified master position.
Helper class to handle automatically marking connections as reusable (via RAII pattern) as well handl...
getMaintenanceConnectionRef( $i, $groups=[], $domain=false, $flags=0)
Get a live database handle for a real or virtual (DB_MASTER/DB_REPLICA) server index that can be used...
bool $cliMode
Whether this PHP instance is for a CLI script.
pendingMasterChangeCallers()
Get the list of callers that have pending master changes.
string $hostname
Current server name.
Exception class for replica DB wait errors.
static pickRandom( $weights)
Given an array of non-normalised probabilities, this function will select an element and return the a...
Definition: ArrayUtils.php:66
callable [] $trxRecurringCallbacks
Map of (name => callable)
DatabaseDomain $localDomain
Local DB domain ID and default for selectDB() calls.
getServerInfoStrict( $i, $field=null)
string $localDomainIdAlias
Alternate local DB domain instead of DatabaseDomain::getId()
isNonZeroLoad( $i)
Returns true if the specified index is valid and has non-zero load.
close( $fname=__METHOD__, $owner=null)
Close the database connection.
const DB_REPLICA
Definition: defines.php:25
Database connection, tracking, load balancing, and transaction manager for a cluster.
int null $ownerId
Integer ID of the managing LBFactory instance or null if none.
redefineLocalDomain( $domain)
Close all connection and redefine the local domain for testing or schema creation.
openConnection( $i, $domain=false, $flags=0)
getServerInfo( $i)
Return the server info structure for a given index or false if the index is invalid.
callable $errorLogger
Exception logger.
int [] $maxLagByIndex
Map of (server index => seconds of lag considered "high")
flushReplicaSnapshots( $fname=__METHOD__, $owner=null)
Commit all replica DB transactions so as to flush any REPEATABLE-READ or SSI snapshots.
doWait( $index, $open=false, $timeout=null)
Wait for a given replica DB to catch up to the master pos stored in "waitForPos". ...
getServerConnection( $i, $domain, $flags=0)
static factory( $type, $params=[], $connect=self::NEW_CONNECTED)
Construct a Database subclass instance given a database type and parameters.
Definition: Database.php:389
tablePrefix( $prefix=null)
Get/set the table prefix.
reuseConnection(IDatabase $conn)
Mark a live handle as being available for reuse under a different database domain.
getReplicaResumePos()
Get the highest DB replication position for chronology control purposes.
clearFlag( $flag, $remember=self::REMEMBER_NOTHING)
Clear a flag for this connection.
string $lastError
The last DB selection or connection error.
Database error base class.
Definition: DBError.php:30
isOpen( $index)
Test if the specified index represents an open connection.
ping(&$rtt=null)
Ping the server and try to reconnect if it there is no connection.
int $connectionCounter
Total number of new connections ever made with this instance.
finalizeMasterChanges( $fname=__METHOD__, $owner=null)
Run pre-commit callbacks and defer execution of post-commit callbacks.