MediaWiki  master
ExternalStoreDB.php
Go to the documentation of this file.
1 <?php
30 
41  private $lbFactory;
42 
48  public function __construct( array $params ) {
49  parent::__construct( $params );
50  if ( !isset( $params['lbFactory'] ) || !( $params['lbFactory'] instanceof LBFactory ) ) {
51  throw new InvalidArgumentException( "LBFactory required in 'lbFactory' field." );
52  }
53  $this->lbFactory = $params['lbFactory'];
54  }
55 
64  public function fetchFromURL( $url ) {
65  list( $cluster, $id, $itemID ) = $this->parseURL( $url );
66  $ret = $this->fetchBlob( $cluster, $id, $itemID );
67 
68  if ( $itemID !== false && $ret !== false ) {
69  return $ret->getItem( $itemID );
70  }
71 
72  return $ret;
73  }
74 
84  public function batchFetchFromURLs( array $urls ) {
85  $batched = $inverseUrlMap = [];
86  foreach ( $urls as $url ) {
87  list( $cluster, $id, $itemID ) = $this->parseURL( $url );
88  $batched[$cluster][$id][] = $itemID;
89  // false $itemID gets cast to int, but should be ok
90  // since we do === from the $itemID in $batched
91  $inverseUrlMap[$cluster][$id][$itemID] = $url;
92  }
93  $ret = [];
94  foreach ( $batched as $cluster => $batchByCluster ) {
95  $res = $this->batchFetchBlobs( $cluster, $batchByCluster );
97  foreach ( $res as $id => $blob ) {
98  foreach ( $batchByCluster[$id] as $itemID ) {
99  $url = $inverseUrlMap[$cluster][$id][$itemID];
100  if ( $itemID === false ) {
101  $ret[$url] = $blob;
102  } else {
103  $ret[$url] = $blob->getItem( $itemID );
104  }
105  }
106  }
107  }
108 
109  return $ret;
110  }
111 
115  public function store( $location, $data ) {
116  $dbw = $this->getMaster( $location );
117  $dbw->insert(
118  $this->getTable( $dbw, $location ),
119  [ 'blob_text' => $data ],
120  __METHOD__
121  );
122  $id = $dbw->insertId();
123  if ( !$id ) {
124  throw new MWException( __METHOD__ . ': no insert ID' );
125  }
126 
127  return "DB://$location/$id";
128  }
129 
133  public function isReadOnly( $location ) {
134  if ( parent::isReadOnly( $location ) ) {
135  return true;
136  }
137 
138  $lb = $this->getLoadBalancer( $location );
139  $domainId = $this->getDomainId( $lb->getServerInfo( $lb->getWriterIndex() ) );
140 
141  return ( $lb->getReadOnlyReason( $domainId ) !== false );
142  }
143 
150  private function getLoadBalancer( $cluster ) {
151  return $this->lbFactory->getExternalLB( $cluster );
152  }
153 
161  public function getReplica( $cluster ) {
162  $lb = $this->getLoadBalancer( $cluster );
163 
164  return $lb->getConnectionRef(
165  DB_REPLICA,
166  [],
167  $this->getDomainId( $lb->getServerInfo( $lb->getWriterIndex() ) ),
168  $lb::CONN_TRX_AUTOCOMMIT
169  );
170  }
171 
179  public function getSlave( $cluster ) {
180  wfDeprecated( __METHOD__, '1.34' );
181  return $this->getReplica( $cluster );
182  }
183 
190  public function getMaster( $cluster ) {
191  $lb = $this->getLoadBalancer( $cluster );
192 
193  return $lb->getMaintenanceConnectionRef(
194  DB_MASTER,
195  [],
196  $this->getDomainId( $lb->getServerInfo( $lb->getWriterIndex() ) ),
197  $lb::CONN_TRX_AUTOCOMMIT
198  );
199  }
200 
205  private function getDomainId( array $server ) {
206  if ( $this->isDbDomainExplicit ) {
207  return $this->dbDomain; // explicit foreign domain
208  }
209 
210  if ( isset( $server['dbname'] ) ) {
211  // T200471: for b/c, treat any "dbname" field as forcing which database to use.
212  // MediaWiki/LoadBalancer previously did not enforce any concept of a local DB
213  // domain, but rather assumed that the LB server configuration matched $wgDBname.
214  // This check is useful when the external storage DB for this cluster does not use
215  // the same name as the corresponding "main" DB(s) for wikis.
216  $domain = new DatabaseDomain(
217  $server['dbname'],
218  $server['schema'] ?? null,
219  $server['tablePrefix'] ?? ''
220  );
221 
222  return $domain->getId();
223  }
224 
225  return false; // local LB domain
226  }
227 
235  public function getTable( $db, $cluster = null ) {
236  if ( $cluster !== null ) {
237  $lb = $this->getLoadBalancer( $cluster );
238  $info = $lb->getServerInfo( $lb->getWriterIndex() );
239  if ( isset( $info['blobs table'] ) ) {
240  return $info['blobs table'];
241  }
242  }
243 
244  return $db->getLBInfo( 'blobs table' ) ?? 'blobs'; // b/c
245  }
246 
254  public function initializeTable( $cluster ) {
255  global $IP;
256 
257  static $supportedTypes = [ 'mysql', 'sqlite' ];
258 
259  $dbw = $this->getMaster( $cluster );
260  if ( !in_array( $dbw->getType(), $supportedTypes, true ) ) {
261  throw new DBUnexpectedError( $dbw, "RDBMS type '{$dbw->getType()}' not supported." );
262  }
263 
264  $sqlFilePath = "$IP/maintenance/storage/blobs.sql";
265  $sql = file_get_contents( $sqlFilePath );
266  if ( $sql === false ) {
267  throw new RuntimeException( "Failed to read '$sqlFilePath'." );
268  }
269 
270  $rawTable = $this->getTable( $dbw, $cluster ); // e.g. "blobs_cluster23"
271  $encTable = $dbw->tableName( $rawTable );
272  $dbw->query(
273  str_replace(
274  [ '/*$wgDBprefix*/blobs', '/*_*/blobs' ],
275  [ $encTable, $encTable ],
276  $sql
277  ),
278  __METHOD__,
279  $dbw::QUERY_IGNORE_DBO_TRX
280  );
281  }
282 
292  private function fetchBlob( $cluster, $id, $itemID ) {
299  static $externalBlobCache = [];
300 
301  $cacheID = ( $itemID === false ) ? "$cluster/$id" : "$cluster/$id/";
302  $cacheID = "$cacheID@{$this->dbDomain}";
303 
304  if ( isset( $externalBlobCache[$cacheID] ) ) {
305  $this->logger->debug( "ExternalStoreDB::fetchBlob cache hit on $cacheID" );
306 
307  return $externalBlobCache[$cacheID];
308  }
309 
310  $this->logger->debug( "ExternalStoreDB::fetchBlob cache miss on $cacheID" );
311 
312  $dbr = $this->getReplica( $cluster );
313  $ret = $dbr->selectField(
314  $this->getTable( $dbr, $cluster ),
315  'blob_text',
316  [ 'blob_id' => $id ],
317  __METHOD__
318  );
319  if ( $ret === false ) {
320  $this->logger->info( "ExternalStoreDB::fetchBlob master fallback on $cacheID" );
321  // Try the master
322  $dbw = $this->getMaster( $cluster );
323  $ret = $dbw->selectField(
324  $this->getTable( $dbw, $cluster ),
325  'blob_text',
326  [ 'blob_id' => $id ],
327  __METHOD__
328  );
329  if ( $ret === false ) {
330  $this->logger->error( "ExternalStoreDB::fetchBlob master failed to find $cacheID" );
331  }
332  }
333  if ( $itemID !== false && $ret !== false ) {
334  // Unserialise object; caller extracts item
335  $ret = unserialize( $ret );
336  }
337 
338  $externalBlobCache = [ $cacheID => $ret ];
339 
340  return $ret;
341  }
342 
351  private function batchFetchBlobs( $cluster, array $ids ) {
352  $dbr = $this->getReplica( $cluster );
353  $res = $dbr->select(
354  $this->getTable( $dbr, $cluster ),
355  [ 'blob_id', 'blob_text' ],
356  [ 'blob_id' => array_keys( $ids ) ],
357  __METHOD__
358  );
359 
360  $ret = [];
361  if ( $res !== false ) {
362  $this->mergeBatchResult( $ret, $ids, $res );
363  }
364  if ( $ids ) {
365  $this->logger->info(
366  __METHOD__ . ": master fallback on '$cluster' for: " .
367  implode( ',', array_keys( $ids ) )
368  );
369  // Try the master
370  $dbw = $this->getMaster( $cluster );
371  $res = $dbw->select(
372  $this->getTable( $dbr, $cluster ),
373  [ 'blob_id', 'blob_text' ],
374  [ 'blob_id' => array_keys( $ids ) ],
375  __METHOD__ );
376  if ( $res === false ) {
377  $this->logger->error( __METHOD__ . ": master failed on '$cluster'" );
378  } else {
379  $this->mergeBatchResult( $ret, $ids, $res );
380  }
381  }
382  if ( $ids ) {
383  $this->logger->error(
384  __METHOD__ . ": master on '$cluster' failed locating items: " .
385  implode( ',', array_keys( $ids ) )
386  );
387  }
388 
389  return $ret;
390  }
391 
398  private function mergeBatchResult( array &$ret, array &$ids, $res ) {
399  foreach ( $res as $row ) {
400  $id = $row->blob_id;
401  $itemIDs = $ids[$id];
402  unset( $ids[$id] ); // to track if everything is found
403  if ( count( $itemIDs ) === 1 && reset( $itemIDs ) === false ) {
404  // single result stored per blob
405  $ret[$id] = $row->blob_text;
406  } else {
407  // multi result stored per blob
408  $ret[$id] = unserialize( $row->blob_text );
409  }
410  }
411  }
412 
417  protected function parseURL( $url ) {
418  $path = explode( '/', $url );
419 
420  return [
421  $path[2], // cluster
422  $path[3], // id
423  $path[4] ?? false // itemID
424  ];
425  }
426 }
ExternalStoreDB\batchFetchBlobs
batchFetchBlobs( $cluster, array $ids)
Fetch multiple blob items out of the database.
Definition: ExternalStoreDB.php:351
ExternalStoreMedium
Key/value blob storage for a particular storage medium type (e.g.
Definition: ExternalStoreMedium.php:38
ExternalStoreDB\__construct
__construct(array $params)
Definition: ExternalStoreDB.php:48
ExternalStoreMedium\$params
array $params
Usage context options for this instance.
Definition: ExternalStoreMedium.php:40
HistoryBlob
Base class for general text storage via the "object" flag in old_flags, or two-part external storage ...
Definition: HistoryBlob.php:28
ExternalStoreDB\getSlave
getSlave( $cluster)
Get a replica DB connection for the specified cluster.
Definition: ExternalStoreDB.php:179
ExternalStoreDB\parseURL
parseURL( $url)
Definition: ExternalStoreDB.php:417
ExternalStoreDB
DB accessible external objects.
Definition: ExternalStoreDB.php:39
ExternalStoreDB\store
store( $location, $data)
Insert a data item into a given location.The location name The data item string|bool The URL of the s...
Definition: ExternalStoreDB.php:115
ExternalStoreDB\batchFetchFromURLs
batchFetchFromURLs(array $urls)
Fetch data from given external store URLs.
Definition: ExternalStoreDB.php:84
ExternalStoreDB\$lbFactory
LBFactory $lbFactory
Definition: ExternalStoreDB.php:41
ExternalStoreMedium\$dbDomain
string $dbDomain
Default database domain to store content under.
Definition: ExternalStoreMedium.php:42
$res
$res
Definition: testCompression.php:57
ExternalStoreDB\fetchBlob
fetchBlob( $cluster, $id, $itemID)
Fetch a blob item out of the database; a cache of the last-loaded blob will be kept so that multiple ...
Definition: ExternalStoreDB.php:292
Wikimedia\Rdbms\IDatabase
Basic database interface for live and lazy-loaded relation database handles.
Definition: IDatabase.php:38
ExternalStoreDB\getDomainId
getDomainId(array $server)
Definition: ExternalStoreDB.php:205
$dbr
$dbr
Definition: testCompression.php:54
Wikimedia\Rdbms\MaintainableDBConnRef
Helper class to handle automatically marking connections as reusable (via RAII pattern) as well handl...
Definition: MaintainableDBConnRef.php:13
ExternalStoreDB\getLoadBalancer
getLoadBalancer( $cluster)
Get a LoadBalancer for the specified cluster.
Definition: ExternalStoreDB.php:150
MWException
MediaWiki exception.
Definition: MWException.php:29
wfDeprecated
wfDeprecated( $function, $version=false, $component=false, $callerOffset=2)
Logs a warning that $function is deprecated.
Definition: GlobalFunctions.php:1033
$blob
$blob
Definition: testCompression.php:70
ExternalStoreDB\isReadOnly
isReadOnly( $location)
Check if a given location is read-only.The location name bool Whether this location is read-only 1....
Definition: ExternalStoreDB.php:133
DB_REPLICA
const DB_REPLICA
Definition: defines.php:25
DB_MASTER
const DB_MASTER
Definition: defines.php:26
ExternalStoreDB\fetchFromURL
fetchFromURL( $url)
The provided URL is in the form of DB://cluster/id or DB://cluster/id/itemid for concatened storage.
Definition: ExternalStoreDB.php:64
ExternalStoreDB\getReplica
getReplica( $cluster)
Get a replica DB connection for the specified cluster.
Definition: ExternalStoreDB.php:161
Wikimedia\Rdbms\DBUnexpectedError
@newable Stable to extend
Definition: DBUnexpectedError.php:29
Wikimedia\Rdbms\DBConnRef
Helper class used for automatically marking an IDatabase connection as reusable (once it no longer ma...
Definition: DBConnRef.php:29
unserialize
unserialize( $serialized)
Definition: ApiMessageTrait.php:146
Wikimedia\Rdbms\LBFactory
An interface for generating database load balancers.
Definition: LBFactory.php:41
$path
$path
Definition: NoLocalSettings.php:25
ExternalStoreDB\initializeTable
initializeTable( $cluster)
Create the appropriate blobs table on this cluster.
Definition: ExternalStoreDB.php:254
Wikimedia\Rdbms\DatabaseDomain
Class to handle database/schema/prefix specifications for IDatabase.
Definition: DatabaseDomain.php:40
$IP
$IP
Definition: WebStart.php:49
ExternalStoreDB\mergeBatchResult
mergeBatchResult(array &$ret, array &$ids, $res)
Helper function for self::batchFetchBlobs for merging master/replica DB results.
Definition: ExternalStoreDB.php:398
ExternalStoreDB\getTable
getTable( $db, $cluster=null)
Get the 'blobs' table name for this database.
Definition: ExternalStoreDB.php:235
ExternalStoreDB\getMaster
getMaster( $cluster)
Get a master database connection for the specified cluster.
Definition: ExternalStoreDB.php:190
Wikimedia\Rdbms\ILoadBalancer
Database cluster connection, tracking, load balancing, and transaction manager interface.
Definition: ILoadBalancer.php:81