MediaWiki  master
ExternalStoreDB.php
Go to the documentation of this file.
1 <?php
30 
41  private $lbFactory;
42 
48  public function __construct( array $params ) {
49  parent::__construct( $params );
50  if ( !isset( $params['lbFactory'] ) || !( $params['lbFactory'] instanceof LBFactory ) ) {
51  throw new InvalidArgumentException( "LBFactory required in 'lbFactory' field." );
52  }
53  $this->lbFactory = $params['lbFactory'];
54  }
55 
64  public function fetchFromURL( $url ) {
65  list( $cluster, $id, $itemID ) = $this->parseURL( $url );
66  $ret = $this->fetchBlob( $cluster, $id, $itemID );
67 
68  if ( $itemID !== false && $ret !== false ) {
69  return $ret->getItem( $itemID );
70  }
71 
72  return $ret;
73  }
74 
84  public function batchFetchFromURLs( array $urls ) {
85  $batched = $inverseUrlMap = [];
86  foreach ( $urls as $url ) {
87  list( $cluster, $id, $itemID ) = $this->parseURL( $url );
88  $batched[$cluster][$id][] = $itemID;
89  // false $itemID gets cast to int, but should be ok
90  // since we do === from the $itemID in $batched
91  $inverseUrlMap[$cluster][$id][$itemID] = $url;
92  }
93  $ret = [];
94  foreach ( $batched as $cluster => $batchByCluster ) {
95  $res = $this->batchFetchBlobs( $cluster, $batchByCluster );
97  foreach ( $res as $id => $blob ) {
98  foreach ( $batchByCluster[$id] as $itemID ) {
99  $url = $inverseUrlMap[$cluster][$id][$itemID];
100  if ( $itemID === false ) {
101  $ret[$url] = $blob;
102  } else {
103  $ret[$url] = $blob->getItem( $itemID );
104  }
105  }
106  }
107  }
108 
109  return $ret;
110  }
111 
115  public function store( $location, $data ) {
116  $dbw = $this->getMaster( $location );
117  $dbw->insert(
118  $this->getTable( $dbw, $location ),
119  [ 'blob_text' => $data ],
120  __METHOD__
121  );
122  $id = $dbw->insertId();
123  if ( !$id ) {
124  throw new MWException( __METHOD__ . ': no insert ID' );
125  }
126 
127  return "DB://$location/$id";
128  }
129 
133  public function isReadOnly( $location ) {
134  if ( parent::isReadOnly( $location ) ) {
135  return true;
136  }
137 
138  $lb = $this->getLoadBalancer( $location );
139  $domainId = $this->getDomainId( $lb->getServerInfo( $lb->getWriterIndex() ) );
140 
141  return ( $lb->getReadOnlyReason( $domainId ) !== false );
142  }
143 
150  private function getLoadBalancer( $cluster ) {
151  return $this->lbFactory->getExternalLB( $cluster );
152  }
153 
161  public function getReplica( $cluster ) {
162  $lb = $this->getLoadBalancer( $cluster );
163 
164  return $lb->getConnectionRef(
165  DB_REPLICA,
166  [],
167  $this->getDomainId( $lb->getServerInfo( $lb->getWriterIndex() ) ),
168  $lb::CONN_TRX_AUTOCOMMIT
169  );
170  }
171 
179  public function getSlave( $cluster ) {
180  return $this->getReplica( $cluster );
181  }
182 
189  public function getMaster( $cluster ) {
190  $lb = $this->getLoadBalancer( $cluster );
191 
192  return $lb->getMaintenanceConnectionRef(
193  DB_MASTER,
194  [],
195  $this->getDomainId( $lb->getServerInfo( $lb->getWriterIndex() ) ),
196  $lb::CONN_TRX_AUTOCOMMIT
197  );
198  }
199 
204  private function getDomainId( array $server ) {
205  if ( $this->isDbDomainExplicit ) {
206  return $this->dbDomain; // explicit foreign domain
207  }
208 
209  if ( isset( $server['dbname'] ) ) {
210  // T200471: for b/c, treat any "dbname" field as forcing which database to use.
211  // MediaWiki/LoadBalancer previously did not enforce any concept of a local DB
212  // domain, but rather assumed that the LB server configuration matched $wgDBname.
213  // This check is useful when the external storage DB for this cluster does not use
214  // the same name as the corresponding "main" DB(s) for wikis.
215  $domain = new DatabaseDomain(
216  $server['dbname'],
217  $server['schema'] ?? null,
218  $server['tablePrefix'] ?? ''
219  );
220 
221  return $domain->getId();
222  }
223 
224  return false; // local LB domain
225  }
226 
234  public function getTable( $db, $cluster = null ) {
235  if ( $cluster !== null ) {
236  $lb = $this->getLoadBalancer( $cluster );
237  $info = $lb->getServerInfo( $lb->getWriterIndex() );
238  if ( isset( $info['blobs table'] ) ) {
239  return $info['blobs table'];
240  }
241  }
242 
243  return $db->getLBInfo( 'blobs table' ) ?? 'blobs'; // b/c
244  }
245 
253  public function initializeTable( $cluster ) {
254  global $IP;
255 
256  static $supportedTypes = [ 'mysql', 'sqlite' ];
257 
258  $dbw = $this->getMaster( $cluster );
259  if ( !in_array( $dbw->getType(), $supportedTypes, true ) ) {
260  throw new DBUnexpectedError( $dbw, "RDBMS type '{$dbw->getType()}' not supported." );
261  }
262 
263  $sqlFilePath = "$IP/maintenance/storage/blobs.sql";
264  $sql = file_get_contents( $sqlFilePath );
265  if ( $sql === false ) {
266  throw new RuntimeException( "Failed to read '$sqlFilePath'." );
267  }
268 
269  $rawTable = $this->getTable( $dbw, $cluster ); // e.g. "blobs_cluster23"
270  $encTable = $dbw->tableName( $rawTable );
271  $dbw->query(
272  str_replace(
273  [ '/*$wgDBprefix*/blobs', '/*_*/blobs' ],
274  [ $encTable, $encTable ],
275  $sql
276  ),
277  __METHOD__,
278  $dbw::QUERY_IGNORE_DBO_TRX
279  );
280  }
281 
291  private function fetchBlob( $cluster, $id, $itemID ) {
298  static $externalBlobCache = [];
299 
300  $cacheID = ( $itemID === false ) ? "$cluster/$id" : "$cluster/$id/";
301  $cacheID = "$cacheID@{$this->dbDomain}";
302 
303  if ( isset( $externalBlobCache[$cacheID] ) ) {
304  $this->logger->debug( "ExternalStoreDB::fetchBlob cache hit on $cacheID" );
305 
306  return $externalBlobCache[$cacheID];
307  }
308 
309  $this->logger->debug( "ExternalStoreDB::fetchBlob cache miss on $cacheID" );
310 
311  $dbr = $this->getReplica( $cluster );
312  $ret = $dbr->selectField(
313  $this->getTable( $dbr, $cluster ),
314  'blob_text',
315  [ 'blob_id' => $id ],
316  __METHOD__
317  );
318  if ( $ret === false ) {
319  $this->logger->info( "ExternalStoreDB::fetchBlob master fallback on $cacheID" );
320  // Try the master
321  $dbw = $this->getMaster( $cluster );
322  $ret = $dbw->selectField(
323  $this->getTable( $dbw, $cluster ),
324  'blob_text',
325  [ 'blob_id' => $id ],
326  __METHOD__
327  );
328  if ( $ret === false ) {
329  $this->logger->error( "ExternalStoreDB::fetchBlob master failed to find $cacheID" );
330  }
331  }
332  if ( $itemID !== false && $ret !== false ) {
333  // Unserialise object; caller extracts item
334  $ret = unserialize( $ret );
335  }
336 
337  $externalBlobCache = [ $cacheID => $ret ];
338 
339  return $ret;
340  }
341 
350  private function batchFetchBlobs( $cluster, array $ids ) {
351  $dbr = $this->getReplica( $cluster );
352  $res = $dbr->select(
353  $this->getTable( $dbr, $cluster ),
354  [ 'blob_id', 'blob_text' ],
355  [ 'blob_id' => array_keys( $ids ) ],
356  __METHOD__
357  );
358 
359  $ret = [];
360  if ( $res !== false ) {
361  $this->mergeBatchResult( $ret, $ids, $res );
362  }
363  if ( $ids ) {
364  $this->logger->info(
365  __METHOD__ . ": master fallback on '$cluster' for: " .
366  implode( ',', array_keys( $ids ) )
367  );
368  // Try the master
369  $dbw = $this->getMaster( $cluster );
370  $res = $dbw->select(
371  $this->getTable( $dbr, $cluster ),
372  [ 'blob_id', 'blob_text' ],
373  [ 'blob_id' => array_keys( $ids ) ],
374  __METHOD__ );
375  if ( $res === false ) {
376  $this->logger->error( __METHOD__ . ": master failed on '$cluster'" );
377  } else {
378  $this->mergeBatchResult( $ret, $ids, $res );
379  }
380  }
381  if ( $ids ) {
382  $this->logger->error(
383  __METHOD__ . ": master on '$cluster' failed locating items: " .
384  implode( ',', array_keys( $ids ) )
385  );
386  }
387 
388  return $ret;
389  }
390 
397  private function mergeBatchResult( array &$ret, array &$ids, $res ) {
398  foreach ( $res as $row ) {
399  $id = $row->blob_id;
400  $itemIDs = $ids[$id];
401  unset( $ids[$id] ); // to track if everything is found
402  if ( count( $itemIDs ) === 1 && reset( $itemIDs ) === false ) {
403  // single result stored per blob
404  $ret[$id] = $row->blob_text;
405  } else {
406  // multi result stored per blob
407  $ret[$id] = unserialize( $row->blob_text );
408  }
409  }
410  }
411 
416  protected function parseURL( $url ) {
417  $path = explode( '/', $url );
418 
419  return [
420  $path[2], // cluster
421  $path[3], // id
422  $path[4] ?? false // itemID
423  ];
424  }
425 }
getDomainId(array $server)
Key/value blob storage for a particular storage medium type (e.g.
$IP
Definition: WebStart.php:41
An interface for generating database load balancers.
Definition: LBFactory.php:40
getReplica( $cluster)
Get a replica DB connection for the specified cluster.
fetchFromURL( $url)
The provided URL is in the form of DB://cluster/id or DB://cluster/id/itemid for concatened storage...
getSlave( $cluster)
Get a replica DB connection for the specified cluster.
batchFetchBlobs( $cluster, array $ids)
Fetch multiple blob items out of the database.
const DB_MASTER
Definition: defines.php:26
getTable( $db, $cluster=null)
Get the &#39;blobs&#39; table name for this database.
batchFetchFromURLs(array $urls)
Fetch data from given external store URLs.
LBFactory $lbFactory
getLoadBalancer( $cluster)
Get a LoadBalancer for the specified cluster.
unserialize( $serialized)
initializeTable( $cluster)
Create the appropriate blobs table on this cluster.
__construct(array $params)
mergeBatchResult(array &$ret, array &$ids, $res)
Helper function for self::batchFetchBlobs for merging master/replica DB results.
string $dbDomain
Default database domain to store content under.
Class to handle database/schema/prefix specifications for IDatabase.
array $params
Usage context options for this instance.
store( $location, $data)
isReadOnly( $location)
const DB_REPLICA
Definition: defines.php:25
getMaster( $cluster)
Get a master database connection for the specified cluster.
DB accessible external objects.
fetchBlob( $cluster, $id, $itemID)
Fetch a blob item out of the database; a cache of the last-loaded blob will be kept so that multiple ...
return true
Definition: router.php:92