MediaWiki master
ExternalStoreDB.php
Go to the documentation of this file.
1<?php
27use Wikimedia\ScopedCallback;
28
40 private $lbFactory;
41
47 public function __construct( array $params ) {
48 parent::__construct( $params );
49 if ( !isset( $params['lbFactory'] ) || !( $params['lbFactory'] instanceof LBFactory ) ) {
50 throw new InvalidArgumentException( "LBFactory required in 'lbFactory' field." );
51 }
52 $this->lbFactory = $params['lbFactory'];
53 }
54
65 public function fetchFromURL( $url ) {
66 [ $cluster, $id, $itemID ] = $this->parseURL( $url );
67 $ret = $this->fetchBlob( $cluster, $id, $itemID );
68
69 if ( $itemID !== false && $ret !== false ) {
70 return $ret->getItem( $itemID );
71 }
72
73 return $ret;
74 }
75
86 public function batchFetchFromURLs( array $urls ) {
87 $batched = $inverseUrlMap = [];
88 foreach ( $urls as $url ) {
89 [ $cluster, $id, $itemID ] = $this->parseURL( $url );
90 $batched[$cluster][$id][] = $itemID;
91 // false $itemID gets cast to int, but should be ok
92 // since we do === from the $itemID in $batched
93 $inverseUrlMap[$cluster][$id][$itemID] = $url;
94 }
95 $ret = [];
96 foreach ( $batched as $cluster => $batchByCluster ) {
97 $res = $this->batchFetchBlobs( $cluster, $batchByCluster );
99 foreach ( $res as $id => $blob ) {
100 foreach ( $batchByCluster[$id] as $itemID ) {
101 $url = $inverseUrlMap[$cluster][$id][$itemID];
102 if ( $itemID === false ) {
103 $ret[$url] = $blob;
104 } else {
105 $ret[$url] = $blob->getItem( $itemID );
106 }
107 }
108 }
109 }
110
111 return $ret;
112 }
113
117 public function store( $location, $data ) {
118 $dbw = $this->getPrimary( $location );
119 $dbw->newInsertQueryBuilder()
120 ->insertInto( $this->getTable( $dbw, $location ) )
121 ->row( [ 'blob_text' => $data ] )
122 ->caller( __METHOD__ )->execute();
123 $id = $dbw->insertId();
124 if ( !$id ) {
125 throw new ExternalStoreException( __METHOD__ . ': no insert ID' );
126 }
127
128 return "DB://$location/$id";
129 }
130
134 public function isReadOnly( $location ) {
135 if ( parent::isReadOnly( $location ) ) {
136 return true;
137 }
138
139 return ( $this->getLoadBalancer( $location )->getReadOnlyReason() !== false );
140 }
141
148 private function getLoadBalancer( $cluster ) {
149 return $this->lbFactory->getExternalLB( $cluster );
150 }
151
159 public function getReplica( $cluster ) {
160 $lb = $this->getLoadBalancer( $cluster );
161
162 return $lb->getConnectionRef(
164 [],
165 $this->getDomainId( $lb->getServerInfo( $lb->getWriterIndex() ) ),
166 $lb::CONN_TRX_AUTOCOMMIT
167 );
168 }
169
177 public function getPrimary( $cluster ) {
178 $lb = $this->getLoadBalancer( $cluster );
179
180 return $lb->getMaintenanceConnectionRef(
182 [],
183 $this->getDomainId( $lb->getServerInfo( $lb->getWriterIndex() ) ),
184 $lb::CONN_TRX_AUTOCOMMIT
185 );
186 }
187
192 private function getDomainId( array $server ) {
193 if ( $this->isDbDomainExplicit ) {
194 return $this->dbDomain; // explicit foreign domain
195 }
196
197 if ( isset( $server['dbname'] ) ) {
198 // T200471: for b/c, treat any "dbname" field as forcing which database to use.
199 // MediaWiki/LoadBalancer previously did not enforce any concept of a local DB
200 // domain, but rather assumed that the LB server configuration matched $wgDBname.
201 // This check is useful when the external storage DB for this cluster does not use
202 // the same name as the corresponding "main" DB(s) for wikis.
203 $domain = new DatabaseDomain(
204 $server['dbname'],
205 $server['schema'] ?? null,
206 $server['tablePrefix'] ?? ''
207 );
208
209 return $domain->getId();
210 }
211
212 return false; // local LB domain
213 }
214
222 public function getTable( $db, $cluster = null ) {
223 if ( $cluster !== null ) {
224 $lb = $this->getLoadBalancer( $cluster );
225 $info = $lb->getServerInfo( $lb->getWriterIndex() );
226 if ( isset( $info['blobs table'] ) ) {
227 return $info['blobs table'];
228 }
229 }
230
231 return $db->getLBInfo( 'blobs table' ) ?? 'blobs'; // b/c
232 }
233
241 public function initializeTable( $cluster ) {
242 global $IP;
243
244 static $supportedTypes = [ 'mysql', 'sqlite' ];
245
246 $dbw = $this->getPrimary( $cluster );
247 if ( !in_array( $dbw->getType(), $supportedTypes, true ) ) {
248 throw new DBUnexpectedError( $dbw, "RDBMS type '{$dbw->getType()}' not supported." );
249 }
250
251 $sqlFilePath = "$IP/maintenance/storage/blobs.sql";
252 $sql = file_get_contents( $sqlFilePath );
253 if ( $sql === false ) {
254 throw new RuntimeException( "Failed to read '$sqlFilePath'." );
255 }
256
257 $rawTable = $this->getTable( $dbw, $cluster ); // e.g. "blobs_cluster23"
258 $encTable = $dbw->tableName( $rawTable );
259 $dbw->query(
260 str_replace(
261 [ '/*$wgDBprefix*/blobs', '/*_*/blobs' ],
262 [ $encTable, $encTable ],
263 $sql
264 ),
265 __METHOD__,
266 $dbw::QUERY_IGNORE_DBO_TRX
267 );
268 }
269
279 private function fetchBlob( $cluster, $id, $itemID ) {
286 static $externalBlobCache = [];
287
288 $cacheID = ( $itemID === false ) ? "$cluster/$id" : "$cluster/$id/";
289 $cacheID = "$cacheID@{$this->dbDomain}";
290
291 if ( isset( $externalBlobCache[$cacheID] ) ) {
292 $this->logger->debug( __METHOD__ . ": cache hit on $cacheID" );
293
294 return $externalBlobCache[$cacheID];
295 }
296
297 $this->logger->debug( __METHOD__ . ": cache miss on $cacheID" );
298
299 $dbr = $this->getReplica( $cluster );
300 $ret = $dbr->newSelectQueryBuilder()
301 ->select( 'blob_text' )
302 ->from( $this->getTable( $dbr, $cluster ) )
303 ->where( [ 'blob_id' => $id ] )
304 ->caller( __METHOD__ )->fetchField();
305 if ( $ret === false ) {
306 // Try the primary DB
307 $this->logger->warning( __METHOD__ . ": primary DB fallback on $cacheID" );
308 $trxProfiler = $this->lbFactory->getTransactionProfiler();
309 $scope = $trxProfiler->silenceForScope( $trxProfiler::EXPECTATION_REPLICAS_ONLY );
310 $dbw = $this->getPrimary( $cluster );
311 $ret = $dbw->newSelectQueryBuilder()
312 ->select( 'blob_text' )
313 ->from( $this->getTable( $dbw, $cluster ) )
314 ->where( [ 'blob_id' => $id ] )
315 ->caller( __METHOD__ )->fetchField();
316 ScopedCallback::consume( $scope );
317 if ( $ret === false ) {
318 $this->logger->warning( __METHOD__ . ": primary DB failed to find $cacheID" );
319 }
320 }
321 if ( $itemID !== false && $ret !== false ) {
322 // Unserialise object; caller extracts item
323 $ret = HistoryBlobUtils::unserialize( $ret );
324 }
325
326 $externalBlobCache = [ $cacheID => $ret ];
327
328 return $ret;
329 }
330
339 private function batchFetchBlobs( $cluster, array $ids ) {
340 $dbr = $this->getReplica( $cluster );
341 $res = $dbr->newSelectQueryBuilder()
342 ->select( [ 'blob_id', 'blob_text' ] )
343 ->from( $this->getTable( $dbr, $cluster ) )
344 ->where( [ 'blob_id' => array_keys( $ids ) ] )
345 ->caller( __METHOD__ )
346 ->fetchResultSet();
347
348 $ret = [];
349 if ( $res !== false ) {
350 $this->mergeBatchResult( $ret, $ids, $res );
351 }
352 if ( $ids ) {
353 // Try the primary
354 $this->logger->info(
355 __METHOD__ . ": primary fallback on '$cluster' for: " .
356 implode( ',', array_keys( $ids ) )
357 );
358 $trxProfiler = $this->lbFactory->getTransactionProfiler();
359 $scope = $trxProfiler->silenceForScope( $trxProfiler::EXPECTATION_REPLICAS_ONLY );
360 $dbw = $this->getPrimary( $cluster );
361 $res = $dbw->newSelectQueryBuilder()
362 ->select( [ 'blob_id', 'blob_text' ] )
363 ->from( $this->getTable( $dbr, $cluster ) )
364 ->where( [ 'blob_id' => array_keys( $ids ) ] )
365 ->caller( __METHOD__ )
366 ->fetchResultSet();
367 ScopedCallback::consume( $scope );
368 if ( $res === false ) {
369 $this->logger->error( __METHOD__ . ": primary failed on '$cluster'" );
370 } else {
371 $this->mergeBatchResult( $ret, $ids, $res );
372 }
373 }
374 if ( $ids ) {
375 $this->logger->error(
376 __METHOD__ . ": primary on '$cluster' failed locating items: " .
377 implode( ',', array_keys( $ids ) )
378 );
379 }
380
381 return $ret;
382 }
383
390 private function mergeBatchResult( array &$ret, array &$ids, $res ) {
391 foreach ( $res as $row ) {
392 $id = $row->blob_id;
393 $itemIDs = $ids[$id];
394 unset( $ids[$id] ); // to track if everything is found
395 if ( count( $itemIDs ) === 1 && reset( $itemIDs ) === false ) {
396 // single result stored per blob
397 $ret[$id] = $row->blob_text;
398 } else {
399 // multi result stored per blob
400 $ret[$id] = HistoryBlobUtils::unserialize( $row->blob_text );
401 }
402 }
403 }
404
409 protected function parseURL( $url ) {
410 $path = explode( '/', $url );
411
412 return [
413 $path[2], // cluster
414 $path[3], // id
415 $path[4] ?? false // itemID
416 ];
417 }
418}
if(!defined( 'MEDIAWIKI')) if(ini_get('mbstring.func_overload')) if(!defined( 'MW_ENTRY_POINT')) global $IP
Environment checks.
Definition Setup.php:98
External storage in a SQL database.
getPrimary( $cluster)
Get a primary database connection for the specified cluster.
__construct(array $params)
getReplica( $cluster)
Get a replica DB connection for the specified cluster.
initializeTable( $cluster)
Create the appropriate blobs table on this cluster.
fetchFromURL( $url)
Fetch data from given external store URL.
getTable( $db, $cluster=null)
Get the 'blobs' table name for this database.
store( $location, $data)
Insert a data item into a given location.string|bool The URL of the stored data item,...
batchFetchFromURLs(array $urls)
Fetch multiple URLs from given external store.
isReadOnly( $location)
Check if a given location is read-only.bool Whether this location is read-only 1.31
Base class for external storage.
array $params
Usage context options for this instance.
string $dbDomain
Default database domain to store content under.
static unserialize(string $str, bool $allowDouble=false)
Unserialize a HistoryBlob.
Helper class used for automatically marking an IDatabase connection as reusable (once it no longer ma...
Definition DBConnRef.php:29
Class to handle database/schema/prefix specifications for IDatabase.
Base class for general text storage via the "object" flag in old_flags, or two-part external storage ...
Basic database interface for live and lazy-loaded relation database handles.
Definition IDatabase.php:36
This class is a delegate to ILBFactory for a given database cluster.
const DB_REPLICA
Definition defines.php:26
const DB_PRIMARY
Definition defines.php:28