27 use Wikimedia\ScopedCallback;
50 throw new InvalidArgumentException(
"LBFactory required in 'lbFactory' field." );
52 $this->lbFactory =
$params[
'lbFactory'];
66 [ $cluster, $id, $itemID ] = $this->
parseURL( $url );
67 $ret = $this->fetchBlob( $cluster, $id, $itemID );
69 if ( $itemID !==
false && $ret !==
false ) {
70 return $ret->getItem( $itemID );
87 $batched = $inverseUrlMap = [];
88 foreach ( $urls as $url ) {
89 [ $cluster, $id, $itemID ] = $this->
parseURL( $url );
90 $batched[$cluster][$id][] = $itemID;
93 $inverseUrlMap[$cluster][$id][$itemID] = $url;
96 foreach ( $batched as $cluster => $batchByCluster ) {
97 $res = $this->batchFetchBlobs( $cluster, $batchByCluster );
100 foreach ( $batchByCluster[$id] as $itemID ) {
101 $url = $inverseUrlMap[$cluster][$id][$itemID];
102 if ( $itemID ===
false ) {
105 $ret[$url] =
$blob->getItem( $itemID );
117 public function store( $location, $data ) {
121 [
'blob_text' => $data ],
124 $id = $dbw->insertId();
126 throw new MWException( __METHOD__ .
': no insert ID' );
129 return "DB://$location/$id";
136 if ( parent::isReadOnly( $location ) ) {
140 return ( $this->getLoadBalancer( $location )->getReadOnlyReason() !==
false );
149 private function getLoadBalancer( $cluster ) {
150 return $this->lbFactory->getExternalLB( $cluster );
161 $lb = $this->getLoadBalancer( $cluster );
163 return $lb->getConnectionRef(
166 $this->getDomainId( $lb->getServerInfo( $lb->getWriterIndex() ) ),
167 $lb::CONN_TRX_AUTOCOMMIT
179 $lb = $this->getLoadBalancer( $cluster );
181 return $lb->getMaintenanceConnectionRef(
184 $this->getDomainId( $lb->getServerInfo( $lb->getWriterIndex() ) ),
185 $lb::CONN_TRX_AUTOCOMMIT
193 private function getDomainId( array $server ) {
194 if ( $this->isDbDomainExplicit ) {
198 if ( isset( $server[
'dbname'] ) ) {
206 $server[
'schema'] ??
null,
207 $server[
'tablePrefix'] ??
''
210 return $domain->getId();
224 if ( $cluster !==
null ) {
225 $lb = $this->getLoadBalancer( $cluster );
226 $info = $lb->getServerInfo( $lb->getWriterIndex() );
227 if ( isset( $info[
'blobs table'] ) ) {
228 return $info[
'blobs table'];
232 return $db->getLBInfo(
'blobs table' ) ??
'blobs';
245 static $supportedTypes = [
'mysql',
'sqlite' ];
248 if ( !in_array( $dbw->getType(), $supportedTypes,
true ) ) {
249 throw new DBUnexpectedError( $dbw,
"RDBMS type '{$dbw->getType()}' not supported." );
252 $sqlFilePath =
"$IP/maintenance/storage/blobs.sql";
253 $sql = file_get_contents( $sqlFilePath );
254 if ( $sql ===
false ) {
255 throw new RuntimeException(
"Failed to read '$sqlFilePath'." );
258 $rawTable = $this->
getTable( $dbw, $cluster );
259 $encTable = $dbw->tableName( $rawTable );
262 [
'/*$wgDBprefix*/blobs',
'/*_*/blobs' ],
263 [ $encTable, $encTable ],
267 $dbw::QUERY_IGNORE_DBO_TRX
280 private function fetchBlob( $cluster, $id, $itemID ) {
287 static $externalBlobCache = [];
289 $cacheID = ( $itemID === false ) ?
"$cluster/$id" :
"$cluster/$id/";
290 $cacheID =
"$cacheID@{$this->dbDomain}";
292 if ( isset( $externalBlobCache[$cacheID] ) ) {
293 $this->logger->debug( __METHOD__ .
": cache hit on $cacheID" );
295 return $externalBlobCache[$cacheID];
298 $this->logger->debug( __METHOD__ .
": cache miss on $cacheID" );
301 $ret =
$dbr->selectField(
304 [
'blob_id' => $id ],
307 if ( $ret ===
false ) {
309 $this->logger->warning( __METHOD__ .
": primary DB fallback on $cacheID" );
310 $trxProfiler = $this->lbFactory->getTransactionProfiler();
311 $scope = $trxProfiler->silenceForScope( $trxProfiler::EXPECTATION_REPLICAS_ONLY );
313 $ret = $dbw->selectField(
316 [
'blob_id' => $id ],
319 ScopedCallback::consume( $scope );
320 if ( $ret ===
false ) {
321 $this->logger->warning( __METHOD__ .
": primary DB failed to find $cacheID" );
324 if ( $itemID !==
false && $ret !==
false ) {
329 $externalBlobCache = [ $cacheID => $ret ];
342 private function batchFetchBlobs( $cluster, array $ids ) {
344 $res =
$dbr->newSelectQueryBuilder()
345 ->select( [
'blob_id',
'blob_text' ] )
347 ->where( [
'blob_id' => array_keys( $ids ) ] )
348 ->caller( __METHOD__ )
352 if (
$res !==
false ) {
353 $this->mergeBatchResult( $ret, $ids,
$res );
358 __METHOD__ .
": primary fallback on '$cluster' for: " .
359 implode(
',', array_keys( $ids ) )
361 $trxProfiler = $this->lbFactory->getTransactionProfiler();
362 $scope = $trxProfiler->silenceForScope( $trxProfiler::EXPECTATION_REPLICAS_ONLY );
364 $res = $dbw->newSelectQueryBuilder()
365 ->select( [
'blob_id',
'blob_text' ] )
367 ->where( [
'blob_id' => array_keys( $ids ) ] )
368 ->caller( __METHOD__ )
370 ScopedCallback::consume( $scope );
371 if (
$res ===
false ) {
372 $this->logger->error( __METHOD__ .
": primary failed on '$cluster'" );
374 $this->mergeBatchResult( $ret, $ids,
$res );
378 $this->logger->error(
379 __METHOD__ .
": primary on '$cluster' failed locating items: " .
380 implode(
',', array_keys( $ids ) )
393 private function mergeBatchResult( array &$ret, array &$ids,
$res ) {
394 foreach (
$res as $row ) {
396 $itemIDs = $ids[$id];
398 if ( count( $itemIDs ) === 1 && reset( $itemIDs ) ===
false ) {
400 $ret[$id] = $row->blob_text;
413 $path = explode(
'/', $url );
if(!defined( 'MEDIAWIKI')) if(ini_get( 'mbstring.func_overload')) if(!defined( 'MW_ENTRY_POINT')) global $IP
Environment checks.
External storage in a SQL database.
getPrimary( $cluster)
Get a primary database connection for the specified cluster.
__construct(array $params)
getReplica( $cluster)
Get a replica DB connection for the specified cluster.
initializeTable( $cluster)
Create the appropriate blobs table on this cluster.
fetchFromURL( $url)
Fetch data from given external store URL.
getTable( $db, $cluster=null)
Get the 'blobs' table name for this database.
store( $location, $data)
Insert a data item into a given location.The location name The data item string|bool The URL of the s...
batchFetchFromURLs(array $urls)
Fetch multiple URLs from given external store.
isReadOnly( $location)
Check if a given location is read-only.The location name bool Whether this location is read-only 1....
Base class for external storage.
array $params
Usage context options for this instance.
string $dbDomain
Default database domain to store content under.
static unserialize(string $str, bool $allowDouble=false)
Unserialize a HistoryBlob.
Class to handle database/schema/prefix specifications for IDatabase.