27 use Wikimedia\ScopedCallback;
50 throw new InvalidArgumentException(
"LBFactory required in 'lbFactory' field." );
52 $this->lbFactory =
$params[
'lbFactory'];
66 [ $cluster, $id, $itemID ] = $this->
parseURL( $url );
67 $ret = $this->fetchBlob( $cluster, $id, $itemID );
69 if ( $itemID !==
false && $ret !==
false ) {
70 return $ret->getItem( $itemID );
87 $batched = $inverseUrlMap = [];
88 foreach ( $urls as $url ) {
89 [ $cluster, $id, $itemID ] = $this->
parseURL( $url );
90 $batched[$cluster][$id][] = $itemID;
93 $inverseUrlMap[$cluster][$id][$itemID] = $url;
96 foreach ( $batched as $cluster => $batchByCluster ) {
97 $res = $this->batchFetchBlobs( $cluster, $batchByCluster );
99 foreach ( $res as $id => $blob ) {
100 foreach ( $batchByCluster[$id] as $itemID ) {
101 $url = $inverseUrlMap[$cluster][$id][$itemID];
102 if ( $itemID ===
false ) {
105 $ret[$url] = $blob->getItem( $itemID );
117 public function store( $location, $data ) {
119 $dbw->newInsertQueryBuilder()
120 ->insertInto( $this->
getTable( $dbw, $location ) )
121 ->row( [
'blob_text' => $data ] )
122 ->caller( __METHOD__ )->execute();
123 $id = $dbw->insertId();
128 return "DB://$location/$id";
135 if ( parent::isReadOnly( $location ) ) {
139 return ( $this->getLoadBalancer( $location )->getReadOnlyReason() !==
false );
148 private function getLoadBalancer( $cluster ) {
149 return $this->lbFactory->getExternalLB( $cluster );
160 $lb = $this->getLoadBalancer( $cluster );
162 return $lb->getConnectionRef(
165 $this->getDomainId( $lb->getServerInfo( $lb->getWriterIndex() ) ),
166 $lb::CONN_TRX_AUTOCOMMIT
178 $lb = $this->getLoadBalancer( $cluster );
180 return $lb->getMaintenanceConnectionRef(
183 $this->getDomainId( $lb->getServerInfo( $lb->getWriterIndex() ) ),
184 $lb::CONN_TRX_AUTOCOMMIT
192 private function getDomainId( array $server ) {
193 if ( $this->isDbDomainExplicit ) {
197 if ( isset( $server[
'dbname'] ) ) {
205 $server[
'schema'] ??
null,
206 $server[
'tablePrefix'] ??
''
209 return $domain->getId();
223 if ( $cluster !==
null ) {
224 $lb = $this->getLoadBalancer( $cluster );
225 $info = $lb->getServerInfo( $lb->getWriterIndex() );
226 if ( isset( $info[
'blobs table'] ) ) {
227 return $info[
'blobs table'];
231 return $db->getLBInfo(
'blobs table' ) ??
'blobs';
244 static $supportedTypes = [
'mysql',
'sqlite' ];
247 if ( !in_array( $dbw->getType(), $supportedTypes,
true ) ) {
248 throw new DBUnexpectedError( $dbw,
"RDBMS type '{$dbw->getType()}' not supported." );
251 $sqlFilePath =
"$IP/maintenance/storage/blobs.sql";
252 $sql = file_get_contents( $sqlFilePath );
253 if ( $sql ===
false ) {
254 throw new RuntimeException(
"Failed to read '$sqlFilePath'." );
257 $rawTable = $this->
getTable( $dbw, $cluster );
258 $encTable = $dbw->tableName( $rawTable );
261 [
'/*$wgDBprefix*/blobs',
'/*_*/blobs' ],
262 [ $encTable, $encTable ],
266 $dbw::QUERY_IGNORE_DBO_TRX
279 private function fetchBlob( $cluster, $id, $itemID ) {
286 static $externalBlobCache = [];
288 $cacheID = ( $itemID === false ) ?
"$cluster/$id" :
"$cluster/$id/";
289 $cacheID =
"$cacheID@{$this->dbDomain}";
291 if ( isset( $externalBlobCache[$cacheID] ) ) {
292 $this->logger->debug( __METHOD__ .
": cache hit on $cacheID" );
294 return $externalBlobCache[$cacheID];
297 $this->logger->debug( __METHOD__ .
": cache miss on $cacheID" );
300 $ret = $dbr->newSelectQueryBuilder()
301 ->select(
'blob_text' )
302 ->from( $this->
getTable( $dbr, $cluster ) )
303 ->where( [
'blob_id' => $id ] )
304 ->caller( __METHOD__ )->fetchField();
305 if ( $ret ===
false ) {
307 $this->logger->warning( __METHOD__ .
": primary DB fallback on $cacheID" );
308 $trxProfiler = $this->lbFactory->getTransactionProfiler();
309 $scope = $trxProfiler->silenceForScope( $trxProfiler::EXPECTATION_REPLICAS_ONLY );
311 $ret = $dbw->newSelectQueryBuilder()
312 ->select(
'blob_text' )
313 ->from( $this->
getTable( $dbw, $cluster ) )
314 ->where( [
'blob_id' => $id ] )
315 ->caller( __METHOD__ )->fetchField();
316 ScopedCallback::consume( $scope );
317 if ( $ret ===
false ) {
318 $this->logger->warning( __METHOD__ .
": primary DB failed to find $cacheID" );
321 if ( $itemID !==
false && $ret !==
false ) {
326 $externalBlobCache = [ $cacheID => $ret ];
339 private function batchFetchBlobs( $cluster, array $ids ) {
341 $res = $dbr->newSelectQueryBuilder()
342 ->select( [
'blob_id',
'blob_text' ] )
343 ->from( $this->
getTable( $dbr, $cluster ) )
344 ->where( [
'blob_id' => array_keys( $ids ) ] )
345 ->caller( __METHOD__ )
349 if ( $res !==
false ) {
350 $this->mergeBatchResult( $ret, $ids, $res );
355 __METHOD__ .
": primary fallback on '$cluster' for: " .
356 implode(
',', array_keys( $ids ) )
358 $trxProfiler = $this->lbFactory->getTransactionProfiler();
359 $scope = $trxProfiler->silenceForScope( $trxProfiler::EXPECTATION_REPLICAS_ONLY );
361 $res = $dbw->newSelectQueryBuilder()
362 ->select( [
'blob_id',
'blob_text' ] )
363 ->from( $this->
getTable( $dbr, $cluster ) )
364 ->where( [
'blob_id' => array_keys( $ids ) ] )
365 ->caller( __METHOD__ )
367 ScopedCallback::consume( $scope );
368 if ( $res ===
false ) {
369 $this->logger->error( __METHOD__ .
": primary failed on '$cluster'" );
371 $this->mergeBatchResult( $ret, $ids, $res );
375 $this->logger->error(
376 __METHOD__ .
": primary on '$cluster' failed locating items: " .
377 implode(
',', array_keys( $ids ) )
390 private function mergeBatchResult( array &$ret, array &$ids, $res ) {
391 foreach ( $res as $row ) {
393 $itemIDs = $ids[$id];
395 if ( count( $itemIDs ) === 1 && reset( $itemIDs ) ===
false ) {
397 $ret[$id] = $row->blob_text;
410 $path = explode(
'/', $url );
if(!defined( 'MEDIAWIKI')) if(ini_get( 'mbstring.func_overload')) if(!defined( 'MW_ENTRY_POINT')) global $IP
Environment checks.
External storage in a SQL database.
getPrimary( $cluster)
Get a primary database connection for the specified cluster.
__construct(array $params)
getReplica( $cluster)
Get a replica DB connection for the specified cluster.
initializeTable( $cluster)
Create the appropriate blobs table on this cluster.
fetchFromURL( $url)
Fetch data from given external store URL.
getTable( $db, $cluster=null)
Get the 'blobs' table name for this database.
store( $location, $data)
Insert a data item into a given location.The location name The data item string|bool The URL of the s...
batchFetchFromURLs(array $urls)
Fetch multiple URLs from given external store.
isReadOnly( $location)
Check if a given location is read-only.The location name bool Whether this location is read-only 1....
Base class for external storage.
array $params
Usage context options for this instance.
string $dbDomain
Default database domain to store content under.
static unserialize(string $str, bool $allowDouble=false)
Unserialize a HistoryBlob.
Class to handle database/schema/prefix specifications for IDatabase.