61 private $dbLoadBalancer;
66 private $extStoreAccess;
86 private $compressBlobs =
false;
91 private $legacyEncoding =
false;
96 private $useExternalStore =
false;
115 $this->dbLoadBalancer = $dbLoadBalancer;
116 $this->extStoreAccess = $extStoreAccess;
117 $this->cache = $cache;
118 $this->dbDomain = $dbDomain;
125 return $this->cacheExpiry;
132 $this->cacheExpiry = $cacheExpiry;
139 return $this->compressBlobs;
146 $this->compressBlobs = $compressBlobs;
154 return $this->legacyEncoding;
166 $this->legacyEncoding = $legacyEncoding;
173 return $this->useExternalStore;
180 $this->useExternalStore = $useExternalStore;
186 private function getDBLoadBalancer() {
187 return $this->dbLoadBalancer;
195 private function getDBConnection( $index ) {
196 $lb = $this->getDBLoadBalancer();
197 return $lb->getConnectionRef( $index, [], $this->dbDomain );
213 # Write to external storage if required
214 if ( $this->useExternalStore ) {
217 $data = $this->extStoreAccess->insert( $data, [
'domain' => $this->dbDomain ] );
227 $flags .=
'external';
236 $dbw->newInsertQueryBuilder()
237 ->insertInto(
'text' )
238 ->row( [
'old_text' => $data,
'old_flags' => $flags ] )
239 ->caller( __METHOD__ )->execute();
241 $textId = $dbw->insertId();
258 public function getBlob( $blobAddress, $queryFlags = 0 ) {
259 Assert::parameterType(
'string', $blobAddress,
'$blobAddress' );
262 $blob = $this->cache->getWithSetCallback(
264 $this->getCacheTTL(),
265 function ( $unused, &$ttl, &$setOpts ) use ( $blobAddress, $queryFlags, &$error ) {
267 [ $result, $errors ] = $this->fetchBlobs( [ $blobAddress ], $queryFlags );
269 $error = $errors[$blobAddress] ??
null;
271 $ttl = WANObjectCache::TTL_UNCACHEABLE;
273 return $result[$blobAddress];
275 $this->getCacheOptions()
279 if ( $error[0] ===
'badrevision' ) {
286 Assert::postcondition( is_string( $blob ),
'Blob must not be null' );
306 [ $blobsByAddress, $errors ] = $this->fetchBlobs( $blobAddresses, $queryFlags );
308 $blobsByAddress = array_map(
static function ( $blob ) {
309 return $blob ===
false ? null : $blob;
310 }, $blobsByAddress );
312 $result = StatusValue::newGood( $blobsByAddress );
313 foreach ( $errors as $error ) {
315 $result->warning( ...$error );
334 private function fetchBlobs( $blobAddresses, $queryFlags ) {
335 $textIdToBlobAddress = [];
338 foreach ( $blobAddresses as $blobAddress ) {
341 }
catch ( InvalidArgumentException $ex ) {
342 throw new BlobAccessException(
343 $ex->getMessage() .
'. Use findBadBlobs.php to remedy.',
350 if ( $schema ===
'bad' ) {
354 .
": loading known-bad content ($blobAddress), returning empty string"
356 $result[$blobAddress] =
'';
357 $errors[$blobAddress] = [
359 'The content of this revision is missing or corrupted (bad schema)'
361 } elseif ( $schema ===
'tt' ) {
362 $textId = intval( $id );
364 if ( $textId < 1 || $id !== (
string)$textId ) {
365 $errors[$blobAddress] = [
367 "Bad blob address: $blobAddress. Use findBadBlobs.php to remedy."
369 $result[$blobAddress] =
false;
372 $textIdToBlobAddress[$textId] = $blobAddress;
374 $errors[$blobAddress] = [
376 "Unknown blob address schema: $schema. Use findBadBlobs.php to remedy."
378 $result[$blobAddress] =
false;
382 $textIds = array_keys( $textIdToBlobAddress );
384 return [ $result, $errors ];
389 ? IDBAccessObject::READ_LATEST_IMMUTABLE
391 [ $index, $options, $fallbackIndex, $fallbackOptions ] =
392 self::getDBOptions( $queryFlags );
394 $dbConnection = $this->getDBConnection( $index );
395 $rows = $dbConnection->newSelectQueryBuilder()
396 ->select( [
'old_id',
'old_text',
'old_flags' ] )
398 ->where( [
'old_id' => $textIds ] )
399 ->options( $options )
400 ->caller( __METHOD__ )->fetchResultSet();
401 $numRows = $rows->numRows();
405 if ( $numRows !== count( $textIds ) && $fallbackIndex !==
null ) {
406 $fetchedTextIds = [];
407 foreach ( $rows as $row ) {
408 $fetchedTextIds[] = $row->old_id;
410 $missingTextIds = array_diff( $textIds, $fetchedTextIds );
411 $dbConnection = $this->getDBConnection( $fallbackIndex );
412 $rowsFromFallback = $dbConnection->newSelectQueryBuilder()
413 ->select( [
'old_id',
'old_text',
'old_flags' ] )
415 ->where( [
'old_id' => $missingTextIds ] )
416 ->options( $fallbackOptions )
417 ->caller( __METHOD__ )->fetchResultSet();
418 $appendIterator =
new AppendIterator();
419 $appendIterator->append( $rows );
420 $appendIterator->append( $rowsFromFallback );
421 $rows = $appendIterator;
424 foreach ( $rows as $row ) {
425 $blobAddress = $textIdToBlobAddress[$row->old_id];
427 if ( $row->old_text !==
null ) {
428 $blob = $this->
expandBlob( $row->old_text, $row->old_flags, $blobAddress );
430 if ( $blob ===
false ) {
431 $errors[$blobAddress] = [
433 "Bad data in text row {$row->old_id}. Use findBadBlobs.php to remedy."
436 $result[$blobAddress] = $blob;
440 if ( count( $result ) !== count( $blobAddresses ) ) {
441 foreach ( $blobAddresses as $blobAddress ) {
442 if ( !isset( $result[$blobAddress ] ) ) {
443 $errors[$blobAddress] = [
445 "Unable to fetch blob at $blobAddress. Use findBadBlobs.php to remedy."
447 $result[$blobAddress] =
false;
451 return [ $result, $errors ];
454 private static function getDBOptions( $bitfield ) {
455 if ( DBAccessObjectUtils::hasFlags( $bitfield, IDBAccessObject::READ_LATEST_IMMUTABLE ) ) {
458 } elseif ( DBAccessObjectUtils::hasFlags( $bitfield, IDBAccessObject::READ_LATEST ) ) {
460 $fallbackIndex =
null;
463 $fallbackIndex =
null;
466 $lockingOptions = [];
467 if ( DBAccessObjectUtils::hasFlags( $bitfield, IDBAccessObject::READ_EXCLUSIVE ) ) {
468 $lockingOptions[] =
'FOR UPDATE';
469 } elseif ( DBAccessObjectUtils::hasFlags( $bitfield, IDBAccessObject::READ_LOCKING ) ) {
470 $lockingOptions[] =
'LOCK IN SHARE MODE';
473 if ( $fallbackIndex !==
null ) {
475 $fallbackOptions = $lockingOptions;
477 $options = $lockingOptions;
478 $fallbackOptions = [];
481 return [ $index, $options, $fallbackIndex, $fallbackOptions ];
495 return $this->cache->makeGlobalKey(
497 $this->dbLoadBalancer->resolveDomainID( $this->dbDomain ),
507 private function getCacheOptions() {
510 'pcTTL' => WANObjectCache::TTL_PROC_LONG,
511 'segmentable' => true
535 public function expandBlob( $raw, $flags, $blobAddress =
null ) {
536 if ( is_string( $flags ) ) {
539 if ( in_array(
'error', $flags ) ) {
541 "The content of this revision is missing or corrupted (error flag)"
546 if ( in_array(
'external', $flags ) ) {
548 $parts = explode(
'://', $url, 2 );
549 if ( count( $parts ) == 1 || $parts[1] ==
'' ) {
553 if ( $blobAddress ) {
555 return $this->cache->getWithSetCallback(
557 $this->getCacheTTL(),
558 function () use ( $url, $flags ) {
560 $blob = $this->extStoreAccess
561 ->fetchFromURL( $url, [
'domain' => $this->dbDomain ] );
563 return $blob ===
false ? false : $this->
decompressData( $blob, $flags );
565 $this->getCacheOptions()
568 $blob = $this->extStoreAccess->fetchFromURL( $url, [
'domain' => $this->dbDomain ] );
569 return $blob ===
false ? false : $this->
decompressData( $blob, $flags );
598 $blobFlags[] =
'utf-8';
600 if ( $this->compressBlobs ) {
601 if ( function_exists(
'gzdeflate' ) ) {
602 $deflated = gzdeflate( $blob );
604 if ( $deflated ===
false ) {
608 $blobFlags[] =
'gzip';
611 wfDebug( __METHOD__ .
" -- no zlib support, not compressing" );
614 return implode(
',', $blobFlags );
633 if ( in_array(
'error', $blobFlags ) ) {
638 if ( in_array(
'gzip', $blobFlags ) ) {
639 # Deal with optional compression of archived pages.
640 # This can be done periodically via maintenance/compressOld.php, and
641 # as pages are saved if $wgCompressRevisions is set.
642 $blob = gzinflate( $blob );
644 if ( $blob ===
false ) {
645 wfWarn( __METHOD__ .
': gzinflate() failed' );
650 if ( in_array(
'object', $blobFlags ) ) {
651 # Generic compressed storage
657 $blob = $obj->getText();
661 if ( $blob !==
false && $this->legacyEncoding
662 && !in_array(
'utf-8', $blobFlags ) && !in_array(
'utf8', $blobFlags )
664 # Old revisions kept around in a legacy encoding?
665 # Upconvert on demand.
666 # ("utf8" checked for compatibility with some broken
667 # conversion scripts 2008-12-30)
669 # *input* string. We just ignore those too.
672 AtEase::suppressWarnings();
673 $blob = iconv( $this->legacyEncoding,
'UTF-8//IGNORE', $blob );
674 AtEase::restoreWarnings();
687 private function getCacheTTL() {
688 $cache = $this->cache;
690 if ( $cache->
getQoS( $cache::ATTR_DURABILITY ) >= $cache::QOS_DURABILITY_RDBMS ) {
692 $ttl = $cache::TTL_UNCACHEABLE;
694 $ttl = $this->cacheExpiry ?: $cache::TTL_UNCACHEABLE;
723 if ( $schema !==
'tt' ) {
727 $textId = intval( $id );
729 if ( !$textId || $id !== (
string)$textId ) {
730 throw new InvalidArgumentException(
"Malformed text_id: $id" );
760 return $flagsString ===
'' ? [] : explode(
',', $flagsString );
774 if ( !preg_match(
'/^([-+.\w]+):([^\s?]+)(\?([^\s]*))?$/', $address, $m ) ) {
775 throw new InvalidArgumentException(
"Bad blob address: $address" );
778 $schema = strtolower( $m[1] );
782 return [ $schema, $id, $parameters ];
786 if ( $this->useExternalStore && $this->extStoreAccess->isReadOnly() ) {
790 return ( $this->getDBLoadBalancer()->getReadOnlyReason() !==
false );
static unserialize(string $str, bool $allowDouble=false)
Unserialize a HistoryBlob.