SqlBlobStore.php

Go to the documentation of this file.

<?php
namespace MediaWiki\Storage;
 
use AppendIterator;
use DBAccessObjectUtils;
use IDBAccessObject;
use IExpiringStore;
use InvalidArgumentException;
use MWException;
use StatusValue;
use WANObjectCache;
use ExternalStoreAccess;
use Wikimedia\Assert\Assert;
use Wikimedia\AtEase\AtEase;
use Wikimedia\Rdbms\IDatabase;
use Wikimedia\Rdbms\ILoadBalancer;
 
class SqlBlobStore implements IDBAccessObject, BlobStore {
 
    // Note: the name has been taken unchanged from the Revision class.
    const TEXT_CACHE_GROUP = 'revisiontext:10';
 
    private $dbLoadBalancer;
 
    private $extStoreAccess;
 
    private $cache;
 
    private $dbDomain;
 
    private $cacheExpiry = 604800; // 7 days
 
    private $compressBlobs = false;
 
    private $legacyEncoding = false;
 
    private $useExternalStore = false;
 
    public function __construct(
        ILoadBalancer $dbLoadBalancer,
        ExternalStoreAccess $extStoreAccess,
        WANObjectCache $cache,
        $dbDomain = false
    ) {
        $this->dbLoadBalancer = $dbLoadBalancer;
        $this->extStoreAccess = $extStoreAccess;
        $this->cache = $cache;
        $this->dbDomain = $dbDomain;
    }
 
    public function getCacheExpiry() {
        return $this->cacheExpiry;
    }
 
    public function setCacheExpiry( $cacheExpiry ) {
        Assert::parameterType( 'integer', $cacheExpiry, '$cacheExpiry' );
 
        $this->cacheExpiry = $cacheExpiry;
    }
 
    public function getCompressBlobs() {
        return $this->compressBlobs;
    }
 
    public function setCompressBlobs( $compressBlobs ) {
        $this->compressBlobs = $compressBlobs;
    }
 
    public function getLegacyEncoding() {
        return $this->legacyEncoding;
    }
 
    public function getLegacyEncodingConversionLang() {
        wfDeprecated( __METHOD__ );
        return null;
    }
 
    public function setLegacyEncoding( $legacyEncoding ) {
        Assert::parameterType( 'string', $legacyEncoding, '$legacyEncoding' );
 
        $this->legacyEncoding = $legacyEncoding;
    }
 
    public function getUseExternalStore() {
        return $this->useExternalStore;
    }
 
    public function setUseExternalStore( $useExternalStore ) {
        Assert::parameterType( 'boolean', $useExternalStore, '$useExternalStore' );
 
        $this->useExternalStore = $useExternalStore;
    }
 
    private function getDBLoadBalancer() {
        return $this->dbLoadBalancer;
    }
 
    private function getDBConnection( $index ) {
        $lb = $this->getDBLoadBalancer();
        return $lb->getConnectionRef( $index, [], $this->dbDomain );
    }
 
    public function storeBlob( $data, $hints = [] ) {
        try {
            $flags = $this->compressData( $data );
 
            # Write to external storage if required
            if ( $this->useExternalStore ) {
                // Store and get the URL
                $data = $this->extStoreAccess->insert( $data, [ 'domain' => $this->dbDomain ] );
                if ( !$data ) {
                    throw new BlobAccessException( "Failed to store text to external storage" );
                }
                if ( $flags ) {
                    $flags .= ',';
                }
                $flags .= 'external';
 
                // TODO: we could also return an address for the external store directly here.
                // That would mean bypassing the text table entirely when the external store is
                // used. We'll need to assess expected fallout before doing that.
            }
 
            $dbw = $this->getDBConnection( DB_MASTER );
 
            $old_id = $dbw->nextSequenceValue( 'text_old_id_seq' );
            $dbw->insert(
                'text',
                [
                    'old_id' => $old_id,
                    'old_text' => $data,
                    'old_flags' => $flags,
                ],
                __METHOD__
            );
 
            $textId = $dbw->insertId();
 
            return self::makeAddressFromTextId( $textId );
        } catch ( MWException $e ) {
            throw new BlobAccessException( $e->getMessage(), 0, $e );
        }
    }
 
    public function getBlob( $blobAddress, $queryFlags = 0 ) {
        Assert::parameterType( 'string', $blobAddress, '$blobAddress' );
 
        $error = null;
        $blob = $this->cache->getWithSetCallback(
            $this->getCacheKey( $blobAddress ),
            $this->getCacheTTL(),
            function ( $unused, &$ttl, &$setOpts ) use ( $blobAddress, $queryFlags, &$error ) {
                // Ignore $setOpts; blobs are immutable and negatives are not cached
                list( $result, $errors ) = $this->fetchBlobs( [ $blobAddress ], $queryFlags );
                // No negative caching; negative hits on text rows may be due to corrupted replica DBs
                $error = $errors[$blobAddress] ?? null;
                return $result[$blobAddress];
            },
            [ 'pcGroup' => self::TEXT_CACHE_GROUP, 'pcTTL' => IExpiringStore::TTL_PROC_LONG ]
        );
 
        if ( $error ) {
            throw new BlobAccessException( $error );
        }
 
        Assert::postcondition( is_string( $blob ), 'Blob must not be null' );
        return $blob;
    }
 
    public function getBlobBatch( $blobAddresses, $queryFlags = 0 ) {
        $errors = null;
        $addressByCacheKey = $this->cache->makeMultiKeys(
            $blobAddresses,
            function ( $blobAddress ) {
                return $this->getCacheKey( $blobAddress );
            }
        );
        $blobsByCacheKey = $this->cache->getMultiWithUnionSetCallback(
            $addressByCacheKey,
            $this->getCacheTTL(),
            function ( array $blobAddresses, array &$ttls, array &$setOpts ) use ( $queryFlags, &$errors ) {
                // Ignore $setOpts; blobs are immutable and negatives are not cached
                list( $result, $errors ) = $this->fetchBlobs( $blobAddresses, $queryFlags );
                return $result;
            },
            [ 'pcGroup' => self::TEXT_CACHE_GROUP, 'pcTTL' => IExpiringStore::TTL_PROC_LONG ]
        );
 
        // Remap back to incoming blob addresses. The return value of the
        // WANObjectCache::getMultiWithUnionSetCallback is keyed on the internal
        // keys from WANObjectCache::makeMultiKeys, so we need to remap them
        // before returning to the client.
        $blobsByAddress = [];
        foreach ( $blobsByCacheKey as $cacheKey => $blob ) {
            $blobsByAddress[ $addressByCacheKey[ $cacheKey ] ] = $blob !== false ? $blob : null;
        }
 
        $result = StatusValue::newGood( $blobsByAddress );
        if ( $errors ) {
            foreach ( $errors as $error ) {
                $result->warning( 'internalerror', $error );
            }
        }
        return $result;
    }
 
    private function fetchBlobs( $blobAddresses, $queryFlags ) {
        $textIdToBlobAddress = [];
        $result = [];
        $errors = [];
        foreach ( $blobAddresses as $blobAddress ) {
            list( $schema, $id ) = self::splitBlobAddress( $blobAddress );
            //TODO: MCR: also support 'ex' schema with ExternalStore URLs, plus flags encoded in the URL!
            if ( $schema === 'tt' ) {
                $textId = intval( $id );
                $textIdToBlobAddress[$textId] = $blobAddress;
            } else {
                $errors[$blobAddress] = "Unknown blob address schema: $schema";
                $result[$blobAddress] = false;
                continue;
            }
 
            if ( !$textId || $id !== (string)$textId ) {
                $errors[$blobAddress] = "Bad blob address: $blobAddress";
                $result[$blobAddress] = false;
            }
        }
 
        $textIds = array_keys( $textIdToBlobAddress );
        if ( !$textIds ) {
            return [ $result, $errors ];
        }
        // Callers doing updates will pass in READ_LATEST as usual. Since the text/blob tables
        // do not normally get rows changed around, set READ_LATEST_IMMUTABLE in those cases.
        $queryFlags |= DBAccessObjectUtils::hasFlags( $queryFlags, self::READ_LATEST )
            ? self::READ_LATEST_IMMUTABLE
            : 0;
        list( $index, $options, $fallbackIndex, $fallbackOptions ) =
            DBAccessObjectUtils::getDBOptions( $queryFlags );
        // Text data is immutable; check replica DBs first.
        $dbConnection = $this->getDBConnection( $index );
        $rows = $dbConnection->select(
            'text',
            [ 'old_id', 'old_text', 'old_flags' ],
            [ 'old_id' => $textIds ],
            __METHOD__,
            $options
        );
 
        // Fallback to DB_MASTER in some cases if not all the rows were found, using the appropriate
        // options, such as FOR UPDATE to avoid missing rows due to REPEATABLE-READ.
        if ( $dbConnection->numRows( $rows ) !== count( $textIds ) && $fallbackIndex !== null ) {
            $fetchedTextIds = [];
            foreach ( $rows as $row ) {
                $fetchedTextIds[] = $row->old_id;
            }
            $missingTextIds = array_diff( $textIds, $fetchedTextIds );
            $dbConnection = $this->getDBConnection( $fallbackIndex );
            $rowsFromFallback = $dbConnection->select(
                'text',
                [ 'old_id', 'old_text', 'old_flags' ],
                [ 'old_id' => $missingTextIds ],
                __METHOD__,
                $fallbackOptions
            );
            $appendIterator = new AppendIterator();
            $appendIterator->append( $rows );
            $appendIterator->append( $rowsFromFallback );
            $rows = $appendIterator;
        }
 
        foreach ( $rows as $row ) {
            $blobAddress = $textIdToBlobAddress[$row->old_id];
            $blob = $this->expandBlob( $row->old_text, $row->old_flags, $blobAddress );
            if ( $blob === false ) {
                $errors[$blobAddress] = "Bad data in text row {$row->old_id}.";
            }
            $result[$blobAddress] = $blob;
        }
 
        // If we're still missing some of the rows, set errors for missing blobs.
        if ( count( $result ) !== count( $blobAddresses ) ) {
            foreach ( $blobAddresses as $blobAddress ) {
                if ( !isset( $result[$blobAddress ] ) ) {
                    $errors[$blobAddress] = "Unable to fetch blob at $blobAddress";
                    $result[$blobAddress] = false;
                }
            }
        }
        return [ $result, $errors ];
    }
 
    private function getCacheKey( $blobAddress ) {
        return $this->cache->makeGlobalKey(
            'SqlBlobStore-blob',
            $this->dbLoadBalancer->resolveDomainID( $this->dbDomain ),
            $blobAddress
        );
    }
 
    public function expandBlob( $raw, $flags, $cacheKey = null ) {
        if ( is_string( $flags ) ) {
            $flags = explode( ',', $flags );
        }
 
        // Use external methods for external objects, text in table is URL-only then
        if ( in_array( 'external', $flags ) ) {
            $url = $raw;
            $parts = explode( '://', $url, 2 );
            if ( count( $parts ) == 1 || $parts[1] == '' ) {
                return false;
            }
 
            if ( $cacheKey ) {
                // The cached value should be decompressed, so handle that and return here.
                return $this->cache->getWithSetCallback(
                    $this->getCacheKey( $cacheKey ),
                    $this->getCacheTTL(),
                    function () use ( $url, $flags ) {
                        // Ignore $setOpts; blobs are immutable and negatives are not cached
                        $blob = $this->extStoreAccess
                            ->fetchFromURL( $url, [ 'domain' => $this->dbDomain ] );
 
                        return $blob === false ? false : $this->decompressData( $blob, $flags );
                    },
                    [ 'pcGroup' => self::TEXT_CACHE_GROUP, 'pcTTL' => WANObjectCache::TTL_PROC_LONG ]
                );
            } else {
                $blob = $this->extStoreAccess->fetchFromURL( $url, [ 'domain' => $this->dbDomain ] );
                return $blob === false ? false : $this->decompressData( $blob, $flags );
            }
        } else {
            return $this->decompressData( $raw, $flags );
        }
    }
 
    public function compressData( &$blob ) {
        $blobFlags = [];
 
        // Revisions not marked as UTF-8 will have legacy decoding applied by decompressData().
        // XXX: if $this->legacyEncoding is not set, we could skip this. That would however be
        // risky, since $this->legacyEncoding being set in the future would lead to data corruption.
        $blobFlags[] = 'utf-8';
 
        if ( $this->compressBlobs ) {
            if ( function_exists( 'gzdeflate' ) ) {
                $deflated = gzdeflate( $blob );
 
                if ( $deflated === false ) {
                    wfLogWarning( __METHOD__ . ': gzdeflate() failed' );
                } else {
                    $blob = $deflated;
                    $blobFlags[] = 'gzip';
                }
            } else {
                wfDebug( __METHOD__ . " -- no zlib support, not compressing\n" );
            }
        }
        return implode( ',', $blobFlags );
    }
 
    public function decompressData( $blob, array $blobFlags ) {
        // Revision::decompressRevisionText accepted false here, so defend against that
        Assert::parameterType( 'string', $blob, '$blob' );
 
        if ( in_array( 'error', $blobFlags ) ) {
            // Error row, return false
            return false;
        }
 
        if ( in_array( 'gzip', $blobFlags ) ) {
            # Deal with optional compression of archived pages.
            # This can be done periodically via maintenance/compressOld.php, and
            # as pages are saved if $wgCompressRevisions is set.
            $blob = gzinflate( $blob );
 
            if ( $blob === false ) {
                wfWarn( __METHOD__ . ': gzinflate() failed' );
                return false;
            }
        }
 
        if ( in_array( 'object', $blobFlags ) ) {
            # Generic compressed storage
            $obj = unserialize( $blob );
            if ( !is_object( $obj ) ) {
                // Invalid object
                return false;
            }
            $blob = $obj->getText();
        }
 
        // Needed to support old revisions left over from from the 1.4 / 1.5 migration.
        if ( $blob !== false && $this->legacyEncoding
            && !in_array( 'utf-8', $blobFlags ) && !in_array( 'utf8', $blobFlags )
        ) {
            # Old revisions kept around in a legacy encoding?
            # Upconvert on demand.
            # ("utf8" checked for compatibility with some broken
            #  conversion scripts 2008-12-30)
            # Even with //IGNORE iconv can whine about illegal characters in
            # *input* string. We just ignore those too.
            # REF: https://bugs.php.net/bug.php?id=37166
            # REF: https://phabricator.wikimedia.org/T18885
            AtEase::suppressWarnings();
            $blob = iconv( $this->legacyEncoding, 'UTF-8//IGNORE', $blob );
            AtEase::restoreWarnings();
        }
 
        return $blob;
    }
 
    private function getCacheTTL() {
        if ( $this->cache->getQoS( WANObjectCache::ATTR_EMULATION )
                <= WANObjectCache::QOS_EMULATION_SQL
        ) {
            // Do not cache RDBMs blobs in...the RDBMs store
            $ttl = WANObjectCache::TTL_UNCACHEABLE;
        } else {
            $ttl = $this->cacheExpiry ?: WANObjectCache::TTL_UNCACHEABLE;
        }
 
        return $ttl;
    }
 
    public function getTextIdFromAddress( $address ) {
        list( $schema, $id, ) = self::splitBlobAddress( $address );
 
        if ( $schema !== 'tt' ) {
            return null;
        }
 
        $textId = intval( $id );
 
        if ( !$textId || $id !== (string)$textId ) {
            throw new InvalidArgumentException( "Malformed text_id: $id" );
        }
 
        return $textId;
    }
 
    public static function makeAddressFromTextId( $id ) {
        return 'tt:' . $id;
    }
 
    public static function splitBlobAddress( $address ) {
        if ( !preg_match( '/^(\w+):(\w+)(\?(.*))?$/', $address, $m ) ) {
            throw new InvalidArgumentException( "Bad blob address: $address" );
        }
 
        $schema = strtolower( $m[1] );
        $id = $m[2];
        $parameters = isset( $m[4] ) ? wfCgiToArray( $m[4] ) : [];
 
        return [ $schema, $id, $parameters ];
    }
 
    public function isReadOnly() {
        if ( $this->useExternalStore && $this->extStoreAccess->isReadOnly() ) {
            return true;
        }
 
        return ( $this->getDBLoadBalancer()->getReadOnlyReason() !== false );
    }
}