HistoryBlob.php

Go to the documentation of this file.

<?php
interface HistoryBlob {
    function addItem( $text );

    function getItem( $key );

    function setText( $text );

    function getText();
}

class ConcatenatedGzipHistoryBlob implements HistoryBlob {
    public $mVersion = 0, $mCompressed = false, $mItems = [], $mDefaultHash = '';
    public $mSize = 0;
    public $mMaxSize = 10000000;
    public $mMaxCount = 100;

    public function __construct() {
        if ( !function_exists( 'gzdeflate' ) ) {
            throw new MWException( "Need zlib support to read or write this "
                . "kind of history object (ConcatenatedGzipHistoryBlob)\n" );
        }
    }

    public function addItem( $text ) {
        $this->uncompress();
        $hash = md5( $text );
        if ( !isset( $this->mItems[$hash] ) ) {
            $this->mItems[$hash] = $text;
            $this->mSize += strlen( $text );
        }
        return $hash;
    }

    public function getItem( $hash ) {
        $this->uncompress();
        if ( array_key_exists( $hash, $this->mItems ) ) {
            return $this->mItems[$hash];
        } else {
            return false;
        }
    }

    public function setText( $text ) {
        $this->uncompress();
        $this->mDefaultHash = $this->addItem( $text );
    }

    public function getText() {
        $this->uncompress();
        return $this->getItem( $this->mDefaultHash );
    }

    public function removeItem( $hash ) {
        $this->mSize -= strlen( $this->mItems[$hash] );
        unset( $this->mItems[$hash] );
    }

    public function compress() {
        if ( !$this->mCompressed ) {
            $this->mItems = gzdeflate( serialize( $this->mItems ) );
            $this->mCompressed = true;
        }
    }

    public function uncompress() {
        if ( $this->mCompressed ) {
            $this->mItems = unserialize( gzinflate( $this->mItems ) );
            $this->mCompressed = false;
        }
    }

    function __sleep() {
        $this->compress();
        return [ 'mVersion', 'mCompressed', 'mItems', 'mDefaultHash' ];
    }

    function __wakeup() {
        $this->uncompress();
    }

    public function isHappy() {
        return $this->mSize < $this->mMaxSize
            && count( $this->mItems ) < $this->mMaxCount;
    }
}

class HistoryBlobStub {
    protected static $blobCache = [];

    public $mOldId;

    public $mHash;

    public $mRef;

    function __construct( $hash = '', $oldid = 0 ) {
        $this->mHash = $hash;
    }

    function setLocation( $id ) {
        $this->mOldId = $id;
    }

    function setReferrer( $id ) {
        $this->mRef = $id;
    }

    function getReferrer() {
        return $this->mRef;
    }

    function getText() {
        if ( isset( self::$blobCache[$this->mOldId] ) ) {
            $obj = self::$blobCache[$this->mOldId];
        } else {
            $dbr = wfGetDB( DB_REPLICA );
            $row = $dbr->selectRow(
                'text',
                [ 'old_flags', 'old_text' ],
                [ 'old_id' => $this->mOldId ]
            );

            if ( !$row ) {
                return false;
            }

            $flags = explode( ',', $row->old_flags );
            if ( in_array( 'external', $flags ) ) {
                $url = $row->old_text;
                $parts = explode( '://', $url, 2 );
                if ( !isset( $parts[1] ) || $parts[1] == '' ) {
                    return false;
                }
                $row->old_text = ExternalStore::fetchFromURL( $url );

            }

            if ( !in_array( 'object', $flags ) ) {
                return false;
            }

            if ( in_array( 'gzip', $flags ) ) {
                // This shouldn't happen, but a bug in the compress script
                // may at times gzip-compress a HistoryBlob object row.
                $obj = unserialize( gzinflate( $row->old_text ) );
            } else {
                $obj = unserialize( $row->old_text );
            }

            if ( !is_object( $obj ) ) {
                // Correct for old double-serialization bug.
                $obj = unserialize( $obj );
            }

            // Save this item for reference; if pulling many
            // items in a row we'll likely use it again.
            $obj->uncompress();
            self::$blobCache = [ $this->mOldId => $obj ];
        }

        return $obj->getItem( $this->mHash );
    }

    function getHash() {
        return $this->mHash;
    }
}

class HistoryBlobCurStub {
    public $mCurId;

    function __construct( $curid = 0 ) {
        $this->mCurId = $curid;
    }

    function setLocation( $id ) {
        $this->mCurId = $id;
    }

    function getText() {
        $dbr = wfGetDB( DB_REPLICA );
        $row = $dbr->selectRow( 'cur', [ 'cur_text' ], [ 'cur_id' => $this->mCurId ] );
        if ( !$row ) {
            return false;
        }
        return $row->cur_text;
    }
}

class DiffHistoryBlob implements HistoryBlob {
    public $mItems = [];

    public $mSize = 0;

    public $mDiffs;

    public $mDiffMap;

    public $mDefaultKey;

    public $mCompressed;

    public $mFrozen = false;

    public $mMaxSize = 10000000;

    public $mMaxCount = 100;

    const XDL_BDOP_INS = 1;
    const XDL_BDOP_CPY = 2;
    const XDL_BDOP_INSB = 3;

    function __construct() {
        if ( !function_exists( 'gzdeflate' ) ) {
            throw new MWException( "Need zlib support to read or write DiffHistoryBlob\n" );
        }
    }

    function addItem( $text ) {
        if ( $this->mFrozen ) {
            throw new MWException( __METHOD__ . ": Cannot add more items after sleep/wakeup" );
        }

        $this->mItems[] = $text;
        $this->mSize += strlen( $text );
        $this->mDiffs = null; // later
        return count( $this->mItems ) - 1;
    }

    function getItem( $key ) {
        return $this->mItems[$key];
    }

    function setText( $text ) {
        $this->mDefaultKey = $this->addItem( $text );
    }

    function getText() {
        return $this->getItem( $this->mDefaultKey );
    }

    function compress() {
        if ( !function_exists( 'xdiff_string_rabdiff' ) ) {
            throw new MWException( "Need xdiff 1.5+ support to write DiffHistoryBlob\n" );
        }
        if ( isset( $this->mDiffs ) ) {
            // Already compressed
            return;
        }
        if ( !count( $this->mItems ) ) {
            // Empty
            return;
        }

        // Create two diff sequences: one for main text and one for small text
        $sequences = [
            'small' => [
                'tail' => '',
                'diffs' => [],
                'map' => [],
            ],
            'main' => [
                'tail' => '',
                'diffs' => [],
                'map' => [],
            ],
        ];
        $smallFactor = 0.5;

        $mItemsCount = count( $this->mItems );
        for ( $i = 0; $i < $mItemsCount; $i++ ) {
            $text = $this->mItems[$i];
            if ( $i == 0 ) {
                $seqName = 'main';
            } else {
                $mainTail = $sequences['main']['tail'];
                if ( strlen( $text ) < strlen( $mainTail ) * $smallFactor ) {
                    $seqName = 'small';
                } else {
                    $seqName = 'main';
                }
            }
            $seq =& $sequences[$seqName];
            $tail = $seq['tail'];
            $diff = $this->diff( $tail, $text );
            $seq['diffs'][] = $diff;
            $seq['map'][] = $i;
            $seq['tail'] = $text;
        }
        unset( $seq ); // unlink dangerous alias

        // Knit the sequences together
        $tail = '';
        $this->mDiffs = [];
        $this->mDiffMap = [];
        foreach ( $sequences as $seq ) {
            if ( !count( $seq['diffs'] ) ) {
                continue;
            }
            if ( $tail === '' ) {
                $this->mDiffs[] = $seq['diffs'][0];
            } else {
                $head = $this->patch( '', $seq['diffs'][0] );
                $this->mDiffs[] = $this->diff( $tail, $head );
            }
            $this->mDiffMap[] = $seq['map'][0];
            $diffsCount = count( $seq['diffs'] );
            for ( $i = 1; $i < $diffsCount; $i++ ) {
                $this->mDiffs[] = $seq['diffs'][$i];
                $this->mDiffMap[] = $seq['map'][$i];
            }
            $tail = $seq['tail'];
        }
    }

    function diff( $t1, $t2 ) {
        # Need to do a null concatenation with warnings off, due to bugs in the current version of xdiff
        # "String is not zero-terminated"
        MediaWiki\suppressWarnings();
        $diff = xdiff_string_rabdiff( $t1, $t2 ) . '';
        MediaWiki\restoreWarnings();
        return $diff;
    }

    function patch( $base, $diff ) {
        if ( function_exists( 'xdiff_string_bpatch' ) ) {
            MediaWiki\suppressWarnings();
            $text = xdiff_string_bpatch( $base, $diff ) . '';
            MediaWiki\restoreWarnings();
            return $text;
        }

        # Pure PHP implementation

        $header = unpack( 'Vofp/Vcsize', substr( $diff, 0, 8 ) );

        # Check the checksum if hash extension is available
        $ofp = $this->xdiffAdler32( $base );
        if ( $ofp !== false && $ofp !== substr( $diff, 0, 4 ) ) {
            wfDebug( __METHOD__ . ": incorrect base checksum\n" );
            return false;
        }
        if ( $header['csize'] != strlen( $base ) ) {
            wfDebug( __METHOD__ . ": incorrect base length\n" );
            return false;
        }

        $p = 8;
        $out = '';
        while ( $p < strlen( $diff ) ) {
            $x = unpack( 'Cop', substr( $diff, $p, 1 ) );
            $op = $x['op'];
            ++$p;
            switch ( $op ) {
            case self::XDL_BDOP_INS:
                $x = unpack( 'Csize', substr( $diff, $p, 1 ) );
                $p++;
                $out .= substr( $diff, $p, $x['size'] );
                $p += $x['size'];
                break;
            case self::XDL_BDOP_INSB:
                $x = unpack( 'Vcsize', substr( $diff, $p, 4 ) );
                $p += 4;
                $out .= substr( $diff, $p, $x['csize'] );
                $p += $x['csize'];
                break;
            case self::XDL_BDOP_CPY:
                $x = unpack( 'Voff/Vcsize', substr( $diff, $p, 8 ) );
                $p += 8;
                $out .= substr( $base, $x['off'], $x['csize'] );
                break;
            default:
                wfDebug( __METHOD__ . ": invalid op\n" );
                return false;
            }
        }
        return $out;
    }

    function xdiffAdler32( $s ) {
        if ( !function_exists( 'hash' ) ) {
            return false;
        }

        static $init;
        if ( $init === null ) {
            $init = str_repeat( "\xf0", 205 ) . "\xee" . str_repeat( "\xf0", 67 ) . "\x02";
        }

        // The real Adler-32 checksum of $init is zero, so it initialises the
        // state to zero, as it is at the start of LibXDiff's checksum
        // algorithm. Appending the subject string then simulates LibXDiff.
        return strrev( hash( 'adler32', $init . $s, true ) );
    }

    function uncompress() {
        if ( !$this->mDiffs ) {
            return;
        }
        $tail = '';
        $mDiffsCount = count( $this->mDiffs );
        for ( $diffKey = 0; $diffKey < $mDiffsCount; $diffKey++ ) {
            $textKey = $this->mDiffMap[$diffKey];
            $text = $this->patch( $tail, $this->mDiffs[$diffKey] );
            $this->mItems[$textKey] = $text;
            $tail = $text;
        }
    }

    function __sleep() {
        $this->compress();
        if ( !count( $this->mItems ) ) {
            // Empty object
            $info = false;
        } else {
            // Take forward differences to improve the compression ratio for sequences
            $map = '';
            $prev = 0;
            foreach ( $this->mDiffMap as $i ) {
                if ( $map !== '' ) {
                    $map .= ',';
                }
                $map .= $i - $prev;
                $prev = $i;
            }
            $info = [
                'diffs' => $this->mDiffs,
                'map' => $map
            ];
        }
        if ( isset( $this->mDefaultKey ) ) {
            $info['default'] = $this->mDefaultKey;
        }
        $this->mCompressed = gzdeflate( serialize( $info ) );
        return [ 'mCompressed' ];
    }

    function __wakeup() {
        // addItem() doesn't work if mItems is partially filled from mDiffs
        $this->mFrozen = true;
        $info = unserialize( gzinflate( $this->mCompressed ) );
        unset( $this->mCompressed );

        if ( !$info ) {
            // Empty object
            return;
        }

        if ( isset( $info['default'] ) ) {
            $this->mDefaultKey = $info['default'];
        }
        $this->mDiffs = $info['diffs'];
        if ( isset( $info['base'] ) ) {
            // Old format
            $this->mDiffMap = range( 0, count( $this->mDiffs ) - 1 );
            array_unshift( $this->mDiffs,
                pack( 'VVCV', 0, 0, self::XDL_BDOP_INSB, strlen( $info['base'] ) ) .
                $info['base'] );
        } else {
            // New format
            $map = explode( ',', $info['map'] );
            $cur = 0;
            $this->mDiffMap = [];
            foreach ( $map as $i ) {
                $cur += $i;
                $this->mDiffMap[] = $cur;
            }
        }
        $this->uncompress();
    }

    function isHappy() {
        return $this->mSize < $this->mMaxSize
            && count( $this->mItems ) < $this->mMaxCount;
    }

}