BacklinkCache.php

Go to the documentation of this file.

<?php
use Wikimedia\Rdbms\FakeResultWrapper;
use Wikimedia\Rdbms\IDatabase;
use Wikimedia\Rdbms\IResultWrapper;
use MediaWiki\MediaWikiServices;
 
class BacklinkCache {
    protected static $instance;
 
    protected $partitionCache = [];
 
    protected $fullResultCache = [];
 
    protected $wanCache;
 
    protected $db;
 
    protected $title;
 
    const CACHE_EXPIRY = 3600;
 
    public function __construct( Title $title ) {
        $this->title = $title;
        $this->wanCache = MediaWikiServices::getInstance()->getMainWANObjectCache();
    }
 
    public static function get( Title $title ) {
        if ( !self::$instance || !self::$instance->title->equals( $title ) ) {
            self::$instance = new self( $title );
        }
        return self::$instance;
    }
 
    function __sleep() {
        return [ 'partitionCache', 'fullResultCache', 'title' ];
    }
 
    public function clear() {
        $this->partitionCache = [];
        $this->fullResultCache = [];
        $this->wanCache->touchCheckKey( $this->makeCheckKey() );
        $this->db = null;
    }
 
    public function setDB( $db ) {
        $this->db = $db;
    }
 
    protected function getDB() {
        if ( $this->db === null ) {
            $this->db = wfGetDB( DB_REPLICA );
        }
 
        return $this->db;
    }
 
    public function getLinks( $table, $startId = false, $endId = false, $max = INF ) {
        return TitleArray::newFromResult( $this->queryLinks( $table, $startId, $endId, $max ) );
    }
 
    protected function queryLinks( $table, $startId, $endId, $max, $select = 'all' ) {
        $fromField = $this->getPrefix( $table ) . '_from';
 
        if ( !$startId && !$endId && is_infinite( $max )
            && isset( $this->fullResultCache[$table] )
        ) {
            wfDebug( __METHOD__ . ": got results from cache\n" );
            $res = $this->fullResultCache[$table];
        } else {
            wfDebug( __METHOD__ . ": got results from DB\n" );
            $conds = $this->getConditions( $table );
            // Use the from field in the condition rather than the joined page_id,
            // because databases are stupid and don't necessarily propagate indexes.
            if ( $startId ) {
                $conds[] = "$fromField >= " . intval( $startId );
            }
            if ( $endId ) {
                $conds[] = "$fromField <= " . intval( $endId );
            }
            $options = [ 'ORDER BY' => $fromField ];
            if ( is_finite( $max ) && $max > 0 ) {
                $options['LIMIT'] = $max;
            }
 
            if ( $select === 'ids' ) {
                // Just select from the backlink table and ignore the page JOIN
                $res = $this->getDB()->select(
                    $table,
                    [ $this->getPrefix( $table ) . '_from AS page_id' ],
                    array_filter( $conds, function ( $clause ) { // kind of janky
                        return !preg_match( '/(\b|=)page_id(\b|=)/', $clause );
                    } ),
                    __METHOD__,
                    $options
                );
            } else {
                // Select from the backlink table and JOIN with page title information
                $res = $this->getDB()->select(
                    [ $table, 'page' ],
                    [ 'page_namespace', 'page_title', 'page_id' ],
                    $conds,
                    __METHOD__,
                    array_merge( [ 'STRAIGHT_JOIN' ], $options )
                );
            }
 
            if ( $select === 'all' && !$startId && !$endId && $res->numRows() < $max ) {
                // The full results fit within the limit, so cache them
                $this->fullResultCache[$table] = $res;
            } else {
                wfDebug( __METHOD__ . ": results from DB were uncacheable\n" );
            }
        }
 
        return $res;
    }
 
    protected function getPrefix( $table ) {
        static $prefixes = [
            'pagelinks' => 'pl',
            'imagelinks' => 'il',
            'categorylinks' => 'cl',
            'templatelinks' => 'tl',
            'redirect' => 'rd',
        ];
 
        if ( isset( $prefixes[$table] ) ) {
            return $prefixes[$table];
        } else {
            $prefix = null;
            Hooks::run( 'BacklinkCacheGetPrefix', [ $table, &$prefix ] );
            if ( $prefix ) {
                return $prefix;
            } else {
                throw new MWException( "Invalid table \"$table\" in " . __CLASS__ );
            }
        }
    }
 
    protected function getConditions( $table ) {
        $prefix = $this->getPrefix( $table );
 
        switch ( $table ) {
            case 'pagelinks':
            case 'templatelinks':
                $conds = [
                    "{$prefix}_namespace" => $this->title->getNamespace(),
                    "{$prefix}_title" => $this->title->getDBkey(),
                    "page_id={$prefix}_from"
                ];
                break;
            case 'redirect':
                $conds = [
                    "{$prefix}_namespace" => $this->title->getNamespace(),
                    "{$prefix}_title" => $this->title->getDBkey(),
                    $this->getDB()->makeList( [
                        "{$prefix}_interwiki" => '',
                        "{$prefix}_interwiki IS NULL",
                    ], LIST_OR ),
                    "page_id={$prefix}_from"
                ];
                break;
            case 'imagelinks':
            case 'categorylinks':
                $conds = [
                    "{$prefix}_to" => $this->title->getDBkey(),
                    "page_id={$prefix}_from"
                ];
                break;
            default:
                $conds = null;
                Hooks::run( 'BacklinkCacheGetConditions', [ $table, $this->title, &$conds ] );
                if ( !$conds ) {
                    throw new MWException( "Invalid table \"$table\" in " . __CLASS__ );
                }
        }
 
        return $conds;
    }
 
    public function hasLinks( $table ) {
        return ( $this->getNumLinks( $table, 1 ) > 0 );
    }
 
    public function getNumLinks( $table, $max = INF ) {
        global $wgUpdateRowsPerJob;
 
        // 1) try partition cache ...
        if ( isset( $this->partitionCache[$table] ) ) {
            $entry = reset( $this->partitionCache[$table] );
 
            return min( $max, $entry['numRows'] );
        }
 
        // 2) ... then try full result cache ...
        if ( isset( $this->fullResultCache[$table] ) ) {
            return min( $max, $this->fullResultCache[$table]->numRows() );
        }
 
        $memcKey = $this->wanCache->makeKey(
            'numbacklinks',
            md5( $this->title->getPrefixedDBkey() ),
            $table
        );
 
        // 3) ... fallback to memcached ...
        $curTTL = INF;
        $count = $this->wanCache->get(
            $memcKey,
            $curTTL,
            [
                $this->makeCheckKey()
            ]
        );
        if ( $count && ( $curTTL > 0 ) ) {
            return min( $max, $count );
        }
 
        // 4) fetch from the database ...
        if ( is_infinite( $max ) ) { // no limit at all
            // Use partition() since it will batch the query and skip the JOIN.
            // Use $wgUpdateRowsPerJob just to encourage cache reuse for jobs.
            $this->partition( $table, $wgUpdateRowsPerJob ); // updates $this->partitionCache
            return $this->partitionCache[$table][$wgUpdateRowsPerJob]['numRows'];
        } else { // probably some sane limit
            // Fetch the full title info, since the caller will likely need it next
            $count = $this->getLinks( $table, false, false, $max )->count();
            if ( $count < $max ) { // full count
                $this->wanCache->set( $memcKey, $count, self::CACHE_EXPIRY );
            }
        }
 
        return min( $max, $count );
    }
 
    public function partition( $table, $batchSize ) {
        // 1) try partition cache ...
        if ( isset( $this->partitionCache[$table][$batchSize] ) ) {
            wfDebug( __METHOD__ . ": got from partition cache\n" );
 
            return $this->partitionCache[$table][$batchSize]['batches'];
        }
 
        $this->partitionCache[$table][$batchSize] = false;
        $cacheEntry =& $this->partitionCache[$table][$batchSize];
 
        // 2) ... then try full result cache ...
        if ( isset( $this->fullResultCache[$table] ) ) {
            $cacheEntry = $this->partitionResult( $this->fullResultCache[$table], $batchSize );
            wfDebug( __METHOD__ . ": got from full result cache\n" );
 
            return $cacheEntry['batches'];
        }
 
        $memcKey = $this->wanCache->makeKey(
            'backlinks',
            md5( $this->title->getPrefixedDBkey() ),
            $table,
            $batchSize
        );
 
        // 3) ... fallback to memcached ...
        $curTTL = 0;
        $memcValue = $this->wanCache->get(
            $memcKey,
            $curTTL,
            [
                $this->makeCheckKey()
            ]
        );
        if ( is_array( $memcValue ) && ( $curTTL > 0 ) ) {
            $cacheEntry = $memcValue;
            wfDebug( __METHOD__ . ": got from memcached $memcKey\n" );
 
            return $cacheEntry['batches'];
        }
 
        // 4) ... finally fetch from the slow database :(
        $cacheEntry = [ 'numRows' => 0, 'batches' => [] ]; // final result
        // Do the selects in batches to avoid client-side OOMs (T45452).
        // Use a LIMIT that plays well with $batchSize to keep equal sized partitions.
        $selectSize = max( $batchSize, 200000 - ( 200000 % $batchSize ) );
        $start = false;
        do {
            $res = $this->queryLinks( $table, $start, false, $selectSize, 'ids' );
            $partitions = $this->partitionResult( $res, $batchSize, false );
            // Merge the link count and range partitions for this chunk
            $cacheEntry['numRows'] += $partitions['numRows'];
            $cacheEntry['batches'] = array_merge( $cacheEntry['batches'], $partitions['batches'] );
            if ( count( $partitions['batches'] ) ) {
                list( , $lEnd ) = end( $partitions['batches'] );
                $start = $lEnd + 1; // pick up after this inclusive range
            }
        } while ( $partitions['numRows'] >= $selectSize );
        // Make sure the first range has start=false and the last one has end=false
        if ( count( $cacheEntry['batches'] ) ) {
            $cacheEntry['batches'][0][0] = false;
            $cacheEntry['batches'][count( $cacheEntry['batches'] ) - 1][1] = false;
        }
 
        // Save partitions to memcached
        $this->wanCache->set( $memcKey, $cacheEntry, self::CACHE_EXPIRY );
 
        // Save backlink count to memcached
        $memcKey = $this->wanCache->makeKey(
            'numbacklinks',
            md5( $this->title->getPrefixedDBkey() ),
            $table
        );
        $this->wanCache->set( $memcKey, $cacheEntry['numRows'], self::CACHE_EXPIRY );
 
        wfDebug( __METHOD__ . ": got from database\n" );
 
        return $cacheEntry['batches'];
    }
 
    protected function partitionResult( $res, $batchSize, $isComplete = true ) {
        $batches = [];
        $numRows = $res->numRows();
        $numBatches = ceil( $numRows / $batchSize );
 
        for ( $i = 0; $i < $numBatches; $i++ ) {
            if ( $i == 0 && $isComplete ) {
                $start = false;
            } else {
                $rowNum = $i * $batchSize;
                $res->seek( $rowNum );
                $row = $res->fetchObject();
                $start = (int)$row->page_id;
            }
 
            if ( $i == ( $numBatches - 1 ) && $isComplete ) {
                $end = false;
            } else {
                $rowNum = min( $numRows - 1, ( $i + 1 ) * $batchSize - 1 );
                $res->seek( $rowNum );
                $row = $res->fetchObject();
                $end = (int)$row->page_id;
            }
 
            # Sanity check order
            if ( $start && $end && $start > $end ) {
                throw new MWException( __METHOD__ . ': Internal error: query result out of order' );
            }
 
            $batches[] = [ $start, $end ];
        }
 
        return [ 'numRows' => $numRows, 'batches' => $batches ];
    }
 
    public function getCascadeProtectedLinks() {
        $dbr = $this->getDB();
 
        // @todo: use UNION without breaking tests that use temp tables
        $resSets = [];
        $resSets[] = $dbr->select(
            [ 'templatelinks', 'page_restrictions', 'page' ],
            [ 'page_namespace', 'page_title', 'page_id' ],
            [
                'tl_namespace' => $this->title->getNamespace(),
                'tl_title' => $this->title->getDBkey(),
                'tl_from = pr_page',
                'pr_cascade' => 1,
                'page_id = tl_from'
            ],
            __METHOD__,
            [ 'DISTINCT' ]
        );
        if ( $this->title->getNamespace() == NS_FILE ) {
            $resSets[] = $dbr->select(
                [ 'imagelinks', 'page_restrictions', 'page' ],
                [ 'page_namespace', 'page_title', 'page_id' ],
                [
                    'il_to' => $this->title->getDBkey(),
                    'il_from = pr_page',
                    'pr_cascade' => 1,
                    'page_id = il_from'
                ],
                __METHOD__,
                [ 'DISTINCT' ]
            );
        }
 
        // Combine and de-duplicate the results
        $mergedRes = [];
        foreach ( $resSets as $res ) {
            foreach ( $res as $row ) {
                $mergedRes[$row->page_id] = $row;
            }
        }
 
        return TitleArray::newFromResult(
            new FakeResultWrapper( array_values( $mergedRes ) ) );
    }
 
    private function makeCheckKey() {
        return $this->wanCache->makeKey(
            'backlinks',
            md5( $this->title->getPrefixedDBkey() )
        );
    }
}