BaseBlacklist.php

Go to the documentation of this file.

<?php
 
use MediaWiki\MediaWikiServices;
use MediaWiki\Storage\SlotRecord;
 
abstract class BaseBlacklist {
    public $files = [];
 
    protected $regexes = false;
 
    public $warningChance = 100;
 
    public $warningTime = 600;
 
    public $expiryTime = 900;
 
    private static $blacklistTypes = [
        'spam' => 'SpamBlacklist',
        'email' => 'EmailBlacklist',
    ];
 
    private static $instances = [];
 
    public function __construct( $settings = [] ) {
        foreach ( $settings as $name => $value ) {
            $this->$name = $value;
        }
    }
 
    abstract public function filter( array $links, Title $title, $preventLog = false );
 
    public static function addBlacklistType( $type, $class ) {
        self::$blacklistTypes[$type] = $class;
    }
 
    public static function getBlacklistTypes() {
        return self::$blacklistTypes;
    }
 
    public static function getSpamBlacklist() {
        return self::getInstance( 'spam' );
    }
 
    public static function getEmailBlacklist() {
        return self::getInstance( 'email' );
    }
 
    public static function getInstance( $type ) {
        if ( !isset( self::$blacklistTypes[$type] ) ) {
            throw new Exception( "Invalid blacklist type '$type' passed to " . __METHOD__ );
        }
 
        if ( !isset( self::$instances[$type] ) ) {
            global $wgBlacklistSettings;
 
            // Prevent notices
            if ( !isset( $wgBlacklistSettings[$type] ) ) {
                $wgBlacklistSettings[$type] = [];
            }
 
            $class = self::$blacklistTypes[$type];
            self::$instances[$type] = new $class( $wgBlacklistSettings[$type] );
        }
 
        return self::$instances[$type];
    }
 
    abstract protected function getBlacklistType();
 
    public static function isLocalSource( Title $title ) {
        global $wgDBname, $wgBlacklistSettings;
 
        if ( $title->inNamespace( NS_MEDIAWIKI ) ) {
            $sources = [];
            foreach ( self::$blacklistTypes as $type => $class ) {
                $type = ucfirst( $type );
                $sources += [
                    "$type-blacklist",
                    "$type-whitelist"
                ];
            }
 
            if ( in_array( $title->getDBkey(), $sources ) ) {
                return true;
            }
        }
 
        $thisHttp = wfExpandUrl( $title->getFullUrl( 'action=raw' ), PROTO_HTTP );
        $thisHttpRegex = '/^' . preg_quote( $thisHttp, '/' ) . '(?:&.*)?$/';
 
        $files = [];
        foreach ( self::$blacklistTypes as $type => $class ) {
            if ( isset( $wgBlacklistSettings[$type]['files'] ) ) {
                $files += $wgBlacklistSettings[$type]['files'];
            }
        }
 
        // @phan-suppress-next-line PhanTypeMismatchForeach += makes Phan think $files is a number
        foreach ( $files as $fileName ) {
            $matches = [];
            if ( preg_match( '/^DB: (\w*) (.*)$/', $fileName, $matches ) ) {
                if ( $wgDBname === $matches[1] ) {
                    if ( $matches[2] === $title->getPrefixedDbKey() ) {
                        // Local DB fetch of this page...
                        return true;
                    }
                }
            } elseif ( preg_match( $thisHttpRegex, $fileName ) ) {
                // Raw view of this page
                return true;
            }
        }
 
        return false;
    }
 
    public static function getTypeFromTitle( Title $title ) {
        $contLang = MediaWikiServices::getInstance()->getContentLanguage();
 
        $types = array_map( [ $contLang, 'ucfirst' ], array_keys( self::$blacklistTypes ) );
        $regex = '/(' . implode( '|', $types ) . ')-(?:blacklist|whitelist)/';
 
        if ( preg_match( $regex, $title->getDBkey(), $m ) ) {
            return strtolower( $m[1] );
        }
 
        return false;
    }
 
    public function getBlacklists() {
        if ( $this->regexes === false ) {
            $this->regexes = array_merge(
                $this->getLocalBlacklists(),
                $this->getSharedBlacklists()
            );
        }
        return $this->regexes;
    }
 
    public function getLocalBlacklists() {
        $that = $this;
        $type = $this->getBlacklistType();
        $cache = MediaWikiServices::getInstance()->getMainWANObjectCache();
 
        return $cache->getWithSetCallback(
            $cache->makeKey( 'spamblacklist', $type, 'blacklist-regex' ),
            $this->expiryTime,
            function () use ( $that, $type ) {
                return SpamRegexBatch::regexesFromMessage( "{$type}-blacklist", $that );
            }
        );
    }
 
    public function getWhitelists() {
        $that = $this;
        $type = $this->getBlacklistType();
        $cache = MediaWikiServices::getInstance()->getMainWANObjectCache();
 
        return $cache->getWithSetCallback(
            $cache->makeKey( 'spamblacklist', $type, 'whitelist-regex' ),
            $this->expiryTime,
            function () use ( $that, $type ) {
                return SpamRegexBatch::regexesFromMessage( "{$type}-whitelist", $that );
            }
        );
    }
 
    private function getSharedBlacklists() {
        $listType = $this->getBlacklistType();
 
        wfDebugLog( 'SpamBlacklist', "Loading $listType regex..." );
 
        if ( !$this->files ) {
            # No lists
            wfDebugLog( 'SpamBlacklist', "no files specified\n" );
            return [];
        }
 
        $miss = false;
 
        $that = $this;
        $cache = MediaWikiServices::getInstance()->getMainWANObjectCache();
        $regexes = $cache->getWithSetCallback(
            // This used to be cached per-site, but that could be bad on a shared
            // server where not all wikis have the same configuration.
            $cache->makeKey( 'spamblacklist', $listType, 'shared-blacklist-regex' ),
            $this->expiryTime,
            function () use ( $that, &$miss ) {
                $miss = true;
                return $that->buildSharedBlacklists();
            }
        );
 
        if ( !$miss ) {
            wfDebugLog( 'SpamBlacklist', "Got shared spam regexes from cache\n" );
        }
 
        return $regexes;
    }
 
    public function clearCache() {
        $listType = $this->getBlacklistType();
 
        $cache = MediaWikiServices::getInstance()->getMainWANObjectCache();
        $cache->delete( $cache->makeKey( 'spamblacklist', $listType, 'shared-blacklist-regex' ) );
        $cache->delete( $cache->makeKey( 'spamblacklist', $listType, 'blacklist-regex' ) );
        $cache->delete( $cache->makeKey( 'spamblacklist', $listType, 'whitelist-regex' ) );
 
        wfDebugLog( 'SpamBlacklist', "$listType blacklist local cache cleared.\n" );
    }
 
    private function buildSharedBlacklists() {
        $regexes = [];
        $listType = $this->getBlacklistType();
        # Load lists
        wfDebugLog( 'SpamBlacklist', "Constructing $listType blacklist\n" );
        foreach ( $this->files as $fileName ) {
            $matches = [];
            if ( preg_match( '/^DB: ([\w-]*) (.*)$/', $fileName, $matches ) ) {
                $text = $this->getArticleText( $matches[1], $matches[2] );
            } elseif ( preg_match( '/^(https?:)?\/\//', $fileName ) ) {
                $text = $this->getHttpText( $fileName );
            } else {
                $text = file_get_contents( $fileName );
                wfDebugLog( 'SpamBlacklist', "got from file $fileName\n" );
            }
 
            // Build a separate batch of regexes from each source.
            // While in theory we could squeeze a little efficiency
            // out of combining multiple sources in one regex, if
            // there's a bad line in one of them we'll gain more
            // from only having to break that set into smaller pieces.
            $regexes = array_merge(
                $regexes,
                SpamRegexBatch::regexesFromText( $text, $this, $fileName )
            );
        }
 
        return $regexes;
    }
 
    private function getHttpText( $fileName ) {
        global $wgDBname, $messageMemc;
        $listType = $this->getBlacklistType();
 
        # HTTP request
        # To keep requests to a minimum, we save results into $messageMemc, which is
        # similar to $wgMemc except almost certain to exist. By default, it is stored
        # in the database
        # There are two keys, when the warning key expires, a random thread will refresh
        # the real key. This reduces the chance of multiple requests under high traffic
        # conditions.
        $key = "{$listType}_blacklist_file:$fileName";
        $warningKey = "$wgDBname:{$listType}filewarning:$fileName";
        $httpText = $messageMemc->get( $key );
        $warning = $messageMemc->get( $warningKey );
 
        if ( !is_string( $httpText ) || ( !$warning && !mt_rand( 0, $this->warningChance ) ) ) {
            wfDebugLog( 'SpamBlacklist', "Loading $listType blacklist from $fileName\n" );
            $httpText = Http::get( $fileName );
            if ( $httpText === false ) {
                wfDebugLog( 'SpamBlacklist', "Error loading $listType blacklist from $fileName\n" );
            }
            $messageMemc->set( $warningKey, 1, $this->warningTime );
            $messageMemc->set( $key, $httpText, $this->expiryTime );
        } else {
            wfDebugLog( 'SpamBlacklist', "Got $listType blacklist from HTTP cache for $fileName\n" );
        }
        return $httpText;
    }
 
    private function getArticleText( $wiki, $pagename ) {
        wfDebugLog( 'SpamBlacklist',
            "Fetching {$this->getBlacklistType()} blacklist from '$pagename' on '$wiki'...\n" );
 
        $services = MediaWikiServices::getInstance();
 
        // XXX: We do not know about custom namespaces on the target wiki here!
        $title = $services->getTitleParser()->parseTitle( $pagename );
        $store = $services->getRevisionStoreFactory()->getRevisionStore( $wiki );
        $rev = $store->getRevisionByTitle( $title );
 
        $content = $rev ? $rev->getContent( SlotRecord::MAIN ) : null;
 
        if ( !( $content instanceof TextContent ) ) {
            return false;
        }
 
        return $content->getText();
    }
 
    public function getRegexStart() {
        return '/[a-z0-9_\-.]*';
    }
 
    public function getRegexEnd( $batchSize ) {
        return ( $batchSize > 0 ) ? '/Sim' : '/im';
    }
 
    public function warmCachesForFilter( Title $title, array $entries ) {
        // subclass this
    }
}