LinkFilter.php

Go to the documentation of this file.

<?php
namespace MediaWiki\ExternalLinks;
 
use Content;
use MediaWiki\MainConfigNames;
use MediaWiki\MediaWikiServices;
use StringUtils;
use TextContent;
use Wikimedia\IPUtils;
use Wikimedia\Rdbms\AndExpressionGroup;
use Wikimedia\Rdbms\IExpression;
use Wikimedia\Rdbms\LikeMatch;
use Wikimedia\Rdbms\LikeValue;
use Wikimedia\Rdbms\OrExpressionGroup;
 
class LinkFilter {
    public static function matchEntry( Content $content, $filterEntry, $protocol = 'http://' ) {
        if ( !( $content instanceof TextContent ) ) {
            // TODO: handle other types of content too.
            //      Maybe create ContentHandler::matchFilter( LinkFilter ).
            //      Think about a common base class for LinkFilter and MagicWord.
            return 0;
        }
 
        $text = $content->getText();
        $regex = self::makeRegex( $filterEntry, $protocol );
        return preg_match( $regex, $text );
    }
 
    private static function makeRegex( $filterEntry, $protocol ) {
        $regex = '!' . preg_quote( $protocol, '!' );
        if ( substr( $filterEntry, 0, 2 ) == '*.' ) {
            $regex .= '(?:[A-Za-z0-9.-]+\.|)';
            $filterEntry = substr( $filterEntry, 2 );
        }
        $regex .= preg_quote( $filterEntry, '!' ) . '!Si';
        return $regex;
    }
 
    private static function indexifyHost( $host, $reverse = true ) {
        // Canonicalize.
        $host = rawurldecode( $host );
        if ( $host !== '' ) {
            $tmp = idn_to_utf8( $host );
            if ( $tmp !== false ) {
                $host = $tmp;
            }
        }
        $okChars = 'a-zA-Z0-9\\-._~!$&\'()*+,;=';
        if ( StringUtils::isUtf8( $host ) ) {
            // Save a little space by not percent-encoding valid UTF-8 bytes
            $okChars .= '\x80-\xf4';
        }
        $host = preg_replace_callback(
            '<[^' . $okChars . ']+>',
            static fn ( $m ) => rawurlencode( $m[0] ),
            strtolower( $host )
        );
 
        // IPv6? RFC 3986 syntax.
        if ( preg_match( '/^\[([0-9a-f:*]+)\]$/', rawurldecode( $host ), $m ) ) {
            $ip = $m[1];
            if ( IPUtils::isValid( $ip ) ) {
                if ( !$reverse ) {
                    return '[' . IPUtils::sanitizeIP( $ip ) . ']';
                }
                return 'V6.' . implode( '.', explode( ':', IPUtils::sanitizeIP( $ip ) ) ) . '.';
            }
            if ( substr( $ip, -2 ) === ':*' ) {
                $cutIp = substr( $ip, 0, -2 );
                if ( IPUtils::isValid( "{$cutIp}::" ) ) {
                    // Wildcard IP doesn't contain "::", so multiple parts can be wild
                    $ct = count( explode( ':', $ip ) ) - 1;
                    if ( !$reverse ) {
                        return '[' . IPUtils::sanitizeIP( "{$cutIp}::" ) . ']';
                    }
                    return 'V6.' .
                        implode( '.', array_slice( explode( ':', IPUtils::sanitizeIP( "{$cutIp}::" ) ), 0, $ct ) ) .
                        '.*.';
                }
                if ( IPUtils::isValid( "{$cutIp}:1" ) ) {
                    // Wildcard IP does contain "::", so only the last part is wild
                    if ( !$reverse ) {
                        return '[' . IPUtils::sanitizeIP( "{$cutIp}:1" ) . ']';
                    }
                    return 'V6.' .
                        substr( implode( '.', explode( ':', IPUtils::sanitizeIP( "{$cutIp}:1" ) ) ), 0, -1 ) .
                        '*.';
                }
            }
        }
 
        // Regularize explicit specification of the DNS root.
        // Browsers seem to do this for IPv4 literals too.
        if ( substr( $host, -1 ) === '.' ) {
            $host = substr( $host, 0, -1 );
        }
 
        // IPv4?
        $b = '(?:0*25[0-5]|0*2[0-4][0-9]|0*1[0-9][0-9]|0*[0-9]?[0-9])';
        if ( preg_match( "/^(?:{$b}\.){3}{$b}$|^(?:{$b}\.){1,3}\*$/", $host ) ) {
            if ( !$reverse ) {
                return $host;
            }
            return 'V4.' . implode( '.', array_map( static function ( $v ) {
                return $v === '*' ? $v : (int)$v;
            }, explode( '.', $host ) ) ) . '.';
        }
 
        // Must be a host name.
        if ( $reverse ) {
            return implode( '.', array_reverse( explode( '.', $host ) ) ) . '.';
        } else {
            return $host;
        }
    }
 
    public static function makeIndexes( $url, $reverseDomain = true ) {
        // NOTE: refreshExternallinksIndex.php assumes that only protocol-relative URLs return more
        // than one index, and that the indexes for protocol-relative URLs only vary in the "http://"
        // versus "https://" prefix. If you change that, you'll likely need to update
        // refreshExternallinksIndex.php accordingly.
 
        $bits = wfParseUrl( $url );
        if ( !$bits ) {
            return [];
        }
 
        // URI RFC identifies the email/server part of mailto or news protocol as 'path',
        // while we want to match the email's domain or news server the same way we are
        // matching hosts for other URLs.
        if ( in_array( $bits['scheme'], [ 'mailto', 'news' ] ) ) {
            // (T347574) Only set host if it's not already set (if // is used)
            if ( array_key_exists( 'path', $bits ) ) {
                $bits['host'] = $bits['path'];
            }
            $bits['path'] = '';
        }
 
        // Reverse the labels in the hostname, convert to lower case, unless it's an IP.
        // For emails turn it into "domain.reversed@localpart"
        if ( $bits['scheme'] == 'mailto' ) {
            $mailparts = explode( '@', $bits['host'], 2 );
            if ( count( $mailparts ) === 2 ) {
                $domainpart = self::indexifyHost( $mailparts[1], $reverseDomain );
            } else {
                // No @, assume it's a local part with no domain
                $domainpart = '';
            }
            if ( $reverseDomain ) {
                $bits['host'] = $domainpart . '@' . $mailparts[0];
            } else {
                $bits['host'] = $mailparts[0] . '@' . $domainpart;
            }
        } else {
            $bits['host'] = self::indexifyHost( $bits['host'], $reverseDomain );
        }
 
        // Reconstruct the pseudo-URL
        $index = $bits['scheme'] . $bits['delimiter'] . $bits['host'];
        // Leave out user and password. Add the port, path, query and fragment
        if ( isset( $bits['port'] ) ) {
            $index .= ':' . $bits['port'];
        }
        $index2 = $bits['path'] ?? '/';
        if ( isset( $bits['query'] ) ) {
            $index2 .= '?' . $bits['query'];
        }
        if ( isset( $bits['fragment'] ) ) {
            $index2 .= '#' . $bits['fragment'];
        }
 
        if ( $bits['scheme'] == '' ) {
            return [ [ "https:$index", $index2 ] ];
        } else {
            return [ [ $index, $index2 ] ];
        }
    }
 
    public static function getIndexedUrlsNonReversed( $urls ) {
        $newLinks = [];
        foreach ( $urls as $url ) {
            $indexes = self::makeIndexes( $url, false );
            if ( !$indexes ) {
                continue;
            }
            foreach ( $indexes as $index ) {
                $newLinks[] = $index[0] . $index[1];
            }
        }
        return $newLinks;
    }
 
    public static function reverseIndexes( $domainIndex ) {
        $bits = wfParseUrl( $domainIndex );
        if ( !$bits ) {
            return '';
        }
 
        // Reverse the labels in the hostname, convert to lower case, unless it's an IP.
        // For emails turn it into "domain.reversed@localpart"
        if ( $bits['scheme'] == 'mailto' ) {
            $mailparts = explode( '@', $bits['path'], 2 );
            if ( count( $mailparts ) === 2 ) {
                $domainpart = rtrim( self::reverseDomain( $mailparts[0] ), '.' );
            } else {
                // No @, assume it's a local part with no domain
                $domainpart = '';
            }
            $bits['host'] = $mailparts[1] . '@' . $domainpart;
        } else {
            $bits['host'] = rtrim( self::reverseDomain( $bits['host'] ), '.' );
        }
 
        $index = $bits['scheme'] . $bits['delimiter'] . $bits['host'];
        if ( isset( $bits['port'] ) && $bits['port'] ) {
            $index .= ':' . $bits['port'];
        }
        return $index;
    }
 
    private static function reverseDomain( $domain ) {
        if ( substr( $domain, 0, 3 ) === 'V6.' ) {
            $ipv6 = str_replace( '.', ':', trim( substr( $domain, 3 ), '.' ) );
            if ( IPUtils::isValid( $ipv6 ) ) {
                return '[' . $ipv6 . ']';
            }
        } elseif ( substr( $domain, 0, 3 ) === 'V4.' ) {
            $ipv4 = trim( substr( $domain, 3 ), '.' );
            if ( IPUtils::isValid( $ipv4 ) ) {
                return $ipv4;
            }
        }
        return self::indexifyHost( $domain );
    }
 
    public static function getQueryConditions( $filterEntry, array $options = [] ) {
        $options += [
            'protocol' => [ 'http://', 'https://' ],
            'oneWildcard' => false,
            'db' => null,
        ];
        $domainGaps = MediaWikiServices::getInstance()->getMainConfig()->get(
            MainConfigNames::ExternalLinksDomainGaps
        );
 
        if ( is_string( $options['protocol'] ) ) {
            $options['protocol'] = [ $options['protocol'] ];
        } elseif ( $options['protocol'] === null ) {
            $options['protocol'] = [ 'http://', 'https://' ];
        }
 
        $domainConditions = [];
        $db = $options['db'] ?: MediaWikiServices::getInstance()->getDBLoadBalancerFactory()->getReplicaDatabase();
        foreach ( $options['protocol'] as $protocol ) {
            $like = self::makeLikeArray( $filterEntry, $protocol );
            if ( $like === false ) {
                continue;
            }
            [ $likeDomain, $likePath ] = $like;
            $trimmedlikeDomain = self::keepOneWildcard( $likeDomain );
            if ( $trimmedlikeDomain[count( $trimmedlikeDomain ) - 1] instanceof LikeMatch ) {
                array_pop( $trimmedlikeDomain );
            }
            $index1 = implode( '', $trimmedlikeDomain );
            $thisDomainConditions = [];
            if ( $options['oneWildcard'] && $likePath[0] != '/' ) {
                $thisDomainConditions[] = $db->expr( 'el_to_domain_index', '=', $index1 );
            } else {
                $thisDomainConditions[] = $db->expr(
                    'el_to_domain_index',
                    IExpression::LIKE,
                    new LikeValue( $index1, $db->anyString() )
                );
            }
            foreach ( $domainGaps[$index1] ?? [] as $from => $to ) {
                $thisDomainConditions[] = $db->expr( 'el_id', '<', $from )->or( 'el_id', '>', $to );
            }
            $domainConditions[] = new AndExpressionGroup( ...$thisDomainConditions );
 
        }
        if ( !$domainConditions ) {
            return false;
        }
        // @phan-suppress-next-line PhanPossiblyUndeclaredVariable
        $trimmedlikePath = self::keepOneWildcard( $likePath );
        if ( $trimmedlikePath[count( $trimmedlikePath ) - 1] instanceof LikeMatch ) {
            array_pop( $trimmedlikePath );
        }
        $index2 = implode( '', $trimmedlikePath );
 
        return [
            new OrExpressionGroup( ...$domainConditions ),
            $db->expr( 'el_to_path', IExpression::LIKE, new LikeValue( $index2, $db->anyString() ) ),
        ];
    }
 
    public static function getProtocolPrefix( $protocol ) {
        // Find the right prefix
        $urlProtocols = MediaWikiServices::getInstance()->getMainConfig()
                                         ->get( MainConfigNames::UrlProtocols );
        if ( $protocol && !in_array( $protocol, $urlProtocols ) ) {
            foreach ( $urlProtocols as $p ) {
                if ( str_starts_with( $p, $protocol ) ) {
                    $protocol = $p;
                    break;
                }
            }
 
            return $protocol;
        } else {
            return null;
        }
    }
 
    public static function prepareProtocols() {
        $urlProtocols = MediaWikiServices::getInstance()->getMainConfig()
                                         ->get( MainConfigNames::UrlProtocols );
        $protocols = [ '' ];
        foreach ( $urlProtocols as $p ) {
            if ( $p !== '//' ) {
                $protocols[] = substr( $p, 0, strpos( $p, ':' ) );
            }
        }
 
        return $protocols;
    }
 
    public static function makeLikeArray( $filterEntry, $protocol = 'http://' ) {
        $db = MediaWikiServices::getInstance()->getDBLoadBalancerFactory()->getReplicaDatabase();
        $likeDomain = [];
        $likePath = [];
 
        $target = $protocol . $filterEntry;
        $bits = wfParseUrl( $target );
        if ( !$bits ) {
            return false;
        }
 
        // URI RFC identifies the email/server part of mailto or news protocol as 'path',
        // while we want to match the email's domain or news server the same way we are
        // matching hosts for other URLs.
        if ( in_array( $bits['scheme'], [ 'mailto', 'news' ] ) ) {
            $bits['host'] = $bits['path'];
            $bits['path'] = '';
        }
 
        $subdomains = false;
        if ( $bits['scheme'] === 'mailto' && strpos( $bits['host'], '@' ) ) {
            // Email address with domain and non-empty local part
            $mailparts = explode( '@', $bits['host'], 2 );
            $domainpart = self::indexifyHost( $mailparts[1] );
            if ( $mailparts[0] === '*' ) {
                $subdomains = true;
                $bits['host'] = $domainpart . '@';
            } else {
                $bits['host'] = $domainpart . '@' . $mailparts[0];
            }
        } else {
            // Non-email, or email with only a domain part.
            $bits['host'] = self::indexifyHost( $bits['host'] );
            if ( substr( $bits['host'], -3 ) === '.*.' ) {
                $subdomains = true;
                $bits['host'] = substr( $bits['host'], 0, -2 );
            }
        }
 
        $likeDomain[] = $bits['scheme'] . $bits['delimiter'] . $bits['host'];
 
        if ( $subdomains ) {
            $likeDomain[] = $db->anyString();
        }
 
        if ( isset( $bits['port'] ) ) {
            $likeDomain[] = ':' . $bits['port'];
        }
        if ( isset( $bits['path'] ) ) {
            $likePath[] = $bits['path'];
        } else {
            $likePath[] = '/';
        }
        if ( isset( $bits['query'] ) ) {
            $likePath[] = '?' . $bits['query'];
        }
        if ( isset( $bits['fragment'] ) ) {
            $likePath[] = '#' . $bits['fragment'];
        }
        $likePath[] = $db->anyString();
 
        // Check for stray asterisks: asterisk only allowed at the start of the domain
        foreach ( array_merge( $likeDomain, $likePath ) as $likepart ) {
            if ( !( $likepart instanceof LikeMatch ) && strpos( $likepart, '*' ) !== false ) {
                return false;
            }
        }
 
        return [ $likeDomain, $likePath ];
    }
 
    public static function keepOneWildcard( $arr ) {
        if ( !is_array( $arr ) ) {
            return $arr;
        }
 
        foreach ( $arr as $key => $value ) {
            if ( $value instanceof LikeMatch ) {
                return array_slice( $arr, 0, $key + 1 );
            }
        }
 
        return $arr;
    }
}
 
class_alias( LinkFilter::class, 'LinkFilter' );