Sanitizer.php

Go to the documentation of this file.

<?php
use MediaWiki\MediaWikiServices;
use Wikimedia\RemexHtml\HTMLData;
use Wikimedia\RemexHtml\Tokenizer\Tokenizer as RemexTokenizer;
 
class Sanitizer {
    private const CHAR_REFS_REGEX =
        '/&([A-Za-z0-9\x80-\xff]+;)
         |&\#([0-9]+);
         |&\#[xX]([0-9A-Fa-f]+);
         |(&)/x';
 
    private const ELEMENT_BITS_REGEX = '!^(/?)([A-Za-z][^\t\n\v />\0]*+)([^>]*?)(/?>)([^<]*)$!';
 
    private const EVIL_URI_PATTERN = '!(^|\s|\*/\s*)(javascript|vbscript)([^\w]|$)!i';
    private const XMLNS_ATTRIBUTE_PATTERN = "/^xmlns:[:A-Z_a-z-.0-9]+$/";
 
    public const ID_PRIMARY = 0;
 
    public const ID_FALLBACK = 1;
 
    private const MW_ENTITY_ALIASES = [
        'רלמ;' => 'rlm;',
        'رلم;' => 'rlm;',
    ];
 
    private static $attribsRegex;
 
    private static function getAttribsRegex() {
        if ( self::$attribsRegex === null ) {
            $spaceChars = '\x09\x0a\x0c\x0d\x20';
            $space = "[{$spaceChars}]";
            $attrib = "[^{$spaceChars}\/>=]";
            $attribFirst = "(?:{$attrib}|=)";
            self::$attribsRegex =
                "/({$attribFirst}{$attrib}*)
                    ($space*=$space*
                    (?:
                        # The attribute value: quoted or alone
                        \"([^\"]*)(?:\"|\$)
                        | '([^']*)(?:'|\$)
                        | (((?!$space|>).)*)
                    )
                )?/sxu";
        }
        return self::$attribsRegex;
    }
 
    private static $attribNameRegex;
 
    private static function getAttribNameRegex() {
        if ( self::$attribNameRegex === null ) {
            $attribFirst = "[:_\p{L}\p{N}]";
            $attrib = "[:_\.\-\p{L}\p{N}]";
            self::$attribNameRegex = "/^({$attribFirst}{$attrib}*)$/sxu";
        }
        return self::$attribNameRegex;
    }
 
    public static function getRecognizedTagData( $extratags = [], $removetags = [] ) {
        global $wgAllowImageTag;
 
        static $htmlpairsStatic, $htmlsingle, $htmlsingleonly, $htmlnest, $tabletags,
            $htmllist, $listtags, $htmlsingleallowed, $htmlelementsStatic, $staticInitialised;
 
        // Base our staticInitialised variable off of the global config state so that if the globals
        // are changed (like in the screwed up test system) we will re-initialise the settings.
        $globalContext = $wgAllowImageTag;
        if ( !$staticInitialised || $staticInitialised != $globalContext ) {
            $htmlpairsStatic = [ # Tags that must be closed
                'b', 'bdi', 'del', 'i', 'ins', 'u', 'font', 'big', 'small', 'sub', 'sup', 'h1',
                'h2', 'h3', 'h4', 'h5', 'h6', 'cite', 'code', 'em', 's',
                'strike', 'strong', 'tt', 'var', 'div', 'center',
                'blockquote', 'ol', 'ul', 'dl', 'table', 'caption', 'pre',
                'ruby', 'rb', 'rp', 'rt', 'rtc', 'p', 'span', 'abbr', 'dfn',
                'kbd', 'samp', 'data', 'time', 'mark'
            ];
            $htmlsingle = [
                'br', 'wbr', 'hr', 'li', 'dt', 'dd', 'meta', 'link'
            ];
 
            # Elements that cannot have close tags. This is (not coincidentally)
            # also the list of tags for which the HTML 5 parsing algorithm
            # requires you to "acknowledge the token's self-closing flag", i.e.
            # a self-closing tag like <br/> is not an HTML 5 parse error only
            # for this list.
            $htmlsingleonly = [
                'br', 'wbr', 'hr', 'meta', 'link'
            ];
 
            $htmlnest = [ # Tags that can be nested--??
                'table', 'tr', 'td', 'th', 'div', 'blockquote', 'ol', 'ul',
                'li', 'dl', 'dt', 'dd', 'font', 'big', 'small', 'sub', 'sup', 'span',
                'var', 'kbd', 'samp', 'em', 'strong', 'q', 'ruby', 'bdo'
            ];
            $tabletags = [ # Can only appear inside table, we will close them
                'td', 'th', 'tr',
            ];
            $htmllist = [ # Tags used by list
                'ul', 'ol',
            ];
            $listtags = [ # Tags that can appear in a list
                'li',
            ];
 
            if ( $wgAllowImageTag ) {
                wfDeprecatedMsg( 'Setting $wgAllowImageTag to true ' .
                    'is deprecated since MediaWiki 1.35', '1.35', false, false );
                $htmlsingle[] = 'img';
                $htmlsingleonly[] = 'img';
            }
 
            $htmlsingleallowed = array_unique( array_merge( $htmlsingle, $tabletags ) );
            $htmlelementsStatic = array_unique( array_merge( $htmlsingle, $htmlpairsStatic, $htmlnest ) );
 
            # Convert them all to hashtables for faster lookup
            $vars = [ 'htmlpairsStatic', 'htmlsingle', 'htmlsingleonly', 'htmlnest', 'tabletags',
                'htmllist', 'listtags', 'htmlsingleallowed', 'htmlelementsStatic' ];
            foreach ( $vars as $var ) {
                $$var = array_fill_keys( $$var, true );
            }
            $staticInitialised = $globalContext;
        }
 
        # Populate $htmlpairs and $htmlelements with the $extratags and $removetags arrays
        $extratags = array_fill_keys( $extratags, true );
        $removetags = array_fill_keys( $removetags, true );
        $htmlpairs = array_merge( $extratags, $htmlpairsStatic );
        $htmlelements = array_diff_key( array_merge( $extratags, $htmlelementsStatic ), $removetags );
 
        return [
            'htmlpairs' => $htmlpairs,
            'htmlsingle' => $htmlsingle,
            'htmlsingleonly' => $htmlsingleonly,
            'htmlnest' => $htmlnest,
            'tabletags' => $tabletags,
            'htmllist' => $htmllist,
            'listtags' => $listtags,
            'htmlsingleallowed' => $htmlsingleallowed,
            'htmlelements' => $htmlelements,
        ];
    }
 
    public static function removeHTMLtags( $text, $processCallback = null,
        $args = [], $extratags = [], $removetags = []
    ) {
        $tagData = self::getRecognizedTagData( $extratags, $removetags );
        $htmlpairs = $tagData['htmlpairs'];
        $htmlsingle = $tagData['htmlsingle'];
        $htmlsingleonly = $tagData['htmlsingleonly'];
        $htmlnest = $tagData['htmlnest'];
        $tabletags = $tagData['tabletags'];
        $htmllist = $tagData['htmllist'];
        $listtags = $tagData['listtags'];
        $htmlsingleallowed = $tagData['htmlsingleallowed'];
        $htmlelements = $tagData['htmlelements'];
 
        # Remove HTML comments
        $text = self::removeHTMLcomments( $text );
        $bits = explode( '<', $text );
        $text = str_replace( '>', '&gt;', array_shift( $bits ) );
 
        # this might be possible using remex tidy itself
        foreach ( $bits as $x ) {
            if ( preg_match( self::ELEMENT_BITS_REGEX, $x, $regs ) ) {
                list( /* $qbar */, $slash, $t, $params, $brace, $rest ) = $regs;
 
                $badtag = false;
                $t = strtolower( $t );
                if ( isset( $htmlelements[$t] ) ) {
                    if ( is_callable( $processCallback ) ) {
                        call_user_func_array( $processCallback, [ &$params, $args ] );
                    }
 
                    if ( $brace == '/>' && !( isset( $htmlsingle[$t] ) || isset( $htmlsingleonly[$t] ) ) ) {
                        // Remove the self-closing slash, to be consistent
                        // with HTML5 semantics. T134423
                        $brace = '>';
                    }
                    if ( !self::validateTag( $params, $t ) ) {
                        $badtag = true;
                    }
 
                    $newparams = self::fixTagAttributes( $params, $t );
                    if ( !$badtag ) {
                        if ( $brace === '/>' && !isset( $htmlsingleonly[$t] ) ) {
                            # Interpret self-closing tags as empty tags even when
                            # HTML 5 would interpret them as start tags. Such input
                            # is commonly seen on Wikimedia wikis with this intention.
                            $brace = "></$t>";
                        }
 
                        $rest = str_replace( '>', '&gt;', $rest );
                        $text .= "<$slash$t$newparams$brace$rest";
                        continue;
                    }
                }
            }
            $text .= '&lt;' . str_replace( '>', '&gt;', $x );
        }
        return $text;
    }
 
    public static function removeHTMLcomments( $text ) {
        while ( ( $start = strpos( $text, '<!--' ) ) !== false ) {
            $end = strpos( $text, '-->', $start + 4 );
            if ( $end === false ) {
                # Unterminated comment; bail out
                break;
            }
 
            $end += 3;
 
            # Trim space and newline if the comment is both
            # preceded and followed by a newline
            $spaceStart = max( $start - 1, 0 );
            $spaceLen = $end - $spaceStart;
            while ( substr( $text, $spaceStart, 1 ) === ' ' && $spaceStart > 0 ) {
                $spaceStart--;
                $spaceLen++;
            }
            while ( substr( $text, $spaceStart + $spaceLen, 1 ) === ' ' ) {
                $spaceLen++;
            }
            if ( substr( $text, $spaceStart, 1 ) === "\n"
                && substr( $text, $spaceStart + $spaceLen, 1 ) === "\n" ) {
                # Remove the comment, leading and trailing
                # spaces, and leave only one newline.
                $text = substr_replace( $text, "\n", $spaceStart, $spaceLen + 1 );
            } else {
                # Remove just the comment.
                $text = substr_replace( $text, '', $start, $end - $start );
            }
        }
        return $text;
    }
 
    private static function validateTag( $params, $element ) {
        $params = self::decodeTagAttributes( $params );
 
        if ( $element == 'meta' || $element == 'link' ) {
            if ( !isset( $params['itemprop'] ) ) {
                // <meta> and <link> must have an itemprop="" otherwise they are not valid or safe in content
                return false;
            }
            if ( $element == 'meta' && !isset( $params['content'] ) ) {
                // <meta> must have a content="" for the itemprop
                return false;
            }
            if ( $element == 'link' && !isset( $params['href'] ) ) {
                // <link> must have an associated href=""
                return false;
            }
        }
 
        return true;
    }
 
    public static function validateTagAttributes( $attribs, $element ) {
        return self::validateAttributes( $attribs,
            self::attributesAllowedInternal( $element ) );
    }
 
    public static function validateAttributes( $attribs, $allowed ) {
        if ( isset( $allowed[0] ) ) {
            // Calling this function with a sequential array is
            // deprecated.  For now just convert it.
            wfDeprecated( __METHOD__ . ' with sequential array', '1.35' );
            $allowed = array_fill_keys( $allowed, true );
        }
        $hrefExp = '/^(' . wfUrlProtocols() . ')[^\s]+$/';
 
        $out = [];
        foreach ( $attribs as $attribute => $value ) {
            # Allow XML namespace declaration to allow RDFa
            if ( preg_match( self::XMLNS_ATTRIBUTE_PATTERN, $attribute ) ) {
                if ( !preg_match( self::EVIL_URI_PATTERN, $value ) ) {
                    $out[$attribute] = $value;
                }
 
                continue;
            }
 
            # Allow any attribute beginning with "data-"
            # However:
            # * Disallow data attributes used by MediaWiki code
            # * Ensure that the attribute is not namespaced by banning
            #   colons.
            if ( (
                !preg_match( '/^data-[^:]*$/i', $attribute ) &&
                !array_key_exists( $attribute, $allowed )
            ) || self::isReservedDataAttribute( $attribute ) ) {
                continue;
            }
 
            # Strip javascript "expression" from stylesheets.
            # https://msdn.microsoft.com/en-us/library/ms537634.aspx
            if ( $attribute == 'style' ) {
                $value = self::checkCss( $value );
            }
 
            # Escape HTML id attributes
            if ( $attribute === 'id' ) {
                $value = self::escapeIdForAttribute( $value, self::ID_PRIMARY );
            }
 
            # Escape HTML id reference lists
            if ( $attribute === 'aria-describedby'
                || $attribute === 'aria-flowto'
                || $attribute === 'aria-labelledby'
                || $attribute === 'aria-owns'
            ) {
                $value = self::escapeIdReferenceListInternal( $value );
            }
 
            // RDFa and microdata properties allow URLs, URIs and/or CURIs.
            if ( $attribute === 'rel' || $attribute === 'rev'
                # RDFa
                || $attribute === 'about' || $attribute === 'property'
                || $attribute === 'resource' || $attribute === 'datatype'
                || $attribute === 'typeof'
                # HTML5 microdata
                || $attribute === 'itemid' || $attribute === 'itemprop'
                || $attribute === 'itemref' || $attribute === 'itemscope'
                || $attribute === 'itemtype'
            ) {
                // Paranoia. Allow "simple" values but suppress javascript
                if ( preg_match( self::EVIL_URI_PATTERN, $value ) ) {
                    continue;
                }
            }
 
            # NOTE: even though elements using href/src are not allowed directly, supply
            #       validation code that can be used by tag hook handlers, etc
            if ( $attribute === 'href' || $attribute === 'src' || $attribute === 'poster' ) {
                if ( !preg_match( $hrefExp, $value ) ) {
                    continue; // drop any href or src attributes not using an allowed protocol.
                    // NOTE: this also drops all relative URLs
                }
            }
 
            if ( $attribute === 'tabindex' && $value !== '0' ) {
                // Only allow tabindex of 0, which is useful for accessibility.
                continue;
            }
 
            // If this attribute was previously set, override it.
            // Output should only have one attribute of each name.
            $out[$attribute] = $value;
        }
 
        # itemtype, itemid, itemref don't make sense without itemscope
        if ( !array_key_exists( 'itemscope', $out ) ) {
            unset( $out['itemtype'] );
            unset( $out['itemid'] );
            unset( $out['itemref'] );
        }
        # TODO: Strip itemprop if we aren't descendants of an itemscope or pointed to by an itemref.
 
        return $out;
    }
 
    public static function isReservedDataAttribute( $attr ) {
        // data-ooui is reserved for ooui.
        // data-mw and data-parsoid are reserved for parsoid.
        // data-mw-<name here> is reserved for extensions (or core) if
        // they need to communicate some data to the client and want to be
        // sure that it isn't coming from an untrusted user.
        // We ignore the possibility of namespaces since user-generated HTML
        // can't use them anymore.
        return (bool)preg_match( '/^data-(ooui|mw|parsoid)/i', $attr );
    }
 
    public static function mergeAttributes( $a, $b ) {
        $out = array_merge( $a, $b );
        if ( isset( $a['class'] ) && isset( $b['class'] )
            && is_string( $a['class'] ) && is_string( $b['class'] )
            && $a['class'] !== $b['class']
        ) {
            $classes = preg_split( '/\s+/', "{$a['class']} {$b['class']}",
                -1, PREG_SPLIT_NO_EMPTY );
            $out['class'] = implode( ' ', array_unique( $classes ) );
        }
        return $out;
    }
 
    public static function normalizeCss( $value ) {
        // Decode character references like &#123;
        $value = self::decodeCharReferences( $value );
 
        // Decode escape sequences and line continuation
        // See the grammar in the CSS 2 spec, appendix D.
        // This has to be done AFTER decoding character references.
        // This means it isn't possible for this function to return
        // unsanitized escape sequences. It is possible to manufacture
        // input that contains character references that decode to
        // escape sequences that decode to character references, but
        // it's OK for the return value to contain character references
        // because the caller is supposed to escape those anyway.
        static $decodeRegex;
        if ( !$decodeRegex ) {
            $space = '[\\x20\\t\\r\\n\\f]';
            $nl = '(?:\\n|\\r\\n|\\r|\\f)';
            $backslash = '\\\\';
            $decodeRegex = "/ $backslash
                (?:
                    ($nl) |  # 1. Line continuation
                    ([0-9A-Fa-f]{1,6})$space? |  # 2. character number
                    (.) | # 3. backslash cancelling special meaning
                    () | # 4. backslash at end of string
                )/xu";
        }
        $value = preg_replace_callback( $decodeRegex,
            [ __CLASS__, 'cssDecodeCallback' ], $value );
 
        // Let the value through if it's nothing but a single comment, to
        // allow other functions which may reject it to pass some error
        // message through.
        if ( !preg_match( '! ^ \s* /\* [^*\\/]* \*/ \s* $ !x', $value ) ) {
            // Remove any comments; IE gets token splitting wrong
            // This must be done AFTER decoding character references and
            // escape sequences, because those steps can introduce comments
            // This step cannot introduce character references or escape
            // sequences, because it replaces comments with spaces rather
            // than removing them completely.
            $value = StringUtils::delimiterReplace( '/*', '*/', ' ', $value );
 
            // Remove anything after a comment-start token, to guard against
            // incorrect client implementations.
            $commentPos = strpos( $value, '/*' );
            if ( $commentPos !== false ) {
                $value = substr( $value, 0, $commentPos );
            }
        }
 
        return $value;
    }
 
    public static function checkCss( $value ) {
        $value = self::normalizeCss( $value );
 
        // Reject problematic keywords and control characters
        if ( preg_match( '/[\000-\010\013\016-\037\177]/', $value ) ||
            strpos( $value, UtfNormal\Constants::UTF8_REPLACEMENT ) !== false ) {
            return '/* invalid control char */';
        } elseif ( preg_match(
            '! expression
                | filter\s*:
                | accelerator\s*:
                | -o-link\s*:
                | -o-link-source\s*:
                | -o-replace\s*:
                | url\s*\‍(
                | image\s*\‍(
                | image-set\s*\‍(
                | attr\s*\‍([^)]+[\s,]+url
                | var\s*\‍(
            !ix', $value ) ) {
            return '/* insecure input */';
        }
        return $value;
    }
 
    private static function cssDecodeCallback( $matches ) {
        if ( $matches[1] !== '' ) {
            // Line continuation
            return '';
        } elseif ( $matches[2] !== '' ) {
            $char = UtfNormal\Utils::codepointToUtf8( hexdec( $matches[2] ) );
        } elseif ( $matches[3] !== '' ) {
            $char = $matches[3];
        } else {
            $char = '\\';
        }
        if ( $char == "\n" || $char == '"' || $char == "'" || $char == '\\' ) {
            // These characters need to be escaped in strings
            // Clean up the escape sequence to avoid parsing errors by clients
            return '\\' . dechex( ord( $char ) ) . ' ';
        } else {
            // Decode unnecessary escape
            return $char;
        }
    }
 
    public static function fixTagAttributes( $text, $element, $sorted = false ) {
        if ( trim( $text ) == '' ) {
            return '';
        }
 
        $decoded = self::decodeTagAttributes( $text );
        $stripped = self::validateTagAttributes( $decoded, $element );
 
        if ( $sorted ) {
            ksort( $stripped );
        }
 
        return self::safeEncodeTagAttributes( $stripped );
    }
 
    public static function encodeAttribute( $text ) {
        $encValue = htmlspecialchars( $text, ENT_QUOTES );
 
        // Whitespace is normalized during attribute decoding,
        // so if we've been passed non-spaces we must encode them
        // ahead of time or they won't be preserved.
        $encValue = strtr( $encValue, [
            "\n" => '&#10;',
            "\r" => '&#13;',
            "\t" => '&#9;',
        ] );
 
        return $encValue;
    }
 
    public static function armorFrenchSpaces( $text, $space = '&#160;' ) {
        // Replace $ with \$ and \ with \\
        $space = preg_replace( '#(?<!\\\\‍)(\\$|\\\\‍)#', '\\\\$1', $space );
        $fixtags = [
            # French spaces, last one Guillemet-left
            # only if it isn't followed by a word character.
            '/ (?=[?:;!%»›](?!\w))/u' => "$space",
            # French spaces, Guillemet-right
            '/([«‹]) /u' => "\\1$space",
        ];
        return preg_replace( array_keys( $fixtags ), array_values( $fixtags ), $text );
    }
 
    public static function safeEncodeAttribute( $text ) {
        $encValue = self::encodeAttribute( $text );
 
        # Templates and links may be expanded in later parsing,
        # creating invalid or dangerous output. Suppress this.
        $encValue = strtr( $encValue, [
            '<'    => '&lt;',   // This should never happen,
            '>'    => '&gt;',   // we've received invalid input
            '"'    => '&quot;', // which should have been escaped.
            '{'    => '&#123;',
            '}'    => '&#125;', // prevent unpaired language conversion syntax
            '['    => '&#91;',
            ']'    => '&#93;',
            "''"   => '&#39;&#39;',
            'ISBN' => '&#73;SBN',
            'RFC'  => '&#82;FC',
            'PMID' => '&#80;MID',
            '|'    => '&#124;',
            '__'   => '&#95;_',
        ] );
 
        # Stupid hack
        $encValue = preg_replace_callback(
            '/((?i)' . wfUrlProtocols() . ')/',
            static function ( $matches ) {
                return str_replace( ':', '&#58;', $matches[1] );
            },
            $encValue );
        return $encValue;
    }
 
    public static function escapeIdForAttribute( $id, $mode = self::ID_PRIMARY ) {
        global $wgFragmentMode;
 
        if ( !isset( $wgFragmentMode[$mode] ) ) {
            if ( $mode === self::ID_PRIMARY ) {
                throw new UnexpectedValueException( '$wgFragmentMode is configured with no primary mode' );
            }
            return false;
        }
 
        $internalMode = $wgFragmentMode[$mode];
 
        return self::escapeIdInternal( $id, $internalMode );
    }
 
    public static function escapeIdForLink( $id ) {
        global $wgFragmentMode;
 
        if ( !isset( $wgFragmentMode[self::ID_PRIMARY] ) ) {
            throw new UnexpectedValueException( '$wgFragmentMode is configured with no primary mode' );
        }
 
        $mode = $wgFragmentMode[self::ID_PRIMARY];
 
        $id = self::escapeIdInternalUrl( $id, $mode );
 
        return $id;
    }
 
    public static function escapeIdForExternalInterwiki( $id ) {
        global $wgExternalInterwikiFragmentMode;
 
        $id = self::escapeIdInternalUrl( $id, $wgExternalInterwikiFragmentMode );
 
        return $id;
    }
 
    private static function escapeIdInternalUrl( $id, $mode ) {
        $id = self::escapeIdInternal( $id, $mode );
        if ( $mode === 'html5' ) {
            $id = preg_replace( '/%([a-fA-F0-9]{2})/', '%25$1', $id );
        }
        return $id;
    }
 
    private static function escapeIdInternal( $id, $mode ) {
        // Truncate overly-long IDs.  This isn't an HTML limit, it's just
        // griefer protection. [T251506]
        $id = mb_substr( $id, 0, 1024 );
 
        switch ( $mode ) {
            case 'html5':
                // html5 spec says ids must not have any of the following:
                // U+0009 TAB, U+000A LF, U+000C FF, U+000D CR, or U+0020 SPACE
                // In practice, in wikitext, only tab, LF, CR (and SPACE) are
                // possible using either Lua or html entities.
                $id = str_replace( [ "\t", "\n", "\f", "\r", " " ], '_', $id );
                break;
            case 'legacy':
                // This corresponds to 'noninitial' mode of the former escapeId()
                static $replace = [
                    '%3A' => ':',
                    '%' => '.'
                ];
 
                $id = urlencode( str_replace( ' ', '_', $id ) );
                $id = strtr( $id, $replace );
                break;
            default:
                throw new InvalidArgumentException( "Invalid mode '$mode' passed to '" . __METHOD__ );
        }
 
        return $id;
    }
 
    public static function escapeIdReferenceList( $referenceString ) {
        wfDeprecated( __METHOD__, '1.36' );
        return self::escapeIdReferenceListInternal( $referenceString );
    }
 
    private static function escapeIdReferenceListInternal( $referenceString ) {
        # Explode the space delimited list string into an array of tokens
        $references = preg_split( '/\s+/', "{$referenceString}", -1, PREG_SPLIT_NO_EMPTY );
 
        # Escape each token as an id
        foreach ( $references as &$ref ) {
            $ref = self::escapeIdForAttribute( $ref );
        }
 
        # Merge the array back to a space delimited list string
        # If the array is empty, the result will be an empty string ('')
        $referenceString = implode( ' ', $references );
 
        return $referenceString;
    }
 
    public static function escapeClass( $class ) {
        // Convert ugly stuff to underscores and kill underscores in ugly places
        return rtrim( preg_replace(
            [ '/(^[0-9\\-])|[\\x00-\\x20!"#$%&\'()*+,.\\/:;<=>?@[\\]^`{|}~]|\\xC2\\xA0/', '/_+/' ],
            '_',
            $class ), '_' );
    }
 
    public static function escapeHtmlAllowEntities( $html ) {
        $html = self::decodeCharReferences( $html );
        # It seems wise to escape ' as well as ", as a matter of course.  Can't
        # hurt. Use ENT_SUBSTITUTE so that incorrectly truncated multibyte characters
        # don't cause the entire string to disappear.
        $html = htmlspecialchars( $html, ENT_QUOTES | ENT_SUBSTITUTE );
        return $html;
    }
 
    public static function decodeTagAttributes( $text ) {
        if ( trim( $text ) == '' ) {
            return [];
        }
 
        $pairs = [];
        if ( !preg_match_all(
            self::getAttribsRegex(),
            $text,
            $pairs,
            PREG_SET_ORDER ) ) {
            return [];
        }
 
        $attribs = [];
        foreach ( $pairs as $set ) {
            $attribute = strtolower( $set[1] );
 
            // Filter attribute names with unacceptable characters
            if ( !preg_match( self::getAttribNameRegex(), $attribute ) ) {
                continue;
            }
 
            $value = self::getTagAttributeCallback( $set );
 
            // Normalize whitespace
            $value = preg_replace( '/[\t\r\n ]+/', ' ', $value );
            $value = trim( $value );
 
            // Decode character references
            $attribs[$attribute] = self::decodeCharReferences( $value );
        }
        return $attribs;
    }
 
    public static function safeEncodeTagAttributes( $assoc_array ) {
        $attribs = [];
        foreach ( $assoc_array as $attribute => $value ) {
            $encAttribute = htmlspecialchars( $attribute );
            $encValue = self::safeEncodeAttribute( $value );
 
            $attribs[] = "$encAttribute=\"$encValue\"";
        }
        return count( $attribs ) ? ' ' . implode( ' ', $attribs ) : '';
    }
 
    private static function getTagAttributeCallback( $set ) {
        if ( isset( $set[5] ) ) {
            # No quotes.
            return $set[5];
        } elseif ( isset( $set[4] ) ) {
            # Single-quoted
            return $set[4];
        } elseif ( isset( $set[3] ) ) {
            # Double-quoted
            return $set[3];
        } elseif ( !isset( $set[2] ) ) {
            # In XHTML, attributes must have a value so return an empty string.
            # See "Empty attribute syntax",
            # https://www.w3.org/TR/html5/syntax.html#syntax-attribute-name
            return "";
        } else {
            throw new MWException( "Tag conditions not met. This should never happen and is a bug." );
        }
    }
 
    private static function normalizeWhitespace( $text ) {
        return trim( preg_replace(
            '/(?:\r\n|[\x20\x0d\x0a\x09])+/',
            ' ',
            $text ) );
    }
 
    public static function normalizeSectionNameWhitespace( $section ) {
        return trim( preg_replace( '/[ _]+/', ' ', $section ) );
    }
 
    public static function normalizeCharReferences( $text ) {
        return preg_replace_callback(
            self::CHAR_REFS_REGEX,
            [ self::class, 'normalizeCharReferencesCallback' ],
            $text );
    }
 
    private static function normalizeCharReferencesCallback( $matches ) {
        $ret = null;
        if ( $matches[1] != '' ) {
            $ret = self::normalizeEntity( $matches[1] );
        } elseif ( $matches[2] != '' ) {
            $ret = self::decCharReference( $matches[2] );
        } elseif ( $matches[3] != '' ) {
            $ret = self::hexCharReference( $matches[3] );
        }
        if ( $ret === null ) {
            return htmlspecialchars( $matches[0] );
        } else {
            return $ret;
        }
    }
 
    private static function normalizeEntity( $name ) {
        if ( isset( self::MW_ENTITY_ALIASES[$name] ) ) {
            // Non-standard MediaWiki-specific entities
            return '&' . self::MW_ENTITY_ALIASES[$name];
        } elseif ( in_array( $name, [ 'lt;', 'gt;', 'amp;', 'quot;' ], true ) ) {
            // Keep these in word form
            return "&$name";
        } elseif ( isset( HTMLData::$namedEntityTranslations[$name] ) ) {
            // Beware: some entities expand to more than 1 codepoint
            return preg_replace_callback( '/./Ssu', static function ( $m ) {
                return '&#' . UtfNormal\Utils::utf8ToCodepoint( $m[0] ) . ';';
            }, HTMLData::$namedEntityTranslations[$name] );
        } else {
            return "&amp;$name";
        }
    }
 
    private static function decCharReference( $codepoint ) {
        $point = intval( $codepoint );
        if ( self::validateCodepoint( $point ) ) {
            return sprintf( '&#%d;', $point );
        } else {
            return null;
        }
    }
 
    private static function hexCharReference( $codepoint ) {
        $point = hexdec( $codepoint );
        if ( self::validateCodepoint( $point ) ) {
            return sprintf( '&#x%x;', $point );
        } else {
            return null;
        }
    }
 
    private static function validateCodepoint( $codepoint ) {
        # U+000C is valid in HTML5 but not allowed in XML.
        # U+000D is valid in XML but not allowed in HTML5.
        # U+007F - U+009F are disallowed in HTML5 (control characters).
        return $codepoint == 0x09
            || $codepoint == 0x0a
            || ( $codepoint >= 0x20 && $codepoint <= 0x7e )
            || ( $codepoint >= 0xa0 && $codepoint <= 0xd7ff )
            || ( $codepoint >= 0xe000 && $codepoint <= 0xfffd )
            || ( $codepoint >= 0x10000 && $codepoint <= 0x10ffff );
    }
 
    public static function decodeCharReferences( $text ) {
        return preg_replace_callback(
            self::CHAR_REFS_REGEX,
            [ self::class, 'decodeCharReferencesCallback' ],
            $text );
    }
 
    public static function decodeCharReferencesAndNormalize( $text ) {
        $text = preg_replace_callback(
            self::CHAR_REFS_REGEX,
            [ self::class, 'decodeCharReferencesCallback' ],
            $text,
            -1, // limit
            $count
        );
 
        if ( $count ) {
            return MediaWikiServices::getInstance()->getContentLanguage()->normalize( $text );
        } else {
            return $text;
        }
    }
 
    private static function decodeCharReferencesCallback( $matches ) {
        if ( $matches[1] != '' ) {
            return self::decodeEntity( $matches[1] );
        } elseif ( $matches[2] != '' ) {
            return self::decodeChar( intval( $matches[2] ) );
        } elseif ( $matches[3] != '' ) {
            return self::decodeChar( hexdec( $matches[3] ) );
        }
        # Last case should be an ampersand by itself
        return $matches[0];
    }
 
    private static function decodeChar( $codepoint ) {
        if ( self::validateCodepoint( $codepoint ) ) {
            return UtfNormal\Utils::codepointToUtf8( $codepoint );
        } else {
            return UtfNormal\Constants::UTF8_REPLACEMENT;
        }
    }
 
    private static function decodeEntity( $name ) {
        // These are MediaWiki-specific entities, not in the HTML standard
        if ( isset( self::MW_ENTITY_ALIASES[$name] ) ) {
            $name = self::MW_ENTITY_ALIASES[$name];
        }
        $trans = HTMLData::$namedEntityTranslations[$name] ?? null;
        return $trans ?? "&$name";
    }
 
    private static function attributesAllowedInternal( $element ) {
        $list = self::setupAttributesAllowedInternal();
        return $list[$element] ?? [];
    }
 
    private static function setupAttributesAllowedInternal() {
        static $allowed;
 
        if ( $allowed !== null ) {
            return $allowed;
        }
 
        // For lookup efficiency flip each attributes array so the keys are
        // the valid attributes.
        $merge = static function ( $a, $b, $c = [] ) {
            return array_merge(
                $a,
                array_fill_keys( $b, true ),
                array_fill_keys( $c, true ) );
        };
        $common = $merge( [], [
            # HTML
            'id',
            'class',
            'style',
            'lang',
            'dir',
            'title',
            'tabindex',
 
            # WAI-ARIA
            'aria-describedby',
            'aria-flowto',
            'aria-hidden',
            'aria-label',
            'aria-labelledby',
            'aria-owns',
            'role',
 
            # RDFa
            # These attributes are specified in section 9 of
            # https://www.w3.org/TR/2008/REC-rdfa-syntax-20081014
            'about',
            'property',
            'resource',
            'datatype',
            'typeof',
 
            # Microdata. These are specified by
            # https://html.spec.whatwg.org/multipage/microdata.html#the-microdata-model
            'itemid',
            'itemprop',
            'itemref',
            'itemscope',
            'itemtype',
        ] );
 
        $block = $merge( $common, [ 'align' ] );
 
        $tablealign = [ 'align', 'valign' ];
        $tablecell = [
            'abbr',
            'axis',
            'headers',
            'scope',
            'rowspan',
            'colspan',
            'nowrap', # deprecated
            'width', # deprecated
            'height', # deprecated
            'bgcolor', # deprecated
        ];
 
        # Numbers refer to sections in HTML 4.01 standard describing the element.
        # See: https://www.w3.org/TR/html4/
        $allowed = [
            # 7.5.4
            'div'        => $block,
            'center'     => $common, # deprecated
            'span'       => $common,
 
            # 7.5.5
            'h1'         => $block,
            'h2'         => $block,
            'h3'         => $block,
            'h4'         => $block,
            'h5'         => $block,
            'h6'         => $block,
 
            # 7.5.6
            # address
 
            # 8.2.4
            'bdo'        => $common,
 
            # 9.2.1
            'em'         => $common,
            'strong'     => $common,
            'cite'       => $common,
            'dfn'        => $common,
            'code'       => $common,
            'samp'       => $common,
            'kbd'        => $common,
            'var'        => $common,
            'abbr'       => $common,
            # acronym
 
            # 9.2.2
            'blockquote' => $merge( $common, [ 'cite' ] ),
            'q'          => $merge( $common, [ 'cite' ] ),
 
            # 9.2.3
            'sub'        => $common,
            'sup'        => $common,
 
            # 9.3.1
            'p'          => $block,
 
            # 9.3.2
            'br'         => $merge( $common, [ 'clear' ] ),
 
            # https://www.w3.org/TR/html5/text-level-semantics.html#the-wbr-element
            'wbr'        => $common,
 
            # 9.3.4
            'pre'        => $merge( $common, [ 'width' ] ),
 
            # 9.4
            'ins'        => $merge( $common, [ 'cite', 'datetime' ] ),
            'del'        => $merge( $common, [ 'cite', 'datetime' ] ),
 
            # 10.2
            'ul'         => $merge( $common, [ 'type' ] ),
            'ol'         => $merge( $common, [ 'type', 'start', 'reversed' ] ),
            'li'         => $merge( $common, [ 'type', 'value' ] ),
 
            # 10.3
            'dl'         => $common,
            'dd'         => $common,
            'dt'         => $common,
 
            # 11.2.1
            'table'      => $merge( $common,
                                [ 'summary', 'width', 'border', 'frame',
                                        'rules', 'cellspacing', 'cellpadding',
                                        'align', 'bgcolor',
                                ] ),
 
            # 11.2.2
            'caption'    => $block,
 
            # 11.2.3
            'thead'      => $common,
            'tfoot'      => $common,
            'tbody'      => $common,
 
            # 11.2.4
            'colgroup'   => $merge( $common, [ 'span' ] ),
            'col'        => $merge( $common, [ 'span' ] ),
 
            # 11.2.5
            'tr'         => $merge( $common, [ 'bgcolor' ], $tablealign ),
 
            # 11.2.6
            'td'         => $merge( $common, $tablecell, $tablealign ),
            'th'         => $merge( $common, $tablecell, $tablealign ),
 
            # 12.2
            # NOTE: <a> is not allowed directly, but this list of allowed
            # attributes is used from the Parser object
            'a'          => $merge( $common, [ 'href', 'rel', 'rev' ] ), # rel/rev esp. for RDFa
 
            # 13.2
            # Not usually allowed, but may be used for extension-style hooks
            # such as <math> when it is rasterized, or if $wgAllowImageTag is
            # true
            'img'        => $merge( $common, [ 'alt', 'src', 'width', 'height', 'srcset' ] ),
            # Attributes for A/V tags added in T163583 / T133673
            'audio'      => $merge( $common, [ 'controls', 'preload', 'width', 'height' ] ),
            'video'      => $merge( $common, [ 'poster', 'controls', 'preload', 'width', 'height' ] ),
            'source'     => $merge( $common, [ 'type', 'src' ] ),
            'track'      => $merge( $common, [ 'type', 'src', 'srclang', 'kind', 'label' ] ),
 
            # 15.2.1
            'tt'         => $common,
            'b'          => $common,
            'i'          => $common,
            'big'        => $common,
            'small'      => $common,
            'strike'     => $common,
            's'          => $common,
            'u'          => $common,
 
            # 15.2.2
            'font'       => $merge( $common, [ 'size', 'color', 'face' ] ),
            # basefont
 
            # 15.3
            'hr'         => $merge( $common, [ 'width' ] ),
 
            # HTML Ruby annotation text module, simple ruby only.
            # https://www.w3.org/TR/html5/text-level-semantics.html#the-ruby-element
            'ruby'       => $common,
            # rbc
            'rb'         => $common,
            'rp'         => $common,
            'rt'         => $common, # $merge( $common, [ 'rbspan' ] ),
            'rtc'        => $common,
 
            # MathML root element, where used for extensions
            # 'title' may not be 100% valid here; it's XHTML
            # https://www.w3.org/TR/REC-MathML/
            'math'       => $merge( [], [ 'class', 'style', 'id', 'title' ] ),
 
            // HTML 5 section 4.5
            'figure'     => $common,
            'figcaption' => $common,
 
            # HTML 5 section 4.6
            'bdi' => $common,
 
            # HTML5 elements, defined by:
            # https://html.spec.whatwg.org/multipage/semantics.html#the-data-element
            'data' => $merge( $common, [ 'value' ] ),
            'time' => $merge( $common, [ 'datetime' ] ),
            'mark' => $common,
 
            // meta and link are only permitted by removeHTMLtags when Microdata
            // is enabled so we don't bother adding a conditional to hide these
            // Also meta and link are only valid in WikiText as Microdata elements
            // (ie: validateTag rejects tags missing the attributes needed for Microdata)
            // So we don't bother including $common attributes that have no purpose.
            'meta' => $merge( [], [ 'itemprop', 'content' ] ),
            'link' => $merge( [], [ 'itemprop', 'href', 'title' ] ),
 
            # HTML 5 section 4.3.5
            'aside' => $common,
        ];
 
        return $allowed;
    }
 
    public static function stripAllTags( $html ) {
        // Use RemexHtml to tokenize $html and extract the text
        $handler = new RemexStripTagHandler;
        $tokenizer = new RemexTokenizer( $handler, $html, [
            'ignoreErrors' => true,
            // don't ignore char refs, we want them to be decoded
            'ignoreNulls' => true,
            'skipPreprocess' => true,
        ] );
        $tokenizer->execute();
        $text = $handler->getResult();
 
        $text = self::normalizeWhitespace( $text );
        return $text;
    }
 
    public static function hackDocType() {
        $out = "<!DOCTYPE html [\n";
        foreach ( HTMLData::$namedEntityTranslations as $entity => $translation ) {
            if ( substr( $entity, -1 ) !== ';' ) {
                // Some HTML entities omit the trailing semicolon;
                // wikitext does not permit these.
                continue;
            }
            $name = substr( $entity, 0, -1 );
            $expansion = self::normalizeEntity( $entity );
            if ( $entity === $expansion ) {
                // Skip &lt; &gt; etc
                continue;
            }
            $out .= "<!ENTITY $name \"$expansion\">";
        }
        $out .= "]>\n";
        return $out;
    }
 
    public static function cleanUrl( $url ) {
        # Normalize any HTML entities in input. They will be
        # re-escaped by makeExternalLink().
        $url = self::decodeCharReferences( $url );
 
        # Escape any control characters introduced by the above step
        $url = preg_replace_callback( '/[\][<>"\\x00-\\x20\\x7F\|]/',
            [ __CLASS__, 'cleanUrlCallback' ], $url );
 
        # Validate hostname portion
        $matches = [];
        if ( preg_match( '!^([^:]+:)(//[^/]+)?(.*)$!iD', $url, $matches ) ) {
            list( /* $whole */, $protocol, $host, $rest ) = $matches;
 
            // Characters that will be ignored in IDNs.
            // https://datatracker.ietf.org/doc/html/rfc8264#section-9.13
            // https://www.unicode.org/Public/UCD/latest/ucd/DerivedCoreProperties.txt
            // Strip them before further processing so deny lists and such work.
            $strip = "/
                \\s|      # general whitespace
                \u{00AD}|               # SOFT HYPHEN
                \u{034F}|               # COMBINING GRAPHEME JOINER
                \u{061C}|               # ARABIC LETTER MARK
                [\u{115F}-\u{1160}]|    # HANGUL CHOSEONG FILLER..
                            # HANGUL JUNGSEONG FILLER
                [\u{17B4}-\u{17B5}]|    # KHMER VOWEL INHERENT AQ..
                            # KHMER VOWEL INHERENT AA
                [\u{180B}-\u{180D}]|    # MONGOLIAN FREE VARIATION SELECTOR ONE..
                            # MONGOLIAN FREE VARIATION SELECTOR THREE
                \u{180E}|               # MONGOLIAN VOWEL SEPARATOR
                [\u{200B}-\u{200F}]|    # ZERO WIDTH SPACE..
                            # RIGHT-TO-LEFT MARK
                [\u{202A}-\u{202E}]|    # LEFT-TO-RIGHT EMBEDDING..
                            # RIGHT-TO-LEFT OVERRIDE
                [\u{2060}-\u{2064}]|    # WORD JOINER..
                            # INVISIBLE PLUS
                \u{2065}|               # <reserved-2065>
                [\u{2066}-\u{206F}]|    # LEFT-TO-RIGHT ISOLATE..
                            # NOMINAL DIGIT SHAPES
                \u{3164}|               # HANGUL FILLER
                [\u{FE00}-\u{FE0F}]|    # VARIATION SELECTOR-1..
                            # VARIATION SELECTOR-16
                \u{FEFF}|               # ZERO WIDTH NO-BREAK SPACE
                \u{FFA0}|               # HALFWIDTH HANGUL FILLER
                [\u{FFF0}-\u{FFF8}]|    # <reserved-FFF0>..
                            # <reserved-FFF8>
                [\u{1BCA0}-\u{1BCA3}]|  # SHORTHAND FORMAT LETTER OVERLAP..
                            # SHORTHAND FORMAT UP STEP
                [\u{1D173}-\u{1D17A}]|  # MUSICAL SYMBOL BEGIN BEAM..
                            # MUSICAL SYMBOL END PHRASE
                \u{E0000}|              # <reserved-E0000>
                \u{E0001}|              # LANGUAGE TAG
                [\u{E0002}-\u{E001F}]|  # <reserved-E0002>..
                            # <reserved-E001F>
                [\u{E0020}-\u{E007F}]|  # TAG SPACE..
                            # CANCEL TAG
                [\u{E0080}-\u{E00FF}]|  # <reserved-E0080>..
                            # <reserved-E00FF>
                [\u{E0100}-\u{E01EF}]|  # VARIATION SELECTOR-17..
                            # VARIATION SELECTOR-256
                [\u{E01F0}-\u{E0FFF}]|  # <reserved-E01F0>..
                            # <reserved-E0FFF>
                /xuD";
 
            $host = preg_replace( $strip, '', $host );
 
            // IPv6 host names are bracketed with [].  Url-decode these.
            if ( substr_compare( "//%5B", $host, 0, 5 ) === 0 &&
                preg_match( '!^//%5B([0-9A-Fa-f:.]+)%5D((:\d+)?)$!', $host, $matches )
            ) {
                $host = '//[' . $matches[1] . ']' . $matches[2];
            }
 
            // @todo FIXME: Validate hostnames here
 
            return $protocol . $host . $rest;
        } else {
            return $url;
        }
    }
 
    private static function cleanUrlCallback( $matches ) {
        return urlencode( $matches[0] );
    }
 
    public static function validateEmail( $addr ) {
        $result = null;
        // TODO This method should be non-static, and have a HookRunner injected
        if ( !Hooks::runner()->onIsValidEmailAddr( $addr, $result ) ) {
            return $result;
        }
 
        // Please note strings below are enclosed in brackets [], this make the
        // hyphen "-" a range indicator. Hence it is double backslashed below.
        // See T28948
        $rfc5322_atext = "a-z0-9!#$%&'*+\\-\/=?^_`{|}~";
        $rfc1034_ldh_str = "a-z0-9\\-";
 
        $html5_email_regexp = "/
        ^                      # start of string
        [$rfc5322_atext\\.]+    # user part which is liberal :p
        @                      # 'apostrophe'
        [$rfc1034_ldh_str]+       # First domain part
        (\\.[$rfc1034_ldh_str]+)*  # Following part prefixed with a dot
        $                      # End of string
        /ix"; // case Insensitive, eXtended
 
        return (bool)preg_match( $html5_email_regexp, $addr );
    }
}