Sanitizer.php

Go to the documentation of this file.

<?php
use MediaWiki\MediaWikiServices;

class Sanitizer {
    const CHAR_REFS_REGEX =
        '/&([A-Za-z0-9\x80-\xff]+);
         |&\#([0-9]+);
         |&\#[xX]([0-9A-Fa-f]+);
         |(&)/x';

    const ELEMENT_BITS_REGEX = '!^(/?)([A-Za-z][^\t\n\v />\0]*+)([^>]*?)(/?>)([^<]*)$!';

    const EVIL_URI_PATTERN = '!(^|\s|\*/\s*)(javascript|vbscript)([^\w]|$)!i';
    const XMLNS_ATTRIBUTE_PATTERN = "/^xmlns:[:A-Z_a-z-.0-9]+$/";

    const ID_PRIMARY = 0;

    const ID_FALLBACK = 1;

    private const HTML_ENTITIES = [
        'Aacute'   => 193,
        'aacute'   => 225,
        'Acirc'    => 194,
        'acirc'    => 226,
        'acute'    => 180,
        'AElig'    => 198,
        'aelig'    => 230,
        'Agrave'   => 192,
        'agrave'   => 224,
        'alefsym'  => 8501,
        'Alpha'    => 913,
        'alpha'    => 945,
        'amp'      => 38,
        'and'      => 8743,
        'ang'      => 8736,
        'apos'     => 39, // New in XHTML & HTML 5; avoid in output for compatibility with IE.
        'Aring'    => 197,
        'aring'    => 229,
        'asymp'    => 8776,
        'Atilde'   => 195,
        'atilde'   => 227,
        'Auml'     => 196,
        'auml'     => 228,
        'bdquo'    => 8222,
        'Beta'     => 914,
        'beta'     => 946,
        'brvbar'   => 166,
        'bull'     => 8226,
        'cap'      => 8745,
        'Ccedil'   => 199,
        'ccedil'   => 231,
        'cedil'    => 184,
        'cent'     => 162,
        'Chi'      => 935,
        'chi'      => 967,
        'circ'     => 710,
        'clubs'    => 9827,
        'cong'     => 8773,
        'copy'     => 169,
        'crarr'    => 8629,
        'cup'      => 8746,
        'curren'   => 164,
        'dagger'   => 8224,
        'Dagger'   => 8225,
        'darr'     => 8595,
        'dArr'     => 8659,
        'deg'      => 176,
        'Delta'    => 916,
        'delta'    => 948,
        'diams'    => 9830,
        'divide'   => 247,
        'Eacute'   => 201,
        'eacute'   => 233,
        'Ecirc'    => 202,
        'ecirc'    => 234,
        'Egrave'   => 200,
        'egrave'   => 232,
        'empty'    => 8709,
        'emsp'     => 8195,
        'ensp'     => 8194,
        'Epsilon'  => 917,
        'epsilon'  => 949,
        'equiv'    => 8801,
        'Eta'      => 919,
        'eta'      => 951,
        'ETH'      => 208,
        'eth'      => 240,
        'Euml'     => 203,
        'euml'     => 235,
        'euro'     => 8364,
        'exist'    => 8707,
        'fnof'     => 402,
        'forall'   => 8704,
        'frac12'   => 189,
        'frac14'   => 188,
        'frac34'   => 190,
        'frasl'    => 8260,
        'Gamma'    => 915,
        'gamma'    => 947,
        'ge'       => 8805,
        'gt'       => 62,
        'harr'     => 8596,
        'hArr'     => 8660,
        'hearts'   => 9829,
        'hellip'   => 8230,
        'Iacute'   => 205,
        'iacute'   => 237,
        'Icirc'    => 206,
        'icirc'    => 238,
        'iexcl'    => 161,
        'Igrave'   => 204,
        'igrave'   => 236,
        'image'    => 8465,
        'infin'    => 8734,
        'int'      => 8747,
        'Iota'     => 921,
        'iota'     => 953,
        'iquest'   => 191,
        'isin'     => 8712,
        'Iuml'     => 207,
        'iuml'     => 239,
        'Kappa'    => 922,
        'kappa'    => 954,
        'Lambda'   => 923,
        'lambda'   => 955,
        'lang'     => 9001,
        'laquo'    => 171,
        'larr'     => 8592,
        'lArr'     => 8656,
        'lceil'    => 8968,
        'ldquo'    => 8220,
        'le'       => 8804,
        'lfloor'   => 8970,
        'lowast'   => 8727,
        'loz'      => 9674,
        'lrm'      => 8206,
        'lsaquo'   => 8249,
        'lsquo'    => 8216,
        'lt'       => 60,
        'macr'     => 175,
        'mdash'    => 8212,
        'micro'    => 181,
        'middot'   => 183,
        'minus'    => 8722,
        'Mu'       => 924,
        'mu'       => 956,
        'nabla'    => 8711,
        'nbsp'     => 160,
        'ndash'    => 8211,
        'ne'       => 8800,
        'ni'       => 8715,
        'not'      => 172,
        'notin'    => 8713,
        'nsub'     => 8836,
        'Ntilde'   => 209,
        'ntilde'   => 241,
        'Nu'       => 925,
        'nu'       => 957,
        'Oacute'   => 211,
        'oacute'   => 243,
        'Ocirc'    => 212,
        'ocirc'    => 244,
        'OElig'    => 338,
        'oelig'    => 339,
        'Ograve'   => 210,
        'ograve'   => 242,
        'oline'    => 8254,
        'Omega'    => 937,
        'omega'    => 969,
        'Omicron'  => 927,
        'omicron'  => 959,
        'oplus'    => 8853,
        'or'       => 8744,
        'ordf'     => 170,
        'ordm'     => 186,
        'Oslash'   => 216,
        'oslash'   => 248,
        'Otilde'   => 213,
        'otilde'   => 245,
        'otimes'   => 8855,
        'Ouml'     => 214,
        'ouml'     => 246,
        'para'     => 182,
        'part'     => 8706,
        'permil'   => 8240,
        'perp'     => 8869,
        'Phi'      => 934,
        'phi'      => 966,
        'Pi'       => 928,
        'pi'       => 960,
        'piv'      => 982,
        'plusmn'   => 177,
        'pound'    => 163,
        'prime'    => 8242,
        'Prime'    => 8243,
        'prod'     => 8719,
        'prop'     => 8733,
        'Psi'      => 936,
        'psi'      => 968,
        'quot'     => 34,
        'radic'    => 8730,
        'rang'     => 9002,
        'raquo'    => 187,
        'rarr'     => 8594,
        'rArr'     => 8658,
        'rceil'    => 8969,
        'rdquo'    => 8221,
        'real'     => 8476,
        'reg'      => 174,
        'rfloor'   => 8971,
        'Rho'      => 929,
        'rho'      => 961,
        'rlm'      => 8207,
        'rsaquo'   => 8250,
        'rsquo'    => 8217,
        'sbquo'    => 8218,
        'Scaron'   => 352,
        'scaron'   => 353,
        'sdot'     => 8901,
        'sect'     => 167,
        'shy'      => 173,
        'Sigma'    => 931,
        'sigma'    => 963,
        'sigmaf'   => 962,
        'sim'      => 8764,
        'spades'   => 9824,
        'sub'      => 8834,
        'sube'     => 8838,
        'sum'      => 8721,
        'sup'      => 8835,
        'sup1'     => 185,
        'sup2'     => 178,
        'sup3'     => 179,
        'supe'     => 8839,
        'szlig'    => 223,
        'Tau'      => 932,
        'tau'      => 964,
        'there4'   => 8756,
        'Theta'    => 920,
        'theta'    => 952,
        'thetasym' => 977,
        'thinsp'   => 8201,
        'THORN'    => 222,
        'thorn'    => 254,
        'tilde'    => 732,
        'times'    => 215,
        'trade'    => 8482,
        'Uacute'   => 218,
        'uacute'   => 250,
        'uarr'     => 8593,
        'uArr'     => 8657,
        'Ucirc'    => 219,
        'ucirc'    => 251,
        'Ugrave'   => 217,
        'ugrave'   => 249,
        'uml'      => 168,
        'upsih'    => 978,
        'Upsilon'  => 933,
        'upsilon'  => 965,
        'Uuml'     => 220,
        'uuml'     => 252,
        'weierp'   => 8472,
        'Xi'       => 926,
        'xi'       => 958,
        'Yacute'   => 221,
        'yacute'   => 253,
        'yen'      => 165,
        'Yuml'     => 376,
        'yuml'     => 255,
        'Zeta'     => 918,
        'zeta'     => 950,
        'zwj'      => 8205,
        'zwnj'     => 8204
    ];

    private const HTML_ENTITY_ALIASES = [
        'רלמ' => 'rlm',
        'رلم' => 'rlm',
    ];

    private static $attribsRegex;

    static function getAttribsRegex() {
        if ( self::$attribsRegex === null ) {
            $spaceChars = '\x09\x0a\x0c\x0d\x20';
            $space = "[{$spaceChars}]";
            $attrib = "[^{$spaceChars}\/>=]";
            $attribFirst = "(?:{$attrib}|=)";
            self::$attribsRegex =
                "/({$attribFirst}{$attrib}*)
                    ($space*=$space*
                    (?:
                        # The attribute value: quoted or alone
                        \"([^\"]*)(?:\"|\$)
                        | '([^']*)(?:'|\$)
                        | (((?!$space|>).)*)
                    )
                )?/sxu";
        }
        return self::$attribsRegex;
    }

    private static $attribNameRegex;

    static function getAttribNameRegex() {
        if ( self::$attribNameRegex === null ) {
            $attribFirst = "[:_\p{L}\p{N}]";
            $attrib = "[:_\.\-\p{L}\p{N}]";
            self::$attribNameRegex = "/^({$attribFirst}{$attrib}*)$/sxu";
        }
        return self::$attribNameRegex;
    }

    public static function getRecognizedTagData( $extratags = [], $removetags = [] ) {
        global $wgAllowImageTag;

        static $htmlpairsStatic, $htmlsingle, $htmlsingleonly, $htmlnest, $tabletags,
            $htmllist, $listtags, $htmlsingleallowed, $htmlelementsStatic, $staticInitialised;

        // Base our staticInitialised variable off of the global config state so that if the globals
        // are changed (like in the screwed up test system) we will re-initialise the settings.
        $globalContext = $wgAllowImageTag;
        if ( !$staticInitialised || $staticInitialised != $globalContext ) {
            $htmlpairsStatic = [ # Tags that must be closed
                'b', 'bdi', 'del', 'i', 'ins', 'u', 'font', 'big', 'small', 'sub', 'sup', 'h1',
                'h2', 'h3', 'h4', 'h5', 'h6', 'cite', 'code', 'em', 's',
                'strike', 'strong', 'tt', 'var', 'div', 'center',
                'blockquote', 'ol', 'ul', 'dl', 'table', 'caption', 'pre',
                'ruby', 'rb', 'rp', 'rt', 'rtc', 'p', 'span', 'abbr', 'dfn',
                'kbd', 'samp', 'data', 'time', 'mark'
            ];
            $htmlsingle = [
                'br', 'wbr', 'hr', 'li', 'dt', 'dd', 'meta', 'link'
            ];

            # Elements that cannot have close tags. This is (not coincidentally)
            # also the list of tags for which the HTML 5 parsing algorithm
            # requires you to "acknowledge the token's self-closing flag", i.e.
            # a self-closing tag like <br/> is not an HTML 5 parse error only
            # for this list.
            $htmlsingleonly = [
                'br', 'wbr', 'hr', 'meta', 'link'
            ];

            $htmlnest = [ # Tags that can be nested--??
                'table', 'tr', 'td', 'th', 'div', 'blockquote', 'ol', 'ul',
                'li', 'dl', 'dt', 'dd', 'font', 'big', 'small', 'sub', 'sup', 'span',
                'var', 'kbd', 'samp', 'em', 'strong', 'q', 'ruby', 'bdo'
            ];
            $tabletags = [ # Can only appear inside table, we will close them
                'td', 'th', 'tr',
            ];
            $htmllist = [ # Tags used by list
                'ul', 'ol',
            ];
            $listtags = [ # Tags that can appear in a list
                'li',
            ];

            if ( $wgAllowImageTag ) {
                $htmlsingle[] = 'img';
                $htmlsingleonly[] = 'img';
            }

            $htmlsingleallowed = array_unique( array_merge( $htmlsingle, $tabletags ) );
            $htmlelementsStatic = array_unique( array_merge( $htmlsingle, $htmlpairsStatic, $htmlnest ) );

            # Convert them all to hashtables for faster lookup
            $vars = [ 'htmlpairsStatic', 'htmlsingle', 'htmlsingleonly', 'htmlnest', 'tabletags',
                'htmllist', 'listtags', 'htmlsingleallowed', 'htmlelementsStatic' ];
            foreach ( $vars as $var ) {
                $$var = array_flip( $$var );
            }
            $staticInitialised = $globalContext;
        }

        # Populate $htmlpairs and $htmlelements with the $extratags and $removetags arrays
        $extratags = array_flip( $extratags );
        $removetags = array_flip( $removetags );
        $htmlpairs = array_merge( $extratags, $htmlpairsStatic );
        $htmlelements = array_diff_key( array_merge( $extratags, $htmlelementsStatic ), $removetags );

        return [
            'htmlpairs' => $htmlpairs,
            'htmlsingle' => $htmlsingle,
            'htmlsingleonly' => $htmlsingleonly,
            'htmlnest' => $htmlnest,
            'tabletags' => $tabletags,
            'htmllist' => $htmllist,
            'listtags' => $listtags,
            'htmlsingleallowed' => $htmlsingleallowed,
            'htmlelements' => $htmlelements,
        ];
    }

    public static function removeHTMLtags( $text, $processCallback = null,
        $args = [], $extratags = [], $removetags = [], $warnCallback = null
    ) {
        $tagData = self::getRecognizedTagData( $extratags, $removetags );
        $htmlpairs = $tagData['htmlpairs'];
        $htmlsingle = $tagData['htmlsingle'];
        $htmlsingleonly = $tagData['htmlsingleonly'];
        $htmlnest = $tagData['htmlnest'];
        $tabletags = $tagData['tabletags'];
        $htmllist = $tagData['htmllist'];
        $listtags = $tagData['listtags'];
        $htmlsingleallowed = $tagData['htmlsingleallowed'];
        $htmlelements = $tagData['htmlelements'];

        # Remove HTML comments
        $text = self::removeHTMLcomments( $text );
        $bits = explode( '<', $text );
        $text = str_replace( '>', '&gt;', array_shift( $bits ) );
        if ( !MWTidy::isEnabled() ) {
            wfDeprecated( 'disabling tidy', '1.33' );
            $tagstack = $tablestack = [];
            foreach ( $bits as $x ) {
                $regs = [];
                # $slash: Does the current element start with a '/'?
                # $t: Current element name
                # $params: String between element name and >
                # $brace: Ending '>' or '/>'
                # $rest: Everything until the next element of $bits
                if ( preg_match( self::ELEMENT_BITS_REGEX, $x, $regs ) ) {
                    list( /* $qbar */, $slash, $t, $params, $brace, $rest ) = $regs;
                } else {
                    $slash = $t = $params = $brace = $rest = null;
                }

                $badtag = false;
                $t = strtolower( $t );
                if ( isset( $htmlelements[$t] ) ) {
                    # Check our stack
                    if ( $slash && isset( $htmlsingleonly[$t] ) ) {
                        $badtag = true;
                    } elseif ( $slash ) {
                        # Closing a tag... is it the one we just opened?
                        Wikimedia\suppressWarnings();
                        $ot = array_pop( $tagstack );
                        Wikimedia\restoreWarnings();

                        if ( $ot != $t ) {
                            if ( isset( $htmlsingleallowed[$ot] ) ) {
                                # Pop all elements with an optional close tag
                                # and see if we find a match below them
                                $optstack = [];
                                array_push( $optstack, $ot );
                                Wikimedia\suppressWarnings();
                                $ot = array_pop( $tagstack );
                                Wikimedia\restoreWarnings();
                                while ( $ot != $t && isset( $htmlsingleallowed[$ot] ) ) {
                                    array_push( $optstack, $ot );
                                    Wikimedia\suppressWarnings();
                                    $ot = array_pop( $tagstack );
                                    Wikimedia\restoreWarnings();
                                }
                                if ( $t != $ot ) {
                                    # No match. Push the optional elements back again
                                    $badtag = true;
                                    Wikimedia\suppressWarnings();
                                    $ot = array_pop( $optstack );
                                    Wikimedia\restoreWarnings();
                                    while ( $ot ) {
                                        array_push( $tagstack, $ot );
                                        Wikimedia\suppressWarnings();
                                        $ot = array_pop( $optstack );
                                        Wikimedia\restoreWarnings();
                                    }
                                }
                            } else {
                                Wikimedia\suppressWarnings();
                                array_push( $tagstack, $ot );
                                Wikimedia\restoreWarnings();

                                # <li> can be nested in <ul> or <ol>, skip those cases:
                                if ( !isset( $htmllist[$ot] ) || !isset( $listtags[$t] ) ) {
                                    $badtag = true;
                                }
                            }
                        } elseif ( $t == 'table' ) {
                            $tagstack = array_pop( $tablestack );
                        }
                        $newparams = '';
                    } else {
                        # Keep track for later
                        if ( isset( $tabletags[$t] ) && !in_array( 'table', $tagstack ) ) {
                            $badtag = true;
                        } elseif ( in_array( $t, $tagstack ) && !isset( $htmlnest[$t] ) ) {
                            $badtag = true;
                        #  Is it a self closed htmlpair ? (T7487)
                        } elseif ( $brace == '/>' && isset( $htmlpairs[$t] ) ) {
                            // Eventually we'll just remove the self-closing
                            // slash, in order to be consistent with HTML5
                            // semantics.
                            // $brace = '>';
                            // For now, let's just warn authors to clean up.
                            if ( is_callable( $warnCallback ) ) {
                                call_user_func_array( $warnCallback, [ 'deprecated-self-close-category' ] );
                            }
                            $badtag = true;
                        } elseif ( isset( $htmlsingleonly[$t] ) ) {
                            # Hack to force empty tag for unclosable elements
                            $brace = '/>';
                        } elseif ( isset( $htmlsingle[$t] ) ) {
                            # Hack to not close $htmlsingle tags
                            $brace = null;
                            # Still need to push this optionally-closed tag to
                            # the tag stack so that we can match end tags
                            # instead of marking them as bad.
                            array_push( $tagstack, $t );
                        } elseif ( isset( $tabletags[$t] ) && in_array( $t, $tagstack ) ) {
                            // New table tag but forgot to close the previous one
                            $text .= "</$t>";
                        } else {
                            if ( $t == 'table' ) {
                                array_push( $tablestack, $tagstack );
                                $tagstack = [];
                            }
                            array_push( $tagstack, $t );
                        }

                        # Replace any variables or template parameters with
                        # plaintext results.
                        if ( is_callable( $processCallback ) ) {
                            call_user_func_array( $processCallback, [ &$params, $args ] );
                        }

                        if ( !self::validateTag( $params, $t ) ) {
                            $badtag = true;
                        }

                        # Strip non-approved attributes from the tag
                        $newparams = self::fixTagAttributes( $params, $t );
                    }
                    if ( !$badtag ) {
                        $rest = str_replace( '>', '&gt;', $rest );
                        $close = ( $brace == '/>' && !$slash ) ? ' /' : '';
                        $text .= "<$slash$t$newparams$close>$rest";
                        continue;
                    }
                }
                $text .= '&lt;' . str_replace( '>', '&gt;', $x );
            }
            # Close off any remaining tags
            while ( is_array( $tagstack ) && ( $t = array_pop( $tagstack ) ) ) {
                $text .= "</$t>\n";
                if ( $t == 'table' ) {
                    $tagstack = array_pop( $tablestack );
                }
            }
        } else {
            # this might be possible using tidy itself
            foreach ( $bits as $x ) {
                if ( preg_match( self::ELEMENT_BITS_REGEX, $x, $regs ) ) {
                    list( /* $qbar */, $slash, $t, $params, $brace, $rest ) = $regs;

                    $badtag = false;
                    $t = strtolower( $t );
                    if ( isset( $htmlelements[$t] ) ) {
                        if ( is_callable( $processCallback ) ) {
                            call_user_func_array( $processCallback, [ &$params, $args ] );
                        }

                        if ( $brace == '/>' && !( isset( $htmlsingle[$t] ) || isset( $htmlsingleonly[$t] ) ) ) {
                            // Eventually we'll just remove the self-closing
                            // slash, in order to be consistent with HTML5
                            // semantics.
                            // $brace = '>';
                            // For now, let's just warn authors to clean up.
                            if ( is_callable( $warnCallback ) ) {
                                call_user_func_array( $warnCallback, [ 'deprecated-self-close-category' ] );
                            }
                        }
                        if ( !self::validateTag( $params, $t ) ) {
                            $badtag = true;
                        }

                        $newparams = self::fixTagAttributes( $params, $t );
                        if ( !$badtag ) {
                            if ( $brace === '/>' && !isset( $htmlsingleonly[$t] ) ) {
                                # Interpret self-closing tags as empty tags even when
                                # HTML 5 would interpret them as start tags. Such input
                                # is commonly seen on Wikimedia wikis with this intention.
                                $brace = "></$t>";
                            }

                            $rest = str_replace( '>', '&gt;', $rest );
                            $text .= "<$slash$t$newparams$brace$rest";
                            continue;
                        }
                    }
                }
                $text .= '&lt;' . str_replace( '>', '&gt;', $x );
            }
        }
        return $text;
    }

    public static function removeHTMLcomments( $text ) {
        while ( ( $start = strpos( $text, '<!--' ) ) !== false ) {
            $end = strpos( $text, '-->', $start + 4 );
            if ( $end === false ) {
                # Unterminated comment; bail out
                break;
            }

            $end += 3;

            # Trim space and newline if the comment is both
            # preceded and followed by a newline
            $spaceStart = max( $start - 1, 0 );
            $spaceLen = $end - $spaceStart;
            while ( substr( $text, $spaceStart, 1 ) === ' ' && $spaceStart > 0 ) {
                $spaceStart--;
                $spaceLen++;
            }
            while ( substr( $text, $spaceStart + $spaceLen, 1 ) === ' ' ) {
                $spaceLen++;
            }
            if ( substr( $text, $spaceStart, 1 ) === "\n"
                && substr( $text, $spaceStart + $spaceLen, 1 ) === "\n" ) {
                # Remove the comment, leading and trailing
                # spaces, and leave only one newline.
                $text = substr_replace( $text, "\n", $spaceStart, $spaceLen + 1 );
            } else {
                # Remove just the comment.
                $text = substr_replace( $text, '', $start, $end - $start );
            }
        }
        return $text;
    }

    static function validateTag( $params, $element ) {
        $params = self::decodeTagAttributes( $params );

        if ( $element == 'meta' || $element == 'link' ) {
            if ( !isset( $params['itemprop'] ) ) {
                // <meta> and <link> must have an itemprop="" otherwise they are not valid or safe in content
                return false;
            }
            if ( $element == 'meta' && !isset( $params['content'] ) ) {
                // <meta> must have a content="" for the itemprop
                return false;
            }
            if ( $element == 'link' && !isset( $params['href'] ) ) {
                // <link> must have an associated href=""
                return false;
            }
        }

        return true;
    }

    static function validateTagAttributes( $attribs, $element ) {
        return self::validateAttributes( $attribs,
            self::attributeWhitelistInternal( $element ) );
    }

    static function validateAttributes( $attribs, $whitelist ) {
        if ( isset( $whitelist[0] ) ) {
            // We would like to eventually deprecate calling this
            // function with a sequential array, but for now just
            // convert it.
            $whitelist = array_flip( $whitelist );
        }
        $hrefExp = '/^(' . wfUrlProtocols() . ')[^\s]+$/';

        $out = [];
        foreach ( $attribs as $attribute => $value ) {
            # Allow XML namespace declaration to allow RDFa
            if ( preg_match( self::XMLNS_ATTRIBUTE_PATTERN, $attribute ) ) {
                if ( !preg_match( self::EVIL_URI_PATTERN, $value ) ) {
                    $out[$attribute] = $value;
                }

                continue;
            }

            # Allow any attribute beginning with "data-"
            # However:
            # * Disallow data attributes used by MediaWiki code
            # * Ensure that the attribute is not namespaced by banning
            #   colons.
            if ( (
                !preg_match( '/^data-[^:]*$/i', $attribute ) &&
                !array_key_exists( $attribute, $whitelist )
            ) || self::isReservedDataAttribute( $attribute ) ) {
                continue;
            }

            # Strip javascript "expression" from stylesheets.
            # https://msdn.microsoft.com/en-us/library/ms537634.aspx
            if ( $attribute == 'style' ) {
                $value = self::checkCss( $value );
            }

            # Escape HTML id attributes
            if ( $attribute === 'id' ) {
                $value = self::escapeIdForAttribute( $value, self::ID_PRIMARY );
            }

            # Escape HTML id reference lists
            if ( $attribute === 'aria-describedby'
                || $attribute === 'aria-flowto'
                || $attribute === 'aria-labelledby'
                || $attribute === 'aria-owns'
            ) {
                $value = self::escapeIdReferenceList( $value );
            }

            // RDFa and microdata properties allow URLs, URIs and/or CURIs.
            // Check them for sanity.
            if ( $attribute === 'rel' || $attribute === 'rev'
                # RDFa
                || $attribute === 'about' || $attribute === 'property'
                || $attribute === 'resource' || $attribute === 'datatype'
                || $attribute === 'typeof'
                # HTML5 microdata
                || $attribute === 'itemid' || $attribute === 'itemprop'
                || $attribute === 'itemref' || $attribute === 'itemscope'
                || $attribute === 'itemtype'
            ) {
                // Paranoia. Allow "simple" values but suppress javascript
                if ( preg_match( self::EVIL_URI_PATTERN, $value ) ) {
                    continue;
                }
            }

            # NOTE: even though elements using href/src are not allowed directly, supply
            #       validation code that can be used by tag hook handlers, etc
            if ( $attribute === 'href' || $attribute === 'src' || $attribute === 'poster' ) {
                if ( !preg_match( $hrefExp, $value ) ) {
                    continue; // drop any href or src attributes not using an allowed protocol.
                    // NOTE: this also drops all relative URLs
                }
            }

            // If this attribute was previously set, override it.
            // Output should only have one attribute of each name.
            $out[$attribute] = $value;
        }

        # itemtype, itemid, itemref don't make sense without itemscope
        if ( !array_key_exists( 'itemscope', $out ) ) {
            unset( $out['itemtype'] );
            unset( $out['itemid'] );
            unset( $out['itemref'] );
        }
        # TODO: Strip itemprop if we aren't descendants of an itemscope or pointed to by an itemref.

        return $out;
    }

    public static function isReservedDataAttribute( $attr ) {
        // data-ooui is reserved for ooui.
        // data-mw and data-parsoid are reserved for parsoid.
        // data-mw-<name here> is reserved for extensions (or core) if
        // they need to communicate some data to the client and want to be
        // sure that it isn't coming from an untrusted user.
        // We ignore the possibility of namespaces since user-generated HTML
        // can't use them anymore.
        return (bool)preg_match( '/^data-(ooui|mw|parsoid)/i', $attr );
    }

    static function mergeAttributes( $a, $b ) {
        $out = array_merge( $a, $b );
        if ( isset( $a['class'] ) && isset( $b['class'] )
            && is_string( $a['class'] ) && is_string( $b['class'] )
            && $a['class'] !== $b['class']
        ) {
            $classes = preg_split( '/\s+/', "{$a['class']} {$b['class']}",
                -1, PREG_SPLIT_NO_EMPTY );
            $out['class'] = implode( ' ', array_unique( $classes ) );
        }
        return $out;
    }

    public static function normalizeCss( $value ) {
        // Decode character references like &#123;
        $value = self::decodeCharReferences( $value );

        // Decode escape sequences and line continuation
        // See the grammar in the CSS 2 spec, appendix D.
        // This has to be done AFTER decoding character references.
        // This means it isn't possible for this function to return
        // unsanitized escape sequences. It is possible to manufacture
        // input that contains character references that decode to
        // escape sequences that decode to character references, but
        // it's OK for the return value to contain character references
        // because the caller is supposed to escape those anyway.
        static $decodeRegex;
        if ( !$decodeRegex ) {
            $space = '[\\x20\\t\\r\\n\\f]';
            $nl = '(?:\\n|\\r\\n|\\r|\\f)';
            $backslash = '\\\\';
            $decodeRegex = "/ $backslash
                (?:
                    ($nl) |  # 1. Line continuation
                    ([0-9A-Fa-f]{1,6})$space? |  # 2. character number
                    (.) | # 3. backslash cancelling special meaning
                    () | # 4. backslash at end of string
                )/xu";
        }
        $value = preg_replace_callback( $decodeRegex,
            [ __CLASS__, 'cssDecodeCallback' ], $value );

        // Let the value through if it's nothing but a single comment, to
        // allow other functions which may reject it to pass some error
        // message through.
        if ( !preg_match( '! ^ \s* /\* [^*\\/]* \*/ \s* $ !x', $value ) ) {
            // Remove any comments; IE gets token splitting wrong
            // This must be done AFTER decoding character references and
            // escape sequences, because those steps can introduce comments
            // This step cannot introduce character references or escape
            // sequences, because it replaces comments with spaces rather
            // than removing them completely.
            $value = StringUtils::delimiterReplace( '/*', '*/', ' ', $value );

            // Remove anything after a comment-start token, to guard against
            // incorrect client implementations.
            $commentPos = strpos( $value, '/*' );
            if ( $commentPos !== false ) {
                $value = substr( $value, 0, $commentPos );
            }
        }

        return $value;
    }

    static function checkCss( $value ) {
        $value = self::normalizeCss( $value );

        // Reject problematic keywords and control characters
        if ( preg_match( '/[\000-\010\013\016-\037\177]/', $value ) ||
            strpos( $value, UtfNormal\Constants::UTF8_REPLACEMENT ) !== false ) {
            return '/* invalid control char */';
        } elseif ( preg_match(
            '! expression
                | filter\s*:
                | accelerator\s*:
                | -o-link\s*:
                | -o-link-source\s*:
                | -o-replace\s*:
                | url\s*\(
                | image\s*\(
                | image-set\s*\(
                | attr\s*\([^)]+[\s,]+url
                | var\s*\(
            !ix', $value ) ) {
            return '/* insecure input */';
        }
        return $value;
    }

    static function cssDecodeCallback( $matches ) {
        if ( $matches[1] !== '' ) {
            // Line continuation
            return '';
        } elseif ( $matches[2] !== '' ) {
            $char = UtfNormal\Utils::codepointToUtf8( hexdec( $matches[2] ) );
        } elseif ( $matches[3] !== '' ) {
            $char = $matches[3];
        } else {
            $char = '\\';
        }
        if ( $char == "\n" || $char == '"' || $char == "'" || $char == '\\' ) {
            // These characters need to be escaped in strings
            // Clean up the escape sequence to avoid parsing errors by clients
            return '\\' . dechex( ord( $char ) ) . ' ';
        } else {
            // Decode unnecessary escape
            return $char;
        }
    }

    static function fixTagAttributes( $text, $element, $sorted = false ) {
        if ( trim( $text ) == '' ) {
            return '';
        }

        $decoded = self::decodeTagAttributes( $text );
        $stripped = self::validateTagAttributes( $decoded, $element );

        if ( $sorted ) {
            ksort( $stripped );
        }

        return self::safeEncodeTagAttributes( $stripped );
    }

    static function encodeAttribute( $text ) {
        $encValue = htmlspecialchars( $text, ENT_QUOTES );

        // Whitespace is normalized during attribute decoding,
        // so if we've been passed non-spaces we must encode them
        // ahead of time or they won't be preserved.
        $encValue = strtr( $encValue, [
            "\n" => '&#10;',
            "\r" => '&#13;',
            "\t" => '&#9;',
        ] );

        return $encValue;
    }

    public static function armorFrenchSpaces( $text, $space = '&#160;' ) {
        // Replace $ with \$ and \ with \\
        $space = preg_replace( '#(?<!\\\\)(\\$|\\\\)#', '\\\\$1', $space );
        $fixtags = [
            # French spaces, last one Guillemet-left
            # only if there is something before the space
            # and a non-word character after the punctuation.
            '/(?<=\S) (?=[?:;!%»›](?!\w))/u' => "$space",
            # French spaces, Guillemet-right
            '/([«‹]) /u' => "\\1$space",
        ];
        return preg_replace( array_keys( $fixtags ), array_values( $fixtags ), $text );
    }

    static function safeEncodeAttribute( $text ) {
        $encValue = self::encodeAttribute( $text );

        # Templates and links may be expanded in later parsing,
        # creating invalid or dangerous output. Suppress this.
        $encValue = strtr( $encValue, [
            '<'    => '&lt;',   // This should never happen,
            '>'    => '&gt;',   // we've received invalid input
            '"'    => '&quot;', // which should have been escaped.
            '{'    => '&#123;',
            '}'    => '&#125;', // prevent unpaired language conversion syntax
            '['    => '&#91;',
            ']'    => '&#93;',
            "''"   => '&#39;&#39;',
            'ISBN' => '&#73;SBN',
            'RFC'  => '&#82;FC',
            'PMID' => '&#80;MID',
            '|'    => '&#124;',
            '__'   => '&#95;_',
        ] );

        # Armor against French spaces detection (T5158)
        $encValue = self::armorFrenchSpaces( $encValue, '&#32;' );

        # Stupid hack
        $encValue = preg_replace_callback(
            '/((?i)' . wfUrlProtocols() . ')/',
            function ( $matches ) {
                return str_replace( ':', '&#58;', $matches[1] );
            },
            $encValue );
        return $encValue;
    }

    static function escapeId( $id, $options = [] ) {
        $options = (array)$options;

        // HTML4-style escaping
        static $replace = [
            '%3A' => ':',
            '%' => '.'
        ];

        $id = urlencode( strtr( $id, ' ', '_' ) );
        $id = strtr( $id, $replace );

        if ( !preg_match( '/^[a-zA-Z]/', $id ) && !in_array( 'noninitial', $options ) ) {
            // Initial character must be a letter!
            $id = "x$id";
        }
        return $id;
    }

    public static function escapeIdForAttribute( $id, $mode = self::ID_PRIMARY ) {
        global $wgFragmentMode;

        if ( !isset( $wgFragmentMode[$mode] ) ) {
            if ( $mode === self::ID_PRIMARY ) {
                throw new UnexpectedValueException( '$wgFragmentMode is configured with no primary mode' );
            }
            return false;
        }

        $internalMode = $wgFragmentMode[$mode];

        return self::escapeIdInternal( $id, $internalMode );
    }

    public static function escapeIdForLink( $id ) {
        global $wgFragmentMode;

        if ( !isset( $wgFragmentMode[self::ID_PRIMARY] ) ) {
            throw new UnexpectedValueException( '$wgFragmentMode is configured with no primary mode' );
        }

        $mode = $wgFragmentMode[self::ID_PRIMARY];

        $id = self::escapeIdInternal( $id, $mode );

        return $id;
    }

    public static function escapeIdForExternalInterwiki( $id ) {
        global $wgExternalInterwikiFragmentMode;

        $id = self::escapeIdInternal( $id, $wgExternalInterwikiFragmentMode );

        return $id;
    }

    private static function escapeIdInternal( $id, $mode ) {
        switch ( $mode ) {
            case 'html5':
                $id = str_replace( ' ', '_', $id );
                break;
            case 'legacy':
                // This corresponds to 'noninitial' mode of the old escapeId()
                static $replace = [
                    '%3A' => ':',
                    '%' => '.'
                ];

                $id = urlencode( str_replace( ' ', '_', $id ) );
                $id = strtr( $id, $replace );
                break;
            default:
                throw new InvalidArgumentException( "Invalid mode '$mode' passed to '" . __METHOD__ );
        }

        return $id;
    }

    public static function escapeIdReferenceList( $referenceString ) {
        # Explode the space delimited list string into an array of tokens
        $references = preg_split( '/\s+/', "{$referenceString}", -1, PREG_SPLIT_NO_EMPTY );

        # Escape each token as an id
        foreach ( $references as &$ref ) {
            $ref = self::escapeIdForAttribute( $ref );
        }

        # Merge the array back to a space delimited list string
        # If the array is empty, the result will be an empty string ('')
        $referenceString = implode( ' ', $references );

        return $referenceString;
    }

    static function escapeClass( $class ) {
        // Convert ugly stuff to underscores and kill underscores in ugly places
        return rtrim( preg_replace(
            [ '/(^[0-9\\-])|[\\x00-\\x20!"#$%&\'()*+,.\\/:;<=>?@[\\]^`{|}~]|\\xC2\\xA0/', '/_+/' ],
            '_',
            $class ), '_' );
    }

    static function escapeHtmlAllowEntities( $html ) {
        $html = self::decodeCharReferences( $html );
        # It seems wise to escape ' as well as ", as a matter of course.  Can't
        # hurt. Use ENT_SUBSTITUTE so that incorrectly truncated multibyte characters
        # don't cause the entire string to disappear.
        $html = htmlspecialchars( $html, ENT_QUOTES | ENT_SUBSTITUTE );
        return $html;
    }

    public static function decodeTagAttributes( $text ) {
        if ( trim( $text ) == '' ) {
            return [];
        }

        $pairs = [];
        if ( !preg_match_all(
            self::getAttribsRegex(),
            $text,
            $pairs,
            PREG_SET_ORDER ) ) {
            return [];
        }

        $attribs = [];
        foreach ( $pairs as $set ) {
            $attribute = strtolower( $set[1] );

            // Filter attribute names with unacceptable characters
            if ( !preg_match( self::getAttribNameRegex(), $attribute ) ) {
                continue;
            }

            $value = self::getTagAttributeCallback( $set );

            // Normalize whitespace
            $value = preg_replace( '/[\t\r\n ]+/', ' ', $value );
            $value = trim( $value );

            // Decode character references
            $attribs[$attribute] = self::decodeCharReferences( $value );
        }
        return $attribs;
    }

    public static function safeEncodeTagAttributes( $assoc_array ) {
        $attribs = [];
        foreach ( $assoc_array as $attribute => $value ) {
            $encAttribute = htmlspecialchars( $attribute );
            $encValue = self::safeEncodeAttribute( $value );

            $attribs[] = "$encAttribute=\"$encValue\"";
        }
        return count( $attribs ) ? ' ' . implode( ' ', $attribs ) : '';
    }

    private static function getTagAttributeCallback( $set ) {
        if ( isset( $set[5] ) ) {
            # No quotes.
            return $set[5];
        } elseif ( isset( $set[4] ) ) {
            # Single-quoted
            return $set[4];
        } elseif ( isset( $set[3] ) ) {
            # Double-quoted
            return $set[3];
        } elseif ( !isset( $set[2] ) ) {
            # In XHTML, attributes must have a value so return an empty string.
            # See "Empty attribute syntax",
            # https://www.w3.org/TR/html5/syntax.html#syntax-attribute-name
            return "";
        } else {
            throw new MWException( "Tag conditions not met. This should never happen and is a bug." );
        }
    }

    private static function normalizeWhitespace( $text ) {
        return trim( preg_replace(
            '/(?:\r\n|[\x20\x0d\x0a\x09])+/',
            ' ',
            $text ) );
    }

    static function normalizeSectionNameWhitespace( $section ) {
        return trim( preg_replace( '/[ _]+/', ' ', $section ) );
    }

    static function normalizeCharReferences( $text ) {
        return preg_replace_callback(
            self::CHAR_REFS_REGEX,
            [ self::class, 'normalizeCharReferencesCallback' ],
            $text );
    }

    static function normalizeCharReferencesCallback( $matches ) {
        $ret = null;
        if ( $matches[1] != '' ) {
            $ret = self::normalizeEntity( $matches[1] );
        } elseif ( $matches[2] != '' ) {
            $ret = self::decCharReference( $matches[2] );
        } elseif ( $matches[3] != '' ) {
            $ret = self::hexCharReference( $matches[3] );
        }
        if ( is_null( $ret ) ) {
            return htmlspecialchars( $matches[0] );
        } else {
            return $ret;
        }
    }

    static function normalizeEntity( $name ) {
        if ( isset( self::HTML_ENTITY_ALIASES[$name] ) ) {
            return '&' . self::HTML_ENTITY_ALIASES[$name] . ';';
        } elseif ( in_array( $name, [ 'lt', 'gt', 'amp', 'quot' ] ) ) {
            return "&$name;";
        } elseif ( isset( self::HTML_ENTITIES[$name] ) ) {
            return '&#' . self::HTML_ENTITIES[$name] . ';';
        } else {
            return "&amp;$name;";
        }
    }

    static function decCharReference( $codepoint ) {
        $point = intval( $codepoint );
        if ( self::validateCodepoint( $point ) ) {
            return sprintf( '&#%d;', $point );
        } else {
            return null;
        }
    }

    static function hexCharReference( $codepoint ) {
        $point = hexdec( $codepoint );
        if ( self::validateCodepoint( $point ) ) {
            return sprintf( '&#x%x;', $point );
        } else {
            return null;
        }
    }

    private static function validateCodepoint( $codepoint ) {
        # U+000C is valid in HTML5 but not allowed in XML.
        # U+000D is valid in XML but not allowed in HTML5.
        # U+007F - U+009F are disallowed in HTML5 (control characters).
        return $codepoint == 0x09
            || $codepoint == 0x0a
            || ( $codepoint >= 0x20 && $codepoint <= 0x7e )
            || ( $codepoint >= 0xa0 && $codepoint <= 0xd7ff )
            || ( $codepoint >= 0xe000 && $codepoint <= 0xfffd )
            || ( $codepoint >= 0x10000 && $codepoint <= 0x10ffff );
    }

    public static function decodeCharReferences( $text ) {
        return preg_replace_callback(
            self::CHAR_REFS_REGEX,
            [ self::class, 'decodeCharReferencesCallback' ],
            $text );
    }

    public static function decodeCharReferencesAndNormalize( $text ) {
        $text = preg_replace_callback(
            self::CHAR_REFS_REGEX,
            [ self::class, 'decodeCharReferencesCallback' ],
            $text,
            -1, //limit
            $count
        );

        if ( $count ) {
            return MediaWikiServices::getInstance()->getContentLanguage()->normalize( $text );
        } else {
            return $text;
        }
    }

    static function decodeCharReferencesCallback( $matches ) {
        if ( $matches[1] != '' ) {
            return self::decodeEntity( $matches[1] );
        } elseif ( $matches[2] != '' ) {
            return self::decodeChar( intval( $matches[2] ) );
        } elseif ( $matches[3] != '' ) {
            return self::decodeChar( hexdec( $matches[3] ) );
        }
        # Last case should be an ampersand by itself
        return $matches[0];
    }

    static function decodeChar( $codepoint ) {
        if ( self::validateCodepoint( $codepoint ) ) {
            return UtfNormal\Utils::codepointToUtf8( $codepoint );
        } else {
            return UtfNormal\Constants::UTF8_REPLACEMENT;
        }
    }

    static function decodeEntity( $name ) {
        if ( isset( self::HTML_ENTITY_ALIASES[$name] ) ) {
            $name = self::HTML_ENTITY_ALIASES[$name];
        }
        if ( isset( self::HTML_ENTITIES[$name] ) ) {
            return UtfNormal\Utils::codepointToUtf8( self::HTML_ENTITIES[$name] );
        } else {
            return "&$name;";
        }
    }

    static function attributeWhitelist( $element ) {
        wfDeprecated( __METHOD__, '1.34' );
        $list = self::setupAttributeWhitelist();
        return $list[$element] ?? [];
    }

    private static function attributeWhitelistInternal( $element ) {
        $list = self::setupAttributeWhitelistInternal();
        return $list[$element] ?? [];
    }

    static function setupAttributeWhitelist() {
        wfDeprecated( __METHOD__, '1.34' );
        $wlist = self::setupAttributeWhitelistInternal();
        // This method is expected to return a sequential array as the
        // value for each HTML element key.
        return array_map( function ( $v ) {
            return array_keys( $v );
        }, $wlist );
    }

    private static function setupAttributeWhitelistInternal() {
        static $whitelist;

        if ( $whitelist !== null ) {
            return $whitelist;
        }

        // For lookup efficiency flip each attributes array so the keys are
        // the valid attributes.
        $merge = function ( $a, $b, $c = [] ) {
            return array_merge( $a, array_flip( $b ), array_flip( $c ) );
        };
        $common = $merge( [], [
            # HTML
            'id',
            'class',
            'style',
            'lang',
            'dir',
            'title',

            # WAI-ARIA
            'aria-describedby',
            'aria-flowto',
            'aria-label',
            'aria-labelledby',
            'aria-owns',
            'role',

            # RDFa
            # These attributes are specified in section 9 of
            # https://www.w3.org/TR/2008/REC-rdfa-syntax-20081014
            'about',
            'property',
            'resource',
            'datatype',
            'typeof',

            # Microdata. These are specified by
            # https://html.spec.whatwg.org/multipage/microdata.html#the-microdata-model
            'itemid',
            'itemprop',
            'itemref',
            'itemscope',
            'itemtype',
        ] );

        $block = $merge( $common, [ 'align' ] );

        $tablealign = [ 'align', 'valign' ];
        $tablecell = [
            'abbr',
            'axis',
            'headers',
            'scope',
            'rowspan',
            'colspan',
            'nowrap', # deprecated
            'width', # deprecated
            'height', # deprecated
            'bgcolor', # deprecated
        ];

        # Numbers refer to sections in HTML 4.01 standard describing the element.
        # See: https://www.w3.org/TR/html4/
        $whitelist = [
            # 7.5.4
            'div'        => $block,
            'center'     => $common, # deprecated
            'span'       => $common,

            # 7.5.5
            'h1'         => $block,
            'h2'         => $block,
            'h3'         => $block,
            'h4'         => $block,
            'h5'         => $block,
            'h6'         => $block,

            # 7.5.6
            # address

            # 8.2.4
            'bdo'        => $common,

            # 9.2.1
            'em'         => $common,
            'strong'     => $common,
            'cite'       => $common,
            'dfn'        => $common,
            'code'       => $common,
            'samp'       => $common,
            'kbd'        => $common,
            'var'        => $common,
            'abbr'       => $common,
            # acronym

            # 9.2.2
            'blockquote' => $merge( $common, [ 'cite' ] ),
            'q'          => $merge( $common, [ 'cite' ] ),

            # 9.2.3
            'sub'        => $common,
            'sup'        => $common,

            # 9.3.1
            'p'          => $block,

            # 9.3.2
            'br'         => $merge( $common, [ 'clear' ] ),

            # https://www.w3.org/TR/html5/text-level-semantics.html#the-wbr-element
            'wbr'        => $common,

            # 9.3.4
            'pre'        => $merge( $common, [ 'width' ] ),

            # 9.4
            'ins'        => $merge( $common, [ 'cite', 'datetime' ] ),
            'del'        => $merge( $common, [ 'cite', 'datetime' ] ),

            # 10.2
            'ul'         => $merge( $common, [ 'type' ] ),
            'ol'         => $merge( $common, [ 'type', 'start', 'reversed' ] ),
            'li'         => $merge( $common, [ 'type', 'value' ] ),

            # 10.3
            'dl'         => $common,
            'dd'         => $common,
            'dt'         => $common,

            # 11.2.1
            'table'      => $merge( $common,
                                [ 'summary', 'width', 'border', 'frame',
                                        'rules', 'cellspacing', 'cellpadding',
                                        'align', 'bgcolor',
                                ] ),

            # 11.2.2
            'caption'    => $block,

            # 11.2.3
            'thead'      => $common,
            'tfoot'      => $common,
            'tbody'      => $common,

            # 11.2.4
            'colgroup'   => $merge( $common, [ 'span' ] ),
            'col'        => $merge( $common, [ 'span' ] ),

            # 11.2.5
            'tr'         => $merge( $common, [ 'bgcolor' ], $tablealign ),

            # 11.2.6
            'td'         => $merge( $common, $tablecell, $tablealign ),
            'th'         => $merge( $common, $tablecell, $tablealign ),

            # 12.2
            # NOTE: <a> is not allowed directly, but the attrib
            # whitelist is used from the Parser object
            'a'          => $merge( $common, [ 'href', 'rel', 'rev' ] ), # rel/rev esp. for RDFa

            # 13.2
            # Not usually allowed, but may be used for extension-style hooks
            # such as <math> when it is rasterized, or if $wgAllowImageTag is
            # true
            'img'        => $merge( $common, [ 'alt', 'src', 'width', 'height', 'srcset' ] ),
            # Attributes for A/V tags added in T163583 / T133673
            'audio'      => $merge( $common, [ 'controls', 'preload', 'width', 'height' ] ),
            'video'      => $merge( $common, [ 'poster', 'controls', 'preload', 'width', 'height' ] ),
            'source'     => $merge( $common, [ 'type', 'src' ] ),
            'track'      => $merge( $common, [ 'type', 'src', 'srclang', 'kind', 'label' ] ),

            # 15.2.1
            'tt'         => $common,
            'b'          => $common,
            'i'          => $common,
            'big'        => $common,
            'small'      => $common,
            'strike'     => $common,
            's'          => $common,
            'u'          => $common,

            # 15.2.2
            'font'       => $merge( $common, [ 'size', 'color', 'face' ] ),
            # basefont

            # 15.3
            'hr'         => $merge( $common, [ 'width' ] ),

            # HTML Ruby annotation text module, simple ruby only.
            # https://www.w3.org/TR/html5/text-level-semantics.html#the-ruby-element
            'ruby'       => $common,
            # rbc
            'rb'         => $common,
            'rp'         => $common,
            'rt'         => $common, # $merge( $common, [ 'rbspan' ] ),
            'rtc'        => $common,

            # MathML root element, where used for extensions
            # 'title' may not be 100% valid here; it's XHTML
            # https://www.w3.org/TR/REC-MathML/
            'math'       => $merge( [], [ 'class', 'style', 'id', 'title' ] ),

            // HTML 5 section 4.5
            'figure'     => $common,
            'figure-inline' => $common, # T118520
            'figcaption' => $common,

            # HTML 5 section 4.6
            'bdi' => $common,

            # HTML5 elements, defined by:
            # https://html.spec.whatwg.org/multipage/semantics.html#the-data-element
            'data' => $merge( $common, [ 'value' ] ),
            'time' => $merge( $common, [ 'datetime' ] ),
            'mark' => $common,

            // meta and link are only permitted by removeHTMLtags when Microdata
            // is enabled so we don't bother adding a conditional to hide these
            // Also meta and link are only valid in WikiText as Microdata elements
            // (ie: validateTag rejects tags missing the attributes needed for Microdata)
            // So we don't bother including $common attributes that have no purpose.
            'meta' => $merge( [], [ 'itemprop', 'content' ] ),
            'link' => $merge( [], [ 'itemprop', 'href', 'title' ] ),
        ];

        return $whitelist;
    }

    static function stripAllTags( $html ) {
        // Use RemexHtml to tokenize $html and extract the text
        $handler = new RemexStripTagHandler;
        $tokenizer = new RemexHtml\Tokenizer\Tokenizer( $handler, $html, [
            'ignoreErrors' => true,
            // don't ignore char refs, we want them to be decoded
            'ignoreNulls' => true,
            'skipPreprocess' => true,
        ] );
        $tokenizer->execute();
        $text = $handler->getResult();

        $text = self::normalizeWhitespace( $text );
        return $text;
    }

    static function hackDocType() {
        $out = "<!DOCTYPE html [\n";
        foreach ( self::HTML_ENTITIES as $entity => $codepoint ) {
            $out .= "<!ENTITY $entity \"&#$codepoint;\">";
        }
        $out .= "]>\n";
        return $out;
    }

    static function cleanUrl( $url ) {
        # Normalize any HTML entities in input. They will be
        # re-escaped by makeExternalLink().
        $url = self::decodeCharReferences( $url );

        # Escape any control characters introduced by the above step
        $url = preg_replace_callback( '/[\][<>"\\x00-\\x20\\x7F\|]/',
            [ __CLASS__, 'cleanUrlCallback' ], $url );

        # Validate hostname portion
        $matches = [];
        if ( preg_match( '!^([^:]+:)(//[^/]+)?(.*)$!iD', $url, $matches ) ) {
            list( /* $whole */, $protocol, $host, $rest ) = $matches;

            // Characters that will be ignored in IDNs.
            // https://tools.ietf.org/html/rfc3454#section-3.1
            // Strip them before further processing so blacklists and such work.
            $strip = "/
                \\s|          # general whitespace
                \xc2\xad|     # 00ad SOFT HYPHEN
                \xe1\xa0\x86| # 1806 MONGOLIAN TODO SOFT HYPHEN
                \xe2\x80\x8b| # 200b ZERO WIDTH SPACE
                \xe2\x81\xa0| # 2060 WORD JOINER
                \xef\xbb\xbf| # feff ZERO WIDTH NO-BREAK SPACE
                \xcd\x8f|     # 034f COMBINING GRAPHEME JOINER
                \xe1\xa0\x8b| # 180b MONGOLIAN FREE VARIATION SELECTOR ONE
                \xe1\xa0\x8c| # 180c MONGOLIAN FREE VARIATION SELECTOR TWO
                \xe1\xa0\x8d| # 180d MONGOLIAN FREE VARIATION SELECTOR THREE
                \xe2\x80\x8c| # 200c ZERO WIDTH NON-JOINER
                \xe2\x80\x8d| # 200d ZERO WIDTH JOINER
                [\xef\xb8\x80-\xef\xb8\x8f] # fe00-fe0f VARIATION SELECTOR-1-16
                /xuD";

            $host = preg_replace( $strip, '', $host );

            // IPv6 host names are bracketed with [].  Url-decode these.
            if ( substr_compare( "//%5B", $host, 0, 5 ) === 0 &&
                preg_match( '!^//%5B([0-9A-Fa-f:.]+)%5D((:\d+)?)$!', $host, $matches )
            ) {
                $host = '//[' . $matches[1] . ']' . $matches[2];
            }

            // @todo FIXME: Validate hostnames here

            return $protocol . $host . $rest;
        } else {
            return $url;
        }
    }

    static function cleanUrlCallback( $matches ) {
        return urlencode( $matches[0] );
    }

    public static function validateEmail( $addr ) {
        $result = null;
        if ( !Hooks::run( 'isValidEmailAddr', [ $addr, &$result ] ) ) {
            return $result;
        }

        // Please note strings below are enclosed in brackets [], this make the
        // hyphen "-" a range indicator. Hence it is double backslashed below.
        // See T28948
        $rfc5322_atext = "a-z0-9!#$%&'*+\\-\/=?^_`{|}~";
        $rfc1034_ldh_str = "a-z0-9\\-";

        $html5_email_regexp = "/
        ^                      # start of string
        [$rfc5322_atext\\.]+    # user part which is liberal :p
        @                      # 'apostrophe'
        [$rfc1034_ldh_str]+       # First domain part
        (\\.[$rfc1034_ldh_str]+)*  # Following part prefixed with a dot
        $                      # End of string
        /ix"; // case Insensitive, eXtended

        return (bool)preg_match( $html5_email_regexp, $addr );
    }
}