Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
62.31% |
81 / 130 |
|
56.25% |
9 / 16 |
CRAP | |
0.00% |
0 / 1 |
| Utils | |
62.31% |
81 / 130 |
|
56.25% |
9 / 16 |
177.57 | |
0.00% |
0 / 1 |
| convert | |
66.67% |
8 / 12 |
|
0.00% |
0 / 1 |
15.48 | |||
| wikitextToHTML | |
0.00% |
0 / 6 |
|
0.00% |
0 / 1 |
2 | |||
| htmlToWikitext | |
0.00% |
0 / 10 |
|
0.00% |
0 / 1 |
6 | |||
| htmlToPlaintext | |
100.00% |
4 / 4 |
|
100.00% |
1 / 1 |
2 | |||
| commentParser | |
77.78% |
7 / 9 |
|
0.00% |
0 / 1 |
5.27 | |||
| createDOM | |
54.84% |
17 / 31 |
|
0.00% |
0 / 1 |
7.30 | |||
| onFlowAddModules | |
0.00% |
0 / 5 |
|
0.00% |
0 / 1 |
2 | |||
| saferSaveXML | |
100.00% |
3 / 3 |
|
100.00% |
1 / 1 |
1 | |||
| getInnerHtml | |
100.00% |
8 / 8 |
|
100.00% |
1 / 1 |
4 | |||
| getOuterHtml | |
100.00% |
4 / 4 |
|
100.00% |
1 / 1 |
2 | |||
| encodeHeadInfo | |
100.00% |
9 / 9 |
|
100.00% |
1 / 1 |
4 | |||
| decodeHeadInfo | |
100.00% |
9 / 9 |
|
100.00% |
1 / 1 |
2 | |||
| getParsoidVersion | |
100.00% |
4 / 4 |
|
100.00% |
1 / 1 |
2 | |||
| createRelativeTitle | |
100.00% |
4 / 4 |
|
100.00% |
1 / 1 |
4 | |||
| getLanguageConverter | |
100.00% |
4 / 4 |
|
100.00% |
1 / 1 |
1 | |||
| getConvertedTitle | |
0.00% |
0 / 8 |
|
0.00% |
0 / 1 |
6 | |||
| 1 | <?php |
| 2 | |
| 3 | namespace Flow\Conversion; |
| 4 | |
| 5 | use DOMDocument; |
| 6 | use DOMElement; |
| 7 | use DOMNode; |
| 8 | use Flow\Exception\NoParserException; |
| 9 | use Flow\Exception\WikitextException; |
| 10 | use Flow\Parsoid\ContentFixer; |
| 11 | use Flow\Parsoid\Fixer\EmptyNodeFixer; |
| 12 | use MediaWiki\Content\TextContent; |
| 13 | use MediaWiki\Content\WikitextContent; |
| 14 | use MediaWiki\Html\Html; |
| 15 | use MediaWiki\Language\ILanguageConverter; |
| 16 | use MediaWiki\Language\Language; |
| 17 | use MediaWiki\MediaWikiServices; |
| 18 | use MediaWiki\Output\OutputPage; |
| 19 | use MediaWiki\Parser\ParserOptions; |
| 20 | use MediaWiki\Parser\Sanitizer; |
| 21 | use MediaWiki\Title\Title; |
| 22 | |
| 23 | abstract class Utils { |
| 24 | |
| 25 | public const PARSOID_VERSION = '2.0.0'; |
| 26 | |
| 27 | /** |
| 28 | * Convert from/to wikitext <=> html or topic-title-wikitext => topic-title-html. |
| 29 | * Only these pairs are supported. html => wikitext requires Parsoid, and |
| 30 | * topic-title-html => topic-title-wikitext is not supported. |
| 31 | * |
| 32 | * @param string $from Format of content to convert: html|wikitext|topic-title-wikitext |
| 33 | * @param string $to Format to convert to: html|wikitext|topic-title-html |
| 34 | * @param string $content |
| 35 | * @param Title $title |
| 36 | * @return string |
| 37 | * @throws WikitextException When the requested conversion is unsupported |
| 38 | * @throws NoParserException When the conversion fails |
| 39 | * @return-taint none |
| 40 | */ |
| 41 | public static function convert( $from, $to, $content, Title $title ) { |
| 42 | if ( $from === $to || $content === '' ) { |
| 43 | return $content; |
| 44 | } |
| 45 | |
| 46 | if ( $from === 'wt' ) { |
| 47 | $from = 'wikitext'; |
| 48 | } |
| 49 | |
| 50 | if ( $from == 'wikitext' && $to == 'html' ) { |
| 51 | return self::wikitextToHTML( $content, $title ); |
| 52 | } elseif ( $from == 'html' && $to == 'wikitext' ) { |
| 53 | return self::htmlToWikitext( $content, $title ); |
| 54 | } elseif ( $from === 'topic-title-wikitext' && |
| 55 | ( $to === 'topic-title-html' || $to === 'topic-title-plaintext' ) ) { |
| 56 | // FIXME: links need to be proceed by findVariantLinks or equivant function |
| 57 | return self::getLanguageConverter()->convert( self::commentParser( $from, $to, $content ) ); |
| 58 | } else { |
| 59 | return self::commentParser( $from, $to, $content ); |
| 60 | } |
| 61 | } |
| 62 | |
| 63 | /** |
| 64 | * @param string $wikitext |
| 65 | * @param Title $title |
| 66 | * |
| 67 | * @return string The converted wikitext to HTML |
| 68 | */ |
| 69 | private static function wikitextToHTML( string $wikitext, Title $title ) { |
| 70 | $parserOptions = ParserOptions::newFromAnon(); |
| 71 | $parserOptions->setRenderReason( __METHOD__ ); |
| 72 | |
| 73 | $parserFactory = MediaWikiServices::getInstance()->getParsoidParserFactory()->create(); |
| 74 | $parserOutput = $parserFactory->parse( $wikitext, $title, $parserOptions ); |
| 75 | |
| 76 | // $parserOutput->getText() will strip off the body tag, but we want to retain here. |
| 77 | // So we'll call ->getRawText() here and modify the HTML by ourselves. |
| 78 | preg_match( "#<body[^>]*>(.*?)</body>#s", $parserOutput->getRawText(), $html ); |
| 79 | |
| 80 | return $html[0]; |
| 81 | } |
| 82 | |
| 83 | /** |
| 84 | * @param string $html |
| 85 | * @param Title $title |
| 86 | * |
| 87 | * @return string The converted HTML to Wikitext |
| 88 | * @throws WikitextException When the conversion is unsupported |
| 89 | */ |
| 90 | private static function htmlToWikitext( string $html, Title $title ) { |
| 91 | $transform = MediaWikiServices::getInstance()->getHtmlTransformFactory() |
| 92 | ->getHtmlToContentTransform( $html, $title ); |
| 93 | |
| 94 | $transform->setOptions( [ |
| 95 | 'contentmodel' => CONTENT_MODEL_WIKITEXT, |
| 96 | 'offsetType' => 'byte' |
| 97 | ] ); |
| 98 | |
| 99 | /** @var TextContent $content */ |
| 100 | $content = $transform->htmlToContent(); |
| 101 | |
| 102 | if ( !$content instanceof WikitextContent ) { |
| 103 | throw new WikitextException( 'Conversion to Wikitext failed' ); |
| 104 | } |
| 105 | |
| 106 | return trim( $content->getTextForSearchIndex() ); |
| 107 | } |
| 108 | |
| 109 | /** |
| 110 | * Basic conversion of html to plaintext for use in recent changes, history, |
| 111 | * and other places where a roundtrip is undesired. |
| 112 | * |
| 113 | * @param string $html |
| 114 | * @param int|null $truncateLength Maximum length in characters (including ellipses) or null for whole string. |
| 115 | * @param Language|null $lang Language to use for truncation. Defaults to $wgLang |
| 116 | * @return string plaintext |
| 117 | */ |
| 118 | public static function htmlToPlaintext( $html, ?int $truncateLength = null, ?Language $lang = null ) { |
| 119 | /** @var Language $wgLang */ |
| 120 | global $wgLang; |
| 121 | |
| 122 | $plain = trim( Sanitizer::stripAllTags( $html ) ); |
| 123 | |
| 124 | // Fallback to some large-ish value for truncation. |
| 125 | $truncateLength ??= 10000; |
| 126 | |
| 127 | $lang = $lang ?: $wgLang; |
| 128 | return $lang->truncateForVisual( $plain, $truncateLength ); |
| 129 | } |
| 130 | |
| 131 | /** |
| 132 | * Convert from/to topic-title-wikitext/topic-title-html using |
| 133 | * MediaWiki\CommentFormatter\CommentFormatter::formatLinks |
| 134 | * |
| 135 | * @param string $from Format of content to convert: topic-title-wikitext |
| 136 | * @param string $to Format of content to convert to: topic-title-html |
| 137 | * @param string $content Content to convert, in topic-title-wikitext format. |
| 138 | * @return string $content in HTML |
| 139 | * @throws WikitextException |
| 140 | */ |
| 141 | protected static function commentParser( $from, $to, $content ) { |
| 142 | if ( |
| 143 | $from !== 'topic-title-wikitext' || |
| 144 | ( $to !== 'topic-title-html' && $to !== 'topic-title-plaintext' ) |
| 145 | ) { |
| 146 | throw new WikitextException( "Conversion from '$from' to '$to' was requested, " . |
| 147 | "but this is not supported." ); |
| 148 | } |
| 149 | |
| 150 | $html = MediaWikiServices::getInstance()->getCommentFormatter() |
| 151 | ->formatLinks( Sanitizer::escapeHtmlAllowEntities( $content ) ); |
| 152 | if ( $to === 'topic-title-plaintext' ) { |
| 153 | return self::htmlToPlaintext( $html ); |
| 154 | } else { |
| 155 | return $html; |
| 156 | } |
| 157 | } |
| 158 | |
| 159 | /** |
| 160 | * Turns given $content string into a DOMDocument object. |
| 161 | * |
| 162 | * Note that, by default, $content will be prefixed with <?xml encoding="utf-8"?> to force |
| 163 | * libxml to interpret the content as UTF-8. If for some reason you don't want this to happen, |
| 164 | * or you are certain that your input already has <?xml encoding="utf-8"?> or |
| 165 | * <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> , then you can disable |
| 166 | * this behavior by setting $utf8Fragment=false to disable this behavior. |
| 167 | * |
| 168 | * Some libxml errors are forgivable, libxml errors that aren't |
| 169 | * ignored will throw a WikitextException. |
| 170 | * |
| 171 | * The default error codes allowed are: |
| 172 | * 9 - allow illegal characters (they are removed, but this option means it |
| 173 | * doesn't trigger an error. |
| 174 | * 76 - allow unexpected end tag. This is typically old wikitext using deprecated tags. |
| 175 | * 513 - allow multiple tags with same id |
| 176 | * 801 - allow unrecognized tags like figcaption |
| 177 | * |
| 178 | * @param string $content |
| 179 | * @param bool $utf8Fragment If true, prefix $content with <?xml encoding="utf-8"?> |
| 180 | * @param array $ignoreErrorCodes |
| 181 | * @return DOMDocument |
| 182 | * @throws WikitextException |
| 183 | * @see http://www.xmlsoft.org/html/libxml-xmlerror.html |
| 184 | */ |
| 185 | public static function createDOM( |
| 186 | $content, |
| 187 | $utf8Fragment = true, |
| 188 | array $ignoreErrorCodes = [ 9, 76, 513, 801 ] |
| 189 | ) { |
| 190 | $dom = new DOMDocument(); |
| 191 | |
| 192 | $loadEntities = false; |
| 193 | if ( LIBXML_VERSION < 20900 ) { |
| 194 | // Otherwise the parser may attempt to load the dtd from an external source. |
| 195 | // See: https://www.mediawiki.org/wiki/XML_External_Entity_Processing |
| 196 | $loadEntities = libxml_disable_entity_loader( true ); |
| 197 | } |
| 198 | |
| 199 | // don't output warnings |
| 200 | $useErrors = libxml_use_internal_errors( true ); |
| 201 | |
| 202 | // Work around DOMDocument's morbid insistence on using iso-8859-1 |
| 203 | // Even $dom = new DOMDocument( '1.0', 'utf-8' ); doesn't work, you have to specify |
| 204 | // encoding ="utf-8" in the string fed to loadHTML() |
| 205 | $html = ( $utf8Fragment ? '<?xml encoding="utf-8"?>' : '' ) . $content; |
| 206 | $dom->loadHTML( $html, LIBXML_PARSEHUGE ); |
| 207 | |
| 208 | if ( LIBXML_VERSION < 20900 ) { |
| 209 | libxml_disable_entity_loader( $loadEntities ); |
| 210 | } |
| 211 | |
| 212 | // check error codes; if not in the supplied list of ignorable errors, |
| 213 | // throw an exception |
| 214 | $errors = array_filter( |
| 215 | libxml_get_errors(), |
| 216 | static function ( $error ) use( $ignoreErrorCodes ) { |
| 217 | return !in_array( $error->code, $ignoreErrorCodes ); |
| 218 | } |
| 219 | ); |
| 220 | |
| 221 | // restore libxml state before anything else |
| 222 | libxml_clear_errors(); |
| 223 | libxml_use_internal_errors( $useErrors ); |
| 224 | |
| 225 | if ( $errors ) { |
| 226 | throw new WikitextException( |
| 227 | implode( |
| 228 | "\n", |
| 229 | array_map( |
| 230 | static function ( $error ) { |
| 231 | return $error->message; |
| 232 | }, |
| 233 | $errors |
| 234 | ) |
| 235 | ) . "\n\nFrom source content:\n" . $content, |
| 236 | 'process-wikitext' |
| 237 | ); |
| 238 | } |
| 239 | |
| 240 | return $dom; |
| 241 | } |
| 242 | |
| 243 | /** |
| 244 | * Handler for FlowAddModules, avoids rest of Flow having to be aware if |
| 245 | * Parsoid is in use. |
| 246 | * |
| 247 | * @param OutputPage $out |
| 248 | * @return bool |
| 249 | */ |
| 250 | public static function onFlowAddModules( OutputPage $out ) { |
| 251 | // The module is only necessary when we are using parsoid. |
| 252 | // XXX We only need the Parsoid CSS if some content being |
| 253 | // rendered has getContentFormat() === 'html'. |
| 254 | $out->addModuleStyles( [ |
| 255 | 'mediawiki.skinning.content.parsoid', |
| 256 | 'ext.cite.parsoid.styles', |
| 257 | ] ); |
| 258 | |
| 259 | return true; |
| 260 | } |
| 261 | |
| 262 | /** |
| 263 | * Saves a document using saveXML, but avoid escaping style blocks with CDATA. |
| 264 | * This is not needed in HTML and breaks the CSS. |
| 265 | * |
| 266 | * @param DOMDocument $doc |
| 267 | * @param DOMNode|null $node the specific node to save |
| 268 | * @return string HTML |
| 269 | */ |
| 270 | public static function saferSaveXML( DOMDocument $doc, ?DOMNode $node = null ) { |
| 271 | $html = $doc->saveXML( $node ); |
| 272 | // This regex is only safe as long as attribute values get escaped > chars |
| 273 | // This is checked by the testcases |
| 274 | $html = preg_replace( '/<style([^>]*)><!\[CDATA\[/i', '<style\1>', $html ); |
| 275 | return preg_replace( '/\]\]><\/style>/i', '</style>', $html ); |
| 276 | } |
| 277 | |
| 278 | /** |
| 279 | * Retrieves the html of the node's children. |
| 280 | * |
| 281 | * @param DOMNode|null $node |
| 282 | * @return string html of the nodes children |
| 283 | */ |
| 284 | public static function getInnerHtml( ?DOMNode $node = null ) { |
| 285 | $html = ''; |
| 286 | if ( $node ) { |
| 287 | $dom = $node instanceof DOMDocument ? $node : $node->ownerDocument; |
| 288 | // Don't use saveHTML(), it has bugs (T217766); instead use XML serialization |
| 289 | // with a workaround for empty non-void nodes |
| 290 | $fixer = new ContentFixer( new EmptyNodeFixer ); |
| 291 | $fixer->applyToDom( $dom, Title::newMainPage() ); |
| 292 | |
| 293 | foreach ( $node->childNodes as $child ) { |
| 294 | $html .= self::saferSaveXML( $dom, $child ); |
| 295 | } |
| 296 | } |
| 297 | return $html; |
| 298 | } |
| 299 | |
| 300 | /** |
| 301 | * Gets the HTML of a node. This is like getInnterHtml(), but includes the node's tag itself too. |
| 302 | * @param DOMNode $node |
| 303 | * @return string HTML |
| 304 | */ |
| 305 | public static function getOuterHtml( DOMNode $node ) { |
| 306 | $dom = $node instanceof DOMDocument ? $node : $node->ownerDocument; |
| 307 | // Don't use saveHTML(), it has bugs (T217766); instead use XML serialization |
| 308 | // with a workaround for empty non-void nodes |
| 309 | $fixer = new ContentFixer( new EmptyNodeFixer ); |
| 310 | $fixer->applyToDom( $dom, Title::newMainPage() ); |
| 311 | return self::saferSaveXML( $dom, $node ); |
| 312 | } |
| 313 | |
| 314 | /** |
| 315 | * Encode information from the <head> tag as attributes on the <body> tag, then |
| 316 | * drop the <head>. |
| 317 | * |
| 318 | * Specifically, add the Parsoid version number in the parsoid-version attribute; |
| 319 | * put the href of the <base> tag in the base-url attribute; |
| 320 | * and remove the class attribute from the <body>. |
| 321 | * |
| 322 | * @param string $html |
| 323 | * @return string HTML with <head> information encoded as attributes on the <body> |
| 324 | * @throws WikitextException |
| 325 | * @suppress PhanUndeclaredMethod,PhanTypeMismatchArgumentNullable Apparently a phan bug / wrong built-in PHP stubs |
| 326 | */ |
| 327 | public static function encodeHeadInfo( $html ) { |
| 328 | $dom = ContentFixer::createDOM( $html ); |
| 329 | $body = $dom->getElementsByTagName( 'body' )->item( 0 ); |
| 330 | $head = $dom->getElementsByTagName( 'head' )->item( 0 ); |
| 331 | $base = $head ? $head->getElementsByTagName( 'base' )->item( 0 ) : null; |
| 332 | $body->setAttribute( 'parsoid-version', self::PARSOID_VERSION ); |
| 333 | if ( $base instanceof DOMElement && $base->getAttribute( 'href' ) ) { |
| 334 | $body->setAttribute( 'base-url', $base->getAttribute( 'href' ) ); |
| 335 | } |
| 336 | // The class attribute is not used by us and is wastefully long, remove it |
| 337 | $body->removeAttribute( 'class' ); |
| 338 | return self::getOuterHtml( $body ); |
| 339 | } |
| 340 | |
| 341 | /** |
| 342 | * Put the base URI from the <body>'s base-url attribute back in the <head> as a <base> tag. |
| 343 | * This reverses (part of) the transformation done by encodeHeadInfo(). |
| 344 | * |
| 345 | * @param string $html HTML (may be a full document, <body> tag or unwrapped <body> contents) |
| 346 | * @return string HTML (<html> tag with <head> and <body>) with the <base> tag restored |
| 347 | * @throws WikitextException |
| 348 | * @suppress PhanUndeclaredMethod,PhanTypeMismatchArgumentNullable Apparently a phan bug / wrong built-in PHP stubs |
| 349 | */ |
| 350 | public static function decodeHeadInfo( $html ) { |
| 351 | $dom = ContentFixer::createDOM( $html ); |
| 352 | $body = $dom->getElementsByTagName( 'body' )->item( 0 ); |
| 353 | $baseUrl = $body->getAttribute( 'base-url' ); |
| 354 | return Html::rawElement( 'html', [], |
| 355 | Html::rawElement( 'head', [], |
| 356 | // Only set base href if there's a value to set. |
| 357 | $baseUrl ? Html::element( 'base', [ 'href' => $baseUrl ] ) : '' |
| 358 | ) . |
| 359 | self::getOuterHtml( $body ) |
| 360 | ); |
| 361 | } |
| 362 | |
| 363 | /** |
| 364 | * Get the Parsoid version from HTML content stored in the database. |
| 365 | * This interprets the transformation done by encodeHeadInfo(). |
| 366 | * |
| 367 | * @param string $html |
| 368 | * @return string|null Parsoid version number, or null if none found |
| 369 | * @suppress PhanUndeclaredMethod Apparently a phan bug / wrong built-in PHP stubs |
| 370 | */ |
| 371 | public static function getParsoidVersion( $html ) { |
| 372 | $dom = ContentFixer::createDOM( $html ); |
| 373 | $body = $dom->getElementsByTagName( 'body' )->item( 0 ); |
| 374 | $version = $body->getAttribute( 'parsoid-version' ); |
| 375 | return $version !== '' ? $version : null; |
| 376 | } |
| 377 | |
| 378 | /** |
| 379 | * Subpage links from Parsoid don't contain any direct context, its applied via |
| 380 | * a <base href="..."> tag, so here we apply a similar rule resolving against |
| 381 | * $title |
| 382 | * |
| 383 | * @param string $text |
| 384 | * @param Title $title Title to resolve relative links against |
| 385 | * @return Title|null |
| 386 | */ |
| 387 | public static function createRelativeTitle( $text, Title $title ) { |
| 388 | // currently parsoid always uses enough ../ or ./ to go |
| 389 | // back to the root, a bit of a kludge but just assume we |
| 390 | // can strip and will end up with a non-relative text. |
| 391 | $text = preg_replace( '|^(\.\.?/)+|', '', $text ); |
| 392 | |
| 393 | if ( $text && ( $text[0] === '/' || $text[0] === '#' ) ) { |
| 394 | return Title::newFromText( $title->getDBkey() . $text, $title->getNamespace() ); |
| 395 | } |
| 396 | |
| 397 | return Title::newFromText( $text ); |
| 398 | } |
| 399 | |
| 400 | /** |
| 401 | * @since 1.35 |
| 402 | * @return ILanguageConverter |
| 403 | */ |
| 404 | private static function getLanguageConverter(): ILanguageConverter { |
| 405 | $services = MediaWikiServices::getInstance(); |
| 406 | return $services |
| 407 | ->getLanguageConverterFactory() |
| 408 | ->getLanguageConverter( $services->getContentLanguage() ); |
| 409 | } |
| 410 | |
| 411 | /** |
| 412 | * @since 1.35 |
| 413 | * @param Title $title Title to convert to language variant |
| 414 | * @return string Converted title |
| 415 | */ |
| 416 | public static function getConvertedTitle( Title $title ) { |
| 417 | $ns = $title->getNamespace(); |
| 418 | $titleText = $title->getText(); |
| 419 | $langConv = self::getLanguageConverter(); |
| 420 | $variant = $langConv->getPreferredVariant(); |
| 421 | $convertedNamespace = $langConv->convertNamespace( $ns, $variant ); |
| 422 | if ( $convertedNamespace ) { |
| 423 | return $convertedNamespace . ':' . $langConv->translate( $titleText, $variant ); |
| 424 | } else { |
| 425 | return $langConv->translate( $titleText, $variant ); |
| 426 | } |
| 427 | } |
| 428 | } |