Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
0.00% |
0 / 691 |
|
0.00% |
0 / 33 |
CRAP | |
0.00% |
0 / 1 |
| WikitextSerializer | |
0.00% |
0 / 691 |
|
0.00% |
0 / 33 |
78680 | |
0.00% |
0 / 1 |
| __construct | |
0.00% |
0 / 4 |
|
0.00% |
0 / 1 |
2 | |||
| linkHandler | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| languageVariantHandler | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| escapeWikitext | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| domToWikitext | |
0.00% |
0 / 3 |
|
0.00% |
0 / 1 |
2 | |||
| htmlToWikitext | |
0.00% |
0 / 4 |
|
0.00% |
0 / 1 |
2 | |||
| getAttributeKey | |
0.00% |
0 / 8 |
|
0.00% |
0 / 1 |
20 | |||
| getAttributeValue | |
0.00% |
0 / 11 |
|
0.00% |
0 / 1 |
42 | |||
| getAttributeValueAsShadowInfo | |
0.00% |
0 / 9 |
|
0.00% |
0 / 1 |
6 | |||
| serializedImageAttrVal | |
0.00% |
0 / 2 |
|
0.00% |
0 / 1 |
6 | |||
| serializedAttrVal | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| tagNeedsEscaping | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| wrapAngleBracket | |
0.00% |
0 / 7 |
|
0.00% |
0 / 1 |
20 | |||
| serializeHTMLTag | |
0.00% |
0 / 19 |
|
0.00% |
0 / 1 |
72 | |||
| serializeHTMLEndTag | |
0.00% |
0 / 13 |
|
0.00% |
0 / 1 |
42 | |||
| serializeAttributes | |
0.00% |
0 / 56 |
|
0.00% |
0 / 1 |
992 | |||
| handleLIHackIfApplicable | |
0.00% |
0 / 7 |
|
0.00% |
0 / 1 |
42 | |||
| formatStringSubst | |
0.00% |
0 / 10 |
|
0.00% |
0 / 1 |
20 | |||
| createParamComparator | |
0.00% |
0 / 56 |
|
0.00% |
0 / 1 |
342 | |||
| serializePart | |
0.00% |
0 / 124 |
|
0.00% |
0 / 1 |
1892 | |||
| serializeFromParts | |
0.00% |
0 / 38 |
|
0.00% |
0 / 1 |
156 | |||
| serializeExtensionStartTag | |
0.00% |
0 / 17 |
|
0.00% |
0 / 1 |
30 | |||
| defaultExtensionHandler | |
0.00% |
0 / 18 |
|
0.00% |
0 / 1 |
20 | |||
| serializeText | |
0.00% |
0 / 13 |
|
0.00% |
0 / 1 |
42 | |||
| serializeTextNode | |
0.00% |
0 / 4 |
|
0.00% |
0 / 1 |
2 | |||
| emitWikitext | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| serializeNodeInternal | |
0.00% |
0 / 63 |
|
0.00% |
0 / 1 |
870 | |||
| serializeNode | |
0.00% |
0 / 62 |
|
0.00% |
0 / 1 |
306 | |||
| stripUnnecessaryHeadingNowikis | |
0.00% |
0 / 11 |
|
0.00% |
0 / 1 |
20 | |||
| stripUnnecessaryIndentPreNowikis | |
0.00% |
0 / 32 |
|
0.00% |
0 / 1 |
72 | |||
| stripUnnecessaryQuoteNowikis | |
0.00% |
0 / 63 |
|
0.00% |
0 / 1 |
1482 | |||
| serializeDOM | |
0.00% |
0 / 30 |
|
0.00% |
0 / 1 |
156 | |||
| trace | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| 1 | <?php |
| 2 | declare( strict_types = 1 ); |
| 3 | |
| 4 | namespace Wikimedia\Parsoid\Html2Wt; |
| 5 | |
| 6 | use Closure; |
| 7 | use Exception; |
| 8 | use stdClass; |
| 9 | use Wikimedia\Assert\Assert; |
| 10 | use Wikimedia\Parsoid\Config\Env; |
| 11 | use Wikimedia\Parsoid\Core\InternalException; |
| 12 | use Wikimedia\Parsoid\DOM\Comment; |
| 13 | use Wikimedia\Parsoid\DOM\Document; |
| 14 | use Wikimedia\Parsoid\DOM\DocumentFragment; |
| 15 | use Wikimedia\Parsoid\DOM\Element; |
| 16 | use Wikimedia\Parsoid\DOM\Node; |
| 17 | use Wikimedia\Parsoid\DOM\Text; |
| 18 | use Wikimedia\Parsoid\Html2Wt\ConstrainedText\ConstrainedText; |
| 19 | use Wikimedia\Parsoid\Html2Wt\DOMHandlers\DOMHandler; |
| 20 | use Wikimedia\Parsoid\Html2Wt\DOMHandlers\DOMHandlerFactory; |
| 21 | use Wikimedia\Parsoid\Tokens\KV; |
| 22 | use Wikimedia\Parsoid\Tokens\TagTk; |
| 23 | use Wikimedia\Parsoid\Tokens\Token; |
| 24 | use Wikimedia\Parsoid\Utils\ContentUtils; |
| 25 | use Wikimedia\Parsoid\Utils\DiffDOMUtils; |
| 26 | use Wikimedia\Parsoid\Utils\DOMCompat; |
| 27 | use Wikimedia\Parsoid\Utils\DOMDataUtils; |
| 28 | use Wikimedia\Parsoid\Utils\DOMUtils; |
| 29 | use Wikimedia\Parsoid\Utils\PHPUtils; |
| 30 | use Wikimedia\Parsoid\Utils\Title; |
| 31 | use Wikimedia\Parsoid\Utils\TokenUtils; |
| 32 | use Wikimedia\Parsoid\Utils\Utils; |
| 33 | use Wikimedia\Parsoid\Utils\WTUtils; |
| 34 | use Wikimedia\Parsoid\Wikitext\Consts; |
| 35 | |
| 36 | /** |
| 37 | * Wikitext to HTML serializer. |
| 38 | * Serializes a chunk of tokens or an HTML DOM to MediaWiki's wikitext flavor. |
| 39 | * |
| 40 | * This serializer is designed to eventually |
| 41 | * - accept arbitrary HTML and |
| 42 | * - serialize that to wikitext in a way that round-trips back to the same |
| 43 | * HTML DOM as far as possible within the limitations of wikitext. |
| 44 | * |
| 45 | * Not much effort has been invested so far on supporting |
| 46 | * non-Parsoid/VE-generated HTML. Some of this involves adaptively switching |
| 47 | * between wikitext and HTML representations based on the values of attributes |
| 48 | * and DOM context. A few special cases are already handled adaptively |
| 49 | * (multi-paragraph list item contents are serialized as HTML tags for |
| 50 | * example, generic A elements are serialized to HTML A tags), but in general |
| 51 | * support for this is mostly missing. |
| 52 | * |
| 53 | * Example issue: |
| 54 | * ``` |
| 55 | * <h1><p>foo</p></h1> will serialize to =\nfoo\n= whereas the |
| 56 | * correct serialized output would be: =<p>foo</p>= |
| 57 | * ``` |
| 58 | * |
| 59 | * What to do about this? |
| 60 | * - add a generic 'can this HTML node be serialized to wikitext in this |
| 61 | * context' detection method and use that to adaptively switch between |
| 62 | * wikitext and HTML serialization. |
| 63 | * |
| 64 | */ |
| 65 | class WikitextSerializer { |
| 66 | |
| 67 | /** @var string[] */ |
| 68 | private const IGNORED_ATTRIBUTES = [ |
| 69 | 'data-parsoid' => true, |
| 70 | 'data-ve-changed' => true, |
| 71 | 'data-parsoid-changed' => true, |
| 72 | 'data-parsoid-diff' => true, |
| 73 | 'data-parsoid-serialize' => true, |
| 74 | DOMDataUtils::DATA_OBJECT_ATTR_NAME => true, |
| 75 | ]; |
| 76 | |
| 77 | /** @var string[] attribute name => value regexp */ |
| 78 | private const PARSOID_ATTRIBUTES = [ |
| 79 | 'about' => '/^#mwt\d+$/D', |
| 80 | 'typeof' => '/(^|\s)mw:\S+/', |
| 81 | ]; |
| 82 | |
| 83 | /** @var string Regexp */ |
| 84 | private const TRAILING_COMMENT_OR_WS_AFTER_NL_REGEXP |
| 85 | = '/\n(\s|' . Utils::COMMENT_REGEXP_FRAGMENT . ')*$/D'; |
| 86 | |
| 87 | /** @var string Regexp */ |
| 88 | private const FORMATSTRING_REGEXP = |
| 89 | '/^(\n)?(\{\{ *_+)(\n? *\|\n? *_+ *= *)(_+)(\n? *\}\})(\n)?$/D'; |
| 90 | |
| 91 | /** @var string Regexp for testing whether nowiki added around heading-like wikitext is needed */ |
| 92 | private const COMMENT_OR_WS_REGEXP = '/^(\s|' . Utils::COMMENT_REGEXP_FRAGMENT . ')*$/D'; |
| 93 | |
| 94 | /** @var string Regexp for testing whether nowiki added around heading-like wikitext is needed */ |
| 95 | private const HEADING_NOWIKI_REGEXP = '/^(?:' . Utils::COMMENT_REGEXP_FRAGMENT . ')*' |
| 96 | . '<nowiki>(=+[^=]+=+)<\/nowiki>(.+)$/D'; |
| 97 | |
| 98 | /** @var array string[] */ |
| 99 | private static $separatorREs = [ |
| 100 | 'pureSepRE' => '/^[ \t\r\n]*$/D', |
| 101 | 'sepPrefixWithNlsRE' => '/^[ \t]*\n+[ \t\r\n]*/', |
| 102 | 'sepSuffixWithNlsRE' => '/\n[ \t\r\n]*$/D', |
| 103 | ]; |
| 104 | |
| 105 | /** @var WikitextEscapeHandlers */ |
| 106 | public $wteHandlers; |
| 107 | |
| 108 | /** @var Env */ |
| 109 | public $env; |
| 110 | |
| 111 | /** @var SerializerState */ |
| 112 | private $state; |
| 113 | |
| 114 | /** @var string Log type for trace() */ |
| 115 | private $logType; |
| 116 | |
| 117 | /** |
| 118 | * @param Env $env |
| 119 | * @param array $options List of options for serialization: |
| 120 | * - logType: (string) |
| 121 | * - extName: (string) |
| 122 | */ |
| 123 | public function __construct( Env $env, $options ) { |
| 124 | $this->env = $env; |
| 125 | $this->logType = $options['logType'] ?? 'trace/wts'; |
| 126 | $this->state = new SerializerState( $this, $options ); |
| 127 | $this->wteHandlers = new WikitextEscapeHandlers( $env, $options['extName'] ?? null ); |
| 128 | } |
| 129 | |
| 130 | /** |
| 131 | * Main link handler. |
| 132 | * @param Element $node |
| 133 | * Used in multiple tag handlers (<a> and <link>), and hence added as top-level method |
| 134 | */ |
| 135 | public function linkHandler( Element $node ): void { |
| 136 | LinkHandlerUtils::linkHandler( $this->state, $node ); |
| 137 | } |
| 138 | |
| 139 | /** |
| 140 | * @param Element $node |
| 141 | */ |
| 142 | public function languageVariantHandler( Node $node ): void { |
| 143 | LanguageVariantHandler::handleLanguageVariant( $this->state, $node ); |
| 144 | } |
| 145 | |
| 146 | /** |
| 147 | * Escape wikitext-like strings in '$text' so that $text renders as a plain string |
| 148 | * when rendered as HTML. The escaping is done based on the context in which $text |
| 149 | * is present (ex: start-of-line, in a link, etc.) |
| 150 | * |
| 151 | * @param SerializerState $state |
| 152 | * @param string $text |
| 153 | * @param array $opts |
| 154 | * - node: (Node) |
| 155 | * - isLastChild: (bool) |
| 156 | * @return string |
| 157 | */ |
| 158 | public function escapeWikitext( SerializerState $state, string $text, array $opts ): string { |
| 159 | return $this->wteHandlers->escapeWikitext( $state, $text, $opts ); |
| 160 | } |
| 161 | |
| 162 | public function domToWikitext( |
| 163 | array $opts, DocumentFragment $node |
| 164 | ): string { |
| 165 | $opts['logType'] = $this->logType; |
| 166 | $serializer = new WikitextSerializer( $this->env, $opts ); |
| 167 | return $serializer->serializeDOM( $node ); |
| 168 | } |
| 169 | |
| 170 | public function htmlToWikitext( array $opts, string $html ): string { |
| 171 | $domFragment = ContentUtils::createAndLoadDocumentFragment( |
| 172 | $this->env->topLevelDoc, $html, [ 'markNew' => true ] |
| 173 | ); |
| 174 | return $this->domToWikitext( $opts, $domFragment ); |
| 175 | } |
| 176 | |
| 177 | public function getAttributeKey( Element $node, string $key ): string { |
| 178 | $tplAttrs = DOMDataUtils::getDataMw( $node )->attribs ?? []; |
| 179 | foreach ( $tplAttrs as $attr ) { |
| 180 | // If this attribute's key is generated content, |
| 181 | // serialize HTML back to generator wikitext. |
| 182 | // PORT-FIXME: bool check might not be safe. Need documentation on attrib format. |
| 183 | if ( ( $attr[0]->txt ?? null ) === $key && isset( $attr[0]->html ) ) { |
| 184 | return $this->htmlToWikitext( [ |
| 185 | 'env' => $this->env, |
| 186 | 'onSOL' => false, |
| 187 | ], $attr[0]->html ); |
| 188 | } |
| 189 | } |
| 190 | return $key; |
| 191 | } |
| 192 | |
| 193 | /** |
| 194 | * @param Element $node |
| 195 | * @param string $key Attribute name. |
| 196 | * @return ?string The wikitext value, or null if the attribute is not present. |
| 197 | */ |
| 198 | public function getAttributeValue( Element $node, string $key ): ?string { |
| 199 | $tplAttrs = DOMDataUtils::getDataMw( $node )->attribs ?? []; |
| 200 | foreach ( $tplAttrs as $attr ) { |
| 201 | // If this attribute's value is generated content, |
| 202 | // serialize HTML back to generator wikitext. |
| 203 | // PORT-FIXME: not type safe. Need documentation on attrib format. |
| 204 | if ( ( $attr[0] === $key || ( $attr[0]->txt ?? null ) === $key ) |
| 205 | // Only return here if the value is generated (ie. .html), |
| 206 | // it may just be in .txt form. |
| 207 | && isset( $attr[1]->html ) |
| 208 | // !== null is required. html:"" will serialize to "" and |
| 209 | // will be returned here. This is used to suppress the =".." |
| 210 | // string in the attribute in scenarios where the template |
| 211 | // generates a "k=v" string. |
| 212 | // Ex: <div {{1x|1=style='color:red'}}>foo</div> |
| 213 | && $attr[1]->html !== null |
| 214 | ) { |
| 215 | return $this->htmlToWikitext( [ |
| 216 | 'env' => $this->env, |
| 217 | 'onSOL' => false, |
| 218 | 'inAttribute' => true, |
| 219 | ], $attr[1]->html ); |
| 220 | } |
| 221 | } |
| 222 | return null; |
| 223 | } |
| 224 | |
| 225 | /** |
| 226 | * @param Element $node |
| 227 | * @param string $key |
| 228 | * @return array|null A tuple in {@link WTSUtils::getShadowInfo()} format, |
| 229 | * with an extra 'fromDataMW' flag. |
| 230 | */ |
| 231 | public function getAttributeValueAsShadowInfo( Element $node, string $key ): ?array { |
| 232 | $v = $this->getAttributeValue( $node, $key ); |
| 233 | if ( $v === null ) { |
| 234 | return $v; |
| 235 | } |
| 236 | return [ |
| 237 | 'value' => $v, |
| 238 | 'modified' => false, |
| 239 | 'fromsrc' => true, |
| 240 | 'fromDataMW' => true, |
| 241 | ]; |
| 242 | } |
| 243 | |
| 244 | /** |
| 245 | * @param Element $dataMWnode |
| 246 | * @param Element $htmlAttrNode |
| 247 | * @param string $key |
| 248 | * @return array A tuple in {@link WTSUtils::getShadowInfo()} format, |
| 249 | * possibly with an extra 'fromDataMW' flag. |
| 250 | */ |
| 251 | public function serializedImageAttrVal( |
| 252 | Element $dataMWnode, Element $htmlAttrNode, string $key |
| 253 | ): array { |
| 254 | $v = $this->getAttributeValueAsShadowInfo( $dataMWnode, $key ); |
| 255 | return $v ?: WTSUtils::getAttributeShadowInfo( $htmlAttrNode, $key ); |
| 256 | } |
| 257 | |
| 258 | public function serializedAttrVal( Element $node, string $name ): array { |
| 259 | return $this->serializedImageAttrVal( $node, $node, $name ); |
| 260 | } |
| 261 | |
| 262 | /** |
| 263 | * Check if token needs escaping |
| 264 | * |
| 265 | * @param string $name |
| 266 | * @return bool |
| 267 | */ |
| 268 | public function tagNeedsEscaping( string $name ): bool { |
| 269 | return WTUtils::isAnnOrExtTag( $this->env, $name ); |
| 270 | } |
| 271 | |
| 272 | public function wrapAngleBracket( Token $token, string $inner ): string { |
| 273 | if ( |
| 274 | $this->tagNeedsEscaping( $token->getName() ) && |
| 275 | !( |
| 276 | // Allow for html tags that shadow extension tags found in source |
| 277 | // to roundtrip. They only parse as html tags if they are unclosed, |
| 278 | // since extension tags bail on parsing without closing tags. |
| 279 | // |
| 280 | // This only applies when wrapAngleBracket() is being called for |
| 281 | // start tags, but we wouldn't be here if it was autoInsertedEnd |
| 282 | // anyways. |
| 283 | isset( Consts::$Sanitizer['AllowedLiteralTags'][$token->getName()] ) && |
| 284 | !empty( $token->dataParsoid->autoInsertedEnd ) |
| 285 | ) |
| 286 | ) { |
| 287 | return "<{$inner}>"; |
| 288 | } |
| 289 | return "<$inner>"; |
| 290 | } |
| 291 | |
| 292 | public function serializeHTMLTag( Element $node, bool $wrapperUnmodified ): string { |
| 293 | // TODO(arlolra): As of 1.3.0, html pre is considered an extension |
| 294 | // and wrapped in encapsulation. When that version is no longer |
| 295 | // accepted for serialization, we can remove this backwards |
| 296 | // compatibility code. |
| 297 | // |
| 298 | // 'inHTMLPre' flag has to be updated always, |
| 299 | // even when we are selsering in the wrapperUnmodified case. |
| 300 | $token = WTSUtils::mkTagTk( $node ); |
| 301 | if ( $token->getName() === 'pre' ) { |
| 302 | // html-syntax pre is very similar to nowiki |
| 303 | $this->state->inHTMLPre = true; |
| 304 | } |
| 305 | |
| 306 | if ( $wrapperUnmodified ) { |
| 307 | $dsr = DOMDataUtils::getDataParsoid( $node )->dsr; |
| 308 | return $this->state->getOrigSrc( $dsr->start, $dsr->innerStart() ) ?? ''; |
| 309 | } |
| 310 | |
| 311 | $da = $token->dataParsoid; |
| 312 | if ( !empty( $da->autoInsertedStart ) ) { |
| 313 | return ''; |
| 314 | } |
| 315 | |
| 316 | $close = ''; |
| 317 | if ( ( Utils::isVoidElement( $token->getName() ) && empty( $da->noClose ) ) || |
| 318 | !empty( $da->selfClose ) |
| 319 | ) { |
| 320 | $close = ' /'; |
| 321 | } |
| 322 | |
| 323 | $sAttribs = $this->serializeAttributes( $node, $token ); |
| 324 | if ( strlen( $sAttribs ) > 0 ) { |
| 325 | $sAttribs = ' ' . $sAttribs; |
| 326 | } |
| 327 | |
| 328 | // srcTagName cannot be '' so, it is okay to use ?? operator |
| 329 | $tokenName = $da->srcTagName ?? $token->getName(); |
| 330 | $inner = "{$tokenName}{$sAttribs}{$close}"; |
| 331 | return $this->wrapAngleBracket( $token, $inner ); |
| 332 | } |
| 333 | |
| 334 | /** |
| 335 | * @param Element $node |
| 336 | * @param bool $wrapperUnmodified |
| 337 | * @return string |
| 338 | */ |
| 339 | public function serializeHTMLEndTag( Element $node, $wrapperUnmodified ): string { |
| 340 | if ( $wrapperUnmodified ) { |
| 341 | $dsr = DOMDataUtils::getDataParsoid( $node )->dsr; |
| 342 | return $this->state->getOrigSrc( $dsr->innerEnd(), $dsr->end ) ?? ''; |
| 343 | } |
| 344 | |
| 345 | $token = WTSUtils::mkEndTagTk( $node ); |
| 346 | if ( $token->getName() === 'pre' ) { |
| 347 | $this->state->inHTMLPre = false; |
| 348 | } |
| 349 | |
| 350 | // srcTagName cannot be '' so, it is okay to use ?? operator |
| 351 | $tokenName = $token->dataParsoid->srcTagName ?? $token->getName(); |
| 352 | $ret = ''; |
| 353 | |
| 354 | if ( empty( $token->dataParsoid->autoInsertedEnd ) |
| 355 | && !Utils::isVoidElement( $token->getName() ) |
| 356 | && empty( $token->dataParsoid->selfClose ) |
| 357 | ) { |
| 358 | $ret = $this->wrapAngleBracket( $token, "/{$tokenName}" ); |
| 359 | } |
| 360 | |
| 361 | return $ret; |
| 362 | } |
| 363 | |
| 364 | public function serializeAttributes( Element $node, Token $token, bool $isWt = false ): string { |
| 365 | $attribs = $token->attribs; |
| 366 | |
| 367 | $out = []; |
| 368 | foreach ( $attribs as $kv ) { |
| 369 | // Tokens created during html2wt don't have nested tokens for keys. |
| 370 | // But, they could be integers but we want strings below. |
| 371 | $k = (string)$kv->k; |
| 372 | $v = null; |
| 373 | $vInfo = null; |
| 374 | |
| 375 | // Unconditionally ignore |
| 376 | // (all of the IGNORED_ATTRIBUTES should be filtered out earlier, |
| 377 | // but ignore them here too just to make sure.) |
| 378 | if ( isset( self::IGNORED_ATTRIBUTES[$k] ) || $k === 'data-mw' ) { |
| 379 | continue; |
| 380 | } |
| 381 | |
| 382 | // Ignore parsoid-like ids. They may have been left behind |
| 383 | // by clients and shouldn't be serialized. This can also happen |
| 384 | // in v2/v3 API when there is no matching data-parsoid entry found |
| 385 | // for this id. |
| 386 | if ( $k === 'id' && preg_match( '/^mw[\w-]{2,}$/D', $kv->v ) ) { |
| 387 | if ( WTUtils::isNewElt( $node ) ) { |
| 388 | // Parsoid id found on element without a matching data-parsoid. Drop it! |
| 389 | } else { |
| 390 | $vInfo = $token->getAttributeShadowInfo( $k ); |
| 391 | if ( !$vInfo['modified'] && $vInfo['fromsrc'] ) { |
| 392 | $out[] = $k . '=' . '"' . str_replace( '"', '"', $vInfo['value'] ) . '"'; |
| 393 | } |
| 394 | } |
| 395 | continue; |
| 396 | } |
| 397 | |
| 398 | // Parsoid auto-generates ids for headings and they should |
| 399 | // be stripped out, except if this is not auto-generated id. |
| 400 | if ( $k === 'id' && DOMUtils::isHeading( $node ) ) { |
| 401 | if ( !empty( DOMDataUtils::getDataParsoid( $node )->reusedId ) ) { |
| 402 | $vInfo = $token->getAttributeShadowInfo( $k ); |
| 403 | // PORT-FIXME: is this safe? value could be a token or token array |
| 404 | $out[] = $k . '="' . str_replace( '"', '"', $vInfo['value'] ) . '"'; |
| 405 | } |
| 406 | continue; |
| 407 | } |
| 408 | |
| 409 | // Strip Parsoid-inserted class="mw-empty-elt" attributes |
| 410 | if ( $k === 'class' |
| 411 | && isset( Consts::$Output['FlaggedEmptyElts'][DOMCompat::nodeName( $node )] ) |
| 412 | ) { |
| 413 | $kv->v = preg_replace( '/\bmw-empty-elt\b/', '', $kv->v, 1 ); |
| 414 | if ( !$kv->v ) { |
| 415 | continue; |
| 416 | } |
| 417 | } |
| 418 | |
| 419 | // Strip other Parsoid-generated values |
| 420 | // |
| 421 | // FIXME: Given that we are currently escaping about/typeof keys |
| 422 | // that show up in wikitext, we could unconditionally strip these |
| 423 | // away right now. |
| 424 | $parsoidValueRegExp = self::PARSOID_ATTRIBUTES[$k] ?? null; |
| 425 | if ( $parsoidValueRegExp && preg_match( $parsoidValueRegExp, $kv->v ) ) { |
| 426 | $v = preg_replace( $parsoidValueRegExp, '', $kv->v ); |
| 427 | if ( $v ) { |
| 428 | $out[] = $k . '="' . $v . '"'; |
| 429 | } |
| 430 | continue; |
| 431 | } |
| 432 | |
| 433 | if ( strlen( $k ) > 0 ) { |
| 434 | $vInfo = $token->getAttributeShadowInfo( $k ); |
| 435 | $v = $vInfo['value']; |
| 436 | // Deal with k/v's that were template-generated |
| 437 | $kk = $this->getAttributeKey( $node, $k ); |
| 438 | // Pass in $k, not $kk since $kk can potentially |
| 439 | // be original wikitext source for 'k' rather than |
| 440 | // the string value of the key. |
| 441 | $vv = $this->getAttributeValue( $node, $k ) ?? $v; |
| 442 | // Remove encapsulation from protected attributes |
| 443 | // in pegTokenizer.pegjs:generic_newline_attribute |
| 444 | $kk = preg_replace( '/^data-x-/i', '', $kk, 1 ); |
| 445 | // PORT-FIXME: is this type safe? $vv could be a ConstrainedText |
| 446 | if ( $vv !== null && strlen( $vv ) > 0 ) { |
| 447 | if ( !$vInfo['fromsrc'] && !$isWt ) { |
| 448 | // Escape wikitext entities |
| 449 | $vv = str_replace( '>', '>', Utils::escapeWtEntities( $vv ) ); |
| 450 | } |
| 451 | $out[] = $kk . '="' . str_replace( '"', '"', $vv ) . '"'; |
| 452 | } elseif ( preg_match( '/[{<]/', $kk ) ) { |
| 453 | // Templated, <*include*>, or <ext-tag> generated |
| 454 | $out[] = $kk; |
| 455 | } else { |
| 456 | $out[] = $kk . '=""'; |
| 457 | } |
| 458 | continue; |
| 459 | // PORT-FIXME: is this type safe? $k->v could be a Token or Token array |
| 460 | } elseif ( strlen( $kv->v ) ) { |
| 461 | // not very likely.. |
| 462 | $out[] = $kv->v; |
| 463 | } |
| 464 | } |
| 465 | |
| 466 | // SSS FIXME: It can be reasonably argued that we can permanently delete |
| 467 | // dangerous and unacceptable attributes in the interest of safety/security |
| 468 | // and the resultant dirty diffs should be acceptable. But, this is |
| 469 | // something to do in the future once we have passed the initial tests |
| 470 | // of parsoid acceptance. |
| 471 | // |
| 472 | // 'a' data attribs -- look for attributes that were removed |
| 473 | // as part of sanitization and add them back |
| 474 | $dataParsoid = $token->dataParsoid; |
| 475 | if ( isset( $dataParsoid->a ) && isset( $dataParsoid->sa ) ) { |
| 476 | $aKeys = array_keys( $dataParsoid->a ); |
| 477 | foreach ( $aKeys as $k ) { |
| 478 | // Attrib not present -- sanitized away! |
| 479 | if ( !KV::lookupKV( $attribs, (string)$k ) ) { |
| 480 | $v = $dataParsoid->sa[$k] ?? null; |
| 481 | // PORT-FIXME check type |
| 482 | if ( $v !== null && $v !== '' ) { |
| 483 | $out[] = $k . '="' . str_replace( '"', '"', $v ) . '"'; |
| 484 | } else { |
| 485 | // at least preserve the key |
| 486 | $out[] = $k; |
| 487 | } |
| 488 | } |
| 489 | } |
| 490 | } |
| 491 | // XXX: round-trip optional whitespace / line breaks etc |
| 492 | return implode( ' ', $out ); |
| 493 | } |
| 494 | |
| 495 | /** |
| 496 | * FIXME: Get rid of this function after content version 2.2.0 has expired from caches. |
| 497 | * |
| 498 | * @param Element $node |
| 499 | */ |
| 500 | public function handleLIHackIfApplicable( Element $node ): void { |
| 501 | $liHackSrc = DOMDataUtils::getDataParsoid( $node )->liHackSrc ?? null; |
| 502 | $prev = DiffDOMUtils::previousNonSepSibling( $node ); |
| 503 | |
| 504 | // If we are dealing with an LI hack, then we must ensure that |
| 505 | // we are dealing with either |
| 506 | // |
| 507 | // 1. A node with no previous sibling inside of a list. |
| 508 | // |
| 509 | // 2. A node whose previous sibling is a list element. |
| 510 | if ( $liHackSrc !== null |
| 511 | // Case 1 |
| 512 | && ( ( $prev === null && DOMUtils::isList( $node->parentNode ) ) |
| 513 | // Case 2 |
| 514 | || ( $prev !== null && DOMUtils::isListItem( $prev ) ) ) |
| 515 | ) { |
| 516 | $this->state->emitChunk( $liHackSrc, $node ); |
| 517 | } |
| 518 | } |
| 519 | |
| 520 | private function formatStringSubst( string $format, string $value, bool $forceTrim ): string { |
| 521 | // PORT-FIXME: JS is more agressive and removes various unicode whitespaces |
| 522 | // (most notably nbsp). Does that matter? |
| 523 | if ( $forceTrim ) { |
| 524 | $value = trim( $value ); |
| 525 | } |
| 526 | return preg_replace_callback( '/_+/', static function ( $m ) use ( $value ) { |
| 527 | if ( $value === '' ) { |
| 528 | return $value; |
| 529 | } |
| 530 | $hole = $m[0]; |
| 531 | $holeLen = strlen( $hole ); |
| 532 | $valueLen = mb_strlen( $value ); |
| 533 | return $holeLen <= $valueLen ? $value : $value . str_repeat( ' ', $holeLen - $valueLen ); |
| 534 | }, $format, 1 ); |
| 535 | } |
| 536 | |
| 537 | /** |
| 538 | * Generates a template parameter sort function that tries to preserve existing ordering |
| 539 | * but also to follow the order prescribed by the templatedata. |
| 540 | * @param array $dpArgInfo |
| 541 | * @param ?array $tplData |
| 542 | * @param array $dataMwKeys |
| 543 | * @return Closure |
| 544 | */ |
| 545 | private function createParamComparator( |
| 546 | array $dpArgInfo, ?array $tplData, array $dataMwKeys |
| 547 | ): Closure { |
| 548 | // Record order of parameters in new data-mw |
| 549 | $newOrder = []; |
| 550 | foreach ( $dataMwKeys as $i => $key ) { |
| 551 | $newOrder[$key] = [ 'order' => $i ]; |
| 552 | } |
| 553 | // Record order of parameters in templatedata (if present) |
| 554 | $tplDataOrder = []; |
| 555 | $aliasMap = []; |
| 556 | $keys = []; |
| 557 | if ( $tplData && isset( $tplData['paramOrder'] ) ) { |
| 558 | foreach ( $tplData['paramOrder'] as $i => $key ) { |
| 559 | $tplDataOrder[$key] = [ 'order' => $i ]; |
| 560 | $aliasMap[$key] = [ 'key' => $key, 'order' => -1 ]; |
| 561 | $keys[] = $key; |
| 562 | // Aliases have the same sort order as the main name. |
| 563 | $aliases = $tplData['params'][$key]['aliases'] ?? []; |
| 564 | foreach ( $aliases as $j => $alias ) { |
| 565 | $aliasMap[$alias] = [ 'key' => $key, 'order' => $j ]; |
| 566 | } |
| 567 | } |
| 568 | } |
| 569 | // Record order of parameters in original wikitext (from data-parsoid) |
| 570 | $origOrder = []; |
| 571 | foreach ( $dpArgInfo as $i => $argInfo ) { |
| 572 | $origOrder[$argInfo->k] = [ 'order' => $i, 'dist' => 0 ]; |
| 573 | } |
| 574 | // Canonical parameter key gets the same order as an alias parameter |
| 575 | // found in the original wikitext. |
| 576 | foreach ( $dpArgInfo as $i => $argInfo ) { |
| 577 | $canon = $aliasMap[$argInfo->k] ?? null; |
| 578 | if ( $canon !== null && !array_key_exists( $canon['key'], $origOrder ) ) { |
| 579 | $origOrder[$canon['key']] = $origOrder[$argInfo->k]; |
| 580 | } |
| 581 | } |
| 582 | // Find the closest "original parameter" for each templatedata parameter, |
| 583 | // so that newly-added parameters are placed near the parameters which |
| 584 | // templatedata says they should be adjacent to. |
| 585 | $nearestOrder = $origOrder; |
| 586 | $reduceF = static function ( $acc, $val ) use ( &$origOrder, &$nearestOrder ) { |
| 587 | if ( isset( $origOrder[$val] ) ) { |
| 588 | $acc = $origOrder[$val]; |
| 589 | } |
| 590 | if ( !( isset( $nearestOrder[$val] ) && $nearestOrder[$val]['dist'] < $acc['dist'] ) ) { |
| 591 | $nearestOrder[$val] = $acc; |
| 592 | } |
| 593 | return [ 'order' => $acc['order'], 'dist' => $acc['dist'] + 1 ]; |
| 594 | }; |
| 595 | // Find closest original parameter before the key. |
| 596 | // @phan-suppress-next-line PhanPluginUseReturnValueInternalKnown |
| 597 | array_reduce( $keys, $reduceF, [ 'order' => -1, 'dist' => 2 * count( $keys ) ] ); |
| 598 | // Find closest original parameter after the key. |
| 599 | // @phan-suppress-next-line PhanPluginUseReturnValueInternalKnown |
| 600 | array_reduce( array_reverse( $keys ), $reduceF, |
| 601 | [ 'order' => count( $origOrder ), 'dist' => count( $keys ) ] ); |
| 602 | |
| 603 | // Helper function to return a large number if the given key isn't |
| 604 | // in the sort order map |
| 605 | $big = max( count( $nearestOrder ), count( $newOrder ) ); |
| 606 | $defaultGet = static function ( $map, $key1, $key2 = null ) use ( &$big ) { |
| 607 | $key = ( !$key2 || isset( $map[$key1] ) ) ? $key1 : $key2; |
| 608 | return $map[$key]['order'] ?? $big; |
| 609 | }; |
| 610 | |
| 611 | return static function ( $a, $b ) use ( |
| 612 | &$aliasMap, &$defaultGet, &$nearestOrder, &$tplDataOrder, &$newOrder |
| 613 | ) { |
| 614 | $aCanon = $aliasMap[$a] ?? [ 'key' => $a, 'order' => -1 ]; |
| 615 | $bCanon = $aliasMap[$b] ?? [ 'key' => $b, 'order' => -1 ]; |
| 616 | // primary key is `nearestOrder` (nearest original parameter) |
| 617 | $aOrder = $defaultGet( $nearestOrder, $a, $aCanon['key'] ); |
| 618 | $bOrder = $defaultGet( $nearestOrder, $b, $bCanon['key'] ); |
| 619 | if ( $aOrder !== $bOrder ) { |
| 620 | return $aOrder - $bOrder; |
| 621 | } |
| 622 | // secondary key is templatedata order |
| 623 | if ( $aCanon['key'] === $bCanon['key'] ) { |
| 624 | return $aCanon['order'] - $bCanon['order']; |
| 625 | } |
| 626 | $aOrder = $defaultGet( $tplDataOrder, $aCanon['key'] ); |
| 627 | $bOrder = $defaultGet( $tplDataOrder, $bCanon['key'] ); |
| 628 | if ( $aOrder !== $bOrder ) { |
| 629 | return $aOrder - $bOrder; |
| 630 | } |
| 631 | // tertiary key is original input order (makes sort stable) |
| 632 | $aOrder = $defaultGet( $newOrder, $a ); |
| 633 | $bOrder = $defaultGet( $newOrder, $b ); |
| 634 | return $aOrder - $bOrder; |
| 635 | }; |
| 636 | } |
| 637 | |
| 638 | /** |
| 639 | * Serialize part of a templatelike expression. |
| 640 | * @param SerializerState $state |
| 641 | * @param string $buf |
| 642 | * @param Element $node |
| 643 | * @param string $type The type of the part to be serialized. One of template, templatearg, |
| 644 | * parserfunction. |
| 645 | * @param stdClass $part The expression fragment to serialize. See $srcParts |
| 646 | * in serializeFromParts() for format. |
| 647 | * @param ?array $tplData Templatedata, see |
| 648 | * https://github.com/wikimedia/mediawiki-extensions-TemplateData/blob/master/Specification.md |
| 649 | * @param mixed $prevPart Previous part. See $srcParts in serializeFromParts(). |
| 650 | * @param mixed $nextPart Next part. See $srcParts in serializeFromParts(). |
| 651 | * @return string |
| 652 | */ |
| 653 | private function serializePart( |
| 654 | SerializerState $state, string $buf, Element $node, string $type, stdClass $part, |
| 655 | ?array $tplData, $prevPart, $nextPart |
| 656 | ): string { |
| 657 | // Parse custom format specification, if present. |
| 658 | $defaultBlockSpc = "{{_\n| _ = _\n}}"; // "block" |
| 659 | $defaultInlineSpc = '{{_|_=_}}'; // "inline" |
| 660 | |
| 661 | $format = isset( $tplData['format'] ) ? strtolower( $tplData['format'] ) : null; |
| 662 | if ( $format === 'block' ) { |
| 663 | $format = $defaultBlockSpc; |
| 664 | } elseif ( $format === 'inline' ) { |
| 665 | $format = $defaultInlineSpc; |
| 666 | } |
| 667 | // Check format string for validity. |
| 668 | preg_match( self::FORMATSTRING_REGEXP, $format ?? '', $parsedFormat ); |
| 669 | if ( !$parsedFormat ) { |
| 670 | preg_match( self::FORMATSTRING_REGEXP, $defaultInlineSpc, $parsedFormat ); |
| 671 | $format = null; // Indicates that no valid custom format was present. |
| 672 | } |
| 673 | $formatSOL = $parsedFormat[1] ?? ''; |
| 674 | $formatStart = $parsedFormat[2] ?? ''; |
| 675 | $formatParamName = $parsedFormat[3] ?? ''; |
| 676 | $formatParamValue = $parsedFormat[4] ?? ''; |
| 677 | $formatEnd = $parsedFormat[5] ?? ''; |
| 678 | $formatEOL = $parsedFormat[6] ?? ''; |
| 679 | $forceTrim = ( $format !== null ) || WTUtils::isNewElt( $node ); |
| 680 | |
| 681 | // Shoehorn formatting of top-level templatearg wikitext into this code. |
| 682 | if ( $type === 'templatearg' ) { |
| 683 | $formatStart = preg_replace( '/{{/', '{{{', $formatStart, 1 ); |
| 684 | $formatEnd = preg_replace( '/}}/', '}}}', $formatEnd, 1 ); |
| 685 | } |
| 686 | |
| 687 | // handle SOL newline requirement |
| 688 | if ( $formatSOL && !str_ends_with( ( $prevPart !== null ) ? $buf : ( $state->sep->src ?? '' ), "\n" ) ) { |
| 689 | $buf .= "\n"; |
| 690 | } |
| 691 | |
| 692 | // open the transclusion |
| 693 | $tgt = $part->target; |
| 694 | '@phan-var stdClass $tgt'; |
| 695 | $buf .= $this->formatStringSubst( $formatStart, $tgt->wt, $forceTrim ); |
| 696 | |
| 697 | // Short-circuit transclusions without params |
| 698 | $paramKeys = array_keys( get_object_vars( $part->params ) ); |
| 699 | if ( !$paramKeys ) { |
| 700 | if ( substr( $formatEnd, 0, 1 ) === "\n" ) { |
| 701 | $formatEnd = substr( $formatEnd, 1 ); |
| 702 | } |
| 703 | return $buf . $formatEnd; |
| 704 | } |
| 705 | |
| 706 | // Trim whitespace from data-mw keys to deal with non-compliant |
| 707 | // clients. Make sure param info is accessible for the stripped key |
| 708 | // since later code will be using the stripped key always. |
| 709 | $tplKeysFromDataMw = array_map( static function ( $key ) use ( $part ) { |
| 710 | $strippedKey = trim( (string)$key ); |
| 711 | if ( $key !== $strippedKey ) { |
| 712 | $part->params->{$strippedKey} = $part->params->{$key}; |
| 713 | } |
| 714 | return $strippedKey; |
| 715 | }, $paramKeys ); |
| 716 | |
| 717 | // Per-parameter info from data-parsoid for pre-existing parameters |
| 718 | $dp = DOMDataUtils::getDataParsoid( $node ); |
| 719 | // Account for clients not setting the `i`, see T238721 |
| 720 | $dpArgInfo = isset( $part->i ) ? ( $dp->pi[$part->i] ?? [] ) : []; |
| 721 | |
| 722 | // Build a key -> arg info map |
| 723 | $dpArgInfoMap = []; |
| 724 | foreach ( $dpArgInfo as $info ) { |
| 725 | $dpArgInfoMap[$info->k] = $info; |
| 726 | } |
| 727 | |
| 728 | // 1. Process all parameters and build a map of |
| 729 | // arg-name -> [serializeAsNamed, name, value] |
| 730 | // |
| 731 | // 2. Serialize tpl args in required order |
| 732 | // |
| 733 | // 3. Format them according to formatParamName/formatParamValue |
| 734 | |
| 735 | $kvMap = []; |
| 736 | foreach ( $tplKeysFromDataMw as $key ) { |
| 737 | $param = $part->params->{$key}; |
| 738 | $argInfo = $dpArgInfoMap[$key] ?? []; |
| 739 | |
| 740 | // TODO: Other formats? |
| 741 | // Only consider the html parameter if the wikitext one |
| 742 | // isn't present at all. If it's present but empty, |
| 743 | // that's still considered a valid parameter. |
| 744 | if ( property_exists( $param, 'wt' ) ) { |
| 745 | $value = $param->wt; |
| 746 | } elseif ( property_exists( $param, 'html' ) ) { |
| 747 | $value = $this->htmlToWikitext( [ 'env' => $this->env ], $param->html ); |
| 748 | } else { |
| 749 | $this->env->log( |
| 750 | 'error', |
| 751 | "params in data-mw part is missing wt/html for $key. " . |
| 752 | "Serializing as empty string.", |
| 753 | "data-mw part: " . json_encode( $part ) |
| 754 | ); |
| 755 | $value = ""; |
| 756 | } |
| 757 | |
| 758 | Assert::invariant( is_string( $value ), "For param: $key, wt property should be a string ' |
| 759 | . 'but got: $value" ); |
| 760 | |
| 761 | $serializeAsNamed = !empty( $argInfo->named ); |
| 762 | |
| 763 | // The name is usually equal to the parameter key, but |
| 764 | // if there's a key->wt attribute, use that. |
| 765 | $name = null; |
| 766 | if ( isset( $param->key->wt ) ) { |
| 767 | $name = $param->key->wt; |
| 768 | // And make it appear even if there wasn't any data-parsoid information. |
| 769 | $serializeAsNamed = true; |
| 770 | } else { |
| 771 | $name = $key; |
| 772 | } |
| 773 | |
| 774 | // Use 'k' as the key, not 'name'. |
| 775 | // |
| 776 | // The normalized form of 'k' is used as the key in both |
| 777 | // data-parsoid and data-mw. The full non-normalized form |
| 778 | // is present in '$param->key->wt' |
| 779 | $kvMap[$key] = [ 'serializeAsNamed' => $serializeAsNamed, 'name' => $name, 'value' => $value ]; |
| 780 | } |
| 781 | |
| 782 | $argOrder = array_keys( $kvMap ); |
| 783 | usort( $argOrder, $this->createParamComparator( $dpArgInfo, $tplData, $argOrder ) ); |
| 784 | |
| 785 | $argIndex = 1; |
| 786 | $numericIndex = 1; |
| 787 | |
| 788 | $numPositionalArgs = 0; |
| 789 | foreach ( $dpArgInfo as $pi ) { |
| 790 | if ( isset( $part->params->{$pi->k} ) && empty( $pi->named ) ) { |
| 791 | $numPositionalArgs++; |
| 792 | } |
| 793 | } |
| 794 | |
| 795 | $argBuf = []; |
| 796 | foreach ( $argOrder as $param ) { |
| 797 | $kv = $kvMap[$param]; |
| 798 | // Add nowiki escapes for the arg value, as required |
| 799 | $escapedValue = $this->wteHandlers->escapeTplArgWT( $kv['value'], [ |
| 800 | 'serializeAsNamed' => $kv['serializeAsNamed'] || $param !== $numericIndex, |
| 801 | 'type' => $type, |
| 802 | 'argPositionalIndex' => $numericIndex, |
| 803 | 'numPositionalArgs' => $numPositionalArgs, |
| 804 | 'argIndex' => $argIndex++, |
| 805 | 'numArgs' => count( $tplKeysFromDataMw ), |
| 806 | ] ); |
| 807 | if ( $escapedValue['serializeAsNamed'] ) { |
| 808 | // WS trimming for values of named args |
| 809 | $argBuf[] = [ 'dpKey' => $param, 'name' => $kv['name'], 'value' => trim( $escapedValue['v'] ) ]; |
| 810 | } else { |
| 811 | $numericIndex++; |
| 812 | // No WS trimming for positional args |
| 813 | $argBuf[] = [ 'dpKey' => $param, 'name' => null, 'value' => $escapedValue['v'] ]; |
| 814 | } |
| 815 | } |
| 816 | |
| 817 | // If no explicit format is provided, default format is: |
| 818 | // - 'inline' for new args |
| 819 | // - whatever format is available from data-parsoid for old args |
| 820 | // (aka, overriding formatParamName/formatParamValue) |
| 821 | // |
| 822 | // If an unedited node OR if paramFormat is unspecified, |
| 823 | // this strategy prevents unnecessary normalization |
| 824 | // of edited transclusions which don't have valid |
| 825 | // templatedata formatting information. |
| 826 | |
| 827 | // "magic case": If the format string ends with a newline, an extra newline is added |
| 828 | // between the template name and the first parameter. |
| 829 | |
| 830 | foreach ( $argBuf as $arg ) { |
| 831 | $name = $arg['name']; |
| 832 | $val = $arg['value']; |
| 833 | if ( $name === null ) { |
| 834 | // We are serializing a positional parameter. |
| 835 | // Whitespace is significant for these and |
| 836 | // formatting would change semantics. |
| 837 | $name = ''; |
| 838 | $modFormatParamName = '|_'; |
| 839 | $modFormatParamValue = '_'; |
| 840 | } elseif ( $name === '' ) { |
| 841 | // No spacing for blank parameters ({{foo|=bar}}) |
| 842 | // This should be an edge case and probably only for |
| 843 | // inline-formatted templates, but we are consciously |
| 844 | // forcing this default here. Can revisit if this is |
| 845 | // ever a problem. |
| 846 | $modFormatParamName = '|_='; |
| 847 | $modFormatParamValue = '_'; |
| 848 | } else { |
| 849 | // Preserve existing spacing, esp if there was a comment |
| 850 | // embedded in it. Otherwise, follow TemplateData's lead. |
| 851 | // NOTE: In either case, we are forcibly normalizing |
| 852 | // non-block-formatted transclusions into block formats |
| 853 | // by adding missing newlines. |
| 854 | $spc = $dpArgInfoMap[$arg['dpKey']]->spc ?? null; |
| 855 | if ( $spc && ( !$format || preg_match( Utils::COMMENT_REGEXP, $spc[3] ?? '' ) ) ) { |
| 856 | $nl = ( substr( $formatParamName, 0, 1 ) === "\n" ) ? "\n" : ''; |
| 857 | $modFormatParamName = $nl . '|' . $spc[0] . '_' . $spc[1] . '=' . $spc[2]; |
| 858 | $modFormatParamValue = '_' . $spc[3]; |
| 859 | } else { |
| 860 | $modFormatParamName = $formatParamName; |
| 861 | $modFormatParamValue = $formatParamValue; |
| 862 | } |
| 863 | } |
| 864 | |
| 865 | // Don't create duplicate newlines. |
| 866 | $trailing = preg_match( self::TRAILING_COMMENT_OR_WS_AFTER_NL_REGEXP, $buf ); |
| 867 | if ( $trailing && substr( $formatParamName, 0, 1 ) === "\n" ) { |
| 868 | $modFormatParamName = substr( $formatParamName, 1 ); |
| 869 | } |
| 870 | |
| 871 | $buf .= $this->formatStringSubst( $modFormatParamName, $name, $forceTrim ); |
| 872 | $buf .= $this->formatStringSubst( $modFormatParamValue, $val, $forceTrim ); |
| 873 | } |
| 874 | |
| 875 | // Don't create duplicate newlines. |
| 876 | if ( preg_match( self::TRAILING_COMMENT_OR_WS_AFTER_NL_REGEXP, $buf ) |
| 877 | && substr( $formatEnd, 0, 1 ) === "\n" |
| 878 | ) { |
| 879 | $buf .= substr( $formatEnd, 1 ); |
| 880 | } else { |
| 881 | $buf .= $formatEnd; |
| 882 | } |
| 883 | |
| 884 | if ( $formatEOL ) { |
| 885 | if ( $nextPart === null ) { |
| 886 | // This is the last part of the block. Add the \n only |
| 887 | // if the next non-comment node is not a text node |
| 888 | // of if the text node doesn't have a leading \n. |
| 889 | $next = DiffDOMUtils::nextNonDeletedSibling( $node ); |
| 890 | while ( $next instanceof Comment ) { |
| 891 | $next = DiffDOMUtils::nextNonDeletedSibling( $next ); |
| 892 | } |
| 893 | if ( !( $next instanceof Text ) || substr( $next->nodeValue, 0, 1 ) !== "\n" ) { |
| 894 | $buf .= "\n"; |
| 895 | } |
| 896 | } elseif ( !is_string( $nextPart ) || substr( $nextPart, 0, 1 ) !== "\n" ) { |
| 897 | // If nextPart is another template, and it wants a leading nl, |
| 898 | // this \n we add here will count towards that because of the |
| 899 | // formatSOL check at the top. |
| 900 | $buf .= "\n"; |
| 901 | } |
| 902 | } |
| 903 | |
| 904 | return $buf; |
| 905 | } |
| 906 | |
| 907 | /** |
| 908 | * Serialize a template from its parts. |
| 909 | * @param SerializerState $state |
| 910 | * @param Element $node |
| 911 | * @param list<stdClass|string> $srcParts Template parts from TemplateInfo::getDataMw() |
| 912 | * @return string |
| 913 | */ |
| 914 | public function serializeFromParts( |
| 915 | SerializerState $state, Element $node, array $srcParts |
| 916 | ): string { |
| 917 | $useTplData = WTUtils::isNewElt( $node ) || DiffUtils::hasDiffMarkers( $node ); |
| 918 | $buf = ''; |
| 919 | foreach ( $srcParts as $i => $part ) { |
| 920 | if ( is_string( $part ) ) { |
| 921 | $buf .= $part; |
| 922 | continue; |
| 923 | } |
| 924 | |
| 925 | $prevPart = $srcParts[$i - 1] ?? null; |
| 926 | $nextPart = $srcParts[$i + 1] ?? null; |
| 927 | |
| 928 | $isTplArg = isset( $part->templatearg ); |
| 929 | $tpl = $part->templatearg ?? $part->template ?? null; |
| 930 | |
| 931 | if ( !isset( $tpl->target->wt ) ) { |
| 932 | // Maybe we should just raise a ClientError |
| 933 | $this->env->log( 'error', 'data-mw.parts array is malformed: ', |
| 934 | DOMCompat::getOuterHTML( $node ), PHPUtils::jsonEncode( $srcParts ) ); |
| 935 | continue; |
| 936 | } |
| 937 | |
| 938 | // Account for clients leaving off the params array, presumably when empty. |
| 939 | // See T291741 |
| 940 | $tpl->params ??= (object)[]; |
| 941 | |
| 942 | if ( $isTplArg ) { |
| 943 | $buf = $this->serializePart( |
| 944 | $state, $buf, $node, 'templatearg', $tpl, null, $prevPart, |
| 945 | $nextPart |
| 946 | ); |
| 947 | continue; |
| 948 | } |
| 949 | |
| 950 | // transclusion: tpl or parser function |
| 951 | $tplHref = $tpl->target->href ?? null; |
| 952 | $isTpl = is_string( $tplHref ); |
| 953 | $type = $isTpl ? 'template' : 'parserfunction'; |
| 954 | |
| 955 | // While the API supports fetching multiple template data objects in one call, |
| 956 | // we will fetch one at a time to benefit from cached responses. |
| 957 | // |
| 958 | // Fetch template data for the template |
| 959 | $tplData = null; |
| 960 | $apiResp = null; |
| 961 | if ( $isTpl && $useTplData ) { |
| 962 | try { |
| 963 | $title = Title::newFromText( |
| 964 | PHPUtils::stripPrefix( Utils::decodeURIComponent( $tplHref ), './' ), |
| 965 | $this->env->getSiteConfig() |
| 966 | ); |
| 967 | $tplData = $this->env->getDataAccess()->fetchTemplateData( $this->env->getPageConfig(), $title ); |
| 968 | } catch ( Exception $err ) { |
| 969 | // Log the error, and use default serialization mode. |
| 970 | // Better to misformat a transclusion than to lose an edit. |
| 971 | $this->env->log( 'error/html2wt/tpldata', $err ); |
| 972 | } |
| 973 | } |
| 974 | // If the template doesn't exist, or does but has no TemplateData, ignore it |
| 975 | if ( !empty( $tplData['missing'] ) || !empty( $tplData['notemplatedata'] ) ) { |
| 976 | $tplData = null; |
| 977 | } |
| 978 | $buf = $this->serializePart( $state, $buf, $node, $type, $tpl, $tplData, $prevPart, $nextPart ); |
| 979 | } |
| 980 | return $buf; |
| 981 | } |
| 982 | |
| 983 | public function serializeExtensionStartTag( Element $node, SerializerState $state ): string { |
| 984 | $dataMw = DOMDataUtils::getDataMw( $node ); |
| 985 | $extTagName = $dataMw->name; |
| 986 | |
| 987 | // Serialize extension attributes in normalized form as: |
| 988 | // key='value' |
| 989 | // FIXME: with no dataParsoid, shadow info will mark it as new |
| 990 | $attrs = (array)( $dataMw->attrs ?? [] ); |
| 991 | $extTok = new TagTk( $extTagName, array_map( static function ( $key ) use ( $attrs ) { |
| 992 | return new KV( $key, $attrs[$key] ); |
| 993 | }, array_keys( $attrs ) ) ); |
| 994 | |
| 995 | $about = DOMCompat::getAttribute( $node, 'about' ); |
| 996 | if ( $about !== null ) { |
| 997 | $extTok->addAttribute( 'about', $about ); |
| 998 | } |
| 999 | $typeof = DOMCompat::getAttribute( $node, 'typeof' ); |
| 1000 | if ( $typeof !== null ) { |
| 1001 | $extTok->addAttribute( 'typeof', $typeof ); |
| 1002 | } |
| 1003 | |
| 1004 | $attrStr = $this->serializeAttributes( $node, $extTok ); |
| 1005 | $src = '<' . $extTagName; |
| 1006 | if ( $attrStr ) { |
| 1007 | $src .= ' ' . $attrStr; |
| 1008 | } |
| 1009 | return $src . ( !empty( $dataMw->body ) ? '>' : ' />' ); |
| 1010 | } |
| 1011 | |
| 1012 | public function defaultExtensionHandler( Element $node, SerializerState $state ): string { |
| 1013 | $dp = DOMDataUtils::getDataParsoid( $node ); |
| 1014 | $dataMw = DOMDataUtils::getDataMw( $node ); |
| 1015 | $src = $this->serializeExtensionStartTag( $node, $state ); |
| 1016 | if ( !isset( $dataMw->body ) ) { |
| 1017 | return $src; // We self-closed this already. |
| 1018 | } elseif ( is_string( $dataMw->body->extsrc ?? null ) ) { |
| 1019 | $src .= $dataMw->body->extsrc; |
| 1020 | } elseif ( isset( $dp->src ) ) { |
| 1021 | $this->env->log( |
| 1022 | 'error/html2wt/ext', |
| 1023 | 'Extension data-mw missing for: ' . DOMCompat::getOuterHTML( $node ) |
| 1024 | ); |
| 1025 | return $dp->src; |
| 1026 | } else { |
| 1027 | $this->env->log( |
| 1028 | 'error/html2wt/ext', |
| 1029 | 'Extension src unavailable for: ' . DOMCompat::getOuterHTML( $node ) |
| 1030 | ); |
| 1031 | } |
| 1032 | return $src . '</' . $dataMw->name . '>'; |
| 1033 | } |
| 1034 | |
| 1035 | /** |
| 1036 | * Consolidate separator handling when emitting text. |
| 1037 | * @param string $res |
| 1038 | * @param Node $node |
| 1039 | */ |
| 1040 | private function serializeText( string $res, Node $node ): void { |
| 1041 | $state = $this->state; |
| 1042 | |
| 1043 | // Deal with trailing separator-like text (at least 1 newline and other whitespace) |
| 1044 | preg_match( self::$separatorREs['sepSuffixWithNlsRE'], $res, $newSepMatch ); |
| 1045 | $res = preg_replace( self::$separatorREs['sepSuffixWithNlsRE'], '', $res, 1 ); |
| 1046 | |
| 1047 | if ( !$state->inIndentPre ) { |
| 1048 | // Strip leading newlines and other whitespace |
| 1049 | if ( preg_match( self::$separatorREs['sepPrefixWithNlsRE'], $res, $match ) ) { |
| 1050 | $state->appendSep( $match[0] ); |
| 1051 | $res = substr( $res, strlen( $match[0] ) ); |
| 1052 | } |
| 1053 | } |
| 1054 | |
| 1055 | if ( $state->needsEscaping ) { |
| 1056 | $res = Utils::escapeWtEntities( $res ); |
| 1057 | } |
| 1058 | $state->emitChunk( $res, $node ); |
| 1059 | |
| 1060 | // Move trailing newlines into the next separator |
| 1061 | if ( $newSepMatch ) { |
| 1062 | if ( !$state->sep->src ) { |
| 1063 | $state->appendSep( $newSepMatch[0] ); |
| 1064 | } else { |
| 1065 | /* SSS FIXME: what are we doing with the stripped NLs?? */ |
| 1066 | } |
| 1067 | } |
| 1068 | } |
| 1069 | |
| 1070 | /** |
| 1071 | * Serialize the content of a text node |
| 1072 | * @param Node $node |
| 1073 | * @return Node|null |
| 1074 | */ |
| 1075 | private function serializeTextNode( Node $node ): ?Node { |
| 1076 | $this->state->needsEscaping = true; |
| 1077 | $this->serializeText( $node->nodeValue, $node ); |
| 1078 | $this->state->needsEscaping = false; |
| 1079 | return $node->nextSibling; |
| 1080 | } |
| 1081 | |
| 1082 | /** |
| 1083 | * Emit non-separator wikitext that does not need to be escaped. |
| 1084 | * @param string $res |
| 1085 | * @param Node $node |
| 1086 | */ |
| 1087 | public function emitWikitext( string $res, Node $node ): void { |
| 1088 | $this->serializeText( $res, $node ); |
| 1089 | } |
| 1090 | |
| 1091 | /** |
| 1092 | * DOM-based serialization |
| 1093 | * @param Element $node |
| 1094 | * @param DOMHandler $domHandler |
| 1095 | * @return Node|null |
| 1096 | */ |
| 1097 | private function serializeNodeInternal( Element $node, DOMHandler $domHandler ) { |
| 1098 | // To serialize a node from source, the node should satisfy these |
| 1099 | // conditions: |
| 1100 | // |
| 1101 | // 1. It should not have a diff marker or be in a modified subtree |
| 1102 | // WTS should not be in a subtree with a modification flag that |
| 1103 | // applies to every node of a subtree (rather than an indication |
| 1104 | // that some node in the subtree is modified). |
| 1105 | // |
| 1106 | // 2. It should continue to be valid in any surrounding edited context |
| 1107 | // For some nodes, modification of surrounding context |
| 1108 | // can change serialized output of this node |
| 1109 | // (ex: <td>s and whether you emit | or || for them) |
| 1110 | // |
| 1111 | // 3. It should have valid, usable DSR |
| 1112 | // |
| 1113 | // 4. Either it has non-zero positive DSR width, or meets one of the |
| 1114 | // following: |
| 1115 | // |
| 1116 | // 4a. It is content like <p><br/><p> or an automatically-inserted |
| 1117 | // wikitext <references/> (HTML <ol>) (will have dsr-width 0) |
| 1118 | // 4b. it is fostered content (will have dsr-width 0) |
| 1119 | // 4c. it is misnested content (will have dsr-width 0) |
| 1120 | // |
| 1121 | // SSS FIXME: Additionally, we can guard against buggy DSR with |
| 1122 | // some validity checks. We can test that non-sep src content |
| 1123 | // leading wikitext markup corresponds to the node type. |
| 1124 | // |
| 1125 | // Ex: If node.nodeName is 'UL', then src[0] should be '*' |
| 1126 | // |
| 1127 | // TO BE DONE |
| 1128 | |
| 1129 | $state = $this->state; |
| 1130 | $wrapperUnmodified = false; |
| 1131 | $dp = DOMDataUtils::getDataParsoid( $node ); |
| 1132 | |
| 1133 | if ( $state->selserMode |
| 1134 | && !$state->inInsertedContent |
| 1135 | && WTSUtils::origSrcValidInEditedContext( $state, $node ) |
| 1136 | && Utils::isValidDSR( $dp->dsr ?? null ) |
| 1137 | && ( $dp->dsr->end > $dp->dsr->start |
| 1138 | // FIXME: <p><br/></p> |
| 1139 | // nodes that have dsr width 0 because currently, |
| 1140 | // we emit newlines outside the p-nodes. So, this check |
| 1141 | // tries to handle that scenario. |
| 1142 | || ( |
| 1143 | $dp->dsr->end === $dp->dsr->start && ( |
| 1144 | in_array( DOMCompat::nodeName( $node ), [ 'p', 'br' ], true ) |
| 1145 | || !empty( DOMDataUtils::getDataMw( $node )->autoGenerated ) |
| 1146 | // FIXME: This is only necessary while outputContentVersion |
| 1147 | // 2.1.2 - 2.2.0 are still valid |
| 1148 | || DOMUtils::hasTypeOf( $node, 'mw:Placeholder/StrippedTag' ) |
| 1149 | ) |
| 1150 | ) |
| 1151 | || !empty( $dp->fostered ) |
| 1152 | || !empty( $dp->misnested ) |
| 1153 | ) |
| 1154 | ) { |
| 1155 | if ( !DiffUtils::hasDiffMarkers( $node ) ) { |
| 1156 | // If this HTML node will disappear in wikitext because of |
| 1157 | // zero width, then the separator constraints will carry over |
| 1158 | // to the node's children. |
| 1159 | // |
| 1160 | // Since we dont recurse into 'node' in selser mode, we update the |
| 1161 | // separator constraintInfo to apply to 'node' and its first child. |
| 1162 | // |
| 1163 | // We could clear constraintInfo altogether which would be |
| 1164 | // correct (but could normalize separators and introduce dirty |
| 1165 | // diffs unnecessarily). |
| 1166 | |
| 1167 | $state->currNodeUnmodified = true; |
| 1168 | |
| 1169 | if ( WTUtils::isZeroWidthWikitextElt( $node ) |
| 1170 | && $node->hasChildNodes() |
| 1171 | && ( $state->sep->constraints['constraintInfo']['sepType'] ?? null ) === 'sibling' |
| 1172 | ) { |
| 1173 | $state->sep->constraints['constraintInfo']['onSOL'] = $state->onSOL; |
| 1174 | $state->sep->constraints['constraintInfo']['sepType'] = 'parent-child'; |
| 1175 | $state->sep->constraints['constraintInfo']['nodeA'] = $node; |
| 1176 | $state->sep->constraints['constraintInfo']['nodeB'] = $node->firstChild; |
| 1177 | } |
| 1178 | |
| 1179 | $out = $state->getOrigSrc( $dp->dsr->start, $dp->dsr->end ) ?? ''; |
| 1180 | |
| 1181 | $this->trace( 'ORIG-src with DSR', static function () use ( $dp, $out ) { |
| 1182 | return '[' . $dp->dsr->start . ',' . $dp->dsr->end . '] = ' |
| 1183 | . PHPUtils::jsonEncode( $out ); |
| 1184 | } ); |
| 1185 | |
| 1186 | // When reusing source, we should only suppress serializing |
| 1187 | // to a single line for the cases we've allowed in normal serialization. |
| 1188 | // <a> tags might look surprising here, but, here is the rationale. |
| 1189 | // If some link syntax (wikilink, extlink, etc.) accepted a newline |
| 1190 | // originally, we can safely let it through here. There is no need to have |
| 1191 | // specific checks for wikilnks / extlinks / ... etc. The only concern is |
| 1192 | // if the surrounding context in which this link-syntax is embedded also |
| 1193 | // breaks the link syntax. There is no such syntax right now. |
| 1194 | // FIXME: Note the limitation here, that if these nodes are nested |
| 1195 | // in something as trivial as an i / b, the suppression won't happen |
| 1196 | // and we'll dirty the text. |
| 1197 | $suppressSLC = WTUtils::isFirstEncapsulationWrapperNode( $node ) |
| 1198 | || DOMUtils::hasTypeOf( $node, 'mw:Nowiki' ) |
| 1199 | || in_array( DOMCompat::nodeName( $node ), [ 'dl', 'ul', 'ol', 'a' ], true ) |
| 1200 | || ( DOMCompat::nodeName( $node ) === 'table' |
| 1201 | && DOMCompat::nodeName( $node->parentNode ) === 'dd' |
| 1202 | && DiffDOMUtils::previousNonSepSibling( $node ) === null ); |
| 1203 | |
| 1204 | // Use selser to serialize this text! The original |
| 1205 | // wikitext is `out`. But first allow |
| 1206 | // `ConstrainedText.fromSelSer` to figure out the right |
| 1207 | // type of ConstrainedText chunk(s) to use to represent |
| 1208 | // `out`, based on the node type. Since we might actually |
| 1209 | // have to break this wikitext into multiple chunks, |
| 1210 | // `fromSelSer` returns an array. |
| 1211 | if ( $suppressSLC ) { |
| 1212 | $state->singleLineContext->disable(); |
| 1213 | } |
| 1214 | foreach ( ConstrainedText::fromSelSer( $out, $node, $dp, $this->env ) as $ct ) { |
| 1215 | $state->emitChunk( $ct, $ct->node ); |
| 1216 | } |
| 1217 | if ( $suppressSLC ) { |
| 1218 | $state->singleLineContext->pop(); |
| 1219 | } |
| 1220 | |
| 1221 | // Skip over encapsulated content since it has already been |
| 1222 | // serialized. |
| 1223 | if ( WTUtils::isFirstEncapsulationWrapperNode( $node ) ) { |
| 1224 | return WTUtils::skipOverEncapsulatedContent( $node ); |
| 1225 | } else { |
| 1226 | return $node->nextSibling; |
| 1227 | } |
| 1228 | } |
| 1229 | |
| 1230 | $wrapperUnmodified = DiffUtils::onlySubtreeChanged( $node ) && |
| 1231 | WTSUtils::hasValidTagWidths( $dp->dsr ?? null ); |
| 1232 | } |
| 1233 | |
| 1234 | $state->currNodeUnmodified = false; |
| 1235 | |
| 1236 | $currentInsertedState = $state->inInsertedContent; |
| 1237 | |
| 1238 | $inInsertedContent = $state->selserMode && DiffUtils::hasInsertedDiffMark( $node ); |
| 1239 | |
| 1240 | if ( $inInsertedContent ) { |
| 1241 | $state->inInsertedContent = true; |
| 1242 | } |
| 1243 | |
| 1244 | $next = $domHandler->handle( $node, $state, $wrapperUnmodified ); |
| 1245 | |
| 1246 | if ( $inInsertedContent ) { |
| 1247 | $state->inInsertedContent = $currentInsertedState; |
| 1248 | } |
| 1249 | |
| 1250 | return $next; |
| 1251 | } |
| 1252 | |
| 1253 | /** |
| 1254 | * Internal worker. Recursively serialize a DOM subtree. |
| 1255 | * @private |
| 1256 | * @param Node $node |
| 1257 | * @return ?Node |
| 1258 | */ |
| 1259 | public function serializeNode( Node $node ): ?Node { |
| 1260 | $nodeName = DOMCompat::nodeName( $node ); |
| 1261 | $domHandler = $method = null; |
| 1262 | $domHandlerFactory = new DOMHandlerFactory(); |
| 1263 | $state = $this->state; |
| 1264 | $state->currNode = $node; |
| 1265 | |
| 1266 | if ( $state->selserMode ) { |
| 1267 | $this->trace( |
| 1268 | static function () use ( $node ) { |
| 1269 | return WTSUtils::traceNodeName( $node ); |
| 1270 | }, |
| 1271 | '; prev-unmodified: ', $state->prevNodeUnmodified, |
| 1272 | '; SOL: ', $state->onSOL ); |
| 1273 | } else { |
| 1274 | $this->trace( |
| 1275 | static function () use ( $node ) { |
| 1276 | return WTSUtils::traceNodeName( $node ); |
| 1277 | }, |
| 1278 | '; SOL: ', $state->onSOL ); |
| 1279 | } |
| 1280 | |
| 1281 | switch ( $node->nodeType ) { |
| 1282 | case XML_ELEMENT_NODE: |
| 1283 | '@phan-var Element $node';/** @var Element $node */ |
| 1284 | // Ignore DiffMarker metas, but clear unmodified node state |
| 1285 | if ( DiffUtils::isDiffMarker( $node ) ) { |
| 1286 | $state->updateModificationFlags( $node ); |
| 1287 | // `state.sep.lastSourceNode` is cleared here so that removed |
| 1288 | // separators between otherwise unmodified nodes don't get |
| 1289 | // restored. |
| 1290 | $state->updateSep( $node ); |
| 1291 | return $node->nextSibling; |
| 1292 | } |
| 1293 | $domHandler = $domHandlerFactory->getDOMHandler( $node ); |
| 1294 | $method = [ $this, 'serializeNodeInternal' ]; |
| 1295 | break; |
| 1296 | case XML_TEXT_NODE: |
| 1297 | // This code assumes that the DOM is in normalized form with no |
| 1298 | // run of text nodes. |
| 1299 | // Accumulate whitespace from the text node into state.sep.src |
| 1300 | $text = $node->nodeValue; |
| 1301 | if ( !$state->inIndentPre |
| 1302 | // PORT-FIXME: original uses this->state->serializer->separatorREs |
| 1303 | // but that does not seem useful |
| 1304 | && preg_match( self::$separatorREs['pureSepRE'], $text ) |
| 1305 | ) { |
| 1306 | $state->appendSep( $text ); |
| 1307 | return $node->nextSibling; |
| 1308 | } |
| 1309 | if ( $state->selserMode ) { |
| 1310 | $prev = $node->previousSibling; |
| 1311 | if ( !$state->inInsertedContent && ( |
| 1312 | ( !$prev && DOMUtils::atTheTop( $node->parentNode ) ) || |
| 1313 | ( $prev && !DiffUtils::isDiffMarker( $prev ) ) |
| 1314 | ) ) { |
| 1315 | $state->currNodeUnmodified = true; |
| 1316 | } else { |
| 1317 | $state->currNodeUnmodified = false; |
| 1318 | } |
| 1319 | } |
| 1320 | |
| 1321 | $domHandler = new DOMHandler( false ); |
| 1322 | $method = [ $this, 'serializeTextNode' ]; |
| 1323 | break; |
| 1324 | case XML_COMMENT_NODE: |
| 1325 | // Merge this into separators |
| 1326 | $state->appendSep( WTSUtils::commentWT( $node->nodeValue ) ); |
| 1327 | return $node->nextSibling; |
| 1328 | default: |
| 1329 | throw new InternalException( 'Unhandled node type: ' . $node->nodeType ); |
| 1330 | } |
| 1331 | |
| 1332 | $prev = DiffDOMUtils::previousNonSepSibling( $node ) ?: $node->parentNode; |
| 1333 | $this->env->log( 'debug/wts', 'Before constraints for ' . $nodeName ); |
| 1334 | $state->separators->updateSeparatorConstraints( |
| 1335 | $prev, $domHandlerFactory->getDOMHandler( $prev ), |
| 1336 | $node, $domHandler |
| 1337 | ); |
| 1338 | |
| 1339 | $this->env->log( 'debug/wts', 'Calling serialization handler for ' . $nodeName ); |
| 1340 | $nextNode = call_user_func( $method, $node, $domHandler ); |
| 1341 | |
| 1342 | $next = DiffDOMUtils::nextNonSepSibling( $node ) ?: $node->parentNode; |
| 1343 | $this->env->log( 'debug/wts', 'After constraints for ' . $nodeName ); |
| 1344 | $state->separators->updateSeparatorConstraints( |
| 1345 | $node, $domHandler, |
| 1346 | $next, $domHandlerFactory->getDOMHandler( $next ) |
| 1347 | ); |
| 1348 | |
| 1349 | // Update modification flags |
| 1350 | $state->updateModificationFlags( $node ); |
| 1351 | |
| 1352 | return $nextNode; |
| 1353 | } |
| 1354 | |
| 1355 | private function stripUnnecessaryHeadingNowikis( string $line ): string { |
| 1356 | $state = $this->state; |
| 1357 | if ( !$state->hasHeadingEscapes ) { |
| 1358 | return $line; |
| 1359 | } |
| 1360 | |
| 1361 | $escaper = static function ( string $wt ) use ( $state ) { |
| 1362 | $ret = $state->serializer->wteHandlers->escapedText( $state, false, $wt, false, true ); |
| 1363 | return $ret; |
| 1364 | }; |
| 1365 | |
| 1366 | preg_match( self::HEADING_NOWIKI_REGEXP, $line, $match ); |
| 1367 | if ( $match && !preg_match( self::COMMENT_OR_WS_REGEXP, $match[2] ) ) { |
| 1368 | // The nowikiing was spurious since the trailing = is not in EOL position |
| 1369 | return $escaper( $match[1] ) . $match[2]; |
| 1370 | } else { |
| 1371 | // All is good. |
| 1372 | return $line; |
| 1373 | } |
| 1374 | } |
| 1375 | |
| 1376 | private function stripUnnecessaryIndentPreNowikis(): void { |
| 1377 | // FIXME: The solTransparentWikitextRegexp includes redirects, which really |
| 1378 | // only belong at the SOF and should be unique. See the "New redirect" test. |
| 1379 | $noWikiRegexp = '@^' |
| 1380 | . PHPUtils::reStrip( $this->env->getSiteConfig()->solTransparentWikitextNoWsRegexp(), '@' ) |
| 1381 | . '((?i:<nowiki>\s+</nowiki>))([^\n]*(?:\n|$))' . '@Dm'; |
| 1382 | $pieces = preg_split( $noWikiRegexp, $this->state->out, -1, PREG_SPLIT_DELIM_CAPTURE ); |
| 1383 | $out = $pieces[0]; |
| 1384 | for ( $i = 1; $i < count( $pieces ); $i += 4 ) { |
| 1385 | $out .= $pieces[$i]; |
| 1386 | $nowiki = $pieces[$i + 1]; |
| 1387 | $rest = $pieces[$i + 2]; |
| 1388 | // Ignore comments |
| 1389 | preg_match_all( '/<[^!][^<>]*>/', $rest, $htmlTags ); |
| 1390 | |
| 1391 | // Not required if just sol transparent wt. |
| 1392 | $reqd = !preg_match( $this->env->getSiteConfig()->solTransparentWikitextRegexp(), $rest ); |
| 1393 | |
| 1394 | if ( $reqd ) { |
| 1395 | foreach ( $htmlTags[0] as $j => $rawTagName ) { |
| 1396 | // Strip </, attributes, and > to get the tagname |
| 1397 | $tagName = preg_replace( '/<\/?|\s.*|>/', '', $rawTagName ); |
| 1398 | if ( !isset( Consts::$HTML['HTML5Tags'][$tagName] ) ) { |
| 1399 | // If we encounter any tag that is not a html5 tag, |
| 1400 | // it could be an extension tag. We could do a more complex |
| 1401 | // regexp or tokenize the string to determine if any block tags |
| 1402 | // show up outside the extension tag. But, for now, we just |
| 1403 | // conservatively bail and leave the nowiki as is. |
| 1404 | $reqd = true; |
| 1405 | break; |
| 1406 | } elseif ( TokenUtils::isWikitextBlockTag( $tagName ) ) { |
| 1407 | // FIXME: Extension tags shadowing html5 tags might not |
| 1408 | // have block semantics. |
| 1409 | // Block tags on a line suppress nowikis |
| 1410 | $reqd = false; |
| 1411 | } |
| 1412 | } |
| 1413 | } |
| 1414 | |
| 1415 | if ( !$reqd ) { |
| 1416 | $nowiki = preg_replace( '#^<nowiki>(\s+)</nowiki>#', '$1', $nowiki, 1 ); |
| 1417 | } else { |
| 1418 | $solTransparentWikitextNoWsRegexpFragment = PHPUtils::reStrip( |
| 1419 | $this->env->getSiteConfig()->solTransparentWikitextNoWsRegexp(), '/' ); |
| 1420 | $wsReplacementRE = '/^(' . $solTransparentWikitextNoWsRegexpFragment . ')\s+/'; |
| 1421 | // Replace all leading whitespace |
| 1422 | do { |
| 1423 | $oldRest = $rest; |
| 1424 | $rest = preg_replace( $wsReplacementRE, '$1', $rest ); |
| 1425 | } while ( $rest !== $oldRest ); |
| 1426 | |
| 1427 | // Protect against sol-sensitive wikitext characters |
| 1428 | $solCharsTest = '/^' . $solTransparentWikitextNoWsRegexpFragment . '[=*#:;]/'; |
| 1429 | $nowiki = preg_replace( '#^<nowiki>(\s+)</nowiki>#', |
| 1430 | preg_match( $solCharsTest, $rest ) ? '<nowiki/>' : '', $nowiki, 1 ); |
| 1431 | } |
| 1432 | $out = $out . $nowiki . $rest . $pieces[$i + 3]; |
| 1433 | } |
| 1434 | $this->state->out = $out; |
| 1435 | } |
| 1436 | |
| 1437 | /** |
| 1438 | * This implements a heuristic to strip two common sources of <nowiki/>s. |
| 1439 | * When <i> and <b> tags are matched up properly, |
| 1440 | * - any single ' char before <i> or <b> does not need <nowiki/> protection. |
| 1441 | * - any single ' char before </i> or </b> does not need <nowiki/> protection. |
| 1442 | * @param string $line |
| 1443 | * @return string |
| 1444 | */ |
| 1445 | private function stripUnnecessaryQuoteNowikis( string $line ): string { |
| 1446 | if ( !$this->state->hasQuoteNowikis ) { |
| 1447 | return $line; |
| 1448 | } |
| 1449 | |
| 1450 | // Optimization: We are interested in <nowiki/>s before quote chars. |
| 1451 | // So, skip this if we don't have both. |
| 1452 | if ( !( preg_match( '#<nowiki\s*/>#', $line ) && preg_match( "/'/", $line ) ) ) { |
| 1453 | return $line; |
| 1454 | } |
| 1455 | |
| 1456 | // * Split out all the [[ ]] {{ }} '' ''' ''''' <..> </...> |
| 1457 | // parens in the regexp mean that the split segments will |
| 1458 | // be spliced into the result array as the odd elements. |
| 1459 | // * If we match up the tags properly and we see opening |
| 1460 | // <i> / <b> / <i><b> tags preceded by a '<nowiki/>, we |
| 1461 | // can remove all those nowikis. |
| 1462 | // Ex: '<nowiki/>''foo'' bar '<nowiki/>'''baz''' |
| 1463 | // * If we match up the tags properly and we see closing |
| 1464 | // <i> / <b> / <i><b> tags preceded by a '<nowiki/>, we |
| 1465 | // can remove all those nowikis. |
| 1466 | // Ex: ''foo'<nowiki/>'' bar '''baz'<nowiki/>''' |
| 1467 | // phpcs:ignore Generic.Files.LineLength.TooLong |
| 1468 | $p = preg_split( "#('''''|'''|''|\[\[|\]\]|\{\{|\}\}|<\w+(?:\s+[^>]*?|\s*?)/?>|</\w+\s*>)#", $line, -1, PREG_SPLIT_DELIM_CAPTURE ); |
| 1469 | |
| 1470 | // Which nowiki do we strip out? |
| 1471 | $nowikiIndex = -1; |
| 1472 | |
| 1473 | // Verify that everything else is properly paired up. |
| 1474 | $stack = []; |
| 1475 | $quotesOnStack = 0; |
| 1476 | $n = count( $p ); |
| 1477 | $nonHtmlTag = null; |
| 1478 | for ( $j = 1; $j < $n; $j += 2 ) { |
| 1479 | // For HTML tags, pull out just the tag name for clearer code below. |
| 1480 | preg_match( '#^<(/?\w+)#', $p[$j], $matches ); |
| 1481 | $tag = mb_strtolower( $matches[1] ?? $p[$j] ); |
| 1482 | $tagLen = strlen( $tag ); |
| 1483 | $selfClose = false; |
| 1484 | if ( str_ends_with( $p[$j], '/>' ) ) { |
| 1485 | $tag .= '/'; |
| 1486 | $selfClose = true; |
| 1487 | } |
| 1488 | |
| 1489 | // Ignore non-html-tag (<nowiki> OR extension tag) blocks |
| 1490 | if ( !$nonHtmlTag ) { |
| 1491 | if ( isset( $this->env->getSiteConfig()->getExtensionTagNameMap()[$tag] ) ) { |
| 1492 | $nonHtmlTag = $tag; |
| 1493 | continue; |
| 1494 | } |
| 1495 | } else { |
| 1496 | if ( $tagLen > 0 && $tag[0] === '/' && substr( $tag, 1 ) === $nonHtmlTag ) { |
| 1497 | $nonHtmlTag = null; |
| 1498 | } |
| 1499 | continue; |
| 1500 | } |
| 1501 | |
| 1502 | if ( $tag === ']]' ) { |
| 1503 | if ( array_pop( $stack ) !== '[[' ) { |
| 1504 | return $line; |
| 1505 | } |
| 1506 | } elseif ( $tag === '}}' ) { |
| 1507 | if ( array_pop( $stack ) !== '{{' ) { |
| 1508 | return $line; |
| 1509 | } |
| 1510 | } elseif ( $tagLen > 0 && $tag[0] === '/' ) { // closing html tag |
| 1511 | // match html/ext tags |
| 1512 | $openTag = array_pop( $stack ); |
| 1513 | if ( $tag !== ( '/' . $openTag ) ) { |
| 1514 | return $line; |
| 1515 | } |
| 1516 | } elseif ( $tag === 'nowiki/' ) { |
| 1517 | // We only want to process: |
| 1518 | // - trailing single quotes (bar') |
| 1519 | // - or single quotes by themselves without a preceding '' sequence |
| 1520 | if ( substr( $p[$j - 1], -1 ) === "'" |
| 1521 | && !( $p[$j - 1] === "'" && $j > 1 && substr( $p[$j - 2], -2 ) === "''" ) |
| 1522 | // Consider <b>foo<i>bar'</i>baz</b> or <b>foo'<i>bar'</i>baz</b>. |
| 1523 | // The <nowiki/> before the <i> or </i> cannot be stripped |
| 1524 | // if the <i> is embedded inside another quote. |
| 1525 | && ( $quotesOnStack === 0 |
| 1526 | // The only strippable scenario with a single quote elt on stack |
| 1527 | // is: ''bar'<nowiki/>'' |
| 1528 | // -> ["", "''", "bar'", "<nowiki/>", "", "''"] |
| 1529 | || ( $quotesOnStack === 1 |
| 1530 | && $j + 2 < $n |
| 1531 | && $p[$j + 1] === '' |
| 1532 | && $p[$j + 2][0] === "'" |
| 1533 | && $p[$j + 2] === PHPUtils::lastItem( $stack ) ) ) |
| 1534 | ) { |
| 1535 | $nowikiIndex = $j; |
| 1536 | } |
| 1537 | continue; |
| 1538 | } elseif ( $selfClose || $tag === 'br' ) { |
| 1539 | // Skip over self-closing tags or what should have been self-closed. |
| 1540 | // ( While we could do this for all void tags defined in |
| 1541 | // mediawiki.wikitext.constants.js, <br> is the most common |
| 1542 | // culprit. ) |
| 1543 | continue; |
| 1544 | } elseif ( $tagLen > 0 && $tag[0] === "'" && PHPUtils::lastItem( $stack ) === $tag ) { |
| 1545 | array_pop( $stack ); |
| 1546 | $quotesOnStack--; |
| 1547 | } else { |
| 1548 | $stack[] = $tag; |
| 1549 | if ( $tagLen > 0 && $tag[0] === "'" ) { |
| 1550 | $quotesOnStack++; |
| 1551 | } |
| 1552 | } |
| 1553 | } |
| 1554 | |
| 1555 | if ( count( $stack ) ) { |
| 1556 | return $line; |
| 1557 | } |
| 1558 | |
| 1559 | if ( $nowikiIndex !== -1 ) { |
| 1560 | // We can only remove the final trailing nowiki. |
| 1561 | // |
| 1562 | // HTML : <i>'foo'</i> |
| 1563 | // line : ''<nowiki/>'foo'<nowiki/>'' |
| 1564 | $p[$nowikiIndex] = ''; |
| 1565 | return implode( '', $p ); |
| 1566 | } else { |
| 1567 | return $line; |
| 1568 | } |
| 1569 | } |
| 1570 | |
| 1571 | /** |
| 1572 | * Serialize an HTML DOM. |
| 1573 | * |
| 1574 | * WARNING: You probably want to use WikitextContentModelHandler::fromDOM instead. |
| 1575 | * |
| 1576 | * @param Document|DocumentFragment $node |
| 1577 | * @param bool $selserMode |
| 1578 | * @return string |
| 1579 | */ |
| 1580 | public function serializeDOM( |
| 1581 | Node $node, bool $selserMode = false |
| 1582 | ): string { |
| 1583 | Assert::parameterType( |
| 1584 | Document::class . '|' . DocumentFragment::class, |
| 1585 | $node, '$node' ); |
| 1586 | |
| 1587 | if ( $node instanceof Document ) { |
| 1588 | $node = DOMCompat::getBody( $node ); |
| 1589 | } |
| 1590 | |
| 1591 | $this->logType = $selserMode ? 'trace/selser' : 'trace/wts'; |
| 1592 | |
| 1593 | $state = $this->state; |
| 1594 | $state->initMode( $selserMode ); |
| 1595 | |
| 1596 | $domNormalizer = new DOMNormalizer( $state ); |
| 1597 | $domNormalizer->normalize( $node ); |
| 1598 | |
| 1599 | if ( $this->env->hasDumpFlag( 'dom:post-normal' ) ) { |
| 1600 | $options = [ 'storeDiffMark' => true ]; |
| 1601 | $this->env->writeDump( ContentUtils::dumpDOM( $node, 'DOM: post-normal', $options ) ); |
| 1602 | } |
| 1603 | |
| 1604 | $state->kickOffSerialize( $node ); |
| 1605 | |
| 1606 | if ( $state->hasIndentPreNowikis ) { |
| 1607 | // FIXME: Perhaps this can be done on a per-line basis |
| 1608 | // rather than do one post-pass on the entire document. |
| 1609 | $this->stripUnnecessaryIndentPreNowikis(); |
| 1610 | } |
| 1611 | |
| 1612 | $splitLines = $state->selserMode |
| 1613 | || $state->hasQuoteNowikis |
| 1614 | || $state->hasSelfClosingNowikis |
| 1615 | || $state->hasHeadingEscapes; |
| 1616 | |
| 1617 | if ( $splitLines ) { |
| 1618 | $state->out = implode( "\n", array_map( function ( $line ) { |
| 1619 | // FIXME: Perhaps this can be done on a per-line basis |
| 1620 | // rather than do one post-pass on the entire document. |
| 1621 | $line = $this->stripUnnecessaryQuoteNowikis( $line ); |
| 1622 | |
| 1623 | return $this->stripUnnecessaryHeadingNowikis( $line ); |
| 1624 | }, explode( "\n", $state->out ) ) ); |
| 1625 | } |
| 1626 | |
| 1627 | if ( $state->redirectText && $state->redirectText !== 'unbuffered' ) { |
| 1628 | $firstLine = explode( "\n", $state->out, 1 )[0]; |
| 1629 | $nl = preg_match( '/^(\s|$)/D', $firstLine ) ? '' : "\n"; |
| 1630 | $state->out = $state->redirectText . $nl . $state->out; |
| 1631 | } |
| 1632 | |
| 1633 | return $state->out; |
| 1634 | } |
| 1635 | |
| 1636 | /** |
| 1637 | * @note Porting note: this replaces the pattern $serializer->env->log( $serializer->logType, ... ) |
| 1638 | * @param mixed ...$args |
| 1639 | */ |
| 1640 | public function trace( ...$args ) { |
| 1641 | $this->env->log( $this->logType, ...$args ); |
| 1642 | } |
| 1643 | |
| 1644 | } |