Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
23.47% |
23 / 98 |
|
14.29% |
1 / 7 |
CRAP | |
0.00% |
0 / 1 |
| ContentModelHandler | |
23.47% |
23 / 98 |
|
14.29% |
1 / 7 |
130.75 | |
0.00% |
0 / 1 |
| __construct | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| canonicalizeDOM | |
0.00% |
0 / 16 |
|
0.00% |
0 / 1 |
6 | |||
| setupSelser | |
0.00% |
0 / 13 |
|
0.00% |
0 / 1 |
6 | |||
| processIndicators | |
0.00% |
0 / 9 |
|
0.00% |
0 / 1 |
12 | |||
| toDOM | |
0.00% |
0 / 26 |
|
0.00% |
0 / 1 |
12 | |||
| preprocessEditedDOM | |
0.00% |
0 / 10 |
|
0.00% |
0 / 1 |
12 | |||
| fromDOM | |
100.00% |
23 / 23 |
|
100.00% |
1 / 1 |
2 | |||
| 1 | <?php |
| 2 | declare( strict_types = 1 ); |
| 3 | |
| 4 | namespace Wikimedia\Parsoid\Wikitext; |
| 5 | |
| 6 | use Wikimedia\Assert\Assert; |
| 7 | use Wikimedia\Parsoid\Config\Env; |
| 8 | use Wikimedia\Parsoid\Core\ContentModelHandler as IContentModelHandler; |
| 9 | use Wikimedia\Parsoid\Core\DOMCompat; |
| 10 | use Wikimedia\Parsoid\Core\DomPageBundle; |
| 11 | use Wikimedia\Parsoid\Core\SelectiveUpdateData; |
| 12 | use Wikimedia\Parsoid\DOM\Document; |
| 13 | use Wikimedia\Parsoid\Ext\DOMProcessor as ExtDOMProcessor; |
| 14 | use Wikimedia\Parsoid\Ext\ParsoidExtensionAPI; |
| 15 | use Wikimedia\Parsoid\Html2Wt\RemoveRedLinks; |
| 16 | use Wikimedia\Parsoid\Html2Wt\SelectiveSerializer; |
| 17 | use Wikimedia\Parsoid\Html2Wt\WikitextSerializer; |
| 18 | use Wikimedia\Parsoid\Utils\ContentUtils; |
| 19 | use Wikimedia\Parsoid\Utils\DOMDataUtils; |
| 20 | use Wikimedia\Parsoid\Utils\DOMTraverser; |
| 21 | use Wikimedia\Parsoid\Utils\Timing; |
| 22 | |
| 23 | class ContentModelHandler extends IContentModelHandler { |
| 24 | |
| 25 | /** @var Env */ |
| 26 | private $env; |
| 27 | |
| 28 | /** |
| 29 | * Sneak an environment in here since it's not exposed as part of the |
| 30 | * ParsoidExtensionAPI |
| 31 | * |
| 32 | * @param Env $env |
| 33 | */ |
| 34 | public function __construct( Env $env ) { |
| 35 | $this->env = $env; |
| 36 | } |
| 37 | |
| 38 | /** |
| 39 | * Bring DOM to expected canonical form |
| 40 | */ |
| 41 | private function canonicalizeDOM( |
| 42 | Env $env, Document $doc, bool $isSelectiveUpdate |
| 43 | ): void { |
| 44 | Assert::invariant( |
| 45 | DOMDataUtils::isPreparedAndLoaded( $doc ), |
| 46 | "doc should already be prepared and loaded" |
| 47 | ); |
| 48 | $body = DOMCompat::getBody( $doc ); |
| 49 | |
| 50 | // Update DSR offsets if necessary. |
| 51 | ContentUtils::convertOffsets( |
| 52 | $env, $doc, $env->getRequestOffsetType(), 'byte' |
| 53 | ); |
| 54 | |
| 55 | // Strip <section> and mw:FallbackId <span> tags, if present, |
| 56 | // as well as extended annotation wrappers. |
| 57 | // This ensures that we can accept HTML from CX / VE |
| 58 | // and other clients that might have stripped them. |
| 59 | ContentUtils::stripUnnecessaryWrappersAndSyntheticNodes( $body ); |
| 60 | |
| 61 | if ( !$isSelectiveUpdate ) { |
| 62 | $redLinkRemover = new DOMTraverser( |
| 63 | traverseWithTplInfo: false, |
| 64 | applyToAttributeEmbeddedHTML: true, |
| 65 | ); |
| 66 | $redLinkRemover->addHandler( 'a', [ RemoveRedLinks::class, 'handler' ] ); |
| 67 | $redLinkRemover->traverse( $env->getSiteConfig(), $body, null ); |
| 68 | } |
| 69 | } |
| 70 | |
| 71 | /** |
| 72 | * Fetch prior DOM for selser. |
| 73 | * |
| 74 | * @param ParsoidExtensionAPI $extApi |
| 75 | * @param SelectiveUpdateData $selserData |
| 76 | */ |
| 77 | private function setupSelser( |
| 78 | ParsoidExtensionAPI $extApi, SelectiveUpdateData $selserData |
| 79 | ): void { |
| 80 | $env = $this->env; |
| 81 | |
| 82 | // Why is it safe to use a reparsed dom for dom diff'ing? |
| 83 | // (Since that's the only use of `env.page.dom`) |
| 84 | // |
| 85 | // There are two types of non-determinism to discuss: |
| 86 | // |
| 87 | // * The first is from parsoid generated ids. At this point, |
| 88 | // data-attributes have already been applied so there's no chance |
| 89 | // that variability in the ids used to associate data-attributes |
| 90 | // will lead to data being applied to the wrong nodes. |
| 91 | // |
| 92 | // Further, although about ids will differ, they belong to the set |
| 93 | // of ignorable attributes in the dom differ. |
| 94 | // |
| 95 | // * Templates, and encapsulated content in general, are the second. |
| 96 | // Since that content can change in between parses, the resulting |
| 97 | // dom might not be the same. However, because dom diffing on |
| 98 | // on those regions only uses data-mw for comparision (which will |
| 99 | // remain constant between parses), this also shouldn't be an |
| 100 | // issue. |
| 101 | // |
| 102 | // There is one caveat. Because encapsulated content isn't |
| 103 | // guaranteed to be "balanced", the template affected regions |
| 104 | // may change between parses. This should be rare. |
| 105 | // |
| 106 | // We therefore consider this safe since it won't corrupt the page |
| 107 | // and, at worst, mixed up diff'ing annotations can end up with an |
| 108 | // unfaithful serialization of the edit. |
| 109 | // |
| 110 | // However, in cases where original content is not returned by the |
| 111 | // client / RESTBase, selective serialization cannot proceed and |
| 112 | // we're forced to fallback to normalizing the entire page. This has |
| 113 | // proved unacceptable to editors as is and, as we lean heavier on |
| 114 | // selser, will only get worse over time. |
| 115 | // |
| 116 | // So, we're forced to trade off the correctness for usability. |
| 117 | if ( $selserData->revHTML === null ) { |
| 118 | $env->log( "warn/html2wt", "Missing selserData->revHTML. Regenerating." ); |
| 119 | |
| 120 | // FIXME(T266838): Create a new Env for this parse? Something is |
| 121 | // needed to avoid this rigmarole. |
| 122 | $topLevelDoc = $env->getTopLevelDoc(); |
| 123 | // This effectively parses $selserData->revText for us because |
| 124 | // $selserData->revText = $env->getPageconfig()->getPageMainContent() |
| 125 | $env->setupTopLevelDoc(); |
| 126 | $doc = $this->toDOM( $extApi ); |
| 127 | |
| 128 | // Now set up doc again for html2wt |
| 129 | $env->setupTopLevelDoc( $topLevelDoc ); |
| 130 | DOMDataUtils::getBag( $topLevelDoc )->serializeNewEmptyDp = false; |
| 131 | } else { |
| 132 | $doc = ContentUtils::createAndLoadDocument( |
| 133 | $selserData->revHTML, siteConfig: $env->getSiteConfig(), |
| 134 | ); |
| 135 | } |
| 136 | |
| 137 | $this->canonicalizeDOM( $env, $doc, false ); |
| 138 | $selserData->revDOM = $doc; |
| 139 | } |
| 140 | |
| 141 | private function processIndicators( Document $doc, ParsoidExtensionAPI $extApi ): void { |
| 142 | // Erroneous indicators without names will be <span>s |
| 143 | $indicators = DOMCompat::querySelectorAll( $doc, 'meta[typeof~="mw:Extension/indicator"]' ); |
| 144 | $iData = []; |
| 145 | |
| 146 | // https://www.mediawiki.org/wiki/Help:Page_status_indicators#Adding_page_status_indicators |
| 147 | // says that last one wins. But, that may just be documentation of the |
| 148 | // implementation vs. being a deliberate strategy. |
| 149 | // |
| 150 | // The indicators are ordered by depth-first pre-order DOM traversal. |
| 151 | // This ensures that the indicators are in document textual order. |
| 152 | // Given that, the for-loop below implements "last-one-wins" semantics |
| 153 | // for indicators that use the same name key. |
| 154 | foreach ( $indicators as $meta ) { |
| 155 | $dmw = DOMDataUtils::getDataMw( $meta ); |
| 156 | $name = $dmw->getExtAttrib( 'name' ); |
| 157 | $iData[$name] = $dmw->html; |
| 158 | unset( $dmw->html ); |
| 159 | } |
| 160 | |
| 161 | // set indicator metadata for unique keys |
| 162 | // (Note that this could easily be updated in the future to pass |
| 163 | // a DocumentFragment directly, instead of converting to a string.) |
| 164 | foreach ( $iData as $name => $html ) { |
| 165 | $extApi->getMetadata()->setIndicator( (string)$name, $extApi->domToHtml( $html, true ) ); |
| 166 | } |
| 167 | } |
| 168 | |
| 169 | /** |
| 170 | * @inheritDoc |
| 171 | */ |
| 172 | public function toDOM( |
| 173 | ParsoidExtensionAPI $extApi, ?SelectiveUpdateData $selectiveUpdateData = null |
| 174 | ): Document { |
| 175 | $env = $this->env; |
| 176 | $pipelineFactory = $env->getPipelineFactory(); |
| 177 | |
| 178 | if ( $selectiveUpdateData ) { |
| 179 | $doc = ContentUtils::createAndLoadDocument( |
| 180 | $selectiveUpdateData->revHTML, |
| 181 | [ 'serializeNewEmptyDp' => true ], // isSelectiveUpdate |
| 182 | siteConfig: $env->getSiteConfig(), |
| 183 | ); |
| 184 | Assert::invariant( |
| 185 | !DomPageBundle::isSingleDocument( $doc ), |
| 186 | "toplevelDoc should not be a single-document page bundle" |
| 187 | ); |
| 188 | $env->setupTopLevelDoc( $doc ); |
| 189 | $this->canonicalizeDOM( $env, $env->getTopLevelDoc(), true ); |
| 190 | $selectiveUpdateData->revDOM = $doc; |
| 191 | $doc = $pipelineFactory->selectiveDOMUpdate( $selectiveUpdateData ); |
| 192 | } else { |
| 193 | $doc = $pipelineFactory->parse( |
| 194 | // @phan-suppress-next-line PhanDeprecatedFunction not ready for topFrame yet |
| 195 | $env->getPageConfig()->getPageMainContent() |
| 196 | ); |
| 197 | } |
| 198 | |
| 199 | // Hardcoded support for indicators |
| 200 | // TODO: Eventually we'll want to apply this to selective updates as well |
| 201 | if ( !$selectiveUpdateData ) { |
| 202 | $this->processIndicators( $doc, $extApi ); |
| 203 | } |
| 204 | |
| 205 | Assert::invariant( |
| 206 | DOMDataUtils::isPreparedAndLoaded( $doc ), |
| 207 | "toDOM should return a prepared and loaded doc" |
| 208 | ); |
| 209 | return $doc; |
| 210 | } |
| 211 | |
| 212 | /** |
| 213 | * Preprocess the edited DOM as required before attempting to convert it to wikitext |
| 214 | * 1. The edited DOM (represented by body) might not be in canonical form |
| 215 | * because Parsoid might be providing server-side management of global state |
| 216 | * for extensions. To address this and bring the DOM back to canonical form, |
| 217 | * we run extension-provided handlers. The original DOM isn't subject to this problem. |
| 218 | * FIXME: But, this is not the only reason an extension might register a preprocessor. |
| 219 | * How do we know when to run a preprocessor on both original & edited DOMs? |
| 220 | * 2. We need to do this after all data attributes have been loaded. |
| 221 | * 3. We need to do this before we run dom-diffs to eliminate spurious diffs. |
| 222 | * |
| 223 | * @param Env $env |
| 224 | * @param Document $doc |
| 225 | */ |
| 226 | private function preprocessEditedDOM( Env $env, Document $doc ): void { |
| 227 | $siteConfig = $env->getSiteConfig(); |
| 228 | |
| 229 | // Run any registered DOM preprocessors |
| 230 | foreach ( $siteConfig->getExtDOMProcessors() as $extName => $domProcs ) { |
| 231 | foreach ( $domProcs as $classNameOrSpec ) { |
| 232 | $c = $siteConfig->getObjectFactory()->createObject( $classNameOrSpec, [ |
| 233 | 'allowClassName' => true, |
| 234 | 'assertClass' => ExtDOMProcessor::class, |
| 235 | ] ); |
| 236 | $c->htmlPreprocess( |
| 237 | new ParsoidExtensionAPI( $env ), DOMCompat::getBody( $doc ) |
| 238 | ); |
| 239 | } |
| 240 | } |
| 241 | } |
| 242 | |
| 243 | /** |
| 244 | * @inheritDoc |
| 245 | */ |
| 246 | public function fromDOM( |
| 247 | ParsoidExtensionAPI $extApi, ?SelectiveUpdateData $selectiveUpdateData = null |
| 248 | ): string { |
| 249 | $env = $this->env; |
| 250 | $siteConfig = $env->getSiteConfig(); |
| 251 | $setupTiming = Timing::start( $siteConfig ); |
| 252 | |
| 253 | $this->canonicalizeDOM( $env, $env->getTopLevelDoc(), false ); |
| 254 | |
| 255 | $serializerOpts = [ 'selserData' => $selectiveUpdateData ]; |
| 256 | if ( $selectiveUpdateData ) { |
| 257 | $serializer = new SelectiveSerializer( $env, $serializerOpts ); |
| 258 | $this->setupSelser( $extApi, $selectiveUpdateData ); |
| 259 | $wtsType = 'selser'; |
| 260 | } else { |
| 261 | // Fallback |
| 262 | $serializer = new WikitextSerializer( $env, $serializerOpts ); |
| 263 | $wtsType = 'noselser'; |
| 264 | } |
| 265 | |
| 266 | $setupTiming->end( 'html2wt.setup', 'html2wt_setup_seconds', [] ); |
| 267 | |
| 268 | $preprocTiming = Timing::start( $siteConfig ); |
| 269 | $this->preprocessEditedDOM( $env, $env->getTopLevelDoc() ); |
| 270 | $preprocTiming->end( 'html2wt.preprocess', 'html2wt_preprocess_seconds', [] ); |
| 271 | |
| 272 | $serializeTiming = Timing::start( $siteConfig ); |
| 273 | $res = $serializer->serializeDOM( $env->getTopLevelDoc() ); |
| 274 | $serializeTiming->end( |
| 275 | "html2wt.{$wtsType}.serialize", |
| 276 | "html2wt_serialize_seconds", |
| 277 | [ 'wts' => $wtsType ] |
| 278 | ); |
| 279 | |
| 280 | return $res; |
| 281 | } |
| 282 | |
| 283 | } |