Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
0.00% |
0 / 131 |
|
0.00% |
0 / 5 |
CRAP | |
0.00% |
0 / 1 |
| ComputeSelectiveStats | |
0.00% |
0 / 131 |
|
0.00% |
0 / 5 |
1260 | |
0.00% |
0 / 1 |
| classify | |
0.00% |
0 / 93 |
|
0.00% |
0 / 1 |
506 | |||
| pc2wt | |
0.00% |
0 / 1 |
|
0.00% |
0 / 1 |
2 | |||
| filterUserAgent | |
0.00% |
0 / 29 |
|
0.00% |
0 / 1 |
30 | |||
| bool2str | |
0.00% |
0 / 3 |
|
0.00% |
0 / 1 |
12 | |||
| int2str | |
0.00% |
0 / 5 |
|
0.00% |
0 / 1 |
20 | |||
| 1 | <?php |
| 2 | declare( strict_types = 1 ); |
| 3 | |
| 4 | namespace Wikimedia\Parsoid\Utils; |
| 5 | |
| 6 | use Wikimedia\Parsoid\Config\Env; |
| 7 | use Wikimedia\Parsoid\Config\PageConfig; |
| 8 | use Wikimedia\Parsoid\Core\DOMCompat; |
| 9 | use Wikimedia\Parsoid\Core\DomPageBundle; |
| 10 | use Wikimedia\Parsoid\Core\HtmlPageBundle; |
| 11 | use Wikimedia\Parsoid\DOM\Element; |
| 12 | use Wikimedia\Parsoid\Html2Wt\DiffUtils; |
| 13 | use Wikimedia\Parsoid\Html2Wt\DOMDiff; |
| 14 | use Wikimedia\Parsoid\NodeData\DataParsoid; |
| 15 | use Wikimedia\Parsoid\NodeData\TemplateInfo; |
| 16 | |
| 17 | /** |
| 18 | * This file contains code to classify opportunities for selective |
| 19 | * update and collect statistics. |
| 20 | */ |
| 21 | class ComputeSelectiveStats { |
| 22 | |
| 23 | /** |
| 24 | * @phpcs:ignore Generic.Files.LineLength.TooLong |
| 25 | * @return array{type: string, same_wt: string, rev_diff: string, changed_sections: string, changed_template_sites: string, changed_template_names: string} |
| 26 | */ |
| 27 | public static function classify( |
| 28 | Env $env, |
| 29 | ?PageConfig $oldPage, ?HtmlPageBundle $oldPb, |
| 30 | PageConfig $newPage, HtmlPageBundle $newPb |
| 31 | ): array { |
| 32 | // Default labels (ensure keys are consistent & in consistent order). |
| 33 | // Each label key should be a valid label name accepted by StatsLib, |
| 34 | // i.e. an alphanumeric string that does not include dashes (T394053). |
| 35 | $labels = [ |
| 36 | 'type' => 'missing-prev', |
| 37 | 'same_wt' => 'unknown', |
| 38 | 'rev_diff' => 'unknown', |
| 39 | 'changed_sections' => 'unknown', |
| 40 | 'changed_template_sites' => 'unknown', |
| 41 | 'changed_template_names' => 'unknown', |
| 42 | ]; |
| 43 | if ( $oldPage === null || $oldPb === null ) { |
| 44 | return $labels; |
| 45 | } |
| 46 | $oldWt = self::pc2wt( $oldPage ); |
| 47 | $newWt = self::pc2wt( $newPage ); |
| 48 | |
| 49 | // Compare wikitext in both revisions |
| 50 | $labels['same_wt'] = self::bool2str( $oldWt == $newWt ); |
| 51 | |
| 52 | // Compare revision IDs |
| 53 | $oldRev = $oldPage->getRevisionId(); |
| 54 | $newRev = $newPage->getRevisionId(); |
| 55 | if ( $oldRev === $newRev ) { |
| 56 | // same revision (template update, most likely) |
| 57 | $labels['rev_diff'] = '0'; |
| 58 | } elseif ( $oldRev === $newPage->getParentRevisionId() ) { |
| 59 | // "normal edit": new revision is the one after old revision |
| 60 | $labels['rev_diff'] = '1'; |
| 61 | } elseif ( $newRev === $oldPage->getParentRevisionId() ) { |
| 62 | // new revision is the one *before* old revision |
| 63 | // This is probably a render triggered from RevisionOutputCache |
| 64 | // of the previous revision where the "oldRev" is coming from |
| 65 | // the parser cache and is thus the latest. This may happen |
| 66 | // during races, vandalism patrol, HTML diffing, etc. |
| 67 | $labels['rev_diff'] = 'minus1'; |
| 68 | } |
| 69 | |
| 70 | // Parse to DOM and diff |
| 71 | $oldDoc = DomPageBundle::fromHtmlPageBundle( $oldPb )->toDom( |
| 72 | siteConfig: $env->getSiteConfig() |
| 73 | ); |
| 74 | $newDoc = DomPageBundle::fromHtmlPageBundle( $newPb )->toDom( |
| 75 | siteConfig: $env->getSiteConfig() |
| 76 | ); |
| 77 | $dd = new DOMDiff( $env ); |
| 78 | // Don't skip over template content! |
| 79 | $dd->skipEncapsulatedContent = false; |
| 80 | // Ignore differences in data-parsoid 'dsr' and 'tmp' |
| 81 | $cleanDP = static function ( DataParsoid $dp ): DataParsoid { |
| 82 | $dp = clone $dp; |
| 83 | foreach ( [ 'tmp', 'tsr', 'dsr', 'extTagOffsets', 'extLinkContentOffsets' ] as $prop ) { |
| 84 | unset( $dp->$prop ); |
| 85 | } |
| 86 | return $dp; |
| 87 | }; |
| 88 | $dd->specializedAttribHandlers['data-parsoid'] = static function ( |
| 89 | Element $nA, DataParsoid $vA, Element $nB, DataParsoid $vB |
| 90 | ) use ( $cleanDP ): bool { |
| 91 | // This is deliberately a not-strict equality comparisong between |
| 92 | // two DataParsoid objects. |
| 93 | // @phan-suppress-next-line PhanPluginComparisonObjectEqualityNotStrict |
| 94 | return $cleanDP( $vA ) == $cleanDP( $vB ); |
| 95 | }; |
| 96 | // Ignore differences in 'id' attributes, since these are a side-effect |
| 97 | // of data-parsoid/page bundle encapsulation. |
| 98 | $dd->specializedAttribHandlers['id'] = static function ( |
| 99 | Element $nA, string $vA, Element $nB, string $vB |
| 100 | ): bool { |
| 101 | // XXX we can't really tell synthethic ID attributes created by |
| 102 | // DOMDataUtils::storeInPageBundle() from "real" ID attributes |
| 103 | // in user wikitext. Hackishly ignore differences in any ID |
| 104 | // attributes that begin with 'mw' even though technically you |
| 105 | // could have a <span id="mw-something'> in wikitext, and change |
| 106 | // that to <span id='mw-different-thing'> and with this attribute |
| 107 | // handler DOM diff wouldn't flag the change. In theory we should |
| 108 | // be using shadow attributes to record when an id was synthetic. |
| 109 | if ( CounterType::NODE_DATA_ID->matches( $vA ) && |
| 110 | CounterType::NODE_DATA_ID->matches( $vB ) |
| 111 | ) { |
| 112 | return true; // equal enough |
| 113 | } |
| 114 | return $vA === $vB; |
| 115 | }; |
| 116 | [ 'isEmpty' => $emptyDiff ] = $dd->diff( |
| 117 | DOMCompat::getBody( $oldDoc ), |
| 118 | DOMCompat::getBody( $newDoc ) |
| 119 | ); |
| 120 | if ( $oldWt === $newWt ) { |
| 121 | // old and new wikitext identical. is html also identical? |
| 122 | $labels['type'] = $emptyDiff ? 'no-op' : 'template-update'; |
| 123 | } else { |
| 124 | $labels['type'] = 'page-update'; |
| 125 | } |
| 126 | |
| 127 | // Use a DOMTraverser to count how many sections and templates were |
| 128 | // modified. (Skip attribute embedded HTML for now.) |
| 129 | $dt = new DOMTraverser( true ); |
| 130 | $sectionsModified = 0; |
| 131 | $dt->addHandler( 'section', static function ( Element $el ) use ( &$sectionsModified ) { |
| 132 | if ( WTUtils::isParsoidSectionTag( $el ) && !DiffUtils::subtreeUnchanged( $el ) ) { |
| 133 | $sectionsModified++; |
| 134 | } |
| 135 | return true; |
| 136 | } ); |
| 137 | $templatesModified = 0; |
| 138 | $namedTemplates = []; |
| 139 | $dt->addHandler( null, static function ( $el, $state ) use ( &$templatesModified, &$namedTemplates ) { |
| 140 | if ( !( $el instanceof Element ) ) { |
| 141 | return true; |
| 142 | } |
| 143 | if ( |
| 144 | $el === ( $state->tplInfo->first ?? null ) && |
| 145 | DOMUtils::hasTypeOf( $el, 'mw:Transclusion' ) |
| 146 | ) { |
| 147 | $changed = false; |
| 148 | $about = DOMCompat::getAttribute( $el, 'about' ); |
| 149 | foreach ( WTUtils::getAboutSiblings( $el, $about ) as $sib ) { |
| 150 | // Note that we might miss a change here in a sibling |
| 151 | // which is fosterable IEW, since that's !Element. |
| 152 | if ( |
| 153 | $sib instanceof Element && |
| 154 | !DiffUtils::subtreeUnchanged( $sib ) |
| 155 | ) { |
| 156 | $changed = true; |
| 157 | break; |
| 158 | } |
| 159 | } |
| 160 | |
| 161 | // Compute the number of templates modified |
| 162 | if ( $changed ) { |
| 163 | $templatesModified++; |
| 164 | $dataMw = DOMDataUtils::getDataMw( $el ); |
| 165 | $name = null; |
| 166 | foreach ( $dataMw->parts ?? [] as $part ) { |
| 167 | if ( $part instanceof TemplateInfo ) { |
| 168 | $name ??= $part->href; |
| 169 | } |
| 170 | } |
| 171 | $namedTemplates[$name ?? 'unknown'] = true; |
| 172 | } |
| 173 | // Don't recurse into templates, just tabulate top-level |
| 174 | $state->tplInfo->clear = true; |
| 175 | return $state->tplInfo->last->nextSibling; |
| 176 | } |
| 177 | return true; |
| 178 | } ); |
| 179 | # do the traversal |
| 180 | $dt->traverse( null, DOMCompat::getBody( $newDoc ), new DTState( $env ) ); |
| 181 | |
| 182 | # report changed sections as '0', '1', or '2+' |
| 183 | $labels['changed_sections'] = self::int2str( $sectionsModified, 2 ); |
| 184 | # report changed templates as '0', '1', or '2+' |
| 185 | $labels['changed_template_sites'] = self::int2str( $templatesModified, 2 ); |
| 186 | # report the count of the *names* of the templates that were updated. |
| 187 | $labels['changed_template_names'] = self::int2str( count( $namedTemplates ), 2 ); |
| 188 | |
| 189 | // TODO: sum up the time spent on modified (vs unmodified) templates |
| 190 | |
| 191 | return $labels; |
| 192 | } |
| 193 | |
| 194 | // ----------- Helper functions --------------- |
| 195 | |
| 196 | /** Convert a PageConfig to a wikitext string. */ |
| 197 | private static function pc2wt( PageConfig $pc ): string { |
| 198 | return $pc->getRevisionContent()->getContent( 'main' ); |
| 199 | } |
| 200 | |
| 201 | // See https://www.mediawiki.org/wiki/Manual:Stats#Cardinality |
| 202 | |
| 203 | /** Restrict the cardinality of user agent labels */ |
| 204 | public static function filterUserAgent( ?string $userAgent ): string { |
| 205 | static $acceptableAgents = [ |
| 206 | 'ChangePropagation_JobQueue_WMF' => true, |
| 207 | 'ChangePropagation_WMF' => true, |
| 208 | 'Mobileapps_WMF' => true, |
| 209 | 'RESTBase_WMF' => true, |
| 210 | 'C_WikiAPI' => true, |
| 211 | 'Java_7_0_for_MediaWikiAPI' => true, |
| 212 | ]; |
| 213 | static $agentPrefixes = [ |
| 214 | 'MediaWiki_API', |
| 215 | 'MediaWiki_Bot', |
| 216 | 'Mozilla_4_0', |
| 217 | 'Mozilla_5_0', |
| 218 | 'Mozilla', |
| 219 | 'REST_API_Crawler_Google', |
| 220 | 'IABot', |
| 221 | 'Rust_mediawiki_API', |
| 222 | 'ChangePropagation', // fallback |
| 223 | ]; |
| 224 | if ( $userAgent === null ) { |
| 225 | return 'unknown'; |
| 226 | } |
| 227 | // Replace non-alphanumeric characters, the same way that core does |
| 228 | // See mediawiki-core:includes/libs/Stats/StatsUtils::normalizeString() |
| 229 | $userAgent = preg_replace( '/\W+/', '_', $userAgent ); |
| 230 | $userAgent = trim( $userAgent, "_" ); |
| 231 | if ( $acceptableAgents[$userAgent] ?? false ) { |
| 232 | return $userAgent; |
| 233 | } |
| 234 | foreach ( $agentPrefixes as $prefix ) { |
| 235 | if ( str_starts_with( $userAgent, $prefix ) ) { |
| 236 | return $prefix; |
| 237 | } |
| 238 | } |
| 239 | return 'other'; |
| 240 | } |
| 241 | |
| 242 | /** |
| 243 | * Convert a boolean to a string for labelling purposes. |
| 244 | * |
| 245 | * @phan-return 'false'|'true'|'unknown' |
| 246 | */ |
| 247 | private static function bool2str( ?bool $val ): string { |
| 248 | return ( $val === true ) ? 'true' : ( |
| 249 | ( $val === false ) ? 'false' : 'unknown' |
| 250 | ); |
| 251 | } |
| 252 | |
| 253 | /** |
| 254 | * Convert an integer to a string for labelling purposes, |
| 255 | * restricting its cardinality. |
| 256 | */ |
| 257 | private static function int2str( ?int $val, ?int $limit = null ): string { |
| 258 | if ( $val === null ) { |
| 259 | return 'unknown'; |
| 260 | } |
| 261 | if ( $limit !== null && $val >= $limit ) { |
| 262 | return "{$limit}plus"; |
| 263 | } |
| 264 | return "$val"; |
| 265 | } |
| 266 | } |