Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
44.32% |
117 / 264 |
|
35.71% |
5 / 14 |
CRAP | |
0.00% |
0 / 1 |
| Database | |
44.32% |
117 / 264 |
|
35.71% |
5 / 14 |
638.76 | |
0.00% |
0 / 1 |
| __construct | |
100.00% |
4 / 4 |
|
100.00% |
1 / 1 |
1 | |||
| getDBConnectionRef | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| getReplicaDBConnection | |
100.00% |
1 / 1 |
|
100.00% |
1 / 1 |
1 | |||
| getFromId | |
0.00% |
0 / 10 |
|
0.00% |
0 / 1 |
6 | |||
| makeLintError | |
57.14% |
8 / 14 |
|
0.00% |
0 / 1 |
2.31 | |||
| getForPage | |
92.31% |
12 / 13 |
|
0.00% |
0 / 1 |
3.00 | |||
| buildErrorRow | |
47.62% |
10 / 21 |
|
0.00% |
0 / 1 |
8.59 | |||
| countByCat | |
100.00% |
6 / 6 |
|
100.00% |
1 / 1 |
3 | |||
| setForPage | |
97.83% |
45 / 46 |
|
0.00% |
0 / 1 |
13 | |||
| getTotalsEstimate | |
64.71% |
11 / 17 |
|
0.00% |
0 / 1 |
2.18 | |||
| getTotalsForPage | |
82.35% |
14 / 17 |
|
0.00% |
0 / 1 |
4.09 | |||
| getTotals | |
100.00% |
5 / 5 |
|
100.00% |
1 / 1 |
2 | |||
| migrateNamespace | |
0.00% |
0 / 57 |
|
0.00% |
0 / 1 |
72 | |||
| migrateTemplateAndTagInfo | |
0.00% |
0 / 52 |
|
0.00% |
0 / 1 |
132 | |||
| 1 | <?php |
| 2 | /** |
| 3 | * This program is free software; you can redistribute it and/or modify |
| 4 | * it under the terms of the GNU General Public License as published by |
| 5 | * the Free Software Foundation; either version 2 of the License, or |
| 6 | * (at your option) any later version. |
| 7 | * |
| 8 | * This program is distributed in the hope that it will be useful, |
| 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 11 | * GNU General Public License for more details. |
| 12 | * |
| 13 | * You should have received a copy of the GNU General Public License along |
| 14 | * with this program; if not, write to the Free Software Foundation, Inc., |
| 15 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
| 16 | * http://www.gnu.org/copyleft/gpl.html |
| 17 | * |
| 18 | * @file |
| 19 | */ |
| 20 | |
| 21 | namespace MediaWiki\Linter; |
| 22 | |
| 23 | use FormatJson; |
| 24 | use MediaWiki\Config\ServiceOptions; |
| 25 | use MediaWiki\Logger\LoggerFactory; |
| 26 | use stdClass; |
| 27 | use Wikimedia\Rdbms\IDatabase; |
| 28 | use Wikimedia\Rdbms\IReadableDatabase; |
| 29 | use Wikimedia\Rdbms\LBFactory; |
| 30 | use Wikimedia\Rdbms\SelectQueryBuilder; |
| 31 | |
| 32 | /** |
| 33 | * Database logic |
| 34 | */ |
| 35 | class Database { |
| 36 | public const CONSTRUCTOR_OPTIONS = [ |
| 37 | 'LinterWriteNamespaceColumnStage', |
| 38 | 'LinterWriteTagAndTemplateColumnsStage', |
| 39 | ]; |
| 40 | |
| 41 | /** |
| 42 | * Maximum number of errors to save per category, |
| 43 | * for a page, the rest are just dropped |
| 44 | */ |
| 45 | public const MAX_PER_CAT = 20; |
| 46 | public const MAX_ACCURATE_COUNT = 20; |
| 47 | |
| 48 | /** |
| 49 | * The linter_tag field has a maximum length of 32 characters, linter_template field a maximum of 255 characters |
| 50 | * so to ensure the length is not exceeded, the tag and template strings are truncated a few bytes below that limit |
| 51 | */ |
| 52 | public const MAX_TAG_LENGTH = 30; |
| 53 | public const MAX_TEMPLATE_LENGTH = 250; |
| 54 | |
| 55 | private ServiceOptions $options; |
| 56 | private CategoryManager $categoryManager; |
| 57 | private LBFactory $dbLoadBalancerFactory; |
| 58 | |
| 59 | /** |
| 60 | * @param ServiceOptions $options |
| 61 | * @param CategoryManager $categoryManager |
| 62 | * @param LBFactory $dbLoadBalancerFactory |
| 63 | */ |
| 64 | public function __construct( |
| 65 | ServiceOptions $options, |
| 66 | CategoryManager $categoryManager, |
| 67 | LBFactory $dbLoadBalancerFactory |
| 68 | ) { |
| 69 | $options->assertRequiredOptions( self::CONSTRUCTOR_OPTIONS ); |
| 70 | $this->options = $options; |
| 71 | $this->categoryManager = $categoryManager; |
| 72 | $this->dbLoadBalancerFactory = $dbLoadBalancerFactory; |
| 73 | } |
| 74 | |
| 75 | /** |
| 76 | * @param int $mode DB_PRIMARY or DB_REPLICA |
| 77 | * @return IDatabase |
| 78 | */ |
| 79 | public function getDBConnectionRef( int $mode ): IDatabase { |
| 80 | return $this->dbLoadBalancerFactory->getMainLB()->getConnection( $mode ); |
| 81 | } |
| 82 | |
| 83 | /** |
| 84 | * @return IReadableDatabase |
| 85 | */ |
| 86 | public function getReplicaDBConnection(): IReadableDatabase { |
| 87 | return $this->dbLoadBalancerFactory->getReplicaDatabase(); |
| 88 | } |
| 89 | |
| 90 | /** |
| 91 | * Get a specific LintError by id |
| 92 | * |
| 93 | * @param int $id linter_id |
| 94 | * @return bool|LintError |
| 95 | */ |
| 96 | public function getFromId( int $id ) { |
| 97 | $row = $this->getReplicaDBConnection()->newSelectQueryBuilder() |
| 98 | ->select( [ 'linter_cat', 'linter_params', 'linter_start', 'linter_end' ] ) |
| 99 | ->from( 'linter' ) |
| 100 | ->where( [ 'linter_id' => $id ] ) |
| 101 | ->caller( __METHOD__ ) |
| 102 | ->fetchRow(); |
| 103 | |
| 104 | if ( $row ) { |
| 105 | $row->linter_id = $id; |
| 106 | return self::makeLintError( $this->categoryManager, $row ); |
| 107 | } else { |
| 108 | return false; |
| 109 | } |
| 110 | } |
| 111 | |
| 112 | /** |
| 113 | * Turn a database row into a LintError object |
| 114 | * |
| 115 | * @param CategoryManager $categoryManager |
| 116 | * @param stdClass $row |
| 117 | * @return LintError|bool false on error |
| 118 | */ |
| 119 | public static function makeLintError( CategoryManager $categoryManager, $row ) { |
| 120 | try { |
| 121 | $name = $categoryManager->getCategoryName( $row->linter_cat ); |
| 122 | } catch ( MissingCategoryException $e ) { |
| 123 | LoggerFactory::getInstance( 'Linter' )->error( |
| 124 | 'Could not find name for id: {linter_cat}', |
| 125 | [ 'linter_cat' => $row->linter_cat ] |
| 126 | ); |
| 127 | return false; |
| 128 | } |
| 129 | return new LintError( |
| 130 | $name, |
| 131 | [ (int)$row->linter_start, (int)$row->linter_end ], |
| 132 | $row->linter_params, |
| 133 | $row->linter_cat, |
| 134 | (int)$row->linter_id |
| 135 | ); |
| 136 | } |
| 137 | |
| 138 | /** |
| 139 | * Get all the lint errors for a page |
| 140 | * |
| 141 | * @param int $pageId |
| 142 | * @return LintError[] |
| 143 | */ |
| 144 | public function getForPage( int $pageId ) { |
| 145 | $rows = $this->getReplicaDBConnection()->newSelectQueryBuilder() |
| 146 | ->select( [ 'linter_id', 'linter_cat', 'linter_start', 'linter_end', 'linter_params' ] ) |
| 147 | ->from( 'linter' ) |
| 148 | ->where( [ 'linter_page' => $pageId ] ) |
| 149 | ->caller( __METHOD__ ) |
| 150 | ->fetchResultSet(); |
| 151 | |
| 152 | $result = []; |
| 153 | foreach ( $rows as $row ) { |
| 154 | $error = self::makeLintError( $this->categoryManager, $row ); |
| 155 | if ( !$error ) { |
| 156 | continue; |
| 157 | } |
| 158 | $result[$error->id()] = $error; |
| 159 | } |
| 160 | |
| 161 | return $result; |
| 162 | } |
| 163 | |
| 164 | /** |
| 165 | * Convert a LintError object into an array for |
| 166 | * inserting/querying in the database |
| 167 | * |
| 168 | * @param int $pageId |
| 169 | * @param int $namespaceId |
| 170 | * @param LintError $error |
| 171 | * @return array |
| 172 | */ |
| 173 | private function buildErrorRow( int $pageId, int $namespaceId, LintError $error ) { |
| 174 | $result = [ |
| 175 | 'linter_page' => $pageId, |
| 176 | 'linter_cat' => $this->categoryManager->getCategoryId( $error->category, $error->catId ), |
| 177 | 'linter_params' => FormatJson::encode( $error->params, false, FormatJson::ALL_OK ), |
| 178 | 'linter_start' => $error->location[ 0 ], |
| 179 | 'linter_end' => $error->location[ 1 ] |
| 180 | ]; |
| 181 | |
| 182 | // To enable 756101 |
| 183 | // |
| 184 | // During the addition of this column to the table, the initial value |
| 185 | // of null allows the migrate stage code to determine the needs to fill |
| 186 | // in the field for that record, as the record was created prior to the |
| 187 | // write stage code being active and filling it in during record |
| 188 | // creation. Once the migrate code runs once no nulls should exist in |
| 189 | // this field for any record, and if the migrate code times out during |
| 190 | // execution, can be restarted and continue without duplicating work. |
| 191 | // The final code that enables the use of this field during records |
| 192 | // search will depend on this fields index being valid for all records. |
| 193 | if ( $this->options->get( 'LinterWriteNamespaceColumnStage' ) ) { |
| 194 | $result[ 'linter_namespace' ] = $namespaceId; |
| 195 | } |
| 196 | |
| 197 | // To enable 720130 |
| 198 | if ( $this->options->get( 'LinterWriteTagAndTemplateColumnsStage' ) ) { |
| 199 | $templateInfo = $error->templateInfo ?? ''; |
| 200 | if ( is_array( $templateInfo ) ) { |
| 201 | if ( isset( $templateInfo[ 'multiPartTemplateBlock' ] ) ) { |
| 202 | $templateInfo = 'multi-part-template-block'; |
| 203 | } else { |
| 204 | $templateInfo = $templateInfo[ 'name' ] ?? ''; |
| 205 | } |
| 206 | } |
| 207 | $templateInfo = mb_strcut( $templateInfo, 0, self::MAX_TEMPLATE_LENGTH ); |
| 208 | $result[ 'linter_template' ] = $templateInfo; |
| 209 | |
| 210 | $tagInfo = $error->tagInfo ?? ''; |
| 211 | $tagInfo = mb_strcut( $tagInfo, 0, self::MAX_TAG_LENGTH ); |
| 212 | $result[ 'linter_tag' ] = $tagInfo; |
| 213 | } |
| 214 | |
| 215 | return $result; |
| 216 | } |
| 217 | |
| 218 | /** |
| 219 | * @param LintError[] $errors |
| 220 | * @return array |
| 221 | */ |
| 222 | private function countByCat( array $errors ) { |
| 223 | $count = []; |
| 224 | foreach ( $errors as $error ) { |
| 225 | if ( !isset( $count[$error->category] ) ) { |
| 226 | $count[$error->category] = 1; |
| 227 | } else { |
| 228 | $count[$error->category] += 1; |
| 229 | } |
| 230 | } |
| 231 | |
| 232 | return $count; |
| 233 | } |
| 234 | |
| 235 | /** |
| 236 | * Save the specified lint errors in the |
| 237 | * database |
| 238 | * |
| 239 | * @param int $pageId |
| 240 | * @param int $namespaceId |
| 241 | * @param LintError[] $errors |
| 242 | * @return array [ 'deleted' => [ cat => count ], 'added' => [ cat => count ] ] |
| 243 | */ |
| 244 | public function setForPage( int $pageId, int $namespaceId, $errors ) { |
| 245 | $previous = $this->getForPage( $pageId ); |
| 246 | $dbw = $this->getDBConnectionRef( DB_PRIMARY ); |
| 247 | if ( !$previous && !$errors ) { |
| 248 | return [ 'deleted' => [], 'added' => [] ]; |
| 249 | } elseif ( !$previous && $errors ) { |
| 250 | $toInsert = array_values( $errors ); |
| 251 | $toDelete = []; |
| 252 | } elseif ( $previous && !$errors ) { |
| 253 | $dbw->newDeleteQueryBuilder() |
| 254 | ->deleteFrom( 'linter' ) |
| 255 | ->where( [ 'linter_page' => $pageId ] ) |
| 256 | ->caller( __METHOD__ ) |
| 257 | ->execute(); |
| 258 | return [ 'deleted' => $this->countByCat( $previous ), 'added' => [] ]; |
| 259 | } else { |
| 260 | $toInsert = []; |
| 261 | $toDelete = $previous; |
| 262 | // Diff previous and errors |
| 263 | foreach ( $errors as $error ) { |
| 264 | $uniqueId = $error->id(); |
| 265 | if ( isset( $previous[$uniqueId] ) ) { |
| 266 | unset( $toDelete[$uniqueId] ); |
| 267 | } else { |
| 268 | $toInsert[] = $error; |
| 269 | } |
| 270 | } |
| 271 | } |
| 272 | |
| 273 | if ( $toDelete ) { |
| 274 | $ids = []; |
| 275 | foreach ( $toDelete as $lintError ) { |
| 276 | if ( $lintError->lintId ) { |
| 277 | $ids[] = $lintError->lintId; |
| 278 | } |
| 279 | } |
| 280 | $dbw->newDeleteQueryBuilder() |
| 281 | ->deleteFrom( 'linter' ) |
| 282 | ->where( [ 'linter_id' => $ids ] ) |
| 283 | ->caller( __METHOD__ ) |
| 284 | ->execute(); |
| 285 | } |
| 286 | |
| 287 | if ( $toInsert ) { |
| 288 | // Insert into db, ignoring any duplicate key errors |
| 289 | // since they're the same lint error |
| 290 | $dbw->newInsertQueryBuilder() |
| 291 | ->insertInto( 'linter' ) |
| 292 | ->ignore() |
| 293 | ->rows( |
| 294 | array_map( function ( LintError $error ) use ( $pageId, $namespaceId ) { |
| 295 | return $this->buildErrorRow( $pageId, $namespaceId, $error ); |
| 296 | }, $toInsert ) |
| 297 | ) |
| 298 | ->caller( __METHOD__ ) |
| 299 | ->execute(); |
| 300 | } |
| 301 | |
| 302 | return [ |
| 303 | 'deleted' => $this->countByCat( $toDelete ), |
| 304 | 'added' => $this->countByCat( $toInsert ), |
| 305 | ]; |
| 306 | } |
| 307 | |
| 308 | /** |
| 309 | * Get an estimate of how many rows are there for the |
| 310 | * specified category with EXPLAIN SELECT COUNT(*). |
| 311 | * If the category actually has no rows, then 0 will |
| 312 | * be returned. |
| 313 | * |
| 314 | * @param int $catId |
| 315 | * @return int |
| 316 | */ |
| 317 | private function getTotalsEstimate( $catId ) { |
| 318 | $dbr = $this->getReplicaDBConnection(); |
| 319 | // First see if there are no rows, or a moderate number |
| 320 | // within the limit specified by the MAX_ACCURATE_COUNT. |
| 321 | // The distinction between 0, a few and a lot is important |
| 322 | // to determine first, as estimateRowCount seem to never |
| 323 | // return 0 or accurate low error counts. |
| 324 | $rows = $dbr->newSelectQueryBuilder() |
| 325 | ->select( '*' ) |
| 326 | ->from( 'linter' ) |
| 327 | ->where( [ 'linter_cat' => $catId ] ) |
| 328 | // Select 1 more so we can see if we're over the max limit |
| 329 | ->limit( self::MAX_ACCURATE_COUNT + 1 ) |
| 330 | ->caller( __METHOD__ ) |
| 331 | ->fetchRowCount(); |
| 332 | // Return an accurate count if the number of errors is |
| 333 | // below the maximum accurate count limit |
| 334 | if ( $rows <= self::MAX_ACCURATE_COUNT ) { |
| 335 | return $rows; |
| 336 | } |
| 337 | // Now we can just estimate if the maximum accurate count limit |
| 338 | // was returned, which isn't the actual count but the limit reached |
| 339 | return $dbr->newSelectQueryBuilder() |
| 340 | ->select( '*' ) |
| 341 | ->from( 'linter' ) |
| 342 | ->where( [ 'linter_cat' => $catId ] ) |
| 343 | ->caller( __METHOD__ ) |
| 344 | ->estimateRowCount(); |
| 345 | } |
| 346 | |
| 347 | /** |
| 348 | * This uses COUNT(*), which is accurate, but can be significantly |
| 349 | * slower depending upon how many rows are in the database. |
| 350 | * |
| 351 | * @param int $pageId |
| 352 | * @return int[] |
| 353 | */ |
| 354 | public function getTotalsForPage( int $pageId ): array { |
| 355 | $rows = $this->getReplicaDBConnection()->newSelectQueryBuilder() |
| 356 | ->select( [ 'linter_cat', 'COUNT(*) AS count' ] ) |
| 357 | ->from( 'linter' ) |
| 358 | ->where( [ 'linter_page' => $pageId ] ) |
| 359 | ->caller( __METHOD__ ) |
| 360 | ->groupBy( 'linter_cat' ) |
| 361 | ->fetchResultSet(); |
| 362 | |
| 363 | // Initialize zero values |
| 364 | $categories = $this->categoryManager->getVisibleCategories(); |
| 365 | $ret = array_fill_keys( $categories, 0 ); |
| 366 | foreach ( $rows as $row ) { |
| 367 | try { |
| 368 | $catName = $this->categoryManager->getCategoryName( $row->linter_cat ); |
| 369 | } catch ( MissingCategoryException $e ) { |
| 370 | continue; |
| 371 | } |
| 372 | // Only set visible categories. Alternatively, we could add another |
| 373 | // where clause to the selection above. |
| 374 | if ( !in_array( $catName, $categories, true ) ) { |
| 375 | continue; |
| 376 | } |
| 377 | $ret[$catName] = (int)$row->count; |
| 378 | } |
| 379 | return $ret; |
| 380 | } |
| 381 | |
| 382 | /** |
| 383 | * @return int[] |
| 384 | */ |
| 385 | public function getTotals() { |
| 386 | $ret = []; |
| 387 | foreach ( $this->categoryManager->getVisibleCategories() as $cat ) { |
| 388 | $id = $this->categoryManager->getCategoryId( $cat ); |
| 389 | $ret[$cat] = $this->getTotalsEstimate( $id ); |
| 390 | } |
| 391 | |
| 392 | return $ret; |
| 393 | } |
| 394 | |
| 395 | /** |
| 396 | * This code migrates namespace ID identified by the Linter records linter_page |
| 397 | * field and populates the new linter_namespace field if it is unpopulated. |
| 398 | * This code is intended to be run once though it could be run multiple times |
| 399 | * using `--force` if needed via the maintenance script. |
| 400 | * It is safe to run more than once, and will quickly exit if no records need updating. |
| 401 | * |
| 402 | * @param int $pageBatchSize |
| 403 | * @param int $linterBatchSize |
| 404 | * @param int $sleep |
| 405 | * @param bool $bypassConfig |
| 406 | * @return int number of pages updated, each with one or more linter records |
| 407 | */ |
| 408 | public function migrateNamespace( |
| 409 | int $pageBatchSize, int $linterBatchSize, int $sleep, |
| 410 | bool $bypassConfig = false |
| 411 | ): int { |
| 412 | // code used by phpunit test, bypassed when run as a maintenance script |
| 413 | if ( !$bypassConfig ) { |
| 414 | if ( !$this->options->get( 'LinterWriteNamespaceColumnStage' ) ) { |
| 415 | return 0; |
| 416 | } |
| 417 | } |
| 418 | if ( gettype( $sleep ) !== 'integer' || $sleep < 0 ) { |
| 419 | $sleep = 0; |
| 420 | } |
| 421 | |
| 422 | $logger = LoggerFactory::getInstance( 'MigrateNamespaceChannel' ); |
| 423 | |
| 424 | $lbFactory = $this->dbLoadBalancerFactory; |
| 425 | $dbw = $this->getDBConnectionRef( DB_PRIMARY ); |
| 426 | $dbread = $this->getDBConnectionRef( DB_REPLICA ); |
| 427 | |
| 428 | $logger->info( "Migrate namespace starting\n" ); |
| 429 | |
| 430 | $updated = 0; |
| 431 | $lastElement = 0; |
| 432 | do { |
| 433 | // Gather some unique pageId values in linter table records into an array |
| 434 | $linterPages = []; |
| 435 | |
| 436 | $result = $dbw->newSelectQueryBuilder() |
| 437 | ->select( 'linter_page' ) |
| 438 | ->distinct() |
| 439 | ->from( 'linter' ) |
| 440 | ->where( $dbw->expr( 'linter_page', '>', $lastElement ) ) |
| 441 | ->andWhere( [ 'linter_namespace' => null ] ) |
| 442 | ->orderBy( 'linter_page' ) |
| 443 | ->limit( $linterBatchSize ) |
| 444 | ->caller( __METHOD__ ) |
| 445 | ->fetchResultSet(); |
| 446 | |
| 447 | foreach ( $result as $row ) { |
| 448 | $lastElement = intval( $row->linter_page ); |
| 449 | $linterPages[] = $lastElement; |
| 450 | } |
| 451 | $linterPagesLength = count( $linterPages ); |
| 452 | |
| 453 | $pageStartIndex = 0; |
| 454 | do { |
| 455 | $pageIdBatch = array_slice( $linterPages, $pageStartIndex, $pageBatchSize ); |
| 456 | |
| 457 | if ( count( $pageIdBatch ) > 0 ) { |
| 458 | |
| 459 | $pageResults = $dbread->newSelectQueryBuilder() |
| 460 | ->select( [ 'page_id', 'page_namespace' ] ) |
| 461 | ->from( 'page' ) |
| 462 | ->where( [ 'page_id' => $pageIdBatch ] ) |
| 463 | ->caller( __METHOD__ ) |
| 464 | ->fetchResultSet(); |
| 465 | |
| 466 | foreach ( $pageResults as $pageRow ) { |
| 467 | $pageId = intval( $pageRow->page_id ); |
| 468 | $namespaceId = intval( $pageRow->page_namespace ); |
| 469 | |
| 470 | // If a record about to be updated has been removed by another process, |
| 471 | // the update will not error, and continue updating the existing records. |
| 472 | $dbw->newUpdateQueryBuilder() |
| 473 | ->update( 'linter' ) |
| 474 | ->set( [ 'linter_namespace' => $namespaceId ] ) |
| 475 | ->where( [ |
| 476 | 'linter_namespace' => null, |
| 477 | 'linter_page' => $pageId |
| 478 | ] ) |
| 479 | ->caller( __METHOD__ ) |
| 480 | ->execute(); |
| 481 | $updated += $dbw->affectedRows(); |
| 482 | } |
| 483 | |
| 484 | // Sleep between batches for replication to catch up |
| 485 | $lbFactory->waitForReplication(); |
| 486 | sleep( $sleep ); |
| 487 | } |
| 488 | |
| 489 | $pageStartIndex += $pageBatchSize; |
| 490 | } while ( $linterPagesLength > $pageStartIndex ); |
| 491 | |
| 492 | $logger->info( 'Migrated ' . $updated . " page IDs\n" ); |
| 493 | |
| 494 | } while ( $linterPagesLength > 0 ); |
| 495 | |
| 496 | $logger->info( "Migrate namespace finished!\n" ); |
| 497 | |
| 498 | return $updated; |
| 499 | } |
| 500 | |
| 501 | /** |
| 502 | * This code migrates the content of Linter record linter_params to linter_template and |
| 503 | * linter_tag fields if they are unpopulated or stale. |
| 504 | * This code should only be run once and thereafter disabled but must run to completion. |
| 505 | * It can be restarted if interrupted and will pick up where new divergences are found. |
| 506 | * Note: When linter_params are not set, the content is set to '[]' indicating no content |
| 507 | * and the code also handles a null linter_params field if found. |
| 508 | * This code is only run once by maintenance script migrateTagTemplate.php |
| 509 | * |
| 510 | * @param int $batchSize |
| 511 | * @param int $sleep |
| 512 | * @param bool $bypassConfig |
| 513 | * @return int |
| 514 | */ |
| 515 | public function migrateTemplateAndTagInfo( |
| 516 | int $batchSize, int $sleep, bool $bypassConfig = false |
| 517 | ): int { |
| 518 | // code used by phpunit test, bypassed when run as a maintenance script |
| 519 | if ( !$bypassConfig ) { |
| 520 | if ( !$this->options->get( 'LinterWriteTagAndTemplateColumnsStage' ) ) { |
| 521 | return 0; |
| 522 | } |
| 523 | } |
| 524 | if ( gettype( $sleep ) !== 'integer' || $sleep < 0 ) { |
| 525 | $sleep = 0; |
| 526 | } |
| 527 | |
| 528 | $logger = LoggerFactory::getInstance( 'MigrateTagAndTemplateChannel' ); |
| 529 | |
| 530 | $lbFactory = $this->dbLoadBalancerFactory; |
| 531 | $dbw = $this->getDBConnectionRef( DB_PRIMARY ); |
| 532 | |
| 533 | $logger->info( "Migration of linter_params field to linter_tag and linter_template fields starting\n" ); |
| 534 | |
| 535 | $updated = 0; |
| 536 | $lastElement = 0; |
| 537 | do { |
| 538 | $results = $dbw->newSelectQueryBuilder() |
| 539 | ->select( [ 'linter_id', 'linter_params', 'linter_template', 'linter_tag' ] ) |
| 540 | ->from( 'linter' ) |
| 541 | ->where( [ |
| 542 | $dbw->expr( "linter_params", '!=', '[]' ), |
| 543 | $dbw->expr( "linter_params", '!=', null ), |
| 544 | $dbw->expr( "linter_id", '>', $lastElement ) |
| 545 | ] ) |
| 546 | ->orderBy( 'linter_id', selectQueryBuilder::SORT_ASC ) |
| 547 | ->limit( $batchSize ) |
| 548 | ->caller( __METHOD__ ) |
| 549 | ->fetchResultSet(); |
| 550 | $linterBatchLength = 0; |
| 551 | |
| 552 | foreach ( $results as $row ) { |
| 553 | $linter_id = intval( $row->linter_id ); |
| 554 | $lastElement = $linter_id; |
| 555 | $linter_params = FormatJson::decode( $row->linter_params ); |
| 556 | $templateInfo = $linter_params->templateInfo ?? ''; |
| 557 | if ( is_object( $templateInfo ) ) { |
| 558 | if ( isset( $templateInfo->multiPartTemplateBlock ) ) { |
| 559 | $templateInfo = 'multi-part-template-block'; |
| 560 | } else { |
| 561 | $templateInfo = $templateInfo->name ?? ''; |
| 562 | } |
| 563 | } |
| 564 | $templateInfo = mb_strcut( $templateInfo, 0, self::MAX_TEMPLATE_LENGTH ); |
| 565 | |
| 566 | $tagInfo = $linter_params->name ?? ''; |
| 567 | $tagInfo = mb_strcut( $tagInfo, 0, self::MAX_TAG_LENGTH ); |
| 568 | |
| 569 | // compare the content of linter_params to the template and tag field contents |
| 570 | // and if they diverge, update the field with the correct template and tag info. |
| 571 | // This behavior allows this function to be restarted should it be interrupted |
| 572 | // and avoids repeating database record updates that are already correct due to |
| 573 | // having been populated when the error record was created with the new recordLintError |
| 574 | // write code that populates the template and tag fields, or for records populated |
| 575 | // during a previous but interrupted run of this migrate code. |
| 576 | if ( $templateInfo != $row->linter_template || $tagInfo != $row->linter_tag ) { |
| 577 | // If the record about to be updated has been removed by another process, |
| 578 | // the update will not do anything and just return with no records updated. |
| 579 | $dbw->newUpdateQueryBuilder() |
| 580 | ->update( 'linter' ) |
| 581 | ->set( [ 'linter_template' => $templateInfo, 'linter_tag' => $tagInfo, ] ) |
| 582 | ->where( [ 'linter_id' => $linter_id ] ) |
| 583 | ->caller( __METHOD__ ) |
| 584 | ->execute(); |
| 585 | $updated += $dbw->affectedRows(); |
| 586 | } |
| 587 | $linterBatchLength++; |
| 588 | } |
| 589 | |
| 590 | // Sleep between batches for replication to catch up |
| 591 | $lbFactory->waitForReplication(); |
| 592 | if ( $sleep > 0 ) { |
| 593 | sleep( $sleep ); |
| 594 | } |
| 595 | |
| 596 | $logger->info( 'Migrated ' . $updated . " linter IDs\n" ); |
| 597 | |
| 598 | } while ( $linterBatchLength > 0 ); |
| 599 | |
| 600 | $logger->info( "Migrate linter_params to linter_tag and linter_template fields finished!\n" ); |
| 601 | |
| 602 | return $updated; |
| 603 | } |
| 604 | |
| 605 | } |