48 private $numRowsProcessed = 0;
57 private $verboseStats;
63 private $collationName;
78 private $namespaceInfo;
81 parent::__construct();
84This script will find all rows in the categorylinks table whose collation is
86repopulate cl_sortkey
using the page title and cl_sortkey_prefix. If all
87collations are up-to-date, it will
do nothing.
92 $this->
addOption(
'force',
'Run on all rows, even if the collation is ' .
93 'supposed to be up-to-date.',
false,
false,
'f' );
94 $this->
addOption(
'previous-collation',
'Set the previous value of ' .
95 '$wgCategoryCollation here to speed up this script, especially if your ' .
96 'categorylinks table is large. This will only update rows with that ' .
97 'collation, though, so it may miss out-of-date rows with a different, ' .
98 'even older collation.',
false,
true );
99 $this->
addOption(
'target-collation',
'Set this to the new collation type to ' .
100 'use instead of $wgCategoryCollation. Usually you should not use this, ' .
101 'you should just update $wgCategoryCollation in LocalSettings.php.',
103 $this->
addOption(
'target-table',
'Copy rows from categorylinks into the ' .
104 'specified table instead of updating them in place.',
false,
true );
105 $this->
addOption(
'remote',
'Use Shellbox to calculate the new sort keys ' .
107 $this->
addOption(
'dry-run',
'Don\'t actually change the collations, just ' .
108 'compile statistics.' );
109 $this->
addOption(
'verbose-stats',
'Show more statistics.' );
115 private function init() {
117 $this->namespaceInfo = $services->getNamespaceInfo();
118 $this->lbFactory = $services->getDBLoadBalancerFactory();
120 if ( $this->
hasOption(
'target-collation' ) ) {
121 $this->collationName = $this->
getOption(
'target-collation' );
123 $this->collationName = $this->
getConfig()->get( MainConfigNames::CategoryCollation );
126 $realCollationName =
'remote-' . $this->collationName;
128 $realCollationName = $this->collationName;
130 $this->collation = $services->getCollationFactory()->makeCollation( $realCollationName );
134 $this->collation->getSortKey(
'MediaWiki' );
136 $this->force = $this->
getOption(
'force' );
137 $this->dryRun = $this->
getOption(
'dry-run' );
138 $this->verboseStats = $this->
getOption(
'verbose-stats' );
141 $this->targetTable = $this->
getOption(
'target-table' );
148 if ( $this->targetTable ) {
149 if ( !$this->dbw->tableExists( $this->targetTable, __METHOD__ ) ) {
150 $this->
output(
"Creating table {$this->targetTable}\n" );
152 'CREATE TABLE ' . $this->dbw->tableName( $this->targetTable ) .
153 ' LIKE ' . $this->dbw->tableName(
'categorylinks' ),
163 if ( $this->
hasOption(
'previous-collation' ) ) {
164 $orderBy =
'cl_to, cl_type, cl_from';
166 $orderBy =
'cl_collation, cl_to, cl_type, cl_from';
169 $collationConds = [];
170 if ( !$this->force && !$this->targetTable ) {
171 if ( $this->
hasOption(
'previous-collation' ) ) {
172 $collationConds[
'cl_collation'] = $this->
getOption(
'previous-collation' );
175 0 =>
'cl_collation != ' . $this->dbr->addQuotes( $this->collationName )
179 $count = $this->dbr->estimateRowCount(
186 if ( $count < 1000000 ) {
187 $count = $this->dbr->newSelectQueryBuilder()
188 ->select(
'COUNT(*)' )
189 ->from(
'categorylinks' )
190 ->where( $collationConds )
191 ->caller( __METHOD__ )->fetchField();
194 $this->
output(
"Collations up-to-date.\n" );
198 if ( $this->dryRun ) {
199 $this->
output(
"$count rows would be updated.\n" );
201 $this->
output(
"Fixing collation for $count rows.\n" );
206 $this->
output(
"Selecting next $batchSize rows..." );
210 if ( $this->dbw->getType() ===
'mysql' ) {
211 $clType =
'cl_type+0 AS "cl_type_numeric"';
215 $res = $this->dbw->newSelectQueryBuilder()
217 'cl_from',
'cl_to',
'cl_sortkey_prefix',
'cl_collation',
218 'cl_sortkey', $clType,
'cl_timestamp',
219 'page_namespace',
'page_title'
221 ->from(
'categorylinks' )
223 ->straightJoin(
'page',
null,
'cl_from = page_id' )
224 ->where( $collationConds )
225 ->andWhere( $batchConds )
226 ->limit( $batchSize )
227 ->orderBy( $orderBy )
228 ->caller( __METHOD__ )->fetchResultSet();
229 $this->
output(
" processing..." );
231 if ( $res->numRows() ) {
232 if ( $this->targetTable ) {
233 $this->copyBatch( $res );
235 $this->updateBatch( $res );
237 $res->seek( $res->numRows() - 1 );
238 $lastRow = $res->fetchObject();
239 $batchConds = [ $this->getBatchCondition( $lastRow, $this->dbw ) ];
242 if ( $this->dryRun ) {
243 $this->
output(
"{$this->numRowsProcessed} rows would be updated so far.\n" );
245 $this->
output(
"{$this->numRowsProcessed} done.\n" );
247 }
while ( $res->numRows() == $batchSize );
249 if ( !$this->dryRun ) {
250 $this->
output(
"{$this->numRowsProcessed} rows processed\n" );
253 if ( $this->verboseStats ) {
255 $this->showSortKeySizeHistogram();
266 private function getBatchCondition( $row, $dbw ) {
267 if ( $this->
hasOption(
'previous-collation' ) ) {
268 $fields = [
'cl_to',
'cl_type',
'cl_from' ];
270 $fields = [
'cl_collation',
'cl_to',
'cl_type',
'cl_from' ];
273 foreach ( $fields as $field ) {
274 if ( $dbw->
getType() ===
'mysql' && $field ===
'cl_type' ) {
277 $value = intval( $row->cl_type_numeric );
279 $value = $row->$field;
281 $conds[ $field ] = $value;
292 private function updateBatch( $res ) {
293 if ( !$this->dryRun ) {
296 foreach ( $res as $row ) {
297 $title = Title::newFromRow( $row );
298 if ( !$row->cl_collation ) {
299 # This is an old-style row, so the sortkey needs to be
301 if ( $row->cl_sortkey == $title->getText()
302 || $row->cl_sortkey == $title->getPrefixedText()
306 # Custom sortkey, use it as a prefix
307 $prefix = $row->cl_sortkey;
310 $prefix = $row->cl_sortkey_prefix;
312 # cl_type will be wrong for lots of pages if cl_collation is 0,
313 # so let's update it while we're here.
314 $type = $this->namespaceInfo->getCategoryLinkType( $row->page_namespace );
315 $newSortKey = $this->collation->getSortKey(
316 $title->getCategorySortkey( $prefix ) );
317 $this->updateSortKeySizeHistogram( $newSortKey );
319 $newSortKey = substr( $newSortKey, 0, 230 );
321 if ( $this->dryRun ) {
324 $this->numRowsProcessed += ( $row->cl_sortkey !== $newSortKey );
329 'cl_sortkey' => $newSortKey,
330 'cl_sortkey_prefix' => $prefix,
331 'cl_collation' => $this->collationName,
333 'cl_timestamp = cl_timestamp',
335 [
'cl_from' => $row->cl_from,
'cl_to' => $row->cl_to ],
338 $this->numRowsProcessed++;
341 if ( !$this->dryRun ) {
351 private function copyBatch( $res ) {
353 foreach ( $res as $row ) {
354 $title = Title::newFromRow( $row );
355 $sortKeyInputs[] = $title->getCategorySortkey( $row->cl_sortkey_prefix );
357 $sortKeys = $this->collation->getSortKeys( $sortKeyInputs );
359 foreach ( $res as $i => $row ) {
360 if ( !isset( $sortKeys[$i] ) ) {
361 throw new RuntimeException(
'Unable to get sort key' );
363 $newSortKey = $sortKeys[$i];
364 $this->updateSortKeySizeHistogram( $newSortKey );
366 $newSortKey = substr( $newSortKey, 0, 230 );
367 $type = $this->namespaceInfo->getCategoryLinkType( $row->page_namespace );
369 'cl_from' => $row->cl_from,
370 'cl_to' => $row->cl_to,
371 'cl_sortkey' => $newSortKey,
372 'cl_sortkey_prefix' => $row->cl_sortkey_prefix,
373 'cl_collation' => $this->collationName,
375 'cl_timestamp' => $row->cl_timestamp
378 if ( $this->dryRun ) {
379 $this->numRowsProcessed += count( $rowsToInsert );
382 $this->dbw->insert( $this->targetTable, $rowsToInsert, __METHOD__, [
'IGNORE' ] );
383 $this->numRowsProcessed += $this->dbw->affectedRows();
393 private function updateSortKeySizeHistogram( $key ) {
394 if ( !$this->verboseStats ) {
397 $length = strlen( $key );
398 if ( !isset( $this->sizeHistogram[$length] ) ) {
399 $this->sizeHistogram[$length] = 0;
401 $this->sizeHistogram[$length]++;
407 private function showSortKeySizeHistogram() {
408 if ( !$this->sizeHistogram ) {
411 $maxLength = max( array_keys( $this->sizeHistogram ) );
412 if ( $maxLength == 0 ) {
416 $coarseHistogram = array_fill( 0, $numBins, 0 );
417 $coarseBoundaries = [];
419 for ( $i = 0; $i < $numBins - 1; $i++ ) {
420 $boundary += $maxLength / $numBins;
421 $coarseBoundaries[$i] = round( $boundary );
423 $coarseBoundaries[$numBins - 1] = $maxLength + 1;
425 for ( $i = 0; $i <= $maxLength; $i++ ) {
429 $val = $this->sizeHistogram[$i] ?? 0;
430 for ( $coarseIndex = 0; $coarseIndex < $numBins - 1; $coarseIndex++ ) {
432 if ( $coarseBoundaries[$coarseIndex] > $i ) {
433 $coarseHistogram[$coarseIndex] += $val;
437 if ( $coarseIndex == $numBins - 1 ) {
438 $coarseHistogram[$coarseIndex] += $val;
443 $this->
output(
"Sort key size histogram\nRaw data: $raw\n\n" );
445 $maxBinVal = max( $coarseHistogram );
446 $scale = (int)( 60 / $maxBinVal );
448 for ( $coarseIndex = 0; $coarseIndex < $numBins; $coarseIndex++ ) {
449 $val = $coarseHistogram[$coarseIndex] ?? 0;
451 $boundary = $coarseBoundaries[$coarseIndex];
452 $this->
output( sprintf(
"%-10s %-10d |%s\n",
453 $prevBoundary .
'-' . ( $boundary - 1 ) .
': ',
455 str_repeat(
'*', $scale * $val ) ) );
456 $prevBoundary = $boundary;
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
beginTransaction(IDatabase $dbw, $fname)
Begin a transaction on a DB.
commitTransaction(IDatabase $dbw, $fname)
Commit the transaction on a DB handle and wait for replica DBs to catch up.
output( $out, $channel=null)
Throw some output to the user.
hasOption( $name)
Checks to see if a particular option was set.
getServiceContainer()
Returns the main service container.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.