49 public $sizeHistogram = [];
52 private $numRowsProcessed = 0;
61 private $verboseStats;
67 private $collationName;
72 private bool $normalization =
false;
81 private $namespaceInfo;
84 parent::__construct();
87This script will find all rows in the categorylinks table whose collation is
89repopulate cl_sortkey
using the page title and cl_sortkey_prefix. If all
90collations are up-to-date, it will
do nothing.
95 $this->
addOption(
'force',
'Run on all rows, even if the collation is ' .
96 'supposed to be up-to-date.',
false,
false,
'f' );
97 $this->
addOption(
'previous-collation',
'Set the previous value of ' .
98 '$wgCategoryCollation here to speed up this script, especially if your ' .
99 'categorylinks table is large. This will only update rows with that ' .
100 'collation, though, so it may miss out-of-date rows with a different, ' .
101 'even older collation.',
false,
true );
102 $this->
addOption(
'target-collation',
'Set this to the new collation type to ' .
103 'use instead of $wgCategoryCollation. Usually you should not use this, ' .
104 'you should just update $wgCategoryCollation in LocalSettings.php.',
106 $this->
addOption(
'target-table',
'Copy rows from categorylinks into the ' .
107 'specified table instead of updating them in place.',
false,
true );
108 $this->
addOption(
'only-migrate-normalization',
'Only backfill cl_collation_id ' .
109 'field from cl_collation',
false );
110 $this->
addOption(
'remote',
'Use Shellbox to calculate the new sort keys ' .
112 $this->
addOption(
'dry-run',
'Don\'t actually change the collations, just ' .
113 'compile statistics.' );
114 $this->
addOption(
'verbose-stats',
'Show more statistics.' );
120 private function init() {
121 $services = $this->getServiceContainer();
122 $this->namespaceInfo = $services->getNamespaceInfo();
124 if ( $this->hasOption(
'target-collation' ) ) {
125 $this->collationName = $this->getOption(
'target-collation' );
127 $this->collationName = $this->
getConfig()->get( MainConfigNames::CategoryCollation );
130 $realCollationName =
'remote-' . $this->collationName;
132 $realCollationName = $this->collationName;
134 $this->collation = $services->getCollationFactory()->makeCollation( $realCollationName );
138 $this->collation->getSortKey(
'MediaWiki' );
140 $this->force = $this->
getOption(
'force' );
141 $this->dryRun = $this->
getOption(
'dry-run' );
142 $this->verboseStats = $this->
getOption(
'verbose-stats' );
145 $this->targetTable = $this->
getOption(
'target-table' );
146 $this->normalization = $this->
getOption(
'only-migrate-normalization',
false );
153 if ( $this->normalization ) {
154 $this->runNormalizationMigration();
158 if ( $this->targetTable ) {
159 if ( !$this->dbw->tableExists( $this->targetTable, __METHOD__ ) ) {
160 $this->
output(
"Creating table {$this->targetTable}\n" );
162 'CREATE TABLE ' . $this->dbw->tableName( $this->targetTable ) .
163 ' LIKE ' . $this->dbw->tableName(
'categorylinks' ),
169 $collationConds = [];
170 if ( !$this->force && !$this->targetTable ) {
171 if ( $this->
hasOption(
'previous-collation' ) ) {
172 $collationConds[
'cl_collation'] = $this->
getOption(
'previous-collation' );
174 $collationConds[] = $this->dbr->expr(
'cl_collation',
'!=', $this->collationName );
177 $maxPageId = (int)$this->dbr->newSelectQueryBuilder()
178 ->select(
'MAX(page_id)' )
180 ->caller( __METHOD__ )->fetchField();
183 $this->
output(
"Selecting next $batchSize pages from cl_from = $batchValue... " );
187 if ( $this->dbw->getType() ===
'mysql' ) {
188 $clType =
'cl_type+0 AS "cl_type_numeric"';
192 $res = $this->dbw->newSelectQueryBuilder()
194 'cl_from',
'cl_to',
'cl_sortkey_prefix',
'cl_collation',
195 'cl_sortkey', $clType,
'cl_timestamp',
196 'page_namespace',
'page_title'
198 ->from(
'categorylinks' )
200 ->straightJoin(
'page',
null,
'cl_from = page_id' )
201 ->where( $collationConds )
203 $this->dbw->expr(
'cl_from',
'>=', $batchValue )
204 ->and(
'cl_from',
'<', $batchValue + $this->getBatchSize() )
206 ->orderBy(
'cl_from' )
207 ->caller( __METHOD__ )->fetchResultSet();
208 $this->
output(
"processing... " );
211 if ( $this->targetTable ) {
212 $this->copyBatch( $res );
214 $this->updateBatch( $res );
219 if ( $this->dryRun ) {
220 $this->
output(
"{$this->numRowsProcessed} rows would be updated so far.\n" );
222 $this->
output(
"{$this->numRowsProcessed} done.\n" );
224 }
while ( $maxPageId >= $batchValue );
226 if ( !$this->dryRun ) {
227 $this->
output(
"{$this->numRowsProcessed} rows processed\n" );
230 if ( $this->verboseStats ) {
232 $this->showSortKeySizeHistogram();
240 if ( !$this->dryRun ) {
241 $this->beginTransaction( $this->dbw, __METHOD__ );
243 foreach ( $res as $row ) {
244 $title = Title::newFromRow( $row );
245 if ( !$row->cl_collation ) {
246 # This is an old-style row, so the sortkey needs to be
248 if ( $row->cl_sortkey === $title->getText()
249 || $row->cl_sortkey === $title->getPrefixedText()
253 # Custom sortkey, so use it as a prefix
254 $prefix = $row->cl_sortkey;
257 $prefix = $row->cl_sortkey_prefix;
259 # cl_type will be wrong for lots of pages if cl_collation is 0,
260 # so let's update it while we're here.
261 $type = $this->namespaceInfo->getCategoryLinkType( $row->page_namespace );
262 $newSortKey = $this->collation->getSortKey(
263 $title->getCategorySortkey( $prefix ) );
264 $this->updateSortKeySizeHistogram( $newSortKey );
266 $newSortKey = substr( $newSortKey, 0, 230 );
268 if ( $this->dryRun ) {
271 $this->numRowsProcessed += ( $row->cl_sortkey !== $newSortKey );
273 $this->dbw->newUpdateQueryBuilder()
274 ->update(
'categorylinks' )
276 'cl_sortkey' => $newSortKey,
277 'cl_sortkey_prefix' => $prefix,
278 'cl_collation' => $this->collationName,
280 'cl_timestamp = cl_timestamp',
282 ->where( [
'cl_from' => $row->cl_from,
'cl_to' => $row->cl_to ] )
283 ->caller( __METHOD__ )
285 $this->numRowsProcessed++;
288 if ( !$this->dryRun ) {
298 foreach ( $res as $row ) {
299 $title = Title::newFromRow( $row );
300 $sortKeyInputs[] = $title->getCategorySortkey( $row->cl_sortkey_prefix );
302 $sortKeys = $this->collation->getSortKeys( $sortKeyInputs );
304 foreach ( $res as $i => $row ) {
305 if ( !isset( $sortKeys[$i] ) ) {
306 throw new RuntimeException(
'Unable to get sort key' );
308 $newSortKey = $sortKeys[$i];
309 $this->updateSortKeySizeHistogram( $newSortKey );
311 $newSortKey = substr( $newSortKey, 0, 230 );
312 $type = $this->namespaceInfo->getCategoryLinkType( $row->page_namespace );
314 'cl_from' => $row->cl_from,
315 'cl_to' => $row->cl_to,
316 'cl_sortkey' => $newSortKey,
317 'cl_sortkey_prefix' => $row->cl_sortkey_prefix,
318 'cl_collation' => $this->collationName,
320 'cl_timestamp' => $row->cl_timestamp
323 if ( $this->dryRun ) {
324 $this->numRowsProcessed += count( $rowsToInsert );
327 $this->dbw->newInsertQueryBuilder()
328 ->insertInto( $this->targetTable )
330 ->rows( $rowsToInsert )
331 ->caller( __METHOD__ )->execute();
332 $this->numRowsProcessed += $this->dbw->affectedRows();
340 private function updateSortKeySizeHistogram(
string $key ) {
341 if ( !$this->verboseStats ) {
344 $length = strlen( $key );
345 if ( !isset( $this->sizeHistogram[$length] ) ) {
346 $this->sizeHistogram[$length] = 0;
348 $this->sizeHistogram[$length]++;
354 private function showSortKeySizeHistogram() {
355 if ( !$this->sizeHistogram ) {
358 $maxLength = max( array_keys( $this->sizeHistogram ) );
359 if ( $maxLength === 0 ) {
363 $coarseHistogram = array_fill( 0, $numBins, 0 );
364 $coarseBoundaries = [];
366 for ( $i = 0; $i < $numBins - 1; $i++ ) {
367 $boundary += $maxLength / $numBins;
368 $coarseBoundaries[$i] = round( $boundary );
370 $coarseBoundaries[$numBins - 1] = $maxLength + 1;
372 for ( $i = 0; $i <= $maxLength; $i++ ) {
376 $val = $this->sizeHistogram[$i] ?? 0;
377 for ( $coarseIndex = 0; $coarseIndex < $numBins - 1; $coarseIndex++ ) {
379 if ( $coarseBoundaries[$coarseIndex] > $i ) {
380 $coarseHistogram[$coarseIndex] += $val;
384 if ( $coarseIndex === ( $numBins - 1 ) ) {
385 $coarseHistogram[$coarseIndex] += $val;
390 $this->
output(
"Sort key size histogram\nRaw data: $raw\n\n" );
392 $maxBinVal = max( $coarseHistogram );
393 $scale = (int)( 60 / $maxBinVal );
395 for ( $coarseIndex = 0; $coarseIndex < $numBins; $coarseIndex++ ) {
396 $val = $coarseHistogram[$coarseIndex] ?? 0;
398 $boundary = $coarseBoundaries[$coarseIndex];
400 sprintf(
"%-10s %-10d |%s\n",
401 $prevBoundary .
'-' . ( $boundary - 1 ) .
': ',
403 str_repeat(
'*', $scale * $val )
406 $prevBoundary = $boundary;
410 private function runNormalizationMigration() {
411 $maxPageId = (int)$this->dbr->newSelectQueryBuilder()
412 ->select(
'MAX(page_id)' )
414 ->caller( __METHOD__ )->fetchField();
421 LoggerFactory::getInstance(
'SecondaryDataUpdate' ),
427 $this->
output(
"Selecting next $batchSize pages from cl_from = $batchValue... " );
429 $res = $this->dbw->newSelectQueryBuilder()
430 ->select( [
'cl_collation' ] )
432 ->from(
'categorylinks' )
433 ->where( [
'cl_collation_id' => 0 ] )
435 $this->dbw->expr(
'cl_from',
'>=', $batchValue )
436 ->and(
'cl_from',
'<', $batchValue + $this->getBatchSize() )
438 ->orderBy(
'cl_from' )
439 ->caller( __METHOD__ )->fetchResultSet();
440 $this->
output(
"processing... " );
442 if ( $res->
numRows() && !$this->dryRun ) {
443 foreach ( $res as $row ) {
444 $collationName = $row->cl_collation;
445 $collationId = $collationNameStore->acquireId( $collationName );
446 $this->dbw->newUpdateQueryBuilder()
447 ->update(
'categorylinks' )
448 ->set( [
'cl_collation_id' => $collationId ] )
449 ->where( [
'cl_collation' => $collationName ] )
451 $this->dbw->expr(
'cl_from',
'>=', $batchValue )
452 ->and(
'cl_from',
'<', $batchValue + $this->getBatchSize() )
454 ->caller( __METHOD__ )->execute();
455 $this->numRowsProcessed += $this->dbw->affectedRows();
460 $this->
output(
"{$this->numRowsProcessed} done.\n" );
461 }
while ( $maxPageId >= $batchValue );
463 if ( !$this->dryRun ) {
464 $this->
output(
"{$this->numRowsProcessed} rows processed\n" );