44 public $sizeHistogram = [];
47 private $numRowsProcessed = 0;
56 private $verboseStats;
62 private $collationName;
74 private $namespaceInfo;
77 parent::__construct();
80This script will find all rows in the categorylinks table whose collation is
82repopulate cl_sortkey
using the page title and cl_sortkey_prefix. If all
83collations are up-to-date, it will
do nothing.
88 $this->
addOption(
'force',
'Run on all rows, even if the collation is ' .
89 'supposed to be up-to-date.',
false,
false,
'f' );
90 $this->
addOption(
'previous-collation',
'Set the previous value of ' .
91 '$wgCategoryCollation here to speed up this script, especially if your ' .
92 'categorylinks table is large. This will only update rows with that ' .
93 'collation, though, so it may miss out-of-date rows with a different, ' .
94 'even older collation.',
false,
true );
95 $this->
addOption(
'target-collation',
'Set this to the new collation type to ' .
96 'use instead of $wgCategoryCollation. Usually you should not use this, ' .
97 'you should just update $wgCategoryCollation in LocalSettings.php.',
99 $this->
addOption(
'target-table',
'Copy rows from categorylinks into the ' .
100 'specified table instead of updating them in place.',
false,
true );
101 $this->
addOption(
'remote',
'Use Shellbox to calculate the new sort keys ' .
103 $this->
addOption(
'dry-run',
'Don\'t actually change the collations, just ' .
104 'compile statistics.' );
105 $this->
addOption(
'verbose-stats',
'Show more statistics.' );
111 private function init() {
112 $services = $this->getServiceContainer();
113 $this->namespaceInfo = $services->getNamespaceInfo();
115 if ( $this->hasOption(
'target-collation' ) ) {
116 $this->collationName = $this->getOption(
'target-collation' );
118 $this->collationName = $this->
getConfig()->get( MainConfigNames::CategoryCollation );
121 $realCollationName =
'remote-' . $this->collationName;
123 $realCollationName = $this->collationName;
125 $this->collation = $services->getCollationFactory()->makeCollation( $realCollationName );
129 $this->collation->getSortKey(
'MediaWiki' );
131 $this->force = $this->
getOption(
'force' );
132 $this->dryRun = $this->
getOption(
'dry-run' );
133 $this->verboseStats = $this->
getOption(
'verbose-stats' );
136 $this->targetTable = $this->
getOption(
'target-table' );
143 if ( $this->targetTable ) {
144 if ( !$this->dbw->tableExists( $this->targetTable, __METHOD__ ) ) {
145 $this->
output(
"Creating table {$this->targetTable}\n" );
147 'CREATE TABLE ' . $this->dbw->tableName( $this->targetTable ) .
148 ' LIKE ' . $this->dbw->tableName(
'categorylinks' ),
154 $collationConds = [];
155 if ( !$this->force && !$this->targetTable ) {
156 if ( $this->
hasOption(
'previous-collation' ) ) {
157 $collationConds[
'cl_collation'] = $this->
getOption(
'previous-collation' );
159 $collationConds[] = $this->dbr->expr(
'cl_collation',
'!=', $this->collationName );
162 $maxPageId = (int)$this->dbr->newSelectQueryBuilder()
163 ->select(
'MAX(page_id)' )
165 ->caller( __METHOD__ )->fetchField();
168 $this->
output(
"Selecting next $batchSize pages from cl_from = $batchValue... " );
172 if ( $this->dbw->getType() ===
'mysql' ) {
173 $clType =
'cl_type+0 AS "cl_type_numeric"';
177 $res = $this->dbw->newSelectQueryBuilder()
179 'cl_from',
'cl_to',
'cl_sortkey_prefix',
'cl_collation',
180 'cl_sortkey', $clType,
'cl_timestamp',
181 'page_namespace',
'page_title'
183 ->from(
'categorylinks' )
185 ->straightJoin(
'page',
null,
'cl_from = page_id' )
186 ->where( $collationConds )
188 $this->dbw->expr(
'cl_from',
'>=', $batchValue )
189 ->and(
'cl_from',
'<', $batchValue + $this->getBatchSize() )
191 ->orderBy(
'cl_from' )
192 ->caller( __METHOD__ )->fetchResultSet();
193 $this->
output(
"processing... " );
196 if ( $this->targetTable ) {
197 $this->copyBatch( $res );
199 $this->updateBatch( $res );
204 if ( $this->dryRun ) {
205 $this->
output(
"{$this->numRowsProcessed} rows would be updated so far.\n" );
207 $this->
output(
"{$this->numRowsProcessed} done.\n" );
209 }
while ( $maxPageId >= $batchValue );
211 if ( !$this->dryRun ) {
212 $this->
output(
"{$this->numRowsProcessed} rows processed\n" );
215 if ( $this->verboseStats ) {
217 $this->showSortKeySizeHistogram();
225 if ( !$this->dryRun ) {
226 $this->beginTransaction( $this->dbw, __METHOD__ );
228 foreach ( $res as $row ) {
229 $title = Title::newFromRow( $row );
230 if ( !$row->cl_collation ) {
231 # This is an old-style row, so the sortkey needs to be
233 if ( $row->cl_sortkey === $title->getText()
234 || $row->cl_sortkey === $title->getPrefixedText()
238 # Custom sortkey, so use it as a prefix
239 $prefix = $row->cl_sortkey;
242 $prefix = $row->cl_sortkey_prefix;
244 # cl_type will be wrong for lots of pages if cl_collation is 0,
245 # so let's update it while we're here.
246 $type = $this->namespaceInfo->getCategoryLinkType( $row->page_namespace );
247 $newSortKey = $this->collation->getSortKey(
248 $title->getCategorySortkey( $prefix ) );
249 $this->updateSortKeySizeHistogram( $newSortKey );
251 $newSortKey = substr( $newSortKey, 0, 230 );
253 if ( $this->dryRun ) {
256 $this->numRowsProcessed += ( $row->cl_sortkey !== $newSortKey );
258 $this->dbw->newUpdateQueryBuilder()
259 ->update(
'categorylinks' )
261 'cl_sortkey' => $newSortKey,
262 'cl_sortkey_prefix' => $prefix,
263 'cl_collation' => $this->collationName,
265 'cl_timestamp = cl_timestamp',
267 ->where( [
'cl_from' => $row->cl_from,
'cl_to' => $row->cl_to ] )
268 ->caller( __METHOD__ )
270 $this->numRowsProcessed++;
273 if ( !$this->dryRun ) {
283 foreach ( $res as $row ) {
284 $title = Title::newFromRow( $row );
285 $sortKeyInputs[] = $title->getCategorySortkey( $row->cl_sortkey_prefix );
287 $sortKeys = $this->collation->getSortKeys( $sortKeyInputs );
289 foreach ( $res as $i => $row ) {
290 if ( !isset( $sortKeys[$i] ) ) {
291 throw new RuntimeException(
'Unable to get sort key' );
293 $newSortKey = $sortKeys[$i];
294 $this->updateSortKeySizeHistogram( $newSortKey );
296 $newSortKey = substr( $newSortKey, 0, 230 );
297 $type = $this->namespaceInfo->getCategoryLinkType( $row->page_namespace );
299 'cl_from' => $row->cl_from,
300 'cl_to' => $row->cl_to,
301 'cl_sortkey' => $newSortKey,
302 'cl_sortkey_prefix' => $row->cl_sortkey_prefix,
303 'cl_collation' => $this->collationName,
305 'cl_timestamp' => $row->cl_timestamp
308 if ( $this->dryRun ) {
309 $this->numRowsProcessed += count( $rowsToInsert );
312 $this->dbw->newInsertQueryBuilder()
313 ->insertInto( $this->targetTable )
315 ->rows( $rowsToInsert )
316 ->caller( __METHOD__ )->execute();
317 $this->numRowsProcessed += $this->dbw->affectedRows();
325 private function updateSortKeySizeHistogram(
string $key ) {
326 if ( !$this->verboseStats ) {
329 $length = strlen( $key );
330 if ( !isset( $this->sizeHistogram[$length] ) ) {
331 $this->sizeHistogram[$length] = 0;
333 $this->sizeHistogram[$length]++;
339 private function showSortKeySizeHistogram() {
340 if ( !$this->sizeHistogram ) {
343 $maxLength = max( array_keys( $this->sizeHistogram ) );
344 if ( $maxLength === 0 ) {
348 $coarseHistogram = array_fill( 0, $numBins, 0 );
349 $coarseBoundaries = [];
351 for ( $i = 0; $i < $numBins - 1; $i++ ) {
352 $boundary += $maxLength / $numBins;
353 $coarseBoundaries[$i] = round( $boundary );
355 $coarseBoundaries[$numBins - 1] = $maxLength + 1;
357 for ( $i = 0; $i <= $maxLength; $i++ ) {
361 $val = $this->sizeHistogram[$i] ?? 0;
362 for ( $coarseIndex = 0; $coarseIndex < $numBins - 1; $coarseIndex++ ) {
364 if ( $coarseBoundaries[$coarseIndex] > $i ) {
365 $coarseHistogram[$coarseIndex] += $val;
369 if ( $coarseIndex === ( $numBins - 1 ) ) {
370 $coarseHistogram[$coarseIndex] += $val;
375 $this->
output(
"Sort key size histogram\nRaw data: $raw\n\n" );
377 $maxBinVal = max( $coarseHistogram );
378 $scale = (int)( 60 / $maxBinVal );
380 for ( $coarseIndex = 0; $coarseIndex < $numBins; $coarseIndex++ ) {
381 $val = $coarseHistogram[$coarseIndex] ?? 0;
383 $boundary = $coarseBoundaries[$coarseIndex];
385 sprintf(
"%-10s %-10d |%s\n",
386 $prevBoundary .
'-' . ( $boundary - 1 ) .
': ',
388 str_repeat(
'*', $scale * $val )
391 $prevBoundary = $boundary;
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
beginTransaction(IDatabase $dbw, $fname)
Begin a transaction on a DB.
commitTransaction(IDatabase $dbw, $fname)
Commit the transaction on a DB handle and wait for replica DBs to catch up.
output( $out, $channel=null)
Throw some output to the user.
hasOption( $name)
Checks to see if a particular option was set.
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.