27require_once __DIR__ .
'/Maintenance.php';
44 parent::__construct();
48This script will find all rows in the categorylinks
table whose collation is
49out-of-date (cl_collation !=
'$wgCategoryCollation') and repopulate cl_sortkey
50using the page title and cl_sortkey_prefix. If all collations are
51up-to-date, it will
do nothing.
55 $this->
addOption(
'force',
'Run on all rows, even if the collation is ' .
56 'supposed to be up-to-date.',
false,
false,
'f' );
57 $this->
addOption(
'previous-collation',
'Set the previous value of ' .
58 '$wgCategoryCollation here to speed up this script, especially if your ' .
59 'categorylinks table is large. This will only update rows with that ' .
60 'collation, though, so it may miss out-of-date rows with a different, ' .
61 'even older collation.',
false,
true );
62 $this->
addOption(
'target-collation',
'Set this to the new collation type to ' .
63 'use instead of $wgCategoryCollation. Usually you should not use this, ' .
64 'you should just update $wgCategoryCollation in LocalSettings.php.',
66 $this->
addOption(
'dry-run',
'Don\'t actually change the collations, just ' .
67 'compile statistics.' );
68 $this->
addOption(
'verbose-stats',
'Show more statistics.' );
78 $verboseStats = $this->
getOption(
'verbose-stats' );
79 if ( $this->
hasOption(
'target-collation' ) ) {
80 $collationName = $this->
getOption(
'target-collation' );
89 $collation->getFirstLetter(
'MediaWiki' );
95 if ( $this->
hasOption(
'previous-collation' ) ) {
96 $orderBy =
'cl_to, cl_type, cl_from';
98 $orderBy =
'cl_collation, cl_to, cl_type, cl_from';
102 'ORDER BY' => $orderBy,
107 $collationConds = [];
109 if ( $this->
hasOption(
'previous-collation' ) ) {
110 $collationConds[
'cl_collation'] = $this->
getOption(
'previous-collation' );
112 $collationConds = [ 0 =>
113 'cl_collation != ' . $dbw->addQuotes( $collationName )
117 $count =
$dbr->estimateRowCount(
124 if ( $count < 1000000 ) {
125 $count =
$dbr->selectField(
133 $this->
output(
"Collations up-to-date.\n" );
138 $this->
output(
"$count rows would be updated.\n" );
140 $this->
output(
"Fixing collation for $count rows.\n" );
148 $this->
output(
"Selecting next " . self::BATCH_SIZE .
" rows..." );
152 if ( $dbw->getType() ===
'mysql' ) {
153 $clType =
'cl_type+0 AS "cl_type_numeric"';
158 [
'categorylinks',
'page' ],
159 [
'cl_from',
'cl_to',
'cl_sortkey_prefix',
'cl_collation',
160 'cl_sortkey', $clType,
161 'page_namespace',
'page_title'
163 array_merge( $collationConds, $batchConds, [
'cl_from = page_id' ] ),
167 $this->
output(
" processing..." );
172 foreach (
$res as $row ) {
173 $title = Title::newFromRow( $row );
174 if ( !$row->cl_collation ) {
175 # This is an old-style row, so the sortkey needs to be
177 if ( $row->cl_sortkey == $title->getText()
178 || $row->cl_sortkey == $title->getPrefixedText()
182 # Custom sortkey, use it as a prefix
183 $prefix = $row->cl_sortkey;
186 $prefix = $row->cl_sortkey_prefix;
188 # cl_type will be wrong for lots of pages if cl_collation is 0,
189 # so let's update it while we're here.
192 } elseif ( $title->getNamespace() ==
NS_FILE ) {
197 $newSortKey = $collation->getSortKey(
198 $title->getCategorySortkey( $prefix ) );
199 if ( $verboseStats ) {
207 'cl_sortkey' => $newSortKey,
208 'cl_sortkey_prefix' => $prefix,
209 'cl_collation' => $collationName,
211 'cl_timestamp = cl_timestamp',
213 [
'cl_from' => $row->cl_from,
'cl_to' => $row->cl_to ],
225 $count +=
$res->numRows();
226 $this->
output(
"$count done.\n" );
228 if ( !$dryRun && ++$batchCount % self::SYNC_INTERVAL == 0 ) {
229 $this->
output(
"Waiting for replica DBs ... " );
231 $this->
output(
"done\n" );
233 }
while (
$res->numRows() == self::BATCH_SIZE );
235 $this->
output(
"$count rows processed\n" );
237 if ( $verboseStats ) {
251 if ( $this->
hasOption(
'previous-collation' ) ) {
252 $fields = [
'cl_to',
'cl_type',
'cl_from' ];
254 $fields = [
'cl_collation',
'cl_to',
'cl_type',
'cl_from' ];
259 foreach ( $fields as $field ) {
260 if ( $dbw->getType() ===
'mysql' && $field ===
'cl_type' ) {
263 $encValue = intval( $row->cl_type_numeric );
265 $encValue = $dbw->addQuotes( $row->$field );
267 $inequality =
"$field > $encValue";
268 $equality =
"$field = $encValue";
274 $cond .=
" OR ($prefix AND $inequality)";
275 $prefix .=
" AND $equality";
283 $length = strlen( $key );
284 if ( !isset( $this->sizeHistogram[$length] ) ) {
285 $this->sizeHistogram[$length] = 0;
287 $this->sizeHistogram[$length]++;
291 $maxLength = max( array_keys( $this->sizeHistogram ) );
292 if ( $maxLength == 0 ) {
296 $coarseHistogram = array_fill( 0, $numBins, 0 );
297 $coarseBoundaries = [];
299 for ( $i = 0; $i < $numBins - 1; $i++ ) {
300 $boundary += $maxLength / $numBins;
301 $coarseBoundaries[$i] = round( $boundary );
303 $coarseBoundaries[$numBins - 1] = $maxLength + 1;
305 for ( $i = 0; $i <= $maxLength; $i++ ) {
309 if ( !isset( $this->sizeHistogram[$i] ) ) {
312 $val = $this->sizeHistogram[$i];
314 for ( $coarseIndex = 0; $coarseIndex < $numBins - 1; $coarseIndex++ ) {
315 if ( $coarseBoundaries[$coarseIndex] > $i ) {
316 $coarseHistogram[$coarseIndex] += $val;
320 if ( $coarseIndex == $numBins - 1 ) {
321 $coarseHistogram[$coarseIndex] += $val;
326 $this->
output(
"Sort key size histogram\nRaw data: $raw\n\n" );
328 $maxBinVal = max( $coarseHistogram );
329 $scale = 60 / $maxBinVal;
331 for ( $coarseIndex = 0; $coarseIndex < $numBins; $coarseIndex++ ) {
332 if ( !isset( $coarseHistogram[$coarseIndex] ) ) {
335 $val = $coarseHistogram[$coarseIndex];
337 $boundary = $coarseBoundaries[$coarseIndex];
338 $this->
output( sprintf(
"%-10s %-10d |%s\n",
339 $prevBoundary .
'-' . ( $boundary - 1 ) .
': ',
341 str_repeat(
'*', $scale * $val ) ) );
342 $prevBoundary = $boundary;
$wgCategoryCollation
Specify how category names should be sorted, when listed on a category page.
wfWaitForSlaves( $ifWritesSince=null, $wiki=false, $cluster=false, $timeout=null)
Waits for the replica DBs to catch up to the master position.
static factory( $collationName)
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
beginTransaction(IDatabase $dbw, $fname)
Begin a transcation on a DB.
commitTransaction(IDatabase $dbw, $fname)
Commit the transcation on a DB handle and wait for replica DBs to catch up.
getDB( $db, $groups=[], $wiki=false)
Returns a database to be used by current maintenance script.
hasOption( $name)
Checks to see if a particular param exists.
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
Maintenance script that will find all rows in the categorylinks table whose collation is out-of-date.
showSortKeySizeHistogram()
execute()
Do the actual work.
__construct()
Default constructor.
updateSortKeySizeHistogram( $key)
getBatchCondition( $row, $dbw)
Return an SQL expression selecting rows which sort above the given row, assuming an ordering of cl_co...
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global then executing the whole list after the page is displayed We don t do anything smart like collating updates to the same table or such because the list is almost always going to have just one item on if so it s not worth the trouble Since there is a job queue in the jobs table
design txt This is a brief overview of the new design More thorough and up to date information is available on the documentation wiki at etc Handles the details of getting and saving to the user table of the and dealing with sessions and cookies OutputPage Encapsulates the entire HTML page that will be sent in response to any server request It is used by calling its functions to add in any and then calling output() to send it all. It could be easily changed to send incrementally if that becomes useful
null means default in associative array with keys and values unescaped Should be merged with default with a value of false meaning to suppress the attribute in associative array with keys and values unescaped & $options
require_once RUN_MAINTENANCE_IF_MAIN