MediaWiki REL1_30
updateCollation.php
Go to the documentation of this file.
1<?php
27require_once __DIR__ . '/Maintenance.php';
28
30
38 const BATCH_SIZE = 100; // Number of rows to process in one batch
39 const SYNC_INTERVAL = 5; // Wait for replica DBs after this many batches
40
41 public $sizeHistogram = [];
42
43 public function __construct() {
44 parent::__construct();
45
47 $this->addDescription( <<<TEXT
48This script will find all rows in the categorylinks table whose collation is
49out-of-date (cl_collation != '$wgCategoryCollation') and repopulate cl_sortkey
50using the page title and cl_sortkey_prefix. If all collations are
51up-to-date, it will do nothing.
52TEXT
53 );
54
55 $this->addOption( 'force', 'Run on all rows, even if the collation is ' .
56 'supposed to be up-to-date.', false, false, 'f' );
57 $this->addOption( 'previous-collation', 'Set the previous value of ' .
58 '$wgCategoryCollation here to speed up this script, especially if your ' .
59 'categorylinks table is large. This will only update rows with that ' .
60 'collation, though, so it may miss out-of-date rows with a different, ' .
61 'even older collation.', false, true );
62 $this->addOption( 'target-collation', 'Set this to the new collation type to ' .
63 'use instead of $wgCategoryCollation. Usually you should not use this, ' .
64 'you should just update $wgCategoryCollation in LocalSettings.php.',
65 false, true );
66 $this->addOption( 'dry-run', 'Don\'t actually change the collations, just ' .
67 'compile statistics.' );
68 $this->addOption( 'verbose-stats', 'Show more statistics.' );
69 }
70
71 public function execute() {
73
74 $dbw = $this->getDB( DB_MASTER );
75 $dbr = $this->getDB( DB_REPLICA );
76 $force = $this->getOption( 'force' );
77 $dryRun = $this->getOption( 'dry-run' );
78 $verboseStats = $this->getOption( 'verbose-stats' );
79 if ( $this->hasOption( 'target-collation' ) ) {
80 $collationName = $this->getOption( 'target-collation' );
81 $collation = Collation::factory( $collationName );
82 } else {
83 $collationName = $wgCategoryCollation;
84 $collation = Collation::singleton();
85 }
86
87 // Collation sanity check: in some cases the constructor will work,
88 // but this will raise an exception, breaking all category pages
89 $collation->getFirstLetter( 'MediaWiki' );
90
91 // Locally at least, (my local is a rather old version of mysql)
92 // mysql seems to filesort if there is both an equality
93 // (but not for an inequality) condition on cl_collation in the
94 // WHERE and it is also the first item in the ORDER BY.
95 if ( $this->hasOption( 'previous-collation' ) ) {
96 $orderBy = 'cl_to, cl_type, cl_from';
97 } else {
98 $orderBy = 'cl_collation, cl_to, cl_type, cl_from';
99 }
100 $options = [
101 'LIMIT' => self::BATCH_SIZE,
102 'ORDER BY' => $orderBy,
103 'STRAIGHT_JOIN' // per T58041
104 ];
105
106 if ( $force ) {
107 $collationConds = [];
108 } else {
109 if ( $this->hasOption( 'previous-collation' ) ) {
110 $collationConds['cl_collation'] = $this->getOption( 'previous-collation' );
111 } else {
112 $collationConds = [ 0 =>
113 'cl_collation != ' . $dbw->addQuotes( $collationName )
114 ];
115 }
116
117 $count = $dbr->estimateRowCount(
118 'categorylinks',
119 '*',
120 $collationConds,
121 __METHOD__
122 );
123 // Improve estimate if feasible
124 if ( $count < 1000000 ) {
125 $count = $dbr->selectField(
126 'categorylinks',
127 'COUNT(*)',
128 $collationConds,
129 __METHOD__
130 );
131 }
132 if ( $count == 0 ) {
133 $this->output( "Collations up-to-date.\n" );
134
135 return;
136 }
137 if ( $dryRun ) {
138 $this->output( "$count rows would be updated.\n" );
139 } else {
140 $this->output( "Fixing collation for $count rows.\n" );
141 }
143 }
144 $count = 0;
145 $batchCount = 0;
146 $batchConds = [];
147 do {
148 $this->output( "Selecting next " . self::BATCH_SIZE . " rows..." );
149
150 // cl_type must be selected as a number for proper paging because
151 // enums suck.
152 if ( $dbw->getType() === 'mysql' ) {
153 $clType = 'cl_type+0 AS "cl_type_numeric"';
154 } else {
155 $clType = 'cl_type';
156 }
157 $res = $dbw->select(
158 [ 'categorylinks', 'page' ],
159 [ 'cl_from', 'cl_to', 'cl_sortkey_prefix', 'cl_collation',
160 'cl_sortkey', $clType,
161 'page_namespace', 'page_title'
162 ],
163 array_merge( $collationConds, $batchConds, [ 'cl_from = page_id' ] ),
164 __METHOD__,
166 );
167 $this->output( " processing..." );
168
169 if ( !$dryRun ) {
170 $this->beginTransaction( $dbw, __METHOD__ );
171 }
172 foreach ( $res as $row ) {
173 $title = Title::newFromRow( $row );
174 if ( !$row->cl_collation ) {
175 # This is an old-style row, so the sortkey needs to be
176 # converted.
177 if ( $row->cl_sortkey == $title->getText()
178 || $row->cl_sortkey == $title->getPrefixedText()
179 ) {
180 $prefix = '';
181 } else {
182 # Custom sortkey, use it as a prefix
183 $prefix = $row->cl_sortkey;
184 }
185 } else {
186 $prefix = $row->cl_sortkey_prefix;
187 }
188 # cl_type will be wrong for lots of pages if cl_collation is 0,
189 # so let's update it while we're here.
190 if ( $title->getNamespace() == NS_CATEGORY ) {
191 $type = 'subcat';
192 } elseif ( $title->getNamespace() == NS_FILE ) {
193 $type = 'file';
194 } else {
195 $type = 'page';
196 }
197 $newSortKey = $collation->getSortKey(
198 $title->getCategorySortkey( $prefix ) );
199 if ( $verboseStats ) {
200 $this->updateSortKeySizeHistogram( $newSortKey );
201 }
202
203 if ( !$dryRun ) {
204 $dbw->update(
205 'categorylinks',
206 [
207 'cl_sortkey' => $newSortKey,
208 'cl_sortkey_prefix' => $prefix,
209 'cl_collation' => $collationName,
210 'cl_type' => $type,
211 'cl_timestamp = cl_timestamp',
212 ],
213 [ 'cl_from' => $row->cl_from, 'cl_to' => $row->cl_to ],
214 __METHOD__
215 );
216 }
217 if ( $row ) {
218 $batchConds = [ $this->getBatchCondition( $row, $dbw ) ];
219 }
220 }
221 if ( !$dryRun ) {
222 $this->commitTransaction( $dbw, __METHOD__ );
223 }
224
225 $count += $res->numRows();
226 $this->output( "$count done.\n" );
227
228 if ( !$dryRun && ++$batchCount % self::SYNC_INTERVAL == 0 ) {
229 $this->output( "Waiting for replica DBs ... " );
231 $this->output( "done\n" );
232 }
233 } while ( $res->numRows() == self::BATCH_SIZE );
234
235 $this->output( "$count rows processed\n" );
236
237 if ( $verboseStats ) {
238 $this->output( "\n" );
240 }
241 }
242
250 function getBatchCondition( $row, $dbw ) {
251 if ( $this->hasOption( 'previous-collation' ) ) {
252 $fields = [ 'cl_to', 'cl_type', 'cl_from' ];
253 } else {
254 $fields = [ 'cl_collation', 'cl_to', 'cl_type', 'cl_from' ];
255 }
256 $first = true;
257 $cond = false;
258 $prefix = false;
259 foreach ( $fields as $field ) {
260 if ( $dbw->getType() === 'mysql' && $field === 'cl_type' ) {
261 // Range conditions with enums are weird in mysql
262 // This must be a numeric literal, or it won't work.
263 $encValue = intval( $row->cl_type_numeric );
264 } else {
265 $encValue = $dbw->addQuotes( $row->$field );
266 }
267 $inequality = "$field > $encValue";
268 $equality = "$field = $encValue";
269 if ( $first ) {
270 $cond = $inequality;
271 $prefix = $equality;
272 $first = false;
273 } else {
274 $cond .= " OR ($prefix AND $inequality)";
275 $prefix .= " AND $equality";
276 }
277 }
278
279 return $cond;
280 }
281
282 function updateSortKeySizeHistogram( $key ) {
283 $length = strlen( $key );
284 if ( !isset( $this->sizeHistogram[$length] ) ) {
285 $this->sizeHistogram[$length] = 0;
286 }
287 $this->sizeHistogram[$length]++;
288 }
289
291 $maxLength = max( array_keys( $this->sizeHistogram ) );
292 if ( $maxLength == 0 ) {
293 return;
294 }
295 $numBins = 20;
296 $coarseHistogram = array_fill( 0, $numBins, 0 );
297 $coarseBoundaries = [];
298 $boundary = 0;
299 for ( $i = 0; $i < $numBins - 1; $i++ ) {
300 $boundary += $maxLength / $numBins;
301 $coarseBoundaries[$i] = round( $boundary );
302 }
303 $coarseBoundaries[$numBins - 1] = $maxLength + 1;
304 $raw = '';
305 for ( $i = 0; $i <= $maxLength; $i++ ) {
306 if ( $raw !== '' ) {
307 $raw .= ', ';
308 }
309 if ( !isset( $this->sizeHistogram[$i] ) ) {
310 $val = 0;
311 } else {
312 $val = $this->sizeHistogram[$i];
313 }
314 for ( $coarseIndex = 0; $coarseIndex < $numBins - 1; $coarseIndex++ ) {
315 if ( $coarseBoundaries[$coarseIndex] > $i ) {
316 $coarseHistogram[$coarseIndex] += $val;
317 break;
318 }
319 }
320 if ( $coarseIndex == $numBins - 1 ) {
321 $coarseHistogram[$coarseIndex] += $val;
322 }
323 $raw .= $val;
324 }
325
326 $this->output( "Sort key size histogram\nRaw data: $raw\n\n" );
327
328 $maxBinVal = max( $coarseHistogram );
329 $scale = 60 / $maxBinVal;
330 $prevBoundary = 0;
331 for ( $coarseIndex = 0; $coarseIndex < $numBins; $coarseIndex++ ) {
332 if ( !isset( $coarseHistogram[$coarseIndex] ) ) {
333 $val = 0;
334 } else {
335 $val = $coarseHistogram[$coarseIndex];
336 }
337 $boundary = $coarseBoundaries[$coarseIndex];
338 $this->output( sprintf( "%-10s %-10d |%s\n",
339 $prevBoundary . '-' . ( $boundary - 1 ) . ': ',
340 $val,
341 str_repeat( '*', $scale * $val ) ) );
342 $prevBoundary = $boundary;
343 }
344 }
345}
346
347$maintClass = "UpdateCollation";
348require_once RUN_MAINTENANCE_IF_MAIN;
$wgCategoryCollation
Specify how category names should be sorted, when listed on a category page.
wfWaitForSlaves( $ifWritesSince=null, $wiki=false, $cluster=false, $timeout=null)
Waits for the replica DBs to catch up to the master position.
static singleton()
Definition Collation.php:34
static factory( $collationName)
Definition Collation.php:48
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
beginTransaction(IDatabase $dbw, $fname)
Begin a transcation on a DB.
commitTransaction(IDatabase $dbw, $fname)
Commit the transcation on a DB handle and wait for replica DBs to catch up.
getDB( $db, $groups=[], $wiki=false)
Returns a database to be used by current maintenance script.
hasOption( $name)
Checks to see if a particular param exists.
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
Maintenance script that will find all rows in the categorylinks table whose collation is out-of-date.
execute()
Do the actual work.
__construct()
Default constructor.
updateSortKeySizeHistogram( $key)
getBatchCondition( $row, $dbw)
Return an SQL expression selecting rows which sort above the given row, assuming an ordering of cl_co...
if(! $regexes) $dbr
Definition cleanup.php:94
$res
Definition database.txt:21
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global then executing the whole list after the page is displayed We don t do anything smart like collating updates to the same table or such because the list is almost always going to have just one item on if so it s not worth the trouble Since there is a job queue in the jobs table
Definition deferred.txt:16
design txt This is a brief overview of the new design More thorough and up to date information is available on the documentation wiki at etc Handles the details of getting and saving to the user table of the and dealing with sessions and cookies OutputPage Encapsulates the entire HTML page that will be sent in response to any server request It is used by calling its functions to add in any and then calling output() to send it all. It could be easily changed to send incrementally if that becomes useful
null means default in associative array with keys and values unescaped Should be merged with default with a value of false meaning to suppress the attribute in associative array with keys and values unescaped & $options
Definition hooks.txt:1971
const NS_FILE
Definition Defines.php:71
const NS_CATEGORY
Definition Defines.php:79
Basic database interface for live and lazy-loaded relation database handles.
Definition IDatabase.php:40
require_once RUN_MAINTENANCE_IF_MAIN
const DB_REPLICA
Definition defines.php:25
const DB_MASTER
Definition defines.php:26