MediaWiki master
recountCategories.php
Go to the documentation of this file.
1<?php
26
27// @codeCoverageIgnoreStart
28require_once __DIR__ . '/Maintenance.php';
29// @codeCoverageIgnoreEnd
30
39 private $minimumId;
40
41 public function __construct() {
42 parent::__construct();
43 $this->addDescription( <<<'TEXT'
44This script refreshes the category membership counts stored in the category
45table. As time passes, these counts often drift from the actual number of
46category members. The script identifies rows where the value in the category
47table does not match the number of categorylinks rows for that category, and
48updates the category table accordingly.
49
50To fully refresh the data in the category table, you need to run this script
51for all three modes. Alternatively, just one mode can be run if required.
52TEXT
53 );
54 $this->addOption(
55 'mode',
56 '(REQUIRED) Which category count column to recompute: "pages", "subcats", "files" or "all".',
57 true,
58 true
59 );
60 $this->addOption(
61 'begin',
62 'Only recount categories with cat_id greater than the given value',
63 false,
64 true
65 );
66 $this->addOption(
67 'throttle',
68 'Wait this many milliseconds after each batch. Default: 0',
69 false,
70 true
71 );
72
73 $this->addOption(
74 'skip-cleanup',
75 'Skip running cleanupEmptyCategories if the "page" mode is selected',
76 false,
77 false
78 );
79
80 $this->setBatchSize( 500 );
81 }
82
83 public function execute() {
84 $originalMode = $this->getOption( 'mode' );
85 if ( !in_array( $originalMode, [ 'pages', 'subcats', 'files', 'all' ] ) ) {
86 $this->fatalError( 'Please specify a valid mode: one of "pages", "subcats", "files" or "all".' );
87 }
88
89 if ( $originalMode === 'all' ) {
90 $modes = [ 'pages', 'subcats', 'files' ];
91 } else {
92 $modes = [ $originalMode ];
93 }
94
95 foreach ( $modes as $mode ) {
96 $this->output( "Starting to recount {$mode} counts.\n" );
97 $this->minimumId = intval( $this->getOption( 'begin', 0 ) );
98
99 // do the work, batch by batch
100 $affectedRows = 0;
101 // phpcs:ignore Generic.CodeAnalysis.AssignmentInCondition.FoundInWhileCondition
102 while ( ( $result = $this->doWork( $mode ) ) !== false ) {
103 $affectedRows += $result;
104 usleep( $this->getOption( 'throttle', 0 ) * 1000 );
105 }
106
107 $this->output( "Updated the {$mode} counts of $affectedRows categories.\n" );
108 }
109
110 // Finished
111 $this->output( "Done!\n" );
112 if ( $originalMode !== 'all' ) {
113 $this->output( "Now run the script using the other --mode options if you haven't already.\n" );
114 }
115
116 if ( in_array( 'pages', $modes ) ) {
117 if ( $this->hasOption( 'skip-cleanup' ) ) {
118 $this->output(
119 "Also run 'php cleanupEmptyCategories.php --mode remove' to remove empty,\n" .
120 "nonexistent categories from the category table.\n\n" );
121 } else {
122 $this->output( "Running cleanupEmptyCategories.php\n" );
123 $cleanup = $this->runChild( CleanupEmptyCategories::class );
124 '@phan-var CleanupEmptyCategories $cleanup';
125 // Pass no options into the child because of a parameter collision between "mode", which
126 // both scripts use but set to different values. We'll just use the defaults.
127 $cleanup->loadParamsAndArgs( $this->mSelf, [], [] );
128 // Force execution because we want to run it regardless of whether it's been run before.
129 $cleanup->setForce( true );
130 $cleanup->execute();
131 }
132 }
133 }
134
135 protected function doWork( string $mode ): int|false {
136 $this->output( "Finding up to {$this->getBatchSize()} drifted rows " .
137 "greater than cat_id {$this->minimumId}...\n" );
138
139 $dbr = $this->getDB( DB_REPLICA, 'vslow' );
140
141 $migrationStage = $this->getServiceContainer()->getMainConfig()->get(
142 MainConfigNames::CategoryLinksSchemaMigrationStage
143 );
144
145 if ( $migrationStage & SCHEMA_COMPAT_READ_OLD ) {
146 $queryBuilder = $dbr->newSelectQueryBuilder()
147 ->select( 'COUNT(*)' )
148 ->from( 'categorylinks' )
149 ->where( 'cl_to = cat_title' );
150 } else {
151 $queryBuilder = $dbr->newSelectQueryBuilder()
152 ->select( 'COUNT(*)' )
153 ->from( 'categorylinks' )
154 ->join( 'linktarget', null, 'cl_target_id = lt_id' )
155 ->where( [
156 'lt_title' => 'cat_title',
157 'lt_namespace' => NS_CATEGORY,
158 ] );
159 }
160
161 if ( $mode === 'subcats' ) {
162 $queryBuilder->andWhere( [ 'cl_type' => 'subcat' ] );
163 } elseif ( $mode === 'files' ) {
164 $queryBuilder->andWhere( [ 'cl_type' => 'file' ] );
165 }
166
167 $countingSubquery = $queryBuilder->caller( __METHOD__ )->getSQL();
168
169 // First, let's find out which categories have drifted and need to be updated.
170 // The query counts the categorylinks for each category on the replica DB,
171 // but this data can't be used for updating the master, so we don't include it
172 // in the results.
173 $idsToUpdate = $dbr->newSelectQueryBuilder()
174 ->select( 'cat_id' )
175 ->from( 'category' )
176 ->where( [ $dbr->expr( 'cat_id', '>', (int)$this->minimumId ), "cat_{$mode} != ($countingSubquery)" ] )
177 ->limit( $this->getBatchSize() )
178 ->caller( __METHOD__ )->fetchFieldValues();
179 if ( !$idsToUpdate ) {
180 return false;
181 }
182 $this->output( "Updating cat_{$mode} field on " .
183 count( $idsToUpdate ) . " rows...\n" );
184
185 // In the next batch, start where this query left off. The rows selected
186 // in this iteration shouldn't be selected again after being updated, but
187 // we still keep track of where we are up to, as extra protection against
188 // infinite loops.
189 $this->minimumId = end( $idsToUpdate );
190
191 // Now, on master, find the correct counts for these categories.
192 $dbw = $this->getPrimaryDB();
193 $res = $dbw->newSelectQueryBuilder()
194 ->select( [ 'cat_id', 'count' => "($countingSubquery)" ] )
195 ->from( 'category' )
196 ->where( [ 'cat_id' => $idsToUpdate ] )
197 ->caller( __METHOD__ )->fetchResultSet();
198
199 // Update the category counts on the rows we just identified.
200 // This logic is equivalent to Category::refreshCounts, except here, we
201 // don't remove rows when cat_pages is zero and the category description page
202 // doesn't exist - instead we print a suggestion to run
203 // cleanupEmptyCategories.php.
204 $affectedRows = 0;
205 foreach ( $res as $row ) {
206 $dbw->newUpdateQueryBuilder()
207 ->update( 'category' )
208 ->set( [ "cat_{$mode}" => $row->count ] )
209 ->where( [
210 'cat_id' => $row->cat_id,
211 $dbw->expr( "cat_{$mode}", '!=', (int)$row->count ),
212 ] )
213 ->caller( __METHOD__ )
214 ->execute();
215 $affectedRows += $dbw->affectedRows();
216 }
217
218 $this->waitForReplication();
219
220 return $affectedRows;
221 }
222}
223
224// @codeCoverageIgnoreStart
225$maintClass = RecountCategories::class;
226require_once RUN_MAINTENANCE_IF_MAIN;
227// @codeCoverageIgnoreEnd
const SCHEMA_COMPAT_READ_OLD
Definition Defines.php:304
const NS_CATEGORY
Definition Defines.php:79
A class containing constants representing the names of configuration variables.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
getBatchSize()
Returns batch size.
output( $out, $channel=null)
Throw some output to the user.
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
runChild( $maintClass, $classFile=null)
Returns an instance of the given maintenance script, with all of the current arguments passed to it.
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
waitForReplication()
Wait for replica DB servers to catch up.
hasOption( $name)
Checks to see if a particular option was set.
getOption( $name, $default=null)
Get an option, or return the default.
getServiceContainer()
Returns the main service container.
addDescription( $text)
Set the description text.
Maintenance script that refreshes category membership counts in the category table.
__construct()
Default constructor.
execute()
Do the actual work.
const DB_REPLICA
Definition defines.php:26