MediaWiki master
cleanupInvalidDbKeys.php
Go to the documentation of this file.
1<?php
24// @codeCoverageIgnoreStart
25require_once __DIR__ . '/Maintenance.php';
26// @codeCoverageIgnoreEnd
27
33
42 protected static $tables = [
43 // Data tables
44 [ 'page', 'page' ],
45 [ 'redirect', 'rd', 'idField' => 'rd_from' ],
46 [ 'archive', 'ar' ],
47 [ 'logging', 'log' ],
48 [ 'protected_titles', 'pt', 'idField' => 0 ],
49 [ 'category', 'cat', 'nsField' => 14 ],
50 [ 'recentchanges', 'rc' ],
51 [ 'watchlist', 'wl' ],
52 // The querycache tables' qc(c)_title and qcc_titletwo may contain titles,
53 // but also usernames or other things like that, so we leave them alone
54
55 // Links tables
56 [ 'pagelinks', 'pl', 'idField' => 'pl_from' ],
57 [ 'templatelinks', 'tl', 'idField' => 'tl_from' ],
58 [ 'categorylinks', 'cl', 'idField' => 'cl_from', 'nsField' => 14, 'titleField' => 'cl_to' ],
59 [ 'imagelinks', 'il', 'idField' => 'il_from', 'nsField' => 6, 'titleField' => 'il_to' ],
60 ];
61
62 public function __construct() {
63 parent::__construct();
64 $this->addDescription( <<<'TEXT'
65This script cleans up the title fields in various tables to remove entries that
66will be rejected by the constructor of TitleValue. This constructor throws an
67exception when invalid data is encountered, which will not normally occur on
68regular page views, but can happen on query special pages.
69
70The script targets titles matching the regular expression /^_|[ \r\n\t]|_$/.
71Because any foreign key relationships involving these titles will already be
72broken, the titles are corrected to a valid version or the rows are deleted
73entirely, depending on the table.
74
75The script runs with the expectation that STDOUT is redirected to a file.
76TEXT
77 );
78 $this->addOption( 'fix', 'Actually clean up invalid titles. If this parameter is ' .
79 'not specified, the script will report invalid titles but not clean them up.',
80 false, false );
81 $this->addOption( 'table', 'The table(s) to process. This option can be specified ' .
82 'more than once (e.g. -t category -t watchlist). If not specified, all available ' .
83 'tables will be processed. Available tables are: ' .
84 implode( ', ', array_column( static::$tables, 0 ) ), false, true, 't', true );
85
86 $this->setBatchSize( 500 );
87 }
88
89 public function execute() {
90 $tablesToProcess = $this->getOption( 'table' );
91 foreach ( static::$tables as $tableParams ) {
92 if ( !$tablesToProcess || in_array( $tableParams[0], $tablesToProcess ) ) {
93 $this->cleanupTable( $tableParams );
94 }
95 }
96
97 $this->outputStatus( 'Done!' );
98 if ( $this->hasOption( 'fix' ) ) {
99 $dbDomain = WikiMap::getCurrentWikiDbDomain()->getId();
100 $this->outputStatus( " Cleaned up invalid DB keys on $dbDomain!\n" );
101 }
102 }
103
111 protected function outputStatus( $str, $channel = null ) {
112 // Make it easier to find progress lines in the STDOUT log
113 if ( trim( $str ) ) {
114 fwrite( STDOUT, '*** ' . trim( $str ) . "\n" );
115 }
116 fwrite( STDERR, $str );
117 }
118
124 protected function writeToReport( $str ) {
125 fwrite( STDOUT, $str );
126 }
127
133 protected function cleanupTable( $tableParams ) {
134 [ $table, $prefix ] = $tableParams;
135 $idField = $tableParams['idField'] ?? "{$prefix}_id";
136 $nsField = $tableParams['nsField'] ?? "{$prefix}_namespace";
137 $titleField = $tableParams['titleField'] ?? "{$prefix}_title";
138
139 $this->outputStatus( "Looking for invalid $titleField entries in $table...\n" );
140
141 // Do all the select queries on the replicas, as they are slow (they use
142 // unanchored LIKEs). Naturally this could cause problems if rows are
143 // modified after selecting and before deleting/updating, but working on
144 // the hypothesis that invalid rows will be old and in all likelihood
145 // unreferenced, we should be fine to do it like this.
146 $dbr = $this->getDB( DB_REPLICA, 'vslow' );
147 $linksMigration = $this->getServiceContainer()->getLinksMigration();
148 $joinConds = [];
149 $tables = [ $table ];
150 if ( isset( $linksMigration::$mapping[$table] ) ) {
151 [ $nsField, $titleField ] = $linksMigration->getTitleFields( $table );
152 $joinConds = $linksMigration->getQueryInfo( $table )['joins'];
153 $tables = $linksMigration->getQueryInfo( $table )['tables'];
154 }
155
156 // Find all TitleValue-invalid titles.
157 $percent = $dbr->anyString();
158 // The REGEXP operator is not cross-DBMS, so we have to use lots of LIKEs
159 $likeExpr = $dbr
160 ->expr( $titleField, IExpression::LIKE, new LikeValue( $percent, ' ', $percent ) )
161 ->or( $titleField, IExpression::LIKE, new LikeValue( $percent, "\r", $percent ) )
162 ->or( $titleField, IExpression::LIKE, new LikeValue( $percent, "\n", $percent ) )
163 ->or( $titleField, IExpression::LIKE, new LikeValue( $percent, "\t", $percent ) )
164 ->or( $titleField, IExpression::LIKE, new LikeValue( '_', $percent ) )
165 ->or( $titleField, IExpression::LIKE, new LikeValue( $percent, '_' ) );
166 $res = $dbr->newSelectQueryBuilder()
167 ->select( [
168 'id' => $idField,
169 'ns' => $nsField,
170 'title' => $titleField,
171 ] )
172 ->tables( $tables )
173 ->where( $likeExpr )
174 ->joinConds( $joinConds )
175 ->limit( $this->getBatchSize() )
176 ->caller( __METHOD__ )
177 ->fetchResultSet();
178
179 $this->outputStatus( "Number of invalid rows: " . $res->numRows() . "\n" );
180 if ( !$res->numRows() ) {
181 $this->outputStatus( "\n" );
182 return;
183 }
184
185 // Write a table of titles to the report file. Also keep a list of the found
186 // IDs, as we might need it later for DB updates
187 $this->writeToReport( sprintf( "%10s | ns | dbkey\n", $idField ) );
188 $ids = [];
189 foreach ( $res as $row ) {
190 $this->writeToReport( sprintf( "%10d | %3d | %s\n", $row->id, $row->ns, $row->title ) );
191 $ids[] = $row->id;
192 }
193
194 // If we're doing a dry run, output the new titles we would use for the UPDATE
195 // queries (if relevant), and finish
196 if ( !$this->hasOption( 'fix' ) ) {
197 if ( $table === 'logging' || $table === 'archive' ) {
198 $this->writeToReport( "The following updates would be run with the --fix flag:\n" );
199 foreach ( $res as $row ) {
200 $newTitle = self::makeValidTitle( $row->title );
201 $this->writeToReport(
202 "$idField={$row->id}: update '{$row->title}' to '$newTitle'\n" );
203 }
204 }
205
206 if ( $table !== 'page' && $table !== 'redirect' ) {
207 $this->outputStatus( "Run with --fix to clean up these rows\n" );
208 }
209 $this->outputStatus( "\n" );
210 return;
211 }
212
213 $services = $this->getServiceContainer();
214
215 // Fix the bad data, using different logic for the various tables
216 $dbw = $this->getPrimaryDB();
217 switch ( $table ) {
218 case 'page':
219 case 'redirect':
220 // This shouldn't happen on production wikis, and we already have a script
221 // to handle 'page' rows anyway, so just notify the user and let them decide
222 // what to do next.
223 $this->outputStatus( <<<TEXT
224IMPORTANT: This script does not fix invalid entries in the $table table.
225Consider repairing these rows, and rows in related tables, by hand.
226You may like to run, or borrow logic from, the cleanupTitles.php script.
227
228TEXT
229 );
230 break;
231
232 case 'archive':
233 case 'logging':
234 // Rename the title to a corrected equivalent. Any foreign key relationships
235 // to the page_title field are already broken, so this will just make sure
236 // users can still access the log entries/deleted revisions from the interface
237 // using a valid page title.
238 $this->outputStatus(
239 "Updating these rows, setting $titleField to the closest valid DB key...\n" );
240 $affectedRowCount = 0;
241 foreach ( $res as $row ) {
242 $newTitle = self::makeValidTitle( $row->title );
243 $this->writeToReport(
244 "$idField={$row->id}: updating '{$row->title}' to '$newTitle'\n" );
245
246 $dbw->newUpdateQueryBuilder()
247 ->update( $table )
248 ->set( [ $titleField => $newTitle ] )
249 ->where( [ $idField => $row->id ] )
250 ->caller( __METHOD__ )
251 ->execute();
252 $affectedRowCount += $dbw->affectedRows();
253 }
254 $this->waitForReplication();
255 $this->outputStatus( "Updated $affectedRowCount rows on $table.\n" );
256
257 break;
258
259 case 'recentchanges':
260 case 'watchlist':
261 case 'category':
262 // Since these broken titles can't exist, there's really nothing to watch,
263 // nothing can be categorised in them, and they can't have been changed
264 // recently, so we can just remove these rows.
265 $this->outputStatus( "Deleting invalid $table rows...\n" );
266 $dbw->newDeleteQueryBuilder()
267 ->deleteFrom( $table )
268 ->where( [ $idField => $ids ] )
269 ->caller( __METHOD__ )->execute();
270 $this->waitForReplication();
271 $this->outputStatus( 'Deleted ' . $dbw->affectedRows() . " rows from $table.\n" );
272 break;
273
274 case 'protected_titles':
275 // Since these broken titles can't exist, there's really nothing to protect,
276 // so we can just remove these rows. Made more complicated by this table
277 // not having an ID field
278 $this->outputStatus( "Deleting invalid $table rows...\n" );
279 $affectedRowCount = 0;
280 foreach ( $res as $row ) {
281 $dbw->newDeleteQueryBuilder()
282 ->deleteFrom( $table )
283 ->where( [ $nsField => $row->ns, $titleField => $row->title ] )
284 ->caller( __METHOD__ )->execute();
285 $affectedRowCount += $dbw->affectedRows();
286 }
287 $this->waitForReplication();
288 $this->outputStatus( "Deleted $affectedRowCount rows from $table.\n" );
289 break;
290
291 case 'pagelinks':
292 case 'templatelinks':
293 case 'categorylinks':
294 case 'imagelinks':
295 // Update links tables for each page where these bogus links are supposedly
296 // located. If the invalid rows don't go away after these jobs go through,
297 // they're probably being added by a buggy hook.
298 $this->outputStatus( "Queueing link update jobs for the pages in $idField...\n" );
299 $linksMigration = $this->getServiceContainer()->getLinksMigration();
300 $wikiPageFactory = $services->getWikiPageFactory();
301 foreach ( $res as $row ) {
302 $wp = $wikiPageFactory->newFromID( $row->id );
303 if ( $wp ) {
304 RefreshLinks::fixLinksFromArticle( $row->id );
305 } else {
306 if ( isset( $linksMigration::$mapping[$table] ) ) {
307 $conds = $linksMigration->getLinksConditions(
308 $table,
309 Title::makeTitle( $row->ns, $row->title )
310 );
311 } else {
312 $conds = [ $nsField => $row->ns, $titleField => $row->title ];
313 }
314 // This link entry points to a nonexistent page, so just get rid of it
315 $dbw->newDeleteQueryBuilder()
316 ->deleteFrom( $table )
317 ->where( array_merge( [ $idField => $row->id ], $conds ) )
318 ->caller( __METHOD__ )->execute();
319 }
320 }
321 $this->waitForReplication();
322 $this->outputStatus( "Link update jobs have been added to the job queue.\n" );
323 break;
324 }
325
326 $this->outputStatus( "\n" );
327 }
328
335 protected static function makeValidTitle( $invalidTitle ) {
336 return strtr( trim( $invalidTitle, '_' ),
337 [ ' ' => '_', "\r" => '', "\n" => '', "\t" => '_' ] );
338 }
339}
340
341// @codeCoverageIgnoreStart
342$maintClass = CleanupInvalidDbKeys::class;
343require_once RUN_MAINTENANCE_IF_MAIN;
344// @codeCoverageIgnoreEnd
run()
Run the job.
Maintenance script that cleans up invalid titles in various tables.
static makeValidTitle( $invalidTitle)
Fix possible validation issues in the given title (DB key).
static array[] $tables
List of tables to clean up, and the field prefix for that table.
cleanupTable( $tableParams)
Identifies, and optionally cleans up, invalid titles.
outputStatus( $str, $channel=null)
Prints text to STDOUT, and STDERR if STDOUT was redirected to a file.
writeToReport( $str)
Prints text to STDOUT.
__construct()
Default constructor.
execute()
Do the actual work.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
getBatchSize()
Returns batch size.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
waitForReplication()
Wait for replica DB servers to catch up.
hasOption( $name)
Checks to see if a particular option was set.
getOption( $name, $default=null)
Get an option, or return the default.
getServiceContainer()
Returns the main service container.
addDescription( $text)
Set the description text.
Represents a title within MediaWiki.
Definition Title.php:78
Tools for dealing with other locally-hosted wikis.
Definition WikiMap.php:31
Content of like value.
Definition LikeValue.php:14
const DB_REPLICA
Definition defines.php:26