MediaWiki  master
cleanupInvalidDbKeys.php
Go to the documentation of this file.
1 <?php
24 require_once __DIR__ . '/Maintenance.php';
25 
29 
38  protected static $tables = [
39  // Data tables
40  [ 'page', 'page' ],
41  [ 'redirect', 'rd', 'idField' => 'rd_from' ],
42  [ 'archive', 'ar' ],
43  [ 'logging', 'log' ],
44  [ 'protected_titles', 'pt', 'idField' => 0 ],
45  [ 'category', 'cat', 'nsField' => 14 ],
46  [ 'recentchanges', 'rc' ],
47  [ 'watchlist', 'wl' ],
48  // The querycache tables' qc(c)_title and qcc_titletwo may contain titles,
49  // but also usernames or other things like that, so we leave them alone
50 
51  // Links tables
52  [ 'pagelinks', 'pl', 'idField' => 'pl_from' ],
53  [ 'templatelinks', 'tl', 'idField' => 'tl_from' ],
54  [ 'categorylinks', 'cl', 'idField' => 'cl_from', 'nsField' => 14, 'titleField' => 'cl_to' ],
55  ];
56 
57  public function __construct() {
58  parent::__construct();
59  $this->addDescription( <<<'TEXT'
60 This script cleans up the title fields in various tables to remove entries that
61 will be rejected by the constructor of TitleValue. This constructor throws an
62 exception when invalid data is encountered, which will not normally occur on
63 regular page views, but can happen on query special pages.
64 
65 The script targets titles matching the regular expression /^_|[ \r\n\t]|_$/.
66 Because any foreign key relationships involving these titles will already be
67 broken, the titles are corrected to a valid version or the rows are deleted
68 entirely, depending on the table.
69 
70 The script runs with the expectation that STDOUT is redirected to a file.
71 TEXT
72  );
73  $this->addOption( 'fix', 'Actually clean up invalid titles. If this parameter is ' .
74  'not specified, the script will report invalid titles but not clean them up.',
75  false, false );
76  $this->addOption( 'table', 'The table(s) to process. This option can be specified ' .
77  'more than once (e.g. -t category -t watchlist). If not specified, all available ' .
78  'tables will be processed. Available tables are: ' .
79  implode( ', ', array_column( static::$tables, 0 ) ), false, true, 't', true );
80 
81  $this->setBatchSize( 500 );
82  }
83 
84  public function execute() {
85  $tablesToProcess = $this->getOption( 'table' );
86  foreach ( static::$tables as $tableParams ) {
87  if ( !$tablesToProcess || in_array( $tableParams[0], $tablesToProcess ) ) {
88  $this->cleanupTable( $tableParams );
89  }
90  }
91 
92  $this->outputStatus( 'Done!' );
93  if ( $this->hasOption( 'fix' ) ) {
94  $dbDomain = WikiMap::getCurrentWikiDbDomain()->getId();
95  $this->outputStatus( " Cleaned up invalid DB keys on $dbDomain!\n" );
96  }
97  }
98 
106  protected function outputStatus( $str, $channel = null ) {
107  // Make it easier to find progress lines in the STDOUT log
108  if ( trim( $str ) ) {
109  fwrite( STDOUT, '*** ' . trim( $str ) . "\n" );
110  }
111  fwrite( STDERR, $str );
112  }
113 
119  protected function writeToReport( $str ) {
120  fwrite( STDOUT, $str );
121  }
122 
128  protected function cleanupTable( $tableParams ) {
129  [ $table, $prefix ] = $tableParams;
130  $idField = $tableParams['idField'] ?? "{$prefix}_id";
131  $nsField = $tableParams['nsField'] ?? "{$prefix}_namespace";
132  $titleField = $tableParams['titleField'] ?? "{$prefix}_title";
133 
134  $this->outputStatus( "Looking for invalid $titleField entries in $table...\n" );
135 
136  // Do all the select queries on the replicas, as they are slow (they use
137  // unanchored LIKEs). Naturally this could cause problems if rows are
138  // modified after selecting and before deleting/updating, but working on
139  // the hypothesis that invalid rows will be old and in all likelihood
140  // unreferenced, we should be fine to do it like this.
141  $dbr = $this->getDB( DB_REPLICA, 'vslow' );
142  $linksMigration = MediaWikiServices::getInstance()->getLinksMigration();
143  $joinConds = [];
144  $tables = [ $table ];
145  if ( isset( $linksMigration::$mapping[$table] ) ) {
146  [ $nsField,$titleField ] = $linksMigration->getTitleFields( $table );
147  $joinConds = $linksMigration->getQueryInfo( $table )['joins'];
148  $tables = $linksMigration->getQueryInfo( $table )['tables'];
149  }
150 
151  // Find all TitleValue-invalid titles.
152  $percent = $dbr->anyString();
153  $res = $dbr->newSelectQueryBuilder()
154  ->select( [
155  'id' => $idField,
156  'ns' => $nsField,
157  'title' => $titleField,
158  ] )
159  ->tables( $tables )
160  // The REGEXP operator is not cross-DBMS, so we have to use lots of LIKEs
161  ->where( $dbr->makeList( [
162  $titleField . $dbr->buildLike( $percent, ' ', $percent ),
163  $titleField . $dbr->buildLike( $percent, "\r", $percent ),
164  $titleField . $dbr->buildLike( $percent, "\n", $percent ),
165  $titleField . $dbr->buildLike( $percent, "\t", $percent ),
166  $titleField . $dbr->buildLike( '_', $percent ),
167  $titleField . $dbr->buildLike( $percent, '_' ),
168  ], LIST_OR ) )
169  ->joinConds( $joinConds )
170  ->limit( $this->getBatchSize() )
171  ->caller( __METHOD__ )
172  ->fetchResultSet();
173 
174  $this->outputStatus( "Number of invalid rows: " . $res->numRows() . "\n" );
175  if ( !$res->numRows() ) {
176  $this->outputStatus( "\n" );
177  return;
178  }
179 
180  // Write a table of titles to the report file. Also keep a list of the found
181  // IDs, as we might need it later for DB updates
182  $this->writeToReport( sprintf( "%10s | ns | dbkey\n", $idField ) );
183  $ids = [];
184  foreach ( $res as $row ) {
185  $this->writeToReport( sprintf( "%10d | %3d | %s\n", $row->id, $row->ns, $row->title ) );
186  $ids[] = $row->id;
187  }
188 
189  // If we're doing a dry run, output the new titles we would use for the UPDATE
190  // queries (if relevant), and finish
191  if ( !$this->hasOption( 'fix' ) ) {
192  if ( $table === 'logging' || $table === 'archive' ) {
193  $this->writeToReport( "The following updates would be run with the --fix flag:\n" );
194  foreach ( $res as $row ) {
195  $newTitle = self::makeValidTitle( $row->title );
196  $this->writeToReport(
197  "$idField={$row->id}: update '{$row->title}' to '$newTitle'\n" );
198  }
199  }
200 
201  if ( $table !== 'page' && $table !== 'redirect' ) {
202  $this->outputStatus( "Run with --fix to clean up these rows\n" );
203  }
204  $this->outputStatus( "\n" );
205  return;
206  }
207 
208  $services = MediaWikiServices::getInstance();
209 
210  // Fix the bad data, using different logic for the various tables
211  $dbw = $this->getDB( DB_PRIMARY );
212  switch ( $table ) {
213  case 'page':
214  case 'redirect':
215  // This shouldn't happen on production wikis, and we already have a script
216  // to handle 'page' rows anyway, so just notify the user and let them decide
217  // what to do next.
218  $this->outputStatus( <<<TEXT
219 IMPORTANT: This script does not fix invalid entries in the $table table.
220 Consider repairing these rows, and rows in related tables, by hand.
221 You may like to run, or borrow logic from, the cleanupTitles.php script.
222 
223 TEXT
224  );
225  break;
226 
227  case 'archive':
228  case 'logging':
229  // Rename the title to a corrected equivalent. Any foreign key relationships
230  // to the page_title field are already broken, so this will just make sure
231  // users can still access the log entries/deleted revisions from the interface
232  // using a valid page title.
233  $this->outputStatus(
234  "Updating these rows, setting $titleField to the closest valid DB key...\n" );
235  $affectedRowCount = 0;
236  foreach ( $res as $row ) {
237  $newTitle = self::makeValidTitle( $row->title );
238  $this->writeToReport(
239  "$idField={$row->id}: updating '{$row->title}' to '$newTitle'\n" );
240 
241  $dbw->update( $table,
242  [ $titleField => $newTitle ],
243  [ $idField => $row->id ],
244  __METHOD__ );
245  $affectedRowCount += $dbw->affectedRows();
246  }
247  $this->waitForReplication();
248  $this->outputStatus( "Updated $affectedRowCount rows on $table.\n" );
249 
250  break;
251 
252  case 'recentchanges':
253  case 'watchlist':
254  case 'category':
255  // Since these broken titles can't exist, there's really nothing to watch,
256  // nothing can be categorised in them, and they can't have been changed
257  // recently, so we can just remove these rows.
258  $this->outputStatus( "Deleting invalid $table rows...\n" );
259  $dbw->delete( $table, [ $idField => $ids ], __METHOD__ );
260  $this->waitForReplication();
261  $this->outputStatus( 'Deleted ' . $dbw->affectedRows() . " rows from $table.\n" );
262  break;
263 
264  case 'protected_titles':
265  // Since these broken titles can't exist, there's really nothing to protect,
266  // so we can just remove these rows. Made more complicated by this table
267  // not having an ID field
268  $this->outputStatus( "Deleting invalid $table rows...\n" );
269  $affectedRowCount = 0;
270  foreach ( $res as $row ) {
271  $dbw->delete( $table,
272  [ $nsField => $row->ns, $titleField => $row->title ],
273  __METHOD__ );
274  $affectedRowCount += $dbw->affectedRows();
275  }
276  $this->waitForReplication();
277  $this->outputStatus( "Deleted $affectedRowCount rows from $table.\n" );
278  break;
279 
280  case 'pagelinks':
281  case 'templatelinks':
282  case 'categorylinks':
283  // Update links tables for each page where these bogus links are supposedly
284  // located. If the invalid rows don't go away after these jobs go through,
285  // they're probably being added by a buggy hook.
286  $this->outputStatus( "Queueing link update jobs for the pages in $idField...\n" );
287  $linksMigration = MediaWikiServices::getInstance()->getLinksMigration();
288  $wikiPageFactory = $services->getWikiPageFactory();
289  foreach ( $res as $row ) {
290  $wp = $wikiPageFactory->newFromID( $row->id );
291  if ( $wp ) {
293  } else {
294  if ( isset( $linksMigration::$mapping[$table] ) ) {
295  $conds = $linksMigration->getLinksConditions(
296  $table,
297  Title::makeTitle( $row->ns, $row->title )
298  );
299  } else {
300  $conds = [ $nsField => $row->ns, $titleField => $row->title ];
301  }
302  // This link entry points to a nonexistent page, so just get rid of it
303  $dbw->delete( $table,
304  array_merge( [ $idField => $row->id ], $conds ),
305  __METHOD__ );
306  }
307  }
308  $this->waitForReplication();
309  $this->outputStatus( "Link update jobs have been added to the job queue.\n" );
310  break;
311  }
312 
313  $this->outputStatus( "\n" );
314  }
315 
322  protected static function makeValidTitle( $invalidTitle ) {
323  return strtr( trim( $invalidTitle, '_' ),
324  [ ' ' => '_', "\r" => '', "\n" => '', "\t" => '_' ] );
325  }
326 }
327 
328 $maintClass = CleanupInvalidDbKeys::class;
329 require_once RUN_MAINTENANCE_IF_MAIN;
const LIST_OR
Definition: Defines.php:46
Maintenance script that cleans up invalid titles in various tables.
static makeValidTitle( $invalidTitle)
Fix possible validation issues in the given title (DB key).
static array[] $tables
List of tables to clean up, and the field prefix for that table.
cleanupTable( $tableParams)
Identifies, and optionally cleans up, invalid titles.
outputStatus( $str, $channel=null)
Prints text to STDOUT, and STDERR if STDOUT was redirected to a file.
writeToReport( $str)
Prints text to STDOUT.
__construct()
Default constructor.
execute()
Do the actual work.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:66
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
waitForReplication()
Wait for replica DBs to catch up.
hasOption( $name)
Checks to see if a particular option was set.
getBatchSize()
Returns batch size.
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
setBatchSize( $s=0)
Service locator for MediaWiki core services.
Represents a title within MediaWiki.
Definition: Title.php:82
Helper tools for dealing with other locally-hosted wikis.
Definition: WikiMap.php:33
Represents a page (or page fragment) title within MediaWiki.
Definition: TitleValue.php:40
const DB_REPLICA
Definition: defines.php:26
const DB_PRIMARY
Definition: defines.php:28