MediaWiki  master
cleanupInvalidDbKeys.php
Go to the documentation of this file.
1 <?php
24 require_once __DIR__ . '/Maintenance.php';
25 
28 
37  protected static $tables = [
38  // Data tables
39  [ 'page', 'page' ],
40  [ 'redirect', 'rd', 'idField' => 'rd_from' ],
41  [ 'archive', 'ar' ],
42  [ 'logging', 'log' ],
43  [ 'protected_titles', 'pt', 'idField' => 0 ],
44  [ 'category', 'cat', 'nsField' => 14 ],
45  [ 'recentchanges', 'rc' ],
46  [ 'watchlist', 'wl' ],
47  // The querycache tables' qc(c)_title and qcc_titletwo may contain titles,
48  // but also usernames or other things like that, so we leave them alone
49 
50  // Links tables
51  [ 'pagelinks', 'pl', 'idField' => 'pl_from' ],
52  [ 'templatelinks', 'tl', 'idField' => 'tl_from' ],
53  [ 'categorylinks', 'cl', 'idField' => 'cl_from', 'nsField' => 14, 'titleField' => 'cl_to' ],
54  ];
55 
56  public function __construct() {
57  parent::__construct();
58  $this->addDescription( <<<'TEXT'
59 This script cleans up the title fields in various tables to remove entries that
60 will be rejected by the constructor of TitleValue. This constructor throws an
61 exception when invalid data is encountered, which will not normally occur on
62 regular page views, but can happen on query special pages.
63 
64 The script targets titles matching the regular expression /^_|[ \r\n\t]|_$/.
65 Because any foreign key relationships involving these titles will already be
66 broken, the titles are corrected to a valid version or the rows are deleted
67 entirely, depending on the table.
68 
69 The script runs with the expectation that STDOUT is redirected to a file.
70 TEXT
71  );
72  $this->addOption( 'fix', 'Actually clean up invalid titles. If this parameter is ' .
73  'not specified, the script will report invalid titles but not clean them up.',
74  false, false );
75  $this->addOption( 'table', 'The table(s) to process. This option can be specified ' .
76  'more than once (e.g. -t category -t watchlist). If not specified, all available ' .
77  'tables will be processed. Available tables are: ' .
78  implode( ', ', array_column( static::$tables, 0 ) ), false, true, 't', true );
79 
80  $this->setBatchSize( 500 );
81  }
82 
83  public function execute() {
84  $tablesToProcess = $this->getOption( 'table' );
85  foreach ( static::$tables as $tableParams ) {
86  if ( !$tablesToProcess || in_array( $tableParams[0], $tablesToProcess ) ) {
87  $this->cleanupTable( $tableParams );
88  }
89  }
90 
91  $this->outputStatus( 'Done!' );
92  if ( $this->hasOption( 'fix' ) ) {
93  $dbDomain = WikiMap::getCurrentWikiDbDomain()->getId();
94  $this->outputStatus( " Cleaned up invalid DB keys on $dbDomain!\n" );
95  }
96  }
97 
105  protected function outputStatus( $str, $channel = null ) {
106  // Make it easier to find progress lines in the STDOUT log
107  if ( trim( $str ) ) {
108  fwrite( STDOUT, '*** ' . trim( $str ) . "\n" );
109  }
110  fwrite( STDERR, $str );
111  }
112 
118  protected function writeToReport( $str ) {
119  fwrite( STDOUT, $str );
120  }
121 
127  protected function cleanupTable( $tableParams ) {
128  [ $table, $prefix ] = $tableParams;
129  $idField = $tableParams['idField'] ?? "{$prefix}_id";
130  $nsField = $tableParams['nsField'] ?? "{$prefix}_namespace";
131  $titleField = $tableParams['titleField'] ?? "{$prefix}_title";
132 
133  $this->outputStatus( "Looking for invalid $titleField entries in $table...\n" );
134 
135  // Do all the select queries on the replicas, as they are slow (they use
136  // unanchored LIKEs). Naturally this could cause problems if rows are
137  // modified after selecting and before deleting/updating, but working on
138  // the hypothesis that invalid rows will be old and in all likelihood
139  // unreferenced, we should be fine to do it like this.
140  $dbr = $this->getDB( DB_REPLICA, 'vslow' );
141  $linksMigration = $this->getServiceContainer()->getLinksMigration();
142  $joinConds = [];
143  $tables = [ $table ];
144  if ( isset( $linksMigration::$mapping[$table] ) ) {
145  [ $nsField, $titleField ] = $linksMigration->getTitleFields( $table );
146  $joinConds = $linksMigration->getQueryInfo( $table )['joins'];
147  $tables = $linksMigration->getQueryInfo( $table )['tables'];
148  }
149 
150  // Find all TitleValue-invalid titles.
151  $percent = $dbr->anyString();
152  $res = $dbr->newSelectQueryBuilder()
153  ->select( [
154  'id' => $idField,
155  'ns' => $nsField,
156  'title' => $titleField,
157  ] )
158  ->tables( $tables )
159  // The REGEXP operator is not cross-DBMS, so we have to use lots of LIKEs
160  ->where( $dbr->makeList( [
161  $titleField . $dbr->buildLike( $percent, ' ', $percent ),
162  $titleField . $dbr->buildLike( $percent, "\r", $percent ),
163  $titleField . $dbr->buildLike( $percent, "\n", $percent ),
164  $titleField . $dbr->buildLike( $percent, "\t", $percent ),
165  $titleField . $dbr->buildLike( '_', $percent ),
166  $titleField . $dbr->buildLike( $percent, '_' ),
167  ], LIST_OR ) )
168  ->joinConds( $joinConds )
169  ->limit( $this->getBatchSize() )
170  ->caller( __METHOD__ )
171  ->fetchResultSet();
172 
173  $this->outputStatus( "Number of invalid rows: " . $res->numRows() . "\n" );
174  if ( !$res->numRows() ) {
175  $this->outputStatus( "\n" );
176  return;
177  }
178 
179  // Write a table of titles to the report file. Also keep a list of the found
180  // IDs, as we might need it later for DB updates
181  $this->writeToReport( sprintf( "%10s | ns | dbkey\n", $idField ) );
182  $ids = [];
183  foreach ( $res as $row ) {
184  $this->writeToReport( sprintf( "%10d | %3d | %s\n", $row->id, $row->ns, $row->title ) );
185  $ids[] = $row->id;
186  }
187 
188  // If we're doing a dry run, output the new titles we would use for the UPDATE
189  // queries (if relevant), and finish
190  if ( !$this->hasOption( 'fix' ) ) {
191  if ( $table === 'logging' || $table === 'archive' ) {
192  $this->writeToReport( "The following updates would be run with the --fix flag:\n" );
193  foreach ( $res as $row ) {
194  $newTitle = self::makeValidTitle( $row->title );
195  $this->writeToReport(
196  "$idField={$row->id}: update '{$row->title}' to '$newTitle'\n" );
197  }
198  }
199 
200  if ( $table !== 'page' && $table !== 'redirect' ) {
201  $this->outputStatus( "Run with --fix to clean up these rows\n" );
202  }
203  $this->outputStatus( "\n" );
204  return;
205  }
206 
207  $services = $this->getServiceContainer();
208 
209  // Fix the bad data, using different logic for the various tables
210  $dbw = $this->getDB( DB_PRIMARY );
211  switch ( $table ) {
212  case 'page':
213  case 'redirect':
214  // This shouldn't happen on production wikis, and we already have a script
215  // to handle 'page' rows anyway, so just notify the user and let them decide
216  // what to do next.
217  $this->outputStatus( <<<TEXT
218 IMPORTANT: This script does not fix invalid entries in the $table table.
219 Consider repairing these rows, and rows in related tables, by hand.
220 You may like to run, or borrow logic from, the cleanupTitles.php script.
221 
222 TEXT
223  );
224  break;
225 
226  case 'archive':
227  case 'logging':
228  // Rename the title to a corrected equivalent. Any foreign key relationships
229  // to the page_title field are already broken, so this will just make sure
230  // users can still access the log entries/deleted revisions from the interface
231  // using a valid page title.
232  $this->outputStatus(
233  "Updating these rows, setting $titleField to the closest valid DB key...\n" );
234  $affectedRowCount = 0;
235  foreach ( $res as $row ) {
236  $newTitle = self::makeValidTitle( $row->title );
237  $this->writeToReport(
238  "$idField={$row->id}: updating '{$row->title}' to '$newTitle'\n" );
239 
240  $dbw->update( $table,
241  [ $titleField => $newTitle ],
242  [ $idField => $row->id ],
243  __METHOD__ );
244  $affectedRowCount += $dbw->affectedRows();
245  }
246  $this->waitForReplication();
247  $this->outputStatus( "Updated $affectedRowCount rows on $table.\n" );
248 
249  break;
250 
251  case 'recentchanges':
252  case 'watchlist':
253  case 'category':
254  // Since these broken titles can't exist, there's really nothing to watch,
255  // nothing can be categorised in them, and they can't have been changed
256  // recently, so we can just remove these rows.
257  $this->outputStatus( "Deleting invalid $table rows...\n" );
258  $dbw->delete( $table, [ $idField => $ids ], __METHOD__ );
259  $this->waitForReplication();
260  $this->outputStatus( 'Deleted ' . $dbw->affectedRows() . " rows from $table.\n" );
261  break;
262 
263  case 'protected_titles':
264  // Since these broken titles can't exist, there's really nothing to protect,
265  // so we can just remove these rows. Made more complicated by this table
266  // not having an ID field
267  $this->outputStatus( "Deleting invalid $table rows...\n" );
268  $affectedRowCount = 0;
269  foreach ( $res as $row ) {
270  $dbw->delete( $table,
271  [ $nsField => $row->ns, $titleField => $row->title ],
272  __METHOD__ );
273  $affectedRowCount += $dbw->affectedRows();
274  }
275  $this->waitForReplication();
276  $this->outputStatus( "Deleted $affectedRowCount rows from $table.\n" );
277  break;
278 
279  case 'pagelinks':
280  case 'templatelinks':
281  case 'categorylinks':
282  // Update links tables for each page where these bogus links are supposedly
283  // located. If the invalid rows don't go away after these jobs go through,
284  // they're probably being added by a buggy hook.
285  $this->outputStatus( "Queueing link update jobs for the pages in $idField...\n" );
286  $linksMigration = $this->getServiceContainer()->getLinksMigration();
287  $wikiPageFactory = $services->getWikiPageFactory();
288  foreach ( $res as $row ) {
289  $wp = $wikiPageFactory->newFromID( $row->id );
290  if ( $wp ) {
292  } else {
293  if ( isset( $linksMigration::$mapping[$table] ) ) {
294  $conds = $linksMigration->getLinksConditions(
295  $table,
296  Title::makeTitle( $row->ns, $row->title )
297  );
298  } else {
299  $conds = [ $nsField => $row->ns, $titleField => $row->title ];
300  }
301  // This link entry points to a nonexistent page, so just get rid of it
302  $dbw->delete( $table,
303  array_merge( [ $idField => $row->id ], $conds ),
304  __METHOD__ );
305  }
306  }
307  $this->waitForReplication();
308  $this->outputStatus( "Link update jobs have been added to the job queue.\n" );
309  break;
310  }
311 
312  $this->outputStatus( "\n" );
313  }
314 
321  protected static function makeValidTitle( $invalidTitle ) {
322  return strtr( trim( $invalidTitle, '_' ),
323  [ ' ' => '_', "\r" => '', "\n" => '', "\t" => '_' ] );
324  }
325 }
326 
327 $maintClass = CleanupInvalidDbKeys::class;
328 require_once RUN_MAINTENANCE_IF_MAIN;
const LIST_OR
Definition: Defines.php:46
Maintenance script that cleans up invalid titles in various tables.
static makeValidTitle( $invalidTitle)
Fix possible validation issues in the given title (DB key).
static array[] $tables
List of tables to clean up, and the field prefix for that table.
cleanupTable( $tableParams)
Identifies, and optionally cleans up, invalid titles.
outputStatus( $str, $channel=null)
Prints text to STDOUT, and STDERR if STDOUT was redirected to a file.
writeToReport( $str)
Prints text to STDOUT.
__construct()
Default constructor.
execute()
Do the actual work.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:66
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
waitForReplication()
Wait for replica DBs to catch up.
hasOption( $name)
Checks to see if a particular option was set.
getServiceContainer()
Returns the main service container.
getBatchSize()
Returns batch size.
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
setBatchSize( $s=0)
Represents a title within MediaWiki.
Definition: Title.php:76
Tools for dealing with other locally-hosted wikis.
Definition: WikiMap.php:31
const DB_REPLICA
Definition: defines.php:26
const DB_PRIMARY
Definition: defines.php:28