MediaWiki  master
cleanupInvalidDbKeys.php
Go to the documentation of this file.
1 <?php
24 require_once __DIR__ . '/Maintenance.php';
25 
34  protected static $tables = [
35  // Data tables
36  [ 'page', 'page' ],
37  [ 'redirect', 'rd', 'idField' => 'rd_from' ],
38  [ 'archive', 'ar' ],
39  [ 'logging', 'log' ],
40  [ 'protected_titles', 'pt', 'idField' => 0 ],
41  [ 'category', 'cat', 'nsField' => 14 ],
42  [ 'recentchanges', 'rc' ],
43  [ 'watchlist', 'wl' ],
44  // The querycache tables' qc(c)_title and qcc_titletwo may contain titles,
45  // but also usernames or other things like that, so we leave them alone
46 
47  // Links tables
48  [ 'pagelinks', 'pl', 'idField' => 'pl_from' ],
49  [ 'templatelinks', 'tl', 'idField' => 'tl_from' ],
50  [ 'categorylinks', 'cl', 'idField' => 'cl_from', 'nsField' => 14, 'titleField' => 'cl_to' ],
51  ];
52 
53  public function __construct() {
54  parent::__construct();
55  $this->addDescription( <<<'TEXT'
56 This script cleans up the title fields in various tables to remove entries that
57 will be rejected by the constructor of TitleValue. This constructor throws an
58 exception when invalid data is encountered, which will not normally occur on
59 regular page views, but can happen on query special pages.
60 
61 The script targets titles matching the regular expression /^_|[ \r\n\t]|_$/.
62 Because any foreign key relationships involving these titles will already be
63 broken, the titles are corrected to a valid version or the rows are deleted
64 entirely, depending on the table.
65 
66 The script runs with the expectation that STDOUT is redirected to a file.
67 TEXT
68  );
69  $this->addOption( 'fix', 'Actually clean up invalid titles. If this parameter is ' .
70  'not specified, the script will report invalid titles but not clean them up.',
71  false, false );
72  $this->addOption( 'table', 'The table(s) to process. This option can be specified ' .
73  'more than once (e.g. -t category -t watchlist). If not specified, all available ' .
74  'tables will be processed. Available tables are: ' .
75  implode( ', ', array_column( static::$tables, 0 ) ), false, true, 't', true );
76 
77  $this->setBatchSize( 500 );
78  }
79 
80  public function execute() {
81  $tablesToProcess = $this->getOption( 'table' );
82  foreach ( static::$tables as $tableParams ) {
83  if ( !$tablesToProcess || in_array( $tableParams[0], $tablesToProcess ) ) {
84  $this->cleanupTable( $tableParams );
85  }
86  }
87 
88  $this->outputStatus( 'Done!' );
89  if ( $this->hasOption( 'fix' ) ) {
90  $dbDomain = WikiMap::getCurrentWikiDbDomain()->getId();
91  $this->outputStatus( " Cleaned up invalid DB keys on $dbDomain!\n" );
92  }
93  }
94 
102  protected function outputStatus( $str, $channel = null ) {
103  // Make it easier to find progress lines in the STDOUT log
104  if ( trim( $str ) ) {
105  fwrite( STDOUT, '*** ' . trim( $str ) . "\n" );
106  }
107  fwrite( STDERR, $str );
108  }
109 
115  protected function writeToReport( $str ) {
116  fwrite( STDOUT, $str );
117  }
118 
124  protected function cleanupTable( $tableParams ) {
125  list( $table, $prefix ) = $tableParams;
126  $idField = $tableParams['idField'] ?? "{$prefix}_id";
127  $nsField = $tableParams['nsField'] ?? "{$prefix}_namespace";
128  $titleField = $tableParams['titleField'] ?? "{$prefix}_title";
129 
130  $this->outputStatus( "Looking for invalid $titleField entries in $table...\n" );
131 
132  // Do all the select queries on the replicas, as they are slow (they use
133  // unanchored LIKEs). Naturally this could cause problems if rows are
134  // modified after selecting and before deleting/updating, but working on
135  // the hypothesis that invalid rows will be old and in all likelihood
136  // unreferenced, we should be fine to do it like this.
137  $dbr = $this->getDB( DB_REPLICA, 'vslow' );
138 
139  // Find all TitleValue-invalid titles.
140  $percent = $dbr->anyString(); // DBMS-agnostic equivalent of '%' LIKE wildcard
141  $res = $dbr->select(
142  $table,
143  [
144  'id' => $idField,
145  'ns' => $nsField,
146  'title' => $titleField,
147  ],
148  // The REGEXP operator is not cross-DBMS, so we have to use lots of LIKEs
149  [ $dbr->makeList( [
150  $titleField . $dbr->buildLike( $percent, ' ', $percent ),
151  $titleField . $dbr->buildLike( $percent, "\r", $percent ),
152  $titleField . $dbr->buildLike( $percent, "\n", $percent ),
153  $titleField . $dbr->buildLike( $percent, "\t", $percent ),
154  $titleField . $dbr->buildLike( '_', $percent ),
155  $titleField . $dbr->buildLike( $percent, '_' ),
156  ], LIST_OR ) ],
157  __METHOD__,
158  [ 'LIMIT' => $this->getBatchSize() ]
159  );
160 
161  $this->outputStatus( "Number of invalid rows: " . $res->numRows() . "\n" );
162  if ( !$res->numRows() ) {
163  $this->outputStatus( "\n" );
164  return;
165  }
166 
167  // Write a table of titles to the report file. Also keep a list of the found
168  // IDs, as we might need it later for DB updates
169  $this->writeToReport( sprintf( "%10s | ns | dbkey\n", $idField ) );
170  $ids = [];
171  foreach ( $res as $row ) {
172  $this->writeToReport( sprintf( "%10d | %3d | %s\n", $row->id, $row->ns, $row->title ) );
173  $ids[] = $row->id;
174  }
175 
176  // If we're doing a dry run, output the new titles we would use for the UPDATE
177  // queries (if relevant), and finish
178  if ( !$this->hasOption( 'fix' ) ) {
179  if ( $table === 'logging' || $table === 'archive' ) {
180  $this->writeToReport( "The following updates would be run with the --fix flag:\n" );
181  foreach ( $res as $row ) {
182  $newTitle = self::makeValidTitle( $row->title );
183  $this->writeToReport(
184  "$idField={$row->id}: update '{$row->title}' to '$newTitle'\n" );
185  }
186  }
187 
188  if ( $table !== 'page' && $table !== 'redirect' ) {
189  $this->outputStatus( "Run with --fix to clean up these rows\n" );
190  }
191  $this->outputStatus( "\n" );
192  return;
193  }
194 
195  // Fix the bad data, using different logic for the various tables
196  $dbw = $this->getDB( DB_MASTER );
197  switch ( $table ) {
198  case 'page':
199  case 'redirect':
200  // This shouldn't happen on production wikis, and we already have a script
201  // to handle 'page' rows anyway, so just notify the user and let them decide
202  // what to do next.
203  $this->outputStatus( <<<TEXT
204 IMPORTANT: This script does not fix invalid entries in the $table table.
205 Consider repairing these rows, and rows in related tables, by hand.
206 You may like to run, or borrow logic from, the cleanupTitles.php script.
207 
208 TEXT
209  );
210  break;
211 
212  case 'archive':
213  case 'logging':
214  // Rename the title to a corrected equivalent. Any foreign key relationships
215  // to the page_title field are already broken, so this will just make sure
216  // users can still access the log entries/deleted revisions from the interface
217  // using a valid page title.
218  $this->outputStatus(
219  "Updating these rows, setting $titleField to the closest valid DB key...\n" );
220  $affectedRowCount = 0;
221  foreach ( $res as $row ) {
222  $newTitle = self::makeValidTitle( $row->title );
223  $this->writeToReport(
224  "$idField={$row->id}: updating '{$row->title}' to '$newTitle'\n" );
225 
226  $dbw->update( $table,
227  [ $titleField => $newTitle ],
228  [ $idField => $row->id ],
229  __METHOD__ );
230  $affectedRowCount += $dbw->affectedRows();
231  }
232  wfWaitForSlaves();
233  $this->outputStatus( "Updated $affectedRowCount rows on $table.\n" );
234 
235  break;
236 
237  case 'recentchanges':
238  case 'watchlist':
239  case 'category':
240  // Since these broken titles can't exist, there's really nothing to watch,
241  // nothing can be categorised in them, and they can't have been changed
242  // recently, so we can just remove these rows.
243  $this->outputStatus( "Deleting invalid $table rows...\n" );
244  $dbw->delete( $table, [ $idField => $ids ], __METHOD__ );
245  wfWaitForSlaves();
246  $this->outputStatus( 'Deleted ' . $dbw->affectedRows() . " rows from $table.\n" );
247  break;
248 
249  case 'protected_titles':
250  // Since these broken titles can't exist, there's really nothing to protect,
251  // so we can just remove these rows. Made more complicated by this table
252  // not having an ID field
253  $this->outputStatus( "Deleting invalid $table rows...\n" );
254  $affectedRowCount = 0;
255  foreach ( $res as $row ) {
256  $dbw->delete( $table,
257  [ $nsField => $row->ns, $titleField => $row->title ],
258  __METHOD__ );
259  $affectedRowCount += $dbw->affectedRows();
260  }
261  wfWaitForSlaves();
262  $this->outputStatus( "Deleted $affectedRowCount rows from $table.\n" );
263  break;
264 
265  case 'pagelinks':
266  case 'templatelinks':
267  case 'categorylinks':
268  // Update links tables for each page where these bogus links are supposedly
269  // located. If the invalid rows don't go away after these jobs go through,
270  // they're probably being added by a buggy hook.
271  $this->outputStatus( "Queueing link update jobs for the pages in $idField...\n" );
272  foreach ( $res as $row ) {
273  $wp = WikiPage::newFromID( $row->id );
274  if ( $wp ) {
276  } else {
277  // This link entry points to a nonexistent page, so just get rid of it
278  $dbw->delete( $table,
279  [ $idField => $row->id, $nsField => $row->ns, $titleField => $row->title ],
280  __METHOD__ );
281  }
282  }
283  wfWaitForSlaves();
284  $this->outputStatus( "Link update jobs have been added to the job queue.\n" );
285  break;
286  }
287 
288  $this->outputStatus( "\n" );
289  }
290 
297  protected static function makeValidTitle( $invalidTitle ) {
298  return strtr( trim( $invalidTitle, '_' ),
299  [ ' ' => '_', "\r" => '', "\n" => '', "\t" => '_' ] );
300  }
301 }
302 
303 $maintClass = CleanupInvalidDbKeys::class;
304 require_once RUN_MAINTENANCE_IF_MAIN;
const RUN_MAINTENANCE_IF_MAIN
Definition: Maintenance.php:39
getOption( $name, $default=null)
Get an option, or return the default.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:86
Represents a page (or page fragment) title within MediaWiki.
Definition: TitleValue.php:37
setBatchSize( $s=0)
Set the batch size.
hasOption( $name)
Checks to see if a particular option exists.
cleanupTable( $tableParams)
Identifies, and optionally cleans up, invalid titles.
const DB_MASTER
Definition: defines.php:26
static array [] $tables
List of tables to clean up, and the field prefix for that table.
writeToReport( $str)
Prints text to STDOUT.
wfWaitForSlaves( $ifWritesSince=null, $wiki=false, $cluster=false, $timeout=null)
Waits for the replica DBs to catch up to the master position.
addDescription( $text)
Set the description text.
static array [] __construct()
static makeValidTitle( $invalidTitle)
Fix possible validation issues in the given title (DB key).
const LIST_OR
Definition: Defines.php:42
outputStatus( $str, $channel=null)
Prints text to STDOUT, and STDERR if STDOUT was redirected to a file.
static getCurrentWikiDbDomain()
Definition: WikiMap.php:293
static newFromID( $id, $from='fromdb')
Constructor from a page id.
Definition: WikiPage.php:180
getBatchSize()
Returns batch size.
Maintenance script that cleans up invalid titles in various tables.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
const DB_REPLICA
Definition: defines.php:25
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
while(( $__line=Maintenance::readconsole()) !==false) print n
Definition: eval.php:64