MediaWiki  master
cleanupInvalidDbKeys.php
Go to the documentation of this file.
1 <?php
24 require_once __DIR__ . '/Maintenance.php';
25 
27 
36  protected static $tables = [
37  // Data tables
38  [ 'page', 'page' ],
39  [ 'redirect', 'rd', 'idField' => 'rd_from' ],
40  [ 'archive', 'ar' ],
41  [ 'logging', 'log' ],
42  [ 'protected_titles', 'pt', 'idField' => 0 ],
43  [ 'category', 'cat', 'nsField' => 14 ],
44  [ 'recentchanges', 'rc' ],
45  [ 'watchlist', 'wl' ],
46  // The querycache tables' qc(c)_title and qcc_titletwo may contain titles,
47  // but also usernames or other things like that, so we leave them alone
48 
49  // Links tables
50  [ 'pagelinks', 'pl', 'idField' => 'pl_from' ],
51  [ 'templatelinks', 'tl', 'idField' => 'tl_from' ],
52  [ 'categorylinks', 'cl', 'idField' => 'cl_from', 'nsField' => 14, 'titleField' => 'cl_to' ],
53  ];
54 
55  public function __construct() {
56  parent::__construct();
57  $this->addDescription( <<<'TEXT'
58 This script cleans up the title fields in various tables to remove entries that
59 will be rejected by the constructor of TitleValue. This constructor throws an
60 exception when invalid data is encountered, which will not normally occur on
61 regular page views, but can happen on query special pages.
62 
63 The script targets titles matching the regular expression /^_|[ \r\n\t]|_$/.
64 Because any foreign key relationships involving these titles will already be
65 broken, the titles are corrected to a valid version or the rows are deleted
66 entirely, depending on the table.
67 
68 The script runs with the expectation that STDOUT is redirected to a file.
69 TEXT
70  );
71  $this->addOption( 'fix', 'Actually clean up invalid titles. If this parameter is ' .
72  'not specified, the script will report invalid titles but not clean them up.',
73  false, false );
74  $this->addOption( 'table', 'The table(s) to process. This option can be specified ' .
75  'more than once (e.g. -t category -t watchlist). If not specified, all available ' .
76  'tables will be processed. Available tables are: ' .
77  implode( ', ', array_column( static::$tables, 0 ) ), false, true, 't', true );
78 
79  $this->setBatchSize( 500 );
80  }
81 
82  public function execute() {
83  $tablesToProcess = $this->getOption( 'table' );
84  foreach ( static::$tables as $tableParams ) {
85  if ( !$tablesToProcess || in_array( $tableParams[0], $tablesToProcess ) ) {
86  $this->cleanupTable( $tableParams );
87  }
88  }
89 
90  $this->outputStatus( 'Done!' );
91  if ( $this->hasOption( 'fix' ) ) {
92  $dbDomain = WikiMap::getCurrentWikiDbDomain()->getId();
93  $this->outputStatus( " Cleaned up invalid DB keys on $dbDomain!\n" );
94  }
95  }
96 
104  protected function outputStatus( $str, $channel = null ) {
105  // Make it easier to find progress lines in the STDOUT log
106  if ( trim( $str ) ) {
107  fwrite( STDOUT, '*** ' . trim( $str ) . "\n" );
108  }
109  fwrite( STDERR, $str );
110  }
111 
117  protected function writeToReport( $str ) {
118  fwrite( STDOUT, $str );
119  }
120 
126  protected function cleanupTable( $tableParams ) {
127  list( $table, $prefix ) = $tableParams;
128  $idField = $tableParams['idField'] ?? "{$prefix}_id";
129  $nsField = $tableParams['nsField'] ?? "{$prefix}_namespace";
130  $titleField = $tableParams['titleField'] ?? "{$prefix}_title";
131 
132  $this->outputStatus( "Looking for invalid $titleField entries in $table...\n" );
133 
134  // Do all the select queries on the replicas, as they are slow (they use
135  // unanchored LIKEs). Naturally this could cause problems if rows are
136  // modified after selecting and before deleting/updating, but working on
137  // the hypothesis that invalid rows will be old and in all likelihood
138  // unreferenced, we should be fine to do it like this.
139  $dbr = $this->getDB( DB_REPLICA, 'vslow' );
140 
141  // Find all TitleValue-invalid titles.
142  $percent = $dbr->anyString();
143  $res = $dbr->select(
144  $table,
145  [
146  'id' => $idField,
147  'ns' => $nsField,
148  'title' => $titleField,
149  ],
150  // The REGEXP operator is not cross-DBMS, so we have to use lots of LIKEs
151  [ $dbr->makeList( [
152  $titleField . $dbr->buildLike( $percent, ' ', $percent ),
153  $titleField . $dbr->buildLike( $percent, "\r", $percent ),
154  $titleField . $dbr->buildLike( $percent, "\n", $percent ),
155  $titleField . $dbr->buildLike( $percent, "\t", $percent ),
156  $titleField . $dbr->buildLike( '_', $percent ),
157  $titleField . $dbr->buildLike( $percent, '_' ),
158  ], LIST_OR ) ],
159  __METHOD__,
160  [ 'LIMIT' => $this->getBatchSize() ]
161  );
162 
163  $this->outputStatus( "Number of invalid rows: " . $res->numRows() . "\n" );
164  if ( !$res->numRows() ) {
165  $this->outputStatus( "\n" );
166  return;
167  }
168 
169  // Write a table of titles to the report file. Also keep a list of the found
170  // IDs, as we might need it later for DB updates
171  $this->writeToReport( sprintf( "%10s | ns | dbkey\n", $idField ) );
172  $ids = [];
173  foreach ( $res as $row ) {
174  $this->writeToReport( sprintf( "%10d | %3d | %s\n", $row->id, $row->ns, $row->title ) );
175  $ids[] = $row->id;
176  }
177 
178  // If we're doing a dry run, output the new titles we would use for the UPDATE
179  // queries (if relevant), and finish
180  if ( !$this->hasOption( 'fix' ) ) {
181  if ( $table === 'logging' || $table === 'archive' ) {
182  $this->writeToReport( "The following updates would be run with the --fix flag:\n" );
183  foreach ( $res as $row ) {
184  $newTitle = self::makeValidTitle( $row->title );
185  $this->writeToReport(
186  "$idField={$row->id}: update '{$row->title}' to '$newTitle'\n" );
187  }
188  }
189 
190  if ( $table !== 'page' && $table !== 'redirect' ) {
191  $this->outputStatus( "Run with --fix to clean up these rows\n" );
192  }
193  $this->outputStatus( "\n" );
194  return;
195  }
196 
197  $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
198 
199  // Fix the bad data, using different logic for the various tables
200  $dbw = $this->getDB( DB_MASTER );
201  switch ( $table ) {
202  case 'page':
203  case 'redirect':
204  // This shouldn't happen on production wikis, and we already have a script
205  // to handle 'page' rows anyway, so just notify the user and let them decide
206  // what to do next.
207  $this->outputStatus( <<<TEXT
208 IMPORTANT: This script does not fix invalid entries in the $table table.
209 Consider repairing these rows, and rows in related tables, by hand.
210 You may like to run, or borrow logic from, the cleanupTitles.php script.
211 
212 TEXT
213  );
214  break;
215 
216  case 'archive':
217  case 'logging':
218  // Rename the title to a corrected equivalent. Any foreign key relationships
219  // to the page_title field are already broken, so this will just make sure
220  // users can still access the log entries/deleted revisions from the interface
221  // using a valid page title.
222  $this->outputStatus(
223  "Updating these rows, setting $titleField to the closest valid DB key...\n" );
224  $affectedRowCount = 0;
225  foreach ( $res as $row ) {
226  $newTitle = self::makeValidTitle( $row->title );
227  $this->writeToReport(
228  "$idField={$row->id}: updating '{$row->title}' to '$newTitle'\n" );
229 
230  $dbw->update( $table,
231  [ $titleField => $newTitle ],
232  [ $idField => $row->id ],
233  __METHOD__ );
234  $affectedRowCount += $dbw->affectedRows();
235  }
236  $lbFactory->waitForReplication();
237  $this->outputStatus( "Updated $affectedRowCount rows on $table.\n" );
238 
239  break;
240 
241  case 'recentchanges':
242  case 'watchlist':
243  case 'category':
244  // Since these broken titles can't exist, there's really nothing to watch,
245  // nothing can be categorised in them, and they can't have been changed
246  // recently, so we can just remove these rows.
247  $this->outputStatus( "Deleting invalid $table rows...\n" );
248  $dbw->delete( $table, [ $idField => $ids ], __METHOD__ );
249  $lbFactory->waitForReplication();
250  $this->outputStatus( 'Deleted ' . $dbw->affectedRows() . " rows from $table.\n" );
251  break;
252 
253  case 'protected_titles':
254  // Since these broken titles can't exist, there's really nothing to protect,
255  // so we can just remove these rows. Made more complicated by this table
256  // not having an ID field
257  $this->outputStatus( "Deleting invalid $table rows...\n" );
258  $affectedRowCount = 0;
259  foreach ( $res as $row ) {
260  $dbw->delete( $table,
261  [ $nsField => $row->ns, $titleField => $row->title ],
262  __METHOD__ );
263  $affectedRowCount += $dbw->affectedRows();
264  }
265  $lbFactory->waitForReplication();
266  $this->outputStatus( "Deleted $affectedRowCount rows from $table.\n" );
267  break;
268 
269  case 'pagelinks':
270  case 'templatelinks':
271  case 'categorylinks':
272  // Update links tables for each page where these bogus links are supposedly
273  // located. If the invalid rows don't go away after these jobs go through,
274  // they're probably being added by a buggy hook.
275  $this->outputStatus( "Queueing link update jobs for the pages in $idField...\n" );
276  foreach ( $res as $row ) {
277  $wp = WikiPage::newFromID( $row->id );
278  if ( $wp ) {
280  } else {
281  // This link entry points to a nonexistent page, so just get rid of it
282  $dbw->delete( $table,
283  [ $idField => $row->id, $nsField => $row->ns, $titleField => $row->title ],
284  __METHOD__ );
285  }
286  }
287  $lbFactory->waitForReplication();
288  $this->outputStatus( "Link update jobs have been added to the job queue.\n" );
289  break;
290  }
291 
292  $this->outputStatus( "\n" );
293  }
294 
301  protected static function makeValidTitle( $invalidTitle ) {
302  return strtr( trim( $invalidTitle, '_' ),
303  [ ' ' => '_', "\r" => '', "\n" => '', "\t" => '_' ] );
304  }
305 }
306 
307 $maintClass = CleanupInvalidDbKeys::class;
308 require_once RUN_MAINTENANCE_IF_MAIN;
RUN_MAINTENANCE_IF_MAIN
const RUN_MAINTENANCE_IF_MAIN
Definition: Maintenance.php:38
WikiMap\getCurrentWikiDbDomain
static getCurrentWikiDbDomain()
Definition: WikiMap.php:293
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:154
n
while(( $__line=Maintenance::readconsole()) !==false) print n
Definition: eval.php:64
Maintenance\addDescription
addDescription( $text)
Set the description text.
Definition: Maintenance.php:327
CleanupInvalidDbKeys\outputStatus
outputStatus( $str, $channel=null)
Prints text to STDOUT, and STDERR if STDOUT was redirected to a file.
Definition: cleanupInvalidDbKeys.php:104
Maintenance
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:55
CleanupInvalidDbKeys\writeToReport
writeToReport( $str)
Prints text to STDOUT.
Definition: cleanupInvalidDbKeys.php:117
$res
$res
Definition: testCompression.php:57
$dbr
$dbr
Definition: testCompression.php:54
CleanupInvalidDbKeys\makeValidTitle
static makeValidTitle( $invalidTitle)
Fix possible validation issues in the given title (DB key).
Definition: cleanupInvalidDbKeys.php:301
LIST_OR
const LIST_OR
Definition: Defines.php:51
CleanupInvalidDbKeys\execute
execute()
Do the actual work.
Definition: cleanupInvalidDbKeys.php:82
Maintenance\addOption
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
Definition: Maintenance.php:245
CleanupInvalidDbKeys\__construct
__construct()
Default constructor.
Definition: cleanupInvalidDbKeys.php:55
DB_REPLICA
const DB_REPLICA
Definition: defines.php:25
DB_MASTER
const DB_MASTER
Definition: defines.php:26
WikiPage\newFromID
static newFromID( $id, $from='fromdb')
Constructor from a page id.
Definition: WikiPage.php:171
Maintenance\getDB
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
Definition: Maintenance.php:1366
CleanupInvalidDbKeys\cleanupTable
cleanupTable( $tableParams)
Identifies, and optionally cleans up, invalid titles.
Definition: cleanupInvalidDbKeys.php:126
CleanupInvalidDbKeys\$tables
static array[] $tables
List of tables to clean up, and the field prefix for that table.
Definition: cleanupInvalidDbKeys.php:36
Maintenance\getOption
getOption( $name, $default=null)
Get an option, or return the default.
Definition: Maintenance.php:281
Maintenance\hasOption
hasOption( $name)
Checks to see if a particular option exists.
Definition: Maintenance.php:266
$maintClass
$maintClass
Definition: cleanupInvalidDbKeys.php:307
Maintenance\setBatchSize
setBatchSize( $s=0)
Set the batch size.
Definition: Maintenance.php:374
CleanupInvalidDbKeys
Maintenance script that cleans up invalid titles in various tables.
Definition: cleanupInvalidDbKeys.php:34
TitleValue
Represents a page (or page fragment) title within MediaWiki.
Definition: TitleValue.php:39