MediaWiki  master
cleanupInvalidDbKeys.php
Go to the documentation of this file.
1 <?php
24 require_once __DIR__ . '/Maintenance.php';
25 
34  protected static $tables = [
35  // Data tables
36  [ 'page', 'page' ],
37  [ 'redirect', 'rd', 'idField' => 'rd_from' ],
38  [ 'archive', 'ar' ],
39  [ 'logging', 'log' ],
40  [ 'protected_titles', 'pt', 'idField' => 0 ],
41  [ 'category', 'cat', 'nsField' => 14 ],
42  [ 'recentchanges', 'rc' ],
43  [ 'watchlist', 'wl' ],
44  // The querycache tables' qc(c)_title and qcc_titletwo may contain titles,
45  // but also usernames or other things like that, so we leave them alone
46 
47  // Links tables
48  [ 'pagelinks', 'pl', 'idField' => 'pl_from' ],
49  [ 'templatelinks', 'tl', 'idField' => 'tl_from' ],
50  [ 'categorylinks', 'cl', 'idField' => 'cl_from', 'nsField' => 14, 'titleField' => 'cl_to' ],
51  ];
52 
53  public function __construct() {
54  parent::__construct();
55  $this->addDescription( <<<'TEXT'
56 This script cleans up the title fields in various tables to remove entries that
57 will be rejected by the constructor of TitleValue. This constructor throws an
58 exception when invalid data is encountered, which will not normally occur on
59 regular page views, but can happen on query special pages.
60 
61 The script targets titles matching the regular expression /^_|[ \r\n\t]|_$/.
62 Because any foreign key relationships involving these titles will already be
63 broken, the titles are corrected to a valid version or the rows are deleted
64 entirely, depending on the table.
65 
66 The script runs with the expectation that STDOUT is redirected to a file.
67 TEXT
68  );
69  $this->addOption( 'fix', 'Actually clean up invalid titles. If this parameter is ' .
70  'not specified, the script will report invalid titles but not clean them up.',
71  false, false );
72  $this->addOption( 'table', 'The table(s) to process. This option can be specified ' .
73  'more than once (e.g. -t category -t watchlist). If not specified, all available ' .
74  'tables will be processed. Available tables are: ' .
75  implode( ', ', array_column( static::$tables, 0 ) ), false, true, 't', true );
76 
77  $this->setBatchSize( 500 );
78  }
79 
80  public function execute() {
81  $tablesToProcess = $this->getOption( 'table' );
82  foreach ( static::$tables as $tableParams ) {
83  if ( !$tablesToProcess || in_array( $tableParams[0], $tablesToProcess ) ) {
84  $this->cleanupTable( $tableParams );
85  }
86  }
87 
88  $this->outputStatus( 'Done!' );
89  if ( $this->hasOption( 'fix' ) ) {
90  $this->outputStatus( ' Cleaned up invalid DB keys on ' . wfWikiID() . "!\n" );
91  }
92  }
93 
101  protected function outputStatus( $str, $channel = null ) {
102  // Make it easier to find progress lines in the STDOUT log
103  if ( trim( $str ) ) {
104  fwrite( STDOUT, '*** ' . trim( $str ) . "\n" );
105  }
106  fwrite( STDERR, $str );
107  }
108 
114  protected function writeToReport( $str ) {
115  fwrite( STDOUT, $str );
116  }
117 
123  protected function cleanupTable( $tableParams ) {
124  list( $table, $prefix ) = $tableParams;
125  $idField = $tableParams['idField'] ?? "{$prefix}_id";
126  $nsField = $tableParams['nsField'] ?? "{$prefix}_namespace";
127  $titleField = $tableParams['titleField'] ?? "{$prefix}_title";
128 
129  $this->outputStatus( "Looking for invalid $titleField entries in $table...\n" );
130 
131  // Do all the select queries on the replicas, as they are slow (they use
132  // unanchored LIKEs). Naturally this could cause problems if rows are
133  // modified after selecting and before deleting/updating, but working on
134  // the hypothesis that invalid rows will be old and in all likelihood
135  // unreferenced, we should be fine to do it like this.
136  $dbr = $this->getDB( DB_REPLICA, 'vslow' );
137 
138  // Find all TitleValue-invalid titles.
139  $percent = $dbr->anyString(); // DBMS-agnostic equivalent of '%' LIKE wildcard
140  $res = $dbr->select(
141  $table,
142  [
143  'id' => $idField,
144  'ns' => $nsField,
145  'title' => $titleField,
146  ],
147  // The REGEXP operator is not cross-DBMS, so we have to use lots of LIKEs
148  [ $dbr->makeList( [
149  $titleField . $dbr->buildLike( $percent, ' ', $percent ),
150  $titleField . $dbr->buildLike( $percent, "\r", $percent ),
151  $titleField . $dbr->buildLike( $percent, "\n", $percent ),
152  $titleField . $dbr->buildLike( $percent, "\t", $percent ),
153  $titleField . $dbr->buildLike( '_', $percent ),
154  $titleField . $dbr->buildLike( $percent, '_' ),
155  ], LIST_OR ) ],
156  __METHOD__,
157  [ 'LIMIT' => $this->getBatchSize() ]
158  );
159 
160  $this->outputStatus( "Number of invalid rows: " . $res->numRows() . "\n" );
161  if ( !$res->numRows() ) {
162  $this->outputStatus( "\n" );
163  return;
164  }
165 
166  // Write a table of titles to the report file. Also keep a list of the found
167  // IDs, as we might need it later for DB updates
168  $this->writeToReport( sprintf( "%10s | ns | dbkey\n", $idField ) );
169  $ids = [];
170  foreach ( $res as $row ) {
171  $this->writeToReport( sprintf( "%10d | %3d | %s\n", $row->id, $row->ns, $row->title ) );
172  $ids[] = $row->id;
173  }
174 
175  // If we're doing a dry run, output the new titles we would use for the UPDATE
176  // queries (if relevant), and finish
177  if ( !$this->hasOption( 'fix' ) ) {
178  if ( $table === 'logging' || $table === 'archive' ) {
179  $this->writeToReport( "The following updates would be run with the --fix flag:\n" );
180  foreach ( $res as $row ) {
181  $newTitle = self::makeValidTitle( $row->title );
182  $this->writeToReport(
183  "$idField={$row->id}: update '{$row->title}' to '$newTitle'\n" );
184  }
185  }
186 
187  if ( $table !== 'page' && $table !== 'redirect' ) {
188  $this->outputStatus( "Run with --fix to clean up these rows\n" );
189  }
190  $this->outputStatus( "\n" );
191  return;
192  }
193 
194  // Fix the bad data, using different logic for the various tables
195  $dbw = $this->getDB( DB_MASTER );
196  switch ( $table ) {
197  case 'page':
198  case 'redirect':
199  // This shouldn't happen on production wikis, and we already have a script
200  // to handle 'page' rows anyway, so just notify the user and let them decide
201  // what to do next.
202  $this->outputStatus( <<<TEXT
203 IMPORTANT: This script does not fix invalid entries in the $table table.
204 Consider repairing these rows, and rows in related tables, by hand.
205 You may like to run, or borrow logic from, the cleanupTitles.php script.
206 
207 TEXT
208  );
209  break;
210 
211  case 'archive':
212  case 'logging':
213  // Rename the title to a corrected equivalent. Any foreign key relationships
214  // to the page_title field are already broken, so this will just make sure
215  // users can still access the log entries/deleted revisions from the interface
216  // using a valid page title.
217  $this->outputStatus(
218  "Updating these rows, setting $titleField to the closest valid DB key...\n" );
219  $affectedRowCount = 0;
220  foreach ( $res as $row ) {
221  $newTitle = self::makeValidTitle( $row->title );
222  $this->writeToReport(
223  "$idField={$row->id}: updating '{$row->title}' to '$newTitle'\n" );
224 
225  $dbw->update( $table,
226  [ $titleField => $newTitle ],
227  [ $idField => $row->id ],
228  __METHOD__ );
229  $affectedRowCount += $dbw->affectedRows();
230  }
231  wfWaitForSlaves();
232  $this->outputStatus( "Updated $affectedRowCount rows on $table.\n" );
233 
234  break;
235 
236  case 'recentchanges':
237  case 'watchlist':
238  case 'category':
239  // Since these broken titles can't exist, there's really nothing to watch,
240  // nothing can be categorised in them, and they can't have been changed
241  // recently, so we can just remove these rows.
242  $this->outputStatus( "Deleting invalid $table rows...\n" );
243  $dbw->delete( $table, [ $idField => $ids ], __METHOD__ );
244  wfWaitForSlaves();
245  $this->outputStatus( 'Deleted ' . $dbw->affectedRows() . " rows from $table.\n" );
246  break;
247 
248  case 'protected_titles':
249  // Since these broken titles can't exist, there's really nothing to protect,
250  // so we can just remove these rows. Made more complicated by this table
251  // not having an ID field
252  $this->outputStatus( "Deleting invalid $table rows...\n" );
253  $affectedRowCount = 0;
254  foreach ( $res as $row ) {
255  $dbw->delete( $table,
256  [ $nsField => $row->ns, $titleField => $row->title ],
257  __METHOD__ );
258  $affectedRowCount += $dbw->affectedRows();
259  }
260  wfWaitForSlaves();
261  $this->outputStatus( "Deleted $affectedRowCount rows from $table.\n" );
262  break;
263 
264  case 'pagelinks':
265  case 'templatelinks':
266  case 'categorylinks':
267  // Update links tables for each page where these bogus links are supposedly
268  // located. If the invalid rows don't go away after these jobs go through,
269  // they're probably being added by a buggy hook.
270  $this->outputStatus( "Queueing link update jobs for the pages in $idField...\n" );
271  foreach ( $res as $row ) {
272  $wp = WikiPage::newFromID( $row->id );
273  if ( $wp ) {
275  } else {
276  // This link entry points to a nonexistent page, so just get rid of it
277  $dbw->delete( $table,
278  [ $idField => $row->id, $nsField => $row->ns, $titleField => $row->title ],
279  __METHOD__ );
280  }
281  }
282  wfWaitForSlaves();
283  $this->outputStatus( "Link update jobs have been added to the job queue.\n" );
284  break;
285  }
286 
287  $this->outputStatus( "\n" );
288  }
289 
296  protected static function makeValidTitle( $invalidTitle ) {
297  return strtr( trim( $invalidTitle, '_' ),
298  [ ' ' => '_', "\r" => '', "\n" => '', "\t" => '_' ] );
299  }
300 }
301 
303 require_once RUN_MAINTENANCE_IF_MAIN;
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global then executing the whole list after the page is displayed We don t do anything smart like collating updates to the same table or such because the list is almost always going to have just one item on if that
Definition: deferred.txt:11
and how to run hooks for an and one after Each event has a preferably in CamelCase For ArticleDelete hook A clump of code and data that should be run when an event happens This can be either a function and a chunk of data
Definition: hooks.txt:6
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition: deferred.txt:11
static configuration should be added through ResourceLoaderGetConfigVars instead can be used to get the real title e g db for database replication lag or jobqueue for job queue size converted to pseudo seconds It is possible to add more fields and they will be returned to the user in the API response after the basic globals have been set up
Definition: hooks.txt:2217
getOption( $name, $default=null)
Get an option, or return the default.
In both all secondary updates will be triggered handle like object that caches derived data representing a and can trigger updates of cached copies of that e g in the links tables
Definition: pageupdater.txt:78
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for and distribution as defined by Sections through of this document Licensor shall mean the copyright owner or entity authorized by the copyright owner that is granting the License Legal Entity shall mean the union of the acting entity and all other entities that control are controlled by or are under common control with that entity For the purposes of this definition control direct or to cause the direction or management of such whether by contract or including but not limited to software source documentation and configuration files Object form shall mean any form resulting from mechanical transformation or translation of a Source including but not limited to compiled object generated and conversions to other media types Work shall mean the work of whether in Source or Object made available under the as indicated by a copyright notice that is included in or attached to the whether in Source or Object that is based or other modifications as a an original work of authorship For the purposes of this Derivative Works shall not include works that remain separable from
globals txt Globals are evil The original MediaWiki code relied on globals for processing context far too often MediaWiki development since then has been a story of slowly moving context out of global variables and into objects Storing processing context in object member variables allows those objects to be reused in a much more flexible way Consider the elegance of
database rows
Definition: globals.txt:10
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: maintenance.txt:39
null for the local wiki Added in
Definition: hooks.txt:1585
either a unescaped string or a HtmlArmor object after in associative array form externallinks including delete and has completed for all link tables whether this was an auto creation use $formDescriptor instead default is conds Array Extra conditions for the No matching items in log is displayed if loglist is empty msgKey Array If you want a nice box with a set this to the key of the message First element is the message key
Definition: hooks.txt:2151
setBatchSize( $s=0)
Set the batch size.
hasOption( $name)
Checks to see if a particular option exists.
require_once RUN_MAINTENANCE_IF_MAIN
Definition: maintenance.txt:50
target page
cleanupTable( $tableParams)
Identifies, and optionally cleans up, invalid titles.
title
const DB_MASTER
Definition: defines.php:26
this hook is for auditing only RecentChangesLinked and Watchlist Do not use this to implement individual filters if they are compatible with the ChangesListFilter and ChangesListFilterGroup structure use sub classes of those in conjunction with the ChangesListSpecialPageStructuredFilters hook This hook can be used to implement filters that do not implement that or custom behavior that is not an individual filter e g Watchlist & $tables
Definition: hooks.txt:979
globals will be eliminated from MediaWiki entirely
Definition: globals.txt:25
This document provides an overview of the usage of PageUpdater and that is
Definition: pageupdater.txt:3
The ContentHandler facility adds support for arbitrary content types on wiki instead of relying on wikitext for everything It was introduced in MediaWiki Each kind of and so on Built in content types are
Prior to version
Definition: maintenance.txt:1
static array [] $tables
List of tables to clean up, and the field prefix for that table.
Using a hook running we can avoid having all this option specific stuff in our mainline code Using the function We ve cleaned up the code here by removing clumps of infrequently used code and moving them off somewhere else It s much easier for someone working with this code to see what s _really_ going and make changes or fix bugs In we can take all the code that deals with the little used title reversing we can concentrate it all in an extension file
Definition: hooks.txt:91
writeToReport( $str)
Prints text to STDOUT.
return true to allow those checks to occur
Definition: hooks.txt:1473
$res
Definition: database.txt:21
wfWaitForSlaves( $ifWritesSince=null, $wiki=false, $cluster=false, $timeout=null)
Waits for the replica DBs to catch up to the master position.
addDescription( $text)
Set the description text.
and(b) You must cause any modified files to carry prominent notices stating that You changed the files
this hook is for auditing only or null if authentication failed before getting that far or null if we can t even determine that When $user is not null
Definition: hooks.txt:780
getDB( $db, $groups=[], $wiki=false)
Returns a database to be used by current maintenance script.
static array [] __construct()
wfWikiID()
Get an ASCII string identifying this wiki This is used as a prefix in memcached keys.
Apache License January AND DISTRIBUTION Definitions License shall mean the terms and conditions for and distribution as defined by Sections through of this document Licensor shall mean the copyright owner or entity authorized by the copyright owner that is granting the License Legal Entity shall mean the union of the acting entity and all other entities that control are controlled by or are under common control with that entity For the purposes of this definition control direct or to cause the direction or management of such whether by contract or including but not limited to software source documentation and configuration files Object form shall mean any form resulting from mechanical transformation or translation of a Source including but not limited to compiled object generated and conversions to other media types Work shall mean the work of whether in Source or Object made available under the as indicated by a copyright notice that is included in or attached to the whether in Source or Object that is based or other modifications as a an original work of authorship For the purposes of this Derivative Works shall not include works that remain separable or merely the Work and Derivative Works thereof Contribution shall mean any work of including the original version of the Work and any modifications or additions to that Work or Derivative Works that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner For the purposes of this submitted means any form of or written communication sent to the Licensor or its including but not limited to communication on electronic mailing source code control and issue tracking systems that are managed by
null means default in associative array with keys and values unescaped Should be merged with default with a value of false meaning to suppress the attribute in associative array with keys and values unescaped broken
Definition: hooks.txt:1982
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
static makeValidTitle( $invalidTitle)
Fix possible validation issues in the given title (DB key).
const LIST_OR
Definition: Defines.php:46
outputStatus( $str, $channel=null)
Prints text to STDOUT, and STDERR if STDOUT was redirected to a file.
static newFromID( $id, $from='fromdb')
Constructor from a page id.
Definition: WikiPage.php:176
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
The ContentHandler facility adds support for arbitrary content types on wiki pages
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global then executing the whole list after the page is displayed We don t do anything smart like collating updates to the same table or such because the list is almost always going to have just one item on if so it s not worth the trouble Since there is a job queue in the jobs table
Definition: deferred.txt:11
you have access to all of the normal MediaWiki so you can get a DB use the etc For full docs on the Maintenance class
Definition: maintenance.txt:52
getBatchSize()
Returns batch size.
Using a hook running we can avoid having all this option specific stuff in our mainline code Using the function We ve cleaned up the code here by removing clumps of infrequently used code and moving them off somewhere else It s much easier for someone working with this code to see what s _really_ going on
Definition: hooks.txt:77
Maintenance script that cleans up invalid titles in various tables.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
const DB_REPLICA
Definition: defines.php:25
</td >< td > &</td >< td > t want your writing to be edited mercilessly and redistributed at will
For a write query
Definition: database.txt:26
while(( $__line=Maintenance::readconsole()) !==false) print n
Definition: eval.php:64
For a write use something like
Definition: database.txt:26