MediaWiki  master
convertLinks.php
Go to the documentation of this file.
1 <?php
25 
26 require_once __DIR__ . '/Maintenance.php';
27 
36 class ConvertLinks extends Maintenance {
37  private $logPerformance;
38 
39  public function __construct() {
40  parent::__construct();
41  $this->addDescription(
42  'Convert from the old links schema (string->ID) to the new schema (ID->ID). '
43  . 'The wiki should be put into read-only mode while this script executes' );
44 
45  $this->addArg( 'logperformance', "Log performance to perfLogFilename.", false );
46  $this->addArg(
47  'perfLogFilename',
48  "Filename where performance is logged if --logperformance was set "
49  . "(defaults to 'convLinksPerf.txt').",
50  false
51  );
52  $this->addArg(
53  'keep-links-table',
54  "Don't overwrite the old links table with the new one, leave the new table at links_temp.",
55  false
56  );
57  $this->addArg(
58  'nokeys',
59  /* (What about InnoDB?) */
60  "Don't create keys, and so allow duplicates in the new links table.\n"
61  . "This gives a huge speed improvement for very large links tables which are MyISAM.",
62  false
63  );
64  }
65 
66  public function getDbType() {
67  return Maintenance::DB_ADMIN;
68  }
69 
70  public function execute() {
71  $dbw = $this->getDB( DB_MASTER );
72 
73  $type = $dbw->getType();
74  if ( $type != 'mysql' ) {
75  $this->output( "Link table conversion not necessary for $type\n" );
76 
77  return;
78  }
79 
80  # counters etc
81  $numBadLinks = $curRowsRead = 0;
82 
83  # total tuples INSERTed into links_temp
84  $totalTuplesInserted = 0;
85 
86  # whether or not to give progress reports while reading IDs from cur table
87  $reportCurReadProgress = true;
88 
89  # number of rows between progress reports
90  $curReadReportInterval = 1000;
91 
92  # whether or not to give progress reports during conversion
93  $reportLinksConvProgress = true;
94 
95  # number of rows per INSERT
96  $linksConvInsertInterval = 1000;
97 
98  $initialRowOffset = 0;
99 
100  # not used yet; highest row number from links table to process
101  # $finalRowOffset = 0;
102 
103  $this->logPerformance = $this->hasOption( 'logperformance' );
104  $perfLogFilename = $this->getArg( 1, "convLinksPerf.txt" );
105  $overwriteLinksTable = !$this->hasOption( 'keep-links-table' );
106  $noKeys = $this->hasOption( 'noKeys' );
107 
108  # --------------------------------------------------------------------
109 
110  list( $cur, $links, $links_temp, $links_backup ) =
111  $dbw->tableNamesN( 'cur', 'links', 'links_temp', 'links_backup' );
112 
113  if ( $dbw->tableExists( 'pagelinks', __METHOD__ ) ) {
114  $this->output( "...have pagelinks; skipping old links table updates\n" );
115 
116  return;
117  }
118 
119  $res = $dbw->query( "SELECT l_from FROM $links LIMIT 1", __METHOD__ );
120  // @phan-suppress-next-line PhanUndeclaredMethod
121  if ( $dbw->fieldType( $res, 0 ) == "int" ) {
122  $this->output( "Schema already converted\n" );
123 
124  return;
125  }
126 
127  $res = $dbw->query( "SELECT COUNT(*) AS count FROM $links", __METHOD__ );
128  $row = $dbw->fetchObject( $res );
129  $numRows = $row->count;
130 
131  if ( $numRows == 0 ) {
132  $this->output( "Updating schema (no rows to convert)...\n" );
133  $this->createTempTable();
134  } else {
135  $fh = false;
136  if ( $this->logPerformance ) {
137  $fh = fopen( $perfLogFilename, "w" );
138  if ( !$fh ) {
139  $this->error( "Couldn't open $perfLogFilename" );
140  $this->logPerformance = false;
141  }
142  }
143  $baseTime = $startTime = microtime( true );
144  # Create a title -> cur_id map
145  $this->output( "Loading IDs from $cur table...\n" );
146  $this->performanceLog( $fh, "Reading $numRows rows from cur table...\n" );
147  $this->performanceLog( $fh, "rows read vs seconds elapsed:\n" );
148  $contentLang = MediaWikiServices::getInstance()->getContentLanguage();
149 
150  $ids = [];
151  $lastId = 0;
152  do {
153  $res = $dbw->query(
154  "SELECT cur_namespace,cur_title,cur_id FROM $cur " .
155  "WHERE cur_id > $lastId ORDER BY cur_id LIMIT 10000",
156  __METHOD__
157  );
158  foreach ( $res as $row ) {
159  $title = $row->cur_title;
160  if ( $row->cur_namespace ) {
161  $title = $contentLang->getNsText( $row->cur_namespace ) . ":$title";
162  }
163  $ids[$title] = $row->cur_id;
164  $curRowsRead++;
165  if ( $reportCurReadProgress ) {
166  if ( ( $curRowsRead % $curReadReportInterval ) == 0 ) {
167  $this->performanceLog(
168  $fh,
169  $curRowsRead . " " . ( microtime( true ) - $baseTime ) . "\n"
170  );
171  $this->output( "\t$curRowsRead rows of $cur table read.\n" );
172  }
173  }
174  $lastId = $row->cur_id;
175  }
176  } while ( $res->numRows() > 0 );
177  $this->output( "Finished loading IDs.\n\n" );
178  $this->performanceLog(
179  $fh,
180  "Took " . ( microtime( true ) - $baseTime ) . " seconds to load IDs.\n\n"
181  );
182 
183  # --------------------------------------------------------------------
184 
185  # Now, step through the links table (in chunks of $linksConvInsertInterval rows),
186  # convert, and write to the new table.
187  $this->createTempTable();
188  $this->performanceLog( $fh, "Resetting timer.\n\n" );
189  $baseTime = microtime( true );
190  $this->output( "Processing $numRows rows from $links table...\n" );
191  $this->performanceLog( $fh, "Processing $numRows rows from $links table...\n" );
192  $this->performanceLog( $fh, "rows inserted vs seconds elapsed:\n" );
193 
194  for ( $rowOffset = $initialRowOffset; $rowOffset < $numRows;
195  $rowOffset += $linksConvInsertInterval
196  ) {
197  $sqlRead = "SELECT * FROM $links ";
198  $sqlRead = $dbw->limitResult( $sqlRead, $linksConvInsertInterval, $rowOffset );
199  $res = $dbw->query( $sqlRead, __METHOD__ );
200  if ( $noKeys ) {
201  $sqlWrite = [ "INSERT INTO $links_temp (l_from,l_to) VALUES " ];
202  } else {
203  $sqlWrite = [ "INSERT IGNORE INTO $links_temp (l_from,l_to) VALUES " ];
204  }
205 
206  $tuplesAdded = 0; # no tuples added to INSERT yet
207  foreach ( $res as $row ) {
208  $fromTitle = $row->l_from;
209  if ( array_key_exists( $fromTitle, $ids ) ) { # valid title
210  $from = $ids[$fromTitle];
211  $to = $row->l_to;
212  if ( $tuplesAdded != 0 ) {
213  $sqlWrite[] = ",";
214  }
215  $sqlWrite[] = "($from,$to)";
216  $tuplesAdded++;
217  } else { # invalid title
218  $numBadLinks++;
219  }
220  }
221  # $this->output( "rowOffset: $rowOffset\ttuplesAdded: "
222  # . "$tuplesAdded\tnumBadLinks: $numBadLinks\n" );
223  if ( $tuplesAdded != 0 ) {
224  if ( $reportLinksConvProgress ) {
225  $this->output( "Inserting $tuplesAdded tuples into $links_temp..." );
226  }
227  $dbw->query( implode( "", $sqlWrite ), __METHOD__ );
228  $totalTuplesInserted += $tuplesAdded;
229  if ( $reportLinksConvProgress ) {
230  $this->output( " done. Total $totalTuplesInserted tuples inserted.\n" );
231  $this->performanceLog(
232  $fh,
233  $totalTuplesInserted . " " . ( microtime( true ) - $baseTime ) . "\n"
234  );
235  }
236  }
237  }
238  $this->output( "$totalTuplesInserted valid titles and "
239  . "$numBadLinks invalid titles were processed.\n\n" );
240  $this->performanceLog(
241  $fh,
242  "$totalTuplesInserted valid titles and $numBadLinks invalid titles were processed.\n"
243  );
244  $this->performanceLog(
245  $fh,
246  "Total execution time: " . ( microtime( true ) - $startTime ) . " seconds.\n"
247  );
248  if ( $this->logPerformance ) {
249  fclose( $fh );
250  }
251  }
252  # --------------------------------------------------------------------
253 
254  if ( $overwriteLinksTable ) {
255  # Check for existing links_backup, and delete it if it exists.
256  $this->output( "Dropping backup links table if it exists..." );
257  $dbw->query( "DROP TABLE IF EXISTS $links_backup", __METHOD__ );
258  $this->output( " done.\n" );
259 
260  # Swap in the new table, and move old links table to links_backup
261  $this->output( "Swapping tables '$links' to '$links_backup'; '$links_temp' to '$links'..." );
262  $dbw->query( "RENAME TABLE links TO $links_backup, $links_temp TO $links", __METHOD__ );
263  $this->output( " done.\n\n" );
264 
265  $this->output( "Conversion complete. The old table remains at $links_backup;\n" );
266  $this->output( "delete at your leisure.\n" );
267  } else {
268  $this->output( "Conversion complete. The converted table is at $links_temp;\n" );
269  $this->output( "the original links table is unchanged.\n" );
270  }
271  }
272 
273  private function createTempTable() {
274  $dbConn = $this->getDB( DB_MASTER );
275 
276  if ( !( $dbConn->isOpen() ) ) {
277  $this->output( "Opening connection to database failed.\n" );
278 
279  return;
280  }
281  $links_temp = $dbConn->tableName( 'links_temp' );
282 
283  $this->output( "Dropping temporary links table if it exists..." );
284  $dbConn->query( "DROP TABLE IF EXISTS $links_temp", __METHOD__ );
285  $this->output( " done.\n" );
286 
287  $this->output( "Creating temporary links table..." );
288  if ( $this->hasOption( 'noKeys' ) ) {
289  $dbConn->query( "CREATE TABLE $links_temp ( " .
290  "l_from int(8) unsigned NOT NULL default '0', " .
291  "l_to int(8) unsigned NOT NULL default '0')", __METHOD__ );
292  } else {
293  $dbConn->query( "CREATE TABLE $links_temp ( " .
294  "l_from int(8) unsigned NOT NULL default '0', " .
295  "l_to int(8) unsigned NOT NULL default '0', " .
296  "UNIQUE KEY l_from(l_from,l_to), " .
297  "KEY (l_to))", __METHOD__ );
298  }
299  $this->output( " done.\n\n" );
300  }
301 
302  private function performanceLog( $fh, $text ) {
303  if ( $this->logPerformance ) {
304  fwrite( $fh, $text );
305  }
306  }
307 }
308 
309 $maintClass = ConvertLinks::class;
310 require_once RUN_MAINTENANCE_IF_MAIN;
RUN_MAINTENANCE_IF_MAIN
const RUN_MAINTENANCE_IF_MAIN
Definition: Maintenance.php:38
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:154
Maintenance\addDescription
addDescription( $text)
Set the description text.
Definition: Maintenance.php:327
Maintenance
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: Maintenance.php:55
$res
$res
Definition: testCompression.php:57
$title
$title
Definition: testCompression.php:38
DB_MASTER
const DB_MASTER
Definition: defines.php:26
Maintenance\DB_ADMIN
const DB_ADMIN
Definition: Maintenance.php:62
Maintenance\getDB
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
Definition: Maintenance.php:1366
Maintenance\addArg
addArg( $arg, $description, $required=true)
Add some args that are needed.
Definition: Maintenance.php:298
Maintenance\error
error( $err, $die=0)
Throw an error to the user.
Definition: Maintenance.php:463
Maintenance\output
output( $out, $channel=null)
Throw some output to the user.
Definition: Maintenance.php:434
Maintenance\hasOption
hasOption( $name)
Checks to see if a particular option exists.
Definition: Maintenance.php:266
Maintenance\getArg
getArg( $argId=0, $default=null)
Get an argument.
Definition: Maintenance.php:351
$type
$type
Definition: testCompression.php:52