MediaWiki  master
MigrateActors.php
Go to the documentation of this file.
1 <?php
25 
26 require_once __DIR__ . '/../Maintenance.php';
27 
35 
36  protected $tables = null;
37 
38  public function __construct() {
39  parent::__construct();
40  $this->addDescription( 'Migrates actors from pre-1.31 columns to the \'actor\' table' );
41  $this->addOption( 'tables', 'List of tables to process, comma-separated', false, true );
42  $this->setBatchSize( 100 );
43  }
44 
45  protected function getUpdateKey() {
46  return __CLASS__;
47  }
48 
49  protected function doTable( $table ) {
50  return $this->tables === null || in_array( $table, $this->tables, true );
51  }
52 
53  protected function doDBUpdates() {
54  $tables = $this->getOption( 'tables' );
55  if ( $tables !== null ) {
56  $this->tables = explode( ',', $tables );
57  }
58 
59  if ( $this->doTable( 'user' ) ) {
60  $this->output( "Creating actor entries for all registered users\n" );
61  $end = 0;
62  $dbw = $this->getDB( DB_MASTER );
63  $max = $dbw->selectField( 'user', 'MAX(user_id)', '', __METHOD__ );
64  $count = 0;
65  while ( $end < $max ) {
66  $start = $end + 1;
67  $end = min( $start + $this->mBatchSize, $max );
68  $this->output( "... $start - $end\n" );
69  $dbw->insertSelect(
70  'actor',
71  'user',
72  [ 'actor_user' => 'user_id', 'actor_name' => 'user_name' ],
73  [ "user_id >= $start", "user_id <= $end" ],
74  __METHOD__,
75  [ 'IGNORE' ],
76  [ 'ORDER BY' => [ 'user_id' ] ]
77  );
78  $count += $dbw->affectedRows();
80  }
81  $this->output( "Completed actor creation, added $count new actor(s)\n" );
82  } else {
83  $this->output( "Checking that actors exist for all registered users\n" );
84  $dbr = $this->getDB( DB_REPLICA, [ 'vslow' ] );
85  $anyMissing = $dbr->selectField(
86  [ 'user', 'actor' ],
87  '1',
88  [ 'actor_id' => null ],
89  __METHOD__,
90  [ 'LIMIT 1' ],
91  [ 'actor' => [ 'LEFT JOIN', 'actor_user = user_id' ] ]
92  );
93  if ( $anyMissing ) {
94  $this->error( 'Some users lack actors; run without --tables or include `user` in --tables.' );
95  return false;
96  }
97  $this->output( "Ok, continuing.\n" );
98  }
99 
100  $errors = 0;
101  $errors += $this->migrateToTemp(
102  'revision', 'rev_id', [ 'revactor_timestamp' => 'rev_timestamp', 'revactor_page' => 'rev_page' ],
103  'rev_user', 'rev_user_text', 'revactor_rev', 'revactor_actor'
104  );
105  $errors += $this->migrate( 'archive', 'ar_id', 'ar_user', 'ar_user_text', 'ar_actor' );
106  $errors += $this->migrate( 'ipblocks', 'ipb_id', 'ipb_by', 'ipb_by_text', 'ipb_by_actor' );
107  $errors += $this->migrate( 'image', 'img_name', 'img_user', 'img_user_text', 'img_actor' );
108  $errors += $this->migrate(
109  'oldimage', [ 'oi_name', 'oi_timestamp' ], 'oi_user', 'oi_user_text', 'oi_actor'
110  );
111  $errors += $this->migrate( 'filearchive', 'fa_id', 'fa_user', 'fa_user_text', 'fa_actor' );
112  $errors += $this->migrate( 'recentchanges', 'rc_id', 'rc_user', 'rc_user_text', 'rc_actor' );
113  $errors += $this->migrate( 'logging', 'log_id', 'log_user', 'log_user_text', 'log_actor' );
114 
115  $errors += $this->migrateLogSearch();
116 
117  return $errors === 0;
118  }
119 
127  private function makeNextCond( $dbw, $primaryKey, $row ) {
128  $next = '';
129  $display = [];
130  for ( $i = count( $primaryKey ) - 1; $i >= 0; $i-- ) {
131  $field = $primaryKey[$i];
132  $display[] = $field . '=' . $row->$field;
133  $value = $dbw->addQuotes( $row->$field );
134  if ( $next === '' ) {
135  $next = "$field > $value";
136  } else {
137  $next = "$field > $value OR $field = $value AND ($next)";
138  }
139  }
140  $display = implode( ' ', array_reverse( $display ) );
141  return [ $next, $display ];
142  }
143 
151  private function makeActorIdSubquery( $dbw, $userField, $nameField ) {
152  $idSubquery = $dbw->buildSelectSubquery(
153  'actor',
154  'actor_id',
155  [ "$userField = actor_user" ],
156  __METHOD__
157  );
158  $nameSubquery = $dbw->buildSelectSubquery(
159  'actor',
160  'actor_id',
161  [ "$nameField = actor_name" ],
162  __METHOD__
163  );
164  return "CASE WHEN $userField = 0 OR $userField IS NULL THEN $nameSubquery ELSE $idSubquery END";
165  }
166 
178  private function addActorsForRows(
179  IDatabase $dbw, $nameField, array &$rows, array &$complainedAboutUsers, &$countErrors
180  ) {
181  $needActors = [];
182  $countActors = 0;
183 
184  $keep = [];
185  foreach ( $rows as $index => $row ) {
186  $keep[$index] = true;
187  if ( $row->actor_id === null ) {
188  // All registered users should have an actor_id already. So
189  // if we have a usable name here, it means they didn't run
190  // maintenance/cleanupUsersWithNoId.php
191  $name = $row->$nameField;
192  if ( User::isUsableName( $name ) ) {
193  if ( !isset( $complainedAboutUsers[$name] ) ) {
194  $complainedAboutUsers[$name] = true;
195  $this->error(
196  "User name \"$name\" is usable, cannot create an anonymous actor for it."
197  . " Run maintenance/cleanupUsersWithNoId.php to fix this situation.\n"
198  );
199  }
200  unset( $keep[$index] );
201  $countErrors++;
202  } else {
203  $needActors[$name] = 0;
204  }
205  }
206  }
207  $rows = array_intersect_key( $rows, $keep );
208 
209  if ( $needActors ) {
210  $dbw->insert(
211  'actor',
212  array_map( function ( $v ) {
213  return [
214  'actor_name' => $v,
215  ];
216  }, array_keys( $needActors ) ),
217  __METHOD__
218  );
219  $countActors += $dbw->affectedRows();
220 
221  $res = $dbw->select(
222  'actor',
223  [ 'actor_id', 'actor_name' ],
224  [ 'actor_name' => array_map( 'strval', array_keys( $needActors ) ) ],
225  __METHOD__
226  );
227  foreach ( $res as $row ) {
228  $needActors[$row->actor_name] = $row->actor_id;
229  }
230  foreach ( $rows as $row ) {
231  if ( $row->actor_id === null ) {
232  $row->actor_id = $needActors[$row->$nameField];
233  }
234  }
235  }
236 
237  return $countActors;
238  }
239 
253  protected function migrate( $table, $primaryKey, $userField, $nameField, $actorField ) {
254  if ( !$this->doTable( $table ) ) {
255  $this->output( "Skipping $table, not included in --tables\n" );
256  return 0;
257  }
258 
259  $dbw = $this->getDB( DB_MASTER );
260  if ( !$dbw->fieldExists( $table, $userField, __METHOD__ ) ) {
261  $this->output( "No need to migrate $table.$userField, field does not exist\n" );
262  return 0;
263  }
264 
265  $complainedAboutUsers = [];
266 
267  $primaryKey = (array)$primaryKey;
268  $pkFilter = array_flip( $primaryKey );
269  $this->output(
270  "Beginning migration of $table.$userField and $table.$nameField to $table.$actorField\n"
271  );
272  wfWaitForSlaves();
273 
274  $actorIdSubquery = $this->makeActorIdSubquery( $dbw, $userField, $nameField );
275  $next = '1=1';
276  $countUpdated = 0;
277  $countActors = 0;
278  $countErrors = 0;
279  while ( true ) {
280  // Fetch the rows needing update
281  $res = $dbw->select(
282  $table,
283  array_merge( $primaryKey, [ $userField, $nameField, 'actor_id' => $actorIdSubquery ] ),
284  [
285  $actorField => 0,
286  $next,
287  ],
288  __METHOD__,
289  [
290  'ORDER BY' => $primaryKey,
291  'LIMIT' => $this->mBatchSize,
292  ]
293  );
294  if ( !$res->numRows() ) {
295  break;
296  }
297 
298  // Insert new actors for rows that need one
299  $rows = iterator_to_array( $res );
300  $lastRow = end( $rows );
301  $countActors += $this->addActorsForRows(
302  $dbw, $nameField, $rows, $complainedAboutUsers, $countErrors
303  );
304 
305  // Update the existing rows
306  foreach ( $rows as $row ) {
307  if ( !$row->actor_id ) {
308  list( , $display ) = $this->makeNextCond( $dbw, $primaryKey, $row );
309  $this->error(
310  "Could not make actor for row with $display "
311  . "$userField={$row->$userField} $nameField={$row->$nameField}\n"
312  );
313  $countErrors++;
314  continue;
315  }
316  $dbw->update(
317  $table,
318  [
319  $actorField => $row->actor_id,
320  ],
321  array_intersect_key( (array)$row, $pkFilter ) + [
322  $actorField => 0
323  ],
324  __METHOD__
325  );
326  $countUpdated += $dbw->affectedRows();
327  }
328 
329  list( $next, $display ) = $this->makeNextCond( $dbw, $primaryKey, $lastRow );
330  $this->output( "... $display\n" );
331  wfWaitForSlaves();
332  }
333 
334  $this->output(
335  "Completed migration, updated $countUpdated row(s) with $countActors new actor(s), "
336  . "$countErrors error(s)\n"
337  );
338  return $countErrors;
339  }
340 
358  protected function migrateToTemp(
359  $table, $primaryKey, $extra, $userField, $nameField, $newPrimaryKey, $actorField
360  ) {
361  if ( !$this->doTable( $table ) ) {
362  $this->output( "Skipping $table, not included in --tables\n" );
363  return 0;
364  }
365 
366  $dbw = $this->getDB( DB_MASTER );
367  if ( !$dbw->fieldExists( $table, $userField, __METHOD__ ) ) {
368  $this->output( "No need to migrate $table.$userField, field does not exist\n" );
369  return 0;
370  }
371 
372  $complainedAboutUsers = [];
373 
374  $newTable = $table . '_actor_temp';
375  $this->output(
376  "Beginning migration of $table.$userField and $table.$nameField to $newTable.$actorField\n"
377  );
378  wfWaitForSlaves();
379 
380  $actorIdSubquery = $this->makeActorIdSubquery( $dbw, $userField, $nameField );
381  $next = [];
382  $countUpdated = 0;
383  $countActors = 0;
384  $countErrors = 0;
385  while ( true ) {
386  // Fetch the rows needing update
387  $res = $dbw->select(
388  [ $table, $newTable ],
389  [ $primaryKey, $userField, $nameField, 'actor_id' => $actorIdSubquery ] + $extra,
390  // @phan-suppress-next-line PhanSuspiciousBinaryAddLists
391  [ $newPrimaryKey => null ] + $next,
392  __METHOD__,
393  [
394  'ORDER BY' => $primaryKey,
395  'LIMIT' => $this->mBatchSize,
396  ],
397  [
398  $newTable => [ 'LEFT JOIN', "{$primaryKey}={$newPrimaryKey}" ],
399  ]
400  );
401  if ( !$res->numRows() ) {
402  break;
403  }
404 
405  // Insert new actors for rows that need one
406  $rows = iterator_to_array( $res );
407  $lastRow = end( $rows );
408  $countActors += $this->addActorsForRows(
409  $dbw, $nameField, $rows, $complainedAboutUsers, $countErrors
410  );
411 
412  // Update rows
413  if ( $rows ) {
414  $inserts = [];
415  $updates = [];
416  foreach ( $rows as $row ) {
417  if ( !$row->actor_id ) {
418  list( , $display ) = $this->makeNextCond( $dbw, [ $primaryKey ], $row );
419  $this->error(
420  "Could not make actor for row with $display "
421  . "$userField={$row->$userField} $nameField={$row->$nameField}\n"
422  );
423  $countErrors++;
424  continue;
425  }
426  $ins = [
427  $newPrimaryKey => $row->$primaryKey,
428  $actorField => $row->actor_id,
429  ];
430  foreach ( $extra as $to => $from ) {
431  $ins[$to] = $row->$to; // It's aliased
432  }
433  $inserts[] = $ins;
434  $updates[] = $row->$primaryKey;
435  }
436  $this->beginTransaction( $dbw, __METHOD__ );
437  $dbw->insert( $newTable, $inserts, __METHOD__ );
438  $countUpdated += $dbw->affectedRows();
439  $this->commitTransaction( $dbw, __METHOD__ );
440  }
441 
442  // Calculate the "next" condition
443  list( $n, $display ) = $this->makeNextCond( $dbw, [ $primaryKey ], $lastRow );
444  $next = [ $n ];
445  $this->output( "... $display\n" );
446  }
447 
448  $this->output(
449  "Completed migration, updated $countUpdated row(s) with $countActors new actor(s), "
450  . "$countErrors error(s)\n"
451  );
452  return $countErrors;
453  }
454 
459  protected function migrateLogSearch() {
460  if ( !$this->doTable( 'log_search' ) ) {
461  $this->output( "Skipping log_search, not included in --tables\n" );
462  return 0;
463  }
464 
465  $complainedAboutUsers = [];
466 
467  $primaryKey = [ 'ls_value', 'ls_log_id' ];
468  $this->output( "Beginning migration of log_search\n" );
469  wfWaitForSlaves();
470 
471  $dbw = $this->getDB( DB_MASTER );
472  $countInserted = 0;
473  $countActors = 0;
474  $countErrors = 0;
475 
476  $anyBad = $dbw->selectField(
477  'log_search',
478  '1',
479  [ 'ls_field' => 'target_author_actor', 'ls_value' => '' ],
480  __METHOD__,
481  [ 'LIMIT' => 1 ]
482  );
483  if ( $anyBad ) {
484  $this->output( "... Deleting bogus rows due to T215525\n" );
485  $dbw->delete(
486  'log_search',
487  [ 'ls_field' => 'target_author_actor', 'ls_value' => '' ],
488  __METHOD__
489  );
490  $ct = $dbw->affectedRows();
491  $this->output( "... Deleted $ct bogus row(s) from T215525\n" );
492  wfWaitForSlaves();
493  }
494 
495  $next = '1=1';
496  while ( true ) {
497  // Fetch the rows needing update
498  $res = $dbw->select(
499  [ 'log_search', 'actor' ],
500  [ 'ls_value', 'ls_log_id', 'actor_id' ],
501  [
502  'ls_field' => 'target_author_id',
503  $next
504  ],
505  __METHOD__,
506  [
507  'ORDER BY' => $primaryKey,
508  'LIMIT' => $this->mBatchSize,
509  ],
510  [ 'actor' => [ 'LEFT JOIN', 'actor_user = ' . $dbw->buildIntegerCast( 'ls_value' ) ] ]
511  );
512  if ( !$res->numRows() ) {
513  break;
514  }
515 
516  // Insert a 'target_author_actor' for each 'target_author_id'
517  $ins = [];
518  foreach ( $res as $row ) {
519  $lastRow = $row;
520  if ( !$row->actor_id ) {
521  list( , $display ) = $this->makeNextCond( $dbw, $primaryKey, $row );
522  $this->error( "No actor for target_author_id row with $display\n" );
523  $countErrors++;
524  continue;
525  }
526  $ins[] = [
527  'ls_field' => 'target_author_actor',
528  'ls_value' => $row->actor_id,
529  'ls_log_id' => $row->ls_log_id,
530  ];
531  }
532  $dbw->insert( 'log_search', $ins, __METHOD__, [ 'IGNORE' ] );
533  $countInserted += $dbw->affectedRows();
534 
535  list( $next, $display ) = $this->makeNextCond( $dbw, $primaryKey, $lastRow );
536  $this->output( "... target_author_id, $display\n" );
537  wfWaitForSlaves();
538  }
539 
540  $next = '1=1';
541  while ( true ) {
542  // Fetch the rows needing update
543  $res = $dbw->select(
544  [ 'log_search', 'actor' ],
545  [ 'ls_value', 'ls_log_id', 'actor_id' ],
546  [
547  'ls_field' => 'target_author_ip',
548  $next
549  ],
550  __METHOD__,
551  [
552  'ORDER BY' => $primaryKey,
553  'LIMIT' => $this->mBatchSize,
554  ],
555  [ 'actor' => [ 'LEFT JOIN', 'ls_value = actor_name' ] ]
556  );
557  if ( !$res->numRows() ) {
558  break;
559  }
560 
561  // Insert new actors for rows that need one
562  $rows = iterator_to_array( $res );
563  $lastRow = end( $rows );
564  $countActors += $this->addActorsForRows(
565  $dbw, 'ls_value', $rows, $complainedAboutUsers, $countErrors
566  );
567 
568  // Insert a 'target_author_actor' for each 'target_author_ip'
569  $ins = [];
570  foreach ( $rows as $row ) {
571  if ( !$row->actor_id ) {
572  list( , $display ) = $this->makeNextCond( $dbw, $primaryKey, $row );
573  $this->error( "Could not make actor for target_author_ip row with $display\n" );
574  $countErrors++;
575  continue;
576  }
577  $ins[] = [
578  'ls_field' => 'target_author_actor',
579  'ls_value' => $row->actor_id,
580  'ls_log_id' => $row->ls_log_id,
581  ];
582  }
583  $dbw->insert( 'log_search', $ins, __METHOD__, [ 'IGNORE' ] );
584  $countInserted += $dbw->affectedRows();
585 
586  list( $next, $display ) = $this->makeNextCond( $dbw, $primaryKey, $lastRow );
587  $this->output( "... target_author_ip, $display\n" );
588  wfWaitForSlaves();
589  }
590 
591  $this->output(
592  "Completed migration, inserted $countInserted row(s) with $countActors new actor(s), "
593  . "$countErrors error(s)\n"
594  );
595  return $countErrors;
596  }
597 }
Wikimedia\Rdbms\IDatabase\affectedRows
affectedRows()
Get the number of rows affected by the last write query.
MigrateActors\migrateLogSearch
migrateLogSearch()
Migrate actors in the log_search table.
Definition: MigrateActors.php:459
Maintenance\addDescription
addDescription( $text)
Set the description text.
Definition: Maintenance.php:354
$res
$res
Definition: testCompression.php:54
wfWaitForSlaves
wfWaitForSlaves( $ifWritesSince=null, $wiki=false, $cluster=false, $timeout=null)
Waits for the replica DBs to catch up to the master position.
Definition: GlobalFunctions.php:2705
Wikimedia\Rdbms\IDatabase
Basic database interface for live and lazy-loaded relation database handles.
Definition: IDatabase.php:38
$dbr
$dbr
Definition: testCompression.php:52
Maintenance\beginTransaction
beginTransaction(IDatabase $dbw, $fname)
Begin a transcation on a DB.
Definition: Maintenance.php:1439
MigrateActors\$tables
$tables
Definition: MigrateActors.php:36
LoggedUpdateMaintenance
Class for scripts that perform database maintenance and want to log the update in updatelog so we can...
Definition: Maintenance.php:1740
Maintenance\addOption
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
Definition: Maintenance.php:273
MigrateActors\migrateToTemp
migrateToTemp( $table, $primaryKey, $extra, $userField, $nameField, $newPrimaryKey, $actorField)
Migrate actors in a table to a temporary table.
Definition: MigrateActors.php:358
DB_REPLICA
const DB_REPLICA
Definition: defines.php:25
DB_MASTER
const DB_MASTER
Definition: defines.php:26
MigrateActors\makeActorIdSubquery
makeActorIdSubquery( $dbw, $userField, $nameField)
Make the subqueries for actor_id
Definition: MigrateActors.php:151
MigrateActors\doDBUpdates
doDBUpdates()
Do the actual work.
Definition: MigrateActors.php:53
MigrateActors\getUpdateKey
getUpdateKey()
Get the update key name to go in the update log table.
Definition: MigrateActors.php:45
MigrateActors\migrate
migrate( $table, $primaryKey, $userField, $nameField, $actorField)
Migrate actors in a table.
Definition: MigrateActors.php:253
Maintenance\commitTransaction
commitTransaction(IDatabase $dbw, $fname)
Commit the transcation on a DB handle and wait for replica DBs to catch up.
Definition: Maintenance.php:1454
Maintenance\getDB
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
Definition: Maintenance.php:1409
Wikimedia\Rdbms\IDatabase\insert
insert( $table, $rows, $fname=__METHOD__, $options=[])
INSERT wrapper, inserts an array into a table.
Maintenance\getOption
getOption( $name, $default=null)
Get an option, or return the default.
Definition: Maintenance.php:308
Wikimedia\Rdbms\IDatabase\select
select( $table, $vars, $conds='', $fname=__METHOD__, $options=[], $join_conds=[])
Execute a SELECT query constructed using the various parameters provided.
MigrateActors
Maintenance script that migrates actors from pre-1.31 columns to the 'actor' table.
Definition: MigrateActors.php:34
Maintenance\error
error( $err, $die=0)
Throw an error to the user.
Definition: Maintenance.php:487
Maintenance\output
output( $out, $channel=null)
Throw some output to the user.
Definition: Maintenance.php:459
MigrateActors\addActorsForRows
addActorsForRows(IDatabase $dbw, $nameField, array &$rows, array &$complainedAboutUsers, &$countErrors)
Add actors for anons in a set of rows.
Definition: MigrateActors.php:178
MigrateActors\makeNextCond
makeNextCond( $dbw, $primaryKey, $row)
Calculate a "next" condition and a display string.
Definition: MigrateActors.php:127
User\isUsableName
static isUsableName( $name)
Usernames which fail to pass this function will be blocked from user login and new account registrati...
Definition: User.php:1023
MigrateActors\__construct
__construct()
Default constructor.
Definition: MigrateActors.php:38
MigrateActors\doTable
doTable( $table)
Definition: MigrateActors.php:49
Maintenance\setBatchSize
setBatchSize( $s=0)
Set the batch size.
Definition: Maintenance.php:400