MediaWiki  master
migrateComments.php
Go to the documentation of this file.
1 <?php
26 
27 require_once __DIR__ . '/Maintenance.php';
28 
36  public function __construct() {
37  parent::__construct();
38  $this->addDescription( 'Migrates comments from pre-1.30 columns to the \'comment\' table' );
39  $this->setBatchSize( 100 );
40  }
41 
42  protected function getUpdateKey() {
43  return __CLASS__;
44  }
45 
46  protected function updateSkippedMessage() {
47  return 'comments already migrated.';
48  }
49 
50  protected function doDBUpdates() {
51  $this->migrateToTemp(
52  'revision', 'rev_id', 'rev_comment', 'revcomment_rev', 'revcomment_comment_id'
53  );
54  $this->migrate( 'archive', 'ar_id', 'ar_comment' );
55  $this->migrate( 'ipblocks', 'ipb_id', 'ipb_reason' );
56  $this->migrate( 'image', 'img_name', 'img_description' );
57  $this->migrate( 'oldimage', [ 'oi_name', 'oi_timestamp' ], 'oi_description' );
58  $this->migrate( 'filearchive', 'fa_id', 'fa_deleted_reason' );
59  $this->migrate( 'filearchive', 'fa_id', 'fa_description' );
60  $this->migrate( 'recentchanges', 'rc_id', 'rc_comment' );
61  $this->migrate( 'logging', 'log_id', 'log_comment' );
62  $this->migrate( 'protected_titles', [ 'pt_namespace', 'pt_title' ], 'pt_reason' );
63  return true;
64  }
65 
72  private function loadCommentIDs( IDatabase $dbw, array &$comments ) {
73  $count = 0;
74  $needComments = $comments;
75 
76  while ( true ) {
77  $where = [];
78  foreach ( $needComments as $need => $dummy ) {
79  $where[] = $dbw->makeList(
80  [
81  'comment_hash' => CommentStore::hash( $need, null ),
82  'comment_text' => $need,
83  ],
84  LIST_AND
85  );
86  }
87 
88  $res = $dbw->select(
89  'comment',
90  [ 'comment_id', 'comment_text' ],
91  [
92  $dbw->makeList( $where, LIST_OR ),
93  'comment_data' => null,
94  ],
95  __METHOD__
96  );
97  foreach ( $res as $row ) {
98  $comments[$row->comment_text] = $row->comment_id;
99  unset( $needComments[$row->comment_text] );
100  }
101 
102  if ( !$needComments ) {
103  break;
104  }
105 
106  $dbw->insert(
107  'comment',
108  array_map( function ( $v ) {
109  return [
110  'comment_hash' => CommentStore::hash( $v, null ),
111  'comment_text' => $v,
112  ];
113  }, array_keys( $needComments ) ),
114  __METHOD__
115  );
116  $count += $dbw->affectedRows();
117  }
118  return $count;
119  }
120 
132  protected function migrate( $table, $primaryKey, $oldField ) {
133  $dbw = $this->getDB( DB_MASTER );
134  if ( !$dbw->fieldExists( $table, $oldField, __METHOD__ ) ) {
135  $this->output( "No need to migrate $table.$oldField, field does not exist\n" );
136  return;
137  }
138 
139  $newField = $oldField . '_id';
140  $primaryKey = (array)$primaryKey;
141  $pkFilter = array_flip( $primaryKey );
142  $this->output( "Beginning migration of $table.$oldField to $table.$newField\n" );
143  $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
144  $lbFactory->waitForReplication();
145 
146  $next = '1=1';
147  $countUpdated = 0;
148  $countComments = 0;
149  while ( true ) {
150  // Fetch the rows needing update
151  $res = $dbw->select(
152  $table,
153  array_merge( $primaryKey, [ $oldField ] ),
154  [
155  $newField => 0,
156  $next,
157  ],
158  __METHOD__,
159  [
160  'ORDER BY' => $primaryKey,
161  'LIMIT' => $this->getBatchSize(),
162  ]
163  );
164  if ( !$res->numRows() ) {
165  break;
166  }
167 
168  // Collect the distinct comments from those rows
169  $comments = [];
170  foreach ( $res as $row ) {
171  $comments[$row->$oldField] = 0;
172  }
173  $countComments += $this->loadCommentIDs( $dbw, $comments );
174 
175  // Update the existing rows
176  foreach ( $res as $row ) {
177  $dbw->update(
178  $table,
179  [
180  $newField => $comments[$row->$oldField],
181  $oldField => '',
182  ],
183  array_intersect_key( (array)$row, $pkFilter ) + [
184  $newField => 0
185  ],
186  __METHOD__
187  );
188  $countUpdated += $dbw->affectedRows();
189  }
190 
191  // Calculate the "next" condition
192  $next = '';
193  $prompt = [];
194  for ( $i = count( $primaryKey ) - 1; $i >= 0; $i-- ) {
195  $field = $primaryKey[$i];
196  $prompt[] = $row->$field;
197  $value = $dbw->addQuotes( $row->$field );
198  if ( $next === '' ) {
199  $next = "$field > $value";
200  } else {
201  $next = "$field > $value OR $field = $value AND ($next)";
202  }
203  }
204  $prompt = implode( ' ', array_reverse( $prompt ) );
205  $this->output( "... $prompt\n" );
206  $lbFactory->waitForReplication();
207  }
208 
209  $this->output(
210  "Completed migration, updated $countUpdated row(s) with $countComments new comment(s)\n"
211  );
212  }
213 
229  protected function migrateToTemp( $table, $primaryKey, $oldField, $newPrimaryKey, $newField ) {
230  $dbw = $this->getDB( DB_MASTER );
231  if ( !$dbw->fieldExists( $table, $oldField, __METHOD__ ) ) {
232  $this->output( "No need to migrate $table.$oldField, field does not exist\n" );
233  return;
234  }
235 
236  $newTable = $table . '_comment_temp';
237  $this->output( "Beginning migration of $table.$oldField to $newTable.$newField\n" );
238  MediaWikiServices::getInstance()->getDBLoadBalancerFactory()->waitForReplication();
239 
240  $dbw = $this->getDB( DB_MASTER );
241  $next = [];
242  $countUpdated = 0;
243  $countComments = 0;
244  while ( true ) {
245  // Fetch the rows needing update
246  $res = $dbw->select(
247  [ $table, $newTable ],
248  [ $primaryKey, $oldField ],
249  // @phan-suppress-next-line PhanSuspiciousBinaryAddLists
250  [ $newPrimaryKey => null ] + $next,
251  __METHOD__,
252  [
253  'ORDER BY' => $primaryKey,
254  'LIMIT' => $this->getBatchSize(),
255  ],
256  [ $newTable => [ 'LEFT JOIN', "{$primaryKey}={$newPrimaryKey}" ] ]
257  );
258  if ( !$res->numRows() ) {
259  break;
260  }
261 
262  // Collect the distinct comments from those rows
263  $comments = [];
264  foreach ( $res as $row ) {
265  $comments[$row->$oldField] = 0;
266  }
267  $countComments += $this->loadCommentIDs( $dbw, $comments );
268 
269  // Update rows
270  $inserts = [];
271  $updates = [];
272  foreach ( $res as $row ) {
273  $inserts[] = [
274  $newPrimaryKey => $row->$primaryKey,
275  $newField => $comments[$row->$oldField]
276  ];
277  $updates[] = $row->$primaryKey;
278  }
279  $this->beginTransaction( $dbw, __METHOD__ );
280  $dbw->insert( $newTable, $inserts, __METHOD__ );
281  $dbw->update( $table, [ $oldField => '' ], [ $primaryKey => $updates ], __METHOD__ );
282  $countUpdated += $dbw->affectedRows();
283  $this->commitTransaction( $dbw, __METHOD__ );
284 
285  // Calculate the "next" condition
286  $next = [ $primaryKey . ' > ' . $dbw->addQuotes( $row->$primaryKey ) ];
287  $this->output( "... {$row->$primaryKey}\n" );
288  }
289 
290  $this->output(
291  "Completed migration, updated $countUpdated row(s) with $countComments new comment(s)\n"
292  );
293  }
294 }
295 
296 $maintClass = MigrateComments::class;
297 require_once RUN_MAINTENANCE_IF_MAIN;
RUN_MAINTENANCE_IF_MAIN
const RUN_MAINTENANCE_IF_MAIN
Definition: Maintenance.php:38
Wikimedia\Rdbms\IDatabase\affectedRows
affectedRows()
Get the number of rows affected by the last write query.
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:146
MigrateComments\doDBUpdates
doDBUpdates()
Do the actual work.
Definition: migrateComments.php:50
Maintenance\addDescription
addDescription( $text)
Set the description text.
Definition: Maintenance.php:323
$res
$res
Definition: testCompression.php:57
MigrateComments\migrate
migrate( $table, $primaryKey, $oldField)
Migrate comments in a table.
Definition: migrateComments.php:132
LIST_AND
const LIST_AND
Definition: Defines.php:48
Wikimedia\Rdbms\IDatabase
Basic database interface for live and lazy-loaded relation database handles.
Definition: IDatabase.php:38
MigrateComments
Maintenance script that migrates comments from pre-1.30 columns to the 'comment' table.
Definition: migrateComments.php:35
Maintenance\beginTransaction
beginTransaction(IDatabase $dbw, $fname)
Begin a transcation on a DB.
Definition: Maintenance.php:1380
LIST_OR
const LIST_OR
Definition: Defines.php:51
$maintClass
$maintClass
Definition: migrateComments.php:296
LoggedUpdateMaintenance
Class for scripts that perform database maintenance and want to log the update in updatelog so we can...
Definition: LoggedUpdateMaintenance.php:26
DB_MASTER
const DB_MASTER
Definition: defines.php:26
MigrateComments\updateSkippedMessage
updateSkippedMessage()
Message to show that the update was done already and was just skipped.
Definition: migrateComments.php:46
MigrateComments\getUpdateKey
getUpdateKey()
Get the update key name to go in the update log table.
Definition: migrateComments.php:42
Maintenance\commitTransaction
commitTransaction(IDatabase $dbw, $fname)
Commit the transcation on a DB handle and wait for replica DBs to catch up.
Definition: Maintenance.php:1395
Maintenance\getDB
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
Definition: Maintenance.php:1350
Wikimedia\Rdbms\IDatabase\insert
insert( $table, $rows, $fname=__METHOD__, $options=[])
Insert the given row(s) into a table.
CommentStore\hash
static hash( $text, $data)
Hashing function for comment storage.
Definition: CommentStore.php:640
Maintenance\getBatchSize
getBatchSize()
Returns batch size.
Definition: Maintenance.php:362
Wikimedia\Rdbms\IDatabase\select
select( $table, $vars, $conds='', $fname=__METHOD__, $options=[], $join_conds=[])
Execute a SELECT query constructed using the various parameters provided.
Maintenance\output
output( $out, $channel=null)
Throw some output to the user.
Definition: Maintenance.php:429
MigrateComments\migrateToTemp
migrateToTemp( $table, $primaryKey, $oldField, $newPrimaryKey, $newField)
Migrate comments in a table to a temporary table.
Definition: migrateComments.php:229
Wikimedia\Rdbms\IDatabase\makeList
makeList(array $a, $mode=self::LIST_COMMA)
Makes an encoded list of strings from an array.
MigrateComments\__construct
__construct()
Default constructor.
Definition: migrateComments.php:36
MigrateComments\loadCommentIDs
loadCommentIDs(IDatabase $dbw, array &$comments)
Fetch comment IDs for a set of comments.
Definition: migrateComments.php:72
Maintenance\setBatchSize
setBatchSize( $s=0)
Set the batch size.
Definition: Maintenance.php:370