MediaWiki  master
migrateComments.php
Go to the documentation of this file.
1 <?php
26 
27 require_once __DIR__ . '/Maintenance.php';
28 
36  public function __construct() {
37  parent::__construct();
38  $this->addDescription( 'Migrates comments from pre-1.30 columns to the \'comment\' table' );
39  $this->setBatchSize( 100 );
40  }
41 
42  protected function getUpdateKey() {
43  return __CLASS__;
44  }
45 
46  protected function updateSkippedMessage() {
47  return 'comments already migrated.';
48  }
49 
50  protected function doDBUpdates() {
51  $this->migrateToTemp(
52  'revision', 'rev_id', 'rev_comment', 'revcomment_rev', 'revcomment_comment_id'
53  );
54  $this->migrate( 'archive', 'ar_id', 'ar_comment' );
55  $this->migrate( 'ipblocks', 'ipb_id', 'ipb_reason' );
56  $this->migrate( 'image', 'img_name', 'img_description' );
57  $this->migrate( 'oldimage', [ 'oi_name', 'oi_timestamp' ], 'oi_description' );
58  $this->migrate( 'filearchive', 'fa_id', 'fa_deleted_reason' );
59  $this->migrate( 'filearchive', 'fa_id', 'fa_description' );
60  $this->migrate( 'recentchanges', 'rc_id', 'rc_comment' );
61  $this->migrate( 'logging', 'log_id', 'log_comment' );
62  $this->migrate( 'protected_titles', [ 'pt_namespace', 'pt_title' ], 'pt_reason' );
63  return true;
64  }
65 
72  private function loadCommentIDs( IDatabase $dbw, array &$comments ) {
73  $count = 0;
74  $needComments = $comments;
75 
76  while ( true ) {
77  $where = [];
78  foreach ( $needComments as $need => $dummy ) {
79  $need = (string)$need; // T268887
80  $where[] = $dbw->makeList(
81  [
82  'comment_hash' => CommentStore::hash( $need, null ),
83  'comment_text' => $need,
84  ],
85  LIST_AND
86  );
87  }
88 
89  $res = $dbw->select(
90  'comment',
91  [ 'comment_id', 'comment_text' ],
92  [
93  $dbw->makeList( $where, LIST_OR ),
94  'comment_data' => null,
95  ],
96  __METHOD__
97  );
98  foreach ( $res as $row ) {
99  $comments[$row->comment_text] = $row->comment_id;
100  unset( $needComments[$row->comment_text] );
101  }
102 
103  if ( !$needComments ) {
104  break;
105  }
106 
107  $dbw->insert(
108  'comment',
109  array_map( static function ( $v ) {
110  return [
111  'comment_hash' => CommentStore::hash( $v, null ),
112  'comment_text' => $v,
113  ];
114  }, array_keys( $needComments ) ),
115  __METHOD__
116  );
117  $count += $dbw->affectedRows();
118  }
119  return $count;
120  }
121 
133  protected function migrate( $table, $primaryKey, $oldField ) {
134  $dbw = $this->getDB( DB_PRIMARY );
135  if ( !$dbw->fieldExists( $table, $oldField, __METHOD__ ) ) {
136  $this->output( "No need to migrate $table.$oldField, field does not exist\n" );
137  return;
138  }
139 
140  $newField = $oldField . '_id';
141  $primaryKey = (array)$primaryKey;
142  $pkFilter = array_fill_keys( $primaryKey, true );
143  $this->output( "Beginning migration of $table.$oldField to $table.$newField\n" );
144  $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
145  $lbFactory->waitForReplication();
146 
147  $next = '1=1';
148  $countUpdated = 0;
149  $countComments = 0;
150  while ( true ) {
151  // Fetch the rows needing update
152  $res = $dbw->select(
153  $table,
154  array_merge( $primaryKey, [ $oldField ] ),
155  [
156  $newField => 0,
157  $next,
158  ],
159  __METHOD__,
160  [
161  'ORDER BY' => $primaryKey,
162  'LIMIT' => $this->getBatchSize(),
163  ]
164  );
165  if ( !$res->numRows() ) {
166  break;
167  }
168 
169  // Collect the distinct comments from those rows
170  $comments = [];
171  foreach ( $res as $row ) {
172  $comments[$row->$oldField] = 0;
173  }
174  $countComments += $this->loadCommentIDs( $dbw, $comments );
175 
176  // Update the existing rows
177  foreach ( $res as $row ) {
178  $dbw->update(
179  $table,
180  [
181  $newField => $comments[$row->$oldField],
182  $oldField => '',
183  ],
184  array_intersect_key( (array)$row, $pkFilter ) + [
185  $newField => 0
186  ],
187  __METHOD__
188  );
189  $countUpdated += $dbw->affectedRows();
190  }
191 
192  // Calculate the "next" condition
193  $next = '';
194  $prompt = [];
195  for ( $i = count( $primaryKey ) - 1; $i >= 0; $i-- ) {
196  $field = $primaryKey[$i];
197  $prompt[] = $row->$field;
198  $value = $dbw->addQuotes( $row->$field );
199  if ( $next === '' ) {
200  $next = "$field > $value";
201  } else {
202  $next = "$field > $value OR $field = $value AND ($next)";
203  }
204  }
205  $prompt = implode( ' ', array_reverse( $prompt ) );
206  $this->output( "... $prompt\n" );
207  $lbFactory->waitForReplication();
208  }
209 
210  $this->output(
211  "Completed migration, updated $countUpdated row(s) with $countComments new comment(s)\n"
212  );
213  }
214 
230  protected function migrateToTemp( $table, $primaryKey, $oldField, $newPrimaryKey, $newField ) {
231  $dbw = $this->getDB( DB_PRIMARY );
232  if ( !$dbw->fieldExists( $table, $oldField, __METHOD__ ) ) {
233  $this->output( "No need to migrate $table.$oldField, field does not exist\n" );
234  return;
235  }
236 
237  $newTable = $table . '_comment_temp';
238  $this->output( "Beginning migration of $table.$oldField to $newTable.$newField\n" );
239  MediaWikiServices::getInstance()->getDBLoadBalancerFactory()->waitForReplication();
240 
241  $dbw = $this->getDB( DB_PRIMARY );
242  $next = [];
243  $countUpdated = 0;
244  $countComments = 0;
245  while ( true ) {
246  // Fetch the rows needing update
247  $res = $dbw->select(
248  [ $table, $newTable ],
249  [ $primaryKey, $oldField ],
250  [ $newPrimaryKey => null ] + $next,
251  __METHOD__,
252  [
253  'ORDER BY' => $primaryKey,
254  'LIMIT' => $this->getBatchSize(),
255  ],
256  [ $newTable => [ 'LEFT JOIN', "{$primaryKey}={$newPrimaryKey}" ] ]
257  );
258  if ( !$res->numRows() ) {
259  break;
260  }
261 
262  // Collect the distinct comments from those rows
263  $comments = [];
264  foreach ( $res as $row ) {
265  $comments[$row->$oldField] = 0;
266  }
267  $countComments += $this->loadCommentIDs( $dbw, $comments );
268 
269  // Update rows
270  $inserts = [];
271  $updates = [];
272  foreach ( $res as $row ) {
273  $inserts[] = [
274  $newPrimaryKey => $row->$primaryKey,
275  $newField => $comments[$row->$oldField]
276  ];
277  $updates[] = $row->$primaryKey;
278  }
279  $this->beginTransaction( $dbw, __METHOD__ );
280  $dbw->insert( $newTable, $inserts, __METHOD__ );
281  $dbw->update( $table, [ $oldField => '' ], [ $primaryKey => $updates ], __METHOD__ );
282  $countUpdated += $dbw->affectedRows();
283  $this->commitTransaction( $dbw, __METHOD__ );
284 
285  // Calculate the "next" condition
286  $next = [ $primaryKey . ' > ' . $dbw->addQuotes( $row->$primaryKey ) ];
287  $this->output( "... {$row->$primaryKey}\n" );
288  }
289 
290  $this->output(
291  "Completed migration, updated $countUpdated row(s) with $countComments new comment(s)\n"
292  );
293  }
294 }
295 
296 $maintClass = MigrateComments::class;
297 require_once RUN_MAINTENANCE_IF_MAIN;
LIST_OR
const LIST_OR
Definition: Defines.php:46
Wikimedia\Rdbms\IDatabase\affectedRows
affectedRows()
Get the number of rows affected by the last write query.
MediaWiki\MediaWikiServices
MediaWikiServices is the service locator for the application scope of MediaWiki.
Definition: MediaWikiServices.php:193
MigrateComments\doDBUpdates
doDBUpdates()
Do the actual work.
Definition: migrateComments.php:50
Maintenance\addDescription
addDescription( $text)
Set the description text.
Definition: Maintenance.php:329
LIST_AND
const LIST_AND
Definition: Defines.php:43
$res
$res
Definition: testCompression.php:57
MigrateComments\migrate
migrate( $table, $primaryKey, $oldField)
Migrate comments in a table.
Definition: migrateComments.php:133
Wikimedia\Rdbms\IDatabase
Basic database interface for live and lazy-loaded relation database handles.
Definition: IDatabase.php:38
MigrateComments
Maintenance script that migrates comments from pre-1.30 columns to the 'comment' table.
Definition: migrateComments.php:35
Maintenance\beginTransaction
beginTransaction(IDatabase $dbw, $fname)
Begin a transaction on a DB.
Definition: Maintenance.php:1393
$maintClass
$maintClass
Definition: migrateComments.php:296
LoggedUpdateMaintenance
Class for scripts that perform database maintenance and want to log the update in updatelog so we can...
Definition: LoggedUpdateMaintenance.php:26
MigrateComments\updateSkippedMessage
updateSkippedMessage()
Message to show that the update was done already and was just skipped.
Definition: migrateComments.php:46
DB_PRIMARY
const DB_PRIMARY
Definition: defines.php:27
MigrateComments\getUpdateKey
getUpdateKey()
Get the update key name to go in the update log table.
Definition: migrateComments.php:42
Maintenance\commitTransaction
commitTransaction(IDatabase $dbw, $fname)
Commit the transaction on a DB handle and wait for replica DBs to catch up.
Definition: Maintenance.php:1408
Maintenance\getDB
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
Definition: Maintenance.php:1362
Wikimedia\Rdbms\IDatabase\insert
insert( $table, $rows, $fname=__METHOD__, $options=[])
Insert the given row(s) into a table.
CommentStore\hash
static hash( $text, $data)
Hashing function for comment storage.
Definition: CommentStore.php:647
Maintenance\getBatchSize
getBatchSize()
Returns batch size.
Definition: Maintenance.php:368
Wikimedia\Rdbms\IDatabase\select
select( $table, $vars, $conds='', $fname=__METHOD__, $options=[], $join_conds=[])
Execute a SELECT query constructed using the various parameters provided.
Maintenance\output
output( $out, $channel=null)
Throw some output to the user.
Definition: Maintenance.php:435
MigrateComments\migrateToTemp
migrateToTemp( $table, $primaryKey, $oldField, $newPrimaryKey, $newField)
Migrate comments in a table to a temporary table.
Definition: migrateComments.php:230
Wikimedia\Rdbms\IDatabase\makeList
makeList(array $a, $mode=self::LIST_COMMA)
Makes an encoded list of strings from an array.
MigrateComments\__construct
__construct()
Default constructor.
Definition: migrateComments.php:36
MigrateComments\loadCommentIDs
loadCommentIDs(IDatabase $dbw, array &$comments)
Fetch comment IDs for a set of comments.
Definition: migrateComments.php:72
Maintenance\setBatchSize
setBatchSize( $s=0)
Definition: Maintenance.php:375