MediaWiki  master
deduplicateArchiveRevId.php
Go to the documentation of this file.
1 <?php
2 
4 
5 require_once __DIR__ . '/Maintenance.php';
6 
15 
16  private $deleted = 0;
17  private $reassigned = 0;
18 
19  public function __construct() {
20  parent::__construct();
21  $this->addDescription(
22  'Clean up duplicate ar_rev_id, both within archive and between archive and revision.'
23  );
24  $this->setBatchSize( 10000 );
25  }
26 
27  protected function getUpdateKey() {
28  return __CLASS__;
29  }
30 
31  protected function doDBUpdates() {
32  $this->output( "Deduplicating ar_rev_id...\n" );
33  $dbw = $this->getDB( DB_PRIMARY );
34  // If this is a new install, we don't need to do anything here.
35  if ( PopulateArchiveRevId::isNewInstall( $dbw ) ) {
36  $this->output( "New install, nothing to do here.\n" );
37  return true;
38  }
39 
41 
42  $minId = $dbw->newSelectQueryBuilder()
43  ->select( 'MIN(ar_rev_id)' )
44  ->from( 'archive' )
45  ->options( [] )
46  ->caller( __METHOD__ )
47  ->fetchField();
48  $maxId = $dbw->newSelectQueryBuilder()
49  ->select( 'MAX(ar_rev_id)' )
50  ->from( 'archive' )
51  ->options( [] )
52  ->caller( __METHOD__ )
53  ->fetchField();
54  $batchSize = $this->getBatchSize();
55 
56  $revActorQuery = ActorMigration::newMigration()->getJoin( 'rev_user' );
57 
58  for ( $id = $minId; $id <= $maxId; $id += $batchSize ) {
59  $endId = min( $maxId, $id + $batchSize - 1 );
60 
61  $this->beginTransaction( $dbw, __METHOD__ );
62 
63  // Lock the archive and revision table rows for the IDs we're checking
64  // to try to prevent deletions or undeletions from confusing things.
65  $dbw->newSelectQueryBuilder()
66  ->select( '1' )
67  ->from( 'archive' )
68  ->where( [ 'ar_rev_id >= ' . (int)$id, 'ar_rev_id <= ' . (int)$endId ] )
69  ->caller( __METHOD__ )
70  ->forUpdate()
71  ->fetchRowCount();
72  $dbw->newSelectQueryBuilder()
73  ->select( '1' )
74  ->from( 'archive' )
75  ->where( [ 'ar_rev_id >= ' . (int)$id, 'ar_rev_id <= ' . (int)$endId ] )
76  ->caller( __METHOD__ )
77  ->lockInShareMode()
78  ->fetchRowCount();
79 
80  // Figure out the ar_rev_ids we actually need to look at
81  $res = $dbw->newSelectQueryBuilder()
82  ->select( [ 'rev_id', 'rev_timestamp', 'rev_sha1' ] + $revActorQuery['fields'] )
83  ->tables( [ 'archive', 'revision' ] + $revActorQuery['tables'] )
84  ->where( [ 'ar_rev_id >= ' . (int)$id, 'ar_rev_id <= ' . (int)$endId ] )
85  ->caller( __METHOD__ )
86  ->distinct()
87  ->joinConds( [ 'revision', [ 'ar_rev_id = rev_id' ] ] + $revActorQuery['joins'] )
88  ->fetchResultSet();
89  $revRows = [];
90  foreach ( $res as $row ) {
91  $revRows[$row->rev_id] = $row;
92  }
93 
94  $arRevIds = $dbw->newSelectQueryBuilder()
95  ->select( 'ar_rev_id' )
96  ->from( 'archive' )
97  ->where( [ 'ar_rev_id >= ' . (int)$id, 'ar_rev_id <= ' . (int)$endId ] )
98  ->caller( __METHOD__ )
99  ->groupBy( 'ar_rev_id' )
100  ->having( 'COUNT(*) > 1' )
101  ->fetchFieldValues();
102  $arRevIds = array_values( array_unique( array_merge( $arRevIds, array_keys( $revRows ) ) ) );
103 
104  if ( $arRevIds ) {
105  $this->processArRevIds( $dbw, $arRevIds, $revRows );
106  }
107 
108  $this->output( "... $id-$endId\n" );
109  $this->commitTransaction( $dbw, __METHOD__ );
110  }
111 
112  $this->output(
113  "Finished deduplicating ar_rev_id. $this->deleted rows deleted, "
114  . "$this->reassigned assigned new IDs.\n"
115  );
116  return true;
117  }
118 
125  private function processArRevIds( IDatabase $dbw, array $arRevIds, array $revRows ) {
126  // Select all the data we need for deduplication
127  $res = $dbw->newSelectQueryBuilder()
128  ->select( [ 'ar_id', 'ar_rev_id', 'ar_namespace', 'ar_title', 'ar_actor',
129  'ar_timestamp', 'ar_sha1' ] )
130  ->from( 'archive' )
131  ->where( [ 'ar_rev_id' => $arRevIds ] )
132  ->caller( __METHOD__ )
133  ->fetchResultSet();
134 
135  // Determine which rows we need to delete or reassign
136  $seen = [];
137  $toDelete = [];
138  $toReassign = [];
139  foreach ( $res as $row ) {
140  // Revision-table row exists?
141  if ( isset( $revRows[$row->ar_rev_id] ) ) {
142  $revRow = $revRows[$row->ar_rev_id];
143 
144  // Record the rev_id as seen, so the code below will always delete or reassign.
145  if ( !isset( $seen[$revRow->rev_id] ) ) {
146  $seen[$revRow->rev_id] = [
147  'first' => "revision row",
148  ];
149  }
150 
151  // Delete the archive row if it seems to be the same regardless
152  // of page, because moves can change IDs and titles.
153  if ( $row->ar_timestamp === $revRow->rev_timestamp &&
154  $row->ar_sha1 === $revRow->rev_sha1 &&
155  $row->ar_actor === $revRow->rev_actor
156  ) {
157  $this->output(
158  "Row $row->ar_id duplicates revision row for rev_id $revRow->rev_id, deleting\n"
159  );
160  $toDelete[] = $row->ar_id;
161  continue;
162  }
163  }
164 
165  $key = $this->getSeenKey( $row );
166  if ( !isset( $seen[$row->ar_rev_id] ) ) {
167  // This rev_id hasn't even been seen yet, nothing to do besides record it.
168  $seen[$row->ar_rev_id] = [
169  'first' => "archive row $row->ar_id",
170  $key => $row->ar_id,
171  ];
172  } elseif ( !isset( $seen[$row->ar_rev_id][$key] ) ) {
173  // The rev_id was seen, but not this particular change. Reassign it.
174  $seen[$row->ar_rev_id][$key] = $row->ar_id;
175  $this->output(
176  "Row $row->ar_id conflicts with {$seen[$row->ar_rev_id]['first']} "
177  . "for rev_id $row->ar_rev_id, reassigning\n"
178  );
179  $toReassign[] = $row->ar_id;
180  } else {
181  // The rev_id was seen with a row that matches this change. Delete it.
182  $this->output(
183  "Row $row->ar_id duplicates archive row {$seen[$row->ar_rev_id][$key]} "
184  . "for rev_id $row->ar_rev_id, deleting\n"
185  );
186  $toDelete[] = $row->ar_id;
187  }
188  }
189 
190  // Perform the updates
191  if ( $toDelete ) {
192  $dbw->delete( 'archive', [ 'ar_id' => $toDelete ], __METHOD__ );
193  $this->deleted += $dbw->affectedRows();
194  }
195  if ( $toReassign ) {
196  $this->reassigned += PopulateArchiveRevId::reassignArRevIds( $dbw, $toReassign );
197  }
198  }
199 
205  private function getSeenKey( $row ) {
206  return implode( "\n", [
207  $row->ar_namespace,
208  $row->ar_title,
209  $row->ar_timestamp,
210  $row->ar_sha1,
211  $row->ar_actor,
212  ] );
213  }
214 
215 }
216 
217 $maintClass = DeduplicateArchiveRevId::class;
218 require_once RUN_MAINTENANCE_IF_MAIN;
static newMigration()
Static constructor.
Maintenance script that cleans up archive rows with duplicated ar_rev_id, both within archive and bet...
__construct()
Default constructor.
getUpdateKey()
Get the update key name to go in the update log table.
doDBUpdates()
Do the actual work.
Class for scripts that perform database maintenance and want to log the update in updatelog so we can...
getDB( $db, $groups=[], $dbDomain=false)
Returns a database to be used by current maintenance script.
beginTransaction(IDatabase $dbw, $fname)
Begin a transaction on a DB.
commitTransaction(IDatabase $dbw, $fname)
Commit the transaction on a DB handle and wait for replica DBs to catch up.
output( $out, $channel=null)
Throw some output to the user.
getBatchSize()
Returns batch size.
addDescription( $text)
Set the description text.
setBatchSize( $s=0)
static checkMysqlAutoIncrementBug(IDatabase $dbw)
Check for (and work around) a MySQL auto-increment bug.
static reassignArRevIds(IDatabase $dbw, array $arIds, array $conds=[])
Assign new ar_rev_ids to a set of ar_ids.
static isNewInstall(IDatabase $dbw)
Basic database interface for live and lazy-loaded relation database handles.
Definition: IDatabase.php:39
affectedRows()
Get the number of rows affected by the last write query.
delete( $table, $conds, $fname=__METHOD__)
Delete all rows in a table that match a condition.
newSelectQueryBuilder()
Create an empty SelectQueryBuilder which can be used to run queries against this connection.
const DB_PRIMARY
Definition: defines.php:28