MediaWiki REL1_39
deduplicateArchiveRevId.php
Go to the documentation of this file.
1<?php
2
4
5require_once __DIR__ . '/Maintenance.php';
6
15
16 private $deleted = 0;
17 private $reassigned = 0;
18
19 public function __construct() {
20 parent::__construct();
21 $this->addDescription(
22 'Clean up duplicate ar_rev_id, both within archive and between archive and revision.'
23 );
24 $this->setBatchSize( 10000 );
25 }
26
27 protected function getUpdateKey() {
28 return __CLASS__;
29 }
30
31 protected function doDBUpdates() {
32 $this->output( "Deduplicating ar_rev_id...\n" );
33 $dbw = $this->getDB( DB_PRIMARY );
34 // If this is a new install, we don't need to do anything here.
36 $this->output( "New install, nothing to do here.\n" );
37 return true;
38 }
39
41
42 $minId = $dbw->newSelectQueryBuilder()
43 ->select( 'MIN(ar_rev_id)' )
44 ->from( 'archive' )
45 ->options( [] )
46 ->caller( __METHOD__ )
47 ->fetchField();
48 $maxId = $dbw->newSelectQueryBuilder()
49 ->select( 'MAX(ar_rev_id)' )
50 ->from( 'archive' )
51 ->options( [] )
52 ->caller( __METHOD__ )
53 ->fetchField();
54 $batchSize = $this->getBatchSize();
55
56 $revActorQuery = ActorMigration::newMigration()->getJoin( 'rev_user' );
57
58 for ( $id = $minId; $id <= $maxId; $id += $batchSize ) {
59 $endId = min( $maxId, $id + $batchSize - 1 );
60
61 $this->beginTransaction( $dbw, __METHOD__ );
62
63 // Lock the archive and revision table rows for the IDs we're checking
64 // to try to prevent deletions or undeletions from confusing things.
66 ->select( '1' )
67 ->from( 'archive' )
68 ->where( [ 'ar_rev_id >= ' . (int)$id, 'ar_rev_id <= ' . (int)$endId ] )
69 ->caller( __METHOD__ )
70 ->forUpdate()
71 ->fetchRowCount();
73 ->select( '1' )
74 ->from( 'archive' )
75 ->where( [ 'ar_rev_id >= ' . (int)$id, 'ar_rev_id <= ' . (int)$endId ] )
76 ->caller( __METHOD__ )
77 ->lockInShareMode()
78 ->fetchRowCount();
79
80 // Figure out the ar_rev_ids we actually need to look at
81 $res = $dbw->select(
82 [ 'archive', 'revision' ] + $revActorQuery['tables'],
83 [ 'rev_id', 'rev_timestamp', 'rev_sha1' ] + $revActorQuery['fields'],
84 [ 'ar_rev_id >= ' . (int)$id, 'ar_rev_id <= ' . (int)$endId ],
85 __METHOD__,
86 [ 'DISTINCT' ],
87 [ 'revision' => [ 'JOIN', 'ar_rev_id = rev_id' ] ] + $revActorQuery['joins']
88 );
89 $revRows = [];
90 foreach ( $res as $row ) {
91 $revRows[$row->rev_id] = $row;
92 }
93
94 $arRevIds = $dbw->newSelectQueryBuilder()
95 ->select( 'ar_rev_id' )
96 ->from( 'archive' )
97 ->where( [ 'ar_rev_id >= ' . (int)$id, 'ar_rev_id <= ' . (int)$endId ] )
98 ->caller( __METHOD__ )
99 ->groupBy( 'ar_rev_id' )
100 ->having( 'COUNT(*) > 1' )
101 ->fetchFieldValues();
102 $arRevIds = array_values( array_unique( array_merge( $arRevIds, array_keys( $revRows ) ) ) );
103
104 if ( $arRevIds ) {
105 $this->processArRevIds( $dbw, $arRevIds, $revRows );
106 }
107
108 $this->output( "... $id-$endId\n" );
109 $this->commitTransaction( $dbw, __METHOD__ );
110 }
111
112 $this->output(
113 "Finished deduplicating ar_rev_id. $this->deleted rows deleted, "
114 . "$this->reassigned assigned new IDs.\n"
115 );
116 return true;
117 }
118
125 private function processArRevIds( IDatabase $dbw, array $arRevIds, array $revRows ) {
126 // Select all the data we need for deduplication
128 ->select( [ 'ar_id', 'ar_rev_id', 'ar_namespace', 'ar_title', 'ar_actor',
129 'ar_timestamp', 'ar_sha1' ] )
130 ->from( 'archive' )
131 ->where( [ 'ar_rev_id' => $arRevIds ] )
132 ->caller( __METHOD__ )
133 ->fetchResultSet();
134
135 // Determine which rows we need to delete or reassign
136 $seen = [];
137 $toDelete = [];
138 $toReassign = [];
139 foreach ( $res as $row ) {
140 // Revision-table row exists?
141 if ( isset( $revRows[$row->ar_rev_id] ) ) {
142 $revRow = $revRows[$row->ar_rev_id];
143
144 // Record the rev_id as seen, so the code below will always delete or reassign.
145 if ( !isset( $seen[$revRow->rev_id] ) ) {
146 $seen[$revRow->rev_id] = [
147 'first' => "revision row",
148 ];
149 }
150
151 // Delete the archive row if it seems to be the same regardless
152 // of page, because moves can change IDs and titles.
153 if ( $row->ar_timestamp === $revRow->rev_timestamp &&
154 $row->ar_sha1 === $revRow->rev_sha1 &&
155 $row->ar_actor === $revRow->rev_actor
156 ) {
157 $this->output(
158 "Row $row->ar_id duplicates revision row for rev_id $revRow->rev_id, deleting\n"
159 );
160 $toDelete[] = $row->ar_id;
161 continue;
162 }
163 }
164
165 $key = $this->getSeenKey( $row );
166 if ( !isset( $seen[$row->ar_rev_id] ) ) {
167 // This rev_id hasn't even been seen yet, nothing to do besides record it.
168 $seen[$row->ar_rev_id] = [
169 'first' => "archive row $row->ar_id",
170 $key => $row->ar_id,
171 ];
172 } elseif ( !isset( $seen[$row->ar_rev_id][$key] ) ) {
173 // The rev_id was seen, but not this particular change. Reassign it.
174 $seen[$row->ar_rev_id][$key] = $row->ar_id;
175 $this->output(
176 "Row $row->ar_id conflicts with {$seen[$row->ar_rev_id]['first']} "
177 . "for rev_id $row->ar_rev_id, reassigning\n"
178 );
179 $toReassign[] = $row->ar_id;
180 } else {
181 // The rev_id was seen with a row that matches this change. Delete it.
182 $this->output(
183 "Row $row->ar_id duplicates archive row {$seen[$row->ar_rev_id][$key]} "
184 . "for rev_id $row->ar_rev_id, deleting\n"
185 );
186 $toDelete[] = $row->ar_id;
187 }
188 }
189
190 // Perform the updates
191 if ( $toDelete ) {
192 $dbw->delete( 'archive', [ 'ar_id' => $toDelete ], __METHOD__ );
193 $this->deleted += $dbw->affectedRows();
194 }
195 if ( $toReassign ) {
196 $this->reassigned += PopulateArchiveRevId::reassignArRevIds( $dbw, $toReassign );
197 }
198 }
199
205 private function getSeenKey( $row ) {
206 return implode( "\n", [
207 $row->ar_namespace,
208 $row->ar_title,
209 $row->ar_timestamp,
210 $row->ar_sha1,
211 $row->ar_actor,
212 ] );
213 }
214
215}
216
217$maintClass = DeduplicateArchiveRevId::class;
218require_once RUN_MAINTENANCE_IF_MAIN;
getDB()
Maintenance script that cleans up archive rows with duplicated ar_rev_id, both within archive and bet...
__construct()
Default constructor.
getUpdateKey()
Get the update key name to go in the update log table.
Class for scripts that perform database maintenance and want to log the update in updatelog so we can...
beginTransaction(IDatabase $dbw, $fname)
Begin a transaction on a DB.
commitTransaction(IDatabase $dbw, $fname)
Commit the transaction on a DB handle and wait for replica DBs to catch up.
output( $out, $channel=null)
Throw some output to the user.
getBatchSize()
Returns batch size.
addDescription( $text)
Set the description text.
setBatchSize( $s=0)
static checkMysqlAutoIncrementBug(IDatabase $dbw)
Check for (and work around) a MySQL auto-increment bug.
static reassignArRevIds(IDatabase $dbw, array $arIds, array $conds=[])
Assign new ar_rev_ids to a set of ar_ids.
static isNewInstall(IDatabase $dbw)
Basic database interface for live and lazy-loaded relation database handles.
Definition IDatabase.php:39
select( $table, $vars, $conds='', $fname=__METHOD__, $options=[], $join_conds=[])
Execute a SELECT query constructed using the various parameters provided.
affectedRows()
Get the number of rows affected by the last write query.
delete( $table, $conds, $fname=__METHOD__)
Delete all rows in a table that match a condition.
newSelectQueryBuilder()
Create an empty SelectQueryBuilder which can be used to run queries against this connection.
const DB_PRIMARY
Definition defines.php:28