MediaWiki  1.30.0
fixT22757.php
Go to the documentation of this file.
1 <?php
24 require_once __DIR__ . '/../Maintenance.php';
25 
31 class FixT22757 extends Maintenance {
32  public $batchSize = 10000;
33  public $mapCache = [];
34  public $mapCacheSize = 0;
35  public $maxMapCacheSize = 1000000;
36 
37  function __construct() {
38  parent::__construct();
39  $this->addDescription( 'Script to fix T22757 assuming that blob_tracking is intact' );
40  $this->addOption( 'dry-run', 'Report only' );
41  $this->addOption( 'start', 'old_id to start at', false, true );
42  }
43 
44  function execute() {
45  $dbr = $this->getDB( DB_REPLICA );
46  $dbw = $this->getDB( DB_MASTER );
47 
48  $dryRun = $this->getOption( 'dry-run' );
49  if ( $dryRun ) {
50  print "Dry run only.\n";
51  }
52 
53  $startId = $this->getOption( 'start', 0 );
54  $numGood = 0;
55  $numFixed = 0;
56  $numBad = 0;
57 
58  $totalRevs = $dbr->selectField( 'text', 'MAX(old_id)', false, __METHOD__ );
59 
60  // In MySQL 4.1+, the binary field old_text has a non-working LOWER() function
61  $lowerLeft = 'LOWER(CONVERT(LEFT(old_text,22) USING latin1))';
62 
63  while ( true ) {
64  print "ID: $startId / $totalRevs\r";
65 
66  $res = $dbr->select(
67  'text',
68  [ 'old_id', 'old_flags', 'old_text' ],
69  [
70  'old_id > ' . intval( $startId ),
71  'old_flags LIKE \'%object%\' AND old_flags NOT LIKE \'%external%\'',
72  "$lowerLeft = 'o:15:\"historyblobstub\"'",
73  ],
74  __METHOD__,
75  [
76  'ORDER BY' => 'old_id',
77  'LIMIT' => $this->batchSize,
78  ]
79  );
80 
81  if ( !$res->numRows() ) {
82  break;
83  }
84 
85  $secondaryIds = [];
86  $stubs = [];
87 
88  foreach ( $res as $row ) {
89  $startId = $row->old_id;
90 
91  // Basic sanity checks
92  $obj = unserialize( $row->old_text );
93  if ( $obj === false ) {
94  print "{$row->old_id}: unrecoverable: cannot unserialize\n";
95  ++$numBad;
96  continue;
97  }
98 
99  if ( !is_object( $obj ) ) {
100  print "{$row->old_id}: unrecoverable: unserialized to type " .
101  gettype( $obj ) . ", possible double-serialization\n";
102  ++$numBad;
103  continue;
104  }
105 
106  if ( strtolower( get_class( $obj ) ) !== 'historyblobstub' ) {
107  print "{$row->old_id}: unrecoverable: unexpected object class " .
108  get_class( $obj ) . "\n";
109  ++$numBad;
110  continue;
111  }
112 
113  // Process flags
114  $flags = explode( ',', $row->old_flags );
115  if ( in_array( 'utf-8', $flags ) || in_array( 'utf8', $flags ) ) {
116  $legacyEncoding = false;
117  } else {
118  $legacyEncoding = true;
119  }
120 
121  // Queue the stub for future batch processing
122  $id = intval( $obj->mOldId );
123  $secondaryIds[] = $id;
124  $stubs[$row->old_id] = [
125  'legacyEncoding' => $legacyEncoding,
126  'secondaryId' => $id,
127  'hash' => $obj->mHash,
128  ];
129  }
130 
131  $secondaryIds = array_unique( $secondaryIds );
132 
133  if ( !count( $secondaryIds ) ) {
134  continue;
135  }
136 
137  // Run the batch query on blob_tracking
138  $res = $dbr->select(
139  'blob_tracking',
140  '*',
141  [
142  'bt_text_id' => $secondaryIds,
143  ],
144  __METHOD__
145  );
146  $trackedBlobs = [];
147  foreach ( $res as $row ) {
148  $trackedBlobs[$row->bt_text_id] = $row;
149  }
150 
151  // Process the stubs
152  foreach ( $stubs as $primaryId => $stub ) {
153  $secondaryId = $stub['secondaryId'];
154  if ( !isset( $trackedBlobs[$secondaryId] ) ) {
155  // No tracked blob. Work out what went wrong
156  $secondaryRow = $dbr->selectRow(
157  'text',
158  [ 'old_flags', 'old_text' ],
159  [ 'old_id' => $secondaryId ],
160  __METHOD__
161  );
162  if ( !$secondaryRow ) {
163  print "$primaryId: unrecoverable: secondary row is missing\n";
164  ++$numBad;
165  } elseif ( $this->isUnbrokenStub( $stub, $secondaryRow ) ) {
166  // Not broken yet, and not in the tracked clusters so it won't get
167  // broken by the current RCT run.
168  ++$numGood;
169  } elseif ( strpos( $secondaryRow->old_flags, 'external' ) !== false ) {
170  print "$primaryId: unrecoverable: secondary gone to {$secondaryRow->old_text}\n";
171  ++$numBad;
172  } else {
173  print "$primaryId: unrecoverable: miscellaneous corruption of secondary row\n";
174  ++$numBad;
175  }
176  unset( $stubs[$primaryId] );
177  continue;
178  }
179  $trackRow = $trackedBlobs[$secondaryId];
180 
181  // Check that the specified text really is available in the tracked source row
182  $url = "DB://{$trackRow->bt_cluster}/{$trackRow->bt_blob_id}/{$stub['hash']}";
183  $text = ExternalStore::fetchFromURL( $url );
184  if ( $text === false ) {
185  print "$primaryId: unrecoverable: source text missing\n";
186  ++$numBad;
187  unset( $stubs[$primaryId] );
188  continue;
189  }
190  if ( md5( $text ) !== $stub['hash'] ) {
191  print "$primaryId: unrecoverable: content hashes do not match\n";
192  ++$numBad;
193  unset( $stubs[$primaryId] );
194  continue;
195  }
196 
197  // Find the page_id and rev_id
198  // The page is probably the same as the page of the secondary row
199  $pageId = intval( $trackRow->bt_page );
200  if ( !$pageId ) {
201  $revId = $pageId = 0;
202  } else {
203  $revId = $this->findTextIdInPage( $pageId, $primaryId );
204  if ( !$revId ) {
205  // Actually an orphan
206  $pageId = $revId = 0;
207  }
208  }
209 
210  $newFlags = $stub['legacyEncoding'] ? 'external' : 'external,utf-8';
211 
212  if ( !$dryRun ) {
213  // Reset the text row to point to the original copy
214  $this->beginTransaction( $dbw, __METHOD__ );
215  $dbw->update(
216  'text',
217  // SET
218  [
219  'old_flags' => $newFlags,
220  'old_text' => $url
221  ],
222  // WHERE
223  [ 'old_id' => $primaryId ],
224  __METHOD__
225  );
226 
227  // Add a blob_tracking row so that the new reference can be recompressed
228  // without needing to run trackBlobs.php again
229  $dbw->insert( 'blob_tracking',
230  [
231  'bt_page' => $pageId,
232  'bt_rev_id' => $revId,
233  'bt_text_id' => $primaryId,
234  'bt_cluster' => $trackRow->bt_cluster,
235  'bt_blob_id' => $trackRow->bt_blob_id,
236  'bt_cgz_hash' => $stub['hash'],
237  'bt_new_url' => null,
238  'bt_moved' => 0,
239  ],
240  __METHOD__
241  );
242  $this->commitTransaction( $dbw, __METHOD__ );
243  $this->waitForSlaves();
244  }
245 
246  print "$primaryId: resolved to $url\n";
247  ++$numFixed;
248  }
249  }
250 
251  print "\n";
252  print "Fixed: $numFixed\n";
253  print "Unrecoverable: $numBad\n";
254  print "Good stubs: $numGood\n";
255  }
256 
257  function waitForSlaves() {
258  static $iteration = 0;
259  ++$iteration;
260  if ( ++$iteration > 50 == 0 ) {
261  wfWaitForSlaves();
262  $iteration = 0;
263  }
264  }
265 
266  function findTextIdInPage( $pageId, $textId ) {
267  $ids = $this->getRevTextMap( $pageId );
268  if ( !isset( $ids[$textId] ) ) {
269  return null;
270  } else {
271  return $ids[$textId];
272  }
273  }
274 
275  function getRevTextMap( $pageId ) {
276  if ( !isset( $this->mapCache[$pageId] ) ) {
277  // Limit cache size
278  while ( $this->mapCacheSize > $this->maxMapCacheSize ) {
279  $key = key( $this->mapCache );
280  $this->mapCacheSize -= count( $this->mapCache[$key] );
281  unset( $this->mapCache[$key] );
282  }
283 
284  $dbr = $this->getDB( DB_REPLICA );
285  $map = [];
286  $res = $dbr->select( 'revision',
287  [ 'rev_id', 'rev_text_id' ],
288  [ 'rev_page' => $pageId ],
289  __METHOD__
290  );
291  foreach ( $res as $row ) {
292  $map[$row->rev_text_id] = $row->rev_id;
293  }
294  $this->mapCache[$pageId] = $map;
295  $this->mapCacheSize += count( $map );
296  }
297 
298  return $this->mapCache[$pageId];
299  }
300 
308  function isUnbrokenStub( $stub, $secondaryRow ) {
309  $flags = explode( ',', $secondaryRow->old_flags );
310  $text = $secondaryRow->old_text;
311  if ( in_array( 'external', $flags ) ) {
312  $url = $text;
313  MediaWiki\suppressWarnings();
314  list( /* $proto */, $path ) = explode( '://', $url, 2 );
315  MediaWiki\restoreWarnings();
316 
317  if ( $path == "" ) {
318  return false;
319  }
320  $text = ExternalStore::fetchFromURL( $url );
321  }
322  if ( !in_array( 'object', $flags ) ) {
323  return false;
324  }
325 
326  if ( in_array( 'gzip', $flags ) ) {
327  $obj = unserialize( gzinflate( $text ) );
328  } else {
329  $obj = unserialize( $text );
330  }
331 
332  if ( !is_object( $obj ) ) {
333  // Correct for old double-serialization bug.
334  $obj = unserialize( $obj );
335  }
336 
337  if ( !is_object( $obj ) ) {
338  return false;
339  }
340 
341  $obj->uncompress();
342  $text = $obj->getItem( $stub['hash'] );
343 
344  return $text !== false;
345  }
346 }
347 
348 $maintClass = 'FixT22757';
349 require_once RUN_MAINTENANCE_IF_MAIN;
FixT22757\findTextIdInPage
findTextIdInPage( $pageId, $textId)
Definition: fixT22757.php:266
FixT22757
Maintenance script to fix T22757.
Definition: fixT22757.php:31
FixT22757\waitForSlaves
waitForSlaves()
Definition: fixT22757.php:257
captcha-old.count
count
Definition: captcha-old.py:249
FixT22757\execute
execute()
Do the actual work.
Definition: fixT22757.php:44
FixT22757\$mapCache
$mapCache
Definition: fixT22757.php:33
Maintenance\addDescription
addDescription( $text)
Set the description text.
Definition: Maintenance.php:287
ExternalStore\fetchFromURL
static fetchFromURL( $url, array $params=[])
Fetch data from given URL.
Definition: ExternalStore.php:75
unserialize
unserialize( $serialized)
Definition: ApiMessage.php:185
RUN_MAINTENANCE_IF_MAIN
require_once RUN_MAINTENANCE_IF_MAIN
Definition: maintenance.txt:50
$maintClass
$maintClass
Definition: fixT22757.php:348
$res
$res
Definition: database.txt:21
Maintenance
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
Definition: maintenance.txt:39
wfWaitForSlaves
wfWaitForSlaves( $ifWritesSince=null, $wiki=false, $cluster=false, $timeout=null)
Waits for the replica DBs to catch up to the master position.
Definition: GlobalFunctions.php:3010
php
injection txt This is an overview of how MediaWiki makes use of dependency injection The design described here grew from the discussion of RFC T384 The term dependency this means that anything an object needs to operate should be injected from the the object itself should only know narrow no concrete implementation of the logic it relies on The requirement to inject everything typically results in an architecture that based on two main types of and essentially stateless service objects that use other service objects to operate on the value objects As of the beginning MediaWiki is only starting to use the DI approach Much of the code still relies on global state or direct resulting in a highly cyclical dependency which acts as the top level factory for services in MediaWiki which can be used to gain access to default instances of various services MediaWikiServices however also allows new services to be defined and default services to be redefined Services are defined or redefined by providing a callback the instantiator that will return a new instance of the service When it will create an instance of MediaWikiServices and populate it with the services defined in the files listed by thereby bootstrapping the DI framework Per $wgServiceWiringFiles lists includes ServiceWiring php
Definition: injection.txt:35
Maintenance\beginTransaction
beginTransaction(IDatabase $dbw, $fname)
Begin a transcation on a DB.
Definition: Maintenance.php:1278
key
design txt This is a brief overview of the new design More thorough and up to date information is available on the documentation wiki at etc Handles the details of getting and saving to the user table of the and dealing with sessions and cookies OutputPage Encapsulates the entire HTML page that will be sent in response to any server request It is used by calling its functions to add in any and then calling but I prefer the flexibility This should also do the output encoding The system allocates a global one in $wgOut Title Represents the title of an and does all the work of translating among various forms such as plain database key
Definition: design.txt:25
FixT22757\getRevTextMap
getRevTextMap( $pageId)
Definition: fixT22757.php:275
Maintenance\addOption
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
Definition: Maintenance.php:215
DB_REPLICA
const DB_REPLICA
Definition: defines.php:25
DB_MASTER
const DB_MASTER
Definition: defines.php:26
list
deferred txt A few of the database updates required by various functions here can be deferred until after the result page is displayed to the user For updating the view updating the linked to tables after a etc PHP does not yet have any way to tell the server to actually return and disconnect while still running these but it might have such a feature in the future We handle these by creating a deferred update object and putting those objects on a global list
Definition: deferred.txt:11
FixT22757\$maxMapCacheSize
$maxMapCacheSize
Definition: fixT22757.php:35
Maintenance\commitTransaction
commitTransaction(IDatabase $dbw, $fname)
Commit the transcation on a DB handle and wait for replica DBs to catch up.
Definition: Maintenance.php:1293
FixT22757\__construct
__construct()
Default constructor.
Definition: fixT22757.php:37
FixT22757\$mapCacheSize
$mapCacheSize
Definition: fixT22757.php:34
Maintenance\getOption
getOption( $name, $default=null)
Get an option, or return the default.
Definition: Maintenance.php:250
$dbr
if(! $regexes) $dbr
Definition: cleanup.php:94
$path
$path
Definition: NoLocalSettings.php:26
as
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
Maintenance\getDB
getDB( $db, $groups=[], $wiki=false)
Returns a database to be used by current maintenance script.
Definition: Maintenance.php:1251
FixT22757\isUnbrokenStub
isUnbrokenStub( $stub, $secondaryRow)
This is based on part of HistoryBlobStub::getText().
Definition: fixT22757.php:308
FixT22757\$batchSize
$batchSize
Definition: fixT22757.php:32
$flags
it s the revision text itself In either if gzip is the revision text is gzipped $flags
Definition: hooks.txt:2801