57 parent::__construct();
59 $this->
addOption(
'type',
'Set compression type to either: gzip|concat',
false,
true,
't' );
62 'Maximum number of revisions in a concat chunk',
69 'Earliest date to check for uncompressed revisions',
74 $this->
addOption(
'end-date',
'Latest revision date to compress',
false,
true,
'e' );
77 'The id to start from (gzip -> text table, concat -> page table)',
84 'Store specified revisions in an external cluster (untested)',
90 'The page_id to stop at (only when using concat compression type)',
99 if ( !function_exists(
"gzdeflate" ) ) {
100 $this->
fatalError(
"You must enable zlib support in PHP to compress old revisions!\n" .
101 "Please see https://www.php.net/manual/en/ref.zlib.php\n" );
104 $type = $this->
getOption(
'type',
'concat' );
105 $chunkSize = $this->
getOption(
'chunksize', 20 );
106 $startId = $this->
getOption(
'startid', 0 );
107 $beginDate = $this->
getOption(
'begin-date',
'' );
108 $endDate = $this->
getOption(
'end-date',
'' );
109 $extDB = $this->
getOption(
'extdb',
'' );
110 $endId = $this->
getOption(
'endid',
false );
112 if ( $type !=
'concat' && $type !=
'gzip' ) {
113 $this->
error(
"Type \"{$type}\" not supported" );
116 if ( $extDB !=
'' ) {
117 $this->
output(
"Compressing database {$wgDBname} to external cluster {$extDB}\n"
118 . str_repeat(
'-', 76 ) .
"\n\n" );
120 $this->
output(
"Compressing database {$wgDBname}\n"
121 . str_repeat(
'-', 76 ) .
"\n\n" );
125 if ( $type ==
'concat' ) {
126 $success = $this->compressWithConcat( $startId, $chunkSize, $beginDate,
127 $endDate, $extDB, $endId );
129 $this->compressOldPages( $startId, $extDB );
133 $this->
output(
"Done.\n" );
143 private function compressOldPages( $start = 0, $extdb =
'' ) {
145 $this->
output(
"Starting from old_id $start...\n" );
148 $res = $dbw->newSelectQueryBuilder()
149 ->select( [
'old_id',
'old_flags',
'old_text' ] )
152 ->where(
"old_id>=$start" )
153 ->orderBy(
'old_id' )
154 ->limit( $chunksize )
155 ->caller( __METHOD__ )->fetchResultSet();
157 if ( $res->numRows() == 0 ) {
163 foreach ( $res as $row ) {
164 # print " {$row->old_id} - {$row->old_namespace}:{$row->old_title}\n";
165 $this->compressPage( $row, $extdb );
166 $last = $row->old_id;
169 $start = $last + 1; # Deletion may leave
long empty stretches
170 $this->
output(
"$start...\n" );
181 private function compressPage( $row, $extdb ) {
182 if ( strpos( $row->old_flags,
'gzip' ) !==
false
183 || strpos( $row->old_flags,
'object' ) !==
false
185 # print "Already compressed row {$row->old_id}\n";
189 $flags = $row->old_flags ?
"{$row->old_flags},gzip" :
"gzip";
190 $compress = gzdeflate( $row->old_text );
192 # Store in external storage if required
193 if ( $extdb !==
'' ) {
196 $storeObj = $esFactory->getStore(
'DB' );
197 $compress = $storeObj->store( $extdb, $compress );
198 if ( $compress ===
false ) {
199 $this->
error(
"Unable to store object" );
206 $dbw->newUpdateQueryBuilder()
209 'old_flags' => $flags,
210 'old_text' => $compress
213 'old_id' => $row->old_id
215 ->caller( __METHOD__ )
232 private function compressWithConcat( $startId, $maxChunkSize, $beginDate,
233 $endDate, $extdb =
"", $maxPageId =
false
238 # Set up external storage
239 if ( $extdb !=
'' ) {
242 $storeObj = $esFactory->getStore(
'DB' );
246 ->getBlobStoreFactory()
249 # Get all articles by page_id
251 $maxPageId = $dbr->newSelectQueryBuilder()
252 ->select(
'max(page_id)' )
254 ->caller( __METHOD__ )->fetchField();
256 $this->
output(
"Starting from $startId of $maxPageId\n" );
269 # For each article, get a list of revisions which fit the criteria
271 # No recompression, use a condition on old_flags
272 # Don't compress object type entities, because that might produce data loss when
273 # overwriting bulk storage concat rows. Don't compress external references, because
274 # the script doesn't yet delete rows from external storage.
276 $queryBuilderTemplate = $dbw->newSelectQueryBuilder()
277 ->select( [
'rev_id',
'old_id',
'old_flags',
'old_text' ] )
280 ->join(
'slots',
null,
'rev_id=slot_revision_id' )
281 ->join(
'content',
null,
'content_id=slot_content_id' )
282 ->join(
'text',
null,
'SUBSTRING(content_address, 4)=old_id' )
286 IExpression::NOT_LIKE,
287 new LikeValue( $dbr->anyString(),
'object', $dbr->anyString() )
290 IExpression::NOT_LIKE,
291 new LikeValue( $dbr->anyString(),
'external', $dbr->anyString() )
295 'slot_role_id' => $slotRoleStore->getId( SlotRecord::MAIN ),
296 'SUBSTRING(content_address, 1, 3)=' . $dbr->addQuotes(
'tt:' ),
300 if ( !preg_match(
'/^\d{14}$/', $beginDate ) ) {
301 $this->
error(
"Invalid begin date \"$beginDate\"\n" );
305 $queryBuilderTemplate->andWhere(
"rev_timestamp>'" . $beginDate .
"'" );
308 if ( !preg_match(
'/^\d{14}$/', $endDate ) ) {
309 $this->
error(
"Invalid end date \"$endDate\"\n" );
313 $queryBuilderTemplate->andWhere(
"rev_timestamp<'" . $endDate .
"'" );
316 for ( $pageId = $startId; $pageId <= $maxPageId; $pageId++ ) {
323 $pageRow = $dbr->newSelectQueryBuilder()
324 ->select( [
'page_id',
'page_namespace',
'page_title',
'rev_timestamp' ] )
326 ->straightJoin(
'revision',
null,
'page_latest = rev_id' )
327 ->where( $pageConds )
328 ->andWhere( [
'page_id' => $pageId ] )
329 ->caller( __METHOD__ )->fetchRow();
330 if ( $pageRow ===
false ) {
335 $titleObj = Title::makeTitle( $pageRow->page_namespace, $pageRow->page_title );
336 $this->
output(
"$pageId\t" . $titleObj->getPrefixedDBkey() .
" " );
339 $queryBuilder = clone $queryBuilderTemplate;
340 $revRes = $queryBuilder->where(
342 'rev_page' => $pageRow->page_id,
346 'rev_timestamp < ' . (
int)$pageRow->rev_timestamp
348 ->caller( __METHOD__ )->fetchResultSet();
351 foreach ( $revRes as $revRow ) {
355 if ( count( $revs ) < 2 ) {
356 # No revisions matching, no further processing
363 while ( $i < count( $revs ) ) {
364 if ( $i < count( $revs ) - $maxChunkSize ) {
365 $thisChunkSize = $maxChunkSize;
367 $thisChunkSize = count( $revs ) - $i;
374 $primaryOldid = $revs[$i]->old_id;
376 # Get the text of each revision and add it to the object
377 for ( $j = 0; $j < $thisChunkSize && $chunk->isHappy(); $j++ ) {
378 $oldid = $revs[$i + $j]->old_id;
380 # Get text. We do not need the full `extractBlob` since the query is built
381 # to fetch non-externalstore blobs.
382 $text = $blobStore->decompressData(
383 $revs[$i + $j]->old_text,
384 explode(
',', $revs[$i + $j]->old_flags )
387 if ( $text ===
false ) {
388 $this->
error(
"\nError, unable to get text in old_id $oldid" );
389 # $dbw->delete( 'old', [ 'old_id' => $oldid ] );
392 if ( $extdb ==
"" && $j == 0 ) {
393 $chunk->setText( $text );
396 # Don't make a stub if it's going to be longer than the article
397 # Stubs are typically about 100 bytes
398 if ( strlen( $text ) < 120 ) {
403 $stub->setLocation( $primaryOldid );
404 $stub->setReferrer( $oldid );
413 # If we couldn't actually use any stubs because the pages were too small, do nothing
415 if ( $extdb !=
"" ) {
416 # Move blob objects to External Storage
418 $stored = $storeObj->store( $extdb, serialize( $chunk ) );
419 if ( $stored ===
false ) {
420 $this->
error(
"Unable to store object" );
424 # Store External Storage URLs instead of Stub placeholders
425 foreach ( $stubs as $stub ) {
426 if ( $stub ===
false ) {
429 # $stored should provide base path to a BLOB
430 $url = $stored .
"/" . $stub->getHash();
431 $dbw->newUpdateQueryBuilder()
435 'old_flags' =>
'external,utf-8',
438 'old_id' => $stub->getReferrer(),
440 ->caller( __METHOD__ )
444 # Store the main object locally
445 $dbw->newUpdateQueryBuilder()
448 'old_text' => serialize( $chunk ),
449 'old_flags' =>
'object,utf-8',
452 'old_id' => $primaryOldid
454 ->caller( __METHOD__ )
457 # Store the stub objects
458 for ( $j = 1; $j < $thisChunkSize; $j++ ) {
459 # Skip if not compressing and don't overwrite the first revision
460 if ( $stubs[$j] !==
false && $revs[$i + $j]->old_id != $primaryOldid ) {
461 $dbw->newUpdateQueryBuilder()
464 'old_text' => serialize( $stubs[$j] ),
465 'old_flags' =>
'object,utf-8',
468 'old_id' => $revs[$i + $j]->old_id
470 ->caller( __METHOD__ )
479 $i += $thisChunkSize;
Concatenated gzip (CGZ) storage Improves compression ratio by concatenating like objects before gzipp...
Pointer object for an item within a CGZ blob stored in the text table.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
error( $err, $die=0)
Throw an error to the user.
beginTransaction(IDatabase $dbw, $fname)
Begin a transaction on a DB.
commitTransaction(IDatabase $dbw, $fname)
Commit the transaction on a DB handle and wait for replica DBs to catch up.
output( $out, $channel=null)
Throw some output to the user.
waitForReplication()
Wait for replica DBs to catch up.
getServiceContainer()
Returns the main service container.
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.