57 parent::__construct();
59 $this->
addOption(
'type',
'Set compression type to either: gzip|concat',
false,
true,
't' );
62 'Maximum number of revisions in a concat chunk',
69 'Earliest date to check for uncompressed revisions',
74 $this->
addOption(
'end-date',
'Latest revision date to compress',
false,
true,
'e' );
77 'The id to start from (gzip -> text table, concat -> page table)',
84 'Store specified revisions in an external cluster (untested)',
90 'The page_id to stop at (only when using concat compression type)',
99 if ( !function_exists(
"gzdeflate" ) ) {
100 $this->
fatalError(
"You must enable zlib support in PHP to compress old revisions!\n" .
101 "Please see https://www.php.net/manual/en/ref.zlib.php\n" );
104 $type = $this->
getOption(
'type',
'concat' );
105 $chunkSize = $this->
getOption(
'chunksize', 20 );
106 $startId = $this->
getOption(
'startid', 0 );
107 $beginDate = $this->
getOption(
'begin-date',
'' );
108 $endDate = $this->
getOption(
'end-date',
'' );
109 $extDB = $this->
getOption(
'extdb',
'' );
110 $endId = $this->
getOption(
'endid',
false );
112 if ( $type !=
'concat' && $type !=
'gzip' ) {
113 $this->
error(
"Type \"{$type}\" not supported" );
116 if ( $extDB !=
'' ) {
117 $this->
output(
"Compressing database {$wgDBname} to external cluster {$extDB}\n"
118 . str_repeat(
'-', 76 ) .
"\n\n" );
120 $this->
output(
"Compressing database {$wgDBname}\n"
121 . str_repeat(
'-', 76 ) .
"\n\n" );
125 if ( $type ==
'concat' ) {
126 $success = $this->compressWithConcat( $startId, $chunkSize, $beginDate,
127 $endDate, $extDB, $endId );
129 $this->compressOldPages( $startId, $extDB );
133 $this->
output(
"Done.\n" );
143 private function compressOldPages( $start = 0, $extdb =
'' ) {
145 $this->
output(
"Starting from old_id $start...\n" );
148 $res = $dbw->newSelectQueryBuilder()
149 ->select( [
'old_id',
'old_flags',
'old_text' ] )
152 ->where(
"old_id>=$start" )
153 ->orderBy(
'old_id' )
154 ->limit( $chunksize )
155 ->caller( __METHOD__ )->fetchResultSet();
157 if ( $res->numRows() == 0 ) {
163 foreach ( $res as $row ) {
164 # print " {$row->old_id} - {$row->old_namespace}:{$row->old_title}\n";
165 $this->compressPage( $row, $extdb );
166 $last = $row->old_id;
169 $start = $last + 1; # Deletion may leave
long empty stretches
170 $this->
output(
"$start...\n" );
181 private function compressPage( $row, $extdb ) {
182 if ( strpos( $row->old_flags,
'gzip' ) !==
false
183 || strpos( $row->old_flags,
'object' ) !==
false
185 # print "Already compressed row {$row->old_id}\n";
189 $flags = $row->old_flags ?
"{$row->old_flags},gzip" :
"gzip";
190 $compress = gzdeflate( $row->old_text );
192 # Store in external storage if required
193 if ( $extdb !==
'' ) {
196 $storeObj = $esFactory->getStore(
'DB' );
197 $compress = $storeObj->store( $extdb, $compress );
198 if ( $compress ===
false ) {
199 $this->
error(
"Unable to store object" );
206 $dbw->update(
'text',
208 'old_flags' => $flags,
209 'old_text' => $compress
211 'old_id' => $row->old_id
230 private function compressWithConcat( $startId, $maxChunkSize, $beginDate,
231 $endDate, $extdb =
"", $maxPageId =
false
236 # Set up external storage
237 if ( $extdb !=
'' ) {
240 $storeObj = $esFactory->getStore(
'DB' );
244 ->getBlobStoreFactory()
247 # Get all articles by page_id
249 $maxPageId = $dbr->newSelectQueryBuilder()
250 ->select(
'max(page_id)' )
252 ->caller( __METHOD__ )->fetchField();
254 $this->
output(
"Starting from $startId of $maxPageId\n" );
267 # For each article, get a list of revisions which fit the criteria
269 # No recompression, use a condition on old_flags
270 # Don't compress object type entities, because that might produce data loss when
271 # overwriting bulk storage concat rows. Don't compress external references, because
272 # the script doesn't yet delete rows from external storage.
274 $queryBuilderTemplate = $dbw->newSelectQueryBuilder()
275 ->select( [
'rev_id',
'old_id',
'old_flags',
'old_text' ] )
278 ->join(
'slots',
null,
'rev_id=slot_revision_id' )
279 ->join(
'content',
null,
'content_id=slot_content_id' )
280 ->join(
'text',
null,
'SUBSTRING(content_address, 4)=old_id' )
284 IExpression::NOT_LIKE,
285 new LikeValue( $dbr->anyString(),
'object', $dbr->anyString() )
288 IExpression::NOT_LIKE,
289 new LikeValue( $dbr->anyString(),
'external', $dbr->anyString() )
293 'slot_role_id' => $slotRoleStore->getId( SlotRecord::MAIN ),
294 'SUBSTRING(content_address, 1, 3)=' . $dbr->addQuotes(
'tt:' ),
298 if ( !preg_match(
'/^\d{14}$/', $beginDate ) ) {
299 $this->
error(
"Invalid begin date \"$beginDate\"\n" );
303 $queryBuilderTemplate->andWhere(
"rev_timestamp>'" . $beginDate .
"'" );
306 if ( !preg_match(
'/^\d{14}$/', $endDate ) ) {
307 $this->
error(
"Invalid end date \"$endDate\"\n" );
311 $queryBuilderTemplate->andWhere(
"rev_timestamp<'" . $endDate .
"'" );
314 for ( $pageId = $startId; $pageId <= $maxPageId; $pageId++ ) {
321 $pageRow = $dbr->newSelectQueryBuilder()
322 ->select( [
'page_id',
'page_namespace',
'page_title',
'rev_timestamp' ] )
324 ->straightJoin(
'revision',
null,
'page_latest = rev_id' )
325 ->where( $pageConds )
326 ->andWhere( [
'page_id' => $pageId ] )
327 ->caller( __METHOD__ )->fetchRow();
328 if ( $pageRow ===
false ) {
333 $titleObj = Title::makeTitle( $pageRow->page_namespace, $pageRow->page_title );
334 $this->
output(
"$pageId\t" . $titleObj->getPrefixedDBkey() .
" " );
337 $queryBuilder = clone $queryBuilderTemplate;
338 $revRes = $queryBuilder->where(
340 'rev_page' => $pageRow->page_id,
344 'rev_timestamp < ' . (
int)$pageRow->rev_timestamp
346 ->caller( __METHOD__ )->fetchResultSet();
349 foreach ( $revRes as $revRow ) {
353 if ( count( $revs ) < 2 ) {
354 # No revisions matching, no further processing
361 while ( $i < count( $revs ) ) {
362 if ( $i < count( $revs ) - $maxChunkSize ) {
363 $thisChunkSize = $maxChunkSize;
365 $thisChunkSize = count( $revs ) - $i;
372 $primaryOldid = $revs[$i]->old_id;
374 # Get the text of each revision and add it to the object
375 for ( $j = 0; $j < $thisChunkSize && $chunk->isHappy(); $j++ ) {
376 $oldid = $revs[$i + $j]->old_id;
378 # Get text. We do not need the full `extractBlob` since the query is built
379 # to fetch non-externalstore blobs.
380 $text = $blobStore->decompressData(
381 $revs[$i + $j]->old_text,
382 explode(
',', $revs[$i + $j]->old_flags )
385 if ( $text ===
false ) {
386 $this->
error(
"\nError, unable to get text in old_id $oldid" );
387 # $dbw->delete( 'old', [ 'old_id' => $oldid ] );
390 if ( $extdb ==
"" && $j == 0 ) {
391 $chunk->setText( $text );
394 # Don't make a stub if it's going to be longer than the article
395 # Stubs are typically about 100 bytes
396 if ( strlen( $text ) < 120 ) {
401 $stub->setLocation( $primaryOldid );
402 $stub->setReferrer( $oldid );
411 # If we couldn't actually use any stubs because the pages were too small, do nothing
413 if ( $extdb !=
"" ) {
414 # Move blob objects to External Storage
416 $stored = $storeObj->store( $extdb, serialize( $chunk ) );
417 if ( $stored ===
false ) {
418 $this->
error(
"Unable to store object" );
422 # Store External Storage URLs instead of Stub placeholders
423 foreach ( $stubs as $stub ) {
424 if ( $stub ===
false ) {
427 # $stored should provide base path to a BLOB
428 $url = $stored .
"/" . $stub->getHash();
429 $dbw->update(
'text',
432 'old_flags' =>
'external,utf-8',
434 'old_id' => $stub->getReferrer(),
440 # Store the main object locally
441 $dbw->update(
'text',
443 'old_text' => serialize( $chunk ),
444 'old_flags' =>
'object,utf-8',
446 'old_id' => $primaryOldid
451 # Store the stub objects
452 for ( $j = 1; $j < $thisChunkSize; $j++ ) {
453 # Skip if not compressing and don't overwrite the first revision
454 if ( $stubs[$j] !==
false && $revs[$i + $j]->old_id != $primaryOldid ) {
455 $dbw->update(
'text',
457 'old_text' => serialize( $stubs[$j] ),
458 'old_flags' =>
'object,utf-8',
460 'old_id' => $revs[$i + $j]->old_id
471 $i += $thisChunkSize;
Concatenated gzip (CGZ) storage Improves compression ratio by concatenating like objects before gzipp...
Pointer object for an item within a CGZ blob stored in the text table.
Abstract maintenance class for quickly writing and churning out maintenance scripts with minimal effo...
error( $err, $die=0)
Throw an error to the user.
beginTransaction(IDatabase $dbw, $fname)
Begin a transaction on a DB.
commitTransaction(IDatabase $dbw, $fname)
Commit the transaction on a DB handle and wait for replica DBs to catch up.
output( $out, $channel=null)
Throw some output to the user.
waitForReplication()
Wait for replica DBs to catch up.
getServiceContainer()
Returns the main service container.
addDescription( $text)
Set the description text.
addOption( $name, $description, $required=false, $withArg=false, $shortName=false, $multiOccurrence=false)
Add a parameter to the script.
getOption( $name, $default=null)
Get an option, or return the default.
fatalError( $msg, $exitCode=1)
Output a message and terminate the current script.