60 parent::__construct();
62 $this->
addOption(
'type',
'Set compression type to either: gzip|concat',
false,
true,
't' );
65 'Maximum number of revisions in a concat chunk',
72 'Earliest date to check for uncompressed revisions',
77 $this->
addOption(
'end-date',
'Latest revision date to compress',
false,
true,
'e' );
80 'The id to start from (gzip -> text table, concat -> page table)',
87 'Store specified revisions in an external cluster (untested)',
93 'The page_id to stop at (only when using concat compression type)',
102 if ( !function_exists(
"gzdeflate" ) ) {
103 $this->
fatalError(
"You must enable zlib support in PHP to compress old revisions!\n" .
104 "Please see https://www.php.net/manual/en/ref.zlib.php\n" );
107 $type = $this->
getOption(
'type',
'concat' );
108 $chunkSize = $this->
getOption(
'chunksize', 20 );
109 $startId = $this->
getOption(
'startid', 0 );
110 $beginDate = $this->
getOption(
'begin-date',
'' );
111 $endDate = $this->
getOption(
'end-date',
'' );
112 $extDB = $this->
getOption(
'extdb',
'' );
113 $endId = $this->
getOption(
'endid',
false );
115 if ( $type !=
'concat' && $type !=
'gzip' ) {
116 $this->
error(
"Type \"{$type}\" not supported" );
119 if ( $extDB !=
'' ) {
120 $this->
output(
"Compressing database {$wgDBname} to external cluster {$extDB}\n"
121 . str_repeat(
'-', 76 ) .
"\n\n" );
123 $this->
output(
"Compressing database {$wgDBname}\n"
124 . str_repeat(
'-', 76 ) .
"\n\n" );
128 if ( $type ==
'concat' ) {
129 $success = $this->compressWithConcat( $startId, $chunkSize, $beginDate,
130 $endDate, $extDB, $endId );
132 $this->compressOldPages( $startId, $extDB );
136 $this->
output(
"Done.\n" );
146 private function compressOldPages( $start = 0, $extdb =
'' ) {
148 $this->
output(
"Starting from old_id $start...\n" );
151 $res = $dbw->newSelectQueryBuilder()
152 ->select( [
'old_id',
'old_flags',
'old_text' ] )
155 ->where(
"old_id>=$start" )
156 ->orderBy(
'old_id' )
157 ->limit( $chunksize )
158 ->caller( __METHOD__ )->fetchResultSet();
160 if ( $res->numRows() == 0 ) {
166 foreach ( $res as $row ) {
167 # print " {$row->old_id} - {$row->old_namespace}:{$row->old_title}\n";
168 $this->compressPage( $row, $extdb );
169 $last = $row->old_id;
172 $start = $last + 1; # Deletion may leave
long empty stretches
173 $this->
output(
"$start...\n" );
184 private function compressPage( $row, $extdb ) {
185 if ( strpos( $row->old_flags,
'gzip' ) !==
false
186 || strpos( $row->old_flags,
'object' ) !==
false
188 # print "Already compressed row {$row->old_id}\n";
192 $flags = $row->old_flags ?
"{$row->old_flags},gzip" :
"gzip";
193 $compress = gzdeflate( $row->old_text );
195 # Store in external storage if required
196 if ( $extdb !==
'' ) {
199 $storeObj = $esFactory->getStore(
'DB' );
200 $compress = $storeObj->store( $extdb, $compress );
201 if ( $compress ===
false ) {
202 $this->
error(
"Unable to store object" );
209 $dbw->newUpdateQueryBuilder()
212 'old_flags' => $flags,
213 'old_text' => $compress
216 'old_id' => $row->old_id
218 ->caller( __METHOD__ )
235 private function compressWithConcat( $startId, $maxChunkSize, $beginDate,
236 $endDate, $extdb =
"", $maxPageId =
false
241 # Set up external storage
242 if ( $extdb !=
'' ) {
245 $storeObj = $esFactory->getStore(
'DB' );
249 ->getBlobStoreFactory()
252 # Get all articles by page_id
254 $maxPageId = $dbr->newSelectQueryBuilder()
255 ->select(
'max(page_id)' )
257 ->caller( __METHOD__ )->fetchField();
259 $this->
output(
"Starting from $startId of $maxPageId\n" );
272 # For each article, get a list of revisions which fit the criteria
274 # No recompression, use a condition on old_flags
275 # Don't compress object type entities, because that might produce data loss when
276 # overwriting bulk storage concat rows. Don't compress external references, because
277 # the script doesn't yet delete rows from external storage.
279 $queryBuilderTemplate = $dbw->newSelectQueryBuilder()
280 ->select( [
'rev_id',
'old_id',
'old_flags',
'old_text' ] )
283 ->join(
'slots',
null,
'rev_id=slot_revision_id' )
284 ->join(
'content',
null,
'content_id=slot_content_id' )
285 ->join(
'text',
null,
'SUBSTRING(content_address, 4)=old_id' )
289 IExpression::NOT_LIKE,
290 new LikeValue( $dbr->anyString(),
'object', $dbr->anyString() )
293 IExpression::NOT_LIKE,
294 new LikeValue( $dbr->anyString(),
'external', $dbr->anyString() )
298 'slot_role_id' => $slotRoleStore->getId( SlotRecord::MAIN ),
299 'SUBSTRING(content_address, 1, 3)=' . $dbr->addQuotes(
'tt:' ),
303 if ( !preg_match(
'/^\d{14}$/', $beginDate ) ) {
304 $this->
error(
"Invalid begin date \"$beginDate\"\n" );
308 $queryBuilderTemplate->andWhere( $dbr->expr(
'rev_timestamp',
'>', $beginDate ) );
311 if ( !preg_match(
'/^\d{14}$/', $endDate ) ) {
312 $this->
error(
"Invalid end date \"$endDate\"\n" );
316 $queryBuilderTemplate->andWhere( $dbr->expr(
'rev_timestamp',
'<', $endDate ) );
319 for ( $pageId = $startId; $pageId <= $maxPageId; $pageId++ ) {
326 $pageRow = $dbr->newSelectQueryBuilder()
327 ->select( [
'page_id',
'page_namespace',
'page_title',
'rev_timestamp' ] )
329 ->straightJoin(
'revision',
null,
'page_latest = rev_id' )
330 ->where( $pageConds )
331 ->andWhere( [
'page_id' => $pageId ] )
332 ->caller( __METHOD__ )->fetchRow();
333 if ( $pageRow ===
false ) {
338 $titleObj = Title::makeTitle( $pageRow->page_namespace, $pageRow->page_title );
339 $this->
output(
"$pageId\t" . $titleObj->getPrefixedDBkey() .
" " );
342 $queryBuilder = clone $queryBuilderTemplate;
343 $revRes = $queryBuilder->where(
345 'rev_page' => $pageRow->page_id,
349 $dbr->expr(
'rev_timestamp',
'<', (
int)$pageRow->rev_timestamp ),
351 ->caller( __METHOD__ )->fetchResultSet();
354 foreach ( $revRes as $revRow ) {
358 if ( count( $revs ) < 2 ) {
359 # No revisions matching, no further processing
366 while ( $i < count( $revs ) ) {
367 if ( $i < count( $revs ) - $maxChunkSize ) {
368 $thisChunkSize = $maxChunkSize;
370 $thisChunkSize = count( $revs ) - $i;
377 $primaryOldid = $revs[$i]->old_id;
379 # Get the text of each revision and add it to the object
380 for ( $j = 0; $j < $thisChunkSize && $chunk->isHappy(); $j++ ) {
381 $oldid = $revs[$i + $j]->old_id;
383 # Get text. We do not need the full `extractBlob` since the query is built
384 # to fetch non-externalstore blobs.
385 $text = $blobStore->decompressData(
386 $revs[$i + $j]->old_text,
387 explode(
',', $revs[$i + $j]->old_flags )
390 if ( $text ===
false ) {
391 $this->
error(
"\nError, unable to get text in old_id $oldid" );
392 # $dbw->delete( 'old', [ 'old_id' => $oldid ] );
395 if ( $extdb ==
"" && $j == 0 ) {
396 $chunk->setText( $text );
399 # Don't make a stub if it's going to be longer than the article
400 # Stubs are typically about 100 bytes
401 if ( strlen( $text ) < 120 ) {
406 $stub->setLocation( $primaryOldid );
407 $stub->setReferrer( $oldid );
416 # If we couldn't actually use any stubs because the pages were too small, do nothing
418 if ( $extdb !=
"" ) {
419 # Move blob objects to External Storage
421 $stored = $storeObj->store( $extdb, serialize( $chunk ) );
422 if ( $stored ===
false ) {
423 $this->
error(
"Unable to store object" );
427 # Store External Storage URLs instead of Stub placeholders
428 foreach ( $stubs as $stub ) {
429 if ( $stub ===
false ) {
432 # $stored should provide base path to a BLOB
433 $url = $stored .
"/" . $stub->getHash();
434 $dbw->newUpdateQueryBuilder()
438 'old_flags' =>
'external,utf-8',
441 'old_id' => $stub->getReferrer(),
443 ->caller( __METHOD__ )
447 # Store the main object locally
448 $dbw->newUpdateQueryBuilder()
451 'old_text' => serialize( $chunk ),
452 'old_flags' =>
'object,utf-8',
455 'old_id' => $primaryOldid
457 ->caller( __METHOD__ )
460 # Store the stub objects
461 for ( $j = 1; $j < $thisChunkSize; $j++ ) {
462 # Skip if not compressing and don't overwrite the first revision
463 if ( $stubs[$j] !==
false && $revs[$i + $j]->old_id != $primaryOldid ) {
464 $dbw->newUpdateQueryBuilder()
467 'old_text' => serialize( $stubs[$j] ),
468 'old_flags' =>
'object,utf-8',
471 'old_id' => $revs[$i + $j]->old_id
473 ->caller( __METHOD__ )
482 $i += $thisChunkSize;
Concatenated gzip (CGZ) storage Improves compression ratio by concatenating like objects before gzipp...
Pointer object for an item within a CGZ blob stored in the text table.