MediaWiki  1.23.2
HistoryBlob.php
Go to the documentation of this file.
1 <?php
28 interface HistoryBlob
29 {
39  function addItem( $text );
40 
48  function getItem( $key );
49 
60  function setText( $text );
61 
67  function getText();
68 }
69 
75 {
76  public $mVersion = 0, $mCompressed = false, $mItems = array(), $mDefaultHash = '';
77  public $mSize = 0;
78  public $mMaxSize = 10000000;
79  public $mMaxCount = 100;
80 
84  public function __construct() {
85  if ( !function_exists( 'gzdeflate' ) ) {
86  throw new MWException( "Need zlib support to read or write this kind of history object (ConcatenatedGzipHistoryBlob)\n" );
87  }
88  }
89 
94  public function addItem( $text ) {
95  $this->uncompress();
96  $hash = md5( $text );
97  if ( !isset( $this->mItems[$hash] ) ) {
98  $this->mItems[$hash] = $text;
99  $this->mSize += strlen( $text );
100  }
101  return $hash;
102  }
103 
108  public function getItem( $hash ) {
109  $this->uncompress();
110  if ( array_key_exists( $hash, $this->mItems ) ) {
111  return $this->mItems[$hash];
112  } else {
113  return false;
114  }
115  }
116 
121  public function setText( $text ) {
122  $this->uncompress();
123  $this->mDefaultHash = $this->addItem( $text );
124  }
125 
129  public function getText() {
130  $this->uncompress();
131  return $this->getItem( $this->mDefaultHash );
132  }
133 
139  public function removeItem( $hash ) {
140  $this->mSize -= strlen( $this->mItems[$hash] );
141  unset( $this->mItems[$hash] );
142  }
143 
147  public function compress() {
148  if ( !$this->mCompressed ) {
149  $this->mItems = gzdeflate( serialize( $this->mItems ) );
150  $this->mCompressed = true;
151  }
152  }
153 
157  public function uncompress() {
158  if ( $this->mCompressed ) {
159  $this->mItems = unserialize( gzinflate( $this->mItems ) );
160  $this->mCompressed = false;
161  }
162  }
163 
167  function __sleep() {
168  $this->compress();
169  return array( 'mVersion', 'mCompressed', 'mItems', 'mDefaultHash' );
170  }
171 
172  function __wakeup() {
173  $this->uncompress();
174  }
175 
182  public function isHappy() {
183  return $this->mSize < $this->mMaxSize
184  && count( $this->mItems ) < $this->mMaxCount;
185  }
186 }
187 
198  protected static $blobCache = array();
199 
201 
206  function __construct( $hash = '', $oldid = 0 ) {
207  $this->mHash = $hash;
208  }
209 
214  function setLocation( $id ) {
215  $this->mOldId = $id;
216  }
217 
221  function setReferrer( $id ) {
222  $this->mRef = $id;
223  }
224 
228  function getReferrer() {
229  return $this->mRef;
230  }
231 
235  function getText() {
236  if ( isset( self::$blobCache[$this->mOldId] ) ) {
237  $obj = self::$blobCache[$this->mOldId];
238  } else {
239  $dbr = wfGetDB( DB_SLAVE );
240  $row = $dbr->selectRow( 'text', array( 'old_flags', 'old_text' ), array( 'old_id' => $this->mOldId ) );
241  if ( !$row ) {
242  return false;
243  }
244  $flags = explode( ',', $row->old_flags );
245  if ( in_array( 'external', $flags ) ) {
246  $url = $row->old_text;
247  $parts = explode( '://', $url, 2 );
248  if ( !isset( $parts[1] ) || $parts[1] == '' ) {
249  return false;
250  }
251  $row->old_text = ExternalStore::fetchFromUrl( $url );
252 
253  }
254  if ( !in_array( 'object', $flags ) ) {
255  return false;
256  }
257 
258  if ( in_array( 'gzip', $flags ) ) {
259  // This shouldn't happen, but a bug in the compress script
260  // may at times gzip-compress a HistoryBlob object row.
261  $obj = unserialize( gzinflate( $row->old_text ) );
262  } else {
263  $obj = unserialize( $row->old_text );
264  }
265 
266  if ( !is_object( $obj ) ) {
267  // Correct for old double-serialization bug.
268  $obj = unserialize( $obj );
269  }
270 
271  // Save this item for reference; if pulling many
272  // items in a row we'll likely use it again.
273  $obj->uncompress();
274  self::$blobCache = array( $this->mOldId => $obj );
275  }
276  return $obj->getItem( $this->mHash );
277  }
278 
284  function getHash() {
285  return $this->mHash;
286  }
287 }
288 
298  var $mCurId;
299 
303  function __construct( $curid = 0 ) {
304  $this->mCurId = $curid;
305  }
306 
313  function setLocation( $id ) {
314  $this->mCurId = $id;
315  }
316 
320  function getText() {
321  $dbr = wfGetDB( DB_SLAVE );
322  $row = $dbr->selectRow( 'cur', array( 'cur_text' ), array( 'cur_id' => $this->mCurId ) );
323  if ( !$row ) {
324  return false;
325  }
326  return $row->cur_text;
327  }
328 }
329 
334 class DiffHistoryBlob implements HistoryBlob {
336  var $mItems = array();
337 
339  var $mSize = 0;
340 
349  var $mDiffs;
350 
353 
358 
363 
367  var $mFrozen = false;
368 
373  var $mMaxSize = 10000000;
374 
378  var $mMaxCount = 100;
379 
381  const XDL_BDOP_INS = 1;
382  const XDL_BDOP_CPY = 2;
383  const XDL_BDOP_INSB = 3;
384 
385  function __construct() {
386  if ( !function_exists( 'gzdeflate' ) ) {
387  throw new MWException( "Need zlib support to read or write DiffHistoryBlob\n" );
388  }
389  }
390 
396  function addItem( $text ) {
397  if ( $this->mFrozen ) {
398  throw new MWException( __METHOD__ . ": Cannot add more items after sleep/wakeup" );
399  }
400 
401  $this->mItems[] = $text;
402  $this->mSize += strlen( $text );
403  $this->mDiffs = null; // later
404  return count( $this->mItems ) - 1;
405  }
406 
411  function getItem( $key ) {
412  return $this->mItems[$key];
413  }
414 
418  function setText( $text ) {
419  $this->mDefaultKey = $this->addItem( $text );
420  }
421 
425  function getText() {
426  return $this->getItem( $this->mDefaultKey );
427  }
428 
432  function compress() {
433  if ( !function_exists( 'xdiff_string_rabdiff' ) ) {
434  throw new MWException( "Need xdiff 1.5+ support to write DiffHistoryBlob\n" );
435  }
436  if ( isset( $this->mDiffs ) ) {
437  // Already compressed
438  return;
439  }
440  if ( !count( $this->mItems ) ) {
441  // Empty
442  return;
443  }
444 
445  // Create two diff sequences: one for main text and one for small text
446  $sequences = array(
447  'small' => array(
448  'tail' => '',
449  'diffs' => array(),
450  'map' => array(),
451  ),
452  'main' => array(
453  'tail' => '',
454  'diffs' => array(),
455  'map' => array(),
456  ),
457  );
458  $smallFactor = 0.5;
459 
460  for ( $i = 0; $i < count( $this->mItems ); $i++ ) {
461  $text = $this->mItems[$i];
462  if ( $i == 0 ) {
463  $seqName = 'main';
464  } else {
465  $mainTail = $sequences['main']['tail'];
466  if ( strlen( $text ) < strlen( $mainTail ) * $smallFactor ) {
467  $seqName = 'small';
468  } else {
469  $seqName = 'main';
470  }
471  }
472  $seq =& $sequences[$seqName];
473  $tail = $seq['tail'];
474  $diff = $this->diff( $tail, $text );
475  $seq['diffs'][] = $diff;
476  $seq['map'][] = $i;
477  $seq['tail'] = $text;
478  }
479  unset( $seq ); // unlink dangerous alias
480 
481  // Knit the sequences together
482  $tail = '';
483  $this->mDiffs = array();
484  $this->mDiffMap = array();
485  foreach ( $sequences as $seq ) {
486  if ( !count( $seq['diffs'] ) ) {
487  continue;
488  }
489  if ( $tail === '' ) {
490  $this->mDiffs[] = $seq['diffs'][0];
491  } else {
492  $head = $this->patch( '', $seq['diffs'][0] );
493  $this->mDiffs[] = $this->diff( $tail, $head );
494  }
495  $this->mDiffMap[] = $seq['map'][0];
496  for ( $i = 1; $i < count( $seq['diffs'] ); $i++ ) {
497  $this->mDiffs[] = $seq['diffs'][$i];
498  $this->mDiffMap[] = $seq['map'][$i];
499  }
500  $tail = $seq['tail'];
501  }
502  }
503 
509  function diff( $t1, $t2 ) {
510  # Need to do a null concatenation with warnings off, due to bugs in the current version of xdiff
511  # "String is not zero-terminated"
513  $diff = xdiff_string_rabdiff( $t1, $t2 ) . '';
515  return $diff;
516  }
517 
523  function patch( $base, $diff ) {
524  if ( function_exists( 'xdiff_string_bpatch' ) ) {
526  $text = xdiff_string_bpatch( $base, $diff ) . '';
528  return $text;
529  }
530 
531  # Pure PHP implementation
532 
533  $header = unpack( 'Vofp/Vcsize', substr( $diff, 0, 8 ) );
534 
535  # Check the checksum if hash extension is available
536  $ofp = $this->xdiffAdler32( $base );
537  if ( $ofp !== false && $ofp !== substr( $diff, 0, 4 ) ) {
538  wfDebug( __METHOD__ . ": incorrect base checksum\n" );
539  return false;
540  }
541  if ( $header['csize'] != strlen( $base ) ) {
542  wfDebug( __METHOD__ . ": incorrect base length\n" );
543  return false;
544  }
545 
546  $p = 8;
547  $out = '';
548  while ( $p < strlen( $diff ) ) {
549  $x = unpack( 'Cop', substr( $diff, $p, 1 ) );
550  $op = $x['op'];
551  ++$p;
552  switch ( $op ) {
553  case self::XDL_BDOP_INS:
554  $x = unpack( 'Csize', substr( $diff, $p, 1 ) );
555  $p++;
556  $out .= substr( $diff, $p, $x['size'] );
557  $p += $x['size'];
558  break;
559  case self::XDL_BDOP_INSB:
560  $x = unpack( 'Vcsize', substr( $diff, $p, 4 ) );
561  $p += 4;
562  $out .= substr( $diff, $p, $x['csize'] );
563  $p += $x['csize'];
564  break;
565  case self::XDL_BDOP_CPY:
566  $x = unpack( 'Voff/Vcsize', substr( $diff, $p, 8 ) );
567  $p += 8;
568  $out .= substr( $base, $x['off'], $x['csize'] );
569  break;
570  default:
571  wfDebug( __METHOD__ . ": invalid op\n" );
572  return false;
573  }
574  }
575  return $out;
576  }
577 
585  function xdiffAdler32( $s ) {
586  if ( !function_exists( 'hash' ) ) {
587  return false;
588  }
589 
590  static $init;
591  if ( $init === null ) {
592  $init = str_repeat( "\xf0", 205 ) . "\xee" . str_repeat( "\xf0", 67 ) . "\x02";
593  }
594 
595  // The real Adler-32 checksum of $init is zero, so it initialises the
596  // state to zero, as it is at the start of LibXDiff's checksum
597  // algorithm. Appending the subject string then simulates LibXDiff.
598  return strrev( hash( 'adler32', $init . $s, true ) );
599  }
600 
601  function uncompress() {
602  if ( !$this->mDiffs ) {
603  return;
604  }
605  $tail = '';
606  for ( $diffKey = 0; $diffKey < count( $this->mDiffs ); $diffKey++ ) {
607  $textKey = $this->mDiffMap[$diffKey];
608  $text = $this->patch( $tail, $this->mDiffs[$diffKey] );
609  $this->mItems[$textKey] = $text;
610  $tail = $text;
611  }
612  }
613 
617  function __sleep() {
618  $this->compress();
619  if ( !count( $this->mItems ) ) {
620  // Empty object
621  $info = false;
622  } else {
623  // Take forward differences to improve the compression ratio for sequences
624  $map = '';
625  $prev = 0;
626  foreach ( $this->mDiffMap as $i ) {
627  if ( $map !== '' ) {
628  $map .= ',';
629  }
630  $map .= $i - $prev;
631  $prev = $i;
632  }
633  $info = array(
634  'diffs' => $this->mDiffs,
635  'map' => $map
636  );
637  }
638  if ( isset( $this->mDefaultKey ) ) {
639  $info['default'] = $this->mDefaultKey;
640  }
641  $this->mCompressed = gzdeflate( serialize( $info ) );
642  return array( 'mCompressed' );
643  }
644 
645  function __wakeup() {
646  // addItem() doesn't work if mItems is partially filled from mDiffs
647  $this->mFrozen = true;
648  $info = unserialize( gzinflate( $this->mCompressed ) );
649  unset( $this->mCompressed );
650 
651  if ( !$info ) {
652  // Empty object
653  return;
654  }
655 
656  if ( isset( $info['default'] ) ) {
657  $this->mDefaultKey = $info['default'];
658  }
659  $this->mDiffs = $info['diffs'];
660  if ( isset( $info['base'] ) ) {
661  // Old format
662  $this->mDiffMap = range( 0, count( $this->mDiffs ) - 1 );
663  array_unshift( $this->mDiffs,
664  pack( 'VVCV', 0, 0, self::XDL_BDOP_INSB, strlen( $info['base'] ) ) .
665  $info['base'] );
666  } else {
667  // New format
668  $map = explode( ',', $info['map'] );
669  $cur = 0;
670  $this->mDiffMap = array();
671  foreach ( $map as $i ) {
672  $cur += $i;
673  $this->mDiffMap[] = $cur;
674  }
675  }
676  $this->uncompress();
677  }
678 
685  function isHappy() {
686  return $this->mSize < $this->mMaxSize
687  && count( $this->mItems ) < $this->mMaxCount;
688  }
689 
690 }
HistoryBlob
Base class for general text storage via the "object" flag in old_flags, or two-part external storage ...
Definition: HistoryBlob.php:28
DiffHistoryBlob\__sleep
__sleep()
Definition: HistoryBlob.php:617
DiffHistoryBlob\$mDiffs
$mDiffs
Array of diffs.
Definition: HistoryBlob.php:349
DiffHistoryBlob\$mMaxCount
$mMaxCount
The maximum number of text items before the object becomes sad.
Definition: HistoryBlob.php:378
HistoryBlobCurStub\$mCurId
$mCurId
Definition: HistoryBlob.php:298
DiffHistoryBlob\XDL_BDOP_CPY
const XDL_BDOP_CPY
Definition: HistoryBlob.php:382
ConcatenatedGzipHistoryBlob\$mMaxSize
$mMaxSize
Definition: HistoryBlob.php:78
php
skin txt MediaWiki includes four core it has been set as the default in MediaWiki since the replacing Monobook it had been been the default skin since before being replaced by Vector largely rewritten in while keeping its appearance Several legacy skins were removed in the as the burden of supporting them became too heavy to bear Those in etc for skin dependent CSS etc for skin dependent JavaScript These can also be customised on a per user by etc This feature has led to a wide variety of user styles becoming that gallery is a good place to ending in php
Definition: skin.txt:62
ConcatenatedGzipHistoryBlob\__sleep
__sleep()
Definition: HistoryBlob.php:167
HistoryBlob\getText
getText()
Get default text.
ConcatenatedGzipHistoryBlob\isHappy
isHappy()
Helper function for compression jobs Returns true until the object is "full" and ready to be committe...
Definition: HistoryBlob.php:182
HistoryBlob\getItem
getItem( $key)
Get item by key, or false if the key is not present.
HistoryBlobStub
Pointer object for an item within a CGZ blob stored in the text table.
Definition: HistoryBlob.php:191
HistoryBlobStub\getHash
getHash()
Get the content hash.
Definition: HistoryBlob.php:284
wfGetDB
& wfGetDB( $db, $groups=array(), $wiki=false)
Get a Database object.
Definition: GlobalFunctions.php:3650
ConcatenatedGzipHistoryBlob
Concatenated gzip (CGZ) storage Improves compression ratio by concatenating like objects before gzipp...
Definition: HistoryBlob.php:74
wfSuppressWarnings
wfSuppressWarnings( $end=false)
Reference-counted warning suppression.
Definition: GlobalFunctions.php:2387
HistoryBlobStub\$mRef
$mRef
Definition: HistoryBlob.php:200
DiffHistoryBlob\__construct
__construct()
Definition: HistoryBlob.php:385
HistoryBlob\addItem
addItem( $text)
Adds an item of text, returns a stub object which points to the item.
$s
$s
Definition: mergeMessageFileList.php:156
ConcatenatedGzipHistoryBlob\removeItem
removeItem( $hash)
Remove an item.
Definition: HistoryBlob.php:139
HistoryBlobStub\setReferrer
setReferrer( $id)
Sets the location (old_id) of the referring object.
Definition: HistoryBlob.php:221
DiffHistoryBlob\diff
diff( $t1, $t2)
Definition: HistoryBlob.php:509
DiffHistoryBlob\$mSize
$mSize
Total uncompressed size.
Definition: HistoryBlob.php:339
$flags
it s the revision text itself In either if gzip is the revision text is gzipped $flags
Definition: hooks.txt:2113
HistoryBlobStub\$mOldId
$mOldId
Definition: HistoryBlob.php:200
DiffHistoryBlob\getItem
getItem( $key)
Definition: HistoryBlob.php:411
DiffHistoryBlob\uncompress
uncompress()
Definition: HistoryBlob.php:601
DiffHistoryBlob\XDL_BDOP_INS
const XDL_BDOP_INS
Constants from xdiff.h.
Definition: HistoryBlob.php:381
$dbr
$dbr
Definition: testCompression.php:48
HistoryBlobCurStub\setLocation
setLocation( $id)
Sets the location (cur_id) of the main object to which this object points.
Definition: HistoryBlob.php:313
ConcatenatedGzipHistoryBlob\$mMaxCount
$mMaxCount
Definition: HistoryBlob.php:79
ConcatenatedGzipHistoryBlob\$mVersion
$mVersion
Definition: HistoryBlob.php:76
HistoryBlobCurStub\__construct
__construct( $curid=0)
Definition: HistoryBlob.php:303
MWException
MediaWiki exception.
Definition: MWException.php:26
$out
$out
Definition: UtfNormalGenerate.php:167
wfRestoreWarnings
wfRestoreWarnings()
Restore error level to previous value.
Definition: GlobalFunctions.php:2417
ConcatenatedGzipHistoryBlob\getText
getText()
Definition: HistoryBlob.php:129
DiffHistoryBlob\$mMaxSize
$mMaxSize
The maximum uncompressed size before the object becomes sad Should be less than max_allowed_packet.
Definition: HistoryBlob.php:373
HistoryBlobStub\$mHash
$mHash
Definition: HistoryBlob.php:200
array
the array() calling protocol came about after MediaWiki 1.4rc1.
List of Api Query prop modules.
ConcatenatedGzipHistoryBlob\$mItems
$mItems
Definition: HistoryBlob.php:76
HistoryBlobStub\$blobCache
static $blobCache
One-step cache variable to hold base blobs; operations that pull multiple revisions may often pull mu...
Definition: HistoryBlob.php:198
ConcatenatedGzipHistoryBlob\__construct
__construct()
Constructor.
Definition: HistoryBlob.php:84
DiffHistoryBlob\__wakeup
__wakeup()
Definition: HistoryBlob.php:645
wfDebug
wfDebug( $text, $dest='all')
Sends a line to the debug log if enabled or, optionally, to a comment in output.
Definition: GlobalFunctions.php:933
ConcatenatedGzipHistoryBlob\$mDefaultHash
$mDefaultHash
Definition: HistoryBlob.php:76
HistoryBlob\setText
setText( $text)
Set the "default text" This concept is an odd property of the current DB schema, whereby each text it...
HistoryBlobStub\__construct
__construct( $hash='', $oldid=0)
Definition: HistoryBlob.php:206
HistoryBlobStub\getText
getText()
Definition: HistoryBlob.php:235
DiffHistoryBlob\$mItems
$mItems
Uncompressed item cache.
Definition: HistoryBlob.php:336
ConcatenatedGzipHistoryBlob\$mCompressed
$mCompressed
Definition: HistoryBlob.php:76
DiffHistoryBlob\isHappy
isHappy()
Helper function for compression jobs Returns true until the object is "full" and ready to be committe...
Definition: HistoryBlob.php:685
DiffHistoryBlob\addItem
addItem( $text)
Definition: HistoryBlob.php:396
ConcatenatedGzipHistoryBlob\uncompress
uncompress()
Uncompress bulk data.
Definition: HistoryBlob.php:157
ConcatenatedGzipHistoryBlob\$mSize
$mSize
Definition: HistoryBlob.php:77
DiffHistoryBlob\XDL_BDOP_INSB
const XDL_BDOP_INSB
Definition: HistoryBlob.php:383
DiffHistoryBlob\setText
setText( $text)
Definition: HistoryBlob.php:418
DiffHistoryBlob\$mFrozen
$mFrozen
True if the object is locked against further writes.
Definition: HistoryBlob.php:367
ConcatenatedGzipHistoryBlob\getItem
getItem( $hash)
Definition: HistoryBlob.php:108
DiffHistoryBlob\$mDiffMap
$mDiffMap
The diff map, see above.
Definition: HistoryBlob.php:352
$hash
return false to override stock group addition can be modified try getUserPermissionsErrors userCan checks are continued by internal code can override on output return false to not delete it return false to override the default password checks & $hash
Definition: hooks.txt:2697
DiffHistoryBlob\getText
getText()
Definition: HistoryBlob.php:425
HistoryBlobStub\setLocation
setLocation( $id)
Sets the location (old_id) of the main object to which this object points.
Definition: HistoryBlob.php:214
DB_SLAVE
const DB_SLAVE
Definition: Defines.php:55
ConcatenatedGzipHistoryBlob\__wakeup
__wakeup()
Definition: HistoryBlob.php:172
HistoryBlobCurStub
To speed up conversion from 1.4 to 1.5 schema, text rows can refer to the leftover cur table as the b...
Definition: HistoryBlob.php:297
DiffHistoryBlob\xdiffAdler32
xdiffAdler32( $s)
Compute a binary "Adler-32" checksum as defined by LibXDiff, i.e.
Definition: HistoryBlob.php:585
as
This document is intended to provide useful advice for parties seeking to redistribute MediaWiki to end users It s targeted particularly at maintainers for Linux since it s been observed that distribution packages of MediaWiki often break We ve consistently had to recommend that users seeking support use official tarballs instead of their distribution s and this often solves whatever problem the user is having It would be nice if this could such as
Definition: distributors.txt:9
ConcatenatedGzipHistoryBlob\addItem
addItem( $text)
Definition: HistoryBlob.php:94
DiffHistoryBlob\patch
patch( $base, $diff)
Definition: HistoryBlob.php:523
ConcatenatedGzipHistoryBlob\compress
compress()
Compress the bulk data in the object.
Definition: HistoryBlob.php:147
DiffHistoryBlob\$mCompressed
$mCompressed
Compressed storage.
Definition: HistoryBlob.php:362
DiffHistoryBlob
Diff-based history compression Requires xdiff 1.5+ and zlib.
Definition: HistoryBlob.php:334
ConcatenatedGzipHistoryBlob\setText
setText( $text)
Definition: HistoryBlob.php:121
DiffHistoryBlob\compress
compress()
Definition: HistoryBlob.php:432
HistoryBlobCurStub\getText
getText()
Definition: HistoryBlob.php:320
HistoryBlobStub\getReferrer
getReferrer()
Gets the location of the referring object.
Definition: HistoryBlob.php:228
DiffHistoryBlob\$mDefaultKey
$mDefaultKey
The key for getText()
Definition: HistoryBlob.php:357