includes/HistoryBlob.php

   1 <?php
   2
   3 /**
   4  * Base class for general text storage via the "object" flag in old_flags, or
   5  * two-part external storage URLs. Used for represent efficient concatenated
   6  * storage, and migration-related pointer objects.
   7  */
   8 interface HistoryBlob
   9 {
  10         /**
  11          * Adds an item of text, returns a stub object which points to the item.
  12          * You must call setLocation() on the stub object before storing it to the
  13          * database
  14          *
  15          * @return String: the key for getItem()
  16          */
  17         function addItem( $text );
  18
  19         /**
  20          * Get item by key, or false if the key is not present
  21          *
  22          * @return String or false
  23          */
  24         function getItem( $key );
  25
  26         /**
  27          * Set the "default text"
  28          * This concept is an odd property of the current DB schema, whereby each text item has a revision
  29          * associated with it. The default text is the text of the associated revision. There may, however,
  30          * be other revisions in the same object.
  31          *
  32          * Default text is not required for two-part external storage URLs.
  33          */
  34         function setText( $text );
  35
  36         /**
  37          * Get default text. This is called from Revision::getRevisionText()
  38          *
  39          * @return String
  40          */
  41         function getText();
  42 }
  43
  44 /**
  45  * Concatenated gzip (CGZ) storage
  46  * Improves compression ratio by concatenating like objects before gzipping
  47  */
  48 class ConcatenatedGzipHistoryBlob implements HistoryBlob
  49 {
  50         public $mVersion = 0, $mCompressed = false, $mItems = array(), $mDefaultHash = '';
  51         public $mSize = 0;
  52         public $mMaxSize = 10000000;
  53         public $mMaxCount = 100;
  54
  55         /** Constructor */
  56         public function __construct() {
  57                 if ( !function_exists( 'gzdeflate' ) ) {
  58                         throw new MWException( "Need zlib support to read or write this kind of history object (ConcatenatedGzipHistoryBlob)\n" );
  59                 }
  60         }
  61
  62         public function addItem( $text ) {
  63                 $this->uncompress();
  64                 $hash = md5( $text );
  65                 if ( !isset( $this->mItems[$hash] ) ) {
  66                         $this->mItems[$hash] = $text;
  67                         $this->mSize += strlen( $text );
  68                 }
  69                 return $hash;
  70         }
  71
  72         public function getItem( $hash ) {
  73                 $this->uncompress();
  74                 if ( array_key_exists( $hash, $this->mItems ) ) {
  75                         return $this->mItems[$hash];
  76                 } else {
  77                         return false;
  78                 }
  79         }
  80
  81         public function setText( $text ) {
  82                 $this->uncompress();
  83                 $this->mDefaultHash = $this->addItem( $text );
  84         }
  85
  86         public function getText() {
  87                 $this->uncompress();
  88                 return $this->getItem( $this->mDefaultHash );
  89         }
  90
  91         /**
  92          * Remove an item
  93          */
  94         public function removeItem( $hash ) {
  95                 $this->mSize -= strlen( $this->mItems[$hash] );
  96                 unset( $this->mItems[$hash] );
  97         }
  98
  99         /**
 100          * Compress the bulk data in the object
 101          */
 102         public function compress() {
 103                 if ( !$this->mCompressed  ) {
 104                         $this->mItems = gzdeflate( serialize( $this->mItems ) );
 105                         $this->mCompressed = true;
 106                 }
 107         }
 108
 109         /**
 110          * Uncompress bulk data
 111          */
 112         public function uncompress() {
 113                 if ( $this->mCompressed ) {
 114                         $this->mItems = unserialize( gzinflate( $this->mItems ) );
 115                         $this->mCompressed = false;
 116                 }
 117         }
 118
 119
 120         function __sleep() {
 121                 $this->compress();
 122                 return array( 'mVersion', 'mCompressed', 'mItems', 'mDefaultHash' );
 123         }
 124
 125         function __wakeup() {
 126                 $this->uncompress();
 127         }
 128
 129         /**
 130          * Helper function for compression jobs
 131          * Returns true until the object is "full" and ready to be committed
 132          */
 133         public function isHappy() {
 134                 return $this->mSize < $this->mMaxSize
 135                         && count( $this->mItems ) < $this->mMaxCount;
 136         }
 137 }
 138
 139
 140
 141
 142 /**
 143  * Pointer object for an item within a CGZ blob stored in the text table.
 144  */
 145 class HistoryBlobStub {
 146         /**
 147          * One-step cache variable to hold base blobs; operations that
 148          * pull multiple revisions may often pull multiple times from
 149          * the same blob. By keeping the last-used one open, we avoid
 150          * redundant unserialization and decompression overhead.
 151          */
 152         protected static $blobCache = array();
 153
 154         var $mOldId, $mHash, $mRef;
 155
 156         /**
 157          * @param $hash Strng: the content hash of the text
 158          * @param $oldid Integer: the old_id for the CGZ object
 159          */
 160         function __construct( $hash = '', $oldid = 0 ) {
 161                 $this->mHash = $hash;
 162         }
 163
 164         /**
 165          * Sets the location (old_id) of the main object to which this object
 166          * points
 167          */
 168         function setLocation( $id ) {
 169                 $this->mOldId = $id;
 170         }
 171
 172         /**
 173          * Sets the location (old_id) of the referring object
 174          */
 175         function setReferrer( $id ) {
 176                 $this->mRef = $id;
 177         }
 178
 179         /**
 180          * Gets the location of the referring object
 181          */
 182         function getReferrer() {
 183                 return $this->mRef;
 184         }
 185
 186         function getText() {
 187                 $fname = 'HistoryBlobStub::getText';
 188
 189                 if( isset( self::$blobCache[$this->mOldId] ) ) {
 190                         $obj = self::$blobCache[$this->mOldId];
 191                 } else {
 192                         $dbr = wfGetDB( DB_SLAVE );
 193                         $row = $dbr->selectRow( 'text', array( 'old_flags', 'old_text' ), array( 'old_id' => $this->mOldId ) );
 194                         if( !$row ) {
 195                                 return false;
 196                         }
 197                         $flags = explode( ',', $row->old_flags );
 198                         if( in_array( 'external', $flags ) ) {
 199                                 $url=$row->old_text;
 200                                 @list( /* $proto */ ,$path)=explode('://',$url,2);
 201                                 if ($path=="") {
 202                                         wfProfileOut( $fname );
 203                                         return false;
 204                                 }
 205                                 $row->old_text=ExternalStore::fetchFromUrl($url);
 206
 207                         }
 208                         if( !in_array( 'object', $flags ) ) {
 209                                 return false;
 210                         }
 211
 212                         if( in_array( 'gzip', $flags ) ) {
 213                                 // This shouldn't happen, but a bug in the compress script
 214                                 // may at times gzip-compress a HistoryBlob object row.
 215                                 $obj = unserialize( gzinflate( $row->old_text ) );
 216                         } else {
 217                                 $obj = unserialize( $row->old_text );
 218                         }
 219
 220                         if( !is_object( $obj ) ) {
 221                                 // Correct for old double-serialization bug.
 222                                 $obj = unserialize( $obj );
 223                         }
 224
 225                         // Save this item for reference; if pulling many
 226                         // items in a row we'll likely use it again.
 227                         $obj->uncompress();
 228                         self::$blobCache = array( $this->mOldId => $obj );
 229                 }
 230                 return $obj->getItem( $this->mHash );
 231         }
 232
 233         /**
 234          * Get the content hash
 235          */
 236         function getHash() {
 237                 return $this->mHash;
 238         }
 239 }
 240
 241
 242 /**
 243  * To speed up conversion from 1.4 to 1.5 schema, text rows can refer to the
 244  * leftover cur table as the backend. This avoids expensively copying hundreds
 245  * of megabytes of data during the conversion downtime.
 246  *
 247  * Serialized HistoryBlobCurStub objects will be inserted into the text table
 248  * on conversion if $wgFastSchemaUpgrades is set to true.
 249  */
 250 class HistoryBlobCurStub {
 251         var $mCurId;
 252
 253         /**
 254          * @param $curid Integer: the cur_id pointed to
 255          */
 256         function __construct( $curid = 0 ) {
 257                 $this->mCurId = $curid;
 258         }
 259
 260         /**
 261          * Sets the location (cur_id) of the main object to which this object
 262          * points
 263          */
 264         function setLocation( $id ) {
 265                 $this->mCurId = $id;
 266         }
 267
 268         function getText() {
 269                 $dbr = wfGetDB( DB_SLAVE );
 270                 $row = $dbr->selectRow( 'cur', array( 'cur_text' ), array( 'cur_id' => $this->mCurId ) );
 271                 if( !$row ) {
 272                         return false;
 273                 }
 274                 return $row->cur_text;
 275         }
 276 }
 277
 278 /**
 279  * Diff-based history compression
 280  * Requires xdiff 1.5+ and zlib
 281  */
 282 class DiffHistoryBlob implements HistoryBlob {
 283         /** Uncompressed item cache */
 284         var $mItems = array();
 285
 286         /** Total uncompressed size */
 287         var $mSize = 0;
 288
 289         /**
 290          * Array of diffs. If a diff D from A to B is notated D = B - A, and Z is
 291          * an empty string:
 292          *
 293          *              { item[map[i]] - item[map[i-1]]   where i > 0
 294          *    diff[i] = {
 295          *              { item[map[i]] - Z                where i = 0
 296          */
 297         var $mDiffs;
 298
 299         /** The diff map, see above */
 300         var $mDiffMap;
 301
 302         /**
 303          * The key for getText()
 304          */
 305         var $mDefaultKey;
 306
 307         /**
 308          * Compressed storage
 309          */
 310         var $mCompressed;
 311
 312         /**
 313          * True if the object is locked against further writes
 314          */
 315         var $mFrozen = false;
 316
 317         /**
 318          * The maximum uncompressed size before the object becomes sad
 319          * Should be less than max_allowed_packet
 320          */
 321         var $mMaxSize = 10000000;
 322
 323         /**
 324          * The maximum number of text items before the object becomes sad
 325          */
 326         var $mMaxCount = 100;
 327
 328         /** Constants from xdiff.h */
 329         const XDL_BDOP_INS = 1;
 330         const XDL_BDOP_CPY = 2;
 331         const XDL_BDOP_INSB = 3;
 332
 333         function __construct() {
 334                 if ( !function_exists( 'gzdeflate' ) ) {
 335                         throw new MWException( "Need zlib support to read or write DiffHistoryBlob\n" );
 336                 }
 337         }
 338
 339         function addItem( $text ) {
 340                 if ( $this->mFrozen ) {
 341                         throw new MWException( __METHOD__.": Cannot add more items after sleep/wakeup" );
 342                 }
 343
 344                 $this->mItems[] = $text;
 345                 $this->mSize += strlen( $text );
 346                 $this->mDiffs = null; // later
 347                 return count( $this->mItems ) - 1;
 348         }
 349
 350         function getItem( $key ) {
 351                 return $this->mItems[$key];
 352         }
 353
 354         function setText( $text ) {
 355                 $this->mDefaultKey = $this->addItem( $text );
 356         }
 357
 358         function getText() {
 359                 return $this->getItem( $this->mDefaultKey );
 360         }
 361
 362         function compress() {
 363                 if ( !function_exists( 'xdiff_string_rabdiff' ) ){
 364                         throw new MWException( "Need xdiff 1.5+ support to write DiffHistoryBlob\n" );
 365                 }
 366                 if ( isset( $this->mDiffs ) ) {
 367                         // Already compressed
 368                         return;
 369                 }
 370                 if ( !count( $this->mItems ) ) {
 371                         // Empty
 372                         return;
 373                 }
 374
 375                 // Create two diff sequences: one for main text and one for small text
 376                 $sequences = array(
 377                         'small' => array(
 378                                 'tail' => '',
 379                                 'diffs' => array(),
 380                                 'map' => array(),
 381                         ),
 382                         'main' => array(
 383                                 'tail' => '',
 384                                 'diffs' => array(),
 385                                 'map' => array(),
 386                         ),
 387                 );
 388                 $smallFactor = 0.5;
 389
 390                 for ( $i = 0; $i < count( $this->mItems ); $i++ ) {
 391                         $text = $this->mItems[$i];
 392                         if ( $i == 0 ) {
 393                                 $seqName = 'main';
 394                         } else {
 395                                 $mainTail = $sequences['main']['tail'];
 396                                 if ( strlen( $text ) < strlen( $mainTail ) * $smallFactor ) {
 397                                         $seqName = 'small';
 398                                 } else {
 399                                         $seqName = 'main';
 400                                 }
 401                         }
 402                         $seq =& $sequences[$seqName];
 403                         $tail = $seq['tail'];
 404                         $diff = $this->diff( $tail, $text );
 405                         $seq['diffs'][] = $diff;
 406                         $seq['map'][] = $i;
 407                         $seq['tail'] = $text;
 408                 }
 409                 unset( $seq ); // unlink dangerous alias
 410
 411                 // Knit the sequences together
 412                 $tail = '';
 413                 $this->mDiffs = array();
 414                 $this->mDiffMap = array();
 415                 foreach ( $sequences as $seq ) {
 416                         if ( !count( $seq['diffs'] ) ) {
 417                                 continue;
 418                         }
 419                         if ( $tail === '' ) {
 420                                 $this->mDiffs[] = $seq['diffs'][0];
 421                         } else {
 422                                 $head = $this->patch( '', $seq['diffs'][0] );
 423                                 $this->mDiffs[] = $this->diff( $tail, $head );
 424                         }
 425                         $this->mDiffMap[] = $seq['map'][0];
 426                         for ( $i = 1; $i < count( $seq['diffs'] ); $i++ ) {
 427                                 $this->mDiffs[] = $seq['diffs'][$i];
 428                                 $this->mDiffMap[] = $seq['map'][$i];
 429                         }
 430                         $tail = $seq['tail'];
 431                 }
 432         }
 433
 434         function diff( $t1, $t2 ) {
 435                 # Need to do a null concatenation with warnings off, due to bugs in the current version of xdiff
 436                 # "String is not zero-terminated"
 437                 wfSuppressWarnings();
 438                 $diff = xdiff_string_rabdiff( $t1, $t2 ) . '';
 439                 wfRestoreWarnings();
 440                 return $diff;
 441         }
 442
 443         function patch( $base, $diff ) {
 444                 if ( function_exists( 'xdiff_string_bpatch' ) ) {
 445                         wfSuppressWarnings();
 446                         $text = xdiff_string_bpatch( $base, $diff ) . '';
 447                         wfRestoreWarnings();
 448                         return $text;
 449                 }
 450
 451                 # Pure PHP implementation
 452
 453                 $header = unpack( 'Vofp/Vcsize', substr( $diff, 0, 8 ) );
 454
 455                 # Check the checksum if mhash is available
 456                 if ( extension_loaded( 'mhash' ) ) {
 457                         $ofp = mhash( MHASH_ADLER32, $base );
 458                         if ( $ofp !== substr( $diff, 0, 4 ) ) {
 459                                 wfDebug( __METHOD__. ": incorrect base checksum\n" );
 460                                 return false;
 461                         }
 462                 }
 463                 if ( $header['csize'] != strlen( $base ) ) {
 464                         wfDebug( __METHOD__. ": incorrect base length\n" );
 465                         return false;
 466                 }
 467
 468                 $p = 8;
 469                 $out = '';
 470                 while ( $p < strlen( $diff ) ) {
 471                         $x = unpack( 'Cop', substr( $diff, $p, 1 ) );
 472                         $op = $x['op'];
 473                         ++$p;
 474                         switch ( $op ) {
 475                         case self::XDL_BDOP_INS:
 476                                 $x = unpack( 'Csize', substr( $diff, $p, 1 ) );
 477                                 $p++;
 478                                 $out .= substr( $diff, $p, $x['size'] );
 479                                 $p += $x['size'];
 480                                 break;
 481                         case self::XDL_BDOP_INSB:
 482                                 $x = unpack( 'Vcsize', substr( $diff, $p, 4 ) );
 483                                 $p += 4;
 484                                 $out .= substr( $diff, $p, $x['csize'] );
 485                                 $p += $x['csize'];
 486                                 break;
 487                         case self::XDL_BDOP_CPY:
 488                                 $x = unpack( 'Voff/Vcsize', substr( $diff, $p, 8 ) );
 489                                 $p += 8;
 490                                 $out .= substr( $base, $x['off'], $x['csize'] );
 491                                 break;
 492                         default:
 493                                 wfDebug( __METHOD__.": invalid op\n" );
 494                                 return false;
 495                         }
 496                 }
 497                 return $out;
 498         }
 499
 500         function uncompress() {
 501                 if ( !$this->mDiffs ) {
 502                         return;
 503                 }
 504                 $tail = '';
 505                 for ( $diffKey = 0; $diffKey < count( $this->mDiffs ); $diffKey++ ) {
 506                         $textKey = $this->mDiffMap[$diffKey];
 507                         $text = $this->patch( $tail, $this->mDiffs[$diffKey] );
 508                         $this->mItems[$textKey] = $text;
 509                         $tail = $text;
 510                 }
 511         }
 512
 513         function __sleep() {
 514                 $this->compress();
 515                 if ( !count( $this->mItems ) ) {
 516                         // Empty object
 517                         $info = false;
 518                 } else {
 519                         // Take forward differences to improve the compression ratio for sequences
 520                         $map = '';
 521                         $prev = 0;
 522                         foreach ( $this->mDiffMap as $i ) {
 523                                 if ( $map !== '' ) {
 524                                         $map .= ',';
 525                                 }
 526                                 $map .= $i - $prev;
 527                                 $prev = $i;
 528                         }
 529                         $info = array(
 530                                 'diffs' => $this->mDiffs,
 531                                 'map' => $map
 532                         );
 533                 }
 534                 if ( isset( $this->mDefaultKey ) ) {
 535                         $info['default'] = $this->mDefaultKey;
 536                 }
 537                 $this->mCompressed = gzdeflate( serialize( $info ) );
 538                 return array( 'mCompressed' );
 539         }
 540
 541         function __wakeup() {
 542                 // addItem() doesn't work if mItems is partially filled from mDiffs
 543                 $this->mFrozen = true;
 544                 $info = unserialize( gzinflate( $this->mCompressed ) );
 545                 unset( $this->mCompressed );
 546
 547                 if ( !$info ) {
 548                         // Empty object
 549                         return;
 550                 }
 551
 552                 if ( isset( $info['default'] ) ) {
 553                         $this->mDefaultKey = $info['default'];
 554                 }
 555                 $this->mDiffs = $info['diffs'];
 556                 if ( isset( $info['base'] ) ) {
 557                         // Old format
 558                         $this->mDiffMap = range( 0, count( $this->mDiffs ) - 1 );
 559                         array_unshift( $this->mDiffs,
 560                                 pack( 'VVCV', 0, 0, self::XDL_BDOP_INSB, strlen( $info['base'] ) ) .
 561                                 $info['base'] );
 562                 } else {
 563                         // New format
 564                         $map = explode( ',', $info['map'] );
 565                         $cur = 0;
 566                         $this->mDiffMap = array();
 567                         foreach ( $map as $i ) {
 568                                 $cur += $i;
 569                                 $this->mDiffMap[] = $cur;
 570                         }
 571                 }
 572                 $this->uncompress();
 573         }
 574
 575         /**
 576          * Helper function for compression jobs
 577          * Returns true until the object is "full" and ready to be committed
 578          */
 579         function isHappy() {
 580                 return $this->mSize < $this->mMaxSize
 581                         && count( $this->mItems ) < $this->mMaxCount;
 582         }
 583
 584 }