includes/HistoryBlob.php

   1 <?php
   2
   3 /**
   4  * Base class for general text storage via the "object" flag in old_flags, or
   5  * two-part external storage URLs. Used for represent efficient concatenated
   6  * storage, and migration-related pointer objects.
   7  */
   8 interface HistoryBlob
   9 {
  10         /**
  11          * Adds an item of text, returns a stub object which points to the item.
  12          * You must call setLocation() on the stub object before storing it to the
  13          * database
  14          * Returns the key for getItem()
  15          */
  16         public function addItem( $text );
  17
  18         /**
  19          * Get item by key, or false if the key is not present
  20          */
  21         public function getItem( $key );
  22
  23         /**
  24          * Set the "default text"
  25          * This concept is an odd property of the current DB schema, whereby each text item has a revision
  26          * associated with it. The default text is the text of the associated revision. There may, however,
  27          * be other revisions in the same object.
  28          *
  29          * Default text is not required for two-part external storage URLs.
  30          */
  31         public function setText( $text );
  32
  33         /**
  34          * Get default text. This is called from Revision::getRevisionText()
  35          */
  36         function getText();
  37 }
  38
  39 /**
  40  * Concatenated gzip (CGZ) storage
  41  * Improves compression ratio by concatenating like objects before gzipping
  42  */
  43 class ConcatenatedGzipHistoryBlob implements HistoryBlob
  44 {
  45         public $mVersion = 0, $mCompressed = false, $mItems = array(), $mDefaultHash = '';
  46         public $mFast = 0, $mSize = 0;
  47
  48         /** Constructor */
  49         public function ConcatenatedGzipHistoryBlob() {
  50                 if ( !function_exists( 'gzdeflate' ) ) {
  51                         throw new MWException( "Need zlib support to read or write this kind of history object (ConcatenatedGzipHistoryBlob)\n" );
  52                 }
  53         }
  54
  55         public function addItem( $text ) {
  56                 $this->uncompress();
  57                 $hash = md5( $text );
  58                 $this->mItems[$hash] = $text;
  59                 $this->mSize += strlen( $text );
  60
  61                 return $hash;
  62         }
  63
  64         public function getItem( $hash ) {
  65                 $this->uncompress();
  66                 if ( array_key_exists( $hash, $this->mItems ) ) {
  67                         return $this->mItems[$hash];
  68                 } else {
  69                         return false;
  70                 }
  71         }
  72
  73         public function setText( $text ) {
  74                 $this->uncompress();
  75                 $this->mDefaultHash = $this->addItem( $text );
  76         }
  77
  78         public function getText() {
  79                 $this->uncompress();
  80                 return $this->getItem( $this->mDefaultHash );
  81         }
  82
  83         /**
  84          * Remove an item
  85          */
  86         public function removeItem( $hash ) {
  87                 $this->mSize -= strlen( $this->mItems[$hash] );
  88                 unset( $this->mItems[$hash] );
  89         }
  90
  91         /**
  92          * Compress the bulk data in the object
  93          */
  94         public function compress() {
  95                 if ( !$this->mCompressed  ) {
  96                         $this->mItems = gzdeflate( serialize( $this->mItems ) );
  97                         $this->mCompressed = true;
  98                 }
  99         }
 100
 101         /**
 102          * Uncompress bulk data
 103          */
 104         public function uncompress() {
 105                 if ( $this->mCompressed ) {
 106                         $this->mItems = unserialize( gzinflate( $this->mItems ) );
 107                         $this->mCompressed = false;
 108                 }
 109         }
 110
 111
 112         function __sleep() {
 113                 $this->compress();
 114                 return array( 'mVersion', 'mCompressed', 'mItems', 'mDefaultHash' );
 115         }
 116
 117         function __wakeup() {
 118                 $this->uncompress();
 119         }
 120
 121         /**
 122          * Helper function for compression jobs
 123          * Returns true until the object is "full" and ready to be committed
 124          */
 125         public function isHappy( $maxFactor, $factorThreshold ) {
 126                 if ( count( $this->mItems ) == 0 ) {
 127                         return true;
 128                 }
 129                 if ( !$this->mFast ) {
 130                         $this->uncompress();
 131                         $record = serialize( $this->mItems );
 132                         $size = strlen( $record );
 133                         $avgUncompressed = $size / count( $this->mItems );
 134                         $compressed = strlen( gzdeflate( $record ) );
 135
 136                         if ( $compressed < $factorThreshold * 1024 ) {
 137                                 return true;
 138                         } else {
 139                                 return $avgUncompressed * $maxFactor < $compressed;
 140                         }
 141                 } else {
 142                         return count( $this->mItems ) <= 10;
 143                 }
 144         }
 145 }
 146
 147
 148 /**
 149  * One-step cache variable to hold base blobs; operations that
 150  * pull multiple revisions may often pull multiple times from
 151  * the same blob. By keeping the last-used one open, we avoid
 152  * redundant unserialization and decompression overhead.
 153  */
 154 global $wgBlobCache;
 155 $wgBlobCache = array();
 156
 157
 158 /**
 159  * Pointer object for an item within a CGZ blob stored in the text table.
 160  */
 161 class HistoryBlobStub {
 162         var $mOldId, $mHash, $mRef;
 163
 164         /**
 165          * @param string $hash The content hash of the text
 166          * @param integer $oldid The old_id for the CGZ object
 167          */
 168         function HistoryBlobStub( $hash = '', $oldid = 0 ) {
 169                 $this->mHash = $hash;
 170         }
 171
 172         /**
 173          * Sets the location (old_id) of the main object to which this object
 174          * points
 175          */
 176         function setLocation( $id ) {
 177                 $this->mOldId = $id;
 178         }
 179
 180         /**
 181          * Sets the location (old_id) of the referring object
 182          */
 183         function setReferrer( $id ) {
 184                 $this->mRef = $id;
 185         }
 186
 187         /**
 188          * Gets the location of the referring object
 189          */
 190         function getReferrer() {
 191                 return $this->mRef;
 192         }
 193
 194         function getText() {
 195                 $fname = 'HistoryBlobStub::getText';
 196                 global $wgBlobCache;
 197                 if( isset( $wgBlobCache[$this->mOldId] ) ) {
 198                         $obj = $wgBlobCache[$this->mOldId];
 199                 } else {
 200                         $dbr = wfGetDB( DB_SLAVE );
 201                         $row = $dbr->selectRow( 'text', array( 'old_flags', 'old_text' ), array( 'old_id' => $this->mOldId ) );
 202                         if( !$row ) {
 203                                 return false;
 204                         }
 205                         $flags = explode( ',', $row->old_flags );
 206                         if( in_array( 'external', $flags ) ) {
 207                                 $url=$row->old_text;
 208                                 @list( /* $proto */ ,$path)=explode('://',$url,2);
 209                                 if ($path=="") {
 210                                         wfProfileOut( $fname );
 211                                         return false;
 212                                 }
 213                                 $row->old_text=ExternalStore::fetchFromUrl($url);
 214
 215                         }
 216                         if( !in_array( 'object', $flags ) ) {
 217                                 return false;
 218                         }
 219
 220                         if( in_array( 'gzip', $flags ) ) {
 221                                 // This shouldn't happen, but a bug in the compress script
 222                                 // may at times gzip-compress a HistoryBlob object row.
 223                                 $obj = unserialize( gzinflate( $row->old_text ) );
 224                         } else {
 225                                 $obj = unserialize( $row->old_text );
 226                         }
 227
 228                         if( !is_object( $obj ) ) {
 229                                 // Correct for old double-serialization bug.
 230                                 $obj = unserialize( $obj );
 231                         }
 232
 233                         // Save this item for reference; if pulling many
 234                         // items in a row we'll likely use it again.
 235                         $obj->uncompress();
 236                         $wgBlobCache = array( $this->mOldId => $obj );
 237                 }
 238                 return $obj->getItem( $this->mHash );
 239         }
 240
 241         /**
 242          * Get the content hash
 243          */
 244         function getHash() {
 245                 return $this->mHash;
 246         }
 247 }
 248
 249
 250 /**
 251  * To speed up conversion from 1.4 to 1.5 schema, text rows can refer to the
 252  * leftover cur table as the backend. This avoids expensively copying hundreds
 253  * of megabytes of data during the conversion downtime.
 254  *
 255  * Serialized HistoryBlobCurStub objects will be inserted into the text table
 256  * on conversion if $wgFastSchemaUpgrades is set to true.
 257  */
 258 class HistoryBlobCurStub {
 259         var $mCurId;
 260
 261         /**
 262          * @param integer $curid The cur_id pointed to
 263          */
 264         function HistoryBlobCurStub( $curid = 0 ) {
 265                 $this->mCurId = $curid;
 266         }
 267
 268         /**
 269          * Sets the location (cur_id) of the main object to which this object
 270          * points
 271          */
 272         function setLocation( $id ) {
 273                 $this->mCurId = $id;
 274         }
 275
 276         function getText() {
 277                 $dbr = wfGetDB( DB_SLAVE );
 278                 $row = $dbr->selectRow( 'cur', array( 'cur_text' ), array( 'cur_id' => $this->mCurId ) );
 279                 if( !$row ) {
 280                         return false;
 281                 }
 282                 return $row->cur_text;
 283         }
 284 }
 285
 286 /**
 287  * Diff-based history compression
 288  * Requires xdiff 1.5+ and zlib
 289  */
 290 class DiffHistoryBlob implements HistoryBlob {
 291         /** Uncompressed item cache */
 292         var $mItems = array();
 293
 294         /**
 295          * Array of diffs, where $this->mDiffs[0] is the diff between
 296          * $this->mDiffs[0] and $this->mDiffs[1]
 297          */
 298         var $mDiffs = array();
 299
 300         /**
 301          * The key for getText()
 302          */
 303         var $mDefaultKey;
 304
 305         /**
 306          * Compressed storage
 307          */
 308         var $mCompressed;
 309
 310         /**
 311          * True if the object is locked against further writes
 312          */
 313         var $mFrozen = false;
 314
 315
 316         function __construct() {
 317                 if ( !function_exists( 'xdiff_string_bdiff' ) ){
 318                         throw new MWException( "Need xdiff 1.5+ support to read or write DiffHistoryBlob\n" );
 319                 }
 320                 if ( !function_exists( 'gzdeflate' ) ) {
 321                         throw new MWException( "Need zlib support to read or write DiffHistoryBlob\n" );
 322                 }
 323         }
 324
 325         function addItem( $text ) {
 326                 if ( $this->mFrozen ) {
 327                         throw new MWException( __METHOD__.": Cannot add more items after sleep/wakeup" );
 328                 }
 329
 330                 $this->mItems[] = $text;
 331                 $i = count( $this->mItems ) - 1;
 332                 if ( $i > 0 ) {
 333                         # Need to do a null concatenation with warnings off, due to bugs in the current version of xdiff
 334                         # "String is not zero-terminated"
 335                         wfSuppressWarnings();
 336                         $this->mDiffs[] = xdiff_string_bdiff( $this->mItems[$i-1], $text ) . '';
 337                         wfRestoreWarnings();
 338                 }
 339                 return $i;
 340         }
 341
 342         function getItem( $key ) {
 343                 if ( $key > count( $this->mDiffs ) + 1 ) {
 344                         return false;
 345                 }
 346                 $key = intval( $key );
 347                 if ( $key == 0 ) {
 348                         return $this->mItems[0];
 349                 }
 350
 351                 $last = count( $this->mItems ) - 1;
 352                 for ( $i = $last + 1; $i <= $key; $i++ ) {
 353                         # Need to do a null concatenation with warnings off, due to bugs in the current version of xdiff
 354                         # "String is not zero-terminated"
 355                         wfSuppressWarnings();
 356                         $this->mItems[$i] = xdiff_string_bpatch( $this->mItems[$i - 1], $this->mDiffs[$i - 1] ) . '';
 357                         wfRestoreWarnings();
 358                 }
 359                 return $this->mItems[$key];
 360         }
 361
 362         function setText( $text ) {
 363                 $this->mDefaultKey = $this->addItem( $text );
 364         }
 365
 366         function getText() {
 367                 return $this->getItem( $this->mDefaultKey );
 368         }
 369
 370         function __sleep() {
 371                 if ( !isset( $this->mItems[0] ) ) {
 372                         // Empty object
 373                         $info = false;
 374                 } else {
 375                         $info = array(
 376                                 'base' => $this->mItems[0],
 377                                 'diffs' => $this->mDiffs
 378                         );
 379                 }
 380                 if ( isset( $this->mDefaultKey ) ) {
 381                         $info['default'] = $this->mDefaultKey;
 382                 }
 383                 $this->mCompressed = gzdeflate( serialize( $info ) );
 384                 return array( 'mCompressed' );
 385         }
 386
 387         function __wakeup() {
 388                 // addItem() doesn't work if mItems is partially filled from mDiffs
 389                 $this->mFrozen = true;
 390                 $info = unserialize( gzinflate( $this->mCompressed ) );
 391                 unset( $this->mCompressed );
 392
 393                 if ( !$info ) {
 394                         // Empty object
 395                         return;
 396                 }
 397
 398                 if ( isset( $info['default'] ) ) {
 399                         $this->mDefaultKey = $info['default'];
 400                 }
 401                 $this->mItems[0] = $info['base'];
 402                 $this->mDiffs = $info['diffs'];
 403         }
 404 }