includes/BagOStuff.php

   1 <?php
   2 #
   3 # Copyright (C) 2003-2004 Brion Vibber <brion@pobox.com>
   4 # http://www.mediawiki.org/
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License along
  17 # with this program; if not, write to the Free Software Foundation, Inc.,
  18 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  19 # http://www.gnu.org/copyleft/gpl.html
  20 /**
  21  *
  22  */
  23
  24 /**
  25  * Simple generic object store
  26  *
  27  * interface is intended to be more or less compatible with
  28  * the PHP memcached client.
  29  *
  30  * backends for local hash array and SQL table included:
  31  * <code>
  32  *   $bag = new HashBagOStuff();
  33  *   $bag = new MysqlBagOStuff($tablename); # connect to db first
  34  * </code>
  35  *
  36  */
  37 class BagOStuff {
  38         var $debugmode;
  39
  40         function __construct() {
  41                 $this->set_debug( false );
  42         }
  43
  44         function set_debug($bool) {
  45                 $this->debugmode = $bool;
  46         }
  47
  48         /* *** THE GUTS OF THE OPERATION *** */
  49         /* Override these with functional things in subclasses */
  50
  51         function get($key) {
  52                 /* stub */
  53                 return false;
  54         }
  55
  56         function set($key, $value, $exptime=0) {
  57                 /* stub */
  58                 return false;
  59         }
  60
  61         function delete($key, $time=0) {
  62                 /* stub */
  63                 return false;
  64         }
  65
  66         function lock($key, $timeout = 0) {
  67                 /* stub */
  68                 return true;
  69         }
  70
  71         function unlock($key) {
  72                 /* stub */
  73                 return true;
  74         }
  75
  76         function keys() {
  77                 /* stub */
  78                 return array();
  79         }
  80
  81         /* *** Emulated functions *** */
  82         /* Better performance can likely be got with custom written versions */
  83         function get_multi($keys) {
  84                 $out = array();
  85                 foreach($keys as $key)
  86                         $out[$key] = $this->get($key);
  87                 return $out;
  88         }
  89
  90         function set_multi($hash, $exptime=0) {
  91                 foreach($hash as $key => $value)
  92                         $this->set($key, $value, $exptime);
  93         }
  94
  95         function add($key, $value, $exptime=0) {
  96                 if( $this->get($key) == false ) {
  97                         $this->set($key, $value, $exptime);
  98                         return true;
  99                 }
 100         }
 101
 102         function add_multi($hash, $exptime=0) {
 103                 foreach($hash as $key => $value)
 104                         $this->add($key, $value, $exptime);
 105         }
 106
 107         function delete_multi($keys, $time=0) {
 108                 foreach($keys as $key)
 109                         $this->delete($key, $time);
 110         }
 111
 112         function replace($key, $value, $exptime=0) {
 113                 if( $this->get($key) !== false )
 114                         $this->set($key, $value, $exptime);
 115         }
 116
 117         function incr($key, $value=1) {
 118                 if ( !$this->lock($key) ) {
 119                         return false;
 120                 }
 121                 $value = intval($value);
 122                 if($value < 0) $value = 0;
 123
 124                 $n = false;
 125                 if( ($n = $this->get($key)) !== false ) {
 126                         $n += $value;
 127                         $this->set($key, $n); // exptime?
 128                 }
 129                 $this->unlock($key);
 130                 return $n;
 131         }
 132
 133         function decr($key, $value=1) {
 134                 if ( !$this->lock($key) ) {
 135                         return false;
 136                 }
 137                 $value = intval($value);
 138                 if($value < 0) $value = 0;
 139
 140                 $m = false;
 141                 if( ($n = $this->get($key)) !== false ) {
 142                         $m = $n - $value;
 143                         if($m < 0) $m = 0;
 144                         $this->set($key, $m); // exptime?
 145                 }
 146                 $this->unlock($key);
 147                 return $m;
 148         }
 149
 150         function _debug($text) {
 151                 if($this->debugmode)
 152                         wfDebug("BagOStuff debug: $text\n");
 153         }
 154
 155         /**
 156          * Convert an optionally relative time to an absolute time
 157          */
 158         static function convertExpiry( $exptime ) {
 159                 if(($exptime != 0) && ($exptime < 3600*24*30)) {
 160                         return time() + $exptime;
 161                 } else {
 162                         return $exptime;
 163                 }
 164         }
 165 }
 166
 167
 168 /**
 169  * Functional versions!
 170  * @todo document
 171  */
 172 class HashBagOStuff extends BagOStuff {
 173         /*
 174            This is a test of the interface, mainly. It stores
 175            things in an associative array, which is not going to
 176            persist between program runs.
 177         */
 178         var $bag;
 179
 180         function __construct() {
 181                 $this->bag = array();
 182         }
 183
 184         function _expire($key) {
 185                 $et = $this->bag[$key][1];
 186                 if(($et == 0) || ($et > time()))
 187                         return false;
 188                 $this->delete($key);
 189                 return true;
 190         }
 191
 192         function get($key) {
 193                 if(!$this->bag[$key])
 194                         return false;
 195                 if($this->_expire($key))
 196                         return false;
 197                 return $this->bag[$key][0];
 198         }
 199
 200         function set($key,$value,$exptime=0) {
 201                 $this->bag[$key] = array( $value, BagOStuff::convertExpiry( $exptime ) );
 202         }
 203
 204         function delete($key,$time=0) {
 205                 if(!$this->bag[$key])
 206                         return false;
 207                 unset($this->bag[$key]);
 208                 return true;
 209         }
 210
 211         function keys() {
 212                 return array_keys( $this->bag );
 213         }
 214 }
 215
 216 /*
 217 CREATE TABLE objectcache (
 218   keyname char(255) binary not null default '',
 219   value mediumblob,
 220   exptime datetime,
 221   unique key (keyname),
 222   key (exptime)
 223 );
 224 */
 225
 226 /**
 227  * @todo document
 228  * @abstract
 229  */
 230 abstract class SqlBagOStuff extends BagOStuff {
 231         var $table;
 232         var $lastexpireall = 0;
 233
 234         function __construct($tablename = 'objectcache') {
 235                 $this->table = $tablename;
 236         }
 237
 238         function get($key) {
 239                 /* expire old entries if any */
 240                 $this->garbageCollect();
 241
 242                 $res = $this->_query(
 243                         "SELECT value,exptime FROM $0 WHERE keyname='$1'", $key);
 244                 if(!$res) {
 245                         $this->_debug("get: ** error: " . $this->_dberror($res) . " **");
 246                         return false;
 247                 }
 248                 if($row=$this->_fetchobject($res)) {
 249                         $this->_debug("get: retrieved data; exp time is " . $row->exptime);
 250                         if ( $row->exptime != $this->_maxdatetime() &&
 251                           wfTimestamp( TS_UNIX, $row->exptime ) < time() )
 252                         {
 253                                 $this->_debug("get: key has expired, deleting");
 254                                 $this->delete($key);
 255                                 return false;
 256                         }
 257                         return $this->_unserialize($this->_blobdecode($row->value));
 258                 } else {
 259                         $this->_debug('get: no matching rows');
 260                 }
 261                 return false;
 262         }
 263
 264         function set($key,$value,$exptime=0) {
 265                 if ( wfReadOnly() ) {
 266                         return false;
 267                 }
 268                 $exptime = intval($exptime);
 269                 if($exptime < 0) $exptime = 0;
 270                 if($exptime == 0) {
 271                         $exp = $this->_maxdatetime();
 272                 } else {
 273                         if($exptime < 3.16e8) # ~10 years
 274                                 $exptime += time();
 275                         $exp = $this->_fromunixtime($exptime);
 276                 }
 277                 $this->delete( $key );
 278                 $this->_doinsert($this->getTableName(), array(
 279                                         'keyname' => $key,
 280                                         'value' => $this->_blobencode($this->_serialize($value)),
 281                                         'exptime' => $exp
 282                                 ));
 283                 return true; /* ? */
 284         }
 285
 286         function delete($key,$time=0) {
 287                 if ( wfReadOnly() ) {
 288                         return false;
 289                 }
 290                 $this->_query(
 291                         "DELETE FROM $0 WHERE keyname='$1'", $key );
 292                 return true; /* ? */
 293         }
 294
 295         function keys() {
 296                 $res = $this->_query( "SELECT keyname FROM $0" );
 297                 if(!$res) {
 298                         $this->_debug("keys: ** error: " . $this->_dberror($res) . " **");
 299                         return array();
 300                 }
 301                 $result = array();
 302                 while( $row = $this->_fetchobject($res) ) {
 303                         $result[] = $row->keyname;
 304                 }
 305                 return $result;
 306         }
 307
 308         function getTableName() {
 309                 return $this->table;
 310         }
 311
 312         function _query($sql) {
 313                 $reps = func_get_args();
 314                 $reps[0] = $this->getTableName();
 315                 // ewwww
 316                 for($i=0;$i<count($reps);$i++) {
 317                         $sql = str_replace(
 318                                 '$' . $i,
 319                                 $i > 0 ? $this->_strencode($reps[$i]) : $reps[$i],
 320                                 $sql);
 321                 }
 322                 $res = $this->_doquery($sql);
 323                 if($res == false) {
 324                         $this->_debug('query failed: ' . $this->_dberror($res));
 325                 }
 326                 return $res;
 327         }
 328
 329         function _strencode($str) {
 330                 /* Protect strings in SQL */
 331                 return str_replace( "'", "''", $str );
 332         }
 333         function _blobencode($str) {
 334                 return $str;
 335         }
 336         function _blobdecode($str) {
 337                 return $str;
 338         }
 339
 340         abstract function _doinsert($table, $vals);
 341         abstract function _doquery($sql);
 342
 343         function _freeresult($result) {
 344                 /* stub */
 345                 return false;
 346         }
 347
 348         function _dberror($result) {
 349                 /* stub */
 350                 return 'unknown error';
 351         }
 352
 353         abstract function _maxdatetime();
 354         abstract function _fromunixtime($ts);
 355
 356         function garbageCollect() {
 357                 /* Ignore 99% of requests */
 358                 if ( !mt_rand( 0, 100 ) ) {
 359                         $nowtime = time();
 360                         /* Avoid repeating the delete within a few seconds */
 361                         if ( $nowtime > ($this->lastexpireall + 1) ) {
 362                                 $this->lastexpireall = $nowtime;
 363                                 $this->expireall();
 364                         }
 365                 }
 366         }
 367
 368         function expireall() {
 369                 /* Remove any items that have expired */
 370                 if ( wfReadOnly() ) {
 371                         return false;
 372                 }
 373                 $now = $this->_fromunixtime( time() );
 374                 $this->_query( "DELETE FROM $0 WHERE exptime < '$now'" );
 375         }
 376
 377         function deleteall() {
 378                 /* Clear *all* items from cache table */
 379                 if ( wfReadOnly() ) {
 380                         return false;
 381                 }
 382                 $this->_query( "DELETE FROM $0" );
 383         }
 384
 385         /**
 386          * Serialize an object and, if possible, compress the representation.
 387          * On typical message and page data, this can provide a 3X decrease
 388          * in storage requirements.
 389          *
 390          * @param mixed $data
 391          * @return string
 392          */
 393         function _serialize( &$data ) {
 394                 $serial = serialize( $data );
 395                 if( function_exists( 'gzdeflate' ) ) {
 396                         return gzdeflate( $serial );
 397                 } else {
 398                         return $serial;
 399                 }
 400         }
 401
 402         /**
 403          * Unserialize and, if necessary, decompress an object.
 404          * @param string $serial
 405          * @return mixed
 406          */
 407         function _unserialize( $serial ) {
 408                 if( function_exists( 'gzinflate' ) ) {
 409                         $decomp = @gzinflate( $serial );
 410                         if( false !== $decomp ) {
 411                                 $serial = $decomp;
 412                         }
 413                 }
 414                 $ret = unserialize( $serial );
 415                 return $ret;
 416         }
 417 }
 418
 419 /**
 420  * @todo document
 421  */
 422 class MediaWikiBagOStuff extends SqlBagOStuff {
 423         var $tableInitialised = false;
 424
 425         function _doquery($sql) {
 426                 $dbw = wfGetDB( DB_MASTER );
 427                 return $dbw->query($sql, 'MediaWikiBagOStuff::_doquery');
 428         }
 429         function _doinsert($t, $v) {
 430                 $dbw = wfGetDB( DB_MASTER );
 431                 return $dbw->insert($t, $v, 'MediaWikiBagOStuff::_doinsert',
 432                         array( 'IGNORE' ) );
 433         }
 434         function _fetchobject($result) {
 435                 $dbw = wfGetDB( DB_MASTER );
 436                 return $dbw->fetchObject($result);
 437         }
 438         function _freeresult($result) {
 439                 $dbw = wfGetDB( DB_MASTER );
 440                 return $dbw->freeResult($result);
 441         }
 442         function _dberror($result) {
 443                 $dbw = wfGetDB( DB_MASTER );
 444                 return $dbw->lastError();
 445         }
 446         function _maxdatetime() {
 447                 if ( time() > 0x7fffffff ) {
 448                         return $this->_fromunixtime( 1<<62 );
 449                 } else {
 450                         return $this->_fromunixtime( 0x7fffffff );
 451                 }
 452         }
 453         function _fromunixtime($ts) {
 454                 $dbw = wfGetDB(DB_MASTER);
 455                 return $dbw->timestamp($ts);
 456         }
 457         function _strencode($s) {
 458                 $dbw = wfGetDB( DB_MASTER );
 459                 return $dbw->strencode($s);
 460         }
 461         function _blobencode($s) {
 462                 $dbw = wfGetDB( DB_MASTER );
 463                 return $dbw->encodeBlob($s);
 464         }
 465         function _blobdecode($s) {
 466                 $dbw = wfGetDB( DB_MASTER );
 467                 return $dbw->decodeBlob($s);
 468         }
 469         function getTableName() {
 470                 if ( !$this->tableInitialised ) {
 471                         $dbw = wfGetDB( DB_MASTER );
 472                         /* This is actually a hack, we should be able
 473                            to use Language classes here... or not */
 474                         if (!$dbw)
 475                                 throw new MWException("Could not connect to database");
 476                         $this->table = $dbw->tableName( $this->table );
 477                         $this->tableInitialised = true;
 478                 }
 479                 return $this->table;
 480         }
 481 }
 482
 483 /**
 484  * This is a wrapper for Turck MMCache's shared memory functions.
 485  *
 486  * You can store objects with mmcache_put() and mmcache_get(), but Turck seems
 487  * to use a weird custom serializer that randomly segfaults. So we wrap calls
 488  * with serialize()/unserialize().
 489  *
 490  * The thing I noticed about the Turck serialized data was that unlike ordinary
 491  * serialize(), it contained the names of methods, and judging by the amount of
 492  * binary data, perhaps even the bytecode of the methods themselves. It may be
 493  * that Turck's serializer is faster, so a possible future extension would be
 494  * to use it for arrays but not for objects.
 495  *
 496  */
 497 class TurckBagOStuff extends BagOStuff {
 498         function get($key) {
 499                 $val = mmcache_get( $key );
 500                 if ( is_string( $val ) ) {
 501                         $val = unserialize( $val );
 502                 }
 503                 return $val;
 504         }
 505
 506         function set($key, $value, $exptime=0) {
 507                 mmcache_put( $key, serialize( $value ), $exptime );
 508                 return true;
 509         }
 510
 511         function delete($key, $time=0) {
 512                 mmcache_rm( $key );
 513                 return true;
 514         }
 515
 516         function lock($key, $waitTimeout = 0 ) {
 517                 mmcache_lock( $key );
 518                 return true;
 519         }
 520
 521         function unlock($key) {
 522                 mmcache_unlock( $key );
 523                 return true;
 524         }
 525 }
 526
 527 /**
 528  * This is a wrapper for APC's shared memory functions
 529  *
 530  */
 531 class APCBagOStuff extends BagOStuff {
 532         function get($key) {
 533                 $val = apc_fetch($key);
 534                 if ( is_string( $val ) ) {
 535                         $val = unserialize( $val );
 536                 }
 537                 return $val;
 538         }
 539
 540         function set($key, $value, $exptime=0) {
 541                 apc_store($key, serialize($value), $exptime);
 542                 return true;
 543         }
 544
 545         function delete($key, $time=0) {
 546                 apc_delete($key);
 547                 return true;
 548         }
 549 }
 550
 551
 552 /**
 553  * This is a wrapper for eAccelerator's shared memory functions.
 554  *
 555  * This is basically identical to the Turck MMCache version,
 556  * mostly because eAccelerator is based on Turck MMCache.
 557  *
 558  */
 559 class eAccelBagOStuff extends BagOStuff {
 560         function get($key) {
 561                 $val = eaccelerator_get( $key );
 562                 if ( is_string( $val ) ) {
 563                         $val = unserialize( $val );
 564                 }
 565                 return $val;
 566         }
 567
 568         function set($key, $value, $exptime=0) {
 569                 eaccelerator_put( $key, serialize( $value ), $exptime );
 570                 return true;
 571         }
 572
 573         function delete($key, $time=0) {
 574                 eaccelerator_rm( $key );
 575                 return true;
 576         }
 577
 578         function lock($key, $waitTimeout = 0 ) {
 579                 eaccelerator_lock( $key );
 580                 return true;
 581         }
 582
 583         function unlock($key) {
 584                 eaccelerator_unlock( $key );
 585                 return true;
 586         }
 587 }
 588
 589 /**
 590  * Wrapper for XCache object caching functions; identical interface
 591  * to the APC wrapper
 592  */
 593 class XCacheBagOStuff extends BagOStuff {
 594
 595         /**
 596          * Get a value from the XCache object cache
 597          *
 598          * @param string $key Cache key
 599          * @return mixed
 600          */
 601         public function get( $key ) {
 602                 $val = xcache_get( $key );
 603                 if( is_string( $val ) )
 604                         $val = unserialize( $val );
 605                 return $val;
 606         }
 607
 608         /**
 609          * Store a value in the XCache object cache
 610          *
 611          * @param string $key Cache key
 612          * @param mixed $value Object to store
 613          * @param int $expire Expiration time
 614          * @return bool
 615          */
 616         public function set( $key, $value, $expire = 0 ) {
 617                 xcache_set( $key, serialize( $value ), $expire );
 618                 return true;
 619         }
 620
 621         /**
 622          * Remove a value from the XCache object cache
 623          *
 624          * @param string $key Cache key
 625          * @param int $time Not used in this implementation
 626          * @return bool
 627          */
 628         public function delete( $key, $time = 0 ) {
 629                 xcache_unset( $key );
 630                 return true;
 631         }
 632
 633 }
 634
 635 /**
 636  * @todo document
 637  */
 638 class DBABagOStuff extends BagOStuff {
 639         var $mHandler, $mFile, $mReader, $mWriter, $mDisabled;
 640
 641         function __construct( $handler = 'db3', $dir = false ) {
 642                 if ( $dir === false ) {
 643                         global $wgTmpDirectory;
 644                         $dir = $wgTmpDirectory;
 645                 }
 646                 $this->mFile = "$dir/mw-cache-" . wfWikiID();
 647                 $this->mFile .= '.db';
 648                 wfDebug( __CLASS__.": using cache file {$this->mFile}\n" );
 649                 $this->mHandler = $handler;
 650         }
 651
 652         /**
 653          * Encode value and expiry for storage
 654          */
 655         function encode( $value, $expiry ) {
 656                 # Convert to absolute time
 657                 $expiry = BagOStuff::convertExpiry( $expiry );
 658                 return sprintf( '%010u', intval( $expiry ) ) . ' ' . serialize( $value );
 659         }
 660
 661         /**
 662          * @return list containing value first and expiry second
 663          */
 664         function decode( $blob ) {
 665                 if ( !is_string( $blob ) ) {
 666                         return array( null, 0 );
 667                 } else {
 668                         return array(
 669                                 unserialize( substr( $blob, 11 ) ),
 670                                 intval( substr( $blob, 0, 10 ) )
 671                         );
 672                 }
 673         }
 674
 675         function getReader() {
 676                 if ( file_exists( $this->mFile ) ) {
 677                         $handle = dba_open( $this->mFile, 'rl', $this->mHandler );
 678                 } else {
 679                         $handle = $this->getWriter();
 680                 }
 681                 if ( !$handle ) {
 682                         wfDebug( "Unable to open DBA cache file {$this->mFile}\n" );
 683                 }
 684                 return $handle;
 685         }
 686
 687         function getWriter() {
 688                 $handle = dba_open( $this->mFile, 'cl', $this->mHandler );
 689                 if ( !$handle ) {
 690                         wfDebug( "Unable to open DBA cache file {$this->mFile}\n" );
 691                 }
 692                 return $handle;
 693         }
 694
 695         function get( $key ) {
 696                 wfProfileIn( __METHOD__ );
 697                 wfDebug( __METHOD__."($key)\n" );
 698                 $handle = $this->getReader();
 699                 if ( !$handle ) {
 700                         return null;
 701                 }
 702                 $val = dba_fetch( $key, $handle );
 703                 list( $val, $expiry ) = $this->decode( $val );
 704                 # Must close ASAP because locks are held
 705                 dba_close( $handle );
 706
 707                 if ( !is_null( $val ) && $expiry && $expiry < time() ) {
 708                         # Key is expired, delete it
 709                         $handle = $this->getWriter();
 710                         dba_delete( $key, $handle );
 711                         dba_close( $handle );
 712                         wfDebug( __METHOD__.": $key expired\n" );
 713                         $val = null;
 714                 }
 715                 wfProfileOut( __METHOD__ );
 716                 return $val;
 717         }
 718
 719         function set( $key, $value, $exptime=0 ) {
 720                 wfProfileIn( __METHOD__ );
 721                 wfDebug( __METHOD__."($key)\n" );
 722                 $blob = $this->encode( $value, $exptime );
 723                 $handle = $this->getWriter();
 724                 if ( !$handle ) {
 725                         return false;
 726                 }
 727                 $ret = dba_replace( $key, $blob, $handle );
 728                 dba_close( $handle );
 729                 wfProfileOut( __METHOD__ );
 730                 return $ret;
 731         }
 732
 733         function delete( $key, $time = 0 ) {
 734                 wfProfileIn( __METHOD__ );
 735                 wfDebug( __METHOD__."($key)\n" );
 736                 $handle = $this->getWriter();
 737                 if ( !$handle ) {
 738                         return false;
 739                 }
 740                 $ret = dba_delete( $key, $handle );
 741                 dba_close( $handle );
 742                 wfProfileOut( __METHOD__ );
 743                 return $ret;
 744         }
 745
 746         function add( $key, $value, $exptime = 0 ) {
 747                 wfProfileIn( __METHOD__ );
 748                 $blob = $this->encode( $value, $exptime );
 749                 $handle = $this->getWriter();
 750                 if ( !$handle ) {
 751                         return false;
 752                 }
 753                 $ret = dba_insert( $key, $blob, $handle );
 754                 # Insert failed, check to see if it failed due to an expired key
 755                 if ( !$ret ) {
 756                         list( $value, $expiry ) = $this->decode( dba_fetch( $key, $handle ) );
 757                         if ( $expiry < time() ) {
 758                                 # Yes expired, delete and try again
 759                                 dba_delete( $key, $handle );
 760                                 $ret = dba_insert( $key, $blob, $handle );
 761                                 # This time if it failed then it will be handled by the caller like any other race
 762                         }
 763                 }
 764
 765                 dba_close( $handle );
 766                 wfProfileOut( __METHOD__ );
 767                 return $ret;
 768         }
 769
 770         function keys() {
 771                 $reader = $this->getReader();
 772                 $k1 = dba_firstkey( $reader );
 773                 if( !$k1 ) {
 774                         return array();
 775                 }
 776                 $result[] = $k1;
 777                 while( $key = dba_nextkey( $reader ) ) {
 778                         $result[] = $key;
 779                 }
 780                 return $result;
 781         }
 782 }
 783
 784