includes/PoolCounter.php

   1 <?php
   2 /**
   3  * Provides of semaphore semantics for restricting the number
   4  * of workers that may be concurrently performing the same task.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License along
  17  * with this program; if not, write to the Free Software Foundation, Inc.,
  18  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  19  * http://www.gnu.org/copyleft/gpl.html
  20  *
  21  * @file
  22  */
  23
  24 /**
  25  * When you have many workers (threads/servers) giving service, and a
  26  * cached item expensive to produce expires, you may get several workers
  27  * doing the job at the same time.
  28  *
  29  * Given enough requests and the item expiring fast (non-cacheable,
  30  * lots of edits...) that single work can end up unfairly using most (all)
  31  * of the cpu of the pool. This is also known as 'Michael Jackson effect'
  32  * since this effect triggered on the english wikipedia on the day Michael
  33  * Jackson died, the biographical article got hit with several edits per
  34  * minutes and hundreds of read hits.
  35  *
  36  * The PoolCounter provides semaphore semantics for restricting the number
  37  * of workers that may be concurrently performing such single task.
  38  *
  39  * By default PoolCounter_Stub is used, which provides no locking. You
  40  * can get a useful one in the PoolCounter extension.
  41  */
  42 abstract class PoolCounter {
  43         /* Return codes */
  44         const LOCKED = 1; /* Lock acquired */
  45         const RELEASED = 2; /* Lock released */
  46         const DONE = 3; /* Another worker did the work for you */
  47
  48         const ERROR = -1; /* Indeterminate error */
  49         const NOT_LOCKED = -2; /* Called release() with no lock held */
  50         const QUEUE_FULL = -3; /* There are already maxqueue workers on this lock */
  51         const TIMEOUT = -4; /* Timeout exceeded */
  52         const LOCK_HELD = -5; /* Cannot acquire another lock while you have one lock held */
  53
  54         /** @var string All workers with the same key share the lock */
  55         protected $key;
  56         /** @var integer Maximum number of workers doing the task simultaneously */
  57         protected $workers;
  58         /** @var integer If this number of workers are already working/waiting, fail instead of wait */
  59         protected $maxqueue;
  60         /** @var float Maximum time in seconds to wait for the lock */
  61         protected $timeout;
  62
  63         /**
  64          * @param array $conf
  65          * @param string $type
  66          * @param string $key
  67          */
  68         protected function __construct( $conf, $type, $key ) {
  69                 $this->key = $key;
  70                 $this->workers = $conf['workers'];
  71                 $this->maxqueue = $conf['maxqueue'];
  72                 $this->timeout = $conf['timeout'];
  73         }
  74
  75         /**
  76          * Create a Pool counter. This should only be called from the PoolWorks.
  77          *
  78          * @param $type
  79          * @param $key
  80          *
  81          * @return PoolCounter
  82          */
  83         public static function factory( $type, $key ) {
  84                 global $wgPoolCounterConf;
  85                 if ( !isset( $wgPoolCounterConf[$type] ) ) {
  86                         return new PoolCounter_Stub;
  87                 }
  88                 $conf = $wgPoolCounterConf[$type];
  89                 $class = $conf['class'];
  90
  91                 return new $class( $conf, $type, $key );
  92         }
  93
  94         /**
  95          * I want to do this task and I need to do it myself.
  96          *
  97          * @return Status Value is one of Locked/Error
  98          */
  99         abstract public function acquireForMe();
 100
 101         /**
 102          * I want to do this task, but if anyone else does it
 103          * instead, it's also fine for me. I will read its cached data.
 104          *
 105          * @return Status Value is one of Locked/Done/Error
 106          */
 107         abstract public function acquireForAnyone();
 108
 109         /**
 110          * I have successfully finished my task.
 111          * Lets another one grab the lock, and returns the workers
 112          * waiting on acquireForAnyone()
 113          *
 114          * @return Status value is one of Released/NotLocked/Error
 115          */
 116         abstract public function release();
 117 }
 118
 119 class PoolCounter_Stub extends PoolCounter {
 120         public function __construct() {
 121                 /* No parameters needed */
 122         }
 123
 124         public function acquireForMe() {
 125                 return Status::newGood( PoolCounter::LOCKED );
 126         }
 127
 128         public function acquireForAnyone() {
 129                 return Status::newGood( PoolCounter::LOCKED );
 130         }
 131
 132         public function release() {
 133                 return Status::newGood( PoolCounter::RELEASED );
 134         }
 135 }
 136
 137 /**
 138  * Class for dealing with PoolCounters using class members
 139  */
 140 abstract class PoolCounterWork {
 141         protected $cacheable = false; //Does this override getCachedWork() ?
 142
 143         /**
 144          * @param string $type The type of PoolCounter to use
 145          * @param string $key Key that identifies the queue this work is placed on
 146          */
 147         public function __construct( $type, $key ) {
 148                 $this->poolCounter = PoolCounter::factory( $type, $key );
 149         }
 150
 151         /**
 152          * Actually perform the work, caching it if needed
 153          * @return mixed work result or false
 154          */
 155         abstract public function doWork();
 156
 157         /**
 158          * Retrieve the work from cache
 159          * @return mixed work result or false
 160          */
 161         public function getCachedWork() {
 162                 return false;
 163         }
 164
 165         /**
 166          * A work not so good (eg. expired one) but better than an error
 167          * message.
 168          * @return mixed work result or false
 169          */
 170         public function fallback() {
 171                 return false;
 172         }
 173
 174         /**
 175          * Do something with the error, like showing it to the user.
 176          * @return bool
 177          */
 178         function error( $status ) {
 179                 return false;
 180         }
 181
 182         /**
 183          * Log an error
 184          *
 185          * @param $status Status
 186          * @return void
 187          */
 188         function logError( $status ) {
 189                 wfDebugLog( 'poolcounter', $status->getWikiText() );
 190         }
 191
 192         /**
 193          * Get the result of the work (whatever it is), or the result of the error() function.
 194          * This returns the result of the first applicable method that returns a non-false value,
 195          * where the methods are checked in the following order:
 196          *   - a) doWork()       : Applies if the work is exclusive or no another process
 197          *                         is doing it, and on the condition that either this process
 198          *                         successfully entered the pool or the pool counter is down.
 199          *   - b) doCachedWork() : Applies if the work is cacheable and this blocked on another
 200          *                         process which finished the work.
 201          *   - c) fallback()     : Applies for all remaining cases.
 202          * If these all fall through (by returning false), then the result of error() is returned.
 203          *
 204          * @param $skipcache bool
 205          * @return mixed
 206          */
 207         public function execute( $skipcache = false ) {
 208                 if ( $this->cacheable && !$skipcache ) {
 209                         $status = $this->poolCounter->acquireForAnyone();
 210                 } else {
 211                         $status = $this->poolCounter->acquireForMe();
 212                 }
 213
 214                 if ( !$status->isOK() ) {
 215                         // Respond gracefully to complete server breakage: just log it and do the work
 216                         $this->logError( $status );
 217                         return $this->doWork();
 218                 }
 219
 220                 switch ( $status->value ) {
 221                         case PoolCounter::LOCKED:
 222                                 $result = $this->doWork();
 223                                 $this->poolCounter->release();
 224                                 return $result;
 225
 226                         case PoolCounter::DONE:
 227                                 $result = $this->getCachedWork();
 228                                 if ( $result === false ) {
 229                                         /* That someone else work didn't serve us.
 230                                          * Acquire the lock for me
 231                                          */
 232                                         return $this->execute( true );
 233                                 }
 234                                 return $result;
 235
 236                         case PoolCounter::QUEUE_FULL:
 237                         case PoolCounter::TIMEOUT:
 238                                 $result = $this->fallback();
 239
 240                                 if ( $result !== false ) {
 241                                         return $result;
 242                                 }
 243                                 /* no break */
 244
 245                         /* These two cases should never be hit... */
 246                         case PoolCounter::ERROR:
 247                         default:
 248                                 $errors = array(
 249                                         PoolCounter::QUEUE_FULL => 'pool-queuefull',
 250                                         PoolCounter::TIMEOUT => 'pool-timeout' );
 251
 252                                 $status = Status::newFatal( isset( $errors[$status->value] )
 253                                         ? $errors[$status->value]
 254                                         : 'pool-errorunknown' );
 255                                 $this->logError( $status );
 256                                 return $this->error( $status );
 257                 }
 258         }
 259 }
 260
 261 /**
 262  * Convenience class for dealing with PoolCounters using callbacks
 263  * @since 1.22
 264  */
 265 class PoolCounterWorkViaCallback extends PoolCounterWork {
 266         /** @var callable */
 267         protected $doWork;
 268         /** @var callable|null */
 269         protected $doCachedWork;
 270         /** @var callable|null */
 271         protected $fallback;
 272         /** @var callable|null */
 273         protected $error;
 274
 275         /**
 276          * Build a PoolCounterWork class from a type, key, and callback map.
 277          *
 278          * The callback map must at least have a callback for the 'doWork' method.
 279          * Additionally, callbacks can be provided for the 'doCachedWork', 'fallback',
 280          * and 'error' methods. Methods without callbacks will be no-ops that return false.
 281          * If a 'doCachedWork' callback is provided, then execute() may wait for any prior
 282          * process in the pool to finish and reuse its cached result.
 283          *
 284          * @param string $type
 285          * @param string $key
 286          * @param array $callbacks Map of callbacks
 287          * @throws MWException
 288          */
 289         public function __construct( $type, $key, array $callbacks ) {
 290                 parent::__construct( $type, $key );
 291                 foreach ( array( 'doWork', 'doCachedWork', 'fallback', 'error' ) as $name ) {
 292                         if ( isset( $callbacks[$name] ) ) {
 293                                 if ( !is_callable( $callbacks[$name] ) ) {
 294                                         throw new MWException( "Invalid callback provided for '$name' function." );
 295                                 }
 296                                 $this->$name = $callbacks[$name];
 297                         }
 298                 }
 299                 if ( !isset( $this->doWork ) ) {
 300                         throw new MWException( "No callback provided for 'doWork' function." );
 301                 }
 302                 $this->cacheable = isset( $this->doCachedWork );
 303         }
 304
 305         public function doWork() {
 306                 return call_user_func_array( $this->doWork, array() );
 307         }
 308
 309         public function getCachedWork() {
 310                 if ( $this->doCachedWork ) {
 311                         return call_user_func_array( $this->doCachedWork, array() );
 312                 }
 313                 return false;
 314         }
 315
 316         function fallback() {
 317                 if ( $this->fallback ) {
 318                         return call_user_func_array( $this->fallback, array() );
 319                 }
 320                 return false;
 321         }
 322
 323         function error( $status ) {
 324                 if ( $this->error ) {
 325                         return call_user_func_array( $this->error, array( $status ) );
 326                 }
 327                 return false;
 328         }
 329 }