3 * Provides of semaphore semantics for restricting the number
4 * of workers that may be concurrently performing the same task.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License along
17 * with this program; if not, write to the Free Software Foundation, Inc.,
18 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 * http://www.gnu.org/copyleft/gpl.html
25 * When you have many workers (threads/servers) giving service, and a
26 * cached item expensive to produce expires, you may get several workers
27 * doing the job at the same time.
29 * Given enough requests and the item expiring fast (non-cacheable,
30 * lots of edits...) that single work can end up unfairly using most (all)
31 * of the cpu of the pool. This is also known as 'Michael Jackson effect'
32 * since this effect triggered on the english wikipedia on the day Michael
33 * Jackson died, the biographical article got hit with several edits per
34 * minutes and hundreds of read hits.
36 * The PoolCounter provides semaphore semantics for restricting the number
37 * of workers that may be concurrently performing such single task.
39 * By default PoolCounter_Stub is used, which provides no locking. You
40 * can get a useful one in the PoolCounter extension.
42 abstract class PoolCounter
{
44 const LOCKED
= 1; /* Lock acquired */
45 const RELEASED
= 2; /* Lock released */
46 const DONE
= 3; /* Another worker did the work for you */
48 const ERROR
= -1; /* Indeterminate error */
49 const NOT_LOCKED
= -2; /* Called release() with no lock held */
50 const QUEUE_FULL
= -3; /* There are already maxqueue workers on this lock */
51 const TIMEOUT
= -4; /* Timeout exceeded */
52 const LOCK_HELD
= -5; /* Cannot acquire another lock while you have one lock held */
54 /** @var string All workers with the same key share the lock */
56 /** @var integer Maximum number of workers doing the task simultaneously */
58 /** @var integer If this number of workers are already working/waiting, fail instead of wait */
60 /** @var float Maximum time in seconds to wait for the lock */
68 protected function __construct( $conf, $type, $key ) {
70 $this->workers
= $conf['workers'];
71 $this->maxqueue
= $conf['maxqueue'];
72 $this->timeout
= $conf['timeout'];
76 * Create a Pool counter. This should only be called from the PoolWorks.
83 public static function factory( $type, $key ) {
84 global $wgPoolCounterConf;
85 if ( !isset( $wgPoolCounterConf[$type] ) ) {
86 return new PoolCounter_Stub
;
88 $conf = $wgPoolCounterConf[$type];
89 $class = $conf['class'];
91 return new $class( $conf, $type, $key );
97 public function getKey() {
102 * I want to do this task and I need to do it myself.
104 * @return Status Value is one of Locked/Error
106 abstract public function acquireForMe();
109 * I want to do this task, but if anyone else does it
110 * instead, it's also fine for me. I will read its cached data.
112 * @return Status Value is one of Locked/Done/Error
114 abstract public function acquireForAnyone();
117 * I have successfully finished my task.
118 * Lets another one grab the lock, and returns the workers
119 * waiting on acquireForAnyone()
121 * @return Status value is one of Released/NotLocked/Error
123 abstract public function release();
126 class PoolCounter_Stub
extends PoolCounter
{
127 public function __construct() {
128 /* No parameters needed */
131 public function acquireForMe() {
132 return Status
::newGood( PoolCounter
::LOCKED
);
135 public function acquireForAnyone() {
136 return Status
::newGood( PoolCounter
::LOCKED
);
139 public function release() {
140 return Status
::newGood( PoolCounter
::RELEASED
);
145 * Class for dealing with PoolCounters using class members
147 abstract class PoolCounterWork
{
148 protected $cacheable = false; //Does this override getCachedWork() ?
151 * @param string $type The type of PoolCounter to use
152 * @param string $key Key that identifies the queue this work is placed on
154 public function __construct( $type, $key ) {
155 $this->poolCounter
= PoolCounter
::factory( $type, $key );
159 * Actually perform the work, caching it if needed
160 * @return mixed work result or false
162 abstract public function doWork();
165 * Retrieve the work from cache
166 * @return mixed work result or false
168 public function getCachedWork() {
173 * A work not so good (eg. expired one) but better than an error
175 * @return mixed work result or false
177 public function fallback() {
182 * Do something with the error, like showing it to the user.
185 public function error( $status ) {
192 * @param $status Status
195 public function logError( $status ) {
196 $key = $this->poolCounter
->getKey();
198 wfDebugLog( 'poolcounter', "Pool key '$key': "
199 . $status->getMessage()->inLanguage( 'en' )->useDatabase( false )->text() );
203 * Get the result of the work (whatever it is), or the result of the error() function.
204 * This returns the result of the first applicable method that returns a non-false value,
205 * where the methods are checked in the following order:
206 * - a) doWork() : Applies if the work is exclusive or no another process
207 * is doing it, and on the condition that either this process
208 * successfully entered the pool or the pool counter is down.
209 * - b) doCachedWork() : Applies if the work is cacheable and this blocked on another
210 * process which finished the work.
211 * - c) fallback() : Applies for all remaining cases.
212 * If these all fall through (by returning false), then the result of error() is returned.
214 * @param $skipcache bool
217 public function execute( $skipcache = false ) {
218 if ( $this->cacheable
&& !$skipcache ) {
219 $status = $this->poolCounter
->acquireForAnyone();
221 $status = $this->poolCounter
->acquireForMe();
224 if ( !$status->isOK() ) {
225 // Respond gracefully to complete server breakage: just log it and do the work
226 $this->logError( $status );
227 return $this->doWork();
230 switch ( $status->value
) {
231 case PoolCounter
::LOCKED
:
232 $result = $this->doWork();
233 $this->poolCounter
->release();
236 case PoolCounter
::DONE
:
237 $result = $this->getCachedWork();
238 if ( $result === false ) {
239 /* That someone else work didn't serve us.
240 * Acquire the lock for me
242 return $this->execute( true );
246 case PoolCounter
::QUEUE_FULL
:
247 case PoolCounter
::TIMEOUT
:
248 $result = $this->fallback();
250 if ( $result !== false ) {
255 /* These two cases should never be hit... */
256 case PoolCounter
::ERROR
:
259 PoolCounter
::QUEUE_FULL
=> 'pool-queuefull',
260 PoolCounter
::TIMEOUT
=> 'pool-timeout' );
262 $status = Status
::newFatal( isset( $errors[$status->value
] )
263 ?
$errors[$status->value
]
264 : 'pool-errorunknown' );
265 $this->logError( $status );
266 return $this->error( $status );
272 * Convenience class for dealing with PoolCounters using callbacks
275 class PoolCounterWorkViaCallback
extends PoolCounterWork
{
278 /** @var callable|null */
279 protected $doCachedWork;
280 /** @var callable|null */
282 /** @var callable|null */
286 * Build a PoolCounterWork class from a type, key, and callback map.
288 * The callback map must at least have a callback for the 'doWork' method.
289 * Additionally, callbacks can be provided for the 'doCachedWork', 'fallback',
290 * and 'error' methods. Methods without callbacks will be no-ops that return false.
291 * If a 'doCachedWork' callback is provided, then execute() may wait for any prior
292 * process in the pool to finish and reuse its cached result.
294 * @param string $type
296 * @param array $callbacks Map of callbacks
297 * @throws MWException
299 public function __construct( $type, $key, array $callbacks ) {
300 parent
::__construct( $type, $key );
301 foreach ( array( 'doWork', 'doCachedWork', 'fallback', 'error' ) as $name ) {
302 if ( isset( $callbacks[$name] ) ) {
303 if ( !is_callable( $callbacks[$name] ) ) {
304 throw new MWException( "Invalid callback provided for '$name' function." );
306 $this->$name = $callbacks[$name];
309 if ( !isset( $this->doWork
) ) {
310 throw new MWException( "No callback provided for 'doWork' function." );
312 $this->cacheable
= isset( $this->doCachedWork
);
315 public function doWork() {
316 return call_user_func_array( $this->doWork
, array() );
319 public function getCachedWork() {
320 if ( $this->doCachedWork
) {
321 return call_user_func_array( $this->doCachedWork
, array() );
326 public function fallback() {
327 if ( $this->fallback
) {
328 return call_user_func_array( $this->fallback
, array() );
333 public function error( $status ) {
334 if ( $this->error
) {
335 return call_user_func_array( $this->error
, array( $status ) );