Merge "Combine JavaScript and JSON encoding logic"
[mediawiki.git] / includes / job / JobQueueRedis.php
blobbd23174a8d187b87b3ddb5cd9dbdba3c3d2f0b68
1 <?php
2 /**
3 * Redis-backed job queue code.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
20 * @file
21 * @author Aaron Schulz
24 /**
25 * Class to handle job queues stored in Redis
27 * This is faster, less resource intensive, queue that JobQueueDB.
28 * All data for a queue using this class is placed into one redis server.
30 * There are eight main redis keys used to track jobs:
31 * - l-unclaimed : A list of job IDs used for ready unclaimed jobs
32 * - z-claimed : A sorted set of (job ID, UNIX timestamp as score) used for job retries
33 * - z-abandoned : A sorted set of (job ID, UNIX timestamp as score) used for broken jobs
34 * - z-delayed : A sorted set of (job ID, UNIX timestamp as score) used for delayed jobs
35 * - h-idBySha1 : A hash of (SHA1 => job ID) for unclaimed jobs used for de-duplication
36 * - h-sha1Byid : A hash of (job ID => SHA1) for unclaimed jobs used for de-duplication
37 * - h-attempts : A hash of (job ID => attempt count) used for job claiming/retries
38 * - h-data : A hash of (job ID => serialized blobs) for job storage
39 * A job ID can be in only one of z-delayed, l-unclaimed, z-claimed, and z-abandoned.
40 * If an ID appears in any of those lists, it should have a h-data entry for its ID.
41 * If a job has a SHA1 de-duplication value and its ID is in l-unclaimed or z-delayed, then
42 * there should be no other such jobs with that SHA1. Every h-idBySha1 entry has an h-sha1Byid
43 * entry and every h-sha1Byid must refer to an ID that is l-unclaimed. If a job has its
44 * ID in z-claimed or z-abandoned, then it must also have an h-attempts entry for its ID.
46 * Additionally, "rootjob:* keys track "root jobs" used for additional de-duplication.
47 * Aside from root job keys, all keys have no expiry, and are only removed when jobs are run.
48 * All the keys are prefixed with the relevant wiki ID information.
50 * This class requires Redis 2.6 as it makes use Lua scripts for fast atomic operations.
51 * Additionally, it should be noted that redis has different persistence modes, such
52 * as rdb snapshots, journaling, and no persistent. Appropriate configuration should be
53 * made on the servers based on what queues are using it and what tolerance they have.
55 * @ingroup JobQueue
56 * @ingroup Redis
57 * @since 1.21
59 class JobQueueRedis extends JobQueue {
60 /** @var RedisConnectionPool */
61 protected $redisPool;
63 protected $server; // string; server address
65 const MAX_AGE_PRUNE = 604800; // integer; seconds a job can live once claimed (7 days)
67 protected $key; // string; key to prefix the queue keys with (used for testing)
69 /**
70 * @params include:
71 * - redisConfig : An array of parameters to RedisConnectionPool::__construct().
72 * Note that the serializer option is ignored "none" is always used.
73 * - redisServer : A hostname/port combination or the absolute path of a UNIX socket.
74 * If a hostname is specified but no port, the standard port number
75 * 6379 will be used. Required.
76 * @param array $params
78 public function __construct( array $params ) {
79 parent::__construct( $params );
80 $params['redisConfig']['serializer'] = 'none'; // make it easy to use Lua
81 $this->server = $params['redisServer'];
82 $this->redisPool = RedisConnectionPool::singleton( $params['redisConfig'] );
85 protected function supportedOrders() {
86 return array( 'timestamp', 'fifo' );
89 protected function optimalOrder() {
90 return 'fifo';
93 protected function supportsDelayedJobs() {
94 return true;
97 /**
98 * @see JobQueue::doIsEmpty()
99 * @return bool
100 * @throws MWException
102 protected function doIsEmpty() {
103 return $this->doGetSize() == 0;
107 * @see JobQueue::doGetSize()
108 * @return integer
109 * @throws MWException
111 protected function doGetSize() {
112 $conn = $this->getConnection();
113 try {
114 return $conn->lSize( $this->getQueueKey( 'l-unclaimed' ) );
115 } catch ( RedisException $e ) {
116 $this->throwRedisException( $this->server, $conn, $e );
121 * @see JobQueue::doGetAcquiredCount()
122 * @return integer
123 * @throws MWException
125 protected function doGetAcquiredCount() {
126 if ( $this->claimTTL <= 0 ) {
127 return 0; // no acknowledgements
129 $conn = $this->getConnection();
130 try {
131 $conn->multi( Redis::PIPELINE );
132 $conn->zSize( $this->getQueueKey( 'z-claimed' ) );
133 $conn->zSize( $this->getQueueKey( 'z-abandoned' ) );
134 return array_sum( $conn->exec() );
135 } catch ( RedisException $e ) {
136 $this->throwRedisException( $this->server, $conn, $e );
141 * @see JobQueue::doGetDelayedCount()
142 * @return integer
143 * @throws MWException
145 protected function doGetDelayedCount() {
146 if ( !$this->checkDelay ) {
147 return 0; // no delayed jobs
149 $conn = $this->getConnection();
150 try {
151 return $conn->zSize( $this->getQueueKey( 'z-delayed' ) );
152 } catch ( RedisException $e ) {
153 $this->throwRedisException( $this->server, $conn, $e );
158 * @see JobQueue::doBatchPush()
159 * @param array $jobs
160 * @param $flags
161 * @return bool
162 * @throws MWException
164 protected function doBatchPush( array $jobs, $flags ) {
165 // Convert the jobs into field maps (de-duplicated against each other)
166 $items = array(); // (job ID => job fields map)
167 foreach ( $jobs as $job ) {
168 $item = $this->getNewJobFields( $job );
169 if ( strlen( $item['sha1'] ) ) { // hash identifier => de-duplicate
170 $items[$item['sha1']] = $item;
171 } else {
172 $items[$item['uuid']] = $item;
176 if ( !count( $items ) ) {
177 return true; // nothing to do
180 $conn = $this->getConnection();
181 try {
182 // Actually push the non-duplicate jobs into the queue...
183 if ( $flags & self::QoS_Atomic ) {
184 $batches = array( $items ); // all or nothing
185 } else {
186 $batches = array_chunk( $items, 500 ); // avoid tying up the server
188 $failed = 0;
189 $pushed = 0;
190 foreach ( $batches as $itemBatch ) {
191 $added = $this->pushBlobs( $conn, $itemBatch );
192 if ( is_int( $added ) ) {
193 $pushed += $added;
194 } else {
195 $failed += count( $itemBatch );
198 if ( $failed > 0 ) {
199 wfDebugLog( 'JobQueueRedis', "Could not insert {$failed} {$this->type} job(s)." );
200 return false;
202 wfIncrStats( 'job-insert', count( $items ) );
203 wfIncrStats( 'job-insert-duplicate', count( $items ) - $failed - $pushed );
204 } catch ( RedisException $e ) {
205 $this->throwRedisException( $this->server, $conn, $e );
208 return true;
212 * @param RedisConnRef $conn
213 * @param array $items List of results from JobQueueRedis::getNewJobFields()
214 * @return integer Number of jobs inserted (duplicates are ignored)
215 * @throws RedisException
217 protected function pushBlobs( RedisConnRef $conn, array $items ) {
218 $args = array(); // ([id, sha1, blob [, id, sha1, blob ... ] ] )
219 foreach ( $items as $item ) {
220 $args[] = (string)$item['uuid'];
221 $args[] = (string)$item['sha1'];
222 $args[] = (string)$item['rtimestamp'];
223 $args[] = (string)serialize( $item );
225 static $script =
226 <<<LUA
227 if #ARGV % 4 ~= 0 then return redis.error_reply('Unmatched arguments') end
228 local pushed = 0
229 for i = 1,#ARGV,4 do
230 local id,sha1,rtimestamp,blob = ARGV[i],ARGV[i+1],ARGV[i+2],ARGV[i+3]
231 if sha1 == '' or redis.call('hExists',KEYS[3],sha1) == 0 then
232 if 1*rtimestamp > 0 then
233 -- Insert into delayed queue (release time as score)
234 redis.call('zAdd',KEYS[4],rtimestamp,id)
235 else
236 -- Insert into unclaimed queue
237 redis.call('lPush',KEYS[1],id)
239 if sha1 ~= '' then
240 redis.call('hSet',KEYS[2],id,sha1)
241 redis.call('hSet',KEYS[3],sha1,id)
243 redis.call('hSet',KEYS[5],id,blob)
244 pushed = pushed + 1
247 return pushed
248 LUA;
249 return $this->redisEval( $conn, $script,
250 array_merge(
251 array(
252 $this->getQueueKey( 'l-unclaimed' ), # KEYS[1]
253 $this->getQueueKey( 'h-sha1ById' ), # KEYS[2]
254 $this->getQueueKey( 'h-idBySha1' ), # KEYS[3]
255 $this->getQueueKey( 'z-delayed' ), # KEYS[4]
256 $this->getQueueKey( 'h-data' ), # KEYS[5]
258 $args
260 5 # number of first argument(s) that are keys
265 * @see JobQueue::doPop()
266 * @return Job|bool
267 * @throws MWException
269 protected function doPop() {
270 $job = false;
272 // Push ready delayed jobs into the queue every 10 jobs to spread the load.
273 // This is also done as a periodic task, but we don't want too much done at once.
274 if ( $this->checkDelay && mt_rand( 0, 9 ) == 0 ) {
275 $this->releaseReadyDelayedJobs();
278 $conn = $this->getConnection();
279 try {
280 do {
281 if ( $this->claimTTL > 0 ) {
282 // Keep the claimed job list down for high-traffic queues
283 if ( mt_rand( 0, 99 ) == 0 ) {
284 $this->recycleAndDeleteStaleJobs();
286 $blob = $this->popAndAcquireBlob( $conn );
287 } else {
288 $blob = $this->popAndDeleteBlob( $conn );
290 if ( $blob === false ) {
291 break; // no jobs; nothing to do
294 wfIncrStats( 'job-pop' );
295 $item = unserialize( $blob );
296 if ( $item === false ) {
297 wfDebugLog( 'JobQueueRedis', "Could not unserialize {$this->type} job." );
298 continue;
301 // If $item is invalid, recycleAndDeleteStaleJobs() will cleanup as needed
302 $job = $this->getJobFromFields( $item ); // may be false
303 } while ( !$job ); // job may be false if invalid
304 } catch ( RedisException $e ) {
305 $this->throwRedisException( $this->server, $conn, $e );
308 return $job;
312 * @param RedisConnRef $conn
313 * @return array serialized string or false
314 * @throws RedisException
316 protected function popAndDeleteBlob( RedisConnRef $conn ) {
317 static $script =
318 <<<LUA
319 -- Pop an item off the queue
320 local id = redis.call('rpop',KEYS[1])
321 if not id then return false end
322 -- Get the job data and remove it
323 local item = redis.call('hGet',KEYS[4],id)
324 redis.call('hDel',KEYS[4],id)
325 -- Allow new duplicates of this job
326 local sha1 = redis.call('hGet',KEYS[2],id)
327 if sha1 then redis.call('hDel',KEYS[3],sha1) end
328 redis.call('hDel',KEYS[2],id)
329 -- Return the job data
330 return item
331 LUA;
332 return $this->redisEval( $conn, $script,
333 array(
334 $this->getQueueKey( 'l-unclaimed' ), # KEYS[1]
335 $this->getQueueKey( 'h-sha1ById' ), # KEYS[2]
336 $this->getQueueKey( 'h-idBySha1' ), # KEYS[3]
337 $this->getQueueKey( 'h-data' ), # KEYS[4]
339 4 # number of first argument(s) that are keys
344 * @param RedisConnRef $conn
345 * @return array serialized string or false
346 * @throws RedisException
348 protected function popAndAcquireBlob( RedisConnRef $conn ) {
349 static $script =
350 <<<LUA
351 -- Pop an item off the queue
352 local id = redis.call('rPop',KEYS[1])
353 if not id then return false end
354 -- Allow new duplicates of this job
355 local sha1 = redis.call('hGet',KEYS[2],id)
356 if sha1 then redis.call('hDel',KEYS[3],sha1) end
357 redis.call('hDel',KEYS[2],id)
358 -- Mark the jobs as claimed and return it
359 redis.call('zAdd',KEYS[4],ARGV[1],id)
360 redis.call('hIncrBy',KEYS[5],id,1)
361 return redis.call('hGet',KEYS[6],id)
362 LUA;
363 return $this->redisEval( $conn, $script,
364 array(
365 $this->getQueueKey( 'l-unclaimed' ), # KEYS[1]
366 $this->getQueueKey( 'h-sha1ById' ), # KEYS[2]
367 $this->getQueueKey( 'h-idBySha1' ), # KEYS[3]
368 $this->getQueueKey( 'z-claimed' ), # KEYS[4]
369 $this->getQueueKey( 'h-attempts' ), # KEYS[5]
370 $this->getQueueKey( 'h-data' ), # KEYS[6]
371 time(), # ARGV[1] (injected to be replication-safe)
373 6 # number of first argument(s) that are keys
378 * @see JobQueue::doAck()
379 * @param Job $job
380 * @return Job|bool
381 * @throws MWException
383 protected function doAck( Job $job ) {
384 if ( $this->claimTTL > 0 ) {
385 $conn = $this->getConnection();
386 try {
387 // Get the exact field map this Job came from, regardless of whether
388 // the job was transformed into a DuplicateJob or anything of the sort.
389 $item = $job->metadata['sourceFields'];
391 static $script =
392 <<<LUA
393 -- Unmark the job as claimed
394 redis.call('zRem',KEYS[1],ARGV[1])
395 redis.call('hDel',KEYS[2],ARGV[1])
396 -- Delete the job data itself
397 return redis.call('hDel',KEYS[3],ARGV[1])
398 LUA;
399 $res = $this->redisEval( $conn, $script,
400 array(
401 $this->getQueueKey( 'z-claimed' ), # KEYS[1]
402 $this->getQueueKey( 'h-attempts' ), # KEYS[2]
403 $this->getQueueKey( 'h-data' ), # KEYS[3]
404 $item['uuid'] # ARGV[1]
406 3 # number of first argument(s) that are keys
409 if ( !$res ) {
410 wfDebugLog( 'JobQueueRedis', "Could not acknowledge {$this->type} job." );
411 return false;
413 } catch ( RedisException $e ) {
414 $this->throwRedisException( $this->server, $conn, $e );
417 return true;
421 * @see JobQueue::doDeduplicateRootJob()
422 * @param Job $job
423 * @return bool
424 * @throws MWException
426 protected function doDeduplicateRootJob( Job $job ) {
427 $params = $job->getParams();
428 if ( !isset( $params['rootJobSignature'] ) ) {
429 throw new MWException( "Cannot register root job; missing 'rootJobSignature'." );
430 } elseif ( !isset( $params['rootJobTimestamp'] ) ) {
431 throw new MWException( "Cannot register root job; missing 'rootJobTimestamp'." );
433 $key = $this->getRootJobCacheKey( $params['rootJobSignature'] );
435 $conn = $this->getConnection();
436 try {
437 $timestamp = $conn->get( $key ); // current last timestamp of this job
438 if ( $timestamp && $timestamp >= $params['rootJobTimestamp'] ) {
439 return true; // a newer version of this root job was enqueued
441 // Update the timestamp of the last root job started at the location...
442 return $conn->set( $key, $params['rootJobTimestamp'], self::ROOTJOB_TTL ); // 2 weeks
443 } catch ( RedisException $e ) {
444 $this->throwRedisException( $this->server, $conn, $e );
449 * @see JobQueue::doIsRootJobOldDuplicate()
450 * @param Job $job
451 * @return bool
453 protected function doIsRootJobOldDuplicate( Job $job ) {
454 $params = $job->getParams();
455 if ( !isset( $params['rootJobSignature'] ) ) {
456 return false; // job has no de-deplication info
457 } elseif ( !isset( $params['rootJobTimestamp'] ) ) {
458 wfDebugLog( 'JobQueueRedis', "Cannot check root job; missing 'rootJobTimestamp'." );
459 return false;
462 $conn = $this->getConnection();
463 try {
464 // Get the last time this root job was enqueued
465 $timestamp = $conn->get( $this->getRootJobCacheKey( $params['rootJobSignature'] ) );
466 } catch ( RedisException $e ) {
467 $this->throwRedisException( $this->server, $conn, $e );
470 // Check if a new root job was started at the location after this one's...
471 return ( $timestamp && $timestamp > $params['rootJobTimestamp'] );
475 * @see JobQueue::getAllQueuedJobs()
476 * @return Iterator
478 public function getAllQueuedJobs() {
479 $conn = $this->getConnection();
480 if ( !$conn ) {
481 throw new MWException( "Unable to connect to redis server." );
483 try {
484 $that = $this;
485 return new MappedIterator(
486 $conn->lRange( $this->getQueueKey( 'l-unclaimed' ), 0, -1 ),
487 function( $uid ) use ( $that, $conn ) {
488 return $that->getJobFromUidInternal( $uid, $conn );
491 } catch ( RedisException $e ) {
492 $this->throwRedisException( $this->server, $conn, $e );
497 * @see JobQueue::getAllQueuedJobs()
498 * @return Iterator
500 public function getAllDelayedJobs() {
501 $conn = $this->getConnection();
502 if ( !$conn ) {
503 throw new MWException( "Unable to connect to redis server." );
505 try {
506 $that = $this;
507 return new MappedIterator( // delayed jobs
508 $conn->zRange( $this->getQueueKey( 'z-delayed' ), 0, -1 ),
509 function( $uid ) use ( $that, $conn ) {
510 return $that->getJobFromUidInternal( $uid, $conn );
513 } catch ( RedisException $e ) {
514 $this->throwRedisException( $this->server, $conn, $e );
519 * This function should not be called outside JobQueueRedis
521 * @param $uid string
522 * @param $conn RedisConnRef
523 * @return Job
524 * @throws MWException
526 public function getJobFromUidInternal( $uid, RedisConnRef $conn ) {
527 try {
528 $item = unserialize( $conn->hGet( $this->getQueueKey( 'h-data' ), $uid ) );
529 if ( !is_array( $item ) ) { // this shouldn't happen
530 throw new MWException( "Could not find job with ID '$uid'." );
532 $title = Title::makeTitle( $item['namespace'], $item['title'] );
533 $job = Job::factory( $item['type'], $title, $item['params'] );
534 $job->metadata['sourceFields'] = $item;
535 return $job;
536 } catch ( RedisException $e ) {
537 $this->throwRedisException( $this->server, $conn, $e );
542 * Release any ready delayed jobs into the queue
544 * @return integer Number of jobs released
545 * @throws MWException
547 public function releaseReadyDelayedJobs() {
548 $count = 0;
550 $conn = $this->getConnection();
551 try {
552 static $script =
553 <<<LUA
554 -- Get the list of ready delayed jobs, sorted by readiness
555 local ids = redis.call('zRangeByScore',KEYS[1],0,ARGV[1])
556 -- Migrate the jobs from the "delayed" set to the "unclaimed" list
557 for k,id in ipairs(ids) do
558 redis.call('lPush',KEYS[2],id)
559 redis.call('zRem',KEYS[1],id)
561 return #ids
562 LUA;
563 $count += (int)$this->redisEval( $conn, $script,
564 array(
565 $this->getQueueKey( 'z-delayed' ), // KEYS[1]
566 $this->getQueueKey( 'l-unclaimed' ), // KEYS[2]
567 time() // ARGV[1]; max "delay until" UNIX timestamp
569 2 # first two arguments are keys
571 } catch ( RedisException $e ) {
572 $this->throwRedisException( $this->server, $conn, $e );
575 return $count;
579 * Recycle or destroy any jobs that have been claimed for too long
581 * @return integer Number of jobs recycled/deleted
582 * @throws MWException
584 public function recycleAndDeleteStaleJobs() {
585 if ( $this->claimTTL <= 0 ) { // sanity
586 throw new MWException( "Cannot recycle jobs since acknowledgements are disabled." );
588 $count = 0;
589 // For each job item that can be retried, we need to add it back to the
590 // main queue and remove it from the list of currenty claimed job items.
591 // For those that cannot, they are marked as dead and kept around for
592 // investigation and manual job restoration but are eventually deleted.
593 $conn = $this->getConnection();
594 try {
595 $now = time();
596 static $script =
597 <<<LUA
598 local released,abandoned,pruned = 0,0,0
599 -- Get all non-dead jobs that have an expired claim on them.
600 -- The score for each item is the last claim timestamp (UNIX).
601 local staleClaims = redis.call('zRangeByScore',KEYS[1],0,ARGV[1],'WITHSCORES')
602 for id,timestamp in ipairs(staleClaims) do
603 local attempts = redis.call('hGet',KEYS[2],id)
604 if attempts < ARGV[3] then
605 -- Claim expired and retries left: re-enqueue the job
606 redis.call('lPush',KEYS[3],id)
607 redis.call('hIncrBy',KEYS[2],id,1)
608 released = released + 1
609 else
610 -- Claim expired and no retries left: mark the job as dead
611 redis.call('zAdd',KEYS[5],timestamp,id)
612 abandoned = abandoned + 1
614 redis.call('zRem',KEYS[1],id)
616 -- Get all of the dead jobs that have been marked as dead for too long.
617 -- The score for each item is the last claim timestamp (UNIX).
618 local deadClaims = redis.call('zRangeByScore',KEYS[5],0,ARGV[2],'WITHSCORES')
619 for id,timestamp in ipairs(deadClaims) do
620 -- Stale and out of retries: remove any traces of the job
621 redis.call('zRem',KEYS[5],id)
622 redis.call('hDel',KEYS[2],id)
623 redis.call('hDel',KEYS[4],id)
624 pruned = pruned + 1
626 return {released,abandoned,pruned}
627 LUA;
628 $res = $this->redisEval( $conn, $script,
629 array(
630 $this->getQueueKey( 'z-claimed' ), # KEYS[1]
631 $this->getQueueKey( 'h-attempts' ), # KEYS[2]
632 $this->getQueueKey( 'l-unclaimed' ), # KEYS[3]
633 $this->getQueueKey( 'h-data' ), # KEYS[4]
634 $this->getQueueKey( 'z-abandoned' ), # KEYS[5]
635 $now - $this->claimTTL, # ARGV[1]
636 $now - self::MAX_AGE_PRUNE, # ARGV[2]
637 $this->maxTries # ARGV[3]
639 5 # number of first argument(s) that are keys
641 if ( $res ) {
642 list( $released, $abandoned, $pruned ) = $res;
643 $count += $released + $pruned;
644 wfIncrStats( 'job-recycle', count( $released ) );
646 } catch ( RedisException $e ) {
647 $this->throwRedisException( $this->server, $conn, $e );
650 return $count;
654 * @return Array
656 protected function doGetPeriodicTasks() {
657 $tasks = array();
658 if ( $this->claimTTL > 0 ) {
659 $tasks['recycleAndDeleteStaleJobs'] = array(
660 'callback' => array( $this, 'recycleAndDeleteStaleJobs' ),
661 'period' => ceil( $this->claimTTL / 2 )
664 if ( $this->checkDelay ) {
665 $tasks['releaseReadyDelayedJobs'] = array(
666 'callback' => array( $this, 'releaseReadyDelayedJobs' ),
667 'period' => 300 // 5 minutes
670 return $tasks;
674 * @param RedisConnRef $conn
675 * @param string $script
676 * @param array $params
677 * @param integer $numKeys
678 * @return mixed
680 protected function redisEval( RedisConnRef $conn, $script, array $params, $numKeys ) {
681 $sha1 = sha1( $script ); // 40 char hex
683 // Try to run the server-side cached copy of the script
684 $conn->clearLastError();
685 $res = $conn->evalSha( $sha1, $params, $numKeys );
686 // If the script is not in cache, use eval() to retry and cache it
687 if ( $conn->getLastError() && $conn->script( 'exists', $sha1 ) === array( 0 ) ) {
688 $conn->clearLastError();
689 $res = $conn->eval( $script, $params, $numKeys );
690 wfDebugLog( 'JobQueueRedis', "Used eval() for Lua script $sha1." );
693 if ( $conn->getLastError() ) { // script bug?
694 wfDebugLog( 'JobQueueRedis', "Lua script error: " . $conn->getLastError() );
697 return $res;
701 * @param $job Job
702 * @return array
704 protected function getNewJobFields( Job $job ) {
705 return array(
706 // Fields that describe the nature of the job
707 'type' => $job->getType(),
708 'namespace' => $job->getTitle()->getNamespace(),
709 'title' => $job->getTitle()->getDBkey(),
710 'params' => $job->getParams(),
711 // Some jobs cannot run until a "release timestamp"
712 'rtimestamp' => $job->getReleaseTimestamp() ?: 0,
713 // Additional job metadata
714 'uuid' => UIDGenerator::newRawUUIDv4( UIDGenerator::QUICK_RAND ),
715 'sha1' => $job->ignoreDuplicates()
716 ? wfBaseConvert( sha1( serialize( $job->getDeduplicationInfo() ) ), 16, 36, 31 )
717 : '',
718 'timestamp' => time() // UNIX timestamp
723 * @param $fields array
724 * @return Job|bool
726 protected function getJobFromFields( array $fields ) {
727 $title = Title::makeTitleSafe( $fields['namespace'], $fields['title'] );
728 if ( $title ) {
729 $job = Job::factory( $fields['type'], $title, $fields['params'] );
730 $job->metadata['sourceFields'] = $fields;
731 return $job;
733 return false;
737 * Get a connection to the server that handles all sub-queues for this queue
739 * @return Array (server name, Redis instance)
740 * @throws MWException
742 protected function getConnection() {
743 $conn = $this->redisPool->getConnection( $this->server );
744 if ( !$conn ) {
745 throw new MWException( "Unable to connect to redis server." );
747 return $conn;
751 * @param $server string
752 * @param $conn RedisConnRef
753 * @param $e RedisException
754 * @throws MWException
756 protected function throwRedisException( $server, RedisConnRef $conn, $e ) {
757 $this->redisPool->handleException( $server, $conn, $e );
758 throw new MWException( "Redis server error: {$e->getMessage()}\n" );
762 * @param $prop string
763 * @return string
765 private function getQueueKey( $prop ) {
766 list( $db, $prefix ) = wfSplitWikiID( $this->wiki );
767 if ( strlen( $this->key ) ) { // namespaced queue (for testing)
768 return wfForeignMemcKey( $db, $prefix, 'jobqueue', $this->type, $this->key, $prop );
769 } else {
770 return wfForeignMemcKey( $db, $prefix, 'jobqueue', $this->type, $prop );
775 * @param $key string
776 * @return void
778 public function setTestingPrefix( $key ) {
779 $this->key = $key;