tests/phpunit/includes/jobqueue/JobQueueTest.php

   1 <?php
   2
   3 /**
   4  * @group JobQueue
   5  * @group medium
   6  * @group Database
   7  */
   8 class JobQueueTest extends MediaWikiTestCase {
   9         protected $key;
  10         protected $queueRand, $queueRandTTL, $queueFifo, $queueFifoTTL;
  11
  12         function __construct( $name = null, array $data = array(), $dataName = '' ) {
  13                 parent::__construct( $name, $data, $dataName );
  14
  15                 $this->tablesUsed[] = 'job';
  16         }
  17
  18         protected function setUp() {
  19                 global $wgJobTypeConf;
  20                 parent::setUp();
  21
  22                 $this->setMwGlobals( 'wgMemc', new HashBagOStuff() );
  23
  24                 if ( $this->getCliArg( 'use-jobqueue' ) ) {
  25                         $name = $this->getCliArg( 'use-jobqueue' );
  26                         if ( !isset( $wgJobTypeConf[$name] ) ) {
  27                                 throw new MWException( "No \$wgJobTypeConf entry for '$name'." );
  28                         }
  29                         $baseConfig = $wgJobTypeConf[$name];
  30                 } else {
  31                         $baseConfig = array( 'class' => 'JobQueueDB' );
  32                 }
  33                 $baseConfig['type'] = 'null';
  34                 $baseConfig['wiki'] = wfWikiID();
  35                 $variants = array(
  36                         'queueRand' => array( 'order' => 'random', 'claimTTL' => 0 ),
  37                         'queueRandTTL' => array( 'order' => 'random', 'claimTTL' => 10 ),
  38                         'queueTimestamp' => array( 'order' => 'timestamp', 'claimTTL' => 0 ),
  39                         'queueTimestampTTL' => array( 'order' => 'timestamp', 'claimTTL' => 10 ),
  40                         'queueFifo' => array( 'order' => 'fifo', 'claimTTL' => 0 ),
  41                         'queueFifoTTL' => array( 'order' => 'fifo', 'claimTTL' => 10 ),
  42                 );
  43                 foreach ( $variants as $q => $settings ) {
  44                         try {
  45                                 $this->$q = JobQueue::factory( $settings + $baseConfig );
  46                                 if ( !( $this->$q instanceof JobQueueDB ) ) {
  47                                         $this->$q->setTestingPrefix( 'unittests-' . wfRandomString( 32 ) );
  48                                 }
  49                         } catch ( MWException $e ) {
  50                                 // unsupported?
  51                                 // @todo What if it was another error?
  52                         };
  53                 }
  54         }
  55
  56         protected function tearDown() {
  57                 parent::tearDown();
  58                 foreach (
  59                         array(
  60                                 'queueRand', 'queueRandTTL', 'queueTimestamp', 'queueTimestampTTL',
  61                                 'queueFifo', 'queueFifoTTL'
  62                         ) as $q
  63                 ) {
  64                         if ( $this->$q ) {
  65                                 $this->$q->delete();
  66                         }
  67                         $this->$q = null;
  68                 }
  69         }
  70
  71         /**
  72          * @dataProvider provider_queueLists
  73          * @covers JobQueue::getWiki
  74          */
  75         public function testGetWiki( $queue, $recycles, $desc ) {
  76                 $queue = $this->$queue;
  77                 if ( !$queue ) {
  78                         $this->markTestSkipped( $desc );
  79                 }
  80                 $this->assertEquals( wfWikiID(), $queue->getWiki(), "Proper wiki ID ($desc)" );
  81         }
  82
  83         /**
  84          * @dataProvider provider_queueLists
  85          * @covers JobQueue::getType
  86          */
  87         public function testGetType( $queue, $recycles, $desc ) {
  88                 $queue = $this->$queue;
  89                 if ( !$queue ) {
  90                         $this->markTestSkipped( $desc );
  91                 }
  92                 $this->assertEquals( 'null', $queue->getType(), "Proper job type ($desc)" );
  93         }
  94
  95         /**
  96          * @dataProvider provider_queueLists
  97          * @covers JobQueue
  98          */
  99         public function testBasicOperations( $queue, $recycles, $desc ) {
 100                 $queue = $this->$queue;
 101                 if ( !$queue ) {
 102                         $this->markTestSkipped( $desc );
 103                 }
 104
 105                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 106
 107                 $queue->flushCaches();
 108                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 109                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 110
 111                 $this->assertNull( $queue->push( $this->newJob() ), "Push worked ($desc)" );
 112                 $this->assertNull( $queue->batchPush( array( $this->newJob() ) ), "Push worked ($desc)" );
 113
 114                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 115
 116                 $queue->flushCaches();
 117                 $this->assertEquals( 2, $queue->getSize(), "Queue size is correct ($desc)" );
 118                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 119                 $jobs = iterator_to_array( $queue->getAllQueuedJobs() );
 120                 $this->assertEquals( 2, count( $jobs ), "Queue iterator size is correct ($desc)" );
 121
 122                 $job1 = $queue->pop();
 123                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 124
 125                 $queue->flushCaches();
 126                 $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
 127
 128                 $queue->flushCaches();
 129                 if ( $recycles ) {
 130                         $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
 131                 } else {
 132                         $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
 133                 }
 134
 135                 $job2 = $queue->pop();
 136                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 137                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 138
 139                 $queue->flushCaches();
 140                 if ( $recycles ) {
 141                         $this->assertEquals( 2, $queue->getAcquiredCount(), "Active job count ($desc)" );
 142                 } else {
 143                         $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
 144                 }
 145
 146                 $queue->ack( $job1 );
 147
 148                 $queue->flushCaches();
 149                 if ( $recycles ) {
 150                         $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
 151                 } else {
 152                         $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
 153                 }
 154
 155                 $queue->ack( $job2 );
 156
 157                 $queue->flushCaches();
 158                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
 159
 160                 $this->assertNull( $queue->batchPush( array( $this->newJob(), $this->newJob() ) ),
 161                         "Push worked ($desc)" );
 162                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 163
 164                 $queue->delete();
 165                 $queue->flushCaches();
 166                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 167                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 168         }
 169
 170         /**
 171          * @dataProvider provider_queueLists
 172          * @covers JobQueue
 173          */
 174         public function testBasicDeduplication( $queue, $recycles, $desc ) {
 175                 $queue = $this->$queue;
 176                 if ( !$queue ) {
 177                         $this->markTestSkipped( $desc );
 178                 }
 179
 180                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 181
 182                 $queue->flushCaches();
 183                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 184                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 185
 186                 $this->assertNull(
 187                         $queue->batchPush(
 188                                 array( $this->newDedupedJob(), $this->newDedupedJob(), $this->newDedupedJob() )
 189                         ),
 190                         "Push worked ($desc)" );
 191
 192                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 193
 194                 $queue->flushCaches();
 195                 $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
 196                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 197
 198                 $this->assertNull(
 199                         $queue->batchPush(
 200                                 array( $this->newDedupedJob(), $this->newDedupedJob(), $this->newDedupedJob() )
 201                         ),
 202                         "Push worked ($desc)"
 203                 );
 204
 205                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 206
 207                 $queue->flushCaches();
 208                 $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
 209                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 210
 211                 $job1 = $queue->pop();
 212                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 213
 214                 $queue->flushCaches();
 215                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 216                 if ( $recycles ) {
 217                         $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
 218                 } else {
 219                         $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
 220                 }
 221
 222                 $queue->ack( $job1 );
 223
 224                 $queue->flushCaches();
 225                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
 226         }
 227
 228         /**
 229          * @dataProvider provider_queueLists
 230          * @covers JobQueue
 231          */
 232         public function testRootDeduplication( $queue, $recycles, $desc ) {
 233                 $queue = $this->$queue;
 234                 if ( !$queue ) {
 235                         $this->markTestSkipped( $desc );
 236                 }
 237
 238                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 239
 240                 $queue->flushCaches();
 241                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 242                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 243
 244                 $id = wfRandomString( 32 );
 245                 $root1 = Job::newRootJobParams( "nulljobspam:$id" ); // task ID/timestamp
 246                 for ( $i = 0; $i < 5; ++$i ) {
 247                         $this->assertNull( $queue->push( $this->newJob( 0, $root1 ) ), "Push worked ($desc)" );
 248                 }
 249                 $queue->deduplicateRootJob( $this->newJob( 0, $root1 ) );
 250
 251                 $root2 = $root1;
 252                 # Add a second to UNIX epoch and format back to TS_MW
 253                 $root2_ts = strtotime( $root2['rootJobTimestamp'] );
 254                 $root2_ts++;
 255                 $root2['rootJobTimestamp'] = wfTimestamp( TS_MW, $root2_ts );
 256
 257                 $this->assertNotEquals( $root1['rootJobTimestamp'], $root2['rootJobTimestamp'],
 258                         "Root job signatures have different timestamps." );
 259                 for ( $i = 0; $i < 5; ++$i ) {
 260                         $this->assertNull( $queue->push( $this->newJob( 0, $root2 ) ), "Push worked ($desc)" );
 261                 }
 262                 $queue->deduplicateRootJob( $this->newJob( 0, $root2 ) );
 263
 264                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 265
 266                 $queue->flushCaches();
 267                 $this->assertEquals( 10, $queue->getSize(), "Queue size is correct ($desc)" );
 268                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 269
 270                 $dupcount = 0;
 271                 $jobs = array();
 272                 do {
 273                         $job = $queue->pop();
 274                         if ( $job ) {
 275                                 $jobs[] = $job;
 276                                 $queue->ack( $job );
 277                         }
 278                         if ( $job instanceof DuplicateJob ) {
 279                                 ++$dupcount;
 280                         }
 281                 } while ( $job );
 282
 283                 $this->assertEquals( 10, count( $jobs ), "Correct number of jobs popped ($desc)" );
 284                 $this->assertEquals( 5, $dupcount, "Correct number of duplicate jobs popped ($desc)" );
 285         }
 286
 287         /**
 288          * @dataProvider provider_fifoQueueLists
 289          * @covers JobQueue
 290          */
 291         public function testJobOrder( $queue, $recycles, $desc ) {
 292                 $queue = $this->$queue;
 293                 if ( !$queue ) {
 294                         $this->markTestSkipped( $desc );
 295                 }
 296
 297                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 298
 299                 $queue->flushCaches();
 300                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 301                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 302
 303                 for ( $i = 0; $i < 10; ++$i ) {
 304                         $this->assertNull( $queue->push( $this->newJob( $i ) ), "Push worked ($desc)" );
 305                 }
 306
 307                 for ( $i = 0; $i < 10; ++$i ) {
 308                         $job = $queue->pop();
 309                         $this->assertTrue( $job instanceof Job, "Jobs popped from queue ($desc)" );
 310                         $params = $job->getParams();
 311                         $this->assertEquals( $i, $params['i'], "Job popped from queue is FIFO ($desc)" );
 312                         $queue->ack( $job );
 313                 }
 314
 315                 $this->assertFalse( $queue->pop(), "Queue is not empty ($desc)" );
 316
 317                 $queue->flushCaches();
 318                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 319                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 320         }
 321
 322         public static function provider_queueLists() {
 323                 return array(
 324                         array( 'queueRand', false, 'Random queue without ack()' ),
 325                         array( 'queueRandTTL', true, 'Random queue with ack()' ),
 326                         array( 'queueTimestamp', false, 'Time ordered queue without ack()' ),
 327                         array( 'queueTimestampTTL', true, 'Time ordered queue with ack()' ),
 328                         array( 'queueFifo', false, 'FIFO ordered queue without ack()' ),
 329                         array( 'queueFifoTTL', true, 'FIFO ordered queue with ack()' )
 330                 );
 331         }
 332
 333         public static function provider_fifoQueueLists() {
 334                 return array(
 335                         array( 'queueFifo', false, 'Ordered queue without ack()' ),
 336                         array( 'queueFifoTTL', true, 'Ordered queue with ack()' )
 337                 );
 338         }
 339
 340         function newJob( $i = 0, $rootJob = array() ) {
 341                 return new NullJob( Title::newMainPage(),
 342                         array( 'lives' => 0, 'usleep' => 0, 'removeDuplicates' => 0, 'i' => $i ) + $rootJob );
 343         }
 344
 345         function newDedupedJob( $i = 0, $rootJob = array() ) {
 346                 return new NullJob( Title::newMainPage(),
 347                         array( 'lives' => 0, 'usleep' => 0, 'removeDuplicates' => 1, 'i' => $i ) + $rootJob );
 348         }
 349 }