hbase-common/src/test/java/org/apache/hadoop/hbase/TestChoreService.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertFalse;
  22 import static org.junit.Assert.assertTrue;
  23
  24 import java.util.concurrent.TimeUnit;
  25 import org.apache.hadoop.hbase.TestChoreService.ScheduledChoreSamples.CountingChore;
  26 import org.apache.hadoop.hbase.TestChoreService.ScheduledChoreSamples.DoNothingChore;
  27 import org.apache.hadoop.hbase.TestChoreService.ScheduledChoreSamples.FailInitialChore;
  28 import org.apache.hadoop.hbase.TestChoreService.ScheduledChoreSamples.SampleStopper;
  29 import org.apache.hadoop.hbase.TestChoreService.ScheduledChoreSamples.SleepingChore;
  30 import org.apache.hadoop.hbase.TestChoreService.ScheduledChoreSamples.SlowChore;
  31 import org.apache.hadoop.hbase.testclassification.SmallTests;
  32 import org.junit.ClassRule;
  33 import org.junit.Test;
  34 import org.junit.experimental.categories.Category;
  35 import org.slf4j.Logger;
  36 import org.slf4j.LoggerFactory;
  37
  38 @Category(SmallTests.class)
  39 public class TestChoreService {
  40
  41   @ClassRule
  42   public static final HBaseClassTestRule CLASS_RULE =
  43       HBaseClassTestRule.forClass(TestChoreService.class);
  44
  45   public static final Logger log = LoggerFactory.getLogger(TestChoreService.class);
  46
  47   /**
  48    * A few ScheduledChore samples that are useful for testing with ChoreService
  49    */
  50   public static class ScheduledChoreSamples {
  51     /**
  52      * Straight forward stopper implementation that is used by default when one is not provided
  53      */
  54     public static class SampleStopper implements Stoppable {
  55       private boolean stopped = false;
  56
  57       @Override
  58       public void stop(String why) {
  59         stopped = true;
  60       }
  61
  62       @Override
  63       public boolean isStopped() {
  64         return stopped;
  65       }
  66     }
  67
  68     /**
  69      * Sleeps for longer than the scheduled period. This chore always misses its scheduled periodic
  70      * executions
  71      */
  72     public static class SlowChore extends ScheduledChore {
  73       public SlowChore(String name, int period) {
  74         this(name, new SampleStopper(), period);
  75       }
  76
  77       public SlowChore(String name, Stoppable stopper, int period) {
  78         super(name, stopper, period);
  79       }
  80
  81       @Override
  82       protected boolean initialChore() {
  83         try {
  84           Thread.sleep(getPeriod() * 2);
  85         } catch (InterruptedException e) {
  86           log.warn("", e);
  87         }
  88         return true;
  89       }
  90
  91       @Override
  92       protected void chore() {
  93         try {
  94           Thread.sleep(getPeriod() * 2);
  95         } catch (InterruptedException e) {
  96           log.warn("", e);
  97         }
  98       }
  99     }
 100
 101     /**
 102      * Lightweight ScheduledChore used primarily to fill the scheduling queue in tests
 103      */
 104     public static class DoNothingChore extends ScheduledChore {
 105       public DoNothingChore(String name, int period) {
 106         super(name, new SampleStopper(), period);
 107       }
 108
 109       public DoNothingChore(String name, Stoppable stopper, int period) {
 110         super(name, stopper, period);
 111       }
 112
 113       @Override
 114       protected void chore() {
 115         // DO NOTHING
 116       }
 117
 118     }
 119
 120     public static class SleepingChore extends ScheduledChore {
 121       private int sleepTime;
 122
 123       public SleepingChore(String name, int chorePeriod, int sleepTime) {
 124         this(name, new SampleStopper(), chorePeriod, sleepTime);
 125       }
 126
 127       public SleepingChore(String name, Stoppable stopper, int period, int sleepTime) {
 128         super(name, stopper, period);
 129         this.sleepTime = sleepTime;
 130       }
 131
 132       @Override
 133       protected boolean initialChore() {
 134         try {
 135           Thread.sleep(sleepTime);
 136         } catch (InterruptedException e) {
 137           log.warn("", e);
 138         }
 139         return true;
 140       }
 141
 142       @Override
 143       protected void chore() {
 144         try {
 145           Thread.sleep(sleepTime);
 146         } catch (Exception e) {
 147           log.warn("", e);
 148         }
 149       }
 150     }
 151
 152     public static class CountingChore extends ScheduledChore {
 153       private int countOfChoreCalls;
 154       private boolean outputOnTicks = false;
 155
 156       public CountingChore(String name, int period) {
 157         this(name, new SampleStopper(), period);
 158       }
 159
 160       public CountingChore(String name, Stoppable stopper, int period) {
 161         this(name, stopper, period, false);
 162       }
 163
 164       public CountingChore(String name, Stoppable stopper, int period,
 165           final boolean outputOnTicks) {
 166         super(name, stopper, period);
 167         this.countOfChoreCalls = 0;
 168         this.outputOnTicks = outputOnTicks;
 169       }
 170
 171       @Override
 172       protected boolean initialChore() {
 173         countOfChoreCalls++;
 174         if (outputOnTicks) {
 175           outputTickCount();
 176         }
 177         return true;
 178       }
 179
 180       @Override
 181       protected void chore() {
 182         countOfChoreCalls++;
 183         if (outputOnTicks) {
 184           outputTickCount();
 185         }
 186       }
 187
 188       private void outputTickCount() {
 189         log.info("Chore: " + getName() + ". Count of chore calls: " + countOfChoreCalls);
 190       }
 191
 192       public int getCountOfChoreCalls() {
 193         return countOfChoreCalls;
 194       }
 195
 196       public boolean isOutputtingOnTicks() {
 197         return outputOnTicks;
 198       }
 199
 200       public void setOutputOnTicks(boolean o) {
 201         outputOnTicks = o;
 202       }
 203     }
 204
 205     /**
 206      * A Chore that will try to execute the initial chore a few times before succeeding. Once the
 207      * initial chore is complete the chore cancels itself
 208      */
 209     public static class FailInitialChore extends ScheduledChore {
 210       private int numberOfFailures;
 211       private int failureThreshold;
 212
 213       /**
 214        * @param failThreshold Number of times the Chore fails when trying to execute initialChore
 215        *          before succeeding.
 216        */
 217       public FailInitialChore(String name, int period, int failThreshold) {
 218         this(name, new SampleStopper(), period, failThreshold);
 219       }
 220
 221       public FailInitialChore(String name, Stoppable stopper, int period, int failThreshold) {
 222         super(name, stopper, period);
 223         numberOfFailures = 0;
 224         failureThreshold = failThreshold;
 225       }
 226
 227       @Override
 228       protected boolean initialChore() {
 229         if (numberOfFailures < failureThreshold) {
 230           numberOfFailures++;
 231           return false;
 232         } else {
 233           return true;
 234         }
 235       }
 236
 237       @Override
 238       protected void chore() {
 239         assertTrue(numberOfFailures == failureThreshold);
 240         cancel(false);
 241       }
 242
 243     }
 244   }
 245
 246   @Test
 247   public void testInitialChorePrecedence() throws InterruptedException {
 248     ChoreService service = new ChoreService("testInitialChorePrecedence");
 249
 250     final int period = 100;
 251     final int failureThreshold = 5;
 252
 253     try {
 254       ScheduledChore chore = new FailInitialChore("chore", period, failureThreshold);
 255       service.scheduleChore(chore);
 256
 257       int loopCount = 0;
 258       boolean brokeOutOfLoop = false;
 259
 260       while (!chore.isInitialChoreComplete() && chore.isScheduled()) {
 261         Thread.sleep(failureThreshold * period);
 262         loopCount++;
 263         if (loopCount > 3) {
 264           brokeOutOfLoop = true;
 265           break;
 266         }
 267       }
 268
 269       assertFalse(brokeOutOfLoop);
 270     } finally {
 271       shutdownService(service);
 272     }
 273   }
 274
 275   @Test
 276   public void testCancelChore() throws InterruptedException {
 277     final int period = 100;
 278     ScheduledChore chore1 = new DoNothingChore("chore1", period);
 279     ChoreService service = new ChoreService("testCancelChore");
 280     try {
 281       service.scheduleChore(chore1);
 282       assertTrue(chore1.isScheduled());
 283
 284       chore1.cancel(true);
 285       assertFalse(chore1.isScheduled());
 286       assertTrue(service.getNumberOfScheduledChores() == 0);
 287     } finally {
 288       shutdownService(service);
 289     }
 290   }
 291
 292   @Test
 293   public void testScheduledChoreConstruction() {
 294     final String NAME = "chore";
 295     final int PERIOD = 100;
 296     final long VALID_DELAY = 0;
 297     final long INVALID_DELAY = -100;
 298     final TimeUnit UNIT = TimeUnit.NANOSECONDS;
 299
 300     ScheduledChore chore1 =
 301         new ScheduledChore(NAME, new SampleStopper(), PERIOD, VALID_DELAY, UNIT) {
 302       @Override
 303       protected void chore() {
 304         // DO NOTHING
 305       }
 306     };
 307
 308     assertEquals("Name construction failed", NAME, chore1.getName());
 309     assertEquals("Period construction failed", PERIOD, chore1.getPeriod());
 310     assertEquals("Initial Delay construction failed", VALID_DELAY, chore1.getInitialDelay());
 311     assertEquals("TimeUnit construction failed", UNIT, chore1.getTimeUnit());
 312
 313     ScheduledChore invalidDelayChore =
 314         new ScheduledChore(NAME, new SampleStopper(), PERIOD, INVALID_DELAY, UNIT) {
 315       @Override
 316       protected void chore() {
 317         // DO NOTHING
 318       }
 319     };
 320
 321     assertEquals("Initial Delay should be set to 0 when invalid", 0,
 322       invalidDelayChore.getInitialDelay());
 323   }
 324
 325   @Test
 326   public void testChoreServiceConstruction() throws InterruptedException {
 327     final int corePoolSize = 10;
 328     final int defaultCorePoolSize = ChoreService.MIN_CORE_POOL_SIZE;
 329
 330     ChoreService customInit =
 331         new ChoreService("testChoreServiceConstruction_custom", corePoolSize, false);
 332     try {
 333       assertEquals(corePoolSize, customInit.getCorePoolSize());
 334     } finally {
 335       shutdownService(customInit);
 336     }
 337
 338     ChoreService defaultInit = new ChoreService("testChoreServiceConstruction_default");
 339     try {
 340       assertEquals(defaultCorePoolSize, defaultInit.getCorePoolSize());
 341     } finally {
 342       shutdownService(defaultInit);
 343     }
 344
 345     ChoreService invalidInit = new ChoreService("testChoreServiceConstruction_invalid", -10, false);
 346     try {
 347       assertEquals(defaultCorePoolSize, invalidInit.getCorePoolSize());
 348     } finally {
 349       shutdownService(invalidInit);
 350     }
 351   }
 352
 353   @Test
 354   public void testFrequencyOfChores() throws InterruptedException {
 355     final int period = 100;
 356     // Small delta that acts as time buffer (allowing chores to complete if running slowly)
 357     final int delta = period/5;
 358     ChoreService service = new ChoreService("testFrequencyOfChores");
 359     CountingChore chore = new CountingChore("countingChore", period);
 360     try {
 361       service.scheduleChore(chore);
 362
 363       Thread.sleep(10 * period + delta);
 364       assertEquals("10 periods have elapsed.", 11, chore.getCountOfChoreCalls());
 365
 366       Thread.sleep(10 * period + delta);
 367       assertEquals("20 periods have elapsed.", 21, chore.getCountOfChoreCalls());
 368     } finally {
 369       shutdownService(service);
 370     }
 371   }
 372
 373   public void shutdownService(ChoreService service) throws InterruptedException {
 374     service.shutdown();
 375     while (!service.isTerminated()) {
 376       Thread.sleep(100);
 377     }
 378   }
 379
 380   @Test
 381   public void testForceTrigger() throws InterruptedException {
 382     final int period = 100;
 383     final int delta = period/10;
 384     ChoreService service = new ChoreService("testForceTrigger");
 385     final CountingChore chore = new CountingChore("countingChore", period);
 386     try {
 387       service.scheduleChore(chore);
 388       Thread.sleep(10 * period + delta);
 389
 390       assertEquals("10 periods have elapsed.", 11, chore.getCountOfChoreCalls());
 391
 392       // Force five runs of the chore to occur, sleeping between triggers to ensure the
 393       // chore has time to run
 394       chore.triggerNow();
 395       Thread.sleep(delta);
 396       chore.triggerNow();
 397       Thread.sleep(delta);
 398       chore.triggerNow();
 399       Thread.sleep(delta);
 400       chore.triggerNow();
 401       Thread.sleep(delta);
 402       chore.triggerNow();
 403       Thread.sleep(delta);
 404
 405       assertEquals("Trigger was called 5 times after 10 periods.", 16,
 406           chore.getCountOfChoreCalls());
 407
 408       Thread.sleep(10 * period + delta);
 409
 410       // Be loosey-goosey. It used to be '26' but it was a big flakey relying on timing.
 411       assertTrue("Expected at least 16 invocations, instead got " + chore.getCountOfChoreCalls(),
 412           chore.getCountOfChoreCalls() > 16);
 413     } finally {
 414       shutdownService(service);
 415     }
 416   }
 417
 418   @Test
 419   public void testCorePoolIncrease() throws InterruptedException {
 420     final int initialCorePoolSize = 3;
 421     ChoreService service = new ChoreService("testCorePoolIncrease", initialCorePoolSize, false);
 422
 423     try {
 424       assertEquals("Setting core pool size gave unexpected results.", initialCorePoolSize,
 425         service.getCorePoolSize());
 426
 427       final int slowChorePeriod = 100;
 428       SlowChore slowChore1 = new SlowChore("slowChore1", slowChorePeriod);
 429       SlowChore slowChore2 = new SlowChore("slowChore2", slowChorePeriod);
 430       SlowChore slowChore3 = new SlowChore("slowChore3", slowChorePeriod);
 431
 432       service.scheduleChore(slowChore1);
 433       service.scheduleChore(slowChore2);
 434       service.scheduleChore(slowChore3);
 435
 436       Thread.sleep(slowChorePeriod * 10);
 437       assertEquals("Should not create more pools than scheduled chores", 3,
 438         service.getCorePoolSize());
 439
 440       SlowChore slowChore4 = new SlowChore("slowChore4", slowChorePeriod);
 441       service.scheduleChore(slowChore4);
 442
 443       Thread.sleep(slowChorePeriod * 10);
 444       assertEquals("Chores are missing their start time. Should expand core pool size", 4,
 445         service.getCorePoolSize());
 446
 447       SlowChore slowChore5 = new SlowChore("slowChore5", slowChorePeriod);
 448       service.scheduleChore(slowChore5);
 449
 450       Thread.sleep(slowChorePeriod * 10);
 451       assertEquals("Chores are missing their start time. Should expand core pool size", 5,
 452         service.getCorePoolSize());
 453     } finally {
 454       shutdownService(service);
 455     }
 456   }
 457
 458   @Test
 459   public void testCorePoolDecrease() throws InterruptedException {
 460     final int initialCorePoolSize = 3;
 461     ChoreService service = new ChoreService("testCorePoolDecrease", initialCorePoolSize, false);
 462     final int chorePeriod = 100;
 463     try {
 464       // Slow chores always miss their start time and thus the core pool size should be at least as
 465       // large as the number of running slow chores
 466       SlowChore slowChore1 = new SlowChore("slowChore1", chorePeriod);
 467       SlowChore slowChore2 = new SlowChore("slowChore2", chorePeriod);
 468       SlowChore slowChore3 = new SlowChore("slowChore3", chorePeriod);
 469
 470       service.scheduleChore(slowChore1);
 471       service.scheduleChore(slowChore2);
 472       service.scheduleChore(slowChore3);
 473
 474       Thread.sleep(chorePeriod * 10);
 475       assertEquals("Should not create more pools than scheduled chores",
 476         service.getNumberOfScheduledChores(), service.getCorePoolSize());
 477
 478       SlowChore slowChore4 = new SlowChore("slowChore4", chorePeriod);
 479       service.scheduleChore(slowChore4);
 480       Thread.sleep(chorePeriod * 10);
 481       assertEquals("Chores are missing their start time. Should expand core pool size",
 482         service.getNumberOfScheduledChores(), service.getCorePoolSize());
 483
 484       SlowChore slowChore5 = new SlowChore("slowChore5", chorePeriod);
 485       service.scheduleChore(slowChore5);
 486       Thread.sleep(chorePeriod * 10);
 487       assertEquals("Chores are missing their start time. Should expand core pool size",
 488         service.getNumberOfScheduledChores(), service.getCorePoolSize());
 489       assertEquals(5, service.getNumberOfChoresMissingStartTime());
 490
 491       // Now we begin to cancel the chores that caused an increase in the core thread pool of the
 492       // ChoreService. These cancellations should cause a decrease in the core thread pool.
 493       slowChore5.cancel();
 494       Thread.sleep(chorePeriod * 10);
 495       assertEquals(Math.max(ChoreService.MIN_CORE_POOL_SIZE, service.getNumberOfScheduledChores()),
 496         service.getCorePoolSize());
 497       assertEquals(4, service.getNumberOfChoresMissingStartTime());
 498
 499       slowChore4.cancel();
 500       Thread.sleep(chorePeriod * 10);
 501       assertEquals(Math.max(ChoreService.MIN_CORE_POOL_SIZE, service.getNumberOfScheduledChores()),
 502         service.getCorePoolSize());
 503       assertEquals(3, service.getNumberOfChoresMissingStartTime());
 504
 505       slowChore3.cancel();
 506       Thread.sleep(chorePeriod * 10);
 507       assertEquals(Math.max(ChoreService.MIN_CORE_POOL_SIZE, service.getNumberOfScheduledChores()),
 508         service.getCorePoolSize());
 509       assertEquals(2, service.getNumberOfChoresMissingStartTime());
 510
 511       slowChore2.cancel();
 512       Thread.sleep(chorePeriod * 10);
 513       assertEquals(Math.max(ChoreService.MIN_CORE_POOL_SIZE, service.getNumberOfScheduledChores()),
 514         service.getCorePoolSize());
 515       assertEquals(1, service.getNumberOfChoresMissingStartTime());
 516
 517       slowChore1.cancel();
 518       Thread.sleep(chorePeriod * 10);
 519       assertEquals(Math.max(ChoreService.MIN_CORE_POOL_SIZE, service.getNumberOfScheduledChores()),
 520         service.getCorePoolSize());
 521       assertEquals(0, service.getNumberOfChoresMissingStartTime());
 522     } finally {
 523       shutdownService(service);
 524     }
 525   }
 526
 527   @Test
 528   public void testNumberOfRunningChores() throws InterruptedException {
 529     ChoreService service = new ChoreService("testNumberOfRunningChores");
 530
 531     final int period = 100;
 532     final int sleepTime = 5;
 533
 534     try {
 535       DoNothingChore dn1 = new DoNothingChore("dn1", period);
 536       DoNothingChore dn2 = new DoNothingChore("dn2", period);
 537       DoNothingChore dn3 = new DoNothingChore("dn3", period);
 538       DoNothingChore dn4 = new DoNothingChore("dn4", period);
 539       DoNothingChore dn5 = new DoNothingChore("dn5", period);
 540
 541       service.scheduleChore(dn1);
 542       service.scheduleChore(dn2);
 543       service.scheduleChore(dn3);
 544       service.scheduleChore(dn4);
 545       service.scheduleChore(dn5);
 546
 547       Thread.sleep(sleepTime);
 548       assertEquals("Scheduled chore mismatch", 5, service.getNumberOfScheduledChores());
 549
 550       dn1.cancel();
 551       Thread.sleep(sleepTime);
 552       assertEquals("Scheduled chore mismatch", 4, service.getNumberOfScheduledChores());
 553
 554       dn2.cancel();
 555       dn3.cancel();
 556       dn4.cancel();
 557       Thread.sleep(sleepTime);
 558       assertEquals("Scheduled chore mismatch", 1, service.getNumberOfScheduledChores());
 559
 560       dn5.cancel();
 561       Thread.sleep(sleepTime);
 562       assertEquals("Scheduled chore mismatch", 0, service.getNumberOfScheduledChores());
 563     } finally {
 564       shutdownService(service);
 565     }
 566   }
 567
 568   @Test
 569   public void testNumberOfChoresMissingStartTime() throws InterruptedException {
 570     ChoreService service = new ChoreService("testNumberOfChoresMissingStartTime");
 571
 572     final int period = 100;
 573     final int sleepTime = 5 * period;
 574
 575     try {
 576       // Slow chores sleep for a length of time LONGER than their period. Thus, SlowChores
 577       // ALWAYS miss their start time since their execution takes longer than their period
 578       SlowChore sc1 = new SlowChore("sc1", period);
 579       SlowChore sc2 = new SlowChore("sc2", period);
 580       SlowChore sc3 = new SlowChore("sc3", period);
 581       SlowChore sc4 = new SlowChore("sc4", period);
 582       SlowChore sc5 = new SlowChore("sc5", period);
 583
 584       service.scheduleChore(sc1);
 585       service.scheduleChore(sc2);
 586       service.scheduleChore(sc3);
 587       service.scheduleChore(sc4);
 588       service.scheduleChore(sc5);
 589
 590       Thread.sleep(sleepTime);
 591       assertEquals(5, service.getNumberOfChoresMissingStartTime());
 592
 593       sc1.cancel();
 594       Thread.sleep(sleepTime);
 595       assertEquals(4, service.getNumberOfChoresMissingStartTime());
 596
 597       sc2.cancel();
 598       sc3.cancel();
 599       sc4.cancel();
 600       Thread.sleep(sleepTime);
 601       assertEquals(1, service.getNumberOfChoresMissingStartTime());
 602
 603       sc5.cancel();
 604       Thread.sleep(sleepTime);
 605       assertEquals(0, service.getNumberOfChoresMissingStartTime());
 606     } finally {
 607       shutdownService(service);
 608     }
 609   }
 610
 611   /**
 612    * ChoreServices should never have a core pool size that exceeds the number of chores that have
 613    * been scheduled with the service. For example, if 4 ScheduledChores are scheduled with a
 614    * ChoreService, the number of threads in the ChoreService's core pool should never exceed 4
 615    */
 616   @Test
 617   public void testMaximumChoreServiceThreads() throws InterruptedException {
 618     ChoreService service = new ChoreService("testMaximumChoreServiceThreads");
 619
 620     final int period = 100;
 621     final int sleepTime = 5 * period;
 622
 623     try {
 624       // Slow chores sleep for a length of time LONGER than their period. Thus, SlowChores
 625       // ALWAYS miss their start time since their execution takes longer than their period.
 626       // Chores that miss their start time will trigger the onChoreMissedStartTime callback
 627       // in the ChoreService. This callback will try to increase the number of core pool
 628       // threads.
 629       SlowChore sc1 = new SlowChore("sc1", period);
 630       SlowChore sc2 = new SlowChore("sc2", period);
 631       SlowChore sc3 = new SlowChore("sc3", period);
 632       SlowChore sc4 = new SlowChore("sc4", period);
 633       SlowChore sc5 = new SlowChore("sc5", period);
 634
 635       service.scheduleChore(sc1);
 636       service.scheduleChore(sc2);
 637       service.scheduleChore(sc3);
 638       service.scheduleChore(sc4);
 639       service.scheduleChore(sc5);
 640
 641       Thread.sleep(sleepTime);
 642       assertTrue(service.getCorePoolSize() <= service.getNumberOfScheduledChores());
 643
 644       SlowChore sc6 = new SlowChore("sc6", period);
 645       SlowChore sc7 = new SlowChore("sc7", period);
 646       SlowChore sc8 = new SlowChore("sc8", period);
 647       SlowChore sc9 = new SlowChore("sc9", period);
 648       SlowChore sc10 = new SlowChore("sc10", period);
 649
 650       service.scheduleChore(sc6);
 651       service.scheduleChore(sc7);
 652       service.scheduleChore(sc8);
 653       service.scheduleChore(sc9);
 654       service.scheduleChore(sc10);
 655
 656       Thread.sleep(sleepTime);
 657       assertTrue(service.getCorePoolSize() <= service.getNumberOfScheduledChores());
 658     } finally {
 659       shutdownService(service);
 660     }
 661   }
 662
 663   @Test
 664   public void testChangingChoreServices() throws InterruptedException {
 665     final int period = 100;
 666     final int sleepTime = 10;
 667     ChoreService service1 = new ChoreService("testChangingChoreServices_1");
 668     ChoreService service2 = new ChoreService("testChangingChoreServices_2");
 669     ScheduledChore chore = new DoNothingChore("sample", period);
 670
 671     try {
 672       assertFalse(chore.isScheduled());
 673       assertFalse(service1.isChoreScheduled(chore));
 674       assertFalse(service2.isChoreScheduled(chore));
 675       assertTrue(chore.getChoreServicer() == null);
 676
 677       service1.scheduleChore(chore);
 678       Thread.sleep(sleepTime);
 679       assertTrue(chore.isScheduled());
 680       assertTrue(service1.isChoreScheduled(chore));
 681       assertFalse(service2.isChoreScheduled(chore));
 682       assertFalse(chore.getChoreServicer() == null);
 683
 684       service2.scheduleChore(chore);
 685       Thread.sleep(sleepTime);
 686       assertTrue(chore.isScheduled());
 687       assertFalse(service1.isChoreScheduled(chore));
 688       assertTrue(service2.isChoreScheduled(chore));
 689       assertFalse(chore.getChoreServicer() == null);
 690
 691       chore.cancel();
 692       assertFalse(chore.isScheduled());
 693       assertFalse(service1.isChoreScheduled(chore));
 694       assertFalse(service2.isChoreScheduled(chore));
 695       assertTrue(chore.getChoreServicer() == null);
 696     } finally {
 697       shutdownService(service1);
 698       shutdownService(service2);
 699     }
 700   }
 701
 702   @Test
 703   public void testStopperForScheduledChores() throws InterruptedException {
 704     ChoreService service = new ChoreService("testStopperForScheduledChores");
 705     Stoppable stopperForGroup1 = new SampleStopper();
 706     Stoppable stopperForGroup2 = new SampleStopper();
 707     final int period = 100;
 708     final int delta = period/10;
 709
 710     try {
 711       ScheduledChore chore1_group1 = new DoNothingChore("c1g1", stopperForGroup1, period);
 712       ScheduledChore chore2_group1 = new DoNothingChore("c2g1", stopperForGroup1, period);
 713       ScheduledChore chore3_group1 = new DoNothingChore("c3g1", stopperForGroup1, period);
 714
 715       ScheduledChore chore1_group2 = new DoNothingChore("c1g2", stopperForGroup2, period);
 716       ScheduledChore chore2_group2 = new DoNothingChore("c2g2", stopperForGroup2, period);
 717       ScheduledChore chore3_group2 = new DoNothingChore("c3g2", stopperForGroup2, period);
 718
 719       service.scheduleChore(chore1_group1);
 720       service.scheduleChore(chore2_group1);
 721       service.scheduleChore(chore3_group1);
 722       service.scheduleChore(chore1_group2);
 723       service.scheduleChore(chore2_group2);
 724       service.scheduleChore(chore3_group2);
 725
 726       Thread.sleep(delta);
 727       Thread.sleep(10 * period);
 728       assertTrue(chore1_group1.isScheduled());
 729       assertTrue(chore2_group1.isScheduled());
 730       assertTrue(chore3_group1.isScheduled());
 731       assertTrue(chore1_group2.isScheduled());
 732       assertTrue(chore2_group2.isScheduled());
 733       assertTrue(chore3_group2.isScheduled());
 734
 735       stopperForGroup1.stop("test stopping group 1");
 736       Thread.sleep(period);
 737       assertFalse(chore1_group1.isScheduled());
 738       assertFalse(chore2_group1.isScheduled());
 739       assertFalse(chore3_group1.isScheduled());
 740       assertTrue(chore1_group2.isScheduled());
 741       assertTrue(chore2_group2.isScheduled());
 742       assertTrue(chore3_group2.isScheduled());
 743
 744       stopperForGroup2.stop("test stopping group 2");
 745       Thread.sleep(period);
 746       assertFalse(chore1_group1.isScheduled());
 747       assertFalse(chore2_group1.isScheduled());
 748       assertFalse(chore3_group1.isScheduled());
 749       assertFalse(chore1_group2.isScheduled());
 750       assertFalse(chore2_group2.isScheduled());
 751       assertFalse(chore3_group2.isScheduled());
 752     } finally {
 753       shutdownService(service);
 754     }
 755   }
 756
 757   @Test
 758   public void testShutdownCancelsScheduledChores() throws InterruptedException {
 759     final int period = 100;
 760     ChoreService service = new ChoreService("testShutdownCancelsScheduledChores");
 761     ScheduledChore successChore1 = new DoNothingChore("sc1", period);
 762     ScheduledChore successChore2 = new DoNothingChore("sc2", period);
 763     ScheduledChore successChore3 = new DoNothingChore("sc3", period);
 764
 765     try {
 766       assertTrue(service.scheduleChore(successChore1));
 767       assertTrue(successChore1.isScheduled());
 768       assertTrue(service.scheduleChore(successChore2));
 769       assertTrue(successChore2.isScheduled());
 770       assertTrue(service.scheduleChore(successChore3));
 771       assertTrue(successChore3.isScheduled());
 772     } finally {
 773       shutdownService(service);
 774     }
 775
 776     assertFalse(successChore1.isScheduled());
 777     assertFalse(successChore2.isScheduled());
 778     assertFalse(successChore3.isScheduled());
 779   }
 780
 781   @Test
 782   public void testShutdownWorksWhileChoresAreExecuting() throws InterruptedException {
 783     final int period = 100;
 784     final int sleep = 5 * period;
 785     ChoreService service = new ChoreService("testShutdownWorksWhileChoresAreExecuting");
 786     ScheduledChore slowChore1 = new SleepingChore("sc1", period, sleep);
 787     ScheduledChore slowChore2 = new SleepingChore("sc2", period, sleep);
 788     ScheduledChore slowChore3 = new SleepingChore("sc3", period, sleep);
 789     try {
 790       assertTrue(service.scheduleChore(slowChore1));
 791       assertTrue(service.scheduleChore(slowChore2));
 792       assertTrue(service.scheduleChore(slowChore3));
 793
 794       Thread.sleep(sleep / 2);
 795       shutdownService(service);
 796
 797       assertFalse(slowChore1.isScheduled());
 798       assertFalse(slowChore2.isScheduled());
 799       assertFalse(slowChore3.isScheduled());
 800       assertTrue(service.isShutdown());
 801
 802       Thread.sleep(5);
 803       assertTrue(service.isTerminated());
 804     } finally {
 805       shutdownService(service);
 806     }
 807   }
 808
 809   @Test
 810   public void testShutdownRejectsNewSchedules() throws InterruptedException {
 811     final int period = 100;
 812     ChoreService service = new ChoreService("testShutdownRejectsNewSchedules");
 813     ScheduledChore successChore1 = new DoNothingChore("sc1", period);
 814     ScheduledChore successChore2 = new DoNothingChore("sc2", period);
 815     ScheduledChore successChore3 = new DoNothingChore("sc3", period);
 816     ScheduledChore failChore1 = new DoNothingChore("fc1", period);
 817     ScheduledChore failChore2 = new DoNothingChore("fc2", period);
 818     ScheduledChore failChore3 = new DoNothingChore("fc3", period);
 819
 820     try {
 821       assertTrue(service.scheduleChore(successChore1));
 822       assertTrue(successChore1.isScheduled());
 823       assertTrue(service.scheduleChore(successChore2));
 824       assertTrue(successChore2.isScheduled());
 825       assertTrue(service.scheduleChore(successChore3));
 826       assertTrue(successChore3.isScheduled());
 827     } finally {
 828       shutdownService(service);
 829     }
 830
 831     assertFalse(service.scheduleChore(failChore1));
 832     assertFalse(failChore1.isScheduled());
 833     assertFalse(service.scheduleChore(failChore2));
 834     assertFalse(failChore2.isScheduled());
 835     assertFalse(service.scheduleChore(failChore3));
 836     assertFalse(failChore3.isScheduled());
 837   }
 838 }