hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerOnlineConfigChange.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.regionserver;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertFalse;
  22 import static org.junit.Assert.assertNotNull;
  23 import static org.junit.Assert.assertTrue;
  24
  25 import java.io.IOException;
  26 import java.util.concurrent.TimeUnit;
  27 import org.apache.hadoop.conf.Configuration;
  28 import org.apache.hadoop.hbase.HBaseClassTestRule;
  29 import org.apache.hadoop.hbase.HBaseTestingUtil;
  30 import org.apache.hadoop.hbase.HConstants;
  31 import org.apache.hadoop.hbase.TableName;
  32 import org.apache.hadoop.hbase.Waiter;
  33 import org.apache.hadoop.hbase.client.Admin;
  34 import org.apache.hadoop.hbase.client.Connection;
  35 import org.apache.hadoop.hbase.client.ConnectionFactory;
  36 import org.apache.hadoop.hbase.client.RegionInfo;
  37 import org.apache.hadoop.hbase.client.RegionLocator;
  38 import org.apache.hadoop.hbase.client.Table;
  39 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
  40 import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
  41 import org.apache.hadoop.hbase.testclassification.MediumTests;
  42 import org.apache.hadoop.hbase.util.Bytes;
  43 import org.junit.AfterClass;
  44 import org.junit.Before;
  45 import org.junit.BeforeClass;
  46 import org.junit.ClassRule;
  47 import org.junit.Test;
  48 import org.junit.experimental.categories.Category;
  49 import org.slf4j.Logger;
  50 import org.slf4j.LoggerFactory;
  51
  52 /**
  53  * Verify that the Online config Changes on the HRegionServer side are actually
  54  * happening. We should add tests for important configurations which will be
  55  * changed online.
  56  */
  57
  58 @Category({MediumTests.class})
  59 public class TestRegionServerOnlineConfigChange {
  60
  61   @ClassRule
  62   public static final HBaseClassTestRule CLASS_RULE =
  63       HBaseClassTestRule.forClass(TestRegionServerOnlineConfigChange.class);
  64
  65   private static final Logger LOG =
  66           LoggerFactory.getLogger(TestRegionServerOnlineConfigChange.class.getName());
  67   private static final long WAIT_TIMEOUT = TimeUnit.MINUTES.toMillis(2);
  68   private static HBaseTestingUtil hbaseTestingUtility = new HBaseTestingUtil();
  69   private static Configuration conf = null;
  70
  71   private static Table t1 = null;
  72   private static HRegionServer rs1 = null;
  73   private static byte[] r1name = null;
  74   private static Region r1 = null;
  75
  76   private final static String table1Str = "table1";
  77   private final static String columnFamily1Str = "columnFamily1";
  78   private final static TableName TABLE1 = TableName.valueOf(table1Str);
  79   private final static byte[] COLUMN_FAMILY1 = Bytes.toBytes(columnFamily1Str);
  80   private final static long MAX_FILE_SIZE = 20 * 1024 * 1024L;
  81
  82
  83   @BeforeClass
  84   public static void setUpBeforeClass() throws Exception {
  85     conf = hbaseTestingUtility.getConfiguration();
  86     hbaseTestingUtility.startMiniCluster(2);
  87     t1 = hbaseTestingUtility.createTable(
  88       TableDescriptorBuilder.newBuilder(TABLE1).setMaxFileSize(MAX_FILE_SIZE).build(),
  89       new byte[][] { COLUMN_FAMILY1 }, conf);
  90   }
  91
  92   @AfterClass
  93   public static void tearDown() throws Exception {
  94     hbaseTestingUtility.shutdownMiniCluster();
  95   }
  96
  97   @Before
  98   public void setUp() throws Exception {
  99     try (RegionLocator locator = hbaseTestingUtility.getConnection().getRegionLocator(TABLE1)) {
 100       RegionInfo firstHRI = locator.getAllRegionLocations().get(0).getRegion();
 101       r1name = firstHRI.getRegionName();
 102       rs1 = hbaseTestingUtility.getHBaseCluster().getRegionServer(
 103         hbaseTestingUtility.getHBaseCluster().getServerWith(r1name));
 104       r1 = rs1.getRegion(r1name);
 105     }
 106   }
 107
 108   /**
 109    * Check if the number of compaction threads changes online
 110    */
 111   @Test
 112   public void testNumCompactionThreadsOnlineChange() {
 113     assertNotNull(rs1.getCompactSplitThread());
 114     int newNumSmallThreads =
 115       rs1.getCompactSplitThread().getSmallCompactionThreadNum() + 1;
 116     int newNumLargeThreads =
 117       rs1.getCompactSplitThread().getLargeCompactionThreadNum() + 1;
 118
 119     conf.setInt("hbase.regionserver.thread.compaction.small",
 120       newNumSmallThreads);
 121     conf.setInt("hbase.regionserver.thread.compaction.large",
 122       newNumLargeThreads);
 123     rs1.getConfigurationManager().notifyAllObservers(conf);
 124
 125     assertEquals(newNumSmallThreads,
 126       rs1.getCompactSplitThread().getSmallCompactionThreadNum());
 127     assertEquals(newNumLargeThreads,
 128       rs1.getCompactSplitThread().getLargeCompactionThreadNum());
 129   }
 130
 131   /**
 132    * Test that the configurations in the CompactionConfiguration class change
 133    * properly.
 134    *
 135    * @throws IOException
 136    */
 137   @Test
 138   public void testCompactionConfigurationOnlineChange() throws IOException {
 139     String strPrefix = "hbase.hstore.compaction.";
 140     Store s = r1.getStore(COLUMN_FAMILY1);
 141     if (!(s instanceof HStore)) {
 142       LOG.error("Can't test the compaction configuration of HStore class. "
 143           + "Got a different implementation other than HStore");
 144       return;
 145     }
 146     HStore hstore = (HStore)s;
 147
 148     // Set the new compaction ratio to a different value.
 149     double newCompactionRatio =
 150             hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatio() + 0.1;
 151     conf.setFloat(strPrefix + "ratio", (float)newCompactionRatio);
 152
 153     // Notify all the observers, which includes the Store object.
 154     rs1.getConfigurationManager().notifyAllObservers(conf);
 155
 156     // Check if the compaction ratio got updated in the Compaction Configuration
 157     assertEquals(newCompactionRatio,
 158                  hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatio(),
 159                  0.00001);
 160
 161     // Check if the off peak compaction ratio gets updated.
 162     double newOffPeakCompactionRatio =
 163         hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatioOffPeak() + 0.1;
 164     conf.setFloat(strPrefix + "ratio.offpeak",
 165             (float)newOffPeakCompactionRatio);
 166     rs1.getConfigurationManager().notifyAllObservers(conf);
 167     assertEquals(newOffPeakCompactionRatio,
 168         hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatioOffPeak(),
 169                  0.00001);
 170
 171     // Check if the throttle point gets updated.
 172     long newThrottlePoint =
 173         hstore.getStoreEngine().getCompactionPolicy().getConf().getThrottlePoint() + 10;
 174     conf.setLong("hbase.regionserver.thread.compaction.throttle",
 175                   newThrottlePoint);
 176     rs1.getConfigurationManager().notifyAllObservers(conf);
 177     assertEquals(newThrottlePoint,
 178         hstore.getStoreEngine().getCompactionPolicy().getConf().getThrottlePoint());
 179
 180     // Check if the minFilesToCompact gets updated.
 181     int newMinFilesToCompact =
 182             hstore.getStoreEngine().getCompactionPolicy().getConf().getMinFilesToCompact() + 1;
 183     conf.setLong(strPrefix + "min", newMinFilesToCompact);
 184     rs1.getConfigurationManager().notifyAllObservers(conf);
 185     assertEquals(newMinFilesToCompact,
 186         hstore.getStoreEngine().getCompactionPolicy().getConf().getMinFilesToCompact());
 187
 188     // Check if the maxFilesToCompact gets updated.
 189     int newMaxFilesToCompact =
 190             hstore.getStoreEngine().getCompactionPolicy().getConf().getMaxFilesToCompact() + 1;
 191     conf.setLong(strPrefix + "max", newMaxFilesToCompact);
 192     rs1.getConfigurationManager().notifyAllObservers(conf);
 193     assertEquals(newMaxFilesToCompact,
 194         hstore.getStoreEngine().getCompactionPolicy().getConf().getMaxFilesToCompact());
 195
 196     // Check OffPeak hours is updated in an online fashion.
 197     conf.setLong(CompactionConfiguration.HBASE_HSTORE_OFFPEAK_START_HOUR, 6);
 198     conf.setLong(CompactionConfiguration.HBASE_HSTORE_OFFPEAK_END_HOUR, 7);
 199     rs1.getConfigurationManager().notifyAllObservers(conf);
 200     assertFalse(hstore.getOffPeakHours().isOffPeakHour(4));
 201
 202     // Check if the minCompactSize gets updated.
 203     long newMinCompactSize =
 204             hstore.getStoreEngine().getCompactionPolicy().getConf().getMinCompactSize() + 1;
 205     conf.setLong(strPrefix + "min.size", newMinCompactSize);
 206     rs1.getConfigurationManager().notifyAllObservers(conf);
 207     assertEquals(newMinCompactSize,
 208                  hstore.getStoreEngine().getCompactionPolicy().getConf().getMinCompactSize());
 209
 210     // Check if the maxCompactSize gets updated.
 211     long newMaxCompactSize =
 212             hstore.getStoreEngine().getCompactionPolicy().getConf().getMaxCompactSize() - 1;
 213     conf.setLong(strPrefix + "max.size", newMaxCompactSize);
 214     rs1.getConfigurationManager().notifyAllObservers(conf);
 215     assertEquals(newMaxCompactSize,
 216                  hstore.getStoreEngine().getCompactionPolicy().getConf().getMaxCompactSize());
 217     // Check if the offPeakMaxCompactSize gets updated.
 218     long newOffpeakMaxCompactSize =
 219             hstore.getStoreEngine().getCompactionPolicy().getConf().getOffPeakMaxCompactSize() - 1;
 220     conf.setLong(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY,
 221       newOffpeakMaxCompactSize);
 222     rs1.getConfigurationManager().notifyAllObservers(conf);
 223     assertEquals(newOffpeakMaxCompactSize,
 224                  hstore.getStoreEngine().getCompactionPolicy().getConf().getOffPeakMaxCompactSize());
 225
 226     // Check if majorCompactionPeriod gets updated.
 227     long newMajorCompactionPeriod =
 228             hstore.getStoreEngine().getCompactionPolicy().getConf().getMajorCompactionPeriod() + 10;
 229     conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, newMajorCompactionPeriod);
 230     rs1.getConfigurationManager().notifyAllObservers(conf);
 231     assertEquals(newMajorCompactionPeriod,
 232             hstore.getStoreEngine().getCompactionPolicy().getConf().getMajorCompactionPeriod());
 233
 234     // Check if majorCompactionJitter gets updated.
 235     float newMajorCompactionJitter =
 236         hstore.getStoreEngine().getCompactionPolicy().getConf().getMajorCompactionJitter() + 0.02F;
 237     conf.setFloat("hbase.hregion.majorcompaction.jitter",
 238                   newMajorCompactionJitter);
 239     rs1.getConfigurationManager().notifyAllObservers(conf);
 240     assertEquals(newMajorCompactionJitter,
 241       hstore.getStoreEngine().getCompactionPolicy().getConf().getMajorCompactionJitter(), 0.00001);
 242   }
 243
 244   @Test
 245   public void removeClosedRegionFromConfigurationManager() throws Exception {
 246     try (Connection connection = ConnectionFactory.createConnection(conf)) {
 247       Admin admin = connection.getAdmin();
 248       assertTrue("The open region doesn't register as a ConfigurationObserver",
 249         rs1.getConfigurationManager().containsObserver(r1));
 250       admin.move(r1name);
 251       hbaseTestingUtility.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
 252         @Override public boolean evaluate() throws Exception {
 253           return rs1.getOnlineRegion(r1name) == null;
 254         }
 255       });
 256       assertFalse("The closed region is not removed from ConfigurationManager",
 257         rs1.getConfigurationManager().containsObserver(r1));
 258       admin.move(r1name, rs1.getServerName());
 259       hbaseTestingUtility.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
 260         @Override public boolean evaluate() throws Exception {
 261           return rs1.getOnlineRegion(r1name) != null;
 262         }
 263       });
 264     }
 265   }
 266
 267   @Test
 268   public void testStoreConfigurationOnlineChange() {
 269     rs1.getConfigurationManager().notifyAllObservers(conf);
 270     long actualMaxFileSize = r1.getStore(COLUMN_FAMILY1).getReadOnlyConfiguration()
 271         .getLong(TableDescriptorBuilder.MAX_FILESIZE, -1);
 272     assertEquals(MAX_FILE_SIZE, actualMaxFileSize);
 273   }
 274 }