hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerOnlineConfigChange.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.regionserver;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertFalse;
  22 import static org.junit.Assert.assertNotNull;
  23 import static org.junit.Assert.assertNull;
  24 import static org.junit.Assert.assertTrue;
  25
  26 import java.io.IOException;
  27 import java.util.concurrent.TimeUnit;
  28 import org.apache.hadoop.conf.Configuration;
  29 import org.apache.hadoop.hbase.HBaseClassTestRule;
  30 import org.apache.hadoop.hbase.HBaseTestingUtil;
  31 import org.apache.hadoop.hbase.HConstants;
  32 import org.apache.hadoop.hbase.JMXListener;
  33 import org.apache.hadoop.hbase.TableName;
  34 import org.apache.hadoop.hbase.Waiter;
  35 import org.apache.hadoop.hbase.client.Admin;
  36 import org.apache.hadoop.hbase.client.Connection;
  37 import org.apache.hadoop.hbase.client.ConnectionFactory;
  38 import org.apache.hadoop.hbase.client.RegionInfo;
  39 import org.apache.hadoop.hbase.client.RegionLocator;
  40 import org.apache.hadoop.hbase.client.Table;
  41 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
  42 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
  43 import org.apache.hadoop.hbase.master.HMaster;
  44 import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
  45 import org.apache.hadoop.hbase.testclassification.MediumTests;
  46 import org.apache.hadoop.hbase.util.Bytes;
  47 import org.junit.AfterClass;
  48 import org.junit.Before;
  49 import org.junit.BeforeClass;
  50 import org.junit.ClassRule;
  51 import org.junit.Test;
  52 import org.junit.experimental.categories.Category;
  53 import org.slf4j.Logger;
  54 import org.slf4j.LoggerFactory;
  55
  56 /**
  57  * Verify that the Online config Changes on the HRegionServer side are actually
  58  * happening. We should add tests for important configurations which will be
  59  * changed online.
  60  */
  61
  62 @Category({MediumTests.class})
  63 public class TestRegionServerOnlineConfigChange {
  64
  65   @ClassRule
  66   public static final HBaseClassTestRule CLASS_RULE =
  67       HBaseClassTestRule.forClass(TestRegionServerOnlineConfigChange.class);
  68
  69   private static final Logger LOG =
  70           LoggerFactory.getLogger(TestRegionServerOnlineConfigChange.class.getName());
  71   private static final long WAIT_TIMEOUT = TimeUnit.MINUTES.toMillis(2);
  72   private static HBaseTestingUtil hbaseTestingUtility = new HBaseTestingUtil();
  73   private static Configuration conf = null;
  74
  75   private static Table t1 = null;
  76   private static HRegionServer rs1 = null;
  77   private static HMaster hMaster = null;
  78   private static byte[] r1name = null;
  79   private static Region r1 = null;
  80
  81   private final static String table1Str = "table1";
  82   private final static String columnFamily1Str = "columnFamily1";
  83   private final static TableName TABLE1 = TableName.valueOf(table1Str);
  84   private final static byte[] COLUMN_FAMILY1 = Bytes.toBytes(columnFamily1Str);
  85   private final static long MAX_FILE_SIZE = 20 * 1024 * 1024L;
  86
  87
  88   @BeforeClass
  89   public static void setUpBeforeClass() throws Exception {
  90     conf = hbaseTestingUtility.getConfiguration();
  91     hbaseTestingUtility.startMiniCluster(2);
  92     t1 = hbaseTestingUtility.createTable(
  93       TableDescriptorBuilder.newBuilder(TABLE1).setMaxFileSize(MAX_FILE_SIZE).build(),
  94       new byte[][] { COLUMN_FAMILY1 }, conf);
  95   }
  96
  97   @AfterClass
  98   public static void tearDown() throws Exception {
  99     hbaseTestingUtility.shutdownMiniCluster();
 100   }
 101
 102   @Before
 103   public void setUp() throws Exception {
 104     try (RegionLocator locator = hbaseTestingUtility.getConnection().getRegionLocator(TABLE1)) {
 105       RegionInfo firstHRI = locator.getAllRegionLocations().get(0).getRegion();
 106       r1name = firstHRI.getRegionName();
 107       rs1 = hbaseTestingUtility.getHBaseCluster().getRegionServer(
 108         hbaseTestingUtility.getHBaseCluster().getServerWith(r1name));
 109       r1 = rs1.getRegion(r1name);
 110       hMaster = hbaseTestingUtility.getHBaseCluster().getMaster();
 111     }
 112   }
 113
 114   /**
 115    * Check if the number of compaction threads changes online
 116    */
 117   @Test
 118   public void testNumCompactionThreadsOnlineChange() {
 119     assertNotNull(rs1.getCompactSplitThread());
 120     int newNumSmallThreads =
 121       rs1.getCompactSplitThread().getSmallCompactionThreadNum() + 1;
 122     int newNumLargeThreads =
 123       rs1.getCompactSplitThread().getLargeCompactionThreadNum() + 1;
 124
 125     conf.setInt("hbase.regionserver.thread.compaction.small",
 126       newNumSmallThreads);
 127     conf.setInt("hbase.regionserver.thread.compaction.large",
 128       newNumLargeThreads);
 129     rs1.getConfigurationManager().notifyAllObservers(conf);
 130
 131     assertEquals(newNumSmallThreads,
 132       rs1.getCompactSplitThread().getSmallCompactionThreadNum());
 133     assertEquals(newNumLargeThreads,
 134       rs1.getCompactSplitThread().getLargeCompactionThreadNum());
 135   }
 136
 137   /**
 138    * Test that the configurations in the CompactionConfiguration class change
 139    * properly.
 140    *
 141    * @throws IOException
 142    */
 143   @Test
 144   public void testCompactionConfigurationOnlineChange() throws IOException {
 145     String strPrefix = "hbase.hstore.compaction.";
 146     Store s = r1.getStore(COLUMN_FAMILY1);
 147     if (!(s instanceof HStore)) {
 148       LOG.error("Can't test the compaction configuration of HStore class. "
 149           + "Got a different implementation other than HStore");
 150       return;
 151     }
 152     HStore hstore = (HStore)s;
 153
 154     // Set the new compaction ratio to a different value.
 155     double newCompactionRatio =
 156             hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatio() + 0.1;
 157     conf.setFloat(strPrefix + "ratio", (float)newCompactionRatio);
 158
 159     // Notify all the observers, which includes the Store object.
 160     rs1.getConfigurationManager().notifyAllObservers(conf);
 161
 162     // Check if the compaction ratio got updated in the Compaction Configuration
 163     assertEquals(newCompactionRatio,
 164                  hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatio(),
 165                  0.00001);
 166
 167     // Check if the off peak compaction ratio gets updated.
 168     double newOffPeakCompactionRatio =
 169         hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatioOffPeak() + 0.1;
 170     conf.setFloat(strPrefix + "ratio.offpeak",
 171             (float)newOffPeakCompactionRatio);
 172     rs1.getConfigurationManager().notifyAllObservers(conf);
 173     assertEquals(newOffPeakCompactionRatio,
 174         hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatioOffPeak(),
 175                  0.00001);
 176
 177     // Check if the throttle point gets updated.
 178     long newThrottlePoint =
 179         hstore.getStoreEngine().getCompactionPolicy().getConf().getThrottlePoint() + 10;
 180     conf.setLong("hbase.regionserver.thread.compaction.throttle",
 181                   newThrottlePoint);
 182     rs1.getConfigurationManager().notifyAllObservers(conf);
 183     assertEquals(newThrottlePoint,
 184         hstore.getStoreEngine().getCompactionPolicy().getConf().getThrottlePoint());
 185
 186     // Check if the minFilesToCompact gets updated.
 187     int newMinFilesToCompact =
 188             hstore.getStoreEngine().getCompactionPolicy().getConf().getMinFilesToCompact() + 1;
 189     conf.setLong(strPrefix + "min", newMinFilesToCompact);
 190     rs1.getConfigurationManager().notifyAllObservers(conf);
 191     assertEquals(newMinFilesToCompact,
 192         hstore.getStoreEngine().getCompactionPolicy().getConf().getMinFilesToCompact());
 193
 194     // Check if the maxFilesToCompact gets updated.
 195     int newMaxFilesToCompact =
 196             hstore.getStoreEngine().getCompactionPolicy().getConf().getMaxFilesToCompact() + 1;
 197     conf.setLong(strPrefix + "max", newMaxFilesToCompact);
 198     rs1.getConfigurationManager().notifyAllObservers(conf);
 199     assertEquals(newMaxFilesToCompact,
 200         hstore.getStoreEngine().getCompactionPolicy().getConf().getMaxFilesToCompact());
 201
 202     // Check OffPeak hours is updated in an online fashion.
 203     conf.setLong(CompactionConfiguration.HBASE_HSTORE_OFFPEAK_START_HOUR, 6);
 204     conf.setLong(CompactionConfiguration.HBASE_HSTORE_OFFPEAK_END_HOUR, 7);
 205     rs1.getConfigurationManager().notifyAllObservers(conf);
 206     assertFalse(hstore.getOffPeakHours().isOffPeakHour(4));
 207
 208     // Check if the minCompactSize gets updated.
 209     long newMinCompactSize =
 210             hstore.getStoreEngine().getCompactionPolicy().getConf().getMinCompactSize() + 1;
 211     conf.setLong(strPrefix + "min.size", newMinCompactSize);
 212     rs1.getConfigurationManager().notifyAllObservers(conf);
 213     assertEquals(newMinCompactSize,
 214                  hstore.getStoreEngine().getCompactionPolicy().getConf().getMinCompactSize());
 215
 216     // Check if the maxCompactSize gets updated.
 217     long newMaxCompactSize =
 218             hstore.getStoreEngine().getCompactionPolicy().getConf().getMaxCompactSize() - 1;
 219     conf.setLong(strPrefix + "max.size", newMaxCompactSize);
 220     rs1.getConfigurationManager().notifyAllObservers(conf);
 221     assertEquals(newMaxCompactSize,
 222                  hstore.getStoreEngine().getCompactionPolicy().getConf().getMaxCompactSize());
 223     // Check if the offPeakMaxCompactSize gets updated.
 224     long newOffpeakMaxCompactSize =
 225             hstore.getStoreEngine().getCompactionPolicy().getConf().getOffPeakMaxCompactSize() - 1;
 226     conf.setLong(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY,
 227       newOffpeakMaxCompactSize);
 228     rs1.getConfigurationManager().notifyAllObservers(conf);
 229     assertEquals(newOffpeakMaxCompactSize,
 230                  hstore.getStoreEngine().getCompactionPolicy().getConf().getOffPeakMaxCompactSize());
 231
 232     // Check if majorCompactionPeriod gets updated.
 233     long newMajorCompactionPeriod =
 234             hstore.getStoreEngine().getCompactionPolicy().getConf().getMajorCompactionPeriod() + 10;
 235     conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, newMajorCompactionPeriod);
 236     rs1.getConfigurationManager().notifyAllObservers(conf);
 237     assertEquals(newMajorCompactionPeriod,
 238             hstore.getStoreEngine().getCompactionPolicy().getConf().getMajorCompactionPeriod());
 239
 240     // Check if majorCompactionJitter gets updated.
 241     float newMajorCompactionJitter =
 242         hstore.getStoreEngine().getCompactionPolicy().getConf().getMajorCompactionJitter() + 0.02F;
 243     conf.setFloat("hbase.hregion.majorcompaction.jitter",
 244                   newMajorCompactionJitter);
 245     rs1.getConfigurationManager().notifyAllObservers(conf);
 246     assertEquals(newMajorCompactionJitter,
 247       hstore.getStoreEngine().getCompactionPolicy().getConf().getMajorCompactionJitter(), 0.00001);
 248   }
 249
 250   @Test
 251   public void removeClosedRegionFromConfigurationManager() throws Exception {
 252     try (Connection connection = ConnectionFactory.createConnection(conf)) {
 253       Admin admin = connection.getAdmin();
 254       assertTrue("The open region doesn't register as a ConfigurationObserver",
 255         rs1.getConfigurationManager().containsObserver(r1));
 256       admin.move(r1name);
 257       hbaseTestingUtility.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
 258         @Override public boolean evaluate() throws Exception {
 259           return rs1.getOnlineRegion(r1name) == null;
 260         }
 261       });
 262       assertFalse("The closed region is not removed from ConfigurationManager",
 263         rs1.getConfigurationManager().containsObserver(r1));
 264       admin.move(r1name, rs1.getServerName());
 265       hbaseTestingUtility.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
 266         @Override public boolean evaluate() throws Exception {
 267           return rs1.getOnlineRegion(r1name) != null;
 268         }
 269       });
 270     }
 271   }
 272
 273   @Test
 274   public void testStoreConfigurationOnlineChange() {
 275     rs1.getConfigurationManager().notifyAllObservers(conf);
 276     long actualMaxFileSize = r1.getStore(COLUMN_FAMILY1).getReadOnlyConfiguration()
 277         .getLong(TableDescriptorBuilder.MAX_FILESIZE, -1);
 278     assertEquals(MAX_FILE_SIZE, actualMaxFileSize);
 279   }
 280
 281   @Test
 282   public void testCoprocessorConfigurationOnlineChange() {
 283     assertNull(rs1.getRegionServerCoprocessorHost().findCoprocessor(JMXListener.class.getName()));
 284     conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, JMXListener.class.getName());
 285     rs1.getConfigurationManager().notifyAllObservers(conf);
 286     assertNotNull(
 287       rs1.getRegionServerCoprocessorHost().findCoprocessor(JMXListener.class.getName()));
 288   }
 289
 290   @Test
 291   public void testCoprocessorConfigurationOnlineChangeOnMaster() {
 292     assertNull(hMaster.getMasterCoprocessorHost().findCoprocessor(JMXListener.class.getName()));
 293     conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, JMXListener.class.getName());
 294     assertFalse(hMaster.isInMaintenanceMode());
 295     hMaster.getConfigurationManager().notifyAllObservers(conf);
 296     assertNotNull(
 297       hMaster.getMasterCoprocessorHost().findCoprocessor(JMXListener.class.getName()));
 298   }
 299
 300 }