hbase-server/src/test/java/org/apache/hadoop/hbase/TestStochasticBalancerJmxMetrics.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase;
  19
  20 import static org.junit.Assert.assertTrue;
  21
  22 import java.io.IOException;
  23 import java.util.HashSet;
  24 import java.util.Hashtable;
  25 import java.util.Iterator;
  26 import java.util.List;
  27 import java.util.Map;
  28 import java.util.Random;
  29 import java.util.Set;
  30 import javax.management.MBeanAttributeInfo;
  31 import javax.management.MBeanInfo;
  32 import javax.management.MBeanServerConnection;
  33 import javax.management.ObjectInstance;
  34 import javax.management.ObjectName;
  35 import javax.management.remote.JMXConnector;
  36 import javax.management.remote.JMXConnectorFactory;
  37 import org.apache.hadoop.conf.Configuration;
  38 import org.apache.hadoop.hbase.client.RegionInfo;
  39 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
  40 import org.apache.hadoop.hbase.master.balancer.BalancerTestBase;
  41 import org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer;
  42 import org.apache.hadoop.hbase.testclassification.MediumTests;
  43 import org.apache.hadoop.hbase.testclassification.MiscTests;
  44 import org.apache.hadoop.hbase.util.Threads;
  45 import org.apache.hadoop.net.DNSToSwitchMapping;
  46 import org.junit.AfterClass;
  47 import org.junit.BeforeClass;
  48 import org.junit.ClassRule;
  49 import org.junit.FixMethodOrder;
  50 import org.junit.Ignore;
  51 import org.junit.Test;
  52 import org.junit.experimental.categories.Category;
  53 import org.junit.runners.MethodSorters;
  54 import org.slf4j.Logger;
  55 import org.slf4j.LoggerFactory;
  56
  57 @Category({ MiscTests.class, MediumTests.class })
  58 @FixMethodOrder(MethodSorters.NAME_ASCENDING)
  59 @Ignore
  60 public class TestStochasticBalancerJmxMetrics extends BalancerTestBase {
  61
  62   @ClassRule
  63   public static final HBaseClassTestRule CLASS_RULE =
  64       HBaseClassTestRule.forClass(TestStochasticBalancerJmxMetrics.class);
  65
  66   private static final Logger LOG = LoggerFactory.getLogger(TestStochasticBalancerJmxMetrics.class);
  67   private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
  68   private static int connectorPort = 61120;
  69   private static StochasticLoadBalancer loadBalancer;
  70   /**
  71    * a simple cluster for testing JMX.
  72    */
  73   private static int[] mockCluster_ensemble = new int[] { 0, 1, 2, 3 };
  74   private static int[] mockCluster_pertable_1 = new int[] { 0, 1, 2 };
  75   private static int[] mockCluster_pertable_2 = new int[] { 3, 1, 1 };
  76   private static int[] mockCluster_pertable_namespace = new int[] { 1, 3, 1 };
  77
  78   private static final String TABLE_NAME_1 = "Table1";
  79   private static final String TABLE_NAME_2 = "Table2";
  80   private static final String TABLE_NAME_NAMESPACE = "hbase:namespace";
  81
  82   private static Configuration conf = null;
  83
  84   /**
  85    * Setup the environment for the test.
  86    */
  87   @BeforeClass
  88   public static void setupBeforeClass() throws Exception {
  89
  90     conf = UTIL.getConfiguration();
  91
  92     conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class);
  93     conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 0.75f);
  94     conf.setFloat("hbase.regions.slop", 0.0f);
  95     conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, JMXListener.class.getName());
  96     Random rand = new Random();
  97     for (int i = 0; i < 10; i++) {
  98       do {
  99         int sign = i % 2 == 0 ? 1 : -1;
 100         connectorPort += sign * rand.nextInt(100);
 101       } while (!HBaseTestingUtility.available(connectorPort));
 102       try {
 103         conf.setInt("regionserver.rmi.registry.port", connectorPort);
 104
 105         UTIL.startMiniCluster();
 106         break;
 107       } catch (Exception e) {
 108         LOG.debug("Encountered exception when starting cluster. Trying port " + connectorPort, e);
 109         try {
 110           // this is to avoid "IllegalStateException: A mini-cluster is already running"
 111           UTIL.shutdownMiniCluster();
 112         } catch (Exception ex) {
 113           LOG.debug("Encountered exception shutting down cluster", ex);
 114         }
 115       }
 116     }
 117   }
 118
 119   @AfterClass
 120   public static void tearDownAfterClass() throws Exception {
 121     UTIL.shutdownMiniCluster();
 122   }
 123
 124   /**
 125    * In Ensemble mode, there should be only one ensemble table
 126    */
 127   @Test
 128   public void testJmxMetrics_EnsembleMode() throws Exception {
 129     loadBalancer = new StochasticLoadBalancer();
 130
 131     conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, false);
 132     loadBalancer.setConf(conf);
 133
 134     TableName tableName = HConstants.ENSEMBLE_TABLE_NAME;
 135     Map<ServerName, List<RegionInfo>> clusterState = mockClusterServers(mockCluster_ensemble);
 136     loadBalancer.balanceTable(tableName, clusterState);
 137
 138     String[] tableNames = new String[] { tableName.getNameAsString() };
 139     String[] functionNames = loadBalancer.getCostFunctionNames();
 140     Set<String> jmxMetrics = readJmxMetricsWithRetry();
 141     Set<String> expectedMetrics = getExpectedJmxMetrics(tableNames, functionNames);
 142
 143     // printMetrics(jmxMetrics, "existing metrics in ensemble mode");
 144     // printMetrics(expectedMetrics, "expected metrics in ensemble mode");
 145
 146     // assert that every expected is in the JMX
 147     for (String expected : expectedMetrics) {
 148       assertTrue("Metric " + expected + " can not be found in JMX in ensemble mode.",
 149         jmxMetrics.contains(expected));
 150     }
 151   }
 152
 153   /**
 154    * In per-table mode, each table has a set of metrics
 155    */
 156   @Test
 157   public void testJmxMetrics_PerTableMode() throws Exception {
 158     loadBalancer = new StochasticLoadBalancer();
 159
 160     conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, true);
 161     loadBalancer.setConf(conf);
 162
 163     // NOTE the size is normally set in setClusterMetrics, for test purpose, we set it manually
 164     // Tables: hbase:namespace, table1, table2
 165     // Functions: costFunctions, overall
 166     String[] functionNames = loadBalancer.getCostFunctionNames();
 167     loadBalancer.updateMetricsSize(3 * (functionNames.length + 1));
 168
 169     // table 1
 170     TableName tableName = TableName.valueOf(TABLE_NAME_1);
 171     Map<ServerName, List<RegionInfo>> clusterState = mockClusterServers(mockCluster_pertable_1);
 172     loadBalancer.balanceTable(tableName, clusterState);
 173
 174     // table 2
 175     tableName = TableName.valueOf(TABLE_NAME_2);
 176     clusterState = mockClusterServers(mockCluster_pertable_2);
 177     loadBalancer.balanceTable(tableName, clusterState);
 178
 179     // table hbase:namespace
 180     tableName = TableName.valueOf(TABLE_NAME_NAMESPACE);
 181     clusterState = mockClusterServers(mockCluster_pertable_namespace);
 182     loadBalancer.balanceTable(tableName, clusterState);
 183
 184     String[] tableNames = new String[] { TABLE_NAME_1, TABLE_NAME_2, TABLE_NAME_NAMESPACE };
 185     Set<String> jmxMetrics = readJmxMetricsWithRetry();
 186     Set<String> expectedMetrics = getExpectedJmxMetrics(tableNames, functionNames);
 187
 188     // printMetrics(jmxMetrics, "existing metrics in per-table mode");
 189     // printMetrics(expectedMetrics, "expected metrics in per-table mode");
 190
 191     // assert that every expected is in the JMX
 192     for (String expected : expectedMetrics) {
 193       assertTrue("Metric " + expected + " can not be found in JMX in per-table mode.",
 194         jmxMetrics.contains(expected));
 195     }
 196   }
 197
 198   private Set<String> readJmxMetricsWithRetry() throws IOException {
 199     final int count = 0;
 200     for (int i = 0; i < 10; i++) {
 201       Set<String> metrics = readJmxMetrics();
 202       if (metrics != null) return metrics;
 203       LOG.warn("Failed to get jmxmetrics... sleeping, retrying; " + i + " of " + count + " times");
 204       Threads.sleep(1000);
 205     }
 206     return null;
 207   }
 208
 209   /**
 210    * Read the attributes from Hadoop->HBase->Master->Balancer in JMX
 211    * @throws IOException
 212    */
 213   private Set<String> readJmxMetrics() throws IOException {
 214     JMXConnector connector = null;
 215     ObjectName target = null;
 216     MBeanServerConnection mb = null;
 217     try {
 218       connector =
 219           JMXConnectorFactory.connect(JMXListener.buildJMXServiceURL(connectorPort, connectorPort));
 220       mb = connector.getMBeanServerConnection();
 221
 222       Hashtable<String, String> pairs = new Hashtable<>();
 223       pairs.put("service", "HBase");
 224       pairs.put("name", "Master");
 225       pairs.put("sub", "Balancer");
 226       target = new ObjectName("Hadoop", pairs);
 227       MBeanInfo beanInfo = mb.getMBeanInfo(target);
 228
 229       Set<String> existingAttrs = new HashSet<>();
 230       for (MBeanAttributeInfo attrInfo : beanInfo.getAttributes()) {
 231         existingAttrs.add(attrInfo.getName());
 232       }
 233       return existingAttrs;
 234     } catch (Exception e) {
 235       LOG.warn("Failed to get bean!!! " + target, e);
 236       if (mb != null) {
 237         Set<ObjectInstance> instances = mb.queryMBeans(null, null);
 238         Iterator<ObjectInstance> iterator = instances.iterator();
 239         System.out.println("MBean Found:");
 240         while (iterator.hasNext()) {
 241           ObjectInstance instance = iterator.next();
 242           System.out.println("Class Name: " + instance.getClassName());
 243           System.out.println("Object Name: " + instance.getObjectName());
 244         }
 245       }
 246     } finally {
 247       if (connector != null) {
 248         try {
 249           connector.close();
 250         } catch (Exception e) {
 251           e.printStackTrace();
 252         }
 253       }
 254     }
 255     return null;
 256   }
 257
 258   /**
 259    * Given the tables and functions, return metrics names that should exist in JMX
 260    */
 261   private Set<String> getExpectedJmxMetrics(String[] tableNames, String[] functionNames) {
 262     Set<String> ret = new HashSet<>();
 263
 264     for (String tableName : tableNames) {
 265       ret.add(StochasticLoadBalancer.composeAttributeName(tableName, "Overall"));
 266       for (String functionName : functionNames) {
 267         String metricsName = StochasticLoadBalancer.composeAttributeName(tableName, functionName);
 268         ret.add(metricsName);
 269       }
 270     }
 271
 272     return ret;
 273   }
 274
 275   private static void printMetrics(Set<String> metrics, String info) {
 276     if (null != info) LOG.info("++++ ------ " + info + " ------");
 277
 278     LOG.info("++++ metrics count = " + metrics.size());
 279     for (String str : metrics) {
 280       LOG.info(" ++++ " + str);
 281     }
 282   }
 283 }