hbase-server/src/test/java/org/apache/hadoop/hbase/TestClientClusterMetrics.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase;
  19
  20 import java.io.IOException;
  21 import java.security.PrivilegedAction;
  22 import java.util.EnumSet;
  23 import java.util.List;
  24 import java.util.Map;
  25 import java.util.Optional;
  26 import java.util.concurrent.CompletableFuture;
  27 import java.util.concurrent.atomic.AtomicInteger;
  28
  29 import org.apache.hadoop.conf.Configuration;
  30 import org.apache.hadoop.hbase.ClusterMetrics.Option;
  31 import org.apache.hadoop.hbase.Waiter.Predicate;
  32 import org.apache.hadoop.hbase.client.Admin;
  33 import org.apache.hadoop.hbase.client.AsyncAdmin;
  34 import org.apache.hadoop.hbase.client.AsyncConnection;
  35 import org.apache.hadoop.hbase.client.ClusterConnectionFactory;
  36 import org.apache.hadoop.hbase.client.Connection;
  37 import org.apache.hadoop.hbase.client.ConnectionFactory;
  38 import org.apache.hadoop.hbase.client.Get;
  39 import org.apache.hadoop.hbase.client.Put;
  40 import org.apache.hadoop.hbase.client.RegionInfoBuilder;
  41 import org.apache.hadoop.hbase.client.RegionStatesCount;
  42 import org.apache.hadoop.hbase.client.Result;
  43 import org.apache.hadoop.hbase.client.Scan;
  44 import org.apache.hadoop.hbase.client.Table;
  45 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
  46 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
  47 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
  48 import org.apache.hadoop.hbase.coprocessor.MasterObserver;
  49 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
  50 import org.apache.hadoop.hbase.filter.FilterAllFilter;
  51 import org.apache.hadoop.hbase.master.HMaster;
  52 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
  53 import org.apache.hadoop.hbase.regionserver.HRegionServer;
  54 import org.apache.hadoop.hbase.regionserver.MetricsUserAggregateFactory;
  55 import org.apache.hadoop.hbase.security.User;
  56 import org.apache.hadoop.hbase.security.UserProvider;
  57 import org.apache.hadoop.hbase.testclassification.MediumTests;
  58 import org.apache.hadoop.hbase.util.Bytes;
  59 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
  60 import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
  61 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
  62 import org.junit.AfterClass;
  63 import org.junit.Assert;
  64 import org.junit.BeforeClass;
  65 import org.junit.ClassRule;
  66 import org.junit.Test;
  67 import org.junit.experimental.categories.Category;
  68
  69 @Category(MediumTests.class)
  70 public class TestClientClusterMetrics {
  71
  72   @ClassRule
  73   public static final HBaseClassTestRule CLASS_RULE =
  74       HBaseClassTestRule.forClass(TestClientClusterMetrics.class);
  75
  76   private static HBaseTestingUtil UTIL;
  77   private static Admin ADMIN;
  78   private final static int SLAVES = 5;
  79   private final static int MASTERS = 3;
  80   private static SingleProcessHBaseCluster CLUSTER;
  81   private static HRegionServer DEAD;
  82   private static final TableName TABLE_NAME = TableName.valueOf("test");
  83   private static final byte[] CF = Bytes.toBytes("cf");
  84
  85   // We need to promote the visibility of tryRegionServerReport for this test
  86   public static class MyRegionServer
  87       extends SingleProcessHBaseCluster.MiniHBaseClusterRegionServer {
  88     public MyRegionServer(Configuration conf) throws IOException, InterruptedException {
  89       super(conf);
  90     }
  91     @Override
  92     public void tryRegionServerReport(long reportStartTime, long reportEndTime)
  93         throws IOException {
  94       super.tryRegionServerReport(reportStartTime, reportEndTime);
  95     }
  96   }
  97
  98   @BeforeClass
  99   public static void setUpBeforeClass() throws Exception {
 100     Configuration conf = HBaseConfiguration.create();
 101     conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MyObserver.class.getName());
 102     UTIL = new HBaseTestingUtil(conf);
 103     StartTestingClusterOption option = StartTestingClusterOption.builder()
 104         .rsClass(TestClientClusterMetrics.MyRegionServer.class)
 105         .numMasters(MASTERS).numRegionServers(SLAVES).numDataNodes(SLAVES).build();
 106     UTIL.startMiniCluster(option);
 107     CLUSTER = UTIL.getHBaseCluster();
 108     CLUSTER.waitForActiveAndReadyMaster();
 109     ADMIN = UTIL.getAdmin();
 110     // Kill one region server
 111     List<RegionServerThread> rsts = CLUSTER.getLiveRegionServerThreads();
 112     RegionServerThread rst = rsts.get(rsts.size() - 1);
 113     DEAD = rst.getRegionServer();
 114     DEAD.stop("Test dead servers metrics");
 115     while (rst.isAlive()) {
 116       Thread.sleep(500);
 117     }
 118   }
 119
 120   @Test
 121   public void testDefaults() throws Exception {
 122     ClusterMetrics origin = ADMIN.getClusterMetrics();
 123     ClusterMetrics defaults = ADMIN.getClusterMetrics(EnumSet.allOf(Option.class));
 124     Assert.assertEquals(origin.getHBaseVersion(), defaults.getHBaseVersion());
 125     Assert.assertEquals(origin.getClusterId(), defaults.getClusterId());
 126     Assert.assertEquals(origin.getAverageLoad(), defaults.getAverageLoad(), 0);
 127     Assert.assertEquals(origin.getBackupMasterNames().size(),
 128         defaults.getBackupMasterNames().size());
 129     Assert.assertEquals(origin.getDeadServerNames().size(), defaults.getDeadServerNames().size());
 130     Assert.assertEquals(origin.getRegionCount(), defaults.getRegionCount());
 131     Assert.assertEquals(origin.getLiveServerMetrics().size(),
 132         defaults.getLiveServerMetrics().size());
 133     Assert.assertEquals(origin.getMasterInfoPort(), defaults.getMasterInfoPort());
 134     Assert.assertEquals(origin.getServersName().size(), defaults.getServersName().size());
 135     Assert.assertEquals(ADMIN.getRegionServers().size(), defaults.getServersName().size());
 136   }
 137
 138   @Test
 139   public void testAsyncClient() throws Exception {
 140     try (AsyncConnection asyncConnect = ConnectionFactory.createAsyncConnection(
 141       UTIL.getConfiguration()).get()) {
 142       AsyncAdmin asyncAdmin = asyncConnect.getAdmin();
 143       CompletableFuture<ClusterMetrics> originFuture =
 144         asyncAdmin.getClusterMetrics();
 145       CompletableFuture<ClusterMetrics> defaultsFuture =
 146         asyncAdmin.getClusterMetrics(EnumSet.allOf(Option.class));
 147       ClusterMetrics origin = originFuture.get();
 148       ClusterMetrics defaults = defaultsFuture.get();
 149       Assert.assertEquals(origin.getHBaseVersion(), defaults.getHBaseVersion());
 150       Assert.assertEquals(origin.getClusterId(), defaults.getClusterId());
 151       Assert.assertEquals(origin.getHBaseVersion(), defaults.getHBaseVersion());
 152       Assert.assertEquals(origin.getClusterId(), defaults.getClusterId());
 153       Assert.assertEquals(origin.getAverageLoad(), defaults.getAverageLoad(), 0);
 154       Assert.assertEquals(origin.getBackupMasterNames().size(),
 155         defaults.getBackupMasterNames().size());
 156       Assert.assertEquals(origin.getDeadServerNames().size(), defaults.getDeadServerNames().size());
 157       Assert.assertEquals(origin.getRegionCount(), defaults.getRegionCount());
 158       Assert.assertEquals(origin.getLiveServerMetrics().size(),
 159         defaults.getLiveServerMetrics().size());
 160       Assert.assertEquals(origin.getMasterInfoPort(), defaults.getMasterInfoPort());
 161       Assert.assertEquals(origin.getServersName().size(), defaults.getServersName().size());
 162       origin.getTableRegionStatesCount().forEach(((tableName, regionStatesCount) -> {
 163         RegionStatesCount defaultRegionStatesCount = defaults.getTableRegionStatesCount()
 164           .get(tableName);
 165         Assert.assertEquals(defaultRegionStatesCount, regionStatesCount);
 166       }));
 167     }
 168   }
 169
 170   @Test
 171   public void testLiveAndDeadServersStatus() throws Exception {
 172     // Count the number of live regionservers
 173     List<RegionServerThread> regionserverThreads = CLUSTER.getLiveRegionServerThreads();
 174     int numRs = 0;
 175     int len = regionserverThreads.size();
 176     for (int i = 0; i < len; i++) {
 177       if (regionserverThreads.get(i).isAlive()) {
 178         numRs++;
 179       }
 180     }
 181     // Depending on the (random) order of unit execution we may run this unit before the
 182     // minicluster is fully up and recovered from the RS shutdown done during test init.
 183     Waiter.waitFor(CLUSTER.getConfiguration(), 10 * 1000, 100, new Predicate<Exception>() {
 184       @Override
 185       public boolean evaluate() throws Exception {
 186         ClusterMetrics metrics = ADMIN.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS));
 187         Assert.assertNotNull(metrics);
 188         return metrics.getRegionCount() > 0;
 189       }
 190     });
 191     // Retrieve live servers and dead servers info.
 192     EnumSet<Option> options =
 193         EnumSet.of(Option.LIVE_SERVERS, Option.DEAD_SERVERS, Option.SERVERS_NAME);
 194     ClusterMetrics metrics = ADMIN.getClusterMetrics(options);
 195     Assert.assertNotNull(metrics);
 196     // exclude a dead region server
 197     Assert.assertEquals(SLAVES -1, numRs);
 198     // live servers = nums of regionservers
 199     // By default, HMaster don't carry any regions so it won't report its load.
 200     // Hence, it won't be in the server list.
 201     Assert.assertEquals(numRs, metrics.getLiveServerMetrics().size());
 202     Assert.assertTrue(metrics.getRegionCount() > 0);
 203     Assert.assertNotNull(metrics.getDeadServerNames());
 204     Assert.assertEquals(1, metrics.getDeadServerNames().size());
 205     ServerName deadServerName = metrics.getDeadServerNames().iterator().next();
 206     Assert.assertEquals(DEAD.getServerName(), deadServerName);
 207     Assert.assertNotNull(metrics.getServersName());
 208     Assert.assertEquals(numRs, metrics.getServersName().size());
 209   }
 210
 211   @Test
 212   public void testRegionStatesCount() throws Exception {
 213     Table table = UTIL.createTable(TABLE_NAME, CF);
 214     table.put(new Put(Bytes.toBytes("k1"))
 215       .addColumn(CF, Bytes.toBytes("q1"), Bytes.toBytes("v1")));
 216     table.put(new Put(Bytes.toBytes("k2"))
 217       .addColumn(CF, Bytes.toBytes("q2"), Bytes.toBytes("v2")));
 218     table.put(new Put(Bytes.toBytes("k3"))
 219       .addColumn(CF, Bytes.toBytes("q3"), Bytes.toBytes("v3")));
 220
 221     ClusterMetrics metrics = ADMIN.getClusterMetrics();
 222     Assert.assertEquals(metrics.getTableRegionStatesCount().size(), 2);
 223     Assert.assertEquals(metrics.getTableRegionStatesCount().get(TableName.META_TABLE_NAME)
 224       .getRegionsInTransition(), 0);
 225     Assert.assertEquals(metrics.getTableRegionStatesCount().get(TableName.META_TABLE_NAME)
 226       .getOpenRegions(), 1);
 227     Assert.assertEquals(metrics.getTableRegionStatesCount().get(TableName.META_TABLE_NAME)
 228       .getTotalRegions(), 1);
 229     Assert.assertEquals(metrics.getTableRegionStatesCount().get(TableName.META_TABLE_NAME)
 230       .getClosedRegions(), 0);
 231     Assert.assertEquals(metrics.getTableRegionStatesCount().get(TableName.META_TABLE_NAME)
 232       .getSplitRegions(), 0);
 233     Assert.assertEquals(metrics.getTableRegionStatesCount().get(TABLE_NAME)
 234       .getRegionsInTransition(), 0);
 235     Assert.assertEquals(metrics.getTableRegionStatesCount().get(TABLE_NAME)
 236       .getOpenRegions(), 1);
 237     Assert.assertEquals(metrics.getTableRegionStatesCount().get(TABLE_NAME)
 238       .getTotalRegions(), 1);
 239
 240     UTIL.deleteTable(TABLE_NAME);
 241   }
 242
 243   @Test
 244   public void testRegionStatesWithSplit() throws Exception {
 245     int startRowNum = 20;
 246     int rowCount = 80;
 247     Table table = UTIL.createTable(TABLE_NAME, CF);
 248     table.put(new Put(Bytes.toBytes("k1"))
 249       .addColumn(CF, Bytes.toBytes("q1"), Bytes.toBytes("v1")));
 250     table.put(new Put(Bytes.toBytes("k2"))
 251       .addColumn(CF, Bytes.toBytes("q2"), Bytes.toBytes("v2")));
 252
 253     insertData(TABLE_NAME, startRowNum, rowCount);
 254
 255     ClusterMetrics metrics = ADMIN.getClusterMetrics();
 256     Assert.assertEquals(metrics.getTableRegionStatesCount().size(), 2);
 257     Assert.assertEquals(metrics.getTableRegionStatesCount().get(TableName.META_TABLE_NAME)
 258       .getRegionsInTransition(), 0);
 259     Assert.assertEquals(metrics.getTableRegionStatesCount().get(TableName.META_TABLE_NAME)
 260       .getOpenRegions(), 1);
 261     Assert.assertEquals(metrics.getTableRegionStatesCount().get(TableName.META_TABLE_NAME)
 262       .getTotalRegions(), 1);
 263     Assert.assertEquals(metrics.getTableRegionStatesCount().get(TABLE_NAME)
 264       .getRegionsInTransition(), 0);
 265     Assert.assertEquals(metrics.getTableRegionStatesCount().get(TABLE_NAME)
 266       .getOpenRegions(), 1);
 267     Assert.assertEquals(metrics.getTableRegionStatesCount().get(TABLE_NAME)
 268       .getTotalRegions(), 1);
 269
 270     int splitRowNum = startRowNum + rowCount / 2;
 271     byte[] splitKey = Bytes.toBytes("" + splitRowNum);
 272
 273     // Split region of the table
 274     ADMIN.split(TABLE_NAME, splitKey);
 275
 276     metrics = ADMIN.getClusterMetrics();
 277     Assert.assertEquals(metrics.getTableRegionStatesCount().size(), 2);
 278     Assert.assertEquals(metrics.getTableRegionStatesCount().get(TableName.META_TABLE_NAME)
 279       .getRegionsInTransition(), 0);
 280     Assert.assertEquals(metrics.getTableRegionStatesCount().get(TableName.META_TABLE_NAME)
 281       .getOpenRegions(), 1);
 282     Assert.assertEquals(metrics.getTableRegionStatesCount().get(TableName.META_TABLE_NAME)
 283       .getTotalRegions(), 1);
 284     Assert.assertEquals(metrics.getTableRegionStatesCount().get(TABLE_NAME)
 285       .getRegionsInTransition(), 0);
 286     Assert.assertEquals(metrics.getTableRegionStatesCount().get(TABLE_NAME)
 287       .getOpenRegions(), 2);
 288     Assert.assertEquals(metrics.getTableRegionStatesCount().get(TABLE_NAME)
 289       .getTotalRegions(), 3);
 290     Assert.assertEquals(metrics.getTableRegionStatesCount().get(TABLE_NAME)
 291       .getSplitRegions(), 1);
 292     Assert.assertEquals(metrics.getTableRegionStatesCount().get(TABLE_NAME)
 293       .getClosedRegions(), 0);
 294
 295     UTIL.deleteTable(TABLE_NAME);
 296   }
 297
 298   @Test public void testMasterAndBackupMastersStatus() throws Exception {
 299     // get all the master threads
 300     List<MasterThread> masterThreads = CLUSTER.getMasterThreads();
 301     int numActive = 0;
 302     int activeIndex = 0;
 303     ServerName activeName = null;
 304     HMaster active = null;
 305     for (int i = 0; i < masterThreads.size(); i++) {
 306       if (masterThreads.get(i).getMaster().isActiveMaster()) {
 307         numActive++;
 308         activeIndex = i;
 309         active = masterThreads.get(activeIndex).getMaster();
 310         activeName = active.getServerName();
 311       }
 312     }
 313     Assert.assertNotNull(active);
 314     Assert.assertEquals(1, numActive);
 315     Assert.assertEquals(MASTERS, masterThreads.size());
 316     // Retrieve master and backup masters infos only.
 317     EnumSet<Option> options = EnumSet.of(Option.MASTER, Option.BACKUP_MASTERS);
 318     ClusterMetrics metrics = ADMIN.getClusterMetrics(options);
 319     Assert.assertTrue(metrics.getMasterName().equals(activeName));
 320     Assert.assertEquals(MASTERS - 1, metrics.getBackupMasterNames().size());
 321   }
 322
 323   @Test
 324   public void testUserMetrics() throws Exception {
 325     Configuration conf = UTIL.getConfiguration();
 326     // If metrics for users is not enabled, this test doesn't  make sense.
 327     if (!conf.getBoolean(MetricsUserAggregateFactory.METRIC_USER_ENABLED_CONF,
 328         MetricsUserAggregateFactory.DEFAULT_METRIC_USER_ENABLED_CONF)) {
 329       return;
 330     }
 331     User userFoo = User.createUserForTesting(conf, "FOO_USER_METRIC_TEST", new String[0]);
 332     User userBar = User.createUserForTesting(conf, "BAR_USER_METRIC_TEST", new String[0]);
 333     User userTest = User.createUserForTesting(conf, "TEST_USER_METRIC_TEST", new String[0]);
 334     UTIL.createTable(TABLE_NAME, CF);
 335     waitForUsersMetrics(0);
 336     long writeMetaMetricBeforeNextuser = getMetaMetrics().getWriteRequestCount();
 337     userFoo.runAs(new PrivilegedAction<Void>() {
 338       @Override public Void run() {
 339         try {
 340           doPut();
 341         } catch (IOException e) {
 342           Assert.fail("Exception:" + e.getMessage());
 343         }
 344         return null;
 345       }
 346     });
 347     waitForUsersMetrics(1);
 348     long writeMetaMetricForUserFoo =
 349         getMetaMetrics().getWriteRequestCount() - writeMetaMetricBeforeNextuser;
 350     long readMetaMetricBeforeNextuser = getMetaMetrics().getReadRequestCount();
 351     userBar.runAs(new PrivilegedAction<Void>() {
 352       @Override public Void run() {
 353         try {
 354           doGet();
 355         } catch (IOException e) {
 356           Assert.fail("Exception:" + e.getMessage());
 357         }
 358         return null;
 359       }
 360     });
 361     waitForUsersMetrics(2);
 362     long readMetaMetricForUserBar =
 363         getMetaMetrics().getReadRequestCount() - readMetaMetricBeforeNextuser;
 364     long filteredMetaReqeust = getMetaMetrics().getFilteredReadRequestCount();
 365     userTest.runAs(new PrivilegedAction<Void>() {
 366       @Override public Void run() {
 367         try {
 368           Table table = createConnection(UTIL.getConfiguration()).getTable(TABLE_NAME);
 369           for (Result result : table.getScanner(new Scan().setFilter(new FilterAllFilter()))) {
 370             Assert.fail("Should have filtered all rows");
 371           }
 372         } catch (IOException e) {
 373           Assert.fail("Exception:" + e.getMessage());
 374         }
 375         return null;
 376       }
 377     });
 378     waitForUsersMetrics(3);
 379     long filteredMetaReqeustForTestUser =
 380         getMetaMetrics().getFilteredReadRequestCount() - filteredMetaReqeust;
 381     Map<byte[], UserMetrics> userMap =
 382         ADMIN.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().values()
 383             .iterator().next().getUserMetrics();
 384     for (byte[] user : userMap.keySet()) {
 385       switch (Bytes.toString(user)) {
 386         case "FOO_USER_METRIC_TEST":
 387           Assert.assertEquals(1,
 388               userMap.get(user).getWriteRequestCount() - writeMetaMetricForUserFoo);
 389           break;
 390         case "BAR_USER_METRIC_TEST":
 391           Assert
 392               .assertEquals(1, userMap.get(user).getReadRequestCount() - readMetaMetricForUserBar);
 393           Assert.assertEquals(0, userMap.get(user).getWriteRequestCount());
 394           break;
 395         case "TEST_USER_METRIC_TEST":
 396           Assert.assertEquals(1,
 397               userMap.get(user).getFilteredReadRequests() - filteredMetaReqeustForTestUser);
 398           Assert.assertEquals(0, userMap.get(user).getWriteRequestCount());
 399           break;
 400         default:
 401           //current user
 402           Assert.assertEquals(UserProvider.instantiate(conf).getCurrent().getName(),
 403               Bytes.toString(user));
 404           //Read/write count because of Meta operations
 405           Assert.assertTrue(userMap.get(user).getReadRequestCount() > 1);
 406           break;
 407       }
 408     }
 409     UTIL.deleteTable(TABLE_NAME);
 410   }
 411
 412   @Test
 413   public void testServerTasks() throws Exception {
 414     // TaskMonitor is a singleton per VM, so will be shared among all minicluster "servers",
 415     // so we only need to look at the first live server's results to find it.
 416     final String testTaskName = "TEST TASK";
 417     TaskMonitor.get().createStatus(testTaskName).setStatus("Testing 1... 2... 3...");
 418     // Of course, first we must trigger regionserver reports.
 419     final long now = EnvironmentEdgeManager.currentTime();
 420     final long last = now - 1000; // fake a period, or someone might div by zero
 421     for (RegionServerThread rs: CLUSTER.getRegionServerThreads()) {
 422       ((MyRegionServer)rs.getRegionServer()).tryRegionServerReport(last, now);
 423     }
 424     // Get status now
 425     ClusterMetrics clusterMetrics = ADMIN.getClusterMetrics(EnumSet.of(Option.TASKS));
 426     // The test task will be in the master metrics list
 427     boolean found = false;
 428     for (ServerTask task: clusterMetrics.getMasterTasks()) {
 429       if (testTaskName.equals(task.getDescription())) {
 430         // Found it
 431         found = true;
 432         break;
 433       }
 434     }
 435     Assert.assertTrue("Expected task not found in master task list", found);
 436     // Get the tasks information (carried in server metrics)
 437     found = false;
 438     for (ServerMetrics serverMetrics: clusterMetrics.getLiveServerMetrics().values()) {
 439       if (serverMetrics.getTasks() != null) {
 440         for (ServerTask task: serverMetrics.getTasks()) {
 441           if (testTaskName.equals(task.getDescription())) {
 442             // Found it
 443             found = true;
 444             break;
 445           }
 446         }
 447       }
 448     }
 449     // We will fall through here if getClusterMetrics(TASKS) did not correctly process the
 450     // task list.
 451     Assert.assertTrue("Expected task not found in server load", found);
 452   }
 453
 454   private RegionMetrics getMetaMetrics() throws IOException {
 455     for (ServerMetrics serverMetrics : ADMIN.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
 456         .getLiveServerMetrics().values()) {
 457       RegionMetrics metaMetrics = serverMetrics.getRegionMetrics()
 458           .get(RegionInfoBuilder.FIRST_META_REGIONINFO.getRegionName());
 459       if (metaMetrics != null) {
 460         return metaMetrics;
 461       }
 462     }
 463     Assert.fail("Should have find meta metrics");
 464     return null;
 465   }
 466
 467   private void waitForUsersMetrics(int noOfUsers) throws Exception {
 468     //Sleep for metrics to get updated on master
 469     Thread.sleep(5000);
 470     Waiter.waitFor(CLUSTER.getConfiguration(), 10 * 1000, 100, new Predicate<Exception>() {
 471       @Override public boolean evaluate() throws Exception {
 472         Map<byte[], UserMetrics> metrics =
 473             ADMIN.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().values()
 474                 .iterator().next().getUserMetrics();
 475         Assert.assertNotNull(metrics);
 476         //including current user + noOfUsers
 477         return metrics.keySet().size() > noOfUsers;
 478       }
 479     });
 480   }
 481
 482   private void doPut() throws IOException {
 483     Table table = createConnection(UTIL.getConfiguration()).getTable(TABLE_NAME);
 484     table.put(new Put(Bytes.toBytes("a")).addColumn(CF, Bytes.toBytes("col1"), Bytes.toBytes("1")));
 485
 486   }
 487
 488   private void doGet() throws IOException {
 489     Table table = createConnection(UTIL.getConfiguration()).getTable(TABLE_NAME);
 490     table.get(new Get(Bytes.toBytes("a")).addColumn(CF, Bytes.toBytes("col1")));
 491
 492   }
 493
 494   private Connection createConnection(Configuration conf) throws IOException {
 495     User user = UserProvider.instantiate(conf).getCurrent();
 496     return ClusterConnectionFactory.createAsyncClusterConnection(conf, null, user).toConnection();
 497   }
 498
 499   @Test
 500   public void testOtherStatusInfos() throws Exception {
 501     EnumSet<Option> options =
 502         EnumSet.of(Option.MASTER_COPROCESSORS, Option.HBASE_VERSION,
 503                    Option.CLUSTER_ID, Option.BALANCER_ON);
 504     ClusterMetrics metrics = ADMIN.getClusterMetrics(options);
 505     Assert.assertEquals(1, metrics.getMasterCoprocessorNames().size());
 506     Assert.assertNotNull(metrics.getHBaseVersion());
 507     Assert.assertNotNull(metrics.getClusterId());
 508     Assert.assertTrue(metrics.getAverageLoad() == 0.0);
 509     Assert.assertNotNull(metrics.getBalancerOn());
 510   }
 511
 512   @AfterClass
 513   public static void tearDownAfterClass() throws Exception {
 514     if (ADMIN != null) {
 515       ADMIN.close();
 516     }
 517     UTIL.shutdownMiniCluster();
 518   }
 519
 520   @Test
 521   public void testObserver() throws IOException {
 522     int preCount = MyObserver.PRE_COUNT.get();
 523     int postCount = MyObserver.POST_COUNT.get();
 524     Assert.assertTrue(ADMIN.getClusterMetrics().getMasterCoprocessorNames().stream()
 525         .anyMatch(s -> s.equals(MyObserver.class.getSimpleName())));
 526     Assert.assertEquals(preCount + 1, MyObserver.PRE_COUNT.get());
 527     Assert.assertEquals(postCount + 1, MyObserver.POST_COUNT.get());
 528   }
 529
 530   private static void insertData(final TableName tableName, int startRow, int rowCount)
 531       throws IOException {
 532     Table t = UTIL.getConnection().getTable(tableName);
 533     Put p;
 534     for (int i = 0; i < rowCount; i++) {
 535       p = new Put(Bytes.toBytes("" + (startRow + i)));
 536       p.addColumn(CF, Bytes.toBytes("val1"), Bytes.toBytes(i));
 537       t.put(p);
 538     }
 539   }
 540
 541   public static class MyObserver implements MasterCoprocessor, MasterObserver {
 542     private static final AtomicInteger PRE_COUNT = new AtomicInteger(0);
 543     private static final AtomicInteger POST_COUNT = new AtomicInteger(0);
 544
 545     @Override public Optional<MasterObserver> getMasterObserver() {
 546       return Optional.of(this);
 547     }
 548
 549     @Override public void preGetClusterMetrics(ObserverContext<MasterCoprocessorEnvironment> ctx)
 550         throws IOException {
 551       PRE_COUNT.incrementAndGet();
 552     }
 553
 554     @Override public void postGetClusterMetrics(ObserverContext<MasterCoprocessorEnvironment> ctx,
 555         ClusterMetrics metrics) throws IOException {
 556       POST_COUNT.incrementAndGet();
 557     }
 558   }
 559 }