hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java

   1 /*
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.master;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertNotNull;
  22 import static org.junit.Assert.assertNull;
  23 import static org.junit.Assert.assertTrue;
  24 import static org.junit.Assert.fail;
  25
  26 import java.io.IOException;
  27 import java.net.InetSocketAddress;
  28 import java.util.ArrayList;
  29 import java.util.Arrays;
  30 import java.util.Collection;
  31 import java.util.HashMap;
  32 import java.util.List;
  33 import java.util.Map;
  34 import java.util.Random;
  35 import java.util.concurrent.atomic.AtomicInteger;
  36 import org.apache.hadoop.conf.Configuration;
  37 import org.apache.hadoop.hbase.CatalogFamilyFormat;
  38 import org.apache.hadoop.hbase.ClientMetaTableAccessor;
  39 import org.apache.hadoop.hbase.HBaseClassTestRule;
  40 import org.apache.hadoop.hbase.HBaseTestingUtility;
  41 import org.apache.hadoop.hbase.HConstants;
  42 import org.apache.hadoop.hbase.HRegionLocation;
  43 import org.apache.hadoop.hbase.MetaTableAccessor;
  44 import org.apache.hadoop.hbase.MiniHBaseCluster;
  45 import org.apache.hadoop.hbase.NamespaceDescriptor;
  46 import org.apache.hadoop.hbase.ServerName;
  47 import org.apache.hadoop.hbase.TableName;
  48 import org.apache.hadoop.hbase.client.Admin;
  49 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
  50 import org.apache.hadoop.hbase.client.Connection;
  51 import org.apache.hadoop.hbase.client.RegionInfo;
  52 import org.apache.hadoop.hbase.client.RegionLocator;
  53 import org.apache.hadoop.hbase.client.Result;
  54 import org.apache.hadoop.hbase.client.TableDescriptor;
  55 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
  56 import org.apache.hadoop.hbase.favored.FavoredNodeAssignmentHelper;
  57 import org.apache.hadoop.hbase.favored.FavoredNodeLoadBalancer;
  58 import org.apache.hadoop.hbase.favored.FavoredNodesPlan;
  59 import org.apache.hadoop.hbase.favored.FavoredNodesPlan.Position;
  60 import org.apache.hadoop.hbase.regionserver.HRegion;
  61 import org.apache.hadoop.hbase.regionserver.HRegionServer;
  62 import org.apache.hadoop.hbase.regionserver.Region;
  63 import org.apache.hadoop.hbase.testclassification.MasterTests;
  64 import org.apache.hadoop.hbase.testclassification.MediumTests;
  65 import org.apache.hadoop.hbase.util.Bytes;
  66 import org.apache.hadoop.hbase.util.Pair;
  67 import org.apache.zookeeper.KeeperException;
  68 import org.junit.AfterClass;
  69 import org.junit.BeforeClass;
  70 import org.junit.ClassRule;
  71 import org.junit.Ignore;
  72 import org.junit.Test;
  73 import org.junit.experimental.categories.Category;
  74 import org.slf4j.Logger;
  75 import org.slf4j.LoggerFactory;
  76
  77 @Category({MasterTests.class, MediumTests.class})
  78 public class TestRegionPlacement {
  79
  80   @ClassRule
  81   public static final HBaseClassTestRule CLASS_RULE =
  82       HBaseClassTestRule.forClass(TestRegionPlacement.class);
  83
  84   private static final Logger LOG = LoggerFactory.getLogger(TestRegionPlacement.class);
  85   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
  86   private final static int SLAVES = 10;
  87   private static Connection CONNECTION;
  88   private static Admin admin;
  89   private static RegionPlacementMaintainer rp;
  90   private static Position[] positions = Position.values();
  91   private int lastRegionOnPrimaryRSCount = 0;
  92   private int REGION_NUM = 10;
  93   private Map<RegionInfo, ServerName[]> favoredNodesAssignmentPlan = new HashMap<>();
  94
  95   @BeforeClass
  96   public static void setupBeforeClass() throws Exception {
  97     Configuration conf = TEST_UTIL.getConfiguration();
  98     // Enable the favored nodes based load balancer
  99     conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
 100         FavoredNodeLoadBalancer.class, LoadBalancer.class);
 101     conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
 102     TEST_UTIL.startMiniCluster(SLAVES);
 103     CONNECTION = TEST_UTIL.getConnection();
 104     admin = CONNECTION.getAdmin();
 105     rp = new RegionPlacementMaintainer(conf);
 106   }
 107
 108   @AfterClass
 109   public static void tearDownAfterClass() throws Exception {
 110     TEST_UTIL.shutdownMiniCluster();
 111   }
 112
 113   @Ignore ("Test for unfinished feature") @Test
 114   public void testRegionPlacement() throws Exception {
 115     String tableStr = "testRegionAssignment";
 116     TableName table = TableName.valueOf(tableStr);
 117     // Create a table with REGION_NUM regions.
 118     createTable(table, REGION_NUM);
 119
 120     TEST_UTIL.waitTableAvailable(table);
 121
 122     // Verify all the user regions are assigned to the primary region server
 123     // based on the plan
 124     verifyRegionOnPrimaryRS(REGION_NUM);
 125
 126     FavoredNodesPlan currentPlan = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
 127     // Verify all the region server are update with the latest favored nodes
 128     verifyRegionServerUpdated(currentPlan);
 129     // Test Case 2: To verify whether the region placement tools can
 130     // correctly update the new assignment plan to hbase:meta and Region Server.
 131     // The new assignment plan is generated by shuffle the existing assignment
 132     // plan by switching PRIMARY, SECONDARY and TERTIARY nodes.
 133     // Shuffle the plan by switching the secondary region server with
 134     // the tertiary.
 135
 136     // Shuffle the secondary with tertiary favored nodes
 137     FavoredNodesPlan shuffledPlan = this.shuffleAssignmentPlan(currentPlan,
 138         FavoredNodesPlan.Position.SECONDARY, FavoredNodesPlan.Position.TERTIARY);
 139     // Let the region placement update the hbase:meta and Region Servers
 140     rp.updateAssignmentPlan(shuffledPlan);
 141
 142     // Verify the region assignment. There are supposed to no region reassignment
 143     // All the regions are still on the primary region server
 144     verifyRegionAssignment(shuffledPlan,0, REGION_NUM);
 145
 146     // Shuffle the plan by switching the primary with secondary and
 147     // verify the region reassignment is consistent with the plan.
 148     shuffledPlan = this.shuffleAssignmentPlan(currentPlan,
 149         FavoredNodesPlan.Position.PRIMARY, FavoredNodesPlan.Position.SECONDARY);
 150
 151     // Let the region placement update the hbase:meta and Region Servers
 152     rp.updateAssignmentPlan(shuffledPlan);
 153
 154     verifyRegionAssignment(shuffledPlan, REGION_NUM, REGION_NUM);
 155
 156     // also verify that the AssignmentVerificationReport has the correct information
 157     RegionPlacementMaintainer rp = new RegionPlacementMaintainer(TEST_UTIL.getConfiguration());
 158     // we are interested in only one table (and hence one report)
 159     rp.setTargetTableName(new String[]{tableStr});
 160     List<AssignmentVerificationReport> reports = rp.verifyRegionPlacement(false);
 161     AssignmentVerificationReport report = reports.get(0);
 162     assertTrue(report.getRegionsWithoutValidFavoredNodes().isEmpty());
 163     assertTrue(report.getNonFavoredAssignedRegions().isEmpty());
 164     assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
 165     assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) != 0);
 166     assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) == 0);
 167     assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) == 0);
 168     assertTrue(report.getUnassignedRegions().isEmpty());
 169
 170     // Check when a RS stops, the regions get assigned to their secondary/tertiary
 171     killRandomServerAndVerifyAssignment();
 172
 173     // also verify that the AssignmentVerificationReport has the correct information
 174     reports = rp.verifyRegionPlacement(false);
 175     report = reports.get(0);
 176     assertTrue(report.getRegionsWithoutValidFavoredNodes().isEmpty());
 177     assertTrue(report.getNonFavoredAssignedRegions().isEmpty());
 178     assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
 179     assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) > 0);
 180     assertTrue("secondary " +
 181     report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) + " tertiary "
 182         + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY),
 183         (report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) > 0
 184         || report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) > 0));
 185     assertTrue((report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) +
 186         report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) +
 187         report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY)) == REGION_NUM);
 188     RegionPlacementMaintainer.printAssignmentPlan(currentPlan);
 189   }
 190
 191   private void killRandomServerAndVerifyAssignment()
 192       throws IOException, InterruptedException, KeeperException {
 193     ServerName serverToKill = null;
 194     int killIndex = 0;
 195     Random random = new Random(System.currentTimeMillis());
 196     ServerName metaServer = TEST_UTIL.getHBaseCluster().getServerHoldingMeta();
 197     LOG.debug("Server holding meta " + metaServer);
 198     boolean isNamespaceServer = false;
 199     do {
 200       // kill a random non-meta server carrying at least one region
 201       killIndex = random.nextInt(SLAVES);
 202       serverToKill = TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getServerName();
 203       Collection<HRegion> regs =
 204           TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getOnlineRegionsLocalContext();
 205       isNamespaceServer = false;
 206       for (HRegion r : regs) {
 207         if (r.getRegionInfo().getTable().getNamespaceAsString()
 208             .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
 209           isNamespaceServer = true;
 210           break;
 211         }
 212       }
 213     } while (ServerName.isSameAddress(metaServer, serverToKill) || isNamespaceServer ||
 214         TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getNumberOfOnlineRegions() == 0);
 215     LOG.debug("Stopping RS " + serverToKill);
 216     Map<RegionInfo, Pair<ServerName, ServerName>> regionsToVerify = new HashMap<>();
 217     // mark the regions to track
 218     for (Map.Entry<RegionInfo, ServerName[]> entry : favoredNodesAssignmentPlan.entrySet()) {
 219       ServerName s = entry.getValue()[0];
 220       if (ServerName.isSameAddress(s, serverToKill)) {
 221         regionsToVerify.put(entry.getKey(), new Pair<>(entry.getValue()[1], entry.getValue()[2]));
 222         LOG.debug("Adding " + entry.getKey() + " with sedcondary/tertiary " +
 223             entry.getValue()[1] + " " + entry.getValue()[2]);
 224       }
 225     }
 226     int orig = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getNumRegionsOpened();
 227     TEST_UTIL.getHBaseCluster().stopRegionServer(serverToKill);
 228     TEST_UTIL.getHBaseCluster().waitForRegionServerToStop(serverToKill, 60000);
 229     int curr = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getNumRegionsOpened();
 230     while (curr - orig < regionsToVerify.size()) {
 231       LOG.debug("Waiting for " + regionsToVerify.size() + " to come online " +
 232           " Current #regions " + curr + " Original #regions " + orig);
 233       Thread.sleep(200);
 234       curr = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getNumRegionsOpened();
 235     }
 236     // now verify
 237     for (Map.Entry<RegionInfo, Pair<ServerName, ServerName>> entry : regionsToVerify.entrySet()) {
 238       ServerName newDestination = TEST_UTIL.getHBaseCluster().getMaster()
 239           .getAssignmentManager().getRegionStates().getRegionServerOfRegion(entry.getKey());
 240       Pair<ServerName, ServerName> secondaryTertiaryServers = entry.getValue();
 241       LOG.debug("New destination for region " + entry.getKey().getEncodedName() +
 242           " " + newDestination +". Secondary/Tertiary are " + secondaryTertiaryServers.getFirst()
 243           + "/" + secondaryTertiaryServers.getSecond());
 244       if (!(ServerName.isSameAddress(newDestination, secondaryTertiaryServers.getFirst())||
 245           ServerName.isSameAddress(newDestination, secondaryTertiaryServers.getSecond()))){
 246         fail("Region " + entry.getKey() + " not present on any of the expected servers");
 247       }
 248     }
 249     // start(reinstate) region server since we killed one before
 250     TEST_UTIL.getHBaseCluster().startRegionServer();
 251   }
 252
 253   /**
 254    * Used to test the correctness of this class.
 255    */
 256   @Ignore ("Test for unfinished feature") @Test
 257   public void testRandomizedMatrix() {
 258     int rows = 100;
 259     int cols = 100;
 260     float[][] matrix = new float[rows][cols];
 261     Random random = new Random();
 262     for (int i = 0; i < rows; i++) {
 263       for (int j = 0; j < cols; j++) {
 264         matrix[i][j] = random.nextFloat();
 265       }
 266     }
 267
 268     // Test that inverting a transformed matrix gives the original matrix.
 269     RegionPlacementMaintainer.RandomizedMatrix rm =
 270       new RegionPlacementMaintainer.RandomizedMatrix(rows, cols);
 271     float[][] transformed = rm.transform(matrix);
 272     float[][] invertedTransformed = rm.invert(transformed);
 273     for (int i = 0; i < rows; i++) {
 274       for (int j = 0; j < cols; j++) {
 275         if (matrix[i][j] != invertedTransformed[i][j]) {
 276           throw new RuntimeException();
 277         }
 278       }
 279     }
 280
 281     // Test that the indices on a transformed matrix can be inverted to give
 282     // the same values on the original matrix.
 283     int[] transformedIndices = new int[rows];
 284     for (int i = 0; i < rows; i++) {
 285       transformedIndices[i] = random.nextInt(cols);
 286     }
 287     int[] invertedTransformedIndices = rm.invertIndices(transformedIndices);
 288     float[] transformedValues = new float[rows];
 289     float[] invertedTransformedValues = new float[rows];
 290     for (int i = 0; i < rows; i++) {
 291       transformedValues[i] = transformed[i][transformedIndices[i]];
 292       invertedTransformedValues[i] = matrix[i][invertedTransformedIndices[i]];
 293     }
 294     Arrays.sort(transformedValues);
 295     Arrays.sort(invertedTransformedValues);
 296     if (!Arrays.equals(transformedValues, invertedTransformedValues)) {
 297       throw new RuntimeException();
 298     }
 299   }
 300
 301   /**
 302    * Shuffle the assignment plan by switching two favored node positions.
 303    * @param plan The assignment plan
 304    * @param p1 The first switch position
 305    * @param p2 The second switch position
 306    * @return the shuffled assignment plan
 307    */
 308   private FavoredNodesPlan shuffleAssignmentPlan(FavoredNodesPlan plan,
 309       FavoredNodesPlan.Position p1, FavoredNodesPlan.Position p2) throws IOException {
 310     FavoredNodesPlan shuffledPlan = new FavoredNodesPlan();
 311
 312     Map<String, RegionInfo> regionToHRegion =
 313         rp.getRegionAssignmentSnapshot().getRegionNameToRegionInfoMap();
 314     for (Map.Entry<String, List<ServerName>> entry :
 315       plan.getAssignmentMap().entrySet()) {
 316
 317       // copy the server list from the original plan
 318       List<ServerName> shuffledServerList = new ArrayList<>();
 319       shuffledServerList.addAll(entry.getValue());
 320
 321       // start to shuffle
 322       shuffledServerList.set(p1.ordinal(), entry.getValue().get(p2.ordinal()));
 323       shuffledServerList.set(p2.ordinal(), entry.getValue().get(p1.ordinal()));
 324
 325       // update the plan
 326       shuffledPlan.updateFavoredNodesMap(regionToHRegion.get(entry.getKey()), shuffledServerList);
 327     }
 328     return shuffledPlan;
 329   }
 330
 331   /**
 332    * To verify the region assignment status.
 333    * It will check the assignment plan consistency between hbase:meta and
 334    * region servers.
 335    * Also it will verify weather the number of region movement and
 336    * the number regions on the primary region server are expected
 337    *
 338    * @param plan
 339    * @param regionMovementNum
 340    * @param numRegionsOnPrimaryRS
 341    * @throws InterruptedException
 342    * @throws IOException
 343    */
 344   private void verifyRegionAssignment(FavoredNodesPlan plan,
 345       int regionMovementNum, int numRegionsOnPrimaryRS)
 346   throws InterruptedException, IOException {
 347     // Verify the assignment plan in hbase:meta is consistent with the expected plan.
 348     verifyMETAUpdated(plan);
 349
 350     // Verify the number of region movement is expected
 351     verifyRegionMovementNum(regionMovementNum);
 352
 353     // Verify the number of regions is assigned to the primary region server
 354     // based on the plan is expected
 355     verifyRegionOnPrimaryRS(numRegionsOnPrimaryRS);
 356
 357     // Verify all the online region server are updated with the assignment plan
 358     verifyRegionServerUpdated(plan);
 359   }
 360
 361   /**
 362    * Verify the meta has updated to the latest assignment plan
 363    * @param expectedPlan the region assignment plan
 364    * @throws IOException if an IO problem is encountered
 365    */
 366   private void verifyMETAUpdated(FavoredNodesPlan expectedPlan)
 367   throws IOException {
 368     FavoredNodesPlan planFromMETA = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
 369     assertTrue("The assignment plan is NOT consistent with the expected plan ",
 370         planFromMETA.equals(expectedPlan));
 371   }
 372
 373   /**
 374    * Verify the number of region movement is expected
 375    */
 376   private void verifyRegionMovementNum(int expected)
 377       throws InterruptedException, IOException {
 378     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
 379     HMaster m = cluster.getMaster();
 380     int lastRegionOpenedCount = m.getAssignmentManager().getNumRegionsOpened();
 381     // get the assignments start to execute
 382     m.balance();
 383
 384     int retry = 10;
 385     long sleep = 3000;
 386     int attempt = 0;
 387     int currentRegionOpened, regionMovement;
 388     do {
 389       currentRegionOpened = m.getAssignmentManager().getNumRegionsOpened();
 390       regionMovement= currentRegionOpened - lastRegionOpenedCount;
 391       LOG.debug("There are " + regionMovement + "/" + expected +
 392           " regions moved after " + attempt + " attempts");
 393       Thread.sleep((++attempt) * sleep);
 394     } while (regionMovement != expected && attempt <= retry);
 395
 396     // update the lastRegionOpenedCount
 397     lastRegionOpenedCount = currentRegionOpened;
 398
 399     assertEquals("There are only " + regionMovement + " instead of "
 400           + expected + " region movement for " + attempt + " attempts", expected, regionMovement);
 401   }
 402
 403   /**
 404    * Verify the number of user regions is assigned to the primary
 405    * region server based on the plan is expected
 406    * @param expectedNum the expected number of assigned regions
 407    * @throws IOException
 408    */
 409   private void verifyRegionOnPrimaryRS(int expectedNum)
 410       throws IOException {
 411     lastRegionOnPrimaryRSCount = getNumRegionisOnPrimaryRS();
 412     assertEquals("Only " +  expectedNum + " of user regions running " +
 413         "on the primary region server", expectedNum ,
 414         lastRegionOnPrimaryRSCount);
 415   }
 416
 417   /**
 418    * Verify all the online region servers has been updated to the
 419    * latest assignment plan
 420    * @param plan
 421    * @throws IOException
 422    */
 423   private void verifyRegionServerUpdated(FavoredNodesPlan plan) throws IOException {
 424     // Verify all region servers contain the correct favored nodes information
 425     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
 426     for (int i = 0; i < SLAVES; i++) {
 427       HRegionServer rs = cluster.getRegionServer(i);
 428       for (Region region: rs.getRegions(TableName.valueOf("testRegionAssignment"))) {
 429         InetSocketAddress[] favoredSocketAddress = rs.getFavoredNodesForRegion(
 430             region.getRegionInfo().getEncodedName());
 431         String regionName = region.getRegionInfo().getRegionNameAsString();
 432         List<ServerName> favoredServerList = plan.getAssignmentMap().get(regionName);
 433
 434         // All regions are supposed to have favored nodes,
 435         // except for hbase:meta and ROOT
 436         if (favoredServerList == null) {
 437           TableDescriptor desc = region.getTableDescriptor();
 438           // Verify they are ROOT and hbase:meta regions since no favored nodes
 439           assertNull(favoredSocketAddress);
 440           assertTrue("User region " +
 441               region.getTableDescriptor().getTableName() +
 442               " should have favored nodes", desc.isMetaRegion());
 443         } else {
 444           // For user region, the favored nodes in the region server should be
 445           // identical to favored nodes in the assignmentPlan
 446           assertTrue(favoredSocketAddress.length == favoredServerList.size());
 447           assertTrue(favoredServerList.size() > 0);
 448           for (int j = 0; j < favoredServerList.size(); j++) {
 449             InetSocketAddress addrFromRS = favoredSocketAddress[j];
 450             InetSocketAddress addrFromPlan = InetSocketAddress.createUnresolved(
 451                 favoredServerList.get(j).getHostname(), favoredServerList.get(j).getPort());
 452
 453             assertNotNull(addrFromRS);
 454             assertNotNull(addrFromPlan);
 455             assertTrue("Region server " + rs.getServerName().getAddress()
 456                 + " has the " + positions[j] +
 457                 " for region " + region.getRegionInfo().getRegionNameAsString() + " is " +
 458                 addrFromRS + " which is inconsistent with the plan "
 459                 + addrFromPlan, addrFromRS.equals(addrFromPlan));
 460           }
 461         }
 462       }
 463     }
 464   }
 465
 466   /**
 467    * Check whether regions are assigned to servers consistent with the explicit
 468    * hints that are persisted in the hbase:meta table.
 469    * Also keep track of the number of the regions are assigned to the
 470    * primary region server.
 471    * @return the number of regions are assigned to the primary region server
 472    * @throws IOException
 473    */
 474   private int getNumRegionisOnPrimaryRS() throws IOException {
 475     final AtomicInteger regionOnPrimaryNum = new AtomicInteger(0);
 476     final AtomicInteger totalRegionNum = new AtomicInteger(0);
 477     LOG.info("The start of region placement verification");
 478     ClientMetaTableAccessor.Visitor visitor = new ClientMetaTableAccessor.Visitor() {
 479       @Override
 480       public boolean visit(Result result) throws IOException {
 481         try {
 482           @SuppressWarnings("deprecation")
 483           RegionInfo info = CatalogFamilyFormat.getRegionInfo(result);
 484           if(info.getTable().getNamespaceAsString()
 485               .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
 486             return true;
 487           }
 488           byte[] server = result.getValue(HConstants.CATALOG_FAMILY,
 489               HConstants.SERVER_QUALIFIER);
 490           byte[] favoredNodes = result.getValue(HConstants.CATALOG_FAMILY,
 491               FavoredNodeAssignmentHelper.FAVOREDNODES_QUALIFIER);
 492           // Add the favored nodes into assignment plan
 493           ServerName[] favoredServerList =
 494               FavoredNodeAssignmentHelper.getFavoredNodesList(favoredNodes);
 495           favoredNodesAssignmentPlan.put(info, favoredServerList);
 496
 497           Position[] positions = Position.values();
 498           if (info != null) {
 499             totalRegionNum.incrementAndGet();
 500             if (server != null) {
 501               ServerName serverName =
 502                   ServerName.valueOf(Bytes.toString(server), -1);
 503               if (favoredNodes != null) {
 504                 String placement = "[NOT FAVORED NODE]";
 505                 for (int i = 0; i < favoredServerList.length; i++) {
 506                   if (favoredServerList[i].equals(serverName)) {
 507                     placement = positions[i].toString();
 508                     if (i == Position.PRIMARY.ordinal()) {
 509                       regionOnPrimaryNum.incrementAndGet();
 510                     }
 511                     break;
 512                   }
 513                 }
 514                 LOG.info(info.getRegionNameAsString() + " on " +
 515                     serverName + " " + placement);
 516               } else {
 517                 LOG.info(info.getRegionNameAsString() + " running on " +
 518                     serverName + " but there is no favored region server");
 519               }
 520             } else {
 521               LOG.info(info.getRegionNameAsString() +
 522                   " not assigned to any server");
 523             }
 524           }
 525           return true;
 526         } catch (RuntimeException e) {
 527           LOG.error("Result=" + result);
 528           throw e;
 529         }
 530       }
 531     };
 532     MetaTableAccessor.fullScanRegions(CONNECTION, visitor);
 533     LOG.info("There are " + regionOnPrimaryNum.intValue() + " out of " +
 534         totalRegionNum.intValue() + " regions running on the primary" +
 535         " region servers" );
 536     return regionOnPrimaryNum.intValue() ;
 537   }
 538
 539   /**
 540    * Create a table with specified table name and region number.
 541    * @param tableName the name of the table to be created
 542    * @param regionNum number of regions to create
 543    * @throws IOException
 544    */
 545   private static void createTable(TableName tableName, int regionNum)
 546       throws IOException {
 547     int expectedRegions = regionNum;
 548     byte[][] splitKeys = new byte[expectedRegions - 1][];
 549     for (int i = 1; i < expectedRegions; i++) {
 550       byte splitKey = (byte) i;
 551       splitKeys[i - 1] = new byte[] { splitKey, splitKey, splitKey };
 552     }
 553
 554     TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
 555       new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
 556
 557     tableDescriptor.setColumnFamily(
 558       new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(
 559         HConstants.CATALOG_FAMILY));
 560     admin.createTable(tableDescriptor, splitKeys);
 561
 562     try (RegionLocator r = CONNECTION.getRegionLocator(tableName)) {
 563       List<HRegionLocation> regions = r.getAllRegionLocations();
 564       assertEquals("Tried to create " + expectedRegions + " regions "
 565           + "but only found " + regions.size(), expectedRegions, regions.size());
 566     }
 567   }
 568 }