hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java

   1 /*
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.master;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertNotNull;
  22 import static org.junit.Assert.assertNull;
  23 import static org.junit.Assert.assertTrue;
  24 import static org.junit.Assert.fail;
  25
  26 import java.io.IOException;
  27 import java.net.InetSocketAddress;
  28 import java.util.ArrayList;
  29 import java.util.Arrays;
  30 import java.util.Collection;
  31 import java.util.HashMap;
  32 import java.util.List;
  33 import java.util.Map;
  34 import java.util.Random;
  35 import java.util.concurrent.atomic.AtomicInteger;
  36 import org.apache.hadoop.conf.Configuration;
  37 import org.apache.hadoop.hbase.HBaseClassTestRule;
  38 import org.apache.hadoop.hbase.HBaseTestingUtility;
  39 import org.apache.hadoop.hbase.HConstants;
  40 import org.apache.hadoop.hbase.HRegionLocation;
  41 import org.apache.hadoop.hbase.MetaTableAccessor;
  42 import org.apache.hadoop.hbase.MiniHBaseCluster;
  43 import org.apache.hadoop.hbase.NamespaceDescriptor;
  44 import org.apache.hadoop.hbase.ServerName;
  45 import org.apache.hadoop.hbase.TableName;
  46 import org.apache.hadoop.hbase.client.Admin;
  47 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
  48 import org.apache.hadoop.hbase.client.Connection;
  49 import org.apache.hadoop.hbase.client.RegionInfo;
  50 import org.apache.hadoop.hbase.client.RegionLocator;
  51 import org.apache.hadoop.hbase.client.Result;
  52 import org.apache.hadoop.hbase.client.TableDescriptor;
  53 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
  54 import org.apache.hadoop.hbase.favored.FavoredNodeAssignmentHelper;
  55 import org.apache.hadoop.hbase.favored.FavoredNodeLoadBalancer;
  56 import org.apache.hadoop.hbase.favored.FavoredNodesPlan;
  57 import org.apache.hadoop.hbase.favored.FavoredNodesPlan.Position;
  58 import org.apache.hadoop.hbase.regionserver.HRegion;
  59 import org.apache.hadoop.hbase.regionserver.HRegionServer;
  60 import org.apache.hadoop.hbase.regionserver.Region;
  61 import org.apache.hadoop.hbase.testclassification.MasterTests;
  62 import org.apache.hadoop.hbase.testclassification.MediumTests;
  63 import org.apache.hadoop.hbase.util.Bytes;
  64 import org.apache.hadoop.hbase.util.Pair;
  65 import org.apache.zookeeper.KeeperException;
  66 import org.junit.AfterClass;
  67 import org.junit.BeforeClass;
  68 import org.junit.ClassRule;
  69 import org.junit.Ignore;
  70 import org.junit.Test;
  71 import org.junit.experimental.categories.Category;
  72 import org.slf4j.Logger;
  73 import org.slf4j.LoggerFactory;
  74
  75 @Category({MasterTests.class, MediumTests.class})
  76 public class TestRegionPlacement {
  77
  78   @ClassRule
  79   public static final HBaseClassTestRule CLASS_RULE =
  80       HBaseClassTestRule.forClass(TestRegionPlacement.class);
  81
  82   private static final Logger LOG = LoggerFactory.getLogger(TestRegionPlacement.class);
  83   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
  84   private final static int SLAVES = 10;
  85   private static Connection CONNECTION;
  86   private static Admin admin;
  87   private static RegionPlacementMaintainer rp;
  88   private static Position[] positions = Position.values();
  89   private int lastRegionOnPrimaryRSCount = 0;
  90   private int REGION_NUM = 10;
  91   private Map<RegionInfo, ServerName[]> favoredNodesAssignmentPlan = new HashMap<>();
  92
  93   @BeforeClass
  94   public static void setupBeforeClass() throws Exception {
  95     Configuration conf = TEST_UTIL.getConfiguration();
  96     // Enable the favored nodes based load balancer
  97     conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
  98         FavoredNodeLoadBalancer.class, LoadBalancer.class);
  99     conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
 100     TEST_UTIL.startMiniCluster(SLAVES);
 101     CONNECTION = TEST_UTIL.getConnection();
 102     admin = CONNECTION.getAdmin();
 103     rp = new RegionPlacementMaintainer(conf);
 104   }
 105
 106   @AfterClass
 107   public static void tearDownAfterClass() throws Exception {
 108     TEST_UTIL.shutdownMiniCluster();
 109   }
 110
 111   @Ignore ("Test for unfinished feature") @Test
 112   public void testRegionPlacement() throws Exception {
 113     String tableStr = "testRegionAssignment";
 114     TableName table = TableName.valueOf(tableStr);
 115     // Create a table with REGION_NUM regions.
 116     createTable(table, REGION_NUM);
 117
 118     TEST_UTIL.waitTableAvailable(table);
 119
 120     // Verify all the user regions are assigned to the primary region server
 121     // based on the plan
 122     verifyRegionOnPrimaryRS(REGION_NUM);
 123
 124     FavoredNodesPlan currentPlan = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
 125     // Verify all the region server are update with the latest favored nodes
 126     verifyRegionServerUpdated(currentPlan);
 127     // Test Case 2: To verify whether the region placement tools can
 128     // correctly update the new assignment plan to hbase:meta and Region Server.
 129     // The new assignment plan is generated by shuffle the existing assignment
 130     // plan by switching PRIMARY, SECONDARY and TERTIARY nodes.
 131     // Shuffle the plan by switching the secondary region server with
 132     // the tertiary.
 133
 134     // Shuffle the secondary with tertiary favored nodes
 135     FavoredNodesPlan shuffledPlan = this.shuffleAssignmentPlan(currentPlan,
 136         FavoredNodesPlan.Position.SECONDARY, FavoredNodesPlan.Position.TERTIARY);
 137     // Let the region placement update the hbase:meta and Region Servers
 138     rp.updateAssignmentPlan(shuffledPlan);
 139
 140     // Verify the region assignment. There are supposed to no region reassignment
 141     // All the regions are still on the primary region server
 142     verifyRegionAssignment(shuffledPlan,0, REGION_NUM);
 143
 144     // Shuffle the plan by switching the primary with secondary and
 145     // verify the region reassignment is consistent with the plan.
 146     shuffledPlan = this.shuffleAssignmentPlan(currentPlan,
 147         FavoredNodesPlan.Position.PRIMARY, FavoredNodesPlan.Position.SECONDARY);
 148
 149     // Let the region placement update the hbase:meta and Region Servers
 150     rp.updateAssignmentPlan(shuffledPlan);
 151
 152     verifyRegionAssignment(shuffledPlan, REGION_NUM, REGION_NUM);
 153
 154     // also verify that the AssignmentVerificationReport has the correct information
 155     RegionPlacementMaintainer rp = new RegionPlacementMaintainer(TEST_UTIL.getConfiguration());
 156     // we are interested in only one table (and hence one report)
 157     rp.setTargetTableName(new String[]{tableStr});
 158     List<AssignmentVerificationReport> reports = rp.verifyRegionPlacement(false);
 159     AssignmentVerificationReport report = reports.get(0);
 160     assertTrue(report.getRegionsWithoutValidFavoredNodes().isEmpty());
 161     assertTrue(report.getNonFavoredAssignedRegions().isEmpty());
 162     assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
 163     assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) != 0);
 164     assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) == 0);
 165     assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) == 0);
 166     assertTrue(report.getUnassignedRegions().isEmpty());
 167
 168     // Check when a RS stops, the regions get assigned to their secondary/tertiary
 169     killRandomServerAndVerifyAssignment();
 170
 171     // also verify that the AssignmentVerificationReport has the correct information
 172     reports = rp.verifyRegionPlacement(false);
 173     report = reports.get(0);
 174     assertTrue(report.getRegionsWithoutValidFavoredNodes().isEmpty());
 175     assertTrue(report.getNonFavoredAssignedRegions().isEmpty());
 176     assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
 177     assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) > 0);
 178     assertTrue("secondary " +
 179     report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) + " tertiary "
 180         + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY),
 181         (report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) > 0
 182         || report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) > 0));
 183     assertTrue((report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) +
 184         report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) +
 185         report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY)) == REGION_NUM);
 186     RegionPlacementMaintainer.printAssignmentPlan(currentPlan);
 187   }
 188
 189   private void killRandomServerAndVerifyAssignment()
 190       throws IOException, InterruptedException, KeeperException {
 191     ServerName serverToKill = null;
 192     int killIndex = 0;
 193     Random random = new Random(System.currentTimeMillis());
 194     ServerName metaServer = TEST_UTIL.getHBaseCluster().getServerHoldingMeta();
 195     LOG.debug("Server holding meta " + metaServer);
 196     boolean isNamespaceServer = false;
 197     do {
 198       // kill a random non-meta server carrying at least one region
 199       killIndex = random.nextInt(SLAVES);
 200       serverToKill = TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getServerName();
 201       Collection<HRegion> regs =
 202           TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getOnlineRegionsLocalContext();
 203       isNamespaceServer = false;
 204       for (HRegion r : regs) {
 205         if (r.getRegionInfo().getTable().getNamespaceAsString()
 206             .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
 207           isNamespaceServer = true;
 208           break;
 209         }
 210       }
 211     } while (ServerName.isSameAddress(metaServer, serverToKill) || isNamespaceServer ||
 212         TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getNumberOfOnlineRegions() == 0);
 213     LOG.debug("Stopping RS " + serverToKill);
 214     Map<RegionInfo, Pair<ServerName, ServerName>> regionsToVerify = new HashMap<>();
 215     // mark the regions to track
 216     for (Map.Entry<RegionInfo, ServerName[]> entry : favoredNodesAssignmentPlan.entrySet()) {
 217       ServerName s = entry.getValue()[0];
 218       if (ServerName.isSameAddress(s, serverToKill)) {
 219         regionsToVerify.put(entry.getKey(), new Pair<>(entry.getValue()[1], entry.getValue()[2]));
 220         LOG.debug("Adding " + entry.getKey() + " with sedcondary/tertiary " +
 221             entry.getValue()[1] + " " + entry.getValue()[2]);
 222       }
 223     }
 224     int orig = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getNumRegionsOpened();
 225     TEST_UTIL.getHBaseCluster().stopRegionServer(serverToKill);
 226     TEST_UTIL.getHBaseCluster().waitForRegionServerToStop(serverToKill, 60000);
 227     int curr = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getNumRegionsOpened();
 228     while (curr - orig < regionsToVerify.size()) {
 229       LOG.debug("Waiting for " + regionsToVerify.size() + " to come online " +
 230           " Current #regions " + curr + " Original #regions " + orig);
 231       Thread.sleep(200);
 232       curr = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getNumRegionsOpened();
 233     }
 234     // now verify
 235     for (Map.Entry<RegionInfo, Pair<ServerName, ServerName>> entry : regionsToVerify.entrySet()) {
 236       ServerName newDestination = TEST_UTIL.getHBaseCluster().getMaster()
 237           .getAssignmentManager().getRegionStates().getRegionServerOfRegion(entry.getKey());
 238       Pair<ServerName, ServerName> secondaryTertiaryServers = entry.getValue();
 239       LOG.debug("New destination for region " + entry.getKey().getEncodedName() +
 240           " " + newDestination +". Secondary/Tertiary are " + secondaryTertiaryServers.getFirst()
 241           + "/" + secondaryTertiaryServers.getSecond());
 242       if (!(ServerName.isSameAddress(newDestination, secondaryTertiaryServers.getFirst())||
 243           ServerName.isSameAddress(newDestination, secondaryTertiaryServers.getSecond()))){
 244         fail("Region " + entry.getKey() + " not present on any of the expected servers");
 245       }
 246     }
 247     // start(reinstate) region server since we killed one before
 248     TEST_UTIL.getHBaseCluster().startRegionServer();
 249   }
 250
 251   /**
 252    * Used to test the correctness of this class.
 253    */
 254   @Ignore ("Test for unfinished feature") @Test
 255   public void testRandomizedMatrix() {
 256     int rows = 100;
 257     int cols = 100;
 258     float[][] matrix = new float[rows][cols];
 259     Random random = new Random();
 260     for (int i = 0; i < rows; i++) {
 261       for (int j = 0; j < cols; j++) {
 262         matrix[i][j] = random.nextFloat();
 263       }
 264     }
 265
 266     // Test that inverting a transformed matrix gives the original matrix.
 267     RegionPlacementMaintainer.RandomizedMatrix rm =
 268       new RegionPlacementMaintainer.RandomizedMatrix(rows, cols);
 269     float[][] transformed = rm.transform(matrix);
 270     float[][] invertedTransformed = rm.invert(transformed);
 271     for (int i = 0; i < rows; i++) {
 272       for (int j = 0; j < cols; j++) {
 273         if (matrix[i][j] != invertedTransformed[i][j]) {
 274           throw new RuntimeException();
 275         }
 276       }
 277     }
 278
 279     // Test that the indices on a transformed matrix can be inverted to give
 280     // the same values on the original matrix.
 281     int[] transformedIndices = new int[rows];
 282     for (int i = 0; i < rows; i++) {
 283       transformedIndices[i] = random.nextInt(cols);
 284     }
 285     int[] invertedTransformedIndices = rm.invertIndices(transformedIndices);
 286     float[] transformedValues = new float[rows];
 287     float[] invertedTransformedValues = new float[rows];
 288     for (int i = 0; i < rows; i++) {
 289       transformedValues[i] = transformed[i][transformedIndices[i]];
 290       invertedTransformedValues[i] = matrix[i][invertedTransformedIndices[i]];
 291     }
 292     Arrays.sort(transformedValues);
 293     Arrays.sort(invertedTransformedValues);
 294     if (!Arrays.equals(transformedValues, invertedTransformedValues)) {
 295       throw new RuntimeException();
 296     }
 297   }
 298
 299   /**
 300    * Shuffle the assignment plan by switching two favored node positions.
 301    * @param plan The assignment plan
 302    * @param p1 The first switch position
 303    * @param p2 The second switch position
 304    * @return the shuffled assignment plan
 305    */
 306   private FavoredNodesPlan shuffleAssignmentPlan(FavoredNodesPlan plan,
 307       FavoredNodesPlan.Position p1, FavoredNodesPlan.Position p2) throws IOException {
 308     FavoredNodesPlan shuffledPlan = new FavoredNodesPlan();
 309
 310     Map<String, RegionInfo> regionToHRegion =
 311         rp.getRegionAssignmentSnapshot().getRegionNameToRegionInfoMap();
 312     for (Map.Entry<String, List<ServerName>> entry :
 313       plan.getAssignmentMap().entrySet()) {
 314
 315       // copy the server list from the original plan
 316       List<ServerName> shuffledServerList = new ArrayList<>();
 317       shuffledServerList.addAll(entry.getValue());
 318
 319       // start to shuffle
 320       shuffledServerList.set(p1.ordinal(), entry.getValue().get(p2.ordinal()));
 321       shuffledServerList.set(p2.ordinal(), entry.getValue().get(p1.ordinal()));
 322
 323       // update the plan
 324       shuffledPlan.updateFavoredNodesMap(regionToHRegion.get(entry.getKey()), shuffledServerList);
 325     }
 326     return shuffledPlan;
 327   }
 328
 329   /**
 330    * To verify the region assignment status.
 331    * It will check the assignment plan consistency between hbase:meta and
 332    * region servers.
 333    * Also it will verify weather the number of region movement and
 334    * the number regions on the primary region server are expected
 335    *
 336    * @param plan
 337    * @param regionMovementNum
 338    * @param numRegionsOnPrimaryRS
 339    * @throws InterruptedException
 340    * @throws IOException
 341    */
 342   private void verifyRegionAssignment(FavoredNodesPlan plan,
 343       int regionMovementNum, int numRegionsOnPrimaryRS)
 344   throws InterruptedException, IOException {
 345     // Verify the assignment plan in hbase:meta is consistent with the expected plan.
 346     verifyMETAUpdated(plan);
 347
 348     // Verify the number of region movement is expected
 349     verifyRegionMovementNum(regionMovementNum);
 350
 351     // Verify the number of regions is assigned to the primary region server
 352     // based on the plan is expected
 353     verifyRegionOnPrimaryRS(numRegionsOnPrimaryRS);
 354
 355     // Verify all the online region server are updated with the assignment plan
 356     verifyRegionServerUpdated(plan);
 357   }
 358
 359   /**
 360    * Verify the meta has updated to the latest assignment plan
 361    * @param expectedPlan the region assignment plan
 362    * @throws IOException if an IO problem is encountered
 363    */
 364   private void verifyMETAUpdated(FavoredNodesPlan expectedPlan)
 365   throws IOException {
 366     FavoredNodesPlan planFromMETA = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
 367     assertTrue("The assignment plan is NOT consistent with the expected plan ",
 368         planFromMETA.equals(expectedPlan));
 369   }
 370
 371   /**
 372    * Verify the number of region movement is expected
 373    */
 374   private void verifyRegionMovementNum(int expected)
 375       throws InterruptedException, IOException {
 376     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
 377     HMaster m = cluster.getMaster();
 378     int lastRegionOpenedCount = m.getAssignmentManager().getNumRegionsOpened();
 379     // get the assignments start to execute
 380     m.balance();
 381
 382     int retry = 10;
 383     long sleep = 3000;
 384     int attempt = 0;
 385     int currentRegionOpened, regionMovement;
 386     do {
 387       currentRegionOpened = m.getAssignmentManager().getNumRegionsOpened();
 388       regionMovement= currentRegionOpened - lastRegionOpenedCount;
 389       LOG.debug("There are " + regionMovement + "/" + expected +
 390           " regions moved after " + attempt + " attempts");
 391       Thread.sleep((++attempt) * sleep);
 392     } while (regionMovement != expected && attempt <= retry);
 393
 394     // update the lastRegionOpenedCount
 395     lastRegionOpenedCount = currentRegionOpened;
 396
 397     assertEquals("There are only " + regionMovement + " instead of "
 398           + expected + " region movement for " + attempt + " attempts", expected, regionMovement);
 399   }
 400
 401   /**
 402    * Verify the number of user regions is assigned to the primary
 403    * region server based on the plan is expected
 404    * @param expectedNum the expected number of assigned regions
 405    * @throws IOException
 406    */
 407   private void verifyRegionOnPrimaryRS(int expectedNum)
 408       throws IOException {
 409     lastRegionOnPrimaryRSCount = getNumRegionisOnPrimaryRS();
 410     assertEquals("Only " +  expectedNum + " of user regions running " +
 411         "on the primary region server", expectedNum ,
 412         lastRegionOnPrimaryRSCount);
 413   }
 414
 415   /**
 416    * Verify all the online region servers has been updated to the
 417    * latest assignment plan
 418    * @param plan
 419    * @throws IOException
 420    */
 421   private void verifyRegionServerUpdated(FavoredNodesPlan plan) throws IOException {
 422     // Verify all region servers contain the correct favored nodes information
 423     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
 424     for (int i = 0; i < SLAVES; i++) {
 425       HRegionServer rs = cluster.getRegionServer(i);
 426       for (Region region: rs.getRegions(TableName.valueOf("testRegionAssignment"))) {
 427         InetSocketAddress[] favoredSocketAddress = rs.getFavoredNodesForRegion(
 428             region.getRegionInfo().getEncodedName());
 429         String regionName = region.getRegionInfo().getRegionNameAsString();
 430         List<ServerName> favoredServerList = plan.getAssignmentMap().get(regionName);
 431
 432         // All regions are supposed to have favored nodes,
 433         // except for hbase:meta and ROOT
 434         if (favoredServerList == null) {
 435           TableDescriptor desc = region.getTableDescriptor();
 436           // Verify they are ROOT and hbase:meta regions since no favored nodes
 437           assertNull(favoredSocketAddress);
 438           assertTrue("User region " +
 439               region.getTableDescriptor().getTableName() +
 440               " should have favored nodes", desc.isMetaRegion());
 441         } else {
 442           // For user region, the favored nodes in the region server should be
 443           // identical to favored nodes in the assignmentPlan
 444           assertTrue(favoredSocketAddress.length == favoredServerList.size());
 445           assertTrue(favoredServerList.size() > 0);
 446           for (int j = 0; j < favoredServerList.size(); j++) {
 447             InetSocketAddress addrFromRS = favoredSocketAddress[j];
 448             InetSocketAddress addrFromPlan = InetSocketAddress.createUnresolved(
 449                 favoredServerList.get(j).getHostname(), favoredServerList.get(j).getPort());
 450
 451             assertNotNull(addrFromRS);
 452             assertNotNull(addrFromPlan);
 453             assertTrue("Region server " + rs.getServerName().getHostAndPort()
 454                 + " has the " + positions[j] +
 455                 " for region " + region.getRegionInfo().getRegionNameAsString() + " is " +
 456                 addrFromRS + " which is inconsistent with the plan "
 457                 + addrFromPlan, addrFromRS.equals(addrFromPlan));
 458           }
 459         }
 460       }
 461     }
 462   }
 463
 464   /**
 465    * Check whether regions are assigned to servers consistent with the explicit
 466    * hints that are persisted in the hbase:meta table.
 467    * Also keep track of the number of the regions are assigned to the
 468    * primary region server.
 469    * @return the number of regions are assigned to the primary region server
 470    * @throws IOException
 471    */
 472   private int getNumRegionisOnPrimaryRS() throws IOException {
 473     final AtomicInteger regionOnPrimaryNum = new AtomicInteger(0);
 474     final AtomicInteger totalRegionNum = new AtomicInteger(0);
 475     LOG.info("The start of region placement verification");
 476     MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() {
 477       @Override
 478       public boolean visit(Result result) throws IOException {
 479         try {
 480           @SuppressWarnings("deprecation")
 481           RegionInfo info = MetaTableAccessor.getRegionInfo(result);
 482           if(info.getTable().getNamespaceAsString()
 483               .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
 484             return true;
 485           }
 486           byte[] server = result.getValue(HConstants.CATALOG_FAMILY,
 487               HConstants.SERVER_QUALIFIER);
 488           byte[] favoredNodes = result.getValue(HConstants.CATALOG_FAMILY,
 489               FavoredNodeAssignmentHelper.FAVOREDNODES_QUALIFIER);
 490           // Add the favored nodes into assignment plan
 491           ServerName[] favoredServerList =
 492               FavoredNodeAssignmentHelper.getFavoredNodesList(favoredNodes);
 493           favoredNodesAssignmentPlan.put(info, favoredServerList);
 494
 495           Position[] positions = Position.values();
 496           if (info != null) {
 497             totalRegionNum.incrementAndGet();
 498             if (server != null) {
 499               ServerName serverName =
 500                   ServerName.valueOf(Bytes.toString(server), -1);
 501               if (favoredNodes != null) {
 502                 String placement = "[NOT FAVORED NODE]";
 503                 for (int i = 0; i < favoredServerList.length; i++) {
 504                   if (favoredServerList[i].equals(serverName)) {
 505                     placement = positions[i].toString();
 506                     if (i == Position.PRIMARY.ordinal()) {
 507                       regionOnPrimaryNum.incrementAndGet();
 508                     }
 509                     break;
 510                   }
 511                 }
 512                 LOG.info(info.getRegionNameAsString() + " on " +
 513                     serverName + " " + placement);
 514               } else {
 515                 LOG.info(info.getRegionNameAsString() + " running on " +
 516                     serverName + " but there is no favored region server");
 517               }
 518             } else {
 519               LOG.info(info.getRegionNameAsString() +
 520                   " not assigned to any server");
 521             }
 522           }
 523           return true;
 524         } catch (RuntimeException e) {
 525           LOG.error("Result=" + result);
 526           throw e;
 527         }
 528       }
 529     };
 530     MetaTableAccessor.fullScanRegions(CONNECTION, visitor);
 531     LOG.info("There are " + regionOnPrimaryNum.intValue() + " out of " +
 532         totalRegionNum.intValue() + " regions running on the primary" +
 533         " region servers" );
 534     return regionOnPrimaryNum.intValue() ;
 535   }
 536
 537   /**
 538    * Create a table with specified table name and region number.
 539    * @param tableName the name of the table to be created
 540    * @param regionNum number of regions to create
 541    * @throws IOException
 542    */
 543   private static void createTable(TableName tableName, int regionNum)
 544       throws IOException {
 545     int expectedRegions = regionNum;
 546     byte[][] splitKeys = new byte[expectedRegions - 1][];
 547     for (int i = 1; i < expectedRegions; i++) {
 548       byte splitKey = (byte) i;
 549       splitKeys[i - 1] = new byte[] { splitKey, splitKey, splitKey };
 550     }
 551
 552     TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
 553       new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
 554
 555     tableDescriptor.setColumnFamily(
 556       new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(
 557         HConstants.CATALOG_FAMILY));
 558     admin.createTable(tableDescriptor, splitKeys);
 559
 560     try (RegionLocator r = CONNECTION.getRegionLocator(tableName)) {
 561       List<HRegionLocation> regions = r.getAllRegionLocations();
 562       assertEquals("Tried to create " + expectedRegions + " regions "
 563           + "but only found " + regions.size(), expectedRegions, regions.size());
 564     }
 565   }
 566 }