hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java

   1 /*
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.regionserver;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertFalse;
  22 import static org.junit.Assert.assertTrue;
  23 import static org.junit.Assert.fail;
  24
  25 import java.io.IOException;
  26 import java.util.ArrayList;
  27 import java.util.Arrays;
  28 import java.util.List;
  29 import java.util.Objects;
  30 import java.util.concurrent.atomic.AtomicBoolean;
  31 import java.util.stream.Collectors;
  32 import org.apache.commons.lang3.RandomUtils;
  33 import org.apache.hadoop.conf.Configuration;
  34 import org.apache.hadoop.fs.FileSystem;
  35 import org.apache.hadoop.fs.Path;
  36 import org.apache.hadoop.hbase.CatalogFamilyFormat;
  37 import org.apache.hadoop.hbase.HBaseClassTestRule;
  38 import org.apache.hadoop.hbase.HBaseTestingUtil;
  39 import org.apache.hadoop.hbase.MetaTableAccessor;
  40 import org.apache.hadoop.hbase.ServerName;
  41 import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
  42 import org.apache.hadoop.hbase.StartTestingClusterOption;
  43 import org.apache.hadoop.hbase.TableName;
  44 import org.apache.hadoop.hbase.UnknownRegionException;
  45 import org.apache.hadoop.hbase.client.Admin;
  46 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
  47 import org.apache.hadoop.hbase.client.DoNotRetryRegionException;
  48 import org.apache.hadoop.hbase.client.Put;
  49 import org.apache.hadoop.hbase.client.RegionInfo;
  50 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
  51 import org.apache.hadoop.hbase.client.Result;
  52 import org.apache.hadoop.hbase.client.ResultScanner;
  53 import org.apache.hadoop.hbase.client.Scan;
  54 import org.apache.hadoop.hbase.client.Table;
  55 import org.apache.hadoop.hbase.client.TableDescriptor;
  56 import org.apache.hadoop.hbase.exceptions.MergeRegionException;
  57 import org.apache.hadoop.hbase.master.HMaster;
  58 import org.apache.hadoop.hbase.master.MasterRpcServices;
  59 import org.apache.hadoop.hbase.master.RegionState;
  60 import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
  61 import org.apache.hadoop.hbase.master.assignment.RegionStates;
  62 import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
  63 import org.apache.hadoop.hbase.testclassification.LargeTests;
  64 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
  65 import org.apache.hadoop.hbase.util.Bytes;
  66 import org.apache.hadoop.hbase.util.CommonFSUtils;
  67 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
  68 import org.apache.hadoop.hbase.util.FutureUtils;
  69 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
  70 import org.apache.hadoop.hbase.util.Pair;
  71 import org.apache.hadoop.hbase.util.PairOfSameType;
  72 import org.apache.hadoop.hbase.util.Threads;
  73 import org.apache.hadoop.util.StringUtils;
  74 import org.apache.zookeeper.KeeperException;
  75 import org.junit.AfterClass;
  76 import org.junit.BeforeClass;
  77 import org.junit.ClassRule;
  78 import org.junit.Rule;
  79 import org.junit.Test;
  80 import org.junit.experimental.categories.Category;
  81 import org.junit.rules.TestName;
  82 import org.slf4j.Logger;
  83 import org.slf4j.LoggerFactory;
  84
  85 import org.apache.hbase.thirdparty.com.google.common.base.Joiner;
  86 import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
  87 import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
  88
  89 import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
  90 import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest;
  91 import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse;
  92
  93 @Category({RegionServerTests.class, LargeTests.class})
  94 public class TestRegionMergeTransactionOnCluster {
  95
  96   @ClassRule
  97   public static final HBaseClassTestRule CLASS_RULE =
  98       HBaseClassTestRule.forClass(TestRegionMergeTransactionOnCluster.class);
  99
 100   private static final Logger LOG =
 101       LoggerFactory.getLogger(TestRegionMergeTransactionOnCluster.class);
 102
 103   @Rule public TestName name = new TestName();
 104
 105   private static final int NB_SERVERS = 3;
 106
 107   private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
 108   private static final byte[] QUALIFIER = Bytes.toBytes("q");
 109
 110   private static byte[] ROW = Bytes.toBytes("testRow");
 111   private static final int INITIAL_REGION_NUM = 10;
 112   private static final int ROWSIZE = 200;
 113   private static byte[][] ROWS = makeN(ROW, ROWSIZE);
 114
 115   private static int waitTime = 60 * 1000;
 116
 117   static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
 118
 119   private static HMaster MASTER;
 120   private static Admin ADMIN;
 121
 122   @BeforeClass
 123   public static void beforeAllTests() throws Exception {
 124     // Start a cluster
 125     StartTestingClusterOption option = StartTestingClusterOption.builder()
 126         .masterClass(MyMaster.class).numRegionServers(NB_SERVERS).numDataNodes(NB_SERVERS).build();
 127     TEST_UTIL.startMiniCluster(option);
 128     SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
 129     MASTER = cluster.getMaster();
 130     MASTER.balanceSwitch(false);
 131     ADMIN = TEST_UTIL.getConnection().getAdmin();
 132   }
 133
 134   @AfterClass
 135   public static void afterAllTests() throws Exception {
 136     TEST_UTIL.shutdownMiniCluster();
 137     if (ADMIN != null) {
 138       ADMIN.close();
 139     }
 140   }
 141
 142   @Test
 143   public void testWholesomeMerge() throws Exception {
 144     LOG.info("Starting " + name.getMethodName());
 145     final TableName tableName = TableName.valueOf(name.getMethodName());
 146
 147     try {
 148       // Create table and load data.
 149       Table table = createTableAndLoadData(MASTER, tableName);
 150       // Merge 1st and 2nd region
 151       mergeRegionsAndVerifyRegionNum(MASTER, tableName, 0, 1, INITIAL_REGION_NUM - 1);
 152
 153       // Merge 2nd and 3th region
 154       PairOfSameType<RegionInfo> mergedRegions =
 155         mergeRegionsAndVerifyRegionNum(MASTER, tableName, 1, 2, INITIAL_REGION_NUM - 2);
 156
 157       verifyRowCount(table, ROWSIZE);
 158
 159       // Randomly choose one of the two merged regions
 160       RegionInfo hri = RandomUtils.nextBoolean() ? mergedRegions.getFirst() : mergedRegions.getSecond();
 161       SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
 162       AssignmentManager am = cluster.getMaster().getAssignmentManager();
 163       RegionStates regionStates = am.getRegionStates();
 164
 165       // We should not be able to assign it again
 166       am.assign(hri);
 167       assertFalse("Merged region can't be assigned", regionStates.isRegionInTransition(hri));
 168
 169       // We should not be able to unassign it either
 170       am.unassign(hri);
 171       assertFalse("Merged region can't be unassigned", regionStates.isRegionInTransition(hri));
 172
 173       table.close();
 174     } finally {
 175       TEST_UTIL.deleteTable(tableName);
 176     }
 177   }
 178
 179   /**
 180    * Not really restarting the master. Simulate it by clear of new region
 181    * state since it is not persisted, will be lost after master restarts.
 182    */
 183   @Test
 184   public void testMergeAndRestartingMaster() throws Exception {
 185     final TableName tableName = TableName.valueOf(name.getMethodName());
 186
 187     try {
 188       // Create table and load data.
 189       Table table = createTableAndLoadData(MASTER, tableName);
 190
 191       try {
 192         MyMasterRpcServices.enabled.set(true);
 193
 194         // Merge 1st and 2nd region
 195         mergeRegionsAndVerifyRegionNum(MASTER, tableName, 0, 1, INITIAL_REGION_NUM - 1);
 196       } finally {
 197         MyMasterRpcServices.enabled.set(false);
 198       }
 199
 200       table.close();
 201     } finally {
 202       TEST_UTIL.deleteTable(tableName);
 203     }
 204   }
 205
 206   @Test
 207   public void testCleanMergeReference() throws Exception {
 208     LOG.info("Starting " + name.getMethodName());
 209     ADMIN.catalogJanitorSwitch(false);
 210     final TableName tableName = TableName.valueOf(name.getMethodName());
 211     try {
 212       // Create table and load data.
 213       Table table = createTableAndLoadData(MASTER, tableName);
 214       // Merge 1st and 2nd region
 215       mergeRegionsAndVerifyRegionNum(MASTER, tableName, 0, 1, INITIAL_REGION_NUM - 1);
 216       verifyRowCount(table, ROWSIZE);
 217       table.close();
 218
 219       List<Pair<RegionInfo, ServerName>> tableRegions = MetaTableAccessor
 220           .getTableRegionsAndLocations(MASTER.getConnection(), tableName);
 221       RegionInfo mergedRegionInfo = tableRegions.get(0).getFirst();
 222       TableDescriptor tableDescriptor = MASTER.getTableDescriptors().get(
 223           tableName);
 224       Result mergedRegionResult = MetaTableAccessor.getRegionResult(
 225         MASTER.getConnection(), mergedRegionInfo.getRegionName());
 226
 227       // contains merge reference in META
 228       assertTrue(CatalogFamilyFormat.hasMergeRegions(mergedRegionResult.rawCells()));
 229
 230       // merging regions' directory are in the file system all the same
 231       List<RegionInfo> p = CatalogFamilyFormat.getMergeRegions(mergedRegionResult.rawCells());
 232       RegionInfo regionA = p.get(0);
 233       RegionInfo regionB = p.get(1);
 234       FileSystem fs = MASTER.getMasterFileSystem().getFileSystem();
 235       Path rootDir = MASTER.getMasterFileSystem().getRootDir();
 236
 237       Path tabledir = CommonFSUtils.getTableDir(rootDir, mergedRegionInfo.getTable());
 238       Path regionAdir = new Path(tabledir, regionA.getEncodedName());
 239       Path regionBdir = new Path(tabledir, regionB.getEncodedName());
 240       assertTrue(fs.exists(regionAdir));
 241       assertTrue(fs.exists(regionBdir));
 242
 243       ColumnFamilyDescriptor[] columnFamilies = tableDescriptor.getColumnFamilies();
 244       HRegionFileSystem hrfs = new HRegionFileSystem(
 245         TEST_UTIL.getConfiguration(), fs, tabledir, mergedRegionInfo);
 246       int count = 0;
 247       for(ColumnFamilyDescriptor colFamily : columnFamilies) {
 248         count += hrfs.getStoreFiles(colFamily.getNameAsString()).size();
 249       }
 250       ADMIN.compactRegion(mergedRegionInfo.getRegionName());
 251       // clean up the merged region store files
 252       // wait until merged region have reference file
 253       long timeout = EnvironmentEdgeManager.currentTime() + waitTime;
 254       int newcount = 0;
 255       while (EnvironmentEdgeManager.currentTime() < timeout) {
 256         for(ColumnFamilyDescriptor colFamily : columnFamilies) {
 257           newcount += hrfs.getStoreFiles(colFamily.getNameAsString()).size();
 258         }
 259         if(newcount > count) {
 260           break;
 261         }
 262         Thread.sleep(50);
 263       }
 264       assertTrue(newcount > count);
 265       List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
 266           .getRegionServerThreads();
 267       for (RegionServerThread rs : regionServerThreads) {
 268         CompactedHFilesDischarger cleaner = new CompactedHFilesDischarger(100, null,
 269             rs.getRegionServer(), false);
 270         cleaner.chore();
 271         Thread.sleep(1000);
 272       }
 273       while (EnvironmentEdgeManager.currentTime() < timeout) {
 274         int newcount1 = 0;
 275         for(ColumnFamilyDescriptor colFamily : columnFamilies) {
 276           newcount1 += hrfs.getStoreFiles(colFamily.getNameAsString()).size();
 277         }
 278         if(newcount1 <= 1) {
 279           break;
 280         }
 281         Thread.sleep(50);
 282       }
 283       // run CatalogJanitor to clean merge references in hbase:meta and archive the
 284       // files of merging regions
 285       int cleaned = 0;
 286       while (cleaned == 0) {
 287         cleaned = ADMIN.runCatalogJanitor();
 288         LOG.debug("catalog janitor returned " + cleaned);
 289         Thread.sleep(50);
 290         // Cleanup is async so wait till all procedures are done running.
 291         ProcedureTestingUtility.waitNoProcedureRunning(
 292             TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor());
 293       }
 294       // We used to check for existence of region in fs but sometimes the region dir was
 295       // cleaned up by the time we got here making the test sometimes flakey.
 296       assertTrue(cleaned > 0);
 297
 298       // Wait around a bit to give stuff a chance to complete.
 299       while (true) {
 300         mergedRegionResult = MetaTableAccessor
 301           .getRegionResult(TEST_UTIL.getConnection(), mergedRegionInfo.getRegionName());
 302         if (CatalogFamilyFormat.hasMergeRegions(mergedRegionResult.rawCells())) {
 303           LOG.info("Waiting on cleanup of merge columns {}",
 304             Arrays.asList(mergedRegionResult.rawCells()).stream().
 305               map(c -> c.toString()).collect(Collectors.joining(",")));
 306           Threads.sleep(50);
 307         } else {
 308           break;
 309         }
 310       }
 311       assertFalse(CatalogFamilyFormat.hasMergeRegions(mergedRegionResult.rawCells()));
 312     } finally {
 313       ADMIN.catalogJanitorSwitch(true);
 314       TEST_UTIL.deleteTable(tableName);
 315     }
 316   }
 317
 318   /**
 319    * This test tests 1, merging region not online;
 320    * 2, merging same two regions; 3, merging unknown regions.
 321    * They are in one test case so that we don't have to create
 322    * many tables, and these tests are simple.
 323    */
 324   @Test
 325   public void testMerge() throws Exception {
 326     LOG.info("Starting " + name.getMethodName());
 327     final TableName tableName = TableName.valueOf(name.getMethodName());
 328     final Admin admin = TEST_UTIL.getAdmin();
 329
 330     try {
 331       // Create table and load data.
 332       Table table = createTableAndLoadData(MASTER, tableName);
 333       AssignmentManager am = MASTER.getAssignmentManager();
 334       List<RegionInfo> regions = am.getRegionStates().getRegionsOfTable(tableName);
 335       // Fake offline one region
 336       RegionInfo a = regions.get(0);
 337       RegionInfo b = regions.get(1);
 338       am.unassign(b);
 339       am.offlineRegion(b);
 340       try {
 341         // Merge offline region. Region a is offline here
 342         FutureUtils.get(
 343           admin.mergeRegionsAsync(a.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), false));
 344         fail("Offline regions should not be able to merge");
 345       } catch (DoNotRetryRegionException ie) {
 346         System.out.println(ie);
 347         assertTrue(ie instanceof MergeRegionException);
 348       }
 349
 350       try {
 351         // Merge the same region: b and b.
 352         FutureUtils
 353           .get(admin.mergeRegionsAsync(b.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), true));
 354         fail("A region should not be able to merge with itself, even forcfully");
 355       } catch (IOException ie) {
 356         assertTrue("Exception should mention regions not online",
 357           StringUtils.stringifyException(ie).contains("region to itself") &&
 358             ie instanceof MergeRegionException);
 359       }
 360
 361       try {
 362         // Merge unknown regions
 363         FutureUtils.get(admin.mergeRegionsAsync(Bytes.toBytes("-f1"), Bytes.toBytes("-f2"), true));
 364         fail("Unknown region could not be merged");
 365       } catch (IOException ie) {
 366         assertTrue("UnknownRegionException should be thrown", ie instanceof UnknownRegionException);
 367       }
 368       table.close();
 369     } finally {
 370       TEST_UTIL.deleteTable(tableName);
 371     }
 372   }
 373
 374   @Test
 375   public void testMergeWithReplicas() throws Exception {
 376     final TableName tableName = TableName.valueOf(name.getMethodName());
 377     try {
 378       // Create table and load data.
 379       Table table = createTableAndLoadData(MASTER, tableName, 5, 2);
 380       List<Pair<RegionInfo, ServerName>> initialRegionToServers =
 381         MetaTableAccessor.getTableRegionsAndLocations(TEST_UTIL.getConnection(), tableName);
 382       // Merge 1st and 2nd region
 383       PairOfSameType<RegionInfo> mergedRegions =
 384         mergeRegionsAndVerifyRegionNum(MASTER, tableName, 0, 2, 5 * 2 - 2);
 385       List<Pair<RegionInfo, ServerName>> currentRegionToServers =
 386         MetaTableAccessor.getTableRegionsAndLocations(TEST_UTIL.getConnection(), tableName);
 387       List<RegionInfo> initialRegions = new ArrayList<>();
 388       for (Pair<RegionInfo, ServerName> p : initialRegionToServers) {
 389         initialRegions.add(p.getFirst());
 390       }
 391       List<RegionInfo> currentRegions = new ArrayList<>();
 392       for (Pair<RegionInfo, ServerName> p : currentRegionToServers) {
 393         currentRegions.add(p.getFirst());
 394       }
 395       assertTrue(initialRegions.contains(mergedRegions.getFirst())); //this is the first region
 396       assertTrue(initialRegions.contains(RegionReplicaUtil
 397         .getRegionInfoForReplica(mergedRegions.getFirst(), 1))); //this is the replica of the first region
 398       assertTrue(initialRegions.contains(mergedRegions.getSecond())); //this is the second region
 399       assertTrue(initialRegions.contains(RegionReplicaUtil
 400         .getRegionInfoForReplica(mergedRegions.getSecond(), 1))); //this is the replica of the second region
 401       assertTrue(!initialRegions.contains(currentRegions.get(0))); //this is the new region
 402       assertTrue(!initialRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(currentRegions.get(0), 1))); //replica of the new region
 403       assertTrue(currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(currentRegions.get(0), 1))); //replica of the new region
 404       assertTrue(!currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(mergedRegions.getFirst(), 1))); //replica of the merged region
 405       assertTrue(!currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(mergedRegions.getSecond(), 1))); //replica of the merged region
 406       table.close();
 407     } finally {
 408       TEST_UTIL.deleteTable(tableName);
 409     }
 410   }
 411
 412   private PairOfSameType<RegionInfo> mergeRegionsAndVerifyRegionNum(
 413       HMaster master, TableName tablename,
 414       int regionAnum, int regionBnum, int expectedRegionNum) throws Exception {
 415     PairOfSameType<RegionInfo> mergedRegions =
 416       requestMergeRegion(master, tablename, regionAnum, regionBnum);
 417     waitAndVerifyRegionNum(master, tablename, expectedRegionNum);
 418     return mergedRegions;
 419   }
 420
 421   private PairOfSameType<RegionInfo> requestMergeRegion(
 422       HMaster master, TableName tablename,
 423       int regionAnum, int regionBnum) throws Exception {
 424     List<Pair<RegionInfo, ServerName>> tableRegions = MetaTableAccessor
 425         .getTableRegionsAndLocations(
 426             TEST_UTIL.getConnection(), tablename);
 427     RegionInfo regionA = tableRegions.get(regionAnum).getFirst();
 428     RegionInfo regionB = tableRegions.get(regionBnum).getFirst();
 429     ADMIN.mergeRegionsAsync(
 430       regionA.getEncodedNameAsBytes(),
 431       regionB.getEncodedNameAsBytes(), false);
 432     return new PairOfSameType<>(regionA, regionB);
 433   }
 434
 435   private void waitAndVerifyRegionNum(HMaster master, TableName tablename,
 436       int expectedRegionNum) throws Exception {
 437     List<Pair<RegionInfo, ServerName>> tableRegionsInMeta;
 438     List<RegionInfo> tableRegionsInMaster;
 439     long timeout = EnvironmentEdgeManager.currentTime() + waitTime;
 440     while (EnvironmentEdgeManager.currentTime() < timeout) {
 441       tableRegionsInMeta =
 442           MetaTableAccessor.getTableRegionsAndLocations(TEST_UTIL.getConnection(), tablename);
 443       tableRegionsInMaster =
 444           master.getAssignmentManager().getRegionStates().getRegionsOfTable(tablename);
 445       LOG.info(Objects.toString(tableRegionsInMaster));
 446       LOG.info(Objects.toString(tableRegionsInMeta));
 447       int tableRegionsInMetaSize = tableRegionsInMeta.size();
 448       int tableRegionsInMasterSize = tableRegionsInMaster.size();
 449       if (tableRegionsInMetaSize == expectedRegionNum
 450           && tableRegionsInMasterSize == expectedRegionNum) {
 451         break;
 452       }
 453       Thread.sleep(250);
 454     }
 455
 456     tableRegionsInMeta = MetaTableAccessor.getTableRegionsAndLocations(
 457         TEST_UTIL.getConnection(), tablename);
 458     LOG.info("Regions after merge:" + Joiner.on(',').join(tableRegionsInMeta));
 459     assertEquals(expectedRegionNum, tableRegionsInMeta.size());
 460   }
 461
 462   private Table createTableAndLoadData(HMaster master, TableName tablename)
 463       throws Exception {
 464     return createTableAndLoadData(master, tablename, INITIAL_REGION_NUM, 1);
 465   }
 466
 467   private Table createTableAndLoadData(HMaster master, TableName tablename,
 468       int numRegions, int replication) throws Exception {
 469     assertTrue("ROWSIZE must > numregions:" + numRegions, ROWSIZE > numRegions);
 470     byte[][] splitRows = new byte[numRegions - 1][];
 471     for (int i = 0; i < splitRows.length; i++) {
 472       splitRows[i] = ROWS[(i + 1) * ROWSIZE / numRegions];
 473     }
 474
 475     Table table = TEST_UTIL.createTable(tablename, FAMILYNAME, splitRows);
 476     LOG.info("Created " + table.getName());
 477     if (replication > 1) {
 478       HBaseTestingUtil.setReplicas(ADMIN, tablename, replication);
 479       LOG.info("Set replication of " + replication + " on " + table.getName());
 480     }
 481     loadData(table);
 482     LOG.info("Loaded " + table.getName());
 483     verifyRowCount(table, ROWSIZE);
 484     LOG.info("Verified " + table.getName());
 485
 486     List<Pair<RegionInfo, ServerName>> tableRegions;
 487     TEST_UTIL.waitUntilAllRegionsAssigned(tablename);
 488     LOG.info("All regions assigned for table - " + table.getName());
 489     tableRegions = MetaTableAccessor.getTableRegionsAndLocations(
 490         TEST_UTIL.getConnection(), tablename);
 491     assertEquals("Wrong number of regions in table " + tablename,
 492         numRegions * replication, tableRegions.size());
 493     LOG.info(tableRegions.size() + "Regions after load: " + Joiner.on(',').join(tableRegions));
 494     assertEquals(numRegions * replication, tableRegions.size());
 495     return table;
 496   }
 497
 498   private static byte[][] makeN(byte[] base, int n) {
 499     byte[][] ret = new byte[n][];
 500     for (int i = 0; i < n; i++) {
 501       ret[i] = Bytes.add(base, Bytes.toBytes(String.format("%04d", i)));
 502     }
 503     return ret;
 504   }
 505
 506   private void loadData(Table table) throws IOException {
 507     for (int i = 0; i < ROWSIZE; i++) {
 508       Put put = new Put(ROWS[i]);
 509       put.addColumn(FAMILYNAME, QUALIFIER, Bytes.toBytes(i));
 510       table.put(put);
 511     }
 512   }
 513
 514   private void verifyRowCount(Table table, int expectedRegionNum)
 515       throws IOException {
 516     ResultScanner scanner = table.getScanner(new Scan());
 517     int rowCount = 0;
 518     while (scanner.next() != null) {
 519       rowCount++;
 520     }
 521     assertEquals(expectedRegionNum, rowCount);
 522     scanner.close();
 523   }
 524
 525   // Make it public so that JVMClusterUtil can access it.
 526   public static class MyMaster extends HMaster {
 527     public MyMaster(Configuration conf) throws IOException, KeeperException, InterruptedException {
 528       super(conf);
 529     }
 530
 531     @Override
 532     protected MasterRpcServices createRpcServices() throws IOException {
 533       return new MyMasterRpcServices(this);
 534     }
 535   }
 536
 537   static class MyMasterRpcServices extends MasterRpcServices {
 538     static AtomicBoolean enabled = new AtomicBoolean(false);
 539
 540     private HMaster myMaster;
 541     public MyMasterRpcServices(HMaster master) throws IOException {
 542       super(master);
 543       myMaster = master;
 544     }
 545
 546     @Override
 547     public ReportRegionStateTransitionResponse reportRegionStateTransition(RpcController c,
 548         ReportRegionStateTransitionRequest req) throws ServiceException {
 549       ReportRegionStateTransitionResponse resp = super.reportRegionStateTransition(c, req);
 550       if (enabled.get() && req.getTransition(0).getTransitionCode()
 551           == TransitionCode.READY_TO_MERGE && !resp.hasErrorMessage()) {
 552         RegionStates regionStates = myMaster.getAssignmentManager().getRegionStates();
 553         for (RegionState regionState: regionStates.getRegionsStateInTransition()) {
 554           // Find the merging_new region and remove it
 555           if (regionState.isMergingNew()) {
 556             regionStates.deleteRegion(regionState.getRegion());
 557           }
 558         }
 559       }
 560       return resp;
 561     }
 562   }
 563 }