hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java

   1 /*
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.regionserver;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertFalse;
  22 import static org.junit.Assert.assertTrue;
  23 import static org.junit.Assert.fail;
  24
  25 import java.io.IOException;
  26 import java.util.ArrayList;
  27 import java.util.Arrays;
  28 import java.util.List;
  29 import java.util.Objects;
  30 import java.util.concurrent.ThreadLocalRandom;
  31 import java.util.concurrent.atomic.AtomicBoolean;
  32 import java.util.stream.Collectors;
  33 import org.apache.hadoop.conf.Configuration;
  34 import org.apache.hadoop.fs.FileSystem;
  35 import org.apache.hadoop.fs.Path;
  36 import org.apache.hadoop.hbase.CatalogFamilyFormat;
  37 import org.apache.hadoop.hbase.HBaseClassTestRule;
  38 import org.apache.hadoop.hbase.HBaseTestingUtil;
  39 import org.apache.hadoop.hbase.MetaTableAccessor;
  40 import org.apache.hadoop.hbase.ServerName;
  41 import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
  42 import org.apache.hadoop.hbase.StartTestingClusterOption;
  43 import org.apache.hadoop.hbase.TableName;
  44 import org.apache.hadoop.hbase.UnknownRegionException;
  45 import org.apache.hadoop.hbase.client.Admin;
  46 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
  47 import org.apache.hadoop.hbase.client.DoNotRetryRegionException;
  48 import org.apache.hadoop.hbase.client.Put;
  49 import org.apache.hadoop.hbase.client.RegionInfo;
  50 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
  51 import org.apache.hadoop.hbase.client.Result;
  52 import org.apache.hadoop.hbase.client.ResultScanner;
  53 import org.apache.hadoop.hbase.client.Scan;
  54 import org.apache.hadoop.hbase.client.Table;
  55 import org.apache.hadoop.hbase.client.TableDescriptor;
  56 import org.apache.hadoop.hbase.exceptions.MergeRegionException;
  57 import org.apache.hadoop.hbase.master.HMaster;
  58 import org.apache.hadoop.hbase.master.MasterRpcServices;
  59 import org.apache.hadoop.hbase.master.RegionState;
  60 import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
  61 import org.apache.hadoop.hbase.master.assignment.RegionStates;
  62 import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
  63 import org.apache.hadoop.hbase.testclassification.LargeTests;
  64 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
  65 import org.apache.hadoop.hbase.util.Bytes;
  66 import org.apache.hadoop.hbase.util.CommonFSUtils;
  67 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
  68 import org.apache.hadoop.hbase.util.FutureUtils;
  69 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
  70 import org.apache.hadoop.hbase.util.Pair;
  71 import org.apache.hadoop.hbase.util.PairOfSameType;
  72 import org.apache.hadoop.hbase.util.Threads;
  73 import org.apache.hadoop.util.StringUtils;
  74 import org.apache.zookeeper.KeeperException;
  75 import org.junit.AfterClass;
  76 import org.junit.BeforeClass;
  77 import org.junit.ClassRule;
  78 import org.junit.Rule;
  79 import org.junit.Test;
  80 import org.junit.experimental.categories.Category;
  81 import org.junit.rules.TestName;
  82 import org.slf4j.Logger;
  83 import org.slf4j.LoggerFactory;
  84 import org.apache.hbase.thirdparty.com.google.common.base.Joiner;
  85 import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
  86 import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
  87 import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
  88 import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest;
  89 import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse;
  90
  91 @Category({RegionServerTests.class, LargeTests.class})
  92 public class TestRegionMergeTransactionOnCluster {
  93
  94   @ClassRule
  95   public static final HBaseClassTestRule CLASS_RULE =
  96       HBaseClassTestRule.forClass(TestRegionMergeTransactionOnCluster.class);
  97
  98   private static final Logger LOG =
  99       LoggerFactory.getLogger(TestRegionMergeTransactionOnCluster.class);
 100
 101   @Rule public TestName name = new TestName();
 102
 103   private static final int NB_SERVERS = 3;
 104
 105   private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
 106   private static final byte[] QUALIFIER = Bytes.toBytes("q");
 107
 108   private static byte[] ROW = Bytes.toBytes("testRow");
 109   private static final int INITIAL_REGION_NUM = 10;
 110   private static final int ROWSIZE = 200;
 111   private static byte[][] ROWS = makeN(ROW, ROWSIZE);
 112
 113   private static int waitTime = 60 * 1000;
 114
 115   static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
 116
 117   private static HMaster MASTER;
 118   private static Admin ADMIN;
 119
 120   @BeforeClass
 121   public static void beforeAllTests() throws Exception {
 122     // Start a cluster
 123     StartTestingClusterOption option = StartTestingClusterOption.builder()
 124         .masterClass(MyMaster.class).numRegionServers(NB_SERVERS).numDataNodes(NB_SERVERS).build();
 125     TEST_UTIL.startMiniCluster(option);
 126     SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
 127     MASTER = cluster.getMaster();
 128     MASTER.balanceSwitch(false);
 129     ADMIN = TEST_UTIL.getConnection().getAdmin();
 130   }
 131
 132   @AfterClass
 133   public static void afterAllTests() throws Exception {
 134     TEST_UTIL.shutdownMiniCluster();
 135     if (ADMIN != null) {
 136       ADMIN.close();
 137     }
 138   }
 139
 140   @Test
 141   public void testWholesomeMerge() throws Exception {
 142     LOG.info("Starting " + name.getMethodName());
 143     final TableName tableName = TableName.valueOf(name.getMethodName());
 144
 145     try {
 146       // Create table and load data.
 147       Table table = createTableAndLoadData(MASTER, tableName);
 148       // Merge 1st and 2nd region
 149       mergeRegionsAndVerifyRegionNum(MASTER, tableName, 0, 1, INITIAL_REGION_NUM - 1);
 150
 151       // Merge 2nd and 3th region
 152       PairOfSameType<RegionInfo> mergedRegions =
 153         mergeRegionsAndVerifyRegionNum(MASTER, tableName, 1, 2, INITIAL_REGION_NUM - 2);
 154
 155       verifyRowCount(table, ROWSIZE);
 156
 157       // Randomly choose one of the two merged regions
 158       RegionInfo hri = ThreadLocalRandom.current().nextBoolean() ? mergedRegions.getFirst() :
 159         mergedRegions.getSecond();
 160       SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
 161       AssignmentManager am = cluster.getMaster().getAssignmentManager();
 162       RegionStates regionStates = am.getRegionStates();
 163
 164       // We should not be able to assign it again
 165       am.assign(hri);
 166       assertFalse("Merged region can't be assigned", regionStates.isRegionInTransition(hri));
 167
 168       // We should not be able to unassign it either
 169       am.unassign(hri);
 170       assertFalse("Merged region can't be unassigned", regionStates.isRegionInTransition(hri));
 171
 172       table.close();
 173     } finally {
 174       TEST_UTIL.deleteTable(tableName);
 175     }
 176   }
 177
 178   /**
 179    * Not really restarting the master. Simulate it by clear of new region
 180    * state since it is not persisted, will be lost after master restarts.
 181    */
 182   @Test
 183   public void testMergeAndRestartingMaster() throws Exception {
 184     final TableName tableName = TableName.valueOf(name.getMethodName());
 185
 186     try {
 187       // Create table and load data.
 188       Table table = createTableAndLoadData(MASTER, tableName);
 189
 190       try {
 191         MyMasterRpcServices.enabled.set(true);
 192
 193         // Merge 1st and 2nd region
 194         mergeRegionsAndVerifyRegionNum(MASTER, tableName, 0, 1, INITIAL_REGION_NUM - 1);
 195       } finally {
 196         MyMasterRpcServices.enabled.set(false);
 197       }
 198
 199       table.close();
 200     } finally {
 201       TEST_UTIL.deleteTable(tableName);
 202     }
 203   }
 204
 205   @Test
 206   public void testCleanMergeReference() throws Exception {
 207     LOG.info("Starting " + name.getMethodName());
 208     ADMIN.catalogJanitorSwitch(false);
 209     final TableName tableName = TableName.valueOf(name.getMethodName());
 210     try {
 211       // Create table and load data.
 212       Table table = createTableAndLoadData(MASTER, tableName);
 213       // Merge 1st and 2nd region
 214       mergeRegionsAndVerifyRegionNum(MASTER, tableName, 0, 1, INITIAL_REGION_NUM - 1);
 215       verifyRowCount(table, ROWSIZE);
 216       table.close();
 217
 218       List<Pair<RegionInfo, ServerName>> tableRegions = MetaTableAccessor
 219           .getTableRegionsAndLocations(MASTER.getConnection(), tableName);
 220       RegionInfo mergedRegionInfo = tableRegions.get(0).getFirst();
 221       TableDescriptor tableDescriptor = MASTER.getTableDescriptors().get(
 222           tableName);
 223       Result mergedRegionResult = MetaTableAccessor.getRegionResult(
 224         MASTER.getConnection(), mergedRegionInfo.getRegionName());
 225
 226       // contains merge reference in META
 227       assertTrue(CatalogFamilyFormat.hasMergeRegions(mergedRegionResult.rawCells()));
 228
 229       // merging regions' directory are in the file system all the same
 230       List<RegionInfo> p = CatalogFamilyFormat.getMergeRegions(mergedRegionResult.rawCells());
 231       RegionInfo regionA = p.get(0);
 232       RegionInfo regionB = p.get(1);
 233       FileSystem fs = MASTER.getMasterFileSystem().getFileSystem();
 234       Path rootDir = MASTER.getMasterFileSystem().getRootDir();
 235
 236       Path tabledir = CommonFSUtils.getTableDir(rootDir, mergedRegionInfo.getTable());
 237       Path regionAdir = new Path(tabledir, regionA.getEncodedName());
 238       Path regionBdir = new Path(tabledir, regionB.getEncodedName());
 239       assertTrue(fs.exists(regionAdir));
 240       assertTrue(fs.exists(regionBdir));
 241
 242       ColumnFamilyDescriptor[] columnFamilies = tableDescriptor.getColumnFamilies();
 243       HRegionFileSystem hrfs = new HRegionFileSystem(
 244         TEST_UTIL.getConfiguration(), fs, tabledir, mergedRegionInfo);
 245       int count = 0;
 246       for(ColumnFamilyDescriptor colFamily : columnFamilies) {
 247         count += hrfs.getStoreFiles(colFamily.getNameAsString()).size();
 248       }
 249       ADMIN.compactRegion(mergedRegionInfo.getRegionName());
 250       // clean up the merged region store files
 251       // wait until merged region have reference file
 252       long timeout = EnvironmentEdgeManager.currentTime() + waitTime;
 253       int newcount = 0;
 254       while (EnvironmentEdgeManager.currentTime() < timeout) {
 255         for(ColumnFamilyDescriptor colFamily : columnFamilies) {
 256           newcount += hrfs.getStoreFiles(colFamily.getNameAsString()).size();
 257         }
 258         if(newcount > count) {
 259           break;
 260         }
 261         Thread.sleep(50);
 262       }
 263       assertTrue(newcount > count);
 264       List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
 265           .getRegionServerThreads();
 266       for (RegionServerThread rs : regionServerThreads) {
 267         CompactedHFilesDischarger cleaner = new CompactedHFilesDischarger(100, null,
 268             rs.getRegionServer(), false);
 269         cleaner.chore();
 270         Thread.sleep(1000);
 271       }
 272       while (EnvironmentEdgeManager.currentTime() < timeout) {
 273         int newcount1 = 0;
 274         for(ColumnFamilyDescriptor colFamily : columnFamilies) {
 275           newcount1 += hrfs.getStoreFiles(colFamily.getNameAsString()).size();
 276         }
 277         if(newcount1 <= 1) {
 278           break;
 279         }
 280         Thread.sleep(50);
 281       }
 282       // run CatalogJanitor to clean merge references in hbase:meta and archive the
 283       // files of merging regions
 284       int cleaned = 0;
 285       while (cleaned == 0) {
 286         cleaned = ADMIN.runCatalogJanitor();
 287         LOG.debug("catalog janitor returned " + cleaned);
 288         Thread.sleep(50);
 289         // Cleanup is async so wait till all procedures are done running.
 290         ProcedureTestingUtility.waitNoProcedureRunning(
 291             TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor());
 292       }
 293       // We used to check for existence of region in fs but sometimes the region dir was
 294       // cleaned up by the time we got here making the test sometimes flakey.
 295       assertTrue(cleaned > 0);
 296
 297       // Wait around a bit to give stuff a chance to complete.
 298       while (true) {
 299         mergedRegionResult = MetaTableAccessor
 300           .getRegionResult(TEST_UTIL.getConnection(), mergedRegionInfo.getRegionName());
 301         if (CatalogFamilyFormat.hasMergeRegions(mergedRegionResult.rawCells())) {
 302           LOG.info("Waiting on cleanup of merge columns {}",
 303             Arrays.asList(mergedRegionResult.rawCells()).stream().
 304               map(c -> c.toString()).collect(Collectors.joining(",")));
 305           Threads.sleep(50);
 306         } else {
 307           break;
 308         }
 309       }
 310       assertFalse(CatalogFamilyFormat.hasMergeRegions(mergedRegionResult.rawCells()));
 311     } finally {
 312       ADMIN.catalogJanitorSwitch(true);
 313       TEST_UTIL.deleteTable(tableName);
 314     }
 315   }
 316
 317   /**
 318    * This test tests 1, merging region not online;
 319    * 2, merging same two regions; 3, merging unknown regions.
 320    * They are in one test case so that we don't have to create
 321    * many tables, and these tests are simple.
 322    */
 323   @Test
 324   public void testMerge() throws Exception {
 325     LOG.info("Starting " + name.getMethodName());
 326     final TableName tableName = TableName.valueOf(name.getMethodName());
 327     final Admin admin = TEST_UTIL.getAdmin();
 328
 329     try {
 330       // Create table and load data.
 331       Table table = createTableAndLoadData(MASTER, tableName);
 332       AssignmentManager am = MASTER.getAssignmentManager();
 333       List<RegionInfo> regions = am.getRegionStates().getRegionsOfTable(tableName);
 334       // Fake offline one region
 335       RegionInfo a = regions.get(0);
 336       RegionInfo b = regions.get(1);
 337       am.unassign(b);
 338       am.offlineRegion(b);
 339       try {
 340         // Merge offline region. Region a is offline here
 341         FutureUtils.get(
 342           admin.mergeRegionsAsync(a.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), false));
 343         fail("Offline regions should not be able to merge");
 344       } catch (DoNotRetryRegionException ie) {
 345         System.out.println(ie);
 346         assertTrue(ie instanceof MergeRegionException);
 347       }
 348
 349       try {
 350         // Merge the same region: b and b.
 351         FutureUtils
 352           .get(admin.mergeRegionsAsync(b.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), true));
 353         fail("A region should not be able to merge with itself, even forcfully");
 354       } catch (IOException ie) {
 355         assertTrue("Exception should mention regions not online",
 356           StringUtils.stringifyException(ie).contains("region to itself") &&
 357             ie instanceof MergeRegionException);
 358       }
 359
 360       try {
 361         // Merge unknown regions
 362         FutureUtils.get(admin.mergeRegionsAsync(Bytes.toBytes("-f1"), Bytes.toBytes("-f2"), true));
 363         fail("Unknown region could not be merged");
 364       } catch (IOException ie) {
 365         assertTrue("UnknownRegionException should be thrown", ie instanceof UnknownRegionException);
 366       }
 367       table.close();
 368     } finally {
 369       TEST_UTIL.deleteTable(tableName);
 370     }
 371   }
 372
 373   @Test
 374   public void testMergeWithReplicas() throws Exception {
 375     final TableName tableName = TableName.valueOf(name.getMethodName());
 376     try {
 377       // Create table and load data.
 378       Table table = createTableAndLoadData(MASTER, tableName, 5, 2);
 379       List<Pair<RegionInfo, ServerName>> initialRegionToServers =
 380         MetaTableAccessor.getTableRegionsAndLocations(TEST_UTIL.getConnection(), tableName);
 381       // Merge 1st and 2nd region
 382       PairOfSameType<RegionInfo> mergedRegions =
 383         mergeRegionsAndVerifyRegionNum(MASTER, tableName, 0, 2, 5 * 2 - 2);
 384       List<Pair<RegionInfo, ServerName>> currentRegionToServers =
 385         MetaTableAccessor.getTableRegionsAndLocations(TEST_UTIL.getConnection(), tableName);
 386       List<RegionInfo> initialRegions = new ArrayList<>();
 387       for (Pair<RegionInfo, ServerName> p : initialRegionToServers) {
 388         initialRegions.add(p.getFirst());
 389       }
 390       List<RegionInfo> currentRegions = new ArrayList<>();
 391       for (Pair<RegionInfo, ServerName> p : currentRegionToServers) {
 392         currentRegions.add(p.getFirst());
 393       }
 394       assertTrue(initialRegions.contains(mergedRegions.getFirst())); //this is the first region
 395       assertTrue(initialRegions.contains(RegionReplicaUtil
 396         .getRegionInfoForReplica(mergedRegions.getFirst(), 1))); //this is the replica of the first region
 397       assertTrue(initialRegions.contains(mergedRegions.getSecond())); //this is the second region
 398       assertTrue(initialRegions.contains(RegionReplicaUtil
 399         .getRegionInfoForReplica(mergedRegions.getSecond(), 1))); //this is the replica of the second region
 400       assertTrue(!initialRegions.contains(currentRegions.get(0))); //this is the new region
 401       assertTrue(!initialRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(currentRegions.get(0), 1))); //replica of the new region
 402       assertTrue(currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(currentRegions.get(0), 1))); //replica of the new region
 403       assertTrue(!currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(mergedRegions.getFirst(), 1))); //replica of the merged region
 404       assertTrue(!currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(mergedRegions.getSecond(), 1))); //replica of the merged region
 405       table.close();
 406     } finally {
 407       TEST_UTIL.deleteTable(tableName);
 408     }
 409   }
 410
 411   private PairOfSameType<RegionInfo> mergeRegionsAndVerifyRegionNum(
 412       HMaster master, TableName tablename,
 413       int regionAnum, int regionBnum, int expectedRegionNum) throws Exception {
 414     PairOfSameType<RegionInfo> mergedRegions =
 415       requestMergeRegion(master, tablename, regionAnum, regionBnum);
 416     waitAndVerifyRegionNum(master, tablename, expectedRegionNum);
 417     return mergedRegions;
 418   }
 419
 420   private PairOfSameType<RegionInfo> requestMergeRegion(
 421       HMaster master, TableName tablename,
 422       int regionAnum, int regionBnum) throws Exception {
 423     List<Pair<RegionInfo, ServerName>> tableRegions = MetaTableAccessor
 424         .getTableRegionsAndLocations(
 425             TEST_UTIL.getConnection(), tablename);
 426     RegionInfo regionA = tableRegions.get(regionAnum).getFirst();
 427     RegionInfo regionB = tableRegions.get(regionBnum).getFirst();
 428     ADMIN.mergeRegionsAsync(
 429       regionA.getEncodedNameAsBytes(),
 430       regionB.getEncodedNameAsBytes(), false);
 431     return new PairOfSameType<>(regionA, regionB);
 432   }
 433
 434   private void waitAndVerifyRegionNum(HMaster master, TableName tablename,
 435       int expectedRegionNum) throws Exception {
 436     List<Pair<RegionInfo, ServerName>> tableRegionsInMeta;
 437     List<RegionInfo> tableRegionsInMaster;
 438     long timeout = EnvironmentEdgeManager.currentTime() + waitTime;
 439     while (EnvironmentEdgeManager.currentTime() < timeout) {
 440       tableRegionsInMeta =
 441           MetaTableAccessor.getTableRegionsAndLocations(TEST_UTIL.getConnection(), tablename);
 442       tableRegionsInMaster =
 443           master.getAssignmentManager().getRegionStates().getRegionsOfTable(tablename);
 444       LOG.info(Objects.toString(tableRegionsInMaster));
 445       LOG.info(Objects.toString(tableRegionsInMeta));
 446       int tableRegionsInMetaSize = tableRegionsInMeta.size();
 447       int tableRegionsInMasterSize = tableRegionsInMaster.size();
 448       if (tableRegionsInMetaSize == expectedRegionNum
 449           && tableRegionsInMasterSize == expectedRegionNum) {
 450         break;
 451       }
 452       Thread.sleep(250);
 453     }
 454
 455     tableRegionsInMeta = MetaTableAccessor.getTableRegionsAndLocations(
 456         TEST_UTIL.getConnection(), tablename);
 457     LOG.info("Regions after merge:" + Joiner.on(',').join(tableRegionsInMeta));
 458     assertEquals(expectedRegionNum, tableRegionsInMeta.size());
 459   }
 460
 461   private Table createTableAndLoadData(HMaster master, TableName tablename)
 462       throws Exception {
 463     return createTableAndLoadData(master, tablename, INITIAL_REGION_NUM, 1);
 464   }
 465
 466   private Table createTableAndLoadData(HMaster master, TableName tablename,
 467       int numRegions, int replication) throws Exception {
 468     assertTrue("ROWSIZE must > numregions:" + numRegions, ROWSIZE > numRegions);
 469     byte[][] splitRows = new byte[numRegions - 1][];
 470     for (int i = 0; i < splitRows.length; i++) {
 471       splitRows[i] = ROWS[(i + 1) * ROWSIZE / numRegions];
 472     }
 473
 474     Table table = TEST_UTIL.createTable(tablename, FAMILYNAME, splitRows);
 475     LOG.info("Created " + table.getName());
 476     if (replication > 1) {
 477       HBaseTestingUtil.setReplicas(ADMIN, tablename, replication);
 478       LOG.info("Set replication of " + replication + " on " + table.getName());
 479     }
 480     loadData(table);
 481     LOG.info("Loaded " + table.getName());
 482     verifyRowCount(table, ROWSIZE);
 483     LOG.info("Verified " + table.getName());
 484
 485     List<Pair<RegionInfo, ServerName>> tableRegions;
 486     TEST_UTIL.waitUntilAllRegionsAssigned(tablename);
 487     LOG.info("All regions assigned for table - " + table.getName());
 488     tableRegions = MetaTableAccessor.getTableRegionsAndLocations(
 489         TEST_UTIL.getConnection(), tablename);
 490     assertEquals("Wrong number of regions in table " + tablename,
 491         numRegions * replication, tableRegions.size());
 492     LOG.info(tableRegions.size() + "Regions after load: " + Joiner.on(',').join(tableRegions));
 493     assertEquals(numRegions * replication, tableRegions.size());
 494     return table;
 495   }
 496
 497   private static byte[][] makeN(byte[] base, int n) {
 498     byte[][] ret = new byte[n][];
 499     for (int i = 0; i < n; i++) {
 500       ret[i] = Bytes.add(base, Bytes.toBytes(String.format("%04d", i)));
 501     }
 502     return ret;
 503   }
 504
 505   private void loadData(Table table) throws IOException {
 506     for (int i = 0; i < ROWSIZE; i++) {
 507       Put put = new Put(ROWS[i]);
 508       put.addColumn(FAMILYNAME, QUALIFIER, Bytes.toBytes(i));
 509       table.put(put);
 510     }
 511   }
 512
 513   private void verifyRowCount(Table table, int expectedRegionNum)
 514       throws IOException {
 515     ResultScanner scanner = table.getScanner(new Scan());
 516     int rowCount = 0;
 517     while (scanner.next() != null) {
 518       rowCount++;
 519     }
 520     assertEquals(expectedRegionNum, rowCount);
 521     scanner.close();
 522   }
 523
 524   // Make it public so that JVMClusterUtil can access it.
 525   public static class MyMaster extends HMaster {
 526     public MyMaster(Configuration conf) throws IOException, KeeperException, InterruptedException {
 527       super(conf);
 528     }
 529
 530     @Override
 531     protected MasterRpcServices createRpcServices() throws IOException {
 532       return new MyMasterRpcServices(this);
 533     }
 534   }
 535
 536   static class MyMasterRpcServices extends MasterRpcServices {
 537     static AtomicBoolean enabled = new AtomicBoolean(false);
 538
 539     private HMaster myMaster;
 540     public MyMasterRpcServices(HMaster master) throws IOException {
 541       super(master);
 542       myMaster = master;
 543     }
 544
 545     @Override
 546     public ReportRegionStateTransitionResponse reportRegionStateTransition(RpcController c,
 547         ReportRegionStateTransitionRequest req) throws ServiceException {
 548       ReportRegionStateTransitionResponse resp = super.reportRegionStateTransition(c, req);
 549       if (enabled.get() && req.getTransition(0).getTransitionCode()
 550           == TransitionCode.READY_TO_MERGE && !resp.hasErrorMessage()) {
 551         RegionStates regionStates = myMaster.getAssignmentManager().getRegionStates();
 552         for (RegionState regionState: regionStates.getRegionsStateInTransition()) {
 553           // Find the merging_new region and remove it
 554           if (regionState.isMergingNew()) {
 555             regionStates.deleteRegion(regionState.getRegion());
 556           }
 557         }
 558       }
 559       return resp;
 560     }
 561   }
 562 }