hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaShutdownHandler.java

   1 /*
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.master;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertNotEquals;
  22 import static org.junit.Assert.assertTrue;
  23
  24 import java.io.IOException;
  25 import org.apache.hadoop.conf.Configuration;
  26 import org.apache.hadoop.hbase.HBaseClassTestRule;
  27 import org.apache.hadoop.hbase.HBaseTestingUtil;
  28 import org.apache.hadoop.hbase.ServerName;
  29 import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
  30 import org.apache.hadoop.hbase.SingleProcessHBaseCluster.MiniHBaseClusterRegionServer;
  31 import org.apache.hadoop.hbase.StartTestingClusterOption;
  32 import org.apache.hadoop.hbase.Waiter;
  33 import org.apache.hadoop.hbase.client.RegionInfoBuilder;
  34 import org.apache.hadoop.hbase.master.assignment.RegionStates;
  35 import org.apache.hadoop.hbase.regionserver.HRegionServer;
  36 import org.apache.hadoop.hbase.testclassification.MediumTests;
  37 import org.apache.hadoop.hbase.util.Bytes;
  38 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
  39 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
  40 import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
  41 import org.apache.zookeeper.KeeperException;
  42 import org.junit.AfterClass;
  43 import org.junit.BeforeClass;
  44 import org.junit.ClassRule;
  45 import org.junit.Test;
  46 import org.junit.experimental.categories.Category;
  47 import org.slf4j.Logger;
  48 import org.slf4j.LoggerFactory;
  49
  50 /**
  51  * Tests handling of meta-carrying region server failover.
  52  */
  53 @Category(MediumTests.class)
  54 public class TestMetaShutdownHandler {
  55   private static final Logger LOG = LoggerFactory.getLogger(TestMetaShutdownHandler.class);
  56   @ClassRule
  57   public static final HBaseClassTestRule CLASS_RULE =
  58       HBaseClassTestRule.forClass(TestMetaShutdownHandler.class);
  59
  60   private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
  61   final static Configuration conf = TEST_UTIL.getConfiguration();
  62
  63   @BeforeClass
  64   public static void setUpBeforeClass() throws Exception {
  65     StartTestingClusterOption option = StartTestingClusterOption.builder()
  66         .numRegionServers(3).rsClass(MyRegionServer.class).numDataNodes(3).build();
  67     TEST_UTIL.startMiniCluster(option);
  68   }
  69
  70   @AfterClass
  71   public static void tearDownAfterClass() throws Exception {
  72     TEST_UTIL.shutdownMiniCluster();
  73   }
  74
  75   /**
  76    * This test will test the expire handling of a meta-carrying
  77    * region server.
  78    * After HBaseMiniCluster is up, we will delete the ephemeral
  79    * node of the meta-carrying region server, which will trigger
  80    * the expire of this region server on the master.
  81    * On the other hand, we will slow down the abort process on
  82    * the region server so that it is still up during the master SSH.
  83    * We will check that the master SSH is still successfully done.
  84    */
  85   @Test
  86   public void testExpireMetaRegionServer() throws Exception {
  87     SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
  88     HMaster master = cluster.getMaster();
  89     RegionStates regionStates = master.getAssignmentManager().getRegionStates();
  90     ServerName metaServerName =
  91       regionStates.getRegionServerOfRegion(RegionInfoBuilder.FIRST_META_REGIONINFO);
  92     if (master.getServerName().equals(metaServerName) || metaServerName == null ||
  93       !metaServerName.equals(cluster.getServerHoldingMeta())) {
  94       // Move meta off master
  95       metaServerName =
  96         cluster.getLiveRegionServerThreads().get(0).getRegionServer().getServerName();
  97       master.move(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(),
  98         Bytes.toBytes(metaServerName.getServerName()));
  99       TEST_UTIL.waitUntilNoRegionsInTransition(60000);
 100       metaServerName =
 101         regionStates.getRegionServerOfRegion(RegionInfoBuilder.FIRST_META_REGIONINFO);
 102     }
 103     RegionState metaState = MetaTableLocator.getMetaRegionState(master.getZooKeeper());
 104     assertEquals("Wrong state for meta!", RegionState.State.OPEN, metaState.getState());
 105     assertNotEquals("Meta is on master!", metaServerName, master.getServerName());
 106     HRegionServer metaRegionServer = cluster.getRegionServer(metaServerName);
 107
 108     // Delete the ephemeral node of the meta-carrying region server.
 109     // This is trigger the expire of this region server on the master.
 110     String rsEphemeralNodePath =
 111         ZNodePaths.joinZNode(master.getZooKeeper().getZNodePaths().rsZNode,
 112                 metaServerName.toString());
 113     ZKUtil.deleteNode(master.getZooKeeper(), rsEphemeralNodePath);
 114     LOG.info("Deleted the znode for the RegionServer hosting hbase:meta; waiting on SSH");
 115     // Wait for SSH to finish
 116     final ServerManager serverManager = master.getServerManager();
 117     final ServerName priorMetaServerName = metaServerName;
 118     TEST_UTIL.waitFor(60000, 100, () -> metaRegionServer.isStopped());
 119     TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
 120       @Override
 121       public boolean evaluate() throws Exception {
 122         return !serverManager.isServerOnline(priorMetaServerName)
 123             && !serverManager.areDeadServersInProgress();
 124       }
 125     });
 126     LOG.info("Past wait on RIT");
 127     TEST_UTIL.waitUntilNoRegionsInTransition(60000);
 128     // Now, make sure meta is assigned
 129     assertTrue("Meta should be assigned",
 130       regionStates.isRegionOnline(RegionInfoBuilder.FIRST_META_REGIONINFO));
 131     // Now, make sure meta is registered in zk
 132     metaState = MetaTableLocator.getMetaRegionState(master.getZooKeeper());
 133     assertEquals("Meta should not be in transition", RegionState.State.OPEN, metaState.getState());
 134     assertEquals("Meta should be assigned", metaState.getServerName(),
 135       regionStates.getRegionServerOfRegion(RegionInfoBuilder.FIRST_META_REGIONINFO));
 136     assertNotEquals("Meta should be assigned on a different server", metaState.getServerName(),
 137       metaServerName);
 138   }
 139
 140   public static class MyRegionServer extends MiniHBaseClusterRegionServer {
 141
 142     public MyRegionServer(Configuration conf) throws IOException, KeeperException,
 143         InterruptedException {
 144       super(conf);
 145     }
 146
 147     @Override
 148     public void abort(String reason, Throwable cause) {
 149       // sleep to slow down the region server abort
 150       try {
 151         Thread.sleep(30*1000);
 152       } catch (InterruptedException e) {
 153         return;
 154       }
 155       super.abort(reason, cause);
 156     }
 157   }
 158 }