hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicasShutdownHandling.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.client;
  19
  20 import static org.junit.Assert.assertArrayEquals;
  21 import static org.junit.Assert.assertTrue;
  22
  23 import java.util.Arrays;
  24 import java.util.List;
  25 import org.apache.hadoop.conf.Configuration;
  26 import org.apache.hadoop.hbase.HBaseClassTestRule;
  27 import org.apache.hadoop.hbase.HBaseTestingUtil;
  28 import org.apache.hadoop.hbase.HConstants;
  29 import org.apache.hadoop.hbase.HRegionLocation;
  30 import org.apache.hadoop.hbase.MetaTableAccessor;
  31 import org.apache.hadoop.hbase.ServerName;
  32 import org.apache.hadoop.hbase.TableName;
  33 import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore;
  34 import org.apache.hadoop.hbase.testclassification.MediumTests;
  35 import org.apache.hadoop.hbase.testclassification.MiscTests;
  36 import org.apache.hadoop.hbase.util.Bytes;
  37 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
  38 import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
  39 import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
  40 import org.junit.BeforeClass;
  41 import org.junit.ClassRule;
  42 import org.junit.Test;
  43 import org.junit.experimental.categories.Category;
  44 import org.slf4j.Logger;
  45 import org.slf4j.LoggerFactory;
  46
  47 import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
  48
  49 @Category({ MiscTests.class, MediumTests.class })
  50 public class TestMetaWithReplicasShutdownHandling extends MetaWithReplicasTestBase {
  51
  52   @ClassRule
  53   public static final HBaseClassTestRule CLASS_RULE =
  54     HBaseClassTestRule.forClass(TestMetaWithReplicasShutdownHandling.class);
  55
  56   private static final Logger LOG =
  57     LoggerFactory.getLogger(TestMetaWithReplicasShutdownHandling.class);
  58
  59   @BeforeClass
  60   public static void setUp() throws Exception {
  61     startCluster();
  62   }
  63
  64   @Test
  65   public void testShutdownHandling() throws Exception {
  66     // This test creates a table, flushes the meta (with 3 replicas), kills the
  67     // server holding the primary meta replica. Then it does a put/get into/from
  68     // the test table. The put/get operations would use the replicas to locate the
  69     // location of the test table's region
  70     shutdownMetaAndDoValidations(TEST_UTIL);
  71   }
  72
  73   public static void shutdownMetaAndDoValidations(HBaseTestingUtil util) throws Exception {
  74     // This test creates a table, flushes the meta (with 3 replicas), kills the
  75     // server holding the primary meta replica. Then it does a put/get into/from
  76     // the test table. The put/get operations would use the replicas to locate the
  77     // location of the test table's region
  78     ZKWatcher zkw = util.getZooKeeperWatcher();
  79     Configuration conf = util.getConfiguration();
  80     conf.setBoolean(HConstants.USE_META_REPLICAS, true);
  81
  82     String baseZNode =
  83       conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
  84     String primaryMetaZnode =
  85       ZNodePaths.joinZNode(baseZNode, conf.get("zookeeper.znode.metaserver", "meta-region-server"));
  86     byte[] data = ZKUtil.getData(zkw, primaryMetaZnode);
  87     ServerName primary = ProtobufUtil.toServerName(data);
  88     LOG.info("Primary=" + primary.toString());
  89
  90     TableName TABLE = TableName.valueOf("testShutdownHandling");
  91     byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") };
  92     if (util.getAdmin().tableExists(TABLE)) {
  93       util.getAdmin().disableTable(TABLE);
  94       util.getAdmin().deleteTable(TABLE);
  95     }
  96     byte[] row = Bytes.toBytes("test");
  97     ServerName master = null;
  98     try (Connection c = ConnectionFactory.createConnection(util.getConfiguration())) {
  99       try (Table htable = util.createTable(TABLE, FAMILIES)) {
 100         util.getAdmin().flush(TableName.META_TABLE_NAME);
 101         Thread.sleep(
 102           conf.getInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 30000) * 6);
 103         List<RegionInfo> regions = MetaTableAccessor.getTableRegions(c, TABLE);
 104         HRegionLocation hrl = MetaTableAccessor.getRegionLocation(c, regions.get(0));
 105         // Ensure that the primary server for test table is not the same one as the primary
 106         // of the meta region since we will be killing the srv holding the meta's primary...
 107         // We want to be able to write to the test table even when the meta is not present ..
 108         // If the servers are the same, then move the test table's region out of the server
 109         // to another random server
 110         if (hrl.getServerName().equals(primary)) {
 111           util.getAdmin().move(hrl.getRegion().getEncodedNameAsBytes());
 112           // wait for the move to complete
 113           do {
 114             Thread.sleep(10);
 115             hrl = MetaTableAccessor.getRegionLocation(c, regions.get(0));
 116           } while (primary.equals(hrl.getServerName()));
 117           util.getAdmin().flush(TableName.META_TABLE_NAME);
 118           Thread.sleep(
 119             conf.getInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 30000) * 3);
 120         }
 121         // Ensure all metas are not on same hbase:meta replica=0 server!
 122
 123         master = util.getHBaseClusterInterface().getClusterMetrics().getMasterName();
 124         // kill the master so that regionserver recovery is not triggered at all
 125         // for the meta server
 126         LOG.info("Stopping master=" + master.toString());
 127         util.getHBaseClusterInterface().stopMaster(master);
 128         util.getHBaseClusterInterface().waitForMasterToStop(master, 60000);
 129         LOG.info("Master " + master + " stopped!");
 130         if (!master.equals(primary)) {
 131           util.getHBaseClusterInterface().killRegionServer(primary);
 132           util.getHBaseClusterInterface().waitForRegionServerToStop(primary, 60000);
 133         }
 134         c.clearRegionLocationCache();
 135       }
 136       LOG.info("Running GETs");
 137       try (Table htable = c.getTable(TABLE)) {
 138         Put put = new Put(row);
 139         put.addColumn(Bytes.toBytes("foo"), row, row);
 140         BufferedMutator m = c.getBufferedMutator(TABLE);
 141         m.mutate(put);
 142         m.flush();
 143         // Try to do a get of the row that was just put
 144         Result r = htable.get(new Get(row));
 145         assertTrue(Arrays.equals(r.getRow(), row));
 146         // now start back the killed servers and disable use of replicas. That would mean
 147         // calls go to the primary
 148         LOG.info("Starting Master");
 149         util.getHBaseClusterInterface().startMaster(master.getHostname(), 0);
 150         util.getHBaseClusterInterface().startRegionServer(primary.getHostname(), 0);
 151         util.getHBaseClusterInterface().waitForActiveAndReadyMaster();
 152         LOG.info("Master active!");
 153         c.clearRegionLocationCache();
 154       }
 155     }
 156     conf.setBoolean(HConstants.USE_META_REPLICAS, false);
 157     LOG.info("Running GETs no replicas");
 158     try (Connection c = ConnectionFactory.createConnection(conf);
 159       Table htable = c.getTable(TABLE)) {
 160       Result r = htable.get(new Get(row));
 161       assertArrayEquals(row, r.getRow());
 162     }
 163   }
 164 }