hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFailedMetaReplicaAssigment.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.client;
  19
  20 import static org.junit.Assert.assertFalse;
  21 import static org.junit.Assert.assertNull;
  22 import static org.junit.Assert.assertTrue;
  23
  24 import java.io.IOException;
  25 import java.util.ArrayList;
  26 import java.util.Collections;
  27 import java.util.List;
  28 import org.apache.hadoop.conf.Configuration;
  29 import org.apache.hadoop.hbase.HBaseClassTestRule;
  30 import org.apache.hadoop.hbase.HBaseTestingUtil;
  31 import org.apache.hadoop.hbase.HConstants;
  32 import org.apache.hadoop.hbase.StartTestingClusterOption;
  33 import org.apache.hadoop.hbase.master.HMaster;
  34 import org.apache.hadoop.hbase.master.MasterServices;
  35 import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
  36 import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
  37 import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure;
  38 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
  39 import org.apache.hadoop.hbase.master.region.MasterRegion;
  40 import org.apache.hadoop.hbase.procedure2.Procedure;
  41 import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
  42 import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
  43 import org.apache.hadoop.hbase.testclassification.MediumTests;
  44 import org.apache.hadoop.hbase.testclassification.MiscTests;
  45 import org.junit.AfterClass;
  46 import org.junit.BeforeClass;
  47 import org.junit.ClassRule;
  48 import org.junit.Test;
  49 import org.junit.experimental.categories.Category;
  50
  51 @Category({ MiscTests.class, MediumTests.class })
  52 public class TestFailedMetaReplicaAssigment {
  53
  54   @ClassRule
  55   public static final HBaseClassTestRule CLASS_RULE =
  56     HBaseClassTestRule.forClass(TestFailedMetaReplicaAssigment.class);
  57
  58   private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
  59
  60   @BeforeClass
  61   public static void setUp() throws Exception {
  62     // using our rigged master, to force a failed meta replica assignment when start up master
  63     // this test can be removed once we remove the HConstants.META_REPLICAS_NUM config.
  64     Configuration conf = TEST_UTIL.getConfiguration();
  65     conf.setInt(HConstants.META_REPLICAS_NUM, 3);
  66     StartTestingClusterOption option =
  67       StartTestingClusterOption.builder().numAlwaysStandByMasters(1).numMasters(1)
  68         .numRegionServers(1).masterClass(BrokenMetaReplicaMaster.class).build();
  69     TEST_UTIL.startMiniCluster(option);
  70   }
  71
  72   @AfterClass
  73   public static void tearDown() throws IOException {
  74     TEST_UTIL.shutdownMiniCluster();
  75   }
  76
  77   @Test
  78   public void testFailedReplicaAssignment() throws InterruptedException {
  79     HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
  80     // waiting for master to come up
  81     TEST_UTIL.waitFor(30000, () -> master.isInitialized());
  82
  83     AssignmentManager am = master.getAssignmentManager();
  84     // showing one of the replicas got assigned
  85     RegionInfo metaReplicaHri =
  86       RegionReplicaUtil.getRegionInfoForReplica(RegionInfoBuilder.FIRST_META_REGIONINFO, 1);
  87     // we use assignAsync so we need to wait a bit
  88     TEST_UTIL.waitFor(30000, () -> {
  89       RegionStateNode metaReplicaRegionNode =
  90         am.getRegionStates().getOrCreateRegionStateNode(metaReplicaHri);
  91       return metaReplicaRegionNode.getRegionLocation() != null;
  92     });
  93     // showing one of the replicas failed to be assigned
  94     RegionInfo metaReplicaHri2 =
  95       RegionReplicaUtil.getRegionInfoForReplica(RegionInfoBuilder.FIRST_META_REGIONINFO, 2);
  96     RegionStateNode metaReplicaRegionNode2 =
  97       am.getRegionStates().getOrCreateRegionStateNode(metaReplicaHri2);
  98     // wait for several seconds to make sure that it is not assigned
  99     for (int i = 0; i < 3; i++) {
 100       Thread.sleep(2000);
 101       assertNull(metaReplicaRegionNode2.getRegionLocation());
 102     }
 103
 104     // showing master is active and running
 105     assertFalse(master.isStopping());
 106     assertFalse(master.isStopped());
 107     assertTrue(master.isActiveMaster());
 108   }
 109
 110   public static class BrokenTransitRegionStateProcedure extends TransitRegionStateProcedure {
 111
 112     public BrokenTransitRegionStateProcedure() {
 113       super(null, null, null, false, TransitionType.ASSIGN);
 114     }
 115
 116     public BrokenTransitRegionStateProcedure(MasterProcedureEnv env, RegionInfo hri) {
 117       super(env, hri, null, false, TransitionType.ASSIGN);
 118     }
 119
 120     @Override
 121     protected Procedure[] execute(MasterProcedureEnv env)
 122       throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException {
 123       throw new ProcedureSuspendedException("Never end procedure!");
 124     }
 125   }
 126
 127   public static class BrokenMetaReplicaMaster extends HMaster {
 128     public BrokenMetaReplicaMaster(final Configuration conf) throws IOException {
 129       super(conf);
 130     }
 131
 132     @Override
 133     public AssignmentManager createAssignmentManager(MasterServices master,
 134       MasterRegion masterRegion) {
 135       return new BrokenMasterMetaAssignmentManager(master, masterRegion);
 136     }
 137   }
 138
 139   public static class BrokenMasterMetaAssignmentManager extends AssignmentManager {
 140     MasterServices master;
 141
 142     public BrokenMasterMetaAssignmentManager(final MasterServices master,
 143       MasterRegion masterRegion) {
 144       super(master, masterRegion);
 145       this.master = master;
 146     }
 147
 148     @Override
 149     public TransitRegionStateProcedure[] createAssignProcedures(List<RegionInfo> hris) {
 150       List<TransitRegionStateProcedure> procs = new ArrayList<>();
 151       for (RegionInfo hri : hris) {
 152         if (hri.isMetaRegion() && hri.getReplicaId() == 2) {
 153           RegionStateNode regionNode = getRegionStates().getOrCreateRegionStateNode(hri);
 154           regionNode.lock();
 155           try {
 156             procs.add(regionNode.setProcedure(new BrokenTransitRegionStateProcedure(
 157               master.getMasterProcedureExecutor().getEnvironment(), hri)));
 158           } finally {
 159             regionNode.unlock();
 160           }
 161         } else {
 162           procs.add(super.createAssignProcedures(Collections.singletonList(hri))[0]);
 163         }
 164       }
 165       return procs.toArray(new TransitRegionStateProcedure[0]);
 166     }
 167   }
 168 }