hbase-server/src/test/java/org/apache/hadoop/hbase/client/RegionReplicaTestHelper.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.client;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertNotNull;
  22 import static org.junit.Assert.fail;
  23
  24 import java.io.IOException;
  25 import java.util.ArrayList;
  26 import java.util.List;
  27 import java.util.Optional;
  28 import org.apache.hadoop.conf.Configuration;
  29 import org.apache.hadoop.hbase.HBaseTestingUtil;
  30 import org.apache.hadoop.hbase.HRegionLocation;
  31 import org.apache.hadoop.hbase.NotServingRegionException;
  32 import org.apache.hadoop.hbase.RegionLocations;
  33 import org.apache.hadoop.hbase.ServerName;
  34 import org.apache.hadoop.hbase.TableName;
  35 import org.apache.hadoop.hbase.Waiter;
  36 import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
  37 import org.apache.hadoop.hbase.regionserver.Region;
  38 import org.apache.hadoop.hbase.util.JVMClusterUtil;
  39
  40 public final class RegionReplicaTestHelper {
  41
  42   private RegionReplicaTestHelper() {
  43   }
  44
  45   // waits for all replicas to have region location
  46   static void waitUntilAllMetaReplicasAreReady(HBaseTestingUtil util,
  47     ConnectionRegistry registry) throws IOException {
  48     Configuration conf = util.getConfiguration();
  49     int regionReplicaCount =
  50       util.getAdmin().getDescriptor(TableName.META_TABLE_NAME).getRegionReplication();
  51     Waiter.waitFor(conf, conf.getLong("hbase.client.sync.wait.timeout.msec", 60000), 200, true,
  52       new ExplainingPredicate<IOException>() {
  53         @Override
  54         public String explainFailure() {
  55           return "Not all meta replicas get assigned";
  56         }
  57
  58         @Override
  59         public boolean evaluate() {
  60           try {
  61             RegionLocations locs = registry.getMetaRegionLocations().get();
  62             if (locs.size() < regionReplicaCount) {
  63               return false;
  64             }
  65             for (int i = 0; i < regionReplicaCount; i++) {
  66               HRegionLocation loc = locs.getRegionLocation(i);
  67               // Wait until the replica is served by a region server. There could be delay between
  68               // the replica being available to the connection and region server opening it.
  69               Optional<ServerName> rsCarryingReplica =
  70                   getRSCarryingReplica(util, loc.getRegion().getTable(), i);
  71               if (!rsCarryingReplica.isPresent()) {
  72                 return false;
  73               }
  74             }
  75             return true;
  76           } catch (Exception e) {
  77             TestZKConnectionRegistry.LOG.warn("Failed to get meta region locations", e);
  78             return false;
  79           }
  80         }
  81       });
  82   }
  83
  84   static Optional<ServerName> getRSCarryingReplica(HBaseTestingUtil util, TableName tableName,
  85       int replicaId) {
  86     return util.getHBaseCluster().getRegionServerThreads().stream().map(t -> t.getRegionServer())
  87       .filter(rs -> rs.getRegions(tableName).stream()
  88         .anyMatch(r -> r.getRegionInfo().getReplicaId() == replicaId))
  89       .findAny().map(rs -> rs.getServerName());
  90   }
  91
  92   /**
  93    * Return the new location.
  94    */
  95   static ServerName moveRegion(HBaseTestingUtil util, HRegionLocation currentLoc)
  96       throws Exception {
  97     ServerName serverName = currentLoc.getServerName();
  98     RegionInfo regionInfo = currentLoc.getRegion();
  99     TableName tableName = regionInfo.getTable();
 100     int replicaId = regionInfo.getReplicaId();
 101     ServerName newServerName = util.getHBaseCluster().getRegionServerThreads().stream()
 102       .map(t -> t.getRegionServer().getServerName()).filter(sn -> !sn.equals(serverName)).findAny()
 103       .get();
 104     util.getAdmin().move(regionInfo.getEncodedNameAsBytes(), newServerName);
 105     util.waitFor(30000, new ExplainingPredicate<Exception>() {
 106
 107       @Override
 108       public boolean evaluate() throws Exception {
 109         Optional<ServerName> newServerName = getRSCarryingReplica(util, tableName, replicaId);
 110         return newServerName.isPresent() && !newServerName.get().equals(serverName);
 111       }
 112
 113       @Override
 114       public String explainFailure() throws Exception {
 115         return regionInfo.getRegionNameAsString() + " is still on " + serverName;
 116       }
 117     });
 118     return newServerName;
 119   }
 120
 121   interface Locator {
 122     RegionLocations getRegionLocations(TableName tableName, int replicaId, boolean reload)
 123         throws Exception;
 124
 125     void updateCachedLocationOnError(HRegionLocation loc, Throwable error) throws Exception;
 126   }
 127
 128   static void testLocator(HBaseTestingUtil util, TableName tableName, Locator locator)
 129       throws Exception {
 130     RegionLocations locs =
 131       locator.getRegionLocations(tableName, RegionReplicaUtil.DEFAULT_REPLICA_ID, false);
 132     assertEquals(3, locs.size());
 133     for (int i = 0; i < 3; i++) {
 134       HRegionLocation loc = locs.getRegionLocation(i);
 135       assertNotNull(loc);
 136       ServerName serverName = getRSCarryingReplica(util, tableName, i).get();
 137       assertEquals(serverName, loc.getServerName());
 138     }
 139     ServerName newServerName = moveRegion(util, locs.getDefaultRegionLocation());
 140     // The cached location should not be changed
 141     assertEquals(locs.getDefaultRegionLocation().getServerName(),
 142       locator.getRegionLocations(tableName, RegionReplicaUtil.DEFAULT_REPLICA_ID, false)
 143         .getDefaultRegionLocation().getServerName());
 144     // should get the new location when reload = true
 145     // when meta replica LoadBalance mode is enabled, it may delay a bit.
 146     util.waitFor(3000, new ExplainingPredicate<Exception>() {
 147       @Override
 148       public boolean evaluate() throws Exception {
 149         ServerName sn = locator.getRegionLocations(tableName, RegionReplicaUtil.DEFAULT_REPLICA_ID,
 150           true).getDefaultRegionLocation().getServerName();
 151         return newServerName.equals(sn);
 152       }
 153
 154       @Override
 155       public String explainFailure() throws Exception {
 156         return "New location does not show up in meta (replica) region";
 157       }
 158     });
 159
 160     assertEquals(newServerName,
 161       locator.getRegionLocations(tableName, RegionReplicaUtil.DEFAULT_REPLICA_ID, true)
 162         .getDefaultRegionLocation().getServerName());
 163     // the cached location should be replaced
 164     assertEquals(newServerName,
 165       locator.getRegionLocations(tableName, RegionReplicaUtil.DEFAULT_REPLICA_ID, false)
 166         .getDefaultRegionLocation().getServerName());
 167
 168     ServerName newServerName1 = moveRegion(util, locs.getRegionLocation(1));
 169     ServerName newServerName2 = moveRegion(util, locs.getRegionLocation(2));
 170
 171     // The cached location should not be change
 172     assertEquals(locs.getRegionLocation(1).getServerName(),
 173       locator.getRegionLocations(tableName, 1, false).getRegionLocation(1).getServerName());
 174     // clear the cached location for replica 1
 175     locator.updateCachedLocationOnError(locs.getRegionLocation(1), new NotServingRegionException());
 176     // the cached location for replica 2 should not be changed
 177     assertEquals(locs.getRegionLocation(2).getServerName(),
 178       locator.getRegionLocations(tableName, 2, false).getRegionLocation(2).getServerName());
 179     // should get the new location as we have cleared the old location
 180     assertEquals(newServerName1,
 181       locator.getRegionLocations(tableName, 1, false).getRegionLocation(1).getServerName());
 182     // as we will get the new location for replica 2 at once, we should also get the new location
 183     // for replica 2
 184     assertEquals(newServerName2,
 185       locator.getRegionLocations(tableName, 2, false).getRegionLocation(2).getServerName());
 186   }
 187
 188   public static void assertReplicaDistributed(HBaseTestingUtil util, Table t)
 189     throws IOException {
 190     if (t.getDescriptor().getRegionReplication() <= 1) {
 191       return;
 192     }
 193     List<RegionInfo> regionInfos = new ArrayList<>();
 194     for (JVMClusterUtil.RegionServerThread rs : util.getMiniHBaseCluster()
 195       .getRegionServerThreads()) {
 196       regionInfos.clear();
 197       for (Region r : rs.getRegionServer().getRegions(t.getName())) {
 198         if (contains(regionInfos, r.getRegionInfo())) {
 199           fail("Replica regions should be assigned to different region servers");
 200         } else {
 201           regionInfos.add(r.getRegionInfo());
 202         }
 203       }
 204     }
 205   }
 206
 207   private static boolean contains(List<RegionInfo> regionInfos, RegionInfo regionInfo) {
 208     for (RegionInfo info : regionInfos) {
 209       if (RegionReplicaUtil.isReplicasForSameRegion(info, regionInfo)) {
 210         return true;
 211       }
 212     }
 213     return false;
 214   }
 215 }