hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.rsgroup;
  19
  20 import java.io.IOException;
  21 import java.util.ArrayList;
  22 import java.util.Collections;
  23 import java.util.HashMap;
  24 import java.util.HashSet;
  25 import java.util.Iterator;
  26 import java.util.LinkedList;
  27 import java.util.List;
  28 import java.util.Map;
  29 import java.util.Set;
  30 import java.util.function.Function;
  31
  32 import org.apache.commons.lang3.StringUtils;
  33 import org.apache.hadoop.hbase.DoNotRetryIOException;
  34 import org.apache.hadoop.hbase.NamespaceDescriptor;
  35 import org.apache.hadoop.hbase.ServerName;
  36 import org.apache.hadoop.hbase.TableName;
  37 import org.apache.hadoop.hbase.client.RegionInfo;
  38 import org.apache.hadoop.hbase.constraint.ConstraintException;
  39 import org.apache.hadoop.hbase.master.HMaster;
  40 import org.apache.hadoop.hbase.master.LoadBalancer;
  41 import org.apache.hadoop.hbase.master.MasterServices;
  42 import org.apache.hadoop.hbase.master.RegionPlan;
  43 import org.apache.hadoop.hbase.master.RegionState;
  44 import org.apache.hadoop.hbase.master.ServerManager;
  45 import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
  46 import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
  47 import org.apache.hadoop.hbase.net.Address;
  48 import org.apache.yetus.audience.InterfaceAudience;
  49 import org.slf4j.Logger;
  50 import org.slf4j.LoggerFactory;
  51
  52 import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
  53
  54 /**
  55  * Service to support Region Server Grouping (HBase-6721).
  56  */
  57 @InterfaceAudience.Private
  58 public class RSGroupAdminServer implements RSGroupAdmin {
  59   private static final Logger LOG = LoggerFactory.getLogger(RSGroupAdminServer.class);
  60   static final String KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE = "should keep at least " +
  61           "one server in 'default' RSGroup.";
  62
  63   private MasterServices master;
  64   private final RSGroupInfoManager rsGroupInfoManager;
  65
  66   /** Define the config key of retries threshold when movements failed */
  67   //made package private for testing
  68   static final String FAILED_MOVE_MAX_RETRY = "hbase.rsgroup.move.max.retry";
  69
  70   /** Define the default number of retries */
  71   //made package private for testing
  72   static final int DEFAULT_MAX_RETRY_VALUE = 50;
  73
  74   private int moveMaxRetry;
  75
  76   public RSGroupAdminServer(MasterServices master, RSGroupInfoManager rsGroupInfoManager) {
  77     this.master = master;
  78     this.rsGroupInfoManager = rsGroupInfoManager;
  79     this.moveMaxRetry = master.getConfiguration().getInt(FAILED_MOVE_MAX_RETRY,
  80         DEFAULT_MAX_RETRY_VALUE);
  81   }
  82
  83   @Override
  84   public RSGroupInfo getRSGroupInfo(String groupName) throws IOException {
  85     return rsGroupInfoManager.getRSGroup(groupName);
  86   }
  87
  88   @Override
  89   public RSGroupInfo getRSGroupInfoOfTable(TableName tableName) throws IOException {
  90     // We are reading across two Maps in the below with out synchronizing across
  91     // them; should be safe most of the time.
  92     String groupName = rsGroupInfoManager.getRSGroupOfTable(tableName);
  93     return groupName == null? null: rsGroupInfoManager.getRSGroup(groupName);
  94   }
  95
  96   private void checkOnlineServersOnly(Set<Address> servers) throws ConstraintException {
  97     // This uglyness is because we only have Address, not ServerName.
  98     // Online servers are keyed by ServerName.
  99     Set<Address> onlineServers = new HashSet<>();
 100     for(ServerName server: master.getServerManager().getOnlineServers().keySet()) {
 101       onlineServers.add(server.getAddress());
 102     }
 103     for (Address address: servers) {
 104       if (!onlineServers.contains(address)) {
 105         throw new ConstraintException(
 106             "Server " + address + " is not an online server in 'default' RSGroup.");
 107       }
 108     }
 109   }
 110
 111   /**
 112    * Check passed name. Fail if nulls or if corresponding RSGroupInfo not found.
 113    * @return The RSGroupInfo named <code>name</code>
 114    */
 115   private RSGroupInfo getAndCheckRSGroupInfo(String name) throws IOException {
 116     if (StringUtils.isEmpty(name)) {
 117       throw new ConstraintException("RSGroup cannot be null.");
 118     }
 119     RSGroupInfo rsGroupInfo = getRSGroupInfo(name);
 120     if (rsGroupInfo == null) {
 121       throw new ConstraintException("RSGroup does not exist: " + name);
 122     }
 123     return rsGroupInfo;
 124   }
 125
 126   /**
 127    * @return List of Regions associated with this <code>server</code>.
 128    */
 129   private List<RegionInfo> getRegions(final Address server) {
 130     LinkedList<RegionInfo> regions = new LinkedList<>();
 131     for (Map.Entry<RegionInfo, ServerName> el :
 132         master.getAssignmentManager().getRegionStates().getRegionAssignments().entrySet()) {
 133       if (el.getValue() == null) {
 134         continue;
 135       }
 136
 137       if (el.getValue().getAddress().equals(server)) {
 138         addRegion(regions, el.getKey());
 139       }
 140     }
 141     for (RegionStateNode state : master.getAssignmentManager().getRegionsInTransition()) {
 142       if (state.getRegionLocation() != null &&
 143           state.getRegionLocation().getAddress().equals(server)) {
 144         addRegion(regions, state.getRegionInfo());
 145       }
 146     }
 147     return regions;
 148   }
 149
 150   private void addRegion(final LinkedList<RegionInfo> regions, RegionInfo hri) {
 151     // If meta, move it last otherwise other unassigns fail because meta is not
 152     // online for them to update state in. This is dodgy. Needs to be made more
 153     // robust. See TODO below.
 154     if (hri.isMetaRegion()) {
 155       regions.addLast(hri);
 156     } else {
 157       regions.addFirst(hri);
 158     }
 159   }
 160
 161   /**
 162    * Check servers and tables.
 163    *
 164    * @param servers servers to move
 165    * @param tables tables to move
 166    * @param targetGroupName target group name
 167    * @throws IOException if nulls or if servers and tables not belong to the same group
 168    */
 169   private void checkServersAndTables(Set<Address> servers, Set<TableName> tables,
 170                                      String targetGroupName) throws IOException {
 171     // Presume first server's source group. Later ensure all servers are from this group.
 172     Address firstServer = servers.iterator().next();
 173     RSGroupInfo tmpSrcGrp = rsGroupInfoManager.getRSGroupOfServer(firstServer);
 174     if (tmpSrcGrp == null) {
 175       // Be careful. This exception message is tested for in TestRSGroupsBase...
 176       throw new ConstraintException("Source RSGroup for server " + firstServer
 177               + " does not exist.");
 178     }
 179     RSGroupInfo srcGrp = new RSGroupInfo(tmpSrcGrp);
 180
 181     // Only move online servers
 182     checkOnlineServersOnly(servers);
 183
 184     // Ensure all servers are of same rsgroup.
 185     for (Address server: servers) {
 186       String tmpGroup = rsGroupInfoManager.getRSGroupOfServer(server).getName();
 187       if (!tmpGroup.equals(srcGrp.getName())) {
 188         throw new ConstraintException("Move server request should only come from one source " +
 189                 "RSGroup. Expecting only " + srcGrp.getName() + " but contains " + tmpGroup);
 190       }
 191     }
 192
 193     // Ensure all tables and servers are of same rsgroup.
 194     for (TableName table : tables) {
 195       String tmpGroup = rsGroupInfoManager.getRSGroupOfTable(table);
 196       if (!tmpGroup.equals(srcGrp.getName())) {
 197         throw new ConstraintException("Move table request should only come from one source " +
 198                 "RSGroup. Expecting only " + srcGrp.getName() + " but contains " + tmpGroup);
 199       }
 200     }
 201
 202     if (srcGrp.getServers().size() <= servers.size() && srcGrp.getTables().size() > tables.size()) {
 203       throw new ConstraintException("Cannot leave a RSGroup " + srcGrp.getName() +
 204               " that contains tables without servers to host them.");
 205     }
 206   }
 207
 208   /**
 209    * Move every region from servers which are currently located on these servers,
 210    * but should not be located there.
 211    *
 212    * @param servers the servers that will move to new group
 213    * @param targetGroupName the target group name
 214    * @throws IOException if moving the server and tables fail
 215    */
 216   private void moveServerRegionsFromGroup(Set<Address> servers, String targetGroupName)
 217     throws IOException {
 218     moveRegionsBetweenGroups(servers, targetGroupName,
 219       rs -> getRegions(rs),
 220       info -> {
 221         try {
 222           RSGroupInfo group = getRSGroupInfo(targetGroupName);
 223           return group.containsTable(info.getTable());
 224         } catch (IOException e) {
 225           e.printStackTrace();
 226           return false;
 227         }
 228       },
 229       rs -> rs.getHostname());
 230   }
 231
 232   /**
 233    * Moves regions of tables which are not on target group servers.
 234    *
 235    * @param tables the tables that will move to new group
 236    * @param targetGroupName the target group name
 237    * @throws IOException if moving the region fails
 238    */
 239   private void moveTableRegionsToGroup(Set<TableName> tables, String targetGroupName)
 240     throws IOException {
 241     moveRegionsBetweenGroups(tables, targetGroupName,
 242       table -> {
 243         if (master.getAssignmentManager().isTableDisabled(table)) {
 244           return new ArrayList<>();
 245         }
 246         return master.getAssignmentManager().getRegionStates().getRegionsOfTable(table);
 247       },
 248       info -> {
 249         try {
 250           RSGroupInfo group = getRSGroupInfo(targetGroupName);
 251           ServerName sn =
 252               master.getAssignmentManager().getRegionStates().getRegionServerOfRegion(info);
 253           return group.containsServer(sn.getAddress());
 254         } catch (IOException e) {
 255           e.printStackTrace();
 256           return false;
 257         }
 258       },
 259       table -> table.getNameWithNamespaceInclAsString());
 260   }
 261
 262   private <T> void moveRegionsBetweenGroups(Set<T> regionsOwners, String targetGroupName,
 263       Function<T, List<RegionInfo>> getRegionsInfo, Function<RegionInfo, Boolean> validation,
 264       Function<T, String> getOwnerName) throws IOException {
 265     boolean hasRegionsToMove;
 266     int retry = 0;
 267     Set<T> allOwners = new HashSet<>(regionsOwners);
 268     Set<String> failedRegions = new HashSet<>();
 269     IOException toThrow = null;
 270     do {
 271       hasRegionsToMove = false;
 272       for (Iterator<T> iter = allOwners.iterator(); iter.hasNext(); ) {
 273         T owner = iter.next();
 274         // Get regions that are associated with this server and filter regions by group tables.
 275         for (RegionInfo region : getRegionsInfo.apply(owner)) {
 276           if (!validation.apply(region)) {
 277             LOG.info("Moving region {}, which do not belong to RSGroup {}",
 278                 region.getShortNameToLog(), targetGroupName);
 279             try {
 280               this.master.getAssignmentManager().move(region);
 281               failedRegions.remove(region.getRegionNameAsString());
 282             } catch (IOException ioe) {
 283               LOG.debug("Move region {} from group failed, will retry, current retry time is {}",
 284                   region.getShortNameToLog(), retry, ioe);
 285               toThrow = ioe;
 286               failedRegions.add(region.getRegionNameAsString());
 287             }
 288             if (master.getAssignmentManager().getRegionStates().
 289                 getRegionState(region).isFailedOpen()) {
 290               continue;
 291             }
 292             hasRegionsToMove = true;
 293           }
 294         }
 295
 296         if (!hasRegionsToMove) {
 297           LOG.info("No more regions to move from {} to RSGroup", getOwnerName.apply(owner));
 298           iter.remove();
 299         }
 300       }
 301
 302       retry++;
 303       try {
 304         rsGroupInfoManager.wait(1000);
 305       } catch (InterruptedException e) {
 306         LOG.warn("Sleep interrupted", e);
 307         Thread.currentThread().interrupt();
 308       }
 309     } while (hasRegionsToMove && retry <= moveMaxRetry);
 310
 311     //has up to max retry time or there are no more regions to move
 312     if (hasRegionsToMove) {
 313       // print failed moved regions, for later process conveniently
 314       String msg = String
 315           .format("move regions for group %s failed, failed regions: %s", targetGroupName,
 316               failedRegions);
 317       LOG.error(msg);
 318       throw new DoNotRetryIOException(
 319           msg + ", just record the last failed region's cause, more details in server log",
 320           toThrow);
 321     }
 322   }
 323
 324   @edu.umd.cs.findbugs.annotations.SuppressWarnings(
 325       value="RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE",
 326       justification="Ignoring complaint because don't know what it is complaining about")
 327   @Override
 328   public void moveServers(Set<Address> servers, String targetGroupName) throws IOException {
 329     if (servers == null) {
 330       throw new ConstraintException("The list of servers to move cannot be null.");
 331     }
 332     if (servers.isEmpty()) {
 333       // For some reason this difference between null servers and isEmpty is important distinction.
 334       // TODO. Why? Stuff breaks if I equate them.
 335       return;
 336     }
 337     //check target group
 338     getAndCheckRSGroupInfo(targetGroupName);
 339
 340     // Hold a lock on the manager instance while moving servers to prevent
 341     // another writer changing our state while we are working.
 342     synchronized (rsGroupInfoManager) {
 343       // Presume first server's source group. Later ensure all servers are from this group.
 344       Address firstServer = servers.iterator().next();
 345       RSGroupInfo srcGrp = rsGroupInfoManager.getRSGroupOfServer(firstServer);
 346       if (srcGrp == null) {
 347         // Be careful. This exception message is tested for in TestRSGroupsBase...
 348         throw new ConstraintException("Source RSGroup for server " + firstServer
 349             + " does not exist.");
 350       }
 351       // Only move online servers (when moving from 'default') or servers from other
 352       // groups. This prevents bogus servers from entering groups
 353       if (RSGroupInfo.DEFAULT_GROUP.equals(srcGrp.getName())) {
 354         if (srcGrp.getServers().size() <= servers.size()) {
 355           throw new ConstraintException(KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE);
 356         }
 357         checkOnlineServersOnly(servers);
 358       }
 359       // Ensure all servers are of same rsgroup.
 360       for (Address server: servers) {
 361         String tmpGroup = rsGroupInfoManager.getRSGroupOfServer(server).getName();
 362         if (!tmpGroup.equals(srcGrp.getName())) {
 363           throw new ConstraintException("Move server request should only come from one source " +
 364               "RSGroup. Expecting only " + srcGrp.getName() + " but contains " + tmpGroup);
 365         }
 366       }
 367       if (srcGrp.getServers().size() <= servers.size() && srcGrp.getTables().size() > 0) {
 368         throw new ConstraintException("Cannot leave a RSGroup " + srcGrp.getName() +
 369             " that contains tables without servers to host them.");
 370       }
 371
 372       // MovedServers may be < passed in 'servers'.
 373       Set<Address> movedServers = rsGroupInfoManager.moveServers(servers, srcGrp.getName(),
 374           targetGroupName);
 375       moveServerRegionsFromGroup(movedServers, targetGroupName);
 376       LOG.info("Move servers done: {} => {}", srcGrp.getName(), targetGroupName);
 377     }
 378   }
 379
 380   @Override
 381   public void moveTables(Set<TableName> tables, String targetGroup) throws IOException {
 382     if (tables == null) {
 383       throw new ConstraintException("The list of servers cannot be null.");
 384     }
 385     if (tables.size() < 1) {
 386       LOG.debug("moveTables() passed an empty set. Ignoring.");
 387       return;
 388     }
 389
 390     // Hold a lock on the manager instance while moving servers to prevent
 391     // another writer changing our state while we are working.
 392     synchronized (rsGroupInfoManager) {
 393       if(targetGroup != null) {
 394         RSGroupInfo destGroup = rsGroupInfoManager.getRSGroup(targetGroup);
 395         if(destGroup == null) {
 396           throw new ConstraintException("Target " + targetGroup + " RSGroup does not exist.");
 397         }
 398         if(destGroup.getServers().size() < 1) {
 399           throw new ConstraintException("Target RSGroup must have at least one server.");
 400         }
 401       }
 402       rsGroupInfoManager.moveTables(tables, targetGroup);
 403
 404       // targetGroup is null when a table is being deleted. In this case no further
 405       // action is required.
 406       if (targetGroup != null) {
 407         moveTableRegionsToGroup(tables, targetGroup);
 408       }
 409     }
 410   }
 411
 412   @Override
 413   public void addRSGroup(String name) throws IOException {
 414     rsGroupInfoManager.addRSGroup(new RSGroupInfo(name));
 415   }
 416
 417   @Override
 418   public void removeRSGroup(String name) throws IOException {
 419     // Hold a lock on the manager instance while moving servers to prevent
 420     // another writer changing our state while we are working.
 421     synchronized (rsGroupInfoManager) {
 422       RSGroupInfo rsGroupInfo = rsGroupInfoManager.getRSGroup(name);
 423       if (rsGroupInfo == null) {
 424         throw new ConstraintException("RSGroup " + name + " does not exist");
 425       }
 426       int tableCount = rsGroupInfo.getTables().size();
 427       if (tableCount > 0) {
 428         throw new ConstraintException("RSGroup " + name + " has " + tableCount +
 429             " tables; you must remove these tables from the rsgroup before " +
 430             "the rsgroup can be removed.");
 431       }
 432       int serverCount = rsGroupInfo.getServers().size();
 433       if (serverCount > 0) {
 434         throw new ConstraintException("RSGroup " + name + " has " + serverCount +
 435             " servers; you must remove these servers from the RSGroup before" +
 436             "the RSGroup can be removed.");
 437       }
 438       for (NamespaceDescriptor ns : master.getClusterSchema().getNamespaces()) {
 439         String nsGroup = ns.getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP);
 440         if (nsGroup != null && nsGroup.equals(name)) {
 441           throw new ConstraintException(
 442             "RSGroup " + name + " is referenced by namespace: " + ns.getName());
 443         }
 444       }
 445       rsGroupInfoManager.removeRSGroup(name);
 446     }
 447   }
 448
 449   @Override
 450   public boolean balanceRSGroup(String groupName) throws IOException {
 451     ServerManager serverManager = master.getServerManager();
 452     LoadBalancer balancer = master.getLoadBalancer();
 453
 454     synchronized (balancer) {
 455       // If balance not true, don't run balancer.
 456       if (!((HMaster) master).isBalancerOn()) {
 457         return false;
 458       }
 459
 460       if (getRSGroupInfo(groupName) == null) {
 461         throw new ConstraintException("RSGroup does not exist: "+groupName);
 462       }
 463       // Only allow one balance run at at time.
 464       Map<String, RegionState> groupRIT = rsGroupGetRegionsInTransition(groupName);
 465       if (groupRIT.size() > 0) {
 466         LOG.debug("Not running balancer because {} region(s) in transition: {}", groupRIT.size(),
 467             StringUtils.abbreviate(
 468               master.getAssignmentManager().getRegionStates().getRegionsInTransition().toString(),
 469               256));
 470         return false;
 471       }
 472       if (serverManager.areDeadServersInProgress()) {
 473         LOG.debug("Not running balancer because processing dead regionserver(s): {}",
 474             serverManager.getDeadServers());
 475         return false;
 476       }
 477
 478       //We balance per group instead of per table
 479       List<RegionPlan> plans = new ArrayList<>();
 480       for(Map.Entry<TableName, Map<ServerName, List<RegionInfo>>> tableMap:
 481           getRSGroupAssignmentsByTable(groupName).entrySet()) {
 482         LOG.info("Creating partial plan for table {} : {}", tableMap.getKey(), tableMap.getValue());
 483         List<RegionPlan> partialPlans = balancer.balanceCluster(tableMap.getValue());
 484         LOG.info("Partial plan for table {} : {}", tableMap.getKey(), partialPlans);
 485         if (partialPlans != null) {
 486           plans.addAll(partialPlans);
 487         }
 488       }
 489       boolean balancerRan = !plans.isEmpty();
 490       if (balancerRan) {
 491         LOG.info("RSGroup balance {} starting with plan count: {}", groupName, plans.size());
 492         master.executeRegionPlansWithThrottling(plans);
 493         LOG.info("RSGroup balance " + groupName + " completed");
 494       }
 495       return balancerRan;
 496     }
 497   }
 498
 499   @Override
 500   public List<RSGroupInfo> listRSGroups() throws IOException {
 501     return rsGroupInfoManager.listRSGroups();
 502   }
 503
 504   @Override
 505   public RSGroupInfo getRSGroupOfServer(Address hostPort) throws IOException {
 506     return rsGroupInfoManager.getRSGroupOfServer(hostPort);
 507   }
 508
 509   @Override
 510   public void moveServersAndTables(Set<Address> servers, Set<TableName> tables, String targetGroup)
 511       throws IOException {
 512     if (servers == null || servers.isEmpty()) {
 513       throw new ConstraintException("The list of servers to move cannot be null or empty.");
 514     }
 515     if (tables == null || tables.isEmpty()) {
 516       throw new ConstraintException("The list of tables to move cannot be null or empty.");
 517     }
 518
 519     //check target group
 520     getAndCheckRSGroupInfo(targetGroup);
 521
 522     // Hold a lock on the manager instance while moving servers and tables to prevent
 523     // another writer changing our state while we are working.
 524     synchronized (rsGroupInfoManager) {
 525       //check servers and tables status
 526       checkServersAndTables(servers, tables, targetGroup);
 527
 528       //Move servers and tables to a new group.
 529       String srcGroup = getRSGroupOfServer(servers.iterator().next()).getName();
 530       rsGroupInfoManager.moveServersAndTables(servers, tables, srcGroup, targetGroup);
 531
 532       //move regions on these servers which do not belong to group tables
 533       moveServerRegionsFromGroup(servers, targetGroup);
 534       //move regions of these tables which are not on group servers
 535       moveTableRegionsToGroup(tables, targetGroup);
 536     }
 537     LOG.info("Move servers and tables done. Severs: {}, Tables: {} => {}", servers, tables,
 538         targetGroup);
 539   }
 540
 541   @Override
 542   public void removeServers(Set<Address> servers) throws IOException {
 543     {
 544       if (servers == null || servers.isEmpty()) {
 545         throw new ConstraintException("The set of servers to remove cannot be null or empty.");
 546       }
 547       // Hold a lock on the manager instance while moving servers to prevent
 548       // another writer changing our state while we are working.
 549       synchronized (rsGroupInfoManager) {
 550         //check the set of servers
 551         checkForDeadOrOnlineServers(servers);
 552         rsGroupInfoManager.removeServers(servers);
 553         LOG.info("Remove decommissioned servers {} from RSGroup done", servers);
 554       }
 555     }
 556   }
 557
 558   private Map<String, RegionState> rsGroupGetRegionsInTransition(String groupName)
 559       throws IOException {
 560     Map<String, RegionState> rit = Maps.newTreeMap();
 561     AssignmentManager am = master.getAssignmentManager();
 562     for(TableName tableName : getRSGroupInfo(groupName).getTables()) {
 563       for(RegionInfo regionInfo: am.getRegionStates().getRegionsOfTable(tableName)) {
 564         RegionState state = am.getRegionStates().getRegionTransitionState(regionInfo);
 565         if(state != null) {
 566           rit.put(regionInfo.getEncodedName(), state);
 567         }
 568       }
 569     }
 570     return rit;
 571   }
 572
 573   private Map<TableName, Map<ServerName, List<RegionInfo>>>
 574       getRSGroupAssignmentsByTable(String groupName) throws IOException {
 575     Map<TableName, Map<ServerName, List<RegionInfo>>> result = Maps.newHashMap();
 576     RSGroupInfo rsGroupInfo = getRSGroupInfo(groupName);
 577     Map<TableName, Map<ServerName, List<RegionInfo>>> assignments = Maps.newHashMap();
 578     for(Map.Entry<RegionInfo, ServerName> entry:
 579         master.getAssignmentManager().getRegionStates().getRegionAssignments().entrySet()) {
 580       TableName currTable = entry.getKey().getTable();
 581       ServerName currServer = entry.getValue();
 582       RegionInfo currRegion = entry.getKey();
 583       if (rsGroupInfo.getTables().contains(currTable)) {
 584         assignments.putIfAbsent(currTable, new HashMap<>());
 585         assignments.get(currTable).putIfAbsent(currServer, new ArrayList<>());
 586         assignments.get(currTable).get(currServer).add(currRegion);
 587       }
 588     }
 589
 590     Map<ServerName, List<RegionInfo>> serverMap = Maps.newHashMap();
 591     for(ServerName serverName: master.getServerManager().getOnlineServers().keySet()) {
 592       if(rsGroupInfo.getServers().contains(serverName.getAddress())) {
 593         serverMap.put(serverName, Collections.emptyList());
 594       }
 595     }
 596
 597     // add all tables that are members of the group
 598     for(TableName tableName : rsGroupInfo.getTables()) {
 599       if(assignments.containsKey(tableName)) {
 600         result.put(tableName, new HashMap<>());
 601         result.get(tableName).putAll(serverMap);
 602         result.get(tableName).putAll(assignments.get(tableName));
 603         LOG.debug("Adding assignments for {}: {}", tableName, assignments.get(tableName));
 604       }
 605     }
 606
 607     return result;
 608   }
 609
 610   /**
 611    * Check if the set of servers are belong to dead servers list or online servers list.
 612    * @param servers servers to remove
 613    */
 614   private void checkForDeadOrOnlineServers(Set<Address> servers) throws ConstraintException {
 615     // This uglyness is because we only have Address, not ServerName.
 616     Set<Address> onlineServers = new HashSet<>();
 617     List<ServerName> drainingServers = master.getServerManager().getDrainingServersList();
 618     for (ServerName server : master.getServerManager().getOnlineServers().keySet()) {
 619       // Only online but not decommissioned servers are really online
 620       if (!drainingServers.contains(server)) {
 621         onlineServers.add(server.getAddress());
 622       }
 623     }
 624
 625     Set<Address> deadServers = new HashSet<>();
 626     for(ServerName server: master.getServerManager().getDeadServers().copyServerNames()) {
 627       deadServers.add(server.getAddress());
 628     }
 629
 630     for (Address address: servers) {
 631       if (onlineServers.contains(address)) {
 632         throw new ConstraintException(
 633             "Server " + address + " is an online server, not allowed to remove.");
 634       }
 635       if (deadServers.contains(address)) {
 636         throw new ConstraintException(
 637             "Server " + address + " is on the dead servers list,"
 638                 + " Maybe it will come back again, not allowed to remove.");
 639       }
 640     }
 641   }
 642 }