hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.rsgroup;
  19
  20 import java.io.IOException;
  21 import java.util.ArrayList;
  22 import java.util.Collections;
  23 import java.util.HashMap;
  24 import java.util.HashSet;
  25 import java.util.Iterator;
  26 import java.util.LinkedList;
  27 import java.util.List;
  28 import java.util.Map;
  29 import java.util.Set;
  30
  31 import org.apache.commons.lang3.StringUtils;
  32 import org.apache.hadoop.hbase.NamespaceDescriptor;
  33 import org.apache.hadoop.hbase.ServerName;
  34 import org.apache.hadoop.hbase.TableName;
  35 import org.apache.hadoop.hbase.client.RegionInfo;
  36 import org.apache.hadoop.hbase.constraint.ConstraintException;
  37 import org.apache.hadoop.hbase.master.HMaster;
  38 import org.apache.hadoop.hbase.master.LoadBalancer;
  39 import org.apache.hadoop.hbase.master.MasterServices;
  40 import org.apache.hadoop.hbase.master.RegionPlan;
  41 import org.apache.hadoop.hbase.master.RegionState;
  42 import org.apache.hadoop.hbase.master.ServerManager;
  43 import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
  44 import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
  45 import org.apache.hadoop.hbase.net.Address;
  46 import org.apache.yetus.audience.InterfaceAudience;
  47 import org.slf4j.Logger;
  48 import org.slf4j.LoggerFactory;
  49
  50 import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
  51 import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
  52
  53 /**
  54  * Service to support Region Server Grouping (HBase-6721).
  55  */
  56 @InterfaceAudience.Private
  57 public class RSGroupAdminServer implements RSGroupAdmin {
  58   private static final Logger LOG = LoggerFactory.getLogger(RSGroupAdminServer.class);
  59   public static final String KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE = "should keep at least " +
  60           "one server in 'default' RSGroup.";
  61
  62   private MasterServices master;
  63   private final RSGroupInfoManager rsGroupInfoManager;
  64
  65   public RSGroupAdminServer(MasterServices master, RSGroupInfoManager rsGroupInfoManager)
  66       throws IOException {
  67     this.master = master;
  68     this.rsGroupInfoManager = rsGroupInfoManager;
  69   }
  70
  71   @Override
  72   public RSGroupInfo getRSGroupInfo(String groupName) throws IOException {
  73     return rsGroupInfoManager.getRSGroup(groupName);
  74   }
  75
  76   @Override
  77   public RSGroupInfo getRSGroupInfoOfTable(TableName tableName) throws IOException {
  78     // We are reading across two Maps in the below with out synchronizing across
  79     // them; should be safe most of the time.
  80     String groupName = rsGroupInfoManager.getRSGroupOfTable(tableName);
  81     return groupName == null? null: rsGroupInfoManager.getRSGroup(groupName);
  82   }
  83
  84   private void checkOnlineServersOnly(Set<Address> servers) throws ConstraintException {
  85     // This uglyness is because we only have Address, not ServerName.
  86     // Online servers are keyed by ServerName.
  87     Set<Address> onlineServers = new HashSet<>();
  88     for(ServerName server: master.getServerManager().getOnlineServers().keySet()) {
  89       onlineServers.add(server.getAddress());
  90     }
  91     for (Address address: servers) {
  92       if (!onlineServers.contains(address)) {
  93         throw new ConstraintException(
  94             "Server " + address + " is not an online server in 'default' RSGroup.");
  95       }
  96     }
  97   }
  98
  99   /**
 100    * Check passed name. Fail if nulls or if corresponding RSGroupInfo not found.
 101    * @return The RSGroupInfo named <code>name</code>
 102    */
 103   private RSGroupInfo getAndCheckRSGroupInfo(String name)
 104   throws IOException {
 105     if (StringUtils.isEmpty(name)) {
 106       throw new ConstraintException("RSGroup cannot be null.");
 107     }
 108     RSGroupInfo rsGroupInfo = getRSGroupInfo(name);
 109     if (rsGroupInfo == null) {
 110       throw new ConstraintException("RSGroup does not exist: " + name);
 111     }
 112     return rsGroupInfo;
 113   }
 114
 115   /**
 116    * @return List of Regions associated with this <code>server</code>.
 117    */
 118   private List<RegionInfo> getRegions(final Address server) {
 119     LinkedList<RegionInfo> regions = new LinkedList<>();
 120     for (Map.Entry<RegionInfo, ServerName> el :
 121         master.getAssignmentManager().getRegionStates().getRegionAssignments().entrySet()) {
 122       if (el.getValue() == null) {
 123         continue;
 124       }
 125
 126       if (el.getValue().getAddress().equals(server)) {
 127         addRegion(regions, el.getKey());
 128       }
 129     }
 130     for (RegionStateNode state : master.getAssignmentManager().getRegionsInTransition()) {
 131       if (state.getRegionLocation().getAddress().equals(server)) {
 132         addRegion(regions, state.getRegionInfo());
 133       }
 134     }
 135     return regions;
 136   }
 137
 138   private void addRegion(final LinkedList<RegionInfo> regions, RegionInfo hri) {
 139     // If meta, move it last otherwise other unassigns fail because meta is not
 140     // online for them to update state in. This is dodgy. Needs to be made more
 141     // robust. See TODO below.
 142     if (hri.isMetaRegion()) {
 143       regions.addLast(hri);
 144     } else {
 145       regions.addFirst(hri);
 146     }
 147   }
 148
 149   /**
 150    * Check servers and tables.
 151    *
 152    * @param servers servers to move
 153    * @param tables tables to move
 154    * @param targetGroupName target group name
 155    * @throws IOException if nulls or if servers and tables not belong to the same group
 156    */
 157   private void checkServersAndTables(Set<Address> servers, Set<TableName> tables,
 158                                      String targetGroupName) throws IOException {
 159     // Presume first server's source group. Later ensure all servers are from this group.
 160     Address firstServer = servers.iterator().next();
 161     RSGroupInfo tmpSrcGrp = rsGroupInfoManager.getRSGroupOfServer(firstServer);
 162     if (tmpSrcGrp == null) {
 163       // Be careful. This exception message is tested for in TestRSGroupsBase...
 164       throw new ConstraintException("Source RSGroup for server " + firstServer
 165               + " does not exist.");
 166     }
 167     RSGroupInfo srcGrp = new RSGroupInfo(tmpSrcGrp);
 168     if (srcGrp.getName().equals(targetGroupName)) {
 169       throw new ConstraintException("Target RSGroup " + targetGroupName +
 170               " is same as source " + srcGrp.getName() + " RSGroup.");
 171     }
 172     // Only move online servers
 173     checkOnlineServersOnly(servers);
 174
 175     // Ensure all servers are of same rsgroup.
 176     for (Address server: servers) {
 177       String tmpGroup = rsGroupInfoManager.getRSGroupOfServer(server).getName();
 178       if (!tmpGroup.equals(srcGrp.getName())) {
 179         throw new ConstraintException("Move server request should only come from one source " +
 180                 "RSGroup. Expecting only " + srcGrp.getName() + " but contains " + tmpGroup);
 181       }
 182     }
 183
 184     // Ensure all tables and servers are of same rsgroup.
 185     for (TableName table : tables) {
 186       String tmpGroup = rsGroupInfoManager.getRSGroupOfTable(table);
 187       if (!tmpGroup.equals(srcGrp.getName())) {
 188         throw new ConstraintException("Move table request should only come from one source " +
 189                 "RSGroup. Expecting only " + srcGrp.getName() + " but contains " + tmpGroup);
 190       }
 191     }
 192
 193     if (srcGrp.getServers().size() <= servers.size() && srcGrp.getTables().size() > tables.size()) {
 194       throw new ConstraintException("Cannot leave a RSGroup " + srcGrp.getName() +
 195               " that contains tables without servers to host them.");
 196     }
 197   }
 198
 199   /**
 200    * Moves every region from servers which are currently located on these servers,
 201    * but should not be located there.
 202    * @param servers the servers that will move to new group
 203    * @param tables these tables will be kept on the servers, others will be moved
 204    * @param targetGroupName the target group name
 205    * @throws IOException if moving the server and tables fail
 206    */
 207   private void moveRegionsFromServers(Set<Address> servers, Set<TableName> tables,
 208       String targetGroupName) throws IOException {
 209     boolean foundRegionsToMove;
 210     RSGroupInfo targetGrp = getRSGroupInfo(targetGroupName);
 211     Set<Address> allSevers = new HashSet<>(servers);
 212     do {
 213       foundRegionsToMove = false;
 214       for (Iterator<Address> iter = allSevers.iterator(); iter.hasNext();) {
 215         Address rs = iter.next();
 216         // Get regions that are associated with this server and filter regions by tables.
 217         List<RegionInfo> regions = new ArrayList<>();
 218         for (RegionInfo region : getRegions(rs)) {
 219           if (!tables.contains(region.getTable())) {
 220             regions.add(region);
 221           }
 222         }
 223
 224         LOG.info("Moving " + regions.size() + " region(s) from " + rs +
 225             " for server move to " + targetGroupName);
 226         if (!regions.isEmpty()) {
 227           for (RegionInfo region: regions) {
 228             // Regions might get assigned from tables of target group so we need to filter
 229             if (!targetGrp.containsTable(region.getTable())) {
 230               this.master.getAssignmentManager().move(region);
 231               if (master.getAssignmentManager().getRegionStates().
 232                   getRegionState(region).isFailedOpen()) {
 233                 continue;
 234               }
 235               foundRegionsToMove = true;
 236             }
 237           }
 238         }
 239         if (!foundRegionsToMove) {
 240           iter.remove();
 241         }
 242       }
 243       try {
 244         rsGroupInfoManager.wait(1000);
 245       } catch (InterruptedException e) {
 246         LOG.warn("Sleep interrupted", e);
 247         Thread.currentThread().interrupt();
 248       }
 249     } while (foundRegionsToMove);
 250   }
 251
 252   /**
 253    * Moves every region of tables which should be kept on the servers,
 254    * but currently they are located on other servers.
 255    * @param servers the regions of these servers will be kept on the servers, others will be moved
 256    * @param tables the tables that will move to new group
 257    * @param targetGroupName the target group name
 258    * @throws IOException if moving the region fails
 259    */
 260   private void moveRegionsToServers(Set<Address> servers, Set<TableName> tables,
 261       String targetGroupName) throws IOException {
 262     for (TableName table: tables) {
 263       LOG.info("Moving region(s) from " + table + " for table move to " + targetGroupName);
 264       for (RegionInfo region : master.getAssignmentManager().getRegionStates()
 265               .getRegionsOfTable(table)) {
 266         ServerName sn = master.getAssignmentManager().getRegionStates()
 267                 .getRegionServerOfRegion(region);
 268         if (!servers.contains(sn.getAddress())) {
 269           master.getAssignmentManager().move(region);
 270         }
 271       }
 272     }
 273   }
 274
 275   @edu.umd.cs.findbugs.annotations.SuppressWarnings(
 276       value="RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE",
 277       justification="Ignoring complaint because don't know what it is complaining about")
 278   @Override
 279   public void moveServers(Set<Address> servers, String targetGroupName)
 280   throws IOException {
 281     if (servers == null) {
 282       throw new ConstraintException("The list of servers to move cannot be null.");
 283     }
 284     if (servers.isEmpty()) {
 285       // For some reason this difference between null servers and isEmpty is important distinction.
 286       // TODO. Why? Stuff breaks if I equate them.
 287       return;
 288     }
 289     RSGroupInfo targetGrp = getAndCheckRSGroupInfo(targetGroupName);
 290
 291     // Hold a lock on the manager instance while moving servers to prevent
 292     // another writer changing our state while we are working.
 293     synchronized (rsGroupInfoManager) {
 294       // Presume first server's source group. Later ensure all servers are from this group.
 295       Address firstServer = servers.iterator().next();
 296       RSGroupInfo srcGrp = rsGroupInfoManager.getRSGroupOfServer(firstServer);
 297       if (srcGrp == null) {
 298         // Be careful. This exception message is tested for in TestRSGroupsBase...
 299         throw new ConstraintException("Source RSGroup for server " + firstServer
 300             + " does not exist.");
 301       }
 302       if (srcGrp.getName().equals(targetGroupName)) {
 303         throw new ConstraintException("Target RSGroup " + targetGroupName +
 304             " is same as source " + srcGrp + " RSGroup.");
 305       }
 306       // Only move online servers (when moving from 'default') or servers from other
 307       // groups. This prevents bogus servers from entering groups
 308       if (RSGroupInfo.DEFAULT_GROUP.equals(srcGrp.getName())) {
 309         if (srcGrp.getServers().size() <= servers.size()) {
 310           throw new ConstraintException(KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE);
 311         }
 312         checkOnlineServersOnly(servers);
 313       }
 314       // Ensure all servers are of same rsgroup.
 315       for (Address server: servers) {
 316         String tmpGroup = rsGroupInfoManager.getRSGroupOfServer(server).getName();
 317         if (!tmpGroup.equals(srcGrp.getName())) {
 318           throw new ConstraintException("Move server request should only come from one source " +
 319               "RSGroup. Expecting only " + srcGrp.getName() + " but contains " + tmpGroup);
 320         }
 321       }
 322       if (srcGrp.getServers().size() <= servers.size() && srcGrp.getTables().size() > 0) {
 323         throw new ConstraintException("Cannot leave a RSGroup " + srcGrp.getName() +
 324             " that contains tables without servers to host them.");
 325       }
 326
 327       // MovedServers may be < passed in 'servers'.
 328       Set<Address> movedServers = rsGroupInfoManager.moveServers(servers, srcGrp.getName(),
 329           targetGroupName);
 330       List<Address> editableMovedServers = Lists.newArrayList(movedServers);
 331       boolean foundRegionsToMove;
 332       do {
 333         foundRegionsToMove = false;
 334         for (Iterator<Address> iter = editableMovedServers.iterator(); iter.hasNext();) {
 335           Address rs = iter.next();
 336           // Get regions that are associated with this server.
 337           List<RegionInfo> regions = getRegions(rs);
 338
 339           LOG.info("Moving " + regions.size() + " region(s) from " + rs +
 340               " for server move to " + targetGroupName);
 341
 342           for (RegionInfo region: regions) {
 343             // Regions might get assigned from tables of target group so we need to filter
 344             if (targetGrp.containsTable(region.getTable())) {
 345               continue;
 346             }
 347             LOG.info("Moving region " + region.getShortNameToLog());
 348             this.master.getAssignmentManager().move(region);
 349             if (master.getAssignmentManager().getRegionStates().
 350                 getRegionState(region).isFailedOpen()) {
 351               // If region is in FAILED_OPEN state, it won't recover, not without
 352               // operator intervention... in hbase-2.0.0 at least. Continue rather
 353               // than mark region as 'foundRegionsToMove'.
 354               continue;
 355             }
 356             foundRegionsToMove = true;
 357           }
 358           if (!foundRegionsToMove) {
 359             iter.remove();
 360           }
 361         }
 362         try {
 363           rsGroupInfoManager.wait(1000);
 364         } catch (InterruptedException e) {
 365           LOG.warn("Sleep interrupted", e);
 366           Thread.currentThread().interrupt();
 367         }
 368       } while (foundRegionsToMove);
 369
 370       LOG.info("Move server done: " + srcGrp.getName() + "=>" + targetGroupName);
 371     }
 372   }
 373
 374   @Override
 375   public void moveTables(Set<TableName> tables, String targetGroup) throws IOException {
 376     if (tables == null) {
 377       throw new ConstraintException("The list of servers cannot be null.");
 378     }
 379     if (tables.size() < 1) {
 380       LOG.debug("moveTables() passed an empty set. Ignoring.");
 381       return;
 382     }
 383
 384     // Hold a lock on the manager instance while moving servers to prevent
 385     // another writer changing our state while we are working.
 386     synchronized (rsGroupInfoManager) {
 387       if(targetGroup != null) {
 388         RSGroupInfo destGroup = rsGroupInfoManager.getRSGroup(targetGroup);
 389         if(destGroup == null) {
 390           throw new ConstraintException("Target " + targetGroup + " RSGroup does not exist.");
 391         }
 392         if(destGroup.getServers().size() < 1) {
 393           throw new ConstraintException("Target RSGroup must have at least one server.");
 394         }
 395       }
 396
 397       for (TableName table : tables) {
 398         String srcGroup = rsGroupInfoManager.getRSGroupOfTable(table);
 399         if(srcGroup != null && srcGroup.equals(targetGroup)) {
 400           throw new ConstraintException(
 401               "Source RSGroup " + srcGroup + " is same as target " + targetGroup +
 402               " RSGroup for table " + table);
 403         }
 404         LOG.info("Moving table " + table.getNameAsString() + " to RSGroup " + targetGroup);
 405       }
 406       rsGroupInfoManager.moveTables(tables, targetGroup);
 407
 408       // targetGroup is null when a table is being deleted. In this case no further
 409       // action is required.
 410       if (targetGroup != null) {
 411         for (TableName table: tables) {
 412           if (master.getAssignmentManager().isTableDisabled(table)) {
 413             LOG.debug("Skipping move regions because the table" + table + " is disabled.");
 414             continue;
 415           }
 416           for (RegionInfo region :
 417               master.getAssignmentManager().getRegionStates().getRegionsOfTable(table)) {
 418             LOG.info("Moving region " + region.getShortNameToLog() +
 419                 " to RSGroup " + targetGroup);
 420             master.getAssignmentManager().move(region);
 421           }
 422         }
 423       }
 424     }
 425   }
 426
 427   @Override
 428   public void addRSGroup(String name) throws IOException {
 429     rsGroupInfoManager.addRSGroup(new RSGroupInfo(name));
 430   }
 431
 432   @Override
 433   public void removeRSGroup(String name) throws IOException {
 434     // Hold a lock on the manager instance while moving servers to prevent
 435     // another writer changing our state while we are working.
 436     synchronized (rsGroupInfoManager) {
 437       RSGroupInfo rsGroupInfo = rsGroupInfoManager.getRSGroup(name);
 438       if (rsGroupInfo == null) {
 439         throw new ConstraintException("RSGroup " + name + " does not exist");
 440       }
 441       int tableCount = rsGroupInfo.getTables().size();
 442       if (tableCount > 0) {
 443         throw new ConstraintException("RSGroup " + name + " has " + tableCount +
 444             " tables; you must remove these tables from the rsgroup before " +
 445             "the rsgroup can be removed.");
 446       }
 447       int serverCount = rsGroupInfo.getServers().size();
 448       if (serverCount > 0) {
 449         throw new ConstraintException("RSGroup " + name + " has " + serverCount +
 450             " servers; you must remove these servers from the RSGroup before" +
 451             "the RSGroup can be removed.");
 452       }
 453       for (NamespaceDescriptor ns: master.getClusterSchema().getNamespaces()) {
 454         String nsGroup = ns.getConfigurationValue(rsGroupInfo.NAMESPACE_DESC_PROP_GROUP);
 455         if (nsGroup != null &&  nsGroup.equals(name)) {
 456           throw new ConstraintException("RSGroup " + name + " is referenced by namespace: " +
 457               ns.getName());
 458         }
 459       }
 460       rsGroupInfoManager.removeRSGroup(name);
 461     }
 462   }
 463
 464   @Override
 465   public boolean balanceRSGroup(String groupName) throws IOException {
 466     ServerManager serverManager = master.getServerManager();
 467     AssignmentManager assignmentManager = master.getAssignmentManager();
 468     LoadBalancer balancer = master.getLoadBalancer();
 469
 470     synchronized (balancer) {
 471       // If balance not true, don't run balancer.
 472       if (!((HMaster) master).isBalancerOn()) {
 473         return false;
 474       }
 475
 476       if (getRSGroupInfo(groupName) == null) {
 477         throw new ConstraintException("RSGroup does not exist: "+groupName);
 478       }
 479       // Only allow one balance run at at time.
 480       Map<String, RegionState> groupRIT = rsGroupGetRegionsInTransition(groupName);
 481       if (groupRIT.size() > 0) {
 482         LOG.debug("Not running balancer because " + groupRIT.size() + " region(s) in transition: " +
 483           StringUtils.abbreviate(
 484               master.getAssignmentManager().getRegionStates().getRegionsInTransition().toString(),
 485               256));
 486         return false;
 487       }
 488       if (serverManager.areDeadServersInProgress()) {
 489         LOG.debug("Not running balancer because processing dead regionserver(s): " +
 490             serverManager.getDeadServers());
 491         return false;
 492       }
 493
 494       //We balance per group instead of per table
 495       List<RegionPlan> plans = new ArrayList<>();
 496       for(Map.Entry<TableName, Map<ServerName, List<RegionInfo>>> tableMap:
 497           getRSGroupAssignmentsByTable(groupName).entrySet()) {
 498         LOG.info("Creating partial plan for table " + tableMap.getKey() + ": "
 499             + tableMap.getValue());
 500         List<RegionPlan> partialPlans = balancer.balanceCluster(tableMap.getValue());
 501         LOG.info("Partial plan for table " + tableMap.getKey() + ": " + partialPlans);
 502         if (partialPlans != null) {
 503           plans.addAll(partialPlans);
 504         }
 505       }
 506       long startTime = System.currentTimeMillis();
 507       boolean balancerRan = !plans.isEmpty();
 508       if (balancerRan) {
 509         LOG.info("RSGroup balance " + groupName + " starting with plan count: " + plans.size());
 510         for (RegionPlan plan: plans) {
 511           LOG.info("balance " + plan);
 512           assignmentManager.moveAsync(plan);
 513         }
 514         LOG.info("RSGroup balance " + groupName + " completed after " +
 515             (System.currentTimeMillis()-startTime) + " seconds");
 516       }
 517       return balancerRan;
 518     }
 519   }
 520
 521   @Override
 522   public List<RSGroupInfo> listRSGroups() throws IOException {
 523     return rsGroupInfoManager.listRSGroups();
 524   }
 525
 526   @Override
 527   public RSGroupInfo getRSGroupOfServer(Address hostPort) throws IOException {
 528     return rsGroupInfoManager.getRSGroupOfServer(hostPort);
 529   }
 530
 531   @Override
 532   public void moveServersAndTables(Set<Address> servers, Set<TableName> tables, String targetGroup)
 533       throws IOException {
 534     if (servers == null || servers.isEmpty()) {
 535       throw new ConstraintException("The list of servers to move cannot be null or empty.");
 536     }
 537     if (tables == null || tables.isEmpty()) {
 538       throw new ConstraintException("The list of tables to move cannot be null or empty.");
 539     }
 540
 541     //check target group
 542     getAndCheckRSGroupInfo(targetGroup);
 543
 544     // Hold a lock on the manager instance while moving servers and tables to prevent
 545     // another writer changing our state while we are working.
 546     synchronized (rsGroupInfoManager) {
 547       //check servers and tables status
 548       checkServersAndTables(servers, tables, targetGroup);
 549
 550       //Move servers and tables to a new group.
 551       String srcGroup = getRSGroupOfServer(servers.iterator().next()).getName();
 552       rsGroupInfoManager.moveServersAndTables(servers, tables, srcGroup, targetGroup);
 553
 554       //move regions which should not belong to these tables
 555       moveRegionsFromServers(servers, tables, targetGroup);
 556       //move regions which should belong to these servers
 557       moveRegionsToServers(servers, tables, targetGroup);
 558     }
 559     LOG.info("Move servers and tables done. Severs :"
 560             + servers + " , Tables : " + tables + " => " +  targetGroup);
 561   }
 562
 563   @Override
 564   public void removeServers(Set<Address> servers) throws IOException {
 565     {
 566       if (servers == null || servers.isEmpty()) {
 567         throw new ConstraintException("The set of servers to remove cannot be null or empty.");
 568       }
 569       // Hold a lock on the manager instance while moving servers to prevent
 570       // another writer changing our state while we are working.
 571       synchronized (rsGroupInfoManager) {
 572         //check the set of servers
 573         checkForDeadOrOnlineServers(servers);
 574         rsGroupInfoManager.removeServers(servers);
 575         LOG.info("Remove decommissioned servers " + servers + " from rsgroup done.");
 576       }
 577     }
 578   }
 579
 580   private Map<String, RegionState> rsGroupGetRegionsInTransition(String groupName)
 581       throws IOException {
 582     Map<String, RegionState> rit = Maps.newTreeMap();
 583     AssignmentManager am = master.getAssignmentManager();
 584     for(TableName tableName : getRSGroupInfo(groupName).getTables()) {
 585       for(RegionInfo regionInfo: am.getRegionStates().getRegionsOfTable(tableName)) {
 586         RegionState state = am.getRegionStates().getRegionTransitionState(regionInfo);
 587         if(state != null) {
 588           rit.put(regionInfo.getEncodedName(), state);
 589         }
 590       }
 591     }
 592     return rit;
 593   }
 594
 595   private Map<TableName, Map<ServerName, List<RegionInfo>>>
 596       getRSGroupAssignmentsByTable(String groupName) throws IOException {
 597     Map<TableName, Map<ServerName, List<RegionInfo>>> result = Maps.newHashMap();
 598     RSGroupInfo rsGroupInfo = getRSGroupInfo(groupName);
 599     Map<TableName, Map<ServerName, List<RegionInfo>>> assignments = Maps.newHashMap();
 600     for(Map.Entry<RegionInfo, ServerName> entry:
 601         master.getAssignmentManager().getRegionStates().getRegionAssignments().entrySet()) {
 602       TableName currTable = entry.getKey().getTable();
 603       ServerName currServer = entry.getValue();
 604       RegionInfo currRegion = entry.getKey();
 605       if (rsGroupInfo.getTables().contains(currTable)) {
 606         assignments.putIfAbsent(currTable, new HashMap<>());
 607         assignments.get(currTable).putIfAbsent(currServer, new ArrayList<>());
 608         assignments.get(currTable).get(currServer).add(currRegion);
 609       }
 610     }
 611
 612     Map<ServerName, List<RegionInfo>> serverMap = Maps.newHashMap();
 613     for(ServerName serverName: master.getServerManager().getOnlineServers().keySet()) {
 614       if(rsGroupInfo.getServers().contains(serverName.getAddress())) {
 615         serverMap.put(serverName, Collections.emptyList());
 616       }
 617     }
 618
 619     // add all tables that are members of the group
 620     for(TableName tableName : rsGroupInfo.getTables()) {
 621       if(assignments.containsKey(tableName)) {
 622         result.put(tableName, new HashMap<>());
 623         result.get(tableName).putAll(serverMap);
 624         result.get(tableName).putAll(assignments.get(tableName));
 625         LOG.debug("Adding assignments for " + tableName + ": " + assignments.get(tableName));
 626       }
 627     }
 628
 629     return result;
 630   }
 631
 632   /**
 633    * Check if the set of servers are belong to dead servers list or online servers list.
 634    * @param servers servers to remove
 635    */
 636   private void checkForDeadOrOnlineServers(Set<Address> servers) throws ConstraintException {
 637     // This uglyness is because we only have Address, not ServerName.
 638     Set<Address> onlineServers = new HashSet<>();
 639     List<ServerName> drainingServers = master.getServerManager().getDrainingServersList();
 640     for (ServerName server : master.getServerManager().getOnlineServers().keySet()) {
 641       // Only online but not decommissioned servers are really online
 642       if (!drainingServers.contains(server)) {
 643         onlineServers.add(server.getAddress());
 644       }
 645     }
 646
 647     Set<Address> deadServers = new HashSet<>();
 648     for(ServerName server: master.getServerManager().getDeadServers().copyServerNames()) {
 649       deadServers.add(server.getAddress());
 650     }
 651
 652     for (Address address: servers) {
 653       if (onlineServers.contains(address)) {
 654         throw new ConstraintException(
 655             "Server " + address + " is an online server, not allowed to remove.");
 656       }
 657       if (deadServers.contains(address)) {
 658         throw new ConstraintException(
 659             "Server " + address + " is on the dead servers list,"
 660                 + " Maybe it will come back again, not allowed to remove.");
 661       }
 662     }
 663   }
 664 }