hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin1.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.rsgroup;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertFalse;
  22 import static org.junit.Assert.assertNull;
  23 import static org.junit.Assert.assertTrue;
  24 import static org.junit.Assert.fail;
  25
  26 import java.io.IOException;
  27 import java.util.List;
  28 import java.util.Map;
  29 import java.util.Set;
  30 import org.apache.hadoop.hbase.HBaseClassTestRule;
  31 import org.apache.hadoop.hbase.NamespaceDescriptor;
  32 import org.apache.hadoop.hbase.ServerName;
  33 import org.apache.hadoop.hbase.TableExistsException;
  34 import org.apache.hadoop.hbase.TableName;
  35 import org.apache.hadoop.hbase.TableNotFoundException;
  36 import org.apache.hadoop.hbase.Waiter;
  37 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
  38 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
  39 import org.apache.hadoop.hbase.client.TableDescriptor;
  40 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
  41 import org.apache.hadoop.hbase.constraint.ConstraintException;
  42 import org.apache.hadoop.hbase.master.TableNamespaceManager;
  43 import org.apache.hadoop.hbase.net.Address;
  44 import org.apache.hadoop.hbase.testclassification.MediumTests;
  45 import org.apache.hadoop.hbase.testclassification.RSGroupTests;
  46 import org.apache.hadoop.hbase.util.Bytes;
  47 import org.junit.After;
  48 import org.junit.AfterClass;
  49 import org.junit.Before;
  50 import org.junit.BeforeClass;
  51 import org.junit.ClassRule;
  52 import org.junit.Test;
  53 import org.junit.experimental.categories.Category;
  54 import org.slf4j.Logger;
  55 import org.slf4j.LoggerFactory;
  56
  57 import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
  58
  59 @Category({ RSGroupTests.class, MediumTests.class })
  60 public class TestRSGroupsAdmin1 extends TestRSGroupsBase {
  61
  62   @ClassRule
  63   public static final HBaseClassTestRule CLASS_RULE =
  64     HBaseClassTestRule.forClass(TestRSGroupsAdmin1.class);
  65
  66   private static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsAdmin1.class);
  67
  68   @BeforeClass
  69   public static void setUp() throws Exception {
  70     setUpTestBeforeClass();
  71   }
  72
  73   @AfterClass
  74   public static void tearDown() throws Exception {
  75     tearDownAfterClass();
  76   }
  77
  78   @Before
  79   public void beforeMethod() throws Exception {
  80     setUpBeforeMethod();
  81   }
  82
  83   @After
  84   public void afterMethod() throws Exception {
  85     tearDownAfterMethod();
  86   }
  87
  88   @Test
  89   public void testValidGroupNames() throws IOException {
  90     String[] badNames = { "foo*", "foo@", "-" };
  91     String[] goodNames = { "foo_123" };
  92
  93     for (String entry : badNames) {
  94       try {
  95         ADMIN.addRSGroup(entry);
  96         fail("Expected a constraint exception for: " + entry);
  97       } catch (ConstraintException ex) {
  98         // expected
  99       }
 100     }
 101
 102     for (String entry : goodNames) {
 103       ADMIN.addRSGroup(entry);
 104     }
 105   }
 106
 107   @Test
 108   public void testBogusArgs() throws Exception {
 109     assertNull(ADMIN.getRSGroup(TableName.valueOf("nonexistent")));
 110     assertNull(ADMIN.getRSGroup(Address.fromParts("bogus", 123)));
 111     assertNull(ADMIN.getRSGroup("bogus"));
 112
 113     try {
 114       ADMIN.removeRSGroup("bogus");
 115       fail("Expected removing bogus group to fail");
 116     } catch (ConstraintException ex) {
 117       // expected
 118     }
 119
 120     try {
 121       ADMIN.setRSGroup(Sets.newHashSet(TableName.valueOf("bogustable")), "bogus");
 122       fail("Expected set table to bogus group fail");
 123     } catch (ConstraintException | TableNotFoundException ex) {
 124       // expected
 125     }
 126
 127     try {
 128       ADMIN.moveServersToRSGroup(Sets.newHashSet(Address.fromParts("bogus", 123)), "bogus");
 129       fail("Expected move with bogus group to fail");
 130     } catch (ConstraintException ex) {
 131       // expected
 132     }
 133
 134     try {
 135       ADMIN.balancerSwitch(true, true);
 136       ADMIN.balanceRSGroup("bogus");
 137       ADMIN.balancerSwitch(false, true);
 138       fail("Expected move with bogus group to fail");
 139     } catch (ConstraintException ex) {
 140       // expected
 141     }
 142   }
 143
 144   @Test
 145   public void testNamespaceConstraint() throws Exception {
 146     String nsName = TABLE_PREFIX + "_foo";
 147     String groupName = TABLE_PREFIX + "_foo";
 148     LOG.info("testNamespaceConstraint");
 149     addGroup(groupName, 1);
 150     assertTrue(OBSERVER.preAddRSGroupCalled);
 151     assertTrue(OBSERVER.postAddRSGroupCalled);
 152
 153     ADMIN.createNamespace(NamespaceDescriptor.create(nsName)
 154       .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, groupName).build());
 155     RSGroupInfo rsGroupInfo = ADMIN.getRSGroup(groupName);
 156     ADMIN.moveServersToRSGroup(rsGroupInfo.getServers(), RSGroupInfo.DEFAULT_GROUP);
 157     // test removing a referenced group
 158     try {
 159       ADMIN.removeRSGroup(groupName);
 160       fail("Expected a constraint exception");
 161     } catch (IOException ex) {
 162     }
 163     // test modify group
 164     // changing with the same name is fine
 165     ADMIN.modifyNamespace(NamespaceDescriptor.create(nsName)
 166       .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, groupName).build());
 167     String anotherGroup = TABLE_PREFIX + "_anotherGroup";
 168     ADMIN.addRSGroup(anotherGroup);
 169     // test add non-existent group
 170     ADMIN.deleteNamespace(nsName);
 171     ADMIN.removeRSGroup(groupName);
 172     assertTrue(OBSERVER.preRemoveRSGroupCalled);
 173     assertTrue(OBSERVER.postRemoveRSGroupCalled);
 174     try {
 175       ADMIN.createNamespace(NamespaceDescriptor.create(nsName)
 176         .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, "foo").build());
 177       fail("Expected a constraint exception");
 178     } catch (IOException ex) {
 179     }
 180   }
 181
 182   @Test
 183   public void testFailRemoveGroup() throws IOException, InterruptedException {
 184     int initNumGroups = ADMIN.listRSGroups().size();
 185     addGroup("bar", 3);
 186     TEST_UTIL.createTable(tableName, Bytes.toBytes("f"));
 187     ADMIN.setRSGroup(Sets.newHashSet(tableName), "bar");
 188     RSGroupInfo barGroup = ADMIN.getRSGroup("bar");
 189     // group is not empty therefore it should fail
 190     try {
 191       ADMIN.removeRSGroup(barGroup.getName());
 192       fail("Expected remove group to fail");
 193     } catch (IOException e) {
 194     }
 195     // group cannot lose all it's servers therefore it should fail
 196     try {
 197       ADMIN.moveServersToRSGroup(barGroup.getServers(), RSGroupInfo.DEFAULT_GROUP);
 198       fail("Expected move servers to fail");
 199     } catch (IOException e) {
 200     }
 201
 202     ADMIN.setRSGroup(Sets.newHashSet(ADMIN.listTablesInRSGroup("bar")), RSGroupInfo.DEFAULT_GROUP);
 203     try {
 204       ADMIN.removeRSGroup(barGroup.getName());
 205       fail("Expected move servers to fail");
 206     } catch (IOException e) {
 207     }
 208
 209     ADMIN.moveServersToRSGroup(barGroup.getServers(), RSGroupInfo.DEFAULT_GROUP);
 210     ADMIN.removeRSGroup(barGroup.getName());
 211
 212     assertEquals(initNumGroups, ADMIN.listRSGroups().size());
 213   }
 214
 215   @Test
 216   public void testMultiTableMove() throws Exception {
 217     final TableName tableNameA =
 218       TableName.valueOf(TABLE_PREFIX + getNameWithoutIndex(name.getMethodName()) + "A");
 219     final TableName tableNameB =
 220       TableName.valueOf(TABLE_PREFIX + getNameWithoutIndex(name.getMethodName()) + "B");
 221     final byte[] familyNameBytes = Bytes.toBytes("f");
 222     String newGroupName = getGroupName(getNameWithoutIndex(name.getMethodName()));
 223     final RSGroupInfo newGroup = addGroup(newGroupName, 1);
 224
 225     TEST_UTIL.createTable(tableNameA, familyNameBytes);
 226     TEST_UTIL.createTable(tableNameB, familyNameBytes);
 227     TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
 228       @Override
 229       public boolean evaluate() throws Exception {
 230         List<String> regionsA = getTableRegionMap().get(tableNameA);
 231         if (regionsA == null) {
 232           return false;
 233         }
 234         List<String> regionsB = getTableRegionMap().get(tableNameB);
 235         if (regionsB == null) {
 236           return false;
 237         }
 238         return getTableRegionMap().get(tableNameA).size() >= 1 &&
 239           getTableRegionMap().get(tableNameB).size() >= 1;
 240       }
 241     });
 242
 243     RSGroupInfo tableGrpA = ADMIN.getRSGroup(tableNameA);
 244     assertTrue(tableGrpA.getName().equals(RSGroupInfo.DEFAULT_GROUP));
 245
 246     RSGroupInfo tableGrpB = ADMIN.getRSGroup(tableNameB);
 247     assertTrue(tableGrpB.getName().equals(RSGroupInfo.DEFAULT_GROUP));
 248     // change table's group
 249     LOG.info("Moving table [" + tableNameA + "," + tableNameB + "] to " + newGroup.getName());
 250     ADMIN.setRSGroup(Sets.newHashSet(tableNameA, tableNameB), newGroup.getName());
 251
 252     // verify group change
 253     assertEquals(newGroup.getName(), ADMIN.getRSGroup(tableNameA).getName());
 254
 255     assertEquals(newGroup.getName(), ADMIN.getRSGroup(tableNameB).getName());
 256
 257     // verify tables' not exist in old group
 258     Set<TableName> defaultTables =
 259       Sets.newHashSet(ADMIN.listTablesInRSGroup(RSGroupInfo.DEFAULT_GROUP));
 260     assertFalse(defaultTables.contains(tableNameA));
 261     assertFalse(defaultTables.contains(tableNameB));
 262
 263     // verify tables' exist in new group
 264     Set<TableName> newGroupTables = Sets.newHashSet(ADMIN.listTablesInRSGroup(newGroupName));
 265     assertTrue(newGroupTables.contains(tableNameA));
 266     assertTrue(newGroupTables.contains(tableNameB));
 267   }
 268
 269   @Test
 270   public void testTableMoveTruncateAndDrop() throws Exception {
 271     final byte[] familyNameBytes = Bytes.toBytes("f");
 272     String newGroupName = getGroupName(getNameWithoutIndex(name.getMethodName()));
 273     final RSGroupInfo newGroup = addGroup(newGroupName, 2);
 274
 275     TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 5);
 276     TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
 277       @Override
 278       public boolean evaluate() throws Exception {
 279         List<String> regions = getTableRegionMap().get(tableName);
 280         if (regions == null) {
 281           return false;
 282         }
 283
 284         return getTableRegionMap().get(tableName).size() >= 5;
 285       }
 286     });
 287
 288     RSGroupInfo tableGrp = ADMIN.getRSGroup(tableName);
 289     LOG.info("got table group info is {}", tableGrp);
 290     assertTrue(tableGrp.getName().equals(RSGroupInfo.DEFAULT_GROUP));
 291
 292     // change table's group
 293     LOG.info("Moving table " + tableName + " to " + newGroup.getName());
 294     ADMIN.setRSGroup(Sets.newHashSet(tableName), newGroup.getName());
 295
 296     // verify group change
 297     assertEquals(newGroup.getName(), ADMIN.getRSGroup(tableName).getName());
 298
 299     TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
 300       @Override
 301       public boolean evaluate() throws Exception {
 302         Map<ServerName, List<String>> serverMap = getTableServerRegionMap().get(tableName);
 303         int count = 0;
 304         if (serverMap != null) {
 305           for (ServerName rs : serverMap.keySet()) {
 306             if (newGroup.containsServer(rs.getAddress())) {
 307               count += serverMap.get(rs).size();
 308             }
 309           }
 310         }
 311         return count == 5;
 312       }
 313     });
 314
 315     // test truncate
 316     ADMIN.disableTable(tableName);
 317     ADMIN.truncateTable(tableName, true);
 318     List<TableName> tablesInGroup = ADMIN.listTablesInRSGroup(newGroup.getName());
 319     assertEquals(1, tablesInGroup.size());
 320     assertEquals(tableName, tablesInGroup.get(0));
 321
 322     // verify removed table is removed from group
 323     TEST_UTIL.deleteTable(tableName);
 324     assertEquals(0, ADMIN.listTablesInRSGroup(newGroup.getName()).size());
 325   }
 326
 327   @Test
 328   public void testDisabledTableMove() throws Exception {
 329     final byte[] familyNameBytes = Bytes.toBytes("f");
 330     String newGroupName = getGroupName(getNameWithoutIndex(name.getMethodName()));
 331     final RSGroupInfo newGroup = addGroup(newGroupName, 2);
 332
 333     TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 5);
 334     TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
 335       @Override
 336       public boolean evaluate() throws Exception {
 337         List<String> regions = getTableRegionMap().get(tableName);
 338         if (regions == null) {
 339           return false;
 340         }
 341         return getTableRegionMap().get(tableName).size() >= 5;
 342       }
 343     });
 344
 345     RSGroupInfo tableGrp = ADMIN.getRSGroup(tableName);
 346     assertTrue(tableGrp.getName().equals(RSGroupInfo.DEFAULT_GROUP));
 347
 348     // test disable table
 349     ADMIN.disableTable(tableName);
 350
 351     // change table's group
 352     LOG.info("Moving table " + tableName + " to " + newGroup.getName());
 353     ADMIN.setRSGroup(Sets.newHashSet(tableName), newGroup.getName());
 354
 355     // verify group change
 356     assertEquals(newGroup.getName(), ADMIN.getRSGroup(tableName).getName());
 357   }
 358
 359   @Test
 360   public void testNonExistentTableMove() throws Exception {
 361     TableName tableName =
 362       TableName.valueOf(TABLE_PREFIX + getNameWithoutIndex(name.getMethodName()));
 363     RSGroupInfo tableGrp = ADMIN.getRSGroup(tableName);
 364     assertNull(tableGrp);
 365
 366     // test if table exists already.
 367     boolean exist = ADMIN.tableExists(tableName);
 368     assertFalse(exist);
 369
 370     LOG.info("Moving table " + tableName + " to " + RSGroupInfo.DEFAULT_GROUP);
 371     try {
 372       ADMIN.setRSGroup(Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP);
 373       fail("Table " + tableName + " shouldn't have been successfully moved.");
 374     } catch (IOException ex) {
 375       assertTrue(ex instanceof TableNotFoundException);
 376     }
 377
 378     try {
 379       ADMIN.setRSGroup(Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP);
 380       ADMIN.moveServersToRSGroup(Sets.newHashSet(Address.fromParts("bogus", 123)),
 381         RSGroupInfo.DEFAULT_GROUP);
 382       fail("Table " + tableName + " shouldn't have been successfully moved.");
 383     } catch (IOException ex) {
 384       assertTrue(ex instanceof TableNotFoundException);
 385     }
 386     // verify group change
 387     assertNull(ADMIN.getRSGroup(tableName));
 388   }
 389
 390   @Test
 391   public void testRSGroupListDoesNotContainFailedTableCreation() throws Exception {
 392     toggleQuotaCheckAndRestartMiniCluster(true);
 393     String nsp = "np1";
 394     NamespaceDescriptor nspDesc =
 395       NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5")
 396         .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
 397     ADMIN.createNamespace(nspDesc);
 398     assertEquals(3, ADMIN.listNamespaceDescriptors().length);
 399     ColumnFamilyDescriptor fam1 = ColumnFamilyDescriptorBuilder.of("fam1");
 400     TableDescriptor tableDescOne = TableDescriptorBuilder
 401       .newBuilder(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"))
 402       .setColumnFamily(fam1).build();
 403     ADMIN.createTable(tableDescOne);
 404
 405     TableDescriptor tableDescTwo = TableDescriptorBuilder
 406       .newBuilder(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"))
 407       .setColumnFamily(fam1).build();
 408     boolean constraintViolated = false;
 409
 410     try {
 411       ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 6);
 412       fail("Creation table should fail because of quota violation.");
 413     } catch (Exception exp) {
 414       assertTrue(exp instanceof IOException);
 415       constraintViolated = true;
 416     } finally {
 417       assertTrue("Constraint not violated for table " + tableDescTwo.getTableName(),
 418         constraintViolated);
 419     }
 420     List<RSGroupInfo> rsGroupInfoList = ADMIN.listRSGroups();
 421     boolean foundTable2 = false;
 422     boolean foundTable1 = false;
 423     for (int i = 0; i < rsGroupInfoList.size(); i++) {
 424       Set<TableName> tables =
 425         Sets.newHashSet(ADMIN.listTablesInRSGroup(rsGroupInfoList.get(i).getName()));
 426       if (tables.contains(tableDescTwo.getTableName())) {
 427         foundTable2 = true;
 428       }
 429       if (tables.contains(tableDescOne.getTableName())) {
 430         foundTable1 = true;
 431       }
 432     }
 433     assertFalse("Found table2 in rsgroup list.", foundTable2);
 434     assertTrue("Did not find table1 in rsgroup list", foundTable1);
 435
 436     TEST_UTIL.deleteTable(tableDescOne.getTableName());
 437     ADMIN.deleteNamespace(nspDesc.getName());
 438     toggleQuotaCheckAndRestartMiniCluster(false);
 439
 440   }
 441
 442   @Test
 443   public void testNotMoveTableToNullRSGroupWhenCreatingExistingTable() throws Exception {
 444     // Trigger
 445     TableName tn1 = TableName.valueOf("t1");
 446     TEST_UTIL.createTable(tn1, "cf1");
 447     try {
 448       // Create an existing table to trigger HBASE-21866
 449       TEST_UTIL.createTable(tn1, "cf1");
 450     } catch (TableExistsException teex) {
 451       // Ignore
 452     }
 453
 454     // Wait then verify
 455     // Could not verify until the rollback of CreateTableProcedure is done
 456     // (that is, the coprocessor finishes its work),
 457     // or the table is still in the "default" rsgroup even though HBASE-21866
 458     // is not fixed.
 459     TEST_UTIL.waitFor(5000, new Waiter.Predicate<Exception>() {
 460       @Override
 461       public boolean evaluate() throws Exception {
 462         return MASTER.getMasterProcedureExecutor().getActiveExecutorCount() == 0;
 463       }
 464     });
 465     Set<TableName> tables = Sets.newHashSet(ADMIN.listTablesInRSGroup(RSGroupInfo.DEFAULT_GROUP));
 466     assertTrue("Table 't1' must be in 'default' rsgroup", tables.contains(tn1));
 467
 468     // Cleanup
 469     TEST_UTIL.deleteTable(tn1);
 470   }
 471 }