hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.client;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertFalse;
  22 import static org.junit.Assert.assertThrows;
  23 import static org.junit.Assert.assertTrue;
  24 import static org.junit.Assert.fail;
  25
  26 import java.io.IOException;
  27 import java.util.ArrayList;
  28 import java.util.Collection;
  29 import java.util.Collections;
  30 import java.util.EnumSet;
  31 import java.util.HashMap;
  32 import java.util.HashSet;
  33 import java.util.List;
  34 import java.util.Random;
  35 import java.util.concurrent.atomic.AtomicInteger;
  36 import java.util.stream.Collectors;
  37 import org.apache.hadoop.hbase.ClusterMetrics.Option;
  38 import org.apache.hadoop.hbase.HBaseClassTestRule;
  39 import org.apache.hadoop.hbase.HConstants;
  40 import org.apache.hadoop.hbase.ServerName;
  41 import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
  42 import org.apache.hadoop.hbase.TableExistsException;
  43 import org.apache.hadoop.hbase.TableName;
  44 import org.apache.hadoop.hbase.TableNotDisabledException;
  45 import org.apache.hadoop.hbase.TableNotEnabledException;
  46 import org.apache.hadoop.hbase.TableNotFoundException;
  47 import org.apache.hadoop.hbase.UnknownRegionException;
  48 import org.apache.hadoop.hbase.Waiter.Predicate;
  49 import org.apache.hadoop.hbase.constraint.ConstraintException;
  50 import org.apache.hadoop.hbase.master.HMaster;
  51 import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
  52 import org.apache.hadoop.hbase.regionserver.HRegion;
  53 import org.apache.hadoop.hbase.regionserver.HRegionServer;
  54 import org.apache.hadoop.hbase.regionserver.HStore;
  55 import org.apache.hadoop.hbase.testclassification.ClientTests;
  56 import org.apache.hadoop.hbase.testclassification.LargeTests;
  57 import org.apache.hadoop.hbase.util.Bytes;
  58 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
  59 import org.apache.hadoop.hbase.util.FutureUtils;
  60 import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
  61 import org.junit.Assert;
  62 import org.junit.ClassRule;
  63 import org.junit.Test;
  64 import org.junit.experimental.categories.Category;
  65 import org.slf4j.Logger;
  66 import org.slf4j.LoggerFactory;
  67
  68 import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
  69 import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
  70
  71 /**
  72  * Class to test HBaseAdmin.
  73  * Spins up the minicluster once at test start and then takes it down afterward.
  74  * Add any testing of HBaseAdmin functionality here.
  75  */
  76 @Category({LargeTests.class, ClientTests.class})
  77 public class TestAdmin2 extends TestAdminBase {
  78
  79   @ClassRule
  80   public static final HBaseClassTestRule CLASS_RULE =
  81       HBaseClassTestRule.forClass(TestAdmin2.class);
  82
  83   private static final Logger LOG = LoggerFactory.getLogger(TestAdmin2.class);
  84
  85   @Test
  86   public void testCreateBadTables() throws IOException {
  87     String msg = null;
  88     try {
  89       ADMIN.createTable(TableDescriptorBuilder.newBuilder(TableName.META_TABLE_NAME).build());
  90     } catch (TableExistsException e) {
  91       msg = e.toString();
  92     }
  93     assertTrue("Unexcepted exception message " + msg, msg != null &&
  94       msg.startsWith(TableExistsException.class.getName()) &&
  95       msg.contains(TableName.META_TABLE_NAME.getNameAsString()));
  96
  97     // Now try and do concurrent creation with a bunch of threads.
  98     TableDescriptor tableDescriptor =
  99       TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 100         .setColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build();
 101     int count = 10;
 102     Thread [] threads = new Thread [count];
 103     final AtomicInteger successes = new AtomicInteger(0);
 104     final AtomicInteger failures = new AtomicInteger(0);
 105     final Admin localAdmin = ADMIN;
 106     for (int i = 0; i < count; i++) {
 107       threads[i] = new Thread(Integer.toString(i)) {
 108         @Override
 109         public void run() {
 110           try {
 111             localAdmin.createTable(tableDescriptor);
 112             successes.incrementAndGet();
 113           } catch (TableExistsException e) {
 114             failures.incrementAndGet();
 115           } catch (IOException e) {
 116             throw new RuntimeException("Failed threaded create" + getName(), e);
 117           }
 118         }
 119       };
 120     }
 121     for (int i = 0; i < count; i++) {
 122       threads[i].start();
 123     }
 124     for (int i = 0; i < count; i++) {
 125       while(threads[i].isAlive()) {
 126         try {
 127           Thread.sleep(100);
 128         } catch (InterruptedException e) {
 129           // continue
 130         }
 131       }
 132     }
 133     // All threads are now dead.  Count up how many tables were created and
 134     // how many failed w/ appropriate exception.
 135     assertEquals(1, successes.get());
 136     assertEquals(count - 1, failures.get());
 137   }
 138
 139   /**
 140    * Test for hadoop-1581 'HBASE: Unopenable tablename bug'.
 141    */
 142   @Test
 143   public void testTableNameClash() throws Exception {
 144     final String name = this.name.getMethodName();
 145     TableDescriptor tableDescriptor1 =
 146       TableDescriptorBuilder.newBuilder(TableName.valueOf(name + "SOMEUPPERCASE"))
 147         .setColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build();
 148     TableDescriptor tableDescriptor2 = TableDescriptorBuilder.newBuilder(TableName.valueOf(name))
 149       .setColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build();
 150     ADMIN.createTable(tableDescriptor1);
 151     ADMIN.createTable(tableDescriptor2);
 152     // Before fix, below would fail throwing a NoServerForRegionException.
 153     TEST_UTIL.getConnection().getTable(tableDescriptor2.getTableName()).close();
 154   }
 155
 156   /***
 157    * HMaster.createTable used to be kind of synchronous call
 158    * Thus creating of table with lots of regions can cause RPC timeout
 159    * After the fix to make createTable truly async, RPC timeout shouldn't be an
 160    * issue anymore
 161    */
 162   @Test
 163   public void testCreateTableRPCTimeOut() throws Exception {
 164     final String name = this.name.getMethodName();
 165     int oldTimeout = TEST_UTIL.getConfiguration().
 166       getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
 167     TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 1500);
 168     try {
 169       int expectedRegions = 100;
 170       // Use 80 bit numbers to make sure we aren't limited
 171       byte [] startKey = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
 172       byte [] endKey =   { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 };
 173       Admin hbaseadmin = TEST_UTIL.getAdmin();
 174       TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(TableName.valueOf(name))
 175         .setColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build();
 176       hbaseadmin.createTable(tableDescriptor, startKey, endKey, expectedRegions);
 177     } finally {
 178       TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, oldTimeout);
 179     }
 180   }
 181
 182   /**
 183    * Test read only tables
 184    * @throws Exception
 185    */
 186   @Test
 187   public void testReadOnlyTable() throws Exception {
 188     final TableName name = TableName.valueOf(this.name.getMethodName());
 189     Table table = TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY);
 190     byte[] value = Bytes.toBytes("somedata");
 191     // This used to use an empty row... That must have been a bug
 192     Put put = new Put(value);
 193     put.addColumn(HConstants.CATALOG_FAMILY, HConstants.CATALOG_FAMILY, value);
 194     table.put(put);
 195     table.close();
 196   }
 197
 198   /**
 199    * Test that user table names can contain '-' and '.' so long as they do not
 200    * start with same. HBASE-771
 201    */
 202   @Test
 203   public void testTableNames() throws IOException {
 204     byte[][] illegalNames = new byte[][] { Bytes.toBytes("-bad"), Bytes.toBytes(".bad") };
 205     for (byte[] illegalName : illegalNames) {
 206       assertThrows(
 207         "Did not detect '" + Bytes.toString(illegalName) + "' as an illegal user table name",
 208         IllegalArgumentException.class, () -> TableName.valueOf(illegalName));
 209     }
 210     byte[] legalName = Bytes.toBytes("g-oo.d");
 211     try {
 212       TableName.valueOf(legalName);
 213     } catch (IllegalArgumentException e) {
 214       fail("Legal user table name: '" + Bytes.toString(legalName) +
 215         "' caused IllegalArgumentException: " + e.getMessage());
 216     }
 217   }
 218
 219   /**
 220    * For HADOOP-2579
 221    */
 222   @Test (expected=TableExistsException.class)
 223   public void testTableExistsExceptionWithATable() throws IOException {
 224     final TableName name = TableName.valueOf(this.name.getMethodName());
 225     TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY).close();
 226     TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY);
 227   }
 228
 229   /**
 230    * Can't disable a table if the table isn't in enabled state
 231    */
 232   @Test (expected=TableNotEnabledException.class)
 233   public void testTableNotEnabledExceptionWithATable() throws IOException {
 234     final TableName name = TableName.valueOf(this.name.getMethodName());
 235     TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY).close();
 236     ADMIN.disableTable(name);
 237     ADMIN.disableTable(name);
 238   }
 239
 240   /**
 241    * Can't enable a table if the table isn't in disabled state
 242    */
 243   @Test(expected = TableNotDisabledException.class)
 244   public void testTableNotDisabledExceptionWithATable() throws IOException {
 245     final TableName name = TableName.valueOf(this.name.getMethodName());
 246     try (Table t = TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY)) {
 247       ADMIN.enableTable(name);
 248     }
 249   }
 250
 251   /**
 252    * For HADOOP-2579
 253    */
 254   @Test(expected = TableNotFoundException.class)
 255   public void testTableNotFoundExceptionWithoutAnyTables() throws IOException {
 256     TableName tableName = TableName.valueOf("testTableNotFoundExceptionWithoutAnyTables");
 257     try (Table ht = TEST_UTIL.getConnection().getTable(tableName)) {
 258       ht.get(new Get(Bytes.toBytes("e")));
 259     }
 260   }
 261
 262   @Test
 263   public void testShouldUnassignTheRegion() throws Exception {
 264     final TableName tableName = TableName.valueOf(name.getMethodName());
 265     createTableWithDefaultConf(tableName);
 266
 267     RegionInfo info = null;
 268     HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(tableName);
 269     List<RegionInfo> onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices());
 270     for (RegionInfo regionInfo : onlineRegions) {
 271       if (!regionInfo.getTable().isSystemTable()) {
 272         info = regionInfo;
 273         ADMIN.unassign(regionInfo.getRegionName(), true);
 274       }
 275     }
 276     boolean isInList = ProtobufUtil.getOnlineRegions(
 277       rs.getRSRpcServices()).contains(info);
 278     long timeout = EnvironmentEdgeManager.currentTime() + 10000;
 279     while ((EnvironmentEdgeManager.currentTime() < timeout) && (isInList)) {
 280       Thread.sleep(100);
 281       isInList = ProtobufUtil.getOnlineRegions(
 282         rs.getRSRpcServices()).contains(info);
 283     }
 284
 285     assertFalse("The region should not be present in online regions list.",
 286       isInList);
 287   }
 288
 289   @Test
 290   public void testCloseRegionIfInvalidRegionNameIsPassed() throws Exception {
 291     final String name = this.name.getMethodName();
 292     byte[] tableName = Bytes.toBytes(name);
 293     createTableWithDefaultConf(tableName);
 294
 295     RegionInfo info = null;
 296     HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TableName.valueOf(tableName));
 297     List<RegionInfo> onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices());
 298     for (RegionInfo regionInfo : onlineRegions) {
 299       if (!regionInfo.isMetaRegion()) {
 300         if (regionInfo.getRegionNameAsString().contains(name)) {
 301           info = regionInfo;
 302           assertThrows(UnknownRegionException.class,
 303             () -> ADMIN.unassign(Bytes.toBytes(
 304               "test,,1358563771069.acc1ad1b7962564fc3a43e5907e8db33."), true));
 305         }
 306       }
 307     }
 308     onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices());
 309     assertTrue("The region should be present in online regions list.",
 310         onlineRegions.contains(info));
 311   }
 312
 313   @Test
 314   public void testCloseRegionThatFetchesTheHRIFromMeta() throws Exception {
 315     final TableName tableName = TableName.valueOf(name.getMethodName());
 316     createTableWithDefaultConf(tableName);
 317
 318     RegionInfo info = null;
 319     HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(tableName);
 320     List<RegionInfo> onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices());
 321     for (RegionInfo regionInfo : onlineRegions) {
 322       if (!regionInfo.isMetaRegion()) {
 323         if (regionInfo.getRegionNameAsString().contains("TestHBACloseRegion2")) {
 324           info = regionInfo;
 325           ADMIN.unassign(regionInfo.getRegionName(), true);
 326         }
 327       }
 328     }
 329
 330     boolean isInList = ProtobufUtil.getOnlineRegions(
 331       rs.getRSRpcServices()).contains(info);
 332     long timeout = EnvironmentEdgeManager.currentTime() + 10000;
 333     while ((EnvironmentEdgeManager.currentTime() < timeout) && (isInList)) {
 334       Thread.sleep(100);
 335       isInList = ProtobufUtil.getOnlineRegions(
 336         rs.getRSRpcServices()).contains(info);
 337     }
 338
 339     assertFalse("The region should not be present in online regions list.",
 340       isInList);
 341   }
 342
 343   private Admin createTable(TableName tableName) throws IOException {
 344     Admin admin = TEST_UTIL.getAdmin();
 345
 346     TableDescriptorBuilder tableDescriptorBuilder =
 347       TableDescriptorBuilder.newBuilder(tableName);
 348     ColumnFamilyDescriptor columnFamilyDescriptor =
 349       ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("value")).build();
 350
 351     tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
 352     admin.createTable(tableDescriptorBuilder.build());
 353     return admin;
 354   }
 355
 356   private void createTableWithDefaultConf(byte[] TABLENAME) throws IOException {
 357     createTableWithDefaultConf(TableName.valueOf(TABLENAME));
 358   }
 359
 360   private void createTableWithDefaultConf(TableName TABLENAME) throws IOException {
 361     TableDescriptorBuilder tableDescriptorBuilder =
 362       TableDescriptorBuilder.newBuilder(TABLENAME);
 363     ColumnFamilyDescriptor columnFamilyDescriptor =
 364       ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("value")).build();
 365     tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
 366
 367     ADMIN.createTable(tableDescriptorBuilder.build());
 368   }
 369
 370   /**
 371    * For HBASE-2556
 372    */
 373   @Test
 374   public void testGetTableRegions() throws IOException {
 375     final TableName tableName = TableName.valueOf(name.getMethodName());
 376
 377     int expectedRegions = 10;
 378
 379     // Use 80 bit numbers to make sure we aren't limited
 380     byte [] startKey = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
 381     byte [] endKey =   { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 };
 382
 383
 384     TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
 385       .setColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build();
 386     ADMIN.createTable(tableDescriptor, startKey, endKey, expectedRegions);
 387
 388     List<RegionInfo> RegionInfos = ADMIN.getRegions(tableName);
 389
 390     assertEquals("Tried to create " + expectedRegions + " regions " +
 391         "but only found " + RegionInfos.size(),
 392         expectedRegions, RegionInfos.size());
 393  }
 394
 395   @Test
 396   public void testMoveToPreviouslyAssignedRS() throws IOException, InterruptedException {
 397     SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
 398     HMaster master = cluster.getMaster();
 399     final TableName tableName = TableName.valueOf(name.getMethodName());
 400     Admin localAdmin = createTable(tableName);
 401     List<RegionInfo> tableRegions = localAdmin.getRegions(tableName);
 402     RegionInfo hri = tableRegions.get(0);
 403     AssignmentManager am = master.getAssignmentManager();
 404     ServerName server = am.getRegionStates().getRegionServerOfRegion(hri);
 405     localAdmin.move(hri.getEncodedNameAsBytes(), server);
 406     assertEquals("Current region server and region server before move should be same.", server,
 407       am.getRegionStates().getRegionServerOfRegion(hri));
 408   }
 409
 410   @Test
 411   public void testWALRollWriting() throws Exception {
 412     setUpforLogRolling();
 413     String className = this.getClass().getName();
 414     StringBuilder v = new StringBuilder(className);
 415     while (v.length() < 1000) {
 416       v.append(className);
 417     }
 418     byte[] value = Bytes.toBytes(v.toString());
 419     HRegionServer regionServer = startAndWriteData(TableName.valueOf(name.getMethodName()), value);
 420     LOG.info("after writing there are "
 421         + AbstractFSWALProvider.getNumRolledLogFiles(regionServer.getWAL(null)) + " log files");
 422
 423     // flush all regions
 424     for (HRegion r : regionServer.getOnlineRegionsLocalContext()) {
 425       r.flush(true);
 426     }
 427     ADMIN.rollWALWriter(regionServer.getServerName());
 428     int count = AbstractFSWALProvider.getNumRolledLogFiles(regionServer.getWAL(null));
 429     LOG.info("after flushing all regions and rolling logs there are " +
 430         count + " log files");
 431     assertTrue(("actual count: " + count), count <= 2);
 432   }
 433
 434   private void setUpforLogRolling() {
 435     // Force a region split after every 768KB
 436     TEST_UTIL.getConfiguration().setLong(HConstants.HREGION_MAX_FILESIZE,
 437         768L * 1024L);
 438
 439     // We roll the log after every 32 writes
 440     TEST_UTIL.getConfiguration().setInt("hbase.regionserver.maxlogentries", 32);
 441
 442     TEST_UTIL.getConfiguration().setInt(
 443         "hbase.regionserver.logroll.errors.tolerated", 2);
 444     TEST_UTIL.getConfiguration().setInt("hbase.rpc.timeout", 10 * 1000);
 445
 446     // For less frequently updated regions flush after every 2 flushes
 447     TEST_UTIL.getConfiguration().setInt(
 448         "hbase.hregion.memstore.optionalflushcount", 2);
 449
 450     // We flush the cache after every 8192 bytes
 451     TEST_UTIL.getConfiguration().setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE,
 452         8192);
 453
 454     // Increase the amount of time between client retries
 455     TEST_UTIL.getConfiguration().setLong("hbase.client.pause", 10 * 1000);
 456
 457     // Reduce thread wake frequency so that other threads can get
 458     // a chance to run.
 459     TEST_UTIL.getConfiguration().setInt(HConstants.THREAD_WAKE_FREQUENCY,
 460         2 * 1000);
 461
 462     /**** configuration for testLogRollOnDatanodeDeath ****/
 463     // lower the namenode & datanode heartbeat so the namenode
 464     // quickly detects datanode failures
 465     TEST_UTIL.getConfiguration().setInt("dfs.namenode.heartbeat.recheck-interval", 5000);
 466     TEST_UTIL.getConfiguration().setInt("dfs.heartbeat.interval", 1);
 467     // the namenode might still try to choose the recently-dead datanode
 468     // for a pipeline, so try to a new pipeline multiple times
 469     TEST_UTIL.getConfiguration().setInt("dfs.client.block.write.retries", 30);
 470     TEST_UTIL.getConfiguration().setInt(
 471         "hbase.regionserver.hlog.tolerable.lowreplication", 2);
 472     TEST_UTIL.getConfiguration().setInt(
 473         "hbase.regionserver.hlog.lowreplication.rolllimit", 3);
 474   }
 475
 476   private HRegionServer startAndWriteData(TableName tableName, byte[] value)
 477   throws IOException, InterruptedException {
 478     // When the hbase:meta table can be opened, the region servers are running
 479     TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME).close();
 480
 481     // Create the test table and open it
 482     TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
 483       .setColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build();
 484     ADMIN.createTable(tableDescriptor);
 485     Table table = TEST_UTIL.getConnection().getTable(tableName);
 486
 487     HRegionServer regionServer = TEST_UTIL.getRSForFirstRegionInTable(tableName);
 488     for (int i = 1; i <= 256; i++) { // 256 writes should cause 8 log rolls
 489       Put put = new Put(Bytes.toBytes("row" + String.format("%1$04d", i)));
 490       put.addColumn(HConstants.CATALOG_FAMILY, null, value);
 491       table.put(put);
 492       if (i % 32 == 0) {
 493         // After every 32 writes sleep to let the log roller run
 494         try {
 495           Thread.sleep(2000);
 496         } catch (InterruptedException e) {
 497           // continue
 498         }
 499       }
 500     }
 501
 502     table.close();
 503     return regionServer;
 504   }
 505
 506   @Test
 507   public void testDisableCatalogTable() throws Exception {
 508     try {
 509       ADMIN.disableTable(TableName.META_TABLE_NAME);
 510       fail("Expected to throw ConstraintException");
 511     } catch (ConstraintException e) {
 512     }
 513     // Before the fix for HBASE-6146, the below table creation was failing as the hbase:meta table
 514     // actually getting disabled by the disableTable() call.
 515     TableDescriptor tableDescriptor =
 516       TableDescriptorBuilder.newBuilder(TableName.valueOf(Bytes.toBytes(name.getMethodName())))
 517         .setColumnFamily(ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("cf1"))).build();
 518     TEST_UTIL.getAdmin().createTable(tableDescriptor);
 519   }
 520
 521   @Test
 522   public void testIsEnabledOrDisabledOnUnknownTable() throws Exception {
 523     try {
 524       ADMIN.isTableEnabled(TableName.valueOf(name.getMethodName()));
 525       fail("Test should fail if isTableEnabled called on unknown table.");
 526     } catch (IOException e) {
 527     }
 528
 529     try {
 530       ADMIN.isTableDisabled(TableName.valueOf(name.getMethodName()));
 531       fail("Test should fail if isTableDisabled called on unknown table.");
 532     } catch (IOException e) {
 533     }
 534   }
 535
 536   @Test
 537   public void testBalancer() throws Exception {
 538     boolean initialState = ADMIN.isBalancerEnabled();
 539
 540     // Start the balancer, wait for it.
 541     boolean prevState = ADMIN.balancerSwitch(!initialState, true);
 542
 543     // The previous state should be the original state we observed
 544     assertEquals(initialState, prevState);
 545
 546     // Current state should be opposite of the original
 547     assertEquals(!initialState, ADMIN.isBalancerEnabled());
 548
 549     // Reset it back to what it was
 550     prevState = ADMIN.balancerSwitch(initialState, true);
 551
 552     // The previous state should be the opposite of the initial state
 553     assertEquals(!initialState, prevState);
 554     // Current state should be the original state again
 555     assertEquals(initialState, ADMIN.isBalancerEnabled());
 556   }
 557
 558   @Test
 559   public void testRegionNormalizer() throws Exception {
 560     boolean initialState = ADMIN.isNormalizerEnabled();
 561
 562     // flip state
 563     boolean prevState = ADMIN.normalizerSwitch(!initialState);
 564
 565     // The previous state should be the original state we observed
 566     assertEquals(initialState, prevState);
 567
 568     // Current state should be opposite of the original
 569     assertEquals(!initialState, ADMIN.isNormalizerEnabled());
 570
 571     // Reset it back to what it was
 572     prevState = ADMIN.normalizerSwitch(initialState);
 573
 574     // The previous state should be the opposite of the initial state
 575     assertEquals(!initialState, prevState);
 576     // Current state should be the original state again
 577     assertEquals(initialState, ADMIN.isNormalizerEnabled());
 578   }
 579
 580   @Test
 581   public void testAbortProcedureFail() throws Exception {
 582     Random randomGenerator = new Random();
 583     long procId = randomGenerator.nextLong();
 584
 585     boolean abortResult = ADMIN.abortProcedure(procId, true);
 586     assertFalse(abortResult);
 587   }
 588
 589   @Test
 590   public void testGetProcedures() throws Exception {
 591     String procList = ADMIN.getProcedures();
 592     assertTrue(procList.startsWith("["));
 593   }
 594
 595   @Test
 596   public void testGetLocks() throws Exception {
 597     String lockList = ADMIN.getLocks();
 598     assertTrue(lockList.startsWith("["));
 599   }
 600
 601   @Test
 602   public void testDecommissionRegionServers() throws Exception {
 603     List<ServerName> decommissionedRegionServers = ADMIN.listDecommissionedRegionServers();
 604     assertTrue(decommissionedRegionServers.isEmpty());
 605
 606     final TableName tableName = TableName.valueOf(name.getMethodName());
 607     TEST_UTIL.createMultiRegionTable(tableName, Bytes.toBytes("f"), 6);
 608
 609     ArrayList<ServerName> clusterRegionServers =
 610         new ArrayList<>(ADMIN.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
 611           .getLiveServerMetrics().keySet());
 612
 613     assertEquals(3, clusterRegionServers.size());
 614
 615     HashMap<ServerName, List<RegionInfo>> serversToDecommssion = new HashMap<>();
 616     // Get a server that has meta online. We will decommission two of the servers,
 617     // leaving one online.
 618     int i;
 619     for (i = 0; i < clusterRegionServers.size(); i++) {
 620       List<RegionInfo> regionsOnServer = ADMIN.getRegions(clusterRegionServers.get(i));
 621       if (ADMIN.getRegions(clusterRegionServers.get(i)).stream().anyMatch(p -> p.isMetaRegion())) {
 622         serversToDecommssion.put(clusterRegionServers.get(i), regionsOnServer);
 623         break;
 624       }
 625     }
 626
 627     clusterRegionServers.remove(i);
 628     // Get another server to decommission.
 629     serversToDecommssion.put(clusterRegionServers.get(0),
 630       ADMIN.getRegions(clusterRegionServers.get(0)));
 631
 632     ServerName remainingServer = clusterRegionServers.get(1);
 633
 634     // Decommission
 635     ADMIN.decommissionRegionServers(new ArrayList<ServerName>(serversToDecommssion.keySet()), true);
 636     assertEquals(2, ADMIN.listDecommissionedRegionServers().size());
 637
 638     // Verify the regions have been off the decommissioned servers, all on the one
 639     // remaining server.
 640     for (ServerName server : serversToDecommssion.keySet()) {
 641       for (RegionInfo region : serversToDecommssion.get(server)) {
 642         TEST_UTIL.assertRegionOnServer(region, remainingServer, 10000);
 643       }
 644     }
 645
 646     // Recommission and load the regions.
 647     for (ServerName server : serversToDecommssion.keySet()) {
 648       List<byte[]> encodedRegionNames = serversToDecommssion.get(server).stream()
 649           .map(region -> region.getEncodedNameAsBytes()).collect(Collectors.toList());
 650       ADMIN.recommissionRegionServer(server, encodedRegionNames);
 651     }
 652     assertTrue(ADMIN.listDecommissionedRegionServers().isEmpty());
 653     // Verify the regions have been moved to the recommissioned servers
 654     for (ServerName server : serversToDecommssion.keySet()) {
 655       for (RegionInfo region : serversToDecommssion.get(server)) {
 656         TEST_UTIL.assertRegionOnServer(region, server, 10000);
 657       }
 658     }
 659   }
 660
 661   /**
 662    * TestCase for HBASE-21355
 663    */
 664   @Test
 665   public void testGetRegionInfo() throws Exception {
 666     final TableName tableName = TableName.valueOf(name.getMethodName());
 667     Table table = TEST_UTIL.createTable(tableName, Bytes.toBytes("f"));
 668     for (int i = 0; i < 100; i++) {
 669       table.put(new Put(Bytes.toBytes(i)).addColumn(Bytes.toBytes("f"), Bytes.toBytes("q"),
 670         Bytes.toBytes(i)));
 671     }
 672     ADMIN.flush(tableName);
 673
 674     HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(table.getName());
 675     List<HRegion> regions = rs.getRegions(tableName);
 676     Assert.assertEquals(1, regions.size());
 677
 678     HRegion region = regions.get(0);
 679     byte[] regionName = region.getRegionInfo().getRegionName();
 680     HStore store = region.getStore(Bytes.toBytes("f"));
 681     long expectedStoreFilesSize = store.getStorefilesSize();
 682     Assert.assertNotNull(store);
 683     Assert.assertEquals(expectedStoreFilesSize, store.getSize());
 684     for (int i = 0; i < 10; i++) {
 685       RegionInfo ri = ProtobufUtil
 686         .toRegionInfo(TEST_UTIL.getAsyncConnection().getRegionServerAdmin(rs.getServerName())
 687           .getRegionInfo(RequestConverter.buildGetRegionInfoRequest(regionName)).get()
 688           .getRegionInfo());
 689
 690       Assert.assertEquals(region.getRegionInfo(), ri);
 691
 692       // Make sure that the store size is still the actual file system's store size.
 693       Assert.assertEquals(expectedStoreFilesSize, store.getSize());
 694     }
 695
 696     // Test querying using the encoded name only. When encoded name passed,
 697     // and the target server is the Master, we return the full region name.
 698     // Convenience.
 699     ServerName sn = null;
 700     try (Admin admin = TEST_UTIL.getConnection().getAdmin()) {
 701       sn = admin.getMaster();
 702     }
 703     RegionInfo ri = region.getRegionInfo();
 704     testGetWithRegionName(sn, ri, ri.getEncodedNameAsBytes());
 705     testGetWithRegionName(sn, ri, ri.getRegionName());
 706     // Try querying meta encoded name.
 707     ri = RegionInfoBuilder.FIRST_META_REGIONINFO;
 708     testGetWithRegionName(sn, ri, ri.getEncodedNameAsBytes());
 709     testGetWithRegionName(sn, ri, ri.getRegionName());
 710   }
 711
 712   /**
 713    * Do get of RegionInfo from Master using encoded region name.
 714    */
 715   private void testGetWithRegionName(ServerName sn, RegionInfo inputRI,
 716       byte [] regionName) throws IOException {
 717     RegionInfo ri = ProtobufUtil.toRegionInfo(FutureUtils.get(
 718       TEST_UTIL.getAsyncConnection().getRegionServerAdmin(sn).getRegionInfo(
 719         ProtobufUtil.getGetRegionInfoRequest(regionName))).getRegionInfo());
 720     assertEquals(inputRI, ri);
 721   }
 722
 723   @Test
 724   public void testTableSplitFollowedByModify() throws Exception {
 725     final TableName tableName = TableName.valueOf(name.getMethodName());
 726     TEST_UTIL.createTable(tableName, Bytes.toBytes("f"));
 727
 728     // get the original table region count
 729     List<RegionInfo> regions = ADMIN.getRegions(tableName);
 730     int originalCount = regions.size();
 731     assertEquals(1, originalCount);
 732
 733     // split the table and wait until region count increases
 734     ADMIN.split(tableName, Bytes.toBytes(3));
 735     TEST_UTIL.waitFor(30000, new Predicate<Exception>() {
 736
 737       @Override
 738       public boolean evaluate() throws Exception {
 739         return ADMIN.getRegions(tableName).size() > originalCount;
 740       }
 741     });
 742
 743     // do some table modification
 744     TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(ADMIN.getDescriptor(tableName))
 745         .setMaxFileSize(11111111)
 746         .build();
 747     ADMIN.modifyTable(tableDesc);
 748     assertEquals(11111111, ADMIN.getDescriptor(tableName).getMaxFileSize());
 749   }
 750
 751   @Test
 752   public void testTableMergeFollowedByModify() throws Exception {
 753     final TableName tableName = TableName.valueOf(name.getMethodName());
 754     TEST_UTIL.createTable(tableName, new byte[][] { Bytes.toBytes("f") },
 755       new byte[][] { Bytes.toBytes(3) });
 756
 757     // assert we have at least 2 regions in the table
 758     List<RegionInfo> regions = ADMIN.getRegions(tableName);
 759     int originalCount = regions.size();
 760     assertTrue(originalCount >= 2);
 761
 762     byte[] nameOfRegionA = regions.get(0).getEncodedNameAsBytes();
 763     byte[] nameOfRegionB = regions.get(1).getEncodedNameAsBytes();
 764
 765     // merge the table regions and wait until region count decreases
 766     ADMIN.mergeRegionsAsync(nameOfRegionA, nameOfRegionB, true);
 767     TEST_UTIL.waitFor(30000, new Predicate<Exception>() {
 768
 769       @Override
 770       public boolean evaluate() throws Exception {
 771         return ADMIN.getRegions(tableName).size() < originalCount;
 772       }
 773     });
 774
 775     // do some table modification
 776     TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(ADMIN.getDescriptor(tableName))
 777         .setMaxFileSize(11111111)
 778         .build();
 779     ADMIN.modifyTable(tableDesc);
 780     assertEquals(11111111, ADMIN.getDescriptor(tableName).getMaxFileSize());
 781   }
 782
 783   @Test
 784   public void testSnapshotCleanupAsync() throws Exception {
 785     testSnapshotCleanup(false);
 786   }
 787
 788   @Test
 789   public void testSnapshotCleanupSync() throws Exception {
 790     testSnapshotCleanup(true);
 791   }
 792
 793   private void testSnapshotCleanup(final boolean synchronous) throws IOException {
 794     final boolean initialState = ADMIN.isSnapshotCleanupEnabled();
 795     // Switch the snapshot auto cleanup state to opposite to initial state
 796     boolean prevState = ADMIN.snapshotCleanupSwitch(!initialState, synchronous);
 797     // The previous state should be the original state we observed
 798     assertEquals(initialState, prevState);
 799     // Current state should be opposite of the initial state
 800     assertEquals(!initialState, ADMIN.isSnapshotCleanupEnabled());
 801     // Reset the state back to what it was initially
 802     prevState = ADMIN.snapshotCleanupSwitch(initialState, synchronous);
 803     // The previous state should be the opposite of the initial state
 804     assertEquals(!initialState, prevState);
 805     // Current state should be the original state again
 806     assertEquals(initialState, ADMIN.isSnapshotCleanupEnabled());
 807   }
 808
 809   @Test
 810   public void testSlowLogResponses() throws Exception {
 811     // get all live server names
 812     Collection<ServerName> serverNames = ADMIN.getRegionServers();
 813     List<ServerName> serverNameList = new ArrayList<>(serverNames);
 814
 815     // clean up slowlog responses maintained in memory by RegionServers
 816     List<Boolean> areSlowLogsCleared = ADMIN.clearSlowLogResponses(new HashSet<>(serverNameList));
 817
 818     int countFailedClearSlowResponse = 0;
 819     for (Boolean isSlowLogCleared : areSlowLogsCleared) {
 820       if (!isSlowLogCleared) {
 821         ++countFailedClearSlowResponse;
 822       }
 823     }
 824     Assert.assertEquals(countFailedClearSlowResponse, 0);
 825
 826     List<LogEntry> onlineLogRecords = ADMIN.getLogEntries(new HashSet<>(serverNames),
 827       "SLOW_LOG", ServerType.REGION_SERVER, 100, null);
 828     // after cleanup of slowlog responses, total count of slowlog payloads should be 0
 829     Assert.assertEquals(onlineLogRecords.size(), 0);
 830     List<LogEntry> balancerDecisionRecords =
 831       ADMIN.getLogEntries(null, "BALANCER_DECISION", ServerType.MASTER, 100, null);
 832     Assert.assertEquals(balancerDecisionRecords.size(), 0);
 833   }
 834
 835   @Test
 836   public void testGetRegionServers() throws Exception {
 837     // get all live server names
 838     List<ServerName> serverNames = new ArrayList<>(ADMIN.getRegionServers(true));
 839     Assert.assertEquals(3, serverNames.size());
 840
 841     List<ServerName> serversToDecom = new ArrayList<>();
 842     ServerName serverToDecommission = serverNames.get(0);
 843
 844     serversToDecom.add(serverToDecommission);
 845     ADMIN.decommissionRegionServers(serversToDecom, false);
 846     waitForServerCommissioned(serverToDecommission, true);
 847
 848     Assert.assertEquals(2, ADMIN.getRegionServers(true).size());
 849     Assert.assertEquals(3, ADMIN.getRegionServers(false).size());
 850
 851     ADMIN.recommissionRegionServer(serverToDecommission, Collections.emptyList());
 852     waitForServerCommissioned(null, false);
 853
 854     Assert.assertEquals(3, ADMIN.getRegionServers(true).size());
 855     Assert.assertEquals(3, ADMIN.getRegionServers(false).size());
 856   }
 857
 858   private static void waitForServerCommissioned(ServerName excludeServer,
 859       boolean anyServerDecommissioned) {
 860     TEST_UTIL.waitFor(3000, () -> {
 861       try {
 862         List<ServerName> decomServers = TEST_UTIL.getAdmin().listDecommissionedRegionServers();
 863         if (anyServerDecommissioned) {
 864           return decomServers.size() == 1
 865             && decomServers.get(0).equals(excludeServer);
 866         } else {
 867           return decomServers.size() == 0;
 868         }
 869       } catch (IOException e) {
 870         throw new RuntimeException(e);
 871       }
 872     });
 873   }
 874
 875 }