hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi2.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.client;
  19
  20 import org.apache.hadoop.fs.Path;
  21 import org.apache.hadoop.hbase.HBaseClassTestRule;
  22 import org.apache.hadoop.hbase.TableName;
  23 import org.apache.hadoop.hbase.master.MasterFileSystem;
  24 import org.apache.hadoop.hbase.testclassification.ClientTests;
  25 import org.apache.hadoop.hbase.testclassification.LargeTests;
  26 import org.apache.hadoop.hbase.util.Bytes;
  27 import org.apache.hadoop.hbase.util.FSTableDescriptors;
  28 import org.apache.hadoop.hbase.util.FSUtils;
  29 import org.junit.Assert;
  30 import org.junit.ClassRule;
  31 import org.junit.Test;
  32 import org.junit.experimental.categories.Category;
  33 import org.junit.runner.RunWith;
  34 import org.junit.runners.Parameterized;
  35
  36 import java.util.Optional;
  37 import java.util.Set;
  38
  39 import static org.junit.Assert.assertEquals;
  40 import static org.junit.Assert.assertFalse;
  41 import static org.junit.Assert.assertTrue;
  42
  43 /**
  44  * Class to test asynchronous table admin operations
  45  * @see TestAsyncTableAdminApi This test and it used to be joined it was taking longer than our
  46  * ten minute timeout so they were split.
  47  */
  48 @RunWith(Parameterized.class)
  49 @Category({ LargeTests.class, ClientTests.class })
  50 public class TestAsyncTableAdminApi2 extends TestAsyncAdminBase {
  51
  52   @ClassRule
  53   public static final HBaseClassTestRule CLASS_RULE =
  54       HBaseClassTestRule.forClass(TestAsyncTableAdminApi2.class);
  55
  56   @Test
  57   public void testAddColumnFamily() throws Exception {
  58     // Create a table with two families
  59     TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
  60     builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_0));
  61     admin.createTable(builder.build()).join();
  62     admin.disableTable(tableName).join();
  63     // Verify the table descriptor
  64     verifyTableDescriptor(tableName, FAMILY_0);
  65
  66     // Modify the table removing one family and verify the descriptor
  67     admin.addColumnFamily(tableName, ColumnFamilyDescriptorBuilder.of(FAMILY_1)).join();
  68     verifyTableDescriptor(tableName, FAMILY_0, FAMILY_1);
  69   }
  70
  71   @Test
  72   public void testAddSameColumnFamilyTwice() throws Exception {
  73     // Create a table with one families
  74     TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
  75     builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_0));
  76     admin.createTable(builder.build()).join();
  77     admin.disableTable(tableName).join();
  78     // Verify the table descriptor
  79     verifyTableDescriptor(tableName, FAMILY_0);
  80
  81     // Modify the table removing one family and verify the descriptor
  82     admin.addColumnFamily(tableName, ColumnFamilyDescriptorBuilder.of(FAMILY_1)).join();
  83     verifyTableDescriptor(tableName, FAMILY_0, FAMILY_1);
  84
  85     try {
  86       // Add same column family again - expect failure
  87       this.admin.addColumnFamily(tableName, ColumnFamilyDescriptorBuilder.of(FAMILY_1)).join();
  88       Assert.fail("Delete a non-exist column family should fail");
  89     } catch (Exception e) {
  90       // Expected.
  91     }
  92   }
  93
  94   @Test
  95   public void testModifyColumnFamily() throws Exception {
  96     TableDescriptorBuilder tdBuilder = TableDescriptorBuilder.newBuilder(tableName);
  97     ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.of(FAMILY_0);
  98     int blockSize = cfd.getBlocksize();
  99     admin.createTable(tdBuilder.setColumnFamily(cfd).build()).join();
 100     admin.disableTable(tableName).join();
 101     // Verify the table descriptor
 102     verifyTableDescriptor(tableName, FAMILY_0);
 103
 104     int newBlockSize = 2 * blockSize;
 105     cfd = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY_0).setBlocksize(newBlockSize).build();
 106     // Modify colymn family
 107     admin.modifyColumnFamily(tableName, cfd).join();
 108
 109     TableDescriptor htd = admin.getDescriptor(tableName).get();
 110     ColumnFamilyDescriptor hcfd = htd.getColumnFamily(FAMILY_0);
 111     assertTrue(hcfd.getBlocksize() == newBlockSize);
 112   }
 113
 114   @Test
 115   public void testModifyNonExistingColumnFamily() throws Exception {
 116     TableDescriptorBuilder tdBuilder = TableDescriptorBuilder.newBuilder(tableName);
 117     ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.of(FAMILY_0);
 118     int blockSize = cfd.getBlocksize();
 119     admin.createTable(tdBuilder.setColumnFamily(cfd).build()).join();
 120     admin.disableTable(tableName).join();
 121     // Verify the table descriptor
 122     verifyTableDescriptor(tableName, FAMILY_0);
 123
 124     int newBlockSize = 2 * blockSize;
 125     cfd = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY_1).setBlocksize(newBlockSize).build();
 126
 127     // Modify a column family that is not in the table.
 128     try {
 129       admin.modifyColumnFamily(tableName, cfd).join();
 130       Assert.fail("Modify a non-exist column family should fail");
 131     } catch (Exception e) {
 132       // Expected.
 133     }
 134   }
 135
 136   @Test
 137   public void testDeleteColumnFamily() throws Exception {
 138     // Create a table with two families
 139     TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
 140     builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_0))
 141         .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_1));
 142     admin.createTable(builder.build()).join();
 143     admin.disableTable(tableName).join();
 144     // Verify the table descriptor
 145     verifyTableDescriptor(tableName, FAMILY_0, FAMILY_1);
 146
 147     // Modify the table removing one family and verify the descriptor
 148     admin.deleteColumnFamily(tableName, FAMILY_1).join();
 149     verifyTableDescriptor(tableName, FAMILY_0);
 150   }
 151
 152   @Test
 153   public void testDeleteSameColumnFamilyTwice() throws Exception {
 154     // Create a table with two families
 155     TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
 156     builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_0))
 157         .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_1));
 158     admin.createTable(builder.build()).join();
 159     admin.disableTable(tableName).join();
 160     // Verify the table descriptor
 161     verifyTableDescriptor(tableName, FAMILY_0, FAMILY_1);
 162
 163     // Modify the table removing one family and verify the descriptor
 164     admin.deleteColumnFamily(tableName, FAMILY_1).join();
 165     verifyTableDescriptor(tableName, FAMILY_0);
 166
 167     try {
 168       // Delete again - expect failure
 169       admin.deleteColumnFamily(tableName, FAMILY_1).join();
 170       Assert.fail("Delete a non-exist column family should fail");
 171     } catch (Exception e) {
 172       // Expected.
 173     }
 174   }
 175
 176   private void verifyTableDescriptor(final TableName tableName, final byte[]... families)
 177       throws Exception {
 178     // Verify descriptor from master
 179     TableDescriptor htd = admin.getDescriptor(tableName).get();
 180     verifyTableDescriptor(htd, tableName, families);
 181
 182     // Verify descriptor from HDFS
 183     MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
 184     Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName);
 185     TableDescriptor td = FSTableDescriptors.getTableDescriptorFromFs(mfs.getFileSystem(), tableDir);
 186     verifyTableDescriptor(td, tableName, families);
 187   }
 188
 189   private void verifyTableDescriptor(final TableDescriptor htd, final TableName tableName,
 190       final byte[]... families) {
 191     Set<byte[]> htdFamilies = htd.getColumnFamilyNames();
 192     assertEquals(tableName, htd.getTableName());
 193     assertEquals(families.length, htdFamilies.size());
 194     for (byte[] familyName : families) {
 195       assertTrue("Expected family " + Bytes.toString(familyName), htdFamilies.contains(familyName));
 196     }
 197   }
 198
 199   @Test
 200   public void testCompactionTimestamps() throws Exception {
 201     createTableWithDefaultConf(tableName);
 202     AsyncTable<?> table = ASYNC_CONN.getTable(tableName);
 203     Optional<Long> ts = admin.getLastMajorCompactionTimestamp(tableName).get();
 204     assertFalse(ts.isPresent());
 205     Put p = new Put(Bytes.toBytes("row1"));
 206     p.addColumn(FAMILY, Bytes.toBytes("q"), Bytes.toBytes("v"));
 207     table.put(p).join();
 208     ts = admin.getLastMajorCompactionTimestamp(tableName).get();
 209     // no files written -> no data
 210     assertFalse(ts.isPresent());
 211
 212     admin.flush(tableName).join();
 213     ts = admin.getLastMajorCompactionTimestamp(tableName).get();
 214     // still 0, we flushed a file, but no major compaction happened
 215     assertFalse(ts.isPresent());
 216
 217     byte[] regionName = ASYNC_CONN.getRegionLocator(tableName)
 218         .getRegionLocation(Bytes.toBytes("row1")).get().getRegion().getRegionName();
 219     Optional<Long> ts1 = admin.getLastMajorCompactionTimestampForRegion(regionName).get();
 220     assertFalse(ts1.isPresent());
 221     p = new Put(Bytes.toBytes("row2"));
 222     p.addColumn(FAMILY, Bytes.toBytes("q"), Bytes.toBytes("v"));
 223     table.put(p).join();
 224     admin.flush(tableName).join();
 225     ts1 = admin.getLastMajorCompactionTimestamp(tableName).get();
 226     // make sure the region API returns the same value, as the old file is still around
 227     assertFalse(ts1.isPresent());
 228
 229     for (int i = 0; i < 3; i++) {
 230       table.put(p).join();
 231       admin.flush(tableName).join();
 232     }
 233     admin.majorCompact(tableName).join();
 234     long curt = System.currentTimeMillis();
 235     long waitTime = 10000;
 236     long endt = curt + waitTime;
 237     CompactionState state = admin.getCompactionState(tableName).get();
 238     LOG.info("Current compaction state 1 is " + state);
 239     while (state == CompactionState.NONE && curt < endt) {
 240       Thread.sleep(100);
 241       state = admin.getCompactionState(tableName).get();
 242       curt = System.currentTimeMillis();
 243       LOG.info("Current compaction state 2 is " + state);
 244     }
 245     // Now, should have the right compaction state, let's wait until the compaction is done
 246     if (state == CompactionState.MAJOR) {
 247       state = admin.getCompactionState(tableName).get();
 248       LOG.info("Current compaction state 3 is " + state);
 249       while (state != CompactionState.NONE && curt < endt) {
 250         Thread.sleep(10);
 251         state = admin.getCompactionState(tableName).get();
 252         LOG.info("Current compaction state 4 is " + state);
 253       }
 254     }
 255     // Sleep to wait region server report
 256     Thread
 257         .sleep(TEST_UTIL.getConfiguration().getInt("hbase.regionserver.msginterval", 3 * 1000) * 2);
 258
 259     ts = admin.getLastMajorCompactionTimestamp(tableName).get();
 260     // after a compaction our earliest timestamp will have progressed forward
 261     assertTrue(ts.isPresent());
 262     assertTrue(ts.get() > 0);
 263     // region api still the same
 264     ts1 = admin.getLastMajorCompactionTimestampForRegion(regionName).get();
 265     assertTrue(ts1.isPresent());
 266     assertEquals(ts.get(), ts1.get());
 267     table.put(p).join();
 268     admin.flush(tableName).join();
 269     ts = admin.getLastMajorCompactionTimestamp(tableName).join();
 270     assertTrue(ts.isPresent());
 271     assertEquals(ts.get(), ts1.get());
 272     ts1 = admin.getLastMajorCompactionTimestampForRegion(regionName).get();
 273     assertTrue(ts1.isPresent());
 274     assertEquals(ts.get(), ts1.get());
 275   }
 276 }