HBASE-24609 Move MetaTableAccessor out of hbase-client (#1943)
[hbase.git] / hbase-server / src / test / java / org / apache / hadoop / hbase / client / TestAsyncTableAdminApi.java
blob52ebc168674863991cc154b3530f7aa5f16d04b1
1 /**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
10 * http://www.apache.org/licenses/LICENSE-2.0
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
18 package org.apache.hadoop.hbase.client;
20 import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME;
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25 import java.util.ArrayList;
26 import java.util.HashMap;
27 import java.util.Iterator;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.Optional;
31 import java.util.concurrent.CompletionException;
32 import org.apache.hadoop.hbase.ClientMetaTableAccessor;
33 import org.apache.hadoop.hbase.HBaseClassTestRule;
34 import org.apache.hadoop.hbase.HConstants;
35 import org.apache.hadoop.hbase.HRegionLocation;
36 import org.apache.hadoop.hbase.ServerName;
37 import org.apache.hadoop.hbase.TableExistsException;
38 import org.apache.hadoop.hbase.TableName;
39 import org.apache.hadoop.hbase.TableNotFoundException;
40 import org.apache.hadoop.hbase.master.LoadBalancer;
41 import org.apache.hadoop.hbase.testclassification.ClientTests;
42 import org.apache.hadoop.hbase.testclassification.LargeTests;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.junit.ClassRule;
45 import org.junit.Test;
46 import org.junit.experimental.categories.Category;
47 import org.junit.runner.RunWith;
48 import org.junit.runners.Parameterized;
50 /**
51 * Class to test asynchronous table admin operations.
52 * @see TestAsyncTableAdminApi2 This test and it used to be joined it was taking longer than our
53 * ten minute timeout so they were split.
54 * @see TestAsyncTableAdminApi3 Another split out from this class so each runs under ten minutes.
56 @RunWith(Parameterized.class)
57 @Category({ LargeTests.class, ClientTests.class })
58 public class TestAsyncTableAdminApi extends TestAsyncAdminBase {
60 @ClassRule
61 public static final HBaseClassTestRule CLASS_RULE =
62 HBaseClassTestRule.forClass(TestAsyncTableAdminApi.class);
64 @Test
65 public void testCreateTable() throws Exception {
66 List<TableDescriptor> tables = admin.listTableDescriptors().get();
67 int numTables = tables.size();
68 createTableWithDefaultConf(tableName);
69 tables = admin.listTableDescriptors().get();
70 assertEquals(numTables + 1, tables.size());
71 assertTrue("Table must be enabled.", TEST_UTIL.getHBaseCluster().getMaster()
72 .getTableStateManager().isTableState(tableName, TableState.State.ENABLED));
73 assertEquals(TableState.State.ENABLED, getStateFromMeta(tableName));
76 static TableState.State getStateFromMeta(TableName table) throws Exception {
77 Optional<TableState> state = ClientMetaTableAccessor
78 .getTableState(ASYNC_CONN.getTable(TableName.META_TABLE_NAME), table).get();
79 assertTrue(state.isPresent());
80 return state.get().getState();
83 @Test
84 public void testCreateTableNumberOfRegions() throws Exception {
85 AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME);
87 createTableWithDefaultConf(tableName);
88 List<HRegionLocation> regionLocations = ClientMetaTableAccessor
89 .getTableHRegionLocations(metaTable, tableName).get();
90 assertEquals("Table should have only 1 region", 1, regionLocations.size());
92 final TableName tableName2 = TableName.valueOf(tableName.getNameAsString() + "_2");
93 createTableWithDefaultConf(tableName2, new byte[][] { new byte[] { 42 } });
94 regionLocations = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName2).get();
95 assertEquals("Table should have only 2 region", 2, regionLocations.size());
97 final TableName tableName3 = TableName.valueOf(tableName.getNameAsString() + "_3");
98 TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName3);
99 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
100 admin.createTable(builder.build(), Bytes.toBytes("a"), Bytes.toBytes("z"), 3).join();
101 regionLocations = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName3).get();
102 assertEquals("Table should have only 3 region", 3, regionLocations.size());
104 final TableName tableName4 = TableName.valueOf(tableName.getNameAsString() + "_4");
105 builder = TableDescriptorBuilder.newBuilder(tableName4);
106 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
107 try {
108 admin.createTable(builder.build(), Bytes.toBytes("a"), Bytes.toBytes("z"), 2).join();
109 fail("Should not be able to create a table with only 2 regions using this API.");
110 } catch (CompletionException e) {
111 assertTrue(e.getCause() instanceof IllegalArgumentException);
114 final TableName tableName5 = TableName.valueOf(tableName.getNameAsString() + "_5");
115 builder = TableDescriptorBuilder.newBuilder(tableName5);
116 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
117 admin.createTable(builder.build(), new byte[] { 1 }, new byte[] { 127 }, 16).join();
118 regionLocations = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName5).get();
119 assertEquals("Table should have 16 region", 16, regionLocations.size());
122 @Test
123 public void testCreateTableWithRegions() throws Exception {
124 byte[][] splitKeys = { new byte[] { 1, 1, 1 }, new byte[] { 2, 2, 2 }, new byte[] { 3, 3, 3 },
125 new byte[] { 4, 4, 4 }, new byte[] { 5, 5, 5 }, new byte[] { 6, 6, 6 },
126 new byte[] { 7, 7, 7 }, new byte[] { 8, 8, 8 }, new byte[] { 9, 9, 9 }, };
127 int expectedRegions = splitKeys.length + 1;
128 boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration());
129 createTableWithDefaultConf(tableName, splitKeys);
131 boolean tableAvailable = admin.isTableAvailable(tableName).get();
132 assertTrue("Table should be created with splitKyes + 1 rows in META", tableAvailable);
134 AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME);
135 List<HRegionLocation> regions = ClientMetaTableAccessor
136 .getTableHRegionLocations(metaTable, tableName).get();
137 Iterator<HRegionLocation> hris = regions.iterator();
139 assertEquals(
140 "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(),
141 expectedRegions, regions.size());
142 System.err.println("Found " + regions.size() + " regions");
144 RegionInfo hri;
145 hris = regions.iterator();
146 hri = hris.next().getRegion();
147 assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0);
148 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[0]));
149 hri = hris.next().getRegion();
150 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[0]));
151 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[1]));
152 hri = hris.next().getRegion();
153 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[1]));
154 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[2]));
155 hri = hris.next().getRegion();
156 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[2]));
157 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[3]));
158 hri = hris.next().getRegion();
159 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[3]));
160 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[4]));
161 hri = hris.next().getRegion();
162 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[4]));
163 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[5]));
164 hri = hris.next().getRegion();
165 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[5]));
166 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[6]));
167 hri = hris.next().getRegion();
168 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[6]));
169 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[7]));
170 hri = hris.next().getRegion();
171 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[7]));
172 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[8]));
173 hri = hris.next().getRegion();
174 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[8]));
175 assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0);
176 if (tablesOnMaster) {
177 verifyRoundRobinDistribution(regions, expectedRegions);
180 // Now test using start/end with a number of regions
182 // Use 80 bit numbers to make sure we aren't limited
183 byte[] startKey = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
184 byte[] endKey = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 };
186 // Splitting into 10 regions, we expect (null,1) ... (9, null)
187 // with (1,2) (2,3) (3,4) (4,5) (5,6) (6,7) (7,8) (8,9) in the middle
188 expectedRegions = 10;
189 final TableName tableName2 = TableName.valueOf(tableName.getNameAsString() + "_2");
190 TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName2);
191 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
192 admin.createTable(builder.build(), startKey, endKey, expectedRegions).join();
194 regions = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName2).get();
195 assertEquals(
196 "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(),
197 expectedRegions, regions.size());
198 System.err.println("Found " + regions.size() + " regions");
200 hris = regions.iterator();
201 hri = hris.next().getRegion();
202 assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0);
203 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }));
204 hri = hris.next().getRegion();
205 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }));
206 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 }));
207 hri = hris.next().getRegion();
208 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 }));
209 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 }));
210 hri = hris.next().getRegion();
211 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 }));
212 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 }));
213 hri = hris.next().getRegion();
214 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 }));
215 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 }));
216 hri = hris.next().getRegion();
217 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 }));
218 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 6, 6, 6, 6, 6, 6, 6, 6, 6, 6 }));
219 hri = hris.next().getRegion();
220 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 6, 6, 6, 6, 6, 6, 6, 6, 6, 6 }));
221 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 }));
222 hri = hris.next().getRegion();
223 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 }));
224 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 8, 8, 8, 8, 8, 8, 8, 8, 8, 8 }));
225 hri = hris.next().getRegion();
226 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 8, 8, 8, 8, 8, 8, 8, 8, 8, 8 }));
227 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }));
228 hri = hris.next().getRegion();
229 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }));
230 assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0);
231 if (tablesOnMaster) {
232 // This don't work if master is not carrying regions. FIX. TODO.
233 verifyRoundRobinDistribution(regions, expectedRegions);
236 // Try once more with something that divides into something infinite
237 startKey = new byte[] { 0, 0, 0, 0, 0, 0 };
238 endKey = new byte[] { 1, 0, 0, 0, 0, 0 };
240 expectedRegions = 5;
241 final TableName tableName3 = TableName.valueOf(tableName.getNameAsString() + "_3");
242 builder = TableDescriptorBuilder.newBuilder(tableName3);
243 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
244 admin.createTable(builder.build(), startKey, endKey, expectedRegions).join();
246 regions = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName3)
247 .get();
248 assertEquals(
249 "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(),
250 expectedRegions, regions.size());
251 System.err.println("Found " + regions.size() + " regions");
252 if (tablesOnMaster) {
253 // This don't work if master is not carrying regions. FIX. TODO.
254 verifyRoundRobinDistribution(regions, expectedRegions);
257 // Try an invalid case where there are duplicate split keys
258 splitKeys = new byte[][] { new byte[] { 1, 1, 1 }, new byte[] { 2, 2, 2 },
259 new byte[] { 3, 3, 3 }, new byte[] { 2, 2, 2 } };
260 final TableName tableName4 = TableName.valueOf(tableName.getNameAsString() + "_4");
261 try {
262 createTableWithDefaultConf(tableName4, splitKeys);
263 fail("Should not be able to create this table because of " + "duplicate split keys");
264 } catch (CompletionException e) {
265 assertTrue(e.getCause() instanceof IllegalArgumentException);
269 private void verifyRoundRobinDistribution(List<HRegionLocation> regions, int expectedRegions) {
270 int numRS = TEST_UTIL.getMiniHBaseCluster().getNumLiveRegionServers();
272 Map<ServerName, List<RegionInfo>> server2Regions = new HashMap<>();
273 regions.stream().forEach((loc) -> {
274 ServerName server = loc.getServerName();
275 server2Regions.computeIfAbsent(server, (s) -> new ArrayList<>()).add(loc.getRegion());
277 if (numRS >= 2) {
278 // Ignore the master region server,
279 // which contains less regions by intention.
280 numRS--;
282 float average = (float) expectedRegions / numRS;
283 int min = (int) Math.floor(average);
284 int max = (int) Math.ceil(average);
285 server2Regions.values().forEach((regionList) -> {
286 assertTrue(regionList.size() == min || regionList.size() == max);
290 @Test
291 public void testCreateTableWithOnlyEmptyStartRow() throws Exception {
292 byte[][] splitKeys = new byte[1][];
293 splitKeys[0] = HConstants.EMPTY_BYTE_ARRAY;
294 try {
295 createTableWithDefaultConf(tableName, splitKeys);
296 fail("Test case should fail as empty split key is passed.");
297 } catch (CompletionException e) {
298 assertTrue(e.getCause() instanceof IllegalArgumentException);
302 @Test
303 public void testCreateTableWithEmptyRowInTheSplitKeys() throws Exception {
304 byte[][] splitKeys = new byte[3][];
305 splitKeys[0] = Bytes.toBytes("region1");
306 splitKeys[1] = HConstants.EMPTY_BYTE_ARRAY;
307 splitKeys[2] = Bytes.toBytes("region2");
308 try {
309 createTableWithDefaultConf(tableName, splitKeys);
310 fail("Test case should fail as empty split key is passed.");
311 } catch (CompletionException e) {
312 assertTrue(e.getCause() instanceof IllegalArgumentException);
316 @Test
317 public void testDeleteTable() throws Exception {
318 createTableWithDefaultConf(tableName);
319 assertTrue(admin.tableExists(tableName).get());
320 TEST_UTIL.getAdmin().disableTable(tableName);
321 admin.deleteTable(tableName).join();
322 assertFalse(admin.tableExists(tableName).get());
325 @Test
326 public void testTruncateTable() throws Exception {
327 testTruncateTable(tableName, false);
330 @Test
331 public void testTruncateTablePreservingSplits() throws Exception {
332 testTruncateTable(tableName, true);
335 private void testTruncateTable(final TableName tableName, boolean preserveSplits)
336 throws Exception {
337 byte[][] splitKeys = new byte[2][];
338 splitKeys[0] = Bytes.toBytes(4);
339 splitKeys[1] = Bytes.toBytes(8);
341 // Create & Fill the table
342 createTableWithDefaultConf(tableName, splitKeys);
343 AsyncTable<?> table = ASYNC_CONN.getTable(tableName);
344 int expectedRows = 10;
345 for (int i = 0; i < expectedRows; i++) {
346 byte[] data = Bytes.toBytes(String.valueOf(i));
347 Put put = new Put(data);
348 put.addColumn(FAMILY, null, data);
349 table.put(put).join();
351 assertEquals(10, table.scanAll(new Scan()).get().size());
352 assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
354 // Truncate & Verify
355 admin.disableTable(tableName).join();
356 admin.truncateTable(tableName, preserveSplits).join();
357 assertEquals(0, table.scanAll(new Scan()).get().size());
358 if (preserveSplits) {
359 assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
360 } else {
361 assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
365 @Test
366 public void testCloneTableSchema() throws Exception {
367 final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
368 testCloneTableSchema(tableName, newTableName, false);
371 @Test
372 public void testCloneTableSchemaPreservingSplits() throws Exception {
373 final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
374 testCloneTableSchema(tableName, newTableName, true);
377 private void testCloneTableSchema(final TableName tableName,
378 final TableName newTableName, boolean preserveSplits) throws Exception {
379 byte[][] splitKeys = new byte[2][];
380 splitKeys[0] = Bytes.toBytes(4);
381 splitKeys[1] = Bytes.toBytes(8);
382 int NUM_FAMILYS = 2;
383 int NUM_REGIONS = 3;
384 int BLOCK_SIZE = 1024;
385 int TTL = 86400;
386 boolean BLOCK_CACHE = false;
388 // Create the table
389 TableDescriptor tableDesc = TableDescriptorBuilder
390 .newBuilder(tableName)
391 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_0))
392 .setColumnFamily(ColumnFamilyDescriptorBuilder
393 .newBuilder(FAMILY_1)
394 .setBlocksize(BLOCK_SIZE)
395 .setBlockCacheEnabled(BLOCK_CACHE)
396 .setTimeToLive(TTL)
397 .build()).build();
398 admin.createTable(tableDesc, splitKeys).join();
400 assertEquals(NUM_REGIONS, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
401 assertTrue("Table should be created with splitKyes + 1 rows in META",
402 admin.isTableAvailable(tableName).get());
404 // Clone & Verify
405 admin.cloneTableSchema(tableName, newTableName, preserveSplits).join();
406 TableDescriptor newTableDesc = admin.getDescriptor(newTableName).get();
408 assertEquals(NUM_FAMILYS, newTableDesc.getColumnFamilyCount());
409 assertEquals(BLOCK_SIZE, newTableDesc.getColumnFamily(FAMILY_1).getBlocksize());
410 assertEquals(BLOCK_CACHE, newTableDesc.getColumnFamily(FAMILY_1).isBlockCacheEnabled());
411 assertEquals(TTL, newTableDesc.getColumnFamily(FAMILY_1).getTimeToLive());
412 TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc);
414 if (preserveSplits) {
415 assertEquals(NUM_REGIONS, TEST_UTIL.getHBaseCluster().getRegions(newTableName).size());
416 assertTrue("New table should be created with splitKyes + 1 rows in META",
417 admin.isTableAvailable(newTableName).get());
418 } else {
419 assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(newTableName).size());
423 @Test
424 public void testCloneTableSchemaWithNonExistentSourceTable() throws Exception {
425 final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
426 // test for non-existent source table
427 try {
428 admin.cloneTableSchema(tableName, newTableName, false).join();
429 fail("Should have failed when source table doesn't exist.");
430 } catch (CompletionException e) {
431 assertTrue(e.getCause() instanceof TableNotFoundException);
435 @Test
436 public void testCloneTableSchemaWithExistentDestinationTable() throws Exception {
437 final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
438 byte[] FAMILY_0 = Bytes.toBytes("cf0");
439 TEST_UTIL.createTable(tableName, FAMILY_0);
440 TEST_UTIL.createTable(newTableName, FAMILY_0);
441 // test for existent destination table
442 try {
443 admin.cloneTableSchema(tableName, newTableName, false).join();
444 fail("Should have failed when destination table exists.");
445 } catch (CompletionException e) {
446 assertTrue(e.getCause() instanceof TableExistsException);
450 @Test
451 public void testIsTableAvailableWithInexistantTable() throws Exception {
452 final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
453 // test for inexistant table
454 assertFalse(admin.isTableAvailable(newTableName).get());