HBASE-26582 Prune use of Random and SecureRandom objects (#4118)
[hbase.git] / hbase-server / src / test / java / org / apache / hadoop / hbase / TestMetaTableAccessor.java
blob94236b7b4cde2c1dc790eeb42ebb31edbbb54a43
1 /*
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
10 * http://www.apache.org/licenses/LICENSE-2.0
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
18 package org.apache.hadoop.hbase;
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertNotNull;
22 import static org.junit.Assert.assertNull;
23 import static org.junit.Assert.assertTrue;
24 import static org.mockito.ArgumentMatchers.anyObject;
25 import static org.mockito.Mockito.doReturn;
26 import static org.mockito.Mockito.mock;
27 import static org.mockito.Mockito.reset;
28 import static org.mockito.Mockito.times;
29 import static org.mockito.Mockito.verify;
31 import java.io.IOException;
32 import java.util.Collections;
33 import java.util.List;
34 import java.util.Random;
35 import java.util.concurrent.ThreadLocalRandom;
37 import org.apache.hadoop.conf.Configuration;
38 import org.apache.hadoop.hbase.client.Connection;
39 import org.apache.hadoop.hbase.client.ConnectionFactory;
40 import org.apache.hadoop.hbase.client.Get;
41 import org.apache.hadoop.hbase.client.RegionInfo;
42 import org.apache.hadoop.hbase.client.RegionInfoBuilder;
43 import org.apache.hadoop.hbase.client.RegionLocator;
44 import org.apache.hadoop.hbase.client.Result;
45 import org.apache.hadoop.hbase.client.Table;
46 import org.apache.hadoop.hbase.ipc.CallRunner;
47 import org.apache.hadoop.hbase.ipc.DelegatingRpcScheduler;
48 import org.apache.hadoop.hbase.ipc.PriorityFunction;
49 import org.apache.hadoop.hbase.ipc.RpcScheduler;
50 import org.apache.hadoop.hbase.master.HMaster;
51 import org.apache.hadoop.hbase.regionserver.HRegion;
52 import org.apache.hadoop.hbase.regionserver.HRegionServer;
53 import org.apache.hadoop.hbase.regionserver.SimpleRpcSchedulerFactory;
54 import org.apache.hadoop.hbase.testclassification.MediumTests;
55 import org.apache.hadoop.hbase.testclassification.MiscTests;
56 import org.apache.hadoop.hbase.util.Bytes;
57 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
58 import org.apache.hadoop.hbase.util.Pair;
59 import org.junit.AfterClass;
60 import org.junit.BeforeClass;
61 import org.junit.ClassRule;
62 import org.junit.Rule;
63 import org.junit.Test;
64 import org.junit.experimental.categories.Category;
65 import org.junit.rules.TestName;
66 import org.slf4j.Logger;
67 import org.slf4j.LoggerFactory;
69 import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
71 /**
72 * Test {@link org.apache.hadoop.hbase.MetaTableAccessor}.
74 @Category({ MiscTests.class, MediumTests.class })
75 @SuppressWarnings("deprecation")
76 public class TestMetaTableAccessor {
77 @ClassRule
78 public static final HBaseClassTestRule CLASS_RULE =
79 HBaseClassTestRule.forClass(TestMetaTableAccessor.class);
81 private static final Logger LOG = LoggerFactory.getLogger(TestMetaTableAccessor.class);
82 private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
83 private static Connection connection;
85 @Rule
86 public TestName name = new TestName();
88 @BeforeClass
89 public static void beforeClass() throws Exception {
90 UTIL.startMiniCluster(3);
92 Configuration c = new Configuration(UTIL.getConfiguration());
93 // Tests to 4 retries every 5 seconds. Make it try every 1 second so more
94 // responsive. 1 second is default as is ten retries.
95 c.setLong("hbase.client.pause", 1000);
96 c.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 10);
97 connection = ConnectionFactory.createConnection(c);
100 @AfterClass
101 public static void afterClass() throws Exception {
102 connection.close();
103 UTIL.shutdownMiniCluster();
106 @Test
107 public void testIsMetaWhenAllHealthy() throws InterruptedException {
108 HMaster m = UTIL.getMiniHBaseCluster().getMaster();
109 assertTrue(m.waitForMetaOnline());
112 @Test
113 public void testIsMetaWhenMetaGoesOffline() throws InterruptedException {
114 HMaster m = UTIL.getMiniHBaseCluster().getMaster();
115 int index = UTIL.getMiniHBaseCluster().getServerWithMeta();
116 HRegionServer rsWithMeta = UTIL.getMiniHBaseCluster().getRegionServer(index);
117 rsWithMeta.abort("TESTING");
118 assertTrue(m.waitForMetaOnline());
122 * Does {@link MetaTableAccessor#getRegion(Connection, byte[])} and a write against hbase:meta
123 * while its hosted server is restarted to prove our retrying works.
125 @Test
126 public void testRetrying() throws IOException, InterruptedException {
127 final TableName tableName = TableName.valueOf(name.getMethodName());
128 LOG.info("Started " + tableName);
129 Table t = UTIL.createMultiRegionTable(tableName, HConstants.CATALOG_FAMILY);
130 int regionCount = -1;
131 try (RegionLocator r = UTIL.getConnection().getRegionLocator(tableName)) {
132 regionCount = r.getStartKeys().length;
134 // Test it works getting a region from just made user table.
135 final List<RegionInfo> regions = testGettingTableRegions(connection, tableName, regionCount);
136 MetaTask reader = new MetaTask(connection, "reader") {
137 @Override
138 void metaTask() throws Throwable {
139 testGetRegion(connection, regions.get(0));
140 LOG.info("Read " + regions.get(0).getEncodedName());
143 MetaTask writer = new MetaTask(connection, "writer") {
145 @Override
146 void metaTask() throws IOException {
147 MetaTableAccessor.addRegionsToMeta(connection, Collections.singletonList(regions.get(0)),
149 LOG.info("Wrote " + regions.get(0).getEncodedName());
152 reader.start();
153 writer.start();
155 // We're gonna check how it takes. If it takes too long, we will consider
156 // it as a fail. We can't put that in the @Test tag as we want to close
157 // the threads nicely
158 final long timeOut = 180000;
159 long startTime = EnvironmentEdgeManager.currentTime();
161 try {
162 // Make sure reader and writer are working.
163 assertTrue(reader.isProgressing());
164 assertTrue(writer.isProgressing());
166 // Kill server hosting meta -- twice . See if our reader/writer ride over the
167 // meta moves. They'll need to retry.
168 for (int i = 0; i < 2; i++) {
169 LOG.info("Restart=" + i);
170 UTIL.ensureSomeRegionServersAvailable(2);
171 int index = -1;
172 do {
173 index = UTIL.getMiniHBaseCluster().getServerWithMeta();
174 } while (index == -1 && startTime + timeOut < EnvironmentEdgeManager.currentTime());
176 if (index != -1) {
177 UTIL.getMiniHBaseCluster().abortRegionServer(index);
178 UTIL.getMiniHBaseCluster().waitOnRegionServer(index);
182 assertTrue("reader: " + reader.toString(), reader.isProgressing());
183 assertTrue("writer: " + writer.toString(), writer.isProgressing());
184 } catch (IOException e) {
185 throw e;
186 } finally {
187 reader.stop = true;
188 writer.stop = true;
189 reader.join();
190 writer.join();
191 t.close();
193 long exeTime = EnvironmentEdgeManager.currentTime() - startTime;
194 assertTrue("Timeout: test took " + exeTime / 1000 + " sec", exeTime < timeOut);
198 * Thread that runs a MetaTableAccessor task until asked stop.
200 abstract static class MetaTask extends Thread {
201 boolean stop = false;
202 int count = 0;
203 Throwable t = null;
204 final Connection connection;
206 MetaTask(final Connection connection, final String name) {
207 super(name);
208 this.connection = connection;
211 @Override
212 public void run() {
213 try {
214 while (!this.stop) {
215 LOG.info("Before " + this.getName() + ", count=" + this.count);
216 metaTask();
217 this.count += 1;
218 LOG.info("After " + this.getName() + ", count=" + this.count);
219 Thread.sleep(100);
221 } catch (Throwable t) {
222 LOG.info(this.getName() + " failed", t);
223 this.t = t;
227 boolean isProgressing() throws InterruptedException {
228 int currentCount = this.count;
229 while (currentCount == this.count) {
230 if (!isAlive()) return false;
231 if (this.t != null) return false;
232 Thread.sleep(10);
234 return true;
237 @Override
238 public String toString() {
239 return "count=" + this.count + ", t=" + (this.t == null ? "null" : this.t.toString());
242 abstract void metaTask() throws Throwable;
245 @Test
246 public void testGetRegion() throws IOException, InterruptedException {
247 final String name = this.name.getMethodName();
248 LOG.info("Started " + name);
249 // Test get on non-existent region.
250 Pair<RegionInfo, ServerName> pair =
251 MetaTableAccessor.getRegion(connection, Bytes.toBytes("nonexistent-region"));
252 assertNull(pair);
253 LOG.info("Finished " + name);
256 // Test for the optimization made in HBASE-3650
257 @Test
258 public void testScanMetaForTable() throws IOException, InterruptedException {
259 final TableName tableName = TableName.valueOf(name.getMethodName());
260 LOG.info("Started " + tableName);
263 * Create 2 tables - testScanMetaForTable - testScanMetaForTablf
266 UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
267 // name that is +1 greater than the first one (e+1=f)
268 TableName greaterName = TableName.valueOf("testScanMetaForTablf");
269 UTIL.createTable(greaterName, HConstants.CATALOG_FAMILY);
271 // Now make sure we only get the regions from 1 of the tables at a time
273 assertEquals(1, MetaTableAccessor.getTableRegions(connection, tableName).size());
274 assertEquals(1, MetaTableAccessor.getTableRegions(connection, greaterName).size());
277 private static List<RegionInfo> testGettingTableRegions(final Connection connection,
278 final TableName name, final int regionCount) throws IOException, InterruptedException {
279 List<RegionInfo> regions = MetaTableAccessor.getTableRegions(connection, name);
280 assertEquals(regionCount, regions.size());
281 Pair<RegionInfo, ServerName> pair =
282 MetaTableAccessor.getRegion(connection, regions.get(0).getRegionName());
283 assertEquals(regions.get(0).getEncodedName(), pair.getFirst().getEncodedName());
284 return regions;
287 private static void testGetRegion(final Connection connection, final RegionInfo region)
288 throws IOException, InterruptedException {
289 Pair<RegionInfo, ServerName> pair =
290 MetaTableAccessor.getRegion(connection, region.getRegionName());
291 assertEquals(region.getEncodedName(), pair.getFirst().getEncodedName());
294 @Test
295 public void testMetaLocationsForRegionReplicas() throws IOException {
296 Random rand = ThreadLocalRandom.current();
298 ServerName serverName0 = ServerName.valueOf("foo", 60010, rand.nextLong());
299 ServerName serverName1 = ServerName.valueOf("bar", 60010, rand.nextLong());
300 ServerName serverName100 = ServerName.valueOf("baz", 60010, rand.nextLong());
302 long regionId = EnvironmentEdgeManager.currentTime();
303 RegionInfo primary = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
304 .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false)
305 .setRegionId(regionId).setReplicaId(0).build();
306 RegionInfo replica1 = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
307 .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false)
308 .setRegionId(regionId).setReplicaId(1).build();
309 RegionInfo replica100 = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
310 .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false)
311 .setRegionId(regionId).setReplicaId(100).build();
313 long seqNum0 = rand.nextLong();
314 long seqNum1 = rand.nextLong();
315 long seqNum100 = rand.nextLong();
317 try (Table meta = MetaTableAccessor.getMetaHTable(connection)) {
318 MetaTableAccessor.updateRegionLocation(connection, primary, serverName0, seqNum0,
319 EnvironmentEdgeManager.currentTime());
321 // assert that the server, startcode and seqNum columns are there for the primary region
322 assertMetaLocation(meta, primary.getRegionName(), serverName0, seqNum0, 0, true);
324 // add replica = 1
325 MetaTableAccessor.updateRegionLocation(connection, replica1, serverName1, seqNum1,
326 EnvironmentEdgeManager.currentTime());
327 // check whether the primary is still there
328 assertMetaLocation(meta, primary.getRegionName(), serverName0, seqNum0, 0, true);
329 // now check for replica 1
330 assertMetaLocation(meta, primary.getRegionName(), serverName1, seqNum1, 1, true);
332 // add replica = 1
333 MetaTableAccessor.updateRegionLocation(connection, replica100, serverName100, seqNum100,
334 EnvironmentEdgeManager.currentTime());
335 // check whether the primary is still there
336 assertMetaLocation(meta, primary.getRegionName(), serverName0, seqNum0, 0, true);
337 // check whether the replica 1 is still there
338 assertMetaLocation(meta, primary.getRegionName(), serverName1, seqNum1, 1, true);
339 // now check for replica 1
340 assertMetaLocation(meta, primary.getRegionName(), serverName100, seqNum100, 100, true);
344 public static void assertMetaLocation(Table meta, byte[] row, ServerName serverName, long seqNum,
345 int replicaId, boolean checkSeqNum) throws IOException {
346 Get get = new Get(row);
347 Result result = meta.get(get);
348 assertTrue(Bytes.equals(
349 result.getValue(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getServerColumn(replicaId)),
350 Bytes.toBytes(serverName.getAddress().toString())));
351 assertTrue(Bytes.equals(
352 result.getValue(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getStartCodeColumn(replicaId)),
353 Bytes.toBytes(serverName.getStartcode())));
354 if (checkSeqNum) {
355 assertTrue(Bytes.equals(
356 result.getValue(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getSeqNumColumn(replicaId)),
357 Bytes.toBytes(seqNum)));
361 public static void assertEmptyMetaLocation(Table meta, byte[] row, int replicaId)
362 throws IOException {
363 Get get = new Get(row);
364 Result result = meta.get(get);
365 Cell serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
366 CatalogFamilyFormat.getServerColumn(replicaId));
367 Cell startCodeCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
368 CatalogFamilyFormat.getStartCodeColumn(replicaId));
369 assertNotNull(serverCell);
370 assertNotNull(startCodeCell);
371 assertEquals(0, serverCell.getValueLength());
372 assertEquals(0, startCodeCell.getValueLength());
375 @Test
376 public void testMetaLocationForRegionReplicasIsAddedAtTableCreation() throws IOException {
377 long regionId = EnvironmentEdgeManager.currentTime();
378 RegionInfo primary = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
379 .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false)
380 .setRegionId(regionId).setReplicaId(0).build();
382 Table meta = MetaTableAccessor.getMetaHTable(connection);
383 try {
384 List<RegionInfo> regionInfos = Lists.newArrayList(primary);
385 MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3);
387 assertEmptyMetaLocation(meta, primary.getRegionName(), 1);
388 assertEmptyMetaLocation(meta, primary.getRegionName(), 2);
389 } finally {
390 meta.close();
394 @Test
395 public void testMetaScanner() throws Exception {
396 LOG.info("Starting " + name.getMethodName());
398 final TableName tableName = TableName.valueOf(name.getMethodName());
399 final byte[] FAMILY = Bytes.toBytes("family");
400 final byte[][] SPLIT_KEYS =
401 new byte[][] { Bytes.toBytes("region_a"), Bytes.toBytes("region_b") };
403 UTIL.createTable(tableName, FAMILY, SPLIT_KEYS);
404 Table table = connection.getTable(tableName);
405 // Make sure all the regions are deployed
406 HBaseTestingUtil.countRows(table);
408 ClientMetaTableAccessor.Visitor visitor = mock(ClientMetaTableAccessor.Visitor.class);
409 doReturn(true).when(visitor).visit((Result) anyObject());
411 // Scanning the entire table should give us three rows
412 MetaTableAccessor.scanMetaForTableRegions(connection, visitor, tableName);
413 verify(visitor, times(3)).visit((Result) anyObject());
415 // Scanning the table with a specified empty start row should also
416 // give us three hbase:meta rows
417 reset(visitor);
418 doReturn(true).when(visitor).visit((Result) anyObject());
419 MetaTableAccessor.scanMeta(connection, visitor, tableName, null, 1000);
420 verify(visitor, times(3)).visit((Result) anyObject());
422 // Scanning the table starting in the middle should give us two rows:
423 // region_a and region_b
424 reset(visitor);
425 doReturn(true).when(visitor).visit((Result) anyObject());
426 MetaTableAccessor.scanMeta(connection, visitor, tableName, Bytes.toBytes("region_ac"), 1000);
427 verify(visitor, times(2)).visit((Result) anyObject());
429 // Scanning with a limit of 1 should only give us one row
430 reset(visitor);
431 doReturn(true).when(visitor).visit((Result) anyObject());
432 MetaTableAccessor.scanMeta(connection, visitor, tableName, Bytes.toBytes("region_ac"), 1);
433 verify(visitor, times(1)).visit((Result) anyObject());
434 table.close();
438 * Tests whether maximum of masters system time versus RSs local system time is used
440 @Test
441 public void testMastersSystemTimeIsUsedInUpdateLocations() throws IOException {
442 long regionId = EnvironmentEdgeManager.currentTime();
443 RegionInfo regionInfo = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
444 .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false)
445 .setRegionId(regionId).setReplicaId(0).build();
447 ServerName sn = ServerName.valueOf("bar", 0, 0);
448 try (Table meta = MetaTableAccessor.getMetaHTable(connection)) {
449 List<RegionInfo> regionInfos = Lists.newArrayList(regionInfo);
450 MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 1);
452 long masterSystemTime = EnvironmentEdgeManager.currentTime() + 123456789;
453 MetaTableAccessor.updateRegionLocation(connection, regionInfo, sn, 1, masterSystemTime);
455 Get get = new Get(regionInfo.getRegionName());
456 Result result = meta.get(get);
457 Cell serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
458 CatalogFamilyFormat.getServerColumn(0));
459 Cell startCodeCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
460 CatalogFamilyFormat.getStartCodeColumn(0));
461 Cell seqNumCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
462 CatalogFamilyFormat.getSeqNumColumn(0));
463 assertNotNull(serverCell);
464 assertNotNull(startCodeCell);
465 assertNotNull(seqNumCell);
466 assertTrue(serverCell.getValueLength() > 0);
467 assertTrue(startCodeCell.getValueLength() > 0);
468 assertTrue(seqNumCell.getValueLength() > 0);
469 assertEquals(masterSystemTime, serverCell.getTimestamp());
470 assertEquals(masterSystemTime, startCodeCell.getTimestamp());
471 assertEquals(masterSystemTime, seqNumCell.getTimestamp());
475 public static class SpyingRpcSchedulerFactory extends SimpleRpcSchedulerFactory {
476 @Override
477 public RpcScheduler create(Configuration conf, PriorityFunction priority, Abortable server) {
478 final RpcScheduler delegate = super.create(conf, priority, server);
479 return new SpyingRpcScheduler(delegate);
483 public static class SpyingRpcScheduler extends DelegatingRpcScheduler {
484 long numPriorityCalls = 0;
486 public SpyingRpcScheduler(RpcScheduler delegate) {
487 super(delegate);
490 @Override
491 public boolean dispatch(CallRunner task) throws IOException, InterruptedException {
492 int priority = task.getRpcCall().getPriority();
494 if (priority > HConstants.QOS_THRESHOLD) {
495 numPriorityCalls++;
497 return super.dispatch(task);
501 @Test
502 public void testScanByRegionEncodedNameExistingRegion() throws Exception {
503 final TableName tableName = TableName.valueOf("testScanByRegionEncodedNameExistingRegion");
504 UTIL.createTable(tableName, "cf");
505 final List<HRegion> regions = UTIL.getHBaseCluster().getRegions(tableName);
506 final String encodedName = regions.get(0).getRegionInfo().getEncodedName();
507 final Result result =
508 MetaTableAccessor.scanByRegionEncodedName(UTIL.getConnection(), encodedName);
509 assertNotNull(result);
510 assertTrue(result.advance());
511 final String resultingRowKey = CellUtil.getCellKeyAsString(result.current());
512 assertTrue(resultingRowKey.contains(encodedName));
513 UTIL.deleteTable(tableName);
516 @Test
517 public void testScanByRegionEncodedNameNonExistingRegion() throws Exception {
518 final String encodedName = "nonexistingregion";
519 final Result result =
520 MetaTableAccessor.scanByRegionEncodedName(UTIL.getConnection(), encodedName);
521 assertNull(result);