hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java

   1 /*
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.tool;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertFalse;
  22 import static org.junit.Assert.assertNotEquals;
  23 import static org.junit.Assert.assertNotNull;
  24 import static org.junit.Assert.assertTrue;
  25 import static org.mockito.ArgumentMatchers.argThat;
  26 import static org.mockito.Matchers.anyLong;
  27 import static org.mockito.Matchers.eq;
  28 import static org.mockito.Matchers.isA;
  29 import static org.mockito.Mockito.atLeastOnce;
  30 import static org.mockito.Mockito.never;
  31 import static org.mockito.Mockito.spy;
  32 import static org.mockito.Mockito.times;
  33 import static org.mockito.Mockito.verify;
  34 import java.util.List;
  35 import java.util.Map;
  36 import java.util.concurrent.ExecutorService;
  37 import java.util.concurrent.ScheduledThreadPoolExecutor;
  38 import org.apache.hadoop.hbase.HBaseClassTestRule;
  39 import org.apache.hadoop.hbase.HBaseTestingUtility;
  40 import org.apache.hadoop.hbase.HConstants;
  41 import org.apache.hadoop.hbase.ServerName;
  42 import org.apache.hadoop.hbase.TableName;
  43 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
  44 import org.apache.hadoop.hbase.client.Put;
  45 import org.apache.hadoop.hbase.client.RegionInfo;
  46 import org.apache.hadoop.hbase.client.Table;
  47 import org.apache.hadoop.hbase.testclassification.LargeTests;
  48 import org.apache.hadoop.hbase.util.Bytes;
  49 import org.apache.hadoop.util.ToolRunner;
  50 import org.apache.log4j.Appender;
  51 import org.apache.log4j.LogManager;
  52 import org.apache.log4j.spi.LoggingEvent;
  53 import org.junit.After;
  54 import org.junit.Before;
  55 import org.junit.ClassRule;
  56 import org.junit.Rule;
  57 import org.junit.Test;
  58 import org.junit.experimental.categories.Category;
  59 import org.junit.rules.TestName;
  60 import org.junit.runner.RunWith;
  61 import org.mockito.ArgumentMatcher;
  62 import org.mockito.Mock;
  63 import org.mockito.runners.MockitoJUnitRunner;
  64 import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
  65
  66 @RunWith(MockitoJUnitRunner.class)
  67 @Category({LargeTests.class})
  68 public class TestCanaryTool {
  69
  70   @ClassRule
  71   public static final HBaseClassTestRule CLASS_RULE =
  72       HBaseClassTestRule.forClass(TestCanaryTool.class);
  73
  74   private HBaseTestingUtility testingUtility;
  75   private static final byte[] FAMILY = Bytes.toBytes("f");
  76   private static final byte[] COLUMN = Bytes.toBytes("col");
  77
  78   @Rule
  79   public TestName name = new TestName();
  80
  81   @Before
  82   public void setUp() throws Exception {
  83     testingUtility = new HBaseTestingUtility();
  84     testingUtility.startMiniCluster();
  85     LogManager.getRootLogger().addAppender(mockAppender);
  86   }
  87
  88   @After
  89   public void tearDown() throws Exception {
  90     testingUtility.shutdownMiniCluster();
  91     LogManager.getRootLogger().removeAppender(mockAppender);
  92   }
  93
  94   @Mock
  95   Appender mockAppender;
  96
  97   @Test
  98   public void testBasicZookeeperCanaryWorks() throws Exception {
  99     final String[] args = { "-t", "10000", "-zookeeper" };
 100     testZookeeperCanaryWithArgs(args);
 101   }
 102
 103   @Test
 104   public void testZookeeperCanaryPermittedFailuresArgumentWorks() throws Exception {
 105     final String[] args = { "-t", "10000", "-zookeeper", "-treatFailureAsError",
 106       "-permittedZookeeperFailures", "1" };
 107     testZookeeperCanaryWithArgs(args);
 108   }
 109
 110   @Test
 111   public void testBasicCanaryWorks() throws Exception {
 112     final TableName tableName = TableName.valueOf(name.getMethodName());
 113     Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY });
 114     // insert some test rows
 115     for (int i=0; i<1000; i++) {
 116       byte[] iBytes = Bytes.toBytes(i);
 117       Put p = new Put(iBytes);
 118       p.addColumn(FAMILY, COLUMN, iBytes);
 119       table.put(p);
 120     }
 121     ExecutorService executor = new ScheduledThreadPoolExecutor(1);
 122     CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
 123     CanaryTool canary = new CanaryTool(executor, sink);
 124     String[] args = { "-writeSniffing", "-t", "10000", tableName.getNameAsString() };
 125     assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
 126     assertEquals("verify no read error count", 0, canary.getReadFailures().size());
 127     assertEquals("verify no write error count", 0, canary.getWriteFailures().size());
 128     verify(sink, atLeastOnce()).publishReadTiming(isA(ServerName.class), isA(RegionInfo.class),
 129       isA(ColumnFamilyDescriptor.class), anyLong());
 130   }
 131
 132   @Test
 133   public void testCanaryRegionTaskResult() throws Exception {
 134     TableName tableName = TableName.valueOf("testCanaryRegionTaskResult");
 135     Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY });
 136     // insert some test rows
 137     for (int i=0; i<1000; i++) {
 138       byte[] iBytes = Bytes.toBytes(i);
 139       Put p = new Put(iBytes);
 140       p.addColumn(FAMILY, COLUMN, iBytes);
 141       table.put(p);
 142     }
 143     ExecutorService executor = new ScheduledThreadPoolExecutor(1);
 144     CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
 145     CanaryTool canary = new CanaryTool(executor, sink);
 146     String[] args = { "-writeSniffing", "-t", "10000", "testCanaryRegionTaskResult" };
 147     assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
 148
 149     assertTrue("canary should expect to scan at least 1 region",
 150         sink.getTotalExpectedRegions() > 0);
 151     assertTrue("there should be no read failures", sink.getReadFailureCount() == 0);
 152     assertTrue("there should be no write failures", sink.getWriteFailureCount() == 0);
 153     assertTrue("verify read success count > 0", sink.getReadSuccessCount() > 0);
 154     assertTrue("verify write success count > 0", sink.getWriteSuccessCount() > 0);
 155     verify(sink, atLeastOnce()).publishReadTiming(isA(ServerName.class), isA(RegionInfo.class),
 156         isA(ColumnFamilyDescriptor.class), anyLong());
 157     verify(sink, atLeastOnce()).publishWriteTiming(isA(ServerName.class), isA(RegionInfo.class),
 158         isA(ColumnFamilyDescriptor.class), anyLong());
 159
 160     assertEquals("canary region success count should equal total expected regions",
 161         sink.getReadSuccessCount() + sink.getWriteSuccessCount(), sink.getTotalExpectedRegions());
 162     Map<String, List<CanaryTool.RegionTaskResult>> regionMap = sink.getRegionMap();
 163     assertFalse("verify region map has size > 0", regionMap.isEmpty());
 164
 165     for (String regionName : regionMap.keySet()) {
 166       for (CanaryTool.RegionTaskResult res: regionMap.get(regionName)) {
 167         assertNotNull("verify getRegionNameAsString()", regionName);
 168         assertNotNull("verify getRegionInfo()", res.getRegionInfo());
 169         assertNotNull("verify getTableName()", res.getTableName());
 170         assertNotNull("verify getTableNameAsString()", res.getTableNameAsString());
 171         assertNotNull("verify getServerName()", res.getServerName());
 172         assertNotNull("verify getServerNameAsString()", res.getServerNameAsString());
 173         assertNotNull("verify getColumnFamily()", res.getColumnFamily());
 174         assertNotNull("verify getColumnFamilyNameAsString()", res.getColumnFamilyNameAsString());
 175
 176         if (regionName.contains(CanaryTool.DEFAULT_WRITE_TABLE_NAME.getNameAsString())) {
 177           assertTrue("write to region " + regionName + " succeeded", res.isWriteSuccess());
 178           assertTrue("write took some time", res.getWriteLatency() > -1);
 179         } else {
 180           assertTrue("read from region " + regionName + " succeeded", res.isReadSuccess());
 181           assertTrue("read took some time", res.getReadLatency() > -1);
 182         }
 183       }
 184     }
 185   }
 186
 187   @Test
 188   public void testReadTableTimeouts() throws Exception {
 189     final TableName [] tableNames = new TableName[2];
 190     tableNames[0] = TableName.valueOf(name.getMethodName() + "1");
 191     tableNames[1] = TableName.valueOf(name.getMethodName() + "2");
 192     // Create 2 test tables.
 193     for (int j = 0; j<2; j++) {
 194       Table table = testingUtility.createTable(tableNames[j], new byte[][] { FAMILY });
 195       // insert some test rows
 196       for (int i=0; i<1000; i++) {
 197         byte[] iBytes = Bytes.toBytes(i + j);
 198         Put p = new Put(iBytes);
 199         p.addColumn(FAMILY, COLUMN, iBytes);
 200         table.put(p);
 201       }
 202     }
 203     ExecutorService executor = new ScheduledThreadPoolExecutor(1);
 204     CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
 205     CanaryTool canary = new CanaryTool(executor, sink);
 206     String configuredTimeoutStr = tableNames[0].getNameAsString() + "=" + Long.MAX_VALUE + "," +
 207       tableNames[1].getNameAsString() + "=0";
 208     String[] args = {"-readTableTimeouts", configuredTimeoutStr, name.getMethodName() + "1",
 209       name.getMethodName() + "2"};
 210     assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
 211     verify(sink, times(tableNames.length)).initializeAndGetReadLatencyForTable(isA(String.class));
 212     for (int i=0; i<2; i++) {
 213       assertNotEquals("verify non-null read latency", null,
 214         sink.getReadLatencyMap().get(tableNames[i].getNameAsString()));
 215       assertNotEquals("verify non-zero read latency", 0L,
 216         sink.getReadLatencyMap().get(tableNames[i].getNameAsString()));
 217     }
 218     // One table's timeout is set for 0 ms and thus, should lead to an error.
 219     verify(mockAppender, times(1)).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() {
 220       @Override
 221       public boolean matches(LoggingEvent argument) {
 222         return argument.getRenderedMessage().contains("exceeded the configured read timeout.");
 223       }
 224     }));
 225     verify(mockAppender, times(2)).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() {
 226       @Override
 227       public boolean matches(LoggingEvent argument) {
 228         return argument.getRenderedMessage().contains("Configured read timeout");
 229       }
 230     }));
 231   }
 232
 233   @Test
 234   public void testWriteTableTimeout() throws Exception {
 235     ExecutorService executor = new ScheduledThreadPoolExecutor(1);
 236     CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
 237     CanaryTool canary = new CanaryTool(executor, sink);
 238     String[] args = { "-writeSniffing", "-writeTableTimeout", String.valueOf(Long.MAX_VALUE)};
 239     assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
 240     assertNotEquals("verify non-null write latency", null, sink.getWriteLatency());
 241     assertNotEquals("verify non-zero write latency", 0L, sink.getWriteLatency());
 242     verify(mockAppender, times(1)).doAppend(argThat(
 243         new ArgumentMatcher<LoggingEvent>() {
 244           @Override
 245           public boolean matches(LoggingEvent argument) {
 246             return argument.getRenderedMessage().contains("Configured write timeout");
 247           }
 248         }));
 249   }
 250
 251   //no table created, so there should be no regions
 252   @Test
 253   public void testRegionserverNoRegions() throws Exception {
 254     runRegionserverCanary();
 255     verify(mockAppender).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() {
 256       @Override
 257       public boolean matches(LoggingEvent argument) {
 258         return argument.getRenderedMessage().contains("Regionserver not serving any regions");
 259       }
 260     }));
 261   }
 262
 263   //by creating a table, there shouldn't be any region servers not serving any regions
 264   @Test
 265   public void testRegionserverWithRegions() throws Exception {
 266     final TableName tableName = TableName.valueOf(name.getMethodName());
 267     testingUtility.createTable(tableName, new byte[][] { FAMILY });
 268     runRegionserverCanary();
 269     verify(mockAppender, never()).doAppend(argThat(new ArgumentMatcher<LoggingEvent>() {
 270       @Override
 271       public boolean matches(LoggingEvent argument) {
 272         return argument.getRenderedMessage().contains("Regionserver not serving any regions");
 273       }
 274     }));
 275   }
 276
 277   @Test
 278   public void testRawScanConfig() throws Exception {
 279     final TableName tableName = TableName.valueOf(name.getMethodName());
 280     Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY });
 281     // insert some test rows
 282     for (int i=0; i<1000; i++) {
 283       byte[] iBytes = Bytes.toBytes(i);
 284       Put p = new Put(iBytes);
 285       p.addColumn(FAMILY, COLUMN, iBytes);
 286       table.put(p);
 287     }
 288     ExecutorService executor = new ScheduledThreadPoolExecutor(1);
 289     CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
 290     CanaryTool canary = new CanaryTool(executor, sink);
 291     String[] args = { "-t", "10000", name.getMethodName() };
 292     org.apache.hadoop.conf.Configuration conf =
 293       new org.apache.hadoop.conf.Configuration(testingUtility.getConfiguration());
 294     conf.setBoolean(HConstants.HBASE_CANARY_READ_RAW_SCAN_KEY, true);
 295     assertEquals(0, ToolRunner.run(conf, canary, args));
 296     verify(sink, atLeastOnce())
 297         .publishReadTiming(isA(ServerName.class), isA(RegionInfo.class),
 298         isA(ColumnFamilyDescriptor.class), anyLong());
 299     assertEquals("verify no read error count", 0, canary.getReadFailures().size());
 300   }
 301
 302   private void runRegionserverCanary() throws Exception {
 303     ExecutorService executor = new ScheduledThreadPoolExecutor(1);
 304     CanaryTool canary = new CanaryTool(executor, new CanaryTool.RegionServerStdOutSink());
 305     String[] args = { "-t", "10000", "-regionserver"};
 306     assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
 307     assertEquals("verify no read error count", 0, canary.getReadFailures().size());
 308   }
 309
 310   private void testZookeeperCanaryWithArgs(String[] args) throws Exception {
 311     Integer port =
 312       Iterables.getOnlyElement(testingUtility.getZkCluster().getClientPortList(), null);
 313     String hostPort = testingUtility.getZkCluster().getAddress().toString();
 314     testingUtility.getConfiguration().set(HConstants.ZOOKEEPER_QUORUM, hostPort);
 315     ExecutorService executor = new ScheduledThreadPoolExecutor(2);
 316     CanaryTool.ZookeeperStdOutSink sink = spy(new CanaryTool.ZookeeperStdOutSink());
 317     CanaryTool canary = new CanaryTool(executor, sink);
 318     assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
 319
 320     String baseZnode = testingUtility.getConfiguration()
 321       .get(HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
 322     verify(sink, atLeastOnce()).publishReadTiming(eq(baseZnode), eq(hostPort), anyLong());
 323   }
 324 }