hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultiRespectsLimits.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.client;
  19
  20 import static org.junit.Assert.assertEquals;
  21
  22 import java.util.ArrayList;
  23 import java.util.List;
  24 import java.util.concurrent.ThreadLocalRandom;
  25 import org.apache.hadoop.hbase.Cell;
  26 import org.apache.hadoop.hbase.CellBuilderFactory;
  27 import org.apache.hadoop.hbase.CellBuilderType;
  28 import org.apache.hadoop.hbase.CompatibilityFactory;
  29 import org.apache.hadoop.hbase.HBaseClassTestRule;
  30 import org.apache.hadoop.hbase.HBaseTestingUtil;
  31 import org.apache.hadoop.hbase.HConstants;
  32 import org.apache.hadoop.hbase.TableName;
  33 import org.apache.hadoop.hbase.Waiter;
  34 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
  35 import org.apache.hadoop.hbase.ipc.RpcServerInterface;
  36 import org.apache.hadoop.hbase.logging.Log4jUtils;
  37 import org.apache.hadoop.hbase.metrics.BaseSource;
  38 import org.apache.hadoop.hbase.regionserver.HRegionServer;
  39 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
  40 import org.apache.hadoop.hbase.testclassification.ClientTests;
  41 import org.apache.hadoop.hbase.testclassification.MediumTests;
  42 import org.apache.hadoop.hbase.util.Bytes;
  43 import org.junit.AfterClass;
  44 import org.junit.BeforeClass;
  45 import org.junit.ClassRule;
  46 import org.junit.Rule;
  47 import org.junit.Test;
  48 import org.junit.experimental.categories.Category;
  49 import org.junit.rules.TestName;
  50
  51 /**
  52  * This test sets the multi size WAAAAAY low and then checks to make sure that gets will still make
  53  * progress.
  54  */
  55 @Category({MediumTests.class, ClientTests.class})
  56 public class TestMultiRespectsLimits {
  57
  58   @ClassRule
  59   public static final HBaseClassTestRule CLASS_RULE =
  60       HBaseClassTestRule.forClass(TestMultiRespectsLimits.class);
  61
  62   private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
  63   private static final MetricsAssertHelper METRICS_ASSERT =
  64       CompatibilityFactory.getInstance(MetricsAssertHelper.class);
  65   private final static byte[] FAMILY = Bytes.toBytes("D");
  66   public static final int MAX_SIZE = 100;
  67   private static String LOG_LEVEL;
  68
  69   @Rule
  70   public TestName name = new TestName();
  71
  72   @BeforeClass
  73   public static void setUpBeforeClass() throws Exception {
  74     // disable the debug log to avoid flooding the output
  75     LOG_LEVEL = Log4jUtils.getEffectiveLevel(AsyncRegionLocatorHelper.class.getName());
  76     Log4jUtils.setLogLevel(AsyncRegionLocatorHelper.class.getName(), "INFO");
  77     TEST_UTIL.getConfiguration().setLong(HConstants.HBASE_SERVER_SCANNER_MAX_RESULT_SIZE_KEY,
  78       MAX_SIZE);
  79
  80     // Only start on regionserver so that all regions are on the same server.
  81     TEST_UTIL.startMiniCluster(1);
  82   }
  83
  84   @AfterClass
  85   public static void tearDownAfterClass() throws Exception {
  86     if (LOG_LEVEL != null) {
  87       Log4jUtils.setLogLevel(AsyncRegionLocatorHelper.class.getName(), LOG_LEVEL);
  88     }
  89     TEST_UTIL.shutdownMiniCluster();
  90   }
  91
  92   @Test
  93   public void testMultiLimits() throws Exception {
  94     final TableName tableName = TableName.valueOf(name.getMethodName());
  95     Table t = TEST_UTIL.createTable(tableName, FAMILY);
  96     TEST_UTIL.loadTable(t, FAMILY, false);
  97
  98     // Split the table to make sure that the chunking happens accross regions.
  99     try (final Admin admin = TEST_UTIL.getAdmin()) {
 100       admin.split(tableName);
 101       TEST_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
 102         @Override
 103         public boolean evaluate() throws Exception {
 104           return admin.getRegions(tableName).size() > 1;
 105         }
 106       });
 107     }
 108     List<Get> gets = new ArrayList<>(MAX_SIZE);
 109
 110     for (int i = 0; i < MAX_SIZE; i++) {
 111       gets.add(new Get(HBaseTestingUtil.ROWS[i]));
 112     }
 113
 114     RpcServerInterface rpcServer = TEST_UTIL.getHBaseCluster().getRegionServer(0).getRpcServer();
 115     BaseSource s = rpcServer.getMetrics().getMetricsSource();
 116     long startingExceptions = METRICS_ASSERT.getCounter("exceptions", s);
 117     long startingMultiExceptions = METRICS_ASSERT.getCounter("exceptions.multiResponseTooLarge", s);
 118
 119     Result[] results = t.get(gets);
 120     assertEquals(MAX_SIZE, results.length);
 121
 122     // Cells from TEST_UTIL.loadTable have a length of 27.
 123     // Multiplying by less than that gives an easy lower bound on size.
 124     // However in reality each kv is being reported as much higher than that.
 125     METRICS_ASSERT.assertCounterGt("exceptions",
 126         startingExceptions + ((MAX_SIZE * 25) / MAX_SIZE), s);
 127     METRICS_ASSERT.assertCounterGt("exceptions.multiResponseTooLarge",
 128         startingMultiExceptions + ((MAX_SIZE * 25) / MAX_SIZE), s);
 129   }
 130
 131   @Test
 132   public void testBlockMultiLimits() throws Exception {
 133     final TableName tableName = TableName.valueOf(name.getMethodName());
 134     TEST_UTIL.getAdmin().createTable(
 135       TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(ColumnFamilyDescriptorBuilder
 136         .newBuilder(FAMILY).setDataBlockEncoding(DataBlockEncoding.FAST_DIFF).build()).build());
 137     Table t = TEST_UTIL.getConnection().getTable(tableName);
 138
 139     final HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(0);
 140     RpcServerInterface rpcServer = regionServer.getRpcServer();
 141     BaseSource s = rpcServer.getMetrics().getMetricsSource();
 142     long startingExceptions = METRICS_ASSERT.getCounter("exceptions", s);
 143     long startingMultiExceptions = METRICS_ASSERT.getCounter("exceptions.multiResponseTooLarge", s);
 144
 145     byte[] row = Bytes.toBytes("TEST");
 146     byte[][] cols = new byte[][]{
 147         Bytes.toBytes("0"), // Get this
 148         Bytes.toBytes("1"), // Buffer
 149         Bytes.toBytes("2"), // Buffer
 150         Bytes.toBytes("3"), // Get This
 151         Bytes.toBytes("4"), // Buffer
 152         Bytes.toBytes("5"), // Buffer
 153     };
 154
 155     // Set the value size so that one result will be less than the MAX_SIE
 156     // however the block being reference will be larger than MAX_SIZE.
 157     // This should cause the regionserver to try and send a result immediately.
 158     byte[] value = new byte[MAX_SIZE - 100];
 159     ThreadLocalRandom.current().nextBytes(value);
 160
 161     for (byte[] col:cols) {
 162       Put p = new Put(row);
 163       p.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY)
 164               .setRow(row)
 165               .setFamily(FAMILY)
 166               .setQualifier(col)
 167               .setTimestamp(p.getTimestamp())
 168               .setType(Cell.Type.Put)
 169               .setValue(value)
 170               .build());
 171       t.put(p);
 172     }
 173
 174     // Make sure that a flush happens
 175     try (final Admin admin = TEST_UTIL.getAdmin()) {
 176       admin.flush(tableName);
 177       TEST_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
 178         @Override
 179         public boolean evaluate() throws Exception {
 180           return regionServer.getRegions(tableName).get(0).getMaxFlushedSeqId() > 3;
 181         }
 182       });
 183     }
 184
 185     List<Get> gets = new ArrayList<>(2);
 186     Get g0 = new Get(row);
 187     g0.addColumn(FAMILY, cols[0]);
 188     gets.add(g0);
 189
 190     Get g2 = new Get(row);
 191     g2.addColumn(FAMILY, cols[3]);
 192     gets.add(g2);
 193
 194     Result[] results = t.get(gets);
 195     assertEquals(2, results.length);
 196     METRICS_ASSERT.assertCounterGt("exceptions", startingExceptions, s);
 197     METRICS_ASSERT.assertCounterGt("exceptions.multiResponseTooLarge",
 198         startingMultiExceptions, s);
 199   }
 200 }