hbase-server/src/test/java/org/apache/hadoop/hbase/TestServerSideScanMetricsFromClientSide.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertTrue;
  22
  23 import java.io.IOException;
  24 import java.util.ArrayList;
  25 import java.util.List;
  26 import org.apache.hadoop.hbase.client.Put;
  27 import org.apache.hadoop.hbase.client.Result;
  28 import org.apache.hadoop.hbase.client.ResultScanner;
  29 import org.apache.hadoop.hbase.client.Scan;
  30 import org.apache.hadoop.hbase.client.Table;
  31 import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
  32 import org.apache.hadoop.hbase.client.metrics.ServerSideScanMetrics;
  33 import org.apache.hadoop.hbase.filter.BinaryComparator;
  34 import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
  35 import org.apache.hadoop.hbase.filter.Filter;
  36 import org.apache.hadoop.hbase.filter.FilterList;
  37 import org.apache.hadoop.hbase.filter.FilterList.Operator;
  38 import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
  39 import org.apache.hadoop.hbase.filter.RowFilter;
  40 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
  41 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
  42 import org.apache.hadoop.hbase.testclassification.LargeTests;
  43 import org.apache.hadoop.hbase.util.Bytes;
  44 import org.junit.AfterClass;
  45 import org.junit.BeforeClass;
  46 import org.junit.ClassRule;
  47 import org.junit.Test;
  48 import org.junit.experimental.categories.Category;
  49
  50 @Category(LargeTests.class)
  51 public class TestServerSideScanMetricsFromClientSide {
  52
  53   @ClassRule
  54   public static final HBaseClassTestRule CLASS_RULE =
  55       HBaseClassTestRule.forClass(TestServerSideScanMetricsFromClientSide.class);
  56
  57   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
  58
  59   private static Table TABLE = null;
  60
  61   /**
  62    * Table configuration
  63    */
  64   private static TableName TABLE_NAME = TableName.valueOf("testTable");
  65
  66   private static int NUM_ROWS = 10;
  67   private static byte[] ROW = Bytes.toBytes("testRow");
  68   private static byte[][] ROWS = HTestConst.makeNAscii(ROW, NUM_ROWS);
  69
  70   // Should keep this value below 10 to keep generation of expected kv's simple. If above 10 then
  71   // table/row/cf1/... will be followed by table/row/cf10/... instead of table/row/cf2/... which
  72   // breaks the simple generation of expected kv's
  73   private static int NUM_FAMILIES = 1;
  74   private static byte[] FAMILY = Bytes.toBytes("testFamily");
  75   private static byte[][] FAMILIES = HTestConst.makeNAscii(FAMILY, NUM_FAMILIES);
  76
  77   private static int NUM_QUALIFIERS = 1;
  78   private static byte[] QUALIFIER = Bytes.toBytes("testQualifier");
  79   private static byte[][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, NUM_QUALIFIERS);
  80
  81   private static int VALUE_SIZE = 10;
  82   private static byte[] VALUE = Bytes.createMaxByteArray(VALUE_SIZE);
  83
  84   private static int NUM_COLS = NUM_FAMILIES * NUM_QUALIFIERS;
  85
  86   // Approximation of how large the heap size of cells in our table. Should be accessed through
  87   // getCellHeapSize().
  88   private static long CELL_HEAP_SIZE = -1;
  89
  90   @BeforeClass
  91   public static void setUpBeforeClass() throws Exception {
  92     TEST_UTIL.startMiniCluster(3);
  93     TABLE = createTestTable(TABLE_NAME, ROWS, FAMILIES, QUALIFIERS, VALUE);
  94   }
  95
  96   private static Table createTestTable(TableName name, byte[][] rows, byte[][] families,
  97       byte[][] qualifiers, byte[] cellValue) throws IOException {
  98     Table ht = TEST_UTIL.createTable(name, families);
  99     List<Put> puts = createPuts(rows, families, qualifiers, cellValue);
 100     ht.put(puts);
 101
 102     return ht;
 103   }
 104
 105   @AfterClass
 106   public static void tearDownAfterClass() throws Exception {
 107     TEST_UTIL.shutdownMiniCluster();
 108   }
 109
 110   /**
 111    * Make puts to put the input value into each combination of row, family, and qualifier
 112    * @param rows the rows to use
 113    * @param families the column families to use
 114    * @param qualifiers the column qualifiers to use
 115    * @param value the value to put
 116    * @return the putted input values added in puts
 117    * @throws IOException If an IO problem is encountered
 118    */
 119   private static ArrayList<Put> createPuts(byte[][] rows, byte[][] families, byte[][] qualifiers,
 120       byte[] value) throws IOException {
 121     Put put;
 122     ArrayList<Put> puts = new ArrayList<>();
 123
 124     for (int row = 0; row < rows.length; row++) {
 125       put = new Put(rows[row]);
 126       for (int fam = 0; fam < families.length; fam++) {
 127         for (int qual = 0; qual < qualifiers.length; qual++) {
 128           KeyValue kv = new KeyValue(rows[row], families[fam], qualifiers[qual], qual, value);
 129           put.add(kv);
 130         }
 131       }
 132       puts.add(put);
 133     }
 134
 135     return puts;
 136   }
 137
 138   /**
 139    * @return The approximate heap size of a cell in the test table. All cells should have
 140    *         approximately the same heap size, so the value is cached to avoid repeating the
 141    *         calculation
 142    * @throws Exception on unexpected failure
 143    */
 144   private long getCellHeapSize() throws Exception {
 145     if (CELL_HEAP_SIZE == -1) {
 146       // Do a partial scan that will return a single result with a single cell
 147       Scan scan = new Scan();
 148       scan.setMaxResultSize(1);
 149       scan.setAllowPartialResults(true);
 150       ResultScanner scanner = TABLE.getScanner(scan);
 151
 152       Result result = scanner.next();
 153
 154       assertTrue(result != null);
 155       assertTrue(result.rawCells() != null);
 156       assertTrue(result.rawCells().length == 1);
 157
 158       CELL_HEAP_SIZE = result.rawCells()[0].heapSize();
 159       scanner.close();
 160     }
 161
 162     return CELL_HEAP_SIZE;
 163   }
 164
 165   @Test
 166   public void testRowsSeenMetric() throws Exception {
 167     // Base scan configuration
 168     Scan baseScan;
 169     baseScan = new Scan();
 170     baseScan.setScanMetricsEnabled(true);
 171     testRowsSeenMetric(baseScan);
 172
 173     // Test case that only a single result will be returned per RPC to the serer
 174     baseScan.setCaching(1);
 175     testRowsSeenMetric(baseScan);
 176
 177     // Test case that partial results are returned from the server. At most one cell will be
 178     // contained in each response
 179     baseScan.setMaxResultSize(1);
 180     testRowsSeenMetric(baseScan);
 181
 182     // Test case that size limit is set such that a few cells are returned per partial result from
 183     // the server
 184     baseScan.setCaching(NUM_ROWS);
 185     baseScan.setMaxResultSize(getCellHeapSize() * (NUM_COLS - 1));
 186     testRowsSeenMetric(baseScan);
 187   }
 188
 189   private void testRowsSeenMetric(Scan baseScan) throws Exception {
 190     Scan scan;
 191     scan = new Scan(baseScan);
 192     testMetric(scan, ServerSideScanMetrics.COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME, NUM_ROWS);
 193
 194     for (int i = 0; i < ROWS.length - 1; i++) {
 195       scan = new Scan(baseScan);
 196       scan.withStartRow(ROWS[0]);
 197       scan.withStopRow(ROWS[i + 1]);
 198       testMetric(scan, ServerSideScanMetrics.COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME, i + 1);
 199     }
 200
 201     for (int i = ROWS.length - 1; i > 0; i--) {
 202       scan = new Scan(baseScan);
 203       scan.withStartRow(ROWS[i - 1]);
 204       scan.withStopRow(ROWS[ROWS.length - 1]);
 205       testMetric(scan, ServerSideScanMetrics.COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME, ROWS.length - i);
 206     }
 207
 208     // The filter should filter out all rows, but we still expect to see every row.
 209     Filter filter =
 210         new RowFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("xyz")));
 211     scan = new Scan(baseScan);
 212     scan.setFilter(filter);
 213     testMetric(scan, ServerSideScanMetrics.COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME, ROWS.length);
 214
 215     // Filter should pass on all rows
 216     SingleColumnValueFilter singleColumnValueFilter =
 217         new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS[0], CompareOperator.EQUAL, VALUE);
 218     scan = new Scan(baseScan);
 219     scan.setFilter(singleColumnValueFilter);
 220     testMetric(scan, ServerSideScanMetrics.COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME, ROWS.length);
 221
 222     // Filter should filter out all rows
 223     singleColumnValueFilter =
 224         new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS[0], CompareOperator.NOT_EQUAL, VALUE);
 225     scan = new Scan(baseScan);
 226     scan.setFilter(singleColumnValueFilter);
 227     testMetric(scan, ServerSideScanMetrics.COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME, ROWS.length);
 228   }
 229
 230   @Test
 231   public void testRowsFilteredMetric() throws Exception {
 232     // Base scan configuration
 233     Scan baseScan;
 234     baseScan = new Scan();
 235     baseScan.setScanMetricsEnabled(true);
 236
 237     // Test case where scan uses default values
 238     testRowsFilteredMetric(baseScan);
 239
 240     // Test case where at most one Result is retrieved per RPC
 241     baseScan.setCaching(1);
 242     testRowsFilteredMetric(baseScan);
 243
 244     // Test case where size limit is very restrictive and partial results will be returned from
 245     // server
 246     baseScan.setMaxResultSize(1);
 247     testRowsFilteredMetric(baseScan);
 248
 249     // Test a case where max result size limits response from server to only a few cells (not all
 250     // cells from the row)
 251     baseScan.setCaching(NUM_ROWS);
 252     baseScan.setMaxResultSize(getCellHeapSize() * (NUM_COLS - 1));
 253     testRowsSeenMetric(baseScan);
 254   }
 255
 256   private void testRowsFilteredMetric(Scan baseScan) throws Exception {
 257     testRowsFilteredMetric(baseScan, null, 0);
 258
 259     // Row filter doesn't match any row key. All rows should be filtered
 260     Filter filter =
 261         new RowFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("xyz")));
 262     testRowsFilteredMetric(baseScan, filter, ROWS.length);
 263
 264     // Filter will return results containing only the first key. Number of entire rows filtered
 265     // should be 0.
 266     filter = new FirstKeyOnlyFilter();
 267     testRowsFilteredMetric(baseScan, filter, 0);
 268
 269     // Column prefix will find some matching qualifier on each row. Number of entire rows filtered
 270     // should be 0
 271     filter = new ColumnPrefixFilter(QUALIFIERS[0]);
 272     testRowsFilteredMetric(baseScan, filter, 0);
 273
 274     // Column prefix will NOT find any matching qualifier on any row. All rows should be filtered
 275     filter = new ColumnPrefixFilter(Bytes.toBytes("xyz"));
 276     testRowsFilteredMetric(baseScan, filter, ROWS.length);
 277
 278     // Matching column value should exist in each row. No rows should be filtered.
 279     filter = new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS[0], CompareOperator.EQUAL, VALUE);
 280     testRowsFilteredMetric(baseScan, filter, 0);
 281
 282     // No matching column value should exist in any row. Filter all rows
 283     filter = new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS[0],
 284       CompareOperator.NOT_EQUAL, VALUE);
 285     testRowsFilteredMetric(baseScan, filter, ROWS.length);
 286
 287     List<Filter> filters = new ArrayList<>();
 288     filters.add(new RowFilter(CompareOperator.EQUAL, new BinaryComparator(ROWS[0])));
 289     filters.add(new RowFilter(CompareOperator.EQUAL, new BinaryComparator(ROWS[3])));
 290     int numberOfMatchingRowFilters = filters.size();
 291     filter = new FilterList(Operator.MUST_PASS_ONE, filters);
 292     testRowsFilteredMetric(baseScan, filter, ROWS.length - numberOfMatchingRowFilters);
 293     filters.clear();
 294
 295     // Add a single column value exclude filter for each column... The net effect is that all
 296     // columns will be excluded when scanning on the server side. This will result in an empty cell
 297     // array in RegionScanner#nextInternal which should be interpreted as a row being filtered.
 298     for (int family = 0; family < FAMILIES.length; family++) {
 299       for (int qualifier = 0; qualifier < QUALIFIERS.length; qualifier++) {
 300         filters.add(new SingleColumnValueExcludeFilter(FAMILIES[family], QUALIFIERS[qualifier],
 301           CompareOperator.EQUAL, VALUE));
 302       }
 303     }
 304     filter = new FilterList(Operator.MUST_PASS_ONE, filters);
 305     testRowsFilteredMetric(baseScan, filter, ROWS.length);
 306   }
 307
 308   private void testRowsFilteredMetric(Scan baseScan, Filter filter, int expectedNumFiltered)
 309       throws Exception {
 310     Scan scan = new Scan(baseScan);
 311     if (filter != null) {
 312       scan.setFilter(filter);
 313     }
 314     testMetric(scan, ServerSideScanMetrics.COUNT_OF_ROWS_FILTERED_KEY_METRIC_NAME,
 315       expectedNumFiltered);
 316   }
 317
 318   /**
 319    * Run the scan to completetion and check the metric against the specified value
 320    * @param scan The scan instance to use to record metrics
 321    * @param metricKey The metric key name
 322    * @param expectedValue The expected value of metric
 323    * @throws Exception on unexpected failure
 324    */
 325   private void testMetric(Scan scan, String metricKey, long expectedValue) throws Exception {
 326     assertTrue("Scan should be configured to record metrics", scan.isScanMetricsEnabled());
 327     ResultScanner scanner = TABLE.getScanner(scan);
 328     // Iterate through all the results
 329     while (scanner.next() != null) {
 330     }
 331     scanner.close();
 332     ScanMetrics metrics = scanner.getScanMetrics();
 333     assertTrue("Metrics are null", metrics != null);
 334     assertTrue("Metric : " + metricKey + " does not exist", metrics.hasCounter(metricKey));
 335     final long actualMetricValue = metrics.getCounter(metricKey).get();
 336     assertEquals(
 337       "Metric: " + metricKey + " Expected: " + expectedValue + " Actual: " + actualMetricValue,
 338       expectedValue, actualMetricValue);
 339   }
 340 }