hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java

   1 /*
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.regionserver;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertTrue;
  22
  23 import java.io.IOException;
  24 import java.util.ArrayList;
  25 import java.util.Collection;
  26 import java.util.HashMap;
  27 import java.util.HashSet;
  28 import java.util.List;
  29 import java.util.Set;
  30 import org.apache.hadoop.hbase.Cell;
  31 import org.apache.hadoop.hbase.HBaseClassTestRule;
  32 import org.apache.hadoop.hbase.HBaseTestingUtil;
  33 import org.apache.hadoop.hbase.KeyValue;
  34 import org.apache.hadoop.hbase.KeyValueTestUtil;
  35 import org.apache.hadoop.hbase.TableName;
  36 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
  37 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
  38 import org.apache.hadoop.hbase.client.Durability;
  39 import org.apache.hadoop.hbase.client.Put;
  40 import org.apache.hadoop.hbase.client.RegionInfo;
  41 import org.apache.hadoop.hbase.client.RegionInfoBuilder;
  42 import org.apache.hadoop.hbase.client.Scan;
  43 import org.apache.hadoop.hbase.client.TableDescriptor;
  44 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
  45 import org.apache.hadoop.hbase.testclassification.MediumTests;
  46 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
  47 import org.apache.hadoop.hbase.util.Bytes;
  48 import org.junit.ClassRule;
  49 import org.junit.Rule;
  50 import org.junit.Test;
  51 import org.junit.experimental.categories.Category;
  52 import org.junit.rules.TestName;
  53 import org.slf4j.Logger;
  54 import org.slf4j.LoggerFactory;
  55
  56 @Category({RegionServerTests.class, MediumTests.class})
  57 public class TestColumnSeeking {
  58
  59   @ClassRule
  60   public static final HBaseClassTestRule CLASS_RULE =
  61       HBaseClassTestRule.forClass(TestColumnSeeking.class);
  62
  63   @Rule public TestName name = new TestName();
  64   private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
  65
  66   private static final Logger LOG = LoggerFactory.getLogger(TestColumnSeeking.class);
  67
  68   @SuppressWarnings("unchecked")
  69   @Test
  70   public void testDuplicateVersions() throws IOException {
  71     String family = "Family";
  72     byte[] familyBytes = Bytes.toBytes("Family");
  73     TableName table = TableName.valueOf(name.getMethodName());
  74
  75     ColumnFamilyDescriptor familyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(familyBytes)
  76       .setMaxVersions(1000).setMaxVersions(3).build();
  77     TableDescriptor tableDescriptor =
  78       TableDescriptorBuilder.newBuilder(table).setColumnFamily(familyDescriptor).build();
  79     RegionInfo info = RegionInfoBuilder.newBuilder(table).build();
  80     // Set this so that the archiver writes to the temp dir as well.
  81     HRegion region = TEST_UTIL.createLocalHRegion(info, tableDescriptor);
  82     try {
  83       List<String> rows = generateRandomWords(10, "row");
  84       List<String> allColumns = generateRandomWords(10, "column");
  85       List<String> values = generateRandomWords(100, "value");
  86
  87       long maxTimestamp = 2;
  88       double selectPercent = 0.5;
  89       int numberOfTests = 5;
  90       double flushPercentage = 0.2;
  91       double minorPercentage = 0.2;
  92       double majorPercentage = 0.2;
  93       double putPercentage = 0.2;
  94
  95       HashMap<String, KeyValue> allKVMap = new HashMap<>();
  96
  97       HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
  98       ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
  99
 100       for (int i = 0; i < numberOfTests; i++) {
 101         kvMaps[i] = new HashMap<>();
 102         columnLists[i] = new ArrayList<>();
 103         for (String column : allColumns) {
 104           if (Math.random() < selectPercent) {
 105             columnLists[i].add(column);
 106           }
 107         }
 108       }
 109
 110       for (String value : values) {
 111         for (String row : rows) {
 112           Put p = new Put(Bytes.toBytes(row));
 113           p.setDurability(Durability.SKIP_WAL);
 114           for (String column : allColumns) {
 115             for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
 116               KeyValue kv =
 117                   KeyValueTestUtil.create(row, family, column, timestamp, value);
 118               if (Math.random() < putPercentage) {
 119                 p.add(kv);
 120                 allKVMap.put(kv.getKeyString(), kv);
 121                 for (int i = 0; i < numberOfTests; i++) {
 122                   if (columnLists[i].contains(column)) {
 123                     kvMaps[i].put(kv.getKeyString(), kv);
 124                   }
 125                 }
 126               }
 127             }
 128           }
 129           region.put(p);
 130           if (Math.random() < flushPercentage) {
 131             LOG.info("Flushing... ");
 132             region.flush(true);
 133           }
 134
 135           if (Math.random() < minorPercentage) {
 136             LOG.info("Minor compacting... ");
 137             region.compact(false);
 138           }
 139
 140           if (Math.random() < majorPercentage) {
 141             LOG.info("Major compacting... ");
 142             region.compact(true);
 143           }
 144         }
 145       }
 146
 147       for (int i = 0; i < numberOfTests + 1; i++) {
 148         Collection<KeyValue> kvSet;
 149         Scan scan = new Scan();
 150         scan.readAllVersions();
 151         if (i < numberOfTests) {
 152           if (columnLists[i].isEmpty()) continue; // HBASE-7700
 153           kvSet = kvMaps[i].values();
 154           for (String column : columnLists[i]) {
 155             scan.addColumn(familyBytes, Bytes.toBytes(column));
 156           }
 157           LOG.info("ExplicitColumns scanner");
 158           LOG.info("Columns: " + columnLists[i].size() + "  Keys: "
 159               + kvSet.size());
 160         } else {
 161           kvSet = allKVMap.values();
 162           LOG.info("Wildcard scanner");
 163           LOG.info("Columns: " + allColumns.size() + "  Keys: " + kvSet.size());
 164
 165         }
 166         InternalScanner scanner = region.getScanner(scan);
 167         List<Cell> results = new ArrayList<>();
 168         while (scanner.next(results))
 169           ;
 170         assertEquals(kvSet.size(), results.size());
 171         assertTrue(KeyValueTestUtil.containsIgnoreMvccVersion(results, kvSet));
 172       }
 173     } finally {
 174       HBaseTestingUtil.closeRegionAndWAL(region);
 175     }
 176
 177     HBaseTestingUtil.closeRegionAndWAL(region);
 178   }
 179
 180   @SuppressWarnings("unchecked")
 181   @Test
 182   public void testReseeking() throws IOException {
 183     String family = "Family";
 184     byte[] familyBytes = Bytes.toBytes("Family");
 185     TableName table = TableName.valueOf(name.getMethodName());
 186
 187     TableDescriptorBuilder tableDescriptorBuilder =
 188       TableDescriptorBuilder.newBuilder(table);
 189     ColumnFamilyDescriptor columnFamilyDescriptor =
 190       ColumnFamilyDescriptorBuilder
 191         .newBuilder(Bytes.toBytes(family))
 192         .setMaxVersions(3).build();
 193     tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
 194
 195     RegionInfo info = RegionInfoBuilder.newBuilder(table).build();
 196     HRegion region = TEST_UTIL.createLocalHRegion(info, tableDescriptorBuilder.build());
 197
 198     List<String> rows = generateRandomWords(10, "row");
 199     List<String> allColumns = generateRandomWords(100, "column");
 200
 201     long maxTimestamp = 2;
 202     double selectPercent = 0.5;
 203     int numberOfTests = 5;
 204     double flushPercentage = 0.2;
 205     double minorPercentage = 0.2;
 206     double majorPercentage = 0.2;
 207     double putPercentage = 0.2;
 208
 209     HashMap<String, KeyValue> allKVMap = new HashMap<>();
 210
 211     HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
 212     ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
 213     String valueString = "Value";
 214
 215     for (int i = 0; i < numberOfTests; i++) {
 216       kvMaps[i] = new HashMap<>();
 217       columnLists[i] = new ArrayList<>();
 218       for (String column : allColumns) {
 219         if (Math.random() < selectPercent) {
 220           columnLists[i].add(column);
 221         }
 222       }
 223     }
 224
 225     for (String row : rows) {
 226       Put p = new Put(Bytes.toBytes(row));
 227       p.setDurability(Durability.SKIP_WAL);
 228       for (String column : allColumns) {
 229         for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
 230           KeyValue kv =
 231               KeyValueTestUtil.create(row, family, column, timestamp,
 232                   valueString);
 233           if (Math.random() < putPercentage) {
 234             p.add(kv);
 235             allKVMap.put(kv.getKeyString(), kv);
 236             for (int i = 0; i < numberOfTests; i++) {
 237               if (columnLists[i].contains(column)) {
 238                 kvMaps[i].put(kv.getKeyString(), kv);
 239               }
 240             }
 241           }
 242
 243         }
 244       }
 245       region.put(p);
 246       if (Math.random() < flushPercentage) {
 247         LOG.info("Flushing... ");
 248         region.flush(true);
 249       }
 250
 251       if (Math.random() < minorPercentage) {
 252         LOG.info("Minor compacting... ");
 253         region.compact(false);
 254       }
 255
 256       if (Math.random() < majorPercentage) {
 257         LOG.info("Major compacting... ");
 258         region.compact(true);
 259       }
 260     }
 261
 262     for (int i = 0; i < numberOfTests + 1; i++) {
 263       Collection<KeyValue> kvSet;
 264       Scan scan = new Scan();
 265       scan.readAllVersions();
 266       if (i < numberOfTests) {
 267         if (columnLists[i].isEmpty()) continue; // HBASE-7700
 268         kvSet = kvMaps[i].values();
 269         for (String column : columnLists[i]) {
 270           scan.addColumn(familyBytes, Bytes.toBytes(column));
 271         }
 272         LOG.info("ExplicitColumns scanner");
 273         LOG.info("Columns: " + columnLists[i].size() + "  Keys: "
 274             + kvSet.size());
 275       } else {
 276         kvSet = allKVMap.values();
 277         LOG.info("Wildcard scanner");
 278         LOG.info("Columns: " + allColumns.size() + "  Keys: " + kvSet.size());
 279
 280       }
 281       InternalScanner scanner = region.getScanner(scan);
 282       List<Cell> results = new ArrayList<>();
 283       while (scanner.next(results))
 284         ;
 285       assertEquals(kvSet.size(), results.size());
 286       assertTrue(KeyValueTestUtil.containsIgnoreMvccVersion(results, kvSet));
 287     }
 288
 289     HBaseTestingUtil.closeRegionAndWAL(region);
 290   }
 291
 292   List<String> generateRandomWords(int numberOfWords, String suffix) {
 293     Set<String> wordSet = new HashSet<>();
 294     for (int i = 0; i < numberOfWords; i++) {
 295       int lengthOfWords = (int) (Math.random() * 5) + 1;
 296       char[] wordChar = new char[lengthOfWords];
 297       for (int j = 0; j < wordChar.length; j++) {
 298         wordChar[j] = (char) (Math.random() * 26 + 97);
 299       }
 300       String word;
 301       if (suffix == null) {
 302         word = new String(wordChar);
 303       } else {
 304         word = new String(wordChar) + suffix;
 305       }
 306       wordSet.add(word);
 307     }
 308     List<String> wordList = new ArrayList<>(wordSet);
 309     return wordList;
 310   }
 311
 312 }
 313