hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataBlockEncodingTool.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.regionserver;
  19
  20 import java.io.IOException;
  21
  22 import org.apache.hadoop.conf.Configuration;
  23 import org.apache.hadoop.fs.FileSystem;
  24 import org.apache.hadoop.fs.Path;
  25 import org.apache.hadoop.hbase.ArrayBackedTag;
  26 import org.apache.hadoop.hbase.HBaseClassTestRule;
  27 import org.apache.hadoop.hbase.HBaseConfiguration;
  28 import org.apache.hadoop.hbase.HBaseTestingUtility;
  29 import org.apache.hadoop.hbase.KeyValue;
  30 import org.apache.hadoop.hbase.Tag;
  31 import org.apache.hadoop.hbase.io.compress.Compression;
  32 import org.apache.hadoop.hbase.io.hfile.HFileContext;
  33 import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
  34 import org.apache.hadoop.hbase.testclassification.MiscTests;
  35 import org.apache.hadoop.hbase.testclassification.SmallTests;
  36 import org.apache.hadoop.hbase.util.Bytes;
  37 import org.junit.Before;
  38 import org.junit.ClassRule;
  39 import org.junit.Test;
  40 import org.junit.experimental.categories.Category;
  41
  42 /**
  43  *  Test DataBlockEncodingTool.
  44  */
  45 @Category({MiscTests.class, SmallTests.class})
  46 public class TestDataBlockEncodingTool {
  47
  48   @ClassRule
  49   public static final HBaseClassTestRule CLASS_RULE =
  50       HBaseClassTestRule.forClass(TestDataBlockEncodingTool.class);
  51
  52   private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
  53   private static final String ROOT_DIR =
  54       TEST_UTIL.getDataTestDir("TestDataBlockEncodingTool").toString();
  55   private static final Configuration conf = TEST_UTIL.getConfiguration();
  56   private static FileSystem fs;
  57   private static StoreFileWriter sfw;
  58
  59   @Before
  60   public void setUp() throws IOException {
  61     fs = TEST_UTIL.getTestFileSystem();
  62   }
  63
  64   private void testHFile(String fileName, boolean useTags, boolean allTags) throws IOException {
  65     Path path = new Path(ROOT_DIR, fileName);
  66     try {
  67       createHFileWithTags(path, useTags, allTags);
  68       testDataBlockingTool(path);
  69     } finally {
  70       if (fs.exists(path)) {
  71         fs.delete(path, false);
  72       }
  73     }
  74   }
  75
  76   private void createHFileWithTags(Path path, boolean useTags, boolean allTags) throws IOException {
  77     HFileContext meta = new HFileContextBuilder()
  78         .withBlockSize(64 * 1024)
  79         .withIncludesTags(useTags).build();
  80     sfw =
  81         new StoreFileWriter.Builder(conf, fs)
  82             .withFilePath(path)
  83             .withFileContext(meta).build();
  84     long now = System.currentTimeMillis();
  85     byte[] FAMILY = Bytes.toBytes("cf");
  86     byte[] QUALIFIER = Bytes.toBytes("q");
  87     try {
  88       for (char d = 'a'; d <= 'z'; d++) {
  89         for (char e = 'a'; e <= 'z'; e++) {
  90           byte[] b = new byte[] { (byte) d, (byte) e };
  91           KeyValue kv;
  92           if (useTags) {
  93             if (allTags) {
  94               // Write cells with tags to HFile.
  95               Tag[] tags = new Tag[]{
  96                 new ArrayBackedTag((byte) 0, Bytes.toString(b)),
  97                 new ArrayBackedTag((byte) 0, Bytes.toString(b))};
  98               kv = new KeyValue(b, FAMILY, QUALIFIER, now, b, tags);
  99             } else {
 100               // Write half cells with tags and half without tags to HFile.
 101               if ((e - 'a') % 2 == 0) {
 102                 kv = new KeyValue(b, FAMILY, QUALIFIER, now, b);
 103               } else {
 104                 Tag[] tags = new Tag[]{
 105                   new ArrayBackedTag((byte) 0, Bytes.toString(b)),
 106                   new ArrayBackedTag((byte) 0, Bytes.toString(b))};
 107                 kv = new KeyValue(b, FAMILY, QUALIFIER, now, b, tags);
 108               }
 109             }
 110           } else {
 111             // Write cells without tags to HFile.
 112             kv = new KeyValue(b, FAMILY, QUALIFIER, now, b);
 113           }
 114           sfw.append(kv);
 115         }
 116       }
 117       sfw.appendMetadata(0, false);
 118     } finally {
 119       sfw.close();
 120     }
 121   }
 122
 123   private static void testDataBlockingTool(Path path) throws IOException {
 124     Configuration conf = HBaseConfiguration.create();
 125     int maxKV = Integer.MAX_VALUE;
 126     boolean doVerify = true;
 127     boolean doBenchmark = true;
 128     String testHFilePath = path.toString();
 129     DataBlockEncodingTool.testCodecs(conf, maxKV, testHFilePath,
 130         Compression.Algorithm.GZ.getName(), doBenchmark, doVerify);
 131   }
 132
 133   @Test
 134   public void testHFileAllCellsWithTags() throws IOException {
 135     testHFile("1234567890", true, true);
 136   }
 137
 138   @Test
 139   public void testHFileAllCellsWithoutTags() throws IOException {
 140     testHFile("1234567089", false, false);
 141   }
 142
 143   @Test
 144   public void testHFileHalfCellsWithTags() throws IOException {
 145     testHFile("1234560789", true, false);
 146   }
 147 }