hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCache.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.mob;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertNotNull;
  22
  23 import java.io.IOException;
  24 import java.util.Date;
  25 import org.apache.hadoop.conf.Configuration;
  26 import org.apache.hadoop.fs.FileSystem;
  27 import org.apache.hadoop.fs.Path;
  28 import org.apache.hadoop.hbase.HBaseClassTestRule;
  29 import org.apache.hadoop.hbase.HBaseConfiguration;
  30 import org.apache.hadoop.hbase.HBaseTestingUtility;
  31 import org.apache.hadoop.hbase.HColumnDescriptor;
  32 import org.apache.hadoop.hbase.HConstants;
  33 import org.apache.hadoop.hbase.HRegionInfo;
  34 import org.apache.hadoop.hbase.KeyValue;
  35 import org.apache.hadoop.hbase.TableName;
  36 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
  37 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
  38 import org.apache.hadoop.hbase.client.RegionInfo;
  39 import org.apache.hadoop.hbase.client.RegionInfoBuilder;
  40 import org.apache.hadoop.hbase.client.TableDescriptor;
  41 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
  42 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
  43 import org.apache.hadoop.hbase.regionserver.HMobStore;
  44 import org.apache.hadoop.hbase.regionserver.HRegion;
  45 import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
  46 import org.apache.hadoop.hbase.testclassification.SmallTests;
  47 import org.apache.hadoop.hbase.util.Bytes;
  48 import org.junit.After;
  49 import org.junit.Before;
  50 import org.junit.ClassRule;
  51 import org.junit.Test;
  52 import org.junit.experimental.categories.Category;
  53
  54 @Category(SmallTests.class)
  55 public class TestMobFileCache {
  56
  57   @ClassRule
  58   public static final HBaseClassTestRule CLASS_RULE =
  59       HBaseClassTestRule.forClass(TestMobFileCache.class);
  60
  61   private HBaseTestingUtility UTIL;
  62   private HRegion region;
  63   private Configuration conf;
  64   private MobFileCache mobFileCache;
  65   private Date currentDate = new Date();
  66   private static final String TEST_CACHE_SIZE = "2";
  67   private static final int EXPECTED_CACHE_SIZE_ZERO = 0;
  68   private static final int EXPECTED_CACHE_SIZE_ONE = 1;
  69   private static final int EXPECTED_CACHE_SIZE_TWO = 2;
  70   private static final int EXPECTED_CACHE_SIZE_THREE = 3;
  71   private static final long EXPECTED_REFERENCE_ONE = 1;
  72   private static final long EXPECTED_REFERENCE_TWO = 2;
  73
  74   private static final String TABLE = "tableName";
  75   private static final String FAMILY1 = "family1";
  76   private static final String FAMILY2 = "family2";
  77   private static final String FAMILY3 = "family3";
  78
  79   private static final byte[] ROW = Bytes.toBytes("row");
  80   private static final byte[] ROW2 = Bytes.toBytes("row2");
  81   private static final byte[] VALUE = Bytes.toBytes("value");
  82   private static final byte[] VALUE2 = Bytes.toBytes("value2");
  83   private static final byte[] QF1 = Bytes.toBytes("qf1");
  84   private static final byte[] QF2 = Bytes.toBytes("qf2");
  85   private static final byte[] QF3 = Bytes.toBytes("qf3");
  86
  87   @Before
  88   public void setUp() throws Exception {
  89     UTIL = new HBaseTestingUtility();
  90     conf = UTIL.getConfiguration();
  91     conf.set(MobConstants.MOB_FILE_CACHE_SIZE_KEY, TEST_CACHE_SIZE);
  92     TableDescriptorBuilder tableDescriptorBuilder =
  93       TableDescriptorBuilder.newBuilder(UTIL.createTableDescriptor(
  94         TableName.valueOf("testMobFileCache"), HColumnDescriptor.DEFAULT_MIN_VERSIONS, 3,
  95         HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED));
  96     ColumnFamilyDescriptor columnFamilyDescriptor =
  97       ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(FAMILY1))
  98         .setMobEnabled(true)
  99         .setMobThreshold(0)
 100         .build();
 101     tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
 102     columnFamilyDescriptor =
 103       ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(FAMILY2))
 104         .setMobEnabled(true)
 105         .setMobThreshold(0)
 106         .build();
 107     tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
 108     columnFamilyDescriptor =
 109       ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(FAMILY3))
 110         .setMobEnabled(true)
 111         .setMobThreshold(0)
 112         .build();
 113     tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
 114     TableDescriptor tableDescriptor = tableDescriptorBuilder.build();
 115     RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
 116     mobFileCache = new MobFileCache(conf);
 117     region = HBaseTestingUtility
 118       .createRegionAndWAL(regionInfo, UTIL.getDataTestDir(), conf, tableDescriptor, mobFileCache);
 119   }
 120
 121   @After
 122   public void tearDown() throws Exception {
 123     region.close();
 124     region.getFilesystem().delete(UTIL.getDataTestDir(), true);
 125   }
 126
 127   /**
 128    * Create the mob store file.
 129    */
 130   private Path createMobStoreFile(String family) throws IOException {
 131     return createMobStoreFile(HBaseConfiguration.create(), family);
 132   }
 133
 134   /**
 135    * Create the mob store file
 136    */
 137   private Path createMobStoreFile(Configuration conf, String family) throws IOException {
 138     ColumnFamilyDescriptor columnFamilyDescriptor =
 139       ColumnFamilyDescriptorBuilder
 140         .newBuilder(Bytes.toBytes(family))
 141         .setMaxVersions(4)
 142         .setMobEnabled(true).build();
 143     return createMobStoreFile(columnFamilyDescriptor);
 144   }
 145
 146   /**
 147    * Create the mob store file
 148    */
 149   private Path createMobStoreFile(ColumnFamilyDescriptor columnFamilyDescriptor)
 150       throws IOException {
 151     // Setting up a Store
 152     TableName tn = TableName.valueOf(TABLE);
 153     TableDescriptorBuilder tableDescriptorBuilder =
 154       TableDescriptorBuilder.newBuilder(tn);
 155     tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
 156     HMobStore mobStore = (HMobStore) region.getStore(columnFamilyDescriptor.getName());
 157     KeyValue key1 = new KeyValue(ROW, columnFamilyDescriptor.getName(), QF1, 1, VALUE);
 158     KeyValue key2 = new KeyValue(ROW, columnFamilyDescriptor.getName(), QF2, 1, VALUE);
 159     KeyValue key3 = new KeyValue(ROW2, columnFamilyDescriptor.getName(), QF3, 1, VALUE2);
 160     KeyValue[] keys = new KeyValue[] { key1, key2, key3 };
 161     int maxKeyCount = keys.length;
 162     HRegionInfo regionInfo = new HRegionInfo(tn);
 163     StoreFileWriter mobWriter = mobStore.createWriterInTmp(currentDate,
 164       maxKeyCount, columnFamilyDescriptor.getCompactionCompressionType(),
 165       regionInfo.getStartKey(), false);
 166     Path mobFilePath = mobWriter.getPath();
 167     String fileName = mobFilePath.getName();
 168     mobWriter.append(key1);
 169     mobWriter.append(key2);
 170     mobWriter.append(key3);
 171     mobWriter.close();
 172     String targetPathName = MobUtils.formatDate(currentDate);
 173     Path targetPath = new Path(mobStore.getPath(), targetPathName);
 174     mobStore.commitFile(mobFilePath, targetPath);
 175     return new Path(targetPath, fileName);
 176   }
 177
 178   @Test
 179   public void testMobFileCache() throws Exception {
 180     FileSystem fs = FileSystem.get(conf);
 181     Path file1Path = createMobStoreFile(FAMILY1);
 182     Path file2Path = createMobStoreFile(FAMILY2);
 183     Path file3Path = createMobStoreFile(FAMILY3);
 184
 185     CacheConfig cacheConf = new CacheConfig(conf);
 186     // Before open one file by the MobFileCache
 187     assertEquals(EXPECTED_CACHE_SIZE_ZERO, mobFileCache.getCacheSize());
 188     // Open one file by the MobFileCache
 189     CachedMobFile cachedMobFile1 = (CachedMobFile) mobFileCache.openFile(
 190         fs, file1Path, cacheConf);
 191     assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize());
 192     assertNotNull(cachedMobFile1);
 193     assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile1.getReferenceCount());
 194
 195     // The evict is also managed by a schedule thread pool.
 196     // And its check period is set as 3600 seconds by default.
 197     // This evict should get the lock at the most time
 198     mobFileCache.evict();  // Cache not full, evict it
 199     assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize());
 200     assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile1.getReferenceCount());
 201
 202     mobFileCache.evictFile(file1Path.getName());  // Evict one file
 203     assertEquals(EXPECTED_CACHE_SIZE_ZERO, mobFileCache.getCacheSize());
 204     assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile1.getReferenceCount());
 205
 206     cachedMobFile1.close();  // Close the cached mob file
 207
 208     // Reopen three cached file
 209     cachedMobFile1 = (CachedMobFile) mobFileCache.openFile(
 210         fs, file1Path, cacheConf);
 211     assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize());
 212     CachedMobFile cachedMobFile2 = (CachedMobFile) mobFileCache.openFile(
 213         fs, file2Path, cacheConf);
 214     assertEquals(EXPECTED_CACHE_SIZE_TWO, mobFileCache.getCacheSize());
 215     CachedMobFile cachedMobFile3 = (CachedMobFile) mobFileCache.openFile(
 216         fs, file3Path, cacheConf);
 217     // Before the evict
 218     // Evict the cache, should close the first file 1
 219     assertEquals(EXPECTED_CACHE_SIZE_THREE, mobFileCache.getCacheSize());
 220     assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile1.getReferenceCount());
 221     assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile2.getReferenceCount());
 222     assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile3.getReferenceCount());
 223     mobFileCache.evict();
 224     assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize());
 225     assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile1.getReferenceCount());
 226     assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile2.getReferenceCount());
 227     assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile3.getReferenceCount());
 228   }
 229 }