hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.mob;
  19
  20 import static org.junit.Assert.assertEquals;
  21
  22 import java.util.Random;
  23 import org.apache.hadoop.fs.FileStatus;
  24 import org.apache.hadoop.fs.Path;
  25 import org.apache.hadoop.hbase.HBaseClassTestRule;
  26 import org.apache.hadoop.hbase.HBaseTestingUtility;
  27 import org.apache.hadoop.hbase.TableName;
  28 import org.apache.hadoop.hbase.client.Admin;
  29 import org.apache.hadoop.hbase.client.BufferedMutator;
  30 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
  31 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
  32 import org.apache.hadoop.hbase.client.ConnectionFactory;
  33 import org.apache.hadoop.hbase.client.Put;
  34 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
  35 import org.apache.hadoop.hbase.testclassification.MediumTests;
  36 import org.apache.hadoop.hbase.util.Bytes;
  37 import org.apache.hadoop.util.ToolRunner;
  38 import org.junit.After;
  39 import org.junit.AfterClass;
  40 import org.junit.Before;
  41 import org.junit.BeforeClass;
  42 import org.junit.ClassRule;
  43 import org.junit.Test;
  44 import org.junit.experimental.categories.Category;
  45
  46 @Category(MediumTests.class)
  47 public class TestExpiredMobFileCleaner {
  48
  49   @ClassRule
  50   public static final HBaseClassTestRule CLASS_RULE =
  51       HBaseClassTestRule.forClass(TestExpiredMobFileCleaner.class);
  52
  53   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
  54   private final static TableName tableName = TableName.valueOf("TestExpiredMobFileCleaner");
  55   private final static String family = "family";
  56   private final static byte[] row1 = Bytes.toBytes("row1");
  57   private final static byte[] row2 = Bytes.toBytes("row2");
  58   private final static byte[] qf = Bytes.toBytes("qf");
  59
  60   private static BufferedMutator table;
  61   private static Admin admin;
  62
  63   @BeforeClass
  64   public static void setUpBeforeClass() throws Exception {
  65     TEST_UTIL.getConfiguration().setInt("hfile.format.version", 3);
  66   }
  67
  68   @AfterClass
  69   public static void tearDownAfterClass() throws Exception {
  70
  71   }
  72
  73   @Before
  74   public void setUp() throws Exception {
  75     TEST_UTIL.startMiniCluster(1);
  76   }
  77
  78   @After
  79   public void tearDown() throws Exception {
  80     admin.disableTable(tableName);
  81     admin.deleteTable(tableName);
  82     admin.close();
  83     TEST_UTIL.shutdownMiniCluster();
  84     TEST_UTIL.getTestFileSystem().delete(TEST_UTIL.getDataTestDir(), true);
  85   }
  86
  87   private void init() throws Exception {
  88     TableDescriptorBuilder tableDescriptorBuilder =
  89       TableDescriptorBuilder.newBuilder(tableName);
  90     ColumnFamilyDescriptor columnFamilyDescriptor =
  91       ColumnFamilyDescriptorBuilder
  92         .newBuilder(Bytes.toBytes(family))
  93         .setMobEnabled(true)
  94         .setMobThreshold(3L)
  95         .setMaxVersions(4)
  96         .build();
  97     tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
  98
  99     admin = TEST_UTIL.getAdmin();
 100     admin.createTable(tableDescriptorBuilder.build());
 101     table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())
 102             .getBufferedMutator(tableName);
 103   }
 104
 105   private void modifyColumnExpiryDays(int expireDays) throws Exception {
 106     ColumnFamilyDescriptorBuilder columnFamilyDescriptorBuilder =
 107       ColumnFamilyDescriptorBuilder
 108         .newBuilder(Bytes.toBytes(family))
 109         .setMobEnabled(true)
 110         .setMobThreshold(3L);
 111     // change ttl as expire days to make some row expired
 112     int timeToLive = expireDays * secondsOfDay();
 113     columnFamilyDescriptorBuilder.setTimeToLive(timeToLive);
 114
 115     admin.modifyColumnFamily(tableName, columnFamilyDescriptorBuilder.build());
 116   }
 117
 118   private void putKVAndFlush(BufferedMutator table, byte[] row, byte[] value, long ts)
 119       throws Exception {
 120
 121     Put put = new Put(row, ts);
 122     put.addColumn(Bytes.toBytes(family), qf, value);
 123     table.mutate(put);
 124
 125     table.flush();
 126     admin.flush(tableName);
 127   }
 128
 129   /**
 130    * Creates a 3 day old hfile and an 1 day old hfile then sets expiry to 2 days.
 131    * Verifies that the 3 day old hfile is removed but the 1 day one is still present
 132    * after the expiry based cleaner is run.
 133    */
 134   @Test
 135   public void testCleaner() throws Exception {
 136     init();
 137
 138     Path mobDirPath = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, family);
 139
 140     byte[] dummyData = makeDummyData(600);
 141     long ts = System.currentTimeMillis() - 3 * secondsOfDay() * 1000; // 3 days before
 142     putKVAndFlush(table, row1, dummyData, ts);
 143     FileStatus[] firstFiles = TEST_UTIL.getTestFileSystem().listStatus(mobDirPath);
 144     //the first mob file
 145     assertEquals("Before cleanup without delay 1", 1, firstFiles.length);
 146     String firstFile = firstFiles[0].getPath().getName();
 147
 148     ts = System.currentTimeMillis() - 1 * secondsOfDay() * 1000; // 1 day before
 149     putKVAndFlush(table, row2, dummyData, ts);
 150     FileStatus[] secondFiles = TEST_UTIL.getTestFileSystem().listStatus(mobDirPath);
 151     //now there are 2 mob files
 152     assertEquals("Before cleanup without delay 2", 2, secondFiles.length);
 153     String f1 = secondFiles[0].getPath().getName();
 154     String f2 = secondFiles[1].getPath().getName();
 155     String secondFile = f1.equals(firstFile) ? f2 : f1;
 156
 157     modifyColumnExpiryDays(2); // ttl = 2, make the first row expired
 158
 159     //run the cleaner
 160     String[] args = new String[2];
 161     args[0] = tableName.getNameAsString();
 162     args[1] = family;
 163     ToolRunner.run(TEST_UTIL.getConfiguration(), new ExpiredMobFileCleaner(), args);
 164
 165     FileStatus[] filesAfterClean = TEST_UTIL.getTestFileSystem().listStatus(mobDirPath);
 166     String lastFile = filesAfterClean[0].getPath().getName();
 167     //the first mob fie is removed
 168     assertEquals("After cleanup without delay 1", 1, filesAfterClean.length);
 169     assertEquals("After cleanup without delay 2", secondFile, lastFile);
 170   }
 171
 172   private int secondsOfDay() {
 173     return 24 * 3600;
 174   }
 175
 176   private byte[] makeDummyData(int size) {
 177     byte [] dummyData = new byte[size];
 178     new Random().nextBytes(dummyData);
 179     return dummyData;
 180   }
 181 }