hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionBase.java

   1 /*
   2  *
   3  * Licensed to the Apache Software Foundation (ASF) under one
   4  * or more contributor license agreements.  See the NOTICE file
   5  * distributed with this work for additional information
   6  * regarding copyright ownership.  The ASF licenses this file
   7  * to you under the Apache License, Version 2.0 (the
   8  * "License"); you may not use this file except in compliance
   9  * with the License.  You may obtain a copy of the License at
  10  *
  11  *     http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  */
  19 package org.apache.hadoop.hbase.mob;
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertTrue;
  22 import static org.junit.Assert.fail;
  23
  24 import java.io.IOException;
  25 import java.util.Arrays;
  26 import java.util.Random;
  27
  28 import org.apache.hadoop.conf.Configuration;
  29 import org.apache.hadoop.fs.FileStatus;
  30 import org.apache.hadoop.fs.FileSystem;
  31 import org.apache.hadoop.fs.Path;
  32 import org.apache.hadoop.hbase.HBaseTestingUtility;
  33 import org.apache.hadoop.hbase.TableName;
  34 import org.apache.hadoop.hbase.client.Admin;
  35 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
  36 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
  37 import org.apache.hadoop.hbase.client.CompactionState;
  38 import org.apache.hadoop.hbase.client.Put;
  39 import org.apache.hadoop.hbase.client.Result;
  40 import org.apache.hadoop.hbase.client.ResultScanner;
  41 import org.apache.hadoop.hbase.client.Table;
  42 import org.apache.hadoop.hbase.client.TableDescriptor;
  43 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
  44 import org.apache.hadoop.hbase.util.Bytes;
  45 import org.apache.hadoop.hbase.util.RegionSplitter;
  46 import org.junit.After;
  47 import org.junit.Before;
  48 import org.slf4j.Logger;
  49 import org.slf4j.LoggerFactory;
  50
  51 /**
  52   * Mob file compaction base test.
  53   * 1. Enables batch mode for regular MOB compaction,
  54   *    Sets batch size to 7 regions. (Optional)
  55   * 2. Disables periodic MOB compactions, sets minimum age to archive to 10 sec
  56   * 3. Creates MOB table with 20 regions
  57   * 4. Loads MOB data (randomized keys, 1000 rows), flushes data.
  58   * 5. Repeats 4. two more times
  59   * 6. Verifies that we have 20 *3 = 60 mob files (equals to number of regions x 3)
  60   * 7. Runs major MOB compaction.
  61   * 8. Verifies that number of MOB files in a mob directory is 20 x4 = 80
  62   * 9. Waits for a period of time larger than minimum age to archive
  63   * 10. Runs Mob cleaner chore
  64   * 11 Verifies that number of MOB files in a mob directory is 20.
  65   * 12 Runs scanner and checks all 3 * 1000 rows.
  66  */
  67 @SuppressWarnings("deprecation")
  68 public abstract class TestMobCompactionBase {
  69   private static final Logger LOG =
  70       LoggerFactory.getLogger(TestMobCompactionBase.class);
  71
  72   protected HBaseTestingUtility HTU;
  73
  74   protected final static String famStr = "f1";
  75   protected final static byte[] fam = Bytes.toBytes(famStr);
  76   protected final static byte[] qualifier = Bytes.toBytes("q1");
  77   protected final static long mobLen = 10;
  78   protected final static byte[] mobVal = Bytes
  79       .toBytes("01234567890123456789012345678901234567890123456789012345678901234567890123456789");
  80
  81   protected Configuration conf;
  82   protected TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor;
  83   private ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor;
  84   protected Admin admin;
  85   protected Table table = null;
  86   protected long minAgeToArchive = 10000;
  87   protected int numRegions = 20;
  88   protected int rows = 1000;
  89
  90   protected MobFileCleanerChore cleanerChore;
  91
  92   public TestMobCompactionBase() {
  93   }
  94
  95
  96   @Before
  97   public void setUp() throws Exception {
  98     HTU = new HBaseTestingUtility();
  99     tableDescriptor = HTU.createModifyableTableDescriptor(getClass().getName());
 100     conf = HTU.getConfiguration();
 101
 102     initConf();
 103
 104     HTU.startMiniCluster();
 105     admin = HTU.getAdmin();
 106     cleanerChore = new MobFileCleanerChore();
 107     familyDescriptor = new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(fam);
 108     familyDescriptor.setMobEnabled(true);
 109     familyDescriptor.setMobThreshold(mobLen);
 110     familyDescriptor.setMaxVersions(1);
 111     tableDescriptor.setColumnFamily(familyDescriptor);
 112     RegionSplitter.UniformSplit splitAlgo = new RegionSplitter.UniformSplit();
 113     byte[][] splitKeys = splitAlgo.split(numRegions);
 114     table = HTU.createTable(tableDescriptor, splitKeys);
 115
 116   }
 117
 118   protected void initConf() {
 119
 120     conf.setInt("hfile.format.version", 3);
 121     // Disable automatic MOB compaction
 122     conf.setLong(MobConstants.MOB_COMPACTION_CHORE_PERIOD, 0);
 123     // Disable automatic MOB file cleaner chore
 124     conf.setLong(MobConstants.MOB_CLEANER_PERIOD, 0);
 125     // Set minimum age to archive to 10 sec
 126     conf.setLong(MobConstants.MIN_AGE_TO_ARCHIVE_KEY, minAgeToArchive);
 127     // Set compacted file discharger interval to a half minAgeToArchive
 128     conf.setLong("hbase.hfile.compaction.discharger.interval", minAgeToArchive/2);
 129   }
 130
 131   private void loadData(int num) {
 132
 133     Random r = new Random();
 134     try {
 135       LOG.info("Started loading {} rows", num);
 136       for (int i = 0; i < num; i++) {
 137         byte[] key = new byte[32];
 138         r.nextBytes(key);
 139         Put p = new Put(key);
 140         p.addColumn(fam, qualifier, mobVal);
 141         table.put(p);
 142       }
 143       admin.flush(table.getName());
 144       LOG.info("Finished loading {} rows", num);
 145     } catch (Exception e) {
 146       LOG.error("MOB file compaction chore test FAILED", e);
 147       fail("MOB file compaction chore test FAILED");
 148     }
 149   }
 150
 151   @After
 152   public void tearDown() throws Exception {
 153     admin.disableTable(tableDescriptor.getTableName());
 154     admin.deleteTable(tableDescriptor.getTableName());
 155     HTU.shutdownMiniCluster();
 156   }
 157
 158
 159   public void baseTestMobFileCompaction() throws InterruptedException, IOException {
 160
 161     // Load and flush data 3 times
 162     loadData(rows);
 163     loadData(rows);
 164     loadData(rows);
 165     long num = getNumberOfMobFiles(conf, table.getName(), new String(fam));
 166     assertEquals(numRegions * 3, num);
 167     // Major MOB compact
 168     mobCompact(admin, tableDescriptor, familyDescriptor);
 169     // wait until compaction is complete
 170     while (admin.getCompactionState(tableDescriptor.getTableName()) != CompactionState.NONE) {
 171       Thread.sleep(100);
 172     }
 173
 174     num = getNumberOfMobFiles(conf, table.getName(), new String(fam));
 175     assertEquals(numRegions * 4, num);
 176     // We have guarantee, that compacted file discharger will run during this pause
 177     // because it has interval less than this wait time
 178     LOG.info("Waiting for {}ms", minAgeToArchive + 1000);
 179
 180     Thread.sleep(minAgeToArchive + 1000);
 181     LOG.info("Cleaning up MOB files");
 182     // Cleanup again
 183     cleanerChore.cleanupObsoleteMobFiles(conf, table.getName());
 184
 185     num = getNumberOfMobFiles(conf, table.getName(), new String(fam));
 186     assertEquals(numRegions, num);
 187
 188     long scanned = scanTable();
 189     assertEquals(3 * rows, scanned);
 190
 191   }
 192
 193   protected abstract void mobCompact(Admin admin2, TableDescriptor tableDescriptor,
 194       ColumnFamilyDescriptor familyDescriptor) throws IOException, InterruptedException;
 195
 196
 197   protected  long getNumberOfMobFiles(Configuration conf, TableName tableName, String family)
 198       throws IOException {
 199     FileSystem fs = FileSystem.get(conf);
 200     Path dir = MobUtils.getMobFamilyPath(conf, tableName, family);
 201     FileStatus[] stat = fs.listStatus(dir);
 202     for (FileStatus st : stat) {
 203       LOG.debug("MOB Directory content: {}", st.getPath());
 204     }
 205     LOG.debug("MOB Directory content total files: {}", stat.length);
 206
 207     return stat.length;
 208   }
 209
 210
 211   protected long scanTable() {
 212     try {
 213
 214       Result result;
 215       ResultScanner scanner = table.getScanner(fam);
 216       long counter = 0;
 217       while ((result = scanner.next()) != null) {
 218         assertTrue(Arrays.equals(result.getValue(fam, qualifier), mobVal));
 219         counter++;
 220       }
 221       return counter;
 222     } catch (Exception e) {
 223       LOG.error("MOB file compaction test FAILED", e);
 224       if (HTU != null) {
 225         fail(e.getMessage());
 226       } else {
 227         System.exit(-1);
 228       }
 229     }
 230     return 0;
 231   }
 232 }