hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.regionserver;
  19
  20 import java.io.IOException;
  21 import java.util.ArrayList;
  22 import java.util.List;
  23 import org.apache.hadoop.hbase.HBaseClassTestRule;
  24 import org.apache.hadoop.hbase.HConstants;
  25 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl;
  26 import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
  27 import org.apache.hadoop.hbase.testclassification.SmallTests;
  28 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
  29 import org.apache.hadoop.hbase.util.TimeOffsetEnvironmentEdge;
  30 import org.junit.Assert;
  31 import org.junit.ClassRule;
  32 import org.junit.Test;
  33 import org.junit.experimental.categories.Category;
  34
  35 @Category(SmallTests.class)
  36 public class TestDefaultCompactSelection extends TestCompactionPolicy {
  37
  38   @ClassRule
  39   public static final HBaseClassTestRule CLASS_RULE =
  40       HBaseClassTestRule.forClass(TestDefaultCompactSelection.class);
  41
  42   @Override
  43   protected void config() {
  44     super.config();
  45     // DON'T change this config since all test cases assume HStore.BLOCKING_STOREFILES_KEY is 10.
  46     this.conf.setLong(HStore.BLOCKING_STOREFILES_KEY, 10);
  47   }
  48
  49   @Test
  50   public void testCompactionRatio() throws IOException {
  51     TimeOffsetEnvironmentEdge edge = new TimeOffsetEnvironmentEdge();
  52     EnvironmentEdgeManager.injectEdge(edge);
  53     /**
  54      * NOTE: these tests are specific to describe the implementation of the
  55      * current compaction algorithm.  Developed to ensure that refactoring
  56      * doesn't implicitly alter this.
  57      */
  58     long tooBig = maxSize + 1;
  59
  60     // default case. preserve user ratio on size
  61     compactEquals(sfCreate(100,50,23,12,12), 23, 12, 12);
  62     // less than compact threshold = don't compact
  63     compactEquals(sfCreate(100,50,25,12,12) /* empty */);
  64     // greater than compact size = skip those
  65     compactEquals(sfCreate(tooBig, tooBig, 700, 700, 700), 700, 700, 700);
  66     // big size + threshold
  67     compactEquals(sfCreate(tooBig, tooBig, 700,700) /* empty */);
  68     // small files = don't care about ratio
  69     compactEquals(sfCreate(7,1,1), 7,1,1);
  70
  71     // don't exceed max file compact threshold
  72     // note:  file selection starts with largest to smallest.
  73     compactEquals(sfCreate(7, 6, 5, 4, 3, 2, 1), 5, 4, 3, 2, 1);
  74
  75     compactEquals(sfCreate(50, 10, 10 ,10, 10), 10, 10, 10, 10);
  76
  77     compactEquals(sfCreate(10, 10, 10, 10, 50), 10, 10, 10, 10);
  78
  79     compactEquals(sfCreate(251, 253, 251, maxSize -1), 251, 253, 251);
  80
  81     compactEquals(sfCreate(maxSize -1,maxSize -1,maxSize -1) /* empty */);
  82
  83     // Always try and compact something to get below blocking storefile count
  84     this.conf.setLong("hbase.hstore.compaction.min.size", 1);
  85     store.storeEngine.getCompactionPolicy().setConf(conf);
  86     compactEquals(sfCreate(512,256,128,64,32,16,8,4,2,1), 4,2,1);
  87     this.conf.setLong("hbase.hstore.compaction.min.size", minSize);
  88     store.storeEngine.getCompactionPolicy().setConf(conf);
  89
  90     /* MAJOR COMPACTION */
  91     // if a major compaction has been forced, then compact everything
  92     compactEquals(sfCreate(50,25,12,12), true, 50, 25, 12, 12);
  93     // also choose files < threshold on major compaction
  94     compactEquals(sfCreate(12,12), true, 12, 12);
  95     // even if one of those files is too big
  96     compactEquals(sfCreate(tooBig, 12,12), true, tooBig, 12, 12);
  97     // don't exceed max file compact threshold, even with major compaction
  98     store.forceMajor = true;
  99     compactEquals(sfCreate(7, 6, 5, 4, 3, 2, 1), 5, 4, 3, 2, 1);
 100     store.forceMajor = false;
 101     // if we exceed maxCompactSize, downgrade to minor
 102     // if not, it creates a 'snowball effect' when files >> maxCompactSize:
 103     // the last file in compaction is the aggregate of all previous compactions
 104     compactEquals(sfCreate(100,50,23,12,12), true, 23, 12, 12);
 105     conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 1);
 106     conf.setFloat("hbase.hregion.majorcompaction.jitter", 0);
 107     store.storeEngine.getCompactionPolicy().setConf(conf);
 108     try {
 109       // The modTime of the mocked store file is the current time, so we need to increase the
 110       // timestamp a bit to make sure that now - lowestModTime is greater than major compaction
 111       // period(1ms).
 112       // trigger an aged major compaction
 113       List<HStoreFile> candidates = sfCreate(50, 25, 12, 12);
 114       edge.increment(2);
 115       compactEquals(candidates, 50, 25, 12, 12);
 116       // major sure exceeding maxCompactSize also downgrades aged minors
 117       candidates = sfCreate(100, 50, 23, 12, 12);
 118       edge.increment(2);
 119       compactEquals(candidates, 23, 12, 12);
 120     } finally {
 121       conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 1000*60*60*24);
 122       conf.setFloat("hbase.hregion.majorcompaction.jitter", 0.20F);
 123     }
 124
 125     /* REFERENCES == file is from a region that was split */
 126     // treat storefiles that have references like a major compaction
 127     compactEquals(sfCreate(true, 100,50,25,12,12), 100, 50, 25, 12, 12);
 128     // reference files shouldn't obey max threshold
 129     compactEquals(sfCreate(true, tooBig, 12,12), tooBig, 12, 12);
 130     // reference files should obey max file compact to avoid OOM
 131     compactEquals(sfCreate(true, 7, 6, 5, 4, 3, 2, 1), 7, 6, 5, 4, 3);
 132
 133     // empty case
 134     compactEquals(new ArrayList<>() /* empty */);
 135     // empty case (because all files are too big)
 136     compactEquals(sfCreate(tooBig, tooBig) /* empty */);
 137   }
 138
 139   @Test
 140   public void testOffPeakCompactionRatio() throws IOException {
 141     /*
 142      * NOTE: these tests are specific to describe the implementation of the
 143      * current compaction algorithm.  Developed to ensure that refactoring
 144      * doesn't implicitly alter this.
 145      */
 146     // set an off-peak compaction threshold
 147     this.conf.setFloat("hbase.hstore.compaction.ratio.offpeak", 5.0F);
 148     store.storeEngine.getCompactionPolicy().setConf(this.conf);
 149     // Test with and without the flag.
 150     compactEquals(sfCreate(999, 50, 12, 12, 1), false, true, 50, 12, 12, 1);
 151     compactEquals(sfCreate(999, 50, 12, 12, 1), 12, 12, 1);
 152   }
 153
 154   @Test
 155   public void testStuckStoreCompaction() throws IOException {
 156     // Select the smallest compaction if the store is stuck.
 157     compactEquals(sfCreate(99,99,99,99,99,99, 30,30,30,30), 30, 30, 30);
 158     // If not stuck, standard policy applies.
 159     compactEquals(sfCreate(99,99,99,99,99, 30,30,30,30), 99, 30, 30, 30, 30);
 160
 161     // Add sufficiently small files to compaction, though
 162     compactEquals(sfCreate(99,99,99,99,99,99, 30,30,30,15), 30, 30, 30, 15);
 163     // Prefer earlier compaction to latter if the benefit is not significant
 164     compactEquals(sfCreate(99,99,99,99, 30,26,26,29,25,25), 30, 26, 26);
 165     // Prefer later compaction if the benefit is significant.
 166     compactEquals(sfCreate(99,99,99,99, 27,27,27,20,20,20), 20, 20, 20);
 167   }
 168
 169   @Test
 170   public void testCompactionEmptyHFile() throws IOException {
 171     // Set TTL
 172     ScanInfo oldScanInfo = store.getScanInfo();
 173     ScanInfo newScanInfo = oldScanInfo.customize(oldScanInfo.getMaxVersions(), 600,
 174         oldScanInfo.getKeepDeletedCells());
 175     store.setScanInfo(newScanInfo);
 176     // Do not compact empty store file
 177     List<HStoreFile> candidates = sfCreate(0);
 178     for (HStoreFile file : candidates) {
 179       if (file instanceof MockHStoreFile) {
 180         MockHStoreFile mockFile = (MockHStoreFile) file;
 181         mockFile.setTimeRangeTracker(TimeRangeTracker.create(TimeRangeTracker.Type.SYNC, -1, -1));
 182         mockFile.setEntries(0);
 183       }
 184     }
 185     // Test Default compactions
 186     CompactionRequestImpl result = ((RatioBasedCompactionPolicy) store.storeEngine
 187         .getCompactionPolicy()).selectCompaction(candidates,
 188         new ArrayList<>(), false, false, false);
 189     Assert.assertTrue(result.getFiles().isEmpty());
 190     store.setScanInfo(oldScanInfo);
 191   }
 192 }