hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompactionPolicy.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.regionserver;
  19
  20 import java.io.IOException;
  21 import org.apache.hadoop.hbase.HBaseClassTestRule;
  22 import org.apache.hadoop.hbase.HConstants;
  23 import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
  24 import org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory;
  25 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
  26 import org.apache.hadoop.hbase.testclassification.SmallTests;
  27 import org.junit.ClassRule;
  28 import org.junit.Test;
  29 import org.junit.experimental.categories.Category;
  30
  31 @Category({ RegionServerTests.class, SmallTests.class })
  32 public class TestDateTieredCompactionPolicy extends AbstractTestDateTieredCompactionPolicy {
  33
  34   @ClassRule
  35   public static final HBaseClassTestRule CLASS_RULE =
  36       HBaseClassTestRule.forClass(TestDateTieredCompactionPolicy.class);
  37
  38   @Override
  39   protected void config() {
  40     super.config();
  41
  42     // Set up policy
  43     conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY,
  44       "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine");
  45     conf.setLong(CompactionConfiguration.DATE_TIERED_MAX_AGE_MILLIS_KEY, 100);
  46     conf.setLong(CompactionConfiguration.DATE_TIERED_INCOMING_WINDOW_MIN_KEY, 3);
  47     conf.setLong(ExponentialCompactionWindowFactory.BASE_WINDOW_MILLIS_KEY, 6);
  48     conf.setInt(ExponentialCompactionWindowFactory.WINDOWS_PER_TIER_KEY, 4);
  49     conf.setBoolean(CompactionConfiguration.DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY,
  50       false);
  51
  52     // Special settings for compaction policy per window
  53     this.conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, 2);
  54     this.conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_KEY, 12);
  55     this.conf.setFloat(CompactionConfiguration.HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F);
  56
  57     conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 20);
  58     conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 5);
  59   }
  60
  61   /**
  62    * Test for incoming window
  63    * @throws IOException with error
  64    */
  65   @Test
  66   public void incomingWindow() throws IOException {
  67     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
  68     long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
  69     long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 };
  70
  71     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11, 12, 13 },
  72       new long[] { Long.MIN_VALUE, 12 }, false, true);
  73   }
  74
  75   /**
  76    * Not enough files in incoming window
  77    * @throws IOException with error
  78    */
  79   @Test
  80   public void NotIncomingWindow() throws IOException {
  81     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
  82     long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 };
  83     long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11 };
  84
  85     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 20, 21, 22, 23,
  86         24, 25 }, new long[] { Long.MIN_VALUE, 6}, false, true);
  87   }
  88
  89   /**
  90    * Test for file on the upper bound of incoming window
  91    * @throws IOException with error
  92    */
  93   @Test
  94   public void OnUpperBoundOfIncomingWindow() throws IOException {
  95     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
  96     long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 18 };
  97     long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 };
  98
  99     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11, 12, 13 },
 100       new long[] { Long.MIN_VALUE, 12 }, false, true);
 101   }
 102
 103   /**
 104    * Test for file newer than incoming window
 105    * @throws IOException with error
 106    */
 107   @Test
 108   public void NewerThanIncomingWindow() throws IOException {
 109     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
 110     long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 19 };
 111     long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 };
 112
 113     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11, 12, 13 },
 114       new long[] { Long.MIN_VALUE, 12}, false, true);
 115   }
 116
 117   /**
 118    * If there is no T1 window, we don't build T2
 119    * @throws IOException with error
 120    */
 121   @Test
 122   public void NoT2() throws IOException {
 123     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
 124     long[] maxTimestamps = new long[] { 44, 60, 61, 97, 100, 193 };
 125     long[] sizes = new long[] { 0, 20, 21, 22, 23, 1 };
 126
 127     compactEquals(194, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 22, 23 },
 128       new long[] { Long.MIN_VALUE, 96}, false, true);
 129   }
 130
 131   @Test
 132   public void T1() throws IOException {
 133     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
 134     long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 120, 124, 143, 145, 157 };
 135     long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 30, 31, 32, 2, 1 };
 136
 137     compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 30, 31, 32 },
 138       new long[] { Long.MIN_VALUE, 120 }, false, true);
 139   }
 140
 141   /**
 142    * Apply exploring logic on non-incoming window
 143    * @throws IOException with error
 144    */
 145   @Test
 146   public void RatioT0() throws IOException {
 147     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
 148     long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 };
 149     long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 280, 23, 24, 1 };
 150
 151     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 20, 21, 22 },
 152       new long[] { Long.MIN_VALUE }, false, true);
 153   }
 154
 155   /**
 156    * Also apply ratio-based logic on t2 window
 157    * @throws IOException with error
 158    */
 159   @Test
 160   public void RatioT2() throws IOException {
 161     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
 162     long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 120, 124, 143, 145, 157 };
 163     long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 350, 30, 31, 2, 1 };
 164
 165     compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 30, 31 },
 166       new long[] { Long.MIN_VALUE }, false, true);
 167   }
 168
 169   /**
 170    * The next compaction call after testTieredCompactionRatioT0 is compacted
 171    * @throws IOException with error
 172    */
 173   @Test
 174   public void RatioT0Next() throws IOException {
 175     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
 176     long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 8, 9, 10, 11, 12 };
 177     long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 280, 23, 24, 1 };
 178
 179     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 23, 24 },
 180       new long[] { Long.MIN_VALUE }, false, true);
 181   }
 182
 183   /**
 184    * Older than now(161) - maxAge(100)
 185    * @throws IOException with error
 186    */
 187   @Test
 188   public void olderThanMaxAge() throws IOException {
 189     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
 190     long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 105, 106, 113, 145, 157 };
 191     long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
 192
 193     compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 40, 41, 42, 33,
 194         30, 31 }, new long[] { Long.MIN_VALUE, 96 }, false, true);
 195   }
 196
 197   /**
 198    * Out-of-order data
 199    * @throws IOException with error
 200    */
 201   @Test
 202   public void outOfOrder() throws IOException {
 203     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
 204     long[] maxTimestamps = new long[] { 0, 13, 3, 10, 11, 1, 2, 12, 14, 15 };
 205     long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 28, 23, 24, 1 };
 206
 207     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 31, 32, 33, 34,
 208         22, 28, 23, 24, 1 }, new long[] { Long.MIN_VALUE, 12 }, false, true);
 209   }
 210
 211   /**
 212    * Negative epoch time
 213    * @throws IOException with error
 214    */
 215   @Test
 216   public void negativeEpochtime() throws IOException {
 217     long[] minTimestamps =
 218         new long[] { -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000 };
 219     long[] maxTimestamps = new long[] { -28, -11, -10, -9, -8, -7, -6, -5, -4, -3 };
 220     long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 25, 23, 24, 1 };
 221
 222     compactEquals(1, sfCreate(minTimestamps, maxTimestamps, sizes),
 223       new long[] { 31, 32, 33, 34, 22, 25, 23, 24, 1 },
 224       new long[] { Long.MIN_VALUE, -24 }, false, true);
 225   }
 226
 227   /**
 228    * Major compaction
 229    * @throws IOException with error
 230    */
 231   @Test
 232   public void majorCompation() throws IOException {
 233     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
 234     long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 105, 106, 113, 145, 157 };
 235     long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
 236
 237     compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes),
 238       new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 },
 239       new long[] { Long.MIN_VALUE, 24, 48, 72, 96, 120, 144, 150, 156 }, true, true);
 240   }
 241
 242   /**
 243    * Major Compaction to check min max timestamp falling in the same window and also to check
 244    * boundary condition in which case binary sort gives insertion point as length of the array
 245    * @throws IOException
 246    */
 247   @Test
 248   public void checkMinMaxTimestampSameBoundary() throws IOException {
 249     long[] minTimestamps = new long[] { 0, 26, 50, 90, 98, 122, 145, 151, 158, 166 };
 250     long[] maxTimestamps = new long[] { 12, 46, 70, 95, 100, 140, 148, 155, 162, 174 };
 251     long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2 };
 252
 253     compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes),
 254       new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2 },
 255       new long[] { Long.MIN_VALUE, 24, 48, 72, 96, 120, 144, 150, 156 }, true, true);
 256   }
 257
 258   /**
 259    * Major compaction with negative numbers
 260    * @throws IOException with error
 261    */
 262   @Test
 263   public void negativeForMajor() throws IOException {
 264     long[] minTimestamps =
 265         new long[] { -155, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100 };
 266     long[] maxTimestamps = new long[] { -8, -7, -6, -5, -4, -3, -2, -1, 0, 6, 13 };
 267     long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
 268
 269     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 0, 50, 51, 40,
 270         41, 42, 33, 30, 31, 2, 1 },
 271       new long[] { Long.MIN_VALUE, -144, -120, -96, -72, -48, -24, 0, 6, 12 }, true, true);
 272   }
 273 }