hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionAfterBulkLoad.java

   1 /*
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.regionserver;
  19
  20 import static org.apache.hadoop.hbase.regionserver.HRegion.COMPACTION_AFTER_BULKLOAD_ENABLE;
  21
  22 import static org.junit.Assert.assertEquals;
  23 import static org.mockito.ArgumentMatchers.any;
  24 import static org.mockito.Mockito.mock;
  25 import static org.mockito.Mockito.when;
  26 import static org.mockito.hamcrest.MockitoHamcrest.argThat;
  27 import java.io.IOException;
  28 import java.util.ArrayList;
  29 import java.util.List;
  30 import java.util.concurrent.atomic.AtomicInteger;
  31
  32 import org.apache.hadoop.conf.Configuration;
  33 import org.apache.hadoop.fs.Path;
  34 import org.apache.hadoop.hbase.HBaseClassTestRule;
  35 import org.apache.hadoop.hbase.HBaseConfiguration;
  36 import org.apache.hadoop.hbase.TableName;
  37 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
  38 import org.apache.hadoop.hbase.client.RegionInfo;
  39 import org.apache.hadoop.hbase.client.RegionInfoBuilder;
  40 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
  41 import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
  42 import org.apache.hadoop.hbase.security.User;
  43 import org.apache.hadoop.hbase.testclassification.SmallTests;
  44 import org.apache.hadoop.hbase.util.Bytes;
  45 import org.apache.hadoop.hbase.util.Pair;
  46 import org.apache.hadoop.hbase.wal.WALEdit;
  47 import org.apache.hadoop.hbase.wal.WALKeyImpl;
  48 import org.junit.ClassRule;
  49 import org.junit.Test;
  50 import org.junit.experimental.categories.Category;
  51 import org.mockito.invocation.InvocationOnMock;
  52 import org.mockito.stubbing.Answer;
  53
  54 @Category(SmallTests.class)
  55 public class TestCompactionAfterBulkLoad extends TestBulkloadBase {
  56
  57   @ClassRule
  58   public static final HBaseClassTestRule CLASS_RULE =
  59     HBaseClassTestRule.forClass(TestCompactionAfterBulkLoad.class);
  60
  61   private final RegionServerServices regionServerServices = mock(RegionServerServices.class);
  62   public static AtomicInteger called = new AtomicInteger(0);
  63
  64   @Override
  65   protected HRegion testRegionWithFamiliesAndSpecifiedTableName(TableName tableName,
  66       byte[]... families) throws IOException {
  67     RegionInfo hRegionInfo = RegionInfoBuilder.newBuilder(tableName).build();
  68     TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
  69
  70     for (byte[] family : families) {
  71       builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family));
  72     }
  73     ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null,
  74       MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);
  75     // TODO We need a way to do this without creating files
  76     return HRegion.createHRegion(hRegionInfo, new Path(testFolder.newFolder().toURI()), conf,
  77       builder.build(), log, true, regionServerServices);
  78
  79   }
  80
  81   @Test
  82   public void shouldRequestCompactAllStoresAfterBulkLoad() throws IOException {
  83     final CompactSplit compactSplit = new TestCompactSplit(HBaseConfiguration.create());
  84     called.set(0);
  85     List<Pair<byte[], String>> familyPaths = new ArrayList<>();
  86     // enough hfile to request compaction
  87     for (int i = 0; i < 5; i++) {
  88       familyPaths.addAll(withFamilyPathsFor(family1, family2, family3));
  89     }
  90     try {
  91       conf.setBoolean(COMPACTION_AFTER_BULKLOAD_ENABLE, true);
  92       when(regionServerServices.getConfiguration()).thenReturn(conf);
  93       when(regionServerServices.getCompactionRequestor()).thenReturn(compactSplit);
  94       when(log.appendMarker(any(), any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD))))
  95           .thenAnswer(new Answer() {
  96             @Override
  97             public Object answer(InvocationOnMock invocation) {
  98               WALKeyImpl walKey = invocation.getArgument(1);
  99               MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
 100               if (mvcc != null) {
 101                 MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
 102                 walKey.setWriteEntry(we);
 103               }
 104               return 01L;
 105             }
 106           });
 107
 108       HRegion region = testRegionWithFamilies(family1, family2, family3);
 109       region.bulkLoadHFiles(familyPaths, false, null);
 110       assertEquals(3, called.get());
 111     } finally {
 112       conf.setBoolean(COMPACTION_AFTER_BULKLOAD_ENABLE, false);
 113     }
 114   }
 115
 116   @Test
 117   public void testAvoidRepeatedlyRequestCompactAfterBulkLoad() throws IOException {
 118     final CompactSplit compactSplit = new TestFamily1UnderCompact(HBaseConfiguration.create());
 119     called.set(0);
 120     List<Pair<byte[], String>> familyPaths = new ArrayList<>();
 121     // enough hfile to request compaction
 122     for (int i = 0; i < 5; i++) {
 123       familyPaths.addAll(withFamilyPathsFor(family1, family2, family3));
 124     }
 125     try {
 126       conf.setBoolean(COMPACTION_AFTER_BULKLOAD_ENABLE, true);
 127       when(regionServerServices.getConfiguration()).thenReturn(conf);
 128       when(regionServerServices.getCompactionRequestor()).thenReturn(compactSplit);
 129       when(log.appendMarker(any(), any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD))))
 130         .thenAnswer(new Answer() {
 131           @Override
 132           public Object answer(InvocationOnMock invocation) {
 133             WALKeyImpl walKey = invocation.getArgument(1);
 134             MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
 135             if (mvcc != null) {
 136               MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
 137               walKey.setWriteEntry(we);
 138             }
 139             return 01L;
 140           }
 141         });
 142
 143       HRegion region = testRegionWithFamilies(family1, family2, family3);
 144       region.bulkLoadHFiles(familyPaths, false, null);
 145       // invoke three times for 2 families
 146       assertEquals(2, called.get());
 147     } finally {
 148       conf.setBoolean(COMPACTION_AFTER_BULKLOAD_ENABLE, false);
 149     }
 150   }
 151
 152   private class TestCompactSplit extends CompactSplit {
 153
 154     TestCompactSplit(Configuration conf) {
 155       super(conf);
 156     }
 157
 158     @Override
 159     protected void requestCompactionInternal(HRegion region, HStore store, String why, int priority,
 160       boolean selectNow, CompactionLifeCycleTracker tracker,
 161       CompactionCompleteTracker completeTracker, User user) throws IOException {
 162       called.addAndGet(1);
 163     }
 164   }
 165
 166   private class TestFamily1UnderCompact extends TestCompactSplit {
 167
 168     TestFamily1UnderCompact(Configuration conf) {
 169       super(conf);
 170     }
 171
 172     @Override
 173     public boolean isUnderCompaction(final HStore s) {
 174       if (s.getColumnFamilyName().equals(Bytes.toString(family1))) {
 175         return true;
 176       }
 177       return super.isUnderCompaction(s);
 178     }
 179   }
 180
 181 }