hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionAfterBulkLoad.java

   1 /*
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.regionserver;
  19
  20 import static org.apache.hadoop.hbase.regionserver.HRegion.COMPACTION_AFTER_BULKLOAD_ENABLE;
  21
  22 import static org.junit.Assert.assertEquals;
  23 import static org.mockito.ArgumentMatchers.any;
  24 import static org.mockito.Mockito.mock;
  25 import static org.mockito.Mockito.when;
  26 import static org.mockito.hamcrest.MockitoHamcrest.argThat;
  27 import java.io.IOException;
  28 import java.util.ArrayList;
  29 import java.util.List;
  30 import java.util.concurrent.atomic.AtomicInteger;
  31
  32 import org.apache.hadoop.conf.Configuration;
  33 import org.apache.hadoop.fs.Path;
  34 import org.apache.hadoop.hbase.HBaseClassTestRule;
  35 import org.apache.hadoop.hbase.HBaseConfiguration;
  36 import org.apache.hadoop.hbase.TableName;
  37 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
  38 import org.apache.hadoop.hbase.client.RegionInfo;
  39 import org.apache.hadoop.hbase.client.RegionInfoBuilder;
  40 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
  41 import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
  42 import org.apache.hadoop.hbase.security.User;
  43 import org.apache.hadoop.hbase.testclassification.SmallTests;
  44 import org.apache.hadoop.hbase.util.Bytes;
  45 import org.apache.hadoop.hbase.util.Pair;
  46 import org.apache.hadoop.hbase.wal.WALEdit;
  47 import org.apache.hadoop.hbase.wal.WALKeyImpl;
  48 import org.junit.ClassRule;
  49 import org.junit.Test;
  50 import org.junit.experimental.categories.Category;
  51 import org.mockito.invocation.InvocationOnMock;
  52 import org.mockito.stubbing.Answer;
  53
  54 @Category(SmallTests.class)
  55 public class TestCompactionAfterBulkLoad extends TestBulkloadBase {
  56
  57   @ClassRule
  58   public static final HBaseClassTestRule CLASS_RULE =
  59     HBaseClassTestRule.forClass(TestCompactionAfterBulkLoad.class);
  60
  61   private final RegionServerServices regionServerServices = mock(RegionServerServices.class);
  62   public static AtomicInteger called = new AtomicInteger(0);
  63
  64   public TestCompactionAfterBulkLoad(boolean useFileBasedSFT) {
  65     super(useFileBasedSFT);
  66   }
  67
  68   @Override
  69   protected HRegion testRegionWithFamiliesAndSpecifiedTableName(TableName tableName,
  70       byte[]... families) throws IOException {
  71     RegionInfo hRegionInfo = RegionInfoBuilder.newBuilder(tableName).build();
  72     TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
  73
  74     for (byte[] family : families) {
  75       builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family));
  76     }
  77     ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null,
  78       MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);
  79     // TODO We need a way to do this without creating files
  80     return HRegion.createHRegion(hRegionInfo, new Path(testFolder.newFolder().toURI()), conf,
  81       builder.build(), log, true, regionServerServices);
  82
  83   }
  84
  85   @Test
  86   public void shouldRequestCompactAllStoresAfterBulkLoad() throws IOException {
  87     final CompactSplit compactSplit = new TestCompactSplit(HBaseConfiguration.create());
  88     called.set(0);
  89     List<Pair<byte[], String>> familyPaths = new ArrayList<>();
  90     // enough hfile to request compaction
  91     for (int i = 0; i < 5; i++) {
  92       familyPaths.addAll(withFamilyPathsFor(family1, family2, family3));
  93     }
  94     try {
  95       conf.setBoolean(COMPACTION_AFTER_BULKLOAD_ENABLE, true);
  96       when(regionServerServices.getConfiguration()).thenReturn(conf);
  97       when(regionServerServices.getCompactionRequestor()).thenReturn(compactSplit);
  98       when(log.appendMarker(any(), any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD))))
  99           .thenAnswer(new Answer() {
 100             @Override
 101             public Object answer(InvocationOnMock invocation) {
 102               WALKeyImpl walKey = invocation.getArgument(1);
 103               MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
 104               if (mvcc != null) {
 105                 MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
 106                 walKey.setWriteEntry(we);
 107               }
 108               return 01L;
 109             }
 110           });
 111
 112       HRegion region = testRegionWithFamilies(family1, family2, family3);
 113       region.bulkLoadHFiles(familyPaths, false, null);
 114       assertEquals(3, called.get());
 115     } finally {
 116       conf.setBoolean(COMPACTION_AFTER_BULKLOAD_ENABLE, false);
 117     }
 118   }
 119
 120   @Test
 121   public void testAvoidRepeatedlyRequestCompactAfterBulkLoad() throws IOException {
 122     final CompactSplit compactSplit = new TestFamily1UnderCompact(HBaseConfiguration.create());
 123     called.set(0);
 124     List<Pair<byte[], String>> familyPaths = new ArrayList<>();
 125     // enough hfile to request compaction
 126     for (int i = 0; i < 5; i++) {
 127       familyPaths.addAll(withFamilyPathsFor(family1, family2, family3));
 128     }
 129     try {
 130       conf.setBoolean(COMPACTION_AFTER_BULKLOAD_ENABLE, true);
 131       when(regionServerServices.getConfiguration()).thenReturn(conf);
 132       when(regionServerServices.getCompactionRequestor()).thenReturn(compactSplit);
 133       when(log.appendMarker(any(), any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD))))
 134         .thenAnswer(new Answer() {
 135           @Override
 136           public Object answer(InvocationOnMock invocation) {
 137             WALKeyImpl walKey = invocation.getArgument(1);
 138             MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
 139             if (mvcc != null) {
 140               MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
 141               walKey.setWriteEntry(we);
 142             }
 143             return 01L;
 144           }
 145         });
 146
 147       HRegion region = testRegionWithFamilies(family1, family2, family3);
 148       region.bulkLoadHFiles(familyPaths, false, null);
 149       // invoke three times for 2 families
 150       assertEquals(2, called.get());
 151     } finally {
 152       conf.setBoolean(COMPACTION_AFTER_BULKLOAD_ENABLE, false);
 153     }
 154   }
 155
 156   private class TestCompactSplit extends CompactSplit {
 157
 158     TestCompactSplit(Configuration conf) {
 159       super(conf);
 160     }
 161
 162     @Override
 163     protected void requestCompactionInternal(HRegion region, HStore store, String why, int priority,
 164       boolean selectNow, CompactionLifeCycleTracker tracker,
 165       CompactionCompleteTracker completeTracker, User user) throws IOException {
 166       called.addAndGet(1);
 167     }
 168   }
 169
 170   private class TestFamily1UnderCompact extends TestCompactSplit {
 171
 172     TestFamily1UnderCompact(Configuration conf) {
 173       super(conf);
 174     }
 175
 176     @Override
 177     public boolean isUnderCompaction(final HStore s) {
 178       if (s.getColumnFamilyName().equals(Bytes.toString(family1))) {
 179         return true;
 180       }
 181       return super.isUnderCompaction(s);
 182     }
 183   }
 184
 185 }