hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java

   1 /*
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.regionserver;
  19
  20 import static java.util.Arrays.asList;
  21 import static org.mockito.ArgumentMatchers.any;
  22 import static org.mockito.ArgumentMatchers.anyLong;
  23 import static org.mockito.Mockito.verify;
  24 import static org.mockito.Mockito.when;
  25 import static org.mockito.hamcrest.MockitoHamcrest.argThat;
  26
  27 import java.io.FileNotFoundException;
  28 import java.io.IOException;
  29 import java.util.ArrayList;
  30 import java.util.List;
  31
  32 import org.apache.hadoop.fs.Path;
  33 import org.apache.hadoop.hbase.DoNotRetryIOException;
  34 import org.apache.hadoop.hbase.HBaseClassTestRule;
  35 import org.apache.hadoop.hbase.TableName;
  36 import org.apache.hadoop.hbase.testclassification.SmallTests;
  37 import org.apache.hadoop.hbase.util.Pair;
  38 import org.apache.hadoop.hbase.wal.WALEdit;
  39 import org.apache.hadoop.hbase.wal.WALKeyImpl;
  40 import org.junit.ClassRule;
  41 import org.junit.Test;
  42 import org.junit.experimental.categories.Category;
  43 import org.mockito.invocation.InvocationOnMock;
  44 import org.mockito.stubbing.Answer;
  45
  46
  47 /**
  48  * This class attempts to unit test bulk HLog loading.
  49  */
  50 @Category(SmallTests.class)
  51 public class TestBulkLoad extends TestBulkloadBase {
  52
  53   @ClassRule
  54   public static final HBaseClassTestRule CLASS_RULE =
  55       HBaseClassTestRule.forClass(TestBulkLoad.class);
  56
  57   public TestBulkLoad(boolean useFileBasedSFT) {
  58     super(useFileBasedSFT);
  59   }
  60
  61   @Test
  62   public void verifyBulkLoadEvent() throws IOException {
  63     TableName tableName = TableName.valueOf("test", "test");
  64     List<Pair<byte[], String>> familyPaths = withFamilyPathsFor(family1);
  65     byte[] familyName = familyPaths.get(0).getFirst();
  66     String storeFileName = familyPaths.get(0).getSecond();
  67     storeFileName = (new Path(storeFileName)).getName();
  68     List<String> storeFileNames = new ArrayList<>();
  69     storeFileNames.add(storeFileName);
  70     when(log.appendMarker(any(), any(),
  71       argThat(bulkLogWalEdit(WALEdit.BULK_LOAD, tableName.toBytes(), familyName, storeFileNames))))
  72         .thenAnswer(new Answer() {
  73           @Override
  74           public Object answer(InvocationOnMock invocation) {
  75             WALKeyImpl walKey = invocation.getArgument(1);
  76             MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
  77             if (mvcc != null) {
  78               MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
  79               walKey.setWriteEntry(we);
  80             }
  81             return 01L;
  82           }
  83         });
  84     testRegionWithFamiliesAndSpecifiedTableName(tableName, family1)
  85         .bulkLoadHFiles(familyPaths, false, null);
  86     verify(log).sync(anyLong());
  87   }
  88
  89   @Test
  90   public void bulkHLogShouldThrowNoErrorAndWriteMarkerWithBlankInput() throws IOException {
  91     testRegionWithFamilies(family1).bulkLoadHFiles(new ArrayList<>(),false, null);
  92   }
  93
  94   @Test
  95   public void shouldBulkLoadSingleFamilyHLog() throws IOException {
  96     when(log.appendMarker(any(), any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD))))
  97         .thenAnswer(new Answer() {
  98           @Override
  99           public Object answer(InvocationOnMock invocation) {
 100             WALKeyImpl walKey = invocation.getArgument(1);
 101             MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
 102             if (mvcc != null) {
 103               MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
 104               walKey.setWriteEntry(we);
 105             }
 106             return 01L;
 107           }
 108         });
 109     testRegionWithFamilies(family1).bulkLoadHFiles(withFamilyPathsFor(family1), false, null);
 110     verify(log).sync(anyLong());
 111   }
 112
 113   @Test
 114   public void shouldBulkLoadManyFamilyHLog() throws IOException {
 115     when(log.appendMarker(any(),
 116             any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)))).thenAnswer(new Answer() {
 117               @Override
 118               public Object answer(InvocationOnMock invocation) {
 119                 WALKeyImpl walKey = invocation.getArgument(1);
 120                 MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
 121                 if (mvcc != null) {
 122                   MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
 123                   walKey.setWriteEntry(we);
 124                 }
 125                 return 01L;
 126               }
 127             });
 128     testRegionWithFamilies(family1, family2).bulkLoadHFiles(withFamilyPathsFor(family1, family2),
 129             false, null);
 130     verify(log).sync(anyLong());
 131   }
 132
 133   @Test
 134   public void shouldBulkLoadManyFamilyHLogEvenWhenTableNameNamespaceSpecified() throws IOException {
 135     when(log.appendMarker(any(), any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD))))
 136         .thenAnswer(new Answer() {
 137           @Override
 138           public Object answer(InvocationOnMock invocation) {
 139             WALKeyImpl walKey = invocation.getArgument(1);
 140             MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
 141             if (mvcc != null) {
 142               MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
 143               walKey.setWriteEntry(we);
 144             }
 145             return 01L;
 146           }
 147         });
 148     TableName tableName = TableName.valueOf("test", "test");
 149     testRegionWithFamiliesAndSpecifiedTableName(tableName, family1, family2)
 150         .bulkLoadHFiles(withFamilyPathsFor(family1, family2), false, null);
 151     verify(log).sync(anyLong());
 152   }
 153
 154   @Test(expected = DoNotRetryIOException.class)
 155   public void shouldCrashIfBulkLoadFamiliesNotInTable() throws IOException {
 156     testRegionWithFamilies(family1).bulkLoadHFiles(withFamilyPathsFor(family1, family2), false,
 157       null);
 158   }
 159
 160   // after HBASE-24021 will throw DoNotRetryIOException, not MultipleIOException
 161   @Test(expected = DoNotRetryIOException.class)
 162   public void shouldCrashIfBulkLoadMultiFamiliesNotInTable() throws IOException {
 163     testRegionWithFamilies(family1).bulkLoadHFiles(withFamilyPathsFor(family1, family2, family3),
 164       false, null);
 165   }
 166
 167   @Test(expected = DoNotRetryIOException.class)
 168   public void bulkHLogShouldThrowErrorWhenFamilySpecifiedAndHFileExistsButNotInTableDescriptor()
 169       throws IOException {
 170     testRegionWithFamilies().bulkLoadHFiles(withFamilyPathsFor(family1), false, null);
 171   }
 172
 173   @Test(expected = DoNotRetryIOException.class)
 174   public void shouldThrowErrorIfBadFamilySpecifiedAsFamilyPath() throws IOException {
 175     testRegionWithFamilies()
 176         .bulkLoadHFiles(asList(withInvalidColumnFamilyButProperHFileLocation(family1)),
 177             false, null);
 178   }
 179
 180   @Test(expected = FileNotFoundException.class)
 181   public void shouldThrowErrorIfHFileDoesNotExist() throws IOException {
 182     List<Pair<byte[], String>> list = asList(withMissingHFileForFamily(family1));
 183     testRegionWithFamilies(family1).bulkLoadHFiles(list, false, null);
 184   }
 185
 186   // after HBASE-24021 will throw FileNotFoundException, not MultipleIOException
 187   @Test(expected = FileNotFoundException.class)
 188   public void shouldThrowErrorIfMultiHFileDoesNotExist() throws IOException {
 189     List<Pair<byte[], String>> list = new ArrayList<>();
 190     list.addAll(asList(withMissingHFileForFamily(family1)));
 191     list.addAll(asList(withMissingHFileForFamily(family2)));
 192     testRegionWithFamilies(family1, family2).bulkLoadHFiles(list, false, null);
 193   }
 194 }