hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java

   1 /*
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.regionserver;
  19
  20 import static java.util.Arrays.asList;
  21 import static org.mockito.ArgumentMatchers.any;
  22 import static org.mockito.ArgumentMatchers.anyLong;
  23 import static org.mockito.Mockito.verify;
  24 import static org.mockito.Mockito.when;
  25 import static org.mockito.hamcrest.MockitoHamcrest.argThat;
  26
  27 import java.io.FileNotFoundException;
  28 import java.io.IOException;
  29 import java.util.ArrayList;
  30 import java.util.List;
  31
  32 import org.apache.hadoop.fs.Path;
  33 import org.apache.hadoop.hbase.DoNotRetryIOException;
  34 import org.apache.hadoop.hbase.HBaseClassTestRule;
  35 import org.apache.hadoop.hbase.TableName;
  36 import org.apache.hadoop.hbase.testclassification.SmallTests;
  37 import org.apache.hadoop.hbase.util.Pair;
  38 import org.apache.hadoop.hbase.wal.WALEdit;
  39 import org.apache.hadoop.hbase.wal.WALKeyImpl;
  40 import org.junit.ClassRule;
  41 import org.junit.Test;
  42 import org.junit.experimental.categories.Category;
  43 import org.mockito.invocation.InvocationOnMock;
  44 import org.mockito.stubbing.Answer;
  45
  46
  47 /**
  48  * This class attempts to unit test bulk HLog loading.
  49  */
  50 @Category(SmallTests.class)
  51 public class TestBulkLoad extends TestBulkloadBase {
  52
  53   @ClassRule
  54   public static final HBaseClassTestRule CLASS_RULE =
  55       HBaseClassTestRule.forClass(TestBulkLoad.class);
  56
  57   @Test
  58   public void verifyBulkLoadEvent() throws IOException {
  59     TableName tableName = TableName.valueOf("test", "test");
  60     List<Pair<byte[], String>> familyPaths = withFamilyPathsFor(family1);
  61     byte[] familyName = familyPaths.get(0).getFirst();
  62     String storeFileName = familyPaths.get(0).getSecond();
  63     storeFileName = (new Path(storeFileName)).getName();
  64     List<String> storeFileNames = new ArrayList<>();
  65     storeFileNames.add(storeFileName);
  66     when(log.appendMarker(any(), any(),
  67       argThat(bulkLogWalEdit(WALEdit.BULK_LOAD, tableName.toBytes(), familyName, storeFileNames))))
  68         .thenAnswer(new Answer() {
  69           @Override
  70           public Object answer(InvocationOnMock invocation) {
  71             WALKeyImpl walKey = invocation.getArgument(1);
  72             MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
  73             if (mvcc != null) {
  74               MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
  75               walKey.setWriteEntry(we);
  76             }
  77             return 01L;
  78           }
  79         });
  80     testRegionWithFamiliesAndSpecifiedTableName(tableName, family1)
  81         .bulkLoadHFiles(familyPaths, false, null);
  82     verify(log).sync(anyLong());
  83   }
  84
  85   @Test
  86   public void bulkHLogShouldThrowNoErrorAndWriteMarkerWithBlankInput() throws IOException {
  87     testRegionWithFamilies(family1).bulkLoadHFiles(new ArrayList<>(),false, null);
  88   }
  89
  90   @Test
  91   public void shouldBulkLoadSingleFamilyHLog() throws IOException {
  92     when(log.appendMarker(any(), any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD))))
  93         .thenAnswer(new Answer() {
  94           @Override
  95           public Object answer(InvocationOnMock invocation) {
  96             WALKeyImpl walKey = invocation.getArgument(1);
  97             MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
  98             if (mvcc != null) {
  99               MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
 100               walKey.setWriteEntry(we);
 101             }
 102             return 01L;
 103           }
 104         });
 105     testRegionWithFamilies(family1).bulkLoadHFiles(withFamilyPathsFor(family1), false, null);
 106     verify(log).sync(anyLong());
 107   }
 108
 109   @Test
 110   public void shouldBulkLoadManyFamilyHLog() throws IOException {
 111     when(log.appendMarker(any(),
 112             any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)))).thenAnswer(new Answer() {
 113               @Override
 114               public Object answer(InvocationOnMock invocation) {
 115                 WALKeyImpl walKey = invocation.getArgument(1);
 116                 MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
 117                 if (mvcc != null) {
 118                   MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
 119                   walKey.setWriteEntry(we);
 120                 }
 121                 return 01L;
 122               }
 123             });
 124     testRegionWithFamilies(family1, family2).bulkLoadHFiles(withFamilyPathsFor(family1, family2),
 125             false, null);
 126     verify(log).sync(anyLong());
 127   }
 128
 129   @Test
 130   public void shouldBulkLoadManyFamilyHLogEvenWhenTableNameNamespaceSpecified() throws IOException {
 131     when(log.appendMarker(any(), any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD))))
 132         .thenAnswer(new Answer() {
 133           @Override
 134           public Object answer(InvocationOnMock invocation) {
 135             WALKeyImpl walKey = invocation.getArgument(1);
 136             MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
 137             if (mvcc != null) {
 138               MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
 139               walKey.setWriteEntry(we);
 140             }
 141             return 01L;
 142           }
 143         });
 144     TableName tableName = TableName.valueOf("test", "test");
 145     testRegionWithFamiliesAndSpecifiedTableName(tableName, family1, family2)
 146         .bulkLoadHFiles(withFamilyPathsFor(family1, family2), false, null);
 147     verify(log).sync(anyLong());
 148   }
 149
 150   @Test(expected = DoNotRetryIOException.class)
 151   public void shouldCrashIfBulkLoadFamiliesNotInTable() throws IOException {
 152     testRegionWithFamilies(family1).bulkLoadHFiles(withFamilyPathsFor(family1, family2), false,
 153       null);
 154   }
 155
 156   // after HBASE-24021 will throw DoNotRetryIOException, not MultipleIOException
 157   @Test(expected = DoNotRetryIOException.class)
 158   public void shouldCrashIfBulkLoadMultiFamiliesNotInTable() throws IOException {
 159     testRegionWithFamilies(family1).bulkLoadHFiles(withFamilyPathsFor(family1, family2, family3),
 160       false, null);
 161   }
 162
 163   @Test(expected = DoNotRetryIOException.class)
 164   public void bulkHLogShouldThrowErrorWhenFamilySpecifiedAndHFileExistsButNotInTableDescriptor()
 165       throws IOException {
 166     testRegionWithFamilies().bulkLoadHFiles(withFamilyPathsFor(family1), false, null);
 167   }
 168
 169   @Test(expected = DoNotRetryIOException.class)
 170   public void shouldThrowErrorIfBadFamilySpecifiedAsFamilyPath() throws IOException {
 171     testRegionWithFamilies()
 172         .bulkLoadHFiles(asList(withInvalidColumnFamilyButProperHFileLocation(family1)),
 173             false, null);
 174   }
 175
 176   @Test(expected = FileNotFoundException.class)
 177   public void shouldThrowErrorIfHFileDoesNotExist() throws IOException {
 178     List<Pair<byte[], String>> list = asList(withMissingHFileForFamily(family1));
 179     testRegionWithFamilies(family1).bulkLoadHFiles(list, false, null);
 180   }
 181
 182   // after HBASE-24021 will throw FileNotFoundException, not MultipleIOException
 183   @Test(expected = FileNotFoundException.class)
 184   public void shouldThrowErrorIfMultiHFileDoesNotExist() throws IOException {
 185     List<Pair<byte[], String>> list = new ArrayList<>();
 186     list.addAll(asList(withMissingHFileForFamily(family1)));
 187     list.addAll(asList(withMissingHFileForFamily(family2)));
 188     testRegionWithFamilies(family1, family2).bulkLoadHFiles(list, false, null);
 189   }
 190 }