hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.regionserver;
  19
  20 import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL;
  21 import static org.junit.Assert.assertTrue;
  22 import static org.junit.Assert.fail;
  23
  24 import java.io.IOException;
  25 import java.util.ArrayList;
  26 import java.util.List;
  27 import java.util.UUID;
  28 import java.util.concurrent.TimeUnit;
  29 import org.apache.commons.lang3.mutable.MutableBoolean;
  30 import org.apache.hadoop.fs.FileStatus;
  31 import org.apache.hadoop.fs.FileSystem;
  32 import org.apache.hadoop.fs.FileUtil;
  33 import org.apache.hadoop.fs.Path;
  34 import org.apache.hadoop.hbase.HBaseClassTestRule;
  35 import org.apache.hadoop.hbase.HBaseTestingUtil;
  36 import org.apache.hadoop.hbase.TableName;
  37 import org.apache.hadoop.hbase.TableNameTestRule;
  38 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
  39 import org.apache.hadoop.hbase.client.Put;
  40 import org.apache.hadoop.hbase.client.RegionInfo;
  41 import org.apache.hadoop.hbase.client.RegionInfoBuilder;
  42 import org.apache.hadoop.hbase.client.Table;
  43 import org.apache.hadoop.hbase.client.TableDescriptor;
  44 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
  45 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
  46 import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerForTest;
  47 import org.apache.hadoop.hbase.testclassification.LargeTests;
  48 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
  49 import org.apache.hadoop.hbase.util.Bytes;
  50 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
  51 import org.apache.hadoop.hbase.util.Pair;
  52 import org.junit.AfterClass;
  53 import org.junit.Before;
  54 import org.junit.BeforeClass;
  55 import org.junit.ClassRule;
  56 import org.junit.Rule;
  57 import org.junit.Test;
  58 import org.junit.experimental.categories.Category;
  59
  60
  61 @Category({RegionServerTests.class, LargeTests.class})
  62 public class TestMergesSplitsAddToTracker {
  63
  64   @ClassRule
  65   public static final HBaseClassTestRule CLASS_RULE =
  66     HBaseClassTestRule.forClass(TestMergesSplitsAddToTracker.class);
  67
  68   private static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
  69
  70   private static final String FAMILY_NAME_STR = "info";
  71
  72   private static final byte[] FAMILY_NAME = Bytes.toBytes(FAMILY_NAME_STR);
  73
  74   @Rule
  75   public TableNameTestRule name = new TableNameTestRule();
  76
  77   @BeforeClass
  78   public static void setupClass() throws Exception {
  79     TEST_UTIL.startMiniCluster();
  80   }
  81
  82   @AfterClass
  83   public static void afterClass() throws Exception {
  84     TEST_UTIL.shutdownMiniCluster();
  85   }
  86
  87   @Before
  88   public void setup(){
  89     StoreFileTrackerForTest.clear();
  90   }
  91
  92   private TableName createTable(byte[] splitKey) throws IOException {
  93     TableDescriptor td = TableDescriptorBuilder.newBuilder(name.getTableName())
  94       .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_NAME))
  95       .setValue(TRACKER_IMPL, StoreFileTrackerForTest.class.getName()).build();
  96     if (splitKey != null) {
  97       TEST_UTIL.getAdmin().createTable(td, new byte[][] { splitKey });
  98     } else {
  99       TEST_UTIL.getAdmin().createTable(td);
 100     }
 101     return td.getTableName();
 102   }
 103
 104   @Test
 105   public void testCommitDaughterRegion() throws Exception {
 106     TableName table = createTable(null);
 107     //first put some data in order to have a store file created
 108     putThreeRowsAndFlush(table);
 109     HRegion region = TEST_UTIL.getHBaseCluster().getRegions(table).get(0);
 110     HRegionFileSystem regionFS = region.getStores().get(0).getRegionFileSystem();
 111     RegionInfo daughterA =
 112       RegionInfoBuilder.newBuilder(table).setStartKey(region.getRegionInfo().getStartKey()).
 113         setEndKey(Bytes.toBytes("002")).setSplit(false).
 114         setRegionId(region.getRegionInfo().getRegionId() +
 115           EnvironmentEdgeManager.currentTime()).
 116         build();
 117     RegionInfo daughterB = RegionInfoBuilder.newBuilder(table).setStartKey(Bytes.toBytes("002"))
 118       .setEndKey(region.getRegionInfo().getEndKey()).setSplit(false)
 119       .setRegionId(region.getRegionInfo().getRegionId()).build();
 120     HStoreFile file = (HStoreFile) region.getStore(FAMILY_NAME).getStorefiles().toArray()[0];
 121     List<Path> splitFilesA = new ArrayList<>();
 122     splitFilesA.add(regionFS
 123       .splitStoreFile(daughterA, Bytes.toString(FAMILY_NAME), file,
 124         Bytes.toBytes("002"), false, region.getSplitPolicy()));
 125     List<Path> splitFilesB = new ArrayList<>();
 126     splitFilesB.add(regionFS
 127       .splitStoreFile(daughterB, Bytes.toString(FAMILY_NAME), file,
 128         Bytes.toBytes("002"), true, region.getSplitPolicy()));
 129     MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster().
 130       getMasterProcedureExecutor().getEnvironment();
 131     Path resultA = regionFS.commitDaughterRegion(daughterA, splitFilesA, env);
 132     Path resultB = regionFS.commitDaughterRegion(daughterB, splitFilesB, env);
 133     FileSystem fs = regionFS.getFileSystem();
 134     verifyFilesAreTracked(resultA, fs);
 135     verifyFilesAreTracked(resultB, fs);
 136   }
 137
 138   @Test
 139   public void testCommitMergedRegion() throws Exception {
 140     TableName table = createTable(null);
 141     //splitting the table first
 142     TEST_UTIL.getAdmin().split(table, Bytes.toBytes("002"));
 143     //Add data and flush to create files in the two different regions
 144     putThreeRowsAndFlush(table);
 145     List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table);
 146     HRegion first = regions.get(0);
 147     HRegion second = regions.get(1);
 148     HRegionFileSystem regionFS = first.getRegionFileSystem();
 149
 150     RegionInfo mergeResult =
 151       RegionInfoBuilder.newBuilder(table).setStartKey(first.getRegionInfo().getStartKey())
 152         .setEndKey(second.getRegionInfo().getEndKey()).setSplit(false)
 153         .setRegionId(first.getRegionInfo().getRegionId() +
 154           EnvironmentEdgeManager.currentTime()).build();
 155
 156     HRegionFileSystem mergeFS = HRegionFileSystem.createRegionOnFileSystem(
 157       TEST_UTIL.getHBaseCluster().getMaster().getConfiguration(),
 158       regionFS.getFileSystem(), regionFS.getTableDir(), mergeResult);
 159
 160     List<Path> mergedFiles = new ArrayList<>();
 161     //merge file from first region
 162     mergedFiles.add(mergeFileFromRegion(first, mergeFS));
 163     //merge file from second region
 164     mergedFiles.add(mergeFileFromRegion(second, mergeFS));
 165     MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster().
 166       getMasterProcedureExecutor().getEnvironment();
 167     mergeFS.commitMergedRegion(mergedFiles, env);
 168     //validate
 169     FileSystem fs = first.getRegionFileSystem().getFileSystem();
 170     Path finalMergeDir = new Path(first.getRegionFileSystem().getTableDir(),
 171       mergeResult.getEncodedName());
 172     verifyFilesAreTracked(finalMergeDir, fs);
 173   }
 174
 175   @Test
 176   public void testSplitLoadsFromTracker() throws Exception {
 177     TableName table = createTable(null);
 178     //Add data and flush to create files in the two different regions
 179     putThreeRowsAndFlush(table);
 180     HRegion region = TEST_UTIL.getHBaseCluster().getRegions(table).get(0);
 181     Pair<StoreFileInfo, String> copyResult = copyFileInTheStoreDir(region);
 182     StoreFileInfo fileInfo = copyResult.getFirst();
 183     String copyName = copyResult.getSecond();
 184     //Now splits the region
 185     TEST_UTIL.getAdmin().split(table, Bytes.toBytes("002"));
 186     List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table);
 187     HRegion first = regions.get(0);
 188     validateDaughterRegionsFiles(first, fileInfo.getActiveFileName(), copyName);
 189     HRegion second = regions.get(1);
 190     validateDaughterRegionsFiles(second, fileInfo.getActiveFileName(), copyName);
 191   }
 192
 193   @Test
 194   public void testMergeLoadsFromTracker() throws Exception {
 195     TableName table = createTable(Bytes.toBytes("002"));
 196     //Add data and flush to create files in the two different regions
 197     putThreeRowsAndFlush(table);
 198     List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table);
 199     HRegion first = regions.get(0);
 200     Pair<StoreFileInfo, String> copyResult = copyFileInTheStoreDir(first);
 201     StoreFileInfo fileInfo = copyResult.getFirst();
 202     String copyName = copyResult.getSecond();
 203     //Now merges the first two regions
 204     TEST_UTIL.getAdmin().mergeRegionsAsync(new byte[][]{
 205       first.getRegionInfo().getEncodedNameAsBytes(),
 206       regions.get(1).getRegionInfo().getEncodedNameAsBytes()
 207     }, true).get(10, TimeUnit.SECONDS);
 208     regions = TEST_UTIL.getHBaseCluster().getRegions(table);
 209     HRegion merged = regions.get(0);
 210     validateDaughterRegionsFiles(merged, fileInfo.getActiveFileName(), copyName);
 211   }
 212
 213   private Pair<StoreFileInfo,String> copyFileInTheStoreDir(HRegion region) throws IOException {
 214     Path storeDir = region.getRegionFileSystem().getStoreDir("info");
 215     //gets the single file
 216     StoreFileInfo fileInfo = region.getRegionFileSystem().getStoreFiles("info").get(0);
 217     //make a copy of the valid file staight into the store dir, so that it's not tracked.
 218     String copyName = UUID.randomUUID().toString().replaceAll("-", "");
 219     Path copy = new Path(storeDir, copyName);
 220     FileUtil.copy(region.getFilesystem(), fileInfo.getFileStatus(), region.getFilesystem(),
 221       copy , false, false, TEST_UTIL.getConfiguration());
 222     return new Pair<>(fileInfo, copyName);
 223   }
 224
 225   private void validateDaughterRegionsFiles(HRegion region, String orignalFileName,
 226       String untrackedFile) throws IOException {
 227     //verify there's no link for the untracked, copied file in first region
 228     List<StoreFileInfo> infos = region.getRegionFileSystem().getStoreFiles("info");
 229     final MutableBoolean foundLink = new MutableBoolean(false);
 230     infos.stream().forEach(i -> {
 231       i.getActiveFileName().contains(orignalFileName);
 232       if(i.getActiveFileName().contains(untrackedFile)){
 233         fail();
 234       }
 235       if(i.getActiveFileName().contains(orignalFileName)){
 236         foundLink.setTrue();
 237       }
 238     });
 239     assertTrue(foundLink.booleanValue());
 240   }
 241
 242   private void verifyFilesAreTracked(Path regionDir, FileSystem fs) throws Exception {
 243     for (FileStatus f : fs.listStatus(new Path(regionDir, FAMILY_NAME_STR))) {
 244       assertTrue(
 245         StoreFileTrackerForTest.tracked(regionDir.getName(), FAMILY_NAME_STR, f.getPath()));
 246     }
 247   }
 248
 249   private Path mergeFileFromRegion(HRegion regionToMerge, HRegionFileSystem mergeFS)
 250       throws IOException {
 251     HStoreFile file = (HStoreFile) regionToMerge.getStore(FAMILY_NAME).getStorefiles().toArray()[0];
 252     return mergeFS.mergeStoreFile(regionToMerge.getRegionInfo(), Bytes.toString(FAMILY_NAME), file);
 253   }
 254
 255   private void putThreeRowsAndFlush(TableName table) throws IOException {
 256     Table tbl = TEST_UTIL.getConnection().getTable(table);
 257     Put put = new Put(Bytes.toBytes("001"));
 258     byte[] qualifier = Bytes.toBytes("1");
 259     put.addColumn(FAMILY_NAME, qualifier, Bytes.toBytes(1));
 260     tbl.put(put);
 261     put = new Put(Bytes.toBytes("002"));
 262     put.addColumn(FAMILY_NAME, qualifier, Bytes.toBytes(2));
 263     tbl.put(put);
 264     put = new Put(Bytes.toBytes("003"));
 265     put.addColumn(FAMILY_NAME, qualifier, Bytes.toBytes(2));
 266     tbl.put(put);
 267     TEST_UTIL.flush(table);
 268   }
 269 }