hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMergesSplitsAddToTracker.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.regionserver;
  19
  20 import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.
  21   TRACKER_IMPL;
  22 import static org.junit.Assert.assertTrue;
  23 import static org.junit.Assert.fail;
  24
  25 import java.io.IOException;
  26 import java.util.ArrayList;
  27 import java.util.HashMap;
  28 import java.util.List;
  29 import java.util.UUID;
  30 import java.util.concurrent.TimeUnit;
  31
  32 import org.apache.commons.lang3.mutable.MutableBoolean;
  33 import org.apache.hadoop.fs.FileStatus;
  34 import org.apache.hadoop.fs.FileSystem;
  35 import org.apache.hadoop.fs.FileUtil;
  36 import org.apache.hadoop.fs.Path;
  37 import org.apache.hadoop.hbase.HBaseClassTestRule;
  38 import org.apache.hadoop.hbase.HBaseTestingUtil;
  39 import org.apache.hadoop.hbase.TableName;
  40 import org.apache.hadoop.hbase.client.Put;
  41 import org.apache.hadoop.hbase.client.RegionInfo;
  42 import org.apache.hadoop.hbase.client.RegionInfoBuilder;
  43 import org.apache.hadoop.hbase.client.Table;
  44 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
  45 import org.apache.hadoop.hbase.regionserver.storefiletracker.TestStoreFileTracker;
  46 import org.apache.hadoop.hbase.testclassification.LargeTests;
  47 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
  48 import org.apache.hadoop.hbase.util.Bytes;
  49 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
  50 import org.apache.hadoop.hbase.util.Pair;
  51 import org.junit.AfterClass;
  52 import org.junit.Before;
  53 import org.junit.BeforeClass;
  54 import org.junit.ClassRule;
  55 import org.junit.Rule;
  56 import org.junit.Test;
  57 import org.junit.experimental.categories.Category;
  58 import org.junit.rules.TestName;
  59
  60
  61 @Category({RegionServerTests.class, LargeTests.class})
  62 public class TestMergesSplitsAddToTracker {
  63
  64   @ClassRule
  65   public static final HBaseClassTestRule CLASS_RULE =
  66     HBaseClassTestRule.forClass(TestMergesSplitsAddToTracker.class);
  67
  68   private static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
  69
  70   public static final byte[] FAMILY_NAME = Bytes.toBytes("info");
  71
  72   @Rule
  73   public TestName name = new TestName();
  74
  75   @BeforeClass
  76   public static void setupClass() throws Exception {
  77     TEST_UTIL.getConfiguration().set(TRACKER_IMPL, TestStoreFileTracker.class.getName());
  78     TEST_UTIL.startMiniCluster();
  79   }
  80
  81   @AfterClass
  82   public static void afterClass() throws Exception {
  83     TEST_UTIL.shutdownMiniCluster();
  84   }
  85
  86   @Before
  87   public void setup(){
  88     TestStoreFileTracker.trackedFiles = new HashMap<>();
  89   }
  90
  91   @Test
  92   public void testCommitDaughterRegion() throws Exception {
  93     TableName table = TableName.valueOf(name.getMethodName());
  94     TEST_UTIL.createTable(table, FAMILY_NAME);
  95     //first put some data in order to have a store file created
  96     putThreeRowsAndFlush(table);
  97     HRegion region = TEST_UTIL.getHBaseCluster().getRegions(table).get(0);
  98     HRegionFileSystem regionFS = region.getStores().get(0).getRegionFileSystem();
  99     RegionInfo daughterA =
 100       RegionInfoBuilder.newBuilder(table).setStartKey(region.getRegionInfo().getStartKey()).
 101         setEndKey(Bytes.toBytes("002")).setSplit(false).
 102         setRegionId(region.getRegionInfo().getRegionId() +
 103           EnvironmentEdgeManager.currentTime()).
 104         build();
 105     RegionInfo daughterB = RegionInfoBuilder.newBuilder(table).setStartKey(Bytes.toBytes("002"))
 106       .setEndKey(region.getRegionInfo().getEndKey()).setSplit(false)
 107       .setRegionId(region.getRegionInfo().getRegionId()).build();
 108     HStoreFile file = (HStoreFile) region.getStore(FAMILY_NAME).getStorefiles().toArray()[0];
 109     List<Path> splitFilesA = new ArrayList<>();
 110     splitFilesA.add(regionFS
 111       .splitStoreFile(daughterA, Bytes.toString(FAMILY_NAME), file,
 112         Bytes.toBytes("002"), false, region.getSplitPolicy()));
 113     List<Path> splitFilesB = new ArrayList<>();
 114     splitFilesB.add(regionFS
 115       .splitStoreFile(daughterB, Bytes.toString(FAMILY_NAME), file,
 116         Bytes.toBytes("002"), true, region.getSplitPolicy()));
 117     MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster().
 118       getMasterProcedureExecutor().getEnvironment();
 119     Path resultA = regionFS.commitDaughterRegion(daughterA, splitFilesA, env);
 120     Path resultB = regionFS.commitDaughterRegion(daughterB, splitFilesB, env);
 121     FileSystem fs = regionFS.getFileSystem();
 122     verifyFilesAreTracked(resultA, fs);
 123     verifyFilesAreTracked(resultB, fs);
 124   }
 125
 126   @Test
 127   public void testCommitMergedRegion() throws Exception {
 128     TableName table = TableName.valueOf(name.getMethodName());
 129     TEST_UTIL.createTable(table, FAMILY_NAME);
 130     //splitting the table first
 131     TEST_UTIL.getAdmin().split(table, Bytes.toBytes("002"));
 132     //Add data and flush to create files in the two different regions
 133     putThreeRowsAndFlush(table);
 134     List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table);
 135     HRegion first = regions.get(0);
 136     HRegion second = regions.get(1);
 137     HRegionFileSystem regionFS = first.getRegionFileSystem();
 138
 139     RegionInfo mergeResult =
 140       RegionInfoBuilder.newBuilder(table).setStartKey(first.getRegionInfo().getStartKey())
 141         .setEndKey(second.getRegionInfo().getEndKey()).setSplit(false)
 142         .setRegionId(first.getRegionInfo().getRegionId() +
 143           EnvironmentEdgeManager.currentTime()).build();
 144
 145     HRegionFileSystem mergeFS = HRegionFileSystem.createRegionOnFileSystem(
 146       TEST_UTIL.getHBaseCluster().getMaster().getConfiguration(),
 147       regionFS.getFileSystem(), regionFS.getTableDir(), mergeResult);
 148
 149     List<Path> mergedFiles = new ArrayList<>();
 150     //merge file from first region
 151     mergedFiles.add(mergeFileFromRegion(first, mergeFS));
 152     //merge file from second region
 153     mergedFiles.add(mergeFileFromRegion(second, mergeFS));
 154     MasterProcedureEnv env = TEST_UTIL.getMiniHBaseCluster().getMaster().
 155       getMasterProcedureExecutor().getEnvironment();
 156     mergeFS.commitMergedRegion(mergedFiles, env);
 157     //validate
 158     FileSystem fs = first.getRegionFileSystem().getFileSystem();
 159     Path finalMergeDir = new Path(first.getRegionFileSystem().getTableDir(),
 160       mergeResult.getEncodedName());
 161     verifyFilesAreTracked(finalMergeDir, fs);
 162   }
 163
 164   @Test
 165   public void testSplitLoadsFromTracker() throws Exception {
 166     TableName table = TableName.valueOf(name.getMethodName());
 167     TEST_UTIL.createTable(table, FAMILY_NAME);
 168     //Add data and flush to create files in the two different regions
 169     putThreeRowsAndFlush(table);
 170     HRegion region = TEST_UTIL.getHBaseCluster().getRegions(table).get(0);
 171     Pair<StoreFileInfo, String> copyResult = copyFileInTheStoreDir(region);
 172     StoreFileInfo fileInfo = copyResult.getFirst();
 173     String copyName = copyResult.getSecond();
 174     //Now splits the region
 175     TEST_UTIL.getAdmin().split(table, Bytes.toBytes("002"));
 176     List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table);
 177     HRegion first = regions.get(0);
 178     validateDaughterRegionsFiles(first, fileInfo.getActiveFileName(), copyName);
 179     HRegion second = regions.get(1);
 180     validateDaughterRegionsFiles(second, fileInfo.getActiveFileName(), copyName);
 181   }
 182
 183   @Test
 184   public void testMergeLoadsFromTracker() throws Exception {
 185     TableName table = TableName.valueOf(name.getMethodName());
 186     TEST_UTIL.createTable(table, new byte[][]{FAMILY_NAME},
 187       new byte[][]{Bytes.toBytes("002")});
 188     //Add data and flush to create files in the two different regions
 189     putThreeRowsAndFlush(table);
 190     List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table);
 191     HRegion first = regions.get(0);
 192     Pair<StoreFileInfo, String> copyResult = copyFileInTheStoreDir(first);
 193     StoreFileInfo fileInfo = copyResult.getFirst();
 194     String copyName = copyResult.getSecond();
 195     //Now merges the first two regions
 196     TEST_UTIL.getAdmin().mergeRegionsAsync(new byte[][]{
 197       first.getRegionInfo().getEncodedNameAsBytes(),
 198       regions.get(1).getRegionInfo().getEncodedNameAsBytes()
 199     }, true).get(10, TimeUnit.SECONDS);
 200     regions = TEST_UTIL.getHBaseCluster().getRegions(table);
 201     HRegion merged = regions.get(0);
 202     validateDaughterRegionsFiles(merged, fileInfo.getActiveFileName(), copyName);
 203   }
 204
 205   private Pair<StoreFileInfo,String> copyFileInTheStoreDir(HRegion region) throws IOException {
 206     Path storeDir = region.getRegionFileSystem().getStoreDir("info");
 207     //gets the single file
 208     StoreFileInfo fileInfo = region.getRegionFileSystem().getStoreFiles("info").get(0);
 209     //make a copy of the valid file staight into the store dir, so that it's not tracked.
 210     String copyName = UUID.randomUUID().toString().replaceAll("-", "");
 211     Path copy = new Path(storeDir, copyName);
 212     FileUtil.copy(region.getFilesystem(), fileInfo.getFileStatus(), region.getFilesystem(),
 213       copy , false, false, TEST_UTIL.getConfiguration());
 214     return new Pair<>(fileInfo, copyName);
 215   }
 216
 217   private void validateDaughterRegionsFiles(HRegion region, String orignalFileName,
 218       String untrackedFile) throws IOException {
 219     //verify there's no link for the untracked, copied file in first region
 220     List<StoreFileInfo> infos = region.getRegionFileSystem().getStoreFiles("info");
 221     final MutableBoolean foundLink = new MutableBoolean(false);
 222     infos.stream().forEach(i -> {
 223       i.getActiveFileName().contains(orignalFileName);
 224       if(i.getActiveFileName().contains(untrackedFile)){
 225         fail();
 226       }
 227       if(i.getActiveFileName().contains(orignalFileName)){
 228         foundLink.setTrue();
 229       }
 230     });
 231     assertTrue(foundLink.booleanValue());
 232   }
 233
 234   private void verifyFilesAreTracked(Path regionDir, FileSystem fs) throws Exception {
 235     String storeId = regionDir.getName() + "-info";
 236     for(FileStatus f : fs.listStatus(new Path(regionDir, Bytes.toString(FAMILY_NAME)))){
 237       assertTrue(TestStoreFileTracker.trackedFiles.get(storeId).stream().filter(s ->
 238         s.getPath().equals(f.getPath())).findFirst().isPresent());
 239     }
 240   }
 241
 242   private Path mergeFileFromRegion(HRegion regionToMerge, HRegionFileSystem mergeFS)
 243       throws IOException {
 244     HStoreFile file = (HStoreFile) regionToMerge.getStore(FAMILY_NAME).getStorefiles().toArray()[0];
 245     return mergeFS.mergeStoreFile(regionToMerge.getRegionInfo(), Bytes.toString(FAMILY_NAME), file);
 246   }
 247
 248   private void putThreeRowsAndFlush(TableName table) throws IOException {
 249     Table tbl = TEST_UTIL.getConnection().getTable(table);
 250     Put put = new Put(Bytes.toBytes("001"));
 251     byte[] qualifier = Bytes.toBytes("1");
 252     put.addColumn(FAMILY_NAME, qualifier, Bytes.toBytes(1));
 253     tbl.put(put);
 254     put = new Put(Bytes.toBytes("002"));
 255     put.addColumn(FAMILY_NAME, qualifier, Bytes.toBytes(2));
 256     tbl.put(put);
 257     put = new Put(Bytes.toBytes("003"));
 258     put.addColumn(FAMILY_NAME, qualifier, Bytes.toBytes(2));
 259     tbl.put(put);
 260     TEST_UTIL.flush(table);
 261   }
 262 }