hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaFixer.java

   1 /*
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.master;
  19
  20 import static org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils.isNotEmpty;
  21 import static org.junit.Assert.assertEquals;
  22 import static org.junit.Assert.assertTrue;
  23 import java.io.IOException;
  24 import java.util.Collections;
  25 import java.util.List;
  26 import java.util.function.BooleanSupplier;
  27 import org.apache.hadoop.hbase.HBaseClassTestRule;
  28 import org.apache.hadoop.hbase.HBaseTestingUtility;
  29 import org.apache.hadoop.hbase.HConstants;
  30 import org.apache.hadoop.hbase.MetaTableAccessor;
  31 import org.apache.hadoop.hbase.TableName;
  32 import org.apache.hadoop.hbase.client.RegionInfo;
  33 import org.apache.hadoop.hbase.client.RegionInfoBuilder;
  34 import org.apache.hadoop.hbase.testclassification.LargeTests;
  35 import org.apache.hadoop.hbase.testclassification.MasterTests;
  36 import org.apache.hadoop.hbase.util.Threads;
  37 import org.junit.AfterClass;
  38 import org.junit.BeforeClass;
  39 import org.junit.ClassRule;
  40 import org.junit.Rule;
  41 import org.junit.Test;
  42 import org.junit.experimental.categories.Category;
  43 import org.junit.rules.TestName;
  44
  45 @Category({MasterTests.class, LargeTests.class})
  46 public class TestMetaFixer {
  47   @ClassRule
  48   public static final HBaseClassTestRule CLASS_RULE =
  49       HBaseClassTestRule.forClass(TestMetaFixer.class);
  50   @Rule
  51   public TestName name = new TestName();
  52
  53   private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
  54
  55   @BeforeClass
  56   public static void setupBeforeClass() throws Exception {
  57     TEST_UTIL.startMiniCluster();
  58   }
  59
  60   @AfterClass
  61   public static void tearDownAfterClass() throws Exception {
  62     TEST_UTIL.shutdownMiniCluster();
  63   }
  64
  65   private void deleteRegion(MasterServices services, RegionInfo ri) throws IOException {
  66     MetaTableAccessor.deleteRegionInfo(TEST_UTIL.getConnection(), ri);
  67     // Delete it from Master context too else it sticks around.
  68     services.getAssignmentManager().getRegionStates().deleteRegion(ri);
  69   }
  70
  71   @Test
  72   public void testPlugsHoles() throws Exception {
  73     TableName tn = TableName.valueOf(this.name.getMethodName());
  74     TEST_UTIL.createMultiRegionTable(tn, HConstants.CATALOG_FAMILY);
  75     List<RegionInfo> ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
  76     MasterServices services = TEST_UTIL.getHBaseCluster().getMaster();
  77     int initialSize = services.getAssignmentManager().getRegionStates().getRegionStates().size();
  78     services.getCatalogJanitor().scan();
  79     CatalogJanitor.Report report = services.getCatalogJanitor().getLastReport();
  80     assertTrue(report.isEmpty());
  81     int originalCount = ris.size();
  82     // Remove first, last and middle region. See if hole gets plugged. Table has 26 regions.
  83     deleteRegion(services, ris.get(ris.size() -1));
  84     deleteRegion(services, ris.get(3));
  85     deleteRegion(services, ris.get(0));
  86     assertEquals(initialSize - 3,
  87         services.getAssignmentManager().getRegionStates().getRegionStates().size());
  88     services.getCatalogJanitor().scan();
  89     report = services.getCatalogJanitor().getLastReport();
  90     assertEquals(report.toString(), 3, report.getHoles().size());
  91     MetaFixer fixer = new MetaFixer(services);
  92     fixer.fixHoles(report);
  93     services.getCatalogJanitor().scan();
  94     report = services.getCatalogJanitor().getLastReport();
  95     assertTrue(report.toString(), report.isEmpty());
  96     assertEquals(initialSize,
  97         services.getAssignmentManager().getRegionStates().getRegionStates().size());
  98
  99     // wait for RITs to settle -- those are the fixed regions being assigned -- or until the
 100     // watchdog TestRule terminates the test.
 101     await(50, () -> isNotEmpty(services.getAssignmentManager().getRegionsInTransition()));
 102
 103     ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
 104     assertEquals(originalCount, ris.size());
 105   }
 106
 107   /**
 108    * Just make sure running fixMeta does right thing for the case
 109    * of a single-region Table where the region gets dropped.
 110    * There is nothing much we can do. We can't restore what
 111    * we don't know about (at least from a read of hbase:meta).
 112    */
 113   @Test
 114   public void testOneRegionTable() throws IOException {
 115     TableName tn = TableName.valueOf(this.name.getMethodName());
 116     TEST_UTIL.createTable(tn, HConstants.CATALOG_FAMILY);
 117     List<RegionInfo> ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
 118     MasterServices services = TEST_UTIL.getHBaseCluster().getMaster();
 119     services.getCatalogJanitor().scan();
 120     deleteRegion(services, ris.get(0));
 121     services.getCatalogJanitor().scan();
 122     CatalogJanitor.Report report = services.getCatalogJanitor().getLastReport();
 123     ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
 124     assertTrue(ris.isEmpty());
 125     MetaFixer fixer = new MetaFixer(services);
 126     fixer.fixHoles(report);
 127     report = services.getCatalogJanitor().getLastReport();
 128     assertTrue(report.isEmpty());
 129     ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
 130     assertEquals(0, ris.size());
 131   }
 132
 133   private static void makeOverlap(MasterServices services, RegionInfo a, RegionInfo b)
 134       throws IOException {
 135     RegionInfo overlapRegion = RegionInfoBuilder.newBuilder(a.getTable()).
 136         setStartKey(a.getStartKey()).
 137         setEndKey(b.getEndKey()).
 138         build();
 139     MetaTableAccessor.putsToMetaTable(services.getConnection(),
 140         Collections.singletonList(MetaTableAccessor.makePutFromRegionInfo(overlapRegion,
 141             System.currentTimeMillis())));
 142     // TODO: Add checks at assign time to PREVENT being able to assign over existing assign.
 143     services.getAssignmentManager().assign(overlapRegion);
 144   }
 145
 146   @Test
 147   public void testOverlap() throws Exception {
 148     TableName tn = TableName.valueOf(this.name.getMethodName());
 149     TEST_UTIL.createMultiRegionTable(tn, HConstants.CATALOG_FAMILY);
 150     List<RegionInfo> ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
 151     assertTrue(ris.size() > 5);
 152     MasterServices services = TEST_UTIL.getHBaseCluster().getMaster();
 153     services.getCatalogJanitor().scan();
 154     CatalogJanitor.Report report = services.getCatalogJanitor().getLastReport();
 155     assertTrue(report.isEmpty());
 156     // Make a simple overlap spanning second and third region.
 157     makeOverlap(services, ris.get(1), ris.get(3));
 158     makeOverlap(services, ris.get(2), ris.get(3));
 159     makeOverlap(services, ris.get(2), ris.get(4));
 160     Threads.sleep(10000);
 161     services.getCatalogJanitor().scan();
 162     report = services.getCatalogJanitor().getLastReport();
 163     assertEquals(6, report.getOverlaps().size());
 164     assertEquals(1, MetaFixer.calculateMerges(10, report.getOverlaps()).size());
 165     MetaFixer fixer = new MetaFixer(services);
 166     fixer.fixOverlaps(report);
 167     await(10, () -> {
 168       try {
 169         services.getCatalogJanitor().scan();
 170         final CatalogJanitor.Report postReport = services.getCatalogJanitor().getLastReport();
 171         return postReport.isEmpty();
 172       } catch (Exception e) {
 173         throw new RuntimeException(e);
 174       }
 175     });
 176   }
 177
 178   /**
 179    * Await the successful return of {@code condition}, sleeping {@code sleepMillis} between
 180    * invocations.
 181    */
 182   private static void await(final long sleepMillis, final BooleanSupplier condition)
 183     throws InterruptedException {
 184     try {
 185       while (!condition.getAsBoolean()) {
 186         Thread.sleep(sleepMillis);
 187       }
 188     } catch (RuntimeException e) {
 189       if (e.getCause() instanceof AssertionError) {
 190         throw (AssertionError) e.getCause();
 191       }
 192       throw e;
 193     }
 194   }
 195 }