hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestBoundedRegionGroupingStrategy.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.wal;
  19
  20 import static org.apache.hadoop.hbase.wal.BoundedGroupingStrategy.DEFAULT_NUM_REGION_GROUPS;
  21 import static org.apache.hadoop.hbase.wal.BoundedGroupingStrategy.NUM_REGION_GROUPS;
  22 import static org.apache.hadoop.hbase.wal.RegionGroupingProvider.DELEGATE_PROVIDER;
  23 import static org.apache.hadoop.hbase.wal.RegionGroupingProvider.REGION_GROUPING_STRATEGY;
  24 import static org.apache.hadoop.hbase.wal.WALFactory.WAL_PROVIDER;
  25 import static org.junit.Assert.assertEquals;
  26
  27 import java.io.IOException;
  28 import java.util.Arrays;
  29 import java.util.HashSet;
  30 import java.util.Set;
  31 import java.util.concurrent.ThreadLocalRandom;
  32 import org.apache.hadoop.conf.Configuration;
  33 import org.apache.hadoop.fs.FileStatus;
  34 import org.apache.hadoop.fs.Path;
  35 import org.apache.hadoop.hbase.HBaseClassTestRule;
  36 import org.apache.hadoop.hbase.HBaseTestingUtility;
  37 import org.apache.hadoop.hbase.TableName;
  38 import org.apache.hadoop.hbase.client.RegionInfoBuilder;
  39 import org.apache.hadoop.hbase.testclassification.LargeTests;
  40 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
  41 import org.apache.hadoop.hbase.util.FSUtils;
  42 import org.apache.hadoop.hdfs.DistributedFileSystem;
  43 import org.junit.After;
  44 import org.junit.AfterClass;
  45 import org.junit.Before;
  46 import org.junit.BeforeClass;
  47 import org.junit.ClassRule;
  48 import org.junit.Test;
  49 import org.junit.experimental.categories.Category;
  50 import org.junit.runner.RunWith;
  51 import org.junit.runners.Parameterized;
  52 import org.junit.runners.Parameterized.Parameter;
  53 import org.junit.runners.Parameterized.Parameters;
  54 import org.slf4j.Logger;
  55 import org.slf4j.LoggerFactory;
  56
  57 @RunWith(Parameterized.class)
  58 @Category({ RegionServerTests.class, LargeTests.class })
  59 public class TestBoundedRegionGroupingStrategy {
  60
  61   @ClassRule
  62   public static final HBaseClassTestRule CLASS_RULE =
  63       HBaseClassTestRule.forClass(TestBoundedRegionGroupingStrategy.class);
  64
  65   private static final Logger LOG =
  66       LoggerFactory.getLogger(TestBoundedRegionGroupingStrategy.class);
  67
  68   private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
  69
  70   private static Configuration CONF;
  71   private static DistributedFileSystem FS;
  72
  73   @Parameter
  74   public String walProvider;
  75
  76   @Parameters(name = "{index}: delegate-provider={0}")
  77   public static Iterable<Object[]> data() {
  78     return Arrays.asList(new Object[] { "defaultProvider" }, new Object[] { "asyncfs" });
  79   }
  80
  81   @Before
  82   public void setUp() throws Exception {
  83     CONF.set(DELEGATE_PROVIDER, walProvider);
  84   }
  85
  86   @After
  87   public void tearDown() throws Exception {
  88     FileStatus[] entries = FS.listStatus(new Path("/"));
  89     for (FileStatus dir : entries) {
  90       FS.delete(dir.getPath(), true);
  91     }
  92   }
  93
  94   @BeforeClass
  95   public static void setUpBeforeClass() throws Exception {
  96     CONF = TEST_UTIL.getConfiguration();
  97     // Make block sizes small.
  98     CONF.setInt("dfs.blocksize", 1024 * 1024);
  99     // quicker heartbeat interval for faster DN death notification
 100     CONF.setInt("dfs.namenode.heartbeat.recheck-interval", 5000);
 101     CONF.setInt("dfs.heartbeat.interval", 1);
 102     CONF.setInt("dfs.client.socket-timeout", 5000);
 103
 104     // faster failover with cluster.shutdown();fs.close() idiom
 105     CONF.setInt("hbase.ipc.client.connect.max.retries", 1);
 106     CONF.setInt("dfs.client.block.recovery.retries", 1);
 107     CONF.setInt("hbase.ipc.client.connection.maxidletime", 500);
 108
 109     CONF.setClass(WAL_PROVIDER, RegionGroupingProvider.class, WALProvider.class);
 110     CONF.set(REGION_GROUPING_STRATEGY, RegionGroupingProvider.Strategies.bounded.name());
 111
 112     TEST_UTIL.startMiniDFSCluster(3);
 113
 114     FS = TEST_UTIL.getDFSCluster().getFileSystem();
 115   }
 116
 117   @AfterClass
 118   public static void tearDownAfterClass() throws Exception {
 119     TEST_UTIL.shutdownMiniCluster();
 120   }
 121
 122   /**
 123    * Write to a log file with three concurrent threads and verifying all data is written.
 124    */
 125   @Test
 126   public void testConcurrentWrites() throws Exception {
 127     // Run the WPE tool with three threads writing 3000 edits each concurrently.
 128     // When done, verify that all edits were written.
 129     int errCode = WALPerformanceEvaluation.innerMain(new Configuration(CONF),
 130       new String[] { "-threads", "3", "-verify", "-noclosefs", "-iterations", "3000" });
 131     assertEquals(0, errCode);
 132   }
 133
 134   /**
 135    * Make sure we can successfully run with more regions then our bound.
 136    */
 137   @Test
 138   public void testMoreRegionsThanBound() throws Exception {
 139     final String parallelism = Integer.toString(DEFAULT_NUM_REGION_GROUPS * 2);
 140     int errCode = WALPerformanceEvaluation.innerMain(new Configuration(CONF),
 141       new String[] { "-threads", parallelism, "-verify", "-noclosefs", "-iterations", "3000",
 142           "-regions", parallelism });
 143     assertEquals(0, errCode);
 144   }
 145
 146   @Test
 147   public void testBoundsGreaterThanDefault() throws Exception {
 148     final int temp = CONF.getInt(NUM_REGION_GROUPS, DEFAULT_NUM_REGION_GROUPS);
 149     try {
 150       CONF.setInt(NUM_REGION_GROUPS, temp * 4);
 151       final String parallelism = Integer.toString(temp * 4);
 152       int errCode = WALPerformanceEvaluation.innerMain(new Configuration(CONF),
 153         new String[] { "-threads", parallelism, "-verify", "-noclosefs", "-iterations", "3000",
 154             "-regions", parallelism });
 155       assertEquals(0, errCode);
 156     } finally {
 157       CONF.setInt(NUM_REGION_GROUPS, temp);
 158     }
 159   }
 160
 161   @Test
 162   public void testMoreRegionsThanBoundWithBoundsGreaterThanDefault() throws Exception {
 163     final int temp = CONF.getInt(NUM_REGION_GROUPS, DEFAULT_NUM_REGION_GROUPS);
 164     try {
 165       CONF.setInt(NUM_REGION_GROUPS, temp * 4);
 166       final String parallelism = Integer.toString(temp * 4 * 2);
 167       int errCode = WALPerformanceEvaluation.innerMain(new Configuration(CONF),
 168         new String[] { "-threads", parallelism, "-verify", "-noclosefs", "-iterations", "3000",
 169             "-regions", parallelism });
 170       assertEquals(0, errCode);
 171     } finally {
 172       CONF.setInt(NUM_REGION_GROUPS, temp);
 173     }
 174   }
 175
 176   /**
 177    * Ensure that we can use Set.add to deduplicate WALs
 178    */
 179   @Test
 180   public void setMembershipDedups() throws IOException {
 181     final int temp = CONF.getInt(NUM_REGION_GROUPS, DEFAULT_NUM_REGION_GROUPS);
 182     WALFactory wals = null;
 183     try {
 184       CONF.setInt(NUM_REGION_GROUPS, temp * 4);
 185       // Set HDFS root directory for storing WAL
 186       FSUtils.setRootDir(CONF, TEST_UTIL.getDataTestDirOnTestFS());
 187
 188       wals = new WALFactory(CONF, "setMembershipDedups");
 189       Set<WAL> seen = new HashSet<>(temp * 4);
 190       int count = 0;
 191       // we know that this should see one of the wals more than once
 192       for (int i = 0; i < temp * 8; i++) {
 193         WAL maybeNewWAL = wals.getWAL(RegionInfoBuilder
 194             .newBuilder(TableName.valueOf("Table-" + ThreadLocalRandom.current().nextInt()))
 195             .build());
 196         LOG.info("Iteration " + i + ", checking wal " + maybeNewWAL);
 197         if (seen.add(maybeNewWAL)) {
 198           count++;
 199         }
 200       }
 201       assertEquals("received back a different number of WALs that are not equal() to each other "
 202           + "than the bound we placed.",
 203         temp * 4, count);
 204     } finally {
 205       if (wals != null) {
 206         wals.close();
 207       }
 208       CONF.setInt(NUM_REGION_GROUPS, temp);
 209     }
 210   }
 211 }