hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiLogThreshold.java

   1 /*
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.regionserver;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertFalse;
  22 import static org.junit.Assert.assertTrue;
  23 import static org.mockito.ArgumentMatchers.any;
  24 import static org.mockito.Mockito.doAnswer;
  25 import static org.mockito.Mockito.mock;
  26 import static org.mockito.Mockito.when;
  27
  28 import java.io.IOException;
  29 import java.util.Arrays;
  30 import java.util.List;
  31 import java.util.concurrent.BlockingDeque;
  32 import java.util.concurrent.LinkedBlockingDeque;
  33 import org.apache.hadoop.conf.Configuration;
  34 import org.apache.hadoop.hbase.HBaseClassTestRule;
  35 import org.apache.hadoop.hbase.HBaseTestingUtil;
  36 import org.apache.hadoop.hbase.HConstants;
  37 import org.apache.hadoop.hbase.TableName;
  38 import org.apache.hadoop.hbase.ipc.HBaseRpcController;
  39 import org.apache.hadoop.hbase.testclassification.MediumTests;
  40 import org.apache.hadoop.hbase.util.Bytes;
  41 import org.junit.After;
  42 import org.junit.Before;
  43 import org.junit.ClassRule;
  44 import org.junit.Test;
  45 import org.junit.experimental.categories.Category;
  46 import org.junit.runner.RunWith;
  47 import org.junit.runners.Parameterized;
  48 import org.mockito.Mockito;
  49 import org.mockito.invocation.InvocationOnMock;
  50 import org.mockito.stubbing.Answer;
  51
  52 import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
  53 import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
  54
  55 import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
  56 import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.Action;
  57 import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MultiRequest;
  58 import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.RegionAction;
  59 import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
  60
  61 /**
  62  * Tests logging of large batch commands via Multi. Tests are fast, but uses a mini-cluster (to test
  63  * via "Multi" commands) so classified as MediumTests
  64  */
  65 @RunWith(Parameterized.class)
  66 @Category(MediumTests.class)
  67 public class TestMultiLogThreshold {
  68
  69   @ClassRule
  70   public static final HBaseClassTestRule CLASS_RULE =
  71     HBaseClassTestRule.forClass(TestMultiLogThreshold.class);
  72
  73   private static final TableName NAME = TableName.valueOf("tableName");
  74   private static final byte[] TEST_FAM = Bytes.toBytes("fam");
  75
  76   private HBaseTestingUtil util;
  77   private Configuration conf;
  78   private int threshold;
  79   private HRegionServer rs;
  80   private RSRpcServices services;
  81
  82   private org.apache.logging.log4j.core.Appender appender;
  83
  84   @Parameterized.Parameter
  85   public static boolean rejectLargeBatchOp;
  86
  87   @Parameterized.Parameters
  88   public static List<Object[]> params() {
  89     return Arrays.asList(new Object[] { false }, new Object[] { true });
  90   }
  91
  92   private final class LevelAndMessage {
  93     final org.apache.logging.log4j.Level level;
  94
  95     final String msg;
  96
  97     public LevelAndMessage(org.apache.logging.log4j.Level level, String msg) {
  98       this.level = level;
  99       this.msg = msg;
 100     }
 101
 102   }
 103
 104   // log4j2 will reuse the LogEvent so we need to copy the level and message out.
 105   private BlockingDeque<LevelAndMessage> logs = new LinkedBlockingDeque<>();
 106
 107   @Before
 108   public void setupTest() throws Exception {
 109     util = new HBaseTestingUtil();
 110     conf = util.getConfiguration();
 111     threshold =
 112       conf.getInt(HConstants.BATCH_ROWS_THRESHOLD_NAME, HConstants.BATCH_ROWS_THRESHOLD_DEFAULT);
 113     conf.setBoolean("hbase.rpc.rows.size.threshold.reject", rejectLargeBatchOp);
 114     util.startMiniCluster();
 115     util.createTable(NAME, TEST_FAM);
 116     rs = util.getRSForFirstRegionInTable(NAME);
 117     appender = mock(org.apache.logging.log4j.core.Appender.class);
 118     when(appender.getName()).thenReturn("mockAppender");
 119     when(appender.isStarted()).thenReturn(true);
 120     doAnswer(new Answer<Void>() {
 121
 122       @Override
 123       public Void answer(InvocationOnMock invocation) throws Throwable {
 124         org.apache.logging.log4j.core.LogEvent logEvent =
 125           invocation.getArgument(0, org.apache.logging.log4j.core.LogEvent.class);
 126         logs.add(
 127           new LevelAndMessage(logEvent.getLevel(), logEvent.getMessage().getFormattedMessage()));
 128         return null;
 129       }
 130     }).when(appender).append(any(org.apache.logging.log4j.core.LogEvent.class));
 131     ((org.apache.logging.log4j.core.Logger) org.apache.logging.log4j.LogManager
 132       .getLogger(RSRpcServices.class)).addAppender(appender);
 133   }
 134
 135   @After
 136   public void tearDown() throws Exception {
 137     ((org.apache.logging.log4j.core.Logger) org.apache.logging.log4j.LogManager
 138       .getLogger(RSRpcServices.class)).removeAppender(appender);
 139     util.shutdownMiniCluster();
 140   }
 141
 142   private enum ActionType {
 143     REGION_ACTIONS, ACTIONS
 144   }
 145
 146   /**
 147    * Sends a multi request with a certain amount of rows, will populate Multi command with either
 148    * "rows" number of RegionActions with one Action each or one RegionAction with "rows" number of
 149    * Actions
 150    */
 151   private void sendMultiRequest(int rows, ActionType actionType)
 152     throws ServiceException, IOException {
 153     RpcController rpcc = Mockito.mock(HBaseRpcController.class);
 154     MultiRequest.Builder builder = MultiRequest.newBuilder();
 155     int numRAs = 1;
 156     int numAs = 1;
 157     switch (actionType) {
 158       case REGION_ACTIONS:
 159         numRAs = rows;
 160         break;
 161       case ACTIONS:
 162         numAs = rows;
 163         break;
 164     }
 165     for (int i = 0; i < numRAs; i++) {
 166       RegionAction.Builder rab = RegionAction.newBuilder();
 167       rab.setRegion(RequestConverter.buildRegionSpecifier(
 168         HBaseProtos.RegionSpecifier.RegionSpecifierType.REGION_NAME,
 169         Bytes.toBytes("someStuff" + i)));
 170       for (int j = 0; j < numAs; j++) {
 171         Action.Builder ab = Action.newBuilder();
 172         rab.addAction(ab.build());
 173       }
 174       builder.addRegionAction(rab.build());
 175     }
 176     services = new RSRpcServices(rs);
 177     services.multi(rpcc, builder.build());
 178   }
 179
 180   private void assertLogBatchWarnings(boolean expected) {
 181     boolean actual = false;
 182     for (LevelAndMessage event : logs) {
 183       if (event.level == org.apache.logging.log4j.Level.WARN &&
 184         event.msg.contains("Large batch operation detected")) {
 185         actual = true;
 186         break;
 187       }
 188     }
 189     logs.clear();
 190     assertEquals(expected, actual);
 191   }
 192
 193   @Test
 194   public void testMultiLogThresholdRegionActions() throws ServiceException, IOException {
 195     try {
 196       sendMultiRequest(threshold + 1, ActionType.REGION_ACTIONS);
 197       assertFalse(rejectLargeBatchOp);
 198     } catch (ServiceException e) {
 199       assertTrue(rejectLargeBatchOp);
 200     }
 201     assertLogBatchWarnings(true);
 202
 203     sendMultiRequest(threshold, ActionType.REGION_ACTIONS);
 204     assertLogBatchWarnings(false);
 205
 206     try {
 207       sendMultiRequest(threshold + 1, ActionType.ACTIONS);
 208       assertFalse(rejectLargeBatchOp);
 209     } catch (ServiceException e) {
 210       assertTrue(rejectLargeBatchOp);
 211     }
 212     assertLogBatchWarnings(true);
 213
 214     sendMultiRequest(threshold, ActionType.ACTIONS);
 215     assertLogBatchWarnings(false);
 216   }
 217 }