hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestClusterScopeQuotaThrottle.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.quotas;
  19
  20 import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.doGets;
  21 import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.doPuts;
  22 import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.triggerNamespaceCacheRefresh;
  23 import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.triggerTableCacheRefresh;
  24 import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.triggerUserCacheRefresh;
  25 import static org.junit.Assert.assertEquals;
  26 import static org.junit.Assert.assertTrue;
  27
  28 import java.util.concurrent.TimeUnit;
  29
  30 import org.apache.hadoop.hbase.HBaseClassTestRule;
  31 import org.apache.hadoop.hbase.HBaseTestingUtility;
  32 import org.apache.hadoop.hbase.HConstants;
  33 import org.apache.hadoop.hbase.NamespaceDescriptor;
  34 import org.apache.hadoop.hbase.TableName;
  35 import org.apache.hadoop.hbase.client.Admin;
  36 import org.apache.hadoop.hbase.client.Table;
  37 import org.apache.hadoop.hbase.security.User;
  38 import org.apache.hadoop.hbase.testclassification.LargeTests;
  39 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
  40 import org.apache.hadoop.hbase.util.Bytes;
  41 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
  42 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
  43 import org.junit.After;
  44 import org.junit.AfterClass;
  45 import org.junit.BeforeClass;
  46 import org.junit.ClassRule;
  47 import org.junit.Test;
  48 import org.junit.experimental.categories.Category;
  49
  50 @Category({ RegionServerTests.class, LargeTests.class })
  51 public class TestClusterScopeQuotaThrottle {
  52
  53   @ClassRule
  54   public static final HBaseClassTestRule CLASS_RULE =
  55       HBaseClassTestRule.forClass(TestClusterScopeQuotaThrottle.class);
  56
  57   private final static int REFRESH_TIME = 30 * 60000;
  58   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
  59
  60   private final static TableName[] TABLE_NAMES =
  61       new TableName[] { TableName.valueOf("TestQuotaAdmin0"), TableName.valueOf("TestQuotaAdmin1"),
  62           TableName.valueOf("TestQuotaAdmin2") };
  63   private final static byte[] FAMILY = Bytes.toBytes("cf");
  64   private final static byte[] QUALIFIER = Bytes.toBytes("q");
  65   private final static byte[][] SPLITS = new byte[][] { Bytes.toBytes("1") };
  66   private static Table[] tables;
  67
  68   private final static String NAMESPACE = "TestNs";
  69   private final static TableName TABLE_NAME = TableName.valueOf(NAMESPACE, "TestTable");
  70   private static Table table;
  71
  72   @BeforeClass
  73   public static void setUpBeforeClass() throws Exception {
  74     TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
  75     TEST_UTIL.getConfiguration().setInt(QuotaCache.REFRESH_CONF_KEY, REFRESH_TIME);
  76     TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
  77     TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
  78     TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
  79     TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
  80     TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
  81     TEST_UTIL.startMiniCluster(2);
  82     TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME);
  83     QuotaCache.TEST_FORCE_REFRESH = true;
  84
  85     tables = new Table[TABLE_NAMES.length];
  86     for (int i = 0; i < TABLE_NAMES.length; ++i) {
  87       tables[i] = TEST_UTIL.createTable(TABLE_NAMES[i], FAMILY);
  88       TEST_UTIL.waitTableAvailable(TABLE_NAMES[i]);
  89     }
  90     TEST_UTIL.getAdmin().createNamespace(NamespaceDescriptor.create(NAMESPACE).build());
  91     table = TEST_UTIL.createTable(TABLE_NAME, FAMILY, SPLITS);
  92     TEST_UTIL.waitTableAvailable(TABLE_NAME);
  93   }
  94
  95   @AfterClass
  96   public static void tearDownAfterClass() throws Exception {
  97     EnvironmentEdgeManager.reset();
  98     for (int i = 0; i < tables.length; ++i) {
  99       if (tables[i] != null) {
 100         tables[i].close();
 101         TEST_UTIL.deleteTable(TABLE_NAMES[i]);
 102       }
 103     }
 104     TEST_UTIL.deleteTable(TABLE_NAME);
 105     TEST_UTIL.getAdmin().deleteNamespace(NAMESPACE);
 106     TEST_UTIL.shutdownMiniCluster();
 107   }
 108
 109   @After
 110   public void tearDown() throws Exception {
 111     ThrottleQuotaTestUtil.clearQuotaCache(TEST_UTIL);
 112   }
 113
 114   @Test
 115   public void testNamespaceClusterScopeQuota() throws Exception {
 116     final Admin admin = TEST_UTIL.getAdmin();
 117     final String NAMESPACE = "default";
 118
 119     // Add 10req/min limit for write request in cluster scope
 120     admin.setQuota(QuotaSettingsFactory.throttleNamespace(NAMESPACE, ThrottleType.WRITE_NUMBER, 10,
 121       TimeUnit.MINUTES, QuotaScope.CLUSTER));
 122     // Add 6req/min limit for read request in machine scope
 123     admin.setQuota(QuotaSettingsFactory.throttleNamespace(NAMESPACE, ThrottleType.READ_NUMBER, 6,
 124       TimeUnit.MINUTES, QuotaScope.MACHINE));
 125     triggerNamespaceCacheRefresh(TEST_UTIL, false, TABLE_NAMES[0]);
 126     // should execute at max 5 write requests and at max 3 read requests
 127     assertEquals(5, doPuts(10, FAMILY, QUALIFIER, tables[0]));
 128     assertEquals(6, doGets(10, tables[0]));
 129     // Remove all the limits
 130     admin.setQuota(QuotaSettingsFactory.unthrottleNamespace(NAMESPACE));
 131     triggerNamespaceCacheRefresh(TEST_UTIL, true, TABLE_NAMES[0]);
 132   }
 133
 134   @Test
 135   public void testTableClusterScopeQuota() throws Exception {
 136     final Admin admin = TEST_UTIL.getAdmin();
 137     admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAME, ThrottleType.READ_NUMBER, 20,
 138       TimeUnit.HOURS, QuotaScope.CLUSTER));
 139     triggerTableCacheRefresh(TEST_UTIL, false, TABLE_NAME);
 140     for (RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
 141       for (TableName tableName : rst.getRegionServer().getOnlineTables()) {
 142         if (tableName.getNameAsString().equals(TABLE_NAME.getNameAsString())) {
 143           int rsRegionNum = rst.getRegionServer().getRegions(tableName).size();
 144           if (rsRegionNum == 0) {
 145             // If rs has 0 region, the machine limiter is 0 (20 * 0 / 2)
 146             break;
 147           } else if (rsRegionNum == 1) {
 148             // If rs has 1 region, the machine limiter is 10 (20 * 1 / 2)
 149             // Read rows from 1 region, so can read 10 first time and 0 second time
 150             long count = doGets(20, table);
 151             assertTrue(count == 0 || count == 10);
 152           } else if (rsRegionNum == 2) {
 153             // If rs has 2 regions, the machine limiter is 20 (20 * 2 / 2)
 154             assertEquals(20, doGets(20, table));
 155           }
 156           break;
 157         }
 158       }
 159     }
 160     admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAME));
 161     triggerTableCacheRefresh(TEST_UTIL, true, TABLE_NAME);
 162   }
 163
 164   @Test
 165   public void testUserClusterScopeQuota() throws Exception {
 166     final Admin admin = TEST_UTIL.getAdmin();
 167     final String userName = User.getCurrent().getShortName();
 168
 169     // Add 6req/min limit for read request in cluster scope
 170     admin.setQuota(QuotaSettingsFactory.throttleUser(userName, ThrottleType.READ_NUMBER, 6,
 171       TimeUnit.MINUTES, QuotaScope.CLUSTER));
 172     // Add 6req/min limit for write request in machine scope
 173     admin.setQuota(
 174       QuotaSettingsFactory.throttleUser(userName, ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
 175     triggerUserCacheRefresh(TEST_UTIL, false, TABLE_NAMES);
 176     // should execute at max 6 read requests and at max 3 write write requests
 177     assertEquals(6, doPuts(10, FAMILY, QUALIFIER, tables[0]));
 178     assertEquals(3, doGets(10, tables[0]));
 179     // Remove all the limits
 180     admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
 181     triggerUserCacheRefresh(TEST_UTIL, true, TABLE_NAMES);
 182   }
 183
 184   @Test
 185   public void testUserNamespaceClusterScopeQuota() throws Exception {
 186     final Admin admin = TEST_UTIL.getAdmin();
 187     final String userName = User.getCurrent().getShortName();
 188     final String namespace = TABLE_NAMES[0].getNamespaceAsString();
 189
 190     // Add 10req/min limit for read request in cluster scope
 191     admin.setQuota(QuotaSettingsFactory.throttleUser(userName, namespace, ThrottleType.READ_NUMBER,
 192       10, TimeUnit.MINUTES, QuotaScope.CLUSTER));
 193     // Add 6req/min limit for write request in machine scope
 194     admin.setQuota(QuotaSettingsFactory.throttleUser(userName, namespace, ThrottleType.WRITE_NUMBER,
 195       6, TimeUnit.MINUTES));
 196     triggerUserCacheRefresh(TEST_UTIL, false, TABLE_NAMES[0]);
 197     // should execute at max 5 read requests and at max 6 write requests
 198     assertEquals(5, doGets(10, tables[0]));
 199     assertEquals(6, doPuts(10, FAMILY, QUALIFIER, tables[0]));
 200
 201     // Remove all the limits
 202     admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, namespace));
 203     triggerUserCacheRefresh(TEST_UTIL, true, TABLE_NAMES[0]);
 204   }
 205
 206   @Test
 207   public void testUserTableClusterScopeQuota() throws Exception {
 208     final Admin admin = TEST_UTIL.getAdmin();
 209     final String userName = User.getCurrent().getShortName();
 210     admin.setQuota(QuotaSettingsFactory.throttleUser(userName, TABLE_NAME, ThrottleType.READ_NUMBER,
 211       20, TimeUnit.HOURS, QuotaScope.CLUSTER));
 212     triggerUserCacheRefresh(TEST_UTIL, false, TABLE_NAME);
 213     for (RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
 214       for (TableName tableName : rst.getRegionServer().getOnlineTables()) {
 215         if (tableName.getNameAsString().equals(TABLE_NAME.getNameAsString())) {
 216           int rsRegionNum = rst.getRegionServer().getRegions(tableName).size();
 217           if (rsRegionNum == 0) {
 218             // If rs has 0 region, the machine limiter is 0 (20 * 0 / 2)
 219             break;
 220           } else if (rsRegionNum == 1) {
 221             // If rs has 1 region, the machine limiter is 10 (20 * 1 / 2)
 222             // Read rows from 1 region, so can read 10 first time and 0 second time
 223             long count = doGets(20, table);
 224             assertTrue(count == 0 || count == 10);
 225           } else if (rsRegionNum == 2) {
 226             // If rs has 2 regions, the machine limiter is 20 (20 * 2 / 2)
 227             assertEquals(20, doGets(20, table));
 228           }
 229           break;
 230         }
 231       }
 232     }
 233     admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
 234     triggerUserCacheRefresh(TEST_UTIL, true, TABLE_NAME);
 235   }
 236 }