hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/BusyRegionSplitPolicy.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18
  19 package org.apache.hadoop.hbase.regionserver;
  20
  21 import org.apache.hadoop.conf.Configuration;
  22 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
  23 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
  24 import org.apache.yetus.audience.InterfaceAudience;
  25 import org.slf4j.Logger;
  26 import org.slf4j.LoggerFactory;
  27
  28 /**
  29  * This class represents a split policy which makes the split decision based
  30  * on how busy a region is. The metric that is used here is the fraction of
  31  * total write requests that are blocked due to high memstore utilization.
  32  * This fractional rate is calculated over a running window of
  33  * "hbase.busy.policy.aggWindow" milliseconds. The rate is a time-weighted
  34  * aggregated average of the rate in the current window and the
  35  * true average rate in the previous window.
  36  *
  37  */
  38
  39 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
  40 public class BusyRegionSplitPolicy extends IncreasingToUpperBoundRegionSplitPolicy {
  41
  42   private static final Logger LOG = LoggerFactory.getLogger(BusyRegionSplitPolicy.class);
  43
  44   // Maximum fraction blocked write requests before region is considered for split
  45   private float maxBlockedRequests;
  46   public static final float DEFAULT_MAX_BLOCKED_REQUESTS = 0.2f;
  47
  48   // Minimum age of the region in milliseconds before it is considered for split
  49   private long minAge = -1;
  50   public static final long DEFAULT_MIN_AGE_MS = 600000;  // 10 minutes
  51
  52   // The window time in milliseconds over which the blocked requests rate is calculated
  53   private long aggregationWindow;
  54   public static final long DEFAULT_AGGREGATION_WINDOW = 300000;  // 5 minutes
  55
  56   private HRegion region;
  57   private long prevTime;
  58   private long startTime;
  59   private long writeRequestCount;
  60   private long blockedRequestCount;
  61   private float blockedRate;
  62
  63   @Override
  64   public String toString() {
  65     return "BusyRegionSplitPolicy{" + "maxBlockedRequests=" + maxBlockedRequests + ", minAge="
  66       + minAge + ", aggregationWindow=" + aggregationWindow + ", " + super.toString() + '}';
  67   }
  68
  69   @Override
  70   protected void configureForRegion(final HRegion region) {
  71     super.configureForRegion(region);
  72     this.region = region;
  73     Configuration conf = getConf();
  74
  75     maxBlockedRequests = conf.getFloat("hbase.busy.policy.blockedRequests",
  76         DEFAULT_MAX_BLOCKED_REQUESTS);
  77     minAge = conf.getLong("hbase.busy.policy.minAge", DEFAULT_MIN_AGE_MS);
  78     aggregationWindow = conf.getLong("hbase.busy.policy.aggWindow",
  79         DEFAULT_AGGREGATION_WINDOW);
  80
  81     if (maxBlockedRequests < 0.00001f || maxBlockedRequests > 0.99999f) {
  82       LOG.warn("Threshold for maximum blocked requests is set too low or too high, "
  83           + " resetting to default of " + DEFAULT_MAX_BLOCKED_REQUESTS);
  84       maxBlockedRequests = DEFAULT_MAX_BLOCKED_REQUESTS;
  85     }
  86
  87     if (aggregationWindow <= 0) {
  88       LOG.warn("Aggregation window size is too low: " + aggregationWindow
  89           + ". Resetting it to default of " + DEFAULT_AGGREGATION_WINDOW);
  90       aggregationWindow = DEFAULT_AGGREGATION_WINDOW;
  91     }
  92
  93     init();
  94   }
  95
  96   private synchronized void init() {
  97     startTime = EnvironmentEdgeManager.currentTime();
  98     prevTime = startTime;
  99     blockedRequestCount = region.getBlockedRequestsCount();
 100     writeRequestCount = region.getWriteRequestsCount();
 101   }
 102
 103   @Override
 104   protected boolean shouldSplit() {
 105     float blockedReqRate = updateRate();
 106     if (super.shouldSplit()) {
 107       return true;
 108     }
 109
 110     if (EnvironmentEdgeManager.currentTime() <  startTime + minAge) {
 111       return false;
 112     }
 113
 114     for (HStore store: region.getStores()) {
 115       if (!store.canSplit()) {
 116         return false;
 117       }
 118     }
 119
 120     if (blockedReqRate >= maxBlockedRequests) {
 121       if (LOG.isDebugEnabled()) {
 122         LOG.debug("Going to split region " + region.getRegionInfo().getRegionNameAsString()
 123             + " because it's too busy. Blocked Request rate: " + blockedReqRate);
 124       }
 125       return true;
 126     }
 127
 128     return false;
 129   }
 130
 131   /**
 132    * Update the blocked request rate based on number of blocked and total write requests in the
 133    * last aggregation window, or since last call to this method, whichever is farthest in time.
 134    * Uses weighted rate calculation based on the previous rate and new data.
 135    *
 136    * @return Updated blocked request rate.
 137    */
 138   private synchronized float updateRate() {
 139     float aggBlockedRate;
 140     long curTime = EnvironmentEdgeManager.currentTime();
 141
 142     long newBlockedReqs = region.getBlockedRequestsCount();
 143     long newWriteReqs = region.getWriteRequestsCount();
 144
 145     aggBlockedRate =
 146         (newBlockedReqs - blockedRequestCount) / (newWriteReqs - writeRequestCount + 0.00001f);
 147
 148     if (curTime - prevTime >= aggregationWindow) {
 149       blockedRate = aggBlockedRate;
 150       prevTime = curTime;
 151       blockedRequestCount = newBlockedReqs;
 152       writeRequestCount = newWriteReqs;
 153     } else if (curTime - startTime >= aggregationWindow) {
 154       // Calculate the aggregate blocked rate as the weighted sum of
 155       // previous window's average blocked rate and blocked rate in this window so far.
 156       float timeSlice = (curTime - prevTime) / (aggregationWindow + 0.0f);
 157       aggBlockedRate = (1 - timeSlice) * blockedRate + timeSlice * aggBlockedRate;
 158     } else {
 159       aggBlockedRate = 0.0f;
 160     }
 161     return aggBlockedRate;
 162   }
 163 }