hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/BusyRegionSplitPolicy.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18
  19 package org.apache.hadoop.hbase.regionserver;
  20
  21 import org.apache.commons.logging.Log;
  22 import org.apache.commons.logging.LogFactory;
  23 import org.apache.hadoop.conf.Configuration;
  24 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
  25 import org.apache.hadoop.hbase.classification.InterfaceAudience;
  26 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
  27
  28 /**
  29  * This class represents a split policy which makes the split decision based
  30  * on how busy a region is. The metric that is used here is the fraction of
  31  * total write requests that are blocked due to high memstore utilization.
  32  * This fractional rate is calculated over a running window of
  33  * "hbase.busy.policy.aggWindow" milliseconds. The rate is a time-weighted
  34  * aggregated average of the rate in the current window and the
  35  * true average rate in the previous window.
  36  *
  37  */
  38
  39 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
  40 public class BusyRegionSplitPolicy extends IncreasingToUpperBoundRegionSplitPolicy {
  41
  42   private static final Log LOG = LogFactory.getLog(BusyRegionSplitPolicy.class);
  43
  44   // Maximum fraction blocked write requests before region is considered for split
  45   private float maxBlockedRequests;
  46   public static final float DEFAULT_MAX_BLOCKED_REQUESTS = 0.2f;
  47
  48   // Minimum age of the region in milliseconds before it is considered for split
  49   private long minAge = -1;
  50   public static final long DEFAULT_MIN_AGE_MS = 600000;  // 10 minutes
  51
  52   // The window time in milliseconds over which the blocked requests rate is calculated
  53   private long aggregationWindow;
  54   public static final long DEFAULT_AGGREGATION_WINDOW = 300000;  // 5 minutes
  55
  56   private HRegion region;
  57   private long prevTime;
  58   private long startTime;
  59   private long writeRequestCount;
  60   private long blockedRequestCount;
  61   private float blockedRate;
  62
  63   @Override
  64   protected void configureForRegion(final HRegion region) {
  65     super.configureForRegion(region);
  66     this.region = region;
  67     Configuration conf = getConf();
  68
  69     maxBlockedRequests = conf.getFloat("hbase.busy.policy.blockedRequests",
  70         DEFAULT_MAX_BLOCKED_REQUESTS);
  71     minAge = conf.getLong("hbase.busy.policy.minAge", DEFAULT_MIN_AGE_MS);
  72     aggregationWindow = conf.getLong("hbase.busy.policy.aggWindow",
  73         DEFAULT_AGGREGATION_WINDOW);
  74
  75     if (maxBlockedRequests < 0.00001f || maxBlockedRequests > 0.99999f) {
  76       LOG.warn("Threshold for maximum blocked requests is set too low or too high, "
  77           + " resetting to default of " + DEFAULT_MAX_BLOCKED_REQUESTS);
  78       maxBlockedRequests = DEFAULT_MAX_BLOCKED_REQUESTS;
  79     }
  80
  81     if (aggregationWindow <= 0) {
  82       LOG.warn("Aggregation window size is too low: " + aggregationWindow
  83           + ". Resetting it to default of " + DEFAULT_AGGREGATION_WINDOW);
  84       aggregationWindow = DEFAULT_AGGREGATION_WINDOW;
  85     }
  86
  87     init();
  88   }
  89
  90   private synchronized void init() {
  91     startTime = EnvironmentEdgeManager.currentTime();
  92     prevTime = startTime;
  93     blockedRequestCount = region.getBlockedRequestsCount();
  94     writeRequestCount = region.getWriteRequestsCount();
  95   }
  96
  97   @Override
  98   protected boolean shouldSplit() {
  99     float blockedReqRate = updateRate();
 100     if (super.shouldSplit()) {
 101       return true;
 102     }
 103
 104     if (EnvironmentEdgeManager.currentTime() <  startTime + minAge) {
 105       return false;
 106     }
 107
 108     for (Store store: region.getStores()) {
 109       if (!store.canSplit()) {
 110         return false;
 111       }
 112     }
 113
 114     if (blockedReqRate >= maxBlockedRequests) {
 115       if (LOG.isDebugEnabled()) {
 116         LOG.debug("Going to split region " + region.getRegionInfo().getRegionNameAsString()
 117             + " because it's too busy. Blocked Request rate: " + blockedReqRate);
 118       }
 119       return true;
 120     }
 121
 122     return false;
 123   }
 124
 125   /**
 126    * Update the blocked request rate based on number of blocked and total write requests in the
 127    * last aggregation window, or since last call to this method, whichever is farthest in time.
 128    * Uses weighted rate calculation based on the previous rate and new data.
 129    *
 130    * @return Updated blocked request rate.
 131    */
 132   private synchronized float updateRate() {
 133     float aggBlockedRate;
 134     long curTime = EnvironmentEdgeManager.currentTime();
 135
 136     long newBlockedReqs = region.getBlockedRequestsCount();
 137     long newWriteReqs = region.getWriteRequestsCount();
 138
 139     aggBlockedRate =
 140         (newBlockedReqs - blockedRequestCount) / (newWriteReqs - writeRequestCount + 0.00001f);
 141
 142     if (curTime - prevTime >= aggregationWindow) {
 143       blockedRate = aggBlockedRate;
 144       prevTime = curTime;
 145       blockedRequestCount = newBlockedReqs;
 146       writeRequestCount = newWriteReqs;
 147     } else if (curTime - startTime >= aggregationWindow) {
 148       // Calculate the aggregate blocked rate as the weighted sum of
 149       // previous window's average blocked rate and blocked rate in this window so far.
 150       float timeSlice = (curTime - prevTime) / (aggregationWindow + 0.0f);
 151       aggBlockedRate = (1 - timeSlice) * blockedRate + timeSlice * aggBlockedRate;
 152     } else {
 153       aggBlockedRate = 0.0f;
 154     }
 155     return aggBlockedRate;
 156   }
 157 }