net/url_request/url_request_throttler_entry.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_request_throttler_entry.h"
   6
   7 #include <cmath>
   8
   9 #include "base/logging.h"
  10 #include "base/metrics/field_trial.h"
  11 #include "base/metrics/histogram.h"
  12 #include "base/rand_util.h"
  13 #include "base/strings/string_number_conversions.h"
  14 #include "base/values.h"
  15 #include "net/base/load_flags.h"
  16 #include "net/log/net_log.h"
  17 #include "net/url_request/url_request.h"
  18 #include "net/url_request/url_request_context.h"
  19 #include "net/url_request/url_request_throttler_manager.h"
  20
  21 namespace net {
  22
  23 const int URLRequestThrottlerEntry::kDefaultSlidingWindowPeriodMs = 2000;
  24 const int URLRequestThrottlerEntry::kDefaultMaxSendThreshold = 20;
  25
  26 // This set of back-off parameters will (at maximum values, i.e. without
  27 // the reduction caused by jitter) add 0-41% (distributed uniformly
  28 // in that range) to the "perceived downtime" of the remote server, once
  29 // exponential back-off kicks in and is throttling requests for more than
  30 // about a second at a time.  Once the maximum back-off is reached, the added
  31 // perceived downtime decreases rapidly, percentage-wise.
  32 //
  33 // Another way to put it is that the maximum additional perceived downtime
  34 // with these numbers is a couple of seconds shy of 15 minutes, and such
  35 // a delay would not occur until the remote server has been actually
  36 // unavailable at the end of each back-off period for a total of about
  37 // 48 minutes.
  38 //
  39 // Ignoring the first couple of errors is just a conservative measure to
  40 // avoid false positives.  It should help avoid back-off from kicking in e.g.
  41 // on flaky connections.
  42 const int URLRequestThrottlerEntry::kDefaultNumErrorsToIgnore = 2;
  43 const int URLRequestThrottlerEntry::kDefaultInitialDelayMs = 700;
  44 const double URLRequestThrottlerEntry::kDefaultMultiplyFactor = 1.4;
  45 const double URLRequestThrottlerEntry::kDefaultJitterFactor = 0.4;
  46 const int URLRequestThrottlerEntry::kDefaultMaximumBackoffMs = 15 * 60 * 1000;
  47 const int URLRequestThrottlerEntry::kDefaultEntryLifetimeMs = 2 * 60 * 1000;
  48
  49 // Returns NetLog parameters when a request is rejected by throttling.
  50 scoped_ptr<base::Value> NetLogRejectedRequestCallback(
  51     const std::string* url_id,
  52     int num_failures,
  53     const base::TimeDelta& release_after,
  54     NetLogCaptureMode /* capture_mode */) {
  55   scoped_ptr<base::DictionaryValue> dict(new base::DictionaryValue());
  56   dict->SetString("url", *url_id);
  57   dict->SetInteger("num_failures", num_failures);
  58   dict->SetInteger("release_after_ms",
  59                    static_cast<int>(release_after.InMilliseconds()));
  60   return dict.Pass();
  61 }
  62
  63 URLRequestThrottlerEntry::URLRequestThrottlerEntry(
  64     URLRequestThrottlerManager* manager,
  65     const std::string& url_id)
  66     : sliding_window_period_(
  67           base::TimeDelta::FromMilliseconds(kDefaultSlidingWindowPeriodMs)),
  68       max_send_threshold_(kDefaultMaxSendThreshold),
  69       is_backoff_disabled_(false),
  70       backoff_entry_(&backoff_policy_),
  71       manager_(manager),
  72       url_id_(url_id),
  73       net_log_(BoundNetLog::Make(
  74           manager->net_log(), NetLog::SOURCE_EXPONENTIAL_BACKOFF_THROTTLING)) {
  75   DCHECK(manager_);
  76   Initialize();
  77 }
  78
  79 URLRequestThrottlerEntry::URLRequestThrottlerEntry(
  80     URLRequestThrottlerManager* manager,
  81     const std::string& url_id,
  82     int sliding_window_period_ms,
  83     int max_send_threshold,
  84     int initial_backoff_ms,
  85     double multiply_factor,
  86     double jitter_factor,
  87     int maximum_backoff_ms)
  88     : sliding_window_period_(
  89           base::TimeDelta::FromMilliseconds(sliding_window_period_ms)),
  90       max_send_threshold_(max_send_threshold),
  91       is_backoff_disabled_(false),
  92       backoff_entry_(&backoff_policy_),
  93       manager_(manager),
  94       url_id_(url_id) {
  95   DCHECK_GT(sliding_window_period_ms, 0);
  96   DCHECK_GT(max_send_threshold_, 0);
  97   DCHECK_GE(initial_backoff_ms, 0);
  98   DCHECK_GT(multiply_factor, 0);
  99   DCHECK_GE(jitter_factor, 0.0);
 100   DCHECK_LT(jitter_factor, 1.0);
 101   DCHECK_GE(maximum_backoff_ms, 0);
 102   DCHECK(manager_);
 103
 104   Initialize();
 105   backoff_policy_.initial_delay_ms = initial_backoff_ms;
 106   backoff_policy_.multiply_factor = multiply_factor;
 107   backoff_policy_.jitter_factor = jitter_factor;
 108   backoff_policy_.maximum_backoff_ms = maximum_backoff_ms;
 109   backoff_policy_.entry_lifetime_ms = -1;
 110   backoff_policy_.num_errors_to_ignore = 0;
 111   backoff_policy_.always_use_initial_delay = false;
 112 }
 113
 114 bool URLRequestThrottlerEntry::IsEntryOutdated() const {
 115   // This function is called by the URLRequestThrottlerManager to determine
 116   // whether entries should be discarded from its url_entries_ map.  We
 117   // want to ensure that it does not remove entries from the map while there
 118   // are clients (objects other than the manager) holding references to
 119   // the entry, otherwise separate clients could end up holding separate
 120   // entries for a request to the same URL, which is undesirable.  Therefore,
 121   // if an entry has more than one reference (the map will always hold one),
 122   // it should not be considered outdated.
 123   //
 124   // We considered whether to make URLRequestThrottlerEntry objects
 125   // non-refcounted, but since any means of knowing whether they are
 126   // currently in use by others than the manager would be more or less
 127   // equivalent to a refcount, we kept them refcounted.
 128   if (!HasOneRef())
 129     return false;
 130
 131   // If there are send events in the sliding window period, we still need this
 132   // entry.
 133   if (!send_log_.empty() &&
 134       send_log_.back() + sliding_window_period_ > ImplGetTimeNow()) {
 135     return false;
 136   }
 137
 138   return GetBackoffEntry()->CanDiscard();
 139 }
 140
 141 void URLRequestThrottlerEntry::DisableBackoffThrottling() {
 142   is_backoff_disabled_ = true;
 143 }
 144
 145 void URLRequestThrottlerEntry::DetachManager() {
 146   manager_ = NULL;
 147 }
 148
 149 bool URLRequestThrottlerEntry::ShouldRejectRequest(
 150     const URLRequest& request,
 151     NetworkDelegate* network_delegate) const {
 152   bool reject_request = false;
 153   if (!is_backoff_disabled_ && !ExplicitUserRequest(request.load_flags()) &&
 154       (!network_delegate || network_delegate->CanThrottleRequest(request)) &&
 155       GetBackoffEntry()->ShouldRejectRequest()) {
 156     net_log_.AddEvent(
 157         NetLog::TYPE_THROTTLING_REJECTED_REQUEST,
 158         base::Bind(&NetLogRejectedRequestCallback,
 159                    &url_id_,
 160                    GetBackoffEntry()->failure_count(),
 161                    GetBackoffEntry()->GetTimeUntilRelease()));
 162     reject_request = true;
 163   }
 164
 165   int reject_count = reject_request ? 1 : 0;
 166   UMA_HISTOGRAM_ENUMERATION(
 167       "Throttling.RequestThrottled", reject_count, 2);
 168
 169   return reject_request;
 170 }
 171
 172 int64 URLRequestThrottlerEntry::ReserveSendingTimeForNextRequest(
 173     const base::TimeTicks& earliest_time) {
 174   base::TimeTicks now = ImplGetTimeNow();
 175
 176   // If a lot of requests were successfully made recently,
 177   // sliding_window_release_time_ may be greater than
 178   // exponential_backoff_release_time_.
 179   base::TimeTicks recommended_sending_time =
 180       std::max(std::max(now, earliest_time),
 181                std::max(GetBackoffEntry()->GetReleaseTime(),
 182                         sliding_window_release_time_));
 183
 184   DCHECK(send_log_.empty() ||
 185          recommended_sending_time >= send_log_.back());
 186   // Log the new send event.
 187   send_log_.push(recommended_sending_time);
 188
 189   sliding_window_release_time_ = recommended_sending_time;
 190
 191   // Drop the out-of-date events in the event list.
 192   // We don't need to worry that the queue may become empty during this
 193   // operation, since the last element is sliding_window_release_time_.
 194   while ((send_log_.front() + sliding_window_period_ <=
 195           sliding_window_release_time_) ||
 196          send_log_.size() > static_cast<unsigned>(max_send_threshold_)) {
 197     send_log_.pop();
 198   }
 199
 200   // Check if there are too many send events in recent time.
 201   if (send_log_.size() == static_cast<unsigned>(max_send_threshold_))
 202     sliding_window_release_time_ = send_log_.front() + sliding_window_period_;
 203
 204   return (recommended_sending_time - now).InMillisecondsRoundedUp();
 205 }
 206
 207 base::TimeTicks
 208     URLRequestThrottlerEntry::GetExponentialBackoffReleaseTime() const {
 209   // If a site opts out, it's likely because they have problems that trigger
 210   // the back-off mechanism when it shouldn't be triggered, in which case
 211   // returning the calculated back-off release time would probably be the
 212   // wrong thing to do (i.e. it would likely be too long).  Therefore, we
 213   // return "now" so that retries are not delayed.
 214   if (is_backoff_disabled_)
 215     return ImplGetTimeNow();
 216
 217   return GetBackoffEntry()->GetReleaseTime();
 218 }
 219
 220 void URLRequestThrottlerEntry::UpdateWithResponse(int status_code) {
 221   GetBackoffEntry()->InformOfRequest(IsConsideredSuccess(status_code));
 222 }
 223
 224 void URLRequestThrottlerEntry::ReceivedContentWasMalformed(int response_code) {
 225   // A malformed body can only occur when the request to fetch a resource
 226   // was successful.  Therefore, in such a situation, we will receive one
 227   // call to ReceivedContentWasMalformed() and one call to
 228   // UpdateWithResponse() with a response categorized as "good".  To end
 229   // up counting one failure, we need to count two failures here against
 230   // the one success in UpdateWithResponse().
 231   //
 232   // We do nothing for a response that is already being considered an error
 233   // based on its status code (otherwise we would count 3 errors instead of 1).
 234   if (IsConsideredSuccess(response_code)) {
 235     GetBackoffEntry()->InformOfRequest(false);
 236     GetBackoffEntry()->InformOfRequest(false);
 237   }
 238 }
 239
 240 URLRequestThrottlerEntry::~URLRequestThrottlerEntry() {
 241 }
 242
 243 void URLRequestThrottlerEntry::Initialize() {
 244   sliding_window_release_time_ = base::TimeTicks::Now();
 245   backoff_policy_.num_errors_to_ignore = kDefaultNumErrorsToIgnore;
 246   backoff_policy_.initial_delay_ms = kDefaultInitialDelayMs;
 247   backoff_policy_.multiply_factor = kDefaultMultiplyFactor;
 248   backoff_policy_.jitter_factor = kDefaultJitterFactor;
 249   backoff_policy_.maximum_backoff_ms = kDefaultMaximumBackoffMs;
 250   backoff_policy_.entry_lifetime_ms = kDefaultEntryLifetimeMs;
 251   backoff_policy_.always_use_initial_delay = false;
 252 }
 253
 254 bool URLRequestThrottlerEntry::IsConsideredSuccess(int response_code) {
 255   // We throttle only for the status codes most likely to indicate the server
 256   // is failing because it is too busy or otherwise are likely to be
 257   // because of DDoS.
 258   //
 259   // 500 is the generic error when no better message is suitable, and
 260   //     as such does not necessarily indicate a temporary state, but
 261   //     other status codes cover most of the permanent error states.
 262   // 503 is explicitly documented as a temporary state where the server
 263   //     is either overloaded or down for maintenance.
 264   // 509 is the (non-standard but widely implemented) Bandwidth Limit Exceeded
 265   //     status code, which might indicate DDoS.
 266   //
 267   // We do not back off on 502 or 504, which are reported by gateways
 268   // (proxies) on timeouts or failures, because in many cases these requests
 269   // have not made it to the destination server and so we do not actually
 270   // know that it is down or busy.  One degenerate case could be a proxy on
 271   // localhost, where you are not actually connected to the network.
 272   return !(response_code == 500 || response_code == 503 ||
 273            response_code == 509);
 274 }
 275
 276 base::TimeTicks URLRequestThrottlerEntry::ImplGetTimeNow() const {
 277   return base::TimeTicks::Now();
 278 }
 279
 280 const BackoffEntry* URLRequestThrottlerEntry::GetBackoffEntry() const {
 281   return &backoff_entry_;
 282 }
 283
 284 BackoffEntry* URLRequestThrottlerEntry::GetBackoffEntry() {
 285   return &backoff_entry_;
 286 }
 287
 288 // static
 289 bool URLRequestThrottlerEntry::ExplicitUserRequest(const int load_flags) {
 290   return (load_flags & LOAD_MAYBE_USER_GESTURE) != 0;
 291 }
 292
 293 }  // namespace net