net/url_request/url_request_throttler_manager.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_request_throttler_manager.h"
   6
   7 #include "base/logging.h"
   8 #include "base/metrics/field_trial.h"
   9 #include "base/metrics/histogram.h"
  10 #include "base/strings/string_util.h"
  11 #include "net/base/net_util.h"
  12 #include "net/log/net_log.h"
  13
  14 namespace net {
  15
  16 const unsigned int URLRequestThrottlerManager::kMaximumNumberOfEntries = 1500;
  17 const unsigned int URLRequestThrottlerManager::kRequestsBetweenCollecting = 200;
  18
  19 URLRequestThrottlerManager::URLRequestThrottlerManager()
  20     : requests_since_last_gc_(0),
  21       enable_thread_checks_(false),
  22       logged_for_localhost_disabled_(false),
  23       registered_from_thread_(base::kInvalidThreadId) {
  24   url_id_replacements_.ClearPassword();
  25   url_id_replacements_.ClearUsername();
  26   url_id_replacements_.ClearQuery();
  27   url_id_replacements_.ClearRef();
  28
  29   NetworkChangeNotifier::AddIPAddressObserver(this);
  30   NetworkChangeNotifier::AddConnectionTypeObserver(this);
  31 }
  32
  33 URLRequestThrottlerManager::~URLRequestThrottlerManager() {
  34   NetworkChangeNotifier::RemoveIPAddressObserver(this);
  35   NetworkChangeNotifier::RemoveConnectionTypeObserver(this);
  36
  37   // Since the manager object might conceivably go away before the
  38   // entries, detach the entries' back-pointer to the manager.
  39   UrlEntryMap::iterator i = url_entries_.begin();
  40   while (i != url_entries_.end()) {
  41     if (i->second.get() != NULL) {
  42       i->second->DetachManager();
  43     }
  44     ++i;
  45   }
  46
  47   // Delete all entries.
  48   url_entries_.clear();
  49 }
  50
  51 scoped_refptr<URLRequestThrottlerEntryInterface>
  52     URLRequestThrottlerManager::RegisterRequestUrl(const GURL &url) {
  53   DCHECK(!enable_thread_checks_ || CalledOnValidThread());
  54
  55   // Normalize the url.
  56   std::string url_id = GetIdFromUrl(url);
  57
  58   // Periodically garbage collect old entries.
  59   GarbageCollectEntriesIfNecessary();
  60
  61   // Find the entry in the map or create a new NULL entry.
  62   scoped_refptr<URLRequestThrottlerEntry>& entry = url_entries_[url_id];
  63
  64   // If the entry exists but could be garbage collected at this point, we
  65   // start with a fresh entry so that we possibly back off a bit less
  66   // aggressively (i.e. this resets the error count when the entry's URL
  67   // hasn't been requested in long enough).
  68   if (entry.get() && entry->IsEntryOutdated()) {
  69     entry = NULL;
  70   }
  71
  72   // Create the entry if needed.
  73   if (entry.get() == NULL) {
  74     entry = new URLRequestThrottlerEntry(this, url_id);
  75
  76     // We only disable back-off throttling on an entry that we have
  77     // just constructed.  This is to allow unit tests to explicitly override
  78     // the entry for localhost URLs.  Given that we do not attempt to
  79     // disable throttling for entries already handed out (see comment
  80     // in AddToOptOutList), this is not a problem.
  81     std::string host = url.host();
  82     if (opt_out_hosts_.find(host) != opt_out_hosts_.end() ||
  83         IsLocalhost(host)) {
  84       if (!logged_for_localhost_disabled_ && IsLocalhost(host)) {
  85         logged_for_localhost_disabled_ = true;
  86         net_log_.AddEvent(NetLog::TYPE_THROTTLING_DISABLED_FOR_HOST,
  87                           NetLog::StringCallback("host", &host));
  88       }
  89
  90       // TODO(joi): Once sliding window is separate from back-off throttling,
  91       // we can simply return a dummy implementation of
  92       // URLRequestThrottlerEntryInterface here that never blocks anything (and
  93       // not keep entries in url_entries_ for opted-out sites).
  94       entry->DisableBackoffThrottling();
  95     }
  96   }
  97
  98   return entry;
  99 }
 100
 101 void URLRequestThrottlerManager::AddToOptOutList(const std::string& host) {
 102   // There is an edge case here that we are not handling, to keep things
 103   // simple.  If a host starts adding the opt-out header to its responses
 104   // after there are already one or more entries in url_entries_ for that
 105   // host, the pre-existing entries may still perform back-off throttling.
 106   // In practice, this would almost never occur.
 107   if (opt_out_hosts_.find(host) == opt_out_hosts_.end()) {
 108     UMA_HISTOGRAM_COUNTS("Throttling.SiteOptedOut", 1);
 109
 110     net_log_.EndEvent(NetLog::TYPE_THROTTLING_DISABLED_FOR_HOST,
 111                       NetLog::StringCallback("host", &host));
 112     opt_out_hosts_.insert(host);
 113   }
 114 }
 115
 116 void URLRequestThrottlerManager::OverrideEntryForTests(
 117     const GURL& url,
 118     URLRequestThrottlerEntry* entry) {
 119   // Normalize the url.
 120   std::string url_id = GetIdFromUrl(url);
 121
 122   // Periodically garbage collect old entries.
 123   GarbageCollectEntriesIfNecessary();
 124
 125   url_entries_[url_id] = entry;
 126 }
 127
 128 void URLRequestThrottlerManager::EraseEntryForTests(const GURL& url) {
 129   // Normalize the url.
 130   std::string url_id = GetIdFromUrl(url);
 131   url_entries_.erase(url_id);
 132 }
 133
 134 void URLRequestThrottlerManager::set_enable_thread_checks(bool enable) {
 135   enable_thread_checks_ = enable;
 136 }
 137
 138 bool URLRequestThrottlerManager::enable_thread_checks() const {
 139   return enable_thread_checks_;
 140 }
 141
 142 void URLRequestThrottlerManager::set_net_log(NetLog* net_log) {
 143   DCHECK(net_log);
 144   net_log_ = BoundNetLog::Make(net_log,
 145                                NetLog::SOURCE_EXPONENTIAL_BACKOFF_THROTTLING);
 146 }
 147
 148 NetLog* URLRequestThrottlerManager::net_log() const {
 149   return net_log_.net_log();
 150 }
 151
 152 void URLRequestThrottlerManager::OnIPAddressChanged() {
 153   OnNetworkChange();
 154 }
 155
 156 void URLRequestThrottlerManager::OnConnectionTypeChanged(
 157     NetworkChangeNotifier::ConnectionType type) {
 158   OnNetworkChange();
 159 }
 160
 161 std::string URLRequestThrottlerManager::GetIdFromUrl(const GURL& url) const {
 162   if (!url.is_valid())
 163     return url.possibly_invalid_spec();
 164
 165   GURL id = url.ReplaceComponents(url_id_replacements_);
 166   return base::StringToLowerASCII(id.spec()).c_str();
 167 }
 168
 169 void URLRequestThrottlerManager::GarbageCollectEntriesIfNecessary() {
 170   requests_since_last_gc_++;
 171   if (requests_since_last_gc_ < kRequestsBetweenCollecting)
 172     return;
 173   requests_since_last_gc_ = 0;
 174
 175   GarbageCollectEntries();
 176 }
 177
 178 void URLRequestThrottlerManager::GarbageCollectEntries() {
 179   UrlEntryMap::iterator i = url_entries_.begin();
 180   while (i != url_entries_.end()) {
 181     if ((i->second)->IsEntryOutdated()) {
 182       url_entries_.erase(i++);
 183     } else {
 184       ++i;
 185     }
 186   }
 187
 188   // In case something broke we want to make sure not to grow indefinitely.
 189   while (url_entries_.size() > kMaximumNumberOfEntries) {
 190     url_entries_.erase(url_entries_.begin());
 191   }
 192 }
 193
 194 void URLRequestThrottlerManager::OnNetworkChange() {
 195   // Remove all entries.  Any entries that in-flight requests have a reference
 196   // to will live until those requests end, and these entries may be
 197   // inconsistent with new entries for the same URLs, but since what we
 198   // want is a clean slate for the new connection type, this is OK.
 199   url_entries_.clear();
 200   requests_since_last_gc_ = 0;
 201 }
 202
 203 }  // namespace net