net/url_request/url_request_throttler_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_request_throttler_manager.h"
   6
   7 #include "base/memory/scoped_ptr.h"
   8 #include "base/metrics/histogram_samples.h"
   9 #include "base/pickle.h"
  10 #include "base/stl_util.h"
  11 #include "base/strings/string_number_conversions.h"
  12 #include "base/strings/stringprintf.h"
  13 #include "base/test/histogram_tester.h"
  14 #include "base/time/time.h"
  15 #include "net/base/load_flags.h"
  16 #include "net/base/request_priority.h"
  17 #include "net/base/test_completion_callback.h"
  18 #include "net/url_request/url_request.h"
  19 #include "net/url_request/url_request_context.h"
  20 #include "net/url_request/url_request_test_util.h"
  21 #include "net/url_request/url_request_throttler_test_support.h"
  22 #include "testing/gtest/include/gtest/gtest.h"
  23
  24 using base::TimeDelta;
  25 using base::TimeTicks;
  26
  27 namespace net {
  28
  29 namespace {
  30
  31 const char kRequestThrottledHistogramName[] = "Throttling.RequestThrottled";
  32
  33 class MockURLRequestThrottlerEntry : public URLRequestThrottlerEntry {
  34  public:
  35   explicit MockURLRequestThrottlerEntry(
  36       URLRequestThrottlerManager* manager)
  37       : URLRequestThrottlerEntry(manager, std::string()),
  38         backoff_entry_(&backoff_policy_, &fake_clock_) {
  39     InitPolicy();
  40   }
  41   MockURLRequestThrottlerEntry(
  42       URLRequestThrottlerManager* manager,
  43       const TimeTicks& exponential_backoff_release_time,
  44       const TimeTicks& sliding_window_release_time,
  45       const TimeTicks& fake_now)
  46       : URLRequestThrottlerEntry(manager, std::string()),
  47         fake_clock_(fake_now),
  48         backoff_entry_(&backoff_policy_, &fake_clock_) {
  49     InitPolicy();
  50
  51     set_exponential_backoff_release_time(exponential_backoff_release_time);
  52     set_sliding_window_release_time(sliding_window_release_time);
  53   }
  54
  55   void InitPolicy() {
  56     // Some tests become flaky if we have jitter.
  57     backoff_policy_.jitter_factor = 0.0;
  58
  59     // This lets us avoid having to make multiple failures initially (this
  60     // logic is already tested in the BackoffEntry unit tests).
  61     backoff_policy_.num_errors_to_ignore = 0;
  62   }
  63
  64   const BackoffEntry* GetBackoffEntry() const override {
  65     return &backoff_entry_;
  66   }
  67
  68   BackoffEntry* GetBackoffEntry() override { return &backoff_entry_; }
  69
  70   static bool ExplicitUserRequest(int load_flags) {
  71     return URLRequestThrottlerEntry::ExplicitUserRequest(load_flags);
  72   }
  73
  74   void ResetToBlank(const TimeTicks& time_now) {
  75     fake_clock_.set_now(time_now);
  76
  77     GetBackoffEntry()->Reset();
  78     set_sliding_window_release_time(time_now);
  79   }
  80
  81   // Overridden for tests.
  82   TimeTicks ImplGetTimeNow() const override { return fake_clock_.NowTicks(); }
  83
  84   void set_fake_now(const TimeTicks& now) { fake_clock_.set_now(now); }
  85
  86   void set_exponential_backoff_release_time(const TimeTicks& release_time) {
  87     GetBackoffEntry()->SetCustomReleaseTime(release_time);
  88   }
  89
  90   TimeTicks sliding_window_release_time() const {
  91     return URLRequestThrottlerEntry::sliding_window_release_time();
  92   }
  93
  94   void set_sliding_window_release_time(const TimeTicks& release_time) {
  95     URLRequestThrottlerEntry::set_sliding_window_release_time(release_time);
  96   }
  97
  98  protected:
  99   ~MockURLRequestThrottlerEntry() override {}
 100
 101  private:
 102   mutable TestTickClock fake_clock_;
 103   BackoffEntry backoff_entry_;
 104 };
 105
 106 class MockURLRequestThrottlerManager : public URLRequestThrottlerManager {
 107  public:
 108   MockURLRequestThrottlerManager() : create_entry_index_(0) {}
 109
 110   // Method to process the URL using URLRequestThrottlerManager protected
 111   // method.
 112   std::string DoGetUrlIdFromUrl(const GURL& url) { return GetIdFromUrl(url); }
 113
 114   // Method to use the garbage collecting method of URLRequestThrottlerManager.
 115   void DoGarbageCollectEntries() { GarbageCollectEntries(); }
 116
 117   // Returns the number of entries in the map.
 118   int GetNumberOfEntries() const { return GetNumberOfEntriesForTests(); }
 119
 120   void CreateEntry(bool is_outdated) {
 121     TimeTicks time = TimeTicks::Now();
 122     if (is_outdated) {
 123       time -= TimeDelta::FromMilliseconds(
 124           MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs + 1000);
 125     }
 126     std::string fake_url_string("http://www.fakeurl.com/");
 127     fake_url_string.append(base::IntToString(create_entry_index_++));
 128     GURL fake_url(fake_url_string);
 129     OverrideEntryForTests(
 130         fake_url,
 131         new MockURLRequestThrottlerEntry(this, time, TimeTicks::Now(),
 132                                          TimeTicks::Now()));
 133   }
 134
 135  private:
 136   int create_entry_index_;
 137 };
 138
 139 struct TimeAndBool {
 140   TimeAndBool(const TimeTicks& time_value, bool expected, int line_num) {
 141     time = time_value;
 142     result = expected;
 143     line = line_num;
 144   }
 145   TimeTicks time;
 146   bool result;
 147   int line;
 148 };
 149
 150 struct GurlAndString {
 151   GurlAndString(const GURL& url_value,
 152                 const std::string& expected,
 153                 int line_num) {
 154     url = url_value;
 155     result = expected;
 156     line = line_num;
 157   }
 158   GURL url;
 159   std::string result;
 160   int line;
 161 };
 162
 163 }  // namespace
 164
 165 class URLRequestThrottlerEntryTest : public testing::Test {
 166  protected:
 167   URLRequestThrottlerEntryTest()
 168       : request_(context_.CreateRequest(GURL(), DEFAULT_PRIORITY, NULL)) {}
 169
 170   void SetUp() override;
 171
 172   TimeTicks now_;
 173   MockURLRequestThrottlerManager manager_;  // Dummy object, not used.
 174   scoped_refptr<MockURLRequestThrottlerEntry> entry_;
 175
 176   TestURLRequestContext context_;
 177   scoped_ptr<URLRequest> request_;
 178 };
 179
 180 void URLRequestThrottlerEntryTest::SetUp() {
 181   request_->SetLoadFlags(0);
 182
 183   now_ = TimeTicks::Now();
 184   entry_ = new MockURLRequestThrottlerEntry(&manager_);
 185   entry_->ResetToBlank(now_);
 186 }
 187
 188 std::ostream& operator<<(std::ostream& out, const base::TimeTicks& time) {
 189   return out << time.ToInternalValue();
 190 }
 191
 192 TEST_F(URLRequestThrottlerEntryTest, InterfaceDuringExponentialBackoff) {
 193   base::HistogramTester histogram_tester;
 194   entry_->set_exponential_backoff_release_time(
 195       entry_->ImplGetTimeNow() + TimeDelta::FromMilliseconds(1));
 196   EXPECT_TRUE(entry_->ShouldRejectRequest(*request_));
 197
 198   // Also end-to-end test the load flags exceptions.
 199   request_->SetLoadFlags(LOAD_MAYBE_USER_GESTURE);
 200   EXPECT_FALSE(entry_->ShouldRejectRequest(*request_));
 201
 202   histogram_tester.ExpectBucketCount(kRequestThrottledHistogramName, 0, 1);
 203   histogram_tester.ExpectBucketCount(kRequestThrottledHistogramName, 1, 1);
 204 }
 205
 206 TEST_F(URLRequestThrottlerEntryTest, InterfaceNotDuringExponentialBackoff) {
 207   base::HistogramTester histogram_tester;
 208   entry_->set_exponential_backoff_release_time(entry_->ImplGetTimeNow());
 209   EXPECT_FALSE(entry_->ShouldRejectRequest(*request_));
 210   entry_->set_exponential_backoff_release_time(
 211       entry_->ImplGetTimeNow() - TimeDelta::FromMilliseconds(1));
 212   EXPECT_FALSE(entry_->ShouldRejectRequest(*request_));
 213
 214   histogram_tester.ExpectBucketCount(kRequestThrottledHistogramName, 0, 2);
 215   histogram_tester.ExpectBucketCount(kRequestThrottledHistogramName, 1, 0);
 216 }
 217
 218 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateFailure) {
 219   entry_->UpdateWithResponse(503);
 220   EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(),
 221             entry_->ImplGetTimeNow())
 222       << "A failure should increase the release_time";
 223 }
 224
 225 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccess) {
 226   entry_->UpdateWithResponse(200);
 227   EXPECT_EQ(entry_->GetExponentialBackoffReleaseTime(),
 228             entry_->ImplGetTimeNow())
 229       << "A success should not add any delay";
 230 }
 231
 232 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccessThenFailure) {
 233   entry_->UpdateWithResponse(200);
 234   entry_->UpdateWithResponse(503);
 235   EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(),
 236             entry_->ImplGetTimeNow())
 237       << "This scenario should add delay";
 238   entry_->UpdateWithResponse(200);
 239 }
 240
 241 TEST_F(URLRequestThrottlerEntryTest, IsEntryReallyOutdated) {
 242   TimeDelta lifetime = TimeDelta::FromMilliseconds(
 243       MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs);
 244   const TimeDelta kFiveMs = TimeDelta::FromMilliseconds(5);
 245
 246   TimeAndBool test_values[] = {
 247       TimeAndBool(now_, false, __LINE__),
 248       TimeAndBool(now_ - kFiveMs, false, __LINE__),
 249       TimeAndBool(now_ + kFiveMs, false, __LINE__),
 250       TimeAndBool(now_ - (lifetime - kFiveMs), false, __LINE__),
 251       TimeAndBool(now_ - lifetime, true, __LINE__),
 252       TimeAndBool(now_ - (lifetime + kFiveMs), true, __LINE__)};
 253
 254   for (unsigned int i = 0; i < arraysize(test_values); ++i) {
 255     entry_->set_exponential_backoff_release_time(test_values[i].time);
 256     EXPECT_EQ(entry_->IsEntryOutdated(), test_values[i].result) <<
 257         "Test case #" << i << " line " << test_values[i].line << " failed";
 258   }
 259 }
 260
 261 TEST_F(URLRequestThrottlerEntryTest, MaxAllowedBackoff) {
 262   for (int i = 0; i < 30; ++i) {
 263     entry_->UpdateWithResponse(503);
 264   }
 265
 266   TimeDelta delay = entry_->GetExponentialBackoffReleaseTime() - now_;
 267   EXPECT_EQ(delay.InMilliseconds(),
 268             MockURLRequestThrottlerEntry::kDefaultMaximumBackoffMs);
 269 }
 270
 271 TEST_F(URLRequestThrottlerEntryTest, MalformedContent) {
 272   for (int i = 0; i < 5; ++i)
 273     entry_->UpdateWithResponse(503);
 274
 275   TimeTicks release_after_failures = entry_->GetExponentialBackoffReleaseTime();
 276
 277   // Inform the entry that a response body was malformed, which is supposed to
 278   // increase the back-off time.  Note that we also submit a successful
 279   // UpdateWithResponse to pair with ReceivedContentWasMalformed() since that
 280   // is what happens in practice (if a body is received, then a non-500
 281   // response must also have been received).
 282   entry_->ReceivedContentWasMalformed(200);
 283   entry_->UpdateWithResponse(200);
 284   EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), release_after_failures);
 285 }
 286
 287 TEST_F(URLRequestThrottlerEntryTest, SlidingWindow) {
 288   int max_send = URLRequestThrottlerEntry::kDefaultMaxSendThreshold;
 289   int sliding_window =
 290       URLRequestThrottlerEntry::kDefaultSlidingWindowPeriodMs;
 291
 292   TimeTicks time_1 = entry_->ImplGetTimeNow() +
 293       TimeDelta::FromMilliseconds(sliding_window / 3);
 294   TimeTicks time_2 = entry_->ImplGetTimeNow() +
 295       TimeDelta::FromMilliseconds(2 * sliding_window / 3);
 296   TimeTicks time_3 = entry_->ImplGetTimeNow() +
 297       TimeDelta::FromMilliseconds(sliding_window);
 298   TimeTicks time_4 = entry_->ImplGetTimeNow() +
 299       TimeDelta::FromMilliseconds(sliding_window + 2 * sliding_window / 3);
 300
 301   entry_->set_exponential_backoff_release_time(time_1);
 302
 303   for (int i = 0; i < max_send / 2; ++i) {
 304     EXPECT_EQ(2 * sliding_window / 3,
 305               entry_->ReserveSendingTimeForNextRequest(time_2));
 306   }
 307   EXPECT_EQ(time_2, entry_->sliding_window_release_time());
 308
 309   entry_->set_fake_now(time_3);
 310
 311   for (int i = 0; i < (max_send + 1) / 2; ++i)
 312     EXPECT_EQ(0, entry_->ReserveSendingTimeForNextRequest(TimeTicks()));
 313
 314   EXPECT_EQ(time_4, entry_->sliding_window_release_time());
 315 }
 316
 317 TEST_F(URLRequestThrottlerEntryTest, ExplicitUserRequest) {
 318   ASSERT_FALSE(MockURLRequestThrottlerEntry::ExplicitUserRequest(0));
 319   ASSERT_TRUE(MockURLRequestThrottlerEntry::ExplicitUserRequest(
 320       LOAD_MAYBE_USER_GESTURE));
 321   ASSERT_FALSE(MockURLRequestThrottlerEntry::ExplicitUserRequest(
 322       ~LOAD_MAYBE_USER_GESTURE));
 323 }
 324
 325 class URLRequestThrottlerManagerTest : public testing::Test {
 326  protected:
 327   URLRequestThrottlerManagerTest()
 328       : request_(context_.CreateRequest(GURL(), DEFAULT_PRIORITY, NULL)) {}
 329
 330   void SetUp() override { request_->SetLoadFlags(0); }
 331
 332   void ExpectEntryAllowsAllOnErrorIfOptedOut(
 333       URLRequestThrottlerEntryInterface* entry,
 334       bool opted_out,
 335       const URLRequest& request) {
 336     EXPECT_FALSE(entry->ShouldRejectRequest(request));
 337     for (int i = 0; i < 10; ++i) {
 338       entry->UpdateWithResponse(503);
 339     }
 340     EXPECT_NE(opted_out, entry->ShouldRejectRequest(request));
 341
 342     if (opted_out) {
 343       // We're not mocking out GetTimeNow() in this scenario
 344       // so add a 100 ms buffer to avoid flakiness (that should always
 345       // give enough time to get from the TimeTicks::Now() call here
 346       // to the TimeTicks::Now() call in the entry class).
 347       EXPECT_GT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100),
 348                 entry->GetExponentialBackoffReleaseTime());
 349     } else {
 350       // As above, add 100 ms.
 351       EXPECT_LT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100),
 352                 entry->GetExponentialBackoffReleaseTime());
 353     }
 354   }
 355
 356   // context_ must be declared before request_.
 357   TestURLRequestContext context_;
 358   scoped_ptr<URLRequest> request_;
 359 };
 360
 361 TEST_F(URLRequestThrottlerManagerTest, IsUrlStandardised) {
 362   MockURLRequestThrottlerManager manager;
 363   GurlAndString test_values[] = {
 364       GurlAndString(GURL("http://www.example.com"),
 365                     std::string("http://www.example.com/"),
 366                     __LINE__),
 367       GurlAndString(GURL("http://www.Example.com"),
 368                     std::string("http://www.example.com/"),
 369                     __LINE__),
 370       GurlAndString(GURL("http://www.ex4mple.com/Pr4c71c41"),
 371                     std::string("http://www.ex4mple.com/pr4c71c41"),
 372                     __LINE__),
 373       GurlAndString(GURL("http://www.example.com/0/token/false"),
 374                     std::string("http://www.example.com/0/token/false"),
 375                     __LINE__),
 376       GurlAndString(GURL("http://www.example.com/index.php?code=javascript"),
 377                     std::string("http://www.example.com/index.php"),
 378                     __LINE__),
 379       GurlAndString(GURL("http://www.example.com/index.php?code=1#superEntry"),
 380                     std::string("http://www.example.com/index.php"),
 381                     __LINE__),
 382       GurlAndString(GURL("http://www.example.com/index.php#superEntry"),
 383                     std::string("http://www.example.com/index.php"),
 384                     __LINE__),
 385       GurlAndString(GURL("http://www.example.com:1234/"),
 386                     std::string("http://www.example.com:1234/"),
 387                     __LINE__)};
 388
 389   for (unsigned int i = 0; i < arraysize(test_values); ++i) {
 390     std::string temp = manager.DoGetUrlIdFromUrl(test_values[i].url);
 391     EXPECT_EQ(temp, test_values[i].result) <<
 392         "Test case #" << i << " line " << test_values[i].line << " failed";
 393   }
 394 }
 395
 396 TEST_F(URLRequestThrottlerManagerTest, AreEntriesBeingCollected) {
 397   MockURLRequestThrottlerManager manager;
 398
 399   manager.CreateEntry(true);  // true = Entry is outdated.
 400   manager.CreateEntry(true);
 401   manager.CreateEntry(true);
 402   manager.DoGarbageCollectEntries();
 403   EXPECT_EQ(0, manager.GetNumberOfEntries());
 404
 405   manager.CreateEntry(false);
 406   manager.CreateEntry(false);
 407   manager.CreateEntry(false);
 408   manager.CreateEntry(true);
 409   manager.DoGarbageCollectEntries();
 410   EXPECT_EQ(3, manager.GetNumberOfEntries());
 411 }
 412
 413 TEST_F(URLRequestThrottlerManagerTest, IsHostBeingRegistered) {
 414   MockURLRequestThrottlerManager manager;
 415
 416   manager.RegisterRequestUrl(GURL("http://www.example.com/"));
 417   manager.RegisterRequestUrl(GURL("http://www.google.com/"));
 418   manager.RegisterRequestUrl(GURL("http://www.google.com/index/0"));
 419   manager.RegisterRequestUrl(GURL("http://www.google.com/index/0?code=1"));
 420   manager.RegisterRequestUrl(GURL("http://www.google.com/index/0#lolsaure"));
 421
 422   EXPECT_EQ(3, manager.GetNumberOfEntries());
 423 }
 424
 425 TEST_F(URLRequestThrottlerManagerTest, LocalHostOptedOut) {
 426   MockURLRequestThrottlerManager manager;
 427   // A localhost entry should always be opted out.
 428   scoped_refptr<URLRequestThrottlerEntryInterface> localhost_entry =
 429       manager.RegisterRequestUrl(GURL("http://localhost/hello"));
 430   EXPECT_FALSE(localhost_entry->ShouldRejectRequest(*request_));
 431   for (int i = 0; i < 10; ++i) {
 432     localhost_entry->UpdateWithResponse(503);
 433   }
 434   EXPECT_FALSE(localhost_entry->ShouldRejectRequest(*request_));
 435
 436   // We're not mocking out GetTimeNow() in this scenario
 437   // so add a 100 ms buffer to avoid flakiness (that should always
 438   // give enough time to get from the TimeTicks::Now() call here
 439   // to the TimeTicks::Now() call in the entry class).
 440   EXPECT_GT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100),
 441             localhost_entry->GetExponentialBackoffReleaseTime());
 442 }
 443
 444 TEST_F(URLRequestThrottlerManagerTest, ClearOnNetworkChange) {
 445   for (int i = 0; i < 3; ++i) {
 446     MockURLRequestThrottlerManager manager;
 447     scoped_refptr<URLRequestThrottlerEntryInterface> entry_before =
 448         manager.RegisterRequestUrl(GURL("http://www.example.com/"));
 449     for (int j = 0; j < 10; ++j) {
 450       entry_before->UpdateWithResponse(503);
 451     }
 452     EXPECT_TRUE(entry_before->ShouldRejectRequest(*request_));
 453
 454     switch (i) {
 455       case 0:
 456         manager.OnIPAddressChanged();
 457         break;
 458       case 1:
 459         manager.OnConnectionTypeChanged(
 460             NetworkChangeNotifier::CONNECTION_UNKNOWN);
 461         break;
 462       case 2:
 463         manager.OnConnectionTypeChanged(NetworkChangeNotifier::CONNECTION_NONE);
 464         break;
 465       default:
 466         FAIL();
 467     }
 468
 469     scoped_refptr<URLRequestThrottlerEntryInterface> entry_after =
 470         manager.RegisterRequestUrl(GURL("http://www.example.com/"));
 471     EXPECT_FALSE(entry_after->ShouldRejectRequest(*request_));
 472   }
 473 }
 474
 475 }  // namespace net