net/url_request/url_request_throttler_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_request_throttler_manager.h"
   6
   7 #include "base/memory/scoped_ptr.h"
   8 #include "base/metrics/histogram_samples.h"
   9 #include "base/pickle.h"
  10 #include "base/stl_util.h"
  11 #include "base/strings/string_number_conversions.h"
  12 #include "base/strings/stringprintf.h"
  13 #include "base/test/histogram_tester.h"
  14 #include "base/time/time.h"
  15 #include "net/base/load_flags.h"
  16 #include "net/base/request_priority.h"
  17 #include "net/base/test_completion_callback.h"
  18 #include "net/url_request/url_request.h"
  19 #include "net/url_request/url_request_context.h"
  20 #include "net/url_request/url_request_test_util.h"
  21 #include "net/url_request/url_request_throttler_test_support.h"
  22 #include "testing/gtest/include/gtest/gtest.h"
  23
  24 using base::TimeDelta;
  25 using base::TimeTicks;
  26
  27 namespace net {
  28
  29 namespace {
  30
  31 const char kRequestThrottledHistogramName[] = "Throttling.RequestThrottled";
  32
  33 class MockURLRequestThrottlerEntry : public URLRequestThrottlerEntry {
  34  public:
  35   explicit MockURLRequestThrottlerEntry(
  36       URLRequestThrottlerManager* manager)
  37       : URLRequestThrottlerEntry(manager, std::string()),
  38         backoff_entry_(&backoff_policy_, &fake_clock_) {
  39     InitPolicy();
  40   }
  41   MockURLRequestThrottlerEntry(
  42       URLRequestThrottlerManager* manager,
  43       const TimeTicks& exponential_backoff_release_time,
  44       const TimeTicks& sliding_window_release_time,
  45       const TimeTicks& fake_now)
  46       : URLRequestThrottlerEntry(manager, std::string()),
  47         fake_clock_(fake_now),
  48         backoff_entry_(&backoff_policy_, &fake_clock_) {
  49     InitPolicy();
  50
  51     set_exponential_backoff_release_time(exponential_backoff_release_time);
  52     set_sliding_window_release_time(sliding_window_release_time);
  53   }
  54
  55   void InitPolicy() {
  56     // Some tests become flaky if we have jitter.
  57     backoff_policy_.jitter_factor = 0.0;
  58
  59     // This lets us avoid having to make multiple failures initially (this
  60     // logic is already tested in the BackoffEntry unit tests).
  61     backoff_policy_.num_errors_to_ignore = 0;
  62   }
  63
  64   const BackoffEntry* GetBackoffEntry() const override {
  65     return &backoff_entry_;
  66   }
  67
  68   BackoffEntry* GetBackoffEntry() override { return &backoff_entry_; }
  69
  70   static bool ExplicitUserRequest(int load_flags) {
  71     return URLRequestThrottlerEntry::ExplicitUserRequest(load_flags);
  72   }
  73
  74   void ResetToBlank(const TimeTicks& time_now) {
  75     fake_clock_.set_now(time_now);
  76
  77     GetBackoffEntry()->Reset();
  78     set_sliding_window_release_time(time_now);
  79   }
  80
  81   // Overridden for tests.
  82   TimeTicks ImplGetTimeNow() const override { return fake_clock_.NowTicks(); }
  83
  84   void set_fake_now(const TimeTicks& now) { fake_clock_.set_now(now); }
  85
  86   void set_exponential_backoff_release_time(const TimeTicks& release_time) {
  87     GetBackoffEntry()->SetCustomReleaseTime(release_time);
  88   }
  89
  90   TimeTicks sliding_window_release_time() const {
  91     return URLRequestThrottlerEntry::sliding_window_release_time();
  92   }
  93
  94   void set_sliding_window_release_time(const TimeTicks& release_time) {
  95     URLRequestThrottlerEntry::set_sliding_window_release_time(release_time);
  96   }
  97
  98  protected:
  99   ~MockURLRequestThrottlerEntry() override {}
 100
 101  private:
 102   mutable TestTickClock fake_clock_;
 103   BackoffEntry backoff_entry_;
 104 };
 105
 106 class MockURLRequestThrottlerManager : public URLRequestThrottlerManager {
 107  public:
 108   MockURLRequestThrottlerManager() : create_entry_index_(0) {}
 109
 110   // Method to process the URL using URLRequestThrottlerManager protected
 111   // method.
 112   std::string DoGetUrlIdFromUrl(const GURL& url) { return GetIdFromUrl(url); }
 113
 114   // Method to use the garbage collecting method of URLRequestThrottlerManager.
 115   void DoGarbageCollectEntries() { GarbageCollectEntries(); }
 116
 117   // Returns the number of entries in the map.
 118   int GetNumberOfEntries() const { return GetNumberOfEntriesForTests(); }
 119
 120   void CreateEntry(bool is_outdated) {
 121     TimeTicks time = TimeTicks::Now();
 122     if (is_outdated) {
 123       time -= TimeDelta::FromMilliseconds(
 124           MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs + 1000);
 125     }
 126     std::string fake_url_string("http://www.fakeurl.com/");
 127     fake_url_string.append(base::IntToString(create_entry_index_++));
 128     GURL fake_url(fake_url_string);
 129     OverrideEntryForTests(
 130         fake_url,
 131         new MockURLRequestThrottlerEntry(this, time, TimeTicks::Now(),
 132                                          TimeTicks::Now()));
 133   }
 134
 135  private:
 136   int create_entry_index_;
 137 };
 138
 139 struct TimeAndBool {
 140   TimeAndBool(const TimeTicks& time_value, bool expected, int line_num) {
 141     time = time_value;
 142     result = expected;
 143     line = line_num;
 144   }
 145   TimeTicks time;
 146   bool result;
 147   int line;
 148 };
 149
 150 struct GurlAndString {
 151   GurlAndString(const GURL& url_value,
 152                 const std::string& expected,
 153                 int line_num) {
 154     url = url_value;
 155     result = expected;
 156     line = line_num;
 157   }
 158   GURL url;
 159   std::string result;
 160   int line;
 161 };
 162
 163 }  // namespace
 164
 165 class URLRequestThrottlerEntryTest : public testing::Test {
 166  protected:
 167   URLRequestThrottlerEntryTest()
 168       : request_(context_.CreateRequest(GURL(), DEFAULT_PRIORITY, NULL)) {}
 169
 170   void SetUp() override;
 171
 172   TimeTicks now_;
 173   MockURLRequestThrottlerManager manager_;  // Dummy object, not used.
 174   scoped_refptr<MockURLRequestThrottlerEntry> entry_;
 175
 176   TestURLRequestContext context_;
 177   scoped_ptr<URLRequest> request_;
 178 };
 179
 180 void URLRequestThrottlerEntryTest::SetUp() {
 181   request_->SetLoadFlags(0);
 182
 183   now_ = TimeTicks::Now();
 184   entry_ = new MockURLRequestThrottlerEntry(&manager_);
 185   entry_->ResetToBlank(now_);
 186 }
 187
 188 std::ostream& operator<<(std::ostream& out, const base::TimeTicks& time) {
 189   return out << time.ToInternalValue();
 190 }
 191
 192 TEST_F(URLRequestThrottlerEntryTest, CanThrottleRequest) {
 193   TestNetworkDelegate d;
 194   context_.set_network_delegate(&d);
 195   entry_->set_exponential_backoff_release_time(
 196       entry_->ImplGetTimeNow() + TimeDelta::FromMilliseconds(1));
 197
 198   d.set_can_throttle_requests(false);
 199   EXPECT_FALSE(entry_->ShouldRejectRequest(*request_,
 200                                            context_.network_delegate()));
 201   d.set_can_throttle_requests(true);
 202   EXPECT_TRUE(entry_->ShouldRejectRequest(*request_,
 203                                           context_.network_delegate()));
 204 }
 205
 206 TEST_F(URLRequestThrottlerEntryTest, InterfaceDuringExponentialBackoff) {
 207   base::HistogramTester histogram_tester;
 208   entry_->set_exponential_backoff_release_time(
 209       entry_->ImplGetTimeNow() + TimeDelta::FromMilliseconds(1));
 210   EXPECT_TRUE(entry_->ShouldRejectRequest(*request_,
 211                                           context_.network_delegate()));
 212
 213   // Also end-to-end test the load flags exceptions.
 214   request_->SetLoadFlags(LOAD_MAYBE_USER_GESTURE);
 215   EXPECT_FALSE(entry_->ShouldRejectRequest(*request_,
 216                                            context_.network_delegate()));
 217
 218   histogram_tester.ExpectBucketCount(kRequestThrottledHistogramName, 0, 1);
 219   histogram_tester.ExpectBucketCount(kRequestThrottledHistogramName, 1, 1);
 220 }
 221
 222 TEST_F(URLRequestThrottlerEntryTest, InterfaceNotDuringExponentialBackoff) {
 223   base::HistogramTester histogram_tester;
 224   entry_->set_exponential_backoff_release_time(entry_->ImplGetTimeNow());
 225   EXPECT_FALSE(entry_->ShouldRejectRequest(*request_,
 226                                            context_.network_delegate()));
 227   entry_->set_exponential_backoff_release_time(
 228       entry_->ImplGetTimeNow() - TimeDelta::FromMilliseconds(1));
 229   EXPECT_FALSE(entry_->ShouldRejectRequest(*request_,
 230                                            context_.network_delegate()));
 231
 232   histogram_tester.ExpectBucketCount(kRequestThrottledHistogramName, 0, 2);
 233   histogram_tester.ExpectBucketCount(kRequestThrottledHistogramName, 1, 0);
 234 }
 235
 236 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateFailure) {
 237   entry_->UpdateWithResponse(503);
 238   EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(),
 239             entry_->ImplGetTimeNow())
 240       << "A failure should increase the release_time";
 241 }
 242
 243 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccess) {
 244   entry_->UpdateWithResponse(200);
 245   EXPECT_EQ(entry_->GetExponentialBackoffReleaseTime(),
 246             entry_->ImplGetTimeNow())
 247       << "A success should not add any delay";
 248 }
 249
 250 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccessThenFailure) {
 251   entry_->UpdateWithResponse(200);
 252   entry_->UpdateWithResponse(503);
 253   EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(),
 254             entry_->ImplGetTimeNow())
 255       << "This scenario should add delay";
 256   entry_->UpdateWithResponse(200);
 257 }
 258
 259 TEST_F(URLRequestThrottlerEntryTest, IsEntryReallyOutdated) {
 260   TimeDelta lifetime = TimeDelta::FromMilliseconds(
 261       MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs);
 262   const TimeDelta kFiveMs = TimeDelta::FromMilliseconds(5);
 263
 264   TimeAndBool test_values[] = {
 265       TimeAndBool(now_, false, __LINE__),
 266       TimeAndBool(now_ - kFiveMs, false, __LINE__),
 267       TimeAndBool(now_ + kFiveMs, false, __LINE__),
 268       TimeAndBool(now_ - (lifetime - kFiveMs), false, __LINE__),
 269       TimeAndBool(now_ - lifetime, true, __LINE__),
 270       TimeAndBool(now_ - (lifetime + kFiveMs), true, __LINE__)};
 271
 272   for (unsigned int i = 0; i < arraysize(test_values); ++i) {
 273     entry_->set_exponential_backoff_release_time(test_values[i].time);
 274     EXPECT_EQ(entry_->IsEntryOutdated(), test_values[i].result) <<
 275         "Test case #" << i << " line " << test_values[i].line << " failed";
 276   }
 277 }
 278
 279 TEST_F(URLRequestThrottlerEntryTest, MaxAllowedBackoff) {
 280   for (int i = 0; i < 30; ++i) {
 281     entry_->UpdateWithResponse(503);
 282   }
 283
 284   TimeDelta delay = entry_->GetExponentialBackoffReleaseTime() - now_;
 285   EXPECT_EQ(delay.InMilliseconds(),
 286             MockURLRequestThrottlerEntry::kDefaultMaximumBackoffMs);
 287 }
 288
 289 TEST_F(URLRequestThrottlerEntryTest, MalformedContent) {
 290   for (int i = 0; i < 5; ++i)
 291     entry_->UpdateWithResponse(503);
 292
 293   TimeTicks release_after_failures = entry_->GetExponentialBackoffReleaseTime();
 294
 295   // Inform the entry that a response body was malformed, which is supposed to
 296   // increase the back-off time.  Note that we also submit a successful
 297   // UpdateWithResponse to pair with ReceivedContentWasMalformed() since that
 298   // is what happens in practice (if a body is received, then a non-500
 299   // response must also have been received).
 300   entry_->ReceivedContentWasMalformed(200);
 301   entry_->UpdateWithResponse(200);
 302   EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), release_after_failures);
 303 }
 304
 305 TEST_F(URLRequestThrottlerEntryTest, SlidingWindow) {
 306   int max_send = URLRequestThrottlerEntry::kDefaultMaxSendThreshold;
 307   int sliding_window =
 308       URLRequestThrottlerEntry::kDefaultSlidingWindowPeriodMs;
 309
 310   TimeTicks time_1 = entry_->ImplGetTimeNow() +
 311       TimeDelta::FromMilliseconds(sliding_window / 3);
 312   TimeTicks time_2 = entry_->ImplGetTimeNow() +
 313       TimeDelta::FromMilliseconds(2 * sliding_window / 3);
 314   TimeTicks time_3 = entry_->ImplGetTimeNow() +
 315       TimeDelta::FromMilliseconds(sliding_window);
 316   TimeTicks time_4 = entry_->ImplGetTimeNow() +
 317       TimeDelta::FromMilliseconds(sliding_window + 2 * sliding_window / 3);
 318
 319   entry_->set_exponential_backoff_release_time(time_1);
 320
 321   for (int i = 0; i < max_send / 2; ++i) {
 322     EXPECT_EQ(2 * sliding_window / 3,
 323               entry_->ReserveSendingTimeForNextRequest(time_2));
 324   }
 325   EXPECT_EQ(time_2, entry_->sliding_window_release_time());
 326
 327   entry_->set_fake_now(time_3);
 328
 329   for (int i = 0; i < (max_send + 1) / 2; ++i)
 330     EXPECT_EQ(0, entry_->ReserveSendingTimeForNextRequest(TimeTicks()));
 331
 332   EXPECT_EQ(time_4, entry_->sliding_window_release_time());
 333 }
 334
 335 TEST_F(URLRequestThrottlerEntryTest, ExplicitUserRequest) {
 336   ASSERT_FALSE(MockURLRequestThrottlerEntry::ExplicitUserRequest(0));
 337   ASSERT_TRUE(MockURLRequestThrottlerEntry::ExplicitUserRequest(
 338       LOAD_MAYBE_USER_GESTURE));
 339   ASSERT_FALSE(MockURLRequestThrottlerEntry::ExplicitUserRequest(
 340       ~LOAD_MAYBE_USER_GESTURE));
 341 }
 342
 343 class URLRequestThrottlerManagerTest : public testing::Test {
 344  protected:
 345   URLRequestThrottlerManagerTest()
 346       : request_(context_.CreateRequest(GURL(), DEFAULT_PRIORITY, NULL)) {}
 347
 348   void SetUp() override { request_->SetLoadFlags(0); }
 349
 350   void ExpectEntryAllowsAllOnErrorIfOptedOut(
 351       URLRequestThrottlerEntryInterface* entry,
 352       bool opted_out,
 353       const URLRequest& request) {
 354     EXPECT_FALSE(entry->ShouldRejectRequest(request,
 355                                             context_.network_delegate()));
 356     for (int i = 0; i < 10; ++i) {
 357       entry->UpdateWithResponse(503);
 358     }
 359     EXPECT_NE(opted_out, entry->ShouldRejectRequest(
 360                   request, context_.network_delegate()));
 361
 362     if (opted_out) {
 363       // We're not mocking out GetTimeNow() in this scenario
 364       // so add a 100 ms buffer to avoid flakiness (that should always
 365       // give enough time to get from the TimeTicks::Now() call here
 366       // to the TimeTicks::Now() call in the entry class).
 367       EXPECT_GT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100),
 368                 entry->GetExponentialBackoffReleaseTime());
 369     } else {
 370       // As above, add 100 ms.
 371       EXPECT_LT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100),
 372                 entry->GetExponentialBackoffReleaseTime());
 373     }
 374   }
 375
 376   // context_ must be declared before request_.
 377   TestURLRequestContext context_;
 378   scoped_ptr<URLRequest> request_;
 379 };
 380
 381 TEST_F(URLRequestThrottlerManagerTest, IsUrlStandardised) {
 382   MockURLRequestThrottlerManager manager;
 383   GurlAndString test_values[] = {
 384       GurlAndString(GURL("http://www.example.com"),
 385                     std::string("http://www.example.com/"),
 386                     __LINE__),
 387       GurlAndString(GURL("http://www.Example.com"),
 388                     std::string("http://www.example.com/"),
 389                     __LINE__),
 390       GurlAndString(GURL("http://www.ex4mple.com/Pr4c71c41"),
 391                     std::string("http://www.ex4mple.com/pr4c71c41"),
 392                     __LINE__),
 393       GurlAndString(GURL("http://www.example.com/0/token/false"),
 394                     std::string("http://www.example.com/0/token/false"),
 395                     __LINE__),
 396       GurlAndString(GURL("http://www.example.com/index.php?code=javascript"),
 397                     std::string("http://www.example.com/index.php"),
 398                     __LINE__),
 399       GurlAndString(GURL("http://www.example.com/index.php?code=1#superEntry"),
 400                     std::string("http://www.example.com/index.php"),
 401                     __LINE__),
 402       GurlAndString(GURL("http://www.example.com/index.php#superEntry"),
 403                     std::string("http://www.example.com/index.php"),
 404                     __LINE__),
 405       GurlAndString(GURL("http://www.example.com:1234/"),
 406                     std::string("http://www.example.com:1234/"),
 407                     __LINE__)};
 408
 409   for (unsigned int i = 0; i < arraysize(test_values); ++i) {
 410     std::string temp = manager.DoGetUrlIdFromUrl(test_values[i].url);
 411     EXPECT_EQ(temp, test_values[i].result) <<
 412         "Test case #" << i << " line " << test_values[i].line << " failed";
 413   }
 414 }
 415
 416 TEST_F(URLRequestThrottlerManagerTest, AreEntriesBeingCollected) {
 417   MockURLRequestThrottlerManager manager;
 418
 419   manager.CreateEntry(true);  // true = Entry is outdated.
 420   manager.CreateEntry(true);
 421   manager.CreateEntry(true);
 422   manager.DoGarbageCollectEntries();
 423   EXPECT_EQ(0, manager.GetNumberOfEntries());
 424
 425   manager.CreateEntry(false);
 426   manager.CreateEntry(false);
 427   manager.CreateEntry(false);
 428   manager.CreateEntry(true);
 429   manager.DoGarbageCollectEntries();
 430   EXPECT_EQ(3, manager.GetNumberOfEntries());
 431 }
 432
 433 TEST_F(URLRequestThrottlerManagerTest, IsHostBeingRegistered) {
 434   MockURLRequestThrottlerManager manager;
 435
 436   manager.RegisterRequestUrl(GURL("http://www.example.com/"));
 437   manager.RegisterRequestUrl(GURL("http://www.google.com/"));
 438   manager.RegisterRequestUrl(GURL("http://www.google.com/index/0"));
 439   manager.RegisterRequestUrl(GURL("http://www.google.com/index/0?code=1"));
 440   manager.RegisterRequestUrl(GURL("http://www.google.com/index/0#lolsaure"));
 441
 442   EXPECT_EQ(3, manager.GetNumberOfEntries());
 443 }
 444
 445 TEST_F(URLRequestThrottlerManagerTest, LocalHostOptedOut) {
 446   MockURLRequestThrottlerManager manager;
 447   // A localhost entry should always be opted out.
 448   scoped_refptr<URLRequestThrottlerEntryInterface> localhost_entry =
 449       manager.RegisterRequestUrl(GURL("http://localhost/hello"));
 450   EXPECT_FALSE(localhost_entry->ShouldRejectRequest(
 451       (*request_), context_.network_delegate()));
 452   for (int i = 0; i < 10; ++i) {
 453     localhost_entry->UpdateWithResponse(503);
 454   }
 455   EXPECT_FALSE(localhost_entry->ShouldRejectRequest(
 456       (*request_), context_.network_delegate()));
 457
 458   // We're not mocking out GetTimeNow() in this scenario
 459   // so add a 100 ms buffer to avoid flakiness (that should always
 460   // give enough time to get from the TimeTicks::Now() call here
 461   // to the TimeTicks::Now() call in the entry class).
 462   EXPECT_GT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100),
 463             localhost_entry->GetExponentialBackoffReleaseTime());
 464 }
 465
 466 TEST_F(URLRequestThrottlerManagerTest, ClearOnNetworkChange) {
 467   for (int i = 0; i < 3; ++i) {
 468     MockURLRequestThrottlerManager manager;
 469     scoped_refptr<URLRequestThrottlerEntryInterface> entry_before =
 470         manager.RegisterRequestUrl(GURL("http://www.example.com/"));
 471     for (int j = 0; j < 10; ++j) {
 472       entry_before->UpdateWithResponse(503);
 473     }
 474     EXPECT_TRUE(entry_before->ShouldRejectRequest(*request_,
 475                                                   context_.network_delegate()));
 476
 477     switch (i) {
 478       case 0:
 479         manager.OnIPAddressChanged();
 480         break;
 481       case 1:
 482         manager.OnConnectionTypeChanged(
 483             NetworkChangeNotifier::CONNECTION_UNKNOWN);
 484         break;
 485       case 2:
 486         manager.OnConnectionTypeChanged(NetworkChangeNotifier::CONNECTION_NONE);
 487         break;
 488       default:
 489         FAIL();
 490     }
 491
 492     scoped_refptr<URLRequestThrottlerEntryInterface> entry_after =
 493         manager.RegisterRequestUrl(GURL("http://www.example.com/"));
 494     EXPECT_FALSE(entry_after->ShouldRejectRequest(
 495                      *request_, context_.network_delegate()));
 496   }
 497 }
 498
 499 }  // namespace net