1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "net/url_request/url_request_throttler_manager.h"
7 #include "base/memory/scoped_ptr.h"
8 #include "base/metrics/histogram_samples.h"
9 #include "base/pickle.h"
10 #include "base/stl_util.h"
11 #include "base/strings/string_number_conversions.h"
12 #include "base/strings/stringprintf.h"
13 #include "base/test/histogram_tester.h"
14 #include "base/time/time.h"
15 #include "net/base/load_flags.h"
16 #include "net/base/request_priority.h"
17 #include "net/base/test_completion_callback.h"
18 #include "net/url_request/url_request.h"
19 #include "net/url_request/url_request_context.h"
20 #include "net/url_request/url_request_test_util.h"
21 #include "net/url_request/url_request_throttler_header_interface.h"
22 #include "net/url_request/url_request_throttler_test_support.h"
23 #include "testing/gtest/include/gtest/gtest.h"
25 using base::TimeDelta
;
26 using base::TimeTicks
;
32 const char kRequestThrottledHistogramName
[] = "Throttling.RequestThrottled";
34 class MockURLRequestThrottlerEntry
: public URLRequestThrottlerEntry
{
36 explicit MockURLRequestThrottlerEntry(
37 URLRequestThrottlerManager
* manager
)
38 : URLRequestThrottlerEntry(manager
, std::string()),
39 backoff_entry_(&backoff_policy_
, &fake_clock_
) {
42 MockURLRequestThrottlerEntry(
43 URLRequestThrottlerManager
* manager
,
44 const TimeTicks
& exponential_backoff_release_time
,
45 const TimeTicks
& sliding_window_release_time
,
46 const TimeTicks
& fake_now
)
47 : URLRequestThrottlerEntry(manager
, std::string()),
48 fake_clock_(fake_now
),
49 backoff_entry_(&backoff_policy_
, &fake_clock_
) {
52 set_exponential_backoff_release_time(exponential_backoff_release_time
);
53 set_sliding_window_release_time(sliding_window_release_time
);
57 // Some tests become flaky if we have jitter.
58 backoff_policy_
.jitter_factor
= 0.0;
60 // This lets us avoid having to make multiple failures initially (this
61 // logic is already tested in the BackoffEntry unit tests).
62 backoff_policy_
.num_errors_to_ignore
= 0;
65 const BackoffEntry
* GetBackoffEntry() const override
{
66 return &backoff_entry_
;
69 BackoffEntry
* GetBackoffEntry() override
{ return &backoff_entry_
; }
71 static bool ExplicitUserRequest(int load_flags
) {
72 return URLRequestThrottlerEntry::ExplicitUserRequest(load_flags
);
75 void ResetToBlank(const TimeTicks
& time_now
) {
76 fake_clock_
.set_now(time_now
);
78 GetBackoffEntry()->Reset();
79 set_sliding_window_release_time(time_now
);
82 // Overridden for tests.
83 TimeTicks
ImplGetTimeNow() const override
{ return fake_clock_
.NowTicks(); }
85 void set_fake_now(const TimeTicks
& now
) { fake_clock_
.set_now(now
); }
87 void set_exponential_backoff_release_time(const TimeTicks
& release_time
) {
88 GetBackoffEntry()->SetCustomReleaseTime(release_time
);
91 TimeTicks
sliding_window_release_time() const {
92 return URLRequestThrottlerEntry::sliding_window_release_time();
95 void set_sliding_window_release_time(const TimeTicks
& release_time
) {
96 URLRequestThrottlerEntry::set_sliding_window_release_time(release_time
);
100 ~MockURLRequestThrottlerEntry() override
{}
103 mutable TestTickClock fake_clock_
;
104 BackoffEntry backoff_entry_
;
107 class MockURLRequestThrottlerManager
: public URLRequestThrottlerManager
{
109 MockURLRequestThrottlerManager() : create_entry_index_(0) {}
111 // Method to process the URL using URLRequestThrottlerManager protected
113 std::string
DoGetUrlIdFromUrl(const GURL
& url
) { return GetIdFromUrl(url
); }
115 // Method to use the garbage collecting method of URLRequestThrottlerManager.
116 void DoGarbageCollectEntries() { GarbageCollectEntries(); }
118 // Returns the number of entries in the map.
119 int GetNumberOfEntries() const { return GetNumberOfEntriesForTests(); }
121 void CreateEntry(bool is_outdated
) {
122 TimeTicks time
= TimeTicks::Now();
124 time
-= TimeDelta::FromMilliseconds(
125 MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs
+ 1000);
127 std::string
fake_url_string("http://www.fakeurl.com/");
128 fake_url_string
.append(base::IntToString(create_entry_index_
++));
129 GURL
fake_url(fake_url_string
);
130 OverrideEntryForTests(
132 new MockURLRequestThrottlerEntry(this, time
, TimeTicks::Now(),
137 int create_entry_index_
;
141 TimeAndBool(const TimeTicks
& time_value
, bool expected
, int line_num
) {
151 struct GurlAndString
{
152 GurlAndString(const GURL
& url_value
,
153 const std::string
& expected
,
166 class URLRequestThrottlerEntryTest
: public testing::Test
{
168 URLRequestThrottlerEntryTest()
169 : request_(context_
.CreateRequest(GURL(), DEFAULT_PRIORITY
, NULL
)) {}
171 void SetUp() override
;
174 MockURLRequestThrottlerManager manager_
; // Dummy object, not used.
175 scoped_refptr
<MockURLRequestThrottlerEntry
> entry_
;
177 TestURLRequestContext context_
;
178 scoped_ptr
<URLRequest
> request_
;
181 void URLRequestThrottlerEntryTest::SetUp() {
182 request_
->SetLoadFlags(0);
184 now_
= TimeTicks::Now();
185 entry_
= new MockURLRequestThrottlerEntry(&manager_
);
186 entry_
->ResetToBlank(now_
);
189 std::ostream
& operator<<(std::ostream
& out
, const base::TimeTicks
& time
) {
190 return out
<< time
.ToInternalValue();
193 TEST_F(URLRequestThrottlerEntryTest
, CanThrottleRequest
) {
194 TestNetworkDelegate d
;
195 context_
.set_network_delegate(&d
);
196 entry_
->set_exponential_backoff_release_time(
197 entry_
->ImplGetTimeNow() + TimeDelta::FromMilliseconds(1));
199 d
.set_can_throttle_requests(false);
200 EXPECT_FALSE(entry_
->ShouldRejectRequest(*request_
,
201 context_
.network_delegate()));
202 d
.set_can_throttle_requests(true);
203 EXPECT_TRUE(entry_
->ShouldRejectRequest(*request_
,
204 context_
.network_delegate()));
207 TEST_F(URLRequestThrottlerEntryTest
, InterfaceDuringExponentialBackoff
) {
208 base::HistogramTester histogram_tester
;
209 entry_
->set_exponential_backoff_release_time(
210 entry_
->ImplGetTimeNow() + TimeDelta::FromMilliseconds(1));
211 EXPECT_TRUE(entry_
->ShouldRejectRequest(*request_
,
212 context_
.network_delegate()));
214 // Also end-to-end test the load flags exceptions.
215 request_
->SetLoadFlags(LOAD_MAYBE_USER_GESTURE
);
216 EXPECT_FALSE(entry_
->ShouldRejectRequest(*request_
,
217 context_
.network_delegate()));
219 histogram_tester
.ExpectBucketCount(kRequestThrottledHistogramName
, 0, 1);
220 histogram_tester
.ExpectBucketCount(kRequestThrottledHistogramName
, 1, 1);
223 TEST_F(URLRequestThrottlerEntryTest
, InterfaceNotDuringExponentialBackoff
) {
224 base::HistogramTester histogram_tester
;
225 entry_
->set_exponential_backoff_release_time(entry_
->ImplGetTimeNow());
226 EXPECT_FALSE(entry_
->ShouldRejectRequest(*request_
,
227 context_
.network_delegate()));
228 entry_
->set_exponential_backoff_release_time(
229 entry_
->ImplGetTimeNow() - TimeDelta::FromMilliseconds(1));
230 EXPECT_FALSE(entry_
->ShouldRejectRequest(*request_
,
231 context_
.network_delegate()));
233 histogram_tester
.ExpectBucketCount(kRequestThrottledHistogramName
, 0, 2);
234 histogram_tester
.ExpectBucketCount(kRequestThrottledHistogramName
, 1, 0);
237 TEST_F(URLRequestThrottlerEntryTest
, InterfaceUpdateFailure
) {
238 MockURLRequestThrottlerHeaderAdapter
failure_response(503);
239 entry_
->UpdateWithResponse(std::string(), &failure_response
);
240 EXPECT_GT(entry_
->GetExponentialBackoffReleaseTime(),
241 entry_
->ImplGetTimeNow())
242 << "A failure should increase the release_time";
245 TEST_F(URLRequestThrottlerEntryTest
, InterfaceUpdateSuccess
) {
246 MockURLRequestThrottlerHeaderAdapter
success_response(200);
247 entry_
->UpdateWithResponse(std::string(), &success_response
);
248 EXPECT_EQ(entry_
->GetExponentialBackoffReleaseTime(),
249 entry_
->ImplGetTimeNow())
250 << "A success should not add any delay";
253 TEST_F(URLRequestThrottlerEntryTest
, InterfaceUpdateSuccessThenFailure
) {
254 MockURLRequestThrottlerHeaderAdapter
failure_response(503);
255 MockURLRequestThrottlerHeaderAdapter
success_response(200);
256 entry_
->UpdateWithResponse(std::string(), &success_response
);
257 entry_
->UpdateWithResponse(std::string(), &failure_response
);
258 EXPECT_GT(entry_
->GetExponentialBackoffReleaseTime(),
259 entry_
->ImplGetTimeNow())
260 << "This scenario should add delay";
261 entry_
->UpdateWithResponse(std::string(), &success_response
);
264 TEST_F(URLRequestThrottlerEntryTest
, IsEntryReallyOutdated
) {
265 TimeDelta lifetime
= TimeDelta::FromMilliseconds(
266 MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs
);
267 const TimeDelta kFiveMs
= TimeDelta::FromMilliseconds(5);
269 TimeAndBool test_values
[] = {
270 TimeAndBool(now_
, false, __LINE__
),
271 TimeAndBool(now_
- kFiveMs
, false, __LINE__
),
272 TimeAndBool(now_
+ kFiveMs
, false, __LINE__
),
273 TimeAndBool(now_
- (lifetime
- kFiveMs
), false, __LINE__
),
274 TimeAndBool(now_
- lifetime
, true, __LINE__
),
275 TimeAndBool(now_
- (lifetime
+ kFiveMs
), true, __LINE__
)};
277 for (unsigned int i
= 0; i
< arraysize(test_values
); ++i
) {
278 entry_
->set_exponential_backoff_release_time(test_values
[i
].time
);
279 EXPECT_EQ(entry_
->IsEntryOutdated(), test_values
[i
].result
) <<
280 "Test case #" << i
<< " line " << test_values
[i
].line
<< " failed";
284 TEST_F(URLRequestThrottlerEntryTest
, MaxAllowedBackoff
) {
285 for (int i
= 0; i
< 30; ++i
) {
286 MockURLRequestThrottlerHeaderAdapter
response_adapter(503);
287 entry_
->UpdateWithResponse(std::string(), &response_adapter
);
290 TimeDelta delay
= entry_
->GetExponentialBackoffReleaseTime() - now_
;
291 EXPECT_EQ(delay
.InMilliseconds(),
292 MockURLRequestThrottlerEntry::kDefaultMaximumBackoffMs
);
295 TEST_F(URLRequestThrottlerEntryTest
, MalformedContent
) {
296 MockURLRequestThrottlerHeaderAdapter
response_adapter(503);
297 for (int i
= 0; i
< 5; ++i
)
298 entry_
->UpdateWithResponse(std::string(), &response_adapter
);
300 TimeTicks release_after_failures
= entry_
->GetExponentialBackoffReleaseTime();
302 // Inform the entry that a response body was malformed, which is supposed to
303 // increase the back-off time. Note that we also submit a successful
304 // UpdateWithResponse to pair with ReceivedContentWasMalformed() since that
305 // is what happens in practice (if a body is received, then a non-500
306 // response must also have been received).
307 entry_
->ReceivedContentWasMalformed(200);
308 MockURLRequestThrottlerHeaderAdapter
success_adapter(200);
309 entry_
->UpdateWithResponse(std::string(), &success_adapter
);
310 EXPECT_GT(entry_
->GetExponentialBackoffReleaseTime(), release_after_failures
);
313 TEST_F(URLRequestThrottlerEntryTest
, SlidingWindow
) {
314 int max_send
= URLRequestThrottlerEntry::kDefaultMaxSendThreshold
;
316 URLRequestThrottlerEntry::kDefaultSlidingWindowPeriodMs
;
318 TimeTicks time_1
= entry_
->ImplGetTimeNow() +
319 TimeDelta::FromMilliseconds(sliding_window
/ 3);
320 TimeTicks time_2
= entry_
->ImplGetTimeNow() +
321 TimeDelta::FromMilliseconds(2 * sliding_window
/ 3);
322 TimeTicks time_3
= entry_
->ImplGetTimeNow() +
323 TimeDelta::FromMilliseconds(sliding_window
);
324 TimeTicks time_4
= entry_
->ImplGetTimeNow() +
325 TimeDelta::FromMilliseconds(sliding_window
+ 2 * sliding_window
/ 3);
327 entry_
->set_exponential_backoff_release_time(time_1
);
329 for (int i
= 0; i
< max_send
/ 2; ++i
) {
330 EXPECT_EQ(2 * sliding_window
/ 3,
331 entry_
->ReserveSendingTimeForNextRequest(time_2
));
333 EXPECT_EQ(time_2
, entry_
->sliding_window_release_time());
335 entry_
->set_fake_now(time_3
);
337 for (int i
= 0; i
< (max_send
+ 1) / 2; ++i
)
338 EXPECT_EQ(0, entry_
->ReserveSendingTimeForNextRequest(TimeTicks()));
340 EXPECT_EQ(time_4
, entry_
->sliding_window_release_time());
343 TEST_F(URLRequestThrottlerEntryTest
, ExplicitUserRequest
) {
344 ASSERT_FALSE(MockURLRequestThrottlerEntry::ExplicitUserRequest(0));
345 ASSERT_TRUE(MockURLRequestThrottlerEntry::ExplicitUserRequest(
346 LOAD_MAYBE_USER_GESTURE
));
347 ASSERT_FALSE(MockURLRequestThrottlerEntry::ExplicitUserRequest(
348 ~LOAD_MAYBE_USER_GESTURE
));
351 class URLRequestThrottlerManagerTest
: public testing::Test
{
353 URLRequestThrottlerManagerTest()
354 : request_(context_
.CreateRequest(GURL(), DEFAULT_PRIORITY
, NULL
)) {}
356 void SetUp() override
{ request_
->SetLoadFlags(0); }
358 void ExpectEntryAllowsAllOnErrorIfOptedOut(
359 URLRequestThrottlerEntryInterface
* entry
,
361 const URLRequest
& request
) {
362 EXPECT_FALSE(entry
->ShouldRejectRequest(request
,
363 context_
.network_delegate()));
364 MockURLRequestThrottlerHeaderAdapter
failure_adapter(503);
365 for (int i
= 0; i
< 10; ++i
) {
366 // Host doesn't really matter in this scenario so we skip it.
367 entry
->UpdateWithResponse(std::string(), &failure_adapter
);
369 EXPECT_NE(opted_out
, entry
->ShouldRejectRequest(
370 request
, context_
.network_delegate()));
373 // We're not mocking out GetTimeNow() in this scenario
374 // so add a 100 ms buffer to avoid flakiness (that should always
375 // give enough time to get from the TimeTicks::Now() call here
376 // to the TimeTicks::Now() call in the entry class).
377 EXPECT_GT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100),
378 entry
->GetExponentialBackoffReleaseTime());
380 // As above, add 100 ms.
381 EXPECT_LT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100),
382 entry
->GetExponentialBackoffReleaseTime());
386 // context_ must be declared before request_.
387 TestURLRequestContext context_
;
388 scoped_ptr
<URLRequest
> request_
;
391 TEST_F(URLRequestThrottlerManagerTest
, IsUrlStandardised
) {
392 MockURLRequestThrottlerManager manager
;
393 GurlAndString test_values
[] = {
394 GurlAndString(GURL("http://www.example.com"),
395 std::string("http://www.example.com/"),
397 GurlAndString(GURL("http://www.Example.com"),
398 std::string("http://www.example.com/"),
400 GurlAndString(GURL("http://www.ex4mple.com/Pr4c71c41"),
401 std::string("http://www.ex4mple.com/pr4c71c41"),
403 GurlAndString(GURL("http://www.example.com/0/token/false"),
404 std::string("http://www.example.com/0/token/false"),
406 GurlAndString(GURL("http://www.example.com/index.php?code=javascript"),
407 std::string("http://www.example.com/index.php"),
409 GurlAndString(GURL("http://www.example.com/index.php?code=1#superEntry"),
410 std::string("http://www.example.com/index.php"),
412 GurlAndString(GURL("http://www.example.com/index.php#superEntry"),
413 std::string("http://www.example.com/index.php"),
415 GurlAndString(GURL("http://www.example.com:1234/"),
416 std::string("http://www.example.com:1234/"),
419 for (unsigned int i
= 0; i
< arraysize(test_values
); ++i
) {
420 std::string temp
= manager
.DoGetUrlIdFromUrl(test_values
[i
].url
);
421 EXPECT_EQ(temp
, test_values
[i
].result
) <<
422 "Test case #" << i
<< " line " << test_values
[i
].line
<< " failed";
426 TEST_F(URLRequestThrottlerManagerTest
, AreEntriesBeingCollected
) {
427 MockURLRequestThrottlerManager manager
;
429 manager
.CreateEntry(true); // true = Entry is outdated.
430 manager
.CreateEntry(true);
431 manager
.CreateEntry(true);
432 manager
.DoGarbageCollectEntries();
433 EXPECT_EQ(0, manager
.GetNumberOfEntries());
435 manager
.CreateEntry(false);
436 manager
.CreateEntry(false);
437 manager
.CreateEntry(false);
438 manager
.CreateEntry(true);
439 manager
.DoGarbageCollectEntries();
440 EXPECT_EQ(3, manager
.GetNumberOfEntries());
443 TEST_F(URLRequestThrottlerManagerTest
, IsHostBeingRegistered
) {
444 MockURLRequestThrottlerManager manager
;
446 manager
.RegisterRequestUrl(GURL("http://www.example.com/"));
447 manager
.RegisterRequestUrl(GURL("http://www.google.com/"));
448 manager
.RegisterRequestUrl(GURL("http://www.google.com/index/0"));
449 manager
.RegisterRequestUrl(GURL("http://www.google.com/index/0?code=1"));
450 manager
.RegisterRequestUrl(GURL("http://www.google.com/index/0#lolsaure"));
452 EXPECT_EQ(3, manager
.GetNumberOfEntries());
455 TEST_F(URLRequestThrottlerManagerTest
, OptOutHeader
) {
456 MockURLRequestThrottlerManager manager
;
457 scoped_refptr
<URLRequestThrottlerEntryInterface
> entry
=
458 manager
.RegisterRequestUrl(GURL("http://www.google.com/yodude"));
460 // Fake a response with the opt-out header.
461 MockURLRequestThrottlerHeaderAdapter
response_adapter(
463 MockURLRequestThrottlerEntry::kExponentialThrottlingDisableValue
,
465 entry
->UpdateWithResponse("www.google.com", &response_adapter
);
467 // Ensure that the same entry on error always allows everything.
468 ExpectEntryAllowsAllOnErrorIfOptedOut(entry
.get(), true, *request_
);
470 // Ensure that a freshly created entry (for a different URL on an
471 // already opted-out host) also gets "always allow" behavior.
472 scoped_refptr
<URLRequestThrottlerEntryInterface
> other_entry
=
473 manager
.RegisterRequestUrl(GURL("http://www.google.com/bingobob"));
474 ExpectEntryAllowsAllOnErrorIfOptedOut(other_entry
.get(), true, *request_
);
476 // Fake a response with the opt-out header incorrectly specified.
477 scoped_refptr
<URLRequestThrottlerEntryInterface
> no_opt_out_entry
=
478 manager
.RegisterRequestUrl(GURL("http://www.nike.com/justdoit"));
479 MockURLRequestThrottlerHeaderAdapter
wrong_adapter(
480 std::string(), "yesplease", 200);
481 no_opt_out_entry
->UpdateWithResponse("www.nike.com", &wrong_adapter
);
482 ExpectEntryAllowsAllOnErrorIfOptedOut(
483 no_opt_out_entry
.get(), false, *request_
);
485 // A localhost entry should always be opted out.
486 scoped_refptr
<URLRequestThrottlerEntryInterface
> localhost_entry
=
487 manager
.RegisterRequestUrl(GURL("http://localhost/hello"));
488 ExpectEntryAllowsAllOnErrorIfOptedOut(localhost_entry
.get(), true, *request_
);
491 TEST_F(URLRequestThrottlerManagerTest
, ClearOnNetworkChange
) {
492 for (int i
= 0; i
< 3; ++i
) {
493 MockURLRequestThrottlerManager manager
;
494 scoped_refptr
<URLRequestThrottlerEntryInterface
> entry_before
=
495 manager
.RegisterRequestUrl(GURL("http://www.example.com/"));
496 MockURLRequestThrottlerHeaderAdapter
failure_adapter(503);
497 for (int j
= 0; j
< 10; ++j
) {
498 // Host doesn't really matter in this scenario so we skip it.
499 entry_before
->UpdateWithResponse(std::string(), &failure_adapter
);
501 EXPECT_TRUE(entry_before
->ShouldRejectRequest(*request_
,
502 context_
.network_delegate()));
506 manager
.OnIPAddressChanged();
509 manager
.OnConnectionTypeChanged(
510 NetworkChangeNotifier::CONNECTION_UNKNOWN
);
513 manager
.OnConnectionTypeChanged(NetworkChangeNotifier::CONNECTION_NONE
);
519 scoped_refptr
<URLRequestThrottlerEntryInterface
> entry_after
=
520 manager
.RegisterRequestUrl(GURL("http://www.example.com/"));
521 EXPECT_FALSE(entry_after
->ShouldRejectRequest(
522 *request_
, context_
.network_delegate()));