1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "net/url_request/url_request_throttler_manager.h"
7 #include "base/memory/scoped_ptr.h"
8 #include "base/metrics/histogram_samples.h"
9 #include "base/metrics/statistics_recorder.h"
10 #include "base/pickle.h"
11 #include "base/stl_util.h"
12 #include "base/strings/string_number_conversions.h"
13 #include "base/strings/stringprintf.h"
14 #include "base/test/statistics_delta_reader.h"
15 #include "base/time/time.h"
16 #include "net/base/load_flags.h"
17 #include "net/base/request_priority.h"
18 #include "net/base/test_completion_callback.h"
19 #include "net/url_request/url_request_context.h"
20 #include "net/url_request/url_request_test_util.h"
21 #include "net/url_request/url_request_throttler_header_interface.h"
22 #include "net/url_request/url_request_throttler_test_support.h"
23 #include "testing/gtest/include/gtest/gtest.h"
25 using base::TimeDelta
;
26 using base::TimeTicks
;
32 const char kRequestThrottledHistogramName
[] = "Throttling.RequestThrottled";
34 class MockURLRequestThrottlerEntry
: public URLRequestThrottlerEntry
{
36 explicit MockURLRequestThrottlerEntry(
37 net::URLRequestThrottlerManager
* manager
)
38 : net::URLRequestThrottlerEntry(manager
, std::string()),
39 mock_backoff_entry_(&backoff_policy_
) {
42 MockURLRequestThrottlerEntry(
43 net::URLRequestThrottlerManager
* manager
,
44 const TimeTicks
& exponential_backoff_release_time
,
45 const TimeTicks
& sliding_window_release_time
,
46 const TimeTicks
& fake_now
)
47 : net::URLRequestThrottlerEntry(manager
, std::string()),
48 fake_time_now_(fake_now
),
49 mock_backoff_entry_(&backoff_policy_
) {
52 mock_backoff_entry_
.set_fake_now(fake_now
);
53 set_exponential_backoff_release_time(exponential_backoff_release_time
);
54 set_sliding_window_release_time(sliding_window_release_time
);
58 // Some tests become flaky if we have jitter.
59 backoff_policy_
.jitter_factor
= 0.0;
61 // This lets us avoid having to make multiple failures initially (this
62 // logic is already tested in the BackoffEntry unit tests).
63 backoff_policy_
.num_errors_to_ignore
= 0;
66 virtual const BackoffEntry
* GetBackoffEntry() const OVERRIDE
{
67 return &mock_backoff_entry_
;
70 virtual BackoffEntry
* GetBackoffEntry() OVERRIDE
{
71 return &mock_backoff_entry_
;
74 static bool ExplicitUserRequest(int load_flags
) {
75 return URLRequestThrottlerEntry::ExplicitUserRequest(load_flags
);
78 void ResetToBlank(const TimeTicks
& time_now
) {
79 fake_time_now_
= time_now
;
80 mock_backoff_entry_
.set_fake_now(time_now
);
82 GetBackoffEntry()->Reset();
83 GetBackoffEntry()->SetCustomReleaseTime(time_now
);
84 set_sliding_window_release_time(time_now
);
87 // Overridden for tests.
88 virtual TimeTicks
ImplGetTimeNow() const OVERRIDE
{ return fake_time_now_
; }
90 void set_exponential_backoff_release_time(
91 const base::TimeTicks
& release_time
) {
92 GetBackoffEntry()->SetCustomReleaseTime(release_time
);
95 base::TimeTicks
sliding_window_release_time() const {
96 return URLRequestThrottlerEntry::sliding_window_release_time();
99 void set_sliding_window_release_time(
100 const base::TimeTicks
& release_time
) {
101 URLRequestThrottlerEntry::set_sliding_window_release_time(
105 TimeTicks fake_time_now_
;
106 MockBackoffEntry mock_backoff_entry_
;
109 virtual ~MockURLRequestThrottlerEntry() {}
112 class MockURLRequestThrottlerManager
: public URLRequestThrottlerManager
{
114 MockURLRequestThrottlerManager() : create_entry_index_(0) {}
116 // Method to process the URL using URLRequestThrottlerManager protected
118 std::string
DoGetUrlIdFromUrl(const GURL
& url
) { return GetIdFromUrl(url
); }
120 // Method to use the garbage collecting method of URLRequestThrottlerManager.
121 void DoGarbageCollectEntries() { GarbageCollectEntries(); }
123 // Returns the number of entries in the map.
124 int GetNumberOfEntries() const { return GetNumberOfEntriesForTests(); }
126 void CreateEntry(bool is_outdated
) {
127 TimeTicks time
= TimeTicks::Now();
129 time
-= TimeDelta::FromMilliseconds(
130 MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs
+ 1000);
132 std::string
fake_url_string("http://www.fakeurl.com/");
133 fake_url_string
.append(base::IntToString(create_entry_index_
++));
134 GURL
fake_url(fake_url_string
);
135 OverrideEntryForTests(
137 new MockURLRequestThrottlerEntry(this, time
, TimeTicks::Now(),
142 int create_entry_index_
;
146 TimeAndBool(const TimeTicks
& time_value
, bool expected
, int line_num
) {
156 struct GurlAndString
{
157 GurlAndString(const GURL
& url_value
,
158 const std::string
& expected
,
171 class URLRequestThrottlerEntryTest
: public testing::Test
{
173 URLRequestThrottlerEntryTest()
174 : request_(GURL(), DEFAULT_PRIORITY
, NULL
, &context_
) {}
176 static void SetUpTestCase() {
177 base::StatisticsRecorder::Initialize();
180 virtual void SetUp();
183 MockURLRequestThrottlerManager manager_
; // Dummy object, not used.
184 scoped_refptr
<MockURLRequestThrottlerEntry
> entry_
;
186 TestURLRequestContext context_
;
187 TestURLRequest request_
;
190 void URLRequestThrottlerEntryTest::SetUp() {
191 request_
.SetLoadFlags(0);
193 now_
= TimeTicks::Now();
194 entry_
= new MockURLRequestThrottlerEntry(&manager_
);
195 entry_
->ResetToBlank(now_
);
198 std::ostream
& operator<<(std::ostream
& out
, const base::TimeTicks
& time
) {
199 return out
<< time
.ToInternalValue();
202 TEST_F(URLRequestThrottlerEntryTest
, CanThrottleRequest
) {
203 TestNetworkDelegate d
;
204 context_
.set_network_delegate(&d
);
205 entry_
->set_exponential_backoff_release_time(
206 entry_
->fake_time_now_
+ TimeDelta::FromMilliseconds(1));
208 d
.set_can_throttle_requests(false);
209 EXPECT_FALSE(entry_
->ShouldRejectRequest(request_
));
210 d
.set_can_throttle_requests(true);
211 EXPECT_TRUE(entry_
->ShouldRejectRequest(request_
));
214 TEST_F(URLRequestThrottlerEntryTest
, InterfaceDuringExponentialBackoff
) {
215 base::StatisticsDeltaReader statistics_delta_reader
;
216 entry_
->set_exponential_backoff_release_time(
217 entry_
->fake_time_now_
+ TimeDelta::FromMilliseconds(1));
218 EXPECT_TRUE(entry_
->ShouldRejectRequest(request_
));
220 // Also end-to-end test the load flags exceptions.
221 request_
.SetLoadFlags(LOAD_MAYBE_USER_GESTURE
);
222 EXPECT_FALSE(entry_
->ShouldRejectRequest(request_
));
224 scoped_ptr
<base::HistogramSamples
> samples(
225 statistics_delta_reader
.GetHistogramSamplesSinceCreation(
226 kRequestThrottledHistogramName
));
227 ASSERT_EQ(1, samples
->GetCount(0));
228 ASSERT_EQ(1, samples
->GetCount(1));
231 TEST_F(URLRequestThrottlerEntryTest
, InterfaceNotDuringExponentialBackoff
) {
232 base::StatisticsDeltaReader statistics_delta_reader
;
233 entry_
->set_exponential_backoff_release_time(entry_
->fake_time_now_
);
234 EXPECT_FALSE(entry_
->ShouldRejectRequest(request_
));
235 entry_
->set_exponential_backoff_release_time(
236 entry_
->fake_time_now_
- TimeDelta::FromMilliseconds(1));
237 EXPECT_FALSE(entry_
->ShouldRejectRequest(request_
));
239 scoped_ptr
<base::HistogramSamples
> samples(
240 statistics_delta_reader
.GetHistogramSamplesSinceCreation(
241 kRequestThrottledHistogramName
));
242 ASSERT_EQ(2, samples
->GetCount(0));
243 ASSERT_EQ(0, samples
->GetCount(1));
246 TEST_F(URLRequestThrottlerEntryTest
, InterfaceUpdateFailure
) {
247 MockURLRequestThrottlerHeaderAdapter
failure_response(503);
248 entry_
->UpdateWithResponse(std::string(), &failure_response
);
249 EXPECT_GT(entry_
->GetExponentialBackoffReleaseTime(), entry_
->fake_time_now_
)
250 << "A failure should increase the release_time";
253 TEST_F(URLRequestThrottlerEntryTest
, InterfaceUpdateSuccess
) {
254 MockURLRequestThrottlerHeaderAdapter
success_response(200);
255 entry_
->UpdateWithResponse(std::string(), &success_response
);
256 EXPECT_EQ(entry_
->GetExponentialBackoffReleaseTime(), entry_
->fake_time_now_
)
257 << "A success should not add any delay";
260 TEST_F(URLRequestThrottlerEntryTest
, InterfaceUpdateSuccessThenFailure
) {
261 MockURLRequestThrottlerHeaderAdapter
failure_response(503);
262 MockURLRequestThrottlerHeaderAdapter
success_response(200);
263 entry_
->UpdateWithResponse(std::string(), &success_response
);
264 entry_
->UpdateWithResponse(std::string(), &failure_response
);
265 EXPECT_GT(entry_
->GetExponentialBackoffReleaseTime(), entry_
->fake_time_now_
)
266 << "This scenario should add delay";
267 entry_
->UpdateWithResponse(std::string(), &success_response
);
270 TEST_F(URLRequestThrottlerEntryTest
, IsEntryReallyOutdated
) {
271 TimeDelta lifetime
= TimeDelta::FromMilliseconds(
272 MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs
);
273 const TimeDelta kFiveMs
= TimeDelta::FromMilliseconds(5);
275 TimeAndBool test_values
[] = {
276 TimeAndBool(now_
, false, __LINE__
),
277 TimeAndBool(now_
- kFiveMs
, false, __LINE__
),
278 TimeAndBool(now_
+ kFiveMs
, false, __LINE__
),
279 TimeAndBool(now_
- (lifetime
- kFiveMs
), false, __LINE__
),
280 TimeAndBool(now_
- lifetime
, true, __LINE__
),
281 TimeAndBool(now_
- (lifetime
+ kFiveMs
), true, __LINE__
)};
283 for (unsigned int i
= 0; i
< arraysize(test_values
); ++i
) {
284 entry_
->set_exponential_backoff_release_time(test_values
[i
].time
);
285 EXPECT_EQ(entry_
->IsEntryOutdated(), test_values
[i
].result
) <<
286 "Test case #" << i
<< " line " << test_values
[i
].line
<< " failed";
290 TEST_F(URLRequestThrottlerEntryTest
, MaxAllowedBackoff
) {
291 for (int i
= 0; i
< 30; ++i
) {
292 MockURLRequestThrottlerHeaderAdapter
response_adapter(503);
293 entry_
->UpdateWithResponse(std::string(), &response_adapter
);
296 TimeDelta delay
= entry_
->GetExponentialBackoffReleaseTime() - now_
;
297 EXPECT_EQ(delay
.InMilliseconds(),
298 MockURLRequestThrottlerEntry::kDefaultMaximumBackoffMs
);
301 TEST_F(URLRequestThrottlerEntryTest
, MalformedContent
) {
302 MockURLRequestThrottlerHeaderAdapter
response_adapter(503);
303 for (int i
= 0; i
< 5; ++i
)
304 entry_
->UpdateWithResponse(std::string(), &response_adapter
);
306 TimeTicks release_after_failures
= entry_
->GetExponentialBackoffReleaseTime();
308 // Inform the entry that a response body was malformed, which is supposed to
309 // increase the back-off time. Note that we also submit a successful
310 // UpdateWithResponse to pair with ReceivedContentWasMalformed() since that
311 // is what happens in practice (if a body is received, then a non-500
312 // response must also have been received).
313 entry_
->ReceivedContentWasMalformed(200);
314 MockURLRequestThrottlerHeaderAdapter
success_adapter(200);
315 entry_
->UpdateWithResponse(std::string(), &success_adapter
);
316 EXPECT_GT(entry_
->GetExponentialBackoffReleaseTime(), release_after_failures
);
319 TEST_F(URLRequestThrottlerEntryTest
, SlidingWindow
) {
320 int max_send
= URLRequestThrottlerEntry::kDefaultMaxSendThreshold
;
322 URLRequestThrottlerEntry::kDefaultSlidingWindowPeriodMs
;
324 TimeTicks time_1
= entry_
->fake_time_now_
+
325 TimeDelta::FromMilliseconds(sliding_window
/ 3);
326 TimeTicks time_2
= entry_
->fake_time_now_
+
327 TimeDelta::FromMilliseconds(2 * sliding_window
/ 3);
328 TimeTicks time_3
= entry_
->fake_time_now_
+
329 TimeDelta::FromMilliseconds(sliding_window
);
330 TimeTicks time_4
= entry_
->fake_time_now_
+
331 TimeDelta::FromMilliseconds(sliding_window
+ 2 * sliding_window
/ 3);
333 entry_
->set_exponential_backoff_release_time(time_1
);
335 for (int i
= 0; i
< max_send
/ 2; ++i
) {
336 EXPECT_EQ(2 * sliding_window
/ 3,
337 entry_
->ReserveSendingTimeForNextRequest(time_2
));
339 EXPECT_EQ(time_2
, entry_
->sliding_window_release_time());
341 entry_
->fake_time_now_
= time_3
;
343 for (int i
= 0; i
< (max_send
+ 1) / 2; ++i
)
344 EXPECT_EQ(0, entry_
->ReserveSendingTimeForNextRequest(TimeTicks()));
346 EXPECT_EQ(time_4
, entry_
->sliding_window_release_time());
349 TEST_F(URLRequestThrottlerEntryTest
, ExplicitUserRequest
) {
350 ASSERT_FALSE(MockURLRequestThrottlerEntry::ExplicitUserRequest(0));
351 ASSERT_TRUE(MockURLRequestThrottlerEntry::ExplicitUserRequest(
352 LOAD_MAYBE_USER_GESTURE
));
353 ASSERT_FALSE(MockURLRequestThrottlerEntry::ExplicitUserRequest(
354 ~LOAD_MAYBE_USER_GESTURE
));
357 class URLRequestThrottlerManagerTest
: public testing::Test
{
359 URLRequestThrottlerManagerTest()
360 : request_(GURL(), DEFAULT_PRIORITY
, NULL
, &context_
) {}
362 virtual void SetUp() {
363 request_
.SetLoadFlags(0);
366 // context_ must be declared before request_.
367 TestURLRequestContext context_
;
368 TestURLRequest request_
;
371 TEST_F(URLRequestThrottlerManagerTest
, IsUrlStandardised
) {
372 MockURLRequestThrottlerManager manager
;
373 GurlAndString test_values
[] = {
374 GurlAndString(GURL("http://www.example.com"),
375 std::string("http://www.example.com/"),
377 GurlAndString(GURL("http://www.Example.com"),
378 std::string("http://www.example.com/"),
380 GurlAndString(GURL("http://www.ex4mple.com/Pr4c71c41"),
381 std::string("http://www.ex4mple.com/pr4c71c41"),
383 GurlAndString(GURL("http://www.example.com/0/token/false"),
384 std::string("http://www.example.com/0/token/false"),
386 GurlAndString(GURL("http://www.example.com/index.php?code=javascript"),
387 std::string("http://www.example.com/index.php"),
389 GurlAndString(GURL("http://www.example.com/index.php?code=1#superEntry"),
390 std::string("http://www.example.com/index.php"),
392 GurlAndString(GURL("http://www.example.com/index.php#superEntry"),
393 std::string("http://www.example.com/index.php"),
395 GurlAndString(GURL("http://www.example.com:1234/"),
396 std::string("http://www.example.com:1234/"),
399 for (unsigned int i
= 0; i
< arraysize(test_values
); ++i
) {
400 std::string temp
= manager
.DoGetUrlIdFromUrl(test_values
[i
].url
);
401 EXPECT_EQ(temp
, test_values
[i
].result
) <<
402 "Test case #" << i
<< " line " << test_values
[i
].line
<< " failed";
406 TEST_F(URLRequestThrottlerManagerTest
, AreEntriesBeingCollected
) {
407 MockURLRequestThrottlerManager manager
;
409 manager
.CreateEntry(true); // true = Entry is outdated.
410 manager
.CreateEntry(true);
411 manager
.CreateEntry(true);
412 manager
.DoGarbageCollectEntries();
413 EXPECT_EQ(0, manager
.GetNumberOfEntries());
415 manager
.CreateEntry(false);
416 manager
.CreateEntry(false);
417 manager
.CreateEntry(false);
418 manager
.CreateEntry(true);
419 manager
.DoGarbageCollectEntries();
420 EXPECT_EQ(3, manager
.GetNumberOfEntries());
423 TEST_F(URLRequestThrottlerManagerTest
, IsHostBeingRegistered
) {
424 MockURLRequestThrottlerManager manager
;
426 manager
.RegisterRequestUrl(GURL("http://www.example.com/"));
427 manager
.RegisterRequestUrl(GURL("http://www.google.com/"));
428 manager
.RegisterRequestUrl(GURL("http://www.google.com/index/0"));
429 manager
.RegisterRequestUrl(GURL("http://www.google.com/index/0?code=1"));
430 manager
.RegisterRequestUrl(GURL("http://www.google.com/index/0#lolsaure"));
432 EXPECT_EQ(3, manager
.GetNumberOfEntries());
435 void ExpectEntryAllowsAllOnErrorIfOptedOut(
436 net::URLRequestThrottlerEntryInterface
* entry
,
438 const URLRequest
& request
) {
439 EXPECT_FALSE(entry
->ShouldRejectRequest(request
));
440 MockURLRequestThrottlerHeaderAdapter
failure_adapter(503);
441 for (int i
= 0; i
< 10; ++i
) {
442 // Host doesn't really matter in this scenario so we skip it.
443 entry
->UpdateWithResponse(std::string(), &failure_adapter
);
445 EXPECT_NE(opted_out
, entry
->ShouldRejectRequest(request
));
448 // We're not mocking out GetTimeNow() in this scenario
449 // so add a 100 ms buffer to avoid flakiness (that should always
450 // give enough time to get from the TimeTicks::Now() call here
451 // to the TimeTicks::Now() call in the entry class).
452 EXPECT_GT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100),
453 entry
->GetExponentialBackoffReleaseTime());
455 // As above, add 100 ms.
456 EXPECT_LT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100),
457 entry
->GetExponentialBackoffReleaseTime());
461 TEST_F(URLRequestThrottlerManagerTest
, OptOutHeader
) {
462 MockURLRequestThrottlerManager manager
;
463 scoped_refptr
<net::URLRequestThrottlerEntryInterface
> entry
=
464 manager
.RegisterRequestUrl(GURL("http://www.google.com/yodude"));
466 // Fake a response with the opt-out header.
467 MockURLRequestThrottlerHeaderAdapter
response_adapter(
469 MockURLRequestThrottlerEntry::kExponentialThrottlingDisableValue
,
471 entry
->UpdateWithResponse("www.google.com", &response_adapter
);
473 // Ensure that the same entry on error always allows everything.
474 ExpectEntryAllowsAllOnErrorIfOptedOut(entry
.get(), true, request_
);
476 // Ensure that a freshly created entry (for a different URL on an
477 // already opted-out host) also gets "always allow" behavior.
478 scoped_refptr
<net::URLRequestThrottlerEntryInterface
> other_entry
=
479 manager
.RegisterRequestUrl(GURL("http://www.google.com/bingobob"));
480 ExpectEntryAllowsAllOnErrorIfOptedOut(other_entry
.get(), true, request_
);
482 // Fake a response with the opt-out header incorrectly specified.
483 scoped_refptr
<net::URLRequestThrottlerEntryInterface
> no_opt_out_entry
=
484 manager
.RegisterRequestUrl(GURL("http://www.nike.com/justdoit"));
485 MockURLRequestThrottlerHeaderAdapter
wrong_adapter(
486 std::string(), "yesplease", 200);
487 no_opt_out_entry
->UpdateWithResponse("www.nike.com", &wrong_adapter
);
488 ExpectEntryAllowsAllOnErrorIfOptedOut(
489 no_opt_out_entry
.get(), false, request_
);
491 // A localhost entry should always be opted out.
492 scoped_refptr
<net::URLRequestThrottlerEntryInterface
> localhost_entry
=
493 manager
.RegisterRequestUrl(GURL("http://localhost/hello"));
494 ExpectEntryAllowsAllOnErrorIfOptedOut(localhost_entry
.get(), true, request_
);
497 TEST_F(URLRequestThrottlerManagerTest
, ClearOnNetworkChange
) {
498 for (int i
= 0; i
< 3; ++i
) {
499 MockURLRequestThrottlerManager manager
;
500 scoped_refptr
<net::URLRequestThrottlerEntryInterface
> entry_before
=
501 manager
.RegisterRequestUrl(GURL("http://www.example.com/"));
502 MockURLRequestThrottlerHeaderAdapter
failure_adapter(503);
503 for (int j
= 0; j
< 10; ++j
) {
504 // Host doesn't really matter in this scenario so we skip it.
505 entry_before
->UpdateWithResponse(std::string(), &failure_adapter
);
507 EXPECT_TRUE(entry_before
->ShouldRejectRequest(request_
));
511 manager
.OnIPAddressChanged();
514 manager
.OnConnectionTypeChanged(
515 net::NetworkChangeNotifier::CONNECTION_UNKNOWN
);
518 manager
.OnConnectionTypeChanged(
519 net::NetworkChangeNotifier::CONNECTION_NONE
);
525 scoped_refptr
<net::URLRequestThrottlerEntryInterface
> entry_after
=
526 manager
.RegisterRequestUrl(GURL("http://www.example.com/"));
527 EXPECT_FALSE(entry_after
->ShouldRejectRequest(request_
));