1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "components/precache/core/precache_fetcher.h"
11 #include "base/basictypes.h"
12 #include "base/bind.h"
13 #include "base/callback.h"
14 #include "base/command_line.h"
15 #include "base/compiler_specific.h"
16 #include "base/memory/ref_counted.h"
17 #include "base/memory/scoped_ptr.h"
18 #include "base/run_loop.h"
19 #include "base/test/histogram_tester.h"
20 #include "base/thread_task_runner_handle.h"
21 #include "components/precache/core/precache_switches.h"
22 #include "components/precache/core/proto/precache.pb.h"
23 #include "net/base/load_flags.h"
24 #include "net/http/http_response_headers.h"
25 #include "net/http/http_status_code.h"
26 #include "net/url_request/test_url_fetcher_factory.h"
27 #include "net/url_request/url_request_status.h"
28 #include "net/url_request/url_request_test_util.h"
29 #include "testing/gmock/include/gmock/gmock.h"
30 #include "testing/gtest/include/gtest/gtest.h"
39 const char kConfigURL
[] = "http://config-url.com";
40 const char kManifestURLPrefix
[] = "http://manifest-url-prefix.com/";
41 const char kCustomConfigURL
[] = "http://custom-config-url.com";
42 const char kCustomManifestURLPrefix
[] =
43 "http://custom-manifest-url-prefix.com/";
44 const char kManifestFetchFailureURL
[] =
45 "http://manifest-url-prefix.com/manifest-fetch-failure.com";
46 const char kBadManifestURL
[] =
47 "http://manifest-url-prefix.com/bad-manifest.com";
48 const char kGoodManifestURL
[] =
49 "http://manifest-url-prefix.com/good-manifest.com";
50 const char kCustomGoodManifestURL
[] =
51 "http://custom-manifest-url-prefix.com/good-manifest.com";
52 const char kResourceFetchFailureURL
[] = "http://resource-fetch-failure.com";
53 const char kGoodResourceURL
[] = "http://good-resource.com";
54 const char kForcedStartingURLManifestURL
[] =
55 "http://manifest-url-prefix.com/forced-starting-url.com";
57 class TestURLFetcherCallback
{
59 TestURLFetcherCallback() : total_response_bytes_(0) {}
61 scoped_ptr
<net::FakeURLFetcher
> CreateURLFetcher(
62 const GURL
& url
, net::URLFetcherDelegate
* delegate
,
63 const std::string
& response_data
, net::HttpStatusCode response_code
,
64 net::URLRequestStatus::Status status
) {
65 scoped_ptr
<net::FakeURLFetcher
> fetcher(new net::FakeURLFetcher(
66 url
, delegate
, response_data
, response_code
, status
));
68 total_response_bytes_
+= response_data
.size();
69 requested_urls_
.insert(url
);
71 return fetcher
.Pass();
74 const std::multiset
<GURL
>& requested_urls() const {
75 return requested_urls_
;
78 int total_response_bytes() const { return total_response_bytes_
; }
81 // Multiset with one entry for each URL requested.
82 std::multiset
<GURL
> requested_urls_
;
83 int total_response_bytes_
;
86 class TestPrecacheDelegate
: public PrecacheFetcher::PrecacheDelegate
{
88 TestPrecacheDelegate() : was_on_done_called_(false) {}
90 void OnDone() override
{ was_on_done_called_
= true; }
92 bool was_on_done_called() const {
93 return was_on_done_called_
;
97 bool was_on_done_called_
;
100 class MockURLFetcherFactory
: public net::URLFetcherFactory
{
102 typedef net::URLFetcher
* DoURLFetcher(
105 net::URLFetcher::RequestType request_type
,
106 net::URLFetcherDelegate
* delegate
);
108 scoped_ptr
<net::URLFetcher
> CreateURLFetcher(
111 net::URLFetcher::RequestType request_type
,
112 net::URLFetcherDelegate
* delegate
) override
{
113 return make_scoped_ptr(DoCreateURLFetcher(id
, url
, request_type
, delegate
));
116 // The method to mock out, instead of CreateURLFetcher. This is necessary
117 // because gmock can't handle move-only types such as scoped_ptr.
118 MOCK_METHOD4(DoCreateURLFetcher
, DoURLFetcher
);
120 // A fake successful response. When the action runs, it saves off a pointer to
121 // the FakeURLFetcher in its output parameter for later inspection.
122 testing::Action
<DoURLFetcher
> RespondWith(const std::string
& body
,
123 net::FakeURLFetcher
** fetcher
) {
124 return RespondWith(body
, [](net::FakeURLFetcher
* fetcher
) {
125 fetcher
->set_response_code(net::HTTP_OK
);
129 // A fake custom response. When the action runs, it runs the given modifier to
130 // customize the FakeURLFetcher, and then saves off a pointer to the
131 // FakeURLFetcher in its output parameter for later inspection. The modifier
132 // should be a functor that takes a FakeURLFetcher* and returns void.
133 template <typename F
>
134 testing::Action
<DoURLFetcher
> RespondWith(const std::string
& body
,
136 net::FakeURLFetcher
** fetcher
) {
137 return testing::MakeAction(
138 new FakeResponseAction
<F
>(body
, modifier
, fetcher
));
142 template <typename F
>
143 class FakeResponseAction
: public testing::ActionInterface
<DoURLFetcher
> {
145 FakeResponseAction(const std::string
& body
,
147 net::FakeURLFetcher
** fetcher
)
148 : body_(body
), modifier_(modifier
), fetcher_(fetcher
) {}
150 net::URLFetcher
* Perform(
151 const testing::tuple
<int,
153 net::URLFetcher::RequestType
,
154 net::URLFetcherDelegate
*>& args
) {
155 *fetcher_
= new net::FakeURLFetcher(
156 testing::get
<1>(args
), testing::get
<3>(args
), body_
, net::HTTP_OK
,
157 net::URLRequestStatus::SUCCESS
);
158 modifier_(*fetcher_
);
165 net::FakeURLFetcher
** fetcher_
;
169 class PrecacheFetcherFetcherTest
: public testing::Test
{
171 PrecacheFetcherFetcherTest()
172 : request_context_(new net::TestURLRequestContextGetter(
173 base::ThreadTaskRunnerHandle::Get())),
174 scoped_url_fetcher_factory_(&factory_
),
175 callback_(base::Bind(&PrecacheFetcherFetcherTest::Callback
,
176 base::Unretained(this))),
177 callback_called_(false) {}
179 void Callback(const net::URLFetcher
&) { callback_called_
= true; }
182 base::MessageLoopForUI loop_
;
183 scoped_refptr
<net::TestURLRequestContextGetter
> request_context_
;
184 MockURLFetcherFactory factory_
;
185 net::ScopedURLFetcherFactory scoped_url_fetcher_factory_
;
186 base::Callback
<void(const net::URLFetcher
&)> callback_
;
187 bool callback_called_
;
190 TEST_F(PrecacheFetcherFetcherTest
, Config
) {
191 GURL
url(kConfigURL
);
193 net::FakeURLFetcher
* fetcher
= nullptr;
194 EXPECT_CALL(factory_
, DoCreateURLFetcher(_
, url
, net::URLFetcher::GET
, _
))
195 .WillOnce(factory_
.RespondWith("", &fetcher
));
197 PrecacheFetcher::Fetcher
precache_fetcher(
198 request_context_
.get(), url
, callback_
, false /* is_resource_request */);
200 loop_
.RunUntilIdle();
202 ASSERT_NE(nullptr, fetcher
);
203 EXPECT_EQ(kNoTracking
, fetcher
->GetLoadFlags());
205 EXPECT_EQ(true, callback_called_
);
208 TEST_F(PrecacheFetcherFetcherTest
, ResourceNotInCache
) {
209 GURL
url(kGoodResourceURL
);
211 net::FakeURLFetcher
*fetcher1
= nullptr, *fetcher2
= nullptr;
212 EXPECT_CALL(factory_
, DoCreateURLFetcher(_
, url
, net::URLFetcher::GET
, _
))
213 .WillOnce(factory_
.RespondWith(
215 [](net::FakeURLFetcher
* fetcher
) {
216 fetcher
->set_status(net::URLRequestStatus(
217 net::URLRequestStatus::FAILED
, net::ERR_CACHE_MISS
));
220 .WillOnce(factory_
.RespondWith("", &fetcher2
));
222 PrecacheFetcher::Fetcher
precache_fetcher(
223 request_context_
.get(), url
, callback_
, true /* is_resource_request */);
225 loop_
.RunUntilIdle();
227 ASSERT_NE(nullptr, fetcher1
);
228 EXPECT_EQ(net::LOAD_ONLY_FROM_CACHE
| kNoTracking
, fetcher1
->GetLoadFlags());
229 ASSERT_NE(nullptr, fetcher2
);
230 EXPECT_EQ(net::LOAD_VALIDATE_CACHE
| kNoTracking
, fetcher2
->GetLoadFlags());
232 EXPECT_EQ(true, callback_called_
);
235 TEST_F(PrecacheFetcherFetcherTest
, ResourceHasStrongValidators
) {
236 GURL
url(kGoodResourceURL
);
238 net::FakeURLFetcher
*fetcher1
= nullptr, *fetcher2
= nullptr;
239 EXPECT_CALL(factory_
, DoCreateURLFetcher(_
, url
, net::URLFetcher::GET
, _
))
240 .WillOnce(factory_
.RespondWith(
242 [](net::FakeURLFetcher
* fetcher
) {
243 std::string
raw_headers("HTTP/1.1 200 OK\0ETag: foo\0\0", 27);
244 fetcher
->set_response_headers(
245 make_scoped_refptr(new net::HttpResponseHeaders(raw_headers
)));
248 .WillOnce(factory_
.RespondWith("", &fetcher2
));
250 PrecacheFetcher::Fetcher
precache_fetcher(
251 request_context_
.get(), url
, callback_
, true /* is_resource_request */);
253 loop_
.RunUntilIdle();
255 ASSERT_NE(nullptr, fetcher1
);
256 EXPECT_EQ(net::LOAD_ONLY_FROM_CACHE
| kNoTracking
, fetcher1
->GetLoadFlags());
257 ASSERT_NE(nullptr, fetcher2
);
258 EXPECT_EQ(net::LOAD_VALIDATE_CACHE
| kNoTracking
, fetcher2
->GetLoadFlags());
260 EXPECT_EQ(true, callback_called_
);
263 TEST_F(PrecacheFetcherFetcherTest
, ResourceHasNoValidators
) {
264 GURL
url(kGoodResourceURL
);
266 net::FakeURLFetcher
* fetcher
;
267 EXPECT_CALL(factory_
, DoCreateURLFetcher(_
, url
, net::URLFetcher::GET
, _
))
268 .WillOnce(factory_
.RespondWith("", &fetcher
));
270 PrecacheFetcher::Fetcher
precache_fetcher(
271 request_context_
.get(), url
, callback_
, true /* is_resource_request */);
273 loop_
.RunUntilIdle();
275 EXPECT_EQ(net::LOAD_ONLY_FROM_CACHE
| kNoTracking
, fetcher
->GetLoadFlags());
277 EXPECT_EQ(true, callback_called_
);
280 class PrecacheFetcherTest
: public testing::Test
{
282 PrecacheFetcherTest()
283 : request_context_(new net::TestURLRequestContextGetter(
284 base::ThreadTaskRunnerHandle::Get())),
286 base::Bind(&TestURLFetcherCallback::CreateURLFetcher
,
287 base::Unretained(&url_callback_
))),
288 expected_total_response_bytes_(0) {}
291 void SetDefaultFlags() {
292 base::CommandLine::ForCurrentProcess()->AppendSwitchASCII(
293 switches::kPrecacheConfigSettingsURL
, kConfigURL
);
294 base::CommandLine::ForCurrentProcess()->AppendSwitchASCII(
295 switches::kPrecacheManifestURLPrefix
, kManifestURLPrefix
);
298 base::MessageLoopForUI loop_
;
299 scoped_refptr
<net::TestURLRequestContextGetter
> request_context_
;
300 TestURLFetcherCallback url_callback_
;
301 net::FakeURLFetcherFactory factory_
;
302 TestPrecacheDelegate precache_delegate_
;
303 int expected_total_response_bytes_
;
306 TEST_F(PrecacheFetcherTest
, FullPrecache
) {
309 std::vector
<std::string
> starting_hosts
;
310 starting_hosts
.push_back("manifest-fetch-failure.com");
311 starting_hosts
.push_back("bad-manifest.com");
312 starting_hosts
.push_back("good-manifest.com");
313 starting_hosts
.push_back("not-in-top-3.com");
315 PrecacheConfigurationSettings config
;
316 config
.set_top_sites_count(3);
317 config
.add_forced_site("forced-starting-url.com");
318 // Duplicate starting URL, the manifest for this should only be fetched once.
319 config
.add_forced_site("good-manifest.com");
321 PrecacheManifest good_manifest
;
322 good_manifest
.add_resource()->set_url(kResourceFetchFailureURL
);
323 good_manifest
.add_resource(); // Resource with no URL, should not be fetched.
324 good_manifest
.add_resource()->set_url(kGoodResourceURL
);
326 factory_
.SetFakeResponse(GURL(kConfigURL
), config
.SerializeAsString(),
327 net::HTTP_OK
, net::URLRequestStatus::SUCCESS
);
328 factory_
.SetFakeResponse(GURL(kManifestFetchFailureURL
), "",
329 net::HTTP_INTERNAL_SERVER_ERROR
,
330 net::URLRequestStatus::FAILED
);
331 factory_
.SetFakeResponse(GURL(kBadManifestURL
), "bad protobuf", net::HTTP_OK
,
332 net::URLRequestStatus::SUCCESS
);
333 factory_
.SetFakeResponse(GURL(kGoodManifestURL
),
334 good_manifest
.SerializeAsString(), net::HTTP_OK
,
335 net::URLRequestStatus::SUCCESS
);
336 factory_
.SetFakeResponse(GURL(kResourceFetchFailureURL
),
337 "", net::HTTP_INTERNAL_SERVER_ERROR
,
338 net::URLRequestStatus::FAILED
);
339 factory_
.SetFakeResponse(GURL(kGoodResourceURL
), "good", net::HTTP_OK
,
340 net::URLRequestStatus::SUCCESS
);
341 factory_
.SetFakeResponse(GURL(kForcedStartingURLManifestURL
),
342 PrecacheManifest().SerializeAsString(), net::HTTP_OK
,
343 net::URLRequestStatus::SUCCESS
);
345 base::HistogramTester histogram
;
348 PrecacheFetcher
precache_fetcher(starting_hosts
, request_context_
.get(),
349 GURL(), std::string(),
350 &precache_delegate_
);
351 precache_fetcher
.Start();
353 loop_
.RunUntilIdle();
355 // Destroy the PrecacheFetcher after it has finished, to record metrics.
358 std::multiset
<GURL
> expected_requested_urls
;
359 expected_requested_urls
.insert(GURL(kConfigURL
));
360 expected_requested_urls
.insert(GURL(kManifestFetchFailureURL
));
361 expected_requested_urls
.insert(GURL(kBadManifestURL
));
362 expected_requested_urls
.insert(GURL(kGoodManifestURL
));
363 expected_requested_urls
.insert(GURL(kResourceFetchFailureURL
));
364 expected_requested_urls
.insert(GURL(kGoodResourceURL
));
365 expected_requested_urls
.insert(GURL(kForcedStartingURLManifestURL
));
367 EXPECT_EQ(expected_requested_urls
, url_callback_
.requested_urls());
369 EXPECT_TRUE(precache_delegate_
.was_on_done_called());
371 histogram
.ExpectUniqueSample("Precache.Fetch.PercentCompleted", 100, 1);
372 histogram
.ExpectUniqueSample("Precache.Fetch.ResponseBytes.Total",
373 url_callback_
.total_response_bytes(), 1);
374 histogram
.ExpectTotalCount("Precache.Fetch.TimeToComplete", 1);
377 TEST_F(PrecacheFetcherTest
, CustomURLs
) {
380 std::vector
<std::string
> starting_hosts
;
381 starting_hosts
.push_back("good-manifest.com");
383 PrecacheConfigurationSettings config
;
385 PrecacheManifest good_manifest
;
386 good_manifest
.add_resource()->set_url(kGoodResourceURL
);
388 factory_
.SetFakeResponse(GURL(kCustomConfigURL
), config
.SerializeAsString(),
389 net::HTTP_OK
, net::URLRequestStatus::SUCCESS
);
390 factory_
.SetFakeResponse(GURL(kCustomGoodManifestURL
),
391 good_manifest
.SerializeAsString(), net::HTTP_OK
,
392 net::URLRequestStatus::SUCCESS
);
393 factory_
.SetFakeResponse(GURL(kGoodResourceURL
), "good", net::HTTP_OK
,
394 net::URLRequestStatus::SUCCESS
);
396 PrecacheFetcher
precache_fetcher(
397 starting_hosts
, request_context_
.get(), GURL(kCustomConfigURL
),
398 kCustomManifestURLPrefix
, &precache_delegate_
);
399 precache_fetcher
.Start();
401 loop_
.RunUntilIdle();
403 std::multiset
<GURL
> expected_requested_urls
;
404 expected_requested_urls
.insert(GURL(kCustomConfigURL
));
405 expected_requested_urls
.insert(GURL(kCustomGoodManifestURL
));
406 expected_requested_urls
.insert(GURL(kGoodResourceURL
));
408 EXPECT_EQ(expected_requested_urls
, url_callback_
.requested_urls());
410 EXPECT_TRUE(precache_delegate_
.was_on_done_called());
413 TEST_F(PrecacheFetcherTest
, ConfigFetchFailure
) {
416 std::vector
<std::string
> starting_hosts(1, "good-manifest.com");
418 factory_
.SetFakeResponse(GURL(kConfigURL
), "",
419 net::HTTP_INTERNAL_SERVER_ERROR
,
420 net::URLRequestStatus::FAILED
);
421 factory_
.SetFakeResponse(GURL(kGoodManifestURL
), "", net::HTTP_OK
,
422 net::URLRequestStatus::SUCCESS
);
424 PrecacheFetcher
precache_fetcher(starting_hosts
, request_context_
.get(),
425 GURL(), std::string(), &precache_delegate_
);
426 precache_fetcher
.Start();
428 loop_
.RunUntilIdle();
430 std::multiset
<GURL
> expected_requested_urls
;
431 expected_requested_urls
.insert(GURL(kConfigURL
));
432 expected_requested_urls
.insert(GURL(kGoodManifestURL
));
433 EXPECT_EQ(expected_requested_urls
, url_callback_
.requested_urls());
435 EXPECT_TRUE(precache_delegate_
.was_on_done_called());
438 TEST_F(PrecacheFetcherTest
, BadConfig
) {
441 std::vector
<std::string
> starting_hosts(1, "good-manifest.com");
443 factory_
.SetFakeResponse(GURL(kConfigURL
), "bad protobuf", net::HTTP_OK
,
444 net::URLRequestStatus::SUCCESS
);
445 factory_
.SetFakeResponse(GURL(kGoodManifestURL
), "", net::HTTP_OK
,
446 net::URLRequestStatus::SUCCESS
);
448 PrecacheFetcher
precache_fetcher(starting_hosts
, request_context_
.get(),
449 GURL(), std::string(), &precache_delegate_
);
450 precache_fetcher
.Start();
452 loop_
.RunUntilIdle();
454 std::multiset
<GURL
> expected_requested_urls
;
455 expected_requested_urls
.insert(GURL(kConfigURL
));
456 expected_requested_urls
.insert(GURL(kGoodManifestURL
));
457 EXPECT_EQ(expected_requested_urls
, url_callback_
.requested_urls());
459 EXPECT_TRUE(precache_delegate_
.was_on_done_called());
462 TEST_F(PrecacheFetcherTest
, Cancel
) {
465 std::vector
<std::string
> starting_hosts(1, "starting-url.com");
467 PrecacheConfigurationSettings config
;
468 config
.set_top_sites_count(1);
470 factory_
.SetFakeResponse(GURL(kConfigURL
), config
.SerializeAsString(),
471 net::HTTP_OK
, net::URLRequestStatus::SUCCESS
);
473 base::HistogramTester histogram
;
476 PrecacheFetcher
precache_fetcher(starting_hosts
, request_context_
.get(),
477 GURL(), std::string(),
478 &precache_delegate_
);
479 precache_fetcher
.Start();
481 // Destroy the PrecacheFetcher, to cancel precaching and record metrics.
482 // This should not cause OnDone to be called on the precache delegate.
485 loop_
.RunUntilIdle();
487 std::multiset
<GURL
> expected_requested_urls
;
488 expected_requested_urls
.insert(GURL(kConfigURL
));
489 EXPECT_EQ(expected_requested_urls
, url_callback_
.requested_urls());
491 EXPECT_FALSE(precache_delegate_
.was_on_done_called());
493 histogram
.ExpectUniqueSample("Precache.Fetch.PercentCompleted", 0, 1);
494 histogram
.ExpectUniqueSample("Precache.Fetch.ResponseBytes.Total", 0, 1);
495 histogram
.ExpectTotalCount("Precache.Fetch.TimeToComplete", 0);
498 #if defined(PRECACHE_CONFIG_SETTINGS_URL)
500 // If the default precache configuration settings URL is defined, then test that
501 // it works with the PrecacheFetcher.
502 TEST_F(PrecacheFetcherTest
, PrecacheUsingDefaultConfigSettingsURL
) {
503 std::vector
<std::string
> starting_hosts(1, "starting-url.com");
505 PrecacheConfigurationSettings config
;
506 config
.set_top_sites_count(0);
508 factory_
.SetFakeResponse(GURL(PRECACHE_CONFIG_SETTINGS_URL
),
509 config
.SerializeAsString(), net::HTTP_OK
,
510 net::URLRequestStatus::SUCCESS
);
512 PrecacheFetcher
precache_fetcher(starting_hosts
, request_context_
.get(),
513 GURL(), std::string(), &precache_delegate_
);
514 precache_fetcher
.Start();
516 loop_
.RunUntilIdle();
518 std::multiset
<GURL
> expected_requested_urls
;
519 expected_requested_urls
.insert(GURL(PRECACHE_CONFIG_SETTINGS_URL
));
520 EXPECT_EQ(expected_requested_urls
, url_callback_
.requested_urls());
522 EXPECT_TRUE(precache_delegate_
.was_on_done_called());
525 #endif // PRECACHE_CONFIG_SETTINGS_URL
527 #if defined(PRECACHE_MANIFEST_URL_PREFIX)
529 // If the default precache manifest URL prefix is defined, then test that it
530 // works with the PrecacheFetcher.
531 TEST_F(PrecacheFetcherTest
, PrecacheUsingDefaultManifestURLPrefix
) {
532 base::CommandLine::ForCurrentProcess()->AppendSwitchASCII(
533 switches::kPrecacheConfigSettingsURL
, kConfigURL
);
535 std::vector
<std::string
> starting_hosts(1, "starting-url.com");
537 PrecacheConfigurationSettings config
;
538 config
.set_top_sites_count(1);
540 GURL
manifest_url(PRECACHE_MANIFEST_URL_PREFIX
"starting-url.com");
542 factory_
.SetFakeResponse(GURL(kConfigURL
), config
.SerializeAsString(),
543 net::HTTP_OK
, net::URLRequestStatus::SUCCESS
);
544 factory_
.SetFakeResponse(manifest_url
, PrecacheManifest().SerializeAsString(),
545 net::HTTP_OK
, net::URLRequestStatus::SUCCESS
);
547 PrecacheFetcher
precache_fetcher(starting_hosts
, request_context_
.get(),
548 GURL(), std::string(), &precache_delegate_
);
549 precache_fetcher
.Start();
551 loop_
.RunUntilIdle();
553 std::multiset
<GURL
> expected_requested_urls
;
554 expected_requested_urls
.insert(GURL(kConfigURL
));
555 expected_requested_urls
.insert(manifest_url
);
556 EXPECT_EQ(expected_requested_urls
, url_callback_
.requested_urls());
558 EXPECT_TRUE(precache_delegate_
.was_on_done_called());
561 #endif // PRECACHE_MANIFEST_URL_PREFIX
565 } // namespace precache