net/url_request/url_fetcher_impl_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_fetcher_impl.h"
   6
   7 #include <stdint.h>
   8 #include <string.h>
   9
  10 #include <algorithm>
  11 #include <string>
  12
  13 #include "base/bind.h"
  14 #include "base/files/file_util.h"
  15 #include "base/files/scoped_temp_dir.h"
  16 #include "base/location.h"
  17 #include "base/macros.h"
  18 #include "base/path_service.h"
  19 #include "base/run_loop.h"
  20 #include "base/single_thread_task_runner.h"
  21 #include "base/strings/string_util.h"
  22 #include "base/strings/stringprintf.h"
  23 #include "base/synchronization/waitable_event.h"
  24 #include "base/thread_task_runner_handle.h"
  25 #include "base/threading/thread.h"
  26 #include "build/build_config.h"
  27 #include "crypto/nss_util.h"
  28 #include "net/base/elements_upload_data_stream.h"
  29 #include "net/base/network_change_notifier.h"
  30 #include "net/base/upload_bytes_element_reader.h"
  31 #include "net/base/upload_element_reader.h"
  32 #include "net/base/upload_file_element_reader.h"
  33 #include "net/dns/mock_host_resolver.h"
  34 #include "net/http/http_response_headers.h"
  35 #include "net/test/spawned_test_server/spawned_test_server.h"
  36 #include "net/url_request/url_fetcher_delegate.h"
  37 #include "net/url_request/url_request_context_getter.h"
  38 #include "net/url_request/url_request_test_util.h"
  39 #include "net/url_request/url_request_throttler_manager.h"
  40 #include "testing/gtest/include/gtest/gtest.h"
  41
  42 #if defined(USE_NSS_CERTS) || defined(OS_IOS)
  43 #include "net/cert_net/nss_ocsp.h"
  44 #endif
  45
  46 namespace net {
  47
  48 using base::Time;
  49 using base::TimeDelta;
  50
  51 // TODO(eroman): Add a regression test for http://crbug.com/40505.
  52
  53 namespace {
  54
  55 // TODO(akalin): Move all the test data to somewhere under net/.
  56 const base::FilePath::CharType kDocRoot[] =
  57     FILE_PATH_LITERAL("net/data/url_fetcher_impl_unittest");
  58 const char kTestServerFilePrefix[] = "files/";
  59
  60 // Test server path and response body for the default URL used by many of the
  61 // tests.
  62 const char kDefaultResponsePath[] = "defaultresponse";
  63 const char kDefaultResponseBody[] =
  64     "Default response given for path: /defaultresponse";
  65
  66 // Request body for streams created by CreateUploadStream.
  67 const char kCreateUploadStreamBody[] = "rosebud";
  68
  69 base::FilePath GetUploadFileTestPath() {
  70   base::FilePath path;
  71   PathService::Get(base::DIR_SOURCE_ROOT, &path);
  72   return path.Append(
  73       FILE_PATH_LITERAL("net/data/url_request_unittest/BullRunSpeech.txt"));
  74 }
  75
  76 // Simple URLRequestDelegate that waits for the specified fetcher to complete.
  77 // Can only be used once.
  78 class WaitingURLFetcherDelegate : public URLFetcherDelegate {
  79  public:
  80   WaitingURLFetcherDelegate() : did_complete_(false) {}
  81
  82   void CreateFetcher(
  83       const GURL& url,
  84       URLFetcher::RequestType request_type,
  85       scoped_refptr<net::URLRequestContextGetter> context_getter) {
  86     fetcher_.reset(new URLFetcherImpl(url, request_type, this));
  87     fetcher_->SetRequestContext(context_getter.get());
  88   }
  89
  90   URLFetcher* fetcher() const { return fetcher_.get(); }
  91
  92   // Wait until the request has completed or been canceled.
  93   void StartFetcherAndWait() {
  94     fetcher_->Start();
  95     WaitForComplete();
  96   }
  97
  98   // Wait until the request has completed or been canceled. Does not start the
  99   // request.
 100   void WaitForComplete() { run_loop_.Run(); }
 101
 102   // Cancels the fetch by deleting the fetcher.
 103   void CancelFetch() {
 104     EXPECT_TRUE(fetcher_);
 105     fetcher_.reset();
 106     run_loop_.Quit();
 107   }
 108
 109   // URLFetcherDelegate:
 110   void OnURLFetchComplete(const URLFetcher* source) override {
 111     EXPECT_FALSE(did_complete_);
 112     EXPECT_TRUE(fetcher_);
 113     EXPECT_EQ(fetcher_.get(), source);
 114     did_complete_ = true;
 115     run_loop_.Quit();
 116   }
 117
 118   void OnURLFetchDownloadProgress(const URLFetcher* source,
 119                                   int64 current,
 120                                   int64 total) override {
 121     // Note that the current progress may be greater than the previous progress,
 122     // in the case of retrying the request.
 123     EXPECT_FALSE(did_complete_);
 124     EXPECT_TRUE(fetcher_);
 125     EXPECT_EQ(source, fetcher_.get());
 126
 127     EXPECT_LE(0, current);
 128     // If file size is not known, |total| is -1.
 129     if (total >= 0)
 130       EXPECT_LE(current, total);
 131   }
 132
 133   void OnURLFetchUploadProgress(const URLFetcher* source,
 134                                 int64 current,
 135                                 int64 total) override {
 136     // Note that the current progress may be greater than the previous progress,
 137     // in the case of retrying the request.
 138     EXPECT_FALSE(did_complete_);
 139     EXPECT_TRUE(fetcher_);
 140     EXPECT_EQ(source, fetcher_.get());
 141
 142     EXPECT_LE(0, current);
 143     // If file size is not known, |total| is -1.
 144     if (total >= 0)
 145       EXPECT_LE(current, total);
 146   }
 147
 148   bool did_complete() const { return did_complete_; }
 149
 150  private:
 151   bool did_complete_;
 152
 153   scoped_ptr<URLFetcherImpl> fetcher_;
 154   base::RunLoop run_loop_;
 155
 156   DISALLOW_COPY_AND_ASSIGN(WaitingURLFetcherDelegate);
 157 };
 158
 159 // A TestURLRequestContext with a ThrottleManager and a MockHostResolver.
 160 class FetcherTestURLRequestContext : public TestURLRequestContext {
 161  public:
 162   // All requests for |hanging_domain| will hang on host resolution until the
 163   // mock_resolver()->ResolveAllPending() is called.
 164   explicit FetcherTestURLRequestContext(const std::string& hanging_domain)
 165       : TestURLRequestContext(true), mock_resolver_(new MockHostResolver()) {
 166     mock_resolver_->set_ondemand_mode(true);
 167     mock_resolver_->rules()->AddRule(hanging_domain, "127.0.0.1");
 168     // Pass ownership to ContextStorage to ensure correct destruction order.
 169     context_storage_.set_host_resolver(
 170         scoped_ptr<HostResolver>(mock_resolver_));
 171     context_storage_.set_throttler_manager(new URLRequestThrottlerManager());
 172     Init();
 173   }
 174
 175   MockHostResolver* mock_resolver() { return mock_resolver_; }
 176
 177  private:
 178   MockHostResolver* mock_resolver_;
 179
 180   DISALLOW_COPY_AND_ASSIGN(FetcherTestURLRequestContext);
 181 };
 182
 183 class FetcherTestURLRequestContextGetter : public URLRequestContextGetter {
 184  public:
 185   FetcherTestURLRequestContextGetter(
 186       scoped_refptr<base::SingleThreadTaskRunner> network_task_runner,
 187       const std::string& hanging_domain)
 188       : network_task_runner_(network_task_runner),
 189         hanging_domain_(hanging_domain),
 190         shutting_down_(false) {}
 191
 192   // Sets callback to be invoked when the getter is destroyed.
 193   void set_on_destruction_callback(
 194       const base::Closure& on_destruction_callback) {
 195     on_destruction_callback_ = on_destruction_callback;
 196   }
 197
 198   // URLRequestContextGetter:
 199   FetcherTestURLRequestContext* GetURLRequestContext() override {
 200     // Calling this on the wrong thread may be either a bug in the test or a bug
 201     // in production code.
 202     EXPECT_TRUE(network_task_runner_->BelongsToCurrentThread());
 203
 204     if (shutting_down_)
 205       return nullptr;
 206
 207     if (!context_)
 208       context_.reset(new FetcherTestURLRequestContext(hanging_domain_));
 209     return context_.get();
 210   }
 211
 212   scoped_refptr<base::SingleThreadTaskRunner> GetNetworkTaskRunner()
 213       const override {
 214     return network_task_runner_;
 215   }
 216
 217   // Adds a throttler entry with the specified parameters.  Does this
 218   // synchronously if the context lives on the current thread, or posts a task
 219   // to the relevant thread otherwise.
 220   //
 221   // If |reserve_sending_time_for_next_request|, will start backoff early, as
 222   // if there has already been a request for |url|.
 223   void AddThrottlerEntry(const GURL& url,
 224                          const std::string& url_id,
 225                          int sliding_window_period_ms,
 226                          int max_send_threshold,
 227                          int initial_backoff_ms,
 228                          double multiply_factor,
 229                          double jitter_factor,
 230                          int maximum_backoff_ms,
 231                          bool reserve_sending_time_for_next_request) {
 232     if (!network_task_runner_->RunsTasksOnCurrentThread()) {
 233       network_task_runner_->PostTask(
 234           FROM_HERE,
 235           base::Bind(&FetcherTestURLRequestContextGetter::AddThrottlerEntry,
 236                      this, url, url_id, sliding_window_period_ms,
 237                      max_send_threshold, initial_backoff_ms, multiply_factor,
 238                      jitter_factor, maximum_backoff_ms,
 239                      reserve_sending_time_for_next_request));
 240       return;
 241     }
 242     scoped_refptr<URLRequestThrottlerEntry> entry(new URLRequestThrottlerEntry(
 243         GetURLRequestContext()->throttler_manager(), url_id,
 244         sliding_window_period_ms, max_send_threshold, initial_backoff_ms,
 245         multiply_factor, jitter_factor, maximum_backoff_ms));
 246
 247     GetURLRequestContext()->throttler_manager()->OverrideEntryForTests(
 248         url, entry.get());
 249
 250     if (reserve_sending_time_for_next_request)
 251       entry->ReserveSendingTimeForNextRequest(base::TimeTicks());
 252   }
 253
 254   // Tells the getter to act as if the URLRequestContext is about to be shut
 255   // down.
 256   void Shutdown() {
 257     if (!network_task_runner_->RunsTasksOnCurrentThread()) {
 258       network_task_runner_->PostTask(
 259           FROM_HERE,
 260           base::Bind(&FetcherTestURLRequestContextGetter::Shutdown, this));
 261       return;
 262     }
 263
 264     shutting_down_ = true;
 265     NotifyContextShuttingDown();
 266     // Should now be safe to destroy the context.  Context will check it has no
 267     // pending requests.
 268     context_.reset();
 269   }
 270
 271   // Convenience method to access the context as a FetcherTestURLRequestContext
 272   // without going through GetURLRequestContext.
 273   FetcherTestURLRequestContext* context() {
 274     DCHECK(network_task_runner_->BelongsToCurrentThread());
 275     return context_.get();
 276   }
 277
 278  protected:
 279   ~FetcherTestURLRequestContextGetter() override {
 280     // |context_| may only be deleted on the network thread. Fortunately,
 281     // the parent class already ensures it's deleted on the network thread.
 282     DCHECK(network_task_runner_->BelongsToCurrentThread());
 283     if (!on_destruction_callback_.is_null())
 284       on_destruction_callback_.Run();
 285   }
 286
 287   scoped_refptr<base::SingleThreadTaskRunner> network_task_runner_;
 288   const std::string hanging_domain_;
 289
 290   scoped_ptr<FetcherTestURLRequestContext> context_;
 291   bool shutting_down_;
 292
 293   base::Closure on_destruction_callback_;
 294
 295   DISALLOW_COPY_AND_ASSIGN(FetcherTestURLRequestContextGetter);
 296 };
 297
 298 }  // namespace
 299
 300 class URLFetcherTest : public testing::Test {
 301  public:
 302   URLFetcherTest() : num_upload_streams_created_(0) {}
 303
 304   static int GetNumFetcherCores() {
 305     return URLFetcherImpl::GetNumFetcherCores();
 306   }
 307
 308   // Creates a URLRequestContextGetter with a URLRequestContext that lives on
 309   // the current thread.
 310   scoped_refptr<FetcherTestURLRequestContextGetter>
 311   CreateSameThreadContextGetter() {
 312     return scoped_refptr<FetcherTestURLRequestContextGetter>(
 313         new FetcherTestURLRequestContextGetter(
 314             base::ThreadTaskRunnerHandle::Get(), hanging_url().host()));
 315   }
 316
 317   // Creates a URLRequestContextGetter with a URLRequestContext that lives on
 318   // a separate network thread.
 319   scoped_refptr<FetcherTestURLRequestContextGetter>
 320   CreateCrossThreadContextGetter() {
 321     if (!network_thread_) {
 322       network_thread_.reset(new base::Thread("network thread"));
 323       base::Thread::Options network_thread_options;
 324       network_thread_options.message_loop_type = base::MessageLoop::TYPE_IO;
 325       bool result = network_thread_->StartWithOptions(network_thread_options);
 326       CHECK(result);
 327     }
 328
 329     return scoped_refptr<FetcherTestURLRequestContextGetter>(
 330         new FetcherTestURLRequestContextGetter(network_thread_->task_runner(),
 331                                                hanging_url().host()));
 332   }
 333
 334   // Callback passed to URLFetcher to create upload stream by some tests.
 335   scoped_ptr<UploadDataStream> CreateUploadStream() {
 336     ++num_upload_streams_created_;
 337     std::vector<char> buffer(
 338         kCreateUploadStreamBody,
 339         kCreateUploadStreamBody + strlen(kCreateUploadStreamBody));
 340     return ElementsUploadDataStream::CreateWithReader(
 341         scoped_ptr<UploadElementReader>(
 342             new UploadOwnedBytesElementReader(&buffer)),
 343         0);
 344   }
 345
 346   // Number of streams created by CreateUploadStream.
 347   size_t num_upload_streams_created() const {
 348     return num_upload_streams_created_;
 349   }
 350
 351   // Downloads |file_to_fetch| and checks the contents when done.  If
 352   // |save_to_temporary_file| is true, saves it to a temporary file, and
 353   // |requested_out_path| is ignored. Otherwise, saves it to
 354   // |requested_out_path|. Takes ownership of the file if |take_ownership| is
 355   // true. Deletes file when done.
 356   void SaveFileTest(const char* file_to_fetch,
 357                     bool save_to_temporary_file,
 358                     const base::FilePath& requested_out_path,
 359                     bool take_ownership) {
 360     scoped_ptr<WaitingURLFetcherDelegate> delegate(
 361         new WaitingURLFetcherDelegate());
 362     delegate->CreateFetcher(
 363         test_server_->GetURL(std::string(kTestServerFilePrefix) +
 364                              file_to_fetch),
 365         URLFetcher::GET, CreateSameThreadContextGetter());
 366     if (save_to_temporary_file) {
 367       delegate->fetcher()->SaveResponseToTemporaryFile(
 368           scoped_refptr<base::SingleThreadTaskRunner>(
 369               base::ThreadTaskRunnerHandle::Get()));
 370     } else {
 371       delegate->fetcher()->SaveResponseToFileAtPath(
 372           requested_out_path, scoped_refptr<base::SingleThreadTaskRunner>(
 373                                   base::ThreadTaskRunnerHandle::Get()));
 374     }
 375     delegate->StartFetcherAndWait();
 376
 377     EXPECT_TRUE(delegate->fetcher()->GetStatus().is_success());
 378     EXPECT_EQ(200, delegate->fetcher()->GetResponseCode());
 379
 380     base::FilePath out_path;
 381     EXPECT_TRUE(
 382         delegate->fetcher()->GetResponseAsFilePath(take_ownership, &out_path));
 383     if (!save_to_temporary_file) {
 384       EXPECT_EQ(requested_out_path, out_path);
 385     }
 386
 387     EXPECT_TRUE(base::ContentsEqual(
 388         test_server_->GetDocumentRoot().AppendASCII(file_to_fetch), out_path));
 389
 390     // Delete the delegate and run the message loop to give the fetcher's
 391     // destructor a chance to delete the file.
 392     delegate.reset();
 393     base::RunLoop().RunUntilIdle();
 394
 395     // File should only exist if |take_ownership| was true.
 396     EXPECT_EQ(take_ownership, base::PathExists(out_path));
 397
 398     // Cleanup.
 399     if (base::PathExists(out_path))
 400       base::DeleteFile(out_path, false);
 401   }
 402
 403   // Returns a URL that hangs on DNS resolution when using a context created by
 404   // the test fixture.
 405   const GURL& hanging_url() const { return hanging_url_; }
 406
 407   // testing::Test:
 408   void SetUp() override {
 409     SetUpServer();
 410     ASSERT_TRUE(test_server_->Start());
 411
 412     // URL that will hang when lookups reach the host resolver.
 413     hanging_url_ = GURL(base::StringPrintf(
 414         "http://example.com:%d/%s", test_server_->host_port_pair().port(),
 415         kDefaultResponsePath));
 416     ASSERT_TRUE(hanging_url_.is_valid());
 417
 418 #if defined(USE_NSS_CERTS) || defined(OS_IOS)
 419     crypto::EnsureNSSInit();
 420     EnsureNSSHttpIOInit();
 421 #endif
 422   }
 423
 424   void TearDown() override {
 425 #if defined(USE_NSS_CERTS) || defined(OS_IOS)
 426     ShutdownNSSHttpIO();
 427 #endif
 428   }
 429
 430   // Initializes |test_server_| without starting it.  Allows subclasses to use
 431   // their own server configuration.
 432   virtual void SetUpServer() {
 433     test_server_.reset(new SpawnedTestServer(SpawnedTestServer::TYPE_HTTP,
 434                                              SpawnedTestServer::kLocalhost,
 435                                              base::FilePath(kDocRoot)));
 436   }
 437
 438   // Network thread for cross-thread tests.  Most threads just use the main
 439   // thread for network activity.
 440   scoped_ptr<base::Thread> network_thread_;
 441
 442   scoped_ptr<SpawnedTestServer> test_server_;
 443   GURL hanging_url_;
 444
 445   size_t num_upload_streams_created_;
 446 };
 447
 448 namespace {
 449
 450 // Version of URLFetcherTest that tests bad HTTPS requests.
 451 class URLFetcherBadHTTPSTest : public URLFetcherTest {
 452  public:
 453   URLFetcherBadHTTPSTest() {}
 454
 455   // URLFetcherTest:
 456   void SetUpServer() override {
 457     SpawnedTestServer::SSLOptions ssl_options(
 458         SpawnedTestServer::SSLOptions::CERT_EXPIRED);
 459     test_server_.reset(new SpawnedTestServer(
 460         SpawnedTestServer::TYPE_HTTPS, ssl_options, base::FilePath(kDocRoot)));
 461   }
 462 };
 463
 464 // Create the fetcher on the main thread.  Since network IO will happen on the
 465 // main thread, this will test URLFetcher's ability to do everything on one
 466 // thread.
 467 TEST_F(URLFetcherTest, SameThreadTest) {
 468   WaitingURLFetcherDelegate delegate;
 469   delegate.CreateFetcher(test_server_->GetURL(kDefaultResponsePath),
 470                          URLFetcher::GET, CreateSameThreadContextGetter());
 471   delegate.StartFetcherAndWait();
 472
 473   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 474   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 475   std::string data;
 476   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 477   EXPECT_EQ(kDefaultResponseBody, data);
 478
 479   EXPECT_EQ(static_cast<int64_t>(strlen(kDefaultResponseBody)),
 480             delegate.fetcher()->GetReceivedResponseContentLength());
 481   std::string parsed_headers;
 482   base::ReplaceChars(delegate.fetcher()->GetResponseHeaders()->raw_headers(),
 483                      std::string("\0", 1), "\n\r", &parsed_headers);
 484   EXPECT_EQ(static_cast<int64_t>(parsed_headers.size() +
 485                                  strlen(kDefaultResponseBody)),
 486             delegate.fetcher()->GetTotalReceivedBytes());
 487 }
 488
 489 // Create a separate thread that will create the URLFetcher.  A separate thread
 490 // acts as the network thread.
 491 TEST_F(URLFetcherTest, DifferentThreadsTest) {
 492   WaitingURLFetcherDelegate delegate;
 493   delegate.CreateFetcher(test_server_->GetURL(kDefaultResponsePath),
 494                          URLFetcher::GET, CreateCrossThreadContextGetter());
 495   delegate.StartFetcherAndWait();
 496
 497   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 498   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 499   std::string data;
 500   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 501   EXPECT_EQ(kDefaultResponseBody, data);
 502 }
 503
 504 // Tests to make sure CancelAll() will successfully cancel existing URLFetchers.
 505 TEST_F(URLFetcherTest, CancelAll) {
 506   EXPECT_EQ(0, GetNumFetcherCores());
 507
 508   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
 509       CreateSameThreadContextGetter());
 510   // Force context creation.
 511   context_getter->GetURLRequestContext();
 512   MockHostResolver* mock_resolver = context_getter->context()->mock_resolver();
 513
 514   WaitingURLFetcherDelegate delegate;
 515   delegate.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
 516   delegate.fetcher()->Start();
 517   // Wait for the request to reach the mock resolver and hang, to ensure the
 518   // request has actually started.
 519   base::RunLoop().RunUntilIdle();
 520   EXPECT_TRUE(mock_resolver->has_pending_requests());
 521
 522   EXPECT_EQ(1, URLFetcherTest::GetNumFetcherCores());
 523   URLFetcherImpl::CancelAll();
 524   EXPECT_EQ(0, URLFetcherTest::GetNumFetcherCores());
 525 }
 526
 527 TEST_F(URLFetcherTest, DontRetryOnNetworkChangedByDefault) {
 528   EXPECT_EQ(0, GetNumFetcherCores());
 529
 530   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
 531       CreateSameThreadContextGetter());
 532   // Force context creation.
 533   context_getter->GetURLRequestContext();
 534   MockHostResolver* mock_resolver = context_getter->context()->mock_resolver();
 535
 536   WaitingURLFetcherDelegate delegate;
 537   delegate.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
 538   EXPECT_FALSE(mock_resolver->has_pending_requests());
 539
 540   // This posts a task to start the fetcher.
 541   delegate.fetcher()->Start();
 542   base::RunLoop().RunUntilIdle();
 543
 544   // The fetcher is now running, but is pending the host resolve.
 545   EXPECT_EQ(1, GetNumFetcherCores());
 546   EXPECT_TRUE(mock_resolver->has_pending_requests());
 547   ASSERT_FALSE(delegate.did_complete());
 548
 549   // A network change notification aborts the connect job.
 550   NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
 551   delegate.WaitForComplete();
 552   EXPECT_FALSE(mock_resolver->has_pending_requests());
 553
 554   // And the owner of the fetcher gets the ERR_NETWORK_CHANGED error.
 555   EXPECT_EQ(hanging_url(), delegate.fetcher()->GetOriginalURL());
 556   ASSERT_FALSE(delegate.fetcher()->GetStatus().is_success());
 557   EXPECT_EQ(ERR_NETWORK_CHANGED, delegate.fetcher()->GetStatus().error());
 558 }
 559
 560 TEST_F(URLFetcherTest, RetryOnNetworkChangedAndFail) {
 561   EXPECT_EQ(0, GetNumFetcherCores());
 562
 563   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
 564       CreateSameThreadContextGetter());
 565   // Force context creation.
 566   context_getter->GetURLRequestContext();
 567   MockHostResolver* mock_resolver = context_getter->context()->mock_resolver();
 568
 569   WaitingURLFetcherDelegate delegate;
 570   delegate.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
 571   delegate.fetcher()->SetAutomaticallyRetryOnNetworkChanges(3);
 572   EXPECT_FALSE(mock_resolver->has_pending_requests());
 573
 574   // This posts a task to start the fetcher.
 575   delegate.fetcher()->Start();
 576   base::RunLoop().RunUntilIdle();
 577
 578   // The fetcher is now running, but is pending the host resolve.
 579   EXPECT_EQ(1, GetNumFetcherCores());
 580   EXPECT_TRUE(mock_resolver->has_pending_requests());
 581   ASSERT_FALSE(delegate.did_complete());
 582
 583   // Make it fail 3 times.
 584   for (int i = 0; i < 3; ++i) {
 585     // A network change notification aborts the connect job.
 586     NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
 587     base::RunLoop().RunUntilIdle();
 588
 589     // But the fetcher retries automatically.
 590     EXPECT_EQ(1, GetNumFetcherCores());
 591     EXPECT_TRUE(mock_resolver->has_pending_requests());
 592     ASSERT_FALSE(delegate.did_complete());
 593   }
 594
 595   // A 4th failure doesn't trigger another retry, and propagates the error
 596   // to the owner of the fetcher.
 597   NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
 598   delegate.WaitForComplete();
 599   EXPECT_FALSE(mock_resolver->has_pending_requests());
 600
 601   // And the owner of the fetcher gets the ERR_NETWORK_CHANGED error.
 602   EXPECT_EQ(hanging_url(), delegate.fetcher()->GetOriginalURL());
 603   ASSERT_FALSE(delegate.fetcher()->GetStatus().is_success());
 604   EXPECT_EQ(ERR_NETWORK_CHANGED, delegate.fetcher()->GetStatus().error());
 605 }
 606
 607 TEST_F(URLFetcherTest, RetryOnNetworkChangedAndSucceed) {
 608   EXPECT_EQ(0, GetNumFetcherCores());
 609
 610   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
 611       CreateSameThreadContextGetter());
 612   // Force context creation.
 613   context_getter->GetURLRequestContext();
 614   MockHostResolver* mock_resolver = context_getter->context()->mock_resolver();
 615
 616   WaitingURLFetcherDelegate delegate;
 617   delegate.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
 618   delegate.fetcher()->SetAutomaticallyRetryOnNetworkChanges(3);
 619   EXPECT_FALSE(mock_resolver->has_pending_requests());
 620
 621   // This posts a task to start the fetcher.
 622   delegate.fetcher()->Start();
 623   base::RunLoop().RunUntilIdle();
 624
 625   // The fetcher is now running, but is pending the host resolve.
 626   EXPECT_EQ(1, GetNumFetcherCores());
 627   EXPECT_TRUE(mock_resolver->has_pending_requests());
 628   ASSERT_FALSE(delegate.did_complete());
 629
 630   // Make it fail 3 times.
 631   for (int i = 0; i < 3; ++i) {
 632     // A network change notification aborts the connect job.
 633     NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
 634     base::RunLoop().RunUntilIdle();
 635
 636     // But the fetcher retries automatically.
 637     EXPECT_EQ(1, GetNumFetcherCores());
 638     EXPECT_TRUE(mock_resolver->has_pending_requests());
 639     ASSERT_FALSE(delegate.did_complete());
 640   }
 641
 642   // Now let it succeed by resolving the pending request.
 643   mock_resolver->ResolveAllPending();
 644   delegate.WaitForComplete();
 645   EXPECT_FALSE(mock_resolver->has_pending_requests());
 646
 647   // This time the request succeeded.
 648   EXPECT_EQ(hanging_url(), delegate.fetcher()->GetOriginalURL());
 649   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 650   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 651
 652   std::string data;
 653   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 654   EXPECT_EQ(kDefaultResponseBody, data);
 655 }
 656
 657 TEST_F(URLFetcherTest, PostString) {
 658   const char kUploadData[] = "bobsyeruncle";
 659
 660   WaitingURLFetcherDelegate delegate;
 661   delegate.CreateFetcher(test_server_->GetURL("echo"), URLFetcher::POST,
 662                          CreateSameThreadContextGetter());
 663   delegate.fetcher()->SetUploadData("application/x-www-form-urlencoded",
 664                                     kUploadData);
 665   delegate.StartFetcherAndWait();
 666
 667   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 668   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 669   std::string data;
 670   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 671   EXPECT_EQ(kUploadData, data);
 672 }
 673
 674 TEST_F(URLFetcherTest, PostEmptyString) {
 675   const char kUploadData[] = "";
 676
 677   WaitingURLFetcherDelegate delegate;
 678   delegate.CreateFetcher(test_server_->GetURL("echo"), URLFetcher::POST,
 679                          CreateSameThreadContextGetter());
 680   delegate.fetcher()->SetUploadData("application/x-www-form-urlencoded",
 681                                     kUploadData);
 682   delegate.StartFetcherAndWait();
 683
 684   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 685   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 686   std::string data;
 687   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 688   EXPECT_EQ(kUploadData, data);
 689 }
 690
 691 TEST_F(URLFetcherTest, PostEntireFile) {
 692   base::FilePath upload_path = GetUploadFileTestPath();
 693
 694   WaitingURLFetcherDelegate delegate;
 695   delegate.CreateFetcher(test_server_->GetURL("echo"), URLFetcher::POST,
 696                          CreateSameThreadContextGetter());
 697   delegate.fetcher()->SetUploadFilePath("application/x-www-form-urlencoded",
 698                                         upload_path, 0, kuint64max,
 699                                         base::ThreadTaskRunnerHandle::Get());
 700   delegate.StartFetcherAndWait();
 701
 702   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 703   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 704
 705   std::string expected;
 706   ASSERT_TRUE(base::ReadFileToString(upload_path, &expected));
 707   std::string data;
 708   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 709   EXPECT_EQ(expected, data);
 710 }
 711
 712 TEST_F(URLFetcherTest, PostFileRange) {
 713   const size_t kRangeStart = 30;
 714   const size_t kRangeLength = 100;
 715   base::FilePath upload_path = GetUploadFileTestPath();
 716
 717   WaitingURLFetcherDelegate delegate;
 718   delegate.CreateFetcher(test_server_->GetURL("echo"), URLFetcher::POST,
 719                          CreateSameThreadContextGetter());
 720   delegate.fetcher()->SetUploadFilePath("application/x-www-form-urlencoded",
 721                                         upload_path, kRangeStart, kRangeLength,
 722                                         base::ThreadTaskRunnerHandle::Get());
 723   delegate.StartFetcherAndWait();
 724
 725   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 726   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 727
 728   std::string expected;
 729   ASSERT_TRUE(base::ReadFileToString(upload_path, &expected));
 730   std::string data;
 731   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 732   EXPECT_EQ(expected.substr(kRangeStart, kRangeLength), data);
 733 }
 734
 735 TEST_F(URLFetcherTest, PostWithUploadStreamFactory) {
 736   WaitingURLFetcherDelegate delegate;
 737   delegate.CreateFetcher(test_server_->GetURL("echo"), URLFetcher::POST,
 738                          CreateSameThreadContextGetter());
 739   delegate.fetcher()->SetUploadStreamFactory(
 740       "text/plain",
 741       base::Bind(&URLFetcherTest::CreateUploadStream, base::Unretained(this)));
 742   delegate.StartFetcherAndWait();
 743
 744   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 745   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 746   std::string data;
 747   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 748   EXPECT_EQ(kCreateUploadStreamBody, data);
 749   EXPECT_EQ(1u, num_upload_streams_created());
 750 }
 751
 752 TEST_F(URLFetcherTest, PostWithUploadStreamFactoryAndRetries) {
 753   WaitingURLFetcherDelegate delegate;
 754   delegate.CreateFetcher(test_server_->GetURL("echo?status=500"),
 755                          URLFetcher::POST, CreateSameThreadContextGetter());
 756   delegate.fetcher()->SetAutomaticallyRetryOn5xx(true);
 757   delegate.fetcher()->SetMaxRetriesOn5xx(1);
 758   delegate.fetcher()->SetUploadStreamFactory(
 759       "text/plain",
 760       base::Bind(&URLFetcherTest::CreateUploadStream, base::Unretained(this)));
 761   delegate.StartFetcherAndWait();
 762
 763   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 764   EXPECT_EQ(500, delegate.fetcher()->GetResponseCode());
 765   std::string data;
 766   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 767   EXPECT_EQ(kCreateUploadStreamBody, data);
 768   EXPECT_EQ(2u, num_upload_streams_created());
 769 }
 770
 771 // Checks that upload progress increases over time, never exceeds what's already
 772 // been sent, and adds a chunk whenever all previously appended chunks have
 773 // been uploaded.
 774 class CheckUploadProgressDelegate : public WaitingURLFetcherDelegate {
 775  public:
 776   CheckUploadProgressDelegate()
 777       : chunk_(1 << 16, 'a'), num_chunks_appended_(0), last_seen_progress_(0) {}
 778   ~CheckUploadProgressDelegate() override {}
 779
 780   void OnURLFetchUploadProgress(const URLFetcher* source,
 781                                 int64 current,
 782                                 int64 total) override {
 783     // Run default checks.
 784     WaitingURLFetcherDelegate::OnURLFetchUploadProgress(source, current, total);
 785
 786     EXPECT_LE(last_seen_progress_, current);
 787     EXPECT_LE(current, bytes_appended());
 788     last_seen_progress_ = current;
 789     MaybeAppendChunk();
 790   }
 791
 792   // Append the next chunk if all previously appended chunks have been sent.
 793   void MaybeAppendChunk() {
 794     const int kNumChunks = 5;
 795     if (last_seen_progress_ == bytes_appended() &&
 796         num_chunks_appended_ < kNumChunks) {
 797       ++num_chunks_appended_;
 798       fetcher()->AppendChunkToUpload(chunk_,
 799                                      num_chunks_appended_ == kNumChunks);
 800     }
 801   }
 802
 803  private:
 804   int64 bytes_appended() const { return num_chunks_appended_ * chunk_.size(); }
 805
 806   const std::string chunk_;
 807
 808   int64 num_chunks_appended_;
 809   int64 last_seen_progress_;
 810
 811   DISALLOW_COPY_AND_ASSIGN(CheckUploadProgressDelegate);
 812 };
 813
 814 TEST_F(URLFetcherTest, UploadProgress) {
 815   CheckUploadProgressDelegate delegate;
 816   delegate.CreateFetcher(test_server_->GetURL("echo"), URLFetcher::POST,
 817                          CreateSameThreadContextGetter());
 818   // Use a chunked upload so that the upload can be paused after uploading data.
 819   // Since upload progress uses a timer, the delegate may not receive any
 820   // notification otherwise.
 821   delegate.fetcher()->SetChunkedUpload("application/x-www-form-urlencoded");
 822
 823   delegate.fetcher()->Start();
 824   // Append the first chunk.  Others will be appended automatically in response
 825   // to OnURLFetchUploadProgress events.
 826   delegate.MaybeAppendChunk();
 827   delegate.WaitForComplete();
 828
 829   // Make sure there are no pending events that cause problems when run.
 830   base::RunLoop().RunUntilIdle();
 831
 832   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 833   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 834   EXPECT_TRUE(delegate.did_complete());
 835 }
 836
 837 // Checks that download progress never decreases, never exceeds file size, and
 838 // that file size is correctly reported.
 839 class CheckDownloadProgressDelegate : public WaitingURLFetcherDelegate {
 840  public:
 841   CheckDownloadProgressDelegate(int64 file_size)
 842       : file_size_(file_size), last_seen_progress_(0) {}
 843   ~CheckDownloadProgressDelegate() override {}
 844
 845   void OnURLFetchDownloadProgress(const URLFetcher* source,
 846                                   int64 current,
 847                                   int64 total) override {
 848     // Run default checks.
 849     WaitingURLFetcherDelegate::OnURLFetchDownloadProgress(source, current,
 850                                                           total);
 851
 852     EXPECT_LE(last_seen_progress_, current);
 853     EXPECT_EQ(file_size_, total);
 854     last_seen_progress_ = current;
 855   }
 856
 857  private:
 858   int64 file_size_;
 859   int64 last_seen_progress_;
 860
 861   DISALLOW_COPY_AND_ASSIGN(CheckDownloadProgressDelegate);
 862 };
 863
 864 TEST_F(URLFetcherTest, DownloadProgress) {
 865   // Get a file large enough to require more than one read into
 866   // URLFetcher::Core's IOBuffer.
 867   const char kFileToFetch[] = "animate1.gif";
 868
 869   std::string file_contents;
 870   ASSERT_TRUE(base::ReadFileToString(
 871       test_server_->GetDocumentRoot().AppendASCII(kFileToFetch),
 872       &file_contents));
 873
 874   CheckDownloadProgressDelegate delegate(file_contents.size());
 875   delegate.CreateFetcher(
 876       test_server_->GetURL(std::string(kTestServerFilePrefix) + kFileToFetch),
 877       URLFetcher::GET, CreateSameThreadContextGetter());
 878   delegate.StartFetcherAndWait();
 879
 880   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 881   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 882   std::string data;
 883   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 884   EXPECT_EQ(file_contents, data);
 885 }
 886
 887 class CancelOnUploadProgressDelegate : public WaitingURLFetcherDelegate {
 888  public:
 889   CancelOnUploadProgressDelegate() {}
 890   ~CancelOnUploadProgressDelegate() override {}
 891
 892   void OnURLFetchUploadProgress(const URLFetcher* source,
 893                                 int64 current,
 894                                 int64 total) override {
 895     CancelFetch();
 896   }
 897
 898  private:
 899   DISALLOW_COPY_AND_ASSIGN(CancelOnUploadProgressDelegate);
 900 };
 901
 902 // Check that a fetch can be safely cancelled/deleted during an upload progress
 903 // callback.
 904 TEST_F(URLFetcherTest, CancelInUploadProgressCallback) {
 905   CancelOnUploadProgressDelegate delegate;
 906   delegate.CreateFetcher(test_server_->GetURL("echo"), URLFetcher::POST,
 907                          CreateSameThreadContextGetter());
 908   delegate.fetcher()->SetChunkedUpload("application/x-www-form-urlencoded");
 909   delegate.fetcher()->Start();
 910   // Use a chunked upload so that the upload can be paused after uploading data.
 911   // Since uploads progress uses a timer, may not receive any notification,
 912   // otherwise.
 913   std::string upload_data(1 << 16, 'a');
 914   delegate.fetcher()->AppendChunkToUpload(upload_data, false);
 915   delegate.WaitForComplete();
 916
 917   // Make sure there are no pending events that cause problems when run.
 918   base::RunLoop().RunUntilIdle();
 919
 920   EXPECT_FALSE(delegate.did_complete());
 921   EXPECT_FALSE(delegate.fetcher());
 922 }
 923
 924 class CancelOnDownloadProgressDelegate : public WaitingURLFetcherDelegate {
 925  public:
 926   CancelOnDownloadProgressDelegate() {}
 927   ~CancelOnDownloadProgressDelegate() override {}
 928
 929   void OnURLFetchDownloadProgress(const URLFetcher* source,
 930                                   int64 current,
 931                                   int64 total) override {
 932     CancelFetch();
 933   }
 934
 935  private:
 936   DISALLOW_COPY_AND_ASSIGN(CancelOnDownloadProgressDelegate);
 937 };
 938
 939 // Check that a fetch can be safely cancelled/deleted during a download progress
 940 // callback.
 941 TEST_F(URLFetcherTest, CancelInDownloadProgressCallback) {
 942   // Get a file large enough to require more than one read into
 943   // URLFetcher::Core's IOBuffer.
 944   static const char kFileToFetch[] = "animate1.gif";
 945   CancelOnDownloadProgressDelegate delegate;
 946   delegate.CreateFetcher(
 947       test_server_->GetURL(std::string(kTestServerFilePrefix) + kFileToFetch),
 948       URLFetcher::GET, CreateSameThreadContextGetter());
 949   delegate.StartFetcherAndWait();
 950
 951   // Make sure there are no pending events that cause problems when run.
 952   base::RunLoop().RunUntilIdle();
 953
 954   EXPECT_FALSE(delegate.did_complete());
 955   EXPECT_FALSE(delegate.fetcher());
 956 }
 957
 958 TEST_F(URLFetcherTest, Headers) {
 959   WaitingURLFetcherDelegate delegate;
 960   delegate.CreateFetcher(
 961       test_server_->GetURL("set-header?cache-control: private"),
 962       URLFetcher::GET, CreateSameThreadContextGetter());
 963   delegate.StartFetcherAndWait();
 964
 965   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 966   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 967   std::string header;
 968   ASSERT_TRUE(delegate.fetcher()->GetResponseHeaders()->GetNormalizedHeader(
 969       "cache-control", &header));
 970   EXPECT_EQ("private", header);
 971 }
 972
 973 TEST_F(URLFetcherTest, SocketAddress) {
 974   WaitingURLFetcherDelegate delegate;
 975   delegate.CreateFetcher(test_server_->GetURL(kDefaultResponsePath),
 976                          URLFetcher::GET, CreateSameThreadContextGetter());
 977   delegate.StartFetcherAndWait();
 978
 979   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 980   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 981   EXPECT_EQ(test_server_->host_port_pair().port(),
 982             delegate.fetcher()->GetSocketAddress().port());
 983   EXPECT_EQ(test_server_->host_port_pair().host(),
 984             delegate.fetcher()->GetSocketAddress().host());
 985 }
 986
 987 TEST_F(URLFetcherTest, StopOnRedirect) {
 988   const char kRedirectTarget[] = "http://redirect.target.com";
 989
 990   WaitingURLFetcherDelegate delegate;
 991   delegate.CreateFetcher(
 992       test_server_->GetURL(std::string("server-redirect?") + kRedirectTarget),
 993       URLFetcher::GET, CreateSameThreadContextGetter());
 994   delegate.fetcher()->SetStopOnRedirect(true);
 995   delegate.StartFetcherAndWait();
 996
 997   EXPECT_EQ(GURL(kRedirectTarget), delegate.fetcher()->GetURL());
 998   EXPECT_EQ(URLRequestStatus::CANCELED,
 999             delegate.fetcher()->GetStatus().status());
1000   EXPECT_EQ(ERR_ABORTED, delegate.fetcher()->GetStatus().error());
1001   EXPECT_EQ(301, delegate.fetcher()->GetResponseCode());
1002 }
1003
1004 TEST_F(URLFetcherTest, ThrottleOnRepeatedFetches) {
1005   base::Time start_time = Time::Now();
1006   GURL url(test_server_->GetURL(kDefaultResponsePath));
1007
1008   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
1009       CreateSameThreadContextGetter());
1010
1011   // Registers an entry for test url. It only allows 3 requests to be sent
1012   // in 200 milliseconds.
1013   context_getter->AddThrottlerEntry(
1014       url, std::string() /* url_id */, 200 /* sliding_window_period_ms */,
1015       3 /* max_send_threshold */, 1 /* initial_backoff_ms */,
1016       2.0 /* multiply_factor */, 0.0 /* jitter_factor */,
1017       256 /* maximum_backoff_ms */,
1018       false /* reserve_sending_time_for_next_request*/);
1019
1020   for (int i = 0; i < 20; ++i) {
1021     WaitingURLFetcherDelegate delegate;
1022     delegate.CreateFetcher(url, URLFetcher::GET, context_getter);
1023     delegate.StartFetcherAndWait();
1024
1025     EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
1026     EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
1027   }
1028
1029   // 20 requests were sent. Due to throttling, they should have collectively
1030   // taken over 1 second.
1031   EXPECT_GE(Time::Now() - start_time, base::TimeDelta::FromSeconds(1));
1032 }
1033
1034 TEST_F(URLFetcherTest, ThrottleOn5xxRetries) {
1035   base::Time start_time = Time::Now();
1036   GURL url(test_server_->GetURL("files/server-unavailable.html"));
1037
1038   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
1039       CreateSameThreadContextGetter());
1040
1041   // Registers an entry for test url. The backoff time is calculated by:
1042   //     new_backoff = 2.0 * old_backoff + 0
1043   // and maximum backoff time is 256 milliseconds.
1044   // Maximum retries allowed is set to 11.
1045   context_getter->AddThrottlerEntry(
1046       url, std::string() /* url_id */, 200 /* sliding_window_period_ms */,
1047       3 /* max_send_threshold */, 1 /* initial_backoff_ms */,
1048       2.0 /* multiply_factor */, 0.0 /* jitter_factor */,
1049       256 /* maximum_backoff_ms */,
1050       false /* reserve_sending_time_for_next_request*/);
1051
1052   WaitingURLFetcherDelegate delegate;
1053   delegate.CreateFetcher(url, URLFetcher::GET, context_getter);
1054   delegate.fetcher()->SetAutomaticallyRetryOn5xx(true);
1055   delegate.fetcher()->SetMaxRetriesOn5xx(11);
1056   delegate.StartFetcherAndWait();
1057
1058   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
1059   EXPECT_EQ(503, delegate.fetcher()->GetResponseCode());
1060   std::string data;
1061   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
1062   EXPECT_FALSE(data.empty());
1063
1064   // The request should have been retried 11 times (12 times including the first
1065   // attempt).  Due to throttling, they should have collectively taken over 1
1066   // second.
1067   EXPECT_GE(Time::Now() - start_time, base::TimeDelta::FromSeconds(1));
1068 }
1069
1070 // Tests overload protection, when responses passed through.
1071 TEST_F(URLFetcherTest, ProtectTestPassedThrough) {
1072   base::Time start_time = Time::Now();
1073   GURL url(test_server_->GetURL("files/server-unavailable.html"));
1074
1075   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
1076       CreateSameThreadContextGetter());
1077
1078   // Registers an entry for test url. The backoff time is calculated by:
1079   //     new_backoff = 2.0 * old_backoff + 0
1080   // and maximum backoff time is 150000 milliseconds.
1081   // Maximum retries allowed is set to 11.
1082   // Total time if *not* for not doing automatic backoff would be 150s.
1083   // In reality it should be "as soon as server responds".
1084   context_getter->AddThrottlerEntry(
1085       url, std::string() /* url_id */, 200 /* sliding_window_period_ms */,
1086       3 /* max_send_threshold */, 10000 /* initial_backoff_ms */,
1087       2.0 /* multiply_factor */, 0.0 /* jitter_factor */,
1088       150000 /* maximum_backoff_ms */,
1089       false /* reserve_sending_time_for_next_request*/);
1090
1091   WaitingURLFetcherDelegate delegate;
1092   delegate.CreateFetcher(url, URLFetcher::GET, context_getter);
1093   delegate.fetcher()->SetAutomaticallyRetryOn5xx(false);
1094   delegate.fetcher()->SetMaxRetriesOn5xx(11);
1095   delegate.StartFetcherAndWait();
1096
1097   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
1098   EXPECT_EQ(503, delegate.fetcher()->GetResponseCode());
1099   std::string data;
1100   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
1101   EXPECT_FALSE(data.empty());
1102   EXPECT_GT(delegate.fetcher()->GetBackoffDelay().InMicroseconds(), 0);
1103
1104   // The request should not have been retried at all.  If it had attempted all
1105   // 11 retries, that should have taken 2.5 minutes.
1106   EXPECT_TRUE(Time::Now() - start_time < TimeDelta::FromMinutes(1));
1107 }
1108
1109 // Used to check if a callback has been invoked.
1110 void SetBoolToTrue(bool* ptr) {
1111   *ptr = true;
1112 }
1113
1114 // Make sure that the URLFetcher cancels the URLRequest and releases its context
1115 // getter pointer synchronously when the fetcher and request context live on
1116 // the same thread.
1117 TEST_F(URLFetcherTest, CancelSameThread) {
1118   WaitingURLFetcherDelegate delegate;
1119   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
1120       CreateSameThreadContextGetter());
1121   bool getter_was_destroyed = false;
1122   context_getter->set_on_destruction_callback(
1123       base::Bind(&SetBoolToTrue, &getter_was_destroyed));
1124   delegate.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
1125
1126   // The getter won't be destroyed if the test holds on to a reference to it.
1127   context_getter = nullptr;
1128
1129   delegate.fetcher()->Start();
1130   // Give the fetcher a chance to start the request.
1131   base::RunLoop().RunUntilIdle();
1132   EXPECT_EQ(1, URLFetcherTest::GetNumFetcherCores());
1133
1134   // On same-thread cancel, the request should be canceled and getter destroyed
1135   // synchronously, for safe shutdown.
1136   delegate.CancelFetch();
1137   EXPECT_EQ(0, URLFetcherTest::GetNumFetcherCores());
1138   EXPECT_TRUE(getter_was_destroyed);
1139 }
1140
1141 // Make sure that the URLFetcher releases its context getter pointer on
1142 // cancellation, cross-thread case.
1143 TEST_F(URLFetcherTest, CancelDifferentThreads) {
1144   base::RunLoop run_loop_;
1145
1146   WaitingURLFetcherDelegate delegate;
1147   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
1148       CreateCrossThreadContextGetter());
1149   context_getter->set_on_destruction_callback(base::Bind(
1150       base::IgnoreResult(&base::SingleThreadTaskRunner::PostTask),
1151       base::ThreadTaskRunnerHandle::Get(), FROM_HERE, run_loop_.QuitClosure()));
1152   delegate.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
1153
1154   // The getter won't be destroyed if the test holds on to a reference to it.
1155   context_getter = nullptr;
1156
1157   delegate.fetcher()->Start();
1158   delegate.CancelFetch();
1159   run_loop_.Run();
1160
1161   EXPECT_FALSE(delegate.did_complete());
1162 }
1163
1164 TEST_F(URLFetcherTest, CancelWhileDelayedByThrottleDifferentThreads) {
1165   GURL url = test_server_->GetURL(kDefaultResponsePath);
1166   base::RunLoop run_loop_;
1167
1168   WaitingURLFetcherDelegate delegate;
1169   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
1170       CreateCrossThreadContextGetter());
1171   context_getter->set_on_destruction_callback(base::Bind(
1172       base::IgnoreResult(&base::SingleThreadTaskRunner::PostTask),
1173       base::ThreadTaskRunnerHandle::Get(), FROM_HERE, run_loop_.QuitClosure()));
1174   delegate.CreateFetcher(url, URLFetcher::GET, context_getter);
1175
1176   // Register an entry for test url using a sliding window of 400 seconds, and
1177   // max of 1 request.  Also simulate a request having just started, so the
1178   // next request will be affected by backoff of ~400 seconds.
1179   context_getter->AddThrottlerEntry(
1180       url, std::string() /* url_id */, 400000 /* sliding_window_period_ms */,
1181       1 /* max_send_threshold */, 200000 /* initial_backoff_ms */,
1182       2.0 /* multiply_factor */, 0.0 /* jitter_factor */,
1183       400000 /* maximum_backoff_ms */,
1184       true /* reserve_sending_time_for_next_request*/);
1185
1186   // The getter won't be destroyed if the test holds on to a reference to it.
1187   context_getter = nullptr;
1188
1189   delegate.fetcher()->Start();
1190   delegate.CancelFetch();
1191   run_loop_.Run();
1192
1193   EXPECT_FALSE(delegate.did_complete());
1194 }
1195
1196 // A URLFetcherDelegate that expects to receive a response body of "request1"
1197 // and then reuses the fetcher for the same URL, setting the "test" request
1198 // header to "request2".
1199 class ReuseFetcherDelegate : public WaitingURLFetcherDelegate {
1200  public:
1201   // |second_request_context_getter| is the context getter used for the second
1202   // request. Can't reuse the old one because fetchers release it on completion.
1203   ReuseFetcherDelegate(
1204       scoped_refptr<URLRequestContextGetter> second_request_context_getter)
1205       : first_request_complete_(false),
1206         second_request_context_getter_(second_request_context_getter) {}
1207
1208   ~ReuseFetcherDelegate() override {}
1209
1210   void OnURLFetchComplete(const URLFetcher* source) override {
1211     EXPECT_EQ(fetcher(), source);
1212     if (!first_request_complete_) {
1213       first_request_complete_ = true;
1214       EXPECT_TRUE(fetcher()->GetStatus().is_success());
1215       EXPECT_EQ(200, fetcher()->GetResponseCode());
1216       std::string data;
1217       ASSERT_TRUE(fetcher()->GetResponseAsString(&data));
1218       EXPECT_EQ("request1", data);
1219
1220       fetcher()->SetRequestContext(second_request_context_getter_.get());
1221       fetcher()->SetExtraRequestHeaders("test: request2");
1222       fetcher()->Start();
1223       return;
1224     }
1225     WaitingURLFetcherDelegate::OnURLFetchComplete(source);
1226   }
1227
1228  private:
1229   bool first_request_complete_;
1230   scoped_refptr<URLRequestContextGetter> second_request_context_getter_;
1231
1232   DISALLOW_COPY_AND_ASSIGN(ReuseFetcherDelegate);
1233 };
1234
1235 TEST_F(URLFetcherTest, ReuseFetcherForSameURL) {
1236   // TODO(mmenke):  It's really weird that this is supported, particularly
1237   // some fields can be modified between requests, but some (Like upload body)
1238   // cannot be. Can we get rid of support for this?
1239   scoped_refptr<URLRequestContextGetter> context_getter(
1240       CreateSameThreadContextGetter());
1241   ReuseFetcherDelegate delegate(context_getter);
1242   delegate.CreateFetcher(test_server_->GetURL("echoheader?test"),
1243                          URLFetcher::GET, context_getter);
1244   delegate.fetcher()->SetExtraRequestHeaders("test: request1");
1245   delegate.StartFetcherAndWait();
1246
1247   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
1248   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
1249   std::string data;
1250   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
1251   EXPECT_EQ("request2", data);
1252 }
1253
1254 TEST_F(URLFetcherTest, ShutdownSameThread) {
1255   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
1256       CreateSameThreadContextGetter());
1257
1258   // Create a fetcher and wait for it to create a request.
1259   WaitingURLFetcherDelegate delegate1;
1260   delegate1.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
1261   delegate1.fetcher()->Start();
1262   // Need to spin the loop to ensure the URLRequest is created and started.
1263   base::RunLoop().RunUntilIdle();
1264
1265   // Create and start another fetcher, but don't wait for it to start. The task
1266   // to start the request should be in the message loop.
1267   WaitingURLFetcherDelegate delegate2;
1268   delegate2.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
1269   delegate2.fetcher()->Start();
1270
1271   // Check that shutting down the getter cancels the request synchronously,
1272   // allowing the context to be destroyed.
1273   context_getter->Shutdown();
1274
1275   // Wait for the first fetcher, make sure it failed.
1276   delegate1.WaitForComplete();
1277   EXPECT_FALSE(delegate1.fetcher()->GetStatus().is_success());
1278   EXPECT_EQ(ERR_CONTEXT_SHUT_DOWN, delegate1.fetcher()->GetStatus().error());
1279
1280   // Wait for the second fetcher, make sure it failed.
1281   delegate2.WaitForComplete();
1282   EXPECT_FALSE(delegate2.fetcher()->GetStatus().is_success());
1283   EXPECT_EQ(ERR_CONTEXT_SHUT_DOWN, delegate2.fetcher()->GetStatus().error());
1284
1285   // New fetchers should automatically fail without making new requests. This
1286   // should follow the same path as the second fetcher, but best to be safe.
1287   WaitingURLFetcherDelegate delegate3;
1288   delegate3.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
1289   delegate3.fetcher()->Start();
1290   delegate3.WaitForComplete();
1291   EXPECT_FALSE(delegate3.fetcher()->GetStatus().is_success());
1292   EXPECT_EQ(ERR_CONTEXT_SHUT_DOWN, delegate3.fetcher()->GetStatus().error());
1293 }
1294
1295 TEST_F(URLFetcherTest, ShutdownCrossThread) {
1296   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
1297       CreateCrossThreadContextGetter());
1298
1299   WaitingURLFetcherDelegate delegate1;
1300   delegate1.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
1301   delegate1.fetcher()->Start();
1302   // Check that shutting the context getter lets the context be destroyed safely
1303   // and cancels the request.
1304   context_getter->Shutdown();
1305   delegate1.WaitForComplete();
1306   EXPECT_FALSE(delegate1.fetcher()->GetStatus().is_success());
1307   EXPECT_EQ(ERR_CONTEXT_SHUT_DOWN, delegate1.fetcher()->GetStatus().error());
1308
1309   // New requests should automatically fail without making new requests.
1310   WaitingURLFetcherDelegate delegate2;
1311   delegate2.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
1312   delegate2.StartFetcherAndWait();
1313   EXPECT_FALSE(delegate2.fetcher()->GetStatus().is_success());
1314   EXPECT_EQ(ERR_CONTEXT_SHUT_DOWN, delegate2.fetcher()->GetStatus().error());
1315 }
1316
1317 // Get a small file.
1318 TEST_F(URLFetcherTest, FileTestSmallGet) {
1319   const char kFileToFetch[] = "simple.html";
1320
1321   base::ScopedTempDir temp_dir;
1322   ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
1323   base::FilePath out_path = temp_dir.path().AppendASCII(kFileToFetch);
1324   SaveFileTest(kFileToFetch, false, out_path, false);
1325 }
1326
1327 // Get a file large enough to require more than one read into URLFetcher::Core's
1328 // IOBuffer.
1329 TEST_F(URLFetcherTest, FileTestLargeGet) {
1330   const char kFileToFetch[] = "animate1.gif";
1331
1332   base::ScopedTempDir temp_dir;
1333   ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
1334   base::FilePath out_path = temp_dir.path().AppendASCII(kFileToFetch);
1335   SaveFileTest(kFileToFetch, false, out_path, false);
1336 }
1337
1338 // If the caller takes the ownership of the output file, the file should persist
1339 // even after URLFetcher is gone.
1340 TEST_F(URLFetcherTest, FileTestTakeOwnership) {
1341   const char kFileToFetch[] = "simple.html";
1342
1343   base::ScopedTempDir temp_dir;
1344   ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
1345   base::FilePath out_path = temp_dir.path().AppendASCII(kFileToFetch);
1346   SaveFileTest(kFileToFetch, false, out_path, true);
1347 }
1348
1349 // Test that an existing file can be overwritten be a fetcher.
1350 TEST_F(URLFetcherTest, FileTestOverwriteExisting) {
1351   base::ScopedTempDir temp_dir;
1352   ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
1353
1354   // Create a file before trying to fetch.
1355   const char kFileToFetch[] = "simple.html";
1356   std::string data(10000, '?');  // Meant to be larger than simple.html.
1357   base::FilePath out_path = temp_dir.path().AppendASCII(kFileToFetch);
1358   ASSERT_EQ(static_cast<int>(data.size()),
1359             base::WriteFile(out_path, data.data(), data.size()));
1360   ASSERT_TRUE(base::PathExists(out_path));
1361
1362   SaveFileTest(kFileToFetch, false, out_path, true);
1363 }
1364
1365 // Test trying to overwrite a directory with a file when using a fetcher fails.
1366 TEST_F(URLFetcherTest, FileTestTryToOverwriteDirectory) {
1367   base::ScopedTempDir temp_dir;
1368   ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
1369
1370   // Create a directory before trying to fetch.
1371   static const char kFileToFetch[] = "simple.html";
1372   base::FilePath out_path = temp_dir.path().AppendASCII(kFileToFetch);
1373   ASSERT_TRUE(base::CreateDirectory(out_path));
1374   ASSERT_TRUE(base::PathExists(out_path));
1375
1376   WaitingURLFetcherDelegate delegate;
1377   delegate.CreateFetcher(
1378       test_server_->GetURL(std::string(kTestServerFilePrefix) + kFileToFetch),
1379       URLFetcher::GET, CreateSameThreadContextGetter());
1380   delegate.fetcher()->SaveResponseToFileAtPath(
1381       out_path, scoped_refptr<base::SingleThreadTaskRunner>(
1382                     base::ThreadTaskRunnerHandle::Get()));
1383   delegate.StartFetcherAndWait();
1384
1385   EXPECT_FALSE(delegate.fetcher()->GetStatus().is_success());
1386   EXPECT_EQ(ERR_ACCESS_DENIED, delegate.fetcher()->GetStatus().error());
1387 }
1388
1389 // Get a small file and save it to a temp file.
1390 TEST_F(URLFetcherTest, TempFileTestSmallGet) {
1391   SaveFileTest("simple.html", true, base::FilePath(), false);
1392 }
1393
1394 // Get a file large enough to require more than one read into URLFetcher::Core's
1395 // IOBuffer and save it to a temp file.
1396 TEST_F(URLFetcherTest, TempFileTestLargeGet) {
1397   SaveFileTest("animate1.gif", true, base::FilePath(), false);
1398 }
1399
1400 // If the caller takes the ownership of the temp file, check that the file
1401 // persists even after URLFetcher is gone.
1402 TEST_F(URLFetcherTest, TempFileTestTakeOwnership) {
1403   SaveFileTest("simple.html", true, base::FilePath(), true);
1404 }
1405
1406 TEST_F(URLFetcherBadHTTPSTest, BadHTTPS) {
1407   WaitingURLFetcherDelegate delegate;
1408   delegate.CreateFetcher(test_server_->GetURL(kDefaultResponsePath),
1409                          URLFetcher::GET, CreateSameThreadContextGetter());
1410   delegate.StartFetcherAndWait();
1411
1412   EXPECT_EQ(URLRequestStatus::CANCELED,
1413             delegate.fetcher()->GetStatus().status());
1414   EXPECT_EQ(ERR_ABORTED, delegate.fetcher()->GetStatus().error());
1415   EXPECT_EQ(-1, delegate.fetcher()->GetResponseCode());
1416   EXPECT_TRUE(delegate.fetcher()->GetCookies().empty());
1417   std::string data;
1418   EXPECT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
1419   EXPECT_TRUE(data.empty());
1420 }
1421
1422 }  // namespace
1423
1424 }  // namespace net