net/url_request/url_fetcher_impl_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_fetcher_impl.h"
   6
   7 #include <stdint.h>
   8 #include <string.h>
   9
  10 #include <algorithm>
  11 #include <string>
  12
  13 #include "base/bind.h"
  14 #include "base/files/file_util.h"
  15 #include "base/files/scoped_temp_dir.h"
  16 #include "base/location.h"
  17 #include "base/macros.h"
  18 #include "base/memory/scoped_ptr.h"
  19 #include "base/path_service.h"
  20 #include "base/run_loop.h"
  21 #include "base/single_thread_task_runner.h"
  22 #include "base/strings/string_util.h"
  23 #include "base/strings/stringprintf.h"
  24 #include "base/synchronization/waitable_event.h"
  25 #include "base/thread_task_runner_handle.h"
  26 #include "base/threading/thread.h"
  27 #include "build/build_config.h"
  28 #include "crypto/nss_util.h"
  29 #include "net/base/elements_upload_data_stream.h"
  30 #include "net/base/network_change_notifier.h"
  31 #include "net/base/upload_bytes_element_reader.h"
  32 #include "net/base/upload_element_reader.h"
  33 #include "net/base/upload_file_element_reader.h"
  34 #include "net/dns/mock_host_resolver.h"
  35 #include "net/http/http_response_headers.h"
  36 #include "net/test/spawned_test_server/spawned_test_server.h"
  37 #include "net/url_request/url_fetcher_delegate.h"
  38 #include "net/url_request/url_request_context_getter.h"
  39 #include "net/url_request/url_request_test_util.h"
  40 #include "net/url_request/url_request_throttler_manager.h"
  41 #include "testing/gtest/include/gtest/gtest.h"
  42
  43 #if defined(USE_NSS_CERTS) || defined(OS_IOS)
  44 #include "net/cert_net/nss_ocsp.h"
  45 #endif
  46
  47 namespace net {
  48
  49 using base::Time;
  50 using base::TimeDelta;
  51
  52 // TODO(eroman): Add a regression test for http://crbug.com/40505.
  53
  54 namespace {
  55
  56 // TODO(akalin): Move all the test data to somewhere under net/.
  57 const base::FilePath::CharType kDocRoot[] =
  58     FILE_PATH_LITERAL("net/data/url_fetcher_impl_unittest");
  59 const char kTestServerFilePrefix[] = "files/";
  60
  61 // Test server path and response body for the default URL used by many of the
  62 // tests.
  63 const char kDefaultResponsePath[] = "defaultresponse";
  64 const char kDefaultResponseBody[] =
  65     "Default response given for path: /defaultresponse";
  66
  67 // Request body for streams created by CreateUploadStream.
  68 const char kCreateUploadStreamBody[] = "rosebud";
  69
  70 base::FilePath GetUploadFileTestPath() {
  71   base::FilePath path;
  72   PathService::Get(base::DIR_SOURCE_ROOT, &path);
  73   return path.Append(
  74       FILE_PATH_LITERAL("net/data/url_request_unittest/BullRunSpeech.txt"));
  75 }
  76
  77 // Simple URLRequestDelegate that waits for the specified fetcher to complete.
  78 // Can only be used once.
  79 class WaitingURLFetcherDelegate : public URLFetcherDelegate {
  80  public:
  81   WaitingURLFetcherDelegate() : did_complete_(false) {}
  82
  83   void CreateFetcher(
  84       const GURL& url,
  85       URLFetcher::RequestType request_type,
  86       scoped_refptr<net::URLRequestContextGetter> context_getter) {
  87     fetcher_.reset(new URLFetcherImpl(url, request_type, this));
  88     fetcher_->SetRequestContext(context_getter.get());
  89   }
  90
  91   URLFetcher* fetcher() const { return fetcher_.get(); }
  92
  93   // Wait until the request has completed or been canceled.
  94   void StartFetcherAndWait() {
  95     fetcher_->Start();
  96     WaitForComplete();
  97   }
  98
  99   // Wait until the request has completed or been canceled. Does not start the
 100   // request.
 101   void WaitForComplete() { run_loop_.Run(); }
 102
 103   // Cancels the fetch by deleting the fetcher.
 104   void CancelFetch() {
 105     EXPECT_TRUE(fetcher_);
 106     fetcher_.reset();
 107     run_loop_.Quit();
 108   }
 109
 110   // URLFetcherDelegate:
 111   void OnURLFetchComplete(const URLFetcher* source) override {
 112     EXPECT_FALSE(did_complete_);
 113     EXPECT_TRUE(fetcher_);
 114     EXPECT_EQ(fetcher_.get(), source);
 115     did_complete_ = true;
 116     run_loop_.Quit();
 117   }
 118
 119   void OnURLFetchDownloadProgress(const URLFetcher* source,
 120                                   int64 current,
 121                                   int64 total) override {
 122     // Note that the current progress may be greater than the previous progress,
 123     // in the case of retrying the request.
 124     EXPECT_FALSE(did_complete_);
 125     EXPECT_TRUE(fetcher_);
 126     EXPECT_EQ(source, fetcher_.get());
 127
 128     EXPECT_LE(0, current);
 129     // If file size is not known, |total| is -1.
 130     if (total >= 0)
 131       EXPECT_LE(current, total);
 132   }
 133
 134   void OnURLFetchUploadProgress(const URLFetcher* source,
 135                                 int64 current,
 136                                 int64 total) override {
 137     // Note that the current progress may be greater than the previous progress,
 138     // in the case of retrying the request.
 139     EXPECT_FALSE(did_complete_);
 140     EXPECT_TRUE(fetcher_);
 141     EXPECT_EQ(source, fetcher_.get());
 142
 143     EXPECT_LE(0, current);
 144     // If file size is not known, |total| is -1.
 145     if (total >= 0)
 146       EXPECT_LE(current, total);
 147   }
 148
 149   bool did_complete() const { return did_complete_; }
 150
 151  private:
 152   bool did_complete_;
 153
 154   scoped_ptr<URLFetcherImpl> fetcher_;
 155   base::RunLoop run_loop_;
 156
 157   DISALLOW_COPY_AND_ASSIGN(WaitingURLFetcherDelegate);
 158 };
 159
 160 // A TestURLRequestContext with a ThrottleManager and a MockHostResolver.
 161 class FetcherTestURLRequestContext : public TestURLRequestContext {
 162  public:
 163   // All requests for |hanging_domain| will hang on host resolution until the
 164   // mock_resolver()->ResolveAllPending() is called.
 165   explicit FetcherTestURLRequestContext(const std::string& hanging_domain)
 166       : TestURLRequestContext(true), mock_resolver_(new MockHostResolver()) {
 167     mock_resolver_->set_ondemand_mode(true);
 168     mock_resolver_->rules()->AddRule(hanging_domain, "127.0.0.1");
 169     // Pass ownership to ContextStorage to ensure correct destruction order.
 170     context_storage_.set_host_resolver(
 171         scoped_ptr<HostResolver>(mock_resolver_));
 172     context_storage_.set_throttler_manager(
 173         make_scoped_ptr(new URLRequestThrottlerManager()));
 174     Init();
 175   }
 176
 177   MockHostResolver* mock_resolver() { return mock_resolver_; }
 178
 179  private:
 180   MockHostResolver* mock_resolver_;
 181
 182   DISALLOW_COPY_AND_ASSIGN(FetcherTestURLRequestContext);
 183 };
 184
 185 class FetcherTestURLRequestContextGetter : public URLRequestContextGetter {
 186  public:
 187   FetcherTestURLRequestContextGetter(
 188       scoped_refptr<base::SingleThreadTaskRunner> network_task_runner,
 189       const std::string& hanging_domain)
 190       : network_task_runner_(network_task_runner),
 191         hanging_domain_(hanging_domain),
 192         shutting_down_(false) {}
 193
 194   // Sets callback to be invoked when the getter is destroyed.
 195   void set_on_destruction_callback(
 196       const base::Closure& on_destruction_callback) {
 197     on_destruction_callback_ = on_destruction_callback;
 198   }
 199
 200   // URLRequestContextGetter:
 201   FetcherTestURLRequestContext* GetURLRequestContext() override {
 202     // Calling this on the wrong thread may be either a bug in the test or a bug
 203     // in production code.
 204     EXPECT_TRUE(network_task_runner_->BelongsToCurrentThread());
 205
 206     if (shutting_down_)
 207       return nullptr;
 208
 209     if (!context_)
 210       context_.reset(new FetcherTestURLRequestContext(hanging_domain_));
 211     return context_.get();
 212   }
 213
 214   scoped_refptr<base::SingleThreadTaskRunner> GetNetworkTaskRunner()
 215       const override {
 216     return network_task_runner_;
 217   }
 218
 219   // Adds a throttler entry with the specified parameters.  Does this
 220   // synchronously if the context lives on the current thread, or posts a task
 221   // to the relevant thread otherwise.
 222   //
 223   // If |reserve_sending_time_for_next_request|, will start backoff early, as
 224   // if there has already been a request for |url|.
 225   void AddThrottlerEntry(const GURL& url,
 226                          const std::string& url_id,
 227                          int sliding_window_period_ms,
 228                          int max_send_threshold,
 229                          int initial_backoff_ms,
 230                          double multiply_factor,
 231                          double jitter_factor,
 232                          int maximum_backoff_ms,
 233                          bool reserve_sending_time_for_next_request) {
 234     if (!network_task_runner_->RunsTasksOnCurrentThread()) {
 235       network_task_runner_->PostTask(
 236           FROM_HERE,
 237           base::Bind(&FetcherTestURLRequestContextGetter::AddThrottlerEntry,
 238                      this, url, url_id, sliding_window_period_ms,
 239                      max_send_threshold, initial_backoff_ms, multiply_factor,
 240                      jitter_factor, maximum_backoff_ms,
 241                      reserve_sending_time_for_next_request));
 242       return;
 243     }
 244     scoped_refptr<URLRequestThrottlerEntry> entry(new URLRequestThrottlerEntry(
 245         GetURLRequestContext()->throttler_manager(), url_id,
 246         sliding_window_period_ms, max_send_threshold, initial_backoff_ms,
 247         multiply_factor, jitter_factor, maximum_backoff_ms));
 248
 249     GetURLRequestContext()->throttler_manager()->OverrideEntryForTests(
 250         url, entry.get());
 251
 252     if (reserve_sending_time_for_next_request)
 253       entry->ReserveSendingTimeForNextRequest(base::TimeTicks());
 254   }
 255
 256   // Tells the getter to act as if the URLRequestContext is about to be shut
 257   // down.
 258   void Shutdown() {
 259     if (!network_task_runner_->RunsTasksOnCurrentThread()) {
 260       network_task_runner_->PostTask(
 261           FROM_HERE,
 262           base::Bind(&FetcherTestURLRequestContextGetter::Shutdown, this));
 263       return;
 264     }
 265
 266     shutting_down_ = true;
 267     NotifyContextShuttingDown();
 268     // Should now be safe to destroy the context.  Context will check it has no
 269     // pending requests.
 270     context_.reset();
 271   }
 272
 273   // Convenience method to access the context as a FetcherTestURLRequestContext
 274   // without going through GetURLRequestContext.
 275   FetcherTestURLRequestContext* context() {
 276     DCHECK(network_task_runner_->BelongsToCurrentThread());
 277     return context_.get();
 278   }
 279
 280  protected:
 281   ~FetcherTestURLRequestContextGetter() override {
 282     // |context_| may only be deleted on the network thread. Fortunately,
 283     // the parent class already ensures it's deleted on the network thread.
 284     DCHECK(network_task_runner_->BelongsToCurrentThread());
 285     if (!on_destruction_callback_.is_null())
 286       on_destruction_callback_.Run();
 287   }
 288
 289   scoped_refptr<base::SingleThreadTaskRunner> network_task_runner_;
 290   const std::string hanging_domain_;
 291
 292   scoped_ptr<FetcherTestURLRequestContext> context_;
 293   bool shutting_down_;
 294
 295   base::Closure on_destruction_callback_;
 296
 297   DISALLOW_COPY_AND_ASSIGN(FetcherTestURLRequestContextGetter);
 298 };
 299
 300 }  // namespace
 301
 302 class URLFetcherTest : public testing::Test {
 303  public:
 304   URLFetcherTest() : num_upload_streams_created_(0) {}
 305
 306   static int GetNumFetcherCores() {
 307     return URLFetcherImpl::GetNumFetcherCores();
 308   }
 309
 310   // Creates a URLRequestContextGetter with a URLRequestContext that lives on
 311   // the current thread.
 312   scoped_refptr<FetcherTestURLRequestContextGetter>
 313   CreateSameThreadContextGetter() {
 314     return scoped_refptr<FetcherTestURLRequestContextGetter>(
 315         new FetcherTestURLRequestContextGetter(
 316             base::ThreadTaskRunnerHandle::Get(), hanging_url().host()));
 317   }
 318
 319   // Creates a URLRequestContextGetter with a URLRequestContext that lives on
 320   // a separate network thread.
 321   scoped_refptr<FetcherTestURLRequestContextGetter>
 322   CreateCrossThreadContextGetter() {
 323     if (!network_thread_) {
 324       network_thread_.reset(new base::Thread("network thread"));
 325       base::Thread::Options network_thread_options;
 326       network_thread_options.message_loop_type = base::MessageLoop::TYPE_IO;
 327       bool result = network_thread_->StartWithOptions(network_thread_options);
 328       CHECK(result);
 329     }
 330
 331     return scoped_refptr<FetcherTestURLRequestContextGetter>(
 332         new FetcherTestURLRequestContextGetter(network_thread_->task_runner(),
 333                                                hanging_url().host()));
 334   }
 335
 336   // Callback passed to URLFetcher to create upload stream by some tests.
 337   scoped_ptr<UploadDataStream> CreateUploadStream() {
 338     ++num_upload_streams_created_;
 339     std::vector<char> buffer(
 340         kCreateUploadStreamBody,
 341         kCreateUploadStreamBody + strlen(kCreateUploadStreamBody));
 342     return ElementsUploadDataStream::CreateWithReader(
 343         scoped_ptr<UploadElementReader>(
 344             new UploadOwnedBytesElementReader(&buffer)),
 345         0);
 346   }
 347
 348   // Number of streams created by CreateUploadStream.
 349   size_t num_upload_streams_created() const {
 350     return num_upload_streams_created_;
 351   }
 352
 353   // Downloads |file_to_fetch| and checks the contents when done.  If
 354   // |save_to_temporary_file| is true, saves it to a temporary file, and
 355   // |requested_out_path| is ignored. Otherwise, saves it to
 356   // |requested_out_path|. Takes ownership of the file if |take_ownership| is
 357   // true. Deletes file when done.
 358   void SaveFileTest(const char* file_to_fetch,
 359                     bool save_to_temporary_file,
 360                     const base::FilePath& requested_out_path,
 361                     bool take_ownership) {
 362     scoped_ptr<WaitingURLFetcherDelegate> delegate(
 363         new WaitingURLFetcherDelegate());
 364     delegate->CreateFetcher(
 365         test_server_->GetURL(std::string(kTestServerFilePrefix) +
 366                              file_to_fetch),
 367         URLFetcher::GET, CreateSameThreadContextGetter());
 368     if (save_to_temporary_file) {
 369       delegate->fetcher()->SaveResponseToTemporaryFile(
 370           scoped_refptr<base::SingleThreadTaskRunner>(
 371               base::ThreadTaskRunnerHandle::Get()));
 372     } else {
 373       delegate->fetcher()->SaveResponseToFileAtPath(
 374           requested_out_path, scoped_refptr<base::SingleThreadTaskRunner>(
 375                                   base::ThreadTaskRunnerHandle::Get()));
 376     }
 377     delegate->StartFetcherAndWait();
 378
 379     EXPECT_TRUE(delegate->fetcher()->GetStatus().is_success());
 380     EXPECT_EQ(200, delegate->fetcher()->GetResponseCode());
 381
 382     base::FilePath out_path;
 383     EXPECT_TRUE(
 384         delegate->fetcher()->GetResponseAsFilePath(take_ownership, &out_path));
 385     if (!save_to_temporary_file) {
 386       EXPECT_EQ(requested_out_path, out_path);
 387     }
 388
 389     EXPECT_TRUE(base::ContentsEqual(
 390         test_server_->GetDocumentRoot().AppendASCII(file_to_fetch), out_path));
 391
 392     // Delete the delegate and run the message loop to give the fetcher's
 393     // destructor a chance to delete the file.
 394     delegate.reset();
 395     base::RunLoop().RunUntilIdle();
 396
 397     // File should only exist if |take_ownership| was true.
 398     EXPECT_EQ(take_ownership, base::PathExists(out_path));
 399
 400     // Cleanup.
 401     if (base::PathExists(out_path))
 402       base::DeleteFile(out_path, false);
 403   }
 404
 405   // Returns a URL that hangs on DNS resolution when using a context created by
 406   // the test fixture.
 407   const GURL& hanging_url() const { return hanging_url_; }
 408
 409   // testing::Test:
 410   void SetUp() override {
 411     SetUpServer();
 412     ASSERT_TRUE(test_server_->Start());
 413
 414     // URL that will hang when lookups reach the host resolver.
 415     hanging_url_ = GURL(base::StringPrintf(
 416         "http://example.com:%d/%s", test_server_->host_port_pair().port(),
 417         kDefaultResponsePath));
 418     ASSERT_TRUE(hanging_url_.is_valid());
 419
 420 #if defined(USE_NSS_CERTS) || defined(OS_IOS)
 421     crypto::EnsureNSSInit();
 422     EnsureNSSHttpIOInit();
 423 #endif
 424   }
 425
 426   void TearDown() override {
 427 #if defined(USE_NSS_CERTS) || defined(OS_IOS)
 428     ShutdownNSSHttpIO();
 429 #endif
 430   }
 431
 432   // Initializes |test_server_| without starting it.  Allows subclasses to use
 433   // their own server configuration.
 434   virtual void SetUpServer() {
 435     test_server_.reset(new SpawnedTestServer(SpawnedTestServer::TYPE_HTTP,
 436                                              SpawnedTestServer::kLocalhost,
 437                                              base::FilePath(kDocRoot)));
 438   }
 439
 440   // Network thread for cross-thread tests.  Most threads just use the main
 441   // thread for network activity.
 442   scoped_ptr<base::Thread> network_thread_;
 443
 444   scoped_ptr<SpawnedTestServer> test_server_;
 445   GURL hanging_url_;
 446
 447   size_t num_upload_streams_created_;
 448 };
 449
 450 namespace {
 451
 452 // Version of URLFetcherTest that tests bad HTTPS requests.
 453 class URLFetcherBadHTTPSTest : public URLFetcherTest {
 454  public:
 455   URLFetcherBadHTTPSTest() {}
 456
 457   // URLFetcherTest:
 458   void SetUpServer() override {
 459     SpawnedTestServer::SSLOptions ssl_options(
 460         SpawnedTestServer::SSLOptions::CERT_EXPIRED);
 461     test_server_.reset(new SpawnedTestServer(
 462         SpawnedTestServer::TYPE_HTTPS, ssl_options, base::FilePath(kDocRoot)));
 463   }
 464 };
 465
 466 // Create the fetcher on the main thread.  Since network IO will happen on the
 467 // main thread, this will test URLFetcher's ability to do everything on one
 468 // thread.
 469 TEST_F(URLFetcherTest, SameThreadTest) {
 470   WaitingURLFetcherDelegate delegate;
 471   delegate.CreateFetcher(test_server_->GetURL(kDefaultResponsePath),
 472                          URLFetcher::GET, CreateSameThreadContextGetter());
 473   delegate.StartFetcherAndWait();
 474
 475   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 476   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 477   std::string data;
 478   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 479   EXPECT_EQ(kDefaultResponseBody, data);
 480
 481   EXPECT_EQ(static_cast<int64_t>(strlen(kDefaultResponseBody)),
 482             delegate.fetcher()->GetReceivedResponseContentLength());
 483   std::string parsed_headers;
 484   base::ReplaceChars(delegate.fetcher()->GetResponseHeaders()->raw_headers(),
 485                      std::string("\0", 1), "\n\r", &parsed_headers);
 486   EXPECT_EQ(static_cast<int64_t>(parsed_headers.size() +
 487                                  strlen(kDefaultResponseBody)),
 488             delegate.fetcher()->GetTotalReceivedBytes());
 489 }
 490
 491 // Create a separate thread that will create the URLFetcher.  A separate thread
 492 // acts as the network thread.
 493 TEST_F(URLFetcherTest, DifferentThreadsTest) {
 494   WaitingURLFetcherDelegate delegate;
 495   delegate.CreateFetcher(test_server_->GetURL(kDefaultResponsePath),
 496                          URLFetcher::GET, CreateCrossThreadContextGetter());
 497   delegate.StartFetcherAndWait();
 498
 499   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 500   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 501   std::string data;
 502   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 503   EXPECT_EQ(kDefaultResponseBody, data);
 504 }
 505
 506 // Tests to make sure CancelAll() will successfully cancel existing URLFetchers.
 507 TEST_F(URLFetcherTest, CancelAll) {
 508   EXPECT_EQ(0, GetNumFetcherCores());
 509
 510   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
 511       CreateSameThreadContextGetter());
 512   // Force context creation.
 513   context_getter->GetURLRequestContext();
 514   MockHostResolver* mock_resolver = context_getter->context()->mock_resolver();
 515
 516   WaitingURLFetcherDelegate delegate;
 517   delegate.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
 518   delegate.fetcher()->Start();
 519   // Wait for the request to reach the mock resolver and hang, to ensure the
 520   // request has actually started.
 521   base::RunLoop().RunUntilIdle();
 522   EXPECT_TRUE(mock_resolver->has_pending_requests());
 523
 524   EXPECT_EQ(1, URLFetcherTest::GetNumFetcherCores());
 525   URLFetcherImpl::CancelAll();
 526   EXPECT_EQ(0, URLFetcherTest::GetNumFetcherCores());
 527 }
 528
 529 TEST_F(URLFetcherTest, DontRetryOnNetworkChangedByDefault) {
 530   EXPECT_EQ(0, GetNumFetcherCores());
 531
 532   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
 533       CreateSameThreadContextGetter());
 534   // Force context creation.
 535   context_getter->GetURLRequestContext();
 536   MockHostResolver* mock_resolver = context_getter->context()->mock_resolver();
 537
 538   WaitingURLFetcherDelegate delegate;
 539   delegate.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
 540   EXPECT_FALSE(mock_resolver->has_pending_requests());
 541
 542   // This posts a task to start the fetcher.
 543   delegate.fetcher()->Start();
 544   base::RunLoop().RunUntilIdle();
 545
 546   // The fetcher is now running, but is pending the host resolve.
 547   EXPECT_EQ(1, GetNumFetcherCores());
 548   EXPECT_TRUE(mock_resolver->has_pending_requests());
 549   ASSERT_FALSE(delegate.did_complete());
 550
 551   // A network change notification aborts the connect job.
 552   NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
 553   delegate.WaitForComplete();
 554   EXPECT_FALSE(mock_resolver->has_pending_requests());
 555
 556   // And the owner of the fetcher gets the ERR_NETWORK_CHANGED error.
 557   EXPECT_EQ(hanging_url(), delegate.fetcher()->GetOriginalURL());
 558   ASSERT_FALSE(delegate.fetcher()->GetStatus().is_success());
 559   EXPECT_EQ(ERR_NETWORK_CHANGED, delegate.fetcher()->GetStatus().error());
 560 }
 561
 562 TEST_F(URLFetcherTest, RetryOnNetworkChangedAndFail) {
 563   EXPECT_EQ(0, GetNumFetcherCores());
 564
 565   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
 566       CreateSameThreadContextGetter());
 567   // Force context creation.
 568   context_getter->GetURLRequestContext();
 569   MockHostResolver* mock_resolver = context_getter->context()->mock_resolver();
 570
 571   WaitingURLFetcherDelegate delegate;
 572   delegate.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
 573   delegate.fetcher()->SetAutomaticallyRetryOnNetworkChanges(3);
 574   EXPECT_FALSE(mock_resolver->has_pending_requests());
 575
 576   // This posts a task to start the fetcher.
 577   delegate.fetcher()->Start();
 578   base::RunLoop().RunUntilIdle();
 579
 580   // The fetcher is now running, but is pending the host resolve.
 581   EXPECT_EQ(1, GetNumFetcherCores());
 582   EXPECT_TRUE(mock_resolver->has_pending_requests());
 583   ASSERT_FALSE(delegate.did_complete());
 584
 585   // Make it fail 3 times.
 586   for (int i = 0; i < 3; ++i) {
 587     // A network change notification aborts the connect job.
 588     NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
 589     base::RunLoop().RunUntilIdle();
 590
 591     // But the fetcher retries automatically.
 592     EXPECT_EQ(1, GetNumFetcherCores());
 593     EXPECT_TRUE(mock_resolver->has_pending_requests());
 594     ASSERT_FALSE(delegate.did_complete());
 595   }
 596
 597   // A 4th failure doesn't trigger another retry, and propagates the error
 598   // to the owner of the fetcher.
 599   NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
 600   delegate.WaitForComplete();
 601   EXPECT_FALSE(mock_resolver->has_pending_requests());
 602
 603   // And the owner of the fetcher gets the ERR_NETWORK_CHANGED error.
 604   EXPECT_EQ(hanging_url(), delegate.fetcher()->GetOriginalURL());
 605   ASSERT_FALSE(delegate.fetcher()->GetStatus().is_success());
 606   EXPECT_EQ(ERR_NETWORK_CHANGED, delegate.fetcher()->GetStatus().error());
 607 }
 608
 609 TEST_F(URLFetcherTest, RetryOnNetworkChangedAndSucceed) {
 610   EXPECT_EQ(0, GetNumFetcherCores());
 611
 612   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
 613       CreateSameThreadContextGetter());
 614   // Force context creation.
 615   context_getter->GetURLRequestContext();
 616   MockHostResolver* mock_resolver = context_getter->context()->mock_resolver();
 617
 618   WaitingURLFetcherDelegate delegate;
 619   delegate.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
 620   delegate.fetcher()->SetAutomaticallyRetryOnNetworkChanges(3);
 621   EXPECT_FALSE(mock_resolver->has_pending_requests());
 622
 623   // This posts a task to start the fetcher.
 624   delegate.fetcher()->Start();
 625   base::RunLoop().RunUntilIdle();
 626
 627   // The fetcher is now running, but is pending the host resolve.
 628   EXPECT_EQ(1, GetNumFetcherCores());
 629   EXPECT_TRUE(mock_resolver->has_pending_requests());
 630   ASSERT_FALSE(delegate.did_complete());
 631
 632   // Make it fail 3 times.
 633   for (int i = 0; i < 3; ++i) {
 634     // A network change notification aborts the connect job.
 635     NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
 636     base::RunLoop().RunUntilIdle();
 637
 638     // But the fetcher retries automatically.
 639     EXPECT_EQ(1, GetNumFetcherCores());
 640     EXPECT_TRUE(mock_resolver->has_pending_requests());
 641     ASSERT_FALSE(delegate.did_complete());
 642   }
 643
 644   // Now let it succeed by resolving the pending request.
 645   mock_resolver->ResolveAllPending();
 646   delegate.WaitForComplete();
 647   EXPECT_FALSE(mock_resolver->has_pending_requests());
 648
 649   // This time the request succeeded.
 650   EXPECT_EQ(hanging_url(), delegate.fetcher()->GetOriginalURL());
 651   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 652   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 653
 654   std::string data;
 655   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 656   EXPECT_EQ(kDefaultResponseBody, data);
 657 }
 658
 659 TEST_F(URLFetcherTest, PostString) {
 660   const char kUploadData[] = "bobsyeruncle";
 661
 662   WaitingURLFetcherDelegate delegate;
 663   delegate.CreateFetcher(test_server_->GetURL("echo"), URLFetcher::POST,
 664                          CreateSameThreadContextGetter());
 665   delegate.fetcher()->SetUploadData("application/x-www-form-urlencoded",
 666                                     kUploadData);
 667   delegate.StartFetcherAndWait();
 668
 669   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 670   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 671   std::string data;
 672   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 673   EXPECT_EQ(kUploadData, data);
 674 }
 675
 676 TEST_F(URLFetcherTest, PostEmptyString) {
 677   const char kUploadData[] = "";
 678
 679   WaitingURLFetcherDelegate delegate;
 680   delegate.CreateFetcher(test_server_->GetURL("echo"), URLFetcher::POST,
 681                          CreateSameThreadContextGetter());
 682   delegate.fetcher()->SetUploadData("application/x-www-form-urlencoded",
 683                                     kUploadData);
 684   delegate.StartFetcherAndWait();
 685
 686   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 687   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 688   std::string data;
 689   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 690   EXPECT_EQ(kUploadData, data);
 691 }
 692
 693 TEST_F(URLFetcherTest, PostEntireFile) {
 694   base::FilePath upload_path = GetUploadFileTestPath();
 695
 696   WaitingURLFetcherDelegate delegate;
 697   delegate.CreateFetcher(test_server_->GetURL("echo"), URLFetcher::POST,
 698                          CreateSameThreadContextGetter());
 699   delegate.fetcher()->SetUploadFilePath("application/x-www-form-urlencoded",
 700                                         upload_path, 0, kuint64max,
 701                                         base::ThreadTaskRunnerHandle::Get());
 702   delegate.StartFetcherAndWait();
 703
 704   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 705   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 706
 707   std::string expected;
 708   ASSERT_TRUE(base::ReadFileToString(upload_path, &expected));
 709   std::string data;
 710   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 711   EXPECT_EQ(expected, data);
 712 }
 713
 714 TEST_F(URLFetcherTest, PostFileRange) {
 715   const size_t kRangeStart = 30;
 716   const size_t kRangeLength = 100;
 717   base::FilePath upload_path = GetUploadFileTestPath();
 718
 719   WaitingURLFetcherDelegate delegate;
 720   delegate.CreateFetcher(test_server_->GetURL("echo"), URLFetcher::POST,
 721                          CreateSameThreadContextGetter());
 722   delegate.fetcher()->SetUploadFilePath("application/x-www-form-urlencoded",
 723                                         upload_path, kRangeStart, kRangeLength,
 724                                         base::ThreadTaskRunnerHandle::Get());
 725   delegate.StartFetcherAndWait();
 726
 727   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 728   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 729
 730   std::string expected;
 731   ASSERT_TRUE(base::ReadFileToString(upload_path, &expected));
 732   std::string data;
 733   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 734   EXPECT_EQ(expected.substr(kRangeStart, kRangeLength), data);
 735 }
 736
 737 TEST_F(URLFetcherTest, PostWithUploadStreamFactory) {
 738   WaitingURLFetcherDelegate delegate;
 739   delegate.CreateFetcher(test_server_->GetURL("echo"), URLFetcher::POST,
 740                          CreateSameThreadContextGetter());
 741   delegate.fetcher()->SetUploadStreamFactory(
 742       "text/plain",
 743       base::Bind(&URLFetcherTest::CreateUploadStream, base::Unretained(this)));
 744   delegate.StartFetcherAndWait();
 745
 746   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 747   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 748   std::string data;
 749   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 750   EXPECT_EQ(kCreateUploadStreamBody, data);
 751   EXPECT_EQ(1u, num_upload_streams_created());
 752 }
 753
 754 TEST_F(URLFetcherTest, PostWithUploadStreamFactoryAndRetries) {
 755   WaitingURLFetcherDelegate delegate;
 756   delegate.CreateFetcher(test_server_->GetURL("echo?status=500"),
 757                          URLFetcher::POST, CreateSameThreadContextGetter());
 758   delegate.fetcher()->SetAutomaticallyRetryOn5xx(true);
 759   delegate.fetcher()->SetMaxRetriesOn5xx(1);
 760   delegate.fetcher()->SetUploadStreamFactory(
 761       "text/plain",
 762       base::Bind(&URLFetcherTest::CreateUploadStream, base::Unretained(this)));
 763   delegate.StartFetcherAndWait();
 764
 765   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 766   EXPECT_EQ(500, delegate.fetcher()->GetResponseCode());
 767   std::string data;
 768   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 769   EXPECT_EQ(kCreateUploadStreamBody, data);
 770   EXPECT_EQ(2u, num_upload_streams_created());
 771 }
 772
 773 // Checks that upload progress increases over time, never exceeds what's already
 774 // been sent, and adds a chunk whenever all previously appended chunks have
 775 // been uploaded.
 776 class CheckUploadProgressDelegate : public WaitingURLFetcherDelegate {
 777  public:
 778   CheckUploadProgressDelegate()
 779       : chunk_(1 << 16, 'a'), num_chunks_appended_(0), last_seen_progress_(0) {}
 780   ~CheckUploadProgressDelegate() override {}
 781
 782   void OnURLFetchUploadProgress(const URLFetcher* source,
 783                                 int64 current,
 784                                 int64 total) override {
 785     // Run default checks.
 786     WaitingURLFetcherDelegate::OnURLFetchUploadProgress(source, current, total);
 787
 788     EXPECT_LE(last_seen_progress_, current);
 789     EXPECT_LE(current, bytes_appended());
 790     last_seen_progress_ = current;
 791     MaybeAppendChunk();
 792   }
 793
 794   // Append the next chunk if all previously appended chunks have been sent.
 795   void MaybeAppendChunk() {
 796     const int kNumChunks = 5;
 797     if (last_seen_progress_ == bytes_appended() &&
 798         num_chunks_appended_ < kNumChunks) {
 799       ++num_chunks_appended_;
 800       fetcher()->AppendChunkToUpload(chunk_,
 801                                      num_chunks_appended_ == kNumChunks);
 802     }
 803   }
 804
 805  private:
 806   int64 bytes_appended() const { return num_chunks_appended_ * chunk_.size(); }
 807
 808   const std::string chunk_;
 809
 810   int64 num_chunks_appended_;
 811   int64 last_seen_progress_;
 812
 813   DISALLOW_COPY_AND_ASSIGN(CheckUploadProgressDelegate);
 814 };
 815
 816 TEST_F(URLFetcherTest, UploadProgress) {
 817   CheckUploadProgressDelegate delegate;
 818   delegate.CreateFetcher(test_server_->GetURL("echo"), URLFetcher::POST,
 819                          CreateSameThreadContextGetter());
 820   // Use a chunked upload so that the upload can be paused after uploading data.
 821   // Since upload progress uses a timer, the delegate may not receive any
 822   // notification otherwise.
 823   delegate.fetcher()->SetChunkedUpload("application/x-www-form-urlencoded");
 824
 825   delegate.fetcher()->Start();
 826   // Append the first chunk.  Others will be appended automatically in response
 827   // to OnURLFetchUploadProgress events.
 828   delegate.MaybeAppendChunk();
 829   delegate.WaitForComplete();
 830
 831   // Make sure there are no pending events that cause problems when run.
 832   base::RunLoop().RunUntilIdle();
 833
 834   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 835   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 836   EXPECT_TRUE(delegate.did_complete());
 837 }
 838
 839 // Checks that download progress never decreases, never exceeds file size, and
 840 // that file size is correctly reported.
 841 class CheckDownloadProgressDelegate : public WaitingURLFetcherDelegate {
 842  public:
 843   CheckDownloadProgressDelegate(int64 file_size)
 844       : file_size_(file_size), last_seen_progress_(0) {}
 845   ~CheckDownloadProgressDelegate() override {}
 846
 847   void OnURLFetchDownloadProgress(const URLFetcher* source,
 848                                   int64 current,
 849                                   int64 total) override {
 850     // Run default checks.
 851     WaitingURLFetcherDelegate::OnURLFetchDownloadProgress(source, current,
 852                                                           total);
 853
 854     EXPECT_LE(last_seen_progress_, current);
 855     EXPECT_EQ(file_size_, total);
 856     last_seen_progress_ = current;
 857   }
 858
 859  private:
 860   int64 file_size_;
 861   int64 last_seen_progress_;
 862
 863   DISALLOW_COPY_AND_ASSIGN(CheckDownloadProgressDelegate);
 864 };
 865
 866 TEST_F(URLFetcherTest, DownloadProgress) {
 867   // Get a file large enough to require more than one read into
 868   // URLFetcher::Core's IOBuffer.
 869   const char kFileToFetch[] = "animate1.gif";
 870
 871   std::string file_contents;
 872   ASSERT_TRUE(base::ReadFileToString(
 873       test_server_->GetDocumentRoot().AppendASCII(kFileToFetch),
 874       &file_contents));
 875
 876   CheckDownloadProgressDelegate delegate(file_contents.size());
 877   delegate.CreateFetcher(
 878       test_server_->GetURL(std::string(kTestServerFilePrefix) + kFileToFetch),
 879       URLFetcher::GET, CreateSameThreadContextGetter());
 880   delegate.StartFetcherAndWait();
 881
 882   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 883   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 884   std::string data;
 885   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 886   EXPECT_EQ(file_contents, data);
 887 }
 888
 889 class CancelOnUploadProgressDelegate : public WaitingURLFetcherDelegate {
 890  public:
 891   CancelOnUploadProgressDelegate() {}
 892   ~CancelOnUploadProgressDelegate() override {}
 893
 894   void OnURLFetchUploadProgress(const URLFetcher* source,
 895                                 int64 current,
 896                                 int64 total) override {
 897     CancelFetch();
 898   }
 899
 900  private:
 901   DISALLOW_COPY_AND_ASSIGN(CancelOnUploadProgressDelegate);
 902 };
 903
 904 // Check that a fetch can be safely cancelled/deleted during an upload progress
 905 // callback.
 906 TEST_F(URLFetcherTest, CancelInUploadProgressCallback) {
 907   CancelOnUploadProgressDelegate delegate;
 908   delegate.CreateFetcher(test_server_->GetURL("echo"), URLFetcher::POST,
 909                          CreateSameThreadContextGetter());
 910   delegate.fetcher()->SetChunkedUpload("application/x-www-form-urlencoded");
 911   delegate.fetcher()->Start();
 912   // Use a chunked upload so that the upload can be paused after uploading data.
 913   // Since uploads progress uses a timer, may not receive any notification,
 914   // otherwise.
 915   std::string upload_data(1 << 16, 'a');
 916   delegate.fetcher()->AppendChunkToUpload(upload_data, false);
 917   delegate.WaitForComplete();
 918
 919   // Make sure there are no pending events that cause problems when run.
 920   base::RunLoop().RunUntilIdle();
 921
 922   EXPECT_FALSE(delegate.did_complete());
 923   EXPECT_FALSE(delegate.fetcher());
 924 }
 925
 926 class CancelOnDownloadProgressDelegate : public WaitingURLFetcherDelegate {
 927  public:
 928   CancelOnDownloadProgressDelegate() {}
 929   ~CancelOnDownloadProgressDelegate() override {}
 930
 931   void OnURLFetchDownloadProgress(const URLFetcher* source,
 932                                   int64 current,
 933                                   int64 total) override {
 934     CancelFetch();
 935   }
 936
 937  private:
 938   DISALLOW_COPY_AND_ASSIGN(CancelOnDownloadProgressDelegate);
 939 };
 940
 941 // Check that a fetch can be safely cancelled/deleted during a download progress
 942 // callback.
 943 TEST_F(URLFetcherTest, CancelInDownloadProgressCallback) {
 944   // Get a file large enough to require more than one read into
 945   // URLFetcher::Core's IOBuffer.
 946   static const char kFileToFetch[] = "animate1.gif";
 947   CancelOnDownloadProgressDelegate delegate;
 948   delegate.CreateFetcher(
 949       test_server_->GetURL(std::string(kTestServerFilePrefix) + kFileToFetch),
 950       URLFetcher::GET, CreateSameThreadContextGetter());
 951   delegate.StartFetcherAndWait();
 952
 953   // Make sure there are no pending events that cause problems when run.
 954   base::RunLoop().RunUntilIdle();
 955
 956   EXPECT_FALSE(delegate.did_complete());
 957   EXPECT_FALSE(delegate.fetcher());
 958 }
 959
 960 TEST_F(URLFetcherTest, Headers) {
 961   WaitingURLFetcherDelegate delegate;
 962   delegate.CreateFetcher(
 963       test_server_->GetURL("set-header?cache-control: private"),
 964       URLFetcher::GET, CreateSameThreadContextGetter());
 965   delegate.StartFetcherAndWait();
 966
 967   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 968   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 969   std::string header;
 970   ASSERT_TRUE(delegate.fetcher()->GetResponseHeaders()->GetNormalizedHeader(
 971       "cache-control", &header));
 972   EXPECT_EQ("private", header);
 973 }
 974
 975 TEST_F(URLFetcherTest, SocketAddress) {
 976   WaitingURLFetcherDelegate delegate;
 977   delegate.CreateFetcher(test_server_->GetURL(kDefaultResponsePath),
 978                          URLFetcher::GET, CreateSameThreadContextGetter());
 979   delegate.StartFetcherAndWait();
 980
 981   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 982   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 983   EXPECT_EQ(test_server_->host_port_pair().port(),
 984             delegate.fetcher()->GetSocketAddress().port());
 985   EXPECT_EQ(test_server_->host_port_pair().host(),
 986             delegate.fetcher()->GetSocketAddress().host());
 987 }
 988
 989 TEST_F(URLFetcherTest, StopOnRedirect) {
 990   const char kRedirectTarget[] = "http://redirect.target.com";
 991
 992   WaitingURLFetcherDelegate delegate;
 993   delegate.CreateFetcher(
 994       test_server_->GetURL(std::string("server-redirect?") + kRedirectTarget),
 995       URLFetcher::GET, CreateSameThreadContextGetter());
 996   delegate.fetcher()->SetStopOnRedirect(true);
 997   delegate.StartFetcherAndWait();
 998
 999   EXPECT_EQ(GURL(kRedirectTarget), delegate.fetcher()->GetURL());
1000   EXPECT_EQ(URLRequestStatus::CANCELED,
1001             delegate.fetcher()->GetStatus().status());
1002   EXPECT_EQ(ERR_ABORTED, delegate.fetcher()->GetStatus().error());
1003   EXPECT_EQ(301, delegate.fetcher()->GetResponseCode());
1004 }
1005
1006 TEST_F(URLFetcherTest, ThrottleOnRepeatedFetches) {
1007   base::Time start_time = Time::Now();
1008   GURL url(test_server_->GetURL(kDefaultResponsePath));
1009
1010   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
1011       CreateSameThreadContextGetter());
1012
1013   // Registers an entry for test url. It only allows 3 requests to be sent
1014   // in 200 milliseconds.
1015   context_getter->AddThrottlerEntry(
1016       url, std::string() /* url_id */, 200 /* sliding_window_period_ms */,
1017       3 /* max_send_threshold */, 1 /* initial_backoff_ms */,
1018       2.0 /* multiply_factor */, 0.0 /* jitter_factor */,
1019       256 /* maximum_backoff_ms */,
1020       false /* reserve_sending_time_for_next_request*/);
1021
1022   for (int i = 0; i < 20; ++i) {
1023     WaitingURLFetcherDelegate delegate;
1024     delegate.CreateFetcher(url, URLFetcher::GET, context_getter);
1025     delegate.StartFetcherAndWait();
1026
1027     EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
1028     EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
1029   }
1030
1031   // 20 requests were sent. Due to throttling, they should have collectively
1032   // taken over 1 second.
1033   EXPECT_GE(Time::Now() - start_time, base::TimeDelta::FromSeconds(1));
1034 }
1035
1036 TEST_F(URLFetcherTest, ThrottleOn5xxRetries) {
1037   base::Time start_time = Time::Now();
1038   GURL url(test_server_->GetURL("files/server-unavailable.html"));
1039
1040   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
1041       CreateSameThreadContextGetter());
1042
1043   // Registers an entry for test url. The backoff time is calculated by:
1044   //     new_backoff = 2.0 * old_backoff + 0
1045   // and maximum backoff time is 256 milliseconds.
1046   // Maximum retries allowed is set to 11.
1047   context_getter->AddThrottlerEntry(
1048       url, std::string() /* url_id */, 200 /* sliding_window_period_ms */,
1049       3 /* max_send_threshold */, 1 /* initial_backoff_ms */,
1050       2.0 /* multiply_factor */, 0.0 /* jitter_factor */,
1051       256 /* maximum_backoff_ms */,
1052       false /* reserve_sending_time_for_next_request*/);
1053
1054   WaitingURLFetcherDelegate delegate;
1055   delegate.CreateFetcher(url, URLFetcher::GET, context_getter);
1056   delegate.fetcher()->SetAutomaticallyRetryOn5xx(true);
1057   delegate.fetcher()->SetMaxRetriesOn5xx(11);
1058   delegate.StartFetcherAndWait();
1059
1060   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
1061   EXPECT_EQ(503, delegate.fetcher()->GetResponseCode());
1062   std::string data;
1063   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
1064   EXPECT_FALSE(data.empty());
1065
1066   // The request should have been retried 11 times (12 times including the first
1067   // attempt).  Due to throttling, they should have collectively taken over 1
1068   // second.
1069   EXPECT_GE(Time::Now() - start_time, base::TimeDelta::FromSeconds(1));
1070 }
1071
1072 // Tests overload protection, when responses passed through.
1073 TEST_F(URLFetcherTest, ProtectTestPassedThrough) {
1074   base::Time start_time = Time::Now();
1075   GURL url(test_server_->GetURL("files/server-unavailable.html"));
1076
1077   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
1078       CreateSameThreadContextGetter());
1079
1080   // Registers an entry for test url. The backoff time is calculated by:
1081   //     new_backoff = 2.0 * old_backoff + 0
1082   // and maximum backoff time is 150000 milliseconds.
1083   // Maximum retries allowed is set to 11.
1084   // Total time if *not* for not doing automatic backoff would be 150s.
1085   // In reality it should be "as soon as server responds".
1086   context_getter->AddThrottlerEntry(
1087       url, std::string() /* url_id */, 200 /* sliding_window_period_ms */,
1088       3 /* max_send_threshold */, 10000 /* initial_backoff_ms */,
1089       2.0 /* multiply_factor */, 0.0 /* jitter_factor */,
1090       150000 /* maximum_backoff_ms */,
1091       false /* reserve_sending_time_for_next_request*/);
1092
1093   WaitingURLFetcherDelegate delegate;
1094   delegate.CreateFetcher(url, URLFetcher::GET, context_getter);
1095   delegate.fetcher()->SetAutomaticallyRetryOn5xx(false);
1096   delegate.fetcher()->SetMaxRetriesOn5xx(11);
1097   delegate.StartFetcherAndWait();
1098
1099   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
1100   EXPECT_EQ(503, delegate.fetcher()->GetResponseCode());
1101   std::string data;
1102   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
1103   EXPECT_FALSE(data.empty());
1104   EXPECT_GT(delegate.fetcher()->GetBackoffDelay().InMicroseconds(), 0);
1105
1106   // The request should not have been retried at all.  If it had attempted all
1107   // 11 retries, that should have taken 2.5 minutes.
1108   EXPECT_TRUE(Time::Now() - start_time < TimeDelta::FromMinutes(1));
1109 }
1110
1111 // Used to check if a callback has been invoked.
1112 void SetBoolToTrue(bool* ptr) {
1113   *ptr = true;
1114 }
1115
1116 // Make sure that the URLFetcher cancels the URLRequest and releases its context
1117 // getter pointer synchronously when the fetcher and request context live on
1118 // the same thread.
1119 TEST_F(URLFetcherTest, CancelSameThread) {
1120   WaitingURLFetcherDelegate delegate;
1121   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
1122       CreateSameThreadContextGetter());
1123   bool getter_was_destroyed = false;
1124   context_getter->set_on_destruction_callback(
1125       base::Bind(&SetBoolToTrue, &getter_was_destroyed));
1126   delegate.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
1127
1128   // The getter won't be destroyed if the test holds on to a reference to it.
1129   context_getter = nullptr;
1130
1131   delegate.fetcher()->Start();
1132   // Give the fetcher a chance to start the request.
1133   base::RunLoop().RunUntilIdle();
1134   EXPECT_EQ(1, URLFetcherTest::GetNumFetcherCores());
1135
1136   // On same-thread cancel, the request should be canceled and getter destroyed
1137   // synchronously, for safe shutdown.
1138   delegate.CancelFetch();
1139   EXPECT_EQ(0, URLFetcherTest::GetNumFetcherCores());
1140   EXPECT_TRUE(getter_was_destroyed);
1141 }
1142
1143 // Make sure that the URLFetcher releases its context getter pointer on
1144 // cancellation, cross-thread case.
1145 TEST_F(URLFetcherTest, CancelDifferentThreads) {
1146   base::RunLoop run_loop_;
1147
1148   WaitingURLFetcherDelegate delegate;
1149   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
1150       CreateCrossThreadContextGetter());
1151   context_getter->set_on_destruction_callback(base::Bind(
1152       base::IgnoreResult(&base::SingleThreadTaskRunner::PostTask),
1153       base::ThreadTaskRunnerHandle::Get(), FROM_HERE, run_loop_.QuitClosure()));
1154   delegate.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
1155
1156   // The getter won't be destroyed if the test holds on to a reference to it.
1157   context_getter = nullptr;
1158
1159   delegate.fetcher()->Start();
1160   delegate.CancelFetch();
1161   run_loop_.Run();
1162
1163   EXPECT_FALSE(delegate.did_complete());
1164 }
1165
1166 TEST_F(URLFetcherTest, CancelWhileDelayedByThrottleDifferentThreads) {
1167   GURL url = test_server_->GetURL(kDefaultResponsePath);
1168   base::RunLoop run_loop_;
1169
1170   WaitingURLFetcherDelegate delegate;
1171   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
1172       CreateCrossThreadContextGetter());
1173   context_getter->set_on_destruction_callback(base::Bind(
1174       base::IgnoreResult(&base::SingleThreadTaskRunner::PostTask),
1175       base::ThreadTaskRunnerHandle::Get(), FROM_HERE, run_loop_.QuitClosure()));
1176   delegate.CreateFetcher(url, URLFetcher::GET, context_getter);
1177
1178   // Register an entry for test url using a sliding window of 400 seconds, and
1179   // max of 1 request.  Also simulate a request having just started, so the
1180   // next request will be affected by backoff of ~400 seconds.
1181   context_getter->AddThrottlerEntry(
1182       url, std::string() /* url_id */, 400000 /* sliding_window_period_ms */,
1183       1 /* max_send_threshold */, 200000 /* initial_backoff_ms */,
1184       2.0 /* multiply_factor */, 0.0 /* jitter_factor */,
1185       400000 /* maximum_backoff_ms */,
1186       true /* reserve_sending_time_for_next_request*/);
1187
1188   // The getter won't be destroyed if the test holds on to a reference to it.
1189   context_getter = nullptr;
1190
1191   delegate.fetcher()->Start();
1192   delegate.CancelFetch();
1193   run_loop_.Run();
1194
1195   EXPECT_FALSE(delegate.did_complete());
1196 }
1197
1198 // A URLFetcherDelegate that expects to receive a response body of "request1"
1199 // and then reuses the fetcher for the same URL, setting the "test" request
1200 // header to "request2".
1201 class ReuseFetcherDelegate : public WaitingURLFetcherDelegate {
1202  public:
1203   // |second_request_context_getter| is the context getter used for the second
1204   // request. Can't reuse the old one because fetchers release it on completion.
1205   ReuseFetcherDelegate(
1206       scoped_refptr<URLRequestContextGetter> second_request_context_getter)
1207       : first_request_complete_(false),
1208         second_request_context_getter_(second_request_context_getter) {}
1209
1210   ~ReuseFetcherDelegate() override {}
1211
1212   void OnURLFetchComplete(const URLFetcher* source) override {
1213     EXPECT_EQ(fetcher(), source);
1214     if (!first_request_complete_) {
1215       first_request_complete_ = true;
1216       EXPECT_TRUE(fetcher()->GetStatus().is_success());
1217       EXPECT_EQ(200, fetcher()->GetResponseCode());
1218       std::string data;
1219       ASSERT_TRUE(fetcher()->GetResponseAsString(&data));
1220       EXPECT_EQ("request1", data);
1221
1222       fetcher()->SetRequestContext(second_request_context_getter_.get());
1223       fetcher()->SetExtraRequestHeaders("test: request2");
1224       fetcher()->Start();
1225       return;
1226     }
1227     WaitingURLFetcherDelegate::OnURLFetchComplete(source);
1228   }
1229
1230  private:
1231   bool first_request_complete_;
1232   scoped_refptr<URLRequestContextGetter> second_request_context_getter_;
1233
1234   DISALLOW_COPY_AND_ASSIGN(ReuseFetcherDelegate);
1235 };
1236
1237 TEST_F(URLFetcherTest, ReuseFetcherForSameURL) {
1238   // TODO(mmenke):  It's really weird that this is supported, particularly
1239   // some fields can be modified between requests, but some (Like upload body)
1240   // cannot be. Can we get rid of support for this?
1241   scoped_refptr<URLRequestContextGetter> context_getter(
1242       CreateSameThreadContextGetter());
1243   ReuseFetcherDelegate delegate(context_getter);
1244   delegate.CreateFetcher(test_server_->GetURL("echoheader?test"),
1245                          URLFetcher::GET, context_getter);
1246   delegate.fetcher()->SetExtraRequestHeaders("test: request1");
1247   delegate.StartFetcherAndWait();
1248
1249   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
1250   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
1251   std::string data;
1252   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
1253   EXPECT_EQ("request2", data);
1254 }
1255
1256 TEST_F(URLFetcherTest, ShutdownSameThread) {
1257   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
1258       CreateSameThreadContextGetter());
1259
1260   // Create a fetcher and wait for it to create a request.
1261   WaitingURLFetcherDelegate delegate1;
1262   delegate1.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
1263   delegate1.fetcher()->Start();
1264   // Need to spin the loop to ensure the URLRequest is created and started.
1265   base::RunLoop().RunUntilIdle();
1266
1267   // Create and start another fetcher, but don't wait for it to start. The task
1268   // to start the request should be in the message loop.
1269   WaitingURLFetcherDelegate delegate2;
1270   delegate2.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
1271   delegate2.fetcher()->Start();
1272
1273   // Check that shutting down the getter cancels the request synchronously,
1274   // allowing the context to be destroyed.
1275   context_getter->Shutdown();
1276
1277   // Wait for the first fetcher, make sure it failed.
1278   delegate1.WaitForComplete();
1279   EXPECT_FALSE(delegate1.fetcher()->GetStatus().is_success());
1280   EXPECT_EQ(ERR_CONTEXT_SHUT_DOWN, delegate1.fetcher()->GetStatus().error());
1281
1282   // Wait for the second fetcher, make sure it failed.
1283   delegate2.WaitForComplete();
1284   EXPECT_FALSE(delegate2.fetcher()->GetStatus().is_success());
1285   EXPECT_EQ(ERR_CONTEXT_SHUT_DOWN, delegate2.fetcher()->GetStatus().error());
1286
1287   // New fetchers should automatically fail without making new requests. This
1288   // should follow the same path as the second fetcher, but best to be safe.
1289   WaitingURLFetcherDelegate delegate3;
1290   delegate3.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
1291   delegate3.fetcher()->Start();
1292   delegate3.WaitForComplete();
1293   EXPECT_FALSE(delegate3.fetcher()->GetStatus().is_success());
1294   EXPECT_EQ(ERR_CONTEXT_SHUT_DOWN, delegate3.fetcher()->GetStatus().error());
1295 }
1296
1297 TEST_F(URLFetcherTest, ShutdownCrossThread) {
1298   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
1299       CreateCrossThreadContextGetter());
1300
1301   WaitingURLFetcherDelegate delegate1;
1302   delegate1.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
1303   delegate1.fetcher()->Start();
1304   // Check that shutting the context getter lets the context be destroyed safely
1305   // and cancels the request.
1306   context_getter->Shutdown();
1307   delegate1.WaitForComplete();
1308   EXPECT_FALSE(delegate1.fetcher()->GetStatus().is_success());
1309   EXPECT_EQ(ERR_CONTEXT_SHUT_DOWN, delegate1.fetcher()->GetStatus().error());
1310
1311   // New requests should automatically fail without making new requests.
1312   WaitingURLFetcherDelegate delegate2;
1313   delegate2.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
1314   delegate2.StartFetcherAndWait();
1315   EXPECT_FALSE(delegate2.fetcher()->GetStatus().is_success());
1316   EXPECT_EQ(ERR_CONTEXT_SHUT_DOWN, delegate2.fetcher()->GetStatus().error());
1317 }
1318
1319 // Get a small file.
1320 TEST_F(URLFetcherTest, FileTestSmallGet) {
1321   const char kFileToFetch[] = "simple.html";
1322
1323   base::ScopedTempDir temp_dir;
1324   ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
1325   base::FilePath out_path = temp_dir.path().AppendASCII(kFileToFetch);
1326   SaveFileTest(kFileToFetch, false, out_path, false);
1327 }
1328
1329 // Get a file large enough to require more than one read into URLFetcher::Core's
1330 // IOBuffer.
1331 TEST_F(URLFetcherTest, FileTestLargeGet) {
1332   const char kFileToFetch[] = "animate1.gif";
1333
1334   base::ScopedTempDir temp_dir;
1335   ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
1336   base::FilePath out_path = temp_dir.path().AppendASCII(kFileToFetch);
1337   SaveFileTest(kFileToFetch, false, out_path, false);
1338 }
1339
1340 // If the caller takes the ownership of the output file, the file should persist
1341 // even after URLFetcher is gone.
1342 TEST_F(URLFetcherTest, FileTestTakeOwnership) {
1343   const char kFileToFetch[] = "simple.html";
1344
1345   base::ScopedTempDir temp_dir;
1346   ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
1347   base::FilePath out_path = temp_dir.path().AppendASCII(kFileToFetch);
1348   SaveFileTest(kFileToFetch, false, out_path, true);
1349 }
1350
1351 // Test that an existing file can be overwritten be a fetcher.
1352 TEST_F(URLFetcherTest, FileTestOverwriteExisting) {
1353   base::ScopedTempDir temp_dir;
1354   ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
1355
1356   // Create a file before trying to fetch.
1357   const char kFileToFetch[] = "simple.html";
1358   std::string data(10000, '?');  // Meant to be larger than simple.html.
1359   base::FilePath out_path = temp_dir.path().AppendASCII(kFileToFetch);
1360   ASSERT_EQ(static_cast<int>(data.size()),
1361             base::WriteFile(out_path, data.data(), data.size()));
1362   ASSERT_TRUE(base::PathExists(out_path));
1363
1364   SaveFileTest(kFileToFetch, false, out_path, true);
1365 }
1366
1367 // Test trying to overwrite a directory with a file when using a fetcher fails.
1368 TEST_F(URLFetcherTest, FileTestTryToOverwriteDirectory) {
1369   base::ScopedTempDir temp_dir;
1370   ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
1371
1372   // Create a directory before trying to fetch.
1373   static const char kFileToFetch[] = "simple.html";
1374   base::FilePath out_path = temp_dir.path().AppendASCII(kFileToFetch);
1375   ASSERT_TRUE(base::CreateDirectory(out_path));
1376   ASSERT_TRUE(base::PathExists(out_path));
1377
1378   WaitingURLFetcherDelegate delegate;
1379   delegate.CreateFetcher(
1380       test_server_->GetURL(std::string(kTestServerFilePrefix) + kFileToFetch),
1381       URLFetcher::GET, CreateSameThreadContextGetter());
1382   delegate.fetcher()->SaveResponseToFileAtPath(
1383       out_path, scoped_refptr<base::SingleThreadTaskRunner>(
1384                     base::ThreadTaskRunnerHandle::Get()));
1385   delegate.StartFetcherAndWait();
1386
1387   EXPECT_FALSE(delegate.fetcher()->GetStatus().is_success());
1388   EXPECT_EQ(ERR_ACCESS_DENIED, delegate.fetcher()->GetStatus().error());
1389 }
1390
1391 // Get a small file and save it to a temp file.
1392 TEST_F(URLFetcherTest, TempFileTestSmallGet) {
1393   SaveFileTest("simple.html", true, base::FilePath(), false);
1394 }
1395
1396 // Get a file large enough to require more than one read into URLFetcher::Core's
1397 // IOBuffer and save it to a temp file.
1398 TEST_F(URLFetcherTest, TempFileTestLargeGet) {
1399   SaveFileTest("animate1.gif", true, base::FilePath(), false);
1400 }
1401
1402 // If the caller takes the ownership of the temp file, check that the file
1403 // persists even after URLFetcher is gone.
1404 TEST_F(URLFetcherTest, TempFileTestTakeOwnership) {
1405   SaveFileTest("simple.html", true, base::FilePath(), true);
1406 }
1407
1408 TEST_F(URLFetcherBadHTTPSTest, BadHTTPS) {
1409   WaitingURLFetcherDelegate delegate;
1410   delegate.CreateFetcher(test_server_->GetURL(kDefaultResponsePath),
1411                          URLFetcher::GET, CreateSameThreadContextGetter());
1412   delegate.StartFetcherAndWait();
1413
1414   EXPECT_EQ(URLRequestStatus::CANCELED,
1415             delegate.fetcher()->GetStatus().status());
1416   EXPECT_EQ(ERR_ABORTED, delegate.fetcher()->GetStatus().error());
1417   EXPECT_EQ(-1, delegate.fetcher()->GetResponseCode());
1418   EXPECT_TRUE(delegate.fetcher()->GetCookies().empty());
1419   std::string data;
1420   EXPECT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
1421   EXPECT_TRUE(data.empty());
1422 }
1423
1424 }  // namespace
1425
1426 }  // namespace net