net/url_request/url_fetcher_impl_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_fetcher_impl.h"
   6
   7 #include <algorithm>
   8 #include <string>
   9
  10 #include "base/bind.h"
  11 #include "base/files/file_util.h"
  12 #include "base/files/scoped_temp_dir.h"
  13 #include "base/location.h"
  14 #include "base/macros.h"
  15 #include "base/path_service.h"
  16 #include "base/run_loop.h"
  17 #include "base/single_thread_task_runner.h"
  18 #include "base/strings/stringprintf.h"
  19 #include "base/synchronization/waitable_event.h"
  20 #include "base/thread_task_runner_handle.h"
  21 #include "base/threading/thread.h"
  22 #include "build/build_config.h"
  23 #include "crypto/nss_util.h"
  24 #include "net/base/elements_upload_data_stream.h"
  25 #include "net/base/network_change_notifier.h"
  26 #include "net/base/upload_bytes_element_reader.h"
  27 #include "net/base/upload_element_reader.h"
  28 #include "net/base/upload_file_element_reader.h"
  29 #include "net/dns/mock_host_resolver.h"
  30 #include "net/http/http_response_headers.h"
  31 #include "net/test/spawned_test_server/spawned_test_server.h"
  32 #include "net/url_request/url_fetcher_delegate.h"
  33 #include "net/url_request/url_request_context_getter.h"
  34 #include "net/url_request/url_request_test_util.h"
  35 #include "net/url_request/url_request_throttler_manager.h"
  36 #include "testing/gtest/include/gtest/gtest.h"
  37
  38 #if defined(USE_NSS_CERTS) || defined(OS_IOS)
  39 #include "net/cert_net/nss_ocsp.h"
  40 #endif
  41
  42 namespace net {
  43
  44 using base::Time;
  45 using base::TimeDelta;
  46
  47 // TODO(eroman): Add a regression test for http://crbug.com/40505.
  48
  49 namespace {
  50
  51 // TODO(akalin): Move all the test data to somewhere under net/.
  52 const base::FilePath::CharType kDocRoot[] =
  53     FILE_PATH_LITERAL("net/data/url_fetcher_impl_unittest");
  54 const char kTestServerFilePrefix[] = "files/";
  55
  56 // Test server path and response body for the default URL used by many of the
  57 // tests.
  58 const char kDefaultResponsePath[] = "defaultresponse";
  59 const char kDefaultResponseBody[] =
  60     "Default response given for path: /defaultresponse";
  61
  62 // Request body for streams created by CreateUploadStream.
  63 const char kCreateUploadStreamBody[] = "rosebud";
  64
  65 base::FilePath GetUploadFileTestPath() {
  66   base::FilePath path;
  67   PathService::Get(base::DIR_SOURCE_ROOT, &path);
  68   return path.Append(
  69       FILE_PATH_LITERAL("net/data/url_request_unittest/BullRunSpeech.txt"));
  70 }
  71
  72 // Simple URLRequestDelegate that waits for the specified fetcher to complete.
  73 // Can only be used once.
  74 class WaitingURLFetcherDelegate : public URLFetcherDelegate {
  75  public:
  76   WaitingURLFetcherDelegate() : did_complete_(false) {}
  77
  78   void CreateFetcher(
  79       const GURL& url,
  80       URLFetcher::RequestType request_type,
  81       scoped_refptr<net::URLRequestContextGetter> context_getter) {
  82     fetcher_.reset(new URLFetcherImpl(url, request_type, this));
  83     fetcher_->SetRequestContext(context_getter.get());
  84   }
  85
  86   URLFetcher* fetcher() const { return fetcher_.get(); }
  87
  88   // Wait until the request has completed or been canceled.
  89   void StartFetcherAndWait() {
  90     fetcher_->Start();
  91     WaitForComplete();
  92   }
  93
  94   // Wait until the request has completed or been canceled. Does not start the
  95   // request.
  96   void WaitForComplete() { run_loop_.Run(); }
  97
  98   // Cancels the fetch by deleting the fetcher.
  99   void CancelFetch() {
 100     EXPECT_TRUE(fetcher_);
 101     fetcher_.reset();
 102     run_loop_.Quit();
 103   }
 104
 105   // URLFetcherDelegate:
 106   void OnURLFetchComplete(const URLFetcher* source) override {
 107     EXPECT_FALSE(did_complete_);
 108     EXPECT_TRUE(fetcher_);
 109     EXPECT_EQ(fetcher_.get(), source);
 110     did_complete_ = true;
 111     run_loop_.Quit();
 112   }
 113
 114   void OnURLFetchDownloadProgress(const URLFetcher* source,
 115                                   int64 current,
 116                                   int64 total) override {
 117     // Note that the current progress may be greater than the previous progress,
 118     // in the case of retrying the request.
 119     EXPECT_FALSE(did_complete_);
 120     EXPECT_TRUE(fetcher_);
 121     EXPECT_EQ(source, fetcher_.get());
 122
 123     EXPECT_LE(0, current);
 124     // If file size is not known, |total| is -1.
 125     if (total >= 0)
 126       EXPECT_LE(current, total);
 127   }
 128
 129   void OnURLFetchUploadProgress(const URLFetcher* source,
 130                                 int64 current,
 131                                 int64 total) override {
 132     // Note that the current progress may be greater than the previous progress,
 133     // in the case of retrying the request.
 134     EXPECT_FALSE(did_complete_);
 135     EXPECT_TRUE(fetcher_);
 136     EXPECT_EQ(source, fetcher_.get());
 137
 138     EXPECT_LE(0, current);
 139     // If file size is not known, |total| is -1.
 140     if (total >= 0)
 141       EXPECT_LE(current, total);
 142   }
 143
 144   bool did_complete() const { return did_complete_; }
 145
 146  private:
 147   bool did_complete_;
 148
 149   scoped_ptr<URLFetcherImpl> fetcher_;
 150   base::RunLoop run_loop_;
 151
 152   DISALLOW_COPY_AND_ASSIGN(WaitingURLFetcherDelegate);
 153 };
 154
 155 // A TestURLRequestContext with a ThrottleManager and a MockHostResolver.
 156 class FetcherTestURLRequestContext : public TestURLRequestContext {
 157  public:
 158   // All requests for |hanging_domain| will hang on host resolution until the
 159   // mock_resolver()->ResolveAllPending() is called.
 160   explicit FetcherTestURLRequestContext(const std::string& hanging_domain)
 161       : TestURLRequestContext(true), mock_resolver_(new MockHostResolver()) {
 162     mock_resolver_->set_ondemand_mode(true);
 163     mock_resolver_->rules()->AddRule(hanging_domain, "127.0.0.1");
 164     // Pass ownership to ContextStorage to ensure correct destruction order.
 165     context_storage_.set_host_resolver(
 166         scoped_ptr<HostResolver>(mock_resolver_));
 167     context_storage_.set_throttler_manager(new URLRequestThrottlerManager());
 168     Init();
 169   }
 170
 171   MockHostResolver* mock_resolver() { return mock_resolver_; }
 172
 173  private:
 174   MockHostResolver* mock_resolver_;
 175
 176   DISALLOW_COPY_AND_ASSIGN(FetcherTestURLRequestContext);
 177 };
 178
 179 class FetcherTestURLRequestContextGetter : public URLRequestContextGetter {
 180  public:
 181   FetcherTestURLRequestContextGetter(
 182       scoped_refptr<base::SingleThreadTaskRunner> network_task_runner,
 183       const std::string& hanging_domain)
 184       : network_task_runner_(network_task_runner),
 185         hanging_domain_(hanging_domain),
 186         shutting_down_(false) {}
 187
 188   // Sets callback to be invoked when the getter is destroyed.
 189   void set_on_destruction_callback(
 190       const base::Closure& on_destruction_callback) {
 191     on_destruction_callback_ = on_destruction_callback;
 192   }
 193
 194   // URLRequestContextGetter:
 195   FetcherTestURLRequestContext* GetURLRequestContext() override {
 196     // Calling this on the wrong thread may be either a bug in the test or a bug
 197     // in production code.
 198     EXPECT_TRUE(network_task_runner_->BelongsToCurrentThread());
 199
 200     if (shutting_down_)
 201       return nullptr;
 202
 203     if (!context_)
 204       context_.reset(new FetcherTestURLRequestContext(hanging_domain_));
 205     return context_.get();
 206   }
 207
 208   scoped_refptr<base::SingleThreadTaskRunner> GetNetworkTaskRunner()
 209       const override {
 210     return network_task_runner_;
 211   }
 212
 213   // Adds a throttler entry with the specified parameters.  Does this
 214   // synchronously if the context lives on the current thread, or posts a task
 215   // to the relevant thread otherwise.
 216   //
 217   // If |reserve_sending_time_for_next_request|, will start backoff early, as
 218   // if there has already been a request for |url|.
 219   void AddThrottlerEntry(const GURL& url,
 220                          const std::string& url_id,
 221                          int sliding_window_period_ms,
 222                          int max_send_threshold,
 223                          int initial_backoff_ms,
 224                          double multiply_factor,
 225                          double jitter_factor,
 226                          int maximum_backoff_ms,
 227                          bool reserve_sending_time_for_next_request) {
 228     if (!network_task_runner_->RunsTasksOnCurrentThread()) {
 229       network_task_runner_->PostTask(
 230           FROM_HERE,
 231           base::Bind(&FetcherTestURLRequestContextGetter::AddThrottlerEntry,
 232                      this, url, url_id, sliding_window_period_ms,
 233                      max_send_threshold, initial_backoff_ms, multiply_factor,
 234                      jitter_factor, maximum_backoff_ms,
 235                      reserve_sending_time_for_next_request));
 236       return;
 237     }
 238     scoped_refptr<URLRequestThrottlerEntry> entry(new URLRequestThrottlerEntry(
 239         GetURLRequestContext()->throttler_manager(), url_id,
 240         sliding_window_period_ms, max_send_threshold, initial_backoff_ms,
 241         multiply_factor, jitter_factor, maximum_backoff_ms));
 242
 243     GetURLRequestContext()->throttler_manager()->OverrideEntryForTests(
 244         url, entry.get());
 245
 246     if (reserve_sending_time_for_next_request)
 247       entry->ReserveSendingTimeForNextRequest(base::TimeTicks());
 248   }
 249
 250   // Tells the getter to act as if the URLRequestContext is about to be shut
 251   // down.
 252   void Shutdown() {
 253     if (!network_task_runner_->RunsTasksOnCurrentThread()) {
 254       network_task_runner_->PostTask(
 255           FROM_HERE,
 256           base::Bind(&FetcherTestURLRequestContextGetter::Shutdown, this));
 257       return;
 258     }
 259
 260     shutting_down_ = true;
 261     NotifyContextShuttingDown();
 262     // Should now be safe to destroy the context.  Context will check it has no
 263     // pending requests.
 264     context_.reset();
 265   }
 266
 267   // Convenience method to access the context as a FetcherTestURLRequestContext
 268   // without going through GetURLRequestContext.
 269   FetcherTestURLRequestContext* context() {
 270     DCHECK(network_task_runner_->BelongsToCurrentThread());
 271     return context_.get();
 272   }
 273
 274  protected:
 275   ~FetcherTestURLRequestContextGetter() override {
 276     // |context_| may only be deleted on the network thread. Fortunately,
 277     // the parent class already ensures it's deleted on the network thread.
 278     DCHECK(network_task_runner_->BelongsToCurrentThread());
 279     if (!on_destruction_callback_.is_null())
 280       on_destruction_callback_.Run();
 281   }
 282
 283   scoped_refptr<base::SingleThreadTaskRunner> network_task_runner_;
 284   const std::string hanging_domain_;
 285
 286   scoped_ptr<FetcherTestURLRequestContext> context_;
 287   bool shutting_down_;
 288
 289   base::Closure on_destruction_callback_;
 290
 291   DISALLOW_COPY_AND_ASSIGN(FetcherTestURLRequestContextGetter);
 292 };
 293
 294 }  // namespace
 295
 296 class URLFetcherTest : public testing::Test {
 297  public:
 298   URLFetcherTest() : num_upload_streams_created_(0) {}
 299
 300   static int GetNumFetcherCores() {
 301     return URLFetcherImpl::GetNumFetcherCores();
 302   }
 303
 304   // Creates a URLRequestContextGetter with a URLRequestContext that lives on
 305   // the current thread.
 306   scoped_refptr<FetcherTestURLRequestContextGetter>
 307   CreateSameThreadContextGetter() {
 308     return scoped_refptr<FetcherTestURLRequestContextGetter>(
 309         new FetcherTestURLRequestContextGetter(
 310             base::ThreadTaskRunnerHandle::Get(), hanging_url().host()));
 311   }
 312
 313   // Creates a URLRequestContextGetter with a URLRequestContext that lives on
 314   // a separate network thread.
 315   scoped_refptr<FetcherTestURLRequestContextGetter>
 316   CreateCrossThreadContextGetter() {
 317     if (!network_thread_) {
 318       network_thread_.reset(new base::Thread("network thread"));
 319       base::Thread::Options network_thread_options;
 320       network_thread_options.message_loop_type = base::MessageLoop::TYPE_IO;
 321       bool result = network_thread_->StartWithOptions(network_thread_options);
 322       CHECK(result);
 323     }
 324
 325     return scoped_refptr<FetcherTestURLRequestContextGetter>(
 326         new FetcherTestURLRequestContextGetter(network_thread_->task_runner(),
 327                                                hanging_url().host()));
 328   }
 329
 330   // Callback passed to URLFetcher to create upload stream by some tests.
 331   scoped_ptr<UploadDataStream> CreateUploadStream() {
 332     ++num_upload_streams_created_;
 333     std::vector<char> buffer(
 334         kCreateUploadStreamBody,
 335         kCreateUploadStreamBody + strlen(kCreateUploadStreamBody));
 336     return ElementsUploadDataStream::CreateWithReader(
 337         scoped_ptr<UploadElementReader>(
 338             new UploadOwnedBytesElementReader(&buffer)),
 339         0);
 340   }
 341
 342   // Number of streams created by CreateUploadStream.
 343   size_t num_upload_streams_created() const {
 344     return num_upload_streams_created_;
 345   }
 346
 347   // Downloads |file_to_fetch| and checks the contents when done.  If
 348   // |save_to_temporary_file| is true, saves it to a temporary file, and
 349   // |requested_out_path| is ignored. Otherwise, saves it to
 350   // |requested_out_path|. Takes ownership of the file if |take_ownership| is
 351   // true. Deletes file when done.
 352   void SaveFileTest(const char* file_to_fetch,
 353                     bool save_to_temporary_file,
 354                     const base::FilePath& requested_out_path,
 355                     bool take_ownership) {
 356     scoped_ptr<WaitingURLFetcherDelegate> delegate(
 357         new WaitingURLFetcherDelegate());
 358     delegate->CreateFetcher(
 359         test_server_->GetURL(std::string(kTestServerFilePrefix) +
 360                              file_to_fetch),
 361         URLFetcher::GET, CreateSameThreadContextGetter());
 362     if (save_to_temporary_file) {
 363       delegate->fetcher()->SaveResponseToTemporaryFile(
 364           scoped_refptr<base::SingleThreadTaskRunner>(
 365               base::ThreadTaskRunnerHandle::Get()));
 366     } else {
 367       delegate->fetcher()->SaveResponseToFileAtPath(
 368           requested_out_path, scoped_refptr<base::SingleThreadTaskRunner>(
 369                                   base::ThreadTaskRunnerHandle::Get()));
 370     }
 371     delegate->StartFetcherAndWait();
 372
 373     EXPECT_TRUE(delegate->fetcher()->GetStatus().is_success());
 374     EXPECT_EQ(200, delegate->fetcher()->GetResponseCode());
 375
 376     base::FilePath out_path;
 377     EXPECT_TRUE(
 378         delegate->fetcher()->GetResponseAsFilePath(take_ownership, &out_path));
 379     if (!save_to_temporary_file) {
 380       EXPECT_EQ(requested_out_path, out_path);
 381     }
 382
 383     EXPECT_TRUE(base::ContentsEqual(
 384         test_server_->GetDocumentRoot().AppendASCII(file_to_fetch), out_path));
 385
 386     // Delete the delegate and run the message loop to give the fetcher's
 387     // destructor a chance to delete the file.
 388     delegate.reset();
 389     base::RunLoop().RunUntilIdle();
 390
 391     // File should only exist if |take_ownership| was true.
 392     EXPECT_EQ(take_ownership, base::PathExists(out_path));
 393
 394     // Cleanup.
 395     if (base::PathExists(out_path))
 396       base::DeleteFile(out_path, false);
 397   }
 398
 399   // Returns a URL that hangs on DNS resolution when using a context created by
 400   // the test fixture.
 401   const GURL& hanging_url() const { return hanging_url_; }
 402
 403   // testing::Test:
 404   void SetUp() override {
 405     SetUpServer();
 406     ASSERT_TRUE(test_server_->Start());
 407
 408     // URL that will hang when lookups reach the host resolver.
 409     hanging_url_ = GURL(base::StringPrintf(
 410         "http://example.com:%d/%s", test_server_->host_port_pair().port(),
 411         kDefaultResponsePath));
 412     ASSERT_TRUE(hanging_url_.is_valid());
 413
 414 #if defined(USE_NSS_CERTS) || defined(OS_IOS)
 415     crypto::EnsureNSSInit();
 416     EnsureNSSHttpIOInit();
 417 #endif
 418   }
 419
 420   void TearDown() override {
 421 #if defined(USE_NSS_CERTS) || defined(OS_IOS)
 422     ShutdownNSSHttpIO();
 423 #endif
 424   }
 425
 426   // Initializes |test_server_| without starting it.  Allows subclasses to use
 427   // their own server configuration.
 428   virtual void SetUpServer() {
 429     test_server_.reset(new SpawnedTestServer(SpawnedTestServer::TYPE_HTTP,
 430                                              SpawnedTestServer::kLocalhost,
 431                                              base::FilePath(kDocRoot)));
 432   }
 433
 434   // Network thread for cross-thread tests.  Most threads just use the main
 435   // thread for network activity.
 436   scoped_ptr<base::Thread> network_thread_;
 437
 438   scoped_ptr<SpawnedTestServer> test_server_;
 439   GURL hanging_url_;
 440
 441   size_t num_upload_streams_created_;
 442 };
 443
 444 namespace {
 445
 446 // Version of URLFetcherTest that tests bad HTTPS requests.
 447 class URLFetcherBadHTTPSTest : public URLFetcherTest {
 448  public:
 449   URLFetcherBadHTTPSTest() {}
 450
 451   // URLFetcherTest:
 452   void SetUpServer() override {
 453     SpawnedTestServer::SSLOptions ssl_options(
 454         SpawnedTestServer::SSLOptions::CERT_EXPIRED);
 455     test_server_.reset(new SpawnedTestServer(
 456         SpawnedTestServer::TYPE_HTTPS, ssl_options, base::FilePath(kDocRoot)));
 457   }
 458 };
 459
 460 // Create the fetcher on the main thread.  Since network IO will happen on the
 461 // main thread, this will test URLFetcher's ability to do everything on one
 462 // thread.
 463 TEST_F(URLFetcherTest, SameThreadTest) {
 464   WaitingURLFetcherDelegate delegate;
 465   delegate.CreateFetcher(test_server_->GetURL(kDefaultResponsePath),
 466                          URLFetcher::GET, CreateSameThreadContextGetter());
 467   delegate.StartFetcherAndWait();
 468
 469   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 470   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 471   std::string data;
 472   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 473   EXPECT_EQ(kDefaultResponseBody, data);
 474 }
 475
 476 // Create a separate thread that will create the URLFetcher.  A separate thread
 477 // acts as the network thread.
 478 TEST_F(URLFetcherTest, DifferentThreadsTest) {
 479   WaitingURLFetcherDelegate delegate;
 480   delegate.CreateFetcher(test_server_->GetURL(kDefaultResponsePath),
 481                          URLFetcher::GET, CreateCrossThreadContextGetter());
 482   delegate.StartFetcherAndWait();
 483
 484   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 485   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 486   std::string data;
 487   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 488   EXPECT_EQ(kDefaultResponseBody, data);
 489 }
 490
 491 // Tests to make sure CancelAll() will successfully cancel existing URLFetchers.
 492 TEST_F(URLFetcherTest, CancelAll) {
 493   EXPECT_EQ(0, GetNumFetcherCores());
 494
 495   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
 496       CreateSameThreadContextGetter());
 497   // Force context creation.
 498   context_getter->GetURLRequestContext();
 499   MockHostResolver* mock_resolver = context_getter->context()->mock_resolver();
 500
 501   WaitingURLFetcherDelegate delegate;
 502   delegate.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
 503   delegate.fetcher()->Start();
 504   // Wait for the request to reach the mock resolver and hang, to ensure the
 505   // request has actually started.
 506   base::RunLoop().RunUntilIdle();
 507   EXPECT_TRUE(mock_resolver->has_pending_requests());
 508
 509   EXPECT_EQ(1, URLFetcherTest::GetNumFetcherCores());
 510   URLFetcherImpl::CancelAll();
 511   EXPECT_EQ(0, URLFetcherTest::GetNumFetcherCores());
 512 }
 513
 514 TEST_F(URLFetcherTest, DontRetryOnNetworkChangedByDefault) {
 515   EXPECT_EQ(0, GetNumFetcherCores());
 516
 517   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
 518       CreateSameThreadContextGetter());
 519   // Force context creation.
 520   context_getter->GetURLRequestContext();
 521   MockHostResolver* mock_resolver = context_getter->context()->mock_resolver();
 522
 523   WaitingURLFetcherDelegate delegate;
 524   delegate.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
 525   EXPECT_FALSE(mock_resolver->has_pending_requests());
 526
 527   // This posts a task to start the fetcher.
 528   delegate.fetcher()->Start();
 529   base::RunLoop().RunUntilIdle();
 530
 531   // The fetcher is now running, but is pending the host resolve.
 532   EXPECT_EQ(1, GetNumFetcherCores());
 533   EXPECT_TRUE(mock_resolver->has_pending_requests());
 534   ASSERT_FALSE(delegate.did_complete());
 535
 536   // A network change notification aborts the connect job.
 537   NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
 538   delegate.WaitForComplete();
 539   EXPECT_FALSE(mock_resolver->has_pending_requests());
 540
 541   // And the owner of the fetcher gets the ERR_NETWORK_CHANGED error.
 542   EXPECT_EQ(hanging_url(), delegate.fetcher()->GetOriginalURL());
 543   ASSERT_FALSE(delegate.fetcher()->GetStatus().is_success());
 544   EXPECT_EQ(ERR_NETWORK_CHANGED, delegate.fetcher()->GetStatus().error());
 545 }
 546
 547 TEST_F(URLFetcherTest, RetryOnNetworkChangedAndFail) {
 548   EXPECT_EQ(0, GetNumFetcherCores());
 549
 550   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
 551       CreateSameThreadContextGetter());
 552   // Force context creation.
 553   context_getter->GetURLRequestContext();
 554   MockHostResolver* mock_resolver = context_getter->context()->mock_resolver();
 555
 556   WaitingURLFetcherDelegate delegate;
 557   delegate.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
 558   delegate.fetcher()->SetAutomaticallyRetryOnNetworkChanges(3);
 559   EXPECT_FALSE(mock_resolver->has_pending_requests());
 560
 561   // This posts a task to start the fetcher.
 562   delegate.fetcher()->Start();
 563   base::RunLoop().RunUntilIdle();
 564
 565   // The fetcher is now running, but is pending the host resolve.
 566   EXPECT_EQ(1, GetNumFetcherCores());
 567   EXPECT_TRUE(mock_resolver->has_pending_requests());
 568   ASSERT_FALSE(delegate.did_complete());
 569
 570   // Make it fail 3 times.
 571   for (int i = 0; i < 3; ++i) {
 572     // A network change notification aborts the connect job.
 573     NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
 574     base::RunLoop().RunUntilIdle();
 575
 576     // But the fetcher retries automatically.
 577     EXPECT_EQ(1, GetNumFetcherCores());
 578     EXPECT_TRUE(mock_resolver->has_pending_requests());
 579     ASSERT_FALSE(delegate.did_complete());
 580   }
 581
 582   // A 4th failure doesn't trigger another retry, and propagates the error
 583   // to the owner of the fetcher.
 584   NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
 585   delegate.WaitForComplete();
 586   EXPECT_FALSE(mock_resolver->has_pending_requests());
 587
 588   // And the owner of the fetcher gets the ERR_NETWORK_CHANGED error.
 589   EXPECT_EQ(hanging_url(), delegate.fetcher()->GetOriginalURL());
 590   ASSERT_FALSE(delegate.fetcher()->GetStatus().is_success());
 591   EXPECT_EQ(ERR_NETWORK_CHANGED, delegate.fetcher()->GetStatus().error());
 592 }
 593
 594 TEST_F(URLFetcherTest, RetryOnNetworkChangedAndSucceed) {
 595   EXPECT_EQ(0, GetNumFetcherCores());
 596
 597   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
 598       CreateSameThreadContextGetter());
 599   // Force context creation.
 600   context_getter->GetURLRequestContext();
 601   MockHostResolver* mock_resolver = context_getter->context()->mock_resolver();
 602
 603   WaitingURLFetcherDelegate delegate;
 604   delegate.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
 605   delegate.fetcher()->SetAutomaticallyRetryOnNetworkChanges(3);
 606   EXPECT_FALSE(mock_resolver->has_pending_requests());
 607
 608   // This posts a task to start the fetcher.
 609   delegate.fetcher()->Start();
 610   base::RunLoop().RunUntilIdle();
 611
 612   // The fetcher is now running, but is pending the host resolve.
 613   EXPECT_EQ(1, GetNumFetcherCores());
 614   EXPECT_TRUE(mock_resolver->has_pending_requests());
 615   ASSERT_FALSE(delegate.did_complete());
 616
 617   // Make it fail 3 times.
 618   for (int i = 0; i < 3; ++i) {
 619     // A network change notification aborts the connect job.
 620     NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
 621     base::RunLoop().RunUntilIdle();
 622
 623     // But the fetcher retries automatically.
 624     EXPECT_EQ(1, GetNumFetcherCores());
 625     EXPECT_TRUE(mock_resolver->has_pending_requests());
 626     ASSERT_FALSE(delegate.did_complete());
 627   }
 628
 629   // Now let it succeed by resolving the pending request.
 630   mock_resolver->ResolveAllPending();
 631   delegate.WaitForComplete();
 632   EXPECT_FALSE(mock_resolver->has_pending_requests());
 633
 634   // This time the request succeeded.
 635   EXPECT_EQ(hanging_url(), delegate.fetcher()->GetOriginalURL());
 636   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 637   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 638
 639   std::string data;
 640   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 641   EXPECT_EQ(kDefaultResponseBody, data);
 642 }
 643
 644 TEST_F(URLFetcherTest, PostString) {
 645   const char kUploadData[] = "bobsyeruncle";
 646
 647   WaitingURLFetcherDelegate delegate;
 648   delegate.CreateFetcher(test_server_->GetURL("echo"), URLFetcher::POST,
 649                          CreateSameThreadContextGetter());
 650   delegate.fetcher()->SetUploadData("application/x-www-form-urlencoded",
 651                                     kUploadData);
 652   delegate.StartFetcherAndWait();
 653
 654   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 655   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 656   std::string data;
 657   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 658   EXPECT_EQ(kUploadData, data);
 659 }
 660
 661 TEST_F(URLFetcherTest, PostEmptyString) {
 662   const char kUploadData[] = "";
 663
 664   WaitingURLFetcherDelegate delegate;
 665   delegate.CreateFetcher(test_server_->GetURL("echo"), URLFetcher::POST,
 666                          CreateSameThreadContextGetter());
 667   delegate.fetcher()->SetUploadData("application/x-www-form-urlencoded",
 668                                     kUploadData);
 669   delegate.StartFetcherAndWait();
 670
 671   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 672   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 673   std::string data;
 674   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 675   EXPECT_EQ(kUploadData, data);
 676 }
 677
 678 TEST_F(URLFetcherTest, PostEntireFile) {
 679   base::FilePath upload_path = GetUploadFileTestPath();
 680
 681   WaitingURLFetcherDelegate delegate;
 682   delegate.CreateFetcher(test_server_->GetURL("echo"), URLFetcher::POST,
 683                          CreateSameThreadContextGetter());
 684   delegate.fetcher()->SetUploadFilePath("application/x-www-form-urlencoded",
 685                                         upload_path, 0, kuint64max,
 686                                         base::ThreadTaskRunnerHandle::Get());
 687   delegate.StartFetcherAndWait();
 688
 689   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 690   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 691
 692   std::string expected;
 693   ASSERT_TRUE(base::ReadFileToString(upload_path, &expected));
 694   std::string data;
 695   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 696   EXPECT_EQ(expected, data);
 697 }
 698
 699 TEST_F(URLFetcherTest, PostFileRange) {
 700   const size_t kRangeStart = 30;
 701   const size_t kRangeLength = 100;
 702   base::FilePath upload_path = GetUploadFileTestPath();
 703
 704   WaitingURLFetcherDelegate delegate;
 705   delegate.CreateFetcher(test_server_->GetURL("echo"), URLFetcher::POST,
 706                          CreateSameThreadContextGetter());
 707   delegate.fetcher()->SetUploadFilePath("application/x-www-form-urlencoded",
 708                                         upload_path, kRangeStart, kRangeLength,
 709                                         base::ThreadTaskRunnerHandle::Get());
 710   delegate.StartFetcherAndWait();
 711
 712   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 713   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 714
 715   std::string expected;
 716   ASSERT_TRUE(base::ReadFileToString(upload_path, &expected));
 717   std::string data;
 718   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 719   EXPECT_EQ(expected.substr(kRangeStart, kRangeLength), data);
 720 }
 721
 722 TEST_F(URLFetcherTest, PostWithUploadStreamFactory) {
 723   WaitingURLFetcherDelegate delegate;
 724   delegate.CreateFetcher(test_server_->GetURL("echo"), URLFetcher::POST,
 725                          CreateSameThreadContextGetter());
 726   delegate.fetcher()->SetUploadStreamFactory(
 727       "text/plain",
 728       base::Bind(&URLFetcherTest::CreateUploadStream, base::Unretained(this)));
 729   delegate.StartFetcherAndWait();
 730
 731   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 732   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 733   std::string data;
 734   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 735   EXPECT_EQ(kCreateUploadStreamBody, data);
 736   EXPECT_EQ(1u, num_upload_streams_created());
 737 }
 738
 739 TEST_F(URLFetcherTest, PostWithUploadStreamFactoryAndRetries) {
 740   WaitingURLFetcherDelegate delegate;
 741   delegate.CreateFetcher(test_server_->GetURL("echo?status=500"),
 742                          URLFetcher::POST, CreateSameThreadContextGetter());
 743   delegate.fetcher()->SetAutomaticallyRetryOn5xx(true);
 744   delegate.fetcher()->SetMaxRetriesOn5xx(1);
 745   delegate.fetcher()->SetUploadStreamFactory(
 746       "text/plain",
 747       base::Bind(&URLFetcherTest::CreateUploadStream, base::Unretained(this)));
 748   delegate.StartFetcherAndWait();
 749
 750   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 751   EXPECT_EQ(500, delegate.fetcher()->GetResponseCode());
 752   std::string data;
 753   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 754   EXPECT_EQ(kCreateUploadStreamBody, data);
 755   EXPECT_EQ(2u, num_upload_streams_created());
 756 }
 757
 758 // Checks that upload progress increases over time, never exceeds what's already
 759 // been sent, and adds a chunk whenever all previously appended chunks have
 760 // been uploaded.
 761 class CheckUploadProgressDelegate : public WaitingURLFetcherDelegate {
 762  public:
 763   CheckUploadProgressDelegate()
 764       : chunk_(1 << 16, 'a'), num_chunks_appended_(0), last_seen_progress_(0) {}
 765   ~CheckUploadProgressDelegate() override {}
 766
 767   void OnURLFetchUploadProgress(const URLFetcher* source,
 768                                 int64 current,
 769                                 int64 total) override {
 770     // Run default checks.
 771     WaitingURLFetcherDelegate::OnURLFetchUploadProgress(source, current, total);
 772
 773     EXPECT_LE(last_seen_progress_, current);
 774     EXPECT_LE(current, bytes_appended());
 775     last_seen_progress_ = current;
 776     MaybeAppendChunk();
 777   }
 778
 779   // Append the next chunk if all previously appended chunks have been sent.
 780   void MaybeAppendChunk() {
 781     const int kNumChunks = 5;
 782     if (last_seen_progress_ == bytes_appended() &&
 783         num_chunks_appended_ < kNumChunks) {
 784       ++num_chunks_appended_;
 785       fetcher()->AppendChunkToUpload(chunk_,
 786                                      num_chunks_appended_ == kNumChunks);
 787     }
 788   }
 789
 790  private:
 791   int64 bytes_appended() const { return num_chunks_appended_ * chunk_.size(); }
 792
 793   const std::string chunk_;
 794
 795   int64 num_chunks_appended_;
 796   int64 last_seen_progress_;
 797
 798   DISALLOW_COPY_AND_ASSIGN(CheckUploadProgressDelegate);
 799 };
 800
 801 TEST_F(URLFetcherTest, UploadProgress) {
 802   CheckUploadProgressDelegate delegate;
 803   delegate.CreateFetcher(test_server_->GetURL("echo"), URLFetcher::POST,
 804                          CreateSameThreadContextGetter());
 805   // Use a chunked upload so that the upload can be paused after uploading data.
 806   // Since upload progress uses a timer, the delegate may not receive any
 807   // notification otherwise.
 808   delegate.fetcher()->SetChunkedUpload("application/x-www-form-urlencoded");
 809
 810   delegate.fetcher()->Start();
 811   // Append the first chunk.  Others will be appended automatically in response
 812   // to OnURLFetchUploadProgress events.
 813   delegate.MaybeAppendChunk();
 814   delegate.WaitForComplete();
 815
 816   // Make sure there are no pending events that cause problems when run.
 817   base::RunLoop().RunUntilIdle();
 818
 819   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 820   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 821   EXPECT_TRUE(delegate.did_complete());
 822 }
 823
 824 // Checks that download progress never decreases, never exceeds file size, and
 825 // that file size is correctly reported.
 826 class CheckDownloadProgressDelegate : public WaitingURLFetcherDelegate {
 827  public:
 828   CheckDownloadProgressDelegate(int64 file_size)
 829       : file_size_(file_size), last_seen_progress_(0) {}
 830   ~CheckDownloadProgressDelegate() override {}
 831
 832   void OnURLFetchDownloadProgress(const URLFetcher* source,
 833                                   int64 current,
 834                                   int64 total) override {
 835     // Run default checks.
 836     WaitingURLFetcherDelegate::OnURLFetchDownloadProgress(source, current,
 837                                                           total);
 838
 839     EXPECT_LE(last_seen_progress_, current);
 840     EXPECT_EQ(file_size_, total);
 841     last_seen_progress_ = current;
 842   }
 843
 844  private:
 845   int64 file_size_;
 846   int64 last_seen_progress_;
 847
 848   DISALLOW_COPY_AND_ASSIGN(CheckDownloadProgressDelegate);
 849 };
 850
 851 TEST_F(URLFetcherTest, DownloadProgress) {
 852   // Get a file large enough to require more than one read into
 853   // URLFetcher::Core's IOBuffer.
 854   const char kFileToFetch[] = "animate1.gif";
 855
 856   std::string file_contents;
 857   ASSERT_TRUE(base::ReadFileToString(
 858       test_server_->GetDocumentRoot().AppendASCII(kFileToFetch),
 859       &file_contents));
 860
 861   CheckDownloadProgressDelegate delegate(file_contents.size());
 862   delegate.CreateFetcher(
 863       test_server_->GetURL(std::string(kTestServerFilePrefix) + kFileToFetch),
 864       URLFetcher::GET, CreateSameThreadContextGetter());
 865   delegate.StartFetcherAndWait();
 866
 867   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 868   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 869   std::string data;
 870   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 871   EXPECT_EQ(file_contents, data);
 872 }
 873
 874 class CancelOnUploadProgressDelegate : public WaitingURLFetcherDelegate {
 875  public:
 876   CancelOnUploadProgressDelegate() {}
 877   ~CancelOnUploadProgressDelegate() override {}
 878
 879   void OnURLFetchUploadProgress(const URLFetcher* source,
 880                                 int64 current,
 881                                 int64 total) override {
 882     CancelFetch();
 883   }
 884
 885  private:
 886   DISALLOW_COPY_AND_ASSIGN(CancelOnUploadProgressDelegate);
 887 };
 888
 889 // Check that a fetch can be safely cancelled/deleted during an upload progress
 890 // callback.
 891 TEST_F(URLFetcherTest, CancelInUploadProgressCallback) {
 892   CancelOnUploadProgressDelegate delegate;
 893   delegate.CreateFetcher(test_server_->GetURL("echo"), URLFetcher::POST,
 894                          CreateSameThreadContextGetter());
 895   delegate.fetcher()->SetChunkedUpload("application/x-www-form-urlencoded");
 896   delegate.fetcher()->Start();
 897   // Use a chunked upload so that the upload can be paused after uploading data.
 898   // Since uploads progress uses a timer, may not receive any notification,
 899   // otherwise.
 900   std::string upload_data(1 << 16, 'a');
 901   delegate.fetcher()->AppendChunkToUpload(upload_data, false);
 902   delegate.WaitForComplete();
 903
 904   // Make sure there are no pending events that cause problems when run.
 905   base::RunLoop().RunUntilIdle();
 906
 907   EXPECT_FALSE(delegate.did_complete());
 908   EXPECT_FALSE(delegate.fetcher());
 909 }
 910
 911 class CancelOnDownloadProgressDelegate : public WaitingURLFetcherDelegate {
 912  public:
 913   CancelOnDownloadProgressDelegate() {}
 914   ~CancelOnDownloadProgressDelegate() override {}
 915
 916   void OnURLFetchDownloadProgress(const URLFetcher* source,
 917                                   int64 current,
 918                                   int64 total) override {
 919     CancelFetch();
 920   }
 921
 922  private:
 923   DISALLOW_COPY_AND_ASSIGN(CancelOnDownloadProgressDelegate);
 924 };
 925
 926 // Check that a fetch can be safely cancelled/deleted during a download progress
 927 // callback.
 928 TEST_F(URLFetcherTest, CancelInDownloadProgressCallback) {
 929   // Get a file large enough to require more than one read into
 930   // URLFetcher::Core's IOBuffer.
 931   static const char kFileToFetch[] = "animate1.gif";
 932   CancelOnDownloadProgressDelegate delegate;
 933   delegate.CreateFetcher(
 934       test_server_->GetURL(std::string(kTestServerFilePrefix) + kFileToFetch),
 935       URLFetcher::GET, CreateSameThreadContextGetter());
 936   delegate.StartFetcherAndWait();
 937
 938   // Make sure there are no pending events that cause problems when run.
 939   base::RunLoop().RunUntilIdle();
 940
 941   EXPECT_FALSE(delegate.did_complete());
 942   EXPECT_FALSE(delegate.fetcher());
 943 }
 944
 945 TEST_F(URLFetcherTest, Headers) {
 946   WaitingURLFetcherDelegate delegate;
 947   delegate.CreateFetcher(
 948       test_server_->GetURL("set-header?cache-control: private"),
 949       URLFetcher::GET, CreateSameThreadContextGetter());
 950   delegate.StartFetcherAndWait();
 951
 952   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 953   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 954   std::string header;
 955   ASSERT_TRUE(delegate.fetcher()->GetResponseHeaders()->GetNormalizedHeader(
 956       "cache-control", &header));
 957   EXPECT_EQ("private", header);
 958 }
 959
 960 TEST_F(URLFetcherTest, SocketAddress) {
 961   WaitingURLFetcherDelegate delegate;
 962   delegate.CreateFetcher(test_server_->GetURL(kDefaultResponsePath),
 963                          URLFetcher::GET, CreateSameThreadContextGetter());
 964   delegate.StartFetcherAndWait();
 965
 966   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 967   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 968   EXPECT_EQ(test_server_->host_port_pair().port(),
 969             delegate.fetcher()->GetSocketAddress().port());
 970   EXPECT_EQ(test_server_->host_port_pair().host(),
 971             delegate.fetcher()->GetSocketAddress().host());
 972 }
 973
 974 TEST_F(URLFetcherTest, StopOnRedirect) {
 975   const char kRedirectTarget[] = "http://redirect.target.com";
 976
 977   WaitingURLFetcherDelegate delegate;
 978   delegate.CreateFetcher(
 979       test_server_->GetURL(std::string("server-redirect?") + kRedirectTarget),
 980       URLFetcher::GET, CreateSameThreadContextGetter());
 981   delegate.fetcher()->SetStopOnRedirect(true);
 982   delegate.StartFetcherAndWait();
 983
 984   EXPECT_EQ(GURL(kRedirectTarget), delegate.fetcher()->GetURL());
 985   EXPECT_EQ(URLRequestStatus::CANCELED,
 986             delegate.fetcher()->GetStatus().status());
 987   EXPECT_EQ(ERR_ABORTED, delegate.fetcher()->GetStatus().error());
 988   EXPECT_EQ(301, delegate.fetcher()->GetResponseCode());
 989 }
 990
 991 TEST_F(URLFetcherTest, ThrottleOnRepeatedFetches) {
 992   base::Time start_time = Time::Now();
 993   GURL url(test_server_->GetURL(kDefaultResponsePath));
 994
 995   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
 996       CreateSameThreadContextGetter());
 997
 998   // Registers an entry for test url. It only allows 3 requests to be sent
 999   // in 200 milliseconds.
1000   context_getter->AddThrottlerEntry(
1001       url, std::string() /* url_id */, 200 /* sliding_window_period_ms */,
1002       3 /* max_send_threshold */, 1 /* initial_backoff_ms */,
1003       2.0 /* multiply_factor */, 0.0 /* jitter_factor */,
1004       256 /* maximum_backoff_ms */,
1005       false /* reserve_sending_time_for_next_request*/);
1006
1007   for (int i = 0; i < 20; ++i) {
1008     WaitingURLFetcherDelegate delegate;
1009     delegate.CreateFetcher(url, URLFetcher::GET, context_getter);
1010     delegate.StartFetcherAndWait();
1011
1012     EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
1013     EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
1014   }
1015
1016   // 20 requests were sent. Due to throttling, they should have collectively
1017   // taken over 1 second.
1018   EXPECT_GE(Time::Now() - start_time, base::TimeDelta::FromSeconds(1));
1019 }
1020
1021 TEST_F(URLFetcherTest, ThrottleOn5xxRetries) {
1022   base::Time start_time = Time::Now();
1023   GURL url(test_server_->GetURL("files/server-unavailable.html"));
1024
1025   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
1026       CreateSameThreadContextGetter());
1027
1028   // Registers an entry for test url. The backoff time is calculated by:
1029   //     new_backoff = 2.0 * old_backoff + 0
1030   // and maximum backoff time is 256 milliseconds.
1031   // Maximum retries allowed is set to 11.
1032   context_getter->AddThrottlerEntry(
1033       url, std::string() /* url_id */, 200 /* sliding_window_period_ms */,
1034       3 /* max_send_threshold */, 1 /* initial_backoff_ms */,
1035       2.0 /* multiply_factor */, 0.0 /* jitter_factor */,
1036       256 /* maximum_backoff_ms */,
1037       false /* reserve_sending_time_for_next_request*/);
1038
1039   WaitingURLFetcherDelegate delegate;
1040   delegate.CreateFetcher(url, URLFetcher::GET, context_getter);
1041   delegate.fetcher()->SetAutomaticallyRetryOn5xx(true);
1042   delegate.fetcher()->SetMaxRetriesOn5xx(11);
1043   delegate.StartFetcherAndWait();
1044
1045   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
1046   EXPECT_EQ(503, delegate.fetcher()->GetResponseCode());
1047   std::string data;
1048   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
1049   EXPECT_FALSE(data.empty());
1050
1051   // The request should have been retried 11 times (12 times including the first
1052   // attempt).  Due to throttling, they should have collectively taken over 1
1053   // second.
1054   EXPECT_GE(Time::Now() - start_time, base::TimeDelta::FromSeconds(1));
1055 }
1056
1057 // Tests overload protection, when responses passed through.
1058 TEST_F(URLFetcherTest, ProtectTestPassedThrough) {
1059   base::Time start_time = Time::Now();
1060   GURL url(test_server_->GetURL("files/server-unavailable.html"));
1061
1062   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
1063       CreateSameThreadContextGetter());
1064
1065   // Registers an entry for test url. The backoff time is calculated by:
1066   //     new_backoff = 2.0 * old_backoff + 0
1067   // and maximum backoff time is 150000 milliseconds.
1068   // Maximum retries allowed is set to 11.
1069   // Total time if *not* for not doing automatic backoff would be 150s.
1070   // In reality it should be "as soon as server responds".
1071   context_getter->AddThrottlerEntry(
1072       url, std::string() /* url_id */, 200 /* sliding_window_period_ms */,
1073       3 /* max_send_threshold */, 10000 /* initial_backoff_ms */,
1074       2.0 /* multiply_factor */, 0.0 /* jitter_factor */,
1075       150000 /* maximum_backoff_ms */,
1076       false /* reserve_sending_time_for_next_request*/);
1077
1078   WaitingURLFetcherDelegate delegate;
1079   delegate.CreateFetcher(url, URLFetcher::GET, context_getter);
1080   delegate.fetcher()->SetAutomaticallyRetryOn5xx(false);
1081   delegate.fetcher()->SetMaxRetriesOn5xx(11);
1082   delegate.StartFetcherAndWait();
1083
1084   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
1085   EXPECT_EQ(503, delegate.fetcher()->GetResponseCode());
1086   std::string data;
1087   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
1088   EXPECT_FALSE(data.empty());
1089   EXPECT_GT(delegate.fetcher()->GetBackoffDelay().InMicroseconds(), 0);
1090
1091   // The request should not have been retried at all.  If it had attempted all
1092   // 11 retries, that should have taken 2.5 minutes.
1093   EXPECT_TRUE(Time::Now() - start_time < TimeDelta::FromMinutes(1));
1094 }
1095
1096 // Used to check if a callback has been invoked.
1097 void SetBoolToTrue(bool* ptr) {
1098   *ptr = true;
1099 }
1100
1101 // Make sure that the URLFetcher cancels the URLRequest and releases its context
1102 // getter pointer synchronously when the fetcher and request context live on
1103 // the same thread.
1104 TEST_F(URLFetcherTest, CancelSameThread) {
1105   WaitingURLFetcherDelegate delegate;
1106   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
1107       CreateSameThreadContextGetter());
1108   bool getter_was_destroyed = false;
1109   context_getter->set_on_destruction_callback(
1110       base::Bind(&SetBoolToTrue, &getter_was_destroyed));
1111   delegate.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
1112
1113   // The getter won't be destroyed if the test holds on to a reference to it.
1114   context_getter = nullptr;
1115
1116   delegate.fetcher()->Start();
1117   // Give the fetcher a chance to start the request.
1118   base::RunLoop().RunUntilIdle();
1119   EXPECT_EQ(1, URLFetcherTest::GetNumFetcherCores());
1120
1121   // On same-thread cancel, the request should be canceled and getter destroyed
1122   // synchronously, for safe shutdown.
1123   delegate.CancelFetch();
1124   EXPECT_EQ(0, URLFetcherTest::GetNumFetcherCores());
1125   EXPECT_TRUE(getter_was_destroyed);
1126 }
1127
1128 // Make sure that the URLFetcher releases its context getter pointer on
1129 // cancellation, cross-thread case.
1130 TEST_F(URLFetcherTest, CancelDifferentThreads) {
1131   base::RunLoop run_loop_;
1132
1133   WaitingURLFetcherDelegate delegate;
1134   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
1135       CreateCrossThreadContextGetter());
1136   context_getter->set_on_destruction_callback(base::Bind(
1137       base::IgnoreResult(&base::SingleThreadTaskRunner::PostTask),
1138       base::ThreadTaskRunnerHandle::Get(), FROM_HERE, run_loop_.QuitClosure()));
1139   delegate.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
1140
1141   // The getter won't be destroyed if the test holds on to a reference to it.
1142   context_getter = nullptr;
1143
1144   delegate.fetcher()->Start();
1145   delegate.CancelFetch();
1146   run_loop_.Run();
1147
1148   EXPECT_FALSE(delegate.did_complete());
1149 }
1150
1151 TEST_F(URLFetcherTest, CancelWhileDelayedByThrottleDifferentThreads) {
1152   GURL url = test_server_->GetURL(kDefaultResponsePath);
1153   base::RunLoop run_loop_;
1154
1155   WaitingURLFetcherDelegate delegate;
1156   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
1157       CreateCrossThreadContextGetter());
1158   context_getter->set_on_destruction_callback(base::Bind(
1159       base::IgnoreResult(&base::SingleThreadTaskRunner::PostTask),
1160       base::ThreadTaskRunnerHandle::Get(), FROM_HERE, run_loop_.QuitClosure()));
1161   delegate.CreateFetcher(url, URLFetcher::GET, context_getter);
1162
1163   // Register an entry for test url using a sliding window of 400 seconds, and
1164   // max of 1 request.  Also simulate a request having just started, so the
1165   // next request will be affected by backoff of ~400 seconds.
1166   context_getter->AddThrottlerEntry(
1167       url, std::string() /* url_id */, 400000 /* sliding_window_period_ms */,
1168       1 /* max_send_threshold */, 200000 /* initial_backoff_ms */,
1169       2.0 /* multiply_factor */, 0.0 /* jitter_factor */,
1170       400000 /* maximum_backoff_ms */,
1171       true /* reserve_sending_time_for_next_request*/);
1172
1173   // The getter won't be destroyed if the test holds on to a reference to it.
1174   context_getter = nullptr;
1175
1176   delegate.fetcher()->Start();
1177   delegate.CancelFetch();
1178   run_loop_.Run();
1179
1180   EXPECT_FALSE(delegate.did_complete());
1181 }
1182
1183 // A URLFetcherDelegate that expects to receive a response body of "request1"
1184 // and then reuses the fetcher for the same URL, setting the "test" request
1185 // header to "request2".
1186 class ReuseFetcherDelegate : public WaitingURLFetcherDelegate {
1187  public:
1188   // |second_request_context_getter| is the context getter used for the second
1189   // request. Can't reuse the old one because fetchers release it on completion.
1190   ReuseFetcherDelegate(
1191       scoped_refptr<URLRequestContextGetter> second_request_context_getter)
1192       : first_request_complete_(false),
1193         second_request_context_getter_(second_request_context_getter) {}
1194
1195   ~ReuseFetcherDelegate() override {}
1196
1197   void OnURLFetchComplete(const URLFetcher* source) override {
1198     EXPECT_EQ(fetcher(), source);
1199     if (!first_request_complete_) {
1200       first_request_complete_ = true;
1201       EXPECT_TRUE(fetcher()->GetStatus().is_success());
1202       EXPECT_EQ(200, fetcher()->GetResponseCode());
1203       std::string data;
1204       ASSERT_TRUE(fetcher()->GetResponseAsString(&data));
1205       EXPECT_EQ("request1", data);
1206
1207       fetcher()->SetRequestContext(second_request_context_getter_.get());
1208       fetcher()->SetExtraRequestHeaders("test: request2");
1209       fetcher()->Start();
1210       return;
1211     }
1212     WaitingURLFetcherDelegate::OnURLFetchComplete(source);
1213   }
1214
1215  private:
1216   bool first_request_complete_;
1217   scoped_refptr<URLRequestContextGetter> second_request_context_getter_;
1218
1219   DISALLOW_COPY_AND_ASSIGN(ReuseFetcherDelegate);
1220 };
1221
1222 TEST_F(URLFetcherTest, ReuseFetcherForSameURL) {
1223   // TODO(mmenke):  It's really weird that this is supported, particularly
1224   // some fields can be modified between requests, but some (Like upload body)
1225   // cannot be. Can we get rid of support for this?
1226   scoped_refptr<URLRequestContextGetter> context_getter(
1227       CreateSameThreadContextGetter());
1228   ReuseFetcherDelegate delegate(context_getter);
1229   delegate.CreateFetcher(test_server_->GetURL("echoheader?test"),
1230                          URLFetcher::GET, context_getter);
1231   delegate.fetcher()->SetExtraRequestHeaders("test: request1");
1232   delegate.StartFetcherAndWait();
1233
1234   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
1235   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
1236   std::string data;
1237   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
1238   EXPECT_EQ("request2", data);
1239 }
1240
1241 TEST_F(URLFetcherTest, ShutdownSameThread) {
1242   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
1243       CreateSameThreadContextGetter());
1244
1245   // Create a fetcher and wait for it to create a request.
1246   WaitingURLFetcherDelegate delegate1;
1247   delegate1.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
1248   delegate1.fetcher()->Start();
1249   // Need to spin the loop to ensure the URLRequest is created and started.
1250   base::RunLoop().RunUntilIdle();
1251
1252   // Create and start another fetcher, but don't wait for it to start. The task
1253   // to start the request should be in the message loop.
1254   WaitingURLFetcherDelegate delegate2;
1255   delegate2.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
1256   delegate2.fetcher()->Start();
1257
1258   // Check that shutting down the getter cancels the request synchronously,
1259   // allowing the context to be destroyed.
1260   context_getter->Shutdown();
1261
1262   // Wait for the first fetcher, make sure it failed.
1263   delegate1.WaitForComplete();
1264   EXPECT_FALSE(delegate1.fetcher()->GetStatus().is_success());
1265   EXPECT_EQ(ERR_CONTEXT_SHUT_DOWN, delegate1.fetcher()->GetStatus().error());
1266
1267   // Wait for the second fetcher, make sure it failed.
1268   delegate2.WaitForComplete();
1269   EXPECT_FALSE(delegate2.fetcher()->GetStatus().is_success());
1270   EXPECT_EQ(ERR_CONTEXT_SHUT_DOWN, delegate2.fetcher()->GetStatus().error());
1271
1272   // New fetchers should automatically fail without making new requests. This
1273   // should follow the same path as the second fetcher, but best to be safe.
1274   WaitingURLFetcherDelegate delegate3;
1275   delegate3.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
1276   delegate3.fetcher()->Start();
1277   delegate3.WaitForComplete();
1278   EXPECT_FALSE(delegate3.fetcher()->GetStatus().is_success());
1279   EXPECT_EQ(ERR_CONTEXT_SHUT_DOWN, delegate3.fetcher()->GetStatus().error());
1280 }
1281
1282 TEST_F(URLFetcherTest, ShutdownCrossThread) {
1283   scoped_refptr<FetcherTestURLRequestContextGetter> context_getter(
1284       CreateCrossThreadContextGetter());
1285
1286   WaitingURLFetcherDelegate delegate1;
1287   delegate1.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
1288   delegate1.fetcher()->Start();
1289   // Check that shutting the context getter lets the context be destroyed safely
1290   // and cancels the request.
1291   context_getter->Shutdown();
1292   delegate1.WaitForComplete();
1293   EXPECT_FALSE(delegate1.fetcher()->GetStatus().is_success());
1294   EXPECT_EQ(ERR_CONTEXT_SHUT_DOWN, delegate1.fetcher()->GetStatus().error());
1295
1296   // New requests should automatically fail without making new requests.
1297   WaitingURLFetcherDelegate delegate2;
1298   delegate2.CreateFetcher(hanging_url(), URLFetcher::GET, context_getter);
1299   delegate2.StartFetcherAndWait();
1300   EXPECT_FALSE(delegate2.fetcher()->GetStatus().is_success());
1301   EXPECT_EQ(ERR_CONTEXT_SHUT_DOWN, delegate2.fetcher()->GetStatus().error());
1302 }
1303
1304 // Get a small file.
1305 TEST_F(URLFetcherTest, FileTestSmallGet) {
1306   const char kFileToFetch[] = "simple.html";
1307
1308   base::ScopedTempDir temp_dir;
1309   ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
1310   base::FilePath out_path = temp_dir.path().AppendASCII(kFileToFetch);
1311   SaveFileTest(kFileToFetch, false, out_path, false);
1312 }
1313
1314 // Get a file large enough to require more than one read into URLFetcher::Core's
1315 // IOBuffer.
1316 TEST_F(URLFetcherTest, FileTestLargeGet) {
1317   const char kFileToFetch[] = "animate1.gif";
1318
1319   base::ScopedTempDir temp_dir;
1320   ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
1321   base::FilePath out_path = temp_dir.path().AppendASCII(kFileToFetch);
1322   SaveFileTest(kFileToFetch, false, out_path, false);
1323 }
1324
1325 // If the caller takes the ownership of the output file, the file should persist
1326 // even after URLFetcher is gone.
1327 TEST_F(URLFetcherTest, FileTestTakeOwnership) {
1328   const char kFileToFetch[] = "simple.html";
1329
1330   base::ScopedTempDir temp_dir;
1331   ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
1332   base::FilePath out_path = temp_dir.path().AppendASCII(kFileToFetch);
1333   SaveFileTest(kFileToFetch, false, out_path, true);
1334 }
1335
1336 // Test that an existing file can be overwritten be a fetcher.
1337 TEST_F(URLFetcherTest, FileTestOverwriteExisting) {
1338   base::ScopedTempDir temp_dir;
1339   ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
1340
1341   // Create a file before trying to fetch.
1342   const char kFileToFetch[] = "simple.html";
1343   std::string data(10000, '?');  // Meant to be larger than simple.html.
1344   base::FilePath out_path = temp_dir.path().AppendASCII(kFileToFetch);
1345   ASSERT_EQ(static_cast<int>(data.size()),
1346             base::WriteFile(out_path, data.data(), data.size()));
1347   ASSERT_TRUE(base::PathExists(out_path));
1348
1349   SaveFileTest(kFileToFetch, false, out_path, true);
1350 }
1351
1352 // Test trying to overwrite a directory with a file when using a fetcher fails.
1353 TEST_F(URLFetcherTest, FileTestTryToOverwriteDirectory) {
1354   base::ScopedTempDir temp_dir;
1355   ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
1356
1357   // Create a directory before trying to fetch.
1358   static const char kFileToFetch[] = "simple.html";
1359   base::FilePath out_path = temp_dir.path().AppendASCII(kFileToFetch);
1360   ASSERT_TRUE(base::CreateDirectory(out_path));
1361   ASSERT_TRUE(base::PathExists(out_path));
1362
1363   WaitingURLFetcherDelegate delegate;
1364   delegate.CreateFetcher(
1365       test_server_->GetURL(std::string(kTestServerFilePrefix) + kFileToFetch),
1366       URLFetcher::GET, CreateSameThreadContextGetter());
1367   delegate.fetcher()->SaveResponseToFileAtPath(
1368       out_path, scoped_refptr<base::SingleThreadTaskRunner>(
1369                     base::ThreadTaskRunnerHandle::Get()));
1370   delegate.StartFetcherAndWait();
1371
1372   EXPECT_FALSE(delegate.fetcher()->GetStatus().is_success());
1373   EXPECT_EQ(ERR_ACCESS_DENIED, delegate.fetcher()->GetStatus().error());
1374 }
1375
1376 // Get a small file and save it to a temp file.
1377 TEST_F(URLFetcherTest, TempFileTestSmallGet) {
1378   SaveFileTest("simple.html", true, base::FilePath(), false);
1379 }
1380
1381 // Get a file large enough to require more than one read into URLFetcher::Core's
1382 // IOBuffer and save it to a temp file.
1383 TEST_F(URLFetcherTest, TempFileTestLargeGet) {
1384   SaveFileTest("animate1.gif", true, base::FilePath(), false);
1385 }
1386
1387 // If the caller takes the ownership of the temp file, check that the file
1388 // persists even after URLFetcher is gone.
1389 TEST_F(URLFetcherTest, TempFileTestTakeOwnership) {
1390   SaveFileTest("simple.html", true, base::FilePath(), true);
1391 }
1392
1393 TEST_F(URLFetcherBadHTTPSTest, BadHTTPS) {
1394   WaitingURLFetcherDelegate delegate;
1395   delegate.CreateFetcher(test_server_->GetURL(kDefaultResponsePath),
1396                          URLFetcher::GET, CreateSameThreadContextGetter());
1397   delegate.StartFetcherAndWait();
1398
1399   EXPECT_EQ(URLRequestStatus::CANCELED,
1400             delegate.fetcher()->GetStatus().status());
1401   EXPECT_EQ(ERR_ABORTED, delegate.fetcher()->GetStatus().error());
1402   EXPECT_EQ(-1, delegate.fetcher()->GetResponseCode());
1403   EXPECT_TRUE(delegate.fetcher()->GetCookies().empty());
1404   std::string data;
1405   EXPECT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
1406   EXPECT_TRUE(data.empty());
1407 }
1408
1409 }  // namespace
1410
1411 }  // namespace net