net/url_request/url_fetcher_impl_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_fetcher_impl.h"
   6
   7 #include <algorithm>
   8 #include <string>
   9
  10 #include "base/bind.h"
  11 #include "base/files/file_util.h"
  12 #include "base/files/scoped_temp_dir.h"
  13 #include "base/macros.h"
  14 #include "base/message_loop/message_loop_proxy.h"
  15 #include "base/path_service.h"
  16 #include "base/run_loop.h"
  17 #include "base/strings/stringprintf.h"
  18 #include "base/synchronization/waitable_event.h"
  19 #include "base/threading/thread.h"
  20 #include "build/build_config.h"
  21 #include "crypto/nss_util.h"
  22 #include "net/base/elements_upload_data_stream.h"
  23 #include "net/base/network_change_notifier.h"
  24 #include "net/base/upload_bytes_element_reader.h"
  25 #include "net/base/upload_element_reader.h"
  26 #include "net/base/upload_file_element_reader.h"
  27 #include "net/dns/mock_host_resolver.h"
  28 #include "net/http/http_response_headers.h"
  29 #include "net/test/spawned_test_server/spawned_test_server.h"
  30 #include "net/url_request/url_fetcher_delegate.h"
  31 #include "net/url_request/url_request_context_getter.h"
  32 #include "net/url_request/url_request_test_util.h"
  33 #include "net/url_request/url_request_throttler_manager.h"
  34 #include "testing/gtest/include/gtest/gtest.h"
  35
  36 #if defined(USE_NSS_CERTS) || defined(OS_IOS)
  37 #include "net/cert_net/nss_ocsp.h"
  38 #endif
  39
  40 namespace net {
  41
  42 using base::Time;
  43 using base::TimeDelta;
  44
  45 // TODO(eroman): Add a regression test for http://crbug.com/40505.
  46
  47 namespace {
  48
  49 // TODO(akalin): Move all the test data to somewhere under net/.
  50 const base::FilePath::CharType kDocRoot[] =
  51     FILE_PATH_LITERAL("net/data/url_fetcher_impl_unittest");
  52 const char kTestServerFilePrefix[] = "files/";
  53
  54 // Test server path and response body for the default URL used by many of the
  55 // tests.
  56 const char kDefaultResponsePath[] = "defaultresponse";
  57 const char kDefaultResponseBody[] =
  58     "Default response given for path: /defaultresponse";
  59
  60 // Request body for streams created by CreateUploadStream.
  61 const char kCreateUploadStreamBody[] = "rosebud";
  62
  63 base::FilePath GetUploadFileTestPath() {
  64   base::FilePath path;
  65   PathService::Get(base::DIR_SOURCE_ROOT, &path);
  66   return path.Append(
  67       FILE_PATH_LITERAL("net/data/url_request_unittest/BullRunSpeech.txt"));
  68 }
  69
  70 // Simple URLRequestDelegate that waits for the specified fetcher to complete.
  71 // Can only be used once.
  72 class WaitingURLFetcherDelegate : public URLFetcherDelegate {
  73  public:
  74   WaitingURLFetcherDelegate() : did_complete_(false) {}
  75
  76   // Creates a URLFetcher that runs network tasks on the current message loop.
  77   void CreateFetcherWithContext(const GURL& url,
  78                                 URLFetcher::RequestType request_type,
  79                                 net::URLRequestContext* context) {
  80     CreateFetcherWithContextGetter(
  81         url, request_type, new TrivialURLRequestContextGetter(
  82                                context, base::MessageLoopProxy::current()));
  83   }
  84
  85   void CreateFetcherWithContextGetter(
  86       const GURL& url,
  87       URLFetcher::RequestType request_type,
  88       net::URLRequestContextGetter* context_getter) {
  89     fetcher_.reset(new URLFetcherImpl(url, request_type, this));
  90     fetcher_->SetRequestContext(context_getter);
  91   }
  92
  93   URLFetcher* fetcher() const { return fetcher_.get(); }
  94
  95   // Wait until the request has completed or been canceled.
  96   void StartFetcherAndWait() {
  97     fetcher_->Start();
  98     WaitForComplete();
  99   }
 100
 101   // Wait until the request has completed or been canceled. Does not start the
 102   // request.
 103   void WaitForComplete() { run_loop_.Run(); }
 104
 105   // Cancels the fetch by deleting the fetcher.
 106   void CancelFetch() {
 107     EXPECT_TRUE(fetcher_);
 108     fetcher_.reset();
 109     run_loop_.Quit();
 110   }
 111
 112   // URLFetcherDelegate:
 113   void OnURLFetchComplete(const URLFetcher* source) override {
 114     EXPECT_FALSE(did_complete_);
 115     EXPECT_TRUE(fetcher_);
 116     EXPECT_EQ(fetcher_.get(), source);
 117     did_complete_ = true;
 118     run_loop_.Quit();
 119   }
 120
 121   void OnURLFetchDownloadProgress(const URLFetcher* source,
 122                                   int64 current,
 123                                   int64 total) override {
 124     // Note that the current progress may be greater than the previous progress,
 125     // in the case of retrying the request.
 126     EXPECT_FALSE(did_complete_);
 127     EXPECT_TRUE(fetcher_);
 128     EXPECT_EQ(source, fetcher_.get());
 129
 130     EXPECT_LE(0, current);
 131     // If file size is not known, |total| is -1.
 132     if (total >= 0)
 133       EXPECT_LE(current, total);
 134   }
 135
 136   void OnURLFetchUploadProgress(const URLFetcher* source,
 137                                 int64 current,
 138                                 int64 total) override {
 139     // Note that the current progress may be greater than the previous progress,
 140     // in the case of retrying the request.
 141     EXPECT_FALSE(did_complete_);
 142     EXPECT_TRUE(fetcher_);
 143     EXPECT_EQ(source, fetcher_.get());
 144
 145     EXPECT_LE(0, current);
 146     // If file size is not known, |total| is -1.
 147     if (total >= 0)
 148       EXPECT_LE(current, total);
 149   }
 150
 151   bool did_complete() const { return did_complete_; }
 152
 153  private:
 154   bool did_complete_;
 155
 156   scoped_ptr<URLFetcherImpl> fetcher_;
 157   base::RunLoop run_loop_;
 158
 159   DISALLOW_COPY_AND_ASSIGN(WaitingURLFetcherDelegate);
 160 };
 161
 162 class ThrottlingTestURLRequestContext : public TestURLRequestContext {
 163  public:
 164   ThrottlingTestURLRequestContext() : TestURLRequestContext(true) {
 165     set_throttler_manager(&throttler_manager_);
 166     Init();
 167     DCHECK(throttler_manager() != nullptr);
 168   }
 169
 170  private:
 171   URLRequestThrottlerManager throttler_manager_;
 172 };
 173
 174 class ThrottlingTestURLRequestContextGetter
 175     : public TestURLRequestContextGetter {
 176  public:
 177   ThrottlingTestURLRequestContextGetter(
 178       base::MessageLoopProxy* io_message_loop_proxy,
 179       TestURLRequestContext* request_context)
 180       : TestURLRequestContextGetter(io_message_loop_proxy),
 181         context_(request_context) {
 182   }
 183
 184   // TestURLRequestContextGetter:
 185   TestURLRequestContext* GetURLRequestContext() override { return context_; }
 186
 187  protected:
 188   ~ThrottlingTestURLRequestContextGetter() override {}
 189
 190   TestURLRequestContext* const context_;
 191 };
 192
 193 }  // namespace
 194
 195 class URLFetcherTest : public testing::Test,
 196                        public URLFetcherDelegate {
 197  public:
 198   URLFetcherTest()
 199       : io_message_loop_proxy_(base::MessageLoopProxy::current()),
 200         num_upload_streams_created_(0),
 201         fetcher_(nullptr),
 202         expected_status_code_(200) {}
 203
 204   static int GetNumFetcherCores() {
 205     return URLFetcherImpl::GetNumFetcherCores();
 206   }
 207
 208   // Creates a URLFetcher, using the program's main thread to do IO.
 209   virtual void CreateFetcher(const GURL& url);
 210
 211   // URLFetcherDelegate:
 212   // Subclasses that override this should either call this function or
 213   // CleanupAfterFetchComplete() at the end of their processing, depending on
 214   // whether they want to check for a non-empty HTTP 200 response or not.
 215   void OnURLFetchComplete(const URLFetcher* source) override;
 216
 217   // Deletes |fetcher| and terminates the message loop.
 218   void CleanupAfterFetchComplete();
 219
 220   scoped_refptr<base::MessageLoopProxy> io_message_loop_proxy() {
 221     return io_message_loop_proxy_;
 222   }
 223
 224   TestURLRequestContext* request_context() {
 225     return context_.get();
 226   }
 227
 228   // Callback passed to URLFetcher to create upload stream by some tests.
 229   scoped_ptr<UploadDataStream> CreateUploadStream() {
 230     ++num_upload_streams_created_;
 231     std::vector<char> buffer(
 232         kCreateUploadStreamBody,
 233         kCreateUploadStreamBody + strlen(kCreateUploadStreamBody));
 234     return ElementsUploadDataStream::CreateWithReader(
 235         scoped_ptr<UploadElementReader>(
 236             new UploadOwnedBytesElementReader(&buffer)),
 237         0);
 238   }
 239
 240   // Number of streams created by CreateUploadStream.
 241   size_t num_upload_streams_created() const {
 242     return num_upload_streams_created_;
 243   }
 244
 245   // Downloads |file_to_fetch| and checks the contents when done.  If
 246   // |save_to_temporary_file| is true, saves it to a temporary file, and
 247   // |requested_out_path| is ignored. Otherwise, saves it to
 248   // |requested_out_path|. Takes ownership of the file if |take_ownership| is
 249   // true. Deletes file when done.
 250   void SaveFileTest(const char* file_to_fetch,
 251                     bool save_to_temporary_file,
 252                     const base::FilePath& requested_out_path,
 253                     bool take_ownership) {
 254     scoped_ptr<WaitingURLFetcherDelegate> delegate(
 255         new WaitingURLFetcherDelegate());
 256     delegate->CreateFetcherWithContext(
 257         test_server_->GetURL(std::string(kTestServerFilePrefix) +
 258                              file_to_fetch),
 259         URLFetcher::GET, request_context());
 260     if (save_to_temporary_file) {
 261       delegate->fetcher()->SaveResponseToTemporaryFile(
 262           scoped_refptr<base::MessageLoopProxy>(
 263               base::MessageLoopProxy::current()));
 264     } else {
 265       delegate->fetcher()->SaveResponseToFileAtPath(
 266           requested_out_path, scoped_refptr<base::MessageLoopProxy>(
 267                                   base::MessageLoopProxy::current()));
 268     }
 269     delegate->StartFetcherAndWait();
 270
 271     EXPECT_TRUE(delegate->fetcher()->GetStatus().is_success());
 272     EXPECT_EQ(200, delegate->fetcher()->GetResponseCode());
 273
 274     base::FilePath out_path;
 275     EXPECT_TRUE(
 276         delegate->fetcher()->GetResponseAsFilePath(take_ownership, &out_path));
 277     if (!save_to_temporary_file) {
 278       EXPECT_EQ(requested_out_path, out_path);
 279     }
 280
 281     EXPECT_TRUE(base::ContentsEqual(
 282         test_server_->GetDocumentRoot().AppendASCII(file_to_fetch), out_path));
 283
 284     // Delete the delegate and run the message loop to give the fetcher's
 285     // destructor a chance to delete the file.
 286     delegate.reset();
 287     base::RunLoop().RunUntilIdle();
 288
 289     // File should only exist if |take_ownership| was true.
 290     EXPECT_EQ(take_ownership, base::PathExists(out_path));
 291
 292     // Cleanup.
 293     if (base::PathExists(out_path))
 294       base::DeleteFile(out_path, false);
 295   }
 296
 297   // Returns a URL that hangs on DNS resolution.  Only hangs when using the
 298   // request context returned by request_context().
 299   const GURL& hanging_url() const { return hanging_url_; }
 300
 301   MockHostResolver* resolver() { return &resolver_; }
 302
 303   // testing::Test:
 304   void SetUp() override {
 305     SetUpServer();
 306     ASSERT_TRUE(test_server_->Start());
 307
 308     // Set up host resolver so requests for |hanging_url_| block on an async DNS
 309     // resolver.  Calling resolver()->ResolveAllPending() will resume the hung
 310     // requests.
 311     resolver_.set_ondemand_mode(true);
 312     resolver_.rules()->AddRule("example.com", "127.0.0.1");
 313     hanging_url_ =
 314         GURL(base::StringPrintf("http://example.com:%d/defaultresponse",
 315                                 test_server_->host_port_pair().port()));
 316     ASSERT_TRUE(hanging_url_.is_valid());
 317
 318     context_.reset(new TestURLRequestContext(true));
 319     context_->set_host_resolver(&resolver_);
 320     context_->set_throttler_manager(&throttler_manager_);
 321     context_->Init();
 322
 323 #if defined(USE_NSS_CERTS) || defined(OS_IOS)
 324     crypto::EnsureNSSInit();
 325     EnsureNSSHttpIOInit();
 326 #endif
 327   }
 328
 329   void TearDown() override {
 330 #if defined(USE_NSS_CERTS) || defined(OS_IOS)
 331     ShutdownNSSHttpIO();
 332 #endif
 333   }
 334
 335   // Initializes |test_server_| without starting it.  Allows subclasses to use
 336   // their own server configuration.
 337   virtual void SetUpServer() {
 338     test_server_.reset(new SpawnedTestServer(SpawnedTestServer::TYPE_HTTP,
 339                                              SpawnedTestServer::kLocalhost,
 340                                              base::FilePath(kDocRoot)));
 341   }
 342
 343   // URLFetcher is designed to run on the main UI thread, but in our tests
 344   // we assume that the current thread is the IO thread where the URLFetcher
 345   // dispatches its requests to.  When we wish to simulate being used from
 346   // a UI thread, we dispatch a worker thread to do so.
 347   scoped_refptr<base::MessageLoopProxy> io_message_loop_proxy_;
 348
 349   scoped_ptr<SpawnedTestServer> test_server_;
 350   GURL hanging_url_;
 351
 352   size_t num_upload_streams_created_;
 353
 354   URLFetcherImpl* fetcher_;
 355
 356   MockHostResolver resolver_;
 357   URLRequestThrottlerManager throttler_manager_;
 358   scoped_ptr<TestURLRequestContext> context_;
 359
 360   int expected_status_code_;
 361 };
 362
 363 void URLFetcherTest::CreateFetcher(const GURL& url) {
 364   fetcher_ = new URLFetcherImpl(url, URLFetcher::GET, this);
 365   fetcher_->SetRequestContext(new ThrottlingTestURLRequestContextGetter(
 366       io_message_loop_proxy().get(), request_context()));
 367   fetcher_->Start();
 368 }
 369
 370 void URLFetcherTest::OnURLFetchComplete(const URLFetcher* source) {
 371   EXPECT_TRUE(source->GetStatus().is_success());
 372   EXPECT_EQ(expected_status_code_, source->GetResponseCode());  // HTTP OK
 373
 374   std::string data;
 375   EXPECT_TRUE(source->GetResponseAsString(&data));
 376   EXPECT_FALSE(data.empty());
 377
 378   CleanupAfterFetchComplete();
 379 }
 380
 381 void URLFetcherTest::CleanupAfterFetchComplete() {
 382   delete fetcher_;  // Have to delete this here and not in the destructor,
 383                     // because the destructor won't necessarily run on the
 384                     // same thread that CreateFetcher() did.
 385
 386   io_message_loop_proxy()->PostTask(FROM_HERE,
 387                                     base::MessageLoop::QuitClosure());
 388   // If the current message loop is not the IO loop, it will be shut down when
 389   // the main loop returns and this thread subsequently goes out of scope.
 390 }
 391
 392 namespace {
 393
 394 // Version of URLFetcherTest that tests request cancellation on shutdown.
 395 class URLFetcherCancelTest : public URLFetcherTest {
 396  public:
 397   // URLFetcherTest:
 398   void CreateFetcher(const GURL& url) override;
 399
 400   // URLFetcherDelegate:
 401   void OnURLFetchComplete(const URLFetcher* source) override;
 402
 403   void CancelRequest();
 404 };
 405
 406 // Version of TestURLRequestContext that posts a Quit task to the IO
 407 // thread once it is deleted.
 408 class CancelTestURLRequestContext : public ThrottlingTestURLRequestContext {
 409  public:
 410   explicit CancelTestURLRequestContext() {
 411   }
 412
 413  private:
 414   ~CancelTestURLRequestContext() override {
 415     // The d'tor should execute in the IO thread. Post the quit task to the
 416     // current thread.
 417     base::MessageLoop::current()->PostTask(FROM_HERE,
 418                                            base::MessageLoop::QuitClosure());
 419   }
 420 };
 421
 422 class CancelTestURLRequestContextGetter
 423     : public TestURLRequestContextGetter {
 424  public:
 425   CancelTestURLRequestContextGetter(
 426       base::MessageLoopProxy* io_message_loop_proxy,
 427       const GURL& throttle_for_url)
 428       : TestURLRequestContextGetter(io_message_loop_proxy),
 429         io_message_loop_proxy_(io_message_loop_proxy),
 430         context_created_(false, false),
 431         throttle_for_url_(throttle_for_url) {
 432   }
 433
 434   // TestURLRequestContextGetter:
 435   TestURLRequestContext* GetURLRequestContext() override {
 436     if (!context_.get()) {
 437       context_.reset(new CancelTestURLRequestContext());
 438       DCHECK(context_->throttler_manager());
 439
 440       // Registers an entry for test url. The backoff time is calculated by:
 441       //     new_backoff = 2.0 * old_backoff + 0
 442       // The initial backoff is 2 seconds and maximum backoff is 4 seconds.
 443       // Maximum retries allowed is set to 2.
 444       scoped_refptr<URLRequestThrottlerEntry> entry(
 445           new URLRequestThrottlerEntry(context_->throttler_manager(),
 446                                        std::string(),
 447                                        200,
 448                                        3,
 449                                        2000,
 450                                        2.0,
 451                                        0.0,
 452                                        4000));
 453       context_->throttler_manager()
 454           ->OverrideEntryForTests(throttle_for_url_, entry.get());
 455
 456       context_created_.Signal();
 457     }
 458     return context_.get();
 459   }
 460
 461   virtual scoped_refptr<base::MessageLoopProxy> GetIOMessageLoopProxy() const {
 462     return io_message_loop_proxy_;
 463   }
 464
 465   void WaitForContextCreation() {
 466     context_created_.Wait();
 467   }
 468
 469  protected:
 470   ~CancelTestURLRequestContextGetter() override {}
 471
 472  private:
 473   scoped_ptr<TestURLRequestContext> context_;
 474   scoped_refptr<base::MessageLoopProxy> io_message_loop_proxy_;
 475   base::WaitableEvent context_created_;
 476   GURL throttle_for_url_;
 477 };
 478
 479 // Version of URLFetcherTest that tests bad HTTPS requests.
 480 class URLFetcherBadHTTPSTest : public URLFetcherTest {
 481  public:
 482   URLFetcherBadHTTPSTest() {}
 483
 484   // URLFetcherTest:
 485   void SetUpServer() override {
 486     SpawnedTestServer::SSLOptions ssl_options(
 487         SpawnedTestServer::SSLOptions::CERT_EXPIRED);
 488     test_server_.reset(new SpawnedTestServer(
 489         SpawnedTestServer::TYPE_HTTPS, ssl_options, base::FilePath(kDocRoot)));
 490   }
 491 };
 492
 493 void URLFetcherCancelTest::CreateFetcher(const GURL& url) {
 494   fetcher_ = new URLFetcherImpl(url, URLFetcher::GET, this);
 495   CancelTestURLRequestContextGetter* context_getter =
 496       new CancelTestURLRequestContextGetter(io_message_loop_proxy().get(), url);
 497   fetcher_->SetRequestContext(context_getter);
 498   fetcher_->SetMaxRetriesOn5xx(2);
 499   fetcher_->Start();
 500   // We need to wait for the creation of the URLRequestContext, since we
 501   // rely on it being destroyed as a signal to end the test.
 502   context_getter->WaitForContextCreation();
 503   CancelRequest();
 504 }
 505
 506 void URLFetcherCancelTest::OnURLFetchComplete(
 507     const URLFetcher* source) {
 508   // We should have cancelled the request before completion.
 509   ADD_FAILURE();
 510   CleanupAfterFetchComplete();
 511 }
 512
 513 void URLFetcherCancelTest::CancelRequest() {
 514   delete fetcher_;
 515   // The URLFetcher's test context will post a Quit task once it is
 516   // deleted. So if this test simply hangs, it means cancellation
 517   // did not work.
 518 }
 519
 520 // Create the fetcher on the main thread.  Since network IO will happen on the
 521 // main thread, this will test URLFetcher's ability to do everything on one
 522 // thread.
 523 TEST_F(URLFetcherTest, SameThreadTest) {
 524   WaitingURLFetcherDelegate delegate;
 525   delegate.CreateFetcherWithContext(test_server_->GetURL(kDefaultResponsePath),
 526                                     URLFetcher::GET, request_context());
 527   delegate.StartFetcherAndWait();
 528
 529   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 530   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 531   std::string data;
 532   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 533   EXPECT_EQ(kDefaultResponseBody, data);
 534 }
 535
 536 // Create a separate thread that will create the URLFetcher.  A separate thread
 537 // acts as the network thread.
 538 TEST_F(URLFetcherTest, DifferentThreadsTest) {
 539   base::Thread network_thread("network thread");
 540   base::Thread::Options network_thread_options;
 541   network_thread_options.message_loop_type = base::MessageLoop::TYPE_IO;
 542   ASSERT_TRUE(network_thread.StartWithOptions(network_thread_options));
 543
 544   scoped_refptr<TestURLRequestContextGetter> context_getter(
 545       new TestURLRequestContextGetter(network_thread.task_runner()));
 546   WaitingURLFetcherDelegate delegate;
 547   delegate.CreateFetcherWithContextGetter(
 548       test_server_->GetURL(kDefaultResponsePath), URLFetcher::GET,
 549       new TestURLRequestContextGetter(network_thread.task_runner()));
 550   delegate.StartFetcherAndWait();
 551
 552   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 553   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 554   std::string data;
 555   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 556   EXPECT_EQ(kDefaultResponseBody, data);
 557 }
 558
 559 // Tests to make sure CancelAll() will successfully cancel existing URLFetchers.
 560 TEST_F(URLFetcherTest, CancelAll) {
 561   EXPECT_EQ(0, GetNumFetcherCores());
 562   WaitingURLFetcherDelegate delegate;
 563   delegate.CreateFetcherWithContext(hanging_url(), URLFetcher::GET,
 564                                     request_context());
 565   delegate.fetcher()->Start();
 566   // Wait for the request to reach the mock resolver and hang, to ensure the
 567   // request has actually started.
 568   base::RunLoop().RunUntilIdle();
 569   EXPECT_TRUE(resolver_.has_pending_requests());
 570
 571   EXPECT_EQ(1, URLFetcherTest::GetNumFetcherCores());
 572   URLFetcherImpl::CancelAll();
 573   EXPECT_EQ(0, URLFetcherTest::GetNumFetcherCores());
 574 }
 575
 576 TEST_F(URLFetcherTest, DontRetryOnNetworkChangedByDefault) {
 577   EXPECT_EQ(0, GetNumFetcherCores());
 578   WaitingURLFetcherDelegate delegate;
 579   delegate.CreateFetcherWithContext(hanging_url(), URLFetcher::GET,
 580                                     request_context());
 581   EXPECT_FALSE(resolver_.has_pending_requests());
 582
 583   // This posts a task to start the fetcher.
 584   delegate.fetcher()->Start();
 585   base::RunLoop().RunUntilIdle();
 586
 587   // The fetcher is now running, but is pending the host resolve.
 588   EXPECT_EQ(1, GetNumFetcherCores());
 589   EXPECT_TRUE(resolver_.has_pending_requests());
 590   ASSERT_FALSE(delegate.did_complete());
 591
 592   // A network change notification aborts the connect job.
 593   NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
 594   delegate.WaitForComplete();
 595   EXPECT_FALSE(resolver_.has_pending_requests());
 596
 597   // And the owner of the fetcher gets the ERR_NETWORK_CHANGED error.
 598   EXPECT_EQ(hanging_url(), delegate.fetcher()->GetOriginalURL());
 599   ASSERT_FALSE(delegate.fetcher()->GetStatus().is_success());
 600   EXPECT_EQ(ERR_NETWORK_CHANGED, delegate.fetcher()->GetStatus().error());
 601 }
 602
 603 TEST_F(URLFetcherTest, RetryOnNetworkChangedAndFail) {
 604   EXPECT_EQ(0, GetNumFetcherCores());
 605   WaitingURLFetcherDelegate delegate;
 606   delegate.CreateFetcherWithContext(hanging_url(), URLFetcher::GET,
 607                                     request_context());
 608   delegate.fetcher()->SetAutomaticallyRetryOnNetworkChanges(3);
 609   EXPECT_FALSE(resolver_.has_pending_requests());
 610
 611   // This posts a task to start the fetcher.
 612   delegate.fetcher()->Start();
 613   base::RunLoop().RunUntilIdle();
 614
 615   // The fetcher is now running, but is pending the host resolve.
 616   EXPECT_EQ(1, GetNumFetcherCores());
 617   EXPECT_TRUE(resolver_.has_pending_requests());
 618   ASSERT_FALSE(delegate.did_complete());
 619
 620   // Make it fail 3 times.
 621   for (int i = 0; i < 3; ++i) {
 622     // A network change notification aborts the connect job.
 623     NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
 624     base::RunLoop().RunUntilIdle();
 625
 626     // But the fetcher retries automatically.
 627     EXPECT_EQ(1, GetNumFetcherCores());
 628     EXPECT_TRUE(resolver_.has_pending_requests());
 629     ASSERT_FALSE(delegate.did_complete());
 630   }
 631
 632   // A 4th failure doesn't trigger another retry, and propagates the error
 633   // to the owner of the fetcher.
 634   NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
 635   delegate.WaitForComplete();
 636   EXPECT_FALSE(resolver_.has_pending_requests());
 637
 638   // And the owner of the fetcher gets the ERR_NETWORK_CHANGED error.
 639   EXPECT_EQ(hanging_url(), delegate.fetcher()->GetOriginalURL());
 640   ASSERT_FALSE(delegate.fetcher()->GetStatus().is_success());
 641   EXPECT_EQ(ERR_NETWORK_CHANGED, delegate.fetcher()->GetStatus().error());
 642 }
 643
 644 TEST_F(URLFetcherTest, RetryOnNetworkChangedAndSucceed) {
 645   EXPECT_EQ(0, GetNumFetcherCores());
 646   WaitingURLFetcherDelegate delegate;
 647   delegate.CreateFetcherWithContext(hanging_url(), URLFetcher::GET,
 648                                     request_context());
 649   delegate.fetcher()->SetAutomaticallyRetryOnNetworkChanges(3);
 650   EXPECT_FALSE(resolver_.has_pending_requests());
 651
 652   // This posts a task to start the fetcher.
 653   delegate.fetcher()->Start();
 654   base::RunLoop().RunUntilIdle();
 655
 656   // The fetcher is now running, but is pending the host resolve.
 657   EXPECT_EQ(1, GetNumFetcherCores());
 658   EXPECT_TRUE(resolver_.has_pending_requests());
 659   ASSERT_FALSE(delegate.did_complete());
 660
 661   // Make it fail 3 times.
 662   for (int i = 0; i < 3; ++i) {
 663     // A network change notification aborts the connect job.
 664     NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
 665     base::RunLoop().RunUntilIdle();
 666
 667     // But the fetcher retries automatically.
 668     EXPECT_EQ(1, GetNumFetcherCores());
 669     EXPECT_TRUE(resolver_.has_pending_requests());
 670     ASSERT_FALSE(delegate.did_complete());
 671   }
 672
 673   // Now let it succeed by resolving the pending request.
 674   resolver_.ResolveAllPending();
 675   delegate.WaitForComplete();
 676   EXPECT_FALSE(resolver_.has_pending_requests());
 677
 678   // This time the request succeeded.
 679   EXPECT_EQ(hanging_url(), delegate.fetcher()->GetOriginalURL());
 680   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 681   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 682
 683   std::string data;
 684   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 685   EXPECT_EQ(kDefaultResponseBody, data);
 686 }
 687
 688 TEST_F(URLFetcherTest, PostString) {
 689   const char kUploadData[] = "bobsyeruncle";
 690
 691   WaitingURLFetcherDelegate delegate;
 692   delegate.CreateFetcherWithContext(test_server_->GetURL("echo"),
 693                                     URLFetcher::POST, request_context());
 694   delegate.fetcher()->SetUploadData("application/x-www-form-urlencoded",
 695                                     kUploadData);
 696   delegate.StartFetcherAndWait();
 697
 698   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 699   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 700   std::string data;
 701   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 702   EXPECT_EQ(kUploadData, data);
 703 }
 704
 705 TEST_F(URLFetcherTest, PostEmptyString) {
 706   const char kUploadData[] = "";
 707
 708   WaitingURLFetcherDelegate delegate;
 709   delegate.CreateFetcherWithContext(test_server_->GetURL("echo"),
 710                                     URLFetcher::POST, request_context());
 711   delegate.fetcher()->SetUploadData("application/x-www-form-urlencoded",
 712                                     kUploadData);
 713   delegate.StartFetcherAndWait();
 714
 715   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 716   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 717   std::string data;
 718   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 719   EXPECT_EQ(kUploadData, data);
 720 }
 721
 722 TEST_F(URLFetcherTest, PostEntireFile) {
 723   base::FilePath upload_path = GetUploadFileTestPath();
 724
 725   WaitingURLFetcherDelegate delegate;
 726   delegate.CreateFetcherWithContext(test_server_->GetURL("echo"),
 727                                     URLFetcher::POST, request_context());
 728   delegate.fetcher()->SetUploadFilePath("application/x-www-form-urlencoded",
 729                                         upload_path, 0, kuint64max,
 730                                         base::MessageLoopProxy::current());
 731   delegate.StartFetcherAndWait();
 732
 733   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 734   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 735
 736   std::string expected;
 737   ASSERT_TRUE(base::ReadFileToString(upload_path, &expected));
 738   std::string data;
 739   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 740   EXPECT_EQ(expected, data);
 741 }
 742
 743 TEST_F(URLFetcherTest, PostFileRange) {
 744   const size_t kRangeStart = 30;
 745   const size_t kRangeLength = 100;
 746   base::FilePath upload_path = GetUploadFileTestPath();
 747
 748   WaitingURLFetcherDelegate delegate;
 749   delegate.CreateFetcherWithContext(test_server_->GetURL("echo"),
 750                                     URLFetcher::POST, request_context());
 751   delegate.fetcher()->SetUploadFilePath("application/x-www-form-urlencoded",
 752                                         upload_path, kRangeStart, kRangeLength,
 753                                         base::MessageLoopProxy::current());
 754   delegate.StartFetcherAndWait();
 755
 756   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 757   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 758
 759   std::string expected;
 760   ASSERT_TRUE(base::ReadFileToString(upload_path, &expected));
 761   std::string data;
 762   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 763   EXPECT_EQ(expected.substr(kRangeStart, kRangeLength), data);
 764 }
 765
 766 TEST_F(URLFetcherTest, PostWithUploadStreamFactory) {
 767   WaitingURLFetcherDelegate delegate;
 768   delegate.CreateFetcherWithContext(test_server_->GetURL("echo"),
 769                                     URLFetcher::POST, request_context());
 770   delegate.fetcher()->SetUploadStreamFactory(
 771       "text/plain",
 772       base::Bind(&URLFetcherTest::CreateUploadStream, base::Unretained(this)));
 773   delegate.StartFetcherAndWait();
 774
 775   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 776   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 777   std::string data;
 778   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 779   EXPECT_EQ(kCreateUploadStreamBody, data);
 780   EXPECT_EQ(1u, num_upload_streams_created());
 781 }
 782
 783 TEST_F(URLFetcherTest, PostWithUploadStreamFactoryAndRetries) {
 784   WaitingURLFetcherDelegate delegate;
 785   delegate.CreateFetcherWithContext(test_server_->GetURL("echo?status=500"),
 786                                     URLFetcher::POST, request_context());
 787   delegate.fetcher()->SetAutomaticallyRetryOn5xx(true);
 788   delegate.fetcher()->SetMaxRetriesOn5xx(1);
 789   delegate.fetcher()->SetUploadStreamFactory(
 790       "text/plain",
 791       base::Bind(&URLFetcherTest::CreateUploadStream, base::Unretained(this)));
 792   delegate.StartFetcherAndWait();
 793
 794   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 795   EXPECT_EQ(500, delegate.fetcher()->GetResponseCode());
 796   std::string data;
 797   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 798   EXPECT_EQ(kCreateUploadStreamBody, data);
 799   EXPECT_EQ(2u, num_upload_streams_created());
 800 }
 801
 802 // Checks that upload progress increases over time, never exceeds what's already
 803 // been sent, and adds a chunk whenever all previously appended chunks have
 804 // been uploaded.
 805 class CheckUploadProgressDelegate : public WaitingURLFetcherDelegate {
 806  public:
 807   CheckUploadProgressDelegate()
 808       : chunk_(1 << 16, 'a'), num_chunks_appended_(0), last_seen_progress_(0) {}
 809   ~CheckUploadProgressDelegate() override {}
 810
 811   void OnURLFetchUploadProgress(const URLFetcher* source,
 812                                 int64 current,
 813                                 int64 total) override {
 814     // Run default checks.
 815     WaitingURLFetcherDelegate::OnURLFetchUploadProgress(source, current, total);
 816
 817     EXPECT_LE(last_seen_progress_, current);
 818     EXPECT_LE(current, bytes_appended());
 819     last_seen_progress_ = current;
 820     MaybeAppendChunk();
 821   }
 822
 823   // Append the next chunk if all previously appended chunks have been sent.
 824   void MaybeAppendChunk() {
 825     const int kNumChunks = 5;
 826     if (last_seen_progress_ == bytes_appended() &&
 827         num_chunks_appended_ < kNumChunks) {
 828       ++num_chunks_appended_;
 829       fetcher()->AppendChunkToUpload(chunk_,
 830                                      num_chunks_appended_ == kNumChunks);
 831     }
 832   }
 833
 834  private:
 835   int64 bytes_appended() const { return num_chunks_appended_ * chunk_.size(); }
 836
 837   const std::string chunk_;
 838
 839   int64 num_chunks_appended_;
 840   int64 last_seen_progress_;
 841
 842   DISALLOW_COPY_AND_ASSIGN(CheckUploadProgressDelegate);
 843 };
 844
 845 TEST_F(URLFetcherTest, UploadProgress) {
 846   CheckUploadProgressDelegate delegate;
 847   delegate.CreateFetcherWithContext(test_server_->GetURL("echo"),
 848                                     URLFetcher::POST, request_context());
 849   // Use a chunked upload so that the upload can be paused after uploading data.
 850   // Since upload progress uses a timer, the delegate may not receive any
 851   // notification otherwise.
 852   delegate.fetcher()->SetChunkedUpload("application/x-www-form-urlencoded");
 853
 854   delegate.fetcher()->Start();
 855   // Append the first chunk.  Others will be appended automatically in response
 856   // to OnURLFetchUploadProgress events.
 857   delegate.MaybeAppendChunk();
 858   delegate.WaitForComplete();
 859
 860   // Make sure there are no pending events that cause problems when run.
 861   base::RunLoop().RunUntilIdle();
 862
 863   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 864   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 865   EXPECT_TRUE(delegate.did_complete());
 866 }
 867
 868 // Checks that download progress never decreases, never exceeds file size, and
 869 // that file size is correctly reported.
 870 class CheckDownloadProgressDelegate : public WaitingURLFetcherDelegate {
 871  public:
 872   CheckDownloadProgressDelegate(int64 file_size)
 873       : file_size_(file_size), last_seen_progress_(0) {}
 874   ~CheckDownloadProgressDelegate() override {}
 875
 876   void OnURLFetchDownloadProgress(const URLFetcher* source,
 877                                   int64 current,
 878                                   int64 total) override {
 879     // Run default checks.
 880     WaitingURLFetcherDelegate::OnURLFetchDownloadProgress(source, current,
 881                                                           total);
 882
 883     EXPECT_LE(last_seen_progress_, current);
 884     EXPECT_EQ(file_size_, total);
 885     last_seen_progress_ = current;
 886   }
 887
 888  private:
 889   int64 file_size_;
 890   int64 last_seen_progress_;
 891
 892   DISALLOW_COPY_AND_ASSIGN(CheckDownloadProgressDelegate);
 893 };
 894
 895 TEST_F(URLFetcherTest, DownloadProgress) {
 896   // Get a file large enough to require more than one read into
 897   // URLFetcher::Core's IOBuffer.
 898   const char kFileToFetch[] = "animate1.gif";
 899
 900   std::string file_contents;
 901   ASSERT_TRUE(base::ReadFileToString(
 902       test_server_->GetDocumentRoot().AppendASCII(kFileToFetch),
 903       &file_contents));
 904
 905   CheckDownloadProgressDelegate delegate(file_contents.size());
 906   delegate.CreateFetcherWithContext(
 907       test_server_->GetURL(std::string(kTestServerFilePrefix) + kFileToFetch),
 908       URLFetcher::GET, request_context());
 909   delegate.StartFetcherAndWait();
 910
 911   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 912   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 913   std::string data;
 914   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
 915   EXPECT_EQ(file_contents, data);
 916 }
 917
 918 class CancelOnUploadProgressDelegate : public WaitingURLFetcherDelegate {
 919  public:
 920   CancelOnUploadProgressDelegate() {}
 921   ~CancelOnUploadProgressDelegate() override {}
 922
 923   void OnURLFetchUploadProgress(const URLFetcher* source,
 924                                 int64 current,
 925                                 int64 total) override {
 926     CancelFetch();
 927   }
 928
 929  private:
 930   DISALLOW_COPY_AND_ASSIGN(CancelOnUploadProgressDelegate);
 931 };
 932
 933 // Check that a fetch can be safely cancelled/deleted during an upload progress
 934 // callback.
 935 TEST_F(URLFetcherTest, CancelInUploadProgressCallback) {
 936   CancelOnUploadProgressDelegate delegate;
 937   delegate.CreateFetcherWithContext(test_server_->GetURL("echo"),
 938                                     URLFetcher::POST, request_context());
 939   delegate.fetcher()->SetChunkedUpload("application/x-www-form-urlencoded");
 940   delegate.fetcher()->Start();
 941   // Use a chunked upload so that the upload can be paused after uploading data.
 942   // Since uploads progress uses a timer, may not receive any notification,
 943   // otherwise.
 944   std::string upload_data(1 << 16, 'a');
 945   delegate.fetcher()->AppendChunkToUpload(upload_data, false);
 946   delegate.WaitForComplete();
 947
 948   // Make sure there are no pending events that cause problems when run.
 949   base::RunLoop().RunUntilIdle();
 950
 951   EXPECT_FALSE(delegate.did_complete());
 952   EXPECT_FALSE(delegate.fetcher());
 953 }
 954
 955 class CancelOnDownloadProgressDelegate : public WaitingURLFetcherDelegate {
 956  public:
 957   CancelOnDownloadProgressDelegate() {}
 958   ~CancelOnDownloadProgressDelegate() override {}
 959
 960   void OnURLFetchDownloadProgress(const URLFetcher* source,
 961                                   int64 current,
 962                                   int64 total) override {
 963     CancelFetch();
 964   }
 965
 966  private:
 967   DISALLOW_COPY_AND_ASSIGN(CancelOnDownloadProgressDelegate);
 968 };
 969
 970 // Check that a fetch can be safely cancelled/deleted during a download progress
 971 // callback.
 972 TEST_F(URLFetcherTest, CancelInDownloadProgressCallback) {
 973   // Get a file large enough to require more than one read into
 974   // URLFetcher::Core's IOBuffer.
 975   static const char kFileToFetch[] = "animate1.gif";
 976   CancelOnDownloadProgressDelegate delegate;
 977   delegate.CreateFetcherWithContext(
 978       test_server_->GetURL(std::string(kTestServerFilePrefix) + kFileToFetch),
 979       URLFetcher::GET, request_context());
 980   delegate.StartFetcherAndWait();
 981
 982   // Make sure there are no pending events that cause problems when run.
 983   base::RunLoop().RunUntilIdle();
 984
 985   EXPECT_FALSE(delegate.did_complete());
 986   EXPECT_FALSE(delegate.fetcher());
 987 }
 988
 989 TEST_F(URLFetcherTest, Headers) {
 990   WaitingURLFetcherDelegate delegate;
 991   delegate.CreateFetcherWithContext(
 992       test_server_->GetURL("set-header?cache-control: private"),
 993       URLFetcher::GET, request_context());
 994   delegate.StartFetcherAndWait();
 995
 996   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
 997   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
 998   std::string header;
 999   ASSERT_TRUE(delegate.fetcher()->GetResponseHeaders()->GetNormalizedHeader(
1000       "cache-control", &header));
1001   EXPECT_EQ("private", header);
1002 }
1003
1004 TEST_F(URLFetcherTest, SocketAddress) {
1005   WaitingURLFetcherDelegate delegate;
1006   delegate.CreateFetcherWithContext(test_server_->GetURL(kDefaultResponsePath),
1007                                     URLFetcher::GET, request_context());
1008   delegate.StartFetcherAndWait();
1009
1010   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
1011   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
1012   EXPECT_EQ(test_server_->host_port_pair().port(),
1013             delegate.fetcher()->GetSocketAddress().port());
1014   EXPECT_EQ(test_server_->host_port_pair().host(),
1015             delegate.fetcher()->GetSocketAddress().host());
1016 }
1017
1018 TEST_F(URLFetcherTest, StopOnRedirect) {
1019   const char kRedirectTarget[] = "http://redirect.target.com";
1020
1021   WaitingURLFetcherDelegate delegate;
1022   delegate.CreateFetcherWithContext(
1023       test_server_->GetURL(std::string("server-redirect?") + kRedirectTarget),
1024       URLFetcher::GET, request_context());
1025   delegate.fetcher()->SetStopOnRedirect(true);
1026   delegate.StartFetcherAndWait();
1027
1028   EXPECT_EQ(GURL(kRedirectTarget), delegate.fetcher()->GetURL());
1029   EXPECT_EQ(URLRequestStatus::CANCELED,
1030             delegate.fetcher()->GetStatus().status());
1031   EXPECT_EQ(ERR_ABORTED, delegate.fetcher()->GetStatus().error());
1032   EXPECT_EQ(301, delegate.fetcher()->GetResponseCode());
1033 }
1034
1035 TEST_F(URLFetcherTest, ThrottleOnRepeatedFetches) {
1036   base::Time start_time = Time::Now();
1037   GURL url(test_server_->GetURL(kDefaultResponsePath));
1038
1039   // Registers an entry for test url. It only allows 3 requests to be sent
1040   // in 200 milliseconds.
1041   scoped_refptr<URLRequestThrottlerEntry> entry(new URLRequestThrottlerEntry(
1042       request_context()->throttler_manager(), std::string() /* url_id */,
1043       200 /* sliding_window_period_ms */, 3 /* max_send_threshold */,
1044       1 /* initial_backoff_ms */, 2.0 /* multiply_factor */,
1045       0.0 /* jitter_factor */, 256 /* maximum_backoff_ms */));
1046
1047   request_context()->throttler_manager()
1048       ->OverrideEntryForTests(url, entry.get());
1049
1050   for (int i = 0; i < 20; ++i) {
1051     WaitingURLFetcherDelegate delegate;
1052     delegate.CreateFetcherWithContext(url, URLFetcher::GET, request_context());
1053     delegate.StartFetcherAndWait();
1054
1055     EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
1056     EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
1057   }
1058
1059   // 20 requests were sent. Due to throttling, they should have collectively
1060   // taken over 1 second.
1061   EXPECT_GE(Time::Now() - start_time, base::TimeDelta::FromSeconds(1));
1062 }
1063
1064 TEST_F(URLFetcherTest, ThrottleOn5xxRetries) {
1065   base::Time start_time = Time::Now();
1066   GURL url(test_server_->GetURL("files/server-unavailable.html"));
1067
1068   // Registers an entry for test url. The backoff time is calculated by:
1069   //     new_backoff = 2.0 * old_backoff + 0
1070   // and maximum backoff time is 256 milliseconds.
1071   // Maximum retries allowed is set to 11.
1072   scoped_refptr<URLRequestThrottlerEntry> entry(new URLRequestThrottlerEntry(
1073       request_context()->throttler_manager(), std::string() /* url_id */,
1074       200 /* sliding_window_period_ms */, 3 /* max_send_threshold */,
1075       1 /* initial_backoff_ms */, 2.0 /* multiply_factor */,
1076       0.0 /* jitter_factor */, 256 /* maximum_backoff_ms */));
1077   request_context()->throttler_manager()
1078       ->OverrideEntryForTests(url, entry.get());
1079
1080   request_context()->throttler_manager()->OverrideEntryForTests(url,
1081                                                                 entry.get());
1082
1083   WaitingURLFetcherDelegate delegate;
1084   delegate.CreateFetcherWithContext(url, URLFetcher::GET, request_context());
1085   delegate.fetcher()->SetAutomaticallyRetryOn5xx(true);
1086   delegate.fetcher()->SetMaxRetriesOn5xx(11);
1087   delegate.StartFetcherAndWait();
1088
1089   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
1090   EXPECT_EQ(503, delegate.fetcher()->GetResponseCode());
1091   std::string data;
1092   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
1093   EXPECT_FALSE(data.empty());
1094
1095   // The request should have been retried 11 times (12 times including the first
1096   // attempt).  Due to throttling, they should have collectively taken over 1
1097   // second.
1098   EXPECT_GE(Time::Now() - start_time, base::TimeDelta::FromSeconds(1));
1099 }
1100
1101 // Tests overload protection, when responses passed through.
1102 TEST_F(URLFetcherTest, ProtectTestPassedThrough) {
1103   base::Time start_time = Time::Now();
1104   GURL url(test_server_->GetURL("files/server-unavailable.html"));
1105
1106   // Registers an entry for test url. The backoff time is calculated by:
1107   //     new_backoff = 2.0 * old_backoff + 0
1108   // and maximum backoff time is 150000 milliseconds.
1109   // Maximum retries allowed is set to 11.
1110   scoped_refptr<URLRequestThrottlerEntry> entry(new URLRequestThrottlerEntry(
1111       request_context()->throttler_manager(), std::string() /* url_id */,
1112       200 /* sliding_window_period_ms */, 3 /* max_send_threshold */,
1113       10000 /* initial_backoff_ms */, 2.0 /* multiply_factor */,
1114       0.0 /* jitter_factor */, 150000 /* maximum_backoff_ms */));
1115   // Total time if *not* for not doing automatic backoff would be 150s.
1116   // In reality it should be "as soon as server responds".
1117   request_context()->throttler_manager()
1118       ->OverrideEntryForTests(url, entry.get());
1119
1120   WaitingURLFetcherDelegate delegate;
1121   delegate.CreateFetcherWithContext(url, URLFetcher::GET, request_context());
1122   delegate.fetcher()->SetAutomaticallyRetryOn5xx(false);
1123   delegate.fetcher()->SetMaxRetriesOn5xx(11);
1124   delegate.StartFetcherAndWait();
1125
1126   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
1127   EXPECT_EQ(503, delegate.fetcher()->GetResponseCode());
1128   std::string data;
1129   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
1130   EXPECT_FALSE(data.empty());
1131   EXPECT_GT(delegate.fetcher()->GetBackoffDelay().InMicroseconds(), 0);
1132
1133   // The request should not have been retried at all.  If it had attempted all
1134   // 11 retries, that should have taken 2.5 minutes.
1135   EXPECT_TRUE(Time::Now() - start_time < TimeDelta::FromMinutes(1));
1136 }
1137
1138 TEST_F(URLFetcherCancelTest, ReleasesContext) {
1139   GURL url(test_server_->GetURL("files/server-unavailable.html"));
1140
1141   // Create a separate thread that will create the URLFetcher.  The current
1142   // (main) thread will do the IO, and when the fetch is complete it will
1143   // terminate the main thread's message loop; then the other thread's
1144   // message loop will be shut down automatically as the thread goes out of
1145   // scope.
1146   base::Thread t("URLFetcher test thread");
1147   ASSERT_TRUE(t.Start());
1148   t.message_loop()->PostTask(
1149       FROM_HERE,
1150       base::Bind(&URLFetcherCancelTest::CreateFetcher,
1151                  base::Unretained(this), url));
1152
1153   base::MessageLoop::current()->Run();
1154 }
1155
1156 TEST_F(URLFetcherCancelTest, CancelWhileDelayedStartTaskPending) {
1157   GURL url(test_server_->GetURL("files/server-unavailable.html"));
1158
1159   // Register an entry for test url.
1160   // Using a sliding window of 4 seconds, and max of 1 request, under a fast
1161   // run we expect to have a 4 second delay when posting the Start task.
1162   scoped_refptr<URLRequestThrottlerEntry> entry(
1163       new URLRequestThrottlerEntry(request_context()->throttler_manager(),
1164                                    std::string(),
1165                                    4000,
1166                                    1,
1167                                    2000,
1168                                    2.0,
1169                                    0.0,
1170                                    4000));
1171   request_context()->throttler_manager()
1172       ->OverrideEntryForTests(url, entry.get());
1173   // Fake that a request has just started.
1174   entry->ReserveSendingTimeForNextRequest(base::TimeTicks());
1175
1176   // The next request we try to send will be delayed by ~4 seconds.
1177   // The slower the test runs, the less the delay will be (since it takes the
1178   // time difference from now).
1179
1180   base::Thread t("URLFetcher test thread");
1181   ASSERT_TRUE(t.Start());
1182   t.message_loop()->PostTask(
1183       FROM_HERE,
1184       base::Bind(&URLFetcherTest::CreateFetcher, base::Unretained(this), url));
1185
1186   base::MessageLoop::current()->Run();
1187 }
1188
1189 // A URLFetcherDelegate that expects to receive a response body of "request1"
1190 // and then reuses the fetcher for the same URL, setting the "test" request
1191 // header to "request2".
1192 class ReuseFetcherDelegate : public WaitingURLFetcherDelegate {
1193  public:
1194   // |second_request_context_getter| is the context getter used for the second
1195   // request. Can't reuse the old one because fetchers release it on completion.
1196   ReuseFetcherDelegate(
1197       scoped_refptr<URLRequestContextGetter> second_request_context_getter)
1198       : first_request_complete_(false),
1199         second_request_context_getter_(second_request_context_getter) {}
1200
1201   ~ReuseFetcherDelegate() override {}
1202
1203   void OnURLFetchComplete(const URLFetcher* source) override {
1204     EXPECT_EQ(fetcher(), source);
1205     if (!first_request_complete_) {
1206       first_request_complete_ = true;
1207       EXPECT_TRUE(fetcher()->GetStatus().is_success());
1208       EXPECT_EQ(200, fetcher()->GetResponseCode());
1209       std::string data;
1210       ASSERT_TRUE(fetcher()->GetResponseAsString(&data));
1211       EXPECT_EQ("request1", data);
1212
1213       fetcher()->SetRequestContext(second_request_context_getter_.get());
1214       fetcher()->SetExtraRequestHeaders("test: request2");
1215       fetcher()->Start();
1216       return;
1217     }
1218     WaitingURLFetcherDelegate::OnURLFetchComplete(source);
1219   }
1220
1221  private:
1222   bool first_request_complete_;
1223   scoped_refptr<URLRequestContextGetter> second_request_context_getter_;
1224
1225   DISALLOW_COPY_AND_ASSIGN(ReuseFetcherDelegate);
1226 };
1227
1228 TEST_F(URLFetcherTest, ReuseFetcherForSameURL) {
1229   // TODO(mmenke):  It's really weird that this is supported, particularly
1230   // some fields can be modified between requests, but some (Like upload body)
1231   // cannot be. Can we get rid of support for this?
1232   ReuseFetcherDelegate delegate(new TrivialURLRequestContextGetter(
1233       request_context(), base::MessageLoopProxy::current()));
1234   delegate.CreateFetcherWithContext(test_server_->GetURL("echoheader?test"),
1235                                     URLFetcher::GET, request_context());
1236   delegate.fetcher()->SetExtraRequestHeaders("test: request1");
1237   delegate.StartFetcherAndWait();
1238
1239   EXPECT_TRUE(delegate.fetcher()->GetStatus().is_success());
1240   EXPECT_EQ(200, delegate.fetcher()->GetResponseCode());
1241   std::string data;
1242   ASSERT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
1243   EXPECT_EQ("request2", data);
1244 }
1245
1246 // Get a small file.
1247 TEST_F(URLFetcherTest, FileTestSmallGet) {
1248   const char kFileToFetch[] = "simple.html";
1249
1250   base::ScopedTempDir temp_dir;
1251   ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
1252   base::FilePath out_path = temp_dir.path().AppendASCII(kFileToFetch);
1253   SaveFileTest(kFileToFetch, false, out_path, false);
1254 }
1255
1256 // Get a file large enough to require more than one read into URLFetcher::Core's
1257 // IOBuffer.
1258 TEST_F(URLFetcherTest, FileTestLargeGet) {
1259   const char kFileToFetch[] = "animate1.gif";
1260
1261   base::ScopedTempDir temp_dir;
1262   ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
1263   base::FilePath out_path = temp_dir.path().AppendASCII(kFileToFetch);
1264   SaveFileTest(kFileToFetch, false, out_path, false);
1265 }
1266
1267 // If the caller takes the ownership of the output file, the file should persist
1268 // even after URLFetcher is gone.
1269 TEST_F(URLFetcherTest, FileTestTakeOwnership) {
1270   const char kFileToFetch[] = "simple.html";
1271
1272   base::ScopedTempDir temp_dir;
1273   ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
1274   base::FilePath out_path = temp_dir.path().AppendASCII(kFileToFetch);
1275   SaveFileTest(kFileToFetch, false, out_path, true);
1276 }
1277
1278 // Test that an existing file can be overwritten be a fetcher.
1279 TEST_F(URLFetcherTest, FileTestOverwriteExisting) {
1280   base::ScopedTempDir temp_dir;
1281   ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
1282
1283   // Create a file before trying to fetch.
1284   const char kFileToFetch[] = "simple.html";
1285   std::string data(10000, '?');  // Meant to be larger than simple.html.
1286   base::FilePath out_path = temp_dir.path().AppendASCII(kFileToFetch);
1287   ASSERT_EQ(static_cast<int>(data.size()),
1288             base::WriteFile(out_path, data.data(), data.size()));
1289   ASSERT_TRUE(base::PathExists(out_path));
1290
1291   SaveFileTest(kFileToFetch, false, out_path, true);
1292 }
1293
1294 // Test trying to overwrite a directory with a file when using a fetcher fails.
1295 TEST_F(URLFetcherTest, FileTestTryToOverwriteDirectory) {
1296   base::ScopedTempDir temp_dir;
1297   ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
1298
1299   // Create a directory before trying to fetch.
1300   static const char kFileToFetch[] = "simple.html";
1301   base::FilePath out_path = temp_dir.path().AppendASCII(kFileToFetch);
1302   ASSERT_TRUE(base::CreateDirectory(out_path));
1303   ASSERT_TRUE(base::PathExists(out_path));
1304
1305   WaitingURLFetcherDelegate delegate;
1306   delegate.CreateFetcherWithContext(
1307       test_server_->GetURL(std::string(kTestServerFilePrefix) + kFileToFetch),
1308       URLFetcher::GET, request_context());
1309   delegate.fetcher()->SaveResponseToFileAtPath(
1310       out_path,
1311       scoped_refptr<base::MessageLoopProxy>(base::MessageLoopProxy::current()));
1312   delegate.StartFetcherAndWait();
1313
1314   EXPECT_FALSE(delegate.fetcher()->GetStatus().is_success());
1315   EXPECT_EQ(ERR_ACCESS_DENIED, delegate.fetcher()->GetStatus().error());
1316 }
1317
1318 // Get a small file and save it to a temp file.
1319 TEST_F(URLFetcherTest, TempFileTestSmallGet) {
1320   SaveFileTest("simple.html", true, base::FilePath(), false);
1321 }
1322
1323 // Get a file large enough to require more than one read into URLFetcher::Core's
1324 // IOBuffer and save it to a temp file.
1325 TEST_F(URLFetcherTest, TempFileTestLargeGet) {
1326   SaveFileTest("animate1.gif", true, base::FilePath(), false);
1327 }
1328
1329 // If the caller takes the ownership of the temp file, check that the file
1330 // persists even after URLFetcher is gone.
1331 TEST_F(URLFetcherTest, TempFileTestTakeOwnership) {
1332   SaveFileTest("simple.html", true, base::FilePath(), true);
1333 }
1334
1335 TEST_F(URLFetcherBadHTTPSTest, BadHTTPS) {
1336   WaitingURLFetcherDelegate delegate;
1337   delegate.CreateFetcherWithContext(test_server_->GetURL(kDefaultResponsePath),
1338                                     URLFetcher::GET, request_context());
1339   delegate.StartFetcherAndWait();
1340
1341   EXPECT_EQ(URLRequestStatus::CANCELED,
1342             delegate.fetcher()->GetStatus().status());
1343   EXPECT_EQ(ERR_ABORTED, delegate.fetcher()->GetStatus().error());
1344   EXPECT_EQ(-1, delegate.fetcher()->GetResponseCode());
1345   EXPECT_TRUE(delegate.fetcher()->GetCookies().empty());
1346   std::string data;
1347   EXPECT_TRUE(delegate.fetcher()->GetResponseAsString(&data));
1348   EXPECT_TRUE(data.empty());
1349 }
1350
1351 }  // namespace
1352
1353 }  // namespace net