net/url_request/url_fetcher_core.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_fetcher_core.h"
   6
   7 #include <stdint.h>
   8
   9 #include "base/bind.h"
  10 #include "base/logging.h"
  11 #include "base/metrics/histogram.h"
  12 #include "base/profiler/scoped_tracker.h"
  13 #include "base/sequenced_task_runner.h"
  14 #include "base/single_thread_task_runner.h"
  15 #include "base/stl_util.h"
  16 #include "base/thread_task_runner_handle.h"
  17 #include "base/tracked_objects.h"
  18 #include "net/base/elements_upload_data_stream.h"
  19 #include "net/base/io_buffer.h"
  20 #include "net/base/load_flags.h"
  21 #include "net/base/net_errors.h"
  22 #include "net/base/request_priority.h"
  23 #include "net/base/upload_bytes_element_reader.h"
  24 #include "net/base/upload_data_stream.h"
  25 #include "net/base/upload_file_element_reader.h"
  26 #include "net/http/http_response_headers.h"
  27 #include "net/url_request/redirect_info.h"
  28 #include "net/url_request/url_fetcher_delegate.h"
  29 #include "net/url_request/url_fetcher_response_writer.h"
  30 #include "net/url_request/url_request_context.h"
  31 #include "net/url_request/url_request_context_getter.h"
  32 #include "net/url_request/url_request_throttler_manager.h"
  33
  34 namespace {
  35
  36 const int kBufferSize = 4096;
  37 const int kUploadProgressTimerInterval = 100;
  38 bool g_ignore_certificate_requests = false;
  39
  40 void EmptyCompletionCallback(int result) {}
  41
  42 }  // namespace
  43
  44 namespace net {
  45
  46 // URLFetcherCore::Registry ---------------------------------------------------
  47
  48 URLFetcherCore::Registry::Registry() {}
  49 URLFetcherCore::Registry::~Registry() {}
  50
  51 void URLFetcherCore::Registry::AddURLFetcherCore(URLFetcherCore* core) {
  52   DCHECK(!ContainsKey(fetchers_, core));
  53   fetchers_.insert(core);
  54 }
  55
  56 void URLFetcherCore::Registry::RemoveURLFetcherCore(URLFetcherCore* core) {
  57   DCHECK(ContainsKey(fetchers_, core));
  58   fetchers_.erase(core);
  59 }
  60
  61 void URLFetcherCore::Registry::CancelAll() {
  62   while (!fetchers_.empty())
  63     (*fetchers_.begin())->CancelURLRequest(ERR_ABORTED);
  64 }
  65
  66 // URLFetcherCore -------------------------------------------------------------
  67
  68 // static
  69 base::LazyInstance<URLFetcherCore::Registry>
  70     URLFetcherCore::g_registry = LAZY_INSTANCE_INITIALIZER;
  71
  72 URLFetcherCore::URLFetcherCore(URLFetcher* fetcher,
  73                                const GURL& original_url,
  74                                URLFetcher::RequestType request_type,
  75                                URLFetcherDelegate* d)
  76     : fetcher_(fetcher),
  77       original_url_(original_url),
  78       request_type_(request_type),
  79       delegate_(d),
  80       delegate_task_runner_(base::ThreadTaskRunnerHandle::Get()),
  81       load_flags_(LOAD_NORMAL),
  82       response_code_(URLFetcher::RESPONSE_CODE_INVALID),
  83       buffer_(new IOBuffer(kBufferSize)),
  84       url_request_data_key_(NULL),
  85       was_fetched_via_proxy_(false),
  86       upload_content_set_(false),
  87       upload_range_offset_(0),
  88       upload_range_length_(0),
  89       referrer_policy_(
  90           URLRequest::CLEAR_REFERRER_ON_TRANSITION_FROM_SECURE_TO_INSECURE),
  91       is_chunked_upload_(false),
  92       was_cancelled_(false),
  93       stop_on_redirect_(false),
  94       stopped_on_redirect_(false),
  95       automatically_retry_on_5xx_(true),
  96       num_retries_on_5xx_(0),
  97       max_retries_on_5xx_(0),
  98       num_retries_on_network_changes_(0),
  99       max_retries_on_network_changes_(0),
 100       current_upload_bytes_(-1),
 101       current_response_bytes_(0),
 102       total_response_bytes_(-1) {
 103   CHECK(original_url_.is_valid());
 104 }
 105
 106 void URLFetcherCore::Start() {
 107   DCHECK(delegate_task_runner_.get());
 108   DCHECK(request_context_getter_.get()) << "We need an URLRequestContext!";
 109   if (network_task_runner_.get()) {
 110     DCHECK_EQ(network_task_runner_,
 111               request_context_getter_->GetNetworkTaskRunner());
 112   } else {
 113     network_task_runner_ = request_context_getter_->GetNetworkTaskRunner();
 114   }
 115   DCHECK(network_task_runner_.get()) << "We need an IO task runner";
 116
 117   network_task_runner_->PostTask(
 118       FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
 119 }
 120
 121 void URLFetcherCore::Stop() {
 122   if (delegate_task_runner_.get())  // May be NULL in tests.
 123     DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 124
 125   delegate_ = NULL;
 126   fetcher_ = NULL;
 127   if (!network_task_runner_.get())
 128     return;
 129   if (network_task_runner_->RunsTasksOnCurrentThread()) {
 130     CancelURLRequest(ERR_ABORTED);
 131   } else {
 132     network_task_runner_->PostTask(
 133         FROM_HERE,
 134         base::Bind(&URLFetcherCore::CancelURLRequest, this, ERR_ABORTED));
 135   }
 136 }
 137
 138 void URLFetcherCore::SetUploadData(const std::string& upload_content_type,
 139                                    const std::string& upload_content) {
 140   AssertHasNoUploadData();
 141   DCHECK(!is_chunked_upload_);
 142   DCHECK(upload_content_type_.empty());
 143
 144   // Empty |upload_content_type| is allowed iff the |upload_content| is empty.
 145   DCHECK(upload_content.empty() || !upload_content_type.empty());
 146
 147   upload_content_type_ = upload_content_type;
 148   upload_content_ = upload_content;
 149   upload_content_set_ = true;
 150 }
 151
 152 void URLFetcherCore::SetUploadFilePath(
 153     const std::string& upload_content_type,
 154     const base::FilePath& file_path,
 155     uint64 range_offset,
 156     uint64 range_length,
 157     scoped_refptr<base::TaskRunner> file_task_runner) {
 158   AssertHasNoUploadData();
 159   DCHECK(!is_chunked_upload_);
 160   DCHECK_EQ(upload_range_offset_, 0ULL);
 161   DCHECK_EQ(upload_range_length_, 0ULL);
 162   DCHECK(upload_content_type_.empty());
 163   DCHECK(!upload_content_type.empty());
 164
 165   upload_content_type_ = upload_content_type;
 166   upload_file_path_ = file_path;
 167   upload_range_offset_ = range_offset;
 168   upload_range_length_ = range_length;
 169   upload_file_task_runner_ = file_task_runner;
 170   upload_content_set_ = true;
 171 }
 172
 173 void URLFetcherCore::SetUploadStreamFactory(
 174     const std::string& upload_content_type,
 175     const URLFetcher::CreateUploadStreamCallback& factory) {
 176   AssertHasNoUploadData();
 177   DCHECK(!is_chunked_upload_);
 178   DCHECK(upload_content_type_.empty());
 179
 180   upload_content_type_ = upload_content_type;
 181   upload_stream_factory_ = factory;
 182   upload_content_set_ = true;
 183 }
 184
 185 void URLFetcherCore::SetChunkedUpload(const std::string& content_type) {
 186   if (!is_chunked_upload_) {
 187     AssertHasNoUploadData();
 188     DCHECK(upload_content_type_.empty());
 189   }
 190
 191   // Empty |content_type| is not allowed here, because it is impossible
 192   // to ensure non-empty upload content as it is not yet supplied.
 193   DCHECK(!content_type.empty());
 194
 195   upload_content_type_ = content_type;
 196   upload_content_.clear();
 197   is_chunked_upload_ = true;
 198 }
 199
 200 void URLFetcherCore::AppendChunkToUpload(const std::string& content,
 201                                          bool is_last_chunk) {
 202   DCHECK(delegate_task_runner_.get());
 203   DCHECK(network_task_runner_.get());
 204   network_task_runner_->PostTask(
 205       FROM_HERE,
 206       base::Bind(&URLFetcherCore::CompleteAddingUploadDataChunk, this, content,
 207                  is_last_chunk));
 208 }
 209
 210 void URLFetcherCore::SetLoadFlags(int load_flags) {
 211   load_flags_ = load_flags;
 212 }
 213
 214 int URLFetcherCore::GetLoadFlags() const {
 215   return load_flags_;
 216 }
 217
 218 void URLFetcherCore::SetReferrer(const std::string& referrer) {
 219   referrer_ = referrer;
 220 }
 221
 222 void URLFetcherCore::SetReferrerPolicy(
 223     URLRequest::ReferrerPolicy referrer_policy) {
 224   referrer_policy_ = referrer_policy;
 225 }
 226
 227 void URLFetcherCore::SetExtraRequestHeaders(
 228     const std::string& extra_request_headers) {
 229   extra_request_headers_.Clear();
 230   extra_request_headers_.AddHeadersFromString(extra_request_headers);
 231 }
 232
 233 void URLFetcherCore::AddExtraRequestHeader(const std::string& header_line) {
 234   extra_request_headers_.AddHeaderFromString(header_line);
 235 }
 236
 237 void URLFetcherCore::SetRequestContext(
 238     URLRequestContextGetter* request_context_getter) {
 239   DCHECK(!request_context_getter_.get());
 240   DCHECK(request_context_getter);
 241   request_context_getter_ = request_context_getter;
 242 }
 243
 244 void URLFetcherCore::SetFirstPartyForCookies(
 245     const GURL& first_party_for_cookies) {
 246   DCHECK(first_party_for_cookies_.is_empty());
 247   first_party_for_cookies_ = first_party_for_cookies;
 248 }
 249
 250 void URLFetcherCore::SetURLRequestUserData(
 251     const void* key,
 252     const URLFetcher::CreateDataCallback& create_data_callback) {
 253   DCHECK(key);
 254   DCHECK(!create_data_callback.is_null());
 255   url_request_data_key_ = key;
 256   url_request_create_data_callback_ = create_data_callback;
 257 }
 258
 259 void URLFetcherCore::SetStopOnRedirect(bool stop_on_redirect) {
 260   stop_on_redirect_ = stop_on_redirect;
 261 }
 262
 263 void URLFetcherCore::SetAutomaticallyRetryOn5xx(bool retry) {
 264   automatically_retry_on_5xx_ = retry;
 265 }
 266
 267 void URLFetcherCore::SetMaxRetriesOn5xx(int max_retries) {
 268   max_retries_on_5xx_ = max_retries;
 269 }
 270
 271 int URLFetcherCore::GetMaxRetriesOn5xx() const {
 272   return max_retries_on_5xx_;
 273 }
 274
 275 base::TimeDelta URLFetcherCore::GetBackoffDelay() const {
 276   return backoff_delay_;
 277 }
 278
 279 void URLFetcherCore::SetAutomaticallyRetryOnNetworkChanges(int max_retries) {
 280   max_retries_on_network_changes_ = max_retries;
 281 }
 282
 283 void URLFetcherCore::SaveResponseToFileAtPath(
 284     const base::FilePath& file_path,
 285     scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
 286   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 287   SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
 288       new URLFetcherFileWriter(file_task_runner, file_path)));
 289 }
 290
 291 void URLFetcherCore::SaveResponseToTemporaryFile(
 292     scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
 293   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 294   SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
 295       new URLFetcherFileWriter(file_task_runner, base::FilePath())));
 296 }
 297
 298 void URLFetcherCore::SaveResponseWithWriter(
 299     scoped_ptr<URLFetcherResponseWriter> response_writer) {
 300   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 301   response_writer_ = response_writer.Pass();
 302 }
 303
 304 HttpResponseHeaders* URLFetcherCore::GetResponseHeaders() const {
 305   return response_headers_.get();
 306 }
 307
 308 // TODO(panayiotis): socket_address_ is written in the IO thread,
 309 // if this is accessed in the UI thread, this could result in a race.
 310 // Same for response_headers_ above and was_fetched_via_proxy_ below.
 311 HostPortPair URLFetcherCore::GetSocketAddress() const {
 312   return socket_address_;
 313 }
 314
 315 bool URLFetcherCore::WasFetchedViaProxy() const {
 316   return was_fetched_via_proxy_;
 317 }
 318
 319 const GURL& URLFetcherCore::GetOriginalURL() const {
 320   return original_url_;
 321 }
 322
 323 const GURL& URLFetcherCore::GetURL() const {
 324   return url_;
 325 }
 326
 327 const URLRequestStatus& URLFetcherCore::GetStatus() const {
 328   return status_;
 329 }
 330
 331 int URLFetcherCore::GetResponseCode() const {
 332   return response_code_;
 333 }
 334
 335 const ResponseCookies& URLFetcherCore::GetCookies() const {
 336   return cookies_;
 337 }
 338
 339 void URLFetcherCore::ReceivedContentWasMalformed() {
 340   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 341   if (network_task_runner_.get()) {
 342     network_task_runner_->PostTask(
 343         FROM_HERE, base::Bind(&URLFetcherCore::NotifyMalformedContent, this));
 344   }
 345 }
 346
 347 bool URLFetcherCore::GetResponseAsString(
 348     std::string* out_response_string) const {
 349   URLFetcherStringWriter* string_writer =
 350       response_writer_ ? response_writer_->AsStringWriter() : NULL;
 351   if (!string_writer)
 352     return false;
 353
 354   *out_response_string = string_writer->data();
 355   UMA_HISTOGRAM_MEMORY_KB("UrlFetcher.StringResponseSize",
 356                           (string_writer->data().length() / 1024));
 357   return true;
 358 }
 359
 360 bool URLFetcherCore::GetResponseAsFilePath(bool take_ownership,
 361                                            base::FilePath* out_response_path) {
 362   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 363
 364   URLFetcherFileWriter* file_writer =
 365       response_writer_ ? response_writer_->AsFileWriter() : NULL;
 366   if (!file_writer)
 367     return false;
 368
 369   *out_response_path = file_writer->file_path();
 370
 371   if (take_ownership) {
 372     // Intentionally calling a file_writer_ method directly without posting
 373     // the task to network_task_runner_.
 374     //
 375     // This is for correctly handling the case when file_writer_->DisownFile()
 376     // is soon followed by URLFetcherCore::Stop(). We have to make sure that
 377     // DisownFile takes effect before Stop deletes file_writer_.
 378     //
 379     // This direct call should be thread-safe, since DisownFile itself does no
 380     // file operation. It just flips the state to be referred in destruction.
 381     file_writer->DisownFile();
 382   }
 383   return true;
 384 }
 385
 386 void URLFetcherCore::OnReceivedRedirect(URLRequest* request,
 387                                         const RedirectInfo& redirect_info,
 388                                         bool* defer_redirect) {
 389   DCHECK_EQ(request, request_.get());
 390   DCHECK(network_task_runner_->BelongsToCurrentThread());
 391   if (stop_on_redirect_) {
 392     stopped_on_redirect_ = true;
 393     url_ = redirect_info.new_url;
 394     response_code_ = request_->GetResponseCode();
 395     was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
 396     request->Cancel();
 397     OnReadCompleted(request, 0);
 398   }
 399 }
 400
 401 void URLFetcherCore::OnResponseStarted(URLRequest* request) {
 402   // TODO(vadimt): Remove ScopedTracker below once crbug.com/423948 is fixed.
 403   tracked_objects::ScopedTracker tracking_profile(
 404       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 405           "423948 URLFetcherCore::OnResponseStarted"));
 406
 407   DCHECK_EQ(request, request_.get());
 408   DCHECK(network_task_runner_->BelongsToCurrentThread());
 409   if (request_->status().is_success()) {
 410     response_code_ = request_->GetResponseCode();
 411     response_headers_ = request_->response_headers();
 412     socket_address_ = request_->GetSocketAddress();
 413     was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
 414     total_response_bytes_ = request_->GetExpectedContentSize();
 415   }
 416
 417   ReadResponse();
 418 }
 419
 420 void URLFetcherCore::OnCertificateRequested(
 421     URLRequest* request,
 422     SSLCertRequestInfo* cert_request_info) {
 423   DCHECK_EQ(request, request_.get());
 424   DCHECK(network_task_runner_->BelongsToCurrentThread());
 425
 426   if (g_ignore_certificate_requests) {
 427     request->ContinueWithCertificate(NULL);
 428   } else {
 429     request->Cancel();
 430   }
 431 }
 432
 433 void URLFetcherCore::OnReadCompleted(URLRequest* request,
 434                                      int bytes_read) {
 435   // TODO(vadimt): Remove ScopedTracker below once crbug.com/423948 is fixed.
 436   tracked_objects::ScopedTracker tracking_profile(
 437       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 438           "423948 URLFetcherCore::OnReadCompleted"));
 439
 440   DCHECK(request == request_);
 441   DCHECK(network_task_runner_->BelongsToCurrentThread());
 442
 443   if (!stopped_on_redirect_)
 444     url_ = request->url();
 445   URLRequestThrottlerManager* throttler_manager =
 446       request->context()->throttler_manager();
 447   if (throttler_manager) {
 448     // TODO(vadimt): Remove ScopedTracker below once crbug.com/423948 is fixed.
 449     tracked_objects::ScopedTracker tracking_profile1(
 450         FROM_HERE_WITH_EXPLICIT_FUNCTION(
 451             "423948 URLFetcherCore::OnReadCompleted1"));
 452
 453     url_throttler_entry_ = throttler_manager->RegisterRequestUrl(url_);
 454   }
 455
 456   // TODO(vadimt): Remove ScopedTracker below once crbug.com/423948 is fixed.
 457   tracked_objects::ScopedTracker tracking_profile2(
 458       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 459           "423948 URLFetcherCore::OnReadCompleted2"));
 460
 461   do {
 462     if (!request_->status().is_success() || bytes_read <= 0)
 463       break;
 464
 465     // TODO(vadimt): Remove ScopedTracker below once crbug.com/423948 is fixed.
 466     tracked_objects::ScopedTracker tracking_profile3(
 467         FROM_HERE_WITH_EXPLICIT_FUNCTION(
 468             "423948 URLFetcherCore::OnReadCompleted3"));
 469
 470     current_response_bytes_ += bytes_read;
 471     InformDelegateDownloadProgress();
 472
 473     // TODO(vadimt): Remove ScopedTracker below once crbug.com/423948 is fixed.
 474     tracked_objects::ScopedTracker tracking_profile4(
 475         FROM_HERE_WITH_EXPLICIT_FUNCTION(
 476             "423948 URLFetcherCore::OnReadCompleted4"));
 477
 478     const int result =
 479         WriteBuffer(new DrainableIOBuffer(buffer_.get(), bytes_read));
 480     if (result < 0) {
 481       // Write failed or waiting for write completion.
 482       return;
 483     }
 484   } while (request_->Read(buffer_.get(), kBufferSize, &bytes_read));
 485
 486   const URLRequestStatus status = request_->status();
 487
 488   // TODO(vadimt): Remove ScopedTracker below once crbug.com/423948 is fixed.
 489   tracked_objects::ScopedTracker tracking_profile5(
 490       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 491           "423948 URLFetcherCore::OnReadCompleted5"));
 492
 493   if (status.is_success())
 494     request_->GetResponseCookies(&cookies_);
 495
 496   // See comments re: HEAD requests in ReadResponse().
 497   if (!status.is_io_pending() || request_type_ == URLFetcher::HEAD) {
 498     // TODO(vadimt): Remove ScopedTracker below once crbug.com/423948 is fixed.
 499     tracked_objects::ScopedTracker tracking_profile6(
 500         FROM_HERE_WITH_EXPLICIT_FUNCTION(
 501             "423948 URLFetcherCore::OnReadCompleted6"));
 502
 503     status_ = status;
 504     ReleaseRequest();
 505
 506     // TODO(vadimt): Remove ScopedTracker below once crbug.com/423948 is fixed.
 507     tracked_objects::ScopedTracker tracking_profile7(
 508         FROM_HERE_WITH_EXPLICIT_FUNCTION(
 509             "423948 URLFetcherCore::OnReadCompleted7"));
 510
 511     // No more data to write.
 512     const int result = response_writer_->Finish(
 513         base::Bind(&URLFetcherCore::DidFinishWriting, this));
 514
 515     // TODO(vadimt): Remove ScopedTracker below once crbug.com/423948 is fixed.
 516     tracked_objects::ScopedTracker tracking_profile8(
 517         FROM_HERE_WITH_EXPLICIT_FUNCTION(
 518             "423948 URLFetcherCore::OnReadCompleted8"));
 519
 520     if (result != ERR_IO_PENDING)
 521       DidFinishWriting(result);
 522   }
 523 }
 524
 525 void URLFetcherCore::CancelAll() {
 526   g_registry.Get().CancelAll();
 527 }
 528
 529 int URLFetcherCore::GetNumFetcherCores() {
 530   return g_registry.Get().size();
 531 }
 532
 533 void URLFetcherCore::SetIgnoreCertificateRequests(bool ignored) {
 534   g_ignore_certificate_requests = ignored;
 535 }
 536
 537 URLFetcherCore::~URLFetcherCore() {
 538   // |request_| should be NULL. If not, it's unsafe to delete it here since we
 539   // may not be on the IO thread.
 540   DCHECK(!request_.get());
 541 }
 542
 543 void URLFetcherCore::StartOnIOThread() {
 544   DCHECK(network_task_runner_->BelongsToCurrentThread());
 545
 546   if (!response_writer_)
 547     response_writer_.reset(new URLFetcherStringWriter);
 548
 549   const int result = response_writer_->Initialize(
 550       base::Bind(&URLFetcherCore::DidInitializeWriter, this));
 551   if (result != ERR_IO_PENDING)
 552     DidInitializeWriter(result);
 553 }
 554
 555 void URLFetcherCore::StartURLRequest() {
 556   // TODO(pkasting): Remove ScopedTracker below once crbug.com/456327 is fixed.
 557   tracked_objects::ScopedTracker tracking_profile1(
 558       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 559           "456327 URLFetcherCore::StartURLRequest1"));
 560   DCHECK(network_task_runner_->BelongsToCurrentThread());
 561
 562   if (was_cancelled_) {
 563     // Since StartURLRequest() is posted as a *delayed* task, it may
 564     // run after the URLFetcher was already stopped.
 565     return;
 566   }
 567
 568   DCHECK(request_context_getter_.get());
 569   DCHECK(!request_.get());
 570
 571   g_registry.Get().AddURLFetcherCore(this);
 572   current_response_bytes_ = 0;
 573   request_ = request_context_getter_->GetURLRequestContext()->CreateRequest(
 574       original_url_, DEFAULT_PRIORITY, this);
 575   request_->set_stack_trace(stack_trace_);
 576   int flags = request_->load_flags() | load_flags_;
 577
 578   // TODO(pkasting): Remove ScopedTracker below once crbug.com/456327 is fixed.
 579   tracked_objects::ScopedTracker tracking_profile2(
 580       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 581           "456327 URLFetcherCore::StartURLRequest2"));
 582   if (is_chunked_upload_)
 583     request_->EnableChunkedUpload();
 584   request_->SetLoadFlags(flags);
 585   request_->SetReferrer(referrer_);
 586   request_->set_referrer_policy(referrer_policy_);
 587   request_->set_first_party_for_cookies(first_party_for_cookies_.is_empty() ?
 588       original_url_ : first_party_for_cookies_);
 589   if (url_request_data_key_ && !url_request_create_data_callback_.is_null()) {
 590     request_->SetUserData(url_request_data_key_,
 591                           url_request_create_data_callback_.Run());
 592   }
 593
 594   // TODO(pkasting): Remove ScopedTracker below once crbug.com/456327 is fixed.
 595   tracked_objects::ScopedTracker tracking_profile3(
 596       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 597           "456327 URLFetcherCore::StartURLRequest3"));
 598   switch (request_type_) {
 599     case URLFetcher::GET:
 600       break;
 601
 602     case URLFetcher::POST:
 603     case URLFetcher::PUT:
 604     case URLFetcher::PATCH: {
 605       // Upload content must be set.
 606       DCHECK(is_chunked_upload_ || upload_content_set_);
 607
 608       request_->set_method(
 609           request_type_ == URLFetcher::POST ? "POST" :
 610           request_type_ == URLFetcher::PUT ? "PUT" : "PATCH");
 611       if (!upload_content_type_.empty()) {
 612         extra_request_headers_.SetHeader(HttpRequestHeaders::kContentType,
 613                                          upload_content_type_);
 614       }
 615       if (!upload_content_.empty()) {
 616         scoped_ptr<UploadElementReader> reader(new UploadBytesElementReader(
 617             upload_content_.data(), upload_content_.size()));
 618         request_->set_upload(
 619             ElementsUploadDataStream::CreateWithReader(reader.Pass(), 0));
 620       } else if (!upload_file_path_.empty()) {
 621         scoped_ptr<UploadElementReader> reader(
 622             new UploadFileElementReader(upload_file_task_runner_.get(),
 623                                         upload_file_path_,
 624                                         upload_range_offset_,
 625                                         upload_range_length_,
 626                                         base::Time()));
 627         request_->set_upload(
 628             ElementsUploadDataStream::CreateWithReader(reader.Pass(), 0));
 629       } else if (!upload_stream_factory_.is_null()) {
 630         scoped_ptr<UploadDataStream> stream = upload_stream_factory_.Run();
 631         DCHECK(stream);
 632         request_->set_upload(stream.Pass());
 633       }
 634
 635       current_upload_bytes_ = -1;
 636       // TODO(kinaba): http://crbug.com/118103. Implement upload callback in the
 637       //  layer and avoid using timer here.
 638       upload_progress_checker_timer_.reset(
 639           new base::RepeatingTimer<URLFetcherCore>());
 640       upload_progress_checker_timer_->Start(
 641           FROM_HERE,
 642           base::TimeDelta::FromMilliseconds(kUploadProgressTimerInterval),
 643           this,
 644           &URLFetcherCore::InformDelegateUploadProgress);
 645       break;
 646     }
 647
 648     case URLFetcher::HEAD:
 649       request_->set_method("HEAD");
 650       break;
 651
 652     case URLFetcher::DELETE_REQUEST:
 653       request_->set_method("DELETE");
 654       break;
 655
 656     default:
 657       NOTREACHED();
 658   }
 659
 660   // TODO(pkasting): Remove ScopedTracker below once crbug.com/456327 is fixed.
 661   tracked_objects::ScopedTracker tracking_profile4(
 662       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 663           "456327 URLFetcherCore::StartURLRequest4"));
 664   if (!extra_request_headers_.IsEmpty())
 665     request_->SetExtraRequestHeaders(extra_request_headers_);
 666
 667   request_->Start();
 668 }
 669
 670 void URLFetcherCore::DidInitializeWriter(int result) {
 671   if (result != OK) {
 672     CancelURLRequest(result);
 673     delegate_task_runner_->PostTask(
 674         FROM_HERE,
 675         base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
 676     return;
 677   }
 678   StartURLRequestWhenAppropriate();
 679 }
 680
 681 void URLFetcherCore::StartURLRequestWhenAppropriate() {
 682   DCHECK(network_task_runner_->BelongsToCurrentThread());
 683
 684   if (was_cancelled_)
 685     return;
 686
 687   DCHECK(request_context_getter_.get());
 688
 689   int64 delay = 0;
 690   if (!original_url_throttler_entry_.get()) {
 691     URLRequestThrottlerManager* manager =
 692         request_context_getter_->GetURLRequestContext()->throttler_manager();
 693     if (manager) {
 694       original_url_throttler_entry_ =
 695           manager->RegisterRequestUrl(original_url_);
 696     }
 697   }
 698   if (original_url_throttler_entry_.get()) {
 699     delay = original_url_throttler_entry_->ReserveSendingTimeForNextRequest(
 700         GetBackoffReleaseTime());
 701   }
 702
 703   if (delay == 0) {
 704     StartURLRequest();
 705   } else {
 706     base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
 707         FROM_HERE, base::Bind(&URLFetcherCore::StartURLRequest, this),
 708         base::TimeDelta::FromMilliseconds(delay));
 709   }
 710 }
 711
 712 void URLFetcherCore::CancelURLRequest(int error) {
 713   DCHECK(network_task_runner_->BelongsToCurrentThread());
 714
 715   if (request_.get()) {
 716     request_->CancelWithError(error);
 717     ReleaseRequest();
 718   }
 719
 720   // Set the error manually.
 721   // Normally, calling URLRequest::CancelWithError() results in calling
 722   // OnReadCompleted() with bytes_read = -1 via an asynchronous task posted by
 723   // URLRequestJob::NotifyDone(). But, because the request was released
 724   // immediately after being canceled, the request could not call
 725   // OnReadCompleted() which overwrites |status_| with the error status.
 726   status_.set_status(URLRequestStatus::CANCELED);
 727   status_.set_error(error);
 728
 729   // Release the reference to the request context. There could be multiple
 730   // references to URLFetcher::Core at this point so it may take a while to
 731   // delete the object, but we cannot delay the destruction of the request
 732   // context.
 733   request_context_getter_ = NULL;
 734   first_party_for_cookies_ = GURL();
 735   url_request_data_key_ = NULL;
 736   url_request_create_data_callback_.Reset();
 737   was_cancelled_ = true;
 738 }
 739
 740 void URLFetcherCore::OnCompletedURLRequest(
 741     base::TimeDelta backoff_delay) {
 742   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 743
 744   // Save the status and backoff_delay so that delegates can read it.
 745   if (delegate_) {
 746     backoff_delay_ = backoff_delay;
 747     InformDelegateFetchIsComplete();
 748   }
 749 }
 750
 751 void URLFetcherCore::InformDelegateFetchIsComplete() {
 752   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 753   if (delegate_)
 754     delegate_->OnURLFetchComplete(fetcher_);
 755 }
 756
 757 void URLFetcherCore::NotifyMalformedContent() {
 758   DCHECK(network_task_runner_->BelongsToCurrentThread());
 759   if (url_throttler_entry_.get()) {
 760     int status_code = response_code_;
 761     if (status_code == URLFetcher::RESPONSE_CODE_INVALID) {
 762       // The status code will generally be known by the time clients
 763       // call the |ReceivedContentWasMalformed()| function (which ends up
 764       // calling the current function) but if it's not, we need to assume
 765       // the response was successful so that the total failure count
 766       // used to calculate exponential back-off goes up.
 767       status_code = 200;
 768     }
 769     url_throttler_entry_->ReceivedContentWasMalformed(status_code);
 770   }
 771 }
 772
 773 void URLFetcherCore::DidFinishWriting(int result) {
 774   if (result != OK) {
 775     CancelURLRequest(result);
 776     delegate_task_runner_->PostTask(
 777         FROM_HERE,
 778         base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
 779     return;
 780   }
 781   // If the file was successfully closed, then the URL request is complete.
 782   RetryOrCompleteUrlFetch();
 783 }
 784
 785 void URLFetcherCore::RetryOrCompleteUrlFetch() {
 786   DCHECK(network_task_runner_->BelongsToCurrentThread());
 787   base::TimeDelta backoff_delay;
 788
 789   // Checks the response from server.
 790   if (response_code_ >= 500 ||
 791       status_.error() == ERR_TEMPORARILY_THROTTLED) {
 792     // When encountering a server error, we will send the request again
 793     // after backoff time.
 794     ++num_retries_on_5xx_;
 795
 796     // Note that backoff_delay may be 0 because (a) the
 797     // URLRequestThrottlerManager and related code does not
 798     // necessarily back off on the first error, (b) it only backs off
 799     // on some of the 5xx status codes, (c) not all URLRequestContexts
 800     // have a throttler manager.
 801     base::TimeTicks backoff_release_time = GetBackoffReleaseTime();
 802     backoff_delay = backoff_release_time - base::TimeTicks::Now();
 803     if (backoff_delay < base::TimeDelta())
 804       backoff_delay = base::TimeDelta();
 805
 806     if (automatically_retry_on_5xx_ &&
 807         num_retries_on_5xx_ <= max_retries_on_5xx_) {
 808       StartOnIOThread();
 809       return;
 810     }
 811   } else {
 812     backoff_delay = base::TimeDelta();
 813   }
 814
 815   // Retry if the request failed due to network changes.
 816   if (status_.error() == ERR_NETWORK_CHANGED &&
 817       num_retries_on_network_changes_ < max_retries_on_network_changes_) {
 818     ++num_retries_on_network_changes_;
 819
 820     // Retry soon, after flushing all the current tasks which may include
 821     // further network change observers.
 822     network_task_runner_->PostTask(
 823         FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
 824     return;
 825   }
 826
 827   request_context_getter_ = NULL;
 828   first_party_for_cookies_ = GURL();
 829   url_request_data_key_ = NULL;
 830   url_request_create_data_callback_.Reset();
 831   bool posted = delegate_task_runner_->PostTask(
 832       FROM_HERE,
 833       base::Bind(&URLFetcherCore::OnCompletedURLRequest, this, backoff_delay));
 834
 835   // If the delegate message loop does not exist any more, then the delegate
 836   // should be gone too.
 837   DCHECK(posted || !delegate_);
 838 }
 839
 840 void URLFetcherCore::ReleaseRequest() {
 841   upload_progress_checker_timer_.reset();
 842   request_.reset();
 843   g_registry.Get().RemoveURLFetcherCore(this);
 844 }
 845
 846 base::TimeTicks URLFetcherCore::GetBackoffReleaseTime() {
 847   DCHECK(network_task_runner_->BelongsToCurrentThread());
 848
 849   if (!original_url_throttler_entry_.get())
 850     return base::TimeTicks();
 851
 852   base::TimeTicks original_url_backoff =
 853       original_url_throttler_entry_->GetExponentialBackoffReleaseTime();
 854   base::TimeTicks destination_url_backoff;
 855   if (url_throttler_entry_.get() &&
 856       original_url_throttler_entry_.get() != url_throttler_entry_.get()) {
 857     destination_url_backoff =
 858         url_throttler_entry_->GetExponentialBackoffReleaseTime();
 859   }
 860
 861   return original_url_backoff > destination_url_backoff ?
 862       original_url_backoff : destination_url_backoff;
 863 }
 864
 865 void URLFetcherCore::CompleteAddingUploadDataChunk(
 866     const std::string& content, bool is_last_chunk) {
 867   if (was_cancelled_) {
 868     // Since CompleteAddingUploadDataChunk() is posted as a *delayed* task, it
 869     // may run after the URLFetcher was already stopped.
 870     return;
 871   }
 872   DCHECK(is_chunked_upload_);
 873   DCHECK(request_.get());
 874   DCHECK(!content.empty());
 875   request_->AppendChunkToUpload(content.data(),
 876                                 static_cast<int>(content.length()),
 877                                 is_last_chunk);
 878 }
 879
 880 int URLFetcherCore::WriteBuffer(scoped_refptr<DrainableIOBuffer> data) {
 881   while (data->BytesRemaining() > 0) {
 882     const int result = response_writer_->Write(
 883         data.get(),
 884         data->BytesRemaining(),
 885         base::Bind(&URLFetcherCore::DidWriteBuffer, this, data));
 886     if (result < 0) {
 887       if (result != ERR_IO_PENDING)
 888         DidWriteBuffer(data, result);
 889       return result;
 890     }
 891     data->DidConsume(result);
 892   }
 893   return OK;
 894 }
 895
 896 void URLFetcherCore::DidWriteBuffer(scoped_refptr<DrainableIOBuffer> data,
 897                                     int result) {
 898   if (result < 0) {  // Handle errors.
 899     CancelURLRequest(result);
 900     response_writer_->Finish(base::Bind(&EmptyCompletionCallback));
 901     delegate_task_runner_->PostTask(
 902         FROM_HERE,
 903         base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
 904     return;
 905   }
 906
 907   // Continue writing.
 908   data->DidConsume(result);
 909   if (WriteBuffer(data) < 0)
 910     return;
 911
 912   // Finished writing buffer_. Read some more, unless the request has been
 913   // cancelled and deleted.
 914   DCHECK_EQ(0, data->BytesRemaining());
 915   if (request_.get())
 916     ReadResponse();
 917 }
 918
 919 void URLFetcherCore::ReadResponse() {
 920   // Some servers may treat HEAD requests as GET requests. To free up the
 921   // network connection as soon as possible, signal that the request has
 922   // completed immediately, without trying to read any data back (all we care
 923   // about is the response code and headers, which we already have).
 924   int bytes_read = 0;
 925   if (request_->status().is_success() &&
 926       (request_type_ != URLFetcher::HEAD)) {
 927     if (!request_->Read(buffer_.get(), kBufferSize, &bytes_read))
 928       bytes_read = -1;  // Match OnReadCompleted() interface contract.
 929   }
 930   OnReadCompleted(request_.get(), bytes_read);
 931 }
 932
 933 void URLFetcherCore::InformDelegateUploadProgress() {
 934   DCHECK(network_task_runner_->BelongsToCurrentThread());
 935   if (request_.get()) {
 936     int64 current = request_->GetUploadProgress().position();
 937     if (current_upload_bytes_ != current) {
 938       current_upload_bytes_ = current;
 939       int64 total = -1;
 940       if (!is_chunked_upload_) {
 941         total = static_cast<int64>(request_->GetUploadProgress().size());
 942         // Total may be zero if the UploadDataStream::Init has not been called
 943         // yet. Don't send the upload progress until the size is initialized.
 944         if (!total)
 945           return;
 946       }
 947       delegate_task_runner_->PostTask(
 948           FROM_HERE,
 949           base::Bind(
 950               &URLFetcherCore::InformDelegateUploadProgressInDelegateThread,
 951               this, current, total));
 952     }
 953   }
 954 }
 955
 956 void URLFetcherCore::InformDelegateUploadProgressInDelegateThread(
 957     int64 current, int64 total) {
 958   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 959   if (delegate_)
 960     delegate_->OnURLFetchUploadProgress(fetcher_, current, total);
 961 }
 962
 963 void URLFetcherCore::InformDelegateDownloadProgress() {
 964   // TODO(vadimt): Remove ScopedTracker below once crbug.com/423948 is fixed.
 965   tracked_objects::ScopedTracker tracking_profile1(
 966       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 967           "423948 URLFetcherCore::InformDelegateDownloadProgress1"));
 968
 969   DCHECK(network_task_runner_->BelongsToCurrentThread());
 970
 971   // TODO(vadimt): Remove ScopedTracker below once crbug.com/423948 is fixed.
 972   tracked_objects::ScopedTracker tracking_profile2(
 973       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 974           "423948 URLFetcherCore::InformDelegateDownloadProgress2"));
 975
 976   delegate_task_runner_->PostTask(
 977       FROM_HERE,
 978       base::Bind(
 979           &URLFetcherCore::InformDelegateDownloadProgressInDelegateThread,
 980           this, current_response_bytes_, total_response_bytes_));
 981 }
 982
 983 void URLFetcherCore::InformDelegateDownloadProgressInDelegateThread(
 984     int64 current, int64 total) {
 985   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 986   if (delegate_)
 987     delegate_->OnURLFetchDownloadProgress(fetcher_, current, total);
 988 }
 989
 990 void URLFetcherCore::AssertHasNoUploadData() const {
 991   DCHECK(!upload_content_set_);
 992   DCHECK(upload_content_.empty());
 993   DCHECK(upload_file_path_.empty());
 994   DCHECK(upload_stream_factory_.is_null());
 995 }
 996
 997 }  // namespace net