net/url_request/url_fetcher_core.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_fetcher_core.h"
   6
   7 #include "base/bind.h"
   8 #include "base/logging.h"
   9 #include "base/metrics/histogram.h"
  10 #include "base/sequenced_task_runner.h"
  11 #include "base/single_thread_task_runner.h"
  12 #include "base/stl_util.h"
  13 #include "base/thread_task_runner_handle.h"
  14 #include "base/tracked_objects.h"
  15 #include "net/base/io_buffer.h"
  16 #include "net/base/load_flags.h"
  17 #include "net/base/net_errors.h"
  18 #include "net/base/request_priority.h"
  19 #include "net/base/upload_bytes_element_reader.h"
  20 #include "net/base/upload_data_stream.h"
  21 #include "net/base/upload_file_element_reader.h"
  22 #include "net/http/http_response_headers.h"
  23 #include "net/url_request/url_fetcher_delegate.h"
  24 #include "net/url_request/url_fetcher_response_writer.h"
  25 #include "net/url_request/url_request_context.h"
  26 #include "net/url_request/url_request_context_getter.h"
  27 #include "net/url_request/url_request_throttler_manager.h"
  28
  29 namespace {
  30
  31 const int kBufferSize = 4096;
  32 const int kUploadProgressTimerInterval = 100;
  33 bool g_interception_enabled = false;
  34 bool g_ignore_certificate_requests = false;
  35
  36 void EmptyCompletionCallback(int result) {}
  37
  38 }  // namespace
  39
  40 namespace net {
  41
  42 // URLFetcherCore::Registry ---------------------------------------------------
  43
  44 URLFetcherCore::Registry::Registry() {}
  45 URLFetcherCore::Registry::~Registry() {}
  46
  47 void URLFetcherCore::Registry::AddURLFetcherCore(URLFetcherCore* core) {
  48   DCHECK(!ContainsKey(fetchers_, core));
  49   fetchers_.insert(core);
  50 }
  51
  52 void URLFetcherCore::Registry::RemoveURLFetcherCore(URLFetcherCore* core) {
  53   DCHECK(ContainsKey(fetchers_, core));
  54   fetchers_.erase(core);
  55 }
  56
  57 void URLFetcherCore::Registry::CancelAll() {
  58   while (!fetchers_.empty())
  59     (*fetchers_.begin())->CancelURLRequest(ERR_ABORTED);
  60 }
  61
  62 // URLFetcherCore -------------------------------------------------------------
  63
  64 // static
  65 base::LazyInstance<URLFetcherCore::Registry>
  66     URLFetcherCore::g_registry = LAZY_INSTANCE_INITIALIZER;
  67
  68 URLFetcherCore::URLFetcherCore(URLFetcher* fetcher,
  69                                const GURL& original_url,
  70                                URLFetcher::RequestType request_type,
  71                                URLFetcherDelegate* d)
  72     : fetcher_(fetcher),
  73       original_url_(original_url),
  74       request_type_(request_type),
  75       delegate_(d),
  76       delegate_task_runner_(base::ThreadTaskRunnerHandle::Get()),
  77       load_flags_(LOAD_NORMAL),
  78       response_code_(URLFetcher::RESPONSE_CODE_INVALID),
  79       buffer_(new IOBuffer(kBufferSize)),
  80       url_request_data_key_(NULL),
  81       was_fetched_via_proxy_(false),
  82       upload_content_set_(false),
  83       upload_range_offset_(0),
  84       upload_range_length_(0),
  85       referrer_policy_(
  86           URLRequest::CLEAR_REFERRER_ON_TRANSITION_FROM_SECURE_TO_INSECURE),
  87       is_chunked_upload_(false),
  88       was_cancelled_(false),
  89       stop_on_redirect_(false),
  90       stopped_on_redirect_(false),
  91       automatically_retry_on_5xx_(true),
  92       num_retries_on_5xx_(0),
  93       max_retries_on_5xx_(0),
  94       num_retries_on_network_changes_(0),
  95       max_retries_on_network_changes_(0),
  96       current_upload_bytes_(-1),
  97       current_response_bytes_(0),
  98       total_response_bytes_(-1) {
  99   CHECK(original_url_.is_valid());
 100 }
 101
 102 void URLFetcherCore::Start() {
 103   DCHECK(delegate_task_runner_.get());
 104   DCHECK(request_context_getter_.get()) << "We need an URLRequestContext!";
 105   if (network_task_runner_.get()) {
 106     DCHECK_EQ(network_task_runner_,
 107               request_context_getter_->GetNetworkTaskRunner());
 108   } else {
 109     network_task_runner_ = request_context_getter_->GetNetworkTaskRunner();
 110   }
 111   DCHECK(network_task_runner_.get()) << "We need an IO task runner";
 112
 113   network_task_runner_->PostTask(
 114       FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
 115 }
 116
 117 void URLFetcherCore::Stop() {
 118   if (delegate_task_runner_.get())  // May be NULL in tests.
 119     DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 120
 121   delegate_ = NULL;
 122   fetcher_ = NULL;
 123   if (!network_task_runner_.get())
 124     return;
 125   if (network_task_runner_->RunsTasksOnCurrentThread()) {
 126     CancelURLRequest(ERR_ABORTED);
 127   } else {
 128     network_task_runner_->PostTask(
 129         FROM_HERE,
 130         base::Bind(&URLFetcherCore::CancelURLRequest, this, ERR_ABORTED));
 131   }
 132 }
 133
 134 void URLFetcherCore::SetUploadData(const std::string& upload_content_type,
 135                                    const std::string& upload_content) {
 136   DCHECK(!is_chunked_upload_);
 137   DCHECK(!upload_content_set_);
 138   DCHECK(upload_content_.empty());
 139   DCHECK(upload_file_path_.empty());
 140   DCHECK(upload_content_type_.empty());
 141
 142   // Empty |upload_content_type| is allowed iff the |upload_content| is empty.
 143   DCHECK(upload_content.empty() || !upload_content_type.empty());
 144
 145   upload_content_type_ = upload_content_type;
 146   upload_content_ = upload_content;
 147   upload_content_set_ = true;
 148 }
 149
 150 void URLFetcherCore::SetUploadFilePath(
 151     const std::string& upload_content_type,
 152     const base::FilePath& file_path,
 153     uint64 range_offset,
 154     uint64 range_length,
 155     scoped_refptr<base::TaskRunner> file_task_runner) {
 156   DCHECK(!is_chunked_upload_);
 157   DCHECK(!upload_content_set_);
 158   DCHECK(upload_content_.empty());
 159   DCHECK(upload_file_path_.empty());
 160   DCHECK_EQ(upload_range_offset_, 0ULL);
 161   DCHECK_EQ(upload_range_length_, 0ULL);
 162   DCHECK(upload_content_type_.empty());
 163   DCHECK(!upload_content_type.empty());
 164
 165   upload_content_type_ = upload_content_type;
 166   upload_file_path_ = file_path;
 167   upload_range_offset_ = range_offset;
 168   upload_range_length_ = range_length;
 169   upload_file_task_runner_ = file_task_runner;
 170   upload_content_set_ = true;
 171 }
 172
 173 void URLFetcherCore::SetChunkedUpload(const std::string& content_type) {
 174   DCHECK(is_chunked_upload_ ||
 175          (upload_content_type_.empty() &&
 176           upload_content_.empty()));
 177
 178   // Empty |content_type| is not allowed here, because it is impossible
 179   // to ensure non-empty upload content as it is not yet supplied.
 180   DCHECK(!content_type.empty());
 181
 182   upload_content_type_ = content_type;
 183   upload_content_.clear();
 184   is_chunked_upload_ = true;
 185 }
 186
 187 void URLFetcherCore::AppendChunkToUpload(const std::string& content,
 188                                          bool is_last_chunk) {
 189   DCHECK(delegate_task_runner_.get());
 190   DCHECK(network_task_runner_.get());
 191   network_task_runner_->PostTask(
 192       FROM_HERE,
 193       base::Bind(&URLFetcherCore::CompleteAddingUploadDataChunk, this, content,
 194                  is_last_chunk));
 195 }
 196
 197 void URLFetcherCore::SetLoadFlags(int load_flags) {
 198   load_flags_ = load_flags;
 199 }
 200
 201 int URLFetcherCore::GetLoadFlags() const {
 202   return load_flags_;
 203 }
 204
 205 void URLFetcherCore::SetReferrer(const std::string& referrer) {
 206   referrer_ = referrer;
 207 }
 208
 209 void URLFetcherCore::SetReferrerPolicy(
 210     URLRequest::ReferrerPolicy referrer_policy) {
 211   referrer_policy_ = referrer_policy;
 212 }
 213
 214 void URLFetcherCore::SetExtraRequestHeaders(
 215     const std::string& extra_request_headers) {
 216   extra_request_headers_.Clear();
 217   extra_request_headers_.AddHeadersFromString(extra_request_headers);
 218 }
 219
 220 void URLFetcherCore::AddExtraRequestHeader(const std::string& header_line) {
 221   extra_request_headers_.AddHeaderFromString(header_line);
 222 }
 223
 224 void URLFetcherCore::SetRequestContext(
 225     URLRequestContextGetter* request_context_getter) {
 226   DCHECK(!request_context_getter_.get());
 227   DCHECK(request_context_getter);
 228   request_context_getter_ = request_context_getter;
 229 }
 230
 231 void URLFetcherCore::SetFirstPartyForCookies(
 232     const GURL& first_party_for_cookies) {
 233   DCHECK(first_party_for_cookies_.is_empty());
 234   first_party_for_cookies_ = first_party_for_cookies;
 235 }
 236
 237 void URLFetcherCore::SetURLRequestUserData(
 238     const void* key,
 239     const URLFetcher::CreateDataCallback& create_data_callback) {
 240   DCHECK(key);
 241   DCHECK(!create_data_callback.is_null());
 242   url_request_data_key_ = key;
 243   url_request_create_data_callback_ = create_data_callback;
 244 }
 245
 246 void URLFetcherCore::SetStopOnRedirect(bool stop_on_redirect) {
 247   stop_on_redirect_ = stop_on_redirect;
 248 }
 249
 250 void URLFetcherCore::SetAutomaticallyRetryOn5xx(bool retry) {
 251   automatically_retry_on_5xx_ = retry;
 252 }
 253
 254 void URLFetcherCore::SetMaxRetriesOn5xx(int max_retries) {
 255   max_retries_on_5xx_ = max_retries;
 256 }
 257
 258 int URLFetcherCore::GetMaxRetriesOn5xx() const {
 259   return max_retries_on_5xx_;
 260 }
 261
 262 base::TimeDelta URLFetcherCore::GetBackoffDelay() const {
 263   return backoff_delay_;
 264 }
 265
 266 void URLFetcherCore::SetAutomaticallyRetryOnNetworkChanges(int max_retries) {
 267   max_retries_on_network_changes_ = max_retries;
 268 }
 269
 270 void URLFetcherCore::SaveResponseToFileAtPath(
 271     const base::FilePath& file_path,
 272     scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
 273   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 274   SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
 275       new URLFetcherFileWriter(file_task_runner, file_path)));
 276 }
 277
 278 void URLFetcherCore::SaveResponseToTemporaryFile(
 279     scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
 280   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 281   SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
 282       new URLFetcherFileWriter(file_task_runner, base::FilePath())));
 283 }
 284
 285 void URLFetcherCore::SaveResponseWithWriter(
 286     scoped_ptr<URLFetcherResponseWriter> response_writer) {
 287   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 288   response_writer_ = response_writer.Pass();
 289 }
 290
 291 HttpResponseHeaders* URLFetcherCore::GetResponseHeaders() const {
 292   return response_headers_.get();
 293 }
 294
 295 // TODO(panayiotis): socket_address_ is written in the IO thread,
 296 // if this is accessed in the UI thread, this could result in a race.
 297 // Same for response_headers_ above and was_fetched_via_proxy_ below.
 298 HostPortPair URLFetcherCore::GetSocketAddress() const {
 299   return socket_address_;
 300 }
 301
 302 bool URLFetcherCore::WasFetchedViaProxy() const {
 303   return was_fetched_via_proxy_;
 304 }
 305
 306 const GURL& URLFetcherCore::GetOriginalURL() const {
 307   return original_url_;
 308 }
 309
 310 const GURL& URLFetcherCore::GetURL() const {
 311   return url_;
 312 }
 313
 314 const URLRequestStatus& URLFetcherCore::GetStatus() const {
 315   return status_;
 316 }
 317
 318 int URLFetcherCore::GetResponseCode() const {
 319   return response_code_;
 320 }
 321
 322 const ResponseCookies& URLFetcherCore::GetCookies() const {
 323   return cookies_;
 324 }
 325
 326 void URLFetcherCore::ReceivedContentWasMalformed() {
 327   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 328   if (network_task_runner_.get()) {
 329     network_task_runner_->PostTask(
 330         FROM_HERE, base::Bind(&URLFetcherCore::NotifyMalformedContent, this));
 331   }
 332 }
 333
 334 bool URLFetcherCore::GetResponseAsString(
 335     std::string* out_response_string) const {
 336   URLFetcherStringWriter* string_writer =
 337       response_writer_ ? response_writer_->AsStringWriter() : NULL;
 338   if (!string_writer)
 339     return false;
 340
 341   *out_response_string = string_writer->data();
 342   UMA_HISTOGRAM_MEMORY_KB("UrlFetcher.StringResponseSize",
 343                           (string_writer->data().length() / 1024));
 344   return true;
 345 }
 346
 347 bool URLFetcherCore::GetResponseAsFilePath(bool take_ownership,
 348                                            base::FilePath* out_response_path) {
 349   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 350
 351   URLFetcherFileWriter* file_writer =
 352       response_writer_ ? response_writer_->AsFileWriter() : NULL;
 353   if (!file_writer)
 354     return false;
 355
 356   *out_response_path = file_writer->file_path();
 357
 358   if (take_ownership) {
 359     // Intentionally calling a file_writer_ method directly without posting
 360     // the task to network_task_runner_.
 361     //
 362     // This is for correctly handling the case when file_writer_->DisownFile()
 363     // is soon followed by URLFetcherCore::Stop(). We have to make sure that
 364     // DisownFile takes effect before Stop deletes file_writer_.
 365     //
 366     // This direct call should be thread-safe, since DisownFile itself does no
 367     // file operation. It just flips the state to be referred in destruction.
 368     file_writer->DisownFile();
 369   }
 370   return true;
 371 }
 372
 373 void URLFetcherCore::OnReceivedRedirect(URLRequest* request,
 374                                         const GURL& new_url,
 375                                         bool* defer_redirect) {
 376   DCHECK_EQ(request, request_.get());
 377   DCHECK(network_task_runner_->BelongsToCurrentThread());
 378   if (stop_on_redirect_) {
 379     stopped_on_redirect_ = true;
 380     url_ = new_url;
 381     response_code_ = request_->GetResponseCode();
 382     was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
 383     request->Cancel();
 384     OnReadCompleted(request, 0);
 385   }
 386 }
 387
 388 void URLFetcherCore::OnResponseStarted(URLRequest* request) {
 389   DCHECK_EQ(request, request_.get());
 390   DCHECK(network_task_runner_->BelongsToCurrentThread());
 391   if (request_->status().is_success()) {
 392     response_code_ = request_->GetResponseCode();
 393     response_headers_ = request_->response_headers();
 394     socket_address_ = request_->GetSocketAddress();
 395     was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
 396     total_response_bytes_ = request_->GetExpectedContentSize();
 397   }
 398
 399   ReadResponse();
 400 }
 401
 402 void URLFetcherCore::OnCertificateRequested(
 403     URLRequest* request,
 404     SSLCertRequestInfo* cert_request_info) {
 405   DCHECK_EQ(request, request_.get());
 406   DCHECK(network_task_runner_->BelongsToCurrentThread());
 407
 408   if (g_ignore_certificate_requests) {
 409     request->ContinueWithCertificate(NULL);
 410   } else {
 411     request->Cancel();
 412   }
 413 }
 414
 415 void URLFetcherCore::OnReadCompleted(URLRequest* request,
 416                                      int bytes_read) {
 417   DCHECK(request == request_);
 418   DCHECK(network_task_runner_->BelongsToCurrentThread());
 419
 420   if (!stopped_on_redirect_)
 421     url_ = request->url();
 422   URLRequestThrottlerManager* throttler_manager =
 423       request->context()->throttler_manager();
 424   if (throttler_manager) {
 425     url_throttler_entry_ = throttler_manager->RegisterRequestUrl(url_);
 426   }
 427
 428   do {
 429     if (!request_->status().is_success() || bytes_read <= 0)
 430       break;
 431
 432     current_response_bytes_ += bytes_read;
 433     InformDelegateDownloadProgress();
 434
 435     const int result =
 436         WriteBuffer(new DrainableIOBuffer(buffer_.get(), bytes_read));
 437     if (result < 0) {
 438       // Write failed or waiting for write completion.
 439       return;
 440     }
 441   } while (request_->Read(buffer_.get(), kBufferSize, &bytes_read));
 442
 443   const URLRequestStatus status = request_->status();
 444
 445   if (status.is_success())
 446     request_->GetResponseCookies(&cookies_);
 447
 448   // See comments re: HEAD requests in ReadResponse().
 449   if (!status.is_io_pending() || request_type_ == URLFetcher::HEAD) {
 450     status_ = status;
 451     ReleaseRequest();
 452
 453     // No more data to write.
 454     const int result = response_writer_->Finish(
 455         base::Bind(&URLFetcherCore::DidFinishWriting, this));
 456     if (result != ERR_IO_PENDING)
 457       DidFinishWriting(result);
 458   }
 459 }
 460
 461 void URLFetcherCore::CancelAll() {
 462   g_registry.Get().CancelAll();
 463 }
 464
 465 int URLFetcherCore::GetNumFetcherCores() {
 466   return g_registry.Get().size();
 467 }
 468
 469 void URLFetcherCore::SetEnableInterceptionForTests(bool enabled) {
 470   g_interception_enabled = enabled;
 471 }
 472
 473 void URLFetcherCore::SetIgnoreCertificateRequests(bool ignored) {
 474   g_ignore_certificate_requests = ignored;
 475 }
 476
 477 URLFetcherCore::~URLFetcherCore() {
 478   // |request_| should be NULL.  If not, it's unsafe to delete it here since we
 479   // may not be on the IO thread.
 480   DCHECK(!request_.get());
 481 }
 482
 483 void URLFetcherCore::StartOnIOThread() {
 484   DCHECK(network_task_runner_->BelongsToCurrentThread());
 485
 486   if (!response_writer_)
 487     response_writer_.reset(new URLFetcherStringWriter);
 488
 489   const int result = response_writer_->Initialize(
 490       base::Bind(&URLFetcherCore::DidInitializeWriter, this));
 491   if (result != ERR_IO_PENDING)
 492     DidInitializeWriter(result);
 493 }
 494
 495 void URLFetcherCore::StartURLRequest() {
 496   DCHECK(network_task_runner_->BelongsToCurrentThread());
 497
 498   if (was_cancelled_) {
 499     // Since StartURLRequest() is posted as a *delayed* task, it may
 500     // run after the URLFetcher was already stopped.
 501     return;
 502   }
 503
 504   DCHECK(request_context_getter_.get());
 505   DCHECK(!request_.get());
 506
 507   g_registry.Get().AddURLFetcherCore(this);
 508   current_response_bytes_ = 0;
 509   request_ = request_context_getter_->GetURLRequestContext()->CreateRequest(
 510       original_url_, DEFAULT_PRIORITY, this, NULL);
 511   request_->set_stack_trace(stack_trace_);
 512   int flags = request_->load_flags() | load_flags_;
 513   if (!g_interception_enabled)
 514     flags = flags | LOAD_DISABLE_INTERCEPT;
 515
 516   if (is_chunked_upload_)
 517     request_->EnableChunkedUpload();
 518   request_->SetLoadFlags(flags);
 519   request_->SetReferrer(referrer_);
 520   request_->set_referrer_policy(referrer_policy_);
 521   request_->set_first_party_for_cookies(first_party_for_cookies_.is_empty() ?
 522       original_url_ : first_party_for_cookies_);
 523   if (url_request_data_key_ && !url_request_create_data_callback_.is_null()) {
 524     request_->SetUserData(url_request_data_key_,
 525                           url_request_create_data_callback_.Run());
 526   }
 527
 528   switch (request_type_) {
 529     case URLFetcher::GET:
 530       break;
 531
 532     case URLFetcher::POST:
 533     case URLFetcher::PUT:
 534     case URLFetcher::PATCH:
 535       // Upload content must be set.
 536       DCHECK(is_chunked_upload_ || upload_content_set_);
 537
 538       request_->set_method(
 539           request_type_ == URLFetcher::POST ? "POST" :
 540           request_type_ == URLFetcher::PUT ? "PUT" : "PATCH");
 541       if (!upload_content_type_.empty()) {
 542         extra_request_headers_.SetHeader(HttpRequestHeaders::kContentType,
 543                                          upload_content_type_);
 544       }
 545       if (!upload_content_.empty()) {
 546         scoped_ptr<UploadElementReader> reader(new UploadBytesElementReader(
 547             upload_content_.data(), upload_content_.size()));
 548         request_->set_upload(make_scoped_ptr(
 549             UploadDataStream::CreateWithReader(reader.Pass(), 0)));
 550       } else if (!upload_file_path_.empty()) {
 551         scoped_ptr<UploadElementReader> reader(
 552             new UploadFileElementReader(upload_file_task_runner_.get(),
 553                                         upload_file_path_,
 554                                         upload_range_offset_,
 555                                         upload_range_length_,
 556                                         base::Time()));
 557         request_->set_upload(make_scoped_ptr(
 558             UploadDataStream::CreateWithReader(reader.Pass(), 0)));
 559       }
 560
 561       current_upload_bytes_ = -1;
 562       // TODO(kinaba): http://crbug.com/118103. Implement upload callback in the
 563       //  layer and avoid using timer here.
 564       upload_progress_checker_timer_.reset(
 565           new base::RepeatingTimer<URLFetcherCore>());
 566       upload_progress_checker_timer_->Start(
 567           FROM_HERE,
 568           base::TimeDelta::FromMilliseconds(kUploadProgressTimerInterval),
 569           this,
 570           &URLFetcherCore::InformDelegateUploadProgress);
 571       break;
 572
 573     case URLFetcher::HEAD:
 574       request_->set_method("HEAD");
 575       break;
 576
 577     case URLFetcher::DELETE_REQUEST:
 578       request_->set_method("DELETE");
 579       break;
 580
 581     default:
 582       NOTREACHED();
 583   }
 584
 585   if (!extra_request_headers_.IsEmpty())
 586     request_->SetExtraRequestHeaders(extra_request_headers_);
 587
 588   request_->Start();
 589 }
 590
 591 void URLFetcherCore::DidInitializeWriter(int result) {
 592   if (result != OK) {
 593     CancelURLRequest(result);
 594     delegate_task_runner_->PostTask(
 595         FROM_HERE,
 596         base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
 597     return;
 598   }
 599   StartURLRequestWhenAppropriate();
 600 }
 601
 602 void URLFetcherCore::StartURLRequestWhenAppropriate() {
 603   DCHECK(network_task_runner_->BelongsToCurrentThread());
 604
 605   if (was_cancelled_)
 606     return;
 607
 608   DCHECK(request_context_getter_.get());
 609
 610   int64 delay = 0LL;
 611   if (original_url_throttler_entry_.get() == NULL) {
 612     URLRequestThrottlerManager* manager =
 613         request_context_getter_->GetURLRequestContext()->throttler_manager();
 614     if (manager) {
 615       original_url_throttler_entry_ =
 616           manager->RegisterRequestUrl(original_url_);
 617     }
 618   }
 619   if (original_url_throttler_entry_.get() != NULL) {
 620     delay = original_url_throttler_entry_->ReserveSendingTimeForNextRequest(
 621         GetBackoffReleaseTime());
 622   }
 623
 624   if (delay == 0) {
 625     StartURLRequest();
 626   } else {
 627     base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
 628         FROM_HERE, base::Bind(&URLFetcherCore::StartURLRequest, this),
 629         base::TimeDelta::FromMilliseconds(delay));
 630   }
 631 }
 632
 633 void URLFetcherCore::CancelURLRequest(int error) {
 634   DCHECK(network_task_runner_->BelongsToCurrentThread());
 635
 636   if (request_.get()) {
 637     request_->CancelWithError(error);
 638     ReleaseRequest();
 639   }
 640
 641   // Set the error manually.
 642   // Normally, calling URLRequest::CancelWithError() results in calling
 643   // OnReadCompleted() with bytes_read = -1 via an asynchronous task posted by
 644   // URLRequestJob::NotifyDone(). But, because the request was released
 645   // immediately after being canceled, the request could not call
 646   // OnReadCompleted() which overwrites |status_| with the error status.
 647   status_.set_status(URLRequestStatus::CANCELED);
 648   status_.set_error(error);
 649
 650   // Release the reference to the request context. There could be multiple
 651   // references to URLFetcher::Core at this point so it may take a while to
 652   // delete the object, but we cannot delay the destruction of the request
 653   // context.
 654   request_context_getter_ = NULL;
 655   first_party_for_cookies_ = GURL();
 656   url_request_data_key_ = NULL;
 657   url_request_create_data_callback_.Reset();
 658   was_cancelled_ = true;
 659 }
 660
 661 void URLFetcherCore::OnCompletedURLRequest(
 662     base::TimeDelta backoff_delay) {
 663   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 664
 665   // Save the status and backoff_delay so that delegates can read it.
 666   if (delegate_) {
 667     backoff_delay_ = backoff_delay;
 668     InformDelegateFetchIsComplete();
 669   }
 670 }
 671
 672 void URLFetcherCore::InformDelegateFetchIsComplete() {
 673   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 674   if (delegate_)
 675     delegate_->OnURLFetchComplete(fetcher_);
 676 }
 677
 678 void URLFetcherCore::NotifyMalformedContent() {
 679   DCHECK(network_task_runner_->BelongsToCurrentThread());
 680   if (url_throttler_entry_.get() != NULL) {
 681     int status_code = response_code_;
 682     if (status_code == URLFetcher::RESPONSE_CODE_INVALID) {
 683       // The status code will generally be known by the time clients
 684       // call the |ReceivedContentWasMalformed()| function (which ends up
 685       // calling the current function) but if it's not, we need to assume
 686       // the response was successful so that the total failure count
 687       // used to calculate exponential back-off goes up.
 688       status_code = 200;
 689     }
 690     url_throttler_entry_->ReceivedContentWasMalformed(status_code);
 691   }
 692 }
 693
 694 void URLFetcherCore::DidFinishWriting(int result) {
 695   if (result != OK) {
 696     CancelURLRequest(result);
 697     delegate_task_runner_->PostTask(
 698         FROM_HERE,
 699         base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
 700     return;
 701   }
 702   // If the file was successfully closed, then the URL request is complete.
 703   RetryOrCompleteUrlFetch();
 704 }
 705
 706 void URLFetcherCore::RetryOrCompleteUrlFetch() {
 707   DCHECK(network_task_runner_->BelongsToCurrentThread());
 708   base::TimeDelta backoff_delay;
 709
 710   // Checks the response from server.
 711   if (response_code_ >= 500 ||
 712       status_.error() == ERR_TEMPORARILY_THROTTLED) {
 713     // When encountering a server error, we will send the request again
 714     // after backoff time.
 715     ++num_retries_on_5xx_;
 716
 717     // Note that backoff_delay may be 0 because (a) the
 718     // URLRequestThrottlerManager and related code does not
 719     // necessarily back off on the first error, (b) it only backs off
 720     // on some of the 5xx status codes, (c) not all URLRequestContexts
 721     // have a throttler manager.
 722     base::TimeTicks backoff_release_time = GetBackoffReleaseTime();
 723     backoff_delay = backoff_release_time - base::TimeTicks::Now();
 724     if (backoff_delay < base::TimeDelta())
 725       backoff_delay = base::TimeDelta();
 726
 727     if (automatically_retry_on_5xx_ &&
 728         num_retries_on_5xx_ <= max_retries_on_5xx_) {
 729       StartOnIOThread();
 730       return;
 731     }
 732   } else {
 733     backoff_delay = base::TimeDelta();
 734   }
 735
 736   // Retry if the request failed due to network changes.
 737   if (status_.error() == ERR_NETWORK_CHANGED &&
 738       num_retries_on_network_changes_ < max_retries_on_network_changes_) {
 739     ++num_retries_on_network_changes_;
 740
 741     // Retry soon, after flushing all the current tasks which may include
 742     // further network change observers.
 743     network_task_runner_->PostTask(
 744         FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
 745     return;
 746   }
 747
 748   request_context_getter_ = NULL;
 749   first_party_for_cookies_ = GURL();
 750   url_request_data_key_ = NULL;
 751   url_request_create_data_callback_.Reset();
 752   bool posted = delegate_task_runner_->PostTask(
 753       FROM_HERE,
 754       base::Bind(&URLFetcherCore::OnCompletedURLRequest, this, backoff_delay));
 755
 756   // If the delegate message loop does not exist any more, then the delegate
 757   // should be gone too.
 758   DCHECK(posted || !delegate_);
 759 }
 760
 761 void URLFetcherCore::ReleaseRequest() {
 762   upload_progress_checker_timer_.reset();
 763   request_.reset();
 764   g_registry.Get().RemoveURLFetcherCore(this);
 765 }
 766
 767 base::TimeTicks URLFetcherCore::GetBackoffReleaseTime() {
 768   DCHECK(network_task_runner_->BelongsToCurrentThread());
 769
 770   if (original_url_throttler_entry_.get()) {
 771     base::TimeTicks original_url_backoff =
 772         original_url_throttler_entry_->GetExponentialBackoffReleaseTime();
 773     base::TimeTicks destination_url_backoff;
 774     if (url_throttler_entry_.get() != NULL &&
 775         original_url_throttler_entry_.get() != url_throttler_entry_.get()) {
 776       destination_url_backoff =
 777           url_throttler_entry_->GetExponentialBackoffReleaseTime();
 778     }
 779
 780     return original_url_backoff > destination_url_backoff ?
 781         original_url_backoff : destination_url_backoff;
 782   } else {
 783     return base::TimeTicks();
 784   }
 785 }
 786
 787 void URLFetcherCore::CompleteAddingUploadDataChunk(
 788     const std::string& content, bool is_last_chunk) {
 789   if (was_cancelled_) {
 790     // Since CompleteAddingUploadDataChunk() is posted as a *delayed* task, it
 791     // may run after the URLFetcher was already stopped.
 792     return;
 793   }
 794   DCHECK(is_chunked_upload_);
 795   DCHECK(request_.get());
 796   DCHECK(!content.empty());
 797   request_->AppendChunkToUpload(content.data(),
 798                                 static_cast<int>(content.length()),
 799                                 is_last_chunk);
 800 }
 801
 802 int URLFetcherCore::WriteBuffer(scoped_refptr<DrainableIOBuffer> data) {
 803   while (data->BytesRemaining() > 0) {
 804     const int result = response_writer_->Write(
 805         data.get(),
 806         data->BytesRemaining(),
 807         base::Bind(&URLFetcherCore::DidWriteBuffer, this, data));
 808     if (result < 0) {
 809       if (result != ERR_IO_PENDING)
 810         DidWriteBuffer(data, result);
 811       return result;
 812     }
 813     data->DidConsume(result);
 814   }
 815   return OK;
 816 }
 817
 818 void URLFetcherCore::DidWriteBuffer(scoped_refptr<DrainableIOBuffer> data,
 819                                     int result) {
 820   if (result < 0) {  // Handle errors.
 821     CancelURLRequest(result);
 822     response_writer_->Finish(base::Bind(&EmptyCompletionCallback));
 823     delegate_task_runner_->PostTask(
 824         FROM_HERE,
 825         base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
 826     return;
 827   }
 828
 829   // Continue writing.
 830   data->DidConsume(result);
 831   if (WriteBuffer(data) < 0)
 832     return;
 833
 834   // Finished writing buffer_. Read some more, unless the request has been
 835   // cancelled and deleted.
 836   DCHECK_EQ(0, data->BytesRemaining());
 837   if (request_.get())
 838     ReadResponse();
 839 }
 840
 841 void URLFetcherCore::ReadResponse() {
 842   // Some servers may treat HEAD requests as GET requests.  To free up the
 843   // network connection as soon as possible, signal that the request has
 844   // completed immediately, without trying to read any data back (all we care
 845   // about is the response code and headers, which we already have).
 846   int bytes_read = 0;
 847   if (request_->status().is_success() &&
 848       (request_type_ != URLFetcher::HEAD))
 849     request_->Read(buffer_.get(), kBufferSize, &bytes_read);
 850   OnReadCompleted(request_.get(), bytes_read);
 851 }
 852
 853 void URLFetcherCore::InformDelegateUploadProgress() {
 854   DCHECK(network_task_runner_->BelongsToCurrentThread());
 855   if (request_.get()) {
 856     int64 current = request_->GetUploadProgress().position();
 857     if (current_upload_bytes_ != current) {
 858       current_upload_bytes_ = current;
 859       int64 total = -1;
 860       if (!is_chunked_upload_) {
 861         total = static_cast<int64>(request_->GetUploadProgress().size());
 862         // Total may be zero if the UploadDataStream::Init has not been called
 863         // yet.  Don't send the upload progress until the size is initialized.
 864         if (!total)
 865           return;
 866       }
 867       delegate_task_runner_->PostTask(
 868           FROM_HERE,
 869           base::Bind(
 870               &URLFetcherCore::InformDelegateUploadProgressInDelegateThread,
 871               this, current, total));
 872     }
 873   }
 874 }
 875
 876 void URLFetcherCore::InformDelegateUploadProgressInDelegateThread(
 877     int64 current, int64 total) {
 878   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 879   if (delegate_)
 880     delegate_->OnURLFetchUploadProgress(fetcher_, current, total);
 881 }
 882
 883 void URLFetcherCore::InformDelegateDownloadProgress() {
 884   DCHECK(network_task_runner_->BelongsToCurrentThread());
 885   delegate_task_runner_->PostTask(
 886       FROM_HERE,
 887       base::Bind(
 888           &URLFetcherCore::InformDelegateDownloadProgressInDelegateThread,
 889           this, current_response_bytes_, total_response_bytes_));
 890 }
 891
 892 void URLFetcherCore::InformDelegateDownloadProgressInDelegateThread(
 893     int64 current, int64 total) {
 894   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 895   if (delegate_)
 896     delegate_->OnURLFetchDownloadProgress(fetcher_, current, total);
 897 }
 898
 899 }  // namespace net