net/url_request/url_request.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_request.h"
   6
   7 #include "base/bind.h"
   8 #include "base/bind_helpers.h"
   9 #include "base/callback.h"
  10 #include "base/compiler_specific.h"
  11 #include "base/debug/stack_trace.h"
  12 #include "base/lazy_instance.h"
  13 #include "base/memory/singleton.h"
  14 #include "base/message_loop/message_loop.h"
  15 #include "base/profiler/scoped_tracker.h"
  16 #include "base/stl_util.h"
  17 #include "base/strings/utf_string_conversions.h"
  18 #include "base/synchronization/lock.h"
  19 #include "base/values.h"
  20 #include "net/base/auth.h"
  21 #include "net/base/chunked_upload_data_stream.h"
  22 #include "net/base/host_port_pair.h"
  23 #include "net/base/load_flags.h"
  24 #include "net/base/load_timing_info.h"
  25 #include "net/base/net_errors.h"
  26 #include "net/base/network_change_notifier.h"
  27 #include "net/base/network_delegate.h"
  28 #include "net/base/upload_data_stream.h"
  29 #include "net/http/http_response_headers.h"
  30 #include "net/http/http_util.h"
  31 #include "net/log/net_log.h"
  32 #include "net/ssl/ssl_cert_request_info.h"
  33 #include "net/url_request/redirect_info.h"
  34 #include "net/url_request/url_request_context.h"
  35 #include "net/url_request/url_request_error_job.h"
  36 #include "net/url_request/url_request_job.h"
  37 #include "net/url_request/url_request_job_manager.h"
  38 #include "net/url_request/url_request_netlog_params.h"
  39 #include "net/url_request/url_request_redirect_job.h"
  40 #include "url/gurl.h"
  41 #include "url/origin.h"
  42
  43 using base::Time;
  44 using std::string;
  45
  46 namespace net {
  47
  48 namespace {
  49
  50 // Max number of http redirects to follow.  Same number as gecko.
  51 const int kMaxRedirects = 20;
  52
  53 // Discard headers which have meaning in POST (Content-Length, Content-Type,
  54 // Origin).
  55 void StripPostSpecificHeaders(HttpRequestHeaders* headers) {
  56   // These are headers that may be attached to a POST.
  57   headers->RemoveHeader(HttpRequestHeaders::kContentLength);
  58   headers->RemoveHeader(HttpRequestHeaders::kContentType);
  59   // TODO(jww): This is Origin header removal is probably layering violation and
  60   // should be refactored into //content. See https://crbug.com/471397.
  61   headers->RemoveHeader(HttpRequestHeaders::kOrigin);
  62 }
  63
  64 // TODO(battre): Delete this, see http://crbug.com/89321:
  65 // This counter keeps track of the identifiers used for URL requests so far.
  66 // 0 is reserved to represent an invalid ID.
  67 uint64 g_next_url_request_identifier = 1;
  68
  69 // This lock protects g_next_url_request_identifier.
  70 base::LazyInstance<base::Lock>::Leaky
  71     g_next_url_request_identifier_lock = LAZY_INSTANCE_INITIALIZER;
  72
  73 // Returns an prior unused identifier for URL requests.
  74 uint64 GenerateURLRequestIdentifier() {
  75   base::AutoLock lock(g_next_url_request_identifier_lock.Get());
  76   return g_next_url_request_identifier++;
  77 }
  78
  79 // True once the first URLRequest was started.
  80 bool g_url_requests_started = false;
  81
  82 // True if cookies are accepted by default.
  83 bool g_default_can_use_cookies = true;
  84
  85 // When the URLRequest first assempts load timing information, it has the times
  86 // at which each event occurred.  The API requires the time which the request
  87 // was blocked on each phase.  This function handles the conversion.
  88 //
  89 // In the case of reusing a SPDY session, old proxy results may have been
  90 // reused, so proxy resolution times may be before the request was started.
  91 //
  92 // Due to preconnect and late binding, it is also possible for the connection
  93 // attempt to start before a request has been started, or proxy resolution
  94 // completed.
  95 //
  96 // This functions fixes both those cases.
  97 void ConvertRealLoadTimesToBlockingTimes(LoadTimingInfo* load_timing_info) {
  98   DCHECK(!load_timing_info->request_start.is_null());
  99
 100   // Earliest time possible for the request to be blocking on connect events.
 101   base::TimeTicks block_on_connect = load_timing_info->request_start;
 102
 103   if (!load_timing_info->proxy_resolve_start.is_null()) {
 104     DCHECK(!load_timing_info->proxy_resolve_end.is_null());
 105
 106     // Make sure the proxy times are after request start.
 107     if (load_timing_info->proxy_resolve_start < load_timing_info->request_start)
 108       load_timing_info->proxy_resolve_start = load_timing_info->request_start;
 109     if (load_timing_info->proxy_resolve_end < load_timing_info->request_start)
 110       load_timing_info->proxy_resolve_end = load_timing_info->request_start;
 111
 112     // Connect times must also be after the proxy times.
 113     block_on_connect = load_timing_info->proxy_resolve_end;
 114   }
 115
 116   // Make sure connection times are after start and proxy times.
 117
 118   LoadTimingInfo::ConnectTiming* connect_timing =
 119       &load_timing_info->connect_timing;
 120   if (!connect_timing->dns_start.is_null()) {
 121     DCHECK(!connect_timing->dns_end.is_null());
 122     if (connect_timing->dns_start < block_on_connect)
 123       connect_timing->dns_start = block_on_connect;
 124     if (connect_timing->dns_end < block_on_connect)
 125       connect_timing->dns_end = block_on_connect;
 126   }
 127
 128   if (!connect_timing->connect_start.is_null()) {
 129     DCHECK(!connect_timing->connect_end.is_null());
 130     if (connect_timing->connect_start < block_on_connect)
 131       connect_timing->connect_start = block_on_connect;
 132     if (connect_timing->connect_end < block_on_connect)
 133       connect_timing->connect_end = block_on_connect;
 134   }
 135
 136   if (!connect_timing->ssl_start.is_null()) {
 137     DCHECK(!connect_timing->ssl_end.is_null());
 138     if (connect_timing->ssl_start < block_on_connect)
 139       connect_timing->ssl_start = block_on_connect;
 140     if (connect_timing->ssl_end < block_on_connect)
 141       connect_timing->ssl_end = block_on_connect;
 142   }
 143 }
 144
 145 }  // namespace
 146
 147 ///////////////////////////////////////////////////////////////////////////////
 148 // URLRequest::Delegate
 149
 150 void URLRequest::Delegate::OnReceivedRedirect(URLRequest* request,
 151                                               const RedirectInfo& redirect_info,
 152                                               bool* defer_redirect) {
 153 }
 154
 155 void URLRequest::Delegate::OnAuthRequired(URLRequest* request,
 156                                           AuthChallengeInfo* auth_info) {
 157   request->CancelAuth();
 158 }
 159
 160 void URLRequest::Delegate::OnCertificateRequested(
 161     URLRequest* request,
 162     SSLCertRequestInfo* cert_request_info) {
 163   request->CancelWithError(ERR_SSL_CLIENT_AUTH_CERT_NEEDED);
 164 }
 165
 166 void URLRequest::Delegate::OnSSLCertificateError(URLRequest* request,
 167                                                  const SSLInfo& ssl_info,
 168                                                  bool is_hsts_ok) {
 169   request->Cancel();
 170 }
 171
 172 void URLRequest::Delegate::OnBeforeNetworkStart(URLRequest* request,
 173                                                 bool* defer) {
 174 }
 175
 176 ///////////////////////////////////////////////////////////////////////////////
 177 // URLRequest
 178
 179 URLRequest::~URLRequest() {
 180   Cancel();
 181
 182   if (network_delegate_) {
 183     network_delegate_->NotifyURLRequestDestroyed(this);
 184     if (job_.get())
 185       job_->NotifyURLRequestDestroyed();
 186   }
 187
 188   if (job_.get())
 189     OrphanJob();
 190
 191   int deleted = context_->url_requests()->erase(this);
 192   CHECK_EQ(1, deleted);
 193
 194   int net_error = OK;
 195   // Log error only on failure, not cancellation, as even successful requests
 196   // are "cancelled" on destruction.
 197   if (status_.status() == URLRequestStatus::FAILED)
 198     net_error = status_.error();
 199   net_log_.EndEventWithNetErrorCode(NetLog::TYPE_REQUEST_ALIVE, net_error);
 200 }
 201
 202 void URLRequest::EnableChunkedUpload() {
 203   DCHECK(!upload_data_stream_ || upload_data_stream_->is_chunked());
 204   if (!upload_data_stream_) {
 205     upload_chunked_data_stream_ = new ChunkedUploadDataStream(0);
 206     upload_data_stream_.reset(upload_chunked_data_stream_);
 207   }
 208 }
 209
 210 void URLRequest::AppendChunkToUpload(const char* bytes,
 211                                      int bytes_len,
 212                                      bool is_last_chunk) {
 213   DCHECK(upload_data_stream_);
 214   DCHECK(upload_data_stream_->is_chunked());
 215   upload_chunked_data_stream_->AppendData(bytes, bytes_len, is_last_chunk);
 216 }
 217
 218 void URLRequest::set_upload(scoped_ptr<UploadDataStream> upload) {
 219   upload_data_stream_ = upload.Pass();
 220 }
 221
 222 const UploadDataStream* URLRequest::get_upload() const {
 223   return upload_data_stream_.get();
 224 }
 225
 226 bool URLRequest::has_upload() const {
 227   return upload_data_stream_.get() != NULL;
 228 }
 229
 230 void URLRequest::SetExtraRequestHeaderByName(const string& name,
 231                                              const string& value,
 232                                              bool overwrite) {
 233   DCHECK(!is_pending_ || is_redirecting_);
 234   if (overwrite) {
 235     extra_request_headers_.SetHeader(name, value);
 236   } else {
 237     extra_request_headers_.SetHeaderIfMissing(name, value);
 238   }
 239 }
 240
 241 void URLRequest::RemoveRequestHeaderByName(const string& name) {
 242   DCHECK(!is_pending_ || is_redirecting_);
 243   extra_request_headers_.RemoveHeader(name);
 244 }
 245
 246 void URLRequest::SetExtraRequestHeaders(
 247     const HttpRequestHeaders& headers) {
 248   DCHECK(!is_pending_);
 249   extra_request_headers_ = headers;
 250
 251   // NOTE: This method will likely become non-trivial once the other setters
 252   // for request headers are implemented.
 253 }
 254
 255 bool URLRequest::GetFullRequestHeaders(HttpRequestHeaders* headers) const {
 256   if (!job_.get())
 257     return false;
 258
 259   return job_->GetFullRequestHeaders(headers);
 260 }
 261
 262 int64 URLRequest::GetTotalReceivedBytes() const {
 263   if (!job_.get())
 264     return 0;
 265
 266   return job_->GetTotalReceivedBytes();
 267 }
 268
 269 LoadStateWithParam URLRequest::GetLoadState() const {
 270   // The !blocked_by_.empty() check allows |this| to report it's blocked on a
 271   // delegate before it has been started.
 272   if (calling_delegate_ || !blocked_by_.empty()) {
 273     return LoadStateWithParam(
 274         LOAD_STATE_WAITING_FOR_DELEGATE,
 275         use_blocked_by_as_load_param_ ? base::UTF8ToUTF16(blocked_by_) :
 276                                         base::string16());
 277   }
 278   return LoadStateWithParam(job_.get() ? job_->GetLoadState() : LOAD_STATE_IDLE,
 279                             base::string16());
 280 }
 281
 282 scoped_ptr<base::Value> URLRequest::GetStateAsValue() const {
 283   scoped_ptr<base::DictionaryValue> dict(new base::DictionaryValue());
 284   dict->SetString("url", original_url().possibly_invalid_spec());
 285
 286   if (url_chain_.size() > 1) {
 287     scoped_ptr<base::ListValue> list(new base::ListValue());
 288     for (const GURL& url : url_chain_) {
 289       list->AppendString(url.possibly_invalid_spec());
 290     }
 291     dict->Set("url_chain", list.Pass());
 292   }
 293
 294   dict->SetInteger("load_flags", load_flags_);
 295
 296   LoadStateWithParam load_state = GetLoadState();
 297   dict->SetInteger("load_state", load_state.state);
 298   if (!load_state.param.empty())
 299     dict->SetString("load_state_param", load_state.param);
 300   if (!blocked_by_.empty())
 301     dict->SetString("delegate_info", blocked_by_);
 302
 303   dict->SetString("method", method_);
 304   dict->SetBoolean("has_upload", has_upload());
 305   dict->SetBoolean("is_pending", is_pending_);
 306
 307   // Add the status of the request.  The status should always be IO_PENDING, and
 308   // the error should always be OK, unless something is holding onto a request
 309   // that has finished or a request was leaked.  Neither of these should happen.
 310   switch (status_.status()) {
 311     case URLRequestStatus::SUCCESS:
 312       dict->SetString("status", "SUCCESS");
 313       break;
 314     case URLRequestStatus::IO_PENDING:
 315       dict->SetString("status", "IO_PENDING");
 316       break;
 317     case URLRequestStatus::CANCELED:
 318       dict->SetString("status", "CANCELED");
 319       break;
 320     case URLRequestStatus::FAILED:
 321       dict->SetString("status", "FAILED");
 322       break;
 323   }
 324   if (status_.error() != OK)
 325     dict->SetInteger("net_error", status_.error());
 326   return dict.Pass();
 327 }
 328
 329 void URLRequest::LogBlockedBy(const char* blocked_by) {
 330   DCHECK(blocked_by);
 331   DCHECK_GT(strlen(blocked_by), 0u);
 332
 333   // Only log information to NetLog during startup and certain deferring calls
 334   // to delegates.  For all reads but the first, do nothing.
 335   if (!calling_delegate_ && !response_info_.request_time.is_null())
 336     return;
 337
 338   LogUnblocked();
 339   blocked_by_ = blocked_by;
 340   use_blocked_by_as_load_param_ = false;
 341
 342   net_log_.BeginEvent(
 343       NetLog::TYPE_DELEGATE_INFO,
 344       NetLog::StringCallback("delegate_info", &blocked_by_));
 345 }
 346
 347 void URLRequest::LogAndReportBlockedBy(const char* source) {
 348   LogBlockedBy(source);
 349   use_blocked_by_as_load_param_ = true;
 350 }
 351
 352 void URLRequest::LogUnblocked() {
 353   if (blocked_by_.empty())
 354     return;
 355
 356   net_log_.EndEvent(NetLog::TYPE_DELEGATE_INFO);
 357   blocked_by_.clear();
 358 }
 359
 360 UploadProgress URLRequest::GetUploadProgress() const {
 361   if (!job_.get()) {
 362     // We haven't started or the request was cancelled
 363     return UploadProgress();
 364   }
 365   if (final_upload_progress_.position()) {
 366     // The first job completed and none of the subsequent series of
 367     // GETs when following redirects will upload anything, so we return the
 368     // cached results from the initial job, the POST.
 369     return final_upload_progress_;
 370   }
 371   return job_->GetUploadProgress();
 372 }
 373
 374 void URLRequest::GetResponseHeaderByName(const string& name, string* value) {
 375   DCHECK(value);
 376   if (response_info_.headers.get()) {
 377     response_info_.headers->GetNormalizedHeader(name, value);
 378   } else {
 379     value->clear();
 380   }
 381 }
 382
 383 HostPortPair URLRequest::GetSocketAddress() const {
 384   DCHECK(job_.get());
 385   return job_->GetSocketAddress();
 386 }
 387
 388 HttpResponseHeaders* URLRequest::response_headers() const {
 389   return response_info_.headers.get();
 390 }
 391
 392 void URLRequest::GetLoadTimingInfo(LoadTimingInfo* load_timing_info) const {
 393   *load_timing_info = load_timing_info_;
 394 }
 395
 396 bool URLRequest::GetResponseCookies(ResponseCookies* cookies) {
 397   DCHECK(job_.get());
 398   return job_->GetResponseCookies(cookies);
 399 }
 400
 401 void URLRequest::GetMimeType(string* mime_type) const {
 402   DCHECK(job_.get());
 403   job_->GetMimeType(mime_type);
 404 }
 405
 406 void URLRequest::GetCharset(string* charset) const {
 407   DCHECK(job_.get());
 408   job_->GetCharset(charset);
 409 }
 410
 411 int URLRequest::GetResponseCode() const {
 412   DCHECK(job_.get());
 413   return job_->GetResponseCode();
 414 }
 415
 416 void URLRequest::SetLoadFlags(int flags) {
 417   if ((load_flags_ & LOAD_IGNORE_LIMITS) != (flags & LOAD_IGNORE_LIMITS)) {
 418     DCHECK(!job_.get());
 419     DCHECK(flags & LOAD_IGNORE_LIMITS);
 420     DCHECK_EQ(priority_, MAXIMUM_PRIORITY);
 421   }
 422   load_flags_ = flags;
 423
 424   // This should be a no-op given the above DCHECKs, but do this
 425   // anyway for release mode.
 426   if ((load_flags_ & LOAD_IGNORE_LIMITS) != 0)
 427     SetPriority(MAXIMUM_PRIORITY);
 428 }
 429
 430 // static
 431 void URLRequest::SetDefaultCookiePolicyToBlock() {
 432   CHECK(!g_url_requests_started);
 433   g_default_can_use_cookies = false;
 434 }
 435
 436 // static
 437 bool URLRequest::IsHandledProtocol(const std::string& scheme) {
 438   return URLRequestJobManager::SupportsScheme(scheme);
 439 }
 440
 441 // static
 442 bool URLRequest::IsHandledURL(const GURL& url) {
 443   if (!url.is_valid()) {
 444     // We handle error cases.
 445     return true;
 446   }
 447
 448   return IsHandledProtocol(url.scheme());
 449 }
 450
 451 void URLRequest::set_first_party_for_cookies(
 452     const GURL& first_party_for_cookies) {
 453   DCHECK(!is_pending_);
 454   first_party_for_cookies_ = first_party_for_cookies;
 455 }
 456
 457 void URLRequest::set_first_party_url_policy(
 458     FirstPartyURLPolicy first_party_url_policy) {
 459   DCHECK(!is_pending_);
 460   first_party_url_policy_ = first_party_url_policy;
 461 }
 462
 463 void URLRequest::set_method(const std::string& method) {
 464   DCHECK(!is_pending_);
 465   method_ = method;
 466 }
 467
 468 void URLRequest::SetReferrer(const std::string& referrer) {
 469   DCHECK(!is_pending_);
 470   GURL referrer_url(referrer);
 471   if (referrer_url.is_valid()) {
 472     referrer_ = referrer_url.GetAsReferrer().spec();
 473   } else {
 474     referrer_ = referrer;
 475   }
 476 }
 477
 478 void URLRequest::set_referrer_policy(ReferrerPolicy referrer_policy) {
 479   DCHECK(!is_pending_);
 480   referrer_policy_ = referrer_policy;
 481 }
 482
 483 void URLRequest::set_delegate(Delegate* delegate) {
 484   delegate_ = delegate;
 485 }
 486
 487 void URLRequest::Start() {
 488   // TODO(pkasting): Remove ScopedTracker below once crbug.com/456327 is fixed.
 489   tracked_objects::ScopedTracker tracking_profile(
 490       FROM_HERE_WITH_EXPLICIT_FUNCTION("456327 URLRequest::Start"));
 491
 492   // Some values can be NULL, but the job factory must not be.
 493   DCHECK(context_->job_factory());
 494
 495   // Anything that sets |blocked_by_| before start should have cleaned up after
 496   // itself.
 497   DCHECK(blocked_by_.empty());
 498
 499   g_url_requests_started = true;
 500   response_info_.request_time = base::Time::Now();
 501
 502   load_timing_info_ = LoadTimingInfo();
 503   load_timing_info_.request_start_time = response_info_.request_time;
 504   load_timing_info_.request_start = base::TimeTicks::Now();
 505
 506   // Only notify the delegate for the initial request.
 507   if (network_delegate_) {
 508     // TODO(mmenke): Remove ScopedTracker below once crbug.com/456327 is fixed.
 509     tracked_objects::ScopedTracker tracking_profile25(
 510         FROM_HERE_WITH_EXPLICIT_FUNCTION("456327 URLRequest::Start 2.5"));
 511
 512     OnCallToDelegate();
 513     int error = network_delegate_->NotifyBeforeURLRequest(
 514         this, before_request_callback_, &delegate_redirect_url_);
 515     // If ERR_IO_PENDING is returned, the delegate will invoke
 516     // |before_request_callback_| later.
 517     if (error != ERR_IO_PENDING)
 518       BeforeRequestComplete(error);
 519     return;
 520   }
 521
 522   // TODO(mmenke): Remove ScopedTracker below once crbug.com/456327 is fixed.
 523   tracked_objects::ScopedTracker tracking_profile2(
 524       FROM_HERE_WITH_EXPLICIT_FUNCTION("456327 URLRequest::Start 2"));
 525
 526   StartJob(URLRequestJobManager::GetInstance()->CreateJob(
 527       this, network_delegate_));
 528 }
 529
 530 ///////////////////////////////////////////////////////////////////////////////
 531
 532 URLRequest::URLRequest(const GURL& url,
 533                        RequestPriority priority,
 534                        Delegate* delegate,
 535                        const URLRequestContext* context,
 536                        NetworkDelegate* network_delegate)
 537     : context_(context),
 538       network_delegate_(network_delegate ? network_delegate
 539                                          : context->network_delegate()),
 540       net_log_(
 541           BoundNetLog::Make(context->net_log(), NetLog::SOURCE_URL_REQUEST)),
 542       url_chain_(1, url),
 543       method_("GET"),
 544       referrer_policy_(CLEAR_REFERRER_ON_TRANSITION_FROM_SECURE_TO_INSECURE),
 545       first_party_url_policy_(NEVER_CHANGE_FIRST_PARTY_URL),
 546       load_flags_(LOAD_NORMAL),
 547       delegate_(delegate),
 548       is_pending_(false),
 549       is_redirecting_(false),
 550       redirect_limit_(kMaxRedirects),
 551       priority_(priority),
 552       identifier_(GenerateURLRequestIdentifier()),
 553       calling_delegate_(false),
 554       use_blocked_by_as_load_param_(false),
 555       before_request_callback_(base::Bind(&URLRequest::BeforeRequestComplete,
 556                                           base::Unretained(this))),
 557       has_notified_completion_(false),
 558       received_response_content_length_(0),
 559       creation_time_(base::TimeTicks::Now()),
 560       notified_before_network_start_(false) {
 561   // Sanity check out environment.
 562   DCHECK(base::MessageLoop::current())
 563       << "The current base::MessageLoop must exist";
 564
 565   context->url_requests()->insert(this);
 566   net_log_.BeginEvent(NetLog::TYPE_REQUEST_ALIVE);
 567 }
 568
 569 void URLRequest::BeforeRequestComplete(int error) {
 570   DCHECK(!job_.get());
 571   DCHECK_NE(ERR_IO_PENDING, error);
 572
 573   // Check that there are no callbacks to already canceled requests.
 574   DCHECK_NE(URLRequestStatus::CANCELED, status_.status());
 575
 576   OnCallToDelegateComplete();
 577
 578   if (error != OK) {
 579     std::string source("delegate");
 580     net_log_.AddEvent(NetLog::TYPE_CANCELLED,
 581                       NetLog::StringCallback("source", &source));
 582     StartJob(new URLRequestErrorJob(this, network_delegate_, error));
 583   } else if (!delegate_redirect_url_.is_empty()) {
 584     GURL new_url;
 585     new_url.Swap(&delegate_redirect_url_);
 586
 587     URLRequestRedirectJob* job = new URLRequestRedirectJob(
 588         this, network_delegate_, new_url,
 589         // Use status code 307 to preserve the method, so POST requests work.
 590         URLRequestRedirectJob::REDIRECT_307_TEMPORARY_REDIRECT, "Delegate");
 591     StartJob(job);
 592   } else {
 593     StartJob(URLRequestJobManager::GetInstance()->CreateJob(
 594         this, network_delegate_));
 595   }
 596 }
 597
 598 void URLRequest::StartJob(URLRequestJob* job) {
 599   // TODO(mmenke): Remove ScopedTracker below once crbug.com/456327 is fixed.
 600   tracked_objects::ScopedTracker tracking_profile(
 601       FROM_HERE_WITH_EXPLICIT_FUNCTION("456327 URLRequest::StartJob"));
 602
 603   DCHECK(!is_pending_);
 604   DCHECK(!job_.get());
 605
 606   net_log_.BeginEvent(
 607       NetLog::TYPE_URL_REQUEST_START_JOB,
 608       base::Bind(&NetLogURLRequestStartCallback,
 609                  &url(), &method_, load_flags_, priority_,
 610                  upload_data_stream_ ? upload_data_stream_->identifier() : -1));
 611
 612   job_ = job;
 613   job_->SetExtraRequestHeaders(extra_request_headers_);
 614   job_->SetPriority(priority_);
 615
 616   if (upload_data_stream_.get())
 617     job_->SetUpload(upload_data_stream_.get());
 618
 619   is_pending_ = true;
 620   is_redirecting_ = false;
 621
 622   response_info_.was_cached = false;
 623
 624   if (GURL(referrer_) != URLRequestJob::ComputeReferrerForRedirect(
 625                              referrer_policy_, referrer_, url())) {
 626     if (!network_delegate_ ||
 627         !network_delegate_->CancelURLRequestWithPolicyViolatingReferrerHeader(
 628             *this, url(), GURL(referrer_))) {
 629       referrer_.clear();
 630     } else {
 631       // We need to clear the referrer anyway to avoid an infinite recursion
 632       // when starting the error job.
 633       referrer_.clear();
 634       std::string source("delegate");
 635       net_log_.AddEvent(NetLog::TYPE_CANCELLED,
 636                         NetLog::StringCallback("source", &source));
 637       RestartWithJob(new URLRequestErrorJob(
 638           this, network_delegate_, ERR_BLOCKED_BY_CLIENT));
 639       return;
 640     }
 641   }
 642
 643   // Don't allow errors to be sent from within Start().
 644   // TODO(brettw) this may cause NotifyDone to be sent synchronously,
 645   // we probably don't want this: they should be sent asynchronously so
 646   // the caller does not get reentered.
 647   job_->Start();
 648 }
 649
 650 void URLRequest::Restart() {
 651   // Should only be called if the original job didn't make any progress.
 652   DCHECK(job_.get() && !job_->has_response_started());
 653   RestartWithJob(
 654       URLRequestJobManager::GetInstance()->CreateJob(this, network_delegate_));
 655 }
 656
 657 void URLRequest::RestartWithJob(URLRequestJob *job) {
 658   DCHECK(job->request() == this);
 659   PrepareToRestart();
 660   StartJob(job);
 661 }
 662
 663 void URLRequest::Cancel() {
 664   DoCancel(ERR_ABORTED, SSLInfo());
 665 }
 666
 667 void URLRequest::CancelWithError(int error) {
 668   DoCancel(error, SSLInfo());
 669 }
 670
 671 void URLRequest::CancelWithSSLError(int error, const SSLInfo& ssl_info) {
 672   // This should only be called on a started request.
 673   if (!is_pending_ || !job_.get() || job_->has_response_started()) {
 674     NOTREACHED();
 675     return;
 676   }
 677   DoCancel(error, ssl_info);
 678 }
 679
 680 void URLRequest::DoCancel(int error, const SSLInfo& ssl_info) {
 681   DCHECK(error < 0);
 682   // If cancelled while calling a delegate, clear delegate info.
 683   if (calling_delegate_) {
 684     LogUnblocked();
 685     OnCallToDelegateComplete();
 686   }
 687
 688   // If the URL request already has an error status, then canceling is a no-op.
 689   // Plus, we don't want to change the error status once it has been set.
 690   if (status_.is_success()) {
 691     status_ = URLRequestStatus(URLRequestStatus::CANCELED, error);
 692     response_info_.ssl_info = ssl_info;
 693
 694     // If the request hasn't already been completed, log a cancellation event.
 695     if (!has_notified_completion_) {
 696       // Don't log an error code on ERR_ABORTED, since that's redundant.
 697       net_log_.AddEventWithNetErrorCode(NetLog::TYPE_CANCELLED,
 698                                         error == ERR_ABORTED ? OK : error);
 699     }
 700   }
 701
 702   if (is_pending_ && job_.get())
 703     job_->Kill();
 704
 705   // We need to notify about the end of this job here synchronously. The
 706   // Job sends an asynchronous notification but by the time this is processed,
 707   // our |context_| is NULL.
 708   NotifyRequestCompleted();
 709
 710   // The Job will call our NotifyDone method asynchronously.  This is done so
 711   // that the Delegate implementation can call Cancel without having to worry
 712   // about being called recursively.
 713 }
 714
 715 bool URLRequest::Read(IOBuffer* dest, int dest_size, int* bytes_read) {
 716   DCHECK(job_.get());
 717   DCHECK(bytes_read);
 718   *bytes_read = 0;
 719
 720   // If this is the first read, end the delegate call that may have started in
 721   // OnResponseStarted.
 722   OnCallToDelegateComplete();
 723
 724   // This handles a cancel that happens while paused.
 725   // TODO(ahendrickson): DCHECK() that it is not done after
 726   // http://crbug.com/115705 is fixed.
 727   if (job_->is_done())
 728     return false;
 729
 730   if (dest_size == 0) {
 731     // Caller is not too bright.  I guess we've done what they asked.
 732     return true;
 733   }
 734
 735   // Once the request fails or is cancelled, read will just return 0 bytes
 736   // to indicate end of stream.
 737   if (!status_.is_success()) {
 738     return true;
 739   }
 740
 741   bool rv = job_->Read(dest, dest_size, bytes_read);
 742   // If rv is false, the status cannot be success.
 743   DCHECK(rv || status_.status() != URLRequestStatus::SUCCESS);
 744
 745   if (rv && *bytes_read <= 0 && status_.is_success())
 746     NotifyRequestCompleted();
 747   return rv;
 748 }
 749
 750 void URLRequest::StopCaching() {
 751   DCHECK(job_.get());
 752   job_->StopCaching();
 753 }
 754
 755 void URLRequest::NotifyReceivedRedirect(const RedirectInfo& redirect_info,
 756                                         bool* defer_redirect) {
 757   is_redirecting_ = true;
 758
 759   // TODO(davidben): Pass the full RedirectInfo down to MaybeInterceptRedirect?
 760   URLRequestJob* job =
 761       URLRequestJobManager::GetInstance()->MaybeInterceptRedirect(
 762           this, network_delegate_, redirect_info.new_url);
 763   if (job) {
 764     RestartWithJob(job);
 765   } else if (delegate_) {
 766     OnCallToDelegate();
 767     delegate_->OnReceivedRedirect(this, redirect_info, defer_redirect);
 768     // |this| may be have been destroyed here.
 769   }
 770 }
 771
 772 void URLRequest::NotifyBeforeNetworkStart(bool* defer) {
 773   if (delegate_ && !notified_before_network_start_) {
 774     OnCallToDelegate();
 775     delegate_->OnBeforeNetworkStart(this, defer);
 776     if (!*defer)
 777       OnCallToDelegateComplete();
 778     notified_before_network_start_ = true;
 779   }
 780 }
 781
 782 void URLRequest::ResumeNetworkStart() {
 783   DCHECK(job_.get());
 784   DCHECK(notified_before_network_start_);
 785
 786   OnCallToDelegateComplete();
 787   job_->ResumeNetworkStart();
 788 }
 789
 790 void URLRequest::NotifyResponseStarted() {
 791   int net_error = OK;
 792   if (!status_.is_success())
 793     net_error = status_.error();
 794   net_log_.EndEventWithNetErrorCode(NetLog::TYPE_URL_REQUEST_START_JOB,
 795                                     net_error);
 796
 797   URLRequestJob* job =
 798       URLRequestJobManager::GetInstance()->MaybeInterceptResponse(
 799           this, network_delegate_);
 800   if (job) {
 801     RestartWithJob(job);
 802   } else {
 803     if (delegate_) {
 804       // In some cases (e.g. an event was canceled), we might have sent the
 805       // completion event and receive a NotifyResponseStarted() later.
 806       if (!has_notified_completion_ && status_.is_success()) {
 807         if (network_delegate_)
 808           network_delegate_->NotifyResponseStarted(this);
 809       }
 810
 811       // Notify in case the entire URL Request has been finished.
 812       if (!has_notified_completion_ && !status_.is_success())
 813         NotifyRequestCompleted();
 814
 815       OnCallToDelegate();
 816       delegate_->OnResponseStarted(this);
 817       // Nothing may appear below this line as OnResponseStarted may delete
 818       // |this|.
 819     }
 820   }
 821 }
 822
 823 void URLRequest::FollowDeferredRedirect() {
 824   CHECK(job_.get());
 825   CHECK(status_.is_success());
 826
 827   job_->FollowDeferredRedirect();
 828 }
 829
 830 void URLRequest::SetAuth(const AuthCredentials& credentials) {
 831   DCHECK(job_.get());
 832   DCHECK(job_->NeedsAuth());
 833
 834   job_->SetAuth(credentials);
 835 }
 836
 837 void URLRequest::CancelAuth() {
 838   DCHECK(job_.get());
 839   DCHECK(job_->NeedsAuth());
 840
 841   job_->CancelAuth();
 842 }
 843
 844 void URLRequest::ContinueWithCertificate(X509Certificate* client_cert) {
 845   DCHECK(job_.get());
 846
 847   job_->ContinueWithCertificate(client_cert);
 848 }
 849
 850 void URLRequest::ContinueDespiteLastError() {
 851   DCHECK(job_.get());
 852
 853   job_->ContinueDespiteLastError();
 854 }
 855
 856 void URLRequest::PrepareToRestart() {
 857   DCHECK(job_.get());
 858
 859   // Close the current URL_REQUEST_START_JOB, since we will be starting a new
 860   // one.
 861   net_log_.EndEvent(NetLog::TYPE_URL_REQUEST_START_JOB);
 862
 863   OrphanJob();
 864
 865   response_info_ = HttpResponseInfo();
 866   response_info_.request_time = base::Time::Now();
 867
 868   load_timing_info_ = LoadTimingInfo();
 869   load_timing_info_.request_start_time = response_info_.request_time;
 870   load_timing_info_.request_start = base::TimeTicks::Now();
 871
 872   status_ = URLRequestStatus();
 873   is_pending_ = false;
 874 }
 875
 876 void URLRequest::OrphanJob() {
 877   // When calling this function, please check that URLRequestHttpJob is
 878   // not in between calling NetworkDelegate::NotifyHeadersReceived receiving
 879   // the call back. This is currently guaranteed by the following strategies:
 880   // - OrphanJob is called on JobRestart, in this case the URLRequestJob cannot
 881   //   be receiving any headers at that time.
 882   // - OrphanJob is called in ~URLRequest, in this case
 883   //   NetworkDelegate::NotifyURLRequestDestroyed notifies the NetworkDelegate
 884   //   that the callback becomes invalid.
 885   job_->Kill();
 886   job_->DetachRequest();  // ensures that the job will not call us again
 887   job_ = NULL;
 888 }
 889
 890 int URLRequest::Redirect(const RedirectInfo& redirect_info) {
 891   // Matches call in NotifyReceivedRedirect.
 892   OnCallToDelegateComplete();
 893   if (net_log_.IsCapturing()) {
 894     net_log_.AddEvent(
 895         NetLog::TYPE_URL_REQUEST_REDIRECTED,
 896         NetLog::StringCallback("location",
 897                                &redirect_info.new_url.possibly_invalid_spec()));
 898   }
 899
 900   // TODO(davidben): Pass the full RedirectInfo to the NetworkDelegate.
 901   if (network_delegate_)
 902     network_delegate_->NotifyBeforeRedirect(this, redirect_info.new_url);
 903
 904   if (redirect_limit_ <= 0) {
 905     DVLOG(1) << "disallowing redirect: exceeds limit";
 906     return ERR_TOO_MANY_REDIRECTS;
 907   }
 908
 909   if (!redirect_info.new_url.is_valid())
 910     return ERR_INVALID_URL;
 911
 912   if (!job_->IsSafeRedirect(redirect_info.new_url)) {
 913     DVLOG(1) << "disallowing redirect: unsafe protocol";
 914     return ERR_UNSAFE_REDIRECT;
 915   }
 916
 917   if (!final_upload_progress_.position())
 918     final_upload_progress_ = job_->GetUploadProgress();
 919   PrepareToRestart();
 920
 921   if (redirect_info.new_method != method_) {
 922     // TODO(davidben): This logic still needs to be replicated at the consumers.
 923     if (method_ == "POST") {
 924       // If being switched from POST, must remove headers that were specific to
 925       // the POST and don't have meaning in other methods. For example the
 926       // inclusion of a multipart Content-Type header in GET can cause problems
 927       // with some servers:
 928       // http://code.google.com/p/chromium/issues/detail?id=843
 929       StripPostSpecificHeaders(&extra_request_headers_);
 930     }
 931     upload_data_stream_.reset();
 932     method_ = redirect_info.new_method;
 933   }
 934
 935   // Cross-origin redirects should not result in an Origin header value that is
 936   // equal to the original request's Origin header. This is necessary to prevent
 937   // a reflection of POST requests to bypass CSRF protections. If the header was
 938   // not set to "null", a POST request from origin A to a malicious origin M
 939   // could be redirected by M back to A.
 940   //
 941   // In the Section 4.2, Step 4.10 of the Fetch spec
 942   // (https://fetch.spec.whatwg.org/#concept-http-fetch), it states that on
 943   // cross-origin 301, 302, 303, 307, and 308 redirects, the user agent should
 944   // set the request's origin to an "opaque identifier," which serializes to
 945   // "null." This matches Firefox and IE behavior, although it supercedes the
 946   // suggested behavior in RFC 6454, "The Web Origin Concept."
 947   //
 948   // See also https://crbug.com/465517.
 949   //
 950   // TODO(jww): This is probably layering violation and should be refactored
 951   // into //content. See https://crbug.com/471397.
 952   if (redirect_info.new_url.GetOrigin() != url().GetOrigin() &&
 953       extra_request_headers_.HasHeader(HttpRequestHeaders::kOrigin)) {
 954     extra_request_headers_.SetHeader(HttpRequestHeaders::kOrigin,
 955                                      url::Origin().string());
 956   }
 957
 958   referrer_ = redirect_info.new_referrer;
 959   first_party_for_cookies_ = redirect_info.new_first_party_for_cookies;
 960
 961   url_chain_.push_back(redirect_info.new_url);
 962   --redirect_limit_;
 963
 964   Start();
 965   return OK;
 966 }
 967
 968 const URLRequestContext* URLRequest::context() const {
 969   return context_;
 970 }
 971
 972 int64 URLRequest::GetExpectedContentSize() const {
 973   int64 expected_content_size = -1;
 974   if (job_.get())
 975     expected_content_size = job_->expected_content_size();
 976
 977   return expected_content_size;
 978 }
 979
 980 void URLRequest::SetPriority(RequestPriority priority) {
 981   DCHECK_GE(priority, MINIMUM_PRIORITY);
 982   DCHECK_LE(priority, MAXIMUM_PRIORITY);
 983
 984   if ((load_flags_ & LOAD_IGNORE_LIMITS) && (priority != MAXIMUM_PRIORITY)) {
 985     NOTREACHED();
 986     // Maintain the invariant that requests with IGNORE_LIMITS set
 987     // have MAXIMUM_PRIORITY for release mode.
 988     return;
 989   }
 990
 991   if (priority_ == priority)
 992     return;
 993
 994   priority_ = priority;
 995   if (job_.get()) {
 996     net_log_.AddEvent(NetLog::TYPE_URL_REQUEST_SET_PRIORITY,
 997                       NetLog::IntegerCallback("priority", priority_));
 998     job_->SetPriority(priority_);
 999   }
1000 }
1001
1002 bool URLRequest::GetHSTSRedirect(GURL* redirect_url) const {
1003   const GURL& url = this->url();
1004   bool scheme_is_http = url.SchemeIs("http");
1005   if (!scheme_is_http && !url.SchemeIs("ws"))
1006     return false;
1007   TransportSecurityState* state = context()->transport_security_state();
1008   if (state && state->ShouldUpgradeToSSL(url.host())) {
1009     GURL::Replacements replacements;
1010     const char* new_scheme = scheme_is_http ? "https" : "wss";
1011     replacements.SetSchemeStr(new_scheme);
1012     *redirect_url = url.ReplaceComponents(replacements);
1013     return true;
1014   }
1015   return false;
1016 }
1017
1018 void URLRequest::NotifyAuthRequired(AuthChallengeInfo* auth_info) {
1019   NetworkDelegate::AuthRequiredResponse rv =
1020       NetworkDelegate::AUTH_REQUIRED_RESPONSE_NO_ACTION;
1021   auth_info_ = auth_info;
1022   if (network_delegate_) {
1023     OnCallToDelegate();
1024     rv = network_delegate_->NotifyAuthRequired(
1025         this,
1026         *auth_info,
1027         base::Bind(&URLRequest::NotifyAuthRequiredComplete,
1028                    base::Unretained(this)),
1029         &auth_credentials_);
1030     if (rv == NetworkDelegate::AUTH_REQUIRED_RESPONSE_IO_PENDING)
1031       return;
1032   }
1033
1034   NotifyAuthRequiredComplete(rv);
1035 }
1036
1037 void URLRequest::NotifyAuthRequiredComplete(
1038     NetworkDelegate::AuthRequiredResponse result) {
1039   OnCallToDelegateComplete();
1040
1041   // Check that there are no callbacks to already canceled requests.
1042   DCHECK_NE(URLRequestStatus::CANCELED, status_.status());
1043
1044   // NotifyAuthRequired may be called multiple times, such as
1045   // when an authentication attempt fails. Clear out the data
1046   // so it can be reset on another round.
1047   AuthCredentials credentials = auth_credentials_;
1048   auth_credentials_ = AuthCredentials();
1049   scoped_refptr<AuthChallengeInfo> auth_info;
1050   auth_info.swap(auth_info_);
1051
1052   switch (result) {
1053     case NetworkDelegate::AUTH_REQUIRED_RESPONSE_NO_ACTION:
1054       // Defer to the URLRequest::Delegate, since the NetworkDelegate
1055       // didn't take an action.
1056       if (delegate_)
1057         delegate_->OnAuthRequired(this, auth_info.get());
1058       break;
1059
1060     case NetworkDelegate::AUTH_REQUIRED_RESPONSE_SET_AUTH:
1061       SetAuth(credentials);
1062       break;
1063
1064     case NetworkDelegate::AUTH_REQUIRED_RESPONSE_CANCEL_AUTH:
1065       CancelAuth();
1066       break;
1067
1068     case NetworkDelegate::AUTH_REQUIRED_RESPONSE_IO_PENDING:
1069       NOTREACHED();
1070       break;
1071   }
1072 }
1073
1074 void URLRequest::NotifyCertificateRequested(
1075     SSLCertRequestInfo* cert_request_info) {
1076   if (delegate_)
1077     delegate_->OnCertificateRequested(this, cert_request_info);
1078 }
1079
1080 void URLRequest::NotifySSLCertificateError(const SSLInfo& ssl_info,
1081                                            bool fatal) {
1082   if (delegate_)
1083     delegate_->OnSSLCertificateError(this, ssl_info, fatal);
1084 }
1085
1086 bool URLRequest::CanGetCookies(const CookieList& cookie_list) const {
1087   DCHECK(!(load_flags_ & LOAD_DO_NOT_SEND_COOKIES));
1088   if (network_delegate_) {
1089     return network_delegate_->CanGetCookies(*this, cookie_list);
1090   }
1091   return g_default_can_use_cookies;
1092 }
1093
1094 bool URLRequest::CanSetCookie(const std::string& cookie_line,
1095                               CookieOptions* options) const {
1096   DCHECK(!(load_flags_ & LOAD_DO_NOT_SAVE_COOKIES));
1097   if (network_delegate_) {
1098     return network_delegate_->CanSetCookie(*this, cookie_line, options);
1099   }
1100   return g_default_can_use_cookies;
1101 }
1102
1103 bool URLRequest::CanEnablePrivacyMode() const {
1104   if (network_delegate_) {
1105     return network_delegate_->CanEnablePrivacyMode(url(),
1106                                                    first_party_for_cookies_);
1107   }
1108   return !g_default_can_use_cookies;
1109 }
1110
1111
1112 void URLRequest::NotifyReadCompleted(int bytes_read) {
1113   // Notify in case the entire URL Request has been finished.
1114   if (bytes_read <= 0)
1115     NotifyRequestCompleted();
1116
1117   // Notify NetworkChangeNotifier that we just received network data.
1118   // This is to identify cases where the NetworkChangeNotifier thinks we
1119   // are off-line but we are still receiving network data (crbug.com/124069),
1120   // and to get rough network connection measurements.
1121   if (bytes_read > 0 && !was_cached())
1122     NetworkChangeNotifier::NotifyDataReceived(*this, bytes_read);
1123
1124   if (delegate_)
1125     delegate_->OnReadCompleted(this, bytes_read);
1126
1127   // Nothing below this line as OnReadCompleted may delete |this|.
1128 }
1129
1130 void URLRequest::OnHeadersComplete() {
1131   // Cache load timing information now, as information will be lost once the
1132   // socket is closed and the ClientSocketHandle is Reset, which will happen
1133   // once the body is complete.  The start times should already be populated.
1134   if (job_.get()) {
1135     // Keep a copy of the two times the URLRequest sets.
1136     base::TimeTicks request_start = load_timing_info_.request_start;
1137     base::Time request_start_time = load_timing_info_.request_start_time;
1138
1139     // Clear load times.  Shouldn't be neded, but gives the GetLoadTimingInfo a
1140     // consistent place to start from.
1141     load_timing_info_ = LoadTimingInfo();
1142     job_->GetLoadTimingInfo(&load_timing_info_);
1143
1144     load_timing_info_.request_start = request_start;
1145     load_timing_info_.request_start_time = request_start_time;
1146
1147     ConvertRealLoadTimesToBlockingTimes(&load_timing_info_);
1148   }
1149 }
1150
1151 void URLRequest::NotifyRequestCompleted() {
1152   // TODO(battre): Get rid of this check, according to willchan it should
1153   // not be needed.
1154   if (has_notified_completion_)
1155     return;
1156
1157   is_pending_ = false;
1158   is_redirecting_ = false;
1159   has_notified_completion_ = true;
1160   if (network_delegate_)
1161     network_delegate_->NotifyCompleted(this, job_.get() != NULL);
1162 }
1163
1164 void URLRequest::OnCallToDelegate() {
1165   DCHECK(!calling_delegate_);
1166   DCHECK(blocked_by_.empty());
1167   calling_delegate_ = true;
1168   net_log_.BeginEvent(NetLog::TYPE_URL_REQUEST_DELEGATE);
1169 }
1170
1171 void URLRequest::OnCallToDelegateComplete() {
1172   // This should have been cleared before resuming the request.
1173   DCHECK(blocked_by_.empty());
1174   if (!calling_delegate_)
1175     return;
1176   calling_delegate_ = false;
1177   net_log_.EndEvent(NetLog::TYPE_URL_REQUEST_DELEGATE);
1178 }
1179
1180 void URLRequest::set_stack_trace(const base::debug::StackTrace& stack_trace) {
1181   base::debug::StackTrace* stack_trace_copy =
1182       new base::debug::StackTrace(NULL, 0);
1183   *stack_trace_copy = stack_trace;
1184   stack_trace_.reset(stack_trace_copy);
1185 }
1186
1187 const base::debug::StackTrace* URLRequest::stack_trace() const {
1188   return stack_trace_.get();
1189 }
1190
1191 void URLRequest::GetConnectionAttempts(ConnectionAttempts* out) const {
1192   if (job_)
1193     job_->GetConnectionAttempts(out);
1194   else
1195     out->clear();
1196 }
1197
1198 }  // namespace net