net/url_request/url_request.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_request.h"
   6
   7 #include "base/bind.h"
   8 #include "base/bind_helpers.h"
   9 #include "base/callback.h"
  10 #include "base/compiler_specific.h"
  11 #include "base/debug/stack_trace.h"
  12 #include "base/lazy_instance.h"
  13 #include "base/memory/singleton.h"
  14 #include "base/message_loop/message_loop.h"
  15 #include "base/metrics/stats_counters.h"
  16 #include "base/metrics/user_metrics.h"
  17 #include "base/stl_util.h"
  18 #include "base/strings/utf_string_conversions.h"
  19 #include "base/synchronization/lock.h"
  20 #include "base/values.h"
  21 #include "net/base/auth.h"
  22 #include "net/base/host_port_pair.h"
  23 #include "net/base/load_flags.h"
  24 #include "net/base/load_timing_info.h"
  25 #include "net/base/net_errors.h"
  26 #include "net/base/net_log.h"
  27 #include "net/base/network_change_notifier.h"
  28 #include "net/base/network_delegate.h"
  29 #include "net/base/upload_data_stream.h"
  30 #include "net/http/http_response_headers.h"
  31 #include "net/http/http_util.h"
  32 #include "net/ssl/ssl_cert_request_info.h"
  33 #include "net/url_request/url_request_context.h"
  34 #include "net/url_request/url_request_error_job.h"
  35 #include "net/url_request/url_request_job.h"
  36 #include "net/url_request/url_request_job_manager.h"
  37 #include "net/url_request/url_request_netlog_params.h"
  38 #include "net/url_request/url_request_redirect_job.h"
  39
  40 using base::Time;
  41 using std::string;
  42
  43 namespace net {
  44
  45 namespace {
  46
  47 // Max number of http redirects to follow.  Same number as gecko.
  48 const int kMaxRedirects = 20;
  49
  50 // Discard headers which have meaning in POST (Content-Length, Content-Type,
  51 // Origin).
  52 void StripPostSpecificHeaders(HttpRequestHeaders* headers) {
  53   // These are headers that may be attached to a POST.
  54   headers->RemoveHeader(HttpRequestHeaders::kContentLength);
  55   headers->RemoveHeader(HttpRequestHeaders::kContentType);
  56   headers->RemoveHeader(HttpRequestHeaders::kOrigin);
  57 }
  58
  59 // TODO(battre): Delete this, see http://crbug.com/89321:
  60 // This counter keeps track of the identifiers used for URL requests so far.
  61 // 0 is reserved to represent an invalid ID.
  62 uint64 g_next_url_request_identifier = 1;
  63
  64 // This lock protects g_next_url_request_identifier.
  65 base::LazyInstance<base::Lock>::Leaky
  66     g_next_url_request_identifier_lock = LAZY_INSTANCE_INITIALIZER;
  67
  68 // Returns an prior unused identifier for URL requests.
  69 uint64 GenerateURLRequestIdentifier() {
  70   base::AutoLock lock(g_next_url_request_identifier_lock.Get());
  71   return g_next_url_request_identifier++;
  72 }
  73
  74 // True once the first URLRequest was started.
  75 bool g_url_requests_started = false;
  76
  77 // True if cookies are accepted by default.
  78 bool g_default_can_use_cookies = true;
  79
  80 // When the URLRequest first assempts load timing information, it has the times
  81 // at which each event occurred.  The API requires the time which the request
  82 // was blocked on each phase.  This function handles the conversion.
  83 //
  84 // In the case of reusing a SPDY session, old proxy results may have been
  85 // reused, so proxy resolution times may be before the request was started.
  86 //
  87 // Due to preconnect and late binding, it is also possible for the connection
  88 // attempt to start before a request has been started, or proxy resolution
  89 // completed.
  90 //
  91 // This functions fixes both those cases.
  92 void ConvertRealLoadTimesToBlockingTimes(
  93     net::LoadTimingInfo* load_timing_info) {
  94   DCHECK(!load_timing_info->request_start.is_null());
  95
  96   // Earliest time possible for the request to be blocking on connect events.
  97   base::TimeTicks block_on_connect = load_timing_info->request_start;
  98
  99   if (!load_timing_info->proxy_resolve_start.is_null()) {
 100     DCHECK(!load_timing_info->proxy_resolve_end.is_null());
 101
 102     // Make sure the proxy times are after request start.
 103     if (load_timing_info->proxy_resolve_start < load_timing_info->request_start)
 104       load_timing_info->proxy_resolve_start = load_timing_info->request_start;
 105     if (load_timing_info->proxy_resolve_end < load_timing_info->request_start)
 106       load_timing_info->proxy_resolve_end = load_timing_info->request_start;
 107
 108     // Connect times must also be after the proxy times.
 109     block_on_connect = load_timing_info->proxy_resolve_end;
 110   }
 111
 112   // Make sure connection times are after start and proxy times.
 113
 114   net::LoadTimingInfo::ConnectTiming* connect_timing =
 115       &load_timing_info->connect_timing;
 116   if (!connect_timing->dns_start.is_null()) {
 117     DCHECK(!connect_timing->dns_end.is_null());
 118     if (connect_timing->dns_start < block_on_connect)
 119       connect_timing->dns_start = block_on_connect;
 120     if (connect_timing->dns_end < block_on_connect)
 121       connect_timing->dns_end = block_on_connect;
 122   }
 123
 124   if (!connect_timing->connect_start.is_null()) {
 125     DCHECK(!connect_timing->connect_end.is_null());
 126     if (connect_timing->connect_start < block_on_connect)
 127       connect_timing->connect_start = block_on_connect;
 128     if (connect_timing->connect_end < block_on_connect)
 129       connect_timing->connect_end = block_on_connect;
 130   }
 131
 132   if (!connect_timing->ssl_start.is_null()) {
 133     DCHECK(!connect_timing->ssl_end.is_null());
 134     if (connect_timing->ssl_start < block_on_connect)
 135       connect_timing->ssl_start = block_on_connect;
 136     if (connect_timing->ssl_end < block_on_connect)
 137       connect_timing->ssl_end = block_on_connect;
 138   }
 139 }
 140
 141 }  // namespace
 142
 143 void URLRequest::Deprecated::RegisterRequestInterceptor(
 144     Interceptor* interceptor) {
 145   URLRequest::RegisterRequestInterceptor(interceptor);
 146 }
 147
 148 void URLRequest::Deprecated::UnregisterRequestInterceptor(
 149     Interceptor* interceptor) {
 150   URLRequest::UnregisterRequestInterceptor(interceptor);
 151 }
 152
 153 ///////////////////////////////////////////////////////////////////////////////
 154 // URLRequest::Interceptor
 155
 156 URLRequestJob* URLRequest::Interceptor::MaybeInterceptRedirect(
 157     URLRequest* request,
 158     NetworkDelegate* network_delegate,
 159     const GURL& location) {
 160   return NULL;
 161 }
 162
 163 URLRequestJob* URLRequest::Interceptor::MaybeInterceptResponse(
 164     URLRequest* request, NetworkDelegate* network_delegate) {
 165   return NULL;
 166 }
 167
 168 ///////////////////////////////////////////////////////////////////////////////
 169 // URLRequest::Delegate
 170
 171 void URLRequest::Delegate::OnReceivedRedirect(URLRequest* request,
 172                                               const GURL& new_url,
 173                                               bool* defer_redirect) {
 174 }
 175
 176 void URLRequest::Delegate::OnAuthRequired(URLRequest* request,
 177                                           AuthChallengeInfo* auth_info) {
 178   request->CancelAuth();
 179 }
 180
 181 void URLRequest::Delegate::OnCertificateRequested(
 182     URLRequest* request,
 183     SSLCertRequestInfo* cert_request_info) {
 184   request->Cancel();
 185 }
 186
 187 void URLRequest::Delegate::OnSSLCertificateError(URLRequest* request,
 188                                                  const SSLInfo& ssl_info,
 189                                                  bool is_hsts_ok) {
 190   request->Cancel();
 191 }
 192
 193 void URLRequest::Delegate::OnBeforeNetworkStart(URLRequest* request,
 194                                                 bool* defer) {
 195 }
 196
 197 ///////////////////////////////////////////////////////////////////////////////
 198 // URLRequest
 199
 200 URLRequest::URLRequest(const GURL& url,
 201                        RequestPriority priority,
 202                        Delegate* delegate,
 203                        const URLRequestContext* context)
 204     : identifier_(GenerateURLRequestIdentifier()) {
 205   Init(url, priority, delegate, context, NULL);
 206 }
 207
 208 URLRequest::URLRequest(const GURL& url,
 209                        RequestPriority priority,
 210                        Delegate* delegate,
 211                        const URLRequestContext* context,
 212                        CookieStore* cookie_store)
 213     : identifier_(GenerateURLRequestIdentifier()) {
 214   Init(url, priority, delegate, context, cookie_store);
 215 }
 216
 217 URLRequest::~URLRequest() {
 218   Cancel();
 219
 220   if (network_delegate_) {
 221     network_delegate_->NotifyURLRequestDestroyed(this);
 222     if (job_.get())
 223       job_->NotifyURLRequestDestroyed();
 224   }
 225
 226   if (job_.get())
 227     OrphanJob();
 228
 229   int deleted = context_->url_requests()->erase(this);
 230   CHECK_EQ(1, deleted);
 231
 232   int net_error = OK;
 233   // Log error only on failure, not cancellation, as even successful requests
 234   // are "cancelled" on destruction.
 235   if (status_.status() == URLRequestStatus::FAILED)
 236     net_error = status_.error();
 237   net_log_.EndEventWithNetErrorCode(NetLog::TYPE_REQUEST_ALIVE, net_error);
 238 }
 239
 240 // static
 241 void URLRequest::RegisterRequestInterceptor(Interceptor* interceptor) {
 242   URLRequestJobManager::GetInstance()->RegisterRequestInterceptor(interceptor);
 243 }
 244
 245 // static
 246 void URLRequest::UnregisterRequestInterceptor(Interceptor* interceptor) {
 247   URLRequestJobManager::GetInstance()->UnregisterRequestInterceptor(
 248       interceptor);
 249 }
 250
 251 void URLRequest::Init(const GURL& url,
 252                       RequestPriority priority,
 253                       Delegate* delegate,
 254                       const URLRequestContext* context,
 255                       CookieStore* cookie_store) {
 256   context_ = context;
 257   network_delegate_ = context->network_delegate();
 258   net_log_ = BoundNetLog::Make(context->net_log(), NetLog::SOURCE_URL_REQUEST);
 259   url_chain_.push_back(url);
 260   method_ = "GET";
 261   referrer_policy_ = CLEAR_REFERRER_ON_TRANSITION_FROM_SECURE_TO_INSECURE;
 262   load_flags_ = LOAD_NORMAL;
 263   delegate_ = delegate;
 264   is_pending_ = false;
 265   is_redirecting_ = false;
 266   redirect_limit_ = kMaxRedirects;
 267   priority_ = priority;
 268   calling_delegate_ = false;
 269   use_blocked_by_as_load_param_ =false;
 270   before_request_callback_ = base::Bind(&URLRequest::BeforeRequestComplete,
 271                                         base::Unretained(this));
 272   has_notified_completion_ = false;
 273   received_response_content_length_ = 0;
 274   creation_time_ = base::TimeTicks::Now();
 275   notified_before_network_start_ = false;
 276
 277   SIMPLE_STATS_COUNTER("URLRequestCount");
 278
 279   // Sanity check out environment.
 280   DCHECK(base::MessageLoop::current())
 281       << "The current base::MessageLoop must exist";
 282
 283   CHECK(context);
 284   context->url_requests()->insert(this);
 285   cookie_store_ = cookie_store;
 286   if (cookie_store_ == NULL)
 287     cookie_store_ = context->cookie_store();
 288
 289   net_log_.BeginEvent(NetLog::TYPE_REQUEST_ALIVE);
 290 }
 291
 292 void URLRequest::EnableChunkedUpload() {
 293   DCHECK(!upload_data_stream_ || upload_data_stream_->is_chunked());
 294   if (!upload_data_stream_) {
 295     upload_data_stream_.reset(
 296         new UploadDataStream(UploadDataStream::CHUNKED, 0));
 297   }
 298 }
 299
 300 void URLRequest::AppendChunkToUpload(const char* bytes,
 301                                      int bytes_len,
 302                                      bool is_last_chunk) {
 303   DCHECK(upload_data_stream_);
 304   DCHECK(upload_data_stream_->is_chunked());
 305   DCHECK_GT(bytes_len, 0);
 306   upload_data_stream_->AppendChunk(bytes, bytes_len, is_last_chunk);
 307 }
 308
 309 void URLRequest::set_upload(scoped_ptr<UploadDataStream> upload) {
 310   DCHECK(!upload->is_chunked());
 311   upload_data_stream_ = upload.Pass();
 312 }
 313
 314 const UploadDataStream* URLRequest::get_upload() const {
 315   return upload_data_stream_.get();
 316 }
 317
 318 bool URLRequest::has_upload() const {
 319   return upload_data_stream_.get() != NULL;
 320 }
 321
 322 void URLRequest::SetExtraRequestHeaderById(int id, const string& value,
 323                                            bool overwrite) {
 324   DCHECK(!is_pending_ || is_redirecting_);
 325   NOTREACHED() << "implement me!";
 326 }
 327
 328 void URLRequest::SetExtraRequestHeaderByName(const string& name,
 329                                              const string& value,
 330                                              bool overwrite) {
 331   DCHECK(!is_pending_ || is_redirecting_);
 332   if (overwrite) {
 333     extra_request_headers_.SetHeader(name, value);
 334   } else {
 335     extra_request_headers_.SetHeaderIfMissing(name, value);
 336   }
 337 }
 338
 339 void URLRequest::RemoveRequestHeaderByName(const string& name) {
 340   DCHECK(!is_pending_ || is_redirecting_);
 341   extra_request_headers_.RemoveHeader(name);
 342 }
 343
 344 void URLRequest::SetExtraRequestHeaders(
 345     const HttpRequestHeaders& headers) {
 346   DCHECK(!is_pending_);
 347   extra_request_headers_ = headers;
 348
 349   // NOTE: This method will likely become non-trivial once the other setters
 350   // for request headers are implemented.
 351 }
 352
 353 bool URLRequest::GetFullRequestHeaders(HttpRequestHeaders* headers) const {
 354   if (!job_.get())
 355     return false;
 356
 357   return job_->GetFullRequestHeaders(headers);
 358 }
 359
 360 int64 URLRequest::GetTotalReceivedBytes() const {
 361   if (!job_.get())
 362     return 0;
 363
 364   return job_->GetTotalReceivedBytes();
 365 }
 366
 367 LoadStateWithParam URLRequest::GetLoadState() const {
 368   // The !blocked_by_.empty() check allows |this| to report it's blocked on a
 369   // delegate before it has been started.
 370   if (calling_delegate_ || !blocked_by_.empty()) {
 371     return LoadStateWithParam(
 372         LOAD_STATE_WAITING_FOR_DELEGATE,
 373         use_blocked_by_as_load_param_ ? base::UTF8ToUTF16(blocked_by_) :
 374                                         base::string16());
 375   }
 376   return LoadStateWithParam(job_.get() ? job_->GetLoadState() : LOAD_STATE_IDLE,
 377                             base::string16());
 378 }
 379
 380 base::Value* URLRequest::GetStateAsValue() const {
 381   base::DictionaryValue* dict = new base::DictionaryValue();
 382   dict->SetString("url", original_url().possibly_invalid_spec());
 383
 384   if (url_chain_.size() > 1) {
 385     base::ListValue* list = new base::ListValue();
 386     for (std::vector<GURL>::const_iterator url = url_chain_.begin();
 387          url != url_chain_.end(); ++url) {
 388       list->AppendString(url->possibly_invalid_spec());
 389     }
 390     dict->Set("url_chain", list);
 391   }
 392
 393   dict->SetInteger("load_flags", load_flags_);
 394
 395   LoadStateWithParam load_state = GetLoadState();
 396   dict->SetInteger("load_state", load_state.state);
 397   if (!load_state.param.empty())
 398     dict->SetString("load_state_param", load_state.param);
 399   if (!blocked_by_.empty())
 400     dict->SetString("delegate_info", blocked_by_);
 401
 402   dict->SetString("method", method_);
 403   dict->SetBoolean("has_upload", has_upload());
 404   dict->SetBoolean("is_pending", is_pending_);
 405
 406   // Add the status of the request.  The status should always be IO_PENDING, and
 407   // the error should always be OK, unless something is holding onto a request
 408   // that has finished or a request was leaked.  Neither of these should happen.
 409   switch (status_.status()) {
 410     case URLRequestStatus::SUCCESS:
 411       dict->SetString("status", "SUCCESS");
 412       break;
 413     case URLRequestStatus::IO_PENDING:
 414       dict->SetString("status", "IO_PENDING");
 415       break;
 416     case URLRequestStatus::CANCELED:
 417       dict->SetString("status", "CANCELED");
 418       break;
 419     case URLRequestStatus::FAILED:
 420       dict->SetString("status", "FAILED");
 421       break;
 422   }
 423   if (status_.error() != OK)
 424     dict->SetInteger("net_error", status_.error());
 425   return dict;
 426 }
 427
 428 void URLRequest::LogBlockedBy(const char* blocked_by) {
 429   DCHECK(blocked_by);
 430   DCHECK_GT(strlen(blocked_by), 0u);
 431
 432   // Only log information to NetLog during startup and certain deferring calls
 433   // to delegates.  For all reads but the first, do nothing.
 434   if (!calling_delegate_ && !response_info_.request_time.is_null())
 435     return;
 436
 437   LogUnblocked();
 438   blocked_by_ = blocked_by;
 439   use_blocked_by_as_load_param_ = false;
 440
 441   net_log_.BeginEvent(
 442       NetLog::TYPE_DELEGATE_INFO,
 443       NetLog::StringCallback("delegate_info", &blocked_by_));
 444 }
 445
 446 void URLRequest::LogAndReportBlockedBy(const char* source) {
 447   LogBlockedBy(source);
 448   use_blocked_by_as_load_param_ = true;
 449 }
 450
 451 void URLRequest::LogUnblocked() {
 452   if (blocked_by_.empty())
 453     return;
 454
 455   net_log_.EndEvent(NetLog::TYPE_DELEGATE_INFO);
 456   blocked_by_.clear();
 457 }
 458
 459 UploadProgress URLRequest::GetUploadProgress() const {
 460   if (!job_.get()) {
 461     // We haven't started or the request was cancelled
 462     return UploadProgress();
 463   }
 464   if (final_upload_progress_.position()) {
 465     // The first job completed and none of the subsequent series of
 466     // GETs when following redirects will upload anything, so we return the
 467     // cached results from the initial job, the POST.
 468     return final_upload_progress_;
 469   }
 470   return job_->GetUploadProgress();
 471 }
 472
 473 void URLRequest::GetResponseHeaderById(int id, string* value) {
 474   DCHECK(job_.get());
 475   NOTREACHED() << "implement me!";
 476 }
 477
 478 void URLRequest::GetResponseHeaderByName(const string& name, string* value) {
 479   DCHECK(value);
 480   if (response_info_.headers.get()) {
 481     response_info_.headers->GetNormalizedHeader(name, value);
 482   } else {
 483     value->clear();
 484   }
 485 }
 486
 487 void URLRequest::GetAllResponseHeaders(string* headers) {
 488   DCHECK(headers);
 489   if (response_info_.headers.get()) {
 490     response_info_.headers->GetNormalizedHeaders(headers);
 491   } else {
 492     headers->clear();
 493   }
 494 }
 495
 496 HostPortPair URLRequest::GetSocketAddress() const {
 497   DCHECK(job_.get());
 498   return job_->GetSocketAddress();
 499 }
 500
 501 HttpResponseHeaders* URLRequest::response_headers() const {
 502   return response_info_.headers.get();
 503 }
 504
 505 void URLRequest::GetLoadTimingInfo(LoadTimingInfo* load_timing_info) const {
 506   *load_timing_info = load_timing_info_;
 507 }
 508
 509 bool URLRequest::GetResponseCookies(ResponseCookies* cookies) {
 510   DCHECK(job_.get());
 511   return job_->GetResponseCookies(cookies);
 512 }
 513
 514 void URLRequest::GetMimeType(string* mime_type) const {
 515   DCHECK(job_.get());
 516   job_->GetMimeType(mime_type);
 517 }
 518
 519 void URLRequest::GetCharset(string* charset) const {
 520   DCHECK(job_.get());
 521   job_->GetCharset(charset);
 522 }
 523
 524 int URLRequest::GetResponseCode() const {
 525   DCHECK(job_.get());
 526   return job_->GetResponseCode();
 527 }
 528
 529 void URLRequest::SetLoadFlags(int flags) {
 530   if ((load_flags_ & LOAD_IGNORE_LIMITS) != (flags & LOAD_IGNORE_LIMITS)) {
 531     DCHECK(!job_);
 532     DCHECK(flags & LOAD_IGNORE_LIMITS);
 533     DCHECK_EQ(priority_, MAXIMUM_PRIORITY);
 534   }
 535   load_flags_ = flags;
 536
 537   // This should be a no-op given the above DCHECKs, but do this
 538   // anyway for release mode.
 539   if ((load_flags_ & LOAD_IGNORE_LIMITS) != 0)
 540     SetPriority(MAXIMUM_PRIORITY);
 541 }
 542
 543 // static
 544 void URLRequest::SetDefaultCookiePolicyToBlock() {
 545   CHECK(!g_url_requests_started);
 546   g_default_can_use_cookies = false;
 547 }
 548
 549 // static
 550 bool URLRequest::IsHandledProtocol(const std::string& scheme) {
 551   return URLRequestJobManager::SupportsScheme(scheme);
 552 }
 553
 554 // static
 555 bool URLRequest::IsHandledURL(const GURL& url) {
 556   if (!url.is_valid()) {
 557     // We handle error cases.
 558     return true;
 559   }
 560
 561   return IsHandledProtocol(url.scheme());
 562 }
 563
 564 void URLRequest::set_first_party_for_cookies(
 565     const GURL& first_party_for_cookies) {
 566   first_party_for_cookies_ = first_party_for_cookies;
 567 }
 568
 569 void URLRequest::set_method(const std::string& method) {
 570   DCHECK(!is_pending_);
 571   method_ = method;
 572 }
 573
 574 // static
 575 std::string URLRequest::ComputeMethodForRedirect(
 576     const std::string& method,
 577     int http_status_code) {
 578   // For 303 redirects, all request methods except HEAD are converted to GET,
 579   // as per the latest httpbis draft.  The draft also allows POST requests to
 580   // be converted to GETs when following 301/302 redirects, for historical
 581   // reasons. Most major browsers do this and so shall we.  Both RFC 2616 and
 582   // the httpbis draft say to prompt the user to confirm the generation of new
 583   // requests, other than GET and HEAD requests, but IE omits these prompts and
 584   // so shall we.
 585   // See:  https://tools.ietf.org/html/draft-ietf-httpbis-p2-semantics-17#section-7.3
 586   if ((http_status_code == 303 && method != "HEAD") ||
 587       ((http_status_code == 301 || http_status_code == 302) &&
 588        method == "POST")) {
 589     return "GET";
 590   }
 591   return method;
 592 }
 593
 594 void URLRequest::SetReferrer(const std::string& referrer) {
 595   DCHECK(!is_pending_);
 596   GURL referrer_url(referrer);
 597   if (referrer_url.is_valid()) {
 598     referrer_ = referrer_url.GetAsReferrer().spec();
 599   } else {
 600     referrer_ = referrer;
 601   }
 602 }
 603
 604 void URLRequest::set_referrer_policy(ReferrerPolicy referrer_policy) {
 605   DCHECK(!is_pending_);
 606   referrer_policy_ = referrer_policy;
 607 }
 608
 609 void URLRequest::set_delegate(Delegate* delegate) {
 610   delegate_ = delegate;
 611 }
 612
 613 void URLRequest::Start() {
 614   // Some values can be NULL, but the job factory must not be.
 615   DCHECK(context_->job_factory());
 616
 617   DCHECK_EQ(network_delegate_, context_->network_delegate());
 618   // Anything that sets |blocked_by_| before start should have cleaned up after
 619   // itself.
 620   DCHECK(blocked_by_.empty());
 621
 622   g_url_requests_started = true;
 623   response_info_.request_time = base::Time::Now();
 624
 625   load_timing_info_ = LoadTimingInfo();
 626   load_timing_info_.request_start_time = response_info_.request_time;
 627   load_timing_info_.request_start = base::TimeTicks::Now();
 628
 629   // Only notify the delegate for the initial request.
 630   if (network_delegate_) {
 631     OnCallToDelegate();
 632     int error = network_delegate_->NotifyBeforeURLRequest(
 633         this, before_request_callback_, &delegate_redirect_url_);
 634     // If ERR_IO_PENDING is returned, the delegate will invoke
 635     // |before_request_callback_| later.
 636     if (error != ERR_IO_PENDING)
 637       BeforeRequestComplete(error);
 638     return;
 639   }
 640
 641   StartJob(URLRequestJobManager::GetInstance()->CreateJob(
 642       this, network_delegate_));
 643 }
 644
 645 ///////////////////////////////////////////////////////////////////////////////
 646
 647 void URLRequest::BeforeRequestComplete(int error) {
 648   DCHECK(!job_.get());
 649   DCHECK_NE(ERR_IO_PENDING, error);
 650   DCHECK_EQ(network_delegate_, context_->network_delegate());
 651
 652   // Check that there are no callbacks to already canceled requests.
 653   DCHECK_NE(URLRequestStatus::CANCELED, status_.status());
 654
 655   OnCallToDelegateComplete();
 656
 657   if (error != OK) {
 658     std::string source("delegate");
 659     net_log_.AddEvent(NetLog::TYPE_CANCELLED,
 660                       NetLog::StringCallback("source", &source));
 661     StartJob(new URLRequestErrorJob(this, network_delegate_, error));
 662   } else if (!delegate_redirect_url_.is_empty()) {
 663     GURL new_url;
 664     new_url.Swap(&delegate_redirect_url_);
 665
 666     URLRequestRedirectJob* job = new URLRequestRedirectJob(
 667         this, network_delegate_, new_url,
 668         // Use status code 307 to preserve the method, so POST requests work.
 669         URLRequestRedirectJob::REDIRECT_307_TEMPORARY_REDIRECT, "Delegate");
 670     StartJob(job);
 671   } else {
 672     StartJob(URLRequestJobManager::GetInstance()->CreateJob(
 673         this, network_delegate_));
 674   }
 675 }
 676
 677 void URLRequest::StartJob(URLRequestJob* job) {
 678   DCHECK(!is_pending_);
 679   DCHECK(!job_.get());
 680
 681   net_log_.BeginEvent(
 682       NetLog::TYPE_URL_REQUEST_START_JOB,
 683       base::Bind(&NetLogURLRequestStartCallback,
 684                  &url(), &method_, load_flags_, priority_,
 685                  upload_data_stream_ ? upload_data_stream_->identifier() : -1));
 686
 687   job_ = job;
 688   job_->SetExtraRequestHeaders(extra_request_headers_);
 689   job_->SetPriority(priority_);
 690
 691   if (upload_data_stream_.get())
 692     job_->SetUpload(upload_data_stream_.get());
 693
 694   is_pending_ = true;
 695   is_redirecting_ = false;
 696
 697   response_info_.was_cached = false;
 698
 699   // If the referrer is secure, but the requested URL is not, the referrer
 700   // policy should be something non-default. If you hit this, please file a
 701   // bug.
 702   if (referrer_policy_ ==
 703           CLEAR_REFERRER_ON_TRANSITION_FROM_SECURE_TO_INSECURE &&
 704       GURL(referrer_).SchemeIsSecure() && !url().SchemeIsSecure()) {
 705 #if !defined(OFFICIAL_BUILD)
 706     LOG(FATAL) << "Trying to send secure referrer for insecure load";
 707 #endif
 708     referrer_.clear();
 709     base::RecordAction(
 710         base::UserMetricsAction("Net.URLRequest_StartJob_InvalidReferrer"));
 711   }
 712
 713   // Don't allow errors to be sent from within Start().
 714   // TODO(brettw) this may cause NotifyDone to be sent synchronously,
 715   // we probably don't want this: they should be sent asynchronously so
 716   // the caller does not get reentered.
 717   job_->Start();
 718 }
 719
 720 void URLRequest::Restart() {
 721   // Should only be called if the original job didn't make any progress.
 722   DCHECK(job_.get() && !job_->has_response_started());
 723   RestartWithJob(
 724       URLRequestJobManager::GetInstance()->CreateJob(this, network_delegate_));
 725 }
 726
 727 void URLRequest::RestartWithJob(URLRequestJob *job) {
 728   DCHECK(job->request() == this);
 729   PrepareToRestart();
 730   StartJob(job);
 731 }
 732
 733 void URLRequest::Cancel() {
 734   DoCancel(ERR_ABORTED, SSLInfo());
 735 }
 736
 737 void URLRequest::CancelWithError(int error) {
 738   DoCancel(error, SSLInfo());
 739 }
 740
 741 void URLRequest::CancelWithSSLError(int error, const SSLInfo& ssl_info) {
 742   // This should only be called on a started request.
 743   if (!is_pending_ || !job_.get() || job_->has_response_started()) {
 744     NOTREACHED();
 745     return;
 746   }
 747   DoCancel(error, ssl_info);
 748 }
 749
 750 void URLRequest::DoCancel(int error, const SSLInfo& ssl_info) {
 751   DCHECK(error < 0);
 752   // If cancelled while calling a delegate, clear delegate info.
 753   if (calling_delegate_) {
 754     LogUnblocked();
 755     OnCallToDelegateComplete();
 756   }
 757
 758   // If the URL request already has an error status, then canceling is a no-op.
 759   // Plus, we don't want to change the error status once it has been set.
 760   if (status_.is_success()) {
 761     status_.set_status(URLRequestStatus::CANCELED);
 762     status_.set_error(error);
 763     response_info_.ssl_info = ssl_info;
 764
 765     // If the request hasn't already been completed, log a cancellation event.
 766     if (!has_notified_completion_) {
 767       // Don't log an error code on ERR_ABORTED, since that's redundant.
 768       net_log_.AddEventWithNetErrorCode(NetLog::TYPE_CANCELLED,
 769                                         error == ERR_ABORTED ? OK : error);
 770     }
 771   }
 772
 773   if (is_pending_ && job_.get())
 774     job_->Kill();
 775
 776   // We need to notify about the end of this job here synchronously. The
 777   // Job sends an asynchronous notification but by the time this is processed,
 778   // our |context_| is NULL.
 779   NotifyRequestCompleted();
 780
 781   // The Job will call our NotifyDone method asynchronously.  This is done so
 782   // that the Delegate implementation can call Cancel without having to worry
 783   // about being called recursively.
 784 }
 785
 786 bool URLRequest::Read(IOBuffer* dest, int dest_size, int* bytes_read) {
 787   DCHECK(job_.get());
 788   DCHECK(bytes_read);
 789   *bytes_read = 0;
 790
 791   // If this is the first read, end the delegate call that may have started in
 792   // OnResponseStarted.
 793   OnCallToDelegateComplete();
 794
 795   // This handles a cancel that happens while paused.
 796   // TODO(ahendrickson): DCHECK() that it is not done after
 797   // http://crbug.com/115705 is fixed.
 798   if (job_->is_done())
 799     return false;
 800
 801   if (dest_size == 0) {
 802     // Caller is not too bright.  I guess we've done what they asked.
 803     return true;
 804   }
 805
 806   // Once the request fails or is cancelled, read will just return 0 bytes
 807   // to indicate end of stream.
 808   if (!status_.is_success()) {
 809     return true;
 810   }
 811
 812   bool rv = job_->Read(dest, dest_size, bytes_read);
 813   // If rv is false, the status cannot be success.
 814   DCHECK(rv || status_.status() != URLRequestStatus::SUCCESS);
 815   if (rv && *bytes_read <= 0 && status_.is_success())
 816     NotifyRequestCompleted();
 817   return rv;
 818 }
 819
 820 void URLRequest::StopCaching() {
 821   DCHECK(job_.get());
 822   job_->StopCaching();
 823 }
 824
 825 void URLRequest::NotifyReceivedRedirect(const GURL& location,
 826                                         bool* defer_redirect) {
 827   is_redirecting_ = true;
 828
 829   URLRequestJob* job =
 830       URLRequestJobManager::GetInstance()->MaybeInterceptRedirect(
 831           this, network_delegate_, location);
 832   if (job) {
 833     RestartWithJob(job);
 834   } else if (delegate_) {
 835     OnCallToDelegate();
 836     delegate_->OnReceivedRedirect(this, location, defer_redirect);
 837     // |this| may be have been destroyed here.
 838   }
 839 }
 840
 841 void URLRequest::NotifyBeforeNetworkStart(bool* defer) {
 842   if (delegate_ && !notified_before_network_start_) {
 843     OnCallToDelegate();
 844     delegate_->OnBeforeNetworkStart(this, defer);
 845     if (!*defer)
 846       OnCallToDelegateComplete();
 847     notified_before_network_start_ = true;
 848   }
 849 }
 850
 851 void URLRequest::ResumeNetworkStart() {
 852   DCHECK(job_);
 853   DCHECK(notified_before_network_start_);
 854
 855   OnCallToDelegateComplete();
 856   job_->ResumeNetworkStart();
 857 }
 858
 859 void URLRequest::NotifyResponseStarted() {
 860   int net_error = OK;
 861   if (!status_.is_success())
 862     net_error = status_.error();
 863   net_log_.EndEventWithNetErrorCode(NetLog::TYPE_URL_REQUEST_START_JOB,
 864                                     net_error);
 865
 866   URLRequestJob* job =
 867       URLRequestJobManager::GetInstance()->MaybeInterceptResponse(
 868           this, network_delegate_);
 869   if (job) {
 870     RestartWithJob(job);
 871   } else {
 872     if (delegate_) {
 873       // In some cases (e.g. an event was canceled), we might have sent the
 874       // completion event and receive a NotifyResponseStarted() later.
 875       if (!has_notified_completion_ && status_.is_success()) {
 876         if (network_delegate_)
 877           network_delegate_->NotifyResponseStarted(this);
 878       }
 879
 880       // Notify in case the entire URL Request has been finished.
 881       if (!has_notified_completion_ && !status_.is_success())
 882         NotifyRequestCompleted();
 883
 884       OnCallToDelegate();
 885       delegate_->OnResponseStarted(this);
 886       // Nothing may appear below this line as OnResponseStarted may delete
 887       // |this|.
 888     }
 889   }
 890 }
 891
 892 void URLRequest::FollowDeferredRedirect() {
 893   CHECK(job_.get());
 894   CHECK(status_.is_success());
 895
 896   job_->FollowDeferredRedirect();
 897 }
 898
 899 void URLRequest::SetAuth(const AuthCredentials& credentials) {
 900   DCHECK(job_.get());
 901   DCHECK(job_->NeedsAuth());
 902
 903   job_->SetAuth(credentials);
 904 }
 905
 906 void URLRequest::CancelAuth() {
 907   DCHECK(job_.get());
 908   DCHECK(job_->NeedsAuth());
 909
 910   job_->CancelAuth();
 911 }
 912
 913 void URLRequest::ContinueWithCertificate(X509Certificate* client_cert) {
 914   DCHECK(job_.get());
 915
 916   job_->ContinueWithCertificate(client_cert);
 917 }
 918
 919 void URLRequest::ContinueDespiteLastError() {
 920   DCHECK(job_.get());
 921
 922   job_->ContinueDespiteLastError();
 923 }
 924
 925 void URLRequest::PrepareToRestart() {
 926   DCHECK(job_.get());
 927
 928   // Close the current URL_REQUEST_START_JOB, since we will be starting a new
 929   // one.
 930   net_log_.EndEvent(NetLog::TYPE_URL_REQUEST_START_JOB);
 931
 932   OrphanJob();
 933
 934   response_info_ = HttpResponseInfo();
 935   response_info_.request_time = base::Time::Now();
 936
 937   load_timing_info_ = LoadTimingInfo();
 938   load_timing_info_.request_start_time = response_info_.request_time;
 939   load_timing_info_.request_start = base::TimeTicks::Now();
 940
 941   status_ = URLRequestStatus();
 942   is_pending_ = false;
 943 }
 944
 945 void URLRequest::OrphanJob() {
 946   // When calling this function, please check that URLRequestHttpJob is
 947   // not in between calling NetworkDelegate::NotifyHeadersReceived receiving
 948   // the call back. This is currently guaranteed by the following strategies:
 949   // - OrphanJob is called on JobRestart, in this case the URLRequestJob cannot
 950   //   be receiving any headers at that time.
 951   // - OrphanJob is called in ~URLRequest, in this case
 952   //   NetworkDelegate::NotifyURLRequestDestroyed notifies the NetworkDelegate
 953   //   that the callback becomes invalid.
 954   job_->Kill();
 955   job_->DetachRequest();  // ensures that the job will not call us again
 956   job_ = NULL;
 957 }
 958
 959 int URLRequest::Redirect(const GURL& location, int http_status_code) {
 960   // Matches call in NotifyReceivedRedirect.
 961   OnCallToDelegateComplete();
 962   if (net_log_.IsLogging()) {
 963     net_log_.AddEvent(
 964         NetLog::TYPE_URL_REQUEST_REDIRECTED,
 965         NetLog::StringCallback("location", &location.possibly_invalid_spec()));
 966   }
 967
 968   if (network_delegate_)
 969     network_delegate_->NotifyBeforeRedirect(this, location);
 970
 971   if (redirect_limit_ <= 0) {
 972     DVLOG(1) << "disallowing redirect: exceeds limit";
 973     return ERR_TOO_MANY_REDIRECTS;
 974   }
 975
 976   if (!location.is_valid())
 977     return ERR_INVALID_URL;
 978
 979   if (!job_->IsSafeRedirect(location)) {
 980     DVLOG(1) << "disallowing redirect: unsafe protocol";
 981     return ERR_UNSAFE_REDIRECT;
 982   }
 983
 984   if (!final_upload_progress_.position())
 985     final_upload_progress_ = job_->GetUploadProgress();
 986   PrepareToRestart();
 987
 988   std::string new_method(ComputeMethodForRedirect(method_, http_status_code));
 989   if (new_method != method_) {
 990     if (method_ == "POST") {
 991       // If being switched from POST, must remove headers that were specific to
 992       // the POST and don't have meaning in other methods. For example the
 993       // inclusion of a multipart Content-Type header in GET can cause problems
 994       // with some servers:
 995       // http://code.google.com/p/chromium/issues/detail?id=843
 996       StripPostSpecificHeaders(&extra_request_headers_);
 997     }
 998     upload_data_stream_.reset();
 999     method_.swap(new_method);
1000   }
1001
1002   // Suppress the referrer if we're redirecting out of https.
1003   if (referrer_policy_ ==
1004           CLEAR_REFERRER_ON_TRANSITION_FROM_SECURE_TO_INSECURE &&
1005       GURL(referrer_).SchemeIsSecure() && !location.SchemeIsSecure()) {
1006     referrer_.clear();
1007   }
1008
1009   url_chain_.push_back(location);
1010   --redirect_limit_;
1011
1012   Start();
1013   return OK;
1014 }
1015
1016 const URLRequestContext* URLRequest::context() const {
1017   return context_;
1018 }
1019
1020 int64 URLRequest::GetExpectedContentSize() const {
1021   int64 expected_content_size = -1;
1022   if (job_.get())
1023     expected_content_size = job_->expected_content_size();
1024
1025   return expected_content_size;
1026 }
1027
1028 void URLRequest::SetPriority(RequestPriority priority) {
1029   DCHECK_GE(priority, MINIMUM_PRIORITY);
1030   DCHECK_LE(priority, MAXIMUM_PRIORITY);
1031
1032   if ((load_flags_ & LOAD_IGNORE_LIMITS) && (priority != MAXIMUM_PRIORITY)) {
1033     NOTREACHED();
1034     // Maintain the invariant that requests with IGNORE_LIMITS set
1035     // have MAXIMUM_PRIORITY for release mode.
1036     return;
1037   }
1038
1039   if (priority_ == priority)
1040     return;
1041
1042   priority_ = priority;
1043   if (job_.get()) {
1044     net_log_.AddEvent(NetLog::TYPE_URL_REQUEST_SET_PRIORITY,
1045                       NetLog::IntegerCallback("priority", priority_));
1046     job_->SetPriority(priority_);
1047   }
1048 }
1049
1050 bool URLRequest::GetHSTSRedirect(GURL* redirect_url) const {
1051   const GURL& url = this->url();
1052   if (!url.SchemeIs("http"))
1053     return false;
1054   TransportSecurityState* state = context()->transport_security_state();
1055   if (state &&
1056       state->ShouldUpgradeToSSL(
1057           url.host(),
1058           SSLConfigService::IsSNIAvailable(context()->ssl_config_service()))) {
1059     url::Replacements<char> replacements;
1060     const char kNewScheme[] = "https";
1061     replacements.SetScheme(kNewScheme, url::Component(0, strlen(kNewScheme)));
1062     *redirect_url = url.ReplaceComponents(replacements);
1063     return true;
1064   }
1065   return false;
1066 }
1067
1068 void URLRequest::NotifyAuthRequired(AuthChallengeInfo* auth_info) {
1069   NetworkDelegate::AuthRequiredResponse rv =
1070       NetworkDelegate::AUTH_REQUIRED_RESPONSE_NO_ACTION;
1071   auth_info_ = auth_info;
1072   if (network_delegate_) {
1073     OnCallToDelegate();
1074     rv = network_delegate_->NotifyAuthRequired(
1075         this,
1076         *auth_info,
1077         base::Bind(&URLRequest::NotifyAuthRequiredComplete,
1078                    base::Unretained(this)),
1079         &auth_credentials_);
1080     if (rv == NetworkDelegate::AUTH_REQUIRED_RESPONSE_IO_PENDING)
1081       return;
1082   }
1083
1084   NotifyAuthRequiredComplete(rv);
1085 }
1086
1087 void URLRequest::NotifyAuthRequiredComplete(
1088     NetworkDelegate::AuthRequiredResponse result) {
1089   OnCallToDelegateComplete();
1090
1091   // Check that there are no callbacks to already canceled requests.
1092   DCHECK_NE(URLRequestStatus::CANCELED, status_.status());
1093
1094   // NotifyAuthRequired may be called multiple times, such as
1095   // when an authentication attempt fails. Clear out the data
1096   // so it can be reset on another round.
1097   AuthCredentials credentials = auth_credentials_;
1098   auth_credentials_ = AuthCredentials();
1099   scoped_refptr<AuthChallengeInfo> auth_info;
1100   auth_info.swap(auth_info_);
1101
1102   switch (result) {
1103     case NetworkDelegate::AUTH_REQUIRED_RESPONSE_NO_ACTION:
1104       // Defer to the URLRequest::Delegate, since the NetworkDelegate
1105       // didn't take an action.
1106       if (delegate_)
1107         delegate_->OnAuthRequired(this, auth_info.get());
1108       break;
1109
1110     case NetworkDelegate::AUTH_REQUIRED_RESPONSE_SET_AUTH:
1111       SetAuth(credentials);
1112       break;
1113
1114     case NetworkDelegate::AUTH_REQUIRED_RESPONSE_CANCEL_AUTH:
1115       CancelAuth();
1116       break;
1117
1118     case NetworkDelegate::AUTH_REQUIRED_RESPONSE_IO_PENDING:
1119       NOTREACHED();
1120       break;
1121   }
1122 }
1123
1124 void URLRequest::NotifyCertificateRequested(
1125     SSLCertRequestInfo* cert_request_info) {
1126   if (delegate_)
1127     delegate_->OnCertificateRequested(this, cert_request_info);
1128 }
1129
1130 void URLRequest::NotifySSLCertificateError(const SSLInfo& ssl_info,
1131                                            bool fatal) {
1132   if (delegate_)
1133     delegate_->OnSSLCertificateError(this, ssl_info, fatal);
1134 }
1135
1136 bool URLRequest::CanGetCookies(const CookieList& cookie_list) const {
1137   DCHECK(!(load_flags_ & LOAD_DO_NOT_SEND_COOKIES));
1138   if (network_delegate_) {
1139     return network_delegate_->CanGetCookies(*this, cookie_list);
1140   }
1141   return g_default_can_use_cookies;
1142 }
1143
1144 bool URLRequest::CanSetCookie(const std::string& cookie_line,
1145                               CookieOptions* options) const {
1146   DCHECK(!(load_flags_ & LOAD_DO_NOT_SAVE_COOKIES));
1147   if (network_delegate_) {
1148     return network_delegate_->CanSetCookie(*this, cookie_line, options);
1149   }
1150   return g_default_can_use_cookies;
1151 }
1152
1153 bool URLRequest::CanEnablePrivacyMode() const {
1154   if (network_delegate_) {
1155     return network_delegate_->CanEnablePrivacyMode(url(),
1156                                                    first_party_for_cookies_);
1157   }
1158   return !g_default_can_use_cookies;
1159 }
1160
1161
1162 void URLRequest::NotifyReadCompleted(int bytes_read) {
1163   // Notify in case the entire URL Request has been finished.
1164   if (bytes_read <= 0)
1165     NotifyRequestCompleted();
1166
1167   // Notify NetworkChangeNotifier that we just received network data.
1168   // This is to identify cases where the NetworkChangeNotifier thinks we
1169   // are off-line but we are still receiving network data (crbug.com/124069),
1170   // and to get rough network connection measurements.
1171   if (bytes_read > 0 && !was_cached())
1172     NetworkChangeNotifier::NotifyDataReceived(*this, bytes_read);
1173
1174   if (delegate_)
1175     delegate_->OnReadCompleted(this, bytes_read);
1176
1177   // Nothing below this line as OnReadCompleted may delete |this|.
1178 }
1179
1180 void URLRequest::OnHeadersComplete() {
1181   // Cache load timing information now, as information will be lost once the
1182   // socket is closed and the ClientSocketHandle is Reset, which will happen
1183   // once the body is complete.  The start times should already be populated.
1184   if (job_.get()) {
1185     // Keep a copy of the two times the URLRequest sets.
1186     base::TimeTicks request_start = load_timing_info_.request_start;
1187     base::Time request_start_time = load_timing_info_.request_start_time;
1188
1189     // Clear load times.  Shouldn't be neded, but gives the GetLoadTimingInfo a
1190     // consistent place to start from.
1191     load_timing_info_ = LoadTimingInfo();
1192     job_->GetLoadTimingInfo(&load_timing_info_);
1193
1194     load_timing_info_.request_start = request_start;
1195     load_timing_info_.request_start_time = request_start_time;
1196
1197     ConvertRealLoadTimesToBlockingTimes(&load_timing_info_);
1198   }
1199 }
1200
1201 void URLRequest::NotifyRequestCompleted() {
1202   // TODO(battre): Get rid of this check, according to willchan it should
1203   // not be needed.
1204   if (has_notified_completion_)
1205     return;
1206
1207   is_pending_ = false;
1208   is_redirecting_ = false;
1209   has_notified_completion_ = true;
1210   if (network_delegate_)
1211     network_delegate_->NotifyCompleted(this, job_.get() != NULL);
1212 }
1213
1214 void URLRequest::OnCallToDelegate() {
1215   DCHECK(!calling_delegate_);
1216   DCHECK(blocked_by_.empty());
1217   calling_delegate_ = true;
1218   net_log_.BeginEvent(NetLog::TYPE_URL_REQUEST_DELEGATE);
1219 }
1220
1221 void URLRequest::OnCallToDelegateComplete() {
1222   // This should have been cleared before resuming the request.
1223   DCHECK(blocked_by_.empty());
1224   if (!calling_delegate_)
1225     return;
1226   calling_delegate_ = false;
1227   net_log_.EndEvent(NetLog::TYPE_URL_REQUEST_DELEGATE);
1228 }
1229
1230 void URLRequest::set_stack_trace(const base::debug::StackTrace& stack_trace) {
1231   base::debug::StackTrace* stack_trace_copy =
1232       new base::debug::StackTrace(NULL, 0);
1233   *stack_trace_copy = stack_trace;
1234   stack_trace_.reset(stack_trace_copy);
1235 }
1236
1237 const base::debug::StackTrace* URLRequest::stack_trace() const {
1238   return stack_trace_.get();
1239 }
1240
1241 }  // namespace net