net/url_request/url_request_http_job.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_request_http_job.h"
   6
   7 #include "base/base_switches.h"
   8 #include "base/bind.h"
   9 #include "base/bind_helpers.h"
  10 #include "base/command_line.h"
  11 #include "base/compiler_specific.h"
  12 #include "base/file_version_info.h"
  13 #include "base/location.h"
  14 #include "base/metrics/field_trial.h"
  15 #include "base/metrics/histogram_macros.h"
  16 #include "base/profiler/scoped_tracker.h"
  17 #include "base/rand_util.h"
  18 #include "base/single_thread_task_runner.h"
  19 #include "base/strings/string_util.h"
  20 #include "base/thread_task_runner_handle.h"
  21 #include "base/time/time.h"
  22 #include "base/values.h"
  23 #include "net/base/host_port_pair.h"
  24 #include "net/base/load_flags.h"
  25 #include "net/base/net_errors.h"
  26 #include "net/base/net_util.h"
  27 #include "net/base/network_delegate.h"
  28 #include "net/base/network_quality_estimator.h"
  29 #include "net/base/sdch_manager.h"
  30 #include "net/base/sdch_net_log_params.h"
  31 #include "net/cert/cert_status_flags.h"
  32 #include "net/cookies/cookie_store.h"
  33 #include "net/http/http_content_disposition.h"
  34 #include "net/http/http_network_session.h"
  35 #include "net/http/http_request_headers.h"
  36 #include "net/http/http_response_headers.h"
  37 #include "net/http/http_response_info.h"
  38 #include "net/http/http_status_code.h"
  39 #include "net/http/http_transaction.h"
  40 #include "net/http/http_transaction_factory.h"
  41 #include "net/http/http_util.h"
  42 #include "net/proxy/proxy_info.h"
  43 #include "net/ssl/ssl_cert_request_info.h"
  44 #include "net/ssl/ssl_config_service.h"
  45 #include "net/url_request/http_user_agent_settings.h"
  46 #include "net/url_request/url_request.h"
  47 #include "net/url_request/url_request_backoff_manager.h"
  48 #include "net/url_request/url_request_context.h"
  49 #include "net/url_request/url_request_error_job.h"
  50 #include "net/url_request/url_request_job_factory.h"
  51 #include "net/url_request/url_request_redirect_job.h"
  52 #include "net/url_request/url_request_throttler_manager.h"
  53 #include "net/websockets/websocket_handshake_stream_base.h"
  54
  55 static const char kAvailDictionaryHeader[] = "Avail-Dictionary";
  56
  57 namespace net {
  58
  59 class URLRequestHttpJob::HttpFilterContext : public FilterContext {
  60  public:
  61   explicit HttpFilterContext(URLRequestHttpJob* job);
  62   ~HttpFilterContext() override;
  63
  64   // FilterContext implementation.
  65   bool GetMimeType(std::string* mime_type) const override;
  66   bool GetURL(GURL* gurl) const override;
  67   base::Time GetRequestTime() const override;
  68   bool IsCachedContent() const override;
  69   SdchManager::DictionarySet* SdchDictionariesAdvertised() const override;
  70   int64 GetByteReadCount() const override;
  71   int GetResponseCode() const override;
  72   const URLRequestContext* GetURLRequestContext() const override;
  73   void RecordPacketStats(StatisticSelector statistic) const override;
  74   const BoundNetLog& GetNetLog() const override;
  75
  76  private:
  77   URLRequestHttpJob* job_;
  78
  79   // URLRequestHttpJob may be detached from URLRequest, but we still need to
  80   // return something.
  81   BoundNetLog dummy_log_;
  82
  83   DISALLOW_COPY_AND_ASSIGN(HttpFilterContext);
  84 };
  85
  86 URLRequestHttpJob::HttpFilterContext::HttpFilterContext(URLRequestHttpJob* job)
  87     : job_(job) {
  88   DCHECK(job_);
  89 }
  90
  91 URLRequestHttpJob::HttpFilterContext::~HttpFilterContext() {
  92 }
  93
  94 bool URLRequestHttpJob::HttpFilterContext::GetMimeType(
  95     std::string* mime_type) const {
  96   return job_->GetMimeType(mime_type);
  97 }
  98
  99 bool URLRequestHttpJob::HttpFilterContext::GetURL(GURL* gurl) const {
 100   if (!job_->request())
 101     return false;
 102   *gurl = job_->request()->url();
 103   return true;
 104 }
 105
 106 base::Time URLRequestHttpJob::HttpFilterContext::GetRequestTime() const {
 107   return job_->request() ? job_->request()->request_time() : base::Time();
 108 }
 109
 110 bool URLRequestHttpJob::HttpFilterContext::IsCachedContent() const {
 111   return job_->is_cached_content_;
 112 }
 113
 114 SdchManager::DictionarySet*
 115 URLRequestHttpJob::HttpFilterContext::SdchDictionariesAdvertised() const {
 116   return job_->dictionaries_advertised_.get();
 117 }
 118
 119 int64 URLRequestHttpJob::HttpFilterContext::GetByteReadCount() const {
 120   return job_->prefilter_bytes_read();
 121 }
 122
 123 int URLRequestHttpJob::HttpFilterContext::GetResponseCode() const {
 124   return job_->GetResponseCode();
 125 }
 126
 127 const URLRequestContext*
 128 URLRequestHttpJob::HttpFilterContext::GetURLRequestContext() const {
 129   return job_->request() ? job_->request()->context() : NULL;
 130 }
 131
 132 void URLRequestHttpJob::HttpFilterContext::RecordPacketStats(
 133     StatisticSelector statistic) const {
 134   job_->RecordPacketStats(statistic);
 135 }
 136
 137 const BoundNetLog& URLRequestHttpJob::HttpFilterContext::GetNetLog() const {
 138   return job_->request() ? job_->request()->net_log() : dummy_log_;
 139 }
 140
 141 // TODO(darin): make sure the port blocking code is not lost
 142 // static
 143 URLRequestJob* URLRequestHttpJob::Factory(URLRequest* request,
 144                                           NetworkDelegate* network_delegate,
 145                                           const std::string& scheme) {
 146   DCHECK(scheme == "http" || scheme == "https" || scheme == "ws" ||
 147          scheme == "wss");
 148
 149   if (!request->context()->http_transaction_factory()) {
 150     NOTREACHED() << "requires a valid context";
 151     return new URLRequestErrorJob(
 152         request, network_delegate, ERR_INVALID_ARGUMENT);
 153   }
 154
 155   GURL redirect_url;
 156   if (request->GetHSTSRedirect(&redirect_url)) {
 157     return new URLRequestRedirectJob(
 158         request, network_delegate, redirect_url,
 159         // Use status code 307 to preserve the method, so POST requests work.
 160         URLRequestRedirectJob::REDIRECT_307_TEMPORARY_REDIRECT, "HSTS");
 161   }
 162   return new URLRequestHttpJob(request,
 163                                network_delegate,
 164                                request->context()->http_user_agent_settings());
 165 }
 166
 167 URLRequestHttpJob::URLRequestHttpJob(
 168     URLRequest* request,
 169     NetworkDelegate* network_delegate,
 170     const HttpUserAgentSettings* http_user_agent_settings)
 171     : URLRequestJob(request, network_delegate),
 172       priority_(DEFAULT_PRIORITY),
 173       response_info_(NULL),
 174       response_cookies_save_index_(0),
 175       proxy_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
 176       server_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
 177       start_callback_(base::Bind(&URLRequestHttpJob::OnStartCompleted,
 178                                  base::Unretained(this))),
 179       notify_before_headers_sent_callback_(
 180           base::Bind(&URLRequestHttpJob::NotifyBeforeSendHeadersCallback,
 181                      base::Unretained(this))),
 182       read_in_progress_(false),
 183       throttling_entry_(NULL),
 184       sdch_test_activated_(false),
 185       sdch_test_control_(false),
 186       is_cached_content_(false),
 187       request_creation_time_(),
 188       packet_timing_enabled_(false),
 189       done_(false),
 190       bytes_observed_in_packets_(0),
 191       request_time_snapshot_(),
 192       final_packet_time_(),
 193       filter_context_(new HttpFilterContext(this)),
 194       on_headers_received_callback_(
 195           base::Bind(&URLRequestHttpJob::OnHeadersReceivedCallback,
 196                      base::Unretained(this))),
 197       awaiting_callback_(false),
 198       http_user_agent_settings_(http_user_agent_settings),
 199       backoff_manager_(request->context()->backoff_manager()),
 200       total_received_bytes_from_previous_transactions_(0),
 201       weak_factory_(this) {
 202   URLRequestThrottlerManager* manager = request->context()->throttler_manager();
 203   if (manager)
 204     throttling_entry_ = manager->RegisterRequestUrl(request->url());
 205
 206   ResetTimer();
 207 }
 208
 209 URLRequestHttpJob::~URLRequestHttpJob() {
 210   CHECK(!awaiting_callback_);
 211
 212   DCHECK(!sdch_test_control_ || !sdch_test_activated_);
 213   if (!is_cached_content_) {
 214     if (sdch_test_control_)
 215       RecordPacketStats(FilterContext::SDCH_EXPERIMENT_HOLDBACK);
 216     if (sdch_test_activated_)
 217       RecordPacketStats(FilterContext::SDCH_EXPERIMENT_DECODE);
 218   }
 219   // Make sure SDCH filters are told to emit histogram data while
 220   // filter_context_ is still alive.
 221   DestroyFilters();
 222
 223   DoneWithRequest(ABORTED);
 224 }
 225
 226 void URLRequestHttpJob::SetPriority(RequestPriority priority) {
 227   priority_ = priority;
 228   if (transaction_)
 229     transaction_->SetPriority(priority_);
 230 }
 231
 232 void URLRequestHttpJob::Start() {
 233   // TODO(mmenke): Remove ScopedTracker below once crbug.com/456327 is fixed.
 234   tracked_objects::ScopedTracker tracking_profile(
 235       FROM_HERE_WITH_EXPLICIT_FUNCTION("456327 URLRequestHttpJob::Start"));
 236
 237   DCHECK(!transaction_.get());
 238
 239   // URLRequest::SetReferrer ensures that we do not send username and password
 240   // fields in the referrer.
 241   GURL referrer(request_->referrer());
 242
 243   request_info_.url = request_->url();
 244   request_info_.method = request_->method();
 245   request_info_.load_flags = request_->load_flags();
 246   // Enable privacy mode if cookie settings or flags tell us not send or
 247   // save cookies.
 248   bool enable_privacy_mode =
 249       (request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES) ||
 250       (request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) ||
 251       CanEnablePrivacyMode();
 252   // Privacy mode could still be disabled in OnCookiesLoaded if we are going
 253   // to send previously saved cookies.
 254   request_info_.privacy_mode = enable_privacy_mode ?
 255       PRIVACY_MODE_ENABLED : PRIVACY_MODE_DISABLED;
 256
 257   // Strip Referer from request_info_.extra_headers to prevent, e.g., plugins
 258   // from overriding headers that are controlled using other means. Otherwise a
 259   // plugin could set a referrer although sending the referrer is inhibited.
 260   request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kReferer);
 261
 262   // Our consumer should have made sure that this is a safe referrer. See for
 263   // instance WebCore::FrameLoader::HideReferrer.
 264   if (referrer.is_valid()) {
 265     request_info_.extra_headers.SetHeader(HttpRequestHeaders::kReferer,
 266                                           referrer.spec());
 267   }
 268
 269   request_info_.extra_headers.SetHeaderIfMissing(
 270       HttpRequestHeaders::kUserAgent,
 271       http_user_agent_settings_ ?
 272           http_user_agent_settings_->GetUserAgent() : std::string());
 273
 274   AddExtraHeaders();
 275   AddCookieHeaderAndStart();
 276 }
 277
 278 void URLRequestHttpJob::Kill() {
 279   if (!transaction_.get())
 280     return;
 281
 282   weak_factory_.InvalidateWeakPtrs();
 283   DestroyTransaction();
 284   URLRequestJob::Kill();
 285 }
 286
 287 void URLRequestHttpJob::GetConnectionAttempts(ConnectionAttempts* out) const {
 288   if (transaction_)
 289     transaction_->GetConnectionAttempts(out);
 290   else
 291     out->clear();
 292 }
 293
 294 void URLRequestHttpJob::NotifyBeforeSendProxyHeadersCallback(
 295     const ProxyInfo& proxy_info,
 296     HttpRequestHeaders* request_headers) {
 297   DCHECK(request_headers);
 298   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 299   if (network_delegate()) {
 300     network_delegate()->NotifyBeforeSendProxyHeaders(
 301         request_,
 302         proxy_info,
 303         request_headers);
 304   }
 305 }
 306
 307 void URLRequestHttpJob::NotifyBeforeNetworkStart(bool* defer) {
 308   if (!request_)
 309     return;
 310   if (backoff_manager_) {
 311     if (backoff_manager_->ShouldRejectRequest(request()->url(),
 312                                               request()->request_time())) {
 313       *defer = true;
 314       base::MessageLoop::current()->PostTask(
 315           FROM_HERE,
 316           base::Bind(&URLRequestHttpJob::OnStartCompleted,
 317                      weak_factory_.GetWeakPtr(), ERR_TEMPORARY_BACKOFF));
 318       return;
 319     }
 320   }
 321   URLRequestJob::NotifyBeforeNetworkStart(defer);
 322 }
 323
 324 void URLRequestHttpJob::NotifyHeadersComplete() {
 325   DCHECK(!response_info_);
 326
 327   response_info_ = transaction_->GetResponseInfo();
 328
 329   // Save boolean, as we'll need this info at destruction time, and filters may
 330   // also need this info.
 331   is_cached_content_ = response_info_->was_cached;
 332
 333   if (!is_cached_content_ && throttling_entry_.get())
 334     throttling_entry_->UpdateWithResponse(GetResponseCode());
 335
 336   if (!is_cached_content_)
 337     ProcessBackoffHeader();
 338
 339   // The ordering of these calls is not important.
 340   ProcessStrictTransportSecurityHeader();
 341   ProcessPublicKeyPinsHeader();
 342
 343   // Handle the server notification of a new SDCH dictionary.
 344   SdchManager* sdch_manager(request()->context()->sdch_manager());
 345   if (sdch_manager) {
 346     SdchProblemCode rv = sdch_manager->IsInSupportedDomain(request()->url());
 347     if (rv != SDCH_OK) {
 348       SdchManager::SdchErrorRecovery(rv);
 349       request()->net_log().AddEvent(
 350           NetLog::TYPE_SDCH_DECODING_ERROR,
 351           base::Bind(&NetLogSdchResourceProblemCallback, rv));
 352     } else {
 353       const std::string name = "Get-Dictionary";
 354       std::string url_text;
 355       void* iter = NULL;
 356       // TODO(jar): We need to not fetch dictionaries the first time they are
 357       // seen, but rather wait until we can justify their usefulness.
 358       // For now, we will only fetch the first dictionary, which will at least
 359       // require multiple suggestions before we get additional ones for this
 360       // site. Eventually we should wait until a dictionary is requested
 361       // several times
 362       // before we even download it (so that we don't waste memory or
 363       // bandwidth).
 364       if (GetResponseHeaders()->EnumerateHeader(&iter, name, &url_text)) {
 365         // Resolve suggested URL relative to request url.
 366         GURL sdch_dictionary_url = request_->url().Resolve(url_text);
 367         if (sdch_dictionary_url.is_valid()) {
 368           rv = sdch_manager->OnGetDictionary(request_->url(),
 369                                              sdch_dictionary_url);
 370           if (rv != SDCH_OK) {
 371             SdchManager::SdchErrorRecovery(rv);
 372             request_->net_log().AddEvent(
 373                 NetLog::TYPE_SDCH_DICTIONARY_ERROR,
 374                 base::Bind(&NetLogSdchDictionaryFetchProblemCallback, rv,
 375                            sdch_dictionary_url, false));
 376           }
 377         }
 378       }
 379     }
 380   }
 381
 382   // Handle the server signalling no SDCH encoding.
 383   if (dictionaries_advertised_) {
 384     // We are wary of proxies that discard or damage SDCH encoding. If a server
 385     // explicitly states that this is not SDCH content, then we can correct our
 386     // assumption that this is an SDCH response, and avoid the need to recover
 387     // as though the content is corrupted (when we discover it is not SDCH
 388     // encoded).
 389     std::string sdch_response_status;
 390     void* iter = NULL;
 391     while (GetResponseHeaders()->EnumerateHeader(&iter, "X-Sdch-Encode",
 392                                                  &sdch_response_status)) {
 393       if (sdch_response_status == "0") {
 394         dictionaries_advertised_.reset();
 395         break;
 396       }
 397     }
 398   }
 399
 400   // The HTTP transaction may be restarted several times for the purposes
 401   // of sending authorization information. Each time it restarts, we get
 402   // notified of the headers completion so that we can update the cookie store.
 403   if (transaction_->IsReadyToRestartForAuth()) {
 404     DCHECK(!response_info_->auth_challenge.get());
 405     // TODO(battre): This breaks the webrequest API for
 406     // URLRequestTestHTTP.BasicAuthWithCookies
 407     // where OnBeforeSendHeaders -> OnSendHeaders -> OnBeforeSendHeaders
 408     // occurs.
 409     RestartTransactionWithAuth(AuthCredentials());
 410     return;
 411   }
 412
 413   URLRequestJob::NotifyHeadersComplete();
 414 }
 415
 416 void URLRequestHttpJob::NotifyDone(const URLRequestStatus& status) {
 417   DoneWithRequest(FINISHED);
 418   URLRequestJob::NotifyDone(status);
 419 }
 420
 421 void URLRequestHttpJob::DestroyTransaction() {
 422   DCHECK(transaction_.get());
 423
 424   DoneWithRequest(ABORTED);
 425
 426   total_received_bytes_from_previous_transactions_ +=
 427       transaction_->GetTotalReceivedBytes();
 428   transaction_.reset();
 429   response_info_ = NULL;
 430   receive_headers_end_ = base::TimeTicks();
 431 }
 432
 433 void URLRequestHttpJob::StartTransaction() {
 434   // TODO(mmenke): Remove ScopedTracker below once crbug.com/456327 is fixed.
 435   tracked_objects::ScopedTracker tracking_profile(
 436       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 437           "456327 URLRequestHttpJob::StartTransaction"));
 438
 439   if (network_delegate()) {
 440     OnCallToDelegate();
 441     int rv = network_delegate()->NotifyBeforeSendHeaders(
 442         request_, notify_before_headers_sent_callback_,
 443         &request_info_.extra_headers);
 444     // If an extension blocks the request, we rely on the callback to
 445     // MaybeStartTransactionInternal().
 446     if (rv == ERR_IO_PENDING)
 447       return;
 448     MaybeStartTransactionInternal(rv);
 449     return;
 450   }
 451   StartTransactionInternal();
 452 }
 453
 454 void URLRequestHttpJob::NotifyBeforeSendHeadersCallback(int result) {
 455   // Check that there are no callbacks to already canceled requests.
 456   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 457
 458   MaybeStartTransactionInternal(result);
 459 }
 460
 461 void URLRequestHttpJob::MaybeStartTransactionInternal(int result) {
 462   // TODO(mmenke): Remove ScopedTracker below once crbug.com/456327 is fixed.
 463   tracked_objects::ScopedTracker tracking_profile(
 464       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 465           "456327 URLRequestHttpJob::MaybeStartTransactionInternal"));
 466
 467   OnCallToDelegateComplete();
 468   if (result == OK) {
 469     StartTransactionInternal();
 470   } else {
 471     std::string source("delegate");
 472     request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 473                                  NetLog::StringCallback("source", &source));
 474     NotifyCanceled();
 475     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 476   }
 477 }
 478
 479 void URLRequestHttpJob::StartTransactionInternal() {
 480   // NOTE: This method assumes that request_info_ is already setup properly.
 481
 482   // If we already have a transaction, then we should restart the transaction
 483   // with auth provided by auth_credentials_.
 484
 485   int rv;
 486
 487   if (network_delegate()) {
 488     network_delegate()->NotifySendHeaders(
 489         request_, request_info_.extra_headers);
 490   }
 491
 492   if (transaction_.get()) {
 493     rv = transaction_->RestartWithAuth(auth_credentials_, start_callback_);
 494     auth_credentials_ = AuthCredentials();
 495   } else {
 496     DCHECK(request_->context()->http_transaction_factory());
 497
 498     rv = request_->context()->http_transaction_factory()->CreateTransaction(
 499         priority_, &transaction_);
 500
 501     if (rv == OK && request_info_.url.SchemeIsWSOrWSS()) {
 502       base::SupportsUserData::Data* data = request_->GetUserData(
 503           WebSocketHandshakeStreamBase::CreateHelper::DataKey());
 504       if (data) {
 505         transaction_->SetWebSocketHandshakeStreamCreateHelper(
 506             static_cast<WebSocketHandshakeStreamBase::CreateHelper*>(data));
 507       } else {
 508         rv = ERR_DISALLOWED_URL_SCHEME;
 509       }
 510     }
 511
 512     if (rv == OK) {
 513       transaction_->SetBeforeNetworkStartCallback(
 514           base::Bind(&URLRequestHttpJob::NotifyBeforeNetworkStart,
 515                      base::Unretained(this)));
 516       transaction_->SetBeforeProxyHeadersSentCallback(
 517           base::Bind(&URLRequestHttpJob::NotifyBeforeSendProxyHeadersCallback,
 518                      base::Unretained(this)));
 519
 520       if (!throttling_entry_.get() ||
 521           !throttling_entry_->ShouldRejectRequest(*request_)) {
 522         rv = transaction_->Start(
 523             &request_info_, start_callback_, request_->net_log());
 524         start_time_ = base::TimeTicks::Now();
 525       } else {
 526         // Special error code for the exponential back-off module.
 527         rv = ERR_TEMPORARILY_THROTTLED;
 528       }
 529     }
 530   }
 531
 532   if (rv == ERR_IO_PENDING)
 533     return;
 534
 535   // The transaction started synchronously, but we need to notify the
 536   // URLRequest delegate via the message loop.
 537   base::ThreadTaskRunnerHandle::Get()->PostTask(
 538       FROM_HERE, base::Bind(&URLRequestHttpJob::OnStartCompleted,
 539                             weak_factory_.GetWeakPtr(), rv));
 540 }
 541
 542 void URLRequestHttpJob::AddExtraHeaders() {
 543   SdchManager* sdch_manager = request()->context()->sdch_manager();
 544
 545   // Supply Accept-Encoding field only if it is not already provided.
 546   // It should be provided IF the content is known to have restrictions on
 547   // potential encoding, such as streaming multi-media.
 548   // For details see bug 47381.
 549   // TODO(jar, enal): jpeg files etc. should set up a request header if
 550   // possible. Right now it is done only by buffered_resource_loader and
 551   // simple_data_source.
 552   if (!request_info_.extra_headers.HasHeader(
 553       HttpRequestHeaders::kAcceptEncoding)) {
 554     // We don't support SDCH responses to POST as there is a possibility
 555     // of having SDCH encoded responses returned (e.g. by the cache)
 556     // which we cannot decode, and in those situations, we will need
 557     // to retransmit the request without SDCH, which is illegal for a POST.
 558     bool advertise_sdch = sdch_manager != NULL && request()->method() != "POST";
 559     if (advertise_sdch) {
 560       SdchProblemCode rv = sdch_manager->IsInSupportedDomain(request()->url());
 561       if (rv != SDCH_OK) {
 562         advertise_sdch = false;
 563         SdchManager::SdchErrorRecovery(rv);
 564         request()->net_log().AddEvent(
 565             NetLog::TYPE_SDCH_DECODING_ERROR,
 566             base::Bind(&NetLogSdchResourceProblemCallback, rv));
 567       }
 568     }
 569     if (advertise_sdch) {
 570       dictionaries_advertised_ =
 571           sdch_manager->GetDictionarySet(request_->url());
 572     }
 573
 574     // The AllowLatencyExperiment() is only true if we've successfully done a
 575     // full SDCH compression recently in this browser session for this host.
 576     // Note that for this path, there might be no applicable dictionaries,
 577     // and hence we can't participate in the experiment.
 578     if (dictionaries_advertised_ &&
 579         sdch_manager->AllowLatencyExperiment(request_->url())) {
 580       // We are participating in the test (or control), and hence we'll
 581       // eventually record statistics via either SDCH_EXPERIMENT_DECODE or
 582       // SDCH_EXPERIMENT_HOLDBACK, and we'll need some packet timing data.
 583       packet_timing_enabled_ = true;
 584       if (base::RandDouble() < .01) {
 585         sdch_test_control_ = true;  // 1% probability.
 586         dictionaries_advertised_.reset();
 587         advertise_sdch = false;
 588       } else {
 589         sdch_test_activated_ = true;
 590       }
 591     }
 592
 593     // Supply Accept-Encoding headers first so that it is more likely that they
 594     // will be in the first transmitted packet. This can sometimes make it
 595     // easier to filter and analyze the streams to assure that a proxy has not
 596     // damaged these headers. Some proxies deliberately corrupt Accept-Encoding
 597     // headers.
 598     if (!advertise_sdch) {
 599       // Tell the server what compression formats we support (other than SDCH).
 600       request_info_.extra_headers.SetHeader(
 601           HttpRequestHeaders::kAcceptEncoding, "gzip, deflate");
 602     } else {
 603       // Include SDCH in acceptable list.
 604       request_info_.extra_headers.SetHeader(
 605           HttpRequestHeaders::kAcceptEncoding, "gzip, deflate, sdch");
 606       if (dictionaries_advertised_) {
 607         request_info_.extra_headers.SetHeader(
 608             kAvailDictionaryHeader,
 609             dictionaries_advertised_->GetDictionaryClientHashList());
 610         // Since we're tagging this transaction as advertising a dictionary,
 611         // we'll definitely employ an SDCH filter (or tentative sdch filter)
 612         // when we get a response. When done, we'll record histograms via
 613         // SDCH_DECODE or SDCH_PASSTHROUGH. Hence we need to record packet
 614         // arrival times.
 615         packet_timing_enabled_ = true;
 616       }
 617     }
 618   }
 619
 620   if (http_user_agent_settings_) {
 621     // Only add default Accept-Language if the request didn't have it
 622     // specified.
 623     std::string accept_language =
 624         http_user_agent_settings_->GetAcceptLanguage();
 625     if (!accept_language.empty()) {
 626       request_info_.extra_headers.SetHeaderIfMissing(
 627           HttpRequestHeaders::kAcceptLanguage,
 628           accept_language);
 629     }
 630   }
 631 }
 632
 633 void URLRequestHttpJob::AddCookieHeaderAndStart() {
 634   // No matter what, we want to report our status as IO pending since we will
 635   // be notifying our consumer asynchronously via OnStartCompleted.
 636   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
 637
 638   // If the request was destroyed, then there is no more work to do.
 639   if (!request_)
 640     return;
 641
 642   CookieStore* cookie_store = request_->context()->cookie_store();
 643   if (cookie_store && !(request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES)) {
 644     cookie_store->GetAllCookiesForURLAsync(
 645         request_->url(),
 646         base::Bind(&URLRequestHttpJob::CheckCookiePolicyAndLoad,
 647                    weak_factory_.GetWeakPtr()));
 648   } else {
 649     DoStartTransaction();
 650   }
 651 }
 652
 653 void URLRequestHttpJob::DoLoadCookies() {
 654   CookieOptions options;
 655   options.set_include_httponly();
 656
 657   // TODO(mkwst): Drop this `if` once we decide whether or not to ship
 658   // first-party cookies: https://crbug.com/459154
 659   if (network_delegate() &&
 660       network_delegate()->FirstPartyOnlyCookieExperimentEnabled())
 661     options.set_first_party_url(request_->first_party_for_cookies());
 662   else
 663     options.set_include_first_party_only();
 664
 665   request_->context()->cookie_store()->GetCookiesWithOptionsAsync(
 666       request_->url(), options, base::Bind(&URLRequestHttpJob::OnCookiesLoaded,
 667                                            weak_factory_.GetWeakPtr()));
 668 }
 669
 670 void URLRequestHttpJob::CheckCookiePolicyAndLoad(
 671     const CookieList& cookie_list) {
 672   if (CanGetCookies(cookie_list))
 673     DoLoadCookies();
 674   else
 675     DoStartTransaction();
 676 }
 677
 678 void URLRequestHttpJob::OnCookiesLoaded(const std::string& cookie_line) {
 679   if (!cookie_line.empty()) {
 680     request_info_.extra_headers.SetHeader(
 681         HttpRequestHeaders::kCookie, cookie_line);
 682     // Disable privacy mode as we are sending cookies anyway.
 683     request_info_.privacy_mode = PRIVACY_MODE_DISABLED;
 684   }
 685   DoStartTransaction();
 686 }
 687
 688 void URLRequestHttpJob::DoStartTransaction() {
 689   // We may have been canceled while retrieving cookies.
 690   if (GetStatus().is_success()) {
 691     StartTransaction();
 692   } else {
 693     NotifyCanceled();
 694   }
 695 }
 696
 697 void URLRequestHttpJob::SaveCookiesAndNotifyHeadersComplete(int result) {
 698   // End of the call started in OnStartCompleted.
 699   OnCallToDelegateComplete();
 700
 701   if (result != OK) {
 702     std::string source("delegate");
 703     request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 704                                  NetLog::StringCallback("source", &source));
 705     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 706     return;
 707   }
 708
 709   DCHECK(transaction_.get());
 710
 711   const HttpResponseInfo* response_info = transaction_->GetResponseInfo();
 712   DCHECK(response_info);
 713
 714   response_cookies_.clear();
 715   response_cookies_save_index_ = 0;
 716
 717   FetchResponseCookies(&response_cookies_);
 718
 719   if (!GetResponseHeaders()->GetDateValue(&response_date_))
 720     response_date_ = base::Time();
 721
 722   // Now, loop over the response cookies, and attempt to persist each.
 723   SaveNextCookie();
 724 }
 725
 726 // If the save occurs synchronously, SaveNextCookie will loop and save the next
 727 // cookie. If the save is deferred, the callback is responsible for continuing
 728 // to iterate through the cookies.
 729 // TODO(erikwright): Modify the CookieStore API to indicate via return value
 730 // whether it completed synchronously or asynchronously.
 731 // See http://crbug.com/131066.
 732 void URLRequestHttpJob::SaveNextCookie() {
 733   // No matter what, we want to report our status as IO pending since we will
 734   // be notifying our consumer asynchronously via OnStartCompleted.
 735   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
 736
 737   // Used to communicate with the callback. See the implementation of
 738   // OnCookieSaved.
 739   scoped_refptr<SharedBoolean> callback_pending = new SharedBoolean(false);
 740   scoped_refptr<SharedBoolean> save_next_cookie_running =
 741       new SharedBoolean(true);
 742
 743   if (!(request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) &&
 744       request_->context()->cookie_store() && response_cookies_.size() > 0) {
 745     CookieOptions options;
 746     options.set_include_httponly();
 747     options.set_server_time(response_date_);
 748
 749     CookieStore::SetCookiesCallback callback(base::Bind(
 750         &URLRequestHttpJob::OnCookieSaved, weak_factory_.GetWeakPtr(),
 751         save_next_cookie_running, callback_pending));
 752
 753     // Loop through the cookies as long as SetCookieWithOptionsAsync completes
 754     // synchronously.
 755     while (!callback_pending->data &&
 756            response_cookies_save_index_ < response_cookies_.size()) {
 757       if (CanSetCookie(
 758           response_cookies_[response_cookies_save_index_], &options)) {
 759         callback_pending->data = true;
 760         request_->context()->cookie_store()->SetCookieWithOptionsAsync(
 761             request_->url(), response_cookies_[response_cookies_save_index_],
 762             options, callback);
 763       }
 764       ++response_cookies_save_index_;
 765     }
 766   }
 767
 768   save_next_cookie_running->data = false;
 769
 770   if (!callback_pending->data) {
 771     response_cookies_.clear();
 772     response_cookies_save_index_ = 0;
 773     SetStatus(URLRequestStatus());  // Clear the IO_PENDING status
 774     NotifyHeadersComplete();
 775     return;
 776   }
 777 }
 778
 779 // |save_next_cookie_running| is true when the callback is bound and set to
 780 // false when SaveNextCookie exits, allowing the callback to determine if the
 781 // save occurred synchronously or asynchronously.
 782 // |callback_pending| is false when the callback is invoked and will be set to
 783 // true by the callback, allowing SaveNextCookie to detect whether the save
 784 // occurred synchronously.
 785 // See SaveNextCookie() for more information.
 786 void URLRequestHttpJob::OnCookieSaved(
 787     scoped_refptr<SharedBoolean> save_next_cookie_running,
 788     scoped_refptr<SharedBoolean> callback_pending,
 789     bool cookie_status) {
 790   callback_pending->data = false;
 791
 792   // If we were called synchronously, return.
 793   if (save_next_cookie_running->data) {
 794     return;
 795   }
 796
 797   // We were called asynchronously, so trigger the next save.
 798   // We may have been canceled within OnSetCookie.
 799   if (GetStatus().is_success()) {
 800     SaveNextCookie();
 801   } else {
 802     NotifyCanceled();
 803   }
 804 }
 805
 806 void URLRequestHttpJob::FetchResponseCookies(
 807     std::vector<std::string>* cookies) {
 808   const std::string name = "Set-Cookie";
 809   std::string value;
 810
 811   void* iter = NULL;
 812   HttpResponseHeaders* headers = GetResponseHeaders();
 813   while (headers->EnumerateHeader(&iter, name, &value)) {
 814     if (!value.empty())
 815       cookies->push_back(value);
 816   }
 817 }
 818
 819 void URLRequestHttpJob::ProcessBackoffHeader() {
 820   DCHECK(response_info_);
 821
 822   if (!backoff_manager_)
 823     return;
 824
 825   TransportSecurityState* security_state =
 826       request_->context()->transport_security_state();
 827   const SSLInfo& ssl_info = response_info_->ssl_info;
 828
 829   // Only accept Backoff headers on HTTPS connections that have no
 830   // certificate errors.
 831   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 832       !security_state)
 833     return;
 834
 835   backoff_manager_->UpdateWithResponse(request()->url(), GetResponseHeaders(),
 836                                        base::Time::Now());
 837 }
 838
 839 // NOTE: |ProcessStrictTransportSecurityHeader| and
 840 // |ProcessPublicKeyPinsHeader| have very similar structures, by design.
 841 void URLRequestHttpJob::ProcessStrictTransportSecurityHeader() {
 842   DCHECK(response_info_);
 843   TransportSecurityState* security_state =
 844       request_->context()->transport_security_state();
 845   const SSLInfo& ssl_info = response_info_->ssl_info;
 846
 847   // Only accept HSTS headers on HTTPS connections that have no
 848   // certificate errors.
 849   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 850       !security_state)
 851     return;
 852
 853   // Don't accept HSTS headers when the hostname is an IP address.
 854   if (request_info_.url.HostIsIPAddress())
 855     return;
 856
 857   // http://tools.ietf.org/html/draft-ietf-websec-strict-transport-sec:
 858   //
 859   //   If a UA receives more than one STS header field in a HTTP response
 860   //   message over secure transport, then the UA MUST process only the
 861   //   first such header field.
 862   HttpResponseHeaders* headers = GetResponseHeaders();
 863   std::string value;
 864   if (headers->EnumerateHeader(NULL, "Strict-Transport-Security", &value))
 865     security_state->AddHSTSHeader(request_info_.url.host(), value);
 866 }
 867
 868 void URLRequestHttpJob::ProcessPublicKeyPinsHeader() {
 869   DCHECK(response_info_);
 870   TransportSecurityState* security_state =
 871       request_->context()->transport_security_state();
 872   const SSLInfo& ssl_info = response_info_->ssl_info;
 873
 874   // Only accept HPKP headers on HTTPS connections that have no
 875   // certificate errors.
 876   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 877       !security_state)
 878     return;
 879
 880   // Don't accept HSTS headers when the hostname is an IP address.
 881   if (request_info_.url.HostIsIPAddress())
 882     return;
 883
 884   // http://tools.ietf.org/html/rfc7469:
 885   //
 886   //   If a UA receives more than one PKP header field in an HTTP
 887   //   response message over secure transport, then the UA MUST process
 888   //   only the first such header field.
 889   HttpResponseHeaders* headers = GetResponseHeaders();
 890   std::string value;
 891   if (headers->EnumerateHeader(nullptr, "Public-Key-Pins", &value))
 892     security_state->AddHPKPHeader(request_info_.url.host(), value, ssl_info);
 893   if (headers->EnumerateHeader(nullptr, "Public-Key-Pins-Report-Only",
 894                                &value)) {
 895     security_state->ProcessHPKPReportOnlyHeader(
 896         value, HostPortPair::FromURL(request_info_.url), ssl_info);
 897   }
 898 }
 899
 900 void URLRequestHttpJob::OnStartCompleted(int result) {
 901   RecordTimer();
 902
 903   // If the request was destroyed, then there is no more work to do.
 904   if (!request_)
 905     return;
 906
 907   // If the job is done (due to cancellation), can just ignore this
 908   // notification.
 909   if (done_)
 910     return;
 911
 912   receive_headers_end_ = base::TimeTicks::Now();
 913
 914   // Clear the IO_PENDING status
 915   SetStatus(URLRequestStatus());
 916
 917   const URLRequestContext* context = request_->context();
 918
 919   if (result == OK) {
 920     if (transaction_ && transaction_->GetResponseInfo()) {
 921       SetProxyServer(transaction_->GetResponseInfo()->proxy_server);
 922     }
 923     scoped_refptr<HttpResponseHeaders> headers = GetResponseHeaders();
 924     if (network_delegate()) {
 925       // Note that |this| may not be deleted until
 926       // |on_headers_received_callback_| or
 927       // |NetworkDelegate::URLRequestDestroyed()| has been called.
 928       OnCallToDelegate();
 929       allowed_unsafe_redirect_url_ = GURL();
 930       int error = network_delegate()->NotifyHeadersReceived(
 931           request_,
 932           on_headers_received_callback_,
 933           headers.get(),
 934           &override_response_headers_,
 935           &allowed_unsafe_redirect_url_);
 936       if (error != OK) {
 937         if (error == ERR_IO_PENDING) {
 938           awaiting_callback_ = true;
 939         } else {
 940           std::string source("delegate");
 941           request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 942                                        NetLog::StringCallback("source",
 943                                                               &source));
 944           OnCallToDelegateComplete();
 945           NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, error));
 946         }
 947         return;
 948       }
 949     }
 950
 951     SaveCookiesAndNotifyHeadersComplete(OK);
 952   } else if (IsCertificateError(result)) {
 953     // We encountered an SSL certificate error.
 954     if (result == ERR_SSL_WEAK_SERVER_EPHEMERAL_DH_KEY ||
 955         result == ERR_SSL_PINNED_KEY_NOT_IN_CERT_CHAIN) {
 956       // These are hard failures. They're handled separately and don't have
 957       // the correct cert status, so set it here.
 958       SSLInfo info(transaction_->GetResponseInfo()->ssl_info);
 959       info.cert_status = MapNetErrorToCertStatus(result);
 960       NotifySSLCertificateError(info, true);
 961     } else {
 962       // Maybe overridable, maybe not. Ask the delegate to decide.
 963       TransportSecurityState* state = context->transport_security_state();
 964       const bool fatal =
 965           state && state->ShouldSSLErrorsBeFatal(request_info_.url.host());
 966       NotifySSLCertificateError(
 967           transaction_->GetResponseInfo()->ssl_info, fatal);
 968     }
 969   } else if (result == ERR_SSL_CLIENT_AUTH_CERT_NEEDED) {
 970     NotifyCertificateRequested(
 971         transaction_->GetResponseInfo()->cert_request_info.get());
 972   } else {
 973     // Even on an error, there may be useful information in the response
 974     // info (e.g. whether there's a cached copy).
 975     if (transaction_.get())
 976       response_info_ = transaction_->GetResponseInfo();
 977     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 978   }
 979 }
 980
 981 void URLRequestHttpJob::OnHeadersReceivedCallback(int result) {
 982   awaiting_callback_ = false;
 983
 984   // Check that there are no callbacks to already canceled requests.
 985   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 986
 987   SaveCookiesAndNotifyHeadersComplete(result);
 988 }
 989
 990 void URLRequestHttpJob::OnReadCompleted(int result) {
 991   read_in_progress_ = false;
 992
 993   if (ShouldFixMismatchedContentLength(result))
 994     result = OK;
 995
 996   if (result == OK) {
 997     NotifyDone(URLRequestStatus());
 998   } else if (result < 0) {
 999     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, result));
1000   } else {
1001     // Clear the IO_PENDING status
1002     SetStatus(URLRequestStatus());
1003   }
1004
1005   NotifyReadComplete(result);
1006 }
1007
1008 void URLRequestHttpJob::RestartTransactionWithAuth(
1009     const AuthCredentials& credentials) {
1010   auth_credentials_ = credentials;
1011
1012   // These will be reset in OnStartCompleted.
1013   response_info_ = NULL;
1014   receive_headers_end_ = base::TimeTicks();
1015   response_cookies_.clear();
1016
1017   ResetTimer();
1018
1019   // Update the cookies, since the cookie store may have been updated from the
1020   // headers in the 401/407. Since cookies were already appended to
1021   // extra_headers, we need to strip them out before adding them again.
1022   request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kCookie);
1023
1024   AddCookieHeaderAndStart();
1025 }
1026
1027 void URLRequestHttpJob::SetUpload(UploadDataStream* upload) {
1028   DCHECK(!transaction_.get()) << "cannot change once started";
1029   request_info_.upload_data_stream = upload;
1030 }
1031
1032 void URLRequestHttpJob::SetExtraRequestHeaders(
1033     const HttpRequestHeaders& headers) {
1034   DCHECK(!transaction_.get()) << "cannot change once started";
1035   request_info_.extra_headers.CopyFrom(headers);
1036 }
1037
1038 LoadState URLRequestHttpJob::GetLoadState() const {
1039   return transaction_.get() ?
1040       transaction_->GetLoadState() : LOAD_STATE_IDLE;
1041 }
1042
1043 UploadProgress URLRequestHttpJob::GetUploadProgress() const {
1044   return transaction_.get() ?
1045       transaction_->GetUploadProgress() : UploadProgress();
1046 }
1047
1048 bool URLRequestHttpJob::GetMimeType(std::string* mime_type) const {
1049   DCHECK(transaction_.get());
1050
1051   if (!response_info_)
1052     return false;
1053
1054   HttpResponseHeaders* headers = GetResponseHeaders();
1055   if (!headers)
1056     return false;
1057   return headers->GetMimeType(mime_type);
1058 }
1059
1060 bool URLRequestHttpJob::GetCharset(std::string* charset) {
1061   DCHECK(transaction_.get());
1062
1063   if (!response_info_)
1064     return false;
1065
1066   return GetResponseHeaders()->GetCharset(charset);
1067 }
1068
1069 void URLRequestHttpJob::GetResponseInfo(HttpResponseInfo* info) {
1070   DCHECK(request_);
1071
1072   if (response_info_) {
1073     DCHECK(transaction_.get());
1074
1075     *info = *response_info_;
1076     if (override_response_headers_.get())
1077       info->headers = override_response_headers_;
1078   }
1079 }
1080
1081 void URLRequestHttpJob::GetLoadTimingInfo(
1082     LoadTimingInfo* load_timing_info) const {
1083   // If haven't made it far enough to receive any headers, don't return
1084   // anything. This makes for more consistent behavior in the case of errors.
1085   if (!transaction_ || receive_headers_end_.is_null())
1086     return;
1087   if (transaction_->GetLoadTimingInfo(load_timing_info))
1088     load_timing_info->receive_headers_end = receive_headers_end_;
1089 }
1090
1091 bool URLRequestHttpJob::GetResponseCookies(std::vector<std::string>* cookies) {
1092   DCHECK(transaction_.get());
1093
1094   if (!response_info_)
1095     return false;
1096
1097   // TODO(darin): Why are we extracting response cookies again?  Perhaps we
1098   // should just leverage response_cookies_.
1099
1100   cookies->clear();
1101   FetchResponseCookies(cookies);
1102   return true;
1103 }
1104
1105 int URLRequestHttpJob::GetResponseCode() const {
1106   DCHECK(transaction_.get());
1107
1108   if (!response_info_)
1109     return -1;
1110
1111   return GetResponseHeaders()->response_code();
1112 }
1113
1114 Filter* URLRequestHttpJob::SetupFilter() const {
1115   DCHECK(transaction_.get());
1116   if (!response_info_)
1117     return NULL;
1118
1119   std::vector<Filter::FilterType> encoding_types;
1120   std::string encoding_type;
1121   HttpResponseHeaders* headers = GetResponseHeaders();
1122   void* iter = NULL;
1123   while (headers->EnumerateHeader(&iter, "Content-Encoding", &encoding_type)) {
1124     encoding_types.push_back(Filter::ConvertEncodingToType(encoding_type));
1125   }
1126
1127   // Even if encoding types are empty, there is a chance that we need to add
1128   // some decoding, as some proxies strip encoding completely. In such cases,
1129   // we may need to add (for example) SDCH filtering (when the context suggests
1130   // it is appropriate).
1131   Filter::FixupEncodingTypes(*filter_context_, &encoding_types);
1132
1133   return !encoding_types.empty()
1134       ? Filter::Factory(encoding_types, *filter_context_) : NULL;
1135 }
1136
1137 bool URLRequestHttpJob::CopyFragmentOnRedirect(const GURL& location) const {
1138   // Allow modification of reference fragments by default, unless
1139   // |allowed_unsafe_redirect_url_| is set and equal to the redirect URL.
1140   // When this is the case, we assume that the network delegate has set the
1141   // desired redirect URL (with or without fragment), so it must not be changed
1142   // any more.
1143   return !allowed_unsafe_redirect_url_.is_valid() ||
1144        allowed_unsafe_redirect_url_ != location;
1145 }
1146
1147 bool URLRequestHttpJob::IsSafeRedirect(const GURL& location) {
1148   // HTTP is always safe.
1149   // TODO(pauljensen): Remove once crbug.com/146591 is fixed.
1150   if (location.is_valid() &&
1151       (location.scheme() == "http" || location.scheme() == "https")) {
1152     return true;
1153   }
1154   // Delegates may mark a URL as safe for redirection.
1155   if (allowed_unsafe_redirect_url_.is_valid() &&
1156       allowed_unsafe_redirect_url_ == location) {
1157     return true;
1158   }
1159   // Query URLRequestJobFactory as to whether |location| would be safe to
1160   // redirect to.
1161   return request_->context()->job_factory() &&
1162       request_->context()->job_factory()->IsSafeRedirectTarget(location);
1163 }
1164
1165 bool URLRequestHttpJob::NeedsAuth() {
1166   int code = GetResponseCode();
1167   if (code == -1)
1168     return false;
1169
1170   // Check if we need either Proxy or WWW Authentication. This could happen
1171   // because we either provided no auth info, or provided incorrect info.
1172   switch (code) {
1173     case 407:
1174       if (proxy_auth_state_ == AUTH_STATE_CANCELED)
1175         return false;
1176       proxy_auth_state_ = AUTH_STATE_NEED_AUTH;
1177       return true;
1178     case 401:
1179       if (server_auth_state_ == AUTH_STATE_CANCELED)
1180         return false;
1181       server_auth_state_ = AUTH_STATE_NEED_AUTH;
1182       return true;
1183   }
1184   return false;
1185 }
1186
1187 void URLRequestHttpJob::GetAuthChallengeInfo(
1188     scoped_refptr<AuthChallengeInfo>* result) {
1189   DCHECK(transaction_.get());
1190   DCHECK(response_info_);
1191
1192   // sanity checks:
1193   DCHECK(proxy_auth_state_ == AUTH_STATE_NEED_AUTH ||
1194          server_auth_state_ == AUTH_STATE_NEED_AUTH);
1195   DCHECK((GetResponseHeaders()->response_code() == HTTP_UNAUTHORIZED) ||
1196          (GetResponseHeaders()->response_code() ==
1197           HTTP_PROXY_AUTHENTICATION_REQUIRED));
1198
1199   *result = response_info_->auth_challenge;
1200 }
1201
1202 void URLRequestHttpJob::SetAuth(const AuthCredentials& credentials) {
1203   DCHECK(transaction_.get());
1204
1205   // Proxy gets set first, then WWW.
1206   if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1207     proxy_auth_state_ = AUTH_STATE_HAVE_AUTH;
1208   } else {
1209     DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1210     server_auth_state_ = AUTH_STATE_HAVE_AUTH;
1211   }
1212
1213   RestartTransactionWithAuth(credentials);
1214 }
1215
1216 void URLRequestHttpJob::CancelAuth() {
1217   // Proxy gets set first, then WWW.
1218   if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1219     proxy_auth_state_ = AUTH_STATE_CANCELED;
1220   } else {
1221     DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1222     server_auth_state_ = AUTH_STATE_CANCELED;
1223   }
1224
1225   // These will be reset in OnStartCompleted.
1226   response_info_ = NULL;
1227   receive_headers_end_ = base::TimeTicks::Now();
1228   response_cookies_.clear();
1229
1230   ResetTimer();
1231
1232   // OK, let the consumer read the error page...
1233   //
1234   // Because we set the AUTH_STATE_CANCELED flag, NeedsAuth will return false,
1235   // which will cause the consumer to receive OnResponseStarted instead of
1236   // OnAuthRequired.
1237   //
1238   // We have to do this via InvokeLater to avoid "recursing" the consumer.
1239   //
1240   base::ThreadTaskRunnerHandle::Get()->PostTask(
1241       FROM_HERE, base::Bind(&URLRequestHttpJob::OnStartCompleted,
1242                             weak_factory_.GetWeakPtr(), OK));
1243 }
1244
1245 void URLRequestHttpJob::ContinueWithCertificate(
1246     X509Certificate* client_cert) {
1247   DCHECK(transaction_.get());
1248
1249   DCHECK(!response_info_) << "should not have a response yet";
1250   receive_headers_end_ = base::TimeTicks();
1251
1252   ResetTimer();
1253
1254   // No matter what, we want to report our status as IO pending since we will
1255   // be notifying our consumer asynchronously via OnStartCompleted.
1256   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1257
1258   int rv = transaction_->RestartWithCertificate(client_cert, start_callback_);
1259   if (rv == ERR_IO_PENDING)
1260     return;
1261
1262   // The transaction started synchronously, but we need to notify the
1263   // URLRequest delegate via the message loop.
1264   base::ThreadTaskRunnerHandle::Get()->PostTask(
1265       FROM_HERE, base::Bind(&URLRequestHttpJob::OnStartCompleted,
1266                             weak_factory_.GetWeakPtr(), rv));
1267 }
1268
1269 void URLRequestHttpJob::ContinueDespiteLastError() {
1270   // If the transaction was destroyed, then the job was cancelled.
1271   if (!transaction_.get())
1272     return;
1273
1274   DCHECK(!response_info_) << "should not have a response yet";
1275   receive_headers_end_ = base::TimeTicks();
1276
1277   ResetTimer();
1278
1279   // No matter what, we want to report our status as IO pending since we will
1280   // be notifying our consumer asynchronously via OnStartCompleted.
1281   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1282
1283   int rv = transaction_->RestartIgnoringLastError(start_callback_);
1284   if (rv == ERR_IO_PENDING)
1285     return;
1286
1287   // The transaction started synchronously, but we need to notify the
1288   // URLRequest delegate via the message loop.
1289   base::ThreadTaskRunnerHandle::Get()->PostTask(
1290       FROM_HERE, base::Bind(&URLRequestHttpJob::OnStartCompleted,
1291                             weak_factory_.GetWeakPtr(), rv));
1292 }
1293
1294 void URLRequestHttpJob::ResumeNetworkStart() {
1295   DCHECK(transaction_.get());
1296   transaction_->ResumeNetworkStart();
1297 }
1298
1299 bool URLRequestHttpJob::ShouldFixMismatchedContentLength(int rv) const {
1300   // Some servers send the body compressed, but specify the content length as
1301   // the uncompressed size. Although this violates the HTTP spec we want to
1302   // support it (as IE and FireFox do), but *only* for an exact match.
1303   // See http://crbug.com/79694.
1304   if (rv == ERR_CONTENT_LENGTH_MISMATCH ||
1305       rv == ERR_INCOMPLETE_CHUNKED_ENCODING) {
1306     if (request_ && request_->response_headers()) {
1307       int64 expected_length = request_->response_headers()->GetContentLength();
1308       VLOG(1) << __FUNCTION__ << "() "
1309               << "\"" << request_->url().spec() << "\""
1310               << " content-length = " << expected_length
1311               << " pre total = " << prefilter_bytes_read()
1312               << " post total = " << postfilter_bytes_read();
1313       if (postfilter_bytes_read() == expected_length) {
1314         // Clear the error.
1315         return true;
1316       }
1317     }
1318   }
1319   return false;
1320 }
1321
1322 bool URLRequestHttpJob::ReadRawData(IOBuffer* buf, int buf_size,
1323                                     int* bytes_read) {
1324   DCHECK_NE(buf_size, 0);
1325   DCHECK(bytes_read);
1326   DCHECK(!read_in_progress_);
1327
1328   int rv = transaction_->Read(
1329       buf, buf_size,
1330       base::Bind(&URLRequestHttpJob::OnReadCompleted, base::Unretained(this)));
1331
1332   if (ShouldFixMismatchedContentLength(rv))
1333     rv = 0;
1334
1335   if (rv >= 0) {
1336     *bytes_read = rv;
1337     if (!rv)
1338       DoneWithRequest(FINISHED);
1339     return true;
1340   }
1341
1342   if (rv == ERR_IO_PENDING) {
1343     read_in_progress_ = true;
1344     SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1345   } else {
1346     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, rv));
1347   }
1348
1349   return false;
1350 }
1351
1352 void URLRequestHttpJob::StopCaching() {
1353   if (transaction_.get())
1354     transaction_->StopCaching();
1355 }
1356
1357 bool URLRequestHttpJob::GetFullRequestHeaders(
1358     HttpRequestHeaders* headers) const {
1359   if (!transaction_)
1360     return false;
1361
1362   return transaction_->GetFullRequestHeaders(headers);
1363 }
1364
1365 int64 URLRequestHttpJob::GetTotalReceivedBytes() const {
1366   int64_t total_received_bytes =
1367       total_received_bytes_from_previous_transactions_;
1368   if (transaction_)
1369     total_received_bytes += transaction_->GetTotalReceivedBytes();
1370   return total_received_bytes;
1371 }
1372
1373 void URLRequestHttpJob::DoneReading() {
1374   if (transaction_) {
1375     transaction_->DoneReading();
1376   }
1377   DoneWithRequest(FINISHED);
1378 }
1379
1380 void URLRequestHttpJob::DoneReadingRedirectResponse() {
1381   if (transaction_) {
1382     if (transaction_->GetResponseInfo()->headers->IsRedirect(NULL)) {
1383       // If the original headers indicate a redirect, go ahead and cache the
1384       // response, even if the |override_response_headers_| are a redirect to
1385       // another location.
1386       transaction_->DoneReading();
1387     } else {
1388       // Otherwise, |override_response_headers_| must be non-NULL and contain
1389       // bogus headers indicating a redirect.
1390       DCHECK(override_response_headers_.get());
1391       DCHECK(override_response_headers_->IsRedirect(NULL));
1392       transaction_->StopCaching();
1393     }
1394   }
1395   DoneWithRequest(FINISHED);
1396 }
1397
1398 HostPortPair URLRequestHttpJob::GetSocketAddress() const {
1399   return response_info_ ? response_info_->socket_address : HostPortPair();
1400 }
1401
1402 void URLRequestHttpJob::RecordTimer() {
1403   if (request_creation_time_.is_null()) {
1404     NOTREACHED()
1405         << "The same transaction shouldn't start twice without new timing.";
1406     return;
1407   }
1408
1409   base::TimeDelta to_start = base::Time::Now() - request_creation_time_;
1410   request_creation_time_ = base::Time();
1411
1412   UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpTimeToFirstByte", to_start);
1413 }
1414
1415 void URLRequestHttpJob::ResetTimer() {
1416   if (!request_creation_time_.is_null()) {
1417     NOTREACHED()
1418         << "The timer was reset before it was recorded.";
1419     return;
1420   }
1421   request_creation_time_ = base::Time::Now();
1422 }
1423
1424 void URLRequestHttpJob::UpdatePacketReadTimes() {
1425   if (!packet_timing_enabled_)
1426     return;
1427
1428   DCHECK_GT(prefilter_bytes_read(), bytes_observed_in_packets_);
1429
1430   base::Time now(base::Time::Now());
1431   if (!bytes_observed_in_packets_)
1432     request_time_snapshot_ = now;
1433   final_packet_time_ = now;
1434
1435   bytes_observed_in_packets_ = prefilter_bytes_read();
1436 }
1437
1438 void URLRequestHttpJob::RecordPacketStats(
1439     FilterContext::StatisticSelector statistic) const {
1440   if (!packet_timing_enabled_ || (final_packet_time_ == base::Time()))
1441     return;
1442
1443   base::TimeDelta duration = final_packet_time_ - request_time_snapshot_;
1444   switch (statistic) {
1445     case FilterContext::SDCH_DECODE: {
1446       UMA_HISTOGRAM_CUSTOM_COUNTS("Sdch3.Network_Decode_Bytes_Processed_b",
1447           static_cast<int>(bytes_observed_in_packets_), 500, 100000, 100);
1448       return;
1449     }
1450     case FilterContext::SDCH_PASSTHROUGH: {
1451       // Despite advertising a dictionary, we handled non-sdch compressed
1452       // content.
1453       return;
1454     }
1455
1456     case FilterContext::SDCH_EXPERIMENT_DECODE: {
1457       UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment3_Decode",
1458                                   duration,
1459                                   base::TimeDelta::FromMilliseconds(20),
1460                                   base::TimeDelta::FromMinutes(10), 100);
1461       return;
1462     }
1463     case FilterContext::SDCH_EXPERIMENT_HOLDBACK: {
1464       UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment3_Holdback",
1465                                   duration,
1466                                   base::TimeDelta::FromMilliseconds(20),
1467                                   base::TimeDelta::FromMinutes(10), 100);
1468       return;
1469     }
1470     default:
1471       NOTREACHED();
1472       return;
1473   }
1474 }
1475
1476 void URLRequestHttpJob::RecordPerfHistograms(CompletionCause reason) {
1477   if (start_time_.is_null())
1478     return;
1479
1480   base::TimeDelta total_time = base::TimeTicks::Now() - start_time_;
1481   UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTime", total_time);
1482
1483   if (reason == FINISHED) {
1484     UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeSuccess", total_time);
1485   } else {
1486     UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCancel", total_time);
1487   }
1488
1489   if (response_info_) {
1490     // QUIC (by default) supports https scheme only, thus track https URLs only
1491     // for QUIC.
1492     bool is_https_google = request() && request()->url().SchemeIs("https") &&
1493                            HasGoogleHost(request()->url());
1494     bool used_quic = response_info_->DidUseQuic();
1495     if (is_https_google) {
1496       if (used_quic) {
1497         UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTime.Secure.Quic",
1498                                    total_time);
1499       } else {
1500         UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTime.Secure.NotQuic",
1501                                    total_time);
1502       }
1503     }
1504     if (response_info_->was_cached) {
1505       UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCached", total_time);
1506       if (is_https_google) {
1507         if (used_quic) {
1508           UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTimeCached.Secure.Quic",
1509                                      total_time);
1510         } else {
1511           UMA_HISTOGRAM_MEDIUM_TIMES(
1512               "Net.HttpJob.TotalTimeCached.Secure.NotQuic", total_time);
1513         }
1514       }
1515     } else  {
1516       UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeNotCached", total_time);
1517       if (is_https_google) {
1518         if (used_quic) {
1519           UMA_HISTOGRAM_MEDIUM_TIMES(
1520               "Net.HttpJob.TotalTimeNotCached.Secure.Quic", total_time);
1521         } else {
1522           UMA_HISTOGRAM_MEDIUM_TIMES(
1523               "Net.HttpJob.TotalTimeNotCached.Secure.NotQuic", total_time);
1524         }
1525       }
1526     }
1527   }
1528
1529   if (request_info_.load_flags & LOAD_PREFETCH && !request_->was_cached())
1530     UMA_HISTOGRAM_COUNTS("Net.Prefetch.PrefilterBytesReadFromNetwork",
1531                          prefilter_bytes_read());
1532
1533   start_time_ = base::TimeTicks();
1534 }
1535
1536 void URLRequestHttpJob::DoneWithRequest(CompletionCause reason) {
1537   if (done_)
1538     return;
1539   done_ = true;
1540
1541   // Notify NetworkQualityEstimator.
1542   if (request() && (reason == FINISHED || reason == ABORTED)) {
1543     NetworkQualityEstimator* network_quality_estimator =
1544         request()->context()->network_quality_estimator();
1545     if (network_quality_estimator)
1546       network_quality_estimator->NotifyRequestCompleted(*request());
1547   }
1548
1549   RecordPerfHistograms(reason);
1550   if (request_)
1551     request_->set_received_response_content_length(prefilter_bytes_read());
1552 }
1553
1554 HttpResponseHeaders* URLRequestHttpJob::GetResponseHeaders() const {
1555   DCHECK(transaction_.get());
1556   DCHECK(transaction_->GetResponseInfo());
1557   return override_response_headers_.get() ?
1558              override_response_headers_.get() :
1559              transaction_->GetResponseInfo()->headers.get();
1560 }
1561
1562 void URLRequestHttpJob::NotifyURLRequestDestroyed() {
1563   awaiting_callback_ = false;
1564 }
1565
1566 }  // namespace net