net/url_request/url_request_http_job.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_request_http_job.h"
   6
   7 #include "base/base_switches.h"
   8 #include "base/bind.h"
   9 #include "base/bind_helpers.h"
  10 #include "base/command_line.h"
  11 #include "base/compiler_specific.h"
  12 #include "base/file_version_info.h"
  13 #include "base/location.h"
  14 #include "base/metrics/field_trial.h"
  15 #include "base/metrics/histogram_macros.h"
  16 #include "base/profiler/scoped_tracker.h"
  17 #include "base/rand_util.h"
  18 #include "base/single_thread_task_runner.h"
  19 #include "base/strings/string_util.h"
  20 #include "base/thread_task_runner_handle.h"
  21 #include "base/time/time.h"
  22 #include "base/values.h"
  23 #include "net/base/host_port_pair.h"
  24 #include "net/base/load_flags.h"
  25 #include "net/base/net_errors.h"
  26 #include "net/base/net_util.h"
  27 #include "net/base/network_delegate.h"
  28 #include "net/base/network_quality_estimator.h"
  29 #include "net/base/sdch_manager.h"
  30 #include "net/base/sdch_net_log_params.h"
  31 #include "net/cert/cert_status_flags.h"
  32 #include "net/cookies/cookie_store.h"
  33 #include "net/http/http_content_disposition.h"
  34 #include "net/http/http_network_session.h"
  35 #include "net/http/http_request_headers.h"
  36 #include "net/http/http_response_headers.h"
  37 #include "net/http/http_response_info.h"
  38 #include "net/http/http_status_code.h"
  39 #include "net/http/http_transaction.h"
  40 #include "net/http/http_transaction_factory.h"
  41 #include "net/http/http_util.h"
  42 #include "net/proxy/proxy_info.h"
  43 #include "net/ssl/ssl_cert_request_info.h"
  44 #include "net/ssl/ssl_config_service.h"
  45 #include "net/url_request/http_user_agent_settings.h"
  46 #include "net/url_request/url_request.h"
  47 #include "net/url_request/url_request_backoff_manager.h"
  48 #include "net/url_request/url_request_context.h"
  49 #include "net/url_request/url_request_error_job.h"
  50 #include "net/url_request/url_request_job_factory.h"
  51 #include "net/url_request/url_request_redirect_job.h"
  52 #include "net/url_request/url_request_throttler_manager.h"
  53 #include "net/websockets/websocket_handshake_stream_base.h"
  54
  55 static const char kAvailDictionaryHeader[] = "Avail-Dictionary";
  56
  57 namespace net {
  58
  59 class URLRequestHttpJob::HttpFilterContext : public FilterContext {
  60  public:
  61   explicit HttpFilterContext(URLRequestHttpJob* job);
  62   ~HttpFilterContext() override;
  63
  64   // FilterContext implementation.
  65   bool GetMimeType(std::string* mime_type) const override;
  66   bool GetURL(GURL* gurl) const override;
  67   base::Time GetRequestTime() const override;
  68   bool IsCachedContent() const override;
  69   SdchManager::DictionarySet* SdchDictionariesAdvertised() const override;
  70   int64 GetByteReadCount() const override;
  71   int GetResponseCode() const override;
  72   const URLRequestContext* GetURLRequestContext() const override;
  73   void RecordPacketStats(StatisticSelector statistic) const override;
  74   const BoundNetLog& GetNetLog() const override;
  75
  76  private:
  77   URLRequestHttpJob* job_;
  78
  79   // URLRequestHttpJob may be detached from URLRequest, but we still need to
  80   // return something.
  81   BoundNetLog dummy_log_;
  82
  83   DISALLOW_COPY_AND_ASSIGN(HttpFilterContext);
  84 };
  85
  86 URLRequestHttpJob::HttpFilterContext::HttpFilterContext(URLRequestHttpJob* job)
  87     : job_(job) {
  88   DCHECK(job_);
  89 }
  90
  91 URLRequestHttpJob::HttpFilterContext::~HttpFilterContext() {
  92 }
  93
  94 bool URLRequestHttpJob::HttpFilterContext::GetMimeType(
  95     std::string* mime_type) const {
  96   return job_->GetMimeType(mime_type);
  97 }
  98
  99 bool URLRequestHttpJob::HttpFilterContext::GetURL(GURL* gurl) const {
 100   if (!job_->request())
 101     return false;
 102   *gurl = job_->request()->url();
 103   return true;
 104 }
 105
 106 base::Time URLRequestHttpJob::HttpFilterContext::GetRequestTime() const {
 107   return job_->request() ? job_->request()->request_time() : base::Time();
 108 }
 109
 110 bool URLRequestHttpJob::HttpFilterContext::IsCachedContent() const {
 111   return job_->is_cached_content_;
 112 }
 113
 114 SdchManager::DictionarySet*
 115 URLRequestHttpJob::HttpFilterContext::SdchDictionariesAdvertised() const {
 116   return job_->dictionaries_advertised_.get();
 117 }
 118
 119 int64 URLRequestHttpJob::HttpFilterContext::GetByteReadCount() const {
 120   return job_->prefilter_bytes_read();
 121 }
 122
 123 int URLRequestHttpJob::HttpFilterContext::GetResponseCode() const {
 124   return job_->GetResponseCode();
 125 }
 126
 127 const URLRequestContext*
 128 URLRequestHttpJob::HttpFilterContext::GetURLRequestContext() const {
 129   return job_->request() ? job_->request()->context() : NULL;
 130 }
 131
 132 void URLRequestHttpJob::HttpFilterContext::RecordPacketStats(
 133     StatisticSelector statistic) const {
 134   job_->RecordPacketStats(statistic);
 135 }
 136
 137 const BoundNetLog& URLRequestHttpJob::HttpFilterContext::GetNetLog() const {
 138   return job_->request() ? job_->request()->net_log() : dummy_log_;
 139 }
 140
 141 // TODO(darin): make sure the port blocking code is not lost
 142 // static
 143 URLRequestJob* URLRequestHttpJob::Factory(URLRequest* request,
 144                                           NetworkDelegate* network_delegate,
 145                                           const std::string& scheme) {
 146   DCHECK(scheme == "http" || scheme == "https" || scheme == "ws" ||
 147          scheme == "wss");
 148
 149   if (!request->context()->http_transaction_factory()) {
 150     NOTREACHED() << "requires a valid context";
 151     return new URLRequestErrorJob(
 152         request, network_delegate, ERR_INVALID_ARGUMENT);
 153   }
 154
 155   GURL redirect_url;
 156   if (request->GetHSTSRedirect(&redirect_url)) {
 157     return new URLRequestRedirectJob(
 158         request, network_delegate, redirect_url,
 159         // Use status code 307 to preserve the method, so POST requests work.
 160         URLRequestRedirectJob::REDIRECT_307_TEMPORARY_REDIRECT, "HSTS");
 161   }
 162   return new URLRequestHttpJob(request,
 163                                network_delegate,
 164                                request->context()->http_user_agent_settings());
 165 }
 166
 167 URLRequestHttpJob::URLRequestHttpJob(
 168     URLRequest* request,
 169     NetworkDelegate* network_delegate,
 170     const HttpUserAgentSettings* http_user_agent_settings)
 171     : URLRequestJob(request, network_delegate),
 172       priority_(DEFAULT_PRIORITY),
 173       response_info_(NULL),
 174       response_cookies_save_index_(0),
 175       proxy_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
 176       server_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
 177       start_callback_(base::Bind(&URLRequestHttpJob::OnStartCompleted,
 178                                  base::Unretained(this))),
 179       notify_before_headers_sent_callback_(
 180           base::Bind(&URLRequestHttpJob::NotifyBeforeSendHeadersCallback,
 181                      base::Unretained(this))),
 182       read_in_progress_(false),
 183       throttling_entry_(NULL),
 184       sdch_test_activated_(false),
 185       sdch_test_control_(false),
 186       is_cached_content_(false),
 187       request_creation_time_(),
 188       packet_timing_enabled_(false),
 189       done_(false),
 190       bytes_observed_in_packets_(0),
 191       request_time_snapshot_(),
 192       final_packet_time_(),
 193       filter_context_(new HttpFilterContext(this)),
 194       on_headers_received_callback_(
 195           base::Bind(&URLRequestHttpJob::OnHeadersReceivedCallback,
 196                      base::Unretained(this))),
 197       awaiting_callback_(false),
 198       http_user_agent_settings_(http_user_agent_settings),
 199       backoff_manager_(request->context()->backoff_manager()),
 200       weak_factory_(this) {
 201   URLRequestThrottlerManager* manager = request->context()->throttler_manager();
 202   if (manager)
 203     throttling_entry_ = manager->RegisterRequestUrl(request->url());
 204
 205   ResetTimer();
 206 }
 207
 208 URLRequestHttpJob::~URLRequestHttpJob() {
 209   CHECK(!awaiting_callback_);
 210
 211   DCHECK(!sdch_test_control_ || !sdch_test_activated_);
 212   if (!is_cached_content_) {
 213     if (sdch_test_control_)
 214       RecordPacketStats(FilterContext::SDCH_EXPERIMENT_HOLDBACK);
 215     if (sdch_test_activated_)
 216       RecordPacketStats(FilterContext::SDCH_EXPERIMENT_DECODE);
 217   }
 218   // Make sure SDCH filters are told to emit histogram data while
 219   // filter_context_ is still alive.
 220   DestroyFilters();
 221
 222   DoneWithRequest(ABORTED);
 223 }
 224
 225 void URLRequestHttpJob::SetPriority(RequestPriority priority) {
 226   priority_ = priority;
 227   if (transaction_)
 228     transaction_->SetPriority(priority_);
 229 }
 230
 231 void URLRequestHttpJob::Start() {
 232   // TODO(mmenke): Remove ScopedTracker below once crbug.com/456327 is fixed.
 233   tracked_objects::ScopedTracker tracking_profile(
 234       FROM_HERE_WITH_EXPLICIT_FUNCTION("456327 URLRequestHttpJob::Start"));
 235
 236   DCHECK(!transaction_.get());
 237
 238   // URLRequest::SetReferrer ensures that we do not send username and password
 239   // fields in the referrer.
 240   GURL referrer(request_->referrer());
 241
 242   request_info_.url = request_->url();
 243   request_info_.method = request_->method();
 244   request_info_.load_flags = request_->load_flags();
 245   // Enable privacy mode if cookie settings or flags tell us not send or
 246   // save cookies.
 247   bool enable_privacy_mode =
 248       (request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES) ||
 249       (request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) ||
 250       CanEnablePrivacyMode();
 251   // Privacy mode could still be disabled in OnCookiesLoaded if we are going
 252   // to send previously saved cookies.
 253   request_info_.privacy_mode = enable_privacy_mode ?
 254       PRIVACY_MODE_ENABLED : PRIVACY_MODE_DISABLED;
 255
 256   // Strip Referer from request_info_.extra_headers to prevent, e.g., plugins
 257   // from overriding headers that are controlled using other means. Otherwise a
 258   // plugin could set a referrer although sending the referrer is inhibited.
 259   request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kReferer);
 260
 261   // Our consumer should have made sure that this is a safe referrer. See for
 262   // instance WebCore::FrameLoader::HideReferrer.
 263   if (referrer.is_valid()) {
 264     request_info_.extra_headers.SetHeader(HttpRequestHeaders::kReferer,
 265                                           referrer.spec());
 266   }
 267
 268   request_info_.extra_headers.SetHeaderIfMissing(
 269       HttpRequestHeaders::kUserAgent,
 270       http_user_agent_settings_ ?
 271           http_user_agent_settings_->GetUserAgent() : std::string());
 272
 273   AddExtraHeaders();
 274   AddCookieHeaderAndStart();
 275 }
 276
 277 void URLRequestHttpJob::Kill() {
 278   if (!transaction_.get())
 279     return;
 280
 281   weak_factory_.InvalidateWeakPtrs();
 282   DestroyTransaction();
 283   URLRequestJob::Kill();
 284 }
 285
 286 void URLRequestHttpJob::GetConnectionAttempts(ConnectionAttempts* out) const {
 287   if (transaction_)
 288     transaction_->GetConnectionAttempts(out);
 289   else
 290     out->clear();
 291 }
 292
 293 void URLRequestHttpJob::NotifyBeforeSendProxyHeadersCallback(
 294     const ProxyInfo& proxy_info,
 295     HttpRequestHeaders* request_headers) {
 296   DCHECK(request_headers);
 297   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 298   if (network_delegate()) {
 299     network_delegate()->NotifyBeforeSendProxyHeaders(
 300         request_,
 301         proxy_info,
 302         request_headers);
 303   }
 304 }
 305
 306 void URLRequestHttpJob::NotifyBeforeNetworkStart(bool* defer) {
 307   if (!request_)
 308     return;
 309   if (backoff_manager_) {
 310     if (backoff_manager_->ShouldRejectRequest(request()->url(),
 311                                               request()->request_time())) {
 312       *defer = true;
 313       base::MessageLoop::current()->PostTask(
 314           FROM_HERE,
 315           base::Bind(&URLRequestHttpJob::OnStartCompleted,
 316                      weak_factory_.GetWeakPtr(), ERR_TEMPORARY_BACKOFF));
 317       return;
 318     }
 319   }
 320   URLRequestJob::NotifyBeforeNetworkStart(defer);
 321 }
 322
 323 void URLRequestHttpJob::NotifyHeadersComplete() {
 324   DCHECK(!response_info_);
 325
 326   response_info_ = transaction_->GetResponseInfo();
 327
 328   // Save boolean, as we'll need this info at destruction time, and filters may
 329   // also need this info.
 330   is_cached_content_ = response_info_->was_cached;
 331
 332   if (!is_cached_content_ && throttling_entry_.get())
 333     throttling_entry_->UpdateWithResponse(GetResponseCode());
 334
 335   if (!is_cached_content_)
 336     ProcessBackoffHeader();
 337
 338   // The ordering of these calls is not important.
 339   ProcessStrictTransportSecurityHeader();
 340   ProcessPublicKeyPinsHeader();
 341
 342   // Handle the server notification of a new SDCH dictionary.
 343   SdchManager* sdch_manager(request()->context()->sdch_manager());
 344   if (sdch_manager) {
 345     SdchProblemCode rv = sdch_manager->IsInSupportedDomain(request()->url());
 346     if (rv != SDCH_OK) {
 347       SdchManager::SdchErrorRecovery(rv);
 348       request()->net_log().AddEvent(
 349           NetLog::TYPE_SDCH_DECODING_ERROR,
 350           base::Bind(&NetLogSdchResourceProblemCallback, rv));
 351     } else {
 352       const std::string name = "Get-Dictionary";
 353       std::string url_text;
 354       void* iter = NULL;
 355       // TODO(jar): We need to not fetch dictionaries the first time they are
 356       // seen, but rather wait until we can justify their usefulness.
 357       // For now, we will only fetch the first dictionary, which will at least
 358       // require multiple suggestions before we get additional ones for this
 359       // site. Eventually we should wait until a dictionary is requested
 360       // several times
 361       // before we even download it (so that we don't waste memory or
 362       // bandwidth).
 363       if (GetResponseHeaders()->EnumerateHeader(&iter, name, &url_text)) {
 364         // Resolve suggested URL relative to request url.
 365         GURL sdch_dictionary_url = request_->url().Resolve(url_text);
 366         if (sdch_dictionary_url.is_valid()) {
 367           rv = sdch_manager->OnGetDictionary(request_->url(),
 368                                              sdch_dictionary_url);
 369           if (rv != SDCH_OK) {
 370             SdchManager::SdchErrorRecovery(rv);
 371             request_->net_log().AddEvent(
 372                 NetLog::TYPE_SDCH_DICTIONARY_ERROR,
 373                 base::Bind(&NetLogSdchDictionaryFetchProblemCallback, rv,
 374                            sdch_dictionary_url, false));
 375           }
 376         }
 377       }
 378     }
 379   }
 380
 381   // Handle the server signalling no SDCH encoding.
 382   if (dictionaries_advertised_) {
 383     // We are wary of proxies that discard or damage SDCH encoding. If a server
 384     // explicitly states that this is not SDCH content, then we can correct our
 385     // assumption that this is an SDCH response, and avoid the need to recover
 386     // as though the content is corrupted (when we discover it is not SDCH
 387     // encoded).
 388     std::string sdch_response_status;
 389     void* iter = NULL;
 390     while (GetResponseHeaders()->EnumerateHeader(&iter, "X-Sdch-Encode",
 391                                                  &sdch_response_status)) {
 392       if (sdch_response_status == "0") {
 393         dictionaries_advertised_.reset();
 394         break;
 395       }
 396     }
 397   }
 398
 399   // The HTTP transaction may be restarted several times for the purposes
 400   // of sending authorization information. Each time it restarts, we get
 401   // notified of the headers completion so that we can update the cookie store.
 402   if (transaction_->IsReadyToRestartForAuth()) {
 403     DCHECK(!response_info_->auth_challenge.get());
 404     // TODO(battre): This breaks the webrequest API for
 405     // URLRequestTestHTTP.BasicAuthWithCookies
 406     // where OnBeforeSendHeaders -> OnSendHeaders -> OnBeforeSendHeaders
 407     // occurs.
 408     RestartTransactionWithAuth(AuthCredentials());
 409     return;
 410   }
 411
 412   URLRequestJob::NotifyHeadersComplete();
 413 }
 414
 415 void URLRequestHttpJob::NotifyDone(const URLRequestStatus& status) {
 416   DoneWithRequest(FINISHED);
 417   URLRequestJob::NotifyDone(status);
 418 }
 419
 420 void URLRequestHttpJob::DestroyTransaction() {
 421   DCHECK(transaction_.get());
 422
 423   DoneWithRequest(ABORTED);
 424   transaction_.reset();
 425   response_info_ = NULL;
 426   receive_headers_end_ = base::TimeTicks();
 427 }
 428
 429 void URLRequestHttpJob::StartTransaction() {
 430   // TODO(mmenke): Remove ScopedTracker below once crbug.com/456327 is fixed.
 431   tracked_objects::ScopedTracker tracking_profile(
 432       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 433           "456327 URLRequestHttpJob::StartTransaction"));
 434
 435   if (network_delegate()) {
 436     OnCallToDelegate();
 437     int rv = network_delegate()->NotifyBeforeSendHeaders(
 438         request_, notify_before_headers_sent_callback_,
 439         &request_info_.extra_headers);
 440     // If an extension blocks the request, we rely on the callback to
 441     // MaybeStartTransactionInternal().
 442     if (rv == ERR_IO_PENDING)
 443       return;
 444     MaybeStartTransactionInternal(rv);
 445     return;
 446   }
 447   StartTransactionInternal();
 448 }
 449
 450 void URLRequestHttpJob::NotifyBeforeSendHeadersCallback(int result) {
 451   // Check that there are no callbacks to already canceled requests.
 452   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 453
 454   MaybeStartTransactionInternal(result);
 455 }
 456
 457 void URLRequestHttpJob::MaybeStartTransactionInternal(int result) {
 458   // TODO(mmenke): Remove ScopedTracker below once crbug.com/456327 is fixed.
 459   tracked_objects::ScopedTracker tracking_profile(
 460       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 461           "456327 URLRequestHttpJob::MaybeStartTransactionInternal"));
 462
 463   OnCallToDelegateComplete();
 464   if (result == OK) {
 465     StartTransactionInternal();
 466   } else {
 467     std::string source("delegate");
 468     request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 469                                  NetLog::StringCallback("source", &source));
 470     NotifyCanceled();
 471     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 472   }
 473 }
 474
 475 void URLRequestHttpJob::StartTransactionInternal() {
 476   // NOTE: This method assumes that request_info_ is already setup properly.
 477
 478   // If we already have a transaction, then we should restart the transaction
 479   // with auth provided by auth_credentials_.
 480
 481   int rv;
 482
 483   if (network_delegate()) {
 484     network_delegate()->NotifySendHeaders(
 485         request_, request_info_.extra_headers);
 486   }
 487
 488   if (transaction_.get()) {
 489     rv = transaction_->RestartWithAuth(auth_credentials_, start_callback_);
 490     auth_credentials_ = AuthCredentials();
 491   } else {
 492     DCHECK(request_->context()->http_transaction_factory());
 493
 494     rv = request_->context()->http_transaction_factory()->CreateTransaction(
 495         priority_, &transaction_);
 496
 497     if (rv == OK && request_info_.url.SchemeIsWSOrWSS()) {
 498       base::SupportsUserData::Data* data = request_->GetUserData(
 499           WebSocketHandshakeStreamBase::CreateHelper::DataKey());
 500       if (data) {
 501         transaction_->SetWebSocketHandshakeStreamCreateHelper(
 502             static_cast<WebSocketHandshakeStreamBase::CreateHelper*>(data));
 503       } else {
 504         rv = ERR_DISALLOWED_URL_SCHEME;
 505       }
 506     }
 507
 508     if (rv == OK) {
 509       transaction_->SetBeforeNetworkStartCallback(
 510           base::Bind(&URLRequestHttpJob::NotifyBeforeNetworkStart,
 511                      base::Unretained(this)));
 512       transaction_->SetBeforeProxyHeadersSentCallback(
 513           base::Bind(&URLRequestHttpJob::NotifyBeforeSendProxyHeadersCallback,
 514                      base::Unretained(this)));
 515
 516       if (!throttling_entry_.get() ||
 517           !throttling_entry_->ShouldRejectRequest(*request_)) {
 518         rv = transaction_->Start(
 519             &request_info_, start_callback_, request_->net_log());
 520         start_time_ = base::TimeTicks::Now();
 521       } else {
 522         // Special error code for the exponential back-off module.
 523         rv = ERR_TEMPORARILY_THROTTLED;
 524       }
 525     }
 526   }
 527
 528   if (rv == ERR_IO_PENDING)
 529     return;
 530
 531   // The transaction started synchronously, but we need to notify the
 532   // URLRequest delegate via the message loop.
 533   base::ThreadTaskRunnerHandle::Get()->PostTask(
 534       FROM_HERE, base::Bind(&URLRequestHttpJob::OnStartCompleted,
 535                             weak_factory_.GetWeakPtr(), rv));
 536 }
 537
 538 void URLRequestHttpJob::AddExtraHeaders() {
 539   SdchManager* sdch_manager = request()->context()->sdch_manager();
 540
 541   // Supply Accept-Encoding field only if it is not already provided.
 542   // It should be provided IF the content is known to have restrictions on
 543   // potential encoding, such as streaming multi-media.
 544   // For details see bug 47381.
 545   // TODO(jar, enal): jpeg files etc. should set up a request header if
 546   // possible. Right now it is done only by buffered_resource_loader and
 547   // simple_data_source.
 548   if (!request_info_.extra_headers.HasHeader(
 549       HttpRequestHeaders::kAcceptEncoding)) {
 550     // We don't support SDCH responses to POST as there is a possibility
 551     // of having SDCH encoded responses returned (e.g. by the cache)
 552     // which we cannot decode, and in those situations, we will need
 553     // to retransmit the request without SDCH, which is illegal for a POST.
 554     bool advertise_sdch = sdch_manager != NULL && request()->method() != "POST";
 555     if (advertise_sdch) {
 556       SdchProblemCode rv = sdch_manager->IsInSupportedDomain(request()->url());
 557       if (rv != SDCH_OK) {
 558         advertise_sdch = false;
 559         SdchManager::SdchErrorRecovery(rv);
 560         request()->net_log().AddEvent(
 561             NetLog::TYPE_SDCH_DECODING_ERROR,
 562             base::Bind(&NetLogSdchResourceProblemCallback, rv));
 563       }
 564     }
 565     if (advertise_sdch) {
 566       dictionaries_advertised_ =
 567           sdch_manager->GetDictionarySet(request_->url());
 568     }
 569
 570     // The AllowLatencyExperiment() is only true if we've successfully done a
 571     // full SDCH compression recently in this browser session for this host.
 572     // Note that for this path, there might be no applicable dictionaries,
 573     // and hence we can't participate in the experiment.
 574     if (dictionaries_advertised_ &&
 575         sdch_manager->AllowLatencyExperiment(request_->url())) {
 576       // We are participating in the test (or control), and hence we'll
 577       // eventually record statistics via either SDCH_EXPERIMENT_DECODE or
 578       // SDCH_EXPERIMENT_HOLDBACK, and we'll need some packet timing data.
 579       packet_timing_enabled_ = true;
 580       if (base::RandDouble() < .01) {
 581         sdch_test_control_ = true;  // 1% probability.
 582         dictionaries_advertised_.reset();
 583         advertise_sdch = false;
 584       } else {
 585         sdch_test_activated_ = true;
 586       }
 587     }
 588
 589     // Supply Accept-Encoding headers first so that it is more likely that they
 590     // will be in the first transmitted packet. This can sometimes make it
 591     // easier to filter and analyze the streams to assure that a proxy has not
 592     // damaged these headers. Some proxies deliberately corrupt Accept-Encoding
 593     // headers.
 594     if (!advertise_sdch) {
 595       // Tell the server what compression formats we support (other than SDCH).
 596       request_info_.extra_headers.SetHeader(
 597           HttpRequestHeaders::kAcceptEncoding, "gzip, deflate");
 598     } else {
 599       // Include SDCH in acceptable list.
 600       request_info_.extra_headers.SetHeader(
 601           HttpRequestHeaders::kAcceptEncoding, "gzip, deflate, sdch");
 602       if (dictionaries_advertised_) {
 603         request_info_.extra_headers.SetHeader(
 604             kAvailDictionaryHeader,
 605             dictionaries_advertised_->GetDictionaryClientHashList());
 606         // Since we're tagging this transaction as advertising a dictionary,
 607         // we'll definitely employ an SDCH filter (or tentative sdch filter)
 608         // when we get a response. When done, we'll record histograms via
 609         // SDCH_DECODE or SDCH_PASSTHROUGH. Hence we need to record packet
 610         // arrival times.
 611         packet_timing_enabled_ = true;
 612       }
 613     }
 614   }
 615
 616   if (http_user_agent_settings_) {
 617     // Only add default Accept-Language if the request didn't have it
 618     // specified.
 619     std::string accept_language =
 620         http_user_agent_settings_->GetAcceptLanguage();
 621     if (!accept_language.empty()) {
 622       request_info_.extra_headers.SetHeaderIfMissing(
 623           HttpRequestHeaders::kAcceptLanguage,
 624           accept_language);
 625     }
 626   }
 627 }
 628
 629 void URLRequestHttpJob::AddCookieHeaderAndStart() {
 630   // No matter what, we want to report our status as IO pending since we will
 631   // be notifying our consumer asynchronously via OnStartCompleted.
 632   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
 633
 634   // If the request was destroyed, then there is no more work to do.
 635   if (!request_)
 636     return;
 637
 638   CookieStore* cookie_store = request_->context()->cookie_store();
 639   if (cookie_store && !(request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES)) {
 640     cookie_store->GetAllCookiesForURLAsync(
 641         request_->url(),
 642         base::Bind(&URLRequestHttpJob::CheckCookiePolicyAndLoad,
 643                    weak_factory_.GetWeakPtr()));
 644   } else {
 645     DoStartTransaction();
 646   }
 647 }
 648
 649 void URLRequestHttpJob::DoLoadCookies() {
 650   CookieOptions options;
 651   options.set_include_httponly();
 652
 653   // TODO(mkwst): Drop this `if` once we decide whether or not to ship
 654   // first-party cookies: https://crbug.com/459154
 655   if (network_delegate() &&
 656       network_delegate()->FirstPartyOnlyCookieExperimentEnabled())
 657     options.set_first_party_url(request_->first_party_for_cookies());
 658   else
 659     options.set_include_first_party_only();
 660
 661   request_->context()->cookie_store()->GetCookiesWithOptionsAsync(
 662       request_->url(), options, base::Bind(&URLRequestHttpJob::OnCookiesLoaded,
 663                                            weak_factory_.GetWeakPtr()));
 664 }
 665
 666 void URLRequestHttpJob::CheckCookiePolicyAndLoad(
 667     const CookieList& cookie_list) {
 668   if (CanGetCookies(cookie_list))
 669     DoLoadCookies();
 670   else
 671     DoStartTransaction();
 672 }
 673
 674 void URLRequestHttpJob::OnCookiesLoaded(const std::string& cookie_line) {
 675   if (!cookie_line.empty()) {
 676     request_info_.extra_headers.SetHeader(
 677         HttpRequestHeaders::kCookie, cookie_line);
 678     // Disable privacy mode as we are sending cookies anyway.
 679     request_info_.privacy_mode = PRIVACY_MODE_DISABLED;
 680   }
 681   DoStartTransaction();
 682 }
 683
 684 void URLRequestHttpJob::DoStartTransaction() {
 685   // We may have been canceled while retrieving cookies.
 686   if (GetStatus().is_success()) {
 687     StartTransaction();
 688   } else {
 689     NotifyCanceled();
 690   }
 691 }
 692
 693 void URLRequestHttpJob::SaveCookiesAndNotifyHeadersComplete(int result) {
 694   // End of the call started in OnStartCompleted.
 695   OnCallToDelegateComplete();
 696
 697   if (result != OK) {
 698     std::string source("delegate");
 699     request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 700                                  NetLog::StringCallback("source", &source));
 701     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 702     return;
 703   }
 704
 705   DCHECK(transaction_.get());
 706
 707   const HttpResponseInfo* response_info = transaction_->GetResponseInfo();
 708   DCHECK(response_info);
 709
 710   response_cookies_.clear();
 711   response_cookies_save_index_ = 0;
 712
 713   FetchResponseCookies(&response_cookies_);
 714
 715   if (!GetResponseHeaders()->GetDateValue(&response_date_))
 716     response_date_ = base::Time();
 717
 718   // Now, loop over the response cookies, and attempt to persist each.
 719   SaveNextCookie();
 720 }
 721
 722 // If the save occurs synchronously, SaveNextCookie will loop and save the next
 723 // cookie. If the save is deferred, the callback is responsible for continuing
 724 // to iterate through the cookies.
 725 // TODO(erikwright): Modify the CookieStore API to indicate via return value
 726 // whether it completed synchronously or asynchronously.
 727 // See http://crbug.com/131066.
 728 void URLRequestHttpJob::SaveNextCookie() {
 729   // No matter what, we want to report our status as IO pending since we will
 730   // be notifying our consumer asynchronously via OnStartCompleted.
 731   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
 732
 733   // Used to communicate with the callback. See the implementation of
 734   // OnCookieSaved.
 735   scoped_refptr<SharedBoolean> callback_pending = new SharedBoolean(false);
 736   scoped_refptr<SharedBoolean> save_next_cookie_running =
 737       new SharedBoolean(true);
 738
 739   if (!(request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) &&
 740       request_->context()->cookie_store() && response_cookies_.size() > 0) {
 741     CookieOptions options;
 742     options.set_include_httponly();
 743     options.set_server_time(response_date_);
 744
 745     CookieStore::SetCookiesCallback callback(base::Bind(
 746         &URLRequestHttpJob::OnCookieSaved, weak_factory_.GetWeakPtr(),
 747         save_next_cookie_running, callback_pending));
 748
 749     // Loop through the cookies as long as SetCookieWithOptionsAsync completes
 750     // synchronously.
 751     while (!callback_pending->data &&
 752            response_cookies_save_index_ < response_cookies_.size()) {
 753       if (CanSetCookie(
 754           response_cookies_[response_cookies_save_index_], &options)) {
 755         callback_pending->data = true;
 756         request_->context()->cookie_store()->SetCookieWithOptionsAsync(
 757             request_->url(), response_cookies_[response_cookies_save_index_],
 758             options, callback);
 759       }
 760       ++response_cookies_save_index_;
 761     }
 762   }
 763
 764   save_next_cookie_running->data = false;
 765
 766   if (!callback_pending->data) {
 767     response_cookies_.clear();
 768     response_cookies_save_index_ = 0;
 769     SetStatus(URLRequestStatus());  // Clear the IO_PENDING status
 770     NotifyHeadersComplete();
 771     return;
 772   }
 773 }
 774
 775 // |save_next_cookie_running| is true when the callback is bound and set to
 776 // false when SaveNextCookie exits, allowing the callback to determine if the
 777 // save occurred synchronously or asynchronously.
 778 // |callback_pending| is false when the callback is invoked and will be set to
 779 // true by the callback, allowing SaveNextCookie to detect whether the save
 780 // occurred synchronously.
 781 // See SaveNextCookie() for more information.
 782 void URLRequestHttpJob::OnCookieSaved(
 783     scoped_refptr<SharedBoolean> save_next_cookie_running,
 784     scoped_refptr<SharedBoolean> callback_pending,
 785     bool cookie_status) {
 786   callback_pending->data = false;
 787
 788   // If we were called synchronously, return.
 789   if (save_next_cookie_running->data) {
 790     return;
 791   }
 792
 793   // We were called asynchronously, so trigger the next save.
 794   // We may have been canceled within OnSetCookie.
 795   if (GetStatus().is_success()) {
 796     SaveNextCookie();
 797   } else {
 798     NotifyCanceled();
 799   }
 800 }
 801
 802 void URLRequestHttpJob::FetchResponseCookies(
 803     std::vector<std::string>* cookies) {
 804   const std::string name = "Set-Cookie";
 805   std::string value;
 806
 807   void* iter = NULL;
 808   HttpResponseHeaders* headers = GetResponseHeaders();
 809   while (headers->EnumerateHeader(&iter, name, &value)) {
 810     if (!value.empty())
 811       cookies->push_back(value);
 812   }
 813 }
 814
 815 void URLRequestHttpJob::ProcessBackoffHeader() {
 816   DCHECK(response_info_);
 817
 818   if (!backoff_manager_)
 819     return;
 820
 821   TransportSecurityState* security_state =
 822       request_->context()->transport_security_state();
 823   const SSLInfo& ssl_info = response_info_->ssl_info;
 824
 825   // Only accept Backoff headers on HTTPS connections that have no
 826   // certificate errors.
 827   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 828       !security_state)
 829     return;
 830
 831   backoff_manager_->UpdateWithResponse(request()->url(), GetResponseHeaders(),
 832                                        base::Time::Now());
 833 }
 834
 835 // NOTE: |ProcessStrictTransportSecurityHeader| and
 836 // |ProcessPublicKeyPinsHeader| have very similar structures, by design.
 837 void URLRequestHttpJob::ProcessStrictTransportSecurityHeader() {
 838   DCHECK(response_info_);
 839   TransportSecurityState* security_state =
 840       request_->context()->transport_security_state();
 841   const SSLInfo& ssl_info = response_info_->ssl_info;
 842
 843   // Only accept HSTS headers on HTTPS connections that have no
 844   // certificate errors.
 845   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 846       !security_state)
 847     return;
 848
 849   // Don't accept HSTS headers when the hostname is an IP address.
 850   if (request_info_.url.HostIsIPAddress())
 851     return;
 852
 853   // http://tools.ietf.org/html/draft-ietf-websec-strict-transport-sec:
 854   //
 855   //   If a UA receives more than one STS header field in a HTTP response
 856   //   message over secure transport, then the UA MUST process only the
 857   //   first such header field.
 858   HttpResponseHeaders* headers = GetResponseHeaders();
 859   std::string value;
 860   if (headers->EnumerateHeader(NULL, "Strict-Transport-Security", &value))
 861     security_state->AddHSTSHeader(request_info_.url.host(), value);
 862 }
 863
 864 void URLRequestHttpJob::ProcessPublicKeyPinsHeader() {
 865   DCHECK(response_info_);
 866   TransportSecurityState* security_state =
 867       request_->context()->transport_security_state();
 868   const SSLInfo& ssl_info = response_info_->ssl_info;
 869
 870   // Only accept HPKP headers on HTTPS connections that have no
 871   // certificate errors.
 872   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 873       !security_state)
 874     return;
 875
 876   // Don't accept HSTS headers when the hostname is an IP address.
 877   if (request_info_.url.HostIsIPAddress())
 878     return;
 879
 880   // http://tools.ietf.org/html/rfc7469:
 881   //
 882   //   If a UA receives more than one PKP header field in an HTTP
 883   //   response message over secure transport, then the UA MUST process
 884   //   only the first such header field.
 885   HttpResponseHeaders* headers = GetResponseHeaders();
 886   std::string value;
 887   if (headers->EnumerateHeader(nullptr, "Public-Key-Pins", &value))
 888     security_state->AddHPKPHeader(request_info_.url.host(), value, ssl_info);
 889   if (headers->EnumerateHeader(nullptr, "Public-Key-Pins-Report-Only",
 890                                &value)) {
 891     security_state->ProcessHPKPReportOnlyHeader(
 892         value, HostPortPair::FromURL(request_info_.url), ssl_info);
 893   }
 894 }
 895
 896 void URLRequestHttpJob::OnStartCompleted(int result) {
 897   RecordTimer();
 898
 899   // If the request was destroyed, then there is no more work to do.
 900   if (!request_)
 901     return;
 902
 903   // If the job is done (due to cancellation), can just ignore this
 904   // notification.
 905   if (done_)
 906     return;
 907
 908   receive_headers_end_ = base::TimeTicks::Now();
 909
 910   // Clear the IO_PENDING status
 911   SetStatus(URLRequestStatus());
 912
 913   const URLRequestContext* context = request_->context();
 914
 915   if (result == OK) {
 916     if (transaction_ && transaction_->GetResponseInfo()) {
 917       SetProxyServer(transaction_->GetResponseInfo()->proxy_server);
 918     }
 919     scoped_refptr<HttpResponseHeaders> headers = GetResponseHeaders();
 920     if (network_delegate()) {
 921       // Note that |this| may not be deleted until
 922       // |on_headers_received_callback_| or
 923       // |NetworkDelegate::URLRequestDestroyed()| has been called.
 924       OnCallToDelegate();
 925       allowed_unsafe_redirect_url_ = GURL();
 926       int error = network_delegate()->NotifyHeadersReceived(
 927           request_,
 928           on_headers_received_callback_,
 929           headers.get(),
 930           &override_response_headers_,
 931           &allowed_unsafe_redirect_url_);
 932       if (error != OK) {
 933         if (error == ERR_IO_PENDING) {
 934           awaiting_callback_ = true;
 935         } else {
 936           std::string source("delegate");
 937           request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 938                                        NetLog::StringCallback("source",
 939                                                               &source));
 940           OnCallToDelegateComplete();
 941           NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, error));
 942         }
 943         return;
 944       }
 945     }
 946
 947     SaveCookiesAndNotifyHeadersComplete(OK);
 948   } else if (IsCertificateError(result)) {
 949     // We encountered an SSL certificate error.
 950     if (result == ERR_SSL_WEAK_SERVER_EPHEMERAL_DH_KEY ||
 951         result == ERR_SSL_PINNED_KEY_NOT_IN_CERT_CHAIN) {
 952       // These are hard failures. They're handled separately and don't have
 953       // the correct cert status, so set it here.
 954       SSLInfo info(transaction_->GetResponseInfo()->ssl_info);
 955       info.cert_status = MapNetErrorToCertStatus(result);
 956       NotifySSLCertificateError(info, true);
 957     } else {
 958       // Maybe overridable, maybe not. Ask the delegate to decide.
 959       TransportSecurityState* state = context->transport_security_state();
 960       const bool fatal =
 961           state && state->ShouldSSLErrorsBeFatal(request_info_.url.host());
 962       NotifySSLCertificateError(
 963           transaction_->GetResponseInfo()->ssl_info, fatal);
 964     }
 965   } else if (result == ERR_SSL_CLIENT_AUTH_CERT_NEEDED) {
 966     NotifyCertificateRequested(
 967         transaction_->GetResponseInfo()->cert_request_info.get());
 968   } else {
 969     // Even on an error, there may be useful information in the response
 970     // info (e.g. whether there's a cached copy).
 971     if (transaction_.get())
 972       response_info_ = transaction_->GetResponseInfo();
 973     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 974   }
 975 }
 976
 977 void URLRequestHttpJob::OnHeadersReceivedCallback(int result) {
 978   awaiting_callback_ = false;
 979
 980   // Check that there are no callbacks to already canceled requests.
 981   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 982
 983   SaveCookiesAndNotifyHeadersComplete(result);
 984 }
 985
 986 void URLRequestHttpJob::OnReadCompleted(int result) {
 987   read_in_progress_ = false;
 988
 989   if (ShouldFixMismatchedContentLength(result))
 990     result = OK;
 991
 992   if (result == OK) {
 993     NotifyDone(URLRequestStatus());
 994   } else if (result < 0) {
 995     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, result));
 996   } else {
 997     // Clear the IO_PENDING status
 998     SetStatus(URLRequestStatus());
 999   }
1000
1001   NotifyReadComplete(result);
1002 }
1003
1004 void URLRequestHttpJob::RestartTransactionWithAuth(
1005     const AuthCredentials& credentials) {
1006   auth_credentials_ = credentials;
1007
1008   // These will be reset in OnStartCompleted.
1009   response_info_ = NULL;
1010   receive_headers_end_ = base::TimeTicks();
1011   response_cookies_.clear();
1012
1013   ResetTimer();
1014
1015   // Update the cookies, since the cookie store may have been updated from the
1016   // headers in the 401/407. Since cookies were already appended to
1017   // extra_headers, we need to strip them out before adding them again.
1018   request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kCookie);
1019
1020   AddCookieHeaderAndStart();
1021 }
1022
1023 void URLRequestHttpJob::SetUpload(UploadDataStream* upload) {
1024   DCHECK(!transaction_.get()) << "cannot change once started";
1025   request_info_.upload_data_stream = upload;
1026 }
1027
1028 void URLRequestHttpJob::SetExtraRequestHeaders(
1029     const HttpRequestHeaders& headers) {
1030   DCHECK(!transaction_.get()) << "cannot change once started";
1031   request_info_.extra_headers.CopyFrom(headers);
1032 }
1033
1034 LoadState URLRequestHttpJob::GetLoadState() const {
1035   return transaction_.get() ?
1036       transaction_->GetLoadState() : LOAD_STATE_IDLE;
1037 }
1038
1039 UploadProgress URLRequestHttpJob::GetUploadProgress() const {
1040   return transaction_.get() ?
1041       transaction_->GetUploadProgress() : UploadProgress();
1042 }
1043
1044 bool URLRequestHttpJob::GetMimeType(std::string* mime_type) const {
1045   DCHECK(transaction_.get());
1046
1047   if (!response_info_)
1048     return false;
1049
1050   HttpResponseHeaders* headers = GetResponseHeaders();
1051   if (!headers)
1052     return false;
1053   return headers->GetMimeType(mime_type);
1054 }
1055
1056 bool URLRequestHttpJob::GetCharset(std::string* charset) {
1057   DCHECK(transaction_.get());
1058
1059   if (!response_info_)
1060     return false;
1061
1062   return GetResponseHeaders()->GetCharset(charset);
1063 }
1064
1065 void URLRequestHttpJob::GetResponseInfo(HttpResponseInfo* info) {
1066   DCHECK(request_);
1067
1068   if (response_info_) {
1069     DCHECK(transaction_.get());
1070
1071     *info = *response_info_;
1072     if (override_response_headers_.get())
1073       info->headers = override_response_headers_;
1074   }
1075 }
1076
1077 void URLRequestHttpJob::GetLoadTimingInfo(
1078     LoadTimingInfo* load_timing_info) const {
1079   // If haven't made it far enough to receive any headers, don't return
1080   // anything. This makes for more consistent behavior in the case of errors.
1081   if (!transaction_ || receive_headers_end_.is_null())
1082     return;
1083   if (transaction_->GetLoadTimingInfo(load_timing_info))
1084     load_timing_info->receive_headers_end = receive_headers_end_;
1085 }
1086
1087 bool URLRequestHttpJob::GetResponseCookies(std::vector<std::string>* cookies) {
1088   DCHECK(transaction_.get());
1089
1090   if (!response_info_)
1091     return false;
1092
1093   // TODO(darin): Why are we extracting response cookies again?  Perhaps we
1094   // should just leverage response_cookies_.
1095
1096   cookies->clear();
1097   FetchResponseCookies(cookies);
1098   return true;
1099 }
1100
1101 int URLRequestHttpJob::GetResponseCode() const {
1102   DCHECK(transaction_.get());
1103
1104   if (!response_info_)
1105     return -1;
1106
1107   return GetResponseHeaders()->response_code();
1108 }
1109
1110 Filter* URLRequestHttpJob::SetupFilter() const {
1111   DCHECK(transaction_.get());
1112   if (!response_info_)
1113     return NULL;
1114
1115   std::vector<Filter::FilterType> encoding_types;
1116   std::string encoding_type;
1117   HttpResponseHeaders* headers = GetResponseHeaders();
1118   void* iter = NULL;
1119   while (headers->EnumerateHeader(&iter, "Content-Encoding", &encoding_type)) {
1120     encoding_types.push_back(Filter::ConvertEncodingToType(encoding_type));
1121   }
1122
1123   // Even if encoding types are empty, there is a chance that we need to add
1124   // some decoding, as some proxies strip encoding completely. In such cases,
1125   // we may need to add (for example) SDCH filtering (when the context suggests
1126   // it is appropriate).
1127   Filter::FixupEncodingTypes(*filter_context_, &encoding_types);
1128
1129   return !encoding_types.empty()
1130       ? Filter::Factory(encoding_types, *filter_context_) : NULL;
1131 }
1132
1133 bool URLRequestHttpJob::CopyFragmentOnRedirect(const GURL& location) const {
1134   // Allow modification of reference fragments by default, unless
1135   // |allowed_unsafe_redirect_url_| is set and equal to the redirect URL.
1136   // When this is the case, we assume that the network delegate has set the
1137   // desired redirect URL (with or without fragment), so it must not be changed
1138   // any more.
1139   return !allowed_unsafe_redirect_url_.is_valid() ||
1140        allowed_unsafe_redirect_url_ != location;
1141 }
1142
1143 bool URLRequestHttpJob::IsSafeRedirect(const GURL& location) {
1144   // HTTP is always safe.
1145   // TODO(pauljensen): Remove once crbug.com/146591 is fixed.
1146   if (location.is_valid() &&
1147       (location.scheme() == "http" || location.scheme() == "https")) {
1148     return true;
1149   }
1150   // Delegates may mark a URL as safe for redirection.
1151   if (allowed_unsafe_redirect_url_.is_valid() &&
1152       allowed_unsafe_redirect_url_ == location) {
1153     return true;
1154   }
1155   // Query URLRequestJobFactory as to whether |location| would be safe to
1156   // redirect to.
1157   return request_->context()->job_factory() &&
1158       request_->context()->job_factory()->IsSafeRedirectTarget(location);
1159 }
1160
1161 bool URLRequestHttpJob::NeedsAuth() {
1162   int code = GetResponseCode();
1163   if (code == -1)
1164     return false;
1165
1166   // Check if we need either Proxy or WWW Authentication. This could happen
1167   // because we either provided no auth info, or provided incorrect info.
1168   switch (code) {
1169     case 407:
1170       if (proxy_auth_state_ == AUTH_STATE_CANCELED)
1171         return false;
1172       proxy_auth_state_ = AUTH_STATE_NEED_AUTH;
1173       return true;
1174     case 401:
1175       if (server_auth_state_ == AUTH_STATE_CANCELED)
1176         return false;
1177       server_auth_state_ = AUTH_STATE_NEED_AUTH;
1178       return true;
1179   }
1180   return false;
1181 }
1182
1183 void URLRequestHttpJob::GetAuthChallengeInfo(
1184     scoped_refptr<AuthChallengeInfo>* result) {
1185   DCHECK(transaction_.get());
1186   DCHECK(response_info_);
1187
1188   // sanity checks:
1189   DCHECK(proxy_auth_state_ == AUTH_STATE_NEED_AUTH ||
1190          server_auth_state_ == AUTH_STATE_NEED_AUTH);
1191   DCHECK((GetResponseHeaders()->response_code() == HTTP_UNAUTHORIZED) ||
1192          (GetResponseHeaders()->response_code() ==
1193           HTTP_PROXY_AUTHENTICATION_REQUIRED));
1194
1195   *result = response_info_->auth_challenge;
1196 }
1197
1198 void URLRequestHttpJob::SetAuth(const AuthCredentials& credentials) {
1199   DCHECK(transaction_.get());
1200
1201   // Proxy gets set first, then WWW.
1202   if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1203     proxy_auth_state_ = AUTH_STATE_HAVE_AUTH;
1204   } else {
1205     DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1206     server_auth_state_ = AUTH_STATE_HAVE_AUTH;
1207   }
1208
1209   RestartTransactionWithAuth(credentials);
1210 }
1211
1212 void URLRequestHttpJob::CancelAuth() {
1213   // Proxy gets set first, then WWW.
1214   if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1215     proxy_auth_state_ = AUTH_STATE_CANCELED;
1216   } else {
1217     DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1218     server_auth_state_ = AUTH_STATE_CANCELED;
1219   }
1220
1221   // These will be reset in OnStartCompleted.
1222   response_info_ = NULL;
1223   receive_headers_end_ = base::TimeTicks::Now();
1224   response_cookies_.clear();
1225
1226   ResetTimer();
1227
1228   // OK, let the consumer read the error page...
1229   //
1230   // Because we set the AUTH_STATE_CANCELED flag, NeedsAuth will return false,
1231   // which will cause the consumer to receive OnResponseStarted instead of
1232   // OnAuthRequired.
1233   //
1234   // We have to do this via InvokeLater to avoid "recursing" the consumer.
1235   //
1236   base::ThreadTaskRunnerHandle::Get()->PostTask(
1237       FROM_HERE, base::Bind(&URLRequestHttpJob::OnStartCompleted,
1238                             weak_factory_.GetWeakPtr(), OK));
1239 }
1240
1241 void URLRequestHttpJob::ContinueWithCertificate(
1242     X509Certificate* client_cert) {
1243   DCHECK(transaction_.get());
1244
1245   DCHECK(!response_info_) << "should not have a response yet";
1246   receive_headers_end_ = base::TimeTicks();
1247
1248   ResetTimer();
1249
1250   // No matter what, we want to report our status as IO pending since we will
1251   // be notifying our consumer asynchronously via OnStartCompleted.
1252   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1253
1254   int rv = transaction_->RestartWithCertificate(client_cert, start_callback_);
1255   if (rv == ERR_IO_PENDING)
1256     return;
1257
1258   // The transaction started synchronously, but we need to notify the
1259   // URLRequest delegate via the message loop.
1260   base::ThreadTaskRunnerHandle::Get()->PostTask(
1261       FROM_HERE, base::Bind(&URLRequestHttpJob::OnStartCompleted,
1262                             weak_factory_.GetWeakPtr(), rv));
1263 }
1264
1265 void URLRequestHttpJob::ContinueDespiteLastError() {
1266   // If the transaction was destroyed, then the job was cancelled.
1267   if (!transaction_.get())
1268     return;
1269
1270   DCHECK(!response_info_) << "should not have a response yet";
1271   receive_headers_end_ = base::TimeTicks();
1272
1273   ResetTimer();
1274
1275   // No matter what, we want to report our status as IO pending since we will
1276   // be notifying our consumer asynchronously via OnStartCompleted.
1277   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1278
1279   int rv = transaction_->RestartIgnoringLastError(start_callback_);
1280   if (rv == ERR_IO_PENDING)
1281     return;
1282
1283   // The transaction started synchronously, but we need to notify the
1284   // URLRequest delegate via the message loop.
1285   base::ThreadTaskRunnerHandle::Get()->PostTask(
1286       FROM_HERE, base::Bind(&URLRequestHttpJob::OnStartCompleted,
1287                             weak_factory_.GetWeakPtr(), rv));
1288 }
1289
1290 void URLRequestHttpJob::ResumeNetworkStart() {
1291   DCHECK(transaction_.get());
1292   transaction_->ResumeNetworkStart();
1293 }
1294
1295 bool URLRequestHttpJob::ShouldFixMismatchedContentLength(int rv) const {
1296   // Some servers send the body compressed, but specify the content length as
1297   // the uncompressed size. Although this violates the HTTP spec we want to
1298   // support it (as IE and FireFox do), but *only* for an exact match.
1299   // See http://crbug.com/79694.
1300   if (rv == ERR_CONTENT_LENGTH_MISMATCH ||
1301       rv == ERR_INCOMPLETE_CHUNKED_ENCODING) {
1302     if (request_ && request_->response_headers()) {
1303       int64 expected_length = request_->response_headers()->GetContentLength();
1304       VLOG(1) << __FUNCTION__ << "() "
1305               << "\"" << request_->url().spec() << "\""
1306               << " content-length = " << expected_length
1307               << " pre total = " << prefilter_bytes_read()
1308               << " post total = " << postfilter_bytes_read();
1309       if (postfilter_bytes_read() == expected_length) {
1310         // Clear the error.
1311         return true;
1312       }
1313     }
1314   }
1315   return false;
1316 }
1317
1318 bool URLRequestHttpJob::ReadRawData(IOBuffer* buf, int buf_size,
1319                                     int* bytes_read) {
1320   DCHECK_NE(buf_size, 0);
1321   DCHECK(bytes_read);
1322   DCHECK(!read_in_progress_);
1323
1324   int rv = transaction_->Read(
1325       buf, buf_size,
1326       base::Bind(&URLRequestHttpJob::OnReadCompleted, base::Unretained(this)));
1327
1328   if (ShouldFixMismatchedContentLength(rv))
1329     rv = 0;
1330
1331   if (rv >= 0) {
1332     *bytes_read = rv;
1333     if (!rv)
1334       DoneWithRequest(FINISHED);
1335     return true;
1336   }
1337
1338   if (rv == ERR_IO_PENDING) {
1339     read_in_progress_ = true;
1340     SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1341   } else {
1342     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, rv));
1343   }
1344
1345   return false;
1346 }
1347
1348 void URLRequestHttpJob::StopCaching() {
1349   if (transaction_.get())
1350     transaction_->StopCaching();
1351 }
1352
1353 bool URLRequestHttpJob::GetFullRequestHeaders(
1354     HttpRequestHeaders* headers) const {
1355   if (!transaction_)
1356     return false;
1357
1358   return transaction_->GetFullRequestHeaders(headers);
1359 }
1360
1361 int64 URLRequestHttpJob::GetTotalReceivedBytes() const {
1362   if (!transaction_)
1363     return 0;
1364
1365   return transaction_->GetTotalReceivedBytes();
1366 }
1367
1368 void URLRequestHttpJob::DoneReading() {
1369   if (transaction_) {
1370     transaction_->DoneReading();
1371   }
1372   DoneWithRequest(FINISHED);
1373 }
1374
1375 void URLRequestHttpJob::DoneReadingRedirectResponse() {
1376   if (transaction_) {
1377     if (transaction_->GetResponseInfo()->headers->IsRedirect(NULL)) {
1378       // If the original headers indicate a redirect, go ahead and cache the
1379       // response, even if the |override_response_headers_| are a redirect to
1380       // another location.
1381       transaction_->DoneReading();
1382     } else {
1383       // Otherwise, |override_response_headers_| must be non-NULL and contain
1384       // bogus headers indicating a redirect.
1385       DCHECK(override_response_headers_.get());
1386       DCHECK(override_response_headers_->IsRedirect(NULL));
1387       transaction_->StopCaching();
1388     }
1389   }
1390   DoneWithRequest(FINISHED);
1391 }
1392
1393 HostPortPair URLRequestHttpJob::GetSocketAddress() const {
1394   return response_info_ ? response_info_->socket_address : HostPortPair();
1395 }
1396
1397 void URLRequestHttpJob::RecordTimer() {
1398   if (request_creation_time_.is_null()) {
1399     NOTREACHED()
1400         << "The same transaction shouldn't start twice without new timing.";
1401     return;
1402   }
1403
1404   base::TimeDelta to_start = base::Time::Now() - request_creation_time_;
1405   request_creation_time_ = base::Time();
1406
1407   UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpTimeToFirstByte", to_start);
1408 }
1409
1410 void URLRequestHttpJob::ResetTimer() {
1411   if (!request_creation_time_.is_null()) {
1412     NOTREACHED()
1413         << "The timer was reset before it was recorded.";
1414     return;
1415   }
1416   request_creation_time_ = base::Time::Now();
1417 }
1418
1419 void URLRequestHttpJob::UpdatePacketReadTimes() {
1420   if (!packet_timing_enabled_)
1421     return;
1422
1423   DCHECK_GT(prefilter_bytes_read(), bytes_observed_in_packets_);
1424
1425   base::Time now(base::Time::Now());
1426   if (!bytes_observed_in_packets_)
1427     request_time_snapshot_ = now;
1428   final_packet_time_ = now;
1429
1430   bytes_observed_in_packets_ = prefilter_bytes_read();
1431 }
1432
1433 void URLRequestHttpJob::RecordPacketStats(
1434     FilterContext::StatisticSelector statistic) const {
1435   if (!packet_timing_enabled_ || (final_packet_time_ == base::Time()))
1436     return;
1437
1438   base::TimeDelta duration = final_packet_time_ - request_time_snapshot_;
1439   switch (statistic) {
1440     case FilterContext::SDCH_DECODE: {
1441       UMA_HISTOGRAM_CUSTOM_COUNTS("Sdch3.Network_Decode_Bytes_Processed_b",
1442           static_cast<int>(bytes_observed_in_packets_), 500, 100000, 100);
1443       return;
1444     }
1445     case FilterContext::SDCH_PASSTHROUGH: {
1446       // Despite advertising a dictionary, we handled non-sdch compressed
1447       // content.
1448       return;
1449     }
1450
1451     case FilterContext::SDCH_EXPERIMENT_DECODE: {
1452       UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment3_Decode",
1453                                   duration,
1454                                   base::TimeDelta::FromMilliseconds(20),
1455                                   base::TimeDelta::FromMinutes(10), 100);
1456       return;
1457     }
1458     case FilterContext::SDCH_EXPERIMENT_HOLDBACK: {
1459       UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment3_Holdback",
1460                                   duration,
1461                                   base::TimeDelta::FromMilliseconds(20),
1462                                   base::TimeDelta::FromMinutes(10), 100);
1463       return;
1464     }
1465     default:
1466       NOTREACHED();
1467       return;
1468   }
1469 }
1470
1471 void URLRequestHttpJob::RecordPerfHistograms(CompletionCause reason) {
1472   if (start_time_.is_null())
1473     return;
1474
1475   base::TimeDelta total_time = base::TimeTicks::Now() - start_time_;
1476   UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTime", total_time);
1477
1478   if (reason == FINISHED) {
1479     UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeSuccess", total_time);
1480   } else {
1481     UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCancel", total_time);
1482   }
1483
1484   if (response_info_) {
1485     // QUIC (by default) supports https scheme only, thus track https URLs only
1486     // for QUIC.
1487     bool is_https_google = request() && request()->url().SchemeIs("https") &&
1488                            HasGoogleHost(request()->url());
1489     bool used_quic = response_info_->DidUseQuic();
1490     if (is_https_google) {
1491       if (used_quic) {
1492         UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTime.Secure.Quic",
1493                                    total_time);
1494       } else {
1495         UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTime.Secure.NotQuic",
1496                                    total_time);
1497       }
1498     }
1499     if (response_info_->was_cached) {
1500       UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCached", total_time);
1501       if (is_https_google) {
1502         if (used_quic) {
1503           UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTimeCached.Secure.Quic",
1504                                      total_time);
1505         } else {
1506           UMA_HISTOGRAM_MEDIUM_TIMES(
1507               "Net.HttpJob.TotalTimeCached.Secure.NotQuic", total_time);
1508         }
1509       }
1510     } else  {
1511       UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeNotCached", total_time);
1512       if (is_https_google) {
1513         if (used_quic) {
1514           UMA_HISTOGRAM_MEDIUM_TIMES(
1515               "Net.HttpJob.TotalTimeNotCached.Secure.Quic", total_time);
1516         } else {
1517           UMA_HISTOGRAM_MEDIUM_TIMES(
1518               "Net.HttpJob.TotalTimeNotCached.Secure.NotQuic", total_time);
1519         }
1520       }
1521     }
1522   }
1523
1524   if (request_info_.load_flags & LOAD_PREFETCH && !request_->was_cached())
1525     UMA_HISTOGRAM_COUNTS("Net.Prefetch.PrefilterBytesReadFromNetwork",
1526                          prefilter_bytes_read());
1527
1528   start_time_ = base::TimeTicks();
1529 }
1530
1531 void URLRequestHttpJob::DoneWithRequest(CompletionCause reason) {
1532   if (done_)
1533     return;
1534   done_ = true;
1535
1536   // Notify NetworkQualityEstimator.
1537   if (request() && (reason == FINISHED || reason == ABORTED)) {
1538     NetworkQualityEstimator* network_quality_estimator =
1539         request()->context()->network_quality_estimator();
1540     if (network_quality_estimator)
1541       network_quality_estimator->NotifyRequestCompleted(*request());
1542   }
1543
1544   RecordPerfHistograms(reason);
1545   if (request_)
1546     request_->set_received_response_content_length(prefilter_bytes_read());
1547 }
1548
1549 HttpResponseHeaders* URLRequestHttpJob::GetResponseHeaders() const {
1550   DCHECK(transaction_.get());
1551   DCHECK(transaction_->GetResponseInfo());
1552   return override_response_headers_.get() ?
1553              override_response_headers_.get() :
1554              transaction_->GetResponseInfo()->headers.get();
1555 }
1556
1557 void URLRequestHttpJob::NotifyURLRequestDestroyed() {
1558   awaiting_callback_ = false;
1559 }
1560
1561 }  // namespace net