net/url_request/url_request_http_job.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_request_http_job.h"
   6
   7 #include "base/base_switches.h"
   8 #include "base/bind.h"
   9 #include "base/bind_helpers.h"
  10 #include "base/command_line.h"
  11 #include "base/compiler_specific.h"
  12 #include "base/file_version_info.h"
  13 #include "base/message_loop/message_loop.h"
  14 #include "base/metrics/field_trial.h"
  15 #include "base/metrics/histogram.h"
  16 #include "base/profiler/scoped_tracker.h"
  17 #include "base/rand_util.h"
  18 #include "base/strings/string_util.h"
  19 #include "base/time/time.h"
  20 #include "net/base/host_port_pair.h"
  21 #include "net/base/load_flags.h"
  22 #include "net/base/net_errors.h"
  23 #include "net/base/net_util.h"
  24 #include "net/base/network_delegate.h"
  25 #include "net/base/sdch_manager.h"
  26 #include "net/base/sdch_net_log_params.h"
  27 #include "net/cert/cert_status_flags.h"
  28 #include "net/cookies/cookie_store.h"
  29 #include "net/http/http_content_disposition.h"
  30 #include "net/http/http_network_session.h"
  31 #include "net/http/http_request_headers.h"
  32 #include "net/http/http_response_headers.h"
  33 #include "net/http/http_response_info.h"
  34 #include "net/http/http_status_code.h"
  35 #include "net/http/http_transaction.h"
  36 #include "net/http/http_transaction_factory.h"
  37 #include "net/http/http_util.h"
  38 #include "net/proxy/proxy_info.h"
  39 #include "net/ssl/ssl_cert_request_info.h"
  40 #include "net/ssl/ssl_config_service.h"
  41 #include "net/url_request/fraudulent_certificate_reporter.h"
  42 #include "net/url_request/http_user_agent_settings.h"
  43 #include "net/url_request/url_request.h"
  44 #include "net/url_request/url_request_context.h"
  45 #include "net/url_request/url_request_error_job.h"
  46 #include "net/url_request/url_request_job_factory.h"
  47 #include "net/url_request/url_request_redirect_job.h"
  48 #include "net/url_request/url_request_throttler_header_adapter.h"
  49 #include "net/url_request/url_request_throttler_manager.h"
  50 #include "net/websockets/websocket_handshake_stream_base.h"
  51
  52 static const char kAvailDictionaryHeader[] = "Avail-Dictionary";
  53
  54 namespace net {
  55
  56 class URLRequestHttpJob::HttpFilterContext : public FilterContext {
  57  public:
  58   explicit HttpFilterContext(URLRequestHttpJob* job);
  59   ~HttpFilterContext() override;
  60
  61   // FilterContext implementation.
  62   bool GetMimeType(std::string* mime_type) const override;
  63   bool GetURL(GURL* gurl) const override;
  64   base::Time GetRequestTime() const override;
  65   bool IsCachedContent() const override;
  66   SdchManager::DictionarySet* SdchDictionariesAdvertised() const override;
  67   int64 GetByteReadCount() const override;
  68   int GetResponseCode() const override;
  69   const URLRequestContext* GetURLRequestContext() const override;
  70   void RecordPacketStats(StatisticSelector statistic) const override;
  71   const BoundNetLog& GetNetLog() const override;
  72
  73  private:
  74   URLRequestHttpJob* job_;
  75
  76   // URLRequestHttpJob may be detached from URLRequest, but we still need to
  77   // return something.
  78   BoundNetLog dummy_log_;
  79
  80   DISALLOW_COPY_AND_ASSIGN(HttpFilterContext);
  81 };
  82
  83 URLRequestHttpJob::HttpFilterContext::HttpFilterContext(URLRequestHttpJob* job)
  84     : job_(job) {
  85   DCHECK(job_);
  86 }
  87
  88 URLRequestHttpJob::HttpFilterContext::~HttpFilterContext() {
  89 }
  90
  91 bool URLRequestHttpJob::HttpFilterContext::GetMimeType(
  92     std::string* mime_type) const {
  93   return job_->GetMimeType(mime_type);
  94 }
  95
  96 bool URLRequestHttpJob::HttpFilterContext::GetURL(GURL* gurl) const {
  97   if (!job_->request())
  98     return false;
  99   *gurl = job_->request()->url();
 100   return true;
 101 }
 102
 103 base::Time URLRequestHttpJob::HttpFilterContext::GetRequestTime() const {
 104   return job_->request() ? job_->request()->request_time() : base::Time();
 105 }
 106
 107 bool URLRequestHttpJob::HttpFilterContext::IsCachedContent() const {
 108   return job_->is_cached_content_;
 109 }
 110
 111 SdchManager::DictionarySet*
 112 URLRequestHttpJob::HttpFilterContext::SdchDictionariesAdvertised() const {
 113   return job_->dictionaries_advertised_.get();
 114 }
 115
 116 int64 URLRequestHttpJob::HttpFilterContext::GetByteReadCount() const {
 117   return job_->filter_input_byte_count();
 118 }
 119
 120 int URLRequestHttpJob::HttpFilterContext::GetResponseCode() const {
 121   return job_->GetResponseCode();
 122 }
 123
 124 const URLRequestContext*
 125 URLRequestHttpJob::HttpFilterContext::GetURLRequestContext() const {
 126   return job_->request() ? job_->request()->context() : NULL;
 127 }
 128
 129 void URLRequestHttpJob::HttpFilterContext::RecordPacketStats(
 130     StatisticSelector statistic) const {
 131   job_->RecordPacketStats(statistic);
 132 }
 133
 134 const BoundNetLog& URLRequestHttpJob::HttpFilterContext::GetNetLog() const {
 135   return job_->request() ? job_->request()->net_log() : dummy_log_;
 136 }
 137
 138 // TODO(darin): make sure the port blocking code is not lost
 139 // static
 140 URLRequestJob* URLRequestHttpJob::Factory(URLRequest* request,
 141                                           NetworkDelegate* network_delegate,
 142                                           const std::string& scheme) {
 143   DCHECK(scheme == "http" || scheme == "https" || scheme == "ws" ||
 144          scheme == "wss");
 145
 146   if (!request->context()->http_transaction_factory()) {
 147     NOTREACHED() << "requires a valid context";
 148     return new URLRequestErrorJob(
 149         request, network_delegate, ERR_INVALID_ARGUMENT);
 150   }
 151
 152   GURL redirect_url;
 153   if (request->GetHSTSRedirect(&redirect_url)) {
 154     return new URLRequestRedirectJob(
 155         request, network_delegate, redirect_url,
 156         // Use status code 307 to preserve the method, so POST requests work.
 157         URLRequestRedirectJob::REDIRECT_307_TEMPORARY_REDIRECT, "HSTS");
 158   }
 159   return new URLRequestHttpJob(request,
 160                                network_delegate,
 161                                request->context()->http_user_agent_settings());
 162 }
 163
 164 URLRequestHttpJob::URLRequestHttpJob(
 165     URLRequest* request,
 166     NetworkDelegate* network_delegate,
 167     const HttpUserAgentSettings* http_user_agent_settings)
 168     : URLRequestJob(request, network_delegate),
 169       priority_(DEFAULT_PRIORITY),
 170       response_info_(NULL),
 171       response_cookies_save_index_(0),
 172       proxy_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
 173       server_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
 174       start_callback_(base::Bind(&URLRequestHttpJob::OnStartCompleted,
 175                                  base::Unretained(this))),
 176       notify_before_headers_sent_callback_(
 177           base::Bind(&URLRequestHttpJob::NotifyBeforeSendHeadersCallback,
 178                      base::Unretained(this))),
 179       read_in_progress_(false),
 180       throttling_entry_(NULL),
 181       sdch_test_activated_(false),
 182       sdch_test_control_(false),
 183       is_cached_content_(false),
 184       request_creation_time_(),
 185       packet_timing_enabled_(false),
 186       done_(false),
 187       bytes_observed_in_packets_(0),
 188       request_time_snapshot_(),
 189       final_packet_time_(),
 190       filter_context_(new HttpFilterContext(this)),
 191       on_headers_received_callback_(
 192           base::Bind(&URLRequestHttpJob::OnHeadersReceivedCallback,
 193                      base::Unretained(this))),
 194       awaiting_callback_(false),
 195       http_user_agent_settings_(http_user_agent_settings),
 196       weak_factory_(this) {
 197   URLRequestThrottlerManager* manager = request->context()->throttler_manager();
 198   if (manager)
 199     throttling_entry_ = manager->RegisterRequestUrl(request->url());
 200
 201   ResetTimer();
 202 }
 203
 204 URLRequestHttpJob::~URLRequestHttpJob() {
 205   CHECK(!awaiting_callback_);
 206
 207   DCHECK(!sdch_test_control_ || !sdch_test_activated_);
 208   if (!is_cached_content_) {
 209     if (sdch_test_control_)
 210       RecordPacketStats(FilterContext::SDCH_EXPERIMENT_HOLDBACK);
 211     if (sdch_test_activated_)
 212       RecordPacketStats(FilterContext::SDCH_EXPERIMENT_DECODE);
 213   }
 214   // Make sure SDCH filters are told to emit histogram data while
 215   // filter_context_ is still alive.
 216   DestroyFilters();
 217
 218   DoneWithRequest(ABORTED);
 219 }
 220
 221 void URLRequestHttpJob::SetPriority(RequestPriority priority) {
 222   priority_ = priority;
 223   if (transaction_)
 224     transaction_->SetPriority(priority_);
 225 }
 226
 227 void URLRequestHttpJob::Start() {
 228   DCHECK(!transaction_.get());
 229
 230   // URLRequest::SetReferrer ensures that we do not send username and password
 231   // fields in the referrer.
 232   GURL referrer(request_->referrer());
 233
 234   request_info_.url = request_->url();
 235   request_info_.method = request_->method();
 236   request_info_.load_flags = request_->load_flags();
 237   // Enable privacy mode if cookie settings or flags tell us not send or
 238   // save cookies.
 239   bool enable_privacy_mode =
 240       (request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES) ||
 241       (request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) ||
 242       CanEnablePrivacyMode();
 243   // Privacy mode could still be disabled in OnCookiesLoaded if we are going
 244   // to send previously saved cookies.
 245   request_info_.privacy_mode = enable_privacy_mode ?
 246       PRIVACY_MODE_ENABLED : PRIVACY_MODE_DISABLED;
 247
 248   // Strip Referer from request_info_.extra_headers to prevent, e.g., plugins
 249   // from overriding headers that are controlled using other means. Otherwise a
 250   // plugin could set a referrer although sending the referrer is inhibited.
 251   request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kReferer);
 252
 253   // Our consumer should have made sure that this is a safe referrer. See for
 254   // instance WebCore::FrameLoader::HideReferrer.
 255   if (referrer.is_valid()) {
 256     request_info_.extra_headers.SetHeader(HttpRequestHeaders::kReferer,
 257                                           referrer.spec());
 258   }
 259
 260   request_info_.extra_headers.SetHeaderIfMissing(
 261       HttpRequestHeaders::kUserAgent,
 262       http_user_agent_settings_ ?
 263           http_user_agent_settings_->GetUserAgent() : std::string());
 264
 265   AddExtraHeaders();
 266   AddCookieHeaderAndStart();
 267 }
 268
 269 void URLRequestHttpJob::Kill() {
 270   if (!transaction_.get())
 271     return;
 272
 273   weak_factory_.InvalidateWeakPtrs();
 274   DestroyTransaction();
 275   URLRequestJob::Kill();
 276 }
 277
 278 void URLRequestHttpJob::NotifyBeforeSendProxyHeadersCallback(
 279     const ProxyInfo& proxy_info,
 280     HttpRequestHeaders* request_headers) {
 281   DCHECK(request_headers);
 282   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 283   if (network_delegate()) {
 284     network_delegate()->NotifyBeforeSendProxyHeaders(
 285         request_,
 286         proxy_info,
 287         request_headers);
 288   }
 289 }
 290
 291 void URLRequestHttpJob::NotifyHeadersComplete() {
 292   DCHECK(!response_info_);
 293
 294   response_info_ = transaction_->GetResponseInfo();
 295
 296   // Save boolean, as we'll need this info at destruction time, and filters may
 297   // also need this info.
 298   is_cached_content_ = response_info_->was_cached;
 299
 300   if (!is_cached_content_ && throttling_entry_.get()) {
 301     URLRequestThrottlerHeaderAdapter response_adapter(GetResponseHeaders());
 302     throttling_entry_->UpdateWithResponse(request_info_.url.host(),
 303                                           &response_adapter);
 304   }
 305
 306   // The ordering of these calls is not important.
 307   ProcessStrictTransportSecurityHeader();
 308   ProcessPublicKeyPinsHeader();
 309
 310   // Handle the server notification of a new SDCH dictionary.
 311   SdchManager* sdch_manager(request()->context()->sdch_manager());
 312   if (sdch_manager) {
 313     SdchProblemCode rv = sdch_manager->IsInSupportedDomain(request()->url());
 314     if (rv != SDCH_OK) {
 315       // If SDCH is just disabled, it is not a real error.
 316       if (rv != SDCH_DISABLED && rv != SDCH_SECURE_SCHEME_NOT_SUPPORTED) {
 317         SdchManager::SdchErrorRecovery(rv);
 318         request()->net_log().AddEvent(
 319             NetLog::TYPE_SDCH_DECODING_ERROR,
 320             base::Bind(&NetLogSdchResourceProblemCallback, rv));
 321       }
 322     } else {
 323       const std::string name = "Get-Dictionary";
 324       std::string url_text;
 325       void* iter = NULL;
 326       // TODO(jar): We need to not fetch dictionaries the first time they are
 327       // seen, but rather wait until we can justify their usefulness.
 328       // For now, we will only fetch the first dictionary, which will at least
 329       // require multiple suggestions before we get additional ones for this
 330       // site. Eventually we should wait until a dictionary is requested
 331       // several times
 332       // before we even download it (so that we don't waste memory or
 333       // bandwidth).
 334       if (GetResponseHeaders()->EnumerateHeader(&iter, name, &url_text)) {
 335         // Resolve suggested URL relative to request url.
 336         GURL sdch_dictionary_url = request_->url().Resolve(url_text);
 337         if (sdch_dictionary_url.is_valid()) {
 338           rv = sdch_manager->OnGetDictionary(request_->url(),
 339                                              sdch_dictionary_url);
 340           if (rv != SDCH_OK) {
 341             SdchManager::SdchErrorRecovery(rv);
 342             request_->net_log().AddEvent(
 343                 NetLog::TYPE_SDCH_DICTIONARY_ERROR,
 344                 base::Bind(&NetLogSdchDictionaryFetchProblemCallback, rv,
 345                            sdch_dictionary_url, false));
 346           }
 347         }
 348       }
 349     }
 350   }
 351
 352   // Handle the server signalling no SDCH encoding.
 353   if (dictionaries_advertised_) {
 354     // We are wary of proxies that discard or damage SDCH encoding. If a server
 355     // explicitly states that this is not SDCH content, then we can correct our
 356     // assumption that this is an SDCH response, and avoid the need to recover
 357     // as though the content is corrupted (when we discover it is not SDCH
 358     // encoded).
 359     std::string sdch_response_status;
 360     void* iter = NULL;
 361     while (GetResponseHeaders()->EnumerateHeader(&iter, "X-Sdch-Encode",
 362                                                  &sdch_response_status)) {
 363       if (sdch_response_status == "0") {
 364         dictionaries_advertised_.reset();
 365         break;
 366       }
 367     }
 368   }
 369
 370   // The HTTP transaction may be restarted several times for the purposes
 371   // of sending authorization information. Each time it restarts, we get
 372   // notified of the headers completion so that we can update the cookie store.
 373   if (transaction_->IsReadyToRestartForAuth()) {
 374     DCHECK(!response_info_->auth_challenge.get());
 375     // TODO(battre): This breaks the webrequest API for
 376     // URLRequestTestHTTP.BasicAuthWithCookies
 377     // where OnBeforeSendHeaders -> OnSendHeaders -> OnBeforeSendHeaders
 378     // occurs.
 379     RestartTransactionWithAuth(AuthCredentials());
 380     return;
 381   }
 382
 383   URLRequestJob::NotifyHeadersComplete();
 384 }
 385
 386 void URLRequestHttpJob::NotifyDone(const URLRequestStatus& status) {
 387   DoneWithRequest(FINISHED);
 388   URLRequestJob::NotifyDone(status);
 389 }
 390
 391 void URLRequestHttpJob::DestroyTransaction() {
 392   DCHECK(transaction_.get());
 393
 394   DoneWithRequest(ABORTED);
 395   transaction_.reset();
 396   response_info_ = NULL;
 397   receive_headers_end_ = base::TimeTicks();
 398 }
 399
 400 void URLRequestHttpJob::StartTransaction() {
 401   if (network_delegate()) {
 402     OnCallToDelegate();
 403     int rv = network_delegate()->NotifyBeforeSendHeaders(
 404         request_, notify_before_headers_sent_callback_,
 405         &request_info_.extra_headers);
 406     // If an extension blocks the request, we rely on the callback to
 407     // MaybeStartTransactionInternal().
 408     if (rv == ERR_IO_PENDING)
 409       return;
 410     MaybeStartTransactionInternal(rv);
 411     return;
 412   }
 413   StartTransactionInternal();
 414 }
 415
 416 void URLRequestHttpJob::NotifyBeforeSendHeadersCallback(int result) {
 417   // Check that there are no callbacks to already canceled requests.
 418   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 419
 420   MaybeStartTransactionInternal(result);
 421 }
 422
 423 void URLRequestHttpJob::MaybeStartTransactionInternal(int result) {
 424   OnCallToDelegateComplete();
 425   if (result == OK) {
 426     StartTransactionInternal();
 427   } else {
 428     std::string source("delegate");
 429     request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 430                                  NetLog::StringCallback("source", &source));
 431     NotifyCanceled();
 432     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 433   }
 434 }
 435
 436 void URLRequestHttpJob::StartTransactionInternal() {
 437   // NOTE: This method assumes that request_info_ is already setup properly.
 438
 439   // If we already have a transaction, then we should restart the transaction
 440   // with auth provided by auth_credentials_.
 441
 442   int rv;
 443
 444   if (network_delegate()) {
 445     network_delegate()->NotifySendHeaders(
 446         request_, request_info_.extra_headers);
 447   }
 448
 449   if (transaction_.get()) {
 450     rv = transaction_->RestartWithAuth(auth_credentials_, start_callback_);
 451     auth_credentials_ = AuthCredentials();
 452   } else {
 453     DCHECK(request_->context()->http_transaction_factory());
 454
 455     rv = request_->context()->http_transaction_factory()->CreateTransaction(
 456         priority_, &transaction_);
 457
 458     if (rv == OK && request_info_.url.SchemeIsWSOrWSS()) {
 459       base::SupportsUserData::Data* data = request_->GetUserData(
 460           WebSocketHandshakeStreamBase::CreateHelper::DataKey());
 461       if (data) {
 462         transaction_->SetWebSocketHandshakeStreamCreateHelper(
 463             static_cast<WebSocketHandshakeStreamBase::CreateHelper*>(data));
 464       } else {
 465         rv = ERR_DISALLOWED_URL_SCHEME;
 466       }
 467     }
 468
 469     if (rv == OK) {
 470       transaction_->SetBeforeNetworkStartCallback(
 471           base::Bind(&URLRequestHttpJob::NotifyBeforeNetworkStart,
 472                      base::Unretained(this)));
 473       transaction_->SetBeforeProxyHeadersSentCallback(
 474           base::Bind(&URLRequestHttpJob::NotifyBeforeSendProxyHeadersCallback,
 475                      base::Unretained(this)));
 476
 477       if (!throttling_entry_.get() ||
 478           !throttling_entry_->ShouldRejectRequest(*request_,
 479                                                   network_delegate())) {
 480         rv = transaction_->Start(
 481             &request_info_, start_callback_, request_->net_log());
 482         start_time_ = base::TimeTicks::Now();
 483       } else {
 484         // Special error code for the exponential back-off module.
 485         rv = ERR_TEMPORARILY_THROTTLED;
 486       }
 487     }
 488   }
 489
 490   if (rv == ERR_IO_PENDING)
 491     return;
 492
 493   // The transaction started synchronously, but we need to notify the
 494   // URLRequest delegate via the message loop.
 495   base::MessageLoop::current()->PostTask(
 496       FROM_HERE,
 497       base::Bind(&URLRequestHttpJob::OnStartCompleted,
 498                  weak_factory_.GetWeakPtr(), rv));
 499 }
 500
 501 void URLRequestHttpJob::AddExtraHeaders() {
 502   SdchManager* sdch_manager = request()->context()->sdch_manager();
 503
 504   // Supply Accept-Encoding field only if it is not already provided.
 505   // It should be provided IF the content is known to have restrictions on
 506   // potential encoding, such as streaming multi-media.
 507   // For details see bug 47381.
 508   // TODO(jar, enal): jpeg files etc. should set up a request header if
 509   // possible. Right now it is done only by buffered_resource_loader and
 510   // simple_data_source.
 511   if (!request_info_.extra_headers.HasHeader(
 512       HttpRequestHeaders::kAcceptEncoding)) {
 513     // We don't support SDCH responses to POST as there is a possibility
 514     // of having SDCH encoded responses returned (e.g. by the cache)
 515     // which we cannot decode, and in those situations, we will need
 516     // to retransmit the request without SDCH, which is illegal for a POST.
 517     bool advertise_sdch = sdch_manager != NULL && request()->method() != "POST";
 518     if (advertise_sdch) {
 519       SdchProblemCode rv = sdch_manager->IsInSupportedDomain(request()->url());
 520       if (rv != SDCH_OK) {
 521         advertise_sdch = false;
 522         // If SDCH is just disabled, it is not a real error.
 523         if (rv != SDCH_DISABLED && rv != SDCH_SECURE_SCHEME_NOT_SUPPORTED) {
 524           SdchManager::SdchErrorRecovery(rv);
 525           request()->net_log().AddEvent(
 526               NetLog::TYPE_SDCH_DECODING_ERROR,
 527               base::Bind(&NetLogSdchResourceProblemCallback, rv));
 528         }
 529       }
 530     }
 531     if (advertise_sdch) {
 532       dictionaries_advertised_ =
 533           sdch_manager->GetDictionarySet(request_->url());
 534     }
 535
 536     // The AllowLatencyExperiment() is only true if we've successfully done a
 537     // full SDCH compression recently in this browser session for this host.
 538     // Note that for this path, there might be no applicable dictionaries,
 539     // and hence we can't participate in the experiment.
 540     if (dictionaries_advertised_ &&
 541         sdch_manager->AllowLatencyExperiment(request_->url())) {
 542       // We are participating in the test (or control), and hence we'll
 543       // eventually record statistics via either SDCH_EXPERIMENT_DECODE or
 544       // SDCH_EXPERIMENT_HOLDBACK, and we'll need some packet timing data.
 545       packet_timing_enabled_ = true;
 546       if (base::RandDouble() < .01) {
 547         sdch_test_control_ = true;  // 1% probability.
 548         dictionaries_advertised_.reset();
 549         advertise_sdch = false;
 550       } else {
 551         sdch_test_activated_ = true;
 552       }
 553     }
 554
 555     // Supply Accept-Encoding headers first so that it is more likely that they
 556     // will be in the first transmitted packet. This can sometimes make it
 557     // easier to filter and analyze the streams to assure that a proxy has not
 558     // damaged these headers. Some proxies deliberately corrupt Accept-Encoding
 559     // headers.
 560     if (!advertise_sdch) {
 561       // Tell the server what compression formats we support (other than SDCH).
 562       request_info_.extra_headers.SetHeader(
 563           HttpRequestHeaders::kAcceptEncoding, "gzip, deflate");
 564     } else {
 565       // Include SDCH in acceptable list.
 566       request_info_.extra_headers.SetHeader(
 567           HttpRequestHeaders::kAcceptEncoding, "gzip, deflate, sdch");
 568       if (dictionaries_advertised_) {
 569         request_info_.extra_headers.SetHeader(
 570             kAvailDictionaryHeader,
 571             dictionaries_advertised_->GetDictionaryClientHashList());
 572         // Since we're tagging this transaction as advertising a dictionary,
 573         // we'll definitely employ an SDCH filter (or tentative sdch filter)
 574         // when we get a response. When done, we'll record histograms via
 575         // SDCH_DECODE or SDCH_PASSTHROUGH. Hence we need to record packet
 576         // arrival times.
 577         packet_timing_enabled_ = true;
 578       }
 579     }
 580   }
 581
 582   if (http_user_agent_settings_) {
 583     // Only add default Accept-Language if the request didn't have it
 584     // specified.
 585     std::string accept_language =
 586         http_user_agent_settings_->GetAcceptLanguage();
 587     if (!accept_language.empty()) {
 588       request_info_.extra_headers.SetHeaderIfMissing(
 589           HttpRequestHeaders::kAcceptLanguage,
 590           accept_language);
 591     }
 592   }
 593 }
 594
 595 void URLRequestHttpJob::AddCookieHeaderAndStart() {
 596   // No matter what, we want to report our status as IO pending since we will
 597   // be notifying our consumer asynchronously via OnStartCompleted.
 598   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
 599
 600   // If the request was destroyed, then there is no more work to do.
 601   if (!request_)
 602     return;
 603
 604   CookieStore* cookie_store = request_->context()->cookie_store();
 605   if (cookie_store && !(request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES)) {
 606     cookie_store->GetAllCookiesForURLAsync(
 607         request_->url(),
 608         base::Bind(&URLRequestHttpJob::CheckCookiePolicyAndLoad,
 609                    weak_factory_.GetWeakPtr()));
 610   } else {
 611     DoStartTransaction();
 612   }
 613 }
 614
 615 void URLRequestHttpJob::DoLoadCookies() {
 616   CookieOptions options;
 617   options.set_include_httponly();
 618
 619   // TODO(mkwst): Drop this `if` once we decide whether or not to ship
 620   // first-party cookies: https://crbug.com/459154
 621   if (network_delegate() &&
 622       network_delegate()->FirstPartyOnlyCookieExperimentEnabled())
 623     options.set_first_party_url(request_->first_party_for_cookies());
 624   else
 625     options.set_include_first_party_only();
 626
 627   request_->context()->cookie_store()->GetCookiesWithOptionsAsync(
 628       request_->url(), options, base::Bind(&URLRequestHttpJob::OnCookiesLoaded,
 629                                            weak_factory_.GetWeakPtr()));
 630 }
 631
 632 void URLRequestHttpJob::CheckCookiePolicyAndLoad(
 633     const CookieList& cookie_list) {
 634   if (CanGetCookies(cookie_list))
 635     DoLoadCookies();
 636   else
 637     DoStartTransaction();
 638 }
 639
 640 void URLRequestHttpJob::OnCookiesLoaded(const std::string& cookie_line) {
 641   if (!cookie_line.empty()) {
 642     request_info_.extra_headers.SetHeader(
 643         HttpRequestHeaders::kCookie, cookie_line);
 644     // Disable privacy mode as we are sending cookies anyway.
 645     request_info_.privacy_mode = PRIVACY_MODE_DISABLED;
 646   }
 647   DoStartTransaction();
 648 }
 649
 650 void URLRequestHttpJob::DoStartTransaction() {
 651   // We may have been canceled while retrieving cookies.
 652   if (GetStatus().is_success()) {
 653     StartTransaction();
 654   } else {
 655     NotifyCanceled();
 656   }
 657 }
 658
 659 void URLRequestHttpJob::SaveCookiesAndNotifyHeadersComplete(int result) {
 660   // End of the call started in OnStartCompleted.
 661   OnCallToDelegateComplete();
 662
 663   if (result != net::OK) {
 664     std::string source("delegate");
 665     request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 666                                  NetLog::StringCallback("source", &source));
 667     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 668     return;
 669   }
 670
 671   DCHECK(transaction_.get());
 672
 673   const HttpResponseInfo* response_info = transaction_->GetResponseInfo();
 674   DCHECK(response_info);
 675
 676   response_cookies_.clear();
 677   response_cookies_save_index_ = 0;
 678
 679   FetchResponseCookies(&response_cookies_);
 680
 681   if (!GetResponseHeaders()->GetDateValue(&response_date_))
 682     response_date_ = base::Time();
 683
 684   // Now, loop over the response cookies, and attempt to persist each.
 685   SaveNextCookie();
 686 }
 687
 688 // If the save occurs synchronously, SaveNextCookie will loop and save the next
 689 // cookie. If the save is deferred, the callback is responsible for continuing
 690 // to iterate through the cookies.
 691 // TODO(erikwright): Modify the CookieStore API to indicate via return value
 692 // whether it completed synchronously or asynchronously.
 693 // See http://crbug.com/131066.
 694 void URLRequestHttpJob::SaveNextCookie() {
 695   // No matter what, we want to report our status as IO pending since we will
 696   // be notifying our consumer asynchronously via OnStartCompleted.
 697   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
 698
 699   // Used to communicate with the callback. See the implementation of
 700   // OnCookieSaved.
 701   scoped_refptr<SharedBoolean> callback_pending = new SharedBoolean(false);
 702   scoped_refptr<SharedBoolean> save_next_cookie_running =
 703       new SharedBoolean(true);
 704
 705   if (!(request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) &&
 706       request_->context()->cookie_store() && response_cookies_.size() > 0) {
 707     CookieOptions options;
 708     options.set_include_httponly();
 709     options.set_server_time(response_date_);
 710
 711     net::CookieStore::SetCookiesCallback callback(
 712         base::Bind(&URLRequestHttpJob::OnCookieSaved,
 713                    weak_factory_.GetWeakPtr(),
 714                    save_next_cookie_running,
 715                    callback_pending));
 716
 717     // Loop through the cookies as long as SetCookieWithOptionsAsync completes
 718     // synchronously.
 719     while (!callback_pending->data &&
 720            response_cookies_save_index_ < response_cookies_.size()) {
 721       if (CanSetCookie(
 722           response_cookies_[response_cookies_save_index_], &options)) {
 723         callback_pending->data = true;
 724         request_->context()->cookie_store()->SetCookieWithOptionsAsync(
 725             request_->url(), response_cookies_[response_cookies_save_index_],
 726             options, callback);
 727       }
 728       ++response_cookies_save_index_;
 729     }
 730   }
 731
 732   save_next_cookie_running->data = false;
 733
 734   if (!callback_pending->data) {
 735     response_cookies_.clear();
 736     response_cookies_save_index_ = 0;
 737     SetStatus(URLRequestStatus());  // Clear the IO_PENDING status
 738     NotifyHeadersComplete();
 739     return;
 740   }
 741 }
 742
 743 // |save_next_cookie_running| is true when the callback is bound and set to
 744 // false when SaveNextCookie exits, allowing the callback to determine if the
 745 // save occurred synchronously or asynchronously.
 746 // |callback_pending| is false when the callback is invoked and will be set to
 747 // true by the callback, allowing SaveNextCookie to detect whether the save
 748 // occurred synchronously.
 749 // See SaveNextCookie() for more information.
 750 void URLRequestHttpJob::OnCookieSaved(
 751     scoped_refptr<SharedBoolean> save_next_cookie_running,
 752     scoped_refptr<SharedBoolean> callback_pending,
 753     bool cookie_status) {
 754   callback_pending->data = false;
 755
 756   // If we were called synchronously, return.
 757   if (save_next_cookie_running->data) {
 758     return;
 759   }
 760
 761   // We were called asynchronously, so trigger the next save.
 762   // We may have been canceled within OnSetCookie.
 763   if (GetStatus().is_success()) {
 764     SaveNextCookie();
 765   } else {
 766     NotifyCanceled();
 767   }
 768 }
 769
 770 void URLRequestHttpJob::FetchResponseCookies(
 771     std::vector<std::string>* cookies) {
 772   const std::string name = "Set-Cookie";
 773   std::string value;
 774
 775   void* iter = NULL;
 776   HttpResponseHeaders* headers = GetResponseHeaders();
 777   while (headers->EnumerateHeader(&iter, name, &value)) {
 778     if (!value.empty())
 779       cookies->push_back(value);
 780   }
 781 }
 782
 783 // NOTE: |ProcessStrictTransportSecurityHeader| and
 784 // |ProcessPublicKeyPinsHeader| have very similar structures, by design.
 785 void URLRequestHttpJob::ProcessStrictTransportSecurityHeader() {
 786   DCHECK(response_info_);
 787   TransportSecurityState* security_state =
 788       request_->context()->transport_security_state();
 789   const SSLInfo& ssl_info = response_info_->ssl_info;
 790
 791   // Only accept HSTS headers on HTTPS connections that have no
 792   // certificate errors.
 793   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 794       !security_state)
 795     return;
 796
 797   // http://tools.ietf.org/html/draft-ietf-websec-strict-transport-sec:
 798   //
 799   //   If a UA receives more than one STS header field in a HTTP response
 800   //   message over secure transport, then the UA MUST process only the
 801   //   first such header field.
 802   HttpResponseHeaders* headers = GetResponseHeaders();
 803   std::string value;
 804   if (headers->EnumerateHeader(NULL, "Strict-Transport-Security", &value))
 805     security_state->AddHSTSHeader(request_info_.url.host(), value);
 806 }
 807
 808 void URLRequestHttpJob::ProcessPublicKeyPinsHeader() {
 809   DCHECK(response_info_);
 810   TransportSecurityState* security_state =
 811       request_->context()->transport_security_state();
 812   const SSLInfo& ssl_info = response_info_->ssl_info;
 813
 814   // Only accept HPKP headers on HTTPS connections that have no
 815   // certificate errors.
 816   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 817       !security_state)
 818     return;
 819
 820   // http://tools.ietf.org/html/draft-ietf-websec-key-pinning:
 821   //
 822   //   If a UA receives more than one PKP header field in an HTTP
 823   //   response message over secure transport, then the UA MUST process
 824   //   only the first such header field.
 825   HttpResponseHeaders* headers = GetResponseHeaders();
 826   std::string value;
 827   if (headers->EnumerateHeader(NULL, "Public-Key-Pins", &value))
 828     security_state->AddHPKPHeader(request_info_.url.host(), value, ssl_info);
 829 }
 830
 831 void URLRequestHttpJob::OnStartCompleted(int result) {
 832   // TODO(vadimt): Remove ScopedTracker below once crbug.com/424359 is fixed.
 833   tracked_objects::ScopedTracker tracking_profile(
 834       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 835           "424359 URLRequestHttpJob::OnStartCompleted"));
 836
 837   RecordTimer();
 838
 839   // If the request was destroyed, then there is no more work to do.
 840   if (!request_)
 841     return;
 842
 843   // If the job is done (due to cancellation), can just ignore this
 844   // notification.
 845   if (done_)
 846     return;
 847
 848   receive_headers_end_ = base::TimeTicks::Now();
 849
 850   // Clear the IO_PENDING status
 851   SetStatus(URLRequestStatus());
 852
 853   const URLRequestContext* context = request_->context();
 854
 855   if (result == ERR_SSL_PINNED_KEY_NOT_IN_CERT_CHAIN &&
 856       transaction_->GetResponseInfo() != NULL) {
 857     FraudulentCertificateReporter* reporter =
 858       context->fraudulent_certificate_reporter();
 859     if (reporter != NULL) {
 860       const SSLInfo& ssl_info = transaction_->GetResponseInfo()->ssl_info;
 861       const std::string& host = request_->url().host();
 862
 863       reporter->SendReport(host, ssl_info);
 864     }
 865   }
 866
 867   if (result == OK) {
 868     if (transaction_ && transaction_->GetResponseInfo()) {
 869       SetProxyServer(transaction_->GetResponseInfo()->proxy_server);
 870     }
 871     scoped_refptr<HttpResponseHeaders> headers = GetResponseHeaders();
 872     if (network_delegate()) {
 873       // Note that |this| may not be deleted until
 874       // |on_headers_received_callback_| or
 875       // |NetworkDelegate::URLRequestDestroyed()| has been called.
 876       OnCallToDelegate();
 877       allowed_unsafe_redirect_url_ = GURL();
 878       int error = network_delegate()->NotifyHeadersReceived(
 879           request_,
 880           on_headers_received_callback_,
 881           headers.get(),
 882           &override_response_headers_,
 883           &allowed_unsafe_redirect_url_);
 884       if (error != net::OK) {
 885         if (error == net::ERR_IO_PENDING) {
 886           awaiting_callback_ = true;
 887         } else {
 888           std::string source("delegate");
 889           request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 890                                        NetLog::StringCallback("source",
 891                                                               &source));
 892           OnCallToDelegateComplete();
 893           NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, error));
 894         }
 895         return;
 896       }
 897     }
 898
 899     SaveCookiesAndNotifyHeadersComplete(net::OK);
 900   } else if (IsCertificateError(result)) {
 901     // We encountered an SSL certificate error.
 902     if (result == ERR_SSL_WEAK_SERVER_EPHEMERAL_DH_KEY ||
 903         result == ERR_SSL_PINNED_KEY_NOT_IN_CERT_CHAIN) {
 904       // These are hard failures. They're handled separately and don't have
 905       // the correct cert status, so set it here.
 906       SSLInfo info(transaction_->GetResponseInfo()->ssl_info);
 907       info.cert_status = MapNetErrorToCertStatus(result);
 908       NotifySSLCertificateError(info, true);
 909     } else {
 910       // Maybe overridable, maybe not. Ask the delegate to decide.
 911       TransportSecurityState* state = context->transport_security_state();
 912       const bool fatal =
 913           state && state->ShouldSSLErrorsBeFatal(request_info_.url.host());
 914       NotifySSLCertificateError(
 915           transaction_->GetResponseInfo()->ssl_info, fatal);
 916     }
 917   } else if (result == ERR_SSL_CLIENT_AUTH_CERT_NEEDED) {
 918     NotifyCertificateRequested(
 919         transaction_->GetResponseInfo()->cert_request_info.get());
 920   } else {
 921     // Even on an error, there may be useful information in the response
 922     // info (e.g. whether there's a cached copy).
 923     if (transaction_.get())
 924       response_info_ = transaction_->GetResponseInfo();
 925     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 926   }
 927 }
 928
 929 void URLRequestHttpJob::OnHeadersReceivedCallback(int result) {
 930   awaiting_callback_ = false;
 931
 932   // Check that there are no callbacks to already canceled requests.
 933   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 934
 935   SaveCookiesAndNotifyHeadersComplete(result);
 936 }
 937
 938 void URLRequestHttpJob::OnReadCompleted(int result) {
 939   // TODO(vadimt): Remove ScopedTracker below once crbug.com/424359 is fixed.
 940   tracked_objects::ScopedTracker tracking_profile(
 941       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 942           "424359 URLRequestHttpJob::OnReadCompleted"));
 943
 944   read_in_progress_ = false;
 945
 946   if (ShouldFixMismatchedContentLength(result))
 947     result = OK;
 948
 949   if (result == OK) {
 950     NotifyDone(URLRequestStatus());
 951   } else if (result < 0) {
 952     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, result));
 953   } else {
 954     // Clear the IO_PENDING status
 955     SetStatus(URLRequestStatus());
 956   }
 957
 958   NotifyReadComplete(result);
 959 }
 960
 961 void URLRequestHttpJob::RestartTransactionWithAuth(
 962     const AuthCredentials& credentials) {
 963   auth_credentials_ = credentials;
 964
 965   // These will be reset in OnStartCompleted.
 966   response_info_ = NULL;
 967   receive_headers_end_ = base::TimeTicks();
 968   response_cookies_.clear();
 969
 970   ResetTimer();
 971
 972   // Update the cookies, since the cookie store may have been updated from the
 973   // headers in the 401/407. Since cookies were already appended to
 974   // extra_headers, we need to strip them out before adding them again.
 975   request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kCookie);
 976
 977   AddCookieHeaderAndStart();
 978 }
 979
 980 void URLRequestHttpJob::SetUpload(UploadDataStream* upload) {
 981   DCHECK(!transaction_.get()) << "cannot change once started";
 982   request_info_.upload_data_stream = upload;
 983 }
 984
 985 void URLRequestHttpJob::SetExtraRequestHeaders(
 986     const HttpRequestHeaders& headers) {
 987   DCHECK(!transaction_.get()) << "cannot change once started";
 988   request_info_.extra_headers.CopyFrom(headers);
 989 }
 990
 991 LoadState URLRequestHttpJob::GetLoadState() const {
 992   return transaction_.get() ?
 993       transaction_->GetLoadState() : LOAD_STATE_IDLE;
 994 }
 995
 996 UploadProgress URLRequestHttpJob::GetUploadProgress() const {
 997   return transaction_.get() ?
 998       transaction_->GetUploadProgress() : UploadProgress();
 999 }
1000
1001 bool URLRequestHttpJob::GetMimeType(std::string* mime_type) const {
1002   DCHECK(transaction_.get());
1003
1004   if (!response_info_)
1005     return false;
1006
1007   HttpResponseHeaders* headers = GetResponseHeaders();
1008   if (!headers)
1009     return false;
1010   return headers->GetMimeType(mime_type);
1011 }
1012
1013 bool URLRequestHttpJob::GetCharset(std::string* charset) {
1014   DCHECK(transaction_.get());
1015
1016   if (!response_info_)
1017     return false;
1018
1019   return GetResponseHeaders()->GetCharset(charset);
1020 }
1021
1022 void URLRequestHttpJob::GetResponseInfo(HttpResponseInfo* info) {
1023   DCHECK(request_);
1024
1025   if (response_info_) {
1026     DCHECK(transaction_.get());
1027
1028     *info = *response_info_;
1029     if (override_response_headers_.get())
1030       info->headers = override_response_headers_;
1031   }
1032 }
1033
1034 void URLRequestHttpJob::GetLoadTimingInfo(
1035     LoadTimingInfo* load_timing_info) const {
1036   // If haven't made it far enough to receive any headers, don't return
1037   // anything. This makes for more consistent behavior in the case of errors.
1038   if (!transaction_ || receive_headers_end_.is_null())
1039     return;
1040   if (transaction_->GetLoadTimingInfo(load_timing_info))
1041     load_timing_info->receive_headers_end = receive_headers_end_;
1042 }
1043
1044 bool URLRequestHttpJob::GetResponseCookies(std::vector<std::string>* cookies) {
1045   DCHECK(transaction_.get());
1046
1047   if (!response_info_)
1048     return false;
1049
1050   // TODO(darin): Why are we extracting response cookies again?  Perhaps we
1051   // should just leverage response_cookies_.
1052
1053   cookies->clear();
1054   FetchResponseCookies(cookies);
1055   return true;
1056 }
1057
1058 int URLRequestHttpJob::GetResponseCode() const {
1059   DCHECK(transaction_.get());
1060
1061   if (!response_info_)
1062     return -1;
1063
1064   return GetResponseHeaders()->response_code();
1065 }
1066
1067 Filter* URLRequestHttpJob::SetupFilter() const {
1068   DCHECK(transaction_.get());
1069   if (!response_info_)
1070     return NULL;
1071
1072   std::vector<Filter::FilterType> encoding_types;
1073   std::string encoding_type;
1074   HttpResponseHeaders* headers = GetResponseHeaders();
1075   void* iter = NULL;
1076   while (headers->EnumerateHeader(&iter, "Content-Encoding", &encoding_type)) {
1077     encoding_types.push_back(Filter::ConvertEncodingToType(encoding_type));
1078   }
1079
1080   // Even if encoding types are empty, there is a chance that we need to add
1081   // some decoding, as some proxies strip encoding completely. In such cases,
1082   // we may need to add (for example) SDCH filtering (when the context suggests
1083   // it is appropriate).
1084   Filter::FixupEncodingTypes(*filter_context_, &encoding_types);
1085
1086   return !encoding_types.empty()
1087       ? Filter::Factory(encoding_types, *filter_context_) : NULL;
1088 }
1089
1090 bool URLRequestHttpJob::CopyFragmentOnRedirect(const GURL& location) const {
1091   // Allow modification of reference fragments by default, unless
1092   // |allowed_unsafe_redirect_url_| is set and equal to the redirect URL.
1093   // When this is the case, we assume that the network delegate has set the
1094   // desired redirect URL (with or without fragment), so it must not be changed
1095   // any more.
1096   return !allowed_unsafe_redirect_url_.is_valid() ||
1097        allowed_unsafe_redirect_url_ != location;
1098 }
1099
1100 bool URLRequestHttpJob::IsSafeRedirect(const GURL& location) {
1101   // HTTP is always safe.
1102   // TODO(pauljensen): Remove once crbug.com/146591 is fixed.
1103   if (location.is_valid() &&
1104       (location.scheme() == "http" || location.scheme() == "https")) {
1105     return true;
1106   }
1107   // Delegates may mark a URL as safe for redirection.
1108   if (allowed_unsafe_redirect_url_.is_valid() &&
1109       allowed_unsafe_redirect_url_ == location) {
1110     return true;
1111   }
1112   // Query URLRequestJobFactory as to whether |location| would be safe to
1113   // redirect to.
1114   return request_->context()->job_factory() &&
1115       request_->context()->job_factory()->IsSafeRedirectTarget(location);
1116 }
1117
1118 bool URLRequestHttpJob::NeedsAuth() {
1119   int code = GetResponseCode();
1120   if (code == -1)
1121     return false;
1122
1123   // Check if we need either Proxy or WWW Authentication. This could happen
1124   // because we either provided no auth info, or provided incorrect info.
1125   switch (code) {
1126     case 407:
1127       if (proxy_auth_state_ == AUTH_STATE_CANCELED)
1128         return false;
1129       proxy_auth_state_ = AUTH_STATE_NEED_AUTH;
1130       return true;
1131     case 401:
1132       if (server_auth_state_ == AUTH_STATE_CANCELED)
1133         return false;
1134       server_auth_state_ = AUTH_STATE_NEED_AUTH;
1135       return true;
1136   }
1137   return false;
1138 }
1139
1140 void URLRequestHttpJob::GetAuthChallengeInfo(
1141     scoped_refptr<AuthChallengeInfo>* result) {
1142   DCHECK(transaction_.get());
1143   DCHECK(response_info_);
1144
1145   // sanity checks:
1146   DCHECK(proxy_auth_state_ == AUTH_STATE_NEED_AUTH ||
1147          server_auth_state_ == AUTH_STATE_NEED_AUTH);
1148   DCHECK((GetResponseHeaders()->response_code() == HTTP_UNAUTHORIZED) ||
1149          (GetResponseHeaders()->response_code() ==
1150           HTTP_PROXY_AUTHENTICATION_REQUIRED));
1151
1152   *result = response_info_->auth_challenge;
1153 }
1154
1155 void URLRequestHttpJob::SetAuth(const AuthCredentials& credentials) {
1156   DCHECK(transaction_.get());
1157
1158   // Proxy gets set first, then WWW.
1159   if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1160     proxy_auth_state_ = AUTH_STATE_HAVE_AUTH;
1161   } else {
1162     DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1163     server_auth_state_ = AUTH_STATE_HAVE_AUTH;
1164   }
1165
1166   RestartTransactionWithAuth(credentials);
1167 }
1168
1169 void URLRequestHttpJob::CancelAuth() {
1170   // Proxy gets set first, then WWW.
1171   if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1172     proxy_auth_state_ = AUTH_STATE_CANCELED;
1173   } else {
1174     DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1175     server_auth_state_ = AUTH_STATE_CANCELED;
1176   }
1177
1178   // These will be reset in OnStartCompleted.
1179   response_info_ = NULL;
1180   receive_headers_end_ = base::TimeTicks::Now();
1181   response_cookies_.clear();
1182
1183   ResetTimer();
1184
1185   // OK, let the consumer read the error page...
1186   //
1187   // Because we set the AUTH_STATE_CANCELED flag, NeedsAuth will return false,
1188   // which will cause the consumer to receive OnResponseStarted instead of
1189   // OnAuthRequired.
1190   //
1191   // We have to do this via InvokeLater to avoid "recursing" the consumer.
1192   //
1193   base::MessageLoop::current()->PostTask(
1194       FROM_HERE,
1195       base::Bind(&URLRequestHttpJob::OnStartCompleted,
1196                  weak_factory_.GetWeakPtr(), OK));
1197 }
1198
1199 void URLRequestHttpJob::ContinueWithCertificate(
1200     X509Certificate* client_cert) {
1201   DCHECK(transaction_.get());
1202
1203   DCHECK(!response_info_) << "should not have a response yet";
1204   receive_headers_end_ = base::TimeTicks();
1205
1206   ResetTimer();
1207
1208   // No matter what, we want to report our status as IO pending since we will
1209   // be notifying our consumer asynchronously via OnStartCompleted.
1210   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1211
1212   int rv = transaction_->RestartWithCertificate(client_cert, start_callback_);
1213   if (rv == ERR_IO_PENDING)
1214     return;
1215
1216   // The transaction started synchronously, but we need to notify the
1217   // URLRequest delegate via the message loop.
1218   base::MessageLoop::current()->PostTask(
1219       FROM_HERE,
1220       base::Bind(&URLRequestHttpJob::OnStartCompleted,
1221                  weak_factory_.GetWeakPtr(), rv));
1222 }
1223
1224 void URLRequestHttpJob::ContinueDespiteLastError() {
1225   // If the transaction was destroyed, then the job was cancelled.
1226   if (!transaction_.get())
1227     return;
1228
1229   DCHECK(!response_info_) << "should not have a response yet";
1230   receive_headers_end_ = base::TimeTicks();
1231
1232   ResetTimer();
1233
1234   // No matter what, we want to report our status as IO pending since we will
1235   // be notifying our consumer asynchronously via OnStartCompleted.
1236   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1237
1238   int rv = transaction_->RestartIgnoringLastError(start_callback_);
1239   if (rv == ERR_IO_PENDING)
1240     return;
1241
1242   // The transaction started synchronously, but we need to notify the
1243   // URLRequest delegate via the message loop.
1244   base::MessageLoop::current()->PostTask(
1245       FROM_HERE,
1246       base::Bind(&URLRequestHttpJob::OnStartCompleted,
1247                  weak_factory_.GetWeakPtr(), rv));
1248 }
1249
1250 void URLRequestHttpJob::ResumeNetworkStart() {
1251   DCHECK(transaction_.get());
1252   transaction_->ResumeNetworkStart();
1253 }
1254
1255 bool URLRequestHttpJob::ShouldFixMismatchedContentLength(int rv) const {
1256   // Some servers send the body compressed, but specify the content length as
1257   // the uncompressed size. Although this violates the HTTP spec we want to
1258   // support it (as IE and FireFox do), but *only* for an exact match.
1259   // See http://crbug.com/79694.
1260   if (rv == net::ERR_CONTENT_LENGTH_MISMATCH ||
1261       rv == net::ERR_INCOMPLETE_CHUNKED_ENCODING) {
1262     if (request_ && request_->response_headers()) {
1263       int64 expected_length = request_->response_headers()->GetContentLength();
1264       VLOG(1) << __FUNCTION__ << "() "
1265               << "\"" << request_->url().spec() << "\""
1266               << " content-length = " << expected_length
1267               << " pre total = " << prefilter_bytes_read()
1268               << " post total = " << postfilter_bytes_read();
1269       if (postfilter_bytes_read() == expected_length) {
1270         // Clear the error.
1271         return true;
1272       }
1273     }
1274   }
1275   return false;
1276 }
1277
1278 bool URLRequestHttpJob::ReadRawData(IOBuffer* buf, int buf_size,
1279                                     int* bytes_read) {
1280   // TODO(vadimt): Remove ScopedTracker below once crbug.com/423948 is fixed.
1281   tracked_objects::ScopedTracker tracking_profile1(
1282       FROM_HERE_WITH_EXPLICIT_FUNCTION(
1283           "423948 URLRequestHttpJob::ReadRawData1"));
1284
1285   DCHECK_NE(buf_size, 0);
1286   DCHECK(bytes_read);
1287   DCHECK(!read_in_progress_);
1288
1289   int rv = transaction_->Read(
1290       buf, buf_size,
1291       base::Bind(&URLRequestHttpJob::OnReadCompleted, base::Unretained(this)));
1292
1293   if (ShouldFixMismatchedContentLength(rv))
1294     rv = 0;
1295
1296   if (rv >= 0) {
1297     *bytes_read = rv;
1298     if (!rv) {
1299       // TODO(vadimt): Remove ScopedTracker below once crbug.com/423948 is
1300       // fixed.
1301       tracked_objects::ScopedTracker tracking_profile2(
1302           FROM_HERE_WITH_EXPLICIT_FUNCTION(
1303               "423948 URLRequestHttpJob::ReadRawData2"));
1304
1305       DoneWithRequest(FINISHED);
1306     }
1307     return true;
1308   }
1309
1310   if (rv == ERR_IO_PENDING) {
1311     read_in_progress_ = true;
1312     SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1313   } else {
1314     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, rv));
1315   }
1316
1317   return false;
1318 }
1319
1320 void URLRequestHttpJob::StopCaching() {
1321   if (transaction_.get())
1322     transaction_->StopCaching();
1323 }
1324
1325 bool URLRequestHttpJob::GetFullRequestHeaders(
1326     HttpRequestHeaders* headers) const {
1327   if (!transaction_)
1328     return false;
1329
1330   return transaction_->GetFullRequestHeaders(headers);
1331 }
1332
1333 int64 URLRequestHttpJob::GetTotalReceivedBytes() const {
1334   if (!transaction_)
1335     return 0;
1336
1337   return transaction_->GetTotalReceivedBytes();
1338 }
1339
1340 void URLRequestHttpJob::DoneReading() {
1341   if (transaction_) {
1342     transaction_->DoneReading();
1343   }
1344   DoneWithRequest(FINISHED);
1345 }
1346
1347 void URLRequestHttpJob::DoneReadingRedirectResponse() {
1348   if (transaction_) {
1349     if (transaction_->GetResponseInfo()->headers->IsRedirect(NULL)) {
1350       // If the original headers indicate a redirect, go ahead and cache the
1351       // response, even if the |override_response_headers_| are a redirect to
1352       // another location.
1353       transaction_->DoneReading();
1354     } else {
1355       // Otherwise, |override_response_headers_| must be non-NULL and contain
1356       // bogus headers indicating a redirect.
1357       DCHECK(override_response_headers_.get());
1358       DCHECK(override_response_headers_->IsRedirect(NULL));
1359       transaction_->StopCaching();
1360     }
1361   }
1362   DoneWithRequest(FINISHED);
1363 }
1364
1365 HostPortPair URLRequestHttpJob::GetSocketAddress() const {
1366   return response_info_ ? response_info_->socket_address : HostPortPair();
1367 }
1368
1369 void URLRequestHttpJob::RecordTimer() {
1370   if (request_creation_time_.is_null()) {
1371     NOTREACHED()
1372         << "The same transaction shouldn't start twice without new timing.";
1373     return;
1374   }
1375
1376   base::TimeDelta to_start = base::Time::Now() - request_creation_time_;
1377   request_creation_time_ = base::Time();
1378
1379   UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpTimeToFirstByte", to_start);
1380 }
1381
1382 void URLRequestHttpJob::ResetTimer() {
1383   if (!request_creation_time_.is_null()) {
1384     NOTREACHED()
1385         << "The timer was reset before it was recorded.";
1386     return;
1387   }
1388   request_creation_time_ = base::Time::Now();
1389 }
1390
1391 void URLRequestHttpJob::UpdatePacketReadTimes() {
1392   if (!packet_timing_enabled_)
1393     return;
1394
1395   if (filter_input_byte_count() <= bytes_observed_in_packets_) {
1396     DCHECK_EQ(filter_input_byte_count(), bytes_observed_in_packets_);
1397     return;  // No new bytes have arrived.
1398   }
1399
1400   base::Time now(base::Time::Now());
1401   if (!bytes_observed_in_packets_)
1402     request_time_snapshot_ = now;
1403   final_packet_time_ = now;
1404
1405   bytes_observed_in_packets_ = filter_input_byte_count();
1406 }
1407
1408 void URLRequestHttpJob::RecordPacketStats(
1409     FilterContext::StatisticSelector statistic) const {
1410   if (!packet_timing_enabled_ || (final_packet_time_ == base::Time()))
1411     return;
1412
1413   base::TimeDelta duration = final_packet_time_ - request_time_snapshot_;
1414   switch (statistic) {
1415     case FilterContext::SDCH_DECODE: {
1416       UMA_HISTOGRAM_CUSTOM_COUNTS("Sdch3.Network_Decode_Bytes_Processed_b",
1417           static_cast<int>(bytes_observed_in_packets_), 500, 100000, 100);
1418       return;
1419     }
1420     case FilterContext::SDCH_PASSTHROUGH: {
1421       // Despite advertising a dictionary, we handled non-sdch compressed
1422       // content.
1423       return;
1424     }
1425
1426     case FilterContext::SDCH_EXPERIMENT_DECODE: {
1427       UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment3_Decode",
1428                                   duration,
1429                                   base::TimeDelta::FromMilliseconds(20),
1430                                   base::TimeDelta::FromMinutes(10), 100);
1431       return;
1432     }
1433     case FilterContext::SDCH_EXPERIMENT_HOLDBACK: {
1434       UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment3_Holdback",
1435                                   duration,
1436                                   base::TimeDelta::FromMilliseconds(20),
1437                                   base::TimeDelta::FromMinutes(10), 100);
1438       return;
1439     }
1440     default:
1441       NOTREACHED();
1442       return;
1443   }
1444 }
1445
1446 void URLRequestHttpJob::RecordPerfHistograms(CompletionCause reason) {
1447   if (start_time_.is_null())
1448     return;
1449
1450   base::TimeDelta total_time = base::TimeTicks::Now() - start_time_;
1451   UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTime", total_time);
1452
1453   if (reason == FINISHED) {
1454     UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeSuccess", total_time);
1455   } else {
1456     UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCancel", total_time);
1457   }
1458
1459   if (response_info_) {
1460     if (response_info_->was_cached) {
1461       UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCached", total_time);
1462     } else  {
1463       UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeNotCached", total_time);
1464     }
1465   }
1466
1467   if (request_info_.load_flags & LOAD_PREFETCH && !request_->was_cached())
1468     UMA_HISTOGRAM_COUNTS("Net.Prefetch.PrefilterBytesReadFromNetwork",
1469                          prefilter_bytes_read());
1470
1471   start_time_ = base::TimeTicks();
1472 }
1473
1474 void URLRequestHttpJob::DoneWithRequest(CompletionCause reason) {
1475   if (done_)
1476     return;
1477   done_ = true;
1478   RecordPerfHistograms(reason);
1479   if (reason == FINISHED) {
1480     request_->set_received_response_content_length(prefilter_bytes_read());
1481   }
1482 }
1483
1484 HttpResponseHeaders* URLRequestHttpJob::GetResponseHeaders() const {
1485   DCHECK(transaction_.get());
1486   DCHECK(transaction_->GetResponseInfo());
1487   return override_response_headers_.get() ?
1488              override_response_headers_.get() :
1489              transaction_->GetResponseInfo()->headers.get();
1490 }
1491
1492 void URLRequestHttpJob::NotifyURLRequestDestroyed() {
1493   awaiting_callback_ = false;
1494 }
1495
1496 }  // namespace net