net/url_request/url_request_http_job.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_request_http_job.h"
   6
   7 #include "base/base_switches.h"
   8 #include "base/bind.h"
   9 #include "base/bind_helpers.h"
  10 #include "base/command_line.h"
  11 #include "base/compiler_specific.h"
  12 #include "base/file_version_info.h"
  13 #include "base/message_loop/message_loop.h"
  14 #include "base/metrics/field_trial.h"
  15 #include "base/metrics/histogram.h"
  16 #include "base/profiler/scoped_tracker.h"
  17 #include "base/rand_util.h"
  18 #include "base/strings/string_util.h"
  19 #include "base/time/time.h"
  20 #include "net/base/host_port_pair.h"
  21 #include "net/base/load_flags.h"
  22 #include "net/base/mime_util.h"
  23 #include "net/base/net_errors.h"
  24 #include "net/base/net_util.h"
  25 #include "net/base/network_delegate.h"
  26 #include "net/base/sdch_manager.h"
  27 #include "net/base/sdch_net_log_params.h"
  28 #include "net/cert/cert_status_flags.h"
  29 #include "net/cookies/cookie_store.h"
  30 #include "net/http/http_content_disposition.h"
  31 #include "net/http/http_network_session.h"
  32 #include "net/http/http_request_headers.h"
  33 #include "net/http/http_response_headers.h"
  34 #include "net/http/http_response_info.h"
  35 #include "net/http/http_status_code.h"
  36 #include "net/http/http_transaction.h"
  37 #include "net/http/http_transaction_factory.h"
  38 #include "net/http/http_util.h"
  39 #include "net/proxy/proxy_info.h"
  40 #include "net/ssl/ssl_cert_request_info.h"
  41 #include "net/ssl/ssl_config_service.h"
  42 #include "net/url_request/fraudulent_certificate_reporter.h"
  43 #include "net/url_request/http_user_agent_settings.h"
  44 #include "net/url_request/url_request.h"
  45 #include "net/url_request/url_request_context.h"
  46 #include "net/url_request/url_request_error_job.h"
  47 #include "net/url_request/url_request_job_factory.h"
  48 #include "net/url_request/url_request_redirect_job.h"
  49 #include "net/url_request/url_request_throttler_header_adapter.h"
  50 #include "net/url_request/url_request_throttler_manager.h"
  51 #include "net/websockets/websocket_handshake_stream_base.h"
  52
  53 static const char kAvailDictionaryHeader[] = "Avail-Dictionary";
  54
  55 namespace net {
  56
  57 class URLRequestHttpJob::HttpFilterContext : public FilterContext {
  58  public:
  59   explicit HttpFilterContext(URLRequestHttpJob* job);
  60   ~HttpFilterContext() override;
  61
  62   // FilterContext implementation.
  63   bool GetMimeType(std::string* mime_type) const override;
  64   bool GetURL(GURL* gurl) const override;
  65   bool GetContentDisposition(std::string* disposition) const override;
  66   base::Time GetRequestTime() const override;
  67   bool IsCachedContent() const override;
  68   bool IsDownload() const override;
  69   SdchManager::DictionarySet* SdchDictionariesAdvertised() const override;
  70   int64 GetByteReadCount() const override;
  71   int GetResponseCode() const override;
  72   const URLRequestContext* GetURLRequestContext() const override;
  73   void RecordPacketStats(StatisticSelector statistic) const override;
  74   const BoundNetLog& GetNetLog() const override;
  75
  76  private:
  77   URLRequestHttpJob* job_;
  78
  79   // URLRequestHttpJob may be detached from URLRequest, but we still need to
  80   // return something.
  81   BoundNetLog dummy_log_;
  82
  83   DISALLOW_COPY_AND_ASSIGN(HttpFilterContext);
  84 };
  85
  86 URLRequestHttpJob::HttpFilterContext::HttpFilterContext(URLRequestHttpJob* job)
  87     : job_(job) {
  88   DCHECK(job_);
  89 }
  90
  91 URLRequestHttpJob::HttpFilterContext::~HttpFilterContext() {
  92 }
  93
  94 bool URLRequestHttpJob::HttpFilterContext::GetMimeType(
  95     std::string* mime_type) const {
  96   return job_->GetMimeType(mime_type);
  97 }
  98
  99 bool URLRequestHttpJob::HttpFilterContext::GetURL(GURL* gurl) const {
 100   if (!job_->request())
 101     return false;
 102   *gurl = job_->request()->url();
 103   return true;
 104 }
 105
 106 bool URLRequestHttpJob::HttpFilterContext::GetContentDisposition(
 107     std::string* disposition) const {
 108   HttpResponseHeaders* headers = job_->GetResponseHeaders();
 109   void *iter = NULL;
 110   return headers->EnumerateHeader(&iter, "Content-Disposition", disposition);
 111 }
 112
 113 base::Time URLRequestHttpJob::HttpFilterContext::GetRequestTime() const {
 114   return job_->request() ? job_->request()->request_time() : base::Time();
 115 }
 116
 117 bool URLRequestHttpJob::HttpFilterContext::IsCachedContent() const {
 118   return job_->is_cached_content_;
 119 }
 120
 121 bool URLRequestHttpJob::HttpFilterContext::IsDownload() const {
 122   return (job_->request_info_.load_flags & LOAD_IS_DOWNLOAD) != 0;
 123 }
 124
 125 SdchManager::DictionarySet*
 126 URLRequestHttpJob::HttpFilterContext::SdchDictionariesAdvertised() const {
 127   return job_->dictionaries_advertised_.get();
 128 }
 129
 130 int64 URLRequestHttpJob::HttpFilterContext::GetByteReadCount() const {
 131   return job_->filter_input_byte_count();
 132 }
 133
 134 int URLRequestHttpJob::HttpFilterContext::GetResponseCode() const {
 135   return job_->GetResponseCode();
 136 }
 137
 138 const URLRequestContext*
 139 URLRequestHttpJob::HttpFilterContext::GetURLRequestContext() const {
 140   return job_->request() ? job_->request()->context() : NULL;
 141 }
 142
 143 void URLRequestHttpJob::HttpFilterContext::RecordPacketStats(
 144     StatisticSelector statistic) const {
 145   job_->RecordPacketStats(statistic);
 146 }
 147
 148 const BoundNetLog& URLRequestHttpJob::HttpFilterContext::GetNetLog() const {
 149   return job_->request() ? job_->request()->net_log() : dummy_log_;
 150 }
 151
 152 // TODO(darin): make sure the port blocking code is not lost
 153 // static
 154 URLRequestJob* URLRequestHttpJob::Factory(URLRequest* request,
 155                                           NetworkDelegate* network_delegate,
 156                                           const std::string& scheme) {
 157   DCHECK(scheme == "http" || scheme == "https" || scheme == "ws" ||
 158          scheme == "wss");
 159
 160   if (!request->context()->http_transaction_factory()) {
 161     NOTREACHED() << "requires a valid context";
 162     return new URLRequestErrorJob(
 163         request, network_delegate, ERR_INVALID_ARGUMENT);
 164   }
 165
 166   GURL redirect_url;
 167   if (request->GetHSTSRedirect(&redirect_url)) {
 168     return new URLRequestRedirectJob(
 169         request, network_delegate, redirect_url,
 170         // Use status code 307 to preserve the method, so POST requests work.
 171         URLRequestRedirectJob::REDIRECT_307_TEMPORARY_REDIRECT, "HSTS");
 172   }
 173   return new URLRequestHttpJob(request,
 174                                network_delegate,
 175                                request->context()->http_user_agent_settings());
 176 }
 177
 178 URLRequestHttpJob::URLRequestHttpJob(
 179     URLRequest* request,
 180     NetworkDelegate* network_delegate,
 181     const HttpUserAgentSettings* http_user_agent_settings)
 182     : URLRequestJob(request, network_delegate),
 183       priority_(DEFAULT_PRIORITY),
 184       response_info_(NULL),
 185       response_cookies_save_index_(0),
 186       proxy_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
 187       server_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
 188       start_callback_(base::Bind(&URLRequestHttpJob::OnStartCompleted,
 189                                  base::Unretained(this))),
 190       notify_before_headers_sent_callback_(
 191           base::Bind(&URLRequestHttpJob::NotifyBeforeSendHeadersCallback,
 192                      base::Unretained(this))),
 193       read_in_progress_(false),
 194       throttling_entry_(NULL),
 195       sdch_test_activated_(false),
 196       sdch_test_control_(false),
 197       is_cached_content_(false),
 198       request_creation_time_(),
 199       packet_timing_enabled_(false),
 200       done_(false),
 201       bytes_observed_in_packets_(0),
 202       request_time_snapshot_(),
 203       final_packet_time_(),
 204       filter_context_(new HttpFilterContext(this)),
 205       on_headers_received_callback_(
 206           base::Bind(&URLRequestHttpJob::OnHeadersReceivedCallback,
 207                      base::Unretained(this))),
 208       awaiting_callback_(false),
 209       http_user_agent_settings_(http_user_agent_settings),
 210       weak_factory_(this) {
 211   URLRequestThrottlerManager* manager = request->context()->throttler_manager();
 212   if (manager)
 213     throttling_entry_ = manager->RegisterRequestUrl(request->url());
 214
 215   ResetTimer();
 216 }
 217
 218 URLRequestHttpJob::~URLRequestHttpJob() {
 219   CHECK(!awaiting_callback_);
 220
 221   DCHECK(!sdch_test_control_ || !sdch_test_activated_);
 222   if (!is_cached_content_) {
 223     if (sdch_test_control_)
 224       RecordPacketStats(FilterContext::SDCH_EXPERIMENT_HOLDBACK);
 225     if (sdch_test_activated_)
 226       RecordPacketStats(FilterContext::SDCH_EXPERIMENT_DECODE);
 227   }
 228   // Make sure SDCH filters are told to emit histogram data while
 229   // filter_context_ is still alive.
 230   DestroyFilters();
 231
 232   DoneWithRequest(ABORTED);
 233 }
 234
 235 void URLRequestHttpJob::SetPriority(RequestPriority priority) {
 236   priority_ = priority;
 237   if (transaction_)
 238     transaction_->SetPriority(priority_);
 239 }
 240
 241 void URLRequestHttpJob::Start() {
 242   DCHECK(!transaction_.get());
 243
 244   // URLRequest::SetReferrer ensures that we do not send username and password
 245   // fields in the referrer.
 246   GURL referrer(request_->referrer());
 247
 248   request_info_.url = request_->url();
 249   request_info_.method = request_->method();
 250   request_info_.load_flags = request_->load_flags();
 251   // Enable privacy mode if cookie settings or flags tell us not send or
 252   // save cookies.
 253   bool enable_privacy_mode =
 254       (request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES) ||
 255       (request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) ||
 256       CanEnablePrivacyMode();
 257   // Privacy mode could still be disabled in OnCookiesLoaded if we are going
 258   // to send previously saved cookies.
 259   request_info_.privacy_mode = enable_privacy_mode ?
 260       PRIVACY_MODE_ENABLED : PRIVACY_MODE_DISABLED;
 261
 262   // Strip Referer from request_info_.extra_headers to prevent, e.g., plugins
 263   // from overriding headers that are controlled using other means. Otherwise a
 264   // plugin could set a referrer although sending the referrer is inhibited.
 265   request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kReferer);
 266
 267   // Our consumer should have made sure that this is a safe referrer. See for
 268   // instance WebCore::FrameLoader::HideReferrer.
 269   if (referrer.is_valid()) {
 270     request_info_.extra_headers.SetHeader(HttpRequestHeaders::kReferer,
 271                                           referrer.spec());
 272   }
 273
 274   request_info_.extra_headers.SetHeaderIfMissing(
 275       HttpRequestHeaders::kUserAgent,
 276       http_user_agent_settings_ ?
 277           http_user_agent_settings_->GetUserAgent() : std::string());
 278
 279   AddExtraHeaders();
 280   AddCookieHeaderAndStart();
 281 }
 282
 283 void URLRequestHttpJob::Kill() {
 284   if (!transaction_.get())
 285     return;
 286
 287   weak_factory_.InvalidateWeakPtrs();
 288   DestroyTransaction();
 289   URLRequestJob::Kill();
 290 }
 291
 292 void URLRequestHttpJob::NotifyBeforeSendProxyHeadersCallback(
 293     const ProxyInfo& proxy_info,
 294     HttpRequestHeaders* request_headers) {
 295   DCHECK(request_headers);
 296   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 297   if (network_delegate()) {
 298     network_delegate()->NotifyBeforeSendProxyHeaders(
 299         request_,
 300         proxy_info,
 301         request_headers);
 302   }
 303 }
 304
 305 void URLRequestHttpJob::NotifyHeadersComplete() {
 306   DCHECK(!response_info_);
 307
 308   response_info_ = transaction_->GetResponseInfo();
 309
 310   // Save boolean, as we'll need this info at destruction time, and filters may
 311   // also need this info.
 312   is_cached_content_ = response_info_->was_cached;
 313
 314   if (!is_cached_content_ && throttling_entry_.get()) {
 315     URLRequestThrottlerHeaderAdapter response_adapter(GetResponseHeaders());
 316     throttling_entry_->UpdateWithResponse(request_info_.url.host(),
 317                                           &response_adapter);
 318   }
 319
 320   // The ordering of these calls is not important.
 321   ProcessStrictTransportSecurityHeader();
 322   ProcessPublicKeyPinsHeader();
 323
 324   // Handle the server notification of a new SDCH dictionary.
 325   SdchManager* sdch_manager(request()->context()->sdch_manager());
 326   if (sdch_manager) {
 327     SdchProblemCode rv = sdch_manager->IsInSupportedDomain(request()->url());
 328     if (rv != SDCH_OK) {
 329       // If SDCH is just disabled, it is not a real error.
 330       if (rv != SDCH_DISABLED && rv != SDCH_SECURE_SCHEME_NOT_SUPPORTED) {
 331         SdchManager::SdchErrorRecovery(rv);
 332         request()->net_log().AddEvent(
 333             NetLog::TYPE_SDCH_DECODING_ERROR,
 334             base::Bind(&NetLogSdchResourceProblemCallback, rv));
 335       }
 336     } else {
 337       const std::string name = "Get-Dictionary";
 338       std::string url_text;
 339       void* iter = NULL;
 340       // TODO(jar): We need to not fetch dictionaries the first time they are
 341       // seen, but rather wait until we can justify their usefulness.
 342       // For now, we will only fetch the first dictionary, which will at least
 343       // require multiple suggestions before we get additional ones for this
 344       // site. Eventually we should wait until a dictionary is requested
 345       // several times
 346       // before we even download it (so that we don't waste memory or
 347       // bandwidth).
 348       if (GetResponseHeaders()->EnumerateHeader(&iter, name, &url_text)) {
 349         // Resolve suggested URL relative to request url.
 350         GURL sdch_dictionary_url = request_->url().Resolve(url_text);
 351         if (sdch_dictionary_url.is_valid()) {
 352           rv = sdch_manager->OnGetDictionary(request_->url(),
 353                                              sdch_dictionary_url);
 354           if (rv != SDCH_OK) {
 355             SdchManager::SdchErrorRecovery(rv);
 356             request_->net_log().AddEvent(
 357                 NetLog::TYPE_SDCH_DICTIONARY_ERROR,
 358                 base::Bind(&NetLogSdchDictionaryFetchProblemCallback, rv,
 359                            sdch_dictionary_url, false));
 360           }
 361         }
 362       }
 363     }
 364   }
 365
 366   // Handle the server signalling no SDCH encoding.
 367   if (dictionaries_advertised_) {
 368     // We are wary of proxies that discard or damage SDCH encoding. If a server
 369     // explicitly states that this is not SDCH content, then we can correct our
 370     // assumption that this is an SDCH response, and avoid the need to recover
 371     // as though the content is corrupted (when we discover it is not SDCH
 372     // encoded).
 373     std::string sdch_response_status;
 374     void* iter = NULL;
 375     while (GetResponseHeaders()->EnumerateHeader(&iter, "X-Sdch-Encode",
 376                                                  &sdch_response_status)) {
 377       if (sdch_response_status == "0") {
 378         dictionaries_advertised_.reset();
 379         break;
 380       }
 381     }
 382   }
 383
 384   // The HTTP transaction may be restarted several times for the purposes
 385   // of sending authorization information. Each time it restarts, we get
 386   // notified of the headers completion so that we can update the cookie store.
 387   if (transaction_->IsReadyToRestartForAuth()) {
 388     DCHECK(!response_info_->auth_challenge.get());
 389     // TODO(battre): This breaks the webrequest API for
 390     // URLRequestTestHTTP.BasicAuthWithCookies
 391     // where OnBeforeSendHeaders -> OnSendHeaders -> OnBeforeSendHeaders
 392     // occurs.
 393     RestartTransactionWithAuth(AuthCredentials());
 394     return;
 395   }
 396
 397   URLRequestJob::NotifyHeadersComplete();
 398 }
 399
 400 void URLRequestHttpJob::NotifyDone(const URLRequestStatus& status) {
 401   DoneWithRequest(FINISHED);
 402   URLRequestJob::NotifyDone(status);
 403 }
 404
 405 void URLRequestHttpJob::DestroyTransaction() {
 406   DCHECK(transaction_.get());
 407
 408   DoneWithRequest(ABORTED);
 409   transaction_.reset();
 410   response_info_ = NULL;
 411   receive_headers_end_ = base::TimeTicks();
 412 }
 413
 414 void URLRequestHttpJob::StartTransaction() {
 415   if (network_delegate()) {
 416     OnCallToDelegate();
 417     int rv = network_delegate()->NotifyBeforeSendHeaders(
 418         request_, notify_before_headers_sent_callback_,
 419         &request_info_.extra_headers);
 420     // If an extension blocks the request, we rely on the callback to
 421     // MaybeStartTransactionInternal().
 422     if (rv == ERR_IO_PENDING)
 423       return;
 424     MaybeStartTransactionInternal(rv);
 425     return;
 426   }
 427   StartTransactionInternal();
 428 }
 429
 430 void URLRequestHttpJob::NotifyBeforeSendHeadersCallback(int result) {
 431   // Check that there are no callbacks to already canceled requests.
 432   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 433
 434   MaybeStartTransactionInternal(result);
 435 }
 436
 437 void URLRequestHttpJob::MaybeStartTransactionInternal(int result) {
 438   OnCallToDelegateComplete();
 439   if (result == OK) {
 440     StartTransactionInternal();
 441   } else {
 442     std::string source("delegate");
 443     request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 444                                  NetLog::StringCallback("source", &source));
 445     NotifyCanceled();
 446     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 447   }
 448 }
 449
 450 void URLRequestHttpJob::StartTransactionInternal() {
 451   // NOTE: This method assumes that request_info_ is already setup properly.
 452
 453   // If we already have a transaction, then we should restart the transaction
 454   // with auth provided by auth_credentials_.
 455
 456   int rv;
 457
 458   if (network_delegate()) {
 459     network_delegate()->NotifySendHeaders(
 460         request_, request_info_.extra_headers);
 461   }
 462
 463   if (transaction_.get()) {
 464     rv = transaction_->RestartWithAuth(auth_credentials_, start_callback_);
 465     auth_credentials_ = AuthCredentials();
 466   } else {
 467     DCHECK(request_->context()->http_transaction_factory());
 468
 469     rv = request_->context()->http_transaction_factory()->CreateTransaction(
 470         priority_, &transaction_);
 471
 472     if (rv == OK && request_info_.url.SchemeIsWSOrWSS()) {
 473       base::SupportsUserData::Data* data = request_->GetUserData(
 474           WebSocketHandshakeStreamBase::CreateHelper::DataKey());
 475       if (data) {
 476         transaction_->SetWebSocketHandshakeStreamCreateHelper(
 477             static_cast<WebSocketHandshakeStreamBase::CreateHelper*>(data));
 478       } else {
 479         rv = ERR_DISALLOWED_URL_SCHEME;
 480       }
 481     }
 482
 483     if (rv == OK) {
 484       transaction_->SetBeforeNetworkStartCallback(
 485           base::Bind(&URLRequestHttpJob::NotifyBeforeNetworkStart,
 486                      base::Unretained(this)));
 487       transaction_->SetBeforeProxyHeadersSentCallback(
 488           base::Bind(&URLRequestHttpJob::NotifyBeforeSendProxyHeadersCallback,
 489                      base::Unretained(this)));
 490
 491       if (!throttling_entry_.get() ||
 492           !throttling_entry_->ShouldRejectRequest(*request_,
 493                                                   network_delegate())) {
 494         rv = transaction_->Start(
 495             &request_info_, start_callback_, request_->net_log());
 496         start_time_ = base::TimeTicks::Now();
 497       } else {
 498         // Special error code for the exponential back-off module.
 499         rv = ERR_TEMPORARILY_THROTTLED;
 500       }
 501     }
 502   }
 503
 504   if (rv == ERR_IO_PENDING)
 505     return;
 506
 507   // The transaction started synchronously, but we need to notify the
 508   // URLRequest delegate via the message loop.
 509   base::MessageLoop::current()->PostTask(
 510       FROM_HERE,
 511       base::Bind(&URLRequestHttpJob::OnStartCompleted,
 512                  weak_factory_.GetWeakPtr(), rv));
 513 }
 514
 515 void URLRequestHttpJob::AddExtraHeaders() {
 516   SdchManager* sdch_manager = request()->context()->sdch_manager();
 517
 518   // Supply Accept-Encoding field only if it is not already provided.
 519   // It should be provided IF the content is known to have restrictions on
 520   // potential encoding, such as streaming multi-media.
 521   // For details see bug 47381.
 522   // TODO(jar, enal): jpeg files etc. should set up a request header if
 523   // possible. Right now it is done only by buffered_resource_loader and
 524   // simple_data_source.
 525   if (!request_info_.extra_headers.HasHeader(
 526       HttpRequestHeaders::kAcceptEncoding)) {
 527     // We don't support SDCH responses to POST as there is a possibility
 528     // of having SDCH encoded responses returned (e.g. by the cache)
 529     // which we cannot decode, and in those situations, we will need
 530     // to retransmit the request without SDCH, which is illegal for a POST.
 531     bool advertise_sdch = sdch_manager != NULL && request()->method() != "POST";
 532     if (advertise_sdch) {
 533       SdchProblemCode rv = sdch_manager->IsInSupportedDomain(request()->url());
 534       if (rv != SDCH_OK) {
 535         advertise_sdch = false;
 536         // If SDCH is just disabled, it is not a real error.
 537         if (rv != SDCH_DISABLED && rv != SDCH_SECURE_SCHEME_NOT_SUPPORTED) {
 538           SdchManager::SdchErrorRecovery(rv);
 539           request()->net_log().AddEvent(
 540               NetLog::TYPE_SDCH_DECODING_ERROR,
 541               base::Bind(&NetLogSdchResourceProblemCallback, rv));
 542         }
 543       }
 544     }
 545     if (advertise_sdch) {
 546       dictionaries_advertised_ =
 547           sdch_manager->GetDictionarySet(request_->url());
 548     }
 549
 550     // The AllowLatencyExperiment() is only true if we've successfully done a
 551     // full SDCH compression recently in this browser session for this host.
 552     // Note that for this path, there might be no applicable dictionaries,
 553     // and hence we can't participate in the experiment.
 554     if (dictionaries_advertised_ &&
 555         sdch_manager->AllowLatencyExperiment(request_->url())) {
 556       // We are participating in the test (or control), and hence we'll
 557       // eventually record statistics via either SDCH_EXPERIMENT_DECODE or
 558       // SDCH_EXPERIMENT_HOLDBACK, and we'll need some packet timing data.
 559       packet_timing_enabled_ = true;
 560       if (base::RandDouble() < .01) {
 561         sdch_test_control_ = true;  // 1% probability.
 562         dictionaries_advertised_.reset();
 563         advertise_sdch = false;
 564       } else {
 565         sdch_test_activated_ = true;
 566       }
 567     }
 568
 569     // Supply Accept-Encoding headers first so that it is more likely that they
 570     // will be in the first transmitted packet. This can sometimes make it
 571     // easier to filter and analyze the streams to assure that a proxy has not
 572     // damaged these headers. Some proxies deliberately corrupt Accept-Encoding
 573     // headers.
 574     if (!advertise_sdch) {
 575       // Tell the server what compression formats we support (other than SDCH).
 576       request_info_.extra_headers.SetHeader(
 577           HttpRequestHeaders::kAcceptEncoding, "gzip, deflate");
 578     } else {
 579       // Include SDCH in acceptable list.
 580       request_info_.extra_headers.SetHeader(
 581           HttpRequestHeaders::kAcceptEncoding, "gzip, deflate, sdch");
 582       if (dictionaries_advertised_) {
 583         request_info_.extra_headers.SetHeader(
 584             kAvailDictionaryHeader,
 585             dictionaries_advertised_->GetDictionaryClientHashList());
 586         // Since we're tagging this transaction as advertising a dictionary,
 587         // we'll definitely employ an SDCH filter (or tentative sdch filter)
 588         // when we get a response. When done, we'll record histograms via
 589         // SDCH_DECODE or SDCH_PASSTHROUGH. Hence we need to record packet
 590         // arrival times.
 591         packet_timing_enabled_ = true;
 592       }
 593     }
 594   }
 595
 596   if (http_user_agent_settings_) {
 597     // Only add default Accept-Language if the request didn't have it
 598     // specified.
 599     std::string accept_language =
 600         http_user_agent_settings_->GetAcceptLanguage();
 601     if (!accept_language.empty()) {
 602       request_info_.extra_headers.SetHeaderIfMissing(
 603           HttpRequestHeaders::kAcceptLanguage,
 604           accept_language);
 605     }
 606   }
 607 }
 608
 609 void URLRequestHttpJob::AddCookieHeaderAndStart() {
 610   // No matter what, we want to report our status as IO pending since we will
 611   // be notifying our consumer asynchronously via OnStartCompleted.
 612   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
 613
 614   // If the request was destroyed, then there is no more work to do.
 615   if (!request_)
 616     return;
 617
 618   CookieStore* cookie_store = GetCookieStore();
 619   if (cookie_store && !(request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES)) {
 620     cookie_store->GetAllCookiesForURLAsync(
 621         request_->url(),
 622         base::Bind(&URLRequestHttpJob::CheckCookiePolicyAndLoad,
 623                    weak_factory_.GetWeakPtr()));
 624   } else {
 625     DoStartTransaction();
 626   }
 627 }
 628
 629 void URLRequestHttpJob::DoLoadCookies() {
 630   CookieOptions options;
 631   options.set_include_httponly();
 632
 633   // TODO(mkwst): Drop this `if` once we decide whether or not to ship
 634   // first-party cookies: https://crbug.com/459154
 635   if (network_delegate() &&
 636       network_delegate()->FirstPartyOnlyCookieExperimentEnabled())
 637     options.set_first_party_url(request_->first_party_for_cookies());
 638   else
 639     options.set_include_first_party_only();
 640
 641   GetCookieStore()->GetCookiesWithOptionsAsync(
 642       request_->url(), options,
 643       base::Bind(&URLRequestHttpJob::OnCookiesLoaded,
 644                  weak_factory_.GetWeakPtr()));
 645 }
 646
 647 void URLRequestHttpJob::CheckCookiePolicyAndLoad(
 648     const CookieList& cookie_list) {
 649   if (CanGetCookies(cookie_list))
 650     DoLoadCookies();
 651   else
 652     DoStartTransaction();
 653 }
 654
 655 void URLRequestHttpJob::OnCookiesLoaded(const std::string& cookie_line) {
 656   if (!cookie_line.empty()) {
 657     request_info_.extra_headers.SetHeader(
 658         HttpRequestHeaders::kCookie, cookie_line);
 659     // Disable privacy mode as we are sending cookies anyway.
 660     request_info_.privacy_mode = PRIVACY_MODE_DISABLED;
 661   }
 662   DoStartTransaction();
 663 }
 664
 665 void URLRequestHttpJob::DoStartTransaction() {
 666   // We may have been canceled while retrieving cookies.
 667   if (GetStatus().is_success()) {
 668     StartTransaction();
 669   } else {
 670     NotifyCanceled();
 671   }
 672 }
 673
 674 void URLRequestHttpJob::SaveCookiesAndNotifyHeadersComplete(int result) {
 675   // End of the call started in OnStartCompleted.
 676   OnCallToDelegateComplete();
 677
 678   if (result != net::OK) {
 679     std::string source("delegate");
 680     request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 681                                  NetLog::StringCallback("source", &source));
 682     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 683     return;
 684   }
 685
 686   DCHECK(transaction_.get());
 687
 688   const HttpResponseInfo* response_info = transaction_->GetResponseInfo();
 689   DCHECK(response_info);
 690
 691   response_cookies_.clear();
 692   response_cookies_save_index_ = 0;
 693
 694   FetchResponseCookies(&response_cookies_);
 695
 696   if (!GetResponseHeaders()->GetDateValue(&response_date_))
 697     response_date_ = base::Time();
 698
 699   // Now, loop over the response cookies, and attempt to persist each.
 700   SaveNextCookie();
 701 }
 702
 703 // If the save occurs synchronously, SaveNextCookie will loop and save the next
 704 // cookie. If the save is deferred, the callback is responsible for continuing
 705 // to iterate through the cookies.
 706 // TODO(erikwright): Modify the CookieStore API to indicate via return value
 707 // whether it completed synchronously or asynchronously.
 708 // See http://crbug.com/131066.
 709 void URLRequestHttpJob::SaveNextCookie() {
 710   // No matter what, we want to report our status as IO pending since we will
 711   // be notifying our consumer asynchronously via OnStartCompleted.
 712   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
 713
 714   // Used to communicate with the callback. See the implementation of
 715   // OnCookieSaved.
 716   scoped_refptr<SharedBoolean> callback_pending = new SharedBoolean(false);
 717   scoped_refptr<SharedBoolean> save_next_cookie_running =
 718       new SharedBoolean(true);
 719
 720   if (!(request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) &&
 721       GetCookieStore() && response_cookies_.size() > 0) {
 722     CookieOptions options;
 723     options.set_include_httponly();
 724     options.set_server_time(response_date_);
 725
 726     net::CookieStore::SetCookiesCallback callback(
 727         base::Bind(&URLRequestHttpJob::OnCookieSaved,
 728                    weak_factory_.GetWeakPtr(),
 729                    save_next_cookie_running,
 730                    callback_pending));
 731
 732     // Loop through the cookies as long as SetCookieWithOptionsAsync completes
 733     // synchronously.
 734     while (!callback_pending->data &&
 735            response_cookies_save_index_ < response_cookies_.size()) {
 736       if (CanSetCookie(
 737           response_cookies_[response_cookies_save_index_], &options)) {
 738         callback_pending->data = true;
 739         GetCookieStore()->SetCookieWithOptionsAsync(
 740             request_->url(), response_cookies_[response_cookies_save_index_],
 741             options, callback);
 742       }
 743       ++response_cookies_save_index_;
 744     }
 745   }
 746
 747   save_next_cookie_running->data = false;
 748
 749   if (!callback_pending->data) {
 750     response_cookies_.clear();
 751     response_cookies_save_index_ = 0;
 752     SetStatus(URLRequestStatus());  // Clear the IO_PENDING status
 753     NotifyHeadersComplete();
 754     return;
 755   }
 756 }
 757
 758 // |save_next_cookie_running| is true when the callback is bound and set to
 759 // false when SaveNextCookie exits, allowing the callback to determine if the
 760 // save occurred synchronously or asynchronously.
 761 // |callback_pending| is false when the callback is invoked and will be set to
 762 // true by the callback, allowing SaveNextCookie to detect whether the save
 763 // occurred synchronously.
 764 // See SaveNextCookie() for more information.
 765 void URLRequestHttpJob::OnCookieSaved(
 766     scoped_refptr<SharedBoolean> save_next_cookie_running,
 767     scoped_refptr<SharedBoolean> callback_pending,
 768     bool cookie_status) {
 769   callback_pending->data = false;
 770
 771   // If we were called synchronously, return.
 772   if (save_next_cookie_running->data) {
 773     return;
 774   }
 775
 776   // We were called asynchronously, so trigger the next save.
 777   // We may have been canceled within OnSetCookie.
 778   if (GetStatus().is_success()) {
 779     SaveNextCookie();
 780   } else {
 781     NotifyCanceled();
 782   }
 783 }
 784
 785 void URLRequestHttpJob::FetchResponseCookies(
 786     std::vector<std::string>* cookies) {
 787   const std::string name = "Set-Cookie";
 788   std::string value;
 789
 790   void* iter = NULL;
 791   HttpResponseHeaders* headers = GetResponseHeaders();
 792   while (headers->EnumerateHeader(&iter, name, &value)) {
 793     if (!value.empty())
 794       cookies->push_back(value);
 795   }
 796 }
 797
 798 // NOTE: |ProcessStrictTransportSecurityHeader| and
 799 // |ProcessPublicKeyPinsHeader| have very similar structures, by design.
 800 void URLRequestHttpJob::ProcessStrictTransportSecurityHeader() {
 801   DCHECK(response_info_);
 802   TransportSecurityState* security_state =
 803       request_->context()->transport_security_state();
 804   const SSLInfo& ssl_info = response_info_->ssl_info;
 805
 806   // Only accept HSTS headers on HTTPS connections that have no
 807   // certificate errors.
 808   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 809       !security_state)
 810     return;
 811
 812   // http://tools.ietf.org/html/draft-ietf-websec-strict-transport-sec:
 813   //
 814   //   If a UA receives more than one STS header field in a HTTP response
 815   //   message over secure transport, then the UA MUST process only the
 816   //   first such header field.
 817   HttpResponseHeaders* headers = GetResponseHeaders();
 818   std::string value;
 819   if (headers->EnumerateHeader(NULL, "Strict-Transport-Security", &value))
 820     security_state->AddHSTSHeader(request_info_.url.host(), value);
 821 }
 822
 823 void URLRequestHttpJob::ProcessPublicKeyPinsHeader() {
 824   DCHECK(response_info_);
 825   TransportSecurityState* security_state =
 826       request_->context()->transport_security_state();
 827   const SSLInfo& ssl_info = response_info_->ssl_info;
 828
 829   // Only accept HPKP headers on HTTPS connections that have no
 830   // certificate errors.
 831   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 832       !security_state)
 833     return;
 834
 835   // http://tools.ietf.org/html/draft-ietf-websec-key-pinning:
 836   //
 837   //   If a UA receives more than one PKP header field in an HTTP
 838   //   response message over secure transport, then the UA MUST process
 839   //   only the first such header field.
 840   HttpResponseHeaders* headers = GetResponseHeaders();
 841   std::string value;
 842   if (headers->EnumerateHeader(NULL, "Public-Key-Pins", &value))
 843     security_state->AddHPKPHeader(request_info_.url.host(), value, ssl_info);
 844 }
 845
 846 void URLRequestHttpJob::OnStartCompleted(int result) {
 847   // TODO(vadimt): Remove ScopedTracker below once crbug.com/424359 is fixed.
 848   tracked_objects::ScopedTracker tracking_profile(
 849       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 850           "424359 URLRequestHttpJob::OnStartCompleted"));
 851
 852   RecordTimer();
 853
 854   // If the request was destroyed, then there is no more work to do.
 855   if (!request_)
 856     return;
 857
 858   // If the job is done (due to cancellation), can just ignore this
 859   // notification.
 860   if (done_)
 861     return;
 862
 863   receive_headers_end_ = base::TimeTicks::Now();
 864
 865   // Clear the IO_PENDING status
 866   SetStatus(URLRequestStatus());
 867
 868   const URLRequestContext* context = request_->context();
 869
 870   if (result == ERR_SSL_PINNED_KEY_NOT_IN_CERT_CHAIN &&
 871       transaction_->GetResponseInfo() != NULL) {
 872     FraudulentCertificateReporter* reporter =
 873       context->fraudulent_certificate_reporter();
 874     if (reporter != NULL) {
 875       const SSLInfo& ssl_info = transaction_->GetResponseInfo()->ssl_info;
 876       const std::string& host = request_->url().host();
 877
 878       reporter->SendReport(host, ssl_info);
 879     }
 880   }
 881
 882   if (result == OK) {
 883     if (transaction_ && transaction_->GetResponseInfo()) {
 884       SetProxyServer(transaction_->GetResponseInfo()->proxy_server);
 885     }
 886     scoped_refptr<HttpResponseHeaders> headers = GetResponseHeaders();
 887     if (network_delegate()) {
 888       // Note that |this| may not be deleted until
 889       // |on_headers_received_callback_| or
 890       // |NetworkDelegate::URLRequestDestroyed()| has been called.
 891       OnCallToDelegate();
 892       allowed_unsafe_redirect_url_ = GURL();
 893       int error = network_delegate()->NotifyHeadersReceived(
 894           request_,
 895           on_headers_received_callback_,
 896           headers.get(),
 897           &override_response_headers_,
 898           &allowed_unsafe_redirect_url_);
 899       if (error != net::OK) {
 900         if (error == net::ERR_IO_PENDING) {
 901           awaiting_callback_ = true;
 902         } else {
 903           std::string source("delegate");
 904           request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 905                                        NetLog::StringCallback("source",
 906                                                               &source));
 907           OnCallToDelegateComplete();
 908           NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, error));
 909         }
 910         return;
 911       }
 912     }
 913
 914     SaveCookiesAndNotifyHeadersComplete(net::OK);
 915   } else if (IsCertificateError(result)) {
 916     // We encountered an SSL certificate error.
 917     if (result == ERR_SSL_WEAK_SERVER_EPHEMERAL_DH_KEY ||
 918         result == ERR_SSL_PINNED_KEY_NOT_IN_CERT_CHAIN) {
 919       // These are hard failures. They're handled separately and don't have
 920       // the correct cert status, so set it here.
 921       SSLInfo info(transaction_->GetResponseInfo()->ssl_info);
 922       info.cert_status = MapNetErrorToCertStatus(result);
 923       NotifySSLCertificateError(info, true);
 924     } else {
 925       // Maybe overridable, maybe not. Ask the delegate to decide.
 926       TransportSecurityState* state = context->transport_security_state();
 927       const bool fatal =
 928           state && state->ShouldSSLErrorsBeFatal(request_info_.url.host());
 929       NotifySSLCertificateError(
 930           transaction_->GetResponseInfo()->ssl_info, fatal);
 931     }
 932   } else if (result == ERR_SSL_CLIENT_AUTH_CERT_NEEDED) {
 933     NotifyCertificateRequested(
 934         transaction_->GetResponseInfo()->cert_request_info.get());
 935   } else {
 936     // Even on an error, there may be useful information in the response
 937     // info (e.g. whether there's a cached copy).
 938     if (transaction_.get())
 939       response_info_ = transaction_->GetResponseInfo();
 940     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 941   }
 942 }
 943
 944 void URLRequestHttpJob::OnHeadersReceivedCallback(int result) {
 945   awaiting_callback_ = false;
 946
 947   // Check that there are no callbacks to already canceled requests.
 948   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 949
 950   SaveCookiesAndNotifyHeadersComplete(result);
 951 }
 952
 953 void URLRequestHttpJob::OnReadCompleted(int result) {
 954   // TODO(vadimt): Remove ScopedTracker below once crbug.com/424359 is fixed.
 955   tracked_objects::ScopedTracker tracking_profile(
 956       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 957           "424359 URLRequestHttpJob::OnReadCompleted"));
 958
 959   read_in_progress_ = false;
 960
 961   if (ShouldFixMismatchedContentLength(result))
 962     result = OK;
 963
 964   if (result == OK) {
 965     NotifyDone(URLRequestStatus());
 966   } else if (result < 0) {
 967     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, result));
 968   } else {
 969     // Clear the IO_PENDING status
 970     SetStatus(URLRequestStatus());
 971   }
 972
 973   NotifyReadComplete(result);
 974 }
 975
 976 void URLRequestHttpJob::RestartTransactionWithAuth(
 977     const AuthCredentials& credentials) {
 978   auth_credentials_ = credentials;
 979
 980   // These will be reset in OnStartCompleted.
 981   response_info_ = NULL;
 982   receive_headers_end_ = base::TimeTicks();
 983   response_cookies_.clear();
 984
 985   ResetTimer();
 986
 987   // Update the cookies, since the cookie store may have been updated from the
 988   // headers in the 401/407. Since cookies were already appended to
 989   // extra_headers, we need to strip them out before adding them again.
 990   request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kCookie);
 991
 992   AddCookieHeaderAndStart();
 993 }
 994
 995 void URLRequestHttpJob::SetUpload(UploadDataStream* upload) {
 996   DCHECK(!transaction_.get()) << "cannot change once started";
 997   request_info_.upload_data_stream = upload;
 998 }
 999
1000 void URLRequestHttpJob::SetExtraRequestHeaders(
1001     const HttpRequestHeaders& headers) {
1002   DCHECK(!transaction_.get()) << "cannot change once started";
1003   request_info_.extra_headers.CopyFrom(headers);
1004 }
1005
1006 LoadState URLRequestHttpJob::GetLoadState() const {
1007   return transaction_.get() ?
1008       transaction_->GetLoadState() : LOAD_STATE_IDLE;
1009 }
1010
1011 UploadProgress URLRequestHttpJob::GetUploadProgress() const {
1012   return transaction_.get() ?
1013       transaction_->GetUploadProgress() : UploadProgress();
1014 }
1015
1016 bool URLRequestHttpJob::GetMimeType(std::string* mime_type) const {
1017   DCHECK(transaction_.get());
1018
1019   if (!response_info_)
1020     return false;
1021
1022   HttpResponseHeaders* headers = GetResponseHeaders();
1023   if (!headers)
1024     return false;
1025   return headers->GetMimeType(mime_type);
1026 }
1027
1028 bool URLRequestHttpJob::GetCharset(std::string* charset) {
1029   DCHECK(transaction_.get());
1030
1031   if (!response_info_)
1032     return false;
1033
1034   return GetResponseHeaders()->GetCharset(charset);
1035 }
1036
1037 void URLRequestHttpJob::GetResponseInfo(HttpResponseInfo* info) {
1038   DCHECK(request_);
1039
1040   if (response_info_) {
1041     DCHECK(transaction_.get());
1042
1043     *info = *response_info_;
1044     if (override_response_headers_.get())
1045       info->headers = override_response_headers_;
1046   }
1047 }
1048
1049 void URLRequestHttpJob::GetLoadTimingInfo(
1050     LoadTimingInfo* load_timing_info) const {
1051   // If haven't made it far enough to receive any headers, don't return
1052   // anything. This makes for more consistent behavior in the case of errors.
1053   if (!transaction_ || receive_headers_end_.is_null())
1054     return;
1055   if (transaction_->GetLoadTimingInfo(load_timing_info))
1056     load_timing_info->receive_headers_end = receive_headers_end_;
1057 }
1058
1059 bool URLRequestHttpJob::GetResponseCookies(std::vector<std::string>* cookies) {
1060   DCHECK(transaction_.get());
1061
1062   if (!response_info_)
1063     return false;
1064
1065   // TODO(darin): Why are we extracting response cookies again?  Perhaps we
1066   // should just leverage response_cookies_.
1067
1068   cookies->clear();
1069   FetchResponseCookies(cookies);
1070   return true;
1071 }
1072
1073 int URLRequestHttpJob::GetResponseCode() const {
1074   DCHECK(transaction_.get());
1075
1076   if (!response_info_)
1077     return -1;
1078
1079   return GetResponseHeaders()->response_code();
1080 }
1081
1082 Filter* URLRequestHttpJob::SetupFilter() const {
1083   DCHECK(transaction_.get());
1084   if (!response_info_)
1085     return NULL;
1086
1087   std::vector<Filter::FilterType> encoding_types;
1088   std::string encoding_type;
1089   HttpResponseHeaders* headers = GetResponseHeaders();
1090   void* iter = NULL;
1091   while (headers->EnumerateHeader(&iter, "Content-Encoding", &encoding_type)) {
1092     encoding_types.push_back(Filter::ConvertEncodingToType(encoding_type));
1093   }
1094
1095   // Even if encoding types are empty, there is a chance that we need to add
1096   // some decoding, as some proxies strip encoding completely. In such cases,
1097   // we may need to add (for example) SDCH filtering (when the context suggests
1098   // it is appropriate).
1099   Filter::FixupEncodingTypes(*filter_context_, &encoding_types);
1100
1101   return !encoding_types.empty()
1102       ? Filter::Factory(encoding_types, *filter_context_) : NULL;
1103 }
1104
1105 bool URLRequestHttpJob::CopyFragmentOnRedirect(const GURL& location) const {
1106   // Allow modification of reference fragments by default, unless
1107   // |allowed_unsafe_redirect_url_| is set and equal to the redirect URL.
1108   // When this is the case, we assume that the network delegate has set the
1109   // desired redirect URL (with or without fragment), so it must not be changed
1110   // any more.
1111   return !allowed_unsafe_redirect_url_.is_valid() ||
1112        allowed_unsafe_redirect_url_ != location;
1113 }
1114
1115 bool URLRequestHttpJob::IsSafeRedirect(const GURL& location) {
1116   // HTTP is always safe.
1117   // TODO(pauljensen): Remove once crbug.com/146591 is fixed.
1118   if (location.is_valid() &&
1119       (location.scheme() == "http" || location.scheme() == "https")) {
1120     return true;
1121   }
1122   // Delegates may mark a URL as safe for redirection.
1123   if (allowed_unsafe_redirect_url_.is_valid() &&
1124       allowed_unsafe_redirect_url_ == location) {
1125     return true;
1126   }
1127   // Query URLRequestJobFactory as to whether |location| would be safe to
1128   // redirect to.
1129   return request_->context()->job_factory() &&
1130       request_->context()->job_factory()->IsSafeRedirectTarget(location);
1131 }
1132
1133 bool URLRequestHttpJob::NeedsAuth() {
1134   int code = GetResponseCode();
1135   if (code == -1)
1136     return false;
1137
1138   // Check if we need either Proxy or WWW Authentication. This could happen
1139   // because we either provided no auth info, or provided incorrect info.
1140   switch (code) {
1141     case 407:
1142       if (proxy_auth_state_ == AUTH_STATE_CANCELED)
1143         return false;
1144       proxy_auth_state_ = AUTH_STATE_NEED_AUTH;
1145       return true;
1146     case 401:
1147       if (server_auth_state_ == AUTH_STATE_CANCELED)
1148         return false;
1149       server_auth_state_ = AUTH_STATE_NEED_AUTH;
1150       return true;
1151   }
1152   return false;
1153 }
1154
1155 void URLRequestHttpJob::GetAuthChallengeInfo(
1156     scoped_refptr<AuthChallengeInfo>* result) {
1157   DCHECK(transaction_.get());
1158   DCHECK(response_info_);
1159
1160   // sanity checks:
1161   DCHECK(proxy_auth_state_ == AUTH_STATE_NEED_AUTH ||
1162          server_auth_state_ == AUTH_STATE_NEED_AUTH);
1163   DCHECK((GetResponseHeaders()->response_code() == HTTP_UNAUTHORIZED) ||
1164          (GetResponseHeaders()->response_code() ==
1165           HTTP_PROXY_AUTHENTICATION_REQUIRED));
1166
1167   *result = response_info_->auth_challenge;
1168 }
1169
1170 void URLRequestHttpJob::SetAuth(const AuthCredentials& credentials) {
1171   DCHECK(transaction_.get());
1172
1173   // Proxy gets set first, then WWW.
1174   if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1175     proxy_auth_state_ = AUTH_STATE_HAVE_AUTH;
1176   } else {
1177     DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1178     server_auth_state_ = AUTH_STATE_HAVE_AUTH;
1179   }
1180
1181   RestartTransactionWithAuth(credentials);
1182 }
1183
1184 void URLRequestHttpJob::CancelAuth() {
1185   // Proxy gets set first, then WWW.
1186   if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1187     proxy_auth_state_ = AUTH_STATE_CANCELED;
1188   } else {
1189     DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1190     server_auth_state_ = AUTH_STATE_CANCELED;
1191   }
1192
1193   // These will be reset in OnStartCompleted.
1194   response_info_ = NULL;
1195   receive_headers_end_ = base::TimeTicks::Now();
1196   response_cookies_.clear();
1197
1198   ResetTimer();
1199
1200   // OK, let the consumer read the error page...
1201   //
1202   // Because we set the AUTH_STATE_CANCELED flag, NeedsAuth will return false,
1203   // which will cause the consumer to receive OnResponseStarted instead of
1204   // OnAuthRequired.
1205   //
1206   // We have to do this via InvokeLater to avoid "recursing" the consumer.
1207   //
1208   base::MessageLoop::current()->PostTask(
1209       FROM_HERE,
1210       base::Bind(&URLRequestHttpJob::OnStartCompleted,
1211                  weak_factory_.GetWeakPtr(), OK));
1212 }
1213
1214 void URLRequestHttpJob::ContinueWithCertificate(
1215     X509Certificate* client_cert) {
1216   DCHECK(transaction_.get());
1217
1218   DCHECK(!response_info_) << "should not have a response yet";
1219   receive_headers_end_ = base::TimeTicks();
1220
1221   ResetTimer();
1222
1223   // No matter what, we want to report our status as IO pending since we will
1224   // be notifying our consumer asynchronously via OnStartCompleted.
1225   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1226
1227   int rv = transaction_->RestartWithCertificate(client_cert, start_callback_);
1228   if (rv == ERR_IO_PENDING)
1229     return;
1230
1231   // The transaction started synchronously, but we need to notify the
1232   // URLRequest delegate via the message loop.
1233   base::MessageLoop::current()->PostTask(
1234       FROM_HERE,
1235       base::Bind(&URLRequestHttpJob::OnStartCompleted,
1236                  weak_factory_.GetWeakPtr(), rv));
1237 }
1238
1239 void URLRequestHttpJob::ContinueDespiteLastError() {
1240   // If the transaction was destroyed, then the job was cancelled.
1241   if (!transaction_.get())
1242     return;
1243
1244   DCHECK(!response_info_) << "should not have a response yet";
1245   receive_headers_end_ = base::TimeTicks();
1246
1247   ResetTimer();
1248
1249   // No matter what, we want to report our status as IO pending since we will
1250   // be notifying our consumer asynchronously via OnStartCompleted.
1251   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1252
1253   int rv = transaction_->RestartIgnoringLastError(start_callback_);
1254   if (rv == ERR_IO_PENDING)
1255     return;
1256
1257   // The transaction started synchronously, but we need to notify the
1258   // URLRequest delegate via the message loop.
1259   base::MessageLoop::current()->PostTask(
1260       FROM_HERE,
1261       base::Bind(&URLRequestHttpJob::OnStartCompleted,
1262                  weak_factory_.GetWeakPtr(), rv));
1263 }
1264
1265 void URLRequestHttpJob::ResumeNetworkStart() {
1266   DCHECK(transaction_.get());
1267   transaction_->ResumeNetworkStart();
1268 }
1269
1270 bool URLRequestHttpJob::ShouldFixMismatchedContentLength(int rv) const {
1271   // Some servers send the body compressed, but specify the content length as
1272   // the uncompressed size. Although this violates the HTTP spec we want to
1273   // support it (as IE and FireFox do), but *only* for an exact match.
1274   // See http://crbug.com/79694.
1275   if (rv == net::ERR_CONTENT_LENGTH_MISMATCH ||
1276       rv == net::ERR_INCOMPLETE_CHUNKED_ENCODING) {
1277     if (request_ && request_->response_headers()) {
1278       int64 expected_length = request_->response_headers()->GetContentLength();
1279       VLOG(1) << __FUNCTION__ << "() "
1280               << "\"" << request_->url().spec() << "\""
1281               << " content-length = " << expected_length
1282               << " pre total = " << prefilter_bytes_read()
1283               << " post total = " << postfilter_bytes_read();
1284       if (postfilter_bytes_read() == expected_length) {
1285         // Clear the error.
1286         return true;
1287       }
1288     }
1289   }
1290   return false;
1291 }
1292
1293 bool URLRequestHttpJob::ReadRawData(IOBuffer* buf, int buf_size,
1294                                     int* bytes_read) {
1295   // TODO(vadimt): Remove ScopedTracker below once crbug.com/423948 is fixed.
1296   tracked_objects::ScopedTracker tracking_profile1(
1297       FROM_HERE_WITH_EXPLICIT_FUNCTION(
1298           "423948 URLRequestHttpJob::ReadRawData1"));
1299
1300   DCHECK_NE(buf_size, 0);
1301   DCHECK(bytes_read);
1302   DCHECK(!read_in_progress_);
1303
1304   int rv = transaction_->Read(
1305       buf, buf_size,
1306       base::Bind(&URLRequestHttpJob::OnReadCompleted, base::Unretained(this)));
1307
1308   if (ShouldFixMismatchedContentLength(rv))
1309     rv = 0;
1310
1311   if (rv >= 0) {
1312     *bytes_read = rv;
1313     if (!rv) {
1314       // TODO(vadimt): Remove ScopedTracker below once crbug.com/423948 is
1315       // fixed.
1316       tracked_objects::ScopedTracker tracking_profile2(
1317           FROM_HERE_WITH_EXPLICIT_FUNCTION(
1318               "423948 URLRequestHttpJob::ReadRawData2"));
1319
1320       DoneWithRequest(FINISHED);
1321     }
1322     return true;
1323   }
1324
1325   if (rv == ERR_IO_PENDING) {
1326     read_in_progress_ = true;
1327     SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1328   } else {
1329     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, rv));
1330   }
1331
1332   return false;
1333 }
1334
1335 void URLRequestHttpJob::StopCaching() {
1336   if (transaction_.get())
1337     transaction_->StopCaching();
1338 }
1339
1340 bool URLRequestHttpJob::GetFullRequestHeaders(
1341     HttpRequestHeaders* headers) const {
1342   if (!transaction_)
1343     return false;
1344
1345   return transaction_->GetFullRequestHeaders(headers);
1346 }
1347
1348 int64 URLRequestHttpJob::GetTotalReceivedBytes() const {
1349   if (!transaction_)
1350     return 0;
1351
1352   return transaction_->GetTotalReceivedBytes();
1353 }
1354
1355 void URLRequestHttpJob::DoneReading() {
1356   if (transaction_) {
1357     transaction_->DoneReading();
1358   }
1359   DoneWithRequest(FINISHED);
1360 }
1361
1362 void URLRequestHttpJob::DoneReadingRedirectResponse() {
1363   if (transaction_) {
1364     if (transaction_->GetResponseInfo()->headers->IsRedirect(NULL)) {
1365       // If the original headers indicate a redirect, go ahead and cache the
1366       // response, even if the |override_response_headers_| are a redirect to
1367       // another location.
1368       transaction_->DoneReading();
1369     } else {
1370       // Otherwise, |override_response_headers_| must be non-NULL and contain
1371       // bogus headers indicating a redirect.
1372       DCHECK(override_response_headers_.get());
1373       DCHECK(override_response_headers_->IsRedirect(NULL));
1374       transaction_->StopCaching();
1375     }
1376   }
1377   DoneWithRequest(FINISHED);
1378 }
1379
1380 HostPortPair URLRequestHttpJob::GetSocketAddress() const {
1381   return response_info_ ? response_info_->socket_address : HostPortPair();
1382 }
1383
1384 void URLRequestHttpJob::RecordTimer() {
1385   if (request_creation_time_.is_null()) {
1386     NOTREACHED()
1387         << "The same transaction shouldn't start twice without new timing.";
1388     return;
1389   }
1390
1391   base::TimeDelta to_start = base::Time::Now() - request_creation_time_;
1392   request_creation_time_ = base::Time();
1393
1394   UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpTimeToFirstByte", to_start);
1395 }
1396
1397 void URLRequestHttpJob::ResetTimer() {
1398   if (!request_creation_time_.is_null()) {
1399     NOTREACHED()
1400         << "The timer was reset before it was recorded.";
1401     return;
1402   }
1403   request_creation_time_ = base::Time::Now();
1404 }
1405
1406 void URLRequestHttpJob::UpdatePacketReadTimes() {
1407   if (!packet_timing_enabled_)
1408     return;
1409
1410   if (filter_input_byte_count() <= bytes_observed_in_packets_) {
1411     DCHECK_EQ(filter_input_byte_count(), bytes_observed_in_packets_);
1412     return;  // No new bytes have arrived.
1413   }
1414
1415   base::Time now(base::Time::Now());
1416   if (!bytes_observed_in_packets_)
1417     request_time_snapshot_ = now;
1418   final_packet_time_ = now;
1419
1420   bytes_observed_in_packets_ = filter_input_byte_count();
1421 }
1422
1423 void URLRequestHttpJob::RecordPacketStats(
1424     FilterContext::StatisticSelector statistic) const {
1425   if (!packet_timing_enabled_ || (final_packet_time_ == base::Time()))
1426     return;
1427
1428   base::TimeDelta duration = final_packet_time_ - request_time_snapshot_;
1429   switch (statistic) {
1430     case FilterContext::SDCH_DECODE: {
1431       UMA_HISTOGRAM_CUSTOM_COUNTS("Sdch3.Network_Decode_Bytes_Processed_b",
1432           static_cast<int>(bytes_observed_in_packets_), 500, 100000, 100);
1433       return;
1434     }
1435     case FilterContext::SDCH_PASSTHROUGH: {
1436       // Despite advertising a dictionary, we handled non-sdch compressed
1437       // content.
1438       return;
1439     }
1440
1441     case FilterContext::SDCH_EXPERIMENT_DECODE: {
1442       UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment3_Decode",
1443                                   duration,
1444                                   base::TimeDelta::FromMilliseconds(20),
1445                                   base::TimeDelta::FromMinutes(10), 100);
1446       return;
1447     }
1448     case FilterContext::SDCH_EXPERIMENT_HOLDBACK: {
1449       UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment3_Holdback",
1450                                   duration,
1451                                   base::TimeDelta::FromMilliseconds(20),
1452                                   base::TimeDelta::FromMinutes(10), 100);
1453       return;
1454     }
1455     default:
1456       NOTREACHED();
1457       return;
1458   }
1459 }
1460
1461 void URLRequestHttpJob::RecordPerfHistograms(CompletionCause reason) {
1462   if (start_time_.is_null())
1463     return;
1464
1465   base::TimeDelta total_time = base::TimeTicks::Now() - start_time_;
1466   UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTime", total_time);
1467
1468   if (reason == FINISHED) {
1469     UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeSuccess", total_time);
1470   } else {
1471     UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCancel", total_time);
1472   }
1473
1474   if (response_info_) {
1475     if (response_info_->was_cached) {
1476       UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCached", total_time);
1477     } else  {
1478       UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeNotCached", total_time);
1479     }
1480   }
1481
1482   if (request_info_.load_flags & LOAD_PREFETCH && !request_->was_cached())
1483     UMA_HISTOGRAM_COUNTS("Net.Prefetch.PrefilterBytesReadFromNetwork",
1484                          prefilter_bytes_read());
1485
1486   start_time_ = base::TimeTicks();
1487 }
1488
1489 void URLRequestHttpJob::DoneWithRequest(CompletionCause reason) {
1490   if (done_)
1491     return;
1492   done_ = true;
1493   RecordPerfHistograms(reason);
1494   if (reason == FINISHED) {
1495     request_->set_received_response_content_length(prefilter_bytes_read());
1496   }
1497 }
1498
1499 HttpResponseHeaders* URLRequestHttpJob::GetResponseHeaders() const {
1500   DCHECK(transaction_.get());
1501   DCHECK(transaction_->GetResponseInfo());
1502   return override_response_headers_.get() ?
1503              override_response_headers_.get() :
1504              transaction_->GetResponseInfo()->headers.get();
1505 }
1506
1507 void URLRequestHttpJob::NotifyURLRequestDestroyed() {
1508   awaiting_callback_ = false;
1509 }
1510
1511 }  // namespace net