net/url_request/url_request_http_job.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_request_http_job.h"
   6
   7 #include "base/base_switches.h"
   8 #include "base/bind.h"
   9 #include "base/bind_helpers.h"
  10 #include "base/command_line.h"
  11 #include "base/compiler_specific.h"
  12 #include "base/file_version_info.h"
  13 #include "base/message_loop/message_loop.h"
  14 #include "base/metrics/field_trial.h"
  15 #include "base/metrics/histogram.h"
  16 #include "base/profiler/scoped_tracker.h"
  17 #include "base/rand_util.h"
  18 #include "base/strings/string_util.h"
  19 #include "base/time/time.h"
  20 #include "base/values.h"
  21 #include "net/base/host_port_pair.h"
  22 #include "net/base/load_flags.h"
  23 #include "net/base/net_errors.h"
  24 #include "net/base/net_util.h"
  25 #include "net/base/network_delegate.h"
  26 #include "net/base/sdch_manager.h"
  27 #include "net/base/sdch_net_log_params.h"
  28 #include "net/cert/cert_status_flags.h"
  29 #include "net/cookies/cookie_store.h"
  30 #include "net/http/http_content_disposition.h"
  31 #include "net/http/http_network_session.h"
  32 #include "net/http/http_request_headers.h"
  33 #include "net/http/http_response_headers.h"
  34 #include "net/http/http_response_info.h"
  35 #include "net/http/http_status_code.h"
  36 #include "net/http/http_transaction.h"
  37 #include "net/http/http_transaction_factory.h"
  38 #include "net/http/http_util.h"
  39 #include "net/proxy/proxy_info.h"
  40 #include "net/ssl/ssl_cert_request_info.h"
  41 #include "net/ssl/ssl_config_service.h"
  42 #include "net/url_request/fraudulent_certificate_reporter.h"
  43 #include "net/url_request/http_user_agent_settings.h"
  44 #include "net/url_request/url_request.h"
  45 #include "net/url_request/url_request_context.h"
  46 #include "net/url_request/url_request_error_job.h"
  47 #include "net/url_request/url_request_job_factory.h"
  48 #include "net/url_request/url_request_redirect_job.h"
  49 #include "net/url_request/url_request_throttler_manager.h"
  50 #include "net/websockets/websocket_handshake_stream_base.h"
  51
  52 static const char kAvailDictionaryHeader[] = "Avail-Dictionary";
  53
  54 namespace net {
  55
  56 class URLRequestHttpJob::HttpFilterContext : public FilterContext {
  57  public:
  58   explicit HttpFilterContext(URLRequestHttpJob* job);
  59   ~HttpFilterContext() override;
  60
  61   // FilterContext implementation.
  62   bool GetMimeType(std::string* mime_type) const override;
  63   bool GetURL(GURL* gurl) const override;
  64   base::Time GetRequestTime() const override;
  65   bool IsCachedContent() const override;
  66   SdchManager::DictionarySet* SdchDictionariesAdvertised() const override;
  67   int64 GetByteReadCount() const override;
  68   int GetResponseCode() const override;
  69   const URLRequestContext* GetURLRequestContext() const override;
  70   void RecordPacketStats(StatisticSelector statistic) const override;
  71   const BoundNetLog& GetNetLog() const override;
  72
  73  private:
  74   URLRequestHttpJob* job_;
  75
  76   // URLRequestHttpJob may be detached from URLRequest, but we still need to
  77   // return something.
  78   BoundNetLog dummy_log_;
  79
  80   DISALLOW_COPY_AND_ASSIGN(HttpFilterContext);
  81 };
  82
  83 URLRequestHttpJob::HttpFilterContext::HttpFilterContext(URLRequestHttpJob* job)
  84     : job_(job) {
  85   DCHECK(job_);
  86 }
  87
  88 URLRequestHttpJob::HttpFilterContext::~HttpFilterContext() {
  89 }
  90
  91 bool URLRequestHttpJob::HttpFilterContext::GetMimeType(
  92     std::string* mime_type) const {
  93   return job_->GetMimeType(mime_type);
  94 }
  95
  96 bool URLRequestHttpJob::HttpFilterContext::GetURL(GURL* gurl) const {
  97   if (!job_->request())
  98     return false;
  99   *gurl = job_->request()->url();
 100   return true;
 101 }
 102
 103 base::Time URLRequestHttpJob::HttpFilterContext::GetRequestTime() const {
 104   return job_->request() ? job_->request()->request_time() : base::Time();
 105 }
 106
 107 bool URLRequestHttpJob::HttpFilterContext::IsCachedContent() const {
 108   return job_->is_cached_content_;
 109 }
 110
 111 SdchManager::DictionarySet*
 112 URLRequestHttpJob::HttpFilterContext::SdchDictionariesAdvertised() const {
 113   return job_->dictionaries_advertised_.get();
 114 }
 115
 116 int64 URLRequestHttpJob::HttpFilterContext::GetByteReadCount() const {
 117   return job_->prefilter_bytes_read();
 118 }
 119
 120 int URLRequestHttpJob::HttpFilterContext::GetResponseCode() const {
 121   return job_->GetResponseCode();
 122 }
 123
 124 const URLRequestContext*
 125 URLRequestHttpJob::HttpFilterContext::GetURLRequestContext() const {
 126   return job_->request() ? job_->request()->context() : NULL;
 127 }
 128
 129 void URLRequestHttpJob::HttpFilterContext::RecordPacketStats(
 130     StatisticSelector statistic) const {
 131   job_->RecordPacketStats(statistic);
 132 }
 133
 134 const BoundNetLog& URLRequestHttpJob::HttpFilterContext::GetNetLog() const {
 135   return job_->request() ? job_->request()->net_log() : dummy_log_;
 136 }
 137
 138 // TODO(darin): make sure the port blocking code is not lost
 139 // static
 140 URLRequestJob* URLRequestHttpJob::Factory(URLRequest* request,
 141                                           NetworkDelegate* network_delegate,
 142                                           const std::string& scheme) {
 143   DCHECK(scheme == "http" || scheme == "https" || scheme == "ws" ||
 144          scheme == "wss");
 145
 146   if (!request->context()->http_transaction_factory()) {
 147     NOTREACHED() << "requires a valid context";
 148     return new URLRequestErrorJob(
 149         request, network_delegate, ERR_INVALID_ARGUMENT);
 150   }
 151
 152   GURL redirect_url;
 153   if (request->GetHSTSRedirect(&redirect_url)) {
 154     return new URLRequestRedirectJob(
 155         request, network_delegate, redirect_url,
 156         // Use status code 307 to preserve the method, so POST requests work.
 157         URLRequestRedirectJob::REDIRECT_307_TEMPORARY_REDIRECT, "HSTS");
 158   }
 159   return new URLRequestHttpJob(request,
 160                                network_delegate,
 161                                request->context()->http_user_agent_settings());
 162 }
 163
 164 URLRequestHttpJob::URLRequestHttpJob(
 165     URLRequest* request,
 166     NetworkDelegate* network_delegate,
 167     const HttpUserAgentSettings* http_user_agent_settings)
 168     : URLRequestJob(request, network_delegate),
 169       priority_(DEFAULT_PRIORITY),
 170       response_info_(NULL),
 171       response_cookies_save_index_(0),
 172       proxy_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
 173       server_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
 174       start_callback_(base::Bind(&URLRequestHttpJob::OnStartCompleted,
 175                                  base::Unretained(this))),
 176       notify_before_headers_sent_callback_(
 177           base::Bind(&URLRequestHttpJob::NotifyBeforeSendHeadersCallback,
 178                      base::Unretained(this))),
 179       read_in_progress_(false),
 180       throttling_entry_(NULL),
 181       sdch_test_activated_(false),
 182       sdch_test_control_(false),
 183       is_cached_content_(false),
 184       request_creation_time_(),
 185       packet_timing_enabled_(false),
 186       done_(false),
 187       bytes_observed_in_packets_(0),
 188       request_time_snapshot_(),
 189       final_packet_time_(),
 190       filter_context_(new HttpFilterContext(this)),
 191       on_headers_received_callback_(
 192           base::Bind(&URLRequestHttpJob::OnHeadersReceivedCallback,
 193                      base::Unretained(this))),
 194       awaiting_callback_(false),
 195       http_user_agent_settings_(http_user_agent_settings),
 196       weak_factory_(this) {
 197   URLRequestThrottlerManager* manager = request->context()->throttler_manager();
 198   if (manager)
 199     throttling_entry_ = manager->RegisterRequestUrl(request->url());
 200
 201   ResetTimer();
 202 }
 203
 204 URLRequestHttpJob::~URLRequestHttpJob() {
 205   CHECK(!awaiting_callback_);
 206
 207   DCHECK(!sdch_test_control_ || !sdch_test_activated_);
 208   if (!is_cached_content_) {
 209     if (sdch_test_control_)
 210       RecordPacketStats(FilterContext::SDCH_EXPERIMENT_HOLDBACK);
 211     if (sdch_test_activated_)
 212       RecordPacketStats(FilterContext::SDCH_EXPERIMENT_DECODE);
 213   }
 214   // Make sure SDCH filters are told to emit histogram data while
 215   // filter_context_ is still alive.
 216   DestroyFilters();
 217
 218   DoneWithRequest(ABORTED);
 219 }
 220
 221 void URLRequestHttpJob::SetPriority(RequestPriority priority) {
 222   priority_ = priority;
 223   if (transaction_)
 224     transaction_->SetPriority(priority_);
 225 }
 226
 227 void URLRequestHttpJob::Start() {
 228   // TODO(mmenke): Remove ScopedTracker below once crbug.com/456327 is fixed.
 229   tracked_objects::ScopedTracker tracking_profile(
 230       FROM_HERE_WITH_EXPLICIT_FUNCTION("456327 URLRequestHttpJob::Start"));
 231
 232   DCHECK(!transaction_.get());
 233
 234   // URLRequest::SetReferrer ensures that we do not send username and password
 235   // fields in the referrer.
 236   GURL referrer(request_->referrer());
 237
 238   request_info_.url = request_->url();
 239   request_info_.method = request_->method();
 240   request_info_.load_flags = request_->load_flags();
 241   // Enable privacy mode if cookie settings or flags tell us not send or
 242   // save cookies.
 243   bool enable_privacy_mode =
 244       (request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES) ||
 245       (request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) ||
 246       CanEnablePrivacyMode();
 247   // Privacy mode could still be disabled in OnCookiesLoaded if we are going
 248   // to send previously saved cookies.
 249   request_info_.privacy_mode = enable_privacy_mode ?
 250       PRIVACY_MODE_ENABLED : PRIVACY_MODE_DISABLED;
 251
 252   // Strip Referer from request_info_.extra_headers to prevent, e.g., plugins
 253   // from overriding headers that are controlled using other means. Otherwise a
 254   // plugin could set a referrer although sending the referrer is inhibited.
 255   request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kReferer);
 256
 257   // Our consumer should have made sure that this is a safe referrer. See for
 258   // instance WebCore::FrameLoader::HideReferrer.
 259   if (referrer.is_valid()) {
 260     request_info_.extra_headers.SetHeader(HttpRequestHeaders::kReferer,
 261                                           referrer.spec());
 262   }
 263
 264   request_info_.extra_headers.SetHeaderIfMissing(
 265       HttpRequestHeaders::kUserAgent,
 266       http_user_agent_settings_ ?
 267           http_user_agent_settings_->GetUserAgent() : std::string());
 268
 269   AddExtraHeaders();
 270   AddCookieHeaderAndStart();
 271 }
 272
 273 void URLRequestHttpJob::Kill() {
 274   if (!transaction_.get())
 275     return;
 276
 277   weak_factory_.InvalidateWeakPtrs();
 278   DestroyTransaction();
 279   URLRequestJob::Kill();
 280 }
 281
 282 void URLRequestHttpJob::GetConnectionAttempts(ConnectionAttempts* out) const {
 283   if (transaction_)
 284     transaction_->GetConnectionAttempts(out);
 285   else
 286     out->clear();
 287 }
 288
 289 void URLRequestHttpJob::NotifyBeforeSendProxyHeadersCallback(
 290     const ProxyInfo& proxy_info,
 291     HttpRequestHeaders* request_headers) {
 292   DCHECK(request_headers);
 293   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 294   if (network_delegate()) {
 295     network_delegate()->NotifyBeforeSendProxyHeaders(
 296         request_,
 297         proxy_info,
 298         request_headers);
 299   }
 300 }
 301
 302 void URLRequestHttpJob::NotifyHeadersComplete() {
 303   DCHECK(!response_info_);
 304
 305   response_info_ = transaction_->GetResponseInfo();
 306
 307   // Save boolean, as we'll need this info at destruction time, and filters may
 308   // also need this info.
 309   is_cached_content_ = response_info_->was_cached;
 310
 311   if (!is_cached_content_ && throttling_entry_.get())
 312     throttling_entry_->UpdateWithResponse(GetResponseCode());
 313
 314   // The ordering of these calls is not important.
 315   ProcessStrictTransportSecurityHeader();
 316   ProcessPublicKeyPinsHeader();
 317
 318   // Handle the server notification of a new SDCH dictionary.
 319   SdchManager* sdch_manager(request()->context()->sdch_manager());
 320   if (sdch_manager) {
 321     SdchProblemCode rv = sdch_manager->IsInSupportedDomain(request()->url());
 322     if (rv != SDCH_OK) {
 323       // If SDCH is just disabled, it is not a real error.
 324       if (rv != SDCH_DISABLED && rv != SDCH_SECURE_SCHEME_NOT_SUPPORTED) {
 325         SdchManager::SdchErrorRecovery(rv);
 326         request()->net_log().AddEvent(
 327             NetLog::TYPE_SDCH_DECODING_ERROR,
 328             base::Bind(&NetLogSdchResourceProblemCallback, rv));
 329       }
 330     } else {
 331       const std::string name = "Get-Dictionary";
 332       std::string url_text;
 333       void* iter = NULL;
 334       // TODO(jar): We need to not fetch dictionaries the first time they are
 335       // seen, but rather wait until we can justify their usefulness.
 336       // For now, we will only fetch the first dictionary, which will at least
 337       // require multiple suggestions before we get additional ones for this
 338       // site. Eventually we should wait until a dictionary is requested
 339       // several times
 340       // before we even download it (so that we don't waste memory or
 341       // bandwidth).
 342       if (GetResponseHeaders()->EnumerateHeader(&iter, name, &url_text)) {
 343         // Resolve suggested URL relative to request url.
 344         GURL sdch_dictionary_url = request_->url().Resolve(url_text);
 345         if (sdch_dictionary_url.is_valid()) {
 346           rv = sdch_manager->OnGetDictionary(request_->url(),
 347                                              sdch_dictionary_url);
 348           if (rv != SDCH_OK) {
 349             SdchManager::SdchErrorRecovery(rv);
 350             request_->net_log().AddEvent(
 351                 NetLog::TYPE_SDCH_DICTIONARY_ERROR,
 352                 base::Bind(&NetLogSdchDictionaryFetchProblemCallback, rv,
 353                            sdch_dictionary_url, false));
 354           }
 355         }
 356       }
 357     }
 358   }
 359
 360   // Handle the server signalling no SDCH encoding.
 361   if (dictionaries_advertised_) {
 362     // We are wary of proxies that discard or damage SDCH encoding. If a server
 363     // explicitly states that this is not SDCH content, then we can correct our
 364     // assumption that this is an SDCH response, and avoid the need to recover
 365     // as though the content is corrupted (when we discover it is not SDCH
 366     // encoded).
 367     std::string sdch_response_status;
 368     void* iter = NULL;
 369     while (GetResponseHeaders()->EnumerateHeader(&iter, "X-Sdch-Encode",
 370                                                  &sdch_response_status)) {
 371       if (sdch_response_status == "0") {
 372         dictionaries_advertised_.reset();
 373         break;
 374       }
 375     }
 376   }
 377
 378   // The HTTP transaction may be restarted several times for the purposes
 379   // of sending authorization information. Each time it restarts, we get
 380   // notified of the headers completion so that we can update the cookie store.
 381   if (transaction_->IsReadyToRestartForAuth()) {
 382     DCHECK(!response_info_->auth_challenge.get());
 383     // TODO(battre): This breaks the webrequest API for
 384     // URLRequestTestHTTP.BasicAuthWithCookies
 385     // where OnBeforeSendHeaders -> OnSendHeaders -> OnBeforeSendHeaders
 386     // occurs.
 387     RestartTransactionWithAuth(AuthCredentials());
 388     return;
 389   }
 390
 391   URLRequestJob::NotifyHeadersComplete();
 392 }
 393
 394 void URLRequestHttpJob::NotifyDone(const URLRequestStatus& status) {
 395   DoneWithRequest(FINISHED);
 396   URLRequestJob::NotifyDone(status);
 397 }
 398
 399 void URLRequestHttpJob::DestroyTransaction() {
 400   DCHECK(transaction_.get());
 401
 402   DoneWithRequest(ABORTED);
 403   transaction_.reset();
 404   response_info_ = NULL;
 405   receive_headers_end_ = base::TimeTicks();
 406 }
 407
 408 void URLRequestHttpJob::StartTransaction() {
 409   // TODO(mmenke): Remove ScopedTracker below once crbug.com/456327 is fixed.
 410   tracked_objects::ScopedTracker tracking_profile(
 411       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 412           "456327 URLRequestHttpJob::StartTransaction"));
 413
 414   if (network_delegate()) {
 415     OnCallToDelegate();
 416     int rv = network_delegate()->NotifyBeforeSendHeaders(
 417         request_, notify_before_headers_sent_callback_,
 418         &request_info_.extra_headers);
 419     // If an extension blocks the request, we rely on the callback to
 420     // MaybeStartTransactionInternal().
 421     if (rv == ERR_IO_PENDING)
 422       return;
 423     MaybeStartTransactionInternal(rv);
 424     return;
 425   }
 426   StartTransactionInternal();
 427 }
 428
 429 void URLRequestHttpJob::NotifyBeforeSendHeadersCallback(int result) {
 430   // Check that there are no callbacks to already canceled requests.
 431   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 432
 433   MaybeStartTransactionInternal(result);
 434 }
 435
 436 void URLRequestHttpJob::MaybeStartTransactionInternal(int result) {
 437   // TODO(mmenke): Remove ScopedTracker below once crbug.com/456327 is fixed.
 438   tracked_objects::ScopedTracker tracking_profile(
 439       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 440           "456327 URLRequestHttpJob::MaybeStartTransactionInternal"));
 441
 442   OnCallToDelegateComplete();
 443   if (result == OK) {
 444     StartTransactionInternal();
 445   } else {
 446     std::string source("delegate");
 447     request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 448                                  NetLog::StringCallback("source", &source));
 449     NotifyCanceled();
 450     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 451   }
 452 }
 453
 454 void URLRequestHttpJob::StartTransactionInternal() {
 455   // NOTE: This method assumes that request_info_ is already setup properly.
 456
 457   // If we already have a transaction, then we should restart the transaction
 458   // with auth provided by auth_credentials_.
 459
 460   int rv;
 461
 462   if (network_delegate()) {
 463     network_delegate()->NotifySendHeaders(
 464         request_, request_info_.extra_headers);
 465   }
 466
 467   if (transaction_.get()) {
 468     rv = transaction_->RestartWithAuth(auth_credentials_, start_callback_);
 469     auth_credentials_ = AuthCredentials();
 470   } else {
 471     DCHECK(request_->context()->http_transaction_factory());
 472
 473     rv = request_->context()->http_transaction_factory()->CreateTransaction(
 474         priority_, &transaction_);
 475
 476     if (rv == OK && request_info_.url.SchemeIsWSOrWSS()) {
 477       base::SupportsUserData::Data* data = request_->GetUserData(
 478           WebSocketHandshakeStreamBase::CreateHelper::DataKey());
 479       if (data) {
 480         transaction_->SetWebSocketHandshakeStreamCreateHelper(
 481             static_cast<WebSocketHandshakeStreamBase::CreateHelper*>(data));
 482       } else {
 483         rv = ERR_DISALLOWED_URL_SCHEME;
 484       }
 485     }
 486
 487     if (rv == OK) {
 488       transaction_->SetBeforeNetworkStartCallback(
 489           base::Bind(&URLRequestHttpJob::NotifyBeforeNetworkStart,
 490                      base::Unretained(this)));
 491       transaction_->SetBeforeProxyHeadersSentCallback(
 492           base::Bind(&URLRequestHttpJob::NotifyBeforeSendProxyHeadersCallback,
 493                      base::Unretained(this)));
 494
 495       if (!throttling_entry_.get() ||
 496           !throttling_entry_->ShouldRejectRequest(*request_,
 497                                                   network_delegate())) {
 498         rv = transaction_->Start(
 499             &request_info_, start_callback_, request_->net_log());
 500         start_time_ = base::TimeTicks::Now();
 501       } else {
 502         // Special error code for the exponential back-off module.
 503         rv = ERR_TEMPORARILY_THROTTLED;
 504       }
 505     }
 506   }
 507
 508   if (rv == ERR_IO_PENDING)
 509     return;
 510
 511   // The transaction started synchronously, but we need to notify the
 512   // URLRequest delegate via the message loop.
 513   base::MessageLoop::current()->PostTask(
 514       FROM_HERE,
 515       base::Bind(&URLRequestHttpJob::OnStartCompleted,
 516                  weak_factory_.GetWeakPtr(), rv));
 517 }
 518
 519 void URLRequestHttpJob::AddExtraHeaders() {
 520   SdchManager* sdch_manager = request()->context()->sdch_manager();
 521
 522   // Supply Accept-Encoding field only if it is not already provided.
 523   // It should be provided IF the content is known to have restrictions on
 524   // potential encoding, such as streaming multi-media.
 525   // For details see bug 47381.
 526   // TODO(jar, enal): jpeg files etc. should set up a request header if
 527   // possible. Right now it is done only by buffered_resource_loader and
 528   // simple_data_source.
 529   if (!request_info_.extra_headers.HasHeader(
 530       HttpRequestHeaders::kAcceptEncoding)) {
 531     // We don't support SDCH responses to POST as there is a possibility
 532     // of having SDCH encoded responses returned (e.g. by the cache)
 533     // which we cannot decode, and in those situations, we will need
 534     // to retransmit the request without SDCH, which is illegal for a POST.
 535     bool advertise_sdch = sdch_manager != NULL && request()->method() != "POST";
 536     if (advertise_sdch) {
 537       SdchProblemCode rv = sdch_manager->IsInSupportedDomain(request()->url());
 538       if (rv != SDCH_OK) {
 539         advertise_sdch = false;
 540         // If SDCH is just disabled, it is not a real error.
 541         if (rv != SDCH_DISABLED && rv != SDCH_SECURE_SCHEME_NOT_SUPPORTED) {
 542           SdchManager::SdchErrorRecovery(rv);
 543           request()->net_log().AddEvent(
 544               NetLog::TYPE_SDCH_DECODING_ERROR,
 545               base::Bind(&NetLogSdchResourceProblemCallback, rv));
 546         }
 547       }
 548     }
 549     if (advertise_sdch) {
 550       dictionaries_advertised_ =
 551           sdch_manager->GetDictionarySet(request_->url());
 552     }
 553
 554     // The AllowLatencyExperiment() is only true if we've successfully done a
 555     // full SDCH compression recently in this browser session for this host.
 556     // Note that for this path, there might be no applicable dictionaries,
 557     // and hence we can't participate in the experiment.
 558     if (dictionaries_advertised_ &&
 559         sdch_manager->AllowLatencyExperiment(request_->url())) {
 560       // We are participating in the test (or control), and hence we'll
 561       // eventually record statistics via either SDCH_EXPERIMENT_DECODE or
 562       // SDCH_EXPERIMENT_HOLDBACK, and we'll need some packet timing data.
 563       packet_timing_enabled_ = true;
 564       if (base::RandDouble() < .01) {
 565         sdch_test_control_ = true;  // 1% probability.
 566         dictionaries_advertised_.reset();
 567         advertise_sdch = false;
 568       } else {
 569         sdch_test_activated_ = true;
 570       }
 571     }
 572
 573     // Supply Accept-Encoding headers first so that it is more likely that they
 574     // will be in the first transmitted packet. This can sometimes make it
 575     // easier to filter and analyze the streams to assure that a proxy has not
 576     // damaged these headers. Some proxies deliberately corrupt Accept-Encoding
 577     // headers.
 578     if (!advertise_sdch) {
 579       // Tell the server what compression formats we support (other than SDCH).
 580       request_info_.extra_headers.SetHeader(
 581           HttpRequestHeaders::kAcceptEncoding, "gzip, deflate");
 582     } else {
 583       // Include SDCH in acceptable list.
 584       request_info_.extra_headers.SetHeader(
 585           HttpRequestHeaders::kAcceptEncoding, "gzip, deflate, sdch");
 586       if (dictionaries_advertised_) {
 587         request_info_.extra_headers.SetHeader(
 588             kAvailDictionaryHeader,
 589             dictionaries_advertised_->GetDictionaryClientHashList());
 590         // Since we're tagging this transaction as advertising a dictionary,
 591         // we'll definitely employ an SDCH filter (or tentative sdch filter)
 592         // when we get a response. When done, we'll record histograms via
 593         // SDCH_DECODE or SDCH_PASSTHROUGH. Hence we need to record packet
 594         // arrival times.
 595         packet_timing_enabled_ = true;
 596       }
 597     }
 598   }
 599
 600   if (http_user_agent_settings_) {
 601     // Only add default Accept-Language if the request didn't have it
 602     // specified.
 603     std::string accept_language =
 604         http_user_agent_settings_->GetAcceptLanguage();
 605     if (!accept_language.empty()) {
 606       request_info_.extra_headers.SetHeaderIfMissing(
 607           HttpRequestHeaders::kAcceptLanguage,
 608           accept_language);
 609     }
 610   }
 611 }
 612
 613 void URLRequestHttpJob::AddCookieHeaderAndStart() {
 614   // No matter what, we want to report our status as IO pending since we will
 615   // be notifying our consumer asynchronously via OnStartCompleted.
 616   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
 617
 618   // If the request was destroyed, then there is no more work to do.
 619   if (!request_)
 620     return;
 621
 622   CookieStore* cookie_store = request_->context()->cookie_store();
 623   if (cookie_store && !(request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES)) {
 624     cookie_store->GetAllCookiesForURLAsync(
 625         request_->url(),
 626         base::Bind(&URLRequestHttpJob::CheckCookiePolicyAndLoad,
 627                    weak_factory_.GetWeakPtr()));
 628   } else {
 629     DoStartTransaction();
 630   }
 631 }
 632
 633 void URLRequestHttpJob::DoLoadCookies() {
 634   CookieOptions options;
 635   options.set_include_httponly();
 636
 637   // TODO(mkwst): Drop this `if` once we decide whether or not to ship
 638   // first-party cookies: https://crbug.com/459154
 639   if (network_delegate() &&
 640       network_delegate()->FirstPartyOnlyCookieExperimentEnabled())
 641     options.set_first_party_url(request_->first_party_for_cookies());
 642   else
 643     options.set_include_first_party_only();
 644
 645   request_->context()->cookie_store()->GetCookiesWithOptionsAsync(
 646       request_->url(), options, base::Bind(&URLRequestHttpJob::OnCookiesLoaded,
 647                                            weak_factory_.GetWeakPtr()));
 648 }
 649
 650 void URLRequestHttpJob::CheckCookiePolicyAndLoad(
 651     const CookieList& cookie_list) {
 652   if (CanGetCookies(cookie_list))
 653     DoLoadCookies();
 654   else
 655     DoStartTransaction();
 656 }
 657
 658 void URLRequestHttpJob::OnCookiesLoaded(const std::string& cookie_line) {
 659   if (!cookie_line.empty()) {
 660     request_info_.extra_headers.SetHeader(
 661         HttpRequestHeaders::kCookie, cookie_line);
 662     // Disable privacy mode as we are sending cookies anyway.
 663     request_info_.privacy_mode = PRIVACY_MODE_DISABLED;
 664   }
 665   DoStartTransaction();
 666 }
 667
 668 void URLRequestHttpJob::DoStartTransaction() {
 669   // We may have been canceled while retrieving cookies.
 670   if (GetStatus().is_success()) {
 671     StartTransaction();
 672   } else {
 673     NotifyCanceled();
 674   }
 675 }
 676
 677 void URLRequestHttpJob::SaveCookiesAndNotifyHeadersComplete(int result) {
 678   // End of the call started in OnStartCompleted.
 679   OnCallToDelegateComplete();
 680
 681   if (result != OK) {
 682     std::string source("delegate");
 683     request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 684                                  NetLog::StringCallback("source", &source));
 685     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 686     return;
 687   }
 688
 689   DCHECK(transaction_.get());
 690
 691   const HttpResponseInfo* response_info = transaction_->GetResponseInfo();
 692   DCHECK(response_info);
 693
 694   response_cookies_.clear();
 695   response_cookies_save_index_ = 0;
 696
 697   FetchResponseCookies(&response_cookies_);
 698
 699   if (!GetResponseHeaders()->GetDateValue(&response_date_))
 700     response_date_ = base::Time();
 701
 702   // Now, loop over the response cookies, and attempt to persist each.
 703   SaveNextCookie();
 704 }
 705
 706 // If the save occurs synchronously, SaveNextCookie will loop and save the next
 707 // cookie. If the save is deferred, the callback is responsible for continuing
 708 // to iterate through the cookies.
 709 // TODO(erikwright): Modify the CookieStore API to indicate via return value
 710 // whether it completed synchronously or asynchronously.
 711 // See http://crbug.com/131066.
 712 void URLRequestHttpJob::SaveNextCookie() {
 713   // No matter what, we want to report our status as IO pending since we will
 714   // be notifying our consumer asynchronously via OnStartCompleted.
 715   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
 716
 717   // Used to communicate with the callback. See the implementation of
 718   // OnCookieSaved.
 719   scoped_refptr<SharedBoolean> callback_pending = new SharedBoolean(false);
 720   scoped_refptr<SharedBoolean> save_next_cookie_running =
 721       new SharedBoolean(true);
 722
 723   if (!(request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) &&
 724       request_->context()->cookie_store() && response_cookies_.size() > 0) {
 725     CookieOptions options;
 726     options.set_include_httponly();
 727     options.set_server_time(response_date_);
 728
 729     CookieStore::SetCookiesCallback callback(base::Bind(
 730         &URLRequestHttpJob::OnCookieSaved, weak_factory_.GetWeakPtr(),
 731         save_next_cookie_running, callback_pending));
 732
 733     // Loop through the cookies as long as SetCookieWithOptionsAsync completes
 734     // synchronously.
 735     while (!callback_pending->data &&
 736            response_cookies_save_index_ < response_cookies_.size()) {
 737       if (CanSetCookie(
 738           response_cookies_[response_cookies_save_index_], &options)) {
 739         callback_pending->data = true;
 740         request_->context()->cookie_store()->SetCookieWithOptionsAsync(
 741             request_->url(), response_cookies_[response_cookies_save_index_],
 742             options, callback);
 743       }
 744       ++response_cookies_save_index_;
 745     }
 746   }
 747
 748   save_next_cookie_running->data = false;
 749
 750   if (!callback_pending->data) {
 751     response_cookies_.clear();
 752     response_cookies_save_index_ = 0;
 753     SetStatus(URLRequestStatus());  // Clear the IO_PENDING status
 754     NotifyHeadersComplete();
 755     return;
 756   }
 757 }
 758
 759 // |save_next_cookie_running| is true when the callback is bound and set to
 760 // false when SaveNextCookie exits, allowing the callback to determine if the
 761 // save occurred synchronously or asynchronously.
 762 // |callback_pending| is false when the callback is invoked and will be set to
 763 // true by the callback, allowing SaveNextCookie to detect whether the save
 764 // occurred synchronously.
 765 // See SaveNextCookie() for more information.
 766 void URLRequestHttpJob::OnCookieSaved(
 767     scoped_refptr<SharedBoolean> save_next_cookie_running,
 768     scoped_refptr<SharedBoolean> callback_pending,
 769     bool cookie_status) {
 770   callback_pending->data = false;
 771
 772   // If we were called synchronously, return.
 773   if (save_next_cookie_running->data) {
 774     return;
 775   }
 776
 777   // We were called asynchronously, so trigger the next save.
 778   // We may have been canceled within OnSetCookie.
 779   if (GetStatus().is_success()) {
 780     SaveNextCookie();
 781   } else {
 782     NotifyCanceled();
 783   }
 784 }
 785
 786 void URLRequestHttpJob::FetchResponseCookies(
 787     std::vector<std::string>* cookies) {
 788   const std::string name = "Set-Cookie";
 789   std::string value;
 790
 791   void* iter = NULL;
 792   HttpResponseHeaders* headers = GetResponseHeaders();
 793   while (headers->EnumerateHeader(&iter, name, &value)) {
 794     if (!value.empty())
 795       cookies->push_back(value);
 796   }
 797 }
 798
 799 // NOTE: |ProcessStrictTransportSecurityHeader| and
 800 // |ProcessPublicKeyPinsHeader| have very similar structures, by design.
 801 void URLRequestHttpJob::ProcessStrictTransportSecurityHeader() {
 802   DCHECK(response_info_);
 803   TransportSecurityState* security_state =
 804       request_->context()->transport_security_state();
 805   const SSLInfo& ssl_info = response_info_->ssl_info;
 806
 807   // Only accept HSTS headers on HTTPS connections that have no
 808   // certificate errors.
 809   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 810       !security_state)
 811     return;
 812
 813   // Don't accept HSTS headers when the hostname is an IP address.
 814   if (request_info_.url.HostIsIPAddress())
 815     return;
 816
 817   // http://tools.ietf.org/html/draft-ietf-websec-strict-transport-sec:
 818   //
 819   //   If a UA receives more than one STS header field in a HTTP response
 820   //   message over secure transport, then the UA MUST process only the
 821   //   first such header field.
 822   HttpResponseHeaders* headers = GetResponseHeaders();
 823   std::string value;
 824   if (headers->EnumerateHeader(NULL, "Strict-Transport-Security", &value))
 825     security_state->AddHSTSHeader(request_info_.url.host(), value);
 826 }
 827
 828 void URLRequestHttpJob::ProcessPublicKeyPinsHeader() {
 829   DCHECK(response_info_);
 830   TransportSecurityState* security_state =
 831       request_->context()->transport_security_state();
 832   const SSLInfo& ssl_info = response_info_->ssl_info;
 833
 834   // Only accept HPKP headers on HTTPS connections that have no
 835   // certificate errors.
 836   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 837       !security_state)
 838     return;
 839
 840   // Don't accept HSTS headers when the hostname is an IP address.
 841   if (request_info_.url.HostIsIPAddress())
 842     return;
 843
 844   // http://tools.ietf.org/html/draft-ietf-websec-key-pinning:
 845   //
 846   //   If a UA receives more than one PKP header field in an HTTP
 847   //   response message over secure transport, then the UA MUST process
 848   //   only the first such header field.
 849   HttpResponseHeaders* headers = GetResponseHeaders();
 850   std::string value;
 851   if (headers->EnumerateHeader(NULL, "Public-Key-Pins", &value))
 852     security_state->AddHPKPHeader(request_info_.url.host(), value, ssl_info);
 853 }
 854
 855 void URLRequestHttpJob::OnStartCompleted(int result) {
 856   RecordTimer();
 857
 858   // If the request was destroyed, then there is no more work to do.
 859   if (!request_)
 860     return;
 861
 862   // If the job is done (due to cancellation), can just ignore this
 863   // notification.
 864   if (done_)
 865     return;
 866
 867   receive_headers_end_ = base::TimeTicks::Now();
 868
 869   // Clear the IO_PENDING status
 870   SetStatus(URLRequestStatus());
 871
 872   const URLRequestContext* context = request_->context();
 873
 874   if (result == ERR_SSL_PINNED_KEY_NOT_IN_CERT_CHAIN &&
 875       transaction_->GetResponseInfo() != NULL) {
 876     FraudulentCertificateReporter* reporter =
 877       context->fraudulent_certificate_reporter();
 878     if (reporter != NULL) {
 879       const SSLInfo& ssl_info = transaction_->GetResponseInfo()->ssl_info;
 880       const std::string& host = request_->url().host();
 881
 882       reporter->SendReport(host, ssl_info);
 883     }
 884   }
 885
 886   if (result == OK) {
 887     if (transaction_ && transaction_->GetResponseInfo()) {
 888       SetProxyServer(transaction_->GetResponseInfo()->proxy_server);
 889     }
 890     scoped_refptr<HttpResponseHeaders> headers = GetResponseHeaders();
 891     if (network_delegate()) {
 892       // Note that |this| may not be deleted until
 893       // |on_headers_received_callback_| or
 894       // |NetworkDelegate::URLRequestDestroyed()| has been called.
 895       OnCallToDelegate();
 896       allowed_unsafe_redirect_url_ = GURL();
 897       int error = network_delegate()->NotifyHeadersReceived(
 898           request_,
 899           on_headers_received_callback_,
 900           headers.get(),
 901           &override_response_headers_,
 902           &allowed_unsafe_redirect_url_);
 903       if (error != OK) {
 904         if (error == ERR_IO_PENDING) {
 905           awaiting_callback_ = true;
 906         } else {
 907           std::string source("delegate");
 908           request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 909                                        NetLog::StringCallback("source",
 910                                                               &source));
 911           OnCallToDelegateComplete();
 912           NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, error));
 913         }
 914         return;
 915       }
 916     }
 917
 918     SaveCookiesAndNotifyHeadersComplete(OK);
 919   } else if (IsCertificateError(result)) {
 920     // We encountered an SSL certificate error.
 921     if (result == ERR_SSL_WEAK_SERVER_EPHEMERAL_DH_KEY ||
 922         result == ERR_SSL_PINNED_KEY_NOT_IN_CERT_CHAIN) {
 923       // These are hard failures. They're handled separately and don't have
 924       // the correct cert status, so set it here.
 925       SSLInfo info(transaction_->GetResponseInfo()->ssl_info);
 926       info.cert_status = MapNetErrorToCertStatus(result);
 927       NotifySSLCertificateError(info, true);
 928     } else {
 929       // Maybe overridable, maybe not. Ask the delegate to decide.
 930       TransportSecurityState* state = context->transport_security_state();
 931       const bool fatal =
 932           state && state->ShouldSSLErrorsBeFatal(request_info_.url.host());
 933       NotifySSLCertificateError(
 934           transaction_->GetResponseInfo()->ssl_info, fatal);
 935     }
 936   } else if (result == ERR_SSL_CLIENT_AUTH_CERT_NEEDED) {
 937     NotifyCertificateRequested(
 938         transaction_->GetResponseInfo()->cert_request_info.get());
 939   } else {
 940     // Even on an error, there may be useful information in the response
 941     // info (e.g. whether there's a cached copy).
 942     if (transaction_.get())
 943       response_info_ = transaction_->GetResponseInfo();
 944     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 945   }
 946 }
 947
 948 void URLRequestHttpJob::OnHeadersReceivedCallback(int result) {
 949   awaiting_callback_ = false;
 950
 951   // Check that there are no callbacks to already canceled requests.
 952   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 953
 954   SaveCookiesAndNotifyHeadersComplete(result);
 955 }
 956
 957 void URLRequestHttpJob::OnReadCompleted(int result) {
 958   read_in_progress_ = false;
 959
 960   if (ShouldFixMismatchedContentLength(result))
 961     result = OK;
 962
 963   if (result == OK) {
 964     NotifyDone(URLRequestStatus());
 965   } else if (result < 0) {
 966     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, result));
 967   } else {
 968     // Clear the IO_PENDING status
 969     SetStatus(URLRequestStatus());
 970   }
 971
 972   NotifyReadComplete(result);
 973 }
 974
 975 void URLRequestHttpJob::RestartTransactionWithAuth(
 976     const AuthCredentials& credentials) {
 977   auth_credentials_ = credentials;
 978
 979   // These will be reset in OnStartCompleted.
 980   response_info_ = NULL;
 981   receive_headers_end_ = base::TimeTicks();
 982   response_cookies_.clear();
 983
 984   ResetTimer();
 985
 986   // Update the cookies, since the cookie store may have been updated from the
 987   // headers in the 401/407. Since cookies were already appended to
 988   // extra_headers, we need to strip them out before adding them again.
 989   request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kCookie);
 990
 991   AddCookieHeaderAndStart();
 992 }
 993
 994 void URLRequestHttpJob::SetUpload(UploadDataStream* upload) {
 995   DCHECK(!transaction_.get()) << "cannot change once started";
 996   request_info_.upload_data_stream = upload;
 997 }
 998
 999 void URLRequestHttpJob::SetExtraRequestHeaders(
1000     const HttpRequestHeaders& headers) {
1001   DCHECK(!transaction_.get()) << "cannot change once started";
1002   request_info_.extra_headers.CopyFrom(headers);
1003 }
1004
1005 LoadState URLRequestHttpJob::GetLoadState() const {
1006   return transaction_.get() ?
1007       transaction_->GetLoadState() : LOAD_STATE_IDLE;
1008 }
1009
1010 UploadProgress URLRequestHttpJob::GetUploadProgress() const {
1011   return transaction_.get() ?
1012       transaction_->GetUploadProgress() : UploadProgress();
1013 }
1014
1015 bool URLRequestHttpJob::GetMimeType(std::string* mime_type) const {
1016   DCHECK(transaction_.get());
1017
1018   if (!response_info_)
1019     return false;
1020
1021   HttpResponseHeaders* headers = GetResponseHeaders();
1022   if (!headers)
1023     return false;
1024   return headers->GetMimeType(mime_type);
1025 }
1026
1027 bool URLRequestHttpJob::GetCharset(std::string* charset) {
1028   DCHECK(transaction_.get());
1029
1030   if (!response_info_)
1031     return false;
1032
1033   return GetResponseHeaders()->GetCharset(charset);
1034 }
1035
1036 void URLRequestHttpJob::GetResponseInfo(HttpResponseInfo* info) {
1037   DCHECK(request_);
1038
1039   if (response_info_) {
1040     DCHECK(transaction_.get());
1041
1042     *info = *response_info_;
1043     if (override_response_headers_.get())
1044       info->headers = override_response_headers_;
1045   }
1046 }
1047
1048 void URLRequestHttpJob::GetLoadTimingInfo(
1049     LoadTimingInfo* load_timing_info) const {
1050   // If haven't made it far enough to receive any headers, don't return
1051   // anything. This makes for more consistent behavior in the case of errors.
1052   if (!transaction_ || receive_headers_end_.is_null())
1053     return;
1054   if (transaction_->GetLoadTimingInfo(load_timing_info))
1055     load_timing_info->receive_headers_end = receive_headers_end_;
1056 }
1057
1058 bool URLRequestHttpJob::GetResponseCookies(std::vector<std::string>* cookies) {
1059   DCHECK(transaction_.get());
1060
1061   if (!response_info_)
1062     return false;
1063
1064   // TODO(darin): Why are we extracting response cookies again?  Perhaps we
1065   // should just leverage response_cookies_.
1066
1067   cookies->clear();
1068   FetchResponseCookies(cookies);
1069   return true;
1070 }
1071
1072 int URLRequestHttpJob::GetResponseCode() const {
1073   DCHECK(transaction_.get());
1074
1075   if (!response_info_)
1076     return -1;
1077
1078   return GetResponseHeaders()->response_code();
1079 }
1080
1081 Filter* URLRequestHttpJob::SetupFilter() const {
1082   DCHECK(transaction_.get());
1083   if (!response_info_)
1084     return NULL;
1085
1086   std::vector<Filter::FilterType> encoding_types;
1087   std::string encoding_type;
1088   HttpResponseHeaders* headers = GetResponseHeaders();
1089   void* iter = NULL;
1090   while (headers->EnumerateHeader(&iter, "Content-Encoding", &encoding_type)) {
1091     encoding_types.push_back(Filter::ConvertEncodingToType(encoding_type));
1092   }
1093
1094   // Even if encoding types are empty, there is a chance that we need to add
1095   // some decoding, as some proxies strip encoding completely. In such cases,
1096   // we may need to add (for example) SDCH filtering (when the context suggests
1097   // it is appropriate).
1098   Filter::FixupEncodingTypes(*filter_context_, &encoding_types);
1099
1100   return !encoding_types.empty()
1101       ? Filter::Factory(encoding_types, *filter_context_) : NULL;
1102 }
1103
1104 bool URLRequestHttpJob::CopyFragmentOnRedirect(const GURL& location) const {
1105   // Allow modification of reference fragments by default, unless
1106   // |allowed_unsafe_redirect_url_| is set and equal to the redirect URL.
1107   // When this is the case, we assume that the network delegate has set the
1108   // desired redirect URL (with or without fragment), so it must not be changed
1109   // any more.
1110   return !allowed_unsafe_redirect_url_.is_valid() ||
1111        allowed_unsafe_redirect_url_ != location;
1112 }
1113
1114 bool URLRequestHttpJob::IsSafeRedirect(const GURL& location) {
1115   // HTTP is always safe.
1116   // TODO(pauljensen): Remove once crbug.com/146591 is fixed.
1117   if (location.is_valid() &&
1118       (location.scheme() == "http" || location.scheme() == "https")) {
1119     return true;
1120   }
1121   // Delegates may mark a URL as safe for redirection.
1122   if (allowed_unsafe_redirect_url_.is_valid() &&
1123       allowed_unsafe_redirect_url_ == location) {
1124     return true;
1125   }
1126   // Query URLRequestJobFactory as to whether |location| would be safe to
1127   // redirect to.
1128   return request_->context()->job_factory() &&
1129       request_->context()->job_factory()->IsSafeRedirectTarget(location);
1130 }
1131
1132 bool URLRequestHttpJob::NeedsAuth() {
1133   int code = GetResponseCode();
1134   if (code == -1)
1135     return false;
1136
1137   // Check if we need either Proxy or WWW Authentication. This could happen
1138   // because we either provided no auth info, or provided incorrect info.
1139   switch (code) {
1140     case 407:
1141       if (proxy_auth_state_ == AUTH_STATE_CANCELED)
1142         return false;
1143       proxy_auth_state_ = AUTH_STATE_NEED_AUTH;
1144       return true;
1145     case 401:
1146       if (server_auth_state_ == AUTH_STATE_CANCELED)
1147         return false;
1148       server_auth_state_ = AUTH_STATE_NEED_AUTH;
1149       return true;
1150   }
1151   return false;
1152 }
1153
1154 void URLRequestHttpJob::GetAuthChallengeInfo(
1155     scoped_refptr<AuthChallengeInfo>* result) {
1156   DCHECK(transaction_.get());
1157   DCHECK(response_info_);
1158
1159   // sanity checks:
1160   DCHECK(proxy_auth_state_ == AUTH_STATE_NEED_AUTH ||
1161          server_auth_state_ == AUTH_STATE_NEED_AUTH);
1162   DCHECK((GetResponseHeaders()->response_code() == HTTP_UNAUTHORIZED) ||
1163          (GetResponseHeaders()->response_code() ==
1164           HTTP_PROXY_AUTHENTICATION_REQUIRED));
1165
1166   *result = response_info_->auth_challenge;
1167 }
1168
1169 void URLRequestHttpJob::SetAuth(const AuthCredentials& credentials) {
1170   DCHECK(transaction_.get());
1171
1172   // Proxy gets set first, then WWW.
1173   if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1174     proxy_auth_state_ = AUTH_STATE_HAVE_AUTH;
1175   } else {
1176     DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1177     server_auth_state_ = AUTH_STATE_HAVE_AUTH;
1178   }
1179
1180   RestartTransactionWithAuth(credentials);
1181 }
1182
1183 void URLRequestHttpJob::CancelAuth() {
1184   // Proxy gets set first, then WWW.
1185   if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1186     proxy_auth_state_ = AUTH_STATE_CANCELED;
1187   } else {
1188     DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1189     server_auth_state_ = AUTH_STATE_CANCELED;
1190   }
1191
1192   // These will be reset in OnStartCompleted.
1193   response_info_ = NULL;
1194   receive_headers_end_ = base::TimeTicks::Now();
1195   response_cookies_.clear();
1196
1197   ResetTimer();
1198
1199   // OK, let the consumer read the error page...
1200   //
1201   // Because we set the AUTH_STATE_CANCELED flag, NeedsAuth will return false,
1202   // which will cause the consumer to receive OnResponseStarted instead of
1203   // OnAuthRequired.
1204   //
1205   // We have to do this via InvokeLater to avoid "recursing" the consumer.
1206   //
1207   base::MessageLoop::current()->PostTask(
1208       FROM_HERE,
1209       base::Bind(&URLRequestHttpJob::OnStartCompleted,
1210                  weak_factory_.GetWeakPtr(), OK));
1211 }
1212
1213 void URLRequestHttpJob::ContinueWithCertificate(
1214     X509Certificate* client_cert) {
1215   DCHECK(transaction_.get());
1216
1217   DCHECK(!response_info_) << "should not have a response yet";
1218   receive_headers_end_ = base::TimeTicks();
1219
1220   ResetTimer();
1221
1222   // No matter what, we want to report our status as IO pending since we will
1223   // be notifying our consumer asynchronously via OnStartCompleted.
1224   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1225
1226   int rv = transaction_->RestartWithCertificate(client_cert, start_callback_);
1227   if (rv == ERR_IO_PENDING)
1228     return;
1229
1230   // The transaction started synchronously, but we need to notify the
1231   // URLRequest delegate via the message loop.
1232   base::MessageLoop::current()->PostTask(
1233       FROM_HERE,
1234       base::Bind(&URLRequestHttpJob::OnStartCompleted,
1235                  weak_factory_.GetWeakPtr(), rv));
1236 }
1237
1238 void URLRequestHttpJob::ContinueDespiteLastError() {
1239   // If the transaction was destroyed, then the job was cancelled.
1240   if (!transaction_.get())
1241     return;
1242
1243   DCHECK(!response_info_) << "should not have a response yet";
1244   receive_headers_end_ = base::TimeTicks();
1245
1246   ResetTimer();
1247
1248   // No matter what, we want to report our status as IO pending since we will
1249   // be notifying our consumer asynchronously via OnStartCompleted.
1250   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1251
1252   int rv = transaction_->RestartIgnoringLastError(start_callback_);
1253   if (rv == ERR_IO_PENDING)
1254     return;
1255
1256   // The transaction started synchronously, but we need to notify the
1257   // URLRequest delegate via the message loop.
1258   base::MessageLoop::current()->PostTask(
1259       FROM_HERE,
1260       base::Bind(&URLRequestHttpJob::OnStartCompleted,
1261                  weak_factory_.GetWeakPtr(), rv));
1262 }
1263
1264 void URLRequestHttpJob::ResumeNetworkStart() {
1265   DCHECK(transaction_.get());
1266   transaction_->ResumeNetworkStart();
1267 }
1268
1269 bool URLRequestHttpJob::ShouldFixMismatchedContentLength(int rv) const {
1270   // Some servers send the body compressed, but specify the content length as
1271   // the uncompressed size. Although this violates the HTTP spec we want to
1272   // support it (as IE and FireFox do), but *only* for an exact match.
1273   // See http://crbug.com/79694.
1274   if (rv == ERR_CONTENT_LENGTH_MISMATCH ||
1275       rv == ERR_INCOMPLETE_CHUNKED_ENCODING) {
1276     if (request_ && request_->response_headers()) {
1277       int64 expected_length = request_->response_headers()->GetContentLength();
1278       VLOG(1) << __FUNCTION__ << "() "
1279               << "\"" << request_->url().spec() << "\""
1280               << " content-length = " << expected_length
1281               << " pre total = " << prefilter_bytes_read()
1282               << " post total = " << postfilter_bytes_read();
1283       if (postfilter_bytes_read() == expected_length) {
1284         // Clear the error.
1285         return true;
1286       }
1287     }
1288   }
1289   return false;
1290 }
1291
1292 bool URLRequestHttpJob::ReadRawData(IOBuffer* buf, int buf_size,
1293                                     int* bytes_read) {
1294   DCHECK_NE(buf_size, 0);
1295   DCHECK(bytes_read);
1296   DCHECK(!read_in_progress_);
1297
1298   int rv = transaction_->Read(
1299       buf, buf_size,
1300       base::Bind(&URLRequestHttpJob::OnReadCompleted, base::Unretained(this)));
1301
1302   if (ShouldFixMismatchedContentLength(rv))
1303     rv = 0;
1304
1305   if (rv >= 0) {
1306     *bytes_read = rv;
1307     if (!rv)
1308       DoneWithRequest(FINISHED);
1309     return true;
1310   }
1311
1312   if (rv == ERR_IO_PENDING) {
1313     read_in_progress_ = true;
1314     SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1315   } else {
1316     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, rv));
1317   }
1318
1319   return false;
1320 }
1321
1322 void URLRequestHttpJob::StopCaching() {
1323   if (transaction_.get())
1324     transaction_->StopCaching();
1325 }
1326
1327 bool URLRequestHttpJob::GetFullRequestHeaders(
1328     HttpRequestHeaders* headers) const {
1329   if (!transaction_)
1330     return false;
1331
1332   return transaction_->GetFullRequestHeaders(headers);
1333 }
1334
1335 int64 URLRequestHttpJob::GetTotalReceivedBytes() const {
1336   if (!transaction_)
1337     return 0;
1338
1339   return transaction_->GetTotalReceivedBytes();
1340 }
1341
1342 void URLRequestHttpJob::DoneReading() {
1343   if (transaction_) {
1344     transaction_->DoneReading();
1345   }
1346   DoneWithRequest(FINISHED);
1347 }
1348
1349 void URLRequestHttpJob::DoneReadingRedirectResponse() {
1350   if (transaction_) {
1351     if (transaction_->GetResponseInfo()->headers->IsRedirect(NULL)) {
1352       // If the original headers indicate a redirect, go ahead and cache the
1353       // response, even if the |override_response_headers_| are a redirect to
1354       // another location.
1355       transaction_->DoneReading();
1356     } else {
1357       // Otherwise, |override_response_headers_| must be non-NULL and contain
1358       // bogus headers indicating a redirect.
1359       DCHECK(override_response_headers_.get());
1360       DCHECK(override_response_headers_->IsRedirect(NULL));
1361       transaction_->StopCaching();
1362     }
1363   }
1364   DoneWithRequest(FINISHED);
1365 }
1366
1367 HostPortPair URLRequestHttpJob::GetSocketAddress() const {
1368   return response_info_ ? response_info_->socket_address : HostPortPair();
1369 }
1370
1371 void URLRequestHttpJob::RecordTimer() {
1372   if (request_creation_time_.is_null()) {
1373     NOTREACHED()
1374         << "The same transaction shouldn't start twice without new timing.";
1375     return;
1376   }
1377
1378   base::TimeDelta to_start = base::Time::Now() - request_creation_time_;
1379   request_creation_time_ = base::Time();
1380
1381   UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpTimeToFirstByte", to_start);
1382 }
1383
1384 void URLRequestHttpJob::ResetTimer() {
1385   if (!request_creation_time_.is_null()) {
1386     NOTREACHED()
1387         << "The timer was reset before it was recorded.";
1388     return;
1389   }
1390   request_creation_time_ = base::Time::Now();
1391 }
1392
1393 void URLRequestHttpJob::UpdatePacketReadTimes() {
1394   if (!packet_timing_enabled_)
1395     return;
1396
1397   DCHECK_GT(prefilter_bytes_read(), bytes_observed_in_packets_);
1398
1399   base::Time now(base::Time::Now());
1400   if (!bytes_observed_in_packets_)
1401     request_time_snapshot_ = now;
1402   final_packet_time_ = now;
1403
1404   bytes_observed_in_packets_ = prefilter_bytes_read();
1405 }
1406
1407 void URLRequestHttpJob::RecordPacketStats(
1408     FilterContext::StatisticSelector statistic) const {
1409   if (!packet_timing_enabled_ || (final_packet_time_ == base::Time()))
1410     return;
1411
1412   base::TimeDelta duration = final_packet_time_ - request_time_snapshot_;
1413   switch (statistic) {
1414     case FilterContext::SDCH_DECODE: {
1415       UMA_HISTOGRAM_CUSTOM_COUNTS("Sdch3.Network_Decode_Bytes_Processed_b",
1416           static_cast<int>(bytes_observed_in_packets_), 500, 100000, 100);
1417       return;
1418     }
1419     case FilterContext::SDCH_PASSTHROUGH: {
1420       // Despite advertising a dictionary, we handled non-sdch compressed
1421       // content.
1422       return;
1423     }
1424
1425     case FilterContext::SDCH_EXPERIMENT_DECODE: {
1426       UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment3_Decode",
1427                                   duration,
1428                                   base::TimeDelta::FromMilliseconds(20),
1429                                   base::TimeDelta::FromMinutes(10), 100);
1430       return;
1431     }
1432     case FilterContext::SDCH_EXPERIMENT_HOLDBACK: {
1433       UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment3_Holdback",
1434                                   duration,
1435                                   base::TimeDelta::FromMilliseconds(20),
1436                                   base::TimeDelta::FromMinutes(10), 100);
1437       return;
1438     }
1439     default:
1440       NOTREACHED();
1441       return;
1442   }
1443 }
1444
1445 void URLRequestHttpJob::RecordPerfHistograms(CompletionCause reason) {
1446   if (start_time_.is_null())
1447     return;
1448
1449   base::TimeDelta total_time = base::TimeTicks::Now() - start_time_;
1450   UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTime", total_time);
1451
1452   if (reason == FINISHED) {
1453     UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeSuccess", total_time);
1454   } else {
1455     UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCancel", total_time);
1456   }
1457
1458   if (response_info_) {
1459     bool is_google = request() && HasGoogleHost(request()->url());
1460     bool used_quic = response_info_->DidUseQuic();
1461     if (is_google) {
1462       if (used_quic) {
1463         UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTime.Quic", total_time);
1464       } else {
1465         UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTime.NotQuic", total_time);
1466       }
1467     }
1468     if (response_info_->was_cached) {
1469       UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCached", total_time);
1470       if (is_google) {
1471         if (used_quic) {
1472           UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTimeCached.Quic",
1473                                      total_time);
1474         } else {
1475           UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTimeCached.NotQuic",
1476                                      total_time);
1477         }
1478       }
1479     } else  {
1480       UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeNotCached", total_time);
1481       if (is_google) {
1482         if (used_quic) {
1483           UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTimeNotCached.Quic",
1484                                      total_time);
1485         } else {
1486           UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTimeNotCached.NotQuic",
1487                                      total_time);
1488         }
1489       }
1490     }
1491   }
1492
1493   if (request_info_.load_flags & LOAD_PREFETCH && !request_->was_cached())
1494     UMA_HISTOGRAM_COUNTS("Net.Prefetch.PrefilterBytesReadFromNetwork",
1495                          prefilter_bytes_read());
1496
1497   start_time_ = base::TimeTicks();
1498 }
1499
1500 void URLRequestHttpJob::DoneWithRequest(CompletionCause reason) {
1501   if (done_)
1502     return;
1503   done_ = true;
1504   RecordPerfHistograms(reason);
1505   if (reason == FINISHED) {
1506     request_->set_received_response_content_length(prefilter_bytes_read());
1507   }
1508 }
1509
1510 HttpResponseHeaders* URLRequestHttpJob::GetResponseHeaders() const {
1511   DCHECK(transaction_.get());
1512   DCHECK(transaction_->GetResponseInfo());
1513   return override_response_headers_.get() ?
1514              override_response_headers_.get() :
1515              transaction_->GetResponseInfo()->headers.get();
1516 }
1517
1518 void URLRequestHttpJob::NotifyURLRequestDestroyed() {
1519   awaiting_callback_ = false;
1520 }
1521
1522 }  // namespace net