net/url_request/url_request_http_job.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_request_http_job.h"
   6
   7 #include "base/base_switches.h"
   8 #include "base/bind.h"
   9 #include "base/bind_helpers.h"
  10 #include "base/command_line.h"
  11 #include "base/compiler_specific.h"
  12 #include "base/file_version_info.h"
  13 #include "base/message_loop/message_loop.h"
  14 #include "base/metrics/field_trial.h"
  15 #include "base/metrics/histogram.h"
  16 #include "base/profiler/scoped_tracker.h"
  17 #include "base/rand_util.h"
  18 #include "base/strings/string_util.h"
  19 #include "base/time/time.h"
  20 #include "net/base/host_port_pair.h"
  21 #include "net/base/load_flags.h"
  22 #include "net/base/net_errors.h"
  23 #include "net/base/net_util.h"
  24 #include "net/base/network_delegate.h"
  25 #include "net/base/sdch_manager.h"
  26 #include "net/base/sdch_net_log_params.h"
  27 #include "net/cert/cert_status_flags.h"
  28 #include "net/cookies/cookie_store.h"
  29 #include "net/http/http_content_disposition.h"
  30 #include "net/http/http_network_session.h"
  31 #include "net/http/http_request_headers.h"
  32 #include "net/http/http_response_headers.h"
  33 #include "net/http/http_response_info.h"
  34 #include "net/http/http_status_code.h"
  35 #include "net/http/http_transaction.h"
  36 #include "net/http/http_transaction_factory.h"
  37 #include "net/http/http_util.h"
  38 #include "net/proxy/proxy_info.h"
  39 #include "net/ssl/ssl_cert_request_info.h"
  40 #include "net/ssl/ssl_config_service.h"
  41 #include "net/url_request/fraudulent_certificate_reporter.h"
  42 #include "net/url_request/http_user_agent_settings.h"
  43 #include "net/url_request/url_request.h"
  44 #include "net/url_request/url_request_context.h"
  45 #include "net/url_request/url_request_error_job.h"
  46 #include "net/url_request/url_request_job_factory.h"
  47 #include "net/url_request/url_request_redirect_job.h"
  48 #include "net/url_request/url_request_throttler_header_adapter.h"
  49 #include "net/url_request/url_request_throttler_manager.h"
  50 #include "net/websockets/websocket_handshake_stream_base.h"
  51
  52 static const char kAvailDictionaryHeader[] = "Avail-Dictionary";
  53
  54 namespace net {
  55
  56 class URLRequestHttpJob::HttpFilterContext : public FilterContext {
  57  public:
  58   explicit HttpFilterContext(URLRequestHttpJob* job);
  59   ~HttpFilterContext() override;
  60
  61   // FilterContext implementation.
  62   bool GetMimeType(std::string* mime_type) const override;
  63   bool GetURL(GURL* gurl) const override;
  64   base::Time GetRequestTime() const override;
  65   bool IsCachedContent() const override;
  66   SdchManager::DictionarySet* SdchDictionariesAdvertised() const override;
  67   int64 GetByteReadCount() const override;
  68   int GetResponseCode() const override;
  69   const URLRequestContext* GetURLRequestContext() const override;
  70   void RecordPacketStats(StatisticSelector statistic) const override;
  71   const BoundNetLog& GetNetLog() const override;
  72
  73  private:
  74   URLRequestHttpJob* job_;
  75
  76   // URLRequestHttpJob may be detached from URLRequest, but we still need to
  77   // return something.
  78   BoundNetLog dummy_log_;
  79
  80   DISALLOW_COPY_AND_ASSIGN(HttpFilterContext);
  81 };
  82
  83 URLRequestHttpJob::HttpFilterContext::HttpFilterContext(URLRequestHttpJob* job)
  84     : job_(job) {
  85   DCHECK(job_);
  86 }
  87
  88 URLRequestHttpJob::HttpFilterContext::~HttpFilterContext() {
  89 }
  90
  91 bool URLRequestHttpJob::HttpFilterContext::GetMimeType(
  92     std::string* mime_type) const {
  93   return job_->GetMimeType(mime_type);
  94 }
  95
  96 bool URLRequestHttpJob::HttpFilterContext::GetURL(GURL* gurl) const {
  97   if (!job_->request())
  98     return false;
  99   *gurl = job_->request()->url();
 100   return true;
 101 }
 102
 103 base::Time URLRequestHttpJob::HttpFilterContext::GetRequestTime() const {
 104   return job_->request() ? job_->request()->request_time() : base::Time();
 105 }
 106
 107 bool URLRequestHttpJob::HttpFilterContext::IsCachedContent() const {
 108   return job_->is_cached_content_;
 109 }
 110
 111 SdchManager::DictionarySet*
 112 URLRequestHttpJob::HttpFilterContext::SdchDictionariesAdvertised() const {
 113   return job_->dictionaries_advertised_.get();
 114 }
 115
 116 int64 URLRequestHttpJob::HttpFilterContext::GetByteReadCount() const {
 117   return job_->prefilter_bytes_read();
 118 }
 119
 120 int URLRequestHttpJob::HttpFilterContext::GetResponseCode() const {
 121   return job_->GetResponseCode();
 122 }
 123
 124 const URLRequestContext*
 125 URLRequestHttpJob::HttpFilterContext::GetURLRequestContext() const {
 126   return job_->request() ? job_->request()->context() : NULL;
 127 }
 128
 129 void URLRequestHttpJob::HttpFilterContext::RecordPacketStats(
 130     StatisticSelector statistic) const {
 131   job_->RecordPacketStats(statistic);
 132 }
 133
 134 const BoundNetLog& URLRequestHttpJob::HttpFilterContext::GetNetLog() const {
 135   return job_->request() ? job_->request()->net_log() : dummy_log_;
 136 }
 137
 138 // TODO(darin): make sure the port blocking code is not lost
 139 // static
 140 URLRequestJob* URLRequestHttpJob::Factory(URLRequest* request,
 141                                           NetworkDelegate* network_delegate,
 142                                           const std::string& scheme) {
 143   DCHECK(scheme == "http" || scheme == "https" || scheme == "ws" ||
 144          scheme == "wss");
 145
 146   if (!request->context()->http_transaction_factory()) {
 147     NOTREACHED() << "requires a valid context";
 148     return new URLRequestErrorJob(
 149         request, network_delegate, ERR_INVALID_ARGUMENT);
 150   }
 151
 152   GURL redirect_url;
 153   if (request->GetHSTSRedirect(&redirect_url)) {
 154     return new URLRequestRedirectJob(
 155         request, network_delegate, redirect_url,
 156         // Use status code 307 to preserve the method, so POST requests work.
 157         URLRequestRedirectJob::REDIRECT_307_TEMPORARY_REDIRECT, "HSTS");
 158   }
 159   return new URLRequestHttpJob(request,
 160                                network_delegate,
 161                                request->context()->http_user_agent_settings());
 162 }
 163
 164 URLRequestHttpJob::URLRequestHttpJob(
 165     URLRequest* request,
 166     NetworkDelegate* network_delegate,
 167     const HttpUserAgentSettings* http_user_agent_settings)
 168     : URLRequestJob(request, network_delegate),
 169       priority_(DEFAULT_PRIORITY),
 170       response_info_(NULL),
 171       response_cookies_save_index_(0),
 172       proxy_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
 173       server_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
 174       start_callback_(base::Bind(&URLRequestHttpJob::OnStartCompleted,
 175                                  base::Unretained(this))),
 176       notify_before_headers_sent_callback_(
 177           base::Bind(&URLRequestHttpJob::NotifyBeforeSendHeadersCallback,
 178                      base::Unretained(this))),
 179       read_in_progress_(false),
 180       throttling_entry_(NULL),
 181       sdch_test_activated_(false),
 182       sdch_test_control_(false),
 183       is_cached_content_(false),
 184       request_creation_time_(),
 185       packet_timing_enabled_(false),
 186       done_(false),
 187       bytes_observed_in_packets_(0),
 188       request_time_snapshot_(),
 189       final_packet_time_(),
 190       filter_context_(new HttpFilterContext(this)),
 191       on_headers_received_callback_(
 192           base::Bind(&URLRequestHttpJob::OnHeadersReceivedCallback,
 193                      base::Unretained(this))),
 194       awaiting_callback_(false),
 195       http_user_agent_settings_(http_user_agent_settings),
 196       weak_factory_(this) {
 197   URLRequestThrottlerManager* manager = request->context()->throttler_manager();
 198   if (manager)
 199     throttling_entry_ = manager->RegisterRequestUrl(request->url());
 200
 201   ResetTimer();
 202 }
 203
 204 URLRequestHttpJob::~URLRequestHttpJob() {
 205   CHECK(!awaiting_callback_);
 206
 207   DCHECK(!sdch_test_control_ || !sdch_test_activated_);
 208   if (!is_cached_content_) {
 209     if (sdch_test_control_)
 210       RecordPacketStats(FilterContext::SDCH_EXPERIMENT_HOLDBACK);
 211     if (sdch_test_activated_)
 212       RecordPacketStats(FilterContext::SDCH_EXPERIMENT_DECODE);
 213   }
 214   // Make sure SDCH filters are told to emit histogram data while
 215   // filter_context_ is still alive.
 216   DestroyFilters();
 217
 218   DoneWithRequest(ABORTED);
 219 }
 220
 221 void URLRequestHttpJob::SetPriority(RequestPriority priority) {
 222   priority_ = priority;
 223   if (transaction_)
 224     transaction_->SetPriority(priority_);
 225 }
 226
 227 void URLRequestHttpJob::Start() {
 228   // TODO(mmenke): Remove ScopedTracker below once crbug.com/456327 is fixed.
 229   tracked_objects::ScopedTracker tracking_profile(
 230       FROM_HERE_WITH_EXPLICIT_FUNCTION("456327 URLRequestHttpJob::Start"));
 231
 232   DCHECK(!transaction_.get());
 233
 234   // URLRequest::SetReferrer ensures that we do not send username and password
 235   // fields in the referrer.
 236   GURL referrer(request_->referrer());
 237
 238   request_info_.url = request_->url();
 239   request_info_.method = request_->method();
 240   request_info_.load_flags = request_->load_flags();
 241   // Enable privacy mode if cookie settings or flags tell us not send or
 242   // save cookies.
 243   bool enable_privacy_mode =
 244       (request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES) ||
 245       (request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) ||
 246       CanEnablePrivacyMode();
 247   // Privacy mode could still be disabled in OnCookiesLoaded if we are going
 248   // to send previously saved cookies.
 249   request_info_.privacy_mode = enable_privacy_mode ?
 250       PRIVACY_MODE_ENABLED : PRIVACY_MODE_DISABLED;
 251
 252   // Strip Referer from request_info_.extra_headers to prevent, e.g., plugins
 253   // from overriding headers that are controlled using other means. Otherwise a
 254   // plugin could set a referrer although sending the referrer is inhibited.
 255   request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kReferer);
 256
 257   // Our consumer should have made sure that this is a safe referrer. See for
 258   // instance WebCore::FrameLoader::HideReferrer.
 259   if (referrer.is_valid()) {
 260     request_info_.extra_headers.SetHeader(HttpRequestHeaders::kReferer,
 261                                           referrer.spec());
 262   }
 263
 264   request_info_.extra_headers.SetHeaderIfMissing(
 265       HttpRequestHeaders::kUserAgent,
 266       http_user_agent_settings_ ?
 267           http_user_agent_settings_->GetUserAgent() : std::string());
 268
 269   AddExtraHeaders();
 270   AddCookieHeaderAndStart();
 271 }
 272
 273 void URLRequestHttpJob::Kill() {
 274   if (!transaction_.get())
 275     return;
 276
 277   weak_factory_.InvalidateWeakPtrs();
 278   DestroyTransaction();
 279   URLRequestJob::Kill();
 280 }
 281
 282 void URLRequestHttpJob::GetConnectionAttempts(ConnectionAttempts* out) const {
 283   if (transaction_)
 284     transaction_->GetConnectionAttempts(out);
 285   else
 286     out->clear();
 287 }
 288
 289 void URLRequestHttpJob::NotifyBeforeSendProxyHeadersCallback(
 290     const ProxyInfo& proxy_info,
 291     HttpRequestHeaders* request_headers) {
 292   DCHECK(request_headers);
 293   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 294   if (network_delegate()) {
 295     network_delegate()->NotifyBeforeSendProxyHeaders(
 296         request_,
 297         proxy_info,
 298         request_headers);
 299   }
 300 }
 301
 302 void URLRequestHttpJob::NotifyHeadersComplete() {
 303   DCHECK(!response_info_);
 304
 305   response_info_ = transaction_->GetResponseInfo();
 306
 307   // Save boolean, as we'll need this info at destruction time, and filters may
 308   // also need this info.
 309   is_cached_content_ = response_info_->was_cached;
 310
 311   if (!is_cached_content_ && throttling_entry_.get()) {
 312     URLRequestThrottlerHeaderAdapter response_adapter(GetResponseHeaders());
 313     throttling_entry_->UpdateWithResponse(request_info_.url.host(),
 314                                           &response_adapter);
 315   }
 316
 317   // The ordering of these calls is not important.
 318   ProcessStrictTransportSecurityHeader();
 319   ProcessPublicKeyPinsHeader();
 320
 321   // Handle the server notification of a new SDCH dictionary.
 322   SdchManager* sdch_manager(request()->context()->sdch_manager());
 323   if (sdch_manager) {
 324     SdchProblemCode rv = sdch_manager->IsInSupportedDomain(request()->url());
 325     if (rv != SDCH_OK) {
 326       // If SDCH is just disabled, it is not a real error.
 327       if (rv != SDCH_DISABLED && rv != SDCH_SECURE_SCHEME_NOT_SUPPORTED) {
 328         SdchManager::SdchErrorRecovery(rv);
 329         request()->net_log().AddEvent(
 330             NetLog::TYPE_SDCH_DECODING_ERROR,
 331             base::Bind(&NetLogSdchResourceProblemCallback, rv));
 332       }
 333     } else {
 334       const std::string name = "Get-Dictionary";
 335       std::string url_text;
 336       void* iter = NULL;
 337       // TODO(jar): We need to not fetch dictionaries the first time they are
 338       // seen, but rather wait until we can justify their usefulness.
 339       // For now, we will only fetch the first dictionary, which will at least
 340       // require multiple suggestions before we get additional ones for this
 341       // site. Eventually we should wait until a dictionary is requested
 342       // several times
 343       // before we even download it (so that we don't waste memory or
 344       // bandwidth).
 345       if (GetResponseHeaders()->EnumerateHeader(&iter, name, &url_text)) {
 346         // Resolve suggested URL relative to request url.
 347         GURL sdch_dictionary_url = request_->url().Resolve(url_text);
 348         if (sdch_dictionary_url.is_valid()) {
 349           rv = sdch_manager->OnGetDictionary(request_->url(),
 350                                              sdch_dictionary_url);
 351           if (rv != SDCH_OK) {
 352             SdchManager::SdchErrorRecovery(rv);
 353             request_->net_log().AddEvent(
 354                 NetLog::TYPE_SDCH_DICTIONARY_ERROR,
 355                 base::Bind(&NetLogSdchDictionaryFetchProblemCallback, rv,
 356                            sdch_dictionary_url, false));
 357           }
 358         }
 359       }
 360     }
 361   }
 362
 363   // Handle the server signalling no SDCH encoding.
 364   if (dictionaries_advertised_) {
 365     // We are wary of proxies that discard or damage SDCH encoding. If a server
 366     // explicitly states that this is not SDCH content, then we can correct our
 367     // assumption that this is an SDCH response, and avoid the need to recover
 368     // as though the content is corrupted (when we discover it is not SDCH
 369     // encoded).
 370     std::string sdch_response_status;
 371     void* iter = NULL;
 372     while (GetResponseHeaders()->EnumerateHeader(&iter, "X-Sdch-Encode",
 373                                                  &sdch_response_status)) {
 374       if (sdch_response_status == "0") {
 375         dictionaries_advertised_.reset();
 376         break;
 377       }
 378     }
 379   }
 380
 381   // The HTTP transaction may be restarted several times for the purposes
 382   // of sending authorization information. Each time it restarts, we get
 383   // notified of the headers completion so that we can update the cookie store.
 384   if (transaction_->IsReadyToRestartForAuth()) {
 385     DCHECK(!response_info_->auth_challenge.get());
 386     // TODO(battre): This breaks the webrequest API for
 387     // URLRequestTestHTTP.BasicAuthWithCookies
 388     // where OnBeforeSendHeaders -> OnSendHeaders -> OnBeforeSendHeaders
 389     // occurs.
 390     RestartTransactionWithAuth(AuthCredentials());
 391     return;
 392   }
 393
 394   URLRequestJob::NotifyHeadersComplete();
 395 }
 396
 397 void URLRequestHttpJob::NotifyDone(const URLRequestStatus& status) {
 398   DoneWithRequest(FINISHED);
 399   URLRequestJob::NotifyDone(status);
 400 }
 401
 402 void URLRequestHttpJob::DestroyTransaction() {
 403   DCHECK(transaction_.get());
 404
 405   DoneWithRequest(ABORTED);
 406   transaction_.reset();
 407   response_info_ = NULL;
 408   receive_headers_end_ = base::TimeTicks();
 409 }
 410
 411 void URLRequestHttpJob::StartTransaction() {
 412   // TODO(mmenke): Remove ScopedTracker below once crbug.com/456327 is fixed.
 413   tracked_objects::ScopedTracker tracking_profile(
 414       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 415           "456327 URLRequestHttpJob::StartTransaction"));
 416
 417   if (network_delegate()) {
 418     OnCallToDelegate();
 419     int rv = network_delegate()->NotifyBeforeSendHeaders(
 420         request_, notify_before_headers_sent_callback_,
 421         &request_info_.extra_headers);
 422     // If an extension blocks the request, we rely on the callback to
 423     // MaybeStartTransactionInternal().
 424     if (rv == ERR_IO_PENDING)
 425       return;
 426     MaybeStartTransactionInternal(rv);
 427     return;
 428   }
 429   StartTransactionInternal();
 430 }
 431
 432 void URLRequestHttpJob::NotifyBeforeSendHeadersCallback(int result) {
 433   // Check that there are no callbacks to already canceled requests.
 434   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 435
 436   MaybeStartTransactionInternal(result);
 437 }
 438
 439 void URLRequestHttpJob::MaybeStartTransactionInternal(int result) {
 440   // TODO(mmenke): Remove ScopedTracker below once crbug.com/456327 is fixed.
 441   tracked_objects::ScopedTracker tracking_profile(
 442       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 443           "456327 URLRequestHttpJob::MaybeStartTransactionInternal"));
 444
 445   OnCallToDelegateComplete();
 446   if (result == OK) {
 447     StartTransactionInternal();
 448   } else {
 449     std::string source("delegate");
 450     request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 451                                  NetLog::StringCallback("source", &source));
 452     NotifyCanceled();
 453     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 454   }
 455 }
 456
 457 void URLRequestHttpJob::StartTransactionInternal() {
 458   // NOTE: This method assumes that request_info_ is already setup properly.
 459
 460   // If we already have a transaction, then we should restart the transaction
 461   // with auth provided by auth_credentials_.
 462
 463   int rv;
 464
 465   if (network_delegate()) {
 466     network_delegate()->NotifySendHeaders(
 467         request_, request_info_.extra_headers);
 468   }
 469
 470   if (transaction_.get()) {
 471     rv = transaction_->RestartWithAuth(auth_credentials_, start_callback_);
 472     auth_credentials_ = AuthCredentials();
 473   } else {
 474     DCHECK(request_->context()->http_transaction_factory());
 475
 476     rv = request_->context()->http_transaction_factory()->CreateTransaction(
 477         priority_, &transaction_);
 478
 479     if (rv == OK && request_info_.url.SchemeIsWSOrWSS()) {
 480       base::SupportsUserData::Data* data = request_->GetUserData(
 481           WebSocketHandshakeStreamBase::CreateHelper::DataKey());
 482       if (data) {
 483         transaction_->SetWebSocketHandshakeStreamCreateHelper(
 484             static_cast<WebSocketHandshakeStreamBase::CreateHelper*>(data));
 485       } else {
 486         rv = ERR_DISALLOWED_URL_SCHEME;
 487       }
 488     }
 489
 490     if (rv == OK) {
 491       transaction_->SetBeforeNetworkStartCallback(
 492           base::Bind(&URLRequestHttpJob::NotifyBeforeNetworkStart,
 493                      base::Unretained(this)));
 494       transaction_->SetBeforeProxyHeadersSentCallback(
 495           base::Bind(&URLRequestHttpJob::NotifyBeforeSendProxyHeadersCallback,
 496                      base::Unretained(this)));
 497
 498       if (!throttling_entry_.get() ||
 499           !throttling_entry_->ShouldRejectRequest(*request_,
 500                                                   network_delegate())) {
 501         rv = transaction_->Start(
 502             &request_info_, start_callback_, request_->net_log());
 503         start_time_ = base::TimeTicks::Now();
 504       } else {
 505         // Special error code for the exponential back-off module.
 506         rv = ERR_TEMPORARILY_THROTTLED;
 507       }
 508     }
 509   }
 510
 511   if (rv == ERR_IO_PENDING)
 512     return;
 513
 514   // The transaction started synchronously, but we need to notify the
 515   // URLRequest delegate via the message loop.
 516   base::MessageLoop::current()->PostTask(
 517       FROM_HERE,
 518       base::Bind(&URLRequestHttpJob::OnStartCompleted,
 519                  weak_factory_.GetWeakPtr(), rv));
 520 }
 521
 522 void URLRequestHttpJob::AddExtraHeaders() {
 523   SdchManager* sdch_manager = request()->context()->sdch_manager();
 524
 525   // Supply Accept-Encoding field only if it is not already provided.
 526   // It should be provided IF the content is known to have restrictions on
 527   // potential encoding, such as streaming multi-media.
 528   // For details see bug 47381.
 529   // TODO(jar, enal): jpeg files etc. should set up a request header if
 530   // possible. Right now it is done only by buffered_resource_loader and
 531   // simple_data_source.
 532   if (!request_info_.extra_headers.HasHeader(
 533       HttpRequestHeaders::kAcceptEncoding)) {
 534     // We don't support SDCH responses to POST as there is a possibility
 535     // of having SDCH encoded responses returned (e.g. by the cache)
 536     // which we cannot decode, and in those situations, we will need
 537     // to retransmit the request without SDCH, which is illegal for a POST.
 538     bool advertise_sdch = sdch_manager != NULL && request()->method() != "POST";
 539     if (advertise_sdch) {
 540       SdchProblemCode rv = sdch_manager->IsInSupportedDomain(request()->url());
 541       if (rv != SDCH_OK) {
 542         advertise_sdch = false;
 543         // If SDCH is just disabled, it is not a real error.
 544         if (rv != SDCH_DISABLED && rv != SDCH_SECURE_SCHEME_NOT_SUPPORTED) {
 545           SdchManager::SdchErrorRecovery(rv);
 546           request()->net_log().AddEvent(
 547               NetLog::TYPE_SDCH_DECODING_ERROR,
 548               base::Bind(&NetLogSdchResourceProblemCallback, rv));
 549         }
 550       }
 551     }
 552     if (advertise_sdch) {
 553       dictionaries_advertised_ =
 554           sdch_manager->GetDictionarySet(request_->url());
 555     }
 556
 557     // The AllowLatencyExperiment() is only true if we've successfully done a
 558     // full SDCH compression recently in this browser session for this host.
 559     // Note that for this path, there might be no applicable dictionaries,
 560     // and hence we can't participate in the experiment.
 561     if (dictionaries_advertised_ &&
 562         sdch_manager->AllowLatencyExperiment(request_->url())) {
 563       // We are participating in the test (or control), and hence we'll
 564       // eventually record statistics via either SDCH_EXPERIMENT_DECODE or
 565       // SDCH_EXPERIMENT_HOLDBACK, and we'll need some packet timing data.
 566       packet_timing_enabled_ = true;
 567       if (base::RandDouble() < .01) {
 568         sdch_test_control_ = true;  // 1% probability.
 569         dictionaries_advertised_.reset();
 570         advertise_sdch = false;
 571       } else {
 572         sdch_test_activated_ = true;
 573       }
 574     }
 575
 576     // Supply Accept-Encoding headers first so that it is more likely that they
 577     // will be in the first transmitted packet. This can sometimes make it
 578     // easier to filter and analyze the streams to assure that a proxy has not
 579     // damaged these headers. Some proxies deliberately corrupt Accept-Encoding
 580     // headers.
 581     if (!advertise_sdch) {
 582       // Tell the server what compression formats we support (other than SDCH).
 583       request_info_.extra_headers.SetHeader(
 584           HttpRequestHeaders::kAcceptEncoding, "gzip, deflate");
 585     } else {
 586       // Include SDCH in acceptable list.
 587       request_info_.extra_headers.SetHeader(
 588           HttpRequestHeaders::kAcceptEncoding, "gzip, deflate, sdch");
 589       if (dictionaries_advertised_) {
 590         request_info_.extra_headers.SetHeader(
 591             kAvailDictionaryHeader,
 592             dictionaries_advertised_->GetDictionaryClientHashList());
 593         // Since we're tagging this transaction as advertising a dictionary,
 594         // we'll definitely employ an SDCH filter (or tentative sdch filter)
 595         // when we get a response. When done, we'll record histograms via
 596         // SDCH_DECODE or SDCH_PASSTHROUGH. Hence we need to record packet
 597         // arrival times.
 598         packet_timing_enabled_ = true;
 599       }
 600     }
 601   }
 602
 603   if (http_user_agent_settings_) {
 604     // Only add default Accept-Language if the request didn't have it
 605     // specified.
 606     std::string accept_language =
 607         http_user_agent_settings_->GetAcceptLanguage();
 608     if (!accept_language.empty()) {
 609       request_info_.extra_headers.SetHeaderIfMissing(
 610           HttpRequestHeaders::kAcceptLanguage,
 611           accept_language);
 612     }
 613   }
 614 }
 615
 616 void URLRequestHttpJob::AddCookieHeaderAndStart() {
 617   // No matter what, we want to report our status as IO pending since we will
 618   // be notifying our consumer asynchronously via OnStartCompleted.
 619   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
 620
 621   // If the request was destroyed, then there is no more work to do.
 622   if (!request_)
 623     return;
 624
 625   CookieStore* cookie_store = request_->context()->cookie_store();
 626   if (cookie_store && !(request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES)) {
 627     cookie_store->GetAllCookiesForURLAsync(
 628         request_->url(),
 629         base::Bind(&URLRequestHttpJob::CheckCookiePolicyAndLoad,
 630                    weak_factory_.GetWeakPtr()));
 631   } else {
 632     DoStartTransaction();
 633   }
 634 }
 635
 636 void URLRequestHttpJob::DoLoadCookies() {
 637   CookieOptions options;
 638   options.set_include_httponly();
 639
 640   // TODO(mkwst): Drop this `if` once we decide whether or not to ship
 641   // first-party cookies: https://crbug.com/459154
 642   if (network_delegate() &&
 643       network_delegate()->FirstPartyOnlyCookieExperimentEnabled())
 644     options.set_first_party_url(request_->first_party_for_cookies());
 645   else
 646     options.set_include_first_party_only();
 647
 648   request_->context()->cookie_store()->GetCookiesWithOptionsAsync(
 649       request_->url(), options, base::Bind(&URLRequestHttpJob::OnCookiesLoaded,
 650                                            weak_factory_.GetWeakPtr()));
 651 }
 652
 653 void URLRequestHttpJob::CheckCookiePolicyAndLoad(
 654     const CookieList& cookie_list) {
 655   if (CanGetCookies(cookie_list))
 656     DoLoadCookies();
 657   else
 658     DoStartTransaction();
 659 }
 660
 661 void URLRequestHttpJob::OnCookiesLoaded(const std::string& cookie_line) {
 662   if (!cookie_line.empty()) {
 663     request_info_.extra_headers.SetHeader(
 664         HttpRequestHeaders::kCookie, cookie_line);
 665     // Disable privacy mode as we are sending cookies anyway.
 666     request_info_.privacy_mode = PRIVACY_MODE_DISABLED;
 667   }
 668   DoStartTransaction();
 669 }
 670
 671 void URLRequestHttpJob::DoStartTransaction() {
 672   // We may have been canceled while retrieving cookies.
 673   if (GetStatus().is_success()) {
 674     StartTransaction();
 675   } else {
 676     NotifyCanceled();
 677   }
 678 }
 679
 680 void URLRequestHttpJob::SaveCookiesAndNotifyHeadersComplete(int result) {
 681   // End of the call started in OnStartCompleted.
 682   OnCallToDelegateComplete();
 683
 684   if (result != OK) {
 685     std::string source("delegate");
 686     request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 687                                  NetLog::StringCallback("source", &source));
 688     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 689     return;
 690   }
 691
 692   DCHECK(transaction_.get());
 693
 694   const HttpResponseInfo* response_info = transaction_->GetResponseInfo();
 695   DCHECK(response_info);
 696
 697   response_cookies_.clear();
 698   response_cookies_save_index_ = 0;
 699
 700   FetchResponseCookies(&response_cookies_);
 701
 702   if (!GetResponseHeaders()->GetDateValue(&response_date_))
 703     response_date_ = base::Time();
 704
 705   // Now, loop over the response cookies, and attempt to persist each.
 706   SaveNextCookie();
 707 }
 708
 709 // If the save occurs synchronously, SaveNextCookie will loop and save the next
 710 // cookie. If the save is deferred, the callback is responsible for continuing
 711 // to iterate through the cookies.
 712 // TODO(erikwright): Modify the CookieStore API to indicate via return value
 713 // whether it completed synchronously or asynchronously.
 714 // See http://crbug.com/131066.
 715 void URLRequestHttpJob::SaveNextCookie() {
 716   // No matter what, we want to report our status as IO pending since we will
 717   // be notifying our consumer asynchronously via OnStartCompleted.
 718   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
 719
 720   // Used to communicate with the callback. See the implementation of
 721   // OnCookieSaved.
 722   scoped_refptr<SharedBoolean> callback_pending = new SharedBoolean(false);
 723   scoped_refptr<SharedBoolean> save_next_cookie_running =
 724       new SharedBoolean(true);
 725
 726   if (!(request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) &&
 727       request_->context()->cookie_store() && response_cookies_.size() > 0) {
 728     CookieOptions options;
 729     options.set_include_httponly();
 730     options.set_server_time(response_date_);
 731
 732     CookieStore::SetCookiesCallback callback(base::Bind(
 733         &URLRequestHttpJob::OnCookieSaved, weak_factory_.GetWeakPtr(),
 734         save_next_cookie_running, callback_pending));
 735
 736     // Loop through the cookies as long as SetCookieWithOptionsAsync completes
 737     // synchronously.
 738     while (!callback_pending->data &&
 739            response_cookies_save_index_ < response_cookies_.size()) {
 740       if (CanSetCookie(
 741           response_cookies_[response_cookies_save_index_], &options)) {
 742         callback_pending->data = true;
 743         request_->context()->cookie_store()->SetCookieWithOptionsAsync(
 744             request_->url(), response_cookies_[response_cookies_save_index_],
 745             options, callback);
 746       }
 747       ++response_cookies_save_index_;
 748     }
 749   }
 750
 751   save_next_cookie_running->data = false;
 752
 753   if (!callback_pending->data) {
 754     response_cookies_.clear();
 755     response_cookies_save_index_ = 0;
 756     SetStatus(URLRequestStatus());  // Clear the IO_PENDING status
 757     NotifyHeadersComplete();
 758     return;
 759   }
 760 }
 761
 762 // |save_next_cookie_running| is true when the callback is bound and set to
 763 // false when SaveNextCookie exits, allowing the callback to determine if the
 764 // save occurred synchronously or asynchronously.
 765 // |callback_pending| is false when the callback is invoked and will be set to
 766 // true by the callback, allowing SaveNextCookie to detect whether the save
 767 // occurred synchronously.
 768 // See SaveNextCookie() for more information.
 769 void URLRequestHttpJob::OnCookieSaved(
 770     scoped_refptr<SharedBoolean> save_next_cookie_running,
 771     scoped_refptr<SharedBoolean> callback_pending,
 772     bool cookie_status) {
 773   callback_pending->data = false;
 774
 775   // If we were called synchronously, return.
 776   if (save_next_cookie_running->data) {
 777     return;
 778   }
 779
 780   // We were called asynchronously, so trigger the next save.
 781   // We may have been canceled within OnSetCookie.
 782   if (GetStatus().is_success()) {
 783     SaveNextCookie();
 784   } else {
 785     NotifyCanceled();
 786   }
 787 }
 788
 789 void URLRequestHttpJob::FetchResponseCookies(
 790     std::vector<std::string>* cookies) {
 791   const std::string name = "Set-Cookie";
 792   std::string value;
 793
 794   void* iter = NULL;
 795   HttpResponseHeaders* headers = GetResponseHeaders();
 796   while (headers->EnumerateHeader(&iter, name, &value)) {
 797     if (!value.empty())
 798       cookies->push_back(value);
 799   }
 800 }
 801
 802 // NOTE: |ProcessStrictTransportSecurityHeader| and
 803 // |ProcessPublicKeyPinsHeader| have very similar structures, by design.
 804 void URLRequestHttpJob::ProcessStrictTransportSecurityHeader() {
 805   DCHECK(response_info_);
 806   TransportSecurityState* security_state =
 807       request_->context()->transport_security_state();
 808   const SSLInfo& ssl_info = response_info_->ssl_info;
 809
 810   // Only accept HSTS headers on HTTPS connections that have no
 811   // certificate errors.
 812   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 813       !security_state)
 814     return;
 815
 816   // Don't accept HSTS headers when the hostname is an IP address.
 817   if (request_info_.url.HostIsIPAddress())
 818     return;
 819
 820   // http://tools.ietf.org/html/draft-ietf-websec-strict-transport-sec:
 821   //
 822   //   If a UA receives more than one STS header field in a HTTP response
 823   //   message over secure transport, then the UA MUST process only the
 824   //   first such header field.
 825   HttpResponseHeaders* headers = GetResponseHeaders();
 826   std::string value;
 827   if (headers->EnumerateHeader(NULL, "Strict-Transport-Security", &value))
 828     security_state->AddHSTSHeader(request_info_.url.host(), value);
 829 }
 830
 831 void URLRequestHttpJob::ProcessPublicKeyPinsHeader() {
 832   DCHECK(response_info_);
 833   TransportSecurityState* security_state =
 834       request_->context()->transport_security_state();
 835   const SSLInfo& ssl_info = response_info_->ssl_info;
 836
 837   // Only accept HPKP headers on HTTPS connections that have no
 838   // certificate errors.
 839   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 840       !security_state)
 841     return;
 842
 843   // Don't accept HSTS headers when the hostname is an IP address.
 844   if (request_info_.url.HostIsIPAddress())
 845     return;
 846
 847   // http://tools.ietf.org/html/draft-ietf-websec-key-pinning:
 848   //
 849   //   If a UA receives more than one PKP header field in an HTTP
 850   //   response message over secure transport, then the UA MUST process
 851   //   only the first such header field.
 852   HttpResponseHeaders* headers = GetResponseHeaders();
 853   std::string value;
 854   if (headers->EnumerateHeader(NULL, "Public-Key-Pins", &value))
 855     security_state->AddHPKPHeader(request_info_.url.host(), value, ssl_info);
 856 }
 857
 858 void URLRequestHttpJob::OnStartCompleted(int result) {
 859   RecordTimer();
 860
 861   // If the request was destroyed, then there is no more work to do.
 862   if (!request_)
 863     return;
 864
 865   // If the job is done (due to cancellation), can just ignore this
 866   // notification.
 867   if (done_)
 868     return;
 869
 870   receive_headers_end_ = base::TimeTicks::Now();
 871
 872   // Clear the IO_PENDING status
 873   SetStatus(URLRequestStatus());
 874
 875   const URLRequestContext* context = request_->context();
 876
 877   if (result == ERR_SSL_PINNED_KEY_NOT_IN_CERT_CHAIN &&
 878       transaction_->GetResponseInfo() != NULL) {
 879     FraudulentCertificateReporter* reporter =
 880       context->fraudulent_certificate_reporter();
 881     if (reporter != NULL) {
 882       const SSLInfo& ssl_info = transaction_->GetResponseInfo()->ssl_info;
 883       const std::string& host = request_->url().host();
 884
 885       reporter->SendReport(host, ssl_info);
 886     }
 887   }
 888
 889   if (result == OK) {
 890     if (transaction_ && transaction_->GetResponseInfo()) {
 891       SetProxyServer(transaction_->GetResponseInfo()->proxy_server);
 892     }
 893     scoped_refptr<HttpResponseHeaders> headers = GetResponseHeaders();
 894     if (network_delegate()) {
 895       // Note that |this| may not be deleted until
 896       // |on_headers_received_callback_| or
 897       // |NetworkDelegate::URLRequestDestroyed()| has been called.
 898       OnCallToDelegate();
 899       allowed_unsafe_redirect_url_ = GURL();
 900       int error = network_delegate()->NotifyHeadersReceived(
 901           request_,
 902           on_headers_received_callback_,
 903           headers.get(),
 904           &override_response_headers_,
 905           &allowed_unsafe_redirect_url_);
 906       if (error != OK) {
 907         if (error == ERR_IO_PENDING) {
 908           awaiting_callback_ = true;
 909         } else {
 910           std::string source("delegate");
 911           request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 912                                        NetLog::StringCallback("source",
 913                                                               &source));
 914           OnCallToDelegateComplete();
 915           NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, error));
 916         }
 917         return;
 918       }
 919     }
 920
 921     SaveCookiesAndNotifyHeadersComplete(OK);
 922   } else if (IsCertificateError(result)) {
 923     // We encountered an SSL certificate error.
 924     if (result == ERR_SSL_WEAK_SERVER_EPHEMERAL_DH_KEY ||
 925         result == ERR_SSL_PINNED_KEY_NOT_IN_CERT_CHAIN) {
 926       // These are hard failures. They're handled separately and don't have
 927       // the correct cert status, so set it here.
 928       SSLInfo info(transaction_->GetResponseInfo()->ssl_info);
 929       info.cert_status = MapNetErrorToCertStatus(result);
 930       NotifySSLCertificateError(info, true);
 931     } else {
 932       // Maybe overridable, maybe not. Ask the delegate to decide.
 933       TransportSecurityState* state = context->transport_security_state();
 934       const bool fatal =
 935           state && state->ShouldSSLErrorsBeFatal(request_info_.url.host());
 936       NotifySSLCertificateError(
 937           transaction_->GetResponseInfo()->ssl_info, fatal);
 938     }
 939   } else if (result == ERR_SSL_CLIENT_AUTH_CERT_NEEDED) {
 940     NotifyCertificateRequested(
 941         transaction_->GetResponseInfo()->cert_request_info.get());
 942   } else {
 943     // Even on an error, there may be useful information in the response
 944     // info (e.g. whether there's a cached copy).
 945     if (transaction_.get())
 946       response_info_ = transaction_->GetResponseInfo();
 947     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 948   }
 949 }
 950
 951 void URLRequestHttpJob::OnHeadersReceivedCallback(int result) {
 952   awaiting_callback_ = false;
 953
 954   // Check that there are no callbacks to already canceled requests.
 955   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 956
 957   SaveCookiesAndNotifyHeadersComplete(result);
 958 }
 959
 960 void URLRequestHttpJob::OnReadCompleted(int result) {
 961   read_in_progress_ = false;
 962
 963   if (ShouldFixMismatchedContentLength(result))
 964     result = OK;
 965
 966   if (result == OK) {
 967     NotifyDone(URLRequestStatus());
 968   } else if (result < 0) {
 969     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, result));
 970   } else {
 971     // Clear the IO_PENDING status
 972     SetStatus(URLRequestStatus());
 973   }
 974
 975   NotifyReadComplete(result);
 976 }
 977
 978 void URLRequestHttpJob::RestartTransactionWithAuth(
 979     const AuthCredentials& credentials) {
 980   auth_credentials_ = credentials;
 981
 982   // These will be reset in OnStartCompleted.
 983   response_info_ = NULL;
 984   receive_headers_end_ = base::TimeTicks();
 985   response_cookies_.clear();
 986
 987   ResetTimer();
 988
 989   // Update the cookies, since the cookie store may have been updated from the
 990   // headers in the 401/407. Since cookies were already appended to
 991   // extra_headers, we need to strip them out before adding them again.
 992   request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kCookie);
 993
 994   AddCookieHeaderAndStart();
 995 }
 996
 997 void URLRequestHttpJob::SetUpload(UploadDataStream* upload) {
 998   DCHECK(!transaction_.get()) << "cannot change once started";
 999   request_info_.upload_data_stream = upload;
1000 }
1001
1002 void URLRequestHttpJob::SetExtraRequestHeaders(
1003     const HttpRequestHeaders& headers) {
1004   DCHECK(!transaction_.get()) << "cannot change once started";
1005   request_info_.extra_headers.CopyFrom(headers);
1006 }
1007
1008 LoadState URLRequestHttpJob::GetLoadState() const {
1009   return transaction_.get() ?
1010       transaction_->GetLoadState() : LOAD_STATE_IDLE;
1011 }
1012
1013 UploadProgress URLRequestHttpJob::GetUploadProgress() const {
1014   return transaction_.get() ?
1015       transaction_->GetUploadProgress() : UploadProgress();
1016 }
1017
1018 bool URLRequestHttpJob::GetMimeType(std::string* mime_type) const {
1019   DCHECK(transaction_.get());
1020
1021   if (!response_info_)
1022     return false;
1023
1024   HttpResponseHeaders* headers = GetResponseHeaders();
1025   if (!headers)
1026     return false;
1027   return headers->GetMimeType(mime_type);
1028 }
1029
1030 bool URLRequestHttpJob::GetCharset(std::string* charset) {
1031   DCHECK(transaction_.get());
1032
1033   if (!response_info_)
1034     return false;
1035
1036   return GetResponseHeaders()->GetCharset(charset);
1037 }
1038
1039 void URLRequestHttpJob::GetResponseInfo(HttpResponseInfo* info) {
1040   DCHECK(request_);
1041
1042   if (response_info_) {
1043     DCHECK(transaction_.get());
1044
1045     *info = *response_info_;
1046     if (override_response_headers_.get())
1047       info->headers = override_response_headers_;
1048   }
1049 }
1050
1051 void URLRequestHttpJob::GetLoadTimingInfo(
1052     LoadTimingInfo* load_timing_info) const {
1053   // If haven't made it far enough to receive any headers, don't return
1054   // anything. This makes for more consistent behavior in the case of errors.
1055   if (!transaction_ || receive_headers_end_.is_null())
1056     return;
1057   if (transaction_->GetLoadTimingInfo(load_timing_info))
1058     load_timing_info->receive_headers_end = receive_headers_end_;
1059 }
1060
1061 bool URLRequestHttpJob::GetResponseCookies(std::vector<std::string>* cookies) {
1062   DCHECK(transaction_.get());
1063
1064   if (!response_info_)
1065     return false;
1066
1067   // TODO(darin): Why are we extracting response cookies again?  Perhaps we
1068   // should just leverage response_cookies_.
1069
1070   cookies->clear();
1071   FetchResponseCookies(cookies);
1072   return true;
1073 }
1074
1075 int URLRequestHttpJob::GetResponseCode() const {
1076   DCHECK(transaction_.get());
1077
1078   if (!response_info_)
1079     return -1;
1080
1081   return GetResponseHeaders()->response_code();
1082 }
1083
1084 Filter* URLRequestHttpJob::SetupFilter() const {
1085   DCHECK(transaction_.get());
1086   if (!response_info_)
1087     return NULL;
1088
1089   std::vector<Filter::FilterType> encoding_types;
1090   std::string encoding_type;
1091   HttpResponseHeaders* headers = GetResponseHeaders();
1092   void* iter = NULL;
1093   while (headers->EnumerateHeader(&iter, "Content-Encoding", &encoding_type)) {
1094     encoding_types.push_back(Filter::ConvertEncodingToType(encoding_type));
1095   }
1096
1097   // Even if encoding types are empty, there is a chance that we need to add
1098   // some decoding, as some proxies strip encoding completely. In such cases,
1099   // we may need to add (for example) SDCH filtering (when the context suggests
1100   // it is appropriate).
1101   Filter::FixupEncodingTypes(*filter_context_, &encoding_types);
1102
1103   return !encoding_types.empty()
1104       ? Filter::Factory(encoding_types, *filter_context_) : NULL;
1105 }
1106
1107 bool URLRequestHttpJob::CopyFragmentOnRedirect(const GURL& location) const {
1108   // Allow modification of reference fragments by default, unless
1109   // |allowed_unsafe_redirect_url_| is set and equal to the redirect URL.
1110   // When this is the case, we assume that the network delegate has set the
1111   // desired redirect URL (with or without fragment), so it must not be changed
1112   // any more.
1113   return !allowed_unsafe_redirect_url_.is_valid() ||
1114        allowed_unsafe_redirect_url_ != location;
1115 }
1116
1117 bool URLRequestHttpJob::IsSafeRedirect(const GURL& location) {
1118   // HTTP is always safe.
1119   // TODO(pauljensen): Remove once crbug.com/146591 is fixed.
1120   if (location.is_valid() &&
1121       (location.scheme() == "http" || location.scheme() == "https")) {
1122     return true;
1123   }
1124   // Delegates may mark a URL as safe for redirection.
1125   if (allowed_unsafe_redirect_url_.is_valid() &&
1126       allowed_unsafe_redirect_url_ == location) {
1127     return true;
1128   }
1129   // Query URLRequestJobFactory as to whether |location| would be safe to
1130   // redirect to.
1131   return request_->context()->job_factory() &&
1132       request_->context()->job_factory()->IsSafeRedirectTarget(location);
1133 }
1134
1135 bool URLRequestHttpJob::NeedsAuth() {
1136   int code = GetResponseCode();
1137   if (code == -1)
1138     return false;
1139
1140   // Check if we need either Proxy or WWW Authentication. This could happen
1141   // because we either provided no auth info, or provided incorrect info.
1142   switch (code) {
1143     case 407:
1144       if (proxy_auth_state_ == AUTH_STATE_CANCELED)
1145         return false;
1146       proxy_auth_state_ = AUTH_STATE_NEED_AUTH;
1147       return true;
1148     case 401:
1149       if (server_auth_state_ == AUTH_STATE_CANCELED)
1150         return false;
1151       server_auth_state_ = AUTH_STATE_NEED_AUTH;
1152       return true;
1153   }
1154   return false;
1155 }
1156
1157 void URLRequestHttpJob::GetAuthChallengeInfo(
1158     scoped_refptr<AuthChallengeInfo>* result) {
1159   DCHECK(transaction_.get());
1160   DCHECK(response_info_);
1161
1162   // sanity checks:
1163   DCHECK(proxy_auth_state_ == AUTH_STATE_NEED_AUTH ||
1164          server_auth_state_ == AUTH_STATE_NEED_AUTH);
1165   DCHECK((GetResponseHeaders()->response_code() == HTTP_UNAUTHORIZED) ||
1166          (GetResponseHeaders()->response_code() ==
1167           HTTP_PROXY_AUTHENTICATION_REQUIRED));
1168
1169   *result = response_info_->auth_challenge;
1170 }
1171
1172 void URLRequestHttpJob::SetAuth(const AuthCredentials& credentials) {
1173   DCHECK(transaction_.get());
1174
1175   // Proxy gets set first, then WWW.
1176   if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1177     proxy_auth_state_ = AUTH_STATE_HAVE_AUTH;
1178   } else {
1179     DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1180     server_auth_state_ = AUTH_STATE_HAVE_AUTH;
1181   }
1182
1183   RestartTransactionWithAuth(credentials);
1184 }
1185
1186 void URLRequestHttpJob::CancelAuth() {
1187   // Proxy gets set first, then WWW.
1188   if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1189     proxy_auth_state_ = AUTH_STATE_CANCELED;
1190   } else {
1191     DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1192     server_auth_state_ = AUTH_STATE_CANCELED;
1193   }
1194
1195   // These will be reset in OnStartCompleted.
1196   response_info_ = NULL;
1197   receive_headers_end_ = base::TimeTicks::Now();
1198   response_cookies_.clear();
1199
1200   ResetTimer();
1201
1202   // OK, let the consumer read the error page...
1203   //
1204   // Because we set the AUTH_STATE_CANCELED flag, NeedsAuth will return false,
1205   // which will cause the consumer to receive OnResponseStarted instead of
1206   // OnAuthRequired.
1207   //
1208   // We have to do this via InvokeLater to avoid "recursing" the consumer.
1209   //
1210   base::MessageLoop::current()->PostTask(
1211       FROM_HERE,
1212       base::Bind(&URLRequestHttpJob::OnStartCompleted,
1213                  weak_factory_.GetWeakPtr(), OK));
1214 }
1215
1216 void URLRequestHttpJob::ContinueWithCertificate(
1217     X509Certificate* client_cert) {
1218   DCHECK(transaction_.get());
1219
1220   DCHECK(!response_info_) << "should not have a response yet";
1221   receive_headers_end_ = base::TimeTicks();
1222
1223   ResetTimer();
1224
1225   // No matter what, we want to report our status as IO pending since we will
1226   // be notifying our consumer asynchronously via OnStartCompleted.
1227   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1228
1229   int rv = transaction_->RestartWithCertificate(client_cert, start_callback_);
1230   if (rv == ERR_IO_PENDING)
1231     return;
1232
1233   // The transaction started synchronously, but we need to notify the
1234   // URLRequest delegate via the message loop.
1235   base::MessageLoop::current()->PostTask(
1236       FROM_HERE,
1237       base::Bind(&URLRequestHttpJob::OnStartCompleted,
1238                  weak_factory_.GetWeakPtr(), rv));
1239 }
1240
1241 void URLRequestHttpJob::ContinueDespiteLastError() {
1242   // If the transaction was destroyed, then the job was cancelled.
1243   if (!transaction_.get())
1244     return;
1245
1246   DCHECK(!response_info_) << "should not have a response yet";
1247   receive_headers_end_ = base::TimeTicks();
1248
1249   ResetTimer();
1250
1251   // No matter what, we want to report our status as IO pending since we will
1252   // be notifying our consumer asynchronously via OnStartCompleted.
1253   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1254
1255   int rv = transaction_->RestartIgnoringLastError(start_callback_);
1256   if (rv == ERR_IO_PENDING)
1257     return;
1258
1259   // The transaction started synchronously, but we need to notify the
1260   // URLRequest delegate via the message loop.
1261   base::MessageLoop::current()->PostTask(
1262       FROM_HERE,
1263       base::Bind(&URLRequestHttpJob::OnStartCompleted,
1264                  weak_factory_.GetWeakPtr(), rv));
1265 }
1266
1267 void URLRequestHttpJob::ResumeNetworkStart() {
1268   DCHECK(transaction_.get());
1269   transaction_->ResumeNetworkStart();
1270 }
1271
1272 bool URLRequestHttpJob::ShouldFixMismatchedContentLength(int rv) const {
1273   // Some servers send the body compressed, but specify the content length as
1274   // the uncompressed size. Although this violates the HTTP spec we want to
1275   // support it (as IE and FireFox do), but *only* for an exact match.
1276   // See http://crbug.com/79694.
1277   if (rv == ERR_CONTENT_LENGTH_MISMATCH ||
1278       rv == ERR_INCOMPLETE_CHUNKED_ENCODING) {
1279     if (request_ && request_->response_headers()) {
1280       int64 expected_length = request_->response_headers()->GetContentLength();
1281       VLOG(1) << __FUNCTION__ << "() "
1282               << "\"" << request_->url().spec() << "\""
1283               << " content-length = " << expected_length
1284               << " pre total = " << prefilter_bytes_read()
1285               << " post total = " << postfilter_bytes_read();
1286       if (postfilter_bytes_read() == expected_length) {
1287         // Clear the error.
1288         return true;
1289       }
1290     }
1291   }
1292   return false;
1293 }
1294
1295 bool URLRequestHttpJob::ReadRawData(IOBuffer* buf, int buf_size,
1296                                     int* bytes_read) {
1297   DCHECK_NE(buf_size, 0);
1298   DCHECK(bytes_read);
1299   DCHECK(!read_in_progress_);
1300
1301   int rv = transaction_->Read(
1302       buf, buf_size,
1303       base::Bind(&URLRequestHttpJob::OnReadCompleted, base::Unretained(this)));
1304
1305   if (ShouldFixMismatchedContentLength(rv))
1306     rv = 0;
1307
1308   if (rv >= 0) {
1309     *bytes_read = rv;
1310     if (!rv)
1311       DoneWithRequest(FINISHED);
1312     return true;
1313   }
1314
1315   if (rv == ERR_IO_PENDING) {
1316     read_in_progress_ = true;
1317     SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1318   } else {
1319     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, rv));
1320   }
1321
1322   return false;
1323 }
1324
1325 void URLRequestHttpJob::StopCaching() {
1326   if (transaction_.get())
1327     transaction_->StopCaching();
1328 }
1329
1330 bool URLRequestHttpJob::GetFullRequestHeaders(
1331     HttpRequestHeaders* headers) const {
1332   if (!transaction_)
1333     return false;
1334
1335   return transaction_->GetFullRequestHeaders(headers);
1336 }
1337
1338 int64 URLRequestHttpJob::GetTotalReceivedBytes() const {
1339   if (!transaction_)
1340     return 0;
1341
1342   return transaction_->GetTotalReceivedBytes();
1343 }
1344
1345 void URLRequestHttpJob::DoneReading() {
1346   if (transaction_) {
1347     transaction_->DoneReading();
1348   }
1349   DoneWithRequest(FINISHED);
1350 }
1351
1352 void URLRequestHttpJob::DoneReadingRedirectResponse() {
1353   if (transaction_) {
1354     if (transaction_->GetResponseInfo()->headers->IsRedirect(NULL)) {
1355       // If the original headers indicate a redirect, go ahead and cache the
1356       // response, even if the |override_response_headers_| are a redirect to
1357       // another location.
1358       transaction_->DoneReading();
1359     } else {
1360       // Otherwise, |override_response_headers_| must be non-NULL and contain
1361       // bogus headers indicating a redirect.
1362       DCHECK(override_response_headers_.get());
1363       DCHECK(override_response_headers_->IsRedirect(NULL));
1364       transaction_->StopCaching();
1365     }
1366   }
1367   DoneWithRequest(FINISHED);
1368 }
1369
1370 HostPortPair URLRequestHttpJob::GetSocketAddress() const {
1371   return response_info_ ? response_info_->socket_address : HostPortPair();
1372 }
1373
1374 void URLRequestHttpJob::RecordTimer() {
1375   if (request_creation_time_.is_null()) {
1376     NOTREACHED()
1377         << "The same transaction shouldn't start twice without new timing.";
1378     return;
1379   }
1380
1381   base::TimeDelta to_start = base::Time::Now() - request_creation_time_;
1382   request_creation_time_ = base::Time();
1383
1384   UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpTimeToFirstByte", to_start);
1385 }
1386
1387 void URLRequestHttpJob::ResetTimer() {
1388   if (!request_creation_time_.is_null()) {
1389     NOTREACHED()
1390         << "The timer was reset before it was recorded.";
1391     return;
1392   }
1393   request_creation_time_ = base::Time::Now();
1394 }
1395
1396 void URLRequestHttpJob::UpdatePacketReadTimes() {
1397   if (!packet_timing_enabled_)
1398     return;
1399
1400   DCHECK_GT(prefilter_bytes_read(), bytes_observed_in_packets_);
1401
1402   base::Time now(base::Time::Now());
1403   if (!bytes_observed_in_packets_)
1404     request_time_snapshot_ = now;
1405   final_packet_time_ = now;
1406
1407   bytes_observed_in_packets_ = prefilter_bytes_read();
1408 }
1409
1410 void URLRequestHttpJob::RecordPacketStats(
1411     FilterContext::StatisticSelector statistic) const {
1412   if (!packet_timing_enabled_ || (final_packet_time_ == base::Time()))
1413     return;
1414
1415   base::TimeDelta duration = final_packet_time_ - request_time_snapshot_;
1416   switch (statistic) {
1417     case FilterContext::SDCH_DECODE: {
1418       UMA_HISTOGRAM_CUSTOM_COUNTS("Sdch3.Network_Decode_Bytes_Processed_b",
1419           static_cast<int>(bytes_observed_in_packets_), 500, 100000, 100);
1420       return;
1421     }
1422     case FilterContext::SDCH_PASSTHROUGH: {
1423       // Despite advertising a dictionary, we handled non-sdch compressed
1424       // content.
1425       return;
1426     }
1427
1428     case FilterContext::SDCH_EXPERIMENT_DECODE: {
1429       UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment3_Decode",
1430                                   duration,
1431                                   base::TimeDelta::FromMilliseconds(20),
1432                                   base::TimeDelta::FromMinutes(10), 100);
1433       return;
1434     }
1435     case FilterContext::SDCH_EXPERIMENT_HOLDBACK: {
1436       UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment3_Holdback",
1437                                   duration,
1438                                   base::TimeDelta::FromMilliseconds(20),
1439                                   base::TimeDelta::FromMinutes(10), 100);
1440       return;
1441     }
1442     default:
1443       NOTREACHED();
1444       return;
1445   }
1446 }
1447
1448 void URLRequestHttpJob::RecordPerfHistograms(CompletionCause reason) {
1449   if (start_time_.is_null())
1450     return;
1451
1452   base::TimeDelta total_time = base::TimeTicks::Now() - start_time_;
1453   UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTime", total_time);
1454
1455   if (reason == FINISHED) {
1456     UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeSuccess", total_time);
1457   } else {
1458     UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCancel", total_time);
1459   }
1460
1461   if (response_info_) {
1462     bool is_google = request() && HasGoogleHost(request()->url());
1463     bool used_quic = response_info_->DidUseQuic();
1464     if (is_google) {
1465       if (used_quic) {
1466         UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTime.Quic", total_time);
1467       } else {
1468         UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTime.NotQuic", total_time);
1469       }
1470     }
1471     if (response_info_->was_cached) {
1472       UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCached", total_time);
1473       if (is_google) {
1474         if (used_quic) {
1475           UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTimeCached.Quic",
1476                                      total_time);
1477         } else {
1478           UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTimeCached.NotQuic",
1479                                      total_time);
1480         }
1481       }
1482     } else  {
1483       UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeNotCached", total_time);
1484       if (is_google) {
1485         if (used_quic) {
1486           UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTimeNotCached.Quic",
1487                                      total_time);
1488         } else {
1489           UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTimeNotCached.NotQuic",
1490                                      total_time);
1491         }
1492       }
1493     }
1494   }
1495
1496   if (request_info_.load_flags & LOAD_PREFETCH && !request_->was_cached())
1497     UMA_HISTOGRAM_COUNTS("Net.Prefetch.PrefilterBytesReadFromNetwork",
1498                          prefilter_bytes_read());
1499
1500   start_time_ = base::TimeTicks();
1501 }
1502
1503 void URLRequestHttpJob::DoneWithRequest(CompletionCause reason) {
1504   if (done_)
1505     return;
1506   done_ = true;
1507   RecordPerfHistograms(reason);
1508   if (reason == FINISHED) {
1509     request_->set_received_response_content_length(prefilter_bytes_read());
1510   }
1511 }
1512
1513 HttpResponseHeaders* URLRequestHttpJob::GetResponseHeaders() const {
1514   DCHECK(transaction_.get());
1515   DCHECK(transaction_->GetResponseInfo());
1516   return override_response_headers_.get() ?
1517              override_response_headers_.get() :
1518              transaction_->GetResponseInfo()->headers.get();
1519 }
1520
1521 void URLRequestHttpJob::NotifyURLRequestDestroyed() {
1522   awaiting_callback_ = false;
1523 }
1524
1525 }  // namespace net