net/url_request/url_request_http_job.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_request_http_job.h"
   6
   7 #include "base/base_switches.h"
   8 #include "base/bind.h"
   9 #include "base/bind_helpers.h"
  10 #include "base/command_line.h"
  11 #include "base/compiler_specific.h"
  12 #include "base/file_version_info.h"
  13 #include "base/location.h"
  14 #include "base/metrics/field_trial.h"
  15 #include "base/metrics/histogram.h"
  16 #include "base/profiler/scoped_tracker.h"
  17 #include "base/rand_util.h"
  18 #include "base/single_thread_task_runner.h"
  19 #include "base/strings/string_util.h"
  20 #include "base/thread_task_runner_handle.h"
  21 #include "base/time/time.h"
  22 #include "base/values.h"
  23 #include "net/base/host_port_pair.h"
  24 #include "net/base/load_flags.h"
  25 #include "net/base/net_errors.h"
  26 #include "net/base/net_util.h"
  27 #include "net/base/network_delegate.h"
  28 #include "net/base/sdch_manager.h"
  29 #include "net/base/sdch_net_log_params.h"
  30 #include "net/cert/cert_status_flags.h"
  31 #include "net/cookies/cookie_store.h"
  32 #include "net/http/http_content_disposition.h"
  33 #include "net/http/http_network_session.h"
  34 #include "net/http/http_request_headers.h"
  35 #include "net/http/http_response_headers.h"
  36 #include "net/http/http_response_info.h"
  37 #include "net/http/http_status_code.h"
  38 #include "net/http/http_transaction.h"
  39 #include "net/http/http_transaction_factory.h"
  40 #include "net/http/http_util.h"
  41 #include "net/proxy/proxy_info.h"
  42 #include "net/ssl/ssl_cert_request_info.h"
  43 #include "net/ssl/ssl_config_service.h"
  44 #include "net/url_request/fraudulent_certificate_reporter.h"
  45 #include "net/url_request/http_user_agent_settings.h"
  46 #include "net/url_request/url_request.h"
  47 #include "net/url_request/url_request_context.h"
  48 #include "net/url_request/url_request_error_job.h"
  49 #include "net/url_request/url_request_job_factory.h"
  50 #include "net/url_request/url_request_redirect_job.h"
  51 #include "net/url_request/url_request_throttler_manager.h"
  52 #include "net/websockets/websocket_handshake_stream_base.h"
  53
  54 static const char kAvailDictionaryHeader[] = "Avail-Dictionary";
  55
  56 namespace net {
  57
  58 class URLRequestHttpJob::HttpFilterContext : public FilterContext {
  59  public:
  60   explicit HttpFilterContext(URLRequestHttpJob* job);
  61   ~HttpFilterContext() override;
  62
  63   // FilterContext implementation.
  64   bool GetMimeType(std::string* mime_type) const override;
  65   bool GetURL(GURL* gurl) const override;
  66   base::Time GetRequestTime() const override;
  67   bool IsCachedContent() const override;
  68   SdchManager::DictionarySet* SdchDictionariesAdvertised() const override;
  69   int64 GetByteReadCount() const override;
  70   int GetResponseCode() const override;
  71   const URLRequestContext* GetURLRequestContext() const override;
  72   void RecordPacketStats(StatisticSelector statistic) const override;
  73   const BoundNetLog& GetNetLog() const override;
  74
  75  private:
  76   URLRequestHttpJob* job_;
  77
  78   // URLRequestHttpJob may be detached from URLRequest, but we still need to
  79   // return something.
  80   BoundNetLog dummy_log_;
  81
  82   DISALLOW_COPY_AND_ASSIGN(HttpFilterContext);
  83 };
  84
  85 URLRequestHttpJob::HttpFilterContext::HttpFilterContext(URLRequestHttpJob* job)
  86     : job_(job) {
  87   DCHECK(job_);
  88 }
  89
  90 URLRequestHttpJob::HttpFilterContext::~HttpFilterContext() {
  91 }
  92
  93 bool URLRequestHttpJob::HttpFilterContext::GetMimeType(
  94     std::string* mime_type) const {
  95   return job_->GetMimeType(mime_type);
  96 }
  97
  98 bool URLRequestHttpJob::HttpFilterContext::GetURL(GURL* gurl) const {
  99   if (!job_->request())
 100     return false;
 101   *gurl = job_->request()->url();
 102   return true;
 103 }
 104
 105 base::Time URLRequestHttpJob::HttpFilterContext::GetRequestTime() const {
 106   return job_->request() ? job_->request()->request_time() : base::Time();
 107 }
 108
 109 bool URLRequestHttpJob::HttpFilterContext::IsCachedContent() const {
 110   return job_->is_cached_content_;
 111 }
 112
 113 SdchManager::DictionarySet*
 114 URLRequestHttpJob::HttpFilterContext::SdchDictionariesAdvertised() const {
 115   return job_->dictionaries_advertised_.get();
 116 }
 117
 118 int64 URLRequestHttpJob::HttpFilterContext::GetByteReadCount() const {
 119   return job_->prefilter_bytes_read();
 120 }
 121
 122 int URLRequestHttpJob::HttpFilterContext::GetResponseCode() const {
 123   return job_->GetResponseCode();
 124 }
 125
 126 const URLRequestContext*
 127 URLRequestHttpJob::HttpFilterContext::GetURLRequestContext() const {
 128   return job_->request() ? job_->request()->context() : NULL;
 129 }
 130
 131 void URLRequestHttpJob::HttpFilterContext::RecordPacketStats(
 132     StatisticSelector statistic) const {
 133   job_->RecordPacketStats(statistic);
 134 }
 135
 136 const BoundNetLog& URLRequestHttpJob::HttpFilterContext::GetNetLog() const {
 137   return job_->request() ? job_->request()->net_log() : dummy_log_;
 138 }
 139
 140 // TODO(darin): make sure the port blocking code is not lost
 141 // static
 142 URLRequestJob* URLRequestHttpJob::Factory(URLRequest* request,
 143                                           NetworkDelegate* network_delegate,
 144                                           const std::string& scheme) {
 145   DCHECK(scheme == "http" || scheme == "https" || scheme == "ws" ||
 146          scheme == "wss");
 147
 148   if (!request->context()->http_transaction_factory()) {
 149     NOTREACHED() << "requires a valid context";
 150     return new URLRequestErrorJob(
 151         request, network_delegate, ERR_INVALID_ARGUMENT);
 152   }
 153
 154   GURL redirect_url;
 155   if (request->GetHSTSRedirect(&redirect_url)) {
 156     return new URLRequestRedirectJob(
 157         request, network_delegate, redirect_url,
 158         // Use status code 307 to preserve the method, so POST requests work.
 159         URLRequestRedirectJob::REDIRECT_307_TEMPORARY_REDIRECT, "HSTS");
 160   }
 161   return new URLRequestHttpJob(request,
 162                                network_delegate,
 163                                request->context()->http_user_agent_settings());
 164 }
 165
 166 URLRequestHttpJob::URLRequestHttpJob(
 167     URLRequest* request,
 168     NetworkDelegate* network_delegate,
 169     const HttpUserAgentSettings* http_user_agent_settings)
 170     : URLRequestJob(request, network_delegate),
 171       priority_(DEFAULT_PRIORITY),
 172       response_info_(NULL),
 173       response_cookies_save_index_(0),
 174       proxy_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
 175       server_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
 176       start_callback_(base::Bind(&URLRequestHttpJob::OnStartCompleted,
 177                                  base::Unretained(this))),
 178       notify_before_headers_sent_callback_(
 179           base::Bind(&URLRequestHttpJob::NotifyBeforeSendHeadersCallback,
 180                      base::Unretained(this))),
 181       read_in_progress_(false),
 182       throttling_entry_(NULL),
 183       sdch_test_activated_(false),
 184       sdch_test_control_(false),
 185       is_cached_content_(false),
 186       request_creation_time_(),
 187       packet_timing_enabled_(false),
 188       done_(false),
 189       bytes_observed_in_packets_(0),
 190       request_time_snapshot_(),
 191       final_packet_time_(),
 192       filter_context_(new HttpFilterContext(this)),
 193       on_headers_received_callback_(
 194           base::Bind(&URLRequestHttpJob::OnHeadersReceivedCallback,
 195                      base::Unretained(this))),
 196       awaiting_callback_(false),
 197       http_user_agent_settings_(http_user_agent_settings),
 198       weak_factory_(this) {
 199   URLRequestThrottlerManager* manager = request->context()->throttler_manager();
 200   if (manager)
 201     throttling_entry_ = manager->RegisterRequestUrl(request->url());
 202
 203   ResetTimer();
 204 }
 205
 206 URLRequestHttpJob::~URLRequestHttpJob() {
 207   CHECK(!awaiting_callback_);
 208
 209   DCHECK(!sdch_test_control_ || !sdch_test_activated_);
 210   if (!is_cached_content_) {
 211     if (sdch_test_control_)
 212       RecordPacketStats(FilterContext::SDCH_EXPERIMENT_HOLDBACK);
 213     if (sdch_test_activated_)
 214       RecordPacketStats(FilterContext::SDCH_EXPERIMENT_DECODE);
 215   }
 216   // Make sure SDCH filters are told to emit histogram data while
 217   // filter_context_ is still alive.
 218   DestroyFilters();
 219
 220   DoneWithRequest(ABORTED);
 221 }
 222
 223 void URLRequestHttpJob::SetPriority(RequestPriority priority) {
 224   priority_ = priority;
 225   if (transaction_)
 226     transaction_->SetPriority(priority_);
 227 }
 228
 229 void URLRequestHttpJob::Start() {
 230   // TODO(mmenke): Remove ScopedTracker below once crbug.com/456327 is fixed.
 231   tracked_objects::ScopedTracker tracking_profile(
 232       FROM_HERE_WITH_EXPLICIT_FUNCTION("456327 URLRequestHttpJob::Start"));
 233
 234   DCHECK(!transaction_.get());
 235
 236   // URLRequest::SetReferrer ensures that we do not send username and password
 237   // fields in the referrer.
 238   GURL referrer(request_->referrer());
 239
 240   request_info_.url = request_->url();
 241   request_info_.method = request_->method();
 242   request_info_.load_flags = request_->load_flags();
 243   // Enable privacy mode if cookie settings or flags tell us not send or
 244   // save cookies.
 245   bool enable_privacy_mode =
 246       (request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES) ||
 247       (request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) ||
 248       CanEnablePrivacyMode();
 249   // Privacy mode could still be disabled in OnCookiesLoaded if we are going
 250   // to send previously saved cookies.
 251   request_info_.privacy_mode = enable_privacy_mode ?
 252       PRIVACY_MODE_ENABLED : PRIVACY_MODE_DISABLED;
 253
 254   // Strip Referer from request_info_.extra_headers to prevent, e.g., plugins
 255   // from overriding headers that are controlled using other means. Otherwise a
 256   // plugin could set a referrer although sending the referrer is inhibited.
 257   request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kReferer);
 258
 259   // Our consumer should have made sure that this is a safe referrer. See for
 260   // instance WebCore::FrameLoader::HideReferrer.
 261   if (referrer.is_valid()) {
 262     request_info_.extra_headers.SetHeader(HttpRequestHeaders::kReferer,
 263                                           referrer.spec());
 264   }
 265
 266   request_info_.extra_headers.SetHeaderIfMissing(
 267       HttpRequestHeaders::kUserAgent,
 268       http_user_agent_settings_ ?
 269           http_user_agent_settings_->GetUserAgent() : std::string());
 270
 271   AddExtraHeaders();
 272   AddCookieHeaderAndStart();
 273 }
 274
 275 void URLRequestHttpJob::Kill() {
 276   if (!transaction_.get())
 277     return;
 278
 279   weak_factory_.InvalidateWeakPtrs();
 280   DestroyTransaction();
 281   URLRequestJob::Kill();
 282 }
 283
 284 void URLRequestHttpJob::GetConnectionAttempts(ConnectionAttempts* out) const {
 285   if (transaction_)
 286     transaction_->GetConnectionAttempts(out);
 287   else
 288     out->clear();
 289 }
 290
 291 void URLRequestHttpJob::NotifyBeforeSendProxyHeadersCallback(
 292     const ProxyInfo& proxy_info,
 293     HttpRequestHeaders* request_headers) {
 294   DCHECK(request_headers);
 295   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 296   if (network_delegate()) {
 297     network_delegate()->NotifyBeforeSendProxyHeaders(
 298         request_,
 299         proxy_info,
 300         request_headers);
 301   }
 302 }
 303
 304 void URLRequestHttpJob::NotifyHeadersComplete() {
 305   DCHECK(!response_info_);
 306
 307   response_info_ = transaction_->GetResponseInfo();
 308
 309   // Save boolean, as we'll need this info at destruction time, and filters may
 310   // also need this info.
 311   is_cached_content_ = response_info_->was_cached;
 312
 313   if (!is_cached_content_ && throttling_entry_.get())
 314     throttling_entry_->UpdateWithResponse(GetResponseCode());
 315
 316   // The ordering of these calls is not important.
 317   ProcessStrictTransportSecurityHeader();
 318   ProcessPublicKeyPinsHeader();
 319
 320   // Handle the server notification of a new SDCH dictionary.
 321   SdchManager* sdch_manager(request()->context()->sdch_manager());
 322   if (sdch_manager) {
 323     SdchProblemCode rv = sdch_manager->IsInSupportedDomain(request()->url());
 324     if (rv != SDCH_OK) {
 325       // If SDCH is just disabled, it is not a real error.
 326       if (rv != SDCH_DISABLED) {
 327         SdchManager::SdchErrorRecovery(rv);
 328         request()->net_log().AddEvent(
 329             NetLog::TYPE_SDCH_DECODING_ERROR,
 330             base::Bind(&NetLogSdchResourceProblemCallback, rv));
 331       }
 332     } else {
 333       const std::string name = "Get-Dictionary";
 334       std::string url_text;
 335       void* iter = NULL;
 336       // TODO(jar): We need to not fetch dictionaries the first time they are
 337       // seen, but rather wait until we can justify their usefulness.
 338       // For now, we will only fetch the first dictionary, which will at least
 339       // require multiple suggestions before we get additional ones for this
 340       // site. Eventually we should wait until a dictionary is requested
 341       // several times
 342       // before we even download it (so that we don't waste memory or
 343       // bandwidth).
 344       if (GetResponseHeaders()->EnumerateHeader(&iter, name, &url_text)) {
 345         // Resolve suggested URL relative to request url.
 346         GURL sdch_dictionary_url = request_->url().Resolve(url_text);
 347         if (sdch_dictionary_url.is_valid()) {
 348           rv = sdch_manager->OnGetDictionary(request_->url(),
 349                                              sdch_dictionary_url);
 350           if (rv != SDCH_OK) {
 351             SdchManager::SdchErrorRecovery(rv);
 352             request_->net_log().AddEvent(
 353                 NetLog::TYPE_SDCH_DICTIONARY_ERROR,
 354                 base::Bind(&NetLogSdchDictionaryFetchProblemCallback, rv,
 355                            sdch_dictionary_url, false));
 356           }
 357         }
 358       }
 359     }
 360   }
 361
 362   // Handle the server signalling no SDCH encoding.
 363   if (dictionaries_advertised_) {
 364     // We are wary of proxies that discard or damage SDCH encoding. If a server
 365     // explicitly states that this is not SDCH content, then we can correct our
 366     // assumption that this is an SDCH response, and avoid the need to recover
 367     // as though the content is corrupted (when we discover it is not SDCH
 368     // encoded).
 369     std::string sdch_response_status;
 370     void* iter = NULL;
 371     while (GetResponseHeaders()->EnumerateHeader(&iter, "X-Sdch-Encode",
 372                                                  &sdch_response_status)) {
 373       if (sdch_response_status == "0") {
 374         dictionaries_advertised_.reset();
 375         break;
 376       }
 377     }
 378   }
 379
 380   // The HTTP transaction may be restarted several times for the purposes
 381   // of sending authorization information. Each time it restarts, we get
 382   // notified of the headers completion so that we can update the cookie store.
 383   if (transaction_->IsReadyToRestartForAuth()) {
 384     DCHECK(!response_info_->auth_challenge.get());
 385     // TODO(battre): This breaks the webrequest API for
 386     // URLRequestTestHTTP.BasicAuthWithCookies
 387     // where OnBeforeSendHeaders -> OnSendHeaders -> OnBeforeSendHeaders
 388     // occurs.
 389     RestartTransactionWithAuth(AuthCredentials());
 390     return;
 391   }
 392
 393   URLRequestJob::NotifyHeadersComplete();
 394 }
 395
 396 void URLRequestHttpJob::NotifyDone(const URLRequestStatus& status) {
 397   DoneWithRequest(FINISHED);
 398   URLRequestJob::NotifyDone(status);
 399 }
 400
 401 void URLRequestHttpJob::DestroyTransaction() {
 402   DCHECK(transaction_.get());
 403
 404   DoneWithRequest(ABORTED);
 405   transaction_.reset();
 406   response_info_ = NULL;
 407   receive_headers_end_ = base::TimeTicks();
 408 }
 409
 410 void URLRequestHttpJob::StartTransaction() {
 411   // TODO(mmenke): Remove ScopedTracker below once crbug.com/456327 is fixed.
 412   tracked_objects::ScopedTracker tracking_profile(
 413       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 414           "456327 URLRequestHttpJob::StartTransaction"));
 415
 416   if (network_delegate()) {
 417     OnCallToDelegate();
 418     int rv = network_delegate()->NotifyBeforeSendHeaders(
 419         request_, notify_before_headers_sent_callback_,
 420         &request_info_.extra_headers);
 421     // If an extension blocks the request, we rely on the callback to
 422     // MaybeStartTransactionInternal().
 423     if (rv == ERR_IO_PENDING)
 424       return;
 425     MaybeStartTransactionInternal(rv);
 426     return;
 427   }
 428   StartTransactionInternal();
 429 }
 430
 431 void URLRequestHttpJob::NotifyBeforeSendHeadersCallback(int result) {
 432   // Check that there are no callbacks to already canceled requests.
 433   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 434
 435   MaybeStartTransactionInternal(result);
 436 }
 437
 438 void URLRequestHttpJob::MaybeStartTransactionInternal(int result) {
 439   // TODO(mmenke): Remove ScopedTracker below once crbug.com/456327 is fixed.
 440   tracked_objects::ScopedTracker tracking_profile(
 441       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 442           "456327 URLRequestHttpJob::MaybeStartTransactionInternal"));
 443
 444   OnCallToDelegateComplete();
 445   if (result == OK) {
 446     StartTransactionInternal();
 447   } else {
 448     std::string source("delegate");
 449     request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 450                                  NetLog::StringCallback("source", &source));
 451     NotifyCanceled();
 452     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 453   }
 454 }
 455
 456 void URLRequestHttpJob::StartTransactionInternal() {
 457   // NOTE: This method assumes that request_info_ is already setup properly.
 458
 459   // If we already have a transaction, then we should restart the transaction
 460   // with auth provided by auth_credentials_.
 461
 462   int rv;
 463
 464   if (network_delegate()) {
 465     network_delegate()->NotifySendHeaders(
 466         request_, request_info_.extra_headers);
 467   }
 468
 469   if (transaction_.get()) {
 470     rv = transaction_->RestartWithAuth(auth_credentials_, start_callback_);
 471     auth_credentials_ = AuthCredentials();
 472   } else {
 473     DCHECK(request_->context()->http_transaction_factory());
 474
 475     rv = request_->context()->http_transaction_factory()->CreateTransaction(
 476         priority_, &transaction_);
 477
 478     if (rv == OK && request_info_.url.SchemeIsWSOrWSS()) {
 479       base::SupportsUserData::Data* data = request_->GetUserData(
 480           WebSocketHandshakeStreamBase::CreateHelper::DataKey());
 481       if (data) {
 482         transaction_->SetWebSocketHandshakeStreamCreateHelper(
 483             static_cast<WebSocketHandshakeStreamBase::CreateHelper*>(data));
 484       } else {
 485         rv = ERR_DISALLOWED_URL_SCHEME;
 486       }
 487     }
 488
 489     if (rv == OK) {
 490       transaction_->SetBeforeNetworkStartCallback(
 491           base::Bind(&URLRequestHttpJob::NotifyBeforeNetworkStart,
 492                      base::Unretained(this)));
 493       transaction_->SetBeforeProxyHeadersSentCallback(
 494           base::Bind(&URLRequestHttpJob::NotifyBeforeSendProxyHeadersCallback,
 495                      base::Unretained(this)));
 496
 497       if (!throttling_entry_.get() ||
 498           !throttling_entry_->ShouldRejectRequest(*request_,
 499                                                   network_delegate())) {
 500         rv = transaction_->Start(
 501             &request_info_, start_callback_, request_->net_log());
 502         start_time_ = base::TimeTicks::Now();
 503       } else {
 504         // Special error code for the exponential back-off module.
 505         rv = ERR_TEMPORARILY_THROTTLED;
 506       }
 507     }
 508   }
 509
 510   if (rv == ERR_IO_PENDING)
 511     return;
 512
 513   // The transaction started synchronously, but we need to notify the
 514   // URLRequest delegate via the message loop.
 515   base::ThreadTaskRunnerHandle::Get()->PostTask(
 516       FROM_HERE, base::Bind(&URLRequestHttpJob::OnStartCompleted,
 517                             weak_factory_.GetWeakPtr(), rv));
 518 }
 519
 520 void URLRequestHttpJob::AddExtraHeaders() {
 521   SdchManager* sdch_manager = request()->context()->sdch_manager();
 522
 523   // Supply Accept-Encoding field only if it is not already provided.
 524   // It should be provided IF the content is known to have restrictions on
 525   // potential encoding, such as streaming multi-media.
 526   // For details see bug 47381.
 527   // TODO(jar, enal): jpeg files etc. should set up a request header if
 528   // possible. Right now it is done only by buffered_resource_loader and
 529   // simple_data_source.
 530   if (!request_info_.extra_headers.HasHeader(
 531       HttpRequestHeaders::kAcceptEncoding)) {
 532     // We don't support SDCH responses to POST as there is a possibility
 533     // of having SDCH encoded responses returned (e.g. by the cache)
 534     // which we cannot decode, and in those situations, we will need
 535     // to retransmit the request without SDCH, which is illegal for a POST.
 536     bool advertise_sdch = sdch_manager != NULL && request()->method() != "POST";
 537     if (advertise_sdch) {
 538       SdchProblemCode rv = sdch_manager->IsInSupportedDomain(request()->url());
 539       if (rv != SDCH_OK) {
 540         advertise_sdch = false;
 541         // If SDCH is just disabled, it is not a real error.
 542         if (rv != SDCH_DISABLED) {
 543           SdchManager::SdchErrorRecovery(rv);
 544           request()->net_log().AddEvent(
 545               NetLog::TYPE_SDCH_DECODING_ERROR,
 546               base::Bind(&NetLogSdchResourceProblemCallback, rv));
 547         }
 548       }
 549     }
 550     if (advertise_sdch) {
 551       dictionaries_advertised_ =
 552           sdch_manager->GetDictionarySet(request_->url());
 553     }
 554
 555     // The AllowLatencyExperiment() is only true if we've successfully done a
 556     // full SDCH compression recently in this browser session for this host.
 557     // Note that for this path, there might be no applicable dictionaries,
 558     // and hence we can't participate in the experiment.
 559     if (dictionaries_advertised_ &&
 560         sdch_manager->AllowLatencyExperiment(request_->url())) {
 561       // We are participating in the test (or control), and hence we'll
 562       // eventually record statistics via either SDCH_EXPERIMENT_DECODE or
 563       // SDCH_EXPERIMENT_HOLDBACK, and we'll need some packet timing data.
 564       packet_timing_enabled_ = true;
 565       if (base::RandDouble() < .01) {
 566         sdch_test_control_ = true;  // 1% probability.
 567         dictionaries_advertised_.reset();
 568         advertise_sdch = false;
 569       } else {
 570         sdch_test_activated_ = true;
 571       }
 572     }
 573
 574     // Supply Accept-Encoding headers first so that it is more likely that they
 575     // will be in the first transmitted packet. This can sometimes make it
 576     // easier to filter and analyze the streams to assure that a proxy has not
 577     // damaged these headers. Some proxies deliberately corrupt Accept-Encoding
 578     // headers.
 579     if (!advertise_sdch) {
 580       // Tell the server what compression formats we support (other than SDCH).
 581       request_info_.extra_headers.SetHeader(
 582           HttpRequestHeaders::kAcceptEncoding, "gzip, deflate");
 583     } else {
 584       // Include SDCH in acceptable list.
 585       request_info_.extra_headers.SetHeader(
 586           HttpRequestHeaders::kAcceptEncoding, "gzip, deflate, sdch");
 587       if (dictionaries_advertised_) {
 588         request_info_.extra_headers.SetHeader(
 589             kAvailDictionaryHeader,
 590             dictionaries_advertised_->GetDictionaryClientHashList());
 591         // Since we're tagging this transaction as advertising a dictionary,
 592         // we'll definitely employ an SDCH filter (or tentative sdch filter)
 593         // when we get a response. When done, we'll record histograms via
 594         // SDCH_DECODE or SDCH_PASSTHROUGH. Hence we need to record packet
 595         // arrival times.
 596         packet_timing_enabled_ = true;
 597       }
 598     }
 599   }
 600
 601   if (http_user_agent_settings_) {
 602     // Only add default Accept-Language if the request didn't have it
 603     // specified.
 604     std::string accept_language =
 605         http_user_agent_settings_->GetAcceptLanguage();
 606     if (!accept_language.empty()) {
 607       request_info_.extra_headers.SetHeaderIfMissing(
 608           HttpRequestHeaders::kAcceptLanguage,
 609           accept_language);
 610     }
 611   }
 612 }
 613
 614 void URLRequestHttpJob::AddCookieHeaderAndStart() {
 615   // No matter what, we want to report our status as IO pending since we will
 616   // be notifying our consumer asynchronously via OnStartCompleted.
 617   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
 618
 619   // If the request was destroyed, then there is no more work to do.
 620   if (!request_)
 621     return;
 622
 623   CookieStore* cookie_store = request_->context()->cookie_store();
 624   if (cookie_store && !(request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES)) {
 625     cookie_store->GetAllCookiesForURLAsync(
 626         request_->url(),
 627         base::Bind(&URLRequestHttpJob::CheckCookiePolicyAndLoad,
 628                    weak_factory_.GetWeakPtr()));
 629   } else {
 630     DoStartTransaction();
 631   }
 632 }
 633
 634 void URLRequestHttpJob::DoLoadCookies() {
 635   CookieOptions options;
 636   options.set_include_httponly();
 637
 638   // TODO(mkwst): Drop this `if` once we decide whether or not to ship
 639   // first-party cookies: https://crbug.com/459154
 640   if (network_delegate() &&
 641       network_delegate()->FirstPartyOnlyCookieExperimentEnabled())
 642     options.set_first_party_url(request_->first_party_for_cookies());
 643   else
 644     options.set_include_first_party_only();
 645
 646   request_->context()->cookie_store()->GetCookiesWithOptionsAsync(
 647       request_->url(), options, base::Bind(&URLRequestHttpJob::OnCookiesLoaded,
 648                                            weak_factory_.GetWeakPtr()));
 649 }
 650
 651 void URLRequestHttpJob::CheckCookiePolicyAndLoad(
 652     const CookieList& cookie_list) {
 653   if (CanGetCookies(cookie_list))
 654     DoLoadCookies();
 655   else
 656     DoStartTransaction();
 657 }
 658
 659 void URLRequestHttpJob::OnCookiesLoaded(const std::string& cookie_line) {
 660   if (!cookie_line.empty()) {
 661     request_info_.extra_headers.SetHeader(
 662         HttpRequestHeaders::kCookie, cookie_line);
 663     // Disable privacy mode as we are sending cookies anyway.
 664     request_info_.privacy_mode = PRIVACY_MODE_DISABLED;
 665   }
 666   DoStartTransaction();
 667 }
 668
 669 void URLRequestHttpJob::DoStartTransaction() {
 670   // We may have been canceled while retrieving cookies.
 671   if (GetStatus().is_success()) {
 672     StartTransaction();
 673   } else {
 674     NotifyCanceled();
 675   }
 676 }
 677
 678 void URLRequestHttpJob::SaveCookiesAndNotifyHeadersComplete(int result) {
 679   // End of the call started in OnStartCompleted.
 680   OnCallToDelegateComplete();
 681
 682   if (result != OK) {
 683     std::string source("delegate");
 684     request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 685                                  NetLog::StringCallback("source", &source));
 686     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 687     return;
 688   }
 689
 690   DCHECK(transaction_.get());
 691
 692   const HttpResponseInfo* response_info = transaction_->GetResponseInfo();
 693   DCHECK(response_info);
 694
 695   response_cookies_.clear();
 696   response_cookies_save_index_ = 0;
 697
 698   FetchResponseCookies(&response_cookies_);
 699
 700   if (!GetResponseHeaders()->GetDateValue(&response_date_))
 701     response_date_ = base::Time();
 702
 703   // Now, loop over the response cookies, and attempt to persist each.
 704   SaveNextCookie();
 705 }
 706
 707 // If the save occurs synchronously, SaveNextCookie will loop and save the next
 708 // cookie. If the save is deferred, the callback is responsible for continuing
 709 // to iterate through the cookies.
 710 // TODO(erikwright): Modify the CookieStore API to indicate via return value
 711 // whether it completed synchronously or asynchronously.
 712 // See http://crbug.com/131066.
 713 void URLRequestHttpJob::SaveNextCookie() {
 714   // No matter what, we want to report our status as IO pending since we will
 715   // be notifying our consumer asynchronously via OnStartCompleted.
 716   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
 717
 718   // Used to communicate with the callback. See the implementation of
 719   // OnCookieSaved.
 720   scoped_refptr<SharedBoolean> callback_pending = new SharedBoolean(false);
 721   scoped_refptr<SharedBoolean> save_next_cookie_running =
 722       new SharedBoolean(true);
 723
 724   if (!(request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) &&
 725       request_->context()->cookie_store() && response_cookies_.size() > 0) {
 726     CookieOptions options;
 727     options.set_include_httponly();
 728     options.set_server_time(response_date_);
 729
 730     CookieStore::SetCookiesCallback callback(base::Bind(
 731         &URLRequestHttpJob::OnCookieSaved, weak_factory_.GetWeakPtr(),
 732         save_next_cookie_running, callback_pending));
 733
 734     // Loop through the cookies as long as SetCookieWithOptionsAsync completes
 735     // synchronously.
 736     while (!callback_pending->data &&
 737            response_cookies_save_index_ < response_cookies_.size()) {
 738       if (CanSetCookie(
 739           response_cookies_[response_cookies_save_index_], &options)) {
 740         callback_pending->data = true;
 741         request_->context()->cookie_store()->SetCookieWithOptionsAsync(
 742             request_->url(), response_cookies_[response_cookies_save_index_],
 743             options, callback);
 744       }
 745       ++response_cookies_save_index_;
 746     }
 747   }
 748
 749   save_next_cookie_running->data = false;
 750
 751   if (!callback_pending->data) {
 752     response_cookies_.clear();
 753     response_cookies_save_index_ = 0;
 754     SetStatus(URLRequestStatus());  // Clear the IO_PENDING status
 755     NotifyHeadersComplete();
 756     return;
 757   }
 758 }
 759
 760 // |save_next_cookie_running| is true when the callback is bound and set to
 761 // false when SaveNextCookie exits, allowing the callback to determine if the
 762 // save occurred synchronously or asynchronously.
 763 // |callback_pending| is false when the callback is invoked and will be set to
 764 // true by the callback, allowing SaveNextCookie to detect whether the save
 765 // occurred synchronously.
 766 // See SaveNextCookie() for more information.
 767 void URLRequestHttpJob::OnCookieSaved(
 768     scoped_refptr<SharedBoolean> save_next_cookie_running,
 769     scoped_refptr<SharedBoolean> callback_pending,
 770     bool cookie_status) {
 771   callback_pending->data = false;
 772
 773   // If we were called synchronously, return.
 774   if (save_next_cookie_running->data) {
 775     return;
 776   }
 777
 778   // We were called asynchronously, so trigger the next save.
 779   // We may have been canceled within OnSetCookie.
 780   if (GetStatus().is_success()) {
 781     SaveNextCookie();
 782   } else {
 783     NotifyCanceled();
 784   }
 785 }
 786
 787 void URLRequestHttpJob::FetchResponseCookies(
 788     std::vector<std::string>* cookies) {
 789   const std::string name = "Set-Cookie";
 790   std::string value;
 791
 792   void* iter = NULL;
 793   HttpResponseHeaders* headers = GetResponseHeaders();
 794   while (headers->EnumerateHeader(&iter, name, &value)) {
 795     if (!value.empty())
 796       cookies->push_back(value);
 797   }
 798 }
 799
 800 // NOTE: |ProcessStrictTransportSecurityHeader| and
 801 // |ProcessPublicKeyPinsHeader| have very similar structures, by design.
 802 void URLRequestHttpJob::ProcessStrictTransportSecurityHeader() {
 803   DCHECK(response_info_);
 804   TransportSecurityState* security_state =
 805       request_->context()->transport_security_state();
 806   const SSLInfo& ssl_info = response_info_->ssl_info;
 807
 808   // Only accept HSTS headers on HTTPS connections that have no
 809   // certificate errors.
 810   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 811       !security_state)
 812     return;
 813
 814   // Don't accept HSTS headers when the hostname is an IP address.
 815   if (request_info_.url.HostIsIPAddress())
 816     return;
 817
 818   // http://tools.ietf.org/html/draft-ietf-websec-strict-transport-sec:
 819   //
 820   //   If a UA receives more than one STS header field in a HTTP response
 821   //   message over secure transport, then the UA MUST process only the
 822   //   first such header field.
 823   HttpResponseHeaders* headers = GetResponseHeaders();
 824   std::string value;
 825   if (headers->EnumerateHeader(NULL, "Strict-Transport-Security", &value))
 826     security_state->AddHSTSHeader(request_info_.url.host(), value);
 827 }
 828
 829 void URLRequestHttpJob::ProcessPublicKeyPinsHeader() {
 830   DCHECK(response_info_);
 831   TransportSecurityState* security_state =
 832       request_->context()->transport_security_state();
 833   const SSLInfo& ssl_info = response_info_->ssl_info;
 834
 835   // Only accept HPKP headers on HTTPS connections that have no
 836   // certificate errors.
 837   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 838       !security_state)
 839     return;
 840
 841   // Don't accept HSTS headers when the hostname is an IP address.
 842   if (request_info_.url.HostIsIPAddress())
 843     return;
 844
 845   // http://tools.ietf.org/html/draft-ietf-websec-key-pinning:
 846   //
 847   //   If a UA receives more than one PKP header field in an HTTP
 848   //   response message over secure transport, then the UA MUST process
 849   //   only the first such header field.
 850   HttpResponseHeaders* headers = GetResponseHeaders();
 851   std::string value;
 852   if (headers->EnumerateHeader(NULL, "Public-Key-Pins", &value))
 853     security_state->AddHPKPHeader(request_info_.url.host(), value, ssl_info);
 854 }
 855
 856 void URLRequestHttpJob::OnStartCompleted(int result) {
 857   RecordTimer();
 858
 859   // If the request was destroyed, then there is no more work to do.
 860   if (!request_)
 861     return;
 862
 863   // If the job is done (due to cancellation), can just ignore this
 864   // notification.
 865   if (done_)
 866     return;
 867
 868   receive_headers_end_ = base::TimeTicks::Now();
 869
 870   // Clear the IO_PENDING status
 871   SetStatus(URLRequestStatus());
 872
 873   const URLRequestContext* context = request_->context();
 874
 875   if (result == ERR_SSL_PINNED_KEY_NOT_IN_CERT_CHAIN &&
 876       transaction_->GetResponseInfo() != NULL) {
 877     FraudulentCertificateReporter* reporter =
 878       context->fraudulent_certificate_reporter();
 879     if (reporter != NULL) {
 880       const SSLInfo& ssl_info = transaction_->GetResponseInfo()->ssl_info;
 881       const std::string& host = request_->url().host();
 882
 883       reporter->SendReport(host, ssl_info);
 884     }
 885   }
 886
 887   if (result == OK) {
 888     if (transaction_ && transaction_->GetResponseInfo()) {
 889       SetProxyServer(transaction_->GetResponseInfo()->proxy_server);
 890     }
 891     scoped_refptr<HttpResponseHeaders> headers = GetResponseHeaders();
 892     if (network_delegate()) {
 893       // Note that |this| may not be deleted until
 894       // |on_headers_received_callback_| or
 895       // |NetworkDelegate::URLRequestDestroyed()| has been called.
 896       OnCallToDelegate();
 897       allowed_unsafe_redirect_url_ = GURL();
 898       int error = network_delegate()->NotifyHeadersReceived(
 899           request_,
 900           on_headers_received_callback_,
 901           headers.get(),
 902           &override_response_headers_,
 903           &allowed_unsafe_redirect_url_);
 904       if (error != OK) {
 905         if (error == ERR_IO_PENDING) {
 906           awaiting_callback_ = true;
 907         } else {
 908           std::string source("delegate");
 909           request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 910                                        NetLog::StringCallback("source",
 911                                                               &source));
 912           OnCallToDelegateComplete();
 913           NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, error));
 914         }
 915         return;
 916       }
 917     }
 918
 919     SaveCookiesAndNotifyHeadersComplete(OK);
 920   } else if (IsCertificateError(result)) {
 921     // We encountered an SSL certificate error.
 922     if (result == ERR_SSL_WEAK_SERVER_EPHEMERAL_DH_KEY ||
 923         result == ERR_SSL_PINNED_KEY_NOT_IN_CERT_CHAIN) {
 924       // These are hard failures. They're handled separately and don't have
 925       // the correct cert status, so set it here.
 926       SSLInfo info(transaction_->GetResponseInfo()->ssl_info);
 927       info.cert_status = MapNetErrorToCertStatus(result);
 928       NotifySSLCertificateError(info, true);
 929     } else {
 930       // Maybe overridable, maybe not. Ask the delegate to decide.
 931       TransportSecurityState* state = context->transport_security_state();
 932       const bool fatal =
 933           state && state->ShouldSSLErrorsBeFatal(request_info_.url.host());
 934       NotifySSLCertificateError(
 935           transaction_->GetResponseInfo()->ssl_info, fatal);
 936     }
 937   } else if (result == ERR_SSL_CLIENT_AUTH_CERT_NEEDED) {
 938     NotifyCertificateRequested(
 939         transaction_->GetResponseInfo()->cert_request_info.get());
 940   } else {
 941     // Even on an error, there may be useful information in the response
 942     // info (e.g. whether there's a cached copy).
 943     if (transaction_.get())
 944       response_info_ = transaction_->GetResponseInfo();
 945     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 946   }
 947 }
 948
 949 void URLRequestHttpJob::OnHeadersReceivedCallback(int result) {
 950   awaiting_callback_ = false;
 951
 952   // Check that there are no callbacks to already canceled requests.
 953   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 954
 955   SaveCookiesAndNotifyHeadersComplete(result);
 956 }
 957
 958 void URLRequestHttpJob::OnReadCompleted(int result) {
 959   read_in_progress_ = false;
 960
 961   if (ShouldFixMismatchedContentLength(result))
 962     result = OK;
 963
 964   if (result == OK) {
 965     NotifyDone(URLRequestStatus());
 966   } else if (result < 0) {
 967     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, result));
 968   } else {
 969     // Clear the IO_PENDING status
 970     SetStatus(URLRequestStatus());
 971   }
 972
 973   NotifyReadComplete(result);
 974 }
 975
 976 void URLRequestHttpJob::RestartTransactionWithAuth(
 977     const AuthCredentials& credentials) {
 978   auth_credentials_ = credentials;
 979
 980   // These will be reset in OnStartCompleted.
 981   response_info_ = NULL;
 982   receive_headers_end_ = base::TimeTicks();
 983   response_cookies_.clear();
 984
 985   ResetTimer();
 986
 987   // Update the cookies, since the cookie store may have been updated from the
 988   // headers in the 401/407. Since cookies were already appended to
 989   // extra_headers, we need to strip them out before adding them again.
 990   request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kCookie);
 991
 992   AddCookieHeaderAndStart();
 993 }
 994
 995 void URLRequestHttpJob::SetUpload(UploadDataStream* upload) {
 996   DCHECK(!transaction_.get()) << "cannot change once started";
 997   request_info_.upload_data_stream = upload;
 998 }
 999
1000 void URLRequestHttpJob::SetExtraRequestHeaders(
1001     const HttpRequestHeaders& headers) {
1002   DCHECK(!transaction_.get()) << "cannot change once started";
1003   request_info_.extra_headers.CopyFrom(headers);
1004 }
1005
1006 LoadState URLRequestHttpJob::GetLoadState() const {
1007   return transaction_.get() ?
1008       transaction_->GetLoadState() : LOAD_STATE_IDLE;
1009 }
1010
1011 UploadProgress URLRequestHttpJob::GetUploadProgress() const {
1012   return transaction_.get() ?
1013       transaction_->GetUploadProgress() : UploadProgress();
1014 }
1015
1016 bool URLRequestHttpJob::GetMimeType(std::string* mime_type) const {
1017   DCHECK(transaction_.get());
1018
1019   if (!response_info_)
1020     return false;
1021
1022   HttpResponseHeaders* headers = GetResponseHeaders();
1023   if (!headers)
1024     return false;
1025   return headers->GetMimeType(mime_type);
1026 }
1027
1028 bool URLRequestHttpJob::GetCharset(std::string* charset) {
1029   DCHECK(transaction_.get());
1030
1031   if (!response_info_)
1032     return false;
1033
1034   return GetResponseHeaders()->GetCharset(charset);
1035 }
1036
1037 void URLRequestHttpJob::GetResponseInfo(HttpResponseInfo* info) {
1038   DCHECK(request_);
1039
1040   if (response_info_) {
1041     DCHECK(transaction_.get());
1042
1043     *info = *response_info_;
1044     if (override_response_headers_.get())
1045       info->headers = override_response_headers_;
1046   }
1047 }
1048
1049 void URLRequestHttpJob::GetLoadTimingInfo(
1050     LoadTimingInfo* load_timing_info) const {
1051   // If haven't made it far enough to receive any headers, don't return
1052   // anything. This makes for more consistent behavior in the case of errors.
1053   if (!transaction_ || receive_headers_end_.is_null())
1054     return;
1055   if (transaction_->GetLoadTimingInfo(load_timing_info))
1056     load_timing_info->receive_headers_end = receive_headers_end_;
1057 }
1058
1059 bool URLRequestHttpJob::GetResponseCookies(std::vector<std::string>* cookies) {
1060   DCHECK(transaction_.get());
1061
1062   if (!response_info_)
1063     return false;
1064
1065   // TODO(darin): Why are we extracting response cookies again?  Perhaps we
1066   // should just leverage response_cookies_.
1067
1068   cookies->clear();
1069   FetchResponseCookies(cookies);
1070   return true;
1071 }
1072
1073 int URLRequestHttpJob::GetResponseCode() const {
1074   DCHECK(transaction_.get());
1075
1076   if (!response_info_)
1077     return -1;
1078
1079   return GetResponseHeaders()->response_code();
1080 }
1081
1082 Filter* URLRequestHttpJob::SetupFilter() const {
1083   DCHECK(transaction_.get());
1084   if (!response_info_)
1085     return NULL;
1086
1087   std::vector<Filter::FilterType> encoding_types;
1088   std::string encoding_type;
1089   HttpResponseHeaders* headers = GetResponseHeaders();
1090   void* iter = NULL;
1091   while (headers->EnumerateHeader(&iter, "Content-Encoding", &encoding_type)) {
1092     encoding_types.push_back(Filter::ConvertEncodingToType(encoding_type));
1093   }
1094
1095   // Even if encoding types are empty, there is a chance that we need to add
1096   // some decoding, as some proxies strip encoding completely. In such cases,
1097   // we may need to add (for example) SDCH filtering (when the context suggests
1098   // it is appropriate).
1099   Filter::FixupEncodingTypes(*filter_context_, &encoding_types);
1100
1101   return !encoding_types.empty()
1102       ? Filter::Factory(encoding_types, *filter_context_) : NULL;
1103 }
1104
1105 bool URLRequestHttpJob::CopyFragmentOnRedirect(const GURL& location) const {
1106   // Allow modification of reference fragments by default, unless
1107   // |allowed_unsafe_redirect_url_| is set and equal to the redirect URL.
1108   // When this is the case, we assume that the network delegate has set the
1109   // desired redirect URL (with or without fragment), so it must not be changed
1110   // any more.
1111   return !allowed_unsafe_redirect_url_.is_valid() ||
1112        allowed_unsafe_redirect_url_ != location;
1113 }
1114
1115 bool URLRequestHttpJob::IsSafeRedirect(const GURL& location) {
1116   // HTTP is always safe.
1117   // TODO(pauljensen): Remove once crbug.com/146591 is fixed.
1118   if (location.is_valid() &&
1119       (location.scheme() == "http" || location.scheme() == "https")) {
1120     return true;
1121   }
1122   // Delegates may mark a URL as safe for redirection.
1123   if (allowed_unsafe_redirect_url_.is_valid() &&
1124       allowed_unsafe_redirect_url_ == location) {
1125     return true;
1126   }
1127   // Query URLRequestJobFactory as to whether |location| would be safe to
1128   // redirect to.
1129   return request_->context()->job_factory() &&
1130       request_->context()->job_factory()->IsSafeRedirectTarget(location);
1131 }
1132
1133 bool URLRequestHttpJob::NeedsAuth() {
1134   int code = GetResponseCode();
1135   if (code == -1)
1136     return false;
1137
1138   // Check if we need either Proxy or WWW Authentication. This could happen
1139   // because we either provided no auth info, or provided incorrect info.
1140   switch (code) {
1141     case 407:
1142       if (proxy_auth_state_ == AUTH_STATE_CANCELED)
1143         return false;
1144       proxy_auth_state_ = AUTH_STATE_NEED_AUTH;
1145       return true;
1146     case 401:
1147       if (server_auth_state_ == AUTH_STATE_CANCELED)
1148         return false;
1149       server_auth_state_ = AUTH_STATE_NEED_AUTH;
1150       return true;
1151   }
1152   return false;
1153 }
1154
1155 void URLRequestHttpJob::GetAuthChallengeInfo(
1156     scoped_refptr<AuthChallengeInfo>* result) {
1157   DCHECK(transaction_.get());
1158   DCHECK(response_info_);
1159
1160   // sanity checks:
1161   DCHECK(proxy_auth_state_ == AUTH_STATE_NEED_AUTH ||
1162          server_auth_state_ == AUTH_STATE_NEED_AUTH);
1163   DCHECK((GetResponseHeaders()->response_code() == HTTP_UNAUTHORIZED) ||
1164          (GetResponseHeaders()->response_code() ==
1165           HTTP_PROXY_AUTHENTICATION_REQUIRED));
1166
1167   *result = response_info_->auth_challenge;
1168 }
1169
1170 void URLRequestHttpJob::SetAuth(const AuthCredentials& credentials) {
1171   DCHECK(transaction_.get());
1172
1173   // Proxy gets set first, then WWW.
1174   if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1175     proxy_auth_state_ = AUTH_STATE_HAVE_AUTH;
1176   } else {
1177     DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1178     server_auth_state_ = AUTH_STATE_HAVE_AUTH;
1179   }
1180
1181   RestartTransactionWithAuth(credentials);
1182 }
1183
1184 void URLRequestHttpJob::CancelAuth() {
1185   // Proxy gets set first, then WWW.
1186   if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1187     proxy_auth_state_ = AUTH_STATE_CANCELED;
1188   } else {
1189     DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1190     server_auth_state_ = AUTH_STATE_CANCELED;
1191   }
1192
1193   // These will be reset in OnStartCompleted.
1194   response_info_ = NULL;
1195   receive_headers_end_ = base::TimeTicks::Now();
1196   response_cookies_.clear();
1197
1198   ResetTimer();
1199
1200   // OK, let the consumer read the error page...
1201   //
1202   // Because we set the AUTH_STATE_CANCELED flag, NeedsAuth will return false,
1203   // which will cause the consumer to receive OnResponseStarted instead of
1204   // OnAuthRequired.
1205   //
1206   // We have to do this via InvokeLater to avoid "recursing" the consumer.
1207   //
1208   base::ThreadTaskRunnerHandle::Get()->PostTask(
1209       FROM_HERE, base::Bind(&URLRequestHttpJob::OnStartCompleted,
1210                             weak_factory_.GetWeakPtr(), OK));
1211 }
1212
1213 void URLRequestHttpJob::ContinueWithCertificate(
1214     X509Certificate* client_cert) {
1215   DCHECK(transaction_.get());
1216
1217   DCHECK(!response_info_) << "should not have a response yet";
1218   receive_headers_end_ = base::TimeTicks();
1219
1220   ResetTimer();
1221
1222   // No matter what, we want to report our status as IO pending since we will
1223   // be notifying our consumer asynchronously via OnStartCompleted.
1224   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1225
1226   int rv = transaction_->RestartWithCertificate(client_cert, start_callback_);
1227   if (rv == ERR_IO_PENDING)
1228     return;
1229
1230   // The transaction started synchronously, but we need to notify the
1231   // URLRequest delegate via the message loop.
1232   base::ThreadTaskRunnerHandle::Get()->PostTask(
1233       FROM_HERE, base::Bind(&URLRequestHttpJob::OnStartCompleted,
1234                             weak_factory_.GetWeakPtr(), rv));
1235 }
1236
1237 void URLRequestHttpJob::ContinueDespiteLastError() {
1238   // If the transaction was destroyed, then the job was cancelled.
1239   if (!transaction_.get())
1240     return;
1241
1242   DCHECK(!response_info_) << "should not have a response yet";
1243   receive_headers_end_ = base::TimeTicks();
1244
1245   ResetTimer();
1246
1247   // No matter what, we want to report our status as IO pending since we will
1248   // be notifying our consumer asynchronously via OnStartCompleted.
1249   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1250
1251   int rv = transaction_->RestartIgnoringLastError(start_callback_);
1252   if (rv == ERR_IO_PENDING)
1253     return;
1254
1255   // The transaction started synchronously, but we need to notify the
1256   // URLRequest delegate via the message loop.
1257   base::ThreadTaskRunnerHandle::Get()->PostTask(
1258       FROM_HERE, base::Bind(&URLRequestHttpJob::OnStartCompleted,
1259                             weak_factory_.GetWeakPtr(), rv));
1260 }
1261
1262 void URLRequestHttpJob::ResumeNetworkStart() {
1263   DCHECK(transaction_.get());
1264   transaction_->ResumeNetworkStart();
1265 }
1266
1267 bool URLRequestHttpJob::ShouldFixMismatchedContentLength(int rv) const {
1268   // Some servers send the body compressed, but specify the content length as
1269   // the uncompressed size. Although this violates the HTTP spec we want to
1270   // support it (as IE and FireFox do), but *only* for an exact match.
1271   // See http://crbug.com/79694.
1272   if (rv == ERR_CONTENT_LENGTH_MISMATCH ||
1273       rv == ERR_INCOMPLETE_CHUNKED_ENCODING) {
1274     if (request_ && request_->response_headers()) {
1275       int64 expected_length = request_->response_headers()->GetContentLength();
1276       VLOG(1) << __FUNCTION__ << "() "
1277               << "\"" << request_->url().spec() << "\""
1278               << " content-length = " << expected_length
1279               << " pre total = " << prefilter_bytes_read()
1280               << " post total = " << postfilter_bytes_read();
1281       if (postfilter_bytes_read() == expected_length) {
1282         // Clear the error.
1283         return true;
1284       }
1285     }
1286   }
1287   return false;
1288 }
1289
1290 bool URLRequestHttpJob::ReadRawData(IOBuffer* buf, int buf_size,
1291                                     int* bytes_read) {
1292   DCHECK_NE(buf_size, 0);
1293   DCHECK(bytes_read);
1294   DCHECK(!read_in_progress_);
1295
1296   int rv = transaction_->Read(
1297       buf, buf_size,
1298       base::Bind(&URLRequestHttpJob::OnReadCompleted, base::Unretained(this)));
1299
1300   if (ShouldFixMismatchedContentLength(rv))
1301     rv = 0;
1302
1303   if (rv >= 0) {
1304     *bytes_read = rv;
1305     if (!rv)
1306       DoneWithRequest(FINISHED);
1307     return true;
1308   }
1309
1310   if (rv == ERR_IO_PENDING) {
1311     read_in_progress_ = true;
1312     SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1313   } else {
1314     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, rv));
1315   }
1316
1317   return false;
1318 }
1319
1320 void URLRequestHttpJob::StopCaching() {
1321   if (transaction_.get())
1322     transaction_->StopCaching();
1323 }
1324
1325 bool URLRequestHttpJob::GetFullRequestHeaders(
1326     HttpRequestHeaders* headers) const {
1327   if (!transaction_)
1328     return false;
1329
1330   return transaction_->GetFullRequestHeaders(headers);
1331 }
1332
1333 int64 URLRequestHttpJob::GetTotalReceivedBytes() const {
1334   if (!transaction_)
1335     return 0;
1336
1337   return transaction_->GetTotalReceivedBytes();
1338 }
1339
1340 void URLRequestHttpJob::DoneReading() {
1341   if (transaction_) {
1342     transaction_->DoneReading();
1343   }
1344   DoneWithRequest(FINISHED);
1345 }
1346
1347 void URLRequestHttpJob::DoneReadingRedirectResponse() {
1348   if (transaction_) {
1349     if (transaction_->GetResponseInfo()->headers->IsRedirect(NULL)) {
1350       // If the original headers indicate a redirect, go ahead and cache the
1351       // response, even if the |override_response_headers_| are a redirect to
1352       // another location.
1353       transaction_->DoneReading();
1354     } else {
1355       // Otherwise, |override_response_headers_| must be non-NULL and contain
1356       // bogus headers indicating a redirect.
1357       DCHECK(override_response_headers_.get());
1358       DCHECK(override_response_headers_->IsRedirect(NULL));
1359       transaction_->StopCaching();
1360     }
1361   }
1362   DoneWithRequest(FINISHED);
1363 }
1364
1365 HostPortPair URLRequestHttpJob::GetSocketAddress() const {
1366   return response_info_ ? response_info_->socket_address : HostPortPair();
1367 }
1368
1369 void URLRequestHttpJob::RecordTimer() {
1370   if (request_creation_time_.is_null()) {
1371     NOTREACHED()
1372         << "The same transaction shouldn't start twice without new timing.";
1373     return;
1374   }
1375
1376   base::TimeDelta to_start = base::Time::Now() - request_creation_time_;
1377   request_creation_time_ = base::Time();
1378
1379   UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpTimeToFirstByte", to_start);
1380 }
1381
1382 void URLRequestHttpJob::ResetTimer() {
1383   if (!request_creation_time_.is_null()) {
1384     NOTREACHED()
1385         << "The timer was reset before it was recorded.";
1386     return;
1387   }
1388   request_creation_time_ = base::Time::Now();
1389 }
1390
1391 void URLRequestHttpJob::UpdatePacketReadTimes() {
1392   if (!packet_timing_enabled_)
1393     return;
1394
1395   DCHECK_GT(prefilter_bytes_read(), bytes_observed_in_packets_);
1396
1397   base::Time now(base::Time::Now());
1398   if (!bytes_observed_in_packets_)
1399     request_time_snapshot_ = now;
1400   final_packet_time_ = now;
1401
1402   bytes_observed_in_packets_ = prefilter_bytes_read();
1403 }
1404
1405 void URLRequestHttpJob::RecordPacketStats(
1406     FilterContext::StatisticSelector statistic) const {
1407   if (!packet_timing_enabled_ || (final_packet_time_ == base::Time()))
1408     return;
1409
1410   base::TimeDelta duration = final_packet_time_ - request_time_snapshot_;
1411   switch (statistic) {
1412     case FilterContext::SDCH_DECODE: {
1413       UMA_HISTOGRAM_CUSTOM_COUNTS("Sdch3.Network_Decode_Bytes_Processed_b",
1414           static_cast<int>(bytes_observed_in_packets_), 500, 100000, 100);
1415       return;
1416     }
1417     case FilterContext::SDCH_PASSTHROUGH: {
1418       // Despite advertising a dictionary, we handled non-sdch compressed
1419       // content.
1420       return;
1421     }
1422
1423     case FilterContext::SDCH_EXPERIMENT_DECODE: {
1424       UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment3_Decode",
1425                                   duration,
1426                                   base::TimeDelta::FromMilliseconds(20),
1427                                   base::TimeDelta::FromMinutes(10), 100);
1428       return;
1429     }
1430     case FilterContext::SDCH_EXPERIMENT_HOLDBACK: {
1431       UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment3_Holdback",
1432                                   duration,
1433                                   base::TimeDelta::FromMilliseconds(20),
1434                                   base::TimeDelta::FromMinutes(10), 100);
1435       return;
1436     }
1437     default:
1438       NOTREACHED();
1439       return;
1440   }
1441 }
1442
1443 void URLRequestHttpJob::RecordPerfHistograms(CompletionCause reason) {
1444   if (start_time_.is_null())
1445     return;
1446
1447   base::TimeDelta total_time = base::TimeTicks::Now() - start_time_;
1448   UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTime", total_time);
1449
1450   if (reason == FINISHED) {
1451     UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeSuccess", total_time);
1452   } else {
1453     UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCancel", total_time);
1454   }
1455
1456   if (response_info_) {
1457     bool is_google = request() && HasGoogleHost(request()->url());
1458     bool used_quic = response_info_->DidUseQuic();
1459     if (is_google) {
1460       if (used_quic) {
1461         UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTime.Quic", total_time);
1462       } else {
1463         UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTime.NotQuic", total_time);
1464       }
1465     }
1466     if (response_info_->was_cached) {
1467       UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCached", total_time);
1468       if (is_google) {
1469         if (used_quic) {
1470           UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTimeCached.Quic",
1471                                      total_time);
1472         } else {
1473           UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTimeCached.NotQuic",
1474                                      total_time);
1475         }
1476       }
1477     } else  {
1478       UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeNotCached", total_time);
1479       if (is_google) {
1480         if (used_quic) {
1481           UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTimeNotCached.Quic",
1482                                      total_time);
1483         } else {
1484           UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTimeNotCached.NotQuic",
1485                                      total_time);
1486         }
1487       }
1488     }
1489   }
1490
1491   if (request_info_.load_flags & LOAD_PREFETCH && !request_->was_cached())
1492     UMA_HISTOGRAM_COUNTS("Net.Prefetch.PrefilterBytesReadFromNetwork",
1493                          prefilter_bytes_read());
1494
1495   start_time_ = base::TimeTicks();
1496 }
1497
1498 void URLRequestHttpJob::DoneWithRequest(CompletionCause reason) {
1499   if (done_)
1500     return;
1501   done_ = true;
1502   RecordPerfHistograms(reason);
1503   if (reason == FINISHED) {
1504     request_->set_received_response_content_length(prefilter_bytes_read());
1505   }
1506 }
1507
1508 HttpResponseHeaders* URLRequestHttpJob::GetResponseHeaders() const {
1509   DCHECK(transaction_.get());
1510   DCHECK(transaction_->GetResponseInfo());
1511   return override_response_headers_.get() ?
1512              override_response_headers_.get() :
1513              transaction_->GetResponseInfo()->headers.get();
1514 }
1515
1516 void URLRequestHttpJob::NotifyURLRequestDestroyed() {
1517   awaiting_callback_ = false;
1518 }
1519
1520 }  // namespace net