net/url_request/url_request_http_job.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_request_http_job.h"
   6
   7 #include "base/base_switches.h"
   8 #include "base/bind.h"
   9 #include "base/bind_helpers.h"
  10 #include "base/command_line.h"
  11 #include "base/compiler_specific.h"
  12 #include "base/file_version_info.h"
  13 #include "base/location.h"
  14 #include "base/metrics/field_trial.h"
  15 #include "base/metrics/histogram_macros.h"
  16 #include "base/profiler/scoped_tracker.h"
  17 #include "base/rand_util.h"
  18 #include "base/single_thread_task_runner.h"
  19 #include "base/strings/string_util.h"
  20 #include "base/thread_task_runner_handle.h"
  21 #include "base/time/time.h"
  22 #include "base/values.h"
  23 #include "net/base/host_port_pair.h"
  24 #include "net/base/load_flags.h"
  25 #include "net/base/net_errors.h"
  26 #include "net/base/net_util.h"
  27 #include "net/base/network_delegate.h"
  28 #include "net/base/network_quality_estimator.h"
  29 #include "net/base/sdch_manager.h"
  30 #include "net/base/sdch_net_log_params.h"
  31 #include "net/cert/cert_status_flags.h"
  32 #include "net/cookies/cookie_store.h"
  33 #include "net/http/http_content_disposition.h"
  34 #include "net/http/http_network_session.h"
  35 #include "net/http/http_request_headers.h"
  36 #include "net/http/http_response_headers.h"
  37 #include "net/http/http_response_info.h"
  38 #include "net/http/http_status_code.h"
  39 #include "net/http/http_transaction.h"
  40 #include "net/http/http_transaction_factory.h"
  41 #include "net/http/http_util.h"
  42 #include "net/proxy/proxy_info.h"
  43 #include "net/ssl/ssl_cert_request_info.h"
  44 #include "net/ssl/ssl_config_service.h"
  45 #include "net/url_request/fraudulent_certificate_reporter.h"
  46 #include "net/url_request/http_user_agent_settings.h"
  47 #include "net/url_request/url_request.h"
  48 #include "net/url_request/url_request_backoff_manager.h"
  49 #include "net/url_request/url_request_context.h"
  50 #include "net/url_request/url_request_error_job.h"
  51 #include "net/url_request/url_request_job_factory.h"
  52 #include "net/url_request/url_request_redirect_job.h"
  53 #include "net/url_request/url_request_throttler_manager.h"
  54 #include "net/websockets/websocket_handshake_stream_base.h"
  55
  56 static const char kAvailDictionaryHeader[] = "Avail-Dictionary";
  57
  58 namespace net {
  59
  60 class URLRequestHttpJob::HttpFilterContext : public FilterContext {
  61  public:
  62   explicit HttpFilterContext(URLRequestHttpJob* job);
  63   ~HttpFilterContext() override;
  64
  65   // FilterContext implementation.
  66   bool GetMimeType(std::string* mime_type) const override;
  67   bool GetURL(GURL* gurl) const override;
  68   base::Time GetRequestTime() const override;
  69   bool IsCachedContent() const override;
  70   SdchManager::DictionarySet* SdchDictionariesAdvertised() const override;
  71   int64 GetByteReadCount() const override;
  72   int GetResponseCode() const override;
  73   const URLRequestContext* GetURLRequestContext() const override;
  74   void RecordPacketStats(StatisticSelector statistic) const override;
  75   const BoundNetLog& GetNetLog() const override;
  76
  77  private:
  78   URLRequestHttpJob* job_;
  79
  80   // URLRequestHttpJob may be detached from URLRequest, but we still need to
  81   // return something.
  82   BoundNetLog dummy_log_;
  83
  84   DISALLOW_COPY_AND_ASSIGN(HttpFilterContext);
  85 };
  86
  87 URLRequestHttpJob::HttpFilterContext::HttpFilterContext(URLRequestHttpJob* job)
  88     : job_(job) {
  89   DCHECK(job_);
  90 }
  91
  92 URLRequestHttpJob::HttpFilterContext::~HttpFilterContext() {
  93 }
  94
  95 bool URLRequestHttpJob::HttpFilterContext::GetMimeType(
  96     std::string* mime_type) const {
  97   return job_->GetMimeType(mime_type);
  98 }
  99
 100 bool URLRequestHttpJob::HttpFilterContext::GetURL(GURL* gurl) const {
 101   if (!job_->request())
 102     return false;
 103   *gurl = job_->request()->url();
 104   return true;
 105 }
 106
 107 base::Time URLRequestHttpJob::HttpFilterContext::GetRequestTime() const {
 108   return job_->request() ? job_->request()->request_time() : base::Time();
 109 }
 110
 111 bool URLRequestHttpJob::HttpFilterContext::IsCachedContent() const {
 112   return job_->is_cached_content_;
 113 }
 114
 115 SdchManager::DictionarySet*
 116 URLRequestHttpJob::HttpFilterContext::SdchDictionariesAdvertised() const {
 117   return job_->dictionaries_advertised_.get();
 118 }
 119
 120 int64 URLRequestHttpJob::HttpFilterContext::GetByteReadCount() const {
 121   return job_->prefilter_bytes_read();
 122 }
 123
 124 int URLRequestHttpJob::HttpFilterContext::GetResponseCode() const {
 125   return job_->GetResponseCode();
 126 }
 127
 128 const URLRequestContext*
 129 URLRequestHttpJob::HttpFilterContext::GetURLRequestContext() const {
 130   return job_->request() ? job_->request()->context() : NULL;
 131 }
 132
 133 void URLRequestHttpJob::HttpFilterContext::RecordPacketStats(
 134     StatisticSelector statistic) const {
 135   job_->RecordPacketStats(statistic);
 136 }
 137
 138 const BoundNetLog& URLRequestHttpJob::HttpFilterContext::GetNetLog() const {
 139   return job_->request() ? job_->request()->net_log() : dummy_log_;
 140 }
 141
 142 // TODO(darin): make sure the port blocking code is not lost
 143 // static
 144 URLRequestJob* URLRequestHttpJob::Factory(URLRequest* request,
 145                                           NetworkDelegate* network_delegate,
 146                                           const std::string& scheme) {
 147   DCHECK(scheme == "http" || scheme == "https" || scheme == "ws" ||
 148          scheme == "wss");
 149
 150   if (!request->context()->http_transaction_factory()) {
 151     NOTREACHED() << "requires a valid context";
 152     return new URLRequestErrorJob(
 153         request, network_delegate, ERR_INVALID_ARGUMENT);
 154   }
 155
 156   GURL redirect_url;
 157   if (request->GetHSTSRedirect(&redirect_url)) {
 158     return new URLRequestRedirectJob(
 159         request, network_delegate, redirect_url,
 160         // Use status code 307 to preserve the method, so POST requests work.
 161         URLRequestRedirectJob::REDIRECT_307_TEMPORARY_REDIRECT, "HSTS");
 162   }
 163   return new URLRequestHttpJob(request,
 164                                network_delegate,
 165                                request->context()->http_user_agent_settings());
 166 }
 167
 168 URLRequestHttpJob::URLRequestHttpJob(
 169     URLRequest* request,
 170     NetworkDelegate* network_delegate,
 171     const HttpUserAgentSettings* http_user_agent_settings)
 172     : URLRequestJob(request, network_delegate),
 173       priority_(DEFAULT_PRIORITY),
 174       response_info_(NULL),
 175       response_cookies_save_index_(0),
 176       proxy_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
 177       server_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
 178       start_callback_(base::Bind(&URLRequestHttpJob::OnStartCompleted,
 179                                  base::Unretained(this))),
 180       notify_before_headers_sent_callback_(
 181           base::Bind(&URLRequestHttpJob::NotifyBeforeSendHeadersCallback,
 182                      base::Unretained(this))),
 183       read_in_progress_(false),
 184       throttling_entry_(NULL),
 185       sdch_test_activated_(false),
 186       sdch_test_control_(false),
 187       is_cached_content_(false),
 188       request_creation_time_(),
 189       packet_timing_enabled_(false),
 190       done_(false),
 191       bytes_observed_in_packets_(0),
 192       request_time_snapshot_(),
 193       final_packet_time_(),
 194       filter_context_(new HttpFilterContext(this)),
 195       on_headers_received_callback_(
 196           base::Bind(&URLRequestHttpJob::OnHeadersReceivedCallback,
 197                      base::Unretained(this))),
 198       awaiting_callback_(false),
 199       http_user_agent_settings_(http_user_agent_settings),
 200       backoff_manager_(request->context()->backoff_manager()),
 201       weak_factory_(this) {
 202   URLRequestThrottlerManager* manager = request->context()->throttler_manager();
 203   if (manager)
 204     throttling_entry_ = manager->RegisterRequestUrl(request->url());
 205
 206   ResetTimer();
 207 }
 208
 209 URLRequestHttpJob::~URLRequestHttpJob() {
 210   CHECK(!awaiting_callback_);
 211
 212   DCHECK(!sdch_test_control_ || !sdch_test_activated_);
 213   if (!is_cached_content_) {
 214     if (sdch_test_control_)
 215       RecordPacketStats(FilterContext::SDCH_EXPERIMENT_HOLDBACK);
 216     if (sdch_test_activated_)
 217       RecordPacketStats(FilterContext::SDCH_EXPERIMENT_DECODE);
 218   }
 219   // Make sure SDCH filters are told to emit histogram data while
 220   // filter_context_ is still alive.
 221   DestroyFilters();
 222
 223   DoneWithRequest(ABORTED);
 224 }
 225
 226 void URLRequestHttpJob::SetPriority(RequestPriority priority) {
 227   priority_ = priority;
 228   if (transaction_)
 229     transaction_->SetPriority(priority_);
 230 }
 231
 232 void URLRequestHttpJob::Start() {
 233   // TODO(mmenke): Remove ScopedTracker below once crbug.com/456327 is fixed.
 234   tracked_objects::ScopedTracker tracking_profile(
 235       FROM_HERE_WITH_EXPLICIT_FUNCTION("456327 URLRequestHttpJob::Start"));
 236
 237   DCHECK(!transaction_.get());
 238
 239   // URLRequest::SetReferrer ensures that we do not send username and password
 240   // fields in the referrer.
 241   GURL referrer(request_->referrer());
 242
 243   request_info_.url = request_->url();
 244   request_info_.method = request_->method();
 245   request_info_.load_flags = request_->load_flags();
 246   // Enable privacy mode if cookie settings or flags tell us not send or
 247   // save cookies.
 248   bool enable_privacy_mode =
 249       (request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES) ||
 250       (request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) ||
 251       CanEnablePrivacyMode();
 252   // Privacy mode could still be disabled in OnCookiesLoaded if we are going
 253   // to send previously saved cookies.
 254   request_info_.privacy_mode = enable_privacy_mode ?
 255       PRIVACY_MODE_ENABLED : PRIVACY_MODE_DISABLED;
 256
 257   // Strip Referer from request_info_.extra_headers to prevent, e.g., plugins
 258   // from overriding headers that are controlled using other means. Otherwise a
 259   // plugin could set a referrer although sending the referrer is inhibited.
 260   request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kReferer);
 261
 262   // Our consumer should have made sure that this is a safe referrer. See for
 263   // instance WebCore::FrameLoader::HideReferrer.
 264   if (referrer.is_valid()) {
 265     request_info_.extra_headers.SetHeader(HttpRequestHeaders::kReferer,
 266                                           referrer.spec());
 267   }
 268
 269   request_info_.extra_headers.SetHeaderIfMissing(
 270       HttpRequestHeaders::kUserAgent,
 271       http_user_agent_settings_ ?
 272           http_user_agent_settings_->GetUserAgent() : std::string());
 273
 274   AddExtraHeaders();
 275   AddCookieHeaderAndStart();
 276 }
 277
 278 void URLRequestHttpJob::Kill() {
 279   if (!transaction_.get())
 280     return;
 281
 282   weak_factory_.InvalidateWeakPtrs();
 283   DestroyTransaction();
 284   URLRequestJob::Kill();
 285 }
 286
 287 void URLRequestHttpJob::GetConnectionAttempts(ConnectionAttempts* out) const {
 288   if (transaction_)
 289     transaction_->GetConnectionAttempts(out);
 290   else
 291     out->clear();
 292 }
 293
 294 void URLRequestHttpJob::NotifyBeforeSendProxyHeadersCallback(
 295     const ProxyInfo& proxy_info,
 296     HttpRequestHeaders* request_headers) {
 297   DCHECK(request_headers);
 298   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 299   if (network_delegate()) {
 300     network_delegate()->NotifyBeforeSendProxyHeaders(
 301         request_,
 302         proxy_info,
 303         request_headers);
 304   }
 305 }
 306
 307 void URLRequestHttpJob::NotifyBeforeNetworkStart(bool* defer) {
 308   if (!request_)
 309     return;
 310   if (backoff_manager_) {
 311     if (backoff_manager_->ShouldRejectRequest(request()->url(),
 312                                               request()->request_time())) {
 313       *defer = true;
 314       base::MessageLoop::current()->PostTask(
 315           FROM_HERE,
 316           base::Bind(&URLRequestHttpJob::OnStartCompleted,
 317                      weak_factory_.GetWeakPtr(), ERR_TEMPORARY_BACKOFF));
 318       return;
 319     }
 320   }
 321   URLRequestJob::NotifyBeforeNetworkStart(defer);
 322 }
 323
 324 void URLRequestHttpJob::NotifyHeadersComplete() {
 325   DCHECK(!response_info_);
 326
 327   response_info_ = transaction_->GetResponseInfo();
 328
 329   // Save boolean, as we'll need this info at destruction time, and filters may
 330   // also need this info.
 331   is_cached_content_ = response_info_->was_cached;
 332
 333   if (!is_cached_content_ && throttling_entry_.get())
 334     throttling_entry_->UpdateWithResponse(GetResponseCode());
 335
 336   if (!is_cached_content_)
 337     ProcessBackoffHeader();
 338
 339   // The ordering of these calls is not important.
 340   ProcessStrictTransportSecurityHeader();
 341   ProcessPublicKeyPinsHeader();
 342
 343   // Handle the server notification of a new SDCH dictionary.
 344   SdchManager* sdch_manager(request()->context()->sdch_manager());
 345   if (sdch_manager) {
 346     SdchProblemCode rv = sdch_manager->IsInSupportedDomain(request()->url());
 347     if (rv != SDCH_OK) {
 348       // If SDCH is just disabled, it is not a real error.
 349       if (rv != SDCH_DISABLED) {
 350         SdchManager::SdchErrorRecovery(rv);
 351         request()->net_log().AddEvent(
 352             NetLog::TYPE_SDCH_DECODING_ERROR,
 353             base::Bind(&NetLogSdchResourceProblemCallback, rv));
 354       }
 355     } else {
 356       const std::string name = "Get-Dictionary";
 357       std::string url_text;
 358       void* iter = NULL;
 359       // TODO(jar): We need to not fetch dictionaries the first time they are
 360       // seen, but rather wait until we can justify their usefulness.
 361       // For now, we will only fetch the first dictionary, which will at least
 362       // require multiple suggestions before we get additional ones for this
 363       // site. Eventually we should wait until a dictionary is requested
 364       // several times
 365       // before we even download it (so that we don't waste memory or
 366       // bandwidth).
 367       if (GetResponseHeaders()->EnumerateHeader(&iter, name, &url_text)) {
 368         // Resolve suggested URL relative to request url.
 369         GURL sdch_dictionary_url = request_->url().Resolve(url_text);
 370         if (sdch_dictionary_url.is_valid()) {
 371           rv = sdch_manager->OnGetDictionary(request_->url(),
 372                                              sdch_dictionary_url);
 373           if (rv != SDCH_OK) {
 374             SdchManager::SdchErrorRecovery(rv);
 375             request_->net_log().AddEvent(
 376                 NetLog::TYPE_SDCH_DICTIONARY_ERROR,
 377                 base::Bind(&NetLogSdchDictionaryFetchProblemCallback, rv,
 378                            sdch_dictionary_url, false));
 379           }
 380         }
 381       }
 382     }
 383   }
 384
 385   // Handle the server signalling no SDCH encoding.
 386   if (dictionaries_advertised_) {
 387     // We are wary of proxies that discard or damage SDCH encoding. If a server
 388     // explicitly states that this is not SDCH content, then we can correct our
 389     // assumption that this is an SDCH response, and avoid the need to recover
 390     // as though the content is corrupted (when we discover it is not SDCH
 391     // encoded).
 392     std::string sdch_response_status;
 393     void* iter = NULL;
 394     while (GetResponseHeaders()->EnumerateHeader(&iter, "X-Sdch-Encode",
 395                                                  &sdch_response_status)) {
 396       if (sdch_response_status == "0") {
 397         dictionaries_advertised_.reset();
 398         break;
 399       }
 400     }
 401   }
 402
 403   // The HTTP transaction may be restarted several times for the purposes
 404   // of sending authorization information. Each time it restarts, we get
 405   // notified of the headers completion so that we can update the cookie store.
 406   if (transaction_->IsReadyToRestartForAuth()) {
 407     DCHECK(!response_info_->auth_challenge.get());
 408     // TODO(battre): This breaks the webrequest API for
 409     // URLRequestTestHTTP.BasicAuthWithCookies
 410     // where OnBeforeSendHeaders -> OnSendHeaders -> OnBeforeSendHeaders
 411     // occurs.
 412     RestartTransactionWithAuth(AuthCredentials());
 413     return;
 414   }
 415
 416   URLRequestJob::NotifyHeadersComplete();
 417 }
 418
 419 void URLRequestHttpJob::NotifyDone(const URLRequestStatus& status) {
 420   DoneWithRequest(FINISHED);
 421   URLRequestJob::NotifyDone(status);
 422 }
 423
 424 void URLRequestHttpJob::DestroyTransaction() {
 425   DCHECK(transaction_.get());
 426
 427   DoneWithRequest(ABORTED);
 428   transaction_.reset();
 429   response_info_ = NULL;
 430   receive_headers_end_ = base::TimeTicks();
 431 }
 432
 433 void URLRequestHttpJob::StartTransaction() {
 434   // TODO(mmenke): Remove ScopedTracker below once crbug.com/456327 is fixed.
 435   tracked_objects::ScopedTracker tracking_profile(
 436       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 437           "456327 URLRequestHttpJob::StartTransaction"));
 438
 439   if (network_delegate()) {
 440     OnCallToDelegate();
 441     int rv = network_delegate()->NotifyBeforeSendHeaders(
 442         request_, notify_before_headers_sent_callback_,
 443         &request_info_.extra_headers);
 444     // If an extension blocks the request, we rely on the callback to
 445     // MaybeStartTransactionInternal().
 446     if (rv == ERR_IO_PENDING)
 447       return;
 448     MaybeStartTransactionInternal(rv);
 449     return;
 450   }
 451   StartTransactionInternal();
 452 }
 453
 454 void URLRequestHttpJob::NotifyBeforeSendHeadersCallback(int result) {
 455   // Check that there are no callbacks to already canceled requests.
 456   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 457
 458   MaybeStartTransactionInternal(result);
 459 }
 460
 461 void URLRequestHttpJob::MaybeStartTransactionInternal(int result) {
 462   // TODO(mmenke): Remove ScopedTracker below once crbug.com/456327 is fixed.
 463   tracked_objects::ScopedTracker tracking_profile(
 464       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 465           "456327 URLRequestHttpJob::MaybeStartTransactionInternal"));
 466
 467   OnCallToDelegateComplete();
 468   if (result == OK) {
 469     StartTransactionInternal();
 470   } else {
 471     std::string source("delegate");
 472     request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 473                                  NetLog::StringCallback("source", &source));
 474     NotifyCanceled();
 475     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 476   }
 477 }
 478
 479 void URLRequestHttpJob::StartTransactionInternal() {
 480   // NOTE: This method assumes that request_info_ is already setup properly.
 481
 482   // If we already have a transaction, then we should restart the transaction
 483   // with auth provided by auth_credentials_.
 484
 485   int rv;
 486
 487   if (network_delegate()) {
 488     network_delegate()->NotifySendHeaders(
 489         request_, request_info_.extra_headers);
 490   }
 491
 492   if (transaction_.get()) {
 493     rv = transaction_->RestartWithAuth(auth_credentials_, start_callback_);
 494     auth_credentials_ = AuthCredentials();
 495   } else {
 496     DCHECK(request_->context()->http_transaction_factory());
 497
 498     rv = request_->context()->http_transaction_factory()->CreateTransaction(
 499         priority_, &transaction_);
 500
 501     if (rv == OK && request_info_.url.SchemeIsWSOrWSS()) {
 502       base::SupportsUserData::Data* data = request_->GetUserData(
 503           WebSocketHandshakeStreamBase::CreateHelper::DataKey());
 504       if (data) {
 505         transaction_->SetWebSocketHandshakeStreamCreateHelper(
 506             static_cast<WebSocketHandshakeStreamBase::CreateHelper*>(data));
 507       } else {
 508         rv = ERR_DISALLOWED_URL_SCHEME;
 509       }
 510     }
 511
 512     if (rv == OK) {
 513       transaction_->SetBeforeNetworkStartCallback(
 514           base::Bind(&URLRequestHttpJob::NotifyBeforeNetworkStart,
 515                      base::Unretained(this)));
 516       transaction_->SetBeforeProxyHeadersSentCallback(
 517           base::Bind(&URLRequestHttpJob::NotifyBeforeSendProxyHeadersCallback,
 518                      base::Unretained(this)));
 519
 520       if (!throttling_entry_.get() ||
 521           !throttling_entry_->ShouldRejectRequest(*request_)) {
 522         rv = transaction_->Start(
 523             &request_info_, start_callback_, request_->net_log());
 524         start_time_ = base::TimeTicks::Now();
 525       } else {
 526         // Special error code for the exponential back-off module.
 527         rv = ERR_TEMPORARILY_THROTTLED;
 528       }
 529     }
 530   }
 531
 532   if (rv == ERR_IO_PENDING)
 533     return;
 534
 535   // The transaction started synchronously, but we need to notify the
 536   // URLRequest delegate via the message loop.
 537   base::ThreadTaskRunnerHandle::Get()->PostTask(
 538       FROM_HERE, base::Bind(&URLRequestHttpJob::OnStartCompleted,
 539                             weak_factory_.GetWeakPtr(), rv));
 540 }
 541
 542 void URLRequestHttpJob::AddExtraHeaders() {
 543   SdchManager* sdch_manager = request()->context()->sdch_manager();
 544
 545   // Supply Accept-Encoding field only if it is not already provided.
 546   // It should be provided IF the content is known to have restrictions on
 547   // potential encoding, such as streaming multi-media.
 548   // For details see bug 47381.
 549   // TODO(jar, enal): jpeg files etc. should set up a request header if
 550   // possible. Right now it is done only by buffered_resource_loader and
 551   // simple_data_source.
 552   if (!request_info_.extra_headers.HasHeader(
 553       HttpRequestHeaders::kAcceptEncoding)) {
 554     // We don't support SDCH responses to POST as there is a possibility
 555     // of having SDCH encoded responses returned (e.g. by the cache)
 556     // which we cannot decode, and in those situations, we will need
 557     // to retransmit the request without SDCH, which is illegal for a POST.
 558     bool advertise_sdch = sdch_manager != NULL && request()->method() != "POST";
 559     if (advertise_sdch) {
 560       SdchProblemCode rv = sdch_manager->IsInSupportedDomain(request()->url());
 561       if (rv != SDCH_OK) {
 562         advertise_sdch = false;
 563         // If SDCH is just disabled, it is not a real error.
 564         if (rv != SDCH_DISABLED) {
 565           SdchManager::SdchErrorRecovery(rv);
 566           request()->net_log().AddEvent(
 567               NetLog::TYPE_SDCH_DECODING_ERROR,
 568               base::Bind(&NetLogSdchResourceProblemCallback, rv));
 569         }
 570       }
 571     }
 572     if (advertise_sdch) {
 573       dictionaries_advertised_ =
 574           sdch_manager->GetDictionarySet(request_->url());
 575     }
 576
 577     // The AllowLatencyExperiment() is only true if we've successfully done a
 578     // full SDCH compression recently in this browser session for this host.
 579     // Note that for this path, there might be no applicable dictionaries,
 580     // and hence we can't participate in the experiment.
 581     if (dictionaries_advertised_ &&
 582         sdch_manager->AllowLatencyExperiment(request_->url())) {
 583       // We are participating in the test (or control), and hence we'll
 584       // eventually record statistics via either SDCH_EXPERIMENT_DECODE or
 585       // SDCH_EXPERIMENT_HOLDBACK, and we'll need some packet timing data.
 586       packet_timing_enabled_ = true;
 587       if (base::RandDouble() < .01) {
 588         sdch_test_control_ = true;  // 1% probability.
 589         dictionaries_advertised_.reset();
 590         advertise_sdch = false;
 591       } else {
 592         sdch_test_activated_ = true;
 593       }
 594     }
 595
 596     // Supply Accept-Encoding headers first so that it is more likely that they
 597     // will be in the first transmitted packet. This can sometimes make it
 598     // easier to filter and analyze the streams to assure that a proxy has not
 599     // damaged these headers. Some proxies deliberately corrupt Accept-Encoding
 600     // headers.
 601     if (!advertise_sdch) {
 602       // Tell the server what compression formats we support (other than SDCH).
 603       request_info_.extra_headers.SetHeader(
 604           HttpRequestHeaders::kAcceptEncoding, "gzip, deflate");
 605     } else {
 606       // Include SDCH in acceptable list.
 607       request_info_.extra_headers.SetHeader(
 608           HttpRequestHeaders::kAcceptEncoding, "gzip, deflate, sdch");
 609       if (dictionaries_advertised_) {
 610         request_info_.extra_headers.SetHeader(
 611             kAvailDictionaryHeader,
 612             dictionaries_advertised_->GetDictionaryClientHashList());
 613         // Since we're tagging this transaction as advertising a dictionary,
 614         // we'll definitely employ an SDCH filter (or tentative sdch filter)
 615         // when we get a response. When done, we'll record histograms via
 616         // SDCH_DECODE or SDCH_PASSTHROUGH. Hence we need to record packet
 617         // arrival times.
 618         packet_timing_enabled_ = true;
 619       }
 620     }
 621   }
 622
 623   if (http_user_agent_settings_) {
 624     // Only add default Accept-Language if the request didn't have it
 625     // specified.
 626     std::string accept_language =
 627         http_user_agent_settings_->GetAcceptLanguage();
 628     if (!accept_language.empty()) {
 629       request_info_.extra_headers.SetHeaderIfMissing(
 630           HttpRequestHeaders::kAcceptLanguage,
 631           accept_language);
 632     }
 633   }
 634 }
 635
 636 void URLRequestHttpJob::AddCookieHeaderAndStart() {
 637   // No matter what, we want to report our status as IO pending since we will
 638   // be notifying our consumer asynchronously via OnStartCompleted.
 639   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
 640
 641   // If the request was destroyed, then there is no more work to do.
 642   if (!request_)
 643     return;
 644
 645   CookieStore* cookie_store = request_->context()->cookie_store();
 646   if (cookie_store && !(request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES)) {
 647     cookie_store->GetAllCookiesForURLAsync(
 648         request_->url(),
 649         base::Bind(&URLRequestHttpJob::CheckCookiePolicyAndLoad,
 650                    weak_factory_.GetWeakPtr()));
 651   } else {
 652     DoStartTransaction();
 653   }
 654 }
 655
 656 void URLRequestHttpJob::DoLoadCookies() {
 657   CookieOptions options;
 658   options.set_include_httponly();
 659
 660   // TODO(mkwst): Drop this `if` once we decide whether or not to ship
 661   // first-party cookies: https://crbug.com/459154
 662   if (network_delegate() &&
 663       network_delegate()->FirstPartyOnlyCookieExperimentEnabled())
 664     options.set_first_party_url(request_->first_party_for_cookies());
 665   else
 666     options.set_include_first_party_only();
 667
 668   request_->context()->cookie_store()->GetCookiesWithOptionsAsync(
 669       request_->url(), options, base::Bind(&URLRequestHttpJob::OnCookiesLoaded,
 670                                            weak_factory_.GetWeakPtr()));
 671 }
 672
 673 void URLRequestHttpJob::CheckCookiePolicyAndLoad(
 674     const CookieList& cookie_list) {
 675   if (CanGetCookies(cookie_list))
 676     DoLoadCookies();
 677   else
 678     DoStartTransaction();
 679 }
 680
 681 void URLRequestHttpJob::OnCookiesLoaded(const std::string& cookie_line) {
 682   if (!cookie_line.empty()) {
 683     request_info_.extra_headers.SetHeader(
 684         HttpRequestHeaders::kCookie, cookie_line);
 685     // Disable privacy mode as we are sending cookies anyway.
 686     request_info_.privacy_mode = PRIVACY_MODE_DISABLED;
 687   }
 688   DoStartTransaction();
 689 }
 690
 691 void URLRequestHttpJob::DoStartTransaction() {
 692   // We may have been canceled while retrieving cookies.
 693   if (GetStatus().is_success()) {
 694     StartTransaction();
 695   } else {
 696     NotifyCanceled();
 697   }
 698 }
 699
 700 void URLRequestHttpJob::SaveCookiesAndNotifyHeadersComplete(int result) {
 701   // End of the call started in OnStartCompleted.
 702   OnCallToDelegateComplete();
 703
 704   if (result != OK) {
 705     std::string source("delegate");
 706     request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 707                                  NetLog::StringCallback("source", &source));
 708     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 709     return;
 710   }
 711
 712   DCHECK(transaction_.get());
 713
 714   const HttpResponseInfo* response_info = transaction_->GetResponseInfo();
 715   DCHECK(response_info);
 716
 717   response_cookies_.clear();
 718   response_cookies_save_index_ = 0;
 719
 720   FetchResponseCookies(&response_cookies_);
 721
 722   if (!GetResponseHeaders()->GetDateValue(&response_date_))
 723     response_date_ = base::Time();
 724
 725   // Now, loop over the response cookies, and attempt to persist each.
 726   SaveNextCookie();
 727 }
 728
 729 // If the save occurs synchronously, SaveNextCookie will loop and save the next
 730 // cookie. If the save is deferred, the callback is responsible for continuing
 731 // to iterate through the cookies.
 732 // TODO(erikwright): Modify the CookieStore API to indicate via return value
 733 // whether it completed synchronously or asynchronously.
 734 // See http://crbug.com/131066.
 735 void URLRequestHttpJob::SaveNextCookie() {
 736   // No matter what, we want to report our status as IO pending since we will
 737   // be notifying our consumer asynchronously via OnStartCompleted.
 738   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
 739
 740   // Used to communicate with the callback. See the implementation of
 741   // OnCookieSaved.
 742   scoped_refptr<SharedBoolean> callback_pending = new SharedBoolean(false);
 743   scoped_refptr<SharedBoolean> save_next_cookie_running =
 744       new SharedBoolean(true);
 745
 746   if (!(request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) &&
 747       request_->context()->cookie_store() && response_cookies_.size() > 0) {
 748     CookieOptions options;
 749     options.set_include_httponly();
 750     options.set_server_time(response_date_);
 751
 752     CookieStore::SetCookiesCallback callback(base::Bind(
 753         &URLRequestHttpJob::OnCookieSaved, weak_factory_.GetWeakPtr(),
 754         save_next_cookie_running, callback_pending));
 755
 756     // Loop through the cookies as long as SetCookieWithOptionsAsync completes
 757     // synchronously.
 758     while (!callback_pending->data &&
 759            response_cookies_save_index_ < response_cookies_.size()) {
 760       if (CanSetCookie(
 761           response_cookies_[response_cookies_save_index_], &options)) {
 762         callback_pending->data = true;
 763         request_->context()->cookie_store()->SetCookieWithOptionsAsync(
 764             request_->url(), response_cookies_[response_cookies_save_index_],
 765             options, callback);
 766       }
 767       ++response_cookies_save_index_;
 768     }
 769   }
 770
 771   save_next_cookie_running->data = false;
 772
 773   if (!callback_pending->data) {
 774     response_cookies_.clear();
 775     response_cookies_save_index_ = 0;
 776     SetStatus(URLRequestStatus());  // Clear the IO_PENDING status
 777     NotifyHeadersComplete();
 778     return;
 779   }
 780 }
 781
 782 // |save_next_cookie_running| is true when the callback is bound and set to
 783 // false when SaveNextCookie exits, allowing the callback to determine if the
 784 // save occurred synchronously or asynchronously.
 785 // |callback_pending| is false when the callback is invoked and will be set to
 786 // true by the callback, allowing SaveNextCookie to detect whether the save
 787 // occurred synchronously.
 788 // See SaveNextCookie() for more information.
 789 void URLRequestHttpJob::OnCookieSaved(
 790     scoped_refptr<SharedBoolean> save_next_cookie_running,
 791     scoped_refptr<SharedBoolean> callback_pending,
 792     bool cookie_status) {
 793   callback_pending->data = false;
 794
 795   // If we were called synchronously, return.
 796   if (save_next_cookie_running->data) {
 797     return;
 798   }
 799
 800   // We were called asynchronously, so trigger the next save.
 801   // We may have been canceled within OnSetCookie.
 802   if (GetStatus().is_success()) {
 803     SaveNextCookie();
 804   } else {
 805     NotifyCanceled();
 806   }
 807 }
 808
 809 void URLRequestHttpJob::FetchResponseCookies(
 810     std::vector<std::string>* cookies) {
 811   const std::string name = "Set-Cookie";
 812   std::string value;
 813
 814   void* iter = NULL;
 815   HttpResponseHeaders* headers = GetResponseHeaders();
 816   while (headers->EnumerateHeader(&iter, name, &value)) {
 817     if (!value.empty())
 818       cookies->push_back(value);
 819   }
 820 }
 821
 822 void URLRequestHttpJob::ProcessBackoffHeader() {
 823   DCHECK(response_info_);
 824
 825   if (!backoff_manager_)
 826     return;
 827
 828   TransportSecurityState* security_state =
 829       request_->context()->transport_security_state();
 830   const SSLInfo& ssl_info = response_info_->ssl_info;
 831
 832   // Only accept Backoff headers on HTTPS connections that have no
 833   // certificate errors.
 834   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 835       !security_state)
 836     return;
 837
 838   backoff_manager_->UpdateWithResponse(request()->url(), GetResponseHeaders(),
 839                                        base::Time::Now());
 840 }
 841
 842 // NOTE: |ProcessStrictTransportSecurityHeader| and
 843 // |ProcessPublicKeyPinsHeader| have very similar structures, by design.
 844 void URLRequestHttpJob::ProcessStrictTransportSecurityHeader() {
 845   DCHECK(response_info_);
 846   TransportSecurityState* security_state =
 847       request_->context()->transport_security_state();
 848   const SSLInfo& ssl_info = response_info_->ssl_info;
 849
 850   // Only accept HSTS headers on HTTPS connections that have no
 851   // certificate errors.
 852   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 853       !security_state)
 854     return;
 855
 856   // Don't accept HSTS headers when the hostname is an IP address.
 857   if (request_info_.url.HostIsIPAddress())
 858     return;
 859
 860   // http://tools.ietf.org/html/draft-ietf-websec-strict-transport-sec:
 861   //
 862   //   If a UA receives more than one STS header field in a HTTP response
 863   //   message over secure transport, then the UA MUST process only the
 864   //   first such header field.
 865   HttpResponseHeaders* headers = GetResponseHeaders();
 866   std::string value;
 867   if (headers->EnumerateHeader(NULL, "Strict-Transport-Security", &value))
 868     security_state->AddHSTSHeader(request_info_.url.host(), value);
 869 }
 870
 871 void URLRequestHttpJob::ProcessPublicKeyPinsHeader() {
 872   DCHECK(response_info_);
 873   TransportSecurityState* security_state =
 874       request_->context()->transport_security_state();
 875   const SSLInfo& ssl_info = response_info_->ssl_info;
 876
 877   // Only accept HPKP headers on HTTPS connections that have no
 878   // certificate errors.
 879   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 880       !security_state)
 881     return;
 882
 883   // Don't accept HSTS headers when the hostname is an IP address.
 884   if (request_info_.url.HostIsIPAddress())
 885     return;
 886
 887   // http://tools.ietf.org/html/rfc7469:
 888   //
 889   //   If a UA receives more than one PKP header field in an HTTP
 890   //   response message over secure transport, then the UA MUST process
 891   //   only the first such header field.
 892   HttpResponseHeaders* headers = GetResponseHeaders();
 893   std::string value;
 894   if (headers->EnumerateHeader(nullptr, "Public-Key-Pins", &value))
 895     security_state->AddHPKPHeader(request_info_.url.host(), value, ssl_info);
 896   if (headers->EnumerateHeader(nullptr, "Public-Key-Pins-Report-Only",
 897                                &value)) {
 898     security_state->ProcessHPKPReportOnlyHeader(
 899         value, HostPortPair::FromURL(request_info_.url), ssl_info);
 900   }
 901 }
 902
 903 void URLRequestHttpJob::OnStartCompleted(int result) {
 904   RecordTimer();
 905
 906   // If the request was destroyed, then there is no more work to do.
 907   if (!request_)
 908     return;
 909
 910   // If the job is done (due to cancellation), can just ignore this
 911   // notification.
 912   if (done_)
 913     return;
 914
 915   receive_headers_end_ = base::TimeTicks::Now();
 916
 917   // Clear the IO_PENDING status
 918   SetStatus(URLRequestStatus());
 919
 920   const URLRequestContext* context = request_->context();
 921
 922   if (result == ERR_SSL_PINNED_KEY_NOT_IN_CERT_CHAIN &&
 923       transaction_->GetResponseInfo() != NULL) {
 924     FraudulentCertificateReporter* reporter =
 925       context->fraudulent_certificate_reporter();
 926     if (reporter != NULL) {
 927       const SSLInfo& ssl_info = transaction_->GetResponseInfo()->ssl_info;
 928       const std::string& host = request_->url().host();
 929
 930       reporter->SendReport(host, ssl_info);
 931     }
 932   }
 933
 934   if (result == OK) {
 935     if (transaction_ && transaction_->GetResponseInfo()) {
 936       SetProxyServer(transaction_->GetResponseInfo()->proxy_server);
 937     }
 938     scoped_refptr<HttpResponseHeaders> headers = GetResponseHeaders();
 939     if (network_delegate()) {
 940       // Note that |this| may not be deleted until
 941       // |on_headers_received_callback_| or
 942       // |NetworkDelegate::URLRequestDestroyed()| has been called.
 943       OnCallToDelegate();
 944       allowed_unsafe_redirect_url_ = GURL();
 945       int error = network_delegate()->NotifyHeadersReceived(
 946           request_,
 947           on_headers_received_callback_,
 948           headers.get(),
 949           &override_response_headers_,
 950           &allowed_unsafe_redirect_url_);
 951       if (error != OK) {
 952         if (error == ERR_IO_PENDING) {
 953           awaiting_callback_ = true;
 954         } else {
 955           std::string source("delegate");
 956           request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 957                                        NetLog::StringCallback("source",
 958                                                               &source));
 959           OnCallToDelegateComplete();
 960           NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, error));
 961         }
 962         return;
 963       }
 964     }
 965
 966     SaveCookiesAndNotifyHeadersComplete(OK);
 967   } else if (IsCertificateError(result)) {
 968     // We encountered an SSL certificate error.
 969     if (result == ERR_SSL_WEAK_SERVER_EPHEMERAL_DH_KEY ||
 970         result == ERR_SSL_PINNED_KEY_NOT_IN_CERT_CHAIN) {
 971       // These are hard failures. They're handled separately and don't have
 972       // the correct cert status, so set it here.
 973       SSLInfo info(transaction_->GetResponseInfo()->ssl_info);
 974       info.cert_status = MapNetErrorToCertStatus(result);
 975       NotifySSLCertificateError(info, true);
 976     } else {
 977       // Maybe overridable, maybe not. Ask the delegate to decide.
 978       TransportSecurityState* state = context->transport_security_state();
 979       const bool fatal =
 980           state && state->ShouldSSLErrorsBeFatal(request_info_.url.host());
 981       NotifySSLCertificateError(
 982           transaction_->GetResponseInfo()->ssl_info, fatal);
 983     }
 984   } else if (result == ERR_SSL_CLIENT_AUTH_CERT_NEEDED) {
 985     NotifyCertificateRequested(
 986         transaction_->GetResponseInfo()->cert_request_info.get());
 987   } else {
 988     // Even on an error, there may be useful information in the response
 989     // info (e.g. whether there's a cached copy).
 990     if (transaction_.get())
 991       response_info_ = transaction_->GetResponseInfo();
 992     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 993   }
 994 }
 995
 996 void URLRequestHttpJob::OnHeadersReceivedCallback(int result) {
 997   awaiting_callback_ = false;
 998
 999   // Check that there are no callbacks to already canceled requests.
1000   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
1001
1002   SaveCookiesAndNotifyHeadersComplete(result);
1003 }
1004
1005 void URLRequestHttpJob::OnReadCompleted(int result) {
1006   read_in_progress_ = false;
1007
1008   if (ShouldFixMismatchedContentLength(result))
1009     result = OK;
1010
1011   if (result == OK) {
1012     NotifyDone(URLRequestStatus());
1013   } else if (result < 0) {
1014     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, result));
1015   } else {
1016     // Clear the IO_PENDING status
1017     SetStatus(URLRequestStatus());
1018   }
1019
1020   NotifyReadComplete(result);
1021 }
1022
1023 void URLRequestHttpJob::RestartTransactionWithAuth(
1024     const AuthCredentials& credentials) {
1025   auth_credentials_ = credentials;
1026
1027   // These will be reset in OnStartCompleted.
1028   response_info_ = NULL;
1029   receive_headers_end_ = base::TimeTicks();
1030   response_cookies_.clear();
1031
1032   ResetTimer();
1033
1034   // Update the cookies, since the cookie store may have been updated from the
1035   // headers in the 401/407. Since cookies were already appended to
1036   // extra_headers, we need to strip them out before adding them again.
1037   request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kCookie);
1038
1039   AddCookieHeaderAndStart();
1040 }
1041
1042 void URLRequestHttpJob::SetUpload(UploadDataStream* upload) {
1043   DCHECK(!transaction_.get()) << "cannot change once started";
1044   request_info_.upload_data_stream = upload;
1045 }
1046
1047 void URLRequestHttpJob::SetExtraRequestHeaders(
1048     const HttpRequestHeaders& headers) {
1049   DCHECK(!transaction_.get()) << "cannot change once started";
1050   request_info_.extra_headers.CopyFrom(headers);
1051 }
1052
1053 LoadState URLRequestHttpJob::GetLoadState() const {
1054   return transaction_.get() ?
1055       transaction_->GetLoadState() : LOAD_STATE_IDLE;
1056 }
1057
1058 UploadProgress URLRequestHttpJob::GetUploadProgress() const {
1059   return transaction_.get() ?
1060       transaction_->GetUploadProgress() : UploadProgress();
1061 }
1062
1063 bool URLRequestHttpJob::GetMimeType(std::string* mime_type) const {
1064   DCHECK(transaction_.get());
1065
1066   if (!response_info_)
1067     return false;
1068
1069   HttpResponseHeaders* headers = GetResponseHeaders();
1070   if (!headers)
1071     return false;
1072   return headers->GetMimeType(mime_type);
1073 }
1074
1075 bool URLRequestHttpJob::GetCharset(std::string* charset) {
1076   DCHECK(transaction_.get());
1077
1078   if (!response_info_)
1079     return false;
1080
1081   return GetResponseHeaders()->GetCharset(charset);
1082 }
1083
1084 void URLRequestHttpJob::GetResponseInfo(HttpResponseInfo* info) {
1085   DCHECK(request_);
1086
1087   if (response_info_) {
1088     DCHECK(transaction_.get());
1089
1090     *info = *response_info_;
1091     if (override_response_headers_.get())
1092       info->headers = override_response_headers_;
1093   }
1094 }
1095
1096 void URLRequestHttpJob::GetLoadTimingInfo(
1097     LoadTimingInfo* load_timing_info) const {
1098   // If haven't made it far enough to receive any headers, don't return
1099   // anything. This makes for more consistent behavior in the case of errors.
1100   if (!transaction_ || receive_headers_end_.is_null())
1101     return;
1102   if (transaction_->GetLoadTimingInfo(load_timing_info))
1103     load_timing_info->receive_headers_end = receive_headers_end_;
1104 }
1105
1106 bool URLRequestHttpJob::GetResponseCookies(std::vector<std::string>* cookies) {
1107   DCHECK(transaction_.get());
1108
1109   if (!response_info_)
1110     return false;
1111
1112   // TODO(darin): Why are we extracting response cookies again?  Perhaps we
1113   // should just leverage response_cookies_.
1114
1115   cookies->clear();
1116   FetchResponseCookies(cookies);
1117   return true;
1118 }
1119
1120 int URLRequestHttpJob::GetResponseCode() const {
1121   DCHECK(transaction_.get());
1122
1123   if (!response_info_)
1124     return -1;
1125
1126   return GetResponseHeaders()->response_code();
1127 }
1128
1129 Filter* URLRequestHttpJob::SetupFilter() const {
1130   DCHECK(transaction_.get());
1131   if (!response_info_)
1132     return NULL;
1133
1134   std::vector<Filter::FilterType> encoding_types;
1135   std::string encoding_type;
1136   HttpResponseHeaders* headers = GetResponseHeaders();
1137   void* iter = NULL;
1138   while (headers->EnumerateHeader(&iter, "Content-Encoding", &encoding_type)) {
1139     encoding_types.push_back(Filter::ConvertEncodingToType(encoding_type));
1140   }
1141
1142   // Even if encoding types are empty, there is a chance that we need to add
1143   // some decoding, as some proxies strip encoding completely. In such cases,
1144   // we may need to add (for example) SDCH filtering (when the context suggests
1145   // it is appropriate).
1146   Filter::FixupEncodingTypes(*filter_context_, &encoding_types);
1147
1148   return !encoding_types.empty()
1149       ? Filter::Factory(encoding_types, *filter_context_) : NULL;
1150 }
1151
1152 bool URLRequestHttpJob::CopyFragmentOnRedirect(const GURL& location) const {
1153   // Allow modification of reference fragments by default, unless
1154   // |allowed_unsafe_redirect_url_| is set and equal to the redirect URL.
1155   // When this is the case, we assume that the network delegate has set the
1156   // desired redirect URL (with or without fragment), so it must not be changed
1157   // any more.
1158   return !allowed_unsafe_redirect_url_.is_valid() ||
1159        allowed_unsafe_redirect_url_ != location;
1160 }
1161
1162 bool URLRequestHttpJob::IsSafeRedirect(const GURL& location) {
1163   // HTTP is always safe.
1164   // TODO(pauljensen): Remove once crbug.com/146591 is fixed.
1165   if (location.is_valid() &&
1166       (location.scheme() == "http" || location.scheme() == "https")) {
1167     return true;
1168   }
1169   // Delegates may mark a URL as safe for redirection.
1170   if (allowed_unsafe_redirect_url_.is_valid() &&
1171       allowed_unsafe_redirect_url_ == location) {
1172     return true;
1173   }
1174   // Query URLRequestJobFactory as to whether |location| would be safe to
1175   // redirect to.
1176   return request_->context()->job_factory() &&
1177       request_->context()->job_factory()->IsSafeRedirectTarget(location);
1178 }
1179
1180 bool URLRequestHttpJob::NeedsAuth() {
1181   int code = GetResponseCode();
1182   if (code == -1)
1183     return false;
1184
1185   // Check if we need either Proxy or WWW Authentication. This could happen
1186   // because we either provided no auth info, or provided incorrect info.
1187   switch (code) {
1188     case 407:
1189       if (proxy_auth_state_ == AUTH_STATE_CANCELED)
1190         return false;
1191       proxy_auth_state_ = AUTH_STATE_NEED_AUTH;
1192       return true;
1193     case 401:
1194       if (server_auth_state_ == AUTH_STATE_CANCELED)
1195         return false;
1196       server_auth_state_ = AUTH_STATE_NEED_AUTH;
1197       return true;
1198   }
1199   return false;
1200 }
1201
1202 void URLRequestHttpJob::GetAuthChallengeInfo(
1203     scoped_refptr<AuthChallengeInfo>* result) {
1204   DCHECK(transaction_.get());
1205   DCHECK(response_info_);
1206
1207   // sanity checks:
1208   DCHECK(proxy_auth_state_ == AUTH_STATE_NEED_AUTH ||
1209          server_auth_state_ == AUTH_STATE_NEED_AUTH);
1210   DCHECK((GetResponseHeaders()->response_code() == HTTP_UNAUTHORIZED) ||
1211          (GetResponseHeaders()->response_code() ==
1212           HTTP_PROXY_AUTHENTICATION_REQUIRED));
1213
1214   *result = response_info_->auth_challenge;
1215 }
1216
1217 void URLRequestHttpJob::SetAuth(const AuthCredentials& credentials) {
1218   DCHECK(transaction_.get());
1219
1220   // Proxy gets set first, then WWW.
1221   if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1222     proxy_auth_state_ = AUTH_STATE_HAVE_AUTH;
1223   } else {
1224     DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1225     server_auth_state_ = AUTH_STATE_HAVE_AUTH;
1226   }
1227
1228   RestartTransactionWithAuth(credentials);
1229 }
1230
1231 void URLRequestHttpJob::CancelAuth() {
1232   // Proxy gets set first, then WWW.
1233   if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1234     proxy_auth_state_ = AUTH_STATE_CANCELED;
1235   } else {
1236     DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1237     server_auth_state_ = AUTH_STATE_CANCELED;
1238   }
1239
1240   // These will be reset in OnStartCompleted.
1241   response_info_ = NULL;
1242   receive_headers_end_ = base::TimeTicks::Now();
1243   response_cookies_.clear();
1244
1245   ResetTimer();
1246
1247   // OK, let the consumer read the error page...
1248   //
1249   // Because we set the AUTH_STATE_CANCELED flag, NeedsAuth will return false,
1250   // which will cause the consumer to receive OnResponseStarted instead of
1251   // OnAuthRequired.
1252   //
1253   // We have to do this via InvokeLater to avoid "recursing" the consumer.
1254   //
1255   base::ThreadTaskRunnerHandle::Get()->PostTask(
1256       FROM_HERE, base::Bind(&URLRequestHttpJob::OnStartCompleted,
1257                             weak_factory_.GetWeakPtr(), OK));
1258 }
1259
1260 void URLRequestHttpJob::ContinueWithCertificate(
1261     X509Certificate* client_cert) {
1262   DCHECK(transaction_.get());
1263
1264   DCHECK(!response_info_) << "should not have a response yet";
1265   receive_headers_end_ = base::TimeTicks();
1266
1267   ResetTimer();
1268
1269   // No matter what, we want to report our status as IO pending since we will
1270   // be notifying our consumer asynchronously via OnStartCompleted.
1271   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1272
1273   int rv = transaction_->RestartWithCertificate(client_cert, start_callback_);
1274   if (rv == ERR_IO_PENDING)
1275     return;
1276
1277   // The transaction started synchronously, but we need to notify the
1278   // URLRequest delegate via the message loop.
1279   base::ThreadTaskRunnerHandle::Get()->PostTask(
1280       FROM_HERE, base::Bind(&URLRequestHttpJob::OnStartCompleted,
1281                             weak_factory_.GetWeakPtr(), rv));
1282 }
1283
1284 void URLRequestHttpJob::ContinueDespiteLastError() {
1285   // If the transaction was destroyed, then the job was cancelled.
1286   if (!transaction_.get())
1287     return;
1288
1289   DCHECK(!response_info_) << "should not have a response yet";
1290   receive_headers_end_ = base::TimeTicks();
1291
1292   ResetTimer();
1293
1294   // No matter what, we want to report our status as IO pending since we will
1295   // be notifying our consumer asynchronously via OnStartCompleted.
1296   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1297
1298   int rv = transaction_->RestartIgnoringLastError(start_callback_);
1299   if (rv == ERR_IO_PENDING)
1300     return;
1301
1302   // The transaction started synchronously, but we need to notify the
1303   // URLRequest delegate via the message loop.
1304   base::ThreadTaskRunnerHandle::Get()->PostTask(
1305       FROM_HERE, base::Bind(&URLRequestHttpJob::OnStartCompleted,
1306                             weak_factory_.GetWeakPtr(), rv));
1307 }
1308
1309 void URLRequestHttpJob::ResumeNetworkStart() {
1310   DCHECK(transaction_.get());
1311   transaction_->ResumeNetworkStart();
1312 }
1313
1314 bool URLRequestHttpJob::ShouldFixMismatchedContentLength(int rv) const {
1315   // Some servers send the body compressed, but specify the content length as
1316   // the uncompressed size. Although this violates the HTTP spec we want to
1317   // support it (as IE and FireFox do), but *only* for an exact match.
1318   // See http://crbug.com/79694.
1319   if (rv == ERR_CONTENT_LENGTH_MISMATCH ||
1320       rv == ERR_INCOMPLETE_CHUNKED_ENCODING) {
1321     if (request_ && request_->response_headers()) {
1322       int64 expected_length = request_->response_headers()->GetContentLength();
1323       VLOG(1) << __FUNCTION__ << "() "
1324               << "\"" << request_->url().spec() << "\""
1325               << " content-length = " << expected_length
1326               << " pre total = " << prefilter_bytes_read()
1327               << " post total = " << postfilter_bytes_read();
1328       if (postfilter_bytes_read() == expected_length) {
1329         // Clear the error.
1330         return true;
1331       }
1332     }
1333   }
1334   return false;
1335 }
1336
1337 bool URLRequestHttpJob::ReadRawData(IOBuffer* buf, int buf_size,
1338                                     int* bytes_read) {
1339   DCHECK_NE(buf_size, 0);
1340   DCHECK(bytes_read);
1341   DCHECK(!read_in_progress_);
1342
1343   int rv = transaction_->Read(
1344       buf, buf_size,
1345       base::Bind(&URLRequestHttpJob::OnReadCompleted, base::Unretained(this)));
1346
1347   if (ShouldFixMismatchedContentLength(rv))
1348     rv = 0;
1349
1350   if (rv >= 0) {
1351     *bytes_read = rv;
1352     if (!rv)
1353       DoneWithRequest(FINISHED);
1354     return true;
1355   }
1356
1357   if (rv == ERR_IO_PENDING) {
1358     read_in_progress_ = true;
1359     SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1360   } else {
1361     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, rv));
1362   }
1363
1364   return false;
1365 }
1366
1367 void URLRequestHttpJob::StopCaching() {
1368   if (transaction_.get())
1369     transaction_->StopCaching();
1370 }
1371
1372 bool URLRequestHttpJob::GetFullRequestHeaders(
1373     HttpRequestHeaders* headers) const {
1374   if (!transaction_)
1375     return false;
1376
1377   return transaction_->GetFullRequestHeaders(headers);
1378 }
1379
1380 int64 URLRequestHttpJob::GetTotalReceivedBytes() const {
1381   if (!transaction_)
1382     return 0;
1383
1384   return transaction_->GetTotalReceivedBytes();
1385 }
1386
1387 void URLRequestHttpJob::DoneReading() {
1388   if (transaction_) {
1389     transaction_->DoneReading();
1390   }
1391   DoneWithRequest(FINISHED);
1392 }
1393
1394 void URLRequestHttpJob::DoneReadingRedirectResponse() {
1395   if (transaction_) {
1396     if (transaction_->GetResponseInfo()->headers->IsRedirect(NULL)) {
1397       // If the original headers indicate a redirect, go ahead and cache the
1398       // response, even if the |override_response_headers_| are a redirect to
1399       // another location.
1400       transaction_->DoneReading();
1401     } else {
1402       // Otherwise, |override_response_headers_| must be non-NULL and contain
1403       // bogus headers indicating a redirect.
1404       DCHECK(override_response_headers_.get());
1405       DCHECK(override_response_headers_->IsRedirect(NULL));
1406       transaction_->StopCaching();
1407     }
1408   }
1409   DoneWithRequest(FINISHED);
1410 }
1411
1412 HostPortPair URLRequestHttpJob::GetSocketAddress() const {
1413   return response_info_ ? response_info_->socket_address : HostPortPair();
1414 }
1415
1416 void URLRequestHttpJob::RecordTimer() {
1417   if (request_creation_time_.is_null()) {
1418     NOTREACHED()
1419         << "The same transaction shouldn't start twice without new timing.";
1420     return;
1421   }
1422
1423   base::TimeDelta to_start = base::Time::Now() - request_creation_time_;
1424   request_creation_time_ = base::Time();
1425
1426   UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpTimeToFirstByte", to_start);
1427 }
1428
1429 void URLRequestHttpJob::ResetTimer() {
1430   if (!request_creation_time_.is_null()) {
1431     NOTREACHED()
1432         << "The timer was reset before it was recorded.";
1433     return;
1434   }
1435   request_creation_time_ = base::Time::Now();
1436 }
1437
1438 void URLRequestHttpJob::UpdatePacketReadTimes() {
1439   if (!packet_timing_enabled_)
1440     return;
1441
1442   DCHECK_GT(prefilter_bytes_read(), bytes_observed_in_packets_);
1443
1444   base::Time now(base::Time::Now());
1445   if (!bytes_observed_in_packets_)
1446     request_time_snapshot_ = now;
1447   final_packet_time_ = now;
1448
1449   bytes_observed_in_packets_ = prefilter_bytes_read();
1450 }
1451
1452 void URLRequestHttpJob::RecordPacketStats(
1453     FilterContext::StatisticSelector statistic) const {
1454   if (!packet_timing_enabled_ || (final_packet_time_ == base::Time()))
1455     return;
1456
1457   base::TimeDelta duration = final_packet_time_ - request_time_snapshot_;
1458   switch (statistic) {
1459     case FilterContext::SDCH_DECODE: {
1460       UMA_HISTOGRAM_CUSTOM_COUNTS("Sdch3.Network_Decode_Bytes_Processed_b",
1461           static_cast<int>(bytes_observed_in_packets_), 500, 100000, 100);
1462       return;
1463     }
1464     case FilterContext::SDCH_PASSTHROUGH: {
1465       // Despite advertising a dictionary, we handled non-sdch compressed
1466       // content.
1467       return;
1468     }
1469
1470     case FilterContext::SDCH_EXPERIMENT_DECODE: {
1471       UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment3_Decode",
1472                                   duration,
1473                                   base::TimeDelta::FromMilliseconds(20),
1474                                   base::TimeDelta::FromMinutes(10), 100);
1475       return;
1476     }
1477     case FilterContext::SDCH_EXPERIMENT_HOLDBACK: {
1478       UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment3_Holdback",
1479                                   duration,
1480                                   base::TimeDelta::FromMilliseconds(20),
1481                                   base::TimeDelta::FromMinutes(10), 100);
1482       return;
1483     }
1484     default:
1485       NOTREACHED();
1486       return;
1487   }
1488 }
1489
1490 void URLRequestHttpJob::RecordPerfHistograms(CompletionCause reason) {
1491   if (start_time_.is_null())
1492     return;
1493
1494   base::TimeDelta total_time = base::TimeTicks::Now() - start_time_;
1495   UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTime", total_time);
1496
1497   if (reason == FINISHED) {
1498     UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeSuccess", total_time);
1499   } else {
1500     UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCancel", total_time);
1501   }
1502
1503   if (response_info_) {
1504     // QUIC (by default) supports https scheme only, thus track https URLs only
1505     // for QUIC.
1506     bool is_https_google = request() && request()->url().SchemeIs("https") &&
1507                            HasGoogleHost(request()->url());
1508     bool used_quic = response_info_->DidUseQuic();
1509     if (is_https_google) {
1510       if (used_quic) {
1511         UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTime.Secure.Quic",
1512                                    total_time);
1513       } else {
1514         UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTime.Secure.NotQuic",
1515                                    total_time);
1516       }
1517     }
1518     if (response_info_->was_cached) {
1519       UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCached", total_time);
1520       if (is_https_google) {
1521         if (used_quic) {
1522           UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTimeCached.Secure.Quic",
1523                                      total_time);
1524         } else {
1525           UMA_HISTOGRAM_MEDIUM_TIMES(
1526               "Net.HttpJob.TotalTimeCached.Secure.NotQuic", total_time);
1527         }
1528       }
1529     } else  {
1530       UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeNotCached", total_time);
1531       if (is_https_google) {
1532         if (used_quic) {
1533           UMA_HISTOGRAM_MEDIUM_TIMES(
1534               "Net.HttpJob.TotalTimeNotCached.Secure.Quic", total_time);
1535         } else {
1536           UMA_HISTOGRAM_MEDIUM_TIMES(
1537               "Net.HttpJob.TotalTimeNotCached.Secure.NotQuic", total_time);
1538         }
1539       }
1540     }
1541   }
1542
1543   if (request_info_.load_flags & LOAD_PREFETCH && !request_->was_cached())
1544     UMA_HISTOGRAM_COUNTS("Net.Prefetch.PrefilterBytesReadFromNetwork",
1545                          prefilter_bytes_read());
1546
1547   start_time_ = base::TimeTicks();
1548 }
1549
1550 void URLRequestHttpJob::DoneWithRequest(CompletionCause reason) {
1551   if (done_)
1552     return;
1553   done_ = true;
1554
1555   // Notify NetworkQualityEstimator.
1556   if (request() && (reason == FINISHED || reason == ABORTED)) {
1557     NetworkQualityEstimator* network_quality_estimator =
1558         request()->context()->network_quality_estimator();
1559     if (network_quality_estimator)
1560       network_quality_estimator->NotifyRequestCompleted(*request());
1561   }
1562
1563   RecordPerfHistograms(reason);
1564   if (request_)
1565     request_->set_received_response_content_length(prefilter_bytes_read());
1566 }
1567
1568 HttpResponseHeaders* URLRequestHttpJob::GetResponseHeaders() const {
1569   DCHECK(transaction_.get());
1570   DCHECK(transaction_->GetResponseInfo());
1571   return override_response_headers_.get() ?
1572              override_response_headers_.get() :
1573              transaction_->GetResponseInfo()->headers.get();
1574 }
1575
1576 void URLRequestHttpJob::NotifyURLRequestDestroyed() {
1577   awaiting_callback_ = false;
1578 }
1579
1580 }  // namespace net