net/url_request/url_request_http_job.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_request_http_job.h"
   6
   7 #include "base/base_switches.h"
   8 #include "base/bind.h"
   9 #include "base/bind_helpers.h"
  10 #include "base/command_line.h"
  11 #include "base/compiler_specific.h"
  12 #include "base/file_version_info.h"
  13 #include "base/location.h"
  14 #include "base/metrics/field_trial.h"
  15 #include "base/metrics/histogram_macros.h"
  16 #include "base/profiler/scoped_tracker.h"
  17 #include "base/rand_util.h"
  18 #include "base/single_thread_task_runner.h"
  19 #include "base/strings/string_util.h"
  20 #include "base/thread_task_runner_handle.h"
  21 #include "base/time/time.h"
  22 #include "base/values.h"
  23 #include "net/base/host_port_pair.h"
  24 #include "net/base/load_flags.h"
  25 #include "net/base/net_errors.h"
  26 #include "net/base/net_util.h"
  27 #include "net/base/network_delegate.h"
  28 #include "net/base/sdch_manager.h"
  29 #include "net/base/sdch_net_log_params.h"
  30 #include "net/cert/cert_status_flags.h"
  31 #include "net/cookies/cookie_store.h"
  32 #include "net/http/http_content_disposition.h"
  33 #include "net/http/http_network_session.h"
  34 #include "net/http/http_request_headers.h"
  35 #include "net/http/http_response_headers.h"
  36 #include "net/http/http_response_info.h"
  37 #include "net/http/http_status_code.h"
  38 #include "net/http/http_transaction.h"
  39 #include "net/http/http_transaction_factory.h"
  40 #include "net/http/http_util.h"
  41 #include "net/proxy/proxy_info.h"
  42 #include "net/ssl/ssl_cert_request_info.h"
  43 #include "net/ssl/ssl_config_service.h"
  44 #include "net/url_request/fraudulent_certificate_reporter.h"
  45 #include "net/url_request/http_user_agent_settings.h"
  46 #include "net/url_request/url_request.h"
  47 #include "net/url_request/url_request_backoff_manager.h"
  48 #include "net/url_request/url_request_context.h"
  49 #include "net/url_request/url_request_error_job.h"
  50 #include "net/url_request/url_request_job_factory.h"
  51 #include "net/url_request/url_request_redirect_job.h"
  52 #include "net/url_request/url_request_throttler_manager.h"
  53 #include "net/websockets/websocket_handshake_stream_base.h"
  54
  55 static const char kAvailDictionaryHeader[] = "Avail-Dictionary";
  56
  57 namespace net {
  58
  59 class URLRequestHttpJob::HttpFilterContext : public FilterContext {
  60  public:
  61   explicit HttpFilterContext(URLRequestHttpJob* job);
  62   ~HttpFilterContext() override;
  63
  64   // FilterContext implementation.
  65   bool GetMimeType(std::string* mime_type) const override;
  66   bool GetURL(GURL* gurl) const override;
  67   base::Time GetRequestTime() const override;
  68   bool IsCachedContent() const override;
  69   SdchManager::DictionarySet* SdchDictionariesAdvertised() const override;
  70   int64 GetByteReadCount() const override;
  71   int GetResponseCode() const override;
  72   const URLRequestContext* GetURLRequestContext() const override;
  73   void RecordPacketStats(StatisticSelector statistic) const override;
  74   const BoundNetLog& GetNetLog() const override;
  75
  76  private:
  77   URLRequestHttpJob* job_;
  78
  79   // URLRequestHttpJob may be detached from URLRequest, but we still need to
  80   // return something.
  81   BoundNetLog dummy_log_;
  82
  83   DISALLOW_COPY_AND_ASSIGN(HttpFilterContext);
  84 };
  85
  86 URLRequestHttpJob::HttpFilterContext::HttpFilterContext(URLRequestHttpJob* job)
  87     : job_(job) {
  88   DCHECK(job_);
  89 }
  90
  91 URLRequestHttpJob::HttpFilterContext::~HttpFilterContext() {
  92 }
  93
  94 bool URLRequestHttpJob::HttpFilterContext::GetMimeType(
  95     std::string* mime_type) const {
  96   return job_->GetMimeType(mime_type);
  97 }
  98
  99 bool URLRequestHttpJob::HttpFilterContext::GetURL(GURL* gurl) const {
 100   if (!job_->request())
 101     return false;
 102   *gurl = job_->request()->url();
 103   return true;
 104 }
 105
 106 base::Time URLRequestHttpJob::HttpFilterContext::GetRequestTime() const {
 107   return job_->request() ? job_->request()->request_time() : base::Time();
 108 }
 109
 110 bool URLRequestHttpJob::HttpFilterContext::IsCachedContent() const {
 111   return job_->is_cached_content_;
 112 }
 113
 114 SdchManager::DictionarySet*
 115 URLRequestHttpJob::HttpFilterContext::SdchDictionariesAdvertised() const {
 116   return job_->dictionaries_advertised_.get();
 117 }
 118
 119 int64 URLRequestHttpJob::HttpFilterContext::GetByteReadCount() const {
 120   return job_->prefilter_bytes_read();
 121 }
 122
 123 int URLRequestHttpJob::HttpFilterContext::GetResponseCode() const {
 124   return job_->GetResponseCode();
 125 }
 126
 127 const URLRequestContext*
 128 URLRequestHttpJob::HttpFilterContext::GetURLRequestContext() const {
 129   return job_->request() ? job_->request()->context() : NULL;
 130 }
 131
 132 void URLRequestHttpJob::HttpFilterContext::RecordPacketStats(
 133     StatisticSelector statistic) const {
 134   job_->RecordPacketStats(statistic);
 135 }
 136
 137 const BoundNetLog& URLRequestHttpJob::HttpFilterContext::GetNetLog() const {
 138   return job_->request() ? job_->request()->net_log() : dummy_log_;
 139 }
 140
 141 // TODO(darin): make sure the port blocking code is not lost
 142 // static
 143 URLRequestJob* URLRequestHttpJob::Factory(URLRequest* request,
 144                                           NetworkDelegate* network_delegate,
 145                                           const std::string& scheme) {
 146   DCHECK(scheme == "http" || scheme == "https" || scheme == "ws" ||
 147          scheme == "wss");
 148
 149   if (!request->context()->http_transaction_factory()) {
 150     NOTREACHED() << "requires a valid context";
 151     return new URLRequestErrorJob(
 152         request, network_delegate, ERR_INVALID_ARGUMENT);
 153   }
 154
 155   GURL redirect_url;
 156   if (request->GetHSTSRedirect(&redirect_url)) {
 157     return new URLRequestRedirectJob(
 158         request, network_delegate, redirect_url,
 159         // Use status code 307 to preserve the method, so POST requests work.
 160         URLRequestRedirectJob::REDIRECT_307_TEMPORARY_REDIRECT, "HSTS");
 161   }
 162   return new URLRequestHttpJob(request,
 163                                network_delegate,
 164                                request->context()->http_user_agent_settings());
 165 }
 166
 167 URLRequestHttpJob::URLRequestHttpJob(
 168     URLRequest* request,
 169     NetworkDelegate* network_delegate,
 170     const HttpUserAgentSettings* http_user_agent_settings)
 171     : URLRequestJob(request, network_delegate),
 172       priority_(DEFAULT_PRIORITY),
 173       response_info_(NULL),
 174       response_cookies_save_index_(0),
 175       proxy_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
 176       server_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
 177       start_callback_(base::Bind(&URLRequestHttpJob::OnStartCompleted,
 178                                  base::Unretained(this))),
 179       notify_before_headers_sent_callback_(
 180           base::Bind(&URLRequestHttpJob::NotifyBeforeSendHeadersCallback,
 181                      base::Unretained(this))),
 182       read_in_progress_(false),
 183       throttling_entry_(NULL),
 184       sdch_test_activated_(false),
 185       sdch_test_control_(false),
 186       is_cached_content_(false),
 187       request_creation_time_(),
 188       packet_timing_enabled_(false),
 189       done_(false),
 190       bytes_observed_in_packets_(0),
 191       request_time_snapshot_(),
 192       final_packet_time_(),
 193       filter_context_(new HttpFilterContext(this)),
 194       on_headers_received_callback_(
 195           base::Bind(&URLRequestHttpJob::OnHeadersReceivedCallback,
 196                      base::Unretained(this))),
 197       awaiting_callback_(false),
 198       http_user_agent_settings_(http_user_agent_settings),
 199       backoff_manager_(request->context()->backoff_manager()),
 200       weak_factory_(this) {
 201   URLRequestThrottlerManager* manager = request->context()->throttler_manager();
 202   if (manager)
 203     throttling_entry_ = manager->RegisterRequestUrl(request->url());
 204
 205   ResetTimer();
 206 }
 207
 208 URLRequestHttpJob::~URLRequestHttpJob() {
 209   CHECK(!awaiting_callback_);
 210
 211   DCHECK(!sdch_test_control_ || !sdch_test_activated_);
 212   if (!is_cached_content_) {
 213     if (sdch_test_control_)
 214       RecordPacketStats(FilterContext::SDCH_EXPERIMENT_HOLDBACK);
 215     if (sdch_test_activated_)
 216       RecordPacketStats(FilterContext::SDCH_EXPERIMENT_DECODE);
 217   }
 218   // Make sure SDCH filters are told to emit histogram data while
 219   // filter_context_ is still alive.
 220   DestroyFilters();
 221
 222   DoneWithRequest(ABORTED);
 223 }
 224
 225 void URLRequestHttpJob::SetPriority(RequestPriority priority) {
 226   priority_ = priority;
 227   if (transaction_)
 228     transaction_->SetPriority(priority_);
 229 }
 230
 231 void URLRequestHttpJob::Start() {
 232   // TODO(mmenke): Remove ScopedTracker below once crbug.com/456327 is fixed.
 233   tracked_objects::ScopedTracker tracking_profile(
 234       FROM_HERE_WITH_EXPLICIT_FUNCTION("456327 URLRequestHttpJob::Start"));
 235
 236   DCHECK(!transaction_.get());
 237
 238   // URLRequest::SetReferrer ensures that we do not send username and password
 239   // fields in the referrer.
 240   GURL referrer(request_->referrer());
 241
 242   request_info_.url = request_->url();
 243   request_info_.method = request_->method();
 244   request_info_.load_flags = request_->load_flags();
 245   // Enable privacy mode if cookie settings or flags tell us not send or
 246   // save cookies.
 247   bool enable_privacy_mode =
 248       (request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES) ||
 249       (request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) ||
 250       CanEnablePrivacyMode();
 251   // Privacy mode could still be disabled in OnCookiesLoaded if we are going
 252   // to send previously saved cookies.
 253   request_info_.privacy_mode = enable_privacy_mode ?
 254       PRIVACY_MODE_ENABLED : PRIVACY_MODE_DISABLED;
 255
 256   // Strip Referer from request_info_.extra_headers to prevent, e.g., plugins
 257   // from overriding headers that are controlled using other means. Otherwise a
 258   // plugin could set a referrer although sending the referrer is inhibited.
 259   request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kReferer);
 260
 261   // Our consumer should have made sure that this is a safe referrer. See for
 262   // instance WebCore::FrameLoader::HideReferrer.
 263   if (referrer.is_valid()) {
 264     request_info_.extra_headers.SetHeader(HttpRequestHeaders::kReferer,
 265                                           referrer.spec());
 266   }
 267
 268   request_info_.extra_headers.SetHeaderIfMissing(
 269       HttpRequestHeaders::kUserAgent,
 270       http_user_agent_settings_ ?
 271           http_user_agent_settings_->GetUserAgent() : std::string());
 272
 273   AddExtraHeaders();
 274   AddCookieHeaderAndStart();
 275 }
 276
 277 void URLRequestHttpJob::Kill() {
 278   if (!transaction_.get())
 279     return;
 280
 281   weak_factory_.InvalidateWeakPtrs();
 282   DestroyTransaction();
 283   URLRequestJob::Kill();
 284 }
 285
 286 void URLRequestHttpJob::GetConnectionAttempts(ConnectionAttempts* out) const {
 287   if (transaction_)
 288     transaction_->GetConnectionAttempts(out);
 289   else
 290     out->clear();
 291 }
 292
 293 void URLRequestHttpJob::NotifyBeforeSendProxyHeadersCallback(
 294     const ProxyInfo& proxy_info,
 295     HttpRequestHeaders* request_headers) {
 296   DCHECK(request_headers);
 297   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 298   if (network_delegate()) {
 299     network_delegate()->NotifyBeforeSendProxyHeaders(
 300         request_,
 301         proxy_info,
 302         request_headers);
 303   }
 304 }
 305
 306 void URLRequestHttpJob::NotifyBeforeNetworkStart(bool* defer) {
 307   if (!request_)
 308     return;
 309   if (backoff_manager_) {
 310     if (backoff_manager_->ShouldRejectRequest(request()->url(),
 311                                               request()->request_time())) {
 312       *defer = true;
 313       base::MessageLoop::current()->PostTask(
 314           FROM_HERE,
 315           base::Bind(&URLRequestHttpJob::OnStartCompleted,
 316                      weak_factory_.GetWeakPtr(), ERR_TEMPORARY_BACKOFF));
 317       return;
 318     }
 319   }
 320   URLRequestJob::NotifyBeforeNetworkStart(defer);
 321 }
 322
 323 void URLRequestHttpJob::NotifyHeadersComplete() {
 324   DCHECK(!response_info_);
 325
 326   response_info_ = transaction_->GetResponseInfo();
 327
 328   // Save boolean, as we'll need this info at destruction time, and filters may
 329   // also need this info.
 330   is_cached_content_ = response_info_->was_cached;
 331
 332   if (!is_cached_content_ && throttling_entry_.get())
 333     throttling_entry_->UpdateWithResponse(GetResponseCode());
 334
 335   if (!is_cached_content_)
 336     ProcessBackoffHeader();
 337
 338   // The ordering of these calls is not important.
 339   ProcessStrictTransportSecurityHeader();
 340   ProcessPublicKeyPinsHeader();
 341
 342   // Handle the server notification of a new SDCH dictionary.
 343   SdchManager* sdch_manager(request()->context()->sdch_manager());
 344   if (sdch_manager) {
 345     SdchProblemCode rv = sdch_manager->IsInSupportedDomain(request()->url());
 346     if (rv != SDCH_OK) {
 347       // If SDCH is just disabled, it is not a real error.
 348       if (rv != SDCH_DISABLED) {
 349         SdchManager::SdchErrorRecovery(rv);
 350         request()->net_log().AddEvent(
 351             NetLog::TYPE_SDCH_DECODING_ERROR,
 352             base::Bind(&NetLogSdchResourceProblemCallback, rv));
 353       }
 354     } else {
 355       const std::string name = "Get-Dictionary";
 356       std::string url_text;
 357       void* iter = NULL;
 358       // TODO(jar): We need to not fetch dictionaries the first time they are
 359       // seen, but rather wait until we can justify their usefulness.
 360       // For now, we will only fetch the first dictionary, which will at least
 361       // require multiple suggestions before we get additional ones for this
 362       // site. Eventually we should wait until a dictionary is requested
 363       // several times
 364       // before we even download it (so that we don't waste memory or
 365       // bandwidth).
 366       if (GetResponseHeaders()->EnumerateHeader(&iter, name, &url_text)) {
 367         // Resolve suggested URL relative to request url.
 368         GURL sdch_dictionary_url = request_->url().Resolve(url_text);
 369         if (sdch_dictionary_url.is_valid()) {
 370           rv = sdch_manager->OnGetDictionary(request_->url(),
 371                                              sdch_dictionary_url);
 372           if (rv != SDCH_OK) {
 373             SdchManager::SdchErrorRecovery(rv);
 374             request_->net_log().AddEvent(
 375                 NetLog::TYPE_SDCH_DICTIONARY_ERROR,
 376                 base::Bind(&NetLogSdchDictionaryFetchProblemCallback, rv,
 377                            sdch_dictionary_url, false));
 378           }
 379         }
 380       }
 381     }
 382   }
 383
 384   // Handle the server signalling no SDCH encoding.
 385   if (dictionaries_advertised_) {
 386     // We are wary of proxies that discard or damage SDCH encoding. If a server
 387     // explicitly states that this is not SDCH content, then we can correct our
 388     // assumption that this is an SDCH response, and avoid the need to recover
 389     // as though the content is corrupted (when we discover it is not SDCH
 390     // encoded).
 391     std::string sdch_response_status;
 392     void* iter = NULL;
 393     while (GetResponseHeaders()->EnumerateHeader(&iter, "X-Sdch-Encode",
 394                                                  &sdch_response_status)) {
 395       if (sdch_response_status == "0") {
 396         dictionaries_advertised_.reset();
 397         break;
 398       }
 399     }
 400   }
 401
 402   // The HTTP transaction may be restarted several times for the purposes
 403   // of sending authorization information. Each time it restarts, we get
 404   // notified of the headers completion so that we can update the cookie store.
 405   if (transaction_->IsReadyToRestartForAuth()) {
 406     DCHECK(!response_info_->auth_challenge.get());
 407     // TODO(battre): This breaks the webrequest API for
 408     // URLRequestTestHTTP.BasicAuthWithCookies
 409     // where OnBeforeSendHeaders -> OnSendHeaders -> OnBeforeSendHeaders
 410     // occurs.
 411     RestartTransactionWithAuth(AuthCredentials());
 412     return;
 413   }
 414
 415   URLRequestJob::NotifyHeadersComplete();
 416 }
 417
 418 void URLRequestHttpJob::NotifyDone(const URLRequestStatus& status) {
 419   DoneWithRequest(FINISHED);
 420   URLRequestJob::NotifyDone(status);
 421 }
 422
 423 void URLRequestHttpJob::DestroyTransaction() {
 424   DCHECK(transaction_.get());
 425
 426   DoneWithRequest(ABORTED);
 427   transaction_.reset();
 428   response_info_ = NULL;
 429   receive_headers_end_ = base::TimeTicks();
 430 }
 431
 432 void URLRequestHttpJob::StartTransaction() {
 433   // TODO(mmenke): Remove ScopedTracker below once crbug.com/456327 is fixed.
 434   tracked_objects::ScopedTracker tracking_profile(
 435       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 436           "456327 URLRequestHttpJob::StartTransaction"));
 437
 438   if (network_delegate()) {
 439     OnCallToDelegate();
 440     int rv = network_delegate()->NotifyBeforeSendHeaders(
 441         request_, notify_before_headers_sent_callback_,
 442         &request_info_.extra_headers);
 443     // If an extension blocks the request, we rely on the callback to
 444     // MaybeStartTransactionInternal().
 445     if (rv == ERR_IO_PENDING)
 446       return;
 447     MaybeStartTransactionInternal(rv);
 448     return;
 449   }
 450   StartTransactionInternal();
 451 }
 452
 453 void URLRequestHttpJob::NotifyBeforeSendHeadersCallback(int result) {
 454   // Check that there are no callbacks to already canceled requests.
 455   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 456
 457   MaybeStartTransactionInternal(result);
 458 }
 459
 460 void URLRequestHttpJob::MaybeStartTransactionInternal(int result) {
 461   // TODO(mmenke): Remove ScopedTracker below once crbug.com/456327 is fixed.
 462   tracked_objects::ScopedTracker tracking_profile(
 463       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 464           "456327 URLRequestHttpJob::MaybeStartTransactionInternal"));
 465
 466   OnCallToDelegateComplete();
 467   if (result == OK) {
 468     StartTransactionInternal();
 469   } else {
 470     std::string source("delegate");
 471     request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 472                                  NetLog::StringCallback("source", &source));
 473     NotifyCanceled();
 474     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 475   }
 476 }
 477
 478 void URLRequestHttpJob::StartTransactionInternal() {
 479   // NOTE: This method assumes that request_info_ is already setup properly.
 480
 481   // If we already have a transaction, then we should restart the transaction
 482   // with auth provided by auth_credentials_.
 483
 484   int rv;
 485
 486   if (network_delegate()) {
 487     network_delegate()->NotifySendHeaders(
 488         request_, request_info_.extra_headers);
 489   }
 490
 491   if (transaction_.get()) {
 492     rv = transaction_->RestartWithAuth(auth_credentials_, start_callback_);
 493     auth_credentials_ = AuthCredentials();
 494   } else {
 495     DCHECK(request_->context()->http_transaction_factory());
 496
 497     rv = request_->context()->http_transaction_factory()->CreateTransaction(
 498         priority_, &transaction_);
 499
 500     if (rv == OK && request_info_.url.SchemeIsWSOrWSS()) {
 501       base::SupportsUserData::Data* data = request_->GetUserData(
 502           WebSocketHandshakeStreamBase::CreateHelper::DataKey());
 503       if (data) {
 504         transaction_->SetWebSocketHandshakeStreamCreateHelper(
 505             static_cast<WebSocketHandshakeStreamBase::CreateHelper*>(data));
 506       } else {
 507         rv = ERR_DISALLOWED_URL_SCHEME;
 508       }
 509     }
 510
 511     if (rv == OK) {
 512       transaction_->SetBeforeNetworkStartCallback(
 513           base::Bind(&URLRequestHttpJob::NotifyBeforeNetworkStart,
 514                      base::Unretained(this)));
 515       transaction_->SetBeforeProxyHeadersSentCallback(
 516           base::Bind(&URLRequestHttpJob::NotifyBeforeSendProxyHeadersCallback,
 517                      base::Unretained(this)));
 518
 519       if (!throttling_entry_.get() ||
 520           !throttling_entry_->ShouldRejectRequest(*request_)) {
 521         rv = transaction_->Start(
 522             &request_info_, start_callback_, request_->net_log());
 523         start_time_ = base::TimeTicks::Now();
 524       } else {
 525         // Special error code for the exponential back-off module.
 526         rv = ERR_TEMPORARILY_THROTTLED;
 527       }
 528     }
 529   }
 530
 531   if (rv == ERR_IO_PENDING)
 532     return;
 533
 534   // The transaction started synchronously, but we need to notify the
 535   // URLRequest delegate via the message loop.
 536   base::ThreadTaskRunnerHandle::Get()->PostTask(
 537       FROM_HERE, base::Bind(&URLRequestHttpJob::OnStartCompleted,
 538                             weak_factory_.GetWeakPtr(), rv));
 539 }
 540
 541 void URLRequestHttpJob::AddExtraHeaders() {
 542   SdchManager* sdch_manager = request()->context()->sdch_manager();
 543
 544   // Supply Accept-Encoding field only if it is not already provided.
 545   // It should be provided IF the content is known to have restrictions on
 546   // potential encoding, such as streaming multi-media.
 547   // For details see bug 47381.
 548   // TODO(jar, enal): jpeg files etc. should set up a request header if
 549   // possible. Right now it is done only by buffered_resource_loader and
 550   // simple_data_source.
 551   if (!request_info_.extra_headers.HasHeader(
 552       HttpRequestHeaders::kAcceptEncoding)) {
 553     // We don't support SDCH responses to POST as there is a possibility
 554     // of having SDCH encoded responses returned (e.g. by the cache)
 555     // which we cannot decode, and in those situations, we will need
 556     // to retransmit the request without SDCH, which is illegal for a POST.
 557     bool advertise_sdch = sdch_manager != NULL && request()->method() != "POST";
 558     if (advertise_sdch) {
 559       SdchProblemCode rv = sdch_manager->IsInSupportedDomain(request()->url());
 560       if (rv != SDCH_OK) {
 561         advertise_sdch = false;
 562         // If SDCH is just disabled, it is not a real error.
 563         if (rv != SDCH_DISABLED) {
 564           SdchManager::SdchErrorRecovery(rv);
 565           request()->net_log().AddEvent(
 566               NetLog::TYPE_SDCH_DECODING_ERROR,
 567               base::Bind(&NetLogSdchResourceProblemCallback, rv));
 568         }
 569       }
 570     }
 571     if (advertise_sdch) {
 572       dictionaries_advertised_ =
 573           sdch_manager->GetDictionarySet(request_->url());
 574     }
 575
 576     // The AllowLatencyExperiment() is only true if we've successfully done a
 577     // full SDCH compression recently in this browser session for this host.
 578     // Note that for this path, there might be no applicable dictionaries,
 579     // and hence we can't participate in the experiment.
 580     if (dictionaries_advertised_ &&
 581         sdch_manager->AllowLatencyExperiment(request_->url())) {
 582       // We are participating in the test (or control), and hence we'll
 583       // eventually record statistics via either SDCH_EXPERIMENT_DECODE or
 584       // SDCH_EXPERIMENT_HOLDBACK, and we'll need some packet timing data.
 585       packet_timing_enabled_ = true;
 586       if (base::RandDouble() < .01) {
 587         sdch_test_control_ = true;  // 1% probability.
 588         dictionaries_advertised_.reset();
 589         advertise_sdch = false;
 590       } else {
 591         sdch_test_activated_ = true;
 592       }
 593     }
 594
 595     // Supply Accept-Encoding headers first so that it is more likely that they
 596     // will be in the first transmitted packet. This can sometimes make it
 597     // easier to filter and analyze the streams to assure that a proxy has not
 598     // damaged these headers. Some proxies deliberately corrupt Accept-Encoding
 599     // headers.
 600     if (!advertise_sdch) {
 601       // Tell the server what compression formats we support (other than SDCH).
 602       request_info_.extra_headers.SetHeader(
 603           HttpRequestHeaders::kAcceptEncoding, "gzip, deflate");
 604     } else {
 605       // Include SDCH in acceptable list.
 606       request_info_.extra_headers.SetHeader(
 607           HttpRequestHeaders::kAcceptEncoding, "gzip, deflate, sdch");
 608       if (dictionaries_advertised_) {
 609         request_info_.extra_headers.SetHeader(
 610             kAvailDictionaryHeader,
 611             dictionaries_advertised_->GetDictionaryClientHashList());
 612         // Since we're tagging this transaction as advertising a dictionary,
 613         // we'll definitely employ an SDCH filter (or tentative sdch filter)
 614         // when we get a response. When done, we'll record histograms via
 615         // SDCH_DECODE or SDCH_PASSTHROUGH. Hence we need to record packet
 616         // arrival times.
 617         packet_timing_enabled_ = true;
 618       }
 619     }
 620   }
 621
 622   if (http_user_agent_settings_) {
 623     // Only add default Accept-Language if the request didn't have it
 624     // specified.
 625     std::string accept_language =
 626         http_user_agent_settings_->GetAcceptLanguage();
 627     if (!accept_language.empty()) {
 628       request_info_.extra_headers.SetHeaderIfMissing(
 629           HttpRequestHeaders::kAcceptLanguage,
 630           accept_language);
 631     }
 632   }
 633 }
 634
 635 void URLRequestHttpJob::AddCookieHeaderAndStart() {
 636   // No matter what, we want to report our status as IO pending since we will
 637   // be notifying our consumer asynchronously via OnStartCompleted.
 638   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
 639
 640   // If the request was destroyed, then there is no more work to do.
 641   if (!request_)
 642     return;
 643
 644   CookieStore* cookie_store = request_->context()->cookie_store();
 645   if (cookie_store && !(request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES)) {
 646     cookie_store->GetAllCookiesForURLAsync(
 647         request_->url(),
 648         base::Bind(&URLRequestHttpJob::CheckCookiePolicyAndLoad,
 649                    weak_factory_.GetWeakPtr()));
 650   } else {
 651     DoStartTransaction();
 652   }
 653 }
 654
 655 void URLRequestHttpJob::DoLoadCookies() {
 656   CookieOptions options;
 657   options.set_include_httponly();
 658
 659   // TODO(mkwst): Drop this `if` once we decide whether or not to ship
 660   // first-party cookies: https://crbug.com/459154
 661   if (network_delegate() &&
 662       network_delegate()->FirstPartyOnlyCookieExperimentEnabled())
 663     options.set_first_party_url(request_->first_party_for_cookies());
 664   else
 665     options.set_include_first_party_only();
 666
 667   request_->context()->cookie_store()->GetCookiesWithOptionsAsync(
 668       request_->url(), options, base::Bind(&URLRequestHttpJob::OnCookiesLoaded,
 669                                            weak_factory_.GetWeakPtr()));
 670 }
 671
 672 void URLRequestHttpJob::CheckCookiePolicyAndLoad(
 673     const CookieList& cookie_list) {
 674   if (CanGetCookies(cookie_list))
 675     DoLoadCookies();
 676   else
 677     DoStartTransaction();
 678 }
 679
 680 void URLRequestHttpJob::OnCookiesLoaded(const std::string& cookie_line) {
 681   if (!cookie_line.empty()) {
 682     request_info_.extra_headers.SetHeader(
 683         HttpRequestHeaders::kCookie, cookie_line);
 684     // Disable privacy mode as we are sending cookies anyway.
 685     request_info_.privacy_mode = PRIVACY_MODE_DISABLED;
 686   }
 687   DoStartTransaction();
 688 }
 689
 690 void URLRequestHttpJob::DoStartTransaction() {
 691   // We may have been canceled while retrieving cookies.
 692   if (GetStatus().is_success()) {
 693     StartTransaction();
 694   } else {
 695     NotifyCanceled();
 696   }
 697 }
 698
 699 void URLRequestHttpJob::SaveCookiesAndNotifyHeadersComplete(int result) {
 700   // End of the call started in OnStartCompleted.
 701   OnCallToDelegateComplete();
 702
 703   if (result != OK) {
 704     std::string source("delegate");
 705     request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 706                                  NetLog::StringCallback("source", &source));
 707     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 708     return;
 709   }
 710
 711   DCHECK(transaction_.get());
 712
 713   const HttpResponseInfo* response_info = transaction_->GetResponseInfo();
 714   DCHECK(response_info);
 715
 716   response_cookies_.clear();
 717   response_cookies_save_index_ = 0;
 718
 719   FetchResponseCookies(&response_cookies_);
 720
 721   if (!GetResponseHeaders()->GetDateValue(&response_date_))
 722     response_date_ = base::Time();
 723
 724   // Now, loop over the response cookies, and attempt to persist each.
 725   SaveNextCookie();
 726 }
 727
 728 // If the save occurs synchronously, SaveNextCookie will loop and save the next
 729 // cookie. If the save is deferred, the callback is responsible for continuing
 730 // to iterate through the cookies.
 731 // TODO(erikwright): Modify the CookieStore API to indicate via return value
 732 // whether it completed synchronously or asynchronously.
 733 // See http://crbug.com/131066.
 734 void URLRequestHttpJob::SaveNextCookie() {
 735   // No matter what, we want to report our status as IO pending since we will
 736   // be notifying our consumer asynchronously via OnStartCompleted.
 737   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
 738
 739   // Used to communicate with the callback. See the implementation of
 740   // OnCookieSaved.
 741   scoped_refptr<SharedBoolean> callback_pending = new SharedBoolean(false);
 742   scoped_refptr<SharedBoolean> save_next_cookie_running =
 743       new SharedBoolean(true);
 744
 745   if (!(request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) &&
 746       request_->context()->cookie_store() && response_cookies_.size() > 0) {
 747     CookieOptions options;
 748     options.set_include_httponly();
 749     options.set_server_time(response_date_);
 750
 751     CookieStore::SetCookiesCallback callback(base::Bind(
 752         &URLRequestHttpJob::OnCookieSaved, weak_factory_.GetWeakPtr(),
 753         save_next_cookie_running, callback_pending));
 754
 755     // Loop through the cookies as long as SetCookieWithOptionsAsync completes
 756     // synchronously.
 757     while (!callback_pending->data &&
 758            response_cookies_save_index_ < response_cookies_.size()) {
 759       if (CanSetCookie(
 760           response_cookies_[response_cookies_save_index_], &options)) {
 761         callback_pending->data = true;
 762         request_->context()->cookie_store()->SetCookieWithOptionsAsync(
 763             request_->url(), response_cookies_[response_cookies_save_index_],
 764             options, callback);
 765       }
 766       ++response_cookies_save_index_;
 767     }
 768   }
 769
 770   save_next_cookie_running->data = false;
 771
 772   if (!callback_pending->data) {
 773     response_cookies_.clear();
 774     response_cookies_save_index_ = 0;
 775     SetStatus(URLRequestStatus());  // Clear the IO_PENDING status
 776     NotifyHeadersComplete();
 777     return;
 778   }
 779 }
 780
 781 // |save_next_cookie_running| is true when the callback is bound and set to
 782 // false when SaveNextCookie exits, allowing the callback to determine if the
 783 // save occurred synchronously or asynchronously.
 784 // |callback_pending| is false when the callback is invoked and will be set to
 785 // true by the callback, allowing SaveNextCookie to detect whether the save
 786 // occurred synchronously.
 787 // See SaveNextCookie() for more information.
 788 void URLRequestHttpJob::OnCookieSaved(
 789     scoped_refptr<SharedBoolean> save_next_cookie_running,
 790     scoped_refptr<SharedBoolean> callback_pending,
 791     bool cookie_status) {
 792   callback_pending->data = false;
 793
 794   // If we were called synchronously, return.
 795   if (save_next_cookie_running->data) {
 796     return;
 797   }
 798
 799   // We were called asynchronously, so trigger the next save.
 800   // We may have been canceled within OnSetCookie.
 801   if (GetStatus().is_success()) {
 802     SaveNextCookie();
 803   } else {
 804     NotifyCanceled();
 805   }
 806 }
 807
 808 void URLRequestHttpJob::FetchResponseCookies(
 809     std::vector<std::string>* cookies) {
 810   const std::string name = "Set-Cookie";
 811   std::string value;
 812
 813   void* iter = NULL;
 814   HttpResponseHeaders* headers = GetResponseHeaders();
 815   while (headers->EnumerateHeader(&iter, name, &value)) {
 816     if (!value.empty())
 817       cookies->push_back(value);
 818   }
 819 }
 820
 821 void URLRequestHttpJob::ProcessBackoffHeader() {
 822   DCHECK(response_info_);
 823
 824   if (!backoff_manager_)
 825     return;
 826
 827   TransportSecurityState* security_state =
 828       request_->context()->transport_security_state();
 829   const SSLInfo& ssl_info = response_info_->ssl_info;
 830
 831   // Only accept Backoff headers on HTTPS connections that have no
 832   // certificate errors.
 833   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 834       !security_state)
 835     return;
 836
 837   backoff_manager_->UpdateWithResponse(request()->url(), GetResponseHeaders(),
 838                                        base::Time::Now());
 839 }
 840
 841 // NOTE: |ProcessStrictTransportSecurityHeader| and
 842 // |ProcessPublicKeyPinsHeader| have very similar structures, by design.
 843 void URLRequestHttpJob::ProcessStrictTransportSecurityHeader() {
 844   DCHECK(response_info_);
 845   TransportSecurityState* security_state =
 846       request_->context()->transport_security_state();
 847   const SSLInfo& ssl_info = response_info_->ssl_info;
 848
 849   // Only accept HSTS headers on HTTPS connections that have no
 850   // certificate errors.
 851   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 852       !security_state)
 853     return;
 854
 855   // Don't accept HSTS headers when the hostname is an IP address.
 856   if (request_info_.url.HostIsIPAddress())
 857     return;
 858
 859   // http://tools.ietf.org/html/draft-ietf-websec-strict-transport-sec:
 860   //
 861   //   If a UA receives more than one STS header field in a HTTP response
 862   //   message over secure transport, then the UA MUST process only the
 863   //   first such header field.
 864   HttpResponseHeaders* headers = GetResponseHeaders();
 865   std::string value;
 866   if (headers->EnumerateHeader(NULL, "Strict-Transport-Security", &value))
 867     security_state->AddHSTSHeader(request_info_.url.host(), value);
 868 }
 869
 870 void URLRequestHttpJob::ProcessPublicKeyPinsHeader() {
 871   DCHECK(response_info_);
 872   TransportSecurityState* security_state =
 873       request_->context()->transport_security_state();
 874   const SSLInfo& ssl_info = response_info_->ssl_info;
 875
 876   // Only accept HPKP headers on HTTPS connections that have no
 877   // certificate errors.
 878   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 879       !security_state)
 880     return;
 881
 882   // Don't accept HSTS headers when the hostname is an IP address.
 883   if (request_info_.url.HostIsIPAddress())
 884     return;
 885
 886   // http://tools.ietf.org/html/rfc7469:
 887   //
 888   //   If a UA receives more than one PKP header field in an HTTP
 889   //   response message over secure transport, then the UA MUST process
 890   //   only the first such header field.
 891   HttpResponseHeaders* headers = GetResponseHeaders();
 892   std::string value;
 893   if (headers->EnumerateHeader(nullptr, "Public-Key-Pins", &value))
 894     security_state->AddHPKPHeader(request_info_.url.host(), value, ssl_info);
 895   if (headers->EnumerateHeader(nullptr, "Public-Key-Pins-Report-Only",
 896                                &value)) {
 897     security_state->ProcessHPKPReportOnlyHeader(
 898         value, HostPortPair::FromURL(request_info_.url), ssl_info);
 899   }
 900 }
 901
 902 void URLRequestHttpJob::OnStartCompleted(int result) {
 903   RecordTimer();
 904
 905   // If the request was destroyed, then there is no more work to do.
 906   if (!request_)
 907     return;
 908
 909   // If the job is done (due to cancellation), can just ignore this
 910   // notification.
 911   if (done_)
 912     return;
 913
 914   receive_headers_end_ = base::TimeTicks::Now();
 915
 916   // Clear the IO_PENDING status
 917   SetStatus(URLRequestStatus());
 918
 919   const URLRequestContext* context = request_->context();
 920
 921   if (result == ERR_SSL_PINNED_KEY_NOT_IN_CERT_CHAIN &&
 922       transaction_->GetResponseInfo() != NULL) {
 923     FraudulentCertificateReporter* reporter =
 924       context->fraudulent_certificate_reporter();
 925     if (reporter != NULL) {
 926       const SSLInfo& ssl_info = transaction_->GetResponseInfo()->ssl_info;
 927       const std::string& host = request_->url().host();
 928
 929       reporter->SendReport(host, ssl_info);
 930     }
 931   }
 932
 933   if (result == OK) {
 934     if (transaction_ && transaction_->GetResponseInfo()) {
 935       SetProxyServer(transaction_->GetResponseInfo()->proxy_server);
 936     }
 937     scoped_refptr<HttpResponseHeaders> headers = GetResponseHeaders();
 938     if (network_delegate()) {
 939       // Note that |this| may not be deleted until
 940       // |on_headers_received_callback_| or
 941       // |NetworkDelegate::URLRequestDestroyed()| has been called.
 942       OnCallToDelegate();
 943       allowed_unsafe_redirect_url_ = GURL();
 944       int error = network_delegate()->NotifyHeadersReceived(
 945           request_,
 946           on_headers_received_callback_,
 947           headers.get(),
 948           &override_response_headers_,
 949           &allowed_unsafe_redirect_url_);
 950       if (error != OK) {
 951         if (error == ERR_IO_PENDING) {
 952           awaiting_callback_ = true;
 953         } else {
 954           std::string source("delegate");
 955           request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 956                                        NetLog::StringCallback("source",
 957                                                               &source));
 958           OnCallToDelegateComplete();
 959           NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, error));
 960         }
 961         return;
 962       }
 963     }
 964
 965     SaveCookiesAndNotifyHeadersComplete(OK);
 966   } else if (IsCertificateError(result)) {
 967     // We encountered an SSL certificate error.
 968     if (result == ERR_SSL_WEAK_SERVER_EPHEMERAL_DH_KEY ||
 969         result == ERR_SSL_PINNED_KEY_NOT_IN_CERT_CHAIN) {
 970       // These are hard failures. They're handled separately and don't have
 971       // the correct cert status, so set it here.
 972       SSLInfo info(transaction_->GetResponseInfo()->ssl_info);
 973       info.cert_status = MapNetErrorToCertStatus(result);
 974       NotifySSLCertificateError(info, true);
 975     } else {
 976       // Maybe overridable, maybe not. Ask the delegate to decide.
 977       TransportSecurityState* state = context->transport_security_state();
 978       const bool fatal =
 979           state && state->ShouldSSLErrorsBeFatal(request_info_.url.host());
 980       NotifySSLCertificateError(
 981           transaction_->GetResponseInfo()->ssl_info, fatal);
 982     }
 983   } else if (result == ERR_SSL_CLIENT_AUTH_CERT_NEEDED) {
 984     NotifyCertificateRequested(
 985         transaction_->GetResponseInfo()->cert_request_info.get());
 986   } else {
 987     // Even on an error, there may be useful information in the response
 988     // info (e.g. whether there's a cached copy).
 989     if (transaction_.get())
 990       response_info_ = transaction_->GetResponseInfo();
 991     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 992   }
 993 }
 994
 995 void URLRequestHttpJob::OnHeadersReceivedCallback(int result) {
 996   awaiting_callback_ = false;
 997
 998   // Check that there are no callbacks to already canceled requests.
 999   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
1000
1001   SaveCookiesAndNotifyHeadersComplete(result);
1002 }
1003
1004 void URLRequestHttpJob::OnReadCompleted(int result) {
1005   read_in_progress_ = false;
1006
1007   if (ShouldFixMismatchedContentLength(result))
1008     result = OK;
1009
1010   if (result == OK) {
1011     NotifyDone(URLRequestStatus());
1012   } else if (result < 0) {
1013     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, result));
1014   } else {
1015     // Clear the IO_PENDING status
1016     SetStatus(URLRequestStatus());
1017   }
1018
1019   NotifyReadComplete(result);
1020 }
1021
1022 void URLRequestHttpJob::RestartTransactionWithAuth(
1023     const AuthCredentials& credentials) {
1024   auth_credentials_ = credentials;
1025
1026   // These will be reset in OnStartCompleted.
1027   response_info_ = NULL;
1028   receive_headers_end_ = base::TimeTicks();
1029   response_cookies_.clear();
1030
1031   ResetTimer();
1032
1033   // Update the cookies, since the cookie store may have been updated from the
1034   // headers in the 401/407. Since cookies were already appended to
1035   // extra_headers, we need to strip them out before adding them again.
1036   request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kCookie);
1037
1038   AddCookieHeaderAndStart();
1039 }
1040
1041 void URLRequestHttpJob::SetUpload(UploadDataStream* upload) {
1042   DCHECK(!transaction_.get()) << "cannot change once started";
1043   request_info_.upload_data_stream = upload;
1044 }
1045
1046 void URLRequestHttpJob::SetExtraRequestHeaders(
1047     const HttpRequestHeaders& headers) {
1048   DCHECK(!transaction_.get()) << "cannot change once started";
1049   request_info_.extra_headers.CopyFrom(headers);
1050 }
1051
1052 LoadState URLRequestHttpJob::GetLoadState() const {
1053   return transaction_.get() ?
1054       transaction_->GetLoadState() : LOAD_STATE_IDLE;
1055 }
1056
1057 UploadProgress URLRequestHttpJob::GetUploadProgress() const {
1058   return transaction_.get() ?
1059       transaction_->GetUploadProgress() : UploadProgress();
1060 }
1061
1062 bool URLRequestHttpJob::GetMimeType(std::string* mime_type) const {
1063   DCHECK(transaction_.get());
1064
1065   if (!response_info_)
1066     return false;
1067
1068   HttpResponseHeaders* headers = GetResponseHeaders();
1069   if (!headers)
1070     return false;
1071   return headers->GetMimeType(mime_type);
1072 }
1073
1074 bool URLRequestHttpJob::GetCharset(std::string* charset) {
1075   DCHECK(transaction_.get());
1076
1077   if (!response_info_)
1078     return false;
1079
1080   return GetResponseHeaders()->GetCharset(charset);
1081 }
1082
1083 void URLRequestHttpJob::GetResponseInfo(HttpResponseInfo* info) {
1084   DCHECK(request_);
1085
1086   if (response_info_) {
1087     DCHECK(transaction_.get());
1088
1089     *info = *response_info_;
1090     if (override_response_headers_.get())
1091       info->headers = override_response_headers_;
1092   }
1093 }
1094
1095 void URLRequestHttpJob::GetLoadTimingInfo(
1096     LoadTimingInfo* load_timing_info) const {
1097   // If haven't made it far enough to receive any headers, don't return
1098   // anything. This makes for more consistent behavior in the case of errors.
1099   if (!transaction_ || receive_headers_end_.is_null())
1100     return;
1101   if (transaction_->GetLoadTimingInfo(load_timing_info))
1102     load_timing_info->receive_headers_end = receive_headers_end_;
1103 }
1104
1105 bool URLRequestHttpJob::GetResponseCookies(std::vector<std::string>* cookies) {
1106   DCHECK(transaction_.get());
1107
1108   if (!response_info_)
1109     return false;
1110
1111   // TODO(darin): Why are we extracting response cookies again?  Perhaps we
1112   // should just leverage response_cookies_.
1113
1114   cookies->clear();
1115   FetchResponseCookies(cookies);
1116   return true;
1117 }
1118
1119 int URLRequestHttpJob::GetResponseCode() const {
1120   DCHECK(transaction_.get());
1121
1122   if (!response_info_)
1123     return -1;
1124
1125   return GetResponseHeaders()->response_code();
1126 }
1127
1128 Filter* URLRequestHttpJob::SetupFilter() const {
1129   DCHECK(transaction_.get());
1130   if (!response_info_)
1131     return NULL;
1132
1133   std::vector<Filter::FilterType> encoding_types;
1134   std::string encoding_type;
1135   HttpResponseHeaders* headers = GetResponseHeaders();
1136   void* iter = NULL;
1137   while (headers->EnumerateHeader(&iter, "Content-Encoding", &encoding_type)) {
1138     encoding_types.push_back(Filter::ConvertEncodingToType(encoding_type));
1139   }
1140
1141   // Even if encoding types are empty, there is a chance that we need to add
1142   // some decoding, as some proxies strip encoding completely. In such cases,
1143   // we may need to add (for example) SDCH filtering (when the context suggests
1144   // it is appropriate).
1145   Filter::FixupEncodingTypes(*filter_context_, &encoding_types);
1146
1147   return !encoding_types.empty()
1148       ? Filter::Factory(encoding_types, *filter_context_) : NULL;
1149 }
1150
1151 bool URLRequestHttpJob::CopyFragmentOnRedirect(const GURL& location) const {
1152   // Allow modification of reference fragments by default, unless
1153   // |allowed_unsafe_redirect_url_| is set and equal to the redirect URL.
1154   // When this is the case, we assume that the network delegate has set the
1155   // desired redirect URL (with or without fragment), so it must not be changed
1156   // any more.
1157   return !allowed_unsafe_redirect_url_.is_valid() ||
1158        allowed_unsafe_redirect_url_ != location;
1159 }
1160
1161 bool URLRequestHttpJob::IsSafeRedirect(const GURL& location) {
1162   // HTTP is always safe.
1163   // TODO(pauljensen): Remove once crbug.com/146591 is fixed.
1164   if (location.is_valid() &&
1165       (location.scheme() == "http" || location.scheme() == "https")) {
1166     return true;
1167   }
1168   // Delegates may mark a URL as safe for redirection.
1169   if (allowed_unsafe_redirect_url_.is_valid() &&
1170       allowed_unsafe_redirect_url_ == location) {
1171     return true;
1172   }
1173   // Query URLRequestJobFactory as to whether |location| would be safe to
1174   // redirect to.
1175   return request_->context()->job_factory() &&
1176       request_->context()->job_factory()->IsSafeRedirectTarget(location);
1177 }
1178
1179 bool URLRequestHttpJob::NeedsAuth() {
1180   int code = GetResponseCode();
1181   if (code == -1)
1182     return false;
1183
1184   // Check if we need either Proxy or WWW Authentication. This could happen
1185   // because we either provided no auth info, or provided incorrect info.
1186   switch (code) {
1187     case 407:
1188       if (proxy_auth_state_ == AUTH_STATE_CANCELED)
1189         return false;
1190       proxy_auth_state_ = AUTH_STATE_NEED_AUTH;
1191       return true;
1192     case 401:
1193       if (server_auth_state_ == AUTH_STATE_CANCELED)
1194         return false;
1195       server_auth_state_ = AUTH_STATE_NEED_AUTH;
1196       return true;
1197   }
1198   return false;
1199 }
1200
1201 void URLRequestHttpJob::GetAuthChallengeInfo(
1202     scoped_refptr<AuthChallengeInfo>* result) {
1203   DCHECK(transaction_.get());
1204   DCHECK(response_info_);
1205
1206   // sanity checks:
1207   DCHECK(proxy_auth_state_ == AUTH_STATE_NEED_AUTH ||
1208          server_auth_state_ == AUTH_STATE_NEED_AUTH);
1209   DCHECK((GetResponseHeaders()->response_code() == HTTP_UNAUTHORIZED) ||
1210          (GetResponseHeaders()->response_code() ==
1211           HTTP_PROXY_AUTHENTICATION_REQUIRED));
1212
1213   *result = response_info_->auth_challenge;
1214 }
1215
1216 void URLRequestHttpJob::SetAuth(const AuthCredentials& credentials) {
1217   DCHECK(transaction_.get());
1218
1219   // Proxy gets set first, then WWW.
1220   if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1221     proxy_auth_state_ = AUTH_STATE_HAVE_AUTH;
1222   } else {
1223     DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1224     server_auth_state_ = AUTH_STATE_HAVE_AUTH;
1225   }
1226
1227   RestartTransactionWithAuth(credentials);
1228 }
1229
1230 void URLRequestHttpJob::CancelAuth() {
1231   // Proxy gets set first, then WWW.
1232   if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1233     proxy_auth_state_ = AUTH_STATE_CANCELED;
1234   } else {
1235     DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1236     server_auth_state_ = AUTH_STATE_CANCELED;
1237   }
1238
1239   // These will be reset in OnStartCompleted.
1240   response_info_ = NULL;
1241   receive_headers_end_ = base::TimeTicks::Now();
1242   response_cookies_.clear();
1243
1244   ResetTimer();
1245
1246   // OK, let the consumer read the error page...
1247   //
1248   // Because we set the AUTH_STATE_CANCELED flag, NeedsAuth will return false,
1249   // which will cause the consumer to receive OnResponseStarted instead of
1250   // OnAuthRequired.
1251   //
1252   // We have to do this via InvokeLater to avoid "recursing" the consumer.
1253   //
1254   base::ThreadTaskRunnerHandle::Get()->PostTask(
1255       FROM_HERE, base::Bind(&URLRequestHttpJob::OnStartCompleted,
1256                             weak_factory_.GetWeakPtr(), OK));
1257 }
1258
1259 void URLRequestHttpJob::ContinueWithCertificate(
1260     X509Certificate* client_cert) {
1261   DCHECK(transaction_.get());
1262
1263   DCHECK(!response_info_) << "should not have a response yet";
1264   receive_headers_end_ = base::TimeTicks();
1265
1266   ResetTimer();
1267
1268   // No matter what, we want to report our status as IO pending since we will
1269   // be notifying our consumer asynchronously via OnStartCompleted.
1270   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1271
1272   int rv = transaction_->RestartWithCertificate(client_cert, start_callback_);
1273   if (rv == ERR_IO_PENDING)
1274     return;
1275
1276   // The transaction started synchronously, but we need to notify the
1277   // URLRequest delegate via the message loop.
1278   base::ThreadTaskRunnerHandle::Get()->PostTask(
1279       FROM_HERE, base::Bind(&URLRequestHttpJob::OnStartCompleted,
1280                             weak_factory_.GetWeakPtr(), rv));
1281 }
1282
1283 void URLRequestHttpJob::ContinueDespiteLastError() {
1284   // If the transaction was destroyed, then the job was cancelled.
1285   if (!transaction_.get())
1286     return;
1287
1288   DCHECK(!response_info_) << "should not have a response yet";
1289   receive_headers_end_ = base::TimeTicks();
1290
1291   ResetTimer();
1292
1293   // No matter what, we want to report our status as IO pending since we will
1294   // be notifying our consumer asynchronously via OnStartCompleted.
1295   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1296
1297   int rv = transaction_->RestartIgnoringLastError(start_callback_);
1298   if (rv == ERR_IO_PENDING)
1299     return;
1300
1301   // The transaction started synchronously, but we need to notify the
1302   // URLRequest delegate via the message loop.
1303   base::ThreadTaskRunnerHandle::Get()->PostTask(
1304       FROM_HERE, base::Bind(&URLRequestHttpJob::OnStartCompleted,
1305                             weak_factory_.GetWeakPtr(), rv));
1306 }
1307
1308 void URLRequestHttpJob::ResumeNetworkStart() {
1309   DCHECK(transaction_.get());
1310   transaction_->ResumeNetworkStart();
1311 }
1312
1313 bool URLRequestHttpJob::ShouldFixMismatchedContentLength(int rv) const {
1314   // Some servers send the body compressed, but specify the content length as
1315   // the uncompressed size. Although this violates the HTTP spec we want to
1316   // support it (as IE and FireFox do), but *only* for an exact match.
1317   // See http://crbug.com/79694.
1318   if (rv == ERR_CONTENT_LENGTH_MISMATCH ||
1319       rv == ERR_INCOMPLETE_CHUNKED_ENCODING) {
1320     if (request_ && request_->response_headers()) {
1321       int64 expected_length = request_->response_headers()->GetContentLength();
1322       VLOG(1) << __FUNCTION__ << "() "
1323               << "\"" << request_->url().spec() << "\""
1324               << " content-length = " << expected_length
1325               << " pre total = " << prefilter_bytes_read()
1326               << " post total = " << postfilter_bytes_read();
1327       if (postfilter_bytes_read() == expected_length) {
1328         // Clear the error.
1329         return true;
1330       }
1331     }
1332   }
1333   return false;
1334 }
1335
1336 bool URLRequestHttpJob::ReadRawData(IOBuffer* buf, int buf_size,
1337                                     int* bytes_read) {
1338   DCHECK_NE(buf_size, 0);
1339   DCHECK(bytes_read);
1340   DCHECK(!read_in_progress_);
1341
1342   int rv = transaction_->Read(
1343       buf, buf_size,
1344       base::Bind(&URLRequestHttpJob::OnReadCompleted, base::Unretained(this)));
1345
1346   if (ShouldFixMismatchedContentLength(rv))
1347     rv = 0;
1348
1349   if (rv >= 0) {
1350     *bytes_read = rv;
1351     if (!rv)
1352       DoneWithRequest(FINISHED);
1353     return true;
1354   }
1355
1356   if (rv == ERR_IO_PENDING) {
1357     read_in_progress_ = true;
1358     SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1359   } else {
1360     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, rv));
1361   }
1362
1363   return false;
1364 }
1365
1366 void URLRequestHttpJob::StopCaching() {
1367   if (transaction_.get())
1368     transaction_->StopCaching();
1369 }
1370
1371 bool URLRequestHttpJob::GetFullRequestHeaders(
1372     HttpRequestHeaders* headers) const {
1373   if (!transaction_)
1374     return false;
1375
1376   return transaction_->GetFullRequestHeaders(headers);
1377 }
1378
1379 int64 URLRequestHttpJob::GetTotalReceivedBytes() const {
1380   if (!transaction_)
1381     return 0;
1382
1383   return transaction_->GetTotalReceivedBytes();
1384 }
1385
1386 void URLRequestHttpJob::DoneReading() {
1387   if (transaction_) {
1388     transaction_->DoneReading();
1389   }
1390   DoneWithRequest(FINISHED);
1391 }
1392
1393 void URLRequestHttpJob::DoneReadingRedirectResponse() {
1394   if (transaction_) {
1395     if (transaction_->GetResponseInfo()->headers->IsRedirect(NULL)) {
1396       // If the original headers indicate a redirect, go ahead and cache the
1397       // response, even if the |override_response_headers_| are a redirect to
1398       // another location.
1399       transaction_->DoneReading();
1400     } else {
1401       // Otherwise, |override_response_headers_| must be non-NULL and contain
1402       // bogus headers indicating a redirect.
1403       DCHECK(override_response_headers_.get());
1404       DCHECK(override_response_headers_->IsRedirect(NULL));
1405       transaction_->StopCaching();
1406     }
1407   }
1408   DoneWithRequest(FINISHED);
1409 }
1410
1411 HostPortPair URLRequestHttpJob::GetSocketAddress() const {
1412   return response_info_ ? response_info_->socket_address : HostPortPair();
1413 }
1414
1415 void URLRequestHttpJob::RecordTimer() {
1416   if (request_creation_time_.is_null()) {
1417     NOTREACHED()
1418         << "The same transaction shouldn't start twice without new timing.";
1419     return;
1420   }
1421
1422   base::TimeDelta to_start = base::Time::Now() - request_creation_time_;
1423   request_creation_time_ = base::Time();
1424
1425   UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpTimeToFirstByte", to_start);
1426 }
1427
1428 void URLRequestHttpJob::ResetTimer() {
1429   if (!request_creation_time_.is_null()) {
1430     NOTREACHED()
1431         << "The timer was reset before it was recorded.";
1432     return;
1433   }
1434   request_creation_time_ = base::Time::Now();
1435 }
1436
1437 void URLRequestHttpJob::UpdatePacketReadTimes() {
1438   if (!packet_timing_enabled_)
1439     return;
1440
1441   DCHECK_GT(prefilter_bytes_read(), bytes_observed_in_packets_);
1442
1443   base::Time now(base::Time::Now());
1444   if (!bytes_observed_in_packets_)
1445     request_time_snapshot_ = now;
1446   final_packet_time_ = now;
1447
1448   bytes_observed_in_packets_ = prefilter_bytes_read();
1449 }
1450
1451 void URLRequestHttpJob::RecordPacketStats(
1452     FilterContext::StatisticSelector statistic) const {
1453   if (!packet_timing_enabled_ || (final_packet_time_ == base::Time()))
1454     return;
1455
1456   base::TimeDelta duration = final_packet_time_ - request_time_snapshot_;
1457   switch (statistic) {
1458     case FilterContext::SDCH_DECODE: {
1459       UMA_HISTOGRAM_CUSTOM_COUNTS("Sdch3.Network_Decode_Bytes_Processed_b",
1460           static_cast<int>(bytes_observed_in_packets_), 500, 100000, 100);
1461       return;
1462     }
1463     case FilterContext::SDCH_PASSTHROUGH: {
1464       // Despite advertising a dictionary, we handled non-sdch compressed
1465       // content.
1466       return;
1467     }
1468
1469     case FilterContext::SDCH_EXPERIMENT_DECODE: {
1470       UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment3_Decode",
1471                                   duration,
1472                                   base::TimeDelta::FromMilliseconds(20),
1473                                   base::TimeDelta::FromMinutes(10), 100);
1474       return;
1475     }
1476     case FilterContext::SDCH_EXPERIMENT_HOLDBACK: {
1477       UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment3_Holdback",
1478                                   duration,
1479                                   base::TimeDelta::FromMilliseconds(20),
1480                                   base::TimeDelta::FromMinutes(10), 100);
1481       return;
1482     }
1483     default:
1484       NOTREACHED();
1485       return;
1486   }
1487 }
1488
1489 void URLRequestHttpJob::RecordPerfHistograms(CompletionCause reason) {
1490   if (start_time_.is_null())
1491     return;
1492
1493   base::TimeDelta total_time = base::TimeTicks::Now() - start_time_;
1494   UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTime", total_time);
1495
1496   if (reason == FINISHED) {
1497     UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeSuccess", total_time);
1498   } else {
1499     UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCancel", total_time);
1500   }
1501
1502   if (response_info_) {
1503     // QUIC (by default) supports https scheme only, thus track https URLs only
1504     // for QUIC.
1505     bool is_https_google = request() && request()->url().SchemeIs("https") &&
1506                            HasGoogleHost(request()->url());
1507     bool used_quic = response_info_->DidUseQuic();
1508     if (is_https_google) {
1509       if (used_quic) {
1510         UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTime.Secure.Quic",
1511                                    total_time);
1512       } else {
1513         UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTime.Secure.NotQuic",
1514                                    total_time);
1515       }
1516     }
1517     if (response_info_->was_cached) {
1518       UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCached", total_time);
1519       if (is_https_google) {
1520         if (used_quic) {
1521           UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpJob.TotalTimeCached.Secure.Quic",
1522                                      total_time);
1523         } else {
1524           UMA_HISTOGRAM_MEDIUM_TIMES(
1525               "Net.HttpJob.TotalTimeCached.Secure.NotQuic", total_time);
1526         }
1527       }
1528     } else  {
1529       UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeNotCached", total_time);
1530       if (is_https_google) {
1531         if (used_quic) {
1532           UMA_HISTOGRAM_MEDIUM_TIMES(
1533               "Net.HttpJob.TotalTimeNotCached.Secure.Quic", total_time);
1534         } else {
1535           UMA_HISTOGRAM_MEDIUM_TIMES(
1536               "Net.HttpJob.TotalTimeNotCached.Secure.NotQuic", total_time);
1537         }
1538       }
1539     }
1540   }
1541
1542   if (request_info_.load_flags & LOAD_PREFETCH && !request_->was_cached())
1543     UMA_HISTOGRAM_COUNTS("Net.Prefetch.PrefilterBytesReadFromNetwork",
1544                          prefilter_bytes_read());
1545
1546   start_time_ = base::TimeTicks();
1547 }
1548
1549 void URLRequestHttpJob::DoneWithRequest(CompletionCause reason) {
1550   if (done_)
1551     return;
1552   done_ = true;
1553   RecordPerfHistograms(reason);
1554   if (request_)
1555     request_->set_received_response_content_length(prefilter_bytes_read());
1556 }
1557
1558 HttpResponseHeaders* URLRequestHttpJob::GetResponseHeaders() const {
1559   DCHECK(transaction_.get());
1560   DCHECK(transaction_->GetResponseInfo());
1561   return override_response_headers_.get() ?
1562              override_response_headers_.get() :
1563              transaction_->GetResponseInfo()->headers.get();
1564 }
1565
1566 void URLRequestHttpJob::NotifyURLRequestDestroyed() {
1567   awaiting_callback_ = false;
1568 }
1569
1570 }  // namespace net