net/url_request/url_request_http_job.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_request_http_job.h"
   6
   7 #include "base/base_switches.h"
   8 #include "base/bind.h"
   9 #include "base/bind_helpers.h"
  10 #include "base/command_line.h"
  11 #include "base/compiler_specific.h"
  12 #include "base/file_util.h"
  13 #include "base/file_version_info.h"
  14 #include "base/message_loop.h"
  15 #include "base/metrics/field_trial.h"
  16 #include "base/metrics/histogram.h"
  17 #include "base/rand_util.h"
  18 #include "base/string_util.h"
  19 #include "base/time.h"
  20 #include "net/base/cert_status_flags.h"
  21 #include "net/base/filter.h"
  22 #include "net/base/host_port_pair.h"
  23 #include "net/base/load_flags.h"
  24 #include "net/base/mime_util.h"
  25 #include "net/base/net_errors.h"
  26 #include "net/base/net_util.h"
  27 #include "net/base/network_delegate.h"
  28 #include "net/base/sdch_manager.h"
  29 #include "net/base/ssl_cert_request_info.h"
  30 #include "net/base/ssl_config_service.h"
  31 #include "net/cookies/cookie_monster.h"
  32 #include "net/http/http_network_session.h"
  33 #include "net/http/http_request_headers.h"
  34 #include "net/http/http_response_headers.h"
  35 #include "net/http/http_response_info.h"
  36 #include "net/http/http_status_code.h"
  37 #include "net/http/http_transaction.h"
  38 #include "net/http/http_transaction_delegate.h"
  39 #include "net/http/http_transaction_factory.h"
  40 #include "net/http/http_util.h"
  41 #include "net/url_request/fraudulent_certificate_reporter.h"
  42 #include "net/url_request/http_user_agent_settings.h"
  43 #include "net/url_request/url_request.h"
  44 #include "net/url_request/url_request_context.h"
  45 #include "net/url_request/url_request_error_job.h"
  46 #include "net/url_request/url_request_redirect_job.h"
  47 #include "net/url_request/url_request_throttler_header_adapter.h"
  48 #include "net/url_request/url_request_throttler_manager.h"
  49
  50 static const char kAvailDictionaryHeader[] = "Avail-Dictionary";
  51
  52 namespace net {
  53
  54 class URLRequestHttpJob::HttpFilterContext : public FilterContext {
  55  public:
  56   explicit HttpFilterContext(URLRequestHttpJob* job);
  57   virtual ~HttpFilterContext();
  58
  59   // FilterContext implementation.
  60   virtual bool GetMimeType(std::string* mime_type) const OVERRIDE;
  61   virtual bool GetURL(GURL* gurl) const OVERRIDE;
  62   virtual base::Time GetRequestTime() const OVERRIDE;
  63   virtual bool IsCachedContent() const OVERRIDE;
  64   virtual bool IsDownload() const OVERRIDE;
  65   virtual bool IsSdchResponse() const OVERRIDE;
  66   virtual int64 GetByteReadCount() const OVERRIDE;
  67   virtual int GetResponseCode() const OVERRIDE;
  68   virtual void RecordPacketStats(StatisticSelector statistic) const OVERRIDE;
  69
  70   // Method to allow us to reset filter context for a response that should have
  71   // been SDCH encoded when there is an update due to an explicit HTTP header.
  72   void ResetSdchResponseToFalse();
  73
  74  private:
  75   URLRequestHttpJob* job_;
  76
  77   DISALLOW_COPY_AND_ASSIGN(HttpFilterContext);
  78 };
  79
  80 class URLRequestHttpJob::HttpTransactionDelegateImpl
  81     : public HttpTransactionDelegate {
  82  public:
  83   explicit HttpTransactionDelegateImpl(URLRequest* request)
  84       : request_(request),
  85         network_delegate_(request->context()->network_delegate()),
  86         cache_active_(false),
  87         network_active_(false) {
  88   }
  89   virtual ~HttpTransactionDelegateImpl() {
  90     OnDetachRequest();
  91   }
  92   void OnDetachRequest() {
  93     if (request_ == NULL || network_delegate_ == NULL)
  94       return;
  95     network_delegate_->NotifyRequestWaitStateChange(
  96         *request_,
  97         NetworkDelegate::REQUEST_WAIT_STATE_RESET);
  98     cache_active_ = false;
  99     network_active_ = false;
 100     request_ = NULL;
 101   }
 102   virtual void OnCacheActionStart() OVERRIDE {
 103     if (request_ == NULL || network_delegate_ == NULL)
 104       return;
 105     DCHECK(!cache_active_ && !network_active_);
 106     cache_active_ = true;
 107     network_delegate_->NotifyRequestWaitStateChange(
 108         *request_,
 109         NetworkDelegate::REQUEST_WAIT_STATE_CACHE_START);
 110   }
 111   virtual void OnCacheActionFinish() OVERRIDE {
 112     if (request_ == NULL || network_delegate_ == NULL)
 113       return;
 114     DCHECK(cache_active_ && !network_active_);
 115     cache_active_ = false;
 116     network_delegate_->NotifyRequestWaitStateChange(
 117         *request_,
 118         NetworkDelegate::REQUEST_WAIT_STATE_CACHE_FINISH);
 119   }
 120   virtual void OnNetworkActionStart() OVERRIDE {
 121     if (request_ == NULL || network_delegate_ == NULL)
 122       return;
 123     DCHECK(!cache_active_ && !network_active_);
 124     network_active_ = true;
 125     network_delegate_->NotifyRequestWaitStateChange(
 126         *request_,
 127         NetworkDelegate::REQUEST_WAIT_STATE_NETWORK_START);
 128   }
 129   virtual void OnNetworkActionFinish() OVERRIDE {
 130     if (request_ == NULL || network_delegate_ == NULL)
 131       return;
 132     DCHECK(!cache_active_ && network_active_);
 133     network_active_ = false;
 134     network_delegate_->NotifyRequestWaitStateChange(
 135         *request_,
 136         NetworkDelegate::REQUEST_WAIT_STATE_NETWORK_FINISH);
 137   }
 138  private:
 139   URLRequest* request_;
 140   NetworkDelegate* network_delegate_;
 141   bool cache_active_;
 142   bool network_active_;
 143 };
 144
 145 URLRequestHttpJob::HttpFilterContext::HttpFilterContext(URLRequestHttpJob* job)
 146     : job_(job) {
 147   DCHECK(job_);
 148 }
 149
 150 URLRequestHttpJob::HttpFilterContext::~HttpFilterContext() {
 151 }
 152
 153 bool URLRequestHttpJob::HttpFilterContext::GetMimeType(
 154     std::string* mime_type) const {
 155   return job_->GetMimeType(mime_type);
 156 }
 157
 158 bool URLRequestHttpJob::HttpFilterContext::GetURL(GURL* gurl) const {
 159   if (!job_->request())
 160     return false;
 161   *gurl = job_->request()->url();
 162   return true;
 163 }
 164
 165 base::Time URLRequestHttpJob::HttpFilterContext::GetRequestTime() const {
 166   return job_->request() ? job_->request()->request_time() : base::Time();
 167 }
 168
 169 bool URLRequestHttpJob::HttpFilterContext::IsCachedContent() const {
 170   return job_->is_cached_content_;
 171 }
 172
 173 bool URLRequestHttpJob::HttpFilterContext::IsDownload() const {
 174   return (job_->request_info_.load_flags & LOAD_IS_DOWNLOAD) != 0;
 175 }
 176
 177 void URLRequestHttpJob::HttpFilterContext::ResetSdchResponseToFalse() {
 178   DCHECK(job_->sdch_dictionary_advertised_);
 179   job_->sdch_dictionary_advertised_ = false;
 180 }
 181
 182 bool URLRequestHttpJob::HttpFilterContext::IsSdchResponse() const {
 183   return job_->sdch_dictionary_advertised_;
 184 }
 185
 186 int64 URLRequestHttpJob::HttpFilterContext::GetByteReadCount() const {
 187   return job_->filter_input_byte_count();
 188 }
 189
 190 int URLRequestHttpJob::HttpFilterContext::GetResponseCode() const {
 191   return job_->GetResponseCode();
 192 }
 193
 194 void URLRequestHttpJob::HttpFilterContext::RecordPacketStats(
 195     StatisticSelector statistic) const {
 196   job_->RecordPacketStats(statistic);
 197 }
 198
 199 // TODO(darin): make sure the port blocking code is not lost
 200 // static
 201 URLRequestJob* URLRequestHttpJob::Factory(URLRequest* request,
 202                                           NetworkDelegate* network_delegate,
 203                                           const std::string& scheme) {
 204   DCHECK(scheme == "http" || scheme == "https");
 205
 206   if (!request->context()->http_transaction_factory()) {
 207     NOTREACHED() << "requires a valid context";
 208     return new URLRequestErrorJob(
 209         request, network_delegate, ERR_INVALID_ARGUMENT);
 210   }
 211
 212   GURL redirect_url;
 213   if (request->GetHSTSRedirect(&redirect_url)) {
 214     return new URLRequestRedirectJob(
 215         request, network_delegate, redirect_url,
 216         // Use status code 307 to preserve the method, so POST requests work.
 217         URLRequestRedirectJob::REDIRECT_307_TEMPORARY_REDIRECT);
 218   }
 219   return new URLRequestHttpJob(request,
 220                                network_delegate,
 221                                request->context()->http_user_agent_settings());
 222 }
 223
 224
 225 URLRequestHttpJob::URLRequestHttpJob(
 226     URLRequest* request,
 227     NetworkDelegate* network_delegate,
 228     const HttpUserAgentSettings* http_user_agent_settings)
 229     : URLRequestJob(request, network_delegate),
 230       response_info_(NULL),
 231       response_cookies_save_index_(0),
 232       proxy_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
 233       server_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
 234       ALLOW_THIS_IN_INITIALIZER_LIST(start_callback_(
 235           base::Bind(&URLRequestHttpJob::OnStartCompleted,
 236                      base::Unretained(this)))),
 237       ALLOW_THIS_IN_INITIALIZER_LIST(notify_before_headers_sent_callback_(
 238           base::Bind(&URLRequestHttpJob::NotifyBeforeSendHeadersCallback,
 239                      base::Unretained(this)))),
 240       read_in_progress_(false),
 241       transaction_(NULL),
 242       throttling_entry_(NULL),
 243       sdch_dictionary_advertised_(false),
 244       sdch_test_activated_(false),
 245       sdch_test_control_(false),
 246       is_cached_content_(false),
 247       request_creation_time_(),
 248       packet_timing_enabled_(false),
 249       done_(false),
 250       bytes_observed_in_packets_(0),
 251       request_time_snapshot_(),
 252       final_packet_time_(),
 253       ALLOW_THIS_IN_INITIALIZER_LIST(
 254           filter_context_(new HttpFilterContext(this))),
 255       ALLOW_THIS_IN_INITIALIZER_LIST(weak_factory_(this)),
 256       ALLOW_THIS_IN_INITIALIZER_LIST(on_headers_received_callback_(
 257           base::Bind(&URLRequestHttpJob::OnHeadersReceivedCallback,
 258                      base::Unretained(this)))),
 259       awaiting_callback_(false),
 260       http_transaction_delegate_(new HttpTransactionDelegateImpl(request)),
 261       http_user_agent_settings_(http_user_agent_settings) {
 262   URLRequestThrottlerManager* manager = request->context()->throttler_manager();
 263   if (manager)
 264     throttling_entry_ = manager->RegisterRequestUrl(request->url());
 265
 266   ResetTimer();
 267 }
 268
 269 void URLRequestHttpJob::NotifyHeadersComplete() {
 270   DCHECK(!response_info_);
 271
 272   response_info_ = transaction_->GetResponseInfo();
 273
 274   // Save boolean, as we'll need this info at destruction time, and filters may
 275   // also need this info.
 276   is_cached_content_ = response_info_->was_cached;
 277
 278   if (!is_cached_content_ && throttling_entry_) {
 279     URLRequestThrottlerHeaderAdapter response_adapter(GetResponseHeaders());
 280     throttling_entry_->UpdateWithResponse(request_info_.url.host(),
 281                                           &response_adapter);
 282   }
 283
 284   // The ordering of these calls is not important.
 285   ProcessStrictTransportSecurityHeader();
 286   ProcessPublicKeyPinsHeader();
 287
 288   if (SdchManager::Global() &&
 289       SdchManager::Global()->IsInSupportedDomain(request_->url())) {
 290     const std::string name = "Get-Dictionary";
 291     std::string url_text;
 292     void* iter = NULL;
 293     // TODO(jar): We need to not fetch dictionaries the first time they are
 294     // seen, but rather wait until we can justify their usefulness.
 295     // For now, we will only fetch the first dictionary, which will at least
 296     // require multiple suggestions before we get additional ones for this site.
 297     // Eventually we should wait until a dictionary is requested several times
 298     // before we even download it (so that we don't waste memory or bandwidth).
 299     if (GetResponseHeaders()->EnumerateHeader(&iter, name, &url_text)) {
 300       // request_->url() won't be valid in the destructor, so we use an
 301       // alternate copy.
 302       DCHECK_EQ(request_->url(), request_info_.url);
 303       // Resolve suggested URL relative to request url.
 304       sdch_dictionary_url_ = request_info_.url.Resolve(url_text);
 305     }
 306   }
 307
 308   // The HTTP transaction may be restarted several times for the purposes
 309   // of sending authorization information. Each time it restarts, we get
 310   // notified of the headers completion so that we can update the cookie store.
 311   if (transaction_->IsReadyToRestartForAuth()) {
 312     DCHECK(!response_info_->auth_challenge.get());
 313     // TODO(battre): This breaks the webrequest API for
 314     // URLRequestTestHTTP.BasicAuthWithCookies
 315     // where OnBeforeSendHeaders -> OnSendHeaders -> OnBeforeSendHeaders
 316     // occurs.
 317     RestartTransactionWithAuth(AuthCredentials());
 318     return;
 319   }
 320
 321   URLRequestJob::NotifyHeadersComplete();
 322 }
 323
 324 void URLRequestHttpJob::NotifyDone(const URLRequestStatus& status) {
 325   DoneWithRequest(FINISHED);
 326   URLRequestJob::NotifyDone(status);
 327 }
 328
 329 void URLRequestHttpJob::DestroyTransaction() {
 330   DCHECK(transaction_.get());
 331
 332   DoneWithRequest(ABORTED);
 333   transaction_.reset();
 334   response_info_ = NULL;
 335 }
 336
 337 void URLRequestHttpJob::StartTransaction() {
 338   if (request_->context()->network_delegate()) {
 339     int rv = request_->context()->network_delegate()->NotifyBeforeSendHeaders(
 340         request_, notify_before_headers_sent_callback_,
 341         &request_info_.extra_headers);
 342     // If an extension blocks the request, we rely on the callback to
 343     // MaybeStartTransactionInternal().
 344     if (rv == ERR_IO_PENDING) {
 345       SetBlockedOnDelegate();
 346       return;
 347     }
 348     MaybeStartTransactionInternal(rv);
 349     return;
 350   }
 351   StartTransactionInternal();
 352 }
 353
 354 void URLRequestHttpJob::NotifyBeforeSendHeadersCallback(int result) {
 355   SetUnblockedOnDelegate();
 356
 357   // Check that there are no callbacks to already canceled requests.
 358   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 359
 360   MaybeStartTransactionInternal(result);
 361 }
 362
 363 void URLRequestHttpJob::MaybeStartTransactionInternal(int result) {
 364   if (result == OK) {
 365     StartTransactionInternal();
 366   } else {
 367     std::string source("delegate");
 368     request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 369                                  NetLog::StringCallback("source", &source));
 370     NotifyCanceled();
 371     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 372   }
 373 }
 374
 375 void URLRequestHttpJob::StartTransactionInternal() {
 376   // NOTE: This method assumes that request_info_ is already setup properly.
 377
 378   // If we already have a transaction, then we should restart the transaction
 379   // with auth provided by auth_credentials_.
 380
 381   int rv;
 382
 383   if (request_->context()->network_delegate()) {
 384     request_->context()->network_delegate()->NotifySendHeaders(
 385         request_, request_info_.extra_headers);
 386   }
 387
 388   if (transaction_.get()) {
 389     rv = transaction_->RestartWithAuth(auth_credentials_, start_callback_);
 390     auth_credentials_ = AuthCredentials();
 391   } else {
 392     DCHECK(request_->context()->http_transaction_factory());
 393
 394     rv = request_->context()->http_transaction_factory()->CreateTransaction(
 395         &transaction_, http_transaction_delegate_.get());
 396     if (rv == OK) {
 397       if (!throttling_entry_ ||
 398           !throttling_entry_->ShouldRejectRequest(*request_)) {
 399         rv = transaction_->Start(
 400             &request_info_, start_callback_, request_->net_log());
 401         start_time_ = base::TimeTicks::Now();
 402       } else {
 403         // Special error code for the exponential back-off module.
 404         rv = ERR_TEMPORARILY_THROTTLED;
 405       }
 406     }
 407   }
 408
 409   if (rv == ERR_IO_PENDING)
 410     return;
 411
 412   // The transaction started synchronously, but we need to notify the
 413   // URLRequest delegate via the message loop.
 414   MessageLoop::current()->PostTask(
 415       FROM_HERE,
 416       base::Bind(&URLRequestHttpJob::OnStartCompleted,
 417                  weak_factory_.GetWeakPtr(), rv));
 418 }
 419
 420 void URLRequestHttpJob::AddExtraHeaders() {
 421   // Supply Accept-Encoding field only if it is not already provided.
 422   // It should be provided IF the content is known to have restrictions on
 423   // potential encoding, such as streaming multi-media.
 424   // For details see bug 47381.
 425   // TODO(jar, enal): jpeg files etc. should set up a request header if
 426   // possible. Right now it is done only by buffered_resource_loader and
 427   // simple_data_source.
 428   if (!request_info_.extra_headers.HasHeader(
 429       HttpRequestHeaders::kAcceptEncoding)) {
 430     bool advertise_sdch = SdchManager::Global() &&
 431         SdchManager::Global()->IsInSupportedDomain(request_->url());
 432     std::string avail_dictionaries;
 433     if (advertise_sdch) {
 434       SdchManager::Global()->GetAvailDictionaryList(request_->url(),
 435                                                     &avail_dictionaries);
 436
 437       // The AllowLatencyExperiment() is only true if we've successfully done a
 438       // full SDCH compression recently in this browser session for this host.
 439       // Note that for this path, there might be no applicable dictionaries,
 440       // and hence we can't participate in the experiment.
 441       if (!avail_dictionaries.empty() &&
 442           SdchManager::Global()->AllowLatencyExperiment(request_->url())) {
 443         // We are participating in the test (or control), and hence we'll
 444         // eventually record statistics via either SDCH_EXPERIMENT_DECODE or
 445         // SDCH_EXPERIMENT_HOLDBACK, and we'll need some packet timing data.
 446         packet_timing_enabled_ = true;
 447         if (base::RandDouble() < .01) {
 448           sdch_test_control_ = true;  // 1% probability.
 449           advertise_sdch = false;
 450         } else {
 451           sdch_test_activated_ = true;
 452         }
 453       }
 454     }
 455
 456     // Supply Accept-Encoding headers first so that it is more likely that they
 457     // will be in the first transmitted packet.  This can sometimes make it
 458     // easier to filter and analyze the streams to assure that a proxy has not
 459     // damaged these headers.  Some proxies deliberately corrupt Accept-Encoding
 460     // headers.
 461     if (!advertise_sdch) {
 462       // Tell the server what compression formats we support (other than SDCH).
 463       request_info_.extra_headers.SetHeader(
 464           HttpRequestHeaders::kAcceptEncoding, "gzip,deflate");
 465     } else {
 466       // Include SDCH in acceptable list.
 467       request_info_.extra_headers.SetHeader(
 468           HttpRequestHeaders::kAcceptEncoding, "gzip,deflate,sdch");
 469       if (!avail_dictionaries.empty()) {
 470         request_info_.extra_headers.SetHeader(
 471             kAvailDictionaryHeader,
 472             avail_dictionaries);
 473         sdch_dictionary_advertised_ = true;
 474         // Since we're tagging this transaction as advertising a dictionary,
 475         // we'll definitely employ an SDCH filter (or tentative sdch filter)
 476         // when we get a response.  When done, we'll record histograms via
 477         // SDCH_DECODE or SDCH_PASSTHROUGH.  Hence we need to record packet
 478         // arrival times.
 479         packet_timing_enabled_ = true;
 480       }
 481     }
 482   }
 483
 484   if (http_user_agent_settings_) {
 485     // Only add default Accept-Language and Accept-Charset if the request
 486     // didn't have them specified.
 487     std::string accept_language =
 488         http_user_agent_settings_->GetAcceptLanguage();
 489     if (!accept_language.empty()) {
 490       request_info_.extra_headers.SetHeaderIfMissing(
 491           HttpRequestHeaders::kAcceptLanguage,
 492           accept_language);
 493     }
 494     std::string accept_charset = http_user_agent_settings_->GetAcceptCharset();
 495     if (!accept_charset.empty()) {
 496       request_info_.extra_headers.SetHeaderIfMissing(
 497           HttpRequestHeaders::kAcceptCharset,
 498           accept_charset);
 499     }
 500   }
 501 }
 502
 503 void URLRequestHttpJob::AddCookieHeaderAndStart() {
 504   // No matter what, we want to report our status as IO pending since we will
 505   // be notifying our consumer asynchronously via OnStartCompleted.
 506   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
 507
 508   // If the request was destroyed, then there is no more work to do.
 509   if (!request_)
 510     return;
 511
 512   CookieStore* cookie_store = request_->context()->cookie_store();
 513   if (cookie_store && !(request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES)) {
 514     net::CookieMonster* cookie_monster = cookie_store->GetCookieMonster();
 515     if (cookie_monster) {
 516       cookie_monster->GetAllCookiesForURLAsync(
 517           request_->url(),
 518           base::Bind(&URLRequestHttpJob::CheckCookiePolicyAndLoad,
 519                      weak_factory_.GetWeakPtr()));
 520     } else {
 521       CheckCookiePolicyAndLoad(CookieList());
 522     }
 523   } else {
 524     DoStartTransaction();
 525   }
 526 }
 527
 528 void URLRequestHttpJob::DoLoadCookies() {
 529   CookieOptions options;
 530   options.set_include_httponly();
 531   request_->context()->cookie_store()->GetCookiesWithInfoAsync(
 532       request_->url(), options,
 533       base::Bind(&URLRequestHttpJob::OnCookiesLoaded,
 534                  weak_factory_.GetWeakPtr()));
 535 }
 536
 537 void URLRequestHttpJob::CheckCookiePolicyAndLoad(
 538     const CookieList& cookie_list) {
 539   if (CanGetCookies(cookie_list))
 540     DoLoadCookies();
 541   else
 542     DoStartTransaction();
 543 }
 544
 545 void URLRequestHttpJob::OnCookiesLoaded(
 546     const std::string& cookie_line,
 547     const std::vector<net::CookieStore::CookieInfo>& cookie_infos) {
 548   if (!cookie_line.empty()) {
 549     request_info_.extra_headers.SetHeader(
 550         HttpRequestHeaders::kCookie, cookie_line);
 551   }
 552   DoStartTransaction();
 553 }
 554
 555 void URLRequestHttpJob::DoStartTransaction() {
 556   // We may have been canceled while retrieving cookies.
 557   if (GetStatus().is_success()) {
 558     StartTransaction();
 559   } else {
 560     NotifyCanceled();
 561   }
 562 }
 563
 564 void URLRequestHttpJob::SaveCookiesAndNotifyHeadersComplete(int result) {
 565   if (result != net::OK) {
 566     std::string source("delegate");
 567     request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 568                                  NetLog::StringCallback("source", &source));
 569     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 570     return;
 571   }
 572
 573   DCHECK(transaction_.get());
 574
 575   const HttpResponseInfo* response_info = transaction_->GetResponseInfo();
 576   DCHECK(response_info);
 577
 578   response_cookies_.clear();
 579   response_cookies_save_index_ = 0;
 580
 581   FetchResponseCookies(&response_cookies_);
 582
 583   if (!GetResponseHeaders()->GetDateValue(&response_date_))
 584     response_date_ = base::Time();
 585
 586   // Now, loop over the response cookies, and attempt to persist each.
 587   SaveNextCookie();
 588 }
 589
 590 // If the save occurs synchronously, SaveNextCookie will loop and save the next
 591 // cookie. If the save is deferred, the callback is responsible for continuing
 592 // to iterate through the cookies.
 593 // TODO(erikwright): Modify the CookieStore API to indicate via return value
 594 // whether it completed synchronously or asynchronously.
 595 // See http://crbug.com/131066.
 596 void URLRequestHttpJob::SaveNextCookie() {
 597   // No matter what, we want to report our status as IO pending since we will
 598   // be notifying our consumer asynchronously via OnStartCompleted.
 599   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
 600
 601   // Used to communicate with the callback. See the implementation of
 602   // OnCookieSaved.
 603   scoped_refptr<SharedBoolean> callback_pending = new SharedBoolean(false);
 604   scoped_refptr<SharedBoolean> save_next_cookie_running =
 605       new SharedBoolean(true);
 606
 607   if (!(request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) &&
 608       request_->context()->cookie_store() &&
 609       response_cookies_.size() > 0) {
 610     CookieOptions options;
 611     options.set_include_httponly();
 612     options.set_server_time(response_date_);
 613
 614     net::CookieStore::SetCookiesCallback callback(
 615         base::Bind(&URLRequestHttpJob::OnCookieSaved,
 616                    weak_factory_.GetWeakPtr(),
 617                    save_next_cookie_running,
 618                    callback_pending));
 619
 620     // Loop through the cookies as long as SetCookieWithOptionsAsync completes
 621     // synchronously.
 622     while (!callback_pending->data &&
 623            response_cookies_save_index_ < response_cookies_.size()) {
 624       if (CanSetCookie(
 625           response_cookies_[response_cookies_save_index_], &options)) {
 626         callback_pending->data = true;
 627         request_->context()->cookie_store()->SetCookieWithOptionsAsync(
 628             request_->url(), response_cookies_[response_cookies_save_index_],
 629             options, callback);
 630       }
 631       ++response_cookies_save_index_;
 632     }
 633   }
 634
 635   save_next_cookie_running->data = false;
 636
 637   if (!callback_pending->data) {
 638     response_cookies_.clear();
 639     response_cookies_save_index_ = 0;
 640     SetStatus(URLRequestStatus());  // Clear the IO_PENDING status
 641     NotifyHeadersComplete();
 642     return;
 643   }
 644 }
 645
 646 // |save_next_cookie_running| is true when the callback is bound and set to
 647 // false when SaveNextCookie exits, allowing the callback to determine if the
 648 // save occurred synchronously or asynchronously.
 649 // |callback_pending| is false when the callback is invoked and will be set to
 650 // true by the callback, allowing SaveNextCookie to detect whether the save
 651 // occurred synchronously.
 652 // See SaveNextCookie() for more information.
 653 void URLRequestHttpJob::OnCookieSaved(
 654     scoped_refptr<SharedBoolean> save_next_cookie_running,
 655     scoped_refptr<SharedBoolean> callback_pending,
 656     bool cookie_status) {
 657   callback_pending->data = false;
 658
 659   // If we were called synchronously, return.
 660   if (save_next_cookie_running->data) {
 661     return;
 662   }
 663
 664   // We were called asynchronously, so trigger the next save.
 665   // We may have been canceled within OnSetCookie.
 666   if (GetStatus().is_success()) {
 667     SaveNextCookie();
 668   } else {
 669     NotifyCanceled();
 670   }
 671 }
 672
 673 void URLRequestHttpJob::FetchResponseCookies(
 674     std::vector<std::string>* cookies) {
 675   const std::string name = "Set-Cookie";
 676   std::string value;
 677
 678   void* iter = NULL;
 679   HttpResponseHeaders* headers = GetResponseHeaders();
 680   while (headers->EnumerateHeader(&iter, name, &value)) {
 681     if (!value.empty())
 682       cookies->push_back(value);
 683   }
 684 }
 685
 686 // NOTE: |ProcessStrictTransportSecurityHeader| and
 687 // |ProcessPublicKeyPinsHeader| have very similar structures, by design.
 688 void URLRequestHttpJob::ProcessStrictTransportSecurityHeader() {
 689   DCHECK(response_info_);
 690   TransportSecurityState* security_state =
 691       request_->context()->transport_security_state();
 692   const SSLInfo& ssl_info = response_info_->ssl_info;
 693
 694   // Only accept HSTS headers on HTTPS connections that have no
 695   // certificate errors.
 696   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 697       !security_state)
 698     return;
 699
 700   // http://tools.ietf.org/html/draft-ietf-websec-strict-transport-sec:
 701   //
 702   //   If a UA receives more than one STS header field in a HTTP response
 703   //   message over secure transport, then the UA MUST process only the
 704   //   first such header field.
 705   HttpResponseHeaders* headers = GetResponseHeaders();
 706   std::string value;
 707   if (headers->EnumerateHeader(NULL, "Strict-Transport-Security", &value))
 708     security_state->AddHSTSHeader(request_info_.url.host(), value);
 709 }
 710
 711 void URLRequestHttpJob::ProcessPublicKeyPinsHeader() {
 712   DCHECK(response_info_);
 713   TransportSecurityState* security_state =
 714       request_->context()->transport_security_state();
 715   const SSLInfo& ssl_info = response_info_->ssl_info;
 716
 717   // Only accept HPKP headers on HTTPS connections that have no
 718   // certificate errors.
 719   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 720       !security_state)
 721     return;
 722
 723   // http://tools.ietf.org/html/draft-ietf-websec-key-pinning:
 724   //
 725   //   If a UA receives more than one PKP header field in an HTTP
 726   //   response message over secure transport, then the UA MUST process
 727   //   only the first such header field.
 728   HttpResponseHeaders* headers = GetResponseHeaders();
 729   std::string value;
 730   if (headers->EnumerateHeader(NULL, "Public-Key-Pins", &value))
 731     security_state->AddHPKPHeader(request_info_.url.host(), value, ssl_info);
 732 }
 733
 734 void URLRequestHttpJob::OnStartCompleted(int result) {
 735   RecordTimer();
 736
 737   // If the request was destroyed, then there is no more work to do.
 738   if (!request_)
 739     return;
 740
 741   // If the transaction was destroyed, then the job was cancelled, and
 742   // we can just ignore this notification.
 743   if (!transaction_.get())
 744     return;
 745
 746   // Clear the IO_PENDING status
 747   SetStatus(URLRequestStatus());
 748
 749   const URLRequestContext* context = request_->context();
 750
 751   if (result == ERR_SSL_PINNED_KEY_NOT_IN_CERT_CHAIN &&
 752       transaction_->GetResponseInfo() != NULL) {
 753     FraudulentCertificateReporter* reporter =
 754       context->fraudulent_certificate_reporter();
 755     if (reporter != NULL) {
 756       const SSLInfo& ssl_info = transaction_->GetResponseInfo()->ssl_info;
 757       bool sni_available = SSLConfigService::IsSNIAvailable(
 758           context->ssl_config_service());
 759       const std::string& host = request_->url().host();
 760
 761       reporter->SendReport(host, ssl_info, sni_available);
 762     }
 763   }
 764
 765   if (result == OK) {
 766     scoped_refptr<HttpResponseHeaders> headers = GetResponseHeaders();
 767     if (context->network_delegate()) {
 768       // Note that |this| may not be deleted until
 769       // |on_headers_received_callback_| or
 770       // |NetworkDelegate::URLRequestDestroyed()| has been called.
 771       int error = context->network_delegate()->
 772           NotifyHeadersReceived(request_, on_headers_received_callback_,
 773                                 headers, &override_response_headers_);
 774       if (error != net::OK) {
 775         if (error == net::ERR_IO_PENDING) {
 776           awaiting_callback_ = true;
 777           SetBlockedOnDelegate();
 778         } else {
 779           std::string source("delegate");
 780           request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 781                                        NetLog::StringCallback("source",
 782                                                               &source));
 783           NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, error));
 784         }
 785         return;
 786       }
 787     }
 788
 789     SaveCookiesAndNotifyHeadersComplete(net::OK);
 790   } else if (IsCertificateError(result)) {
 791     // We encountered an SSL certificate error.  Ask our delegate to decide
 792     // what we should do.
 793
 794     TransportSecurityState::DomainState domain_state;
 795     const URLRequestContext* context = request_->context();
 796     const bool fatal =
 797         context->transport_security_state() &&
 798         context->transport_security_state()->GetDomainState(
 799             request_info_.url.host(),
 800             SSLConfigService::IsSNIAvailable(context->ssl_config_service()),
 801             &domain_state);
 802     NotifySSLCertificateError(transaction_->GetResponseInfo()->ssl_info, fatal);
 803   } else if (result == ERR_SSL_CLIENT_AUTH_CERT_NEEDED) {
 804     NotifyCertificateRequested(
 805         transaction_->GetResponseInfo()->cert_request_info);
 806   } else {
 807     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 808   }
 809 }
 810
 811 void URLRequestHttpJob::OnHeadersReceivedCallback(int result) {
 812   SetUnblockedOnDelegate();
 813   awaiting_callback_ = false;
 814
 815   // Check that there are no callbacks to already canceled requests.
 816   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 817
 818   SaveCookiesAndNotifyHeadersComplete(result);
 819 }
 820
 821 void URLRequestHttpJob::OnReadCompleted(int result) {
 822   read_in_progress_ = false;
 823
 824   if (ShouldFixMismatchedContentLength(result))
 825     result = OK;
 826
 827   if (result == OK) {
 828     NotifyDone(URLRequestStatus());
 829   } else if (result < 0) {
 830     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, result));
 831   } else {
 832     // Clear the IO_PENDING status
 833     SetStatus(URLRequestStatus());
 834   }
 835
 836   NotifyReadComplete(result);
 837 }
 838
 839 void URLRequestHttpJob::RestartTransactionWithAuth(
 840     const AuthCredentials& credentials) {
 841   auth_credentials_ = credentials;
 842
 843   // These will be reset in OnStartCompleted.
 844   response_info_ = NULL;
 845   response_cookies_.clear();
 846
 847   ResetTimer();
 848
 849   // Update the cookies, since the cookie store may have been updated from the
 850   // headers in the 401/407. Since cookies were already appended to
 851   // extra_headers, we need to strip them out before adding them again.
 852   request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kCookie);
 853
 854   AddCookieHeaderAndStart();
 855 }
 856
 857 void URLRequestHttpJob::SetUpload(UploadDataStream* upload) {
 858   DCHECK(!transaction_.get()) << "cannot change once started";
 859   request_info_.upload_data_stream = upload;
 860 }
 861
 862 void URLRequestHttpJob::SetExtraRequestHeaders(
 863     const HttpRequestHeaders& headers) {
 864   DCHECK(!transaction_.get()) << "cannot change once started";
 865   request_info_.extra_headers.CopyFrom(headers);
 866 }
 867
 868 void URLRequestHttpJob::Start() {
 869   DCHECK(!transaction_.get());
 870
 871   // Ensure that we do not send username and password fields in the referrer.
 872   GURL referrer(request_->GetSanitizedReferrer());
 873
 874   request_info_.url = request_->url();
 875   request_info_.method = request_->method();
 876   request_info_.load_flags = request_->load_flags();
 877   request_info_.priority = request_->priority();
 878   request_info_.request_id = request_->identifier();
 879
 880   // Strip Referer from request_info_.extra_headers to prevent, e.g., plugins
 881   // from overriding headers that are controlled using other means. Otherwise a
 882   // plugin could set a referrer although sending the referrer is inhibited.
 883   request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kReferer);
 884
 885   // Our consumer should have made sure that this is a safe referrer.  See for
 886   // instance WebCore::FrameLoader::HideReferrer.
 887   if (referrer.is_valid()) {
 888     request_info_.extra_headers.SetHeader(HttpRequestHeaders::kReferer,
 889                                           referrer.spec());
 890   }
 891
 892   request_info_.extra_headers.SetHeaderIfMissing(
 893       HttpRequestHeaders::kUserAgent,
 894       http_user_agent_settings_ ?
 895           http_user_agent_settings_->GetUserAgent(request_->url()) :
 896           EmptyString());
 897
 898   AddExtraHeaders();
 899   AddCookieHeaderAndStart();
 900 }
 901
 902 void URLRequestHttpJob::Kill() {
 903   http_transaction_delegate_->OnDetachRequest();
 904
 905   if (!transaction_.get())
 906     return;
 907
 908   weak_factory_.InvalidateWeakPtrs();
 909   DestroyTransaction();
 910   URLRequestJob::Kill();
 911 }
 912
 913 LoadState URLRequestHttpJob::GetLoadState() const {
 914   return transaction_.get() ?
 915       transaction_->GetLoadState() : LOAD_STATE_IDLE;
 916 }
 917
 918 UploadProgress URLRequestHttpJob::GetUploadProgress() const {
 919   return transaction_.get() ?
 920       transaction_->GetUploadProgress() : UploadProgress();
 921 }
 922
 923 bool URLRequestHttpJob::GetMimeType(std::string* mime_type) const {
 924   DCHECK(transaction_.get());
 925
 926   if (!response_info_)
 927     return false;
 928
 929   return GetResponseHeaders()->GetMimeType(mime_type);
 930 }
 931
 932 bool URLRequestHttpJob::GetCharset(std::string* charset) {
 933   DCHECK(transaction_.get());
 934
 935   if (!response_info_)
 936     return false;
 937
 938   return GetResponseHeaders()->GetCharset(charset);
 939 }
 940
 941 void URLRequestHttpJob::GetResponseInfo(HttpResponseInfo* info) {
 942   DCHECK(request_);
 943   DCHECK(transaction_.get());
 944
 945   if (response_info_) {
 946     *info = *response_info_;
 947     if (override_response_headers_)
 948       info->headers = override_response_headers_;
 949   }
 950 }
 951
 952 bool URLRequestHttpJob::GetResponseCookies(std::vector<std::string>* cookies) {
 953   DCHECK(transaction_.get());
 954
 955   if (!response_info_)
 956     return false;
 957
 958   // TODO(darin): Why are we extracting response cookies again?  Perhaps we
 959   // should just leverage response_cookies_.
 960
 961   cookies->clear();
 962   FetchResponseCookies(cookies);
 963   return true;
 964 }
 965
 966 int URLRequestHttpJob::GetResponseCode() const {
 967   DCHECK(transaction_.get());
 968
 969   if (!response_info_)
 970     return -1;
 971
 972   return GetResponseHeaders()->response_code();
 973 }
 974
 975 Filter* URLRequestHttpJob::SetupFilter() const {
 976   DCHECK(transaction_.get());
 977   if (!response_info_)
 978     return NULL;
 979
 980   std::vector<Filter::FilterType> encoding_types;
 981   std::string encoding_type;
 982   HttpResponseHeaders* headers = GetResponseHeaders();
 983   void* iter = NULL;
 984   while (headers->EnumerateHeader(&iter, "Content-Encoding", &encoding_type)) {
 985     encoding_types.push_back(Filter::ConvertEncodingToType(encoding_type));
 986   }
 987
 988   if (filter_context_->IsSdchResponse()) {
 989     // We are wary of proxies that discard or damage SDCH encoding.  If a server
 990     // explicitly states that this is not SDCH content, then we can correct our
 991     // assumption that this is an SDCH response, and avoid the need to recover
 992     // as though the content is corrupted (when we discover it is not SDCH
 993     // encoded).
 994     std::string sdch_response_status;
 995     iter = NULL;
 996     while (headers->EnumerateHeader(&iter, "X-Sdch-Encode",
 997                                     &sdch_response_status)) {
 998       if (sdch_response_status == "0") {
 999         filter_context_->ResetSdchResponseToFalse();
1000         break;
1001       }
1002     }
1003   }
1004
1005   // Even if encoding types are empty, there is a chance that we need to add
1006   // some decoding, as some proxies strip encoding completely. In such cases,
1007   // we may need to add (for example) SDCH filtering (when the context suggests
1008   // it is appropriate).
1009   Filter::FixupEncodingTypes(*filter_context_, &encoding_types);
1010
1011   return !encoding_types.empty()
1012       ? Filter::Factory(encoding_types, *filter_context_) : NULL;
1013 }
1014
1015 bool URLRequestHttpJob::IsSafeRedirect(const GURL& location) {
1016   // We only allow redirects to certain "safe" protocols.  This does not
1017   // restrict redirects to externally handled protocols.  Our consumer would
1018   // need to take care of those.
1019
1020   if (!URLRequest::IsHandledURL(location))
1021     return true;
1022
1023   static const char* kSafeSchemes[] = {
1024     "http",
1025     "https",
1026     "ftp"
1027   };
1028
1029   for (size_t i = 0; i < arraysize(kSafeSchemes); ++i) {
1030     if (location.SchemeIs(kSafeSchemes[i]))
1031       return true;
1032   }
1033
1034   return false;
1035 }
1036
1037 bool URLRequestHttpJob::NeedsAuth() {
1038   int code = GetResponseCode();
1039   if (code == -1)
1040     return false;
1041
1042   // Check if we need either Proxy or WWW Authentication.  This could happen
1043   // because we either provided no auth info, or provided incorrect info.
1044   switch (code) {
1045     case 407:
1046       if (proxy_auth_state_ == AUTH_STATE_CANCELED)
1047         return false;
1048       proxy_auth_state_ = AUTH_STATE_NEED_AUTH;
1049       return true;
1050     case 401:
1051       if (server_auth_state_ == AUTH_STATE_CANCELED)
1052         return false;
1053       server_auth_state_ = AUTH_STATE_NEED_AUTH;
1054       return true;
1055   }
1056   return false;
1057 }
1058
1059 void URLRequestHttpJob::GetAuthChallengeInfo(
1060     scoped_refptr<AuthChallengeInfo>* result) {
1061   DCHECK(transaction_.get());
1062   DCHECK(response_info_);
1063
1064   // sanity checks:
1065   DCHECK(proxy_auth_state_ == AUTH_STATE_NEED_AUTH ||
1066          server_auth_state_ == AUTH_STATE_NEED_AUTH);
1067   DCHECK((GetResponseHeaders()->response_code() == HTTP_UNAUTHORIZED) ||
1068          (GetResponseHeaders()->response_code() ==
1069           HTTP_PROXY_AUTHENTICATION_REQUIRED));
1070
1071   *result = response_info_->auth_challenge;
1072 }
1073
1074 void URLRequestHttpJob::SetAuth(const AuthCredentials& credentials) {
1075   DCHECK(transaction_.get());
1076
1077   // Proxy gets set first, then WWW.
1078   if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1079     proxy_auth_state_ = AUTH_STATE_HAVE_AUTH;
1080   } else {
1081     DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1082     server_auth_state_ = AUTH_STATE_HAVE_AUTH;
1083   }
1084
1085   RestartTransactionWithAuth(credentials);
1086 }
1087
1088 void URLRequestHttpJob::CancelAuth() {
1089   // Proxy gets set first, then WWW.
1090   if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1091     proxy_auth_state_ = AUTH_STATE_CANCELED;
1092   } else {
1093     DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1094     server_auth_state_ = AUTH_STATE_CANCELED;
1095   }
1096
1097   // These will be reset in OnStartCompleted.
1098   response_info_ = NULL;
1099   response_cookies_.clear();
1100
1101   ResetTimer();
1102
1103   // OK, let the consumer read the error page...
1104   //
1105   // Because we set the AUTH_STATE_CANCELED flag, NeedsAuth will return false,
1106   // which will cause the consumer to receive OnResponseStarted instead of
1107   // OnAuthRequired.
1108   //
1109   // We have to do this via InvokeLater to avoid "recursing" the consumer.
1110   //
1111   MessageLoop::current()->PostTask(
1112       FROM_HERE,
1113       base::Bind(&URLRequestHttpJob::OnStartCompleted,
1114                  weak_factory_.GetWeakPtr(), OK));
1115 }
1116
1117 void URLRequestHttpJob::ContinueWithCertificate(
1118     X509Certificate* client_cert) {
1119   DCHECK(transaction_.get());
1120
1121   DCHECK(!response_info_) << "should not have a response yet";
1122
1123   ResetTimer();
1124
1125   // No matter what, we want to report our status as IO pending since we will
1126   // be notifying our consumer asynchronously via OnStartCompleted.
1127   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1128
1129   int rv = transaction_->RestartWithCertificate(client_cert, start_callback_);
1130   if (rv == ERR_IO_PENDING)
1131     return;
1132
1133   // The transaction started synchronously, but we need to notify the
1134   // URLRequest delegate via the message loop.
1135   MessageLoop::current()->PostTask(
1136       FROM_HERE,
1137       base::Bind(&URLRequestHttpJob::OnStartCompleted,
1138                  weak_factory_.GetWeakPtr(), rv));
1139 }
1140
1141 void URLRequestHttpJob::ContinueDespiteLastError() {
1142   // If the transaction was destroyed, then the job was cancelled.
1143   if (!transaction_.get())
1144     return;
1145
1146   DCHECK(!response_info_) << "should not have a response yet";
1147
1148   ResetTimer();
1149
1150   // No matter what, we want to report our status as IO pending since we will
1151   // be notifying our consumer asynchronously via OnStartCompleted.
1152   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1153
1154   int rv = transaction_->RestartIgnoringLastError(start_callback_);
1155   if (rv == ERR_IO_PENDING)
1156     return;
1157
1158   // The transaction started synchronously, but we need to notify the
1159   // URLRequest delegate via the message loop.
1160   MessageLoop::current()->PostTask(
1161       FROM_HERE,
1162       base::Bind(&URLRequestHttpJob::OnStartCompleted,
1163                  weak_factory_.GetWeakPtr(), rv));
1164 }
1165
1166 bool URLRequestHttpJob::ShouldFixMismatchedContentLength(int rv) const {
1167   // Some servers send the body compressed, but specify the content length as
1168   // the uncompressed size.  Although this violates the HTTP spec we want to
1169   // support it (as IE and FireFox do), but *only* for an exact match.
1170   // See http://crbug.com/79694.
1171   if (rv == net::ERR_CONTENT_LENGTH_MISMATCH ||
1172       rv == net::ERR_INCOMPLETE_CHUNKED_ENCODING) {
1173     if (request_ && request_->response_headers()) {
1174       int64 expected_length = request_->response_headers()->GetContentLength();
1175       VLOG(1) << __FUNCTION__ << "() "
1176               << "\"" << request_->url().spec() << "\""
1177               << " content-length = " << expected_length
1178               << " pre total = " << prefilter_bytes_read()
1179               << " post total = " << postfilter_bytes_read();
1180       if (postfilter_bytes_read() == expected_length) {
1181         // Clear the error.
1182         return true;
1183       }
1184     }
1185   }
1186   return false;
1187 }
1188
1189 bool URLRequestHttpJob::ReadRawData(IOBuffer* buf, int buf_size,
1190                                     int* bytes_read) {
1191   DCHECK_NE(buf_size, 0);
1192   DCHECK(bytes_read);
1193   DCHECK(!read_in_progress_);
1194
1195   int rv = transaction_->Read(
1196       buf, buf_size,
1197       base::Bind(&URLRequestHttpJob::OnReadCompleted, base::Unretained(this)));
1198
1199   if (ShouldFixMismatchedContentLength(rv))
1200     rv = 0;
1201
1202   if (rv >= 0) {
1203     *bytes_read = rv;
1204     if (!rv)
1205       DoneWithRequest(FINISHED);
1206     return true;
1207   }
1208
1209   if (rv == ERR_IO_PENDING) {
1210     read_in_progress_ = true;
1211     SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1212   } else {
1213     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, rv));
1214   }
1215
1216   return false;
1217 }
1218
1219 void URLRequestHttpJob::StopCaching() {
1220   if (transaction_.get())
1221     transaction_->StopCaching();
1222 }
1223
1224 void URLRequestHttpJob::DoneReading() {
1225   if (transaction_.get())
1226     transaction_->DoneReading();
1227   DoneWithRequest(FINISHED);
1228 }
1229
1230 HostPortPair URLRequestHttpJob::GetSocketAddress() const {
1231   return response_info_ ? response_info_->socket_address : HostPortPair();
1232 }
1233
1234 URLRequestHttpJob::~URLRequestHttpJob() {
1235   CHECK(!awaiting_callback_);
1236
1237   DCHECK(!sdch_test_control_ || !sdch_test_activated_);
1238   if (!is_cached_content_) {
1239     if (sdch_test_control_)
1240       RecordPacketStats(FilterContext::SDCH_EXPERIMENT_HOLDBACK);
1241     if (sdch_test_activated_)
1242       RecordPacketStats(FilterContext::SDCH_EXPERIMENT_DECODE);
1243   }
1244   // Make sure SDCH filters are told to emit histogram data while
1245   // filter_context_ is still alive.
1246   DestroyFilters();
1247
1248   if (sdch_dictionary_url_.is_valid()) {
1249     // Prior to reaching the destructor, request_ has been set to a NULL
1250     // pointer, so request_->url() is no longer valid in the destructor, and we
1251     // use an alternate copy |request_info_.url|.
1252     SdchManager* manager = SdchManager::Global();
1253     // To be extra safe, since this is a "different time" from when we decided
1254     // to get the dictionary, we'll validate that an SdchManager is available.
1255     // At shutdown time, care is taken to be sure that we don't delete this
1256     // globally useful instance "too soon," so this check is just defensive
1257     // coding to assure that IF the system is shutting down, we don't have any
1258     // problem if the manager was deleted ahead of time.
1259     if (manager)  // Defensive programming.
1260       manager->FetchDictionary(request_info_.url, sdch_dictionary_url_);
1261   }
1262   DoneWithRequest(ABORTED);
1263 }
1264
1265 void URLRequestHttpJob::RecordTimer() {
1266   if (request_creation_time_.is_null()) {
1267     NOTREACHED()
1268         << "The same transaction shouldn't start twice without new timing.";
1269     return;
1270   }
1271
1272   base::TimeDelta to_start = base::Time::Now() - request_creation_time_;
1273   request_creation_time_ = base::Time();
1274
1275   UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpTimeToFirstByte", to_start);
1276
1277   static const bool use_overlapped_read_histogram =
1278       base::FieldTrialList::TrialExists("OverlappedReadImpact");
1279   if (use_overlapped_read_histogram) {
1280     UMA_HISTOGRAM_MEDIUM_TIMES(
1281         base::FieldTrial::MakeName("Net.HttpTimeToFirstByte",
1282                                    "OverlappedReadImpact"),
1283         to_start);
1284   }
1285
1286   static const bool use_warm_socket_impact_histogram =
1287       base::FieldTrialList::TrialExists("WarmSocketImpact");
1288   if (use_warm_socket_impact_histogram) {
1289     UMA_HISTOGRAM_MEDIUM_TIMES(
1290         base::FieldTrial::MakeName("Net.HttpTimeToFirstByte",
1291                                    "WarmSocketImpact"),
1292         to_start);
1293   }
1294
1295   static const bool use_prefetch_histogram =
1296       base::FieldTrialList::TrialExists("Prefetch");
1297   if (use_prefetch_histogram) {
1298     UMA_HISTOGRAM_MEDIUM_TIMES(
1299         base::FieldTrial::MakeName("Net.HttpTimeToFirstByte",
1300                                    "Prefetch"),
1301         to_start);
1302   }
1303   static const bool use_prerender_histogram =
1304       base::FieldTrialList::TrialExists("Prerender");
1305   if (use_prerender_histogram) {
1306     UMA_HISTOGRAM_MEDIUM_TIMES(
1307         base::FieldTrial::MakeName("Net.HttpTimeToFirstByte",
1308                                    "Prerender"),
1309         to_start);
1310   }
1311 }
1312
1313 void URLRequestHttpJob::ResetTimer() {
1314   if (!request_creation_time_.is_null()) {
1315     NOTREACHED()
1316         << "The timer was reset before it was recorded.";
1317     return;
1318   }
1319   request_creation_time_ = base::Time::Now();
1320 }
1321
1322 void URLRequestHttpJob::UpdatePacketReadTimes() {
1323   if (!packet_timing_enabled_)
1324     return;
1325
1326   if (filter_input_byte_count() <= bytes_observed_in_packets_) {
1327     DCHECK_EQ(filter_input_byte_count(), bytes_observed_in_packets_);
1328     return;  // No new bytes have arrived.
1329   }
1330
1331   final_packet_time_ = base::Time::Now();
1332   if (!bytes_observed_in_packets_)
1333     request_time_snapshot_ = request_ ? request_->request_time() : base::Time();
1334
1335   bytes_observed_in_packets_ = filter_input_byte_count();
1336 }
1337
1338 void URLRequestHttpJob::RecordPacketStats(
1339     FilterContext::StatisticSelector statistic) const {
1340   if (!packet_timing_enabled_ || (final_packet_time_ == base::Time()))
1341     return;
1342
1343   base::TimeDelta duration = final_packet_time_ - request_time_snapshot_;
1344   switch (statistic) {
1345     case FilterContext::SDCH_DECODE: {
1346       UMA_HISTOGRAM_CUSTOM_COUNTS("Sdch3.Network_Decode_Bytes_Processed_b",
1347           static_cast<int>(bytes_observed_in_packets_), 500, 100000, 100);
1348       return;
1349     }
1350     case FilterContext::SDCH_PASSTHROUGH: {
1351       // Despite advertising a dictionary, we handled non-sdch compressed
1352       // content.
1353       return;
1354     }
1355
1356     case FilterContext::SDCH_EXPERIMENT_DECODE: {
1357       UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment2_Decode",
1358                                   duration,
1359                                   base::TimeDelta::FromMilliseconds(20),
1360                                   base::TimeDelta::FromMinutes(10), 100);
1361       return;
1362     }
1363     case FilterContext::SDCH_EXPERIMENT_HOLDBACK: {
1364       UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment2_Holdback",
1365                                   duration,
1366                                   base::TimeDelta::FromMilliseconds(20),
1367                                   base::TimeDelta::FromMinutes(10), 100);
1368       return;
1369     }
1370     default:
1371       NOTREACHED();
1372       return;
1373   }
1374 }
1375
1376 // The common type of histogram we use for all compression-tracking histograms.
1377 #define COMPRESSION_HISTOGRAM(name, sample) \
1378     do { \
1379       UMA_HISTOGRAM_CUSTOM_COUNTS("Net.Compress." name, sample, \
1380                                   500, 1000000, 100); \
1381     } while (0)
1382
1383 void URLRequestHttpJob::RecordCompressionHistograms() {
1384   DCHECK(request_);
1385   if (!request_)
1386     return;
1387
1388   if (is_cached_content_ ||                // Don't record cached content
1389       !GetStatus().is_success() ||         // Don't record failed content
1390       !IsCompressibleContent() ||          // Only record compressible content
1391       !prefilter_bytes_read())       // Zero-byte responses aren't useful.
1392     return;
1393
1394   // Miniature requests aren't really compressible.  Don't count them.
1395   const int kMinSize = 16;
1396   if (prefilter_bytes_read() < kMinSize)
1397     return;
1398
1399   // Only record for http or https urls.
1400   bool is_http = request_->url().SchemeIs("http");
1401   bool is_https = request_->url().SchemeIs("https");
1402   if (!is_http && !is_https)
1403     return;
1404
1405   int compressed_B = prefilter_bytes_read();
1406   int decompressed_B = postfilter_bytes_read();
1407   bool was_filtered = HasFilter();
1408
1409   // We want to record how often downloaded resources are compressed.
1410   // But, we recognize that different protocols may have different
1411   // properties.  So, for each request, we'll put it into one of 3
1412   // groups:
1413   //      a) SSL resources
1414   //         Proxies cannot tamper with compression headers with SSL.
1415   //      b) Non-SSL, loaded-via-proxy resources
1416   //         In this case, we know a proxy might have interfered.
1417   //      c) Non-SSL, loaded-without-proxy resources
1418   //         In this case, we know there was no explicit proxy.  However,
1419   //         it is possible that a transparent proxy was still interfering.
1420   //
1421   // For each group, we record the same 3 histograms.
1422
1423   if (is_https) {
1424     if (was_filtered) {
1425       COMPRESSION_HISTOGRAM("SSL.BytesBeforeCompression", compressed_B);
1426       COMPRESSION_HISTOGRAM("SSL.BytesAfterCompression", decompressed_B);
1427     } else {
1428       COMPRESSION_HISTOGRAM("SSL.ShouldHaveBeenCompressed", decompressed_B);
1429     }
1430     return;
1431   }
1432
1433   if (request_->was_fetched_via_proxy()) {
1434     if (was_filtered) {
1435       COMPRESSION_HISTOGRAM("Proxy.BytesBeforeCompression", compressed_B);
1436       COMPRESSION_HISTOGRAM("Proxy.BytesAfterCompression", decompressed_B);
1437     } else {
1438       COMPRESSION_HISTOGRAM("Proxy.ShouldHaveBeenCompressed", decompressed_B);
1439     }
1440     return;
1441   }
1442
1443   if (was_filtered) {
1444     COMPRESSION_HISTOGRAM("NoProxy.BytesBeforeCompression", compressed_B);
1445     COMPRESSION_HISTOGRAM("NoProxy.BytesAfterCompression", decompressed_B);
1446   } else {
1447     COMPRESSION_HISTOGRAM("NoProxy.ShouldHaveBeenCompressed", decompressed_B);
1448   }
1449 }
1450
1451 bool URLRequestHttpJob::IsCompressibleContent() const {
1452   std::string mime_type;
1453   return GetMimeType(&mime_type) &&
1454       (IsSupportedJavascriptMimeType(mime_type.c_str()) ||
1455        IsSupportedNonImageMimeType(mime_type.c_str()));
1456 }
1457
1458 void URLRequestHttpJob::RecordPerfHistograms(CompletionCause reason) {
1459   if (start_time_.is_null())
1460     return;
1461
1462   base::TimeDelta total_time = base::TimeTicks::Now() - start_time_;
1463   UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTime", total_time);
1464
1465   if (reason == FINISHED) {
1466     UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeSuccess", total_time);
1467   } else {
1468     UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCancel", total_time);
1469   }
1470
1471   if (response_info_) {
1472     if (response_info_->was_cached) {
1473       UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCached", total_time);
1474     } else  {
1475       UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeNotCached", total_time);
1476     }
1477   }
1478
1479   static const bool use_overlapped_read_histogram =
1480       base::FieldTrialList::TrialExists("OverlappedReadImpact");
1481   if (use_overlapped_read_histogram) {
1482     UMA_HISTOGRAM_TIMES(
1483         base::FieldTrial::MakeName("Net.HttpJob.TotalTime",
1484                                    "OverlappedReadImpact"),
1485         total_time);
1486
1487     if (reason == FINISHED) {
1488       UMA_HISTOGRAM_TIMES(
1489           base::FieldTrial::MakeName("Net.HttpJob.TotalTimeSuccess",
1490                                      "OverlappedReadImpact"),
1491           total_time);
1492     } else {
1493       UMA_HISTOGRAM_TIMES(
1494           base::FieldTrial::MakeName("Net.HttpJob.TotalTimeCancel",
1495                                      "OverlappedReadImpact"),
1496           total_time);
1497     }
1498
1499     if (response_info_) {
1500       if (response_info_->was_cached) {
1501         UMA_HISTOGRAM_TIMES(
1502             base::FieldTrial::MakeName("Net.HttpJob.TotalTimeCached",
1503                                        "OverlappedReadImpact"),
1504             total_time);
1505       } else  {
1506         UMA_HISTOGRAM_TIMES(
1507             base::FieldTrial::MakeName("Net.HttpJob.TotalTimeNotCached",
1508                                        "OverlappedReadImpact"),
1509             total_time);
1510       }
1511     }
1512   }
1513
1514   static const bool cache_sensitivity_analysis =
1515       base::FieldTrialList::TrialExists("CacheSensitivityAnalysis");
1516   if (cache_sensitivity_analysis) {
1517     UMA_HISTOGRAM_TIMES(
1518         base::FieldTrial::MakeName("Net.HttpJob.TotalTime",
1519                                    "CacheSensitivityAnalysis"),
1520         total_time);
1521
1522     if (reason == FINISHED) {
1523       UMA_HISTOGRAM_TIMES(
1524           base::FieldTrial::MakeName("Net.HttpJob.TotalTimeSuccess",
1525                                      "CacheSensitivityAnalysis"),
1526           total_time);
1527     } else {
1528       UMA_HISTOGRAM_TIMES(
1529           base::FieldTrial::MakeName("Net.HttpJob.TotalTimeCancel",
1530                                      "CacheSensitivityAnalysis"),
1531           total_time);
1532     }
1533
1534     if (response_info_) {
1535       if (response_info_->was_cached) {
1536         UMA_HISTOGRAM_TIMES(
1537             base::FieldTrial::MakeName("Net.HttpJob.TotalTimeCached",
1538                                        "CacheSensitivityAnalysis"),
1539             total_time);
1540       } else  {
1541         UMA_HISTOGRAM_TIMES(
1542             base::FieldTrial::MakeName("Net.HttpJob.TotalTimeNotCached",
1543                                        "CacheSensitivityAnalysis"),
1544             total_time);
1545       }
1546     }
1547   }
1548
1549   start_time_ = base::TimeTicks();
1550 }
1551
1552 void URLRequestHttpJob::DoneWithRequest(CompletionCause reason) {
1553   if (done_)
1554     return;
1555   done_ = true;
1556   RecordPerfHistograms(reason);
1557   if (reason == FINISHED) {
1558     request_->set_received_response_content_length(prefilter_bytes_read());
1559     RecordCompressionHistograms();
1560   }
1561 }
1562
1563 HttpResponseHeaders* URLRequestHttpJob::GetResponseHeaders() const {
1564   DCHECK(transaction_.get());
1565   DCHECK(transaction_->GetResponseInfo());
1566   return override_response_headers_.get() ?
1567       override_response_headers_ :
1568       transaction_->GetResponseInfo()->headers;
1569 }
1570
1571 void URLRequestHttpJob::NotifyURLRequestDestroyed() {
1572   awaiting_callback_ = false;
1573 }
1574
1575 void URLRequestHttpJob::OnDetachRequest() {
1576   http_transaction_delegate_->OnDetachRequest();
1577 }
1578
1579 }  // namespace net