content/browser/appcache/appcache_update_job.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/appcache/appcache_update_job.h"
   6
   7 #include "base/bind.h"
   8 #include "base/bind_helpers.h"
   9 #include "base/compiler_specific.h"
  10 #include "base/message_loop/message_loop.h"
  11 #include "base/profiler/scoped_tracker.h"
  12 #include "base/strings/string_util.h"
  13 #include "base/strings/stringprintf.h"
  14 #include "content/browser/appcache/appcache_group.h"
  15 #include "content/browser/appcache/appcache_histograms.h"
  16 #include "net/base/io_buffer.h"
  17 #include "net/base/load_flags.h"
  18 #include "net/base/net_errors.h"
  19 #include "net/base/request_priority.h"
  20 #include "net/http/http_request_headers.h"
  21 #include "net/http/http_response_headers.h"
  22 #include "net/url_request/url_request_context.h"
  23
  24 namespace content {
  25
  26 static const int kBufferSize = 32768;
  27 static const size_t kMaxConcurrentUrlFetches = 2;
  28 static const int kMax503Retries = 3;
  29
  30 static std::string FormatUrlErrorMessage(
  31       const char* format, const GURL& url,
  32       AppCacheUpdateJob::ResultType error,
  33       int response_code) {
  34     // Show the net response code if we have one.
  35     int code = response_code;
  36     if (error != AppCacheUpdateJob::SERVER_ERROR)
  37       code = static_cast<int>(error);
  38     return base::StringPrintf(format, code, url.spec().c_str());
  39 }
  40
  41 // Helper class for collecting hosts per frontend when sending notifications
  42 // so that only one notification is sent for all hosts using the same frontend.
  43 class HostNotifier {
  44  public:
  45   typedef std::vector<int> HostIds;
  46   typedef std::map<AppCacheFrontend*, HostIds> NotifyHostMap;
  47
  48   // Caller is responsible for ensuring there will be no duplicate hosts.
  49   void AddHost(AppCacheHost* host) {
  50     std::pair<NotifyHostMap::iterator , bool> ret = hosts_to_notify.insert(
  51         NotifyHostMap::value_type(host->frontend(), HostIds()));
  52     ret.first->second.push_back(host->host_id());
  53   }
  54
  55   void AddHosts(const std::set<AppCacheHost*>& hosts) {
  56     for (std::set<AppCacheHost*>::const_iterator it = hosts.begin();
  57          it != hosts.end(); ++it) {
  58       AddHost(*it);
  59     }
  60   }
  61
  62   void SendNotifications(AppCacheEventID event_id) {
  63     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  64          it != hosts_to_notify.end(); ++it) {
  65       AppCacheFrontend* frontend = it->first;
  66       frontend->OnEventRaised(it->second, event_id);
  67     }
  68   }
  69
  70   void SendProgressNotifications(
  71       const GURL& url, int num_total, int num_complete) {
  72     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  73          it != hosts_to_notify.end(); ++it) {
  74       AppCacheFrontend* frontend = it->first;
  75       frontend->OnProgressEventRaised(it->second, url,
  76                                       num_total, num_complete);
  77     }
  78   }
  79
  80   void SendErrorNotifications(const AppCacheErrorDetails& details) {
  81     DCHECK(!details.message.empty());
  82     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  83          it != hosts_to_notify.end(); ++it) {
  84       AppCacheFrontend* frontend = it->first;
  85       frontend->OnErrorEventRaised(it->second, details);
  86     }
  87   }
  88
  89   void SendLogMessage(const std::string& message) {
  90     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  91          it != hosts_to_notify.end(); ++it) {
  92       AppCacheFrontend* frontend = it->first;
  93       for (HostIds::iterator id = it->second.begin();
  94            id != it->second.end(); ++id) {
  95         frontend->OnLogMessage(*id, APPCACHE_LOG_WARNING, message);
  96       }
  97     }
  98   }
  99
 100  private:
 101   NotifyHostMap hosts_to_notify;
 102 };
 103
 104 AppCacheUpdateJob::UrlToFetch::UrlToFetch(const GURL& url,
 105                                           bool checked,
 106                                           AppCacheResponseInfo* info)
 107     : url(url),
 108       storage_checked(checked),
 109       existing_response_info(info) {
 110 }
 111
 112 AppCacheUpdateJob::UrlToFetch::~UrlToFetch() {
 113 }
 114
 115 // Helper class to fetch resources. Depending on the fetch type,
 116 // can either fetch to an in-memory string or write the response
 117 // data out to the disk cache.
 118 AppCacheUpdateJob::URLFetcher::URLFetcher(const GURL& url,
 119                                           FetchType fetch_type,
 120                                           AppCacheUpdateJob* job)
 121     : url_(url),
 122       job_(job),
 123       fetch_type_(fetch_type),
 124       retry_503_attempts_(0),
 125       buffer_(new net::IOBuffer(kBufferSize)),
 126       request_(job->service_->request_context()
 127                    ->CreateRequest(url, net::DEFAULT_PRIORITY, this, NULL)),
 128       result_(UPDATE_OK),
 129       redirect_response_code_(-1) {}
 130
 131 AppCacheUpdateJob::URLFetcher::~URLFetcher() {
 132 }
 133
 134 void AppCacheUpdateJob::URLFetcher::Start() {
 135   request_->set_first_party_for_cookies(job_->manifest_url_);
 136   if (existing_response_headers_.get())
 137     AddConditionalHeaders(existing_response_headers_.get());
 138   request_->Start();
 139 }
 140
 141 void AppCacheUpdateJob::URLFetcher::OnReceivedRedirect(
 142     net::URLRequest* request,
 143     const net::RedirectInfo& redirect_info,
 144     bool* defer_redirect) {
 145   DCHECK(request_ == request);
 146   // Redirect is not allowed by the update process.
 147   job_->MadeProgress();
 148   redirect_response_code_ = request->GetResponseCode();
 149   request->Cancel();
 150   result_ = REDIRECT_ERROR;
 151   OnResponseCompleted();
 152 }
 153
 154 void AppCacheUpdateJob::URLFetcher::OnResponseStarted(
 155     net::URLRequest *request) {
 156   // TODO(vadimt): Remove ScopedTracker below once crbug.com/422516 is fixed.
 157   tracked_objects::ScopedTracker tracking_profile(
 158       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 159           "422516 AppCacheUpdateJob::URLFetcher::OnResponseStarted"));
 160
 161   DCHECK(request == request_);
 162   int response_code = -1;
 163   if (request->status().is_success()) {
 164     response_code = request->GetResponseCode();
 165     job_->MadeProgress();
 166   }
 167
 168   if ((response_code / 100) != 2) {
 169     if (response_code > 0)
 170       result_ = SERVER_ERROR;
 171     else
 172       result_ = NETWORK_ERROR;
 173     OnResponseCompleted();
 174     return;
 175   }
 176
 177   if (url_.SchemeIsSecure()) {
 178     // Do not cache content with cert errors.
 179     // Also, we willfully violate the HTML5 spec at this point in order
 180     // to support the appcaching of cross-origin HTTPS resources.
 181     // We've opted for a milder constraint and allow caching unless
 182     // the resource has a "no-store" header. A spec change has been
 183     // requested on the whatwg list.
 184     // See http://code.google.com/p/chromium/issues/detail?id=69594
 185     // TODO(michaeln): Consider doing this for cross-origin HTTP too.
 186     const net::HttpNetworkSession::Params* session_params =
 187         request->context()->GetNetworkSessionParams();
 188     bool ignore_cert_errors = session_params &&
 189                               session_params->ignore_certificate_errors;
 190     if ((net::IsCertStatusError(request->ssl_info().cert_status) &&
 191             !ignore_cert_errors) ||
 192         (url_.GetOrigin() != job_->manifest_url_.GetOrigin() &&
 193             request->response_headers()->
 194                 HasHeaderValue("cache-control", "no-store"))) {
 195       DCHECK_EQ(-1, redirect_response_code_);
 196       request->Cancel();
 197       result_ = SECURITY_ERROR;
 198       OnResponseCompleted();
 199       return;
 200     }
 201   }
 202
 203   // Write response info to storage for URL fetches. Wait for async write
 204   // completion before reading any response data.
 205   if (fetch_type_ == URL_FETCH || fetch_type_ == MASTER_ENTRY_FETCH) {
 206     response_writer_.reset(job_->CreateResponseWriter());
 207     scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
 208         new HttpResponseInfoIOBuffer(
 209             new net::HttpResponseInfo(request->response_info())));
 210     response_writer_->WriteInfo(
 211         io_buffer.get(),
 212         base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
 213   } else {
 214     ReadResponseData();
 215   }
 216 }
 217
 218 void AppCacheUpdateJob::URLFetcher::OnReadCompleted(
 219     net::URLRequest* request, int bytes_read) {
 220   // TODO(vadimt): Remove ScopedTracker below once crbug.com/422516 is fixed.
 221   tracked_objects::ScopedTracker tracking_profile(
 222       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 223           "422516 AppCacheUpdateJob::URLFetcher::OnReadCompleted"));
 224
 225   DCHECK(request_ == request);
 226   bool data_consumed = true;
 227   if (request->status().is_success() && bytes_read > 0) {
 228     job_->MadeProgress();
 229     data_consumed = ConsumeResponseData(bytes_read);
 230     if (data_consumed) {
 231       bytes_read = 0;
 232       while (request->Read(buffer_.get(), kBufferSize, &bytes_read)) {
 233         if (bytes_read > 0) {
 234           data_consumed = ConsumeResponseData(bytes_read);
 235           if (!data_consumed)
 236             break;  // wait for async data processing, then read more
 237         } else {
 238           break;
 239         }
 240       }
 241     }
 242   }
 243   if (data_consumed && !request->status().is_io_pending()) {
 244     DCHECK_EQ(UPDATE_OK, result_);
 245     OnResponseCompleted();
 246   }
 247 }
 248
 249 void AppCacheUpdateJob::URLFetcher::AddConditionalHeaders(
 250     const net::HttpResponseHeaders* headers) {
 251   DCHECK(request_.get() && headers);
 252   net::HttpRequestHeaders extra_headers;
 253
 254   // Add If-Modified-Since header if response info has Last-Modified header.
 255   const std::string last_modified = "Last-Modified";
 256   std::string last_modified_value;
 257   headers->EnumerateHeader(NULL, last_modified, &last_modified_value);
 258   if (!last_modified_value.empty()) {
 259     extra_headers.SetHeader(net::HttpRequestHeaders::kIfModifiedSince,
 260                             last_modified_value);
 261   }
 262
 263   // Add If-None-Match header if response info has ETag header.
 264   const std::string etag = "ETag";
 265   std::string etag_value;
 266   headers->EnumerateHeader(NULL, etag, &etag_value);
 267   if (!etag_value.empty()) {
 268     extra_headers.SetHeader(net::HttpRequestHeaders::kIfNoneMatch,
 269                             etag_value);
 270   }
 271   if (!extra_headers.IsEmpty())
 272     request_->SetExtraRequestHeaders(extra_headers);
 273 }
 274
 275 void  AppCacheUpdateJob::URLFetcher::OnWriteComplete(int result) {
 276   if (result < 0) {
 277     request_->Cancel();
 278     result_ = DISKCACHE_ERROR;
 279     OnResponseCompleted();
 280     return;
 281   }
 282   ReadResponseData();
 283 }
 284
 285 void AppCacheUpdateJob::URLFetcher::ReadResponseData() {
 286   InternalUpdateState state = job_->internal_state_;
 287   if (state == CACHE_FAILURE || state == CANCELLED || state == COMPLETED)
 288     return;
 289   int bytes_read = 0;
 290   request_->Read(buffer_.get(), kBufferSize, &bytes_read);
 291   OnReadCompleted(request_.get(), bytes_read);
 292 }
 293
 294 // Returns false if response data is processed asynchronously, in which
 295 // case ReadResponseData will be invoked when it is safe to continue
 296 // reading more response data from the request.
 297 bool AppCacheUpdateJob::URLFetcher::ConsumeResponseData(int bytes_read) {
 298   DCHECK_GT(bytes_read, 0);
 299   switch (fetch_type_) {
 300     case MANIFEST_FETCH:
 301     case MANIFEST_REFETCH:
 302       manifest_data_.append(buffer_->data(), bytes_read);
 303       break;
 304     case URL_FETCH:
 305     case MASTER_ENTRY_FETCH:
 306       DCHECK(response_writer_.get());
 307       response_writer_->WriteData(
 308           buffer_.get(),
 309           bytes_read,
 310           base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
 311       return false;  // wait for async write completion to continue reading
 312     default:
 313       NOTREACHED();
 314   }
 315   return true;
 316 }
 317
 318 void AppCacheUpdateJob::URLFetcher::OnResponseCompleted() {
 319   if (request_->status().is_success())
 320     job_->MadeProgress();
 321
 322   // Retry for 503s where retry-after is 0.
 323   if (request_->status().is_success() &&
 324       request_->GetResponseCode() == 503 &&
 325       MaybeRetryRequest()) {
 326     return;
 327   }
 328
 329   switch (fetch_type_) {
 330     case MANIFEST_FETCH:
 331       job_->HandleManifestFetchCompleted(this);
 332       break;
 333     case URL_FETCH:
 334       job_->HandleUrlFetchCompleted(this);
 335       break;
 336     case MASTER_ENTRY_FETCH:
 337       job_->HandleMasterEntryFetchCompleted(this);
 338       break;
 339     case MANIFEST_REFETCH:
 340       job_->HandleManifestRefetchCompleted(this);
 341       break;
 342     default:
 343       NOTREACHED();
 344   }
 345
 346   delete this;
 347 }
 348
 349 bool AppCacheUpdateJob::URLFetcher::MaybeRetryRequest() {
 350   if (retry_503_attempts_ >= kMax503Retries ||
 351       !request_->response_headers()->HasHeaderValue("retry-after", "0")) {
 352     return false;
 353   }
 354   ++retry_503_attempts_;
 355   result_ = UPDATE_OK;
 356   request_ = job_->service_->request_context()->CreateRequest(
 357       url_, net::DEFAULT_PRIORITY, this, NULL);
 358   Start();
 359   return true;
 360 }
 361
 362 AppCacheUpdateJob::AppCacheUpdateJob(AppCacheServiceImpl* service,
 363                                      AppCacheGroup* group)
 364     : service_(service),
 365       manifest_url_(group->manifest_url()),
 366       group_(group),
 367       update_type_(UNKNOWN_TYPE),
 368       internal_state_(FETCH_MANIFEST),
 369       master_entries_completed_(0),
 370       url_fetches_completed_(0),
 371       manifest_fetcher_(NULL),
 372       manifest_has_valid_mime_type_(false),
 373       stored_state_(UNSTORED),
 374       storage_(service->storage()) {
 375     service_->AddObserver(this);
 376 }
 377
 378 AppCacheUpdateJob::~AppCacheUpdateJob() {
 379   if (service_)
 380     service_->RemoveObserver(this);
 381   if (internal_state_ != COMPLETED)
 382     Cancel();
 383
 384   DCHECK(!manifest_fetcher_);
 385   DCHECK(pending_url_fetches_.empty());
 386   DCHECK(!inprogress_cache_.get());
 387   DCHECK(pending_master_entries_.empty());
 388   DCHECK(master_entry_fetches_.empty());
 389
 390   if (group_)
 391     group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE);
 392 }
 393
 394 void AppCacheUpdateJob::StartUpdate(AppCacheHost* host,
 395                                     const GURL& new_master_resource) {
 396   DCHECK(group_->update_job() == this);
 397   DCHECK(!group_->is_obsolete());
 398
 399   bool is_new_pending_master_entry = false;
 400   if (!new_master_resource.is_empty()) {
 401     DCHECK(new_master_resource == host->pending_master_entry_url());
 402     DCHECK(!new_master_resource.has_ref());
 403     DCHECK(new_master_resource.GetOrigin() == manifest_url_.GetOrigin());
 404
 405     // Cannot add more to this update if already terminating.
 406     if (IsTerminating()) {
 407       group_->QueueUpdate(host, new_master_resource);
 408       return;
 409     }
 410
 411     std::pair<PendingMasters::iterator, bool> ret =
 412         pending_master_entries_.insert(
 413             PendingMasters::value_type(new_master_resource, PendingHosts()));
 414     is_new_pending_master_entry = ret.second;
 415     ret.first->second.push_back(host);
 416     host->AddObserver(this);
 417   }
 418
 419   // Notify host (if any) if already checking or downloading.
 420   AppCacheGroup::UpdateAppCacheStatus update_status = group_->update_status();
 421   if (update_status == AppCacheGroup::CHECKING ||
 422       update_status == AppCacheGroup::DOWNLOADING) {
 423     if (host) {
 424       NotifySingleHost(host, APPCACHE_CHECKING_EVENT);
 425       if (update_status == AppCacheGroup::DOWNLOADING)
 426         NotifySingleHost(host, APPCACHE_DOWNLOADING_EVENT);
 427
 428       // Add to fetch list or an existing entry if already fetched.
 429       if (!new_master_resource.is_empty()) {
 430         AddMasterEntryToFetchList(host, new_master_resource,
 431                                   is_new_pending_master_entry);
 432       }
 433     }
 434     return;
 435   }
 436
 437   // Begin update process for the group.
 438   MadeProgress();
 439   group_->SetUpdateAppCacheStatus(AppCacheGroup::CHECKING);
 440   if (group_->HasCache()) {
 441     update_type_ = UPGRADE_ATTEMPT;
 442     NotifyAllAssociatedHosts(APPCACHE_CHECKING_EVENT);
 443   } else {
 444     update_type_ = CACHE_ATTEMPT;
 445     DCHECK(host);
 446     NotifySingleHost(host, APPCACHE_CHECKING_EVENT);
 447   }
 448
 449   if (!new_master_resource.is_empty()) {
 450     AddMasterEntryToFetchList(host, new_master_resource,
 451                               is_new_pending_master_entry);
 452   }
 453
 454   FetchManifest(true);
 455 }
 456
 457 AppCacheResponseWriter* AppCacheUpdateJob::CreateResponseWriter() {
 458   AppCacheResponseWriter* writer =
 459       storage_->CreateResponseWriter(manifest_url_,
 460                                                 group_->group_id());
 461   stored_response_ids_.push_back(writer->response_id());
 462   return writer;
 463 }
 464
 465 void AppCacheUpdateJob::HandleCacheFailure(
 466     const AppCacheErrorDetails& error_details,
 467     ResultType result,
 468     const GURL& failed_resource_url) {
 469   // 6.9.4 cache failure steps 2-8.
 470   DCHECK(internal_state_ != CACHE_FAILURE);
 471   DCHECK(!error_details.message.empty());
 472   DCHECK(result != UPDATE_OK);
 473   internal_state_ = CACHE_FAILURE;
 474   LogHistogramStats(result, failed_resource_url);
 475   CancelAllUrlFetches();
 476   CancelAllMasterEntryFetches(error_details);
 477   NotifyAllError(error_details);
 478   DiscardInprogressCache();
 479   internal_state_ = COMPLETED;
 480   DeleteSoon();  // To unwind the stack prior to deletion.
 481 }
 482
 483 void AppCacheUpdateJob::FetchManifest(bool is_first_fetch) {
 484   DCHECK(!manifest_fetcher_);
 485   manifest_fetcher_ = new URLFetcher(
 486      manifest_url_,
 487      is_first_fetch ? URLFetcher::MANIFEST_FETCH :
 488                       URLFetcher::MANIFEST_REFETCH,
 489      this);
 490
 491   // Add any necessary Http headers before sending fetch request.
 492   if (is_first_fetch) {
 493     AppCacheEntry* entry = (update_type_ == UPGRADE_ATTEMPT) ?
 494         group_->newest_complete_cache()->GetEntry(manifest_url_) : NULL;
 495     if (entry) {
 496       // Asynchronously load response info for manifest from newest cache.
 497       storage_->LoadResponseInfo(manifest_url_, group_->group_id(),
 498                                  entry->response_id(), this);
 499     } else {
 500       manifest_fetcher_->Start();
 501     }
 502   } else {
 503     DCHECK(internal_state_ == REFETCH_MANIFEST);
 504     DCHECK(manifest_response_info_.get());
 505     manifest_fetcher_->set_existing_response_headers(
 506         manifest_response_info_->headers.get());
 507     manifest_fetcher_->Start();
 508   }
 509 }
 510
 511
 512 void AppCacheUpdateJob::HandleManifestFetchCompleted(
 513     URLFetcher* fetcher) {
 514   DCHECK_EQ(internal_state_, FETCH_MANIFEST);
 515   DCHECK_EQ(manifest_fetcher_, fetcher);
 516   manifest_fetcher_ = NULL;
 517
 518   net::URLRequest* request = fetcher->request();
 519   int response_code = -1;
 520   bool is_valid_response_code = false;
 521   if (request->status().is_success()) {
 522     response_code = request->GetResponseCode();
 523     is_valid_response_code = (response_code / 100 == 2);
 524
 525     std::string mime_type;
 526     request->GetMimeType(&mime_type);
 527     manifest_has_valid_mime_type_ = (mime_type == "text/cache-manifest");
 528   }
 529
 530   if (is_valid_response_code) {
 531     manifest_data_ = fetcher->manifest_data();
 532     manifest_response_info_.reset(
 533         new net::HttpResponseInfo(request->response_info()));
 534     if (update_type_ == UPGRADE_ATTEMPT)
 535       CheckIfManifestChanged();  // continues asynchronously
 536     else
 537       ContinueHandleManifestFetchCompleted(true);
 538   } else if (response_code == 304 && update_type_ == UPGRADE_ATTEMPT) {
 539     ContinueHandleManifestFetchCompleted(false);
 540   } else if ((response_code == 404 || response_code == 410) &&
 541              update_type_ == UPGRADE_ATTEMPT) {
 542     storage_->MakeGroupObsolete(group_, this, response_code);  // async
 543   } else {
 544     const char* kFormatString = "Manifest fetch failed (%d) %s";
 545     std::string message = FormatUrlErrorMessage(
 546         kFormatString, manifest_url_, fetcher->result(), response_code);
 547     HandleCacheFailure(AppCacheErrorDetails(message,
 548                                     APPCACHE_MANIFEST_ERROR,
 549                                     manifest_url_,
 550                                     response_code,
 551                                     false /*is_cross_origin*/),
 552                        fetcher->result(),
 553                        GURL());
 554   }
 555 }
 556
 557 void AppCacheUpdateJob::OnGroupMadeObsolete(AppCacheGroup* group,
 558                                             bool success,
 559                                             int response_code) {
 560   DCHECK(master_entry_fetches_.empty());
 561   CancelAllMasterEntryFetches(AppCacheErrorDetails(
 562       "The cache has been made obsolete, "
 563       "the manifest file returned 404 or 410",
 564       APPCACHE_MANIFEST_ERROR,
 565       GURL(),
 566       response_code,
 567       false /*is_cross_origin*/));
 568   if (success) {
 569     DCHECK(group->is_obsolete());
 570     NotifyAllAssociatedHosts(APPCACHE_OBSOLETE_EVENT);
 571     internal_state_ = COMPLETED;
 572     MaybeCompleteUpdate();
 573   } else {
 574     // Treat failure to mark group obsolete as a cache failure.
 575     HandleCacheFailure(AppCacheErrorDetails(
 576         "Failed to mark the cache as obsolete",
 577         APPCACHE_UNKNOWN_ERROR,
 578         GURL(),
 579         0,
 580         false /*is_cross_origin*/),
 581                        DB_ERROR,
 582                        GURL());
 583   }
 584 }
 585
 586 void AppCacheUpdateJob::ContinueHandleManifestFetchCompleted(bool changed) {
 587   DCHECK(internal_state_ == FETCH_MANIFEST);
 588
 589   if (!changed) {
 590     DCHECK(update_type_ == UPGRADE_ATTEMPT);
 591     internal_state_ = NO_UPDATE;
 592
 593     // Wait for pending master entries to download.
 594     FetchMasterEntries();
 595     MaybeCompleteUpdate();  // if not done, run async 6.9.4 step 7 substeps
 596     return;
 597   }
 598
 599   AppCacheManifest manifest;
 600   if (!ParseManifest(manifest_url_, manifest_data_.data(),
 601                      manifest_data_.length(),
 602                      manifest_has_valid_mime_type_ ?
 603                         PARSE_MANIFEST_ALLOWING_INTERCEPTS :
 604                         PARSE_MANIFEST_PER_STANDARD,
 605                      manifest)) {
 606     const char* kFormatString = "Failed to parse manifest %s";
 607     const std::string message = base::StringPrintf(kFormatString,
 608         manifest_url_.spec().c_str());
 609     HandleCacheFailure(
 610         AppCacheErrorDetails(
 611             message, APPCACHE_SIGNATURE_ERROR, GURL(), 0,
 612             false /*is_cross_origin*/),
 613         MANIFEST_ERROR,
 614         GURL());
 615     VLOG(1) << message;
 616     return;
 617   }
 618
 619   // Proceed with update process. Section 6.9.4 steps 8-20.
 620   internal_state_ = DOWNLOADING;
 621   inprogress_cache_ = new AppCache(storage_, storage_->NewCacheId());
 622   BuildUrlFileList(manifest);
 623   inprogress_cache_->InitializeWithManifest(&manifest);
 624
 625   // Associate all pending master hosts with the newly created cache.
 626   for (PendingMasters::iterator it = pending_master_entries_.begin();
 627        it != pending_master_entries_.end(); ++it) {
 628     PendingHosts& hosts = it->second;
 629     for (PendingHosts::iterator host_it = hosts.begin();
 630          host_it != hosts.end(); ++host_it) {
 631       (*host_it)
 632           ->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
 633     }
 634   }
 635
 636   if (manifest.did_ignore_intercept_namespaces) {
 637     // Must be done after associating all pending master hosts.
 638     std::string message(
 639         "Ignoring the INTERCEPT section of the application cache manifest "
 640         "because the content type is not text/cache-manifest");
 641     LogConsoleMessageToAll(message);
 642   }
 643
 644   group_->SetUpdateAppCacheStatus(AppCacheGroup::DOWNLOADING);
 645   NotifyAllAssociatedHosts(APPCACHE_DOWNLOADING_EVENT);
 646   FetchUrls();
 647   FetchMasterEntries();
 648   MaybeCompleteUpdate();  // if not done, continues when async fetches complete
 649 }
 650
 651 void AppCacheUpdateJob::HandleUrlFetchCompleted(URLFetcher* fetcher) {
 652   DCHECK(internal_state_ == DOWNLOADING);
 653
 654   net::URLRequest* request = fetcher->request();
 655   const GURL& url = request->original_url();
 656   pending_url_fetches_.erase(url);
 657   NotifyAllProgress(url);
 658   ++url_fetches_completed_;
 659
 660   int response_code = request->status().is_success()
 661                           ? request->GetResponseCode()
 662                           : fetcher->redirect_response_code();
 663
 664   AppCacheEntry& entry = url_file_list_.find(url)->second;
 665
 666   if (response_code / 100 == 2) {
 667     // Associate storage with the new entry.
 668     DCHECK(fetcher->response_writer());
 669     entry.set_response_id(fetcher->response_writer()->response_id());
 670     entry.set_response_size(fetcher->response_writer()->amount_written());
 671     if (!inprogress_cache_->AddOrModifyEntry(url, entry))
 672       duplicate_response_ids_.push_back(entry.response_id());
 673
 674     // TODO(michaeln): Check for <html manifest=xxx>
 675     // See http://code.google.com/p/chromium/issues/detail?id=97930
 676     // if (entry.IsMaster() && !(entry.IsExplicit() || fallback || intercept))
 677     //   if (!manifestAttribute) skip it
 678
 679     // Foreign entries will be detected during cache selection.
 680     // Note: 6.9.4, step 17.9 possible optimization: if resource is HTML or XML
 681     // file whose root element is an html element with a manifest attribute
 682     // whose value doesn't match the manifest url of the application cache
 683     // being processed, mark the entry as being foreign.
 684   } else {
 685     VLOG(1) << "Request status: " << request->status().status()
 686             << " error: " << request->status().error()
 687             << " response code: " << response_code;
 688     if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept()) {
 689       if (response_code == 304 && fetcher->existing_entry().has_response_id()) {
 690         // Keep the existing response.
 691         entry.set_response_id(fetcher->existing_entry().response_id());
 692         entry.set_response_size(fetcher->existing_entry().response_size());
 693         inprogress_cache_->AddOrModifyEntry(url, entry);
 694       } else {
 695         const char* kFormatString = "Resource fetch failed (%d) %s";
 696         std::string message = FormatUrlErrorMessage(
 697             kFormatString, url, fetcher->result(), response_code);
 698         ResultType result = fetcher->result();
 699         bool is_cross_origin = url.GetOrigin() != manifest_url_.GetOrigin();
 700         switch (result) {
 701           case DISKCACHE_ERROR:
 702             HandleCacheFailure(
 703                 AppCacheErrorDetails(
 704                     message, APPCACHE_UNKNOWN_ERROR, GURL(), 0,
 705                     is_cross_origin),
 706                 result,
 707                 url);
 708             break;
 709           case NETWORK_ERROR:
 710             HandleCacheFailure(
 711                 AppCacheErrorDetails(message, APPCACHE_RESOURCE_ERROR, url, 0,
 712                     is_cross_origin),
 713                 result,
 714                 url);
 715             break;
 716           default:
 717             HandleCacheFailure(AppCacheErrorDetails(message,
 718                                             APPCACHE_RESOURCE_ERROR,
 719                                             url,
 720                                             response_code,
 721                                             is_cross_origin),
 722                                result,
 723                                url);
 724             break;
 725         }
 726         return;
 727       }
 728     } else if (response_code == 404 || response_code == 410) {
 729       // Entry is skipped.  They are dropped from the cache.
 730     } else if (update_type_ == UPGRADE_ATTEMPT &&
 731                fetcher->existing_entry().has_response_id()) {
 732       // Keep the existing response.
 733       // TODO(michaeln): Not sure this is a good idea. This is spec compliant
 734       // but the old resource may or may not be compatible with the new contents
 735       // of the cache. Impossible to know one way or the other.
 736       entry.set_response_id(fetcher->existing_entry().response_id());
 737       entry.set_response_size(fetcher->existing_entry().response_size());
 738       inprogress_cache_->AddOrModifyEntry(url, entry);
 739     }
 740   }
 741
 742   // Fetch another URL now that one request has completed.
 743   DCHECK(internal_state_ != CACHE_FAILURE);
 744   FetchUrls();
 745   MaybeCompleteUpdate();
 746 }
 747
 748 void AppCacheUpdateJob::HandleMasterEntryFetchCompleted(
 749     URLFetcher* fetcher) {
 750   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
 751
 752   // TODO(jennb): Handle downloads completing during cache failure when update
 753   // no longer fetches master entries directly. For now, we cancel all pending
 754   // master entry fetches when entering cache failure state so this will never
 755   // be called in CACHE_FAILURE state.
 756
 757   net::URLRequest* request = fetcher->request();
 758   const GURL& url = request->original_url();
 759   master_entry_fetches_.erase(url);
 760   ++master_entries_completed_;
 761
 762   int response_code = request->status().is_success()
 763       ? request->GetResponseCode() : -1;
 764
 765   PendingMasters::iterator found = pending_master_entries_.find(url);
 766   DCHECK(found != pending_master_entries_.end());
 767   PendingHosts& hosts = found->second;
 768
 769   // Section 6.9.4. No update case: step 7.3, else step 22.
 770   if (response_code / 100 == 2) {
 771     // Add fetched master entry to the appropriate cache.
 772     AppCache* cache = inprogress_cache_.get() ? inprogress_cache_.get()
 773                                               : group_->newest_complete_cache();
 774     DCHECK(fetcher->response_writer());
 775     AppCacheEntry master_entry(AppCacheEntry::MASTER,
 776                                fetcher->response_writer()->response_id(),
 777                                fetcher->response_writer()->amount_written());
 778     if (cache->AddOrModifyEntry(url, master_entry))
 779       added_master_entries_.push_back(url);
 780     else
 781       duplicate_response_ids_.push_back(master_entry.response_id());
 782
 783     // In no-update case, associate host with the newest cache.
 784     if (!inprogress_cache_.get()) {
 785       // TODO(michaeln): defer until the updated cache has been stored
 786       DCHECK(cache == group_->newest_complete_cache());
 787       for (PendingHosts::iterator host_it = hosts.begin();
 788            host_it != hosts.end(); ++host_it) {
 789         (*host_it)->AssociateCompleteCache(cache);
 790       }
 791     }
 792   } else {
 793     HostNotifier host_notifier;
 794     for (PendingHosts::iterator host_it = hosts.begin();
 795          host_it != hosts.end(); ++host_it) {
 796       AppCacheHost* host = *host_it;
 797       host_notifier.AddHost(host);
 798
 799       // In downloading case, disassociate host from inprogress cache.
 800       if (inprogress_cache_.get())
 801         host->AssociateNoCache(GURL());
 802
 803       host->RemoveObserver(this);
 804     }
 805     hosts.clear();
 806
 807     const char* kFormatString = "Manifest fetch failed (%d) %s";
 808     std::string message = FormatUrlErrorMessage(
 809         kFormatString, request->url(), fetcher->result(), response_code);
 810     host_notifier.SendErrorNotifications(
 811         AppCacheErrorDetails(message,
 812                      APPCACHE_MANIFEST_ERROR,
 813                      request->url(),
 814                      response_code,
 815                      false /*is_cross_origin*/));
 816
 817     // In downloading case, update result is different if all master entries
 818     // failed vs. only some failing.
 819     if (inprogress_cache_.get()) {
 820       // Only count successful downloads to know if all master entries failed.
 821       pending_master_entries_.erase(found);
 822       --master_entries_completed_;
 823
 824       // Section 6.9.4, step 22.3.
 825       if (update_type_ == CACHE_ATTEMPT && pending_master_entries_.empty()) {
 826         HandleCacheFailure(AppCacheErrorDetails(message,
 827                                         APPCACHE_MANIFEST_ERROR,
 828                                         request->url(),
 829                                         response_code,
 830                                         false /*is_cross_origin*/),
 831                            fetcher->result(),
 832                            GURL());
 833         return;
 834       }
 835     }
 836   }
 837
 838   DCHECK(internal_state_ != CACHE_FAILURE);
 839   FetchMasterEntries();
 840   MaybeCompleteUpdate();
 841 }
 842
 843 void AppCacheUpdateJob::HandleManifestRefetchCompleted(
 844     URLFetcher* fetcher) {
 845   DCHECK(internal_state_ == REFETCH_MANIFEST);
 846   DCHECK(manifest_fetcher_ == fetcher);
 847   manifest_fetcher_ = NULL;
 848
 849   net::URLRequest* request = fetcher->request();
 850   int response_code = request->status().is_success()
 851       ? request->GetResponseCode() : -1;
 852   if (response_code == 304 || manifest_data_ == fetcher->manifest_data()) {
 853     // Only need to store response in storage if manifest is not already
 854     // an entry in the cache.
 855     AppCacheEntry* entry = inprogress_cache_->GetEntry(manifest_url_);
 856     if (entry) {
 857       entry->add_types(AppCacheEntry::MANIFEST);
 858       StoreGroupAndCache();
 859     } else {
 860       manifest_response_writer_.reset(CreateResponseWriter());
 861       scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
 862           new HttpResponseInfoIOBuffer(manifest_response_info_.release()));
 863       manifest_response_writer_->WriteInfo(
 864           io_buffer.get(),
 865           base::Bind(&AppCacheUpdateJob::OnManifestInfoWriteComplete,
 866                      base::Unretained(this)));
 867     }
 868   } else {
 869     VLOG(1) << "Request status: " << request->status().status()
 870             << " error: " << request->status().error()
 871             << " response code: " << response_code;
 872     ScheduleUpdateRetry(kRerunDelayMs);
 873     if (response_code == 200) {
 874       HandleCacheFailure(AppCacheErrorDetails("Manifest changed during update",
 875                                       APPCACHE_CHANGED_ERROR,
 876                                       GURL(),
 877                                       0,
 878                                       false /*is_cross_origin*/),
 879                          MANIFEST_ERROR,
 880                          GURL());
 881     } else {
 882       const char* kFormatString = "Manifest re-fetch failed (%d) %s";
 883       std::string message = FormatUrlErrorMessage(
 884           kFormatString, manifest_url_, fetcher->result(), response_code);
 885       HandleCacheFailure(AppCacheErrorDetails(message,
 886                                       APPCACHE_MANIFEST_ERROR,
 887                                       GURL(),
 888                                       response_code,
 889                                       false /*is_cross_origin*/),
 890                          fetcher->result(),
 891                          GURL());
 892     }
 893   }
 894 }
 895
 896 void AppCacheUpdateJob::OnManifestInfoWriteComplete(int result) {
 897   if (result > 0) {
 898     scoped_refptr<net::StringIOBuffer> io_buffer(
 899         new net::StringIOBuffer(manifest_data_));
 900     manifest_response_writer_->WriteData(
 901         io_buffer.get(),
 902         manifest_data_.length(),
 903         base::Bind(&AppCacheUpdateJob::OnManifestDataWriteComplete,
 904                    base::Unretained(this)));
 905   } else {
 906     HandleCacheFailure(
 907         AppCacheErrorDetails("Failed to write the manifest headers to storage",
 908                      APPCACHE_UNKNOWN_ERROR,
 909                      GURL(),
 910                      0,
 911                      false /*is_cross_origin*/),
 912         DISKCACHE_ERROR,
 913         GURL());
 914   }
 915 }
 916
 917 void AppCacheUpdateJob::OnManifestDataWriteComplete(int result) {
 918   if (result > 0) {
 919     AppCacheEntry entry(AppCacheEntry::MANIFEST,
 920         manifest_response_writer_->response_id(),
 921         manifest_response_writer_->amount_written());
 922     if (!inprogress_cache_->AddOrModifyEntry(manifest_url_, entry))
 923       duplicate_response_ids_.push_back(entry.response_id());
 924     StoreGroupAndCache();
 925   } else {
 926     HandleCacheFailure(
 927         AppCacheErrorDetails("Failed to write the manifest data to storage",
 928                      APPCACHE_UNKNOWN_ERROR,
 929                      GURL(),
 930                      0,
 931                      false /*is_cross_origin*/),
 932         DISKCACHE_ERROR,
 933         GURL());
 934   }
 935 }
 936
 937 void AppCacheUpdateJob::StoreGroupAndCache() {
 938   DCHECK(stored_state_ == UNSTORED);
 939   stored_state_ = STORING;
 940   scoped_refptr<AppCache> newest_cache;
 941   if (inprogress_cache_.get())
 942     newest_cache.swap(inprogress_cache_);
 943   else
 944     newest_cache = group_->newest_complete_cache();
 945   newest_cache->set_update_time(base::Time::Now());
 946
 947   // TODO(michaeln): dcheck is fishing for clues to crbug/95101
 948   DCHECK_EQ(manifest_url_, group_->manifest_url());
 949   storage_->StoreGroupAndNewestCache(group_, newest_cache.get(), this);
 950 }
 951
 952 void AppCacheUpdateJob::OnGroupAndNewestCacheStored(AppCacheGroup* group,
 953                                                     AppCache* newest_cache,
 954                                                     bool success,
 955                                                     bool would_exceed_quota) {
 956   DCHECK(stored_state_ == STORING);
 957   if (success) {
 958     stored_state_ = STORED;
 959     MaybeCompleteUpdate();  // will definitely complete
 960   } else {
 961     stored_state_ = UNSTORED;
 962
 963     // Restore inprogress_cache_ to get the proper events delivered
 964     // and the proper cleanup to occur.
 965     if (newest_cache != group->newest_complete_cache())
 966       inprogress_cache_ = newest_cache;
 967
 968     ResultType result = DB_ERROR;
 969     AppCacheErrorReason reason = APPCACHE_UNKNOWN_ERROR;
 970     std::string message("Failed to commit new cache to storage");
 971     if (would_exceed_quota) {
 972       message.append(", would exceed quota");
 973       result = QUOTA_ERROR;
 974       reason = APPCACHE_QUOTA_ERROR;
 975     }
 976     HandleCacheFailure(
 977         AppCacheErrorDetails(message, reason, GURL(), 0,
 978             false /*is_cross_origin*/),
 979         result,
 980         GURL());
 981   }
 982 }
 983
 984 void AppCacheUpdateJob::NotifySingleHost(AppCacheHost* host,
 985                                          AppCacheEventID event_id) {
 986   std::vector<int> ids(1, host->host_id());
 987   host->frontend()->OnEventRaised(ids, event_id);
 988 }
 989
 990 void AppCacheUpdateJob::NotifyAllAssociatedHosts(AppCacheEventID event_id) {
 991   HostNotifier host_notifier;
 992   AddAllAssociatedHostsToNotifier(&host_notifier);
 993   host_notifier.SendNotifications(event_id);
 994 }
 995
 996 void AppCacheUpdateJob::NotifyAllProgress(const GURL& url) {
 997   HostNotifier host_notifier;
 998   AddAllAssociatedHostsToNotifier(&host_notifier);
 999   host_notifier.SendProgressNotifications(
1000       url, url_file_list_.size(), url_fetches_completed_);
1001 }
1002
1003 void AppCacheUpdateJob::NotifyAllFinalProgress() {
1004   DCHECK(url_file_list_.size() == url_fetches_completed_);
1005   NotifyAllProgress(GURL());
1006 }
1007
1008 void AppCacheUpdateJob::NotifyAllError(const AppCacheErrorDetails& details) {
1009   HostNotifier host_notifier;
1010   AddAllAssociatedHostsToNotifier(&host_notifier);
1011   host_notifier.SendErrorNotifications(details);
1012 }
1013
1014 void AppCacheUpdateJob::LogConsoleMessageToAll(const std::string& message) {
1015   HostNotifier host_notifier;
1016   AddAllAssociatedHostsToNotifier(&host_notifier);
1017   host_notifier.SendLogMessage(message);
1018 }
1019
1020 void AppCacheUpdateJob::AddAllAssociatedHostsToNotifier(
1021     HostNotifier* host_notifier) {
1022   // Collect hosts so we only send one notification per frontend.
1023   // A host can only be associated with a single cache so no need to worry
1024   // about duplicate hosts being added to the notifier.
1025   if (inprogress_cache_.get()) {
1026     DCHECK(internal_state_ == DOWNLOADING || internal_state_ == CACHE_FAILURE);
1027     host_notifier->AddHosts(inprogress_cache_->associated_hosts());
1028   }
1029
1030   AppCacheGroup::Caches old_caches = group_->old_caches();
1031   for (AppCacheGroup::Caches::const_iterator it = old_caches.begin();
1032        it != old_caches.end(); ++it) {
1033     host_notifier->AddHosts((*it)->associated_hosts());
1034   }
1035
1036   AppCache* newest_cache = group_->newest_complete_cache();
1037   if (newest_cache)
1038     host_notifier->AddHosts(newest_cache->associated_hosts());
1039 }
1040
1041 void AppCacheUpdateJob::OnDestructionImminent(AppCacheHost* host) {
1042   // The host is about to be deleted; remove from our collection.
1043   PendingMasters::iterator found =
1044       pending_master_entries_.find(host->pending_master_entry_url());
1045   DCHECK(found != pending_master_entries_.end());
1046   PendingHosts& hosts = found->second;
1047   PendingHosts::iterator it = std::find(hosts.begin(), hosts.end(), host);
1048   DCHECK(it != hosts.end());
1049   hosts.erase(it);
1050 }
1051
1052 void AppCacheUpdateJob::OnServiceReinitialized(
1053     AppCacheStorageReference* old_storage_ref) {
1054   // We continue to use the disabled instance, but arrange for its
1055   // deletion when its no longer needed.
1056   if (old_storage_ref->storage() == storage_)
1057     disabled_storage_reference_ = old_storage_ref;
1058 }
1059
1060 void AppCacheUpdateJob::CheckIfManifestChanged() {
1061   DCHECK(update_type_ == UPGRADE_ATTEMPT);
1062   AppCacheEntry* entry = NULL;
1063   if (group_->newest_complete_cache())
1064     entry = group_->newest_complete_cache()->GetEntry(manifest_url_);
1065   if (!entry) {
1066     // TODO(michaeln): This is just a bandaid to avoid a crash.
1067     // http://code.google.com/p/chromium/issues/detail?id=95101
1068     if (service_->storage() == storage_) {
1069       // Use a local variable because service_ is reset in HandleCacheFailure.
1070       AppCacheServiceImpl* service = service_;
1071       HandleCacheFailure(
1072           AppCacheErrorDetails("Manifest entry not found in existing cache",
1073                        APPCACHE_UNKNOWN_ERROR,
1074                        GURL(),
1075                        0,
1076                        false /*is_cross_origin*/),
1077           DB_ERROR,
1078           GURL());
1079       AppCacheHistograms::AddMissingManifestEntrySample();
1080       service->DeleteAppCacheGroup(manifest_url_, net::CompletionCallback());
1081     }
1082     return;
1083   }
1084
1085   // Load manifest data from storage to compare against fetched manifest.
1086   manifest_response_reader_.reset(
1087       storage_->CreateResponseReader(manifest_url_,
1088                                      group_->group_id(),
1089                                      entry->response_id()));
1090   read_manifest_buffer_ = new net::IOBuffer(kBufferSize);
1091   manifest_response_reader_->ReadData(
1092       read_manifest_buffer_.get(),
1093       kBufferSize,
1094       base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
1095                  base::Unretained(this)));  // async read
1096 }
1097
1098 void AppCacheUpdateJob::OnManifestDataReadComplete(int result) {
1099   if (result > 0) {
1100     loaded_manifest_data_.append(read_manifest_buffer_->data(), result);
1101     manifest_response_reader_->ReadData(
1102         read_manifest_buffer_.get(),
1103         kBufferSize,
1104         base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
1105                    base::Unretained(this)));  // read more
1106   } else {
1107     read_manifest_buffer_ = NULL;
1108     manifest_response_reader_.reset();
1109     ContinueHandleManifestFetchCompleted(
1110         result < 0 || manifest_data_ != loaded_manifest_data_);
1111   }
1112 }
1113
1114 void AppCacheUpdateJob::BuildUrlFileList(const AppCacheManifest& manifest) {
1115   for (base::hash_set<std::string>::const_iterator it =
1116            manifest.explicit_urls.begin();
1117        it != manifest.explicit_urls.end(); ++it) {
1118     AddUrlToFileList(GURL(*it), AppCacheEntry::EXPLICIT);
1119   }
1120
1121   const std::vector<AppCacheNamespace>& intercepts =
1122       manifest.intercept_namespaces;
1123   for (std::vector<AppCacheNamespace>::const_iterator it = intercepts.begin();
1124        it != intercepts.end(); ++it) {
1125     int flags = AppCacheEntry::INTERCEPT;
1126     if (it->is_executable)
1127       flags |= AppCacheEntry::EXECUTABLE;
1128     AddUrlToFileList(it->target_url, flags);
1129   }
1130
1131   const std::vector<AppCacheNamespace>& fallbacks =
1132       manifest.fallback_namespaces;
1133   for (std::vector<AppCacheNamespace>::const_iterator it = fallbacks.begin();
1134        it != fallbacks.end(); ++it) {
1135      AddUrlToFileList(it->target_url, AppCacheEntry::FALLBACK);
1136   }
1137
1138   // Add all master entries from newest complete cache.
1139   if (update_type_ == UPGRADE_ATTEMPT) {
1140     const AppCache::EntryMap& entries =
1141         group_->newest_complete_cache()->entries();
1142     for (AppCache::EntryMap::const_iterator it = entries.begin();
1143          it != entries.end(); ++it) {
1144       const AppCacheEntry& entry = it->second;
1145       if (entry.IsMaster())
1146         AddUrlToFileList(it->first, AppCacheEntry::MASTER);
1147     }
1148   }
1149 }
1150
1151 void AppCacheUpdateJob::AddUrlToFileList(const GURL& url, int type) {
1152   std::pair<AppCache::EntryMap::iterator, bool> ret = url_file_list_.insert(
1153       AppCache::EntryMap::value_type(url, AppCacheEntry(type)));
1154
1155   if (ret.second)
1156     urls_to_fetch_.push_back(UrlToFetch(url, false, NULL));
1157   else
1158     ret.first->second.add_types(type);  // URL already exists. Merge types.
1159 }
1160
1161 void AppCacheUpdateJob::FetchUrls() {
1162   DCHECK(internal_state_ == DOWNLOADING);
1163
1164   // Fetch each URL in the list according to section 6.9.4 step 17.1-17.3.
1165   // Fetch up to the concurrent limit. Other fetches will be triggered as each
1166   // each fetch completes.
1167   while (pending_url_fetches_.size() < kMaxConcurrentUrlFetches &&
1168          !urls_to_fetch_.empty()) {
1169     UrlToFetch url_to_fetch = urls_to_fetch_.front();
1170     urls_to_fetch_.pop_front();
1171
1172     AppCache::EntryMap::iterator it = url_file_list_.find(url_to_fetch.url);
1173     DCHECK(it != url_file_list_.end());
1174     AppCacheEntry& entry = it->second;
1175     if (ShouldSkipUrlFetch(entry)) {
1176       NotifyAllProgress(url_to_fetch.url);
1177       ++url_fetches_completed_;
1178     } else if (AlreadyFetchedEntry(url_to_fetch.url, entry.types())) {
1179       NotifyAllProgress(url_to_fetch.url);
1180       ++url_fetches_completed_;  // saved a URL request
1181     } else if (!url_to_fetch.storage_checked &&
1182                MaybeLoadFromNewestCache(url_to_fetch.url, entry)) {
1183       // Continues asynchronously after data is loaded from newest cache.
1184     } else {
1185       URLFetcher* fetcher = new URLFetcher(
1186           url_to_fetch.url, URLFetcher::URL_FETCH, this);
1187       if (url_to_fetch.existing_response_info.get()) {
1188         DCHECK(group_->newest_complete_cache());
1189         AppCacheEntry* existing_entry =
1190             group_->newest_complete_cache()->GetEntry(url_to_fetch.url);
1191         DCHECK(existing_entry);
1192         DCHECK(existing_entry->response_id() ==
1193                url_to_fetch.existing_response_info->response_id());
1194         fetcher->set_existing_response_headers(
1195             url_to_fetch.existing_response_info->http_response_info()->headers
1196                 .get());
1197         fetcher->set_existing_entry(*existing_entry);
1198       }
1199       fetcher->Start();
1200       pending_url_fetches_.insert(
1201           PendingUrlFetches::value_type(url_to_fetch.url, fetcher));
1202     }
1203   }
1204 }
1205
1206 void AppCacheUpdateJob::CancelAllUrlFetches() {
1207   // Cancel any pending URL requests.
1208   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
1209        it != pending_url_fetches_.end(); ++it) {
1210     delete it->second;
1211   }
1212
1213   url_fetches_completed_ +=
1214       pending_url_fetches_.size() + urls_to_fetch_.size();
1215   pending_url_fetches_.clear();
1216   urls_to_fetch_.clear();
1217 }
1218
1219 bool AppCacheUpdateJob::ShouldSkipUrlFetch(const AppCacheEntry& entry) {
1220   // 6.6.4 Step 17
1221   // If the resource URL being processed was flagged as neither an
1222   // "explicit entry" nor or a "fallback entry", then the user agent
1223   // may skip this URL.
1224   if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept())
1225     return false;
1226
1227   // TODO(jennb): decide if entry should be skipped to expire it from cache
1228   return false;
1229 }
1230
1231 bool AppCacheUpdateJob::AlreadyFetchedEntry(const GURL& url,
1232                                             int entry_type) {
1233   DCHECK(internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE);
1234   AppCacheEntry* existing =
1235       inprogress_cache_.get() ? inprogress_cache_->GetEntry(url)
1236                               : group_->newest_complete_cache()->GetEntry(url);
1237   if (existing) {
1238     existing->add_types(entry_type);
1239     return true;
1240   }
1241   return false;
1242 }
1243
1244 void AppCacheUpdateJob::AddMasterEntryToFetchList(AppCacheHost* host,
1245                                                   const GURL& url,
1246                                                   bool is_new) {
1247   DCHECK(!IsTerminating());
1248
1249   if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE) {
1250     AppCache* cache;
1251     if (inprogress_cache_.get()) {
1252       // always associate
1253       host->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
1254       cache = inprogress_cache_.get();
1255     } else {
1256       cache = group_->newest_complete_cache();
1257     }
1258
1259     // Update existing entry if it has already been fetched.
1260     AppCacheEntry* entry = cache->GetEntry(url);
1261     if (entry) {
1262       entry->add_types(AppCacheEntry::MASTER);
1263       if (internal_state_ == NO_UPDATE && !inprogress_cache_.get()) {
1264         // only associate if have entry
1265         host->AssociateCompleteCache(cache);
1266       }
1267       if (is_new)
1268         ++master_entries_completed_;  // pretend fetching completed
1269       return;
1270     }
1271   }
1272
1273   // Add to fetch list if not already fetching.
1274   if (master_entry_fetches_.find(url) == master_entry_fetches_.end()) {
1275     master_entries_to_fetch_.insert(url);
1276     if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE)
1277       FetchMasterEntries();
1278   }
1279 }
1280
1281 void AppCacheUpdateJob::FetchMasterEntries() {
1282   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
1283
1284   // Fetch each master entry in the list, up to the concurrent limit.
1285   // Additional fetches will be triggered as each fetch completes.
1286   while (master_entry_fetches_.size() < kMaxConcurrentUrlFetches &&
1287          !master_entries_to_fetch_.empty()) {
1288     const GURL& url = *master_entries_to_fetch_.begin();
1289
1290     if (AlreadyFetchedEntry(url, AppCacheEntry::MASTER)) {
1291       ++master_entries_completed_;  // saved a URL request
1292
1293       // In no update case, associate hosts to newest cache in group
1294       // now that master entry has been "successfully downloaded".
1295       if (internal_state_ == NO_UPDATE) {
1296         // TODO(michaeln): defer until the updated cache has been stored.
1297         DCHECK(!inprogress_cache_.get());
1298         AppCache* cache = group_->newest_complete_cache();
1299         PendingMasters::iterator found = pending_master_entries_.find(url);
1300         DCHECK(found != pending_master_entries_.end());
1301         PendingHosts& hosts = found->second;
1302         for (PendingHosts::iterator host_it = hosts.begin();
1303              host_it != hosts.end(); ++host_it) {
1304           (*host_it)->AssociateCompleteCache(cache);
1305         }
1306       }
1307     } else {
1308       URLFetcher* fetcher = new URLFetcher(
1309           url, URLFetcher::MASTER_ENTRY_FETCH, this);
1310       fetcher->Start();
1311       master_entry_fetches_.insert(PendingUrlFetches::value_type(url, fetcher));
1312     }
1313
1314     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
1315   }
1316 }
1317
1318 void AppCacheUpdateJob::CancelAllMasterEntryFetches(
1319     const AppCacheErrorDetails& error_details) {
1320   // For now, cancel all in-progress fetches for master entries and pretend
1321   // all master entries fetches have completed.
1322   // TODO(jennb): Delete this when update no longer fetches master entries
1323   // directly.
1324
1325   // Cancel all in-progress fetches.
1326   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
1327        it != master_entry_fetches_.end(); ++it) {
1328     delete it->second;
1329     master_entries_to_fetch_.insert(it->first);  // back in unfetched list
1330   }
1331   master_entry_fetches_.clear();
1332
1333   master_entries_completed_ += master_entries_to_fetch_.size();
1334
1335   // Cache failure steps, step 2.
1336   // Pretend all master entries that have not yet been fetched have completed
1337   // downloading. Unassociate hosts from any appcache and send ERROR event.
1338   HostNotifier host_notifier;
1339   while (!master_entries_to_fetch_.empty()) {
1340     const GURL& url = *master_entries_to_fetch_.begin();
1341     PendingMasters::iterator found = pending_master_entries_.find(url);
1342     DCHECK(found != pending_master_entries_.end());
1343     PendingHosts& hosts = found->second;
1344     for (PendingHosts::iterator host_it = hosts.begin();
1345          host_it != hosts.end(); ++host_it) {
1346       AppCacheHost* host = *host_it;
1347       host->AssociateNoCache(GURL());
1348       host_notifier.AddHost(host);
1349       host->RemoveObserver(this);
1350     }
1351     hosts.clear();
1352
1353     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
1354   }
1355   host_notifier.SendErrorNotifications(error_details);
1356 }
1357
1358 bool AppCacheUpdateJob::MaybeLoadFromNewestCache(const GURL& url,
1359                                                  AppCacheEntry& entry) {
1360   if (update_type_ != UPGRADE_ATTEMPT)
1361     return false;
1362
1363   AppCache* newest = group_->newest_complete_cache();
1364   AppCacheEntry* copy_me = newest->GetEntry(url);
1365   if (!copy_me || !copy_me->has_response_id())
1366     return false;
1367
1368   // Load HTTP headers for entry from newest cache.
1369   loading_responses_.insert(
1370       LoadingResponses::value_type(copy_me->response_id(), url));
1371   storage_->LoadResponseInfo(manifest_url_, group_->group_id(),
1372                              copy_me->response_id(),
1373                              this);
1374   // Async: wait for OnResponseInfoLoaded to complete.
1375   return true;
1376 }
1377
1378 void AppCacheUpdateJob::OnResponseInfoLoaded(
1379     AppCacheResponseInfo* response_info, int64 response_id) {
1380   const net::HttpResponseInfo* http_info = response_info ?
1381       response_info->http_response_info() : NULL;
1382
1383   // Needed response info for a manifest fetch request.
1384   if (internal_state_ == FETCH_MANIFEST) {
1385     if (http_info)
1386       manifest_fetcher_->set_existing_response_headers(
1387           http_info->headers.get());
1388     manifest_fetcher_->Start();
1389     return;
1390   }
1391
1392   LoadingResponses::iterator found = loading_responses_.find(response_id);
1393   DCHECK(found != loading_responses_.end());
1394   const GURL& url = found->second;
1395
1396   if (!http_info) {
1397     LoadFromNewestCacheFailed(url, NULL);  // no response found
1398   } else {
1399     // Check if response can be re-used according to HTTP caching semantics.
1400     // Responses with a "vary" header get treated as expired.
1401     const std::string name = "vary";
1402     std::string value;
1403     void* iter = NULL;
1404     if (!http_info->headers.get() ||
1405         http_info->headers->RequiresValidation(http_info->request_time,
1406                                                http_info->response_time,
1407                                                base::Time::Now()) ||
1408         http_info->headers->EnumerateHeader(&iter, name, &value)) {
1409       LoadFromNewestCacheFailed(url, response_info);
1410     } else {
1411       DCHECK(group_->newest_complete_cache());
1412       AppCacheEntry* copy_me = group_->newest_complete_cache()->GetEntry(url);
1413       DCHECK(copy_me);
1414       DCHECK(copy_me->response_id() == response_id);
1415
1416       AppCache::EntryMap::iterator it = url_file_list_.find(url);
1417       DCHECK(it != url_file_list_.end());
1418       AppCacheEntry& entry = it->second;
1419       entry.set_response_id(response_id);
1420       entry.set_response_size(copy_me->response_size());
1421       inprogress_cache_->AddOrModifyEntry(url, entry);
1422       NotifyAllProgress(url);
1423       ++url_fetches_completed_;
1424     }
1425   }
1426   loading_responses_.erase(found);
1427
1428   MaybeCompleteUpdate();
1429 }
1430
1431 void AppCacheUpdateJob::LoadFromNewestCacheFailed(
1432     const GURL& url, AppCacheResponseInfo* response_info) {
1433   if (internal_state_ == CACHE_FAILURE)
1434     return;
1435
1436   // Re-insert url at front of fetch list. Indicate storage has been checked.
1437   urls_to_fetch_.push_front(UrlToFetch(url, true, response_info));
1438   FetchUrls();
1439 }
1440
1441 void AppCacheUpdateJob::MaybeCompleteUpdate() {
1442   DCHECK(internal_state_ != CACHE_FAILURE);
1443
1444   // Must wait for any pending master entries or url fetches to complete.
1445   if (master_entries_completed_ != pending_master_entries_.size() ||
1446       url_fetches_completed_ != url_file_list_.size()) {
1447     DCHECK(internal_state_ != COMPLETED);
1448     return;
1449   }
1450
1451   switch (internal_state_) {
1452     case NO_UPDATE:
1453       if (master_entries_completed_ > 0) {
1454         switch (stored_state_) {
1455           case UNSTORED:
1456             StoreGroupAndCache();
1457             return;
1458           case STORING:
1459             return;
1460           case STORED:
1461             break;
1462         }
1463       }
1464       // 6.9.4 steps 7.3-7.7.
1465       NotifyAllAssociatedHosts(APPCACHE_NO_UPDATE_EVENT);
1466       DiscardDuplicateResponses();
1467       internal_state_ = COMPLETED;
1468       break;
1469     case DOWNLOADING:
1470       internal_state_ = REFETCH_MANIFEST;
1471       FetchManifest(false);
1472       break;
1473     case REFETCH_MANIFEST:
1474       DCHECK(stored_state_ == STORED);
1475       NotifyAllFinalProgress();
1476       if (update_type_ == CACHE_ATTEMPT)
1477         NotifyAllAssociatedHosts(APPCACHE_CACHED_EVENT);
1478       else
1479         NotifyAllAssociatedHosts(APPCACHE_UPDATE_READY_EVENT);
1480       DiscardDuplicateResponses();
1481       internal_state_ = COMPLETED;
1482       LogHistogramStats(UPDATE_OK, GURL());
1483       break;
1484     case CACHE_FAILURE:
1485       NOTREACHED();  // See HandleCacheFailure
1486       break;
1487     default:
1488       break;
1489   }
1490
1491   // Let the stack unwind before deletion to make it less risky as this
1492   // method is called from multiple places in this file.
1493   if (internal_state_ == COMPLETED)
1494     DeleteSoon();
1495 }
1496
1497 void AppCacheUpdateJob::ScheduleUpdateRetry(int delay_ms) {
1498   // TODO(jennb): post a delayed task with the "same parameters" as this job
1499   // to retry the update at a later time. Need group, URLs of pending master
1500   // entries and their hosts.
1501 }
1502
1503 void AppCacheUpdateJob::Cancel() {
1504   internal_state_ = CANCELLED;
1505
1506   LogHistogramStats(CANCELLED_ERROR, GURL());
1507
1508   if (manifest_fetcher_) {
1509     delete manifest_fetcher_;
1510     manifest_fetcher_ = NULL;
1511   }
1512
1513   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
1514        it != pending_url_fetches_.end(); ++it) {
1515     delete it->second;
1516   }
1517   pending_url_fetches_.clear();
1518
1519   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
1520        it != master_entry_fetches_.end(); ++it) {
1521     delete it->second;
1522   }
1523   master_entry_fetches_.clear();
1524
1525   ClearPendingMasterEntries();
1526   DiscardInprogressCache();
1527
1528   // Delete response writer to avoid any callbacks.
1529   if (manifest_response_writer_)
1530     manifest_response_writer_.reset();
1531
1532   storage_->CancelDelegateCallbacks(this);
1533 }
1534
1535 void AppCacheUpdateJob::ClearPendingMasterEntries() {
1536   for (PendingMasters::iterator it = pending_master_entries_.begin();
1537        it != pending_master_entries_.end(); ++it) {
1538     PendingHosts& hosts = it->second;
1539     for (PendingHosts::iterator host_it = hosts.begin();
1540          host_it != hosts.end(); ++host_it) {
1541       (*host_it)->RemoveObserver(this);
1542     }
1543   }
1544
1545   pending_master_entries_.clear();
1546 }
1547
1548 void AppCacheUpdateJob::DiscardInprogressCache() {
1549   if (stored_state_ == STORING) {
1550     // We can make no assumptions about whether the StoreGroupAndCacheTask
1551     // actually completed or not. This condition should only be reachable
1552     // during shutdown. Free things up and return to do no harm.
1553     inprogress_cache_ = NULL;
1554     added_master_entries_.clear();
1555     return;
1556   }
1557
1558   storage_->DoomResponses(manifest_url_, stored_response_ids_);
1559
1560   if (!inprogress_cache_.get()) {
1561     // We have to undo the changes we made, if any, to the existing cache.
1562     if (group_ && group_->newest_complete_cache()) {
1563       for (std::vector<GURL>::iterator iter = added_master_entries_.begin();
1564            iter != added_master_entries_.end(); ++iter) {
1565         group_->newest_complete_cache()->RemoveEntry(*iter);
1566       }
1567     }
1568     added_master_entries_.clear();
1569     return;
1570   }
1571
1572   AppCache::AppCacheHosts& hosts = inprogress_cache_->associated_hosts();
1573   while (!hosts.empty())
1574     (*hosts.begin())->AssociateNoCache(GURL());
1575
1576   inprogress_cache_ = NULL;
1577   added_master_entries_.clear();
1578 }
1579
1580 void AppCacheUpdateJob::DiscardDuplicateResponses() {
1581   storage_->DoomResponses(manifest_url_, duplicate_response_ids_);
1582 }
1583
1584 void AppCacheUpdateJob::LogHistogramStats(
1585       ResultType result, const GURL& failed_resource_url) {
1586   AppCacheHistograms::CountUpdateJobResult(result, manifest_url_.GetOrigin());
1587   if (result == UPDATE_OK)
1588     return;
1589
1590   int percent_complete = 0;
1591   if (url_file_list_.size() > 0) {
1592     size_t actual_fetches_completed = url_fetches_completed_;
1593     if (!failed_resource_url.is_empty() && actual_fetches_completed)
1594       --actual_fetches_completed;
1595     percent_complete = (static_cast<double>(actual_fetches_completed) /
1596                             static_cast<double>(url_file_list_.size())) * 100.0;
1597     percent_complete = std::min(percent_complete, 99);
1598   }
1599
1600   bool was_making_progress =
1601       base::Time::Now() - last_progress_time_ <
1602           base::TimeDelta::FromMinutes(5);
1603
1604   bool off_origin_resource_failure =
1605       !failed_resource_url.is_empty() &&
1606           (failed_resource_url.GetOrigin() != manifest_url_.GetOrigin());
1607
1608   AppCacheHistograms::LogUpdateFailureStats(
1609       manifest_url_.GetOrigin(),
1610       percent_complete,
1611       was_making_progress,
1612       off_origin_resource_failure);
1613 }
1614
1615 void AppCacheUpdateJob::DeleteSoon() {
1616   ClearPendingMasterEntries();
1617   manifest_response_writer_.reset();
1618   storage_->CancelDelegateCallbacks(this);
1619   service_->RemoveObserver(this);
1620   service_ = NULL;
1621
1622   // Break the connection with the group so the group cannot call delete
1623   // on this object after we've posted a task to delete ourselves.
1624   group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE);
1625   group_ = NULL;
1626
1627   base::MessageLoop::current()->DeleteSoon(FROM_HERE, this);
1628 }
1629
1630 }  // namespace content