webkit/browser/appcache/appcache_update_job.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "webkit/browser/appcache/appcache_update_job.h"
   6
   7 #include "base/bind.h"
   8 #include "base/bind_helpers.h"
   9 #include "base/compiler_specific.h"
  10 #include "base/message_loop/message_loop.h"
  11 #include "base/strings/string_util.h"
  12 #include "base/strings/stringprintf.h"
  13 #include "net/base/io_buffer.h"
  14 #include "net/base/load_flags.h"
  15 #include "net/base/net_errors.h"
  16 #include "net/http/http_request_headers.h"
  17 #include "net/http/http_response_headers.h"
  18 #include "net/url_request/url_request_context.h"
  19 #include "webkit/browser/appcache/appcache_group.h"
  20 #include "webkit/browser/appcache/appcache_histograms.h"
  21
  22 namespace appcache {
  23
  24 static const int kBufferSize = 32768;
  25 static const size_t kMaxConcurrentUrlFetches = 2;
  26 static const int kMax503Retries = 3;
  27
  28 // Helper class for collecting hosts per frontend when sending notifications
  29 // so that only one notification is sent for all hosts using the same frontend.
  30 class HostNotifier {
  31  public:
  32   typedef std::vector<int> HostIds;
  33   typedef std::map<AppCacheFrontend*, HostIds> NotifyHostMap;
  34
  35   // Caller is responsible for ensuring there will be no duplicate hosts.
  36   void AddHost(AppCacheHost* host) {
  37     std::pair<NotifyHostMap::iterator , bool> ret = hosts_to_notify.insert(
  38         NotifyHostMap::value_type(host->frontend(), HostIds()));
  39     ret.first->second.push_back(host->host_id());
  40   }
  41
  42   void AddHosts(const std::set<AppCacheHost*>& hosts) {
  43     for (std::set<AppCacheHost*>::const_iterator it = hosts.begin();
  44          it != hosts.end(); ++it) {
  45       AddHost(*it);
  46     }
  47   }
  48
  49   void SendNotifications(EventID event_id) {
  50     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  51          it != hosts_to_notify.end(); ++it) {
  52       AppCacheFrontend* frontend = it->first;
  53       frontend->OnEventRaised(it->second, event_id);
  54     }
  55   }
  56
  57   void SendProgressNotifications(
  58       const GURL& url, int num_total, int num_complete) {
  59     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  60          it != hosts_to_notify.end(); ++it) {
  61       AppCacheFrontend* frontend = it->first;
  62       frontend->OnProgressEventRaised(it->second, url,
  63                                       num_total, num_complete);
  64     }
  65   }
  66
  67   void SendErrorNotifications(const std::string& error_message) {
  68     DCHECK(!error_message.empty());
  69     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  70          it != hosts_to_notify.end(); ++it) {
  71       AppCacheFrontend* frontend = it->first;
  72       frontend->OnErrorEventRaised(it->second, error_message);
  73     }
  74   }
  75
  76  private:
  77   NotifyHostMap hosts_to_notify;
  78 };
  79
  80 AppCacheUpdateJob::UrlToFetch::UrlToFetch(const GURL& url,
  81                                           bool checked,
  82                                           AppCacheResponseInfo* info)
  83     : url(url),
  84       storage_checked(checked),
  85       existing_response_info(info) {
  86 }
  87
  88 AppCacheUpdateJob::UrlToFetch::~UrlToFetch() {
  89 }
  90
  91 // Helper class to fetch resources. Depending on the fetch type,
  92 // can either fetch to an in-memory string or write the response
  93 // data out to the disk cache.
  94 AppCacheUpdateJob::URLFetcher::URLFetcher(
  95     const GURL& url, FetchType fetch_type, AppCacheUpdateJob* job)
  96     : url_(url),
  97       job_(job),
  98       fetch_type_(fetch_type),
  99       retry_503_attempts_(0),
 100       buffer_(new net::IOBuffer(kBufferSize)),
 101       request_(job->service_->request_context()->CreateRequest(url, this)) {
 102 }
 103
 104 AppCacheUpdateJob::URLFetcher::~URLFetcher() {
 105 }
 106
 107 void AppCacheUpdateJob::URLFetcher::Start() {
 108   request_->set_first_party_for_cookies(job_->manifest_url_);
 109   request_->set_load_flags(request_->load_flags() |
 110                            net::LOAD_DISABLE_INTERCEPT);
 111   if (existing_response_headers_.get())
 112     AddConditionalHeaders(existing_response_headers_.get());
 113   request_->Start();
 114 }
 115
 116 void AppCacheUpdateJob::URLFetcher::OnReceivedRedirect(
 117     net::URLRequest* request, const GURL& new_url,  bool* defer_redirect) {
 118   DCHECK(request_ == request);
 119   // Redirect is not allowed by the update process.
 120   request->Cancel();
 121   OnResponseCompleted();
 122 }
 123
 124 void AppCacheUpdateJob::URLFetcher::OnResponseStarted(
 125     net::URLRequest *request) {
 126   DCHECK(request == request_);
 127   if (request->status().is_success() &&
 128       (request->GetResponseCode() / 100) == 2) {
 129
 130     // See http://code.google.com/p/chromium/issues/detail?id=69594
 131     // We willfully violate the HTML5 spec at this point in order
 132     // to support the appcaching of cross-origin HTTPS resources.
 133     // We've opted for a milder constraint and allow caching unless
 134     // the resource has a "no-store" header. A spec change has been
 135     // requested on the whatwg list.
 136     // TODO(michaeln): Consider doing this for cross-origin HTTP resources too.
 137     if (url_.SchemeIsSecure() &&
 138         url_.GetOrigin() != job_->manifest_url_.GetOrigin()) {
 139       if (request->response_headers()->
 140               HasHeaderValue("cache-control", "no-store")) {
 141         request->Cancel();
 142         OnResponseCompleted();
 143         return;
 144       }
 145     }
 146
 147     // Write response info to storage for URL fetches. Wait for async write
 148     // completion before reading any response data.
 149     if (fetch_type_ == URL_FETCH || fetch_type_ == MASTER_ENTRY_FETCH) {
 150       response_writer_.reset(job_->CreateResponseWriter());
 151       scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
 152           new HttpResponseInfoIOBuffer(
 153               new net::HttpResponseInfo(request->response_info())));
 154       response_writer_->WriteInfo(
 155           io_buffer.get(),
 156           base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
 157     } else {
 158       ReadResponseData();
 159     }
 160   } else {
 161     OnResponseCompleted();
 162   }
 163 }
 164
 165 void AppCacheUpdateJob::URLFetcher::OnReadCompleted(
 166     net::URLRequest* request, int bytes_read) {
 167   DCHECK(request_ == request);
 168   bool data_consumed = true;
 169   if (request->status().is_success() && bytes_read > 0) {
 170     data_consumed = ConsumeResponseData(bytes_read);
 171     if (data_consumed) {
 172       bytes_read = 0;
 173       while (request->Read(buffer_.get(), kBufferSize, &bytes_read)) {
 174         if (bytes_read > 0) {
 175           data_consumed = ConsumeResponseData(bytes_read);
 176           if (!data_consumed)
 177             break;  // wait for async data processing, then read more
 178         } else {
 179           break;
 180         }
 181       }
 182     }
 183   }
 184   if (data_consumed && !request->status().is_io_pending())
 185     OnResponseCompleted();
 186 }
 187
 188 void AppCacheUpdateJob::URLFetcher::AddConditionalHeaders(
 189     const net::HttpResponseHeaders* headers) {
 190   DCHECK(request_.get() && headers);
 191   net::HttpRequestHeaders extra_headers;
 192
 193   // Add If-Modified-Since header if response info has Last-Modified header.
 194   const std::string last_modified = "Last-Modified";
 195   std::string last_modified_value;
 196   headers->EnumerateHeader(NULL, last_modified, &last_modified_value);
 197   if (!last_modified_value.empty()) {
 198     extra_headers.SetHeader(net::HttpRequestHeaders::kIfModifiedSince,
 199                             last_modified_value);
 200   }
 201
 202   // Add If-None-Match header if response info has ETag header.
 203   const std::string etag = "ETag";
 204   std::string etag_value;
 205   headers->EnumerateHeader(NULL, etag, &etag_value);
 206   if (!etag_value.empty()) {
 207     extra_headers.SetHeader(net::HttpRequestHeaders::kIfNoneMatch,
 208                             etag_value);
 209   }
 210   if (!extra_headers.IsEmpty())
 211     request_->SetExtraRequestHeaders(extra_headers);
 212 }
 213
 214 void  AppCacheUpdateJob::URLFetcher::OnWriteComplete(int result) {
 215   if (result < 0) {
 216     request_->Cancel();
 217     OnResponseCompleted();
 218     return;
 219   }
 220   ReadResponseData();
 221 }
 222
 223 void AppCacheUpdateJob::URLFetcher::ReadResponseData() {
 224   InternalUpdateState state = job_->internal_state_;
 225   if (state == CACHE_FAILURE || state == CANCELLED || state == COMPLETED)
 226     return;
 227   int bytes_read = 0;
 228   request_->Read(buffer_.get(), kBufferSize, &bytes_read);
 229   OnReadCompleted(request_.get(), bytes_read);
 230 }
 231
 232 // Returns false if response data is processed asynchronously, in which
 233 // case ReadResponseData will be invoked when it is safe to continue
 234 // reading more response data from the request.
 235 bool AppCacheUpdateJob::URLFetcher::ConsumeResponseData(int bytes_read) {
 236   DCHECK_GT(bytes_read, 0);
 237   switch (fetch_type_) {
 238     case MANIFEST_FETCH:
 239     case MANIFEST_REFETCH:
 240       manifest_data_.append(buffer_->data(), bytes_read);
 241       break;
 242     case URL_FETCH:
 243     case MASTER_ENTRY_FETCH:
 244       DCHECK(response_writer_.get());
 245       response_writer_->WriteData(
 246           buffer_.get(),
 247           bytes_read,
 248           base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
 249       return false;  // wait for async write completion to continue reading
 250     default:
 251       NOTREACHED();
 252   }
 253   return true;
 254 }
 255
 256 void AppCacheUpdateJob::URLFetcher::OnResponseCompleted() {
 257   // Retry for 503s where retry-after is 0.
 258   if (request_->status().is_success() &&
 259       request_->GetResponseCode() == 503 &&
 260       MaybeRetryRequest()) {
 261     return;
 262   }
 263
 264   switch (fetch_type_) {
 265     case MANIFEST_FETCH:
 266       job_->HandleManifestFetchCompleted(this);
 267       break;
 268     case URL_FETCH:
 269       job_->HandleUrlFetchCompleted(this);
 270       break;
 271     case MASTER_ENTRY_FETCH:
 272       job_->HandleMasterEntryFetchCompleted(this);
 273       break;
 274     case MANIFEST_REFETCH:
 275       job_->HandleManifestRefetchCompleted(this);
 276       break;
 277     default:
 278       NOTREACHED();
 279   }
 280
 281   delete this;
 282 }
 283
 284 bool AppCacheUpdateJob::URLFetcher::MaybeRetryRequest() {
 285   if (retry_503_attempts_ >= kMax503Retries ||
 286       !request_->response_headers()->HasHeaderValue("retry-after", "0")) {
 287     return false;
 288   }
 289   ++retry_503_attempts_;
 290   request_.reset(job_->service_->request_context()->CreateRequest(url_, this));
 291   Start();
 292   return true;
 293 }
 294
 295 AppCacheUpdateJob::AppCacheUpdateJob(AppCacheService* service,
 296                                      AppCacheGroup* group)
 297     : service_(service),
 298       manifest_url_(group->manifest_url()),
 299       group_(group),
 300       update_type_(UNKNOWN_TYPE),
 301       internal_state_(FETCH_MANIFEST),
 302       master_entries_completed_(0),
 303       url_fetches_completed_(0),
 304       manifest_fetcher_(NULL),
 305       stored_state_(UNSTORED) {
 306 }
 307
 308 AppCacheUpdateJob::~AppCacheUpdateJob() {
 309   if (internal_state_ != COMPLETED)
 310     Cancel();
 311
 312   DCHECK(!manifest_fetcher_);
 313   DCHECK(pending_url_fetches_.empty());
 314   DCHECK(!inprogress_cache_.get());
 315   DCHECK(pending_master_entries_.empty());
 316   DCHECK(master_entry_fetches_.empty());
 317
 318   if (group_)
 319     group_->SetUpdateStatus(AppCacheGroup::IDLE);
 320 }
 321
 322 void AppCacheUpdateJob::StartUpdate(AppCacheHost* host,
 323                                     const GURL& new_master_resource) {
 324   DCHECK(group_->update_job() == this);
 325   DCHECK(!group_->is_obsolete());
 326
 327   bool is_new_pending_master_entry = false;
 328   if (!new_master_resource.is_empty()) {
 329     DCHECK(new_master_resource == host->pending_master_entry_url());
 330     DCHECK(!new_master_resource.has_ref());
 331     DCHECK(new_master_resource.GetOrigin() == manifest_url_.GetOrigin());
 332
 333     // Cannot add more to this update if already terminating.
 334     if (IsTerminating()) {
 335       group_->QueueUpdate(host, new_master_resource);
 336       return;
 337     }
 338
 339     std::pair<PendingMasters::iterator, bool> ret =
 340         pending_master_entries_.insert(
 341             PendingMasters::value_type(new_master_resource, PendingHosts()));
 342     is_new_pending_master_entry = ret.second;
 343     ret.first->second.push_back(host);
 344     host->AddObserver(this);
 345   }
 346
 347   // Notify host (if any) if already checking or downloading.
 348   AppCacheGroup::UpdateStatus update_status = group_->update_status();
 349   if (update_status == AppCacheGroup::CHECKING ||
 350       update_status == AppCacheGroup::DOWNLOADING) {
 351     if (host) {
 352       NotifySingleHost(host, CHECKING_EVENT);
 353       if (update_status == AppCacheGroup::DOWNLOADING)
 354         NotifySingleHost(host, DOWNLOADING_EVENT);
 355
 356       // Add to fetch list or an existing entry if already fetched.
 357       if (!new_master_resource.is_empty()) {
 358         AddMasterEntryToFetchList(host, new_master_resource,
 359                                   is_new_pending_master_entry);
 360       }
 361     }
 362     return;
 363   }
 364
 365   // Begin update process for the group.
 366   group_->SetUpdateStatus(AppCacheGroup::CHECKING);
 367   if (group_->HasCache()) {
 368     update_type_ = UPGRADE_ATTEMPT;
 369     NotifyAllAssociatedHosts(CHECKING_EVENT);
 370   } else {
 371     update_type_ = CACHE_ATTEMPT;
 372     DCHECK(host);
 373     NotifySingleHost(host, CHECKING_EVENT);
 374   }
 375
 376   if (!new_master_resource.is_empty()) {
 377     AddMasterEntryToFetchList(host, new_master_resource,
 378                               is_new_pending_master_entry);
 379   }
 380
 381   FetchManifest(true);
 382 }
 383
 384 AppCacheResponseWriter* AppCacheUpdateJob::CreateResponseWriter() {
 385   AppCacheResponseWriter* writer =
 386       service_->storage()->CreateResponseWriter(manifest_url_,
 387                                                 group_->group_id());
 388   stored_response_ids_.push_back(writer->response_id());
 389   return writer;
 390 }
 391
 392 void AppCacheUpdateJob::HandleCacheFailure(const std::string& error_message) {
 393   // 6.9.4 cache failure steps 2-8.
 394   DCHECK(internal_state_ != CACHE_FAILURE);
 395   DCHECK(!error_message.empty());
 396   internal_state_ = CACHE_FAILURE;
 397   CancelAllUrlFetches();
 398   CancelAllMasterEntryFetches(error_message);
 399   NotifyAllError(error_message);
 400   DiscardInprogressCache();
 401   internal_state_ = COMPLETED;
 402   DeleteSoon();  // To unwind the stack prior to deletion.
 403 }
 404
 405 void AppCacheUpdateJob::FetchManifest(bool is_first_fetch) {
 406   DCHECK(!manifest_fetcher_);
 407   manifest_fetcher_ = new URLFetcher(
 408      manifest_url_,
 409      is_first_fetch ? URLFetcher::MANIFEST_FETCH :
 410                       URLFetcher::MANIFEST_REFETCH,
 411      this);
 412
 413   // Add any necessary Http headers before sending fetch request.
 414   if (is_first_fetch) {
 415     AppCacheEntry* entry = (update_type_ == UPGRADE_ATTEMPT) ?
 416         group_->newest_complete_cache()->GetEntry(manifest_url_) : NULL;
 417     if (entry) {
 418       // Asynchronously load response info for manifest from newest cache.
 419       service_->storage()->LoadResponseInfo(manifest_url_, group_->group_id(),
 420                                             entry->response_id(), this);
 421     } else {
 422       manifest_fetcher_->Start();
 423     }
 424   } else {
 425     DCHECK(internal_state_ == REFETCH_MANIFEST);
 426     DCHECK(manifest_response_info_.get());
 427     manifest_fetcher_->set_existing_response_headers(
 428         manifest_response_info_->headers.get());
 429     manifest_fetcher_->Start();
 430   }
 431 }
 432
 433
 434 void AppCacheUpdateJob::HandleManifestFetchCompleted(
 435     URLFetcher* fetcher) {
 436   DCHECK_EQ(internal_state_, FETCH_MANIFEST);
 437   DCHECK_EQ(manifest_fetcher_, fetcher);
 438   manifest_fetcher_ = NULL;
 439
 440   net::URLRequest* request = fetcher->request();
 441   int response_code = -1;
 442   bool is_valid_response_code = false;
 443   if (request->status().is_success()) {
 444     response_code = request->GetResponseCode();
 445     is_valid_response_code = (response_code / 100 == 2);
 446   }
 447
 448   if (is_valid_response_code) {
 449     manifest_data_ = fetcher->manifest_data();
 450     manifest_response_info_.reset(
 451         new net::HttpResponseInfo(request->response_info()));
 452     if (update_type_ == UPGRADE_ATTEMPT)
 453       CheckIfManifestChanged();  // continues asynchronously
 454     else
 455       ContinueHandleManifestFetchCompleted(true);
 456   } else if (response_code == 304 && update_type_ == UPGRADE_ATTEMPT) {
 457     ContinueHandleManifestFetchCompleted(false);
 458   } else if ((response_code == 404 || response_code == 410) &&
 459              update_type_ == UPGRADE_ATTEMPT) {
 460     service_->storage()->MakeGroupObsolete(group_, this);  // async
 461   } else {
 462     const char* kFormatString = "Manifest fetch failed (%d) %s";
 463     std::string message = base::StringPrintf(kFormatString, response_code,
 464                                              manifest_url_.spec().c_str());
 465     HandleCacheFailure(message);
 466   }
 467 }
 468
 469 void AppCacheUpdateJob::OnGroupMadeObsolete(AppCacheGroup* group,
 470                                             bool success) {
 471   DCHECK(master_entry_fetches_.empty());
 472   CancelAllMasterEntryFetches("The cache has been made obsolete, "
 473                               "the manifest file returned 404 or 410");
 474   if (success) {
 475     DCHECK(group->is_obsolete());
 476     NotifyAllAssociatedHosts(OBSOLETE_EVENT);
 477     internal_state_ = COMPLETED;
 478     MaybeCompleteUpdate();
 479   } else {
 480     // Treat failure to mark group obsolete as a cache failure.
 481     HandleCacheFailure("Failed to mark the cache as obsolete");
 482   }
 483 }
 484
 485 void AppCacheUpdateJob::ContinueHandleManifestFetchCompleted(bool changed) {
 486   DCHECK(internal_state_ == FETCH_MANIFEST);
 487
 488   if (!changed) {
 489     DCHECK(update_type_ == UPGRADE_ATTEMPT);
 490     internal_state_ = NO_UPDATE;
 491
 492     // Wait for pending master entries to download.
 493     FetchMasterEntries();
 494     MaybeCompleteUpdate();  // if not done, run async 6.9.4 step 7 substeps
 495     return;
 496   }
 497
 498   Manifest manifest;
 499   if (!ParseManifest(manifest_url_, manifest_data_.data(),
 500                      manifest_data_.length(), manifest)) {
 501     const char* kFormatString = "Failed to parse manifest %s";
 502     const std::string message = base::StringPrintf(kFormatString,
 503         manifest_url_.spec().c_str());
 504     HandleCacheFailure(message);
 505     VLOG(1) << message;
 506     return;
 507   }
 508
 509   // Proceed with update process. Section 6.9.4 steps 8-20.
 510   internal_state_ = DOWNLOADING;
 511   inprogress_cache_ = new AppCache(service_->storage(),
 512                                    service_->storage()->NewCacheId());
 513   BuildUrlFileList(manifest);
 514   inprogress_cache_->InitializeWithManifest(&manifest);
 515
 516   // Associate all pending master hosts with the newly created cache.
 517   for (PendingMasters::iterator it = pending_master_entries_.begin();
 518        it != pending_master_entries_.end(); ++it) {
 519     PendingHosts& hosts = it->second;
 520     for (PendingHosts::iterator host_it = hosts.begin();
 521          host_it != hosts.end(); ++host_it) {
 522       (*host_it)
 523           ->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
 524     }
 525   }
 526
 527   group_->SetUpdateStatus(AppCacheGroup::DOWNLOADING);
 528   NotifyAllAssociatedHosts(DOWNLOADING_EVENT);
 529   FetchUrls();
 530   FetchMasterEntries();
 531   MaybeCompleteUpdate();  // if not done, continues when async fetches complete
 532 }
 533
 534 void AppCacheUpdateJob::HandleUrlFetchCompleted(URLFetcher* fetcher) {
 535   DCHECK(internal_state_ == DOWNLOADING);
 536
 537   net::URLRequest* request = fetcher->request();
 538   const GURL& url = request->original_url();
 539   pending_url_fetches_.erase(url);
 540   NotifyAllProgress(url);
 541   ++url_fetches_completed_;
 542
 543   int response_code = request->status().is_success()
 544       ? request->GetResponseCode() : -1;
 545   AppCacheEntry& entry = url_file_list_.find(url)->second;
 546
 547   if (response_code / 100 == 2) {
 548     // Associate storage with the new entry.
 549     DCHECK(fetcher->response_writer());
 550     entry.set_response_id(fetcher->response_writer()->response_id());
 551     entry.set_response_size(fetcher->response_writer()->amount_written());
 552     if (!inprogress_cache_->AddOrModifyEntry(url, entry))
 553       duplicate_response_ids_.push_back(entry.response_id());
 554
 555     // TODO(michaeln): Check for <html manifest=xxx>
 556     // See http://code.google.com/p/chromium/issues/detail?id=97930
 557     // if (entry.IsMaster() && !(entry.IsExplicit() || fallback || intercept))
 558     //   if (!manifestAttribute) skip it
 559
 560     // Foreign entries will be detected during cache selection.
 561     // Note: 6.9.4, step 17.9 possible optimization: if resource is HTML or XML
 562     // file whose root element is an html element with a manifest attribute
 563     // whose value doesn't match the manifest url of the application cache
 564     // being processed, mark the entry as being foreign.
 565   } else {
 566     VLOG(1) << "Request status: " << request->status().status()
 567             << " error: " << request->status().error()
 568             << " response code: " << response_code;
 569     if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept()) {
 570       if (response_code == 304 && fetcher->existing_entry().has_response_id()) {
 571         // Keep the existing response.
 572         entry.set_response_id(fetcher->existing_entry().response_id());
 573         entry.set_response_size(fetcher->existing_entry().response_size());
 574         inprogress_cache_->AddOrModifyEntry(url, entry);
 575       } else {
 576         const char* kFormatString = "Resource fetch failed (%d) %s";
 577         const std::string message = base::StringPrintf(kFormatString,
 578             response_code, url.spec().c_str());
 579         HandleCacheFailure(message);
 580         return;
 581       }
 582     } else if (response_code == 404 || response_code == 410) {
 583       // Entry is skipped.  They are dropped from the cache.
 584     } else if (update_type_ == UPGRADE_ATTEMPT &&
 585                fetcher->existing_entry().has_response_id()) {
 586       // Keep the existing response.
 587       // TODO(michaeln): Not sure this is a good idea. This is spec compliant
 588       // but the old resource may or may not be compatible with the new contents
 589       // of the cache. Impossible to know one way or the other.
 590       entry.set_response_id(fetcher->existing_entry().response_id());
 591       entry.set_response_size(fetcher->existing_entry().response_size());
 592       inprogress_cache_->AddOrModifyEntry(url, entry);
 593     }
 594   }
 595
 596   // Fetch another URL now that one request has completed.
 597   DCHECK(internal_state_ != CACHE_FAILURE);
 598   FetchUrls();
 599   MaybeCompleteUpdate();
 600 }
 601
 602 void AppCacheUpdateJob::HandleMasterEntryFetchCompleted(
 603     URLFetcher* fetcher) {
 604   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
 605
 606   // TODO(jennb): Handle downloads completing during cache failure when update
 607   // no longer fetches master entries directly. For now, we cancel all pending
 608   // master entry fetches when entering cache failure state so this will never
 609   // be called in CACHE_FAILURE state.
 610
 611   net::URLRequest* request = fetcher->request();
 612   const GURL& url = request->original_url();
 613   master_entry_fetches_.erase(url);
 614   ++master_entries_completed_;
 615
 616   int response_code = request->status().is_success()
 617       ? request->GetResponseCode() : -1;
 618
 619   PendingMasters::iterator found = pending_master_entries_.find(url);
 620   DCHECK(found != pending_master_entries_.end());
 621   PendingHosts& hosts = found->second;
 622
 623   // Section 6.9.4. No update case: step 7.3, else step 22.
 624   if (response_code / 100 == 2) {
 625     // Add fetched master entry to the appropriate cache.
 626     AppCache* cache = inprogress_cache_.get() ? inprogress_cache_.get()
 627                                               : group_->newest_complete_cache();
 628     DCHECK(fetcher->response_writer());
 629     AppCacheEntry master_entry(AppCacheEntry::MASTER,
 630                                fetcher->response_writer()->response_id(),
 631                                fetcher->response_writer()->amount_written());
 632     if (cache->AddOrModifyEntry(url, master_entry))
 633       added_master_entries_.push_back(url);
 634     else
 635       duplicate_response_ids_.push_back(master_entry.response_id());
 636
 637     // In no-update case, associate host with the newest cache.
 638     if (!inprogress_cache_.get()) {
 639       // TODO(michaeln): defer until the updated cache has been stored
 640       DCHECK(cache == group_->newest_complete_cache());
 641       for (PendingHosts::iterator host_it = hosts.begin();
 642            host_it != hosts.end(); ++host_it) {
 643         (*host_it)->AssociateCompleteCache(cache);
 644       }
 645     }
 646   } else {
 647     HostNotifier host_notifier;
 648     for (PendingHosts::iterator host_it = hosts.begin();
 649          host_it != hosts.end(); ++host_it) {
 650       AppCacheHost* host = *host_it;
 651       host_notifier.AddHost(host);
 652
 653       // In downloading case, disassociate host from inprogress cache.
 654       if (inprogress_cache_.get())
 655         host->AssociateNoCache(GURL());
 656
 657       host->RemoveObserver(this);
 658     }
 659     hosts.clear();
 660
 661     const char* kFormatString = "Master entry fetch failed (%d) %s";
 662     const std::string message = base::StringPrintf(kFormatString,
 663         response_code, request->url().spec().c_str());
 664     host_notifier.SendErrorNotifications(message);
 665
 666     // In downloading case, update result is different if all master entries
 667     // failed vs. only some failing.
 668     if (inprogress_cache_.get()) {
 669       // Only count successful downloads to know if all master entries failed.
 670       pending_master_entries_.erase(found);
 671       --master_entries_completed_;
 672
 673       // Section 6.9.4, step 22.3.
 674       if (update_type_ == CACHE_ATTEMPT && pending_master_entries_.empty()) {
 675         HandleCacheFailure(message);
 676         return;
 677       }
 678     }
 679   }
 680
 681   DCHECK(internal_state_ != CACHE_FAILURE);
 682   FetchMasterEntries();
 683   MaybeCompleteUpdate();
 684 }
 685
 686 void AppCacheUpdateJob::HandleManifestRefetchCompleted(
 687     URLFetcher* fetcher) {
 688   DCHECK(internal_state_ == REFETCH_MANIFEST);
 689   DCHECK(manifest_fetcher_ == fetcher);
 690   manifest_fetcher_ = NULL;
 691
 692   net::URLRequest* request = fetcher->request();
 693   int response_code = request->status().is_success()
 694       ? request->GetResponseCode() : -1;
 695   if (response_code == 304 || manifest_data_ == fetcher->manifest_data()) {
 696     // Only need to store response in storage if manifest is not already
 697     // an entry in the cache.
 698     AppCacheEntry* entry = inprogress_cache_->GetEntry(manifest_url_);
 699     if (entry) {
 700       entry->add_types(AppCacheEntry::MANIFEST);
 701       StoreGroupAndCache();
 702     } else {
 703       manifest_response_writer_.reset(CreateResponseWriter());
 704       scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
 705           new HttpResponseInfoIOBuffer(manifest_response_info_.release()));
 706       manifest_response_writer_->WriteInfo(
 707           io_buffer.get(),
 708           base::Bind(&AppCacheUpdateJob::OnManifestInfoWriteComplete,
 709                      base::Unretained(this)));
 710     }
 711   } else {
 712     VLOG(1) << "Request status: " << request->status().status()
 713             << " error: " << request->status().error()
 714             << " response code: " << response_code;
 715     ScheduleUpdateRetry(kRerunDelayMs);
 716     HandleCacheFailure("Manifest changed during update, scheduling retry");
 717   }
 718 }
 719
 720 void AppCacheUpdateJob::OnManifestInfoWriteComplete(int result) {
 721   if (result > 0) {
 722     scoped_refptr<net::StringIOBuffer> io_buffer(
 723         new net::StringIOBuffer(manifest_data_));
 724     manifest_response_writer_->WriteData(
 725         io_buffer.get(),
 726         manifest_data_.length(),
 727         base::Bind(&AppCacheUpdateJob::OnManifestDataWriteComplete,
 728                    base::Unretained(this)));
 729   } else {
 730     HandleCacheFailure("Failed to write the manifest headers to storage");
 731   }
 732 }
 733
 734 void AppCacheUpdateJob::OnManifestDataWriteComplete(int result) {
 735   if (result > 0) {
 736     AppCacheEntry entry(AppCacheEntry::MANIFEST,
 737         manifest_response_writer_->response_id(),
 738         manifest_response_writer_->amount_written());
 739     if (!inprogress_cache_->AddOrModifyEntry(manifest_url_, entry))
 740       duplicate_response_ids_.push_back(entry.response_id());
 741     StoreGroupAndCache();
 742   } else {
 743     HandleCacheFailure("Failed to write the manifest data to storage");
 744   }
 745 }
 746
 747 void AppCacheUpdateJob::StoreGroupAndCache() {
 748   DCHECK(stored_state_ == UNSTORED);
 749   stored_state_ = STORING;
 750   scoped_refptr<AppCache> newest_cache;
 751   if (inprogress_cache_.get())
 752     newest_cache.swap(inprogress_cache_);
 753   else
 754     newest_cache = group_->newest_complete_cache();
 755   newest_cache->set_update_time(base::Time::Now());
 756
 757   // TODO(michaeln): dcheck is fishing for clues to crbug/95101
 758   DCHECK_EQ(manifest_url_, group_->manifest_url());
 759   service_->storage()
 760       ->StoreGroupAndNewestCache(group_, newest_cache.get(), this);  // async
 761 }
 762
 763 void AppCacheUpdateJob::OnGroupAndNewestCacheStored(AppCacheGroup* group,
 764                                                     AppCache* newest_cache,
 765                                                     bool success,
 766                                                     bool would_exceed_quota) {
 767   DCHECK(stored_state_ == STORING);
 768   if (success) {
 769     stored_state_ = STORED;
 770     MaybeCompleteUpdate();  // will definitely complete
 771   } else {
 772     // Restore inprogress_cache_ to get the proper events delivered
 773     // and the proper cleanup to occur.
 774     if (newest_cache != group->newest_complete_cache())
 775       inprogress_cache_ = newest_cache;
 776
 777     std::string message("Failed to commit new cache to storage");
 778     if (would_exceed_quota)
 779       message.append(", would exceed quota");
 780     HandleCacheFailure(message);
 781   }
 782 }
 783
 784 void AppCacheUpdateJob::NotifySingleHost(AppCacheHost* host,
 785                                          EventID event_id) {
 786   std::vector<int> ids(1, host->host_id());
 787   host->frontend()->OnEventRaised(ids, event_id);
 788 }
 789
 790 void AppCacheUpdateJob::NotifyAllAssociatedHosts(EventID event_id) {
 791   HostNotifier host_notifier;
 792   AddAllAssociatedHostsToNotifier(&host_notifier);
 793   host_notifier.SendNotifications(event_id);
 794 }
 795
 796 void AppCacheUpdateJob::NotifyAllProgress(const GURL& url) {
 797   HostNotifier host_notifier;
 798   AddAllAssociatedHostsToNotifier(&host_notifier);
 799   host_notifier.SendProgressNotifications(
 800       url, url_file_list_.size(), url_fetches_completed_);
 801 }
 802
 803 void AppCacheUpdateJob::NotifyAllFinalProgress() {
 804   DCHECK(url_file_list_.size() == url_fetches_completed_);
 805   NotifyAllProgress(GURL());
 806 }
 807
 808 void AppCacheUpdateJob::NotifyAllError(const std::string& error_message) {
 809   HostNotifier host_notifier;
 810   AddAllAssociatedHostsToNotifier(&host_notifier);
 811   host_notifier.SendErrorNotifications(error_message);
 812 }
 813
 814 void AppCacheUpdateJob::AddAllAssociatedHostsToNotifier(
 815     HostNotifier* host_notifier) {
 816   // Collect hosts so we only send one notification per frontend.
 817   // A host can only be associated with a single cache so no need to worry
 818   // about duplicate hosts being added to the notifier.
 819   if (inprogress_cache_.get()) {
 820     DCHECK(internal_state_ == DOWNLOADING || internal_state_ == CACHE_FAILURE);
 821     host_notifier->AddHosts(inprogress_cache_->associated_hosts());
 822   }
 823
 824   AppCacheGroup::Caches old_caches = group_->old_caches();
 825   for (AppCacheGroup::Caches::const_iterator it = old_caches.begin();
 826        it != old_caches.end(); ++it) {
 827     host_notifier->AddHosts((*it)->associated_hosts());
 828   }
 829
 830   AppCache* newest_cache = group_->newest_complete_cache();
 831   if (newest_cache)
 832     host_notifier->AddHosts(newest_cache->associated_hosts());
 833 }
 834
 835 void AppCacheUpdateJob::OnDestructionImminent(AppCacheHost* host) {
 836   // The host is about to be deleted; remove from our collection.
 837   PendingMasters::iterator found =
 838       pending_master_entries_.find(host->pending_master_entry_url());
 839   DCHECK(found != pending_master_entries_.end());
 840   PendingHosts& hosts = found->second;
 841   PendingHosts::iterator it = std::find(hosts.begin(), hosts.end(), host);
 842   DCHECK(it != hosts.end());
 843   hosts.erase(it);
 844 }
 845
 846 void AppCacheUpdateJob::CheckIfManifestChanged() {
 847   DCHECK(update_type_ == UPGRADE_ATTEMPT);
 848   AppCacheEntry* entry =
 849       group_->newest_complete_cache()->GetEntry(manifest_url_);
 850   if (!entry) {
 851     // TODO(michaeln): This is just a bandaid to avoid a crash.
 852     // http://code.google.com/p/chromium/issues/detail?id=95101
 853     HandleCacheFailure("Manifest entry not found in existing cache");
 854     AppCacheHistograms::AddMissingManifestEntrySample();
 855     service_->DeleteAppCacheGroup(manifest_url_, net::CompletionCallback());
 856     return;
 857   }
 858
 859   // Load manifest data from storage to compare against fetched manifest.
 860   manifest_response_reader_.reset(
 861       service_->storage()->CreateResponseReader(manifest_url_,
 862                                                 group_->group_id(),
 863                                                 entry->response_id()));
 864   read_manifest_buffer_ = new net::IOBuffer(kBufferSize);
 865   manifest_response_reader_->ReadData(
 866       read_manifest_buffer_.get(),
 867       kBufferSize,
 868       base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
 869                  base::Unretained(this)));  // async read
 870 }
 871
 872 void AppCacheUpdateJob::OnManifestDataReadComplete(int result) {
 873   if (result > 0) {
 874     loaded_manifest_data_.append(read_manifest_buffer_->data(), result);
 875     manifest_response_reader_->ReadData(
 876         read_manifest_buffer_.get(),
 877         kBufferSize,
 878         base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
 879                    base::Unretained(this)));  // read more
 880   } else {
 881     read_manifest_buffer_ = NULL;
 882     manifest_response_reader_.reset();
 883     ContinueHandleManifestFetchCompleted(
 884         result < 0 || manifest_data_ != loaded_manifest_data_);
 885   }
 886 }
 887
 888 void AppCacheUpdateJob::BuildUrlFileList(const Manifest& manifest) {
 889   for (base::hash_set<std::string>::const_iterator it =
 890            manifest.explicit_urls.begin();
 891        it != manifest.explicit_urls.end(); ++it) {
 892     AddUrlToFileList(GURL(*it), AppCacheEntry::EXPLICIT);
 893   }
 894
 895   const std::vector<Namespace>& intercepts =
 896       manifest.intercept_namespaces;
 897   for (std::vector<Namespace>::const_iterator it = intercepts.begin();
 898        it != intercepts.end(); ++it) {
 899     int flags = AppCacheEntry::INTERCEPT;
 900     if (it->is_executable)
 901       flags |= AppCacheEntry::EXECUTABLE;
 902     AddUrlToFileList(it->target_url, flags);
 903   }
 904
 905   const std::vector<Namespace>& fallbacks =
 906       manifest.fallback_namespaces;
 907   for (std::vector<Namespace>::const_iterator it = fallbacks.begin();
 908        it != fallbacks.end(); ++it) {
 909      AddUrlToFileList(it->target_url, AppCacheEntry::FALLBACK);
 910   }
 911
 912   // Add all master entries from newest complete cache.
 913   if (update_type_ == UPGRADE_ATTEMPT) {
 914     const AppCache::EntryMap& entries =
 915         group_->newest_complete_cache()->entries();
 916     for (AppCache::EntryMap::const_iterator it = entries.begin();
 917          it != entries.end(); ++it) {
 918       const AppCacheEntry& entry = it->second;
 919       if (entry.IsMaster())
 920         AddUrlToFileList(it->first, AppCacheEntry::MASTER);
 921     }
 922   }
 923 }
 924
 925 void AppCacheUpdateJob::AddUrlToFileList(const GURL& url, int type) {
 926   std::pair<AppCache::EntryMap::iterator, bool> ret = url_file_list_.insert(
 927       AppCache::EntryMap::value_type(url, AppCacheEntry(type)));
 928
 929   if (ret.second)
 930     urls_to_fetch_.push_back(UrlToFetch(url, false, NULL));
 931   else
 932     ret.first->second.add_types(type);  // URL already exists. Merge types.
 933 }
 934
 935 void AppCacheUpdateJob::FetchUrls() {
 936   DCHECK(internal_state_ == DOWNLOADING);
 937
 938   // Fetch each URL in the list according to section 6.9.4 step 17.1-17.3.
 939   // Fetch up to the concurrent limit. Other fetches will be triggered as each
 940   // each fetch completes.
 941   while (pending_url_fetches_.size() < kMaxConcurrentUrlFetches &&
 942          !urls_to_fetch_.empty()) {
 943     UrlToFetch url_to_fetch = urls_to_fetch_.front();
 944     urls_to_fetch_.pop_front();
 945
 946     AppCache::EntryMap::iterator it = url_file_list_.find(url_to_fetch.url);
 947     DCHECK(it != url_file_list_.end());
 948     AppCacheEntry& entry = it->second;
 949     if (ShouldSkipUrlFetch(entry)) {
 950       NotifyAllProgress(url_to_fetch.url);
 951       ++url_fetches_completed_;
 952     } else if (AlreadyFetchedEntry(url_to_fetch.url, entry.types())) {
 953       NotifyAllProgress(url_to_fetch.url);
 954       ++url_fetches_completed_;  // saved a URL request
 955     } else if (!url_to_fetch.storage_checked &&
 956                MaybeLoadFromNewestCache(url_to_fetch.url, entry)) {
 957       // Continues asynchronously after data is loaded from newest cache.
 958     } else {
 959       URLFetcher* fetcher = new URLFetcher(
 960           url_to_fetch.url, URLFetcher::URL_FETCH, this);
 961       if (url_to_fetch.existing_response_info.get()) {
 962         DCHECK(group_->newest_complete_cache());
 963         AppCacheEntry* existing_entry =
 964             group_->newest_complete_cache()->GetEntry(url_to_fetch.url);
 965         DCHECK(existing_entry);
 966         DCHECK(existing_entry->response_id() ==
 967                url_to_fetch.existing_response_info->response_id());
 968         fetcher->set_existing_response_headers(
 969             url_to_fetch.existing_response_info->http_response_info()->headers
 970                 .get());
 971         fetcher->set_existing_entry(*existing_entry);
 972       }
 973       fetcher->Start();
 974       pending_url_fetches_.insert(
 975           PendingUrlFetches::value_type(url_to_fetch.url, fetcher));
 976     }
 977   }
 978 }
 979
 980 void AppCacheUpdateJob::CancelAllUrlFetches() {
 981   // Cancel any pending URL requests.
 982   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
 983        it != pending_url_fetches_.end(); ++it) {
 984     delete it->second;
 985   }
 986
 987   url_fetches_completed_ +=
 988       pending_url_fetches_.size() + urls_to_fetch_.size();
 989   pending_url_fetches_.clear();
 990   urls_to_fetch_.clear();
 991 }
 992
 993 bool AppCacheUpdateJob::ShouldSkipUrlFetch(const AppCacheEntry& entry) {
 994   // 6.6.4 Step 17
 995   // If the resource URL being processed was flagged as neither an
 996   // "explicit entry" nor or a "fallback entry", then the user agent
 997   // may skip this URL.
 998   if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept())
 999     return false;
1000
1001   // TODO(jennb): decide if entry should be skipped to expire it from cache
1002   return false;
1003 }
1004
1005 bool AppCacheUpdateJob::AlreadyFetchedEntry(const GURL& url,
1006                                             int entry_type) {
1007   DCHECK(internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE);
1008   AppCacheEntry* existing =
1009       inprogress_cache_.get() ? inprogress_cache_->GetEntry(url)
1010                               : group_->newest_complete_cache()->GetEntry(url);
1011   if (existing) {
1012     existing->add_types(entry_type);
1013     return true;
1014   }
1015   return false;
1016 }
1017
1018 void AppCacheUpdateJob::AddMasterEntryToFetchList(AppCacheHost* host,
1019                                                   const GURL& url,
1020                                                   bool is_new) {
1021   DCHECK(!IsTerminating());
1022
1023   if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE) {
1024     AppCache* cache;
1025     if (inprogress_cache_.get()) {
1026       // always associate
1027       host->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
1028       cache = inprogress_cache_.get();
1029     } else {
1030       cache = group_->newest_complete_cache();
1031     }
1032
1033     // Update existing entry if it has already been fetched.
1034     AppCacheEntry* entry = cache->GetEntry(url);
1035     if (entry) {
1036       entry->add_types(AppCacheEntry::MASTER);
1037       if (internal_state_ == NO_UPDATE && !inprogress_cache_.get()) {
1038         // only associate if have entry
1039         host->AssociateCompleteCache(cache);
1040       }
1041       if (is_new)
1042         ++master_entries_completed_;  // pretend fetching completed
1043       return;
1044     }
1045   }
1046
1047   // Add to fetch list if not already fetching.
1048   if (master_entry_fetches_.find(url) == master_entry_fetches_.end()) {
1049     master_entries_to_fetch_.insert(url);
1050     if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE)
1051       FetchMasterEntries();
1052   }
1053 }
1054
1055 void AppCacheUpdateJob::FetchMasterEntries() {
1056   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
1057
1058   // Fetch each master entry in the list, up to the concurrent limit.
1059   // Additional fetches will be triggered as each fetch completes.
1060   while (master_entry_fetches_.size() < kMaxConcurrentUrlFetches &&
1061          !master_entries_to_fetch_.empty()) {
1062     const GURL& url = *master_entries_to_fetch_.begin();
1063
1064     if (AlreadyFetchedEntry(url, AppCacheEntry::MASTER)) {
1065       ++master_entries_completed_;  // saved a URL request
1066
1067       // In no update case, associate hosts to newest cache in group
1068       // now that master entry has been "successfully downloaded".
1069       if (internal_state_ == NO_UPDATE) {
1070         // TODO(michaeln): defer until the updated cache has been stored.
1071         DCHECK(!inprogress_cache_.get());
1072         AppCache* cache = group_->newest_complete_cache();
1073         PendingMasters::iterator found = pending_master_entries_.find(url);
1074         DCHECK(found != pending_master_entries_.end());
1075         PendingHosts& hosts = found->second;
1076         for (PendingHosts::iterator host_it = hosts.begin();
1077              host_it != hosts.end(); ++host_it) {
1078           (*host_it)->AssociateCompleteCache(cache);
1079         }
1080       }
1081     } else {
1082       URLFetcher* fetcher = new URLFetcher(
1083           url, URLFetcher::MASTER_ENTRY_FETCH, this);
1084       fetcher->Start();
1085       master_entry_fetches_.insert(PendingUrlFetches::value_type(url, fetcher));
1086     }
1087
1088     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
1089   }
1090 }
1091
1092 void AppCacheUpdateJob::CancelAllMasterEntryFetches(
1093     const std::string& error_message) {
1094   // For now, cancel all in-progress fetches for master entries and pretend
1095   // all master entries fetches have completed.
1096   // TODO(jennb): Delete this when update no longer fetches master entries
1097   // directly.
1098
1099   // Cancel all in-progress fetches.
1100   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
1101        it != master_entry_fetches_.end(); ++it) {
1102     delete it->second;
1103     master_entries_to_fetch_.insert(it->first);  // back in unfetched list
1104   }
1105   master_entry_fetches_.clear();
1106
1107   master_entries_completed_ += master_entries_to_fetch_.size();
1108
1109   // Cache failure steps, step 2.
1110   // Pretend all master entries that have not yet been fetched have completed
1111   // downloading. Unassociate hosts from any appcache and send ERROR event.
1112   HostNotifier host_notifier;
1113   while (!master_entries_to_fetch_.empty()) {
1114     const GURL& url = *master_entries_to_fetch_.begin();
1115     PendingMasters::iterator found = pending_master_entries_.find(url);
1116     DCHECK(found != pending_master_entries_.end());
1117     PendingHosts& hosts = found->second;
1118     for (PendingHosts::iterator host_it = hosts.begin();
1119          host_it != hosts.end(); ++host_it) {
1120       AppCacheHost* host = *host_it;
1121       host->AssociateNoCache(GURL());
1122       host_notifier.AddHost(host);
1123       host->RemoveObserver(this);
1124     }
1125     hosts.clear();
1126
1127     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
1128   }
1129   host_notifier.SendErrorNotifications(error_message);
1130 }
1131
1132 bool AppCacheUpdateJob::MaybeLoadFromNewestCache(const GURL& url,
1133                                                  AppCacheEntry& entry) {
1134   if (update_type_ != UPGRADE_ATTEMPT)
1135     return false;
1136
1137   AppCache* newest = group_->newest_complete_cache();
1138   AppCacheEntry* copy_me = newest->GetEntry(url);
1139   if (!copy_me || !copy_me->has_response_id())
1140     return false;
1141
1142   // Load HTTP headers for entry from newest cache.
1143   loading_responses_.insert(
1144       LoadingResponses::value_type(copy_me->response_id(), url));
1145   service_->storage()->LoadResponseInfo(manifest_url_, group_->group_id(),
1146                                         copy_me->response_id(),
1147                                         this);
1148   // Async: wait for OnResponseInfoLoaded to complete.
1149   return true;
1150 }
1151
1152 void AppCacheUpdateJob::OnResponseInfoLoaded(
1153     AppCacheResponseInfo* response_info, int64 response_id) {
1154   const net::HttpResponseInfo* http_info = response_info ?
1155       response_info->http_response_info() : NULL;
1156
1157   // Needed response info for a manifest fetch request.
1158   if (internal_state_ == FETCH_MANIFEST) {
1159     if (http_info)
1160       manifest_fetcher_->set_existing_response_headers(
1161           http_info->headers.get());
1162     manifest_fetcher_->Start();
1163     return;
1164   }
1165
1166   LoadingResponses::iterator found = loading_responses_.find(response_id);
1167   DCHECK(found != loading_responses_.end());
1168   const GURL& url = found->second;
1169
1170   if (!http_info) {
1171     LoadFromNewestCacheFailed(url, NULL);  // no response found
1172   } else {
1173     // Check if response can be re-used according to HTTP caching semantics.
1174     // Responses with a "vary" header get treated as expired.
1175     const std::string name = "vary";
1176     std::string value;
1177     void* iter = NULL;
1178     if (!http_info->headers.get() ||
1179         http_info->headers->RequiresValidation(http_info->request_time,
1180                                                http_info->response_time,
1181                                                base::Time::Now()) ||
1182         http_info->headers->EnumerateHeader(&iter, name, &value)) {
1183       LoadFromNewestCacheFailed(url, response_info);
1184     } else {
1185       DCHECK(group_->newest_complete_cache());
1186       AppCacheEntry* copy_me = group_->newest_complete_cache()->GetEntry(url);
1187       DCHECK(copy_me);
1188       DCHECK(copy_me->response_id() == response_id);
1189
1190       AppCache::EntryMap::iterator it = url_file_list_.find(url);
1191       DCHECK(it != url_file_list_.end());
1192       AppCacheEntry& entry = it->second;
1193       entry.set_response_id(response_id);
1194       entry.set_response_size(copy_me->response_size());
1195       inprogress_cache_->AddOrModifyEntry(url, entry);
1196       NotifyAllProgress(url);
1197       ++url_fetches_completed_;
1198     }
1199   }
1200   loading_responses_.erase(found);
1201
1202   MaybeCompleteUpdate();
1203 }
1204
1205 void AppCacheUpdateJob::LoadFromNewestCacheFailed(
1206     const GURL& url, AppCacheResponseInfo* response_info) {
1207   if (internal_state_ == CACHE_FAILURE)
1208     return;
1209
1210   // Re-insert url at front of fetch list. Indicate storage has been checked.
1211   urls_to_fetch_.push_front(UrlToFetch(url, true, response_info));
1212   FetchUrls();
1213 }
1214
1215 void AppCacheUpdateJob::MaybeCompleteUpdate() {
1216   DCHECK(internal_state_ != CACHE_FAILURE);
1217
1218   // Must wait for any pending master entries or url fetches to complete.
1219   if (master_entries_completed_ != pending_master_entries_.size() ||
1220       url_fetches_completed_ != url_file_list_.size()) {
1221     DCHECK(internal_state_ != COMPLETED);
1222     return;
1223   }
1224
1225   switch (internal_state_) {
1226     case NO_UPDATE:
1227       if (master_entries_completed_ > 0) {
1228         switch (stored_state_) {
1229           case UNSTORED:
1230             StoreGroupAndCache();
1231             return;
1232           case STORING:
1233             return;
1234           case STORED:
1235             break;
1236         }
1237       }
1238       // 6.9.4 steps 7.3-7.7.
1239       NotifyAllAssociatedHosts(NO_UPDATE_EVENT);
1240       DiscardDuplicateResponses();
1241       internal_state_ = COMPLETED;
1242       break;
1243     case DOWNLOADING:
1244       internal_state_ = REFETCH_MANIFEST;
1245       FetchManifest(false);
1246       break;
1247     case REFETCH_MANIFEST:
1248       DCHECK(stored_state_ == STORED);
1249       NotifyAllFinalProgress();
1250       if (update_type_ == CACHE_ATTEMPT)
1251         NotifyAllAssociatedHosts(CACHED_EVENT);
1252       else
1253         NotifyAllAssociatedHosts(UPDATE_READY_EVENT);
1254       DiscardDuplicateResponses();
1255       internal_state_ = COMPLETED;
1256       break;
1257     case CACHE_FAILURE:
1258       NOTREACHED();  // See HandleCacheFailure
1259       break;
1260     default:
1261       break;
1262   }
1263
1264   // Let the stack unwind before deletion to make it less risky as this
1265   // method is called from multiple places in this file.
1266   if (internal_state_ == COMPLETED)
1267     DeleteSoon();
1268 }
1269
1270 void AppCacheUpdateJob::ScheduleUpdateRetry(int delay_ms) {
1271   // TODO(jennb): post a delayed task with the "same parameters" as this job
1272   // to retry the update at a later time. Need group, URLs of pending master
1273   // entries and their hosts.
1274 }
1275
1276 void AppCacheUpdateJob::Cancel() {
1277   internal_state_ = CANCELLED;
1278
1279   if (manifest_fetcher_) {
1280     delete manifest_fetcher_;
1281     manifest_fetcher_ = NULL;
1282   }
1283
1284   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
1285        it != pending_url_fetches_.end(); ++it) {
1286     delete it->second;
1287   }
1288   pending_url_fetches_.clear();
1289
1290   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
1291        it != master_entry_fetches_.end(); ++it) {
1292     delete it->second;
1293   }
1294   master_entry_fetches_.clear();
1295
1296   ClearPendingMasterEntries();
1297   DiscardInprogressCache();
1298
1299   // Delete response writer to avoid any callbacks.
1300   if (manifest_response_writer_)
1301     manifest_response_writer_.reset();
1302
1303   service_->storage()->CancelDelegateCallbacks(this);
1304 }
1305
1306 void AppCacheUpdateJob::ClearPendingMasterEntries() {
1307   for (PendingMasters::iterator it = pending_master_entries_.begin();
1308        it != pending_master_entries_.end(); ++it) {
1309     PendingHosts& hosts = it->second;
1310     for (PendingHosts::iterator host_it = hosts.begin();
1311          host_it != hosts.end(); ++host_it) {
1312       (*host_it)->RemoveObserver(this);
1313     }
1314   }
1315
1316   pending_master_entries_.clear();
1317 }
1318
1319 void AppCacheUpdateJob::DiscardInprogressCache() {
1320   service_->storage()->DoomResponses(manifest_url_, stored_response_ids_);
1321
1322   if (!inprogress_cache_.get()) {
1323     // We have to undo the changes we made, if any, to the existing cache.
1324     for (std::vector<GURL>::iterator iter = added_master_entries_.begin();
1325          iter != added_master_entries_.end(); ++iter) {
1326       DCHECK(group_->newest_complete_cache());
1327       group_->newest_complete_cache()->RemoveEntry(*iter);
1328     }
1329     return;
1330   }
1331
1332   AppCache::AppCacheHosts& hosts = inprogress_cache_->associated_hosts();
1333   while (!hosts.empty())
1334     (*hosts.begin())->AssociateNoCache(GURL());
1335
1336   inprogress_cache_ = NULL;
1337 }
1338
1339 void AppCacheUpdateJob::DiscardDuplicateResponses() {
1340   service_->storage()->DoomResponses(manifest_url_, duplicate_response_ids_);
1341 }
1342
1343 void AppCacheUpdateJob::DeleteSoon() {
1344   ClearPendingMasterEntries();
1345   manifest_response_writer_.reset();
1346   service_->storage()->CancelDelegateCallbacks(this);
1347
1348   // Break the connection with the group so the group cannot call delete
1349   // on this object after we've posted a task to delete ourselves.
1350   group_->SetUpdateStatus(AppCacheGroup::IDLE);
1351   group_ = NULL;
1352
1353   base::MessageLoop::current()->DeleteSoon(FROM_HERE, this);
1354 }
1355
1356 }  // namespace appcache