content/browser/appcache/appcache_update_job.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/appcache/appcache_update_job.h"
   6
   7 #include "base/bind.h"
   8 #include "base/bind_helpers.h"
   9 #include "base/compiler_specific.h"
  10 #include "base/message_loop/message_loop.h"
  11 #include "base/strings/string_util.h"
  12 #include "base/strings/stringprintf.h"
  13 #include "content/browser/appcache/appcache_group.h"
  14 #include "content/browser/appcache/appcache_histograms.h"
  15 #include "net/base/io_buffer.h"
  16 #include "net/base/load_flags.h"
  17 #include "net/base/net_errors.h"
  18 #include "net/base/request_priority.h"
  19 #include "net/http/http_request_headers.h"
  20 #include "net/http/http_response_headers.h"
  21 #include "net/url_request/url_request_context.h"
  22
  23 namespace content {
  24
  25 static const int kBufferSize = 32768;
  26 static const size_t kMaxConcurrentUrlFetches = 2;
  27 static const int kMax503Retries = 3;
  28
  29 static std::string FormatUrlErrorMessage(
  30       const char* format, const GURL& url,
  31       AppCacheUpdateJob::ResultType error,
  32       int response_code) {
  33     // Show the net response code if we have one.
  34     int code = response_code;
  35     if (error != AppCacheUpdateJob::SERVER_ERROR)
  36       code = static_cast<int>(error);
  37     return base::StringPrintf(format, code, url.spec().c_str());
  38 }
  39
  40 // Helper class for collecting hosts per frontend when sending notifications
  41 // so that only one notification is sent for all hosts using the same frontend.
  42 class HostNotifier {
  43  public:
  44   typedef std::vector<int> HostIds;
  45   typedef std::map<AppCacheFrontend*, HostIds> NotifyHostMap;
  46
  47   // Caller is responsible for ensuring there will be no duplicate hosts.
  48   void AddHost(AppCacheHost* host) {
  49     std::pair<NotifyHostMap::iterator , bool> ret = hosts_to_notify.insert(
  50         NotifyHostMap::value_type(host->frontend(), HostIds()));
  51     ret.first->second.push_back(host->host_id());
  52   }
  53
  54   void AddHosts(const std::set<AppCacheHost*>& hosts) {
  55     for (std::set<AppCacheHost*>::const_iterator it = hosts.begin();
  56          it != hosts.end(); ++it) {
  57       AddHost(*it);
  58     }
  59   }
  60
  61   void SendNotifications(AppCacheEventID event_id) {
  62     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  63          it != hosts_to_notify.end(); ++it) {
  64       AppCacheFrontend* frontend = it->first;
  65       frontend->OnEventRaised(it->second, event_id);
  66     }
  67   }
  68
  69   void SendProgressNotifications(
  70       const GURL& url, int num_total, int num_complete) {
  71     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  72          it != hosts_to_notify.end(); ++it) {
  73       AppCacheFrontend* frontend = it->first;
  74       frontend->OnProgressEventRaised(it->second, url,
  75                                       num_total, num_complete);
  76     }
  77   }
  78
  79   void SendErrorNotifications(const AppCacheErrorDetails& details) {
  80     DCHECK(!details.message.empty());
  81     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  82          it != hosts_to_notify.end(); ++it) {
  83       AppCacheFrontend* frontend = it->first;
  84       frontend->OnErrorEventRaised(it->second, details);
  85     }
  86   }
  87
  88   void SendLogMessage(const std::string& message) {
  89     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  90          it != hosts_to_notify.end(); ++it) {
  91       AppCacheFrontend* frontend = it->first;
  92       for (HostIds::iterator id = it->second.begin();
  93            id != it->second.end(); ++id) {
  94         frontend->OnLogMessage(*id, APPCACHE_LOG_WARNING, message);
  95       }
  96     }
  97   }
  98
  99  private:
 100   NotifyHostMap hosts_to_notify;
 101 };
 102
 103 AppCacheUpdateJob::UrlToFetch::UrlToFetch(const GURL& url,
 104                                           bool checked,
 105                                           AppCacheResponseInfo* info)
 106     : url(url),
 107       storage_checked(checked),
 108       existing_response_info(info) {
 109 }
 110
 111 AppCacheUpdateJob::UrlToFetch::~UrlToFetch() {
 112 }
 113
 114 // Helper class to fetch resources. Depending on the fetch type,
 115 // can either fetch to an in-memory string or write the response
 116 // data out to the disk cache.
 117 AppCacheUpdateJob::URLFetcher::URLFetcher(const GURL& url,
 118                                           FetchType fetch_type,
 119                                           AppCacheUpdateJob* job)
 120     : url_(url),
 121       job_(job),
 122       fetch_type_(fetch_type),
 123       retry_503_attempts_(0),
 124       buffer_(new net::IOBuffer(kBufferSize)),
 125       request_(job->service_->request_context()
 126                    ->CreateRequest(url, net::DEFAULT_PRIORITY, this, NULL)),
 127       result_(UPDATE_OK),
 128       redirect_response_code_(-1) {}
 129
 130 AppCacheUpdateJob::URLFetcher::~URLFetcher() {
 131 }
 132
 133 void AppCacheUpdateJob::URLFetcher::Start() {
 134   request_->set_first_party_for_cookies(job_->manifest_url_);
 135   request_->SetLoadFlags(request_->load_flags() |
 136                          net::LOAD_DISABLE_INTERCEPT);
 137   if (existing_response_headers_.get())
 138     AddConditionalHeaders(existing_response_headers_.get());
 139   request_->Start();
 140 }
 141
 142 void AppCacheUpdateJob::URLFetcher::OnReceivedRedirect(
 143     net::URLRequest* request,
 144     const net::RedirectInfo& redirect_info,
 145     bool* defer_redirect) {
 146   DCHECK(request_ == request);
 147   // Redirect is not allowed by the update process.
 148   job_->MadeProgress();
 149   redirect_response_code_ = request->GetResponseCode();
 150   request->Cancel();
 151   result_ = REDIRECT_ERROR;
 152   OnResponseCompleted();
 153 }
 154
 155 void AppCacheUpdateJob::URLFetcher::OnResponseStarted(
 156     net::URLRequest *request) {
 157   DCHECK(request == request_);
 158   int response_code = -1;
 159   if (request->status().is_success()) {
 160     response_code = request->GetResponseCode();
 161     job_->MadeProgress();
 162   }
 163   if ((response_code / 100) == 2) {
 164
 165     // See http://code.google.com/p/chromium/issues/detail?id=69594
 166     // We willfully violate the HTML5 spec at this point in order
 167     // to support the appcaching of cross-origin HTTPS resources.
 168     // We've opted for a milder constraint and allow caching unless
 169     // the resource has a "no-store" header. A spec change has been
 170     // requested on the whatwg list.
 171     // TODO(michaeln): Consider doing this for cross-origin HTTP resources too.
 172     if (url_.SchemeIsSecure() &&
 173         url_.GetOrigin() != job_->manifest_url_.GetOrigin()) {
 174       if (request->response_headers()->
 175               HasHeaderValue("cache-control", "no-store")) {
 176         DCHECK_EQ(-1, redirect_response_code_);
 177         request->Cancel();
 178         result_ = SERVER_ERROR;  // Not the best match?
 179         OnResponseCompleted();
 180         return;
 181       }
 182     }
 183
 184     // Write response info to storage for URL fetches. Wait for async write
 185     // completion before reading any response data.
 186     if (fetch_type_ == URL_FETCH || fetch_type_ == MASTER_ENTRY_FETCH) {
 187       response_writer_.reset(job_->CreateResponseWriter());
 188       scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
 189           new HttpResponseInfoIOBuffer(
 190               new net::HttpResponseInfo(request->response_info())));
 191       response_writer_->WriteInfo(
 192           io_buffer.get(),
 193           base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
 194     } else {
 195       ReadResponseData();
 196     }
 197   } else {
 198     if (response_code > 0)
 199       result_ = SERVER_ERROR;
 200     else
 201       result_ = NETWORK_ERROR;
 202     OnResponseCompleted();
 203   }
 204 }
 205
 206 void AppCacheUpdateJob::URLFetcher::OnReadCompleted(
 207     net::URLRequest* request, int bytes_read) {
 208   DCHECK(request_ == request);
 209   bool data_consumed = true;
 210   if (request->status().is_success() && bytes_read > 0) {
 211     job_->MadeProgress();
 212     data_consumed = ConsumeResponseData(bytes_read);
 213     if (data_consumed) {
 214       bytes_read = 0;
 215       while (request->Read(buffer_.get(), kBufferSize, &bytes_read)) {
 216         if (bytes_read > 0) {
 217           data_consumed = ConsumeResponseData(bytes_read);
 218           if (!data_consumed)
 219             break;  // wait for async data processing, then read more
 220         } else {
 221           break;
 222         }
 223       }
 224     }
 225   }
 226   if (data_consumed && !request->status().is_io_pending()) {
 227     DCHECK_EQ(UPDATE_OK, result_);
 228     OnResponseCompleted();
 229   }
 230 }
 231
 232 void AppCacheUpdateJob::URLFetcher::AddConditionalHeaders(
 233     const net::HttpResponseHeaders* headers) {
 234   DCHECK(request_.get() && headers);
 235   net::HttpRequestHeaders extra_headers;
 236
 237   // Add If-Modified-Since header if response info has Last-Modified header.
 238   const std::string last_modified = "Last-Modified";
 239   std::string last_modified_value;
 240   headers->EnumerateHeader(NULL, last_modified, &last_modified_value);
 241   if (!last_modified_value.empty()) {
 242     extra_headers.SetHeader(net::HttpRequestHeaders::kIfModifiedSince,
 243                             last_modified_value);
 244   }
 245
 246   // Add If-None-Match header if response info has ETag header.
 247   const std::string etag = "ETag";
 248   std::string etag_value;
 249   headers->EnumerateHeader(NULL, etag, &etag_value);
 250   if (!etag_value.empty()) {
 251     extra_headers.SetHeader(net::HttpRequestHeaders::kIfNoneMatch,
 252                             etag_value);
 253   }
 254   if (!extra_headers.IsEmpty())
 255     request_->SetExtraRequestHeaders(extra_headers);
 256 }
 257
 258 void  AppCacheUpdateJob::URLFetcher::OnWriteComplete(int result) {
 259   if (result < 0) {
 260     request_->Cancel();
 261     result_ = DISKCACHE_ERROR;
 262     OnResponseCompleted();
 263     return;
 264   }
 265   ReadResponseData();
 266 }
 267
 268 void AppCacheUpdateJob::URLFetcher::ReadResponseData() {
 269   InternalUpdateState state = job_->internal_state_;
 270   if (state == CACHE_FAILURE || state == CANCELLED || state == COMPLETED)
 271     return;
 272   int bytes_read = 0;
 273   request_->Read(buffer_.get(), kBufferSize, &bytes_read);
 274   OnReadCompleted(request_.get(), bytes_read);
 275 }
 276
 277 // Returns false if response data is processed asynchronously, in which
 278 // case ReadResponseData will be invoked when it is safe to continue
 279 // reading more response data from the request.
 280 bool AppCacheUpdateJob::URLFetcher::ConsumeResponseData(int bytes_read) {
 281   DCHECK_GT(bytes_read, 0);
 282   switch (fetch_type_) {
 283     case MANIFEST_FETCH:
 284     case MANIFEST_REFETCH:
 285       manifest_data_.append(buffer_->data(), bytes_read);
 286       break;
 287     case URL_FETCH:
 288     case MASTER_ENTRY_FETCH:
 289       DCHECK(response_writer_.get());
 290       response_writer_->WriteData(
 291           buffer_.get(),
 292           bytes_read,
 293           base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
 294       return false;  // wait for async write completion to continue reading
 295     default:
 296       NOTREACHED();
 297   }
 298   return true;
 299 }
 300
 301 void AppCacheUpdateJob::URLFetcher::OnResponseCompleted() {
 302   if (request_->status().is_success())
 303     job_->MadeProgress();
 304
 305   // Retry for 503s where retry-after is 0.
 306   if (request_->status().is_success() &&
 307       request_->GetResponseCode() == 503 &&
 308       MaybeRetryRequest()) {
 309     return;
 310   }
 311
 312   switch (fetch_type_) {
 313     case MANIFEST_FETCH:
 314       job_->HandleManifestFetchCompleted(this);
 315       break;
 316     case URL_FETCH:
 317       job_->HandleUrlFetchCompleted(this);
 318       break;
 319     case MASTER_ENTRY_FETCH:
 320       job_->HandleMasterEntryFetchCompleted(this);
 321       break;
 322     case MANIFEST_REFETCH:
 323       job_->HandleManifestRefetchCompleted(this);
 324       break;
 325     default:
 326       NOTREACHED();
 327   }
 328
 329   delete this;
 330 }
 331
 332 bool AppCacheUpdateJob::URLFetcher::MaybeRetryRequest() {
 333   if (retry_503_attempts_ >= kMax503Retries ||
 334       !request_->response_headers()->HasHeaderValue("retry-after", "0")) {
 335     return false;
 336   }
 337   ++retry_503_attempts_;
 338   result_ = UPDATE_OK;
 339   request_ = job_->service_->request_context()->CreateRequest(
 340       url_, net::DEFAULT_PRIORITY, this, NULL);
 341   Start();
 342   return true;
 343 }
 344
 345 AppCacheUpdateJob::AppCacheUpdateJob(AppCacheServiceImpl* service,
 346                                      AppCacheGroup* group)
 347     : service_(service),
 348       manifest_url_(group->manifest_url()),
 349       group_(group),
 350       update_type_(UNKNOWN_TYPE),
 351       internal_state_(FETCH_MANIFEST),
 352       master_entries_completed_(0),
 353       url_fetches_completed_(0),
 354       manifest_fetcher_(NULL),
 355       manifest_has_valid_mime_type_(false),
 356       stored_state_(UNSTORED),
 357       storage_(service->storage()) {
 358     service_->AddObserver(this);
 359 }
 360
 361 AppCacheUpdateJob::~AppCacheUpdateJob() {
 362   if (service_)
 363     service_->RemoveObserver(this);
 364   if (internal_state_ != COMPLETED)
 365     Cancel();
 366
 367   DCHECK(!manifest_fetcher_);
 368   DCHECK(pending_url_fetches_.empty());
 369   DCHECK(!inprogress_cache_.get());
 370   DCHECK(pending_master_entries_.empty());
 371   DCHECK(master_entry_fetches_.empty());
 372
 373   if (group_)
 374     group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE);
 375 }
 376
 377 void AppCacheUpdateJob::StartUpdate(AppCacheHost* host,
 378                                     const GURL& new_master_resource) {
 379   DCHECK(group_->update_job() == this);
 380   DCHECK(!group_->is_obsolete());
 381
 382   bool is_new_pending_master_entry = false;
 383   if (!new_master_resource.is_empty()) {
 384     DCHECK(new_master_resource == host->pending_master_entry_url());
 385     DCHECK(!new_master_resource.has_ref());
 386     DCHECK(new_master_resource.GetOrigin() == manifest_url_.GetOrigin());
 387
 388     // Cannot add more to this update if already terminating.
 389     if (IsTerminating()) {
 390       group_->QueueUpdate(host, new_master_resource);
 391       return;
 392     }
 393
 394     std::pair<PendingMasters::iterator, bool> ret =
 395         pending_master_entries_.insert(
 396             PendingMasters::value_type(new_master_resource, PendingHosts()));
 397     is_new_pending_master_entry = ret.second;
 398     ret.first->second.push_back(host);
 399     host->AddObserver(this);
 400   }
 401
 402   // Notify host (if any) if already checking or downloading.
 403   AppCacheGroup::UpdateAppCacheStatus update_status = group_->update_status();
 404   if (update_status == AppCacheGroup::CHECKING ||
 405       update_status == AppCacheGroup::DOWNLOADING) {
 406     if (host) {
 407       NotifySingleHost(host, APPCACHE_CHECKING_EVENT);
 408       if (update_status == AppCacheGroup::DOWNLOADING)
 409         NotifySingleHost(host, APPCACHE_DOWNLOADING_EVENT);
 410
 411       // Add to fetch list or an existing entry if already fetched.
 412       if (!new_master_resource.is_empty()) {
 413         AddMasterEntryToFetchList(host, new_master_resource,
 414                                   is_new_pending_master_entry);
 415       }
 416     }
 417     return;
 418   }
 419
 420   // Begin update process for the group.
 421   MadeProgress();
 422   group_->SetUpdateAppCacheStatus(AppCacheGroup::CHECKING);
 423   if (group_->HasCache()) {
 424     update_type_ = UPGRADE_ATTEMPT;
 425     NotifyAllAssociatedHosts(APPCACHE_CHECKING_EVENT);
 426   } else {
 427     update_type_ = CACHE_ATTEMPT;
 428     DCHECK(host);
 429     NotifySingleHost(host, APPCACHE_CHECKING_EVENT);
 430   }
 431
 432   if (!new_master_resource.is_empty()) {
 433     AddMasterEntryToFetchList(host, new_master_resource,
 434                               is_new_pending_master_entry);
 435   }
 436
 437   FetchManifest(true);
 438 }
 439
 440 AppCacheResponseWriter* AppCacheUpdateJob::CreateResponseWriter() {
 441   AppCacheResponseWriter* writer =
 442       storage_->CreateResponseWriter(manifest_url_,
 443                                                 group_->group_id());
 444   stored_response_ids_.push_back(writer->response_id());
 445   return writer;
 446 }
 447
 448 void AppCacheUpdateJob::HandleCacheFailure(
 449     const AppCacheErrorDetails& error_details,
 450     ResultType result,
 451     const GURL& failed_resource_url) {
 452   // 6.9.4 cache failure steps 2-8.
 453   DCHECK(internal_state_ != CACHE_FAILURE);
 454   DCHECK(!error_details.message.empty());
 455   DCHECK(result != UPDATE_OK);
 456   internal_state_ = CACHE_FAILURE;
 457   LogHistogramStats(result, failed_resource_url);
 458   CancelAllUrlFetches();
 459   CancelAllMasterEntryFetches(error_details);
 460   NotifyAllError(error_details);
 461   DiscardInprogressCache();
 462   internal_state_ = COMPLETED;
 463   DeleteSoon();  // To unwind the stack prior to deletion.
 464 }
 465
 466 void AppCacheUpdateJob::FetchManifest(bool is_first_fetch) {
 467   DCHECK(!manifest_fetcher_);
 468   manifest_fetcher_ = new URLFetcher(
 469      manifest_url_,
 470      is_first_fetch ? URLFetcher::MANIFEST_FETCH :
 471                       URLFetcher::MANIFEST_REFETCH,
 472      this);
 473
 474   // Add any necessary Http headers before sending fetch request.
 475   if (is_first_fetch) {
 476     AppCacheEntry* entry = (update_type_ == UPGRADE_ATTEMPT) ?
 477         group_->newest_complete_cache()->GetEntry(manifest_url_) : NULL;
 478     if (entry) {
 479       // Asynchronously load response info for manifest from newest cache.
 480       storage_->LoadResponseInfo(manifest_url_, group_->group_id(),
 481                                  entry->response_id(), this);
 482     } else {
 483       manifest_fetcher_->Start();
 484     }
 485   } else {
 486     DCHECK(internal_state_ == REFETCH_MANIFEST);
 487     DCHECK(manifest_response_info_.get());
 488     manifest_fetcher_->set_existing_response_headers(
 489         manifest_response_info_->headers.get());
 490     manifest_fetcher_->Start();
 491   }
 492 }
 493
 494
 495 void AppCacheUpdateJob::HandleManifestFetchCompleted(
 496     URLFetcher* fetcher) {
 497   DCHECK_EQ(internal_state_, FETCH_MANIFEST);
 498   DCHECK_EQ(manifest_fetcher_, fetcher);
 499   manifest_fetcher_ = NULL;
 500
 501   net::URLRequest* request = fetcher->request();
 502   int response_code = -1;
 503   bool is_valid_response_code = false;
 504   if (request->status().is_success()) {
 505     response_code = request->GetResponseCode();
 506     is_valid_response_code = (response_code / 100 == 2);
 507
 508     std::string mime_type;
 509     request->GetMimeType(&mime_type);
 510     manifest_has_valid_mime_type_ = (mime_type == "text/cache-manifest");
 511   }
 512
 513   if (is_valid_response_code) {
 514     manifest_data_ = fetcher->manifest_data();
 515     manifest_response_info_.reset(
 516         new net::HttpResponseInfo(request->response_info()));
 517     if (update_type_ == UPGRADE_ATTEMPT)
 518       CheckIfManifestChanged();  // continues asynchronously
 519     else
 520       ContinueHandleManifestFetchCompleted(true);
 521   } else if (response_code == 304 && update_type_ == UPGRADE_ATTEMPT) {
 522     ContinueHandleManifestFetchCompleted(false);
 523   } else if ((response_code == 404 || response_code == 410) &&
 524              update_type_ == UPGRADE_ATTEMPT) {
 525     storage_->MakeGroupObsolete(group_, this, response_code);  // async
 526   } else {
 527     const char* kFormatString = "Manifest fetch failed (%d) %s";
 528     std::string message = FormatUrlErrorMessage(
 529         kFormatString, manifest_url_, fetcher->result(), response_code);
 530     HandleCacheFailure(AppCacheErrorDetails(message,
 531                                     APPCACHE_MANIFEST_ERROR,
 532                                     manifest_url_,
 533                                     response_code,
 534                                     false /*is_cross_origin*/),
 535                        fetcher->result(),
 536                        GURL());
 537   }
 538 }
 539
 540 void AppCacheUpdateJob::OnGroupMadeObsolete(AppCacheGroup* group,
 541                                             bool success,
 542                                             int response_code) {
 543   DCHECK(master_entry_fetches_.empty());
 544   CancelAllMasterEntryFetches(AppCacheErrorDetails(
 545       "The cache has been made obsolete, "
 546       "the manifest file returned 404 or 410",
 547       APPCACHE_MANIFEST_ERROR,
 548       GURL(),
 549       response_code,
 550       false /*is_cross_origin*/));
 551   if (success) {
 552     DCHECK(group->is_obsolete());
 553     NotifyAllAssociatedHosts(APPCACHE_OBSOLETE_EVENT);
 554     internal_state_ = COMPLETED;
 555     MaybeCompleteUpdate();
 556   } else {
 557     // Treat failure to mark group obsolete as a cache failure.
 558     HandleCacheFailure(AppCacheErrorDetails(
 559         "Failed to mark the cache as obsolete",
 560         APPCACHE_UNKNOWN_ERROR,
 561         GURL(),
 562         0,
 563         false /*is_cross_origin*/),
 564                        DB_ERROR,
 565                        GURL());
 566   }
 567 }
 568
 569 void AppCacheUpdateJob::ContinueHandleManifestFetchCompleted(bool changed) {
 570   DCHECK(internal_state_ == FETCH_MANIFEST);
 571
 572   if (!changed) {
 573     DCHECK(update_type_ == UPGRADE_ATTEMPT);
 574     internal_state_ = NO_UPDATE;
 575
 576     // Wait for pending master entries to download.
 577     FetchMasterEntries();
 578     MaybeCompleteUpdate();  // if not done, run async 6.9.4 step 7 substeps
 579     return;
 580   }
 581
 582   Manifest manifest;
 583   if (!ParseManifest(manifest_url_, manifest_data_.data(),
 584                      manifest_data_.length(),
 585                      manifest_has_valid_mime_type_ ?
 586                         PARSE_MANIFEST_ALLOWING_INTERCEPTS :
 587                         PARSE_MANIFEST_PER_STANDARD,
 588                      manifest)) {
 589     const char* kFormatString = "Failed to parse manifest %s";
 590     const std::string message = base::StringPrintf(kFormatString,
 591         manifest_url_.spec().c_str());
 592     HandleCacheFailure(
 593         AppCacheErrorDetails(
 594             message, APPCACHE_SIGNATURE_ERROR, GURL(), 0,
 595             false /*is_cross_origin*/),
 596         MANIFEST_ERROR,
 597         GURL());
 598     VLOG(1) << message;
 599     return;
 600   }
 601
 602   // Proceed with update process. Section 6.9.4 steps 8-20.
 603   internal_state_ = DOWNLOADING;
 604   inprogress_cache_ = new AppCache(storage_, storage_->NewCacheId());
 605   BuildUrlFileList(manifest);
 606   inprogress_cache_->InitializeWithManifest(&manifest);
 607
 608   // Associate all pending master hosts with the newly created cache.
 609   for (PendingMasters::iterator it = pending_master_entries_.begin();
 610        it != pending_master_entries_.end(); ++it) {
 611     PendingHosts& hosts = it->second;
 612     for (PendingHosts::iterator host_it = hosts.begin();
 613          host_it != hosts.end(); ++host_it) {
 614       (*host_it)
 615           ->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
 616     }
 617   }
 618
 619   if (manifest.did_ignore_intercept_namespaces) {
 620     // Must be done after associating all pending master hosts.
 621     std::string message(
 622         "Ignoring the INTERCEPT section of the application cache manifest "
 623         "because the content type is not text/cache-manifest");
 624     LogConsoleMessageToAll(message);
 625   }
 626
 627   group_->SetUpdateAppCacheStatus(AppCacheGroup::DOWNLOADING);
 628   NotifyAllAssociatedHosts(APPCACHE_DOWNLOADING_EVENT);
 629   FetchUrls();
 630   FetchMasterEntries();
 631   MaybeCompleteUpdate();  // if not done, continues when async fetches complete
 632 }
 633
 634 void AppCacheUpdateJob::HandleUrlFetchCompleted(URLFetcher* fetcher) {
 635   DCHECK(internal_state_ == DOWNLOADING);
 636
 637   net::URLRequest* request = fetcher->request();
 638   const GURL& url = request->original_url();
 639   pending_url_fetches_.erase(url);
 640   NotifyAllProgress(url);
 641   ++url_fetches_completed_;
 642
 643   int response_code = request->status().is_success()
 644                           ? request->GetResponseCode()
 645                           : fetcher->redirect_response_code();
 646
 647   AppCacheEntry& entry = url_file_list_.find(url)->second;
 648
 649   if (response_code / 100 == 2) {
 650     // Associate storage with the new entry.
 651     DCHECK(fetcher->response_writer());
 652     entry.set_response_id(fetcher->response_writer()->response_id());
 653     entry.set_response_size(fetcher->response_writer()->amount_written());
 654     if (!inprogress_cache_->AddOrModifyEntry(url, entry))
 655       duplicate_response_ids_.push_back(entry.response_id());
 656
 657     // TODO(michaeln): Check for <html manifest=xxx>
 658     // See http://code.google.com/p/chromium/issues/detail?id=97930
 659     // if (entry.IsMaster() && !(entry.IsExplicit() || fallback || intercept))
 660     //   if (!manifestAttribute) skip it
 661
 662     // Foreign entries will be detected during cache selection.
 663     // Note: 6.9.4, step 17.9 possible optimization: if resource is HTML or XML
 664     // file whose root element is an html element with a manifest attribute
 665     // whose value doesn't match the manifest url of the application cache
 666     // being processed, mark the entry as being foreign.
 667   } else {
 668     VLOG(1) << "Request status: " << request->status().status()
 669             << " error: " << request->status().error()
 670             << " response code: " << response_code;
 671     if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept()) {
 672       if (response_code == 304 && fetcher->existing_entry().has_response_id()) {
 673         // Keep the existing response.
 674         entry.set_response_id(fetcher->existing_entry().response_id());
 675         entry.set_response_size(fetcher->existing_entry().response_size());
 676         inprogress_cache_->AddOrModifyEntry(url, entry);
 677       } else {
 678         const char* kFormatString = "Resource fetch failed (%d) %s";
 679         std::string message = FormatUrlErrorMessage(
 680             kFormatString, url, fetcher->result(), response_code);
 681         ResultType result = fetcher->result();
 682         bool is_cross_origin = url.GetOrigin() != manifest_url_.GetOrigin();
 683         switch (result) {
 684           case DISKCACHE_ERROR:
 685             HandleCacheFailure(
 686                 AppCacheErrorDetails(
 687                     message, APPCACHE_UNKNOWN_ERROR, GURL(), 0,
 688                     is_cross_origin),
 689                 result,
 690                 url);
 691             break;
 692           case NETWORK_ERROR:
 693             HandleCacheFailure(
 694                 AppCacheErrorDetails(message, APPCACHE_RESOURCE_ERROR, url, 0,
 695                     is_cross_origin),
 696                 result,
 697                 url);
 698             break;
 699           default:
 700             HandleCacheFailure(AppCacheErrorDetails(message,
 701                                             APPCACHE_RESOURCE_ERROR,
 702                                             url,
 703                                             response_code,
 704                                             is_cross_origin),
 705                                result,
 706                                url);
 707             break;
 708         }
 709         return;
 710       }
 711     } else if (response_code == 404 || response_code == 410) {
 712       // Entry is skipped.  They are dropped from the cache.
 713     } else if (update_type_ == UPGRADE_ATTEMPT &&
 714                fetcher->existing_entry().has_response_id()) {
 715       // Keep the existing response.
 716       // TODO(michaeln): Not sure this is a good idea. This is spec compliant
 717       // but the old resource may or may not be compatible with the new contents
 718       // of the cache. Impossible to know one way or the other.
 719       entry.set_response_id(fetcher->existing_entry().response_id());
 720       entry.set_response_size(fetcher->existing_entry().response_size());
 721       inprogress_cache_->AddOrModifyEntry(url, entry);
 722     }
 723   }
 724
 725   // Fetch another URL now that one request has completed.
 726   DCHECK(internal_state_ != CACHE_FAILURE);
 727   FetchUrls();
 728   MaybeCompleteUpdate();
 729 }
 730
 731 void AppCacheUpdateJob::HandleMasterEntryFetchCompleted(
 732     URLFetcher* fetcher) {
 733   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
 734
 735   // TODO(jennb): Handle downloads completing during cache failure when update
 736   // no longer fetches master entries directly. For now, we cancel all pending
 737   // master entry fetches when entering cache failure state so this will never
 738   // be called in CACHE_FAILURE state.
 739
 740   net::URLRequest* request = fetcher->request();
 741   const GURL& url = request->original_url();
 742   master_entry_fetches_.erase(url);
 743   ++master_entries_completed_;
 744
 745   int response_code = request->status().is_success()
 746       ? request->GetResponseCode() : -1;
 747
 748   PendingMasters::iterator found = pending_master_entries_.find(url);
 749   DCHECK(found != pending_master_entries_.end());
 750   PendingHosts& hosts = found->second;
 751
 752   // Section 6.9.4. No update case: step 7.3, else step 22.
 753   if (response_code / 100 == 2) {
 754     // Add fetched master entry to the appropriate cache.
 755     AppCache* cache = inprogress_cache_.get() ? inprogress_cache_.get()
 756                                               : group_->newest_complete_cache();
 757     DCHECK(fetcher->response_writer());
 758     AppCacheEntry master_entry(AppCacheEntry::MASTER,
 759                                fetcher->response_writer()->response_id(),
 760                                fetcher->response_writer()->amount_written());
 761     if (cache->AddOrModifyEntry(url, master_entry))
 762       added_master_entries_.push_back(url);
 763     else
 764       duplicate_response_ids_.push_back(master_entry.response_id());
 765
 766     // In no-update case, associate host with the newest cache.
 767     if (!inprogress_cache_.get()) {
 768       // TODO(michaeln): defer until the updated cache has been stored
 769       DCHECK(cache == group_->newest_complete_cache());
 770       for (PendingHosts::iterator host_it = hosts.begin();
 771            host_it != hosts.end(); ++host_it) {
 772         (*host_it)->AssociateCompleteCache(cache);
 773       }
 774     }
 775   } else {
 776     HostNotifier host_notifier;
 777     for (PendingHosts::iterator host_it = hosts.begin();
 778          host_it != hosts.end(); ++host_it) {
 779       AppCacheHost* host = *host_it;
 780       host_notifier.AddHost(host);
 781
 782       // In downloading case, disassociate host from inprogress cache.
 783       if (inprogress_cache_.get())
 784         host->AssociateNoCache(GURL());
 785
 786       host->RemoveObserver(this);
 787     }
 788     hosts.clear();
 789
 790     const char* kFormatString = "Manifest fetch failed (%d) %s";
 791     std::string message = FormatUrlErrorMessage(
 792         kFormatString, request->url(), fetcher->result(), response_code);
 793     host_notifier.SendErrorNotifications(
 794         AppCacheErrorDetails(message,
 795                      APPCACHE_MANIFEST_ERROR,
 796                      request->url(),
 797                      response_code,
 798                      false /*is_cross_origin*/));
 799
 800     // In downloading case, update result is different if all master entries
 801     // failed vs. only some failing.
 802     if (inprogress_cache_.get()) {
 803       // Only count successful downloads to know if all master entries failed.
 804       pending_master_entries_.erase(found);
 805       --master_entries_completed_;
 806
 807       // Section 6.9.4, step 22.3.
 808       if (update_type_ == CACHE_ATTEMPT && pending_master_entries_.empty()) {
 809         HandleCacheFailure(AppCacheErrorDetails(message,
 810                                         APPCACHE_MANIFEST_ERROR,
 811                                         request->url(),
 812                                         response_code,
 813                                         false /*is_cross_origin*/),
 814                            fetcher->result(),
 815                            GURL());
 816         return;
 817       }
 818     }
 819   }
 820
 821   DCHECK(internal_state_ != CACHE_FAILURE);
 822   FetchMasterEntries();
 823   MaybeCompleteUpdate();
 824 }
 825
 826 void AppCacheUpdateJob::HandleManifestRefetchCompleted(
 827     URLFetcher* fetcher) {
 828   DCHECK(internal_state_ == REFETCH_MANIFEST);
 829   DCHECK(manifest_fetcher_ == fetcher);
 830   manifest_fetcher_ = NULL;
 831
 832   net::URLRequest* request = fetcher->request();
 833   int response_code = request->status().is_success()
 834       ? request->GetResponseCode() : -1;
 835   if (response_code == 304 || manifest_data_ == fetcher->manifest_data()) {
 836     // Only need to store response in storage if manifest is not already
 837     // an entry in the cache.
 838     AppCacheEntry* entry = inprogress_cache_->GetEntry(manifest_url_);
 839     if (entry) {
 840       entry->add_types(AppCacheEntry::MANIFEST);
 841       StoreGroupAndCache();
 842     } else {
 843       manifest_response_writer_.reset(CreateResponseWriter());
 844       scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
 845           new HttpResponseInfoIOBuffer(manifest_response_info_.release()));
 846       manifest_response_writer_->WriteInfo(
 847           io_buffer.get(),
 848           base::Bind(&AppCacheUpdateJob::OnManifestInfoWriteComplete,
 849                      base::Unretained(this)));
 850     }
 851   } else {
 852     VLOG(1) << "Request status: " << request->status().status()
 853             << " error: " << request->status().error()
 854             << " response code: " << response_code;
 855     ScheduleUpdateRetry(kRerunDelayMs);
 856     if (response_code == 200) {
 857       HandleCacheFailure(AppCacheErrorDetails("Manifest changed during update",
 858                                       APPCACHE_CHANGED_ERROR,
 859                                       GURL(),
 860                                       0,
 861                                       false /*is_cross_origin*/),
 862                          MANIFEST_ERROR,
 863                          GURL());
 864     } else {
 865       const char* kFormatString = "Manifest re-fetch failed (%d) %s";
 866       std::string message = FormatUrlErrorMessage(
 867           kFormatString, manifest_url_, fetcher->result(), response_code);
 868       HandleCacheFailure(AppCacheErrorDetails(message,
 869                                       APPCACHE_MANIFEST_ERROR,
 870                                       GURL(),
 871                                       response_code,
 872                                       false /*is_cross_origin*/),
 873                          fetcher->result(),
 874                          GURL());
 875     }
 876   }
 877 }
 878
 879 void AppCacheUpdateJob::OnManifestInfoWriteComplete(int result) {
 880   if (result > 0) {
 881     scoped_refptr<net::StringIOBuffer> io_buffer(
 882         new net::StringIOBuffer(manifest_data_));
 883     manifest_response_writer_->WriteData(
 884         io_buffer.get(),
 885         manifest_data_.length(),
 886         base::Bind(&AppCacheUpdateJob::OnManifestDataWriteComplete,
 887                    base::Unretained(this)));
 888   } else {
 889     HandleCacheFailure(
 890         AppCacheErrorDetails("Failed to write the manifest headers to storage",
 891                      APPCACHE_UNKNOWN_ERROR,
 892                      GURL(),
 893                      0,
 894                      false /*is_cross_origin*/),
 895         DISKCACHE_ERROR,
 896         GURL());
 897   }
 898 }
 899
 900 void AppCacheUpdateJob::OnManifestDataWriteComplete(int result) {
 901   if (result > 0) {
 902     AppCacheEntry entry(AppCacheEntry::MANIFEST,
 903         manifest_response_writer_->response_id(),
 904         manifest_response_writer_->amount_written());
 905     if (!inprogress_cache_->AddOrModifyEntry(manifest_url_, entry))
 906       duplicate_response_ids_.push_back(entry.response_id());
 907     StoreGroupAndCache();
 908   } else {
 909     HandleCacheFailure(
 910         AppCacheErrorDetails("Failed to write the manifest data to storage",
 911                      APPCACHE_UNKNOWN_ERROR,
 912                      GURL(),
 913                      0,
 914                      false /*is_cross_origin*/),
 915         DISKCACHE_ERROR,
 916         GURL());
 917   }
 918 }
 919
 920 void AppCacheUpdateJob::StoreGroupAndCache() {
 921   DCHECK(stored_state_ == UNSTORED);
 922   stored_state_ = STORING;
 923   scoped_refptr<AppCache> newest_cache;
 924   if (inprogress_cache_.get())
 925     newest_cache.swap(inprogress_cache_);
 926   else
 927     newest_cache = group_->newest_complete_cache();
 928   newest_cache->set_update_time(base::Time::Now());
 929
 930   // TODO(michaeln): dcheck is fishing for clues to crbug/95101
 931   DCHECK_EQ(manifest_url_, group_->manifest_url());
 932   storage_->StoreGroupAndNewestCache(group_, newest_cache.get(), this);
 933 }
 934
 935 void AppCacheUpdateJob::OnGroupAndNewestCacheStored(AppCacheGroup* group,
 936                                                     AppCache* newest_cache,
 937                                                     bool success,
 938                                                     bool would_exceed_quota) {
 939   DCHECK(stored_state_ == STORING);
 940   if (success) {
 941     stored_state_ = STORED;
 942     MaybeCompleteUpdate();  // will definitely complete
 943   } else {
 944     stored_state_ = UNSTORED;
 945
 946     // Restore inprogress_cache_ to get the proper events delivered
 947     // and the proper cleanup to occur.
 948     if (newest_cache != group->newest_complete_cache())
 949       inprogress_cache_ = newest_cache;
 950
 951     ResultType result = DB_ERROR;
 952     AppCacheErrorReason reason = APPCACHE_UNKNOWN_ERROR;
 953     std::string message("Failed to commit new cache to storage");
 954     if (would_exceed_quota) {
 955       message.append(", would exceed quota");
 956       result = QUOTA_ERROR;
 957       reason = APPCACHE_QUOTA_ERROR;
 958     }
 959     HandleCacheFailure(
 960         AppCacheErrorDetails(message, reason, GURL(), 0,
 961             false /*is_cross_origin*/),
 962         result,
 963         GURL());
 964   }
 965 }
 966
 967 void AppCacheUpdateJob::NotifySingleHost(AppCacheHost* host,
 968                                          AppCacheEventID event_id) {
 969   std::vector<int> ids(1, host->host_id());
 970   host->frontend()->OnEventRaised(ids, event_id);
 971 }
 972
 973 void AppCacheUpdateJob::NotifyAllAssociatedHosts(AppCacheEventID event_id) {
 974   HostNotifier host_notifier;
 975   AddAllAssociatedHostsToNotifier(&host_notifier);
 976   host_notifier.SendNotifications(event_id);
 977 }
 978
 979 void AppCacheUpdateJob::NotifyAllProgress(const GURL& url) {
 980   HostNotifier host_notifier;
 981   AddAllAssociatedHostsToNotifier(&host_notifier);
 982   host_notifier.SendProgressNotifications(
 983       url, url_file_list_.size(), url_fetches_completed_);
 984 }
 985
 986 void AppCacheUpdateJob::NotifyAllFinalProgress() {
 987   DCHECK(url_file_list_.size() == url_fetches_completed_);
 988   NotifyAllProgress(GURL());
 989 }
 990
 991 void AppCacheUpdateJob::NotifyAllError(const AppCacheErrorDetails& details) {
 992   HostNotifier host_notifier;
 993   AddAllAssociatedHostsToNotifier(&host_notifier);
 994   host_notifier.SendErrorNotifications(details);
 995 }
 996
 997 void AppCacheUpdateJob::LogConsoleMessageToAll(const std::string& message) {
 998   HostNotifier host_notifier;
 999   AddAllAssociatedHostsToNotifier(&host_notifier);
1000   host_notifier.SendLogMessage(message);
1001 }
1002
1003 void AppCacheUpdateJob::AddAllAssociatedHostsToNotifier(
1004     HostNotifier* host_notifier) {
1005   // Collect hosts so we only send one notification per frontend.
1006   // A host can only be associated with a single cache so no need to worry
1007   // about duplicate hosts being added to the notifier.
1008   if (inprogress_cache_.get()) {
1009     DCHECK(internal_state_ == DOWNLOADING || internal_state_ == CACHE_FAILURE);
1010     host_notifier->AddHosts(inprogress_cache_->associated_hosts());
1011   }
1012
1013   AppCacheGroup::Caches old_caches = group_->old_caches();
1014   for (AppCacheGroup::Caches::const_iterator it = old_caches.begin();
1015        it != old_caches.end(); ++it) {
1016     host_notifier->AddHosts((*it)->associated_hosts());
1017   }
1018
1019   AppCache* newest_cache = group_->newest_complete_cache();
1020   if (newest_cache)
1021     host_notifier->AddHosts(newest_cache->associated_hosts());
1022 }
1023
1024 void AppCacheUpdateJob::OnDestructionImminent(AppCacheHost* host) {
1025   // The host is about to be deleted; remove from our collection.
1026   PendingMasters::iterator found =
1027       pending_master_entries_.find(host->pending_master_entry_url());
1028   DCHECK(found != pending_master_entries_.end());
1029   PendingHosts& hosts = found->second;
1030   PendingHosts::iterator it = std::find(hosts.begin(), hosts.end(), host);
1031   DCHECK(it != hosts.end());
1032   hosts.erase(it);
1033 }
1034
1035 void AppCacheUpdateJob::OnServiceReinitialized(
1036     AppCacheStorageReference* old_storage_ref) {
1037   // We continue to use the disabled instance, but arrange for its
1038   // deletion when its no longer needed.
1039   if (old_storage_ref->storage() == storage_)
1040     disabled_storage_reference_ = old_storage_ref;
1041 }
1042
1043 void AppCacheUpdateJob::CheckIfManifestChanged() {
1044   DCHECK(update_type_ == UPGRADE_ATTEMPT);
1045   AppCacheEntry* entry = NULL;
1046   if (group_->newest_complete_cache())
1047     entry = group_->newest_complete_cache()->GetEntry(manifest_url_);
1048   if (!entry) {
1049     // TODO(michaeln): This is just a bandaid to avoid a crash.
1050     // http://code.google.com/p/chromium/issues/detail?id=95101
1051     if (service_->storage() == storage_) {
1052       // Use a local variable because service_ is reset in HandleCacheFailure.
1053       AppCacheServiceImpl* service = service_;
1054       HandleCacheFailure(
1055           AppCacheErrorDetails("Manifest entry not found in existing cache",
1056                        APPCACHE_UNKNOWN_ERROR,
1057                        GURL(),
1058                        0,
1059                        false /*is_cross_origin*/),
1060           DB_ERROR,
1061           GURL());
1062       AppCacheHistograms::AddMissingManifestEntrySample();
1063       service->DeleteAppCacheGroup(manifest_url_, net::CompletionCallback());
1064     }
1065     return;
1066   }
1067
1068   // Load manifest data from storage to compare against fetched manifest.
1069   manifest_response_reader_.reset(
1070       storage_->CreateResponseReader(manifest_url_,
1071                                      group_->group_id(),
1072                                      entry->response_id()));
1073   read_manifest_buffer_ = new net::IOBuffer(kBufferSize);
1074   manifest_response_reader_->ReadData(
1075       read_manifest_buffer_.get(),
1076       kBufferSize,
1077       base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
1078                  base::Unretained(this)));  // async read
1079 }
1080
1081 void AppCacheUpdateJob::OnManifestDataReadComplete(int result) {
1082   if (result > 0) {
1083     loaded_manifest_data_.append(read_manifest_buffer_->data(), result);
1084     manifest_response_reader_->ReadData(
1085         read_manifest_buffer_.get(),
1086         kBufferSize,
1087         base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
1088                    base::Unretained(this)));  // read more
1089   } else {
1090     read_manifest_buffer_ = NULL;
1091     manifest_response_reader_.reset();
1092     ContinueHandleManifestFetchCompleted(
1093         result < 0 || manifest_data_ != loaded_manifest_data_);
1094   }
1095 }
1096
1097 void AppCacheUpdateJob::BuildUrlFileList(const Manifest& manifest) {
1098   for (base::hash_set<std::string>::const_iterator it =
1099            manifest.explicit_urls.begin();
1100        it != manifest.explicit_urls.end(); ++it) {
1101     AddUrlToFileList(GURL(*it), AppCacheEntry::EXPLICIT);
1102   }
1103
1104   const std::vector<AppCacheNamespace>& intercepts =
1105       manifest.intercept_namespaces;
1106   for (std::vector<AppCacheNamespace>::const_iterator it = intercepts.begin();
1107        it != intercepts.end(); ++it) {
1108     int flags = AppCacheEntry::INTERCEPT;
1109     if (it->is_executable)
1110       flags |= AppCacheEntry::EXECUTABLE;
1111     AddUrlToFileList(it->target_url, flags);
1112   }
1113
1114   const std::vector<AppCacheNamespace>& fallbacks =
1115       manifest.fallback_namespaces;
1116   for (std::vector<AppCacheNamespace>::const_iterator it = fallbacks.begin();
1117        it != fallbacks.end(); ++it) {
1118      AddUrlToFileList(it->target_url, AppCacheEntry::FALLBACK);
1119   }
1120
1121   // Add all master entries from newest complete cache.
1122   if (update_type_ == UPGRADE_ATTEMPT) {
1123     const AppCache::EntryMap& entries =
1124         group_->newest_complete_cache()->entries();
1125     for (AppCache::EntryMap::const_iterator it = entries.begin();
1126          it != entries.end(); ++it) {
1127       const AppCacheEntry& entry = it->second;
1128       if (entry.IsMaster())
1129         AddUrlToFileList(it->first, AppCacheEntry::MASTER);
1130     }
1131   }
1132 }
1133
1134 void AppCacheUpdateJob::AddUrlToFileList(const GURL& url, int type) {
1135   std::pair<AppCache::EntryMap::iterator, bool> ret = url_file_list_.insert(
1136       AppCache::EntryMap::value_type(url, AppCacheEntry(type)));
1137
1138   if (ret.second)
1139     urls_to_fetch_.push_back(UrlToFetch(url, false, NULL));
1140   else
1141     ret.first->second.add_types(type);  // URL already exists. Merge types.
1142 }
1143
1144 void AppCacheUpdateJob::FetchUrls() {
1145   DCHECK(internal_state_ == DOWNLOADING);
1146
1147   // Fetch each URL in the list according to section 6.9.4 step 17.1-17.3.
1148   // Fetch up to the concurrent limit. Other fetches will be triggered as each
1149   // each fetch completes.
1150   while (pending_url_fetches_.size() < kMaxConcurrentUrlFetches &&
1151          !urls_to_fetch_.empty()) {
1152     UrlToFetch url_to_fetch = urls_to_fetch_.front();
1153     urls_to_fetch_.pop_front();
1154
1155     AppCache::EntryMap::iterator it = url_file_list_.find(url_to_fetch.url);
1156     DCHECK(it != url_file_list_.end());
1157     AppCacheEntry& entry = it->second;
1158     if (ShouldSkipUrlFetch(entry)) {
1159       NotifyAllProgress(url_to_fetch.url);
1160       ++url_fetches_completed_;
1161     } else if (AlreadyFetchedEntry(url_to_fetch.url, entry.types())) {
1162       NotifyAllProgress(url_to_fetch.url);
1163       ++url_fetches_completed_;  // saved a URL request
1164     } else if (!url_to_fetch.storage_checked &&
1165                MaybeLoadFromNewestCache(url_to_fetch.url, entry)) {
1166       // Continues asynchronously after data is loaded from newest cache.
1167     } else {
1168       URLFetcher* fetcher = new URLFetcher(
1169           url_to_fetch.url, URLFetcher::URL_FETCH, this);
1170       if (url_to_fetch.existing_response_info.get()) {
1171         DCHECK(group_->newest_complete_cache());
1172         AppCacheEntry* existing_entry =
1173             group_->newest_complete_cache()->GetEntry(url_to_fetch.url);
1174         DCHECK(existing_entry);
1175         DCHECK(existing_entry->response_id() ==
1176                url_to_fetch.existing_response_info->response_id());
1177         fetcher->set_existing_response_headers(
1178             url_to_fetch.existing_response_info->http_response_info()->headers
1179                 .get());
1180         fetcher->set_existing_entry(*existing_entry);
1181       }
1182       fetcher->Start();
1183       pending_url_fetches_.insert(
1184           PendingUrlFetches::value_type(url_to_fetch.url, fetcher));
1185     }
1186   }
1187 }
1188
1189 void AppCacheUpdateJob::CancelAllUrlFetches() {
1190   // Cancel any pending URL requests.
1191   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
1192        it != pending_url_fetches_.end(); ++it) {
1193     delete it->second;
1194   }
1195
1196   url_fetches_completed_ +=
1197       pending_url_fetches_.size() + urls_to_fetch_.size();
1198   pending_url_fetches_.clear();
1199   urls_to_fetch_.clear();
1200 }
1201
1202 bool AppCacheUpdateJob::ShouldSkipUrlFetch(const AppCacheEntry& entry) {
1203   // 6.6.4 Step 17
1204   // If the resource URL being processed was flagged as neither an
1205   // "explicit entry" nor or a "fallback entry", then the user agent
1206   // may skip this URL.
1207   if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept())
1208     return false;
1209
1210   // TODO(jennb): decide if entry should be skipped to expire it from cache
1211   return false;
1212 }
1213
1214 bool AppCacheUpdateJob::AlreadyFetchedEntry(const GURL& url,
1215                                             int entry_type) {
1216   DCHECK(internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE);
1217   AppCacheEntry* existing =
1218       inprogress_cache_.get() ? inprogress_cache_->GetEntry(url)
1219                               : group_->newest_complete_cache()->GetEntry(url);
1220   if (existing) {
1221     existing->add_types(entry_type);
1222     return true;
1223   }
1224   return false;
1225 }
1226
1227 void AppCacheUpdateJob::AddMasterEntryToFetchList(AppCacheHost* host,
1228                                                   const GURL& url,
1229                                                   bool is_new) {
1230   DCHECK(!IsTerminating());
1231
1232   if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE) {
1233     AppCache* cache;
1234     if (inprogress_cache_.get()) {
1235       // always associate
1236       host->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
1237       cache = inprogress_cache_.get();
1238     } else {
1239       cache = group_->newest_complete_cache();
1240     }
1241
1242     // Update existing entry if it has already been fetched.
1243     AppCacheEntry* entry = cache->GetEntry(url);
1244     if (entry) {
1245       entry->add_types(AppCacheEntry::MASTER);
1246       if (internal_state_ == NO_UPDATE && !inprogress_cache_.get()) {
1247         // only associate if have entry
1248         host->AssociateCompleteCache(cache);
1249       }
1250       if (is_new)
1251         ++master_entries_completed_;  // pretend fetching completed
1252       return;
1253     }
1254   }
1255
1256   // Add to fetch list if not already fetching.
1257   if (master_entry_fetches_.find(url) == master_entry_fetches_.end()) {
1258     master_entries_to_fetch_.insert(url);
1259     if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE)
1260       FetchMasterEntries();
1261   }
1262 }
1263
1264 void AppCacheUpdateJob::FetchMasterEntries() {
1265   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
1266
1267   // Fetch each master entry in the list, up to the concurrent limit.
1268   // Additional fetches will be triggered as each fetch completes.
1269   while (master_entry_fetches_.size() < kMaxConcurrentUrlFetches &&
1270          !master_entries_to_fetch_.empty()) {
1271     const GURL& url = *master_entries_to_fetch_.begin();
1272
1273     if (AlreadyFetchedEntry(url, AppCacheEntry::MASTER)) {
1274       ++master_entries_completed_;  // saved a URL request
1275
1276       // In no update case, associate hosts to newest cache in group
1277       // now that master entry has been "successfully downloaded".
1278       if (internal_state_ == NO_UPDATE) {
1279         // TODO(michaeln): defer until the updated cache has been stored.
1280         DCHECK(!inprogress_cache_.get());
1281         AppCache* cache = group_->newest_complete_cache();
1282         PendingMasters::iterator found = pending_master_entries_.find(url);
1283         DCHECK(found != pending_master_entries_.end());
1284         PendingHosts& hosts = found->second;
1285         for (PendingHosts::iterator host_it = hosts.begin();
1286              host_it != hosts.end(); ++host_it) {
1287           (*host_it)->AssociateCompleteCache(cache);
1288         }
1289       }
1290     } else {
1291       URLFetcher* fetcher = new URLFetcher(
1292           url, URLFetcher::MASTER_ENTRY_FETCH, this);
1293       fetcher->Start();
1294       master_entry_fetches_.insert(PendingUrlFetches::value_type(url, fetcher));
1295     }
1296
1297     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
1298   }
1299 }
1300
1301 void AppCacheUpdateJob::CancelAllMasterEntryFetches(
1302     const AppCacheErrorDetails& error_details) {
1303   // For now, cancel all in-progress fetches for master entries and pretend
1304   // all master entries fetches have completed.
1305   // TODO(jennb): Delete this when update no longer fetches master entries
1306   // directly.
1307
1308   // Cancel all in-progress fetches.
1309   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
1310        it != master_entry_fetches_.end(); ++it) {
1311     delete it->second;
1312     master_entries_to_fetch_.insert(it->first);  // back in unfetched list
1313   }
1314   master_entry_fetches_.clear();
1315
1316   master_entries_completed_ += master_entries_to_fetch_.size();
1317
1318   // Cache failure steps, step 2.
1319   // Pretend all master entries that have not yet been fetched have completed
1320   // downloading. Unassociate hosts from any appcache and send ERROR event.
1321   HostNotifier host_notifier;
1322   while (!master_entries_to_fetch_.empty()) {
1323     const GURL& url = *master_entries_to_fetch_.begin();
1324     PendingMasters::iterator found = pending_master_entries_.find(url);
1325     DCHECK(found != pending_master_entries_.end());
1326     PendingHosts& hosts = found->second;
1327     for (PendingHosts::iterator host_it = hosts.begin();
1328          host_it != hosts.end(); ++host_it) {
1329       AppCacheHost* host = *host_it;
1330       host->AssociateNoCache(GURL());
1331       host_notifier.AddHost(host);
1332       host->RemoveObserver(this);
1333     }
1334     hosts.clear();
1335
1336     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
1337   }
1338   host_notifier.SendErrorNotifications(error_details);
1339 }
1340
1341 bool AppCacheUpdateJob::MaybeLoadFromNewestCache(const GURL& url,
1342                                                  AppCacheEntry& entry) {
1343   if (update_type_ != UPGRADE_ATTEMPT)
1344     return false;
1345
1346   AppCache* newest = group_->newest_complete_cache();
1347   AppCacheEntry* copy_me = newest->GetEntry(url);
1348   if (!copy_me || !copy_me->has_response_id())
1349     return false;
1350
1351   // Load HTTP headers for entry from newest cache.
1352   loading_responses_.insert(
1353       LoadingResponses::value_type(copy_me->response_id(), url));
1354   storage_->LoadResponseInfo(manifest_url_, group_->group_id(),
1355                              copy_me->response_id(),
1356                              this);
1357   // Async: wait for OnResponseInfoLoaded to complete.
1358   return true;
1359 }
1360
1361 void AppCacheUpdateJob::OnResponseInfoLoaded(
1362     AppCacheResponseInfo* response_info, int64 response_id) {
1363   const net::HttpResponseInfo* http_info = response_info ?
1364       response_info->http_response_info() : NULL;
1365
1366   // Needed response info for a manifest fetch request.
1367   if (internal_state_ == FETCH_MANIFEST) {
1368     if (http_info)
1369       manifest_fetcher_->set_existing_response_headers(
1370           http_info->headers.get());
1371     manifest_fetcher_->Start();
1372     return;
1373   }
1374
1375   LoadingResponses::iterator found = loading_responses_.find(response_id);
1376   DCHECK(found != loading_responses_.end());
1377   const GURL& url = found->second;
1378
1379   if (!http_info) {
1380     LoadFromNewestCacheFailed(url, NULL);  // no response found
1381   } else {
1382     // Check if response can be re-used according to HTTP caching semantics.
1383     // Responses with a "vary" header get treated as expired.
1384     const std::string name = "vary";
1385     std::string value;
1386     void* iter = NULL;
1387     if (!http_info->headers.get() ||
1388         http_info->headers->RequiresValidation(http_info->request_time,
1389                                                http_info->response_time,
1390                                                base::Time::Now()) ||
1391         http_info->headers->EnumerateHeader(&iter, name, &value)) {
1392       LoadFromNewestCacheFailed(url, response_info);
1393     } else {
1394       DCHECK(group_->newest_complete_cache());
1395       AppCacheEntry* copy_me = group_->newest_complete_cache()->GetEntry(url);
1396       DCHECK(copy_me);
1397       DCHECK(copy_me->response_id() == response_id);
1398
1399       AppCache::EntryMap::iterator it = url_file_list_.find(url);
1400       DCHECK(it != url_file_list_.end());
1401       AppCacheEntry& entry = it->second;
1402       entry.set_response_id(response_id);
1403       entry.set_response_size(copy_me->response_size());
1404       inprogress_cache_->AddOrModifyEntry(url, entry);
1405       NotifyAllProgress(url);
1406       ++url_fetches_completed_;
1407     }
1408   }
1409   loading_responses_.erase(found);
1410
1411   MaybeCompleteUpdate();
1412 }
1413
1414 void AppCacheUpdateJob::LoadFromNewestCacheFailed(
1415     const GURL& url, AppCacheResponseInfo* response_info) {
1416   if (internal_state_ == CACHE_FAILURE)
1417     return;
1418
1419   // Re-insert url at front of fetch list. Indicate storage has been checked.
1420   urls_to_fetch_.push_front(UrlToFetch(url, true, response_info));
1421   FetchUrls();
1422 }
1423
1424 void AppCacheUpdateJob::MaybeCompleteUpdate() {
1425   DCHECK(internal_state_ != CACHE_FAILURE);
1426
1427   // Must wait for any pending master entries or url fetches to complete.
1428   if (master_entries_completed_ != pending_master_entries_.size() ||
1429       url_fetches_completed_ != url_file_list_.size()) {
1430     DCHECK(internal_state_ != COMPLETED);
1431     return;
1432   }
1433
1434   switch (internal_state_) {
1435     case NO_UPDATE:
1436       if (master_entries_completed_ > 0) {
1437         switch (stored_state_) {
1438           case UNSTORED:
1439             StoreGroupAndCache();
1440             return;
1441           case STORING:
1442             return;
1443           case STORED:
1444             break;
1445         }
1446       }
1447       // 6.9.4 steps 7.3-7.7.
1448       NotifyAllAssociatedHosts(APPCACHE_NO_UPDATE_EVENT);
1449       DiscardDuplicateResponses();
1450       internal_state_ = COMPLETED;
1451       break;
1452     case DOWNLOADING:
1453       internal_state_ = REFETCH_MANIFEST;
1454       FetchManifest(false);
1455       break;
1456     case REFETCH_MANIFEST:
1457       DCHECK(stored_state_ == STORED);
1458       NotifyAllFinalProgress();
1459       if (update_type_ == CACHE_ATTEMPT)
1460         NotifyAllAssociatedHosts(APPCACHE_CACHED_EVENT);
1461       else
1462         NotifyAllAssociatedHosts(APPCACHE_UPDATE_READY_EVENT);
1463       DiscardDuplicateResponses();
1464       internal_state_ = COMPLETED;
1465       LogHistogramStats(UPDATE_OK, GURL());
1466       break;
1467     case CACHE_FAILURE:
1468       NOTREACHED();  // See HandleCacheFailure
1469       break;
1470     default:
1471       break;
1472   }
1473
1474   // Let the stack unwind before deletion to make it less risky as this
1475   // method is called from multiple places in this file.
1476   if (internal_state_ == COMPLETED)
1477     DeleteSoon();
1478 }
1479
1480 void AppCacheUpdateJob::ScheduleUpdateRetry(int delay_ms) {
1481   // TODO(jennb): post a delayed task with the "same parameters" as this job
1482   // to retry the update at a later time. Need group, URLs of pending master
1483   // entries and their hosts.
1484 }
1485
1486 void AppCacheUpdateJob::Cancel() {
1487   internal_state_ = CANCELLED;
1488
1489   LogHistogramStats(CANCELLED_ERROR, GURL());
1490
1491   if (manifest_fetcher_) {
1492     delete manifest_fetcher_;
1493     manifest_fetcher_ = NULL;
1494   }
1495
1496   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
1497        it != pending_url_fetches_.end(); ++it) {
1498     delete it->second;
1499   }
1500   pending_url_fetches_.clear();
1501
1502   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
1503        it != master_entry_fetches_.end(); ++it) {
1504     delete it->second;
1505   }
1506   master_entry_fetches_.clear();
1507
1508   ClearPendingMasterEntries();
1509   DiscardInprogressCache();
1510
1511   // Delete response writer to avoid any callbacks.
1512   if (manifest_response_writer_)
1513     manifest_response_writer_.reset();
1514
1515   storage_->CancelDelegateCallbacks(this);
1516 }
1517
1518 void AppCacheUpdateJob::ClearPendingMasterEntries() {
1519   for (PendingMasters::iterator it = pending_master_entries_.begin();
1520        it != pending_master_entries_.end(); ++it) {
1521     PendingHosts& hosts = it->second;
1522     for (PendingHosts::iterator host_it = hosts.begin();
1523          host_it != hosts.end(); ++host_it) {
1524       (*host_it)->RemoveObserver(this);
1525     }
1526   }
1527
1528   pending_master_entries_.clear();
1529 }
1530
1531 void AppCacheUpdateJob::DiscardInprogressCache() {
1532   if (stored_state_ == STORING) {
1533     // We can make no assumptions about whether the StoreGroupAndCacheTask
1534     // actually completed or not. This condition should only be reachable
1535     // during shutdown. Free things up and return to do no harm.
1536     inprogress_cache_ = NULL;
1537     added_master_entries_.clear();
1538     return;
1539   }
1540
1541   storage_->DoomResponses(manifest_url_, stored_response_ids_);
1542
1543   if (!inprogress_cache_.get()) {
1544     // We have to undo the changes we made, if any, to the existing cache.
1545     if (group_ && group_->newest_complete_cache()) {
1546       for (std::vector<GURL>::iterator iter = added_master_entries_.begin();
1547            iter != added_master_entries_.end(); ++iter) {
1548         group_->newest_complete_cache()->RemoveEntry(*iter);
1549       }
1550     }
1551     added_master_entries_.clear();
1552     return;
1553   }
1554
1555   AppCache::AppCacheHosts& hosts = inprogress_cache_->associated_hosts();
1556   while (!hosts.empty())
1557     (*hosts.begin())->AssociateNoCache(GURL());
1558
1559   inprogress_cache_ = NULL;
1560   added_master_entries_.clear();
1561 }
1562
1563 void AppCacheUpdateJob::DiscardDuplicateResponses() {
1564   storage_->DoomResponses(manifest_url_, duplicate_response_ids_);
1565 }
1566
1567 void AppCacheUpdateJob::LogHistogramStats(
1568       ResultType result, const GURL& failed_resource_url) {
1569   AppCacheHistograms::CountUpdateJobResult(result, manifest_url_.GetOrigin());
1570   if (result == UPDATE_OK)
1571     return;
1572
1573   int percent_complete = 0;
1574   if (url_file_list_.size() > 0) {
1575     size_t actual_fetches_completed = url_fetches_completed_;
1576     if (!failed_resource_url.is_empty() && actual_fetches_completed)
1577       --actual_fetches_completed;
1578     percent_complete = (static_cast<double>(actual_fetches_completed) /
1579                             static_cast<double>(url_file_list_.size())) * 100.0;
1580     percent_complete = std::min(percent_complete, 99);
1581   }
1582
1583   bool was_making_progress =
1584       base::Time::Now() - last_progress_time_ <
1585           base::TimeDelta::FromMinutes(5);
1586
1587   bool off_origin_resource_failure =
1588       !failed_resource_url.is_empty() &&
1589           (failed_resource_url.GetOrigin() != manifest_url_.GetOrigin());
1590
1591   AppCacheHistograms::LogUpdateFailureStats(
1592       manifest_url_.GetOrigin(),
1593       percent_complete,
1594       was_making_progress,
1595       off_origin_resource_failure);
1596 }
1597
1598 void AppCacheUpdateJob::DeleteSoon() {
1599   ClearPendingMasterEntries();
1600   manifest_response_writer_.reset();
1601   storage_->CancelDelegateCallbacks(this);
1602   service_->RemoveObserver(this);
1603   service_ = NULL;
1604
1605   // Break the connection with the group so the group cannot call delete
1606   // on this object after we've posted a task to delete ourselves.
1607   group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE);
1608   group_ = NULL;
1609
1610   base::MessageLoop::current()->DeleteSoon(FROM_HERE, this);
1611 }
1612
1613 }  // namespace content