content/browser/appcache/appcache_update_job.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/appcache/appcache_update_job.h"
   6
   7 #include "base/bind.h"
   8 #include "base/bind_helpers.h"
   9 #include "base/compiler_specific.h"
  10 #include "base/strings/string_util.h"
  11 #include "base/strings/stringprintf.h"
  12 #include "base/thread_task_runner_handle.h"
  13 #include "content/browser/appcache/appcache_group.h"
  14 #include "content/browser/appcache/appcache_histograms.h"
  15 #include "content/public/browser/browser_thread.h"
  16 #include "net/base/io_buffer.h"
  17 #include "net/base/load_flags.h"
  18 #include "net/base/net_errors.h"
  19 #include "net/base/request_priority.h"
  20 #include "net/http/http_request_headers.h"
  21 #include "net/http/http_response_headers.h"
  22 #include "net/url_request/url_request_context.h"
  23
  24 namespace content {
  25
  26 static const int kBufferSize = 32768;
  27 static const size_t kMaxConcurrentUrlFetches = 2;
  28 static const int kMax503Retries = 3;
  29
  30 static std::string FormatUrlErrorMessage(
  31       const char* format, const GURL& url,
  32       AppCacheUpdateJob::ResultType error,
  33       int response_code) {
  34     // Show the net response code if we have one.
  35     int code = response_code;
  36     if (error != AppCacheUpdateJob::SERVER_ERROR)
  37       code = static_cast<int>(error);
  38     return base::StringPrintf(format, code, url.spec().c_str());
  39 }
  40
  41 // Helper class for collecting hosts per frontend when sending notifications
  42 // so that only one notification is sent for all hosts using the same frontend.
  43 class HostNotifier {
  44  public:
  45   typedef std::vector<int> HostIds;
  46   typedef std::map<AppCacheFrontend*, HostIds> NotifyHostMap;
  47
  48   // Caller is responsible for ensuring there will be no duplicate hosts.
  49   void AddHost(AppCacheHost* host) {
  50     std::pair<NotifyHostMap::iterator , bool> ret = hosts_to_notify.insert(
  51         NotifyHostMap::value_type(host->frontend(), HostIds()));
  52     ret.first->second.push_back(host->host_id());
  53   }
  54
  55   void AddHosts(const std::set<AppCacheHost*>& hosts) {
  56     for (std::set<AppCacheHost*>::const_iterator it = hosts.begin();
  57          it != hosts.end(); ++it) {
  58       AddHost(*it);
  59     }
  60   }
  61
  62   void SendNotifications(AppCacheEventID event_id) {
  63     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  64          it != hosts_to_notify.end(); ++it) {
  65       AppCacheFrontend* frontend = it->first;
  66       frontend->OnEventRaised(it->second, event_id);
  67     }
  68   }
  69
  70   void SendProgressNotifications(
  71       const GURL& url, int num_total, int num_complete) {
  72     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  73          it != hosts_to_notify.end(); ++it) {
  74       AppCacheFrontend* frontend = it->first;
  75       frontend->OnProgressEventRaised(it->second, url,
  76                                       num_total, num_complete);
  77     }
  78   }
  79
  80   void SendErrorNotifications(const AppCacheErrorDetails& details) {
  81     DCHECK(!details.message.empty());
  82     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  83          it != hosts_to_notify.end(); ++it) {
  84       AppCacheFrontend* frontend = it->first;
  85       frontend->OnErrorEventRaised(it->second, details);
  86     }
  87   }
  88
  89   void SendLogMessage(const std::string& message) {
  90     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  91          it != hosts_to_notify.end(); ++it) {
  92       AppCacheFrontend* frontend = it->first;
  93       for (HostIds::iterator id = it->second.begin();
  94            id != it->second.end(); ++id) {
  95         frontend->OnLogMessage(*id, APPCACHE_LOG_WARNING, message);
  96       }
  97     }
  98   }
  99
 100  private:
 101   NotifyHostMap hosts_to_notify;
 102 };
 103
 104 AppCacheUpdateJob::UrlToFetch::UrlToFetch(const GURL& url,
 105                                           bool checked,
 106                                           AppCacheResponseInfo* info)
 107     : url(url),
 108       storage_checked(checked),
 109       existing_response_info(info) {
 110 }
 111
 112 AppCacheUpdateJob::UrlToFetch::~UrlToFetch() {
 113 }
 114
 115 // Helper class to fetch resources. Depending on the fetch type,
 116 // can either fetch to an in-memory string or write the response
 117 // data out to the disk cache.
 118 AppCacheUpdateJob::URLFetcher::URLFetcher(const GURL& url,
 119                                           FetchType fetch_type,
 120                                           AppCacheUpdateJob* job)
 121     : url_(url),
 122       job_(job),
 123       fetch_type_(fetch_type),
 124       retry_503_attempts_(0),
 125       buffer_(new net::IOBuffer(kBufferSize)),
 126       request_(job->service_->request_context()
 127                    ->CreateRequest(url, net::DEFAULT_PRIORITY, this)),
 128       result_(UPDATE_OK),
 129       redirect_response_code_(-1) {}
 130
 131 AppCacheUpdateJob::URLFetcher::~URLFetcher() {
 132 }
 133
 134 void AppCacheUpdateJob::URLFetcher::Start() {
 135   request_->set_first_party_for_cookies(job_->manifest_url_);
 136   if (existing_response_headers_.get())
 137     AddConditionalHeaders(existing_response_headers_.get());
 138   request_->Start();
 139 }
 140
 141 void AppCacheUpdateJob::URLFetcher::OnReceivedRedirect(
 142     net::URLRequest* request,
 143     const net::RedirectInfo& redirect_info,
 144     bool* defer_redirect) {
 145   DCHECK(request_ == request);
 146   // Redirect is not allowed by the update process.
 147   job_->MadeProgress();
 148   redirect_response_code_ = request->GetResponseCode();
 149   request->Cancel();
 150   result_ = REDIRECT_ERROR;
 151   OnResponseCompleted();
 152 }
 153
 154 void AppCacheUpdateJob::URLFetcher::OnResponseStarted(
 155     net::URLRequest *request) {
 156   DCHECK(request == request_);
 157   int response_code = -1;
 158   if (request->status().is_success()) {
 159     response_code = request->GetResponseCode();
 160     job_->MadeProgress();
 161   }
 162
 163   if ((response_code / 100) != 2) {
 164     if (response_code > 0)
 165       result_ = SERVER_ERROR;
 166     else
 167       result_ = NETWORK_ERROR;
 168     OnResponseCompleted();
 169     return;
 170   }
 171
 172   if (url_.SchemeIsSecure()) {
 173     // Do not cache content with cert errors.
 174     // Also, we willfully violate the HTML5 spec at this point in order
 175     // to support the appcaching of cross-origin HTTPS resources.
 176     // We've opted for a milder constraint and allow caching unless
 177     // the resource has a "no-store" header. A spec change has been
 178     // requested on the whatwg list.
 179     // See http://code.google.com/p/chromium/issues/detail?id=69594
 180     // TODO(michaeln): Consider doing this for cross-origin HTTP too.
 181     const net::HttpNetworkSession::Params* session_params =
 182         request->context()->GetNetworkSessionParams();
 183     bool ignore_cert_errors = session_params &&
 184                               session_params->ignore_certificate_errors;
 185     if ((net::IsCertStatusError(request->ssl_info().cert_status) &&
 186             !ignore_cert_errors) ||
 187         (url_.GetOrigin() != job_->manifest_url_.GetOrigin() &&
 188             request->response_headers()->
 189                 HasHeaderValue("cache-control", "no-store"))) {
 190       DCHECK_EQ(-1, redirect_response_code_);
 191       request->Cancel();
 192       result_ = SECURITY_ERROR;
 193       OnResponseCompleted();
 194       return;
 195     }
 196   }
 197
 198   // Write response info to storage for URL fetches. Wait for async write
 199   // completion before reading any response data.
 200   if (fetch_type_ == URL_FETCH || fetch_type_ == MASTER_ENTRY_FETCH) {
 201     response_writer_.reset(job_->CreateResponseWriter());
 202     scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
 203         new HttpResponseInfoIOBuffer(
 204             new net::HttpResponseInfo(request->response_info())));
 205     response_writer_->WriteInfo(
 206         io_buffer.get(),
 207         base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
 208   } else {
 209     ReadResponseData();
 210   }
 211 }
 212
 213 void AppCacheUpdateJob::URLFetcher::OnReadCompleted(
 214     net::URLRequest* request, int bytes_read) {
 215   DCHECK(request_ == request);
 216   bool data_consumed = true;
 217   if (request->status().is_success() && bytes_read > 0) {
 218     job_->MadeProgress();
 219     data_consumed = ConsumeResponseData(bytes_read);
 220     if (data_consumed) {
 221       bytes_read = 0;
 222       while (request->Read(buffer_.get(), kBufferSize, &bytes_read)) {
 223         if (bytes_read > 0) {
 224           data_consumed = ConsumeResponseData(bytes_read);
 225           if (!data_consumed)
 226             break;  // wait for async data processing, then read more
 227         } else {
 228           break;
 229         }
 230       }
 231     }
 232   }
 233   if (data_consumed && !request->status().is_io_pending()) {
 234     DCHECK_EQ(UPDATE_OK, result_);
 235     OnResponseCompleted();
 236   }
 237 }
 238
 239 void AppCacheUpdateJob::URLFetcher::AddConditionalHeaders(
 240     const net::HttpResponseHeaders* headers) {
 241   DCHECK(request_.get() && headers);
 242   net::HttpRequestHeaders extra_headers;
 243
 244   // Add If-Modified-Since header if response info has Last-Modified header.
 245   const std::string last_modified = "Last-Modified";
 246   std::string last_modified_value;
 247   headers->EnumerateHeader(NULL, last_modified, &last_modified_value);
 248   if (!last_modified_value.empty()) {
 249     extra_headers.SetHeader(net::HttpRequestHeaders::kIfModifiedSince,
 250                             last_modified_value);
 251   }
 252
 253   // Add If-None-Match header if response info has ETag header.
 254   const std::string etag = "ETag";
 255   std::string etag_value;
 256   headers->EnumerateHeader(NULL, etag, &etag_value);
 257   if (!etag_value.empty()) {
 258     extra_headers.SetHeader(net::HttpRequestHeaders::kIfNoneMatch,
 259                             etag_value);
 260   }
 261   if (!extra_headers.IsEmpty())
 262     request_->SetExtraRequestHeaders(extra_headers);
 263 }
 264
 265 void  AppCacheUpdateJob::URLFetcher::OnWriteComplete(int result) {
 266   if (result < 0) {
 267     request_->Cancel();
 268     result_ = DISKCACHE_ERROR;
 269     OnResponseCompleted();
 270     return;
 271   }
 272   ReadResponseData();
 273 }
 274
 275 void AppCacheUpdateJob::URLFetcher::ReadResponseData() {
 276   InternalUpdateState state = job_->internal_state_;
 277   if (state == CACHE_FAILURE || state == CANCELLED || state == COMPLETED)
 278     return;
 279   int bytes_read = 0;
 280   request_->Read(buffer_.get(), kBufferSize, &bytes_read);
 281   OnReadCompleted(request_.get(), bytes_read);
 282 }
 283
 284 // Returns false if response data is processed asynchronously, in which
 285 // case ReadResponseData will be invoked when it is safe to continue
 286 // reading more response data from the request.
 287 bool AppCacheUpdateJob::URLFetcher::ConsumeResponseData(int bytes_read) {
 288   DCHECK_GT(bytes_read, 0);
 289   switch (fetch_type_) {
 290     case MANIFEST_FETCH:
 291     case MANIFEST_REFETCH:
 292       manifest_data_.append(buffer_->data(), bytes_read);
 293       break;
 294     case URL_FETCH:
 295     case MASTER_ENTRY_FETCH:
 296       DCHECK(response_writer_.get());
 297       response_writer_->WriteData(
 298           buffer_.get(),
 299           bytes_read,
 300           base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
 301       return false;  // wait for async write completion to continue reading
 302     default:
 303       NOTREACHED();
 304   }
 305   return true;
 306 }
 307
 308 void AppCacheUpdateJob::URLFetcher::OnResponseCompleted() {
 309   if (request_->status().is_success())
 310     job_->MadeProgress();
 311
 312   // Retry for 503s where retry-after is 0.
 313   if (request_->status().is_success() &&
 314       request_->GetResponseCode() == 503 &&
 315       MaybeRetryRequest()) {
 316     return;
 317   }
 318
 319   switch (fetch_type_) {
 320     case MANIFEST_FETCH:
 321       job_->HandleManifestFetchCompleted(this);
 322       break;
 323     case URL_FETCH:
 324       job_->HandleUrlFetchCompleted(this);
 325       break;
 326     case MASTER_ENTRY_FETCH:
 327       job_->HandleMasterEntryFetchCompleted(this);
 328       break;
 329     case MANIFEST_REFETCH:
 330       job_->HandleManifestRefetchCompleted(this);
 331       break;
 332     default:
 333       NOTREACHED();
 334   }
 335
 336   delete this;
 337 }
 338
 339 bool AppCacheUpdateJob::URLFetcher::MaybeRetryRequest() {
 340   if (retry_503_attempts_ >= kMax503Retries ||
 341       !request_->response_headers()->HasHeaderValue("retry-after", "0")) {
 342     return false;
 343   }
 344   ++retry_503_attempts_;
 345   result_ = UPDATE_OK;
 346   request_ = job_->service_->request_context()->CreateRequest(
 347       url_, net::DEFAULT_PRIORITY, this);
 348   Start();
 349   return true;
 350 }
 351
 352 AppCacheUpdateJob::AppCacheUpdateJob(AppCacheServiceImpl* service,
 353                                      AppCacheGroup* group)
 354     : service_(service),
 355       manifest_url_(group->manifest_url()),
 356       group_(group),
 357       update_type_(UNKNOWN_TYPE),
 358       internal_state_(FETCH_MANIFEST),
 359       master_entries_completed_(0),
 360       url_fetches_completed_(0),
 361       manifest_fetcher_(NULL),
 362       manifest_has_valid_mime_type_(false),
 363       stored_state_(UNSTORED),
 364       storage_(service->storage()),
 365       weak_factory_(this) {
 366     service_->AddObserver(this);
 367 }
 368
 369 AppCacheUpdateJob::~AppCacheUpdateJob() {
 370   if (service_)
 371     service_->RemoveObserver(this);
 372   if (internal_state_ != COMPLETED)
 373     Cancel();
 374
 375   DCHECK(!manifest_fetcher_);
 376   DCHECK(pending_url_fetches_.empty());
 377   DCHECK(!inprogress_cache_.get());
 378   DCHECK(pending_master_entries_.empty());
 379   DCHECK(master_entry_fetches_.empty());
 380
 381   if (group_)
 382     group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE);
 383 }
 384
 385 void AppCacheUpdateJob::StartUpdate(AppCacheHost* host,
 386                                     const GURL& new_master_resource) {
 387   DCHECK(group_->update_job() == this);
 388   DCHECK(!group_->is_obsolete());
 389
 390   bool is_new_pending_master_entry = false;
 391   if (!new_master_resource.is_empty()) {
 392     DCHECK(new_master_resource == host->pending_master_entry_url());
 393     DCHECK(!new_master_resource.has_ref());
 394     DCHECK(new_master_resource.GetOrigin() == manifest_url_.GetOrigin());
 395
 396     // Cannot add more to this update if already terminating.
 397     if (IsTerminating()) {
 398       group_->QueueUpdate(host, new_master_resource);
 399       return;
 400     }
 401
 402     std::pair<PendingMasters::iterator, bool> ret =
 403         pending_master_entries_.insert(
 404             PendingMasters::value_type(new_master_resource, PendingHosts()));
 405     is_new_pending_master_entry = ret.second;
 406     ret.first->second.push_back(host);
 407     host->AddObserver(this);
 408   }
 409
 410   // Notify host (if any) if already checking or downloading.
 411   AppCacheGroup::UpdateAppCacheStatus update_status = group_->update_status();
 412   if (update_status == AppCacheGroup::CHECKING ||
 413       update_status == AppCacheGroup::DOWNLOADING) {
 414     if (host) {
 415       NotifySingleHost(host, APPCACHE_CHECKING_EVENT);
 416       if (update_status == AppCacheGroup::DOWNLOADING)
 417         NotifySingleHost(host, APPCACHE_DOWNLOADING_EVENT);
 418
 419       // Add to fetch list or an existing entry if already fetched.
 420       if (!new_master_resource.is_empty()) {
 421         AddMasterEntryToFetchList(host, new_master_resource,
 422                                   is_new_pending_master_entry);
 423       }
 424     }
 425     return;
 426   }
 427
 428   // Begin update process for the group.
 429   MadeProgress();
 430   group_->SetUpdateAppCacheStatus(AppCacheGroup::CHECKING);
 431   if (group_->HasCache()) {
 432     update_type_ = UPGRADE_ATTEMPT;
 433     NotifyAllAssociatedHosts(APPCACHE_CHECKING_EVENT);
 434   } else {
 435     update_type_ = CACHE_ATTEMPT;
 436     DCHECK(host);
 437     NotifySingleHost(host, APPCACHE_CHECKING_EVENT);
 438   }
 439
 440   if (!new_master_resource.is_empty()) {
 441     AddMasterEntryToFetchList(host, new_master_resource,
 442                               is_new_pending_master_entry);
 443   }
 444
 445   BrowserThread::PostAfterStartupTask(
 446       FROM_HERE, base::ThreadTaskRunnerHandle::Get(),
 447       base::Bind(&AppCacheUpdateJob::FetchManifest, weak_factory_.GetWeakPtr(),
 448                  true));
 449 }
 450
 451 AppCacheResponseWriter* AppCacheUpdateJob::CreateResponseWriter() {
 452   AppCacheResponseWriter* writer =
 453       storage_->CreateResponseWriter(manifest_url_,
 454                                                 group_->group_id());
 455   stored_response_ids_.push_back(writer->response_id());
 456   return writer;
 457 }
 458
 459 void AppCacheUpdateJob::HandleCacheFailure(
 460     const AppCacheErrorDetails& error_details,
 461     ResultType result,
 462     const GURL& failed_resource_url) {
 463   // 6.9.4 cache failure steps 2-8.
 464   DCHECK(internal_state_ != CACHE_FAILURE);
 465   DCHECK(!error_details.message.empty());
 466   DCHECK(result != UPDATE_OK);
 467   internal_state_ = CACHE_FAILURE;
 468   LogHistogramStats(result, failed_resource_url);
 469   CancelAllUrlFetches();
 470   CancelAllMasterEntryFetches(error_details);
 471   NotifyAllError(error_details);
 472   DiscardInprogressCache();
 473   internal_state_ = COMPLETED;
 474   DeleteSoon();  // To unwind the stack prior to deletion.
 475 }
 476
 477 void AppCacheUpdateJob::FetchManifest(bool is_first_fetch) {
 478   DCHECK(!manifest_fetcher_);
 479   manifest_fetcher_ = new URLFetcher(
 480      manifest_url_,
 481      is_first_fetch ? URLFetcher::MANIFEST_FETCH :
 482                       URLFetcher::MANIFEST_REFETCH,
 483      this);
 484
 485   // Add any necessary Http headers before sending fetch request.
 486   if (is_first_fetch) {
 487     AppCacheEntry* entry = (update_type_ == UPGRADE_ATTEMPT) ?
 488         group_->newest_complete_cache()->GetEntry(manifest_url_) : NULL;
 489     if (entry) {
 490       // Asynchronously load response info for manifest from newest cache.
 491       storage_->LoadResponseInfo(manifest_url_, group_->group_id(),
 492                                  entry->response_id(), this);
 493     } else {
 494       manifest_fetcher_->Start();
 495     }
 496   } else {
 497     DCHECK(internal_state_ == REFETCH_MANIFEST);
 498     DCHECK(manifest_response_info_.get());
 499     manifest_fetcher_->set_existing_response_headers(
 500         manifest_response_info_->headers.get());
 501     manifest_fetcher_->Start();
 502   }
 503 }
 504
 505
 506 void AppCacheUpdateJob::HandleManifestFetchCompleted(
 507     URLFetcher* fetcher) {
 508   DCHECK_EQ(internal_state_, FETCH_MANIFEST);
 509   DCHECK_EQ(manifest_fetcher_, fetcher);
 510   manifest_fetcher_ = NULL;
 511
 512   net::URLRequest* request = fetcher->request();
 513   int response_code = -1;
 514   bool is_valid_response_code = false;
 515   if (request->status().is_success()) {
 516     response_code = request->GetResponseCode();
 517     is_valid_response_code = (response_code / 100 == 2);
 518
 519     std::string mime_type;
 520     request->GetMimeType(&mime_type);
 521     manifest_has_valid_mime_type_ = (mime_type == "text/cache-manifest");
 522   }
 523
 524   if (is_valid_response_code) {
 525     manifest_data_ = fetcher->manifest_data();
 526     manifest_response_info_.reset(
 527         new net::HttpResponseInfo(request->response_info()));
 528     if (update_type_ == UPGRADE_ATTEMPT)
 529       CheckIfManifestChanged();  // continues asynchronously
 530     else
 531       ContinueHandleManifestFetchCompleted(true);
 532   } else if (response_code == 304 && update_type_ == UPGRADE_ATTEMPT) {
 533     ContinueHandleManifestFetchCompleted(false);
 534   } else if ((response_code == 404 || response_code == 410) &&
 535              update_type_ == UPGRADE_ATTEMPT) {
 536     storage_->MakeGroupObsolete(group_, this, response_code);  // async
 537   } else {
 538     const char* kFormatString = "Manifest fetch failed (%d) %s";
 539     std::string message = FormatUrlErrorMessage(
 540         kFormatString, manifest_url_, fetcher->result(), response_code);
 541     HandleCacheFailure(AppCacheErrorDetails(message,
 542                                     APPCACHE_MANIFEST_ERROR,
 543                                     manifest_url_,
 544                                     response_code,
 545                                     false /*is_cross_origin*/),
 546                        fetcher->result(),
 547                        GURL());
 548   }
 549 }
 550
 551 void AppCacheUpdateJob::OnGroupMadeObsolete(AppCacheGroup* group,
 552                                             bool success,
 553                                             int response_code) {
 554   DCHECK(master_entry_fetches_.empty());
 555   CancelAllMasterEntryFetches(AppCacheErrorDetails(
 556       "The cache has been made obsolete, "
 557       "the manifest file returned 404 or 410",
 558       APPCACHE_MANIFEST_ERROR,
 559       GURL(),
 560       response_code,
 561       false /*is_cross_origin*/));
 562   if (success) {
 563     DCHECK(group->is_obsolete());
 564     NotifyAllAssociatedHosts(APPCACHE_OBSOLETE_EVENT);
 565     internal_state_ = COMPLETED;
 566     MaybeCompleteUpdate();
 567   } else {
 568     // Treat failure to mark group obsolete as a cache failure.
 569     HandleCacheFailure(AppCacheErrorDetails(
 570         "Failed to mark the cache as obsolete",
 571         APPCACHE_UNKNOWN_ERROR,
 572         GURL(),
 573         0,
 574         false /*is_cross_origin*/),
 575                        DB_ERROR,
 576                        GURL());
 577   }
 578 }
 579
 580 void AppCacheUpdateJob::ContinueHandleManifestFetchCompleted(bool changed) {
 581   DCHECK(internal_state_ == FETCH_MANIFEST);
 582
 583   if (!changed) {
 584     DCHECK(update_type_ == UPGRADE_ATTEMPT);
 585     internal_state_ = NO_UPDATE;
 586
 587     // Wait for pending master entries to download.
 588     FetchMasterEntries();
 589     MaybeCompleteUpdate();  // if not done, run async 6.9.4 step 7 substeps
 590     return;
 591   }
 592
 593   AppCacheManifest manifest;
 594   if (!ParseManifest(manifest_url_, manifest_data_.data(),
 595                      manifest_data_.length(),
 596                      manifest_has_valid_mime_type_ ?
 597                         PARSE_MANIFEST_ALLOWING_INTERCEPTS :
 598                         PARSE_MANIFEST_PER_STANDARD,
 599                      manifest)) {
 600     const char* kFormatString = "Failed to parse manifest %s";
 601     const std::string message = base::StringPrintf(kFormatString,
 602         manifest_url_.spec().c_str());
 603     HandleCacheFailure(
 604         AppCacheErrorDetails(
 605             message, APPCACHE_SIGNATURE_ERROR, GURL(), 0,
 606             false /*is_cross_origin*/),
 607         MANIFEST_ERROR,
 608         GURL());
 609     VLOG(1) << message;
 610     return;
 611   }
 612
 613   // Proceed with update process. Section 6.9.4 steps 8-20.
 614   internal_state_ = DOWNLOADING;
 615   inprogress_cache_ = new AppCache(storage_, storage_->NewCacheId());
 616   BuildUrlFileList(manifest);
 617   inprogress_cache_->InitializeWithManifest(&manifest);
 618
 619   // Associate all pending master hosts with the newly created cache.
 620   for (PendingMasters::iterator it = pending_master_entries_.begin();
 621        it != pending_master_entries_.end(); ++it) {
 622     PendingHosts& hosts = it->second;
 623     for (PendingHosts::iterator host_it = hosts.begin();
 624          host_it != hosts.end(); ++host_it) {
 625       (*host_it)
 626           ->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
 627     }
 628   }
 629
 630   if (manifest.did_ignore_intercept_namespaces) {
 631     // Must be done after associating all pending master hosts.
 632     std::string message(
 633         "Ignoring the INTERCEPT section of the application cache manifest "
 634         "because the content type is not text/cache-manifest");
 635     LogConsoleMessageToAll(message);
 636   }
 637
 638   group_->SetUpdateAppCacheStatus(AppCacheGroup::DOWNLOADING);
 639   NotifyAllAssociatedHosts(APPCACHE_DOWNLOADING_EVENT);
 640   FetchUrls();
 641   FetchMasterEntries();
 642   MaybeCompleteUpdate();  // if not done, continues when async fetches complete
 643 }
 644
 645 void AppCacheUpdateJob::HandleUrlFetchCompleted(URLFetcher* fetcher) {
 646   DCHECK(internal_state_ == DOWNLOADING);
 647
 648   net::URLRequest* request = fetcher->request();
 649   const GURL& url = request->original_url();
 650   pending_url_fetches_.erase(url);
 651   NotifyAllProgress(url);
 652   ++url_fetches_completed_;
 653
 654   int response_code = request->status().is_success()
 655                           ? request->GetResponseCode()
 656                           : fetcher->redirect_response_code();
 657
 658   AppCacheEntry& entry = url_file_list_.find(url)->second;
 659
 660   if (response_code / 100 == 2) {
 661     // Associate storage with the new entry.
 662     DCHECK(fetcher->response_writer());
 663     entry.set_response_id(fetcher->response_writer()->response_id());
 664     entry.set_response_size(fetcher->response_writer()->amount_written());
 665     if (!inprogress_cache_->AddOrModifyEntry(url, entry))
 666       duplicate_response_ids_.push_back(entry.response_id());
 667
 668     // TODO(michaeln): Check for <html manifest=xxx>
 669     // See http://code.google.com/p/chromium/issues/detail?id=97930
 670     // if (entry.IsMaster() && !(entry.IsExplicit() || fallback || intercept))
 671     //   if (!manifestAttribute) skip it
 672
 673     // Foreign entries will be detected during cache selection.
 674     // Note: 6.9.4, step 17.9 possible optimization: if resource is HTML or XML
 675     // file whose root element is an html element with a manifest attribute
 676     // whose value doesn't match the manifest url of the application cache
 677     // being processed, mark the entry as being foreign.
 678   } else {
 679     VLOG(1) << "Request status: " << request->status().status()
 680             << " error: " << request->status().error()
 681             << " response code: " << response_code;
 682     if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept()) {
 683       if (response_code == 304 && fetcher->existing_entry().has_response_id()) {
 684         // Keep the existing response.
 685         entry.set_response_id(fetcher->existing_entry().response_id());
 686         entry.set_response_size(fetcher->existing_entry().response_size());
 687         inprogress_cache_->AddOrModifyEntry(url, entry);
 688       } else {
 689         const char* kFormatString = "Resource fetch failed (%d) %s";
 690         std::string message = FormatUrlErrorMessage(
 691             kFormatString, url, fetcher->result(), response_code);
 692         ResultType result = fetcher->result();
 693         bool is_cross_origin = url.GetOrigin() != manifest_url_.GetOrigin();
 694         switch (result) {
 695           case DISKCACHE_ERROR:
 696             HandleCacheFailure(
 697                 AppCacheErrorDetails(
 698                     message, APPCACHE_UNKNOWN_ERROR, GURL(), 0,
 699                     is_cross_origin),
 700                 result,
 701                 url);
 702             break;
 703           case NETWORK_ERROR:
 704             HandleCacheFailure(
 705                 AppCacheErrorDetails(message, APPCACHE_RESOURCE_ERROR, url, 0,
 706                     is_cross_origin),
 707                 result,
 708                 url);
 709             break;
 710           default:
 711             HandleCacheFailure(AppCacheErrorDetails(message,
 712                                             APPCACHE_RESOURCE_ERROR,
 713                                             url,
 714                                             response_code,
 715                                             is_cross_origin),
 716                                result,
 717                                url);
 718             break;
 719         }
 720         return;
 721       }
 722     } else if (response_code == 404 || response_code == 410) {
 723       // Entry is skipped.  They are dropped from the cache.
 724     } else if (update_type_ == UPGRADE_ATTEMPT &&
 725                fetcher->existing_entry().has_response_id()) {
 726       // Keep the existing response.
 727       // TODO(michaeln): Not sure this is a good idea. This is spec compliant
 728       // but the old resource may or may not be compatible with the new contents
 729       // of the cache. Impossible to know one way or the other.
 730       entry.set_response_id(fetcher->existing_entry().response_id());
 731       entry.set_response_size(fetcher->existing_entry().response_size());
 732       inprogress_cache_->AddOrModifyEntry(url, entry);
 733     }
 734   }
 735
 736   // Fetch another URL now that one request has completed.
 737   DCHECK(internal_state_ != CACHE_FAILURE);
 738   FetchUrls();
 739   MaybeCompleteUpdate();
 740 }
 741
 742 void AppCacheUpdateJob::HandleMasterEntryFetchCompleted(
 743     URLFetcher* fetcher) {
 744   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
 745
 746   // TODO(jennb): Handle downloads completing during cache failure when update
 747   // no longer fetches master entries directly. For now, we cancel all pending
 748   // master entry fetches when entering cache failure state so this will never
 749   // be called in CACHE_FAILURE state.
 750
 751   net::URLRequest* request = fetcher->request();
 752   const GURL& url = request->original_url();
 753   master_entry_fetches_.erase(url);
 754   ++master_entries_completed_;
 755
 756   int response_code = request->status().is_success()
 757       ? request->GetResponseCode() : -1;
 758
 759   PendingMasters::iterator found = pending_master_entries_.find(url);
 760   DCHECK(found != pending_master_entries_.end());
 761   PendingHosts& hosts = found->second;
 762
 763   // Section 6.9.4. No update case: step 7.3, else step 22.
 764   if (response_code / 100 == 2) {
 765     // Add fetched master entry to the appropriate cache.
 766     AppCache* cache = inprogress_cache_.get() ? inprogress_cache_.get()
 767                                               : group_->newest_complete_cache();
 768     DCHECK(fetcher->response_writer());
 769     AppCacheEntry master_entry(AppCacheEntry::MASTER,
 770                                fetcher->response_writer()->response_id(),
 771                                fetcher->response_writer()->amount_written());
 772     if (cache->AddOrModifyEntry(url, master_entry))
 773       added_master_entries_.push_back(url);
 774     else
 775       duplicate_response_ids_.push_back(master_entry.response_id());
 776
 777     // In no-update case, associate host with the newest cache.
 778     if (!inprogress_cache_.get()) {
 779       // TODO(michaeln): defer until the updated cache has been stored
 780       DCHECK(cache == group_->newest_complete_cache());
 781       for (PendingHosts::iterator host_it = hosts.begin();
 782            host_it != hosts.end(); ++host_it) {
 783         (*host_it)->AssociateCompleteCache(cache);
 784       }
 785     }
 786   } else {
 787     HostNotifier host_notifier;
 788     for (PendingHosts::iterator host_it = hosts.begin();
 789          host_it != hosts.end(); ++host_it) {
 790       AppCacheHost* host = *host_it;
 791       host_notifier.AddHost(host);
 792
 793       // In downloading case, disassociate host from inprogress cache.
 794       if (inprogress_cache_.get())
 795         host->AssociateNoCache(GURL());
 796
 797       host->RemoveObserver(this);
 798     }
 799     hosts.clear();
 800
 801     const char* kFormatString = "Manifest fetch failed (%d) %s";
 802     std::string message = FormatUrlErrorMessage(
 803         kFormatString, request->url(), fetcher->result(), response_code);
 804     host_notifier.SendErrorNotifications(
 805         AppCacheErrorDetails(message,
 806                      APPCACHE_MANIFEST_ERROR,
 807                      request->url(),
 808                      response_code,
 809                      false /*is_cross_origin*/));
 810
 811     // In downloading case, update result is different if all master entries
 812     // failed vs. only some failing.
 813     if (inprogress_cache_.get()) {
 814       // Only count successful downloads to know if all master entries failed.
 815       pending_master_entries_.erase(found);
 816       --master_entries_completed_;
 817
 818       // Section 6.9.4, step 22.3.
 819       if (update_type_ == CACHE_ATTEMPT && pending_master_entries_.empty()) {
 820         HandleCacheFailure(AppCacheErrorDetails(message,
 821                                         APPCACHE_MANIFEST_ERROR,
 822                                         request->url(),
 823                                         response_code,
 824                                         false /*is_cross_origin*/),
 825                            fetcher->result(),
 826                            GURL());
 827         return;
 828       }
 829     }
 830   }
 831
 832   DCHECK(internal_state_ != CACHE_FAILURE);
 833   FetchMasterEntries();
 834   MaybeCompleteUpdate();
 835 }
 836
 837 void AppCacheUpdateJob::HandleManifestRefetchCompleted(
 838     URLFetcher* fetcher) {
 839   DCHECK(internal_state_ == REFETCH_MANIFEST);
 840   DCHECK(manifest_fetcher_ == fetcher);
 841   manifest_fetcher_ = NULL;
 842
 843   net::URLRequest* request = fetcher->request();
 844   int response_code = request->status().is_success()
 845       ? request->GetResponseCode() : -1;
 846   if (response_code == 304 || manifest_data_ == fetcher->manifest_data()) {
 847     // Only need to store response in storage if manifest is not already
 848     // an entry in the cache.
 849     AppCacheEntry* entry = inprogress_cache_->GetEntry(manifest_url_);
 850     if (entry) {
 851       entry->add_types(AppCacheEntry::MANIFEST);
 852       StoreGroupAndCache();
 853     } else {
 854       manifest_response_writer_.reset(CreateResponseWriter());
 855       scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
 856           new HttpResponseInfoIOBuffer(manifest_response_info_.release()));
 857       manifest_response_writer_->WriteInfo(
 858           io_buffer.get(),
 859           base::Bind(&AppCacheUpdateJob::OnManifestInfoWriteComplete,
 860                      base::Unretained(this)));
 861     }
 862   } else {
 863     VLOG(1) << "Request status: " << request->status().status()
 864             << " error: " << request->status().error()
 865             << " response code: " << response_code;
 866     ScheduleUpdateRetry(kRerunDelayMs);
 867     if (response_code == 200) {
 868       HandleCacheFailure(AppCacheErrorDetails("Manifest changed during update",
 869                                       APPCACHE_CHANGED_ERROR,
 870                                       GURL(),
 871                                       0,
 872                                       false /*is_cross_origin*/),
 873                          MANIFEST_ERROR,
 874                          GURL());
 875     } else {
 876       const char* kFormatString = "Manifest re-fetch failed (%d) %s";
 877       std::string message = FormatUrlErrorMessage(
 878           kFormatString, manifest_url_, fetcher->result(), response_code);
 879       HandleCacheFailure(AppCacheErrorDetails(message,
 880                                       APPCACHE_MANIFEST_ERROR,
 881                                       GURL(),
 882                                       response_code,
 883                                       false /*is_cross_origin*/),
 884                          fetcher->result(),
 885                          GURL());
 886     }
 887   }
 888 }
 889
 890 void AppCacheUpdateJob::OnManifestInfoWriteComplete(int result) {
 891   if (result > 0) {
 892     scoped_refptr<net::StringIOBuffer> io_buffer(
 893         new net::StringIOBuffer(manifest_data_));
 894     manifest_response_writer_->WriteData(
 895         io_buffer.get(),
 896         manifest_data_.length(),
 897         base::Bind(&AppCacheUpdateJob::OnManifestDataWriteComplete,
 898                    base::Unretained(this)));
 899   } else {
 900     HandleCacheFailure(
 901         AppCacheErrorDetails("Failed to write the manifest headers to storage",
 902                      APPCACHE_UNKNOWN_ERROR,
 903                      GURL(),
 904                      0,
 905                      false /*is_cross_origin*/),
 906         DISKCACHE_ERROR,
 907         GURL());
 908   }
 909 }
 910
 911 void AppCacheUpdateJob::OnManifestDataWriteComplete(int result) {
 912   if (result > 0) {
 913     AppCacheEntry entry(AppCacheEntry::MANIFEST,
 914         manifest_response_writer_->response_id(),
 915         manifest_response_writer_->amount_written());
 916     if (!inprogress_cache_->AddOrModifyEntry(manifest_url_, entry))
 917       duplicate_response_ids_.push_back(entry.response_id());
 918     StoreGroupAndCache();
 919   } else {
 920     HandleCacheFailure(
 921         AppCacheErrorDetails("Failed to write the manifest data to storage",
 922                      APPCACHE_UNKNOWN_ERROR,
 923                      GURL(),
 924                      0,
 925                      false /*is_cross_origin*/),
 926         DISKCACHE_ERROR,
 927         GURL());
 928   }
 929 }
 930
 931 void AppCacheUpdateJob::StoreGroupAndCache() {
 932   DCHECK(stored_state_ == UNSTORED);
 933   stored_state_ = STORING;
 934   scoped_refptr<AppCache> newest_cache;
 935   if (inprogress_cache_.get())
 936     newest_cache.swap(inprogress_cache_);
 937   else
 938     newest_cache = group_->newest_complete_cache();
 939   newest_cache->set_update_time(base::Time::Now());
 940
 941   // TODO(michaeln): dcheck is fishing for clues to crbug/95101
 942   DCHECK_EQ(manifest_url_, group_->manifest_url());
 943   storage_->StoreGroupAndNewestCache(group_, newest_cache.get(), this);
 944 }
 945
 946 void AppCacheUpdateJob::OnGroupAndNewestCacheStored(AppCacheGroup* group,
 947                                                     AppCache* newest_cache,
 948                                                     bool success,
 949                                                     bool would_exceed_quota) {
 950   DCHECK(stored_state_ == STORING);
 951   if (success) {
 952     stored_state_ = STORED;
 953     MaybeCompleteUpdate();  // will definitely complete
 954   } else {
 955     stored_state_ = UNSTORED;
 956
 957     // Restore inprogress_cache_ to get the proper events delivered
 958     // and the proper cleanup to occur.
 959     if (newest_cache != group->newest_complete_cache())
 960       inprogress_cache_ = newest_cache;
 961
 962     ResultType result = DB_ERROR;
 963     AppCacheErrorReason reason = APPCACHE_UNKNOWN_ERROR;
 964     std::string message("Failed to commit new cache to storage");
 965     if (would_exceed_quota) {
 966       message.append(", would exceed quota");
 967       result = QUOTA_ERROR;
 968       reason = APPCACHE_QUOTA_ERROR;
 969     }
 970     HandleCacheFailure(
 971         AppCacheErrorDetails(message, reason, GURL(), 0,
 972             false /*is_cross_origin*/),
 973         result,
 974         GURL());
 975   }
 976 }
 977
 978 void AppCacheUpdateJob::NotifySingleHost(AppCacheHost* host,
 979                                          AppCacheEventID event_id) {
 980   std::vector<int> ids(1, host->host_id());
 981   host->frontend()->OnEventRaised(ids, event_id);
 982 }
 983
 984 void AppCacheUpdateJob::NotifyAllAssociatedHosts(AppCacheEventID event_id) {
 985   HostNotifier host_notifier;
 986   AddAllAssociatedHostsToNotifier(&host_notifier);
 987   host_notifier.SendNotifications(event_id);
 988 }
 989
 990 void AppCacheUpdateJob::NotifyAllProgress(const GURL& url) {
 991   HostNotifier host_notifier;
 992   AddAllAssociatedHostsToNotifier(&host_notifier);
 993   host_notifier.SendProgressNotifications(
 994       url, url_file_list_.size(), url_fetches_completed_);
 995 }
 996
 997 void AppCacheUpdateJob::NotifyAllFinalProgress() {
 998   DCHECK(url_file_list_.size() == url_fetches_completed_);
 999   NotifyAllProgress(GURL());
1000 }
1001
1002 void AppCacheUpdateJob::NotifyAllError(const AppCacheErrorDetails& details) {
1003   HostNotifier host_notifier;
1004   AddAllAssociatedHostsToNotifier(&host_notifier);
1005   host_notifier.SendErrorNotifications(details);
1006 }
1007
1008 void AppCacheUpdateJob::LogConsoleMessageToAll(const std::string& message) {
1009   HostNotifier host_notifier;
1010   AddAllAssociatedHostsToNotifier(&host_notifier);
1011   host_notifier.SendLogMessage(message);
1012 }
1013
1014 void AppCacheUpdateJob::AddAllAssociatedHostsToNotifier(
1015     HostNotifier* host_notifier) {
1016   // Collect hosts so we only send one notification per frontend.
1017   // A host can only be associated with a single cache so no need to worry
1018   // about duplicate hosts being added to the notifier.
1019   if (inprogress_cache_.get()) {
1020     DCHECK(internal_state_ == DOWNLOADING || internal_state_ == CACHE_FAILURE);
1021     host_notifier->AddHosts(inprogress_cache_->associated_hosts());
1022   }
1023
1024   AppCacheGroup::Caches old_caches = group_->old_caches();
1025   for (AppCacheGroup::Caches::const_iterator it = old_caches.begin();
1026        it != old_caches.end(); ++it) {
1027     host_notifier->AddHosts((*it)->associated_hosts());
1028   }
1029
1030   AppCache* newest_cache = group_->newest_complete_cache();
1031   if (newest_cache)
1032     host_notifier->AddHosts(newest_cache->associated_hosts());
1033 }
1034
1035 void AppCacheUpdateJob::OnDestructionImminent(AppCacheHost* host) {
1036   // The host is about to be deleted; remove from our collection.
1037   PendingMasters::iterator found =
1038       pending_master_entries_.find(host->pending_master_entry_url());
1039   DCHECK(found != pending_master_entries_.end());
1040   PendingHosts& hosts = found->second;
1041   PendingHosts::iterator it = std::find(hosts.begin(), hosts.end(), host);
1042   DCHECK(it != hosts.end());
1043   hosts.erase(it);
1044 }
1045
1046 void AppCacheUpdateJob::OnServiceReinitialized(
1047     AppCacheStorageReference* old_storage_ref) {
1048   // We continue to use the disabled instance, but arrange for its
1049   // deletion when its no longer needed.
1050   if (old_storage_ref->storage() == storage_)
1051     disabled_storage_reference_ = old_storage_ref;
1052 }
1053
1054 void AppCacheUpdateJob::CheckIfManifestChanged() {
1055   DCHECK(update_type_ == UPGRADE_ATTEMPT);
1056   AppCacheEntry* entry = NULL;
1057   if (group_->newest_complete_cache())
1058     entry = group_->newest_complete_cache()->GetEntry(manifest_url_);
1059   if (!entry) {
1060     // TODO(michaeln): This is just a bandaid to avoid a crash.
1061     // http://code.google.com/p/chromium/issues/detail?id=95101
1062     if (service_->storage() == storage_) {
1063       // Use a local variable because service_ is reset in HandleCacheFailure.
1064       AppCacheServiceImpl* service = service_;
1065       HandleCacheFailure(
1066           AppCacheErrorDetails("Manifest entry not found in existing cache",
1067                        APPCACHE_UNKNOWN_ERROR,
1068                        GURL(),
1069                        0,
1070                        false /*is_cross_origin*/),
1071           DB_ERROR,
1072           GURL());
1073       AppCacheHistograms::AddMissingManifestEntrySample();
1074       service->DeleteAppCacheGroup(manifest_url_, net::CompletionCallback());
1075     }
1076     return;
1077   }
1078
1079   // Load manifest data from storage to compare against fetched manifest.
1080   manifest_response_reader_.reset(
1081       storage_->CreateResponseReader(manifest_url_,
1082                                      group_->group_id(),
1083                                      entry->response_id()));
1084   read_manifest_buffer_ = new net::IOBuffer(kBufferSize);
1085   manifest_response_reader_->ReadData(
1086       read_manifest_buffer_.get(),
1087       kBufferSize,
1088       base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
1089                  base::Unretained(this)));  // async read
1090 }
1091
1092 void AppCacheUpdateJob::OnManifestDataReadComplete(int result) {
1093   if (result > 0) {
1094     loaded_manifest_data_.append(read_manifest_buffer_->data(), result);
1095     manifest_response_reader_->ReadData(
1096         read_manifest_buffer_.get(),
1097         kBufferSize,
1098         base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
1099                    base::Unretained(this)));  // read more
1100   } else {
1101     read_manifest_buffer_ = NULL;
1102     manifest_response_reader_.reset();
1103     ContinueHandleManifestFetchCompleted(
1104         result < 0 || manifest_data_ != loaded_manifest_data_);
1105   }
1106 }
1107
1108 void AppCacheUpdateJob::BuildUrlFileList(const AppCacheManifest& manifest) {
1109   for (base::hash_set<std::string>::const_iterator it =
1110            manifest.explicit_urls.begin();
1111        it != manifest.explicit_urls.end(); ++it) {
1112     AddUrlToFileList(GURL(*it), AppCacheEntry::EXPLICIT);
1113   }
1114
1115   const std::vector<AppCacheNamespace>& intercepts =
1116       manifest.intercept_namespaces;
1117   for (std::vector<AppCacheNamespace>::const_iterator it = intercepts.begin();
1118        it != intercepts.end(); ++it) {
1119     int flags = AppCacheEntry::INTERCEPT;
1120     if (it->is_executable)
1121       flags |= AppCacheEntry::EXECUTABLE;
1122     AddUrlToFileList(it->target_url, flags);
1123   }
1124
1125   const std::vector<AppCacheNamespace>& fallbacks =
1126       manifest.fallback_namespaces;
1127   for (std::vector<AppCacheNamespace>::const_iterator it = fallbacks.begin();
1128        it != fallbacks.end(); ++it) {
1129      AddUrlToFileList(it->target_url, AppCacheEntry::FALLBACK);
1130   }
1131
1132   // Add all master entries from newest complete cache.
1133   if (update_type_ == UPGRADE_ATTEMPT) {
1134     const AppCache::EntryMap& entries =
1135         group_->newest_complete_cache()->entries();
1136     for (AppCache::EntryMap::const_iterator it = entries.begin();
1137          it != entries.end(); ++it) {
1138       const AppCacheEntry& entry = it->second;
1139       if (entry.IsMaster())
1140         AddUrlToFileList(it->first, AppCacheEntry::MASTER);
1141     }
1142   }
1143 }
1144
1145 void AppCacheUpdateJob::AddUrlToFileList(const GURL& url, int type) {
1146   std::pair<AppCache::EntryMap::iterator, bool> ret = url_file_list_.insert(
1147       AppCache::EntryMap::value_type(url, AppCacheEntry(type)));
1148
1149   if (ret.second)
1150     urls_to_fetch_.push_back(UrlToFetch(url, false, NULL));
1151   else
1152     ret.first->second.add_types(type);  // URL already exists. Merge types.
1153 }
1154
1155 void AppCacheUpdateJob::FetchUrls() {
1156   DCHECK(internal_state_ == DOWNLOADING);
1157
1158   // Fetch each URL in the list according to section 6.9.4 step 17.1-17.3.
1159   // Fetch up to the concurrent limit. Other fetches will be triggered as each
1160   // each fetch completes.
1161   while (pending_url_fetches_.size() < kMaxConcurrentUrlFetches &&
1162          !urls_to_fetch_.empty()) {
1163     UrlToFetch url_to_fetch = urls_to_fetch_.front();
1164     urls_to_fetch_.pop_front();
1165
1166     AppCache::EntryMap::iterator it = url_file_list_.find(url_to_fetch.url);
1167     DCHECK(it != url_file_list_.end());
1168     AppCacheEntry& entry = it->second;
1169     if (ShouldSkipUrlFetch(entry)) {
1170       NotifyAllProgress(url_to_fetch.url);
1171       ++url_fetches_completed_;
1172     } else if (AlreadyFetchedEntry(url_to_fetch.url, entry.types())) {
1173       NotifyAllProgress(url_to_fetch.url);
1174       ++url_fetches_completed_;  // saved a URL request
1175     } else if (!url_to_fetch.storage_checked &&
1176                MaybeLoadFromNewestCache(url_to_fetch.url, entry)) {
1177       // Continues asynchronously after data is loaded from newest cache.
1178     } else {
1179       URLFetcher* fetcher = new URLFetcher(
1180           url_to_fetch.url, URLFetcher::URL_FETCH, this);
1181       if (url_to_fetch.existing_response_info.get()) {
1182         DCHECK(group_->newest_complete_cache());
1183         AppCacheEntry* existing_entry =
1184             group_->newest_complete_cache()->GetEntry(url_to_fetch.url);
1185         DCHECK(existing_entry);
1186         DCHECK(existing_entry->response_id() ==
1187                url_to_fetch.existing_response_info->response_id());
1188         fetcher->set_existing_response_headers(
1189             url_to_fetch.existing_response_info->http_response_info()->headers
1190                 .get());
1191         fetcher->set_existing_entry(*existing_entry);
1192       }
1193       fetcher->Start();
1194       pending_url_fetches_.insert(
1195           PendingUrlFetches::value_type(url_to_fetch.url, fetcher));
1196     }
1197   }
1198 }
1199
1200 void AppCacheUpdateJob::CancelAllUrlFetches() {
1201   // Cancel any pending URL requests.
1202   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
1203        it != pending_url_fetches_.end(); ++it) {
1204     delete it->second;
1205   }
1206
1207   url_fetches_completed_ +=
1208       pending_url_fetches_.size() + urls_to_fetch_.size();
1209   pending_url_fetches_.clear();
1210   urls_to_fetch_.clear();
1211 }
1212
1213 bool AppCacheUpdateJob::ShouldSkipUrlFetch(const AppCacheEntry& entry) {
1214   // 6.6.4 Step 17
1215   // If the resource URL being processed was flagged as neither an
1216   // "explicit entry" nor or a "fallback entry", then the user agent
1217   // may skip this URL.
1218   if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept())
1219     return false;
1220
1221   // TODO(jennb): decide if entry should be skipped to expire it from cache
1222   return false;
1223 }
1224
1225 bool AppCacheUpdateJob::AlreadyFetchedEntry(const GURL& url,
1226                                             int entry_type) {
1227   DCHECK(internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE);
1228   AppCacheEntry* existing =
1229       inprogress_cache_.get() ? inprogress_cache_->GetEntry(url)
1230                               : group_->newest_complete_cache()->GetEntry(url);
1231   if (existing) {
1232     existing->add_types(entry_type);
1233     return true;
1234   }
1235   return false;
1236 }
1237
1238 void AppCacheUpdateJob::AddMasterEntryToFetchList(AppCacheHost* host,
1239                                                   const GURL& url,
1240                                                   bool is_new) {
1241   DCHECK(!IsTerminating());
1242
1243   if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE) {
1244     AppCache* cache;
1245     if (inprogress_cache_.get()) {
1246       // always associate
1247       host->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
1248       cache = inprogress_cache_.get();
1249     } else {
1250       cache = group_->newest_complete_cache();
1251     }
1252
1253     // Update existing entry if it has already been fetched.
1254     AppCacheEntry* entry = cache->GetEntry(url);
1255     if (entry) {
1256       entry->add_types(AppCacheEntry::MASTER);
1257       if (internal_state_ == NO_UPDATE && !inprogress_cache_.get()) {
1258         // only associate if have entry
1259         host->AssociateCompleteCache(cache);
1260       }
1261       if (is_new)
1262         ++master_entries_completed_;  // pretend fetching completed
1263       return;
1264     }
1265   }
1266
1267   // Add to fetch list if not already fetching.
1268   if (master_entry_fetches_.find(url) == master_entry_fetches_.end()) {
1269     master_entries_to_fetch_.insert(url);
1270     if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE)
1271       FetchMasterEntries();
1272   }
1273 }
1274
1275 void AppCacheUpdateJob::FetchMasterEntries() {
1276   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
1277
1278   // Fetch each master entry in the list, up to the concurrent limit.
1279   // Additional fetches will be triggered as each fetch completes.
1280   while (master_entry_fetches_.size() < kMaxConcurrentUrlFetches &&
1281          !master_entries_to_fetch_.empty()) {
1282     const GURL& url = *master_entries_to_fetch_.begin();
1283
1284     if (AlreadyFetchedEntry(url, AppCacheEntry::MASTER)) {
1285       ++master_entries_completed_;  // saved a URL request
1286
1287       // In no update case, associate hosts to newest cache in group
1288       // now that master entry has been "successfully downloaded".
1289       if (internal_state_ == NO_UPDATE) {
1290         // TODO(michaeln): defer until the updated cache has been stored.
1291         DCHECK(!inprogress_cache_.get());
1292         AppCache* cache = group_->newest_complete_cache();
1293         PendingMasters::iterator found = pending_master_entries_.find(url);
1294         DCHECK(found != pending_master_entries_.end());
1295         PendingHosts& hosts = found->second;
1296         for (PendingHosts::iterator host_it = hosts.begin();
1297              host_it != hosts.end(); ++host_it) {
1298           (*host_it)->AssociateCompleteCache(cache);
1299         }
1300       }
1301     } else {
1302       URLFetcher* fetcher = new URLFetcher(
1303           url, URLFetcher::MASTER_ENTRY_FETCH, this);
1304       fetcher->Start();
1305       master_entry_fetches_.insert(PendingUrlFetches::value_type(url, fetcher));
1306     }
1307
1308     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
1309   }
1310 }
1311
1312 void AppCacheUpdateJob::CancelAllMasterEntryFetches(
1313     const AppCacheErrorDetails& error_details) {
1314   // For now, cancel all in-progress fetches for master entries and pretend
1315   // all master entries fetches have completed.
1316   // TODO(jennb): Delete this when update no longer fetches master entries
1317   // directly.
1318
1319   // Cancel all in-progress fetches.
1320   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
1321        it != master_entry_fetches_.end(); ++it) {
1322     delete it->second;
1323     master_entries_to_fetch_.insert(it->first);  // back in unfetched list
1324   }
1325   master_entry_fetches_.clear();
1326
1327   master_entries_completed_ += master_entries_to_fetch_.size();
1328
1329   // Cache failure steps, step 2.
1330   // Pretend all master entries that have not yet been fetched have completed
1331   // downloading. Unassociate hosts from any appcache and send ERROR event.
1332   HostNotifier host_notifier;
1333   while (!master_entries_to_fetch_.empty()) {
1334     const GURL& url = *master_entries_to_fetch_.begin();
1335     PendingMasters::iterator found = pending_master_entries_.find(url);
1336     DCHECK(found != pending_master_entries_.end());
1337     PendingHosts& hosts = found->second;
1338     for (PendingHosts::iterator host_it = hosts.begin();
1339          host_it != hosts.end(); ++host_it) {
1340       AppCacheHost* host = *host_it;
1341       host->AssociateNoCache(GURL());
1342       host_notifier.AddHost(host);
1343       host->RemoveObserver(this);
1344     }
1345     hosts.clear();
1346
1347     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
1348   }
1349   host_notifier.SendErrorNotifications(error_details);
1350 }
1351
1352 bool AppCacheUpdateJob::MaybeLoadFromNewestCache(const GURL& url,
1353                                                  AppCacheEntry& entry) {
1354   if (update_type_ != UPGRADE_ATTEMPT)
1355     return false;
1356
1357   AppCache* newest = group_->newest_complete_cache();
1358   AppCacheEntry* copy_me = newest->GetEntry(url);
1359   if (!copy_me || !copy_me->has_response_id())
1360     return false;
1361
1362   // Load HTTP headers for entry from newest cache.
1363   loading_responses_.insert(
1364       LoadingResponses::value_type(copy_me->response_id(), url));
1365   storage_->LoadResponseInfo(manifest_url_, group_->group_id(),
1366                              copy_me->response_id(),
1367                              this);
1368   // Async: wait for OnResponseInfoLoaded to complete.
1369   return true;
1370 }
1371
1372 void AppCacheUpdateJob::OnResponseInfoLoaded(
1373     AppCacheResponseInfo* response_info, int64 response_id) {
1374   const net::HttpResponseInfo* http_info = response_info ?
1375       response_info->http_response_info() : NULL;
1376
1377   // Needed response info for a manifest fetch request.
1378   if (internal_state_ == FETCH_MANIFEST) {
1379     if (http_info)
1380       manifest_fetcher_->set_existing_response_headers(
1381           http_info->headers.get());
1382     manifest_fetcher_->Start();
1383     return;
1384   }
1385
1386   LoadingResponses::iterator found = loading_responses_.find(response_id);
1387   DCHECK(found != loading_responses_.end());
1388   const GURL& url = found->second;
1389
1390   if (!http_info) {
1391     LoadFromNewestCacheFailed(url, NULL);  // no response found
1392   } else {
1393     // Check if response can be re-used according to HTTP caching semantics.
1394     // Responses with a "vary" header get treated as expired.
1395     const std::string name = "vary";
1396     std::string value;
1397     void* iter = NULL;
1398     if (!http_info->headers.get() ||
1399         http_info->headers->RequiresValidation(http_info->request_time,
1400                                                http_info->response_time,
1401                                                base::Time::Now()) ||
1402         http_info->headers->EnumerateHeader(&iter, name, &value)) {
1403       LoadFromNewestCacheFailed(url, response_info);
1404     } else {
1405       DCHECK(group_->newest_complete_cache());
1406       AppCacheEntry* copy_me = group_->newest_complete_cache()->GetEntry(url);
1407       DCHECK(copy_me);
1408       DCHECK(copy_me->response_id() == response_id);
1409
1410       AppCache::EntryMap::iterator it = url_file_list_.find(url);
1411       DCHECK(it != url_file_list_.end());
1412       AppCacheEntry& entry = it->second;
1413       entry.set_response_id(response_id);
1414       entry.set_response_size(copy_me->response_size());
1415       inprogress_cache_->AddOrModifyEntry(url, entry);
1416       NotifyAllProgress(url);
1417       ++url_fetches_completed_;
1418     }
1419   }
1420   loading_responses_.erase(found);
1421
1422   MaybeCompleteUpdate();
1423 }
1424
1425 void AppCacheUpdateJob::LoadFromNewestCacheFailed(
1426     const GURL& url, AppCacheResponseInfo* response_info) {
1427   if (internal_state_ == CACHE_FAILURE)
1428     return;
1429
1430   // Re-insert url at front of fetch list. Indicate storage has been checked.
1431   urls_to_fetch_.push_front(UrlToFetch(url, true, response_info));
1432   FetchUrls();
1433 }
1434
1435 void AppCacheUpdateJob::MaybeCompleteUpdate() {
1436   DCHECK(internal_state_ != CACHE_FAILURE);
1437
1438   // Must wait for any pending master entries or url fetches to complete.
1439   if (master_entries_completed_ != pending_master_entries_.size() ||
1440       url_fetches_completed_ != url_file_list_.size()) {
1441     DCHECK(internal_state_ != COMPLETED);
1442     return;
1443   }
1444
1445   switch (internal_state_) {
1446     case NO_UPDATE:
1447       if (master_entries_completed_ > 0) {
1448         switch (stored_state_) {
1449           case UNSTORED:
1450             StoreGroupAndCache();
1451             return;
1452           case STORING:
1453             return;
1454           case STORED:
1455             break;
1456         }
1457       }
1458       // 6.9.4 steps 7.3-7.7.
1459       NotifyAllAssociatedHosts(APPCACHE_NO_UPDATE_EVENT);
1460       DiscardDuplicateResponses();
1461       internal_state_ = COMPLETED;
1462       break;
1463     case DOWNLOADING:
1464       internal_state_ = REFETCH_MANIFEST;
1465       FetchManifest(false);
1466       break;
1467     case REFETCH_MANIFEST:
1468       DCHECK(stored_state_ == STORED);
1469       NotifyAllFinalProgress();
1470       if (update_type_ == CACHE_ATTEMPT)
1471         NotifyAllAssociatedHosts(APPCACHE_CACHED_EVENT);
1472       else
1473         NotifyAllAssociatedHosts(APPCACHE_UPDATE_READY_EVENT);
1474       DiscardDuplicateResponses();
1475       internal_state_ = COMPLETED;
1476       LogHistogramStats(UPDATE_OK, GURL());
1477       break;
1478     case CACHE_FAILURE:
1479       NOTREACHED();  // See HandleCacheFailure
1480       break;
1481     default:
1482       break;
1483   }
1484
1485   // Let the stack unwind before deletion to make it less risky as this
1486   // method is called from multiple places in this file.
1487   if (internal_state_ == COMPLETED)
1488     DeleteSoon();
1489 }
1490
1491 void AppCacheUpdateJob::ScheduleUpdateRetry(int delay_ms) {
1492   // TODO(jennb): post a delayed task with the "same parameters" as this job
1493   // to retry the update at a later time. Need group, URLs of pending master
1494   // entries and their hosts.
1495 }
1496
1497 void AppCacheUpdateJob::Cancel() {
1498   internal_state_ = CANCELLED;
1499
1500   LogHistogramStats(CANCELLED_ERROR, GURL());
1501
1502   if (manifest_fetcher_) {
1503     delete manifest_fetcher_;
1504     manifest_fetcher_ = NULL;
1505   }
1506
1507   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
1508        it != pending_url_fetches_.end(); ++it) {
1509     delete it->second;
1510   }
1511   pending_url_fetches_.clear();
1512
1513   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
1514        it != master_entry_fetches_.end(); ++it) {
1515     delete it->second;
1516   }
1517   master_entry_fetches_.clear();
1518
1519   ClearPendingMasterEntries();
1520   DiscardInprogressCache();
1521
1522   // Delete response writer to avoid any callbacks.
1523   if (manifest_response_writer_)
1524     manifest_response_writer_.reset();
1525
1526   storage_->CancelDelegateCallbacks(this);
1527 }
1528
1529 void AppCacheUpdateJob::ClearPendingMasterEntries() {
1530   for (PendingMasters::iterator it = pending_master_entries_.begin();
1531        it != pending_master_entries_.end(); ++it) {
1532     PendingHosts& hosts = it->second;
1533     for (PendingHosts::iterator host_it = hosts.begin();
1534          host_it != hosts.end(); ++host_it) {
1535       (*host_it)->RemoveObserver(this);
1536     }
1537   }
1538
1539   pending_master_entries_.clear();
1540 }
1541
1542 void AppCacheUpdateJob::DiscardInprogressCache() {
1543   if (stored_state_ == STORING) {
1544     // We can make no assumptions about whether the StoreGroupAndCacheTask
1545     // actually completed or not. This condition should only be reachable
1546     // during shutdown. Free things up and return to do no harm.
1547     inprogress_cache_ = NULL;
1548     added_master_entries_.clear();
1549     return;
1550   }
1551
1552   storage_->DoomResponses(manifest_url_, stored_response_ids_);
1553
1554   if (!inprogress_cache_.get()) {
1555     // We have to undo the changes we made, if any, to the existing cache.
1556     if (group_ && group_->newest_complete_cache()) {
1557       for (std::vector<GURL>::iterator iter = added_master_entries_.begin();
1558            iter != added_master_entries_.end(); ++iter) {
1559         group_->newest_complete_cache()->RemoveEntry(*iter);
1560       }
1561     }
1562     added_master_entries_.clear();
1563     return;
1564   }
1565
1566   AppCache::AppCacheHosts& hosts = inprogress_cache_->associated_hosts();
1567   while (!hosts.empty())
1568     (*hosts.begin())->AssociateNoCache(GURL());
1569
1570   inprogress_cache_ = NULL;
1571   added_master_entries_.clear();
1572 }
1573
1574 void AppCacheUpdateJob::DiscardDuplicateResponses() {
1575   storage_->DoomResponses(manifest_url_, duplicate_response_ids_);
1576 }
1577
1578 void AppCacheUpdateJob::LogHistogramStats(
1579       ResultType result, const GURL& failed_resource_url) {
1580   AppCacheHistograms::CountUpdateJobResult(result, manifest_url_.GetOrigin());
1581   if (result == UPDATE_OK)
1582     return;
1583
1584   int percent_complete = 0;
1585   if (url_file_list_.size() > 0) {
1586     size_t actual_fetches_completed = url_fetches_completed_;
1587     if (!failed_resource_url.is_empty() && actual_fetches_completed)
1588       --actual_fetches_completed;
1589     percent_complete = (static_cast<double>(actual_fetches_completed) /
1590                             static_cast<double>(url_file_list_.size())) * 100.0;
1591     percent_complete = std::min(percent_complete, 99);
1592   }
1593
1594   bool was_making_progress =
1595       base::Time::Now() - last_progress_time_ <
1596           base::TimeDelta::FromMinutes(5);
1597
1598   bool off_origin_resource_failure =
1599       !failed_resource_url.is_empty() &&
1600           (failed_resource_url.GetOrigin() != manifest_url_.GetOrigin());
1601
1602   AppCacheHistograms::LogUpdateFailureStats(
1603       manifest_url_.GetOrigin(),
1604       percent_complete,
1605       was_making_progress,
1606       off_origin_resource_failure);
1607 }
1608
1609 void AppCacheUpdateJob::DeleteSoon() {
1610   ClearPendingMasterEntries();
1611   manifest_response_writer_.reset();
1612   storage_->CancelDelegateCallbacks(this);
1613   service_->RemoveObserver(this);
1614   service_ = NULL;
1615
1616   // Break the connection with the group so the group cannot call delete
1617   // on this object after we've posted a task to delete ourselves.
1618   group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE);
1619   group_ = NULL;
1620
1621   base::MessageLoop::current()->DeleteSoon(FROM_HERE, this);
1622 }
1623
1624 }  // namespace content