content/browser/appcache/appcache_update_job.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/appcache/appcache_update_job.h"
   6
   7 #include "base/bind.h"
   8 #include "base/bind_helpers.h"
   9 #include "base/compiler_specific.h"
  10 #include "base/message_loop/message_loop.h"
  11 #include "base/strings/string_util.h"
  12 #include "base/strings/stringprintf.h"
  13 #include "content/browser/appcache/appcache_group.h"
  14 #include "content/browser/appcache/appcache_histograms.h"
  15 #include "net/base/io_buffer.h"
  16 #include "net/base/load_flags.h"
  17 #include "net/base/net_errors.h"
  18 #include "net/base/request_priority.h"
  19 #include "net/http/http_request_headers.h"
  20 #include "net/http/http_response_headers.h"
  21 #include "net/url_request/url_request_context.h"
  22
  23 namespace content {
  24
  25 static const int kBufferSize = 32768;
  26 static const size_t kMaxConcurrentUrlFetches = 2;
  27 static const int kMax503Retries = 3;
  28
  29 static std::string FormatUrlErrorMessage(
  30       const char* format, const GURL& url,
  31       AppCacheUpdateJob::ResultType error,
  32       int response_code) {
  33     // Show the net response code if we have one.
  34     int code = response_code;
  35     if (error != AppCacheUpdateJob::SERVER_ERROR)
  36       code = static_cast<int>(error);
  37     return base::StringPrintf(format, code, url.spec().c_str());
  38 }
  39
  40 // Helper class for collecting hosts per frontend when sending notifications
  41 // so that only one notification is sent for all hosts using the same frontend.
  42 class HostNotifier {
  43  public:
  44   typedef std::vector<int> HostIds;
  45   typedef std::map<AppCacheFrontend*, HostIds> NotifyHostMap;
  46
  47   // Caller is responsible for ensuring there will be no duplicate hosts.
  48   void AddHost(AppCacheHost* host) {
  49     std::pair<NotifyHostMap::iterator , bool> ret = hosts_to_notify.insert(
  50         NotifyHostMap::value_type(host->frontend(), HostIds()));
  51     ret.first->second.push_back(host->host_id());
  52   }
  53
  54   void AddHosts(const std::set<AppCacheHost*>& hosts) {
  55     for (std::set<AppCacheHost*>::const_iterator it = hosts.begin();
  56          it != hosts.end(); ++it) {
  57       AddHost(*it);
  58     }
  59   }
  60
  61   void SendNotifications(AppCacheEventID event_id) {
  62     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  63          it != hosts_to_notify.end(); ++it) {
  64       AppCacheFrontend* frontend = it->first;
  65       frontend->OnEventRaised(it->second, event_id);
  66     }
  67   }
  68
  69   void SendProgressNotifications(
  70       const GURL& url, int num_total, int num_complete) {
  71     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  72          it != hosts_to_notify.end(); ++it) {
  73       AppCacheFrontend* frontend = it->first;
  74       frontend->OnProgressEventRaised(it->second, url,
  75                                       num_total, num_complete);
  76     }
  77   }
  78
  79   void SendErrorNotifications(const AppCacheErrorDetails& details) {
  80     DCHECK(!details.message.empty());
  81     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  82          it != hosts_to_notify.end(); ++it) {
  83       AppCacheFrontend* frontend = it->first;
  84       frontend->OnErrorEventRaised(it->second, details);
  85     }
  86   }
  87
  88   void SendLogMessage(const std::string& message) {
  89     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  90          it != hosts_to_notify.end(); ++it) {
  91       AppCacheFrontend* frontend = it->first;
  92       for (HostIds::iterator id = it->second.begin();
  93            id != it->second.end(); ++id) {
  94         frontend->OnLogMessage(*id, APPCACHE_LOG_WARNING, message);
  95       }
  96     }
  97   }
  98
  99  private:
 100   NotifyHostMap hosts_to_notify;
 101 };
 102
 103 AppCacheUpdateJob::UrlToFetch::UrlToFetch(const GURL& url,
 104                                           bool checked,
 105                                           AppCacheResponseInfo* info)
 106     : url(url),
 107       storage_checked(checked),
 108       existing_response_info(info) {
 109 }
 110
 111 AppCacheUpdateJob::UrlToFetch::~UrlToFetch() {
 112 }
 113
 114 // Helper class to fetch resources. Depending on the fetch type,
 115 // can either fetch to an in-memory string or write the response
 116 // data out to the disk cache.
 117 AppCacheUpdateJob::URLFetcher::URLFetcher(const GURL& url,
 118                                           FetchType fetch_type,
 119                                           AppCacheUpdateJob* job)
 120     : url_(url),
 121       job_(job),
 122       fetch_type_(fetch_type),
 123       retry_503_attempts_(0),
 124       buffer_(new net::IOBuffer(kBufferSize)),
 125       request_(job->service_->request_context()
 126                    ->CreateRequest(url, net::DEFAULT_PRIORITY, this)),
 127       result_(UPDATE_OK),
 128       redirect_response_code_(-1) {}
 129
 130 AppCacheUpdateJob::URLFetcher::~URLFetcher() {
 131 }
 132
 133 void AppCacheUpdateJob::URLFetcher::Start() {
 134   request_->set_first_party_for_cookies(job_->manifest_url_);
 135   if (existing_response_headers_.get())
 136     AddConditionalHeaders(existing_response_headers_.get());
 137   request_->Start();
 138 }
 139
 140 void AppCacheUpdateJob::URLFetcher::OnReceivedRedirect(
 141     net::URLRequest* request,
 142     const net::RedirectInfo& redirect_info,
 143     bool* defer_redirect) {
 144   DCHECK(request_ == request);
 145   // Redirect is not allowed by the update process.
 146   job_->MadeProgress();
 147   redirect_response_code_ = request->GetResponseCode();
 148   request->Cancel();
 149   result_ = REDIRECT_ERROR;
 150   OnResponseCompleted();
 151 }
 152
 153 void AppCacheUpdateJob::URLFetcher::OnResponseStarted(
 154     net::URLRequest *request) {
 155   DCHECK(request == request_);
 156   int response_code = -1;
 157   if (request->status().is_success()) {
 158     response_code = request->GetResponseCode();
 159     job_->MadeProgress();
 160   }
 161
 162   if ((response_code / 100) != 2) {
 163     if (response_code > 0)
 164       result_ = SERVER_ERROR;
 165     else
 166       result_ = NETWORK_ERROR;
 167     OnResponseCompleted();
 168     return;
 169   }
 170
 171   if (url_.SchemeIsSecure()) {
 172     // Do not cache content with cert errors.
 173     // Also, we willfully violate the HTML5 spec at this point in order
 174     // to support the appcaching of cross-origin HTTPS resources.
 175     // We've opted for a milder constraint and allow caching unless
 176     // the resource has a "no-store" header. A spec change has been
 177     // requested on the whatwg list.
 178     // See http://code.google.com/p/chromium/issues/detail?id=69594
 179     // TODO(michaeln): Consider doing this for cross-origin HTTP too.
 180     const net::HttpNetworkSession::Params* session_params =
 181         request->context()->GetNetworkSessionParams();
 182     bool ignore_cert_errors = session_params &&
 183                               session_params->ignore_certificate_errors;
 184     if ((net::IsCertStatusError(request->ssl_info().cert_status) &&
 185             !ignore_cert_errors) ||
 186         (url_.GetOrigin() != job_->manifest_url_.GetOrigin() &&
 187             request->response_headers()->
 188                 HasHeaderValue("cache-control", "no-store"))) {
 189       DCHECK_EQ(-1, redirect_response_code_);
 190       request->Cancel();
 191       result_ = SECURITY_ERROR;
 192       OnResponseCompleted();
 193       return;
 194     }
 195   }
 196
 197   // Write response info to storage for URL fetches. Wait for async write
 198   // completion before reading any response data.
 199   if (fetch_type_ == URL_FETCH || fetch_type_ == MASTER_ENTRY_FETCH) {
 200     response_writer_.reset(job_->CreateResponseWriter());
 201     scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
 202         new HttpResponseInfoIOBuffer(
 203             new net::HttpResponseInfo(request->response_info())));
 204     response_writer_->WriteInfo(
 205         io_buffer.get(),
 206         base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
 207   } else {
 208     ReadResponseData();
 209   }
 210 }
 211
 212 void AppCacheUpdateJob::URLFetcher::OnReadCompleted(
 213     net::URLRequest* request, int bytes_read) {
 214   DCHECK(request_ == request);
 215   bool data_consumed = true;
 216   if (request->status().is_success() && bytes_read > 0) {
 217     job_->MadeProgress();
 218     data_consumed = ConsumeResponseData(bytes_read);
 219     if (data_consumed) {
 220       bytes_read = 0;
 221       while (request->Read(buffer_.get(), kBufferSize, &bytes_read)) {
 222         if (bytes_read > 0) {
 223           data_consumed = ConsumeResponseData(bytes_read);
 224           if (!data_consumed)
 225             break;  // wait for async data processing, then read more
 226         } else {
 227           break;
 228         }
 229       }
 230     }
 231   }
 232   if (data_consumed && !request->status().is_io_pending()) {
 233     DCHECK_EQ(UPDATE_OK, result_);
 234     OnResponseCompleted();
 235   }
 236 }
 237
 238 void AppCacheUpdateJob::URLFetcher::AddConditionalHeaders(
 239     const net::HttpResponseHeaders* headers) {
 240   DCHECK(request_.get() && headers);
 241   net::HttpRequestHeaders extra_headers;
 242
 243   // Add If-Modified-Since header if response info has Last-Modified header.
 244   const std::string last_modified = "Last-Modified";
 245   std::string last_modified_value;
 246   headers->EnumerateHeader(NULL, last_modified, &last_modified_value);
 247   if (!last_modified_value.empty()) {
 248     extra_headers.SetHeader(net::HttpRequestHeaders::kIfModifiedSince,
 249                             last_modified_value);
 250   }
 251
 252   // Add If-None-Match header if response info has ETag header.
 253   const std::string etag = "ETag";
 254   std::string etag_value;
 255   headers->EnumerateHeader(NULL, etag, &etag_value);
 256   if (!etag_value.empty()) {
 257     extra_headers.SetHeader(net::HttpRequestHeaders::kIfNoneMatch,
 258                             etag_value);
 259   }
 260   if (!extra_headers.IsEmpty())
 261     request_->SetExtraRequestHeaders(extra_headers);
 262 }
 263
 264 void  AppCacheUpdateJob::URLFetcher::OnWriteComplete(int result) {
 265   if (result < 0) {
 266     request_->Cancel();
 267     result_ = DISKCACHE_ERROR;
 268     OnResponseCompleted();
 269     return;
 270   }
 271   ReadResponseData();
 272 }
 273
 274 void AppCacheUpdateJob::URLFetcher::ReadResponseData() {
 275   InternalUpdateState state = job_->internal_state_;
 276   if (state == CACHE_FAILURE || state == CANCELLED || state == COMPLETED)
 277     return;
 278   int bytes_read = 0;
 279   request_->Read(buffer_.get(), kBufferSize, &bytes_read);
 280   OnReadCompleted(request_.get(), bytes_read);
 281 }
 282
 283 // Returns false if response data is processed asynchronously, in which
 284 // case ReadResponseData will be invoked when it is safe to continue
 285 // reading more response data from the request.
 286 bool AppCacheUpdateJob::URLFetcher::ConsumeResponseData(int bytes_read) {
 287   DCHECK_GT(bytes_read, 0);
 288   switch (fetch_type_) {
 289     case MANIFEST_FETCH:
 290     case MANIFEST_REFETCH:
 291       manifest_data_.append(buffer_->data(), bytes_read);
 292       break;
 293     case URL_FETCH:
 294     case MASTER_ENTRY_FETCH:
 295       DCHECK(response_writer_.get());
 296       response_writer_->WriteData(
 297           buffer_.get(),
 298           bytes_read,
 299           base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
 300       return false;  // wait for async write completion to continue reading
 301     default:
 302       NOTREACHED();
 303   }
 304   return true;
 305 }
 306
 307 void AppCacheUpdateJob::URLFetcher::OnResponseCompleted() {
 308   if (request_->status().is_success())
 309     job_->MadeProgress();
 310
 311   // Retry for 503s where retry-after is 0.
 312   if (request_->status().is_success() &&
 313       request_->GetResponseCode() == 503 &&
 314       MaybeRetryRequest()) {
 315     return;
 316   }
 317
 318   switch (fetch_type_) {
 319     case MANIFEST_FETCH:
 320       job_->HandleManifestFetchCompleted(this);
 321       break;
 322     case URL_FETCH:
 323       job_->HandleUrlFetchCompleted(this);
 324       break;
 325     case MASTER_ENTRY_FETCH:
 326       job_->HandleMasterEntryFetchCompleted(this);
 327       break;
 328     case MANIFEST_REFETCH:
 329       job_->HandleManifestRefetchCompleted(this);
 330       break;
 331     default:
 332       NOTREACHED();
 333   }
 334
 335   delete this;
 336 }
 337
 338 bool AppCacheUpdateJob::URLFetcher::MaybeRetryRequest() {
 339   if (retry_503_attempts_ >= kMax503Retries ||
 340       !request_->response_headers()->HasHeaderValue("retry-after", "0")) {
 341     return false;
 342   }
 343   ++retry_503_attempts_;
 344   result_ = UPDATE_OK;
 345   request_ = job_->service_->request_context()->CreateRequest(
 346       url_, net::DEFAULT_PRIORITY, this);
 347   Start();
 348   return true;
 349 }
 350
 351 AppCacheUpdateJob::AppCacheUpdateJob(AppCacheServiceImpl* service,
 352                                      AppCacheGroup* group)
 353     : service_(service),
 354       manifest_url_(group->manifest_url()),
 355       group_(group),
 356       update_type_(UNKNOWN_TYPE),
 357       internal_state_(FETCH_MANIFEST),
 358       master_entries_completed_(0),
 359       url_fetches_completed_(0),
 360       manifest_fetcher_(NULL),
 361       manifest_has_valid_mime_type_(false),
 362       stored_state_(UNSTORED),
 363       storage_(service->storage()) {
 364     service_->AddObserver(this);
 365 }
 366
 367 AppCacheUpdateJob::~AppCacheUpdateJob() {
 368   if (service_)
 369     service_->RemoveObserver(this);
 370   if (internal_state_ != COMPLETED)
 371     Cancel();
 372
 373   DCHECK(!manifest_fetcher_);
 374   DCHECK(pending_url_fetches_.empty());
 375   DCHECK(!inprogress_cache_.get());
 376   DCHECK(pending_master_entries_.empty());
 377   DCHECK(master_entry_fetches_.empty());
 378
 379   if (group_)
 380     group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE);
 381 }
 382
 383 void AppCacheUpdateJob::StartUpdate(AppCacheHost* host,
 384                                     const GURL& new_master_resource) {
 385   DCHECK(group_->update_job() == this);
 386   DCHECK(!group_->is_obsolete());
 387
 388   bool is_new_pending_master_entry = false;
 389   if (!new_master_resource.is_empty()) {
 390     DCHECK(new_master_resource == host->pending_master_entry_url());
 391     DCHECK(!new_master_resource.has_ref());
 392     DCHECK(new_master_resource.GetOrigin() == manifest_url_.GetOrigin());
 393
 394     // Cannot add more to this update if already terminating.
 395     if (IsTerminating()) {
 396       group_->QueueUpdate(host, new_master_resource);
 397       return;
 398     }
 399
 400     std::pair<PendingMasters::iterator, bool> ret =
 401         pending_master_entries_.insert(
 402             PendingMasters::value_type(new_master_resource, PendingHosts()));
 403     is_new_pending_master_entry = ret.second;
 404     ret.first->second.push_back(host);
 405     host->AddObserver(this);
 406   }
 407
 408   // Notify host (if any) if already checking or downloading.
 409   AppCacheGroup::UpdateAppCacheStatus update_status = group_->update_status();
 410   if (update_status == AppCacheGroup::CHECKING ||
 411       update_status == AppCacheGroup::DOWNLOADING) {
 412     if (host) {
 413       NotifySingleHost(host, APPCACHE_CHECKING_EVENT);
 414       if (update_status == AppCacheGroup::DOWNLOADING)
 415         NotifySingleHost(host, APPCACHE_DOWNLOADING_EVENT);
 416
 417       // Add to fetch list or an existing entry if already fetched.
 418       if (!new_master_resource.is_empty()) {
 419         AddMasterEntryToFetchList(host, new_master_resource,
 420                                   is_new_pending_master_entry);
 421       }
 422     }
 423     return;
 424   }
 425
 426   // Begin update process for the group.
 427   MadeProgress();
 428   group_->SetUpdateAppCacheStatus(AppCacheGroup::CHECKING);
 429   if (group_->HasCache()) {
 430     update_type_ = UPGRADE_ATTEMPT;
 431     NotifyAllAssociatedHosts(APPCACHE_CHECKING_EVENT);
 432   } else {
 433     update_type_ = CACHE_ATTEMPT;
 434     DCHECK(host);
 435     NotifySingleHost(host, APPCACHE_CHECKING_EVENT);
 436   }
 437
 438   if (!new_master_resource.is_empty()) {
 439     AddMasterEntryToFetchList(host, new_master_resource,
 440                               is_new_pending_master_entry);
 441   }
 442
 443   FetchManifest(true);
 444 }
 445
 446 AppCacheResponseWriter* AppCacheUpdateJob::CreateResponseWriter() {
 447   AppCacheResponseWriter* writer =
 448       storage_->CreateResponseWriter(manifest_url_,
 449                                                 group_->group_id());
 450   stored_response_ids_.push_back(writer->response_id());
 451   return writer;
 452 }
 453
 454 void AppCacheUpdateJob::HandleCacheFailure(
 455     const AppCacheErrorDetails& error_details,
 456     ResultType result,
 457     const GURL& failed_resource_url) {
 458   // 6.9.4 cache failure steps 2-8.
 459   DCHECK(internal_state_ != CACHE_FAILURE);
 460   DCHECK(!error_details.message.empty());
 461   DCHECK(result != UPDATE_OK);
 462   internal_state_ = CACHE_FAILURE;
 463   LogHistogramStats(result, failed_resource_url);
 464   CancelAllUrlFetches();
 465   CancelAllMasterEntryFetches(error_details);
 466   NotifyAllError(error_details);
 467   DiscardInprogressCache();
 468   internal_state_ = COMPLETED;
 469   DeleteSoon();  // To unwind the stack prior to deletion.
 470 }
 471
 472 void AppCacheUpdateJob::FetchManifest(bool is_first_fetch) {
 473   DCHECK(!manifest_fetcher_);
 474   manifest_fetcher_ = new URLFetcher(
 475      manifest_url_,
 476      is_first_fetch ? URLFetcher::MANIFEST_FETCH :
 477                       URLFetcher::MANIFEST_REFETCH,
 478      this);
 479
 480   // Add any necessary Http headers before sending fetch request.
 481   if (is_first_fetch) {
 482     AppCacheEntry* entry = (update_type_ == UPGRADE_ATTEMPT) ?
 483         group_->newest_complete_cache()->GetEntry(manifest_url_) : NULL;
 484     if (entry) {
 485       // Asynchronously load response info for manifest from newest cache.
 486       storage_->LoadResponseInfo(manifest_url_, group_->group_id(),
 487                                  entry->response_id(), this);
 488     } else {
 489       manifest_fetcher_->Start();
 490     }
 491   } else {
 492     DCHECK(internal_state_ == REFETCH_MANIFEST);
 493     DCHECK(manifest_response_info_.get());
 494     manifest_fetcher_->set_existing_response_headers(
 495         manifest_response_info_->headers.get());
 496     manifest_fetcher_->Start();
 497   }
 498 }
 499
 500
 501 void AppCacheUpdateJob::HandleManifestFetchCompleted(
 502     URLFetcher* fetcher) {
 503   DCHECK_EQ(internal_state_, FETCH_MANIFEST);
 504   DCHECK_EQ(manifest_fetcher_, fetcher);
 505   manifest_fetcher_ = NULL;
 506
 507   net::URLRequest* request = fetcher->request();
 508   int response_code = -1;
 509   bool is_valid_response_code = false;
 510   if (request->status().is_success()) {
 511     response_code = request->GetResponseCode();
 512     is_valid_response_code = (response_code / 100 == 2);
 513
 514     std::string mime_type;
 515     request->GetMimeType(&mime_type);
 516     manifest_has_valid_mime_type_ = (mime_type == "text/cache-manifest");
 517   }
 518
 519   if (is_valid_response_code) {
 520     manifest_data_ = fetcher->manifest_data();
 521     manifest_response_info_.reset(
 522         new net::HttpResponseInfo(request->response_info()));
 523     if (update_type_ == UPGRADE_ATTEMPT)
 524       CheckIfManifestChanged();  // continues asynchronously
 525     else
 526       ContinueHandleManifestFetchCompleted(true);
 527   } else if (response_code == 304 && update_type_ == UPGRADE_ATTEMPT) {
 528     ContinueHandleManifestFetchCompleted(false);
 529   } else if ((response_code == 404 || response_code == 410) &&
 530              update_type_ == UPGRADE_ATTEMPT) {
 531     storage_->MakeGroupObsolete(group_, this, response_code);  // async
 532   } else {
 533     const char* kFormatString = "Manifest fetch failed (%d) %s";
 534     std::string message = FormatUrlErrorMessage(
 535         kFormatString, manifest_url_, fetcher->result(), response_code);
 536     HandleCacheFailure(AppCacheErrorDetails(message,
 537                                     APPCACHE_MANIFEST_ERROR,
 538                                     manifest_url_,
 539                                     response_code,
 540                                     false /*is_cross_origin*/),
 541                        fetcher->result(),
 542                        GURL());
 543   }
 544 }
 545
 546 void AppCacheUpdateJob::OnGroupMadeObsolete(AppCacheGroup* group,
 547                                             bool success,
 548                                             int response_code) {
 549   DCHECK(master_entry_fetches_.empty());
 550   CancelAllMasterEntryFetches(AppCacheErrorDetails(
 551       "The cache has been made obsolete, "
 552       "the manifest file returned 404 or 410",
 553       APPCACHE_MANIFEST_ERROR,
 554       GURL(),
 555       response_code,
 556       false /*is_cross_origin*/));
 557   if (success) {
 558     DCHECK(group->is_obsolete());
 559     NotifyAllAssociatedHosts(APPCACHE_OBSOLETE_EVENT);
 560     internal_state_ = COMPLETED;
 561     MaybeCompleteUpdate();
 562   } else {
 563     // Treat failure to mark group obsolete as a cache failure.
 564     HandleCacheFailure(AppCacheErrorDetails(
 565         "Failed to mark the cache as obsolete",
 566         APPCACHE_UNKNOWN_ERROR,
 567         GURL(),
 568         0,
 569         false /*is_cross_origin*/),
 570                        DB_ERROR,
 571                        GURL());
 572   }
 573 }
 574
 575 void AppCacheUpdateJob::ContinueHandleManifestFetchCompleted(bool changed) {
 576   DCHECK(internal_state_ == FETCH_MANIFEST);
 577
 578   if (!changed) {
 579     DCHECK(update_type_ == UPGRADE_ATTEMPT);
 580     internal_state_ = NO_UPDATE;
 581
 582     // Wait for pending master entries to download.
 583     FetchMasterEntries();
 584     MaybeCompleteUpdate();  // if not done, run async 6.9.4 step 7 substeps
 585     return;
 586   }
 587
 588   AppCacheManifest manifest;
 589   if (!ParseManifest(manifest_url_, manifest_data_.data(),
 590                      manifest_data_.length(),
 591                      manifest_has_valid_mime_type_ ?
 592                         PARSE_MANIFEST_ALLOWING_INTERCEPTS :
 593                         PARSE_MANIFEST_PER_STANDARD,
 594                      manifest)) {
 595     const char* kFormatString = "Failed to parse manifest %s";
 596     const std::string message = base::StringPrintf(kFormatString,
 597         manifest_url_.spec().c_str());
 598     HandleCacheFailure(
 599         AppCacheErrorDetails(
 600             message, APPCACHE_SIGNATURE_ERROR, GURL(), 0,
 601             false /*is_cross_origin*/),
 602         MANIFEST_ERROR,
 603         GURL());
 604     VLOG(1) << message;
 605     return;
 606   }
 607
 608   // Proceed with update process. Section 6.9.4 steps 8-20.
 609   internal_state_ = DOWNLOADING;
 610   inprogress_cache_ = new AppCache(storage_, storage_->NewCacheId());
 611   BuildUrlFileList(manifest);
 612   inprogress_cache_->InitializeWithManifest(&manifest);
 613
 614   // Associate all pending master hosts with the newly created cache.
 615   for (PendingMasters::iterator it = pending_master_entries_.begin();
 616        it != pending_master_entries_.end(); ++it) {
 617     PendingHosts& hosts = it->second;
 618     for (PendingHosts::iterator host_it = hosts.begin();
 619          host_it != hosts.end(); ++host_it) {
 620       (*host_it)
 621           ->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
 622     }
 623   }
 624
 625   if (manifest.did_ignore_intercept_namespaces) {
 626     // Must be done after associating all pending master hosts.
 627     std::string message(
 628         "Ignoring the INTERCEPT section of the application cache manifest "
 629         "because the content type is not text/cache-manifest");
 630     LogConsoleMessageToAll(message);
 631   }
 632
 633   group_->SetUpdateAppCacheStatus(AppCacheGroup::DOWNLOADING);
 634   NotifyAllAssociatedHosts(APPCACHE_DOWNLOADING_EVENT);
 635   FetchUrls();
 636   FetchMasterEntries();
 637   MaybeCompleteUpdate();  // if not done, continues when async fetches complete
 638 }
 639
 640 void AppCacheUpdateJob::HandleUrlFetchCompleted(URLFetcher* fetcher) {
 641   DCHECK(internal_state_ == DOWNLOADING);
 642
 643   net::URLRequest* request = fetcher->request();
 644   const GURL& url = request->original_url();
 645   pending_url_fetches_.erase(url);
 646   NotifyAllProgress(url);
 647   ++url_fetches_completed_;
 648
 649   int response_code = request->status().is_success()
 650                           ? request->GetResponseCode()
 651                           : fetcher->redirect_response_code();
 652
 653   AppCacheEntry& entry = url_file_list_.find(url)->second;
 654
 655   if (response_code / 100 == 2) {
 656     // Associate storage with the new entry.
 657     DCHECK(fetcher->response_writer());
 658     entry.set_response_id(fetcher->response_writer()->response_id());
 659     entry.set_response_size(fetcher->response_writer()->amount_written());
 660     if (!inprogress_cache_->AddOrModifyEntry(url, entry))
 661       duplicate_response_ids_.push_back(entry.response_id());
 662
 663     // TODO(michaeln): Check for <html manifest=xxx>
 664     // See http://code.google.com/p/chromium/issues/detail?id=97930
 665     // if (entry.IsMaster() && !(entry.IsExplicit() || fallback || intercept))
 666     //   if (!manifestAttribute) skip it
 667
 668     // Foreign entries will be detected during cache selection.
 669     // Note: 6.9.4, step 17.9 possible optimization: if resource is HTML or XML
 670     // file whose root element is an html element with a manifest attribute
 671     // whose value doesn't match the manifest url of the application cache
 672     // being processed, mark the entry as being foreign.
 673   } else {
 674     VLOG(1) << "Request status: " << request->status().status()
 675             << " error: " << request->status().error()
 676             << " response code: " << response_code;
 677     if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept()) {
 678       if (response_code == 304 && fetcher->existing_entry().has_response_id()) {
 679         // Keep the existing response.
 680         entry.set_response_id(fetcher->existing_entry().response_id());
 681         entry.set_response_size(fetcher->existing_entry().response_size());
 682         inprogress_cache_->AddOrModifyEntry(url, entry);
 683       } else {
 684         const char* kFormatString = "Resource fetch failed (%d) %s";
 685         std::string message = FormatUrlErrorMessage(
 686             kFormatString, url, fetcher->result(), response_code);
 687         ResultType result = fetcher->result();
 688         bool is_cross_origin = url.GetOrigin() != manifest_url_.GetOrigin();
 689         switch (result) {
 690           case DISKCACHE_ERROR:
 691             HandleCacheFailure(
 692                 AppCacheErrorDetails(
 693                     message, APPCACHE_UNKNOWN_ERROR, GURL(), 0,
 694                     is_cross_origin),
 695                 result,
 696                 url);
 697             break;
 698           case NETWORK_ERROR:
 699             HandleCacheFailure(
 700                 AppCacheErrorDetails(message, APPCACHE_RESOURCE_ERROR, url, 0,
 701                     is_cross_origin),
 702                 result,
 703                 url);
 704             break;
 705           default:
 706             HandleCacheFailure(AppCacheErrorDetails(message,
 707                                             APPCACHE_RESOURCE_ERROR,
 708                                             url,
 709                                             response_code,
 710                                             is_cross_origin),
 711                                result,
 712                                url);
 713             break;
 714         }
 715         return;
 716       }
 717     } else if (response_code == 404 || response_code == 410) {
 718       // Entry is skipped.  They are dropped from the cache.
 719     } else if (update_type_ == UPGRADE_ATTEMPT &&
 720                fetcher->existing_entry().has_response_id()) {
 721       // Keep the existing response.
 722       // TODO(michaeln): Not sure this is a good idea. This is spec compliant
 723       // but the old resource may or may not be compatible with the new contents
 724       // of the cache. Impossible to know one way or the other.
 725       entry.set_response_id(fetcher->existing_entry().response_id());
 726       entry.set_response_size(fetcher->existing_entry().response_size());
 727       inprogress_cache_->AddOrModifyEntry(url, entry);
 728     }
 729   }
 730
 731   // Fetch another URL now that one request has completed.
 732   DCHECK(internal_state_ != CACHE_FAILURE);
 733   FetchUrls();
 734   MaybeCompleteUpdate();
 735 }
 736
 737 void AppCacheUpdateJob::HandleMasterEntryFetchCompleted(
 738     URLFetcher* fetcher) {
 739   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
 740
 741   // TODO(jennb): Handle downloads completing during cache failure when update
 742   // no longer fetches master entries directly. For now, we cancel all pending
 743   // master entry fetches when entering cache failure state so this will never
 744   // be called in CACHE_FAILURE state.
 745
 746   net::URLRequest* request = fetcher->request();
 747   const GURL& url = request->original_url();
 748   master_entry_fetches_.erase(url);
 749   ++master_entries_completed_;
 750
 751   int response_code = request->status().is_success()
 752       ? request->GetResponseCode() : -1;
 753
 754   PendingMasters::iterator found = pending_master_entries_.find(url);
 755   DCHECK(found != pending_master_entries_.end());
 756   PendingHosts& hosts = found->second;
 757
 758   // Section 6.9.4. No update case: step 7.3, else step 22.
 759   if (response_code / 100 == 2) {
 760     // Add fetched master entry to the appropriate cache.
 761     AppCache* cache = inprogress_cache_.get() ? inprogress_cache_.get()
 762                                               : group_->newest_complete_cache();
 763     DCHECK(fetcher->response_writer());
 764     AppCacheEntry master_entry(AppCacheEntry::MASTER,
 765                                fetcher->response_writer()->response_id(),
 766                                fetcher->response_writer()->amount_written());
 767     if (cache->AddOrModifyEntry(url, master_entry))
 768       added_master_entries_.push_back(url);
 769     else
 770       duplicate_response_ids_.push_back(master_entry.response_id());
 771
 772     // In no-update case, associate host with the newest cache.
 773     if (!inprogress_cache_.get()) {
 774       // TODO(michaeln): defer until the updated cache has been stored
 775       DCHECK(cache == group_->newest_complete_cache());
 776       for (PendingHosts::iterator host_it = hosts.begin();
 777            host_it != hosts.end(); ++host_it) {
 778         (*host_it)->AssociateCompleteCache(cache);
 779       }
 780     }
 781   } else {
 782     HostNotifier host_notifier;
 783     for (PendingHosts::iterator host_it = hosts.begin();
 784          host_it != hosts.end(); ++host_it) {
 785       AppCacheHost* host = *host_it;
 786       host_notifier.AddHost(host);
 787
 788       // In downloading case, disassociate host from inprogress cache.
 789       if (inprogress_cache_.get())
 790         host->AssociateNoCache(GURL());
 791
 792       host->RemoveObserver(this);
 793     }
 794     hosts.clear();
 795
 796     const char* kFormatString = "Manifest fetch failed (%d) %s";
 797     std::string message = FormatUrlErrorMessage(
 798         kFormatString, request->url(), fetcher->result(), response_code);
 799     host_notifier.SendErrorNotifications(
 800         AppCacheErrorDetails(message,
 801                      APPCACHE_MANIFEST_ERROR,
 802                      request->url(),
 803                      response_code,
 804                      false /*is_cross_origin*/));
 805
 806     // In downloading case, update result is different if all master entries
 807     // failed vs. only some failing.
 808     if (inprogress_cache_.get()) {
 809       // Only count successful downloads to know if all master entries failed.
 810       pending_master_entries_.erase(found);
 811       --master_entries_completed_;
 812
 813       // Section 6.9.4, step 22.3.
 814       if (update_type_ == CACHE_ATTEMPT && pending_master_entries_.empty()) {
 815         HandleCacheFailure(AppCacheErrorDetails(message,
 816                                         APPCACHE_MANIFEST_ERROR,
 817                                         request->url(),
 818                                         response_code,
 819                                         false /*is_cross_origin*/),
 820                            fetcher->result(),
 821                            GURL());
 822         return;
 823       }
 824     }
 825   }
 826
 827   DCHECK(internal_state_ != CACHE_FAILURE);
 828   FetchMasterEntries();
 829   MaybeCompleteUpdate();
 830 }
 831
 832 void AppCacheUpdateJob::HandleManifestRefetchCompleted(
 833     URLFetcher* fetcher) {
 834   DCHECK(internal_state_ == REFETCH_MANIFEST);
 835   DCHECK(manifest_fetcher_ == fetcher);
 836   manifest_fetcher_ = NULL;
 837
 838   net::URLRequest* request = fetcher->request();
 839   int response_code = request->status().is_success()
 840       ? request->GetResponseCode() : -1;
 841   if (response_code == 304 || manifest_data_ == fetcher->manifest_data()) {
 842     // Only need to store response in storage if manifest is not already
 843     // an entry in the cache.
 844     AppCacheEntry* entry = inprogress_cache_->GetEntry(manifest_url_);
 845     if (entry) {
 846       entry->add_types(AppCacheEntry::MANIFEST);
 847       StoreGroupAndCache();
 848     } else {
 849       manifest_response_writer_.reset(CreateResponseWriter());
 850       scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
 851           new HttpResponseInfoIOBuffer(manifest_response_info_.release()));
 852       manifest_response_writer_->WriteInfo(
 853           io_buffer.get(),
 854           base::Bind(&AppCacheUpdateJob::OnManifestInfoWriteComplete,
 855                      base::Unretained(this)));
 856     }
 857   } else {
 858     VLOG(1) << "Request status: " << request->status().status()
 859             << " error: " << request->status().error()
 860             << " response code: " << response_code;
 861     ScheduleUpdateRetry(kRerunDelayMs);
 862     if (response_code == 200) {
 863       HandleCacheFailure(AppCacheErrorDetails("Manifest changed during update",
 864                                       APPCACHE_CHANGED_ERROR,
 865                                       GURL(),
 866                                       0,
 867                                       false /*is_cross_origin*/),
 868                          MANIFEST_ERROR,
 869                          GURL());
 870     } else {
 871       const char* kFormatString = "Manifest re-fetch failed (%d) %s";
 872       std::string message = FormatUrlErrorMessage(
 873           kFormatString, manifest_url_, fetcher->result(), response_code);
 874       HandleCacheFailure(AppCacheErrorDetails(message,
 875                                       APPCACHE_MANIFEST_ERROR,
 876                                       GURL(),
 877                                       response_code,
 878                                       false /*is_cross_origin*/),
 879                          fetcher->result(),
 880                          GURL());
 881     }
 882   }
 883 }
 884
 885 void AppCacheUpdateJob::OnManifestInfoWriteComplete(int result) {
 886   if (result > 0) {
 887     scoped_refptr<net::StringIOBuffer> io_buffer(
 888         new net::StringIOBuffer(manifest_data_));
 889     manifest_response_writer_->WriteData(
 890         io_buffer.get(),
 891         manifest_data_.length(),
 892         base::Bind(&AppCacheUpdateJob::OnManifestDataWriteComplete,
 893                    base::Unretained(this)));
 894   } else {
 895     HandleCacheFailure(
 896         AppCacheErrorDetails("Failed to write the manifest headers to storage",
 897                      APPCACHE_UNKNOWN_ERROR,
 898                      GURL(),
 899                      0,
 900                      false /*is_cross_origin*/),
 901         DISKCACHE_ERROR,
 902         GURL());
 903   }
 904 }
 905
 906 void AppCacheUpdateJob::OnManifestDataWriteComplete(int result) {
 907   if (result > 0) {
 908     AppCacheEntry entry(AppCacheEntry::MANIFEST,
 909         manifest_response_writer_->response_id(),
 910         manifest_response_writer_->amount_written());
 911     if (!inprogress_cache_->AddOrModifyEntry(manifest_url_, entry))
 912       duplicate_response_ids_.push_back(entry.response_id());
 913     StoreGroupAndCache();
 914   } else {
 915     HandleCacheFailure(
 916         AppCacheErrorDetails("Failed to write the manifest data to storage",
 917                      APPCACHE_UNKNOWN_ERROR,
 918                      GURL(),
 919                      0,
 920                      false /*is_cross_origin*/),
 921         DISKCACHE_ERROR,
 922         GURL());
 923   }
 924 }
 925
 926 void AppCacheUpdateJob::StoreGroupAndCache() {
 927   DCHECK(stored_state_ == UNSTORED);
 928   stored_state_ = STORING;
 929   scoped_refptr<AppCache> newest_cache;
 930   if (inprogress_cache_.get())
 931     newest_cache.swap(inprogress_cache_);
 932   else
 933     newest_cache = group_->newest_complete_cache();
 934   newest_cache->set_update_time(base::Time::Now());
 935
 936   // TODO(michaeln): dcheck is fishing for clues to crbug/95101
 937   DCHECK_EQ(manifest_url_, group_->manifest_url());
 938   storage_->StoreGroupAndNewestCache(group_, newest_cache.get(), this);
 939 }
 940
 941 void AppCacheUpdateJob::OnGroupAndNewestCacheStored(AppCacheGroup* group,
 942                                                     AppCache* newest_cache,
 943                                                     bool success,
 944                                                     bool would_exceed_quota) {
 945   DCHECK(stored_state_ == STORING);
 946   if (success) {
 947     stored_state_ = STORED;
 948     MaybeCompleteUpdate();  // will definitely complete
 949   } else {
 950     stored_state_ = UNSTORED;
 951
 952     // Restore inprogress_cache_ to get the proper events delivered
 953     // and the proper cleanup to occur.
 954     if (newest_cache != group->newest_complete_cache())
 955       inprogress_cache_ = newest_cache;
 956
 957     ResultType result = DB_ERROR;
 958     AppCacheErrorReason reason = APPCACHE_UNKNOWN_ERROR;
 959     std::string message("Failed to commit new cache to storage");
 960     if (would_exceed_quota) {
 961       message.append(", would exceed quota");
 962       result = QUOTA_ERROR;
 963       reason = APPCACHE_QUOTA_ERROR;
 964     }
 965     HandleCacheFailure(
 966         AppCacheErrorDetails(message, reason, GURL(), 0,
 967             false /*is_cross_origin*/),
 968         result,
 969         GURL());
 970   }
 971 }
 972
 973 void AppCacheUpdateJob::NotifySingleHost(AppCacheHost* host,
 974                                          AppCacheEventID event_id) {
 975   std::vector<int> ids(1, host->host_id());
 976   host->frontend()->OnEventRaised(ids, event_id);
 977 }
 978
 979 void AppCacheUpdateJob::NotifyAllAssociatedHosts(AppCacheEventID event_id) {
 980   HostNotifier host_notifier;
 981   AddAllAssociatedHostsToNotifier(&host_notifier);
 982   host_notifier.SendNotifications(event_id);
 983 }
 984
 985 void AppCacheUpdateJob::NotifyAllProgress(const GURL& url) {
 986   HostNotifier host_notifier;
 987   AddAllAssociatedHostsToNotifier(&host_notifier);
 988   host_notifier.SendProgressNotifications(
 989       url, url_file_list_.size(), url_fetches_completed_);
 990 }
 991
 992 void AppCacheUpdateJob::NotifyAllFinalProgress() {
 993   DCHECK(url_file_list_.size() == url_fetches_completed_);
 994   NotifyAllProgress(GURL());
 995 }
 996
 997 void AppCacheUpdateJob::NotifyAllError(const AppCacheErrorDetails& details) {
 998   HostNotifier host_notifier;
 999   AddAllAssociatedHostsToNotifier(&host_notifier);
1000   host_notifier.SendErrorNotifications(details);
1001 }
1002
1003 void AppCacheUpdateJob::LogConsoleMessageToAll(const std::string& message) {
1004   HostNotifier host_notifier;
1005   AddAllAssociatedHostsToNotifier(&host_notifier);
1006   host_notifier.SendLogMessage(message);
1007 }
1008
1009 void AppCacheUpdateJob::AddAllAssociatedHostsToNotifier(
1010     HostNotifier* host_notifier) {
1011   // Collect hosts so we only send one notification per frontend.
1012   // A host can only be associated with a single cache so no need to worry
1013   // about duplicate hosts being added to the notifier.
1014   if (inprogress_cache_.get()) {
1015     DCHECK(internal_state_ == DOWNLOADING || internal_state_ == CACHE_FAILURE);
1016     host_notifier->AddHosts(inprogress_cache_->associated_hosts());
1017   }
1018
1019   AppCacheGroup::Caches old_caches = group_->old_caches();
1020   for (AppCacheGroup::Caches::const_iterator it = old_caches.begin();
1021        it != old_caches.end(); ++it) {
1022     host_notifier->AddHosts((*it)->associated_hosts());
1023   }
1024
1025   AppCache* newest_cache = group_->newest_complete_cache();
1026   if (newest_cache)
1027     host_notifier->AddHosts(newest_cache->associated_hosts());
1028 }
1029
1030 void AppCacheUpdateJob::OnDestructionImminent(AppCacheHost* host) {
1031   // The host is about to be deleted; remove from our collection.
1032   PendingMasters::iterator found =
1033       pending_master_entries_.find(host->pending_master_entry_url());
1034   DCHECK(found != pending_master_entries_.end());
1035   PendingHosts& hosts = found->second;
1036   PendingHosts::iterator it = std::find(hosts.begin(), hosts.end(), host);
1037   DCHECK(it != hosts.end());
1038   hosts.erase(it);
1039 }
1040
1041 void AppCacheUpdateJob::OnServiceReinitialized(
1042     AppCacheStorageReference* old_storage_ref) {
1043   // We continue to use the disabled instance, but arrange for its
1044   // deletion when its no longer needed.
1045   if (old_storage_ref->storage() == storage_)
1046     disabled_storage_reference_ = old_storage_ref;
1047 }
1048
1049 void AppCacheUpdateJob::CheckIfManifestChanged() {
1050   DCHECK(update_type_ == UPGRADE_ATTEMPT);
1051   AppCacheEntry* entry = NULL;
1052   if (group_->newest_complete_cache())
1053     entry = group_->newest_complete_cache()->GetEntry(manifest_url_);
1054   if (!entry) {
1055     // TODO(michaeln): This is just a bandaid to avoid a crash.
1056     // http://code.google.com/p/chromium/issues/detail?id=95101
1057     if (service_->storage() == storage_) {
1058       // Use a local variable because service_ is reset in HandleCacheFailure.
1059       AppCacheServiceImpl* service = service_;
1060       HandleCacheFailure(
1061           AppCacheErrorDetails("Manifest entry not found in existing cache",
1062                        APPCACHE_UNKNOWN_ERROR,
1063                        GURL(),
1064                        0,
1065                        false /*is_cross_origin*/),
1066           DB_ERROR,
1067           GURL());
1068       AppCacheHistograms::AddMissingManifestEntrySample();
1069       service->DeleteAppCacheGroup(manifest_url_, net::CompletionCallback());
1070     }
1071     return;
1072   }
1073
1074   // Load manifest data from storage to compare against fetched manifest.
1075   manifest_response_reader_.reset(
1076       storage_->CreateResponseReader(manifest_url_,
1077                                      group_->group_id(),
1078                                      entry->response_id()));
1079   read_manifest_buffer_ = new net::IOBuffer(kBufferSize);
1080   manifest_response_reader_->ReadData(
1081       read_manifest_buffer_.get(),
1082       kBufferSize,
1083       base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
1084                  base::Unretained(this)));  // async read
1085 }
1086
1087 void AppCacheUpdateJob::OnManifestDataReadComplete(int result) {
1088   if (result > 0) {
1089     loaded_manifest_data_.append(read_manifest_buffer_->data(), result);
1090     manifest_response_reader_->ReadData(
1091         read_manifest_buffer_.get(),
1092         kBufferSize,
1093         base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
1094                    base::Unretained(this)));  // read more
1095   } else {
1096     read_manifest_buffer_ = NULL;
1097     manifest_response_reader_.reset();
1098     ContinueHandleManifestFetchCompleted(
1099         result < 0 || manifest_data_ != loaded_manifest_data_);
1100   }
1101 }
1102
1103 void AppCacheUpdateJob::BuildUrlFileList(const AppCacheManifest& manifest) {
1104   for (base::hash_set<std::string>::const_iterator it =
1105            manifest.explicit_urls.begin();
1106        it != manifest.explicit_urls.end(); ++it) {
1107     AddUrlToFileList(GURL(*it), AppCacheEntry::EXPLICIT);
1108   }
1109
1110   const std::vector<AppCacheNamespace>& intercepts =
1111       manifest.intercept_namespaces;
1112   for (std::vector<AppCacheNamespace>::const_iterator it = intercepts.begin();
1113        it != intercepts.end(); ++it) {
1114     int flags = AppCacheEntry::INTERCEPT;
1115     if (it->is_executable)
1116       flags |= AppCacheEntry::EXECUTABLE;
1117     AddUrlToFileList(it->target_url, flags);
1118   }
1119
1120   const std::vector<AppCacheNamespace>& fallbacks =
1121       manifest.fallback_namespaces;
1122   for (std::vector<AppCacheNamespace>::const_iterator it = fallbacks.begin();
1123        it != fallbacks.end(); ++it) {
1124      AddUrlToFileList(it->target_url, AppCacheEntry::FALLBACK);
1125   }
1126
1127   // Add all master entries from newest complete cache.
1128   if (update_type_ == UPGRADE_ATTEMPT) {
1129     const AppCache::EntryMap& entries =
1130         group_->newest_complete_cache()->entries();
1131     for (AppCache::EntryMap::const_iterator it = entries.begin();
1132          it != entries.end(); ++it) {
1133       const AppCacheEntry& entry = it->second;
1134       if (entry.IsMaster())
1135         AddUrlToFileList(it->first, AppCacheEntry::MASTER);
1136     }
1137   }
1138 }
1139
1140 void AppCacheUpdateJob::AddUrlToFileList(const GURL& url, int type) {
1141   std::pair<AppCache::EntryMap::iterator, bool> ret = url_file_list_.insert(
1142       AppCache::EntryMap::value_type(url, AppCacheEntry(type)));
1143
1144   if (ret.second)
1145     urls_to_fetch_.push_back(UrlToFetch(url, false, NULL));
1146   else
1147     ret.first->second.add_types(type);  // URL already exists. Merge types.
1148 }
1149
1150 void AppCacheUpdateJob::FetchUrls() {
1151   DCHECK(internal_state_ == DOWNLOADING);
1152
1153   // Fetch each URL in the list according to section 6.9.4 step 17.1-17.3.
1154   // Fetch up to the concurrent limit. Other fetches will be triggered as each
1155   // each fetch completes.
1156   while (pending_url_fetches_.size() < kMaxConcurrentUrlFetches &&
1157          !urls_to_fetch_.empty()) {
1158     UrlToFetch url_to_fetch = urls_to_fetch_.front();
1159     urls_to_fetch_.pop_front();
1160
1161     AppCache::EntryMap::iterator it = url_file_list_.find(url_to_fetch.url);
1162     DCHECK(it != url_file_list_.end());
1163     AppCacheEntry& entry = it->second;
1164     if (ShouldSkipUrlFetch(entry)) {
1165       NotifyAllProgress(url_to_fetch.url);
1166       ++url_fetches_completed_;
1167     } else if (AlreadyFetchedEntry(url_to_fetch.url, entry.types())) {
1168       NotifyAllProgress(url_to_fetch.url);
1169       ++url_fetches_completed_;  // saved a URL request
1170     } else if (!url_to_fetch.storage_checked &&
1171                MaybeLoadFromNewestCache(url_to_fetch.url, entry)) {
1172       // Continues asynchronously after data is loaded from newest cache.
1173     } else {
1174       URLFetcher* fetcher = new URLFetcher(
1175           url_to_fetch.url, URLFetcher::URL_FETCH, this);
1176       if (url_to_fetch.existing_response_info.get()) {
1177         DCHECK(group_->newest_complete_cache());
1178         AppCacheEntry* existing_entry =
1179             group_->newest_complete_cache()->GetEntry(url_to_fetch.url);
1180         DCHECK(existing_entry);
1181         DCHECK(existing_entry->response_id() ==
1182                url_to_fetch.existing_response_info->response_id());
1183         fetcher->set_existing_response_headers(
1184             url_to_fetch.existing_response_info->http_response_info()->headers
1185                 .get());
1186         fetcher->set_existing_entry(*existing_entry);
1187       }
1188       fetcher->Start();
1189       pending_url_fetches_.insert(
1190           PendingUrlFetches::value_type(url_to_fetch.url, fetcher));
1191     }
1192   }
1193 }
1194
1195 void AppCacheUpdateJob::CancelAllUrlFetches() {
1196   // Cancel any pending URL requests.
1197   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
1198        it != pending_url_fetches_.end(); ++it) {
1199     delete it->second;
1200   }
1201
1202   url_fetches_completed_ +=
1203       pending_url_fetches_.size() + urls_to_fetch_.size();
1204   pending_url_fetches_.clear();
1205   urls_to_fetch_.clear();
1206 }
1207
1208 bool AppCacheUpdateJob::ShouldSkipUrlFetch(const AppCacheEntry& entry) {
1209   // 6.6.4 Step 17
1210   // If the resource URL being processed was flagged as neither an
1211   // "explicit entry" nor or a "fallback entry", then the user agent
1212   // may skip this URL.
1213   if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept())
1214     return false;
1215
1216   // TODO(jennb): decide if entry should be skipped to expire it from cache
1217   return false;
1218 }
1219
1220 bool AppCacheUpdateJob::AlreadyFetchedEntry(const GURL& url,
1221                                             int entry_type) {
1222   DCHECK(internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE);
1223   AppCacheEntry* existing =
1224       inprogress_cache_.get() ? inprogress_cache_->GetEntry(url)
1225                               : group_->newest_complete_cache()->GetEntry(url);
1226   if (existing) {
1227     existing->add_types(entry_type);
1228     return true;
1229   }
1230   return false;
1231 }
1232
1233 void AppCacheUpdateJob::AddMasterEntryToFetchList(AppCacheHost* host,
1234                                                   const GURL& url,
1235                                                   bool is_new) {
1236   DCHECK(!IsTerminating());
1237
1238   if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE) {
1239     AppCache* cache;
1240     if (inprogress_cache_.get()) {
1241       // always associate
1242       host->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
1243       cache = inprogress_cache_.get();
1244     } else {
1245       cache = group_->newest_complete_cache();
1246     }
1247
1248     // Update existing entry if it has already been fetched.
1249     AppCacheEntry* entry = cache->GetEntry(url);
1250     if (entry) {
1251       entry->add_types(AppCacheEntry::MASTER);
1252       if (internal_state_ == NO_UPDATE && !inprogress_cache_.get()) {
1253         // only associate if have entry
1254         host->AssociateCompleteCache(cache);
1255       }
1256       if (is_new)
1257         ++master_entries_completed_;  // pretend fetching completed
1258       return;
1259     }
1260   }
1261
1262   // Add to fetch list if not already fetching.
1263   if (master_entry_fetches_.find(url) == master_entry_fetches_.end()) {
1264     master_entries_to_fetch_.insert(url);
1265     if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE)
1266       FetchMasterEntries();
1267   }
1268 }
1269
1270 void AppCacheUpdateJob::FetchMasterEntries() {
1271   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
1272
1273   // Fetch each master entry in the list, up to the concurrent limit.
1274   // Additional fetches will be triggered as each fetch completes.
1275   while (master_entry_fetches_.size() < kMaxConcurrentUrlFetches &&
1276          !master_entries_to_fetch_.empty()) {
1277     const GURL& url = *master_entries_to_fetch_.begin();
1278
1279     if (AlreadyFetchedEntry(url, AppCacheEntry::MASTER)) {
1280       ++master_entries_completed_;  // saved a URL request
1281
1282       // In no update case, associate hosts to newest cache in group
1283       // now that master entry has been "successfully downloaded".
1284       if (internal_state_ == NO_UPDATE) {
1285         // TODO(michaeln): defer until the updated cache has been stored.
1286         DCHECK(!inprogress_cache_.get());
1287         AppCache* cache = group_->newest_complete_cache();
1288         PendingMasters::iterator found = pending_master_entries_.find(url);
1289         DCHECK(found != pending_master_entries_.end());
1290         PendingHosts& hosts = found->second;
1291         for (PendingHosts::iterator host_it = hosts.begin();
1292              host_it != hosts.end(); ++host_it) {
1293           (*host_it)->AssociateCompleteCache(cache);
1294         }
1295       }
1296     } else {
1297       URLFetcher* fetcher = new URLFetcher(
1298           url, URLFetcher::MASTER_ENTRY_FETCH, this);
1299       fetcher->Start();
1300       master_entry_fetches_.insert(PendingUrlFetches::value_type(url, fetcher));
1301     }
1302
1303     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
1304   }
1305 }
1306
1307 void AppCacheUpdateJob::CancelAllMasterEntryFetches(
1308     const AppCacheErrorDetails& error_details) {
1309   // For now, cancel all in-progress fetches for master entries and pretend
1310   // all master entries fetches have completed.
1311   // TODO(jennb): Delete this when update no longer fetches master entries
1312   // directly.
1313
1314   // Cancel all in-progress fetches.
1315   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
1316        it != master_entry_fetches_.end(); ++it) {
1317     delete it->second;
1318     master_entries_to_fetch_.insert(it->first);  // back in unfetched list
1319   }
1320   master_entry_fetches_.clear();
1321
1322   master_entries_completed_ += master_entries_to_fetch_.size();
1323
1324   // Cache failure steps, step 2.
1325   // Pretend all master entries that have not yet been fetched have completed
1326   // downloading. Unassociate hosts from any appcache and send ERROR event.
1327   HostNotifier host_notifier;
1328   while (!master_entries_to_fetch_.empty()) {
1329     const GURL& url = *master_entries_to_fetch_.begin();
1330     PendingMasters::iterator found = pending_master_entries_.find(url);
1331     DCHECK(found != pending_master_entries_.end());
1332     PendingHosts& hosts = found->second;
1333     for (PendingHosts::iterator host_it = hosts.begin();
1334          host_it != hosts.end(); ++host_it) {
1335       AppCacheHost* host = *host_it;
1336       host->AssociateNoCache(GURL());
1337       host_notifier.AddHost(host);
1338       host->RemoveObserver(this);
1339     }
1340     hosts.clear();
1341
1342     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
1343   }
1344   host_notifier.SendErrorNotifications(error_details);
1345 }
1346
1347 bool AppCacheUpdateJob::MaybeLoadFromNewestCache(const GURL& url,
1348                                                  AppCacheEntry& entry) {
1349   if (update_type_ != UPGRADE_ATTEMPT)
1350     return false;
1351
1352   AppCache* newest = group_->newest_complete_cache();
1353   AppCacheEntry* copy_me = newest->GetEntry(url);
1354   if (!copy_me || !copy_me->has_response_id())
1355     return false;
1356
1357   // Load HTTP headers for entry from newest cache.
1358   loading_responses_.insert(
1359       LoadingResponses::value_type(copy_me->response_id(), url));
1360   storage_->LoadResponseInfo(manifest_url_, group_->group_id(),
1361                              copy_me->response_id(),
1362                              this);
1363   // Async: wait for OnResponseInfoLoaded to complete.
1364   return true;
1365 }
1366
1367 void AppCacheUpdateJob::OnResponseInfoLoaded(
1368     AppCacheResponseInfo* response_info, int64 response_id) {
1369   const net::HttpResponseInfo* http_info = response_info ?
1370       response_info->http_response_info() : NULL;
1371
1372   // Needed response info for a manifest fetch request.
1373   if (internal_state_ == FETCH_MANIFEST) {
1374     if (http_info)
1375       manifest_fetcher_->set_existing_response_headers(
1376           http_info->headers.get());
1377     manifest_fetcher_->Start();
1378     return;
1379   }
1380
1381   LoadingResponses::iterator found = loading_responses_.find(response_id);
1382   DCHECK(found != loading_responses_.end());
1383   const GURL& url = found->second;
1384
1385   if (!http_info) {
1386     LoadFromNewestCacheFailed(url, NULL);  // no response found
1387   } else {
1388     // Check if response can be re-used according to HTTP caching semantics.
1389     // Responses with a "vary" header get treated as expired.
1390     const std::string name = "vary";
1391     std::string value;
1392     void* iter = NULL;
1393     if (!http_info->headers.get() ||
1394         http_info->headers->RequiresValidation(http_info->request_time,
1395                                                http_info->response_time,
1396                                                base::Time::Now()) ||
1397         http_info->headers->EnumerateHeader(&iter, name, &value)) {
1398       LoadFromNewestCacheFailed(url, response_info);
1399     } else {
1400       DCHECK(group_->newest_complete_cache());
1401       AppCacheEntry* copy_me = group_->newest_complete_cache()->GetEntry(url);
1402       DCHECK(copy_me);
1403       DCHECK(copy_me->response_id() == response_id);
1404
1405       AppCache::EntryMap::iterator it = url_file_list_.find(url);
1406       DCHECK(it != url_file_list_.end());
1407       AppCacheEntry& entry = it->second;
1408       entry.set_response_id(response_id);
1409       entry.set_response_size(copy_me->response_size());
1410       inprogress_cache_->AddOrModifyEntry(url, entry);
1411       NotifyAllProgress(url);
1412       ++url_fetches_completed_;
1413     }
1414   }
1415   loading_responses_.erase(found);
1416
1417   MaybeCompleteUpdate();
1418 }
1419
1420 void AppCacheUpdateJob::LoadFromNewestCacheFailed(
1421     const GURL& url, AppCacheResponseInfo* response_info) {
1422   if (internal_state_ == CACHE_FAILURE)
1423     return;
1424
1425   // Re-insert url at front of fetch list. Indicate storage has been checked.
1426   urls_to_fetch_.push_front(UrlToFetch(url, true, response_info));
1427   FetchUrls();
1428 }
1429
1430 void AppCacheUpdateJob::MaybeCompleteUpdate() {
1431   DCHECK(internal_state_ != CACHE_FAILURE);
1432
1433   // Must wait for any pending master entries or url fetches to complete.
1434   if (master_entries_completed_ != pending_master_entries_.size() ||
1435       url_fetches_completed_ != url_file_list_.size()) {
1436     DCHECK(internal_state_ != COMPLETED);
1437     return;
1438   }
1439
1440   switch (internal_state_) {
1441     case NO_UPDATE:
1442       if (master_entries_completed_ > 0) {
1443         switch (stored_state_) {
1444           case UNSTORED:
1445             StoreGroupAndCache();
1446             return;
1447           case STORING:
1448             return;
1449           case STORED:
1450             break;
1451         }
1452       }
1453       // 6.9.4 steps 7.3-7.7.
1454       NotifyAllAssociatedHosts(APPCACHE_NO_UPDATE_EVENT);
1455       DiscardDuplicateResponses();
1456       internal_state_ = COMPLETED;
1457       break;
1458     case DOWNLOADING:
1459       internal_state_ = REFETCH_MANIFEST;
1460       FetchManifest(false);
1461       break;
1462     case REFETCH_MANIFEST:
1463       DCHECK(stored_state_ == STORED);
1464       NotifyAllFinalProgress();
1465       if (update_type_ == CACHE_ATTEMPT)
1466         NotifyAllAssociatedHosts(APPCACHE_CACHED_EVENT);
1467       else
1468         NotifyAllAssociatedHosts(APPCACHE_UPDATE_READY_EVENT);
1469       DiscardDuplicateResponses();
1470       internal_state_ = COMPLETED;
1471       LogHistogramStats(UPDATE_OK, GURL());
1472       break;
1473     case CACHE_FAILURE:
1474       NOTREACHED();  // See HandleCacheFailure
1475       break;
1476     default:
1477       break;
1478   }
1479
1480   // Let the stack unwind before deletion to make it less risky as this
1481   // method is called from multiple places in this file.
1482   if (internal_state_ == COMPLETED)
1483     DeleteSoon();
1484 }
1485
1486 void AppCacheUpdateJob::ScheduleUpdateRetry(int delay_ms) {
1487   // TODO(jennb): post a delayed task with the "same parameters" as this job
1488   // to retry the update at a later time. Need group, URLs of pending master
1489   // entries and their hosts.
1490 }
1491
1492 void AppCacheUpdateJob::Cancel() {
1493   internal_state_ = CANCELLED;
1494
1495   LogHistogramStats(CANCELLED_ERROR, GURL());
1496
1497   if (manifest_fetcher_) {
1498     delete manifest_fetcher_;
1499     manifest_fetcher_ = NULL;
1500   }
1501
1502   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
1503        it != pending_url_fetches_.end(); ++it) {
1504     delete it->second;
1505   }
1506   pending_url_fetches_.clear();
1507
1508   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
1509        it != master_entry_fetches_.end(); ++it) {
1510     delete it->second;
1511   }
1512   master_entry_fetches_.clear();
1513
1514   ClearPendingMasterEntries();
1515   DiscardInprogressCache();
1516
1517   // Delete response writer to avoid any callbacks.
1518   if (manifest_response_writer_)
1519     manifest_response_writer_.reset();
1520
1521   storage_->CancelDelegateCallbacks(this);
1522 }
1523
1524 void AppCacheUpdateJob::ClearPendingMasterEntries() {
1525   for (PendingMasters::iterator it = pending_master_entries_.begin();
1526        it != pending_master_entries_.end(); ++it) {
1527     PendingHosts& hosts = it->second;
1528     for (PendingHosts::iterator host_it = hosts.begin();
1529          host_it != hosts.end(); ++host_it) {
1530       (*host_it)->RemoveObserver(this);
1531     }
1532   }
1533
1534   pending_master_entries_.clear();
1535 }
1536
1537 void AppCacheUpdateJob::DiscardInprogressCache() {
1538   if (stored_state_ == STORING) {
1539     // We can make no assumptions about whether the StoreGroupAndCacheTask
1540     // actually completed or not. This condition should only be reachable
1541     // during shutdown. Free things up and return to do no harm.
1542     inprogress_cache_ = NULL;
1543     added_master_entries_.clear();
1544     return;
1545   }
1546
1547   storage_->DoomResponses(manifest_url_, stored_response_ids_);
1548
1549   if (!inprogress_cache_.get()) {
1550     // We have to undo the changes we made, if any, to the existing cache.
1551     if (group_ && group_->newest_complete_cache()) {
1552       for (std::vector<GURL>::iterator iter = added_master_entries_.begin();
1553            iter != added_master_entries_.end(); ++iter) {
1554         group_->newest_complete_cache()->RemoveEntry(*iter);
1555       }
1556     }
1557     added_master_entries_.clear();
1558     return;
1559   }
1560
1561   AppCache::AppCacheHosts& hosts = inprogress_cache_->associated_hosts();
1562   while (!hosts.empty())
1563     (*hosts.begin())->AssociateNoCache(GURL());
1564
1565   inprogress_cache_ = NULL;
1566   added_master_entries_.clear();
1567 }
1568
1569 void AppCacheUpdateJob::DiscardDuplicateResponses() {
1570   storage_->DoomResponses(manifest_url_, duplicate_response_ids_);
1571 }
1572
1573 void AppCacheUpdateJob::LogHistogramStats(
1574       ResultType result, const GURL& failed_resource_url) {
1575   AppCacheHistograms::CountUpdateJobResult(result, manifest_url_.GetOrigin());
1576   if (result == UPDATE_OK)
1577     return;
1578
1579   int percent_complete = 0;
1580   if (url_file_list_.size() > 0) {
1581     size_t actual_fetches_completed = url_fetches_completed_;
1582     if (!failed_resource_url.is_empty() && actual_fetches_completed)
1583       --actual_fetches_completed;
1584     percent_complete = (static_cast<double>(actual_fetches_completed) /
1585                             static_cast<double>(url_file_list_.size())) * 100.0;
1586     percent_complete = std::min(percent_complete, 99);
1587   }
1588
1589   bool was_making_progress =
1590       base::Time::Now() - last_progress_time_ <
1591           base::TimeDelta::FromMinutes(5);
1592
1593   bool off_origin_resource_failure =
1594       !failed_resource_url.is_empty() &&
1595           (failed_resource_url.GetOrigin() != manifest_url_.GetOrigin());
1596
1597   AppCacheHistograms::LogUpdateFailureStats(
1598       manifest_url_.GetOrigin(),
1599       percent_complete,
1600       was_making_progress,
1601       off_origin_resource_failure);
1602 }
1603
1604 void AppCacheUpdateJob::DeleteSoon() {
1605   ClearPendingMasterEntries();
1606   manifest_response_writer_.reset();
1607   storage_->CancelDelegateCallbacks(this);
1608   service_->RemoveObserver(this);
1609   service_ = NULL;
1610
1611   // Break the connection with the group so the group cannot call delete
1612   // on this object after we've posted a task to delete ourselves.
1613   group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE);
1614   group_ = NULL;
1615
1616   base::MessageLoop::current()->DeleteSoon(FROM_HERE, this);
1617 }
1618
1619 }  // namespace content