content/browser/appcache/appcache_update_job.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/appcache/appcache_update_job.h"
   6
   7 #include "base/bind.h"
   8 #include "base/bind_helpers.h"
   9 #include "base/compiler_specific.h"
  10 #include "base/message_loop/message_loop.h"
  11 #include "base/strings/string_util.h"
  12 #include "base/strings/stringprintf.h"
  13 #include "content/browser/appcache/appcache_group.h"
  14 #include "content/browser/appcache/appcache_histograms.h"
  15 #include "net/base/io_buffer.h"
  16 #include "net/base/load_flags.h"
  17 #include "net/base/net_errors.h"
  18 #include "net/base/request_priority.h"
  19 #include "net/http/http_request_headers.h"
  20 #include "net/http/http_response_headers.h"
  21 #include "net/url_request/url_request_context.h"
  22
  23 namespace content {
  24
  25 static const int kBufferSize = 32768;
  26 static const size_t kMaxConcurrentUrlFetches = 2;
  27 static const int kMax503Retries = 3;
  28
  29 static std::string FormatUrlErrorMessage(
  30       const char* format, const GURL& url,
  31       AppCacheUpdateJob::ResultType error,
  32       int response_code) {
  33     // Show the net response code if we have one.
  34     int code = response_code;
  35     if (error != AppCacheUpdateJob::SERVER_ERROR)
  36       code = static_cast<int>(error);
  37     return base::StringPrintf(format, code, url.spec().c_str());
  38 }
  39
  40 // Helper class for collecting hosts per frontend when sending notifications
  41 // so that only one notification is sent for all hosts using the same frontend.
  42 class HostNotifier {
  43  public:
  44   typedef std::vector<int> HostIds;
  45   typedef std::map<AppCacheFrontend*, HostIds> NotifyHostMap;
  46
  47   // Caller is responsible for ensuring there will be no duplicate hosts.
  48   void AddHost(AppCacheHost* host) {
  49     std::pair<NotifyHostMap::iterator , bool> ret = hosts_to_notify.insert(
  50         NotifyHostMap::value_type(host->frontend(), HostIds()));
  51     ret.first->second.push_back(host->host_id());
  52   }
  53
  54   void AddHosts(const std::set<AppCacheHost*>& hosts) {
  55     for (std::set<AppCacheHost*>::const_iterator it = hosts.begin();
  56          it != hosts.end(); ++it) {
  57       AddHost(*it);
  58     }
  59   }
  60
  61   void SendNotifications(AppCacheEventID event_id) {
  62     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  63          it != hosts_to_notify.end(); ++it) {
  64       AppCacheFrontend* frontend = it->first;
  65       frontend->OnEventRaised(it->second, event_id);
  66     }
  67   }
  68
  69   void SendProgressNotifications(
  70       const GURL& url, int num_total, int num_complete) {
  71     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  72          it != hosts_to_notify.end(); ++it) {
  73       AppCacheFrontend* frontend = it->first;
  74       frontend->OnProgressEventRaised(it->second, url,
  75                                       num_total, num_complete);
  76     }
  77   }
  78
  79   void SendErrorNotifications(const AppCacheErrorDetails& details) {
  80     DCHECK(!details.message.empty());
  81     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  82          it != hosts_to_notify.end(); ++it) {
  83       AppCacheFrontend* frontend = it->first;
  84       frontend->OnErrorEventRaised(it->second, details);
  85     }
  86   }
  87
  88   void SendLogMessage(const std::string& message) {
  89     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  90          it != hosts_to_notify.end(); ++it) {
  91       AppCacheFrontend* frontend = it->first;
  92       for (HostIds::iterator id = it->second.begin();
  93            id != it->second.end(); ++id) {
  94         frontend->OnLogMessage(*id, APPCACHE_LOG_WARNING, message);
  95       }
  96     }
  97   }
  98
  99  private:
 100   NotifyHostMap hosts_to_notify;
 101 };
 102
 103 AppCacheUpdateJob::UrlToFetch::UrlToFetch(const GURL& url,
 104                                           bool checked,
 105                                           AppCacheResponseInfo* info)
 106     : url(url),
 107       storage_checked(checked),
 108       existing_response_info(info) {
 109 }
 110
 111 AppCacheUpdateJob::UrlToFetch::~UrlToFetch() {
 112 }
 113
 114 // Helper class to fetch resources. Depending on the fetch type,
 115 // can either fetch to an in-memory string or write the response
 116 // data out to the disk cache.
 117 AppCacheUpdateJob::URLFetcher::URLFetcher(const GURL& url,
 118                                           FetchType fetch_type,
 119                                           AppCacheUpdateJob* job)
 120     : url_(url),
 121       job_(job),
 122       fetch_type_(fetch_type),
 123       retry_503_attempts_(0),
 124       buffer_(new net::IOBuffer(kBufferSize)),
 125       request_(job->service_->request_context()
 126                    ->CreateRequest(url, net::DEFAULT_PRIORITY, this, NULL)),
 127       result_(UPDATE_OK),
 128       redirect_response_code_(-1) {}
 129
 130 AppCacheUpdateJob::URLFetcher::~URLFetcher() {
 131 }
 132
 133 void AppCacheUpdateJob::URLFetcher::Start() {
 134   request_->set_first_party_for_cookies(job_->manifest_url_);
 135   request_->SetLoadFlags(request_->load_flags() |
 136                          net::LOAD_DISABLE_INTERCEPT);
 137   if (existing_response_headers_.get())
 138     AddConditionalHeaders(existing_response_headers_.get());
 139   request_->Start();
 140 }
 141
 142 void AppCacheUpdateJob::URLFetcher::OnReceivedRedirect(
 143     net::URLRequest* request,
 144     const net::RedirectInfo& redirect_info,
 145     bool* defer_redirect) {
 146   DCHECK(request_ == request);
 147   // Redirect is not allowed by the update process.
 148   job_->MadeProgress();
 149   redirect_response_code_ = request->GetResponseCode();
 150   request->Cancel();
 151   result_ = REDIRECT_ERROR;
 152   OnResponseCompleted();
 153 }
 154
 155 void AppCacheUpdateJob::URLFetcher::OnResponseStarted(
 156     net::URLRequest *request) {
 157   DCHECK(request == request_);
 158   int response_code = -1;
 159   if (request->status().is_success()) {
 160     response_code = request->GetResponseCode();
 161     job_->MadeProgress();
 162   }
 163
 164   if ((response_code / 100) != 2) {
 165     if (response_code > 0)
 166       result_ = SERVER_ERROR;
 167     else
 168       result_ = NETWORK_ERROR;
 169     OnResponseCompleted();
 170     return;
 171   }
 172
 173   if (url_.SchemeIsSecure()) {
 174     // Do not cache content with cert errors.
 175     // Also, we willfully violate the HTML5 spec at this point in order
 176     // to support the appcaching of cross-origin HTTPS resources.
 177     // We've opted for a milder constraint and allow caching unless
 178     // the resource has a "no-store" header. A spec change has been
 179     // requested on the whatwg list.
 180     // See http://code.google.com/p/chromium/issues/detail?id=69594
 181     // TODO(michaeln): Consider doing this for cross-origin HTTP too.
 182     if (net::IsCertStatusError(request->ssl_info().cert_status) ||
 183         (url_.GetOrigin() != job_->manifest_url_.GetOrigin() &&
 184             request->response_headers()->
 185                 HasHeaderValue("cache-control", "no-store"))) {
 186       DCHECK_EQ(-1, redirect_response_code_);
 187       request->Cancel();
 188       result_ = SECURITY_ERROR;
 189       OnResponseCompleted();
 190       return;
 191     }
 192   }
 193
 194   // Write response info to storage for URL fetches. Wait for async write
 195   // completion before reading any response data.
 196   if (fetch_type_ == URL_FETCH || fetch_type_ == MASTER_ENTRY_FETCH) {
 197     response_writer_.reset(job_->CreateResponseWriter());
 198     scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
 199         new HttpResponseInfoIOBuffer(
 200             new net::HttpResponseInfo(request->response_info())));
 201     response_writer_->WriteInfo(
 202         io_buffer.get(),
 203         base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
 204   } else {
 205     ReadResponseData();
 206   }
 207 }
 208
 209 void AppCacheUpdateJob::URLFetcher::OnReadCompleted(
 210     net::URLRequest* request, int bytes_read) {
 211   DCHECK(request_ == request);
 212   bool data_consumed = true;
 213   if (request->status().is_success() && bytes_read > 0) {
 214     job_->MadeProgress();
 215     data_consumed = ConsumeResponseData(bytes_read);
 216     if (data_consumed) {
 217       bytes_read = 0;
 218       while (request->Read(buffer_.get(), kBufferSize, &bytes_read)) {
 219         if (bytes_read > 0) {
 220           data_consumed = ConsumeResponseData(bytes_read);
 221           if (!data_consumed)
 222             break;  // wait for async data processing, then read more
 223         } else {
 224           break;
 225         }
 226       }
 227     }
 228   }
 229   if (data_consumed && !request->status().is_io_pending()) {
 230     DCHECK_EQ(UPDATE_OK, result_);
 231     OnResponseCompleted();
 232   }
 233 }
 234
 235 void AppCacheUpdateJob::URLFetcher::AddConditionalHeaders(
 236     const net::HttpResponseHeaders* headers) {
 237   DCHECK(request_.get() && headers);
 238   net::HttpRequestHeaders extra_headers;
 239
 240   // Add If-Modified-Since header if response info has Last-Modified header.
 241   const std::string last_modified = "Last-Modified";
 242   std::string last_modified_value;
 243   headers->EnumerateHeader(NULL, last_modified, &last_modified_value);
 244   if (!last_modified_value.empty()) {
 245     extra_headers.SetHeader(net::HttpRequestHeaders::kIfModifiedSince,
 246                             last_modified_value);
 247   }
 248
 249   // Add If-None-Match header if response info has ETag header.
 250   const std::string etag = "ETag";
 251   std::string etag_value;
 252   headers->EnumerateHeader(NULL, etag, &etag_value);
 253   if (!etag_value.empty()) {
 254     extra_headers.SetHeader(net::HttpRequestHeaders::kIfNoneMatch,
 255                             etag_value);
 256   }
 257   if (!extra_headers.IsEmpty())
 258     request_->SetExtraRequestHeaders(extra_headers);
 259 }
 260
 261 void  AppCacheUpdateJob::URLFetcher::OnWriteComplete(int result) {
 262   if (result < 0) {
 263     request_->Cancel();
 264     result_ = DISKCACHE_ERROR;
 265     OnResponseCompleted();
 266     return;
 267   }
 268   ReadResponseData();
 269 }
 270
 271 void AppCacheUpdateJob::URLFetcher::ReadResponseData() {
 272   InternalUpdateState state = job_->internal_state_;
 273   if (state == CACHE_FAILURE || state == CANCELLED || state == COMPLETED)
 274     return;
 275   int bytes_read = 0;
 276   request_->Read(buffer_.get(), kBufferSize, &bytes_read);
 277   OnReadCompleted(request_.get(), bytes_read);
 278 }
 279
 280 // Returns false if response data is processed asynchronously, in which
 281 // case ReadResponseData will be invoked when it is safe to continue
 282 // reading more response data from the request.
 283 bool AppCacheUpdateJob::URLFetcher::ConsumeResponseData(int bytes_read) {
 284   DCHECK_GT(bytes_read, 0);
 285   switch (fetch_type_) {
 286     case MANIFEST_FETCH:
 287     case MANIFEST_REFETCH:
 288       manifest_data_.append(buffer_->data(), bytes_read);
 289       break;
 290     case URL_FETCH:
 291     case MASTER_ENTRY_FETCH:
 292       DCHECK(response_writer_.get());
 293       response_writer_->WriteData(
 294           buffer_.get(),
 295           bytes_read,
 296           base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
 297       return false;  // wait for async write completion to continue reading
 298     default:
 299       NOTREACHED();
 300   }
 301   return true;
 302 }
 303
 304 void AppCacheUpdateJob::URLFetcher::OnResponseCompleted() {
 305   if (request_->status().is_success())
 306     job_->MadeProgress();
 307
 308   // Retry for 503s where retry-after is 0.
 309   if (request_->status().is_success() &&
 310       request_->GetResponseCode() == 503 &&
 311       MaybeRetryRequest()) {
 312     return;
 313   }
 314
 315   switch (fetch_type_) {
 316     case MANIFEST_FETCH:
 317       job_->HandleManifestFetchCompleted(this);
 318       break;
 319     case URL_FETCH:
 320       job_->HandleUrlFetchCompleted(this);
 321       break;
 322     case MASTER_ENTRY_FETCH:
 323       job_->HandleMasterEntryFetchCompleted(this);
 324       break;
 325     case MANIFEST_REFETCH:
 326       job_->HandleManifestRefetchCompleted(this);
 327       break;
 328     default:
 329       NOTREACHED();
 330   }
 331
 332   delete this;
 333 }
 334
 335 bool AppCacheUpdateJob::URLFetcher::MaybeRetryRequest() {
 336   if (retry_503_attempts_ >= kMax503Retries ||
 337       !request_->response_headers()->HasHeaderValue("retry-after", "0")) {
 338     return false;
 339   }
 340   ++retry_503_attempts_;
 341   result_ = UPDATE_OK;
 342   request_ = job_->service_->request_context()->CreateRequest(
 343       url_, net::DEFAULT_PRIORITY, this, NULL);
 344   Start();
 345   return true;
 346 }
 347
 348 AppCacheUpdateJob::AppCacheUpdateJob(AppCacheServiceImpl* service,
 349                                      AppCacheGroup* group)
 350     : service_(service),
 351       manifest_url_(group->manifest_url()),
 352       group_(group),
 353       update_type_(UNKNOWN_TYPE),
 354       internal_state_(FETCH_MANIFEST),
 355       master_entries_completed_(0),
 356       url_fetches_completed_(0),
 357       manifest_fetcher_(NULL),
 358       manifest_has_valid_mime_type_(false),
 359       stored_state_(UNSTORED),
 360       storage_(service->storage()) {
 361     service_->AddObserver(this);
 362 }
 363
 364 AppCacheUpdateJob::~AppCacheUpdateJob() {
 365   if (service_)
 366     service_->RemoveObserver(this);
 367   if (internal_state_ != COMPLETED)
 368     Cancel();
 369
 370   DCHECK(!manifest_fetcher_);
 371   DCHECK(pending_url_fetches_.empty());
 372   DCHECK(!inprogress_cache_.get());
 373   DCHECK(pending_master_entries_.empty());
 374   DCHECK(master_entry_fetches_.empty());
 375
 376   if (group_)
 377     group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE);
 378 }
 379
 380 void AppCacheUpdateJob::StartUpdate(AppCacheHost* host,
 381                                     const GURL& new_master_resource) {
 382   DCHECK(group_->update_job() == this);
 383   DCHECK(!group_->is_obsolete());
 384
 385   bool is_new_pending_master_entry = false;
 386   if (!new_master_resource.is_empty()) {
 387     DCHECK(new_master_resource == host->pending_master_entry_url());
 388     DCHECK(!new_master_resource.has_ref());
 389     DCHECK(new_master_resource.GetOrigin() == manifest_url_.GetOrigin());
 390
 391     // Cannot add more to this update if already terminating.
 392     if (IsTerminating()) {
 393       group_->QueueUpdate(host, new_master_resource);
 394       return;
 395     }
 396
 397     std::pair<PendingMasters::iterator, bool> ret =
 398         pending_master_entries_.insert(
 399             PendingMasters::value_type(new_master_resource, PendingHosts()));
 400     is_new_pending_master_entry = ret.second;
 401     ret.first->second.push_back(host);
 402     host->AddObserver(this);
 403   }
 404
 405   // Notify host (if any) if already checking or downloading.
 406   AppCacheGroup::UpdateAppCacheStatus update_status = group_->update_status();
 407   if (update_status == AppCacheGroup::CHECKING ||
 408       update_status == AppCacheGroup::DOWNLOADING) {
 409     if (host) {
 410       NotifySingleHost(host, APPCACHE_CHECKING_EVENT);
 411       if (update_status == AppCacheGroup::DOWNLOADING)
 412         NotifySingleHost(host, APPCACHE_DOWNLOADING_EVENT);
 413
 414       // Add to fetch list or an existing entry if already fetched.
 415       if (!new_master_resource.is_empty()) {
 416         AddMasterEntryToFetchList(host, new_master_resource,
 417                                   is_new_pending_master_entry);
 418       }
 419     }
 420     return;
 421   }
 422
 423   // Begin update process for the group.
 424   MadeProgress();
 425   group_->SetUpdateAppCacheStatus(AppCacheGroup::CHECKING);
 426   if (group_->HasCache()) {
 427     update_type_ = UPGRADE_ATTEMPT;
 428     NotifyAllAssociatedHosts(APPCACHE_CHECKING_EVENT);
 429   } else {
 430     update_type_ = CACHE_ATTEMPT;
 431     DCHECK(host);
 432     NotifySingleHost(host, APPCACHE_CHECKING_EVENT);
 433   }
 434
 435   if (!new_master_resource.is_empty()) {
 436     AddMasterEntryToFetchList(host, new_master_resource,
 437                               is_new_pending_master_entry);
 438   }
 439
 440   FetchManifest(true);
 441 }
 442
 443 AppCacheResponseWriter* AppCacheUpdateJob::CreateResponseWriter() {
 444   AppCacheResponseWriter* writer =
 445       storage_->CreateResponseWriter(manifest_url_,
 446                                                 group_->group_id());
 447   stored_response_ids_.push_back(writer->response_id());
 448   return writer;
 449 }
 450
 451 void AppCacheUpdateJob::HandleCacheFailure(
 452     const AppCacheErrorDetails& error_details,
 453     ResultType result,
 454     const GURL& failed_resource_url) {
 455   // 6.9.4 cache failure steps 2-8.
 456   DCHECK(internal_state_ != CACHE_FAILURE);
 457   DCHECK(!error_details.message.empty());
 458   DCHECK(result != UPDATE_OK);
 459   internal_state_ = CACHE_FAILURE;
 460   LogHistogramStats(result, failed_resource_url);
 461   CancelAllUrlFetches();
 462   CancelAllMasterEntryFetches(error_details);
 463   NotifyAllError(error_details);
 464   DiscardInprogressCache();
 465   internal_state_ = COMPLETED;
 466   DeleteSoon();  // To unwind the stack prior to deletion.
 467 }
 468
 469 void AppCacheUpdateJob::FetchManifest(bool is_first_fetch) {
 470   DCHECK(!manifest_fetcher_);
 471   manifest_fetcher_ = new URLFetcher(
 472      manifest_url_,
 473      is_first_fetch ? URLFetcher::MANIFEST_FETCH :
 474                       URLFetcher::MANIFEST_REFETCH,
 475      this);
 476
 477   // Add any necessary Http headers before sending fetch request.
 478   if (is_first_fetch) {
 479     AppCacheEntry* entry = (update_type_ == UPGRADE_ATTEMPT) ?
 480         group_->newest_complete_cache()->GetEntry(manifest_url_) : NULL;
 481     if (entry) {
 482       // Asynchronously load response info for manifest from newest cache.
 483       storage_->LoadResponseInfo(manifest_url_, group_->group_id(),
 484                                  entry->response_id(), this);
 485     } else {
 486       manifest_fetcher_->Start();
 487     }
 488   } else {
 489     DCHECK(internal_state_ == REFETCH_MANIFEST);
 490     DCHECK(manifest_response_info_.get());
 491     manifest_fetcher_->set_existing_response_headers(
 492         manifest_response_info_->headers.get());
 493     manifest_fetcher_->Start();
 494   }
 495 }
 496
 497
 498 void AppCacheUpdateJob::HandleManifestFetchCompleted(
 499     URLFetcher* fetcher) {
 500   DCHECK_EQ(internal_state_, FETCH_MANIFEST);
 501   DCHECK_EQ(manifest_fetcher_, fetcher);
 502   manifest_fetcher_ = NULL;
 503
 504   net::URLRequest* request = fetcher->request();
 505   int response_code = -1;
 506   bool is_valid_response_code = false;
 507   if (request->status().is_success()) {
 508     response_code = request->GetResponseCode();
 509     is_valid_response_code = (response_code / 100 == 2);
 510
 511     std::string mime_type;
 512     request->GetMimeType(&mime_type);
 513     manifest_has_valid_mime_type_ = (mime_type == "text/cache-manifest");
 514   }
 515
 516   if (is_valid_response_code) {
 517     manifest_data_ = fetcher->manifest_data();
 518     manifest_response_info_.reset(
 519         new net::HttpResponseInfo(request->response_info()));
 520     if (update_type_ == UPGRADE_ATTEMPT)
 521       CheckIfManifestChanged();  // continues asynchronously
 522     else
 523       ContinueHandleManifestFetchCompleted(true);
 524   } else if (response_code == 304 && update_type_ == UPGRADE_ATTEMPT) {
 525     ContinueHandleManifestFetchCompleted(false);
 526   } else if ((response_code == 404 || response_code == 410) &&
 527              update_type_ == UPGRADE_ATTEMPT) {
 528     storage_->MakeGroupObsolete(group_, this, response_code);  // async
 529   } else {
 530     const char* kFormatString = "Manifest fetch failed (%d) %s";
 531     std::string message = FormatUrlErrorMessage(
 532         kFormatString, manifest_url_, fetcher->result(), response_code);
 533     HandleCacheFailure(AppCacheErrorDetails(message,
 534                                     APPCACHE_MANIFEST_ERROR,
 535                                     manifest_url_,
 536                                     response_code,
 537                                     false /*is_cross_origin*/),
 538                        fetcher->result(),
 539                        GURL());
 540   }
 541 }
 542
 543 void AppCacheUpdateJob::OnGroupMadeObsolete(AppCacheGroup* group,
 544                                             bool success,
 545                                             int response_code) {
 546   DCHECK(master_entry_fetches_.empty());
 547   CancelAllMasterEntryFetches(AppCacheErrorDetails(
 548       "The cache has been made obsolete, "
 549       "the manifest file returned 404 or 410",
 550       APPCACHE_MANIFEST_ERROR,
 551       GURL(),
 552       response_code,
 553       false /*is_cross_origin*/));
 554   if (success) {
 555     DCHECK(group->is_obsolete());
 556     NotifyAllAssociatedHosts(APPCACHE_OBSOLETE_EVENT);
 557     internal_state_ = COMPLETED;
 558     MaybeCompleteUpdate();
 559   } else {
 560     // Treat failure to mark group obsolete as a cache failure.
 561     HandleCacheFailure(AppCacheErrorDetails(
 562         "Failed to mark the cache as obsolete",
 563         APPCACHE_UNKNOWN_ERROR,
 564         GURL(),
 565         0,
 566         false /*is_cross_origin*/),
 567                        DB_ERROR,
 568                        GURL());
 569   }
 570 }
 571
 572 void AppCacheUpdateJob::ContinueHandleManifestFetchCompleted(bool changed) {
 573   DCHECK(internal_state_ == FETCH_MANIFEST);
 574
 575   if (!changed) {
 576     DCHECK(update_type_ == UPGRADE_ATTEMPT);
 577     internal_state_ = NO_UPDATE;
 578
 579     // Wait for pending master entries to download.
 580     FetchMasterEntries();
 581     MaybeCompleteUpdate();  // if not done, run async 6.9.4 step 7 substeps
 582     return;
 583   }
 584
 585   AppCacheManifest manifest;
 586   if (!ParseManifest(manifest_url_, manifest_data_.data(),
 587                      manifest_data_.length(),
 588                      manifest_has_valid_mime_type_ ?
 589                         PARSE_MANIFEST_ALLOWING_INTERCEPTS :
 590                         PARSE_MANIFEST_PER_STANDARD,
 591                      manifest)) {
 592     const char* kFormatString = "Failed to parse manifest %s";
 593     const std::string message = base::StringPrintf(kFormatString,
 594         manifest_url_.spec().c_str());
 595     HandleCacheFailure(
 596         AppCacheErrorDetails(
 597             message, APPCACHE_SIGNATURE_ERROR, GURL(), 0,
 598             false /*is_cross_origin*/),
 599         MANIFEST_ERROR,
 600         GURL());
 601     VLOG(1) << message;
 602     return;
 603   }
 604
 605   // Proceed with update process. Section 6.9.4 steps 8-20.
 606   internal_state_ = DOWNLOADING;
 607   inprogress_cache_ = new AppCache(storage_, storage_->NewCacheId());
 608   BuildUrlFileList(manifest);
 609   inprogress_cache_->InitializeWithManifest(&manifest);
 610
 611   // Associate all pending master hosts with the newly created cache.
 612   for (PendingMasters::iterator it = pending_master_entries_.begin();
 613        it != pending_master_entries_.end(); ++it) {
 614     PendingHosts& hosts = it->second;
 615     for (PendingHosts::iterator host_it = hosts.begin();
 616          host_it != hosts.end(); ++host_it) {
 617       (*host_it)
 618           ->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
 619     }
 620   }
 621
 622   if (manifest.did_ignore_intercept_namespaces) {
 623     // Must be done after associating all pending master hosts.
 624     std::string message(
 625         "Ignoring the INTERCEPT section of the application cache manifest "
 626         "because the content type is not text/cache-manifest");
 627     LogConsoleMessageToAll(message);
 628   }
 629
 630   group_->SetUpdateAppCacheStatus(AppCacheGroup::DOWNLOADING);
 631   NotifyAllAssociatedHosts(APPCACHE_DOWNLOADING_EVENT);
 632   FetchUrls();
 633   FetchMasterEntries();
 634   MaybeCompleteUpdate();  // if not done, continues when async fetches complete
 635 }
 636
 637 void AppCacheUpdateJob::HandleUrlFetchCompleted(URLFetcher* fetcher) {
 638   DCHECK(internal_state_ == DOWNLOADING);
 639
 640   net::URLRequest* request = fetcher->request();
 641   const GURL& url = request->original_url();
 642   pending_url_fetches_.erase(url);
 643   NotifyAllProgress(url);
 644   ++url_fetches_completed_;
 645
 646   int response_code = request->status().is_success()
 647                           ? request->GetResponseCode()
 648                           : fetcher->redirect_response_code();
 649
 650   AppCacheEntry& entry = url_file_list_.find(url)->second;
 651
 652   if (response_code / 100 == 2) {
 653     // Associate storage with the new entry.
 654     DCHECK(fetcher->response_writer());
 655     entry.set_response_id(fetcher->response_writer()->response_id());
 656     entry.set_response_size(fetcher->response_writer()->amount_written());
 657     if (!inprogress_cache_->AddOrModifyEntry(url, entry))
 658       duplicate_response_ids_.push_back(entry.response_id());
 659
 660     // TODO(michaeln): Check for <html manifest=xxx>
 661     // See http://code.google.com/p/chromium/issues/detail?id=97930
 662     // if (entry.IsMaster() && !(entry.IsExplicit() || fallback || intercept))
 663     //   if (!manifestAttribute) skip it
 664
 665     // Foreign entries will be detected during cache selection.
 666     // Note: 6.9.4, step 17.9 possible optimization: if resource is HTML or XML
 667     // file whose root element is an html element with a manifest attribute
 668     // whose value doesn't match the manifest url of the application cache
 669     // being processed, mark the entry as being foreign.
 670   } else {
 671     VLOG(1) << "Request status: " << request->status().status()
 672             << " error: " << request->status().error()
 673             << " response code: " << response_code;
 674     if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept()) {
 675       if (response_code == 304 && fetcher->existing_entry().has_response_id()) {
 676         // Keep the existing response.
 677         entry.set_response_id(fetcher->existing_entry().response_id());
 678         entry.set_response_size(fetcher->existing_entry().response_size());
 679         inprogress_cache_->AddOrModifyEntry(url, entry);
 680       } else {
 681         const char* kFormatString = "Resource fetch failed (%d) %s";
 682         std::string message = FormatUrlErrorMessage(
 683             kFormatString, url, fetcher->result(), response_code);
 684         ResultType result = fetcher->result();
 685         bool is_cross_origin = url.GetOrigin() != manifest_url_.GetOrigin();
 686         switch (result) {
 687           case DISKCACHE_ERROR:
 688             HandleCacheFailure(
 689                 AppCacheErrorDetails(
 690                     message, APPCACHE_UNKNOWN_ERROR, GURL(), 0,
 691                     is_cross_origin),
 692                 result,
 693                 url);
 694             break;
 695           case NETWORK_ERROR:
 696             HandleCacheFailure(
 697                 AppCacheErrorDetails(message, APPCACHE_RESOURCE_ERROR, url, 0,
 698                     is_cross_origin),
 699                 result,
 700                 url);
 701             break;
 702           default:
 703             HandleCacheFailure(AppCacheErrorDetails(message,
 704                                             APPCACHE_RESOURCE_ERROR,
 705                                             url,
 706                                             response_code,
 707                                             is_cross_origin),
 708                                result,
 709                                url);
 710             break;
 711         }
 712         return;
 713       }
 714     } else if (response_code == 404 || response_code == 410) {
 715       // Entry is skipped.  They are dropped from the cache.
 716     } else if (update_type_ == UPGRADE_ATTEMPT &&
 717                fetcher->existing_entry().has_response_id()) {
 718       // Keep the existing response.
 719       // TODO(michaeln): Not sure this is a good idea. This is spec compliant
 720       // but the old resource may or may not be compatible with the new contents
 721       // of the cache. Impossible to know one way or the other.
 722       entry.set_response_id(fetcher->existing_entry().response_id());
 723       entry.set_response_size(fetcher->existing_entry().response_size());
 724       inprogress_cache_->AddOrModifyEntry(url, entry);
 725     }
 726   }
 727
 728   // Fetch another URL now that one request has completed.
 729   DCHECK(internal_state_ != CACHE_FAILURE);
 730   FetchUrls();
 731   MaybeCompleteUpdate();
 732 }
 733
 734 void AppCacheUpdateJob::HandleMasterEntryFetchCompleted(
 735     URLFetcher* fetcher) {
 736   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
 737
 738   // TODO(jennb): Handle downloads completing during cache failure when update
 739   // no longer fetches master entries directly. For now, we cancel all pending
 740   // master entry fetches when entering cache failure state so this will never
 741   // be called in CACHE_FAILURE state.
 742
 743   net::URLRequest* request = fetcher->request();
 744   const GURL& url = request->original_url();
 745   master_entry_fetches_.erase(url);
 746   ++master_entries_completed_;
 747
 748   int response_code = request->status().is_success()
 749       ? request->GetResponseCode() : -1;
 750
 751   PendingMasters::iterator found = pending_master_entries_.find(url);
 752   DCHECK(found != pending_master_entries_.end());
 753   PendingHosts& hosts = found->second;
 754
 755   // Section 6.9.4. No update case: step 7.3, else step 22.
 756   if (response_code / 100 == 2) {
 757     // Add fetched master entry to the appropriate cache.
 758     AppCache* cache = inprogress_cache_.get() ? inprogress_cache_.get()
 759                                               : group_->newest_complete_cache();
 760     DCHECK(fetcher->response_writer());
 761     AppCacheEntry master_entry(AppCacheEntry::MASTER,
 762                                fetcher->response_writer()->response_id(),
 763                                fetcher->response_writer()->amount_written());
 764     if (cache->AddOrModifyEntry(url, master_entry))
 765       added_master_entries_.push_back(url);
 766     else
 767       duplicate_response_ids_.push_back(master_entry.response_id());
 768
 769     // In no-update case, associate host with the newest cache.
 770     if (!inprogress_cache_.get()) {
 771       // TODO(michaeln): defer until the updated cache has been stored
 772       DCHECK(cache == group_->newest_complete_cache());
 773       for (PendingHosts::iterator host_it = hosts.begin();
 774            host_it != hosts.end(); ++host_it) {
 775         (*host_it)->AssociateCompleteCache(cache);
 776       }
 777     }
 778   } else {
 779     HostNotifier host_notifier;
 780     for (PendingHosts::iterator host_it = hosts.begin();
 781          host_it != hosts.end(); ++host_it) {
 782       AppCacheHost* host = *host_it;
 783       host_notifier.AddHost(host);
 784
 785       // In downloading case, disassociate host from inprogress cache.
 786       if (inprogress_cache_.get())
 787         host->AssociateNoCache(GURL());
 788
 789       host->RemoveObserver(this);
 790     }
 791     hosts.clear();
 792
 793     const char* kFormatString = "Manifest fetch failed (%d) %s";
 794     std::string message = FormatUrlErrorMessage(
 795         kFormatString, request->url(), fetcher->result(), response_code);
 796     host_notifier.SendErrorNotifications(
 797         AppCacheErrorDetails(message,
 798                      APPCACHE_MANIFEST_ERROR,
 799                      request->url(),
 800                      response_code,
 801                      false /*is_cross_origin*/));
 802
 803     // In downloading case, update result is different if all master entries
 804     // failed vs. only some failing.
 805     if (inprogress_cache_.get()) {
 806       // Only count successful downloads to know if all master entries failed.
 807       pending_master_entries_.erase(found);
 808       --master_entries_completed_;
 809
 810       // Section 6.9.4, step 22.3.
 811       if (update_type_ == CACHE_ATTEMPT && pending_master_entries_.empty()) {
 812         HandleCacheFailure(AppCacheErrorDetails(message,
 813                                         APPCACHE_MANIFEST_ERROR,
 814                                         request->url(),
 815                                         response_code,
 816                                         false /*is_cross_origin*/),
 817                            fetcher->result(),
 818                            GURL());
 819         return;
 820       }
 821     }
 822   }
 823
 824   DCHECK(internal_state_ != CACHE_FAILURE);
 825   FetchMasterEntries();
 826   MaybeCompleteUpdate();
 827 }
 828
 829 void AppCacheUpdateJob::HandleManifestRefetchCompleted(
 830     URLFetcher* fetcher) {
 831   DCHECK(internal_state_ == REFETCH_MANIFEST);
 832   DCHECK(manifest_fetcher_ == fetcher);
 833   manifest_fetcher_ = NULL;
 834
 835   net::URLRequest* request = fetcher->request();
 836   int response_code = request->status().is_success()
 837       ? request->GetResponseCode() : -1;
 838   if (response_code == 304 || manifest_data_ == fetcher->manifest_data()) {
 839     // Only need to store response in storage if manifest is not already
 840     // an entry in the cache.
 841     AppCacheEntry* entry = inprogress_cache_->GetEntry(manifest_url_);
 842     if (entry) {
 843       entry->add_types(AppCacheEntry::MANIFEST);
 844       StoreGroupAndCache();
 845     } else {
 846       manifest_response_writer_.reset(CreateResponseWriter());
 847       scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
 848           new HttpResponseInfoIOBuffer(manifest_response_info_.release()));
 849       manifest_response_writer_->WriteInfo(
 850           io_buffer.get(),
 851           base::Bind(&AppCacheUpdateJob::OnManifestInfoWriteComplete,
 852                      base::Unretained(this)));
 853     }
 854   } else {
 855     VLOG(1) << "Request status: " << request->status().status()
 856             << " error: " << request->status().error()
 857             << " response code: " << response_code;
 858     ScheduleUpdateRetry(kRerunDelayMs);
 859     if (response_code == 200) {
 860       HandleCacheFailure(AppCacheErrorDetails("Manifest changed during update",
 861                                       APPCACHE_CHANGED_ERROR,
 862                                       GURL(),
 863                                       0,
 864                                       false /*is_cross_origin*/),
 865                          MANIFEST_ERROR,
 866                          GURL());
 867     } else {
 868       const char* kFormatString = "Manifest re-fetch failed (%d) %s";
 869       std::string message = FormatUrlErrorMessage(
 870           kFormatString, manifest_url_, fetcher->result(), response_code);
 871       HandleCacheFailure(AppCacheErrorDetails(message,
 872                                       APPCACHE_MANIFEST_ERROR,
 873                                       GURL(),
 874                                       response_code,
 875                                       false /*is_cross_origin*/),
 876                          fetcher->result(),
 877                          GURL());
 878     }
 879   }
 880 }
 881
 882 void AppCacheUpdateJob::OnManifestInfoWriteComplete(int result) {
 883   if (result > 0) {
 884     scoped_refptr<net::StringIOBuffer> io_buffer(
 885         new net::StringIOBuffer(manifest_data_));
 886     manifest_response_writer_->WriteData(
 887         io_buffer.get(),
 888         manifest_data_.length(),
 889         base::Bind(&AppCacheUpdateJob::OnManifestDataWriteComplete,
 890                    base::Unretained(this)));
 891   } else {
 892     HandleCacheFailure(
 893         AppCacheErrorDetails("Failed to write the manifest headers to storage",
 894                      APPCACHE_UNKNOWN_ERROR,
 895                      GURL(),
 896                      0,
 897                      false /*is_cross_origin*/),
 898         DISKCACHE_ERROR,
 899         GURL());
 900   }
 901 }
 902
 903 void AppCacheUpdateJob::OnManifestDataWriteComplete(int result) {
 904   if (result > 0) {
 905     AppCacheEntry entry(AppCacheEntry::MANIFEST,
 906         manifest_response_writer_->response_id(),
 907         manifest_response_writer_->amount_written());
 908     if (!inprogress_cache_->AddOrModifyEntry(manifest_url_, entry))
 909       duplicate_response_ids_.push_back(entry.response_id());
 910     StoreGroupAndCache();
 911   } else {
 912     HandleCacheFailure(
 913         AppCacheErrorDetails("Failed to write the manifest data to storage",
 914                      APPCACHE_UNKNOWN_ERROR,
 915                      GURL(),
 916                      0,
 917                      false /*is_cross_origin*/),
 918         DISKCACHE_ERROR,
 919         GURL());
 920   }
 921 }
 922
 923 void AppCacheUpdateJob::StoreGroupAndCache() {
 924   DCHECK(stored_state_ == UNSTORED);
 925   stored_state_ = STORING;
 926   scoped_refptr<AppCache> newest_cache;
 927   if (inprogress_cache_.get())
 928     newest_cache.swap(inprogress_cache_);
 929   else
 930     newest_cache = group_->newest_complete_cache();
 931   newest_cache->set_update_time(base::Time::Now());
 932
 933   // TODO(michaeln): dcheck is fishing for clues to crbug/95101
 934   DCHECK_EQ(manifest_url_, group_->manifest_url());
 935   storage_->StoreGroupAndNewestCache(group_, newest_cache.get(), this);
 936 }
 937
 938 void AppCacheUpdateJob::OnGroupAndNewestCacheStored(AppCacheGroup* group,
 939                                                     AppCache* newest_cache,
 940                                                     bool success,
 941                                                     bool would_exceed_quota) {
 942   DCHECK(stored_state_ == STORING);
 943   if (success) {
 944     stored_state_ = STORED;
 945     MaybeCompleteUpdate();  // will definitely complete
 946   } else {
 947     stored_state_ = UNSTORED;
 948
 949     // Restore inprogress_cache_ to get the proper events delivered
 950     // and the proper cleanup to occur.
 951     if (newest_cache != group->newest_complete_cache())
 952       inprogress_cache_ = newest_cache;
 953
 954     ResultType result = DB_ERROR;
 955     AppCacheErrorReason reason = APPCACHE_UNKNOWN_ERROR;
 956     std::string message("Failed to commit new cache to storage");
 957     if (would_exceed_quota) {
 958       message.append(", would exceed quota");
 959       result = QUOTA_ERROR;
 960       reason = APPCACHE_QUOTA_ERROR;
 961     }
 962     HandleCacheFailure(
 963         AppCacheErrorDetails(message, reason, GURL(), 0,
 964             false /*is_cross_origin*/),
 965         result,
 966         GURL());
 967   }
 968 }
 969
 970 void AppCacheUpdateJob::NotifySingleHost(AppCacheHost* host,
 971                                          AppCacheEventID event_id) {
 972   std::vector<int> ids(1, host->host_id());
 973   host->frontend()->OnEventRaised(ids, event_id);
 974 }
 975
 976 void AppCacheUpdateJob::NotifyAllAssociatedHosts(AppCacheEventID event_id) {
 977   HostNotifier host_notifier;
 978   AddAllAssociatedHostsToNotifier(&host_notifier);
 979   host_notifier.SendNotifications(event_id);
 980 }
 981
 982 void AppCacheUpdateJob::NotifyAllProgress(const GURL& url) {
 983   HostNotifier host_notifier;
 984   AddAllAssociatedHostsToNotifier(&host_notifier);
 985   host_notifier.SendProgressNotifications(
 986       url, url_file_list_.size(), url_fetches_completed_);
 987 }
 988
 989 void AppCacheUpdateJob::NotifyAllFinalProgress() {
 990   DCHECK(url_file_list_.size() == url_fetches_completed_);
 991   NotifyAllProgress(GURL());
 992 }
 993
 994 void AppCacheUpdateJob::NotifyAllError(const AppCacheErrorDetails& details) {
 995   HostNotifier host_notifier;
 996   AddAllAssociatedHostsToNotifier(&host_notifier);
 997   host_notifier.SendErrorNotifications(details);
 998 }
 999
1000 void AppCacheUpdateJob::LogConsoleMessageToAll(const std::string& message) {
1001   HostNotifier host_notifier;
1002   AddAllAssociatedHostsToNotifier(&host_notifier);
1003   host_notifier.SendLogMessage(message);
1004 }
1005
1006 void AppCacheUpdateJob::AddAllAssociatedHostsToNotifier(
1007     HostNotifier* host_notifier) {
1008   // Collect hosts so we only send one notification per frontend.
1009   // A host can only be associated with a single cache so no need to worry
1010   // about duplicate hosts being added to the notifier.
1011   if (inprogress_cache_.get()) {
1012     DCHECK(internal_state_ == DOWNLOADING || internal_state_ == CACHE_FAILURE);
1013     host_notifier->AddHosts(inprogress_cache_->associated_hosts());
1014   }
1015
1016   AppCacheGroup::Caches old_caches = group_->old_caches();
1017   for (AppCacheGroup::Caches::const_iterator it = old_caches.begin();
1018        it != old_caches.end(); ++it) {
1019     host_notifier->AddHosts((*it)->associated_hosts());
1020   }
1021
1022   AppCache* newest_cache = group_->newest_complete_cache();
1023   if (newest_cache)
1024     host_notifier->AddHosts(newest_cache->associated_hosts());
1025 }
1026
1027 void AppCacheUpdateJob::OnDestructionImminent(AppCacheHost* host) {
1028   // The host is about to be deleted; remove from our collection.
1029   PendingMasters::iterator found =
1030       pending_master_entries_.find(host->pending_master_entry_url());
1031   DCHECK(found != pending_master_entries_.end());
1032   PendingHosts& hosts = found->second;
1033   PendingHosts::iterator it = std::find(hosts.begin(), hosts.end(), host);
1034   DCHECK(it != hosts.end());
1035   hosts.erase(it);
1036 }
1037
1038 void AppCacheUpdateJob::OnServiceReinitialized(
1039     AppCacheStorageReference* old_storage_ref) {
1040   // We continue to use the disabled instance, but arrange for its
1041   // deletion when its no longer needed.
1042   if (old_storage_ref->storage() == storage_)
1043     disabled_storage_reference_ = old_storage_ref;
1044 }
1045
1046 void AppCacheUpdateJob::CheckIfManifestChanged() {
1047   DCHECK(update_type_ == UPGRADE_ATTEMPT);
1048   AppCacheEntry* entry = NULL;
1049   if (group_->newest_complete_cache())
1050     entry = group_->newest_complete_cache()->GetEntry(manifest_url_);
1051   if (!entry) {
1052     // TODO(michaeln): This is just a bandaid to avoid a crash.
1053     // http://code.google.com/p/chromium/issues/detail?id=95101
1054     if (service_->storage() == storage_) {
1055       // Use a local variable because service_ is reset in HandleCacheFailure.
1056       AppCacheServiceImpl* service = service_;
1057       HandleCacheFailure(
1058           AppCacheErrorDetails("Manifest entry not found in existing cache",
1059                        APPCACHE_UNKNOWN_ERROR,
1060                        GURL(),
1061                        0,
1062                        false /*is_cross_origin*/),
1063           DB_ERROR,
1064           GURL());
1065       AppCacheHistograms::AddMissingManifestEntrySample();
1066       service->DeleteAppCacheGroup(manifest_url_, net::CompletionCallback());
1067     }
1068     return;
1069   }
1070
1071   // Load manifest data from storage to compare against fetched manifest.
1072   manifest_response_reader_.reset(
1073       storage_->CreateResponseReader(manifest_url_,
1074                                      group_->group_id(),
1075                                      entry->response_id()));
1076   read_manifest_buffer_ = new net::IOBuffer(kBufferSize);
1077   manifest_response_reader_->ReadData(
1078       read_manifest_buffer_.get(),
1079       kBufferSize,
1080       base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
1081                  base::Unretained(this)));  // async read
1082 }
1083
1084 void AppCacheUpdateJob::OnManifestDataReadComplete(int result) {
1085   if (result > 0) {
1086     loaded_manifest_data_.append(read_manifest_buffer_->data(), result);
1087     manifest_response_reader_->ReadData(
1088         read_manifest_buffer_.get(),
1089         kBufferSize,
1090         base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
1091                    base::Unretained(this)));  // read more
1092   } else {
1093     read_manifest_buffer_ = NULL;
1094     manifest_response_reader_.reset();
1095     ContinueHandleManifestFetchCompleted(
1096         result < 0 || manifest_data_ != loaded_manifest_data_);
1097   }
1098 }
1099
1100 void AppCacheUpdateJob::BuildUrlFileList(const AppCacheManifest& manifest) {
1101   for (base::hash_set<std::string>::const_iterator it =
1102            manifest.explicit_urls.begin();
1103        it != manifest.explicit_urls.end(); ++it) {
1104     AddUrlToFileList(GURL(*it), AppCacheEntry::EXPLICIT);
1105   }
1106
1107   const std::vector<AppCacheNamespace>& intercepts =
1108       manifest.intercept_namespaces;
1109   for (std::vector<AppCacheNamespace>::const_iterator it = intercepts.begin();
1110        it != intercepts.end(); ++it) {
1111     int flags = AppCacheEntry::INTERCEPT;
1112     if (it->is_executable)
1113       flags |= AppCacheEntry::EXECUTABLE;
1114     AddUrlToFileList(it->target_url, flags);
1115   }
1116
1117   const std::vector<AppCacheNamespace>& fallbacks =
1118       manifest.fallback_namespaces;
1119   for (std::vector<AppCacheNamespace>::const_iterator it = fallbacks.begin();
1120        it != fallbacks.end(); ++it) {
1121      AddUrlToFileList(it->target_url, AppCacheEntry::FALLBACK);
1122   }
1123
1124   // Add all master entries from newest complete cache.
1125   if (update_type_ == UPGRADE_ATTEMPT) {
1126     const AppCache::EntryMap& entries =
1127         group_->newest_complete_cache()->entries();
1128     for (AppCache::EntryMap::const_iterator it = entries.begin();
1129          it != entries.end(); ++it) {
1130       const AppCacheEntry& entry = it->second;
1131       if (entry.IsMaster())
1132         AddUrlToFileList(it->first, AppCacheEntry::MASTER);
1133     }
1134   }
1135 }
1136
1137 void AppCacheUpdateJob::AddUrlToFileList(const GURL& url, int type) {
1138   std::pair<AppCache::EntryMap::iterator, bool> ret = url_file_list_.insert(
1139       AppCache::EntryMap::value_type(url, AppCacheEntry(type)));
1140
1141   if (ret.second)
1142     urls_to_fetch_.push_back(UrlToFetch(url, false, NULL));
1143   else
1144     ret.first->second.add_types(type);  // URL already exists. Merge types.
1145 }
1146
1147 void AppCacheUpdateJob::FetchUrls() {
1148   DCHECK(internal_state_ == DOWNLOADING);
1149
1150   // Fetch each URL in the list according to section 6.9.4 step 17.1-17.3.
1151   // Fetch up to the concurrent limit. Other fetches will be triggered as each
1152   // each fetch completes.
1153   while (pending_url_fetches_.size() < kMaxConcurrentUrlFetches &&
1154          !urls_to_fetch_.empty()) {
1155     UrlToFetch url_to_fetch = urls_to_fetch_.front();
1156     urls_to_fetch_.pop_front();
1157
1158     AppCache::EntryMap::iterator it = url_file_list_.find(url_to_fetch.url);
1159     DCHECK(it != url_file_list_.end());
1160     AppCacheEntry& entry = it->second;
1161     if (ShouldSkipUrlFetch(entry)) {
1162       NotifyAllProgress(url_to_fetch.url);
1163       ++url_fetches_completed_;
1164     } else if (AlreadyFetchedEntry(url_to_fetch.url, entry.types())) {
1165       NotifyAllProgress(url_to_fetch.url);
1166       ++url_fetches_completed_;  // saved a URL request
1167     } else if (!url_to_fetch.storage_checked &&
1168                MaybeLoadFromNewestCache(url_to_fetch.url, entry)) {
1169       // Continues asynchronously after data is loaded from newest cache.
1170     } else {
1171       URLFetcher* fetcher = new URLFetcher(
1172           url_to_fetch.url, URLFetcher::URL_FETCH, this);
1173       if (url_to_fetch.existing_response_info.get()) {
1174         DCHECK(group_->newest_complete_cache());
1175         AppCacheEntry* existing_entry =
1176             group_->newest_complete_cache()->GetEntry(url_to_fetch.url);
1177         DCHECK(existing_entry);
1178         DCHECK(existing_entry->response_id() ==
1179                url_to_fetch.existing_response_info->response_id());
1180         fetcher->set_existing_response_headers(
1181             url_to_fetch.existing_response_info->http_response_info()->headers
1182                 .get());
1183         fetcher->set_existing_entry(*existing_entry);
1184       }
1185       fetcher->Start();
1186       pending_url_fetches_.insert(
1187           PendingUrlFetches::value_type(url_to_fetch.url, fetcher));
1188     }
1189   }
1190 }
1191
1192 void AppCacheUpdateJob::CancelAllUrlFetches() {
1193   // Cancel any pending URL requests.
1194   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
1195        it != pending_url_fetches_.end(); ++it) {
1196     delete it->second;
1197   }
1198
1199   url_fetches_completed_ +=
1200       pending_url_fetches_.size() + urls_to_fetch_.size();
1201   pending_url_fetches_.clear();
1202   urls_to_fetch_.clear();
1203 }
1204
1205 bool AppCacheUpdateJob::ShouldSkipUrlFetch(const AppCacheEntry& entry) {
1206   // 6.6.4 Step 17
1207   // If the resource URL being processed was flagged as neither an
1208   // "explicit entry" nor or a "fallback entry", then the user agent
1209   // may skip this URL.
1210   if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept())
1211     return false;
1212
1213   // TODO(jennb): decide if entry should be skipped to expire it from cache
1214   return false;
1215 }
1216
1217 bool AppCacheUpdateJob::AlreadyFetchedEntry(const GURL& url,
1218                                             int entry_type) {
1219   DCHECK(internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE);
1220   AppCacheEntry* existing =
1221       inprogress_cache_.get() ? inprogress_cache_->GetEntry(url)
1222                               : group_->newest_complete_cache()->GetEntry(url);
1223   if (existing) {
1224     existing->add_types(entry_type);
1225     return true;
1226   }
1227   return false;
1228 }
1229
1230 void AppCacheUpdateJob::AddMasterEntryToFetchList(AppCacheHost* host,
1231                                                   const GURL& url,
1232                                                   bool is_new) {
1233   DCHECK(!IsTerminating());
1234
1235   if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE) {
1236     AppCache* cache;
1237     if (inprogress_cache_.get()) {
1238       // always associate
1239       host->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
1240       cache = inprogress_cache_.get();
1241     } else {
1242       cache = group_->newest_complete_cache();
1243     }
1244
1245     // Update existing entry if it has already been fetched.
1246     AppCacheEntry* entry = cache->GetEntry(url);
1247     if (entry) {
1248       entry->add_types(AppCacheEntry::MASTER);
1249       if (internal_state_ == NO_UPDATE && !inprogress_cache_.get()) {
1250         // only associate if have entry
1251         host->AssociateCompleteCache(cache);
1252       }
1253       if (is_new)
1254         ++master_entries_completed_;  // pretend fetching completed
1255       return;
1256     }
1257   }
1258
1259   // Add to fetch list if not already fetching.
1260   if (master_entry_fetches_.find(url) == master_entry_fetches_.end()) {
1261     master_entries_to_fetch_.insert(url);
1262     if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE)
1263       FetchMasterEntries();
1264   }
1265 }
1266
1267 void AppCacheUpdateJob::FetchMasterEntries() {
1268   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
1269
1270   // Fetch each master entry in the list, up to the concurrent limit.
1271   // Additional fetches will be triggered as each fetch completes.
1272   while (master_entry_fetches_.size() < kMaxConcurrentUrlFetches &&
1273          !master_entries_to_fetch_.empty()) {
1274     const GURL& url = *master_entries_to_fetch_.begin();
1275
1276     if (AlreadyFetchedEntry(url, AppCacheEntry::MASTER)) {
1277       ++master_entries_completed_;  // saved a URL request
1278
1279       // In no update case, associate hosts to newest cache in group
1280       // now that master entry has been "successfully downloaded".
1281       if (internal_state_ == NO_UPDATE) {
1282         // TODO(michaeln): defer until the updated cache has been stored.
1283         DCHECK(!inprogress_cache_.get());
1284         AppCache* cache = group_->newest_complete_cache();
1285         PendingMasters::iterator found = pending_master_entries_.find(url);
1286         DCHECK(found != pending_master_entries_.end());
1287         PendingHosts& hosts = found->second;
1288         for (PendingHosts::iterator host_it = hosts.begin();
1289              host_it != hosts.end(); ++host_it) {
1290           (*host_it)->AssociateCompleteCache(cache);
1291         }
1292       }
1293     } else {
1294       URLFetcher* fetcher = new URLFetcher(
1295           url, URLFetcher::MASTER_ENTRY_FETCH, this);
1296       fetcher->Start();
1297       master_entry_fetches_.insert(PendingUrlFetches::value_type(url, fetcher));
1298     }
1299
1300     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
1301   }
1302 }
1303
1304 void AppCacheUpdateJob::CancelAllMasterEntryFetches(
1305     const AppCacheErrorDetails& error_details) {
1306   // For now, cancel all in-progress fetches for master entries and pretend
1307   // all master entries fetches have completed.
1308   // TODO(jennb): Delete this when update no longer fetches master entries
1309   // directly.
1310
1311   // Cancel all in-progress fetches.
1312   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
1313        it != master_entry_fetches_.end(); ++it) {
1314     delete it->second;
1315     master_entries_to_fetch_.insert(it->first);  // back in unfetched list
1316   }
1317   master_entry_fetches_.clear();
1318
1319   master_entries_completed_ += master_entries_to_fetch_.size();
1320
1321   // Cache failure steps, step 2.
1322   // Pretend all master entries that have not yet been fetched have completed
1323   // downloading. Unassociate hosts from any appcache and send ERROR event.
1324   HostNotifier host_notifier;
1325   while (!master_entries_to_fetch_.empty()) {
1326     const GURL& url = *master_entries_to_fetch_.begin();
1327     PendingMasters::iterator found = pending_master_entries_.find(url);
1328     DCHECK(found != pending_master_entries_.end());
1329     PendingHosts& hosts = found->second;
1330     for (PendingHosts::iterator host_it = hosts.begin();
1331          host_it != hosts.end(); ++host_it) {
1332       AppCacheHost* host = *host_it;
1333       host->AssociateNoCache(GURL());
1334       host_notifier.AddHost(host);
1335       host->RemoveObserver(this);
1336     }
1337     hosts.clear();
1338
1339     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
1340   }
1341   host_notifier.SendErrorNotifications(error_details);
1342 }
1343
1344 bool AppCacheUpdateJob::MaybeLoadFromNewestCache(const GURL& url,
1345                                                  AppCacheEntry& entry) {
1346   if (update_type_ != UPGRADE_ATTEMPT)
1347     return false;
1348
1349   AppCache* newest = group_->newest_complete_cache();
1350   AppCacheEntry* copy_me = newest->GetEntry(url);
1351   if (!copy_me || !copy_me->has_response_id())
1352     return false;
1353
1354   // Load HTTP headers for entry from newest cache.
1355   loading_responses_.insert(
1356       LoadingResponses::value_type(copy_me->response_id(), url));
1357   storage_->LoadResponseInfo(manifest_url_, group_->group_id(),
1358                              copy_me->response_id(),
1359                              this);
1360   // Async: wait for OnResponseInfoLoaded to complete.
1361   return true;
1362 }
1363
1364 void AppCacheUpdateJob::OnResponseInfoLoaded(
1365     AppCacheResponseInfo* response_info, int64 response_id) {
1366   const net::HttpResponseInfo* http_info = response_info ?
1367       response_info->http_response_info() : NULL;
1368
1369   // Needed response info for a manifest fetch request.
1370   if (internal_state_ == FETCH_MANIFEST) {
1371     if (http_info)
1372       manifest_fetcher_->set_existing_response_headers(
1373           http_info->headers.get());
1374     manifest_fetcher_->Start();
1375     return;
1376   }
1377
1378   LoadingResponses::iterator found = loading_responses_.find(response_id);
1379   DCHECK(found != loading_responses_.end());
1380   const GURL& url = found->second;
1381
1382   if (!http_info) {
1383     LoadFromNewestCacheFailed(url, NULL);  // no response found
1384   } else {
1385     // Check if response can be re-used according to HTTP caching semantics.
1386     // Responses with a "vary" header get treated as expired.
1387     const std::string name = "vary";
1388     std::string value;
1389     void* iter = NULL;
1390     if (!http_info->headers.get() ||
1391         http_info->headers->RequiresValidation(http_info->request_time,
1392                                                http_info->response_time,
1393                                                base::Time::Now()) ||
1394         http_info->headers->EnumerateHeader(&iter, name, &value)) {
1395       LoadFromNewestCacheFailed(url, response_info);
1396     } else {
1397       DCHECK(group_->newest_complete_cache());
1398       AppCacheEntry* copy_me = group_->newest_complete_cache()->GetEntry(url);
1399       DCHECK(copy_me);
1400       DCHECK(copy_me->response_id() == response_id);
1401
1402       AppCache::EntryMap::iterator it = url_file_list_.find(url);
1403       DCHECK(it != url_file_list_.end());
1404       AppCacheEntry& entry = it->second;
1405       entry.set_response_id(response_id);
1406       entry.set_response_size(copy_me->response_size());
1407       inprogress_cache_->AddOrModifyEntry(url, entry);
1408       NotifyAllProgress(url);
1409       ++url_fetches_completed_;
1410     }
1411   }
1412   loading_responses_.erase(found);
1413
1414   MaybeCompleteUpdate();
1415 }
1416
1417 void AppCacheUpdateJob::LoadFromNewestCacheFailed(
1418     const GURL& url, AppCacheResponseInfo* response_info) {
1419   if (internal_state_ == CACHE_FAILURE)
1420     return;
1421
1422   // Re-insert url at front of fetch list. Indicate storage has been checked.
1423   urls_to_fetch_.push_front(UrlToFetch(url, true, response_info));
1424   FetchUrls();
1425 }
1426
1427 void AppCacheUpdateJob::MaybeCompleteUpdate() {
1428   DCHECK(internal_state_ != CACHE_FAILURE);
1429
1430   // Must wait for any pending master entries or url fetches to complete.
1431   if (master_entries_completed_ != pending_master_entries_.size() ||
1432       url_fetches_completed_ != url_file_list_.size()) {
1433     DCHECK(internal_state_ != COMPLETED);
1434     return;
1435   }
1436
1437   switch (internal_state_) {
1438     case NO_UPDATE:
1439       if (master_entries_completed_ > 0) {
1440         switch (stored_state_) {
1441           case UNSTORED:
1442             StoreGroupAndCache();
1443             return;
1444           case STORING:
1445             return;
1446           case STORED:
1447             break;
1448         }
1449       }
1450       // 6.9.4 steps 7.3-7.7.
1451       NotifyAllAssociatedHosts(APPCACHE_NO_UPDATE_EVENT);
1452       DiscardDuplicateResponses();
1453       internal_state_ = COMPLETED;
1454       break;
1455     case DOWNLOADING:
1456       internal_state_ = REFETCH_MANIFEST;
1457       FetchManifest(false);
1458       break;
1459     case REFETCH_MANIFEST:
1460       DCHECK(stored_state_ == STORED);
1461       NotifyAllFinalProgress();
1462       if (update_type_ == CACHE_ATTEMPT)
1463         NotifyAllAssociatedHosts(APPCACHE_CACHED_EVENT);
1464       else
1465         NotifyAllAssociatedHosts(APPCACHE_UPDATE_READY_EVENT);
1466       DiscardDuplicateResponses();
1467       internal_state_ = COMPLETED;
1468       LogHistogramStats(UPDATE_OK, GURL());
1469       break;
1470     case CACHE_FAILURE:
1471       NOTREACHED();  // See HandleCacheFailure
1472       break;
1473     default:
1474       break;
1475   }
1476
1477   // Let the stack unwind before deletion to make it less risky as this
1478   // method is called from multiple places in this file.
1479   if (internal_state_ == COMPLETED)
1480     DeleteSoon();
1481 }
1482
1483 void AppCacheUpdateJob::ScheduleUpdateRetry(int delay_ms) {
1484   // TODO(jennb): post a delayed task with the "same parameters" as this job
1485   // to retry the update at a later time. Need group, URLs of pending master
1486   // entries and their hosts.
1487 }
1488
1489 void AppCacheUpdateJob::Cancel() {
1490   internal_state_ = CANCELLED;
1491
1492   LogHistogramStats(CANCELLED_ERROR, GURL());
1493
1494   if (manifest_fetcher_) {
1495     delete manifest_fetcher_;
1496     manifest_fetcher_ = NULL;
1497   }
1498
1499   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
1500        it != pending_url_fetches_.end(); ++it) {
1501     delete it->second;
1502   }
1503   pending_url_fetches_.clear();
1504
1505   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
1506        it != master_entry_fetches_.end(); ++it) {
1507     delete it->second;
1508   }
1509   master_entry_fetches_.clear();
1510
1511   ClearPendingMasterEntries();
1512   DiscardInprogressCache();
1513
1514   // Delete response writer to avoid any callbacks.
1515   if (manifest_response_writer_)
1516     manifest_response_writer_.reset();
1517
1518   storage_->CancelDelegateCallbacks(this);
1519 }
1520
1521 void AppCacheUpdateJob::ClearPendingMasterEntries() {
1522   for (PendingMasters::iterator it = pending_master_entries_.begin();
1523        it != pending_master_entries_.end(); ++it) {
1524     PendingHosts& hosts = it->second;
1525     for (PendingHosts::iterator host_it = hosts.begin();
1526          host_it != hosts.end(); ++host_it) {
1527       (*host_it)->RemoveObserver(this);
1528     }
1529   }
1530
1531   pending_master_entries_.clear();
1532 }
1533
1534 void AppCacheUpdateJob::DiscardInprogressCache() {
1535   if (stored_state_ == STORING) {
1536     // We can make no assumptions about whether the StoreGroupAndCacheTask
1537     // actually completed or not. This condition should only be reachable
1538     // during shutdown. Free things up and return to do no harm.
1539     inprogress_cache_ = NULL;
1540     added_master_entries_.clear();
1541     return;
1542   }
1543
1544   storage_->DoomResponses(manifest_url_, stored_response_ids_);
1545
1546   if (!inprogress_cache_.get()) {
1547     // We have to undo the changes we made, if any, to the existing cache.
1548     if (group_ && group_->newest_complete_cache()) {
1549       for (std::vector<GURL>::iterator iter = added_master_entries_.begin();
1550            iter != added_master_entries_.end(); ++iter) {
1551         group_->newest_complete_cache()->RemoveEntry(*iter);
1552       }
1553     }
1554     added_master_entries_.clear();
1555     return;
1556   }
1557
1558   AppCache::AppCacheHosts& hosts = inprogress_cache_->associated_hosts();
1559   while (!hosts.empty())
1560     (*hosts.begin())->AssociateNoCache(GURL());
1561
1562   inprogress_cache_ = NULL;
1563   added_master_entries_.clear();
1564 }
1565
1566 void AppCacheUpdateJob::DiscardDuplicateResponses() {
1567   storage_->DoomResponses(manifest_url_, duplicate_response_ids_);
1568 }
1569
1570 void AppCacheUpdateJob::LogHistogramStats(
1571       ResultType result, const GURL& failed_resource_url) {
1572   AppCacheHistograms::CountUpdateJobResult(result, manifest_url_.GetOrigin());
1573   if (result == UPDATE_OK)
1574     return;
1575
1576   int percent_complete = 0;
1577   if (url_file_list_.size() > 0) {
1578     size_t actual_fetches_completed = url_fetches_completed_;
1579     if (!failed_resource_url.is_empty() && actual_fetches_completed)
1580       --actual_fetches_completed;
1581     percent_complete = (static_cast<double>(actual_fetches_completed) /
1582                             static_cast<double>(url_file_list_.size())) * 100.0;
1583     percent_complete = std::min(percent_complete, 99);
1584   }
1585
1586   bool was_making_progress =
1587       base::Time::Now() - last_progress_time_ <
1588           base::TimeDelta::FromMinutes(5);
1589
1590   bool off_origin_resource_failure =
1591       !failed_resource_url.is_empty() &&
1592           (failed_resource_url.GetOrigin() != manifest_url_.GetOrigin());
1593
1594   AppCacheHistograms::LogUpdateFailureStats(
1595       manifest_url_.GetOrigin(),
1596       percent_complete,
1597       was_making_progress,
1598       off_origin_resource_failure);
1599 }
1600
1601 void AppCacheUpdateJob::DeleteSoon() {
1602   ClearPendingMasterEntries();
1603   manifest_response_writer_.reset();
1604   storage_->CancelDelegateCallbacks(this);
1605   service_->RemoveObserver(this);
1606   service_ = NULL;
1607
1608   // Break the connection with the group so the group cannot call delete
1609   // on this object after we've posted a task to delete ourselves.
1610   group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE);
1611   group_ = NULL;
1612
1613   base::MessageLoop::current()->DeleteSoon(FROM_HERE, this);
1614 }
1615
1616 }  // namespace content