content/browser/appcache/appcache_update_job.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/appcache/appcache_update_job.h"
   6
   7 #include "base/bind.h"
   8 #include "base/bind_helpers.h"
   9 #include "base/compiler_specific.h"
  10 #include "base/strings/string_util.h"
  11 #include "base/strings/stringprintf.h"
  12 #include "base/thread_task_runner_handle.h"
  13 #include "content/browser/appcache/appcache_group.h"
  14 #include "content/browser/appcache/appcache_histograms.h"
  15 #include "content/public/browser/browser_thread.h"
  16 #include "net/base/io_buffer.h"
  17 #include "net/base/load_flags.h"
  18 #include "net/base/net_errors.h"
  19 #include "net/base/request_priority.h"
  20 #include "net/http/http_request_headers.h"
  21 #include "net/http/http_response_headers.h"
  22 #include "net/url_request/url_request_context.h"
  23
  24 namespace content {
  25
  26 namespace {
  27
  28 const int kBufferSize = 32768;
  29 const size_t kMaxConcurrentUrlFetches = 2;
  30 const int kMax503Retries = 3;
  31
  32 std::string FormatUrlErrorMessage(
  33       const char* format, const GURL& url,
  34       AppCacheUpdateJob::ResultType error,
  35       int response_code) {
  36     // Show the net response code if we have one.
  37     int code = response_code;
  38     if (error != AppCacheUpdateJob::SERVER_ERROR)
  39       code = static_cast<int>(error);
  40     return base::StringPrintf(format, code, url.spec().c_str());
  41 }
  42
  43 bool IsEvictableError(AppCacheUpdateJob::ResultType result,
  44                       const AppCacheErrorDetails& details) {
  45   switch (result) {
  46     case AppCacheUpdateJob::DB_ERROR:
  47     case AppCacheUpdateJob::DISKCACHE_ERROR:
  48     case AppCacheUpdateJob::QUOTA_ERROR:
  49     case AppCacheUpdateJob::NETWORK_ERROR:
  50     case AppCacheUpdateJob::CANCELLED_ERROR:
  51       return false;
  52
  53     case AppCacheUpdateJob::REDIRECT_ERROR:
  54     case AppCacheUpdateJob::SERVER_ERROR:
  55     case AppCacheUpdateJob::SECURITY_ERROR:
  56       return true;
  57
  58     case AppCacheUpdateJob::MANIFEST_ERROR:
  59       return details.reason == APPCACHE_SIGNATURE_ERROR;
  60
  61     default:
  62       NOTREACHED();
  63       return true;
  64   }
  65 }
  66
  67 void EmptyCompletionCallback(int result) {}
  68
  69 }  // namespace
  70
  71 // Helper class for collecting hosts per frontend when sending notifications
  72 // so that only one notification is sent for all hosts using the same frontend.
  73 class HostNotifier {
  74  public:
  75   typedef std::vector<int> HostIds;
  76   typedef std::map<AppCacheFrontend*, HostIds> NotifyHostMap;
  77
  78   // Caller is responsible for ensuring there will be no duplicate hosts.
  79   void AddHost(AppCacheHost* host) {
  80     std::pair<NotifyHostMap::iterator , bool> ret = hosts_to_notify.insert(
  81         NotifyHostMap::value_type(host->frontend(), HostIds()));
  82     ret.first->second.push_back(host->host_id());
  83   }
  84
  85   void AddHosts(const std::set<AppCacheHost*>& hosts) {
  86     for (std::set<AppCacheHost*>::const_iterator it = hosts.begin();
  87          it != hosts.end(); ++it) {
  88       AddHost(*it);
  89     }
  90   }
  91
  92   void SendNotifications(AppCacheEventID event_id) {
  93     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  94          it != hosts_to_notify.end(); ++it) {
  95       AppCacheFrontend* frontend = it->first;
  96       frontend->OnEventRaised(it->second, event_id);
  97     }
  98   }
  99
 100   void SendProgressNotifications(
 101       const GURL& url, int num_total, int num_complete) {
 102     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
 103          it != hosts_to_notify.end(); ++it) {
 104       AppCacheFrontend* frontend = it->first;
 105       frontend->OnProgressEventRaised(it->second, url,
 106                                       num_total, num_complete);
 107     }
 108   }
 109
 110   void SendErrorNotifications(const AppCacheErrorDetails& details) {
 111     DCHECK(!details.message.empty());
 112     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
 113          it != hosts_to_notify.end(); ++it) {
 114       AppCacheFrontend* frontend = it->first;
 115       frontend->OnErrorEventRaised(it->second, details);
 116     }
 117   }
 118
 119   void SendLogMessage(const std::string& message) {
 120     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
 121          it != hosts_to_notify.end(); ++it) {
 122       AppCacheFrontend* frontend = it->first;
 123       for (HostIds::iterator id = it->second.begin();
 124            id != it->second.end(); ++id) {
 125         frontend->OnLogMessage(*id, APPCACHE_LOG_WARNING, message);
 126       }
 127     }
 128   }
 129
 130  private:
 131   NotifyHostMap hosts_to_notify;
 132 };
 133
 134 AppCacheUpdateJob::UrlToFetch::UrlToFetch(const GURL& url,
 135                                           bool checked,
 136                                           AppCacheResponseInfo* info)
 137     : url(url),
 138       storage_checked(checked),
 139       existing_response_info(info) {
 140 }
 141
 142 AppCacheUpdateJob::UrlToFetch::~UrlToFetch() {
 143 }
 144
 145 // Helper class to fetch resources. Depending on the fetch type,
 146 // can either fetch to an in-memory string or write the response
 147 // data out to the disk cache.
 148 AppCacheUpdateJob::URLFetcher::URLFetcher(const GURL& url,
 149                                           FetchType fetch_type,
 150                                           AppCacheUpdateJob* job)
 151     : url_(url),
 152       job_(job),
 153       fetch_type_(fetch_type),
 154       retry_503_attempts_(0),
 155       buffer_(new net::IOBuffer(kBufferSize)),
 156       request_(job->service_->request_context()
 157                    ->CreateRequest(url, net::DEFAULT_PRIORITY, this)),
 158       result_(UPDATE_OK),
 159       redirect_response_code_(-1) {}
 160
 161 AppCacheUpdateJob::URLFetcher::~URLFetcher() {
 162 }
 163
 164 void AppCacheUpdateJob::URLFetcher::Start() {
 165   request_->set_first_party_for_cookies(job_->manifest_url_);
 166   if (fetch_type_ == MANIFEST_FETCH && job_->doing_full_update_check_)
 167     request_->SetLoadFlags(request_->load_flags() | net::LOAD_BYPASS_CACHE);
 168   else if (existing_response_headers_.get())
 169     AddConditionalHeaders(existing_response_headers_.get());
 170   request_->Start();
 171 }
 172
 173 void AppCacheUpdateJob::URLFetcher::OnReceivedRedirect(
 174     net::URLRequest* request,
 175     const net::RedirectInfo& redirect_info,
 176     bool* defer_redirect) {
 177   DCHECK(request_ == request);
 178   // Redirect is not allowed by the update process.
 179   job_->MadeProgress();
 180   redirect_response_code_ = request->GetResponseCode();
 181   request->Cancel();
 182   result_ = REDIRECT_ERROR;
 183   OnResponseCompleted();
 184 }
 185
 186 void AppCacheUpdateJob::URLFetcher::OnResponseStarted(
 187     net::URLRequest *request) {
 188   DCHECK(request == request_);
 189   int response_code = -1;
 190   if (request->status().is_success()) {
 191     response_code = request->GetResponseCode();
 192     job_->MadeProgress();
 193   }
 194
 195   if ((response_code / 100) != 2) {
 196     if (response_code > 0)
 197       result_ = SERVER_ERROR;
 198     else
 199       result_ = NETWORK_ERROR;
 200     OnResponseCompleted();
 201     return;
 202   }
 203
 204   if (url_.SchemeIsCryptographic()) {
 205     // Do not cache content with cert errors.
 206     // Also, we willfully violate the HTML5 spec at this point in order
 207     // to support the appcaching of cross-origin HTTPS resources.
 208     // We've opted for a milder constraint and allow caching unless
 209     // the resource has a "no-store" header. A spec change has been
 210     // requested on the whatwg list.
 211     // See http://code.google.com/p/chromium/issues/detail?id=69594
 212     // TODO(michaeln): Consider doing this for cross-origin HTTP too.
 213     const net::HttpNetworkSession::Params* session_params =
 214         request->context()->GetNetworkSessionParams();
 215     bool ignore_cert_errors = session_params &&
 216                               session_params->ignore_certificate_errors;
 217     if ((net::IsCertStatusError(request->ssl_info().cert_status) &&
 218             !ignore_cert_errors) ||
 219         (url_.GetOrigin() != job_->manifest_url_.GetOrigin() &&
 220             request->response_headers()->
 221                 HasHeaderValue("cache-control", "no-store"))) {
 222       DCHECK_EQ(-1, redirect_response_code_);
 223       request->Cancel();
 224       result_ = SECURITY_ERROR;
 225       OnResponseCompleted();
 226       return;
 227     }
 228   }
 229
 230   // Write response info to storage for URL fetches. Wait for async write
 231   // completion before reading any response data.
 232   if (fetch_type_ == URL_FETCH || fetch_type_ == MASTER_ENTRY_FETCH) {
 233     response_writer_.reset(job_->CreateResponseWriter());
 234     scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
 235         new HttpResponseInfoIOBuffer(
 236             new net::HttpResponseInfo(request->response_info())));
 237     response_writer_->WriteInfo(
 238         io_buffer.get(),
 239         base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
 240   } else {
 241     ReadResponseData();
 242   }
 243 }
 244
 245 void AppCacheUpdateJob::URLFetcher::OnReadCompleted(
 246     net::URLRequest* request, int bytes_read) {
 247   DCHECK(request_ == request);
 248   bool data_consumed = true;
 249   if (request->status().is_success() && bytes_read > 0) {
 250     job_->MadeProgress();
 251     data_consumed = ConsumeResponseData(bytes_read);
 252     if (data_consumed) {
 253       bytes_read = 0;
 254       while (request->Read(buffer_.get(), kBufferSize, &bytes_read)) {
 255         if (bytes_read > 0) {
 256           data_consumed = ConsumeResponseData(bytes_read);
 257           if (!data_consumed)
 258             break;  // wait for async data processing, then read more
 259         } else {
 260           break;
 261         }
 262       }
 263     }
 264   }
 265   if (data_consumed && !request->status().is_io_pending()) {
 266     DCHECK_EQ(UPDATE_OK, result_);
 267     OnResponseCompleted();
 268   }
 269 }
 270
 271 void AppCacheUpdateJob::URLFetcher::AddConditionalHeaders(
 272     const net::HttpResponseHeaders* headers) {
 273   DCHECK(request_.get() && headers);
 274   net::HttpRequestHeaders extra_headers;
 275
 276   // Add If-Modified-Since header if response info has Last-Modified header.
 277   const std::string last_modified = "Last-Modified";
 278   std::string last_modified_value;
 279   headers->EnumerateHeader(NULL, last_modified, &last_modified_value);
 280   if (!last_modified_value.empty()) {
 281     extra_headers.SetHeader(net::HttpRequestHeaders::kIfModifiedSince,
 282                             last_modified_value);
 283   }
 284
 285   // Add If-None-Match header if response info has ETag header.
 286   const std::string etag = "ETag";
 287   std::string etag_value;
 288   headers->EnumerateHeader(NULL, etag, &etag_value);
 289   if (!etag_value.empty()) {
 290     extra_headers.SetHeader(net::HttpRequestHeaders::kIfNoneMatch,
 291                             etag_value);
 292   }
 293   if (!extra_headers.IsEmpty())
 294     request_->SetExtraRequestHeaders(extra_headers);
 295 }
 296
 297 void  AppCacheUpdateJob::URLFetcher::OnWriteComplete(int result) {
 298   if (result < 0) {
 299     request_->Cancel();
 300     result_ = DISKCACHE_ERROR;
 301     OnResponseCompleted();
 302     return;
 303   }
 304   ReadResponseData();
 305 }
 306
 307 void AppCacheUpdateJob::URLFetcher::ReadResponseData() {
 308   InternalUpdateState state = job_->internal_state_;
 309   if (state == CACHE_FAILURE || state == CANCELLED || state == COMPLETED)
 310     return;
 311   int bytes_read = 0;
 312   request_->Read(buffer_.get(), kBufferSize, &bytes_read);
 313   OnReadCompleted(request_.get(), bytes_read);
 314 }
 315
 316 // Returns false if response data is processed asynchronously, in which
 317 // case ReadResponseData will be invoked when it is safe to continue
 318 // reading more response data from the request.
 319 bool AppCacheUpdateJob::URLFetcher::ConsumeResponseData(int bytes_read) {
 320   DCHECK_GT(bytes_read, 0);
 321   switch (fetch_type_) {
 322     case MANIFEST_FETCH:
 323     case MANIFEST_REFETCH:
 324       manifest_data_.append(buffer_->data(), bytes_read);
 325       break;
 326     case URL_FETCH:
 327     case MASTER_ENTRY_FETCH:
 328       DCHECK(response_writer_.get());
 329       response_writer_->WriteData(
 330           buffer_.get(),
 331           bytes_read,
 332           base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
 333       return false;  // wait for async write completion to continue reading
 334     default:
 335       NOTREACHED();
 336   }
 337   return true;
 338 }
 339
 340 void AppCacheUpdateJob::URLFetcher::OnResponseCompleted() {
 341   if (request_->status().is_success())
 342     job_->MadeProgress();
 343
 344   // Retry for 503s where retry-after is 0.
 345   if (request_->status().is_success() &&
 346       request_->GetResponseCode() == 503 &&
 347       MaybeRetryRequest()) {
 348     return;
 349   }
 350
 351   switch (fetch_type_) {
 352     case MANIFEST_FETCH:
 353       job_->HandleManifestFetchCompleted(this);
 354       break;
 355     case URL_FETCH:
 356       job_->HandleUrlFetchCompleted(this);
 357       break;
 358     case MASTER_ENTRY_FETCH:
 359       job_->HandleMasterEntryFetchCompleted(this);
 360       break;
 361     case MANIFEST_REFETCH:
 362       job_->HandleManifestRefetchCompleted(this);
 363       break;
 364     default:
 365       NOTREACHED();
 366   }
 367
 368   delete this;
 369 }
 370
 371 bool AppCacheUpdateJob::URLFetcher::MaybeRetryRequest() {
 372   if (retry_503_attempts_ >= kMax503Retries ||
 373       !request_->response_headers()->HasHeaderValue("retry-after", "0")) {
 374     return false;
 375   }
 376   ++retry_503_attempts_;
 377   result_ = UPDATE_OK;
 378   request_ = job_->service_->request_context()->CreateRequest(
 379       url_, net::DEFAULT_PRIORITY, this);
 380   Start();
 381   return true;
 382 }
 383
 384 AppCacheUpdateJob::AppCacheUpdateJob(AppCacheServiceImpl* service,
 385                                      AppCacheGroup* group)
 386     : service_(service),
 387       manifest_url_(group->manifest_url()),
 388       group_(group),
 389       update_type_(UNKNOWN_TYPE),
 390       internal_state_(FETCH_MANIFEST),
 391       doing_full_update_check_(false),
 392       master_entries_completed_(0),
 393       url_fetches_completed_(0),
 394       manifest_fetcher_(NULL),
 395       manifest_has_valid_mime_type_(false),
 396       stored_state_(UNSTORED),
 397       storage_(service->storage()),
 398       weak_factory_(this) {
 399     service_->AddObserver(this);
 400 }
 401
 402 AppCacheUpdateJob::~AppCacheUpdateJob() {
 403   if (service_)
 404     service_->RemoveObserver(this);
 405   if (internal_state_ != COMPLETED)
 406     Cancel();
 407
 408   DCHECK(!manifest_fetcher_);
 409   DCHECK(pending_url_fetches_.empty());
 410   DCHECK(!inprogress_cache_.get());
 411   DCHECK(pending_master_entries_.empty());
 412   DCHECK(master_entry_fetches_.empty());
 413
 414   if (group_)
 415     group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE);
 416 }
 417
 418 void AppCacheUpdateJob::StartUpdate(AppCacheHost* host,
 419                                     const GURL& new_master_resource) {
 420   DCHECK(group_->update_job() == this);
 421   DCHECK(!group_->is_obsolete());
 422
 423   bool is_new_pending_master_entry = false;
 424   if (!new_master_resource.is_empty()) {
 425     DCHECK(new_master_resource == host->pending_master_entry_url());
 426     DCHECK(!new_master_resource.has_ref());
 427     DCHECK(new_master_resource.GetOrigin() == manifest_url_.GetOrigin());
 428
 429     // Cannot add more to this update if already terminating.
 430     if (IsTerminating()) {
 431       group_->QueueUpdate(host, new_master_resource);
 432       return;
 433     }
 434
 435     std::pair<PendingMasters::iterator, bool> ret =
 436         pending_master_entries_.insert(
 437             PendingMasters::value_type(new_master_resource, PendingHosts()));
 438     is_new_pending_master_entry = ret.second;
 439     ret.first->second.push_back(host);
 440     host->AddObserver(this);
 441   }
 442
 443   // Notify host (if any) if already checking or downloading.
 444   AppCacheGroup::UpdateAppCacheStatus update_status = group_->update_status();
 445   if (update_status == AppCacheGroup::CHECKING ||
 446       update_status == AppCacheGroup::DOWNLOADING) {
 447     if (host) {
 448       NotifySingleHost(host, APPCACHE_CHECKING_EVENT);
 449       if (update_status == AppCacheGroup::DOWNLOADING)
 450         NotifySingleHost(host, APPCACHE_DOWNLOADING_EVENT);
 451
 452       // Add to fetch list or an existing entry if already fetched.
 453       if (!new_master_resource.is_empty()) {
 454         AddMasterEntryToFetchList(host, new_master_resource,
 455                                   is_new_pending_master_entry);
 456       }
 457     }
 458     return;
 459   }
 460
 461   // Begin update process for the group.
 462   MadeProgress();
 463   group_->SetUpdateAppCacheStatus(AppCacheGroup::CHECKING);
 464   if (group_->HasCache()) {
 465     base::TimeDelta kFullUpdateInterval = base::TimeDelta::FromHours(24);
 466     update_type_ = UPGRADE_ATTEMPT;
 467     base::TimeDelta time_since_last_check =
 468         base::Time::Now() - group_->last_full_update_check_time();
 469     doing_full_update_check_ = time_since_last_check > kFullUpdateInterval;
 470     NotifyAllAssociatedHosts(APPCACHE_CHECKING_EVENT);
 471   } else {
 472     update_type_ = CACHE_ATTEMPT;
 473     doing_full_update_check_ = true;
 474     DCHECK(host);
 475     NotifySingleHost(host, APPCACHE_CHECKING_EVENT);
 476   }
 477
 478   if (!new_master_resource.is_empty()) {
 479     AddMasterEntryToFetchList(host, new_master_resource,
 480                               is_new_pending_master_entry);
 481   }
 482
 483   BrowserThread::PostAfterStartupTask(
 484       FROM_HERE, base::ThreadTaskRunnerHandle::Get(),
 485       base::Bind(&AppCacheUpdateJob::FetchManifest, weak_factory_.GetWeakPtr(),
 486                  true));
 487 }
 488
 489 AppCacheResponseWriter* AppCacheUpdateJob::CreateResponseWriter() {
 490   AppCacheResponseWriter* writer =
 491       storage_->CreateResponseWriter(manifest_url_,
 492                                                 group_->group_id());
 493   stored_response_ids_.push_back(writer->response_id());
 494   return writer;
 495 }
 496
 497 void AppCacheUpdateJob::HandleCacheFailure(
 498     const AppCacheErrorDetails& error_details,
 499     ResultType result,
 500     const GURL& failed_resource_url) {
 501   // 6.9.4 cache failure steps 2-8.
 502   DCHECK(internal_state_ != CACHE_FAILURE);
 503   DCHECK(!error_details.message.empty());
 504   DCHECK(result != UPDATE_OK);
 505   internal_state_ = CACHE_FAILURE;
 506   LogHistogramStats(result, failed_resource_url);
 507   CancelAllUrlFetches();
 508   CancelAllMasterEntryFetches(error_details);
 509   NotifyAllError(error_details);
 510   DiscardInprogressCache();
 511   internal_state_ = COMPLETED;
 512
 513   if (update_type_ == CACHE_ATTEMPT ||
 514       !IsEvictableError(result, error_details) ||
 515       service_->storage() != storage_) {
 516     DeleteSoon();
 517     return;
 518   }
 519
 520   if (group_->first_evictable_error_time().is_null()) {
 521     group_->set_first_evictable_error_time(base::Time::Now());
 522     storage_->StoreEvictionTimes(group_);
 523     DeleteSoon();
 524     return;
 525   }
 526
 527   base::TimeDelta kMaxEvictableErrorDuration = base::TimeDelta::FromDays(14);
 528   base::TimeDelta error_duration =
 529       base::Time::Now() - group_->first_evictable_error_time();
 530   if (error_duration > kMaxEvictableErrorDuration) {
 531     // Break the connection with the group prior to calling
 532     // DeleteAppCacheGroup, otherwise that method would delete |this|
 533     // and we need the stack to unwind prior to deletion.
 534     group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE);
 535     group_ = NULL;
 536     service_->DeleteAppCacheGroup(manifest_url_,
 537                                   base::Bind(EmptyCompletionCallback));
 538   }
 539
 540   DeleteSoon();  // To unwind the stack prior to deletion.
 541 }
 542
 543 void AppCacheUpdateJob::FetchManifest(bool is_first_fetch) {
 544   DCHECK(!manifest_fetcher_);
 545   manifest_fetcher_ = new URLFetcher(
 546      manifest_url_,
 547      is_first_fetch ? URLFetcher::MANIFEST_FETCH :
 548                       URLFetcher::MANIFEST_REFETCH,
 549      this);
 550
 551   if (is_first_fetch) {
 552     // Maybe load the cached headers to make a condiditional request.
 553     AppCacheEntry* entry = (update_type_ == UPGRADE_ATTEMPT) ?
 554         group_->newest_complete_cache()->GetEntry(manifest_url_) : NULL;
 555     if (entry && !doing_full_update_check_) {
 556       // Asynchronously load response info for manifest from newest cache.
 557       storage_->LoadResponseInfo(manifest_url_, group_->group_id(),
 558                                  entry->response_id(), this);
 559       return;
 560     }
 561     manifest_fetcher_->Start();
 562     return;
 563   }
 564
 565   DCHECK(internal_state_ == REFETCH_MANIFEST);
 566   DCHECK(manifest_response_info_.get());
 567   manifest_fetcher_->set_existing_response_headers(
 568       manifest_response_info_->headers.get());
 569   manifest_fetcher_->Start();
 570 }
 571
 572
 573 void AppCacheUpdateJob::HandleManifestFetchCompleted(
 574     URLFetcher* fetcher) {
 575   DCHECK_EQ(internal_state_, FETCH_MANIFEST);
 576   DCHECK_EQ(manifest_fetcher_, fetcher);
 577   manifest_fetcher_ = NULL;
 578
 579   net::URLRequest* request = fetcher->request();
 580   int response_code = -1;
 581   bool is_valid_response_code = false;
 582   if (request->status().is_success()) {
 583     response_code = request->GetResponseCode();
 584     is_valid_response_code = (response_code / 100 == 2);
 585
 586     std::string mime_type;
 587     request->GetMimeType(&mime_type);
 588     manifest_has_valid_mime_type_ = (mime_type == "text/cache-manifest");
 589   }
 590
 591   if (is_valid_response_code) {
 592     manifest_data_ = fetcher->manifest_data();
 593     manifest_response_info_.reset(
 594         new net::HttpResponseInfo(request->response_info()));
 595     if (update_type_ == UPGRADE_ATTEMPT)
 596       CheckIfManifestChanged();  // continues asynchronously
 597     else
 598       ContinueHandleManifestFetchCompleted(true);
 599   } else if (response_code == 304 && update_type_ == UPGRADE_ATTEMPT) {
 600     ContinueHandleManifestFetchCompleted(false);
 601   } else if ((response_code == 404 || response_code == 410) &&
 602              update_type_ == UPGRADE_ATTEMPT) {
 603     storage_->MakeGroupObsolete(group_, this, response_code);  // async
 604   } else {
 605     const char* kFormatString = "Manifest fetch failed (%d) %s";
 606     std::string message = FormatUrlErrorMessage(
 607         kFormatString, manifest_url_, fetcher->result(), response_code);
 608     HandleCacheFailure(AppCacheErrorDetails(message,
 609                                     APPCACHE_MANIFEST_ERROR,
 610                                     manifest_url_,
 611                                     response_code,
 612                                     false /*is_cross_origin*/),
 613                        fetcher->result(),
 614                        GURL());
 615   }
 616 }
 617
 618 void AppCacheUpdateJob::OnGroupMadeObsolete(AppCacheGroup* group,
 619                                             bool success,
 620                                             int response_code) {
 621   DCHECK(master_entry_fetches_.empty());
 622   CancelAllMasterEntryFetches(AppCacheErrorDetails(
 623       "The cache has been made obsolete, "
 624       "the manifest file returned 404 or 410",
 625       APPCACHE_MANIFEST_ERROR,
 626       GURL(),
 627       response_code,
 628       false /*is_cross_origin*/));
 629   if (success) {
 630     DCHECK(group->is_obsolete());
 631     NotifyAllAssociatedHosts(APPCACHE_OBSOLETE_EVENT);
 632     internal_state_ = COMPLETED;
 633     MaybeCompleteUpdate();
 634   } else {
 635     // Treat failure to mark group obsolete as a cache failure.
 636     HandleCacheFailure(AppCacheErrorDetails(
 637                            "Failed to mark the cache as obsolete",
 638                            APPCACHE_UNKNOWN_ERROR, GURL(),  0,
 639                            false /*is_cross_origin*/),
 640                        DB_ERROR,
 641                        GURL());
 642   }
 643 }
 644
 645 void AppCacheUpdateJob::ContinueHandleManifestFetchCompleted(bool changed) {
 646   DCHECK(internal_state_ == FETCH_MANIFEST);
 647
 648   if (!changed) {
 649     DCHECK(update_type_ == UPGRADE_ATTEMPT);
 650     internal_state_ = NO_UPDATE;
 651
 652     // Wait for pending master entries to download.
 653     FetchMasterEntries();
 654     MaybeCompleteUpdate();  // if not done, run async 6.9.4 step 7 substeps
 655     return;
 656   }
 657
 658   AppCacheManifest manifest;
 659   if (!ParseManifest(manifest_url_, manifest_data_.data(),
 660                      manifest_data_.length(),
 661                      manifest_has_valid_mime_type_ ?
 662                         PARSE_MANIFEST_ALLOWING_INTERCEPTS :
 663                         PARSE_MANIFEST_PER_STANDARD,
 664                      manifest)) {
 665     const char* kFormatString = "Failed to parse manifest %s";
 666     const std::string message = base::StringPrintf(kFormatString,
 667         manifest_url_.spec().c_str());
 668     HandleCacheFailure(
 669         AppCacheErrorDetails(
 670             message, APPCACHE_SIGNATURE_ERROR, GURL(), 0,
 671             false /*is_cross_origin*/),
 672         MANIFEST_ERROR,
 673         GURL());
 674     VLOG(1) << message;
 675     return;
 676   }
 677
 678   // Proceed with update process. Section 6.9.4 steps 8-20.
 679   internal_state_ = DOWNLOADING;
 680   inprogress_cache_ = new AppCache(storage_, storage_->NewCacheId());
 681   BuildUrlFileList(manifest);
 682   inprogress_cache_->InitializeWithManifest(&manifest);
 683
 684   // Associate all pending master hosts with the newly created cache.
 685   for (PendingMasters::iterator it = pending_master_entries_.begin();
 686        it != pending_master_entries_.end(); ++it) {
 687     PendingHosts& hosts = it->second;
 688     for (PendingHosts::iterator host_it = hosts.begin();
 689          host_it != hosts.end(); ++host_it) {
 690       (*host_it)
 691           ->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
 692     }
 693   }
 694
 695   if (manifest.did_ignore_intercept_namespaces) {
 696     // Must be done after associating all pending master hosts.
 697     std::string message(
 698         "Ignoring the INTERCEPT section of the application cache manifest "
 699         "because the content type is not text/cache-manifest");
 700     LogConsoleMessageToAll(message);
 701   }
 702
 703   group_->SetUpdateAppCacheStatus(AppCacheGroup::DOWNLOADING);
 704   NotifyAllAssociatedHosts(APPCACHE_DOWNLOADING_EVENT);
 705   FetchUrls();
 706   FetchMasterEntries();
 707   MaybeCompleteUpdate();  // if not done, continues when async fetches complete
 708 }
 709
 710 void AppCacheUpdateJob::HandleUrlFetchCompleted(URLFetcher* fetcher) {
 711   DCHECK(internal_state_ == DOWNLOADING);
 712
 713   net::URLRequest* request = fetcher->request();
 714   const GURL& url = request->original_url();
 715   pending_url_fetches_.erase(url);
 716   NotifyAllProgress(url);
 717   ++url_fetches_completed_;
 718
 719   int response_code = request->status().is_success()
 720                           ? request->GetResponseCode()
 721                           : fetcher->redirect_response_code();
 722
 723   AppCacheEntry& entry = url_file_list_.find(url)->second;
 724
 725   if (response_code / 100 == 2) {
 726     // Associate storage with the new entry.
 727     DCHECK(fetcher->response_writer());
 728     entry.set_response_id(fetcher->response_writer()->response_id());
 729     entry.set_response_size(fetcher->response_writer()->amount_written());
 730     if (!inprogress_cache_->AddOrModifyEntry(url, entry))
 731       duplicate_response_ids_.push_back(entry.response_id());
 732
 733     // TODO(michaeln): Check for <html manifest=xxx>
 734     // See http://code.google.com/p/chromium/issues/detail?id=97930
 735     // if (entry.IsMaster() && !(entry.IsExplicit() || fallback || intercept))
 736     //   if (!manifestAttribute) skip it
 737
 738     // Foreign entries will be detected during cache selection.
 739     // Note: 6.9.4, step 17.9 possible optimization: if resource is HTML or XML
 740     // file whose root element is an html element with a manifest attribute
 741     // whose value doesn't match the manifest url of the application cache
 742     // being processed, mark the entry as being foreign.
 743   } else {
 744     VLOG(1) << "Request status: " << request->status().status()
 745             << " error: " << request->status().error()
 746             << " response code: " << response_code;
 747     if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept()) {
 748       if (response_code == 304 && fetcher->existing_entry().has_response_id()) {
 749         // Keep the existing response.
 750         entry.set_response_id(fetcher->existing_entry().response_id());
 751         entry.set_response_size(fetcher->existing_entry().response_size());
 752         inprogress_cache_->AddOrModifyEntry(url, entry);
 753       } else {
 754         const char* kFormatString = "Resource fetch failed (%d) %s";
 755         std::string message = FormatUrlErrorMessage(
 756             kFormatString, url, fetcher->result(), response_code);
 757         ResultType result = fetcher->result();
 758         bool is_cross_origin = url.GetOrigin() != manifest_url_.GetOrigin();
 759         switch (result) {
 760           case DISKCACHE_ERROR:
 761             HandleCacheFailure(
 762                 AppCacheErrorDetails(
 763                     message, APPCACHE_UNKNOWN_ERROR, GURL(), 0,
 764                     is_cross_origin),
 765                 result,
 766                 url);
 767             break;
 768           case NETWORK_ERROR:
 769             HandleCacheFailure(
 770                 AppCacheErrorDetails(message, APPCACHE_RESOURCE_ERROR, url, 0,
 771                     is_cross_origin),
 772                 result,
 773                 url);
 774             break;
 775           default:
 776             HandleCacheFailure(AppCacheErrorDetails(message,
 777                                             APPCACHE_RESOURCE_ERROR,
 778                                             url,
 779                                             response_code,
 780                                             is_cross_origin),
 781                                result,
 782                                url);
 783             break;
 784         }
 785         return;
 786       }
 787     } else if (response_code == 404 || response_code == 410) {
 788       // Entry is skipped.  They are dropped from the cache.
 789     } else if (update_type_ == UPGRADE_ATTEMPT &&
 790                fetcher->existing_entry().has_response_id()) {
 791       // Keep the existing response.
 792       // TODO(michaeln): Not sure this is a good idea. This is spec compliant
 793       // but the old resource may or may not be compatible with the new contents
 794       // of the cache. Impossible to know one way or the other.
 795       entry.set_response_id(fetcher->existing_entry().response_id());
 796       entry.set_response_size(fetcher->existing_entry().response_size());
 797       inprogress_cache_->AddOrModifyEntry(url, entry);
 798     }
 799   }
 800
 801   // Fetch another URL now that one request has completed.
 802   DCHECK(internal_state_ != CACHE_FAILURE);
 803   FetchUrls();
 804   MaybeCompleteUpdate();
 805 }
 806
 807 void AppCacheUpdateJob::HandleMasterEntryFetchCompleted(
 808     URLFetcher* fetcher) {
 809   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
 810
 811   // TODO(jennb): Handle downloads completing during cache failure when update
 812   // no longer fetches master entries directly. For now, we cancel all pending
 813   // master entry fetches when entering cache failure state so this will never
 814   // be called in CACHE_FAILURE state.
 815
 816   net::URLRequest* request = fetcher->request();
 817   const GURL& url = request->original_url();
 818   master_entry_fetches_.erase(url);
 819   ++master_entries_completed_;
 820
 821   int response_code = request->status().is_success()
 822       ? request->GetResponseCode() : -1;
 823
 824   PendingMasters::iterator found = pending_master_entries_.find(url);
 825   DCHECK(found != pending_master_entries_.end());
 826   PendingHosts& hosts = found->second;
 827
 828   // Section 6.9.4. No update case: step 7.3, else step 22.
 829   if (response_code / 100 == 2) {
 830     // Add fetched master entry to the appropriate cache.
 831     AppCache* cache = inprogress_cache_.get() ? inprogress_cache_.get()
 832                                               : group_->newest_complete_cache();
 833     DCHECK(fetcher->response_writer());
 834     AppCacheEntry master_entry(AppCacheEntry::MASTER,
 835                                fetcher->response_writer()->response_id(),
 836                                fetcher->response_writer()->amount_written());
 837     if (cache->AddOrModifyEntry(url, master_entry))
 838       added_master_entries_.push_back(url);
 839     else
 840       duplicate_response_ids_.push_back(master_entry.response_id());
 841
 842     // In no-update case, associate host with the newest cache.
 843     if (!inprogress_cache_.get()) {
 844       // TODO(michaeln): defer until the updated cache has been stored
 845       DCHECK(cache == group_->newest_complete_cache());
 846       for (PendingHosts::iterator host_it = hosts.begin();
 847            host_it != hosts.end(); ++host_it) {
 848         (*host_it)->AssociateCompleteCache(cache);
 849       }
 850     }
 851   } else {
 852     HostNotifier host_notifier;
 853     for (PendingHosts::iterator host_it = hosts.begin();
 854          host_it != hosts.end(); ++host_it) {
 855       AppCacheHost* host = *host_it;
 856       host_notifier.AddHost(host);
 857
 858       // In downloading case, disassociate host from inprogress cache.
 859       if (inprogress_cache_.get())
 860         host->AssociateNoCache(GURL());
 861
 862       host->RemoveObserver(this);
 863     }
 864     hosts.clear();
 865
 866     const char* kFormatString = "Manifest fetch failed (%d) %s";
 867     std::string message = FormatUrlErrorMessage(
 868         kFormatString, request->url(), fetcher->result(), response_code);
 869     host_notifier.SendErrorNotifications(
 870         AppCacheErrorDetails(message,
 871                      APPCACHE_MANIFEST_ERROR,
 872                      request->url(),
 873                      response_code,
 874                      false /*is_cross_origin*/));
 875
 876     // In downloading case, update result is different if all master entries
 877     // failed vs. only some failing.
 878     if (inprogress_cache_.get()) {
 879       // Only count successful downloads to know if all master entries failed.
 880       pending_master_entries_.erase(found);
 881       --master_entries_completed_;
 882
 883       // Section 6.9.4, step 22.3.
 884       if (update_type_ == CACHE_ATTEMPT && pending_master_entries_.empty()) {
 885         HandleCacheFailure(AppCacheErrorDetails(message,
 886                                         APPCACHE_MANIFEST_ERROR,
 887                                         request->url(),
 888                                         response_code,
 889                                         false /*is_cross_origin*/),
 890                            fetcher->result(),
 891                            GURL());
 892         return;
 893       }
 894     }
 895   }
 896
 897   DCHECK(internal_state_ != CACHE_FAILURE);
 898   FetchMasterEntries();
 899   MaybeCompleteUpdate();
 900 }
 901
 902 void AppCacheUpdateJob::HandleManifestRefetchCompleted(
 903     URLFetcher* fetcher) {
 904   DCHECK(internal_state_ == REFETCH_MANIFEST);
 905   DCHECK(manifest_fetcher_ == fetcher);
 906   manifest_fetcher_ = NULL;
 907
 908   net::URLRequest* request = fetcher->request();
 909   int response_code = request->status().is_success()
 910       ? request->GetResponseCode() : -1;
 911   if (response_code == 304 || manifest_data_ == fetcher->manifest_data()) {
 912     // Only need to store response in storage if manifest is not already
 913     // an entry in the cache.
 914     AppCacheEntry* entry = inprogress_cache_->GetEntry(manifest_url_);
 915     if (entry) {
 916       entry->add_types(AppCacheEntry::MANIFEST);
 917       StoreGroupAndCache();
 918     } else {
 919       manifest_response_writer_.reset(CreateResponseWriter());
 920       scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
 921           new HttpResponseInfoIOBuffer(manifest_response_info_.release()));
 922       manifest_response_writer_->WriteInfo(
 923           io_buffer.get(),
 924           base::Bind(&AppCacheUpdateJob::OnManifestInfoWriteComplete,
 925                      base::Unretained(this)));
 926     }
 927   } else {
 928     VLOG(1) << "Request status: " << request->status().status()
 929             << " error: " << request->status().error()
 930             << " response code: " << response_code;
 931     ScheduleUpdateRetry(kRerunDelayMs);
 932     if (response_code == 200) {
 933       HandleCacheFailure(AppCacheErrorDetails("Manifest changed during update",
 934                                       APPCACHE_CHANGED_ERROR,
 935                                       GURL(),
 936                                       0,
 937                                       false /*is_cross_origin*/),
 938                          MANIFEST_ERROR,
 939                          GURL());
 940     } else {
 941       const char* kFormatString = "Manifest re-fetch failed (%d) %s";
 942       std::string message = FormatUrlErrorMessage(
 943           kFormatString, manifest_url_, fetcher->result(), response_code);
 944       HandleCacheFailure(AppCacheErrorDetails(message,
 945                                       APPCACHE_MANIFEST_ERROR,
 946                                       GURL(),
 947                                       response_code,
 948                                       false /*is_cross_origin*/),
 949                          fetcher->result(),
 950                          GURL());
 951     }
 952   }
 953 }
 954
 955 void AppCacheUpdateJob::OnManifestInfoWriteComplete(int result) {
 956   if (result > 0) {
 957     scoped_refptr<net::StringIOBuffer> io_buffer(
 958         new net::StringIOBuffer(manifest_data_));
 959     manifest_response_writer_->WriteData(
 960         io_buffer.get(),
 961         manifest_data_.length(),
 962         base::Bind(&AppCacheUpdateJob::OnManifestDataWriteComplete,
 963                    base::Unretained(this)));
 964   } else {
 965     HandleCacheFailure(
 966         AppCacheErrorDetails("Failed to write the manifest headers to storage",
 967                      APPCACHE_UNKNOWN_ERROR,
 968                      GURL(),
 969                      0,
 970                      false /*is_cross_origin*/),
 971         DISKCACHE_ERROR,
 972         GURL());
 973   }
 974 }
 975
 976 void AppCacheUpdateJob::OnManifestDataWriteComplete(int result) {
 977   if (result > 0) {
 978     AppCacheEntry entry(AppCacheEntry::MANIFEST,
 979         manifest_response_writer_->response_id(),
 980         manifest_response_writer_->amount_written());
 981     if (!inprogress_cache_->AddOrModifyEntry(manifest_url_, entry))
 982       duplicate_response_ids_.push_back(entry.response_id());
 983     StoreGroupAndCache();
 984   } else {
 985     HandleCacheFailure(
 986         AppCacheErrorDetails("Failed to write the manifest data to storage",
 987                      APPCACHE_UNKNOWN_ERROR,
 988                      GURL(),
 989                      0,
 990                      false /*is_cross_origin*/),
 991         DISKCACHE_ERROR,
 992         GURL());
 993   }
 994 }
 995
 996 void AppCacheUpdateJob::StoreGroupAndCache() {
 997   DCHECK(stored_state_ == UNSTORED);
 998   stored_state_ = STORING;
 999
1000   scoped_refptr<AppCache> newest_cache;
1001   if (inprogress_cache_.get())
1002     newest_cache.swap(inprogress_cache_);
1003   else
1004     newest_cache = group_->newest_complete_cache();
1005   newest_cache->set_update_time(base::Time::Now());
1006
1007   group_->set_first_evictable_error_time(base::Time());
1008   if (doing_full_update_check_)
1009     group_->set_last_full_update_check_time(base::Time::Now());
1010
1011   storage_->StoreGroupAndNewestCache(group_, newest_cache.get(), this);
1012 }
1013
1014 void AppCacheUpdateJob::OnGroupAndNewestCacheStored(AppCacheGroup* group,
1015                                                     AppCache* newest_cache,
1016                                                     bool success,
1017                                                     bool would_exceed_quota) {
1018   DCHECK(stored_state_ == STORING);
1019   if (success) {
1020     stored_state_ = STORED;
1021     MaybeCompleteUpdate();  // will definitely complete
1022     return;
1023   }
1024
1025   stored_state_ = UNSTORED;
1026
1027   // Restore inprogress_cache_ to get the proper events delivered
1028   // and the proper cleanup to occur.
1029   if (newest_cache != group->newest_complete_cache())
1030     inprogress_cache_ = newest_cache;
1031
1032   ResultType result = DB_ERROR;
1033   AppCacheErrorReason reason = APPCACHE_UNKNOWN_ERROR;
1034   std::string message("Failed to commit new cache to storage");
1035   if (would_exceed_quota) {
1036     message.append(", would exceed quota");
1037     result = QUOTA_ERROR;
1038     reason = APPCACHE_QUOTA_ERROR;
1039   }
1040   HandleCacheFailure(
1041       AppCacheErrorDetails(message, reason, GURL(), 0,
1042           false /*is_cross_origin*/),
1043       result,
1044       GURL());
1045 }
1046
1047 void AppCacheUpdateJob::NotifySingleHost(AppCacheHost* host,
1048                                          AppCacheEventID event_id) {
1049   std::vector<int> ids(1, host->host_id());
1050   host->frontend()->OnEventRaised(ids, event_id);
1051 }
1052
1053 void AppCacheUpdateJob::NotifyAllAssociatedHosts(AppCacheEventID event_id) {
1054   HostNotifier host_notifier;
1055   AddAllAssociatedHostsToNotifier(&host_notifier);
1056   host_notifier.SendNotifications(event_id);
1057 }
1058
1059 void AppCacheUpdateJob::NotifyAllProgress(const GURL& url) {
1060   HostNotifier host_notifier;
1061   AddAllAssociatedHostsToNotifier(&host_notifier);
1062   host_notifier.SendProgressNotifications(
1063       url, url_file_list_.size(), url_fetches_completed_);
1064 }
1065
1066 void AppCacheUpdateJob::NotifyAllFinalProgress() {
1067   DCHECK(url_file_list_.size() == url_fetches_completed_);
1068   NotifyAllProgress(GURL());
1069 }
1070
1071 void AppCacheUpdateJob::NotifyAllError(const AppCacheErrorDetails& details) {
1072   HostNotifier host_notifier;
1073   AddAllAssociatedHostsToNotifier(&host_notifier);
1074   host_notifier.SendErrorNotifications(details);
1075 }
1076
1077 void AppCacheUpdateJob::LogConsoleMessageToAll(const std::string& message) {
1078   HostNotifier host_notifier;
1079   AddAllAssociatedHostsToNotifier(&host_notifier);
1080   host_notifier.SendLogMessage(message);
1081 }
1082
1083 void AppCacheUpdateJob::AddAllAssociatedHostsToNotifier(
1084     HostNotifier* host_notifier) {
1085   // Collect hosts so we only send one notification per frontend.
1086   // A host can only be associated with a single cache so no need to worry
1087   // about duplicate hosts being added to the notifier.
1088   if (inprogress_cache_.get()) {
1089     DCHECK(internal_state_ == DOWNLOADING || internal_state_ == CACHE_FAILURE);
1090     host_notifier->AddHosts(inprogress_cache_->associated_hosts());
1091   }
1092
1093   AppCacheGroup::Caches old_caches = group_->old_caches();
1094   for (AppCacheGroup::Caches::const_iterator it = old_caches.begin();
1095        it != old_caches.end(); ++it) {
1096     host_notifier->AddHosts((*it)->associated_hosts());
1097   }
1098
1099   AppCache* newest_cache = group_->newest_complete_cache();
1100   if (newest_cache)
1101     host_notifier->AddHosts(newest_cache->associated_hosts());
1102 }
1103
1104 void AppCacheUpdateJob::OnDestructionImminent(AppCacheHost* host) {
1105   // The host is about to be deleted; remove from our collection.
1106   PendingMasters::iterator found =
1107       pending_master_entries_.find(host->pending_master_entry_url());
1108   DCHECK(found != pending_master_entries_.end());
1109   PendingHosts& hosts = found->second;
1110   PendingHosts::iterator it = std::find(hosts.begin(), hosts.end(), host);
1111   DCHECK(it != hosts.end());
1112   hosts.erase(it);
1113 }
1114
1115 void AppCacheUpdateJob::OnServiceReinitialized(
1116     AppCacheStorageReference* old_storage_ref) {
1117   // We continue to use the disabled instance, but arrange for its
1118   // deletion when its no longer needed.
1119   if (old_storage_ref->storage() == storage_)
1120     disabled_storage_reference_ = old_storage_ref;
1121 }
1122
1123 void AppCacheUpdateJob::CheckIfManifestChanged() {
1124   DCHECK(update_type_ == UPGRADE_ATTEMPT);
1125   AppCacheEntry* entry = NULL;
1126   if (group_->newest_complete_cache())
1127     entry = group_->newest_complete_cache()->GetEntry(manifest_url_);
1128   if (!entry) {
1129     // TODO(michaeln): This is just a bandaid to avoid a crash.
1130     // http://code.google.com/p/chromium/issues/detail?id=95101
1131     if (service_->storage() == storage_) {
1132       // Use a local variable because service_ is reset in HandleCacheFailure.
1133       AppCacheServiceImpl* service = service_;
1134       HandleCacheFailure(
1135           AppCacheErrorDetails("Manifest entry not found in existing cache",
1136                        APPCACHE_UNKNOWN_ERROR,
1137                        GURL(),
1138                        0,
1139                        false /*is_cross_origin*/),
1140           DB_ERROR,
1141           GURL());
1142       AppCacheHistograms::AddMissingManifestEntrySample();
1143       service->DeleteAppCacheGroup(manifest_url_, net::CompletionCallback());
1144     }
1145     return;
1146   }
1147
1148   // Load manifest data from storage to compare against fetched manifest.
1149   manifest_response_reader_.reset(
1150       storage_->CreateResponseReader(manifest_url_,
1151                                      group_->group_id(),
1152                                      entry->response_id()));
1153   read_manifest_buffer_ = new net::IOBuffer(kBufferSize);
1154   manifest_response_reader_->ReadData(
1155       read_manifest_buffer_.get(),
1156       kBufferSize,
1157       base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
1158                  base::Unretained(this)));  // async read
1159 }
1160
1161 void AppCacheUpdateJob::OnManifestDataReadComplete(int result) {
1162   if (result > 0) {
1163     loaded_manifest_data_.append(read_manifest_buffer_->data(), result);
1164     manifest_response_reader_->ReadData(
1165         read_manifest_buffer_.get(),
1166         kBufferSize,
1167         base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
1168                    base::Unretained(this)));  // read more
1169   } else {
1170     read_manifest_buffer_ = NULL;
1171     manifest_response_reader_.reset();
1172     ContinueHandleManifestFetchCompleted(
1173         result < 0 || manifest_data_ != loaded_manifest_data_);
1174   }
1175 }
1176
1177 void AppCacheUpdateJob::BuildUrlFileList(const AppCacheManifest& manifest) {
1178   for (base::hash_set<std::string>::const_iterator it =
1179            manifest.explicit_urls.begin();
1180        it != manifest.explicit_urls.end(); ++it) {
1181     AddUrlToFileList(GURL(*it), AppCacheEntry::EXPLICIT);
1182   }
1183
1184   const std::vector<AppCacheNamespace>& intercepts =
1185       manifest.intercept_namespaces;
1186   for (std::vector<AppCacheNamespace>::const_iterator it = intercepts.begin();
1187        it != intercepts.end(); ++it) {
1188     int flags = AppCacheEntry::INTERCEPT;
1189     if (it->is_executable)
1190       flags |= AppCacheEntry::EXECUTABLE;
1191     AddUrlToFileList(it->target_url, flags);
1192   }
1193
1194   const std::vector<AppCacheNamespace>& fallbacks =
1195       manifest.fallback_namespaces;
1196   for (std::vector<AppCacheNamespace>::const_iterator it = fallbacks.begin();
1197        it != fallbacks.end(); ++it) {
1198      AddUrlToFileList(it->target_url, AppCacheEntry::FALLBACK);
1199   }
1200
1201   // Add all master entries from newest complete cache.
1202   if (update_type_ == UPGRADE_ATTEMPT) {
1203     const AppCache::EntryMap& entries =
1204         group_->newest_complete_cache()->entries();
1205     for (AppCache::EntryMap::const_iterator it = entries.begin();
1206          it != entries.end(); ++it) {
1207       const AppCacheEntry& entry = it->second;
1208       if (entry.IsMaster())
1209         AddUrlToFileList(it->first, AppCacheEntry::MASTER);
1210     }
1211   }
1212 }
1213
1214 void AppCacheUpdateJob::AddUrlToFileList(const GURL& url, int type) {
1215   std::pair<AppCache::EntryMap::iterator, bool> ret = url_file_list_.insert(
1216       AppCache::EntryMap::value_type(url, AppCacheEntry(type)));
1217
1218   if (ret.second)
1219     urls_to_fetch_.push_back(UrlToFetch(url, false, NULL));
1220   else
1221     ret.first->second.add_types(type);  // URL already exists. Merge types.
1222 }
1223
1224 void AppCacheUpdateJob::FetchUrls() {
1225   DCHECK(internal_state_ == DOWNLOADING);
1226
1227   // Fetch each URL in the list according to section 6.9.4 step 17.1-17.3.
1228   // Fetch up to the concurrent limit. Other fetches will be triggered as each
1229   // each fetch completes.
1230   while (pending_url_fetches_.size() < kMaxConcurrentUrlFetches &&
1231          !urls_to_fetch_.empty()) {
1232     UrlToFetch url_to_fetch = urls_to_fetch_.front();
1233     urls_to_fetch_.pop_front();
1234
1235     AppCache::EntryMap::iterator it = url_file_list_.find(url_to_fetch.url);
1236     DCHECK(it != url_file_list_.end());
1237     AppCacheEntry& entry = it->second;
1238     if (ShouldSkipUrlFetch(entry)) {
1239       NotifyAllProgress(url_to_fetch.url);
1240       ++url_fetches_completed_;
1241     } else if (AlreadyFetchedEntry(url_to_fetch.url, entry.types())) {
1242       NotifyAllProgress(url_to_fetch.url);
1243       ++url_fetches_completed_;  // saved a URL request
1244     } else if (!url_to_fetch.storage_checked &&
1245                MaybeLoadFromNewestCache(url_to_fetch.url, entry)) {
1246       // Continues asynchronously after data is loaded from newest cache.
1247     } else {
1248       URLFetcher* fetcher = new URLFetcher(
1249           url_to_fetch.url, URLFetcher::URL_FETCH, this);
1250       if (url_to_fetch.existing_response_info.get()) {
1251         DCHECK(group_->newest_complete_cache());
1252         AppCacheEntry* existing_entry =
1253             group_->newest_complete_cache()->GetEntry(url_to_fetch.url);
1254         DCHECK(existing_entry);
1255         DCHECK(existing_entry->response_id() ==
1256                url_to_fetch.existing_response_info->response_id());
1257         fetcher->set_existing_response_headers(
1258             url_to_fetch.existing_response_info->http_response_info()->headers
1259                 .get());
1260         fetcher->set_existing_entry(*existing_entry);
1261       }
1262       fetcher->Start();
1263       pending_url_fetches_.insert(
1264           PendingUrlFetches::value_type(url_to_fetch.url, fetcher));
1265     }
1266   }
1267 }
1268
1269 void AppCacheUpdateJob::CancelAllUrlFetches() {
1270   // Cancel any pending URL requests.
1271   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
1272        it != pending_url_fetches_.end(); ++it) {
1273     delete it->second;
1274   }
1275
1276   url_fetches_completed_ +=
1277       pending_url_fetches_.size() + urls_to_fetch_.size();
1278   pending_url_fetches_.clear();
1279   urls_to_fetch_.clear();
1280 }
1281
1282 bool AppCacheUpdateJob::ShouldSkipUrlFetch(const AppCacheEntry& entry) {
1283   // 6.6.4 Step 17
1284   // If the resource URL being processed was flagged as neither an
1285   // "explicit entry" nor or a "fallback entry", then the user agent
1286   // may skip this URL.
1287   if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept())
1288     return false;
1289
1290   // TODO(jennb): decide if entry should be skipped to expire it from cache
1291   return false;
1292 }
1293
1294 bool AppCacheUpdateJob::AlreadyFetchedEntry(const GURL& url,
1295                                             int entry_type) {
1296   DCHECK(internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE);
1297   AppCacheEntry* existing =
1298       inprogress_cache_.get() ? inprogress_cache_->GetEntry(url)
1299                               : group_->newest_complete_cache()->GetEntry(url);
1300   if (existing) {
1301     existing->add_types(entry_type);
1302     return true;
1303   }
1304   return false;
1305 }
1306
1307 void AppCacheUpdateJob::AddMasterEntryToFetchList(AppCacheHost* host,
1308                                                   const GURL& url,
1309                                                   bool is_new) {
1310   DCHECK(!IsTerminating());
1311
1312   if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE) {
1313     AppCache* cache;
1314     if (inprogress_cache_.get()) {
1315       // always associate
1316       host->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
1317       cache = inprogress_cache_.get();
1318     } else {
1319       cache = group_->newest_complete_cache();
1320     }
1321
1322     // Update existing entry if it has already been fetched.
1323     AppCacheEntry* entry = cache->GetEntry(url);
1324     if (entry) {
1325       entry->add_types(AppCacheEntry::MASTER);
1326       if (internal_state_ == NO_UPDATE && !inprogress_cache_.get()) {
1327         // only associate if have entry
1328         host->AssociateCompleteCache(cache);
1329       }
1330       if (is_new)
1331         ++master_entries_completed_;  // pretend fetching completed
1332       return;
1333     }
1334   }
1335
1336   // Add to fetch list if not already fetching.
1337   if (master_entry_fetches_.find(url) == master_entry_fetches_.end()) {
1338     master_entries_to_fetch_.insert(url);
1339     if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE)
1340       FetchMasterEntries();
1341   }
1342 }
1343
1344 void AppCacheUpdateJob::FetchMasterEntries() {
1345   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
1346
1347   // Fetch each master entry in the list, up to the concurrent limit.
1348   // Additional fetches will be triggered as each fetch completes.
1349   while (master_entry_fetches_.size() < kMaxConcurrentUrlFetches &&
1350          !master_entries_to_fetch_.empty()) {
1351     const GURL& url = *master_entries_to_fetch_.begin();
1352
1353     if (AlreadyFetchedEntry(url, AppCacheEntry::MASTER)) {
1354       ++master_entries_completed_;  // saved a URL request
1355
1356       // In no update case, associate hosts to newest cache in group
1357       // now that master entry has been "successfully downloaded".
1358       if (internal_state_ == NO_UPDATE) {
1359         // TODO(michaeln): defer until the updated cache has been stored.
1360         DCHECK(!inprogress_cache_.get());
1361         AppCache* cache = group_->newest_complete_cache();
1362         PendingMasters::iterator found = pending_master_entries_.find(url);
1363         DCHECK(found != pending_master_entries_.end());
1364         PendingHosts& hosts = found->second;
1365         for (PendingHosts::iterator host_it = hosts.begin();
1366              host_it != hosts.end(); ++host_it) {
1367           (*host_it)->AssociateCompleteCache(cache);
1368         }
1369       }
1370     } else {
1371       URLFetcher* fetcher = new URLFetcher(
1372           url, URLFetcher::MASTER_ENTRY_FETCH, this);
1373       fetcher->Start();
1374       master_entry_fetches_.insert(PendingUrlFetches::value_type(url, fetcher));
1375     }
1376
1377     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
1378   }
1379 }
1380
1381 void AppCacheUpdateJob::CancelAllMasterEntryFetches(
1382     const AppCacheErrorDetails& error_details) {
1383   // For now, cancel all in-progress fetches for master entries and pretend
1384   // all master entries fetches have completed.
1385   // TODO(jennb): Delete this when update no longer fetches master entries
1386   // directly.
1387
1388   // Cancel all in-progress fetches.
1389   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
1390        it != master_entry_fetches_.end(); ++it) {
1391     delete it->second;
1392     master_entries_to_fetch_.insert(it->first);  // back in unfetched list
1393   }
1394   master_entry_fetches_.clear();
1395
1396   master_entries_completed_ += master_entries_to_fetch_.size();
1397
1398   // Cache failure steps, step 2.
1399   // Pretend all master entries that have not yet been fetched have completed
1400   // downloading. Unassociate hosts from any appcache and send ERROR event.
1401   HostNotifier host_notifier;
1402   while (!master_entries_to_fetch_.empty()) {
1403     const GURL& url = *master_entries_to_fetch_.begin();
1404     PendingMasters::iterator found = pending_master_entries_.find(url);
1405     DCHECK(found != pending_master_entries_.end());
1406     PendingHosts& hosts = found->second;
1407     for (PendingHosts::iterator host_it = hosts.begin();
1408          host_it != hosts.end(); ++host_it) {
1409       AppCacheHost* host = *host_it;
1410       host->AssociateNoCache(GURL());
1411       host_notifier.AddHost(host);
1412       host->RemoveObserver(this);
1413     }
1414     hosts.clear();
1415
1416     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
1417   }
1418   host_notifier.SendErrorNotifications(error_details);
1419 }
1420
1421 bool AppCacheUpdateJob::MaybeLoadFromNewestCache(const GURL& url,
1422                                                  AppCacheEntry& entry) {
1423   if (update_type_ != UPGRADE_ATTEMPT)
1424     return false;
1425
1426   AppCache* newest = group_->newest_complete_cache();
1427   AppCacheEntry* copy_me = newest->GetEntry(url);
1428   if (!copy_me || !copy_me->has_response_id())
1429     return false;
1430
1431   // Load HTTP headers for entry from newest cache.
1432   loading_responses_.insert(
1433       LoadingResponses::value_type(copy_me->response_id(), url));
1434   storage_->LoadResponseInfo(manifest_url_, group_->group_id(),
1435                              copy_me->response_id(),
1436                              this);
1437   // Async: wait for OnResponseInfoLoaded to complete.
1438   return true;
1439 }
1440
1441 void AppCacheUpdateJob::OnResponseInfoLoaded(
1442     AppCacheResponseInfo* response_info, int64 response_id) {
1443   const net::HttpResponseInfo* http_info = response_info ?
1444       response_info->http_response_info() : NULL;
1445
1446   // Needed response info for a manifest fetch request.
1447   if (internal_state_ == FETCH_MANIFEST) {
1448     if (http_info)
1449       manifest_fetcher_->set_existing_response_headers(
1450           http_info->headers.get());
1451     manifest_fetcher_->Start();
1452     return;
1453   }
1454
1455   LoadingResponses::iterator found = loading_responses_.find(response_id);
1456   DCHECK(found != loading_responses_.end());
1457   const GURL& url = found->second;
1458
1459   if (!http_info) {
1460     LoadFromNewestCacheFailed(url, NULL);  // no response found
1461   } else {
1462     // Check if response can be re-used according to HTTP caching semantics.
1463     // Responses with a "vary" header get treated as expired.
1464     const std::string name = "vary";
1465     std::string value;
1466     void* iter = NULL;
1467     if (!http_info->headers.get() ||
1468         http_info->headers->RequiresValidation(http_info->request_time,
1469                                                http_info->response_time,
1470                                                base::Time::Now()) ||
1471         http_info->headers->EnumerateHeader(&iter, name, &value)) {
1472       LoadFromNewestCacheFailed(url, response_info);
1473     } else {
1474       DCHECK(group_->newest_complete_cache());
1475       AppCacheEntry* copy_me = group_->newest_complete_cache()->GetEntry(url);
1476       DCHECK(copy_me);
1477       DCHECK(copy_me->response_id() == response_id);
1478
1479       AppCache::EntryMap::iterator it = url_file_list_.find(url);
1480       DCHECK(it != url_file_list_.end());
1481       AppCacheEntry& entry = it->second;
1482       entry.set_response_id(response_id);
1483       entry.set_response_size(copy_me->response_size());
1484       inprogress_cache_->AddOrModifyEntry(url, entry);
1485       NotifyAllProgress(url);
1486       ++url_fetches_completed_;
1487     }
1488   }
1489   loading_responses_.erase(found);
1490
1491   MaybeCompleteUpdate();
1492 }
1493
1494 void AppCacheUpdateJob::LoadFromNewestCacheFailed(
1495     const GURL& url, AppCacheResponseInfo* response_info) {
1496   if (internal_state_ == CACHE_FAILURE)
1497     return;
1498
1499   // Re-insert url at front of fetch list. Indicate storage has been checked.
1500   urls_to_fetch_.push_front(UrlToFetch(url, true, response_info));
1501   FetchUrls();
1502 }
1503
1504 void AppCacheUpdateJob::MaybeCompleteUpdate() {
1505   DCHECK(internal_state_ != CACHE_FAILURE);
1506
1507   // Must wait for any pending master entries or url fetches to complete.
1508   if (master_entries_completed_ != pending_master_entries_.size() ||
1509       url_fetches_completed_ != url_file_list_.size()) {
1510     DCHECK(internal_state_ != COMPLETED);
1511     return;
1512   }
1513
1514   switch (internal_state_) {
1515     case NO_UPDATE:
1516       if (master_entries_completed_ > 0) {
1517         switch (stored_state_) {
1518           case UNSTORED:
1519             StoreGroupAndCache();
1520             return;
1521           case STORING:
1522             return;
1523           case STORED:
1524             break;
1525         }
1526       } else {
1527         bool times_changed = false;
1528         if (!group_->first_evictable_error_time().is_null()) {
1529           group_->set_first_evictable_error_time(base::Time());
1530           times_changed = true;
1531         }
1532         if (doing_full_update_check_) {
1533           group_->set_last_full_update_check_time(base::Time::Now());
1534           times_changed = true;
1535         }
1536         if (times_changed)
1537           storage_->StoreEvictionTimes(group_);
1538       }
1539       // 6.9.4 steps 7.3-7.7.
1540       NotifyAllAssociatedHosts(APPCACHE_NO_UPDATE_EVENT);
1541       DiscardDuplicateResponses();
1542       internal_state_ = COMPLETED;
1543       break;
1544     case DOWNLOADING:
1545       internal_state_ = REFETCH_MANIFEST;
1546       FetchManifest(false);
1547       break;
1548     case REFETCH_MANIFEST:
1549       DCHECK(stored_state_ == STORED);
1550       NotifyAllFinalProgress();
1551       if (update_type_ == CACHE_ATTEMPT)
1552         NotifyAllAssociatedHosts(APPCACHE_CACHED_EVENT);
1553       else
1554         NotifyAllAssociatedHosts(APPCACHE_UPDATE_READY_EVENT);
1555       DiscardDuplicateResponses();
1556       internal_state_ = COMPLETED;
1557       LogHistogramStats(UPDATE_OK, GURL());
1558       break;
1559     case CACHE_FAILURE:
1560       NOTREACHED();  // See HandleCacheFailure
1561       break;
1562     default:
1563       break;
1564   }
1565
1566   // Let the stack unwind before deletion to make it less risky as this
1567   // method is called from multiple places in this file.
1568   if (internal_state_ == COMPLETED)
1569     DeleteSoon();
1570 }
1571
1572 void AppCacheUpdateJob::ScheduleUpdateRetry(int delay_ms) {
1573   // TODO(jennb): post a delayed task with the "same parameters" as this job
1574   // to retry the update at a later time. Need group, URLs of pending master
1575   // entries and their hosts.
1576 }
1577
1578 void AppCacheUpdateJob::Cancel() {
1579   internal_state_ = CANCELLED;
1580
1581   LogHistogramStats(CANCELLED_ERROR, GURL());
1582
1583   if (manifest_fetcher_) {
1584     delete manifest_fetcher_;
1585     manifest_fetcher_ = NULL;
1586   }
1587
1588   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
1589        it != pending_url_fetches_.end(); ++it) {
1590     delete it->second;
1591   }
1592   pending_url_fetches_.clear();
1593
1594   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
1595        it != master_entry_fetches_.end(); ++it) {
1596     delete it->second;
1597   }
1598   master_entry_fetches_.clear();
1599
1600   ClearPendingMasterEntries();
1601   DiscardInprogressCache();
1602
1603   // Delete response writer to avoid any callbacks.
1604   if (manifest_response_writer_)
1605     manifest_response_writer_.reset();
1606
1607   storage_->CancelDelegateCallbacks(this);
1608 }
1609
1610 void AppCacheUpdateJob::ClearPendingMasterEntries() {
1611   for (PendingMasters::iterator it = pending_master_entries_.begin();
1612        it != pending_master_entries_.end(); ++it) {
1613     PendingHosts& hosts = it->second;
1614     for (PendingHosts::iterator host_it = hosts.begin();
1615          host_it != hosts.end(); ++host_it) {
1616       (*host_it)->RemoveObserver(this);
1617     }
1618   }
1619
1620   pending_master_entries_.clear();
1621 }
1622
1623 void AppCacheUpdateJob::DiscardInprogressCache() {
1624   if (stored_state_ == STORING) {
1625     // We can make no assumptions about whether the StoreGroupAndCacheTask
1626     // actually completed or not. This condition should only be reachable
1627     // during shutdown. Free things up and return to do no harm.
1628     inprogress_cache_ = NULL;
1629     added_master_entries_.clear();
1630     return;
1631   }
1632
1633   storage_->DoomResponses(manifest_url_, stored_response_ids_);
1634
1635   if (!inprogress_cache_.get()) {
1636     // We have to undo the changes we made, if any, to the existing cache.
1637     if (group_ && group_->newest_complete_cache()) {
1638       for (std::vector<GURL>::iterator iter = added_master_entries_.begin();
1639            iter != added_master_entries_.end(); ++iter) {
1640         group_->newest_complete_cache()->RemoveEntry(*iter);
1641       }
1642     }
1643     added_master_entries_.clear();
1644     return;
1645   }
1646
1647   AppCache::AppCacheHosts& hosts = inprogress_cache_->associated_hosts();
1648   while (!hosts.empty())
1649     (*hosts.begin())->AssociateNoCache(GURL());
1650
1651   inprogress_cache_ = NULL;
1652   added_master_entries_.clear();
1653 }
1654
1655 void AppCacheUpdateJob::DiscardDuplicateResponses() {
1656   storage_->DoomResponses(manifest_url_, duplicate_response_ids_);
1657 }
1658
1659 void AppCacheUpdateJob::LogHistogramStats(
1660       ResultType result, const GURL& failed_resource_url) {
1661   AppCacheHistograms::CountUpdateJobResult(result, manifest_url_.GetOrigin());
1662   if (result == UPDATE_OK)
1663     return;
1664
1665   int percent_complete = 0;
1666   if (url_file_list_.size() > 0) {
1667     size_t actual_fetches_completed = url_fetches_completed_;
1668     if (!failed_resource_url.is_empty() && actual_fetches_completed)
1669       --actual_fetches_completed;
1670     percent_complete = (static_cast<double>(actual_fetches_completed) /
1671                             static_cast<double>(url_file_list_.size())) * 100.0;
1672     percent_complete = std::min(percent_complete, 99);
1673   }
1674
1675   bool was_making_progress =
1676       base::Time::Now() - last_progress_time_ <
1677           base::TimeDelta::FromMinutes(5);
1678
1679   bool off_origin_resource_failure =
1680       !failed_resource_url.is_empty() &&
1681           (failed_resource_url.GetOrigin() != manifest_url_.GetOrigin());
1682
1683   AppCacheHistograms::LogUpdateFailureStats(
1684       manifest_url_.GetOrigin(),
1685       percent_complete,
1686       was_making_progress,
1687       off_origin_resource_failure);
1688 }
1689
1690 void AppCacheUpdateJob::DeleteSoon() {
1691   ClearPendingMasterEntries();
1692   manifest_response_writer_.reset();
1693   storage_->CancelDelegateCallbacks(this);
1694   service_->RemoveObserver(this);
1695   service_ = NULL;
1696
1697   // Break the connection with the group so the group cannot call delete
1698   // on this object after we've posted a task to delete ourselves.
1699   if (group_) {
1700     group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE);
1701     group_ = NULL;
1702   }
1703
1704   base::MessageLoop::current()->DeleteSoon(FROM_HERE, this);
1705 }
1706
1707 }  // namespace content