components/precache/core/precache_fetcher.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/precache/core/precache_fetcher.h"
   6
   7 #include <string>
   8
   9 #include "base/bind.h"
  10 #include "base/callback.h"
  11 #include "base/command_line.h"
  12 #include "base/compiler_specific.h"
  13 #include "base/containers/hash_tables.h"
  14 #include "components/precache/core/precache_switches.h"
  15 #include "components/precache/core/proto/precache.pb.h"
  16 #include "net/base/escape.h"
  17 #include "net/base/load_flags.h"
  18 #include "net/url_request/url_fetcher.h"
  19 #include "net/url_request/url_fetcher_delegate.h"
  20 #include "net/url_request/url_request_context_getter.h"
  21 #include "net/url_request/url_request_status.h"
  22
  23 using net::URLFetcher;
  24
  25 namespace precache {
  26
  27 namespace {
  28
  29 GURL GetConfigURL() {
  30   const CommandLine& command_line = *CommandLine::ForCurrentProcess();
  31   if (command_line.HasSwitch(switches::kPrecacheConfigSettingsURL)) {
  32     return GURL(
  33         command_line.GetSwitchValueASCII(switches::kPrecacheConfigSettingsURL));
  34   }
  35
  36 #if defined(PRECACHE_CONFIG_SETTINGS_URL)
  37   return GURL(PRECACHE_CONFIG_SETTINGS_URL);
  38 #else
  39   // The precache config settings URL could not be determined, so return an
  40   // empty, invalid GURL.
  41   return GURL();
  42 #endif
  43 }
  44
  45 std::string GetManifestURLPrefix() {
  46   const CommandLine& command_line = *CommandLine::ForCurrentProcess();
  47   if (command_line.HasSwitch(switches::kPrecacheManifestURLPrefix)) {
  48     return command_line.GetSwitchValueASCII(
  49         switches::kPrecacheManifestURLPrefix);
  50   }
  51
  52 #if defined(PRECACHE_MANIFEST_URL_PREFIX)
  53   return PRECACHE_MANIFEST_URL_PREFIX;
  54 #else
  55   // The precache manifest URL prefix could not be determined, so return an
  56   // empty string.
  57   return std::string();
  58 #endif
  59 }
  60
  61 // Construct the URL of the precache manifest for the given starting URL.
  62 // The server is expecting a request for a URL consisting of the manifest URL
  63 // prefix followed by the doubly escaped starting URL.
  64 GURL ConstructManifestURL(const GURL& starting_url) {
  65   return GURL(
  66       GetManifestURLPrefix() +
  67       net::EscapeQueryParamValue(
  68           net::EscapeQueryParamValue(starting_url.spec(), false), false));
  69 }
  70
  71 // Attempts to parse a protobuf message from the response string of a
  72 // URLFetcher. If parsing is successful, the message parameter will contain the
  73 // parsed protobuf and this function will return true. Otherwise, returns false.
  74 bool ParseProtoFromFetchResponse(const URLFetcher& source,
  75                                  ::google::protobuf::MessageLite* message) {
  76   std::string response_string;
  77
  78   if (!source.GetStatus().is_success()) {
  79     DLOG(WARNING) << "Fetch failed: " << source.GetOriginalURL().spec();
  80     return false;
  81   }
  82   if (!source.GetResponseAsString(&response_string)) {
  83     DLOG(WARNING) << "No response string present: "
  84                   << source.GetOriginalURL().spec();
  85     return false;
  86   }
  87   if (!message->ParseFromString(response_string)) {
  88     DLOG(WARNING) << "Unable to parse proto served from "
  89                   << source.GetOriginalURL().spec();
  90     return false;
  91   }
  92   return true;
  93 }
  94
  95 }  // namespace
  96
  97 // Class that fetches a URL, and runs the specified callback when the fetch is
  98 // complete. This class exists so that a different method can be run in
  99 // response to different kinds of fetches, e.g. OnConfigFetchComplete when
 100 // configuration settings are fetched, OnManifestFetchComplete when a manifest
 101 // is fetched, etc.
 102 class PrecacheFetcher::Fetcher : public net::URLFetcherDelegate {
 103  public:
 104   // Construct a new Fetcher. This will create and start a new URLFetcher for
 105   // the specified URL using the specified request context.
 106   Fetcher(net::URLRequestContextGetter* request_context, const GURL& url,
 107           const base::Callback<void(const URLFetcher&)>& callback);
 108   virtual ~Fetcher() {}
 109   virtual void OnURLFetchComplete(const URLFetcher* source) OVERRIDE;
 110
 111  private:
 112   const base::Callback<void(const URLFetcher&)> callback_;
 113   scoped_ptr<URLFetcher> url_fetcher_;
 114
 115   DISALLOW_COPY_AND_ASSIGN(Fetcher);
 116 };
 117
 118 PrecacheFetcher::Fetcher::Fetcher(
 119     net::URLRequestContextGetter* request_context, const GURL& url,
 120     const base::Callback<void(const URLFetcher&)>& callback)
 121     : callback_(callback) {
 122   url_fetcher_.reset(URLFetcher::Create(url, URLFetcher::GET, this));
 123   url_fetcher_->SetRequestContext(request_context);
 124   url_fetcher_->SetLoadFlags(net::LOAD_DO_NOT_PROMPT_FOR_LOGIN);
 125   url_fetcher_->Start();
 126 }
 127
 128 void PrecacheFetcher::Fetcher::OnURLFetchComplete(const URLFetcher* source) {
 129   callback_.Run(*source);
 130 }
 131
 132 PrecacheFetcher::PrecacheFetcher(
 133     const std::list<GURL>& starting_urls,
 134     net::URLRequestContextGetter* request_context,
 135     PrecacheFetcher::PrecacheDelegate* precache_delegate)
 136     : starting_urls_(starting_urls),
 137       request_context_(request_context),
 138       precache_delegate_(precache_delegate) {
 139   DCHECK(request_context_);    // Request context must be non-NULL.
 140   DCHECK(precache_delegate_);  // Precache delegate must be non-NULL.
 141
 142   DCHECK_NE(GURL(), GetConfigURL())
 143       << "Could not determine the precache config settings URL.";
 144   DCHECK_NE(std::string(), GetManifestURLPrefix())
 145       << "Could not determine the precache manifest URL prefix.";
 146 }
 147
 148 PrecacheFetcher::~PrecacheFetcher() {
 149 }
 150
 151 void PrecacheFetcher::Start() {
 152   DCHECK(!fetcher_);  // Start shouldn't be called repeatedly.
 153
 154   GURL config_url = GetConfigURL();
 155   DCHECK(config_url.is_valid());
 156
 157   // Fetch the precache configuration settings from the server.
 158   fetcher_.reset(new Fetcher(request_context_, config_url,
 159                              base::Bind(&PrecacheFetcher::OnConfigFetchComplete,
 160                                         base::Unretained(this))));
 161 }
 162
 163 void PrecacheFetcher::StartNextFetch() {
 164   if (!resource_urls_to_fetch_.empty()) {
 165     // Fetch the next resource URL.
 166     fetcher_.reset(
 167         new Fetcher(request_context_, resource_urls_to_fetch_.front(),
 168                     base::Bind(&PrecacheFetcher::OnResourceFetchComplete,
 169                                base::Unretained(this))));
 170
 171     resource_urls_to_fetch_.pop_front();
 172     return;
 173   }
 174
 175   if (!manifest_urls_to_fetch_.empty()) {
 176     // Fetch the next manifest URL.
 177     fetcher_.reset(
 178         new Fetcher(request_context_, manifest_urls_to_fetch_.front(),
 179                     base::Bind(&PrecacheFetcher::OnManifestFetchComplete,
 180                                base::Unretained(this))));
 181
 182     manifest_urls_to_fetch_.pop_front();
 183     return;
 184   }
 185
 186   // There are no more URLs to fetch, so end the precache cycle.
 187   precache_delegate_->OnDone();
 188   // OnDone may have deleted this PrecacheFetcher, so don't do anything after it
 189   // is called.
 190 }
 191
 192 void PrecacheFetcher::OnConfigFetchComplete(const URLFetcher& source) {
 193   PrecacheConfigurationSettings config;
 194
 195   if (ParseProtoFromFetchResponse(source, &config)) {
 196     // Keep track of starting URLs that manifests are being fetched for, in
 197     // order to remove duplicates. This is a hash set on strings, and not GURLs,
 198     // because there is no hash function defined for GURL.
 199     base::hash_set<std::string> unique_starting_urls;
 200
 201     // Attempt to fetch manifests for starting URLs up to the maximum top sites
 202     // count. If a manifest does not exist for a particular starting URL, then
 203     // the fetch will fail, and that starting URL will be ignored.
 204     int64 rank = 0;
 205     for (std::list<GURL>::const_iterator it = starting_urls_.begin();
 206          it != starting_urls_.end() && rank < config.top_sites_count();
 207          ++it, ++rank) {
 208       if (unique_starting_urls.find(it->spec()) == unique_starting_urls.end()) {
 209         // Only add a fetch for the manifest URL if this manifest isn't already
 210         // going to be fetched.
 211         manifest_urls_to_fetch_.push_back(ConstructManifestURL(*it));
 212         unique_starting_urls.insert(it->spec());
 213       }
 214     }
 215
 216     for (int i = 0; i < config.forced_starting_url_size(); ++i) {
 217       // Convert the string URL into a GURL and take the spec() of it so that
 218       // the URL string gets canonicalized.
 219       GURL url(config.forced_starting_url(i));
 220       if (unique_starting_urls.find(url.spec()) == unique_starting_urls.end()) {
 221         // Only add a fetch for the manifest URL if this manifest isn't already
 222         // going to be fetched.
 223         manifest_urls_to_fetch_.push_back(ConstructManifestURL(url));
 224         unique_starting_urls.insert(url.spec());
 225       }
 226     }
 227   }
 228
 229   StartNextFetch();
 230 }
 231
 232 void PrecacheFetcher::OnManifestFetchComplete(const URLFetcher& source) {
 233   PrecacheManifest manifest;
 234
 235   if (ParseProtoFromFetchResponse(source, &manifest)) {
 236     for (int i = 0; i < manifest.resource_size(); ++i) {
 237       if (manifest.resource(i).has_url()) {
 238         resource_urls_to_fetch_.push_back(GURL(manifest.resource(i).url()));
 239       }
 240     }
 241   }
 242
 243   StartNextFetch();
 244 }
 245
 246 void PrecacheFetcher::OnResourceFetchComplete(const URLFetcher& source) {
 247   // The resource has already been put in the cache during the fetch process, so
 248   // nothing more needs to be done for the resource.
 249   StartNextFetch();
 250 }
 251
 252 }  // namespace precache