components/precache/core/precache_fetcher.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/precache/core/precache_fetcher.h"
   6
   7 #include <string>
   8 #include <vector>
   9
  10 #include "base/bind.h"
  11 #include "base/callback.h"
  12 #include "base/command_line.h"
  13 #include "base/compiler_specific.h"
  14 #include "base/containers/hash_tables.h"
  15 #include "components/precache/core/precache_switches.h"
  16 #include "components/precache/core/proto/precache.pb.h"
  17 #include "net/base/escape.h"
  18 #include "net/base/load_flags.h"
  19 #include "net/url_request/url_fetcher.h"
  20 #include "net/url_request/url_fetcher_delegate.h"
  21 #include "net/url_request/url_request_context_getter.h"
  22 #include "net/url_request/url_request_status.h"
  23
  24 using net::URLFetcher;
  25
  26 namespace precache {
  27
  28 namespace {
  29
  30 GURL GetConfigURL() {
  31   const base::CommandLine& command_line =
  32       *base::CommandLine::ForCurrentProcess();
  33   if (command_line.HasSwitch(switches::kPrecacheConfigSettingsURL)) {
  34     return GURL(
  35         command_line.GetSwitchValueASCII(switches::kPrecacheConfigSettingsURL));
  36   }
  37
  38 #if defined(PRECACHE_CONFIG_SETTINGS_URL)
  39   return GURL(PRECACHE_CONFIG_SETTINGS_URL);
  40 #else
  41   // The precache config settings URL could not be determined, so return an
  42   // empty, invalid GURL.
  43   return GURL();
  44 #endif
  45 }
  46
  47 std::string GetDefaultManifestURLPrefix() {
  48   const base::CommandLine& command_line =
  49       *base::CommandLine::ForCurrentProcess();
  50   if (command_line.HasSwitch(switches::kPrecacheManifestURLPrefix)) {
  51     return command_line.GetSwitchValueASCII(
  52         switches::kPrecacheManifestURLPrefix);
  53   }
  54
  55 #if defined(PRECACHE_MANIFEST_URL_PREFIX)
  56   return PRECACHE_MANIFEST_URL_PREFIX;
  57 #else
  58   // The precache manifest URL prefix could not be determined, so return an
  59   // empty string.
  60   return std::string();
  61 #endif
  62 }
  63
  64 // Construct the URL of the precache manifest for the given name (either host or
  65 // URL). The server is expecting a request for a URL consisting of the manifest
  66 // URL prefix followed by the doubly escaped name.
  67 std::string ConstructManifestURL(const std::string& prefix,
  68                                  const std::string& name) {
  69   return prefix + net::EscapeQueryParamValue(
  70                       net::EscapeQueryParamValue(name, false), false);
  71 }
  72
  73 // Attempts to parse a protobuf message from the response string of a
  74 // URLFetcher. If parsing is successful, the message parameter will contain the
  75 // parsed protobuf and this function will return true. Otherwise, returns false.
  76 bool ParseProtoFromFetchResponse(const URLFetcher& source,
  77                                  ::google::protobuf::MessageLite* message) {
  78   std::string response_string;
  79
  80   if (!source.GetStatus().is_success()) {
  81     DLOG(WARNING) << "Fetch failed: " << source.GetOriginalURL().spec();
  82     return false;
  83   }
  84   if (!source.GetResponseAsString(&response_string)) {
  85     DLOG(WARNING) << "No response string present: "
  86                   << source.GetOriginalURL().spec();
  87     return false;
  88   }
  89   if (!message->ParseFromString(response_string)) {
  90     DLOG(WARNING) << "Unable to parse proto served from "
  91                   << source.GetOriginalURL().spec();
  92     return false;
  93   }
  94   return true;
  95 }
  96
  97 }  // namespace
  98
  99 // Class that fetches a URL, and runs the specified callback when the fetch is
 100 // complete. This class exists so that a different method can be run in
 101 // response to different kinds of fetches, e.g. OnConfigFetchComplete when
 102 // configuration settings are fetched, OnManifestFetchComplete when a manifest
 103 // is fetched, etc.
 104 class PrecacheFetcher::Fetcher : public net::URLFetcherDelegate {
 105  public:
 106   // Construct a new Fetcher. This will create and start a new URLFetcher for
 107   // the specified URL using the specified request context.
 108   Fetcher(net::URLRequestContextGetter* request_context, const GURL& url,
 109           const base::Callback<void(const URLFetcher&)>& callback);
 110   ~Fetcher() override {}
 111   void OnURLFetchComplete(const URLFetcher* source) override;
 112
 113  private:
 114   const base::Callback<void(const URLFetcher&)> callback_;
 115   scoped_ptr<URLFetcher> url_fetcher_;
 116
 117   DISALLOW_COPY_AND_ASSIGN(Fetcher);
 118 };
 119
 120 PrecacheFetcher::Fetcher::Fetcher(
 121     net::URLRequestContextGetter* request_context, const GURL& url,
 122     const base::Callback<void(const URLFetcher&)>& callback)
 123     : callback_(callback) {
 124   url_fetcher_ = URLFetcher::Create(url, URLFetcher::GET, this);
 125   url_fetcher_->SetRequestContext(request_context);
 126   url_fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES |
 127                              net::LOAD_DO_NOT_SEND_COOKIES);
 128   url_fetcher_->Start();
 129 }
 130
 131 void PrecacheFetcher::Fetcher::OnURLFetchComplete(const URLFetcher* source) {
 132   callback_.Run(*source);
 133 }
 134
 135 PrecacheFetcher::PrecacheFetcher(
 136     const std::vector<std::string>& starting_hosts,
 137     net::URLRequestContextGetter* request_context,
 138     const std::string& manifest_url_prefix,
 139     PrecacheFetcher::PrecacheDelegate* precache_delegate)
 140     : starting_hosts_(starting_hosts),
 141       request_context_(request_context),
 142       manifest_url_prefix_(manifest_url_prefix),
 143       precache_delegate_(precache_delegate) {
 144   DCHECK(request_context_.get());  // Request context must be non-NULL.
 145   DCHECK(precache_delegate_);  // Precache delegate must be non-NULL.
 146
 147   DCHECK_NE(GURL(), GetConfigURL())
 148       << "Could not determine the precache config settings URL.";
 149   DCHECK_NE(std::string(), GetDefaultManifestURLPrefix())
 150       << "Could not determine the default precache manifest URL prefix.";
 151 }
 152
 153 PrecacheFetcher::~PrecacheFetcher() {
 154 }
 155
 156 void PrecacheFetcher::Start() {
 157   DCHECK(!fetcher_);  // Start shouldn't be called repeatedly.
 158
 159   GURL config_url = GetConfigURL();
 160   DCHECK(config_url.is_valid());
 161
 162   // Fetch the precache configuration settings from the server.
 163   fetcher_.reset(new Fetcher(request_context_.get(),
 164                              config_url,
 165                              base::Bind(&PrecacheFetcher::OnConfigFetchComplete,
 166                                         base::Unretained(this))));
 167 }
 168
 169 void PrecacheFetcher::StartNextFetch() {
 170   if (!resource_urls_to_fetch_.empty()) {
 171     // Fetch the next resource URL.
 172     fetcher_.reset(
 173         new Fetcher(request_context_.get(),
 174                     resource_urls_to_fetch_.front(),
 175                     base::Bind(&PrecacheFetcher::OnResourceFetchComplete,
 176                                base::Unretained(this))));
 177
 178     resource_urls_to_fetch_.pop_front();
 179     return;
 180   }
 181
 182   if (!manifest_urls_to_fetch_.empty()) {
 183     // Fetch the next manifest URL.
 184     fetcher_.reset(
 185         new Fetcher(request_context_.get(),
 186                     manifest_urls_to_fetch_.front(),
 187                     base::Bind(&PrecacheFetcher::OnManifestFetchComplete,
 188                                base::Unretained(this))));
 189
 190     manifest_urls_to_fetch_.pop_front();
 191     return;
 192   }
 193
 194   // There are no more URLs to fetch, so end the precache cycle.
 195   precache_delegate_->OnDone();
 196   // OnDone may have deleted this PrecacheFetcher, so don't do anything after it
 197   // is called.
 198 }
 199
 200 void PrecacheFetcher::OnConfigFetchComplete(const URLFetcher& source) {
 201   // Attempt to parse the config proto. On failure, continue on with the default
 202   // configuration.
 203   PrecacheConfigurationSettings config;
 204   ParseProtoFromFetchResponse(source, &config);
 205
 206   std::string prefix = manifest_url_prefix_.empty()
 207                            ? GetDefaultManifestURLPrefix()
 208                            : manifest_url_prefix_;
 209   DCHECK_NE(std::string(), prefix)
 210       << "Could not determine the precache manifest URL prefix.";
 211
 212   // Keep track of manifest URLs that are being fetched, in order to remove
 213   // duplicates.
 214   base::hash_set<std::string> unique_manifest_urls;
 215
 216   // Attempt to fetch manifests for starting hosts up to the maximum top sites
 217   // count. If a manifest does not exist for a particular starting host, then
 218   // the fetch will fail, and that starting host will be ignored.
 219   int64 rank = 0;
 220   for (const std::string& host : starting_hosts_) {
 221     ++rank;
 222     if (rank > config.top_sites_count())
 223       break;
 224     unique_manifest_urls.insert(ConstructManifestURL(prefix, host));
 225   }
 226
 227   for (const std::string& url : config.forced_site())
 228     unique_manifest_urls.insert(ConstructManifestURL(prefix, url));
 229
 230   for (const std::string& manifest_url : unique_manifest_urls)
 231     manifest_urls_to_fetch_.push_back(GURL(manifest_url));
 232
 233   StartNextFetch();
 234 }
 235
 236 void PrecacheFetcher::OnManifestFetchComplete(const URLFetcher& source) {
 237   PrecacheManifest manifest;
 238
 239   if (ParseProtoFromFetchResponse(source, &manifest)) {
 240     for (int i = 0; i < manifest.resource_size(); ++i) {
 241       if (manifest.resource(i).has_url()) {
 242         resource_urls_to_fetch_.push_back(GURL(manifest.resource(i).url()));
 243       }
 244     }
 245   }
 246
 247   StartNextFetch();
 248 }
 249
 250 void PrecacheFetcher::OnResourceFetchComplete(const URLFetcher& source) {
 251   // The resource has already been put in the cache during the fetch process, so
 252   // nothing more needs to be done for the resource.
 253   StartNextFetch();
 254 }
 255
 256 }  // namespace precache