components/precache/core/precache_fetcher.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/precache/core/precache_fetcher.h"
   6
   7 #include <string>
   8 #include <vector>
   9
  10 #include "base/bind.h"
  11 #include "base/callback.h"
  12 #include "base/command_line.h"
  13 #include "base/compiler_specific.h"
  14 #include "base/containers/hash_tables.h"
  15 #include "components/precache/core/precache_switches.h"
  16 #include "components/precache/core/proto/precache.pb.h"
  17 #include "net/base/escape.h"
  18 #include "net/base/load_flags.h"
  19 #include "net/url_request/url_fetcher.h"
  20 #include "net/url_request/url_fetcher_delegate.h"
  21 #include "net/url_request/url_request_context_getter.h"
  22 #include "net/url_request/url_request_status.h"
  23
  24 using net::URLFetcher;
  25
  26 namespace precache {
  27
  28 namespace {
  29
  30 GURL GetConfigURL() {
  31   const base::CommandLine& command_line =
  32       *base::CommandLine::ForCurrentProcess();
  33   if (command_line.HasSwitch(switches::kPrecacheConfigSettingsURL)) {
  34     return GURL(
  35         command_line.GetSwitchValueASCII(switches::kPrecacheConfigSettingsURL));
  36   }
  37
  38 #if defined(PRECACHE_CONFIG_SETTINGS_URL)
  39   return GURL(PRECACHE_CONFIG_SETTINGS_URL);
  40 #else
  41   // The precache config settings URL could not be determined, so return an
  42   // empty, invalid GURL.
  43   return GURL();
  44 #endif
  45 }
  46
  47 std::string GetManifestURLPrefix() {
  48   const base::CommandLine& command_line =
  49       *base::CommandLine::ForCurrentProcess();
  50   if (command_line.HasSwitch(switches::kPrecacheManifestURLPrefix)) {
  51     return command_line.GetSwitchValueASCII(
  52         switches::kPrecacheManifestURLPrefix);
  53   }
  54
  55 #if defined(PRECACHE_MANIFEST_URL_PREFIX)
  56   return PRECACHE_MANIFEST_URL_PREFIX;
  57 #else
  58   // The precache manifest URL prefix could not be determined, so return an
  59   // empty string.
  60   return std::string();
  61 #endif
  62 }
  63
  64 // Construct the URL of the precache manifest for the given name (either host or
  65 // URL). The server is expecting a request for a URL consisting of the manifest
  66 // URL prefix followed by the doubly escaped name.
  67 std::string ConstructManifestURL(const std::string& name) {
  68   return GetManifestURLPrefix() +
  69          net::EscapeQueryParamValue(net::EscapeQueryParamValue(name, false),
  70                                     false);
  71 }
  72
  73 // Attempts to parse a protobuf message from the response string of a
  74 // URLFetcher. If parsing is successful, the message parameter will contain the
  75 // parsed protobuf and this function will return true. Otherwise, returns false.
  76 bool ParseProtoFromFetchResponse(const URLFetcher& source,
  77                                  ::google::protobuf::MessageLite* message) {
  78   std::string response_string;
  79
  80   if (!source.GetStatus().is_success()) {
  81     DLOG(WARNING) << "Fetch failed: " << source.GetOriginalURL().spec();
  82     return false;
  83   }
  84   if (!source.GetResponseAsString(&response_string)) {
  85     DLOG(WARNING) << "No response string present: "
  86                   << source.GetOriginalURL().spec();
  87     return false;
  88   }
  89   if (!message->ParseFromString(response_string)) {
  90     DLOG(WARNING) << "Unable to parse proto served from "
  91                   << source.GetOriginalURL().spec();
  92     return false;
  93   }
  94   return true;
  95 }
  96
  97 }  // namespace
  98
  99 // Class that fetches a URL, and runs the specified callback when the fetch is
 100 // complete. This class exists so that a different method can be run in
 101 // response to different kinds of fetches, e.g. OnConfigFetchComplete when
 102 // configuration settings are fetched, OnManifestFetchComplete when a manifest
 103 // is fetched, etc.
 104 class PrecacheFetcher::Fetcher : public net::URLFetcherDelegate {
 105  public:
 106   // Construct a new Fetcher. This will create and start a new URLFetcher for
 107   // the specified URL using the specified request context.
 108   Fetcher(net::URLRequestContextGetter* request_context, const GURL& url,
 109           const base::Callback<void(const URLFetcher&)>& callback);
 110   ~Fetcher() override {}
 111   void OnURLFetchComplete(const URLFetcher* source) override;
 112
 113  private:
 114   const base::Callback<void(const URLFetcher&)> callback_;
 115   scoped_ptr<URLFetcher> url_fetcher_;
 116
 117   DISALLOW_COPY_AND_ASSIGN(Fetcher);
 118 };
 119
 120 PrecacheFetcher::Fetcher::Fetcher(
 121     net::URLRequestContextGetter* request_context, const GURL& url,
 122     const base::Callback<void(const URLFetcher&)>& callback)
 123     : callback_(callback) {
 124   url_fetcher_ = URLFetcher::Create(url, URLFetcher::GET, this);
 125   url_fetcher_->SetRequestContext(request_context);
 126   url_fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES |
 127                              net::LOAD_DO_NOT_SEND_COOKIES);
 128   url_fetcher_->Start();
 129 }
 130
 131 void PrecacheFetcher::Fetcher::OnURLFetchComplete(const URLFetcher* source) {
 132   callback_.Run(*source);
 133 }
 134
 135 PrecacheFetcher::PrecacheFetcher(
 136     const std::vector<std::string>& starting_hosts,
 137     net::URLRequestContextGetter* request_context,
 138     PrecacheFetcher::PrecacheDelegate* precache_delegate)
 139     : starting_hosts_(starting_hosts),
 140       request_context_(request_context),
 141       precache_delegate_(precache_delegate) {
 142   DCHECK(request_context_.get());  // Request context must be non-NULL.
 143   DCHECK(precache_delegate_);  // Precache delegate must be non-NULL.
 144
 145   DCHECK_NE(GURL(), GetConfigURL())
 146       << "Could not determine the precache config settings URL.";
 147   DCHECK_NE(std::string(), GetManifestURLPrefix())
 148       << "Could not determine the precache manifest URL prefix.";
 149 }
 150
 151 PrecacheFetcher::~PrecacheFetcher() {
 152 }
 153
 154 void PrecacheFetcher::Start() {
 155   DCHECK(!fetcher_);  // Start shouldn't be called repeatedly.
 156
 157   GURL config_url = GetConfigURL();
 158   DCHECK(config_url.is_valid());
 159
 160   // Fetch the precache configuration settings from the server.
 161   fetcher_.reset(new Fetcher(request_context_.get(),
 162                              config_url,
 163                              base::Bind(&PrecacheFetcher::OnConfigFetchComplete,
 164                                         base::Unretained(this))));
 165 }
 166
 167 void PrecacheFetcher::StartNextFetch() {
 168   if (!resource_urls_to_fetch_.empty()) {
 169     // Fetch the next resource URL.
 170     fetcher_.reset(
 171         new Fetcher(request_context_.get(),
 172                     resource_urls_to_fetch_.front(),
 173                     base::Bind(&PrecacheFetcher::OnResourceFetchComplete,
 174                                base::Unretained(this))));
 175
 176     resource_urls_to_fetch_.pop_front();
 177     return;
 178   }
 179
 180   if (!manifest_urls_to_fetch_.empty()) {
 181     // Fetch the next manifest URL.
 182     fetcher_.reset(
 183         new Fetcher(request_context_.get(),
 184                     manifest_urls_to_fetch_.front(),
 185                     base::Bind(&PrecacheFetcher::OnManifestFetchComplete,
 186                                base::Unretained(this))));
 187
 188     manifest_urls_to_fetch_.pop_front();
 189     return;
 190   }
 191
 192   // There are no more URLs to fetch, so end the precache cycle.
 193   precache_delegate_->OnDone();
 194   // OnDone may have deleted this PrecacheFetcher, so don't do anything after it
 195   // is called.
 196 }
 197
 198 void PrecacheFetcher::OnConfigFetchComplete(const URLFetcher& source) {
 199   // Attempt to parse the config proto. On failure, continue on with the default
 200   // configuration.
 201   PrecacheConfigurationSettings config;
 202   ParseProtoFromFetchResponse(source, &config);
 203
 204   // Keep track of manifest URLs that are being fetched, in order to remove
 205   // duplicates.
 206   base::hash_set<std::string> unique_manifest_urls;
 207
 208   // Attempt to fetch manifests for starting hosts up to the maximum top sites
 209   // count. If a manifest does not exist for a particular starting host, then
 210   // the fetch will fail, and that starting host will be ignored.
 211   int64 rank = 0;
 212   for (const std::string& host : starting_hosts_) {
 213     ++rank;
 214     if (rank > config.top_sites_count())
 215       break;
 216     unique_manifest_urls.insert(ConstructManifestURL(host));
 217   }
 218
 219   for (const std::string& url : config.forced_site())
 220     unique_manifest_urls.insert(ConstructManifestURL(url));
 221
 222   for (const std::string& manifest_url : unique_manifest_urls)
 223     manifest_urls_to_fetch_.push_back(GURL(manifest_url));
 224
 225   StartNextFetch();
 226 }
 227
 228 void PrecacheFetcher::OnManifestFetchComplete(const URLFetcher& source) {
 229   PrecacheManifest manifest;
 230
 231   if (ParseProtoFromFetchResponse(source, &manifest)) {
 232     for (int i = 0; i < manifest.resource_size(); ++i) {
 233       if (manifest.resource(i).has_url()) {
 234         resource_urls_to_fetch_.push_back(GURL(manifest.resource(i).url()));
 235       }
 236     }
 237   }
 238
 239   StartNextFetch();
 240 }
 241
 242 void PrecacheFetcher::OnResourceFetchComplete(const URLFetcher& source) {
 243   // The resource has already been put in the cache during the fetch process, so
 244   // nothing more needs to be done for the resource.
 245   StartNextFetch();
 246 }
 247
 248 }  // namespace precache