components/precache/core/precache_fetcher.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/precache/core/precache_fetcher.h"
   6
   7 #include <string>
   8
   9 #include "base/bind.h"
  10 #include "base/callback.h"
  11 #include "base/command_line.h"
  12 #include "base/compiler_specific.h"
  13 #include "base/containers/hash_tables.h"
  14 #include "components/precache/core/precache_switches.h"
  15 #include "components/precache/core/proto/precache.pb.h"
  16 #include "net/base/escape.h"
  17 #include "net/base/load_flags.h"
  18 #include "net/url_request/url_fetcher.h"
  19 #include "net/url_request/url_fetcher_delegate.h"
  20 #include "net/url_request/url_request_context_getter.h"
  21 #include "net/url_request/url_request_status.h"
  22
  23 using net::URLFetcher;
  24
  25 namespace precache {
  26
  27 namespace {
  28
  29 GURL GetConfigURL() {
  30   const base::CommandLine& command_line =
  31       *base::CommandLine::ForCurrentProcess();
  32   if (command_line.HasSwitch(switches::kPrecacheConfigSettingsURL)) {
  33     return GURL(
  34         command_line.GetSwitchValueASCII(switches::kPrecacheConfigSettingsURL));
  35   }
  36
  37 #if defined(PRECACHE_CONFIG_SETTINGS_URL)
  38   return GURL(PRECACHE_CONFIG_SETTINGS_URL);
  39 #else
  40   // The precache config settings URL could not be determined, so return an
  41   // empty, invalid GURL.
  42   return GURL();
  43 #endif
  44 }
  45
  46 std::string GetManifestURLPrefix() {
  47   const base::CommandLine& command_line =
  48       *base::CommandLine::ForCurrentProcess();
  49   if (command_line.HasSwitch(switches::kPrecacheManifestURLPrefix)) {
  50     return command_line.GetSwitchValueASCII(
  51         switches::kPrecacheManifestURLPrefix);
  52   }
  53
  54 #if defined(PRECACHE_MANIFEST_URL_PREFIX)
  55   return PRECACHE_MANIFEST_URL_PREFIX;
  56 #else
  57   // The precache manifest URL prefix could not be determined, so return an
  58   // empty string.
  59   return std::string();
  60 #endif
  61 }
  62
  63 // Construct the URL of the precache manifest for the given starting URL.
  64 // The server is expecting a request for a URL consisting of the manifest URL
  65 // prefix followed by the doubly escaped starting URL.
  66 GURL ConstructManifestURL(const GURL& starting_url) {
  67   return GURL(
  68       GetManifestURLPrefix() +
  69       net::EscapeQueryParamValue(
  70           net::EscapeQueryParamValue(starting_url.spec(), false), false));
  71 }
  72
  73 // Attempts to parse a protobuf message from the response string of a
  74 // URLFetcher. If parsing is successful, the message parameter will contain the
  75 // parsed protobuf and this function will return true. Otherwise, returns false.
  76 bool ParseProtoFromFetchResponse(const URLFetcher& source,
  77                                  ::google::protobuf::MessageLite* message) {
  78   std::string response_string;
  79
  80   if (!source.GetStatus().is_success()) {
  81     DLOG(WARNING) << "Fetch failed: " << source.GetOriginalURL().spec();
  82     return false;
  83   }
  84   if (!source.GetResponseAsString(&response_string)) {
  85     DLOG(WARNING) << "No response string present: "
  86                   << source.GetOriginalURL().spec();
  87     return false;
  88   }
  89   if (!message->ParseFromString(response_string)) {
  90     DLOG(WARNING) << "Unable to parse proto served from "
  91                   << source.GetOriginalURL().spec();
  92     return false;
  93   }
  94   return true;
  95 }
  96
  97 }  // namespace
  98
  99 // Class that fetches a URL, and runs the specified callback when the fetch is
 100 // complete. This class exists so that a different method can be run in
 101 // response to different kinds of fetches, e.g. OnConfigFetchComplete when
 102 // configuration settings are fetched, OnManifestFetchComplete when a manifest
 103 // is fetched, etc.
 104 class PrecacheFetcher::Fetcher : public net::URLFetcherDelegate {
 105  public:
 106   // Construct a new Fetcher. This will create and start a new URLFetcher for
 107   // the specified URL using the specified request context.
 108   Fetcher(net::URLRequestContextGetter* request_context, const GURL& url,
 109           const base::Callback<void(const URLFetcher&)>& callback);
 110   ~Fetcher() override {}
 111   void OnURLFetchComplete(const URLFetcher* source) override;
 112
 113  private:
 114   const base::Callback<void(const URLFetcher&)> callback_;
 115   scoped_ptr<URLFetcher> url_fetcher_;
 116
 117   DISALLOW_COPY_AND_ASSIGN(Fetcher);
 118 };
 119
 120 PrecacheFetcher::Fetcher::Fetcher(
 121     net::URLRequestContextGetter* request_context, const GURL& url,
 122     const base::Callback<void(const URLFetcher&)>& callback)
 123     : callback_(callback) {
 124   url_fetcher_.reset(URLFetcher::Create(url, URLFetcher::GET, this));
 125   url_fetcher_->SetRequestContext(request_context);
 126   url_fetcher_->Start();
 127 }
 128
 129 void PrecacheFetcher::Fetcher::OnURLFetchComplete(const URLFetcher* source) {
 130   callback_.Run(*source);
 131 }
 132
 133 PrecacheFetcher::PrecacheFetcher(
 134     const std::list<GURL>& starting_urls,
 135     net::URLRequestContextGetter* request_context,
 136     PrecacheFetcher::PrecacheDelegate* precache_delegate)
 137     : starting_urls_(starting_urls),
 138       request_context_(request_context),
 139       precache_delegate_(precache_delegate) {
 140   DCHECK(request_context_.get());  // Request context must be non-NULL.
 141   DCHECK(precache_delegate_);  // Precache delegate must be non-NULL.
 142
 143   DCHECK_NE(GURL(), GetConfigURL())
 144       << "Could not determine the precache config settings URL.";
 145   DCHECK_NE(std::string(), GetManifestURLPrefix())
 146       << "Could not determine the precache manifest URL prefix.";
 147 }
 148
 149 PrecacheFetcher::~PrecacheFetcher() {
 150 }
 151
 152 void PrecacheFetcher::Start() {
 153   DCHECK(!fetcher_);  // Start shouldn't be called repeatedly.
 154
 155   GURL config_url = GetConfigURL();
 156   DCHECK(config_url.is_valid());
 157
 158   // Fetch the precache configuration settings from the server.
 159   fetcher_.reset(new Fetcher(request_context_.get(),
 160                              config_url,
 161                              base::Bind(&PrecacheFetcher::OnConfigFetchComplete,
 162                                         base::Unretained(this))));
 163 }
 164
 165 void PrecacheFetcher::StartNextFetch() {
 166   if (!resource_urls_to_fetch_.empty()) {
 167     // Fetch the next resource URL.
 168     fetcher_.reset(
 169         new Fetcher(request_context_.get(),
 170                     resource_urls_to_fetch_.front(),
 171                     base::Bind(&PrecacheFetcher::OnResourceFetchComplete,
 172                                base::Unretained(this))));
 173
 174     resource_urls_to_fetch_.pop_front();
 175     return;
 176   }
 177
 178   if (!manifest_urls_to_fetch_.empty()) {
 179     // Fetch the next manifest URL.
 180     fetcher_.reset(
 181         new Fetcher(request_context_.get(),
 182                     manifest_urls_to_fetch_.front(),
 183                     base::Bind(&PrecacheFetcher::OnManifestFetchComplete,
 184                                base::Unretained(this))));
 185
 186     manifest_urls_to_fetch_.pop_front();
 187     return;
 188   }
 189
 190   // There are no more URLs to fetch, so end the precache cycle.
 191   precache_delegate_->OnDone();
 192   // OnDone may have deleted this PrecacheFetcher, so don't do anything after it
 193   // is called.
 194 }
 195
 196 void PrecacheFetcher::OnConfigFetchComplete(const URLFetcher& source) {
 197   PrecacheConfigurationSettings config;
 198
 199   if (ParseProtoFromFetchResponse(source, &config)) {
 200     // Keep track of starting URLs that manifests are being fetched for, in
 201     // order to remove duplicates. This is a hash set on strings, and not GURLs,
 202     // because there is no hash function defined for GURL.
 203     base::hash_set<std::string> unique_starting_urls;
 204
 205     // Attempt to fetch manifests for starting URLs up to the maximum top sites
 206     // count. If a manifest does not exist for a particular starting URL, then
 207     // the fetch will fail, and that starting URL will be ignored.
 208     int64 rank = 0;
 209     for (std::list<GURL>::const_iterator it = starting_urls_.begin();
 210          it != starting_urls_.end() && rank < config.top_sites_count();
 211          ++it, ++rank) {
 212       if (unique_starting_urls.find(it->spec()) == unique_starting_urls.end()) {
 213         // Only add a fetch for the manifest URL if this manifest isn't already
 214         // going to be fetched.
 215         manifest_urls_to_fetch_.push_back(ConstructManifestURL(*it));
 216         unique_starting_urls.insert(it->spec());
 217       }
 218     }
 219
 220     for (int i = 0; i < config.forced_starting_url_size(); ++i) {
 221       // Convert the string URL into a GURL and take the spec() of it so that
 222       // the URL string gets canonicalized.
 223       GURL url(config.forced_starting_url(i));
 224       if (unique_starting_urls.find(url.spec()) == unique_starting_urls.end()) {
 225         // Only add a fetch for the manifest URL if this manifest isn't already
 226         // going to be fetched.
 227         manifest_urls_to_fetch_.push_back(ConstructManifestURL(url));
 228         unique_starting_urls.insert(url.spec());
 229       }
 230     }
 231   }
 232
 233   StartNextFetch();
 234 }
 235
 236 void PrecacheFetcher::OnManifestFetchComplete(const URLFetcher& source) {
 237   PrecacheManifest manifest;
 238
 239   if (ParseProtoFromFetchResponse(source, &manifest)) {
 240     for (int i = 0; i < manifest.resource_size(); ++i) {
 241       if (manifest.resource(i).has_url()) {
 242         resource_urls_to_fetch_.push_back(GURL(manifest.resource(i).url()));
 243       }
 244     }
 245   }
 246
 247   StartNextFetch();
 248 }
 249
 250 void PrecacheFetcher::OnResourceFetchComplete(const URLFetcher& source) {
 251   // The resource has already been put in the cache during the fetch process, so
 252   // nothing more needs to be done for the resource.
 253   StartNextFetch();
 254 }
 255
 256 }  // namespace precache