components/precache/core/precache_fetcher.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/precache/core/precache_fetcher.h"
   6
   7 #include <string>
   8 #include <vector>
   9
  10 #include "base/bind.h"
  11 #include "base/callback.h"
  12 #include "base/command_line.h"
  13 #include "base/compiler_specific.h"
  14 #include "base/containers/hash_tables.h"
  15 #include "base/metrics/histogram_macros.h"
  16 #include "components/precache/core/precache_switches.h"
  17 #include "components/precache/core/proto/precache.pb.h"
  18 #include "net/base/completion_callback.h"
  19 #include "net/base/escape.h"
  20 #include "net/base/io_buffer.h"
  21 #include "net/base/load_flags.h"
  22 #include "net/base/net_errors.h"
  23 #include "net/url_request/url_fetcher.h"
  24 #include "net/url_request/url_fetcher_delegate.h"
  25 #include "net/url_request/url_fetcher_response_writer.h"
  26 #include "net/url_request/url_request_context_getter.h"
  27 #include "net/url_request/url_request_status.h"
  28
  29 using net::URLFetcher;
  30
  31 namespace precache {
  32
  33 namespace {
  34
  35 // The maximum for the Precache.Fetch.ResponseBytes histogram. We set this to a
  36 // number we expect to be in the 99th percentile for the histogram, give or
  37 // take.
  38 const int kMaxResponseBytes = 100 * 1024 * 1024;
  39
  40 GURL GetConfigURL() {
  41   const base::CommandLine& command_line =
  42       *base::CommandLine::ForCurrentProcess();
  43   if (command_line.HasSwitch(switches::kPrecacheConfigSettingsURL)) {
  44     return GURL(
  45         command_line.GetSwitchValueASCII(switches::kPrecacheConfigSettingsURL));
  46   }
  47
  48 #if defined(PRECACHE_CONFIG_SETTINGS_URL)
  49   return GURL(PRECACHE_CONFIG_SETTINGS_URL);
  50 #else
  51   // The precache config settings URL could not be determined, so return an
  52   // empty, invalid GURL.
  53   return GURL();
  54 #endif
  55 }
  56
  57 std::string GetDefaultManifestURLPrefix() {
  58   const base::CommandLine& command_line =
  59       *base::CommandLine::ForCurrentProcess();
  60   if (command_line.HasSwitch(switches::kPrecacheManifestURLPrefix)) {
  61     return command_line.GetSwitchValueASCII(
  62         switches::kPrecacheManifestURLPrefix);
  63   }
  64
  65 #if defined(PRECACHE_MANIFEST_URL_PREFIX)
  66   return PRECACHE_MANIFEST_URL_PREFIX;
  67 #else
  68   // The precache manifest URL prefix could not be determined, so return an
  69   // empty string.
  70   return std::string();
  71 #endif
  72 }
  73
  74 // Construct the URL of the precache manifest for the given name (either host or
  75 // URL). The server is expecting a request for a URL consisting of the manifest
  76 // URL prefix followed by the doubly escaped name.
  77 std::string ConstructManifestURL(const std::string& prefix,
  78                                  const std::string& name) {
  79   return prefix + net::EscapeQueryParamValue(
  80                       net::EscapeQueryParamValue(name, false), false);
  81 }
  82
  83 // Attempts to parse a protobuf message from the response string of a
  84 // URLFetcher. If parsing is successful, the message parameter will contain the
  85 // parsed protobuf and this function will return true. Otherwise, returns false.
  86 bool ParseProtoFromFetchResponse(const URLFetcher& source,
  87                                  ::google::protobuf::MessageLite* message) {
  88   std::string response_string;
  89
  90   if (!source.GetStatus().is_success()) {
  91     DLOG(WARNING) << "Fetch failed: " << source.GetOriginalURL().spec();
  92     return false;
  93   }
  94   if (!source.GetResponseAsString(&response_string)) {
  95     DLOG(WARNING) << "No response string present: "
  96                   << source.GetOriginalURL().spec();
  97     return false;
  98   }
  99   if (!message->ParseFromString(response_string)) {
 100     DLOG(WARNING) << "Unable to parse proto served from "
 101                   << source.GetOriginalURL().spec();
 102     return false;
 103   }
 104   return true;
 105 }
 106
 107 // URLFetcherResponseWriter that ignores the response body, in order to avoid
 108 // the unnecessary memory usage. Use it rather than the default if you don't
 109 // care about parsing the response body. We use it below as a means to populate
 110 // the cache with requested resource URLs.
 111 class URLFetcherNullWriter : public net::URLFetcherResponseWriter {
 112  public:
 113   int Initialize(const net::CompletionCallback& callback) override {
 114     return net::OK;
 115   }
 116
 117   int Write(net::IOBuffer* buffer,
 118             int num_bytes,
 119             const net::CompletionCallback& callback) override {
 120     return num_bytes;
 121   }
 122
 123   int Finish(const net::CompletionCallback& callback) override {
 124     return net::OK;
 125   }
 126 };
 127
 128 }  // namespace
 129
 130 // Class that fetches a URL, and runs the specified callback when the fetch is
 131 // complete. This class exists so that a different method can be run in
 132 // response to different kinds of fetches, e.g. OnConfigFetchComplete when
 133 // configuration settings are fetched, OnManifestFetchComplete when a manifest
 134 // is fetched, etc.
 135 class PrecacheFetcher::Fetcher : public net::URLFetcherDelegate {
 136  public:
 137   // Construct a new Fetcher. This will create and start a new URLFetcher for
 138   // the specified URL using the specified request context.
 139   Fetcher(net::URLRequestContextGetter* request_context,
 140           const GURL& url,
 141           const base::Callback<void(const URLFetcher&)>& callback,
 142           bool ignore_response_body);
 143   ~Fetcher() override {}
 144   void OnURLFetchDownloadProgress(const URLFetcher* source,
 145                                   int64 current,
 146                                   int64 total) override;
 147   void OnURLFetchComplete(const URLFetcher* source) override;
 148   int response_bytes() { return response_bytes_; }
 149
 150  private:
 151   const base::Callback<void(const URLFetcher&)> callback_;
 152   scoped_ptr<URLFetcher> url_fetcher_;
 153   int response_bytes_;
 154
 155   DISALLOW_COPY_AND_ASSIGN(Fetcher);
 156 };
 157
 158 PrecacheFetcher::Fetcher::Fetcher(
 159     net::URLRequestContextGetter* request_context,
 160     const GURL& url,
 161     const base::Callback<void(const URLFetcher&)>& callback,
 162     bool ignore_response_body)
 163     : callback_(callback), response_bytes_(0) {
 164   url_fetcher_ = URLFetcher::Create(url, URLFetcher::GET, this);
 165   url_fetcher_->SetRequestContext(request_context);
 166   url_fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES |
 167                              net::LOAD_DO_NOT_SEND_COOKIES);
 168   if (ignore_response_body) {
 169     scoped_ptr<URLFetcherNullWriter> null_writer(new URLFetcherNullWriter);
 170     url_fetcher_->SaveResponseWithWriter(null_writer.Pass());
 171   }
 172   url_fetcher_->Start();
 173 }
 174
 175 void PrecacheFetcher::Fetcher::OnURLFetchDownloadProgress(
 176     const URLFetcher* source,
 177     int64 current,
 178     int64 total) {
 179   response_bytes_ = current;
 180 }
 181
 182 void PrecacheFetcher::Fetcher::OnURLFetchComplete(const URLFetcher* source) {
 183   callback_.Run(*source);
 184 }
 185
 186 PrecacheFetcher::PrecacheFetcher(
 187     const std::vector<std::string>& starting_hosts,
 188     net::URLRequestContextGetter* request_context,
 189     const std::string& manifest_url_prefix,
 190     PrecacheFetcher::PrecacheDelegate* precache_delegate)
 191     : starting_hosts_(starting_hosts),
 192       request_context_(request_context),
 193       manifest_url_prefix_(manifest_url_prefix),
 194       precache_delegate_(precache_delegate),
 195       total_response_bytes_(0),
 196       num_manifest_urls_to_fetch_(0) {
 197   DCHECK(request_context_.get());  // Request context must be non-NULL.
 198   DCHECK(precache_delegate_);  // Precache delegate must be non-NULL.
 199
 200   DCHECK_NE(GURL(), GetConfigURL())
 201       << "Could not determine the precache config settings URL.";
 202   DCHECK_NE(std::string(), GetDefaultManifestURLPrefix())
 203       << "Could not determine the default precache manifest URL prefix.";
 204 }
 205
 206 PrecacheFetcher::~PrecacheFetcher() {
 207   // Number of manifests for which we have downloaded all resources.
 208   int manifests_completed =
 209       num_manifest_urls_to_fetch_ - manifest_urls_to_fetch_.size();
 210
 211   // If there are resource URLs left to fetch, the last manifest is not yet
 212   // completed.
 213   if (!resource_urls_to_fetch_.empty())
 214     --manifests_completed;
 215
 216   DCHECK_GE(manifests_completed, 0);
 217   int percent_completed = num_manifest_urls_to_fetch_ == 0
 218                               ? 0
 219                               : (static_cast<double>(manifests_completed) /
 220                                  num_manifest_urls_to_fetch_ * 100);
 221   UMA_HISTOGRAM_PERCENTAGE("Precache.Fetch.PercentCompleted",
 222                            percent_completed);
 223   UMA_HISTOGRAM_CUSTOM_COUNTS("Precache.Fetch.ResponseBytes",
 224                               total_response_bytes_, 1, kMaxResponseBytes, 50);
 225 }
 226
 227 void PrecacheFetcher::Start() {
 228   DCHECK(!fetcher_);  // Start shouldn't be called repeatedly.
 229
 230   GURL config_url = GetConfigURL();
 231   DCHECK(config_url.is_valid());
 232
 233   // Fetch the precache configuration settings from the server.
 234   fetcher_.reset(new Fetcher(request_context_.get(), config_url,
 235                              base::Bind(&PrecacheFetcher::OnConfigFetchComplete,
 236                                         base::Unretained(this)),
 237                              false /* ignore_response_body */));
 238 }
 239
 240 void PrecacheFetcher::StartNextFetch() {
 241   total_response_bytes_ += fetcher_->response_bytes();
 242
 243   if (!resource_urls_to_fetch_.empty()) {
 244     // Fetch the next resource URL.
 245     fetcher_.reset(
 246         new Fetcher(request_context_.get(), resource_urls_to_fetch_.front(),
 247                     base::Bind(&PrecacheFetcher::OnResourceFetchComplete,
 248                                base::Unretained(this)),
 249                     true /* ignore_response_body */));
 250
 251     resource_urls_to_fetch_.pop_front();
 252     return;
 253   }
 254
 255   if (!manifest_urls_to_fetch_.empty()) {
 256     // Fetch the next manifest URL.
 257     fetcher_.reset(
 258         new Fetcher(request_context_.get(), manifest_urls_to_fetch_.front(),
 259                     base::Bind(&PrecacheFetcher::OnManifestFetchComplete,
 260                                base::Unretained(this)),
 261                     false /* ignore_response_body */));
 262
 263     manifest_urls_to_fetch_.pop_front();
 264     return;
 265   }
 266
 267   // There are no more URLs to fetch, so end the precache cycle.
 268   precache_delegate_->OnDone();
 269   // OnDone may have deleted this PrecacheFetcher, so don't do anything after it
 270   // is called.
 271 }
 272
 273 void PrecacheFetcher::OnConfigFetchComplete(const URLFetcher& source) {
 274   // Attempt to parse the config proto. On failure, continue on with the default
 275   // configuration.
 276   PrecacheConfigurationSettings config;
 277   ParseProtoFromFetchResponse(source, &config);
 278
 279   std::string prefix = manifest_url_prefix_.empty()
 280                            ? GetDefaultManifestURLPrefix()
 281                            : manifest_url_prefix_;
 282   DCHECK_NE(std::string(), prefix)
 283       << "Could not determine the precache manifest URL prefix.";
 284
 285   // Keep track of manifest URLs that are being fetched, in order to remove
 286   // duplicates.
 287   base::hash_set<std::string> unique_manifest_urls;
 288
 289   // Attempt to fetch manifests for starting hosts up to the maximum top sites
 290   // count. If a manifest does not exist for a particular starting host, then
 291   // the fetch will fail, and that starting host will be ignored.
 292   int64 rank = 0;
 293   for (const std::string& host : starting_hosts_) {
 294     ++rank;
 295     if (rank > config.top_sites_count())
 296       break;
 297     unique_manifest_urls.insert(ConstructManifestURL(prefix, host));
 298   }
 299
 300   for (const std::string& url : config.forced_site())
 301     unique_manifest_urls.insert(ConstructManifestURL(prefix, url));
 302
 303   for (const std::string& manifest_url : unique_manifest_urls)
 304     manifest_urls_to_fetch_.push_back(GURL(manifest_url));
 305   num_manifest_urls_to_fetch_ = manifest_urls_to_fetch_.size();
 306
 307   StartNextFetch();
 308 }
 309
 310 void PrecacheFetcher::OnManifestFetchComplete(const URLFetcher& source) {
 311   PrecacheManifest manifest;
 312
 313   if (ParseProtoFromFetchResponse(source, &manifest)) {
 314     for (int i = 0; i < manifest.resource_size(); ++i) {
 315       if (manifest.resource(i).has_url()) {
 316         resource_urls_to_fetch_.push_back(GURL(manifest.resource(i).url()));
 317       }
 318     }
 319   }
 320
 321   StartNextFetch();
 322 }
 323
 324 void PrecacheFetcher::OnResourceFetchComplete(const URLFetcher& source) {
 325   // The resource has already been put in the cache during the fetch process, so
 326   // nothing more needs to be done for the resource.
 327   StartNextFetch();
 328 }
 329
 330 }  // namespace precache