components/suggestions/suggestions_service.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/suggestions/suggestions_service.h"
   6
   7 #include <string>
   8
   9 #include "base/memory/scoped_ptr.h"
  10 #include "base/message_loop/message_loop_proxy.h"
  11 #include "base/metrics/histogram.h"
  12 #include "base/metrics/sparse_histogram.h"
  13 #include "base/strings/string_number_conversions.h"
  14 #include "base/strings/string_util.h"
  15 #include "base/time/time.h"
  16 #include "components/pref_registry/pref_registry_syncable.h"
  17 #include "components/suggestions/blacklist_store.h"
  18 #include "components/suggestions/suggestions_store.h"
  19 #include "components/variations/net/variations_http_header_provider.h"
  20 #include "components/variations/variations_associated_data.h"
  21 #include "net/base/escape.h"
  22 #include "net/base/load_flags.h"
  23 #include "net/base/net_errors.h"
  24 #include "net/base/url_util.h"
  25 #include "net/http/http_response_headers.h"
  26 #include "net/http/http_status_code.h"
  27 #include "net/http/http_util.h"
  28 #include "net/url_request/url_fetcher.h"
  29 #include "net/url_request/url_request_status.h"
  30 #include "url/gurl.h"
  31
  32 using base::CancelableClosure;
  33
  34 namespace suggestions {
  35
  36 namespace {
  37
  38 // Used to UMA log the state of the last response from the server.
  39 enum SuggestionsResponseState {
  40   RESPONSE_EMPTY,
  41   RESPONSE_INVALID,
  42   RESPONSE_VALID,
  43   RESPONSE_STATE_SIZE
  44 };
  45
  46 // Will log the supplied response |state|.
  47 void LogResponseState(SuggestionsResponseState state) {
  48   UMA_HISTOGRAM_ENUMERATION("Suggestions.ResponseState", state,
  49                             RESPONSE_STATE_SIZE);
  50 }
  51
  52 // Obtains the experiment parameter under the supplied |key|, or empty string
  53 // if the parameter does not exist.
  54 std::string GetExperimentParam(const std::string& key) {
  55   return variations::GetVariationParamValue(kSuggestionsFieldTrialName, key);
  56 }
  57
  58 GURL BuildBlacklistRequestURL(const std::string& blacklist_url_prefix,
  59                               const GURL& candidate_url) {
  60   return GURL(blacklist_url_prefix +
  61               net::EscapeQueryParamValue(candidate_url.spec(), true));
  62 }
  63
  64 // Runs each callback in |requestors| on |suggestions|, then deallocates
  65 // |requestors|.
  66 void DispatchRequestsAndClear(
  67     const SuggestionsProfile& suggestions,
  68     std::vector<SuggestionsService::ResponseCallback>* requestors) {
  69   std::vector<SuggestionsService::ResponseCallback> temp_requestors;
  70   temp_requestors.swap(*requestors);
  71   std::vector<SuggestionsService::ResponseCallback>::iterator it;
  72   for (it = temp_requestors.begin(); it != temp_requestors.end(); ++it) {
  73     if (!it->is_null()) it->Run(suggestions);
  74   }
  75 }
  76
  77 // Default delay used when scheduling a blacklist request.
  78 const int kBlacklistDefaultDelaySec = 1;
  79
  80 // Multiplier on the delay used when scheduling a blacklist request, in case the
  81 // last observed request was unsuccessful.
  82 const int kBlacklistBackoffMultiplier = 2;
  83
  84 // Maximum valid delay for scheduling a request. Candidate delays larger than
  85 // this are rejected. This means the maximum backoff is at least 300 / 2, i.e.
  86 // 2.5 minutes.
  87 const int kBlacklistMaxDelaySec = 300;  // 5 minutes
  88
  89 }  // namespace
  90
  91 const char kSuggestionsFieldTrialName[] = "ChromeSuggestions";
  92 const char kSuggestionsFieldTrialControlParam[] = "control";
  93 const char kSuggestionsFieldTrialStateEnabled[] = "enabled";
  94
  95 // TODO(mathp): Put this in TemplateURL.
  96 const char kSuggestionsURL[] = "https://www.google.com/chromesuggestions?t=2";
  97 const char kSuggestionsBlacklistURLPrefix[] =
  98     "https://www.google.com/chromesuggestions/blacklist?t=2&url=";
  99 const char kSuggestionsBlacklistURLParam[] = "url";
 100
 101 // The default expiry timeout is 72 hours.
 102 const int64 kDefaultExpiryUsec = 72 * base::Time::kMicrosecondsPerHour;
 103
 104 SuggestionsService::SuggestionsService(
 105     net::URLRequestContextGetter* url_request_context,
 106     scoped_ptr<SuggestionsStore> suggestions_store,
 107     scoped_ptr<ImageManager> thumbnail_manager,
 108     scoped_ptr<BlacklistStore> blacklist_store)
 109     : url_request_context_(url_request_context),
 110       suggestions_store_(suggestions_store.Pass()),
 111       thumbnail_manager_(thumbnail_manager.Pass()),
 112       blacklist_store_(blacklist_store.Pass()),
 113       blacklist_delay_sec_(kBlacklistDefaultDelaySec),
 114       suggestions_url_(kSuggestionsURL),
 115       blacklist_url_prefix_(kSuggestionsBlacklistURLPrefix),
 116       weak_ptr_factory_(this) {}
 117
 118 SuggestionsService::~SuggestionsService() {}
 119
 120 // static
 121 bool SuggestionsService::IsControlGroup() {
 122   return GetExperimentParam(kSuggestionsFieldTrialControlParam) ==
 123          kSuggestionsFieldTrialStateEnabled;
 124 }
 125
 126 void SuggestionsService::FetchSuggestionsData(
 127     SyncState sync_state,
 128     SuggestionsService::ResponseCallback callback) {
 129   DCHECK(thread_checker_.CalledOnValidThread());
 130   waiting_requestors_.push_back(callback);
 131   if (sync_state == SYNC_OR_HISTORY_SYNC_DISABLED) {
 132     // Cancel any ongoing request, to stop interacting with the server.
 133     pending_request_.reset(NULL);
 134     suggestions_store_->ClearSuggestions();
 135     DispatchRequestsAndClear(SuggestionsProfile(), &waiting_requestors_);
 136   } else if (sync_state == INITIALIZED_ENABLED_HISTORY ||
 137              sync_state == NOT_INITIALIZED_ENABLED) {
 138     // Sync is enabled. Serve previously cached suggestions if available, else
 139     // an empty set of suggestions.
 140     ServeFromCache();
 141
 142     // Issue a network request to refresh the suggestions in the cache.
 143     IssueRequestIfNoneOngoing(suggestions_url_);
 144   } else {
 145     NOTREACHED();
 146   }
 147 }
 148
 149 void SuggestionsService::GetPageThumbnail(
 150     const GURL& url,
 151     base::Callback<void(const GURL&, const SkBitmap*)> callback) {
 152   thumbnail_manager_->GetImageForURL(url, callback);
 153 }
 154
 155 void SuggestionsService::BlacklistURL(
 156     const GURL& candidate_url,
 157     const SuggestionsService::ResponseCallback& callback) {
 158   DCHECK(thread_checker_.CalledOnValidThread());
 159   waiting_requestors_.push_back(callback);
 160
 161   // Blacklist locally for immediate effect and serve the requestors.
 162   blacklist_store_->BlacklistUrl(candidate_url);
 163   ServeFromCache();
 164
 165   // Send blacklisting request. Even if this request ends up not being sent
 166   // because of an ongoing request, a blacklist request is later scheduled.
 167   // TODO(mathp): Currently, this will not send a request if there is already
 168   // a request in flight (for suggestions or blacklist). Should we prioritize
 169   // blacklist requests since they actually carry a payload?
 170   IssueRequestIfNoneOngoing(
 171       BuildBlacklistRequestURL(blacklist_url_prefix_, candidate_url));
 172 }
 173
 174 // static
 175 bool SuggestionsService::GetBlacklistedUrl(const net::URLFetcher& request,
 176                                            GURL* url) {
 177   bool is_blacklist_request = StartsWithASCII(request.GetOriginalURL().spec(),
 178                                               kSuggestionsBlacklistURLPrefix,
 179                                               true);
 180   if (!is_blacklist_request) return false;
 181
 182   // Extract the blacklisted URL from the blacklist request.
 183   std::string blacklisted;
 184   if (!net::GetValueForKeyInQuery(
 185           request.GetOriginalURL(),
 186           kSuggestionsBlacklistURLParam,
 187           &blacklisted)) {
 188     return false;
 189   }
 190
 191   GURL blacklisted_url(blacklisted);
 192   blacklisted_url.Swap(url);
 193   return true;
 194 }
 195
 196 // static
 197 void SuggestionsService::RegisterProfilePrefs(
 198     user_prefs::PrefRegistrySyncable* registry) {
 199   SuggestionsStore::RegisterProfilePrefs(registry);
 200   BlacklistStore::RegisterProfilePrefs(registry);
 201 }
 202
 203 void SuggestionsService::SetDefaultExpiryTimestamp(
 204     SuggestionsProfile* suggestions, int64 default_timestamp_usec) {
 205   for (int i = 0; i < suggestions->suggestions_size(); ++i) {
 206     ChromeSuggestion* suggestion = suggestions->mutable_suggestions(i);
 207     // Do not set expiry if the server has already provided a more specific
 208     // expiry time for this suggestion.
 209     if (!suggestion->has_expiry_ts()) {
 210       suggestion->set_expiry_ts(default_timestamp_usec);
 211     }
 212   }
 213 }
 214
 215 void SuggestionsService::IssueRequestIfNoneOngoing(const GURL& url) {
 216   // If there is an ongoing request, let it complete.
 217   if (pending_request_.get()) {
 218     return;
 219   }
 220   pending_request_.reset(CreateSuggestionsRequest(url));
 221   pending_request_->Start();
 222   last_request_started_time_ = base::TimeTicks::Now();
 223 }
 224
 225 net::URLFetcher* SuggestionsService::CreateSuggestionsRequest(const GURL& url) {
 226   net::URLFetcher* request =
 227       net::URLFetcher::Create(0, url, net::URLFetcher::GET, this);
 228   request->SetLoadFlags(net::LOAD_DISABLE_CACHE);
 229   request->SetRequestContext(url_request_context_);
 230   // Add Chrome experiment state to the request headers.
 231   net::HttpRequestHeaders headers;
 232   variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
 233       request->GetOriginalURL(), false, false, &headers);
 234   request->SetExtraRequestHeaders(headers.ToString());
 235   return request;
 236 }
 237
 238 void SuggestionsService::OnURLFetchComplete(const net::URLFetcher* source) {
 239   DCHECK(thread_checker_.CalledOnValidThread());
 240   DCHECK_EQ(pending_request_.get(), source);
 241
 242   // The fetcher will be deleted when the request is handled.
 243   scoped_ptr<const net::URLFetcher> request(pending_request_.release());
 244
 245   const net::URLRequestStatus& request_status = request->GetStatus();
 246   if (request_status.status() != net::URLRequestStatus::SUCCESS) {
 247     // This represents network errors (i.e. the server did not provide a
 248     // response).
 249     UMA_HISTOGRAM_SPARSE_SLOWLY("Suggestions.FailedRequestErrorCode",
 250                                 -request_status.error());
 251     DVLOG(1) << "Suggestions server request failed with error: "
 252              << request_status.error() << ": "
 253              << net::ErrorToString(request_status.error());
 254     ScheduleBlacklistUpload(false);
 255     return;
 256   }
 257
 258   const int response_code = request->GetResponseCode();
 259   UMA_HISTOGRAM_SPARSE_SLOWLY("Suggestions.FetchResponseCode", response_code);
 260   if (response_code != net::HTTP_OK) {
 261     // A non-200 response code means that server has no (longer) suggestions for
 262     // this user. Aggressively clear the cache.
 263     suggestions_store_->ClearSuggestions();
 264     ScheduleBlacklistUpload(false);
 265     return;
 266   }
 267
 268   const base::TimeDelta latency =
 269       base::TimeTicks::Now() - last_request_started_time_;
 270   UMA_HISTOGRAM_MEDIUM_TIMES("Suggestions.FetchSuccessLatency", latency);
 271
 272   // Handle a successful blacklisting.
 273   GURL blacklisted_url;
 274   if (GetBlacklistedUrl(*source, &blacklisted_url)) {
 275     blacklist_store_->RemoveUrl(blacklisted_url);
 276   }
 277
 278   std::string suggestions_data;
 279   bool success = request->GetResponseAsString(&suggestions_data);
 280   DCHECK(success);
 281
 282   // Parse the received suggestions and update the cache, or take proper action
 283   // in the case of invalid response.
 284   SuggestionsProfile suggestions;
 285   if (suggestions_data.empty()) {
 286     LogResponseState(RESPONSE_EMPTY);
 287     suggestions_store_->ClearSuggestions();
 288   } else if (suggestions.ParseFromString(suggestions_data)) {
 289     LogResponseState(RESPONSE_VALID);
 290     int64 now_usec = (base::Time::NowFromSystemTime() - base::Time::UnixEpoch())
 291         .ToInternalValue();
 292     SetDefaultExpiryTimestamp(&suggestions, now_usec + kDefaultExpiryUsec);
 293     suggestions_store_->StoreSuggestions(suggestions);
 294   } else {
 295     LogResponseState(RESPONSE_INVALID);
 296   }
 297
 298   ScheduleBlacklistUpload(true);
 299 }
 300
 301 void SuggestionsService::Shutdown() {
 302   // Cancel pending request, then serve existing requestors from cache.
 303   pending_request_.reset(NULL);
 304   ServeFromCache();
 305 }
 306
 307 void SuggestionsService::ServeFromCache() {
 308   SuggestionsProfile suggestions;
 309   // In case of empty cache or error, |suggestions| stays empty.
 310   suggestions_store_->LoadSuggestions(&suggestions);
 311   thumbnail_manager_->Initialize(suggestions);
 312   FilterAndServe(&suggestions);
 313 }
 314
 315 void SuggestionsService::FilterAndServe(SuggestionsProfile* suggestions) {
 316   blacklist_store_->FilterSuggestions(suggestions);
 317   DispatchRequestsAndClear(*suggestions, &waiting_requestors_);
 318 }
 319
 320 void SuggestionsService::ScheduleBlacklistUpload(bool last_request_successful) {
 321   DCHECK(thread_checker_.CalledOnValidThread());
 322
 323   UpdateBlacklistDelay(last_request_successful);
 324
 325   // Schedule a blacklist upload task.
 326   GURL blacklist_url;
 327   if (blacklist_store_->GetFirstUrlFromBlacklist(&blacklist_url)) {
 328     base::Closure blacklist_cb =
 329         base::Bind(&SuggestionsService::UploadOneFromBlacklist,
 330                    weak_ptr_factory_.GetWeakPtr());
 331     base::MessageLoopProxy::current()->PostDelayedTask(
 332         FROM_HERE, blacklist_cb,
 333         base::TimeDelta::FromSeconds(blacklist_delay_sec_));
 334   }
 335 }
 336
 337 void SuggestionsService::UploadOneFromBlacklist() {
 338   DCHECK(thread_checker_.CalledOnValidThread());
 339
 340   GURL blacklist_url;
 341   if (!blacklist_store_->GetFirstUrlFromBlacklist(&blacklist_url))
 342     return;  // Local blacklist is empty.
 343
 344   // Send blacklisting request. Even if this request ends up not being sent
 345   // because of an ongoing request, a blacklist request is later scheduled.
 346   IssueRequestIfNoneOngoing(
 347       BuildBlacklistRequestURL(blacklist_url_prefix_, blacklist_url));
 348 }
 349
 350 void SuggestionsService::UpdateBlacklistDelay(bool last_request_successful) {
 351   DCHECK(thread_checker_.CalledOnValidThread());
 352
 353   if (last_request_successful) {
 354     blacklist_delay_sec_ = kBlacklistDefaultDelaySec;
 355   } else {
 356     int candidate_delay = blacklist_delay_sec_ * kBlacklistBackoffMultiplier;
 357     if (candidate_delay < kBlacklistMaxDelaySec)
 358       blacklist_delay_sec_ = candidate_delay;
 359   }
 360 }
 361
 362 }  // namespace suggestions