components/suggestions/suggestions_service.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/suggestions/suggestions_service.h"
   6
   7 #include <string>
   8
   9 #include "base/memory/scoped_ptr.h"
  10 #include "base/message_loop/message_loop_proxy.h"
  11 #include "base/metrics/histogram.h"
  12 #include "base/metrics/sparse_histogram.h"
  13 #include "base/strings/string_number_conversions.h"
  14 #include "base/strings/string_util.h"
  15 #include "base/time/time.h"
  16 #include "components/pref_registry/pref_registry_syncable.h"
  17 #include "components/suggestions/blacklist_store.h"
  18 #include "components/suggestions/suggestions_store.h"
  19 #include "components/variations/net/variations_http_header_provider.h"
  20 #include "components/variations/variations_associated_data.h"
  21 #include "net/base/escape.h"
  22 #include "net/base/load_flags.h"
  23 #include "net/base/net_errors.h"
  24 #include "net/base/url_util.h"
  25 #include "net/http/http_response_headers.h"
  26 #include "net/http/http_status_code.h"
  27 #include "net/http/http_util.h"
  28 #include "net/url_request/url_fetcher.h"
  29 #include "net/url_request/url_request_status.h"
  30 #include "url/gurl.h"
  31
  32 using base::CancelableClosure;
  33 using base::TimeDelta;
  34 using base::TimeTicks;
  35
  36 namespace suggestions {
  37
  38 namespace {
  39
  40 // Used to UMA log the state of the last response from the server.
  41 enum SuggestionsResponseState {
  42   RESPONSE_EMPTY,
  43   RESPONSE_INVALID,
  44   RESPONSE_VALID,
  45   RESPONSE_STATE_SIZE
  46 };
  47
  48 // Will log the supplied response |state|.
  49 void LogResponseState(SuggestionsResponseState state) {
  50   UMA_HISTOGRAM_ENUMERATION("Suggestions.ResponseState", state,
  51                             RESPONSE_STATE_SIZE);
  52 }
  53
  54 // Obtains the experiment parameter under the supplied |key|, or empty string
  55 // if the parameter does not exist.
  56 std::string GetExperimentParam(const std::string& key) {
  57   return variations::GetVariationParamValue(kSuggestionsFieldTrialName, key);
  58 }
  59
  60 GURL BuildBlacklistRequestURL(const std::string& blacklist_url_prefix,
  61                               const GURL& candidate_url) {
  62   return GURL(blacklist_url_prefix +
  63               net::EscapeQueryParamValue(candidate_url.spec(), true));
  64 }
  65
  66 // Runs each callback in |requestors| on |suggestions|, then deallocates
  67 // |requestors|.
  68 void DispatchRequestsAndClear(
  69     const SuggestionsProfile& suggestions,
  70     std::vector<SuggestionsService::ResponseCallback>* requestors) {
  71   std::vector<SuggestionsService::ResponseCallback> temp_requestors;
  72   temp_requestors.swap(*requestors);
  73   std::vector<SuggestionsService::ResponseCallback>::iterator it;
  74   for (it = temp_requestors.begin(); it != temp_requestors.end(); ++it) {
  75     if (!it->is_null()) it->Run(suggestions);
  76   }
  77 }
  78
  79 // Default delay used when scheduling a request.
  80 const int kDefaultSchedulingDelaySec = 1;
  81
  82 // Multiplier on the delay used when re-scheduling a failed request.
  83 const int kSchedulingBackoffMultiplier = 2;
  84
  85 // Maximum valid delay for scheduling a request. Candidate delays larger than
  86 // this are rejected. This means the maximum backoff is at least 5 / 2 minutes.
  87 const int kSchedulingMaxDelaySec = 5 * 60;
  88
  89 }  // namespace
  90
  91 const char kSuggestionsFieldTrialName[] = "ChromeSuggestions";
  92 const char kSuggestionsFieldTrialControlParam[] = "control";
  93 const char kSuggestionsFieldTrialStateEnabled[] = "enabled";
  94
  95 // TODO(mathp): Put this in TemplateURL.
  96 const char kSuggestionsURL[] = "https://www.google.com/chromesuggestions?t=2";
  97 const char kSuggestionsBlacklistURLPrefix[] =
  98     "https://www.google.com/chromesuggestions/blacklist?t=2&url=";
  99 const char kSuggestionsBlacklistURLParam[] = "url";
 100
 101 // The default expiry timeout is 72 hours.
 102 const int64 kDefaultExpiryUsec = 72 * base::Time::kMicrosecondsPerHour;
 103
 104 SuggestionsService::SuggestionsService(
 105     net::URLRequestContextGetter* url_request_context,
 106     scoped_ptr<SuggestionsStore> suggestions_store,
 107     scoped_ptr<ImageManager> thumbnail_manager,
 108     scoped_ptr<BlacklistStore> blacklist_store)
 109     : url_request_context_(url_request_context),
 110       suggestions_store_(suggestions_store.Pass()),
 111       thumbnail_manager_(thumbnail_manager.Pass()),
 112       blacklist_store_(blacklist_store.Pass()),
 113       scheduling_delay_(TimeDelta::FromSeconds(kDefaultSchedulingDelaySec)),
 114       suggestions_url_(kSuggestionsURL),
 115       blacklist_url_prefix_(kSuggestionsBlacklistURLPrefix),
 116       weak_ptr_factory_(this) {}
 117
 118 SuggestionsService::~SuggestionsService() {}
 119
 120 // static
 121 bool SuggestionsService::IsControlGroup() {
 122   return GetExperimentParam(kSuggestionsFieldTrialControlParam) ==
 123          kSuggestionsFieldTrialStateEnabled;
 124 }
 125
 126 void SuggestionsService::FetchSuggestionsData(
 127     SyncState sync_state,
 128     SuggestionsService::ResponseCallback callback) {
 129   DCHECK(thread_checker_.CalledOnValidThread());
 130   waiting_requestors_.push_back(callback);
 131   if (sync_state == SYNC_OR_HISTORY_SYNC_DISABLED) {
 132     // Cancel any ongoing request, to stop interacting with the server.
 133     pending_request_.reset(NULL);
 134     suggestions_store_->ClearSuggestions();
 135     DispatchRequestsAndClear(SuggestionsProfile(), &waiting_requestors_);
 136   } else if (sync_state == INITIALIZED_ENABLED_HISTORY ||
 137              sync_state == NOT_INITIALIZED_ENABLED) {
 138     // Sync is enabled. Serve previously cached suggestions if available, else
 139     // an empty set of suggestions.
 140     ServeFromCache();
 141
 142     // Issue a network request to refresh the suggestions in the cache.
 143     IssueRequestIfNoneOngoing(suggestions_url_);
 144   } else {
 145     NOTREACHED();
 146   }
 147 }
 148
 149 void SuggestionsService::GetPageThumbnail(
 150     const GURL& url,
 151     base::Callback<void(const GURL&, const SkBitmap*)> callback) {
 152   thumbnail_manager_->GetImageForURL(url, callback);
 153 }
 154
 155 void SuggestionsService::BlacklistURL(
 156     const GURL& candidate_url,
 157     const SuggestionsService::ResponseCallback& callback,
 158     const base::Closure& fail_callback) {
 159   DCHECK(thread_checker_.CalledOnValidThread());
 160
 161   if (!blacklist_store_->BlacklistUrl(candidate_url)) {
 162     fail_callback.Run();
 163     return;
 164   }
 165
 166   waiting_requestors_.push_back(callback);
 167   ServeFromCache();
 168   // Blacklist uploads are scheduled on any request completion, so only schedule
 169   // an upload if there is no ongoing request.
 170   if (!pending_request_.get()) {
 171     ScheduleBlacklistUpload();
 172   }
 173 }
 174
 175 void SuggestionsService::UndoBlacklistURL(
 176     const GURL& url,
 177     const SuggestionsService::ResponseCallback& callback,
 178     const base::Closure& fail_callback) {
 179   DCHECK(thread_checker_.CalledOnValidThread());
 180   TimeDelta time_delta;
 181   if (blacklist_store_->GetTimeUntilURLReadyForUpload(url, &time_delta) &&
 182       time_delta > TimeDelta::FromSeconds(0) &&
 183       blacklist_store_->RemoveUrl(url)) {
 184     // The URL was not yet candidate for upload to the server and could be
 185     // removed from the blacklist.
 186     waiting_requestors_.push_back(callback);
 187     ServeFromCache();
 188     return;
 189   }
 190   fail_callback.Run();
 191 }
 192
 193 // static
 194 bool SuggestionsService::GetBlacklistedUrl(const net::URLFetcher& request,
 195                                            GURL* url) {
 196   bool is_blacklist_request = StartsWithASCII(request.GetOriginalURL().spec(),
 197                                               kSuggestionsBlacklistURLPrefix,
 198                                               true);
 199   if (!is_blacklist_request) return false;
 200
 201   // Extract the blacklisted URL from the blacklist request.
 202   std::string blacklisted;
 203   if (!net::GetValueForKeyInQuery(
 204           request.GetOriginalURL(),
 205           kSuggestionsBlacklistURLParam,
 206           &blacklisted)) {
 207     return false;
 208   }
 209
 210   GURL blacklisted_url(blacklisted);
 211   blacklisted_url.Swap(url);
 212   return true;
 213 }
 214
 215 // static
 216 void SuggestionsService::RegisterProfilePrefs(
 217     user_prefs::PrefRegistrySyncable* registry) {
 218   SuggestionsStore::RegisterProfilePrefs(registry);
 219   BlacklistStore::RegisterProfilePrefs(registry);
 220 }
 221
 222 void SuggestionsService::SetDefaultExpiryTimestamp(
 223     SuggestionsProfile* suggestions, int64 default_timestamp_usec) {
 224   for (int i = 0; i < suggestions->suggestions_size(); ++i) {
 225     ChromeSuggestion* suggestion = suggestions->mutable_suggestions(i);
 226     // Do not set expiry if the server has already provided a more specific
 227     // expiry time for this suggestion.
 228     if (!suggestion->has_expiry_ts()) {
 229       suggestion->set_expiry_ts(default_timestamp_usec);
 230     }
 231   }
 232 }
 233
 234 void SuggestionsService::IssueRequestIfNoneOngoing(const GURL& url) {
 235   // If there is an ongoing request, let it complete.
 236   if (pending_request_.get()) {
 237     return;
 238   }
 239   pending_request_.reset(CreateSuggestionsRequest(url));
 240   pending_request_->Start();
 241   last_request_started_time_ = TimeTicks::Now();
 242 }
 243
 244 net::URLFetcher* SuggestionsService::CreateSuggestionsRequest(const GURL& url) {
 245   net::URLFetcher* request =
 246       net::URLFetcher::Create(0, url, net::URLFetcher::GET, this);
 247   request->SetLoadFlags(net::LOAD_DISABLE_CACHE);
 248   request->SetRequestContext(url_request_context_);
 249   // Add Chrome experiment state to the request headers.
 250   net::HttpRequestHeaders headers;
 251   variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
 252       request->GetOriginalURL(), false, false, &headers);
 253   request->SetExtraRequestHeaders(headers.ToString());
 254   return request;
 255 }
 256
 257 void SuggestionsService::OnURLFetchComplete(const net::URLFetcher* source) {
 258   DCHECK(thread_checker_.CalledOnValidThread());
 259   DCHECK_EQ(pending_request_.get(), source);
 260
 261   // The fetcher will be deleted when the request is handled.
 262   scoped_ptr<const net::URLFetcher> request(pending_request_.release());
 263
 264   const net::URLRequestStatus& request_status = request->GetStatus();
 265   if (request_status.status() != net::URLRequestStatus::SUCCESS) {
 266     // This represents network errors (i.e. the server did not provide a
 267     // response).
 268     UMA_HISTOGRAM_SPARSE_SLOWLY("Suggestions.FailedRequestErrorCode",
 269                                 -request_status.error());
 270     DVLOG(1) << "Suggestions server request failed with error: "
 271              << request_status.error() << ": "
 272              << net::ErrorToString(request_status.error());
 273     UpdateBlacklistDelay(false);
 274     ScheduleBlacklistUpload();
 275     return;
 276   }
 277
 278   const int response_code = request->GetResponseCode();
 279   UMA_HISTOGRAM_SPARSE_SLOWLY("Suggestions.FetchResponseCode", response_code);
 280   if (response_code != net::HTTP_OK) {
 281     // A non-200 response code means that server has no (longer) suggestions for
 282     // this user. Aggressively clear the cache.
 283     suggestions_store_->ClearSuggestions();
 284     UpdateBlacklistDelay(false);
 285     ScheduleBlacklistUpload();
 286     return;
 287   }
 288
 289   const TimeDelta latency = TimeTicks::Now() - last_request_started_time_;
 290   UMA_HISTOGRAM_MEDIUM_TIMES("Suggestions.FetchSuccessLatency", latency);
 291
 292   // Handle a successful blacklisting.
 293   GURL blacklisted_url;
 294   if (GetBlacklistedUrl(*source, &blacklisted_url)) {
 295     blacklist_store_->RemoveUrl(blacklisted_url);
 296   }
 297
 298   std::string suggestions_data;
 299   bool success = request->GetResponseAsString(&suggestions_data);
 300   DCHECK(success);
 301
 302   // Parse the received suggestions and update the cache, or take proper action
 303   // in the case of invalid response.
 304   SuggestionsProfile suggestions;
 305   if (suggestions_data.empty()) {
 306     LogResponseState(RESPONSE_EMPTY);
 307     suggestions_store_->ClearSuggestions();
 308   } else if (suggestions.ParseFromString(suggestions_data)) {
 309     LogResponseState(RESPONSE_VALID);
 310     int64 now_usec = (base::Time::NowFromSystemTime() - base::Time::UnixEpoch())
 311         .ToInternalValue();
 312     SetDefaultExpiryTimestamp(&suggestions, now_usec + kDefaultExpiryUsec);
 313     suggestions_store_->StoreSuggestions(suggestions);
 314   } else {
 315     LogResponseState(RESPONSE_INVALID);
 316   }
 317
 318   UpdateBlacklistDelay(true);
 319   ScheduleBlacklistUpload();
 320 }
 321
 322 void SuggestionsService::Shutdown() {
 323   // Cancel pending request, then serve existing requestors from cache.
 324   pending_request_.reset(NULL);
 325   ServeFromCache();
 326 }
 327
 328 void SuggestionsService::ServeFromCache() {
 329   SuggestionsProfile suggestions;
 330   // In case of empty cache or error, |suggestions| stays empty.
 331   suggestions_store_->LoadSuggestions(&suggestions);
 332   thumbnail_manager_->Initialize(suggestions);
 333   FilterAndServe(&suggestions);
 334 }
 335
 336 void SuggestionsService::FilterAndServe(SuggestionsProfile* suggestions) {
 337   blacklist_store_->FilterSuggestions(suggestions);
 338   DispatchRequestsAndClear(*suggestions, &waiting_requestors_);
 339 }
 340
 341 void SuggestionsService::ScheduleBlacklistUpload() {
 342   DCHECK(thread_checker_.CalledOnValidThread());
 343   TimeDelta time_delta;
 344   if (blacklist_store_->GetTimeUntilReadyForUpload(&time_delta)) {
 345     // Blacklist cache is not empty: schedule.
 346     base::Closure blacklist_cb =
 347         base::Bind(&SuggestionsService::UploadOneFromBlacklist,
 348                    weak_ptr_factory_.GetWeakPtr());
 349     base::MessageLoopProxy::current()->PostDelayedTask(
 350         FROM_HERE, blacklist_cb, time_delta + scheduling_delay_);
 351   }
 352 }
 353
 354 void SuggestionsService::UploadOneFromBlacklist() {
 355   DCHECK(thread_checker_.CalledOnValidThread());
 356
 357   GURL blacklist_url;
 358   if (blacklist_store_->GetCandidateForUpload(&blacklist_url)) {
 359     // Issue a blacklisting request. Even if this request ends up not being sent
 360     // because of an ongoing request, a blacklist request is later scheduled.
 361     IssueRequestIfNoneOngoing(
 362         BuildBlacklistRequestURL(blacklist_url_prefix_, blacklist_url));
 363     return;
 364   }
 365
 366   // Even though there's no candidate for upload, the blacklist might not be
 367   // empty.
 368   ScheduleBlacklistUpload();
 369 }
 370
 371 void SuggestionsService::UpdateBlacklistDelay(bool last_request_successful) {
 372   DCHECK(thread_checker_.CalledOnValidThread());
 373
 374   if (last_request_successful) {
 375     scheduling_delay_ = TimeDelta::FromSeconds(kDefaultSchedulingDelaySec);
 376   } else {
 377     TimeDelta candidate_delay =
 378         scheduling_delay_ * kSchedulingBackoffMultiplier;
 379     if (candidate_delay < TimeDelta::FromSeconds(kSchedulingMaxDelaySec))
 380       scheduling_delay_ = candidate_delay;
 381   }
 382 }
 383
 384 }  // namespace suggestions