components/suggestions/suggestions_service.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/suggestions/suggestions_service.h"
   6
   7 #include <string>
   8
   9 #include "base/memory/scoped_ptr.h"
  10 #include "base/message_loop/message_loop_proxy.h"
  11 #include "base/metrics/histogram.h"
  12 #include "base/metrics/sparse_histogram.h"
  13 #include "base/strings/string_number_conversions.h"
  14 #include "base/strings/string_util.h"
  15 #include "base/time/time.h"
  16 #include "components/pref_registry/pref_registry_syncable.h"
  17 #include "components/suggestions/blacklist_store.h"
  18 #include "components/suggestions/suggestions_store.h"
  19 #include "components/variations/net/variations_http_header_provider.h"
  20 #include "components/variations/variations_associated_data.h"
  21 #include "net/base/escape.h"
  22 #include "net/base/load_flags.h"
  23 #include "net/base/net_errors.h"
  24 #include "net/base/url_util.h"
  25 #include "net/http/http_response_headers.h"
  26 #include "net/http/http_status_code.h"
  27 #include "net/http/http_util.h"
  28 #include "net/url_request/url_fetcher.h"
  29 #include "net/url_request/url_request_status.h"
  30 #include "url/gurl.h"
  31
  32 using base::CancelableClosure;
  33 using base::TimeDelta;
  34 using base::TimeTicks;
  35
  36 namespace suggestions {
  37
  38 namespace {
  39
  40 // Used to UMA log the state of the last response from the server.
  41 enum SuggestionsResponseState {
  42   RESPONSE_EMPTY,
  43   RESPONSE_INVALID,
  44   RESPONSE_VALID,
  45   RESPONSE_STATE_SIZE
  46 };
  47
  48 // Will log the supplied response |state|.
  49 void LogResponseState(SuggestionsResponseState state) {
  50   UMA_HISTOGRAM_ENUMERATION("Suggestions.ResponseState", state,
  51                             RESPONSE_STATE_SIZE);
  52 }
  53
  54 // Obtains the experiment parameter under the supplied |key|, or empty string
  55 // if the parameter does not exist.
  56 std::string GetExperimentParam(const std::string& key) {
  57   return variations::GetVariationParamValue(kSuggestionsFieldTrialName, key);
  58 }
  59
  60 GURL BuildBlacklistRequestURL(const std::string& blacklist_url_prefix,
  61                               const GURL& candidate_url) {
  62   return GURL(blacklist_url_prefix +
  63               net::EscapeQueryParamValue(candidate_url.spec(), true));
  64 }
  65
  66 // Runs each callback in |requestors| on |suggestions|, then deallocates
  67 // |requestors|.
  68 void DispatchRequestsAndClear(
  69     const SuggestionsProfile& suggestions,
  70     std::vector<SuggestionsService::ResponseCallback>* requestors) {
  71   std::vector<SuggestionsService::ResponseCallback> temp_requestors;
  72   temp_requestors.swap(*requestors);
  73   std::vector<SuggestionsService::ResponseCallback>::iterator it;
  74   for (it = temp_requestors.begin(); it != temp_requestors.end(); ++it) {
  75     if (!it->is_null()) it->Run(suggestions);
  76   }
  77 }
  78
  79 // Default delay used when scheduling a request.
  80 const int kDefaultSchedulingDelaySec = 1;
  81
  82 // Multiplier on the delay used when re-scheduling a failed request.
  83 const int kSchedulingBackoffMultiplier = 2;
  84
  85 // Maximum valid delay for scheduling a request. Candidate delays larger than
  86 // this are rejected. This means the maximum backoff is at least 5 / 2 minutes.
  87 const int kSchedulingMaxDelaySec = 5 * 60;
  88
  89 }  // namespace
  90
  91 const char kSuggestionsFieldTrialName[] = "ChromeSuggestions";
  92 const char kSuggestionsFieldTrialControlParam[] = "control";
  93 const char kSuggestionsFieldTrialStateEnabled[] = "enabled";
  94 const char kSuggestionsFieldTrialStateParam[] = "state";
  95
  96 // TODO(mathp): Put this in TemplateURL.
  97 const char kSuggestionsURL[] = "https://www.google.com/chromesuggestions?t=2";
  98 const char kSuggestionsBlacklistURLPrefix[] =
  99     "https://www.google.com/chromesuggestions/blacklist?t=2&url=";
 100 const char kSuggestionsBlacklistURLParam[] = "url";
 101
 102 // The default expiry timeout is 72 hours.
 103 const int64 kDefaultExpiryUsec = 72 * base::Time::kMicrosecondsPerHour;
 104
 105 SuggestionsService::SuggestionsService(
 106     net::URLRequestContextGetter* url_request_context,
 107     scoped_ptr<SuggestionsStore> suggestions_store,
 108     scoped_ptr<ImageManager> thumbnail_manager,
 109     scoped_ptr<BlacklistStore> blacklist_store)
 110     : url_request_context_(url_request_context),
 111       suggestions_store_(suggestions_store.Pass()),
 112       thumbnail_manager_(thumbnail_manager.Pass()),
 113       blacklist_store_(blacklist_store.Pass()),
 114       scheduling_delay_(TimeDelta::FromSeconds(kDefaultSchedulingDelaySec)),
 115       suggestions_url_(kSuggestionsURL),
 116       blacklist_url_prefix_(kSuggestionsBlacklistURLPrefix),
 117       weak_ptr_factory_(this) {}
 118
 119 SuggestionsService::~SuggestionsService() {}
 120
 121 // static
 122 bool SuggestionsService::IsEnabled() {
 123   return GetExperimentParam(kSuggestionsFieldTrialStateParam) ==
 124       kSuggestionsFieldTrialStateEnabled;
 125 }
 126
 127 // static
 128 bool SuggestionsService::IsControlGroup() {
 129   return GetExperimentParam(kSuggestionsFieldTrialControlParam) ==
 130       kSuggestionsFieldTrialStateEnabled;
 131 }
 132
 133 void SuggestionsService::FetchSuggestionsData(
 134     SyncState sync_state,
 135     SuggestionsService::ResponseCallback callback) {
 136   DCHECK(thread_checker_.CalledOnValidThread());
 137   waiting_requestors_.push_back(callback);
 138   if (sync_state == SYNC_OR_HISTORY_SYNC_DISABLED) {
 139     // Cancel any ongoing request, to stop interacting with the server.
 140     pending_request_.reset(NULL);
 141     suggestions_store_->ClearSuggestions();
 142     DispatchRequestsAndClear(SuggestionsProfile(), &waiting_requestors_);
 143   } else if (sync_state == INITIALIZED_ENABLED_HISTORY ||
 144              sync_state == NOT_INITIALIZED_ENABLED) {
 145     // Sync is enabled. Serve previously cached suggestions if available, else
 146     // an empty set of suggestions.
 147     ServeFromCache();
 148
 149     // Issue a network request to refresh the suggestions in the cache.
 150     IssueRequestIfNoneOngoing(suggestions_url_);
 151   } else {
 152     NOTREACHED();
 153   }
 154 }
 155
 156 void SuggestionsService::GetPageThumbnail(
 157     const GURL& url,
 158     base::Callback<void(const GURL&, const SkBitmap*)> callback) {
 159   thumbnail_manager_->GetImageForURL(url, callback);
 160 }
 161
 162 void SuggestionsService::BlacklistURL(
 163     const GURL& candidate_url,
 164     const SuggestionsService::ResponseCallback& callback,
 165     const base::Closure& fail_callback) {
 166   DCHECK(thread_checker_.CalledOnValidThread());
 167
 168   if (!blacklist_store_->BlacklistUrl(candidate_url)) {
 169     fail_callback.Run();
 170     return;
 171   }
 172
 173   waiting_requestors_.push_back(callback);
 174   ServeFromCache();
 175   // Blacklist uploads are scheduled on any request completion, so only schedule
 176   // an upload if there is no ongoing request.
 177   if (!pending_request_.get()) {
 178     ScheduleBlacklistUpload();
 179   }
 180 }
 181
 182 void SuggestionsService::UndoBlacklistURL(
 183     const GURL& url,
 184     const SuggestionsService::ResponseCallback& callback,
 185     const base::Closure& fail_callback) {
 186   DCHECK(thread_checker_.CalledOnValidThread());
 187   TimeDelta time_delta;
 188   if (blacklist_store_->GetTimeUntilURLReadyForUpload(url, &time_delta) &&
 189       time_delta > TimeDelta::FromSeconds(0) &&
 190       blacklist_store_->RemoveUrl(url)) {
 191     // The URL was not yet candidate for upload to the server and could be
 192     // removed from the blacklist.
 193     waiting_requestors_.push_back(callback);
 194     ServeFromCache();
 195     return;
 196   }
 197   fail_callback.Run();
 198 }
 199
 200 // static
 201 bool SuggestionsService::GetBlacklistedUrl(const net::URLFetcher& request,
 202                                            GURL* url) {
 203   bool is_blacklist_request = StartsWithASCII(request.GetOriginalURL().spec(),
 204                                               kSuggestionsBlacklistURLPrefix,
 205                                               true);
 206   if (!is_blacklist_request) return false;
 207
 208   // Extract the blacklisted URL from the blacklist request.
 209   std::string blacklisted;
 210   if (!net::GetValueForKeyInQuery(
 211           request.GetOriginalURL(),
 212           kSuggestionsBlacklistURLParam,
 213           &blacklisted)) {
 214     return false;
 215   }
 216
 217   GURL blacklisted_url(blacklisted);
 218   blacklisted_url.Swap(url);
 219   return true;
 220 }
 221
 222 // static
 223 void SuggestionsService::RegisterProfilePrefs(
 224     user_prefs::PrefRegistrySyncable* registry) {
 225   SuggestionsStore::RegisterProfilePrefs(registry);
 226   BlacklistStore::RegisterProfilePrefs(registry);
 227 }
 228
 229 void SuggestionsService::SetDefaultExpiryTimestamp(
 230     SuggestionsProfile* suggestions, int64 default_timestamp_usec) {
 231   for (int i = 0; i < suggestions->suggestions_size(); ++i) {
 232     ChromeSuggestion* suggestion = suggestions->mutable_suggestions(i);
 233     // Do not set expiry if the server has already provided a more specific
 234     // expiry time for this suggestion.
 235     if (!suggestion->has_expiry_ts()) {
 236       suggestion->set_expiry_ts(default_timestamp_usec);
 237     }
 238   }
 239 }
 240
 241 void SuggestionsService::IssueRequestIfNoneOngoing(const GURL& url) {
 242   // If there is an ongoing request, let it complete.
 243   if (pending_request_.get()) {
 244     return;
 245   }
 246   pending_request_.reset(CreateSuggestionsRequest(url));
 247   pending_request_->Start();
 248   last_request_started_time_ = TimeTicks::Now();
 249 }
 250
 251 net::URLFetcher* SuggestionsService::CreateSuggestionsRequest(const GURL& url) {
 252   net::URLFetcher* request =
 253       net::URLFetcher::Create(0, url, net::URLFetcher::GET, this);
 254   request->SetLoadFlags(net::LOAD_DISABLE_CACHE);
 255   request->SetRequestContext(url_request_context_);
 256   // Add Chrome experiment state to the request headers.
 257   net::HttpRequestHeaders headers;
 258   variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
 259       request->GetOriginalURL(), false, false, &headers);
 260   request->SetExtraRequestHeaders(headers.ToString());
 261   return request;
 262 }
 263
 264 void SuggestionsService::OnURLFetchComplete(const net::URLFetcher* source) {
 265   DCHECK(thread_checker_.CalledOnValidThread());
 266   DCHECK_EQ(pending_request_.get(), source);
 267
 268   // The fetcher will be deleted when the request is handled.
 269   scoped_ptr<const net::URLFetcher> request(pending_request_.release());
 270
 271   const net::URLRequestStatus& request_status = request->GetStatus();
 272   if (request_status.status() != net::URLRequestStatus::SUCCESS) {
 273     // This represents network errors (i.e. the server did not provide a
 274     // response).
 275     UMA_HISTOGRAM_SPARSE_SLOWLY("Suggestions.FailedRequestErrorCode",
 276                                 -request_status.error());
 277     DVLOG(1) << "Suggestions server request failed with error: "
 278              << request_status.error() << ": "
 279              << net::ErrorToString(request_status.error());
 280     UpdateBlacklistDelay(false);
 281     ScheduleBlacklistUpload();
 282     return;
 283   }
 284
 285   const int response_code = request->GetResponseCode();
 286   UMA_HISTOGRAM_SPARSE_SLOWLY("Suggestions.FetchResponseCode", response_code);
 287   if (response_code != net::HTTP_OK) {
 288     // A non-200 response code means that server has no (longer) suggestions for
 289     // this user. Aggressively clear the cache.
 290     suggestions_store_->ClearSuggestions();
 291     UpdateBlacklistDelay(false);
 292     ScheduleBlacklistUpload();
 293     return;
 294   }
 295
 296   const TimeDelta latency = TimeTicks::Now() - last_request_started_time_;
 297   UMA_HISTOGRAM_MEDIUM_TIMES("Suggestions.FetchSuccessLatency", latency);
 298
 299   // Handle a successful blacklisting.
 300   GURL blacklisted_url;
 301   if (GetBlacklistedUrl(*source, &blacklisted_url)) {
 302     blacklist_store_->RemoveUrl(blacklisted_url);
 303   }
 304
 305   std::string suggestions_data;
 306   bool success = request->GetResponseAsString(&suggestions_data);
 307   DCHECK(success);
 308
 309   // Parse the received suggestions and update the cache, or take proper action
 310   // in the case of invalid response.
 311   SuggestionsProfile suggestions;
 312   if (suggestions_data.empty()) {
 313     LogResponseState(RESPONSE_EMPTY);
 314     suggestions_store_->ClearSuggestions();
 315   } else if (suggestions.ParseFromString(suggestions_data)) {
 316     LogResponseState(RESPONSE_VALID);
 317     int64 now_usec = (base::Time::NowFromSystemTime() - base::Time::UnixEpoch())
 318         .ToInternalValue();
 319     SetDefaultExpiryTimestamp(&suggestions, now_usec + kDefaultExpiryUsec);
 320     suggestions_store_->StoreSuggestions(suggestions);
 321   } else {
 322     LogResponseState(RESPONSE_INVALID);
 323   }
 324
 325   UpdateBlacklistDelay(true);
 326   ScheduleBlacklistUpload();
 327 }
 328
 329 void SuggestionsService::Shutdown() {
 330   // Cancel pending request, then serve existing requestors from cache.
 331   pending_request_.reset(NULL);
 332   ServeFromCache();
 333 }
 334
 335 void SuggestionsService::ServeFromCache() {
 336   SuggestionsProfile suggestions;
 337   // In case of empty cache or error, |suggestions| stays empty.
 338   suggestions_store_->LoadSuggestions(&suggestions);
 339   thumbnail_manager_->Initialize(suggestions);
 340   FilterAndServe(&suggestions);
 341 }
 342
 343 void SuggestionsService::FilterAndServe(SuggestionsProfile* suggestions) {
 344   blacklist_store_->FilterSuggestions(suggestions);
 345   DispatchRequestsAndClear(*suggestions, &waiting_requestors_);
 346 }
 347
 348 void SuggestionsService::ScheduleBlacklistUpload() {
 349   DCHECK(thread_checker_.CalledOnValidThread());
 350   TimeDelta time_delta;
 351   if (blacklist_store_->GetTimeUntilReadyForUpload(&time_delta)) {
 352     // Blacklist cache is not empty: schedule.
 353     base::Closure blacklist_cb =
 354         base::Bind(&SuggestionsService::UploadOneFromBlacklist,
 355                    weak_ptr_factory_.GetWeakPtr());
 356     base::MessageLoopProxy::current()->PostDelayedTask(
 357         FROM_HERE, blacklist_cb, time_delta + scheduling_delay_);
 358   }
 359 }
 360
 361 void SuggestionsService::UploadOneFromBlacklist() {
 362   DCHECK(thread_checker_.CalledOnValidThread());
 363
 364   GURL blacklist_url;
 365   if (blacklist_store_->GetCandidateForUpload(&blacklist_url)) {
 366     // Issue a blacklisting request. Even if this request ends up not being sent
 367     // because of an ongoing request, a blacklist request is later scheduled.
 368     IssueRequestIfNoneOngoing(
 369         BuildBlacklistRequestURL(blacklist_url_prefix_, blacklist_url));
 370     return;
 371   }
 372
 373   // Even though there's no candidate for upload, the blacklist might not be
 374   // empty.
 375   ScheduleBlacklistUpload();
 376 }
 377
 378 void SuggestionsService::UpdateBlacklistDelay(bool last_request_successful) {
 379   DCHECK(thread_checker_.CalledOnValidThread());
 380
 381   if (last_request_successful) {
 382     scheduling_delay_ = TimeDelta::FromSeconds(kDefaultSchedulingDelaySec);
 383   } else {
 384     TimeDelta candidate_delay =
 385         scheduling_delay_ * kSchedulingBackoffMultiplier;
 386     if (candidate_delay < TimeDelta::FromSeconds(kSchedulingMaxDelaySec))
 387       scheduling_delay_ = candidate_delay;
 388   }
 389 }
 390
 391 }  // namespace suggestions