components/suggestions/suggestions_service.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/suggestions/suggestions_service.h"
   6
   7 #include <string>
   8
   9 #include "base/location.h"
  10 #include "base/memory/scoped_ptr.h"
  11 #include "base/metrics/histogram_macros.h"
  12 #include "base/metrics/sparse_histogram.h"
  13 #include "base/single_thread_task_runner.h"
  14 #include "base/strings/string_number_conversions.h"
  15 #include "base/strings/string_util.h"
  16 #include "base/strings/stringprintf.h"
  17 #include "base/thread_task_runner_handle.h"
  18 #include "base/time/time.h"
  19 #include "components/pref_registry/pref_registry_syncable.h"
  20 #include "components/suggestions/blacklist_store.h"
  21 #include "components/suggestions/suggestions_store.h"
  22 #include "components/variations/net/variations_http_header_provider.h"
  23 #include "net/base/escape.h"
  24 #include "net/base/load_flags.h"
  25 #include "net/base/net_errors.h"
  26 #include "net/base/url_util.h"
  27 #include "net/http/http_response_headers.h"
  28 #include "net/http/http_status_code.h"
  29 #include "net/http/http_util.h"
  30 #include "net/url_request/url_fetcher.h"
  31 #include "net/url_request/url_request_status.h"
  32 #include "url/gurl.h"
  33
  34 using base::CancelableClosure;
  35 using base::TimeDelta;
  36 using base::TimeTicks;
  37
  38 namespace suggestions {
  39
  40 namespace {
  41
  42 // Used to UMA log the state of the last response from the server.
  43 enum SuggestionsResponseState {
  44   RESPONSE_EMPTY,
  45   RESPONSE_INVALID,
  46   RESPONSE_VALID,
  47   RESPONSE_STATE_SIZE
  48 };
  49
  50 // Will log the supplied response |state|.
  51 void LogResponseState(SuggestionsResponseState state) {
  52   UMA_HISTOGRAM_ENUMERATION("Suggestions.ResponseState", state,
  53                             RESPONSE_STATE_SIZE);
  54 }
  55
  56 GURL BuildBlacklistRequestURL(const std::string& blacklist_url_prefix,
  57                               const GURL& candidate_url) {
  58   return GURL(blacklist_url_prefix +
  59               net::EscapeQueryParamValue(candidate_url.spec(), true));
  60 }
  61
  62 // Runs each callback in |requestors| on |suggestions|, then deallocates
  63 // |requestors|.
  64 void DispatchRequestsAndClear(
  65     const SuggestionsProfile& suggestions,
  66     std::vector<SuggestionsService::ResponseCallback>* requestors) {
  67   std::vector<SuggestionsService::ResponseCallback> temp_requestors;
  68   temp_requestors.swap(*requestors);
  69   std::vector<SuggestionsService::ResponseCallback>::iterator it;
  70   for (it = temp_requestors.begin(); it != temp_requestors.end(); ++it) {
  71     if (!it->is_null()) it->Run(suggestions);
  72   }
  73 }
  74
  75 // Default delay used when scheduling a request.
  76 const int kDefaultSchedulingDelaySec = 1;
  77
  78 // Multiplier on the delay used when re-scheduling a failed request.
  79 const int kSchedulingBackoffMultiplier = 2;
  80
  81 // Maximum valid delay for scheduling a request. Candidate delays larger than
  82 // this are rejected. This means the maximum backoff is at least 5 / 2 minutes.
  83 const int kSchedulingMaxDelaySec = 5 * 60;
  84
  85 const char kFaviconURL[] =
  86     "https://s2.googleusercontent.com/s2/favicons?domain_url=%s&alt=s&sz=32";
  87
  88 }  // namespace
  89
  90 // TODO(mathp): Put this in TemplateURL.
  91 // TODO(fserb): Add logic to decide the device type of the request.
  92 #if defined(OS_ANDROID) || defined(OS_IOS)
  93 const char kSuggestionsURL[] = "https://www.google.com/chromesuggestions?t=2";
  94 const char kSuggestionsBlacklistURLPrefix[] =
  95     "https://www.google.com/chromesuggestions/blacklist?t=2&url=";
  96 const char kSuggestionsBlacklistClearURL[] =
  97     "https://www.google.com/chromesuggestions/blacklist/clear?t=2";
  98 #else
  99 const char kSuggestionsURL[] = "https://www.google.com/chromesuggestions?t=1";
 100 const char kSuggestionsBlacklistURLPrefix[] =
 101     "https://www.google.com/chromesuggestions/blacklist?t=1&url=";
 102 const char kSuggestionsBlacklistClearURL[] =
 103     "https://www.google.com/chromesuggestions/blacklist/clear?t=1";
 104 #endif
 105 const char kSuggestionsBlacklistURLParam[] = "url";
 106
 107 // The default expiry timeout is 168 hours.
 108 const int64 kDefaultExpiryUsec = 168 * base::Time::kMicrosecondsPerHour;
 109
 110 SuggestionsService::SuggestionsService(
 111     net::URLRequestContextGetter* url_request_context,
 112     scoped_ptr<SuggestionsStore> suggestions_store,
 113     scoped_ptr<ImageManager> thumbnail_manager,
 114     scoped_ptr<BlacklistStore> blacklist_store)
 115     : url_request_context_(url_request_context),
 116       suggestions_store_(suggestions_store.Pass()),
 117       thumbnail_manager_(thumbnail_manager.Pass()),
 118       blacklist_store_(blacklist_store.Pass()),
 119       scheduling_delay_(TimeDelta::FromSeconds(kDefaultSchedulingDelaySec)),
 120       suggestions_url_(kSuggestionsURL),
 121       blacklist_url_prefix_(kSuggestionsBlacklistURLPrefix),
 122       weak_ptr_factory_(this) {}
 123
 124 SuggestionsService::~SuggestionsService() {}
 125
 126 void SuggestionsService::FetchSuggestionsData(
 127     SyncState sync_state,
 128     SuggestionsService::ResponseCallback callback) {
 129   DCHECK(thread_checker_.CalledOnValidThread());
 130   waiting_requestors_.push_back(callback);
 131   if (sync_state == SYNC_OR_HISTORY_SYNC_DISABLED) {
 132     // Cancel any ongoing request, to stop interacting with the server.
 133     pending_request_.reset(NULL);
 134     suggestions_store_->ClearSuggestions();
 135     DispatchRequestsAndClear(SuggestionsProfile(), &waiting_requestors_);
 136   } else if (sync_state == INITIALIZED_ENABLED_HISTORY ||
 137              sync_state == NOT_INITIALIZED_ENABLED) {
 138     // Sync is enabled. Serve previously cached suggestions if available, else
 139     // an empty set of suggestions.
 140     ServeFromCache();
 141
 142     // Issue a network request to refresh the suggestions in the cache.
 143     IssueRequestIfNoneOngoing(suggestions_url_);
 144   } else {
 145     NOTREACHED();
 146   }
 147 }
 148
 149 void SuggestionsService::GetPageThumbnail(
 150     const GURL& url,
 151     base::Callback<void(const GURL&, const SkBitmap*)> callback) {
 152   thumbnail_manager_->GetImageForURL(url, callback);
 153 }
 154
 155 void SuggestionsService::BlacklistURL(
 156     const GURL& candidate_url,
 157     const SuggestionsService::ResponseCallback& callback,
 158     const base::Closure& fail_callback) {
 159   DCHECK(thread_checker_.CalledOnValidThread());
 160
 161   if (!blacklist_store_->BlacklistUrl(candidate_url)) {
 162     fail_callback.Run();
 163     return;
 164   }
 165
 166   waiting_requestors_.push_back(callback);
 167   ServeFromCache();
 168   // Blacklist uploads are scheduled on any request completion, so only schedule
 169   // an upload if there is no ongoing request.
 170   if (!pending_request_.get()) {
 171     ScheduleBlacklistUpload();
 172   }
 173 }
 174
 175 void SuggestionsService::UndoBlacklistURL(
 176     const GURL& url,
 177     const SuggestionsService::ResponseCallback& callback,
 178     const base::Closure& fail_callback) {
 179   DCHECK(thread_checker_.CalledOnValidThread());
 180   TimeDelta time_delta;
 181   if (blacklist_store_->GetTimeUntilURLReadyForUpload(url, &time_delta) &&
 182       time_delta > TimeDelta::FromSeconds(0) &&
 183       blacklist_store_->RemoveUrl(url)) {
 184     // The URL was not yet candidate for upload to the server and could be
 185     // removed from the blacklist.
 186     waiting_requestors_.push_back(callback);
 187     ServeFromCache();
 188     return;
 189   }
 190   fail_callback.Run();
 191 }
 192
 193 void SuggestionsService::ClearBlacklist(const ResponseCallback& callback) {
 194   DCHECK(thread_checker_.CalledOnValidThread());
 195   blacklist_store_->ClearBlacklist();
 196   IssueRequestIfNoneOngoing(GURL(kSuggestionsBlacklistClearURL));
 197   waiting_requestors_.push_back(callback);
 198   ServeFromCache();
 199 }
 200
 201 // static
 202 bool SuggestionsService::GetBlacklistedUrl(const net::URLFetcher& request,
 203                                            GURL* url) {
 204   bool is_blacklist_request = base::StartsWith(
 205       request.GetOriginalURL().spec(), kSuggestionsBlacklistURLPrefix,
 206       base::CompareCase::SENSITIVE);
 207   if (!is_blacklist_request) return false;
 208
 209   // Extract the blacklisted URL from the blacklist request.
 210   std::string blacklisted;
 211   if (!net::GetValueForKeyInQuery(
 212           request.GetOriginalURL(),
 213           kSuggestionsBlacklistURLParam,
 214           &blacklisted)) {
 215     return false;
 216   }
 217
 218   GURL blacklisted_url(blacklisted);
 219   blacklisted_url.Swap(url);
 220   return true;
 221 }
 222
 223 // static
 224 void SuggestionsService::RegisterProfilePrefs(
 225     user_prefs::PrefRegistrySyncable* registry) {
 226   SuggestionsStore::RegisterProfilePrefs(registry);
 227   BlacklistStore::RegisterProfilePrefs(registry);
 228 }
 229
 230 void SuggestionsService::SetDefaultExpiryTimestamp(
 231     SuggestionsProfile* suggestions, int64 default_timestamp_usec) {
 232   for (int i = 0; i < suggestions->suggestions_size(); ++i) {
 233     ChromeSuggestion* suggestion = suggestions->mutable_suggestions(i);
 234     // Do not set expiry if the server has already provided a more specific
 235     // expiry time for this suggestion.
 236     if (!suggestion->has_expiry_ts()) {
 237       suggestion->set_expiry_ts(default_timestamp_usec);
 238     }
 239   }
 240 }
 241
 242 void SuggestionsService::IssueRequestIfNoneOngoing(const GURL& url) {
 243   // If there is an ongoing request, let it complete.
 244   if (pending_request_.get()) {
 245     return;
 246   }
 247   pending_request_ = CreateSuggestionsRequest(url);
 248   pending_request_->Start();
 249   last_request_started_time_ = TimeTicks::Now();
 250 }
 251
 252 scoped_ptr<net::URLFetcher> SuggestionsService::CreateSuggestionsRequest(
 253     const GURL& url) {
 254   scoped_ptr<net::URLFetcher> request =
 255       net::URLFetcher::Create(0, url, net::URLFetcher::GET, this);
 256   request->SetLoadFlags(net::LOAD_DISABLE_CACHE);
 257   request->SetRequestContext(url_request_context_);
 258   // Add Chrome experiment state to the request headers.
 259   net::HttpRequestHeaders headers;
 260   variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
 261       request->GetOriginalURL(), false, false, &headers);
 262   request->SetExtraRequestHeaders(headers.ToString());
 263   return request;
 264 }
 265
 266 void SuggestionsService::OnURLFetchComplete(const net::URLFetcher* source) {
 267   DCHECK(thread_checker_.CalledOnValidThread());
 268   DCHECK_EQ(pending_request_.get(), source);
 269
 270   // The fetcher will be deleted when the request is handled.
 271   scoped_ptr<const net::URLFetcher> request(pending_request_.release());
 272
 273   const net::URLRequestStatus& request_status = request->GetStatus();
 274   if (request_status.status() != net::URLRequestStatus::SUCCESS) {
 275     // This represents network errors (i.e. the server did not provide a
 276     // response).
 277     UMA_HISTOGRAM_SPARSE_SLOWLY("Suggestions.FailedRequestErrorCode",
 278                                 -request_status.error());
 279     DVLOG(1) << "Suggestions server request failed with error: "
 280              << request_status.error() << ": "
 281              << net::ErrorToString(request_status.error());
 282     UpdateBlacklistDelay(false);
 283     ScheduleBlacklistUpload();
 284     return;
 285   }
 286
 287   const int response_code = request->GetResponseCode();
 288   UMA_HISTOGRAM_SPARSE_SLOWLY("Suggestions.FetchResponseCode", response_code);
 289   if (response_code != net::HTTP_OK) {
 290     // A non-200 response code means that server has no (longer) suggestions for
 291     // this user. Aggressively clear the cache.
 292     suggestions_store_->ClearSuggestions();
 293     UpdateBlacklistDelay(false);
 294     ScheduleBlacklistUpload();
 295     return;
 296   }
 297
 298   const TimeDelta latency = TimeTicks::Now() - last_request_started_time_;
 299   UMA_HISTOGRAM_MEDIUM_TIMES("Suggestions.FetchSuccessLatency", latency);
 300
 301   // Handle a successful blacklisting.
 302   GURL blacklisted_url;
 303   if (GetBlacklistedUrl(*source, &blacklisted_url)) {
 304     blacklist_store_->RemoveUrl(blacklisted_url);
 305   }
 306
 307   std::string suggestions_data;
 308   bool success = request->GetResponseAsString(&suggestions_data);
 309   DCHECK(success);
 310
 311   // Parse the received suggestions and update the cache, or take proper action
 312   // in the case of invalid response.
 313   SuggestionsProfile suggestions;
 314   if (suggestions_data.empty()) {
 315     LogResponseState(RESPONSE_EMPTY);
 316     suggestions_store_->ClearSuggestions();
 317   } else if (suggestions.ParseFromString(suggestions_data)) {
 318     LogResponseState(RESPONSE_VALID);
 319     int64 now_usec = (base::Time::NowFromSystemTime() - base::Time::UnixEpoch())
 320         .ToInternalValue();
 321     SetDefaultExpiryTimestamp(&suggestions, now_usec + kDefaultExpiryUsec);
 322     PopulateFaviconUrls(&suggestions);
 323     suggestions_store_->StoreSuggestions(suggestions);
 324   } else {
 325     LogResponseState(RESPONSE_INVALID);
 326   }
 327
 328   UpdateBlacklistDelay(true);
 329   ScheduleBlacklistUpload();
 330 }
 331
 332 void SuggestionsService::PopulateFaviconUrls(SuggestionsProfile* suggestions) {
 333   for (int i = 0; i < suggestions->suggestions_size(); ++i) {
 334     suggestions::ChromeSuggestion* s = suggestions->mutable_suggestions(i);
 335     if (!s->has_favicon_url() || s->favicon_url().empty()) {
 336       s->set_favicon_url(base::StringPrintf(kFaviconURL, s->url().c_str()));
 337     }
 338   }
 339 }
 340
 341 void SuggestionsService::Shutdown() {
 342   // Cancel pending request, then serve existing requestors from cache.
 343   pending_request_.reset(NULL);
 344   ServeFromCache();
 345 }
 346
 347 void SuggestionsService::ServeFromCache() {
 348   SuggestionsProfile suggestions;
 349   // In case of empty cache or error, |suggestions| stays empty.
 350   suggestions_store_->LoadSuggestions(&suggestions);
 351   thumbnail_manager_->Initialize(suggestions);
 352   FilterAndServe(&suggestions);
 353 }
 354
 355 void SuggestionsService::FilterAndServe(SuggestionsProfile* suggestions) {
 356   blacklist_store_->FilterSuggestions(suggestions);
 357   DispatchRequestsAndClear(*suggestions, &waiting_requestors_);
 358 }
 359
 360 void SuggestionsService::ScheduleBlacklistUpload() {
 361   DCHECK(thread_checker_.CalledOnValidThread());
 362   TimeDelta time_delta;
 363   if (blacklist_store_->GetTimeUntilReadyForUpload(&time_delta)) {
 364     // Blacklist cache is not empty: schedule.
 365     base::Closure blacklist_cb =
 366         base::Bind(&SuggestionsService::UploadOneFromBlacklist,
 367                    weak_ptr_factory_.GetWeakPtr());
 368     base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
 369         FROM_HERE, blacklist_cb, time_delta + scheduling_delay_);
 370   }
 371 }
 372
 373 void SuggestionsService::UploadOneFromBlacklist() {
 374   DCHECK(thread_checker_.CalledOnValidThread());
 375
 376   GURL blacklist_url;
 377   if (blacklist_store_->GetCandidateForUpload(&blacklist_url)) {
 378     // Issue a blacklisting request. Even if this request ends up not being sent
 379     // because of an ongoing request, a blacklist request is later scheduled.
 380     IssueRequestIfNoneOngoing(
 381         BuildBlacklistRequestURL(blacklist_url_prefix_, blacklist_url));
 382     return;
 383   }
 384
 385   // Even though there's no candidate for upload, the blacklist might not be
 386   // empty.
 387   ScheduleBlacklistUpload();
 388 }
 389
 390 void SuggestionsService::UpdateBlacklistDelay(bool last_request_successful) {
 391   DCHECK(thread_checker_.CalledOnValidThread());
 392
 393   if (last_request_successful) {
 394     scheduling_delay_ = TimeDelta::FromSeconds(kDefaultSchedulingDelaySec);
 395   } else {
 396     TimeDelta candidate_delay =
 397         scheduling_delay_ * kSchedulingBackoffMultiplier;
 398     if (candidate_delay < TimeDelta::FromSeconds(kSchedulingMaxDelaySec))
 399       scheduling_delay_ = candidate_delay;
 400   }
 401 }
 402
 403 }  // namespace suggestions