components/suggestions/suggestions_service.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/suggestions/suggestions_service.h"
   6
   7 #include <string>
   8
   9 #include "base/location.h"
  10 #include "base/memory/scoped_ptr.h"
  11 #include "base/metrics/histogram_macros.h"
  12 #include "base/metrics/sparse_histogram.h"
  13 #include "base/single_thread_task_runner.h"
  14 #include "base/strings/string_number_conversions.h"
  15 #include "base/strings/string_util.h"
  16 #include "base/strings/stringprintf.h"
  17 #include "base/thread_task_runner_handle.h"
  18 #include "base/time/time.h"
  19 #include "components/pref_registry/pref_registry_syncable.h"
  20 #include "components/suggestions/blacklist_store.h"
  21 #include "components/suggestions/suggestions_store.h"
  22 #include "components/variations/net/variations_http_header_provider.h"
  23 #include "net/base/escape.h"
  24 #include "net/base/load_flags.h"
  25 #include "net/base/net_errors.h"
  26 #include "net/base/url_util.h"
  27 #include "net/http/http_response_headers.h"
  28 #include "net/http/http_status_code.h"
  29 #include "net/http/http_util.h"
  30 #include "net/url_request/url_fetcher.h"
  31 #include "net/url_request/url_request_status.h"
  32 #include "url/gurl.h"
  33
  34 using base::CancelableClosure;
  35 using base::TimeDelta;
  36 using base::TimeTicks;
  37
  38 namespace suggestions {
  39
  40 namespace {
  41
  42 // Used to UMA log the state of the last response from the server.
  43 enum SuggestionsResponseState {
  44   RESPONSE_EMPTY,
  45   RESPONSE_INVALID,
  46   RESPONSE_VALID,
  47   RESPONSE_STATE_SIZE
  48 };
  49
  50 // Will log the supplied response |state|.
  51 void LogResponseState(SuggestionsResponseState state) {
  52   UMA_HISTOGRAM_ENUMERATION("Suggestions.ResponseState", state,
  53                             RESPONSE_STATE_SIZE);
  54 }
  55
  56 GURL BuildBlacklistRequestURL(const std::string& blacklist_url_prefix,
  57                               const GURL& candidate_url) {
  58   return GURL(blacklist_url_prefix +
  59               net::EscapeQueryParamValue(candidate_url.spec(), true));
  60 }
  61
  62 // Runs each callback in |requestors| on |suggestions|, then deallocates
  63 // |requestors|.
  64 void DispatchRequestsAndClear(
  65     const SuggestionsProfile& suggestions,
  66     std::vector<SuggestionsService::ResponseCallback>* requestors) {
  67   std::vector<SuggestionsService::ResponseCallback> temp_requestors;
  68   temp_requestors.swap(*requestors);
  69   std::vector<SuggestionsService::ResponseCallback>::iterator it;
  70   for (it = temp_requestors.begin(); it != temp_requestors.end(); ++it) {
  71     if (!it->is_null()) it->Run(suggestions);
  72   }
  73 }
  74
  75 // Default delay used when scheduling a request.
  76 const int kDefaultSchedulingDelaySec = 1;
  77
  78 // Multiplier on the delay used when re-scheduling a failed request.
  79 const int kSchedulingBackoffMultiplier = 2;
  80
  81 // Maximum valid delay for scheduling a request. Candidate delays larger than
  82 // this are rejected. This means the maximum backoff is at least 5 / 2 minutes.
  83 const int kSchedulingMaxDelaySec = 5 * 60;
  84
  85 const char kFaviconURL[] =
  86     "https://s2.googleusercontent.com/s2/favicons?domain_url=%s&alt=s&sz=32";
  87
  88 const char kPingURL[] =
  89     "https://www.google.com/chromesuggestions/click?q=%lld&cd=%d";
  90 }  // namespace
  91
  92 // TODO(mathp): Put this in TemplateURL.
  93 // TODO(fserb): Add logic to decide the device type of the request.
  94 #if defined(OS_ANDROID) || defined(OS_IOS)
  95 const char kSuggestionsURL[] = "https://www.google.com/chromesuggestions?t=2";
  96 const char kSuggestionsBlacklistURLPrefix[] =
  97     "https://www.google.com/chromesuggestions/blacklist?t=2&url=";
  98 const char kSuggestionsBlacklistClearURL[] =
  99     "https://www.google.com/chromesuggestions/blacklist/clear?t=2";
 100 #else
 101 const char kSuggestionsURL[] = "https://www.google.com/chromesuggestions?t=1";
 102 const char kSuggestionsBlacklistURLPrefix[] =
 103     "https://www.google.com/chromesuggestions/blacklist?t=1&url=";
 104 const char kSuggestionsBlacklistClearURL[] =
 105     "https://www.google.com/chromesuggestions/blacklist/clear?t=1";
 106 #endif
 107 const char kSuggestionsBlacklistURLParam[] = "url";
 108
 109 // The default expiry timeout is 168 hours.
 110 const int64 kDefaultExpiryUsec = 168 * base::Time::kMicrosecondsPerHour;
 111
 112 SuggestionsService::SuggestionsService(
 113     net::URLRequestContextGetter* url_request_context,
 114     scoped_ptr<SuggestionsStore> suggestions_store,
 115     scoped_ptr<ImageManager> thumbnail_manager,
 116     scoped_ptr<BlacklistStore> blacklist_store)
 117     : url_request_context_(url_request_context),
 118       suggestions_store_(suggestions_store.Pass()),
 119       thumbnail_manager_(thumbnail_manager.Pass()),
 120       blacklist_store_(blacklist_store.Pass()),
 121       scheduling_delay_(TimeDelta::FromSeconds(kDefaultSchedulingDelaySec)),
 122       suggestions_url_(kSuggestionsURL),
 123       blacklist_url_prefix_(kSuggestionsBlacklistURLPrefix),
 124       weak_ptr_factory_(this) {}
 125
 126 SuggestionsService::~SuggestionsService() {}
 127
 128 void SuggestionsService::FetchSuggestionsData(
 129     SyncState sync_state,
 130     SuggestionsService::ResponseCallback callback) {
 131   DCHECK(thread_checker_.CalledOnValidThread());
 132   waiting_requestors_.push_back(callback);
 133   if (sync_state == SYNC_OR_HISTORY_SYNC_DISABLED) {
 134     // Cancel any ongoing request, to stop interacting with the server.
 135     pending_request_.reset(NULL);
 136     suggestions_store_->ClearSuggestions();
 137     DispatchRequestsAndClear(SuggestionsProfile(), &waiting_requestors_);
 138   } else if (sync_state == INITIALIZED_ENABLED_HISTORY ||
 139              sync_state == NOT_INITIALIZED_ENABLED) {
 140     // Sync is enabled. Serve previously cached suggestions if available, else
 141     // an empty set of suggestions.
 142     ServeFromCache();
 143
 144     // Issue a network request to refresh the suggestions in the cache.
 145     IssueRequestIfNoneOngoing(suggestions_url_);
 146   } else {
 147     NOTREACHED();
 148   }
 149 }
 150
 151 void SuggestionsService::GetPageThumbnail(
 152     const GURL& url,
 153     const base::Callback<void(const GURL&, const SkBitmap*)>& callback) {
 154   thumbnail_manager_->GetImageForURL(url, callback);
 155 }
 156
 157 void SuggestionsService::GetPageThumbnailWithURL(
 158     const GURL& url,
 159     const GURL& thumbnail_url,
 160     const base::Callback<void(const GURL&, const SkBitmap*)>& callback) {
 161   thumbnail_manager_->AddImageURL(url, thumbnail_url);
 162   GetPageThumbnail(url, callback);
 163 }
 164
 165 void SuggestionsService::BlacklistURL(
 166     const GURL& candidate_url,
 167     const SuggestionsService::ResponseCallback& callback,
 168     const base::Closure& fail_callback) {
 169   DCHECK(thread_checker_.CalledOnValidThread());
 170
 171   if (!blacklist_store_->BlacklistUrl(candidate_url)) {
 172     fail_callback.Run();
 173     return;
 174   }
 175
 176   waiting_requestors_.push_back(callback);
 177   ServeFromCache();
 178   // Blacklist uploads are scheduled on any request completion, so only schedule
 179   // an upload if there is no ongoing request.
 180   if (!pending_request_.get()) {
 181     ScheduleBlacklistUpload();
 182   }
 183 }
 184
 185 void SuggestionsService::UndoBlacklistURL(
 186     const GURL& url,
 187     const SuggestionsService::ResponseCallback& callback,
 188     const base::Closure& fail_callback) {
 189   DCHECK(thread_checker_.CalledOnValidThread());
 190   TimeDelta time_delta;
 191   if (blacklist_store_->GetTimeUntilURLReadyForUpload(url, &time_delta) &&
 192       time_delta > TimeDelta::FromSeconds(0) &&
 193       blacklist_store_->RemoveUrl(url)) {
 194     // The URL was not yet candidate for upload to the server and could be
 195     // removed from the blacklist.
 196     waiting_requestors_.push_back(callback);
 197     ServeFromCache();
 198     return;
 199   }
 200   fail_callback.Run();
 201 }
 202
 203 void SuggestionsService::ClearBlacklist(const ResponseCallback& callback) {
 204   DCHECK(thread_checker_.CalledOnValidThread());
 205   blacklist_store_->ClearBlacklist();
 206   IssueRequestIfNoneOngoing(GURL(kSuggestionsBlacklistClearURL));
 207   waiting_requestors_.push_back(callback);
 208   ServeFromCache();
 209 }
 210
 211 // static
 212 bool SuggestionsService::GetBlacklistedUrl(const net::URLFetcher& request,
 213                                            GURL* url) {
 214   bool is_blacklist_request = base::StartsWith(
 215       request.GetOriginalURL().spec(), kSuggestionsBlacklistURLPrefix,
 216       base::CompareCase::SENSITIVE);
 217   if (!is_blacklist_request) return false;
 218
 219   // Extract the blacklisted URL from the blacklist request.
 220   std::string blacklisted;
 221   if (!net::GetValueForKeyInQuery(
 222           request.GetOriginalURL(),
 223           kSuggestionsBlacklistURLParam,
 224           &blacklisted)) {
 225     return false;
 226   }
 227
 228   GURL blacklisted_url(blacklisted);
 229   blacklisted_url.Swap(url);
 230   return true;
 231 }
 232
 233 // static
 234 void SuggestionsService::RegisterProfilePrefs(
 235     user_prefs::PrefRegistrySyncable* registry) {
 236   SuggestionsStore::RegisterProfilePrefs(registry);
 237   BlacklistStore::RegisterProfilePrefs(registry);
 238 }
 239
 240 void SuggestionsService::SetDefaultExpiryTimestamp(
 241     SuggestionsProfile* suggestions, int64 default_timestamp_usec) {
 242   for (int i = 0; i < suggestions->suggestions_size(); ++i) {
 243     ChromeSuggestion* suggestion = suggestions->mutable_suggestions(i);
 244     // Do not set expiry if the server has already provided a more specific
 245     // expiry time for this suggestion.
 246     if (!suggestion->has_expiry_ts()) {
 247       suggestion->set_expiry_ts(default_timestamp_usec);
 248     }
 249   }
 250 }
 251
 252 void SuggestionsService::IssueRequestIfNoneOngoing(const GURL& url) {
 253   // If there is an ongoing request, let it complete.
 254   if (pending_request_.get()) {
 255     return;
 256   }
 257   pending_request_ = CreateSuggestionsRequest(url);
 258   pending_request_->Start();
 259   last_request_started_time_ = TimeTicks::Now();
 260 }
 261
 262 scoped_ptr<net::URLFetcher> SuggestionsService::CreateSuggestionsRequest(
 263     const GURL& url) {
 264   scoped_ptr<net::URLFetcher> request =
 265       net::URLFetcher::Create(0, url, net::URLFetcher::GET, this);
 266   request->SetLoadFlags(net::LOAD_DISABLE_CACHE);
 267   request->SetRequestContext(url_request_context_);
 268   // Add Chrome experiment state to the request headers.
 269   net::HttpRequestHeaders headers;
 270   variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
 271       request->GetOriginalURL(), false, false, &headers);
 272   request->SetExtraRequestHeaders(headers.ToString());
 273   return request;
 274 }
 275
 276 void SuggestionsService::OnURLFetchComplete(const net::URLFetcher* source) {
 277   DCHECK(thread_checker_.CalledOnValidThread());
 278   DCHECK_EQ(pending_request_.get(), source);
 279
 280   // The fetcher will be deleted when the request is handled.
 281   scoped_ptr<const net::URLFetcher> request(pending_request_.release());
 282
 283   const net::URLRequestStatus& request_status = request->GetStatus();
 284   if (request_status.status() != net::URLRequestStatus::SUCCESS) {
 285     // This represents network errors (i.e. the server did not provide a
 286     // response).
 287     UMA_HISTOGRAM_SPARSE_SLOWLY("Suggestions.FailedRequestErrorCode",
 288                                 -request_status.error());
 289     DVLOG(1) << "Suggestions server request failed with error: "
 290              << request_status.error() << ": "
 291              << net::ErrorToString(request_status.error());
 292     UpdateBlacklistDelay(false);
 293     ScheduleBlacklistUpload();
 294     return;
 295   }
 296
 297   const int response_code = request->GetResponseCode();
 298   UMA_HISTOGRAM_SPARSE_SLOWLY("Suggestions.FetchResponseCode", response_code);
 299   if (response_code != net::HTTP_OK) {
 300     // A non-200 response code means that server has no (longer) suggestions for
 301     // this user. Aggressively clear the cache.
 302     suggestions_store_->ClearSuggestions();
 303     UpdateBlacklistDelay(false);
 304     ScheduleBlacklistUpload();
 305     return;
 306   }
 307
 308   const TimeDelta latency = TimeTicks::Now() - last_request_started_time_;
 309   UMA_HISTOGRAM_MEDIUM_TIMES("Suggestions.FetchSuccessLatency", latency);
 310
 311   // Handle a successful blacklisting.
 312   GURL blacklisted_url;
 313   if (GetBlacklistedUrl(*source, &blacklisted_url)) {
 314     blacklist_store_->RemoveUrl(blacklisted_url);
 315   }
 316
 317   std::string suggestions_data;
 318   bool success = request->GetResponseAsString(&suggestions_data);
 319   DCHECK(success);
 320
 321   // Parse the received suggestions and update the cache, or take proper action
 322   // in the case of invalid response.
 323   SuggestionsProfile suggestions;
 324   if (suggestions_data.empty()) {
 325     LogResponseState(RESPONSE_EMPTY);
 326     suggestions_store_->ClearSuggestions();
 327   } else if (suggestions.ParseFromString(suggestions_data)) {
 328     LogResponseState(RESPONSE_VALID);
 329     int64 now_usec = (base::Time::NowFromSystemTime() - base::Time::UnixEpoch())
 330         .ToInternalValue();
 331     SetDefaultExpiryTimestamp(&suggestions, now_usec + kDefaultExpiryUsec);
 332     PopulateExtraData(&suggestions);
 333     suggestions_store_->StoreSuggestions(suggestions);
 334   } else {
 335     LogResponseState(RESPONSE_INVALID);
 336   }
 337
 338   UpdateBlacklistDelay(true);
 339   ScheduleBlacklistUpload();
 340 }
 341
 342 void SuggestionsService::PopulateExtraData(SuggestionsProfile* suggestions) {
 343   for (int i = 0; i < suggestions->suggestions_size(); ++i) {
 344     suggestions::ChromeSuggestion* s = suggestions->mutable_suggestions(i);
 345     if (!s->has_favicon_url() || s->favicon_url().empty()) {
 346       s->set_favicon_url(base::StringPrintf(kFaviconURL, s->url().c_str()));
 347     }
 348     if (!s->has_impression_url() || s->impression_url().empty()) {
 349       s->set_impression_url(
 350           base::StringPrintf(
 351               kPingURL, static_cast<long long>(suggestions->timestamp()), -1));
 352     }
 353
 354     if (!s->has_click_url() || s->click_url().empty()) {
 355       s->set_click_url(base::StringPrintf(
 356           kPingURL, static_cast<long long>(suggestions->timestamp()), i));
 357     }
 358   }
 359 }
 360
 361 void SuggestionsService::Shutdown() {
 362   // Cancel pending request, then serve existing requestors from cache.
 363   pending_request_.reset(NULL);
 364   ServeFromCache();
 365 }
 366
 367 void SuggestionsService::ServeFromCache() {
 368   SuggestionsProfile suggestions;
 369   // In case of empty cache or error, |suggestions| stays empty.
 370   suggestions_store_->LoadSuggestions(&suggestions);
 371   thumbnail_manager_->Initialize(suggestions);
 372   FilterAndServe(&suggestions);
 373 }
 374
 375 void SuggestionsService::FilterAndServe(SuggestionsProfile* suggestions) {
 376   blacklist_store_->FilterSuggestions(suggestions);
 377   DispatchRequestsAndClear(*suggestions, &waiting_requestors_);
 378 }
 379
 380 void SuggestionsService::ScheduleBlacklistUpload() {
 381   DCHECK(thread_checker_.CalledOnValidThread());
 382   TimeDelta time_delta;
 383   if (blacklist_store_->GetTimeUntilReadyForUpload(&time_delta)) {
 384     // Blacklist cache is not empty: schedule.
 385     base::Closure blacklist_cb =
 386         base::Bind(&SuggestionsService::UploadOneFromBlacklist,
 387                    weak_ptr_factory_.GetWeakPtr());
 388     base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
 389         FROM_HERE, blacklist_cb, time_delta + scheduling_delay_);
 390   }
 391 }
 392
 393 void SuggestionsService::UploadOneFromBlacklist() {
 394   DCHECK(thread_checker_.CalledOnValidThread());
 395
 396   GURL blacklist_url;
 397   if (blacklist_store_->GetCandidateForUpload(&blacklist_url)) {
 398     // Issue a blacklisting request. Even if this request ends up not being sent
 399     // because of an ongoing request, a blacklist request is later scheduled.
 400     IssueRequestIfNoneOngoing(
 401         BuildBlacklistRequestURL(blacklist_url_prefix_, blacklist_url));
 402     return;
 403   }
 404
 405   // Even though there's no candidate for upload, the blacklist might not be
 406   // empty.
 407   ScheduleBlacklistUpload();
 408 }
 409
 410 void SuggestionsService::UpdateBlacklistDelay(bool last_request_successful) {
 411   DCHECK(thread_checker_.CalledOnValidThread());
 412
 413   if (last_request_successful) {
 414     scheduling_delay_ = TimeDelta::FromSeconds(kDefaultSchedulingDelaySec);
 415   } else {
 416     TimeDelta candidate_delay =
 417         scheduling_delay_ * kSchedulingBackoffMultiplier;
 418     if (candidate_delay < TimeDelta::FromSeconds(kSchedulingMaxDelaySec))
 419       scheduling_delay_ = candidate_delay;
 420   }
 421 }
 422
 423 }  // namespace suggestions