components/suggestions/suggestions_service.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/suggestions/suggestions_service.h"
   6
   7 #include <string>
   8
   9 #include "base/location.h"
  10 #include "base/memory/scoped_ptr.h"
  11 #include "base/metrics/histogram_macros.h"
  12 #include "base/metrics/sparse_histogram.h"
  13 #include "base/single_thread_task_runner.h"
  14 #include "base/strings/string_number_conversions.h"
  15 #include "base/strings/string_util.h"
  16 #include "base/strings/stringprintf.h"
  17 #include "base/thread_task_runner_handle.h"
  18 #include "base/time/time.h"
  19 #include "components/data_use_measurement/core/data_use_user_data.h"
  20 #include "components/pref_registry/pref_registry_syncable.h"
  21 #include "components/suggestions/blacklist_store.h"
  22 #include "components/suggestions/suggestions_store.h"
  23 #include "components/variations/net/variations_http_header_provider.h"
  24 #include "net/base/escape.h"
  25 #include "net/base/load_flags.h"
  26 #include "net/base/net_errors.h"
  27 #include "net/base/url_util.h"
  28 #include "net/http/http_response_headers.h"
  29 #include "net/http/http_status_code.h"
  30 #include "net/http/http_util.h"
  31 #include "net/url_request/url_fetcher.h"
  32 #include "net/url_request/url_request_status.h"
  33 #include "url/gurl.h"
  34
  35 using base::CancelableClosure;
  36 using base::TimeDelta;
  37 using base::TimeTicks;
  38
  39 namespace suggestions {
  40
  41 namespace {
  42
  43 // Used to UMA log the state of the last response from the server.
  44 enum SuggestionsResponseState {
  45   RESPONSE_EMPTY,
  46   RESPONSE_INVALID,
  47   RESPONSE_VALID,
  48   RESPONSE_STATE_SIZE
  49 };
  50
  51 // Will log the supplied response |state|.
  52 void LogResponseState(SuggestionsResponseState state) {
  53   UMA_HISTOGRAM_ENUMERATION("Suggestions.ResponseState", state,
  54                             RESPONSE_STATE_SIZE);
  55 }
  56
  57 GURL BuildBlacklistRequestURL(const std::string& blacklist_url_prefix,
  58                               const GURL& candidate_url) {
  59   return GURL(blacklist_url_prefix +
  60               net::EscapeQueryParamValue(candidate_url.spec(), true));
  61 }
  62
  63 // Runs each callback in |requestors| on |suggestions|, then deallocates
  64 // |requestors|.
  65 void DispatchRequestsAndClear(
  66     const SuggestionsProfile& suggestions,
  67     std::vector<SuggestionsService::ResponseCallback>* requestors) {
  68   std::vector<SuggestionsService::ResponseCallback> temp_requestors;
  69   temp_requestors.swap(*requestors);
  70   std::vector<SuggestionsService::ResponseCallback>::iterator it;
  71   for (it = temp_requestors.begin(); it != temp_requestors.end(); ++it) {
  72     if (!it->is_null()) it->Run(suggestions);
  73   }
  74 }
  75
  76 // Default delay used when scheduling a request.
  77 const int kDefaultSchedulingDelaySec = 1;
  78
  79 // Multiplier on the delay used when re-scheduling a failed request.
  80 const int kSchedulingBackoffMultiplier = 2;
  81
  82 // Maximum valid delay for scheduling a request. Candidate delays larger than
  83 // this are rejected. This means the maximum backoff is at least 5 / 2 minutes.
  84 const int kSchedulingMaxDelaySec = 5 * 60;
  85
  86 const char kFaviconURL[] =
  87     "https://s2.googleusercontent.com/s2/favicons?domain_url=%s&alt=s&sz=32";
  88
  89 const char kPingURL[] =
  90     "https://www.google.com/chromesuggestions/click?q=%lld&cd=%d";
  91 }  // namespace
  92
  93 // TODO(mathp): Put this in TemplateURL.
  94 // TODO(fserb): Add logic to decide the device type of the request.
  95 #if defined(OS_ANDROID) || defined(OS_IOS)
  96 const char kSuggestionsURL[] = "https://www.google.com/chromesuggestions?t=2";
  97 const char kSuggestionsBlacklistURLPrefix[] =
  98     "https://www.google.com/chromesuggestions/blacklist?t=2&url=";
  99 const char kSuggestionsBlacklistClearURL[] =
 100     "https://www.google.com/chromesuggestions/blacklist/clear?t=2";
 101 #else
 102 const char kSuggestionsURL[] = "https://www.google.com/chromesuggestions?t=1";
 103 const char kSuggestionsBlacklistURLPrefix[] =
 104     "https://www.google.com/chromesuggestions/blacklist?t=1&url=";
 105 const char kSuggestionsBlacklistClearURL[] =
 106     "https://www.google.com/chromesuggestions/blacklist/clear?t=1";
 107 #endif
 108 const char kSuggestionsBlacklistURLParam[] = "url";
 109
 110 // The default expiry timeout is 168 hours.
 111 const int64 kDefaultExpiryUsec = 168 * base::Time::kMicrosecondsPerHour;
 112
 113 SuggestionsService::SuggestionsService(
 114     net::URLRequestContextGetter* url_request_context,
 115     scoped_ptr<SuggestionsStore> suggestions_store,
 116     scoped_ptr<ImageManager> thumbnail_manager,
 117     scoped_ptr<BlacklistStore> blacklist_store)
 118     : url_request_context_(url_request_context),
 119       suggestions_store_(suggestions_store.Pass()),
 120       thumbnail_manager_(thumbnail_manager.Pass()),
 121       blacklist_store_(blacklist_store.Pass()),
 122       scheduling_delay_(TimeDelta::FromSeconds(kDefaultSchedulingDelaySec)),
 123       suggestions_url_(kSuggestionsURL),
 124       blacklist_url_prefix_(kSuggestionsBlacklistURLPrefix),
 125       weak_ptr_factory_(this) {}
 126
 127 SuggestionsService::~SuggestionsService() {}
 128
 129 void SuggestionsService::FetchSuggestionsData(
 130     SyncState sync_state,
 131     SuggestionsService::ResponseCallback callback) {
 132   DCHECK(thread_checker_.CalledOnValidThread());
 133   waiting_requestors_.push_back(callback);
 134   if (sync_state == SYNC_OR_HISTORY_SYNC_DISABLED) {
 135     // Cancel any ongoing request, to stop interacting with the server.
 136     pending_request_.reset(NULL);
 137     suggestions_store_->ClearSuggestions();
 138     DispatchRequestsAndClear(SuggestionsProfile(), &waiting_requestors_);
 139   } else if (sync_state == INITIALIZED_ENABLED_HISTORY ||
 140              sync_state == NOT_INITIALIZED_ENABLED) {
 141     // Sync is enabled. Serve previously cached suggestions if available, else
 142     // an empty set of suggestions.
 143     ServeFromCache();
 144
 145     // Issue a network request to refresh the suggestions in the cache.
 146     IssueRequestIfNoneOngoing(suggestions_url_);
 147   } else {
 148     NOTREACHED();
 149   }
 150 }
 151
 152 void SuggestionsService::GetPageThumbnail(
 153     const GURL& url,
 154     const base::Callback<void(const GURL&, const SkBitmap*)>& callback) {
 155   thumbnail_manager_->GetImageForURL(url, callback);
 156 }
 157
 158 void SuggestionsService::GetPageThumbnailWithURL(
 159     const GURL& url,
 160     const GURL& thumbnail_url,
 161     const base::Callback<void(const GURL&, const SkBitmap*)>& callback) {
 162   thumbnail_manager_->AddImageURL(url, thumbnail_url);
 163   GetPageThumbnail(url, callback);
 164 }
 165
 166 void SuggestionsService::BlacklistURL(
 167     const GURL& candidate_url,
 168     const SuggestionsService::ResponseCallback& callback,
 169     const base::Closure& fail_callback) {
 170   DCHECK(thread_checker_.CalledOnValidThread());
 171
 172   if (!blacklist_store_->BlacklistUrl(candidate_url)) {
 173     fail_callback.Run();
 174     return;
 175   }
 176
 177   waiting_requestors_.push_back(callback);
 178   ServeFromCache();
 179   // Blacklist uploads are scheduled on any request completion, so only schedule
 180   // an upload if there is no ongoing request.
 181   if (!pending_request_.get()) {
 182     ScheduleBlacklistUpload();
 183   }
 184 }
 185
 186 void SuggestionsService::UndoBlacklistURL(
 187     const GURL& url,
 188     const SuggestionsService::ResponseCallback& callback,
 189     const base::Closure& fail_callback) {
 190   DCHECK(thread_checker_.CalledOnValidThread());
 191   TimeDelta time_delta;
 192   if (blacklist_store_->GetTimeUntilURLReadyForUpload(url, &time_delta) &&
 193       time_delta > TimeDelta::FromSeconds(0) &&
 194       blacklist_store_->RemoveUrl(url)) {
 195     // The URL was not yet candidate for upload to the server and could be
 196     // removed from the blacklist.
 197     waiting_requestors_.push_back(callback);
 198     ServeFromCache();
 199     return;
 200   }
 201   fail_callback.Run();
 202 }
 203
 204 void SuggestionsService::ClearBlacklist(const ResponseCallback& callback) {
 205   DCHECK(thread_checker_.CalledOnValidThread());
 206   blacklist_store_->ClearBlacklist();
 207   IssueRequestIfNoneOngoing(GURL(kSuggestionsBlacklistClearURL));
 208   waiting_requestors_.push_back(callback);
 209   ServeFromCache();
 210 }
 211
 212 // static
 213 bool SuggestionsService::GetBlacklistedUrl(const net::URLFetcher& request,
 214                                            GURL* url) {
 215   bool is_blacklist_request = base::StartsWith(
 216       request.GetOriginalURL().spec(), kSuggestionsBlacklistURLPrefix,
 217       base::CompareCase::SENSITIVE);
 218   if (!is_blacklist_request) return false;
 219
 220   // Extract the blacklisted URL from the blacklist request.
 221   std::string blacklisted;
 222   if (!net::GetValueForKeyInQuery(
 223           request.GetOriginalURL(),
 224           kSuggestionsBlacklistURLParam,
 225           &blacklisted)) {
 226     return false;
 227   }
 228
 229   GURL blacklisted_url(blacklisted);
 230   blacklisted_url.Swap(url);
 231   return true;
 232 }
 233
 234 // static
 235 void SuggestionsService::RegisterProfilePrefs(
 236     user_prefs::PrefRegistrySyncable* registry) {
 237   SuggestionsStore::RegisterProfilePrefs(registry);
 238   BlacklistStore::RegisterProfilePrefs(registry);
 239 }
 240
 241 void SuggestionsService::SetDefaultExpiryTimestamp(
 242     SuggestionsProfile* suggestions, int64 default_timestamp_usec) {
 243   for (int i = 0; i < suggestions->suggestions_size(); ++i) {
 244     ChromeSuggestion* suggestion = suggestions->mutable_suggestions(i);
 245     // Do not set expiry if the server has already provided a more specific
 246     // expiry time for this suggestion.
 247     if (!suggestion->has_expiry_ts()) {
 248       suggestion->set_expiry_ts(default_timestamp_usec);
 249     }
 250   }
 251 }
 252
 253 void SuggestionsService::IssueRequestIfNoneOngoing(const GURL& url) {
 254   // If there is an ongoing request, let it complete.
 255   if (pending_request_.get()) {
 256     return;
 257   }
 258   pending_request_ = CreateSuggestionsRequest(url);
 259   pending_request_->Start();
 260   last_request_started_time_ = TimeTicks::Now();
 261 }
 262
 263 scoped_ptr<net::URLFetcher> SuggestionsService::CreateSuggestionsRequest(
 264     const GURL& url) {
 265   scoped_ptr<net::URLFetcher> request =
 266       net::URLFetcher::Create(0, url, net::URLFetcher::GET, this);
 267   data_use_measurement::DataUseUserData::AttachToFetcher(
 268       request.get(), data_use_measurement::DataUseUserData::SUGGESTIONS);
 269   request->SetLoadFlags(net::LOAD_DISABLE_CACHE);
 270   request->SetRequestContext(url_request_context_);
 271   // Add Chrome experiment state to the request headers.
 272   net::HttpRequestHeaders headers;
 273   variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
 274       request->GetOriginalURL(), false, false, &headers);
 275   request->SetExtraRequestHeaders(headers.ToString());
 276   return request;
 277 }
 278
 279 void SuggestionsService::OnURLFetchComplete(const net::URLFetcher* source) {
 280   DCHECK(thread_checker_.CalledOnValidThread());
 281   DCHECK_EQ(pending_request_.get(), source);
 282
 283   // The fetcher will be deleted when the request is handled.
 284   scoped_ptr<const net::URLFetcher> request(pending_request_.release());
 285
 286   const net::URLRequestStatus& request_status = request->GetStatus();
 287   if (request_status.status() != net::URLRequestStatus::SUCCESS) {
 288     // This represents network errors (i.e. the server did not provide a
 289     // response).
 290     UMA_HISTOGRAM_SPARSE_SLOWLY("Suggestions.FailedRequestErrorCode",
 291                                 -request_status.error());
 292     DVLOG(1) << "Suggestions server request failed with error: "
 293              << request_status.error() << ": "
 294              << net::ErrorToString(request_status.error());
 295     UpdateBlacklistDelay(false);
 296     ScheduleBlacklistUpload();
 297     return;
 298   }
 299
 300   const int response_code = request->GetResponseCode();
 301   UMA_HISTOGRAM_SPARSE_SLOWLY("Suggestions.FetchResponseCode", response_code);
 302   if (response_code != net::HTTP_OK) {
 303     // A non-200 response code means that server has no (longer) suggestions for
 304     // this user. Aggressively clear the cache.
 305     suggestions_store_->ClearSuggestions();
 306     UpdateBlacklistDelay(false);
 307     ScheduleBlacklistUpload();
 308     return;
 309   }
 310
 311   const TimeDelta latency = TimeTicks::Now() - last_request_started_time_;
 312   UMA_HISTOGRAM_MEDIUM_TIMES("Suggestions.FetchSuccessLatency", latency);
 313
 314   // Handle a successful blacklisting.
 315   GURL blacklisted_url;
 316   if (GetBlacklistedUrl(*source, &blacklisted_url)) {
 317     blacklist_store_->RemoveUrl(blacklisted_url);
 318   }
 319
 320   std::string suggestions_data;
 321   bool success = request->GetResponseAsString(&suggestions_data);
 322   DCHECK(success);
 323
 324   // Parse the received suggestions and update the cache, or take proper action
 325   // in the case of invalid response.
 326   SuggestionsProfile suggestions;
 327   if (suggestions_data.empty()) {
 328     LogResponseState(RESPONSE_EMPTY);
 329     suggestions_store_->ClearSuggestions();
 330   } else if (suggestions.ParseFromString(suggestions_data)) {
 331     LogResponseState(RESPONSE_VALID);
 332     int64 now_usec = (base::Time::NowFromSystemTime() - base::Time::UnixEpoch())
 333         .ToInternalValue();
 334     SetDefaultExpiryTimestamp(&suggestions, now_usec + kDefaultExpiryUsec);
 335     PopulateExtraData(&suggestions);
 336     suggestions_store_->StoreSuggestions(suggestions);
 337   } else {
 338     LogResponseState(RESPONSE_INVALID);
 339   }
 340
 341   UpdateBlacklistDelay(true);
 342   ScheduleBlacklistUpload();
 343 }
 344
 345 void SuggestionsService::PopulateExtraData(SuggestionsProfile* suggestions) {
 346   for (int i = 0; i < suggestions->suggestions_size(); ++i) {
 347     suggestions::ChromeSuggestion* s = suggestions->mutable_suggestions(i);
 348     if (!s->has_favicon_url() || s->favicon_url().empty()) {
 349       s->set_favicon_url(base::StringPrintf(kFaviconURL, s->url().c_str()));
 350     }
 351     if (!s->has_impression_url() || s->impression_url().empty()) {
 352       s->set_impression_url(
 353           base::StringPrintf(
 354               kPingURL, static_cast<long long>(suggestions->timestamp()), -1));
 355     }
 356
 357     if (!s->has_click_url() || s->click_url().empty()) {
 358       s->set_click_url(base::StringPrintf(
 359           kPingURL, static_cast<long long>(suggestions->timestamp()), i));
 360     }
 361   }
 362 }
 363
 364 void SuggestionsService::Shutdown() {
 365   // Cancel pending request, then serve existing requestors from cache.
 366   pending_request_.reset(NULL);
 367   ServeFromCache();
 368 }
 369
 370 void SuggestionsService::ServeFromCache() {
 371   SuggestionsProfile suggestions;
 372   // In case of empty cache or error, |suggestions| stays empty.
 373   suggestions_store_->LoadSuggestions(&suggestions);
 374   thumbnail_manager_->Initialize(suggestions);
 375   FilterAndServe(&suggestions);
 376 }
 377
 378 void SuggestionsService::FilterAndServe(SuggestionsProfile* suggestions) {
 379   blacklist_store_->FilterSuggestions(suggestions);
 380   DispatchRequestsAndClear(*suggestions, &waiting_requestors_);
 381 }
 382
 383 void SuggestionsService::ScheduleBlacklistUpload() {
 384   DCHECK(thread_checker_.CalledOnValidThread());
 385   TimeDelta time_delta;
 386   if (blacklist_store_->GetTimeUntilReadyForUpload(&time_delta)) {
 387     // Blacklist cache is not empty: schedule.
 388     base::Closure blacklist_cb =
 389         base::Bind(&SuggestionsService::UploadOneFromBlacklist,
 390                    weak_ptr_factory_.GetWeakPtr());
 391     base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
 392         FROM_HERE, blacklist_cb, time_delta + scheduling_delay_);
 393   }
 394 }
 395
 396 void SuggestionsService::UploadOneFromBlacklist() {
 397   DCHECK(thread_checker_.CalledOnValidThread());
 398
 399   GURL blacklist_url;
 400   if (blacklist_store_->GetCandidateForUpload(&blacklist_url)) {
 401     // Issue a blacklisting request. Even if this request ends up not being sent
 402     // because of an ongoing request, a blacklist request is later scheduled.
 403     IssueRequestIfNoneOngoing(
 404         BuildBlacklistRequestURL(blacklist_url_prefix_, blacklist_url));
 405     return;
 406   }
 407
 408   // Even though there's no candidate for upload, the blacklist might not be
 409   // empty.
 410   ScheduleBlacklistUpload();
 411 }
 412
 413 void SuggestionsService::UpdateBlacklistDelay(bool last_request_successful) {
 414   DCHECK(thread_checker_.CalledOnValidThread());
 415
 416   if (last_request_successful) {
 417     scheduling_delay_ = TimeDelta::FromSeconds(kDefaultSchedulingDelaySec);
 418   } else {
 419     TimeDelta candidate_delay =
 420         scheduling_delay_ * kSchedulingBackoffMultiplier;
 421     if (candidate_delay < TimeDelta::FromSeconds(kSchedulingMaxDelaySec))
 422       scheduling_delay_ = candidate_delay;
 423   }
 424 }
 425
 426 }  // namespace suggestions