components/suggestions/suggestions_service.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/suggestions/suggestions_service.h"
   6
   7 #include <string>
   8
   9 #include "base/location.h"
  10 #include "base/memory/scoped_ptr.h"
  11 #include "base/metrics/histogram_macros.h"
  12 #include "base/metrics/sparse_histogram.h"
  13 #include "base/single_thread_task_runner.h"
  14 #include "base/strings/string_number_conversions.h"
  15 #include "base/strings/string_util.h"
  16 #include "base/thread_task_runner_handle.h"
  17 #include "base/time/time.h"
  18 #include "components/pref_registry/pref_registry_syncable.h"
  19 #include "components/suggestions/blacklist_store.h"
  20 #include "components/suggestions/suggestions_store.h"
  21 #include "components/variations/net/variations_http_header_provider.h"
  22 #include "net/base/escape.h"
  23 #include "net/base/load_flags.h"
  24 #include "net/base/net_errors.h"
  25 #include "net/base/url_util.h"
  26 #include "net/http/http_response_headers.h"
  27 #include "net/http/http_status_code.h"
  28 #include "net/http/http_util.h"
  29 #include "net/url_request/url_fetcher.h"
  30 #include "net/url_request/url_request_status.h"
  31 #include "url/gurl.h"
  32
  33 using base::CancelableClosure;
  34 using base::TimeDelta;
  35 using base::TimeTicks;
  36
  37 namespace suggestions {
  38
  39 namespace {
  40
  41 // Used to UMA log the state of the last response from the server.
  42 enum SuggestionsResponseState {
  43   RESPONSE_EMPTY,
  44   RESPONSE_INVALID,
  45   RESPONSE_VALID,
  46   RESPONSE_STATE_SIZE
  47 };
  48
  49 // Will log the supplied response |state|.
  50 void LogResponseState(SuggestionsResponseState state) {
  51   UMA_HISTOGRAM_ENUMERATION("Suggestions.ResponseState", state,
  52                             RESPONSE_STATE_SIZE);
  53 }
  54
  55 GURL BuildBlacklistRequestURL(const std::string& blacklist_url_prefix,
  56                               const GURL& candidate_url) {
  57   return GURL(blacklist_url_prefix +
  58               net::EscapeQueryParamValue(candidate_url.spec(), true));
  59 }
  60
  61 // Runs each callback in |requestors| on |suggestions|, then deallocates
  62 // |requestors|.
  63 void DispatchRequestsAndClear(
  64     const SuggestionsProfile& suggestions,
  65     std::vector<SuggestionsService::ResponseCallback>* requestors) {
  66   std::vector<SuggestionsService::ResponseCallback> temp_requestors;
  67   temp_requestors.swap(*requestors);
  68   std::vector<SuggestionsService::ResponseCallback>::iterator it;
  69   for (it = temp_requestors.begin(); it != temp_requestors.end(); ++it) {
  70     if (!it->is_null()) it->Run(suggestions);
  71   }
  72 }
  73
  74 // Default delay used when scheduling a request.
  75 const int kDefaultSchedulingDelaySec = 1;
  76
  77 // Multiplier on the delay used when re-scheduling a failed request.
  78 const int kSchedulingBackoffMultiplier = 2;
  79
  80 // Maximum valid delay for scheduling a request. Candidate delays larger than
  81 // this are rejected. This means the maximum backoff is at least 5 / 2 minutes.
  82 const int kSchedulingMaxDelaySec = 5 * 60;
  83
  84 }  // namespace
  85
  86 // TODO(mathp): Put this in TemplateURL.
  87 const char kSuggestionsURL[] = "https://www.google.com/chromesuggestions?t=2";
  88 const char kSuggestionsBlacklistURLPrefix[] =
  89     "https://www.google.com/chromesuggestions/blacklist?t=2&url=";
  90 const char kSuggestionsBlacklistURLParam[] = "url";
  91
  92 // The default expiry timeout is 72 hours.
  93 const int64 kDefaultExpiryUsec = 72 * base::Time::kMicrosecondsPerHour;
  94
  95 SuggestionsService::SuggestionsService(
  96     net::URLRequestContextGetter* url_request_context,
  97     scoped_ptr<SuggestionsStore> suggestions_store,
  98     scoped_ptr<ImageManager> thumbnail_manager,
  99     scoped_ptr<BlacklistStore> blacklist_store)
 100     : url_request_context_(url_request_context),
 101       suggestions_store_(suggestions_store.Pass()),
 102       thumbnail_manager_(thumbnail_manager.Pass()),
 103       blacklist_store_(blacklist_store.Pass()),
 104       scheduling_delay_(TimeDelta::FromSeconds(kDefaultSchedulingDelaySec)),
 105       suggestions_url_(kSuggestionsURL),
 106       blacklist_url_prefix_(kSuggestionsBlacklistURLPrefix),
 107       weak_ptr_factory_(this) {}
 108
 109 SuggestionsService::~SuggestionsService() {}
 110
 111 void SuggestionsService::FetchSuggestionsData(
 112     SyncState sync_state,
 113     SuggestionsService::ResponseCallback callback) {
 114   DCHECK(thread_checker_.CalledOnValidThread());
 115   waiting_requestors_.push_back(callback);
 116   if (sync_state == SYNC_OR_HISTORY_SYNC_DISABLED) {
 117     // Cancel any ongoing request, to stop interacting with the server.
 118     pending_request_.reset(NULL);
 119     suggestions_store_->ClearSuggestions();
 120     DispatchRequestsAndClear(SuggestionsProfile(), &waiting_requestors_);
 121   } else if (sync_state == INITIALIZED_ENABLED_HISTORY ||
 122              sync_state == NOT_INITIALIZED_ENABLED) {
 123     // Sync is enabled. Serve previously cached suggestions if available, else
 124     // an empty set of suggestions.
 125     ServeFromCache();
 126
 127     // Issue a network request to refresh the suggestions in the cache.
 128     IssueRequestIfNoneOngoing(suggestions_url_);
 129   } else {
 130     NOTREACHED();
 131   }
 132 }
 133
 134 void SuggestionsService::GetPageThumbnail(
 135     const GURL& url,
 136     base::Callback<void(const GURL&, const SkBitmap*)> callback) {
 137   thumbnail_manager_->GetImageForURL(url, callback);
 138 }
 139
 140 void SuggestionsService::BlacklistURL(
 141     const GURL& candidate_url,
 142     const SuggestionsService::ResponseCallback& callback,
 143     const base::Closure& fail_callback) {
 144   DCHECK(thread_checker_.CalledOnValidThread());
 145
 146   if (!blacklist_store_->BlacklistUrl(candidate_url)) {
 147     fail_callback.Run();
 148     return;
 149   }
 150
 151   waiting_requestors_.push_back(callback);
 152   ServeFromCache();
 153   // Blacklist uploads are scheduled on any request completion, so only schedule
 154   // an upload if there is no ongoing request.
 155   if (!pending_request_.get()) {
 156     ScheduleBlacklistUpload();
 157   }
 158 }
 159
 160 void SuggestionsService::UndoBlacklistURL(
 161     const GURL& url,
 162     const SuggestionsService::ResponseCallback& callback,
 163     const base::Closure& fail_callback) {
 164   DCHECK(thread_checker_.CalledOnValidThread());
 165   TimeDelta time_delta;
 166   if (blacklist_store_->GetTimeUntilURLReadyForUpload(url, &time_delta) &&
 167       time_delta > TimeDelta::FromSeconds(0) &&
 168       blacklist_store_->RemoveUrl(url)) {
 169     // The URL was not yet candidate for upload to the server and could be
 170     // removed from the blacklist.
 171     waiting_requestors_.push_back(callback);
 172     ServeFromCache();
 173     return;
 174   }
 175   fail_callback.Run();
 176 }
 177
 178 // static
 179 bool SuggestionsService::GetBlacklistedUrl(const net::URLFetcher& request,
 180                                            GURL* url) {
 181   bool is_blacklist_request = base::StartsWithASCII(
 182       request.GetOriginalURL().spec(), kSuggestionsBlacklistURLPrefix, true);
 183   if (!is_blacklist_request) return false;
 184
 185   // Extract the blacklisted URL from the blacklist request.
 186   std::string blacklisted;
 187   if (!net::GetValueForKeyInQuery(
 188           request.GetOriginalURL(),
 189           kSuggestionsBlacklistURLParam,
 190           &blacklisted)) {
 191     return false;
 192   }
 193
 194   GURL blacklisted_url(blacklisted);
 195   blacklisted_url.Swap(url);
 196   return true;
 197 }
 198
 199 // static
 200 void SuggestionsService::RegisterProfilePrefs(
 201     user_prefs::PrefRegistrySyncable* registry) {
 202   SuggestionsStore::RegisterProfilePrefs(registry);
 203   BlacklistStore::RegisterProfilePrefs(registry);
 204 }
 205
 206 void SuggestionsService::SetDefaultExpiryTimestamp(
 207     SuggestionsProfile* suggestions, int64 default_timestamp_usec) {
 208   for (int i = 0; i < suggestions->suggestions_size(); ++i) {
 209     ChromeSuggestion* suggestion = suggestions->mutable_suggestions(i);
 210     // Do not set expiry if the server has already provided a more specific
 211     // expiry time for this suggestion.
 212     if (!suggestion->has_expiry_ts()) {
 213       suggestion->set_expiry_ts(default_timestamp_usec);
 214     }
 215   }
 216 }
 217
 218 void SuggestionsService::IssueRequestIfNoneOngoing(const GURL& url) {
 219   // If there is an ongoing request, let it complete.
 220   if (pending_request_.get()) {
 221     return;
 222   }
 223   pending_request_ = CreateSuggestionsRequest(url);
 224   pending_request_->Start();
 225   last_request_started_time_ = TimeTicks::Now();
 226 }
 227
 228 scoped_ptr<net::URLFetcher> SuggestionsService::CreateSuggestionsRequest(
 229     const GURL& url) {
 230   scoped_ptr<net::URLFetcher> request =
 231       net::URLFetcher::Create(0, url, net::URLFetcher::GET, this);
 232   request->SetLoadFlags(net::LOAD_DISABLE_CACHE);
 233   request->SetRequestContext(url_request_context_);
 234   // Add Chrome experiment state to the request headers.
 235   net::HttpRequestHeaders headers;
 236   variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
 237       request->GetOriginalURL(), false, false, &headers);
 238   request->SetExtraRequestHeaders(headers.ToString());
 239   return request;
 240 }
 241
 242 void SuggestionsService::OnURLFetchComplete(const net::URLFetcher* source) {
 243   DCHECK(thread_checker_.CalledOnValidThread());
 244   DCHECK_EQ(pending_request_.get(), source);
 245
 246   // The fetcher will be deleted when the request is handled.
 247   scoped_ptr<const net::URLFetcher> request(pending_request_.release());
 248
 249   const net::URLRequestStatus& request_status = request->GetStatus();
 250   if (request_status.status() != net::URLRequestStatus::SUCCESS) {
 251     // This represents network errors (i.e. the server did not provide a
 252     // response).
 253     UMA_HISTOGRAM_SPARSE_SLOWLY("Suggestions.FailedRequestErrorCode",
 254                                 -request_status.error());
 255     DVLOG(1) << "Suggestions server request failed with error: "
 256              << request_status.error() << ": "
 257              << net::ErrorToString(request_status.error());
 258     UpdateBlacklistDelay(false);
 259     ScheduleBlacklistUpload();
 260     return;
 261   }
 262
 263   const int response_code = request->GetResponseCode();
 264   UMA_HISTOGRAM_SPARSE_SLOWLY("Suggestions.FetchResponseCode", response_code);
 265   if (response_code != net::HTTP_OK) {
 266     // A non-200 response code means that server has no (longer) suggestions for
 267     // this user. Aggressively clear the cache.
 268     suggestions_store_->ClearSuggestions();
 269     UpdateBlacklistDelay(false);
 270     ScheduleBlacklistUpload();
 271     return;
 272   }
 273
 274   const TimeDelta latency = TimeTicks::Now() - last_request_started_time_;
 275   UMA_HISTOGRAM_MEDIUM_TIMES("Suggestions.FetchSuccessLatency", latency);
 276
 277   // Handle a successful blacklisting.
 278   GURL blacklisted_url;
 279   if (GetBlacklistedUrl(*source, &blacklisted_url)) {
 280     blacklist_store_->RemoveUrl(blacklisted_url);
 281   }
 282
 283   std::string suggestions_data;
 284   bool success = request->GetResponseAsString(&suggestions_data);
 285   DCHECK(success);
 286
 287   // Parse the received suggestions and update the cache, or take proper action
 288   // in the case of invalid response.
 289   SuggestionsProfile suggestions;
 290   if (suggestions_data.empty()) {
 291     LogResponseState(RESPONSE_EMPTY);
 292     suggestions_store_->ClearSuggestions();
 293   } else if (suggestions.ParseFromString(suggestions_data)) {
 294     LogResponseState(RESPONSE_VALID);
 295     int64 now_usec = (base::Time::NowFromSystemTime() - base::Time::UnixEpoch())
 296         .ToInternalValue();
 297     SetDefaultExpiryTimestamp(&suggestions, now_usec + kDefaultExpiryUsec);
 298     suggestions_store_->StoreSuggestions(suggestions);
 299   } else {
 300     LogResponseState(RESPONSE_INVALID);
 301   }
 302
 303   UpdateBlacklistDelay(true);
 304   ScheduleBlacklistUpload();
 305 }
 306
 307 void SuggestionsService::Shutdown() {
 308   // Cancel pending request, then serve existing requestors from cache.
 309   pending_request_.reset(NULL);
 310   ServeFromCache();
 311 }
 312
 313 void SuggestionsService::ServeFromCache() {
 314   SuggestionsProfile suggestions;
 315   // In case of empty cache or error, |suggestions| stays empty.
 316   suggestions_store_->LoadSuggestions(&suggestions);
 317   thumbnail_manager_->Initialize(suggestions);
 318   FilterAndServe(&suggestions);
 319 }
 320
 321 void SuggestionsService::FilterAndServe(SuggestionsProfile* suggestions) {
 322   blacklist_store_->FilterSuggestions(suggestions);
 323   DispatchRequestsAndClear(*suggestions, &waiting_requestors_);
 324 }
 325
 326 void SuggestionsService::ScheduleBlacklistUpload() {
 327   DCHECK(thread_checker_.CalledOnValidThread());
 328   TimeDelta time_delta;
 329   if (blacklist_store_->GetTimeUntilReadyForUpload(&time_delta)) {
 330     // Blacklist cache is not empty: schedule.
 331     base::Closure blacklist_cb =
 332         base::Bind(&SuggestionsService::UploadOneFromBlacklist,
 333                    weak_ptr_factory_.GetWeakPtr());
 334     base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
 335         FROM_HERE, blacklist_cb, time_delta + scheduling_delay_);
 336   }
 337 }
 338
 339 void SuggestionsService::UploadOneFromBlacklist() {
 340   DCHECK(thread_checker_.CalledOnValidThread());
 341
 342   GURL blacklist_url;
 343   if (blacklist_store_->GetCandidateForUpload(&blacklist_url)) {
 344     // Issue a blacklisting request. Even if this request ends up not being sent
 345     // because of an ongoing request, a blacklist request is later scheduled.
 346     IssueRequestIfNoneOngoing(
 347         BuildBlacklistRequestURL(blacklist_url_prefix_, blacklist_url));
 348     return;
 349   }
 350
 351   // Even though there's no candidate for upload, the blacklist might not be
 352   // empty.
 353   ScheduleBlacklistUpload();
 354 }
 355
 356 void SuggestionsService::UpdateBlacklistDelay(bool last_request_successful) {
 357   DCHECK(thread_checker_.CalledOnValidThread());
 358
 359   if (last_request_successful) {
 360     scheduling_delay_ = TimeDelta::FromSeconds(kDefaultSchedulingDelaySec);
 361   } else {
 362     TimeDelta candidate_delay =
 363         scheduling_delay_ * kSchedulingBackoffMultiplier;
 364     if (candidate_delay < TimeDelta::FromSeconds(kSchedulingMaxDelaySec))
 365       scheduling_delay_ = candidate_delay;
 366   }
 367 }
 368
 369 }  // namespace suggestions