components/suggestions/suggestions_service.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/suggestions/suggestions_service.h"
   6
   7 #include <string>
   8
   9 #include "base/memory/scoped_ptr.h"
  10 #include "base/message_loop/message_loop_proxy.h"
  11 #include "base/metrics/histogram.h"
  12 #include "base/metrics/sparse_histogram.h"
  13 #include "base/strings/string_number_conversions.h"
  14 #include "base/strings/string_util.h"
  15 #include "base/time/time.h"
  16 #include "components/pref_registry/pref_registry_syncable.h"
  17 #include "components/suggestions/blacklist_store.h"
  18 #include "components/suggestions/suggestions_store.h"
  19 #include "components/variations/net/variations_http_header_provider.h"
  20 #include "net/base/escape.h"
  21 #include "net/base/load_flags.h"
  22 #include "net/base/net_errors.h"
  23 #include "net/base/url_util.h"
  24 #include "net/http/http_response_headers.h"
  25 #include "net/http/http_status_code.h"
  26 #include "net/http/http_util.h"
  27 #include "net/url_request/url_fetcher.h"
  28 #include "net/url_request/url_request_status.h"
  29 #include "url/gurl.h"
  30
  31 using base::CancelableClosure;
  32 using base::TimeDelta;
  33 using base::TimeTicks;
  34
  35 namespace suggestions {
  36
  37 namespace {
  38
  39 // Used to UMA log the state of the last response from the server.
  40 enum SuggestionsResponseState {
  41   RESPONSE_EMPTY,
  42   RESPONSE_INVALID,
  43   RESPONSE_VALID,
  44   RESPONSE_STATE_SIZE
  45 };
  46
  47 // Will log the supplied response |state|.
  48 void LogResponseState(SuggestionsResponseState state) {
  49   UMA_HISTOGRAM_ENUMERATION("Suggestions.ResponseState", state,
  50                             RESPONSE_STATE_SIZE);
  51 }
  52
  53 GURL BuildBlacklistRequestURL(const std::string& blacklist_url_prefix,
  54                               const GURL& candidate_url) {
  55   return GURL(blacklist_url_prefix +
  56               net::EscapeQueryParamValue(candidate_url.spec(), true));
  57 }
  58
  59 // Runs each callback in |requestors| on |suggestions|, then deallocates
  60 // |requestors|.
  61 void DispatchRequestsAndClear(
  62     const SuggestionsProfile& suggestions,
  63     std::vector<SuggestionsService::ResponseCallback>* requestors) {
  64   std::vector<SuggestionsService::ResponseCallback> temp_requestors;
  65   temp_requestors.swap(*requestors);
  66   std::vector<SuggestionsService::ResponseCallback>::iterator it;
  67   for (it = temp_requestors.begin(); it != temp_requestors.end(); ++it) {
  68     if (!it->is_null()) it->Run(suggestions);
  69   }
  70 }
  71
  72 // Default delay used when scheduling a request.
  73 const int kDefaultSchedulingDelaySec = 1;
  74
  75 // Multiplier on the delay used when re-scheduling a failed request.
  76 const int kSchedulingBackoffMultiplier = 2;
  77
  78 // Maximum valid delay for scheduling a request. Candidate delays larger than
  79 // this are rejected. This means the maximum backoff is at least 5 / 2 minutes.
  80 const int kSchedulingMaxDelaySec = 5 * 60;
  81
  82 }  // namespace
  83
  84 // TODO(mathp): Put this in TemplateURL.
  85 const char kSuggestionsURL[] = "https://www.google.com/chromesuggestions?t=2";
  86 const char kSuggestionsBlacklistURLPrefix[] =
  87     "https://www.google.com/chromesuggestions/blacklist?t=2&url=";
  88 const char kSuggestionsBlacklistURLParam[] = "url";
  89
  90 // The default expiry timeout is 72 hours.
  91 const int64 kDefaultExpiryUsec = 72 * base::Time::kMicrosecondsPerHour;
  92
  93 SuggestionsService::SuggestionsService(
  94     net::URLRequestContextGetter* url_request_context,
  95     scoped_ptr<SuggestionsStore> suggestions_store,
  96     scoped_ptr<ImageManager> thumbnail_manager,
  97     scoped_ptr<BlacklistStore> blacklist_store)
  98     : url_request_context_(url_request_context),
  99       suggestions_store_(suggestions_store.Pass()),
 100       thumbnail_manager_(thumbnail_manager.Pass()),
 101       blacklist_store_(blacklist_store.Pass()),
 102       scheduling_delay_(TimeDelta::FromSeconds(kDefaultSchedulingDelaySec)),
 103       suggestions_url_(kSuggestionsURL),
 104       blacklist_url_prefix_(kSuggestionsBlacklistURLPrefix),
 105       weak_ptr_factory_(this) {}
 106
 107 SuggestionsService::~SuggestionsService() {}
 108
 109 void SuggestionsService::FetchSuggestionsData(
 110     SyncState sync_state,
 111     SuggestionsService::ResponseCallback callback) {
 112   DCHECK(thread_checker_.CalledOnValidThread());
 113   waiting_requestors_.push_back(callback);
 114   if (sync_state == SYNC_OR_HISTORY_SYNC_DISABLED) {
 115     // Cancel any ongoing request, to stop interacting with the server.
 116     pending_request_.reset(NULL);
 117     suggestions_store_->ClearSuggestions();
 118     DispatchRequestsAndClear(SuggestionsProfile(), &waiting_requestors_);
 119   } else if (sync_state == INITIALIZED_ENABLED_HISTORY ||
 120              sync_state == NOT_INITIALIZED_ENABLED) {
 121     // Sync is enabled. Serve previously cached suggestions if available, else
 122     // an empty set of suggestions.
 123     ServeFromCache();
 124
 125     // Issue a network request to refresh the suggestions in the cache.
 126     IssueRequestIfNoneOngoing(suggestions_url_);
 127   } else {
 128     NOTREACHED();
 129   }
 130 }
 131
 132 void SuggestionsService::GetPageThumbnail(
 133     const GURL& url,
 134     base::Callback<void(const GURL&, const SkBitmap*)> callback) {
 135   thumbnail_manager_->GetImageForURL(url, callback);
 136 }
 137
 138 void SuggestionsService::BlacklistURL(
 139     const GURL& candidate_url,
 140     const SuggestionsService::ResponseCallback& callback,
 141     const base::Closure& fail_callback) {
 142   DCHECK(thread_checker_.CalledOnValidThread());
 143
 144   if (!blacklist_store_->BlacklistUrl(candidate_url)) {
 145     fail_callback.Run();
 146     return;
 147   }
 148
 149   waiting_requestors_.push_back(callback);
 150   ServeFromCache();
 151   // Blacklist uploads are scheduled on any request completion, so only schedule
 152   // an upload if there is no ongoing request.
 153   if (!pending_request_.get()) {
 154     ScheduleBlacklistUpload();
 155   }
 156 }
 157
 158 void SuggestionsService::UndoBlacklistURL(
 159     const GURL& url,
 160     const SuggestionsService::ResponseCallback& callback,
 161     const base::Closure& fail_callback) {
 162   DCHECK(thread_checker_.CalledOnValidThread());
 163   TimeDelta time_delta;
 164   if (blacklist_store_->GetTimeUntilURLReadyForUpload(url, &time_delta) &&
 165       time_delta > TimeDelta::FromSeconds(0) &&
 166       blacklist_store_->RemoveUrl(url)) {
 167     // The URL was not yet candidate for upload to the server and could be
 168     // removed from the blacklist.
 169     waiting_requestors_.push_back(callback);
 170     ServeFromCache();
 171     return;
 172   }
 173   fail_callback.Run();
 174 }
 175
 176 // static
 177 bool SuggestionsService::GetBlacklistedUrl(const net::URLFetcher& request,
 178                                            GURL* url) {
 179   bool is_blacklist_request = StartsWithASCII(request.GetOriginalURL().spec(),
 180                                               kSuggestionsBlacklistURLPrefix,
 181                                               true);
 182   if (!is_blacklist_request) return false;
 183
 184   // Extract the blacklisted URL from the blacklist request.
 185   std::string blacklisted;
 186   if (!net::GetValueForKeyInQuery(
 187           request.GetOriginalURL(),
 188           kSuggestionsBlacklistURLParam,
 189           &blacklisted)) {
 190     return false;
 191   }
 192
 193   GURL blacklisted_url(blacklisted);
 194   blacklisted_url.Swap(url);
 195   return true;
 196 }
 197
 198 // static
 199 void SuggestionsService::RegisterProfilePrefs(
 200     user_prefs::PrefRegistrySyncable* registry) {
 201   SuggestionsStore::RegisterProfilePrefs(registry);
 202   BlacklistStore::RegisterProfilePrefs(registry);
 203 }
 204
 205 void SuggestionsService::SetDefaultExpiryTimestamp(
 206     SuggestionsProfile* suggestions, int64 default_timestamp_usec) {
 207   for (int i = 0; i < suggestions->suggestions_size(); ++i) {
 208     ChromeSuggestion* suggestion = suggestions->mutable_suggestions(i);
 209     // Do not set expiry if the server has already provided a more specific
 210     // expiry time for this suggestion.
 211     if (!suggestion->has_expiry_ts()) {
 212       suggestion->set_expiry_ts(default_timestamp_usec);
 213     }
 214   }
 215 }
 216
 217 void SuggestionsService::IssueRequestIfNoneOngoing(const GURL& url) {
 218   // If there is an ongoing request, let it complete.
 219   if (pending_request_.get()) {
 220     return;
 221   }
 222   pending_request_ = CreateSuggestionsRequest(url);
 223   pending_request_->Start();
 224   last_request_started_time_ = TimeTicks::Now();
 225 }
 226
 227 scoped_ptr<net::URLFetcher> SuggestionsService::CreateSuggestionsRequest(
 228     const GURL& url) {
 229   scoped_ptr<net::URLFetcher> request =
 230       net::URLFetcher::Create(0, url, net::URLFetcher::GET, this);
 231   request->SetLoadFlags(net::LOAD_DISABLE_CACHE);
 232   request->SetRequestContext(url_request_context_);
 233   // Add Chrome experiment state to the request headers.
 234   net::HttpRequestHeaders headers;
 235   variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
 236       request->GetOriginalURL(), false, false, &headers);
 237   request->SetExtraRequestHeaders(headers.ToString());
 238   return request;
 239 }
 240
 241 void SuggestionsService::OnURLFetchComplete(const net::URLFetcher* source) {
 242   DCHECK(thread_checker_.CalledOnValidThread());
 243   DCHECK_EQ(pending_request_.get(), source);
 244
 245   // The fetcher will be deleted when the request is handled.
 246   scoped_ptr<const net::URLFetcher> request(pending_request_.release());
 247
 248   const net::URLRequestStatus& request_status = request->GetStatus();
 249   if (request_status.status() != net::URLRequestStatus::SUCCESS) {
 250     // This represents network errors (i.e. the server did not provide a
 251     // response).
 252     UMA_HISTOGRAM_SPARSE_SLOWLY("Suggestions.FailedRequestErrorCode",
 253                                 -request_status.error());
 254     DVLOG(1) << "Suggestions server request failed with error: "
 255              << request_status.error() << ": "
 256              << net::ErrorToString(request_status.error());
 257     UpdateBlacklistDelay(false);
 258     ScheduleBlacklistUpload();
 259     return;
 260   }
 261
 262   const int response_code = request->GetResponseCode();
 263   UMA_HISTOGRAM_SPARSE_SLOWLY("Suggestions.FetchResponseCode", response_code);
 264   if (response_code != net::HTTP_OK) {
 265     // A non-200 response code means that server has no (longer) suggestions for
 266     // this user. Aggressively clear the cache.
 267     suggestions_store_->ClearSuggestions();
 268     UpdateBlacklistDelay(false);
 269     ScheduleBlacklistUpload();
 270     return;
 271   }
 272
 273   const TimeDelta latency = TimeTicks::Now() - last_request_started_time_;
 274   UMA_HISTOGRAM_MEDIUM_TIMES("Suggestions.FetchSuccessLatency", latency);
 275
 276   // Handle a successful blacklisting.
 277   GURL blacklisted_url;
 278   if (GetBlacklistedUrl(*source, &blacklisted_url)) {
 279     blacklist_store_->RemoveUrl(blacklisted_url);
 280   }
 281
 282   std::string suggestions_data;
 283   bool success = request->GetResponseAsString(&suggestions_data);
 284   DCHECK(success);
 285
 286   // Parse the received suggestions and update the cache, or take proper action
 287   // in the case of invalid response.
 288   SuggestionsProfile suggestions;
 289   if (suggestions_data.empty()) {
 290     LogResponseState(RESPONSE_EMPTY);
 291     suggestions_store_->ClearSuggestions();
 292   } else if (suggestions.ParseFromString(suggestions_data)) {
 293     LogResponseState(RESPONSE_VALID);
 294     int64 now_usec = (base::Time::NowFromSystemTime() - base::Time::UnixEpoch())
 295         .ToInternalValue();
 296     SetDefaultExpiryTimestamp(&suggestions, now_usec + kDefaultExpiryUsec);
 297     suggestions_store_->StoreSuggestions(suggestions);
 298   } else {
 299     LogResponseState(RESPONSE_INVALID);
 300   }
 301
 302   UpdateBlacklistDelay(true);
 303   ScheduleBlacklistUpload();
 304 }
 305
 306 void SuggestionsService::Shutdown() {
 307   // Cancel pending request, then serve existing requestors from cache.
 308   pending_request_.reset(NULL);
 309   ServeFromCache();
 310 }
 311
 312 void SuggestionsService::ServeFromCache() {
 313   SuggestionsProfile suggestions;
 314   // In case of empty cache or error, |suggestions| stays empty.
 315   suggestions_store_->LoadSuggestions(&suggestions);
 316   thumbnail_manager_->Initialize(suggestions);
 317   FilterAndServe(&suggestions);
 318 }
 319
 320 void SuggestionsService::FilterAndServe(SuggestionsProfile* suggestions) {
 321   blacklist_store_->FilterSuggestions(suggestions);
 322   DispatchRequestsAndClear(*suggestions, &waiting_requestors_);
 323 }
 324
 325 void SuggestionsService::ScheduleBlacklistUpload() {
 326   DCHECK(thread_checker_.CalledOnValidThread());
 327   TimeDelta time_delta;
 328   if (blacklist_store_->GetTimeUntilReadyForUpload(&time_delta)) {
 329     // Blacklist cache is not empty: schedule.
 330     base::Closure blacklist_cb =
 331         base::Bind(&SuggestionsService::UploadOneFromBlacklist,
 332                    weak_ptr_factory_.GetWeakPtr());
 333     base::MessageLoopProxy::current()->PostDelayedTask(
 334         FROM_HERE, blacklist_cb, time_delta + scheduling_delay_);
 335   }
 336 }
 337
 338 void SuggestionsService::UploadOneFromBlacklist() {
 339   DCHECK(thread_checker_.CalledOnValidThread());
 340
 341   GURL blacklist_url;
 342   if (blacklist_store_->GetCandidateForUpload(&blacklist_url)) {
 343     // Issue a blacklisting request. Even if this request ends up not being sent
 344     // because of an ongoing request, a blacklist request is later scheduled.
 345     IssueRequestIfNoneOngoing(
 346         BuildBlacklistRequestURL(blacklist_url_prefix_, blacklist_url));
 347     return;
 348   }
 349
 350   // Even though there's no candidate for upload, the blacklist might not be
 351   // empty.
 352   ScheduleBlacklistUpload();
 353 }
 354
 355 void SuggestionsService::UpdateBlacklistDelay(bool last_request_successful) {
 356   DCHECK(thread_checker_.CalledOnValidThread());
 357
 358   if (last_request_successful) {
 359     scheduling_delay_ = TimeDelta::FromSeconds(kDefaultSchedulingDelaySec);
 360   } else {
 361     TimeDelta candidate_delay =
 362         scheduling_delay_ * kSchedulingBackoffMultiplier;
 363     if (candidate_delay < TimeDelta::FromSeconds(kSchedulingMaxDelaySec))
 364       scheduling_delay_ = candidate_delay;
 365   }
 366 }
 367
 368 }  // namespace suggestions