components/suggestions/suggestions_service.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/suggestions/suggestions_service.h"
   6
   7 #include <sstream>
   8 #include <string>
   9
  10 #include "base/memory/scoped_ptr.h"
  11 #include "base/message_loop/message_loop_proxy.h"
  12 #include "base/metrics/histogram.h"
  13 #include "base/metrics/sparse_histogram.h"
  14 #include "base/strings/string_number_conversions.h"
  15 #include "base/strings/string_util.h"
  16 #include "base/time/time.h"
  17 #include "components/pref_registry/pref_registry_syncable.h"
  18 #include "components/suggestions/blacklist_store.h"
  19 #include "components/suggestions/suggestions_store.h"
  20 #include "components/variations/variations_associated_data.h"
  21 #include "components/variations/variations_http_header_provider.h"
  22 #include "net/base/escape.h"
  23 #include "net/base/load_flags.h"
  24 #include "net/base/net_errors.h"
  25 #include "net/base/url_util.h"
  26 #include "net/http/http_response_headers.h"
  27 #include "net/http/http_status_code.h"
  28 #include "net/http/http_util.h"
  29 #include "net/url_request/url_fetcher.h"
  30 #include "net/url_request/url_request_status.h"
  31 #include "url/gurl.h"
  32
  33 using base::CancelableClosure;
  34
  35 namespace suggestions {
  36
  37 namespace {
  38
  39 // Used to UMA log the state of the last response from the server.
  40 enum SuggestionsResponseState {
  41   RESPONSE_EMPTY,
  42   RESPONSE_INVALID,
  43   RESPONSE_VALID,
  44   RESPONSE_STATE_SIZE
  45 };
  46
  47 // Will log the supplied response |state|.
  48 void LogResponseState(SuggestionsResponseState state) {
  49   UMA_HISTOGRAM_ENUMERATION("Suggestions.ResponseState", state,
  50                             RESPONSE_STATE_SIZE);
  51 }
  52
  53 // Obtains the experiment parameter under the supplied |key|, or empty string
  54 // if the parameter does not exist.
  55 std::string GetExperimentParam(const std::string& key) {
  56   return variations::GetVariationParamValue(kSuggestionsFieldTrialName, key);
  57 }
  58
  59 GURL BuildBlacklistRequestURL(const std::string& blacklist_url_prefix,
  60                               const GURL& candidate_url) {
  61   return GURL(blacklist_url_prefix +
  62               net::EscapeQueryParamValue(candidate_url.spec(), true));
  63 }
  64
  65 // Runs each callback in |requestors| on |suggestions|, then deallocates
  66 // |requestors|.
  67 void DispatchRequestsAndClear(
  68     const SuggestionsProfile& suggestions,
  69     std::vector<SuggestionsService::ResponseCallback>* requestors) {
  70   std::vector<SuggestionsService::ResponseCallback>::iterator it;
  71   for (it = requestors->begin(); it != requestors->end(); ++it) {
  72     if (!it->is_null()) it->Run(suggestions);
  73   }
  74   std::vector<SuggestionsService::ResponseCallback>().swap(*requestors);
  75 }
  76
  77 const int kDefaultRequestTimeoutMs = 200;
  78
  79 // Default delay used when scheduling a blacklist request.
  80 const int kBlacklistDefaultDelaySec = 1;
  81
  82 // Multiplier on the delay used when scheduling a blacklist request, in case the
  83 // last observed request was unsuccessful.
  84 const int kBlacklistBackoffMultiplier = 2;
  85
  86 // Maximum valid delay for scheduling a request. Candidate delays larger than
  87 // this are rejected. This means the maximum backoff is at least 300 / 2, i.e.
  88 // 2.5 minutes.
  89 const int kBlacklistMaxDelaySec = 300;  // 5 minutes
  90
  91 }  // namespace
  92
  93 const char kSuggestionsFieldTrialName[] = "ChromeSuggestions";
  94 const char kSuggestionsFieldTrialURLParam[] = "url";
  95 const char kSuggestionsFieldTrialCommonParamsParam[] = "common_params";
  96 const char kSuggestionsFieldTrialBlacklistPathParam[] = "blacklist_path";
  97 const char kSuggestionsFieldTrialBlacklistUrlParam[] = "blacklist_url_param";
  98 const char kSuggestionsFieldTrialStateParam[] = "state";
  99 const char kSuggestionsFieldTrialControlParam[] = "control";
 100 const char kSuggestionsFieldTrialStateEnabled[] = "enabled";
 101 const char kSuggestionsFieldTrialTimeoutMs[] = "timeout_ms";
 102
 103 // The default expiry timeout is 72 hours.
 104 const int64 kDefaultExpiryUsec = 72 * base::Time::kMicrosecondsPerHour;
 105
 106 namespace {
 107
 108 std::string GetBlacklistUrlPrefix() {
 109   std::stringstream blacklist_url_prefix_stream;
 110   blacklist_url_prefix_stream
 111       << GetExperimentParam(kSuggestionsFieldTrialURLParam)
 112       << GetExperimentParam(kSuggestionsFieldTrialBlacklistPathParam) << "?"
 113       << GetExperimentParam(kSuggestionsFieldTrialCommonParamsParam) << "&"
 114       << GetExperimentParam(kSuggestionsFieldTrialBlacklistUrlParam) << "=";
 115   return blacklist_url_prefix_stream.str();
 116 }
 117
 118 }  // namespace
 119
 120 SuggestionsService::SuggestionsService(
 121     net::URLRequestContextGetter* url_request_context,
 122     scoped_ptr<SuggestionsStore> suggestions_store,
 123     scoped_ptr<ImageManager> thumbnail_manager,
 124     scoped_ptr<BlacklistStore> blacklist_store)
 125     : suggestions_store_(suggestions_store.Pass()),
 126       blacklist_store_(blacklist_store.Pass()),
 127       thumbnail_manager_(thumbnail_manager.Pass()),
 128       url_request_context_(url_request_context),
 129       blacklist_delay_sec_(kBlacklistDefaultDelaySec),
 130       weak_ptr_factory_(this),
 131       request_timeout_ms_(kDefaultRequestTimeoutMs) {
 132   // Obtain various parameters from Variations.
 133   suggestions_url_ =
 134       GURL(GetExperimentParam(kSuggestionsFieldTrialURLParam) + "?" +
 135            GetExperimentParam(kSuggestionsFieldTrialCommonParamsParam));
 136   blacklist_url_prefix_ = GetBlacklistUrlPrefix();
 137   std::string timeout = GetExperimentParam(kSuggestionsFieldTrialTimeoutMs);
 138   int temp_timeout;
 139   if (!timeout.empty() && base::StringToInt(timeout, &temp_timeout)) {
 140     request_timeout_ms_ = temp_timeout;
 141   }
 142 }
 143
 144 SuggestionsService::~SuggestionsService() {}
 145
 146 // static
 147 bool SuggestionsService::IsEnabled() {
 148   return GetExperimentParam(kSuggestionsFieldTrialStateParam) ==
 149          kSuggestionsFieldTrialStateEnabled;
 150 }
 151
 152 // static
 153 bool SuggestionsService::IsControlGroup() {
 154   return GetExperimentParam(kSuggestionsFieldTrialControlParam) ==
 155          kSuggestionsFieldTrialStateEnabled;
 156 }
 157
 158 void SuggestionsService::FetchSuggestionsData(
 159     SyncState sync_state,
 160     SuggestionsService::ResponseCallback callback) {
 161   DCHECK(thread_checker_.CalledOnValidThread());
 162   if (sync_state == NOT_INITIALIZED_ENABLED) {
 163     // Sync is not initialized yet, but enabled. Serve previously cached
 164     // suggestions if available.
 165     waiting_requestors_.push_back(callback);
 166     ServeFromCache();
 167     return;
 168   } else if (sync_state == SYNC_OR_HISTORY_SYNC_DISABLED) {
 169     // Cancel any ongoing request (and the timeout closure). We must no longer
 170     // interact with the server.
 171     pending_request_.reset(NULL);
 172     pending_timeout_closure_.reset(NULL);
 173     suggestions_store_->ClearSuggestions();
 174     callback.Run(SuggestionsProfile());
 175     DispatchRequestsAndClear(SuggestionsProfile(), &waiting_requestors_);
 176     return;
 177   }
 178
 179   FetchSuggestionsDataNoTimeout(callback);
 180
 181   // Post a task to serve the cached suggestions if the request hasn't completed
 182   // after some time. Cancels the previous such task, if one existed.
 183   pending_timeout_closure_.reset(new CancelableClosure(base::Bind(
 184       &SuggestionsService::OnRequestTimeout, weak_ptr_factory_.GetWeakPtr())));
 185   base::MessageLoopProxy::current()->PostDelayedTask(
 186       FROM_HERE, pending_timeout_closure_->callback(),
 187       base::TimeDelta::FromMilliseconds(request_timeout_ms_));
 188 }
 189
 190 void SuggestionsService::GetPageThumbnail(
 191     const GURL& url,
 192     base::Callback<void(const GURL&, const SkBitmap*)> callback) {
 193   thumbnail_manager_->GetImageForURL(url, callback);
 194 }
 195
 196 void SuggestionsService::BlacklistURL(
 197     const GURL& candidate_url,
 198     const SuggestionsService::ResponseCallback& callback) {
 199   DCHECK(thread_checker_.CalledOnValidThread());
 200   waiting_requestors_.push_back(callback);
 201
 202   // Blacklist locally, for immediate effect.
 203   if (!blacklist_store_->BlacklistUrl(candidate_url)) {
 204     DVLOG(1) << "Failed blacklisting attempt.";
 205     return;
 206   }
 207
 208   // If there's an ongoing request, let it complete.
 209   if (pending_request_.get()) return;
 210   IssueRequest(BuildBlacklistRequestURL(blacklist_url_prefix_, candidate_url));
 211 }
 212
 213 // static
 214 bool SuggestionsService::GetBlacklistedUrl(const net::URLFetcher& request,
 215                                            GURL* url) {
 216   bool is_blacklist_request = StartsWithASCII(request.GetOriginalURL().spec(),
 217                                               GetBlacklistUrlPrefix(), true);
 218   if (!is_blacklist_request) return false;
 219
 220   // Extract the blacklisted URL from the blacklist request.
 221   std::string blacklisted;
 222   if (!net::GetValueForKeyInQuery(
 223           request.GetOriginalURL(),
 224           GetExperimentParam(kSuggestionsFieldTrialBlacklistUrlParam),
 225           &blacklisted))
 226     return false;
 227
 228   GURL blacklisted_url(blacklisted);
 229   blacklisted_url.Swap(url);
 230   return true;
 231 }
 232
 233 // static
 234 void SuggestionsService::RegisterProfilePrefs(
 235     user_prefs::PrefRegistrySyncable* registry) {
 236   SuggestionsStore::RegisterProfilePrefs(registry);
 237   BlacklistStore::RegisterProfilePrefs(registry);
 238 }
 239
 240 void SuggestionsService::SetDefaultExpiryTimestamp(
 241     SuggestionsProfile* suggestions, int64 default_timestamp_usec) {
 242   for (int i = 0; i < suggestions->suggestions_size(); ++i) {
 243     ChromeSuggestion* suggestion = suggestions->mutable_suggestions(i);
 244     // Do not set expiry if the server has already provided a more specific
 245     // expiry time for this suggestion.
 246     if (!suggestion->has_expiry_ts()) {
 247       suggestion->set_expiry_ts(default_timestamp_usec);
 248     }
 249   }
 250 }
 251
 252 void SuggestionsService::FetchSuggestionsDataNoTimeout(
 253     SuggestionsService::ResponseCallback callback) {
 254   DCHECK(thread_checker_.CalledOnValidThread());
 255   if (pending_request_.get()) {
 256     // Request already exists, so just add requestor to queue.
 257     waiting_requestors_.push_back(callback);
 258     return;
 259   }
 260
 261   // Form new request.
 262   DCHECK(waiting_requestors_.empty());
 263   waiting_requestors_.push_back(callback);
 264   IssueRequest(suggestions_url_);
 265 }
 266
 267 void SuggestionsService::IssueRequest(const GURL& url) {
 268   pending_request_.reset(CreateSuggestionsRequest(url));
 269   pending_request_->Start();
 270   last_request_started_time_ = base::TimeTicks::Now();
 271 }
 272
 273 net::URLFetcher* SuggestionsService::CreateSuggestionsRequest(const GURL& url) {
 274   net::URLFetcher* request =
 275       net::URLFetcher::Create(0, url, net::URLFetcher::GET, this);
 276   request->SetLoadFlags(net::LOAD_DISABLE_CACHE);
 277   request->SetRequestContext(url_request_context_);
 278   // Add Chrome experiment state to the request headers.
 279   net::HttpRequestHeaders headers;
 280   variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
 281       request->GetOriginalURL(), false, false, &headers);
 282   request->SetExtraRequestHeaders(headers.ToString());
 283   return request;
 284 }
 285
 286 void SuggestionsService::OnRequestTimeout() {
 287   DCHECK(thread_checker_.CalledOnValidThread());
 288   ServeFromCache();
 289 }
 290
 291 void SuggestionsService::OnURLFetchComplete(const net::URLFetcher* source) {
 292   DCHECK(thread_checker_.CalledOnValidThread());
 293   DCHECK_EQ(pending_request_.get(), source);
 294   // We no longer need the timeout closure. Delete it whether or not it has run.
 295   // If it hasn't, this cancels it.
 296   pending_timeout_closure_.reset();
 297
 298   // The fetcher will be deleted when the request is handled.
 299   scoped_ptr<const net::URLFetcher> request(pending_request_.release());
 300   const net::URLRequestStatus& request_status = request->GetStatus();
 301   if (request_status.status() != net::URLRequestStatus::SUCCESS) {
 302     UMA_HISTOGRAM_SPARSE_SLOWLY("Suggestions.FailedRequestErrorCode",
 303                                 -request_status.error());
 304     DVLOG(1) << "Suggestions server request failed with error: "
 305              << request_status.error() << ": "
 306              << net::ErrorToString(request_status.error());
 307     // Dispatch the cached profile on error.
 308     ServeFromCache();
 309     ScheduleBlacklistUpload(false);
 310     return;
 311   }
 312
 313   // Log the response code.
 314   const int response_code = request->GetResponseCode();
 315   UMA_HISTOGRAM_SPARSE_SLOWLY("Suggestions.FetchResponseCode", response_code);
 316   if (response_code != net::HTTP_OK) {
 317     // Aggressively clear the store.
 318     suggestions_store_->ClearSuggestions();
 319     DispatchRequestsAndClear(SuggestionsProfile(), &waiting_requestors_);
 320     ScheduleBlacklistUpload(false);
 321     return;
 322   }
 323
 324   const base::TimeDelta latency =
 325       base::TimeTicks::Now() - last_request_started_time_;
 326   UMA_HISTOGRAM_MEDIUM_TIMES("Suggestions.FetchSuccessLatency", latency);
 327
 328   // Handle a successful blacklisting.
 329   GURL blacklisted_url;
 330   if (GetBlacklistedUrl(*source, &blacklisted_url)) {
 331     blacklist_store_->RemoveUrl(blacklisted_url);
 332   }
 333
 334   std::string suggestions_data;
 335   bool success = request->GetResponseAsString(&suggestions_data);
 336   DCHECK(success);
 337
 338   // Compute suggestions, and dispatch them to requestors. On error still
 339   // dispatch empty suggestions.
 340   SuggestionsProfile suggestions;
 341   if (suggestions_data.empty()) {
 342     LogResponseState(RESPONSE_EMPTY);
 343     suggestions_store_->ClearSuggestions();
 344   } else if (suggestions.ParseFromString(suggestions_data)) {
 345     LogResponseState(RESPONSE_VALID);
 346     thumbnail_manager_->Initialize(suggestions);
 347
 348     int64 now_usec = (base::Time::NowFromSystemTime() - base::Time::UnixEpoch())
 349         .ToInternalValue();
 350     SetDefaultExpiryTimestamp(&suggestions, now_usec + kDefaultExpiryUsec);
 351     suggestions_store_->StoreSuggestions(suggestions);
 352   } else {
 353     LogResponseState(RESPONSE_INVALID);
 354     suggestions_store_->LoadSuggestions(&suggestions);
 355     thumbnail_manager_->Initialize(suggestions);
 356   }
 357
 358   FilterAndServe(&suggestions);
 359   ScheduleBlacklistUpload(true);
 360 }
 361
 362 void SuggestionsService::Shutdown() {
 363   // Cancel pending request and timeout closure, then serve existing requestors
 364   // from cache.
 365   pending_request_.reset(NULL);
 366   pending_timeout_closure_.reset(NULL);
 367   ServeFromCache();
 368 }
 369
 370 void SuggestionsService::ServeFromCache() {
 371   SuggestionsProfile suggestions;
 372   suggestions_store_->LoadSuggestions(&suggestions);
 373   thumbnail_manager_->Initialize(suggestions);
 374   FilterAndServe(&suggestions);
 375 }
 376
 377 void SuggestionsService::FilterAndServe(SuggestionsProfile* suggestions) {
 378   blacklist_store_->FilterSuggestions(suggestions);
 379   DispatchRequestsAndClear(*suggestions, &waiting_requestors_);
 380 }
 381
 382 void SuggestionsService::ScheduleBlacklistUpload(bool last_request_successful) {
 383   DCHECK(thread_checker_.CalledOnValidThread());
 384
 385   UpdateBlacklistDelay(last_request_successful);
 386
 387   // Schedule a blacklist upload task.
 388   GURL blacklist_url;
 389   if (blacklist_store_->GetFirstUrlFromBlacklist(&blacklist_url)) {
 390     base::Closure blacklist_cb =
 391         base::Bind(&SuggestionsService::UploadOneFromBlacklist,
 392                    weak_ptr_factory_.GetWeakPtr());
 393     base::MessageLoopProxy::current()->PostDelayedTask(
 394         FROM_HERE, blacklist_cb,
 395         base::TimeDelta::FromSeconds(blacklist_delay_sec_));
 396   }
 397 }
 398
 399 void SuggestionsService::UploadOneFromBlacklist() {
 400   DCHECK(thread_checker_.CalledOnValidThread());
 401
 402   // If there's an ongoing request, let it complete.
 403   if (pending_request_.get()) return;
 404
 405   GURL blacklist_url;
 406   if (!blacklist_store_->GetFirstUrlFromBlacklist(&blacklist_url))
 407     return;  // Local blacklist is empty.
 408
 409   // Send blacklisting request.
 410   IssueRequest(BuildBlacklistRequestURL(blacklist_url_prefix_, blacklist_url));
 411 }
 412
 413 void SuggestionsService::UpdateBlacklistDelay(bool last_request_successful) {
 414   DCHECK(thread_checker_.CalledOnValidThread());
 415
 416   if (last_request_successful) {
 417     blacklist_delay_sec_ = kBlacklistDefaultDelaySec;
 418   } else {
 419     int candidate_delay = blacklist_delay_sec_ * kBlacklistBackoffMultiplier;
 420     if (candidate_delay < kBlacklistMaxDelaySec)
 421       blacklist_delay_sec_ = candidate_delay;
 422   }
 423 }
 424
 425 }  // namespace suggestions