chrome/browser/autocomplete/zero_suggest_provider.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/autocomplete/zero_suggest_provider.h"
   6
   7 #include "base/callback.h"
   8 #include "base/i18n/case_conversion.h"
   9 #include "base/json/json_string_value_serializer.h"
  10 #include "base/metrics/histogram.h"
  11 #include "base/metrics/user_metrics.h"
  12 #include "base/prefs/pref_service.h"
  13 #include "base/strings/string16.h"
  14 #include "base/strings/string_util.h"
  15 #include "base/strings/utf_string_conversions.h"
  16 #include "base/time/time.h"
  17 #include "chrome/browser/autocomplete/autocomplete_classifier.h"
  18 #include "chrome/browser/autocomplete/autocomplete_classifier_factory.h"
  19 #include "chrome/browser/autocomplete/chrome_autocomplete_scheme_classifier.h"
  20 #include "chrome/browser/autocomplete/history_url_provider.h"
  21 #include "chrome/browser/autocomplete/search_provider.h"
  22 #include "chrome/browser/history/history_types.h"
  23 #include "chrome/browser/history/top_sites.h"
  24 #include "chrome/browser/profiles/profile.h"
  25 #include "chrome/common/pref_names.h"
  26 #include "components/metrics/proto/omnibox_input_type.pb.h"
  27 #include "components/omnibox/autocomplete_input.h"
  28 #include "components/omnibox/autocomplete_match.h"
  29 #include "components/omnibox/autocomplete_provider_listener.h"
  30 #include "components/omnibox/omnibox_field_trial.h"
  31 #include "components/pref_registry/pref_registry_syncable.h"
  32 #include "components/search_engines/template_url_service.h"
  33 #include "components/variations/variations_http_header_provider.h"
  34 #include "net/base/escape.h"
  35 #include "net/base/load_flags.h"
  36 #include "net/base/net_util.h"
  37 #include "net/http/http_request_headers.h"
  38 #include "net/url_request/url_fetcher.h"
  39 #include "net/url_request/url_request_status.h"
  40 #include "url/gurl.h"
  41
  42 namespace {
  43
  44 // TODO(hfung): The histogram code was copied and modified from
  45 // search_provider.cc.  Refactor and consolidate the code.
  46 // We keep track in a histogram how many suggest requests we send, how
  47 // many suggest requests we invalidate (e.g., due to a user typing
  48 // another character), and how many replies we receive.
  49 // *** ADD NEW ENUMS AFTER ALL PREVIOUSLY DEFINED ONES! ***
  50 //     (excluding the end-of-list enum value)
  51 // We do not want values of existing enums to change or else it screws
  52 // up the statistics.
  53 enum ZeroSuggestRequestsHistogramValue {
  54   ZERO_SUGGEST_REQUEST_SENT = 1,
  55   ZERO_SUGGEST_REQUEST_INVALIDATED,
  56   ZERO_SUGGEST_REPLY_RECEIVED,
  57   ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE
  58 };
  59
  60 void LogOmniboxZeroSuggestRequest(
  61     ZeroSuggestRequestsHistogramValue request_value) {
  62   UMA_HISTOGRAM_ENUMERATION("Omnibox.ZeroSuggestRequests", request_value,
  63                             ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE);
  64 }
  65
  66 // The maximum relevance of the top match from this provider.
  67 const int kDefaultVerbatimZeroSuggestRelevance = 1300;
  68
  69 // Relevance value to use if it was not set explicitly by the server.
  70 const int kDefaultZeroSuggestRelevance = 100;
  71
  72 }  // namespace
  73
  74 // static
  75 ZeroSuggestProvider* ZeroSuggestProvider::Create(
  76     AutocompleteProviderListener* listener,
  77     TemplateURLService* template_url_service,
  78     Profile* profile) {
  79   return new ZeroSuggestProvider(listener, template_url_service, profile);
  80 }
  81
  82 // static
  83 void ZeroSuggestProvider::RegisterProfilePrefs(
  84     user_prefs::PrefRegistrySyncable* registry) {
  85   registry->RegisterStringPref(
  86       prefs::kZeroSuggestCachedResults,
  87       std::string(),
  88       user_prefs::PrefRegistrySyncable::UNSYNCABLE_PREF);
  89 }
  90
  91 void ZeroSuggestProvider::Start(const AutocompleteInput& input,
  92                                 bool minimal_changes) {
  93   matches_.clear();
  94   if (input.type() == metrics::OmniboxInputType::INVALID)
  95     return;
  96
  97   Stop(true);
  98   field_trial_triggered_ = false;
  99   field_trial_triggered_in_session_ = false;
 100   results_from_cache_ = false;
 101   permanent_text_ = input.text();
 102   current_query_ = input.current_url().spec();
 103   current_page_classification_ = input.current_page_classification();
 104   current_url_match_ = MatchForCurrentURL();
 105
 106   const TemplateURL* default_provider =
 107      template_url_service_->GetDefaultSearchProvider();
 108   if (default_provider == NULL)
 109     return;
 110
 111   base::string16 prefix;
 112   TemplateURLRef::SearchTermsArgs search_term_args(prefix);
 113   GURL suggest_url(default_provider->suggestions_url_ref().ReplaceSearchTerms(
 114       search_term_args, template_url_service_->search_terms_data()));
 115   if (!suggest_url.is_valid())
 116     return;
 117
 118   // No need to send the current page URL in personalized suggest field trial.
 119   if (CanSendURL(input.current_url(), suggest_url, default_provider,
 120                  current_page_classification_,
 121                  template_url_service_->search_terms_data(), profile_) &&
 122       !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) {
 123     // Update suggest_url to include the current_page_url.
 124     search_term_args.current_page_url = current_query_;
 125     suggest_url = GURL(default_provider->suggestions_url_ref().
 126                        ReplaceSearchTerms(
 127                            search_term_args,
 128                            template_url_service_->search_terms_data()));
 129   } else if (!CanShowZeroSuggestWithoutSendingURL(suggest_url,
 130                                                   input.current_url())) {
 131     return;
 132   }
 133
 134   done_ = false;
 135   // TODO(jered): Consider adding locally-sourced zero-suggestions here too.
 136   // These may be useful on the NTP or more relevant to the user than server
 137   // suggestions, if based on local browsing history.
 138   MaybeUseCachedSuggestions();
 139   Run(suggest_url);
 140 }
 141
 142 void ZeroSuggestProvider::Stop(bool clear_cached_results) {
 143   if (fetcher_)
 144     LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_INVALIDATED);
 145   fetcher_.reset();
 146   done_ = true;
 147
 148   if (clear_cached_results) {
 149     // We do not call Clear() on |results_| to retain |verbatim_relevance|
 150     // value in the |results_| object. |verbatim_relevance| is used at the
 151     // beginning of the next StartZeroSuggest() call to determine the current
 152     // url match relevance.
 153     results_.suggest_results.clear();
 154     results_.navigation_results.clear();
 155     current_query_.clear();
 156   }
 157 }
 158
 159 void ZeroSuggestProvider::DeleteMatch(const AutocompleteMatch& match) {
 160   if (OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) {
 161     // Remove the deleted match from the cache, so it is not shown to the user
 162     // again. Since we cannot remove just one result, blow away the cache.
 163     profile_->GetPrefs()->SetString(prefs::kZeroSuggestCachedResults,
 164                                     std::string());
 165   }
 166   BaseSearchProvider::DeleteMatch(match);
 167 }
 168
 169 void ZeroSuggestProvider::AddProviderInfo(ProvidersInfo* provider_info) const {
 170   BaseSearchProvider::AddProviderInfo(provider_info);
 171   if (!results_.suggest_results.empty() || !results_.navigation_results.empty())
 172     provider_info->back().set_times_returned_results_in_session(1);
 173 }
 174
 175 void ZeroSuggestProvider::ResetSession() {
 176   // The user has started editing in the omnibox, so leave
 177   // |field_trial_triggered_in_session_| unchanged and set
 178   // |field_trial_triggered_| to false since zero suggest is inactive now.
 179   field_trial_triggered_ = false;
 180 }
 181
 182 ZeroSuggestProvider::ZeroSuggestProvider(
 183   AutocompleteProviderListener* listener,
 184   TemplateURLService* template_url_service,
 185   Profile* profile)
 186     : BaseSearchProvider(template_url_service, profile,
 187                          AutocompleteProvider::TYPE_ZERO_SUGGEST),
 188       listener_(listener),
 189       results_from_cache_(false),
 190       weak_ptr_factory_(this) {
 191 }
 192
 193 ZeroSuggestProvider::~ZeroSuggestProvider() {
 194 }
 195
 196 const TemplateURL* ZeroSuggestProvider::GetTemplateURL(bool is_keyword) const {
 197   // Zero suggest provider should not receive keyword results.
 198   DCHECK(!is_keyword);
 199   return template_url_service_->GetDefaultSearchProvider();
 200 }
 201
 202 const AutocompleteInput ZeroSuggestProvider::GetInput(bool is_keyword) const {
 203   return AutocompleteInput(
 204       base::string16(), base::string16::npos, base::string16(),
 205       GURL(current_query_), current_page_classification_, true, false, false,
 206       true, ChromeAutocompleteSchemeClassifier(profile_));
 207 }
 208
 209 bool ZeroSuggestProvider::ShouldAppendExtraParams(
 210       const SearchSuggestionParser::SuggestResult& result) const {
 211   // We always use the default provider for search, so append the params.
 212   return true;
 213 }
 214
 215 void ZeroSuggestProvider::RecordDeletionResult(bool success) {
 216   if (success) {
 217     base::RecordAction(
 218         base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Success"));
 219   } else {
 220     base::RecordAction(
 221         base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Failure"));
 222   }
 223 }
 224
 225 void ZeroSuggestProvider::OnURLFetchComplete(const net::URLFetcher* source) {
 226   DCHECK(!done_);
 227   DCHECK_EQ(fetcher_.get(), source);
 228
 229   LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REPLY_RECEIVED);
 230
 231   bool results_updated = false;
 232   if (source->GetStatus().is_success() && source->GetResponseCode() == 200) {
 233     std::string json_data = SearchSuggestionParser::ExtractJsonData(source);
 234     scoped_ptr<base::Value> data(
 235         SearchSuggestionParser::DeserializeJsonData(json_data));
 236     if (data) {
 237       if (StoreSuggestionResponse(json_data, *data))
 238         return;
 239       results_updated = ParseSuggestResults(
 240           *data, kDefaultZeroSuggestRelevance, false, &results_);
 241     }
 242   }
 243   fetcher_.reset();
 244   done_ = true;
 245   ConvertResultsToAutocompleteMatches();
 246   listener_->OnProviderUpdate(results_updated);
 247 }
 248
 249 bool ZeroSuggestProvider::StoreSuggestionResponse(
 250     const std::string& json_data,
 251     const base::Value& parsed_data) {
 252   if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial() ||
 253       json_data.empty())
 254     return false;
 255   profile_->GetPrefs()->SetString(prefs::kZeroSuggestCachedResults, json_data);
 256
 257   // If we received an empty result list, we should update the display, as it
 258   // may be showing cached results that should not be shown.
 259   const base::ListValue* root_list = NULL;
 260   const base::ListValue* results_list = NULL;
 261   if (parsed_data.GetAsList(&root_list) &&
 262       root_list->GetList(1, &results_list) &&
 263       results_list->empty())
 264     return false;
 265
 266   // We are finished with the request and want to bail early.
 267   if (results_from_cache_)
 268     done_ = true;
 269
 270   return results_from_cache_;
 271 }
 272
 273 void ZeroSuggestProvider::AddSuggestResultsToMap(
 274     const SearchSuggestionParser::SuggestResults& results,
 275     MatchMap* map) {
 276   for (size_t i = 0; i < results.size(); ++i)
 277     AddMatchToMap(results[i], std::string(), i, false, false, map);
 278 }
 279
 280 AutocompleteMatch ZeroSuggestProvider::NavigationToMatch(
 281     const SearchSuggestionParser::NavigationResult& navigation) {
 282   AutocompleteMatch match(this, navigation.relevance(), false,
 283                           navigation.type());
 284   match.destination_url = navigation.url();
 285
 286   // Zero suggest results should always omit protocols and never appear bold.
 287   const std::string languages(
 288       profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
 289   match.contents = net::FormatUrl(navigation.url(), languages,
 290       net::kFormatUrlOmitAll, net::UnescapeRule::SPACES, NULL, NULL, NULL);
 291   match.fill_into_edit +=
 292       AutocompleteInput::FormattedStringWithEquivalentMeaning(navigation.url(),
 293           match.contents, ChromeAutocompleteSchemeClassifier(profile_));
 294
 295   AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
 296       match.contents.length(), ACMatchClassification::URL,
 297       &match.contents_class);
 298
 299   match.description =
 300       AutocompleteMatch::SanitizeString(navigation.description());
 301   AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
 302       match.description.length(), ACMatchClassification::NONE,
 303       &match.description_class);
 304   return match;
 305 }
 306
 307 void ZeroSuggestProvider::Run(const GURL& suggest_url) {
 308   const int kFetcherID = 1;
 309   fetcher_.reset(
 310       net::URLFetcher::Create(kFetcherID,
 311           suggest_url,
 312           net::URLFetcher::GET, this));
 313   fetcher_->SetRequestContext(profile_->GetRequestContext());
 314   fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES);
 315   // Add Chrome experiment state to the request headers.
 316   net::HttpRequestHeaders headers;
 317   variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
 318       fetcher_->GetOriginalURL(), profile_->IsOffTheRecord(), false, &headers);
 319   fetcher_->SetExtraRequestHeaders(headers.ToString());
 320   fetcher_->Start();
 321
 322   if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
 323     most_visited_urls_.clear();
 324     history::TopSites* ts = profile_->GetTopSites();
 325     if (ts) {
 326       ts->GetMostVisitedURLs(
 327           base::Bind(&ZeroSuggestProvider::OnMostVisitedUrlsAvailable,
 328                      weak_ptr_factory_.GetWeakPtr()), false);
 329     }
 330   }
 331   LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_SENT);
 332 }
 333
 334 void ZeroSuggestProvider::OnMostVisitedUrlsAvailable(
 335     const history::MostVisitedURLList& urls) {
 336   most_visited_urls_ = urls;
 337 }
 338
 339 void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches() {
 340   matches_.clear();
 341
 342   const TemplateURL* default_provider =
 343       template_url_service_->GetDefaultSearchProvider();
 344   // Fail if we can't set the clickthrough URL for query suggestions.
 345   if (default_provider == NULL || !default_provider->SupportsReplacement(
 346           template_url_service_->search_terms_data()))
 347     return;
 348
 349   MatchMap map;
 350   AddSuggestResultsToMap(results_.suggest_results, &map);
 351
 352   const int num_query_results = map.size();
 353   const int num_nav_results = results_.navigation_results.size();
 354   const int num_results = num_query_results + num_nav_results;
 355   UMA_HISTOGRAM_COUNTS("ZeroSuggest.QueryResults", num_query_results);
 356   UMA_HISTOGRAM_COUNTS("ZeroSuggest.URLResults", num_nav_results);
 357   UMA_HISTOGRAM_COUNTS("ZeroSuggest.AllResults", num_results);
 358
 359   // Show Most Visited results after ZeroSuggest response is received.
 360   if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
 361     if (!current_url_match_.destination_url.is_valid())
 362       return;
 363     matches_.push_back(current_url_match_);
 364     int relevance = 600;
 365     if (num_results > 0) {
 366       UMA_HISTOGRAM_COUNTS(
 367           "Omnibox.ZeroSuggest.MostVisitedResultsCounterfactual",
 368           most_visited_urls_.size());
 369     }
 370     const base::string16 current_query_string16(
 371         base::ASCIIToUTF16(current_query_));
 372     const std::string languages(
 373         profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
 374     for (size_t i = 0; i < most_visited_urls_.size(); i++) {
 375       const history::MostVisitedURL& url = most_visited_urls_[i];
 376       SearchSuggestionParser::NavigationResult nav(
 377           ChromeAutocompleteSchemeClassifier(profile_), url.url,
 378           AutocompleteMatchType::NAVSUGGEST, url.title, std::string(), false,
 379           relevance, true, current_query_string16, languages);
 380       matches_.push_back(NavigationToMatch(nav));
 381       --relevance;
 382     }
 383     return;
 384   }
 385
 386   if (num_results == 0)
 387     return;
 388
 389   // TODO(jered): Rip this out once the first match is decoupled from the
 390   // current typing in the omnibox.
 391   matches_.push_back(current_url_match_);
 392
 393   for (MatchMap::const_iterator it(map.begin()); it != map.end(); ++it)
 394     matches_.push_back(it->second);
 395
 396   const SearchSuggestionParser::NavigationResults& nav_results(
 397       results_.navigation_results);
 398   for (SearchSuggestionParser::NavigationResults::const_iterator it(
 399            nav_results.begin()); it != nav_results.end(); ++it)
 400     matches_.push_back(NavigationToMatch(*it));
 401 }
 402
 403 AutocompleteMatch ZeroSuggestProvider::MatchForCurrentURL() {
 404   AutocompleteMatch match;
 405   AutocompleteClassifierFactory::GetForProfile(profile_)->Classify(
 406       permanent_text_, false, true, current_page_classification_, &match, NULL);
 407   match.is_history_what_you_typed_match = false;
 408   match.allowed_to_be_default_match = true;
 409
 410   // The placeholder suggestion for the current URL has high relevance so
 411   // that it is in the first suggestion slot and inline autocompleted. It
 412   // gets dropped as soon as the user types something.
 413   match.relevance = GetVerbatimRelevance();
 414
 415   return match;
 416 }
 417
 418 int ZeroSuggestProvider::GetVerbatimRelevance() const {
 419   return results_.verbatim_relevance >= 0 ?
 420       results_.verbatim_relevance : kDefaultVerbatimZeroSuggestRelevance;
 421 }
 422
 423 bool ZeroSuggestProvider::CanShowZeroSuggestWithoutSendingURL(
 424     const GURL& suggest_url,
 425     const GURL& current_page_url) const {
 426   if (!ZeroSuggestEnabled(suggest_url,
 427                           template_url_service_->GetDefaultSearchProvider(),
 428                           current_page_classification_,
 429                           template_url_service_->search_terms_data(), profile_))
 430     return false;
 431
 432   // If we cannot send URLs, then only the MostVisited and Personalized
 433   // variations can be shown.
 434   if (!OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial() &&
 435       !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
 436     return false;
 437
 438   // Only show zero suggest for HTTP[S] pages.
 439   // TODO(mariakhomenko): We may be able to expand this set to include pages
 440   // with other schemes (e.g. chrome://). That may require improvements to
 441   // the formatting of the verbatim result returned by MatchForCurrentURL().
 442   if (!current_page_url.is_valid() ||
 443       ((current_page_url.scheme() != url::kHttpScheme) &&
 444       (current_page_url.scheme() != url::kHttpsScheme)))
 445     return false;
 446
 447   return true;
 448 }
 449
 450 void ZeroSuggestProvider::MaybeUseCachedSuggestions() {
 451   if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
 452     return;
 453
 454   std::string json_data = profile_->GetPrefs()->GetString(
 455       prefs::kZeroSuggestCachedResults);
 456   if (!json_data.empty()) {
 457     scoped_ptr<base::Value> data(
 458         SearchSuggestionParser::DeserializeJsonData(json_data));
 459     if (data && ParseSuggestResults(
 460             *data, kDefaultZeroSuggestRelevance, false, &results_)) {
 461       ConvertResultsToAutocompleteMatches();
 462       results_from_cache_ = !matches_.empty();
 463     }
 464   }
 465 }