chrome/browser/autocomplete/zero_suggest_provider.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/autocomplete/zero_suggest_provider.h"
   6
   7 #include "base/callback.h"
   8 #include "base/i18n/case_conversion.h"
   9 #include "base/json/json_string_value_serializer.h"
  10 #include "base/metrics/histogram.h"
  11 #include "base/metrics/user_metrics.h"
  12 #include "base/prefs/pref_service.h"
  13 #include "base/strings/string16.h"
  14 #include "base/strings/string_util.h"
  15 #include "base/strings/utf_string_conversions.h"
  16 #include "base/time/time.h"
  17 #include "chrome/browser/autocomplete/autocomplete_classifier.h"
  18 #include "chrome/browser/autocomplete/autocomplete_classifier_factory.h"
  19 #include "chrome/browser/autocomplete/chrome_autocomplete_provider_client.h"
  20 #include "chrome/browser/autocomplete/chrome_autocomplete_scheme_classifier.h"
  21 #include "chrome/browser/autocomplete/history_url_provider.h"
  22 #include "chrome/browser/history/top_sites_factory.h"
  23 #include "chrome/browser/profiles/profile.h"
  24 #include "chrome/browser/search_engines/template_url_service_factory.h"
  25 #include "chrome/common/pref_names.h"
  26 #include "components/history/core/browser/history_types.h"
  27 #include "components/history/core/browser/top_sites.h"
  28 #include "components/metrics/proto/omnibox_input_type.pb.h"
  29 #include "components/omnibox/autocomplete_input.h"
  30 #include "components/omnibox/autocomplete_match.h"
  31 #include "components/omnibox/autocomplete_provider_listener.h"
  32 #include "components/omnibox/omnibox_field_trial.h"
  33 #include "components/omnibox/search_provider.h"
  34 #include "components/pref_registry/pref_registry_syncable.h"
  35 #include "components/search_engines/template_url_service.h"
  36 #include "components/variations/net/variations_http_header_provider.h"
  37 #include "net/base/escape.h"
  38 #include "net/base/load_flags.h"
  39 #include "net/base/net_util.h"
  40 #include "net/http/http_request_headers.h"
  41 #include "net/url_request/url_fetcher.h"
  42 #include "net/url_request/url_request_status.h"
  43 #include "url/gurl.h"
  44
  45 namespace {
  46
  47 // TODO(hfung): The histogram code was copied and modified from
  48 // search_provider.cc.  Refactor and consolidate the code.
  49 // We keep track in a histogram how many suggest requests we send, how
  50 // many suggest requests we invalidate (e.g., due to a user typing
  51 // another character), and how many replies we receive.
  52 // *** ADD NEW ENUMS AFTER ALL PREVIOUSLY DEFINED ONES! ***
  53 //     (excluding the end-of-list enum value)
  54 // We do not want values of existing enums to change or else it screws
  55 // up the statistics.
  56 enum ZeroSuggestRequestsHistogramValue {
  57   ZERO_SUGGEST_REQUEST_SENT = 1,
  58   ZERO_SUGGEST_REQUEST_INVALIDATED,
  59   ZERO_SUGGEST_REPLY_RECEIVED,
  60   ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE
  61 };
  62
  63 void LogOmniboxZeroSuggestRequest(
  64     ZeroSuggestRequestsHistogramValue request_value) {
  65   UMA_HISTOGRAM_ENUMERATION("Omnibox.ZeroSuggestRequests", request_value,
  66                             ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE);
  67 }
  68
  69 // The maximum relevance of the top match from this provider.
  70 const int kDefaultVerbatimZeroSuggestRelevance = 1300;
  71
  72 // Relevance value to use if it was not set explicitly by the server.
  73 const int kDefaultZeroSuggestRelevance = 100;
  74
  75 }  // namespace
  76
  77 // static
  78 ZeroSuggestProvider* ZeroSuggestProvider::Create(
  79     AutocompleteProviderListener* listener,
  80     TemplateURLService* template_url_service,
  81     Profile* profile) {
  82   return new ZeroSuggestProvider(listener, template_url_service, profile);
  83 }
  84
  85 // static
  86 void ZeroSuggestProvider::RegisterProfilePrefs(
  87     user_prefs::PrefRegistrySyncable* registry) {
  88   registry->RegisterStringPref(
  89       prefs::kZeroSuggestCachedResults,
  90       std::string(),
  91       user_prefs::PrefRegistrySyncable::UNSYNCABLE_PREF);
  92 }
  93
  94 void ZeroSuggestProvider::Start(const AutocompleteInput& input,
  95                                 bool minimal_changes,
  96                                 bool called_due_to_focus) {
  97   matches_.clear();
  98   if (!called_due_to_focus ||
  99       input.type() == metrics::OmniboxInputType::INVALID)
 100     return;
 101
 102   Stop(true, false);
 103   field_trial_triggered_ = false;
 104   field_trial_triggered_in_session_ = false;
 105   results_from_cache_ = false;
 106   permanent_text_ = input.text();
 107   current_query_ = input.current_url().spec();
 108   current_page_classification_ = input.current_page_classification();
 109   current_url_match_ = MatchForCurrentURL();
 110
 111   const TemplateURL* default_provider =
 112      template_url_service_->GetDefaultSearchProvider();
 113   if (default_provider == NULL)
 114     return;
 115
 116   base::string16 prefix;
 117   TemplateURLRef::SearchTermsArgs search_term_args(prefix);
 118   GURL suggest_url(default_provider->suggestions_url_ref().ReplaceSearchTerms(
 119       search_term_args, template_url_service_->search_terms_data()));
 120   if (!suggest_url.is_valid())
 121     return;
 122
 123   // No need to send the current page URL in personalized suggest or
 124   // most visited field trials.
 125   if (CanSendURL(input.current_url(), suggest_url, default_provider,
 126                  current_page_classification_,
 127                  template_url_service_->search_terms_data(), client_.get()) &&
 128       !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial() &&
 129       !OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
 130     // Update suggest_url to include the current_page_url.
 131     search_term_args.current_page_url = current_query_;
 132     suggest_url = GURL(default_provider->suggestions_url_ref().
 133                        ReplaceSearchTerms(
 134                            search_term_args,
 135                            template_url_service_->search_terms_data()));
 136   } else if (!ShouldShowNonContextualZeroSuggest(suggest_url,
 137                                                  input.current_url())) {
 138     return;
 139   }
 140
 141   done_ = false;
 142   // TODO(jered): Consider adding locally-sourced zero-suggestions here too.
 143   // These may be useful on the NTP or more relevant to the user than server
 144   // suggestions, if based on local browsing history.
 145   MaybeUseCachedSuggestions();
 146   Run(suggest_url);
 147 }
 148
 149 void ZeroSuggestProvider::Stop(bool clear_cached_results,
 150                                bool due_to_user_inactivity) {
 151   if (fetcher_)
 152     LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_INVALIDATED);
 153   fetcher_.reset();
 154   waiting_for_most_visited_urls_request_ = false;
 155   done_ = true;
 156
 157   if (clear_cached_results) {
 158     // We do not call Clear() on |results_| to retain |verbatim_relevance|
 159     // value in the |results_| object. |verbatim_relevance| is used at the
 160     // beginning of the next OnOmniboxFocused() call to determine the current
 161     // url match relevance.
 162     results_.suggest_results.clear();
 163     results_.navigation_results.clear();
 164     current_query_.clear();
 165     most_visited_urls_.clear();
 166   }
 167 }
 168
 169 void ZeroSuggestProvider::DeleteMatch(const AutocompleteMatch& match) {
 170   if (OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) {
 171     // Remove the deleted match from the cache, so it is not shown to the user
 172     // again. Since we cannot remove just one result, blow away the cache.
 173     profile_->GetPrefs()->SetString(prefs::kZeroSuggestCachedResults,
 174                                     std::string());
 175   }
 176   BaseSearchProvider::DeleteMatch(match);
 177 }
 178
 179 void ZeroSuggestProvider::AddProviderInfo(ProvidersInfo* provider_info) const {
 180   BaseSearchProvider::AddProviderInfo(provider_info);
 181   if (!results_.suggest_results.empty() ||
 182       !results_.navigation_results.empty() ||
 183       !most_visited_urls_.empty())
 184     provider_info->back().set_times_returned_results_in_session(1);
 185 }
 186
 187 void ZeroSuggestProvider::ResetSession() {
 188   // The user has started editing in the omnibox, so leave
 189   // |field_trial_triggered_in_session_| unchanged and set
 190   // |field_trial_triggered_| to false since zero suggest is inactive now.
 191   field_trial_triggered_ = false;
 192 }
 193
 194 ZeroSuggestProvider::ZeroSuggestProvider(
 195   AutocompleteProviderListener* listener,
 196   TemplateURLService* template_url_service,
 197   Profile* profile)
 198     : BaseSearchProvider(template_url_service,
 199                          scoped_ptr<AutocompleteProviderClient>(
 200                              new ChromeAutocompleteProviderClient(profile)),
 201                          AutocompleteProvider::TYPE_ZERO_SUGGEST),
 202       listener_(listener),
 203       profile_(profile),
 204       results_from_cache_(false),
 205       waiting_for_most_visited_urls_request_(false),
 206       weak_ptr_factory_(this) {
 207 }
 208
 209 ZeroSuggestProvider::~ZeroSuggestProvider() {
 210 }
 211
 212 const TemplateURL* ZeroSuggestProvider::GetTemplateURL(bool is_keyword) const {
 213   // Zero suggest provider should not receive keyword results.
 214   DCHECK(!is_keyword);
 215   return template_url_service_->GetDefaultSearchProvider();
 216 }
 217
 218 const AutocompleteInput ZeroSuggestProvider::GetInput(bool is_keyword) const {
 219   return AutocompleteInput(
 220       base::string16(), base::string16::npos, std::string(),
 221       GURL(current_query_), current_page_classification_, true, false, false,
 222       true, ChromeAutocompleteSchemeClassifier(profile_));
 223 }
 224
 225 bool ZeroSuggestProvider::ShouldAppendExtraParams(
 226       const SearchSuggestionParser::SuggestResult& result) const {
 227   // We always use the default provider for search, so append the params.
 228   return true;
 229 }
 230
 231 void ZeroSuggestProvider::RecordDeletionResult(bool success) {
 232   if (success) {
 233     base::RecordAction(
 234         base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Success"));
 235   } else {
 236     base::RecordAction(
 237         base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Failure"));
 238   }
 239 }
 240
 241 void ZeroSuggestProvider::OnURLFetchComplete(const net::URLFetcher* source) {
 242   DCHECK(!done_);
 243   DCHECK_EQ(fetcher_.get(), source);
 244
 245   LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REPLY_RECEIVED);
 246
 247   bool results_updated = false;
 248   if (source->GetStatus().is_success() && source->GetResponseCode() == 200) {
 249     std::string json_data = SearchSuggestionParser::ExtractJsonData(source);
 250     scoped_ptr<base::Value> data(
 251         SearchSuggestionParser::DeserializeJsonData(json_data));
 252     if (data) {
 253       if (StoreSuggestionResponse(json_data, *data))
 254         return;
 255       results_updated = ParseSuggestResults(
 256           *data, kDefaultZeroSuggestRelevance, false, &results_);
 257     }
 258   }
 259   fetcher_.reset();
 260   done_ = true;
 261   ConvertResultsToAutocompleteMatches();
 262   listener_->OnProviderUpdate(results_updated);
 263 }
 264
 265 bool ZeroSuggestProvider::StoreSuggestionResponse(
 266     const std::string& json_data,
 267     const base::Value& parsed_data) {
 268   if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial() ||
 269       json_data.empty())
 270     return false;
 271   profile_->GetPrefs()->SetString(prefs::kZeroSuggestCachedResults, json_data);
 272
 273   // If we received an empty result list, we should update the display, as it
 274   // may be showing cached results that should not be shown.
 275   const base::ListValue* root_list = NULL;
 276   const base::ListValue* results_list = NULL;
 277   if (parsed_data.GetAsList(&root_list) &&
 278       root_list->GetList(1, &results_list) &&
 279       results_list->empty())
 280     return false;
 281
 282   // We are finished with the request and want to bail early.
 283   if (results_from_cache_)
 284     done_ = true;
 285
 286   return results_from_cache_;
 287 }
 288
 289 void ZeroSuggestProvider::AddSuggestResultsToMap(
 290     const SearchSuggestionParser::SuggestResults& results,
 291     MatchMap* map) {
 292   for (size_t i = 0; i < results.size(); ++i)
 293     AddMatchToMap(results[i], std::string(), i, false, false, map);
 294 }
 295
 296 AutocompleteMatch ZeroSuggestProvider::NavigationToMatch(
 297     const SearchSuggestionParser::NavigationResult& navigation) {
 298   AutocompleteMatch match(this, navigation.relevance(), false,
 299                           navigation.type());
 300   match.destination_url = navigation.url();
 301
 302   // Zero suggest results should always omit protocols and never appear bold.
 303   const std::string languages(
 304       profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
 305   match.contents = net::FormatUrl(navigation.url(), languages,
 306       net::kFormatUrlOmitAll, net::UnescapeRule::SPACES, NULL, NULL, NULL);
 307   match.fill_into_edit +=
 308       AutocompleteInput::FormattedStringWithEquivalentMeaning(navigation.url(),
 309           match.contents, ChromeAutocompleteSchemeClassifier(profile_));
 310
 311   AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
 312       match.contents.length(), ACMatchClassification::URL,
 313       &match.contents_class);
 314
 315   match.description =
 316       AutocompleteMatch::SanitizeString(navigation.description());
 317   AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
 318       match.description.length(), ACMatchClassification::NONE,
 319       &match.description_class);
 320   return match;
 321 }
 322
 323 void ZeroSuggestProvider::Run(const GURL& suggest_url) {
 324   if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
 325     most_visited_urls_.clear();
 326     scoped_refptr<history::TopSites> ts =
 327         TopSitesFactory::GetForProfile(profile_);
 328     if (ts) {
 329       waiting_for_most_visited_urls_request_ = true;
 330       ts->GetMostVisitedURLs(
 331           base::Bind(&ZeroSuggestProvider::OnMostVisitedUrlsAvailable,
 332                      weak_ptr_factory_.GetWeakPtr()), false);
 333     }
 334   } else {
 335     const int kFetcherID = 1;
 336     fetcher_.reset(
 337         net::URLFetcher::Create(kFetcherID,
 338             suggest_url,
 339             net::URLFetcher::GET, this));
 340     fetcher_->SetRequestContext(profile_->GetRequestContext());
 341     fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES);
 342     // Add Chrome experiment state to the request headers.
 343     net::HttpRequestHeaders headers;
 344     variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
 345         fetcher_->GetOriginalURL(), profile_->IsOffTheRecord(), false,
 346         &headers);
 347     fetcher_->SetExtraRequestHeaders(headers.ToString());
 348     fetcher_->Start();
 349     LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_SENT);
 350   }
 351 }
 352
 353 void ZeroSuggestProvider::OnMostVisitedUrlsAvailable(
 354     const history::MostVisitedURLList& urls) {
 355   if (!waiting_for_most_visited_urls_request_) return;
 356   most_visited_urls_ = urls;
 357   waiting_for_most_visited_urls_request_ = false;
 358   done_ = true;
 359   ConvertResultsToAutocompleteMatches();
 360   listener_->OnProviderUpdate(true);
 361 }
 362
 363 void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches() {
 364   matches_.clear();
 365
 366   const TemplateURL* default_provider =
 367       template_url_service_->GetDefaultSearchProvider();
 368   // Fail if we can't set the clickthrough URL for query suggestions.
 369   if (default_provider == NULL || !default_provider->SupportsReplacement(
 370           template_url_service_->search_terms_data()))
 371     return;
 372
 373   MatchMap map;
 374   AddSuggestResultsToMap(results_.suggest_results, &map);
 375
 376   const int num_query_results = map.size();
 377   const int num_nav_results = results_.navigation_results.size();
 378   const int num_results = num_query_results + num_nav_results;
 379   UMA_HISTOGRAM_COUNTS("ZeroSuggest.QueryResults", num_query_results);
 380   UMA_HISTOGRAM_COUNTS("ZeroSuggest.URLResults", num_nav_results);
 381   UMA_HISTOGRAM_COUNTS("ZeroSuggest.AllResults", num_results);
 382
 383   // Show Most Visited results after ZeroSuggest response is received.
 384   if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
 385     if (!current_url_match_.destination_url.is_valid())
 386       return;
 387     matches_.push_back(current_url_match_);
 388     int relevance = 600;
 389     if (num_results > 0) {
 390       UMA_HISTOGRAM_COUNTS(
 391           "Omnibox.ZeroSuggest.MostVisitedResultsCounterfactual",
 392           most_visited_urls_.size());
 393     }
 394     const base::string16 current_query_string16(
 395         base::ASCIIToUTF16(current_query_));
 396     const std::string languages(
 397         profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
 398     for (size_t i = 0; i < most_visited_urls_.size(); i++) {
 399       const history::MostVisitedURL& url = most_visited_urls_[i];
 400       SearchSuggestionParser::NavigationResult nav(
 401           ChromeAutocompleteSchemeClassifier(profile_), url.url,
 402           AutocompleteMatchType::NAVSUGGEST, url.title, std::string(), false,
 403           relevance, true, current_query_string16, languages);
 404       matches_.push_back(NavigationToMatch(nav));
 405       --relevance;
 406     }
 407     return;
 408   }
 409
 410   if (num_results == 0)
 411     return;
 412
 413   // TODO(jered): Rip this out once the first match is decoupled from the
 414   // current typing in the omnibox.
 415   matches_.push_back(current_url_match_);
 416
 417   for (MatchMap::const_iterator it(map.begin()); it != map.end(); ++it)
 418     matches_.push_back(it->second);
 419
 420   const SearchSuggestionParser::NavigationResults& nav_results(
 421       results_.navigation_results);
 422   for (SearchSuggestionParser::NavigationResults::const_iterator it(
 423            nav_results.begin()); it != nav_results.end(); ++it)
 424     matches_.push_back(NavigationToMatch(*it));
 425 }
 426
 427 AutocompleteMatch ZeroSuggestProvider::MatchForCurrentURL() {
 428   AutocompleteMatch match;
 429   AutocompleteClassifierFactory::GetForProfile(profile_)->Classify(
 430       permanent_text_, false, true, current_page_classification_, &match, NULL);
 431   match.is_history_what_you_typed_match = false;
 432   match.allowed_to_be_default_match = true;
 433
 434   // The placeholder suggestion for the current URL has high relevance so
 435   // that it is in the first suggestion slot and inline autocompleted. It
 436   // gets dropped as soon as the user types something.
 437   match.relevance = GetVerbatimRelevance();
 438
 439   return match;
 440 }
 441
 442 int ZeroSuggestProvider::GetVerbatimRelevance() const {
 443   return results_.verbatim_relevance >= 0 ?
 444       results_.verbatim_relevance : kDefaultVerbatimZeroSuggestRelevance;
 445 }
 446
 447 bool ZeroSuggestProvider::ShouldShowNonContextualZeroSuggest(
 448     const GURL& suggest_url,
 449     const GURL& current_page_url) const {
 450   if (!ZeroSuggestEnabled(suggest_url,
 451                           template_url_service_->GetDefaultSearchProvider(),
 452                           current_page_classification_,
 453                           template_url_service_->search_terms_data(),
 454                           client_.get()))
 455     return false;
 456
 457   // If we cannot send URLs, then only the MostVisited and Personalized
 458   // variations can be shown.
 459   if (!OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial() &&
 460       !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
 461     return false;
 462
 463   // Only show zero suggest for HTTP[S] pages.
 464   // TODO(mariakhomenko): We may be able to expand this set to include pages
 465   // with other schemes (e.g. chrome://). That may require improvements to
 466   // the formatting of the verbatim result returned by MatchForCurrentURL().
 467   if (!current_page_url.is_valid() ||
 468       ((current_page_url.scheme() != url::kHttpScheme) &&
 469       (current_page_url.scheme() != url::kHttpsScheme)))
 470     return false;
 471
 472   if (OmniboxFieldTrial::InZeroSuggestMostVisitedWithoutSerpFieldTrial() &&
 473       template_url_service_->
 474           IsSearchResultsPageFromDefaultSearchProvider(current_page_url))
 475     return false;
 476
 477   return true;
 478 }
 479
 480 void ZeroSuggestProvider::MaybeUseCachedSuggestions() {
 481   if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
 482     return;
 483
 484   std::string json_data = profile_->GetPrefs()->GetString(
 485       prefs::kZeroSuggestCachedResults);
 486   if (!json_data.empty()) {
 487     scoped_ptr<base::Value> data(
 488         SearchSuggestionParser::DeserializeJsonData(json_data));
 489     if (data && ParseSuggestResults(
 490             *data, kDefaultZeroSuggestRelevance, false, &results_)) {
 491       ConvertResultsToAutocompleteMatches();
 492       results_from_cache_ = !matches_.empty();
 493     }
 494   }
 495 }