components/omnibox/zero_suggest_provider.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/omnibox/zero_suggest_provider.h"
   6
   7 #include "base/callback.h"
   8 #include "base/i18n/case_conversion.h"
   9 #include "base/json/json_string_value_serializer.h"
  10 #include "base/metrics/histogram.h"
  11 #include "base/metrics/user_metrics.h"
  12 #include "base/prefs/pref_service.h"
  13 #include "base/strings/string16.h"
  14 #include "base/strings/string_util.h"
  15 #include "base/strings/utf_string_conversions.h"
  16 #include "base/time/time.h"
  17 #include "components/history/core/browser/history_types.h"
  18 #include "components/history/core/browser/top_sites.h"
  19 #include "components/metrics/proto/omnibox_input_type.pb.h"
  20 #include "components/omnibox/autocomplete_classifier.h"
  21 #include "components/omnibox/autocomplete_input.h"
  22 #include "components/omnibox/autocomplete_match.h"
  23 #include "components/omnibox/autocomplete_provider_listener.h"
  24 #include "components/omnibox/history_url_provider.h"
  25 #include "components/omnibox/omnibox_field_trial.h"
  26 #include "components/omnibox/omnibox_pref_names.h"
  27 #include "components/omnibox/search_provider.h"
  28 #include "components/pref_registry/pref_registry_syncable.h"
  29 #include "components/search_engines/template_url_service.h"
  30 #include "components/variations/net/variations_http_header_provider.h"
  31 #include "net/base/escape.h"
  32 #include "net/base/load_flags.h"
  33 #include "net/base/net_util.h"
  34 #include "net/http/http_request_headers.h"
  35 #include "net/url_request/url_fetcher.h"
  36 #include "net/url_request/url_request_status.h"
  37 #include "url/gurl.h"
  38
  39 namespace {
  40
  41 // TODO(hfung): The histogram code was copied and modified from
  42 // search_provider.cc.  Refactor and consolidate the code.
  43 // We keep track in a histogram how many suggest requests we send, how
  44 // many suggest requests we invalidate (e.g., due to a user typing
  45 // another character), and how many replies we receive.
  46 // *** ADD NEW ENUMS AFTER ALL PREVIOUSLY DEFINED ONES! ***
  47 //     (excluding the end-of-list enum value)
  48 // We do not want values of existing enums to change or else it screws
  49 // up the statistics.
  50 enum ZeroSuggestRequestsHistogramValue {
  51   ZERO_SUGGEST_REQUEST_SENT = 1,
  52   ZERO_SUGGEST_REQUEST_INVALIDATED,
  53   ZERO_SUGGEST_REPLY_RECEIVED,
  54   ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE
  55 };
  56
  57 void LogOmniboxZeroSuggestRequest(
  58     ZeroSuggestRequestsHistogramValue request_value) {
  59   UMA_HISTOGRAM_ENUMERATION("Omnibox.ZeroSuggestRequests", request_value,
  60                             ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE);
  61 }
  62
  63 // The maximum relevance of the top match from this provider.
  64 const int kDefaultVerbatimZeroSuggestRelevance = 1300;
  65
  66 // Relevance value to use if it was not set explicitly by the server.
  67 const int kDefaultZeroSuggestRelevance = 100;
  68
  69 }  // namespace
  70
  71 // static
  72 ZeroSuggestProvider* ZeroSuggestProvider::Create(
  73     AutocompleteProviderClient* client,
  74     AutocompleteProviderListener* listener) {
  75   return new ZeroSuggestProvider(client, listener);
  76 }
  77
  78 // static
  79 void ZeroSuggestProvider::RegisterProfilePrefs(
  80     user_prefs::PrefRegistrySyncable* registry) {
  81   registry->RegisterStringPref(omnibox::kZeroSuggestCachedResults,
  82                                std::string());
  83 }
  84
  85 void ZeroSuggestProvider::Start(const AutocompleteInput& input,
  86                                 bool minimal_changes) {
  87   matches_.clear();
  88   if (!input.from_omnibox_focus() ||
  89       input.type() == metrics::OmniboxInputType::INVALID)
  90     return;
  91
  92   Stop(true, false);
  93   set_field_trial_triggered(false);
  94   set_field_trial_triggered_in_session(false);
  95   results_from_cache_ = false;
  96   permanent_text_ = input.text();
  97   current_query_ = input.current_url().spec();
  98   current_page_classification_ = input.current_page_classification();
  99   current_url_match_ = MatchForCurrentURL();
 100   TemplateURLService* template_url_service = client()->GetTemplateURLService();
 101
 102   const TemplateURL* default_provider =
 103       template_url_service->GetDefaultSearchProvider();
 104   if (default_provider == NULL)
 105     return;
 106
 107   base::string16 prefix;
 108   TemplateURLRef::SearchTermsArgs search_term_args(prefix);
 109   GURL suggest_url(default_provider->suggestions_url_ref().ReplaceSearchTerms(
 110       search_term_args, template_url_service->search_terms_data()));
 111   if (!suggest_url.is_valid())
 112     return;
 113
 114   // No need to send the current page URL in personalized suggest or
 115   // most visited field trials.
 116   if (CanSendURL(input.current_url(), suggest_url, default_provider,
 117                  current_page_classification_,
 118                  template_url_service->search_terms_data(), client()) &&
 119       !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial() &&
 120       !OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
 121     // Update suggest_url to include the current_page_url.
 122     search_term_args.current_page_url = current_query_;
 123     suggest_url =
 124         GURL(default_provider->suggestions_url_ref().ReplaceSearchTerms(
 125             search_term_args, template_url_service->search_terms_data()));
 126   } else if (!ShouldShowNonContextualZeroSuggest(suggest_url,
 127                                                  input.current_url())) {
 128     return;
 129   }
 130
 131   done_ = false;
 132   // TODO(jered): Consider adding locally-sourced zero-suggestions here too.
 133   // These may be useful on the NTP or more relevant to the user than server
 134   // suggestions, if based on local browsing history.
 135   MaybeUseCachedSuggestions();
 136   Run(suggest_url);
 137 }
 138
 139 void ZeroSuggestProvider::Stop(bool clear_cached_results,
 140                                bool due_to_user_inactivity) {
 141   if (fetcher_)
 142     LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_INVALIDATED);
 143   fetcher_.reset();
 144   waiting_for_most_visited_urls_request_ = false;
 145   done_ = true;
 146
 147   if (clear_cached_results) {
 148     // We do not call Clear() on |results_| to retain |verbatim_relevance|
 149     // value in the |results_| object. |verbatim_relevance| is used at the
 150     // beginning of the next call to Start() to determine the current url
 151     // match relevance.
 152     results_.suggest_results.clear();
 153     results_.navigation_results.clear();
 154     current_query_.clear();
 155     most_visited_urls_.clear();
 156   }
 157 }
 158
 159 void ZeroSuggestProvider::DeleteMatch(const AutocompleteMatch& match) {
 160   if (OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) {
 161     // Remove the deleted match from the cache, so it is not shown to the user
 162     // again. Since we cannot remove just one result, blow away the cache.
 163     client()->GetPrefs()->SetString(omnibox::kZeroSuggestCachedResults,
 164                                     std::string());
 165   }
 166   BaseSearchProvider::DeleteMatch(match);
 167 }
 168
 169 void ZeroSuggestProvider::AddProviderInfo(ProvidersInfo* provider_info) const {
 170   BaseSearchProvider::AddProviderInfo(provider_info);
 171   if (!results_.suggest_results.empty() ||
 172       !results_.navigation_results.empty() ||
 173       !most_visited_urls_.empty())
 174     provider_info->back().set_times_returned_results_in_session(1);
 175 }
 176
 177 void ZeroSuggestProvider::ResetSession() {
 178   // The user has started editing in the omnibox, so leave
 179   // |field_trial_triggered_in_session| unchanged and set
 180   // |field_trial_triggered| to false since zero suggest is inactive now.
 181   set_field_trial_triggered(false);
 182 }
 183
 184 ZeroSuggestProvider::ZeroSuggestProvider(AutocompleteProviderClient* client,
 185                                          AutocompleteProviderListener* listener)
 186     : BaseSearchProvider(AutocompleteProvider::TYPE_ZERO_SUGGEST, client),
 187       listener_(listener),
 188       results_from_cache_(false),
 189       waiting_for_most_visited_urls_request_(false),
 190       weak_ptr_factory_(this) {
 191 }
 192
 193 ZeroSuggestProvider::~ZeroSuggestProvider() {
 194 }
 195
 196 const TemplateURL* ZeroSuggestProvider::GetTemplateURL(bool is_keyword) const {
 197   // Zero suggest provider should not receive keyword results.
 198   DCHECK(!is_keyword);
 199   return client()->GetTemplateURLService()->GetDefaultSearchProvider();
 200 }
 201
 202 const AutocompleteInput ZeroSuggestProvider::GetInput(bool is_keyword) const {
 203   // The callers of this method won't look at the AutocompleteInput's
 204   // |from_omnibox_focus| member, so we can set its value to false.
 205   return AutocompleteInput(base::string16(), base::string16::npos,
 206                            std::string(), GURL(current_query_),
 207                            current_page_classification_, true, false, false,
 208                            true, false, client()->GetSchemeClassifier());
 209 }
 210
 211 bool ZeroSuggestProvider::ShouldAppendExtraParams(
 212       const SearchSuggestionParser::SuggestResult& result) const {
 213   // We always use the default provider for search, so append the params.
 214   return true;
 215 }
 216
 217 void ZeroSuggestProvider::RecordDeletionResult(bool success) {
 218   if (success) {
 219     base::RecordAction(
 220         base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Success"));
 221   } else {
 222     base::RecordAction(
 223         base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Failure"));
 224   }
 225 }
 226
 227 void ZeroSuggestProvider::OnURLFetchComplete(const net::URLFetcher* source) {
 228   DCHECK(!done_);
 229   DCHECK_EQ(fetcher_.get(), source);
 230
 231   LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REPLY_RECEIVED);
 232
 233   bool results_updated = false;
 234   if (source->GetStatus().is_success() && source->GetResponseCode() == 200) {
 235     std::string json_data = SearchSuggestionParser::ExtractJsonData(source);
 236     scoped_ptr<base::Value> data(
 237         SearchSuggestionParser::DeserializeJsonData(json_data));
 238     if (data) {
 239       if (StoreSuggestionResponse(json_data, *data))
 240         return;
 241       results_updated = ParseSuggestResults(
 242           *data, kDefaultZeroSuggestRelevance, false, &results_);
 243     }
 244   }
 245   fetcher_.reset();
 246   done_ = true;
 247   ConvertResultsToAutocompleteMatches();
 248   listener_->OnProviderUpdate(results_updated);
 249 }
 250
 251 bool ZeroSuggestProvider::StoreSuggestionResponse(
 252     const std::string& json_data,
 253     const base::Value& parsed_data) {
 254   if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial() ||
 255       json_data.empty())
 256     return false;
 257   client()->GetPrefs()->SetString(omnibox::kZeroSuggestCachedResults,
 258                                   json_data);
 259
 260   // If we received an empty result list, we should update the display, as it
 261   // may be showing cached results that should not be shown.
 262   const base::ListValue* root_list = NULL;
 263   const base::ListValue* results_list = NULL;
 264   if (parsed_data.GetAsList(&root_list) &&
 265       root_list->GetList(1, &results_list) &&
 266       results_list->empty())
 267     return false;
 268
 269   // We are finished with the request and want to bail early.
 270   if (results_from_cache_)
 271     done_ = true;
 272
 273   return results_from_cache_;
 274 }
 275
 276 void ZeroSuggestProvider::AddSuggestResultsToMap(
 277     const SearchSuggestionParser::SuggestResults& results,
 278     MatchMap* map) {
 279   for (size_t i = 0; i < results.size(); ++i)
 280     AddMatchToMap(results[i], std::string(), i, false, false, map);
 281 }
 282
 283 AutocompleteMatch ZeroSuggestProvider::NavigationToMatch(
 284     const SearchSuggestionParser::NavigationResult& navigation) {
 285   AutocompleteMatch match(this, navigation.relevance(), false,
 286                           navigation.type());
 287   match.destination_url = navigation.url();
 288
 289   // Zero suggest results should always omit protocols and never appear bold.
 290   const std::string languages(client()->GetAcceptLanguages());
 291   match.contents = net::FormatUrl(navigation.url(), languages,
 292       net::kFormatUrlOmitAll, net::UnescapeRule::SPACES, NULL, NULL, NULL);
 293   match.fill_into_edit +=
 294       AutocompleteInput::FormattedStringWithEquivalentMeaning(
 295           navigation.url(), match.contents, client()->GetSchemeClassifier());
 296
 297   AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
 298       match.contents.length(), ACMatchClassification::URL,
 299       &match.contents_class);
 300
 301   match.description =
 302       AutocompleteMatch::SanitizeString(navigation.description());
 303   AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
 304       match.description.length(), ACMatchClassification::NONE,
 305       &match.description_class);
 306   return match;
 307 }
 308
 309 void ZeroSuggestProvider::Run(const GURL& suggest_url) {
 310   if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
 311     most_visited_urls_.clear();
 312     scoped_refptr<history::TopSites> ts = client()->GetTopSites();
 313     if (ts) {
 314       waiting_for_most_visited_urls_request_ = true;
 315       ts->GetMostVisitedURLs(
 316           base::Bind(&ZeroSuggestProvider::OnMostVisitedUrlsAvailable,
 317                      weak_ptr_factory_.GetWeakPtr()), false);
 318     }
 319   } else {
 320     const int kFetcherID = 1;
 321     fetcher_ = net::URLFetcher::Create(kFetcherID, suggest_url,
 322                                        net::URLFetcher::GET, this);
 323     fetcher_->SetRequestContext(client()->GetRequestContext());
 324     fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES);
 325     // Add Chrome experiment state to the request headers.
 326     net::HttpRequestHeaders headers;
 327     variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
 328         fetcher_->GetOriginalURL(), client()->IsOffTheRecord(), false,
 329         &headers);
 330     fetcher_->SetExtraRequestHeaders(headers.ToString());
 331     fetcher_->Start();
 332     LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_SENT);
 333   }
 334 }
 335
 336 void ZeroSuggestProvider::OnMostVisitedUrlsAvailable(
 337     const history::MostVisitedURLList& urls) {
 338   if (!waiting_for_most_visited_urls_request_) return;
 339   most_visited_urls_ = urls;
 340   waiting_for_most_visited_urls_request_ = false;
 341   done_ = true;
 342   ConvertResultsToAutocompleteMatches();
 343   listener_->OnProviderUpdate(true);
 344 }
 345
 346 void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches() {
 347   matches_.clear();
 348
 349   TemplateURLService* template_url_service = client()->GetTemplateURLService();
 350   const TemplateURL* default_provider =
 351       template_url_service->GetDefaultSearchProvider();
 352   // Fail if we can't set the clickthrough URL for query suggestions.
 353   if (default_provider == NULL ||
 354       !default_provider->SupportsReplacement(
 355           template_url_service->search_terms_data()))
 356     return;
 357
 358   MatchMap map;
 359   AddSuggestResultsToMap(results_.suggest_results, &map);
 360
 361   const int num_query_results = map.size();
 362   const int num_nav_results = results_.navigation_results.size();
 363   const int num_results = num_query_results + num_nav_results;
 364   UMA_HISTOGRAM_COUNTS("ZeroSuggest.QueryResults", num_query_results);
 365   UMA_HISTOGRAM_COUNTS("ZeroSuggest.URLResults", num_nav_results);
 366   UMA_HISTOGRAM_COUNTS("ZeroSuggest.AllResults", num_results);
 367
 368   // Show Most Visited results after ZeroSuggest response is received.
 369   if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
 370     if (!current_url_match_.destination_url.is_valid())
 371       return;
 372     matches_.push_back(current_url_match_);
 373     int relevance = 600;
 374     if (num_results > 0) {
 375       UMA_HISTOGRAM_COUNTS(
 376           "Omnibox.ZeroSuggest.MostVisitedResultsCounterfactual",
 377           most_visited_urls_.size());
 378     }
 379     const base::string16 current_query_string16(
 380         base::ASCIIToUTF16(current_query_));
 381     const std::string languages(client()->GetAcceptLanguages());
 382     for (size_t i = 0; i < most_visited_urls_.size(); i++) {
 383       const history::MostVisitedURL& url = most_visited_urls_[i];
 384       SearchSuggestionParser::NavigationResult nav(
 385           client()->GetSchemeClassifier(), url.url,
 386           AutocompleteMatchType::NAVSUGGEST, url.title, std::string(), false,
 387           relevance, true, current_query_string16, languages);
 388       matches_.push_back(NavigationToMatch(nav));
 389       --relevance;
 390     }
 391     return;
 392   }
 393
 394   if (num_results == 0)
 395     return;
 396
 397   // TODO(jered): Rip this out once the first match is decoupled from the
 398   // current typing in the omnibox.
 399   matches_.push_back(current_url_match_);
 400
 401   for (MatchMap::const_iterator it(map.begin()); it != map.end(); ++it)
 402     matches_.push_back(it->second);
 403
 404   const SearchSuggestionParser::NavigationResults& nav_results(
 405       results_.navigation_results);
 406   for (SearchSuggestionParser::NavigationResults::const_iterator it(
 407            nav_results.begin()); it != nav_results.end(); ++it)
 408     matches_.push_back(NavigationToMatch(*it));
 409 }
 410
 411 AutocompleteMatch ZeroSuggestProvider::MatchForCurrentURL() {
 412   AutocompleteMatch match;
 413   client()->GetAutocompleteClassifier()->Classify(
 414       permanent_text_, false, true, current_page_classification_, &match, NULL);
 415   match.allowed_to_be_default_match = true;
 416
 417   // The placeholder suggestion for the current URL has high relevance so
 418   // that it is in the first suggestion slot and inline autocompleted. It
 419   // gets dropped as soon as the user types something.
 420   match.relevance = GetVerbatimRelevance();
 421
 422   return match;
 423 }
 424
 425 int ZeroSuggestProvider::GetVerbatimRelevance() const {
 426   return results_.verbatim_relevance >= 0 ?
 427       results_.verbatim_relevance : kDefaultVerbatimZeroSuggestRelevance;
 428 }
 429
 430 bool ZeroSuggestProvider::ShouldShowNonContextualZeroSuggest(
 431     const GURL& suggest_url,
 432     const GURL& current_page_url) const {
 433   const TemplateURLService* template_url_service =
 434       client()->GetTemplateURLService();
 435   if (!ZeroSuggestEnabled(suggest_url,
 436                           template_url_service->GetDefaultSearchProvider(),
 437                           current_page_classification_,
 438                           template_url_service->search_terms_data(), client()))
 439     return false;
 440
 441   // If we cannot send URLs, then only the MostVisited and Personalized
 442   // variations can be shown.
 443   if (!OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial() &&
 444       !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
 445     return false;
 446
 447   // Only show zero suggest for HTTP[S] pages.
 448   // TODO(mariakhomenko): We may be able to expand this set to include pages
 449   // with other schemes (e.g. chrome://). That may require improvements to
 450   // the formatting of the verbatim result returned by MatchForCurrentURL().
 451   if (!current_page_url.is_valid() ||
 452       ((current_page_url.scheme() != url::kHttpScheme) &&
 453       (current_page_url.scheme() != url::kHttpsScheme)))
 454     return false;
 455
 456   if (OmniboxFieldTrial::InZeroSuggestMostVisitedWithoutSerpFieldTrial() &&
 457       client()
 458           ->GetTemplateURLService()
 459           ->IsSearchResultsPageFromDefaultSearchProvider(current_page_url))
 460     return false;
 461
 462   return true;
 463 }
 464
 465 void ZeroSuggestProvider::MaybeUseCachedSuggestions() {
 466   if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
 467     return;
 468
 469   std::string json_data =
 470       client()->GetPrefs()->GetString(omnibox::kZeroSuggestCachedResults);
 471   if (!json_data.empty()) {
 472     scoped_ptr<base::Value> data(
 473         SearchSuggestionParser::DeserializeJsonData(json_data));
 474     if (data && ParseSuggestResults(
 475             *data, kDefaultZeroSuggestRelevance, false, &results_)) {
 476       ConvertResultsToAutocompleteMatches();
 477       results_from_cache_ = !matches_.empty();
 478     }
 479   }
 480 }