components/omnibox/browser/zero_suggest_provider.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/omnibox/browser/zero_suggest_provider.h"
   6
   7 #include "base/callback.h"
   8 #include "base/i18n/case_conversion.h"
   9 #include "base/json/json_string_value_serializer.h"
  10 #include "base/metrics/histogram.h"
  11 #include "base/metrics/user_metrics.h"
  12 #include "base/prefs/pref_service.h"
  13 #include "base/strings/string16.h"
  14 #include "base/strings/string_util.h"
  15 #include "base/strings/utf_string_conversions.h"
  16 #include "base/time/time.h"
  17 #include "components/history/core/browser/history_types.h"
  18 #include "components/history/core/browser/top_sites.h"
  19 #include "components/metrics/proto/omnibox_input_type.pb.h"
  20 #include "components/omnibox/browser/autocomplete_classifier.h"
  21 #include "components/omnibox/browser/autocomplete_input.h"
  22 #include "components/omnibox/browser/autocomplete_match.h"
  23 #include "components/omnibox/browser/autocomplete_provider_listener.h"
  24 #include "components/omnibox/browser/history_url_provider.h"
  25 #include "components/omnibox/browser/omnibox_field_trial.h"
  26 #include "components/omnibox/browser/omnibox_pref_names.h"
  27 #include "components/omnibox/browser/search_provider.h"
  28 #include "components/omnibox/browser/verbatim_match.h"
  29 #include "components/pref_registry/pref_registry_syncable.h"
  30 #include "components/search_engines/template_url_service.h"
  31 #include "components/url_formatter/url_formatter.h"
  32 #include "components/variations/net/variations_http_header_provider.h"
  33 #include "net/base/escape.h"
  34 #include "net/base/load_flags.h"
  35 #include "net/http/http_request_headers.h"
  36 #include "net/url_request/url_fetcher.h"
  37 #include "net/url_request/url_request_status.h"
  38 #include "url/gurl.h"
  39
  40 namespace {
  41
  42 // TODO(hfung): The histogram code was copied and modified from
  43 // search_provider.cc.  Refactor and consolidate the code.
  44 // We keep track in a histogram how many suggest requests we send, how
  45 // many suggest requests we invalidate (e.g., due to a user typing
  46 // another character), and how many replies we receive.
  47 // *** ADD NEW ENUMS AFTER ALL PREVIOUSLY DEFINED ONES! ***
  48 //     (excluding the end-of-list enum value)
  49 // We do not want values of existing enums to change or else it screws
  50 // up the statistics.
  51 enum ZeroSuggestRequestsHistogramValue {
  52   ZERO_SUGGEST_REQUEST_SENT = 1,
  53   ZERO_SUGGEST_REQUEST_INVALIDATED,
  54   ZERO_SUGGEST_REPLY_RECEIVED,
  55   ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE
  56 };
  57
  58 void LogOmniboxZeroSuggestRequest(
  59     ZeroSuggestRequestsHistogramValue request_value) {
  60   UMA_HISTOGRAM_ENUMERATION("Omnibox.ZeroSuggestRequests", request_value,
  61                             ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE);
  62 }
  63
  64 // Relevance value to use if it was not set explicitly by the server.
  65 const int kDefaultZeroSuggestRelevance = 100;
  66
  67 }  // namespace
  68
  69 // static
  70 ZeroSuggestProvider* ZeroSuggestProvider::Create(
  71     AutocompleteProviderClient* client,
  72     AutocompleteProviderListener* listener) {
  73   return new ZeroSuggestProvider(client, listener);
  74 }
  75
  76 // static
  77 void ZeroSuggestProvider::RegisterProfilePrefs(
  78     user_prefs::PrefRegistrySyncable* registry) {
  79   registry->RegisterStringPref(omnibox::kZeroSuggestCachedResults,
  80                                std::string());
  81 }
  82
  83 void ZeroSuggestProvider::Start(const AutocompleteInput& input,
  84                                 bool minimal_changes) {
  85   matches_.clear();
  86   if (!input.from_omnibox_focus() ||
  87       input.type() == metrics::OmniboxInputType::INVALID)
  88     return;
  89
  90   Stop(true, false);
  91   set_field_trial_triggered(false);
  92   set_field_trial_triggered_in_session(false);
  93   results_from_cache_ = false;
  94   permanent_text_ = input.text();
  95   current_query_ = input.current_url().spec();
  96   current_page_classification_ = input.current_page_classification();
  97   current_url_match_ = MatchForCurrentURL();
  98   TemplateURLService* template_url_service = client()->GetTemplateURLService();
  99
 100   const TemplateURL* default_provider =
 101       template_url_service->GetDefaultSearchProvider();
 102   if (default_provider == NULL)
 103     return;
 104
 105   base::string16 prefix;
 106   TemplateURLRef::SearchTermsArgs search_term_args(prefix);
 107   GURL suggest_url(default_provider->suggestions_url_ref().ReplaceSearchTerms(
 108       search_term_args, template_url_service->search_terms_data()));
 109   if (!suggest_url.is_valid())
 110     return;
 111
 112   // No need to send the current page URL in personalized suggest or
 113   // most visited field trials.
 114   if (CanSendURL(input.current_url(), suggest_url, default_provider,
 115                  current_page_classification_,
 116                  template_url_service->search_terms_data(), client()) &&
 117       !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial() &&
 118       !OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
 119     // Update suggest_url to include the current_page_url.
 120     search_term_args.current_page_url = current_query_;
 121     suggest_url =
 122         GURL(default_provider->suggestions_url_ref().ReplaceSearchTerms(
 123             search_term_args, template_url_service->search_terms_data()));
 124   } else if (!ShouldShowNonContextualZeroSuggest(suggest_url,
 125                                                  input.current_url())) {
 126     return;
 127   }
 128
 129   done_ = false;
 130   // TODO(jered): Consider adding locally-sourced zero-suggestions here too.
 131   // These may be useful on the NTP or more relevant to the user than server
 132   // suggestions, if based on local browsing history.
 133   MaybeUseCachedSuggestions();
 134   Run(suggest_url);
 135 }
 136
 137 void ZeroSuggestProvider::Stop(bool clear_cached_results,
 138                                bool due_to_user_inactivity) {
 139   if (fetcher_)
 140     LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_INVALIDATED);
 141   fetcher_.reset();
 142   waiting_for_most_visited_urls_request_ = false;
 143   done_ = true;
 144
 145   if (clear_cached_results) {
 146     // We do not call Clear() on |results_| to retain |verbatim_relevance|
 147     // value in the |results_| object. |verbatim_relevance| is used at the
 148     // beginning of the next call to Start() to determine the current url
 149     // match relevance.
 150     results_.suggest_results.clear();
 151     results_.navigation_results.clear();
 152     current_query_.clear();
 153     most_visited_urls_.clear();
 154   }
 155 }
 156
 157 void ZeroSuggestProvider::DeleteMatch(const AutocompleteMatch& match) {
 158   if (OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) {
 159     // Remove the deleted match from the cache, so it is not shown to the user
 160     // again. Since we cannot remove just one result, blow away the cache.
 161     client()->GetPrefs()->SetString(omnibox::kZeroSuggestCachedResults,
 162                                     std::string());
 163   }
 164   BaseSearchProvider::DeleteMatch(match);
 165 }
 166
 167 void ZeroSuggestProvider::AddProviderInfo(ProvidersInfo* provider_info) const {
 168   BaseSearchProvider::AddProviderInfo(provider_info);
 169   if (!results_.suggest_results.empty() ||
 170       !results_.navigation_results.empty() ||
 171       !most_visited_urls_.empty())
 172     provider_info->back().set_times_returned_results_in_session(1);
 173 }
 174
 175 void ZeroSuggestProvider::ResetSession() {
 176   // The user has started editing in the omnibox, so leave
 177   // |field_trial_triggered_in_session| unchanged and set
 178   // |field_trial_triggered| to false since zero suggest is inactive now.
 179   set_field_trial_triggered(false);
 180 }
 181
 182 ZeroSuggestProvider::ZeroSuggestProvider(AutocompleteProviderClient* client,
 183                                          AutocompleteProviderListener* listener)
 184     : BaseSearchProvider(AutocompleteProvider::TYPE_ZERO_SUGGEST, client),
 185       listener_(listener),
 186       results_from_cache_(false),
 187       waiting_for_most_visited_urls_request_(false),
 188       weak_ptr_factory_(this) {
 189 }
 190
 191 ZeroSuggestProvider::~ZeroSuggestProvider() {
 192 }
 193
 194 const TemplateURL* ZeroSuggestProvider::GetTemplateURL(bool is_keyword) const {
 195   // Zero suggest provider should not receive keyword results.
 196   DCHECK(!is_keyword);
 197   return client()->GetTemplateURLService()->GetDefaultSearchProvider();
 198 }
 199
 200 const AutocompleteInput ZeroSuggestProvider::GetInput(bool is_keyword) const {
 201   // The callers of this method won't look at the AutocompleteInput's
 202   // |from_omnibox_focus| member, so we can set its value to false.
 203   return AutocompleteInput(base::string16(), base::string16::npos,
 204                            std::string(), GURL(current_query_),
 205                            current_page_classification_, true, false, false,
 206                            true, false, client()->GetSchemeClassifier());
 207 }
 208
 209 bool ZeroSuggestProvider::ShouldAppendExtraParams(
 210       const SearchSuggestionParser::SuggestResult& result) const {
 211   // We always use the default provider for search, so append the params.
 212   return true;
 213 }
 214
 215 void ZeroSuggestProvider::RecordDeletionResult(bool success) {
 216   if (success) {
 217     base::RecordAction(
 218         base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Success"));
 219   } else {
 220     base::RecordAction(
 221         base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Failure"));
 222   }
 223 }
 224
 225 void ZeroSuggestProvider::OnURLFetchComplete(const net::URLFetcher* source) {
 226   DCHECK(!done_);
 227   DCHECK_EQ(fetcher_.get(), source);
 228
 229   LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REPLY_RECEIVED);
 230
 231   bool results_updated = false;
 232   if (source->GetStatus().is_success() && source->GetResponseCode() == 200) {
 233     std::string json_data = SearchSuggestionParser::ExtractJsonData(source);
 234     scoped_ptr<base::Value> data(
 235         SearchSuggestionParser::DeserializeJsonData(json_data));
 236     if (data) {
 237       if (StoreSuggestionResponse(json_data, *data))
 238         return;
 239       results_updated = ParseSuggestResults(
 240           *data, kDefaultZeroSuggestRelevance, false, &results_);
 241     }
 242   }
 243   fetcher_.reset();
 244   done_ = true;
 245   ConvertResultsToAutocompleteMatches();
 246   listener_->OnProviderUpdate(results_updated);
 247 }
 248
 249 bool ZeroSuggestProvider::StoreSuggestionResponse(
 250     const std::string& json_data,
 251     const base::Value& parsed_data) {
 252   if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial() ||
 253       json_data.empty())
 254     return false;
 255   client()->GetPrefs()->SetString(omnibox::kZeroSuggestCachedResults,
 256                                   json_data);
 257
 258   // If we received an empty result list, we should update the display, as it
 259   // may be showing cached results that should not be shown.
 260   const base::ListValue* root_list = NULL;
 261   const base::ListValue* results_list = NULL;
 262   if (parsed_data.GetAsList(&root_list) &&
 263       root_list->GetList(1, &results_list) &&
 264       results_list->empty())
 265     return false;
 266
 267   // We are finished with the request and want to bail early.
 268   if (results_from_cache_)
 269     done_ = true;
 270
 271   return results_from_cache_;
 272 }
 273
 274 void ZeroSuggestProvider::AddSuggestResultsToMap(
 275     const SearchSuggestionParser::SuggestResults& results,
 276     MatchMap* map) {
 277   for (size_t i = 0; i < results.size(); ++i)
 278     AddMatchToMap(results[i], std::string(), i, false, false, map);
 279 }
 280
 281 AutocompleteMatch ZeroSuggestProvider::NavigationToMatch(
 282     const SearchSuggestionParser::NavigationResult& navigation) {
 283   AutocompleteMatch match(this, navigation.relevance(), false,
 284                           navigation.type());
 285   match.destination_url = navigation.url();
 286
 287   // Zero suggest results should always omit protocols and never appear bold.
 288   const std::string languages(client()->GetAcceptLanguages());
 289   match.contents = url_formatter::FormatUrl(
 290       navigation.url(), languages, url_formatter::kFormatUrlOmitAll,
 291       net::UnescapeRule::SPACES, nullptr, nullptr, nullptr);
 292   match.fill_into_edit +=
 293       AutocompleteInput::FormattedStringWithEquivalentMeaning(
 294           navigation.url(), match.contents, client()->GetSchemeClassifier());
 295
 296   AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
 297       match.contents.length(), ACMatchClassification::URL,
 298       &match.contents_class);
 299
 300   match.description =
 301       AutocompleteMatch::SanitizeString(navigation.description());
 302   AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
 303       match.description.length(), ACMatchClassification::NONE,
 304       &match.description_class);
 305   return match;
 306 }
 307
 308 void ZeroSuggestProvider::Run(const GURL& suggest_url) {
 309   if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
 310     most_visited_urls_.clear();
 311     scoped_refptr<history::TopSites> ts = client()->GetTopSites();
 312     if (ts) {
 313       waiting_for_most_visited_urls_request_ = true;
 314       ts->GetMostVisitedURLs(
 315           base::Bind(&ZeroSuggestProvider::OnMostVisitedUrlsAvailable,
 316                      weak_ptr_factory_.GetWeakPtr()), false);
 317     }
 318   } else {
 319     const int kFetcherID = 1;
 320     fetcher_ = net::URLFetcher::Create(kFetcherID, suggest_url,
 321                                        net::URLFetcher::GET, this);
 322     fetcher_->SetRequestContext(client()->GetRequestContext());
 323     fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES);
 324     // Add Chrome experiment state to the request headers.
 325     net::HttpRequestHeaders headers;
 326     variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
 327         fetcher_->GetOriginalURL(), client()->IsOffTheRecord(), false,
 328         &headers);
 329     fetcher_->SetExtraRequestHeaders(headers.ToString());
 330     fetcher_->Start();
 331     LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_SENT);
 332   }
 333 }
 334
 335 void ZeroSuggestProvider::OnMostVisitedUrlsAvailable(
 336     const history::MostVisitedURLList& urls) {
 337   if (!waiting_for_most_visited_urls_request_) return;
 338   most_visited_urls_ = urls;
 339   waiting_for_most_visited_urls_request_ = false;
 340   done_ = true;
 341   ConvertResultsToAutocompleteMatches();
 342   listener_->OnProviderUpdate(true);
 343 }
 344
 345 void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches() {
 346   matches_.clear();
 347
 348   TemplateURLService* template_url_service = client()->GetTemplateURLService();
 349   const TemplateURL* default_provider =
 350       template_url_service->GetDefaultSearchProvider();
 351   // Fail if we can't set the clickthrough URL for query suggestions.
 352   if (default_provider == NULL ||
 353       !default_provider->SupportsReplacement(
 354           template_url_service->search_terms_data()))
 355     return;
 356
 357   MatchMap map;
 358   AddSuggestResultsToMap(results_.suggest_results, &map);
 359
 360   const int num_query_results = map.size();
 361   const int num_nav_results = results_.navigation_results.size();
 362   const int num_results = num_query_results + num_nav_results;
 363   UMA_HISTOGRAM_COUNTS("ZeroSuggest.QueryResults", num_query_results);
 364   UMA_HISTOGRAM_COUNTS("ZeroSuggest.URLResults", num_nav_results);
 365   UMA_HISTOGRAM_COUNTS("ZeroSuggest.AllResults", num_results);
 366
 367   // Show Most Visited results after ZeroSuggest response is received.
 368   if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
 369     if (!current_url_match_.destination_url.is_valid())
 370       return;
 371     matches_.push_back(current_url_match_);
 372     int relevance = 600;
 373     if (num_results > 0) {
 374       UMA_HISTOGRAM_COUNTS(
 375           "Omnibox.ZeroSuggest.MostVisitedResultsCounterfactual",
 376           most_visited_urls_.size());
 377     }
 378     const base::string16 current_query_string16(
 379         base::ASCIIToUTF16(current_query_));
 380     const std::string languages(client()->GetAcceptLanguages());
 381     for (size_t i = 0; i < most_visited_urls_.size(); i++) {
 382       const history::MostVisitedURL& url = most_visited_urls_[i];
 383       SearchSuggestionParser::NavigationResult nav(
 384           client()->GetSchemeClassifier(), url.url,
 385           AutocompleteMatchType::NAVSUGGEST, url.title, std::string(), false,
 386           relevance, true, current_query_string16, languages);
 387       matches_.push_back(NavigationToMatch(nav));
 388       --relevance;
 389     }
 390     return;
 391   }
 392
 393   if (num_results == 0)
 394     return;
 395
 396   // TODO(jered): Rip this out once the first match is decoupled from the
 397   // current typing in the omnibox.
 398   matches_.push_back(current_url_match_);
 399
 400   for (MatchMap::const_iterator it(map.begin()); it != map.end(); ++it)
 401     matches_.push_back(it->second);
 402
 403   const SearchSuggestionParser::NavigationResults& nav_results(
 404       results_.navigation_results);
 405   for (SearchSuggestionParser::NavigationResults::const_iterator it(
 406            nav_results.begin()); it != nav_results.end(); ++it)
 407     matches_.push_back(NavigationToMatch(*it));
 408 }
 409
 410 AutocompleteMatch ZeroSuggestProvider::MatchForCurrentURL() {
 411   // The placeholder suggestion for the current URL has high relevance so
 412   // that it is in the first suggestion slot and inline autocompleted. It
 413   // gets dropped as soon as the user types something.
 414   return VerbatimMatchForURL(client(), permanent_text_,
 415                              current_page_classification_,
 416                              results_.verbatim_relevance);
 417 }
 418
 419 bool ZeroSuggestProvider::ShouldShowNonContextualZeroSuggest(
 420     const GURL& suggest_url,
 421     const GURL& current_page_url) const {
 422   const TemplateURLService* template_url_service =
 423       client()->GetTemplateURLService();
 424   if (!ZeroSuggestEnabled(suggest_url,
 425                           template_url_service->GetDefaultSearchProvider(),
 426                           current_page_classification_,
 427                           template_url_service->search_terms_data(), client()))
 428     return false;
 429
 430   // If we cannot send URLs, then only the MostVisited and Personalized
 431   // variations can be shown.
 432   if (!OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial() &&
 433       !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
 434     return false;
 435
 436   // Only show zero suggest for HTTP[S] pages.
 437   // TODO(mariakhomenko): We may be able to expand this set to include pages
 438   // with other schemes (e.g. chrome://). That may require improvements to
 439   // the formatting of the verbatim result returned by MatchForCurrentURL().
 440   if (!current_page_url.is_valid() ||
 441       ((current_page_url.scheme() != url::kHttpScheme) &&
 442       (current_page_url.scheme() != url::kHttpsScheme)))
 443     return false;
 444
 445   if (OmniboxFieldTrial::InZeroSuggestMostVisitedWithoutSerpFieldTrial() &&
 446       client()
 447           ->GetTemplateURLService()
 448           ->IsSearchResultsPageFromDefaultSearchProvider(current_page_url))
 449     return false;
 450
 451   return true;
 452 }
 453
 454 void ZeroSuggestProvider::MaybeUseCachedSuggestions() {
 455   if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
 456     return;
 457
 458   std::string json_data =
 459       client()->GetPrefs()->GetString(omnibox::kZeroSuggestCachedResults);
 460   if (!json_data.empty()) {
 461     scoped_ptr<base::Value> data(
 462         SearchSuggestionParser::DeserializeJsonData(json_data));
 463     if (data && ParseSuggestResults(
 464             *data, kDefaultZeroSuggestRelevance, false, &results_)) {
 465       ConvertResultsToAutocompleteMatches();
 466       results_from_cache_ = !matches_.empty();
 467     }
 468   }
 469 }