components/omnibox/browser/zero_suggest_provider.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/omnibox/browser/zero_suggest_provider.h"
   6
   7 #include "base/callback.h"
   8 #include "base/i18n/case_conversion.h"
   9 #include "base/json/json_string_value_serializer.h"
  10 #include "base/metrics/histogram.h"
  11 #include "base/metrics/user_metrics.h"
  12 #include "base/prefs/pref_service.h"
  13 #include "base/strings/string16.h"
  14 #include "base/strings/string_util.h"
  15 #include "base/strings/utf_string_conversions.h"
  16 #include "base/time/time.h"
  17 #include "components/data_use_measurement/core/data_use_user_data.h"
  18 #include "components/history/core/browser/history_types.h"
  19 #include "components/history/core/browser/top_sites.h"
  20 #include "components/metrics/proto/omnibox_input_type.pb.h"
  21 #include "components/omnibox/browser/autocomplete_classifier.h"
  22 #include "components/omnibox/browser/autocomplete_input.h"
  23 #include "components/omnibox/browser/autocomplete_match.h"
  24 #include "components/omnibox/browser/autocomplete_provider_listener.h"
  25 #include "components/omnibox/browser/history_url_provider.h"
  26 #include "components/omnibox/browser/omnibox_field_trial.h"
  27 #include "components/omnibox/browser/omnibox_pref_names.h"
  28 #include "components/omnibox/browser/search_provider.h"
  29 #include "components/omnibox/browser/verbatim_match.h"
  30 #include "components/pref_registry/pref_registry_syncable.h"
  31 #include "components/search_engines/template_url_service.h"
  32 #include "components/url_formatter/url_formatter.h"
  33 #include "components/variations/net/variations_http_header_provider.h"
  34 #include "net/base/escape.h"
  35 #include "net/base/load_flags.h"
  36 #include "net/http/http_request_headers.h"
  37 #include "net/url_request/url_fetcher.h"
  38 #include "net/url_request/url_request_status.h"
  39 #include "url/gurl.h"
  40
  41 namespace {
  42
  43 // TODO(hfung): The histogram code was copied and modified from
  44 // search_provider.cc.  Refactor and consolidate the code.
  45 // We keep track in a histogram how many suggest requests we send, how
  46 // many suggest requests we invalidate (e.g., due to a user typing
  47 // another character), and how many replies we receive.
  48 // *** ADD NEW ENUMS AFTER ALL PREVIOUSLY DEFINED ONES! ***
  49 //     (excluding the end-of-list enum value)
  50 // We do not want values of existing enums to change or else it screws
  51 // up the statistics.
  52 enum ZeroSuggestRequestsHistogramValue {
  53   ZERO_SUGGEST_REQUEST_SENT = 1,
  54   ZERO_SUGGEST_REQUEST_INVALIDATED,
  55   ZERO_SUGGEST_REPLY_RECEIVED,
  56   ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE
  57 };
  58
  59 void LogOmniboxZeroSuggestRequest(
  60     ZeroSuggestRequestsHistogramValue request_value) {
  61   UMA_HISTOGRAM_ENUMERATION("Omnibox.ZeroSuggestRequests", request_value,
  62                             ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE);
  63 }
  64
  65 // Relevance value to use if it was not set explicitly by the server.
  66 const int kDefaultZeroSuggestRelevance = 100;
  67
  68 }  // namespace
  69
  70 // static
  71 ZeroSuggestProvider* ZeroSuggestProvider::Create(
  72     AutocompleteProviderClient* client,
  73     AutocompleteProviderListener* listener) {
  74   return new ZeroSuggestProvider(client, listener);
  75 }
  76
  77 // static
  78 void ZeroSuggestProvider::RegisterProfilePrefs(
  79     user_prefs::PrefRegistrySyncable* registry) {
  80   registry->RegisterStringPref(omnibox::kZeroSuggestCachedResults,
  81                                std::string());
  82 }
  83
  84 void ZeroSuggestProvider::Start(const AutocompleteInput& input,
  85                                 bool minimal_changes) {
  86   matches_.clear();
  87   if (!input.from_omnibox_focus() ||
  88       input.type() == metrics::OmniboxInputType::INVALID)
  89     return;
  90
  91   Stop(true, false);
  92   set_field_trial_triggered(false);
  93   set_field_trial_triggered_in_session(false);
  94   results_from_cache_ = false;
  95   permanent_text_ = input.text();
  96   current_query_ = input.current_url().spec();
  97   current_page_classification_ = input.current_page_classification();
  98   current_url_match_ = MatchForCurrentURL();
  99   TemplateURLService* template_url_service = client()->GetTemplateURLService();
 100
 101   const TemplateURL* default_provider =
 102       template_url_service->GetDefaultSearchProvider();
 103   if (default_provider == NULL)
 104     return;
 105
 106   base::string16 prefix;
 107   TemplateURLRef::SearchTermsArgs search_term_args(prefix);
 108   GURL suggest_url(default_provider->suggestions_url_ref().ReplaceSearchTerms(
 109       search_term_args, template_url_service->search_terms_data()));
 110   if (!suggest_url.is_valid())
 111     return;
 112
 113   // No need to send the current page URL in personalized suggest or
 114   // most visited field trials.
 115   if (CanSendURL(input.current_url(), suggest_url, default_provider,
 116                  current_page_classification_,
 117                  template_url_service->search_terms_data(), client()) &&
 118       !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial() &&
 119       !OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
 120     // Update suggest_url to include the current_page_url.
 121     search_term_args.current_page_url = current_query_;
 122     suggest_url =
 123         GURL(default_provider->suggestions_url_ref().ReplaceSearchTerms(
 124             search_term_args, template_url_service->search_terms_data()));
 125   } else if (!ShouldShowNonContextualZeroSuggest(suggest_url,
 126                                                  input.current_url())) {
 127     return;
 128   }
 129
 130   done_ = false;
 131   // TODO(jered): Consider adding locally-sourced zero-suggestions here too.
 132   // These may be useful on the NTP or more relevant to the user than server
 133   // suggestions, if based on local browsing history.
 134   MaybeUseCachedSuggestions();
 135   Run(suggest_url);
 136 }
 137
 138 void ZeroSuggestProvider::Stop(bool clear_cached_results,
 139                                bool due_to_user_inactivity) {
 140   if (fetcher_)
 141     LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_INVALIDATED);
 142   fetcher_.reset();
 143   waiting_for_most_visited_urls_request_ = false;
 144   done_ = true;
 145
 146   if (clear_cached_results) {
 147     // We do not call Clear() on |results_| to retain |verbatim_relevance|
 148     // value in the |results_| object. |verbatim_relevance| is used at the
 149     // beginning of the next call to Start() to determine the current url
 150     // match relevance.
 151     results_.suggest_results.clear();
 152     results_.navigation_results.clear();
 153     current_query_.clear();
 154     most_visited_urls_.clear();
 155   }
 156 }
 157
 158 void ZeroSuggestProvider::DeleteMatch(const AutocompleteMatch& match) {
 159   if (OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) {
 160     // Remove the deleted match from the cache, so it is not shown to the user
 161     // again. Since we cannot remove just one result, blow away the cache.
 162     client()->GetPrefs()->SetString(omnibox::kZeroSuggestCachedResults,
 163                                     std::string());
 164   }
 165   BaseSearchProvider::DeleteMatch(match);
 166 }
 167
 168 void ZeroSuggestProvider::AddProviderInfo(ProvidersInfo* provider_info) const {
 169   BaseSearchProvider::AddProviderInfo(provider_info);
 170   if (!results_.suggest_results.empty() ||
 171       !results_.navigation_results.empty() ||
 172       !most_visited_urls_.empty())
 173     provider_info->back().set_times_returned_results_in_session(1);
 174 }
 175
 176 void ZeroSuggestProvider::ResetSession() {
 177   // The user has started editing in the omnibox, so leave
 178   // |field_trial_triggered_in_session| unchanged and set
 179   // |field_trial_triggered| to false since zero suggest is inactive now.
 180   set_field_trial_triggered(false);
 181 }
 182
 183 ZeroSuggestProvider::ZeroSuggestProvider(AutocompleteProviderClient* client,
 184                                          AutocompleteProviderListener* listener)
 185     : BaseSearchProvider(AutocompleteProvider::TYPE_ZERO_SUGGEST, client),
 186       listener_(listener),
 187       results_from_cache_(false),
 188       waiting_for_most_visited_urls_request_(false),
 189       weak_ptr_factory_(this) {
 190 }
 191
 192 ZeroSuggestProvider::~ZeroSuggestProvider() {
 193 }
 194
 195 const TemplateURL* ZeroSuggestProvider::GetTemplateURL(bool is_keyword) const {
 196   // Zero suggest provider should not receive keyword results.
 197   DCHECK(!is_keyword);
 198   return client()->GetTemplateURLService()->GetDefaultSearchProvider();
 199 }
 200
 201 const AutocompleteInput ZeroSuggestProvider::GetInput(bool is_keyword) const {
 202   // The callers of this method won't look at the AutocompleteInput's
 203   // |from_omnibox_focus| member, so we can set its value to false.
 204   return AutocompleteInput(base::string16(), base::string16::npos,
 205                            std::string(), GURL(current_query_),
 206                            current_page_classification_, true, false, false,
 207                            true, false, client()->GetSchemeClassifier());
 208 }
 209
 210 bool ZeroSuggestProvider::ShouldAppendExtraParams(
 211       const SearchSuggestionParser::SuggestResult& result) const {
 212   // We always use the default provider for search, so append the params.
 213   return true;
 214 }
 215
 216 void ZeroSuggestProvider::RecordDeletionResult(bool success) {
 217   if (success) {
 218     base::RecordAction(
 219         base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Success"));
 220   } else {
 221     base::RecordAction(
 222         base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Failure"));
 223   }
 224 }
 225
 226 void ZeroSuggestProvider::OnURLFetchComplete(const net::URLFetcher* source) {
 227   DCHECK(!done_);
 228   DCHECK_EQ(fetcher_.get(), source);
 229
 230   LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REPLY_RECEIVED);
 231
 232   bool results_updated = false;
 233   if (source->GetStatus().is_success() && source->GetResponseCode() == 200) {
 234     std::string json_data = SearchSuggestionParser::ExtractJsonData(source);
 235     scoped_ptr<base::Value> data(
 236         SearchSuggestionParser::DeserializeJsonData(json_data));
 237     if (data) {
 238       if (StoreSuggestionResponse(json_data, *data))
 239         return;
 240       results_updated = ParseSuggestResults(
 241           *data, kDefaultZeroSuggestRelevance, false, &results_);
 242     }
 243   }
 244   fetcher_.reset();
 245   done_ = true;
 246   ConvertResultsToAutocompleteMatches();
 247   listener_->OnProviderUpdate(results_updated);
 248 }
 249
 250 bool ZeroSuggestProvider::StoreSuggestionResponse(
 251     const std::string& json_data,
 252     const base::Value& parsed_data) {
 253   if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial() ||
 254       json_data.empty())
 255     return false;
 256   client()->GetPrefs()->SetString(omnibox::kZeroSuggestCachedResults,
 257                                   json_data);
 258
 259   // If we received an empty result list, we should update the display, as it
 260   // may be showing cached results that should not be shown.
 261   const base::ListValue* root_list = NULL;
 262   const base::ListValue* results_list = NULL;
 263   if (parsed_data.GetAsList(&root_list) &&
 264       root_list->GetList(1, &results_list) &&
 265       results_list->empty())
 266     return false;
 267
 268   // We are finished with the request and want to bail early.
 269   if (results_from_cache_)
 270     done_ = true;
 271
 272   return results_from_cache_;
 273 }
 274
 275 void ZeroSuggestProvider::AddSuggestResultsToMap(
 276     const SearchSuggestionParser::SuggestResults& results,
 277     MatchMap* map) {
 278   for (size_t i = 0; i < results.size(); ++i)
 279     AddMatchToMap(results[i], std::string(), i, false, false, map);
 280 }
 281
 282 AutocompleteMatch ZeroSuggestProvider::NavigationToMatch(
 283     const SearchSuggestionParser::NavigationResult& navigation) {
 284   AutocompleteMatch match(this, navigation.relevance(), false,
 285                           navigation.type());
 286   match.destination_url = navigation.url();
 287
 288   // Zero suggest results should always omit protocols and never appear bold.
 289   const std::string languages(client()->GetAcceptLanguages());
 290   match.contents = url_formatter::FormatUrl(
 291       navigation.url(), languages, url_formatter::kFormatUrlOmitAll,
 292       net::UnescapeRule::SPACES, nullptr, nullptr, nullptr);
 293   match.fill_into_edit +=
 294       AutocompleteInput::FormattedStringWithEquivalentMeaning(
 295           navigation.url(), match.contents, client()->GetSchemeClassifier());
 296
 297   AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
 298       match.contents.length(), ACMatchClassification::URL,
 299       &match.contents_class);
 300
 301   match.description =
 302       AutocompleteMatch::SanitizeString(navigation.description());
 303   AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
 304       match.description.length(), ACMatchClassification::NONE,
 305       &match.description_class);
 306   return match;
 307 }
 308
 309 void ZeroSuggestProvider::Run(const GURL& suggest_url) {
 310   if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
 311     most_visited_urls_.clear();
 312     scoped_refptr<history::TopSites> ts = client()->GetTopSites();
 313     if (ts) {
 314       waiting_for_most_visited_urls_request_ = true;
 315       ts->GetMostVisitedURLs(
 316           base::Bind(&ZeroSuggestProvider::OnMostVisitedUrlsAvailable,
 317                      weak_ptr_factory_.GetWeakPtr()), false);
 318     }
 319   } else {
 320     const int kFetcherID = 1;
 321     fetcher_ = net::URLFetcher::Create(kFetcherID, suggest_url,
 322                                        net::URLFetcher::GET, this);
 323     data_use_measurement::DataUseUserData::AttachToFetcher(
 324         fetcher_.get(), data_use_measurement::DataUseUserData::OMNIBOX);
 325     fetcher_->SetRequestContext(client()->GetRequestContext());
 326     fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES);
 327     // Add Chrome experiment state to the request headers.
 328     net::HttpRequestHeaders headers;
 329     variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
 330         fetcher_->GetOriginalURL(), client()->IsOffTheRecord(), false,
 331         &headers);
 332     fetcher_->SetExtraRequestHeaders(headers.ToString());
 333     fetcher_->Start();
 334     LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_SENT);
 335   }
 336 }
 337
 338 void ZeroSuggestProvider::OnMostVisitedUrlsAvailable(
 339     const history::MostVisitedURLList& urls) {
 340   if (!waiting_for_most_visited_urls_request_) return;
 341   most_visited_urls_ = urls;
 342   waiting_for_most_visited_urls_request_ = false;
 343   done_ = true;
 344   ConvertResultsToAutocompleteMatches();
 345   listener_->OnProviderUpdate(true);
 346 }
 347
 348 void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches() {
 349   matches_.clear();
 350
 351   TemplateURLService* template_url_service = client()->GetTemplateURLService();
 352   const TemplateURL* default_provider =
 353       template_url_service->GetDefaultSearchProvider();
 354   // Fail if we can't set the clickthrough URL for query suggestions.
 355   if (default_provider == NULL ||
 356       !default_provider->SupportsReplacement(
 357           template_url_service->search_terms_data()))
 358     return;
 359
 360   MatchMap map;
 361   AddSuggestResultsToMap(results_.suggest_results, &map);
 362
 363   const int num_query_results = map.size();
 364   const int num_nav_results = results_.navigation_results.size();
 365   const int num_results = num_query_results + num_nav_results;
 366   UMA_HISTOGRAM_COUNTS("ZeroSuggest.QueryResults", num_query_results);
 367   UMA_HISTOGRAM_COUNTS("ZeroSuggest.URLResults", num_nav_results);
 368   UMA_HISTOGRAM_COUNTS("ZeroSuggest.AllResults", num_results);
 369
 370   // Show Most Visited results after ZeroSuggest response is received.
 371   if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
 372     if (!current_url_match_.destination_url.is_valid())
 373       return;
 374     matches_.push_back(current_url_match_);
 375     int relevance = 600;
 376     if (num_results > 0) {
 377       UMA_HISTOGRAM_COUNTS(
 378           "Omnibox.ZeroSuggest.MostVisitedResultsCounterfactual",
 379           most_visited_urls_.size());
 380     }
 381     const base::string16 current_query_string16(
 382         base::ASCIIToUTF16(current_query_));
 383     const std::string languages(client()->GetAcceptLanguages());
 384     for (size_t i = 0; i < most_visited_urls_.size(); i++) {
 385       const history::MostVisitedURL& url = most_visited_urls_[i];
 386       SearchSuggestionParser::NavigationResult nav(
 387           client()->GetSchemeClassifier(), url.url,
 388           AutocompleteMatchType::NAVSUGGEST, url.title, std::string(), false,
 389           relevance, true, current_query_string16, languages);
 390       matches_.push_back(NavigationToMatch(nav));
 391       --relevance;
 392     }
 393     return;
 394   }
 395
 396   if (num_results == 0)
 397     return;
 398
 399   // TODO(jered): Rip this out once the first match is decoupled from the
 400   // current typing in the omnibox.
 401   matches_.push_back(current_url_match_);
 402
 403   for (MatchMap::const_iterator it(map.begin()); it != map.end(); ++it)
 404     matches_.push_back(it->second);
 405
 406   const SearchSuggestionParser::NavigationResults& nav_results(
 407       results_.navigation_results);
 408   for (SearchSuggestionParser::NavigationResults::const_iterator it(
 409            nav_results.begin()); it != nav_results.end(); ++it)
 410     matches_.push_back(NavigationToMatch(*it));
 411 }
 412
 413 AutocompleteMatch ZeroSuggestProvider::MatchForCurrentURL() {
 414   // The placeholder suggestion for the current URL has high relevance so
 415   // that it is in the first suggestion slot and inline autocompleted. It
 416   // gets dropped as soon as the user types something.
 417   return VerbatimMatchForURL(client(), permanent_text_,
 418                              current_page_classification_,
 419                              results_.verbatim_relevance);
 420 }
 421
 422 bool ZeroSuggestProvider::ShouldShowNonContextualZeroSuggest(
 423     const GURL& suggest_url,
 424     const GURL& current_page_url) const {
 425   const TemplateURLService* template_url_service =
 426       client()->GetTemplateURLService();
 427   if (!ZeroSuggestEnabled(suggest_url,
 428                           template_url_service->GetDefaultSearchProvider(),
 429                           current_page_classification_,
 430                           template_url_service->search_terms_data(), client()))
 431     return false;
 432
 433   // If we cannot send URLs, then only the MostVisited and Personalized
 434   // variations can be shown.
 435   if (!OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial() &&
 436       !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
 437     return false;
 438
 439   // Only show zero suggest for HTTP[S] pages.
 440   // TODO(mariakhomenko): We may be able to expand this set to include pages
 441   // with other schemes (e.g. chrome://). That may require improvements to
 442   // the formatting of the verbatim result returned by MatchForCurrentURL().
 443   if (!current_page_url.is_valid() ||
 444       ((current_page_url.scheme() != url::kHttpScheme) &&
 445       (current_page_url.scheme() != url::kHttpsScheme)))
 446     return false;
 447
 448   if (OmniboxFieldTrial::InZeroSuggestMostVisitedWithoutSerpFieldTrial() &&
 449       client()
 450           ->GetTemplateURLService()
 451           ->IsSearchResultsPageFromDefaultSearchProvider(current_page_url))
 452     return false;
 453
 454   return true;
 455 }
 456
 457 void ZeroSuggestProvider::MaybeUseCachedSuggestions() {
 458   if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
 459     return;
 460
 461   std::string json_data =
 462       client()->GetPrefs()->GetString(omnibox::kZeroSuggestCachedResults);
 463   if (!json_data.empty()) {
 464     scoped_ptr<base::Value> data(
 465         SearchSuggestionParser::DeserializeJsonData(json_data));
 466     if (data && ParseSuggestResults(
 467             *data, kDefaultZeroSuggestRelevance, false, &results_)) {
 468       ConvertResultsToAutocompleteMatches();
 469       results_from_cache_ = !matches_.empty();
 470     }
 471   }
 472 }