chrome/browser/autocomplete/zero_suggest_provider.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/autocomplete/zero_suggest_provider.h"
   6
   7 #include "base/callback.h"
   8 #include "base/i18n/case_conversion.h"
   9 #include "base/json/json_string_value_serializer.h"
  10 #include "base/metrics/histogram.h"
  11 #include "base/prefs/pref_service.h"
  12 #include "base/strings/string16.h"
  13 #include "base/strings/string_util.h"
  14 #include "base/strings/utf_string_conversions.h"
  15 #include "base/time/time.h"
  16 #include "chrome/browser/autocomplete/autocomplete_classifier.h"
  17 #include "chrome/browser/autocomplete/autocomplete_classifier_factory.h"
  18 #include "chrome/browser/autocomplete/autocomplete_input.h"
  19 #include "chrome/browser/autocomplete/autocomplete_match.h"
  20 #include "chrome/browser/autocomplete/autocomplete_provider_listener.h"
  21 #include "chrome/browser/autocomplete/history_url_provider.h"
  22 #include "chrome/browser/autocomplete/search_provider.h"
  23 #include "chrome/browser/autocomplete/url_prefix.h"
  24 #include "chrome/browser/history/history_types.h"
  25 #include "chrome/browser/history/top_sites.h"
  26 #include "chrome/browser/metrics/variations/variations_http_header_provider.h"
  27 #include "chrome/browser/omnibox/omnibox_field_trial.h"
  28 #include "chrome/browser/profiles/profile.h"
  29 #include "chrome/browser/search/search.h"
  30 #include "chrome/browser/search_engines/template_url_service.h"
  31 #include "chrome/browser/search_engines/template_url_service_factory.h"
  32 #include "chrome/common/net/url_fixer_upper.h"
  33 #include "chrome/common/pref_names.h"
  34 #include "chrome/common/url_constants.h"
  35 #include "content/public/browser/user_metrics.h"
  36 #include "net/base/escape.h"
  37 #include "net/base/load_flags.h"
  38 #include "net/base/net_util.h"
  39 #include "net/http/http_request_headers.h"
  40 #include "net/url_request/url_fetcher.h"
  41 #include "net/url_request/url_request_status.h"
  42 #include "url/gurl.h"
  43
  44 namespace {
  45
  46 // TODO(hfung): The histogram code was copied and modified from
  47 // search_provider.cc.  Refactor and consolidate the code.
  48 // We keep track in a histogram how many suggest requests we send, how
  49 // many suggest requests we invalidate (e.g., due to a user typing
  50 // another character), and how many replies we receive.
  51 // *** ADD NEW ENUMS AFTER ALL PREVIOUSLY DEFINED ONES! ***
  52 //     (excluding the end-of-list enum value)
  53 // We do not want values of existing enums to change or else it screws
  54 // up the statistics.
  55 enum ZeroSuggestRequestsHistogramValue {
  56   ZERO_SUGGEST_REQUEST_SENT = 1,
  57   ZERO_SUGGEST_REQUEST_INVALIDATED,
  58   ZERO_SUGGEST_REPLY_RECEIVED,
  59   ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE
  60 };
  61
  62 void LogOmniboxZeroSuggestRequest(
  63     ZeroSuggestRequestsHistogramValue request_value) {
  64   UMA_HISTOGRAM_ENUMERATION("Omnibox.ZeroSuggestRequests", request_value,
  65                             ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE);
  66 }
  67
  68 // The maximum relevance of the top match from this provider.
  69 const int kDefaultVerbatimZeroSuggestRelevance = 1300;
  70
  71 // Relevance value to use if it was not set explicitly by the server.
  72 const int kDefaultZeroSuggestRelevance = 100;
  73
  74 }  // namespace
  75
  76 // static
  77 ZeroSuggestProvider* ZeroSuggestProvider::Create(
  78     AutocompleteProviderListener* listener,
  79     Profile* profile) {
  80   return new ZeroSuggestProvider(listener, profile);
  81 }
  82
  83 void ZeroSuggestProvider::Start(const AutocompleteInput& input,
  84                                 bool minimal_changes) {
  85   matches_.clear();
  86   if (input.type() == AutocompleteInput::INVALID)
  87     return;
  88
  89   Stop(true);
  90   field_trial_triggered_ = false;
  91   field_trial_triggered_in_session_ = false;
  92   permanent_text_ = input.text();
  93   current_query_ = input.current_url().spec();
  94   current_page_classification_ = input.current_page_classification();
  95   current_url_match_ = MatchForCurrentURL();
  96
  97   const TemplateURL* default_provider =
  98      template_url_service_->GetDefaultSearchProvider();
  99   if (default_provider == NULL)
 100     return;
 101
 102   base::string16 prefix;
 103   TemplateURLRef::SearchTermsArgs search_term_args(prefix);
 104   GURL suggest_url(default_provider->suggestions_url_ref().ReplaceSearchTerms(
 105       search_term_args));
 106   if (!suggest_url.is_valid())
 107     return;
 108
 109   // No need to send the current page URL in personalized suggest field trial.
 110   if (CanSendURL(input.current_url(), suggest_url, default_provider,
 111                  current_page_classification_, profile_) &&
 112       !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) {
 113     // Update suggest_url to include the current_page_url.
 114     search_term_args.current_page_url = current_query_;
 115     suggest_url = GURL(default_provider->suggestions_url_ref().
 116                        ReplaceSearchTerms(search_term_args));
 117   } else if (!CanShowZeroSuggestWithoutSendingURL(suggest_url,
 118                                                   input.current_url())) {
 119     return;
 120   }
 121
 122   done_ = false;
 123   // TODO(jered): Consider adding locally-sourced zero-suggestions here too.
 124   // These may be useful on the NTP or more relevant to the user than server
 125   // suggestions, if based on local browsing history.
 126   Run(suggest_url);
 127 }
 128
 129 void ZeroSuggestProvider::ResetSession() {
 130   // The user has started editing in the omnibox, so leave
 131   // |field_trial_triggered_in_session_| unchanged and set
 132   // |field_trial_triggered_| to false since zero suggest is inactive now.
 133   field_trial_triggered_ = false;
 134 }
 135
 136 ZeroSuggestProvider::ZeroSuggestProvider(
 137   AutocompleteProviderListener* listener,
 138   Profile* profile)
 139     : BaseSearchProvider(listener, profile,
 140                          AutocompleteProvider::TYPE_ZERO_SUGGEST),
 141       template_url_service_(TemplateURLServiceFactory::GetForProfile(profile)),
 142       weak_ptr_factory_(this) {
 143 }
 144
 145 ZeroSuggestProvider::~ZeroSuggestProvider() {
 146 }
 147
 148 const TemplateURL* ZeroSuggestProvider::GetTemplateURL(bool is_keyword) const {
 149   // Zero suggest provider should not receive keyword results.
 150   DCHECK(!is_keyword);
 151   return template_url_service_->GetDefaultSearchProvider();
 152 }
 153
 154 const AutocompleteInput ZeroSuggestProvider::GetInput(bool is_keyword) const {
 155   return AutocompleteInput(
 156       base::string16(), base::string16::npos, base::string16(),
 157       GURL(current_query_), current_page_classification_, true, false, false,
 158       true);
 159 }
 160
 161 BaseSearchProvider::Results* ZeroSuggestProvider::GetResultsToFill(
 162     bool is_keyword) {
 163   DCHECK(!is_keyword);
 164   return &results_;
 165 }
 166
 167 bool ZeroSuggestProvider::ShouldAppendExtraParams(
 168       const SuggestResult& result) const {
 169   // We always use the default provider for search, so append the params.
 170   return true;
 171 }
 172
 173 void ZeroSuggestProvider::StopSuggest() {
 174   if (suggest_results_pending_ > 0)
 175     LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_INVALIDATED);
 176   suggest_results_pending_ = 0;
 177   fetcher_.reset();
 178 }
 179
 180 void ZeroSuggestProvider::ClearAllResults() {
 181   // We do not call Clear() on |results_| to retain |verbatim_relevance|
 182   // value in the |results_| object. |verbatim_relevance| is used at the
 183   // beginning of the next StartZeroSuggest() call to determine the current url
 184   // match relevance.
 185   results_.suggest_results.clear();
 186   results_.navigation_results.clear();
 187   current_query_.clear();
 188 }
 189
 190 int ZeroSuggestProvider::GetDefaultResultRelevance() const {
 191   return kDefaultZeroSuggestRelevance;
 192 }
 193
 194 void ZeroSuggestProvider::RecordDeletionResult(bool success) {
 195   if (success) {
 196     content::RecordAction(
 197         base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Success"));
 198   } else {
 199     content::RecordAction(
 200         base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Failure"));
 201   }
 202 }
 203
 204 void ZeroSuggestProvider::LogFetchComplete(bool success, bool is_keyword) {
 205   LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REPLY_RECEIVED);
 206 }
 207
 208 bool ZeroSuggestProvider::IsKeywordFetcher(
 209     const net::URLFetcher* fetcher) const {
 210   // ZeroSuggestProvider does not have a keyword provider.
 211   DCHECK_EQ(fetcher, fetcher_.get());
 212   return false;
 213 }
 214
 215 void ZeroSuggestProvider::UpdateMatches() {
 216   done_ = true;
 217   ConvertResultsToAutocompleteMatches();
 218 }
 219
 220 void ZeroSuggestProvider::AddSuggestResultsToMap(
 221     const SuggestResults& results,
 222     MatchMap* map) {
 223   for (size_t i = 0; i < results.size(); ++i)
 224     AddMatchToMap(results[i], std::string(), i, false, map);
 225 }
 226
 227 AutocompleteMatch ZeroSuggestProvider::NavigationToMatch(
 228     const NavigationResult& navigation) {
 229   AutocompleteMatch match(this, navigation.relevance(), false,
 230                           AutocompleteMatchType::NAVSUGGEST);
 231   match.destination_url = navigation.url();
 232
 233   // Zero suggest results should always omit protocols and never appear bold.
 234   const std::string languages(
 235       profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
 236   match.contents = net::FormatUrl(navigation.url(), languages,
 237       net::kFormatUrlOmitAll, net::UnescapeRule::SPACES, NULL, NULL, NULL);
 238   match.fill_into_edit +=
 239       AutocompleteInput::FormattedStringWithEquivalentMeaning(navigation.url(),
 240           match.contents);
 241
 242   AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
 243       match.contents.length(), ACMatchClassification::URL,
 244       &match.contents_class);
 245
 246   match.description =
 247       AutocompleteMatch::SanitizeString(navigation.description());
 248   AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
 249       match.description.length(), ACMatchClassification::NONE,
 250       &match.description_class);
 251   return match;
 252 }
 253
 254 void ZeroSuggestProvider::Run(const GURL& suggest_url) {
 255   suggest_results_pending_ = 0;
 256   const int kFetcherID = 1;
 257   fetcher_.reset(
 258       net::URLFetcher::Create(kFetcherID,
 259           suggest_url,
 260           net::URLFetcher::GET, this));
 261   fetcher_->SetRequestContext(profile_->GetRequestContext());
 262   fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES);
 263   // Add Chrome experiment state to the request headers.
 264   net::HttpRequestHeaders headers;
 265   chrome_variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
 266       fetcher_->GetOriginalURL(), profile_->IsOffTheRecord(), false, &headers);
 267   fetcher_->SetExtraRequestHeaders(headers.ToString());
 268
 269   fetcher_->Start();
 270
 271   if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
 272     most_visited_urls_.clear();
 273     history::TopSites* ts = profile_->GetTopSites();
 274     if (ts) {
 275       ts->GetMostVisitedURLs(
 276           base::Bind(&ZeroSuggestProvider::OnMostVisitedUrlsAvailable,
 277                      weak_ptr_factory_.GetWeakPtr()), false);
 278     }
 279   }
 280   suggest_results_pending_ = 1;
 281   LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_SENT);
 282 }
 283
 284 void ZeroSuggestProvider::OnMostVisitedUrlsAvailable(
 285     const history::MostVisitedURLList& urls) {
 286   most_visited_urls_ = urls;
 287 }
 288
 289 void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches() {
 290   matches_.clear();
 291
 292   const TemplateURL* default_provider =
 293       template_url_service_->GetDefaultSearchProvider();
 294   // Fail if we can't set the clickthrough URL for query suggestions.
 295   if (default_provider == NULL || !default_provider->SupportsReplacement())
 296     return;
 297
 298   MatchMap map;
 299   AddSuggestResultsToMap(results_.suggest_results, &map);
 300
 301   const int num_query_results = map.size();
 302   const int num_nav_results = results_.navigation_results.size();
 303   const int num_results = num_query_results + num_nav_results;
 304   UMA_HISTOGRAM_COUNTS("ZeroSuggest.QueryResults", num_query_results);
 305   UMA_HISTOGRAM_COUNTS("ZeroSuggest.URLResults",  num_nav_results);
 306   UMA_HISTOGRAM_COUNTS("ZeroSuggest.AllResults", num_results);
 307
 308   // Show Most Visited results after ZeroSuggest response is received.
 309   if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
 310     if (!current_url_match_.destination_url.is_valid())
 311       return;
 312     matches_.push_back(current_url_match_);
 313     int relevance = 600;
 314     if (num_results > 0) {
 315       UMA_HISTOGRAM_COUNTS(
 316           "Omnibox.ZeroSuggest.MostVisitedResultsCounterfactual",
 317           most_visited_urls_.size());
 318     }
 319     const base::string16 current_query_string16(
 320         base::ASCIIToUTF16(current_query_));
 321     const std::string languages(
 322         profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
 323     for (size_t i = 0; i < most_visited_urls_.size(); i++) {
 324       const history::MostVisitedURL& url = most_visited_urls_[i];
 325       NavigationResult nav(*this, url.url, url.title, false, relevance, true,
 326           current_query_string16, languages);
 327       matches_.push_back(NavigationToMatch(nav));
 328       --relevance;
 329     }
 330     return;
 331   }
 332
 333   if (num_results == 0)
 334     return;
 335
 336   // TODO(jered): Rip this out once the first match is decoupled from the
 337   // current typing in the omnibox.
 338   matches_.push_back(current_url_match_);
 339
 340   for (MatchMap::const_iterator it(map.begin()); it != map.end(); ++it)
 341     matches_.push_back(it->second);
 342
 343   const NavigationResults& nav_results(results_.navigation_results);
 344   for (NavigationResults::const_iterator it(nav_results.begin());
 345        it != nav_results.end(); ++it)
 346     matches_.push_back(NavigationToMatch(*it));
 347 }
 348
 349 AutocompleteMatch ZeroSuggestProvider::MatchForCurrentURL() {
 350   AutocompleteMatch match;
 351   AutocompleteClassifierFactory::GetForProfile(profile_)->Classify(
 352       permanent_text_, false, true, current_page_classification_, &match, NULL);
 353   match.is_history_what_you_typed_match = false;
 354   match.allowed_to_be_default_match = true;
 355
 356   // The placeholder suggestion for the current URL has high relevance so
 357   // that it is in the first suggestion slot and inline autocompleted. It
 358   // gets dropped as soon as the user types something.
 359   match.relevance = GetVerbatimRelevance();
 360
 361   return match;
 362 }
 363
 364 int ZeroSuggestProvider::GetVerbatimRelevance() const {
 365   return results_.verbatim_relevance >= 0 ?
 366       results_.verbatim_relevance : kDefaultVerbatimZeroSuggestRelevance;
 367 }
 368
 369 bool ZeroSuggestProvider::CanShowZeroSuggestWithoutSendingURL(
 370     const GURL& suggest_url,
 371     const GURL& current_page_url) const {
 372   if (!ZeroSuggestEnabled(suggest_url,
 373                           template_url_service_->GetDefaultSearchProvider(),
 374                           current_page_classification_, profile_))
 375     return false;
 376
 377   // If we cannot send URLs, then only the MostVisited and Personalized
 378   // variations can be shown.
 379   if (!OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial() &&
 380       !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
 381     return false;
 382
 383   // Only show zero suggest for HTTP[S] pages.
 384   // TODO(mariakhomenko): We may be able to expand this set to include pages
 385   // with other schemes (e.g. chrome://). That may require improvements to
 386   // the formatting of the verbatim result returned by MatchForCurrentURL().
 387   if (!current_page_url.is_valid() ||
 388       ((current_page_url.scheme() != content::kHttpScheme) &&
 389       (current_page_url.scheme() != content::kHttpsScheme)))
 390     return false;
 391
 392   return true;
 393 }