chrome/browser/autocomplete/zero_suggest_provider.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/autocomplete/zero_suggest_provider.h"
   6
   7 #include "base/callback.h"
   8 #include "base/i18n/case_conversion.h"
   9 #include "base/json/json_string_value_serializer.h"
  10 #include "base/metrics/histogram.h"
  11 #include "base/prefs/pref_service.h"
  12 #include "base/strings/string16.h"
  13 #include "base/strings/string_util.h"
  14 #include "base/strings/utf_string_conversions.h"
  15 #include "base/time/time.h"
  16 #include "chrome/browser/autocomplete/autocomplete_classifier.h"
  17 #include "chrome/browser/autocomplete/autocomplete_classifier_factory.h"
  18 #include "chrome/browser/autocomplete/autocomplete_input.h"
  19 #include "chrome/browser/autocomplete/autocomplete_match.h"
  20 #include "chrome/browser/autocomplete/autocomplete_provider_listener.h"
  21 #include "chrome/browser/autocomplete/history_url_provider.h"
  22 #include "chrome/browser/autocomplete/search_provider.h"
  23 #include "chrome/browser/autocomplete/url_prefix.h"
  24 #include "chrome/browser/history/history_types.h"
  25 #include "chrome/browser/history/top_sites.h"
  26 #include "chrome/browser/metrics/variations/variations_http_header_provider.h"
  27 #include "chrome/browser/omnibox/omnibox_field_trial.h"
  28 #include "chrome/browser/profiles/profile.h"
  29 #include "chrome/browser/search/search.h"
  30 #include "chrome/browser/search_engines/template_url_service.h"
  31 #include "chrome/browser/search_engines/template_url_service_factory.h"
  32 #include "chrome/common/net/url_fixer_upper.h"
  33 #include "chrome/common/pref_names.h"
  34 #include "chrome/common/url_constants.h"
  35 #include "net/base/escape.h"
  36 #include "net/base/load_flags.h"
  37 #include "net/base/net_util.h"
  38 #include "net/http/http_request_headers.h"
  39 #include "net/http/http_response_headers.h"
  40 #include "net/url_request/url_fetcher.h"
  41 #include "net/url_request/url_request_status.h"
  42 #include "url/gurl.h"
  43
  44 namespace {
  45
  46 // TODO(hfung): The histogram code was copied and modified from
  47 // search_provider.cc.  Refactor and consolidate the code.
  48 // We keep track in a histogram how many suggest requests we send, how
  49 // many suggest requests we invalidate (e.g., due to a user typing
  50 // another character), and how many replies we receive.
  51 // *** ADD NEW ENUMS AFTER ALL PREVIOUSLY DEFINED ONES! ***
  52 //     (excluding the end-of-list enum value)
  53 // We do not want values of existing enums to change or else it screws
  54 // up the statistics.
  55 enum ZeroSuggestRequestsHistogramValue {
  56   ZERO_SUGGEST_REQUEST_SENT = 1,
  57   ZERO_SUGGEST_REQUEST_INVALIDATED,
  58   ZERO_SUGGEST_REPLY_RECEIVED,
  59   ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE
  60 };
  61
  62 void LogOmniboxZeroSuggestRequest(
  63     ZeroSuggestRequestsHistogramValue request_value) {
  64   UMA_HISTOGRAM_ENUMERATION("Omnibox.ZeroSuggestRequests", request_value,
  65                             ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE);
  66 }
  67
  68 // The maximum relevance of the top match from this provider.
  69 const int kDefaultVerbatimZeroSuggestRelevance = 1300;
  70
  71 // Relevance value to use if it was not set explicitly by the server.
  72 const int kDefaultZeroSuggestRelevance = 100;
  73
  74 }  // namespace
  75
  76 // static
  77 ZeroSuggestProvider* ZeroSuggestProvider::Create(
  78     AutocompleteProviderListener* listener,
  79     Profile* profile) {
  80   return new ZeroSuggestProvider(listener, profile);
  81 }
  82
  83 void ZeroSuggestProvider::Start(const AutocompleteInput& input,
  84                                 bool /*minimal_changes*/) {
  85 }
  86
  87 void ZeroSuggestProvider::Stop(bool clear_cached_results) {
  88   if (have_pending_request_)
  89     LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_INVALIDATED);
  90   have_pending_request_ = false;
  91   fetcher_.reset();
  92   done_ = true;
  93   if (clear_cached_results) {
  94     query_matches_map_.clear();
  95     navigation_results_.clear();
  96     current_query_.clear();
  97     matches_.clear();
  98   }
  99 }
 100
 101 void ZeroSuggestProvider::AddProviderInfo(ProvidersInfo* provider_info) const {
 102   provider_info->push_back(metrics::OmniboxEventProto_ProviderInfo());
 103   metrics::OmniboxEventProto_ProviderInfo& new_entry = provider_info->back();
 104   new_entry.set_provider(AsOmniboxEventProviderType());
 105   new_entry.set_provider_done(done_);
 106   std::vector<uint32> field_trial_hashes;
 107   OmniboxFieldTrial::GetActiveSuggestFieldTrialHashes(&field_trial_hashes);
 108   for (size_t i = 0; i < field_trial_hashes.size(); ++i) {
 109     if (field_trial_triggered_)
 110       new_entry.mutable_field_trial_triggered()->Add(field_trial_hashes[i]);
 111     if (field_trial_triggered_in_session_) {
 112       new_entry.mutable_field_trial_triggered_in_session()->Add(
 113           field_trial_hashes[i]);
 114      }
 115   }
 116 }
 117
 118 void ZeroSuggestProvider::ResetSession() {
 119   // The user has started editing in the omnibox, so leave
 120   // |field_trial_triggered_in_session_| unchanged and set
 121   // |field_trial_triggered_| to false since zero suggest is inactive now.
 122   field_trial_triggered_ = false;
 123   Stop(true);
 124 }
 125
 126 void ZeroSuggestProvider::OnURLFetchComplete(const net::URLFetcher* source) {
 127   have_pending_request_ = false;
 128   LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REPLY_RECEIVED);
 129
 130   std::string json_data;
 131   source->GetResponseAsString(&json_data);
 132   const bool request_succeeded =
 133       source->GetStatus().is_success() && source->GetResponseCode() == 200;
 134
 135   if (request_succeeded) {
 136     scoped_ptr<base::Value> data(
 137         SearchProvider::DeserializeJsonData(json_data));
 138     if (data.get())
 139       ParseSuggestResults(*data.get());
 140   }
 141   done_ = true;
 142
 143   ConvertResultsToAutocompleteMatches();
 144   if (!matches_.empty())
 145     listener_->OnProviderUpdate(true);
 146 }
 147
 148 void ZeroSuggestProvider::StartZeroSuggest(
 149     const GURL& current_page_url,
 150     AutocompleteInput::PageClassification page_classification,
 151     const base::string16& permanent_text) {
 152   Stop(true);
 153   field_trial_triggered_ = false;
 154   field_trial_triggered_in_session_ = false;
 155   permanent_text_ = permanent_text;
 156   current_query_ = current_page_url.spec();
 157   current_page_classification_ = page_classification;
 158   current_url_match_ = MatchForCurrentURL();
 159
 160   const TemplateURL* default_provider =
 161      template_url_service_->GetDefaultSearchProvider();
 162   if (default_provider == NULL)
 163     return;
 164   base::string16 prefix;
 165   TemplateURLRef::SearchTermsArgs search_term_args(prefix);
 166   search_term_args.current_page_url = current_query_;
 167   GURL suggest_url(default_provider->suggestions_url_ref().
 168                    ReplaceSearchTerms(search_term_args));
 169   if (!SearchProvider::CanSendURL(
 170           current_page_url, suggest_url,
 171           template_url_service_->GetDefaultSearchProvider(),
 172           page_classification, profile_) ||
 173       !OmniboxFieldTrial::InZeroSuggestFieldTrial())
 174     return;
 175   verbatim_relevance_ = kDefaultVerbatimZeroSuggestRelevance;
 176   done_ = false;
 177   // TODO(jered): Consider adding locally-sourced zero-suggestions here too.
 178   // These may be useful on the NTP or more relevant to the user than server
 179   // suggestions, if based on local browsing history.
 180   Run(suggest_url);
 181 }
 182
 183 ZeroSuggestProvider::ZeroSuggestProvider(
 184   AutocompleteProviderListener* listener,
 185   Profile* profile)
 186     : AutocompleteProvider(listener, profile,
 187           AutocompleteProvider::TYPE_ZERO_SUGGEST),
 188       template_url_service_(TemplateURLServiceFactory::GetForProfile(profile)),
 189       have_pending_request_(false),
 190       verbatim_relevance_(kDefaultVerbatimZeroSuggestRelevance),
 191       field_trial_triggered_(false),
 192       field_trial_triggered_in_session_(false),
 193       weak_ptr_factory_(this) {
 194 }
 195
 196 ZeroSuggestProvider::~ZeroSuggestProvider() {
 197 }
 198
 199 void ZeroSuggestProvider::FillResults(
 200     const base::Value& root_val,
 201     int* verbatim_relevance,
 202     SearchProvider::SuggestResults* suggest_results,
 203     SearchProvider::NavigationResults* navigation_results) {
 204   base::string16 query;
 205   const base::ListValue* root_list = NULL;
 206   const base::ListValue* results = NULL;
 207   const base::ListValue* relevances = NULL;
 208   // The response includes the query, which should be empty for ZeroSuggest
 209   // responses.
 210   if (!root_val.GetAsList(&root_list) || !root_list->GetString(0, &query) ||
 211       (!query.empty()) || !root_list->GetList(1, &results))
 212     return;
 213
 214   // 3rd element: Description list.
 215   const base::ListValue* descriptions = NULL;
 216   root_list->GetList(2, &descriptions);
 217
 218   // 4th element: Disregard the query URL list for now.
 219
 220   // Reset suggested relevance information from the provider.
 221   *verbatim_relevance = kDefaultVerbatimZeroSuggestRelevance;
 222
 223   // 5th element: Optional key-value pairs from the Suggest server.
 224   const base::ListValue* types = NULL;
 225   const base::DictionaryValue* extras = NULL;
 226   if (root_list->GetDictionary(4, &extras)) {
 227     extras->GetList("google:suggesttype", &types);
 228
 229     // Discard this list if its size does not match that of the suggestions.
 230     if (extras->GetList("google:suggestrelevance", &relevances) &&
 231         relevances->GetSize() != results->GetSize())
 232       relevances = NULL;
 233     extras->GetInteger("google:verbatimrelevance", verbatim_relevance);
 234
 235     // Check if the active suggest field trial (if any) has triggered.
 236     bool triggered = false;
 237     extras->GetBoolean("google:fieldtrialtriggered", &triggered);
 238     field_trial_triggered_ |= triggered;
 239     field_trial_triggered_in_session_ |= triggered;
 240   }
 241
 242   // Clear the previous results now that new results are available.
 243   suggest_results->clear();
 244   navigation_results->clear();
 245
 246   base::string16 result, title;
 247   std::string type;
 248   const base::string16 current_query_string16 =
 249       base::ASCIIToUTF16(current_query_);
 250   const std::string languages(
 251       profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
 252   for (size_t index = 0; results->GetString(index, &result); ++index) {
 253     // Google search may return empty suggestions for weird input characters,
 254     // they make no sense at all and can cause problems in our code.
 255     if (result.empty())
 256       continue;
 257
 258     int relevance = kDefaultZeroSuggestRelevance;
 259
 260     // Apply valid suggested relevance scores; discard invalid lists.
 261     if (relevances != NULL && !relevances->GetInteger(index, &relevance))
 262       relevances = NULL;
 263     if (types && types->GetString(index, &type) && (type == "NAVIGATION")) {
 264       // Do not blindly trust the URL coming from the server to be valid.
 265       GURL url(URLFixerUpper::FixupURL(
 266           base::UTF16ToUTF8(result), std::string()));
 267       if (url.is_valid()) {
 268         if (descriptions != NULL)
 269           descriptions->GetString(index, &title);
 270         navigation_results->push_back(SearchProvider::NavigationResult(
 271             *this, url, title, false, relevance, relevances != NULL,
 272             current_query_string16, languages));
 273       }
 274     } else {
 275       suggest_results->push_back(SearchProvider::SuggestResult(
 276           result, AutocompleteMatchType::SEARCH_SUGGEST, result,
 277           base::string16(), std::string(), std::string(), false, relevance,
 278           relevances != NULL, false, current_query_string16));
 279     }
 280   }
 281 }
 282
 283 void ZeroSuggestProvider::AddSuggestResultsToMap(
 284     const SearchProvider::SuggestResults& results,
 285     const TemplateURL* template_url,
 286     SearchProvider::MatchMap* map) {
 287   for (size_t i = 0; i < results.size(); ++i) {
 288     AddMatchToMap(results[i].relevance(), AutocompleteMatchType::SEARCH_SUGGEST,
 289                   template_url, results[i].suggestion(), i, map);
 290   }
 291 }
 292
 293 void ZeroSuggestProvider::AddMatchToMap(int relevance,
 294                                         AutocompleteMatch::Type type,
 295                                         const TemplateURL* template_url,
 296                                         const base::string16& query_string,
 297                                         int accepted_suggestion,
 298                                         SearchProvider::MatchMap* map) {
 299   // Pass in query_string as the input_text to avoid bolding.
 300   SearchProvider::SuggestResult suggestion(
 301       query_string, type, query_string, base::string16(), std::string(),
 302       std::string(), false, relevance, true, false, query_string);
 303   // TODO(samarth|melevin): use the actual omnibox margin here as well instead
 304   // of passing in -1.
 305   AutocompleteMatch match = SearchProvider::CreateSearchSuggestion(
 306       this, AutocompleteInput(), query_string, suggestion, template_url,
 307       accepted_suggestion, -1, true);
 308   if (!match.destination_url.is_valid())
 309     return;
 310
 311   // Try to add |match| to |map|.  If a match for |query_string| is already in
 312   // |map|, replace it if |match| is more relevant.
 313   // NOTE: Keep this ToLower() call in sync with url_database.cc.
 314   SearchProvider::MatchKey match_key(
 315       std::make_pair(base::i18n::ToLower(query_string), std::string()));
 316   const std::pair<SearchProvider::MatchMap::iterator, bool> i(map->insert(
 317       std::make_pair(match_key, match)));
 318   // NOTE: We purposefully do a direct relevance comparison here instead of
 319   // using AutocompleteMatch::MoreRelevant(), so that we'll prefer "items added
 320   // first" rather than "items alphabetically first" when the scores are equal.
 321   // The only case this matters is when a user has results with the same score
 322   // that differ only by capitalization; because the history system returns
 323   // results sorted by recency, this means we'll pick the most recent such
 324   // result even if the precision of our relevance score is too low to
 325   // distinguish the two.
 326   if (!i.second && (match.relevance > i.first->second.relevance))
 327     i.first->second = match;
 328 }
 329
 330 AutocompleteMatch ZeroSuggestProvider::NavigationToMatch(
 331     const SearchProvider::NavigationResult& navigation) {
 332   AutocompleteMatch match(this, navigation.relevance(), false,
 333                           AutocompleteMatchType::NAVSUGGEST);
 334   match.destination_url = navigation.url();
 335
 336   // Zero suggest results should always omit protocols and never appear bold.
 337   const std::string languages(
 338       profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
 339   match.contents = net::FormatUrl(navigation.url(), languages,
 340       net::kFormatUrlOmitAll, net::UnescapeRule::SPACES, NULL, NULL, NULL);
 341   match.fill_into_edit +=
 342       AutocompleteInput::FormattedStringWithEquivalentMeaning(navigation.url(),
 343           match.contents);
 344
 345   AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
 346       match.contents.length(), ACMatchClassification::URL,
 347       &match.contents_class);
 348
 349   match.description =
 350       AutocompleteMatch::SanitizeString(navigation.description());
 351   AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
 352       match.description.length(), ACMatchClassification::NONE,
 353       &match.description_class);
 354   return match;
 355 }
 356
 357 void ZeroSuggestProvider::Run(const GURL& suggest_url) {
 358   have_pending_request_ = false;
 359   const int kFetcherID = 1;
 360   fetcher_.reset(
 361       net::URLFetcher::Create(kFetcherID,
 362           suggest_url,
 363           net::URLFetcher::GET, this));
 364   fetcher_->SetRequestContext(profile_->GetRequestContext());
 365   fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES);
 366   // Add Chrome experiment state to the request headers.
 367   net::HttpRequestHeaders headers;
 368   chrome_variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
 369       fetcher_->GetOriginalURL(), profile_->IsOffTheRecord(), false, &headers);
 370   fetcher_->SetExtraRequestHeaders(headers.ToString());
 371
 372   fetcher_->Start();
 373
 374   if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
 375     most_visited_urls_.clear();
 376     history::TopSites* ts = profile_->GetTopSites();
 377     if (ts) {
 378       ts->GetMostVisitedURLs(
 379           base::Bind(&ZeroSuggestProvider::OnMostVisitedUrlsAvailable,
 380                      weak_ptr_factory_.GetWeakPtr()), false);
 381     }
 382   }
 383   have_pending_request_ = true;
 384   LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_SENT);
 385 }
 386
 387 void ZeroSuggestProvider::ParseSuggestResults(const base::Value& root_val) {
 388   SearchProvider::SuggestResults suggest_results;
 389   FillResults(root_val, &verbatim_relevance_,
 390               &suggest_results, &navigation_results_);
 391
 392   query_matches_map_.clear();
 393   AddSuggestResultsToMap(suggest_results,
 394                          template_url_service_->GetDefaultSearchProvider(),
 395                          &query_matches_map_);
 396 }
 397
 398 void ZeroSuggestProvider::OnMostVisitedUrlsAvailable(
 399     const history::MostVisitedURLList& urls) {
 400   most_visited_urls_ = urls;
 401 }
 402
 403 void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches() {
 404   matches_.clear();
 405
 406   const TemplateURL* default_provider =
 407       template_url_service_->GetDefaultSearchProvider();
 408   // Fail if we can't set the clickthrough URL for query suggestions.
 409   if (default_provider == NULL || !default_provider->SupportsReplacement())
 410     return;
 411
 412   const int num_query_results = query_matches_map_.size();
 413   const int num_nav_results = navigation_results_.size();
 414   const int num_results = num_query_results + num_nav_results;
 415   UMA_HISTOGRAM_COUNTS("ZeroSuggest.QueryResults", num_query_results);
 416   UMA_HISTOGRAM_COUNTS("ZeroSuggest.URLResults",  num_nav_results);
 417   UMA_HISTOGRAM_COUNTS("ZeroSuggest.AllResults", num_results);
 418
 419   // Show Most Visited results after ZeroSuggest response is received.
 420   if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
 421     if (!current_url_match_.destination_url.is_valid())
 422       return;
 423     matches_.push_back(current_url_match_);
 424     int relevance = 600;
 425     if (num_results > 0) {
 426       UMA_HISTOGRAM_COUNTS(
 427           "Omnibox.ZeroSuggest.MostVisitedResultsCounterfactual",
 428           most_visited_urls_.size());
 429     }
 430     const base::string16 current_query_string16(
 431         base::ASCIIToUTF16(current_query_));
 432     const std::string languages(
 433         profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
 434     for (size_t i = 0; i < most_visited_urls_.size(); i++) {
 435       const history::MostVisitedURL& url = most_visited_urls_[i];
 436       SearchProvider::NavigationResult nav(
 437           *this, url.url, url.title, false, relevance, true,
 438           current_query_string16, languages);
 439       matches_.push_back(NavigationToMatch(nav));
 440       --relevance;
 441     }
 442     return;
 443   }
 444
 445   if (num_results == 0)
 446     return;
 447
 448   // TODO(jered): Rip this out once the first match is decoupled from the
 449   // current typing in the omnibox.
 450   matches_.push_back(current_url_match_);
 451
 452   for (SearchProvider::MatchMap::const_iterator it(query_matches_map_.begin());
 453        it != query_matches_map_.end(); ++it)
 454     matches_.push_back(it->second);
 455
 456   for (SearchProvider::NavigationResults::const_iterator it(
 457        navigation_results_.begin()); it != navigation_results_.end(); ++it)
 458     matches_.push_back(NavigationToMatch(*it));
 459 }
 460
 461 AutocompleteMatch ZeroSuggestProvider::MatchForCurrentURL() {
 462   AutocompleteInput input(permanent_text_, base::string16::npos, base::string16(),
 463                           GURL(current_query_), current_page_classification_,
 464                           false, false, true, AutocompleteInput::ALL_MATCHES);
 465
 466   AutocompleteMatch match;
 467   AutocompleteClassifierFactory::GetForProfile(profile_)->Classify(
 468       permanent_text_, false, true, &match, NULL);
 469   match.is_history_what_you_typed_match = false;
 470   match.allowed_to_be_default_match = true;
 471
 472   // The placeholder suggestion for the current URL has high relevance so
 473   // that it is in the first suggestion slot and inline autocompleted. It
 474   // gets dropped as soon as the user types something.
 475   match.relevance = verbatim_relevance_;
 476
 477   return match;
 478 }