components/omnibox/base_search_provider.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/omnibox/base_search_provider.h"
   6
   7 #include "base/i18n/case_conversion.h"
   8 #include "base/strings/string_util.h"
   9 #include "base/strings/utf_string_conversions.h"
  10 #include "components/metrics/proto/omnibox_event.pb.h"
  11 #include "components/metrics/proto/omnibox_input_type.pb.h"
  12 #include "components/omnibox/autocomplete_provider_client.h"
  13 #include "components/omnibox/autocomplete_provider_listener.h"
  14 #include "components/omnibox/omnibox_field_trial.h"
  15 #include "components/search_engines/template_url.h"
  16 #include "components/search_engines/template_url_prepopulate_data.h"
  17 #include "components/search_engines/template_url_service.h"
  18 #include "net/base/registry_controlled_domains/registry_controlled_domain.h"
  19 #include "net/url_request/url_fetcher.h"
  20 #include "net/url_request/url_fetcher_delegate.h"
  21 #include "url/gurl.h"
  22
  23 using metrics::OmniboxEventProto;
  24
  25 // SuggestionDeletionHandler -------------------------------------------------
  26
  27 // This class handles making requests to the server in order to delete
  28 // personalized suggestions.
  29 class SuggestionDeletionHandler : public net::URLFetcherDelegate {
  30  public:
  31   typedef base::Callback<void(bool, SuggestionDeletionHandler*)>
  32       DeletionCompletedCallback;
  33
  34   SuggestionDeletionHandler(
  35       const std::string& deletion_url,
  36       net::URLRequestContextGetter* request_context,
  37       const DeletionCompletedCallback& callback);
  38
  39   virtual ~SuggestionDeletionHandler();
  40
  41  private:
  42   // net::URLFetcherDelegate:
  43   virtual void OnURLFetchComplete(const net::URLFetcher* source) OVERRIDE;
  44
  45   scoped_ptr<net::URLFetcher> deletion_fetcher_;
  46   DeletionCompletedCallback callback_;
  47
  48   DISALLOW_COPY_AND_ASSIGN(SuggestionDeletionHandler);
  49 };
  50
  51 SuggestionDeletionHandler::SuggestionDeletionHandler(
  52     const std::string& deletion_url,
  53     net::URLRequestContextGetter* request_context,
  54     const DeletionCompletedCallback& callback) : callback_(callback) {
  55   GURL url(deletion_url);
  56   DCHECK(url.is_valid());
  57
  58   deletion_fetcher_.reset(net::URLFetcher::Create(
  59       BaseSearchProvider::kDeletionURLFetcherID,
  60       url,
  61       net::URLFetcher::GET,
  62       this));
  63   deletion_fetcher_->SetRequestContext(request_context);
  64   deletion_fetcher_->Start();
  65 }
  66
  67 SuggestionDeletionHandler::~SuggestionDeletionHandler() {
  68 }
  69
  70 void SuggestionDeletionHandler::OnURLFetchComplete(
  71     const net::URLFetcher* source) {
  72   DCHECK(source == deletion_fetcher_.get());
  73   callback_.Run(
  74       source->GetStatus().is_success() && (source->GetResponseCode() == 200),
  75       this);
  76 }
  77
  78 // BaseSearchProvider ---------------------------------------------------------
  79
  80 // static
  81 const int BaseSearchProvider::kDefaultProviderURLFetcherID = 1;
  82 const int BaseSearchProvider::kKeywordProviderURLFetcherID = 2;
  83 const int BaseSearchProvider::kDeletionURLFetcherID = 3;
  84
  85 BaseSearchProvider::BaseSearchProvider(
  86     TemplateURLService* template_url_service,
  87     scoped_ptr<AutocompleteProviderClient> client,
  88     AutocompleteProvider::Type type)
  89     : AutocompleteProvider(type),
  90       template_url_service_(template_url_service),
  91       client_(client.Pass()),
  92       field_trial_triggered_(false),
  93       field_trial_triggered_in_session_(false) {
  94 }
  95
  96 // static
  97 bool BaseSearchProvider::ShouldPrefetch(const AutocompleteMatch& match) {
  98   return match.GetAdditionalInfo(kShouldPrefetchKey) == kTrue;
  99 }
 100
 101 // static
 102 AutocompleteMatch BaseSearchProvider::CreateSearchSuggestion(
 103     const base::string16& suggestion,
 104     AutocompleteMatchType::Type type,
 105     bool from_keyword_provider,
 106     const TemplateURL* template_url,
 107     const SearchTermsData& search_terms_data) {
 108   // This call uses a number of default values.  For instance, it assumes that
 109   // if this match is from a keyword provider than the user is in keyword mode.
 110   return CreateSearchSuggestion(
 111       NULL, AutocompleteInput(), from_keyword_provider,
 112       SearchSuggestionParser::SuggestResult(
 113           suggestion, type, suggestion, base::string16(), base::string16(),
 114           base::string16(), base::string16(), std::string(), std::string(),
 115           from_keyword_provider, 0, false, false, base::string16()),
 116       template_url, search_terms_data, 0, false);
 117 }
 118
 119 void BaseSearchProvider::DeleteMatch(const AutocompleteMatch& match) {
 120   DCHECK(match.deletable);
 121   if (!match.GetAdditionalInfo(BaseSearchProvider::kDeletionUrlKey).empty()) {
 122     deletion_handlers_.push_back(new SuggestionDeletionHandler(
 123         match.GetAdditionalInfo(BaseSearchProvider::kDeletionUrlKey),
 124         client_->RequestContext(),
 125         base::Bind(&BaseSearchProvider::OnDeletionComplete,
 126                    base::Unretained(this))));
 127   }
 128
 129   TemplateURL* template_url =
 130       match.GetTemplateURL(template_url_service_, false);
 131   // This may be NULL if the template corresponding to the keyword has been
 132   // deleted or there is no keyword set.
 133   if (template_url != NULL) {
 134     client_->DeleteMatchingURLsForKeywordFromHistory(template_url->id(),
 135                                                      match.contents);
 136   }
 137
 138   // Immediately update the list of matches to show the match was deleted,
 139   // regardless of whether the server request actually succeeds.
 140   DeleteMatchFromMatches(match);
 141 }
 142
 143 void BaseSearchProvider::AddProviderInfo(ProvidersInfo* provider_info) const {
 144   provider_info->push_back(metrics::OmniboxEventProto_ProviderInfo());
 145   metrics::OmniboxEventProto_ProviderInfo& new_entry = provider_info->back();
 146   new_entry.set_provider(AsOmniboxEventProviderType());
 147   new_entry.set_provider_done(done_);
 148   std::vector<uint32> field_trial_hashes;
 149   OmniboxFieldTrial::GetActiveSuggestFieldTrialHashes(&field_trial_hashes);
 150   for (size_t i = 0; i < field_trial_hashes.size(); ++i) {
 151     if (field_trial_triggered_)
 152       new_entry.mutable_field_trial_triggered()->Add(field_trial_hashes[i]);
 153     if (field_trial_triggered_in_session_) {
 154       new_entry.mutable_field_trial_triggered_in_session()->Add(
 155           field_trial_hashes[i]);
 156     }
 157   }
 158 }
 159
 160 // static
 161 const char BaseSearchProvider::kRelevanceFromServerKey[] =
 162     "relevance_from_server";
 163 const char BaseSearchProvider::kShouldPrefetchKey[] = "should_prefetch";
 164 const char BaseSearchProvider::kSuggestMetadataKey[] = "suggest_metadata";
 165 const char BaseSearchProvider::kDeletionUrlKey[] = "deletion_url";
 166 const char BaseSearchProvider::kTrue[] = "true";
 167 const char BaseSearchProvider::kFalse[] = "false";
 168
 169 BaseSearchProvider::~BaseSearchProvider() {}
 170
 171 void BaseSearchProvider::SetDeletionURL(const std::string& deletion_url,
 172                                         AutocompleteMatch* match) {
 173   if (deletion_url.empty())
 174     return;
 175   if (!template_url_service_)
 176     return;
 177   GURL url =
 178       template_url_service_->GetDefaultSearchProvider()->GenerateSearchURL(
 179           template_url_service_->search_terms_data());
 180   url = url.GetOrigin().Resolve(deletion_url);
 181   if (url.is_valid()) {
 182     match->RecordAdditionalInfo(BaseSearchProvider::kDeletionUrlKey,
 183         url.spec());
 184     match->deletable = true;
 185   }
 186 }
 187
 188 // static
 189 AutocompleteMatch BaseSearchProvider::CreateSearchSuggestion(
 190     AutocompleteProvider* autocomplete_provider,
 191     const AutocompleteInput& input,
 192     const bool in_keyword_mode,
 193     const SearchSuggestionParser::SuggestResult& suggestion,
 194     const TemplateURL* template_url,
 195     const SearchTermsData& search_terms_data,
 196     int accepted_suggestion,
 197     bool append_extra_query_params) {
 198   AutocompleteMatch match(autocomplete_provider, suggestion.relevance(), false,
 199                           suggestion.type());
 200
 201   if (!template_url)
 202     return match;
 203   match.keyword = template_url->keyword();
 204   match.contents = suggestion.match_contents();
 205   match.contents_class = suggestion.match_contents_class();
 206   match.answer_contents = suggestion.answer_contents();
 207   match.answer_type = suggestion.answer_type();
 208   if (suggestion.type() == AutocompleteMatchType::SEARCH_SUGGEST_INFINITE) {
 209     match.RecordAdditionalInfo(
 210         kACMatchPropertyInputText, base::UTF16ToUTF8(input.text()));
 211     match.RecordAdditionalInfo(
 212         kACMatchPropertyContentsPrefix,
 213         base::UTF16ToUTF8(suggestion.match_contents_prefix()));
 214     match.RecordAdditionalInfo(
 215         kACMatchPropertyContentsStartIndex,
 216         static_cast<int>(
 217             suggestion.suggestion().length() - match.contents.length()));
 218   }
 219
 220   if (!suggestion.annotation().empty())
 221     match.description = suggestion.annotation();
 222
 223   // suggestion.match_contents() should have already been collapsed.
 224   match.allowed_to_be_default_match =
 225       (!in_keyword_mode || suggestion.from_keyword_provider()) &&
 226       (base::CollapseWhitespace(input.text(), false) ==
 227        suggestion.match_contents());
 228
 229   // When the user forced a query, we need to make sure all the fill_into_edit
 230   // values preserve that property.  Otherwise, if the user starts editing a
 231   // suggestion, non-Search results will suddenly appear.
 232   if (input.type() == metrics::OmniboxInputType::FORCED_QUERY)
 233     match.fill_into_edit.assign(base::ASCIIToUTF16("?"));
 234   if (suggestion.from_keyword_provider())
 235     match.fill_into_edit.append(match.keyword + base::char16(' '));
 236   if (!input.prevent_inline_autocomplete() &&
 237       (!in_keyword_mode || suggestion.from_keyword_provider()) &&
 238       StartsWith(suggestion.suggestion(), input.text(), false)) {
 239     match.inline_autocompletion =
 240         suggestion.suggestion().substr(input.text().length());
 241     match.allowed_to_be_default_match = true;
 242   }
 243   match.fill_into_edit.append(suggestion.suggestion());
 244
 245   const TemplateURLRef& search_url = template_url->url_ref();
 246   DCHECK(search_url.SupportsReplacement(search_terms_data));
 247   match.search_terms_args.reset(
 248       new TemplateURLRef::SearchTermsArgs(suggestion.suggestion()));
 249   match.search_terms_args->original_query = input.text();
 250   match.search_terms_args->accepted_suggestion = accepted_suggestion;
 251   match.search_terms_args->enable_omnibox_start_margin = true;
 252   match.search_terms_args->suggest_query_params =
 253       suggestion.suggest_query_params();
 254   match.search_terms_args->append_extra_query_params =
 255       append_extra_query_params;
 256   // This is the destination URL sans assisted query stats.  This must be set
 257   // so the AutocompleteController can properly de-dupe; the controller will
 258   // eventually overwrite it before it reaches the user.
 259   match.destination_url =
 260       GURL(search_url.ReplaceSearchTerms(*match.search_terms_args.get(),
 261                                          search_terms_data));
 262
 263   // Search results don't look like URLs.
 264   match.transition = suggestion.from_keyword_provider() ?
 265       content::PAGE_TRANSITION_KEYWORD : content::PAGE_TRANSITION_GENERATED;
 266
 267   return match;
 268 }
 269
 270 // static
 271 bool BaseSearchProvider::ZeroSuggestEnabled(
 272     const GURL& suggest_url,
 273     const TemplateURL* template_url,
 274     OmniboxEventProto::PageClassification page_classification,
 275     const SearchTermsData& search_terms_data,
 276     AutocompleteProviderClient* client) {
 277   if (!OmniboxFieldTrial::InZeroSuggestFieldTrial())
 278     return false;
 279
 280   // Make sure we are sending the suggest request through HTTPS to prevent
 281   // exposing the current page URL or personalized results without encryption.
 282   if (!suggest_url.SchemeIs(url::kHttpsScheme))
 283     return false;
 284
 285   // Don't show zero suggest on the NTP.
 286   // TODO(hfung): Experiment with showing MostVisited zero suggest on NTP
 287   // under the conditions described in crbug.com/305366.
 288   if ((page_classification ==
 289        OmniboxEventProto::INSTANT_NTP_WITH_FAKEBOX_AS_STARTING_FOCUS) ||
 290       (page_classification ==
 291        OmniboxEventProto::INSTANT_NTP_WITH_OMNIBOX_AS_STARTING_FOCUS))
 292     return false;
 293
 294   // Don't run if in incognito mode.
 295   if (client->IsOffTheRecord())
 296     return false;
 297
 298   // Don't run if we can't get preferences or search suggest is not enabled.
 299   if (!client->SearchSuggestEnabled())
 300     return false;
 301
 302   // Only make the request if we know that the provider supports zero suggest
 303   // (currently only the prepopulated Google provider).
 304   if (template_url == NULL ||
 305       !template_url->SupportsReplacement(search_terms_data) ||
 306       TemplateURLPrepopulateData::GetEngineType(
 307           *template_url, search_terms_data) != SEARCH_ENGINE_GOOGLE)
 308     return false;
 309
 310   return true;
 311 }
 312
 313 // static
 314 bool BaseSearchProvider::CanSendURL(
 315     const GURL& current_page_url,
 316     const GURL& suggest_url,
 317     const TemplateURL* template_url,
 318     OmniboxEventProto::PageClassification page_classification,
 319     const SearchTermsData& search_terms_data,
 320     AutocompleteProviderClient* client) {
 321   if (!ZeroSuggestEnabled(suggest_url, template_url, page_classification,
 322                           search_terms_data, client))
 323     return false;
 324
 325   if (!current_page_url.is_valid())
 326     return false;
 327
 328   // Only allow HTTP URLs or HTTPS URLs for the same domain as the search
 329   // provider.
 330   if ((current_page_url.scheme() != url::kHttpScheme) &&
 331       ((current_page_url.scheme() != url::kHttpsScheme) ||
 332        !net::registry_controlled_domains::SameDomainOrHost(
 333            current_page_url, suggest_url,
 334            net::registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES)))
 335     return false;
 336
 337   if (!client->TabSyncEnabledAndUnencrypted())
 338     return false;
 339
 340   return true;
 341 }
 342
 343 void BaseSearchProvider::AddMatchToMap(
 344     const SearchSuggestionParser::SuggestResult& result,
 345     const std::string& metadata,
 346     int accepted_suggestion,
 347     bool mark_as_deletable,
 348     bool in_keyword_mode,
 349     MatchMap* map) {
 350   AutocompleteMatch match = CreateSearchSuggestion(
 351       this, GetInput(result.from_keyword_provider()), in_keyword_mode, result,
 352       GetTemplateURL(result.from_keyword_provider()),
 353       template_url_service_->search_terms_data(), accepted_suggestion,
 354       ShouldAppendExtraParams(result));
 355   if (!match.destination_url.is_valid())
 356     return;
 357   match.search_terms_args->bookmark_bar_pinned = client_->ShowBookmarkBar();
 358   match.RecordAdditionalInfo(kRelevanceFromServerKey,
 359                              result.relevance_from_server() ? kTrue : kFalse);
 360   match.RecordAdditionalInfo(kShouldPrefetchKey,
 361                              result.should_prefetch() ? kTrue : kFalse);
 362   SetDeletionURL(result.deletion_url(), &match);
 363   if (mark_as_deletable)
 364     match.deletable = true;
 365   // Metadata is needed only for prefetching queries.
 366   if (result.should_prefetch())
 367     match.RecordAdditionalInfo(kSuggestMetadataKey, metadata);
 368
 369   // Try to add |match| to |map|.  If a match for this suggestion is
 370   // already in |map|, replace it if |match| is more relevant.
 371   // NOTE: Keep this ToLower() call in sync with url_database.cc.
 372   MatchKey match_key(
 373       std::make_pair(base::i18n::ToLower(result.suggestion()),
 374                      match.search_terms_args->suggest_query_params));
 375   const std::pair<MatchMap::iterator, bool> i(
 376        map->insert(std::make_pair(match_key, match)));
 377
 378   bool should_prefetch = result.should_prefetch();
 379   if (!i.second) {
 380     // NOTE: We purposefully do a direct relevance comparison here instead of
 381     // using AutocompleteMatch::MoreRelevant(), so that we'll prefer "items
 382     // added first" rather than "items alphabetically first" when the scores
 383     // are equal. The only case this matters is when a user has results with
 384     // the same score that differ only by capitalization; because the history
 385     // system returns results sorted by recency, this means we'll pick the most
 386     // recent such result even if the precision of our relevance score is too
 387     // low to distinguish the two.
 388     if (match.relevance > i.first->second.relevance) {
 389       match.duplicate_matches.insert(match.duplicate_matches.end(),
 390                                      i.first->second.duplicate_matches.begin(),
 391                                      i.first->second.duplicate_matches.end());
 392       i.first->second.duplicate_matches.clear();
 393       match.duplicate_matches.push_back(i.first->second);
 394       i.first->second = match;
 395     } else {
 396       i.first->second.duplicate_matches.push_back(match);
 397       if (match.keyword == i.first->second.keyword) {
 398         // Old and new matches are from the same search provider. It is okay to
 399         // record one match's prefetch data onto a different match (for the same
 400         // query string) for the following reasons:
 401         // 1. Because the suggest server only sends down a query string from
 402         // which we construct a URL, rather than sending a full URL, and because
 403         // we construct URLs from query strings in the same way every time, the
 404         // URLs for the two matches will be the same. Therefore, we won't end up
 405         // prefetching something the server didn't intend.
 406         // 2. Presumably the server sets the prefetch bit on a match it things
 407         // is sufficiently relevant that the user is likely to choose it.
 408         // Surely setting the prefetch bit on a match of even higher relevance
 409         // won't violate this assumption.
 410         should_prefetch |= ShouldPrefetch(i.first->second);
 411         i.first->second.RecordAdditionalInfo(kShouldPrefetchKey,
 412                                              should_prefetch ? kTrue : kFalse);
 413         if (should_prefetch)
 414           i.first->second.RecordAdditionalInfo(kSuggestMetadataKey, metadata);
 415       }
 416     }
 417   }
 418 }
 419
 420 bool BaseSearchProvider::ParseSuggestResults(
 421     const base::Value& root_val,
 422     int default_result_relevance,
 423     bool is_keyword_result,
 424     SearchSuggestionParser::Results* results) {
 425   if (!SearchSuggestionParser::ParseSuggestResults(
 426       root_val, GetInput(is_keyword_result),
 427       client_->SchemeClassifier(), default_result_relevance,
 428       client_->AcceptLanguages(), is_keyword_result, results))
 429     return false;
 430
 431   for (std::vector<GURL>::const_iterator it =
 432            results->answers_image_urls.begin();
 433        it != results->answers_image_urls.end(); ++it)
 434     client_->PrefetchImage(*it);
 435
 436   field_trial_triggered_ |= results->field_trial_triggered;
 437   field_trial_triggered_in_session_ |= results->field_trial_triggered;
 438   return true;
 439 }
 440
 441 void BaseSearchProvider::DeleteMatchFromMatches(
 442     const AutocompleteMatch& match) {
 443   for (ACMatches::iterator i(matches_.begin()); i != matches_.end(); ++i) {
 444     // Find the desired match to delete by checking the type and contents.
 445     // We can't check the destination URL, because the autocomplete controller
 446     // may have reformulated that. Not that while checking for matching
 447     // contents works for personalized suggestions, if more match types gain
 448     // deletion support, this algorithm may need to be re-examined.
 449     if (i->contents == match.contents && i->type == match.type) {
 450       matches_.erase(i);
 451       break;
 452     }
 453   }
 454 }
 455
 456 void BaseSearchProvider::OnDeletionComplete(
 457     bool success, SuggestionDeletionHandler* handler) {
 458   RecordDeletionResult(success);
 459   SuggestionDeletionHandlers::iterator it = std::find(
 460       deletion_handlers_.begin(), deletion_handlers_.end(), handler);
 461   DCHECK(it != deletion_handlers_.end());
 462   deletion_handlers_.erase(it);
 463 }