chrome/browser/autocomplete/base_search_provider.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/autocomplete/base_search_provider.h"
   6
   7 #include "base/i18n/case_conversion.h"
   8 #include "base/i18n/icu_string_conversions.h"
   9 #include "base/json/json_string_value_serializer.h"
  10 #include "base/json/json_writer.h"
  11 #include "base/prefs/pref_registry_simple.h"
  12 #include "base/prefs/pref_service.h"
  13 #include "base/strings/string_util.h"
  14 #include "base/strings/utf_string_conversions.h"
  15 #include "chrome/browser/autocomplete/autocomplete_provider_listener.h"
  16 #include "chrome/browser/autocomplete/url_prefix.h"
  17 #include "chrome/browser/history/history_service.h"
  18 #include "chrome/browser/history/history_service_factory.h"
  19 #include "chrome/browser/omnibox/omnibox_field_trial.h"
  20 #include "chrome/browser/profiles/profile.h"
  21 #include "chrome/browser/search/instant_service.h"
  22 #include "chrome/browser/search/instant_service_factory.h"
  23 #include "chrome/browser/search/search.h"
  24 #include "chrome/browser/search_engines/template_url.h"
  25 #include "chrome/browser/search_engines/template_url_prepopulate_data.h"
  26 #include "chrome/browser/search_engines/template_url_service.h"
  27 #include "chrome/browser/search_engines/template_url_service_factory.h"
  28 #include "chrome/browser/sync/profile_sync_service.h"
  29 #include "chrome/browser/sync/profile_sync_service_factory.h"
  30 #include "chrome/common/net/url_fixer_upper.h"
  31 #include "chrome/common/pref_names.h"
  32 #include "components/sync_driver/sync_prefs.h"
  33 #include "content/public/common/url_constants.h"
  34 #include "net/base/escape.h"
  35 #include "net/base/net_util.h"
  36 #include "net/base/registry_controlled_domains/registry_controlled_domain.h"
  37 #include "net/http/http_response_headers.h"
  38 #include "net/url_request/url_fetcher.h"
  39 #include "net/url_request/url_fetcher_delegate.h"
  40 #include "url/gurl.h"
  41
  42 namespace {
  43
  44 AutocompleteMatchType::Type GetAutocompleteMatchType(const std::string& type) {
  45   if (type == "ENTITY")
  46     return AutocompleteMatchType::SEARCH_SUGGEST_ENTITY;
  47   if (type == "INFINITE")
  48     return AutocompleteMatchType::SEARCH_SUGGEST_INFINITE;
  49   if (type == "PERSONALIZED_QUERY")
  50     return AutocompleteMatchType::SEARCH_SUGGEST_PERSONALIZED;
  51   if (type == "PROFILE")
  52     return AutocompleteMatchType::SEARCH_SUGGEST_PROFILE;
  53   if (type == "NAVIGATION")
  54     return AutocompleteMatchType::NAVSUGGEST;
  55   if (type == "PERSONALIZED_NAVIGATION")
  56     return AutocompleteMatchType::NAVSUGGEST_PERSONALIZED;
  57   return AutocompleteMatchType::SEARCH_SUGGEST;
  58 }
  59
  60 } // namespace
  61
  62 // SuggestionDeletionHandler -------------------------------------------------
  63
  64 // This class handles making requests to the server in order to delete
  65 // personalized suggestions.
  66 class SuggestionDeletionHandler : public net::URLFetcherDelegate {
  67  public:
  68   typedef base::Callback<void(bool, SuggestionDeletionHandler*)>
  69       DeletionCompletedCallback;
  70
  71   SuggestionDeletionHandler(
  72       const std::string& deletion_url,
  73       Profile* profile,
  74       const DeletionCompletedCallback& callback);
  75
  76   virtual ~SuggestionDeletionHandler();
  77
  78  private:
  79   // net::URLFetcherDelegate:
  80   virtual void OnURLFetchComplete(const net::URLFetcher* source) OVERRIDE;
  81
  82   scoped_ptr<net::URLFetcher> deletion_fetcher_;
  83   DeletionCompletedCallback callback_;
  84
  85   DISALLOW_COPY_AND_ASSIGN(SuggestionDeletionHandler);
  86 };
  87
  88 SuggestionDeletionHandler::SuggestionDeletionHandler(
  89     const std::string& deletion_url,
  90     Profile* profile,
  91     const DeletionCompletedCallback& callback) : callback_(callback) {
  92   GURL url(deletion_url);
  93   DCHECK(url.is_valid());
  94
  95   deletion_fetcher_.reset(net::URLFetcher::Create(
  96       BaseSearchProvider::kDeletionURLFetcherID,
  97       url,
  98       net::URLFetcher::GET,
  99       this));
 100   deletion_fetcher_->SetRequestContext(profile->GetRequestContext());
 101   deletion_fetcher_->Start();
 102 }
 103
 104 SuggestionDeletionHandler::~SuggestionDeletionHandler() {
 105 }
 106
 107 void SuggestionDeletionHandler::OnURLFetchComplete(
 108     const net::URLFetcher* source) {
 109   DCHECK(source == deletion_fetcher_.get());
 110   callback_.Run(
 111       source->GetStatus().is_success() && (source->GetResponseCode() == 200),
 112       this);
 113 }
 114
 115 // BaseSearchProvider ---------------------------------------------------------
 116
 117 // static
 118 const int BaseSearchProvider::kDefaultProviderURLFetcherID = 1;
 119 const int BaseSearchProvider::kKeywordProviderURLFetcherID = 2;
 120 const int BaseSearchProvider::kDeletionURLFetcherID = 3;
 121
 122 BaseSearchProvider::BaseSearchProvider(AutocompleteProviderListener* listener,
 123                                        Profile* profile,
 124                                        AutocompleteProvider::Type type)
 125     : AutocompleteProvider(listener, profile, type),
 126       field_trial_triggered_(false),
 127       field_trial_triggered_in_session_(false),
 128       suggest_results_pending_(0),
 129       in_app_list_(false) {
 130 }
 131
 132 // static
 133 bool BaseSearchProvider::ShouldPrefetch(const AutocompleteMatch& match) {
 134   return match.GetAdditionalInfo(kShouldPrefetchKey) == kTrue;
 135 }
 136
 137 // static
 138 AutocompleteMatch BaseSearchProvider::CreateSearchSuggestion(
 139     const base::string16& suggestion,
 140     AutocompleteMatchType::Type type,
 141     bool from_keyword_provider,
 142     const TemplateURL* template_url) {
 143   return CreateSearchSuggestion(
 144       NULL, AutocompleteInput(), BaseSearchProvider::SuggestResult(
 145           suggestion, type, suggestion, base::string16(), base::string16(),
 146           base::string16(), base::string16(), std::string(), std::string(),
 147           from_keyword_provider, 0, false, false, base::string16()),
 148       template_url, 0, 0, false, false);
 149 }
 150
 151 void BaseSearchProvider::Stop(bool clear_cached_results) {
 152   StopSuggest();
 153   done_ = true;
 154
 155   if (clear_cached_results)
 156     ClearAllResults();
 157 }
 158
 159 void BaseSearchProvider::DeleteMatch(const AutocompleteMatch& match) {
 160   DCHECK(match.deletable);
 161   if (!match.GetAdditionalInfo(BaseSearchProvider::kDeletionUrlKey).empty()) {
 162     deletion_handlers_.push_back(new SuggestionDeletionHandler(
 163         match.GetAdditionalInfo(BaseSearchProvider::kDeletionUrlKey),
 164         profile_,
 165         base::Bind(&BaseSearchProvider::OnDeletionComplete,
 166                    base::Unretained(this))));
 167   }
 168
 169   HistoryService* const history_service =
 170       HistoryServiceFactory::GetForProfile(profile_, Profile::EXPLICIT_ACCESS);
 171   TemplateURL* template_url = match.GetTemplateURL(profile_, false);
 172   // This may be NULL if the template corresponding to the keyword has been
 173   // deleted or there is no keyword set.
 174   if (template_url != NULL) {
 175     history_service->DeleteMatchingURLsForKeyword(template_url->id(),
 176                                                   match.contents);
 177   }
 178
 179   // Immediately update the list of matches to show the match was deleted,
 180   // regardless of whether the server request actually succeeds.
 181   DeleteMatchFromMatches(match);
 182 }
 183
 184 void BaseSearchProvider::AddProviderInfo(ProvidersInfo* provider_info) const {
 185   provider_info->push_back(metrics::OmniboxEventProto_ProviderInfo());
 186   metrics::OmniboxEventProto_ProviderInfo& new_entry = provider_info->back();
 187   new_entry.set_provider(AsOmniboxEventProviderType());
 188   new_entry.set_provider_done(done_);
 189   std::vector<uint32> field_trial_hashes;
 190   OmniboxFieldTrial::GetActiveSuggestFieldTrialHashes(&field_trial_hashes);
 191   for (size_t i = 0; i < field_trial_hashes.size(); ++i) {
 192     if (field_trial_triggered_)
 193       new_entry.mutable_field_trial_triggered()->Add(field_trial_hashes[i]);
 194     if (field_trial_triggered_in_session_) {
 195       new_entry.mutable_field_trial_triggered_in_session()->Add(
 196           field_trial_hashes[i]);
 197     }
 198   }
 199   ModifyProviderInfo(&new_entry);
 200 }
 201
 202 // static
 203 const char BaseSearchProvider::kRelevanceFromServerKey[] =
 204     "relevance_from_server";
 205 const char BaseSearchProvider::kShouldPrefetchKey[] = "should_prefetch";
 206 const char BaseSearchProvider::kSuggestMetadataKey[] = "suggest_metadata";
 207 const char BaseSearchProvider::kDeletionUrlKey[] = "deletion_url";
 208 const char BaseSearchProvider::kTrue[] = "true";
 209 const char BaseSearchProvider::kFalse[] = "false";
 210
 211 BaseSearchProvider::~BaseSearchProvider() {}
 212
 213 // BaseSearchProvider::Result --------------------------------------------------
 214
 215 BaseSearchProvider::Result::Result(bool from_keyword_provider,
 216                                    int relevance,
 217                                    bool relevance_from_server,
 218                                    AutocompleteMatchType::Type type,
 219                                    const std::string& deletion_url)
 220     : from_keyword_provider_(from_keyword_provider),
 221        type_(type),
 222        relevance_(relevance),
 223        relevance_from_server_(relevance_from_server),
 224        deletion_url_(deletion_url) {}
 225
 226 BaseSearchProvider::Result::~Result() {}
 227
 228 // BaseSearchProvider::SuggestResult -------------------------------------------
 229
 230 BaseSearchProvider::SuggestResult::SuggestResult(
 231     const base::string16& suggestion,
 232     AutocompleteMatchType::Type type,
 233     const base::string16& match_contents,
 234     const base::string16& match_contents_prefix,
 235     const base::string16& annotation,
 236     const base::string16& answer_contents,
 237     const base::string16& answer_type,
 238     const std::string& suggest_query_params,
 239     const std::string& deletion_url,
 240     bool from_keyword_provider,
 241     int relevance,
 242     bool relevance_from_server,
 243     bool should_prefetch,
 244     const base::string16& input_text)
 245     : Result(from_keyword_provider,
 246              relevance,
 247              relevance_from_server,
 248              type,
 249              deletion_url),
 250       suggestion_(suggestion),
 251       match_contents_prefix_(match_contents_prefix),
 252       annotation_(annotation),
 253       suggest_query_params_(suggest_query_params),
 254       answer_contents_(answer_contents),
 255       answer_type_(answer_type),
 256       should_prefetch_(should_prefetch) {
 257   match_contents_ = match_contents;
 258   DCHECK(!match_contents_.empty());
 259   ClassifyMatchContents(true, input_text);
 260 }
 261
 262 BaseSearchProvider::SuggestResult::~SuggestResult() {}
 263
 264 void BaseSearchProvider::SuggestResult::ClassifyMatchContents(
 265     const bool allow_bolding_all,
 266     const base::string16& input_text) {
 267   if (input_text.empty()) {
 268     // In case of zero-suggest results, do not highlight matches.
 269     match_contents_class_.push_back(
 270         ACMatchClassification(0, ACMatchClassification::NONE));
 271     return;
 272   }
 273
 274   base::string16 lookup_text = input_text;
 275   if (type_ == AutocompleteMatchType::SEARCH_SUGGEST_INFINITE) {
 276     const size_t contents_index =
 277         suggestion_.length() - match_contents_.length();
 278     // Ensure the query starts with the input text, and ends with the match
 279     // contents, and the input text has an overlap with contents.
 280     if (StartsWith(suggestion_, input_text, true) &&
 281         EndsWith(suggestion_, match_contents_, true) &&
 282         (input_text.length() > contents_index)) {
 283       lookup_text = input_text.substr(contents_index);
 284     }
 285   }
 286   size_t lookup_position = match_contents_.find(lookup_text);
 287   if (!allow_bolding_all && (lookup_position == base::string16::npos)) {
 288     // Bail if the code below to update the bolding would bold the whole
 289     // string.  Note that the string may already be entirely bolded; if
 290     // so, leave it as is.
 291     return;
 292   }
 293   match_contents_class_.clear();
 294   // We do intra-string highlighting for suggestions - the suggested segment
 295   // will be highlighted, e.g. for input_text = "you" the suggestion may be
 296   // "youtube", so we'll bold the "tube" section: you*tube*.
 297   if (input_text != match_contents_) {
 298     if (lookup_position == base::string16::npos) {
 299       // The input text is not a substring of the query string, e.g. input
 300       // text is "slasdot" and the query string is "slashdot", so we bold the
 301       // whole thing.
 302       match_contents_class_.push_back(
 303           ACMatchClassification(0, ACMatchClassification::MATCH));
 304     } else {
 305       // We don't iterate over the string here annotating all matches because
 306       // it looks odd to have every occurrence of a substring that may be as
 307       // short as a single character highlighted in a query suggestion result,
 308       // e.g. for input text "s" and query string "southwest airlines", it
 309       // looks odd if both the first and last s are highlighted.
 310       if (lookup_position != 0) {
 311         match_contents_class_.push_back(
 312             ACMatchClassification(0, ACMatchClassification::MATCH));
 313       }
 314       match_contents_class_.push_back(
 315           ACMatchClassification(lookup_position, ACMatchClassification::NONE));
 316       size_t next_fragment_position = lookup_position + lookup_text.length();
 317       if (next_fragment_position < match_contents_.length()) {
 318         match_contents_class_.push_back(ACMatchClassification(
 319             next_fragment_position, ACMatchClassification::MATCH));
 320       }
 321     }
 322   } else {
 323     // Otherwise, match_contents_ is a verbatim (what-you-typed) match, either
 324     // for the default provider or a keyword search provider.
 325     match_contents_class_.push_back(
 326         ACMatchClassification(0, ACMatchClassification::NONE));
 327   }
 328 }
 329
 330 bool BaseSearchProvider::SuggestResult::IsInlineable(
 331     const base::string16& input) const {
 332   return StartsWith(suggestion_, input, false);
 333 }
 334
 335 int BaseSearchProvider::SuggestResult::CalculateRelevance(
 336     const AutocompleteInput& input,
 337     bool keyword_provider_requested) const {
 338   if (!from_keyword_provider_ && keyword_provider_requested)
 339     return 100;
 340   return ((input.type() == AutocompleteInput::URL) ? 300 : 600);
 341 }
 342
 343 // BaseSearchProvider::NavigationResult ----------------------------------------
 344
 345 BaseSearchProvider::NavigationResult::NavigationResult(
 346     const AutocompleteProvider& provider,
 347     const GURL& url,
 348     AutocompleteMatchType::Type type,
 349     const base::string16& description,
 350     const std::string& deletion_url,
 351     bool from_keyword_provider,
 352     int relevance,
 353     bool relevance_from_server,
 354     const base::string16& input_text,
 355     const std::string& languages)
 356     : Result(from_keyword_provider,
 357              relevance,
 358              relevance_from_server,
 359              type,
 360              deletion_url),
 361       url_(url),
 362       formatted_url_(AutocompleteInput::FormattedStringWithEquivalentMeaning(
 363           url,
 364           provider.StringForURLDisplay(url, true, false))),
 365       description_(description) {
 366   DCHECK(url_.is_valid());
 367   CalculateAndClassifyMatchContents(true, input_text, languages);
 368 }
 369
 370 BaseSearchProvider::NavigationResult::~NavigationResult() {}
 371
 372 void BaseSearchProvider::NavigationResult::CalculateAndClassifyMatchContents(
 373     const bool allow_bolding_nothing,
 374     const base::string16& input_text,
 375     const std::string& languages) {
 376   if (input_text.empty()) {
 377     // In case of zero-suggest results, do not highlight matches.
 378     match_contents_class_.push_back(
 379         ACMatchClassification(0, ACMatchClassification::NONE));
 380     return;
 381   }
 382
 383   // First look for the user's input inside the formatted url as it would be
 384   // without trimming the scheme, so we can find matches at the beginning of the
 385   // scheme.
 386   const URLPrefix* prefix =
 387       URLPrefix::BestURLPrefix(formatted_url_, input_text);
 388   size_t match_start = (prefix == NULL) ?
 389       formatted_url_.find(input_text) : prefix->prefix.length();
 390   bool trim_http = !AutocompleteInput::HasHTTPScheme(input_text) &&
 391                    (!prefix || (match_start != 0));
 392   const net::FormatUrlTypes format_types =
 393       net::kFormatUrlOmitAll & ~(trim_http ? 0 : net::kFormatUrlOmitHTTP);
 394
 395   base::string16 match_contents = net::FormatUrl(url_, languages, format_types,
 396       net::UnescapeRule::SPACES, NULL, NULL, &match_start);
 397   // If the first match in the untrimmed string was inside a scheme that we
 398   // trimmed, look for a subsequent match.
 399   if (match_start == base::string16::npos)
 400     match_start = match_contents.find(input_text);
 401   // Update |match_contents_| and |match_contents_class_| if it's allowed.
 402   if (allow_bolding_nothing || (match_start != base::string16::npos)) {
 403     match_contents_ = match_contents;
 404     // Safe if |match_start| is npos; also safe if the input is longer than the
 405     // remaining contents after |match_start|.
 406     AutocompleteMatch::ClassifyLocationInString(match_start,
 407         input_text.length(), match_contents_.length(),
 408         ACMatchClassification::URL, &match_contents_class_);
 409   }
 410 }
 411
 412 bool BaseSearchProvider::NavigationResult::IsInlineable(
 413     const base::string16& input) const {
 414   return
 415       URLPrefix::BestURLPrefix(base::UTF8ToUTF16(url_.spec()), input) != NULL;
 416 }
 417
 418 int BaseSearchProvider::NavigationResult::CalculateRelevance(
 419     const AutocompleteInput& input,
 420     bool keyword_provider_requested) const {
 421   return (from_keyword_provider_ || !keyword_provider_requested) ? 800 : 150;
 422 }
 423
 424 // BaseSearchProvider::Results -------------------------------------------------
 425
 426 BaseSearchProvider::Results::Results() : verbatim_relevance(-1) {}
 427
 428 BaseSearchProvider::Results::~Results() {}
 429
 430 void BaseSearchProvider::Results::Clear() {
 431   suggest_results.clear();
 432   navigation_results.clear();
 433   verbatim_relevance = -1;
 434   metadata.clear();
 435 }
 436
 437 bool BaseSearchProvider::Results::HasServerProvidedScores() const {
 438   if (verbatim_relevance >= 0)
 439     return true;
 440
 441   // Right now either all results of one type will be server-scored or they will
 442   // all be locally scored, but in case we change this later, we'll just check
 443   // them all.
 444   for (SuggestResults::const_iterator i(suggest_results.begin());
 445        i != suggest_results.end(); ++i) {
 446     if (i->relevance_from_server())
 447       return true;
 448   }
 449   for (NavigationResults::const_iterator i(navigation_results.begin());
 450        i != navigation_results.end(); ++i) {
 451     if (i->relevance_from_server())
 452       return true;
 453   }
 454
 455   return false;
 456 }
 457
 458 void BaseSearchProvider::SetDeletionURL(const std::string& deletion_url,
 459                                         AutocompleteMatch* match) {
 460   if (deletion_url.empty())
 461     return;
 462   TemplateURLService* template_service =
 463       TemplateURLServiceFactory::GetForProfile(profile_);
 464   if (!template_service)
 465     return;
 466   GURL url = TemplateURLService::GenerateSearchURL(
 467       template_service->GetDefaultSearchProvider());
 468   url = url.GetOrigin().Resolve(deletion_url);
 469   if (url.is_valid()) {
 470     match->RecordAdditionalInfo(BaseSearchProvider::kDeletionUrlKey,
 471         url.spec());
 472     match->deletable = true;
 473   }
 474 }
 475
 476 // BaseSearchProvider ---------------------------------------------------------
 477
 478 // static
 479 AutocompleteMatch BaseSearchProvider::CreateSearchSuggestion(
 480     AutocompleteProvider* autocomplete_provider,
 481     const AutocompleteInput& input,
 482     const SuggestResult& suggestion,
 483     const TemplateURL* template_url,
 484     int accepted_suggestion,
 485     int omnibox_start_margin,
 486     bool append_extra_query_params,
 487     bool from_app_list) {
 488   AutocompleteMatch match(autocomplete_provider, suggestion.relevance(), false,
 489                           suggestion.type());
 490
 491   if (!template_url)
 492     return match;
 493   match.keyword = template_url->keyword();
 494   match.contents = suggestion.match_contents();
 495   match.contents_class = suggestion.match_contents_class();
 496   match.answer_contents = suggestion.answer_contents();
 497   match.answer_type = suggestion.answer_type();
 498   if (suggestion.type() == AutocompleteMatchType::SEARCH_SUGGEST_INFINITE) {
 499     match.RecordAdditionalInfo(
 500         kACMatchPropertyInputText, base::UTF16ToUTF8(input.text()));
 501     match.RecordAdditionalInfo(
 502         kACMatchPropertyContentsPrefix,
 503         base::UTF16ToUTF8(suggestion.match_contents_prefix()));
 504     match.RecordAdditionalInfo(
 505         kACMatchPropertyContentsStartIndex,
 506         static_cast<int>(
 507             suggestion.suggestion().length() - match.contents.length()));
 508   }
 509
 510   if (!suggestion.annotation().empty())
 511     match.description = suggestion.annotation();
 512
 513   // suggestion.match_contents() should have already been collapsed.
 514   match.allowed_to_be_default_match =
 515       (base::CollapseWhitespace(input.text(), false) ==
 516        suggestion.match_contents());
 517
 518   // When the user forced a query, we need to make sure all the fill_into_edit
 519   // values preserve that property.  Otherwise, if the user starts editing a
 520   // suggestion, non-Search results will suddenly appear.
 521   if (input.type() == AutocompleteInput::FORCED_QUERY)
 522     match.fill_into_edit.assign(base::ASCIIToUTF16("?"));
 523   if (suggestion.from_keyword_provider())
 524     match.fill_into_edit.append(match.keyword + base::char16(' '));
 525   if (!input.prevent_inline_autocomplete() &&
 526       StartsWith(suggestion.suggestion(), input.text(), false)) {
 527     match.inline_autocompletion =
 528         suggestion.suggestion().substr(input.text().length());
 529     match.allowed_to_be_default_match = true;
 530   }
 531   match.fill_into_edit.append(suggestion.suggestion());
 532
 533   const TemplateURLRef& search_url = template_url->url_ref();
 534   DCHECK(search_url.SupportsReplacement());
 535   match.search_terms_args.reset(
 536       new TemplateURLRef::SearchTermsArgs(suggestion.suggestion()));
 537   match.search_terms_args->original_query = input.text();
 538   match.search_terms_args->accepted_suggestion = accepted_suggestion;
 539   match.search_terms_args->omnibox_start_margin = omnibox_start_margin;
 540   match.search_terms_args->suggest_query_params =
 541       suggestion.suggest_query_params();
 542   match.search_terms_args->append_extra_query_params =
 543       append_extra_query_params;
 544   match.search_terms_args->from_app_list = from_app_list;
 545   // This is the destination URL sans assisted query stats.  This must be set
 546   // so the AutocompleteController can properly de-dupe; the controller will
 547   // eventually overwrite it before it reaches the user.
 548   match.destination_url =
 549       GURL(search_url.ReplaceSearchTerms(*match.search_terms_args.get()));
 550
 551   // Search results don't look like URLs.
 552   match.transition = suggestion.from_keyword_provider() ?
 553       content::PAGE_TRANSITION_KEYWORD : content::PAGE_TRANSITION_GENERATED;
 554
 555   return match;
 556 }
 557
 558 // static
 559 scoped_ptr<base::Value> BaseSearchProvider::DeserializeJsonData(
 560     std::string json_data) {
 561   // The JSON response should be an array.
 562   for (size_t response_start_index = json_data.find("["), i = 0;
 563        response_start_index != std::string::npos && i < 5;
 564        response_start_index = json_data.find("[", 1), i++) {
 565     // Remove any XSSI guards to allow for JSON parsing.
 566     if (response_start_index > 0)
 567       json_data.erase(0, response_start_index);
 568
 569     JSONStringValueSerializer deserializer(json_data);
 570     deserializer.set_allow_trailing_comma(true);
 571     int error_code = 0;
 572     scoped_ptr<base::Value> data(deserializer.Deserialize(&error_code, NULL));
 573     if (error_code == 0)
 574       return data.Pass();
 575   }
 576   return scoped_ptr<base::Value>();
 577 }
 578
 579 // static
 580 bool BaseSearchProvider::ZeroSuggestEnabled(
 581     const GURL& suggest_url,
 582     const TemplateURL* template_url,
 583     AutocompleteInput::PageClassification page_classification,
 584     Profile* profile) {
 585   if (!OmniboxFieldTrial::InZeroSuggestFieldTrial())
 586     return false;
 587
 588   // Make sure we are sending the suggest request through HTTPS to prevent
 589   // exposing the current page URL or personalized results without encryption.
 590   if (!suggest_url.SchemeIs(url::kHttpsScheme))
 591     return false;
 592
 593   // Don't show zero suggest on the NTP.
 594   // TODO(hfung): Experiment with showing MostVisited zero suggest on NTP
 595   // under the conditions described in crbug.com/305366.
 596   if ((page_classification ==
 597        AutocompleteInput::INSTANT_NTP_WITH_FAKEBOX_AS_STARTING_FOCUS) ||
 598       (page_classification ==
 599        AutocompleteInput::INSTANT_NTP_WITH_OMNIBOX_AS_STARTING_FOCUS))
 600     return false;
 601
 602   // Don't run if there's no profile or in incognito mode.
 603   if (profile == NULL || profile->IsOffTheRecord())
 604     return false;
 605
 606   // Don't run if we can't get preferences or search suggest is not enabled.
 607   PrefService* prefs = profile->GetPrefs();
 608   if (!prefs->GetBoolean(prefs::kSearchSuggestEnabled))
 609     return false;
 610
 611   // Only make the request if we know that the provider supports zero suggest
 612   // (currently only the prepopulated Google provider).
 613   if (template_url == NULL || !template_url->SupportsReplacement() ||
 614       TemplateURLPrepopulateData::GetEngineType(*template_url) !=
 615       SEARCH_ENGINE_GOOGLE)
 616     return false;
 617
 618   return true;
 619 }
 620
 621 // static
 622 bool BaseSearchProvider::CanSendURL(
 623     const GURL& current_page_url,
 624     const GURL& suggest_url,
 625     const TemplateURL* template_url,
 626     AutocompleteInput::PageClassification page_classification,
 627     Profile* profile) {
 628   if (!ZeroSuggestEnabled(suggest_url, template_url, page_classification,
 629                           profile))
 630     return false;
 631
 632   if (!current_page_url.is_valid())
 633     return false;
 634
 635   // Only allow HTTP URLs or HTTPS URLs for the same domain as the search
 636   // provider.
 637   if ((current_page_url.scheme() != url::kHttpScheme) &&
 638       ((current_page_url.scheme() != url::kHttpsScheme) ||
 639        !net::registry_controlled_domains::SameDomainOrHost(
 640            current_page_url, suggest_url,
 641            net::registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES)))
 642     return false;
 643
 644   // Check field trials and settings allow sending the URL on suggest requests.
 645   ProfileSyncService* service =
 646       ProfileSyncServiceFactory::GetInstance()->GetForProfile(profile);
 647   sync_driver::SyncPrefs sync_prefs(profile->GetPrefs());
 648   if (service == NULL ||
 649       !service->IsSyncEnabledAndLoggedIn() ||
 650       !sync_prefs.GetPreferredDataTypes(syncer::UserTypes()).Has(
 651           syncer::PROXY_TABS) ||
 652       service->GetEncryptedDataTypes().Has(syncer::SESSIONS))
 653     return false;
 654
 655   return true;
 656 }
 657
 658 void BaseSearchProvider::OnURLFetchComplete(const net::URLFetcher* source) {
 659   DCHECK(!done_);
 660   suggest_results_pending_--;
 661   DCHECK_GE(suggest_results_pending_, 0);  // Should never go negative.
 662
 663   const bool is_keyword = IsKeywordFetcher(source);
 664
 665   // Ensure the request succeeded and that the provider used is still available.
 666   // A verbatim match cannot be generated without this provider, causing errors.
 667   const bool request_succeeded =
 668       source->GetStatus().is_success() && (source->GetResponseCode() == 200) &&
 669       GetTemplateURL(is_keyword);
 670
 671   LogFetchComplete(request_succeeded, is_keyword);
 672
 673   bool results_updated = false;
 674   if (request_succeeded) {
 675     const net::HttpResponseHeaders* const response_headers =
 676         source->GetResponseHeaders();
 677     std::string json_data;
 678     source->GetResponseAsString(&json_data);
 679
 680     // JSON is supposed to be UTF-8, but some suggest service providers send
 681     // JSON files in non-UTF-8 encodings.  The actual encoding is usually
 682     // specified in the Content-Type header field.
 683     if (response_headers) {
 684       std::string charset;
 685       if (response_headers->GetCharset(&charset)) {
 686         base::string16 data_16;
 687         // TODO(jungshik): Switch to CodePageToUTF8 after it's added.
 688         if (base::CodepageToUTF16(json_data, charset.c_str(),
 689                                   base::OnStringConversionError::FAIL,
 690                                   &data_16))
 691           json_data = base::UTF16ToUTF8(data_16);
 692       }
 693     }
 694
 695     scoped_ptr<base::Value> data(DeserializeJsonData(json_data));
 696     if (data && StoreSuggestionResponse(json_data, *data.get()))
 697       return;
 698
 699     results_updated = data.get() && ParseSuggestResults(
 700         *data.get(), is_keyword, GetResultsToFill(is_keyword));
 701   }
 702
 703   UpdateMatches();
 704   if (done_ || results_updated)
 705     listener_->OnProviderUpdate(results_updated);
 706 }
 707
 708 void BaseSearchProvider::AddMatchToMap(const SuggestResult& result,
 709                                        const std::string& metadata,
 710                                        int accepted_suggestion,
 711                                        bool mark_as_deletable,
 712                                        MatchMap* map) {
 713   InstantService* instant_service =
 714       InstantServiceFactory::GetForProfile(profile_);
 715   // Android and iOS have no InstantService.
 716   const int omnibox_start_margin = instant_service ?
 717       instant_service->omnibox_start_margin() : chrome::kDisableStartMargin;
 718
 719   AutocompleteMatch match = CreateSearchSuggestion(
 720       this, GetInput(result.from_keyword_provider()), result,
 721       GetTemplateURL(result.from_keyword_provider()), accepted_suggestion,
 722       omnibox_start_margin, ShouldAppendExtraParams(result),
 723       in_app_list_);
 724   if (!match.destination_url.is_valid())
 725     return;
 726   match.search_terms_args->bookmark_bar_pinned =
 727       profile_->GetPrefs()->GetBoolean(prefs::kShowBookmarkBar);
 728   match.RecordAdditionalInfo(kRelevanceFromServerKey,
 729                              result.relevance_from_server() ? kTrue : kFalse);
 730   match.RecordAdditionalInfo(kShouldPrefetchKey,
 731                              result.should_prefetch() ? kTrue : kFalse);
 732   SetDeletionURL(result.deletion_url(), &match);
 733   if (mark_as_deletable)
 734     match.deletable = true;
 735   // Metadata is needed only for prefetching queries.
 736   if (result.should_prefetch())
 737     match.RecordAdditionalInfo(kSuggestMetadataKey, metadata);
 738
 739   // Try to add |match| to |map|.  If a match for this suggestion is
 740   // already in |map|, replace it if |match| is more relevant.
 741   // NOTE: Keep this ToLower() call in sync with url_database.cc.
 742   MatchKey match_key(
 743       std::make_pair(base::i18n::ToLower(result.suggestion()),
 744                      match.search_terms_args->suggest_query_params));
 745   const std::pair<MatchMap::iterator, bool> i(
 746        map->insert(std::make_pair(match_key, match)));
 747
 748   bool should_prefetch = result.should_prefetch();
 749   if (!i.second) {
 750     // NOTE: We purposefully do a direct relevance comparison here instead of
 751     // using AutocompleteMatch::MoreRelevant(), so that we'll prefer "items
 752     // added first" rather than "items alphabetically first" when the scores
 753     // are equal. The only case this matters is when a user has results with
 754     // the same score that differ only by capitalization; because the history
 755     // system returns results sorted by recency, this means we'll pick the most
 756     // recent such result even if the precision of our relevance score is too
 757     // low to distinguish the two.
 758     if (match.relevance > i.first->second.relevance) {
 759       match.duplicate_matches.insert(match.duplicate_matches.end(),
 760                                      i.first->second.duplicate_matches.begin(),
 761                                      i.first->second.duplicate_matches.end());
 762       i.first->second.duplicate_matches.clear();
 763       match.duplicate_matches.push_back(i.first->second);
 764       i.first->second = match;
 765     } else {
 766       i.first->second.duplicate_matches.push_back(match);
 767       if (match.keyword == i.first->second.keyword) {
 768         // Old and new matches are from the same search provider. It is okay to
 769         // record one match's prefetch data onto a different match (for the same
 770         // query string) for the following reasons:
 771         // 1. Because the suggest server only sends down a query string from
 772         // which we construct a URL, rather than sending a full URL, and because
 773         // we construct URLs from query strings in the same way every time, the
 774         // URLs for the two matches will be the same. Therefore, we won't end up
 775         // prefetching something the server didn't intend.
 776         // 2. Presumably the server sets the prefetch bit on a match it things
 777         // is sufficiently relevant that the user is likely to choose it.
 778         // Surely setting the prefetch bit on a match of even higher relevance
 779         // won't violate this assumption.
 780         should_prefetch |= ShouldPrefetch(i.first->second);
 781         i.first->second.RecordAdditionalInfo(kShouldPrefetchKey,
 782                                              should_prefetch ? kTrue : kFalse);
 783         if (should_prefetch)
 784           i.first->second.RecordAdditionalInfo(kSuggestMetadataKey, metadata);
 785       }
 786     }
 787   }
 788 }
 789
 790 bool BaseSearchProvider::ParseSuggestResults(const base::Value& root_val,
 791                                              bool is_keyword_result,
 792                                              Results* results) {
 793   base::string16 query;
 794   const base::ListValue* root_list = NULL;
 795   const base::ListValue* results_list = NULL;
 796   const AutocompleteInput& input = GetInput(is_keyword_result);
 797
 798   if (!root_val.GetAsList(&root_list) || !root_list->GetString(0, &query) ||
 799       query != input.text() || !root_list->GetList(1, &results_list))
 800     return false;
 801
 802   // 3rd element: Description list.
 803   const base::ListValue* descriptions = NULL;
 804   root_list->GetList(2, &descriptions);
 805
 806   // 4th element: Disregard the query URL list for now.
 807
 808   // Reset suggested relevance information.
 809   results->verbatim_relevance = -1;
 810
 811   // 5th element: Optional key-value pairs from the Suggest server.
 812   const base::ListValue* types = NULL;
 813   const base::ListValue* relevances = NULL;
 814   const base::ListValue* suggestion_details = NULL;
 815   const base::DictionaryValue* extras = NULL;
 816   int prefetch_index = -1;
 817   if (root_list->GetDictionary(4, &extras)) {
 818     extras->GetList("google:suggesttype", &types);
 819
 820     // Discard this list if its size does not match that of the suggestions.
 821     if (extras->GetList("google:suggestrelevance", &relevances) &&
 822         (relevances->GetSize() != results_list->GetSize()))
 823       relevances = NULL;
 824     extras->GetInteger("google:verbatimrelevance",
 825                        &results->verbatim_relevance);
 826
 827     // Check if the active suggest field trial (if any) has triggered either
 828     // for the default provider or keyword provider.
 829     bool triggered = false;
 830     extras->GetBoolean("google:fieldtrialtriggered", &triggered);
 831     field_trial_triggered_ |= triggered;
 832     field_trial_triggered_in_session_ |= triggered;
 833
 834     const base::DictionaryValue* client_data = NULL;
 835     if (extras->GetDictionary("google:clientdata", &client_data) && client_data)
 836       client_data->GetInteger("phi", &prefetch_index);
 837
 838     if (extras->GetList("google:suggestdetail", &suggestion_details) &&
 839         suggestion_details->GetSize() != results_list->GetSize())
 840       suggestion_details = NULL;
 841
 842     // Store the metadata that came with the response in case we need to pass it
 843     // along with the prefetch query to Instant.
 844     JSONStringValueSerializer json_serializer(&results->metadata);
 845     json_serializer.Serialize(*extras);
 846   }
 847
 848   // Clear the previous results now that new results are available.
 849   results->suggest_results.clear();
 850   results->navigation_results.clear();
 851
 852   base::string16 suggestion;
 853   std::string type;
 854   int relevance = GetDefaultResultRelevance();
 855   // Prohibit navsuggest in FORCED_QUERY mode.  Users wants queries, not URLs.
 856   const bool allow_navsuggest = input.type() != AutocompleteInput::FORCED_QUERY;
 857   const std::string languages(
 858       profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
 859   const base::string16& trimmed_input =
 860       base::CollapseWhitespace(input.text(), false);
 861   for (size_t index = 0; results_list->GetString(index, &suggestion); ++index) {
 862     // Google search may return empty suggestions for weird input characters,
 863     // they make no sense at all and can cause problems in our code.
 864     if (suggestion.empty())
 865       continue;
 866
 867     // Apply valid suggested relevance scores; discard invalid lists.
 868     if (relevances != NULL && !relevances->GetInteger(index, &relevance))
 869       relevances = NULL;
 870     AutocompleteMatchType::Type match_type =
 871         AutocompleteMatchType::SEARCH_SUGGEST;
 872     if (types && types->GetString(index, &type))
 873       match_type = GetAutocompleteMatchType(type);
 874     const base::DictionaryValue* suggestion_detail = NULL;
 875     std::string deletion_url;
 876
 877     if (suggestion_details &&
 878         suggestion_details->GetDictionary(index, &suggestion_detail))
 879       suggestion_detail->GetString("du", &deletion_url);
 880
 881     if ((match_type == AutocompleteMatchType::NAVSUGGEST) ||
 882         (match_type == AutocompleteMatchType::NAVSUGGEST_PERSONALIZED)) {
 883       // Do not blindly trust the URL coming from the server to be valid.
 884       GURL url(URLFixerUpper::FixupURL(
 885           base::UTF16ToUTF8(suggestion), std::string()));
 886       if (url.is_valid() && allow_navsuggest) {
 887         base::string16 title;
 888         if (descriptions != NULL)
 889           descriptions->GetString(index, &title);
 890         results->navigation_results.push_back(NavigationResult(
 891             *this, url, match_type, title, deletion_url, is_keyword_result,
 892             relevance, relevances != NULL, input.text(), languages));
 893       }
 894     } else {
 895       base::string16 match_contents = suggestion;
 896       base::string16 match_contents_prefix;
 897       base::string16 annotation;
 898       base::string16 answer_contents;
 899       base::string16 answer_type;
 900       std::string suggest_query_params;
 901
 902       if (suggestion_details) {
 903         suggestion_details->GetDictionary(index, &suggestion_detail);
 904         if (suggestion_detail) {
 905           suggestion_detail->GetString("t", &match_contents);
 906           suggestion_detail->GetString("mp", &match_contents_prefix);
 907           // Error correction for bad data from server.
 908           if (match_contents.empty())
 909             match_contents = suggestion;
 910           suggestion_detail->GetString("a", &annotation);
 911           suggestion_detail->GetString("q", &suggest_query_params);
 912
 913           // Extract Answers, if provided.
 914           const base::DictionaryValue* answer_json = NULL;
 915           if (suggestion_detail->GetDictionary("ansa", &answer_json)) {
 916             std::string contents;
 917             base::JSONWriter::Write(answer_json, &contents);
 918             answer_contents = base::UTF8ToUTF16(contents);
 919             suggestion_detail->GetString("ansb", &answer_type);
 920           }
 921         }
 922       }
 923
 924       bool should_prefetch = static_cast<int>(index) == prefetch_index;
 925       // TODO(kochi): Improve calculator suggestion presentation.
 926       results->suggest_results.push_back(SuggestResult(
 927           base::CollapseWhitespace(suggestion, false), match_type,
 928           base::CollapseWhitespace(match_contents, false),
 929           match_contents_prefix, annotation, answer_contents, answer_type,
 930           suggest_query_params, deletion_url, is_keyword_result, relevance,
 931           relevances != NULL, should_prefetch, trimmed_input));
 932     }
 933   }
 934   SortResults(is_keyword_result, relevances, results);
 935   return true;
 936 }
 937
 938 void BaseSearchProvider::SortResults(bool is_keyword,
 939                                      const base::ListValue* relevances,
 940                                      Results* results) {
 941 }
 942
 943 bool BaseSearchProvider::StoreSuggestionResponse(
 944     const std::string& json_data,
 945     const base::Value& parsed_data) {
 946   return false;
 947 }
 948
 949 void BaseSearchProvider::ModifyProviderInfo(
 950     metrics::OmniboxEventProto_ProviderInfo* provider_info) const {
 951 }
 952
 953 void BaseSearchProvider::DeleteMatchFromMatches(
 954     const AutocompleteMatch& match) {
 955   for (ACMatches::iterator i(matches_.begin()); i != matches_.end(); ++i) {
 956     // Find the desired match to delete by checking the type and contents.
 957     // We can't check the destination URL, because the autocomplete controller
 958     // may have reformulated that. Not that while checking for matching
 959     // contents works for personalized suggestions, if more match types gain
 960     // deletion support, this algorithm may need to be re-examined.
 961     if (i->contents == match.contents && i->type == match.type) {
 962       matches_.erase(i);
 963       break;
 964     }
 965   }
 966 }
 967
 968 void BaseSearchProvider::OnDeletionComplete(
 969     bool success, SuggestionDeletionHandler* handler) {
 970   RecordDeletionResult(success);
 971   SuggestionDeletionHandlers::iterator it = std::find(
 972       deletion_handlers_.begin(), deletion_handlers_.end(), handler);
 973   DCHECK(it != deletion_handlers_.end());
 974   deletion_handlers_.erase(it);
 975 }