components/omnibox/search_suggestion_parser.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/omnibox/search_suggestion_parser.h"
   6
   7 #include "base/i18n/icu_string_conversions.h"
   8 #include "base/json/json_string_value_serializer.h"
   9 #include "base/json/json_writer.h"
  10 #include "base/logging.h"
  11 #include "base/metrics/histogram.h"
  12 #include "base/strings/string_number_conversions.h"
  13 #include "base/strings/string_util.h"
  14 #include "base/strings/utf_string_conversions.h"
  15 #include "base/values.h"
  16 #include "components/omnibox/autocomplete_input.h"
  17 #include "components/omnibox/url_prefix.h"
  18 #include "components/url_fixer/url_fixer.h"
  19 #include "net/base/net_util.h"
  20 #include "net/http/http_response_headers.h"
  21 #include "net/url_request/url_fetcher.h"
  22 #include "url/url_constants.h"
  23
  24 namespace {
  25
  26 AutocompleteMatchType::Type GetAutocompleteMatchType(const std::string& type) {
  27   if (type == "CALCULATOR")
  28     return AutocompleteMatchType::CALCULATOR;
  29   if (type == "ENTITY")
  30     return AutocompleteMatchType::SEARCH_SUGGEST_ENTITY;
  31   if (type == "TAIL")
  32     return AutocompleteMatchType::SEARCH_SUGGEST_TAIL;
  33   if (type == "PERSONALIZED_QUERY")
  34     return AutocompleteMatchType::SEARCH_SUGGEST_PERSONALIZED;
  35   if (type == "PROFILE")
  36     return AutocompleteMatchType::SEARCH_SUGGEST_PROFILE;
  37   if (type == "NAVIGATION")
  38     return AutocompleteMatchType::NAVSUGGEST;
  39   if (type == "PERSONALIZED_NAVIGATION")
  40     return AutocompleteMatchType::NAVSUGGEST_PERSONALIZED;
  41   return AutocompleteMatchType::SEARCH_SUGGEST;
  42 }
  43
  44 }  // namespace
  45
  46 // SearchSuggestionParser::Result ----------------------------------------------
  47
  48 SearchSuggestionParser::Result::Result(bool from_keyword_provider,
  49                                        int relevance,
  50                                        bool relevance_from_server,
  51                                        AutocompleteMatchType::Type type,
  52                                        const std::string& deletion_url)
  53     : from_keyword_provider_(from_keyword_provider),
  54       type_(type),
  55       relevance_(relevance),
  56       relevance_from_server_(relevance_from_server),
  57       received_after_last_keystroke_(true),
  58       deletion_url_(deletion_url) {}
  59
  60 SearchSuggestionParser::Result::~Result() {}
  61
  62 // SearchSuggestionParser::SuggestResult ---------------------------------------
  63
  64 SearchSuggestionParser::SuggestResult::SuggestResult(
  65     const base::string16& suggestion,
  66     AutocompleteMatchType::Type type,
  67     const base::string16& match_contents,
  68     const base::string16& match_contents_prefix,
  69     const base::string16& annotation,
  70     const base::string16& answer_contents,
  71     const base::string16& answer_type,
  72     scoped_ptr<SuggestionAnswer> answer,
  73     const std::string& suggest_query_params,
  74     const std::string& deletion_url,
  75     bool from_keyword_provider,
  76     int relevance,
  77     bool relevance_from_server,
  78     bool should_prefetch,
  79     const base::string16& input_text)
  80     : Result(from_keyword_provider,
  81              relevance,
  82              relevance_from_server,
  83              type,
  84              deletion_url),
  85       suggestion_(suggestion),
  86       match_contents_prefix_(match_contents_prefix),
  87       annotation_(annotation),
  88       suggest_query_params_(suggest_query_params),
  89       answer_contents_(answer_contents),
  90       answer_type_(answer_type),
  91       answer_(answer.Pass()),
  92       should_prefetch_(should_prefetch) {
  93   match_contents_ = match_contents;
  94   DCHECK(!match_contents_.empty());
  95   ClassifyMatchContents(true, input_text);
  96 }
  97
  98 SearchSuggestionParser::SuggestResult::SuggestResult(
  99     const SuggestResult& result)
 100     : Result(result),
 101       suggestion_(result.suggestion_),
 102       match_contents_prefix_(result.match_contents_prefix_),
 103       annotation_(result.annotation_),
 104       suggest_query_params_(result.suggest_query_params_),
 105       answer_contents_(result.answer_contents_),
 106       answer_type_(result.answer_type_),
 107       answer_(SuggestionAnswer::copy(result.answer_.get())),
 108       should_prefetch_(result.should_prefetch_) {
 109 }
 110
 111 SearchSuggestionParser::SuggestResult::~SuggestResult() {}
 112
 113 SearchSuggestionParser::SuggestResult&
 114     SearchSuggestionParser::SuggestResult::operator=(const SuggestResult& rhs) {
 115   if (this == &rhs)
 116     return *this;
 117
 118   // Assign via parent class first.
 119   Result::operator=(rhs);
 120
 121   suggestion_ = rhs.suggestion_;
 122   match_contents_prefix_ = rhs.match_contents_prefix_;
 123   annotation_ = rhs.annotation_;
 124   suggest_query_params_ = rhs.suggest_query_params_;
 125   answer_contents_ = rhs.answer_contents_;
 126   answer_type_ = rhs.answer_type_;
 127   answer_ = SuggestionAnswer::copy(rhs.answer_.get());
 128   should_prefetch_ = rhs.should_prefetch_;
 129
 130   return *this;
 131 }
 132
 133 void SearchSuggestionParser::SuggestResult::ClassifyMatchContents(
 134     const bool allow_bolding_all,
 135     const base::string16& input_text) {
 136   if (input_text.empty()) {
 137     // In case of zero-suggest results, do not highlight matches.
 138     match_contents_class_.push_back(
 139         ACMatchClassification(0, ACMatchClassification::NONE));
 140     return;
 141   }
 142
 143   base::string16 lookup_text = input_text;
 144   if (type_ == AutocompleteMatchType::SEARCH_SUGGEST_TAIL) {
 145     const size_t contents_index =
 146         suggestion_.length() - match_contents_.length();
 147     // Ensure the query starts with the input text, and ends with the match
 148     // contents, and the input text has an overlap with contents.
 149     if (StartsWith(suggestion_, input_text, true) &&
 150         EndsWith(suggestion_, match_contents_, true) &&
 151         (input_text.length() > contents_index)) {
 152       lookup_text = input_text.substr(contents_index);
 153     }
 154   }
 155   size_t lookup_position = match_contents_.find(lookup_text);
 156   if (!allow_bolding_all && (lookup_position == base::string16::npos)) {
 157     // Bail if the code below to update the bolding would bold the whole
 158     // string.  Note that the string may already be entirely bolded; if
 159     // so, leave it as is.
 160     return;
 161   }
 162   match_contents_class_.clear();
 163   // We do intra-string highlighting for suggestions - the suggested segment
 164   // will be highlighted, e.g. for input_text = "you" the suggestion may be
 165   // "youtube", so we'll bold the "tube" section: you*tube*.
 166   if (input_text != match_contents_) {
 167     if (lookup_position == base::string16::npos) {
 168       // The input text is not a substring of the query string, e.g. input
 169       // text is "slasdot" and the query string is "slashdot", so we bold the
 170       // whole thing.
 171       match_contents_class_.push_back(
 172           ACMatchClassification(0, ACMatchClassification::MATCH));
 173     } else {
 174       // We don't iterate over the string here annotating all matches because
 175       // it looks odd to have every occurrence of a substring that may be as
 176       // short as a single character highlighted in a query suggestion result,
 177       // e.g. for input text "s" and query string "southwest airlines", it
 178       // looks odd if both the first and last s are highlighted.
 179       if (lookup_position != 0) {
 180         match_contents_class_.push_back(
 181             ACMatchClassification(0, ACMatchClassification::MATCH));
 182       }
 183       match_contents_class_.push_back(
 184           ACMatchClassification(lookup_position, ACMatchClassification::NONE));
 185       size_t next_fragment_position = lookup_position + lookup_text.length();
 186       if (next_fragment_position < match_contents_.length()) {
 187         match_contents_class_.push_back(ACMatchClassification(
 188             next_fragment_position, ACMatchClassification::MATCH));
 189       }
 190     }
 191   } else {
 192     // Otherwise, match_contents_ is a verbatim (what-you-typed) match, either
 193     // for the default provider or a keyword search provider.
 194     match_contents_class_.push_back(
 195         ACMatchClassification(0, ACMatchClassification::NONE));
 196   }
 197 }
 198
 199 int SearchSuggestionParser::SuggestResult::CalculateRelevance(
 200     const AutocompleteInput& input,
 201     bool keyword_provider_requested) const {
 202   if (!from_keyword_provider_ && keyword_provider_requested)
 203     return 100;
 204   return ((input.type() == metrics::OmniboxInputType::URL) ? 300 : 600);
 205 }
 206
 207 // SearchSuggestionParser::NavigationResult ------------------------------------
 208
 209 SearchSuggestionParser::NavigationResult::NavigationResult(
 210     const AutocompleteSchemeClassifier& scheme_classifier,
 211     const GURL& url,
 212     AutocompleteMatchType::Type type,
 213     const base::string16& description,
 214     const std::string& deletion_url,
 215     bool from_keyword_provider,
 216     int relevance,
 217     bool relevance_from_server,
 218     const base::string16& input_text,
 219     const std::string& languages)
 220     : Result(from_keyword_provider, relevance, relevance_from_server, type,
 221              deletion_url),
 222       url_(url),
 223       formatted_url_(AutocompleteInput::FormattedStringWithEquivalentMeaning(
 224           url, net::FormatUrl(url, languages,
 225                               net::kFormatUrlOmitAll & ~net::kFormatUrlOmitHTTP,
 226                               net::UnescapeRule::SPACES, NULL, NULL, NULL),
 227           scheme_classifier)),
 228       description_(description) {
 229   DCHECK(url_.is_valid());
 230   CalculateAndClassifyMatchContents(true, input_text, languages);
 231 }
 232
 233 SearchSuggestionParser::NavigationResult::~NavigationResult() {}
 234
 235 void
 236 SearchSuggestionParser::NavigationResult::CalculateAndClassifyMatchContents(
 237     const bool allow_bolding_nothing,
 238     const base::string16& input_text,
 239     const std::string& languages) {
 240   if (input_text.empty()) {
 241     // In case of zero-suggest results, do not highlight matches.
 242     match_contents_class_.push_back(
 243         ACMatchClassification(0, ACMatchClassification::NONE));
 244     return;
 245   }
 246
 247   // First look for the user's input inside the formatted url as it would be
 248   // without trimming the scheme, so we can find matches at the beginning of the
 249   // scheme.
 250   const URLPrefix* prefix =
 251       URLPrefix::BestURLPrefix(formatted_url_, input_text);
 252   size_t match_start = (prefix == NULL) ?
 253       formatted_url_.find(input_text) : prefix->prefix.length();
 254   bool trim_http = !AutocompleteInput::HasHTTPScheme(input_text) &&
 255                    (!prefix || (match_start != 0));
 256   const net::FormatUrlTypes format_types =
 257       net::kFormatUrlOmitAll & ~(trim_http ? 0 : net::kFormatUrlOmitHTTP);
 258
 259   base::string16 match_contents = net::FormatUrl(url_, languages, format_types,
 260       net::UnescapeRule::SPACES, NULL, NULL, &match_start);
 261   // If the first match in the untrimmed string was inside a scheme that we
 262   // trimmed, look for a subsequent match.
 263   if (match_start == base::string16::npos)
 264     match_start = match_contents.find(input_text);
 265   // Update |match_contents_| and |match_contents_class_| if it's allowed.
 266   if (allow_bolding_nothing || (match_start != base::string16::npos)) {
 267     match_contents_ = match_contents;
 268     // Safe if |match_start| is npos; also safe if the input is longer than the
 269     // remaining contents after |match_start|.
 270     AutocompleteMatch::ClassifyLocationInString(match_start,
 271         input_text.length(), match_contents_.length(),
 272         ACMatchClassification::URL, &match_contents_class_);
 273   }
 274 }
 275
 276 int SearchSuggestionParser::NavigationResult::CalculateRelevance(
 277     const AutocompleteInput& input,
 278     bool keyword_provider_requested) const {
 279   return (from_keyword_provider_ || !keyword_provider_requested) ? 800 : 150;
 280 }
 281
 282 // SearchSuggestionParser::Results ---------------------------------------------
 283
 284 SearchSuggestionParser::Results::Results()
 285     : verbatim_relevance(-1),
 286       field_trial_triggered(false),
 287       relevances_from_server(false) {}
 288
 289 SearchSuggestionParser::Results::~Results() {}
 290
 291 void SearchSuggestionParser::Results::Clear() {
 292   suggest_results.clear();
 293   navigation_results.clear();
 294   verbatim_relevance = -1;
 295   metadata.clear();
 296 }
 297
 298 bool SearchSuggestionParser::Results::HasServerProvidedScores() const {
 299   if (verbatim_relevance >= 0)
 300     return true;
 301
 302   // Right now either all results of one type will be server-scored or they will
 303   // all be locally scored, but in case we change this later, we'll just check
 304   // them all.
 305   for (SuggestResults::const_iterator i(suggest_results.begin());
 306        i != suggest_results.end(); ++i) {
 307     if (i->relevance_from_server())
 308       return true;
 309   }
 310   for (NavigationResults::const_iterator i(navigation_results.begin());
 311        i != navigation_results.end(); ++i) {
 312     if (i->relevance_from_server())
 313       return true;
 314   }
 315
 316   return false;
 317 }
 318
 319 // SearchSuggestionParser ------------------------------------------------------
 320
 321 // static
 322 std::string SearchSuggestionParser::ExtractJsonData(
 323     const net::URLFetcher* source) {
 324   const net::HttpResponseHeaders* const response_headers =
 325       source->GetResponseHeaders();
 326   std::string json_data;
 327   source->GetResponseAsString(&json_data);
 328
 329   // JSON is supposed to be UTF-8, but some suggest service providers send
 330   // JSON files in non-UTF-8 encodings.  The actual encoding is usually
 331   // specified in the Content-Type header field.
 332   if (response_headers) {
 333     std::string charset;
 334     if (response_headers->GetCharset(&charset)) {
 335       base::string16 data_16;
 336       // TODO(jungshik): Switch to CodePageToUTF8 after it's added.
 337       if (base::CodepageToUTF16(json_data, charset.c_str(),
 338                                 base::OnStringConversionError::FAIL,
 339                                 &data_16))
 340         json_data = base::UTF16ToUTF8(data_16);
 341     }
 342   }
 343   return json_data;
 344 }
 345
 346 // static
 347 scoped_ptr<base::Value> SearchSuggestionParser::DeserializeJsonData(
 348     base::StringPiece json_data) {
 349   // The JSON response should be an array.
 350   for (size_t response_start_index = json_data.find("["), i = 0;
 351        response_start_index != base::StringPiece::npos && i < 5;
 352        response_start_index = json_data.find("[", 1), i++) {
 353     // Remove any XSSI guards to allow for JSON parsing.
 354     json_data.remove_prefix(response_start_index);
 355
 356     JSONStringValueDeserializer deserializer(json_data);
 357     deserializer.set_allow_trailing_comma(true);
 358     int error_code = 0;
 359     scoped_ptr<base::Value> data(deserializer.Deserialize(&error_code, NULL));
 360     if (error_code == 0)
 361       return data.Pass();
 362   }
 363   return scoped_ptr<base::Value>();
 364 }
 365
 366 // static
 367 bool SearchSuggestionParser::ParseSuggestResults(
 368     const base::Value& root_val,
 369     const AutocompleteInput& input,
 370     const AutocompleteSchemeClassifier& scheme_classifier,
 371     int default_result_relevance,
 372     const std::string& languages,
 373     bool is_keyword_result,
 374     Results* results) {
 375   base::string16 query;
 376   const base::ListValue* root_list = NULL;
 377   const base::ListValue* results_list = NULL;
 378
 379   if (!root_val.GetAsList(&root_list) || !root_list->GetString(0, &query) ||
 380       query != input.text() || !root_list->GetList(1, &results_list))
 381     return false;
 382
 383   // 3rd element: Description list.
 384   const base::ListValue* descriptions = NULL;
 385   root_list->GetList(2, &descriptions);
 386
 387   // 4th element: Disregard the query URL list for now.
 388
 389   // Reset suggested relevance information.
 390   results->verbatim_relevance = -1;
 391
 392   // 5th element: Optional key-value pairs from the Suggest server.
 393   const base::ListValue* types = NULL;
 394   const base::ListValue* relevances = NULL;
 395   const base::ListValue* suggestion_details = NULL;
 396   const base::DictionaryValue* extras = NULL;
 397   int prefetch_index = -1;
 398   if (root_list->GetDictionary(4, &extras)) {
 399     extras->GetList("google:suggesttype", &types);
 400
 401     // Discard this list if its size does not match that of the suggestions.
 402     if (extras->GetList("google:suggestrelevance", &relevances) &&
 403         (relevances->GetSize() != results_list->GetSize()))
 404       relevances = NULL;
 405     extras->GetInteger("google:verbatimrelevance",
 406                        &results->verbatim_relevance);
 407
 408     // Check if the active suggest field trial (if any) has triggered either
 409     // for the default provider or keyword provider.
 410     results->field_trial_triggered = false;
 411     extras->GetBoolean("google:fieldtrialtriggered",
 412                        &results->field_trial_triggered);
 413
 414     const base::DictionaryValue* client_data = NULL;
 415     if (extras->GetDictionary("google:clientdata", &client_data) && client_data)
 416       client_data->GetInteger("phi", &prefetch_index);
 417
 418     if (extras->GetList("google:suggestdetail", &suggestion_details) &&
 419         suggestion_details->GetSize() != results_list->GetSize())
 420       suggestion_details = NULL;
 421
 422     // Store the metadata that came with the response in case we need to pass it
 423     // along with the prefetch query to Instant.
 424     JSONStringValueSerializer json_serializer(&results->metadata);
 425     json_serializer.Serialize(*extras);
 426   }
 427
 428   // Clear the previous results now that new results are available.
 429   results->suggest_results.clear();
 430   results->navigation_results.clear();
 431   results->answers_image_urls.clear();
 432
 433   base::string16 suggestion;
 434   std::string type;
 435   int relevance = default_result_relevance;
 436   // Prohibit navsuggest in FORCED_QUERY mode.  Users wants queries, not URLs.
 437   const bool allow_navsuggest =
 438       input.type() != metrics::OmniboxInputType::FORCED_QUERY;
 439   const base::string16& trimmed_input =
 440       base::CollapseWhitespace(input.text(), false);
 441   for (size_t index = 0; results_list->GetString(index, &suggestion); ++index) {
 442     // Google search may return empty suggestions for weird input characters,
 443     // they make no sense at all and can cause problems in our code.
 444     if (suggestion.empty())
 445       continue;
 446
 447     // Apply valid suggested relevance scores; discard invalid lists.
 448     if (relevances != NULL && !relevances->GetInteger(index, &relevance))
 449       relevances = NULL;
 450     AutocompleteMatchType::Type match_type =
 451         AutocompleteMatchType::SEARCH_SUGGEST;
 452     if (types && types->GetString(index, &type))
 453       match_type = GetAutocompleteMatchType(type);
 454     const base::DictionaryValue* suggestion_detail = NULL;
 455     std::string deletion_url;
 456
 457     if (suggestion_details &&
 458         suggestion_details->GetDictionary(index, &suggestion_detail))
 459       suggestion_detail->GetString("du", &deletion_url);
 460
 461     if ((match_type == AutocompleteMatchType::NAVSUGGEST) ||
 462         (match_type == AutocompleteMatchType::NAVSUGGEST_PERSONALIZED)) {
 463       // Do not blindly trust the URL coming from the server to be valid.
 464       GURL url(
 465           url_fixer::FixupURL(base::UTF16ToUTF8(suggestion), std::string()));
 466       if (url.is_valid() && allow_navsuggest) {
 467         base::string16 title;
 468         if (descriptions != NULL)
 469           descriptions->GetString(index, &title);
 470         results->navigation_results.push_back(NavigationResult(
 471             scheme_classifier, url, match_type, title, deletion_url,
 472             is_keyword_result, relevance, relevances != NULL, input.text(),
 473             languages));
 474       }
 475     } else {
 476       // TODO(dschuyler) If the "= " is no longer sent from the back-end
 477       // then this may be removed.
 478       if ((match_type == AutocompleteMatchType::CALCULATOR) &&
 479           !suggestion.compare(0, 2, base::UTF8ToUTF16("= ")))
 480         suggestion.erase(0, 2);
 481
 482       base::string16 match_contents = suggestion;
 483       base::string16 match_contents_prefix;
 484       base::string16 annotation;
 485       base::string16 answer_contents;
 486       base::string16 answer_type_str;
 487       scoped_ptr<SuggestionAnswer> answer;
 488       std::string suggest_query_params;
 489
 490       if (suggestion_details) {
 491         suggestion_details->GetDictionary(index, &suggestion_detail);
 492         if (suggestion_detail) {
 493           suggestion_detail->GetString("t", &match_contents);
 494           suggestion_detail->GetString("mp", &match_contents_prefix);
 495           // Error correction for bad data from server.
 496           if (match_contents.empty())
 497             match_contents = suggestion;
 498           suggestion_detail->GetString("a", &annotation);
 499           suggestion_detail->GetString("q", &suggest_query_params);
 500
 501           // Extract the Answer, if provided.
 502           const base::DictionaryValue* answer_json = NULL;
 503           if (suggestion_detail->GetDictionary("ansa", &answer_json) &&
 504               suggestion_detail->GetString("ansb", &answer_type_str)) {
 505             bool answer_parsed_successfully = false;
 506             answer = SuggestionAnswer::ParseAnswer(answer_json);
 507             int answer_type = 0;
 508             if (answer && base::StringToInt(answer_type_str, &answer_type)) {
 509               answer_parsed_successfully = true;
 510
 511               answer->set_type(answer_type);
 512               answer->AddImageURLsTo(&results->answers_image_urls);
 513
 514               std::string contents;
 515               base::JSONWriter::Write(answer_json, &contents);
 516               answer_contents = base::UTF8ToUTF16(contents);
 517             } else {
 518               answer_type_str = base::string16();
 519             }
 520             UMA_HISTOGRAM_BOOLEAN("Omnibox.AnswerParseSuccess",
 521                                   answer_parsed_successfully);
 522           }
 523         }
 524       }
 525
 526       bool should_prefetch = static_cast<int>(index) == prefetch_index;
 527       results->suggest_results.push_back(SuggestResult(
 528           base::CollapseWhitespace(suggestion, false), match_type,
 529           base::CollapseWhitespace(match_contents, false),
 530           match_contents_prefix, annotation, answer_contents, answer_type_str,
 531           answer.Pass(), suggest_query_params, deletion_url, is_keyword_result,
 532           relevance, relevances != NULL, should_prefetch, trimmed_input));
 533     }
 534   }
 535   results->relevances_from_server = relevances != NULL;
 536   return true;
 537 }