components/omnibox/search_suggestion_parser.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/omnibox/search_suggestion_parser.h"
   6
   7 #include "base/i18n/icu_string_conversions.h"
   8 #include "base/json/json_string_value_serializer.h"
   9 #include "base/json/json_writer.h"
  10 #include "base/logging.h"
  11 #include "base/strings/string_util.h"
  12 #include "base/strings/utf_string_conversions.h"
  13 #include "base/values.h"
  14 #include "components/omnibox/autocomplete_input.h"
  15 #include "components/omnibox/url_prefix.h"
  16 #include "components/url_fixer/url_fixer.h"
  17 #include "net/base/net_util.h"
  18 #include "net/http/http_response_headers.h"
  19 #include "net/url_request/url_fetcher.h"
  20 #include "url/url_constants.h"
  21
  22 namespace {
  23
  24 AutocompleteMatchType::Type GetAutocompleteMatchType(const std::string& type) {
  25   if (type == "ENTITY")
  26     return AutocompleteMatchType::SEARCH_SUGGEST_ENTITY;
  27   if (type == "INFINITE")
  28     return AutocompleteMatchType::SEARCH_SUGGEST_INFINITE;
  29   if (type == "PERSONALIZED_QUERY")
  30     return AutocompleteMatchType::SEARCH_SUGGEST_PERSONALIZED;
  31   if (type == "PROFILE")
  32     return AutocompleteMatchType::SEARCH_SUGGEST_PROFILE;
  33   if (type == "NAVIGATION")
  34     return AutocompleteMatchType::NAVSUGGEST;
  35   if (type == "PERSONALIZED_NAVIGATION")
  36     return AutocompleteMatchType::NAVSUGGEST_PERSONALIZED;
  37   return AutocompleteMatchType::SEARCH_SUGGEST;
  38 }
  39
  40 }  // namespace
  41
  42 // SearchSuggestionParser::Result ----------------------------------------------
  43
  44 SearchSuggestionParser::Result::Result(bool from_keyword_provider,
  45                                        int relevance,
  46                                        bool relevance_from_server,
  47                                        AutocompleteMatchType::Type type,
  48                                        const std::string& deletion_url)
  49     : from_keyword_provider_(from_keyword_provider),
  50       type_(type),
  51       relevance_(relevance),
  52       relevance_from_server_(relevance_from_server),
  53       deletion_url_(deletion_url) {}
  54
  55 SearchSuggestionParser::Result::~Result() {}
  56
  57 // SearchSuggestionParser::SuggestResult ---------------------------------------
  58
  59 SearchSuggestionParser::SuggestResult::SuggestResult(
  60     const base::string16& suggestion,
  61     AutocompleteMatchType::Type type,
  62     const base::string16& match_contents,
  63     const base::string16& match_contents_prefix,
  64     const base::string16& annotation,
  65     const base::string16& answer_contents,
  66     const base::string16& answer_type,
  67     const std::string& suggest_query_params,
  68     const std::string& deletion_url,
  69     bool from_keyword_provider,
  70     int relevance,
  71     bool relevance_from_server,
  72     bool should_prefetch,
  73     const base::string16& input_text)
  74     : Result(from_keyword_provider,
  75              relevance,
  76              relevance_from_server,
  77              type,
  78              deletion_url),
  79       suggestion_(suggestion),
  80       match_contents_prefix_(match_contents_prefix),
  81       annotation_(annotation),
  82       suggest_query_params_(suggest_query_params),
  83       answer_contents_(answer_contents),
  84       answer_type_(answer_type),
  85       should_prefetch_(should_prefetch) {
  86   match_contents_ = match_contents;
  87   DCHECK(!match_contents_.empty());
  88   ClassifyMatchContents(true, input_text);
  89 }
  90
  91 SearchSuggestionParser::SuggestResult::~SuggestResult() {}
  92
  93 void SearchSuggestionParser::SuggestResult::ClassifyMatchContents(
  94     const bool allow_bolding_all,
  95     const base::string16& input_text) {
  96   if (input_text.empty()) {
  97     // In case of zero-suggest results, do not highlight matches.
  98     match_contents_class_.push_back(
  99         ACMatchClassification(0, ACMatchClassification::NONE));
 100     return;
 101   }
 102
 103   base::string16 lookup_text = input_text;
 104   if (type_ == AutocompleteMatchType::SEARCH_SUGGEST_INFINITE) {
 105     const size_t contents_index =
 106         suggestion_.length() - match_contents_.length();
 107     // Ensure the query starts with the input text, and ends with the match
 108     // contents, and the input text has an overlap with contents.
 109     if (StartsWith(suggestion_, input_text, true) &&
 110         EndsWith(suggestion_, match_contents_, true) &&
 111         (input_text.length() > contents_index)) {
 112       lookup_text = input_text.substr(contents_index);
 113     }
 114   }
 115   size_t lookup_position = match_contents_.find(lookup_text);
 116   if (!allow_bolding_all && (lookup_position == base::string16::npos)) {
 117     // Bail if the code below to update the bolding would bold the whole
 118     // string.  Note that the string may already be entirely bolded; if
 119     // so, leave it as is.
 120     return;
 121   }
 122   match_contents_class_.clear();
 123   // We do intra-string highlighting for suggestions - the suggested segment
 124   // will be highlighted, e.g. for input_text = "you" the suggestion may be
 125   // "youtube", so we'll bold the "tube" section: you*tube*.
 126   if (input_text != match_contents_) {
 127     if (lookup_position == base::string16::npos) {
 128       // The input text is not a substring of the query string, e.g. input
 129       // text is "slasdot" and the query string is "slashdot", so we bold the
 130       // whole thing.
 131       match_contents_class_.push_back(
 132           ACMatchClassification(0, ACMatchClassification::MATCH));
 133     } else {
 134       // We don't iterate over the string here annotating all matches because
 135       // it looks odd to have every occurrence of a substring that may be as
 136       // short as a single character highlighted in a query suggestion result,
 137       // e.g. for input text "s" and query string "southwest airlines", it
 138       // looks odd if both the first and last s are highlighted.
 139       if (lookup_position != 0) {
 140         match_contents_class_.push_back(
 141             ACMatchClassification(0, ACMatchClassification::MATCH));
 142       }
 143       match_contents_class_.push_back(
 144           ACMatchClassification(lookup_position, ACMatchClassification::NONE));
 145       size_t next_fragment_position = lookup_position + lookup_text.length();
 146       if (next_fragment_position < match_contents_.length()) {
 147         match_contents_class_.push_back(ACMatchClassification(
 148             next_fragment_position, ACMatchClassification::MATCH));
 149       }
 150     }
 151   } else {
 152     // Otherwise, match_contents_ is a verbatim (what-you-typed) match, either
 153     // for the default provider or a keyword search provider.
 154     match_contents_class_.push_back(
 155         ACMatchClassification(0, ACMatchClassification::NONE));
 156   }
 157 }
 158
 159 int SearchSuggestionParser::SuggestResult::CalculateRelevance(
 160     const AutocompleteInput& input,
 161     bool keyword_provider_requested) const {
 162   if (!from_keyword_provider_ && keyword_provider_requested)
 163     return 100;
 164   return ((input.type() == metrics::OmniboxInputType::URL) ? 300 : 600);
 165 }
 166
 167 // SearchSuggestionParser::NavigationResult ------------------------------------
 168
 169 SearchSuggestionParser::NavigationResult::NavigationResult(
 170     const AutocompleteSchemeClassifier& scheme_classifier,
 171     const GURL& url,
 172     AutocompleteMatchType::Type type,
 173     const base::string16& description,
 174     const std::string& deletion_url,
 175     bool from_keyword_provider,
 176     int relevance,
 177     bool relevance_from_server,
 178     const base::string16& input_text,
 179     const std::string& languages)
 180     : Result(from_keyword_provider, relevance, relevance_from_server, type,
 181              deletion_url),
 182       url_(url),
 183       formatted_url_(AutocompleteInput::FormattedStringWithEquivalentMeaning(
 184           url, net::FormatUrl(url, languages,
 185                               net::kFormatUrlOmitAll & ~net::kFormatUrlOmitHTTP,
 186                               net::UnescapeRule::SPACES, NULL, NULL, NULL),
 187           scheme_classifier)),
 188       description_(description) {
 189   DCHECK(url_.is_valid());
 190   CalculateAndClassifyMatchContents(true, input_text, languages);
 191 }
 192
 193 SearchSuggestionParser::NavigationResult::~NavigationResult() {}
 194
 195 void
 196 SearchSuggestionParser::NavigationResult::CalculateAndClassifyMatchContents(
 197     const bool allow_bolding_nothing,
 198     const base::string16& input_text,
 199     const std::string& languages) {
 200   if (input_text.empty()) {
 201     // In case of zero-suggest results, do not highlight matches.
 202     match_contents_class_.push_back(
 203         ACMatchClassification(0, ACMatchClassification::NONE));
 204     return;
 205   }
 206
 207   // First look for the user's input inside the formatted url as it would be
 208   // without trimming the scheme, so we can find matches at the beginning of the
 209   // scheme.
 210   const URLPrefix* prefix =
 211       URLPrefix::BestURLPrefix(formatted_url_, input_text);
 212   size_t match_start = (prefix == NULL) ?
 213       formatted_url_.find(input_text) : prefix->prefix.length();
 214   bool trim_http = !AutocompleteInput::HasHTTPScheme(input_text) &&
 215                    (!prefix || (match_start != 0));
 216   const net::FormatUrlTypes format_types =
 217       net::kFormatUrlOmitAll & ~(trim_http ? 0 : net::kFormatUrlOmitHTTP);
 218
 219   base::string16 match_contents = net::FormatUrl(url_, languages, format_types,
 220       net::UnescapeRule::SPACES, NULL, NULL, &match_start);
 221   // If the first match in the untrimmed string was inside a scheme that we
 222   // trimmed, look for a subsequent match.
 223   if (match_start == base::string16::npos)
 224     match_start = match_contents.find(input_text);
 225   // Update |match_contents_| and |match_contents_class_| if it's allowed.
 226   if (allow_bolding_nothing || (match_start != base::string16::npos)) {
 227     match_contents_ = match_contents;
 228     // Safe if |match_start| is npos; also safe if the input is longer than the
 229     // remaining contents after |match_start|.
 230     AutocompleteMatch::ClassifyLocationInString(match_start,
 231         input_text.length(), match_contents_.length(),
 232         ACMatchClassification::URL, &match_contents_class_);
 233   }
 234 }
 235
 236 int SearchSuggestionParser::NavigationResult::CalculateRelevance(
 237     const AutocompleteInput& input,
 238     bool keyword_provider_requested) const {
 239   return (from_keyword_provider_ || !keyword_provider_requested) ? 800 : 150;
 240 }
 241
 242 // SearchSuggestionParser::Results ---------------------------------------------
 243
 244 SearchSuggestionParser::Results::Results()
 245     : verbatim_relevance(-1),
 246       field_trial_triggered(false),
 247       relevances_from_server(false) {}
 248
 249 SearchSuggestionParser::Results::~Results() {}
 250
 251 void SearchSuggestionParser::Results::Clear() {
 252   suggest_results.clear();
 253   navigation_results.clear();
 254   verbatim_relevance = -1;
 255   metadata.clear();
 256 }
 257
 258 bool SearchSuggestionParser::Results::HasServerProvidedScores() const {
 259   if (verbatim_relevance >= 0)
 260     return true;
 261
 262   // Right now either all results of one type will be server-scored or they will
 263   // all be locally scored, but in case we change this later, we'll just check
 264   // them all.
 265   for (SuggestResults::const_iterator i(suggest_results.begin());
 266        i != suggest_results.end(); ++i) {
 267     if (i->relevance_from_server())
 268       return true;
 269   }
 270   for (NavigationResults::const_iterator i(navigation_results.begin());
 271        i != navigation_results.end(); ++i) {
 272     if (i->relevance_from_server())
 273       return true;
 274   }
 275
 276   return false;
 277 }
 278
 279 // SearchSuggestionParser ------------------------------------------------------
 280
 281 // static
 282 std::string SearchSuggestionParser::ExtractJsonData(
 283     const net::URLFetcher* source) {
 284   const net::HttpResponseHeaders* const response_headers =
 285       source->GetResponseHeaders();
 286   std::string json_data;
 287   source->GetResponseAsString(&json_data);
 288
 289   // JSON is supposed to be UTF-8, but some suggest service providers send
 290   // JSON files in non-UTF-8 encodings.  The actual encoding is usually
 291   // specified in the Content-Type header field.
 292   if (response_headers) {
 293     std::string charset;
 294     if (response_headers->GetCharset(&charset)) {
 295       base::string16 data_16;
 296       // TODO(jungshik): Switch to CodePageToUTF8 after it's added.
 297       if (base::CodepageToUTF16(json_data, charset.c_str(),
 298                                 base::OnStringConversionError::FAIL,
 299                                 &data_16))
 300         json_data = base::UTF16ToUTF8(data_16);
 301     }
 302   }
 303   return json_data;
 304 }
 305
 306 // static
 307 scoped_ptr<base::Value> SearchSuggestionParser::DeserializeJsonData(
 308     std::string json_data) {
 309   // The JSON response should be an array.
 310   for (size_t response_start_index = json_data.find("["), i = 0;
 311        response_start_index != std::string::npos && i < 5;
 312        response_start_index = json_data.find("[", 1), i++) {
 313     // Remove any XSSI guards to allow for JSON parsing.
 314     if (response_start_index > 0)
 315       json_data.erase(0, response_start_index);
 316
 317     JSONStringValueSerializer deserializer(json_data);
 318     deserializer.set_allow_trailing_comma(true);
 319     int error_code = 0;
 320     scoped_ptr<base::Value> data(deserializer.Deserialize(&error_code, NULL));
 321     if (error_code == 0)
 322       return data.Pass();
 323   }
 324   return scoped_ptr<base::Value>();
 325 }
 326
 327 // static
 328 bool SearchSuggestionParser::ParseSuggestResults(
 329     const base::Value& root_val,
 330     const AutocompleteInput& input,
 331     const AutocompleteSchemeClassifier& scheme_classifier,
 332     int default_result_relevance,
 333     const std::string& languages,
 334     bool is_keyword_result,
 335     Results* results) {
 336   base::string16 query;
 337   const base::ListValue* root_list = NULL;
 338   const base::ListValue* results_list = NULL;
 339
 340   if (!root_val.GetAsList(&root_list) || !root_list->GetString(0, &query) ||
 341       query != input.text() || !root_list->GetList(1, &results_list))
 342     return false;
 343
 344   // 3rd element: Description list.
 345   const base::ListValue* descriptions = NULL;
 346   root_list->GetList(2, &descriptions);
 347
 348   // 4th element: Disregard the query URL list for now.
 349
 350   // Reset suggested relevance information.
 351   results->verbatim_relevance = -1;
 352
 353   // 5th element: Optional key-value pairs from the Suggest server.
 354   const base::ListValue* types = NULL;
 355   const base::ListValue* relevances = NULL;
 356   const base::ListValue* suggestion_details = NULL;
 357   const base::DictionaryValue* extras = NULL;
 358   int prefetch_index = -1;
 359   if (root_list->GetDictionary(4, &extras)) {
 360     extras->GetList("google:suggesttype", &types);
 361
 362     // Discard this list if its size does not match that of the suggestions.
 363     if (extras->GetList("google:suggestrelevance", &relevances) &&
 364         (relevances->GetSize() != results_list->GetSize()))
 365       relevances = NULL;
 366     extras->GetInteger("google:verbatimrelevance",
 367                        &results->verbatim_relevance);
 368
 369     // Check if the active suggest field trial (if any) has triggered either
 370     // for the default provider or keyword provider.
 371     results->field_trial_triggered = false;
 372     extras->GetBoolean("google:fieldtrialtriggered",
 373                        &results->field_trial_triggered);
 374
 375     const base::DictionaryValue* client_data = NULL;
 376     if (extras->GetDictionary("google:clientdata", &client_data) && client_data)
 377       client_data->GetInteger("phi", &prefetch_index);
 378
 379     if (extras->GetList("google:suggestdetail", &suggestion_details) &&
 380         suggestion_details->GetSize() != results_list->GetSize())
 381       suggestion_details = NULL;
 382
 383     // Store the metadata that came with the response in case we need to pass it
 384     // along with the prefetch query to Instant.
 385     JSONStringValueSerializer json_serializer(&results->metadata);
 386     json_serializer.Serialize(*extras);
 387   }
 388
 389   // Clear the previous results now that new results are available.
 390   results->suggest_results.clear();
 391   results->navigation_results.clear();
 392   results->answers_image_urls.clear();
 393
 394   base::string16 suggestion;
 395   std::string type;
 396   int relevance = default_result_relevance;
 397   // Prohibit navsuggest in FORCED_QUERY mode.  Users wants queries, not URLs.
 398   const bool allow_navsuggest =
 399       input.type() != metrics::OmniboxInputType::FORCED_QUERY;
 400   const base::string16& trimmed_input =
 401       base::CollapseWhitespace(input.text(), false);
 402   for (size_t index = 0; results_list->GetString(index, &suggestion); ++index) {
 403     // Google search may return empty suggestions for weird input characters,
 404     // they make no sense at all and can cause problems in our code.
 405     if (suggestion.empty())
 406       continue;
 407
 408     // Apply valid suggested relevance scores; discard invalid lists.
 409     if (relevances != NULL && !relevances->GetInteger(index, &relevance))
 410       relevances = NULL;
 411     AutocompleteMatchType::Type match_type =
 412         AutocompleteMatchType::SEARCH_SUGGEST;
 413     if (types && types->GetString(index, &type))
 414       match_type = GetAutocompleteMatchType(type);
 415     const base::DictionaryValue* suggestion_detail = NULL;
 416     std::string deletion_url;
 417
 418     if (suggestion_details &&
 419         suggestion_details->GetDictionary(index, &suggestion_detail))
 420       suggestion_detail->GetString("du", &deletion_url);
 421
 422     if ((match_type == AutocompleteMatchType::NAVSUGGEST) ||
 423         (match_type == AutocompleteMatchType::NAVSUGGEST_PERSONALIZED)) {
 424       // Do not blindly trust the URL coming from the server to be valid.
 425       GURL url(
 426           url_fixer::FixupURL(base::UTF16ToUTF8(suggestion), std::string()));
 427       if (url.is_valid() && allow_navsuggest) {
 428         base::string16 title;
 429         if (descriptions != NULL)
 430           descriptions->GetString(index, &title);
 431         results->navigation_results.push_back(NavigationResult(
 432             scheme_classifier, url, match_type, title, deletion_url,
 433             is_keyword_result, relevance, relevances != NULL, input.text(),
 434             languages));
 435       }
 436     } else {
 437       base::string16 match_contents = suggestion;
 438       base::string16 match_contents_prefix;
 439       base::string16 annotation;
 440       base::string16 answer_contents;
 441       base::string16 answer_type;
 442       std::string suggest_query_params;
 443
 444       if (suggestion_details) {
 445         suggestion_details->GetDictionary(index, &suggestion_detail);
 446         if (suggestion_detail) {
 447           suggestion_detail->GetString("t", &match_contents);
 448           suggestion_detail->GetString("mp", &match_contents_prefix);
 449           // Error correction for bad data from server.
 450           if (match_contents.empty())
 451             match_contents = suggestion;
 452           suggestion_detail->GetString("a", &annotation);
 453           suggestion_detail->GetString("q", &suggest_query_params);
 454
 455           // Extract Answers, if provided.
 456           const base::DictionaryValue* answer_json = NULL;
 457           if (suggestion_detail->GetDictionary("ansa", &answer_json)) {
 458             match_type = AutocompleteMatchType::SEARCH_SUGGEST_ANSWER;
 459             GetAnswersImageURLs(answer_json, &results->answers_image_urls);
 460             std::string contents;
 461             base::JSONWriter::Write(answer_json, &contents);
 462             answer_contents = base::UTF8ToUTF16(contents);
 463             suggestion_detail->GetString("ansb", &answer_type);
 464           }
 465         }
 466       }
 467
 468       bool should_prefetch = static_cast<int>(index) == prefetch_index;
 469       // TODO(kochi): Improve calculator suggestion presentation.
 470       results->suggest_results.push_back(SuggestResult(
 471           base::CollapseWhitespace(suggestion, false), match_type,
 472           base::CollapseWhitespace(match_contents, false),
 473           match_contents_prefix, annotation, answer_contents, answer_type,
 474           suggest_query_params, deletion_url, is_keyword_result, relevance,
 475           relevances != NULL, should_prefetch, trimmed_input));
 476     }
 477   }
 478   results->relevances_from_server = relevances != NULL;
 479   return true;
 480 }
 481
 482 // static
 483 void SearchSuggestionParser::GetAnswersImageURLs(
 484     const base::DictionaryValue* answer_json,
 485     std::vector<GURL>* urls) {
 486   DCHECK(answer_json);
 487
 488   const base::ListValue* lines = NULL;
 489   if (!answer_json->GetList("l", &lines) || !lines || lines->GetSize() == 0)
 490     return;
 491
 492   for (base::ListValue::const_iterator iter = lines->begin();
 493        iter != lines->end();
 494        ++iter) {
 495     const base::DictionaryValue* line = NULL;
 496     if (!(*iter)->GetAsDictionary(&line) || !line)
 497       continue;
 498
 499     std::string image_host_and_path;
 500     if (!line->GetString("il.i.d", &image_host_and_path) ||
 501         image_host_and_path.empty())
 502       continue;
 503     // Concatenate scheme and host/path using only ':' as separator. This is
 504     // due to the results delivering strings of the form '//host/path', which
 505     // is web-speak for "use the enclosing page's scheme", but not a valid path
 506     // of an URL.
 507     GURL image_url(
 508         GURL(std::string(url::kHttpsScheme) + ":" + image_host_and_path));
 509     if (image_url.is_valid())
 510       urls->push_back(image_url);
 511   }
 512 }