Clean up extension confirmation prompts and make them consistent between Views and...
[chromium-blink-merge.git] / components / omnibox / search_suggestion_parser.cc
blob5ef420174f23c6de062a356a087f7116b1e2b2ef
1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "components/omnibox/search_suggestion_parser.h"
7 #include <algorithm>
9 #include "base/i18n/icu_string_conversions.h"
10 #include "base/json/json_string_value_serializer.h"
11 #include "base/json/json_writer.h"
12 #include "base/logging.h"
13 #include "base/metrics/histogram.h"
14 #include "base/strings/string_number_conversions.h"
15 #include "base/strings/string_util.h"
16 #include "base/strings/utf_string_conversions.h"
17 #include "base/values.h"
18 #include "components/omnibox/autocomplete_input.h"
19 #include "components/omnibox/url_prefix.h"
20 #include "components/url_fixer/url_fixer.h"
21 #include "net/base/net_util.h"
22 #include "net/http/http_response_headers.h"
23 #include "net/url_request/url_fetcher.h"
24 #include "url/url_constants.h"
26 namespace {
28 AutocompleteMatchType::Type GetAutocompleteMatchType(const std::string& type) {
29 if (type == "CALCULATOR")
30 return AutocompleteMatchType::CALCULATOR;
31 if (type == "ENTITY")
32 return AutocompleteMatchType::SEARCH_SUGGEST_ENTITY;
33 if (type == "TAIL")
34 return AutocompleteMatchType::SEARCH_SUGGEST_TAIL;
35 if (type == "PERSONALIZED_QUERY")
36 return AutocompleteMatchType::SEARCH_SUGGEST_PERSONALIZED;
37 if (type == "PROFILE")
38 return AutocompleteMatchType::SEARCH_SUGGEST_PROFILE;
39 if (type == "NAVIGATION")
40 return AutocompleteMatchType::NAVSUGGEST;
41 if (type == "PERSONALIZED_NAVIGATION")
42 return AutocompleteMatchType::NAVSUGGEST_PERSONALIZED;
43 return AutocompleteMatchType::SEARCH_SUGGEST;
46 } // namespace
48 // SearchSuggestionParser::Result ----------------------------------------------
50 SearchSuggestionParser::Result::Result(bool from_keyword_provider,
51 int relevance,
52 bool relevance_from_server,
53 AutocompleteMatchType::Type type,
54 const std::string& deletion_url)
55 : from_keyword_provider_(from_keyword_provider),
56 type_(type),
57 relevance_(relevance),
58 relevance_from_server_(relevance_from_server),
59 received_after_last_keystroke_(true),
60 deletion_url_(deletion_url) {}
62 SearchSuggestionParser::Result::~Result() {}
64 // SearchSuggestionParser::SuggestResult ---------------------------------------
66 SearchSuggestionParser::SuggestResult::SuggestResult(
67 const base::string16& suggestion,
68 AutocompleteMatchType::Type type,
69 const base::string16& match_contents,
70 const base::string16& match_contents_prefix,
71 const base::string16& annotation,
72 const base::string16& answer_contents,
73 const base::string16& answer_type,
74 scoped_ptr<SuggestionAnswer> answer,
75 const std::string& suggest_query_params,
76 const std::string& deletion_url,
77 bool from_keyword_provider,
78 int relevance,
79 bool relevance_from_server,
80 bool should_prefetch,
81 const base::string16& input_text)
82 : Result(from_keyword_provider,
83 relevance,
84 relevance_from_server,
85 type,
86 deletion_url),
87 suggestion_(suggestion),
88 match_contents_prefix_(match_contents_prefix),
89 annotation_(annotation),
90 suggest_query_params_(suggest_query_params),
91 answer_contents_(answer_contents),
92 answer_type_(answer_type),
93 answer_(answer.Pass()),
94 should_prefetch_(should_prefetch) {
95 match_contents_ = match_contents;
96 DCHECK(!match_contents_.empty());
97 ClassifyMatchContents(true, input_text);
100 SearchSuggestionParser::SuggestResult::SuggestResult(
101 const SuggestResult& result)
102 : Result(result),
103 suggestion_(result.suggestion_),
104 match_contents_prefix_(result.match_contents_prefix_),
105 annotation_(result.annotation_),
106 suggest_query_params_(result.suggest_query_params_),
107 answer_contents_(result.answer_contents_),
108 answer_type_(result.answer_type_),
109 answer_(SuggestionAnswer::copy(result.answer_.get())),
110 should_prefetch_(result.should_prefetch_) {
113 SearchSuggestionParser::SuggestResult::~SuggestResult() {}
115 SearchSuggestionParser::SuggestResult&
116 SearchSuggestionParser::SuggestResult::operator=(const SuggestResult& rhs) {
117 if (this == &rhs)
118 return *this;
120 // Assign via parent class first.
121 Result::operator=(rhs);
123 suggestion_ = rhs.suggestion_;
124 match_contents_prefix_ = rhs.match_contents_prefix_;
125 annotation_ = rhs.annotation_;
126 suggest_query_params_ = rhs.suggest_query_params_;
127 answer_contents_ = rhs.answer_contents_;
128 answer_type_ = rhs.answer_type_;
129 answer_ = SuggestionAnswer::copy(rhs.answer_.get());
130 should_prefetch_ = rhs.should_prefetch_;
132 return *this;
135 void SearchSuggestionParser::SuggestResult::ClassifyMatchContents(
136 const bool allow_bolding_all,
137 const base::string16& input_text) {
138 if (input_text.empty()) {
139 // In case of zero-suggest results, do not highlight matches.
140 match_contents_class_.push_back(
141 ACMatchClassification(0, ACMatchClassification::NONE));
142 return;
145 base::string16 lookup_text = input_text;
146 if (type_ == AutocompleteMatchType::SEARCH_SUGGEST_TAIL) {
147 const size_t contents_index =
148 suggestion_.length() - match_contents_.length();
149 // Ensure the query starts with the input text, and ends with the match
150 // contents, and the input text has an overlap with contents.
151 if (base::StartsWith(suggestion_, input_text, true) &&
152 base::EndsWith(suggestion_, match_contents_, true) &&
153 (input_text.length() > contents_index)) {
154 lookup_text = input_text.substr(contents_index);
157 // Do a case-insensitive search for |lookup_text|.
158 base::string16::const_iterator lookup_position = std::search(
159 match_contents_.begin(), match_contents_.end(), lookup_text.begin(),
160 lookup_text.end(), base::CaseInsensitiveCompare<base::char16>());
161 if (!allow_bolding_all && (lookup_position == match_contents_.end())) {
162 // Bail if the code below to update the bolding would bold the whole
163 // string. Note that the string may already be entirely bolded; if
164 // so, leave it as is.
165 return;
167 match_contents_class_.clear();
168 // We do intra-string highlighting for suggestions - the suggested segment
169 // will be highlighted, e.g. for input_text = "you" the suggestion may be
170 // "youtube", so we'll bold the "tube" section: you*tube*.
171 if (input_text != match_contents_) {
172 if (lookup_position == match_contents_.end()) {
173 // The input text is not a substring of the query string, e.g. input
174 // text is "slasdot" and the query string is "slashdot", so we bold the
175 // whole thing.
176 match_contents_class_.push_back(
177 ACMatchClassification(0, ACMatchClassification::MATCH));
178 } else {
179 // We don't iterate over the string here annotating all matches because
180 // it looks odd to have every occurrence of a substring that may be as
181 // short as a single character highlighted in a query suggestion result,
182 // e.g. for input text "s" and query string "southwest airlines", it
183 // looks odd if both the first and last s are highlighted.
184 const size_t lookup_index = lookup_position - match_contents_.begin();
185 if (lookup_index != 0) {
186 match_contents_class_.push_back(
187 ACMatchClassification(0, ACMatchClassification::MATCH));
189 match_contents_class_.push_back(
190 ACMatchClassification(lookup_index, ACMatchClassification::NONE));
191 size_t next_fragment_position = lookup_index + lookup_text.length();
192 if (next_fragment_position < match_contents_.length()) {
193 match_contents_class_.push_back(ACMatchClassification(
194 next_fragment_position, ACMatchClassification::MATCH));
197 } else {
198 // Otherwise, match_contents_ is a verbatim (what-you-typed) match, either
199 // for the default provider or a keyword search provider.
200 match_contents_class_.push_back(
201 ACMatchClassification(0, ACMatchClassification::NONE));
205 int SearchSuggestionParser::SuggestResult::CalculateRelevance(
206 const AutocompleteInput& input,
207 bool keyword_provider_requested) const {
208 if (!from_keyword_provider_ && keyword_provider_requested)
209 return 100;
210 return ((input.type() == metrics::OmniboxInputType::URL) ? 300 : 600);
213 // SearchSuggestionParser::NavigationResult ------------------------------------
215 SearchSuggestionParser::NavigationResult::NavigationResult(
216 const AutocompleteSchemeClassifier& scheme_classifier,
217 const GURL& url,
218 AutocompleteMatchType::Type type,
219 const base::string16& description,
220 const std::string& deletion_url,
221 bool from_keyword_provider,
222 int relevance,
223 bool relevance_from_server,
224 const base::string16& input_text,
225 const std::string& languages)
226 : Result(from_keyword_provider, relevance, relevance_from_server, type,
227 deletion_url),
228 url_(url),
229 formatted_url_(AutocompleteInput::FormattedStringWithEquivalentMeaning(
230 url, net::FormatUrl(url, languages,
231 net::kFormatUrlOmitAll & ~net::kFormatUrlOmitHTTP,
232 net::UnescapeRule::SPACES, NULL, NULL, NULL),
233 scheme_classifier)),
234 description_(description) {
235 DCHECK(url_.is_valid());
236 CalculateAndClassifyMatchContents(true, input_text, languages);
239 SearchSuggestionParser::NavigationResult::~NavigationResult() {}
241 void
242 SearchSuggestionParser::NavigationResult::CalculateAndClassifyMatchContents(
243 const bool allow_bolding_nothing,
244 const base::string16& input_text,
245 const std::string& languages) {
246 if (input_text.empty()) {
247 // In case of zero-suggest results, do not highlight matches.
248 match_contents_class_.push_back(
249 ACMatchClassification(0, ACMatchClassification::NONE));
250 return;
253 // First look for the user's input inside the formatted url as it would be
254 // without trimming the scheme, so we can find matches at the beginning of the
255 // scheme.
256 const URLPrefix* prefix =
257 URLPrefix::BestURLPrefix(formatted_url_, input_text);
258 size_t match_start = (prefix == NULL) ?
259 formatted_url_.find(input_text) : prefix->prefix.length();
260 bool trim_http = !AutocompleteInput::HasHTTPScheme(input_text) &&
261 (!prefix || (match_start != 0));
262 const net::FormatUrlTypes format_types =
263 net::kFormatUrlOmitAll & ~(trim_http ? 0 : net::kFormatUrlOmitHTTP);
265 base::string16 match_contents = net::FormatUrl(url_, languages, format_types,
266 net::UnescapeRule::SPACES, NULL, NULL, &match_start);
267 // If the first match in the untrimmed string was inside a scheme that we
268 // trimmed, look for a subsequent match.
269 if (match_start == base::string16::npos)
270 match_start = match_contents.find(input_text);
271 // Update |match_contents_| and |match_contents_class_| if it's allowed.
272 if (allow_bolding_nothing || (match_start != base::string16::npos)) {
273 match_contents_ = match_contents;
274 // Safe if |match_start| is npos; also safe if the input is longer than the
275 // remaining contents after |match_start|.
276 AutocompleteMatch::ClassifyLocationInString(match_start,
277 input_text.length(), match_contents_.length(),
278 ACMatchClassification::URL, &match_contents_class_);
282 int SearchSuggestionParser::NavigationResult::CalculateRelevance(
283 const AutocompleteInput& input,
284 bool keyword_provider_requested) const {
285 return (from_keyword_provider_ || !keyword_provider_requested) ? 800 : 150;
288 // SearchSuggestionParser::Results ---------------------------------------------
290 SearchSuggestionParser::Results::Results()
291 : verbatim_relevance(-1),
292 field_trial_triggered(false),
293 relevances_from_server(false) {}
295 SearchSuggestionParser::Results::~Results() {}
297 void SearchSuggestionParser::Results::Clear() {
298 suggest_results.clear();
299 navigation_results.clear();
300 verbatim_relevance = -1;
301 metadata.clear();
304 bool SearchSuggestionParser::Results::HasServerProvidedScores() const {
305 if (verbatim_relevance >= 0)
306 return true;
308 // Right now either all results of one type will be server-scored or they will
309 // all be locally scored, but in case we change this later, we'll just check
310 // them all.
311 for (SuggestResults::const_iterator i(suggest_results.begin());
312 i != suggest_results.end(); ++i) {
313 if (i->relevance_from_server())
314 return true;
316 for (NavigationResults::const_iterator i(navigation_results.begin());
317 i != navigation_results.end(); ++i) {
318 if (i->relevance_from_server())
319 return true;
322 return false;
325 // SearchSuggestionParser ------------------------------------------------------
327 // static
328 std::string SearchSuggestionParser::ExtractJsonData(
329 const net::URLFetcher* source) {
330 const net::HttpResponseHeaders* const response_headers =
331 source->GetResponseHeaders();
332 std::string json_data;
333 source->GetResponseAsString(&json_data);
335 // JSON is supposed to be UTF-8, but some suggest service providers send
336 // JSON files in non-UTF-8 encodings. The actual encoding is usually
337 // specified in the Content-Type header field.
338 if (response_headers) {
339 std::string charset;
340 if (response_headers->GetCharset(&charset)) {
341 base::string16 data_16;
342 // TODO(jungshik): Switch to CodePageToUTF8 after it's added.
343 if (base::CodepageToUTF16(json_data, charset.c_str(),
344 base::OnStringConversionError::FAIL,
345 &data_16))
346 json_data = base::UTF16ToUTF8(data_16);
349 return json_data;
352 // static
353 scoped_ptr<base::Value> SearchSuggestionParser::DeserializeJsonData(
354 base::StringPiece json_data) {
355 // The JSON response should be an array.
356 for (size_t response_start_index = json_data.find("["), i = 0;
357 response_start_index != base::StringPiece::npos && i < 5;
358 response_start_index = json_data.find("[", 1), i++) {
359 // Remove any XSSI guards to allow for JSON parsing.
360 json_data.remove_prefix(response_start_index);
362 JSONStringValueDeserializer deserializer(json_data);
363 deserializer.set_allow_trailing_comma(true);
364 int error_code = 0;
365 scoped_ptr<base::Value> data(deserializer.Deserialize(&error_code, NULL));
366 if (error_code == 0)
367 return data.Pass();
369 return scoped_ptr<base::Value>();
372 // static
373 bool SearchSuggestionParser::ParseSuggestResults(
374 const base::Value& root_val,
375 const AutocompleteInput& input,
376 const AutocompleteSchemeClassifier& scheme_classifier,
377 int default_result_relevance,
378 const std::string& languages,
379 bool is_keyword_result,
380 Results* results) {
381 base::string16 query;
382 const base::ListValue* root_list = NULL;
383 const base::ListValue* results_list = NULL;
385 if (!root_val.GetAsList(&root_list) || !root_list->GetString(0, &query) ||
386 query != input.text() || !root_list->GetList(1, &results_list))
387 return false;
389 // 3rd element: Description list.
390 const base::ListValue* descriptions = NULL;
391 root_list->GetList(2, &descriptions);
393 // 4th element: Disregard the query URL list for now.
395 // Reset suggested relevance information.
396 results->verbatim_relevance = -1;
398 // 5th element: Optional key-value pairs from the Suggest server.
399 const base::ListValue* types = NULL;
400 const base::ListValue* relevances = NULL;
401 const base::ListValue* suggestion_details = NULL;
402 const base::DictionaryValue* extras = NULL;
403 int prefetch_index = -1;
404 if (root_list->GetDictionary(4, &extras)) {
405 extras->GetList("google:suggesttype", &types);
407 // Discard this list if its size does not match that of the suggestions.
408 if (extras->GetList("google:suggestrelevance", &relevances) &&
409 (relevances->GetSize() != results_list->GetSize()))
410 relevances = NULL;
411 extras->GetInteger("google:verbatimrelevance",
412 &results->verbatim_relevance);
414 // Check if the active suggest field trial (if any) has triggered either
415 // for the default provider or keyword provider.
416 results->field_trial_triggered = false;
417 extras->GetBoolean("google:fieldtrialtriggered",
418 &results->field_trial_triggered);
420 const base::DictionaryValue* client_data = NULL;
421 if (extras->GetDictionary("google:clientdata", &client_data) && client_data)
422 client_data->GetInteger("phi", &prefetch_index);
424 if (extras->GetList("google:suggestdetail", &suggestion_details) &&
425 suggestion_details->GetSize() != results_list->GetSize())
426 suggestion_details = NULL;
428 // Store the metadata that came with the response in case we need to pass it
429 // along with the prefetch query to Instant.
430 JSONStringValueSerializer json_serializer(&results->metadata);
431 json_serializer.Serialize(*extras);
434 // Clear the previous results now that new results are available.
435 results->suggest_results.clear();
436 results->navigation_results.clear();
437 results->answers_image_urls.clear();
439 base::string16 suggestion;
440 std::string type;
441 int relevance = default_result_relevance;
442 // Prohibit navsuggest in FORCED_QUERY mode. Users wants queries, not URLs.
443 const bool allow_navsuggest =
444 input.type() != metrics::OmniboxInputType::FORCED_QUERY;
445 const base::string16& trimmed_input =
446 base::CollapseWhitespace(input.text(), false);
447 for (size_t index = 0; results_list->GetString(index, &suggestion); ++index) {
448 // Google search may return empty suggestions for weird input characters,
449 // they make no sense at all and can cause problems in our code.
450 if (suggestion.empty())
451 continue;
453 // Apply valid suggested relevance scores; discard invalid lists.
454 if (relevances != NULL && !relevances->GetInteger(index, &relevance))
455 relevances = NULL;
456 AutocompleteMatchType::Type match_type =
457 AutocompleteMatchType::SEARCH_SUGGEST;
458 if (types && types->GetString(index, &type))
459 match_type = GetAutocompleteMatchType(type);
460 const base::DictionaryValue* suggestion_detail = NULL;
461 std::string deletion_url;
463 if (suggestion_details &&
464 suggestion_details->GetDictionary(index, &suggestion_detail))
465 suggestion_detail->GetString("du", &deletion_url);
467 if ((match_type == AutocompleteMatchType::NAVSUGGEST) ||
468 (match_type == AutocompleteMatchType::NAVSUGGEST_PERSONALIZED)) {
469 // Do not blindly trust the URL coming from the server to be valid.
470 GURL url(
471 url_fixer::FixupURL(base::UTF16ToUTF8(suggestion), std::string()));
472 if (url.is_valid() && allow_navsuggest) {
473 base::string16 title;
474 if (descriptions != NULL)
475 descriptions->GetString(index, &title);
476 results->navigation_results.push_back(NavigationResult(
477 scheme_classifier, url, match_type, title, deletion_url,
478 is_keyword_result, relevance, relevances != NULL, input.text(),
479 languages));
481 } else {
482 // TODO(dschuyler) If the "= " is no longer sent from the back-end
483 // then this may be removed.
484 if ((match_type == AutocompleteMatchType::CALCULATOR) &&
485 !suggestion.compare(0, 2, base::UTF8ToUTF16("= ")))
486 suggestion.erase(0, 2);
488 base::string16 match_contents = suggestion;
489 base::string16 match_contents_prefix;
490 base::string16 annotation;
491 base::string16 answer_contents;
492 base::string16 answer_type_str;
493 scoped_ptr<SuggestionAnswer> answer;
494 std::string suggest_query_params;
496 if (suggestion_details) {
497 suggestion_details->GetDictionary(index, &suggestion_detail);
498 if (suggestion_detail) {
499 suggestion_detail->GetString("t", &match_contents);
500 suggestion_detail->GetString("mp", &match_contents_prefix);
501 // Error correction for bad data from server.
502 if (match_contents.empty())
503 match_contents = suggestion;
504 suggestion_detail->GetString("a", &annotation);
505 suggestion_detail->GetString("q", &suggest_query_params);
507 // Extract the Answer, if provided.
508 const base::DictionaryValue* answer_json = NULL;
509 if (suggestion_detail->GetDictionary("ansa", &answer_json) &&
510 suggestion_detail->GetString("ansb", &answer_type_str)) {
511 bool answer_parsed_successfully = false;
512 answer = SuggestionAnswer::ParseAnswer(answer_json);
513 int answer_type = 0;
514 if (answer && base::StringToInt(answer_type_str, &answer_type)) {
515 answer_parsed_successfully = true;
517 answer->set_type(answer_type);
518 answer->AddImageURLsTo(&results->answers_image_urls);
520 std::string contents;
521 base::JSONWriter::Write(*answer_json, &contents);
522 answer_contents = base::UTF8ToUTF16(contents);
523 } else {
524 answer_type_str = base::string16();
526 UMA_HISTOGRAM_BOOLEAN("Omnibox.AnswerParseSuccess",
527 answer_parsed_successfully);
532 bool should_prefetch = static_cast<int>(index) == prefetch_index;
533 results->suggest_results.push_back(SuggestResult(
534 base::CollapseWhitespace(suggestion, false), match_type,
535 base::CollapseWhitespace(match_contents, false),
536 match_contents_prefix, annotation, answer_contents, answer_type_str,
537 answer.Pass(), suggest_query_params, deletion_url, is_keyword_result,
538 relevance, relevances != NULL, should_prefetch, trimmed_input));
541 results->relevances_from_server = relevances != NULL;
542 return true;