chrome/browser/android/contextualsearch/contextual_search_delegate.cc

   1 // Copyright 2015 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/android/contextualsearch/contextual_search_delegate.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/base64.h"
  10 #include "base/command_line.h"
  11 #include "base/json/json_string_value_serializer.h"
  12 #include "base/strings/string_number_conversions.h"
  13 #include "base/strings/string_util.h"
  14 #include "base/strings/utf_string_conversions.h"
  15 #include "chrome/browser/android/proto/client_discourse_context.pb.h"
  16 #include "chrome/browser/profiles/profile.h"
  17 #include "chrome/browser/profiles/profile_manager.h"
  18 #include "chrome/browser/sync/profile_sync_service.h"
  19 #include "chrome/browser/sync/profile_sync_service_factory.h"
  20 #include "components/search_engines/template_url_service.h"
  21 #include "components/variations/net/variations_http_header_provider.h"
  22 #include "components/variations/variations_associated_data.h"
  23 #include "content/public/browser/android/content_view_core.h"
  24 #include "content/public/browser/web_contents.h"
  25 #include "net/base/escape.h"
  26 #include "net/url_request/url_fetcher.h"
  27 #include "url/gurl.h"
  28
  29 using content::ContentViewCore;
  30
  31 namespace {
  32
  33 const char kContextualSearchFieldTrialName[] = "ContextualSearch";
  34 const char kContextualSearchSurroundingSizeParamName[] = "surrounding_size";
  35 const char kContextualSearchIcingSurroundingSizeParamName[] =
  36     "icing_surrounding_size";
  37 const char kContextualSearchResolverURLParamName[] = "resolver_url";
  38 const char kContextualSearchDoNotSendURLParamName[] = "do_not_send_url";
  39 const char kContextualSearchResponseDisplayTextParam[] = "display_text";
  40 const char kContextualSearchResponseSelectedTextParam[] = "selected_text";
  41 const char kContextualSearchResponseSearchTermParam[] = "search_term";
  42 const char kContextualSearchResponseResolvedTermParam[] = "resolved_term";
  43 const char kContextualSearchPreventPreload[] = "prevent_preload";
  44 const char kContextualSearchMentions[] = "mentions";
  45 const char kContextualSearchServerEndpoint[] = "_/contextualsearch?";
  46 const int kContextualSearchRequestVersion = 2;
  47 const char kContextualSearchResolverUrl[] =
  48     "contextual-search-resolver-url";
  49 // The default size of the content surrounding the selection to gather, allowing
  50 // room for other parameters.
  51 const int kContextualSearchDefaultContentSize = 1536;
  52 const int kContextualSearchDefaultIcingSurroundingSize = 400;
  53 const int kContextualSearchMaxSelection = 100;
  54 // The maximum length of a URL to build.
  55 const int kMaxURLSize = 2048;
  56 const char kXssiEscape[] = ")]}'\n";
  57 const char kDiscourseContextHeaderPrefix[] = "X-Additional-Discourse-Context: ";
  58 const char kDoPreventPreloadValue[] = "1";
  59
  60 // The number of characters that should be shown on each side of the selected
  61 // expression.
  62 const int kSurroundingSizeForUI = 30;
  63
  64 } // namespace
  65
  66 // URLFetcher ID, only used for tests: we only have one kind of fetcher.
  67 const int ContextualSearchDelegate::kContextualSearchURLFetcherID = 1;
  68
  69 // Handles tasks for the ContextualSearchManager in a separable, testable way.
  70 ContextualSearchDelegate::ContextualSearchDelegate(
  71     net::URLRequestContextGetter* url_request_context,
  72     TemplateURLService* template_url_service,
  73     const ContextualSearchDelegate::SearchTermResolutionCallback&
  74         search_term_callback,
  75     const ContextualSearchDelegate::SurroundingTextCallback&
  76         surrounding_callback,
  77     const ContextualSearchDelegate::IcingCallback& icing_callback)
  78     : url_request_context_(url_request_context),
  79       template_url_service_(template_url_service),
  80       search_term_callback_(search_term_callback),
  81       surrounding_callback_(surrounding_callback),
  82       icing_callback_(icing_callback) {
  83 }
  84
  85 ContextualSearchDelegate::~ContextualSearchDelegate() {
  86 }
  87
  88 void ContextualSearchDelegate::StartSearchTermResolutionRequest(
  89     const std::string& selection,
  90     bool use_resolved_search_term,
  91     content::ContentViewCore* content_view_core,
  92     bool may_send_base_page_url) {
  93   GatherSurroundingTextWithCallback(
  94       selection, use_resolved_search_term, content_view_core,
  95       may_send_base_page_url,
  96       base::Bind(&ContextualSearchDelegate::StartSearchTermRequestFromSelection,
  97                  AsWeakPtr()));
  98 }
  99
 100 void ContextualSearchDelegate::GatherAndSaveSurroundingText(
 101     const std::string& selection,
 102     bool use_resolved_search_term,
 103     content::ContentViewCore* content_view_core,
 104     bool may_send_base_page_url) {
 105   GatherSurroundingTextWithCallback(
 106       selection, use_resolved_search_term, content_view_core,
 107       may_send_base_page_url,
 108       base::Bind(&ContextualSearchDelegate::SaveSurroundingText, AsWeakPtr()));
 109   // TODO(donnd): clear the context here, since we're done with it (but risky).
 110 }
 111
 112 void ContextualSearchDelegate::ContinueSearchTermResolutionRequest() {
 113   DCHECK(context_.get());
 114   if (!context_.get())
 115     return;
 116   GURL request_url(BuildRequestUrl());
 117   DCHECK(request_url.is_valid());
 118
 119   // Reset will delete any previous fetcher, and we won't get any callback.
 120   search_term_fetcher_.reset(
 121       net::URLFetcher::Create(kContextualSearchURLFetcherID, request_url,
 122                               net::URLFetcher::GET, this).release());
 123   search_term_fetcher_->SetRequestContext(url_request_context_);
 124
 125   // Add Chrome experiment state to the request headers.
 126   net::HttpRequestHeaders headers;
 127   variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
 128       search_term_fetcher_->GetOriginalURL(),
 129       false,  // Impossible to be incognito at this point.
 130       false,
 131       &headers);
 132   search_term_fetcher_->SetExtraRequestHeaders(headers.ToString());
 133
 134   SetDiscourseContextAndAddToHeader(*context_);
 135
 136   search_term_fetcher_->Start();
 137 }
 138
 139 void ContextualSearchDelegate::OnURLFetchComplete(
 140     const net::URLFetcher* source) {
 141   DCHECK(source == search_term_fetcher_.get());
 142   int response_code = source->GetResponseCode();
 143   std::string search_term;
 144   std::string display_text;
 145   std::string alternate_term;
 146   std::string prevent_preload;
 147   int mention_start = 0;
 148   int mention_end = 0;
 149   int start_adjust = 0;
 150   int end_adjust = 0;
 151
 152   if (source->GetStatus().is_success() && response_code == 200) {
 153     std::string response;
 154     bool has_string_response = source->GetResponseAsString(&response);
 155     DCHECK(has_string_response);
 156     if (has_string_response) {
 157       DecodeSearchTermsFromJsonResponse(response, &search_term, &display_text,
 158                                         &alternate_term, &prevent_preload,
 159                                         &mention_start, &mention_end);
 160       if (mention_start != 0 || mention_end != 0) {
 161         // Sanity check that our selection is non-zero and it is less than
 162         // 100 characters as that would make contextual search bar hide.
 163         // We also check that there is at least one character overlap between
 164         // the new and old selection.
 165         if (mention_start >= mention_end
 166             || (mention_end - mention_start) > kContextualSearchMaxSelection
 167             || mention_end <= context_->start_offset
 168             || mention_start >= context_->end_offset) {
 169           start_adjust = 0;
 170           end_adjust = 0;
 171         } else {
 172           start_adjust = mention_start - context_->start_offset;
 173           end_adjust = mention_end - context_->end_offset;
 174         }
 175       }
 176     }
 177   }
 178   bool is_invalid = response_code == net::URLFetcher::RESPONSE_CODE_INVALID;
 179   search_term_callback_.Run(
 180       is_invalid, response_code, search_term, display_text, alternate_term,
 181       prevent_preload == kDoPreventPreloadValue, start_adjust, end_adjust);
 182
 183   // The ContextualSearchContext is consumed once the request has completed.
 184   context_.reset();
 185 }
 186
 187 // TODO(jeremycho): Remove selected_text and base_page_url CGI parameters.
 188 GURL ContextualSearchDelegate::BuildRequestUrl() {
 189   // TODO(jeremycho): Confirm this is the right way to handle TemplateURL fails.
 190   if (!template_url_service_ ||
 191       !template_url_service_->GetDefaultSearchProvider()) {
 192     return GURL();
 193   }
 194
 195   std::string selected_text_escaped(
 196       net::EscapeQueryParamValue(context_->selected_text, true));
 197   std::string base_page_url_escaped(
 198       net::EscapeQueryParamValue(context_->page_url.spec(), true));
 199   bool use_resolved_search_term = context_->use_resolved_search_term;
 200
 201   // If the request is too long, don't include the base-page URL.
 202   std::string request = GetSearchTermResolutionUrlString(
 203       selected_text_escaped, base_page_url_escaped, use_resolved_search_term);
 204   if (request.length() >= kMaxURLSize) {
 205     request = GetSearchTermResolutionUrlString(
 206           selected_text_escaped, "", use_resolved_search_term);
 207   }
 208   return GURL(request);
 209 }
 210
 211 std::string ContextualSearchDelegate::GetSearchTermResolutionUrlString(
 212     const std::string& selected_text,
 213     const std::string& base_page_url,
 214     const bool use_resolved_search_term) {
 215   TemplateURL* template_url = template_url_service_->GetDefaultSearchProvider();
 216
 217   TemplateURLRef::SearchTermsArgs search_terms_args =
 218       TemplateURLRef::SearchTermsArgs(base::string16());
 219
 220   TemplateURLRef::SearchTermsArgs::ContextualSearchParams params(
 221       kContextualSearchRequestVersion,
 222       selected_text,
 223       base_page_url,
 224       use_resolved_search_term);
 225
 226   search_terms_args.contextual_search_params = params;
 227
 228   std::string request(
 229       template_url->contextual_search_url_ref().ReplaceSearchTerms(
 230           search_terms_args,
 231           template_url_service_->search_terms_data(),
 232           NULL));
 233
 234   // The switch/param should be the URL up to and including the endpoint.
 235   std::string replacement_url;
 236   if (base::CommandLine::ForCurrentProcess()->HasSwitch(
 237       kContextualSearchResolverUrl)) {
 238     replacement_url =
 239         base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
 240             kContextualSearchResolverUrl);
 241   } else {
 242     std::string param_value = variations::GetVariationParamValue(
 243         kContextualSearchFieldTrialName, kContextualSearchResolverURLParamName);
 244     if (!param_value.empty()) replacement_url = param_value;
 245   }
 246
 247   // If a replacement URL was specified above, do the substitution.
 248   if (!replacement_url.empty()) {
 249     size_t pos = request.find(kContextualSearchServerEndpoint);
 250     if (pos != std::string::npos) {
 251       request.replace(0, pos + strlen(kContextualSearchServerEndpoint),
 252                       replacement_url);
 253     }
 254   }
 255   return request;
 256 }
 257
 258 void ContextualSearchDelegate::GatherSurroundingTextWithCallback(
 259     const std::string& selection,
 260     bool use_resolved_search_term,
 261     content::ContentViewCore* content_view_core,
 262     bool may_send_base_page_url,
 263     HandleSurroundingsCallback callback) {
 264   // Immediately cancel any request that's in flight, since we're building a new
 265   // context (and the response disposes of any existing context).
 266   search_term_fetcher_.reset();
 267   // Decide if the URL should be sent with the context.
 268   GURL page_url(content_view_core->GetWebContents()->GetURL());
 269   GURL url_to_send;
 270   if (may_send_base_page_url &&
 271       CanSendPageURL(page_url, ProfileManager::GetActiveUserProfile(),
 272                      template_url_service_)) {
 273     url_to_send = page_url;
 274   }
 275   std::string encoding(content_view_core->GetWebContents()->GetEncoding());
 276   context_.reset(new ContextualSearchContext(
 277       selection, use_resolved_search_term, url_to_send, encoding));
 278   content_view_core->RequestTextSurroundingSelection(
 279       GetSearchTermSurroundingSize(), callback);
 280 }
 281
 282 void ContextualSearchDelegate::StartSearchTermRequestFromSelection(
 283     const base::string16& surrounding_text,
 284     int start_offset,
 285     int end_offset) {
 286   // TODO(donnd): figure out how to gather text surrounding the selection
 287   // for other purposes too: e.g. to determine if we should select the
 288   // word where the user tapped.
 289   if (context_.get()) {
 290     SaveSurroundingText(surrounding_text, start_offset, end_offset);
 291     SendSurroundingText(kSurroundingSizeForUI);
 292     ContinueSearchTermResolutionRequest();
 293   } else {
 294     DVLOG(1) << "ctxs: Null context, ignored!";
 295   }
 296 }
 297
 298 void ContextualSearchDelegate::SaveSurroundingText(
 299     const base::string16& surrounding_text,
 300     int start_offset,
 301     int end_offset) {
 302   DCHECK(context_.get());
 303   // Sometimes the surroundings are 0, 0, '', so fall back on the selection.
 304   // See crbug.com/393100.
 305   if (start_offset == 0 && end_offset == 0 && surrounding_text.length() == 0) {
 306     context_->surrounding_text = base::UTF8ToUTF16(context_->selected_text);
 307     context_->start_offset = 0;
 308     context_->end_offset = context_->selected_text.length();
 309   } else {
 310     context_->surrounding_text = surrounding_text;
 311     context_->start_offset = start_offset;
 312     context_->end_offset = end_offset;
 313   }
 314
 315   // Pin the start and end offsets to ensure they point within the string.
 316   int surrounding_length = context_->surrounding_text.length();
 317   context_->start_offset =
 318       std::min(surrounding_length, std::max(0, context_->start_offset));
 319   context_->end_offset =
 320       std::min(surrounding_length, std::max(0, context_->end_offset));
 321
 322   // Call the Icing callback with a shortened copy of the surroundings.
 323   int icing_surrounding_size = GetIcingSurroundingSize();
 324   size_t selection_start = context_->start_offset;
 325   size_t selection_end = context_->end_offset;
 326   if (icing_surrounding_size >= 0 && selection_start < selection_end) {
 327     int icing_padding_each_side = icing_surrounding_size / 2;
 328     base::string16 icing_surrounding_text = SurroundingTextForIcing(
 329         context_->surrounding_text, icing_padding_each_side, &selection_start,
 330         &selection_end);
 331     if (selection_start < selection_end)
 332       icing_callback_.Run(context_->encoding, icing_surrounding_text,
 333                           selection_start, selection_end);
 334   }
 335 }
 336
 337 void ContextualSearchDelegate::SendSurroundingText(int max_surrounding_chars) {
 338   const base::string16& surrounding = context_->surrounding_text;
 339
 340   // Determine the text before the selection.
 341   int num_before_characters =
 342       std::min(context_->start_offset, max_surrounding_chars);
 343   int start_position = context_->start_offset - num_before_characters;
 344   base::string16 before_text =
 345       surrounding.substr(start_position, num_before_characters);
 346
 347   // Determine the text after the selection.
 348   int surrounding_length = surrounding.length();  // Cast to int.
 349   int num_after_characters = std::min(
 350       surrounding_length - context_->end_offset, max_surrounding_chars);
 351   base::string16 after_text = surrounding.substr(
 352       context_->end_offset, num_after_characters);
 353
 354   base::TrimWhitespace(before_text, base::TRIM_ALL, &before_text);
 355   base::TrimWhitespace(after_text, base::TRIM_ALL, &after_text);
 356   surrounding_callback_.Run(UTF16ToUTF8(before_text), UTF16ToUTF8(after_text));
 357 }
 358
 359 void ContextualSearchDelegate::SetDiscourseContextAndAddToHeader(
 360     const ContextualSearchContext& context) {
 361   discourse_context::ClientDiscourseContext proto;
 362   discourse_context::Display* display = proto.add_display();
 363   display->set_uri(context.page_url.spec());
 364
 365   discourse_context::Media* media = display->mutable_media();
 366   media->set_mime_type(context.encoding);
 367
 368   discourse_context::Selection* selection = display->mutable_selection();
 369   selection->set_content(UTF16ToUTF8(context.surrounding_text));
 370   selection->set_start(context.start_offset);
 371   selection->set_end(context.end_offset);
 372   selection->set_is_uri_encoded(false);
 373
 374   std::string serialized;
 375   proto.SerializeToString(&serialized);
 376
 377   std::string encoded_context;
 378   base::Base64Encode(serialized, &encoded_context);
 379   // The server memoizer expects a web-safe encoding.
 380   std::replace(encoded_context.begin(), encoded_context.end(), '+', '-');
 381   std::replace(encoded_context.begin(), encoded_context.end(), '/', '_');
 382   search_term_fetcher_->AddExtraRequestHeader(
 383       kDiscourseContextHeaderPrefix + encoded_context);
 384 }
 385
 386 bool ContextualSearchDelegate::CanSendPageURL(
 387     const GURL& current_page_url,
 388     Profile* profile,
 389     TemplateURLService* template_url_service) {
 390   // Check whether there is a Finch parameter preventing us from sending the
 391   // page URL.
 392   std::string param_value = variations::GetVariationParamValue(
 393       kContextualSearchFieldTrialName, kContextualSearchDoNotSendURLParamName);
 394   if (!param_value.empty())
 395     return false;
 396
 397   // Ensure that the default search provider is Google.
 398   TemplateURL* default_search_provider =
 399       template_url_service->GetDefaultSearchProvider();
 400   bool is_default_search_provider_google =
 401       default_search_provider &&
 402       default_search_provider->url_ref().HasGoogleBaseURLs(
 403           template_url_service->search_terms_data());
 404   if (!is_default_search_provider_google)
 405     return false;
 406
 407   // Only allow HTTP URLs or HTTPS URLs.
 408   if (current_page_url.scheme() != url::kHttpScheme &&
 409       (current_page_url.scheme() != url::kHttpsScheme))
 410     return false;
 411
 412   // Check that the user has sync enabled, is logged in, and syncs their Chrome
 413   // History.
 414   ProfileSyncService* service =
 415       ProfileSyncServiceFactory::GetInstance()->GetForProfile(profile);
 416   sync_driver::SyncPrefs sync_prefs(profile->GetPrefs());
 417   if (service == NULL || !service->CanSyncStart() ||
 418       !sync_prefs.GetPreferredDataTypes(syncer::UserTypes())
 419            .Has(syncer::PROXY_TABS) ||
 420       !service->GetActiveDataTypes().Has(syncer::HISTORY_DELETE_DIRECTIVES)) {
 421     return false;
 422   }
 423
 424   return true;
 425 }
 426
 427 // Decodes the given response from the search term resolution request and sets
 428 // the value of the given parameters.
 429 void ContextualSearchDelegate::DecodeSearchTermsFromJsonResponse(
 430     const std::string& response,
 431     std::string* search_term,
 432     std::string* display_text,
 433     std::string* alternate_term,
 434     std::string* prevent_preload,
 435     int* mention_start,
 436     int* mention_end) {
 437   bool contains_xssi_escape = response.find(kXssiEscape) == 0;
 438   const std::string& proper_json =
 439       contains_xssi_escape ? response.substr(strlen(kXssiEscape)) : response;
 440   JSONStringValueDeserializer deserializer(proper_json);
 441   scoped_ptr<base::Value> root(deserializer.Deserialize(NULL, NULL));
 442
 443   if (root.get() != NULL && root->IsType(base::Value::TYPE_DICTIONARY)) {
 444     base::DictionaryValue* dict =
 445         static_cast<base::DictionaryValue*>(root.get());
 446     dict->GetString(kContextualSearchPreventPreload, prevent_preload);
 447     dict->GetString(kContextualSearchResponseSearchTermParam, search_term);
 448     // For the display_text, if not present fall back to the "search_term".
 449     if (!dict->GetString(kContextualSearchResponseDisplayTextParam,
 450                          display_text)) {
 451       *display_text = *search_term;
 452     }
 453     // Extract mentions for selection expansion.
 454     base::ListValue* mentions_list;
 455     dict->GetList(kContextualSearchMentions, &mentions_list);
 456     if (mentions_list != NULL && mentions_list->GetSize() >= 2)
 457       ExtractMentionsStartEnd(*mentions_list, mention_start, mention_end);
 458     // If either the selected text or the resolved term is not the search term,
 459     // use it as the alternate term.
 460     std::string selected_text;
 461     dict->GetString(kContextualSearchResponseSelectedTextParam, &selected_text);
 462     if (selected_text != *search_term) {
 463       *alternate_term = selected_text;
 464     } else {
 465       std::string resolved_term;
 466       dict->GetString(kContextualSearchResponseResolvedTermParam,
 467                       &resolved_term);
 468       if (resolved_term != *search_term) {
 469         *alternate_term = resolved_term;
 470       }
 471     }
 472   }
 473 }
 474
 475 // Returns the size of the surroundings to be sent to the server for search term
 476 // resolution.
 477 int ContextualSearchDelegate::GetSearchTermSurroundingSize() {
 478   const std::string param_value = variations::GetVariationParamValue(
 479       kContextualSearchFieldTrialName,
 480       kContextualSearchSurroundingSizeParamName);
 481   int param_length;
 482   if (!param_value.empty() && base::StringToInt(param_value, &param_length))
 483     return param_length;
 484   return kContextualSearchDefaultContentSize;
 485 }
 486
 487 // Extract the Start/End of the mentions in the surrounding text
 488 // for selection-expansion.
 489 void ContextualSearchDelegate::ExtractMentionsStartEnd(
 490     const base::ListValue& mentions_list,
 491     int* startResult,
 492     int* endResult) {
 493   int int_value;
 494   if (mentions_list.GetInteger(0, &int_value))
 495     *startResult = std::max(0, int_value);
 496   if (mentions_list.GetInteger(1, &int_value))
 497     *endResult = std::max(0, int_value);
 498 }
 499
 500 // Returns the size of the surroundings to be sent to Icing.
 501 int ContextualSearchDelegate::GetIcingSurroundingSize() {
 502   std::string param_string = variations::GetVariationParamValue(
 503       kContextualSearchFieldTrialName,
 504       kContextualSearchIcingSurroundingSizeParamName);
 505   if (base::CommandLine::ForCurrentProcess()->HasSwitch(
 506           kContextualSearchIcingSurroundingSizeParamName)) {
 507     param_string = base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
 508         kContextualSearchIcingSurroundingSizeParamName);
 509   }
 510   int param_value;
 511   if (!param_string.empty() && base::StringToInt(param_string, &param_value))
 512     return param_value;
 513   return kContextualSearchDefaultIcingSurroundingSize;
 514 }
 515
 516 base::string16 ContextualSearchDelegate::SurroundingTextForIcing(
 517     const base::string16& surrounding_text,
 518     int padding_each_side,
 519     size_t* start,
 520     size_t* end) {
 521   base::string16 result_text = surrounding_text;
 522   size_t start_offset = *start;
 523   size_t end_offset = *end;
 524   size_t padding_each_side_pinned =
 525       padding_each_side >= 0 ? padding_each_side : 0;
 526   // Now trim the context so the portions before or after the selection
 527   // are within the given limit.
 528   if (start_offset > padding_each_side_pinned) {
 529     // Trim the start.
 530     int trim = start_offset - padding_each_side_pinned;
 531     result_text = result_text.substr(trim);
 532     start_offset -= trim;
 533     end_offset -= trim;
 534   }
 535   if (result_text.length() > end_offset + padding_each_side_pinned) {
 536     // Trim the end.
 537     result_text = result_text.substr(0, end_offset + padding_each_side_pinned);
 538   }
 539   *start = start_offset;
 540   *end = end_offset;
 541   return result_text;
 542 }