chrome/browser/android/contextualsearch/contextual_search_delegate.cc

   1 // Copyright 2015 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/android/contextualsearch/contextual_search_delegate.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/base64.h"
  10 #include "base/command_line.h"
  11 #include "base/json/json_string_value_serializer.h"
  12 #include "base/strings/string_number_conversions.h"
  13 #include "base/strings/string_util.h"
  14 #include "base/strings/utf_string_conversions.h"
  15 #include "chrome/browser/android/proto/client_discourse_context.pb.h"
  16 #include "chrome/browser/profiles/profile.h"
  17 #include "chrome/browser/profiles/profile_manager.h"
  18 #include "chrome/browser/sync/profile_sync_service.h"
  19 #include "chrome/browser/sync/profile_sync_service_factory.h"
  20 #include "components/search_engines/template_url_service.h"
  21 #include "components/variations/net/variations_http_header_provider.h"
  22 #include "components/variations/variations_associated_data.h"
  23 #include "content/public/browser/android/content_view_core.h"
  24 #include "content/public/browser/web_contents.h"
  25 #include "net/base/escape.h"
  26 #include "net/url_request/url_fetcher.h"
  27 #include "url/gurl.h"
  28
  29 using content::ContentViewCore;
  30
  31 namespace {
  32
  33 const char kContextualSearchFieldTrialName[] = "ContextualSearch";
  34 const char kContextualSearchSurroundingSizeParamName[] = "surrounding_size";
  35 const char kContextualSearchIcingSurroundingSizeParamName[] =
  36     "icing_surrounding_size";
  37 const char kContextualSearchResolverURLParamName[] = "resolver_url";
  38 const char kContextualSearchDoNotSendURLParamName[] = "do_not_send_url";
  39 const char kContextualSearchResponseDisplayTextParam[] = "display_text";
  40 const char kContextualSearchResponseSelectedTextParam[] = "selected_text";
  41 const char kContextualSearchResponseSearchTermParam[] = "search_term";
  42 const char kContextualSearchResponseResolvedTermParam[] = "resolved_term";
  43 const char kContextualSearchPreventPreload[] = "prevent_preload";
  44 const char kContextualSearchMentions[] = "mentions";
  45 const char kContextualSearchServerEndpoint[] = "_/contextualsearch?";
  46 const int kContextualSearchRequestVersion = 2;
  47 const char kContextualSearchResolverUrl[] =
  48     "contextual-search-resolver-url";
  49 // The default size of the content surrounding the selection to gather, allowing
  50 // room for other parameters.
  51 const int kContextualSearchDefaultContentSize = 1536;
  52 const int kContextualSearchDefaultIcingSurroundingSize = 400;
  53 const int kContextualSearchMaxSelection = 100;
  54 // The maximum length of a URL to build.
  55 const int kMaxURLSize = 2048;
  56 const char kXssiEscape[] = ")]}'\n";
  57 const char kDiscourseContextHeaderPrefix[] = "X-Additional-Discourse-Context: ";
  58 const char kDoPreventPreloadValue[] = "1";
  59
  60 // The number of characters that should be shown on each side of the selected
  61 // expression.
  62 const int kSurroundingSizeForUI = 30;
  63
  64 } // namespace
  65
  66 // URLFetcher ID, only used for tests: we only have one kind of fetcher.
  67 const int ContextualSearchDelegate::kContextualSearchURLFetcherID = 1;
  68
  69 // Handles tasks for the ContextualSearchManager in a separable, testable way.
  70 ContextualSearchDelegate::ContextualSearchDelegate(
  71     net::URLRequestContextGetter* url_request_context,
  72     TemplateURLService* template_url_service,
  73     const ContextualSearchDelegate::SearchTermResolutionCallback&
  74         search_term_callback,
  75     const ContextualSearchDelegate::SurroundingTextCallback&
  76         surrounding_callback,
  77     const ContextualSearchDelegate::IcingCallback& icing_callback)
  78     : url_request_context_(url_request_context),
  79       template_url_service_(template_url_service),
  80       search_term_callback_(search_term_callback),
  81       surrounding_callback_(surrounding_callback),
  82       icing_callback_(icing_callback) {
  83 }
  84
  85 ContextualSearchDelegate::~ContextualSearchDelegate() {
  86 }
  87
  88 void ContextualSearchDelegate::StartSearchTermResolutionRequest(
  89     const std::string& selection,
  90     bool use_resolved_search_term,
  91     content::ContentViewCore* content_view_core,
  92     bool may_send_base_page_url) {
  93   GatherSurroundingTextWithCallback(
  94       selection, use_resolved_search_term, content_view_core,
  95       may_send_base_page_url,
  96       base::Bind(&ContextualSearchDelegate::StartSearchTermRequestFromSelection,
  97                  AsWeakPtr()));
  98 }
  99
 100 void ContextualSearchDelegate::GatherAndSaveSurroundingText(
 101     const std::string& selection,
 102     bool use_resolved_search_term,
 103     content::ContentViewCore* content_view_core,
 104     bool may_send_base_page_url) {
 105   GatherSurroundingTextWithCallback(
 106       selection, use_resolved_search_term, content_view_core,
 107       may_send_base_page_url,
 108       base::Bind(&ContextualSearchDelegate::SaveSurroundingText, AsWeakPtr()));
 109   // TODO(donnd): clear the context here, since we're done with it (but risky).
 110 }
 111
 112 void ContextualSearchDelegate::ContinueSearchTermResolutionRequest() {
 113   DCHECK(context_.get());
 114   if (!context_.get())
 115     return;
 116   GURL request_url(BuildRequestUrl());
 117   DCHECK(request_url.is_valid());
 118
 119   // Reset will delete any previous fetcher, and we won't get any callback.
 120   search_term_fetcher_.reset(
 121       net::URLFetcher::Create(kContextualSearchURLFetcherID, request_url,
 122                               net::URLFetcher::GET, this).release());
 123   search_term_fetcher_->SetRequestContext(url_request_context_);
 124
 125   // Add Chrome experiment state to the request headers.
 126   net::HttpRequestHeaders headers;
 127   variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
 128       search_term_fetcher_->GetOriginalURL(),
 129       false,  // Impossible to be incognito at this point.
 130       false,
 131       &headers);
 132   search_term_fetcher_->SetExtraRequestHeaders(headers.ToString());
 133
 134   SetDiscourseContextAndAddToHeader(*context_);
 135
 136   search_term_fetcher_->Start();
 137 }
 138
 139 void ContextualSearchDelegate::OnURLFetchComplete(
 140     const net::URLFetcher* source) {
 141   DCHECK(source == search_term_fetcher_.get());
 142   int response_code = source->GetResponseCode();
 143   std::string search_term;
 144   std::string display_text;
 145   std::string alternate_term;
 146   std::string prevent_preload;
 147   int mention_start = 0;
 148   int mention_end = 0;
 149   int start_adjust = 0;
 150   int end_adjust = 0;
 151
 152   if (source->GetStatus().is_success() && response_code == 200) {
 153     std::string response;
 154     bool has_string_response = source->GetResponseAsString(&response);
 155     DCHECK(has_string_response);
 156     if (has_string_response) {
 157       DecodeSearchTermsFromJsonResponse(response, &search_term, &display_text,
 158                                         &alternate_term, &prevent_preload,
 159                                         &mention_start, &mention_end);
 160       if (mention_start != 0 || mention_end != 0) {
 161         // Sanity check that our selection is non-zero and it is less than
 162         // 100 characters as that would make contextual search bar hide.
 163         // We also check that there is at least one character overlap between
 164         // the new and old selection.
 165         if (mention_start >= mention_end
 166             || (mention_end - mention_start) > kContextualSearchMaxSelection
 167             || mention_end <= context_->start_offset
 168             || mention_start >= context_->end_offset) {
 169           start_adjust = 0;
 170           end_adjust = 0;
 171         } else {
 172           start_adjust = mention_start - context_->start_offset;
 173           end_adjust = mention_end - context_->end_offset;
 174         }
 175       }
 176     }
 177   }
 178   bool is_invalid = response_code == net::URLFetcher::RESPONSE_CODE_INVALID;
 179   search_term_callback_.Run(
 180       is_invalid, response_code, search_term, display_text, alternate_term,
 181       prevent_preload == kDoPreventPreloadValue, start_adjust, end_adjust);
 182
 183   // The ContextualSearchContext is consumed once the request has completed.
 184   context_.reset();
 185 }
 186
 187 // TODO(jeremycho): Remove selected_text and base_page_url CGI parameters.
 188 GURL ContextualSearchDelegate::BuildRequestUrl() {
 189   // TODO(jeremycho): Confirm this is the right way to handle TemplateURL fails.
 190   if (!template_url_service_ ||
 191       !template_url_service_->GetDefaultSearchProvider()) {
 192     return GURL();
 193   }
 194
 195   std::string selected_text_escaped(
 196       net::EscapeQueryParamValue(context_->selected_text, true));
 197   std::string base_page_url_escaped(
 198       net::EscapeQueryParamValue(context_->page_url.spec(), true));
 199   bool use_resolved_search_term = context_->use_resolved_search_term;
 200
 201   // If the request is too long, don't include the base-page URL.
 202   std::string request = GetSearchTermResolutionUrlString(
 203       selected_text_escaped, base_page_url_escaped, use_resolved_search_term);
 204   if (request.length() >= kMaxURLSize) {
 205     request = GetSearchTermResolutionUrlString(
 206           selected_text_escaped, "", use_resolved_search_term);
 207   }
 208   return GURL(request);
 209 }
 210
 211 std::string ContextualSearchDelegate::GetSearchTermResolutionUrlString(
 212     const std::string& selected_text,
 213     const std::string& base_page_url,
 214     const bool use_resolved_search_term) {
 215   TemplateURL* template_url = template_url_service_->GetDefaultSearchProvider();
 216
 217   TemplateURLRef::SearchTermsArgs search_terms_args =
 218       TemplateURLRef::SearchTermsArgs(base::string16());
 219
 220   TemplateURLRef::SearchTermsArgs::ContextualSearchParams params(
 221       kContextualSearchRequestVersion,
 222       selected_text,
 223       base_page_url,
 224       use_resolved_search_term);
 225
 226   search_terms_args.contextual_search_params = params;
 227
 228   std::string request(
 229       template_url->contextual_search_url_ref().ReplaceSearchTerms(
 230           search_terms_args,
 231           template_url_service_->search_terms_data(),
 232           NULL));
 233
 234   // The switch/param should be the URL up to and including the endpoint.
 235   std::string replacement_url;
 236   if (base::CommandLine::ForCurrentProcess()->HasSwitch(
 237       kContextualSearchResolverUrl)) {
 238     replacement_url =
 239         base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
 240             kContextualSearchResolverUrl);
 241   } else {
 242     std::string param_value = variations::GetVariationParamValue(
 243         kContextualSearchFieldTrialName, kContextualSearchResolverURLParamName);
 244     if (!param_value.empty()) replacement_url = param_value;
 245   }
 246
 247   // If a replacement URL was specified above, do the substitution.
 248   if (!replacement_url.empty()) {
 249     size_t pos = request.find(kContextualSearchServerEndpoint);
 250     if (pos != std::string::npos) {
 251       request.replace(0, pos + strlen(kContextualSearchServerEndpoint),
 252                       replacement_url);
 253     }
 254   }
 255   return request;
 256 }
 257
 258 void ContextualSearchDelegate::GatherSurroundingTextWithCallback(
 259     const std::string& selection,
 260     bool use_resolved_search_term,
 261     content::ContentViewCore* content_view_core,
 262     bool may_send_base_page_url,
 263     HandleSurroundingsCallback callback) {
 264   // Immediately cancel any request that's in flight, since we're building a new
 265   // context (and the response disposes of any existing context).
 266   search_term_fetcher_.reset();
 267   // Decide if the URL should be sent with the context.
 268   GURL page_url(content_view_core->GetWebContents()->GetURL());
 269   GURL url_to_send;
 270   if (may_send_base_page_url &&
 271       CanSendPageURL(page_url, ProfileManager::GetActiveUserProfile(),
 272                      template_url_service_)) {
 273     url_to_send = page_url;
 274   }
 275   std::string encoding(content_view_core->GetWebContents()->GetEncoding());
 276   context_.reset(new ContextualSearchContext(
 277       selection, use_resolved_search_term, url_to_send, encoding));
 278   content_view_core->RequestTextSurroundingSelection(
 279       GetSearchTermSurroundingSize(), callback);
 280 }
 281
 282 void ContextualSearchDelegate::StartSearchTermRequestFromSelection(
 283     const base::string16& surrounding_text,
 284     int start_offset,
 285     int end_offset) {
 286   // TODO(donnd): figure out how to gather text surrounding the selection
 287   // for other purposes too: e.g. to determine if we should select the
 288   // word where the user tapped.
 289   DCHECK(context_.get());
 290   SaveSurroundingText(surrounding_text, start_offset, end_offset);
 291   SendSurroundingText(kSurroundingSizeForUI);
 292   ContinueSearchTermResolutionRequest();
 293 }
 294
 295 void ContextualSearchDelegate::SaveSurroundingText(
 296     const base::string16& surrounding_text,
 297     int start_offset,
 298     int end_offset) {
 299   DCHECK(context_.get());
 300   // Sometimes the surroundings are 0, 0, '', so fall back on the selection.
 301   // See crbug.com/393100.
 302   if (start_offset == 0 && end_offset == 0 && surrounding_text.length() == 0) {
 303     context_->surrounding_text = base::UTF8ToUTF16(context_->selected_text);
 304     context_->start_offset = 0;
 305     context_->end_offset = context_->selected_text.length();
 306   } else {
 307     context_->surrounding_text = surrounding_text;
 308     context_->start_offset = start_offset;
 309     context_->end_offset = end_offset;
 310   }
 311
 312   // Call the Icing callback, unless it has been disabled.
 313   int icing_surrounding_size = GetIcingSurroundingSize();
 314   size_t selection_start = context_->start_offset;
 315   size_t selection_end = context_->end_offset;
 316   if (icing_surrounding_size >= 0 && selection_start < selection_end) {
 317     int icing_padding_each_side = icing_surrounding_size / 2;
 318     base::string16 icing_surrounding_text = SurroundingTextForIcing(
 319         context_->surrounding_text, icing_padding_each_side, &selection_start,
 320         &selection_end);
 321     if (selection_start < selection_end)
 322       icing_callback_.Run(context_->encoding, icing_surrounding_text,
 323                           selection_start, selection_end);
 324   }
 325 }
 326
 327 void ContextualSearchDelegate::SendSurroundingText(int max_surrounding_chars) {
 328   const base::string16 surrounding = context_->surrounding_text;
 329
 330   // Determine the text before the selection.
 331   int start_position = std::max(
 332       0, context_->start_offset - max_surrounding_chars);
 333   int num_before_characters =
 334       std::min(context_->start_offset, max_surrounding_chars);
 335   base::string16 before_text =
 336       surrounding.substr(start_position, num_before_characters);
 337
 338   // Determine the text after the selection.
 339   int surrounding_size = surrounding.size();  // Cast to int.
 340   int num_after_characters = std::min(
 341       surrounding_size - context_->end_offset, max_surrounding_chars);
 342   base::string16 after_text = surrounding.substr(
 343       context_->end_offset, num_after_characters);
 344
 345   base::TrimWhitespace(before_text, base::TRIM_ALL, &before_text);
 346   base::TrimWhitespace(after_text, base::TRIM_ALL, &after_text);
 347   surrounding_callback_.Run(UTF16ToUTF8(before_text), UTF16ToUTF8(after_text));
 348 }
 349
 350 void ContextualSearchDelegate::SetDiscourseContextAndAddToHeader(
 351     const ContextualSearchContext& context) {
 352   discourse_context::ClientDiscourseContext proto;
 353   discourse_context::Display* display = proto.add_display();
 354   display->set_uri(context.page_url.spec());
 355
 356   discourse_context::Media* media = display->mutable_media();
 357   media->set_mime_type(context.encoding);
 358
 359   discourse_context::Selection* selection = display->mutable_selection();
 360   selection->set_content(UTF16ToUTF8(context.surrounding_text));
 361   selection->set_start(context.start_offset);
 362   selection->set_end(context.end_offset);
 363   selection->set_is_uri_encoded(false);
 364
 365   std::string serialized;
 366   proto.SerializeToString(&serialized);
 367
 368   std::string encoded_context;
 369   base::Base64Encode(serialized, &encoded_context);
 370   // The server memoizer expects a web-safe encoding.
 371   std::replace(encoded_context.begin(), encoded_context.end(), '+', '-');
 372   std::replace(encoded_context.begin(), encoded_context.end(), '/', '_');
 373   search_term_fetcher_->AddExtraRequestHeader(
 374       kDiscourseContextHeaderPrefix + encoded_context);
 375 }
 376
 377 bool ContextualSearchDelegate::CanSendPageURL(
 378     const GURL& current_page_url,
 379     Profile* profile,
 380     TemplateURLService* template_url_service) {
 381   // Check whether there is a Finch parameter preventing us from sending the
 382   // page URL.
 383   std::string param_value = variations::GetVariationParamValue(
 384       kContextualSearchFieldTrialName, kContextualSearchDoNotSendURLParamName);
 385   if (!param_value.empty())
 386     return false;
 387
 388   // Ensure that the default search provider is Google.
 389   TemplateURL* default_search_provider =
 390       template_url_service->GetDefaultSearchProvider();
 391   bool is_default_search_provider_google =
 392       default_search_provider &&
 393       default_search_provider->url_ref().HasGoogleBaseURLs(
 394           template_url_service->search_terms_data());
 395   if (!is_default_search_provider_google)
 396     return false;
 397
 398   // Only allow HTTP URLs or HTTPS URLs.
 399   if (current_page_url.scheme() != url::kHttpScheme &&
 400       (current_page_url.scheme() != url::kHttpsScheme))
 401     return false;
 402
 403   // Check that the user has sync enabled, is logged in, and syncs their Chrome
 404   // History.
 405   ProfileSyncService* service =
 406       ProfileSyncServiceFactory::GetInstance()->GetForProfile(profile);
 407   sync_driver::SyncPrefs sync_prefs(profile->GetPrefs());
 408   if (service == NULL || !service->CanSyncStart() ||
 409       !sync_prefs.GetPreferredDataTypes(syncer::UserTypes())
 410            .Has(syncer::PROXY_TABS) ||
 411       !service->GetActiveDataTypes().Has(syncer::HISTORY_DELETE_DIRECTIVES)) {
 412     return false;
 413   }
 414
 415   return true;
 416 }
 417
 418 // Decodes the given response from the search term resolution request and sets
 419 // the value of the given parameters.
 420 void ContextualSearchDelegate::DecodeSearchTermsFromJsonResponse(
 421     const std::string& response,
 422     std::string* search_term,
 423     std::string* display_text,
 424     std::string* alternate_term,
 425     std::string* prevent_preload,
 426     int* mention_start,
 427     int* mention_end) {
 428   bool contains_xssi_escape = response.find(kXssiEscape) == 0;
 429   const std::string& proper_json =
 430       contains_xssi_escape ? response.substr(strlen(kXssiEscape)) : response;
 431   JSONStringValueDeserializer deserializer(proper_json);
 432   scoped_ptr<base::Value> root(deserializer.Deserialize(NULL, NULL));
 433
 434   if (root.get() != NULL && root->IsType(base::Value::TYPE_DICTIONARY)) {
 435     base::DictionaryValue* dict =
 436         static_cast<base::DictionaryValue*>(root.get());
 437     dict->GetString(kContextualSearchPreventPreload, prevent_preload);
 438     dict->GetString(kContextualSearchResponseSearchTermParam, search_term);
 439     // For the display_text, if not present fall back to the "search_term".
 440     if (!dict->GetString(kContextualSearchResponseDisplayTextParam,
 441                          display_text)) {
 442       *display_text = *search_term;
 443     }
 444     // Extract mentions for selection expansion.
 445     base::ListValue* mentions_list;
 446     dict->GetList(kContextualSearchMentions, &mentions_list);
 447     if (mentions_list != NULL && mentions_list->GetSize() >= 2)
 448       ExtractMentionsStartEnd(*mentions_list, mention_start, mention_end);
 449     // If either the selected text or the resolved term is not the search term,
 450     // use it as the alternate term.
 451     std::string selected_text;
 452     dict->GetString(kContextualSearchResponseSelectedTextParam, &selected_text);
 453     if (selected_text != *search_term) {
 454       *alternate_term = selected_text;
 455     } else {
 456       std::string resolved_term;
 457       dict->GetString(kContextualSearchResponseResolvedTermParam,
 458                       &resolved_term);
 459       if (resolved_term != *search_term) {
 460         *alternate_term = resolved_term;
 461       }
 462     }
 463   }
 464 }
 465
 466 // Returns the size of the surroundings to be sent to the server for search term
 467 // resolution.
 468 int ContextualSearchDelegate::GetSearchTermSurroundingSize() {
 469   const std::string param_value = variations::GetVariationParamValue(
 470       kContextualSearchFieldTrialName,
 471       kContextualSearchSurroundingSizeParamName);
 472   int param_length;
 473   if (!param_value.empty() && base::StringToInt(param_value, &param_length))
 474     return param_length;
 475   return kContextualSearchDefaultContentSize;
 476 }
 477
 478 // Extract the Start/End of the mentions in the surrounding text
 479 // for selection-expansion.
 480 void ContextualSearchDelegate::ExtractMentionsStartEnd(
 481     const base::ListValue& mentions_list,
 482     int* startResult,
 483     int* endResult) {
 484   int int_value;
 485   if (mentions_list.GetInteger(0, &int_value))
 486     *startResult = std::max(0, int_value);
 487   if (mentions_list.GetInteger(1, &int_value))
 488     *endResult = std::max(0, int_value);
 489 }
 490
 491 // Returns the size of the surroundings to be sent to Icing.
 492 int ContextualSearchDelegate::GetIcingSurroundingSize() {
 493   std::string param_string = variations::GetVariationParamValue(
 494       kContextualSearchFieldTrialName,
 495       kContextualSearchIcingSurroundingSizeParamName);
 496   if (base::CommandLine::ForCurrentProcess()->HasSwitch(
 497           kContextualSearchIcingSurroundingSizeParamName)) {
 498     param_string = base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
 499         kContextualSearchIcingSurroundingSizeParamName);
 500   }
 501   int param_value;
 502   if (!param_string.empty() && base::StringToInt(param_string, &param_value))
 503     return param_value;
 504   return kContextualSearchDefaultIcingSurroundingSize;
 505 }
 506
 507 base::string16 ContextualSearchDelegate::SurroundingTextForIcing(
 508     const base::string16& surrounding_text,
 509     int padding_each_side,
 510     size_t* start,
 511     size_t* end) {
 512   base::string16 result_text = surrounding_text;
 513   size_t start_offset = *start;
 514   size_t end_offset = *end;
 515   size_t padding_each_side_pinned =
 516       padding_each_side >= 0 ? padding_each_side : 0;
 517   // Now trim the context so the portions before or after the selection
 518   // are within the given limit.
 519   if (start_offset > padding_each_side_pinned) {
 520     // Trim the start.
 521     int trim = start_offset - padding_each_side_pinned;
 522     result_text = result_text.substr(trim);
 523     start_offset -= trim;
 524     end_offset -= trim;
 525   }
 526   if (result_text.length() > end_offset + padding_each_side_pinned) {
 527     // Trim the end.
 528     result_text = result_text.substr(0, end_offset + padding_each_side_pinned);
 529   }
 530   *start = start_offset;
 531   *end = end_offset;
 532   return result_text;
 533 }