chrome/browser/android/contextualsearch/contextual_search_delegate.cc

   1 // Copyright 2015 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/android/contextualsearch/contextual_search_delegate.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/base64.h"
  10 #include "base/command_line.h"
  11 #include "base/json/json_string_value_serializer.h"
  12 #include "base/strings/string_number_conversions.h"
  13 #include "base/strings/string_util.h"
  14 #include "base/strings/utf_string_conversions.h"
  15 #include "chrome/browser/android/proto/client_discourse_context.pb.h"
  16 #include "chrome/browser/profiles/profile.h"
  17 #include "chrome/browser/profiles/profile_manager.h"
  18 #include "chrome/browser/sync/profile_sync_service.h"
  19 #include "chrome/browser/sync/profile_sync_service_factory.h"
  20 #include "components/search_engines/template_url_service.h"
  21 #include "components/variations/net/variations_http_header_provider.h"
  22 #include "components/variations/variations_associated_data.h"
  23 #include "content/public/browser/android/content_view_core.h"
  24 #include "content/public/browser/web_contents.h"
  25 #include "net/base/escape.h"
  26 #include "net/url_request/url_fetcher.h"
  27 #include "url/gurl.h"
  28
  29 using content::ContentViewCore;
  30
  31 namespace {
  32
  33 const char kContextualSearchFieldTrialName[] = "ContextualSearch";
  34 const char kContextualSearchSurroundingSizeParamName[] = "surrounding_size";
  35 const char kContextualSearchIcingSurroundingSizeParamName[] =
  36     "icing_surrounding_size";
  37 const char kContextualSearchResolverURLParamName[] = "resolver_url";
  38 const char kContextualSearchDoNotSendURLParamName[] = "do_not_send_url";
  39 const char kContextualSearchResponseDisplayTextParam[] = "display_text";
  40 const char kContextualSearchResponseSelectedTextParam[] = "selected_text";
  41 const char kContextualSearchResponseSearchTermParam[] = "search_term";
  42 const char kContextualSearchResponseResolvedTermParam[] = "resolved_term";
  43 const char kContextualSearchPreventPreload[] = "prevent_preload";
  44 const char kContextualSearchServerEndpoint[] = "_/contextualsearch?";
  45 const int kContextualSearchRequestVersion = 2;
  46 const char kContextualSearchResolverUrl[] =
  47     "contextual-search-resolver-url";
  48 // The default size of the content surrounding the selection to gather, allowing
  49 // room for other parameters.
  50 const int kContextualSearchDefaultContentSize = 1536;
  51 const int kContextualSearchDefaultIcingSurroundingSize = 400;
  52 // The maximum length of a URL to build.
  53 const int kMaxURLSize = 2048;
  54 const char kXssiEscape[] = ")]}'\n";
  55 const char kDiscourseContextHeaderPrefix[] = "X-Additional-Discourse-Context: ";
  56 const char kDoPreventPreloadValue[] = "1";
  57
  58 // The number of characters that should be shown on each side of the selected
  59 // expression.
  60 const int kSurroundingSizeForUI = 30;
  61
  62 } // namespace
  63
  64 // URLFetcher ID, only used for tests: we only have one kind of fetcher.
  65 const int ContextualSearchDelegate::kContextualSearchURLFetcherID = 1;
  66
  67 // Handles tasks for the ContextualSearchManager in a separable, testable way.
  68 ContextualSearchDelegate::ContextualSearchDelegate(
  69     net::URLRequestContextGetter* url_request_context,
  70     TemplateURLService* template_url_service,
  71     const ContextualSearchDelegate::SearchTermResolutionCallback&
  72         search_term_callback,
  73     const ContextualSearchDelegate::SurroundingTextCallback&
  74         surrounding_callback,
  75     const ContextualSearchDelegate::IcingCallback& icing_callback)
  76     : url_request_context_(url_request_context),
  77       template_url_service_(template_url_service),
  78       search_term_callback_(search_term_callback),
  79       surrounding_callback_(surrounding_callback),
  80       icing_callback_(icing_callback) {
  81 }
  82
  83 ContextualSearchDelegate::~ContextualSearchDelegate() {
  84 }
  85
  86 void ContextualSearchDelegate::StartSearchTermResolutionRequest(
  87     const std::string& selection,
  88     bool use_resolved_search_term,
  89     content::ContentViewCore* content_view_core) {
  90   GatherSurroundingTextWithCallback(
  91       selection,
  92       use_resolved_search_term,
  93       content_view_core,
  94       base::Bind(&ContextualSearchDelegate::StartSearchTermRequestFromSelection,
  95                  AsWeakPtr()));
  96 }
  97
  98 void ContextualSearchDelegate::GatherAndSaveSurroundingText(
  99     const std::string& selection,
 100     bool use_resolved_search_term,
 101     content::ContentViewCore* content_view_core) {
 102   GatherSurroundingTextWithCallback(
 103       selection,
 104       use_resolved_search_term,
 105       content_view_core,
 106       base::Bind(&ContextualSearchDelegate::SaveSurroundingText, AsWeakPtr()));
 107   // TODO(donnd): clear the context here, since we're done with it (but risky).
 108 }
 109
 110 void ContextualSearchDelegate::ContinueSearchTermResolutionRequest() {
 111   DCHECK(context_.get());
 112   if (!context_.get())
 113     return;
 114   GURL request_url(BuildRequestUrl());
 115   DCHECK(request_url.is_valid());
 116
 117   // Reset will delete any previous fetcher, and we won't get any callback.
 118   search_term_fetcher_.reset(
 119       net::URLFetcher::Create(kContextualSearchURLFetcherID, request_url,
 120                               net::URLFetcher::GET, this).release());
 121   search_term_fetcher_->SetRequestContext(url_request_context_);
 122
 123   // Add Chrome experiment state to the request headers.
 124   net::HttpRequestHeaders headers;
 125   variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
 126       search_term_fetcher_->GetOriginalURL(),
 127       false,  // Impossible to be incognito at this point.
 128       false,
 129       &headers);
 130   search_term_fetcher_->SetExtraRequestHeaders(headers.ToString());
 131
 132   SetDiscourseContextAndAddToHeader(*context_);
 133
 134   search_term_fetcher_->Start();
 135 }
 136
 137 void ContextualSearchDelegate::OnURLFetchComplete(
 138     const net::URLFetcher* source) {
 139   DCHECK(source == search_term_fetcher_.get());
 140   int response_code = source->GetResponseCode();
 141   std::string search_term;
 142   std::string display_text;
 143   std::string alternate_term;
 144   std::string prevent_preload;
 145
 146   if (source->GetStatus().is_success() && response_code == 200) {
 147     std::string response;
 148     bool has_string_response = source->GetResponseAsString(&response);
 149     DCHECK(has_string_response);
 150     if (has_string_response) {
 151       DecodeSearchTermsFromJsonResponse(response, &search_term, &display_text,
 152                                         &alternate_term, &prevent_preload);
 153     }
 154   }
 155   bool is_invalid = response_code == net::URLFetcher::RESPONSE_CODE_INVALID;
 156   search_term_callback_.Run(
 157       is_invalid, response_code, search_term, display_text, alternate_term,
 158       prevent_preload == kDoPreventPreloadValue);
 159
 160   // The ContextualSearchContext is consumed once the request has completed.
 161   context_.reset();
 162 }
 163
 164 // TODO(jeremycho): Remove selected_text and base_page_url CGI parameters.
 165 GURL ContextualSearchDelegate::BuildRequestUrl() {
 166   // TODO(jeremycho): Confirm this is the right way to handle TemplateURL fails.
 167   if (!template_url_service_ ||
 168       !template_url_service_->GetDefaultSearchProvider()) {
 169     return GURL();
 170   }
 171
 172   std::string selected_text_escaped(
 173       net::EscapeQueryParamValue(context_->selected_text, true));
 174   std::string base_page_url_escaped(
 175       net::EscapeQueryParamValue(context_->page_url.spec(), true));
 176   bool use_resolved_search_term = context_->use_resolved_search_term;
 177
 178   // If the request is too long, don't include the base-page URL.
 179   std::string request = GetSearchTermResolutionUrlString(
 180       selected_text_escaped, base_page_url_escaped, use_resolved_search_term);
 181   if (request.length() >= kMaxURLSize) {
 182     request = GetSearchTermResolutionUrlString(
 183           selected_text_escaped, "", use_resolved_search_term);
 184   }
 185   return GURL(request);
 186 }
 187
 188 std::string ContextualSearchDelegate::GetSearchTermResolutionUrlString(
 189     const std::string& selected_text,
 190     const std::string& base_page_url,
 191     const bool use_resolved_search_term) {
 192   TemplateURL* template_url = template_url_service_->GetDefaultSearchProvider();
 193
 194   TemplateURLRef::SearchTermsArgs search_terms_args =
 195       TemplateURLRef::SearchTermsArgs(base::string16());
 196
 197   TemplateURLRef::SearchTermsArgs::ContextualSearchParams params(
 198       kContextualSearchRequestVersion,
 199       selected_text,
 200       base_page_url,
 201       use_resolved_search_term);
 202
 203   search_terms_args.contextual_search_params = params;
 204
 205   std::string request(
 206       template_url->contextual_search_url_ref().ReplaceSearchTerms(
 207           search_terms_args,
 208           template_url_service_->search_terms_data(),
 209           NULL));
 210
 211   // The switch/param should be the URL up to and including the endpoint.
 212   std::string replacement_url;
 213   if (base::CommandLine::ForCurrentProcess()->HasSwitch(
 214       kContextualSearchResolverUrl)) {
 215     replacement_url =
 216         base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
 217             kContextualSearchResolverUrl);
 218   } else {
 219     std::string param_value = variations::GetVariationParamValue(
 220         kContextualSearchFieldTrialName, kContextualSearchResolverURLParamName);
 221     if (!param_value.empty()) replacement_url = param_value;
 222   }
 223
 224   // If a replacement URL was specified above, do the substitution.
 225   if (!replacement_url.empty()) {
 226     size_t pos = request.find(kContextualSearchServerEndpoint);
 227     if (pos != std::string::npos) {
 228       request.replace(0, pos + strlen(kContextualSearchServerEndpoint),
 229                       replacement_url);
 230     }
 231   }
 232   return request;
 233 }
 234
 235 void ContextualSearchDelegate::GatherSurroundingTextWithCallback(
 236     const std::string& selection,
 237     bool use_resolved_search_term,
 238     content::ContentViewCore* content_view_core,
 239     HandleSurroundingsCallback callback) {
 240   // Immediately cancel any request that's in flight, since we're building a new
 241   // context (and the response disposes of any existing context).
 242   search_term_fetcher_.reset();
 243   // Decide if the URL be sent with the context.
 244   GURL page_url(content_view_core->GetWebContents()->GetURL());
 245   GURL url_to_send;
 246   if (CanSendPageURL(page_url,
 247                      ProfileManager::GetActiveUserProfile(),
 248                      template_url_service_)) {
 249     url_to_send = page_url;
 250   }
 251   std::string encoding(content_view_core->GetWebContents()->GetEncoding());
 252   context_.reset(new ContextualSearchContext(
 253       selection, use_resolved_search_term, url_to_send, encoding));
 254   content_view_core->RequestTextSurroundingSelection(
 255       GetSearchTermSurroundingSize(), callback);
 256 }
 257
 258 void ContextualSearchDelegate::StartSearchTermRequestFromSelection(
 259     const base::string16& surrounding_text,
 260     int start_offset,
 261     int end_offset) {
 262   // TODO(donnd): figure out how to gather text surrounding the selection
 263   // for other purposes too: e.g. to determine if we should select the
 264   // word where the user tapped.
 265   DCHECK(context_.get());
 266   SaveSurroundingText(surrounding_text, start_offset, end_offset);
 267   SendSurroundingText(kSurroundingSizeForUI);
 268   ContinueSearchTermResolutionRequest();
 269 }
 270
 271 void ContextualSearchDelegate::SaveSurroundingText(
 272     const base::string16& surrounding_text,
 273     int start_offset,
 274     int end_offset) {
 275   DCHECK(context_.get());
 276   // Sometimes the surroundings are 0, 0, '', so fall back on the selection.
 277   // See crbug.com/393100.
 278   if (start_offset == 0 && end_offset == 0 && surrounding_text.length() == 0) {
 279     context_->surrounding_text = base::UTF8ToUTF16(context_->selected_text);
 280     context_->start_offset = 0;
 281     context_->end_offset = context_->selected_text.length();
 282   } else {
 283     context_->surrounding_text = surrounding_text;
 284     context_->start_offset = start_offset;
 285     context_->end_offset = end_offset;
 286   }
 287
 288   // Call the Icing callback, unless it has been disabled.
 289   int icing_surrounding_size = GetIcingSurroundingSize();
 290   size_t selection_start = context_->start_offset;
 291   size_t selection_end = context_->end_offset;
 292   if (icing_surrounding_size >= 0 && selection_start < selection_end) {
 293     int icing_padding_each_side = icing_surrounding_size / 2;
 294     base::string16 icing_surrounding_text = SurroundingTextForIcing(
 295         context_->surrounding_text, icing_padding_each_side, &selection_start,
 296         &selection_end);
 297     if (selection_start < selection_end)
 298       icing_callback_.Run(context_->encoding, icing_surrounding_text,
 299                           selection_start, selection_end);
 300   }
 301 }
 302
 303 void ContextualSearchDelegate::SendSurroundingText(int max_surrounding_chars) {
 304   const base::string16 surrounding = context_->surrounding_text;
 305
 306   // Determine the text before the selection.
 307   int start_position = std::max(
 308       0, context_->start_offset - max_surrounding_chars);
 309   int num_before_characters =
 310       std::min(context_->start_offset, max_surrounding_chars);
 311   base::string16 before_text =
 312       surrounding.substr(start_position, num_before_characters);
 313
 314   // Determine the text after the selection.
 315   int surrounding_size = surrounding.size();  // Cast to int.
 316   int num_after_characters = std::min(
 317       surrounding_size - context_->end_offset, max_surrounding_chars);
 318   base::string16 after_text = surrounding.substr(
 319       context_->end_offset, num_after_characters);
 320
 321   base::TrimWhitespace(before_text, base::TRIM_ALL, &before_text);
 322   base::TrimWhitespace(after_text, base::TRIM_ALL, &after_text);
 323   surrounding_callback_.Run(UTF16ToUTF8(before_text), UTF16ToUTF8(after_text));
 324 }
 325
 326 void ContextualSearchDelegate::SetDiscourseContextAndAddToHeader(
 327     const ContextualSearchContext& context) {
 328   discourse_context::ClientDiscourseContext proto;
 329   discourse_context::Display* display = proto.add_display();
 330   display->set_uri(context.page_url.spec());
 331
 332   discourse_context::Media* media = display->mutable_media();
 333   media->set_mime_type(context.encoding);
 334
 335   discourse_context::Selection* selection = display->mutable_selection();
 336   selection->set_content(UTF16ToUTF8(context.surrounding_text));
 337   selection->set_start(context.start_offset);
 338   selection->set_end(context.end_offset);
 339   selection->set_is_uri_encoded(false);
 340
 341   std::string serialized;
 342   proto.SerializeToString(&serialized);
 343
 344   std::string encoded_context;
 345   base::Base64Encode(serialized, &encoded_context);
 346   // The server memoizer expects a web-safe encoding.
 347   std::replace(encoded_context.begin(), encoded_context.end(), '+', '-');
 348   std::replace(encoded_context.begin(), encoded_context.end(), '/', '_');
 349   search_term_fetcher_->AddExtraRequestHeader(
 350       kDiscourseContextHeaderPrefix + encoded_context);
 351 }
 352
 353 bool ContextualSearchDelegate::CanSendPageURL(
 354     const GURL& current_page_url,
 355     Profile* profile,
 356     TemplateURLService* template_url_service) {
 357   // Check whether there is a Finch parameter preventing us from sending the
 358   // page URL.
 359   std::string param_value = variations::GetVariationParamValue(
 360       kContextualSearchFieldTrialName, kContextualSearchDoNotSendURLParamName);
 361   if (!param_value.empty())
 362     return false;
 363
 364   // Ensure that the default search provider is Google.
 365   TemplateURL* default_search_provider =
 366       template_url_service->GetDefaultSearchProvider();
 367   bool is_default_search_provider_google =
 368       default_search_provider &&
 369       default_search_provider->url_ref().HasGoogleBaseURLs(
 370           template_url_service->search_terms_data());
 371   if (!is_default_search_provider_google)
 372     return false;
 373
 374   // Only allow HTTP URLs or HTTPS URLs.
 375   if (current_page_url.scheme() != url::kHttpScheme &&
 376       (current_page_url.scheme() != url::kHttpsScheme))
 377     return false;
 378
 379   // Check that the user has sync enabled, is logged in, and syncs their Chrome
 380   // History.
 381   ProfileSyncService* service =
 382       ProfileSyncServiceFactory::GetInstance()->GetForProfile(profile);
 383   sync_driver::SyncPrefs sync_prefs(profile->GetPrefs());
 384   if (service == NULL || !service->IsSyncEnabledAndLoggedIn() ||
 385       !sync_prefs.GetPreferredDataTypes(syncer::UserTypes())
 386            .Has(syncer::PROXY_TABS) ||
 387       !service->GetActiveDataTypes().Has(syncer::HISTORY_DELETE_DIRECTIVES)) {
 388     return false;
 389   }
 390
 391   return true;
 392 }
 393
 394 // Decodes the given response from the search term resolution request and sets
 395 // the value of the given parameters.
 396 void ContextualSearchDelegate::DecodeSearchTermsFromJsonResponse(
 397     const std::string& response,
 398     std::string* search_term,
 399     std::string* display_text,
 400     std::string* alternate_term,
 401     std::string* prevent_preload) {
 402   bool contains_xssi_escape = response.find(kXssiEscape) == 0;
 403   const std::string& proper_json =
 404       contains_xssi_escape ? response.substr(strlen(kXssiEscape)) : response;
 405   JSONStringValueDeserializer deserializer(proper_json);
 406   scoped_ptr<base::Value> root(deserializer.Deserialize(NULL, NULL));
 407
 408   if (root.get() != NULL && root->IsType(base::Value::TYPE_DICTIONARY)) {
 409     base::DictionaryValue* dict =
 410         static_cast<base::DictionaryValue*>(root.get());
 411     dict->GetString(kContextualSearchPreventPreload, prevent_preload);
 412     dict->GetString(kContextualSearchResponseSearchTermParam, search_term);
 413     // For the display_text, if not present fall back to the "search_term".
 414     if (!dict->GetString(kContextualSearchResponseDisplayTextParam,
 415                          display_text)) {
 416       *display_text = *search_term;
 417     }
 418     // If either the selected text or the resolved term is not the search term,
 419     // use it as the alternate term.
 420     std::string selected_text;
 421     dict->GetString(kContextualSearchResponseSelectedTextParam, &selected_text);
 422     if (selected_text != *search_term) {
 423       *alternate_term = selected_text;
 424     } else {
 425       std::string resolved_term;
 426       dict->GetString(kContextualSearchResponseResolvedTermParam,
 427                       &resolved_term);
 428       if (resolved_term != *search_term) {
 429         *alternate_term = resolved_term;
 430       }
 431     }
 432   }
 433 }
 434
 435 // Returns the size of the surroundings to be sent to the server for search term
 436 // resolution.
 437 int ContextualSearchDelegate::GetSearchTermSurroundingSize() {
 438   const std::string param_value = variations::GetVariationParamValue(
 439       kContextualSearchFieldTrialName,
 440       kContextualSearchSurroundingSizeParamName);
 441   int param_length;
 442   if (!param_value.empty() && base::StringToInt(param_value, &param_length))
 443     return param_length;
 444   return kContextualSearchDefaultContentSize;
 445 }
 446
 447 // Returns the size of the surroundings to be sent to Icing.
 448 int ContextualSearchDelegate::GetIcingSurroundingSize() {
 449   std::string param_string = variations::GetVariationParamValue(
 450       kContextualSearchFieldTrialName,
 451       kContextualSearchIcingSurroundingSizeParamName);
 452   if (base::CommandLine::ForCurrentProcess()->HasSwitch(
 453           kContextualSearchIcingSurroundingSizeParamName)) {
 454     param_string = base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
 455         kContextualSearchIcingSurroundingSizeParamName);
 456   }
 457   int param_value;
 458   if (!param_string.empty() && base::StringToInt(param_string, &param_value))
 459     return param_value;
 460   return kContextualSearchDefaultIcingSurroundingSize;
 461 }
 462
 463 base::string16 ContextualSearchDelegate::SurroundingTextForIcing(
 464     const base::string16& surrounding_text,
 465     int padding_each_side,
 466     size_t* start,
 467     size_t* end) {
 468   base::string16 result_text = surrounding_text;
 469   size_t start_offset = *start;
 470   size_t end_offset = *end;
 471   size_t padding_each_side_pinned =
 472       padding_each_side >= 0 ? padding_each_side : 0;
 473   // Now trim the context so the portions before or after the selection
 474   // are within the given limit.
 475   if (start_offset > padding_each_side_pinned) {
 476     // Trim the start.
 477     int trim = start_offset - padding_each_side_pinned;
 478     result_text = result_text.substr(trim);
 479     start_offset -= trim;
 480     end_offset -= trim;
 481   }
 482   if (result_text.length() > end_offset + padding_each_side_pinned) {
 483     // Trim the end.
 484     result_text = result_text.substr(0, end_offset + padding_each_side_pinned);
 485   }
 486   *start = start_offset;
 487   *end = end_offset;
 488   return result_text;
 489 }