components/search_engines/template_url.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/search_engines/template_url.h"
   6
   7 #include <string>
   8 #include <vector>
   9
  10 #include "base/basictypes.h"
  11 #include "base/command_line.h"
  12 #include "base/format_macros.h"
  13 #include "base/i18n/icu_string_conversions.h"
  14 #include "base/i18n/rtl.h"
  15 #include "base/logging.h"
  16 #include "base/metrics/field_trial.h"
  17 #include "base/rand_util.h"
  18 #include "base/strings/string_number_conversions.h"
  19 #include "base/strings/string_piece.h"
  20 #include "base/strings/string_split.h"
  21 #include "base/strings/string_util.h"
  22 #include "base/strings/stringprintf.h"
  23 #include "base/strings/utf_string_conversions.h"
  24 #include "components/google/core/browser/google_util.h"
  25 #include "components/metrics/proto/omnibox_input_type.pb.h"
  26 #include "components/search_engines/search_engines_switches.h"
  27 #include "components/search_engines/search_terms_data.h"
  28 #include "google_apis/google_api_keys.h"
  29 #include "net/base/escape.h"
  30 #include "net/base/mime_util.h"
  31 #include "net/base/net_util.h"
  32 #include "ui/base/device_form_factor.h"
  33 #include "url/gurl.h"
  34
  35 namespace {
  36
  37 // The TemplateURLRef has any number of terms that need to be replaced. Each of
  38 // the terms is enclosed in braces. If the character preceeding the final
  39 // brace is a ?, it indicates the term is optional and can be replaced with
  40 // an empty string.
  41 const char kStartParameter = '{';
  42 const char kEndParameter = '}';
  43 const char kOptional = '?';
  44
  45 // Known parameters found in the URL.
  46 const char kSearchTermsParameter[] = "searchTerms";
  47 const char kSearchTermsParameterFull[] = "{searchTerms}";
  48 const char kSearchTermsParameterFullEscaped[] = "%7BsearchTerms%7D";
  49
  50 // Same as kSearchTermsParameter, with no escaping.
  51 const char kGoogleUnescapedSearchTermsParameter[] =
  52     "google:unescapedSearchTerms";
  53 const char kGoogleUnescapedSearchTermsParameterFull[] =
  54     "{google:unescapedSearchTerms}";
  55
  56 // Display value for kSearchTermsParameter.
  57 const char kDisplaySearchTerms[] = "%s";
  58
  59 // Display value for kGoogleUnescapedSearchTermsParameter.
  60 const char kDisplayUnescapedSearchTerms[] = "%S";
  61
  62 // Used if the count parameter is not optional. Indicates we want 10 search
  63 // results.
  64 const char kDefaultCount[] = "10";
  65
  66 // Used if the output encoding parameter is required.
  67 const char kOutputEncodingType[] = "UTF-8";
  68
  69 // Attempts to encode |terms| and |original_query| in |encoding| and escape
  70 // them.  |terms| may be escaped as path or query depending on |is_in_query|;
  71 // |original_query| is always escaped as query.  Returns whether the encoding
  72 // process succeeded.
  73 bool TryEncoding(const base::string16& terms,
  74                  const base::string16& original_query,
  75                  const char* encoding,
  76                  bool is_in_query,
  77                  base::string16* escaped_terms,
  78                  base::string16* escaped_original_query) {
  79   DCHECK(escaped_terms);
  80   DCHECK(escaped_original_query);
  81   std::string encoded_terms;
  82   if (!base::UTF16ToCodepage(terms, encoding,
  83       base::OnStringConversionError::SKIP, &encoded_terms))
  84     return false;
  85   *escaped_terms = base::UTF8ToUTF16(is_in_query ?
  86       net::EscapeQueryParamValue(encoded_terms, true) :
  87       net::EscapePath(encoded_terms));
  88   if (original_query.empty())
  89     return true;
  90   std::string encoded_original_query;
  91   if (!base::UTF16ToCodepage(original_query, encoding,
  92       base::OnStringConversionError::SKIP, &encoded_original_query))
  93     return false;
  94   *escaped_original_query = base::UTF8ToUTF16(
  95       net::EscapeQueryParamValue(encoded_original_query, true));
  96   return true;
  97 }
  98
  99 // Extract query key and host given a list of parameters coming from the URL
 100 // query or ref.
 101 std::string FindSearchTermsKey(const std::string& params) {
 102   if (params.empty())
 103     return std::string();
 104   url::Component query, key, value;
 105   query.len = static_cast<int>(params.size());
 106   while (url::ExtractQueryKeyValue(params.c_str(), &query, &key, &value)) {
 107     if (key.is_nonempty() && value.is_nonempty()) {
 108       const base::StringPiece value_string(params.c_str() + value.begin,
 109                                            value.len);
 110       if (value_string.find(kSearchTermsParameterFull, 0) !=
 111           base::StringPiece::npos ||
 112           value_string.find(kGoogleUnescapedSearchTermsParameterFull, 0) !=
 113           base::StringPiece::npos) {
 114         return params.substr(key.begin, key.len);
 115       }
 116     }
 117   }
 118   return std::string();
 119 }
 120
 121 // Extract the position of the search terms' parameter in the URL path.
 122 bool FindSearchTermsInPath(const std::string& path,
 123                            url::Component* parameter_position) {
 124   DCHECK(parameter_position);
 125   parameter_position->reset();
 126   const size_t begin = path.find(kSearchTermsParameterFullEscaped);
 127   if (begin == std::string::npos)
 128     return false;
 129   parameter_position->begin = begin;
 130   parameter_position->len = arraysize(kSearchTermsParameterFullEscaped) - 1;
 131   return true;
 132 }
 133
 134 bool IsTemplateParameterString(const std::string& param) {
 135   return (param.length() > 2) && (*(param.begin()) == kStartParameter) &&
 136       (*(param.rbegin()) == kEndParameter);
 137 }
 138
 139 }  // namespace
 140
 141
 142 // TemplateURLRef::SearchTermsArgs --------------------------------------------
 143
 144 TemplateURLRef::SearchTermsArgs::SearchTermsArgs(
 145     const base::string16& search_terms)
 146     : search_terms(search_terms),
 147       input_type(metrics::OmniboxInputType::INVALID),
 148       accepted_suggestion(NO_SUGGESTIONS_AVAILABLE),
 149       cursor_position(base::string16::npos),
 150       enable_omnibox_start_margin(false),
 151       page_classification(metrics::OmniboxEventProto::INVALID_SPEC),
 152       bookmark_bar_pinned(false),
 153       append_extra_query_params(false),
 154       force_instant_results(false),
 155       from_app_list(false),
 156       contextual_search_params(ContextualSearchParams()) {
 157 }
 158
 159 TemplateURLRef::SearchTermsArgs::~SearchTermsArgs() {
 160 }
 161
 162 TemplateURLRef::SearchTermsArgs::ContextualSearchParams::
 163     ContextualSearchParams()
 164     : version(-1),
 165       start(base::string16::npos),
 166       end(base::string16::npos),
 167       resolve(true) {
 168 }
 169
 170 TemplateURLRef::SearchTermsArgs::ContextualSearchParams::
 171     ContextualSearchParams(
 172         const int version,
 173         const std::string& selection,
 174         const std::string& base_page_url,
 175         const bool resolve)
 176     : version(version),
 177       start(base::string16::npos),
 178       end(base::string16::npos),
 179       selection(selection),
 180       base_page_url(base_page_url),
 181       resolve(resolve) {
 182 }
 183
 184 TemplateURLRef::SearchTermsArgs::ContextualSearchParams::
 185     ContextualSearchParams(
 186         const int version,
 187         const size_t start,
 188         const size_t end,
 189         const std::string& selection,
 190         const std::string& content,
 191         const std::string& base_page_url,
 192         const std::string& encoding,
 193         const bool resolve)
 194     : version(version),
 195       start(start),
 196       end(end),
 197       selection(selection),
 198       content(content),
 199       base_page_url(base_page_url),
 200       encoding(encoding),
 201       resolve(resolve) {
 202 }
 203
 204 TemplateURLRef::SearchTermsArgs::ContextualSearchParams::
 205     ~ContextualSearchParams() {
 206 }
 207
 208 // TemplateURLRef -------------------------------------------------------------
 209
 210 TemplateURLRef::TemplateURLRef(const TemplateURL* owner, Type type)
 211     : owner_(owner),
 212       type_(type),
 213       index_in_owner_(0),
 214       parsed_(false),
 215       valid_(false),
 216       supports_replacements_(false),
 217       search_term_position_in_path_(std::string::npos),
 218       search_term_key_location_(url::Parsed::QUERY),
 219       prepopulated_(false) {
 220   DCHECK(owner_);
 221   DCHECK_NE(INDEXED, type_);
 222 }
 223
 224 TemplateURLRef::TemplateURLRef(const TemplateURL* owner, size_t index_in_owner)
 225     : owner_(owner),
 226       type_(INDEXED),
 227       index_in_owner_(index_in_owner),
 228       parsed_(false),
 229       valid_(false),
 230       supports_replacements_(false),
 231       search_term_position_in_path_(std::string::npos),
 232       search_term_key_location_(url::Parsed::QUERY),
 233       prepopulated_(false) {
 234   DCHECK(owner_);
 235   DCHECK_LT(index_in_owner_, owner_->URLCount());
 236 }
 237
 238 TemplateURLRef::~TemplateURLRef() {
 239 }
 240
 241 std::string TemplateURLRef::GetURL() const {
 242   switch (type_) {
 243     case SEARCH:            return owner_->url();
 244     case SUGGEST:           return owner_->suggestions_url();
 245     case INSTANT:           return owner_->instant_url();
 246     case IMAGE:             return owner_->image_url();
 247     case NEW_TAB:           return owner_->new_tab_url();
 248     case CONTEXTUAL_SEARCH: return owner_->contextual_search_url();
 249     case INDEXED:           return owner_->GetURL(index_in_owner_);
 250     default:       NOTREACHED(); return std::string();  // NOLINT
 251   }
 252 }
 253
 254 std::string TemplateURLRef::GetPostParamsString() const {
 255   switch (type_) {
 256     case INDEXED:
 257     case SEARCH:            return owner_->search_url_post_params();
 258     case SUGGEST:           return owner_->suggestions_url_post_params();
 259     case INSTANT:           return owner_->instant_url_post_params();
 260     case NEW_TAB:           return std::string();
 261     case CONTEXTUAL_SEARCH: return std::string();
 262     case IMAGE:             return owner_->image_url_post_params();
 263     default:      NOTREACHED(); return std::string();  // NOLINT
 264   }
 265 }
 266
 267 bool TemplateURLRef::UsesPOSTMethod(
 268     const SearchTermsData& search_terms_data) const {
 269   ParseIfNecessary(search_terms_data);
 270   return !post_params_.empty();
 271 }
 272
 273 bool TemplateURLRef::EncodeFormData(const PostParams& post_params,
 274                                     PostContent* post_content) const {
 275   if (post_params.empty())
 276     return true;
 277   if (!post_content)
 278     return false;
 279
 280   const char kUploadDataMIMEType[] = "multipart/form-data; boundary=";
 281   const char kMultipartBoundary[] = "----+*+----%016" PRIx64 "----+*+----";
 282   // Each name/value pair is stored in a body part which is preceded by a
 283   // boundary delimiter line. Uses random number generator here to create
 284   // a unique boundary delimiter for form data encoding.
 285   std::string boundary = base::StringPrintf(kMultipartBoundary,
 286                                             base::RandUint64());
 287   // Sets the content MIME type.
 288   post_content->first = kUploadDataMIMEType;
 289   post_content->first += boundary;
 290   // Encodes the post parameters.
 291   std::string* post_data = &post_content->second;
 292   post_data->clear();
 293   for (const auto& param : post_params) {
 294     DCHECK(!param.name.empty());
 295     net::AddMultipartValueForUpload(param.name, param.value, boundary,
 296                                     param.content_type, post_data);
 297   }
 298   net::AddMultipartFinalDelimiterForUpload(boundary, post_data);
 299   return true;
 300 }
 301
 302 bool TemplateURLRef::SupportsReplacement(
 303     const SearchTermsData& search_terms_data) const {
 304   ParseIfNecessary(search_terms_data);
 305   return valid_ && supports_replacements_;
 306 }
 307
 308 std::string TemplateURLRef::ReplaceSearchTerms(
 309     const SearchTermsArgs& search_terms_args,
 310     const SearchTermsData& search_terms_data,
 311     PostContent* post_content) const {
 312   ParseIfNecessary(search_terms_data);
 313   if (!valid_)
 314     return std::string();
 315
 316   std::string url(HandleReplacements(search_terms_args, search_terms_data,
 317                                      post_content));
 318
 319   GURL gurl(url);
 320   if (!gurl.is_valid())
 321     return url;
 322
 323   std::vector<std::string> query_params;
 324   if (search_terms_args.append_extra_query_params) {
 325     std::string extra_params(
 326         base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
 327             switches::kExtraSearchQueryParams));
 328     if (!extra_params.empty())
 329       query_params.push_back(extra_params);
 330   }
 331   if (!search_terms_args.suggest_query_params.empty())
 332     query_params.push_back(search_terms_args.suggest_query_params);
 333   if (!gurl.query().empty())
 334     query_params.push_back(gurl.query());
 335
 336   if (query_params.empty())
 337     return url;
 338
 339   GURL::Replacements replacements;
 340   std::string query_str = base::JoinString(query_params, "&");
 341   replacements.SetQueryStr(query_str);
 342   return gurl.ReplaceComponents(replacements).possibly_invalid_spec();
 343 }
 344
 345 bool TemplateURLRef::IsValid(const SearchTermsData& search_terms_data) const {
 346   ParseIfNecessary(search_terms_data);
 347   return valid_;
 348 }
 349
 350 base::string16 TemplateURLRef::DisplayURL(
 351     const SearchTermsData& search_terms_data) const {
 352   ParseIfNecessary(search_terms_data);
 353   std::string result(GetURL());
 354   if (valid_ && !replacements_.empty()) {
 355     base::ReplaceSubstringsAfterOffset(&result, 0,
 356                                        kSearchTermsParameterFull,
 357                                        kDisplaySearchTerms);
 358     base::ReplaceSubstringsAfterOffset(&result, 0,
 359                                        kGoogleUnescapedSearchTermsParameterFull,
 360                                        kDisplayUnescapedSearchTerms);
 361   }
 362   return base::UTF8ToUTF16(result);
 363 }
 364
 365 // static
 366 std::string TemplateURLRef::DisplayURLToURLRef(
 367     const base::string16& display_url) {
 368   std::string result = base::UTF16ToUTF8(display_url);
 369   base::ReplaceSubstringsAfterOffset(&result, 0,
 370                                      kDisplaySearchTerms,
 371                                      kSearchTermsParameterFull);
 372   base::ReplaceSubstringsAfterOffset(&result, 0,
 373                                      kDisplayUnescapedSearchTerms,
 374                                      kGoogleUnescapedSearchTermsParameterFull);
 375   return result;
 376 }
 377
 378 const std::string& TemplateURLRef::GetHost(
 379     const SearchTermsData& search_terms_data) const {
 380   ParseIfNecessary(search_terms_data);
 381   return host_;
 382 }
 383
 384 const std::string& TemplateURLRef::GetPath(
 385     const SearchTermsData& search_terms_data) const {
 386   ParseIfNecessary(search_terms_data);
 387   return path_;
 388 }
 389
 390 const std::string& TemplateURLRef::GetSearchTermKey(
 391     const SearchTermsData& search_terms_data) const {
 392   ParseIfNecessary(search_terms_data);
 393   return search_term_key_;
 394 }
 395
 396 size_t TemplateURLRef::GetSearchTermPositionInPath(
 397     const SearchTermsData& search_terms_data) const {
 398   ParseIfNecessary(search_terms_data);
 399   return search_term_position_in_path_;
 400 }
 401
 402 url::Parsed::ComponentType TemplateURLRef::GetSearchTermKeyLocation(
 403     const SearchTermsData& search_terms_data) const {
 404   ParseIfNecessary(search_terms_data);
 405   return search_term_key_location_;
 406 }
 407
 408 base::string16 TemplateURLRef::SearchTermToString16(
 409     const std::string& term) const {
 410   const std::vector<std::string>& encodings = owner_->input_encodings();
 411   base::string16 result;
 412
 413   net::UnescapeRule::Type unescape_rules =
 414       net::UnescapeRule::SPACES | net::UnescapeRule::URL_SPECIAL_CHARS;
 415   if (search_term_key_location_ != url::Parsed::PATH)
 416     unescape_rules |= net::UnescapeRule::REPLACE_PLUS_WITH_SPACE;
 417
 418   std::string unescaped = net::UnescapeURLComponent(term, unescape_rules);
 419   for (size_t i = 0; i < encodings.size(); ++i) {
 420     if (base::CodepageToUTF16(unescaped, encodings[i].c_str(),
 421                               base::OnStringConversionError::FAIL, &result))
 422       return result;
 423   }
 424
 425   // Always fall back on UTF-8 if it works.
 426   if (base::CodepageToUTF16(unescaped, base::kCodepageUTF8,
 427                             base::OnStringConversionError::FAIL, &result))
 428     return result;
 429
 430   // When nothing worked, just use the escaped text. We have no idea what the
 431   // encoding is. We need to substitute spaces for pluses ourselves since we're
 432   // not sending it through an unescaper.
 433   result = base::UTF8ToUTF16(term);
 434   if (unescape_rules & net::UnescapeRule::REPLACE_PLUS_WITH_SPACE)
 435     std::replace(result.begin(), result.end(), '+', ' ');
 436   return result;
 437 }
 438
 439 bool TemplateURLRef::HasGoogleBaseURLs(
 440     const SearchTermsData& search_terms_data) const {
 441   ParseIfNecessary(search_terms_data);
 442   for (size_t i = 0; i < replacements_.size(); ++i) {
 443     if ((replacements_[i].type == GOOGLE_BASE_URL) ||
 444         (replacements_[i].type == GOOGLE_BASE_SUGGEST_URL))
 445       return true;
 446   }
 447   return false;
 448 }
 449
 450 bool TemplateURLRef::ExtractSearchTermsFromURL(
 451     const GURL& url,
 452     base::string16* search_terms,
 453     const SearchTermsData& search_terms_data,
 454     url::Parsed::ComponentType* search_terms_component,
 455     url::Component* search_terms_position) const {
 456   DCHECK(search_terms);
 457   search_terms->clear();
 458
 459   ParseIfNecessary(search_terms_data);
 460
 461   // We need a search term in the template URL to extract something.
 462   if (search_term_key_.empty() &&
 463       (search_term_key_location_ != url::Parsed::PATH))
 464     return false;
 465
 466   // Fill-in the replacements. We don't care about search terms in the pattern,
 467   // so we use the empty string.
 468   // Currently we assume the search term only shows in URL, not in post params.
 469   GURL pattern(ReplaceSearchTerms(SearchTermsArgs(base::string16()),
 470                                   search_terms_data, NULL));
 471   // Host, path and port must match.
 472   if ((url.port() != pattern.port()) ||
 473       (url.host() != host_) ||
 474       ((url.path() != path_) &&
 475           (search_term_key_location_ != url::Parsed::PATH))) {
 476     return false;
 477   }
 478
 479   std::string source;
 480   url::Component position;
 481
 482   if (search_term_key_location_ == url::Parsed::PATH) {
 483     source = url.path();
 484
 485     // Characters in the path before and after search terms must match.
 486     if (source.length() < path_.length())
 487       return false;
 488     position.begin = search_term_position_in_path_;
 489     position.len = source.length() - path_.length();
 490     if (source.substr(0, position.begin) + source.substr(position.end()) !=
 491         path_)
 492       return false;
 493   } else {
 494     DCHECK(search_term_key_location_ == url::Parsed::QUERY ||
 495            search_term_key_location_ == url::Parsed::REF);
 496     source = (search_term_key_location_ == url::Parsed::QUERY) ?
 497         url.query() : url.ref();
 498
 499     url::Component query, key, value;
 500     query.len = static_cast<int>(source.size());
 501     bool key_found = false;
 502     while (url::ExtractQueryKeyValue(source.c_str(), &query, &key, &value)) {
 503       if (key.is_nonempty()) {
 504         if (source.substr(key.begin, key.len) == search_term_key_) {
 505           // Fail if search term key is found twice.
 506           if (key_found)
 507             return false;
 508           key_found = true;
 509           position = value;
 510         }
 511       }
 512     }
 513     if (!key_found)
 514       return false;
 515   }
 516
 517   // Extract the search term.
 518   *search_terms = SearchTermToString16(
 519       source.substr(position.begin, position.len));
 520   if (search_terms_component)
 521     *search_terms_component = search_term_key_location_;
 522   if (search_terms_position)
 523     *search_terms_position = position;
 524   return true;
 525 }
 526
 527 void TemplateURLRef::InvalidateCachedValues() const {
 528   supports_replacements_ = valid_ = parsed_ = false;
 529   host_.clear();
 530   path_.clear();
 531   search_term_key_.clear();
 532   search_term_position_in_path_ = std::string::npos;
 533   search_term_key_location_ = url::Parsed::QUERY;
 534   replacements_.clear();
 535   post_params_.clear();
 536 }
 537
 538 bool TemplateURLRef::ParseParameter(size_t start,
 539                                     size_t end,
 540                                     std::string* url,
 541                                     Replacements* replacements) const {
 542   DCHECK(start != std::string::npos &&
 543          end != std::string::npos && end > start);
 544   size_t length = end - start - 1;
 545   bool optional = false;
 546   // Make a copy of |url| that can be referenced in StringPieces below. |url| is
 547   // modified, so that can't be used in StringPiece.
 548   const std::string original_url(*url);
 549   if (original_url[end - 1] == kOptional) {
 550     optional = true;
 551     length--;
 552   }
 553
 554   const base::StringPiece parameter(original_url.begin() + start + 1,
 555                                     original_url.begin() + start + 1 + length);
 556   const base::StringPiece full_parameter(original_url.begin() + start,
 557                                          original_url.begin() + end + 1);
 558   // Remove the parameter from the string.  For parameters who replacement is
 559   // constant and already known, just replace them directly.  For other cases,
 560   // like parameters whose values may change over time, use |replacements|.
 561   url->erase(start, end - start + 1);
 562   if (parameter == kSearchTermsParameter) {
 563     replacements->push_back(Replacement(SEARCH_TERMS, start));
 564   } else if (parameter == "count") {
 565     if (!optional)
 566       url->insert(start, kDefaultCount);
 567   } else if (parameter == "google:assistedQueryStats") {
 568     replacements->push_back(Replacement(GOOGLE_ASSISTED_QUERY_STATS, start));
 569   } else if (parameter == "google:baseURL") {
 570     replacements->push_back(Replacement(GOOGLE_BASE_URL, start));
 571   } else if (parameter == "google:baseSuggestURL") {
 572     replacements->push_back(Replacement(GOOGLE_BASE_SUGGEST_URL, start));
 573   } else if (parameter == "google:bookmarkBarPinned") {
 574     replacements->push_back(Replacement(GOOGLE_BOOKMARK_BAR_PINNED, start));
 575   } else if (parameter == "google:currentPageUrl") {
 576     replacements->push_back(Replacement(GOOGLE_CURRENT_PAGE_URL, start));
 577   } else if (parameter == "google:cursorPosition") {
 578     replacements->push_back(Replacement(GOOGLE_CURSOR_POSITION, start));
 579   } else if (parameter == "google:forceInstantResults") {
 580     replacements->push_back(Replacement(GOOGLE_FORCE_INSTANT_RESULTS, start));
 581   } else if (parameter == "google:imageOriginalHeight") {
 582     replacements->push_back(
 583         Replacement(TemplateURLRef::GOOGLE_IMAGE_ORIGINAL_HEIGHT, start));
 584   } else if (parameter == "google:imageOriginalWidth") {
 585     replacements->push_back(
 586         Replacement(TemplateURLRef::GOOGLE_IMAGE_ORIGINAL_WIDTH, start));
 587   } else if (parameter == "google:imageSearchSource") {
 588     replacements->push_back(
 589         Replacement(TemplateURLRef::GOOGLE_IMAGE_SEARCH_SOURCE, start));
 590   } else if (parameter == "google:imageThumbnail") {
 591     replacements->push_back(
 592         Replacement(TemplateURLRef::GOOGLE_IMAGE_THUMBNAIL, start));
 593   } else if (parameter == "google:imageURL") {
 594     replacements->push_back(Replacement(TemplateURLRef::GOOGLE_IMAGE_URL,
 595                                         start));
 596   } else if (parameter == "google:inputType") {
 597     replacements->push_back(Replacement(TemplateURLRef::GOOGLE_INPUT_TYPE,
 598                                         start));
 599   } else if (parameter == "google:instantExtendedEnabledParameter") {
 600     replacements->push_back(Replacement(GOOGLE_INSTANT_EXTENDED_ENABLED,
 601                                         start));
 602   } else if (parameter == "google:instantExtendedEnabledKey") {
 603     url->insert(start, google_util::kInstantExtendedAPIParam);
 604   } else if (parameter == "google:ntpIsThemedParameter") {
 605     replacements->push_back(Replacement(GOOGLE_NTP_IS_THEMED, start));
 606   } else if (parameter == "google:omniboxStartMarginParameter") {
 607     replacements->push_back(Replacement(GOOGLE_OMNIBOX_START_MARGIN, start));
 608   } else if (parameter == "google:contextualSearchVersion") {
 609     replacements->push_back(
 610         Replacement(GOOGLE_CONTEXTUAL_SEARCH_VERSION, start));
 611   } else if (parameter == "google:contextualSearchContextData") {
 612     replacements->push_back(
 613         Replacement(GOOGLE_CONTEXTUAL_SEARCH_CONTEXT_DATA, start));
 614   } else if (parameter == "google:originalQueryForSuggestion") {
 615     replacements->push_back(Replacement(GOOGLE_ORIGINAL_QUERY_FOR_SUGGESTION,
 616                                         start));
 617   } else if (parameter == "google:pageClassification") {
 618     replacements->push_back(Replacement(GOOGLE_PAGE_CLASSIFICATION, start));
 619   } else if (parameter == "google:prefetchQuery") {
 620     replacements->push_back(Replacement(GOOGLE_PREFETCH_QUERY, start));
 621   } else if (parameter == "google:RLZ") {
 622     replacements->push_back(Replacement(GOOGLE_RLZ, start));
 623   } else if (parameter == "google:searchClient") {
 624     replacements->push_back(Replacement(GOOGLE_SEARCH_CLIENT, start));
 625   } else if (parameter == "google:searchFieldtrialParameter") {
 626     replacements->push_back(Replacement(GOOGLE_SEARCH_FIELDTRIAL_GROUP, start));
 627   } else if (parameter == "google:searchVersion") {
 628     replacements->push_back(Replacement(GOOGLE_SEARCH_VERSION, start));
 629   } else if (parameter == "google:sessionToken") {
 630     replacements->push_back(Replacement(GOOGLE_SESSION_TOKEN, start));
 631   } else if (parameter == "google:sourceId") {
 632 #if defined(OS_ANDROID) || defined(OS_IOS)
 633     url->insert(start, "sourceid=chrome-mobile&");
 634 #else
 635     url->insert(start, "sourceid=chrome&");
 636 #endif
 637   } else if (parameter == "google:suggestAPIKeyParameter") {
 638     url->insert(start,
 639                 net::EscapeQueryParamValue(google_apis::GetAPIKey(), false));
 640   } else if (parameter == "google:suggestClient") {
 641     replacements->push_back(Replacement(GOOGLE_SUGGEST_CLIENT, start));
 642   } else if (parameter == "google:suggestRid") {
 643     replacements->push_back(Replacement(GOOGLE_SUGGEST_REQUEST_ID, start));
 644   } else if (parameter == kGoogleUnescapedSearchTermsParameter) {
 645     replacements->push_back(Replacement(GOOGLE_UNESCAPED_SEARCH_TERMS, start));
 646   } else if (parameter == "yandex:searchPath") {
 647     switch (ui::GetDeviceFormFactor()) {
 648       case ui::DEVICE_FORM_FACTOR_DESKTOP:
 649         url->insert(start, "yandsearch");
 650         break;
 651       case ui::DEVICE_FORM_FACTOR_PHONE:
 652         url->insert(start, "touchsearch");
 653         break;
 654       case ui::DEVICE_FORM_FACTOR_TABLET:
 655         url->insert(start, "padsearch");
 656         break;
 657     }
 658   } else if (parameter == "inputEncoding") {
 659     replacements->push_back(Replacement(ENCODING, start));
 660   } else if (parameter == "language") {
 661     replacements->push_back(Replacement(LANGUAGE, start));
 662   } else if (parameter == "outputEncoding") {
 663     if (!optional)
 664       url->insert(start, kOutputEncodingType);
 665   } else if ((parameter == "startIndex") || (parameter == "startPage")) {
 666     // We don't support these.
 667     if (!optional)
 668       url->insert(start, "1");
 669   } else if (!prepopulated_) {
 670     // If it's a prepopulated URL, we know that it's safe to remove unknown
 671     // parameters, so just ignore this and return true below. Otherwise it could
 672     // be some garbage but can also be a javascript block. Put it back.
 673     url->insert(start, full_parameter.data(), full_parameter.size());
 674     return false;
 675   }
 676   return true;
 677 }
 678
 679 std::string TemplateURLRef::ParseURL(const std::string& url,
 680                                      Replacements* replacements,
 681                                      PostParams* post_params,
 682                                      bool* valid) const {
 683   *valid = false;
 684   std::string parsed_url = url;
 685   for (size_t last = 0; last != std::string::npos; ) {
 686     last = parsed_url.find(kStartParameter, last);
 687     if (last != std::string::npos) {
 688       size_t template_end = parsed_url.find(kEndParameter, last);
 689       if (template_end != std::string::npos) {
 690         // Since we allow Javascript in the URL, {} pairs could be nested. Match
 691         // only leaf pairs with supported parameters.
 692         size_t next_template_start = parsed_url.find(kStartParameter, last + 1);
 693         if (next_template_start == std::string::npos ||
 694             next_template_start > template_end) {
 695           // If successful, ParseParameter erases from the string as such no
 696           // need to update |last|. If failed, move |last| to the end of pair.
 697           if (!ParseParameter(last, template_end, &parsed_url, replacements)) {
 698             // |template_end| + 1 may be beyond the end of the string.
 699             last = template_end;
 700           }
 701         } else {
 702           last = next_template_start;
 703         }
 704       } else {
 705         // Open brace without a closing brace, return.
 706         return std::string();
 707       }
 708     }
 709   }
 710
 711   // Handles the post parameters.
 712   const std::string& post_params_string = GetPostParamsString();
 713   if (!post_params_string.empty()) {
 714     for (const base::StringPiece& cur : base::SplitStringPiece(
 715              post_params_string, ",",
 716              base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL)) {
 717       // The '=' delimiter is required and the name must be not empty.
 718       std::vector<std::string> parts = base::SplitString(
 719           cur, "=", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
 720       if ((parts.size() != 2U) || parts[0].empty())
 721         return std::string();
 722
 723       std::string& value = parts[1];
 724       size_t replacements_size = replacements->size();
 725       if (IsTemplateParameterString(value))
 726         ParseParameter(0, value.length() - 1, &value, replacements);
 727       PostParam param = { parts[0], value };
 728       post_params->push_back(param);
 729       // If there was a replacement added, points its index to last added
 730       // PostParam.
 731       if (replacements->size() > replacements_size) {
 732         DCHECK_EQ(replacements_size + 1, replacements->size());
 733         Replacement* r = &replacements->back();
 734         r->is_post_param = true;
 735         r->index = post_params->size() - 1;
 736       }
 737     }
 738     DCHECK(!post_params->empty());
 739   }
 740
 741   *valid = true;
 742   return parsed_url;
 743 }
 744
 745 void TemplateURLRef::ParseIfNecessary(
 746     const SearchTermsData& search_terms_data) const {
 747   if (!parsed_) {
 748     InvalidateCachedValues();
 749     parsed_ = true;
 750     parsed_url_ = ParseURL(GetURL(), &replacements_, &post_params_, &valid_);
 751     supports_replacements_ = false;
 752     if (valid_) {
 753       bool has_only_one_search_term = false;
 754       for (Replacements::const_iterator i = replacements_.begin();
 755            i != replacements_.end(); ++i) {
 756         if ((i->type == SEARCH_TERMS) ||
 757             (i->type == GOOGLE_UNESCAPED_SEARCH_TERMS)) {
 758           if (has_only_one_search_term) {
 759             has_only_one_search_term = false;
 760             break;
 761           }
 762           has_only_one_search_term = true;
 763           supports_replacements_ = true;
 764         }
 765       }
 766       // Only parse the host/key if there is one search term. Technically there
 767       // could be more than one term, but it's uncommon; so we punt.
 768       if (has_only_one_search_term)
 769         ParseHostAndSearchTermKey(search_terms_data);
 770     }
 771   }
 772 }
 773
 774 void TemplateURLRef::ParseHostAndSearchTermKey(
 775     const SearchTermsData& search_terms_data) const {
 776   std::string url_string(GetURL());
 777   base::ReplaceSubstringsAfterOffset(
 778       &url_string, 0, "{google:baseURL}",
 779       search_terms_data.GoogleBaseURLValue());
 780   base::ReplaceSubstringsAfterOffset(
 781       &url_string, 0, "{google:baseSuggestURL}",
 782       search_terms_data.GoogleBaseSuggestURLValue());
 783
 784   search_term_key_.clear();
 785   search_term_position_in_path_ = std::string::npos;
 786   host_.clear();
 787   path_.clear();
 788   search_term_key_location_ = url::Parsed::QUERY;
 789
 790   GURL url(url_string);
 791   if (!url.is_valid())
 792     return;
 793
 794   std::string query_key = FindSearchTermsKey(url.query());
 795   std::string ref_key = FindSearchTermsKey(url.ref());
 796   url::Component parameter_position;
 797   const bool in_query = !query_key.empty();
 798   const bool in_ref = !ref_key.empty();
 799   const bool in_path = FindSearchTermsInPath(url.path(), &parameter_position);
 800   if (in_query ? (in_ref || in_path) : (in_ref == in_path))
 801     return;  // No key or multiple keys found.  We only handle having one key.
 802
 803   host_ = url.host();
 804   path_ = url.path();
 805   if (in_query) {
 806     search_term_key_ = query_key;
 807     search_term_key_location_ = url::Parsed::QUERY;
 808   } else if (in_ref) {
 809     search_term_key_ = ref_key;
 810     search_term_key_location_ = url::Parsed::REF;
 811   } else {
 812     DCHECK(in_path);
 813     DCHECK_GE(parameter_position.begin, 1);  // Path must start with '/'.
 814     search_term_key_location_ = url::Parsed::PATH;
 815     search_term_position_in_path_ = parameter_position.begin;
 816     // Remove the "{searchTerms}" itself from |path_|.
 817     path_.erase(parameter_position.begin, parameter_position.len);
 818   }
 819 }
 820
 821 void TemplateURLRef::HandleReplacement(const std::string& name,
 822                                        const std::string& value,
 823                                        const Replacement& replacement,
 824                                        std::string* url) const {
 825   size_t pos = replacement.index;
 826   if (replacement.is_post_param) {
 827     DCHECK_LT(pos, post_params_.size());
 828     DCHECK(!post_params_[pos].name.empty());
 829     post_params_[pos].value = value;
 830   } else {
 831     url->insert(pos, name.empty() ? value : (name + "=" + value + "&"));
 832   }
 833 }
 834
 835 std::string TemplateURLRef::HandleReplacements(
 836     const SearchTermsArgs& search_terms_args,
 837     const SearchTermsData& search_terms_data,
 838     PostContent* post_content) const {
 839   if (replacements_.empty()) {
 840     if (!post_params_.empty())
 841       EncodeFormData(post_params_, post_content);
 842     return parsed_url_;
 843   }
 844
 845   // Determine if the search terms are in the query or before. We're escaping
 846   // space as '+' in the former case and as '%20' in the latter case.
 847   bool is_in_query = true;
 848   for (Replacements::iterator i = replacements_.begin();
 849        i != replacements_.end(); ++i) {
 850     if (i->type == SEARCH_TERMS) {
 851       base::string16::size_type query_start = parsed_url_.find('?');
 852       is_in_query = query_start != base::string16::npos &&
 853           (static_cast<base::string16::size_type>(i->index) > query_start);
 854       break;
 855     }
 856   }
 857
 858   std::string input_encoding;
 859   base::string16 encoded_terms;
 860   base::string16 encoded_original_query;
 861   owner_->EncodeSearchTerms(search_terms_args, is_in_query, &input_encoding,
 862                             &encoded_terms, &encoded_original_query);
 863
 864   std::string url = parsed_url_;
 865
 866   // replacements_ is ordered in ascending order, as such we need to iterate
 867   // from the back.
 868   for (Replacements::reverse_iterator i = replacements_.rbegin();
 869        i != replacements_.rend(); ++i) {
 870     switch (i->type) {
 871       case ENCODING:
 872         HandleReplacement(std::string(), input_encoding, *i, &url);
 873         break;
 874
 875       case GOOGLE_ASSISTED_QUERY_STATS:
 876         DCHECK(!i->is_post_param);
 877         if (!search_terms_args.assisted_query_stats.empty()) {
 878           // Get the base URL without substituting AQS to avoid infinite
 879           // recursion.  We need the URL to find out if it meets all
 880           // AQS requirements (e.g. HTTPS protocol check).
 881           // See TemplateURLRef::SearchTermsArgs for more details.
 882           SearchTermsArgs search_terms_args_without_aqs(search_terms_args);
 883           search_terms_args_without_aqs.assisted_query_stats.clear();
 884           GURL base_url(ReplaceSearchTerms(
 885               search_terms_args_without_aqs, search_terms_data, NULL));
 886           if (base_url.SchemeIsCryptographic()) {
 887             HandleReplacement(
 888                 "aqs", search_terms_args.assisted_query_stats, *i, &url);
 889           }
 890         }
 891         break;
 892
 893       case GOOGLE_BASE_URL:
 894         DCHECK(!i->is_post_param);
 895         HandleReplacement(
 896             std::string(), search_terms_data.GoogleBaseURLValue(), *i, &url);
 897         break;
 898
 899       case GOOGLE_BASE_SUGGEST_URL:
 900         DCHECK(!i->is_post_param);
 901         HandleReplacement(
 902             std::string(), search_terms_data.GoogleBaseSuggestURLValue(), *i,
 903             &url);
 904         break;
 905
 906       case GOOGLE_BOOKMARK_BAR_PINNED:
 907         if (search_terms_data.IsShowingSearchTermsOnSearchResultsPages()) {
 908           // Log whether the bookmark bar is pinned when the user is seeing
 909           // InstantExtended on the SRP.
 910           DCHECK(!i->is_post_param);
 911           HandleReplacement(
 912               "bmbp", search_terms_args.bookmark_bar_pinned ? "1" : "0", *i,
 913               &url);
 914         }
 915         break;
 916
 917       case GOOGLE_CURRENT_PAGE_URL:
 918         DCHECK(!i->is_post_param);
 919         if (!search_terms_args.current_page_url.empty()) {
 920           const std::string& escaped_current_page_url =
 921               net::EscapeQueryParamValue(search_terms_args.current_page_url,
 922                                          true);
 923           HandleReplacement("url", escaped_current_page_url, *i, &url);
 924         }
 925         break;
 926
 927       case GOOGLE_CURSOR_POSITION:
 928         DCHECK(!i->is_post_param);
 929         if (search_terms_args.cursor_position != base::string16::npos)
 930           HandleReplacement(
 931               "cp",
 932               base::StringPrintf("%" PRIuS, search_terms_args.cursor_position),
 933               *i,
 934               &url);
 935         break;
 936
 937       case GOOGLE_FORCE_INSTANT_RESULTS:
 938         DCHECK(!i->is_post_param);
 939         HandleReplacement(std::string(),
 940                           search_terms_data.ForceInstantResultsParam(
 941                               search_terms_args.force_instant_results),
 942                           *i,
 943                           &url);
 944         break;
 945
 946       case GOOGLE_INPUT_TYPE:
 947         DCHECK(!i->is_post_param);
 948         HandleReplacement(
 949             "oit", base::IntToString(search_terms_args.input_type), *i, &url);
 950         break;
 951
 952       case GOOGLE_INSTANT_EXTENDED_ENABLED:
 953         DCHECK(!i->is_post_param);
 954         HandleReplacement(std::string(),
 955                           search_terms_data.InstantExtendedEnabledParam(
 956                               type_ == SEARCH),
 957                           *i,
 958                           &url);
 959         break;
 960
 961       case GOOGLE_NTP_IS_THEMED:
 962         DCHECK(!i->is_post_param);
 963         HandleReplacement(
 964             std::string(), search_terms_data.NTPIsThemedParam(), *i, &url);
 965         break;
 966
 967       case GOOGLE_OMNIBOX_START_MARGIN:
 968         DCHECK(!i->is_post_param);
 969         if (search_terms_args.enable_omnibox_start_margin) {
 970           int omnibox_start_margin = search_terms_data.OmniboxStartMargin();
 971           if (omnibox_start_margin >= 0) {
 972             HandleReplacement("es_sm", base::IntToString(omnibox_start_margin),
 973                               *i, &url);
 974           }
 975         }
 976         break;
 977
 978       case GOOGLE_CONTEXTUAL_SEARCH_VERSION:
 979         if (search_terms_args.contextual_search_params.version >= 0) {
 980           HandleReplacement(
 981               "ctxs",
 982               base::IntToString(
 983                   search_terms_args.contextual_search_params.version),
 984               *i,
 985               &url);
 986         }
 987         break;
 988
 989       case GOOGLE_CONTEXTUAL_SEARCH_CONTEXT_DATA: {
 990         DCHECK(!i->is_post_param);
 991         std::string context_data;
 992
 993         const SearchTermsArgs::ContextualSearchParams& params =
 994             search_terms_args.contextual_search_params;
 995
 996         if (params.start != std::string::npos) {
 997           context_data.append("ctxs_start=" + base::IntToString(
 998               params.start) + "&");
 999         }
1000
1001         if (params.end != std::string::npos) {
1002           context_data.append("ctxs_end=" + base::IntToString(
1003               params.end) + "&");
1004         }
1005
1006         if (!params.selection.empty())
1007           context_data.append("q=" + params.selection + "&");
1008
1009         if (!params.content.empty())
1010           context_data.append("ctxs_content=" + params.content + "&");
1011
1012         if (!params.base_page_url.empty())
1013           context_data.append("ctxsl_url=" + params.base_page_url + "&");
1014
1015         if (!params.encoding.empty()) {
1016           context_data.append("ctxs_encoding=" + params.encoding + "&");
1017         }
1018
1019         context_data.append(
1020             params.resolve ? "ctxsl_resolve=1" : "ctxsl_resolve=0");
1021
1022         HandleReplacement(std::string(), context_data, *i, &url);
1023         break;
1024       }
1025
1026       case GOOGLE_ORIGINAL_QUERY_FOR_SUGGESTION:
1027         DCHECK(!i->is_post_param);
1028         if (search_terms_args.accepted_suggestion >= 0 ||
1029             !search_terms_args.assisted_query_stats.empty()) {
1030           HandleReplacement(
1031               "oq", base::UTF16ToUTF8(encoded_original_query), *i, &url);
1032         }
1033         break;
1034
1035       case GOOGLE_PAGE_CLASSIFICATION:
1036         if (search_terms_args.page_classification !=
1037             metrics::OmniboxEventProto::INVALID_SPEC) {
1038           HandleReplacement(
1039               "pgcl", base::IntToString(search_terms_args.page_classification),
1040               *i, &url);
1041         }
1042         break;
1043
1044       case GOOGLE_PREFETCH_QUERY: {
1045         const std::string& query = search_terms_args.prefetch_query;
1046         const std::string& type = search_terms_args.prefetch_query_type;
1047         if (!query.empty() && !type.empty()) {
1048           HandleReplacement(
1049               std::string(), "pfq=" + query + "&qha=" + type + "&", *i, &url);
1050         }
1051         break;
1052       }
1053
1054       case GOOGLE_RLZ: {
1055         DCHECK(!i->is_post_param);
1056         // On platforms that don't have RLZ, we still want this branch
1057         // to happen so that we replace the RLZ template with the
1058         // empty string.  (If we don't handle this case, we hit a
1059         // NOTREACHED below.)
1060         base::string16 rlz_string = search_terms_data.GetRlzParameterValue(
1061             search_terms_args.from_app_list);
1062         if (!rlz_string.empty()) {
1063           HandleReplacement("rlz", base::UTF16ToUTF8(rlz_string), *i, &url);
1064         }
1065         break;
1066       }
1067
1068       case GOOGLE_SEARCH_CLIENT: {
1069         DCHECK(!i->is_post_param);
1070         std::string client = search_terms_data.GetSearchClient();
1071         if (!client.empty())
1072           HandleReplacement("client", client, *i, &url);
1073         break;
1074       }
1075
1076       case GOOGLE_SEARCH_FIELDTRIAL_GROUP:
1077         // We are not currently running any fieldtrials that modulate the search
1078         // url.  If we do, then we'd have some conditional insert such as:
1079         // url.insert(i->index, used_www ? "gcx=w&" : "gcx=c&");
1080         break;
1081
1082       case GOOGLE_SEARCH_VERSION:
1083         HandleReplacement("gs_rn", "42", *i, &url);
1084         break;
1085
1086       case GOOGLE_SESSION_TOKEN: {
1087         std::string token = search_terms_args.session_token;
1088         if (!token.empty())
1089           HandleReplacement("psi", token, *i, &url);
1090         break;
1091       }
1092
1093       case GOOGLE_SUGGEST_CLIENT:
1094         HandleReplacement(
1095             std::string(), search_terms_data.GetSuggestClient(), *i, &url);
1096         break;
1097
1098       case GOOGLE_SUGGEST_REQUEST_ID:
1099         HandleReplacement(
1100             std::string(), search_terms_data.GetSuggestRequestIdentifier(), *i,
1101             &url);
1102         break;
1103
1104       case GOOGLE_UNESCAPED_SEARCH_TERMS: {
1105         std::string unescaped_terms;
1106         base::UTF16ToCodepage(search_terms_args.search_terms,
1107                               input_encoding.c_str(),
1108                               base::OnStringConversionError::SKIP,
1109                               &unescaped_terms);
1110         HandleReplacement(std::string(), unescaped_terms, *i, &url);
1111         break;
1112       }
1113
1114       case LANGUAGE:
1115         HandleReplacement(
1116             std::string(), search_terms_data.GetApplicationLocale(), *i, &url);
1117         break;
1118
1119       case SEARCH_TERMS:
1120         HandleReplacement(
1121             std::string(), base::UTF16ToUTF8(encoded_terms), *i, &url);
1122         break;
1123
1124       case GOOGLE_IMAGE_THUMBNAIL:
1125         HandleReplacement(
1126             std::string(), search_terms_args.image_thumbnail_content, *i, &url);
1127         post_params_[i->index].content_type = "image/jpeg";
1128         break;
1129
1130       case GOOGLE_IMAGE_URL:
1131         if (search_terms_args.image_url.is_valid()) {
1132           HandleReplacement(
1133               std::string(), search_terms_args.image_url.spec(), *i, &url);
1134         }
1135         break;
1136
1137       case GOOGLE_IMAGE_ORIGINAL_WIDTH:
1138         if (!search_terms_args.image_original_size.IsEmpty()) {
1139           HandleReplacement(
1140               std::string(),
1141               base::IntToString(search_terms_args.image_original_size.width()),
1142               *i, &url);
1143         }
1144         break;
1145
1146       case GOOGLE_IMAGE_ORIGINAL_HEIGHT:
1147         if (!search_terms_args.image_original_size.IsEmpty()) {
1148           HandleReplacement(
1149               std::string(),
1150               base::IntToString(search_terms_args.image_original_size.height()),
1151               *i, &url);
1152         }
1153         break;
1154
1155       case GOOGLE_IMAGE_SEARCH_SOURCE:
1156         HandleReplacement(
1157             std::string(), search_terms_data.GoogleImageSearchSource(), *i,
1158             &url);
1159         break;
1160
1161       default:
1162         NOTREACHED();
1163         break;
1164     }
1165   }
1166
1167   if (!post_params_.empty())
1168     EncodeFormData(post_params_, post_content);
1169
1170   return url;
1171 }
1172
1173
1174 // TemplateURL ----------------------------------------------------------------
1175
1176 TemplateURL::AssociatedExtensionInfo::AssociatedExtensionInfo(
1177     Type type,
1178     const std::string& extension_id)
1179     : type(type),
1180       extension_id(extension_id),
1181       wants_to_be_default_engine(false) {
1182   DCHECK_NE(NORMAL, type);
1183 }
1184
1185 TemplateURL::AssociatedExtensionInfo::~AssociatedExtensionInfo() {
1186 }
1187
1188 TemplateURL::TemplateURL(const TemplateURLData& data)
1189     : data_(data),
1190       url_ref_(this, TemplateURLRef::SEARCH),
1191       suggestions_url_ref_(this,
1192                            TemplateURLRef::SUGGEST),
1193       instant_url_ref_(this,
1194                        TemplateURLRef::INSTANT),
1195       image_url_ref_(this, TemplateURLRef::IMAGE),
1196       new_tab_url_ref_(this, TemplateURLRef::NEW_TAB),
1197       contextual_search_url_ref_(this, TemplateURLRef::CONTEXTUAL_SEARCH) {
1198   SetPrepopulateId(data_.prepopulate_id);
1199
1200   if (data_.search_terms_replacement_key ==
1201       "{google:instantExtendedEnabledKey}") {
1202     data_.search_terms_replacement_key = google_util::kInstantExtendedAPIParam;
1203   }
1204 }
1205
1206 TemplateURL::~TemplateURL() {
1207 }
1208
1209 // static
1210 base::string16 TemplateURL::GenerateKeyword(
1211     const GURL& url,
1212     const std::string& accept_languages) {
1213   DCHECK(url.is_valid());
1214   // Strip "www." off the front of the keyword; otherwise the keyword won't work
1215   // properly.  See http://code.google.com/p/chromium/issues/detail?id=6984 .
1216   // |url|'s hostname may be IDN-encoded. Before generating |keyword| from it,
1217   // convert to Unicode using the user's accept-languages, so it won't look like
1218   // a confusing punycode string.
1219   base::string16 keyword =
1220       net::StripWWW(net::IDNToUnicode(url.host(), accept_languages));
1221   // Special case: if the host was exactly "www." (not sure this can happen but
1222   // perhaps with some weird intranet and custom DNS server?), ensure we at
1223   // least don't return the empty string.
1224   return keyword.empty() ? base::ASCIIToUTF16("www") : keyword;
1225 }
1226
1227 // static
1228 GURL TemplateURL::GenerateFaviconURL(const GURL& url) {
1229   DCHECK(url.is_valid());
1230   GURL::Replacements rep;
1231
1232   const char favicon_path[] = "/favicon.ico";
1233   int favicon_path_len = arraysize(favicon_path) - 1;
1234
1235   rep.SetPath(favicon_path, url::Component(0, favicon_path_len));
1236   rep.ClearUsername();
1237   rep.ClearPassword();
1238   rep.ClearQuery();
1239   rep.ClearRef();
1240   return url.ReplaceComponents(rep);
1241 }
1242
1243 // static
1244 bool TemplateURL::MatchesData(const TemplateURL* t_url,
1245                               const TemplateURLData* data,
1246                               const SearchTermsData& search_terms_data) {
1247   if (!t_url || !data)
1248     return !t_url && !data;
1249
1250   return (t_url->short_name() == data->short_name()) &&
1251       t_url->HasSameKeywordAs(*data, search_terms_data) &&
1252       (t_url->url() == data->url()) &&
1253       (t_url->suggestions_url() == data->suggestions_url) &&
1254       (t_url->instant_url() == data->instant_url) &&
1255       (t_url->image_url() == data->image_url) &&
1256       (t_url->new_tab_url() == data->new_tab_url) &&
1257       (t_url->search_url_post_params() == data->search_url_post_params) &&
1258       (t_url->suggestions_url_post_params() ==
1259           data->suggestions_url_post_params) &&
1260       (t_url->instant_url_post_params() == data->instant_url_post_params) &&
1261       (t_url->image_url_post_params() == data->image_url_post_params) &&
1262       (t_url->favicon_url() == data->favicon_url) &&
1263       (t_url->safe_for_autoreplace() == data->safe_for_autoreplace) &&
1264       (t_url->show_in_default_list() == data->show_in_default_list) &&
1265       (t_url->input_encodings() == data->input_encodings) &&
1266       (t_url->alternate_urls() == data->alternate_urls) &&
1267       (t_url->search_terms_replacement_key() ==
1268           data->search_terms_replacement_key);
1269 }
1270
1271 base::string16 TemplateURL::AdjustedShortNameForLocaleDirection() const {
1272   base::string16 bidi_safe_short_name = data_.short_name();
1273   base::i18n::AdjustStringForLocaleDirection(&bidi_safe_short_name);
1274   return bidi_safe_short_name;
1275 }
1276
1277 bool TemplateURL::ShowInDefaultList(
1278     const SearchTermsData& search_terms_data) const {
1279   return data_.show_in_default_list &&
1280       url_ref_.SupportsReplacement(search_terms_data);
1281 }
1282
1283 bool TemplateURL::SupportsReplacement(
1284     const SearchTermsData& search_terms_data) const {
1285   return url_ref_.SupportsReplacement(search_terms_data);
1286 }
1287
1288 bool TemplateURL::HasGoogleBaseURLs(
1289     const SearchTermsData& search_terms_data) const {
1290   return url_ref_.HasGoogleBaseURLs(search_terms_data) ||
1291       suggestions_url_ref_.HasGoogleBaseURLs(search_terms_data) ||
1292       instant_url_ref_.HasGoogleBaseURLs(search_terms_data) ||
1293       image_url_ref_.HasGoogleBaseURLs(search_terms_data) ||
1294       new_tab_url_ref_.HasGoogleBaseURLs(search_terms_data);
1295 }
1296
1297 bool TemplateURL::IsGoogleSearchURLWithReplaceableKeyword(
1298     const SearchTermsData& search_terms_data) const {
1299   return (GetType() == NORMAL) &&
1300       url_ref_.HasGoogleBaseURLs(search_terms_data) &&
1301       google_util::IsGoogleHostname(base::UTF16ToUTF8(data_.keyword()),
1302                                     google_util::DISALLOW_SUBDOMAIN);
1303 }
1304
1305 bool TemplateURL::HasSameKeywordAs(
1306     const TemplateURLData& other,
1307     const SearchTermsData& search_terms_data) const {
1308   return (data_.keyword() == other.keyword()) ||
1309       (IsGoogleSearchURLWithReplaceableKeyword(search_terms_data) &&
1310        TemplateURL(other).IsGoogleSearchURLWithReplaceableKeyword(
1311            search_terms_data));
1312 }
1313
1314 TemplateURL::Type TemplateURL::GetType() const {
1315   return extension_info_ ? extension_info_->type : NORMAL;
1316 }
1317
1318 std::string TemplateURL::GetExtensionId() const {
1319   DCHECK(extension_info_);
1320   return extension_info_->extension_id;
1321 }
1322
1323 size_t TemplateURL::URLCount() const {
1324   // Add 1 for the regular search URL.
1325   return data_.alternate_urls.size() + 1;
1326 }
1327
1328 const std::string& TemplateURL::GetURL(size_t index) const {
1329   DCHECK_LT(index, URLCount());
1330
1331   return (index < data_.alternate_urls.size()) ?
1332       data_.alternate_urls[index] : url();
1333 }
1334
1335 bool TemplateURL::ExtractSearchTermsFromURL(
1336     const GURL& url,
1337     const SearchTermsData& search_terms_data,
1338     base::string16* search_terms) const {
1339   return FindSearchTermsInURL(url, search_terms_data, search_terms, NULL, NULL);
1340 }
1341
1342 bool TemplateURL::IsSearchURL(const GURL& url,
1343                               const SearchTermsData& search_terms_data) const {
1344   base::string16 search_terms;
1345   return ExtractSearchTermsFromURL(url, search_terms_data, &search_terms) &&
1346       !search_terms.empty();
1347 }
1348
1349 bool TemplateURL::HasSearchTermsReplacementKey(const GURL& url) const {
1350   // Look for the key both in the query and the ref.
1351   std::string params[] = {url.query(), url.ref()};
1352
1353   for (int i = 0; i < 2; ++i) {
1354     url::Component query, key, value;
1355     query.len = static_cast<int>(params[i].size());
1356     while (url::ExtractQueryKeyValue(params[i].c_str(), &query, &key, &value)) {
1357       if (key.is_nonempty() &&
1358           params[i].substr(key.begin, key.len) ==
1359               search_terms_replacement_key()) {
1360         return true;
1361       }
1362     }
1363   }
1364   return false;
1365 }
1366
1367 bool TemplateURL::ReplaceSearchTermsInURL(
1368     const GURL& url,
1369     const TemplateURLRef::SearchTermsArgs& search_terms_args,
1370     const SearchTermsData& search_terms_data,
1371     GURL* result) {
1372   // TODO(beaudoin): Use AQS from |search_terms_args| too.
1373   url::Parsed::ComponentType search_term_component;
1374   url::Component search_terms_position;
1375   base::string16 search_terms;
1376   if (!FindSearchTermsInURL(url, search_terms_data, &search_terms,
1377                             &search_term_component, &search_terms_position)) {
1378     return false;
1379   }
1380   DCHECK(search_terms_position.is_nonempty());
1381
1382   // Query and ref are encoded in the same way.
1383   const bool is_in_query = (search_term_component != url::Parsed::PATH);
1384
1385   std::string input_encoding;
1386   base::string16 encoded_terms;
1387   base::string16 encoded_original_query;
1388   EncodeSearchTerms(search_terms_args, is_in_query, &input_encoding,
1389                     &encoded_terms, &encoded_original_query);
1390
1391   std::string old_params;
1392   if (search_term_component == url::Parsed::QUERY) {
1393     old_params = url.query();
1394   } else if (search_term_component == url::Parsed::REF) {
1395     old_params = url.ref();
1396   } else {
1397     DCHECK_EQ(search_term_component, url::Parsed::PATH);
1398     old_params = url.path();
1399   }
1400
1401   std::string new_params(old_params, 0, search_terms_position.begin);
1402   new_params += base::UTF16ToUTF8(encoded_terms);
1403   new_params += old_params.substr(search_terms_position.end());
1404   GURL::Replacements replacements;
1405
1406   if (search_term_component == url::Parsed::QUERY) {
1407     replacements.SetQueryStr(new_params);
1408   } else if (search_term_component == url::Parsed::REF) {
1409     replacements.SetRefStr(new_params);
1410   } else {
1411     DCHECK_EQ(search_term_component, url::Parsed::PATH);
1412     replacements.SetPathStr(new_params);
1413   }
1414
1415   *result = url.ReplaceComponents(replacements);
1416   return true;
1417 }
1418
1419 void TemplateURL::EncodeSearchTerms(
1420     const TemplateURLRef::SearchTermsArgs& search_terms_args,
1421     bool is_in_query,
1422     std::string* input_encoding,
1423     base::string16* encoded_terms,
1424     base::string16* encoded_original_query) const {
1425
1426   std::vector<std::string> encodings(input_encodings());
1427   if (std::find(encodings.begin(), encodings.end(), "UTF-8") == encodings.end())
1428     encodings.push_back("UTF-8");
1429   for (std::vector<std::string>::const_iterator i(encodings.begin());
1430        i != encodings.end(); ++i) {
1431     if (TryEncoding(search_terms_args.search_terms,
1432                     search_terms_args.original_query, i->c_str(),
1433                     is_in_query, encoded_terms, encoded_original_query)) {
1434       *input_encoding = *i;
1435       return;
1436     }
1437   }
1438   NOTREACHED();
1439 }
1440
1441 GURL TemplateURL::GenerateSearchURL(
1442     const SearchTermsData& search_terms_data) const {
1443   if (!url_ref_.IsValid(search_terms_data))
1444     return GURL();
1445
1446   if (!url_ref_.SupportsReplacement(search_terms_data))
1447     return GURL(url());
1448
1449   // Use something obscure for the search terms argument so that in the rare
1450   // case the term replaces the URL it's unlikely another keyword would have the
1451   // same url.
1452   // TODO(jnd): Add additional parameters to get post data when the search URL
1453   // has post parameters.
1454   return GURL(url_ref_.ReplaceSearchTerms(
1455       TemplateURLRef::SearchTermsArgs(
1456           base::ASCIIToUTF16("blah.blah.blah.blah.blah")),
1457       search_terms_data, NULL));
1458 }
1459
1460 void TemplateURL::CopyFrom(const TemplateURL& other) {
1461   if (this == &other)
1462     return;
1463
1464   data_ = other.data_;
1465   url_ref_.InvalidateCachedValues();
1466   suggestions_url_ref_.InvalidateCachedValues();
1467   instant_url_ref_.InvalidateCachedValues();
1468   SetPrepopulateId(other.data_.prepopulate_id);
1469 }
1470
1471 void TemplateURL::SetURL(const std::string& url) {
1472   data_.SetURL(url);
1473   url_ref_.InvalidateCachedValues();
1474 }
1475
1476 void TemplateURL::SetPrepopulateId(int id) {
1477   data_.prepopulate_id = id;
1478   const bool prepopulated = id > 0;
1479   url_ref_.prepopulated_ = prepopulated;
1480   suggestions_url_ref_.prepopulated_ = prepopulated;
1481   instant_url_ref_.prepopulated_ = prepopulated;
1482 }
1483
1484 void TemplateURL::ResetKeywordIfNecessary(
1485     const SearchTermsData& search_terms_data,
1486     bool force) {
1487   if (IsGoogleSearchURLWithReplaceableKeyword(search_terms_data) || force) {
1488     DCHECK(GetType() != OMNIBOX_API_EXTENSION);
1489     GURL url(GenerateSearchURL(search_terms_data));
1490     if (url.is_valid())
1491       data_.SetKeyword(
1492           GenerateKeyword(url, search_terms_data.GetAcceptLanguages()));
1493   }
1494 }
1495
1496 bool TemplateURL::FindSearchTermsInURL(
1497     const GURL& url,
1498     const SearchTermsData& search_terms_data,
1499     base::string16* search_terms,
1500     url::Parsed::ComponentType* search_term_component,
1501     url::Component* search_terms_position) const {
1502   DCHECK(search_terms);
1503   search_terms->clear();
1504
1505   // Try to match with every pattern.
1506   for (size_t i = 0; i < URLCount(); ++i) {
1507     TemplateURLRef ref(this, i);
1508     if (ref.ExtractSearchTermsFromURL(url, search_terms, search_terms_data,
1509         search_term_component, search_terms_position)) {
1510       // If ExtractSearchTermsFromURL() returns true and |search_terms| is empty
1511       // it means the pattern matched but no search terms were present. In this
1512       // case we fail immediately without looking for matches in subsequent
1513       // patterns. This means that given patterns
1514       //    [ "http://foo/#q={searchTerms}", "http://foo/?q={searchTerms}" ],
1515       // calling ExtractSearchTermsFromURL() on "http://foo/?q=bar#q=' would
1516       // return false. This is important for at least Google, where such URLs
1517       // are invalid.
1518       return !search_terms->empty();
1519     }
1520   }
1521   return false;
1522 }