components/search_engines/template_url.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/search_engines/template_url.h"
   6
   7 #include <string>
   8 #include <vector>
   9
  10 #include "base/basictypes.h"
  11 #include "base/command_line.h"
  12 #include "base/format_macros.h"
  13 #include "base/i18n/icu_string_conversions.h"
  14 #include "base/i18n/rtl.h"
  15 #include "base/logging.h"
  16 #include "base/metrics/field_trial.h"
  17 #include "base/rand_util.h"
  18 #include "base/strings/string_number_conversions.h"
  19 #include "base/strings/string_piece.h"
  20 #include "base/strings/string_split.h"
  21 #include "base/strings/string_util.h"
  22 #include "base/strings/stringprintf.h"
  23 #include "base/strings/utf_string_conversions.h"
  24 #include "components/google/core/browser/google_util.h"
  25 #include "components/metrics/proto/omnibox_input_type.pb.h"
  26 #include "components/search_engines/search_engines_switches.h"
  27 #include "components/search_engines/search_terms_data.h"
  28 #include "components/url_formatter/url_formatter.h"
  29 #include "google_apis/google_api_keys.h"
  30 #include "net/base/escape.h"
  31 #include "net/base/mime_util.h"
  32 #include "net/base/net_util.h"
  33 #include "ui/base/device_form_factor.h"
  34 #include "url/gurl.h"
  35
  36 namespace {
  37
  38 // The TemplateURLRef has any number of terms that need to be replaced. Each of
  39 // the terms is enclosed in braces. If the character preceeding the final
  40 // brace is a ?, it indicates the term is optional and can be replaced with
  41 // an empty string.
  42 const char kStartParameter = '{';
  43 const char kEndParameter = '}';
  44 const char kOptional = '?';
  45
  46 // Known parameters found in the URL.
  47 const char kSearchTermsParameter[] = "searchTerms";
  48 const char kSearchTermsParameterFull[] = "{searchTerms}";
  49 const char kSearchTermsParameterFullEscaped[] = "%7BsearchTerms%7D";
  50
  51 // Same as kSearchTermsParameter, with no escaping.
  52 const char kGoogleUnescapedSearchTermsParameter[] =
  53     "google:unescapedSearchTerms";
  54 const char kGoogleUnescapedSearchTermsParameterFull[] =
  55     "{google:unescapedSearchTerms}";
  56
  57 // Display value for kSearchTermsParameter.
  58 const char kDisplaySearchTerms[] = "%s";
  59
  60 // Display value for kGoogleUnescapedSearchTermsParameter.
  61 const char kDisplayUnescapedSearchTerms[] = "%S";
  62
  63 // Used if the count parameter is not optional. Indicates we want 10 search
  64 // results.
  65 const char kDefaultCount[] = "10";
  66
  67 // Used if the output encoding parameter is required.
  68 const char kOutputEncodingType[] = "UTF-8";
  69
  70 // Attempts to encode |terms| and |original_query| in |encoding| and escape
  71 // them.  |terms| may be escaped as path or query depending on |is_in_query|;
  72 // |original_query| is always escaped as query.  Returns whether the encoding
  73 // process succeeded.
  74 bool TryEncoding(const base::string16& terms,
  75                  const base::string16& original_query,
  76                  const char* encoding,
  77                  bool is_in_query,
  78                  base::string16* escaped_terms,
  79                  base::string16* escaped_original_query) {
  80   DCHECK(escaped_terms);
  81   DCHECK(escaped_original_query);
  82   std::string encoded_terms;
  83   if (!base::UTF16ToCodepage(terms, encoding,
  84       base::OnStringConversionError::SKIP, &encoded_terms))
  85     return false;
  86   *escaped_terms = base::UTF8ToUTF16(is_in_query ?
  87       net::EscapeQueryParamValue(encoded_terms, true) :
  88       net::EscapePath(encoded_terms));
  89   if (original_query.empty())
  90     return true;
  91   std::string encoded_original_query;
  92   if (!base::UTF16ToCodepage(original_query, encoding,
  93       base::OnStringConversionError::SKIP, &encoded_original_query))
  94     return false;
  95   *escaped_original_query = base::UTF8ToUTF16(
  96       net::EscapeQueryParamValue(encoded_original_query, true));
  97   return true;
  98 }
  99
 100 // Extract query key and host given a list of parameters coming from the URL
 101 // query or ref.
 102 std::string FindSearchTermsKey(const std::string& params) {
 103   if (params.empty())
 104     return std::string();
 105   url::Component query, key, value;
 106   query.len = static_cast<int>(params.size());
 107   while (url::ExtractQueryKeyValue(params.c_str(), &query, &key, &value)) {
 108     if (key.is_nonempty() && value.is_nonempty()) {
 109       const base::StringPiece value_string(params.c_str() + value.begin,
 110                                            value.len);
 111       if (value_string.find(kSearchTermsParameterFull, 0) !=
 112           base::StringPiece::npos ||
 113           value_string.find(kGoogleUnescapedSearchTermsParameterFull, 0) !=
 114           base::StringPiece::npos) {
 115         return params.substr(key.begin, key.len);
 116       }
 117     }
 118   }
 119   return std::string();
 120 }
 121
 122 // Extract the position of the search terms' parameter in the URL path.
 123 bool FindSearchTermsInPath(const std::string& path,
 124                            url::Component* parameter_position) {
 125   DCHECK(parameter_position);
 126   parameter_position->reset();
 127   const size_t begin = path.find(kSearchTermsParameterFullEscaped);
 128   if (begin == std::string::npos)
 129     return false;
 130   parameter_position->begin = begin;
 131   parameter_position->len = arraysize(kSearchTermsParameterFullEscaped) - 1;
 132   return true;
 133 }
 134
 135 bool IsTemplateParameterString(const std::string& param) {
 136   return (param.length() > 2) && (*(param.begin()) == kStartParameter) &&
 137       (*(param.rbegin()) == kEndParameter);
 138 }
 139
 140 }  // namespace
 141
 142
 143 // TemplateURLRef::SearchTermsArgs --------------------------------------------
 144
 145 TemplateURLRef::SearchTermsArgs::SearchTermsArgs(
 146     const base::string16& search_terms)
 147     : search_terms(search_terms),
 148       input_type(metrics::OmniboxInputType::INVALID),
 149       accepted_suggestion(NO_SUGGESTIONS_AVAILABLE),
 150       cursor_position(base::string16::npos),
 151       enable_omnibox_start_margin(false),
 152       page_classification(metrics::OmniboxEventProto::INVALID_SPEC),
 153       bookmark_bar_pinned(false),
 154       append_extra_query_params(false),
 155       force_instant_results(false),
 156       from_app_list(false),
 157       contextual_search_params(ContextualSearchParams()) {
 158 }
 159
 160 TemplateURLRef::SearchTermsArgs::~SearchTermsArgs() {
 161 }
 162
 163 TemplateURLRef::SearchTermsArgs::ContextualSearchParams::
 164     ContextualSearchParams()
 165     : version(-1),
 166       start(base::string16::npos),
 167       end(base::string16::npos),
 168       resolve(true) {
 169 }
 170
 171 TemplateURLRef::SearchTermsArgs::ContextualSearchParams::
 172     ContextualSearchParams(
 173         const int version,
 174         const std::string& selection,
 175         const std::string& base_page_url,
 176         const bool resolve)
 177     : version(version),
 178       start(base::string16::npos),
 179       end(base::string16::npos),
 180       selection(selection),
 181       base_page_url(base_page_url),
 182       resolve(resolve) {
 183 }
 184
 185 TemplateURLRef::SearchTermsArgs::ContextualSearchParams::
 186     ContextualSearchParams(
 187         const int version,
 188         const size_t start,
 189         const size_t end,
 190         const std::string& selection,
 191         const std::string& content,
 192         const std::string& base_page_url,
 193         const std::string& encoding,
 194         const bool resolve)
 195     : version(version),
 196       start(start),
 197       end(end),
 198       selection(selection),
 199       content(content),
 200       base_page_url(base_page_url),
 201       encoding(encoding),
 202       resolve(resolve) {
 203 }
 204
 205 TemplateURLRef::SearchTermsArgs::ContextualSearchParams::
 206     ~ContextualSearchParams() {
 207 }
 208
 209 // TemplateURLRef -------------------------------------------------------------
 210
 211 TemplateURLRef::TemplateURLRef(const TemplateURL* owner, Type type)
 212     : owner_(owner),
 213       type_(type),
 214       index_in_owner_(0),
 215       parsed_(false),
 216       valid_(false),
 217       supports_replacements_(false),
 218       search_term_position_in_path_(std::string::npos),
 219       search_term_key_location_(url::Parsed::QUERY),
 220       prepopulated_(false) {
 221   DCHECK(owner_);
 222   DCHECK_NE(INDEXED, type_);
 223 }
 224
 225 TemplateURLRef::TemplateURLRef(const TemplateURL* owner, size_t index_in_owner)
 226     : owner_(owner),
 227       type_(INDEXED),
 228       index_in_owner_(index_in_owner),
 229       parsed_(false),
 230       valid_(false),
 231       supports_replacements_(false),
 232       search_term_position_in_path_(std::string::npos),
 233       search_term_key_location_(url::Parsed::QUERY),
 234       prepopulated_(false) {
 235   DCHECK(owner_);
 236   DCHECK_LT(index_in_owner_, owner_->URLCount());
 237 }
 238
 239 TemplateURLRef::~TemplateURLRef() {
 240 }
 241
 242 std::string TemplateURLRef::GetURL() const {
 243   switch (type_) {
 244     case SEARCH:            return owner_->url();
 245     case SUGGEST:           return owner_->suggestions_url();
 246     case INSTANT:           return owner_->instant_url();
 247     case IMAGE:             return owner_->image_url();
 248     case NEW_TAB:           return owner_->new_tab_url();
 249     case CONTEXTUAL_SEARCH: return owner_->contextual_search_url();
 250     case INDEXED:           return owner_->GetURL(index_in_owner_);
 251     default:       NOTREACHED(); return std::string();  // NOLINT
 252   }
 253 }
 254
 255 std::string TemplateURLRef::GetPostParamsString() const {
 256   switch (type_) {
 257     case INDEXED:
 258     case SEARCH:            return owner_->search_url_post_params();
 259     case SUGGEST:           return owner_->suggestions_url_post_params();
 260     case INSTANT:           return owner_->instant_url_post_params();
 261     case NEW_TAB:           return std::string();
 262     case CONTEXTUAL_SEARCH: return std::string();
 263     case IMAGE:             return owner_->image_url_post_params();
 264     default:      NOTREACHED(); return std::string();  // NOLINT
 265   }
 266 }
 267
 268 bool TemplateURLRef::UsesPOSTMethod(
 269     const SearchTermsData& search_terms_data) const {
 270   ParseIfNecessary(search_terms_data);
 271   return !post_params_.empty();
 272 }
 273
 274 bool TemplateURLRef::EncodeFormData(const PostParams& post_params,
 275                                     PostContent* post_content) const {
 276   if (post_params.empty())
 277     return true;
 278   if (!post_content)
 279     return false;
 280
 281   const char kUploadDataMIMEType[] = "multipart/form-data; boundary=";
 282   const char kMultipartBoundary[] = "----+*+----%016" PRIx64 "----+*+----";
 283   // Each name/value pair is stored in a body part which is preceded by a
 284   // boundary delimiter line. Uses random number generator here to create
 285   // a unique boundary delimiter for form data encoding.
 286   std::string boundary = base::StringPrintf(kMultipartBoundary,
 287                                             base::RandUint64());
 288   // Sets the content MIME type.
 289   post_content->first = kUploadDataMIMEType;
 290   post_content->first += boundary;
 291   // Encodes the post parameters.
 292   std::string* post_data = &post_content->second;
 293   post_data->clear();
 294   for (const auto& param : post_params) {
 295     DCHECK(!param.name.empty());
 296     net::AddMultipartValueForUpload(param.name, param.value, boundary,
 297                                     param.content_type, post_data);
 298   }
 299   net::AddMultipartFinalDelimiterForUpload(boundary, post_data);
 300   return true;
 301 }
 302
 303 bool TemplateURLRef::SupportsReplacement(
 304     const SearchTermsData& search_terms_data) const {
 305   ParseIfNecessary(search_terms_data);
 306   return valid_ && supports_replacements_;
 307 }
 308
 309 std::string TemplateURLRef::ReplaceSearchTerms(
 310     const SearchTermsArgs& search_terms_args,
 311     const SearchTermsData& search_terms_data,
 312     PostContent* post_content) const {
 313   ParseIfNecessary(search_terms_data);
 314   if (!valid_)
 315     return std::string();
 316
 317   std::string url(HandleReplacements(search_terms_args, search_terms_data,
 318                                      post_content));
 319
 320   GURL gurl(url);
 321   if (!gurl.is_valid())
 322     return url;
 323
 324   std::vector<std::string> query_params;
 325   if (search_terms_args.append_extra_query_params) {
 326     std::string extra_params(
 327         base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
 328             switches::kExtraSearchQueryParams));
 329     if (!extra_params.empty())
 330       query_params.push_back(extra_params);
 331   }
 332   if (!search_terms_args.suggest_query_params.empty())
 333     query_params.push_back(search_terms_args.suggest_query_params);
 334   if (!gurl.query().empty())
 335     query_params.push_back(gurl.query());
 336
 337   if (query_params.empty())
 338     return url;
 339
 340   GURL::Replacements replacements;
 341   std::string query_str = base::JoinString(query_params, "&");
 342   replacements.SetQueryStr(query_str);
 343   return gurl.ReplaceComponents(replacements).possibly_invalid_spec();
 344 }
 345
 346 bool TemplateURLRef::IsValid(const SearchTermsData& search_terms_data) const {
 347   ParseIfNecessary(search_terms_data);
 348   return valid_;
 349 }
 350
 351 base::string16 TemplateURLRef::DisplayURL(
 352     const SearchTermsData& search_terms_data) const {
 353   ParseIfNecessary(search_terms_data);
 354   std::string result(GetURL());
 355   if (valid_ && !replacements_.empty()) {
 356     base::ReplaceSubstringsAfterOffset(&result, 0,
 357                                        kSearchTermsParameterFull,
 358                                        kDisplaySearchTerms);
 359     base::ReplaceSubstringsAfterOffset(&result, 0,
 360                                        kGoogleUnescapedSearchTermsParameterFull,
 361                                        kDisplayUnescapedSearchTerms);
 362   }
 363   return base::UTF8ToUTF16(result);
 364 }
 365
 366 // static
 367 std::string TemplateURLRef::DisplayURLToURLRef(
 368     const base::string16& display_url) {
 369   std::string result = base::UTF16ToUTF8(display_url);
 370   base::ReplaceSubstringsAfterOffset(&result, 0,
 371                                      kDisplaySearchTerms,
 372                                      kSearchTermsParameterFull);
 373   base::ReplaceSubstringsAfterOffset(&result, 0,
 374                                      kDisplayUnescapedSearchTerms,
 375                                      kGoogleUnescapedSearchTermsParameterFull);
 376   return result;
 377 }
 378
 379 const std::string& TemplateURLRef::GetHost(
 380     const SearchTermsData& search_terms_data) const {
 381   ParseIfNecessary(search_terms_data);
 382   return host_;
 383 }
 384
 385 const std::string& TemplateURLRef::GetPath(
 386     const SearchTermsData& search_terms_data) const {
 387   ParseIfNecessary(search_terms_data);
 388   return path_;
 389 }
 390
 391 const std::string& TemplateURLRef::GetSearchTermKey(
 392     const SearchTermsData& search_terms_data) const {
 393   ParseIfNecessary(search_terms_data);
 394   return search_term_key_;
 395 }
 396
 397 size_t TemplateURLRef::GetSearchTermPositionInPath(
 398     const SearchTermsData& search_terms_data) const {
 399   ParseIfNecessary(search_terms_data);
 400   return search_term_position_in_path_;
 401 }
 402
 403 url::Parsed::ComponentType TemplateURLRef::GetSearchTermKeyLocation(
 404     const SearchTermsData& search_terms_data) const {
 405   ParseIfNecessary(search_terms_data);
 406   return search_term_key_location_;
 407 }
 408
 409 base::string16 TemplateURLRef::SearchTermToString16(
 410     const std::string& term) const {
 411   const std::vector<std::string>& encodings = owner_->input_encodings();
 412   base::string16 result;
 413
 414   net::UnescapeRule::Type unescape_rules =
 415       net::UnescapeRule::SPACES | net::UnescapeRule::URL_SPECIAL_CHARS;
 416   if (search_term_key_location_ != url::Parsed::PATH)
 417     unescape_rules |= net::UnescapeRule::REPLACE_PLUS_WITH_SPACE;
 418
 419   std::string unescaped = net::UnescapeURLComponent(term, unescape_rules);
 420   for (size_t i = 0; i < encodings.size(); ++i) {
 421     if (base::CodepageToUTF16(unescaped, encodings[i].c_str(),
 422                               base::OnStringConversionError::FAIL, &result))
 423       return result;
 424   }
 425
 426   // Always fall back on UTF-8 if it works.
 427   if (base::CodepageToUTF16(unescaped, base::kCodepageUTF8,
 428                             base::OnStringConversionError::FAIL, &result))
 429     return result;
 430
 431   // When nothing worked, just use the escaped text. We have no idea what the
 432   // encoding is. We need to substitute spaces for pluses ourselves since we're
 433   // not sending it through an unescaper.
 434   result = base::UTF8ToUTF16(term);
 435   if (unescape_rules & net::UnescapeRule::REPLACE_PLUS_WITH_SPACE)
 436     std::replace(result.begin(), result.end(), '+', ' ');
 437   return result;
 438 }
 439
 440 bool TemplateURLRef::HasGoogleBaseURLs(
 441     const SearchTermsData& search_terms_data) const {
 442   ParseIfNecessary(search_terms_data);
 443   for (size_t i = 0; i < replacements_.size(); ++i) {
 444     if ((replacements_[i].type == GOOGLE_BASE_URL) ||
 445         (replacements_[i].type == GOOGLE_BASE_SUGGEST_URL))
 446       return true;
 447   }
 448   return false;
 449 }
 450
 451 bool TemplateURLRef::ExtractSearchTermsFromURL(
 452     const GURL& url,
 453     base::string16* search_terms,
 454     const SearchTermsData& search_terms_data,
 455     url::Parsed::ComponentType* search_terms_component,
 456     url::Component* search_terms_position) const {
 457   DCHECK(search_terms);
 458   search_terms->clear();
 459
 460   ParseIfNecessary(search_terms_data);
 461
 462   // We need a search term in the template URL to extract something.
 463   if (search_term_key_.empty() &&
 464       (search_term_key_location_ != url::Parsed::PATH))
 465     return false;
 466
 467   // Fill-in the replacements. We don't care about search terms in the pattern,
 468   // so we use the empty string.
 469   // Currently we assume the search term only shows in URL, not in post params.
 470   GURL pattern(ReplaceSearchTerms(SearchTermsArgs(base::string16()),
 471                                   search_terms_data, NULL));
 472   // Host, path and port must match.
 473   if ((url.port() != pattern.port()) ||
 474       (url.host() != host_) ||
 475       ((url.path() != path_) &&
 476           (search_term_key_location_ != url::Parsed::PATH))) {
 477     return false;
 478   }
 479
 480   std::string source;
 481   url::Component position;
 482
 483   if (search_term_key_location_ == url::Parsed::PATH) {
 484     source = url.path();
 485
 486     // Characters in the path before and after search terms must match.
 487     if (source.length() < path_.length())
 488       return false;
 489     position.begin = search_term_position_in_path_;
 490     position.len = source.length() - path_.length();
 491     if (source.substr(0, position.begin) + source.substr(position.end()) !=
 492         path_)
 493       return false;
 494   } else {
 495     DCHECK(search_term_key_location_ == url::Parsed::QUERY ||
 496            search_term_key_location_ == url::Parsed::REF);
 497     source = (search_term_key_location_ == url::Parsed::QUERY) ?
 498         url.query() : url.ref();
 499
 500     url::Component query, key, value;
 501     query.len = static_cast<int>(source.size());
 502     bool key_found = false;
 503     while (url::ExtractQueryKeyValue(source.c_str(), &query, &key, &value)) {
 504       if (key.is_nonempty()) {
 505         if (source.substr(key.begin, key.len) == search_term_key_) {
 506           // Fail if search term key is found twice.
 507           if (key_found)
 508             return false;
 509           key_found = true;
 510           position = value;
 511         }
 512       }
 513     }
 514     if (!key_found)
 515       return false;
 516   }
 517
 518   // Extract the search term.
 519   *search_terms = SearchTermToString16(
 520       source.substr(position.begin, position.len));
 521   if (search_terms_component)
 522     *search_terms_component = search_term_key_location_;
 523   if (search_terms_position)
 524     *search_terms_position = position;
 525   return true;
 526 }
 527
 528 void TemplateURLRef::InvalidateCachedValues() const {
 529   supports_replacements_ = valid_ = parsed_ = false;
 530   host_.clear();
 531   path_.clear();
 532   search_term_key_.clear();
 533   search_term_position_in_path_ = std::string::npos;
 534   search_term_key_location_ = url::Parsed::QUERY;
 535   replacements_.clear();
 536   post_params_.clear();
 537 }
 538
 539 bool TemplateURLRef::ParseParameter(size_t start,
 540                                     size_t end,
 541                                     std::string* url,
 542                                     Replacements* replacements) const {
 543   DCHECK(start != std::string::npos &&
 544          end != std::string::npos && end > start);
 545   size_t length = end - start - 1;
 546   bool optional = false;
 547   // Make a copy of |url| that can be referenced in StringPieces below. |url| is
 548   // modified, so that can't be used in StringPiece.
 549   const std::string original_url(*url);
 550   if (original_url[end - 1] == kOptional) {
 551     optional = true;
 552     length--;
 553   }
 554
 555   const base::StringPiece parameter(original_url.begin() + start + 1,
 556                                     original_url.begin() + start + 1 + length);
 557   const base::StringPiece full_parameter(original_url.begin() + start,
 558                                          original_url.begin() + end + 1);
 559   // Remove the parameter from the string.  For parameters who replacement is
 560   // constant and already known, just replace them directly.  For other cases,
 561   // like parameters whose values may change over time, use |replacements|.
 562   url->erase(start, end - start + 1);
 563   if (parameter == kSearchTermsParameter) {
 564     replacements->push_back(Replacement(SEARCH_TERMS, start));
 565   } else if (parameter == "count") {
 566     if (!optional)
 567       url->insert(start, kDefaultCount);
 568   } else if (parameter == "google:assistedQueryStats") {
 569     replacements->push_back(Replacement(GOOGLE_ASSISTED_QUERY_STATS, start));
 570   } else if (parameter == "google:baseURL") {
 571     replacements->push_back(Replacement(GOOGLE_BASE_URL, start));
 572   } else if (parameter == "google:baseSuggestURL") {
 573     replacements->push_back(Replacement(GOOGLE_BASE_SUGGEST_URL, start));
 574   } else if (parameter == "google:bookmarkBarPinned") {
 575     replacements->push_back(Replacement(GOOGLE_BOOKMARK_BAR_PINNED, start));
 576   } else if (parameter == "google:currentPageUrl") {
 577     replacements->push_back(Replacement(GOOGLE_CURRENT_PAGE_URL, start));
 578   } else if (parameter == "google:cursorPosition") {
 579     replacements->push_back(Replacement(GOOGLE_CURSOR_POSITION, start));
 580   } else if (parameter == "google:forceInstantResults") {
 581     replacements->push_back(Replacement(GOOGLE_FORCE_INSTANT_RESULTS, start));
 582   } else if (parameter == "google:imageOriginalHeight") {
 583     replacements->push_back(
 584         Replacement(TemplateURLRef::GOOGLE_IMAGE_ORIGINAL_HEIGHT, start));
 585   } else if (parameter == "google:imageOriginalWidth") {
 586     replacements->push_back(
 587         Replacement(TemplateURLRef::GOOGLE_IMAGE_ORIGINAL_WIDTH, start));
 588   } else if (parameter == "google:imageSearchSource") {
 589     replacements->push_back(
 590         Replacement(TemplateURLRef::GOOGLE_IMAGE_SEARCH_SOURCE, start));
 591   } else if (parameter == "google:imageThumbnail") {
 592     replacements->push_back(
 593         Replacement(TemplateURLRef::GOOGLE_IMAGE_THUMBNAIL, start));
 594   } else if (parameter == "google:imageURL") {
 595     replacements->push_back(Replacement(TemplateURLRef::GOOGLE_IMAGE_URL,
 596                                         start));
 597   } else if (parameter == "google:inputType") {
 598     replacements->push_back(Replacement(TemplateURLRef::GOOGLE_INPUT_TYPE,
 599                                         start));
 600   } else if (parameter == "google:instantExtendedEnabledParameter") {
 601     replacements->push_back(Replacement(GOOGLE_INSTANT_EXTENDED_ENABLED,
 602                                         start));
 603   } else if (parameter == "google:instantExtendedEnabledKey") {
 604     url->insert(start, google_util::kInstantExtendedAPIParam);
 605   } else if (parameter == "google:ntpIsThemedParameter") {
 606     replacements->push_back(Replacement(GOOGLE_NTP_IS_THEMED, start));
 607   } else if (parameter == "google:omniboxStartMarginParameter") {
 608     replacements->push_back(Replacement(GOOGLE_OMNIBOX_START_MARGIN, start));
 609   } else if (parameter == "google:contextualSearchVersion") {
 610     replacements->push_back(
 611         Replacement(GOOGLE_CONTEXTUAL_SEARCH_VERSION, start));
 612   } else if (parameter == "google:contextualSearchContextData") {
 613     replacements->push_back(
 614         Replacement(GOOGLE_CONTEXTUAL_SEARCH_CONTEXT_DATA, start));
 615   } else if (parameter == "google:originalQueryForSuggestion") {
 616     replacements->push_back(Replacement(GOOGLE_ORIGINAL_QUERY_FOR_SUGGESTION,
 617                                         start));
 618   } else if (parameter == "google:pageClassification") {
 619     replacements->push_back(Replacement(GOOGLE_PAGE_CLASSIFICATION, start));
 620   } else if (parameter == "google:prefetchQuery") {
 621     replacements->push_back(Replacement(GOOGLE_PREFETCH_QUERY, start));
 622   } else if (parameter == "google:RLZ") {
 623     replacements->push_back(Replacement(GOOGLE_RLZ, start));
 624   } else if (parameter == "google:searchClient") {
 625     replacements->push_back(Replacement(GOOGLE_SEARCH_CLIENT, start));
 626   } else if (parameter == "google:searchFieldtrialParameter") {
 627     replacements->push_back(Replacement(GOOGLE_SEARCH_FIELDTRIAL_GROUP, start));
 628   } else if (parameter == "google:searchVersion") {
 629     replacements->push_back(Replacement(GOOGLE_SEARCH_VERSION, start));
 630   } else if (parameter == "google:sessionToken") {
 631     replacements->push_back(Replacement(GOOGLE_SESSION_TOKEN, start));
 632   } else if (parameter == "google:sourceId") {
 633 #if defined(OS_ANDROID) || defined(OS_IOS)
 634     url->insert(start, "sourceid=chrome-mobile&");
 635 #else
 636     url->insert(start, "sourceid=chrome&");
 637 #endif
 638   } else if (parameter == "google:suggestAPIKeyParameter") {
 639     url->insert(start,
 640                 net::EscapeQueryParamValue(google_apis::GetAPIKey(), false));
 641   } else if (parameter == "google:suggestClient") {
 642     replacements->push_back(Replacement(GOOGLE_SUGGEST_CLIENT, start));
 643   } else if (parameter == "google:suggestRid") {
 644     replacements->push_back(Replacement(GOOGLE_SUGGEST_REQUEST_ID, start));
 645   } else if (parameter == kGoogleUnescapedSearchTermsParameter) {
 646     replacements->push_back(Replacement(GOOGLE_UNESCAPED_SEARCH_TERMS, start));
 647   } else if (parameter == "yandex:searchPath") {
 648     switch (ui::GetDeviceFormFactor()) {
 649       case ui::DEVICE_FORM_FACTOR_DESKTOP:
 650         url->insert(start, "yandsearch");
 651         break;
 652       case ui::DEVICE_FORM_FACTOR_PHONE:
 653         url->insert(start, "touchsearch");
 654         break;
 655       case ui::DEVICE_FORM_FACTOR_TABLET:
 656         url->insert(start, "padsearch");
 657         break;
 658     }
 659   } else if (parameter == "inputEncoding") {
 660     replacements->push_back(Replacement(ENCODING, start));
 661   } else if (parameter == "language") {
 662     replacements->push_back(Replacement(LANGUAGE, start));
 663   } else if (parameter == "outputEncoding") {
 664     if (!optional)
 665       url->insert(start, kOutputEncodingType);
 666   } else if ((parameter == "startIndex") || (parameter == "startPage")) {
 667     // We don't support these.
 668     if (!optional)
 669       url->insert(start, "1");
 670   } else if (!prepopulated_) {
 671     // If it's a prepopulated URL, we know that it's safe to remove unknown
 672     // parameters, so just ignore this and return true below. Otherwise it could
 673     // be some garbage but can also be a javascript block. Put it back.
 674     url->insert(start, full_parameter.data(), full_parameter.size());
 675     return false;
 676   }
 677   return true;
 678 }
 679
 680 std::string TemplateURLRef::ParseURL(const std::string& url,
 681                                      Replacements* replacements,
 682                                      PostParams* post_params,
 683                                      bool* valid) const {
 684   *valid = false;
 685   std::string parsed_url = url;
 686   for (size_t last = 0; last != std::string::npos; ) {
 687     last = parsed_url.find(kStartParameter, last);
 688     if (last != std::string::npos) {
 689       size_t template_end = parsed_url.find(kEndParameter, last);
 690       if (template_end != std::string::npos) {
 691         // Since we allow Javascript in the URL, {} pairs could be nested. Match
 692         // only leaf pairs with supported parameters.
 693         size_t next_template_start = parsed_url.find(kStartParameter, last + 1);
 694         if (next_template_start == std::string::npos ||
 695             next_template_start > template_end) {
 696           // If successful, ParseParameter erases from the string as such no
 697           // need to update |last|. If failed, move |last| to the end of pair.
 698           if (!ParseParameter(last, template_end, &parsed_url, replacements)) {
 699             // |template_end| + 1 may be beyond the end of the string.
 700             last = template_end;
 701           }
 702         } else {
 703           last = next_template_start;
 704         }
 705       } else {
 706         // Open brace without a closing brace, return.
 707         return std::string();
 708       }
 709     }
 710   }
 711
 712   // Handles the post parameters.
 713   const std::string& post_params_string = GetPostParamsString();
 714   if (!post_params_string.empty()) {
 715     for (const base::StringPiece& cur : base::SplitStringPiece(
 716              post_params_string, ",",
 717              base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL)) {
 718       // The '=' delimiter is required and the name must be not empty.
 719       std::vector<std::string> parts = base::SplitString(
 720           cur, "=", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
 721       if ((parts.size() != 2U) || parts[0].empty())
 722         return std::string();
 723
 724       std::string& value = parts[1];
 725       size_t replacements_size = replacements->size();
 726       if (IsTemplateParameterString(value))
 727         ParseParameter(0, value.length() - 1, &value, replacements);
 728       PostParam param = { parts[0], value };
 729       post_params->push_back(param);
 730       // If there was a replacement added, points its index to last added
 731       // PostParam.
 732       if (replacements->size() > replacements_size) {
 733         DCHECK_EQ(replacements_size + 1, replacements->size());
 734         Replacement* r = &replacements->back();
 735         r->is_post_param = true;
 736         r->index = post_params->size() - 1;
 737       }
 738     }
 739     DCHECK(!post_params->empty());
 740   }
 741
 742   *valid = true;
 743   return parsed_url;
 744 }
 745
 746 void TemplateURLRef::ParseIfNecessary(
 747     const SearchTermsData& search_terms_data) const {
 748   if (!parsed_) {
 749     InvalidateCachedValues();
 750     parsed_ = true;
 751     parsed_url_ = ParseURL(GetURL(), &replacements_, &post_params_, &valid_);
 752     supports_replacements_ = false;
 753     if (valid_) {
 754       bool has_only_one_search_term = false;
 755       for (Replacements::const_iterator i = replacements_.begin();
 756            i != replacements_.end(); ++i) {
 757         if ((i->type == SEARCH_TERMS) ||
 758             (i->type == GOOGLE_UNESCAPED_SEARCH_TERMS)) {
 759           if (has_only_one_search_term) {
 760             has_only_one_search_term = false;
 761             break;
 762           }
 763           has_only_one_search_term = true;
 764           supports_replacements_ = true;
 765         }
 766       }
 767       // Only parse the host/key if there is one search term. Technically there
 768       // could be more than one term, but it's uncommon; so we punt.
 769       if (has_only_one_search_term)
 770         ParseHostAndSearchTermKey(search_terms_data);
 771     }
 772   }
 773 }
 774
 775 void TemplateURLRef::ParseHostAndSearchTermKey(
 776     const SearchTermsData& search_terms_data) const {
 777   std::string url_string(GetURL());
 778   base::ReplaceSubstringsAfterOffset(
 779       &url_string, 0, "{google:baseURL}",
 780       search_terms_data.GoogleBaseURLValue());
 781   base::ReplaceSubstringsAfterOffset(
 782       &url_string, 0, "{google:baseSuggestURL}",
 783       search_terms_data.GoogleBaseSuggestURLValue());
 784
 785   search_term_key_.clear();
 786   search_term_position_in_path_ = std::string::npos;
 787   host_.clear();
 788   path_.clear();
 789   search_term_key_location_ = url::Parsed::QUERY;
 790
 791   GURL url(url_string);
 792   if (!url.is_valid())
 793     return;
 794
 795   std::string query_key = FindSearchTermsKey(url.query());
 796   std::string ref_key = FindSearchTermsKey(url.ref());
 797   url::Component parameter_position;
 798   const bool in_query = !query_key.empty();
 799   const bool in_ref = !ref_key.empty();
 800   const bool in_path = FindSearchTermsInPath(url.path(), &parameter_position);
 801   if (in_query ? (in_ref || in_path) : (in_ref == in_path))
 802     return;  // No key or multiple keys found.  We only handle having one key.
 803
 804   host_ = url.host();
 805   path_ = url.path();
 806   if (in_query) {
 807     search_term_key_ = query_key;
 808     search_term_key_location_ = url::Parsed::QUERY;
 809   } else if (in_ref) {
 810     search_term_key_ = ref_key;
 811     search_term_key_location_ = url::Parsed::REF;
 812   } else {
 813     DCHECK(in_path);
 814     DCHECK_GE(parameter_position.begin, 1);  // Path must start with '/'.
 815     search_term_key_location_ = url::Parsed::PATH;
 816     search_term_position_in_path_ = parameter_position.begin;
 817     // Remove the "{searchTerms}" itself from |path_|.
 818     path_.erase(parameter_position.begin, parameter_position.len);
 819   }
 820 }
 821
 822 void TemplateURLRef::HandleReplacement(const std::string& name,
 823                                        const std::string& value,
 824                                        const Replacement& replacement,
 825                                        std::string* url) const {
 826   size_t pos = replacement.index;
 827   if (replacement.is_post_param) {
 828     DCHECK_LT(pos, post_params_.size());
 829     DCHECK(!post_params_[pos].name.empty());
 830     post_params_[pos].value = value;
 831   } else {
 832     url->insert(pos, name.empty() ? value : (name + "=" + value + "&"));
 833   }
 834 }
 835
 836 std::string TemplateURLRef::HandleReplacements(
 837     const SearchTermsArgs& search_terms_args,
 838     const SearchTermsData& search_terms_data,
 839     PostContent* post_content) const {
 840   if (replacements_.empty()) {
 841     if (!post_params_.empty())
 842       EncodeFormData(post_params_, post_content);
 843     return parsed_url_;
 844   }
 845
 846   // Determine if the search terms are in the query or before. We're escaping
 847   // space as '+' in the former case and as '%20' in the latter case.
 848   bool is_in_query = true;
 849   for (Replacements::iterator i = replacements_.begin();
 850        i != replacements_.end(); ++i) {
 851     if (i->type == SEARCH_TERMS) {
 852       base::string16::size_type query_start = parsed_url_.find('?');
 853       is_in_query = query_start != base::string16::npos &&
 854           (static_cast<base::string16::size_type>(i->index) > query_start);
 855       break;
 856     }
 857   }
 858
 859   std::string input_encoding;
 860   base::string16 encoded_terms;
 861   base::string16 encoded_original_query;
 862   owner_->EncodeSearchTerms(search_terms_args, is_in_query, &input_encoding,
 863                             &encoded_terms, &encoded_original_query);
 864
 865   std::string url = parsed_url_;
 866
 867   // replacements_ is ordered in ascending order, as such we need to iterate
 868   // from the back.
 869   for (Replacements::reverse_iterator i = replacements_.rbegin();
 870        i != replacements_.rend(); ++i) {
 871     switch (i->type) {
 872       case ENCODING:
 873         HandleReplacement(std::string(), input_encoding, *i, &url);
 874         break;
 875
 876       case GOOGLE_ASSISTED_QUERY_STATS:
 877         DCHECK(!i->is_post_param);
 878         if (!search_terms_args.assisted_query_stats.empty()) {
 879           // Get the base URL without substituting AQS to avoid infinite
 880           // recursion.  We need the URL to find out if it meets all
 881           // AQS requirements (e.g. HTTPS protocol check).
 882           // See TemplateURLRef::SearchTermsArgs for more details.
 883           SearchTermsArgs search_terms_args_without_aqs(search_terms_args);
 884           search_terms_args_without_aqs.assisted_query_stats.clear();
 885           GURL base_url(ReplaceSearchTerms(
 886               search_terms_args_without_aqs, search_terms_data, NULL));
 887           if (base_url.SchemeIsCryptographic()) {
 888             HandleReplacement(
 889                 "aqs", search_terms_args.assisted_query_stats, *i, &url);
 890           }
 891         }
 892         break;
 893
 894       case GOOGLE_BASE_URL:
 895         DCHECK(!i->is_post_param);
 896         HandleReplacement(
 897             std::string(), search_terms_data.GoogleBaseURLValue(), *i, &url);
 898         break;
 899
 900       case GOOGLE_BASE_SUGGEST_URL:
 901         DCHECK(!i->is_post_param);
 902         HandleReplacement(
 903             std::string(), search_terms_data.GoogleBaseSuggestURLValue(), *i,
 904             &url);
 905         break;
 906
 907       case GOOGLE_BOOKMARK_BAR_PINNED:
 908         if (search_terms_data.IsShowingSearchTermsOnSearchResultsPages()) {
 909           // Log whether the bookmark bar is pinned when the user is seeing
 910           // InstantExtended on the SRP.
 911           DCHECK(!i->is_post_param);
 912           HandleReplacement(
 913               "bmbp", search_terms_args.bookmark_bar_pinned ? "1" : "0", *i,
 914               &url);
 915         }
 916         break;
 917
 918       case GOOGLE_CURRENT_PAGE_URL:
 919         DCHECK(!i->is_post_param);
 920         if (!search_terms_args.current_page_url.empty()) {
 921           const std::string& escaped_current_page_url =
 922               net::EscapeQueryParamValue(search_terms_args.current_page_url,
 923                                          true);
 924           HandleReplacement("url", escaped_current_page_url, *i, &url);
 925         }
 926         break;
 927
 928       case GOOGLE_CURSOR_POSITION:
 929         DCHECK(!i->is_post_param);
 930         if (search_terms_args.cursor_position != base::string16::npos)
 931           HandleReplacement(
 932               "cp",
 933               base::StringPrintf("%" PRIuS, search_terms_args.cursor_position),
 934               *i,
 935               &url);
 936         break;
 937
 938       case GOOGLE_FORCE_INSTANT_RESULTS:
 939         DCHECK(!i->is_post_param);
 940         HandleReplacement(std::string(),
 941                           search_terms_data.ForceInstantResultsParam(
 942                               search_terms_args.force_instant_results),
 943                           *i,
 944                           &url);
 945         break;
 946
 947       case GOOGLE_INPUT_TYPE:
 948         DCHECK(!i->is_post_param);
 949         HandleReplacement(
 950             "oit", base::IntToString(search_terms_args.input_type), *i, &url);
 951         break;
 952
 953       case GOOGLE_INSTANT_EXTENDED_ENABLED:
 954         DCHECK(!i->is_post_param);
 955         HandleReplacement(std::string(),
 956                           search_terms_data.InstantExtendedEnabledParam(
 957                               type_ == SEARCH),
 958                           *i,
 959                           &url);
 960         break;
 961
 962       case GOOGLE_NTP_IS_THEMED:
 963         DCHECK(!i->is_post_param);
 964         HandleReplacement(
 965             std::string(), search_terms_data.NTPIsThemedParam(), *i, &url);
 966         break;
 967
 968       case GOOGLE_OMNIBOX_START_MARGIN:
 969         DCHECK(!i->is_post_param);
 970         if (search_terms_args.enable_omnibox_start_margin) {
 971           int omnibox_start_margin = search_terms_data.OmniboxStartMargin();
 972           if (omnibox_start_margin >= 0) {
 973             HandleReplacement("es_sm", base::IntToString(omnibox_start_margin),
 974                               *i, &url);
 975           }
 976         }
 977         break;
 978
 979       case GOOGLE_CONTEXTUAL_SEARCH_VERSION:
 980         if (search_terms_args.contextual_search_params.version >= 0) {
 981           HandleReplacement(
 982               "ctxs",
 983               base::IntToString(
 984                   search_terms_args.contextual_search_params.version),
 985               *i,
 986               &url);
 987         }
 988         break;
 989
 990       case GOOGLE_CONTEXTUAL_SEARCH_CONTEXT_DATA: {
 991         DCHECK(!i->is_post_param);
 992         std::string context_data;
 993
 994         const SearchTermsArgs::ContextualSearchParams& params =
 995             search_terms_args.contextual_search_params;
 996
 997         if (params.start != std::string::npos) {
 998           context_data.append("ctxs_start=" + base::IntToString(
 999               params.start) + "&");
1000         }
1001
1002         if (params.end != std::string::npos) {
1003           context_data.append("ctxs_end=" + base::IntToString(
1004               params.end) + "&");
1005         }
1006
1007         if (!params.selection.empty())
1008           context_data.append("q=" + params.selection + "&");
1009
1010         if (!params.content.empty())
1011           context_data.append("ctxs_content=" + params.content + "&");
1012
1013         if (!params.base_page_url.empty())
1014           context_data.append("ctxsl_url=" + params.base_page_url + "&");
1015
1016         if (!params.encoding.empty()) {
1017           context_data.append("ctxs_encoding=" + params.encoding + "&");
1018         }
1019
1020         context_data.append(
1021             params.resolve ? "ctxsl_resolve=1" : "ctxsl_resolve=0");
1022
1023         HandleReplacement(std::string(), context_data, *i, &url);
1024         break;
1025       }
1026
1027       case GOOGLE_ORIGINAL_QUERY_FOR_SUGGESTION:
1028         DCHECK(!i->is_post_param);
1029         if (search_terms_args.accepted_suggestion >= 0 ||
1030             !search_terms_args.assisted_query_stats.empty()) {
1031           HandleReplacement(
1032               "oq", base::UTF16ToUTF8(encoded_original_query), *i, &url);
1033         }
1034         break;
1035
1036       case GOOGLE_PAGE_CLASSIFICATION:
1037         if (search_terms_args.page_classification !=
1038             metrics::OmniboxEventProto::INVALID_SPEC) {
1039           HandleReplacement(
1040               "pgcl", base::IntToString(search_terms_args.page_classification),
1041               *i, &url);
1042         }
1043         break;
1044
1045       case GOOGLE_PREFETCH_QUERY: {
1046         const std::string& query = search_terms_args.prefetch_query;
1047         const std::string& type = search_terms_args.prefetch_query_type;
1048         if (!query.empty() && !type.empty()) {
1049           HandleReplacement(
1050               std::string(), "pfq=" + query + "&qha=" + type + "&", *i, &url);
1051         }
1052         break;
1053       }
1054
1055       case GOOGLE_RLZ: {
1056         DCHECK(!i->is_post_param);
1057         // On platforms that don't have RLZ, we still want this branch
1058         // to happen so that we replace the RLZ template with the
1059         // empty string.  (If we don't handle this case, we hit a
1060         // NOTREACHED below.)
1061         base::string16 rlz_string = search_terms_data.GetRlzParameterValue(
1062             search_terms_args.from_app_list);
1063         if (!rlz_string.empty()) {
1064           HandleReplacement("rlz", base::UTF16ToUTF8(rlz_string), *i, &url);
1065         }
1066         break;
1067       }
1068
1069       case GOOGLE_SEARCH_CLIENT: {
1070         DCHECK(!i->is_post_param);
1071         std::string client = search_terms_data.GetSearchClient();
1072         if (!client.empty())
1073           HandleReplacement("client", client, *i, &url);
1074         break;
1075       }
1076
1077       case GOOGLE_SEARCH_FIELDTRIAL_GROUP:
1078         // We are not currently running any fieldtrials that modulate the search
1079         // url.  If we do, then we'd have some conditional insert such as:
1080         // url.insert(i->index, used_www ? "gcx=w&" : "gcx=c&");
1081         break;
1082
1083       case GOOGLE_SEARCH_VERSION:
1084         HandleReplacement("gs_rn", "42", *i, &url);
1085         break;
1086
1087       case GOOGLE_SESSION_TOKEN: {
1088         std::string token = search_terms_args.session_token;
1089         if (!token.empty())
1090           HandleReplacement("psi", token, *i, &url);
1091         break;
1092       }
1093
1094       case GOOGLE_SUGGEST_CLIENT:
1095         HandleReplacement(
1096             std::string(), search_terms_data.GetSuggestClient(), *i, &url);
1097         break;
1098
1099       case GOOGLE_SUGGEST_REQUEST_ID:
1100         HandleReplacement(
1101             std::string(), search_terms_data.GetSuggestRequestIdentifier(), *i,
1102             &url);
1103         break;
1104
1105       case GOOGLE_UNESCAPED_SEARCH_TERMS: {
1106         std::string unescaped_terms;
1107         base::UTF16ToCodepage(search_terms_args.search_terms,
1108                               input_encoding.c_str(),
1109                               base::OnStringConversionError::SKIP,
1110                               &unescaped_terms);
1111         HandleReplacement(std::string(), unescaped_terms, *i, &url);
1112         break;
1113       }
1114
1115       case LANGUAGE:
1116         HandleReplacement(
1117             std::string(), search_terms_data.GetApplicationLocale(), *i, &url);
1118         break;
1119
1120       case SEARCH_TERMS:
1121         HandleReplacement(
1122             std::string(), base::UTF16ToUTF8(encoded_terms), *i, &url);
1123         break;
1124
1125       case GOOGLE_IMAGE_THUMBNAIL:
1126         HandleReplacement(
1127             std::string(), search_terms_args.image_thumbnail_content, *i, &url);
1128         post_params_[i->index].content_type = "image/jpeg";
1129         break;
1130
1131       case GOOGLE_IMAGE_URL:
1132         if (search_terms_args.image_url.is_valid()) {
1133           HandleReplacement(
1134               std::string(), search_terms_args.image_url.spec(), *i, &url);
1135         }
1136         break;
1137
1138       case GOOGLE_IMAGE_ORIGINAL_WIDTH:
1139         if (!search_terms_args.image_original_size.IsEmpty()) {
1140           HandleReplacement(
1141               std::string(),
1142               base::IntToString(search_terms_args.image_original_size.width()),
1143               *i, &url);
1144         }
1145         break;
1146
1147       case GOOGLE_IMAGE_ORIGINAL_HEIGHT:
1148         if (!search_terms_args.image_original_size.IsEmpty()) {
1149           HandleReplacement(
1150               std::string(),
1151               base::IntToString(search_terms_args.image_original_size.height()),
1152               *i, &url);
1153         }
1154         break;
1155
1156       case GOOGLE_IMAGE_SEARCH_SOURCE:
1157         HandleReplacement(
1158             std::string(), search_terms_data.GoogleImageSearchSource(), *i,
1159             &url);
1160         break;
1161
1162       default:
1163         NOTREACHED();
1164         break;
1165     }
1166   }
1167
1168   if (!post_params_.empty())
1169     EncodeFormData(post_params_, post_content);
1170
1171   return url;
1172 }
1173
1174
1175 // TemplateURL ----------------------------------------------------------------
1176
1177 TemplateURL::AssociatedExtensionInfo::AssociatedExtensionInfo(
1178     Type type,
1179     const std::string& extension_id)
1180     : type(type),
1181       extension_id(extension_id),
1182       wants_to_be_default_engine(false) {
1183   DCHECK_NE(NORMAL, type);
1184 }
1185
1186 TemplateURL::AssociatedExtensionInfo::~AssociatedExtensionInfo() {
1187 }
1188
1189 TemplateURL::TemplateURL(const TemplateURLData& data)
1190     : data_(data),
1191       url_ref_(this, TemplateURLRef::SEARCH),
1192       suggestions_url_ref_(this,
1193                            TemplateURLRef::SUGGEST),
1194       instant_url_ref_(this,
1195                        TemplateURLRef::INSTANT),
1196       image_url_ref_(this, TemplateURLRef::IMAGE),
1197       new_tab_url_ref_(this, TemplateURLRef::NEW_TAB),
1198       contextual_search_url_ref_(this, TemplateURLRef::CONTEXTUAL_SEARCH) {
1199   SetPrepopulateId(data_.prepopulate_id);
1200
1201   if (data_.search_terms_replacement_key ==
1202       "{google:instantExtendedEnabledKey}") {
1203     data_.search_terms_replacement_key = google_util::kInstantExtendedAPIParam;
1204   }
1205 }
1206
1207 TemplateURL::~TemplateURL() {
1208 }
1209
1210 // static
1211 base::string16 TemplateURL::GenerateKeyword(
1212     const GURL& url,
1213     const std::string& accept_languages) {
1214   DCHECK(url.is_valid());
1215   // Strip "www." off the front of the keyword; otherwise the keyword won't work
1216   // properly.  See http://code.google.com/p/chromium/issues/detail?id=6984 .
1217   // |url|'s hostname may be IDN-encoded. Before generating |keyword| from it,
1218   // convert to Unicode using the user's accept-languages, so it won't look like
1219   // a confusing punycode string.
1220   base::string16 keyword =
1221       net::StripWWW(url_formatter::IDNToUnicode(url.host(), accept_languages));
1222   // Special case: if the host was exactly "www." (not sure this can happen but
1223   // perhaps with some weird intranet and custom DNS server?), ensure we at
1224   // least don't return the empty string.
1225   return keyword.empty() ? base::ASCIIToUTF16("www") : keyword;
1226 }
1227
1228 // static
1229 GURL TemplateURL::GenerateFaviconURL(const GURL& url) {
1230   DCHECK(url.is_valid());
1231   GURL::Replacements rep;
1232
1233   const char favicon_path[] = "/favicon.ico";
1234   int favicon_path_len = arraysize(favicon_path) - 1;
1235
1236   rep.SetPath(favicon_path, url::Component(0, favicon_path_len));
1237   rep.ClearUsername();
1238   rep.ClearPassword();
1239   rep.ClearQuery();
1240   rep.ClearRef();
1241   return url.ReplaceComponents(rep);
1242 }
1243
1244 // static
1245 bool TemplateURL::MatchesData(const TemplateURL* t_url,
1246                               const TemplateURLData* data,
1247                               const SearchTermsData& search_terms_data) {
1248   if (!t_url || !data)
1249     return !t_url && !data;
1250
1251   return (t_url->short_name() == data->short_name()) &&
1252       t_url->HasSameKeywordAs(*data, search_terms_data) &&
1253       (t_url->url() == data->url()) &&
1254       (t_url->suggestions_url() == data->suggestions_url) &&
1255       (t_url->instant_url() == data->instant_url) &&
1256       (t_url->image_url() == data->image_url) &&
1257       (t_url->new_tab_url() == data->new_tab_url) &&
1258       (t_url->search_url_post_params() == data->search_url_post_params) &&
1259       (t_url->suggestions_url_post_params() ==
1260           data->suggestions_url_post_params) &&
1261       (t_url->instant_url_post_params() == data->instant_url_post_params) &&
1262       (t_url->image_url_post_params() == data->image_url_post_params) &&
1263       (t_url->favicon_url() == data->favicon_url) &&
1264       (t_url->safe_for_autoreplace() == data->safe_for_autoreplace) &&
1265       (t_url->show_in_default_list() == data->show_in_default_list) &&
1266       (t_url->input_encodings() == data->input_encodings) &&
1267       (t_url->alternate_urls() == data->alternate_urls) &&
1268       (t_url->search_terms_replacement_key() ==
1269           data->search_terms_replacement_key);
1270 }
1271
1272 base::string16 TemplateURL::AdjustedShortNameForLocaleDirection() const {
1273   base::string16 bidi_safe_short_name = data_.short_name();
1274   base::i18n::AdjustStringForLocaleDirection(&bidi_safe_short_name);
1275   return bidi_safe_short_name;
1276 }
1277
1278 bool TemplateURL::ShowInDefaultList(
1279     const SearchTermsData& search_terms_data) const {
1280   return data_.show_in_default_list &&
1281       url_ref_.SupportsReplacement(search_terms_data);
1282 }
1283
1284 bool TemplateURL::SupportsReplacement(
1285     const SearchTermsData& search_terms_data) const {
1286   return url_ref_.SupportsReplacement(search_terms_data);
1287 }
1288
1289 bool TemplateURL::HasGoogleBaseURLs(
1290     const SearchTermsData& search_terms_data) const {
1291   return url_ref_.HasGoogleBaseURLs(search_terms_data) ||
1292       suggestions_url_ref_.HasGoogleBaseURLs(search_terms_data) ||
1293       instant_url_ref_.HasGoogleBaseURLs(search_terms_data) ||
1294       image_url_ref_.HasGoogleBaseURLs(search_terms_data) ||
1295       new_tab_url_ref_.HasGoogleBaseURLs(search_terms_data);
1296 }
1297
1298 bool TemplateURL::IsGoogleSearchURLWithReplaceableKeyword(
1299     const SearchTermsData& search_terms_data) const {
1300   return (GetType() == NORMAL) &&
1301       url_ref_.HasGoogleBaseURLs(search_terms_data) &&
1302       google_util::IsGoogleHostname(base::UTF16ToUTF8(data_.keyword()),
1303                                     google_util::DISALLOW_SUBDOMAIN);
1304 }
1305
1306 bool TemplateURL::HasSameKeywordAs(
1307     const TemplateURLData& other,
1308     const SearchTermsData& search_terms_data) const {
1309   return (data_.keyword() == other.keyword()) ||
1310       (IsGoogleSearchURLWithReplaceableKeyword(search_terms_data) &&
1311        TemplateURL(other).IsGoogleSearchURLWithReplaceableKeyword(
1312            search_terms_data));
1313 }
1314
1315 TemplateURL::Type TemplateURL::GetType() const {
1316   return extension_info_ ? extension_info_->type : NORMAL;
1317 }
1318
1319 std::string TemplateURL::GetExtensionId() const {
1320   DCHECK(extension_info_);
1321   return extension_info_->extension_id;
1322 }
1323
1324 size_t TemplateURL::URLCount() const {
1325   // Add 1 for the regular search URL.
1326   return data_.alternate_urls.size() + 1;
1327 }
1328
1329 const std::string& TemplateURL::GetURL(size_t index) const {
1330   DCHECK_LT(index, URLCount());
1331
1332   return (index < data_.alternate_urls.size()) ?
1333       data_.alternate_urls[index] : url();
1334 }
1335
1336 bool TemplateURL::ExtractSearchTermsFromURL(
1337     const GURL& url,
1338     const SearchTermsData& search_terms_data,
1339     base::string16* search_terms) const {
1340   return FindSearchTermsInURL(url, search_terms_data, search_terms, NULL, NULL);
1341 }
1342
1343 bool TemplateURL::IsSearchURL(const GURL& url,
1344                               const SearchTermsData& search_terms_data) const {
1345   base::string16 search_terms;
1346   return ExtractSearchTermsFromURL(url, search_terms_data, &search_terms) &&
1347       !search_terms.empty();
1348 }
1349
1350 bool TemplateURL::HasSearchTermsReplacementKey(const GURL& url) const {
1351   // Look for the key both in the query and the ref.
1352   std::string params[] = {url.query(), url.ref()};
1353
1354   for (int i = 0; i < 2; ++i) {
1355     url::Component query, key, value;
1356     query.len = static_cast<int>(params[i].size());
1357     while (url::ExtractQueryKeyValue(params[i].c_str(), &query, &key, &value)) {
1358       if (key.is_nonempty() &&
1359           params[i].substr(key.begin, key.len) ==
1360               search_terms_replacement_key()) {
1361         return true;
1362       }
1363     }
1364   }
1365   return false;
1366 }
1367
1368 bool TemplateURL::ReplaceSearchTermsInURL(
1369     const GURL& url,
1370     const TemplateURLRef::SearchTermsArgs& search_terms_args,
1371     const SearchTermsData& search_terms_data,
1372     GURL* result) {
1373   // TODO(beaudoin): Use AQS from |search_terms_args| too.
1374   url::Parsed::ComponentType search_term_component;
1375   url::Component search_terms_position;
1376   base::string16 search_terms;
1377   if (!FindSearchTermsInURL(url, search_terms_data, &search_terms,
1378                             &search_term_component, &search_terms_position)) {
1379     return false;
1380   }
1381   DCHECK(search_terms_position.is_nonempty());
1382
1383   // Query and ref are encoded in the same way.
1384   const bool is_in_query = (search_term_component != url::Parsed::PATH);
1385
1386   std::string input_encoding;
1387   base::string16 encoded_terms;
1388   base::string16 encoded_original_query;
1389   EncodeSearchTerms(search_terms_args, is_in_query, &input_encoding,
1390                     &encoded_terms, &encoded_original_query);
1391
1392   std::string old_params;
1393   if (search_term_component == url::Parsed::QUERY) {
1394     old_params = url.query();
1395   } else if (search_term_component == url::Parsed::REF) {
1396     old_params = url.ref();
1397   } else {
1398     DCHECK_EQ(search_term_component, url::Parsed::PATH);
1399     old_params = url.path();
1400   }
1401
1402   std::string new_params(old_params, 0, search_terms_position.begin);
1403   new_params += base::UTF16ToUTF8(encoded_terms);
1404   new_params += old_params.substr(search_terms_position.end());
1405   GURL::Replacements replacements;
1406
1407   if (search_term_component == url::Parsed::QUERY) {
1408     replacements.SetQueryStr(new_params);
1409   } else if (search_term_component == url::Parsed::REF) {
1410     replacements.SetRefStr(new_params);
1411   } else {
1412     DCHECK_EQ(search_term_component, url::Parsed::PATH);
1413     replacements.SetPathStr(new_params);
1414   }
1415
1416   *result = url.ReplaceComponents(replacements);
1417   return true;
1418 }
1419
1420 void TemplateURL::EncodeSearchTerms(
1421     const TemplateURLRef::SearchTermsArgs& search_terms_args,
1422     bool is_in_query,
1423     std::string* input_encoding,
1424     base::string16* encoded_terms,
1425     base::string16* encoded_original_query) const {
1426
1427   std::vector<std::string> encodings(input_encodings());
1428   if (std::find(encodings.begin(), encodings.end(), "UTF-8") == encodings.end())
1429     encodings.push_back("UTF-8");
1430   for (std::vector<std::string>::const_iterator i(encodings.begin());
1431        i != encodings.end(); ++i) {
1432     if (TryEncoding(search_terms_args.search_terms,
1433                     search_terms_args.original_query, i->c_str(),
1434                     is_in_query, encoded_terms, encoded_original_query)) {
1435       *input_encoding = *i;
1436       return;
1437     }
1438   }
1439   NOTREACHED();
1440 }
1441
1442 GURL TemplateURL::GenerateSearchURL(
1443     const SearchTermsData& search_terms_data) const {
1444   if (!url_ref_.IsValid(search_terms_data))
1445     return GURL();
1446
1447   if (!url_ref_.SupportsReplacement(search_terms_data))
1448     return GURL(url());
1449
1450   // Use something obscure for the search terms argument so that in the rare
1451   // case the term replaces the URL it's unlikely another keyword would have the
1452   // same url.
1453   // TODO(jnd): Add additional parameters to get post data when the search URL
1454   // has post parameters.
1455   return GURL(url_ref_.ReplaceSearchTerms(
1456       TemplateURLRef::SearchTermsArgs(
1457           base::ASCIIToUTF16("blah.blah.blah.blah.blah")),
1458       search_terms_data, NULL));
1459 }
1460
1461 void TemplateURL::CopyFrom(const TemplateURL& other) {
1462   if (this == &other)
1463     return;
1464
1465   data_ = other.data_;
1466   url_ref_.InvalidateCachedValues();
1467   suggestions_url_ref_.InvalidateCachedValues();
1468   instant_url_ref_.InvalidateCachedValues();
1469   SetPrepopulateId(other.data_.prepopulate_id);
1470 }
1471
1472 void TemplateURL::SetURL(const std::string& url) {
1473   data_.SetURL(url);
1474   url_ref_.InvalidateCachedValues();
1475 }
1476
1477 void TemplateURL::SetPrepopulateId(int id) {
1478   data_.prepopulate_id = id;
1479   const bool prepopulated = id > 0;
1480   url_ref_.prepopulated_ = prepopulated;
1481   suggestions_url_ref_.prepopulated_ = prepopulated;
1482   instant_url_ref_.prepopulated_ = prepopulated;
1483 }
1484
1485 void TemplateURL::ResetKeywordIfNecessary(
1486     const SearchTermsData& search_terms_data,
1487     bool force) {
1488   if (IsGoogleSearchURLWithReplaceableKeyword(search_terms_data) || force) {
1489     DCHECK(GetType() != OMNIBOX_API_EXTENSION);
1490     GURL url(GenerateSearchURL(search_terms_data));
1491     if (url.is_valid())
1492       data_.SetKeyword(
1493           GenerateKeyword(url, search_terms_data.GetAcceptLanguages()));
1494   }
1495 }
1496
1497 bool TemplateURL::FindSearchTermsInURL(
1498     const GURL& url,
1499     const SearchTermsData& search_terms_data,
1500     base::string16* search_terms,
1501     url::Parsed::ComponentType* search_term_component,
1502     url::Component* search_terms_position) const {
1503   DCHECK(search_terms);
1504   search_terms->clear();
1505
1506   // Try to match with every pattern.
1507   for (size_t i = 0; i < URLCount(); ++i) {
1508     TemplateURLRef ref(this, i);
1509     if (ref.ExtractSearchTermsFromURL(url, search_terms, search_terms_data,
1510         search_term_component, search_terms_position)) {
1511       // If ExtractSearchTermsFromURL() returns true and |search_terms| is empty
1512       // it means the pattern matched but no search terms were present. In this
1513       // case we fail immediately without looking for matches in subsequent
1514       // patterns. This means that given patterns
1515       //    [ "http://foo/#q={searchTerms}", "http://foo/?q={searchTerms}" ],
1516       // calling ExtractSearchTermsFromURL() on "http://foo/?q=bar#q=' would
1517       // return false. This is important for at least Google, where such URLs
1518       // are invalid.
1519       return !search_terms->empty();
1520     }
1521   }
1522   return false;
1523 }