components/search_engines/template_url_parser.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/search_engines/template_url_parser.h"
   6
   7 #include <algorithm>
   8 #include <map>
   9 #include <vector>
  10
  11 #include "base/logging.h"
  12 #include "base/memory/scoped_ptr.h"
  13 #include "base/strings/string_number_conversions.h"
  14 #include "base/strings/string_util.h"
  15 #include "base/strings/utf_string_conversions.h"
  16 #include "components/search_engines/template_url.h"
  17 #include "libxml/parser.h"
  18 #include "libxml/xmlwriter.h"
  19 #include "ui/gfx/favicon_size.h"
  20 #include "url/gurl.h"
  21 #include "url/url_constants.h"
  22
  23 namespace {
  24
  25 // NOTE: libxml uses the UTF-8 encoding. As 0-127 of UTF-8 corresponds
  26 // to that of char, the following names are all in terms of char. This avoids
  27 // having to convert to wide, then do comparisons.
  28
  29 // Defines for element names of the OSD document:
  30 const char kURLElement[] = "Url";
  31 const char kParamElement[] = "Param";
  32 const char kShortNameElement[] = "ShortName";
  33 const char kImageElement[] = "Image";
  34 const char kOpenSearchDescriptionElement[] = "OpenSearchDescription";
  35 const char kFirefoxSearchDescriptionElement[] = "SearchPlugin";
  36 const char kInputEncodingElement[] = "InputEncoding";
  37 const char kAliasElement[] = "Alias";
  38
  39 // Various XML attributes used.
  40 const char kURLTypeAttribute[] = "type";
  41 const char kURLTemplateAttribute[] = "template";
  42 const char kImageTypeAttribute[] = "type";
  43 const char kImageWidthAttribute[] = "width";
  44 const char kImageHeightAttribute[] = "height";
  45 const char kParamNameAttribute[] = "name";
  46 const char kParamValueAttribute[] = "value";
  47 const char kParamMethodAttribute[] = "method";
  48
  49 // Mime type for search results.
  50 const char kHTMLType[] = "text/html";
  51
  52 // Mime type for as you type suggestions.
  53 const char kSuggestionType[] = "application/x-suggestions+json";
  54
  55 std::string XMLCharToString(const xmlChar* value) {
  56   return std::string(reinterpret_cast<const char*>(value));
  57 }
  58
  59 // Returns true if input_encoding contains a valid input encoding string. This
  60 // doesn't verify that we have a valid encoding for the string, just that the
  61 // string contains characters that constitute a valid input encoding.
  62 bool IsValidEncodingString(const std::string& input_encoding) {
  63   if (input_encoding.empty())
  64     return false;
  65
  66   if (!IsAsciiAlpha(input_encoding[0]))
  67     return false;
  68
  69   for (size_t i = 1, max = input_encoding.size(); i < max; ++i) {
  70     char c = input_encoding[i];
  71     if (!IsAsciiAlpha(c) && !IsAsciiDigit(c) && c != '.' && c != '_' &&
  72         c != '-') {
  73       return false;
  74     }
  75   }
  76   return true;
  77 }
  78
  79 void AppendParamToQuery(const std::string& key,
  80                         const std::string& value,
  81                         std::string* query) {
  82   if (!query->empty())
  83     query->append("&");
  84   if (!key.empty()) {
  85     query->append(key);
  86     query->append("=");
  87   }
  88   query->append(value);
  89 }
  90
  91 // Returns true if |url| is empty or is a valid URL with a scheme of HTTP[S].
  92 bool IsHTTPRef(const std::string& url) {
  93   if (url.empty())
  94     return true;
  95   GURL gurl(url);
  96   return gurl.is_valid() && (gurl.SchemeIs(url::kHttpScheme) ||
  97                              gurl.SchemeIs(url::kHttpsScheme));
  98 }
  99
 100 }  // namespace
 101
 102
 103 // TemplateURLParsingContext --------------------------------------------------
 104
 105 // To minimize memory overhead while parsing, a SAX style parser is used.
 106 // TemplateURLParsingContext is used to maintain the state we're in the document
 107 // while parsing.
 108 class TemplateURLParsingContext {
 109  public:
 110   // Enum of the known element types.
 111   enum ElementType {
 112     UNKNOWN,
 113     OPEN_SEARCH_DESCRIPTION,
 114     URL,
 115     PARAM,
 116     SHORT_NAME,
 117     IMAGE,
 118     INPUT_ENCODING,
 119     ALIAS,
 120   };
 121
 122   enum Method {
 123     GET,
 124     POST
 125   };
 126
 127   // Key/value of a Param node.
 128   typedef std::pair<std::string, std::string> Param;
 129
 130   explicit TemplateURLParsingContext(
 131       TemplateURLParser::ParameterFilter* parameter_filter);
 132
 133   static void StartElementImpl(void* ctx,
 134                                const xmlChar* name,
 135                                const xmlChar** atts);
 136   static void EndElementImpl(void* ctx, const xmlChar* name);
 137   static void CharactersImpl(void* ctx, const xmlChar* ch, int len);
 138
 139   // Returns a heap-allocated TemplateURL representing the result of parsing.
 140   // This will be NULL if parsing failed or if the results were invalid for some
 141   // reason (e.g. the resulting URL was not HTTP[S], a name wasn't supplied,
 142   // a resulting TemplateURLRef was invalid, etc.).
 143   TemplateURL* GetTemplateURL(const SearchTermsData& search_terms_data,
 144                               bool show_in_default_list);
 145
 146  private:
 147   // Key is UTF8 encoded.
 148   typedef std::map<std::string, ElementType> ElementNameToElementTypeMap;
 149
 150   static void InitMapping();
 151
 152   void ParseURL(const xmlChar** atts);
 153   void ParseImage(const xmlChar** atts);
 154   void ParseParam(const xmlChar** atts);
 155   void ProcessURLParams();
 156
 157   // Returns the current ElementType.
 158   ElementType GetKnownType();
 159
 160   static ElementNameToElementTypeMap* kElementNameToElementTypeMap;
 161
 162   // Data that gets updated as we parse, and is converted to a TemplateURL by
 163   // GetTemplateURL().
 164   TemplateURLData data_;
 165
 166   std::vector<ElementType> elements_;
 167   bool image_is_valid_for_favicon_;
 168
 169   // Character content for the current element.
 170   base::string16 string_;
 171
 172   TemplateURLParser::ParameterFilter* parameter_filter_;
 173
 174   // The list of parameters parsed in the Param nodes of a Url node.
 175   std::vector<Param> extra_params_;
 176
 177   // The HTTP methods used.
 178   Method method_;
 179   Method suggestion_method_;
 180
 181   // If true, we are currently parsing a suggest URL, otherwise it is an HTML
 182   // search.  Note that we don't need a stack as URL nodes cannot be nested.
 183   bool is_suggest_url_;
 184
 185   // If true, the user has set a keyword and we should use it. Otherwise,
 186   // we generate a keyword based on the URL.
 187   bool has_custom_keyword_;
 188
 189   // Whether we should derive the image from the URL (when images are data
 190   // URLs).
 191   bool derive_image_from_url_;
 192
 193   DISALLOW_COPY_AND_ASSIGN(TemplateURLParsingContext);
 194 };
 195
 196 // static
 197 TemplateURLParsingContext::ElementNameToElementTypeMap*
 198     TemplateURLParsingContext::kElementNameToElementTypeMap = NULL;
 199
 200 TemplateURLParsingContext::TemplateURLParsingContext(
 201     TemplateURLParser::ParameterFilter* parameter_filter)
 202     : image_is_valid_for_favicon_(false),
 203       parameter_filter_(parameter_filter),
 204       method_(GET),
 205       suggestion_method_(GET),
 206       is_suggest_url_(false),
 207       has_custom_keyword_(false),
 208       derive_image_from_url_(false) {
 209   if (kElementNameToElementTypeMap == NULL)
 210     InitMapping();
 211 }
 212
 213 // static
 214 void TemplateURLParsingContext::StartElementImpl(void* ctx,
 215                                                  const xmlChar* name,
 216                                                  const xmlChar** atts) {
 217   // Remove the namespace from |name|, ex: os:Url -> Url.
 218   std::string node_name(XMLCharToString(name));
 219   size_t index = node_name.find_first_of(":");
 220   if (index != std::string::npos)
 221     node_name.erase(0, index + 1);
 222
 223   TemplateURLParsingContext* context =
 224       reinterpret_cast<TemplateURLParsingContext*>(ctx);
 225   context->elements_.push_back(
 226     context->kElementNameToElementTypeMap->count(node_name) ?
 227         (*context->kElementNameToElementTypeMap)[node_name] : UNKNOWN);
 228   switch (context->GetKnownType()) {
 229     case TemplateURLParsingContext::URL:
 230       context->extra_params_.clear();
 231       context->ParseURL(atts);
 232       break;
 233     case TemplateURLParsingContext::IMAGE:
 234       context->ParseImage(atts);
 235       break;
 236     case TemplateURLParsingContext::PARAM:
 237       context->ParseParam(atts);
 238       break;
 239     default:
 240       break;
 241   }
 242   context->string_.clear();
 243 }
 244
 245 // static
 246 void TemplateURLParsingContext::EndElementImpl(void* ctx, const xmlChar* name) {
 247   TemplateURLParsingContext* context =
 248       reinterpret_cast<TemplateURLParsingContext*>(ctx);
 249   switch (context->GetKnownType()) {
 250     case TemplateURLParsingContext::URL:
 251       context->ProcessURLParams();
 252       break;
 253     case TemplateURLParsingContext::SHORT_NAME:
 254       context->data_.short_name = context->string_;
 255       break;
 256     case TemplateURLParsingContext::IMAGE: {
 257       GURL image_url(base::UTF16ToUTF8(context->string_));
 258       if (image_url.SchemeIs(url::kDataScheme)) {
 259         // TODO (jcampan): bug 1169256: when dealing with data URL, we need to
 260         // decode the data URL in the renderer. For now, we'll just point to the
 261         // favicon from the URL.
 262         context->derive_image_from_url_ = true;
 263       } else if (context->image_is_valid_for_favicon_ && image_url.is_valid() &&
 264                  (image_url.SchemeIs(url::kHttpScheme) ||
 265                   image_url.SchemeIs(url::kHttpsScheme))) {
 266         context->data_.favicon_url = image_url;
 267       }
 268       context->image_is_valid_for_favicon_ = false;
 269       break;
 270     }
 271     case TemplateURLParsingContext::INPUT_ENCODING: {
 272       std::string input_encoding = base::UTF16ToASCII(context->string_);
 273       if (IsValidEncodingString(input_encoding))
 274         context->data_.input_encodings.push_back(input_encoding);
 275       break;
 276     }
 277     case TemplateURLParsingContext::ALIAS: {
 278       context->data_.SetKeyword(context->string_);
 279       context->has_custom_keyword_ = true;
 280       break;
 281     }
 282     default:
 283       break;
 284   }
 285   context->string_.clear();
 286   context->elements_.pop_back();
 287 }
 288
 289 // static
 290 void TemplateURLParsingContext::CharactersImpl(void* ctx,
 291                                                const xmlChar* ch,
 292                                                int len) {
 293   reinterpret_cast<TemplateURLParsingContext*>(ctx)->string_ +=
 294       base::UTF8ToUTF16(std::string(reinterpret_cast<const char*>(ch), len));
 295 }
 296
 297 TemplateURL* TemplateURLParsingContext::GetTemplateURL(
 298     const SearchTermsData& search_terms_data,
 299     bool show_in_default_list) {
 300   // TODO(jcampan): Support engines that use POST; see http://crbug.com/18107
 301   if (method_ == TemplateURLParsingContext::POST || data_.short_name.empty() ||
 302       !IsHTTPRef(data_.url()) || !IsHTTPRef(data_.suggestions_url))
 303     return NULL;
 304   if (suggestion_method_ == TemplateURLParsingContext::POST)
 305     data_.suggestions_url.clear();
 306
 307   // If the image was a data URL, use the favicon from the search URL instead.
 308   // (see the TODO in EndElementImpl()).
 309   GURL search_url(data_.url());
 310   if (derive_image_from_url_ && data_.favicon_url.is_empty())
 311     data_.favicon_url = TemplateURL::GenerateFaviconURL(search_url);
 312
 313   // Generate a keyword for this search engine if a custom one was not present
 314   // in the imported data.
 315   if (!has_custom_keyword_)
 316     data_.SetKeyword(TemplateURL::GenerateKeyword(search_url));
 317
 318   data_.show_in_default_list = show_in_default_list;
 319
 320   // Bail if the search URL is empty or if either TemplateURLRef is invalid.
 321   scoped_ptr<TemplateURL> template_url(new TemplateURL(data_));
 322   if (template_url->url().empty() ||
 323       !template_url->url_ref().IsValid(search_terms_data) ||
 324       (!template_url->suggestions_url().empty() &&
 325        !template_url->suggestions_url_ref().IsValid(search_terms_data))) {
 326     return NULL;
 327   }
 328
 329   return template_url.release();
 330 }
 331
 332 // static
 333 void TemplateURLParsingContext::InitMapping() {
 334   kElementNameToElementTypeMap = new std::map<std::string, ElementType>;
 335   (*kElementNameToElementTypeMap)[kURLElement] = URL;
 336   (*kElementNameToElementTypeMap)[kParamElement] = PARAM;
 337   (*kElementNameToElementTypeMap)[kShortNameElement] = SHORT_NAME;
 338   (*kElementNameToElementTypeMap)[kImageElement] = IMAGE;
 339   (*kElementNameToElementTypeMap)[kOpenSearchDescriptionElement] =
 340       OPEN_SEARCH_DESCRIPTION;
 341   (*kElementNameToElementTypeMap)[kFirefoxSearchDescriptionElement] =
 342       OPEN_SEARCH_DESCRIPTION;
 343   (*kElementNameToElementTypeMap)[kInputEncodingElement] = INPUT_ENCODING;
 344   (*kElementNameToElementTypeMap)[kAliasElement] = ALIAS;
 345 }
 346
 347 void TemplateURLParsingContext::ParseURL(const xmlChar** atts) {
 348   if (!atts)
 349     return;
 350
 351   std::string template_url;
 352   bool is_post = false;
 353   bool is_html_url = false;
 354   bool is_suggest_url = false;
 355   for (; *atts; atts += 2) {
 356     std::string name(XMLCharToString(*atts));
 357     const xmlChar* value = atts[1];
 358     if (name == kURLTypeAttribute) {
 359       std::string type = XMLCharToString(value);
 360       is_html_url = (type == kHTMLType);
 361       is_suggest_url = (type == kSuggestionType);
 362     } else if (name == kURLTemplateAttribute) {
 363       template_url = XMLCharToString(value);
 364     } else if (name == kParamMethodAttribute) {
 365       is_post = LowerCaseEqualsASCII(XMLCharToString(value), "post");
 366     }
 367   }
 368
 369   if (is_html_url && !template_url.empty()) {
 370     data_.SetURL(template_url);
 371     is_suggest_url_ = false;
 372     if (is_post)
 373       method_ = POST;
 374   } else if (is_suggest_url) {
 375     data_.suggestions_url = template_url;
 376     is_suggest_url_ = true;
 377     if (is_post)
 378       suggestion_method_ = POST;
 379   }
 380 }
 381
 382 void TemplateURLParsingContext::ParseImage(const xmlChar** atts) {
 383   if (!atts)
 384     return;
 385
 386   int width = 0;
 387   int height = 0;
 388   std::string type;
 389   for (; *atts; atts += 2) {
 390     std::string name(XMLCharToString(*atts));
 391     const xmlChar* value = atts[1];
 392     if (name == kImageTypeAttribute) {
 393       type = XMLCharToString(value);
 394     } else if (name == kImageWidthAttribute) {
 395       base::StringToInt(XMLCharToString(value), &width);
 396     } else if (name == kImageHeightAttribute) {
 397       base::StringToInt(XMLCharToString(value), &height);
 398     }
 399   }
 400
 401   image_is_valid_for_favicon_ = (width == gfx::kFaviconSize) &&
 402       (height == gfx::kFaviconSize) &&
 403       ((type == "image/x-icon") || (type == "image/vnd.microsoft.icon"));
 404 }
 405
 406 void TemplateURLParsingContext::ParseParam(const xmlChar** atts) {
 407   if (!atts)
 408     return;
 409
 410   std::string key, value;
 411   for (; *atts; atts += 2) {
 412     std::string name(XMLCharToString(*atts));
 413     const xmlChar* val = atts[1];
 414     if (name == kParamNameAttribute) {
 415       key = XMLCharToString(val);
 416     } else if (name == kParamValueAttribute) {
 417       value = XMLCharToString(val);
 418     }
 419   }
 420
 421   if (!key.empty() &&
 422       (!parameter_filter_ || parameter_filter_->KeepParameter(key, value)))
 423     extra_params_.push_back(Param(key, value));
 424 }
 425
 426 void TemplateURLParsingContext::ProcessURLParams() {
 427   if (!parameter_filter_ && extra_params_.empty())
 428     return;
 429
 430   GURL url(is_suggest_url_ ? data_.suggestions_url : data_.url());
 431   if (url.is_empty())
 432     return;
 433
 434   // If there is a parameter filter, parse the existing URL and remove any
 435   // unwanted parameter.
 436   std::string new_query;
 437   bool modified = false;
 438   if (parameter_filter_) {
 439     url::Component query = url.parsed_for_possibly_invalid_spec().query;
 440     url::Component key, value;
 441     const char* url_spec = url.spec().c_str();
 442     while (url::ExtractQueryKeyValue(url_spec, &query, &key, &value)) {
 443       std::string key_str(url_spec, key.begin, key.len);
 444       std::string value_str(url_spec, value.begin, value.len);
 445       if (parameter_filter_->KeepParameter(key_str, value_str)) {
 446         AppendParamToQuery(key_str, value_str, &new_query);
 447       } else {
 448         modified = true;
 449       }
 450     }
 451   }
 452   if (!modified)
 453     new_query = url.query();
 454
 455   // Add the extra parameters if any.
 456   if (!extra_params_.empty()) {
 457     modified = true;
 458     for (std::vector<Param>::const_iterator iter(extra_params_.begin());
 459          iter != extra_params_.end(); ++iter)
 460       AppendParamToQuery(iter->first, iter->second, &new_query);
 461   }
 462
 463   if (modified) {
 464     GURL::Replacements repl;
 465     repl.SetQueryStr(new_query);
 466     url = url.ReplaceComponents(repl);
 467     if (is_suggest_url_)
 468       data_.suggestions_url = url.spec();
 469     else if (url.is_valid())
 470       data_.SetURL(url.spec());
 471   }
 472 }
 473
 474 TemplateURLParsingContext::ElementType
 475     TemplateURLParsingContext::GetKnownType() {
 476   if (elements_.size() == 2 && elements_[0] == OPEN_SEARCH_DESCRIPTION)
 477     return elements_[1];
 478   // We only expect PARAM nodes under the URL node.
 479   return (elements_.size() == 3 && elements_[0] == OPEN_SEARCH_DESCRIPTION &&
 480       elements_[1] == URL && elements_[2] == PARAM) ? PARAM : UNKNOWN;
 481 }
 482
 483
 484 // TemplateURLParser ----------------------------------------------------------
 485
 486 // static
 487 TemplateURL* TemplateURLParser::Parse(
 488     const SearchTermsData& search_terms_data,
 489     bool show_in_default_list,
 490     const char* data,
 491     size_t length,
 492     TemplateURLParser::ParameterFilter* param_filter) {
 493   // xmlSubstituteEntitiesDefault(1) makes it so that &amp; isn't mapped to
 494   // &#38; . Unfortunately xmlSubstituteEntitiesDefault affects global state.
 495   // If this becomes problematic we'll need to provide our own entity
 496   // type for &amp;, or strip out &#38; by hand after parsing.
 497   int last_sub_entities_value = xmlSubstituteEntitiesDefault(1);
 498   TemplateURLParsingContext context(param_filter);
 499   xmlSAXHandler sax_handler;
 500   memset(&sax_handler, 0, sizeof(sax_handler));
 501   sax_handler.startElement = &TemplateURLParsingContext::StartElementImpl;
 502   sax_handler.endElement = &TemplateURLParsingContext::EndElementImpl;
 503   sax_handler.characters = &TemplateURLParsingContext::CharactersImpl;
 504   int error = xmlSAXUserParseMemory(&sax_handler, &context, data,
 505                                     static_cast<int>(length));
 506   xmlSubstituteEntitiesDefault(last_sub_entities_value);
 507
 508   return error ?
 509       NULL : context.GetTemplateURL(search_terms_data, show_in_default_list);
 510 }