url/url_util.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "url/url_util.h"
   6
   7 #include <string.h>
   8 #include <vector>
   9
  10 #include "base/debug/leak_annotations.h"
  11 #include "base/logging.h"
  12 #include "url/url_canon_internal.h"
  13 #include "url/url_file.h"
  14 #include "url/url_util_internal.h"
  15
  16 namespace url {
  17
  18 namespace {
  19
  20 // ASCII-specific tolower.  The standard library's tolower is locale sensitive,
  21 // so we don't want to use it here.
  22 template<class Char>
  23 inline Char ToLowerASCII(Char c) {
  24   return (c >= 'A' && c <= 'Z') ? (c + ('a' - 'A')) : c;
  25 }
  26
  27 // Backend for LowerCaseEqualsASCII.
  28 template<typename Iter>
  29 inline bool DoLowerCaseEqualsASCII(Iter a_begin, Iter a_end, const char* b) {
  30   for (Iter it = a_begin; it != a_end; ++it, ++b) {
  31     if (!*b || ToLowerASCII(*it) != *b)
  32       return false;
  33   }
  34   return *b == 0;
  35 }
  36
  37 const int kNumStandardURLSchemes = 8;
  38 const char* kStandardURLSchemes[kNumStandardURLSchemes] = {
  39   kHttpScheme,
  40   kHttpsScheme,
  41   kFileScheme,  // Yes, file urls can have a hostname!
  42   kFtpScheme,
  43   kGopherScheme,
  44   kWsScheme,    // WebSocket.
  45   kWssScheme,   // WebSocket secure.
  46   kFileSystemScheme,
  47 };
  48
  49 // List of the currently installed standard schemes. This list is lazily
  50 // initialized by InitStandardSchemes and is leaked on shutdown to prevent
  51 // any destructors from being called that will slow us down or cause problems.
  52 std::vector<const char*>* standard_schemes = NULL;
  53
  54 // See the LockStandardSchemes declaration in the header.
  55 bool standard_schemes_locked = false;
  56
  57 // Ensures that the standard_schemes list is initialized, does nothing if it
  58 // already has values.
  59 void InitStandardSchemes() {
  60   if (standard_schemes)
  61     return;
  62   standard_schemes = new std::vector<const char*>;
  63   for (int i = 0; i < kNumStandardURLSchemes; i++)
  64     standard_schemes->push_back(kStandardURLSchemes[i]);
  65 }
  66
  67 // Given a string and a range inside the string, compares it to the given
  68 // lower-case |compare_to| buffer.
  69 template<typename CHAR>
  70 inline bool DoCompareSchemeComponent(const CHAR* spec,
  71                                      const Component& component,
  72                                      const char* compare_to) {
  73   if (!component.is_nonempty())
  74     return compare_to[0] == 0;  // When component is empty, match empty scheme.
  75   return LowerCaseEqualsASCII(&spec[component.begin],
  76                               &spec[component.end()],
  77                               compare_to);
  78 }
  79
  80 // Returns true if the given scheme identified by |scheme| within |spec| is one
  81 // of the registered "standard" schemes.
  82 template<typename CHAR>
  83 bool DoIsStandard(const CHAR* spec, const Component& scheme) {
  84   if (!scheme.is_nonempty())
  85     return false;  // Empty or invalid schemes are non-standard.
  86
  87   InitStandardSchemes();
  88   for (size_t i = 0; i < standard_schemes->size(); i++) {
  89     if (LowerCaseEqualsASCII(&spec[scheme.begin], &spec[scheme.end()],
  90                              standard_schemes->at(i)))
  91       return true;
  92   }
  93   return false;
  94 }
  95
  96 template<typename CHAR>
  97 bool DoFindAndCompareScheme(const CHAR* str,
  98                             int str_len,
  99                             const char* compare,
 100                             Component* found_scheme) {
 101   // Before extracting scheme, canonicalize the URL to remove any whitespace.
 102   // This matches the canonicalization done in DoCanonicalize function.
 103   RawCanonOutputT<CHAR> whitespace_buffer;
 104   int spec_len;
 105   const CHAR* spec = RemoveURLWhitespace(str, str_len,
 106                                          &whitespace_buffer, &spec_len);
 107
 108   Component our_scheme;
 109   if (!ExtractScheme(spec, spec_len, &our_scheme)) {
 110     // No scheme.
 111     if (found_scheme)
 112       *found_scheme = Component();
 113     return false;
 114   }
 115   if (found_scheme)
 116     *found_scheme = our_scheme;
 117   return DoCompareSchemeComponent(spec, our_scheme, compare);
 118 }
 119
 120 template<typename CHAR>
 121 bool DoCanonicalize(const CHAR* in_spec,
 122                     int in_spec_len,
 123                     bool trim_path_end,
 124                     CharsetConverter* charset_converter,
 125                     CanonOutput* output,
 126                     Parsed* output_parsed) {
 127   // Remove any whitespace from the middle of the relative URL, possibly
 128   // copying to the new buffer.
 129   RawCanonOutputT<CHAR> whitespace_buffer;
 130   int spec_len;
 131   const CHAR* spec = RemoveURLWhitespace(in_spec, in_spec_len,
 132                                          &whitespace_buffer, &spec_len);
 133
 134   Parsed parsed_input;
 135 #ifdef WIN32
 136   // For Windows, we allow things that look like absolute Windows paths to be
 137   // fixed up magically to file URLs. This is done for IE compatability. For
 138   // example, this will change "c:/foo" into a file URL rather than treating
 139   // it as a URL with the protocol "c". It also works for UNC ("\\foo\bar.txt").
 140   // There is similar logic in url_canon_relative.cc for
 141   //
 142   // For Max & Unix, we don't do this (the equivalent would be "/foo/bar" which
 143   // has no meaning as an absolute path name. This is because browsers on Mac
 144   // & Unix don't generally do this, so there is no compatibility reason for
 145   // doing so.
 146   if (DoesBeginUNCPath(spec, 0, spec_len, false) ||
 147       DoesBeginWindowsDriveSpec(spec, 0, spec_len)) {
 148     ParseFileURL(spec, spec_len, &parsed_input);
 149     return CanonicalizeFileURL(spec, spec_len, parsed_input, charset_converter,
 150                                output, output_parsed);
 151   }
 152 #endif
 153
 154   Component scheme;
 155   if (!ExtractScheme(spec, spec_len, &scheme))
 156     return false;
 157
 158   // This is the parsed version of the input URL, we have to canonicalize it
 159   // before storing it in our object.
 160   bool success;
 161   if (DoCompareSchemeComponent(spec, scheme, url::kFileScheme)) {
 162     // File URLs are special.
 163     ParseFileURL(spec, spec_len, &parsed_input);
 164     success = CanonicalizeFileURL(spec, spec_len, parsed_input,
 165                                   charset_converter, output, output_parsed);
 166   } else if (DoCompareSchemeComponent(spec, scheme, url::kFileSystemScheme)) {
 167     // Filesystem URLs are special.
 168     ParseFileSystemURL(spec, spec_len, &parsed_input);
 169     success = CanonicalizeFileSystemURL(spec, spec_len, parsed_input,
 170                                         charset_converter, output,
 171                                         output_parsed);
 172
 173   } else if (DoIsStandard(spec, scheme)) {
 174     // All "normal" URLs.
 175     ParseStandardURL(spec, spec_len, &parsed_input);
 176     success = CanonicalizeStandardURL(spec, spec_len, parsed_input,
 177                                       charset_converter, output, output_parsed);
 178
 179   } else if (DoCompareSchemeComponent(spec, scheme, url::kMailToScheme)) {
 180     // Mailto are treated like a standard url with only a scheme, path, query
 181     ParseMailtoURL(spec, spec_len, &parsed_input);
 182     success = CanonicalizeMailtoURL(spec, spec_len, parsed_input, output,
 183                                     output_parsed);
 184
 185   } else {
 186     // "Weird" URLs like data: and javascript:
 187     ParsePathURL(spec, spec_len, trim_path_end, &parsed_input);
 188     success = CanonicalizePathURL(spec, spec_len, parsed_input, output,
 189                                   output_parsed);
 190   }
 191   return success;
 192 }
 193
 194 template<typename CHAR>
 195 bool DoResolveRelative(const char* base_spec,
 196                        int base_spec_len,
 197                        const Parsed& base_parsed,
 198                        const CHAR* in_relative,
 199                        int in_relative_length,
 200                        CharsetConverter* charset_converter,
 201                        CanonOutput* output,
 202                        Parsed* output_parsed) {
 203   // Remove any whitespace from the middle of the relative URL, possibly
 204   // copying to the new buffer.
 205   RawCanonOutputT<CHAR> whitespace_buffer;
 206   int relative_length;
 207   const CHAR* relative = RemoveURLWhitespace(in_relative, in_relative_length,
 208                                              &whitespace_buffer,
 209                                              &relative_length);
 210   bool base_is_authority_based = false;
 211   bool base_is_hierarchical = false;
 212   if (base_spec &&
 213       base_parsed.scheme.is_nonempty()) {
 214     int after_scheme = base_parsed.scheme.end() + 1;  // Skip past the colon.
 215     int num_slashes = CountConsecutiveSlashes(base_spec, after_scheme,
 216                                               base_spec_len);
 217     base_is_authority_based = num_slashes > 1;
 218     base_is_hierarchical = num_slashes > 0;
 219   }
 220
 221   bool standard_base_scheme =
 222       base_parsed.scheme.is_nonempty() &&
 223       DoIsStandard(base_spec, base_parsed.scheme);
 224
 225   bool is_relative;
 226   Component relative_component;
 227   if (!IsRelativeURL(base_spec, base_parsed, relative, relative_length,
 228                      (base_is_hierarchical || standard_base_scheme),
 229                      &is_relative, &relative_component)) {
 230     // Error resolving.
 231     return false;
 232   }
 233
 234   // Pretend for a moment that |base_spec| is a standard URL. Normally
 235   // non-standard URLs are treated as PathURLs, but if the base has an
 236   // authority we would like to preserve it.
 237   if (is_relative && base_is_authority_based && !standard_base_scheme) {
 238     Parsed base_parsed_authority;
 239     ParseStandardURL(base_spec, base_spec_len, &base_parsed_authority);
 240     if (base_parsed_authority.host.is_nonempty()) {
 241       RawCanonOutputT<char> temporary_output;
 242       bool did_resolve_succeed =
 243           ResolveRelativeURL(base_spec, base_parsed_authority, false, relative,
 244                              relative_component, charset_converter,
 245                              &temporary_output, output_parsed);
 246       // The output_parsed is incorrect at this point (because it was built
 247       // based on base_parsed_authority instead of base_parsed) and needs to be
 248       // re-created.
 249       DoCanonicalize(temporary_output.data(), temporary_output.length(), true,
 250                      charset_converter, output, output_parsed);
 251       return did_resolve_succeed;
 252     }
 253   } else if (is_relative) {
 254     // Relative, resolve and canonicalize.
 255     bool file_base_scheme = base_parsed.scheme.is_nonempty() &&
 256         DoCompareSchemeComponent(base_spec, base_parsed.scheme, kFileScheme);
 257     return ResolveRelativeURL(base_spec, base_parsed, file_base_scheme, relative,
 258                               relative_component, charset_converter, output,
 259                               output_parsed);
 260   }
 261
 262   // Not relative, canonicalize the input.
 263   return DoCanonicalize(relative, relative_length, true, charset_converter,
 264                         output, output_parsed);
 265 }
 266
 267 template<typename CHAR>
 268 bool DoReplaceComponents(const char* spec,
 269                          int spec_len,
 270                          const Parsed& parsed,
 271                          const Replacements<CHAR>& replacements,
 272                          CharsetConverter* charset_converter,
 273                          CanonOutput* output,
 274                          Parsed* out_parsed) {
 275   // If the scheme is overridden, just do a simple string substitution and
 276   // reparse the whole thing. There are lots of edge cases that we really don't
 277   // want to deal with. Like what happens if I replace "http://e:8080/foo"
 278   // with a file. Does it become "file:///E:/8080/foo" where the port number
 279   // becomes part of the path? Parsing that string as a file URL says "yes"
 280   // but almost no sane rule for dealing with the components individually would
 281   // come up with that.
 282   //
 283   // Why allow these crazy cases at all? Programatically, there is almost no
 284   // case for replacing the scheme. The most common case for hitting this is
 285   // in JS when building up a URL using the location object. In this case, the
 286   // JS code expects the string substitution behavior:
 287   //   http://www.w3.org/TR/2008/WD-html5-20080610/structured.html#common3
 288   if (replacements.IsSchemeOverridden()) {
 289     // Canonicalize the new scheme so it is 8-bit and can be concatenated with
 290     // the existing spec.
 291     RawCanonOutput<128> scheme_replaced;
 292     Component scheme_replaced_parsed;
 293     CanonicalizeScheme(replacements.sources().scheme,
 294                        replacements.components().scheme,
 295                        &scheme_replaced, &scheme_replaced_parsed);
 296
 297     // We can assume that the input is canonicalized, which means it always has
 298     // a colon after the scheme (or where the scheme would be).
 299     int spec_after_colon = parsed.scheme.is_valid() ? parsed.scheme.end() + 1
 300                                                     : 1;
 301     if (spec_len - spec_after_colon > 0) {
 302       scheme_replaced.Append(&spec[spec_after_colon],
 303                              spec_len - spec_after_colon);
 304     }
 305
 306     // We now need to completely re-parse the resulting string since its meaning
 307     // may have changed with the different scheme.
 308     RawCanonOutput<128> recanonicalized;
 309     Parsed recanonicalized_parsed;
 310     DoCanonicalize(scheme_replaced.data(), scheme_replaced.length(), true,
 311                    charset_converter,
 312                    &recanonicalized, &recanonicalized_parsed);
 313
 314     // Recurse using the version with the scheme already replaced. This will now
 315     // use the replacement rules for the new scheme.
 316     //
 317     // Warning: this code assumes that ReplaceComponents will re-check all
 318     // components for validity. This is because we can't fail if DoCanonicalize
 319     // failed above since theoretically the thing making it fail could be
 320     // getting replaced here. If ReplaceComponents didn't re-check everything,
 321     // we wouldn't know if something *not* getting replaced is a problem.
 322     // If the scheme-specific replacers are made more intelligent so they don't
 323     // re-check everything, we should instead recanonicalize the whole thing
 324     // after this call to check validity (this assumes replacing the scheme is
 325     // much much less common than other types of replacements, like clearing the
 326     // ref).
 327     Replacements<CHAR> replacements_no_scheme = replacements;
 328     replacements_no_scheme.SetScheme(NULL, Component());
 329     return DoReplaceComponents(recanonicalized.data(), recanonicalized.length(),
 330                                recanonicalized_parsed, replacements_no_scheme,
 331                                charset_converter, output, out_parsed);
 332   }
 333
 334   // If we get here, then we know the scheme doesn't need to be replaced, so can
 335   // just key off the scheme in the spec to know how to do the replacements.
 336   if (DoCompareSchemeComponent(spec, parsed.scheme, url::kFileScheme)) {
 337     return ReplaceFileURL(spec, parsed, replacements, charset_converter, output,
 338                           out_parsed);
 339   }
 340   if (DoCompareSchemeComponent(spec, parsed.scheme, url::kFileSystemScheme)) {
 341     return ReplaceFileSystemURL(spec, parsed, replacements, charset_converter,
 342                                 output, out_parsed);
 343   }
 344   if (DoIsStandard(spec, parsed.scheme)) {
 345     return ReplaceStandardURL(spec, parsed, replacements, charset_converter,
 346                               output, out_parsed);
 347   }
 348   if (DoCompareSchemeComponent(spec, parsed.scheme, url::kMailToScheme)) {
 349     return ReplaceMailtoURL(spec, parsed, replacements, output, out_parsed);
 350   }
 351
 352   // Default is a path URL.
 353   return ReplacePathURL(spec, parsed, replacements, output, out_parsed);
 354 }
 355
 356 }  // namespace
 357
 358 void Initialize() {
 359   InitStandardSchemes();
 360 }
 361
 362 void Shutdown() {
 363   if (standard_schemes) {
 364     delete standard_schemes;
 365     standard_schemes = NULL;
 366   }
 367 }
 368
 369 void AddStandardScheme(const char* new_scheme) {
 370   // If this assert triggers, it means you've called AddStandardScheme after
 371   // LockStandardSchemes have been called (see the header file for
 372   // LockStandardSchemes for more).
 373   //
 374   // This normally means you're trying to set up a new standard scheme too late
 375   // in your application's init process. Locate where your app does this
 376   // initialization and calls LockStandardScheme, and add your new standard
 377   // scheme there.
 378   DCHECK(!standard_schemes_locked) <<
 379       "Trying to add a standard scheme after the list has been locked.";
 380
 381   size_t scheme_len = strlen(new_scheme);
 382   if (scheme_len == 0)
 383     return;
 384
 385   // Dulicate the scheme into a new buffer and add it to the list of standard
 386   // schemes. This pointer will be leaked on shutdown.
 387   char* dup_scheme = new char[scheme_len + 1];
 388   ANNOTATE_LEAKING_OBJECT_PTR(dup_scheme);
 389   memcpy(dup_scheme, new_scheme, scheme_len + 1);
 390
 391   InitStandardSchemes();
 392   standard_schemes->push_back(dup_scheme);
 393 }
 394
 395 void LockStandardSchemes() {
 396   standard_schemes_locked = true;
 397 }
 398
 399 bool IsStandard(const char* spec, const Component& scheme) {
 400   return DoIsStandard(spec, scheme);
 401 }
 402
 403 bool IsStandard(const base::char16* spec, const Component& scheme) {
 404   return DoIsStandard(spec, scheme);
 405 }
 406
 407 bool FindAndCompareScheme(const char* str,
 408                           int str_len,
 409                           const char* compare,
 410                           Component* found_scheme) {
 411   return DoFindAndCompareScheme(str, str_len, compare, found_scheme);
 412 }
 413
 414 bool FindAndCompareScheme(const base::char16* str,
 415                           int str_len,
 416                           const char* compare,
 417                           Component* found_scheme) {
 418   return DoFindAndCompareScheme(str, str_len, compare, found_scheme);
 419 }
 420
 421 bool Canonicalize(const char* spec,
 422                   int spec_len,
 423                   bool trim_path_end,
 424                   CharsetConverter* charset_converter,
 425                   CanonOutput* output,
 426                   Parsed* output_parsed) {
 427   return DoCanonicalize(spec, spec_len, trim_path_end, charset_converter,
 428                         output, output_parsed);
 429 }
 430
 431 bool Canonicalize(const base::char16* spec,
 432                   int spec_len,
 433                   bool trim_path_end,
 434                   CharsetConverter* charset_converter,
 435                   CanonOutput* output,
 436                   Parsed* output_parsed) {
 437   return DoCanonicalize(spec, spec_len, trim_path_end, charset_converter,
 438                         output, output_parsed);
 439 }
 440
 441 bool ResolveRelative(const char* base_spec,
 442                      int base_spec_len,
 443                      const Parsed& base_parsed,
 444                      const char* relative,
 445                      int relative_length,
 446                      CharsetConverter* charset_converter,
 447                      CanonOutput* output,
 448                      Parsed* output_parsed) {
 449   return DoResolveRelative(base_spec, base_spec_len, base_parsed,
 450                            relative, relative_length,
 451                            charset_converter, output, output_parsed);
 452 }
 453
 454 bool ResolveRelative(const char* base_spec,
 455                      int base_spec_len,
 456                      const Parsed& base_parsed,
 457                      const base::char16* relative,
 458                      int relative_length,
 459                      CharsetConverter* charset_converter,
 460                      CanonOutput* output,
 461                      Parsed* output_parsed) {
 462   return DoResolveRelative(base_spec, base_spec_len, base_parsed,
 463                            relative, relative_length,
 464                            charset_converter, output, output_parsed);
 465 }
 466
 467 bool ReplaceComponents(const char* spec,
 468                        int spec_len,
 469                        const Parsed& parsed,
 470                        const Replacements<char>& replacements,
 471                        CharsetConverter* charset_converter,
 472                        CanonOutput* output,
 473                        Parsed* out_parsed) {
 474   return DoReplaceComponents(spec, spec_len, parsed, replacements,
 475                              charset_converter, output, out_parsed);
 476 }
 477
 478 bool ReplaceComponents(const char* spec,
 479                        int spec_len,
 480                        const Parsed& parsed,
 481                        const Replacements<base::char16>& replacements,
 482                        CharsetConverter* charset_converter,
 483                        CanonOutput* output,
 484                        Parsed* out_parsed) {
 485   return DoReplaceComponents(spec, spec_len, parsed, replacements,
 486                              charset_converter, output, out_parsed);
 487 }
 488
 489 // Front-ends for LowerCaseEqualsASCII.
 490 bool LowerCaseEqualsASCII(const char* a_begin,
 491                           const char* a_end,
 492                           const char* b) {
 493   return DoLowerCaseEqualsASCII(a_begin, a_end, b);
 494 }
 495
 496 bool LowerCaseEqualsASCII(const char* a_begin,
 497                           const char* a_end,
 498                           const char* b_begin,
 499                           const char* b_end) {
 500   while (a_begin != a_end && b_begin != b_end &&
 501          ToLowerASCII(*a_begin) == *b_begin) {
 502     a_begin++;
 503     b_begin++;
 504   }
 505   return a_begin == a_end && b_begin == b_end;
 506 }
 507
 508 bool LowerCaseEqualsASCII(const base::char16* a_begin,
 509                           const base::char16* a_end,
 510                           const char* b) {
 511   return DoLowerCaseEqualsASCII(a_begin, a_end, b);
 512 }
 513
 514 void DecodeURLEscapeSequences(const char* input,
 515                               int length,
 516                               CanonOutputW* output) {
 517   RawCanonOutputT<char> unescaped_chars;
 518   for (int i = 0; i < length; i++) {
 519     if (input[i] == '%') {
 520       unsigned char ch;
 521       if (DecodeEscaped(input, &i, length, &ch)) {
 522         unescaped_chars.push_back(ch);
 523       } else {
 524         // Invalid escape sequence, copy the percent literal.
 525         unescaped_chars.push_back('%');
 526       }
 527     } else {
 528       // Regular non-escaped 8-bit character.
 529       unescaped_chars.push_back(input[i]);
 530     }
 531   }
 532
 533   // Convert that 8-bit to UTF-16. It's not clear IE does this at all to
 534   // JavaScript URLs, but Firefox and Safari do.
 535   for (int i = 0; i < unescaped_chars.length(); i++) {
 536     unsigned char uch = static_cast<unsigned char>(unescaped_chars.at(i));
 537     if (uch < 0x80) {
 538       // Non-UTF-8, just append directly
 539       output->push_back(uch);
 540     } else {
 541       // next_ch will point to the last character of the decoded
 542       // character.
 543       int next_character = i;
 544       unsigned code_point;
 545       if (ReadUTFChar(unescaped_chars.data(), &next_character,
 546                       unescaped_chars.length(), &code_point)) {
 547         // Valid UTF-8 character, convert to UTF-16.
 548         AppendUTF16Value(code_point, output);
 549         i = next_character;
 550       } else {
 551         // If there are any sequences that are not valid UTF-8, we keep
 552         // invalid code points and promote to UTF-16. We copy all characters
 553         // from the current position to the end of the identified sequence.
 554         while (i < next_character) {
 555           output->push_back(static_cast<unsigned char>(unescaped_chars.at(i)));
 556           i++;
 557         }
 558         output->push_back(static_cast<unsigned char>(unescaped_chars.at(i)));
 559       }
 560     }
 561   }
 562 }
 563
 564 void EncodeURIComponent(const char* input, int length, CanonOutput* output) {
 565   for (int i = 0; i < length; ++i) {
 566     unsigned char c = static_cast<unsigned char>(input[i]);
 567     if (IsComponentChar(c))
 568       output->push_back(c);
 569     else
 570       AppendEscapedChar(c, output);
 571   }
 572 }
 573
 574 bool CompareSchemeComponent(const char* spec,
 575                             const Component& component,
 576                             const char* compare_to) {
 577   return DoCompareSchemeComponent(spec, component, compare_to);
 578 }
 579
 580 bool CompareSchemeComponent(const base::char16* spec,
 581                             const Component& component,
 582                             const char* compare_to) {
 583   return DoCompareSchemeComponent(spec, component, compare_to);
 584 }
 585
 586 }  // namespace url