Enable icu_use_data_file_flag on Android
[chromium-blink-merge.git] / url / url_util.cc
blob2b4f7add4f98665f549c8ce4b0891e0f99993d36
1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "url/url_util.h"
7 #include <string.h>
8 #include <vector>
10 #include "base/logging.h"
11 #include "url/url_canon_internal.h"
12 #include "url/url_file.h"
13 #include "url/url_util_internal.h"
15 namespace url_util {
17 const char kFileScheme[] = "file";
18 const char kFileSystemScheme[] = "filesystem";
19 const char kMailtoScheme[] = "mailto";
21 namespace {
23 // ASCII-specific tolower. The standard library's tolower is locale sensitive,
24 // so we don't want to use it here.
25 template <class Char> inline Char ToLowerASCII(Char c) {
26 return (c >= 'A' && c <= 'Z') ? (c + ('a' - 'A')) : c;
29 // Backend for LowerCaseEqualsASCII.
30 template<typename Iter>
31 inline bool DoLowerCaseEqualsASCII(Iter a_begin, Iter a_end, const char* b) {
32 for (Iter it = a_begin; it != a_end; ++it, ++b) {
33 if (!*b || ToLowerASCII(*it) != *b)
34 return false;
36 return *b == 0;
39 const int kNumStandardURLSchemes = 8;
40 const char* kStandardURLSchemes[kNumStandardURLSchemes] = {
41 "http",
42 "https",
43 kFileScheme, // Yes, file urls can have a hostname!
44 "ftp",
45 "gopher",
46 "ws", // WebSocket.
47 "wss", // WebSocket secure.
48 kFileSystemScheme,
51 // List of the currently installed standard schemes. This list is lazily
52 // initialized by InitStandardSchemes and is leaked on shutdown to prevent
53 // any destructors from being called that will slow us down or cause problems.
54 std::vector<const char*>* standard_schemes = NULL;
56 // See the LockStandardSchemes declaration in the header.
57 bool standard_schemes_locked = false;
59 // Ensures that the standard_schemes list is initialized, does nothing if it
60 // already has values.
61 void InitStandardSchemes() {
62 if (standard_schemes)
63 return;
64 standard_schemes = new std::vector<const char*>;
65 for (int i = 0; i < kNumStandardURLSchemes; i++)
66 standard_schemes->push_back(kStandardURLSchemes[i]);
69 // Given a string and a range inside the string, compares it to the given
70 // lower-case |compare_to| buffer.
71 template<typename CHAR>
72 inline bool DoCompareSchemeComponent(const CHAR* spec,
73 const url_parse::Component& component,
74 const char* compare_to) {
75 if (!component.is_nonempty())
76 return compare_to[0] == 0; // When component is empty, match empty scheme.
77 return LowerCaseEqualsASCII(&spec[component.begin],
78 &spec[component.end()],
79 compare_to);
82 // Returns true if the given scheme identified by |scheme| within |spec| is one
83 // of the registered "standard" schemes.
84 template<typename CHAR>
85 bool DoIsStandard(const CHAR* spec, const url_parse::Component& scheme) {
86 if (!scheme.is_nonempty())
87 return false; // Empty or invalid schemes are non-standard.
89 InitStandardSchemes();
90 for (size_t i = 0; i < standard_schemes->size(); i++) {
91 if (LowerCaseEqualsASCII(&spec[scheme.begin], &spec[scheme.end()],
92 standard_schemes->at(i)))
93 return true;
95 return false;
98 template<typename CHAR>
99 bool DoFindAndCompareScheme(const CHAR* str,
100 int str_len,
101 const char* compare,
102 url_parse::Component* found_scheme) {
103 // Before extracting scheme, canonicalize the URL to remove any whitespace.
104 // This matches the canonicalization done in DoCanonicalize function.
105 url_canon::RawCanonOutputT<CHAR> whitespace_buffer;
106 int spec_len;
107 const CHAR* spec = RemoveURLWhitespace(str, str_len,
108 &whitespace_buffer, &spec_len);
110 url_parse::Component our_scheme;
111 if (!url_parse::ExtractScheme(spec, spec_len, &our_scheme)) {
112 // No scheme.
113 if (found_scheme)
114 *found_scheme = url_parse::Component();
115 return false;
117 if (found_scheme)
118 *found_scheme = our_scheme;
119 return DoCompareSchemeComponent(spec, our_scheme, compare);
122 template<typename CHAR>
123 bool DoCanonicalize(const CHAR* in_spec, int in_spec_len,
124 bool trim_path_end,
125 url_canon::CharsetConverter* charset_converter,
126 url_canon::CanonOutput* output,
127 url_parse::Parsed* output_parsed) {
128 // Remove any whitespace from the middle of the relative URL, possibly
129 // copying to the new buffer.
130 url_canon::RawCanonOutputT<CHAR> whitespace_buffer;
131 int spec_len;
132 const CHAR* spec = RemoveURLWhitespace(in_spec, in_spec_len,
133 &whitespace_buffer, &spec_len);
135 url_parse::Parsed parsed_input;
136 #ifdef WIN32
137 // For Windows, we allow things that look like absolute Windows paths to be
138 // fixed up magically to file URLs. This is done for IE compatability. For
139 // example, this will change "c:/foo" into a file URL rather than treating
140 // it as a URL with the protocol "c". It also works for UNC ("\\foo\bar.txt").
141 // There is similar logic in url_canon_relative.cc for
143 // For Max & Unix, we don't do this (the equivalent would be "/foo/bar" which
144 // has no meaning as an absolute path name. This is because browsers on Mac
145 // & Unix don't generally do this, so there is no compatibility reason for
146 // doing so.
147 if (url_parse::DoesBeginUNCPath(spec, 0, spec_len, false) ||
148 url_parse::DoesBeginWindowsDriveSpec(spec, 0, spec_len)) {
149 url_parse::ParseFileURL(spec, spec_len, &parsed_input);
150 return url_canon::CanonicalizeFileURL(spec, spec_len, parsed_input,
151 charset_converter,
152 output, output_parsed);
154 #endif
156 url_parse::Component scheme;
157 if (!url_parse::ExtractScheme(spec, spec_len, &scheme))
158 return false;
160 // This is the parsed version of the input URL, we have to canonicalize it
161 // before storing it in our object.
162 bool success;
163 if (DoCompareSchemeComponent(spec, scheme, kFileScheme)) {
164 // File URLs are special.
165 url_parse::ParseFileURL(spec, spec_len, &parsed_input);
166 success = url_canon::CanonicalizeFileURL(spec, spec_len, parsed_input,
167 charset_converter, output,
168 output_parsed);
169 } else if (DoCompareSchemeComponent(spec, scheme, kFileSystemScheme)) {
170 // Filesystem URLs are special.
171 url_parse::ParseFileSystemURL(spec, spec_len, &parsed_input);
172 success = url_canon::CanonicalizeFileSystemURL(spec, spec_len,
173 parsed_input,
174 charset_converter,
175 output, output_parsed);
177 } else if (DoIsStandard(spec, scheme)) {
178 // All "normal" URLs.
179 url_parse::ParseStandardURL(spec, spec_len, &parsed_input);
180 success = url_canon::CanonicalizeStandardURL(spec, spec_len, parsed_input,
181 charset_converter,
182 output, output_parsed);
184 } else if (DoCompareSchemeComponent(spec, scheme, kMailtoScheme)) {
185 // Mailto are treated like a standard url with only a scheme, path, query
186 url_parse::ParseMailtoURL(spec, spec_len, &parsed_input);
187 success = url_canon::CanonicalizeMailtoURL(spec, spec_len, parsed_input,
188 output, output_parsed);
190 } else {
191 // "Weird" URLs like data: and javascript:
192 url_parse::ParsePathURL(spec, spec_len, trim_path_end, &parsed_input);
193 success = url_canon::CanonicalizePathURL(spec, spec_len, parsed_input,
194 output, output_parsed);
196 return success;
199 template<typename CHAR>
200 bool DoResolveRelative(const char* base_spec,
201 int base_spec_len,
202 const url_parse::Parsed& base_parsed,
203 const CHAR* in_relative,
204 int in_relative_length,
205 url_canon::CharsetConverter* charset_converter,
206 url_canon::CanonOutput* output,
207 url_parse::Parsed* output_parsed) {
208 // Remove any whitespace from the middle of the relative URL, possibly
209 // copying to the new buffer.
210 url_canon::RawCanonOutputT<CHAR> whitespace_buffer;
211 int relative_length;
212 const CHAR* relative = RemoveURLWhitespace(in_relative, in_relative_length,
213 &whitespace_buffer,
214 &relative_length);
215 bool base_is_authority_based = false;
216 bool base_is_hierarchical = false;
217 if (base_spec &&
218 base_parsed.scheme.is_nonempty()) {
219 int after_scheme = base_parsed.scheme.end() + 1; // Skip past the colon.
220 int num_slashes = url_parse::CountConsecutiveSlashes(
221 base_spec, after_scheme, base_spec_len);
222 base_is_authority_based = num_slashes > 1;
223 base_is_hierarchical = num_slashes > 0;
226 bool standard_base_scheme =
227 base_parsed.scheme.is_nonempty() &&
228 DoIsStandard(base_spec, base_parsed.scheme);
230 bool is_relative;
231 url_parse::Component relative_component;
232 if (!url_canon::IsRelativeURL(base_spec, base_parsed,
233 relative, relative_length,
234 (base_is_hierarchical || standard_base_scheme),
235 &is_relative,
236 &relative_component)) {
237 // Error resolving.
238 return false;
241 // Pretend for a moment that |base_spec| is a standard URL. Normally
242 // non-standard URLs are treated as PathURLs, but if the base has an
243 // authority we would like to preserve it.
244 if (is_relative && base_is_authority_based && !standard_base_scheme) {
245 url_parse::Parsed base_parsed_authority;
246 ParseStandardURL(base_spec, base_spec_len, &base_parsed_authority);
247 if (base_parsed_authority.host.is_nonempty()) {
248 bool did_resolve_succeed =
249 url_canon::ResolveRelativeURL(base_spec, base_parsed_authority,
250 false, relative,
251 relative_component, charset_converter,
252 output, output_parsed);
253 // The output_parsed is incorrect at this point (because it was built
254 // based on base_parsed_authority instead of base_parsed) and needs to be
255 // re-created.
256 ParsePathURL(output->data(), output->length(), true,
257 output_parsed);
258 return did_resolve_succeed;
260 } else if (is_relative) {
261 // Relative, resolve and canonicalize.
262 bool file_base_scheme = base_parsed.scheme.is_nonempty() &&
263 DoCompareSchemeComponent(base_spec, base_parsed.scheme, kFileScheme);
264 return url_canon::ResolveRelativeURL(base_spec, base_parsed,
265 file_base_scheme, relative,
266 relative_component, charset_converter,
267 output, output_parsed);
270 // Not relative, canonicalize the input.
271 return DoCanonicalize(relative, relative_length, true, charset_converter,
272 output, output_parsed);
275 template<typename CHAR>
276 bool DoReplaceComponents(const char* spec,
277 int spec_len,
278 const url_parse::Parsed& parsed,
279 const url_canon::Replacements<CHAR>& replacements,
280 url_canon::CharsetConverter* charset_converter,
281 url_canon::CanonOutput* output,
282 url_parse::Parsed* out_parsed) {
283 // If the scheme is overridden, just do a simple string substitution and
284 // reparse the whole thing. There are lots of edge cases that we really don't
285 // want to deal with. Like what happens if I replace "http://e:8080/foo"
286 // with a file. Does it become "file:///E:/8080/foo" where the port number
287 // becomes part of the path? Parsing that string as a file URL says "yes"
288 // but almost no sane rule for dealing with the components individually would
289 // come up with that.
291 // Why allow these crazy cases at all? Programatically, there is almost no
292 // case for replacing the scheme. The most common case for hitting this is
293 // in JS when building up a URL using the location object. In this case, the
294 // JS code expects the string substitution behavior:
295 // http://www.w3.org/TR/2008/WD-html5-20080610/structured.html#common3
296 if (replacements.IsSchemeOverridden()) {
297 // Canonicalize the new scheme so it is 8-bit and can be concatenated with
298 // the existing spec.
299 url_canon::RawCanonOutput<128> scheme_replaced;
300 url_parse::Component scheme_replaced_parsed;
301 url_canon::CanonicalizeScheme(
302 replacements.sources().scheme,
303 replacements.components().scheme,
304 &scheme_replaced, &scheme_replaced_parsed);
306 // We can assume that the input is canonicalized, which means it always has
307 // a colon after the scheme (or where the scheme would be).
308 int spec_after_colon = parsed.scheme.is_valid() ? parsed.scheme.end() + 1
309 : 1;
310 if (spec_len - spec_after_colon > 0) {
311 scheme_replaced.Append(&spec[spec_after_colon],
312 spec_len - spec_after_colon);
315 // We now need to completely re-parse the resulting string since its meaning
316 // may have changed with the different scheme.
317 url_canon::RawCanonOutput<128> recanonicalized;
318 url_parse::Parsed recanonicalized_parsed;
319 DoCanonicalize(scheme_replaced.data(), scheme_replaced.length(), true,
320 charset_converter,
321 &recanonicalized, &recanonicalized_parsed);
323 // Recurse using the version with the scheme already replaced. This will now
324 // use the replacement rules for the new scheme.
326 // Warning: this code assumes that ReplaceComponents will re-check all
327 // components for validity. This is because we can't fail if DoCanonicalize
328 // failed above since theoretically the thing making it fail could be
329 // getting replaced here. If ReplaceComponents didn't re-check everything,
330 // we wouldn't know if something *not* getting replaced is a problem.
331 // If the scheme-specific replacers are made more intelligent so they don't
332 // re-check everything, we should instead recanonicalize the whole thing
333 // after this call to check validity (this assumes replacing the scheme is
334 // much much less common than other types of replacements, like clearing the
335 // ref).
336 url_canon::Replacements<CHAR> replacements_no_scheme = replacements;
337 replacements_no_scheme.SetScheme(NULL, url_parse::Component());
338 return DoReplaceComponents(recanonicalized.data(), recanonicalized.length(),
339 recanonicalized_parsed, replacements_no_scheme,
340 charset_converter, output, out_parsed);
343 // If we get here, then we know the scheme doesn't need to be replaced, so can
344 // just key off the scheme in the spec to know how to do the replacements.
345 if (DoCompareSchemeComponent(spec, parsed.scheme, kFileScheme)) {
346 return url_canon::ReplaceFileURL(spec, parsed, replacements,
347 charset_converter, output, out_parsed);
349 if (DoCompareSchemeComponent(spec, parsed.scheme, kFileSystemScheme)) {
350 return url_canon::ReplaceFileSystemURL(spec, parsed, replacements,
351 charset_converter, output,
352 out_parsed);
354 if (DoIsStandard(spec, parsed.scheme)) {
355 return url_canon::ReplaceStandardURL(spec, parsed, replacements,
356 charset_converter, output, out_parsed);
358 if (DoCompareSchemeComponent(spec, parsed.scheme, kMailtoScheme)) {
359 return url_canon::ReplaceMailtoURL(spec, parsed, replacements,
360 output, out_parsed);
363 // Default is a path URL.
364 return url_canon::ReplacePathURL(spec, parsed, replacements,
365 output, out_parsed);
368 } // namespace
370 void Initialize() {
371 InitStandardSchemes();
374 void Shutdown() {
375 if (standard_schemes) {
376 delete standard_schemes;
377 standard_schemes = NULL;
381 void AddStandardScheme(const char* new_scheme) {
382 // If this assert triggers, it means you've called AddStandardScheme after
383 // LockStandardSchemes have been called (see the header file for
384 // LockStandardSchemes for more).
386 // This normally means you're trying to set up a new standard scheme too late
387 // in your application's init process. Locate where your app does this
388 // initialization and calls LockStandardScheme, and add your new standard
389 // scheme there.
390 DCHECK(!standard_schemes_locked) <<
391 "Trying to add a standard scheme after the list has been locked.";
393 size_t scheme_len = strlen(new_scheme);
394 if (scheme_len == 0)
395 return;
397 // Dulicate the scheme into a new buffer and add it to the list of standard
398 // schemes. This pointer will be leaked on shutdown.
399 char* dup_scheme = new char[scheme_len + 1];
400 memcpy(dup_scheme, new_scheme, scheme_len + 1);
402 InitStandardSchemes();
403 standard_schemes->push_back(dup_scheme);
406 void LockStandardSchemes() {
407 standard_schemes_locked = true;
410 bool IsStandard(const char* spec, const url_parse::Component& scheme) {
411 return DoIsStandard(spec, scheme);
414 bool IsStandard(const base::char16* spec, const url_parse::Component& scheme) {
415 return DoIsStandard(spec, scheme);
418 bool FindAndCompareScheme(const char* str,
419 int str_len,
420 const char* compare,
421 url_parse::Component* found_scheme) {
422 return DoFindAndCompareScheme(str, str_len, compare, found_scheme);
425 bool FindAndCompareScheme(const base::char16* str,
426 int str_len,
427 const char* compare,
428 url_parse::Component* found_scheme) {
429 return DoFindAndCompareScheme(str, str_len, compare, found_scheme);
432 bool Canonicalize(const char* spec,
433 int spec_len,
434 bool trim_path_end,
435 url_canon::CharsetConverter* charset_converter,
436 url_canon::CanonOutput* output,
437 url_parse::Parsed* output_parsed) {
438 return DoCanonicalize(spec, spec_len, trim_path_end, charset_converter,
439 output, output_parsed);
442 bool Canonicalize(const base::char16* spec,
443 int spec_len,
444 bool trim_path_end,
445 url_canon::CharsetConverter* charset_converter,
446 url_canon::CanonOutput* output,
447 url_parse::Parsed* output_parsed) {
448 return DoCanonicalize(spec, spec_len, trim_path_end, charset_converter,
449 output, output_parsed);
452 bool ResolveRelative(const char* base_spec,
453 int base_spec_len,
454 const url_parse::Parsed& base_parsed,
455 const char* relative,
456 int relative_length,
457 url_canon::CharsetConverter* charset_converter,
458 url_canon::CanonOutput* output,
459 url_parse::Parsed* output_parsed) {
460 return DoResolveRelative(base_spec, base_spec_len, base_parsed,
461 relative, relative_length,
462 charset_converter, output, output_parsed);
465 bool ResolveRelative(const char* base_spec,
466 int base_spec_len,
467 const url_parse::Parsed& base_parsed,
468 const base::char16* relative,
469 int relative_length,
470 url_canon::CharsetConverter* charset_converter,
471 url_canon::CanonOutput* output,
472 url_parse::Parsed* output_parsed) {
473 return DoResolveRelative(base_spec, base_spec_len, base_parsed,
474 relative, relative_length,
475 charset_converter, output, output_parsed);
478 bool ReplaceComponents(const char* spec,
479 int spec_len,
480 const url_parse::Parsed& parsed,
481 const url_canon::Replacements<char>& replacements,
482 url_canon::CharsetConverter* charset_converter,
483 url_canon::CanonOutput* output,
484 url_parse::Parsed* out_parsed) {
485 return DoReplaceComponents(spec, spec_len, parsed, replacements,
486 charset_converter, output, out_parsed);
489 bool ReplaceComponents(
490 const char* spec,
491 int spec_len,
492 const url_parse::Parsed& parsed,
493 const url_canon::Replacements<base::char16>& replacements,
494 url_canon::CharsetConverter* charset_converter,
495 url_canon::CanonOutput* output,
496 url_parse::Parsed* out_parsed) {
497 return DoReplaceComponents(spec, spec_len, parsed, replacements,
498 charset_converter, output, out_parsed);
501 // Front-ends for LowerCaseEqualsASCII.
502 bool LowerCaseEqualsASCII(const char* a_begin,
503 const char* a_end,
504 const char* b) {
505 return DoLowerCaseEqualsASCII(a_begin, a_end, b);
508 bool LowerCaseEqualsASCII(const char* a_begin,
509 const char* a_end,
510 const char* b_begin,
511 const char* b_end) {
512 while (a_begin != a_end && b_begin != b_end &&
513 ToLowerASCII(*a_begin) == *b_begin) {
514 a_begin++;
515 b_begin++;
517 return a_begin == a_end && b_begin == b_end;
520 bool LowerCaseEqualsASCII(const base::char16* a_begin,
521 const base::char16* a_end,
522 const char* b) {
523 return DoLowerCaseEqualsASCII(a_begin, a_end, b);
526 void DecodeURLEscapeSequences(const char* input, int length,
527 url_canon::CanonOutputW* output) {
528 url_canon::RawCanonOutputT<char> unescaped_chars;
529 for (int i = 0; i < length; i++) {
530 if (input[i] == '%') {
531 unsigned char ch;
532 if (url_canon::DecodeEscaped(input, &i, length, &ch)) {
533 unescaped_chars.push_back(ch);
534 } else {
535 // Invalid escape sequence, copy the percent literal.
536 unescaped_chars.push_back('%');
538 } else {
539 // Regular non-escaped 8-bit character.
540 unescaped_chars.push_back(input[i]);
544 // Convert that 8-bit to UTF-16. It's not clear IE does this at all to
545 // JavaScript URLs, but Firefox and Safari do.
546 for (int i = 0; i < unescaped_chars.length(); i++) {
547 unsigned char uch = static_cast<unsigned char>(unescaped_chars.at(i));
548 if (uch < 0x80) {
549 // Non-UTF-8, just append directly
550 output->push_back(uch);
551 } else {
552 // next_ch will point to the last character of the decoded
553 // character.
554 int next_character = i;
555 unsigned code_point;
556 if (url_canon::ReadUTFChar(unescaped_chars.data(), &next_character,
557 unescaped_chars.length(), &code_point)) {
558 // Valid UTF-8 character, convert to UTF-16.
559 url_canon::AppendUTF16Value(code_point, output);
560 i = next_character;
561 } else {
562 // If there are any sequences that are not valid UTF-8, we keep
563 // invalid code points and promote to UTF-16. We copy all characters
564 // from the current position to the end of the identified sequence.
565 while (i < next_character) {
566 output->push_back(static_cast<unsigned char>(unescaped_chars.at(i)));
567 i++;
569 output->push_back(static_cast<unsigned char>(unescaped_chars.at(i)));
575 void EncodeURIComponent(const char* input, int length,
576 url_canon::CanonOutput* output) {
577 for (int i = 0; i < length; ++i) {
578 unsigned char c = static_cast<unsigned char>(input[i]);
579 if (url_canon::IsComponentChar(c))
580 output->push_back(c);
581 else
582 AppendEscapedChar(c, output);
586 bool CompareSchemeComponent(const char* spec,
587 const url_parse::Component& component,
588 const char* compare_to) {
589 return DoCompareSchemeComponent(spec, component, compare_to);
592 bool CompareSchemeComponent(const base::char16* spec,
593 const url_parse::Component& component,
594 const char* compare_to) {
595 return DoCompareSchemeComponent(spec, component, compare_to);
598 } // namespace url_util