Fix translation extraction failure.
[chromium-blink-merge.git] / url / url_util.cc
blob21bf3cc462ef871470302bc00ca2c033ec70c748
1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "url/url_util.h"
7 #include <string.h>
8 #include <vector>
10 #include "base/debug/leak_annotations.h"
11 #include "base/logging.h"
12 #include "base/strings/string_util.h"
13 #include "url/url_canon_internal.h"
14 #include "url/url_file.h"
15 #include "url/url_util_internal.h"
17 namespace url {
19 namespace {
21 const int kNumStandardURLSchemes = 8;
22 const SchemeWithType kStandardURLSchemes[kNumStandardURLSchemes] = {
23 {kHttpScheme, SCHEME_WITH_PORT},
24 {kHttpsScheme, SCHEME_WITH_PORT},
25 // Yes, file URLs can have a hostname, so file URLs should be handled as
26 // "standard". File URLs never have a port as specified by the SchemeType
27 // field.
28 {kFileScheme, SCHEME_WITHOUT_PORT},
29 {kFtpScheme, SCHEME_WITH_PORT},
30 {kGopherScheme, SCHEME_WITH_PORT},
31 {kWsScheme, SCHEME_WITH_PORT}, // WebSocket.
32 {kWssScheme, SCHEME_WITH_PORT}, // WebSocket secure.
33 {kFileSystemScheme, SCHEME_WITHOUT_AUTHORITY},
36 // List of the currently installed standard schemes. This list is lazily
37 // initialized by InitStandardSchemes and is leaked on shutdown to prevent
38 // any destructors from being called that will slow us down or cause problems.
39 std::vector<SchemeWithType>* standard_schemes = NULL;
41 // See the LockStandardSchemes declaration in the header.
42 bool standard_schemes_locked = false;
44 // This template converts a given character type to the corresponding
45 // StringPiece type.
46 template<typename CHAR> struct CharToStringPiece {
48 template<> struct CharToStringPiece<char> {
49 typedef base::StringPiece Piece;
51 template<> struct CharToStringPiece<base::char16> {
52 typedef base::StringPiece16 Piece;
55 // Ensures that the standard_schemes list is initialized, does nothing if it
56 // already has values.
57 void InitStandardSchemes() {
58 if (standard_schemes)
59 return;
60 standard_schemes = new std::vector<SchemeWithType>;
61 for (int i = 0; i < kNumStandardURLSchemes; i++)
62 standard_schemes->push_back(kStandardURLSchemes[i]);
65 // Given a string and a range inside the string, compares it to the given
66 // lower-case |compare_to| buffer.
67 template<typename CHAR>
68 inline bool DoCompareSchemeComponent(const CHAR* spec,
69 const Component& component,
70 const char* compare_to) {
71 if (!component.is_nonempty())
72 return compare_to[0] == 0; // When component is empty, match empty scheme.
73 return base::LowerCaseEqualsASCII(
74 typename CharToStringPiece<CHAR>::Piece(
75 &spec[component.begin], component.len),
76 compare_to);
79 // Returns true and sets |type| to the SchemeType of the given scheme
80 // identified by |scheme| within |spec| if the scheme is one of the registered
81 // "standard" schemes.
82 template<typename CHAR>
83 bool DoIsStandard(const CHAR* spec,
84 const Component& scheme,
85 SchemeType* type) {
86 if (!scheme.is_nonempty())
87 return false; // Empty or invalid schemes are non-standard.
89 InitStandardSchemes();
90 for (size_t i = 0; i < standard_schemes->size(); i++) {
91 if (base::LowerCaseEqualsASCII(
92 typename CharToStringPiece<CHAR>::Piece(
93 &spec[scheme.begin], scheme.len),
94 standard_schemes->at(i).scheme)) {
95 *type = standard_schemes->at(i).type;
96 return true;
99 return false;
102 template<typename CHAR>
103 bool DoFindAndCompareScheme(const CHAR* str,
104 int str_len,
105 const char* compare,
106 Component* found_scheme) {
107 // Before extracting scheme, canonicalize the URL to remove any whitespace.
108 // This matches the canonicalization done in DoCanonicalize function.
109 RawCanonOutputT<CHAR> whitespace_buffer;
110 int spec_len;
111 const CHAR* spec = RemoveURLWhitespace(str, str_len,
112 &whitespace_buffer, &spec_len);
114 Component our_scheme;
115 if (!ExtractScheme(spec, spec_len, &our_scheme)) {
116 // No scheme.
117 if (found_scheme)
118 *found_scheme = Component();
119 return false;
121 if (found_scheme)
122 *found_scheme = our_scheme;
123 return DoCompareSchemeComponent(spec, our_scheme, compare);
126 template<typename CHAR>
127 bool DoCanonicalize(const CHAR* in_spec,
128 int in_spec_len,
129 bool trim_path_end,
130 CharsetConverter* charset_converter,
131 CanonOutput* output,
132 Parsed* output_parsed) {
133 // Remove any whitespace from the middle of the relative URL, possibly
134 // copying to the new buffer.
135 RawCanonOutputT<CHAR> whitespace_buffer;
136 int spec_len;
137 const CHAR* spec = RemoveURLWhitespace(in_spec, in_spec_len,
138 &whitespace_buffer, &spec_len);
140 Parsed parsed_input;
141 #ifdef WIN32
142 // For Windows, we allow things that look like absolute Windows paths to be
143 // fixed up magically to file URLs. This is done for IE compatibility. For
144 // example, this will change "c:/foo" into a file URL rather than treating
145 // it as a URL with the protocol "c". It also works for UNC ("\\foo\bar.txt").
146 // There is similar logic in url_canon_relative.cc for
148 // For Max & Unix, we don't do this (the equivalent would be "/foo/bar" which
149 // has no meaning as an absolute path name. This is because browsers on Mac
150 // & Unix don't generally do this, so there is no compatibility reason for
151 // doing so.
152 if (DoesBeginUNCPath(spec, 0, spec_len, false) ||
153 DoesBeginWindowsDriveSpec(spec, 0, spec_len)) {
154 ParseFileURL(spec, spec_len, &parsed_input);
155 return CanonicalizeFileURL(spec, spec_len, parsed_input, charset_converter,
156 output, output_parsed);
158 #endif
160 Component scheme;
161 if (!ExtractScheme(spec, spec_len, &scheme))
162 return false;
164 // This is the parsed version of the input URL, we have to canonicalize it
165 // before storing it in our object.
166 bool success;
167 SchemeType unused_scheme_type = SCHEME_WITH_PORT;
168 if (DoCompareSchemeComponent(spec, scheme, url::kFileScheme)) {
169 // File URLs are special.
170 ParseFileURL(spec, spec_len, &parsed_input);
171 success = CanonicalizeFileURL(spec, spec_len, parsed_input,
172 charset_converter, output, output_parsed);
173 } else if (DoCompareSchemeComponent(spec, scheme, url::kFileSystemScheme)) {
174 // Filesystem URLs are special.
175 ParseFileSystemURL(spec, spec_len, &parsed_input);
176 success = CanonicalizeFileSystemURL(spec, spec_len, parsed_input,
177 charset_converter, output,
178 output_parsed);
180 } else if (DoIsStandard(spec, scheme, &unused_scheme_type)) {
181 // All "normal" URLs.
182 ParseStandardURL(spec, spec_len, &parsed_input);
183 success = CanonicalizeStandardURL(spec, spec_len, parsed_input,
184 charset_converter, output, output_parsed);
186 } else if (DoCompareSchemeComponent(spec, scheme, url::kMailToScheme)) {
187 // Mailto URLs are treated like standard URLs, with only a scheme, path,
188 // and query.
189 ParseMailtoURL(spec, spec_len, &parsed_input);
190 success = CanonicalizeMailtoURL(spec, spec_len, parsed_input, output,
191 output_parsed);
193 } else {
194 // "Weird" URLs like data: and javascript:.
195 ParsePathURL(spec, spec_len, trim_path_end, &parsed_input);
196 success = CanonicalizePathURL(spec, spec_len, parsed_input, output,
197 output_parsed);
199 return success;
202 template<typename CHAR>
203 bool DoResolveRelative(const char* base_spec,
204 int base_spec_len,
205 const Parsed& base_parsed,
206 const CHAR* in_relative,
207 int in_relative_length,
208 CharsetConverter* charset_converter,
209 CanonOutput* output,
210 Parsed* output_parsed) {
211 // Remove any whitespace from the middle of the relative URL, possibly
212 // copying to the new buffer.
213 RawCanonOutputT<CHAR> whitespace_buffer;
214 int relative_length;
215 const CHAR* relative = RemoveURLWhitespace(in_relative, in_relative_length,
216 &whitespace_buffer,
217 &relative_length);
218 bool base_is_authority_based = false;
219 bool base_is_hierarchical = false;
220 if (base_spec &&
221 base_parsed.scheme.is_nonempty()) {
222 int after_scheme = base_parsed.scheme.end() + 1; // Skip past the colon.
223 int num_slashes = CountConsecutiveSlashes(base_spec, after_scheme,
224 base_spec_len);
225 base_is_authority_based = num_slashes > 1;
226 base_is_hierarchical = num_slashes > 0;
229 SchemeType unused_scheme_type = SCHEME_WITH_PORT;
230 bool standard_base_scheme =
231 base_parsed.scheme.is_nonempty() &&
232 DoIsStandard(base_spec, base_parsed.scheme, &unused_scheme_type);
234 bool is_relative;
235 Component relative_component;
236 if (!IsRelativeURL(base_spec, base_parsed, relative, relative_length,
237 (base_is_hierarchical || standard_base_scheme),
238 &is_relative, &relative_component)) {
239 // Error resolving.
240 return false;
243 // Pretend for a moment that |base_spec| is a standard URL. Normally
244 // non-standard URLs are treated as PathURLs, but if the base has an
245 // authority we would like to preserve it.
246 if (is_relative && base_is_authority_based && !standard_base_scheme) {
247 Parsed base_parsed_authority;
248 ParseStandardURL(base_spec, base_spec_len, &base_parsed_authority);
249 if (base_parsed_authority.host.is_nonempty()) {
250 RawCanonOutputT<char> temporary_output;
251 bool did_resolve_succeed =
252 ResolveRelativeURL(base_spec, base_parsed_authority, false, relative,
253 relative_component, charset_converter,
254 &temporary_output, output_parsed);
255 // The output_parsed is incorrect at this point (because it was built
256 // based on base_parsed_authority instead of base_parsed) and needs to be
257 // re-created.
258 DoCanonicalize(temporary_output.data(), temporary_output.length(), true,
259 charset_converter, output, output_parsed);
260 return did_resolve_succeed;
262 } else if (is_relative) {
263 // Relative, resolve and canonicalize.
264 bool file_base_scheme = base_parsed.scheme.is_nonempty() &&
265 DoCompareSchemeComponent(base_spec, base_parsed.scheme, kFileScheme);
266 return ResolveRelativeURL(base_spec, base_parsed, file_base_scheme, relative,
267 relative_component, charset_converter, output,
268 output_parsed);
271 // Not relative, canonicalize the input.
272 return DoCanonicalize(relative, relative_length, true, charset_converter,
273 output, output_parsed);
276 template<typename CHAR>
277 bool DoReplaceComponents(const char* spec,
278 int spec_len,
279 const Parsed& parsed,
280 const Replacements<CHAR>& replacements,
281 CharsetConverter* charset_converter,
282 CanonOutput* output,
283 Parsed* out_parsed) {
284 // If the scheme is overridden, just do a simple string substitution and
285 // re-parse the whole thing. There are lots of edge cases that we really don't
286 // want to deal with. Like what happens if I replace "http://e:8080/foo"
287 // with a file. Does it become "file:///E:/8080/foo" where the port number
288 // becomes part of the path? Parsing that string as a file URL says "yes"
289 // but almost no sane rule for dealing with the components individually would
290 // come up with that.
292 // Why allow these crazy cases at all? Programatically, there is almost no
293 // case for replacing the scheme. The most common case for hitting this is
294 // in JS when building up a URL using the location object. In this case, the
295 // JS code expects the string substitution behavior:
296 // http://www.w3.org/TR/2008/WD-html5-20080610/structured.html#common3
297 if (replacements.IsSchemeOverridden()) {
298 // Canonicalize the new scheme so it is 8-bit and can be concatenated with
299 // the existing spec.
300 RawCanonOutput<128> scheme_replaced;
301 Component scheme_replaced_parsed;
302 CanonicalizeScheme(replacements.sources().scheme,
303 replacements.components().scheme,
304 &scheme_replaced, &scheme_replaced_parsed);
306 // We can assume that the input is canonicalized, which means it always has
307 // a colon after the scheme (or where the scheme would be).
308 int spec_after_colon = parsed.scheme.is_valid() ? parsed.scheme.end() + 1
309 : 1;
310 if (spec_len - spec_after_colon > 0) {
311 scheme_replaced.Append(&spec[spec_after_colon],
312 spec_len - spec_after_colon);
315 // We now need to completely re-parse the resulting string since its meaning
316 // may have changed with the different scheme.
317 RawCanonOutput<128> recanonicalized;
318 Parsed recanonicalized_parsed;
319 DoCanonicalize(scheme_replaced.data(), scheme_replaced.length(), true,
320 charset_converter,
321 &recanonicalized, &recanonicalized_parsed);
323 // Recurse using the version with the scheme already replaced. This will now
324 // use the replacement rules for the new scheme.
326 // Warning: this code assumes that ReplaceComponents will re-check all
327 // components for validity. This is because we can't fail if DoCanonicalize
328 // failed above since theoretically the thing making it fail could be
329 // getting replaced here. If ReplaceComponents didn't re-check everything,
330 // we wouldn't know if something *not* getting replaced is a problem.
331 // If the scheme-specific replacers are made more intelligent so they don't
332 // re-check everything, we should instead re-canonicalize the whole thing
333 // after this call to check validity (this assumes replacing the scheme is
334 // much much less common than other types of replacements, like clearing the
335 // ref).
336 Replacements<CHAR> replacements_no_scheme = replacements;
337 replacements_no_scheme.SetScheme(NULL, Component());
338 return DoReplaceComponents(recanonicalized.data(), recanonicalized.length(),
339 recanonicalized_parsed, replacements_no_scheme,
340 charset_converter, output, out_parsed);
343 // If we get here, then we know the scheme doesn't need to be replaced, so can
344 // just key off the scheme in the spec to know how to do the replacements.
345 if (DoCompareSchemeComponent(spec, parsed.scheme, url::kFileScheme)) {
346 return ReplaceFileURL(spec, parsed, replacements, charset_converter, output,
347 out_parsed);
349 if (DoCompareSchemeComponent(spec, parsed.scheme, url::kFileSystemScheme)) {
350 return ReplaceFileSystemURL(spec, parsed, replacements, charset_converter,
351 output, out_parsed);
353 SchemeType unused_scheme_type = SCHEME_WITH_PORT;
354 if (DoIsStandard(spec, parsed.scheme, &unused_scheme_type)) {
355 return ReplaceStandardURL(spec, parsed, replacements, charset_converter,
356 output, out_parsed);
358 if (DoCompareSchemeComponent(spec, parsed.scheme, url::kMailToScheme)) {
359 return ReplaceMailtoURL(spec, parsed, replacements, output, out_parsed);
362 // Default is a path URL.
363 return ReplacePathURL(spec, parsed, replacements, output, out_parsed);
366 } // namespace
368 void Initialize() {
369 InitStandardSchemes();
372 void Shutdown() {
373 if (standard_schemes) {
374 delete standard_schemes;
375 standard_schemes = NULL;
379 void AddStandardScheme(const char* new_scheme,
380 SchemeType type) {
381 // If this assert triggers, it means you've called AddStandardScheme after
382 // LockStandardSchemes have been called (see the header file for
383 // LockStandardSchemes for more).
385 // This normally means you're trying to set up a new standard scheme too late
386 // in your application's init process. Locate where your app does this
387 // initialization and calls LockStandardSchemes, and add your new standard
388 // scheme there.
389 DCHECK(!standard_schemes_locked) <<
390 "Trying to add a standard scheme after the list has been locked.";
392 size_t scheme_len = strlen(new_scheme);
393 if (scheme_len == 0)
394 return;
396 // Duplicate the scheme into a new buffer and add it to the list of standard
397 // schemes. This pointer will be leaked on shutdown.
398 char* dup_scheme = new char[scheme_len + 1];
399 ANNOTATE_LEAKING_OBJECT_PTR(dup_scheme);
400 memcpy(dup_scheme, new_scheme, scheme_len + 1);
402 InitStandardSchemes();
403 SchemeWithType scheme_with_type;
404 scheme_with_type.scheme = dup_scheme;
405 scheme_with_type.type = type;
406 standard_schemes->push_back(scheme_with_type);
409 void LockStandardSchemes() {
410 standard_schemes_locked = true;
413 bool IsStandard(const char* spec, const Component& scheme) {
414 SchemeType unused_scheme_type;
415 return DoIsStandard(spec, scheme, &unused_scheme_type);
418 bool GetStandardSchemeType(const char* spec,
419 const Component& scheme,
420 SchemeType* type) {
421 return DoIsStandard(spec, scheme, type);
424 bool IsStandard(const base::char16* spec, const Component& scheme) {
425 SchemeType unused_scheme_type;
426 return DoIsStandard(spec, scheme, &unused_scheme_type);
429 bool FindAndCompareScheme(const char* str,
430 int str_len,
431 const char* compare,
432 Component* found_scheme) {
433 return DoFindAndCompareScheme(str, str_len, compare, found_scheme);
436 bool FindAndCompareScheme(const base::char16* str,
437 int str_len,
438 const char* compare,
439 Component* found_scheme) {
440 return DoFindAndCompareScheme(str, str_len, compare, found_scheme);
443 bool Canonicalize(const char* spec,
444 int spec_len,
445 bool trim_path_end,
446 CharsetConverter* charset_converter,
447 CanonOutput* output,
448 Parsed* output_parsed) {
449 return DoCanonicalize(spec, spec_len, trim_path_end, charset_converter,
450 output, output_parsed);
453 bool Canonicalize(const base::char16* spec,
454 int spec_len,
455 bool trim_path_end,
456 CharsetConverter* charset_converter,
457 CanonOutput* output,
458 Parsed* output_parsed) {
459 return DoCanonicalize(spec, spec_len, trim_path_end, charset_converter,
460 output, output_parsed);
463 bool ResolveRelative(const char* base_spec,
464 int base_spec_len,
465 const Parsed& base_parsed,
466 const char* relative,
467 int relative_length,
468 CharsetConverter* charset_converter,
469 CanonOutput* output,
470 Parsed* output_parsed) {
471 return DoResolveRelative(base_spec, base_spec_len, base_parsed,
472 relative, relative_length,
473 charset_converter, output, output_parsed);
476 bool ResolveRelative(const char* base_spec,
477 int base_spec_len,
478 const Parsed& base_parsed,
479 const base::char16* relative,
480 int relative_length,
481 CharsetConverter* charset_converter,
482 CanonOutput* output,
483 Parsed* output_parsed) {
484 return DoResolveRelative(base_spec, base_spec_len, base_parsed,
485 relative, relative_length,
486 charset_converter, output, output_parsed);
489 bool ReplaceComponents(const char* spec,
490 int spec_len,
491 const Parsed& parsed,
492 const Replacements<char>& replacements,
493 CharsetConverter* charset_converter,
494 CanonOutput* output,
495 Parsed* out_parsed) {
496 return DoReplaceComponents(spec, spec_len, parsed, replacements,
497 charset_converter, output, out_parsed);
500 bool ReplaceComponents(const char* spec,
501 int spec_len,
502 const Parsed& parsed,
503 const Replacements<base::char16>& replacements,
504 CharsetConverter* charset_converter,
505 CanonOutput* output,
506 Parsed* out_parsed) {
507 return DoReplaceComponents(spec, spec_len, parsed, replacements,
508 charset_converter, output, out_parsed);
511 void DecodeURLEscapeSequences(const char* input,
512 int length,
513 CanonOutputW* output) {
514 RawCanonOutputT<char> unescaped_chars;
515 for (int i = 0; i < length; i++) {
516 if (input[i] == '%') {
517 unsigned char ch;
518 if (DecodeEscaped(input, &i, length, &ch)) {
519 unescaped_chars.push_back(ch);
520 } else {
521 // Invalid escape sequence, copy the percent literal.
522 unescaped_chars.push_back('%');
524 } else {
525 // Regular non-escaped 8-bit character.
526 unescaped_chars.push_back(input[i]);
530 // Convert that 8-bit to UTF-16. It's not clear IE does this at all to
531 // JavaScript URLs, but Firefox and Safari do.
532 for (int i = 0; i < unescaped_chars.length(); i++) {
533 unsigned char uch = static_cast<unsigned char>(unescaped_chars.at(i));
534 if (uch < 0x80) {
535 // Non-UTF-8, just append directly
536 output->push_back(uch);
537 } else {
538 // next_ch will point to the last character of the decoded
539 // character.
540 int next_character = i;
541 unsigned code_point;
542 if (ReadUTFChar(unescaped_chars.data(), &next_character,
543 unescaped_chars.length(), &code_point)) {
544 // Valid UTF-8 character, convert to UTF-16.
545 AppendUTF16Value(code_point, output);
546 i = next_character;
547 } else {
548 // If there are any sequences that are not valid UTF-8, we keep
549 // invalid code points and promote to UTF-16. We copy all characters
550 // from the current position to the end of the identified sequence.
551 while (i < next_character) {
552 output->push_back(static_cast<unsigned char>(unescaped_chars.at(i)));
553 i++;
555 output->push_back(static_cast<unsigned char>(unescaped_chars.at(i)));
561 void EncodeURIComponent(const char* input, int length, CanonOutput* output) {
562 for (int i = 0; i < length; ++i) {
563 unsigned char c = static_cast<unsigned char>(input[i]);
564 if (IsComponentChar(c))
565 output->push_back(c);
566 else
567 AppendEscapedChar(c, output);
571 bool CompareSchemeComponent(const char* spec,
572 const Component& component,
573 const char* compare_to) {
574 return DoCompareSchemeComponent(spec, component, compare_to);
577 bool CompareSchemeComponent(const base::char16* spec,
578 const Component& component,
579 const char* compare_to) {
580 return DoCompareSchemeComponent(spec, component, compare_to);
583 } // namespace url