net/base/net_util.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/base/net_util.h"
   6
   7 #include <errno.h>
   8
   9 #include <algorithm>
  10 #include <limits>
  11 #include <string>
  12
  13 #include "build/build_config.h"
  14
  15 #if defined(OS_WIN)
  16 #include <windows.h>
  17 #include <iphlpapi.h>
  18 #include <winsock2.h>
  19 #include <ws2bth.h>
  20 #pragma comment(lib, "iphlpapi.lib")
  21 #elif defined(OS_POSIX)
  22 #include <fcntl.h>
  23 #include <netdb.h>
  24 #include <netinet/in.h>
  25 #include <unistd.h>
  26 #if !defined(OS_NACL)
  27 #include <net/if.h>
  28 #if !defined(OS_ANDROID)
  29 #include <ifaddrs.h>
  30 #endif  // !defined(OS_NACL)
  31 #endif  // !defined(OS_ANDROID)
  32 #endif  // defined(OS_POSIX)
  33
  34 #include "base/basictypes.h"
  35 #include "base/json/string_escape.h"
  36 #include "base/logging.h"
  37 #include "base/strings/string_piece.h"
  38 #include "base/strings/string_split.h"
  39 #include "base/strings/string_util.h"
  40 #include "base/strings/stringprintf.h"
  41 #include "base/strings/utf_string_conversions.h"
  42 #include "base/sys_byteorder.h"
  43 #include "base/values.h"
  44 #include "net/base/address_list.h"
  45 #include "net/base/dns_util.h"
  46 #include "net/base/ip_address_number.h"
  47 #include "net/base/net_module.h"
  48 #include "net/base/registry_controlled_domains/registry_controlled_domain.h"
  49 #include "net/grit/net_resources.h"
  50 #include "net/http/http_content_disposition.h"
  51 #include "url/gurl.h"
  52 #include "url/third_party/mozilla/url_parse.h"
  53 #include "url/url_canon.h"
  54 #include "url/url_canon_ip.h"
  55
  56 #if defined(OS_ANDROID)
  57 #include "net/android/network_library.h"
  58 #endif
  59 #if defined(OS_WIN)
  60 #include "net/base/winsock_init.h"
  61 #endif
  62
  63 namespace net {
  64
  65 namespace {
  66
  67 std::string NormalizeHostname(const std::string& host) {
  68   std::string result = base::ToLowerASCII(host);
  69   if (!result.empty() && *result.rbegin() == '.')
  70     result.resize(result.size() - 1);
  71   return result;
  72 }
  73
  74 bool IsNormalizedLocalhostTLD(const std::string& host) {
  75   return base::EndsWith(host, ".localhost", base::CompareCase::SENSITIVE);
  76 }
  77
  78 // |host| should be normalized.
  79 bool IsLocalHostname(const std::string& host) {
  80   return host == "localhost" || host == "localhost.localdomain" ||
  81          IsNormalizedLocalhostTLD(host);
  82 }
  83
  84 // |host| should be normalized.
  85 bool IsLocal6Hostname(const std::string& host) {
  86   return host == "localhost6" || host == "localhost6.localdomain6";
  87 }
  88
  89 }  // namespace
  90
  91 std::string CanonicalizeHost(const std::string& host,
  92                              url::CanonHostInfo* host_info) {
  93   // Try to canonicalize the host.
  94   const url::Component raw_host_component(0, static_cast<int>(host.length()));
  95   std::string canon_host;
  96   url::StdStringCanonOutput canon_host_output(&canon_host);
  97   url::CanonicalizeHostVerbose(host.c_str(), raw_host_component,
  98                                &canon_host_output, host_info);
  99
 100   if (host_info->out_host.is_nonempty() &&
 101       host_info->family != url::CanonHostInfo::BROKEN) {
 102     // Success!  Assert that there's no extra garbage.
 103     canon_host_output.Complete();
 104     DCHECK_EQ(host_info->out_host.len, static_cast<int>(canon_host.length()));
 105   } else {
 106     // Empty host, or canonicalization failed.  We'll return empty.
 107     canon_host.clear();
 108   }
 109
 110   return canon_host;
 111 }
 112
 113 std::string GetDirectoryListingHeader(const base::string16& title) {
 114   static const base::StringPiece header(
 115       NetModule::GetResource(IDR_DIR_HEADER_HTML));
 116   // This can be null in unit tests.
 117   DLOG_IF(WARNING, header.empty()) <<
 118       "Missing resource: directory listing header";
 119
 120   std::string result;
 121   if (!header.empty())
 122     result.assign(header.data(), header.size());
 123
 124   result.append("<script>start(");
 125   base::EscapeJSONString(title, true, &result);
 126   result.append(");</script>\n");
 127
 128   return result;
 129 }
 130
 131 inline bool IsHostCharAlphanumeric(char c) {
 132   // We can just check lowercase because uppercase characters have already been
 133   // normalized.
 134   return ((c >= 'a') && (c <= 'z')) || ((c >= '0') && (c <= '9'));
 135 }
 136
 137 bool IsCanonicalizedHostCompliant(const std::string& host) {
 138   if (host.empty())
 139     return false;
 140
 141   bool in_component = false;
 142   bool most_recent_component_started_alphanumeric = false;
 143
 144   for (std::string::const_iterator i(host.begin()); i != host.end(); ++i) {
 145     const char c = *i;
 146     if (!in_component) {
 147       most_recent_component_started_alphanumeric = IsHostCharAlphanumeric(c);
 148       if (!most_recent_component_started_alphanumeric && (c != '-') &&
 149           (c != '_')) {
 150         return false;
 151       }
 152       in_component = true;
 153     } else if (c == '.') {
 154       in_component = false;
 155     } else if (!IsHostCharAlphanumeric(c) && (c != '-') && (c != '_')) {
 156       return false;
 157     }
 158   }
 159
 160   return most_recent_component_started_alphanumeric;
 161 }
 162
 163 base::string16 StripWWW(const base::string16& text) {
 164   const base::string16 www(base::ASCIIToUTF16("www."));
 165   return base::StartsWith(text, www, base::CompareCase::SENSITIVE)
 166       ? text.substr(www.length()) : text;
 167 }
 168
 169 base::string16 StripWWWFromHost(const GURL& url) {
 170   DCHECK(url.is_valid());
 171   return StripWWW(base::ASCIIToUTF16(url.host()));
 172 }
 173
 174 int SetNonBlocking(int fd) {
 175 #if defined(OS_WIN)
 176   unsigned long no_block = 1;
 177   return ioctlsocket(fd, FIONBIO, &no_block);
 178 #elif defined(OS_POSIX)
 179   int flags = fcntl(fd, F_GETFL, 0);
 180   if (-1 == flags)
 181     return flags;
 182   return fcntl(fd, F_SETFL, flags | O_NONBLOCK);
 183 #endif
 184 }
 185
 186 bool ParseHostAndPort(std::string::const_iterator host_and_port_begin,
 187                       std::string::const_iterator host_and_port_end,
 188                       std::string* host,
 189                       int* port) {
 190   if (host_and_port_begin >= host_and_port_end)
 191     return false;
 192
 193   // When using url, we use char*.
 194   const char* auth_begin = &(*host_and_port_begin);
 195   int auth_len = host_and_port_end - host_and_port_begin;
 196
 197   url::Component auth_component(0, auth_len);
 198   url::Component username_component;
 199   url::Component password_component;
 200   url::Component hostname_component;
 201   url::Component port_component;
 202
 203   url::ParseAuthority(auth_begin, auth_component, &username_component,
 204       &password_component, &hostname_component, &port_component);
 205
 206   // There shouldn't be a username/password.
 207   if (username_component.is_valid() || password_component.is_valid())
 208     return false;
 209
 210   if (!hostname_component.is_nonempty())
 211     return false;  // Failed parsing.
 212
 213   int parsed_port_number = -1;
 214   if (port_component.is_nonempty()) {
 215     parsed_port_number = url::ParsePort(auth_begin, port_component);
 216
 217     // If parsing failed, port_number will be either PORT_INVALID or
 218     // PORT_UNSPECIFIED, both of which are negative.
 219     if (parsed_port_number < 0)
 220       return false;  // Failed parsing the port number.
 221   }
 222
 223   if (port_component.len == 0)
 224     return false;  // Reject inputs like "foo:"
 225
 226   unsigned char tmp_ipv6_addr[16];
 227
 228   // If the hostname starts with a bracket, it is either an IPv6 literal or
 229   // invalid. If it is an IPv6 literal then strip the brackets.
 230   if (hostname_component.len > 0 &&
 231       auth_begin[hostname_component.begin] == '[') {
 232     if (auth_begin[hostname_component.end() - 1] == ']' &&
 233         url::IPv6AddressToNumber(
 234             auth_begin, hostname_component, tmp_ipv6_addr)) {
 235       // Strip the brackets.
 236       hostname_component.begin++;
 237       hostname_component.len -= 2;
 238     } else {
 239       return false;
 240     }
 241   }
 242
 243   // Pass results back to caller.
 244   host->assign(auth_begin + hostname_component.begin, hostname_component.len);
 245   *port = parsed_port_number;
 246
 247   return true;  // Success.
 248 }
 249
 250 bool ParseHostAndPort(const std::string& host_and_port,
 251                       std::string* host,
 252                       int* port) {
 253   return ParseHostAndPort(
 254       host_and_port.begin(), host_and_port.end(), host, port);
 255 }
 256
 257 std::string GetHostAndPort(const GURL& url) {
 258   // For IPv6 literals, GURL::host() already includes the brackets so it is
 259   // safe to just append a colon.
 260   return base::StringPrintf("%s:%d", url.host().c_str(),
 261                             url.EffectiveIntPort());
 262 }
 263
 264 std::string GetHostAndOptionalPort(const GURL& url) {
 265   // For IPv6 literals, GURL::host() already includes the brackets
 266   // so it is safe to just append a colon.
 267   if (url.has_port())
 268     return base::StringPrintf("%s:%s", url.host().c_str(), url.port().c_str());
 269   return url.host();
 270 }
 271
 272 bool IsHostnameNonUnique(const std::string& hostname) {
 273   // CanonicalizeHost requires surrounding brackets to parse an IPv6 address.
 274   const std::string host_or_ip = hostname.find(':') != std::string::npos ?
 275       "[" + hostname + "]" : hostname;
 276   url::CanonHostInfo host_info;
 277   std::string canonical_name = CanonicalizeHost(host_or_ip, &host_info);
 278
 279   // If canonicalization fails, then the input is truly malformed. However,
 280   // to avoid mis-reporting bad inputs as "non-unique", treat them as unique.
 281   if (canonical_name.empty())
 282     return false;
 283
 284   // If |hostname| is an IP address, check to see if it's in an IANA-reserved
 285   // range.
 286   if (host_info.IsIPAddress()) {
 287     IPAddressNumber host_addr;
 288     if (!ParseIPLiteralToNumber(hostname.substr(host_info.out_host.begin,
 289                                                 host_info.out_host.len),
 290                                 &host_addr)) {
 291       return false;
 292     }
 293     switch (host_info.family) {
 294       case url::CanonHostInfo::IPV4:
 295       case url::CanonHostInfo::IPV6:
 296         return IsIPAddressReserved(host_addr);
 297       case url::CanonHostInfo::NEUTRAL:
 298       case url::CanonHostInfo::BROKEN:
 299         return false;
 300     }
 301   }
 302
 303   // Check for a registry controlled portion of |hostname|, ignoring private
 304   // registries, as they already chain to ICANN-administered registries,
 305   // and explicitly ignoring unknown registries.
 306   //
 307   // Note: This means that as new gTLDs are introduced on the Internet, they
 308   // will be treated as non-unique until the registry controlled domain list
 309   // is updated. However, because gTLDs are expected to provide significant
 310   // advance notice to deprecate older versions of this code, this an
 311   // acceptable tradeoff.
 312   return 0 == registry_controlled_domains::GetRegistryLength(
 313                   canonical_name,
 314                   registry_controlled_domains::EXCLUDE_UNKNOWN_REGISTRIES,
 315                   registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES);
 316 }
 317
 318 SockaddrStorage::SockaddrStorage(const SockaddrStorage& other)
 319     : addr_len(other.addr_len),
 320       addr(reinterpret_cast<struct sockaddr*>(&addr_storage)) {
 321   memcpy(addr, other.addr, addr_len);
 322 }
 323
 324 void SockaddrStorage::operator=(const SockaddrStorage& other) {
 325   addr_len = other.addr_len;
 326   // addr is already set to &this->addr_storage by default ctor.
 327   memcpy(addr, other.addr, addr_len);
 328 }
 329
 330 // Extracts the address and port portions of a sockaddr.
 331 bool GetIPAddressFromSockAddr(const struct sockaddr* sock_addr,
 332                               socklen_t sock_addr_len,
 333                               const uint8_t** address,
 334                               size_t* address_len,
 335                               uint16_t* port) {
 336   if (sock_addr->sa_family == AF_INET) {
 337     if (sock_addr_len < static_cast<socklen_t>(sizeof(struct sockaddr_in)))
 338       return false;
 339     const struct sockaddr_in* addr =
 340         reinterpret_cast<const struct sockaddr_in*>(sock_addr);
 341     *address = reinterpret_cast<const uint8_t*>(&addr->sin_addr);
 342     *address_len = kIPv4AddressSize;
 343     if (port)
 344       *port = base::NetToHost16(addr->sin_port);
 345     return true;
 346   }
 347
 348   if (sock_addr->sa_family == AF_INET6) {
 349     if (sock_addr_len < static_cast<socklen_t>(sizeof(struct sockaddr_in6)))
 350       return false;
 351     const struct sockaddr_in6* addr =
 352         reinterpret_cast<const struct sockaddr_in6*>(sock_addr);
 353     *address = reinterpret_cast<const uint8_t*>(&addr->sin6_addr);
 354     *address_len = kIPv6AddressSize;
 355     if (port)
 356       *port = base::NetToHost16(addr->sin6_port);
 357     return true;
 358   }
 359
 360 #if defined(OS_WIN)
 361   if (sock_addr->sa_family == AF_BTH) {
 362     if (sock_addr_len < static_cast<socklen_t>(sizeof(SOCKADDR_BTH)))
 363       return false;
 364     const SOCKADDR_BTH* addr =
 365         reinterpret_cast<const SOCKADDR_BTH*>(sock_addr);
 366     *address = reinterpret_cast<const uint8_t*>(&addr->btAddr);
 367     *address_len = kBluetoothAddressSize;
 368     if (port)
 369       *port = static_cast<uint16_t>(addr->port);
 370     return true;
 371   }
 372 #endif
 373
 374   return false;  // Unrecognized |sa_family|.
 375 }
 376
 377 std::string NetAddressToString(const struct sockaddr* sa,
 378                                socklen_t sock_addr_len) {
 379   const uint8_t* address;
 380   size_t address_len;
 381   if (!GetIPAddressFromSockAddr(sa, sock_addr_len, &address,
 382                                 &address_len, NULL)) {
 383     NOTREACHED();
 384     return std::string();
 385   }
 386   return IPAddressToString(address, address_len);
 387 }
 388
 389 std::string NetAddressToStringWithPort(const struct sockaddr* sa,
 390                                        socklen_t sock_addr_len) {
 391   const uint8_t* address;
 392   size_t address_len;
 393   uint16_t port;
 394   if (!GetIPAddressFromSockAddr(sa, sock_addr_len, &address,
 395                                 &address_len, &port)) {
 396     NOTREACHED();
 397     return std::string();
 398   }
 399   return IPAddressToStringWithPort(address, address_len, port);
 400 }
 401
 402 std::string GetHostName() {
 403 #if defined(OS_NACL)
 404   NOTIMPLEMENTED();
 405   return std::string();
 406 #else  // defined(OS_NACL)
 407 #if defined(OS_WIN)
 408   EnsureWinsockInit();
 409 #endif
 410
 411   // Host names are limited to 255 bytes.
 412   char buffer[256];
 413   int result = gethostname(buffer, sizeof(buffer));
 414   if (result != 0) {
 415     DVLOG(1) << "gethostname() failed with " << result;
 416     buffer[0] = '\0';
 417   }
 418   return std::string(buffer);
 419 #endif  // !defined(OS_NACL)
 420 }
 421
 422 void GetIdentityFromURL(const GURL& url,
 423                         base::string16* username,
 424                         base::string16* password) {
 425   UnescapeRule::Type flags =
 426       UnescapeRule::SPACES | UnescapeRule::URL_SPECIAL_CHARS;
 427   *username = UnescapeAndDecodeUTF8URLComponent(url.username(), flags);
 428   *password = UnescapeAndDecodeUTF8URLComponent(url.password(), flags);
 429 }
 430
 431 std::string GetHostOrSpecFromURL(const GURL& url) {
 432   return url.has_host() ? TrimEndingDot(url.host()) : url.spec();
 433 }
 434
 435 GURL SimplifyUrlForRequest(const GURL& url) {
 436   DCHECK(url.is_valid());
 437   GURL::Replacements replacements;
 438   replacements.ClearUsername();
 439   replacements.ClearPassword();
 440   replacements.ClearRef();
 441   return url.ReplaceComponents(replacements);
 442 }
 443
 444 bool HaveOnlyLoopbackAddresses() {
 445 #if defined(OS_ANDROID)
 446   return android::HaveOnlyLoopbackAddresses();
 447 #elif defined(OS_NACL)
 448   NOTIMPLEMENTED();
 449   return false;
 450 #elif defined(OS_POSIX)
 451   struct ifaddrs* interface_addr = NULL;
 452   int rv = getifaddrs(&interface_addr);
 453   if (rv != 0) {
 454     DVLOG(1) << "getifaddrs() failed with errno = " << errno;
 455     return false;
 456   }
 457
 458   bool result = true;
 459   for (struct ifaddrs* interface = interface_addr;
 460        interface != NULL;
 461        interface = interface->ifa_next) {
 462     if (!(IFF_UP & interface->ifa_flags))
 463       continue;
 464     if (IFF_LOOPBACK & interface->ifa_flags)
 465       continue;
 466     const struct sockaddr* addr = interface->ifa_addr;
 467     if (!addr)
 468       continue;
 469     if (addr->sa_family == AF_INET6) {
 470       // Safe cast since this is AF_INET6.
 471       const struct sockaddr_in6* addr_in6 =
 472           reinterpret_cast<const struct sockaddr_in6*>(addr);
 473       const struct in6_addr* sin6_addr = &addr_in6->sin6_addr;
 474       if (IN6_IS_ADDR_LOOPBACK(sin6_addr) || IN6_IS_ADDR_LINKLOCAL(sin6_addr))
 475         continue;
 476     }
 477     if (addr->sa_family != AF_INET6 && addr->sa_family != AF_INET)
 478       continue;
 479
 480     result = false;
 481     break;
 482   }
 483   freeifaddrs(interface_addr);
 484   return result;
 485 #elif defined(OS_WIN)
 486   // TODO(wtc): implement with the GetAdaptersAddresses function.
 487   NOTIMPLEMENTED();
 488   return false;
 489 #else
 490   NOTIMPLEMENTED();
 491   return false;
 492 #endif  // defined(various platforms)
 493 }
 494
 495 AddressFamily GetAddressFamily(const IPAddressNumber& address) {
 496   switch (address.size()) {
 497     case kIPv4AddressSize:
 498       return ADDRESS_FAMILY_IPV4;
 499     case kIPv6AddressSize:
 500       return ADDRESS_FAMILY_IPV6;
 501     default:
 502       return ADDRESS_FAMILY_UNSPECIFIED;
 503   }
 504 }
 505
 506 int ConvertAddressFamily(AddressFamily address_family) {
 507   switch (address_family) {
 508     case ADDRESS_FAMILY_UNSPECIFIED:
 509       return AF_UNSPEC;
 510     case ADDRESS_FAMILY_IPV4:
 511       return AF_INET;
 512     case ADDRESS_FAMILY_IPV6:
 513       return AF_INET6;
 514   }
 515   NOTREACHED();
 516   return AF_UNSPEC;
 517 }
 518
 519 const uint16_t* GetPortFieldFromSockaddr(const struct sockaddr* address,
 520                                          socklen_t address_len) {
 521   if (address->sa_family == AF_INET) {
 522     DCHECK_LE(sizeof(sockaddr_in), static_cast<size_t>(address_len));
 523     const struct sockaddr_in* sockaddr =
 524         reinterpret_cast<const struct sockaddr_in*>(address);
 525     return &sockaddr->sin_port;
 526   } else if (address->sa_family == AF_INET6) {
 527     DCHECK_LE(sizeof(sockaddr_in6), static_cast<size_t>(address_len));
 528     const struct sockaddr_in6* sockaddr =
 529         reinterpret_cast<const struct sockaddr_in6*>(address);
 530     return &sockaddr->sin6_port;
 531   } else {
 532     NOTREACHED();
 533     return NULL;
 534   }
 535 }
 536
 537 int GetPortFromSockaddr(const struct sockaddr* address, socklen_t address_len) {
 538   const uint16_t* port_field = GetPortFieldFromSockaddr(address, address_len);
 539   if (!port_field)
 540     return -1;
 541   return base::NetToHost16(*port_field);
 542 }
 543
 544 bool ResolveLocalHostname(const std::string& host,
 545                           uint16_t port,
 546                           AddressList* address_list) {
 547   static const unsigned char kLocalhostIPv4[] = {127, 0, 0, 1};
 548   static const unsigned char kLocalhostIPv6[] = {
 549       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
 550
 551   std::string normalized_host = NormalizeHostname(host);
 552
 553   address_list->clear();
 554
 555   bool is_local6 = IsLocal6Hostname(normalized_host);
 556   if (!is_local6 && !IsLocalHostname(normalized_host))
 557     return false;
 558
 559   address_list->push_back(
 560       IPEndPoint(IPAddressNumber(kLocalhostIPv6,
 561                                  kLocalhostIPv6 + arraysize(kLocalhostIPv6)),
 562                  port));
 563   if (!is_local6) {
 564     address_list->push_back(
 565         IPEndPoint(IPAddressNumber(kLocalhostIPv4,
 566                                    kLocalhostIPv4 + arraysize(kLocalhostIPv4)),
 567                    port));
 568   }
 569
 570   return true;
 571 }
 572
 573 bool IsLocalhost(const std::string& host) {
 574   std::string normalized_host = NormalizeHostname(host);
 575   if (IsLocalHostname(normalized_host) || IsLocal6Hostname(normalized_host))
 576     return true;
 577
 578   IPAddressNumber ip_number;
 579   if (ParseIPLiteralToNumber(host, &ip_number)) {
 580     size_t size = ip_number.size();
 581     switch (size) {
 582       case kIPv4AddressSize: {
 583         IPAddressNumber localhost_prefix;
 584         localhost_prefix.push_back(127);
 585         for (int i = 0; i < 3; ++i) {
 586           localhost_prefix.push_back(0);
 587         }
 588         return IPNumberMatchesPrefix(ip_number, localhost_prefix, 8);
 589       }
 590
 591       case kIPv6AddressSize: {
 592         struct in6_addr sin6_addr;
 593         memcpy(&sin6_addr, &ip_number[0], kIPv6AddressSize);
 594         return !!IN6_IS_ADDR_LOOPBACK(&sin6_addr);
 595       }
 596
 597       default:
 598         NOTREACHED();
 599     }
 600   }
 601
 602   return false;
 603 }
 604
 605 bool IsLocalhostTLD(const std::string& host) {
 606   return IsNormalizedLocalhostTLD(NormalizeHostname(host));
 607 }
 608
 609 bool HasGoogleHost(const GURL& url) {
 610   static const char* kGoogleHostSuffixes[] = {
 611       ".google.com",
 612       ".youtube.com",
 613       ".gmail.com",
 614       ".doubleclick.net",
 615       ".gstatic.com",
 616       ".googlevideo.com",
 617       ".googleusercontent.com",
 618       ".googlesyndication.com",
 619       ".google-analytics.com",
 620       ".googleadservices.com",
 621       ".googleapis.com",
 622       ".ytimg.com",
 623   };
 624   const std::string& host = url.host();
 625   for (const char* suffix : kGoogleHostSuffixes) {
 626     if (base::EndsWith(host, suffix, base::CompareCase::INSENSITIVE_ASCII))
 627       return true;
 628   }
 629   return false;
 630 }
 631
 632 }  // namespace net