chrome/browser/safe_browsing/safe_browsing_util.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/safe_browsing/safe_browsing_util.h"
   6
   7 #include "base/logging.h"
   8 #include "base/strings/string_util.h"
   9 #include "base/strings/stringprintf.h"
  10 #include "chrome/browser/google/google_util.h"
  11 #include "crypto/sha2.h"
  12 #include "net/base/escape.h"
  13 #include "url/gurl.h"
  14 #include "url/url_util.h"
  15
  16 #if defined(OS_WIN)
  17 #include "chrome/installer/util/browser_distribution.h"
  18 #endif
  19
  20 static const char kReportParams[] = "?tpl=%s&url=%s";
  21
  22 // SBChunk ---------------------------------------------------------------------
  23
  24 SBChunk::SBChunk()
  25     : chunk_number(0),
  26       list_id(0),
  27       is_add(false) {
  28 }
  29
  30 SBChunk::~SBChunk() {}
  31
  32 // SBChunkList -----------------------------------------------------------------
  33
  34 SBChunkList::SBChunkList() {}
  35
  36 SBChunkList::~SBChunkList() {
  37   clear();
  38 }
  39
  40 void SBChunkList::clear() {
  41   for (std::vector<SBChunk>::iterator citer = chunks_.begin();
  42        citer != chunks_.end(); ++citer) {
  43     for (std::deque<SBChunkHost>::iterator hiter = citer->hosts.begin();
  44          hiter != citer->hosts.end(); ++hiter) {
  45       if (hiter->entry) {
  46         hiter->entry->Destroy();
  47         hiter->entry = NULL;
  48       }
  49     }
  50   }
  51   chunks_.clear();
  52 }
  53
  54 // SBListChunkRanges -----------------------------------------------------------
  55
  56 SBListChunkRanges::SBListChunkRanges(const std::string& n) : name(n) {}
  57
  58 // SBChunkDelete ---------------------------------------------------------------
  59
  60 SBChunkDelete::SBChunkDelete() : is_sub_del(false) {}
  61
  62 SBChunkDelete::~SBChunkDelete() {}
  63
  64 // SBEntry ---------------------------------------------------------------------
  65
  66 // static
  67 SBEntry* SBEntry::Create(Type type, int prefix_count) {
  68   int size = Size(type, prefix_count);
  69   SBEntry *rv = static_cast<SBEntry*>(malloc(size));
  70   memset(rv, 0, size);
  71   rv->set_type(type);
  72   rv->set_prefix_count(prefix_count);
  73   return rv;
  74 }
  75
  76 void SBEntry::Destroy() {
  77   free(this);
  78 }
  79
  80 // static
  81 int SBEntry::PrefixSize(Type type) {
  82   switch (type) {
  83     case ADD_PREFIX:
  84       return sizeof(SBPrefix);
  85     case ADD_FULL_HASH:
  86       return sizeof(SBFullHash);
  87     case SUB_PREFIX:
  88       return sizeof(SBSubPrefix);
  89     case SUB_FULL_HASH:
  90       return sizeof(SBSubFullHash);
  91     default:
  92       NOTREACHED();
  93       return 0;
  94   }
  95 }
  96
  97 int SBEntry::Size() const {
  98   return Size(type(), prefix_count());
  99 }
 100
 101 // static
 102 int SBEntry::Size(Type type, int prefix_count) {
 103   return sizeof(Data) + prefix_count * PrefixSize(type);
 104 }
 105
 106 int SBEntry::ChunkIdAtPrefix(int index) const {
 107   if (type() == SUB_PREFIX)
 108     return sub_prefixes_[index].add_chunk;
 109   return (type() == SUB_FULL_HASH) ?
 110       sub_full_hashes_[index].add_chunk : chunk_id();
 111 }
 112
 113 void SBEntry::SetChunkIdAtPrefix(int index, int chunk_id) {
 114   DCHECK(IsSub());
 115
 116   if (type() == SUB_PREFIX)
 117     sub_prefixes_[index].add_chunk = chunk_id;
 118   else
 119     sub_full_hashes_[index].add_chunk = chunk_id;
 120 }
 121
 122 const SBPrefix& SBEntry::PrefixAt(int index) const {
 123   DCHECK(IsPrefix());
 124
 125   return IsAdd() ? add_prefixes_[index] : sub_prefixes_[index].prefix;
 126 }
 127
 128 const SBFullHash& SBEntry::FullHashAt(int index) const {
 129   DCHECK(!IsPrefix());
 130
 131   return IsAdd() ? add_full_hashes_[index] : sub_full_hashes_[index].prefix;
 132 }
 133
 134 void SBEntry::SetPrefixAt(int index, const SBPrefix& prefix) {
 135   DCHECK(IsPrefix());
 136
 137   if (IsAdd())
 138     add_prefixes_[index] = prefix;
 139   else
 140     sub_prefixes_[index].prefix = prefix;
 141 }
 142
 143 void SBEntry::SetFullHashAt(int index, const SBFullHash& full_hash) {
 144   DCHECK(!IsPrefix());
 145
 146   if (IsAdd())
 147     add_full_hashes_[index] = full_hash;
 148   else
 149     sub_full_hashes_[index].prefix = full_hash;
 150 }
 151
 152
 153 // Utility functions -----------------------------------------------------------
 154
 155 namespace {
 156 bool IsKnownList(const std::string& name) {
 157   for (size_t i = 0; i < arraysize(safe_browsing_util::kAllLists); ++i) {
 158     if (!strcmp(safe_browsing_util::kAllLists[i], name.c_str())) {
 159       return true;
 160     }
 161   }
 162   return false;
 163 }
 164 }  // namespace
 165
 166 namespace safe_browsing_util {
 167
 168 // Listnames that browser can process.
 169 const char kMalwareList[] = "goog-malware-shavar";
 170 const char kPhishingList[] = "goog-phish-shavar";
 171 const char kBinUrlList[] = "goog-badbinurl-shavar";
 172 // We don't use the bad binary digest list anymore.  Use a fake listname to be
 173 // sure we don't request it accidentally.
 174 const char kBinHashList[] = "goog-badbin-digestvar-disabled";
 175 const char kCsdWhiteList[] = "goog-csdwhite-sha256";
 176 const char kDownloadWhiteList[] = "goog-downloadwhite-digest256";
 177 const char kExtensionBlacklist[] = "goog-badcrxids-digestvar";
 178 const char kSideEffectFreeWhitelist[] = "goog-sideeffectfree-shavar";
 179 const char kIPBlacklist[] = "goog-badip-digest256";
 180
 181 const char* kAllLists[10] = {
 182   kMalwareList,
 183   kPhishingList,
 184   kBinUrlList,
 185   kBinHashList,
 186   kCsdWhiteList,
 187   kDownloadWhiteList,
 188   kDownloadWhiteList,
 189   kExtensionBlacklist,
 190   kSideEffectFreeWhitelist,
 191   kIPBlacklist,
 192 };
 193
 194 ListType GetListId(const std::string& name) {
 195   ListType id;
 196   if (name == safe_browsing_util::kMalwareList) {
 197     id = MALWARE;
 198   } else if (name == safe_browsing_util::kPhishingList) {
 199     id = PHISH;
 200   } else if (name == safe_browsing_util::kBinUrlList) {
 201     id = BINURL;
 202   } else if (name == safe_browsing_util::kBinHashList) {
 203     id = BINHASH;
 204   } else if (name == safe_browsing_util::kCsdWhiteList) {
 205     id = CSDWHITELIST;
 206   } else if (name == safe_browsing_util::kDownloadWhiteList) {
 207     id = DOWNLOADWHITELIST;
 208   } else if (name == safe_browsing_util::kExtensionBlacklist) {
 209     id = EXTENSIONBLACKLIST;
 210   } else if (name == safe_browsing_util::kSideEffectFreeWhitelist) {
 211     id = SIDEEFFECTFREEWHITELIST;
 212   } else if (name == safe_browsing_util::kIPBlacklist) {
 213     id = IPBLACKLIST;
 214   } else {
 215     id = INVALID;
 216   }
 217   return id;
 218 }
 219
 220 bool GetListName(ListType list_id, std::string* list) {
 221   switch (list_id) {
 222     case MALWARE:
 223       *list = safe_browsing_util::kMalwareList;
 224       break;
 225     case PHISH:
 226       *list = safe_browsing_util::kPhishingList;
 227       break;
 228     case BINURL:
 229       *list = safe_browsing_util::kBinUrlList;
 230       break;
 231     case BINHASH:
 232       *list = safe_browsing_util::kBinHashList;
 233       break;
 234     case CSDWHITELIST:
 235       *list = safe_browsing_util::kCsdWhiteList;
 236       break;
 237     case DOWNLOADWHITELIST:
 238       *list = safe_browsing_util::kDownloadWhiteList;
 239       break;
 240     case EXTENSIONBLACKLIST:
 241       *list = safe_browsing_util::kExtensionBlacklist;
 242       break;
 243     case SIDEEFFECTFREEWHITELIST:
 244       *list = safe_browsing_util::kSideEffectFreeWhitelist;
 245       break;
 246     case IPBLACKLIST:
 247       *list = safe_browsing_util::kIPBlacklist;
 248       break;
 249     default:
 250       return false;
 251   }
 252   DCHECK(IsKnownList(*list));
 253   return true;
 254 }
 255
 256 std::string Unescape(const std::string& url) {
 257   std::string unescaped_str(url);
 258   std::string old_unescaped_str;
 259   const int kMaxLoopIterations = 1024;
 260   int loop_var = 0;
 261   do {
 262     old_unescaped_str = unescaped_str;
 263     unescaped_str = net::UnescapeURLComponent(old_unescaped_str,
 264         net::UnescapeRule::CONTROL_CHARS | net::UnescapeRule::SPACES |
 265         net::UnescapeRule::URL_SPECIAL_CHARS);
 266   } while (unescaped_str != old_unescaped_str && ++loop_var <=
 267            kMaxLoopIterations);
 268
 269   return unescaped_str;
 270 }
 271
 272 std::string Escape(const std::string& url) {
 273   std::string escaped_str;
 274   const char* kHexString = "0123456789ABCDEF";
 275   for (size_t i = 0; i < url.length(); i++) {
 276     unsigned char c = static_cast<unsigned char>(url[i]);
 277     if (c <= ' ' || c > '~' || c == '#' || c == '%') {
 278       escaped_str.push_back('%');
 279       escaped_str.push_back(kHexString[c >> 4]);
 280       escaped_str.push_back(kHexString[c & 0xf]);
 281     } else {
 282       escaped_str.push_back(c);
 283     }
 284   }
 285
 286   return escaped_str;
 287 }
 288
 289 std::string RemoveConsecutiveChars(const std::string& str, const char c) {
 290   std::string output(str);
 291   std::string string_to_find;
 292   std::string::size_type loc = 0;
 293   string_to_find.append(2, c);
 294   while ((loc = output.find(string_to_find, loc)) != std::string::npos) {
 295     output.erase(loc, 1);
 296   }
 297
 298   return output;
 299 }
 300
 301 // Canonicalizes url as per Google Safe Browsing Specification.
 302 // See section 6.1 in
 303 // http://code.google.com/p/google-safe-browsing/wiki/Protocolv2Spec.
 304 void CanonicalizeUrl(const GURL& url,
 305                      std::string* canonicalized_hostname,
 306                      std::string* canonicalized_path,
 307                      std::string* canonicalized_query) {
 308   DCHECK(url.is_valid());
 309
 310   // We only canonicalize "normal" URLs.
 311   if (!url.IsStandard())
 312     return;
 313
 314   // Following canonicalization steps are excluded since url parsing takes care
 315   // of those :-
 316   // 1. Remove any tab (0x09), CR (0x0d), and LF (0x0a) chars from url.
 317   //    (Exclude escaped version of these chars).
 318   // 2. Normalize hostname to 4 dot-seperated decimal values.
 319   // 3. Lowercase hostname.
 320   // 4. Resolve path sequences "/../" and "/./".
 321
 322   // That leaves us with the following :-
 323   // 1. Remove fragment in URL.
 324   GURL url_without_fragment;
 325   GURL::Replacements f_replacements;
 326   f_replacements.ClearRef();
 327   f_replacements.ClearUsername();
 328   f_replacements.ClearPassword();
 329   url_without_fragment = url.ReplaceComponents(f_replacements);
 330
 331   // 2. Do URL unescaping until no more hex encoded characters exist.
 332   std::string url_unescaped_str(Unescape(url_without_fragment.spec()));
 333   url_parse::Parsed parsed;
 334   url_parse::ParseStandardURL(url_unescaped_str.data(),
 335       url_unescaped_str.length(), &parsed);
 336
 337   // 3. In hostname, remove all leading and trailing dots.
 338   const std::string host =
 339       (parsed.host.len > 0)
 340           ? url_unescaped_str.substr(parsed.host.begin, parsed.host.len)
 341           : std::string();
 342   const char kCharsToTrim[] = ".";
 343   std::string host_without_end_dots;
 344   base::TrimString(host, kCharsToTrim, &host_without_end_dots);
 345
 346   // 4. In hostname, replace consecutive dots with a single dot.
 347   std::string host_without_consecutive_dots(RemoveConsecutiveChars(
 348       host_without_end_dots, '.'));
 349
 350   // 5. In path, replace runs of consecutive slashes with a single slash.
 351   std::string path =
 352       (parsed.path.len > 0)
 353           ? url_unescaped_str.substr(parsed.path.begin, parsed.path.len)
 354           : std::string();
 355   std::string path_without_consecutive_slash(RemoveConsecutiveChars(path, '/'));
 356
 357   url_canon::Replacements<char> hp_replacements;
 358   hp_replacements.SetHost(host_without_consecutive_dots.data(),
 359   url_parse::Component(0, host_without_consecutive_dots.length()));
 360   hp_replacements.SetPath(path_without_consecutive_slash.data(),
 361   url_parse::Component(0, path_without_consecutive_slash.length()));
 362
 363   std::string url_unescaped_with_can_hostpath;
 364   url_canon::StdStringCanonOutput output(&url_unescaped_with_can_hostpath);
 365   url_parse::Parsed temp_parsed;
 366   url_util::ReplaceComponents(url_unescaped_str.data(),
 367                               url_unescaped_str.length(), parsed,
 368                               hp_replacements, NULL, &output, &temp_parsed);
 369   output.Complete();
 370
 371   // 6. Step needed to revert escaping done in url_util::ReplaceComponents.
 372   url_unescaped_with_can_hostpath = Unescape(url_unescaped_with_can_hostpath);
 373
 374   // 7. After performing all above steps, percent-escape all chars in url which
 375   // are <= ASCII 32, >= 127, #, %. Escapes must be uppercase hex characters.
 376   std::string escaped_canon_url_str(Escape(url_unescaped_with_can_hostpath));
 377   url_parse::Parsed final_parsed;
 378   url_parse::ParseStandardURL(escaped_canon_url_str.data(),
 379                               escaped_canon_url_str.length(), &final_parsed);
 380
 381   if (canonicalized_hostname && final_parsed.host.len > 0) {
 382     *canonicalized_hostname =
 383         escaped_canon_url_str.substr(final_parsed.host.begin,
 384                                      final_parsed.host.len);
 385   }
 386   if (canonicalized_path && final_parsed.path.len > 0) {
 387     *canonicalized_path = escaped_canon_url_str.substr(final_parsed.path.begin,
 388                                                        final_parsed.path.len);
 389   }
 390   if (canonicalized_query && final_parsed.query.len > 0) {
 391     *canonicalized_query = escaped_canon_url_str.substr(
 392         final_parsed.query.begin, final_parsed.query.len);
 393   }
 394 }
 395
 396 void GenerateHostsToCheck(const GURL& url, std::vector<std::string>* hosts) {
 397   hosts->clear();
 398
 399   std::string canon_host;
 400   CanonicalizeUrl(url, &canon_host, NULL, NULL);
 401
 402   const std::string host = canon_host;  // const sidesteps GCC bugs below!
 403   if (host.empty())
 404     return;
 405
 406   // Per the Safe Browsing Protocol v2 spec, we try the host, and also up to 4
 407   // hostnames formed by starting with the last 5 components and successively
 408   // removing the leading component.  The last component isn't examined alone,
 409   // since it's the TLD or a subcomponent thereof.
 410   //
 411   // Note that we don't need to be clever about stopping at the "real" eTLD --
 412   // the data on the server side has been filtered to ensure it will not
 413   // blacklist a whole TLD, and it's not significantly slower on our side to
 414   // just check too much.
 415   //
 416   // Also note that because we have a simple blacklist, not some sort of complex
 417   // whitelist-in-blacklist or vice versa, it doesn't matter what order we check
 418   // these in.
 419   const size_t kMaxHostsToCheck = 4;
 420   bool skipped_last_component = false;
 421   for (std::string::const_reverse_iterator i(host.rbegin());
 422        i != host.rend() && hosts->size() < kMaxHostsToCheck; ++i) {
 423     if (*i == '.') {
 424       if (skipped_last_component)
 425         hosts->push_back(std::string(i.base(), host.end()));
 426       else
 427         skipped_last_component = true;
 428     }
 429   }
 430   hosts->push_back(host);
 431 }
 432
 433 void GeneratePathsToCheck(const GURL& url, std::vector<std::string>* paths) {
 434   paths->clear();
 435
 436   std::string canon_path;
 437   std::string canon_query;
 438   CanonicalizeUrl(url, NULL, &canon_path, &canon_query);
 439
 440   const std::string path = canon_path;   // const sidesteps GCC bugs below!
 441   const std::string query = canon_query;
 442   if (path.empty())
 443     return;
 444
 445   // Per the Safe Browsing Protocol v2 spec, we try the exact path with/without
 446   // the query parameters, and also up to 4 paths formed by starting at the root
 447   // and adding more path components.
 448   //
 449   // As with the hosts above, it doesn't matter what order we check these in.
 450   const size_t kMaxPathsToCheck = 4;
 451   for (std::string::const_iterator i(path.begin());
 452        i != path.end() && paths->size() < kMaxPathsToCheck; ++i) {
 453     if (*i == '/')
 454       paths->push_back(std::string(path.begin(), i + 1));
 455   }
 456
 457   if (!paths->empty() && paths->back() != path)
 458     paths->push_back(path);
 459
 460   if (!query.empty())
 461     paths->push_back(path + "?" + query);
 462 }
 463
 464 void GeneratePatternsToCheck(const GURL& url, std::vector<std::string>* urls) {
 465   std::vector<std::string> hosts, paths;
 466   GenerateHostsToCheck(url, &hosts);
 467   GeneratePathsToCheck(url, &paths);
 468   for (size_t h = 0; h < hosts.size(); ++h) {
 469     for (size_t p = 0; p < paths.size(); ++p) {
 470       urls->push_back(hosts[h] + paths[p]);
 471     }
 472   }
 473 }
 474
 475 int GetHashIndex(const SBFullHash& hash,
 476                  const std::vector<SBFullHashResult>& full_hashes) {
 477   for (size_t i = 0; i < full_hashes.size(); ++i) {
 478     if (hash == full_hashes[i].hash)
 479       return static_cast<int>(i);
 480   }
 481   return -1;
 482 }
 483
 484 int GetUrlHashIndex(const GURL& url,
 485                     const std::vector<SBFullHashResult>& full_hashes) {
 486   if (full_hashes.empty())
 487     return -1;
 488
 489   std::vector<std::string> patterns;
 490   GeneratePatternsToCheck(url, &patterns);
 491
 492   for (size_t i = 0; i < patterns.size(); ++i) {
 493     SBFullHash key;
 494     crypto::SHA256HashString(patterns[i], key.full_hash, sizeof(SBFullHash));
 495     int index = GetHashIndex(key, full_hashes);
 496     if (index != -1)
 497       return index;
 498   }
 499   return -1;
 500 }
 501
 502 bool IsPhishingList(const std::string& list_name) {
 503   return list_name.compare(kPhishingList) == 0;
 504 }
 505
 506 bool IsMalwareList(const std::string& list_name) {
 507   return list_name.compare(kMalwareList) == 0;
 508 }
 509
 510 bool IsBadbinurlList(const std::string& list_name) {
 511   return list_name.compare(kBinUrlList) == 0;
 512 }
 513
 514 bool IsBadbinhashList(const std::string& list_name) {
 515   return list_name.compare(kBinHashList) == 0;
 516 }
 517
 518 bool IsExtensionList(const std::string& list_name) {
 519   return list_name.compare(kExtensionBlacklist) == 0;
 520 }
 521
 522 GURL GeneratePhishingReportUrl(const std::string& report_page,
 523                                const std::string& url_to_report,
 524                                bool is_client_side_detection) {
 525   const std::string current_esc = net::EscapeQueryParamValue(url_to_report,
 526                                                              true);
 527
 528 #if defined(OS_WIN)
 529   BrowserDistribution* dist = BrowserDistribution::GetDistribution();
 530   std::string client_name(dist->GetSafeBrowsingName());
 531 #else
 532   std::string client_name("googlechrome");
 533 #endif
 534   if (is_client_side_detection)
 535     client_name.append("_csd");
 536
 537   GURL report_url(report_page + base::StringPrintf(kReportParams,
 538                                                    client_name.c_str(),
 539                                                    current_esc.c_str()));
 540   return google_util::AppendGoogleLocaleParam(report_url);
 541 }
 542
 543 SBFullHash StringToSBFullHash(const std::string& hash_in) {
 544   DCHECK_EQ(crypto::kSHA256Length, hash_in.size());
 545   SBFullHash hash_out;
 546   memcpy(hash_out.full_hash, hash_in.data(), crypto::kSHA256Length);
 547   return hash_out;
 548 }
 549
 550 std::string SBFullHashToString(const SBFullHash& hash) {
 551   DCHECK_EQ(crypto::kSHA256Length, sizeof(hash.full_hash));
 552   return std::string(hash.full_hash, sizeof(hash.full_hash));
 553 }
 554
 555 }  // namespace safe_browsing_util