chrome/browser/safe_browsing/safe_browsing_util.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/safe_browsing/safe_browsing_util.h"
   6
   7 #include "base/logging.h"
   8 #include "base/strings/string_util.h"
   9 #include "base/strings/stringprintf.h"
  10 #include "chrome/browser/browser_process.h"
  11 #include "chrome/browser/safe_browsing/chunk.pb.h"
  12 #include "components/google/core/browser/google_util.h"
  13 #include "crypto/sha2.h"
  14 #include "net/base/escape.h"
  15 #include "url/gurl.h"
  16 #include "url/url_util.h"
  17
  18 #if defined(OS_WIN)
  19 #include "chrome/installer/util/browser_distribution.h"
  20 #endif
  21
  22 static const char kReportParams[] = "?tpl=%s&url=%s";
  23
  24 SBFullHash SBFullHashForString(const base::StringPiece& str) {
  25   SBFullHash h;
  26   crypto::SHA256HashString(str, &h.full_hash, sizeof(h.full_hash));
  27   return h;
  28 }
  29
  30 // SBCachedFullHashResult ------------------------------------------------------
  31
  32 SBCachedFullHashResult::SBCachedFullHashResult() {}
  33
  34 SBCachedFullHashResult::SBCachedFullHashResult(
  35     const base::Time& in_expire_after)
  36     : expire_after(in_expire_after) {}
  37
  38 SBCachedFullHashResult::~SBCachedFullHashResult() {}
  39
  40 // SBChunkData -----------------------------------------------------------------
  41
  42 // TODO(shess): Right now this contains a scoped_ptr<ChunkData> so that the
  43 // proto buffer isn't copied all over the place, then these are contained in a
  44 // ScopedVector for purposes of passing things around between tasks.  This seems
  45 // convoluted.  Maybe it would make sense to have an overall container class
  46 // returning references to a nested per-chunk class?
  47
  48 SBChunkData::SBChunkData() {
  49 }
  50
  51 SBChunkData::SBChunkData(safe_browsing::ChunkData* raw_data)
  52     : chunk_data_(raw_data) {
  53   DCHECK(chunk_data_.get());
  54 }
  55
  56 SBChunkData::~SBChunkData() {
  57 }
  58
  59 bool SBChunkData::ParseFrom(const unsigned char* data, size_t length) {
  60   scoped_ptr<safe_browsing::ChunkData> chunk(new safe_browsing::ChunkData());
  61   if (!chunk->ParseFromArray(data, length))
  62     return false;
  63
  64   if (chunk->chunk_type() != safe_browsing::ChunkData::ADD &&
  65       chunk->chunk_type() != safe_browsing::ChunkData::SUB) {
  66     return false;
  67   }
  68
  69   size_t hash_size = 0;
  70   if (chunk->prefix_type() == safe_browsing::ChunkData::PREFIX_4B) {
  71     hash_size = sizeof(SBPrefix);
  72   } else if (chunk->prefix_type() == safe_browsing::ChunkData::FULL_32B) {
  73     hash_size = sizeof(SBFullHash);
  74   } else {
  75     return false;
  76   }
  77
  78   const size_t hash_count = chunk->hashes().size() / hash_size;
  79   if (hash_count * hash_size != chunk->hashes().size())
  80     return false;
  81
  82   if (chunk->chunk_type() == safe_browsing::ChunkData::SUB &&
  83       static_cast<size_t>(chunk->add_numbers_size()) != hash_count) {
  84     return false;
  85   }
  86
  87   chunk_data_.swap(chunk);
  88   return true;
  89 }
  90
  91 int SBChunkData::ChunkNumber() const {
  92   return chunk_data_->chunk_number();
  93 }
  94
  95 bool SBChunkData::IsAdd() const {
  96   return chunk_data_->chunk_type() == safe_browsing::ChunkData::ADD;
  97 }
  98
  99 bool SBChunkData::IsSub() const {
 100   return chunk_data_->chunk_type() == safe_browsing::ChunkData::SUB;
 101 }
 102
 103 int SBChunkData::AddChunkNumberAt(size_t i) const {
 104   DCHECK(IsSub());
 105   DCHECK((IsPrefix() && i < PrefixCount()) ||
 106          (IsFullHash() && i < FullHashCount()));
 107   return chunk_data_->add_numbers(i);
 108 }
 109
 110 bool SBChunkData::IsPrefix() const {
 111   return chunk_data_->prefix_type() == safe_browsing::ChunkData::PREFIX_4B;
 112 }
 113
 114 size_t SBChunkData::PrefixCount() const {
 115   DCHECK(IsPrefix());
 116   return chunk_data_->hashes().size() / sizeof(SBPrefix);
 117 }
 118
 119 SBPrefix SBChunkData::PrefixAt(size_t i) const {
 120   DCHECK(IsPrefix());
 121   DCHECK_LT(i, PrefixCount());
 122
 123   SBPrefix prefix;
 124   memcpy(&prefix, chunk_data_->hashes().data() + i * sizeof(SBPrefix),
 125          sizeof(SBPrefix));
 126   return prefix;
 127 }
 128
 129 bool SBChunkData::IsFullHash() const {
 130   return chunk_data_->prefix_type() == safe_browsing::ChunkData::FULL_32B;
 131 }
 132
 133 size_t SBChunkData::FullHashCount() const {
 134   DCHECK(IsFullHash());
 135   return chunk_data_->hashes().size() / sizeof(SBFullHash);
 136 }
 137
 138 SBFullHash SBChunkData::FullHashAt(size_t i) const {
 139   DCHECK(IsFullHash());
 140   DCHECK_LT(i, FullHashCount());
 141
 142   SBFullHash full_hash;
 143   memcpy(&full_hash, chunk_data_->hashes().data() + i * sizeof(SBFullHash),
 144          sizeof(SBFullHash));
 145   return full_hash;
 146 }
 147
 148 // SBListChunkRanges -----------------------------------------------------------
 149
 150 SBListChunkRanges::SBListChunkRanges(const std::string& n)
 151     : name(n) {
 152 }
 153
 154 // SBChunkDelete ---------------------------------------------------------------
 155
 156 SBChunkDelete::SBChunkDelete() : is_sub_del(false) {}
 157
 158 SBChunkDelete::~SBChunkDelete() {}
 159
 160 // Utility functions -----------------------------------------------------------
 161
 162 namespace {
 163 bool IsKnownList(const std::string& name) {
 164   for (size_t i = 0; i < arraysize(safe_browsing_util::kAllLists); ++i) {
 165     if (!strcmp(safe_browsing_util::kAllLists[i], name.c_str())) {
 166       return true;
 167     }
 168   }
 169   return false;
 170 }
 171 }  // namespace
 172
 173 namespace safe_browsing_util {
 174
 175 // Listnames that browser can process.
 176 // TODO(shess): This shouldn't be OS-driven <http://crbug.com/394379>
 177 #if defined(OS_ANDROID)
 178 // NOTE(shess): This difference is also reflected in the store name in
 179 // safe_browsing_database.cc.
 180 const char kMalwareList[] = "goog-mobilemalware-shavar";
 181 const char kPhishingList[] = "goog-mobilephish-shavar";
 182 #else
 183 const char kMalwareList[] = "goog-malware-shavar";
 184 const char kPhishingList[] = "goog-phish-shavar";
 185 #endif
 186 const char kBinUrlList[] = "goog-badbinurl-shavar";
 187 const char kCsdWhiteList[] = "goog-csdwhite-sha256";
 188 const char kDownloadWhiteList[] = "goog-downloadwhite-digest256";
 189 const char kExtensionBlacklist[] = "goog-badcrxids-digestvar";
 190 const char kIPBlacklist[] = "goog-badip-digest256";
 191 const char kUnwantedUrlList[] = "goog-unwanted-shavar";
 192 const char kInclusionWhitelist[] = "goog-csdinclusionwhite-sha256";
 193
 194 const char* kAllLists[9] = {
 195     kMalwareList,
 196     kPhishingList,
 197     kBinUrlList,
 198     kCsdWhiteList,
 199     kDownloadWhiteList,
 200     kExtensionBlacklist,
 201     kIPBlacklist,
 202     kUnwantedUrlList,
 203     kInclusionWhitelist,
 204 };
 205
 206 ListType GetListId(const base::StringPiece& name) {
 207   ListType id;
 208   if (name == safe_browsing_util::kMalwareList) {
 209     id = MALWARE;
 210   } else if (name == safe_browsing_util::kPhishingList) {
 211     id = PHISH;
 212   } else if (name == safe_browsing_util::kBinUrlList) {
 213     id = BINURL;
 214   } else if (name == safe_browsing_util::kCsdWhiteList) {
 215     id = CSDWHITELIST;
 216   } else if (name == safe_browsing_util::kDownloadWhiteList) {
 217     id = DOWNLOADWHITELIST;
 218   } else if (name == safe_browsing_util::kExtensionBlacklist) {
 219     id = EXTENSIONBLACKLIST;
 220   } else if (name == safe_browsing_util::kIPBlacklist) {
 221     id = IPBLACKLIST;
 222   } else if (name == safe_browsing_util::kUnwantedUrlList) {
 223     id = UNWANTEDURL;
 224   } else if (name == safe_browsing_util::kInclusionWhitelist) {
 225     id = INCLUSIONWHITELIST;
 226   } else {
 227     id = INVALID;
 228   }
 229   return id;
 230 }
 231
 232 bool GetListName(ListType list_id, std::string* list) {
 233   switch (list_id) {
 234     case MALWARE:
 235       *list = safe_browsing_util::kMalwareList;
 236       break;
 237     case PHISH:
 238       *list = safe_browsing_util::kPhishingList;
 239       break;
 240     case BINURL:
 241       *list = safe_browsing_util::kBinUrlList;
 242       break;
 243     case CSDWHITELIST:
 244       *list = safe_browsing_util::kCsdWhiteList;
 245       break;
 246     case DOWNLOADWHITELIST:
 247       *list = safe_browsing_util::kDownloadWhiteList;
 248       break;
 249     case EXTENSIONBLACKLIST:
 250       *list = safe_browsing_util::kExtensionBlacklist;
 251       break;
 252     case IPBLACKLIST:
 253       *list = safe_browsing_util::kIPBlacklist;
 254       break;
 255     case UNWANTEDURL:
 256       *list = safe_browsing_util::kUnwantedUrlList;
 257       break;
 258     case INCLUSIONWHITELIST:
 259       *list = safe_browsing_util::kInclusionWhitelist;
 260       break;
 261     default:
 262       return false;
 263   }
 264   DCHECK(IsKnownList(*list));
 265   return true;
 266 }
 267
 268 std::string Unescape(const std::string& url) {
 269   std::string unescaped_str(url);
 270   std::string old_unescaped_str;
 271   const int kMaxLoopIterations = 1024;
 272   int loop_var = 0;
 273   do {
 274     old_unescaped_str = unescaped_str;
 275     unescaped_str = net::UnescapeURLComponent(
 276         old_unescaped_str, net::UnescapeRule::SPOOFING_AND_CONTROL_CHARS |
 277                                net::UnescapeRule::SPACES |
 278                                net::UnescapeRule::URL_SPECIAL_CHARS);
 279   } while (unescaped_str != old_unescaped_str && ++loop_var <=
 280            kMaxLoopIterations);
 281
 282   return unescaped_str;
 283 }
 284
 285 std::string Escape(const std::string& url) {
 286   std::string escaped_str;
 287   const char* kHexString = "0123456789ABCDEF";
 288   for (size_t i = 0; i < url.length(); i++) {
 289     unsigned char c = static_cast<unsigned char>(url[i]);
 290     if (c <= ' ' || c > '~' || c == '#' || c == '%') {
 291       escaped_str.push_back('%');
 292       escaped_str.push_back(kHexString[c >> 4]);
 293       escaped_str.push_back(kHexString[c & 0xf]);
 294     } else {
 295       escaped_str.push_back(c);
 296     }
 297   }
 298
 299   return escaped_str;
 300 }
 301
 302 std::string RemoveConsecutiveChars(const std::string& str, const char c) {
 303   std::string output(str);
 304   std::string string_to_find;
 305   std::string::size_type loc = 0;
 306   string_to_find.append(2, c);
 307   while ((loc = output.find(string_to_find, loc)) != std::string::npos) {
 308     output.erase(loc, 1);
 309   }
 310
 311   return output;
 312 }
 313
 314 // Canonicalizes url as per Google Safe Browsing Specification.
 315 // See section 6.1 in
 316 // http://code.google.com/p/google-safe-browsing/wiki/Protocolv2Spec.
 317 void CanonicalizeUrl(const GURL& url,
 318                      std::string* canonicalized_hostname,
 319                      std::string* canonicalized_path,
 320                      std::string* canonicalized_query) {
 321   DCHECK(url.is_valid());
 322
 323   // We only canonicalize "normal" URLs.
 324   if (!url.IsStandard())
 325     return;
 326
 327   // Following canonicalization steps are excluded since url parsing takes care
 328   // of those :-
 329   // 1. Remove any tab (0x09), CR (0x0d), and LF (0x0a) chars from url.
 330   //    (Exclude escaped version of these chars).
 331   // 2. Normalize hostname to 4 dot-seperated decimal values.
 332   // 3. Lowercase hostname.
 333   // 4. Resolve path sequences "/../" and "/./".
 334
 335   // That leaves us with the following :-
 336   // 1. Remove fragment in URL.
 337   GURL url_without_fragment;
 338   GURL::Replacements f_replacements;
 339   f_replacements.ClearRef();
 340   f_replacements.ClearUsername();
 341   f_replacements.ClearPassword();
 342   url_without_fragment = url.ReplaceComponents(f_replacements);
 343
 344   // 2. Do URL unescaping until no more hex encoded characters exist.
 345   std::string url_unescaped_str(Unescape(url_without_fragment.spec()));
 346   url::Parsed parsed;
 347   url::ParseStandardURL(url_unescaped_str.data(), url_unescaped_str.length(),
 348                         &parsed);
 349
 350   // 3. In hostname, remove all leading and trailing dots.
 351   const std::string host =
 352       (parsed.host.len > 0)
 353           ? url_unescaped_str.substr(parsed.host.begin, parsed.host.len)
 354           : std::string();
 355   std::string host_without_end_dots;
 356   base::TrimString(host, ".", &host_without_end_dots);
 357
 358   // 4. In hostname, replace consecutive dots with a single dot.
 359   std::string host_without_consecutive_dots(RemoveConsecutiveChars(
 360       host_without_end_dots, '.'));
 361
 362   // 5. In path, replace runs of consecutive slashes with a single slash.
 363   std::string path =
 364       (parsed.path.len > 0)
 365           ? url_unescaped_str.substr(parsed.path.begin, parsed.path.len)
 366           : std::string();
 367   std::string path_without_consecutive_slash(RemoveConsecutiveChars(path, '/'));
 368
 369   url::Replacements<char> hp_replacements;
 370   hp_replacements.SetHost(
 371       host_without_consecutive_dots.data(),
 372       url::Component(0, host_without_consecutive_dots.length()));
 373   hp_replacements.SetPath(
 374       path_without_consecutive_slash.data(),
 375       url::Component(0, path_without_consecutive_slash.length()));
 376
 377   std::string url_unescaped_with_can_hostpath;
 378   url::StdStringCanonOutput output(&url_unescaped_with_can_hostpath);
 379   url::Parsed temp_parsed;
 380   url::ReplaceComponents(url_unescaped_str.data(),
 381                          url_unescaped_str.length(),
 382                          parsed,
 383                          hp_replacements,
 384                          NULL,
 385                          &output,
 386                          &temp_parsed);
 387   output.Complete();
 388
 389   // 6. Step needed to revert escaping done in url::ReplaceComponents.
 390   url_unescaped_with_can_hostpath = Unescape(url_unescaped_with_can_hostpath);
 391
 392   // 7. After performing all above steps, percent-escape all chars in url which
 393   // are <= ASCII 32, >= 127, #, %. Escapes must be uppercase hex characters.
 394   std::string escaped_canon_url_str(Escape(url_unescaped_with_can_hostpath));
 395   url::Parsed final_parsed;
 396   url::ParseStandardURL(escaped_canon_url_str.data(),
 397                         escaped_canon_url_str.length(),
 398                         &final_parsed);
 399
 400   if (canonicalized_hostname && final_parsed.host.len > 0) {
 401     *canonicalized_hostname =
 402         escaped_canon_url_str.substr(final_parsed.host.begin,
 403                                      final_parsed.host.len);
 404   }
 405   if (canonicalized_path && final_parsed.path.len > 0) {
 406     *canonicalized_path = escaped_canon_url_str.substr(final_parsed.path.begin,
 407                                                        final_parsed.path.len);
 408   }
 409   if (canonicalized_query && final_parsed.query.len > 0) {
 410     *canonicalized_query = escaped_canon_url_str.substr(
 411         final_parsed.query.begin, final_parsed.query.len);
 412   }
 413 }
 414
 415 void GenerateHostsToCheck(const GURL& url, std::vector<std::string>* hosts) {
 416   hosts->clear();
 417
 418   std::string canon_host;
 419   CanonicalizeUrl(url, &canon_host, NULL, NULL);
 420
 421   const std::string host = canon_host;  // const sidesteps GCC bugs below!
 422   if (host.empty())
 423     return;
 424
 425   // Per the Safe Browsing Protocol v2 spec, we try the host, and also up to 4
 426   // hostnames formed by starting with the last 5 components and successively
 427   // removing the leading component.  The last component isn't examined alone,
 428   // since it's the TLD or a subcomponent thereof.
 429   //
 430   // Note that we don't need to be clever about stopping at the "real" eTLD --
 431   // the data on the server side has been filtered to ensure it will not
 432   // blacklist a whole TLD, and it's not significantly slower on our side to
 433   // just check too much.
 434   //
 435   // Also note that because we have a simple blacklist, not some sort of complex
 436   // whitelist-in-blacklist or vice versa, it doesn't matter what order we check
 437   // these in.
 438   const size_t kMaxHostsToCheck = 4;
 439   bool skipped_last_component = false;
 440   for (std::string::const_reverse_iterator i(host.rbegin());
 441        i != host.rend() && hosts->size() < kMaxHostsToCheck; ++i) {
 442     if (*i == '.') {
 443       if (skipped_last_component)
 444         hosts->push_back(std::string(i.base(), host.end()));
 445       else
 446         skipped_last_component = true;
 447     }
 448   }
 449   hosts->push_back(host);
 450 }
 451
 452 void GeneratePathsToCheck(const GURL& url, std::vector<std::string>* paths) {
 453   paths->clear();
 454
 455   std::string canon_path;
 456   std::string canon_query;
 457   CanonicalizeUrl(url, NULL, &canon_path, &canon_query);
 458
 459   const std::string path = canon_path;   // const sidesteps GCC bugs below!
 460   const std::string query = canon_query;
 461   if (path.empty())
 462     return;
 463
 464   // Per the Safe Browsing Protocol v2 spec, we try the exact path with/without
 465   // the query parameters, and also up to 4 paths formed by starting at the root
 466   // and adding more path components.
 467   //
 468   // As with the hosts above, it doesn't matter what order we check these in.
 469   const size_t kMaxPathsToCheck = 4;
 470   for (std::string::const_iterator i(path.begin());
 471        i != path.end() && paths->size() < kMaxPathsToCheck; ++i) {
 472     if (*i == '/')
 473       paths->push_back(std::string(path.begin(), i + 1));
 474   }
 475
 476   if (!paths->empty() && paths->back() != path)
 477     paths->push_back(path);
 478
 479   if (!query.empty())
 480     paths->push_back(path + "?" + query);
 481 }
 482
 483 void GeneratePatternsToCheck(const GURL& url, std::vector<std::string>* urls) {
 484   std::vector<std::string> hosts, paths;
 485   GenerateHostsToCheck(url, &hosts);
 486   GeneratePathsToCheck(url, &paths);
 487   for (size_t h = 0; h < hosts.size(); ++h) {
 488     for (size_t p = 0; p < paths.size(); ++p) {
 489       urls->push_back(hosts[h] + paths[p]);
 490     }
 491   }
 492 }
 493
 494 GURL GeneratePhishingReportUrl(const std::string& report_page,
 495                                const std::string& url_to_report,
 496                                bool is_client_side_detection) {
 497   const std::string current_esc = net::EscapeQueryParamValue(url_to_report,
 498                                                              true);
 499
 500 #if defined(OS_WIN)
 501   BrowserDistribution* dist = BrowserDistribution::GetDistribution();
 502   std::string client_name(dist->GetSafeBrowsingName());
 503 #else
 504   std::string client_name("googlechrome");
 505 #endif
 506   if (is_client_side_detection)
 507     client_name.append("_csd");
 508
 509   GURL report_url(report_page + base::StringPrintf(kReportParams,
 510                                                    client_name.c_str(),
 511                                                    current_esc.c_str()));
 512   return google_util::AppendGoogleLocaleParam(
 513       report_url, g_browser_process->GetApplicationLocale());
 514 }
 515
 516 SBFullHash StringToSBFullHash(const std::string& hash_in) {
 517   DCHECK_EQ(crypto::kSHA256Length, hash_in.size());
 518   SBFullHash hash_out;
 519   memcpy(hash_out.full_hash, hash_in.data(), crypto::kSHA256Length);
 520   return hash_out;
 521 }
 522
 523 std::string SBFullHashToString(const SBFullHash& hash) {
 524   DCHECK_EQ(crypto::kSHA256Length, sizeof(hash.full_hash));
 525   return std::string(hash.full_hash, sizeof(hash.full_hash));
 526 }
 527
 528 }  // namespace safe_browsing_util